Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 1 | /* |
| 2 | * GPL HEADER START |
| 3 | * |
| 4 | * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. |
| 5 | * |
| 6 | * This program is free software; you can redistribute it and/or modify |
| 7 | * it under the terms of the GNU General Public License version 2 only, |
| 8 | * as published by the Free Software Foundation. |
| 9 | * |
| 10 | * This program is distributed in the hope that it will be useful, but |
| 11 | * WITHOUT ANY WARRANTY; without even the implied warranty of |
| 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU |
| 13 | * General Public License version 2 for more details (a copy is included |
| 14 | * in the LICENSE file that accompanied this code). |
| 15 | * |
| 16 | * You should have received a copy of the GNU General Public License |
| 17 | * version 2 along with this program; If not, see |
Oleg Drokin | 6a5b99a | 2016-06-14 23:33:40 -0400 | [diff] [blame] | 18 | * http://www.gnu.org/licenses/gpl-2.0.html |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 19 | * |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 20 | * GPL HEADER END |
| 21 | */ |
| 22 | /* |
| 23 | * Copyright (c) 2002, 2010, Oracle and/or its affiliates. All rights reserved. |
| 24 | * Use is subject to license terms. |
| 25 | * |
Andreas Dilger | 1dc563a | 2015-11-08 18:09:37 -0500 | [diff] [blame] | 26 | * Copyright (c) 2010, 2015, Intel Corporation. |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 27 | */ |
| 28 | /* |
| 29 | * This file is part of Lustre, http://www.lustre.org/ |
| 30 | * Lustre is a trademark of Sun Microsystems, Inc. |
| 31 | */ |
| 32 | |
| 33 | #define DEBUG_SUBSYSTEM S_RPC |
Greg Kroah-Hartman | e27db14 | 2014-07-11 22:29:36 -0700 | [diff] [blame] | 34 | #include "../include/obd_support.h" |
| 35 | #include "../include/obd_class.h" |
| 36 | #include "../include/lustre_net.h" |
| 37 | #include "../include/lu_object.h" |
Greg Kroah-Hartman | 9fdaf8c | 2014-07-11 20:51:16 -0700 | [diff] [blame] | 38 | #include "../../include/linux/lnet/types.h" |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 39 | #include "ptlrpc_internal.h" |
| 40 | |
| 41 | /* The following are visible and mutable through /sys/module/ptlrpc */ |
Daniel Machon | b963e72 | 2015-08-08 20:43:25 +0200 | [diff] [blame] | 42 | int test_req_buffer_pressure; |
Peng Tao | 8cc7b4b | 2013-11-21 22:28:30 +0800 | [diff] [blame] | 43 | module_param(test_req_buffer_pressure, int, 0444); |
| 44 | MODULE_PARM_DESC(test_req_buffer_pressure, "set non-zero to put pressure on request buffer pools"); |
| 45 | module_param(at_min, int, 0644); |
| 46 | MODULE_PARM_DESC(at_min, "Adaptive timeout minimum (sec)"); |
| 47 | module_param(at_max, int, 0644); |
| 48 | MODULE_PARM_DESC(at_max, "Adaptive timeout maximum (sec)"); |
| 49 | module_param(at_history, int, 0644); |
| 50 | MODULE_PARM_DESC(at_history, |
| 51 | "Adaptive timeouts remember the slowest event that took place within this period (sec)"); |
| 52 | module_param(at_early_margin, int, 0644); |
| 53 | MODULE_PARM_DESC(at_early_margin, "How soon before an RPC deadline to send an early reply"); |
| 54 | module_param(at_extra, int, 0644); |
| 55 | MODULE_PARM_DESC(at_extra, "How much extra time to give with each early reply"); |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 56 | |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 57 | /* forward ref */ |
| 58 | static int ptlrpc_server_post_idle_rqbds(struct ptlrpc_service_part *svcpt); |
| 59 | static void ptlrpc_server_hpreq_fini(struct ptlrpc_request *req); |
| 60 | static void ptlrpc_at_remove_timed(struct ptlrpc_request *req); |
| 61 | |
| 62 | /** Holds a list of all PTLRPC services */ |
| 63 | LIST_HEAD(ptlrpc_all_services); |
| 64 | /** Used to protect the \e ptlrpc_all_services list */ |
| 65 | struct mutex ptlrpc_all_services_mutex; |
| 66 | |
Zoltán Lajos Kis | a96389d | 2015-08-02 22:36:31 +0200 | [diff] [blame] | 67 | static struct ptlrpc_request_buffer_desc * |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 68 | ptlrpc_alloc_rqbd(struct ptlrpc_service_part *svcpt) |
| 69 | { |
Chris Hanna | d0bfef3 | 2015-06-03 10:28:26 -0400 | [diff] [blame] | 70 | struct ptlrpc_service *svc = svcpt->scp_service; |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 71 | struct ptlrpc_request_buffer_desc *rqbd; |
| 72 | |
Julia Lawall | bae97e8 | 2015-05-03 15:21:44 +0200 | [diff] [blame] | 73 | rqbd = kzalloc_node(sizeof(*rqbd), GFP_NOFS, |
| 74 | cfs_cpt_spread_node(svc->srv_cptable, |
| 75 | svcpt->scp_cpt)); |
Oleg Drokin | 8b38208 | 2016-02-16 00:46:58 -0500 | [diff] [blame] | 76 | if (!rqbd) |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 77 | return NULL; |
| 78 | |
| 79 | rqbd->rqbd_svcpt = svcpt; |
| 80 | rqbd->rqbd_refcount = 0; |
| 81 | rqbd->rqbd_cbid.cbid_fn = request_in_callback; |
| 82 | rqbd->rqbd_cbid.cbid_arg = rqbd; |
| 83 | INIT_LIST_HEAD(&rqbd->rqbd_reqs); |
Oleg Drokin | 6fd5733 | 2015-09-16 12:26:52 -0400 | [diff] [blame] | 84 | rqbd->rqbd_buffer = libcfs_kvzalloc_cpt(svc->srv_cptable, |
| 85 | svcpt->scp_cpt, |
| 86 | svc->srv_buf_size, |
| 87 | GFP_KERNEL); |
Oleg Drokin | 8b38208 | 2016-02-16 00:46:58 -0500 | [diff] [blame] | 88 | if (!rqbd->rqbd_buffer) { |
Julia Lawall | 9ae1059 | 2015-05-01 17:51:12 +0200 | [diff] [blame] | 89 | kfree(rqbd); |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 90 | return NULL; |
| 91 | } |
| 92 | |
| 93 | spin_lock(&svcpt->scp_lock); |
| 94 | list_add(&rqbd->rqbd_list, &svcpt->scp_rqbd_idle); |
| 95 | svcpt->scp_nrqbds_total++; |
| 96 | spin_unlock(&svcpt->scp_lock); |
| 97 | |
| 98 | return rqbd; |
| 99 | } |
| 100 | |
Zoltán Lajos Kis | a96389d | 2015-08-02 22:36:31 +0200 | [diff] [blame] | 101 | static void |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 102 | ptlrpc_free_rqbd(struct ptlrpc_request_buffer_desc *rqbd) |
| 103 | { |
| 104 | struct ptlrpc_service_part *svcpt = rqbd->rqbd_svcpt; |
| 105 | |
| 106 | LASSERT(rqbd->rqbd_refcount == 0); |
| 107 | LASSERT(list_empty(&rqbd->rqbd_reqs)); |
| 108 | |
| 109 | spin_lock(&svcpt->scp_lock); |
| 110 | list_del(&rqbd->rqbd_list); |
| 111 | svcpt->scp_nrqbds_total--; |
| 112 | spin_unlock(&svcpt->scp_lock); |
| 113 | |
Julia Lawall | ee0ec19 | 2015-06-11 14:02:58 +0200 | [diff] [blame] | 114 | kvfree(rqbd->rqbd_buffer); |
Julia Lawall | 9ae1059 | 2015-05-01 17:51:12 +0200 | [diff] [blame] | 115 | kfree(rqbd); |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 116 | } |
| 117 | |
Zoltán Lajos Kis | a96389d | 2015-08-02 22:36:31 +0200 | [diff] [blame] | 118 | static int |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 119 | ptlrpc_grow_req_bufs(struct ptlrpc_service_part *svcpt, int post) |
| 120 | { |
Chris Hanna | d0bfef3 | 2015-06-03 10:28:26 -0400 | [diff] [blame] | 121 | struct ptlrpc_service *svc = svcpt->scp_service; |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 122 | struct ptlrpc_request_buffer_desc *rqbd; |
Chris Hanna | d0bfef3 | 2015-06-03 10:28:26 -0400 | [diff] [blame] | 123 | int rc = 0; |
| 124 | int i; |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 125 | |
| 126 | if (svcpt->scp_rqbd_allocating) |
| 127 | goto try_post; |
| 128 | |
| 129 | spin_lock(&svcpt->scp_lock); |
| 130 | /* check again with lock */ |
| 131 | if (svcpt->scp_rqbd_allocating) { |
| 132 | /* NB: we might allow more than one thread in the future */ |
| 133 | LASSERT(svcpt->scp_rqbd_allocating == 1); |
| 134 | spin_unlock(&svcpt->scp_lock); |
| 135 | goto try_post; |
| 136 | } |
| 137 | |
| 138 | svcpt->scp_rqbd_allocating++; |
| 139 | spin_unlock(&svcpt->scp_lock); |
| 140 | |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 141 | for (i = 0; i < svc->srv_nbuf_per_group; i++) { |
| 142 | /* NB: another thread might have recycled enough rqbds, we |
Oleg Drokin | dadfcda | 2016-02-24 22:00:38 -0500 | [diff] [blame] | 143 | * need to make sure it wouldn't over-allocate, see LU-1212. |
| 144 | */ |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 145 | if (svcpt->scp_nrqbds_posted >= svc->srv_nbuf_per_group) |
| 146 | break; |
| 147 | |
| 148 | rqbd = ptlrpc_alloc_rqbd(svcpt); |
| 149 | |
Oleg Drokin | 8b38208 | 2016-02-16 00:46:58 -0500 | [diff] [blame] | 150 | if (!rqbd) { |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 151 | CERROR("%s: Can't allocate request buffer\n", |
| 152 | svc->srv_name); |
| 153 | rc = -ENOMEM; |
| 154 | break; |
| 155 | } |
| 156 | } |
| 157 | |
| 158 | spin_lock(&svcpt->scp_lock); |
| 159 | |
| 160 | LASSERT(svcpt->scp_rqbd_allocating == 1); |
| 161 | svcpt->scp_rqbd_allocating--; |
| 162 | |
| 163 | spin_unlock(&svcpt->scp_lock); |
| 164 | |
| 165 | CDEBUG(D_RPCTRACE, |
| 166 | "%s: allocate %d new %d-byte reqbufs (%d/%d left), rc = %d\n", |
| 167 | svc->srv_name, i, svc->srv_buf_size, svcpt->scp_nrqbds_posted, |
| 168 | svcpt->scp_nrqbds_total, rc); |
| 169 | |
| 170 | try_post: |
| 171 | if (post && rc == 0) |
| 172 | rc = ptlrpc_server_post_idle_rqbds(svcpt); |
| 173 | |
| 174 | return rc; |
| 175 | } |
| 176 | |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 177 | struct ptlrpc_hr_partition; |
| 178 | |
| 179 | struct ptlrpc_hr_thread { |
| 180 | int hrt_id; /* thread ID */ |
| 181 | spinlock_t hrt_lock; |
| 182 | wait_queue_head_t hrt_waitq; |
| 183 | struct list_head hrt_queue; /* RS queue */ |
| 184 | struct ptlrpc_hr_partition *hrt_partition; |
| 185 | }; |
| 186 | |
| 187 | struct ptlrpc_hr_partition { |
| 188 | /* # of started threads */ |
| 189 | atomic_t hrp_nstarted; |
| 190 | /* # of stopped threads */ |
| 191 | atomic_t hrp_nstopped; |
| 192 | /* cpu partition id */ |
| 193 | int hrp_cpt; |
| 194 | /* round-robin rotor for choosing thread */ |
| 195 | int hrp_rotor; |
| 196 | /* total number of threads on this partition */ |
| 197 | int hrp_nthrs; |
| 198 | /* threads table */ |
| 199 | struct ptlrpc_hr_thread *hrp_thrs; |
| 200 | }; |
| 201 | |
| 202 | #define HRT_RUNNING 0 |
| 203 | #define HRT_STOPPING 1 |
| 204 | |
| 205 | struct ptlrpc_hr_service { |
| 206 | /* CPU partition table, it's just cfs_cpt_table for now */ |
| 207 | struct cfs_cpt_table *hr_cpt_table; |
| 208 | /** controller sleep waitq */ |
| 209 | wait_queue_head_t hr_waitq; |
| 210 | unsigned int hr_stopping; |
| 211 | /** roundrobin rotor for non-affinity service */ |
| 212 | unsigned int hr_rotor; |
| 213 | /* partition data */ |
| 214 | struct ptlrpc_hr_partition **hr_partitions; |
| 215 | }; |
| 216 | |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 217 | /** reply handling service. */ |
| 218 | static struct ptlrpc_hr_service ptlrpc_hr; |
| 219 | |
| 220 | /** |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 221 | * Choose an hr thread to dispatch requests to. |
| 222 | */ |
| 223 | static struct ptlrpc_hr_thread * |
| 224 | ptlrpc_hr_select(struct ptlrpc_service_part *svcpt) |
| 225 | { |
Chris Hanna | d0bfef3 | 2015-06-03 10:28:26 -0400 | [diff] [blame] | 226 | struct ptlrpc_hr_partition *hrp; |
| 227 | unsigned int rotor; |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 228 | |
| 229 | if (svcpt->scp_cpt >= 0 && |
| 230 | svcpt->scp_service->srv_cptable == ptlrpc_hr.hr_cpt_table) { |
| 231 | /* directly match partition */ |
| 232 | hrp = ptlrpc_hr.hr_partitions[svcpt->scp_cpt]; |
| 233 | |
| 234 | } else { |
| 235 | rotor = ptlrpc_hr.hr_rotor++; |
| 236 | rotor %= cfs_cpt_number(ptlrpc_hr.hr_cpt_table); |
| 237 | |
| 238 | hrp = ptlrpc_hr.hr_partitions[rotor]; |
| 239 | } |
| 240 | |
| 241 | rotor = hrp->hrp_rotor++; |
| 242 | return &hrp->hrp_thrs[rotor % hrp->hrp_nthrs]; |
| 243 | } |
| 244 | |
| 245 | /** |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 246 | * Put reply state into a queue for processing because we received |
| 247 | * ACK from the client |
| 248 | */ |
| 249 | void ptlrpc_dispatch_difficult_reply(struct ptlrpc_reply_state *rs) |
| 250 | { |
| 251 | struct ptlrpc_hr_thread *hrt; |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 252 | |
| 253 | LASSERT(list_empty(&rs->rs_list)); |
| 254 | |
| 255 | hrt = ptlrpc_hr_select(rs->rs_svcpt); |
| 256 | |
| 257 | spin_lock(&hrt->hrt_lock); |
| 258 | list_add_tail(&rs->rs_list, &hrt->hrt_queue); |
| 259 | spin_unlock(&hrt->hrt_lock); |
| 260 | |
| 261 | wake_up(&hrt->hrt_waitq); |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 262 | } |
| 263 | |
| 264 | void |
| 265 | ptlrpc_schedule_difficult_reply(struct ptlrpc_reply_state *rs) |
| 266 | { |
Li Xi | 5e42bc9 | 2014-04-27 13:07:06 -0400 | [diff] [blame] | 267 | assert_spin_locked(&rs->rs_svcpt->scp_rep_lock); |
| 268 | assert_spin_locked(&rs->rs_lock); |
Kristina Martsenko | 3949015e | 2013-11-11 21:34:58 +0200 | [diff] [blame] | 269 | LASSERT(rs->rs_difficult); |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 270 | rs->rs_scheduled_ever = 1; /* flag any notification attempt */ |
| 271 | |
| 272 | if (rs->rs_scheduled) { /* being set up or already notified */ |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 273 | return; |
| 274 | } |
| 275 | |
| 276 | rs->rs_scheduled = 1; |
| 277 | list_del_init(&rs->rs_list); |
| 278 | ptlrpc_dispatch_difficult_reply(rs); |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 279 | } |
| 280 | EXPORT_SYMBOL(ptlrpc_schedule_difficult_reply); |
| 281 | |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 282 | static int |
| 283 | ptlrpc_server_post_idle_rqbds(struct ptlrpc_service_part *svcpt) |
| 284 | { |
| 285 | struct ptlrpc_request_buffer_desc *rqbd; |
Chris Hanna | d0bfef3 | 2015-06-03 10:28:26 -0400 | [diff] [blame] | 286 | int rc; |
| 287 | int posted = 0; |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 288 | |
| 289 | for (;;) { |
| 290 | spin_lock(&svcpt->scp_lock); |
| 291 | |
| 292 | if (list_empty(&svcpt->scp_rqbd_idle)) { |
| 293 | spin_unlock(&svcpt->scp_lock); |
| 294 | return posted; |
| 295 | } |
| 296 | |
| 297 | rqbd = list_entry(svcpt->scp_rqbd_idle.next, |
Oleg Drokin | 30c0aa3 | 2016-02-26 01:50:02 -0500 | [diff] [blame] | 298 | struct ptlrpc_request_buffer_desc, |
| 299 | rqbd_list); |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 300 | list_del(&rqbd->rqbd_list); |
| 301 | |
| 302 | /* assume we will post successfully */ |
| 303 | svcpt->scp_nrqbds_posted++; |
| 304 | list_add(&rqbd->rqbd_list, &svcpt->scp_rqbd_posted); |
| 305 | |
| 306 | spin_unlock(&svcpt->scp_lock); |
| 307 | |
| 308 | rc = ptlrpc_register_rqbd(rqbd); |
| 309 | if (rc != 0) |
| 310 | break; |
| 311 | |
| 312 | posted = 1; |
| 313 | } |
| 314 | |
| 315 | spin_lock(&svcpt->scp_lock); |
| 316 | |
| 317 | svcpt->scp_nrqbds_posted--; |
| 318 | list_del(&rqbd->rqbd_list); |
| 319 | list_add_tail(&rqbd->rqbd_list, &svcpt->scp_rqbd_idle); |
| 320 | |
| 321 | /* Don't complain if no request buffers are posted right now; LNET |
Oleg Drokin | dadfcda | 2016-02-24 22:00:38 -0500 | [diff] [blame] | 322 | * won't drop requests because we set the portal lazy! |
| 323 | */ |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 324 | |
| 325 | spin_unlock(&svcpt->scp_lock); |
| 326 | |
| 327 | return -1; |
| 328 | } |
| 329 | |
| 330 | static void ptlrpc_at_timer(unsigned long castmeharder) |
| 331 | { |
| 332 | struct ptlrpc_service_part *svcpt; |
| 333 | |
| 334 | svcpt = (struct ptlrpc_service_part *)castmeharder; |
| 335 | |
| 336 | svcpt->scp_at_check = 1; |
| 337 | svcpt->scp_at_checktime = cfs_time_current(); |
| 338 | wake_up(&svcpt->scp_waitq); |
| 339 | } |
| 340 | |
| 341 | static void |
| 342 | ptlrpc_server_nthreads_check(struct ptlrpc_service *svc, |
| 343 | struct ptlrpc_service_conf *conf) |
| 344 | { |
Chris Hanna | d0bfef3 | 2015-06-03 10:28:26 -0400 | [diff] [blame] | 345 | struct ptlrpc_service_thr_conf *tc = &conf->psc_thr; |
| 346 | unsigned init; |
| 347 | unsigned total; |
| 348 | unsigned nthrs; |
| 349 | int weight; |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 350 | |
| 351 | /* |
| 352 | * Common code for estimating & validating threads number. |
| 353 | * CPT affinity service could have percpt thread-pool instead |
| 354 | * of a global thread-pool, which means user might not always |
| 355 | * get the threads number they give it in conf::tc_nthrs_user |
| 356 | * even they did set. It's because we need to validate threads |
| 357 | * number for each CPT to guarantee each pool will have enough |
| 358 | * threads to keep the service healthy. |
| 359 | */ |
| 360 | init = PTLRPC_NTHRS_INIT + (svc->srv_ops.so_hpreq_handler != NULL); |
| 361 | init = max_t(int, init, tc->tc_nthrs_init); |
| 362 | |
| 363 | /* NB: please see comments in lustre_lnet.h for definition |
Oleg Drokin | dadfcda | 2016-02-24 22:00:38 -0500 | [diff] [blame] | 364 | * details of these members |
| 365 | */ |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 366 | LASSERT(tc->tc_nthrs_max != 0); |
| 367 | |
| 368 | if (tc->tc_nthrs_user != 0) { |
| 369 | /* In case there is a reason to test a service with many |
| 370 | * threads, we give a less strict check here, it can |
Oleg Drokin | dadfcda | 2016-02-24 22:00:38 -0500 | [diff] [blame] | 371 | * be up to 8 * nthrs_max |
| 372 | */ |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 373 | total = min(tc->tc_nthrs_max * 8, tc->tc_nthrs_user); |
| 374 | nthrs = total / svc->srv_ncpts; |
Chris Hanna | d0bfef3 | 2015-06-03 10:28:26 -0400 | [diff] [blame] | 375 | init = max(init, nthrs); |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 376 | goto out; |
| 377 | } |
| 378 | |
| 379 | total = tc->tc_nthrs_max; |
| 380 | if (tc->tc_nthrs_base == 0) { |
| 381 | /* don't care about base threads number per partition, |
Oleg Drokin | dadfcda | 2016-02-24 22:00:38 -0500 | [diff] [blame] | 382 | * this is most for non-affinity service |
| 383 | */ |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 384 | nthrs = total / svc->srv_ncpts; |
| 385 | goto out; |
| 386 | } |
| 387 | |
| 388 | nthrs = tc->tc_nthrs_base; |
| 389 | if (svc->srv_ncpts == 1) { |
Chris Hanna | d0bfef3 | 2015-06-03 10:28:26 -0400 | [diff] [blame] | 390 | int i; |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 391 | |
| 392 | /* NB: Increase the base number if it's single partition |
| 393 | * and total number of cores/HTs is larger or equal to 4. |
Oleg Drokin | dadfcda | 2016-02-24 22:00:38 -0500 | [diff] [blame] | 394 | * result will always < 2 * nthrs_base |
| 395 | */ |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 396 | weight = cfs_cpt_weight(svc->srv_cptable, CFS_CPT_ANY); |
| 397 | for (i = 1; (weight >> (i + 1)) != 0 && /* >= 4 cores/HTs */ |
| 398 | (tc->tc_nthrs_base >> i) != 0; i++) |
| 399 | nthrs += tc->tc_nthrs_base >> i; |
| 400 | } |
| 401 | |
| 402 | if (tc->tc_thr_factor != 0) { |
Chris Hanna | d0bfef3 | 2015-06-03 10:28:26 -0400 | [diff] [blame] | 403 | int factor = tc->tc_thr_factor; |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 404 | const int fade = 4; |
| 405 | |
| 406 | /* |
| 407 | * User wants to increase number of threads with for |
| 408 | * each CPU core/HT, most likely the factor is larger then |
| 409 | * one thread/core because service threads are supposed to |
| 410 | * be blocked by lock or wait for IO. |
| 411 | */ |
| 412 | /* |
| 413 | * Amdahl's law says that adding processors wouldn't give |
| 414 | * a linear increasing of parallelism, so it's nonsense to |
| 415 | * have too many threads no matter how many cores/HTs |
| 416 | * there are. |
| 417 | */ |
Oleg Drokin | 6301647 | 2015-03-02 01:01:46 -0500 | [diff] [blame] | 418 | /* weight is # of HTs */ |
Bartosz Golaszewski | 06931e6 | 2015-05-26 15:11:28 +0200 | [diff] [blame] | 419 | if (cpumask_weight(topology_sibling_cpumask(0)) > 1) { |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 420 | /* depress thread factor for hyper-thread */ |
| 421 | factor = factor - (factor >> 1) + (factor >> 3); |
| 422 | } |
| 423 | |
| 424 | weight = cfs_cpt_weight(svc->srv_cptable, 0); |
| 425 | LASSERT(weight > 0); |
| 426 | |
| 427 | for (; factor > 0 && weight > 0; factor--, weight -= fade) |
| 428 | nthrs += min(weight, fade) * factor; |
| 429 | } |
| 430 | |
| 431 | if (nthrs * svc->srv_ncpts > tc->tc_nthrs_max) { |
| 432 | nthrs = max(tc->tc_nthrs_base, |
| 433 | tc->tc_nthrs_max / svc->srv_ncpts); |
| 434 | } |
| 435 | out: |
| 436 | nthrs = max(nthrs, tc->tc_nthrs_init); |
| 437 | svc->srv_nthrs_cpt_limit = nthrs; |
| 438 | svc->srv_nthrs_cpt_init = init; |
| 439 | |
| 440 | if (nthrs * svc->srv_ncpts > tc->tc_nthrs_max) { |
Joe Perches | 2d00bd1 | 2014-11-23 11:28:50 -0800 | [diff] [blame] | 441 | CDEBUG(D_OTHER, "%s: This service may have more threads (%d) than the given soft limit (%d)\n", |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 442 | svc->srv_name, nthrs * svc->srv_ncpts, |
| 443 | tc->tc_nthrs_max); |
| 444 | } |
| 445 | } |
| 446 | |
| 447 | /** |
| 448 | * Initialize percpt data for a service |
| 449 | */ |
| 450 | static int |
| 451 | ptlrpc_service_part_init(struct ptlrpc_service *svc, |
| 452 | struct ptlrpc_service_part *svcpt, int cpt) |
| 453 | { |
| 454 | struct ptlrpc_at_array *array; |
Chris Hanna | d0bfef3 | 2015-06-03 10:28:26 -0400 | [diff] [blame] | 455 | int size; |
| 456 | int index; |
| 457 | int rc; |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 458 | |
| 459 | svcpt->scp_cpt = cpt; |
| 460 | INIT_LIST_HEAD(&svcpt->scp_threads); |
| 461 | |
| 462 | /* rqbd and incoming request queue */ |
| 463 | spin_lock_init(&svcpt->scp_lock); |
| 464 | INIT_LIST_HEAD(&svcpt->scp_rqbd_idle); |
| 465 | INIT_LIST_HEAD(&svcpt->scp_rqbd_posted); |
| 466 | INIT_LIST_HEAD(&svcpt->scp_req_incoming); |
| 467 | init_waitqueue_head(&svcpt->scp_waitq); |
| 468 | /* history request & rqbd list */ |
| 469 | INIT_LIST_HEAD(&svcpt->scp_hist_reqs); |
| 470 | INIT_LIST_HEAD(&svcpt->scp_hist_rqbds); |
| 471 | |
Masanari Iida | 369e5c9 | 2014-02-08 00:30:36 +0900 | [diff] [blame] | 472 | /* active requests and hp requests */ |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 473 | spin_lock_init(&svcpt->scp_req_lock); |
| 474 | |
| 475 | /* reply states */ |
| 476 | spin_lock_init(&svcpt->scp_rep_lock); |
| 477 | INIT_LIST_HEAD(&svcpt->scp_rep_active); |
| 478 | INIT_LIST_HEAD(&svcpt->scp_rep_idle); |
| 479 | init_waitqueue_head(&svcpt->scp_rep_waitq); |
| 480 | atomic_set(&svcpt->scp_nreps_difficult, 0); |
| 481 | |
| 482 | /* adaptive timeout */ |
| 483 | spin_lock_init(&svcpt->scp_at_lock); |
| 484 | array = &svcpt->scp_at_array; |
| 485 | |
| 486 | size = at_est2timeout(at_max); |
Chris Hanna | d0bfef3 | 2015-06-03 10:28:26 -0400 | [diff] [blame] | 487 | array->paa_size = size; |
| 488 | array->paa_count = 0; |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 489 | array->paa_deadline = -1; |
| 490 | |
| 491 | /* allocate memory for scp_at_array (ptlrpc_at_array) */ |
Julia Lawall | bae97e8 | 2015-05-03 15:21:44 +0200 | [diff] [blame] | 492 | array->paa_reqs_array = |
| 493 | kzalloc_node(sizeof(struct list_head) * size, GFP_NOFS, |
| 494 | cfs_cpt_spread_node(svc->srv_cptable, cpt)); |
Oleg Drokin | 8b38208 | 2016-02-16 00:46:58 -0500 | [diff] [blame] | 495 | if (!array->paa_reqs_array) |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 496 | return -ENOMEM; |
| 497 | |
| 498 | for (index = 0; index < size; index++) |
| 499 | INIT_LIST_HEAD(&array->paa_reqs_array[index]); |
| 500 | |
Julia Lawall | bae97e8 | 2015-05-03 15:21:44 +0200 | [diff] [blame] | 501 | array->paa_reqs_count = |
| 502 | kzalloc_node(sizeof(__u32) * size, GFP_NOFS, |
| 503 | cfs_cpt_spread_node(svc->srv_cptable, cpt)); |
Oleg Drokin | 8b38208 | 2016-02-16 00:46:58 -0500 | [diff] [blame] | 504 | if (!array->paa_reqs_count) |
Julia Lawall | 207e99c | 2015-05-01 21:37:47 +0200 | [diff] [blame] | 505 | goto free_reqs_array; |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 506 | |
Arnd Bergmann | 922da0c | 2015-09-27 16:45:33 -0400 | [diff] [blame] | 507 | setup_timer(&svcpt->scp_at_timer, ptlrpc_at_timer, |
| 508 | (unsigned long)svcpt); |
| 509 | |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 510 | /* At SOW, service time should be quick; 10s seems generous. If client |
Oleg Drokin | dadfcda | 2016-02-24 22:00:38 -0500 | [diff] [blame] | 511 | * timeout is less than this, we'll be sending an early reply. |
| 512 | */ |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 513 | at_init(&svcpt->scp_at_estimate, 10, 0); |
| 514 | |
| 515 | /* assign this before call ptlrpc_grow_req_bufs */ |
| 516 | svcpt->scp_service = svc; |
| 517 | /* Now allocate the request buffers, but don't post them now */ |
| 518 | rc = ptlrpc_grow_req_bufs(svcpt, 0); |
| 519 | /* We shouldn't be under memory pressure at startup, so |
Oleg Drokin | dadfcda | 2016-02-24 22:00:38 -0500 | [diff] [blame] | 520 | * fail if we can't allocate all our buffers at this time. |
| 521 | */ |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 522 | if (rc != 0) |
Julia Lawall | 207e99c | 2015-05-01 21:37:47 +0200 | [diff] [blame] | 523 | goto free_reqs_count; |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 524 | |
| 525 | return 0; |
| 526 | |
Julia Lawall | 207e99c | 2015-05-01 21:37:47 +0200 | [diff] [blame] | 527 | free_reqs_count: |
| 528 | kfree(array->paa_reqs_count); |
| 529 | array->paa_reqs_count = NULL; |
| 530 | free_reqs_array: |
| 531 | kfree(array->paa_reqs_array); |
| 532 | array->paa_reqs_array = NULL; |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 533 | |
| 534 | return -ENOMEM; |
| 535 | } |
| 536 | |
| 537 | /** |
| 538 | * Initialize service on a given portal. |
| 539 | * This includes starting serving threads , allocating and posting rqbds and |
| 540 | * so on. |
| 541 | */ |
| 542 | struct ptlrpc_service * |
| 543 | ptlrpc_register_service(struct ptlrpc_service_conf *conf, |
Oleg Drokin | 328676f | 2015-05-21 15:32:08 -0400 | [diff] [blame] | 544 | struct kset *parent, |
Dmitry Eremin | 700815d | 2015-05-21 15:32:11 -0400 | [diff] [blame] | 545 | struct dentry *debugfs_entry) |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 546 | { |
Chris Hanna | d0bfef3 | 2015-06-03 10:28:26 -0400 | [diff] [blame] | 547 | struct ptlrpc_service_cpt_conf *cconf = &conf->psc_cpt; |
| 548 | struct ptlrpc_service *service; |
| 549 | struct ptlrpc_service_part *svcpt; |
| 550 | struct cfs_cpt_table *cptable; |
| 551 | __u32 *cpts = NULL; |
| 552 | int ncpts; |
| 553 | int cpt; |
| 554 | int rc; |
| 555 | int i; |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 556 | |
| 557 | LASSERT(conf->psc_buf.bc_nbufs > 0); |
| 558 | LASSERT(conf->psc_buf.bc_buf_size >= |
| 559 | conf->psc_buf.bc_req_max_size + SPTLRPC_MAX_PAYLOAD); |
| 560 | LASSERT(conf->psc_thr.tc_ctx_tags != 0); |
| 561 | |
| 562 | cptable = cconf->cc_cptable; |
Oleg Drokin | 8b38208 | 2016-02-16 00:46:58 -0500 | [diff] [blame] | 563 | if (!cptable) |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 564 | cptable = cfs_cpt_table; |
| 565 | |
| 566 | if (!conf->psc_thr.tc_cpu_affinity) { |
| 567 | ncpts = 1; |
| 568 | } else { |
| 569 | ncpts = cfs_cpt_number(cptable); |
Oleg Drokin | 8b38208 | 2016-02-16 00:46:58 -0500 | [diff] [blame] | 570 | if (cconf->cc_pattern) { |
Chris Hanna | d0bfef3 | 2015-06-03 10:28:26 -0400 | [diff] [blame] | 571 | struct cfs_expr_list *el; |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 572 | |
| 573 | rc = cfs_expr_list_parse(cconf->cc_pattern, |
| 574 | strlen(cconf->cc_pattern), |
| 575 | 0, ncpts - 1, &el); |
| 576 | if (rc != 0) { |
| 577 | CERROR("%s: invalid CPT pattern string: %s", |
| 578 | conf->psc_name, cconf->cc_pattern); |
Greg Kroah-Hartman | 0a3bdb0 | 2013-08-03 10:35:28 +0800 | [diff] [blame] | 579 | return ERR_PTR(-EINVAL); |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 580 | } |
| 581 | |
| 582 | rc = cfs_expr_list_values(el, ncpts, &cpts); |
| 583 | cfs_expr_list_free(el); |
| 584 | if (rc <= 0) { |
| 585 | CERROR("%s: failed to parse CPT array %s: %d\n", |
| 586 | conf->psc_name, cconf->cc_pattern, rc); |
Julia Lawall | 207e99c | 2015-05-01 21:37:47 +0200 | [diff] [blame] | 587 | kfree(cpts); |
Greg Kroah-Hartman | 0a3bdb0 | 2013-08-03 10:35:28 +0800 | [diff] [blame] | 588 | return ERR_PTR(rc < 0 ? rc : -EINVAL); |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 589 | } |
| 590 | ncpts = rc; |
| 591 | } |
| 592 | } |
| 593 | |
Julia Lawall | 9ae1059 | 2015-05-01 17:51:12 +0200 | [diff] [blame] | 594 | service = kzalloc(offsetof(struct ptlrpc_service, srv_parts[ncpts]), |
| 595 | GFP_NOFS); |
Julia Lawall | 597851a | 2015-06-20 18:59:10 +0200 | [diff] [blame] | 596 | if (!service) { |
Julia Lawall | 207e99c | 2015-05-01 21:37:47 +0200 | [diff] [blame] | 597 | kfree(cpts); |
Greg Kroah-Hartman | 0a3bdb0 | 2013-08-03 10:35:28 +0800 | [diff] [blame] | 598 | return ERR_PTR(-ENOMEM); |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 599 | } |
| 600 | |
Chris Hanna | d0bfef3 | 2015-06-03 10:28:26 -0400 | [diff] [blame] | 601 | service->srv_cptable = cptable; |
| 602 | service->srv_cpts = cpts; |
| 603 | service->srv_ncpts = ncpts; |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 604 | |
| 605 | service->srv_cpt_bits = 0; /* it's zero already, easy to read... */ |
| 606 | while ((1 << service->srv_cpt_bits) < cfs_cpt_number(cptable)) |
| 607 | service->srv_cpt_bits++; |
| 608 | |
| 609 | /* public members */ |
| 610 | spin_lock_init(&service->srv_lock); |
Chris Hanna | d0bfef3 | 2015-06-03 10:28:26 -0400 | [diff] [blame] | 611 | service->srv_name = conf->psc_name; |
| 612 | service->srv_watchdog_factor = conf->psc_watchdog_factor; |
Masanari Iida | b6da17f | 2014-02-08 00:30:40 +0900 | [diff] [blame] | 613 | INIT_LIST_HEAD(&service->srv_list); /* for safety of cleanup */ |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 614 | |
| 615 | /* buffer configuration */ |
Chris Hanna | d0bfef3 | 2015-06-03 10:28:26 -0400 | [diff] [blame] | 616 | service->srv_nbuf_per_group = test_req_buffer_pressure ? |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 617 | 1 : conf->psc_buf.bc_nbufs; |
Chris Hanna | d0bfef3 | 2015-06-03 10:28:26 -0400 | [diff] [blame] | 618 | service->srv_max_req_size = conf->psc_buf.bc_req_max_size + |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 619 | SPTLRPC_MAX_PAYLOAD; |
Chris Hanna | d0bfef3 | 2015-06-03 10:28:26 -0400 | [diff] [blame] | 620 | service->srv_buf_size = conf->psc_buf.bc_buf_size; |
| 621 | service->srv_rep_portal = conf->psc_buf.bc_rep_portal; |
| 622 | service->srv_req_portal = conf->psc_buf.bc_req_portal; |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 623 | |
| 624 | /* Increase max reply size to next power of two */ |
| 625 | service->srv_max_reply_size = 1; |
| 626 | while (service->srv_max_reply_size < |
| 627 | conf->psc_buf.bc_rep_max_size + SPTLRPC_MAX_PAYLOAD) |
| 628 | service->srv_max_reply_size <<= 1; |
| 629 | |
Chris Hanna | d0bfef3 | 2015-06-03 10:28:26 -0400 | [diff] [blame] | 630 | service->srv_thread_name = conf->psc_thr.tc_thr_name; |
| 631 | service->srv_ctx_tags = conf->psc_thr.tc_ctx_tags; |
| 632 | service->srv_hpreq_ratio = PTLRPC_SVC_HP_RATIO; |
| 633 | service->srv_ops = conf->psc_ops; |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 634 | |
| 635 | for (i = 0; i < ncpts; i++) { |
| 636 | if (!conf->psc_thr.tc_cpu_affinity) |
| 637 | cpt = CFS_CPT_ANY; |
| 638 | else |
Oleg Drokin | 8b38208 | 2016-02-16 00:46:58 -0500 | [diff] [blame] | 639 | cpt = cpts ? cpts[i] : i; |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 640 | |
Julia Lawall | bae97e8 | 2015-05-03 15:21:44 +0200 | [diff] [blame] | 641 | svcpt = kzalloc_node(sizeof(*svcpt), GFP_NOFS, |
| 642 | cfs_cpt_spread_node(cptable, cpt)); |
Oleg Drokin | 8b38208 | 2016-02-16 00:46:58 -0500 | [diff] [blame] | 643 | if (!svcpt) { |
Julia Lawall | a9b3e8f | 2014-09-07 18:18:29 +0200 | [diff] [blame] | 644 | rc = -ENOMEM; |
| 645 | goto failed; |
| 646 | } |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 647 | |
| 648 | service->srv_parts[i] = svcpt; |
| 649 | rc = ptlrpc_service_part_init(service, svcpt, cpt); |
| 650 | if (rc != 0) |
Julia Lawall | a9b3e8f | 2014-09-07 18:18:29 +0200 | [diff] [blame] | 651 | goto failed; |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 652 | } |
| 653 | |
| 654 | ptlrpc_server_nthreads_check(service, conf); |
| 655 | |
| 656 | rc = LNetSetLazyPortal(service->srv_req_portal); |
| 657 | LASSERT(rc == 0); |
| 658 | |
| 659 | mutex_lock(&ptlrpc_all_services_mutex); |
Kristina Martsenko | 3949015e | 2013-11-11 21:34:58 +0200 | [diff] [blame] | 660 | list_add(&service->srv_list, &ptlrpc_all_services); |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 661 | mutex_unlock(&ptlrpc_all_services_mutex); |
| 662 | |
Oleg Drokin | 328676f | 2015-05-21 15:32:08 -0400 | [diff] [blame] | 663 | if (parent) { |
| 664 | rc = ptlrpc_sysfs_register_service(parent, service); |
| 665 | if (rc) |
| 666 | goto failed; |
| 667 | } |
| 668 | |
Dmitry Eremin | 700815d | 2015-05-21 15:32:11 -0400 | [diff] [blame] | 669 | if (!IS_ERR_OR_NULL(debugfs_entry)) |
| 670 | ptlrpc_ldebugfs_register_service(debugfs_entry, service); |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 671 | |
| 672 | rc = ptlrpc_service_nrs_setup(service); |
| 673 | if (rc != 0) |
Julia Lawall | a9b3e8f | 2014-09-07 18:18:29 +0200 | [diff] [blame] | 674 | goto failed; |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 675 | |
| 676 | CDEBUG(D_NET, "%s: Started, listening on portal %d\n", |
| 677 | service->srv_name, service->srv_req_portal); |
| 678 | |
| 679 | rc = ptlrpc_start_threads(service); |
| 680 | if (rc != 0) { |
| 681 | CERROR("Failed to start threads for service %s: %d\n", |
| 682 | service->srv_name, rc); |
Julia Lawall | a9b3e8f | 2014-09-07 18:18:29 +0200 | [diff] [blame] | 683 | goto failed; |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 684 | } |
| 685 | |
Greg Kroah-Hartman | 0a3bdb0 | 2013-08-03 10:35:28 +0800 | [diff] [blame] | 686 | return service; |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 687 | failed: |
| 688 | ptlrpc_unregister_service(service); |
Greg Kroah-Hartman | 0a3bdb0 | 2013-08-03 10:35:28 +0800 | [diff] [blame] | 689 | return ERR_PTR(rc); |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 690 | } |
| 691 | EXPORT_SYMBOL(ptlrpc_register_service); |
| 692 | |
| 693 | /** |
| 694 | * to actually free the request, must be called without holding svc_lock. |
| 695 | * note it's caller's responsibility to unlink req->rq_list. |
| 696 | */ |
| 697 | static void ptlrpc_server_free_request(struct ptlrpc_request *req) |
| 698 | { |
| 699 | LASSERT(atomic_read(&req->rq_refcount) == 0); |
| 700 | LASSERT(list_empty(&req->rq_timed_list)); |
| 701 | |
| 702 | /* DEBUG_REQ() assumes the reply state of a request with a valid |
Oleg Drokin | dadfcda | 2016-02-24 22:00:38 -0500 | [diff] [blame] | 703 | * ref will not be destroyed until that reference is dropped. |
| 704 | */ |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 705 | ptlrpc_req_drop_rs(req); |
| 706 | |
| 707 | sptlrpc_svc_ctx_decref(req); |
| 708 | |
| 709 | if (req != &req->rq_rqbd->rqbd_req) { |
| 710 | /* NB request buffers use an embedded |
| 711 | * req if the incoming req unlinked the |
Oleg Drokin | dadfcda | 2016-02-24 22:00:38 -0500 | [diff] [blame] | 712 | * MD; this isn't one of them! |
| 713 | */ |
Andriy Skulysh | 35b2e1b | 2014-04-27 13:06:35 -0400 | [diff] [blame] | 714 | ptlrpc_request_cache_free(req); |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 715 | } |
| 716 | } |
| 717 | |
| 718 | /** |
| 719 | * drop a reference count of the request. if it reaches 0, we either |
| 720 | * put it into history list, or free it immediately. |
| 721 | */ |
Shraddha Barke | 230a8da1 | 2015-10-08 01:36:07 +0530 | [diff] [blame] | 722 | static void ptlrpc_server_drop_request(struct ptlrpc_request *req) |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 723 | { |
| 724 | struct ptlrpc_request_buffer_desc *rqbd = req->rq_rqbd; |
Chris Hanna | d0bfef3 | 2015-06-03 10:28:26 -0400 | [diff] [blame] | 725 | struct ptlrpc_service_part *svcpt = rqbd->rqbd_svcpt; |
| 726 | struct ptlrpc_service *svc = svcpt->scp_service; |
| 727 | int refcount; |
| 728 | struct list_head *tmp; |
| 729 | struct list_head *nxt; |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 730 | |
| 731 | if (!atomic_dec_and_test(&req->rq_refcount)) |
| 732 | return; |
| 733 | |
| 734 | if (req->rq_at_linked) { |
| 735 | spin_lock(&svcpt->scp_at_lock); |
| 736 | /* recheck with lock, in case it's unlinked by |
Oleg Drokin | dadfcda | 2016-02-24 22:00:38 -0500 | [diff] [blame] | 737 | * ptlrpc_at_check_timed() |
| 738 | */ |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 739 | if (likely(req->rq_at_linked)) |
| 740 | ptlrpc_at_remove_timed(req); |
| 741 | spin_unlock(&svcpt->scp_at_lock); |
| 742 | } |
| 743 | |
| 744 | LASSERT(list_empty(&req->rq_timed_list)); |
| 745 | |
| 746 | /* finalize request */ |
| 747 | if (req->rq_export) { |
| 748 | class_export_put(req->rq_export); |
| 749 | req->rq_export = NULL; |
| 750 | } |
| 751 | |
| 752 | spin_lock(&svcpt->scp_lock); |
| 753 | |
| 754 | list_add(&req->rq_list, &rqbd->rqbd_reqs); |
| 755 | |
| 756 | refcount = --(rqbd->rqbd_refcount); |
| 757 | if (refcount == 0) { |
| 758 | /* request buffer is now idle: add to history */ |
| 759 | list_del(&rqbd->rqbd_list); |
| 760 | |
| 761 | list_add_tail(&rqbd->rqbd_list, &svcpt->scp_hist_rqbds); |
| 762 | svcpt->scp_hist_nrqbds++; |
| 763 | |
| 764 | /* cull some history? |
Oleg Drokin | dadfcda | 2016-02-24 22:00:38 -0500 | [diff] [blame] | 765 | * I expect only about 1 or 2 rqbds need to be recycled here |
| 766 | */ |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 767 | while (svcpt->scp_hist_nrqbds > svc->srv_hist_nrqbds_cpt_max) { |
| 768 | rqbd = list_entry(svcpt->scp_hist_rqbds.next, |
Oleg Drokin | 30c0aa3 | 2016-02-26 01:50:02 -0500 | [diff] [blame] | 769 | struct ptlrpc_request_buffer_desc, |
| 770 | rqbd_list); |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 771 | |
| 772 | list_del(&rqbd->rqbd_list); |
| 773 | svcpt->scp_hist_nrqbds--; |
| 774 | |
| 775 | /* remove rqbd's reqs from svc's req history while |
Oleg Drokin | dadfcda | 2016-02-24 22:00:38 -0500 | [diff] [blame] | 776 | * I've got the service lock |
| 777 | */ |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 778 | list_for_each(tmp, &rqbd->rqbd_reqs) { |
| 779 | req = list_entry(tmp, struct ptlrpc_request, |
Oleg Drokin | 30c0aa3 | 2016-02-26 01:50:02 -0500 | [diff] [blame] | 780 | rq_list); |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 781 | /* Track the highest culled req seq */ |
| 782 | if (req->rq_history_seq > |
| 783 | svcpt->scp_hist_seq_culled) { |
| 784 | svcpt->scp_hist_seq_culled = |
| 785 | req->rq_history_seq; |
| 786 | } |
| 787 | list_del(&req->rq_history_list); |
| 788 | } |
| 789 | |
| 790 | spin_unlock(&svcpt->scp_lock); |
| 791 | |
| 792 | list_for_each_safe(tmp, nxt, &rqbd->rqbd_reqs) { |
| 793 | req = list_entry(rqbd->rqbd_reqs.next, |
Oleg Drokin | 30c0aa3 | 2016-02-26 01:50:02 -0500 | [diff] [blame] | 794 | struct ptlrpc_request, |
| 795 | rq_list); |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 796 | list_del(&req->rq_list); |
| 797 | ptlrpc_server_free_request(req); |
| 798 | } |
| 799 | |
| 800 | spin_lock(&svcpt->scp_lock); |
| 801 | /* |
| 802 | * now all reqs including the embedded req has been |
| 803 | * disposed, schedule request buffer for re-use. |
| 804 | */ |
| 805 | LASSERT(atomic_read(&rqbd->rqbd_req.rq_refcount) == |
| 806 | 0); |
Oleg Drokin | 30c0aa3 | 2016-02-26 01:50:02 -0500 | [diff] [blame] | 807 | list_add_tail(&rqbd->rqbd_list, &svcpt->scp_rqbd_idle); |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 808 | } |
| 809 | |
| 810 | spin_unlock(&svcpt->scp_lock); |
| 811 | } else if (req->rq_reply_state && req->rq_reply_state->rs_prealloc) { |
| 812 | /* If we are low on memory, we are not interested in history */ |
| 813 | list_del(&req->rq_list); |
| 814 | list_del_init(&req->rq_history_list); |
| 815 | |
| 816 | /* Track the highest culled req seq */ |
| 817 | if (req->rq_history_seq > svcpt->scp_hist_seq_culled) |
| 818 | svcpt->scp_hist_seq_culled = req->rq_history_seq; |
| 819 | |
| 820 | spin_unlock(&svcpt->scp_lock); |
| 821 | |
| 822 | ptlrpc_server_free_request(req); |
| 823 | } else { |
| 824 | spin_unlock(&svcpt->scp_lock); |
| 825 | } |
| 826 | } |
| 827 | |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 828 | /** |
| 829 | * to finish a request: stop sending more early replies, and release |
| 830 | * the request. |
| 831 | */ |
| 832 | static void ptlrpc_server_finish_request(struct ptlrpc_service_part *svcpt, |
| 833 | struct ptlrpc_request *req) |
| 834 | { |
| 835 | ptlrpc_server_hpreq_fini(req); |
| 836 | |
Mikhail Pershin | 7018750 | 2016-04-27 21:37:20 -0400 | [diff] [blame] | 837 | if (req->rq_session.lc_thread) { |
| 838 | lu_context_exit(&req->rq_session); |
| 839 | lu_context_fini(&req->rq_session); |
| 840 | } |
| 841 | |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 842 | ptlrpc_server_drop_request(req); |
| 843 | } |
| 844 | |
| 845 | /** |
| 846 | * to finish a active request: stop sending more early replies, and release |
| 847 | * the request. should be called after we finished handling the request. |
| 848 | */ |
| 849 | static void ptlrpc_server_finish_active_request( |
| 850 | struct ptlrpc_service_part *svcpt, |
| 851 | struct ptlrpc_request *req) |
| 852 | { |
| 853 | spin_lock(&svcpt->scp_req_lock); |
| 854 | ptlrpc_nrs_req_stop_nolock(req); |
| 855 | svcpt->scp_nreqs_active--; |
| 856 | if (req->rq_hp) |
| 857 | svcpt->scp_nhreqs_active--; |
| 858 | spin_unlock(&svcpt->scp_req_lock); |
| 859 | |
| 860 | ptlrpc_nrs_req_finalize(req); |
| 861 | |
Oleg Drokin | 8b38208 | 2016-02-16 00:46:58 -0500 | [diff] [blame] | 862 | if (req->rq_export) |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 863 | class_export_rpc_dec(req->rq_export); |
| 864 | |
| 865 | ptlrpc_server_finish_request(svcpt, req); |
| 866 | } |
| 867 | |
| 868 | /** |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 869 | * Sanity check request \a req. |
| 870 | * Return 0 if all is ok, error code otherwise. |
| 871 | */ |
| 872 | static int ptlrpc_check_req(struct ptlrpc_request *req) |
| 873 | { |
Dmitry Eremin | f60d7c3 | 2014-06-22 21:32:09 -0400 | [diff] [blame] | 874 | struct obd_device *obd = req->rq_export->exp_obd; |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 875 | int rc = 0; |
| 876 | |
| 877 | if (unlikely(lustre_msg_get_conn_cnt(req->rq_reqmsg) < |
| 878 | req->rq_export->exp_conn_cnt)) { |
| 879 | DEBUG_REQ(D_RPCTRACE, req, |
| 880 | "DROPPING req from old connection %d < %d", |
| 881 | lustre_msg_get_conn_cnt(req->rq_reqmsg), |
| 882 | req->rq_export->exp_conn_cnt); |
| 883 | return -EEXIST; |
| 884 | } |
Oleg Drokin | 8b38208 | 2016-02-16 00:46:58 -0500 | [diff] [blame] | 885 | if (unlikely(!obd || obd->obd_fail)) { |
Kristina Martsenko | 532118c | 2013-11-11 21:35:03 +0200 | [diff] [blame] | 886 | /* |
| 887 | * Failing over, don't handle any more reqs, send |
| 888 | * error response instead. |
| 889 | */ |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 890 | CDEBUG(D_RPCTRACE, "Dropping req %p for failed obd %s\n", |
Oleg Drokin | 8b38208 | 2016-02-16 00:46:58 -0500 | [diff] [blame] | 891 | req, obd ? obd->obd_name : "unknown"); |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 892 | rc = -ENODEV; |
| 893 | } else if (lustre_msg_get_flags(req->rq_reqmsg) & |
Oleg Drokin | af3ec53 | 2015-09-28 23:44:09 -0400 | [diff] [blame] | 894 | (MSG_REPLAY | MSG_REQ_REPLAY_DONE)) { |
| 895 | DEBUG_REQ(D_ERROR, req, "Invalid replay without recovery"); |
| 896 | class_fail_export(req->rq_export); |
| 897 | rc = -ENODEV; |
| 898 | } else if (lustre_msg_get_transno(req->rq_reqmsg) != 0) { |
| 899 | DEBUG_REQ(D_ERROR, req, |
| 900 | "Invalid req with transno %llu without recovery", |
| 901 | lustre_msg_get_transno(req->rq_reqmsg)); |
| 902 | class_fail_export(req->rq_export); |
| 903 | rc = -ENODEV; |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 904 | } |
| 905 | |
| 906 | if (unlikely(rc < 0)) { |
| 907 | req->rq_status = rc; |
| 908 | ptlrpc_error(req); |
| 909 | } |
| 910 | return rc; |
| 911 | } |
| 912 | |
| 913 | static void ptlrpc_at_set_timer(struct ptlrpc_service_part *svcpt) |
| 914 | { |
| 915 | struct ptlrpc_at_array *array = &svcpt->scp_at_array; |
| 916 | __s32 next; |
| 917 | |
| 918 | if (array->paa_count == 0) { |
Arnd Bergmann | 922da0c | 2015-09-27 16:45:33 -0400 | [diff] [blame] | 919 | del_timer(&svcpt->scp_at_timer); |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 920 | return; |
| 921 | } |
| 922 | |
| 923 | /* Set timer for closest deadline */ |
Arnd Bergmann | 219e6de | 2015-09-27 16:45:30 -0400 | [diff] [blame] | 924 | next = (__s32)(array->paa_deadline - ktime_get_real_seconds() - |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 925 | at_early_margin); |
| 926 | if (next <= 0) { |
| 927 | ptlrpc_at_timer((unsigned long)svcpt); |
| 928 | } else { |
Arnd Bergmann | 922da0c | 2015-09-27 16:45:33 -0400 | [diff] [blame] | 929 | mod_timer(&svcpt->scp_at_timer, cfs_time_shift(next)); |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 930 | CDEBUG(D_INFO, "armed %s at %+ds\n", |
| 931 | svcpt->scp_service->srv_name, next); |
| 932 | } |
| 933 | } |
| 934 | |
| 935 | /* Add rpc to early reply check list */ |
| 936 | static int ptlrpc_at_add_timed(struct ptlrpc_request *req) |
| 937 | { |
| 938 | struct ptlrpc_service_part *svcpt = req->rq_rqbd->rqbd_svcpt; |
| 939 | struct ptlrpc_at_array *array = &svcpt->scp_at_array; |
| 940 | struct ptlrpc_request *rq = NULL; |
| 941 | __u32 index; |
| 942 | |
| 943 | if (AT_OFF) |
Julia Lawall | fbe7c6c | 2014-08-26 22:00:33 +0200 | [diff] [blame] | 944 | return 0; |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 945 | |
| 946 | if (req->rq_no_reply) |
| 947 | return 0; |
| 948 | |
| 949 | if ((lustre_msghdr_get_flags(req->rq_reqmsg) & MSGHDR_AT_SUPPORT) == 0) |
Julia Lawall | fbe7c6c | 2014-08-26 22:00:33 +0200 | [diff] [blame] | 950 | return -ENOSYS; |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 951 | |
| 952 | spin_lock(&svcpt->scp_at_lock); |
| 953 | LASSERT(list_empty(&req->rq_timed_list)); |
| 954 | |
Arnd Bergmann | 219e6de | 2015-09-27 16:45:30 -0400 | [diff] [blame] | 955 | div_u64_rem(req->rq_deadline, array->paa_size, &index); |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 956 | if (array->paa_reqs_count[index] > 0) { |
| 957 | /* latest rpcs will have the latest deadlines in the list, |
Oleg Drokin | dadfcda | 2016-02-24 22:00:38 -0500 | [diff] [blame] | 958 | * so search backward. |
| 959 | */ |
Oleg Drokin | 30c0aa3 | 2016-02-26 01:50:02 -0500 | [diff] [blame] | 960 | list_for_each_entry_reverse(rq, &array->paa_reqs_array[index], |
| 961 | rq_timed_list) { |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 962 | if (req->rq_deadline >= rq->rq_deadline) { |
| 963 | list_add(&req->rq_timed_list, |
Oleg Drokin | 30c0aa3 | 2016-02-26 01:50:02 -0500 | [diff] [blame] | 964 | &rq->rq_timed_list); |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 965 | break; |
| 966 | } |
| 967 | } |
| 968 | } |
| 969 | |
| 970 | /* Add the request at the head of the list */ |
| 971 | if (list_empty(&req->rq_timed_list)) |
Oleg Drokin | 30c0aa3 | 2016-02-26 01:50:02 -0500 | [diff] [blame] | 972 | list_add(&req->rq_timed_list, &array->paa_reqs_array[index]); |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 973 | |
| 974 | spin_lock(&req->rq_lock); |
| 975 | req->rq_at_linked = 1; |
| 976 | spin_unlock(&req->rq_lock); |
| 977 | req->rq_at_index = index; |
| 978 | array->paa_reqs_count[index]++; |
| 979 | array->paa_count++; |
| 980 | if (array->paa_count == 1 || array->paa_deadline > req->rq_deadline) { |
| 981 | array->paa_deadline = req->rq_deadline; |
| 982 | ptlrpc_at_set_timer(svcpt); |
| 983 | } |
| 984 | spin_unlock(&svcpt->scp_at_lock); |
| 985 | |
| 986 | return 0; |
| 987 | } |
| 988 | |
| 989 | static void |
| 990 | ptlrpc_at_remove_timed(struct ptlrpc_request *req) |
| 991 | { |
| 992 | struct ptlrpc_at_array *array; |
| 993 | |
| 994 | array = &req->rq_rqbd->rqbd_svcpt->scp_at_array; |
| 995 | |
| 996 | /* NB: must call with hold svcpt::scp_at_lock */ |
| 997 | LASSERT(!list_empty(&req->rq_timed_list)); |
| 998 | list_del_init(&req->rq_timed_list); |
| 999 | |
| 1000 | spin_lock(&req->rq_lock); |
| 1001 | req->rq_at_linked = 0; |
| 1002 | spin_unlock(&req->rq_lock); |
| 1003 | |
| 1004 | array->paa_reqs_count[req->rq_at_index]--; |
| 1005 | array->paa_count--; |
| 1006 | } |
| 1007 | |
Chris Horn | 31c5e95 | 2016-08-16 16:18:53 -0400 | [diff] [blame] | 1008 | /* |
| 1009 | * Attempt to extend the request deadline by sending an early reply to the |
| 1010 | * client. |
| 1011 | */ |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 1012 | static int ptlrpc_at_send_early_reply(struct ptlrpc_request *req) |
| 1013 | { |
| 1014 | struct ptlrpc_service_part *svcpt = req->rq_rqbd->rqbd_svcpt; |
| 1015 | struct ptlrpc_request *reqcopy; |
| 1016 | struct lustre_msg *reqmsg; |
Arnd Bergmann | 219e6de | 2015-09-27 16:45:30 -0400 | [diff] [blame] | 1017 | long olddl = req->rq_deadline - ktime_get_real_seconds(); |
Mikhail Pershin | be24798 | 2016-09-18 16:38:53 -0400 | [diff] [blame] | 1018 | time64_t newdl; |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 1019 | int rc; |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 1020 | |
| 1021 | /* deadline is when the client expects us to reply, margin is the |
Oleg Drokin | dadfcda | 2016-02-24 22:00:38 -0500 | [diff] [blame] | 1022 | * difference between clients' and servers' expectations |
| 1023 | */ |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 1024 | DEBUG_REQ(D_ADAPTTO, req, |
Joe Perches | 2d00bd1 | 2014-11-23 11:28:50 -0800 | [diff] [blame] | 1025 | "%ssending early reply (deadline %+lds, margin %+lds) for %d+%d", |
| 1026 | AT_OFF ? "AT off - not " : "", |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 1027 | olddl, olddl - at_get(&svcpt->scp_at_estimate), |
| 1028 | at_get(&svcpt->scp_at_estimate), at_extra); |
| 1029 | |
| 1030 | if (AT_OFF) |
Greg Kroah-Hartman | 0a3bdb0 | 2013-08-03 10:35:28 +0800 | [diff] [blame] | 1031 | return 0; |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 1032 | |
| 1033 | if (olddl < 0) { |
Joe Perches | 2d00bd1 | 2014-11-23 11:28:50 -0800 | [diff] [blame] | 1034 | DEBUG_REQ(D_WARNING, req, "Already past deadline (%+lds), not sending early reply. Consider increasing at_early_margin (%d)?", |
| 1035 | olddl, at_early_margin); |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 1036 | |
| 1037 | /* Return an error so we're not re-added to the timed list. */ |
Greg Kroah-Hartman | 0a3bdb0 | 2013-08-03 10:35:28 +0800 | [diff] [blame] | 1038 | return -ETIMEDOUT; |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 1039 | } |
| 1040 | |
Kristina Martsenko | cb68dd2 | 2013-11-11 21:34:59 +0200 | [diff] [blame] | 1041 | if (!(lustre_msghdr_get_flags(req->rq_reqmsg) & MSGHDR_AT_SUPPORT)) { |
Joe Perches | 2d00bd1 | 2014-11-23 11:28:50 -0800 | [diff] [blame] | 1042 | DEBUG_REQ(D_INFO, req, "Wanted to ask client for more time, but no AT support"); |
Greg Kroah-Hartman | 0a3bdb0 | 2013-08-03 10:35:28 +0800 | [diff] [blame] | 1043 | return -ENOSYS; |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 1044 | } |
| 1045 | |
Chris Horn | 31c5e95 | 2016-08-16 16:18:53 -0400 | [diff] [blame] | 1046 | /* |
| 1047 | * We want to extend the request deadline by at_extra seconds, |
| 1048 | * so we set our service estimate to reflect how much time has |
| 1049 | * passed since this request arrived plus an additional |
| 1050 | * at_extra seconds. The client will calculate the new deadline |
| 1051 | * based on this service estimate (plus some additional time to |
| 1052 | * account for network latency). See ptlrpc_at_recv_early_reply |
Oleg Drokin | dadfcda | 2016-02-24 22:00:38 -0500 | [diff] [blame] | 1053 | */ |
Oleg Drokin | af3ec53 | 2015-09-28 23:44:09 -0400 | [diff] [blame] | 1054 | at_measured(&svcpt->scp_at_estimate, at_extra + |
| 1055 | ktime_get_real_seconds() - req->rq_arrival_time.tv_sec); |
Mikhail Pershin | be24798 | 2016-09-18 16:38:53 -0400 | [diff] [blame] | 1056 | newdl = req->rq_arrival_time.tv_sec + at_get(&svcpt->scp_at_estimate); |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 1057 | |
Oleg Drokin | af3ec53 | 2015-09-28 23:44:09 -0400 | [diff] [blame] | 1058 | /* Check to see if we've actually increased the deadline - |
Oleg Drokin | dadfcda | 2016-02-24 22:00:38 -0500 | [diff] [blame] | 1059 | * we may be past adaptive_max |
| 1060 | */ |
Mikhail Pershin | be24798 | 2016-09-18 16:38:53 -0400 | [diff] [blame] | 1061 | if (req->rq_deadline >= newdl) { |
Oleg Drokin | af3ec53 | 2015-09-28 23:44:09 -0400 | [diff] [blame] | 1062 | DEBUG_REQ(D_WARNING, req, "Couldn't add any time (%ld/%lld), not sending early reply\n", |
Mikhail Pershin | be24798 | 2016-09-18 16:38:53 -0400 | [diff] [blame] | 1063 | olddl, newdl - ktime_get_real_seconds()); |
Oleg Drokin | af3ec53 | 2015-09-28 23:44:09 -0400 | [diff] [blame] | 1064 | return -ETIMEDOUT; |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 1065 | } |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 1066 | |
Ann Koehler | 0be19af | 2014-04-27 13:06:36 -0400 | [diff] [blame] | 1067 | reqcopy = ptlrpc_request_cache_alloc(GFP_NOFS); |
Oleg Drokin | 8b38208 | 2016-02-16 00:46:58 -0500 | [diff] [blame] | 1068 | if (!reqcopy) |
Greg Kroah-Hartman | 0a3bdb0 | 2013-08-03 10:35:28 +0800 | [diff] [blame] | 1069 | return -ENOMEM; |
Julia Lawall | ee0ec19 | 2015-06-11 14:02:58 +0200 | [diff] [blame] | 1070 | reqmsg = libcfs_kvzalloc(req->rq_reqlen, GFP_NOFS); |
Julia Lawall | a9b3e8f | 2014-09-07 18:18:29 +0200 | [diff] [blame] | 1071 | if (!reqmsg) { |
| 1072 | rc = -ENOMEM; |
| 1073 | goto out_free; |
| 1074 | } |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 1075 | |
| 1076 | *reqcopy = *req; |
| 1077 | reqcopy->rq_reply_state = NULL; |
| 1078 | reqcopy->rq_rep_swab_mask = 0; |
| 1079 | reqcopy->rq_pack_bulk = 0; |
| 1080 | reqcopy->rq_pack_udesc = 0; |
| 1081 | reqcopy->rq_packed_final = 0; |
| 1082 | sptlrpc_svc_ctx_addref(reqcopy); |
| 1083 | /* We only need the reqmsg for the magic */ |
| 1084 | reqcopy->rq_reqmsg = reqmsg; |
| 1085 | memcpy(reqmsg, req->rq_reqmsg, req->rq_reqlen); |
| 1086 | |
| 1087 | LASSERT(atomic_read(&req->rq_refcount)); |
| 1088 | /** if it is last refcount then early reply isn't needed */ |
| 1089 | if (atomic_read(&req->rq_refcount) == 1) { |
Joe Perches | 2d00bd1 | 2014-11-23 11:28:50 -0800 | [diff] [blame] | 1090 | DEBUG_REQ(D_ADAPTTO, reqcopy, "Normal reply already sent out, abort sending early reply\n"); |
Julia Lawall | a9b3e8f | 2014-09-07 18:18:29 +0200 | [diff] [blame] | 1091 | rc = -EINVAL; |
| 1092 | goto out; |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 1093 | } |
| 1094 | |
| 1095 | /* Connection ref */ |
| 1096 | reqcopy->rq_export = class_conn2export( |
| 1097 | lustre_msg_get_handle(reqcopy->rq_reqmsg)); |
Oleg Drokin | 8b38208 | 2016-02-16 00:46:58 -0500 | [diff] [blame] | 1098 | if (!reqcopy->rq_export) { |
Julia Lawall | a9b3e8f | 2014-09-07 18:18:29 +0200 | [diff] [blame] | 1099 | rc = -ENODEV; |
| 1100 | goto out; |
| 1101 | } |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 1102 | |
| 1103 | /* RPC ref */ |
| 1104 | class_export_rpc_inc(reqcopy->rq_export); |
| 1105 | if (reqcopy->rq_export->exp_obd && |
Julia Lawall | a9b3e8f | 2014-09-07 18:18:29 +0200 | [diff] [blame] | 1106 | reqcopy->rq_export->exp_obd->obd_fail) { |
| 1107 | rc = -ENODEV; |
| 1108 | goto out_put; |
| 1109 | } |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 1110 | |
| 1111 | rc = lustre_pack_reply_flags(reqcopy, 1, NULL, NULL, LPRFL_EARLY_REPLY); |
| 1112 | if (rc) |
Julia Lawall | a9b3e8f | 2014-09-07 18:18:29 +0200 | [diff] [blame] | 1113 | goto out_put; |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 1114 | |
| 1115 | rc = ptlrpc_send_reply(reqcopy, PTLRPC_REPLY_EARLY); |
| 1116 | |
| 1117 | if (!rc) { |
| 1118 | /* Adjust our own deadline to what we told the client */ |
Mikhail Pershin | be24798 | 2016-09-18 16:38:53 -0400 | [diff] [blame] | 1119 | req->rq_deadline = newdl; |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 1120 | req->rq_early_count++; /* number sent, server side */ |
| 1121 | } else { |
| 1122 | DEBUG_REQ(D_ERROR, req, "Early reply send failed %d", rc); |
| 1123 | } |
| 1124 | |
| 1125 | /* Free the (early) reply state from lustre_pack_reply. |
Oleg Drokin | dadfcda | 2016-02-24 22:00:38 -0500 | [diff] [blame] | 1126 | * (ptlrpc_send_reply takes it's own rs ref, so this is safe here) |
| 1127 | */ |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 1128 | ptlrpc_req_drop_rs(reqcopy); |
| 1129 | |
| 1130 | out_put: |
| 1131 | class_export_rpc_dec(reqcopy->rq_export); |
| 1132 | class_export_put(reqcopy->rq_export); |
| 1133 | out: |
| 1134 | sptlrpc_svc_ctx_decref(reqcopy); |
Julia Lawall | ee0ec19 | 2015-06-11 14:02:58 +0200 | [diff] [blame] | 1135 | kvfree(reqmsg); |
Andriy Skulysh | 35b2e1b | 2014-04-27 13:06:35 -0400 | [diff] [blame] | 1136 | out_free: |
| 1137 | ptlrpc_request_cache_free(reqcopy); |
Greg Kroah-Hartman | 0a3bdb0 | 2013-08-03 10:35:28 +0800 | [diff] [blame] | 1138 | return rc; |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 1139 | } |
| 1140 | |
| 1141 | /* Send early replies to everybody expiring within at_early_margin |
Oleg Drokin | dadfcda | 2016-02-24 22:00:38 -0500 | [diff] [blame] | 1142 | * asking for at_extra time |
| 1143 | */ |
Oleg Drokin | 80b6f29 | 2016-02-26 01:49:54 -0500 | [diff] [blame] | 1144 | static void ptlrpc_at_check_timed(struct ptlrpc_service_part *svcpt) |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 1145 | { |
| 1146 | struct ptlrpc_at_array *array = &svcpt->scp_at_array; |
| 1147 | struct ptlrpc_request *rq, *n; |
| 1148 | struct list_head work_list; |
Chris Hanna | d0bfef3 | 2015-06-03 10:28:26 -0400 | [diff] [blame] | 1149 | __u32 index, count; |
Arnd Bergmann | 219e6de | 2015-09-27 16:45:30 -0400 | [diff] [blame] | 1150 | time64_t deadline; |
| 1151 | time64_t now = ktime_get_real_seconds(); |
Greg Kroah-Hartman | b2d201b | 2014-07-12 00:45:51 -0700 | [diff] [blame] | 1152 | long delay; |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 1153 | int first, counter = 0; |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 1154 | |
| 1155 | spin_lock(&svcpt->scp_at_lock); |
| 1156 | if (svcpt->scp_at_check == 0) { |
| 1157 | spin_unlock(&svcpt->scp_at_lock); |
Oleg Drokin | 80b6f29 | 2016-02-26 01:49:54 -0500 | [diff] [blame] | 1158 | return; |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 1159 | } |
| 1160 | delay = cfs_time_sub(cfs_time_current(), svcpt->scp_at_checktime); |
| 1161 | svcpt->scp_at_check = 0; |
| 1162 | |
| 1163 | if (array->paa_count == 0) { |
| 1164 | spin_unlock(&svcpt->scp_at_lock); |
Oleg Drokin | 80b6f29 | 2016-02-26 01:49:54 -0500 | [diff] [blame] | 1165 | return; |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 1166 | } |
| 1167 | |
| 1168 | /* The timer went off, but maybe the nearest rpc already completed. */ |
| 1169 | first = array->paa_deadline - now; |
| 1170 | if (first > at_early_margin) { |
| 1171 | /* We've still got plenty of time. Reset the timer. */ |
| 1172 | ptlrpc_at_set_timer(svcpt); |
| 1173 | spin_unlock(&svcpt->scp_at_lock); |
Oleg Drokin | 80b6f29 | 2016-02-26 01:49:54 -0500 | [diff] [blame] | 1174 | return; |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 1175 | } |
| 1176 | |
| 1177 | /* We're close to a timeout, and we don't know how much longer the |
Oleg Drokin | dadfcda | 2016-02-24 22:00:38 -0500 | [diff] [blame] | 1178 | * server will take. Send early replies to everyone expiring soon. |
| 1179 | */ |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 1180 | INIT_LIST_HEAD(&work_list); |
| 1181 | deadline = -1; |
Arnd Bergmann | 219e6de | 2015-09-27 16:45:30 -0400 | [diff] [blame] | 1182 | div_u64_rem(array->paa_deadline, array->paa_size, &index); |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 1183 | count = array->paa_count; |
| 1184 | while (count > 0) { |
| 1185 | count -= array->paa_reqs_count[index]; |
Oleg Drokin | 30c0aa3 | 2016-02-26 01:50:02 -0500 | [diff] [blame] | 1186 | list_for_each_entry_safe(rq, n, &array->paa_reqs_array[index], |
| 1187 | rq_timed_list) { |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 1188 | if (rq->rq_deadline > now + at_early_margin) { |
| 1189 | /* update the earliest deadline */ |
| 1190 | if (deadline == -1 || |
| 1191 | rq->rq_deadline < deadline) |
| 1192 | deadline = rq->rq_deadline; |
| 1193 | break; |
| 1194 | } |
| 1195 | |
| 1196 | ptlrpc_at_remove_timed(rq); |
| 1197 | /** |
| 1198 | * ptlrpc_server_drop_request() may drop |
| 1199 | * refcount to 0 already. Let's check this and |
| 1200 | * don't add entry to work_list |
| 1201 | */ |
| 1202 | if (likely(atomic_inc_not_zero(&rq->rq_refcount))) |
| 1203 | list_add(&rq->rq_timed_list, &work_list); |
| 1204 | counter++; |
| 1205 | } |
| 1206 | |
| 1207 | if (++index >= array->paa_size) |
| 1208 | index = 0; |
| 1209 | } |
| 1210 | array->paa_deadline = deadline; |
| 1211 | /* we have a new earliest deadline, restart the timer */ |
| 1212 | ptlrpc_at_set_timer(svcpt); |
| 1213 | |
| 1214 | spin_unlock(&svcpt->scp_at_lock); |
| 1215 | |
Joe Perches | 2d00bd1 | 2014-11-23 11:28:50 -0800 | [diff] [blame] | 1216 | CDEBUG(D_ADAPTTO, "timeout in %+ds, asking for %d secs on %d early replies\n", |
| 1217 | first, at_extra, counter); |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 1218 | if (first < 0) { |
| 1219 | /* We're already past request deadlines before we even get a |
Oleg Drokin | dadfcda | 2016-02-24 22:00:38 -0500 | [diff] [blame] | 1220 | * chance to send early replies |
| 1221 | */ |
Joe Perches | 2d00bd1 | 2014-11-23 11:28:50 -0800 | [diff] [blame] | 1222 | LCONSOLE_WARN("%s: This server is not able to keep up with request traffic (cpu-bound).\n", |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 1223 | svcpt->scp_service->srv_name); |
Arnd Bergmann | 219e6de | 2015-09-27 16:45:30 -0400 | [diff] [blame] | 1224 | CWARN("earlyQ=%d reqQ=%d recA=%d, svcEst=%d, delay=%ld(jiff)\n", |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 1225 | counter, svcpt->scp_nreqs_incoming, |
| 1226 | svcpt->scp_nreqs_active, |
| 1227 | at_get(&svcpt->scp_at_estimate), delay); |
| 1228 | } |
| 1229 | |
| 1230 | /* we took additional refcount so entries can't be deleted from list, no |
Oleg Drokin | dadfcda | 2016-02-24 22:00:38 -0500 | [diff] [blame] | 1231 | * locking is needed |
| 1232 | */ |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 1233 | while (!list_empty(&work_list)) { |
| 1234 | rq = list_entry(work_list.next, struct ptlrpc_request, |
Oleg Drokin | 30c0aa3 | 2016-02-26 01:50:02 -0500 | [diff] [blame] | 1235 | rq_timed_list); |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 1236 | list_del_init(&rq->rq_timed_list); |
| 1237 | |
| 1238 | if (ptlrpc_at_send_early_reply(rq) == 0) |
| 1239 | ptlrpc_at_add_timed(rq); |
| 1240 | |
| 1241 | ptlrpc_server_drop_request(rq); |
| 1242 | } |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 1243 | } |
| 1244 | |
| 1245 | /** |
| 1246 | * Put the request to the export list if the request may become |
| 1247 | * a high priority one. |
| 1248 | */ |
| 1249 | static int ptlrpc_server_hpreq_init(struct ptlrpc_service_part *svcpt, |
| 1250 | struct ptlrpc_request *req) |
| 1251 | { |
| 1252 | int rc = 0; |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 1253 | |
| 1254 | if (svcpt->scp_service->srv_ops.so_hpreq_handler) { |
| 1255 | rc = svcpt->scp_service->srv_ops.so_hpreq_handler(req); |
| 1256 | if (rc < 0) |
Greg Kroah-Hartman | 0a3bdb0 | 2013-08-03 10:35:28 +0800 | [diff] [blame] | 1257 | return rc; |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 1258 | LASSERT(rc == 0); |
| 1259 | } |
| 1260 | if (req->rq_export && req->rq_ops) { |
| 1261 | /* Perform request specific check. We should do this check |
| 1262 | * before the request is added into exp_hp_rpcs list otherwise |
Oleg Drokin | dadfcda | 2016-02-24 22:00:38 -0500 | [diff] [blame] | 1263 | * it may hit swab race at LU-1044. |
| 1264 | */ |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 1265 | if (req->rq_ops->hpreq_check) { |
| 1266 | rc = req->rq_ops->hpreq_check(req); |
| 1267 | /** |
| 1268 | * XXX: Out of all current |
| 1269 | * ptlrpc_hpreq_ops::hpreq_check(), only |
| 1270 | * ldlm_cancel_hpreq_check() can return an error code; |
| 1271 | * other functions assert in similar places, which seems |
| 1272 | * odd. What also does not seem right is that handlers |
| 1273 | * for those RPCs do not assert on the same checks, but |
| 1274 | * rather handle the error cases. e.g. see |
| 1275 | * ost_rw_hpreq_check(), and ost_brw_read(), |
| 1276 | * ost_brw_write(). |
| 1277 | */ |
| 1278 | if (rc < 0) |
Greg Kroah-Hartman | 0a3bdb0 | 2013-08-03 10:35:28 +0800 | [diff] [blame] | 1279 | return rc; |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 1280 | LASSERT(rc == 0 || rc == 1); |
| 1281 | } |
| 1282 | |
| 1283 | spin_lock_bh(&req->rq_export->exp_rpc_lock); |
Oleg Drokin | 30c0aa3 | 2016-02-26 01:50:02 -0500 | [diff] [blame] | 1284 | list_add(&req->rq_exp_list, &req->rq_export->exp_hp_rpcs); |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 1285 | spin_unlock_bh(&req->rq_export->exp_rpc_lock); |
| 1286 | } |
| 1287 | |
| 1288 | ptlrpc_nrs_req_initialize(svcpt, req, rc); |
| 1289 | |
Greg Kroah-Hartman | 0a3bdb0 | 2013-08-03 10:35:28 +0800 | [diff] [blame] | 1290 | return rc; |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 1291 | } |
| 1292 | |
| 1293 | /** Remove the request from the export list. */ |
| 1294 | static void ptlrpc_server_hpreq_fini(struct ptlrpc_request *req) |
| 1295 | { |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 1296 | if (req->rq_export && req->rq_ops) { |
| 1297 | /* refresh lock timeout again so that client has more |
Oleg Drokin | dadfcda | 2016-02-24 22:00:38 -0500 | [diff] [blame] | 1298 | * room to send lock cancel RPC. |
| 1299 | */ |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 1300 | if (req->rq_ops->hpreq_fini) |
| 1301 | req->rq_ops->hpreq_fini(req); |
| 1302 | |
| 1303 | spin_lock_bh(&req->rq_export->exp_rpc_lock); |
| 1304 | list_del_init(&req->rq_exp_list); |
| 1305 | spin_unlock_bh(&req->rq_export->exp_rpc_lock); |
| 1306 | } |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 1307 | } |
| 1308 | |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 1309 | static int ptlrpc_server_request_add(struct ptlrpc_service_part *svcpt, |
| 1310 | struct ptlrpc_request *req) |
| 1311 | { |
| 1312 | int rc; |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 1313 | |
| 1314 | rc = ptlrpc_server_hpreq_init(svcpt, req); |
| 1315 | if (rc < 0) |
Greg Kroah-Hartman | 0a3bdb0 | 2013-08-03 10:35:28 +0800 | [diff] [blame] | 1316 | return rc; |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 1317 | |
| 1318 | ptlrpc_nrs_req_add(svcpt, req, !!rc); |
| 1319 | |
Greg Kroah-Hartman | 0a3bdb0 | 2013-08-03 10:35:28 +0800 | [diff] [blame] | 1320 | return 0; |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 1321 | } |
| 1322 | |
| 1323 | /** |
| 1324 | * Allow to handle high priority request |
| 1325 | * User can call it w/o any lock but need to hold |
| 1326 | * ptlrpc_service_part::scp_req_lock to get reliable result |
| 1327 | */ |
| 1328 | static bool ptlrpc_server_allow_high(struct ptlrpc_service_part *svcpt, |
| 1329 | bool force) |
| 1330 | { |
| 1331 | int running = svcpt->scp_nthrs_running; |
| 1332 | |
| 1333 | if (!nrs_svcpt_has_hp(svcpt)) |
| 1334 | return false; |
| 1335 | |
| 1336 | if (force) |
| 1337 | return true; |
| 1338 | |
| 1339 | if (unlikely(svcpt->scp_service->srv_req_portal == MDS_REQUEST_PORTAL && |
| 1340 | CFS_FAIL_PRECHECK(OBD_FAIL_PTLRPC_CANCEL_RESEND))) { |
| 1341 | /* leave just 1 thread for normal RPCs */ |
| 1342 | running = PTLRPC_NTHRS_INIT; |
Oleg Drokin | 8b38208 | 2016-02-16 00:46:58 -0500 | [diff] [blame] | 1343 | if (svcpt->scp_service->srv_ops.so_hpreq_handler) |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 1344 | running += 1; |
| 1345 | } |
| 1346 | |
| 1347 | if (svcpt->scp_nreqs_active >= running - 1) |
| 1348 | return false; |
| 1349 | |
| 1350 | if (svcpt->scp_nhreqs_active == 0) |
| 1351 | return true; |
| 1352 | |
| 1353 | return !ptlrpc_nrs_req_pending_nolock(svcpt, false) || |
| 1354 | svcpt->scp_hreq_count < svcpt->scp_service->srv_hpreq_ratio; |
| 1355 | } |
| 1356 | |
| 1357 | static bool ptlrpc_server_high_pending(struct ptlrpc_service_part *svcpt, |
| 1358 | bool force) |
| 1359 | { |
| 1360 | return ptlrpc_server_allow_high(svcpt, force) && |
| 1361 | ptlrpc_nrs_req_pending_nolock(svcpt, true); |
| 1362 | } |
| 1363 | |
| 1364 | /** |
| 1365 | * Only allow normal priority requests on a service that has a high-priority |
| 1366 | * queue if forced (i.e. cleanup), if there are other high priority requests |
| 1367 | * already being processed (i.e. those threads can service more high-priority |
| 1368 | * requests), or if there are enough idle threads that a later thread can do |
| 1369 | * a high priority request. |
| 1370 | * User can call it w/o any lock but need to hold |
| 1371 | * ptlrpc_service_part::scp_req_lock to get reliable result |
| 1372 | */ |
| 1373 | static bool ptlrpc_server_allow_normal(struct ptlrpc_service_part *svcpt, |
| 1374 | bool force) |
| 1375 | { |
| 1376 | int running = svcpt->scp_nthrs_running; |
Mike Rapoport | 50ffcb7 | 2015-10-13 16:03:40 +0300 | [diff] [blame] | 1377 | |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 1378 | if (unlikely(svcpt->scp_service->srv_req_portal == MDS_REQUEST_PORTAL && |
| 1379 | CFS_FAIL_PRECHECK(OBD_FAIL_PTLRPC_CANCEL_RESEND))) { |
| 1380 | /* leave just 1 thread for normal RPCs */ |
| 1381 | running = PTLRPC_NTHRS_INIT; |
Oleg Drokin | 8b38208 | 2016-02-16 00:46:58 -0500 | [diff] [blame] | 1382 | if (svcpt->scp_service->srv_ops.so_hpreq_handler) |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 1383 | running += 1; |
| 1384 | } |
| 1385 | |
| 1386 | if (force || |
| 1387 | svcpt->scp_nreqs_active < running - 2) |
| 1388 | return true; |
| 1389 | |
| 1390 | if (svcpt->scp_nreqs_active >= running - 1) |
| 1391 | return false; |
| 1392 | |
| 1393 | return svcpt->scp_nhreqs_active > 0 || !nrs_svcpt_has_hp(svcpt); |
| 1394 | } |
| 1395 | |
| 1396 | static bool ptlrpc_server_normal_pending(struct ptlrpc_service_part *svcpt, |
| 1397 | bool force) |
| 1398 | { |
| 1399 | return ptlrpc_server_allow_normal(svcpt, force) && |
| 1400 | ptlrpc_nrs_req_pending_nolock(svcpt, false); |
| 1401 | } |
| 1402 | |
| 1403 | /** |
| 1404 | * Returns true if there are requests available in incoming |
| 1405 | * request queue for processing and it is allowed to fetch them. |
| 1406 | * User can call it w/o any lock but need to hold ptlrpc_service::scp_req_lock |
| 1407 | * to get reliable result |
| 1408 | * \see ptlrpc_server_allow_normal |
| 1409 | * \see ptlrpc_server_allow high |
| 1410 | */ |
| 1411 | static inline bool |
| 1412 | ptlrpc_server_request_pending(struct ptlrpc_service_part *svcpt, bool force) |
| 1413 | { |
| 1414 | return ptlrpc_server_high_pending(svcpt, force) || |
| 1415 | ptlrpc_server_normal_pending(svcpt, force); |
| 1416 | } |
| 1417 | |
| 1418 | /** |
| 1419 | * Fetch a request for processing from queue of unprocessed requests. |
| 1420 | * Favors high-priority requests. |
| 1421 | * Returns a pointer to fetched request. |
| 1422 | */ |
| 1423 | static struct ptlrpc_request * |
| 1424 | ptlrpc_server_request_get(struct ptlrpc_service_part *svcpt, bool force) |
| 1425 | { |
| 1426 | struct ptlrpc_request *req = NULL; |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 1427 | |
| 1428 | spin_lock(&svcpt->scp_req_lock); |
| 1429 | |
| 1430 | if (ptlrpc_server_high_pending(svcpt, force)) { |
| 1431 | req = ptlrpc_nrs_req_get_nolock(svcpt, true, force); |
Oleg Drokin | 8b38208 | 2016-02-16 00:46:58 -0500 | [diff] [blame] | 1432 | if (req) { |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 1433 | svcpt->scp_hreq_count++; |
| 1434 | goto got_request; |
| 1435 | } |
| 1436 | } |
| 1437 | |
| 1438 | if (ptlrpc_server_normal_pending(svcpt, force)) { |
| 1439 | req = ptlrpc_nrs_req_get_nolock(svcpt, false, force); |
Oleg Drokin | 8b38208 | 2016-02-16 00:46:58 -0500 | [diff] [blame] | 1440 | if (req) { |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 1441 | svcpt->scp_hreq_count = 0; |
| 1442 | goto got_request; |
| 1443 | } |
| 1444 | } |
| 1445 | |
| 1446 | spin_unlock(&svcpt->scp_req_lock); |
Greg Kroah-Hartman | 0a3bdb0 | 2013-08-03 10:35:28 +0800 | [diff] [blame] | 1447 | return NULL; |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 1448 | |
| 1449 | got_request: |
| 1450 | svcpt->scp_nreqs_active++; |
| 1451 | if (req->rq_hp) |
| 1452 | svcpt->scp_nhreqs_active++; |
| 1453 | |
| 1454 | spin_unlock(&svcpt->scp_req_lock); |
| 1455 | |
| 1456 | if (likely(req->rq_export)) |
| 1457 | class_export_rpc_inc(req->rq_export); |
| 1458 | |
Greg Kroah-Hartman | 0a3bdb0 | 2013-08-03 10:35:28 +0800 | [diff] [blame] | 1459 | return req; |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 1460 | } |
| 1461 | |
| 1462 | /** |
| 1463 | * Handle freshly incoming reqs, add to timed early reply list, |
| 1464 | * pass on to regular request queue. |
| 1465 | * All incoming requests pass through here before getting into |
| 1466 | * ptlrpc_server_handle_req later on. |
| 1467 | */ |
| 1468 | static int |
| 1469 | ptlrpc_server_handle_req_in(struct ptlrpc_service_part *svcpt, |
| 1470 | struct ptlrpc_thread *thread) |
| 1471 | { |
Chris Hanna | d0bfef3 | 2015-06-03 10:28:26 -0400 | [diff] [blame] | 1472 | struct ptlrpc_service *svc = svcpt->scp_service; |
| 1473 | struct ptlrpc_request *req; |
| 1474 | __u32 deadline; |
| 1475 | int rc; |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 1476 | |
| 1477 | spin_lock(&svcpt->scp_lock); |
| 1478 | if (list_empty(&svcpt->scp_req_incoming)) { |
| 1479 | spin_unlock(&svcpt->scp_lock); |
Greg Kroah-Hartman | 0a3bdb0 | 2013-08-03 10:35:28 +0800 | [diff] [blame] | 1480 | return 0; |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 1481 | } |
| 1482 | |
| 1483 | req = list_entry(svcpt->scp_req_incoming.next, |
Oleg Drokin | 30c0aa3 | 2016-02-26 01:50:02 -0500 | [diff] [blame] | 1484 | struct ptlrpc_request, rq_list); |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 1485 | list_del_init(&req->rq_list); |
| 1486 | svcpt->scp_nreqs_incoming--; |
| 1487 | /* Consider this still a "queued" request as far as stats are |
Oleg Drokin | dadfcda | 2016-02-24 22:00:38 -0500 | [diff] [blame] | 1488 | * concerned |
| 1489 | */ |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 1490 | spin_unlock(&svcpt->scp_lock); |
| 1491 | |
| 1492 | /* go through security check/transform */ |
| 1493 | rc = sptlrpc_svc_unwrap_request(req); |
| 1494 | switch (rc) { |
| 1495 | case SECSVC_OK: |
| 1496 | break; |
| 1497 | case SECSVC_COMPLETE: |
| 1498 | target_send_reply(req, 0, OBD_FAIL_MDS_ALL_REPLY_NET); |
| 1499 | goto err_req; |
| 1500 | case SECSVC_DROP: |
| 1501 | goto err_req; |
| 1502 | default: |
| 1503 | LBUG(); |
| 1504 | } |
| 1505 | |
| 1506 | /* |
| 1507 | * for null-flavored rpc, msg has been unpacked by sptlrpc, although |
| 1508 | * redo it wouldn't be harmful. |
| 1509 | */ |
| 1510 | if (SPTLRPC_FLVR_POLICY(req->rq_flvr.sf_rpc) != SPTLRPC_POLICY_NULL) { |
| 1511 | rc = ptlrpc_unpack_req_msg(req, req->rq_reqlen); |
| 1512 | if (rc != 0) { |
Greg Kroah-Hartman | b0f5aad | 2014-07-12 20:06:04 -0700 | [diff] [blame] | 1513 | CERROR("error unpacking request: ptl %d from %s x%llu\n", |
| 1514 | svc->srv_req_portal, libcfs_id2str(req->rq_peer), |
| 1515 | req->rq_xid); |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 1516 | goto err_req; |
| 1517 | } |
| 1518 | } |
| 1519 | |
| 1520 | rc = lustre_unpack_req_ptlrpc_body(req, MSG_PTLRPC_BODY_OFF); |
| 1521 | if (rc) { |
Greg Kroah-Hartman | b0f5aad | 2014-07-12 20:06:04 -0700 | [diff] [blame] | 1522 | CERROR("error unpacking ptlrpc body: ptl %d from %s x%llu\n", |
| 1523 | svc->srv_req_portal, libcfs_id2str(req->rq_peer), |
| 1524 | req->rq_xid); |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 1525 | goto err_req; |
| 1526 | } |
| 1527 | |
| 1528 | if (OBD_FAIL_CHECK(OBD_FAIL_PTLRPC_DROP_REQ_OPC) && |
| 1529 | lustre_msg_get_opc(req->rq_reqmsg) == cfs_fail_val) { |
Greg Kroah-Hartman | b0f5aad | 2014-07-12 20:06:04 -0700 | [diff] [blame] | 1530 | CERROR("drop incoming rpc opc %u, x%llu\n", |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 1531 | cfs_fail_val, req->rq_xid); |
| 1532 | goto err_req; |
| 1533 | } |
| 1534 | |
| 1535 | rc = -EINVAL; |
| 1536 | if (lustre_msg_get_type(req->rq_reqmsg) != PTL_RPC_MSG_REQUEST) { |
| 1537 | CERROR("wrong packet type received (type=%u) from %s\n", |
| 1538 | lustre_msg_get_type(req->rq_reqmsg), |
| 1539 | libcfs_id2str(req->rq_peer)); |
| 1540 | goto err_req; |
| 1541 | } |
| 1542 | |
Kristina Martsenko | 3949015e | 2013-11-11 21:34:58 +0200 | [diff] [blame] | 1543 | switch (lustre_msg_get_opc(req->rq_reqmsg)) { |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 1544 | case MDS_WRITEPAGE: |
| 1545 | case OST_WRITE: |
| 1546 | req->rq_bulk_write = 1; |
| 1547 | break; |
| 1548 | case MDS_READPAGE: |
| 1549 | case OST_READ: |
| 1550 | case MGS_CONFIG_READ: |
| 1551 | req->rq_bulk_read = 1; |
| 1552 | break; |
| 1553 | } |
| 1554 | |
Greg Kroah-Hartman | b0f5aad | 2014-07-12 20:06:04 -0700 | [diff] [blame] | 1555 | CDEBUG(D_RPCTRACE, "got req x%llu\n", req->rq_xid); |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 1556 | |
| 1557 | req->rq_export = class_conn2export( |
| 1558 | lustre_msg_get_handle(req->rq_reqmsg)); |
| 1559 | if (req->rq_export) { |
| 1560 | rc = ptlrpc_check_req(req); |
| 1561 | if (rc == 0) { |
| 1562 | rc = sptlrpc_target_export_check(req->rq_export, req); |
| 1563 | if (rc) |
Joe Perches | 2d00bd1 | 2014-11-23 11:28:50 -0800 | [diff] [blame] | 1564 | DEBUG_REQ(D_ERROR, req, "DROPPING req with illegal security flavor,"); |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 1565 | } |
| 1566 | |
| 1567 | if (rc) |
| 1568 | goto err_req; |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 1569 | } |
| 1570 | |
| 1571 | /* req_in handling should/must be fast */ |
Arnd Bergmann | 219e6de | 2015-09-27 16:45:30 -0400 | [diff] [blame] | 1572 | if (ktime_get_real_seconds() - req->rq_arrival_time.tv_sec > 5) |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 1573 | DEBUG_REQ(D_WARNING, req, "Slow req_in handling "CFS_DURATION_T"s", |
Arnd Bergmann | 219e6de | 2015-09-27 16:45:30 -0400 | [diff] [blame] | 1574 | (long)(ktime_get_real_seconds() - |
| 1575 | req->rq_arrival_time.tv_sec)); |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 1576 | |
| 1577 | /* Set rpc server deadline and add it to the timed list */ |
| 1578 | deadline = (lustre_msghdr_get_flags(req->rq_reqmsg) & |
| 1579 | MSGHDR_AT_SUPPORT) ? |
| 1580 | /* The max time the client expects us to take */ |
| 1581 | lustre_msg_get_timeout(req->rq_reqmsg) : obd_timeout; |
| 1582 | req->rq_deadline = req->rq_arrival_time.tv_sec + deadline; |
| 1583 | if (unlikely(deadline == 0)) { |
| 1584 | DEBUG_REQ(D_ERROR, req, "Dropping request with 0 timeout"); |
| 1585 | goto err_req; |
| 1586 | } |
| 1587 | |
| 1588 | req->rq_svc_thread = thread; |
Mikhail Pershin | 7018750 | 2016-04-27 21:37:20 -0400 | [diff] [blame] | 1589 | if (thread) { |
| 1590 | /* initialize request session, it is needed for request |
| 1591 | * processing by target |
| 1592 | */ |
| 1593 | rc = lu_context_init(&req->rq_session, |
| 1594 | LCT_SERVER_SESSION | LCT_NOREF); |
| 1595 | if (rc) { |
| 1596 | CERROR("%s: failure to initialize session: rc = %d\n", |
| 1597 | thread->t_name, rc); |
| 1598 | goto err_req; |
| 1599 | } |
| 1600 | req->rq_session.lc_thread = thread; |
| 1601 | lu_context_enter(&req->rq_session); |
| 1602 | req->rq_svc_thread->t_env->le_ses = &req->rq_session; |
| 1603 | } |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 1604 | |
| 1605 | ptlrpc_at_add_timed(req); |
| 1606 | |
| 1607 | /* Move it over to the request processing queue */ |
| 1608 | rc = ptlrpc_server_request_add(svcpt, req); |
| 1609 | if (rc) |
Julia Lawall | a9b3e8f | 2014-09-07 18:18:29 +0200 | [diff] [blame] | 1610 | goto err_req; |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 1611 | |
| 1612 | wake_up(&svcpt->scp_waitq); |
Greg Kroah-Hartman | 0a3bdb0 | 2013-08-03 10:35:28 +0800 | [diff] [blame] | 1613 | return 1; |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 1614 | |
| 1615 | err_req: |
| 1616 | ptlrpc_server_finish_request(svcpt, req); |
| 1617 | |
Greg Kroah-Hartman | 0a3bdb0 | 2013-08-03 10:35:28 +0800 | [diff] [blame] | 1618 | return 1; |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 1619 | } |
| 1620 | |
| 1621 | /** |
| 1622 | * Main incoming request handling logic. |
| 1623 | * Calls handler function from service to do actual processing. |
| 1624 | */ |
| 1625 | static int |
| 1626 | ptlrpc_server_handle_request(struct ptlrpc_service_part *svcpt, |
| 1627 | struct ptlrpc_thread *thread) |
| 1628 | { |
| 1629 | struct ptlrpc_service *svc = svcpt->scp_service; |
| 1630 | struct ptlrpc_request *request; |
Arnd Bergmann | 219e6de | 2015-09-27 16:45:30 -0400 | [diff] [blame] | 1631 | struct timespec64 work_start; |
| 1632 | struct timespec64 work_end; |
| 1633 | struct timespec64 timediff; |
| 1634 | struct timespec64 arrived; |
| 1635 | unsigned long timediff_usecs; |
| 1636 | unsigned long arrived_usecs; |
Chris Hanna | d0bfef3 | 2015-06-03 10:28:26 -0400 | [diff] [blame] | 1637 | int fail_opc = 0; |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 1638 | |
| 1639 | request = ptlrpc_server_request_get(svcpt, false); |
Oleg Drokin | 8b38208 | 2016-02-16 00:46:58 -0500 | [diff] [blame] | 1640 | if (!request) |
Greg Kroah-Hartman | 0a3bdb0 | 2013-08-03 10:35:28 +0800 | [diff] [blame] | 1641 | return 0; |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 1642 | |
| 1643 | if (OBD_FAIL_CHECK(OBD_FAIL_PTLRPC_HPREQ_NOTIMEOUT)) |
| 1644 | fail_opc = OBD_FAIL_PTLRPC_HPREQ_NOTIMEOUT; |
| 1645 | else if (OBD_FAIL_CHECK(OBD_FAIL_PTLRPC_HPREQ_TIMEOUT)) |
| 1646 | fail_opc = OBD_FAIL_PTLRPC_HPREQ_TIMEOUT; |
| 1647 | |
| 1648 | if (unlikely(fail_opc)) { |
| 1649 | if (request->rq_export && request->rq_ops) |
| 1650 | OBD_FAIL_TIMEOUT(fail_opc, 4); |
| 1651 | } |
| 1652 | |
| 1653 | ptlrpc_rqphase_move(request, RQ_PHASE_INTERPRET); |
| 1654 | |
Kristina Martsenko | 3949015e | 2013-11-11 21:34:58 +0200 | [diff] [blame] | 1655 | if (OBD_FAIL_CHECK(OBD_FAIL_PTLRPC_DUMP_LOG)) |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 1656 | libcfs_debug_dumplog(); |
| 1657 | |
Arnd Bergmann | 219e6de | 2015-09-27 16:45:30 -0400 | [diff] [blame] | 1658 | ktime_get_real_ts64(&work_start); |
| 1659 | timediff = timespec64_sub(work_start, request->rq_arrival_time); |
| 1660 | timediff_usecs = timediff.tv_sec * USEC_PER_SEC + |
| 1661 | timediff.tv_nsec / NSEC_PER_USEC; |
Oleg Drokin | 8b38208 | 2016-02-16 00:46:58 -0500 | [diff] [blame] | 1662 | if (likely(svc->srv_stats)) { |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 1663 | lprocfs_counter_add(svc->srv_stats, PTLRPC_REQWAIT_CNTR, |
Arnd Bergmann | 219e6de | 2015-09-27 16:45:30 -0400 | [diff] [blame] | 1664 | timediff_usecs); |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 1665 | lprocfs_counter_add(svc->srv_stats, PTLRPC_REQQDEPTH_CNTR, |
| 1666 | svcpt->scp_nreqs_incoming); |
| 1667 | lprocfs_counter_add(svc->srv_stats, PTLRPC_REQACTIVE_CNTR, |
| 1668 | svcpt->scp_nreqs_active); |
| 1669 | lprocfs_counter_add(svc->srv_stats, PTLRPC_TIMEOUT, |
| 1670 | at_get(&svcpt->scp_at_estimate)); |
| 1671 | } |
| 1672 | |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 1673 | if (likely(request->rq_export)) { |
| 1674 | if (unlikely(ptlrpc_check_req(request))) |
| 1675 | goto put_conn; |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 1676 | } |
| 1677 | |
| 1678 | /* Discard requests queued for longer than the deadline. |
Oleg Drokin | dadfcda | 2016-02-24 22:00:38 -0500 | [diff] [blame] | 1679 | * The deadline is increased if we send an early reply. |
| 1680 | */ |
Arnd Bergmann | 219e6de | 2015-09-27 16:45:30 -0400 | [diff] [blame] | 1681 | if (ktime_get_real_seconds() > request->rq_deadline) { |
Joe Perches | 2d00bd1 | 2014-11-23 11:28:50 -0800 | [diff] [blame] | 1682 | DEBUG_REQ(D_ERROR, request, "Dropping timed-out request from %s: deadline " CFS_DURATION_T ":" CFS_DURATION_T "s ago\n", |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 1683 | libcfs_id2str(request->rq_peer), |
Arnd Bergmann | 219e6de | 2015-09-27 16:45:30 -0400 | [diff] [blame] | 1684 | (long)(request->rq_deadline - |
| 1685 | request->rq_arrival_time.tv_sec), |
| 1686 | (long)(ktime_get_real_seconds() - |
| 1687 | request->rq_deadline)); |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 1688 | goto put_conn; |
| 1689 | } |
| 1690 | |
Joe Perches | 2d00bd1 | 2014-11-23 11:28:50 -0800 | [diff] [blame] | 1691 | CDEBUG(D_RPCTRACE, "Handling RPC pname:cluuid+ref:pid:xid:nid:opc %s:%s+%d:%d:x%llu:%s:%d\n", |
| 1692 | current_comm(), |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 1693 | (request->rq_export ? |
| 1694 | (char *)request->rq_export->exp_client_uuid.uuid : "0"), |
| 1695 | (request->rq_export ? |
| 1696 | atomic_read(&request->rq_export->exp_refcount) : -99), |
| 1697 | lustre_msg_get_status(request->rq_reqmsg), request->rq_xid, |
| 1698 | libcfs_id2str(request->rq_peer), |
| 1699 | lustre_msg_get_opc(request->rq_reqmsg)); |
| 1700 | |
| 1701 | if (lustre_msg_get_opc(request->rq_reqmsg) != OBD_PING) |
| 1702 | CFS_FAIL_TIMEOUT_MS(OBD_FAIL_PTLRPC_PAUSE_REQ, cfs_fail_val); |
| 1703 | |
Mikhail Pershin | 7018750 | 2016-04-27 21:37:20 -0400 | [diff] [blame] | 1704 | CDEBUG(D_NET, "got req %llu\n", request->rq_xid); |
| 1705 | |
| 1706 | /* re-assign request and sesson thread to the current one */ |
| 1707 | request->rq_svc_thread = thread; |
| 1708 | if (thread) { |
| 1709 | LASSERT(request->rq_session.lc_thread); |
| 1710 | request->rq_session.lc_thread = thread; |
| 1711 | request->rq_session.lc_cookie = 0x55; |
| 1712 | thread->t_env->le_ses = &request->rq_session; |
| 1713 | } |
| 1714 | svc->srv_ops.so_req_handler(request); |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 1715 | |
| 1716 | ptlrpc_rqphase_move(request, RQ_PHASE_COMPLETE); |
| 1717 | |
| 1718 | put_conn: |
Arnd Bergmann | 219e6de | 2015-09-27 16:45:30 -0400 | [diff] [blame] | 1719 | if (unlikely(ktime_get_real_seconds() > request->rq_deadline)) { |
Kristina Martsenko | 532118c | 2013-11-11 21:35:03 +0200 | [diff] [blame] | 1720 | DEBUG_REQ(D_WARNING, request, |
Arnd Bergmann | 219e6de | 2015-09-27 16:45:30 -0400 | [diff] [blame] | 1721 | "Request took longer than estimated (%lld:%llds); " |
| 1722 | "client may timeout.", |
| 1723 | (s64)request->rq_deadline - |
| 1724 | request->rq_arrival_time.tv_sec, |
| 1725 | (s64)ktime_get_real_seconds() - request->rq_deadline); |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 1726 | } |
| 1727 | |
Arnd Bergmann | 219e6de | 2015-09-27 16:45:30 -0400 | [diff] [blame] | 1728 | ktime_get_real_ts64(&work_end); |
| 1729 | timediff = timespec64_sub(work_end, work_start); |
| 1730 | timediff_usecs = timediff.tv_sec * USEC_PER_SEC + |
| 1731 | timediff.tv_nsec / NSEC_PER_USEC; |
| 1732 | arrived = timespec64_sub(work_end, request->rq_arrival_time); |
| 1733 | arrived_usecs = arrived.tv_sec * USEC_PER_SEC + |
| 1734 | arrived.tv_nsec / NSEC_PER_USEC; |
Joe Perches | 2d00bd1 | 2014-11-23 11:28:50 -0800 | [diff] [blame] | 1735 | CDEBUG(D_RPCTRACE, "Handled RPC pname:cluuid+ref:pid:xid:nid:opc %s:%s+%d:%d:x%llu:%s:%d Request processed in %ldus (%ldus total) trans %llu rc %d/%d\n", |
| 1736 | current_comm(), |
| 1737 | (request->rq_export ? |
| 1738 | (char *)request->rq_export->exp_client_uuid.uuid : "0"), |
| 1739 | (request->rq_export ? |
| 1740 | atomic_read(&request->rq_export->exp_refcount) : -99), |
| 1741 | lustre_msg_get_status(request->rq_reqmsg), |
| 1742 | request->rq_xid, |
| 1743 | libcfs_id2str(request->rq_peer), |
| 1744 | lustre_msg_get_opc(request->rq_reqmsg), |
Arnd Bergmann | 219e6de | 2015-09-27 16:45:30 -0400 | [diff] [blame] | 1745 | timediff_usecs, |
| 1746 | arrived_usecs, |
Joe Perches | 2d00bd1 | 2014-11-23 11:28:50 -0800 | [diff] [blame] | 1747 | (request->rq_repmsg ? |
| 1748 | lustre_msg_get_transno(request->rq_repmsg) : |
| 1749 | request->rq_transno), |
| 1750 | request->rq_status, |
| 1751 | (request->rq_repmsg ? |
| 1752 | lustre_msg_get_status(request->rq_repmsg) : -999)); |
Oleg Drokin | 8b38208 | 2016-02-16 00:46:58 -0500 | [diff] [blame] | 1753 | if (likely(svc->srv_stats && request->rq_reqmsg)) { |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 1754 | __u32 op = lustre_msg_get_opc(request->rq_reqmsg); |
| 1755 | int opc = opcode_offset(op); |
Mike Rapoport | 50ffcb7 | 2015-10-13 16:03:40 +0300 | [diff] [blame] | 1756 | |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 1757 | if (opc > 0 && !(op == LDLM_ENQUEUE || op == MDS_REINT)) { |
| 1758 | LASSERT(opc < LUSTRE_MAX_OPCODES); |
| 1759 | lprocfs_counter_add(svc->srv_stats, |
| 1760 | opc + EXTRA_MAX_OPCODES, |
Arnd Bergmann | 219e6de | 2015-09-27 16:45:30 -0400 | [diff] [blame] | 1761 | timediff_usecs); |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 1762 | } |
| 1763 | } |
| 1764 | if (unlikely(request->rq_early_count)) { |
| 1765 | DEBUG_REQ(D_ADAPTTO, request, |
Arnd Bergmann | 219e6de | 2015-09-27 16:45:30 -0400 | [diff] [blame] | 1766 | "sent %d early replies before finishing in %llds", |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 1767 | request->rq_early_count, |
Arnd Bergmann | 219e6de | 2015-09-27 16:45:30 -0400 | [diff] [blame] | 1768 | (s64)work_end.tv_sec - |
| 1769 | request->rq_arrival_time.tv_sec); |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 1770 | } |
| 1771 | |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 1772 | ptlrpc_server_finish_active_request(svcpt, request); |
| 1773 | |
Greg Kroah-Hartman | 0a3bdb0 | 2013-08-03 10:35:28 +0800 | [diff] [blame] | 1774 | return 1; |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 1775 | } |
| 1776 | |
| 1777 | /** |
| 1778 | * An internal function to process a single reply state object. |
| 1779 | */ |
| 1780 | static int |
| 1781 | ptlrpc_handle_rs(struct ptlrpc_reply_state *rs) |
| 1782 | { |
| 1783 | struct ptlrpc_service_part *svcpt = rs->rs_svcpt; |
Chris Hanna | d0bfef3 | 2015-06-03 10:28:26 -0400 | [diff] [blame] | 1784 | struct ptlrpc_service *svc = svcpt->scp_service; |
| 1785 | struct obd_export *exp; |
| 1786 | int nlocks; |
| 1787 | int been_handled; |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 1788 | |
| 1789 | exp = rs->rs_export; |
| 1790 | |
Kristina Martsenko | 3949015e | 2013-11-11 21:34:58 +0200 | [diff] [blame] | 1791 | LASSERT(rs->rs_difficult); |
| 1792 | LASSERT(rs->rs_scheduled); |
| 1793 | LASSERT(list_empty(&rs->rs_list)); |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 1794 | |
| 1795 | spin_lock(&exp->exp_lock); |
| 1796 | /* Noop if removed already */ |
Kristina Martsenko | 3949015e | 2013-11-11 21:34:58 +0200 | [diff] [blame] | 1797 | list_del_init(&rs->rs_exp_list); |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 1798 | spin_unlock(&exp->exp_lock); |
| 1799 | |
| 1800 | /* The disk commit callback holds exp_uncommitted_replies_lock while it |
| 1801 | * iterates over newly committed replies, removing them from |
| 1802 | * exp_uncommitted_replies. It then drops this lock and schedules the |
| 1803 | * replies it found for handling here. |
| 1804 | * |
| 1805 | * We can avoid contention for exp_uncommitted_replies_lock between the |
| 1806 | * HRT threads and further commit callbacks by checking rs_committed |
| 1807 | * which is set in the commit callback while it holds both |
| 1808 | * rs_lock and exp_uncommitted_reples. |
| 1809 | * |
| 1810 | * If we see rs_committed clear, the commit callback _may_ not have |
| 1811 | * handled this reply yet and we race with it to grab |
| 1812 | * exp_uncommitted_replies_lock before removing the reply from |
| 1813 | * exp_uncommitted_replies. Note that if we lose the race and the |
| 1814 | * reply has already been removed, list_del_init() is a noop. |
| 1815 | * |
| 1816 | * If we see rs_committed set, we know the commit callback is handling, |
| 1817 | * or has handled this reply since store reordering might allow us to |
| 1818 | * see rs_committed set out of sequence. But since this is done |
| 1819 | * holding rs_lock, we can be sure it has all completed once we hold |
| 1820 | * rs_lock, which we do right next. |
| 1821 | */ |
| 1822 | if (!rs->rs_committed) { |
| 1823 | spin_lock(&exp->exp_uncommitted_replies_lock); |
| 1824 | list_del_init(&rs->rs_obd_list); |
| 1825 | spin_unlock(&exp->exp_uncommitted_replies_lock); |
| 1826 | } |
| 1827 | |
| 1828 | spin_lock(&rs->rs_lock); |
| 1829 | |
| 1830 | been_handled = rs->rs_handled; |
| 1831 | rs->rs_handled = 1; |
| 1832 | |
| 1833 | nlocks = rs->rs_nlocks; /* atomic "steal", but */ |
| 1834 | rs->rs_nlocks = 0; /* locks still on rs_locks! */ |
| 1835 | |
| 1836 | if (nlocks == 0 && !been_handled) { |
| 1837 | /* If we see this, we should already have seen the warning |
Oleg Drokin | dadfcda | 2016-02-24 22:00:38 -0500 | [diff] [blame] | 1838 | * in mds_steal_ack_locks() |
| 1839 | */ |
Greg Kroah-Hartman | f537dd2 | 2014-07-12 18:41:09 -0700 | [diff] [blame] | 1840 | CDEBUG(D_HA, "All locks stolen from rs %p x%lld.t%lld o%d NID %s\n", |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 1841 | rs, |
| 1842 | rs->rs_xid, rs->rs_transno, rs->rs_opc, |
| 1843 | libcfs_nid2str(exp->exp_connection->c_peer.nid)); |
| 1844 | } |
| 1845 | |
| 1846 | if ((!been_handled && rs->rs_on_net) || nlocks > 0) { |
| 1847 | spin_unlock(&rs->rs_lock); |
| 1848 | |
| 1849 | if (!been_handled && rs->rs_on_net) { |
| 1850 | LNetMDUnlink(rs->rs_md_h); |
| 1851 | /* Ignore return code; we're racing with completion */ |
| 1852 | } |
| 1853 | |
| 1854 | while (nlocks-- > 0) |
| 1855 | ldlm_lock_decref(&rs->rs_locks[nlocks], |
| 1856 | rs->rs_modes[nlocks]); |
| 1857 | |
| 1858 | spin_lock(&rs->rs_lock); |
| 1859 | } |
| 1860 | |
| 1861 | rs->rs_scheduled = 0; |
| 1862 | |
| 1863 | if (!rs->rs_on_net) { |
| 1864 | /* Off the net */ |
| 1865 | spin_unlock(&rs->rs_lock); |
| 1866 | |
Kristina Martsenko | 3949015e | 2013-11-11 21:34:58 +0200 | [diff] [blame] | 1867 | class_export_put(exp); |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 1868 | rs->rs_export = NULL; |
Kristina Martsenko | 3949015e | 2013-11-11 21:34:58 +0200 | [diff] [blame] | 1869 | ptlrpc_rs_decref(rs); |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 1870 | if (atomic_dec_and_test(&svcpt->scp_nreps_difficult) && |
| 1871 | svc->srv_is_stopping) |
| 1872 | wake_up_all(&svcpt->scp_waitq); |
Greg Kroah-Hartman | 0a3bdb0 | 2013-08-03 10:35:28 +0800 | [diff] [blame] | 1873 | return 1; |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 1874 | } |
| 1875 | |
| 1876 | /* still on the net; callback will schedule */ |
| 1877 | spin_unlock(&rs->rs_lock); |
Greg Kroah-Hartman | 0a3bdb0 | 2013-08-03 10:35:28 +0800 | [diff] [blame] | 1878 | return 1; |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 1879 | } |
| 1880 | |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 1881 | static void |
| 1882 | ptlrpc_check_rqbd_pool(struct ptlrpc_service_part *svcpt) |
| 1883 | { |
| 1884 | int avail = svcpt->scp_nrqbds_posted; |
| 1885 | int low_water = test_req_buffer_pressure ? 0 : |
| 1886 | svcpt->scp_service->srv_nbuf_per_group / 2; |
| 1887 | |
| 1888 | /* NB I'm not locking; just looking. */ |
| 1889 | |
| 1890 | /* CAVEAT EMPTOR: We might be allocating buffers here because we've |
| 1891 | * allowed the request history to grow out of control. We could put a |
| 1892 | * sanity check on that here and cull some history if we need the |
Oleg Drokin | dadfcda | 2016-02-24 22:00:38 -0500 | [diff] [blame] | 1893 | * space. |
| 1894 | */ |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 1895 | |
| 1896 | if (avail <= low_water) |
| 1897 | ptlrpc_grow_req_bufs(svcpt, 1); |
| 1898 | |
| 1899 | if (svcpt->scp_service->srv_stats) { |
| 1900 | lprocfs_counter_add(svcpt->scp_service->srv_stats, |
| 1901 | PTLRPC_REQBUF_AVAIL_CNTR, avail); |
| 1902 | } |
| 1903 | } |
| 1904 | |
| 1905 | static int |
| 1906 | ptlrpc_retry_rqbds(void *arg) |
| 1907 | { |
Amitoj Kaur Chawla | 864ef62 | 2015-10-15 00:13:30 +0530 | [diff] [blame] | 1908 | struct ptlrpc_service_part *svcpt = arg; |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 1909 | |
| 1910 | svcpt->scp_rqbd_timeout = 0; |
| 1911 | return -ETIMEDOUT; |
| 1912 | } |
| 1913 | |
| 1914 | static inline int |
| 1915 | ptlrpc_threads_enough(struct ptlrpc_service_part *svcpt) |
| 1916 | { |
| 1917 | return svcpt->scp_nreqs_active < |
| 1918 | svcpt->scp_nthrs_running - 1 - |
| 1919 | (svcpt->scp_service->srv_ops.so_hpreq_handler != NULL); |
| 1920 | } |
| 1921 | |
| 1922 | /** |
| 1923 | * allowed to create more threads |
| 1924 | * user can call it w/o any lock but need to hold |
| 1925 | * ptlrpc_service_part::scp_lock to get reliable result |
| 1926 | */ |
| 1927 | static inline int |
| 1928 | ptlrpc_threads_increasable(struct ptlrpc_service_part *svcpt) |
| 1929 | { |
| 1930 | return svcpt->scp_nthrs_running + |
| 1931 | svcpt->scp_nthrs_starting < |
| 1932 | svcpt->scp_service->srv_nthrs_cpt_limit; |
| 1933 | } |
| 1934 | |
| 1935 | /** |
| 1936 | * too many requests and allowed to create more threads |
| 1937 | */ |
| 1938 | static inline int |
| 1939 | ptlrpc_threads_need_create(struct ptlrpc_service_part *svcpt) |
| 1940 | { |
| 1941 | return !ptlrpc_threads_enough(svcpt) && |
| 1942 | ptlrpc_threads_increasable(svcpt); |
| 1943 | } |
| 1944 | |
| 1945 | static inline int |
| 1946 | ptlrpc_thread_stopping(struct ptlrpc_thread *thread) |
| 1947 | { |
| 1948 | return thread_is_stopping(thread) || |
| 1949 | thread->t_svcpt->scp_service->srv_is_stopping; |
| 1950 | } |
| 1951 | |
| 1952 | static inline int |
| 1953 | ptlrpc_rqbd_pending(struct ptlrpc_service_part *svcpt) |
| 1954 | { |
| 1955 | return !list_empty(&svcpt->scp_rqbd_idle) && |
| 1956 | svcpt->scp_rqbd_timeout == 0; |
| 1957 | } |
| 1958 | |
| 1959 | static inline int |
| 1960 | ptlrpc_at_check(struct ptlrpc_service_part *svcpt) |
| 1961 | { |
| 1962 | return svcpt->scp_at_check; |
| 1963 | } |
| 1964 | |
| 1965 | /** |
| 1966 | * requests wait on preprocessing |
| 1967 | * user can call it w/o any lock but need to hold |
| 1968 | * ptlrpc_service_part::scp_lock to get reliable result |
| 1969 | */ |
| 1970 | static inline int |
| 1971 | ptlrpc_server_request_incoming(struct ptlrpc_service_part *svcpt) |
| 1972 | { |
| 1973 | return !list_empty(&svcpt->scp_req_incoming); |
| 1974 | } |
| 1975 | |
| 1976 | static __attribute__((__noinline__)) int |
| 1977 | ptlrpc_wait_event(struct ptlrpc_service_part *svcpt, |
| 1978 | struct ptlrpc_thread *thread) |
| 1979 | { |
| 1980 | /* Don't exit while there are replies to be handled */ |
| 1981 | struct l_wait_info lwi = LWI_TIMEOUT(svcpt->scp_rqbd_timeout, |
| 1982 | ptlrpc_retry_rqbds, svcpt); |
| 1983 | |
Peng Tao | 5d4450c | 2013-07-15 22:27:15 +0800 | [diff] [blame] | 1984 | /* XXX: Add this back when libcfs watchdog is merged upstream |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 1985 | lc_watchdog_disable(thread->t_watchdog); |
Peng Tao | 5d4450c | 2013-07-15 22:27:15 +0800 | [diff] [blame] | 1986 | */ |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 1987 | |
| 1988 | cond_resched(); |
| 1989 | |
| 1990 | l_wait_event_exclusive_head(svcpt->scp_waitq, |
Oleg Drokin | 24c198e | 2016-08-21 18:04:35 -0400 | [diff] [blame] | 1991 | ptlrpc_thread_stopping(thread) || |
| 1992 | ptlrpc_server_request_incoming(svcpt) || |
| 1993 | ptlrpc_server_request_pending(svcpt, |
| 1994 | false) || |
| 1995 | ptlrpc_rqbd_pending(svcpt) || |
| 1996 | ptlrpc_at_check(svcpt), &lwi); |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 1997 | |
| 1998 | if (ptlrpc_thread_stopping(thread)) |
| 1999 | return -EINTR; |
| 2000 | |
Peng Tao | 5d4450c | 2013-07-15 22:27:15 +0800 | [diff] [blame] | 2001 | /* |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 2002 | lc_watchdog_touch(thread->t_watchdog, |
| 2003 | ptlrpc_server_get_timeout(svcpt)); |
Peng Tao | 5d4450c | 2013-07-15 22:27:15 +0800 | [diff] [blame] | 2004 | */ |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 2005 | return 0; |
| 2006 | } |
| 2007 | |
| 2008 | /** |
| 2009 | * Main thread body for service threads. |
| 2010 | * Waits in a loop waiting for new requests to process to appear. |
| 2011 | * Every time an incoming requests is added to its queue, a waitq |
| 2012 | * is woken up and one of the threads will handle it. |
| 2013 | */ |
| 2014 | static int ptlrpc_main(void *arg) |
| 2015 | { |
Amitoj Kaur Chawla | 864ef62 | 2015-10-15 00:13:30 +0530 | [diff] [blame] | 2016 | struct ptlrpc_thread *thread = arg; |
Chris Hanna | d0bfef3 | 2015-06-03 10:28:26 -0400 | [diff] [blame] | 2017 | struct ptlrpc_service_part *svcpt = thread->t_svcpt; |
| 2018 | struct ptlrpc_service *svc = svcpt->scp_service; |
| 2019 | struct ptlrpc_reply_state *rs; |
Greg Kroah-Hartman | c88a6cb | 2013-08-04 07:59:19 +0800 | [diff] [blame] | 2020 | struct group_info *ginfo = NULL; |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 2021 | struct lu_env *env; |
| 2022 | int counter = 0, rc = 0; |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 2023 | |
| 2024 | thread->t_pid = current_pid(); |
| 2025 | unshare_fs_struct(); |
| 2026 | |
| 2027 | /* NB: we will call cfs_cpt_bind() for all threads, because we |
| 2028 | * might want to run lustre server only on a subset of system CPUs, |
Oleg Drokin | dadfcda | 2016-02-24 22:00:38 -0500 | [diff] [blame] | 2029 | * in that case ->scp_cpt is CFS_CPT_ANY |
| 2030 | */ |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 2031 | rc = cfs_cpt_bind(svc->srv_cptable, svcpt->scp_cpt); |
| 2032 | if (rc != 0) { |
| 2033 | CWARN("%s: failed to bind %s on CPT %d\n", |
| 2034 | svc->srv_name, thread->t_name, svcpt->scp_cpt); |
| 2035 | } |
| 2036 | |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 2037 | ginfo = groups_alloc(0); |
| 2038 | if (!ginfo) { |
| 2039 | rc = -ENOMEM; |
| 2040 | goto out; |
| 2041 | } |
| 2042 | |
| 2043 | set_current_groups(ginfo); |
| 2044 | put_group_info(ginfo); |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 2045 | |
Oleg Drokin | 8b38208 | 2016-02-16 00:46:58 -0500 | [diff] [blame] | 2046 | if (svc->srv_ops.so_thr_init) { |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 2047 | rc = svc->srv_ops.so_thr_init(thread); |
| 2048 | if (rc) |
| 2049 | goto out; |
| 2050 | } |
| 2051 | |
Julia Lawall | 9ae1059 | 2015-05-01 17:51:12 +0200 | [diff] [blame] | 2052 | env = kzalloc(sizeof(*env), GFP_NOFS); |
Julia Lawall | 597851a | 2015-06-20 18:59:10 +0200 | [diff] [blame] | 2053 | if (!env) { |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 2054 | rc = -ENOMEM; |
| 2055 | goto out_srv_fini; |
| 2056 | } |
| 2057 | |
| 2058 | rc = lu_context_init(&env->le_ctx, |
Oleg Drokin | cd94f23 | 2016-08-21 18:04:34 -0400 | [diff] [blame] | 2059 | svc->srv_ctx_tags | LCT_REMEMBER | LCT_NOREF); |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 2060 | if (rc) |
| 2061 | goto out_srv_fini; |
| 2062 | |
| 2063 | thread->t_env = env; |
| 2064 | env->le_ctx.lc_thread = thread; |
| 2065 | env->le_ctx.lc_cookie = 0x6; |
| 2066 | |
| 2067 | while (!list_empty(&svcpt->scp_rqbd_idle)) { |
| 2068 | rc = ptlrpc_server_post_idle_rqbds(svcpt); |
| 2069 | if (rc >= 0) |
| 2070 | continue; |
| 2071 | |
| 2072 | CERROR("Failed to post rqbd for %s on CPT %d: %d\n", |
Oleg Drokin | 30c0aa3 | 2016-02-26 01:50:02 -0500 | [diff] [blame] | 2073 | svc->srv_name, svcpt->scp_cpt, rc); |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 2074 | goto out_srv_fini; |
| 2075 | } |
| 2076 | |
| 2077 | /* Alloc reply state structure for this one */ |
Julia Lawall | ee0ec19 | 2015-06-11 14:02:58 +0200 | [diff] [blame] | 2078 | rs = libcfs_kvzalloc(svc->srv_max_reply_size, GFP_NOFS); |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 2079 | if (!rs) { |
| 2080 | rc = -ENOMEM; |
| 2081 | goto out_srv_fini; |
| 2082 | } |
| 2083 | |
| 2084 | spin_lock(&svcpt->scp_lock); |
| 2085 | |
| 2086 | LASSERT(thread_is_starting(thread)); |
| 2087 | thread_clear_flags(thread, SVC_STARTING); |
| 2088 | |
| 2089 | LASSERT(svcpt->scp_nthrs_starting == 1); |
| 2090 | svcpt->scp_nthrs_starting--; |
| 2091 | |
| 2092 | /* SVC_STOPPING may already be set here if someone else is trying |
| 2093 | * to stop the service while this new thread has been dynamically |
| 2094 | * forked. We still set SVC_RUNNING to let our creator know that |
Oleg Drokin | dadfcda | 2016-02-24 22:00:38 -0500 | [diff] [blame] | 2095 | * we are now running, however we will exit as soon as possible |
| 2096 | */ |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 2097 | thread_add_flags(thread, SVC_RUNNING); |
| 2098 | svcpt->scp_nthrs_running++; |
| 2099 | spin_unlock(&svcpt->scp_lock); |
| 2100 | |
| 2101 | /* wake up our creator in case he's still waiting. */ |
| 2102 | wake_up(&thread->t_ctl_waitq); |
| 2103 | |
Peng Tao | 5d4450c | 2013-07-15 22:27:15 +0800 | [diff] [blame] | 2104 | /* |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 2105 | thread->t_watchdog = lc_watchdog_add(ptlrpc_server_get_timeout(svcpt), |
| 2106 | NULL, NULL); |
Peng Tao | 5d4450c | 2013-07-15 22:27:15 +0800 | [diff] [blame] | 2107 | */ |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 2108 | |
| 2109 | spin_lock(&svcpt->scp_rep_lock); |
| 2110 | list_add(&rs->rs_list, &svcpt->scp_rep_idle); |
| 2111 | wake_up(&svcpt->scp_rep_waitq); |
| 2112 | spin_unlock(&svcpt->scp_rep_lock); |
| 2113 | |
| 2114 | CDEBUG(D_NET, "service thread %d (#%d) started\n", thread->t_id, |
| 2115 | svcpt->scp_nthrs_running); |
| 2116 | |
| 2117 | /* XXX maintain a list of all managed devices: insert here */ |
| 2118 | while (!ptlrpc_thread_stopping(thread)) { |
| 2119 | if (ptlrpc_wait_event(svcpt, thread)) |
| 2120 | break; |
| 2121 | |
| 2122 | ptlrpc_check_rqbd_pool(svcpt); |
| 2123 | |
| 2124 | if (ptlrpc_threads_need_create(svcpt)) { |
| 2125 | /* Ignore return code - we tried... */ |
| 2126 | ptlrpc_start_thread(svcpt, 0); |
| 2127 | } |
| 2128 | |
| 2129 | /* Process all incoming reqs before handling any */ |
| 2130 | if (ptlrpc_server_request_incoming(svcpt)) { |
| 2131 | lu_context_enter(&env->le_ctx); |
wang di | 4ee688d | 2013-06-03 21:40:39 +0800 | [diff] [blame] | 2132 | env->le_ses = NULL; |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 2133 | ptlrpc_server_handle_req_in(svcpt, thread); |
| 2134 | lu_context_exit(&env->le_ctx); |
| 2135 | |
| 2136 | /* but limit ourselves in case of flood */ |
| 2137 | if (counter++ < 100) |
| 2138 | continue; |
| 2139 | counter = 0; |
| 2140 | } |
| 2141 | |
| 2142 | if (ptlrpc_at_check(svcpt)) |
| 2143 | ptlrpc_at_check_timed(svcpt); |
| 2144 | |
| 2145 | if (ptlrpc_server_request_pending(svcpt, false)) { |
| 2146 | lu_context_enter(&env->le_ctx); |
| 2147 | ptlrpc_server_handle_request(svcpt, thread); |
| 2148 | lu_context_exit(&env->le_ctx); |
| 2149 | } |
| 2150 | |
| 2151 | if (ptlrpc_rqbd_pending(svcpt) && |
| 2152 | ptlrpc_server_post_idle_rqbds(svcpt) < 0) { |
| 2153 | /* I just failed to repost request buffers. |
| 2154 | * Wait for a timeout (unless something else |
Oleg Drokin | dadfcda | 2016-02-24 22:00:38 -0500 | [diff] [blame] | 2155 | * happens) before I try again |
| 2156 | */ |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 2157 | svcpt->scp_rqbd_timeout = cfs_time_seconds(1) / 10; |
| 2158 | CDEBUG(D_RPCTRACE, "Posted buffers: %d\n", |
| 2159 | svcpt->scp_nrqbds_posted); |
| 2160 | } |
| 2161 | } |
| 2162 | |
Peng Tao | 5d4450c | 2013-07-15 22:27:15 +0800 | [diff] [blame] | 2163 | /* |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 2164 | lc_watchdog_delete(thread->t_watchdog); |
| 2165 | thread->t_watchdog = NULL; |
Peng Tao | 5d4450c | 2013-07-15 22:27:15 +0800 | [diff] [blame] | 2166 | */ |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 2167 | |
| 2168 | out_srv_fini: |
| 2169 | /* |
| 2170 | * deconstruct service specific state created by ptlrpc_start_thread() |
| 2171 | */ |
Oleg Drokin | 8b38208 | 2016-02-16 00:46:58 -0500 | [diff] [blame] | 2172 | if (svc->srv_ops.so_thr_done) |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 2173 | svc->srv_ops.so_thr_done(thread); |
| 2174 | |
Oleg Drokin | 8b38208 | 2016-02-16 00:46:58 -0500 | [diff] [blame] | 2175 | if (env) { |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 2176 | lu_context_fini(&env->le_ctx); |
Julia Lawall | 9ae1059 | 2015-05-01 17:51:12 +0200 | [diff] [blame] | 2177 | kfree(env); |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 2178 | } |
| 2179 | out: |
| 2180 | CDEBUG(D_RPCTRACE, "service thread [ %p : %u ] %d exiting: rc %d\n", |
| 2181 | thread, thread->t_pid, thread->t_id, rc); |
| 2182 | |
| 2183 | spin_lock(&svcpt->scp_lock); |
| 2184 | if (thread_test_and_clear_flags(thread, SVC_STARTING)) |
| 2185 | svcpt->scp_nthrs_starting--; |
| 2186 | |
| 2187 | if (thread_test_and_clear_flags(thread, SVC_RUNNING)) { |
| 2188 | /* must know immediately */ |
| 2189 | svcpt->scp_nthrs_running--; |
| 2190 | } |
| 2191 | |
| 2192 | thread->t_id = rc; |
| 2193 | thread_add_flags(thread, SVC_STOPPED); |
| 2194 | |
| 2195 | wake_up(&thread->t_ctl_waitq); |
| 2196 | spin_unlock(&svcpt->scp_lock); |
| 2197 | |
| 2198 | return rc; |
| 2199 | } |
| 2200 | |
| 2201 | static int hrt_dont_sleep(struct ptlrpc_hr_thread *hrt, |
| 2202 | struct list_head *replies) |
| 2203 | { |
| 2204 | int result; |
| 2205 | |
| 2206 | spin_lock(&hrt->hrt_lock); |
| 2207 | |
| 2208 | list_splice_init(&hrt->hrt_queue, replies); |
| 2209 | result = ptlrpc_hr.hr_stopping || !list_empty(replies); |
| 2210 | |
| 2211 | spin_unlock(&hrt->hrt_lock); |
| 2212 | return result; |
| 2213 | } |
| 2214 | |
| 2215 | /** |
| 2216 | * Main body of "handle reply" function. |
| 2217 | * It processes acked reply states |
| 2218 | */ |
| 2219 | static int ptlrpc_hr_main(void *arg) |
| 2220 | { |
Amitoj Kaur Chawla | 864ef62 | 2015-10-15 00:13:30 +0530 | [diff] [blame] | 2221 | struct ptlrpc_hr_thread *hrt = arg; |
Chris Hanna | d0bfef3 | 2015-06-03 10:28:26 -0400 | [diff] [blame] | 2222 | struct ptlrpc_hr_partition *hrp = hrt->hrt_partition; |
Tapan Prakash T | 24721b2 | 2016-02-28 20:04:15 +0530 | [diff] [blame] | 2223 | LIST_HEAD(replies); |
Chris Hanna | d0bfef3 | 2015-06-03 10:28:26 -0400 | [diff] [blame] | 2224 | char threadname[20]; |
| 2225 | int rc; |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 2226 | |
| 2227 | snprintf(threadname, sizeof(threadname), "ptlrpc_hr%02d_%03d", |
| 2228 | hrp->hrp_cpt, hrt->hrt_id); |
| 2229 | unshare_fs_struct(); |
| 2230 | |
| 2231 | rc = cfs_cpt_bind(ptlrpc_hr.hr_cpt_table, hrp->hrp_cpt); |
| 2232 | if (rc != 0) { |
| 2233 | CWARN("Failed to bind %s on CPT %d of CPT table %p: rc = %d\n", |
| 2234 | threadname, hrp->hrp_cpt, ptlrpc_hr.hr_cpt_table, rc); |
| 2235 | } |
| 2236 | |
| 2237 | atomic_inc(&hrp->hrp_nstarted); |
| 2238 | wake_up(&ptlrpc_hr.hr_waitq); |
| 2239 | |
| 2240 | while (!ptlrpc_hr.hr_stopping) { |
| 2241 | l_wait_condition(hrt->hrt_waitq, hrt_dont_sleep(hrt, &replies)); |
| 2242 | |
| 2243 | while (!list_empty(&replies)) { |
| 2244 | struct ptlrpc_reply_state *rs; |
| 2245 | |
Oleg Drokin | 30c0aa3 | 2016-02-26 01:50:02 -0500 | [diff] [blame] | 2246 | rs = list_entry(replies.prev, struct ptlrpc_reply_state, |
| 2247 | rs_list); |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 2248 | list_del_init(&rs->rs_list); |
| 2249 | ptlrpc_handle_rs(rs); |
| 2250 | } |
| 2251 | } |
| 2252 | |
| 2253 | atomic_inc(&hrp->hrp_nstopped); |
| 2254 | wake_up(&ptlrpc_hr.hr_waitq); |
| 2255 | |
| 2256 | return 0; |
| 2257 | } |
| 2258 | |
| 2259 | static void ptlrpc_stop_hr_threads(void) |
| 2260 | { |
Chris Hanna | d0bfef3 | 2015-06-03 10:28:26 -0400 | [diff] [blame] | 2261 | struct ptlrpc_hr_partition *hrp; |
| 2262 | int i; |
| 2263 | int j; |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 2264 | |
| 2265 | ptlrpc_hr.hr_stopping = 1; |
| 2266 | |
| 2267 | cfs_percpt_for_each(hrp, i, ptlrpc_hr.hr_partitions) { |
Oleg Drokin | 8b38208 | 2016-02-16 00:46:58 -0500 | [diff] [blame] | 2268 | if (!hrp->hrp_thrs) |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 2269 | continue; /* uninitialized */ |
| 2270 | for (j = 0; j < hrp->hrp_nthrs; j++) |
| 2271 | wake_up_all(&hrp->hrp_thrs[j].hrt_waitq); |
| 2272 | } |
| 2273 | |
| 2274 | cfs_percpt_for_each(hrp, i, ptlrpc_hr.hr_partitions) { |
Oleg Drokin | 8b38208 | 2016-02-16 00:46:58 -0500 | [diff] [blame] | 2275 | if (!hrp->hrp_thrs) |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 2276 | continue; /* uninitialized */ |
| 2277 | wait_event(ptlrpc_hr.hr_waitq, |
Oleg Drokin | 30c0aa3 | 2016-02-26 01:50:02 -0500 | [diff] [blame] | 2278 | atomic_read(&hrp->hrp_nstopped) == |
| 2279 | atomic_read(&hrp->hrp_nstarted)); |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 2280 | } |
| 2281 | } |
| 2282 | |
| 2283 | static int ptlrpc_start_hr_threads(void) |
| 2284 | { |
Chris Hanna | d0bfef3 | 2015-06-03 10:28:26 -0400 | [diff] [blame] | 2285 | struct ptlrpc_hr_partition *hrp; |
| 2286 | int i; |
| 2287 | int j; |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 2288 | |
| 2289 | cfs_percpt_for_each(hrp, i, ptlrpc_hr.hr_partitions) { |
Chris Hanna | d0bfef3 | 2015-06-03 10:28:26 -0400 | [diff] [blame] | 2290 | int rc = 0; |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 2291 | |
| 2292 | for (j = 0; j < hrp->hrp_nthrs; j++) { |
| 2293 | struct ptlrpc_hr_thread *hrt = &hrp->hrp_thrs[j]; |
John L. Hammond | 060c282 | 2016-02-15 10:25:49 -0500 | [diff] [blame] | 2294 | struct task_struct *task; |
Mike Rapoport | 50ffcb7 | 2015-10-13 16:03:40 +0300 | [diff] [blame] | 2295 | |
John L. Hammond | 060c282 | 2016-02-15 10:25:49 -0500 | [diff] [blame] | 2296 | task = kthread_run(ptlrpc_hr_main, |
Oleg Drokin | 30c0aa3 | 2016-02-26 01:50:02 -0500 | [diff] [blame] | 2297 | &hrp->hrp_thrs[j], |
| 2298 | "ptlrpc_hr%02d_%03d", |
| 2299 | hrp->hrp_cpt, hrt->hrt_id); |
John L. Hammond | 060c282 | 2016-02-15 10:25:49 -0500 | [diff] [blame] | 2300 | if (IS_ERR(task)) { |
| 2301 | rc = PTR_ERR(task); |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 2302 | break; |
John L. Hammond | 060c282 | 2016-02-15 10:25:49 -0500 | [diff] [blame] | 2303 | } |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 2304 | } |
| 2305 | wait_event(ptlrpc_hr.hr_waitq, |
Oleg Drokin | 30c0aa3 | 2016-02-26 01:50:02 -0500 | [diff] [blame] | 2306 | atomic_read(&hrp->hrp_nstarted) == j); |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 2307 | |
John L. Hammond | 060c282 | 2016-02-15 10:25:49 -0500 | [diff] [blame] | 2308 | if (rc < 0) { |
| 2309 | CERROR("cannot start reply handler thread %d:%d: rc = %d\n", |
| 2310 | i, j, rc); |
| 2311 | ptlrpc_stop_hr_threads(); |
| 2312 | return rc; |
| 2313 | } |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 2314 | } |
Greg Kroah-Hartman | 0a3bdb0 | 2013-08-03 10:35:28 +0800 | [diff] [blame] | 2315 | return 0; |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 2316 | } |
| 2317 | |
| 2318 | static void ptlrpc_svcpt_stop_threads(struct ptlrpc_service_part *svcpt) |
| 2319 | { |
Chris Hanna | d0bfef3 | 2015-06-03 10:28:26 -0400 | [diff] [blame] | 2320 | struct l_wait_info lwi = { 0 }; |
| 2321 | struct ptlrpc_thread *thread; |
Tapan Prakash T | 24721b2 | 2016-02-28 20:04:15 +0530 | [diff] [blame] | 2322 | LIST_HEAD(zombie); |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 2323 | |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 2324 | CDEBUG(D_INFO, "Stopping threads for service %s\n", |
| 2325 | svcpt->scp_service->srv_name); |
| 2326 | |
| 2327 | spin_lock(&svcpt->scp_lock); |
| 2328 | /* let the thread know that we would like it to stop asap */ |
| 2329 | list_for_each_entry(thread, &svcpt->scp_threads, t_link) { |
| 2330 | CDEBUG(D_INFO, "Stopping thread %s #%u\n", |
| 2331 | svcpt->scp_service->srv_thread_name, thread->t_id); |
| 2332 | thread_add_flags(thread, SVC_STOPPING); |
| 2333 | } |
| 2334 | |
| 2335 | wake_up_all(&svcpt->scp_waitq); |
| 2336 | |
| 2337 | while (!list_empty(&svcpt->scp_threads)) { |
| 2338 | thread = list_entry(svcpt->scp_threads.next, |
Oleg Drokin | 30c0aa3 | 2016-02-26 01:50:02 -0500 | [diff] [blame] | 2339 | struct ptlrpc_thread, t_link); |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 2340 | if (thread_is_stopped(thread)) { |
| 2341 | list_del(&thread->t_link); |
| 2342 | list_add(&thread->t_link, &zombie); |
| 2343 | continue; |
| 2344 | } |
| 2345 | spin_unlock(&svcpt->scp_lock); |
| 2346 | |
| 2347 | CDEBUG(D_INFO, "waiting for stopping-thread %s #%u\n", |
| 2348 | svcpt->scp_service->srv_thread_name, thread->t_id); |
| 2349 | l_wait_event(thread->t_ctl_waitq, |
| 2350 | thread_is_stopped(thread), &lwi); |
| 2351 | |
| 2352 | spin_lock(&svcpt->scp_lock); |
| 2353 | } |
| 2354 | |
| 2355 | spin_unlock(&svcpt->scp_lock); |
| 2356 | |
| 2357 | while (!list_empty(&zombie)) { |
| 2358 | thread = list_entry(zombie.next, |
Oleg Drokin | 24c198e | 2016-08-21 18:04:35 -0400 | [diff] [blame] | 2359 | struct ptlrpc_thread, t_link); |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 2360 | list_del(&thread->t_link); |
Julia Lawall | 9ae1059 | 2015-05-01 17:51:12 +0200 | [diff] [blame] | 2361 | kfree(thread); |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 2362 | } |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 2363 | } |
| 2364 | |
| 2365 | /** |
| 2366 | * Stops all threads of a particular service \a svc |
| 2367 | */ |
Shraddha Barke | 230a8da1 | 2015-10-08 01:36:07 +0530 | [diff] [blame] | 2368 | static void ptlrpc_stop_all_threads(struct ptlrpc_service *svc) |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 2369 | { |
| 2370 | struct ptlrpc_service_part *svcpt; |
Chris Hanna | d0bfef3 | 2015-06-03 10:28:26 -0400 | [diff] [blame] | 2371 | int i; |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 2372 | |
| 2373 | ptlrpc_service_for_each_part(svcpt, i, svc) { |
Oleg Drokin | 8b38208 | 2016-02-16 00:46:58 -0500 | [diff] [blame] | 2374 | if (svcpt->scp_service) |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 2375 | ptlrpc_svcpt_stop_threads(svcpt); |
| 2376 | } |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 2377 | } |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 2378 | |
| 2379 | int ptlrpc_start_threads(struct ptlrpc_service *svc) |
| 2380 | { |
Chris Hanna | d0bfef3 | 2015-06-03 10:28:26 -0400 | [diff] [blame] | 2381 | int rc = 0; |
| 2382 | int i; |
| 2383 | int j; |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 2384 | |
| 2385 | /* We require 2 threads min, see note in ptlrpc_server_handle_request */ |
| 2386 | LASSERT(svc->srv_nthrs_cpt_init >= PTLRPC_NTHRS_INIT); |
| 2387 | |
| 2388 | for (i = 0; i < svc->srv_ncpts; i++) { |
| 2389 | for (j = 0; j < svc->srv_nthrs_cpt_init; j++) { |
| 2390 | rc = ptlrpc_start_thread(svc->srv_parts[i], 1); |
| 2391 | if (rc == 0) |
| 2392 | continue; |
| 2393 | |
| 2394 | if (rc != -EMFILE) |
| 2395 | goto failed; |
| 2396 | /* We have enough threads, don't start more. b=15759 */ |
| 2397 | break; |
| 2398 | } |
| 2399 | } |
| 2400 | |
Greg Kroah-Hartman | 0a3bdb0 | 2013-08-03 10:35:28 +0800 | [diff] [blame] | 2401 | return 0; |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 2402 | failed: |
| 2403 | CERROR("cannot start %s thread #%d_%d: rc %d\n", |
| 2404 | svc->srv_thread_name, i, j, rc); |
| 2405 | ptlrpc_stop_all_threads(svc); |
Greg Kroah-Hartman | 0a3bdb0 | 2013-08-03 10:35:28 +0800 | [diff] [blame] | 2406 | return rc; |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 2407 | } |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 2408 | |
| 2409 | int ptlrpc_start_thread(struct ptlrpc_service_part *svcpt, int wait) |
| 2410 | { |
Chris Hanna | d0bfef3 | 2015-06-03 10:28:26 -0400 | [diff] [blame] | 2411 | struct l_wait_info lwi = { 0 }; |
| 2412 | struct ptlrpc_thread *thread; |
| 2413 | struct ptlrpc_service *svc; |
John L. Hammond | 060c282 | 2016-02-15 10:25:49 -0500 | [diff] [blame] | 2414 | struct task_struct *task; |
Chris Hanna | d0bfef3 | 2015-06-03 10:28:26 -0400 | [diff] [blame] | 2415 | int rc; |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 2416 | |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 2417 | svc = svcpt->scp_service; |
| 2418 | |
| 2419 | CDEBUG(D_RPCTRACE, "%s[%d] started %d min %d max %d\n", |
| 2420 | svc->srv_name, svcpt->scp_cpt, svcpt->scp_nthrs_running, |
| 2421 | svc->srv_nthrs_cpt_init, svc->srv_nthrs_cpt_limit); |
| 2422 | |
| 2423 | again: |
| 2424 | if (unlikely(svc->srv_is_stopping)) |
Greg Kroah-Hartman | 0a3bdb0 | 2013-08-03 10:35:28 +0800 | [diff] [blame] | 2425 | return -ESRCH; |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 2426 | |
| 2427 | if (!ptlrpc_threads_increasable(svcpt) || |
| 2428 | (OBD_FAIL_CHECK(OBD_FAIL_TGT_TOOMANY_THREADS) && |
| 2429 | svcpt->scp_nthrs_running == svc->srv_nthrs_cpt_init - 1)) |
Greg Kroah-Hartman | 0a3bdb0 | 2013-08-03 10:35:28 +0800 | [diff] [blame] | 2430 | return -EMFILE; |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 2431 | |
Julia Lawall | bae97e8 | 2015-05-03 15:21:44 +0200 | [diff] [blame] | 2432 | thread = kzalloc_node(sizeof(*thread), GFP_NOFS, |
| 2433 | cfs_cpt_spread_node(svc->srv_cptable, |
| 2434 | svcpt->scp_cpt)); |
Oleg Drokin | 8b38208 | 2016-02-16 00:46:58 -0500 | [diff] [blame] | 2435 | if (!thread) |
Greg Kroah-Hartman | 0a3bdb0 | 2013-08-03 10:35:28 +0800 | [diff] [blame] | 2436 | return -ENOMEM; |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 2437 | init_waitqueue_head(&thread->t_ctl_waitq); |
| 2438 | |
| 2439 | spin_lock(&svcpt->scp_lock); |
| 2440 | if (!ptlrpc_threads_increasable(svcpt)) { |
| 2441 | spin_unlock(&svcpt->scp_lock); |
Julia Lawall | 9ae1059 | 2015-05-01 17:51:12 +0200 | [diff] [blame] | 2442 | kfree(thread); |
Greg Kroah-Hartman | 0a3bdb0 | 2013-08-03 10:35:28 +0800 | [diff] [blame] | 2443 | return -EMFILE; |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 2444 | } |
| 2445 | |
| 2446 | if (svcpt->scp_nthrs_starting != 0) { |
| 2447 | /* serialize starting because some modules (obdfilter) |
Oleg Drokin | dadfcda | 2016-02-24 22:00:38 -0500 | [diff] [blame] | 2448 | * might require unique and contiguous t_id |
| 2449 | */ |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 2450 | LASSERT(svcpt->scp_nthrs_starting == 1); |
| 2451 | spin_unlock(&svcpt->scp_lock); |
Julia Lawall | 9ae1059 | 2015-05-01 17:51:12 +0200 | [diff] [blame] | 2452 | kfree(thread); |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 2453 | if (wait) { |
| 2454 | CDEBUG(D_INFO, "Waiting for creating thread %s #%d\n", |
| 2455 | svc->srv_thread_name, svcpt->scp_thr_nextid); |
| 2456 | schedule(); |
| 2457 | goto again; |
| 2458 | } |
| 2459 | |
| 2460 | CDEBUG(D_INFO, "Creating thread %s #%d race, retry later\n", |
| 2461 | svc->srv_thread_name, svcpt->scp_thr_nextid); |
Greg Kroah-Hartman | 0a3bdb0 | 2013-08-03 10:35:28 +0800 | [diff] [blame] | 2462 | return -EAGAIN; |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 2463 | } |
| 2464 | |
| 2465 | svcpt->scp_nthrs_starting++; |
| 2466 | thread->t_id = svcpt->scp_thr_nextid++; |
| 2467 | thread_add_flags(thread, SVC_STARTING); |
| 2468 | thread->t_svcpt = svcpt; |
| 2469 | |
| 2470 | list_add(&thread->t_link, &svcpt->scp_threads); |
| 2471 | spin_unlock(&svcpt->scp_lock); |
| 2472 | |
| 2473 | if (svcpt->scp_cpt >= 0) { |
Kees Cook | 9edf0f6 | 2013-09-10 21:37:19 -0700 | [diff] [blame] | 2474 | snprintf(thread->t_name, sizeof(thread->t_name), "%s%02d_%03d", |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 2475 | svc->srv_thread_name, svcpt->scp_cpt, thread->t_id); |
| 2476 | } else { |
Kees Cook | 9edf0f6 | 2013-09-10 21:37:19 -0700 | [diff] [blame] | 2477 | snprintf(thread->t_name, sizeof(thread->t_name), "%s_%04d", |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 2478 | svc->srv_thread_name, thread->t_id); |
| 2479 | } |
| 2480 | |
| 2481 | CDEBUG(D_RPCTRACE, "starting thread '%s'\n", thread->t_name); |
John L. Hammond | 060c282 | 2016-02-15 10:25:49 -0500 | [diff] [blame] | 2482 | task = kthread_run(ptlrpc_main, thread, "%s", thread->t_name); |
| 2483 | if (IS_ERR(task)) { |
| 2484 | rc = PTR_ERR(task); |
| 2485 | CERROR("cannot start thread '%s': rc = %d\n", |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 2486 | thread->t_name, rc); |
| 2487 | spin_lock(&svcpt->scp_lock); |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 2488 | --svcpt->scp_nthrs_starting; |
Hiroya Nozaki | 5be8e07 | 2013-07-23 00:06:48 +0800 | [diff] [blame] | 2489 | if (thread_is_stopping(thread)) { |
Masanari Iida | 369e5c9 | 2014-02-08 00:30:36 +0900 | [diff] [blame] | 2490 | /* this ptlrpc_thread is being handled |
Hiroya Nozaki | 5be8e07 | 2013-07-23 00:06:48 +0800 | [diff] [blame] | 2491 | * by ptlrpc_svcpt_stop_threads now |
| 2492 | */ |
| 2493 | thread_add_flags(thread, SVC_STOPPED); |
| 2494 | wake_up(&thread->t_ctl_waitq); |
| 2495 | spin_unlock(&svcpt->scp_lock); |
| 2496 | } else { |
| 2497 | list_del(&thread->t_link); |
| 2498 | spin_unlock(&svcpt->scp_lock); |
Julia Lawall | 9ae1059 | 2015-05-01 17:51:12 +0200 | [diff] [blame] | 2499 | kfree(thread); |
Hiroya Nozaki | 5be8e07 | 2013-07-23 00:06:48 +0800 | [diff] [blame] | 2500 | } |
Greg Kroah-Hartman | 0a3bdb0 | 2013-08-03 10:35:28 +0800 | [diff] [blame] | 2501 | return rc; |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 2502 | } |
| 2503 | |
| 2504 | if (!wait) |
Greg Kroah-Hartman | 0a3bdb0 | 2013-08-03 10:35:28 +0800 | [diff] [blame] | 2505 | return 0; |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 2506 | |
| 2507 | l_wait_event(thread->t_ctl_waitq, |
| 2508 | thread_is_running(thread) || thread_is_stopped(thread), |
| 2509 | &lwi); |
| 2510 | |
| 2511 | rc = thread_is_stopped(thread) ? thread->t_id : 0; |
Greg Kroah-Hartman | 0a3bdb0 | 2013-08-03 10:35:28 +0800 | [diff] [blame] | 2512 | return rc; |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 2513 | } |
| 2514 | |
| 2515 | int ptlrpc_hr_init(void) |
| 2516 | { |
Chris Hanna | d0bfef3 | 2015-06-03 10:28:26 -0400 | [diff] [blame] | 2517 | struct ptlrpc_hr_partition *hrp; |
| 2518 | struct ptlrpc_hr_thread *hrt; |
| 2519 | int rc; |
| 2520 | int i; |
| 2521 | int j; |
| 2522 | int weight; |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 2523 | |
| 2524 | memset(&ptlrpc_hr, 0, sizeof(ptlrpc_hr)); |
| 2525 | ptlrpc_hr.hr_cpt_table = cfs_cpt_table; |
| 2526 | |
| 2527 | ptlrpc_hr.hr_partitions = cfs_percpt_alloc(ptlrpc_hr.hr_cpt_table, |
| 2528 | sizeof(*hrp)); |
Oleg Drokin | 8b38208 | 2016-02-16 00:46:58 -0500 | [diff] [blame] | 2529 | if (!ptlrpc_hr.hr_partitions) |
Greg Kroah-Hartman | 0a3bdb0 | 2013-08-03 10:35:28 +0800 | [diff] [blame] | 2530 | return -ENOMEM; |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 2531 | |
| 2532 | init_waitqueue_head(&ptlrpc_hr.hr_waitq); |
| 2533 | |
Bartosz Golaszewski | 06931e6 | 2015-05-26 15:11:28 +0200 | [diff] [blame] | 2534 | weight = cpumask_weight(topology_sibling_cpumask(0)); |
Peng Tao | 3867ea5 | 2013-07-15 22:27:10 +0800 | [diff] [blame] | 2535 | |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 2536 | cfs_percpt_for_each(hrp, i, ptlrpc_hr.hr_partitions) { |
| 2537 | hrp->hrp_cpt = i; |
| 2538 | |
| 2539 | atomic_set(&hrp->hrp_nstarted, 0); |
| 2540 | atomic_set(&hrp->hrp_nstopped, 0); |
| 2541 | |
| 2542 | hrp->hrp_nthrs = cfs_cpt_weight(ptlrpc_hr.hr_cpt_table, i); |
Peng Tao | 3867ea5 | 2013-07-15 22:27:10 +0800 | [diff] [blame] | 2543 | hrp->hrp_nthrs /= weight; |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 2544 | |
| 2545 | LASSERT(hrp->hrp_nthrs > 0); |
Julia Lawall | bae97e8 | 2015-05-03 15:21:44 +0200 | [diff] [blame] | 2546 | hrp->hrp_thrs = |
| 2547 | kzalloc_node(hrp->hrp_nthrs * sizeof(*hrt), GFP_NOFS, |
Oleg Drokin | 24c198e | 2016-08-21 18:04:35 -0400 | [diff] [blame] | 2548 | cfs_cpt_spread_node(ptlrpc_hr.hr_cpt_table, |
| 2549 | i)); |
Oleg Drokin | 8b38208 | 2016-02-16 00:46:58 -0500 | [diff] [blame] | 2550 | if (!hrp->hrp_thrs) { |
Julia Lawall | a9b3e8f | 2014-09-07 18:18:29 +0200 | [diff] [blame] | 2551 | rc = -ENOMEM; |
| 2552 | goto out; |
| 2553 | } |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 2554 | |
| 2555 | for (j = 0; j < hrp->hrp_nthrs; j++) { |
| 2556 | hrt = &hrp->hrp_thrs[j]; |
| 2557 | |
| 2558 | hrt->hrt_id = j; |
| 2559 | hrt->hrt_partition = hrp; |
| 2560 | init_waitqueue_head(&hrt->hrt_waitq); |
| 2561 | spin_lock_init(&hrt->hrt_lock); |
| 2562 | INIT_LIST_HEAD(&hrt->hrt_queue); |
| 2563 | } |
| 2564 | } |
| 2565 | |
| 2566 | rc = ptlrpc_start_hr_threads(); |
| 2567 | out: |
| 2568 | if (rc != 0) |
| 2569 | ptlrpc_hr_fini(); |
Greg Kroah-Hartman | 0a3bdb0 | 2013-08-03 10:35:28 +0800 | [diff] [blame] | 2570 | return rc; |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 2571 | } |
| 2572 | |
| 2573 | void ptlrpc_hr_fini(void) |
| 2574 | { |
Chris Hanna | d0bfef3 | 2015-06-03 10:28:26 -0400 | [diff] [blame] | 2575 | struct ptlrpc_hr_partition *hrp; |
| 2576 | int i; |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 2577 | |
Oleg Drokin | 8b38208 | 2016-02-16 00:46:58 -0500 | [diff] [blame] | 2578 | if (!ptlrpc_hr.hr_partitions) |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 2579 | return; |
| 2580 | |
| 2581 | ptlrpc_stop_hr_threads(); |
| 2582 | |
| 2583 | cfs_percpt_for_each(hrp, i, ptlrpc_hr.hr_partitions) { |
Markus Elfring | a5cb888 | 2015-07-14 09:35:42 +0200 | [diff] [blame] | 2584 | kfree(hrp->hrp_thrs); |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 2585 | } |
| 2586 | |
| 2587 | cfs_percpt_free(ptlrpc_hr.hr_partitions); |
| 2588 | ptlrpc_hr.hr_partitions = NULL; |
| 2589 | } |
| 2590 | |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 2591 | /** |
| 2592 | * Wait until all already scheduled replies are processed. |
| 2593 | */ |
| 2594 | static void ptlrpc_wait_replies(struct ptlrpc_service_part *svcpt) |
| 2595 | { |
| 2596 | while (1) { |
| 2597 | int rc; |
| 2598 | struct l_wait_info lwi = LWI_TIMEOUT(cfs_time_seconds(10), |
| 2599 | NULL, NULL); |
| 2600 | |
| 2601 | rc = l_wait_event(svcpt->scp_waitq, |
Oleg Drokin | 24c198e | 2016-08-21 18:04:35 -0400 | [diff] [blame] | 2602 | atomic_read(&svcpt->scp_nreps_difficult) == 0, |
| 2603 | &lwi); |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 2604 | if (rc == 0) |
| 2605 | break; |
| 2606 | CWARN("Unexpectedly long timeout %s %p\n", |
| 2607 | svcpt->scp_service->srv_name, svcpt->scp_service); |
| 2608 | } |
| 2609 | } |
| 2610 | |
| 2611 | static void |
| 2612 | ptlrpc_service_del_atimer(struct ptlrpc_service *svc) |
| 2613 | { |
Chris Hanna | d0bfef3 | 2015-06-03 10:28:26 -0400 | [diff] [blame] | 2614 | struct ptlrpc_service_part *svcpt; |
| 2615 | int i; |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 2616 | |
| 2617 | /* early disarm AT timer... */ |
| 2618 | ptlrpc_service_for_each_part(svcpt, i, svc) { |
Oleg Drokin | 8b38208 | 2016-02-16 00:46:58 -0500 | [diff] [blame] | 2619 | if (svcpt->scp_service) |
Arnd Bergmann | 922da0c | 2015-09-27 16:45:33 -0400 | [diff] [blame] | 2620 | del_timer(&svcpt->scp_at_timer); |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 2621 | } |
| 2622 | } |
| 2623 | |
| 2624 | static void |
| 2625 | ptlrpc_service_unlink_rqbd(struct ptlrpc_service *svc) |
| 2626 | { |
Chris Hanna | d0bfef3 | 2015-06-03 10:28:26 -0400 | [diff] [blame] | 2627 | struct ptlrpc_service_part *svcpt; |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 2628 | struct ptlrpc_request_buffer_desc *rqbd; |
Chris Hanna | d0bfef3 | 2015-06-03 10:28:26 -0400 | [diff] [blame] | 2629 | struct l_wait_info lwi; |
| 2630 | int rc; |
| 2631 | int i; |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 2632 | |
| 2633 | /* All history will be culled when the next request buffer is |
Oleg Drokin | dadfcda | 2016-02-24 22:00:38 -0500 | [diff] [blame] | 2634 | * freed in ptlrpc_service_purge_all() |
| 2635 | */ |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 2636 | svc->srv_hist_nrqbds_cpt_max = 0; |
| 2637 | |
| 2638 | rc = LNetClearLazyPortal(svc->srv_req_portal); |
| 2639 | LASSERT(rc == 0); |
| 2640 | |
| 2641 | ptlrpc_service_for_each_part(svcpt, i, svc) { |
Oleg Drokin | 8b38208 | 2016-02-16 00:46:58 -0500 | [diff] [blame] | 2642 | if (!svcpt->scp_service) |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 2643 | break; |
| 2644 | |
| 2645 | /* Unlink all the request buffers. This forces a 'final' |
Oleg Drokin | dadfcda | 2016-02-24 22:00:38 -0500 | [diff] [blame] | 2646 | * event with its 'unlink' flag set for each posted rqbd |
| 2647 | */ |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 2648 | list_for_each_entry(rqbd, &svcpt->scp_rqbd_posted, |
Oleg Drokin | 24c198e | 2016-08-21 18:04:35 -0400 | [diff] [blame] | 2649 | rqbd_list) { |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 2650 | rc = LNetMDUnlink(rqbd->rqbd_md_h); |
| 2651 | LASSERT(rc == 0 || rc == -ENOENT); |
| 2652 | } |
| 2653 | } |
| 2654 | |
| 2655 | ptlrpc_service_for_each_part(svcpt, i, svc) { |
Oleg Drokin | 8b38208 | 2016-02-16 00:46:58 -0500 | [diff] [blame] | 2656 | if (!svcpt->scp_service) |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 2657 | break; |
| 2658 | |
| 2659 | /* Wait for the network to release any buffers |
Oleg Drokin | dadfcda | 2016-02-24 22:00:38 -0500 | [diff] [blame] | 2660 | * it's currently filling |
| 2661 | */ |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 2662 | spin_lock(&svcpt->scp_lock); |
| 2663 | while (svcpt->scp_nrqbds_posted != 0) { |
| 2664 | spin_unlock(&svcpt->scp_lock); |
| 2665 | /* Network access will complete in finite time but |
| 2666 | * the HUGE timeout lets us CWARN for visibility |
Oleg Drokin | dadfcda | 2016-02-24 22:00:38 -0500 | [diff] [blame] | 2667 | * of sluggish LNDs |
| 2668 | */ |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 2669 | lwi = LWI_TIMEOUT_INTERVAL( |
| 2670 | cfs_time_seconds(LONG_UNLINK), |
| 2671 | cfs_time_seconds(1), NULL, NULL); |
| 2672 | rc = l_wait_event(svcpt->scp_waitq, |
| 2673 | svcpt->scp_nrqbds_posted == 0, &lwi); |
| 2674 | if (rc == -ETIMEDOUT) { |
Joe Perches | 2d00bd1 | 2014-11-23 11:28:50 -0800 | [diff] [blame] | 2675 | CWARN("Service %s waiting for request buffers\n", |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 2676 | svcpt->scp_service->srv_name); |
| 2677 | } |
| 2678 | spin_lock(&svcpt->scp_lock); |
| 2679 | } |
| 2680 | spin_unlock(&svcpt->scp_lock); |
| 2681 | } |
| 2682 | } |
| 2683 | |
| 2684 | static void |
| 2685 | ptlrpc_service_purge_all(struct ptlrpc_service *svc) |
| 2686 | { |
Chris Hanna | d0bfef3 | 2015-06-03 10:28:26 -0400 | [diff] [blame] | 2687 | struct ptlrpc_service_part *svcpt; |
| 2688 | struct ptlrpc_request_buffer_desc *rqbd; |
| 2689 | struct ptlrpc_request *req; |
| 2690 | struct ptlrpc_reply_state *rs; |
| 2691 | int i; |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 2692 | |
| 2693 | ptlrpc_service_for_each_part(svcpt, i, svc) { |
Oleg Drokin | 8b38208 | 2016-02-16 00:46:58 -0500 | [diff] [blame] | 2694 | if (!svcpt->scp_service) |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 2695 | break; |
| 2696 | |
| 2697 | spin_lock(&svcpt->scp_rep_lock); |
| 2698 | while (!list_empty(&svcpt->scp_rep_active)) { |
| 2699 | rs = list_entry(svcpt->scp_rep_active.next, |
Oleg Drokin | 30c0aa3 | 2016-02-26 01:50:02 -0500 | [diff] [blame] | 2700 | struct ptlrpc_reply_state, rs_list); |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 2701 | spin_lock(&rs->rs_lock); |
| 2702 | ptlrpc_schedule_difficult_reply(rs); |
| 2703 | spin_unlock(&rs->rs_lock); |
| 2704 | } |
| 2705 | spin_unlock(&svcpt->scp_rep_lock); |
| 2706 | |
| 2707 | /* purge the request queue. NB No new replies (rqbds |
| 2708 | * all unlinked) and no service threads, so I'm the only |
Oleg Drokin | dadfcda | 2016-02-24 22:00:38 -0500 | [diff] [blame] | 2709 | * thread noodling the request queue now |
| 2710 | */ |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 2711 | while (!list_empty(&svcpt->scp_req_incoming)) { |
| 2712 | req = list_entry(svcpt->scp_req_incoming.next, |
Oleg Drokin | 30c0aa3 | 2016-02-26 01:50:02 -0500 | [diff] [blame] | 2713 | struct ptlrpc_request, rq_list); |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 2714 | |
| 2715 | list_del(&req->rq_list); |
| 2716 | svcpt->scp_nreqs_incoming--; |
| 2717 | ptlrpc_server_finish_request(svcpt, req); |
| 2718 | } |
| 2719 | |
| 2720 | while (ptlrpc_server_request_pending(svcpt, true)) { |
| 2721 | req = ptlrpc_server_request_get(svcpt, true); |
| 2722 | ptlrpc_server_finish_active_request(svcpt, req); |
| 2723 | } |
| 2724 | |
| 2725 | LASSERT(list_empty(&svcpt->scp_rqbd_posted)); |
| 2726 | LASSERT(svcpt->scp_nreqs_incoming == 0); |
| 2727 | LASSERT(svcpt->scp_nreqs_active == 0); |
| 2728 | /* history should have been culled by |
Oleg Drokin | dadfcda | 2016-02-24 22:00:38 -0500 | [diff] [blame] | 2729 | * ptlrpc_server_finish_request |
| 2730 | */ |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 2731 | LASSERT(svcpt->scp_hist_nrqbds == 0); |
| 2732 | |
| 2733 | /* Now free all the request buffers since nothing |
Oleg Drokin | dadfcda | 2016-02-24 22:00:38 -0500 | [diff] [blame] | 2734 | * references them any more... |
| 2735 | */ |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 2736 | |
| 2737 | while (!list_empty(&svcpt->scp_rqbd_idle)) { |
| 2738 | rqbd = list_entry(svcpt->scp_rqbd_idle.next, |
Oleg Drokin | 30c0aa3 | 2016-02-26 01:50:02 -0500 | [diff] [blame] | 2739 | struct ptlrpc_request_buffer_desc, |
| 2740 | rqbd_list); |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 2741 | ptlrpc_free_rqbd(rqbd); |
| 2742 | } |
| 2743 | ptlrpc_wait_replies(svcpt); |
| 2744 | |
| 2745 | while (!list_empty(&svcpt->scp_rep_idle)) { |
| 2746 | rs = list_entry(svcpt->scp_rep_idle.next, |
Oleg Drokin | 30c0aa3 | 2016-02-26 01:50:02 -0500 | [diff] [blame] | 2747 | struct ptlrpc_reply_state, |
| 2748 | rs_list); |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 2749 | list_del(&rs->rs_list); |
Julia Lawall | ee0ec19 | 2015-06-11 14:02:58 +0200 | [diff] [blame] | 2750 | kvfree(rs); |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 2751 | } |
| 2752 | } |
| 2753 | } |
| 2754 | |
| 2755 | static void |
| 2756 | ptlrpc_service_free(struct ptlrpc_service *svc) |
| 2757 | { |
Chris Hanna | d0bfef3 | 2015-06-03 10:28:26 -0400 | [diff] [blame] | 2758 | struct ptlrpc_service_part *svcpt; |
| 2759 | struct ptlrpc_at_array *array; |
| 2760 | int i; |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 2761 | |
| 2762 | ptlrpc_service_for_each_part(svcpt, i, svc) { |
Oleg Drokin | 8b38208 | 2016-02-16 00:46:58 -0500 | [diff] [blame] | 2763 | if (!svcpt->scp_service) |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 2764 | break; |
| 2765 | |
| 2766 | /* In case somebody rearmed this in the meantime */ |
Arnd Bergmann | 922da0c | 2015-09-27 16:45:33 -0400 | [diff] [blame] | 2767 | del_timer(&svcpt->scp_at_timer); |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 2768 | array = &svcpt->scp_at_array; |
| 2769 | |
Julia Lawall | 207e99c | 2015-05-01 21:37:47 +0200 | [diff] [blame] | 2770 | kfree(array->paa_reqs_array); |
| 2771 | array->paa_reqs_array = NULL; |
| 2772 | kfree(array->paa_reqs_count); |
| 2773 | array->paa_reqs_count = NULL; |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 2774 | } |
| 2775 | |
| 2776 | ptlrpc_service_for_each_part(svcpt, i, svc) |
Julia Lawall | 9ae1059 | 2015-05-01 17:51:12 +0200 | [diff] [blame] | 2777 | kfree(svcpt); |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 2778 | |
Oleg Drokin | 8b38208 | 2016-02-16 00:46:58 -0500 | [diff] [blame] | 2779 | if (svc->srv_cpts) |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 2780 | cfs_expr_list_values_free(svc->srv_cpts, svc->srv_ncpts); |
| 2781 | |
Julia Lawall | 9ae1059 | 2015-05-01 17:51:12 +0200 | [diff] [blame] | 2782 | kfree(svc); |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 2783 | } |
| 2784 | |
| 2785 | int ptlrpc_unregister_service(struct ptlrpc_service *service) |
| 2786 | { |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 2787 | CDEBUG(D_NET, "%s: tearing down\n", service->srv_name); |
| 2788 | |
| 2789 | service->srv_is_stopping = 1; |
| 2790 | |
| 2791 | mutex_lock(&ptlrpc_all_services_mutex); |
| 2792 | list_del_init(&service->srv_list); |
| 2793 | mutex_unlock(&ptlrpc_all_services_mutex); |
| 2794 | |
| 2795 | ptlrpc_service_del_atimer(service); |
| 2796 | ptlrpc_stop_all_threads(service); |
| 2797 | |
| 2798 | ptlrpc_service_unlink_rqbd(service); |
| 2799 | ptlrpc_service_purge_all(service); |
| 2800 | ptlrpc_service_nrs_cleanup(service); |
| 2801 | |
| 2802 | ptlrpc_lprocfs_unregister_service(service); |
Oleg Drokin | 328676f | 2015-05-21 15:32:08 -0400 | [diff] [blame] | 2803 | ptlrpc_sysfs_unregister_service(service); |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 2804 | |
| 2805 | ptlrpc_service_free(service); |
| 2806 | |
Greg Kroah-Hartman | 0a3bdb0 | 2013-08-03 10:35:28 +0800 | [diff] [blame] | 2807 | return 0; |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 2808 | } |
| 2809 | EXPORT_SYMBOL(ptlrpc_unregister_service); |