David Garcia Quintas | 4bc3463 | 2015-10-07 16:12:35 -0700 | [diff] [blame] | 1 | /* |
| 2 | * |
Jan Tattermusch | 7897ae9 | 2017-06-07 22:57:36 +0200 | [diff] [blame] | 3 | * Copyright 2015 gRPC authors. |
David Garcia Quintas | 4bc3463 | 2015-10-07 16:12:35 -0700 | [diff] [blame] | 4 | * |
Jan Tattermusch | 7897ae9 | 2017-06-07 22:57:36 +0200 | [diff] [blame] | 5 | * Licensed under the Apache License, Version 2.0 (the "License"); |
| 6 | * you may not use this file except in compliance with the License. |
| 7 | * You may obtain a copy of the License at |
David Garcia Quintas | 4bc3463 | 2015-10-07 16:12:35 -0700 | [diff] [blame] | 8 | * |
Jan Tattermusch | 7897ae9 | 2017-06-07 22:57:36 +0200 | [diff] [blame] | 9 | * http://www.apache.org/licenses/LICENSE-2.0 |
David Garcia Quintas | 4bc3463 | 2015-10-07 16:12:35 -0700 | [diff] [blame] | 10 | * |
Jan Tattermusch | 7897ae9 | 2017-06-07 22:57:36 +0200 | [diff] [blame] | 11 | * Unless required by applicable law or agreed to in writing, software |
| 12 | * distributed under the License is distributed on an "AS IS" BASIS, |
| 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 14 | * See the License for the specific language governing permissions and |
| 15 | * limitations under the License. |
David Garcia Quintas | 4bc3463 | 2015-10-07 16:12:35 -0700 | [diff] [blame] | 16 | * |
| 17 | */ |
| 18 | |
Craig Tiller | 9533d04 | 2016-03-25 17:11:06 -0700 | [diff] [blame] | 19 | #include "src/core/lib/iomgr/executor.h" |
David Garcia Quintas | 4bc3463 | 2015-10-07 16:12:35 -0700 | [diff] [blame] | 20 | |
| 21 | #include <string.h> |
| 22 | |
| 23 | #include <grpc/support/alloc.h> |
Craig Tiller | 3e9f98e | 2017-05-12 13:17:47 -0700 | [diff] [blame] | 24 | #include <grpc/support/cpu.h> |
David Garcia Quintas | 4bc3463 | 2015-10-07 16:12:35 -0700 | [diff] [blame] | 25 | #include <grpc/support/log.h> |
| 26 | #include <grpc/support/sync.h> |
| 27 | #include <grpc/support/thd.h> |
Craig Tiller | 3e9f98e | 2017-05-12 13:17:47 -0700 | [diff] [blame] | 28 | #include <grpc/support/tls.h> |
| 29 | #include <grpc/support/useful.h> |
| 30 | |
Craig Tiller | 9533d04 | 2016-03-25 17:11:06 -0700 | [diff] [blame] | 31 | #include "src/core/lib/iomgr/exec_ctx.h" |
Craig Tiller | 3e9f98e | 2017-05-12 13:17:47 -0700 | [diff] [blame] | 32 | #include "src/core/lib/support/spinlock.h" |
David Garcia Quintas | 4bc3463 | 2015-10-07 16:12:35 -0700 | [diff] [blame] | 33 | |
Craig Tiller | 1ed3118 | 2017-05-24 16:42:35 -0700 | [diff] [blame] | 34 | #define MAX_DEPTH 2 |
Craig Tiller | 3e9f98e | 2017-05-12 13:17:47 -0700 | [diff] [blame] | 35 | |
| 36 | typedef struct { |
David Garcia Quintas | 4bc3463 | 2015-10-07 16:12:35 -0700 | [diff] [blame] | 37 | gpr_mu mu; |
Craig Tiller | 3e9f98e | 2017-05-12 13:17:47 -0700 | [diff] [blame] | 38 | gpr_cv cv; |
| 39 | grpc_closure_list elems; |
| 40 | size_t depth; |
| 41 | bool shutdown; |
| 42 | gpr_thd_id id; |
| 43 | } thread_state; |
David Garcia Quintas | 4bc3463 | 2015-10-07 16:12:35 -0700 | [diff] [blame] | 44 | |
Craig Tiller | 3e9f98e | 2017-05-12 13:17:47 -0700 | [diff] [blame] | 45 | static thread_state *g_thread_state; |
| 46 | static size_t g_max_threads; |
| 47 | static gpr_atm g_cur_threads; |
| 48 | static gpr_spinlock g_adding_thread_lock = GPR_SPINLOCK_STATIC_INITIALIZER; |
| 49 | |
| 50 | GPR_TLS_DECL(g_this_thread_state); |
| 51 | |
Craig Tiller | af723b0 | 2017-07-17 17:56:28 -0700 | [diff] [blame] | 52 | static grpc_tracer_flag executor_trace = |
| 53 | GRPC_TRACER_INITIALIZER(false, "executor"); |
| 54 | |
Craig Tiller | 3e9f98e | 2017-05-12 13:17:47 -0700 | [diff] [blame] | 55 | static void executor_thread(void *arg); |
David Garcia Quintas | 4bc3463 | 2015-10-07 16:12:35 -0700 | [diff] [blame] | 56 | |
Craig Tiller | 3e9f98e | 2017-05-12 13:17:47 -0700 | [diff] [blame] | 57 | static size_t run_closures(grpc_exec_ctx *exec_ctx, grpc_closure_list list) { |
| 58 | size_t n = 0; |
David Garcia Quintas | 4bc3463 | 2015-10-07 16:12:35 -0700 | [diff] [blame] | 59 | |
Craig Tiller | 3e9f98e | 2017-05-12 13:17:47 -0700 | [diff] [blame] | 60 | grpc_closure *c = list.head; |
Craig Tiller | 061ef74 | 2016-12-29 10:54:09 -0800 | [diff] [blame] | 61 | while (c != NULL) { |
| 62 | grpc_closure *next = c->next_data.next; |
| 63 | grpc_error *error = c->error_data.error; |
Craig Tiller | af723b0 | 2017-07-17 17:56:28 -0700 | [diff] [blame] | 64 | if (GRPC_TRACER_ON(executor_trace)) { |
| 65 | #ifndef NDEBUG |
| 66 | gpr_log(GPR_DEBUG, "EXECUTOR: run %p [created by %s:%d]", c, |
| 67 | c->file_created, c->line_created); |
| 68 | #else |
| 69 | gpr_log(GPR_DEBUG, "EXECUTOR: run %p", c); |
| 70 | #endif |
| 71 | } |
ncteisen | f8061e8 | 2017-06-09 10:44:42 -0700 | [diff] [blame] | 72 | #ifndef NDEBUG |
Craig Tiller | b9b01ce | 2017-05-12 13:47:10 -0700 | [diff] [blame] | 73 | c->scheduled = false; |
Mark D. Roth | 43f774e | 2017-04-04 16:35:37 -0700 | [diff] [blame] | 74 | #endif |
Craig Tiller | 0b09341 | 2017-01-03 09:49:07 -0800 | [diff] [blame] | 75 | c->cb(exec_ctx, c->cb_arg, error); |
Craig Tiller | 061ef74 | 2016-12-29 10:54:09 -0800 | [diff] [blame] | 76 | GRPC_ERROR_UNREF(error); |
| 77 | c = next; |
Craig Tiller | f21acdd | 2017-06-08 08:09:35 -0700 | [diff] [blame] | 78 | n++; |
Craig Tiller | 061ef74 | 2016-12-29 10:54:09 -0800 | [diff] [blame] | 79 | } |
Craig Tiller | 3e9f98e | 2017-05-12 13:17:47 -0700 | [diff] [blame] | 80 | |
| 81 | return n; |
| 82 | } |
| 83 | |
Craig Tiller | 5e56f00 | 2017-05-16 15:02:50 -0700 | [diff] [blame] | 84 | bool grpc_executor_is_threaded() { |
| 85 | return gpr_atm_no_barrier_load(&g_cur_threads) > 0; |
| 86 | } |
| 87 | |
| 88 | void grpc_executor_set_threading(grpc_exec_ctx *exec_ctx, bool threading) { |
| 89 | gpr_atm cur_threads = gpr_atm_no_barrier_load(&g_cur_threads); |
| 90 | if (threading) { |
| 91 | if (cur_threads > 0) return; |
| 92 | g_max_threads = GPR_MAX(1, 2 * gpr_cpu_num_cores()); |
| 93 | gpr_atm_no_barrier_store(&g_cur_threads, 1); |
| 94 | gpr_tls_init(&g_this_thread_state); |
| 95 | g_thread_state = gpr_zalloc(sizeof(thread_state) * g_max_threads); |
| 96 | for (size_t i = 0; i < g_max_threads; i++) { |
| 97 | gpr_mu_init(&g_thread_state[i].mu); |
| 98 | gpr_cv_init(&g_thread_state[i].cv); |
| 99 | g_thread_state[i].elems = (grpc_closure_list)GRPC_CLOSURE_LIST_INIT; |
| 100 | } |
| 101 | |
| 102 | gpr_thd_options opt = gpr_thd_options_default(); |
| 103 | gpr_thd_options_set_joinable(&opt); |
| 104 | gpr_thd_new(&g_thread_state[0].id, executor_thread, &g_thread_state[0], |
| 105 | &opt); |
| 106 | } else { |
| 107 | if (cur_threads == 0) return; |
| 108 | for (size_t i = 0; i < g_max_threads; i++) { |
| 109 | gpr_mu_lock(&g_thread_state[i].mu); |
| 110 | g_thread_state[i].shutdown = true; |
| 111 | gpr_cv_signal(&g_thread_state[i].cv); |
| 112 | gpr_mu_unlock(&g_thread_state[i].mu); |
| 113 | } |
Craig Tiller | f7c8c9f | 2017-05-17 15:22:05 -0700 | [diff] [blame] | 114 | /* ensure no thread is adding a new thread... once this is past, then |
| 115 | no thread will try to add a new one either (since shutdown is true) */ |
| 116 | gpr_spinlock_lock(&g_adding_thread_lock); |
| 117 | gpr_spinlock_unlock(&g_adding_thread_lock); |
Craig Tiller | 5e56f00 | 2017-05-16 15:02:50 -0700 | [diff] [blame] | 118 | for (gpr_atm i = 0; i < g_cur_threads; i++) { |
| 119 | gpr_thd_join(g_thread_state[i].id); |
| 120 | } |
| 121 | gpr_atm_no_barrier_store(&g_cur_threads, 0); |
| 122 | for (size_t i = 0; i < g_max_threads; i++) { |
| 123 | gpr_mu_destroy(&g_thread_state[i].mu); |
| 124 | gpr_cv_destroy(&g_thread_state[i].cv); |
| 125 | run_closures(exec_ctx, g_thread_state[i].elems); |
| 126 | } |
| 127 | gpr_free(g_thread_state); |
| 128 | gpr_tls_destroy(&g_this_thread_state); |
David Garcia Quintas | 4bc3463 | 2015-10-07 16:12:35 -0700 | [diff] [blame] | 129 | } |
Craig Tiller | 5e56f00 | 2017-05-16 15:02:50 -0700 | [diff] [blame] | 130 | } |
| 131 | |
| 132 | void grpc_executor_init(grpc_exec_ctx *exec_ctx) { |
Craig Tiller | af723b0 | 2017-07-17 17:56:28 -0700 | [diff] [blame] | 133 | grpc_register_tracer(&executor_trace); |
Craig Tiller | b9b01ce | 2017-05-12 13:47:10 -0700 | [diff] [blame] | 134 | gpr_atm_no_barrier_store(&g_cur_threads, 0); |
Craig Tiller | 5e56f00 | 2017-05-16 15:02:50 -0700 | [diff] [blame] | 135 | grpc_executor_set_threading(exec_ctx, true); |
| 136 | } |
| 137 | |
| 138 | void grpc_executor_shutdown(grpc_exec_ctx *exec_ctx) { |
| 139 | grpc_executor_set_threading(exec_ctx, false); |
Craig Tiller | 3e9f98e | 2017-05-12 13:17:47 -0700 | [diff] [blame] | 140 | } |
| 141 | |
| 142 | static void executor_thread(void *arg) { |
| 143 | thread_state *ts = arg; |
| 144 | gpr_tls_set(&g_this_thread_state, (intptr_t)ts); |
| 145 | |
Craig Tiller | 8996208 | 2017-05-12 14:30:42 -0700 | [diff] [blame] | 146 | grpc_exec_ctx exec_ctx = |
| 147 | GRPC_EXEC_CTX_INITIALIZER(0, grpc_never_ready_to_finish, NULL); |
| 148 | |
Craig Tiller | 3e9f98e | 2017-05-12 13:17:47 -0700 | [diff] [blame] | 149 | size_t subtract_depth = 0; |
| 150 | for (;;) { |
Craig Tiller | af723b0 | 2017-07-17 17:56:28 -0700 | [diff] [blame] | 151 | if (GRPC_TRACER_ON(executor_trace)) { |
| 152 | gpr_log(GPR_DEBUG, |
| 153 | "EXECUTOR[%" PRIdPTR "]: step (sub_depth=%" PRIdPTR ")", |
| 154 | ts - g_thread_state, subtract_depth); |
| 155 | } |
Craig Tiller | 3e9f98e | 2017-05-12 13:17:47 -0700 | [diff] [blame] | 156 | gpr_mu_lock(&ts->mu); |
| 157 | ts->depth -= subtract_depth; |
| 158 | while (grpc_closure_list_empty(ts->elems) && !ts->shutdown) { |
| 159 | gpr_cv_wait(&ts->cv, &ts->mu, gpr_inf_future(GPR_CLOCK_REALTIME)); |
| 160 | } |
| 161 | if (ts->shutdown) { |
Craig Tiller | af723b0 | 2017-07-17 17:56:28 -0700 | [diff] [blame] | 162 | if (GRPC_TRACER_ON(executor_trace)) { |
| 163 | gpr_log(GPR_DEBUG, "EXECUTOR[%" PRIdPTR "]: shutdown", |
| 164 | ts - g_thread_state); |
| 165 | } |
Craig Tiller | 3e9f98e | 2017-05-12 13:17:47 -0700 | [diff] [blame] | 166 | gpr_mu_unlock(&ts->mu); |
| 167 | break; |
| 168 | } |
| 169 | grpc_closure_list exec = ts->elems; |
| 170 | ts->elems = (grpc_closure_list)GRPC_CLOSURE_LIST_INIT; |
| 171 | gpr_mu_unlock(&ts->mu); |
Craig Tiller | af723b0 | 2017-07-17 17:56:28 -0700 | [diff] [blame] | 172 | if (GRPC_TRACER_ON(executor_trace)) { |
| 173 | gpr_log(GPR_DEBUG, "EXECUTOR[%" PRIdPTR "]: execute", |
| 174 | ts - g_thread_state); |
| 175 | } |
Craig Tiller | 3e9f98e | 2017-05-12 13:17:47 -0700 | [diff] [blame] | 176 | |
Craig Tiller | 3e9f98e | 2017-05-12 13:17:47 -0700 | [diff] [blame] | 177 | subtract_depth = run_closures(&exec_ctx, exec); |
Craig Tiller | 8996208 | 2017-05-12 14:30:42 -0700 | [diff] [blame] | 178 | grpc_exec_ctx_flush(&exec_ctx); |
Craig Tiller | 3e9f98e | 2017-05-12 13:17:47 -0700 | [diff] [blame] | 179 | } |
Craig Tiller | 8996208 | 2017-05-12 14:30:42 -0700 | [diff] [blame] | 180 | grpc_exec_ctx_finish(&exec_ctx); |
Craig Tiller | 3e9f98e | 2017-05-12 13:17:47 -0700 | [diff] [blame] | 181 | } |
| 182 | |
| 183 | static void executor_push(grpc_exec_ctx *exec_ctx, grpc_closure *closure, |
Craig Tiller | 7a82afd | 2017-07-18 09:40:40 -0700 | [diff] [blame^] | 184 | grpc_error *error, bool is_short) { |
Craig Tiller | 61f96c1 | 2017-05-12 13:36:39 -0700 | [diff] [blame] | 185 | size_t cur_thread_count = (size_t)gpr_atm_no_barrier_load(&g_cur_threads); |
Craig Tiller | b9b01ce | 2017-05-12 13:47:10 -0700 | [diff] [blame] | 186 | if (cur_thread_count == 0) { |
Craig Tiller | af723b0 | 2017-07-17 17:56:28 -0700 | [diff] [blame] | 187 | if (GRPC_TRACER_ON(executor_trace)) { |
| 188 | gpr_log(GPR_DEBUG, "EXECUTOR: schedule %p inline", closure); |
| 189 | } |
Craig Tiller | b9b01ce | 2017-05-12 13:47:10 -0700 | [diff] [blame] | 190 | grpc_closure_list_append(&exec_ctx->closure_list, closure, error); |
| 191 | return; |
| 192 | } |
| 193 | thread_state *ts = (thread_state *)gpr_tls_get(&g_this_thread_state); |
Craig Tiller | 3e9f98e | 2017-05-12 13:17:47 -0700 | [diff] [blame] | 194 | if (ts == NULL) { |
Craig Tiller | 61f96c1 | 2017-05-12 13:36:39 -0700 | [diff] [blame] | 195 | ts = &g_thread_state[GPR_HASH_POINTER(exec_ctx, cur_thread_count)]; |
Craig Tiller | 3e9f98e | 2017-05-12 13:17:47 -0700 | [diff] [blame] | 196 | } |
Craig Tiller | af723b0 | 2017-07-17 17:56:28 -0700 | [diff] [blame] | 197 | if (GRPC_TRACER_ON(executor_trace)) { |
| 198 | gpr_log(GPR_DEBUG, "EXECUTOR: schedule %p to thread %" PRIdPTR, closure, |
| 199 | ts - g_thread_state); |
| 200 | } |
Craig Tiller | 3e9f98e | 2017-05-12 13:17:47 -0700 | [diff] [blame] | 201 | gpr_mu_lock(&ts->mu); |
Craig Tiller | 8996208 | 2017-05-12 14:30:42 -0700 | [diff] [blame] | 202 | if (grpc_closure_list_empty(ts->elems)) { |
| 203 | gpr_cv_signal(&ts->cv); |
| 204 | } |
Craig Tiller | 3e9f98e | 2017-05-12 13:17:47 -0700 | [diff] [blame] | 205 | grpc_closure_list_append(&ts->elems, closure, error); |
| 206 | ts->depth++; |
Craig Tiller | f7c8c9f | 2017-05-17 15:22:05 -0700 | [diff] [blame] | 207 | bool try_new_thread = ts->depth > MAX_DEPTH && |
| 208 | cur_thread_count < g_max_threads && !ts->shutdown; |
Craig Tiller | 747216f | 2017-06-08 08:08:11 -0700 | [diff] [blame] | 209 | gpr_mu_unlock(&ts->mu); |
Craig Tiller | 3e9f98e | 2017-05-12 13:17:47 -0700 | [diff] [blame] | 210 | if (try_new_thread && gpr_spinlock_trylock(&g_adding_thread_lock)) { |
Craig Tiller | 61f96c1 | 2017-05-12 13:36:39 -0700 | [diff] [blame] | 211 | cur_thread_count = (size_t)gpr_atm_no_barrier_load(&g_cur_threads); |
Craig Tiller | 3e9f98e | 2017-05-12 13:17:47 -0700 | [diff] [blame] | 212 | if (cur_thread_count < g_max_threads) { |
| 213 | gpr_atm_no_barrier_store(&g_cur_threads, cur_thread_count + 1); |
| 214 | |
| 215 | gpr_thd_options opt = gpr_thd_options_default(); |
| 216 | gpr_thd_options_set_joinable(&opt); |
| 217 | gpr_thd_new(&g_thread_state[cur_thread_count].id, executor_thread, |
| 218 | &g_thread_state[cur_thread_count], &opt); |
| 219 | } |
| 220 | gpr_spinlock_unlock(&g_adding_thread_lock); |
| 221 | } |
David Garcia Quintas | 4bc3463 | 2015-10-07 16:12:35 -0700 | [diff] [blame] | 222 | } |
Craig Tiller | 91031da | 2016-12-28 15:44:25 -0800 | [diff] [blame] | 223 | |
Craig Tiller | 7a82afd | 2017-07-18 09:40:40 -0700 | [diff] [blame^] | 224 | static void executor_push_short(grpc_exec_ctx *exec_ctx, grpc_closure *closure, |
| 225 | grpc_error *error) { |
| 226 | executor_push(exec_ctx, closure, error, true); |
| 227 | } |
| 228 | |
| 229 | static void executor_push_long(grpc_exec_ctx *exec_ctx, grpc_closure *closure, |
| 230 | grpc_error *error) { |
| 231 | executor_push(exec_ctx, closure, error, false); |
| 232 | } |
| 233 | |
| 234 | static const grpc_closure_scheduler_vtable executor_vtable_short = { |
| 235 | executor_push_short, executor_push_short, "executor"}; |
| 236 | static grpc_closure_scheduler executor_scheduler_short = { |
| 237 | &executor_vtable_short}; |
| 238 | |
| 239 | static const grpc_closure_scheduler_vtable executor_vtable_long = { |
| 240 | executor_push_long, executor_push_long, "executor"}; |
| 241 | static grpc_closure_scheduler executor_scheduler_long = {&executor_vtable_long}; |
| 242 | |
| 243 | grpc_closure_scheduler *grpc_executor_scheduler( |
| 244 | grpc_executor_job_length length) { |
| 245 | return length == GRPC_EXECUTOR_SHORT ? &executor_scheduler_short |
| 246 | : &executor_scheduler_long; |
| 247 | } |