David Garcia Quintas | 4bc3463 | 2015-10-07 16:12:35 -0700 | [diff] [blame] | 1 | /* |
| 2 | * |
Jan Tattermusch | 7897ae9 | 2017-06-07 22:57:36 +0200 | [diff] [blame] | 3 | * Copyright 2015 gRPC authors. |
David Garcia Quintas | 4bc3463 | 2015-10-07 16:12:35 -0700 | [diff] [blame] | 4 | * |
Jan Tattermusch | 7897ae9 | 2017-06-07 22:57:36 +0200 | [diff] [blame] | 5 | * Licensed under the Apache License, Version 2.0 (the "License"); |
| 6 | * you may not use this file except in compliance with the License. |
| 7 | * You may obtain a copy of the License at |
David Garcia Quintas | 4bc3463 | 2015-10-07 16:12:35 -0700 | [diff] [blame] | 8 | * |
Jan Tattermusch | 7897ae9 | 2017-06-07 22:57:36 +0200 | [diff] [blame] | 9 | * http://www.apache.org/licenses/LICENSE-2.0 |
David Garcia Quintas | 4bc3463 | 2015-10-07 16:12:35 -0700 | [diff] [blame] | 10 | * |
Jan Tattermusch | 7897ae9 | 2017-06-07 22:57:36 +0200 | [diff] [blame] | 11 | * Unless required by applicable law or agreed to in writing, software |
| 12 | * distributed under the License is distributed on an "AS IS" BASIS, |
| 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 14 | * See the License for the specific language governing permissions and |
| 15 | * limitations under the License. |
David Garcia Quintas | 4bc3463 | 2015-10-07 16:12:35 -0700 | [diff] [blame] | 16 | * |
| 17 | */ |
| 18 | |
Craig Tiller | 9533d04 | 2016-03-25 17:11:06 -0700 | [diff] [blame] | 19 | #include "src/core/lib/iomgr/executor.h" |
David Garcia Quintas | 4bc3463 | 2015-10-07 16:12:35 -0700 | [diff] [blame] | 20 | |
| 21 | #include <string.h> |
| 22 | |
| 23 | #include <grpc/support/alloc.h> |
Craig Tiller | 3e9f98e | 2017-05-12 13:17:47 -0700 | [diff] [blame] | 24 | #include <grpc/support/cpu.h> |
David Garcia Quintas | 4bc3463 | 2015-10-07 16:12:35 -0700 | [diff] [blame] | 25 | #include <grpc/support/log.h> |
| 26 | #include <grpc/support/sync.h> |
| 27 | #include <grpc/support/thd.h> |
Craig Tiller | 3e9f98e | 2017-05-12 13:17:47 -0700 | [diff] [blame] | 28 | |
Craig Tiller | 57bb9a9 | 2017-08-31 16:44:15 -0700 | [diff] [blame] | 29 | #include "src/core/lib/debug/stats.h" |
Mark D. Roth | dbdf495 | 2018-01-18 11:21:12 -0800 | [diff] [blame] | 30 | #include "src/core/lib/gpr/spinlock.h" |
Vijay Pai | b6cf123 | 2018-01-25 21:02:26 -0800 | [diff] [blame] | 31 | #include "src/core/lib/gpr/tls.h" |
Vijay Pai | d4d0a30 | 2018-01-25 13:24:03 -0800 | [diff] [blame] | 32 | #include "src/core/lib/gpr/useful.h" |
Craig Tiller | 9533d04 | 2016-03-25 17:11:06 -0700 | [diff] [blame] | 33 | #include "src/core/lib/iomgr/exec_ctx.h" |
David Garcia Quintas | 4bc3463 | 2015-10-07 16:12:35 -0700 | [diff] [blame] | 34 | |
Craig Tiller | 2477cf3 | 2017-09-26 12:20:35 -0700 | [diff] [blame] | 35 | #define MAX_DEPTH 2 |
| 36 | |
Craig Tiller | 3e9f98e | 2017-05-12 13:17:47 -0700 | [diff] [blame] | 37 | typedef struct { |
David Garcia Quintas | 4bc3463 | 2015-10-07 16:12:35 -0700 | [diff] [blame] | 38 | gpr_mu mu; |
Craig Tiller | 3e9f98e | 2017-05-12 13:17:47 -0700 | [diff] [blame] | 39 | gpr_cv cv; |
| 40 | grpc_closure_list elems; |
Craig Tiller | 2477cf3 | 2017-09-26 12:20:35 -0700 | [diff] [blame] | 41 | size_t depth; |
Craig Tiller | 3e9f98e | 2017-05-12 13:17:47 -0700 | [diff] [blame] | 42 | bool shutdown; |
Craig Tiller | c2fb83e | 2017-07-18 12:38:25 -0700 | [diff] [blame] | 43 | bool queued_long_job; |
Craig Tiller | 3e9f98e | 2017-05-12 13:17:47 -0700 | [diff] [blame] | 44 | gpr_thd_id id; |
| 45 | } thread_state; |
David Garcia Quintas | 4bc3463 | 2015-10-07 16:12:35 -0700 | [diff] [blame] | 46 | |
Craig Tiller | baa14a9 | 2017-11-03 09:09:36 -0700 | [diff] [blame] | 47 | static thread_state* g_thread_state; |
Craig Tiller | 3e9f98e | 2017-05-12 13:17:47 -0700 | [diff] [blame] | 48 | static size_t g_max_threads; |
| 49 | static gpr_atm g_cur_threads; |
| 50 | static gpr_spinlock g_adding_thread_lock = GPR_SPINLOCK_STATIC_INITIALIZER; |
| 51 | |
| 52 | GPR_TLS_DECL(g_this_thread_state); |
| 53 | |
ncteisen | 72afb76 | 2017-11-10 12:23:12 -0800 | [diff] [blame] | 54 | grpc_core::TraceFlag executor_trace(false, "executor"); |
Craig Tiller | af723b0 | 2017-07-17 17:56:28 -0700 | [diff] [blame] | 55 | |
Craig Tiller | baa14a9 | 2017-11-03 09:09:36 -0700 | [diff] [blame] | 56 | static void executor_thread(void* arg); |
David Garcia Quintas | 4bc3463 | 2015-10-07 16:12:35 -0700 | [diff] [blame] | 57 | |
Yash Tibrewal | 8cf1470 | 2017-12-06 09:47:54 -0800 | [diff] [blame] | 58 | static size_t run_closures(grpc_closure_list list) { |
Craig Tiller | 2477cf3 | 2017-09-26 12:20:35 -0700 | [diff] [blame] | 59 | size_t n = 0; |
David Garcia Quintas | 4bc3463 | 2015-10-07 16:12:35 -0700 | [diff] [blame] | 60 | |
Craig Tiller | baa14a9 | 2017-11-03 09:09:36 -0700 | [diff] [blame] | 61 | grpc_closure* c = list.head; |
Craig Tiller | 4782d92 | 2017-11-10 09:53:21 -0800 | [diff] [blame] | 62 | while (c != nullptr) { |
Craig Tiller | baa14a9 | 2017-11-03 09:09:36 -0700 | [diff] [blame] | 63 | grpc_closure* next = c->next_data.next; |
| 64 | grpc_error* error = c->error_data.error; |
Craig Tiller | 6014e8a | 2017-10-16 13:50:29 -0700 | [diff] [blame] | 65 | if (executor_trace.enabled()) { |
Craig Tiller | af723b0 | 2017-07-17 17:56:28 -0700 | [diff] [blame] | 66 | #ifndef NDEBUG |
Craig Tiller | 2477cf3 | 2017-09-26 12:20:35 -0700 | [diff] [blame] | 67 | gpr_log(GPR_DEBUG, "EXECUTOR: run %p [created by %s:%d]", c, |
| 68 | c->file_created, c->line_created); |
Craig Tiller | af723b0 | 2017-07-17 17:56:28 -0700 | [diff] [blame] | 69 | #else |
Craig Tiller | 2477cf3 | 2017-09-26 12:20:35 -0700 | [diff] [blame] | 70 | gpr_log(GPR_DEBUG, "EXECUTOR: run %p", c); |
Craig Tiller | af723b0 | 2017-07-17 17:56:28 -0700 | [diff] [blame] | 71 | #endif |
Craig Tiller | fb0262b | 2017-09-13 15:19:19 -0700 | [diff] [blame] | 72 | } |
Craig Tiller | 2477cf3 | 2017-09-26 12:20:35 -0700 | [diff] [blame] | 73 | #ifndef NDEBUG |
| 74 | c->scheduled = false; |
| 75 | #endif |
Yash Tibrewal | 8cf1470 | 2017-12-06 09:47:54 -0800 | [diff] [blame] | 76 | c->cb(c->cb_arg, error); |
Craig Tiller | 2477cf3 | 2017-09-26 12:20:35 -0700 | [diff] [blame] | 77 | GRPC_ERROR_UNREF(error); |
| 78 | c = next; |
| 79 | n++; |
Yash Tibrewal | 8cf1470 | 2017-12-06 09:47:54 -0800 | [diff] [blame] | 80 | grpc_core::ExecCtx::Get()->Flush(); |
Craig Tiller | 061ef74 | 2016-12-29 10:54:09 -0800 | [diff] [blame] | 81 | } |
Craig Tiller | 3e9f98e | 2017-05-12 13:17:47 -0700 | [diff] [blame] | 82 | |
Craig Tiller | 2477cf3 | 2017-09-26 12:20:35 -0700 | [diff] [blame] | 83 | return n; |
Craig Tiller | 3e9f98e | 2017-05-12 13:17:47 -0700 | [diff] [blame] | 84 | } |
| 85 | |
Craig Tiller | 5e56f00 | 2017-05-16 15:02:50 -0700 | [diff] [blame] | 86 | bool grpc_executor_is_threaded() { |
| 87 | return gpr_atm_no_barrier_load(&g_cur_threads) > 0; |
| 88 | } |
| 89 | |
Yash Tibrewal | 8cf1470 | 2017-12-06 09:47:54 -0800 | [diff] [blame] | 90 | void grpc_executor_set_threading(bool threading) { |
Craig Tiller | 5e56f00 | 2017-05-16 15:02:50 -0700 | [diff] [blame] | 91 | gpr_atm cur_threads = gpr_atm_no_barrier_load(&g_cur_threads); |
| 92 | if (threading) { |
| 93 | if (cur_threads > 0) return; |
| 94 | g_max_threads = GPR_MAX(1, 2 * gpr_cpu_num_cores()); |
| 95 | gpr_atm_no_barrier_store(&g_cur_threads, 1); |
| 96 | gpr_tls_init(&g_this_thread_state); |
Noah Eisen | 4d20a66 | 2018-02-09 09:34:04 -0800 | [diff] [blame^] | 97 | g_thread_state = static_cast<thread_state*>( |
| 98 | gpr_zalloc(sizeof(thread_state) * g_max_threads)); |
Craig Tiller | 5e56f00 | 2017-05-16 15:02:50 -0700 | [diff] [blame] | 99 | for (size_t i = 0; i < g_max_threads; i++) { |
| 100 | gpr_mu_init(&g_thread_state[i].mu); |
| 101 | gpr_cv_init(&g_thread_state[i].cv); |
Yash Tibrewal | 37fdb73 | 2017-09-25 16:45:02 -0700 | [diff] [blame] | 102 | g_thread_state[i].elems = GRPC_CLOSURE_LIST_INIT; |
Craig Tiller | 5e56f00 | 2017-05-16 15:02:50 -0700 | [diff] [blame] | 103 | } |
| 104 | |
| 105 | gpr_thd_options opt = gpr_thd_options_default(); |
| 106 | gpr_thd_options_set_joinable(&opt); |
Dave MacLachlan | da341bc | 2017-11-30 12:48:22 -0800 | [diff] [blame] | 107 | gpr_thd_new(&g_thread_state[0].id, "grpc_executor", executor_thread, |
Dave MacLachlan | af5c54d | 2017-11-29 16:25:10 -0800 | [diff] [blame] | 108 | &g_thread_state[0], &opt); |
Craig Tiller | 5e56f00 | 2017-05-16 15:02:50 -0700 | [diff] [blame] | 109 | } else { |
| 110 | if (cur_threads == 0) return; |
| 111 | for (size_t i = 0; i < g_max_threads; i++) { |
| 112 | gpr_mu_lock(&g_thread_state[i].mu); |
| 113 | g_thread_state[i].shutdown = true; |
| 114 | gpr_cv_signal(&g_thread_state[i].cv); |
| 115 | gpr_mu_unlock(&g_thread_state[i].mu); |
| 116 | } |
Craig Tiller | f7c8c9f | 2017-05-17 15:22:05 -0700 | [diff] [blame] | 117 | /* ensure no thread is adding a new thread... once this is past, then |
| 118 | no thread will try to add a new one either (since shutdown is true) */ |
| 119 | gpr_spinlock_lock(&g_adding_thread_lock); |
| 120 | gpr_spinlock_unlock(&g_adding_thread_lock); |
Craig Tiller | 5e56f00 | 2017-05-16 15:02:50 -0700 | [diff] [blame] | 121 | for (gpr_atm i = 0; i < g_cur_threads; i++) { |
| 122 | gpr_thd_join(g_thread_state[i].id); |
| 123 | } |
| 124 | gpr_atm_no_barrier_store(&g_cur_threads, 0); |
| 125 | for (size_t i = 0; i < g_max_threads; i++) { |
| 126 | gpr_mu_destroy(&g_thread_state[i].mu); |
| 127 | gpr_cv_destroy(&g_thread_state[i].cv); |
Yash Tibrewal | 8cf1470 | 2017-12-06 09:47:54 -0800 | [diff] [blame] | 128 | run_closures(g_thread_state[i].elems); |
Craig Tiller | 5e56f00 | 2017-05-16 15:02:50 -0700 | [diff] [blame] | 129 | } |
| 130 | gpr_free(g_thread_state); |
| 131 | gpr_tls_destroy(&g_this_thread_state); |
David Garcia Quintas | 4bc3463 | 2015-10-07 16:12:35 -0700 | [diff] [blame] | 132 | } |
Craig Tiller | 5e56f00 | 2017-05-16 15:02:50 -0700 | [diff] [blame] | 133 | } |
| 134 | |
Yash Tibrewal | 8cf1470 | 2017-12-06 09:47:54 -0800 | [diff] [blame] | 135 | void grpc_executor_init() { |
Craig Tiller | b9b01ce | 2017-05-12 13:47:10 -0700 | [diff] [blame] | 136 | gpr_atm_no_barrier_store(&g_cur_threads, 0); |
Yash Tibrewal | 8cf1470 | 2017-12-06 09:47:54 -0800 | [diff] [blame] | 137 | grpc_executor_set_threading(true); |
Craig Tiller | 5e56f00 | 2017-05-16 15:02:50 -0700 | [diff] [blame] | 138 | } |
| 139 | |
Yash Tibrewal | 8cf1470 | 2017-12-06 09:47:54 -0800 | [diff] [blame] | 140 | void grpc_executor_shutdown() { grpc_executor_set_threading(false); } |
Craig Tiller | 3e9f98e | 2017-05-12 13:17:47 -0700 | [diff] [blame] | 141 | |
Craig Tiller | baa14a9 | 2017-11-03 09:09:36 -0700 | [diff] [blame] | 142 | static void executor_thread(void* arg) { |
Noah Eisen | be82e64 | 2018-02-09 09:16:55 -0800 | [diff] [blame] | 143 | thread_state* ts = static_cast<thread_state*>(arg); |
Craig Tiller | 3e9f98e | 2017-05-12 13:17:47 -0700 | [diff] [blame] | 144 | gpr_tls_set(&g_this_thread_state, (intptr_t)ts); |
| 145 | |
Yash Tibrewal | 8cf1470 | 2017-12-06 09:47:54 -0800 | [diff] [blame] | 146 | grpc_core::ExecCtx exec_ctx(0); |
Craig Tiller | 8996208 | 2017-05-12 14:30:42 -0700 | [diff] [blame] | 147 | |
Craig Tiller | 2477cf3 | 2017-09-26 12:20:35 -0700 | [diff] [blame] | 148 | size_t subtract_depth = 0; |
Craig Tiller | 3e9f98e | 2017-05-12 13:17:47 -0700 | [diff] [blame] | 149 | for (;;) { |
Craig Tiller | 6014e8a | 2017-10-16 13:50:29 -0700 | [diff] [blame] | 150 | if (executor_trace.enabled()) { |
Craig Tiller | 2477cf3 | 2017-09-26 12:20:35 -0700 | [diff] [blame] | 151 | gpr_log(GPR_DEBUG, "EXECUTOR[%d]: step (sub_depth=%" PRIdPTR ")", |
Noah Eisen | be82e64 | 2018-02-09 09:16:55 -0800 | [diff] [blame] | 152 | static_cast<int>(ts - g_thread_state), subtract_depth); |
Craig Tiller | af723b0 | 2017-07-17 17:56:28 -0700 | [diff] [blame] | 153 | } |
Craig Tiller | 3e9f98e | 2017-05-12 13:17:47 -0700 | [diff] [blame] | 154 | gpr_mu_lock(&ts->mu); |
Craig Tiller | 2477cf3 | 2017-09-26 12:20:35 -0700 | [diff] [blame] | 155 | ts->depth -= subtract_depth; |
Craig Tiller | 3e9f98e | 2017-05-12 13:17:47 -0700 | [diff] [blame] | 156 | while (grpc_closure_list_empty(ts->elems) && !ts->shutdown) { |
Craig Tiller | 1ab56d8 | 2017-07-19 09:55:57 -0700 | [diff] [blame] | 157 | ts->queued_long_job = false; |
Sree Kuchibhotla | 54961bb | 2017-12-04 12:50:27 -0800 | [diff] [blame] | 158 | gpr_cv_wait(&ts->cv, &ts->mu, gpr_inf_future(GPR_CLOCK_MONOTONIC)); |
Craig Tiller | 3e9f98e | 2017-05-12 13:17:47 -0700 | [diff] [blame] | 159 | } |
| 160 | if (ts->shutdown) { |
Craig Tiller | 6014e8a | 2017-10-16 13:50:29 -0700 | [diff] [blame] | 161 | if (executor_trace.enabled()) { |
Craig Tiller | 7d07994 | 2017-09-05 12:46:48 -0700 | [diff] [blame] | 162 | gpr_log(GPR_DEBUG, "EXECUTOR[%d]: shutdown", |
Noah Eisen | be82e64 | 2018-02-09 09:16:55 -0800 | [diff] [blame] | 163 | static_cast<int>(ts - g_thread_state)); |
Craig Tiller | af723b0 | 2017-07-17 17:56:28 -0700 | [diff] [blame] | 164 | } |
Craig Tiller | 3e9f98e | 2017-05-12 13:17:47 -0700 | [diff] [blame] | 165 | gpr_mu_unlock(&ts->mu); |
| 166 | break; |
| 167 | } |
Yash Tibrewal | 8cf1470 | 2017-12-06 09:47:54 -0800 | [diff] [blame] | 168 | GRPC_STATS_INC_EXECUTOR_QUEUE_DRAINED(); |
Craig Tiller | 2477cf3 | 2017-09-26 12:20:35 -0700 | [diff] [blame] | 169 | grpc_closure_list exec = ts->elems; |
Yash Tibrewal | 37fdb73 | 2017-09-25 16:45:02 -0700 | [diff] [blame] | 170 | ts->elems = GRPC_CLOSURE_LIST_INIT; |
Craig Tiller | 3e9f98e | 2017-05-12 13:17:47 -0700 | [diff] [blame] | 171 | gpr_mu_unlock(&ts->mu); |
Craig Tiller | 6014e8a | 2017-10-16 13:50:29 -0700 | [diff] [blame] | 172 | if (executor_trace.enabled()) { |
Noah Eisen | 4d20a66 | 2018-02-09 09:34:04 -0800 | [diff] [blame^] | 173 | gpr_log(GPR_DEBUG, "EXECUTOR[%d]: execute", |
| 174 | static_cast<int>(ts - g_thread_state)); |
Craig Tiller | af723b0 | 2017-07-17 17:56:28 -0700 | [diff] [blame] | 175 | } |
Craig Tiller | 3e9f98e | 2017-05-12 13:17:47 -0700 | [diff] [blame] | 176 | |
Yash Tibrewal | 8cf1470 | 2017-12-06 09:47:54 -0800 | [diff] [blame] | 177 | grpc_core::ExecCtx::Get()->InvalidateNow(); |
| 178 | subtract_depth = run_closures(exec); |
Craig Tiller | 3e9f98e | 2017-05-12 13:17:47 -0700 | [diff] [blame] | 179 | } |
| 180 | } |
| 181 | |
Yash Tibrewal | 8cf1470 | 2017-12-06 09:47:54 -0800 | [diff] [blame] | 182 | static void executor_push(grpc_closure* closure, grpc_error* error, |
| 183 | bool is_short) { |
Craig Tiller | 2f767eb | 2017-07-20 12:06:14 -0700 | [diff] [blame] | 184 | bool retry_push; |
Craig Tiller | 07d2fa7 | 2017-09-07 13:13:36 -0700 | [diff] [blame] | 185 | if (is_short) { |
Yash Tibrewal | 8cf1470 | 2017-12-06 09:47:54 -0800 | [diff] [blame] | 186 | GRPC_STATS_INC_EXECUTOR_SCHEDULED_SHORT_ITEMS(); |
Craig Tiller | 07d2fa7 | 2017-09-07 13:13:36 -0700 | [diff] [blame] | 187 | } else { |
Yash Tibrewal | 8cf1470 | 2017-12-06 09:47:54 -0800 | [diff] [blame] | 188 | GRPC_STATS_INC_EXECUTOR_SCHEDULED_LONG_ITEMS(); |
Craig Tiller | 07d2fa7 | 2017-09-07 13:13:36 -0700 | [diff] [blame] | 189 | } |
Craig Tiller | 2f767eb | 2017-07-20 12:06:14 -0700 | [diff] [blame] | 190 | do { |
| 191 | retry_push = false; |
Noah Eisen | 4d20a66 | 2018-02-09 09:34:04 -0800 | [diff] [blame^] | 192 | size_t cur_thread_count = |
| 193 | static_cast<size_t> gpr_atm_no_barrier_load(&g_cur_threads); |
Craig Tiller | 2f767eb | 2017-07-20 12:06:14 -0700 | [diff] [blame] | 194 | if (cur_thread_count == 0) { |
Craig Tiller | 6014e8a | 2017-10-16 13:50:29 -0700 | [diff] [blame] | 195 | if (executor_trace.enabled()) { |
Craig Tiller | c2fb83e | 2017-07-18 12:38:25 -0700 | [diff] [blame] | 196 | #ifndef NDEBUG |
Craig Tiller | 2f767eb | 2017-07-20 12:06:14 -0700 | [diff] [blame] | 197 | gpr_log(GPR_DEBUG, "EXECUTOR: schedule %p (created %s:%d) inline", |
| 198 | closure, closure->file_created, closure->line_created); |
Craig Tiller | c2fb83e | 2017-07-18 12:38:25 -0700 | [diff] [blame] | 199 | #else |
Craig Tiller | 2f767eb | 2017-07-20 12:06:14 -0700 | [diff] [blame] | 200 | gpr_log(GPR_DEBUG, "EXECUTOR: schedule %p inline", closure); |
Craig Tiller | c2fb83e | 2017-07-18 12:38:25 -0700 | [diff] [blame] | 201 | #endif |
Craig Tiller | c2fb83e | 2017-07-18 12:38:25 -0700 | [diff] [blame] | 202 | } |
Yash Tibrewal | 8cf1470 | 2017-12-06 09:47:54 -0800 | [diff] [blame] | 203 | grpc_closure_list_append(grpc_core::ExecCtx::Get()->closure_list(), |
| 204 | closure, error); |
Craig Tiller | 2f767eb | 2017-07-20 12:06:14 -0700 | [diff] [blame] | 205 | return; |
Craig Tiller | c2fb83e | 2017-07-18 12:38:25 -0700 | [diff] [blame] | 206 | } |
Craig Tiller | baa14a9 | 2017-11-03 09:09:36 -0700 | [diff] [blame] | 207 | thread_state* ts = (thread_state*)gpr_tls_get(&g_this_thread_state); |
Craig Tiller | 4782d92 | 2017-11-10 09:53:21 -0800 | [diff] [blame] | 208 | if (ts == nullptr) { |
Yash Tibrewal | 8cf1470 | 2017-12-06 09:47:54 -0800 | [diff] [blame] | 209 | ts = &g_thread_state[GPR_HASH_POINTER(grpc_core::ExecCtx::Get(), |
| 210 | cur_thread_count)]; |
Craig Tiller | 022ad3a | 2017-09-07 13:01:56 -0700 | [diff] [blame] | 211 | } else { |
Yash Tibrewal | 8cf1470 | 2017-12-06 09:47:54 -0800 | [diff] [blame] | 212 | GRPC_STATS_INC_EXECUTOR_SCHEDULED_TO_SELF(); |
Craig Tiller | c2fb83e | 2017-07-18 12:38:25 -0700 | [diff] [blame] | 213 | } |
Craig Tiller | baa14a9 | 2017-11-03 09:09:36 -0700 | [diff] [blame] | 214 | thread_state* orig_ts = ts; |
Craig Tiller | 3e9f98e | 2017-05-12 13:17:47 -0700 | [diff] [blame] | 215 | |
Craig Tiller | 2f767eb | 2017-07-20 12:06:14 -0700 | [diff] [blame] | 216 | bool try_new_thread; |
| 217 | for (;;) { |
Craig Tiller | 6014e8a | 2017-10-16 13:50:29 -0700 | [diff] [blame] | 218 | if (executor_trace.enabled()) { |
Craig Tiller | 2f767eb | 2017-07-20 12:06:14 -0700 | [diff] [blame] | 219 | #ifndef NDEBUG |
Craig Tiller | 8af33db | 2017-07-20 16:37:36 -0700 | [diff] [blame] | 220 | gpr_log( |
| 221 | GPR_DEBUG, |
| 222 | "EXECUTOR: try to schedule %p (%s) (created %s:%d) to thread %d", |
| 223 | closure, is_short ? "short" : "long", closure->file_created, |
Noah Eisen | be82e64 | 2018-02-09 09:16:55 -0800 | [diff] [blame] | 224 | closure->line_created, static_cast<int>(ts - g_thread_state)); |
Craig Tiller | 2f767eb | 2017-07-20 12:06:14 -0700 | [diff] [blame] | 225 | #else |
Craig Tiller | 8af33db | 2017-07-20 16:37:36 -0700 | [diff] [blame] | 226 | gpr_log(GPR_DEBUG, "EXECUTOR: try to schedule %p (%s) to thread %d", |
| 227 | closure, is_short ? "short" : "long", |
| 228 | (int)(ts - g_thread_state)); |
Craig Tiller | 2f767eb | 2017-07-20 12:06:14 -0700 | [diff] [blame] | 229 | #endif |
| 230 | } |
| 231 | gpr_mu_lock(&ts->mu); |
| 232 | if (ts->queued_long_job) { |
Craig Tiller | b0ce25e | 2017-09-08 14:42:26 -0700 | [diff] [blame] | 233 | // if there's a long job queued, we never queue anything else to this |
| 234 | // queue (since long jobs can take 'infinite' time and we need to |
| 235 | // guarantee no starvation) |
| 236 | // ... spin through queues and try again |
Craig Tiller | 2f767eb | 2017-07-20 12:06:14 -0700 | [diff] [blame] | 237 | gpr_mu_unlock(&ts->mu); |
Noah Eisen | be82e64 | 2018-02-09 09:16:55 -0800 | [diff] [blame] | 238 | size_t idx = static_cast<size_t>(ts - g_thread_state); |
Craig Tiller | 1890883 | 2017-07-21 13:27:27 -0700 | [diff] [blame] | 239 | ts = &g_thread_state[(idx + 1) % cur_thread_count]; |
Craig Tiller | 2f767eb | 2017-07-20 12:06:14 -0700 | [diff] [blame] | 240 | if (ts == orig_ts) { |
| 241 | retry_push = true; |
| 242 | try_new_thread = true; |
| 243 | break; |
| 244 | } |
| 245 | continue; |
| 246 | } |
ncteisen | c0b00c3 | 2017-12-14 11:30:38 -0800 | [diff] [blame] | 247 | if (grpc_closure_list_empty(ts->elems) && !ts->shutdown) { |
Yash Tibrewal | 8cf1470 | 2017-12-06 09:47:54 -0800 | [diff] [blame] | 248 | GRPC_STATS_INC_EXECUTOR_WAKEUP_INITIATED(); |
Craig Tiller | 2f767eb | 2017-07-20 12:06:14 -0700 | [diff] [blame] | 249 | gpr_cv_signal(&ts->cv); |
| 250 | } |
| 251 | grpc_closure_list_append(&ts->elems, closure, error); |
Craig Tiller | 2477cf3 | 2017-09-26 12:20:35 -0700 | [diff] [blame] | 252 | ts->depth++; |
| 253 | try_new_thread = ts->depth > MAX_DEPTH && |
Craig Tiller | 2f767eb | 2017-07-20 12:06:14 -0700 | [diff] [blame] | 254 | cur_thread_count < g_max_threads && !ts->shutdown; |
| 255 | if (!is_short) ts->queued_long_job = true; |
| 256 | gpr_mu_unlock(&ts->mu); |
| 257 | break; |
Craig Tiller | 3e9f98e | 2017-05-12 13:17:47 -0700 | [diff] [blame] | 258 | } |
Craig Tiller | 2f767eb | 2017-07-20 12:06:14 -0700 | [diff] [blame] | 259 | if (try_new_thread && gpr_spinlock_trylock(&g_adding_thread_lock)) { |
Noah Eisen | 4d20a66 | 2018-02-09 09:34:04 -0800 | [diff] [blame^] | 260 | cur_thread_count = |
| 261 | static_cast<size_t> gpr_atm_no_barrier_load(&g_cur_threads); |
Craig Tiller | 2f767eb | 2017-07-20 12:06:14 -0700 | [diff] [blame] | 262 | if (cur_thread_count < g_max_threads) { |
| 263 | gpr_atm_no_barrier_store(&g_cur_threads, cur_thread_count + 1); |
| 264 | |
| 265 | gpr_thd_options opt = gpr_thd_options_default(); |
| 266 | gpr_thd_options_set_joinable(&opt); |
Dave MacLachlan | af5c54d | 2017-11-29 16:25:10 -0800 | [diff] [blame] | 267 | gpr_thd_new(&g_thread_state[cur_thread_count].id, "gpr_executor", |
| 268 | executor_thread, &g_thread_state[cur_thread_count], &opt); |
Craig Tiller | 2f767eb | 2017-07-20 12:06:14 -0700 | [diff] [blame] | 269 | } |
| 270 | gpr_spinlock_unlock(&g_adding_thread_lock); |
| 271 | } |
Craig Tiller | 07d2fa7 | 2017-09-07 13:13:36 -0700 | [diff] [blame] | 272 | if (retry_push) { |
Yash Tibrewal | 8cf1470 | 2017-12-06 09:47:54 -0800 | [diff] [blame] | 273 | GRPC_STATS_INC_EXECUTOR_PUSH_RETRIES(); |
Craig Tiller | 07d2fa7 | 2017-09-07 13:13:36 -0700 | [diff] [blame] | 274 | } |
Craig Tiller | 2f767eb | 2017-07-20 12:06:14 -0700 | [diff] [blame] | 275 | } while (retry_push); |
David Garcia Quintas | 4bc3463 | 2015-10-07 16:12:35 -0700 | [diff] [blame] | 276 | } |
Craig Tiller | 91031da | 2016-12-28 15:44:25 -0800 | [diff] [blame] | 277 | |
Yash Tibrewal | 8cf1470 | 2017-12-06 09:47:54 -0800 | [diff] [blame] | 278 | static void executor_push_short(grpc_closure* closure, grpc_error* error) { |
| 279 | executor_push(closure, error, true); |
Craig Tiller | 7a82afd | 2017-07-18 09:40:40 -0700 | [diff] [blame] | 280 | } |
| 281 | |
Yash Tibrewal | 8cf1470 | 2017-12-06 09:47:54 -0800 | [diff] [blame] | 282 | static void executor_push_long(grpc_closure* closure, grpc_error* error) { |
| 283 | executor_push(closure, error, false); |
Craig Tiller | 7a82afd | 2017-07-18 09:40:40 -0700 | [diff] [blame] | 284 | } |
| 285 | |
| 286 | static const grpc_closure_scheduler_vtable executor_vtable_short = { |
| 287 | executor_push_short, executor_push_short, "executor"}; |
| 288 | static grpc_closure_scheduler executor_scheduler_short = { |
| 289 | &executor_vtable_short}; |
| 290 | |
| 291 | static const grpc_closure_scheduler_vtable executor_vtable_long = { |
| 292 | executor_push_long, executor_push_long, "executor"}; |
| 293 | static grpc_closure_scheduler executor_scheduler_long = {&executor_vtable_long}; |
| 294 | |
Craig Tiller | baa14a9 | 2017-11-03 09:09:36 -0700 | [diff] [blame] | 295 | grpc_closure_scheduler* grpc_executor_scheduler( |
Craig Tiller | 7a82afd | 2017-07-18 09:40:40 -0700 | [diff] [blame] | 296 | grpc_executor_job_length length) { |
| 297 | return length == GRPC_EXECUTOR_SHORT ? &executor_scheduler_short |
| 298 | : &executor_scheduler_long; |
| 299 | } |