David Garcia Quintas | 4bc3463 | 2015-10-07 16:12:35 -0700 | [diff] [blame] | 1 | /* |
| 2 | * |
Jan Tattermusch | 7897ae9 | 2017-06-07 22:57:36 +0200 | [diff] [blame] | 3 | * Copyright 2015 gRPC authors. |
David Garcia Quintas | 4bc3463 | 2015-10-07 16:12:35 -0700 | [diff] [blame] | 4 | * |
Jan Tattermusch | 7897ae9 | 2017-06-07 22:57:36 +0200 | [diff] [blame] | 5 | * Licensed under the Apache License, Version 2.0 (the "License"); |
| 6 | * you may not use this file except in compliance with the License. |
| 7 | * You may obtain a copy of the License at |
David Garcia Quintas | 4bc3463 | 2015-10-07 16:12:35 -0700 | [diff] [blame] | 8 | * |
Jan Tattermusch | 7897ae9 | 2017-06-07 22:57:36 +0200 | [diff] [blame] | 9 | * http://www.apache.org/licenses/LICENSE-2.0 |
David Garcia Quintas | 4bc3463 | 2015-10-07 16:12:35 -0700 | [diff] [blame] | 10 | * |
Jan Tattermusch | 7897ae9 | 2017-06-07 22:57:36 +0200 | [diff] [blame] | 11 | * Unless required by applicable law or agreed to in writing, software |
| 12 | * distributed under the License is distributed on an "AS IS" BASIS, |
| 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 14 | * See the License for the specific language governing permissions and |
| 15 | * limitations under the License. |
David Garcia Quintas | 4bc3463 | 2015-10-07 16:12:35 -0700 | [diff] [blame] | 16 | * |
| 17 | */ |
| 18 | |
Alexander Polcyn | db3e898 | 2018-02-21 16:59:24 -0800 | [diff] [blame] | 19 | #include <grpc/support/port_platform.h> |
| 20 | |
Craig Tiller | 9533d04 | 2016-03-25 17:11:06 -0700 | [diff] [blame] | 21 | #include "src/core/lib/iomgr/executor.h" |
David Garcia Quintas | 4bc3463 | 2015-10-07 16:12:35 -0700 | [diff] [blame] | 22 | |
| 23 | #include <string.h> |
Sree Kuchibhotla | 7e9d525 | 2018-07-09 14:53:54 -0700 | [diff] [blame] | 24 | #include <functional> |
David Garcia Quintas | 4bc3463 | 2015-10-07 16:12:35 -0700 | [diff] [blame] | 25 | |
| 26 | #include <grpc/support/alloc.h> |
Craig Tiller | 3e9f98e | 2017-05-12 13:17:47 -0700 | [diff] [blame] | 27 | #include <grpc/support/cpu.h> |
David Garcia Quintas | 4bc3463 | 2015-10-07 16:12:35 -0700 | [diff] [blame] | 28 | #include <grpc/support/log.h> |
| 29 | #include <grpc/support/sync.h> |
Craig Tiller | 3e9f98e | 2017-05-12 13:17:47 -0700 | [diff] [blame] | 30 | |
Craig Tiller | 57bb9a9 | 2017-08-31 16:44:15 -0700 | [diff] [blame] | 31 | #include "src/core/lib/debug/stats.h" |
Vijay Pai | b6cf123 | 2018-01-25 21:02:26 -0800 | [diff] [blame] | 32 | #include "src/core/lib/gpr/tls.h" |
Vijay Pai | d4d0a30 | 2018-01-25 13:24:03 -0800 | [diff] [blame] | 33 | #include "src/core/lib/gpr/useful.h" |
Craig Tiller | 9533d04 | 2016-03-25 17:11:06 -0700 | [diff] [blame] | 34 | #include "src/core/lib/iomgr/exec_ctx.h" |
David Garcia Quintas | 4bc3463 | 2015-10-07 16:12:35 -0700 | [diff] [blame] | 35 | |
Craig Tiller | 2477cf3 | 2017-09-26 12:20:35 -0700 | [diff] [blame] | 36 | #define MAX_DEPTH 2 |
| 37 | |
Sree Kuchibhotla | 83d0bfa | 2018-07-10 11:29:43 -0700 | [diff] [blame] | 38 | #define EXECUTOR_TRACE(format, ...) \ |
| 39 | if (executor_trace.enabled()) { \ |
| 40 | gpr_log(GPR_INFO, "EXECUTOR " format, __VA_ARGS__); \ |
Sree Kuchibhotla | 7e9d525 | 2018-07-09 14:53:54 -0700 | [diff] [blame] | 41 | } |
Craig Tiller | 3e9f98e | 2017-05-12 13:17:47 -0700 | [diff] [blame] | 42 | |
ncteisen | 72afb76 | 2017-11-10 12:23:12 -0800 | [diff] [blame] | 43 | grpc_core::TraceFlag executor_trace(false, "executor"); |
Craig Tiller | af723b0 | 2017-07-17 17:56:28 -0700 | [diff] [blame] | 44 | |
Sree Kuchibhotla | 7e9d525 | 2018-07-09 14:53:54 -0700 | [diff] [blame] | 45 | GPR_TLS_DECL(g_this_thread_state); |
David Garcia Quintas | 4bc3463 | 2015-10-07 16:12:35 -0700 | [diff] [blame] | 46 | |
Sree Kuchibhotla | 83d0bfa | 2018-07-10 11:29:43 -0700 | [diff] [blame] | 47 | GrpcExecutor::GrpcExecutor(const char* executor_name) : name_(executor_name) { |
| 48 | adding_thread_lock_ = GPR_SPINLOCK_STATIC_INITIALIZER; |
| 49 | gpr_atm_no_barrier_store(&num_threads_, 0); |
| 50 | max_threads_ = GPR_MAX(1, 2 * gpr_cpu_num_cores()); |
Sree Kuchibhotla | 7e9d525 | 2018-07-09 14:53:54 -0700 | [diff] [blame] | 51 | } |
| 52 | |
| 53 | void GrpcExecutor::Init() { SetThreading(true); } |
| 54 | |
| 55 | size_t GrpcExecutor::RunClosures(grpc_closure_list list) { |
Craig Tiller | 2477cf3 | 2017-09-26 12:20:35 -0700 | [diff] [blame] | 56 | size_t n = 0; |
David Garcia Quintas | 4bc3463 | 2015-10-07 16:12:35 -0700 | [diff] [blame] | 57 | |
Craig Tiller | baa14a9 | 2017-11-03 09:09:36 -0700 | [diff] [blame] | 58 | grpc_closure* c = list.head; |
Craig Tiller | 4782d92 | 2017-11-10 09:53:21 -0800 | [diff] [blame] | 59 | while (c != nullptr) { |
Craig Tiller | baa14a9 | 2017-11-03 09:09:36 -0700 | [diff] [blame] | 60 | grpc_closure* next = c->next_data.next; |
| 61 | grpc_error* error = c->error_data.error; |
Craig Tiller | af723b0 | 2017-07-17 17:56:28 -0700 | [diff] [blame] | 62 | #ifndef NDEBUG |
Sree Kuchibhotla | 83d0bfa | 2018-07-10 11:29:43 -0700 | [diff] [blame] | 63 | EXECUTOR_TRACE("run %p [created by %s:%d]", c, c->file_created, |
Sree Kuchibhotla | 7e9d525 | 2018-07-09 14:53:54 -0700 | [diff] [blame] | 64 | c->line_created); |
Craig Tiller | 2477cf3 | 2017-09-26 12:20:35 -0700 | [diff] [blame] | 65 | c->scheduled = false; |
Sree Kuchibhotla | 7e9d525 | 2018-07-09 14:53:54 -0700 | [diff] [blame] | 66 | #else |
Sree Kuchibhotla | 83d0bfa | 2018-07-10 11:29:43 -0700 | [diff] [blame] | 67 | EXECUTOR_TRACE("run %p", c); |
Craig Tiller | 2477cf3 | 2017-09-26 12:20:35 -0700 | [diff] [blame] | 68 | #endif |
Yash Tibrewal | 8cf1470 | 2017-12-06 09:47:54 -0800 | [diff] [blame] | 69 | c->cb(c->cb_arg, error); |
Craig Tiller | 2477cf3 | 2017-09-26 12:20:35 -0700 | [diff] [blame] | 70 | GRPC_ERROR_UNREF(error); |
| 71 | c = next; |
| 72 | n++; |
Yash Tibrewal | 8cf1470 | 2017-12-06 09:47:54 -0800 | [diff] [blame] | 73 | grpc_core::ExecCtx::Get()->Flush(); |
Craig Tiller | 061ef74 | 2016-12-29 10:54:09 -0800 | [diff] [blame] | 74 | } |
Craig Tiller | 3e9f98e | 2017-05-12 13:17:47 -0700 | [diff] [blame] | 75 | |
Craig Tiller | 2477cf3 | 2017-09-26 12:20:35 -0700 | [diff] [blame] | 76 | return n; |
Craig Tiller | 3e9f98e | 2017-05-12 13:17:47 -0700 | [diff] [blame] | 77 | } |
| 78 | |
Sree Kuchibhotla | 83d0bfa | 2018-07-10 11:29:43 -0700 | [diff] [blame] | 79 | bool GrpcExecutor::IsThreaded() const { |
| 80 | return gpr_atm_no_barrier_load(&num_threads_) > 0; |
Craig Tiller | 5e56f00 | 2017-05-16 15:02:50 -0700 | [diff] [blame] | 81 | } |
| 82 | |
Sree Kuchibhotla | 7e9d525 | 2018-07-09 14:53:54 -0700 | [diff] [blame] | 83 | void GrpcExecutor::SetThreading(bool threading) { |
Sree Kuchibhotla | 7b8a6b6 | 2018-07-11 11:51:46 -0700 | [diff] [blame^] | 84 | gpr_atm curr_num_threads = gpr_atm_no_barrier_load(&num_threads_); |
Sree Kuchibhotla | 7e9d525 | 2018-07-09 14:53:54 -0700 | [diff] [blame] | 85 | |
Craig Tiller | 5e56f00 | 2017-05-16 15:02:50 -0700 | [diff] [blame] | 86 | if (threading) { |
Sree Kuchibhotla | 7e9d525 | 2018-07-09 14:53:54 -0700 | [diff] [blame] | 87 | if (curr_num_threads > 0) return; |
| 88 | |
Sree Kuchibhotla | 83d0bfa | 2018-07-10 11:29:43 -0700 | [diff] [blame] | 89 | GPR_ASSERT(num_threads_ == 0); |
| 90 | gpr_atm_no_barrier_store(&num_threads_, 1); |
Sree Kuchibhotla | 8cc3a00 | 2018-07-10 13:32:35 -0700 | [diff] [blame] | 91 | gpr_tls_init(&g_this_thread_state); |
Sree Kuchibhotla | 83d0bfa | 2018-07-10 11:29:43 -0700 | [diff] [blame] | 92 | thd_state_ = static_cast<ThreadState*>( |
| 93 | gpr_zalloc(sizeof(ThreadState) * max_threads_)); |
Sree Kuchibhotla | 7e9d525 | 2018-07-09 14:53:54 -0700 | [diff] [blame] | 94 | |
Sree Kuchibhotla | 83d0bfa | 2018-07-10 11:29:43 -0700 | [diff] [blame] | 95 | for (size_t i = 0; i < max_threads_; i++) { |
| 96 | gpr_mu_init(&thd_state_[i].mu); |
| 97 | gpr_cv_init(&thd_state_[i].cv); |
| 98 | thd_state_[i].id = i; |
| 99 | thd_state_[i].thd = grpc_core::Thread(); |
| 100 | thd_state_[i].elems = GRPC_CLOSURE_LIST_INIT; |
Craig Tiller | 5e56f00 | 2017-05-16 15:02:50 -0700 | [diff] [blame] | 101 | } |
| 102 | |
Sree Kuchibhotla | 83d0bfa | 2018-07-10 11:29:43 -0700 | [diff] [blame] | 103 | thd_state_[0].thd = |
| 104 | grpc_core::Thread(name_, &GrpcExecutor::ThreadMain, &thd_state_[0]); |
| 105 | thd_state_[0].thd.Start(); |
Sree Kuchibhotla | 02872df | 2018-07-10 14:21:51 -0700 | [diff] [blame] | 106 | } else { // !threading |
Sree Kuchibhotla | 7e9d525 | 2018-07-09 14:53:54 -0700 | [diff] [blame] | 107 | if (curr_num_threads == 0) return; |
| 108 | |
Sree Kuchibhotla | 83d0bfa | 2018-07-10 11:29:43 -0700 | [diff] [blame] | 109 | for (size_t i = 0; i < max_threads_; i++) { |
| 110 | gpr_mu_lock(&thd_state_[i].mu); |
| 111 | thd_state_[i].shutdown = true; |
| 112 | gpr_cv_signal(&thd_state_[i].cv); |
| 113 | gpr_mu_unlock(&thd_state_[i].mu); |
Craig Tiller | 5e56f00 | 2017-05-16 15:02:50 -0700 | [diff] [blame] | 114 | } |
Sree Kuchibhotla | 7e9d525 | 2018-07-09 14:53:54 -0700 | [diff] [blame] | 115 | |
| 116 | /* Ensure no thread is adding a new thread. Once this is past, then no |
| 117 | * thread will try to add a new one either (since shutdown is true) */ |
Sree Kuchibhotla | 83d0bfa | 2018-07-10 11:29:43 -0700 | [diff] [blame] | 118 | gpr_spinlock_lock(&adding_thread_lock_); |
| 119 | gpr_spinlock_unlock(&adding_thread_lock_); |
Sree Kuchibhotla | 7e9d525 | 2018-07-09 14:53:54 -0700 | [diff] [blame] | 120 | |
Sree Kuchibhotla | 7b8a6b6 | 2018-07-11 11:51:46 -0700 | [diff] [blame^] | 121 | curr_num_threads = gpr_atm_no_barrier_load(&num_threads_); |
| 122 | for (gpr_atm i = 0; i < curr_num_threads; i++) { |
Sree Kuchibhotla | 83d0bfa | 2018-07-10 11:29:43 -0700 | [diff] [blame] | 123 | thd_state_[i].thd.Join(); |
Sree Kuchibhotla | 7b8a6b6 | 2018-07-11 11:51:46 -0700 | [diff] [blame^] | 124 | EXECUTOR_TRACE(" Thread %" PRIdPTR " of %" PRIdPTR " joined", i, |
| 125 | curr_num_threads); |
Craig Tiller | 5e56f00 | 2017-05-16 15:02:50 -0700 | [diff] [blame] | 126 | } |
Sree Kuchibhotla | 7e9d525 | 2018-07-09 14:53:54 -0700 | [diff] [blame] | 127 | |
Sree Kuchibhotla | 83d0bfa | 2018-07-10 11:29:43 -0700 | [diff] [blame] | 128 | gpr_atm_no_barrier_store(&num_threads_, 0); |
| 129 | for (size_t i = 0; i < max_threads_; i++) { |
| 130 | gpr_mu_destroy(&thd_state_[i].mu); |
| 131 | gpr_cv_destroy(&thd_state_[i].cv); |
| 132 | RunClosures(thd_state_[i].elems); |
Craig Tiller | 5e56f00 | 2017-05-16 15:02:50 -0700 | [diff] [blame] | 133 | } |
Sree Kuchibhotla | 7e9d525 | 2018-07-09 14:53:54 -0700 | [diff] [blame] | 134 | |
Sree Kuchibhotla | 83d0bfa | 2018-07-10 11:29:43 -0700 | [diff] [blame] | 135 | gpr_free(thd_state_); |
Craig Tiller | 5e56f00 | 2017-05-16 15:02:50 -0700 | [diff] [blame] | 136 | gpr_tls_destroy(&g_this_thread_state); |
David Garcia Quintas | 4bc3463 | 2015-10-07 16:12:35 -0700 | [diff] [blame] | 137 | } |
Craig Tiller | 5e56f00 | 2017-05-16 15:02:50 -0700 | [diff] [blame] | 138 | } |
| 139 | |
Sree Kuchibhotla | 7e9d525 | 2018-07-09 14:53:54 -0700 | [diff] [blame] | 140 | void GrpcExecutor::Shutdown() { SetThreading(false); } |
Craig Tiller | 5e56f00 | 2017-05-16 15:02:50 -0700 | [diff] [blame] | 141 | |
Sree Kuchibhotla | 7e9d525 | 2018-07-09 14:53:54 -0700 | [diff] [blame] | 142 | void GrpcExecutor::ThreadMain(void* arg) { |
Sree Kuchibhotla | 83d0bfa | 2018-07-10 11:29:43 -0700 | [diff] [blame] | 143 | ThreadState* ts = static_cast<ThreadState*>(arg); |
Sree Kuchibhotla | 02872df | 2018-07-10 14:21:51 -0700 | [diff] [blame] | 144 | gpr_tls_set(&g_this_thread_state, reinterpret_cast<intptr_t>(ts)); |
Craig Tiller | 3e9f98e | 2017-05-12 13:17:47 -0700 | [diff] [blame] | 145 | |
kpayson64 | 46a6059 | 2018-06-20 15:18:38 -0700 | [diff] [blame] | 146 | grpc_core::ExecCtx exec_ctx(GRPC_EXEC_CTX_FLAG_IS_INTERNAL_THREAD); |
Craig Tiller | 8996208 | 2017-05-12 14:30:42 -0700 | [diff] [blame] | 147 | |
Craig Tiller | 2477cf3 | 2017-09-26 12:20:35 -0700 | [diff] [blame] | 148 | size_t subtract_depth = 0; |
Craig Tiller | 3e9f98e | 2017-05-12 13:17:47 -0700 | [diff] [blame] | 149 | for (;;) { |
Sree Kuchibhotla | 83d0bfa | 2018-07-10 11:29:43 -0700 | [diff] [blame] | 150 | EXECUTOR_TRACE("[%" PRIdPTR "]: step (sub_depth=%" PRIdPTR ")", ts->id, |
Sree Kuchibhotla | 7e9d525 | 2018-07-09 14:53:54 -0700 | [diff] [blame] | 151 | subtract_depth); |
| 152 | |
Craig Tiller | 3e9f98e | 2017-05-12 13:17:47 -0700 | [diff] [blame] | 153 | gpr_mu_lock(&ts->mu); |
Craig Tiller | 2477cf3 | 2017-09-26 12:20:35 -0700 | [diff] [blame] | 154 | ts->depth -= subtract_depth; |
Sree Kuchibhotla | 7e9d525 | 2018-07-09 14:53:54 -0700 | [diff] [blame] | 155 | // Wait for closures to be enqueued or for the executor to be shutdown |
Craig Tiller | 3e9f98e | 2017-05-12 13:17:47 -0700 | [diff] [blame] | 156 | while (grpc_closure_list_empty(ts->elems) && !ts->shutdown) { |
Craig Tiller | 1ab56d8 | 2017-07-19 09:55:57 -0700 | [diff] [blame] | 157 | ts->queued_long_job = false; |
Sree Kuchibhotla | 54961bb | 2017-12-04 12:50:27 -0800 | [diff] [blame] | 158 | gpr_cv_wait(&ts->cv, &ts->mu, gpr_inf_future(GPR_CLOCK_MONOTONIC)); |
Craig Tiller | 3e9f98e | 2017-05-12 13:17:47 -0700 | [diff] [blame] | 159 | } |
Sree Kuchibhotla | 7e9d525 | 2018-07-09 14:53:54 -0700 | [diff] [blame] | 160 | |
Craig Tiller | 3e9f98e | 2017-05-12 13:17:47 -0700 | [diff] [blame] | 161 | if (ts->shutdown) { |
Sree Kuchibhotla | 83d0bfa | 2018-07-10 11:29:43 -0700 | [diff] [blame] | 162 | EXECUTOR_TRACE("[%" PRIdPTR "]: shutdown", ts->id); |
Craig Tiller | 3e9f98e | 2017-05-12 13:17:47 -0700 | [diff] [blame] | 163 | gpr_mu_unlock(&ts->mu); |
| 164 | break; |
| 165 | } |
Sree Kuchibhotla | 7e9d525 | 2018-07-09 14:53:54 -0700 | [diff] [blame] | 166 | |
Yash Tibrewal | 8cf1470 | 2017-12-06 09:47:54 -0800 | [diff] [blame] | 167 | GRPC_STATS_INC_EXECUTOR_QUEUE_DRAINED(); |
Sree Kuchibhotla | 7e9d525 | 2018-07-09 14:53:54 -0700 | [diff] [blame] | 168 | grpc_closure_list closures = ts->elems; |
Yash Tibrewal | 37fdb73 | 2017-09-25 16:45:02 -0700 | [diff] [blame] | 169 | ts->elems = GRPC_CLOSURE_LIST_INIT; |
Craig Tiller | 3e9f98e | 2017-05-12 13:17:47 -0700 | [diff] [blame] | 170 | gpr_mu_unlock(&ts->mu); |
Sree Kuchibhotla | 7e9d525 | 2018-07-09 14:53:54 -0700 | [diff] [blame] | 171 | |
Sree Kuchibhotla | 83d0bfa | 2018-07-10 11:29:43 -0700 | [diff] [blame] | 172 | EXECUTOR_TRACE("[%" PRIdPTR "]: execute", ts->id); |
Craig Tiller | 3e9f98e | 2017-05-12 13:17:47 -0700 | [diff] [blame] | 173 | |
Yash Tibrewal | 8cf1470 | 2017-12-06 09:47:54 -0800 | [diff] [blame] | 174 | grpc_core::ExecCtx::Get()->InvalidateNow(); |
Sree Kuchibhotla | 7e9d525 | 2018-07-09 14:53:54 -0700 | [diff] [blame] | 175 | subtract_depth = RunClosures(closures); |
Craig Tiller | 3e9f98e | 2017-05-12 13:17:47 -0700 | [diff] [blame] | 176 | } |
| 177 | } |
| 178 | |
Sree Kuchibhotla | 7e9d525 | 2018-07-09 14:53:54 -0700 | [diff] [blame] | 179 | void GrpcExecutor::Enqueue(grpc_closure* closure, grpc_error* error, |
| 180 | bool is_short) { |
Craig Tiller | 2f767eb | 2017-07-20 12:06:14 -0700 | [diff] [blame] | 181 | bool retry_push; |
Craig Tiller | 07d2fa7 | 2017-09-07 13:13:36 -0700 | [diff] [blame] | 182 | if (is_short) { |
Yash Tibrewal | 8cf1470 | 2017-12-06 09:47:54 -0800 | [diff] [blame] | 183 | GRPC_STATS_INC_EXECUTOR_SCHEDULED_SHORT_ITEMS(); |
Craig Tiller | 07d2fa7 | 2017-09-07 13:13:36 -0700 | [diff] [blame] | 184 | } else { |
Yash Tibrewal | 8cf1470 | 2017-12-06 09:47:54 -0800 | [diff] [blame] | 185 | GRPC_STATS_INC_EXECUTOR_SCHEDULED_LONG_ITEMS(); |
Craig Tiller | 07d2fa7 | 2017-09-07 13:13:36 -0700 | [diff] [blame] | 186 | } |
Sree Kuchibhotla | 7e9d525 | 2018-07-09 14:53:54 -0700 | [diff] [blame] | 187 | |
Craig Tiller | 2f767eb | 2017-07-20 12:06:14 -0700 | [diff] [blame] | 188 | do { |
| 189 | retry_push = false; |
Noah Eisen | 4d20a66 | 2018-02-09 09:34:04 -0800 | [diff] [blame] | 190 | size_t cur_thread_count = |
Sree Kuchibhotla | 83d0bfa | 2018-07-10 11:29:43 -0700 | [diff] [blame] | 191 | static_cast<size_t>(gpr_atm_no_barrier_load(&num_threads_)); |
Sree Kuchibhotla | 7e9d525 | 2018-07-09 14:53:54 -0700 | [diff] [blame] | 192 | |
| 193 | // If the number of threads is zero(i.e either the executor is not threaded |
| 194 | // or already shutdown), then queue the closure on the exec context itself |
Craig Tiller | 2f767eb | 2017-07-20 12:06:14 -0700 | [diff] [blame] | 195 | if (cur_thread_count == 0) { |
Craig Tiller | c2fb83e | 2017-07-18 12:38:25 -0700 | [diff] [blame] | 196 | #ifndef NDEBUG |
Sree Kuchibhotla | 83d0bfa | 2018-07-10 11:29:43 -0700 | [diff] [blame] | 197 | EXECUTOR_TRACE("schedule %p (created %s:%d) inline", closure, |
Sree Kuchibhotla | 7e9d525 | 2018-07-09 14:53:54 -0700 | [diff] [blame] | 198 | closure->file_created, closure->line_created); |
Craig Tiller | c2fb83e | 2017-07-18 12:38:25 -0700 | [diff] [blame] | 199 | #else |
Sree Kuchibhotla | 83d0bfa | 2018-07-10 11:29:43 -0700 | [diff] [blame] | 200 | EXECUTOR_TRACE("schedule %p inline", closure); |
Craig Tiller | c2fb83e | 2017-07-18 12:38:25 -0700 | [diff] [blame] | 201 | #endif |
Yash Tibrewal | 8cf1470 | 2017-12-06 09:47:54 -0800 | [diff] [blame] | 202 | grpc_closure_list_append(grpc_core::ExecCtx::Get()->closure_list(), |
| 203 | closure, error); |
Craig Tiller | 2f767eb | 2017-07-20 12:06:14 -0700 | [diff] [blame] | 204 | return; |
Craig Tiller | c2fb83e | 2017-07-18 12:38:25 -0700 | [diff] [blame] | 205 | } |
Sree Kuchibhotla | 7e9d525 | 2018-07-09 14:53:54 -0700 | [diff] [blame] | 206 | |
Sree Kuchibhotla | 83d0bfa | 2018-07-10 11:29:43 -0700 | [diff] [blame] | 207 | ThreadState* ts = (ThreadState*)gpr_tls_get(&g_this_thread_state); |
Craig Tiller | 4782d92 | 2017-11-10 09:53:21 -0800 | [diff] [blame] | 208 | if (ts == nullptr) { |
Sree Kuchibhotla | 83d0bfa | 2018-07-10 11:29:43 -0700 | [diff] [blame] | 209 | ts = &thd_state_[GPR_HASH_POINTER(grpc_core::ExecCtx::Get(), |
| 210 | cur_thread_count)]; |
Craig Tiller | 022ad3a | 2017-09-07 13:01:56 -0700 | [diff] [blame] | 211 | } else { |
Yash Tibrewal | 8cf1470 | 2017-12-06 09:47:54 -0800 | [diff] [blame] | 212 | GRPC_STATS_INC_EXECUTOR_SCHEDULED_TO_SELF(); |
Craig Tiller | c2fb83e | 2017-07-18 12:38:25 -0700 | [diff] [blame] | 213 | } |
Sree Kuchibhotla | 7e9d525 | 2018-07-09 14:53:54 -0700 | [diff] [blame] | 214 | |
Sree Kuchibhotla | 83d0bfa | 2018-07-10 11:29:43 -0700 | [diff] [blame] | 215 | ThreadState* orig_ts = ts; |
Craig Tiller | 3e9f98e | 2017-05-12 13:17:47 -0700 | [diff] [blame] | 216 | |
Sree Kuchibhotla | 83d0bfa | 2018-07-10 11:29:43 -0700 | [diff] [blame] | 217 | bool try_new_thread = false; |
Craig Tiller | 2f767eb | 2017-07-20 12:06:14 -0700 | [diff] [blame] | 218 | for (;;) { |
Craig Tiller | 2f767eb | 2017-07-20 12:06:14 -0700 | [diff] [blame] | 219 | #ifndef NDEBUG |
Sree Kuchibhotla | 7e9d525 | 2018-07-09 14:53:54 -0700 | [diff] [blame] | 220 | EXECUTOR_TRACE( |
Sree Kuchibhotla | 83d0bfa | 2018-07-10 11:29:43 -0700 | [diff] [blame] | 221 | "try to schedule %p (%s) (created %s:%d) to thread " |
| 222 | "%" PRIdPTR, |
Sree Kuchibhotla | 7e9d525 | 2018-07-09 14:53:54 -0700 | [diff] [blame] | 223 | closure, is_short ? "short" : "long", closure->file_created, |
| 224 | closure->line_created, ts->id); |
Craig Tiller | 2f767eb | 2017-07-20 12:06:14 -0700 | [diff] [blame] | 225 | #else |
Sree Kuchibhotla | 83d0bfa | 2018-07-10 11:29:43 -0700 | [diff] [blame] | 226 | EXECUTOR_TRACE("try to schedule %p (%s) to thread %" PRIdPTR, closure, |
Sree Kuchibhotla | 7e9d525 | 2018-07-09 14:53:54 -0700 | [diff] [blame] | 227 | is_short ? "short" : "long", ts->id); |
Craig Tiller | 2f767eb | 2017-07-20 12:06:14 -0700 | [diff] [blame] | 228 | #endif |
Sree Kuchibhotla | 7e9d525 | 2018-07-09 14:53:54 -0700 | [diff] [blame] | 229 | |
Craig Tiller | 2f767eb | 2017-07-20 12:06:14 -0700 | [diff] [blame] | 230 | gpr_mu_lock(&ts->mu); |
| 231 | if (ts->queued_long_job) { |
Craig Tiller | b0ce25e | 2017-09-08 14:42:26 -0700 | [diff] [blame] | 232 | // if there's a long job queued, we never queue anything else to this |
| 233 | // queue (since long jobs can take 'infinite' time and we need to |
Sree Kuchibhotla | 7e9d525 | 2018-07-09 14:53:54 -0700 | [diff] [blame] | 234 | // guarantee no starvation). Spin through queues and try again |
Craig Tiller | 2f767eb | 2017-07-20 12:06:14 -0700 | [diff] [blame] | 235 | gpr_mu_unlock(&ts->mu); |
Sree Kuchibhotla | 7e9d525 | 2018-07-09 14:53:54 -0700 | [diff] [blame] | 236 | size_t idx = ts->id; |
Sree Kuchibhotla | 83d0bfa | 2018-07-10 11:29:43 -0700 | [diff] [blame] | 237 | ts = &thd_state_[(idx + 1) % cur_thread_count]; |
Craig Tiller | 2f767eb | 2017-07-20 12:06:14 -0700 | [diff] [blame] | 238 | if (ts == orig_ts) { |
Sree Kuchibhotla | 7e9d525 | 2018-07-09 14:53:54 -0700 | [diff] [blame] | 239 | // We cycled through all the threads. Retry enqueue again (by creating |
| 240 | // a new thread) |
Craig Tiller | 2f767eb | 2017-07-20 12:06:14 -0700 | [diff] [blame] | 241 | retry_push = true; |
Sree Kuchibhotla | 7e9d525 | 2018-07-09 14:53:54 -0700 | [diff] [blame] | 242 | // TODO (sreek): What if the executor is shutdown OR if |
| 243 | // cur_thread_count is already equal to max_threads ? (currently - as |
| 244 | // of July 2018, we do not run in to this issue because there is only |
| 245 | // one instance of long job in gRPC. This has to be fixed soon) |
Craig Tiller | 2f767eb | 2017-07-20 12:06:14 -0700 | [diff] [blame] | 246 | try_new_thread = true; |
| 247 | break; |
| 248 | } |
Sree Kuchibhotla | 7e9d525 | 2018-07-09 14:53:54 -0700 | [diff] [blame] | 249 | |
Craig Tiller | 2f767eb | 2017-07-20 12:06:14 -0700 | [diff] [blame] | 250 | continue; |
| 251 | } |
Sree Kuchibhotla | 7e9d525 | 2018-07-09 14:53:54 -0700 | [diff] [blame] | 252 | |
| 253 | // == Found the thread state (i.e thread) to enqueue this closure! == |
| 254 | |
| 255 | // Also, if this thread has been waiting for closures, wake it up. |
| 256 | // - If grpc_closure_list_empty() is true and the Executor is not |
| 257 | // shutdown, it means that the thread must be waiting in ThreadMain() |
| 258 | // - Note that gpr_cv_signal() won't immediately wakeup the thread. That |
| 259 | // happens after we release the mutex &ts->mu a few lines below |
ncteisen | c0b00c3 | 2017-12-14 11:30:38 -0800 | [diff] [blame] | 260 | if (grpc_closure_list_empty(ts->elems) && !ts->shutdown) { |
Yash Tibrewal | 8cf1470 | 2017-12-06 09:47:54 -0800 | [diff] [blame] | 261 | GRPC_STATS_INC_EXECUTOR_WAKEUP_INITIATED(); |
Craig Tiller | 2f767eb | 2017-07-20 12:06:14 -0700 | [diff] [blame] | 262 | gpr_cv_signal(&ts->cv); |
| 263 | } |
Sree Kuchibhotla | 7e9d525 | 2018-07-09 14:53:54 -0700 | [diff] [blame] | 264 | |
Craig Tiller | 2f767eb | 2017-07-20 12:06:14 -0700 | [diff] [blame] | 265 | grpc_closure_list_append(&ts->elems, closure, error); |
Sree Kuchibhotla | 7e9d525 | 2018-07-09 14:53:54 -0700 | [diff] [blame] | 266 | |
| 267 | // If we already queued more than MAX_DEPTH number of closures on this |
| 268 | // thread, use this as a hint to create more threads |
Craig Tiller | 2477cf3 | 2017-09-26 12:20:35 -0700 | [diff] [blame] | 269 | ts->depth++; |
| 270 | try_new_thread = ts->depth > MAX_DEPTH && |
Sree Kuchibhotla | 83d0bfa | 2018-07-10 11:29:43 -0700 | [diff] [blame] | 271 | cur_thread_count < max_threads_ && !ts->shutdown; |
Sree Kuchibhotla | 7e9d525 | 2018-07-09 14:53:54 -0700 | [diff] [blame] | 272 | |
| 273 | ts->queued_long_job = !is_short; |
| 274 | |
Craig Tiller | 2f767eb | 2017-07-20 12:06:14 -0700 | [diff] [blame] | 275 | gpr_mu_unlock(&ts->mu); |
| 276 | break; |
Craig Tiller | 3e9f98e | 2017-05-12 13:17:47 -0700 | [diff] [blame] | 277 | } |
Craig Tiller | 2f767eb | 2017-07-20 12:06:14 -0700 | [diff] [blame] | 278 | |
Sree Kuchibhotla | 83d0bfa | 2018-07-10 11:29:43 -0700 | [diff] [blame] | 279 | if (try_new_thread && gpr_spinlock_trylock(&adding_thread_lock_)) { |
Sree Kuchibhotla | 7e9d525 | 2018-07-09 14:53:54 -0700 | [diff] [blame] | 280 | cur_thread_count = |
Sree Kuchibhotla | 83d0bfa | 2018-07-10 11:29:43 -0700 | [diff] [blame] | 281 | static_cast<size_t>(gpr_atm_no_barrier_load(&num_threads_)); |
| 282 | if (cur_thread_count < max_threads_) { |
Sree Kuchibhotla | 7e9d525 | 2018-07-09 14:53:54 -0700 | [diff] [blame] | 283 | // Increment num_threads (Safe to do a no_barrier_store instead of a |
| 284 | // cas because we always increment num_threads under the |
| 285 | // 'adding_thread_lock') |
Sree Kuchibhotla | 83d0bfa | 2018-07-10 11:29:43 -0700 | [diff] [blame] | 286 | gpr_atm_no_barrier_store(&num_threads_, cur_thread_count + 1); |
Sree Kuchibhotla | 7e9d525 | 2018-07-09 14:53:54 -0700 | [diff] [blame] | 287 | |
Sree Kuchibhotla | 83d0bfa | 2018-07-10 11:29:43 -0700 | [diff] [blame] | 288 | thd_state_[cur_thread_count].thd = grpc_core::Thread( |
| 289 | name_, &GrpcExecutor::ThreadMain, &thd_state_[cur_thread_count]); |
| 290 | thd_state_[cur_thread_count].thd.Start(); |
Craig Tiller | 2f767eb | 2017-07-20 12:06:14 -0700 | [diff] [blame] | 291 | } |
Sree Kuchibhotla | 83d0bfa | 2018-07-10 11:29:43 -0700 | [diff] [blame] | 292 | gpr_spinlock_unlock(&adding_thread_lock_); |
Craig Tiller | 2f767eb | 2017-07-20 12:06:14 -0700 | [diff] [blame] | 293 | } |
Sree Kuchibhotla | 7e9d525 | 2018-07-09 14:53:54 -0700 | [diff] [blame] | 294 | |
Craig Tiller | 07d2fa7 | 2017-09-07 13:13:36 -0700 | [diff] [blame] | 295 | if (retry_push) { |
Yash Tibrewal | 8cf1470 | 2017-12-06 09:47:54 -0800 | [diff] [blame] | 296 | GRPC_STATS_INC_EXECUTOR_PUSH_RETRIES(); |
Craig Tiller | 07d2fa7 | 2017-09-07 13:13:36 -0700 | [diff] [blame] | 297 | } |
Craig Tiller | 2f767eb | 2017-07-20 12:06:14 -0700 | [diff] [blame] | 298 | } while (retry_push); |
David Garcia Quintas | 4bc3463 | 2015-10-07 16:12:35 -0700 | [diff] [blame] | 299 | } |
Craig Tiller | 91031da | 2016-12-28 15:44:25 -0800 | [diff] [blame] | 300 | |
Sree Kuchibhotla | 7e9d525 | 2018-07-09 14:53:54 -0700 | [diff] [blame] | 301 | static GrpcExecutor g_global_executor("grpc-executor"); |
| 302 | |
| 303 | void enqueue_long(grpc_closure* closure, grpc_error* error) { |
Sree Kuchibhotla | 83d0bfa | 2018-07-10 11:29:43 -0700 | [diff] [blame] | 304 | g_global_executor.Enqueue(closure, error, false /* is_short */); |
Craig Tiller | 7a82afd | 2017-07-18 09:40:40 -0700 | [diff] [blame] | 305 | } |
| 306 | |
Sree Kuchibhotla | 7e9d525 | 2018-07-09 14:53:54 -0700 | [diff] [blame] | 307 | void enqueue_short(grpc_closure* closure, grpc_error* error) { |
Sree Kuchibhotla | 83d0bfa | 2018-07-10 11:29:43 -0700 | [diff] [blame] | 308 | g_global_executor.Enqueue(closure, error, true /* is_short */); |
Craig Tiller | 7a82afd | 2017-07-18 09:40:40 -0700 | [diff] [blame] | 309 | } |
| 310 | |
Sree Kuchibhotla | 7e9d525 | 2018-07-09 14:53:54 -0700 | [diff] [blame] | 311 | // Short-Job executor scheduler |
| 312 | static const grpc_closure_scheduler_vtable global_executor_vtable_short = { |
| 313 | enqueue_short, enqueue_short, "executor-short"}; |
| 314 | static grpc_closure_scheduler global_scheduler_short = { |
| 315 | &global_executor_vtable_short}; |
Craig Tiller | 7a82afd | 2017-07-18 09:40:40 -0700 | [diff] [blame] | 316 | |
Sree Kuchibhotla | 7e9d525 | 2018-07-09 14:53:54 -0700 | [diff] [blame] | 317 | // Long-job executor scheduler |
| 318 | static const grpc_closure_scheduler_vtable global_executor_vtable_long = { |
| 319 | enqueue_long, enqueue_long, "executor-long"}; |
| 320 | static grpc_closure_scheduler global_scheduler_long = { |
| 321 | &global_executor_vtable_long}; |
| 322 | |
| 323 | void grpc_executor_init() { g_global_executor.Init(); } |
| 324 | |
| 325 | void grpc_executor_shutdown() { g_global_executor.Shutdown(); } |
| 326 | |
| 327 | bool grpc_executor_is_threaded() { return g_global_executor.IsThreaded(); } |
| 328 | |
| 329 | void grpc_executor_set_threading(bool enable) { |
| 330 | g_global_executor.SetThreading(enable); |
| 331 | } |
Craig Tiller | 7a82afd | 2017-07-18 09:40:40 -0700 | [diff] [blame] | 332 | |
Sree Kuchibhotla | 1e69b7c | 2018-07-10 19:30:09 -0700 | [diff] [blame] | 333 | grpc_closure_scheduler* grpc_executor_scheduler(GrpcExecutorJobType job_type) { |
Sree Kuchibhotla | 83d0bfa | 2018-07-10 11:29:43 -0700 | [diff] [blame] | 334 | return job_type == GRPC_EXECUTOR_SHORT ? &global_scheduler_short |
| 335 | : &global_scheduler_long; |
Craig Tiller | 7a82afd | 2017-07-18 09:40:40 -0700 | [diff] [blame] | 336 | } |