Craig Tiller | a26637f | 2016-05-02 13:36:36 -0700 | [diff] [blame] | 1 | /* |
| 2 | * |
Jan Tattermusch | 7897ae9 | 2017-06-07 22:57:36 +0200 | [diff] [blame] | 3 | * Copyright 2016 gRPC authors. |
Craig Tiller | a26637f | 2016-05-02 13:36:36 -0700 | [diff] [blame] | 4 | * |
Jan Tattermusch | 7897ae9 | 2017-06-07 22:57:36 +0200 | [diff] [blame] | 5 | * Licensed under the Apache License, Version 2.0 (the "License"); |
| 6 | * you may not use this file except in compliance with the License. |
| 7 | * You may obtain a copy of the License at |
Craig Tiller | a26637f | 2016-05-02 13:36:36 -0700 | [diff] [blame] | 8 | * |
Jan Tattermusch | 7897ae9 | 2017-06-07 22:57:36 +0200 | [diff] [blame] | 9 | * http://www.apache.org/licenses/LICENSE-2.0 |
Craig Tiller | a26637f | 2016-05-02 13:36:36 -0700 | [diff] [blame] | 10 | * |
Jan Tattermusch | 7897ae9 | 2017-06-07 22:57:36 +0200 | [diff] [blame] | 11 | * Unless required by applicable law or agreed to in writing, software |
| 12 | * distributed under the License is distributed on an "AS IS" BASIS, |
| 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 14 | * See the License for the specific language governing permissions and |
| 15 | * limitations under the License. |
Craig Tiller | a26637f | 2016-05-02 13:36:36 -0700 | [diff] [blame] | 16 | * |
| 17 | */ |
| 18 | |
Craig Tiller | f7cade1 | 2016-07-07 21:41:10 -0700 | [diff] [blame] | 19 | #include "src/core/lib/iomgr/combiner.h" |
Craig Tiller | 5842a5b | 2016-05-02 12:38:57 -0700 | [diff] [blame] | 20 | |
Craig Tiller | 18f09a0 | 2017-02-23 17:10:04 -0800 | [diff] [blame] | 21 | #include <assert.h> |
Yash Tibrewal | fcd26bc | 2017-09-25 15:08:28 -0700 | [diff] [blame] | 22 | #include <inttypes.h> |
Craig Tiller | a26637f | 2016-05-02 13:36:36 -0700 | [diff] [blame] | 23 | #include <string.h> |
| 24 | |
| 25 | #include <grpc/support/alloc.h> |
| 26 | #include <grpc/support/log.h> |
| 27 | |
Craig Tiller | 57bb9a9 | 2017-08-31 16:44:15 -0700 | [diff] [blame] | 28 | #include "src/core/lib/debug/stats.h" |
Craig Tiller | ee4b145 | 2017-05-12 10:56:03 -0700 | [diff] [blame] | 29 | #include "src/core/lib/iomgr/executor.h" |
Craig Tiller | c69d27b | 2016-07-14 13:20:22 -0700 | [diff] [blame] | 30 | #include "src/core/lib/profiling/timers.h" |
| 31 | |
ncteisen | 7712c7c | 2017-07-12 23:11:27 -0700 | [diff] [blame] | 32 | grpc_tracer_flag grpc_combiner_trace = |
| 33 | GRPC_TRACER_INITIALIZER(false, "combiner"); |
Craig Tiller | e7603b8 | 2016-07-18 15:43:42 -0700 | [diff] [blame] | 34 | |
Craig Tiller | 84f75d4 | 2017-05-03 13:06:35 -0700 | [diff] [blame] | 35 | #define GRPC_COMBINER_TRACE(fn) \ |
| 36 | do { \ |
| 37 | if (GRPC_TRACER_ON(grpc_combiner_trace)) { \ |
| 38 | fn; \ |
| 39 | } \ |
Craig Tiller | c3df7b4 | 2016-07-18 15:51:26 -0700 | [diff] [blame] | 40 | } while (0) |
Craig Tiller | e7603b8 | 2016-07-18 15:43:42 -0700 | [diff] [blame] | 41 | |
Craig Tiller | d80a8c9 | 2016-10-10 13:19:56 -0700 | [diff] [blame] | 42 | #define STATE_UNORPHANED 1 |
| 43 | #define STATE_ELEM_COUNT_LOW_BIT 2 |
| 44 | |
Craig Tiller | f7cade1 | 2016-07-07 21:41:10 -0700 | [diff] [blame] | 45 | struct grpc_combiner { |
Craig Tiller | dfd3a8f | 2016-08-24 09:43:45 -0700 | [diff] [blame] | 46 | grpc_combiner *next_combiner_on_this_exec_ctx; |
Craig Tiller | ee4b145 | 2017-05-12 10:56:03 -0700 | [diff] [blame] | 47 | grpc_closure_scheduler scheduler; |
| 48 | grpc_closure_scheduler finally_scheduler; |
Craig Tiller | cf600c9 | 2016-05-03 08:26:56 -0700 | [diff] [blame] | 49 | gpr_mpscq queue; |
Craig Tiller | 68608c5 | 2017-05-18 06:27:29 -0700 | [diff] [blame] | 50 | // either: |
| 51 | // a pointer to the initiating exec ctx if that is the only exec_ctx that has |
| 52 | // ever queued to this combiner, or NULL. If this is non-null, it's not |
| 53 | // dereferencable (since the initiating exec_ctx may have gone out of scope) |
| 54 | gpr_atm initiating_exec_ctx_or_null; |
Craig Tiller | cf600c9 | 2016-05-03 08:26:56 -0700 | [diff] [blame] | 55 | // state is: |
Craig Tiller | d80a8c9 | 2016-10-10 13:19:56 -0700 | [diff] [blame] | 56 | // lower bit - zero if orphaned (STATE_UNORPHANED) |
| 57 | // other bits - number of items queued on the lock (STATE_ELEM_COUNT_LOW_BIT) |
Craig Tiller | cf600c9 | 2016-05-03 08:26:56 -0700 | [diff] [blame] | 58 | gpr_atm state; |
Craig Tiller | dfd3a8f | 2016-08-24 09:43:45 -0700 | [diff] [blame] | 59 | bool time_to_execute_final_list; |
Craig Tiller | a36857d | 2016-07-08 16:57:42 -0700 | [diff] [blame] | 60 | grpc_closure_list final_list; |
Craig Tiller | dfd3a8f | 2016-08-24 09:43:45 -0700 | [diff] [blame] | 61 | grpc_closure offload; |
Craig Tiller | 5634ef6 | 2017-02-09 14:25:32 -0800 | [diff] [blame] | 62 | gpr_refcount refs; |
Craig Tiller | cf600c9 | 2016-05-03 08:26:56 -0700 | [diff] [blame] | 63 | }; |
| 64 | |
Craig Tiller | ee4b145 | 2017-05-12 10:56:03 -0700 | [diff] [blame] | 65 | static void combiner_exec(grpc_exec_ctx *exec_ctx, grpc_closure *closure, |
| 66 | grpc_error *error); |
| 67 | static void combiner_finally_exec(grpc_exec_ctx *exec_ctx, |
Craig Tiller | 91031da | 2016-12-28 15:44:25 -0800 | [diff] [blame] | 68 | grpc_closure *closure, grpc_error *error); |
Craig Tiller | 91031da | 2016-12-28 15:44:25 -0800 | [diff] [blame] | 69 | |
Craig Tiller | ee4b145 | 2017-05-12 10:56:03 -0700 | [diff] [blame] | 70 | static const grpc_closure_scheduler_vtable scheduler = { |
| 71 | combiner_exec, combiner_exec, "combiner:immediately"}; |
| 72 | static const grpc_closure_scheduler_vtable finally_scheduler = { |
| 73 | combiner_finally_exec, combiner_finally_exec, "combiner:finally"}; |
Craig Tiller | 91031da | 2016-12-28 15:44:25 -0800 | [diff] [blame] | 74 | |
Craig Tiller | dfd3a8f | 2016-08-24 09:43:45 -0700 | [diff] [blame] | 75 | static void offload(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error); |
| 76 | |
Craig Tiller | ee4b145 | 2017-05-12 10:56:03 -0700 | [diff] [blame] | 77 | grpc_combiner *grpc_combiner_create(void) { |
Yash Tibrewal | ca3c1c0 | 2017-09-07 22:47:16 -0700 | [diff] [blame] | 78 | grpc_combiner *lock = (grpc_combiner *)gpr_zalloc(sizeof(*lock)); |
Craig Tiller | 5634ef6 | 2017-02-09 14:25:32 -0800 | [diff] [blame] | 79 | gpr_ref_init(&lock->refs, 1); |
Craig Tiller | ee4b145 | 2017-05-12 10:56:03 -0700 | [diff] [blame] | 80 | lock->scheduler.vtable = &scheduler; |
| 81 | lock->finally_scheduler.vtable = &finally_scheduler; |
Craig Tiller | d80a8c9 | 2016-10-10 13:19:56 -0700 | [diff] [blame] | 82 | gpr_atm_no_barrier_store(&lock->state, STATE_UNORPHANED); |
Craig Tiller | ad3c8c1 | 2016-05-02 21:47:30 -0700 | [diff] [blame] | 83 | gpr_mpscq_init(&lock->queue); |
Craig Tiller | a36857d | 2016-07-08 16:57:42 -0700 | [diff] [blame] | 84 | grpc_closure_list_init(&lock->final_list); |
Craig Tiller | 7a82afd | 2017-07-18 09:40:40 -0700 | [diff] [blame] | 85 | GRPC_CLOSURE_INIT(&lock->offload, offload, lock, |
| 86 | grpc_executor_scheduler(GRPC_EXECUTOR_SHORT)); |
Craig Tiller | c3df7b4 | 2016-07-18 15:51:26 -0700 | [diff] [blame] | 87 | GRPC_COMBINER_TRACE(gpr_log(GPR_DEBUG, "C:%p create", lock)); |
Craig Tiller | cf600c9 | 2016-05-03 08:26:56 -0700 | [diff] [blame] | 88 | return lock; |
Craig Tiller | 5842a5b | 2016-05-02 12:38:57 -0700 | [diff] [blame] | 89 | } |
| 90 | |
Craig Tiller | e0221ff | 2016-07-11 15:56:08 -0700 | [diff] [blame] | 91 | static void really_destroy(grpc_exec_ctx *exec_ctx, grpc_combiner *lock) { |
Craig Tiller | c3df7b4 | 2016-07-18 15:51:26 -0700 | [diff] [blame] | 92 | GRPC_COMBINER_TRACE(gpr_log(GPR_DEBUG, "C:%p really_destroy", lock)); |
Craig Tiller | cf600c9 | 2016-05-03 08:26:56 -0700 | [diff] [blame] | 93 | GPR_ASSERT(gpr_atm_no_barrier_load(&lock->state) == 0); |
Craig Tiller | ad3c8c1 | 2016-05-02 21:47:30 -0700 | [diff] [blame] | 94 | gpr_mpscq_destroy(&lock->queue); |
Craig Tiller | 14e3d9b | 2016-05-03 08:33:56 -0700 | [diff] [blame] | 95 | gpr_free(lock); |
Craig Tiller | 5842a5b | 2016-05-02 12:38:57 -0700 | [diff] [blame] | 96 | } |
| 97 | |
Craig Tiller | 5634ef6 | 2017-02-09 14:25:32 -0800 | [diff] [blame] | 98 | static void start_destroy(grpc_exec_ctx *exec_ctx, grpc_combiner *lock) { |
Craig Tiller | d80a8c9 | 2016-10-10 13:19:56 -0700 | [diff] [blame] | 99 | gpr_atm old_state = gpr_atm_full_fetch_add(&lock->state, -STATE_UNORPHANED); |
Craig Tiller | c3df7b4 | 2016-07-18 15:51:26 -0700 | [diff] [blame] | 100 | GRPC_COMBINER_TRACE(gpr_log( |
| 101 | GPR_DEBUG, "C:%p really_destroy old_state=%" PRIdPTR, lock, old_state)); |
Craig Tiller | e7603b8 | 2016-07-18 15:43:42 -0700 | [diff] [blame] | 102 | if (old_state == 1) { |
Craig Tiller | e0221ff | 2016-07-11 15:56:08 -0700 | [diff] [blame] | 103 | really_destroy(exec_ctx, lock); |
Craig Tiller | 5842a5b | 2016-05-02 12:38:57 -0700 | [diff] [blame] | 104 | } |
| 105 | } |
| 106 | |
ncteisen | a135485 | 2017-06-08 16:25:53 -0700 | [diff] [blame] | 107 | #ifndef NDEBUG |
ncteisen | 3bc2558 | 2017-06-09 10:35:35 -0700 | [diff] [blame] | 108 | #define GRPC_COMBINER_DEBUG_SPAM(op, delta) \ |
| 109 | if (GRPC_TRACER_ON(grpc_combiner_trace)) { \ |
| 110 | gpr_log(file, line, GPR_LOG_SEVERITY_DEBUG, \ |
| 111 | "C:%p %s %" PRIdPTR " --> %" PRIdPTR " %s", lock, (op), \ |
| 112 | gpr_atm_no_barrier_load(&lock->refs.count), \ |
| 113 | gpr_atm_no_barrier_load(&lock->refs.count) + (delta), reason); \ |
| 114 | } |
Craig Tiller | 3845e55 | 2017-02-09 15:19:28 -0800 | [diff] [blame] | 115 | #else |
| 116 | #define GRPC_COMBINER_DEBUG_SPAM(op, delta) |
| 117 | #endif |
| 118 | |
| 119 | void grpc_combiner_unref(grpc_exec_ctx *exec_ctx, |
| 120 | grpc_combiner *lock GRPC_COMBINER_DEBUG_ARGS) { |
| 121 | GRPC_COMBINER_DEBUG_SPAM("UNREF", -1); |
Craig Tiller | 5634ef6 | 2017-02-09 14:25:32 -0800 | [diff] [blame] | 122 | if (gpr_unref(&lock->refs)) { |
| 123 | start_destroy(exec_ctx, lock); |
| 124 | } |
| 125 | } |
| 126 | |
Craig Tiller | 3845e55 | 2017-02-09 15:19:28 -0800 | [diff] [blame] | 127 | grpc_combiner *grpc_combiner_ref(grpc_combiner *lock GRPC_COMBINER_DEBUG_ARGS) { |
| 128 | GRPC_COMBINER_DEBUG_SPAM(" REF", 1); |
| 129 | gpr_ref(&lock->refs); |
| 130 | return lock; |
| 131 | } |
Craig Tiller | 5634ef6 | 2017-02-09 14:25:32 -0800 | [diff] [blame] | 132 | |
Craig Tiller | 86037cd0 | 2016-09-02 19:58:43 -0700 | [diff] [blame] | 133 | static void push_last_on_exec_ctx(grpc_exec_ctx *exec_ctx, |
| 134 | grpc_combiner *lock) { |
Craig Tiller | dfd3a8f | 2016-08-24 09:43:45 -0700 | [diff] [blame] | 135 | lock->next_combiner_on_this_exec_ctx = NULL; |
| 136 | if (exec_ctx->active_combiner == NULL) { |
| 137 | exec_ctx->active_combiner = exec_ctx->last_combiner = lock; |
Craig Tiller | 4e43685 | 2016-07-10 14:01:50 -0700 | [diff] [blame] | 138 | } else { |
Craig Tiller | dfd3a8f | 2016-08-24 09:43:45 -0700 | [diff] [blame] | 139 | exec_ctx->last_combiner->next_combiner_on_this_exec_ctx = lock; |
| 140 | exec_ctx->last_combiner = lock; |
Craig Tiller | 4e43685 | 2016-07-10 14:01:50 -0700 | [diff] [blame] | 141 | } |
Craig Tiller | cf600c9 | 2016-05-03 08:26:56 -0700 | [diff] [blame] | 142 | } |
| 143 | |
Craig Tiller | 86037cd0 | 2016-09-02 19:58:43 -0700 | [diff] [blame] | 144 | static void push_first_on_exec_ctx(grpc_exec_ctx *exec_ctx, |
| 145 | grpc_combiner *lock) { |
| 146 | lock->next_combiner_on_this_exec_ctx = exec_ctx->active_combiner; |
| 147 | exec_ctx->active_combiner = lock; |
| 148 | if (lock->next_combiner_on_this_exec_ctx == NULL) { |
| 149 | exec_ctx->last_combiner = lock; |
| 150 | } |
| 151 | } |
| 152 | |
Craig Tiller | ee4b145 | 2017-05-12 10:56:03 -0700 | [diff] [blame] | 153 | #define COMBINER_FROM_CLOSURE_SCHEDULER(closure, scheduler_name) \ |
| 154 | ((grpc_combiner *)(((char *)((closure)->scheduler)) - \ |
| 155 | offsetof(grpc_combiner, scheduler_name))) |
| 156 | |
| 157 | static void combiner_exec(grpc_exec_ctx *exec_ctx, grpc_closure *cl, |
| 158 | grpc_error *error) { |
Craig Tiller | 57bb9a9 | 2017-08-31 16:44:15 -0700 | [diff] [blame] | 159 | GRPC_STATS_INC_COMBINER_LOCKS_SCHEDULED_ITEMS(exec_ctx); |
Craig Tiller | c69d27b | 2016-07-14 13:20:22 -0700 | [diff] [blame] | 160 | GPR_TIMER_BEGIN("combiner.execute", 0); |
Craig Tiller | ee4b145 | 2017-05-12 10:56:03 -0700 | [diff] [blame] | 161 | grpc_combiner *lock = COMBINER_FROM_CLOSURE_SCHEDULER(cl, scheduler); |
Craig Tiller | d80a8c9 | 2016-10-10 13:19:56 -0700 | [diff] [blame] | 162 | gpr_atm last = gpr_atm_full_fetch_add(&lock->state, STATE_ELEM_COUNT_LOW_BIT); |
Craig Tiller | ee4b145 | 2017-05-12 10:56:03 -0700 | [diff] [blame] | 163 | GRPC_COMBINER_TRACE(gpr_log(GPR_DEBUG, |
| 164 | "C:%p grpc_combiner_execute c=%p last=%" PRIdPTR, |
| 165 | lock, cl, last)); |
Craig Tiller | 68608c5 | 2017-05-18 06:27:29 -0700 | [diff] [blame] | 166 | if (last == 1) { |
Craig Tiller | 57bb9a9 | 2017-08-31 16:44:15 -0700 | [diff] [blame] | 167 | GRPC_STATS_INC_COMBINER_LOCKS_INITIATED(exec_ctx); |
Craig Tiller | 68608c5 | 2017-05-18 06:27:29 -0700 | [diff] [blame] | 168 | gpr_atm_no_barrier_store(&lock->initiating_exec_ctx_or_null, |
| 169 | (gpr_atm)exec_ctx); |
| 170 | // first element on this list: add it to the list of combiner locks |
| 171 | // executing within this exec_ctx |
| 172 | push_last_on_exec_ctx(exec_ctx, lock); |
| 173 | } else { |
| 174 | // there may be a race with setting here: if that happens, we may delay |
| 175 | // offload for one or two actions, and that's fine |
| 176 | gpr_atm initiator = |
| 177 | gpr_atm_no_barrier_load(&lock->initiating_exec_ctx_or_null); |
| 178 | if (initiator != 0 && initiator != (gpr_atm)exec_ctx) { |
| 179 | gpr_atm_no_barrier_store(&lock->initiating_exec_ctx_or_null, 0); |
| 180 | } |
| 181 | } |
Craig Tiller | d80a8c9 | 2016-10-10 13:19:56 -0700 | [diff] [blame] | 182 | GPR_ASSERT(last & STATE_UNORPHANED); // ensure lock has not been destroyed |
Craig Tiller | 18f09a0 | 2017-02-23 17:10:04 -0800 | [diff] [blame] | 183 | assert(cl->cb); |
Craig Tiller | ee4b145 | 2017-05-12 10:56:03 -0700 | [diff] [blame] | 184 | cl->error_data.error = error; |
Craig Tiller | dfd3a8f | 2016-08-24 09:43:45 -0700 | [diff] [blame] | 185 | gpr_mpscq_push(&lock->queue, &cl->next_data.atm_next); |
Craig Tiller | c69d27b | 2016-07-14 13:20:22 -0700 | [diff] [blame] | 186 | GPR_TIMER_END("combiner.execute", 0); |
Craig Tiller | 5842a5b | 2016-05-02 12:38:57 -0700 | [diff] [blame] | 187 | } |
Craig Tiller | a36857d | 2016-07-08 16:57:42 -0700 | [diff] [blame] | 188 | |
Craig Tiller | dfd3a8f | 2016-08-24 09:43:45 -0700 | [diff] [blame] | 189 | static void move_next(grpc_exec_ctx *exec_ctx) { |
| 190 | exec_ctx->active_combiner = |
| 191 | exec_ctx->active_combiner->next_combiner_on_this_exec_ctx; |
| 192 | if (exec_ctx->active_combiner == NULL) { |
| 193 | exec_ctx->last_combiner = NULL; |
| 194 | } |
| 195 | } |
| 196 | |
| 197 | static void offload(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) { |
Yash Tibrewal | ca3c1c0 | 2017-09-07 22:47:16 -0700 | [diff] [blame] | 198 | grpc_combiner *lock = (grpc_combiner *)arg; |
Craig Tiller | 86037cd0 | 2016-09-02 19:58:43 -0700 | [diff] [blame] | 199 | push_last_on_exec_ctx(exec_ctx, lock); |
Craig Tiller | dfd3a8f | 2016-08-24 09:43:45 -0700 | [diff] [blame] | 200 | } |
| 201 | |
| 202 | static void queue_offload(grpc_exec_ctx *exec_ctx, grpc_combiner *lock) { |
Craig Tiller | 57bb9a9 | 2017-08-31 16:44:15 -0700 | [diff] [blame] | 203 | GRPC_STATS_INC_COMBINER_LOCKS_OFFLOADED(exec_ctx); |
Craig Tiller | dfd3a8f | 2016-08-24 09:43:45 -0700 | [diff] [blame] | 204 | move_next(exec_ctx); |
Craig Tiller | ee4b145 | 2017-05-12 10:56:03 -0700 | [diff] [blame] | 205 | GRPC_COMBINER_TRACE(gpr_log(GPR_DEBUG, "C:%p queue_offload", lock)); |
ncteisen | 969b46e | 2017-06-08 14:57:11 -0700 | [diff] [blame] | 206 | GRPC_CLOSURE_SCHED(exec_ctx, &lock->offload, GRPC_ERROR_NONE); |
Craig Tiller | dfd3a8f | 2016-08-24 09:43:45 -0700 | [diff] [blame] | 207 | } |
| 208 | |
| 209 | bool grpc_combiner_continue_exec_ctx(grpc_exec_ctx *exec_ctx) { |
| 210 | GPR_TIMER_BEGIN("combiner.continue_exec_ctx", 0); |
| 211 | grpc_combiner *lock = exec_ctx->active_combiner; |
| 212 | if (lock == NULL) { |
| 213 | GPR_TIMER_END("combiner.continue_exec_ctx", 0); |
| 214 | return false; |
| 215 | } |
| 216 | |
Craig Tiller | 3e6c777 | 2017-06-06 08:37:40 -0700 | [diff] [blame] | 217 | bool contended = |
| 218 | gpr_atm_no_barrier_load(&lock->initiating_exec_ctx_or_null) == 0; |
Craig Tiller | e587a5c | 2017-06-06 15:12:33 +0000 | [diff] [blame] | 219 | |
Craig Tiller | ee4b145 | 2017-05-12 10:56:03 -0700 | [diff] [blame] | 220 | GRPC_COMBINER_TRACE(gpr_log(GPR_DEBUG, |
| 221 | "C:%p grpc_combiner_continue_exec_ctx " |
Craig Tiller | e587a5c | 2017-06-06 15:12:33 +0000 | [diff] [blame] | 222 | "contended=%d " |
Craig Tiller | ee4b145 | 2017-05-12 10:56:03 -0700 | [diff] [blame] | 223 | "exec_ctx_ready_to_finish=%d " |
| 224 | "time_to_execute_final_list=%d", |
Craig Tiller | 3e6c777 | 2017-06-06 08:37:40 -0700 | [diff] [blame] | 225 | lock, contended, |
| 226 | grpc_exec_ctx_ready_to_finish(exec_ctx), |
Craig Tiller | ee4b145 | 2017-05-12 10:56:03 -0700 | [diff] [blame] | 227 | lock->time_to_execute_final_list)); |
Craig Tiller | fc2636d | 2016-09-12 09:57:07 -0700 | [diff] [blame] | 228 | |
Craig Tiller | 3e6c777 | 2017-06-06 08:37:40 -0700 | [diff] [blame] | 229 | if (contended && grpc_exec_ctx_ready_to_finish(exec_ctx) && |
| 230 | grpc_executor_is_threaded()) { |
Craig Tiller | dfd3a8f | 2016-08-24 09:43:45 -0700 | [diff] [blame] | 231 | GPR_TIMER_MARK("offload_from_finished_exec_ctx", 0); |
Craig Tiller | 747216f | 2017-06-08 08:08:11 -0700 | [diff] [blame] | 232 | // this execution context wants to move on: schedule remaining work to be |
| 233 | // picked up on the executor |
Craig Tiller | dfd3a8f | 2016-08-24 09:43:45 -0700 | [diff] [blame] | 234 | queue_offload(exec_ctx, lock); |
| 235 | GPR_TIMER_END("combiner.continue_exec_ctx", 0); |
| 236 | return true; |
| 237 | } |
| 238 | |
| 239 | if (!lock->time_to_execute_final_list || |
| 240 | // peek to see if something new has shown up, and execute that with |
| 241 | // priority |
| 242 | (gpr_atm_acq_load(&lock->state) >> 1) > 1) { |
| 243 | gpr_mpscq_node *n = gpr_mpscq_pop(&lock->queue); |
| 244 | GRPC_COMBINER_TRACE( |
| 245 | gpr_log(GPR_DEBUG, "C:%p maybe_finish_one n=%p", lock, n)); |
| 246 | if (n == NULL) { |
Craig Tiller | d80a8c9 | 2016-10-10 13:19:56 -0700 | [diff] [blame] | 247 | // queue is in an inconsistent state: use this as a cue that we should |
Craig Tiller | dfd3a8f | 2016-08-24 09:43:45 -0700 | [diff] [blame] | 248 | // go off and do something else for a while (and come back later) |
| 249 | GPR_TIMER_MARK("delay_busy", 0); |
Craig Tiller | ee4b145 | 2017-05-12 10:56:03 -0700 | [diff] [blame] | 250 | queue_offload(exec_ctx, lock); |
Craig Tiller | dfd3a8f | 2016-08-24 09:43:45 -0700 | [diff] [blame] | 251 | GPR_TIMER_END("combiner.continue_exec_ctx", 0); |
| 252 | return true; |
| 253 | } |
| 254 | GPR_TIMER_BEGIN("combiner.exec1", 0); |
| 255 | grpc_closure *cl = (grpc_closure *)n; |
Craig Tiller | ee4b145 | 2017-05-12 10:56:03 -0700 | [diff] [blame] | 256 | grpc_error *cl_err = cl->error_data.error; |
ncteisen | f8061e8 | 2017-06-09 10:44:42 -0700 | [diff] [blame] | 257 | #ifndef NDEBUG |
Mark D. Roth | 43f774e | 2017-04-04 16:35:37 -0700 | [diff] [blame] | 258 | cl->scheduled = false; |
| 259 | #endif |
Craig Tiller | ee4b145 | 2017-05-12 10:56:03 -0700 | [diff] [blame] | 260 | cl->cb(exec_ctx, cl->cb_arg, cl_err); |
| 261 | GRPC_ERROR_UNREF(cl_err); |
Craig Tiller | dfd3a8f | 2016-08-24 09:43:45 -0700 | [diff] [blame] | 262 | GPR_TIMER_END("combiner.exec1", 0); |
| 263 | } else { |
Craig Tiller | a7cd41c | 2016-08-31 12:59:24 -0700 | [diff] [blame] | 264 | grpc_closure *c = lock->final_list.head; |
| 265 | GPR_ASSERT(c != NULL); |
| 266 | grpc_closure_list_init(&lock->final_list); |
| 267 | int loops = 0; |
| 268 | while (c != NULL) { |
| 269 | GPR_TIMER_BEGIN("combiner.exec_1final", 0); |
| 270 | GRPC_COMBINER_TRACE( |
| 271 | gpr_log(GPR_DEBUG, "C:%p execute_final[%d] c=%p", lock, loops, c)); |
| 272 | grpc_closure *next = c->next_data.next; |
| 273 | grpc_error *error = c->error_data.error; |
ncteisen | f8061e8 | 2017-06-09 10:44:42 -0700 | [diff] [blame] | 274 | #ifndef NDEBUG |
Mark D. Roth | 43f774e | 2017-04-04 16:35:37 -0700 | [diff] [blame] | 275 | c->scheduled = false; |
| 276 | #endif |
Craig Tiller | a7cd41c | 2016-08-31 12:59:24 -0700 | [diff] [blame] | 277 | c->cb(exec_ctx, c->cb_arg, error); |
| 278 | GRPC_ERROR_UNREF(error); |
| 279 | c = next; |
| 280 | GPR_TIMER_END("combiner.exec_1final", 0); |
Craig Tiller | dfd3a8f | 2016-08-24 09:43:45 -0700 | [diff] [blame] | 281 | } |
| 282 | } |
| 283 | |
| 284 | GPR_TIMER_MARK("unref", 0); |
Craig Tiller | 86037cd0 | 2016-09-02 19:58:43 -0700 | [diff] [blame] | 285 | move_next(exec_ctx); |
| 286 | lock->time_to_execute_final_list = false; |
Craig Tiller | d80a8c9 | 2016-10-10 13:19:56 -0700 | [diff] [blame] | 287 | gpr_atm old_state = |
| 288 | gpr_atm_full_fetch_add(&lock->state, -STATE_ELEM_COUNT_LOW_BIT); |
Craig Tiller | dfd3a8f | 2016-08-24 09:43:45 -0700 | [diff] [blame] | 289 | GRPC_COMBINER_TRACE( |
| 290 | gpr_log(GPR_DEBUG, "C:%p finish old_state=%" PRIdPTR, lock, old_state)); |
Craig Tiller | d3ee0d5 | 2016-10-10 14:09:18 -0700 | [diff] [blame] | 291 | // Define a macro to ease readability of the following switch statement. |
| 292 | #define OLD_STATE_WAS(orphaned, elem_count) \ |
| 293 | (((orphaned) ? 0 : STATE_UNORPHANED) | \ |
| 294 | ((elem_count)*STATE_ELEM_COUNT_LOW_BIT)) |
| 295 | // Depending on what the previous state was, we need to perform different |
| 296 | // actions. |
Craig Tiller | dfd3a8f | 2016-08-24 09:43:45 -0700 | [diff] [blame] | 297 | switch (old_state) { |
Craig Tiller | 56f21aa | 2016-09-02 09:29:43 -0700 | [diff] [blame] | 298 | default: |
| 299 | // we have multiple queued work items: just continue executing them |
| 300 | break; |
Craig Tiller | d3ee0d5 | 2016-10-10 14:09:18 -0700 | [diff] [blame] | 301 | case OLD_STATE_WAS(false, 2): |
| 302 | case OLD_STATE_WAS(true, 2): |
Craig Tiller | d80a8c9 | 2016-10-10 13:19:56 -0700 | [diff] [blame] | 303 | // we're down to one queued item: if it's the final list we should do that |
Craig Tiller | dfd3a8f | 2016-08-24 09:43:45 -0700 | [diff] [blame] | 304 | if (!grpc_closure_list_empty(lock->final_list)) { |
| 305 | lock->time_to_execute_final_list = true; |
| 306 | } |
| 307 | break; |
Craig Tiller | d3ee0d5 | 2016-10-10 14:09:18 -0700 | [diff] [blame] | 308 | case OLD_STATE_WAS(false, 1): |
Craig Tiller | d80a8c9 | 2016-10-10 13:19:56 -0700 | [diff] [blame] | 309 | // had one count, one unorphaned --> unlocked unorphaned |
Craig Tiller | dfd3a8f | 2016-08-24 09:43:45 -0700 | [diff] [blame] | 310 | GPR_TIMER_END("combiner.continue_exec_ctx", 0); |
| 311 | return true; |
Craig Tiller | d3ee0d5 | 2016-10-10 14:09:18 -0700 | [diff] [blame] | 312 | case OLD_STATE_WAS(true, 1): |
Craig Tiller | d80a8c9 | 2016-10-10 13:19:56 -0700 | [diff] [blame] | 313 | // and one count, one orphaned --> unlocked and orphaned |
Craig Tiller | dfd3a8f | 2016-08-24 09:43:45 -0700 | [diff] [blame] | 314 | really_destroy(exec_ctx, lock); |
| 315 | GPR_TIMER_END("combiner.continue_exec_ctx", 0); |
| 316 | return true; |
Craig Tiller | d3ee0d5 | 2016-10-10 14:09:18 -0700 | [diff] [blame] | 317 | case OLD_STATE_WAS(false, 0): |
| 318 | case OLD_STATE_WAS(true, 0): |
Craig Tiller | dfd3a8f | 2016-08-24 09:43:45 -0700 | [diff] [blame] | 319 | // these values are illegal - representing an already unlocked or |
| 320 | // deleted lock |
| 321 | GPR_TIMER_END("combiner.continue_exec_ctx", 0); |
| 322 | GPR_UNREACHABLE_CODE(return true); |
| 323 | } |
Craig Tiller | 86037cd0 | 2016-09-02 19:58:43 -0700 | [diff] [blame] | 324 | push_first_on_exec_ctx(exec_ctx, lock); |
Craig Tiller | dfd3a8f | 2016-08-24 09:43:45 -0700 | [diff] [blame] | 325 | GPR_TIMER_END("combiner.continue_exec_ctx", 0); |
| 326 | return true; |
| 327 | } |
| 328 | |
Craig Tiller | e0221ff | 2016-07-11 15:56:08 -0700 | [diff] [blame] | 329 | static void enqueue_finally(grpc_exec_ctx *exec_ctx, void *closure, |
Craig Tiller | 91031da | 2016-12-28 15:44:25 -0800 | [diff] [blame] | 330 | grpc_error *error); |
Craig Tiller | e0221ff | 2016-07-11 15:56:08 -0700 | [diff] [blame] | 331 | |
Craig Tiller | ee4b145 | 2017-05-12 10:56:03 -0700 | [diff] [blame] | 332 | static void combiner_finally_exec(grpc_exec_ctx *exec_ctx, |
| 333 | grpc_closure *closure, grpc_error *error) { |
Craig Tiller | 57bb9a9 | 2017-08-31 16:44:15 -0700 | [diff] [blame] | 334 | GRPC_STATS_INC_COMBINER_LOCKS_SCHEDULED_FINAL_ITEMS(exec_ctx); |
Craig Tiller | ee4b145 | 2017-05-12 10:56:03 -0700 | [diff] [blame] | 335 | grpc_combiner *lock = |
| 336 | COMBINER_FROM_CLOSURE_SCHEDULER(closure, finally_scheduler); |
| 337 | GRPC_COMBINER_TRACE(gpr_log(GPR_DEBUG, |
| 338 | "C:%p grpc_combiner_execute_finally c=%p; ac=%p", |
| 339 | lock, closure, exec_ctx->active_combiner)); |
Craig Tiller | c69d27b | 2016-07-14 13:20:22 -0700 | [diff] [blame] | 340 | GPR_TIMER_BEGIN("combiner.execute_finally", 0); |
Craig Tiller | e0221ff | 2016-07-11 15:56:08 -0700 | [diff] [blame] | 341 | if (exec_ctx->active_combiner != lock) { |
Craig Tiller | c69d27b | 2016-07-14 13:20:22 -0700 | [diff] [blame] | 342 | GPR_TIMER_MARK("slowpath", 0); |
ncteisen | 969b46e | 2017-06-08 14:57:11 -0700 | [diff] [blame] | 343 | GRPC_CLOSURE_SCHED(exec_ctx, |
| 344 | GRPC_CLOSURE_CREATE(enqueue_finally, closure, |
Craig Tiller | ee4b145 | 2017-05-12 10:56:03 -0700 | [diff] [blame] | 345 | grpc_combiner_scheduler(lock)), |
| 346 | error); |
Craig Tiller | c69d27b | 2016-07-14 13:20:22 -0700 | [diff] [blame] | 347 | GPR_TIMER_END("combiner.execute_finally", 0); |
Craig Tiller | e0221ff | 2016-07-11 15:56:08 -0700 | [diff] [blame] | 348 | return; |
| 349 | } |
| 350 | |
Craig Tiller | a36857d | 2016-07-08 16:57:42 -0700 | [diff] [blame] | 351 | if (grpc_closure_list_empty(lock->final_list)) { |
Craig Tiller | d80a8c9 | 2016-10-10 13:19:56 -0700 | [diff] [blame] | 352 | gpr_atm_full_fetch_add(&lock->state, STATE_ELEM_COUNT_LOW_BIT); |
Craig Tiller | a36857d | 2016-07-08 16:57:42 -0700 | [diff] [blame] | 353 | } |
| 354 | grpc_closure_list_append(&lock->final_list, closure, error); |
Craig Tiller | e7603b8 | 2016-07-18 15:43:42 -0700 | [diff] [blame] | 355 | GPR_TIMER_END("combiner.execute_finally", 0); |
Craig Tiller | a36857d | 2016-07-08 16:57:42 -0700 | [diff] [blame] | 356 | } |
Craig Tiller | 91031da | 2016-12-28 15:44:25 -0800 | [diff] [blame] | 357 | |
| 358 | static void enqueue_finally(grpc_exec_ctx *exec_ctx, void *closure, |
| 359 | grpc_error *error) { |
Yash Tibrewal | 52778c4 | 2017-09-11 15:00:11 -0700 | [diff] [blame] | 360 | combiner_finally_exec(exec_ctx, (grpc_closure *)closure, |
| 361 | GRPC_ERROR_REF(error)); |
Craig Tiller | 91031da | 2016-12-28 15:44:25 -0800 | [diff] [blame] | 362 | } |
| 363 | |
Craig Tiller | ee4b145 | 2017-05-12 10:56:03 -0700 | [diff] [blame] | 364 | grpc_closure_scheduler *grpc_combiner_scheduler(grpc_combiner *combiner) { |
| 365 | return &combiner->scheduler; |
Craig Tiller | 91031da | 2016-12-28 15:44:25 -0800 | [diff] [blame] | 366 | } |
| 367 | |
| 368 | grpc_closure_scheduler *grpc_combiner_finally_scheduler( |
Craig Tiller | ee4b145 | 2017-05-12 10:56:03 -0700 | [diff] [blame] | 369 | grpc_combiner *combiner) { |
| 370 | return &combiner->finally_scheduler; |
Craig Tiller | 91031da | 2016-12-28 15:44:25 -0800 | [diff] [blame] | 371 | } |