ctiller | 58393c2 | 2015-01-07 14:03:30 -0800 | [diff] [blame] | 1 | /* |
| 2 | * |
Craig Tiller | a93a25f | 2016-01-28 13:55:49 -0800 | [diff] [blame] | 3 | * Copyright 2015-2016, Google Inc. |
ctiller | 58393c2 | 2015-01-07 14:03:30 -0800 | [diff] [blame] | 4 | * All rights reserved. |
| 5 | * |
| 6 | * Redistribution and use in source and binary forms, with or without |
| 7 | * modification, are permitted provided that the following conditions are |
| 8 | * met: |
| 9 | * |
| 10 | * * Redistributions of source code must retain the above copyright |
| 11 | * notice, this list of conditions and the following disclaimer. |
| 12 | * * Redistributions in binary form must reproduce the above |
| 13 | * copyright notice, this list of conditions and the following disclaimer |
| 14 | * in the documentation and/or other materials provided with the |
| 15 | * distribution. |
| 16 | * * Neither the name of Google Inc. nor the names of its |
| 17 | * contributors may be used to endorse or promote products derived from |
| 18 | * this software without specific prior written permission. |
| 19 | * |
| 20 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS |
| 21 | * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT |
| 22 | * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR |
| 23 | * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT |
| 24 | * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
| 25 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
| 26 | * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
| 27 | * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
| 28 | * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
| 29 | * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
| 30 | * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
| 31 | * |
| 32 | */ |
| 33 | |
Craig Tiller | d14a1a5 | 2015-01-21 15:26:29 -0800 | [diff] [blame] | 34 | #include <grpc/support/port_platform.h> |
| 35 | |
| 36 | #ifdef GPR_POSIX_SOCKET |
| 37 | |
ctiller | 58393c2 | 2015-01-07 14:03:30 -0800 | [diff] [blame] | 38 | #include "src/core/iomgr/pollset_posix.h" |
| 39 | |
| 40 | #include <errno.h> |
ctiller | 58393c2 | 2015-01-07 14:03:30 -0800 | [diff] [blame] | 41 | #include <stdlib.h> |
| 42 | #include <string.h> |
| 43 | #include <unistd.h> |
| 44 | |
David Garcia Quintas | f747bbc | 2015-10-04 23:09:47 -0700 | [diff] [blame] | 45 | #include "src/core/iomgr/timer_internal.h" |
ctiller | 58393c2 | 2015-01-07 14:03:30 -0800 | [diff] [blame] | 46 | #include "src/core/iomgr/fd_posix.h" |
| 47 | #include "src/core/iomgr/iomgr_internal.h" |
| 48 | #include "src/core/iomgr/socket_utils_posix.h" |
Craig Tiller | 1be373c | 2015-04-27 07:58:16 -0700 | [diff] [blame] | 49 | #include "src/core/profiling/timers.h" |
vjpai | 9839d28 | 2015-09-24 17:55:18 -0700 | [diff] [blame] | 50 | #include "src/core/support/block_annotate.h" |
ctiller | 58393c2 | 2015-01-07 14:03:30 -0800 | [diff] [blame] | 51 | #include <grpc/support/alloc.h> |
| 52 | #include <grpc/support/log.h> |
| 53 | #include <grpc/support/thd.h> |
Craig Tiller | 1be373c | 2015-04-27 07:58:16 -0700 | [diff] [blame] | 54 | #include <grpc/support/tls.h> |
ctiller | 58393c2 | 2015-01-07 14:03:30 -0800 | [diff] [blame] | 55 | #include <grpc/support/useful.h> |
| 56 | |
Craig Tiller | a82950e | 2015-09-22 12:33:20 -0700 | [diff] [blame] | 57 | GPR_TLS_DECL(g_current_thread_poller); |
| 58 | GPR_TLS_DECL(g_current_thread_worker); |
Craig Tiller | 1be373c | 2015-04-27 07:58:16 -0700 | [diff] [blame] | 59 | |
Craig Tiller | 191b79c | 2015-09-30 10:49:58 -0700 | [diff] [blame] | 60 | /** Default poll() function - a pointer so that it can be overridden by some |
| 61 | * tests */ |
Craig Tiller | 69f90e6 | 2015-08-06 08:32:35 -0700 | [diff] [blame] | 62 | grpc_poll_function_type grpc_poll_function = poll; |
Craig Tiller | 191b79c | 2015-09-30 10:49:58 -0700 | [diff] [blame] | 63 | |
| 64 | /** The alarm system needs to be able to wakeup 'some poller' sometimes |
| 65 | * (specifically when a new alarm needs to be triggered earlier than the next |
| 66 | * alarm 'epoch'). |
| 67 | * This wakeup_fd gives us something to alert on when such a case occurs. */ |
Craig Tiller | 8afeec8 | 2015-09-28 17:03:34 -0700 | [diff] [blame] | 68 | grpc_wakeup_fd grpc_global_wakeup_fd; |
Craig Tiller | 69f90e6 | 2015-08-06 08:32:35 -0700 | [diff] [blame] | 69 | |
Craig Tiller | a82950e | 2015-09-22 12:33:20 -0700 | [diff] [blame] | 70 | static void remove_worker(grpc_pollset *p, grpc_pollset_worker *worker) { |
Craig Tiller | 5ddbb9d | 2015-07-29 15:58:11 -0700 | [diff] [blame] | 71 | worker->prev->next = worker->next; |
| 72 | worker->next->prev = worker->prev; |
| 73 | } |
| 74 | |
Craig Tiller | a82950e | 2015-09-22 12:33:20 -0700 | [diff] [blame] | 75 | int grpc_pollset_has_workers(grpc_pollset *p) { |
Craig Tiller | 5ddbb9d | 2015-07-29 15:58:11 -0700 | [diff] [blame] | 76 | return p->root_worker.next != &p->root_worker; |
| 77 | } |
| 78 | |
Craig Tiller | a82950e | 2015-09-22 12:33:20 -0700 | [diff] [blame] | 79 | static grpc_pollset_worker *pop_front_worker(grpc_pollset *p) { |
| 80 | if (grpc_pollset_has_workers(p)) { |
| 81 | grpc_pollset_worker *w = p->root_worker.next; |
| 82 | remove_worker(p, w); |
| 83 | return w; |
| 84 | } else { |
| 85 | return NULL; |
| 86 | } |
ctiller | 58393c2 | 2015-01-07 14:03:30 -0800 | [diff] [blame] | 87 | } |
| 88 | |
Craig Tiller | a82950e | 2015-09-22 12:33:20 -0700 | [diff] [blame] | 89 | static void push_back_worker(grpc_pollset *p, grpc_pollset_worker *worker) { |
Craig Tiller | 5ddbb9d | 2015-07-29 15:58:11 -0700 | [diff] [blame] | 90 | worker->next = &p->root_worker; |
| 91 | worker->prev = worker->next->prev; |
| 92 | worker->prev->next = worker->next->prev = worker; |
Craig Tiller | 7d41321 | 2015-02-09 08:00:02 -0800 | [diff] [blame] | 93 | } |
ctiller | 58393c2 | 2015-01-07 14:03:30 -0800 | [diff] [blame] | 94 | |
Craig Tiller | a82950e | 2015-09-22 12:33:20 -0700 | [diff] [blame] | 95 | static void push_front_worker(grpc_pollset *p, grpc_pollset_worker *worker) { |
Craig Tiller | 5ddbb9d | 2015-07-29 15:58:11 -0700 | [diff] [blame] | 96 | worker->prev = &p->root_worker; |
| 97 | worker->next = worker->prev->next; |
| 98 | worker->prev->next = worker->next->prev = worker; |
| 99 | } |
| 100 | |
Craig Tiller | d0a0000 | 2015-10-06 11:30:37 -0700 | [diff] [blame] | 101 | void grpc_pollset_kick_ext(grpc_pollset *p, |
| 102 | grpc_pollset_worker *specific_worker, |
Craig Tiller | 7536af0 | 2015-12-22 13:49:30 -0800 | [diff] [blame] | 103 | uint32_t flags) { |
Craig Tiller | 0ba432d | 2015-10-09 16:57:11 -0700 | [diff] [blame] | 104 | GPR_TIMER_BEGIN("grpc_pollset_kick_ext", 0); |
Craig Tiller | 44011e8 | 2015-10-09 16:16:43 -0700 | [diff] [blame] | 105 | |
Craig Tiller | 4c06b82 | 2015-08-06 08:41:31 -0700 | [diff] [blame] | 106 | /* pollset->mu already held */ |
Craig Tiller | a82950e | 2015-09-22 12:33:20 -0700 | [diff] [blame] | 107 | if (specific_worker != NULL) { |
| 108 | if (specific_worker == GRPC_POLLSET_KICK_BROADCAST) { |
Craig Tiller | 0ba432d | 2015-10-09 16:57:11 -0700 | [diff] [blame] | 109 | GPR_TIMER_BEGIN("grpc_pollset_kick_ext.broadcast", 0); |
Craig Tiller | 988e37f | 2015-10-01 07:53:56 -0700 | [diff] [blame] | 110 | GPR_ASSERT((flags & GRPC_POLLSET_REEVALUATE_POLLING_ON_WAKEUP) == 0); |
Craig Tiller | a82950e | 2015-09-22 12:33:20 -0700 | [diff] [blame] | 111 | for (specific_worker = p->root_worker.next; |
| 112 | specific_worker != &p->root_worker; |
| 113 | specific_worker = specific_worker->next) { |
Craig Tiller | e8b5f62 | 2015-11-02 14:15:03 -0800 | [diff] [blame] | 114 | grpc_wakeup_fd_wakeup(&specific_worker->wakeup_fd->fd); |
Craig Tiller | a82950e | 2015-09-22 12:33:20 -0700 | [diff] [blame] | 115 | } |
| 116 | p->kicked_without_pollers = 1; |
Craig Tiller | 0ba432d | 2015-10-09 16:57:11 -0700 | [diff] [blame] | 117 | GPR_TIMER_END("grpc_pollset_kick_ext.broadcast", 0); |
Craig Tiller | a82950e | 2015-09-22 12:33:20 -0700 | [diff] [blame] | 118 | } else if (gpr_tls_get(&g_current_thread_worker) != |
Craig Tiller | 7536af0 | 2015-12-22 13:49:30 -0800 | [diff] [blame] | 119 | (intptr_t)specific_worker) { |
Craig Tiller | 0ba432d | 2015-10-09 16:57:11 -0700 | [diff] [blame] | 120 | GPR_TIMER_MARK("different_thread_worker", 0); |
Craig Tiller | 988e37f | 2015-10-01 07:53:56 -0700 | [diff] [blame] | 121 | if ((flags & GRPC_POLLSET_REEVALUATE_POLLING_ON_WAKEUP) != 0) { |
| 122 | specific_worker->reevaluate_polling_on_wakeup = 1; |
| 123 | } |
Craig Tiller | 66197ca | 2015-11-02 08:04:10 -0800 | [diff] [blame] | 124 | specific_worker->kicked_specifically = 1; |
Craig Tiller | e8b5f62 | 2015-11-02 14:15:03 -0800 | [diff] [blame] | 125 | grpc_wakeup_fd_wakeup(&specific_worker->wakeup_fd->fd); |
Craig Tiller | 988e37f | 2015-10-01 07:53:56 -0700 | [diff] [blame] | 126 | } else if ((flags & GRPC_POLLSET_CAN_KICK_SELF) != 0) { |
Craig Tiller | 0ba432d | 2015-10-09 16:57:11 -0700 | [diff] [blame] | 127 | GPR_TIMER_MARK("kick_yoself", 0); |
Craig Tiller | 988e37f | 2015-10-01 07:53:56 -0700 | [diff] [blame] | 128 | if ((flags & GRPC_POLLSET_REEVALUATE_POLLING_ON_WAKEUP) != 0) { |
| 129 | specific_worker->reevaluate_polling_on_wakeup = 1; |
| 130 | } |
Craig Tiller | 66197ca | 2015-11-02 08:04:10 -0800 | [diff] [blame] | 131 | specific_worker->kicked_specifically = 1; |
Craig Tiller | e8b5f62 | 2015-11-02 14:15:03 -0800 | [diff] [blame] | 132 | grpc_wakeup_fd_wakeup(&specific_worker->wakeup_fd->fd); |
Craig Tiller | 5ddbb9d | 2015-07-29 15:58:11 -0700 | [diff] [blame] | 133 | } |
Craig Tiller | 7536af0 | 2015-12-22 13:49:30 -0800 | [diff] [blame] | 134 | } else if (gpr_tls_get(&g_current_thread_poller) != (intptr_t)p) { |
Craig Tiller | 988e37f | 2015-10-01 07:53:56 -0700 | [diff] [blame] | 135 | GPR_ASSERT((flags & GRPC_POLLSET_REEVALUATE_POLLING_ON_WAKEUP) == 0); |
Craig Tiller | 0ba432d | 2015-10-09 16:57:11 -0700 | [diff] [blame] | 136 | GPR_TIMER_MARK("kick_anonymous", 0); |
Craig Tiller | a82950e | 2015-09-22 12:33:20 -0700 | [diff] [blame] | 137 | specific_worker = pop_front_worker(p); |
| 138 | if (specific_worker != NULL) { |
Craig Tiller | 7536af0 | 2015-12-22 13:49:30 -0800 | [diff] [blame] | 139 | if (gpr_tls_get(&g_current_thread_worker) == (intptr_t)specific_worker) { |
Craig Tiller | 0ba432d | 2015-10-09 16:57:11 -0700 | [diff] [blame] | 140 | GPR_TIMER_MARK("kick_anonymous_not_self", 0); |
Craig Tiller | 988e37f | 2015-10-01 07:53:56 -0700 | [diff] [blame] | 141 | push_back_worker(p, specific_worker); |
| 142 | specific_worker = pop_front_worker(p); |
Craig Tiller | 57f79d6 | 2015-10-02 14:00:12 -0700 | [diff] [blame] | 143 | if ((flags & GRPC_POLLSET_CAN_KICK_SELF) == 0 && |
| 144 | gpr_tls_get(&g_current_thread_worker) == |
Craig Tiller | 7536af0 | 2015-12-22 13:49:30 -0800 | [diff] [blame] | 145 | (intptr_t)specific_worker) { |
Craig Tiller | 988e37f | 2015-10-01 07:53:56 -0700 | [diff] [blame] | 146 | push_back_worker(p, specific_worker); |
Craig Tiller | 44011e8 | 2015-10-09 16:16:43 -0700 | [diff] [blame] | 147 | specific_worker = NULL; |
Craig Tiller | 988e37f | 2015-10-01 07:53:56 -0700 | [diff] [blame] | 148 | } |
| 149 | } |
Craig Tiller | 44011e8 | 2015-10-09 16:16:43 -0700 | [diff] [blame] | 150 | if (specific_worker != NULL) { |
Craig Tiller | 0ba432d | 2015-10-09 16:57:11 -0700 | [diff] [blame] | 151 | GPR_TIMER_MARK("finally_kick", 0); |
Craig Tiller | 44011e8 | 2015-10-09 16:16:43 -0700 | [diff] [blame] | 152 | push_back_worker(p, specific_worker); |
Craig Tiller | e8b5f62 | 2015-11-02 14:15:03 -0800 | [diff] [blame] | 153 | grpc_wakeup_fd_wakeup(&specific_worker->wakeup_fd->fd); |
Craig Tiller | 44011e8 | 2015-10-09 16:16:43 -0700 | [diff] [blame] | 154 | } |
Craig Tiller | a82950e | 2015-09-22 12:33:20 -0700 | [diff] [blame] | 155 | } else { |
Craig Tiller | 0ba432d | 2015-10-09 16:57:11 -0700 | [diff] [blame] | 156 | GPR_TIMER_MARK("kicked_no_pollers", 0); |
Craig Tiller | a82950e | 2015-09-22 12:33:20 -0700 | [diff] [blame] | 157 | p->kicked_without_pollers = 1; |
Craig Tiller | 5ddbb9d | 2015-07-29 15:58:11 -0700 | [diff] [blame] | 158 | } |
Craig Tiller | a82950e | 2015-09-22 12:33:20 -0700 | [diff] [blame] | 159 | } |
Craig Tiller | 44011e8 | 2015-10-09 16:16:43 -0700 | [diff] [blame] | 160 | |
Craig Tiller | 0ba432d | 2015-10-09 16:57:11 -0700 | [diff] [blame] | 161 | GPR_TIMER_END("grpc_pollset_kick_ext", 0); |
David Klempner | baced4d | 2015-02-10 17:10:15 -0800 | [diff] [blame] | 162 | } |
| 163 | |
Craig Tiller | 988e37f | 2015-10-01 07:53:56 -0700 | [diff] [blame] | 164 | void grpc_pollset_kick(grpc_pollset *p, grpc_pollset_worker *specific_worker) { |
Craig Tiller | b937aa1 | 2015-10-06 11:30:30 -0700 | [diff] [blame] | 165 | grpc_pollset_kick_ext(p, specific_worker, 0); |
Craig Tiller | 988e37f | 2015-10-01 07:53:56 -0700 | [diff] [blame] | 166 | } |
| 167 | |
ctiller | 58393c2 | 2015-01-07 14:03:30 -0800 | [diff] [blame] | 168 | /* global state management */ |
| 169 | |
Craig Tiller | a82950e | 2015-09-22 12:33:20 -0700 | [diff] [blame] | 170 | void grpc_pollset_global_init(void) { |
| 171 | gpr_tls_init(&g_current_thread_poller); |
Craig Tiller | 926c0e9 | 2015-09-28 14:25:38 -0700 | [diff] [blame] | 172 | gpr_tls_init(&g_current_thread_worker); |
Craig Tiller | a82950e | 2015-09-22 12:33:20 -0700 | [diff] [blame] | 173 | grpc_wakeup_fd_global_init(); |
Craig Tiller | 8afeec8 | 2015-09-28 17:03:34 -0700 | [diff] [blame] | 174 | grpc_wakeup_fd_init(&grpc_global_wakeup_fd); |
David Klempner | b505661 | 2015-02-24 14:22:50 -0800 | [diff] [blame] | 175 | } |
| 176 | |
Craig Tiller | a82950e | 2015-09-22 12:33:20 -0700 | [diff] [blame] | 177 | void grpc_pollset_global_shutdown(void) { |
Craig Tiller | 8afeec8 | 2015-09-28 17:03:34 -0700 | [diff] [blame] | 178 | grpc_wakeup_fd_destroy(&grpc_global_wakeup_fd); |
Craig Tiller | a82950e | 2015-09-22 12:33:20 -0700 | [diff] [blame] | 179 | gpr_tls_destroy(&g_current_thread_poller); |
Craig Tiller | 926c0e9 | 2015-09-28 14:25:38 -0700 | [diff] [blame] | 180 | gpr_tls_destroy(&g_current_thread_worker); |
Craig Tiller | e8b5f62 | 2015-11-02 14:15:03 -0800 | [diff] [blame] | 181 | grpc_wakeup_fd_global_destroy(); |
Craig Tiller | 8afeec8 | 2015-09-28 17:03:34 -0700 | [diff] [blame] | 182 | } |
| 183 | |
Craig Tiller | 71a0f9d | 2015-09-28 17:22:01 -0700 | [diff] [blame] | 184 | void grpc_kick_poller(void) { grpc_wakeup_fd_wakeup(&grpc_global_wakeup_fd); } |
ctiller | 58393c2 | 2015-01-07 14:03:30 -0800 | [diff] [blame] | 185 | |
| 186 | /* main interface */ |
| 187 | |
Craig Tiller | a82950e | 2015-09-22 12:33:20 -0700 | [diff] [blame] | 188 | static void become_basic_pollset(grpc_pollset *pollset, grpc_fd *fd_or_null); |
ctiller | 58393c2 | 2015-01-07 14:03:30 -0800 | [diff] [blame] | 189 | |
Craig Tiller | a82950e | 2015-09-22 12:33:20 -0700 | [diff] [blame] | 190 | void grpc_pollset_init(grpc_pollset *pollset) { |
| 191 | gpr_mu_init(&pollset->mu); |
Craig Tiller | 5ddbb9d | 2015-07-29 15:58:11 -0700 | [diff] [blame] | 192 | pollset->root_worker.next = pollset->root_worker.prev = &pollset->root_worker; |
David Klempner | b505661 | 2015-02-24 14:22:50 -0800 | [diff] [blame] | 193 | pollset->in_flight_cbs = 0; |
| 194 | pollset->shutting_down = 0; |
Jan Tattermusch | d6ca8b4 | 2015-06-10 15:58:39 -0700 | [diff] [blame] | 195 | pollset->called_shutdown = 0; |
yang-g | 30e1727 | 2015-11-07 01:11:12 -0800 | [diff] [blame] | 196 | pollset->kicked_without_pollers = 0; |
Craig Tiller | d1bec03 | 2015-09-18 17:29:00 -0700 | [diff] [blame] | 197 | pollset->idle_jobs.head = pollset->idle_jobs.tail = NULL; |
Craig Tiller | e8b5f62 | 2015-11-02 14:15:03 -0800 | [diff] [blame] | 198 | pollset->local_wakeup_cache = NULL; |
| 199 | pollset->kicked_without_pollers = 0; |
| 200 | become_basic_pollset(pollset, NULL); |
| 201 | } |
| 202 | |
| 203 | void grpc_pollset_destroy(grpc_pollset *pollset) { |
| 204 | GPR_ASSERT(pollset->in_flight_cbs == 0); |
| 205 | GPR_ASSERT(!grpc_pollset_has_workers(pollset)); |
| 206 | GPR_ASSERT(pollset->idle_jobs.head == pollset->idle_jobs.tail); |
| 207 | pollset->vtable->destroy(pollset); |
| 208 | gpr_mu_destroy(&pollset->mu); |
| 209 | while (pollset->local_wakeup_cache) { |
| 210 | grpc_cached_wakeup_fd *next = pollset->local_wakeup_cache->next; |
| 211 | grpc_wakeup_fd_destroy(&pollset->local_wakeup_cache->fd); |
| 212 | gpr_free(pollset->local_wakeup_cache); |
| 213 | pollset->local_wakeup_cache = next; |
| 214 | } |
| 215 | } |
| 216 | |
| 217 | void grpc_pollset_reset(grpc_pollset *pollset) { |
| 218 | GPR_ASSERT(pollset->shutting_down); |
| 219 | GPR_ASSERT(pollset->in_flight_cbs == 0); |
| 220 | GPR_ASSERT(!grpc_pollset_has_workers(pollset)); |
| 221 | GPR_ASSERT(pollset->idle_jobs.head == pollset->idle_jobs.tail); |
| 222 | pollset->vtable->destroy(pollset); |
| 223 | pollset->shutting_down = 0; |
| 224 | pollset->called_shutdown = 0; |
| 225 | pollset->kicked_without_pollers = 0; |
Craig Tiller | a82950e | 2015-09-22 12:33:20 -0700 | [diff] [blame] | 226 | become_basic_pollset(pollset, NULL); |
ctiller | 58393c2 | 2015-01-07 14:03:30 -0800 | [diff] [blame] | 227 | } |
| 228 | |
Craig Tiller | a82950e | 2015-09-22 12:33:20 -0700 | [diff] [blame] | 229 | void grpc_pollset_add_fd(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, |
| 230 | grpc_fd *fd) { |
| 231 | gpr_mu_lock(&pollset->mu); |
| 232 | pollset->vtable->add_fd(exec_ctx, pollset, fd, 1); |
Craig Tiller | 4c06b82 | 2015-08-06 08:41:31 -0700 | [diff] [blame] | 233 | /* the following (enabled only in debug) will reacquire and then release |
Craig Tiller | 88a33ef | 2015-12-11 13:17:40 -0800 | [diff] [blame] | 234 | our lock - meaning that if the unlocking flag passed to add_fd above is |
Craig Tiller | 4c06b82 | 2015-08-06 08:41:31 -0700 | [diff] [blame] | 235 | not respected, the code will deadlock (in a way that we have a chance of |
| 236 | debugging) */ |
Craig Tiller | abfaf2a | 2015-07-16 17:23:40 -0700 | [diff] [blame] | 237 | #ifndef NDEBUG |
Craig Tiller | a82950e | 2015-09-22 12:33:20 -0700 | [diff] [blame] | 238 | gpr_mu_lock(&pollset->mu); |
| 239 | gpr_mu_unlock(&pollset->mu); |
Craig Tiller | b0c13ad | 2015-07-16 08:42:31 -0700 | [diff] [blame] | 240 | #endif |
ctiller | 58393c2 | 2015-01-07 14:03:30 -0800 | [diff] [blame] | 241 | } |
| 242 | |
Craig Tiller | a82950e | 2015-09-22 12:33:20 -0700 | [diff] [blame] | 243 | static void finish_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset) { |
Craig Tiller | 6301038 | 2015-09-24 15:00:58 -0700 | [diff] [blame] | 244 | GPR_ASSERT(grpc_closure_list_empty(pollset->idle_jobs)); |
Craig Tiller | a82950e | 2015-09-22 12:33:20 -0700 | [diff] [blame] | 245 | pollset->vtable->finish_shutdown(pollset); |
Craig Tiller | 6c39686 | 2016-01-28 13:53:40 -0800 | [diff] [blame] | 246 | grpc_exec_ctx_enqueue(exec_ctx, pollset->shutdown_done, true, NULL); |
Craig Tiller | 000cd8f | 2015-09-18 07:20:29 -0700 | [diff] [blame] | 247 | } |
| 248 | |
Craig Tiller | a82950e | 2015-09-22 12:33:20 -0700 | [diff] [blame] | 249 | void grpc_pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, |
| 250 | grpc_pollset_worker *worker, gpr_timespec now, |
| 251 | gpr_timespec deadline) { |
ctiller | 58393c2 | 2015-01-07 14:03:30 -0800 | [diff] [blame] | 252 | /* pollset->mu already held */ |
Craig Tiller | 5ddbb9d | 2015-07-29 15:58:11 -0700 | [diff] [blame] | 253 | int added_worker = 0; |
Craig Tiller | d1bec03 | 2015-09-18 17:29:00 -0700 | [diff] [blame] | 254 | int locked = 1; |
Craig Tiller | dc17471 | 2015-10-01 10:25:02 -0700 | [diff] [blame] | 255 | int queued_work = 0; |
Craig Tiller | 6078a7d | 2015-10-06 11:50:21 -0700 | [diff] [blame] | 256 | int keep_polling = 0; |
Craig Tiller | 0ba432d | 2015-10-09 16:57:11 -0700 | [diff] [blame] | 257 | GPR_TIMER_BEGIN("grpc_pollset_work", 0); |
Craig Tiller | 5ddbb9d | 2015-07-29 15:58:11 -0700 | [diff] [blame] | 258 | /* this must happen before we (potentially) drop pollset->mu */ |
| 259 | worker->next = worker->prev = NULL; |
Craig Tiller | dc17471 | 2015-10-01 10:25:02 -0700 | [diff] [blame] | 260 | worker->reevaluate_polling_on_wakeup = 0; |
Craig Tiller | e8b5f62 | 2015-11-02 14:15:03 -0800 | [diff] [blame] | 261 | if (pollset->local_wakeup_cache != NULL) { |
| 262 | worker->wakeup_fd = pollset->local_wakeup_cache; |
| 263 | pollset->local_wakeup_cache = worker->wakeup_fd->next; |
| 264 | } else { |
| 265 | worker->wakeup_fd = gpr_malloc(sizeof(*worker->wakeup_fd)); |
| 266 | grpc_wakeup_fd_init(&worker->wakeup_fd->fd); |
| 267 | } |
Craig Tiller | 66197ca | 2015-11-02 08:04:10 -0800 | [diff] [blame] | 268 | worker->kicked_specifically = 0; |
Craig Tiller | 6078a7d | 2015-10-06 11:50:21 -0700 | [diff] [blame] | 269 | /* If there's work waiting for the pollset to be idle, and the |
| 270 | pollset is idle, then do that work */ |
Craig Tiller | a82950e | 2015-09-22 12:33:20 -0700 | [diff] [blame] | 271 | if (!grpc_pollset_has_workers(pollset) && |
| 272 | !grpc_closure_list_empty(pollset->idle_jobs)) { |
Craig Tiller | e8b5f62 | 2015-11-02 14:15:03 -0800 | [diff] [blame] | 273 | GPR_TIMER_MARK("grpc_pollset_work.idle_jobs", 0); |
Craig Tiller | 6c39686 | 2016-01-28 13:53:40 -0800 | [diff] [blame] | 274 | grpc_exec_ctx_enqueue_list(exec_ctx, &pollset->idle_jobs, NULL); |
Craig Tiller | a82950e | 2015-09-22 12:33:20 -0700 | [diff] [blame] | 275 | goto done; |
| 276 | } |
Craig Tiller | 6078a7d | 2015-10-06 11:50:21 -0700 | [diff] [blame] | 277 | /* Check alarms - these are a global resource so we just ping |
| 278 | each time through on every pollset. |
| 279 | May update deadline to ensure timely wakeups. |
| 280 | TODO(ctiller): can this work be localized? */ |
David Garcia Quintas | f747bbc | 2015-10-04 23:09:47 -0700 | [diff] [blame] | 281 | if (grpc_timer_check(exec_ctx, now, &deadline)) { |
Craig Tiller | e8b5f62 | 2015-11-02 14:15:03 -0800 | [diff] [blame] | 282 | GPR_TIMER_MARK("grpc_pollset_work.alarm_triggered", 0); |
Craig Tiller | 8afeec8 | 2015-09-28 17:03:34 -0700 | [diff] [blame] | 283 | gpr_mu_unlock(&pollset->mu); |
| 284 | locked = 0; |
Craig Tiller | a82950e | 2015-09-22 12:33:20 -0700 | [diff] [blame] | 285 | goto done; |
| 286 | } |
Craig Tiller | 6078a7d | 2015-10-06 11:50:21 -0700 | [diff] [blame] | 287 | /* If we're shutting down then we don't execute any extended work */ |
Craig Tiller | a82950e | 2015-09-22 12:33:20 -0700 | [diff] [blame] | 288 | if (pollset->shutting_down) { |
Craig Tiller | e8b5f62 | 2015-11-02 14:15:03 -0800 | [diff] [blame] | 289 | GPR_TIMER_MARK("grpc_pollset_work.shutting_down", 0); |
Craig Tiller | a82950e | 2015-09-22 12:33:20 -0700 | [diff] [blame] | 290 | goto done; |
| 291 | } |
Craig Tiller | 6078a7d | 2015-10-06 11:50:21 -0700 | [diff] [blame] | 292 | /* Give do_promote priority so we don't starve it out */ |
Craig Tiller | a82950e | 2015-09-22 12:33:20 -0700 | [diff] [blame] | 293 | if (pollset->in_flight_cbs) { |
Craig Tiller | e8b5f62 | 2015-11-02 14:15:03 -0800 | [diff] [blame] | 294 | GPR_TIMER_MARK("grpc_pollset_work.in_flight_cbs", 0); |
Craig Tiller | a82950e | 2015-09-22 12:33:20 -0700 | [diff] [blame] | 295 | gpr_mu_unlock(&pollset->mu); |
| 296 | locked = 0; |
| 297 | goto done; |
| 298 | } |
Craig Tiller | 6078a7d | 2015-10-06 11:50:21 -0700 | [diff] [blame] | 299 | /* Start polling, and keep doing so while we're being asked to |
| 300 | re-evaluate our pollers (this allows poll() based pollers to |
| 301 | ensure they don't miss wakeups) */ |
| 302 | keep_polling = 1; |
| 303 | while (keep_polling) { |
| 304 | keep_polling = 0; |
Craig Tiller | dc17471 | 2015-10-01 10:25:02 -0700 | [diff] [blame] | 305 | if (!pollset->kicked_without_pollers) { |
| 306 | if (!added_worker) { |
| 307 | push_front_worker(pollset, worker); |
| 308 | added_worker = 1; |
Craig Tiller | 7536af0 | 2015-12-22 13:49:30 -0800 | [diff] [blame] | 309 | gpr_tls_set(&g_current_thread_worker, (intptr_t)worker); |
Craig Tiller | dc17471 | 2015-10-01 10:25:02 -0700 | [diff] [blame] | 310 | } |
Craig Tiller | 7536af0 | 2015-12-22 13:49:30 -0800 | [diff] [blame] | 311 | gpr_tls_set(&g_current_thread_poller, (intptr_t)pollset); |
Craig Tiller | 0ba432d | 2015-10-09 16:57:11 -0700 | [diff] [blame] | 312 | GPR_TIMER_BEGIN("maybe_work_and_unlock", 0); |
Craig Tiller | dc17471 | 2015-10-01 10:25:02 -0700 | [diff] [blame] | 313 | pollset->vtable->maybe_work_and_unlock(exec_ctx, pollset, worker, |
| 314 | deadline, now); |
Craig Tiller | 0ba432d | 2015-10-09 16:57:11 -0700 | [diff] [blame] | 315 | GPR_TIMER_END("maybe_work_and_unlock", 0); |
Craig Tiller | dc17471 | 2015-10-01 10:25:02 -0700 | [diff] [blame] | 316 | locked = 0; |
| 317 | gpr_tls_set(&g_current_thread_poller, 0); |
Craig Tiller | dc17471 | 2015-10-01 10:25:02 -0700 | [diff] [blame] | 318 | } else { |
Craig Tiller | e8b5f62 | 2015-11-02 14:15:03 -0800 | [diff] [blame] | 319 | GPR_TIMER_MARK("grpc_pollset_work.kicked_without_pollers", 0); |
Craig Tiller | dc17471 | 2015-10-01 10:25:02 -0700 | [diff] [blame] | 320 | pollset->kicked_without_pollers = 0; |
| 321 | } |
Craig Tiller | 6078a7d | 2015-10-06 11:50:21 -0700 | [diff] [blame] | 322 | /* Finished execution - start cleaning up. |
| 323 | Note that we may arrive here from outside the enclosing while() loop. |
| 324 | In that case we won't loop though as we haven't added worker to the |
| 325 | worker list, which means nobody could ask us to re-evaluate polling). */ |
Craig Tiller | dc17471 | 2015-10-01 10:25:02 -0700 | [diff] [blame] | 326 | done: |
| 327 | if (!locked) { |
| 328 | queued_work |= grpc_exec_ctx_flush(exec_ctx); |
| 329 | gpr_mu_lock(&pollset->mu); |
| 330 | locked = 1; |
| 331 | } |
Craig Tiller | 6078a7d | 2015-10-06 11:50:21 -0700 | [diff] [blame] | 332 | /* If we're forced to re-evaluate polling (via grpc_pollset_kick with |
| 333 | GRPC_POLLSET_REEVALUATE_POLLING_ON_WAKEUP) then we land here and force |
| 334 | a loop */ |
Craig Tiller | dc17471 | 2015-10-01 10:25:02 -0700 | [diff] [blame] | 335 | if (worker->reevaluate_polling_on_wakeup) { |
| 336 | worker->reevaluate_polling_on_wakeup = 0; |
Craig Tiller | 58d05a6 | 2015-10-02 13:59:31 -0700 | [diff] [blame] | 337 | pollset->kicked_without_pollers = 0; |
Craig Tiller | 66197ca | 2015-11-02 08:04:10 -0800 | [diff] [blame] | 338 | if (queued_work || worker->kicked_specifically) { |
Craig Tiller | 6078a7d | 2015-10-06 11:50:21 -0700 | [diff] [blame] | 339 | /* If there's queued work on the list, then set the deadline to be |
| 340 | immediate so we get back out of the polling loop quickly */ |
Craig Tiller | dc17471 | 2015-10-01 10:25:02 -0700 | [diff] [blame] | 341 | deadline = gpr_inf_past(GPR_CLOCK_MONOTONIC); |
| 342 | } |
Craig Tiller | 6078a7d | 2015-10-06 11:50:21 -0700 | [diff] [blame] | 343 | keep_polling = 1; |
Craig Tiller | dc17471 | 2015-10-01 10:25:02 -0700 | [diff] [blame] | 344 | } |
Craig Tiller | a82950e | 2015-09-22 12:33:20 -0700 | [diff] [blame] | 345 | } |
Craig Tiller | a82950e | 2015-09-22 12:33:20 -0700 | [diff] [blame] | 346 | if (added_worker) { |
| 347 | remove_worker(pollset, worker); |
Craig Tiller | cb7bf8a | 2015-10-09 16:18:15 -0700 | [diff] [blame] | 348 | gpr_tls_set(&g_current_thread_worker, 0); |
Craig Tiller | a82950e | 2015-09-22 12:33:20 -0700 | [diff] [blame] | 349 | } |
Craig Tiller | e8b5f62 | 2015-11-02 14:15:03 -0800 | [diff] [blame] | 350 | /* release wakeup fd to the local pool */ |
| 351 | worker->wakeup_fd->next = pollset->local_wakeup_cache; |
| 352 | pollset->local_wakeup_cache = worker->wakeup_fd; |
| 353 | /* check shutdown conditions */ |
Craig Tiller | a82950e | 2015-09-22 12:33:20 -0700 | [diff] [blame] | 354 | if (pollset->shutting_down) { |
| 355 | if (grpc_pollset_has_workers(pollset)) { |
| 356 | grpc_pollset_kick(pollset, NULL); |
| 357 | } else if (!pollset->called_shutdown && pollset->in_flight_cbs == 0) { |
Craig Tiller | 45724b3 | 2015-09-22 10:42:19 -0700 | [diff] [blame] | 358 | pollset->called_shutdown = 1; |
Craig Tiller | a82950e | 2015-09-22 12:33:20 -0700 | [diff] [blame] | 359 | gpr_mu_unlock(&pollset->mu); |
| 360 | finish_shutdown(exec_ctx, pollset); |
| 361 | grpc_exec_ctx_flush(exec_ctx); |
| 362 | /* Continuing to access pollset here is safe -- it is the caller's |
| 363 | * responsibility to not destroy when it has outstanding calls to |
| 364 | * grpc_pollset_work. |
| 365 | * TODO(dklempner): Can we refactor the shutdown logic to avoid this? */ |
| 366 | gpr_mu_lock(&pollset->mu); |
Craig Tiller | 6301038 | 2015-09-24 15:00:58 -0700 | [diff] [blame] | 367 | } else if (!grpc_closure_list_empty(pollset->idle_jobs)) { |
Craig Tiller | 6c39686 | 2016-01-28 13:53:40 -0800 | [diff] [blame] | 368 | grpc_exec_ctx_enqueue_list(exec_ctx, &pollset->idle_jobs, NULL); |
Craig Tiller | e8b5f62 | 2015-11-02 14:15:03 -0800 | [diff] [blame] | 369 | gpr_mu_unlock(&pollset->mu); |
Craig Tiller | 6301038 | 2015-09-24 15:00:58 -0700 | [diff] [blame] | 370 | grpc_exec_ctx_flush(exec_ctx); |
| 371 | gpr_mu_lock(&pollset->mu); |
Craig Tiller | 45724b3 | 2015-09-22 10:42:19 -0700 | [diff] [blame] | 372 | } |
Craig Tiller | a82950e | 2015-09-22 12:33:20 -0700 | [diff] [blame] | 373 | } |
Craig Tiller | 0ba432d | 2015-10-09 16:57:11 -0700 | [diff] [blame] | 374 | GPR_TIMER_END("grpc_pollset_work", 0); |
Craig Tiller | a82950e | 2015-09-22 12:33:20 -0700 | [diff] [blame] | 375 | } |
| 376 | |
| 377 | void grpc_pollset_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, |
| 378 | grpc_closure *closure) { |
Craig Tiller | a82950e | 2015-09-22 12:33:20 -0700 | [diff] [blame] | 379 | GPR_ASSERT(!pollset->shutting_down); |
| 380 | pollset->shutting_down = 1; |
Craig Tiller | e8b5f62 | 2015-11-02 14:15:03 -0800 | [diff] [blame] | 381 | pollset->shutdown_done = closure; |
| 382 | grpc_pollset_kick(pollset, GRPC_POLLSET_KICK_BROADCAST); |
Craig Tiller | 6301038 | 2015-09-24 15:00:58 -0700 | [diff] [blame] | 383 | if (!grpc_pollset_has_workers(pollset)) { |
Craig Tiller | 6c39686 | 2016-01-28 13:53:40 -0800 | [diff] [blame] | 384 | grpc_exec_ctx_enqueue_list(exec_ctx, &pollset->idle_jobs, NULL); |
Craig Tiller | 6301038 | 2015-09-24 15:00:58 -0700 | [diff] [blame] | 385 | } |
Craig Tiller | e8b5f62 | 2015-11-02 14:15:03 -0800 | [diff] [blame] | 386 | if (!pollset->called_shutdown && pollset->in_flight_cbs == 0 && |
| 387 | !grpc_pollset_has_workers(pollset)) { |
| 388 | pollset->called_shutdown = 1; |
Craig Tiller | a82950e | 2015-09-22 12:33:20 -0700 | [diff] [blame] | 389 | finish_shutdown(exec_ctx, pollset); |
| 390 | } |
David Klempner | b505661 | 2015-02-24 14:22:50 -0800 | [diff] [blame] | 391 | } |
| 392 | |
Craig Tiller | a82950e | 2015-09-22 12:33:20 -0700 | [diff] [blame] | 393 | int grpc_poll_deadline_to_millis_timeout(gpr_timespec deadline, |
| 394 | gpr_timespec now) { |
Craig Tiller | 6174b9a | 2015-06-18 08:13:05 -0700 | [diff] [blame] | 395 | gpr_timespec timeout; |
murgatroid99 | 309830f | 2016-02-05 11:30:00 -0800 | [diff] [blame^] | 396 | static const int64_t max_spin_polling_us = 10; |
Craig Tiller | a82950e | 2015-09-22 12:33:20 -0700 | [diff] [blame] | 397 | if (gpr_time_cmp(deadline, gpr_inf_future(deadline.clock_type)) == 0) { |
| 398 | return -1; |
| 399 | } |
| 400 | if (gpr_time_cmp(deadline, gpr_time_add(now, gpr_time_from_micros( |
| 401 | max_spin_polling_us, |
| 402 | GPR_TIMESPAN))) <= 0) { |
| 403 | return 0; |
| 404 | } |
| 405 | timeout = gpr_time_sub(deadline, now); |
| 406 | return gpr_time_to_millis(gpr_time_add( |
Craig Tiller | 8afeec8 | 2015-09-28 17:03:34 -0700 | [diff] [blame] | 407 | timeout, gpr_time_from_nanos(GPR_NS_PER_MS - 1, GPR_TIMESPAN))); |
Craig Tiller | 6174b9a | 2015-06-18 08:13:05 -0700 | [diff] [blame] | 408 | } |
| 409 | |
ctiller | 58393c2 | 2015-01-07 14:03:30 -0800 | [diff] [blame] | 410 | /* |
Craig Tiller | 5ec3bfa | 2015-05-28 14:13:18 -0700 | [diff] [blame] | 411 | * basic_pollset - a vtable that provides polling for zero or one file |
Craig Tiller | 3f529a2c | 2015-05-28 14:00:47 -0700 | [diff] [blame] | 412 | * descriptor via poll() |
ctiller | 58393c2 | 2015-01-07 14:03:30 -0800 | [diff] [blame] | 413 | */ |
| 414 | |
Craig Tiller | a82950e | 2015-09-22 12:33:20 -0700 | [diff] [blame] | 415 | typedef struct grpc_unary_promote_args { |
David Klempner | 7f43eaf | 2015-02-18 17:00:31 -0800 | [diff] [blame] | 416 | const grpc_pollset_vtable *original_vtable; |
| 417 | grpc_pollset *pollset; |
| 418 | grpc_fd *fd; |
Craig Tiller | 3382511 | 2015-09-18 07:44:19 -0700 | [diff] [blame] | 419 | grpc_closure promotion_closure; |
David Klempner | 7f43eaf | 2015-02-18 17:00:31 -0800 | [diff] [blame] | 420 | } grpc_unary_promote_args; |
| 421 | |
Craig Tiller | 6c39686 | 2016-01-28 13:53:40 -0800 | [diff] [blame] | 422 | static void basic_do_promote(grpc_exec_ctx *exec_ctx, void *args, |
| 423 | bool success) { |
David Klempner | 7f43eaf | 2015-02-18 17:00:31 -0800 | [diff] [blame] | 424 | grpc_unary_promote_args *up_args = args; |
| 425 | const grpc_pollset_vtable *original_vtable = up_args->original_vtable; |
| 426 | grpc_pollset *pollset = up_args->pollset; |
| 427 | grpc_fd *fd = up_args->fd; |
David Klempner | 7f43eaf | 2015-02-18 17:00:31 -0800 | [diff] [blame] | 428 | |
David Klempner | b505661 | 2015-02-24 14:22:50 -0800 | [diff] [blame] | 429 | /* |
| 430 | * This is quite tricky. There are a number of cases to keep in mind here: |
| 431 | * 1. fd may have been orphaned |
| 432 | * 2. The pollset may no longer be a unary poller (and we can't let case #1 |
| 433 | * leak to other pollset types!) |
| 434 | * 3. pollset's fd (which may have changed) may have been orphaned |
| 435 | * 4. The pollset may be shutting down. |
| 436 | */ |
| 437 | |
Craig Tiller | a82950e | 2015-09-22 12:33:20 -0700 | [diff] [blame] | 438 | gpr_mu_lock(&pollset->mu); |
David Klempner | 7f43eaf | 2015-02-18 17:00:31 -0800 | [diff] [blame] | 439 | /* First we need to ensure that nobody is polling concurrently */ |
Craig Tiller | a82950e | 2015-09-22 12:33:20 -0700 | [diff] [blame] | 440 | GPR_ASSERT(!grpc_pollset_has_workers(pollset)); |
Craig Tiller | 6bf4501 | 2015-05-15 17:11:44 -0700 | [diff] [blame] | 441 | |
Craig Tiller | a82950e | 2015-09-22 12:33:20 -0700 | [diff] [blame] | 442 | gpr_free(up_args); |
David Klempner | 7f43eaf | 2015-02-18 17:00:31 -0800 | [diff] [blame] | 443 | /* At this point the pollset may no longer be a unary poller. In that case |
| 444 | * we should just call the right add function and be done. */ |
| 445 | /* TODO(klempner): If we're not careful this could cause infinite recursion. |
| 446 | * That's not a problem for now because empty_pollset has a trivial poller |
| 447 | * and we don't have any mechanism to unbecome multipoller. */ |
David Klempner | b505661 | 2015-02-24 14:22:50 -0800 | [diff] [blame] | 448 | pollset->in_flight_cbs--; |
Craig Tiller | a82950e | 2015-09-22 12:33:20 -0700 | [diff] [blame] | 449 | if (pollset->shutting_down) { |
| 450 | /* We don't care about this pollset anymore. */ |
| 451 | if (pollset->in_flight_cbs == 0 && !pollset->called_shutdown) { |
yang-g | 2c1c000 | 2015-10-02 16:15:05 -0700 | [diff] [blame] | 452 | pollset->called_shutdown = 1; |
Craig Tiller | 6301038 | 2015-09-24 15:00:58 -0700 | [diff] [blame] | 453 | finish_shutdown(exec_ctx, pollset); |
David Klempner | b505661 | 2015-02-24 14:22:50 -0800 | [diff] [blame] | 454 | } |
Craig Tiller | a82950e | 2015-09-22 12:33:20 -0700 | [diff] [blame] | 455 | } else if (grpc_fd_is_orphaned(fd)) { |
| 456 | /* Don't try to add it to anything, we'll drop our ref on it below */ |
| 457 | } else if (pollset->vtable != original_vtable) { |
| 458 | pollset->vtable->add_fd(exec_ctx, pollset, fd, 0); |
| 459 | } else if (fd != pollset->data.ptr) { |
| 460 | grpc_fd *fds[2]; |
| 461 | fds[0] = pollset->data.ptr; |
| 462 | fds[1] = fd; |
David Klempner | 7f43eaf | 2015-02-18 17:00:31 -0800 | [diff] [blame] | 463 | |
Craig Tiller | a82950e | 2015-09-22 12:33:20 -0700 | [diff] [blame] | 464 | if (fds[0] && !grpc_fd_is_orphaned(fds[0])) { |
| 465 | grpc_platform_become_multipoller(exec_ctx, pollset, fds, |
| 466 | GPR_ARRAY_SIZE(fds)); |
| 467 | GRPC_FD_UNREF(fds[0], "basicpoll"); |
| 468 | } else { |
| 469 | /* old fd is orphaned and we haven't cleaned it up until now, so remain a |
| 470 | * unary poller */ |
| 471 | /* Note that it is possible that fds[1] is also orphaned at this point. |
| 472 | * That's okay, we'll correct it at the next add or poll. */ |
| 473 | if (fds[0]) GRPC_FD_UNREF(fds[0], "basicpoll"); |
| 474 | pollset->data.ptr = fd; |
| 475 | GRPC_FD_REF(fd, "basicpoll"); |
Craig Tiller | 45724b3 | 2015-09-22 10:42:19 -0700 | [diff] [blame] | 476 | } |
Craig Tiller | a82950e | 2015-09-22 12:33:20 -0700 | [diff] [blame] | 477 | } |
Craig Tiller | 45724b3 | 2015-09-22 10:42:19 -0700 | [diff] [blame] | 478 | |
Craig Tiller | a82950e | 2015-09-22 12:33:20 -0700 | [diff] [blame] | 479 | gpr_mu_unlock(&pollset->mu); |
David Klempner | 7f43eaf | 2015-02-18 17:00:31 -0800 | [diff] [blame] | 480 | |
Craig Tiller | 3f529a2c | 2015-05-28 14:00:47 -0700 | [diff] [blame] | 481 | /* Matching ref in basic_pollset_add_fd */ |
Craig Tiller | a82950e | 2015-09-22 12:33:20 -0700 | [diff] [blame] | 482 | GRPC_FD_UNREF(fd, "basicpoll_add"); |
David Klempner | 7f43eaf | 2015-02-18 17:00:31 -0800 | [diff] [blame] | 483 | } |
| 484 | |
Craig Tiller | a82950e | 2015-09-22 12:33:20 -0700 | [diff] [blame] | 485 | static void basic_pollset_add_fd(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, |
| 486 | grpc_fd *fd, int and_unlock_pollset) { |
David Klempner | 7f43eaf | 2015-02-18 17:00:31 -0800 | [diff] [blame] | 487 | grpc_unary_promote_args *up_args; |
Craig Tiller | a82950e | 2015-09-22 12:33:20 -0700 | [diff] [blame] | 488 | GPR_ASSERT(fd); |
| 489 | if (fd == pollset->data.ptr) goto exit; |
Craig Tiller | 45724b3 | 2015-09-22 10:42:19 -0700 | [diff] [blame] | 490 | |
Craig Tiller | a82950e | 2015-09-22 12:33:20 -0700 | [diff] [blame] | 491 | if (!grpc_pollset_has_workers(pollset)) { |
| 492 | /* Fast path -- no in flight cbs */ |
| 493 | /* TODO(klempner): Comment this out and fix any test failures or establish |
| 494 | * they are due to timing issues */ |
| 495 | grpc_fd *fds[2]; |
| 496 | fds[0] = pollset->data.ptr; |
| 497 | fds[1] = fd; |
Craig Tiller | 45724b3 | 2015-09-22 10:42:19 -0700 | [diff] [blame] | 498 | |
Craig Tiller | a82950e | 2015-09-22 12:33:20 -0700 | [diff] [blame] | 499 | if (fds[0] == NULL) { |
| 500 | pollset->data.ptr = fd; |
| 501 | GRPC_FD_REF(fd, "basicpoll"); |
| 502 | } else if (!grpc_fd_is_orphaned(fds[0])) { |
| 503 | grpc_platform_become_multipoller(exec_ctx, pollset, fds, |
| 504 | GPR_ARRAY_SIZE(fds)); |
| 505 | GRPC_FD_UNREF(fds[0], "basicpoll"); |
| 506 | } else { |
| 507 | /* old fd is orphaned and we haven't cleaned it up until now, so remain a |
| 508 | * unary poller */ |
| 509 | GRPC_FD_UNREF(fds[0], "basicpoll"); |
| 510 | pollset->data.ptr = fd; |
| 511 | GRPC_FD_REF(fd, "basicpoll"); |
Craig Tiller | 45724b3 | 2015-09-22 10:42:19 -0700 | [diff] [blame] | 512 | } |
Craig Tiller | a82950e | 2015-09-22 12:33:20 -0700 | [diff] [blame] | 513 | goto exit; |
| 514 | } |
David Klempner | 7f43eaf | 2015-02-18 17:00:31 -0800 | [diff] [blame] | 515 | |
| 516 | /* Now we need to promote. This needs to happen when we're not polling. Since |
| 517 | * this may be called from poll, the wait needs to happen asynchronously. */ |
Craig Tiller | a82950e | 2015-09-22 12:33:20 -0700 | [diff] [blame] | 518 | GRPC_FD_REF(fd, "basicpoll_add"); |
David Klempner | b505661 | 2015-02-24 14:22:50 -0800 | [diff] [blame] | 519 | pollset->in_flight_cbs++; |
Craig Tiller | a82950e | 2015-09-22 12:33:20 -0700 | [diff] [blame] | 520 | up_args = gpr_malloc(sizeof(*up_args)); |
David Klempner | 7f43eaf | 2015-02-18 17:00:31 -0800 | [diff] [blame] | 521 | up_args->fd = fd; |
| 522 | up_args->original_vtable = pollset->vtable; |
Craig Tiller | ba49645 | 2015-09-21 17:15:19 -0700 | [diff] [blame] | 523 | up_args->pollset = pollset; |
Craig Tiller | b475655 | 2015-06-01 20:33:37 -0700 | [diff] [blame] | 524 | up_args->promotion_closure.cb = basic_do_promote; |
David Garcia Quintas | 284488b | 2015-05-28 16:27:39 -0700 | [diff] [blame] | 525 | up_args->promotion_closure.cb_arg = up_args; |
David Klempner | 7f43eaf | 2015-02-18 17:00:31 -0800 | [diff] [blame] | 526 | |
Craig Tiller | a82950e | 2015-09-22 12:33:20 -0700 | [diff] [blame] | 527 | grpc_closure_list_add(&pollset->idle_jobs, &up_args->promotion_closure, 1); |
| 528 | grpc_pollset_kick(pollset, GRPC_POLLSET_KICK_BROADCAST); |
Craig Tiller | 5c785d4 | 2015-07-14 08:23:43 -0700 | [diff] [blame] | 529 | |
| 530 | exit: |
Craig Tiller | a82950e | 2015-09-22 12:33:20 -0700 | [diff] [blame] | 531 | if (and_unlock_pollset) { |
| 532 | gpr_mu_unlock(&pollset->mu); |
| 533 | } |
ctiller | 58393c2 | 2015-01-07 14:03:30 -0800 | [diff] [blame] | 534 | } |
| 535 | |
Craig Tiller | a82950e | 2015-09-22 12:33:20 -0700 | [diff] [blame] | 536 | static void basic_pollset_maybe_work_and_unlock(grpc_exec_ctx *exec_ctx, |
| 537 | grpc_pollset *pollset, |
| 538 | grpc_pollset_worker *worker, |
| 539 | gpr_timespec deadline, |
| 540 | gpr_timespec now) { |
Craig Tiller | 58d05a6 | 2015-10-02 13:59:31 -0700 | [diff] [blame] | 541 | #define POLLOUT_CHECK (POLLOUT | POLLHUP | POLLERR) |
Craig Tiller | 57f79d6 | 2015-10-02 14:00:12 -0700 | [diff] [blame] | 542 | #define POLLIN_CHECK (POLLIN | POLLHUP | POLLERR) |
Craig Tiller | 58d05a6 | 2015-10-02 13:59:31 -0700 | [diff] [blame] | 543 | |
Craig Tiller | 8afeec8 | 2015-09-28 17:03:34 -0700 | [diff] [blame] | 544 | struct pollfd pfd[3]; |
ctiller | 58393c2 | 2015-01-07 14:03:30 -0800 | [diff] [blame] | 545 | grpc_fd *fd; |
Craig Tiller | 7d41321 | 2015-02-09 08:00:02 -0800 | [diff] [blame] | 546 | grpc_fd_watcher fd_watcher; |
ctiller | 58393c2 | 2015-01-07 14:03:30 -0800 | [diff] [blame] | 547 | int timeout; |
| 548 | int r; |
Craig Tiller | 3121fd4 | 2015-09-10 09:56:20 -0700 | [diff] [blame] | 549 | nfds_t nfds; |
ctiller | 58393c2 | 2015-01-07 14:03:30 -0800 | [diff] [blame] | 550 | |
ctiller | 58393c2 | 2015-01-07 14:03:30 -0800 | [diff] [blame] | 551 | fd = pollset->data.ptr; |
Craig Tiller | a82950e | 2015-09-22 12:33:20 -0700 | [diff] [blame] | 552 | if (fd && grpc_fd_is_orphaned(fd)) { |
| 553 | GRPC_FD_UNREF(fd, "basicpoll"); |
| 554 | fd = pollset->data.ptr = NULL; |
| 555 | } |
| 556 | timeout = grpc_poll_deadline_to_millis_timeout(deadline, now); |
Craig Tiller | 8afeec8 | 2015-09-28 17:03:34 -0700 | [diff] [blame] | 557 | pfd[0].fd = GRPC_WAKEUP_FD_GET_READ_FD(&grpc_global_wakeup_fd); |
ctiller | 58393c2 | 2015-01-07 14:03:30 -0800 | [diff] [blame] | 558 | pfd[0].events = POLLIN; |
| 559 | pfd[0].revents = 0; |
Craig Tiller | e8b5f62 | 2015-11-02 14:15:03 -0800 | [diff] [blame] | 560 | pfd[1].fd = GRPC_WAKEUP_FD_GET_READ_FD(&worker->wakeup_fd->fd); |
Craig Tiller | 8afeec8 | 2015-09-28 17:03:34 -0700 | [diff] [blame] | 561 | pfd[1].events = POLLIN; |
| 562 | pfd[1].revents = 0; |
| 563 | nfds = 2; |
Craig Tiller | a82950e | 2015-09-22 12:33:20 -0700 | [diff] [blame] | 564 | if (fd) { |
Craig Tiller | 8afeec8 | 2015-09-28 17:03:34 -0700 | [diff] [blame] | 565 | pfd[2].fd = fd->fd; |
| 566 | pfd[2].revents = 0; |
Craig Tiller | bae235c | 2015-10-05 11:33:36 -0700 | [diff] [blame] | 567 | GRPC_FD_REF(fd, "basicpoll_begin"); |
Craig Tiller | a82950e | 2015-09-22 12:33:20 -0700 | [diff] [blame] | 568 | gpr_mu_unlock(&pollset->mu); |
Craig Tiller | dc17471 | 2015-10-01 10:25:02 -0700 | [diff] [blame] | 569 | pfd[2].events = (short)grpc_fd_begin_poll(fd, pollset, worker, POLLIN, |
| 570 | POLLOUT, &fd_watcher); |
Craig Tiller | 8afeec8 | 2015-09-28 17:03:34 -0700 | [diff] [blame] | 571 | if (pfd[2].events != 0) { |
Craig Tiller | a82950e | 2015-09-22 12:33:20 -0700 | [diff] [blame] | 572 | nfds++; |
Craig Tiller | 3f529a2c | 2015-05-28 14:00:47 -0700 | [diff] [blame] | 573 | } |
Craig Tiller | a82950e | 2015-09-22 12:33:20 -0700 | [diff] [blame] | 574 | } else { |
| 575 | gpr_mu_unlock(&pollset->mu); |
| 576 | } |
Craig Tiller | 48d01ae | 2015-01-29 22:08:59 -0800 | [diff] [blame] | 577 | |
Vijay Pai | 8c7665e | 2015-09-25 21:40:19 -0700 | [diff] [blame] | 578 | /* TODO(vpai): Consider first doing a 0 timeout poll here to avoid |
| 579 | even going into the blocking annotation if possible */ |
Craig Tiller | baac940 | 2015-05-28 07:39:31 -0700 | [diff] [blame] | 580 | /* poll fd count (argument 2) is shortened by one if we have no events |
| 581 | to poll on - such that it only includes the kicker */ |
Craig Tiller | 0ba432d | 2015-10-09 16:57:11 -0700 | [diff] [blame] | 582 | GPR_TIMER_BEGIN("poll", 0); |
vjpai | 9839d28 | 2015-09-24 17:55:18 -0700 | [diff] [blame] | 583 | GRPC_SCHEDULING_START_BLOCKING_REGION; |
Craig Tiller | a82950e | 2015-09-22 12:33:20 -0700 | [diff] [blame] | 584 | r = grpc_poll_function(pfd, nfds, timeout); |
vjpai | 9839d28 | 2015-09-24 17:55:18 -0700 | [diff] [blame] | 585 | GRPC_SCHEDULING_END_BLOCKING_REGION; |
Craig Tiller | 0ba432d | 2015-10-09 16:57:11 -0700 | [diff] [blame] | 586 | GPR_TIMER_END("poll", 0); |
Craig Tiller | a13752d | 2015-03-06 23:01:38 -0800 | [diff] [blame] | 587 | |
Craig Tiller | a82950e | 2015-09-22 12:33:20 -0700 | [diff] [blame] | 588 | if (r < 0) { |
Tom | 2cb9a61 | 2015-12-02 19:05:35 +0000 | [diff] [blame] | 589 | if (errno != EINTR) { |
| 590 | gpr_log(GPR_ERROR, "poll() failed: %s", strerror(errno)); |
| 591 | } |
Craig Tiller | 58d05a6 | 2015-10-02 13:59:31 -0700 | [diff] [blame] | 592 | if (fd) { |
| 593 | grpc_fd_end_poll(exec_ctx, &fd_watcher, 0, 0); |
ctiller | d9962df | 2015-01-07 15:31:39 -0800 | [diff] [blame] | 594 | } |
Craig Tiller | a82950e | 2015-09-22 12:33:20 -0700 | [diff] [blame] | 595 | } else if (r == 0) { |
Craig Tiller | 58d05a6 | 2015-10-02 13:59:31 -0700 | [diff] [blame] | 596 | if (fd) { |
| 597 | grpc_fd_end_poll(exec_ctx, &fd_watcher, 0, 0); |
| 598 | } |
Craig Tiller | a82950e | 2015-09-22 12:33:20 -0700 | [diff] [blame] | 599 | } else { |
Craig Tiller | 58d05a6 | 2015-10-02 13:59:31 -0700 | [diff] [blame] | 600 | if (pfd[0].revents & POLLIN_CHECK) { |
Craig Tiller | 8afeec8 | 2015-09-28 17:03:34 -0700 | [diff] [blame] | 601 | grpc_wakeup_fd_consume_wakeup(&grpc_global_wakeup_fd); |
| 602 | } |
Craig Tiller | 58d05a6 | 2015-10-02 13:59:31 -0700 | [diff] [blame] | 603 | if (pfd[1].revents & POLLIN_CHECK) { |
Craig Tiller | e8b5f62 | 2015-11-02 14:15:03 -0800 | [diff] [blame] | 604 | grpc_wakeup_fd_consume_wakeup(&worker->wakeup_fd->fd); |
ctiller | 58393c2 | 2015-01-07 14:03:30 -0800 | [diff] [blame] | 605 | } |
Craig Tiller | 8afeec8 | 2015-09-28 17:03:34 -0700 | [diff] [blame] | 606 | if (nfds > 2) { |
Craig Tiller | 58d05a6 | 2015-10-02 13:59:31 -0700 | [diff] [blame] | 607 | grpc_fd_end_poll(exec_ctx, &fd_watcher, pfd[2].revents & POLLIN_CHECK, |
| 608 | pfd[2].revents & POLLOUT_CHECK); |
| 609 | } else if (fd) { |
| 610 | grpc_fd_end_poll(exec_ctx, &fd_watcher, 0, 0); |
ctiller | 58393c2 | 2015-01-07 14:03:30 -0800 | [diff] [blame] | 611 | } |
Craig Tiller | a82950e | 2015-09-22 12:33:20 -0700 | [diff] [blame] | 612 | } |
Craig Tiller | bae235c | 2015-10-05 11:33:36 -0700 | [diff] [blame] | 613 | |
| 614 | if (fd) { |
| 615 | GRPC_FD_UNREF(fd, "basicpoll_begin"); |
| 616 | } |
ctiller | 58393c2 | 2015-01-07 14:03:30 -0800 | [diff] [blame] | 617 | } |
| 618 | |
Craig Tiller | a82950e | 2015-09-22 12:33:20 -0700 | [diff] [blame] | 619 | static void basic_pollset_destroy(grpc_pollset *pollset) { |
| 620 | if (pollset->data.ptr != NULL) { |
| 621 | GRPC_FD_UNREF(pollset->data.ptr, "basicpoll"); |
| 622 | pollset->data.ptr = NULL; |
| 623 | } |
ctiller | 58393c2 | 2015-01-07 14:03:30 -0800 | [diff] [blame] | 624 | } |
| 625 | |
Craig Tiller | 3f529a2c | 2015-05-28 14:00:47 -0700 | [diff] [blame] | 626 | static const grpc_pollset_vtable basic_pollset = { |
Craig Tiller | 620e965 | 2015-12-14 12:02:50 -0800 | [diff] [blame] | 627 | basic_pollset_add_fd, basic_pollset_maybe_work_and_unlock, |
| 628 | basic_pollset_destroy, basic_pollset_destroy}; |
ctiller | 58393c2 | 2015-01-07 14:03:30 -0800 | [diff] [blame] | 629 | |
Craig Tiller | a82950e | 2015-09-22 12:33:20 -0700 | [diff] [blame] | 630 | static void become_basic_pollset(grpc_pollset *pollset, grpc_fd *fd_or_null) { |
Craig Tiller | 3f529a2c | 2015-05-28 14:00:47 -0700 | [diff] [blame] | 631 | pollset->vtable = &basic_pollset; |
Craig Tiller | 3f529a2c | 2015-05-28 14:00:47 -0700 | [diff] [blame] | 632 | pollset->data.ptr = fd_or_null; |
Craig Tiller | a82950e | 2015-09-22 12:33:20 -0700 | [diff] [blame] | 633 | if (fd_or_null != NULL) { |
| 634 | GRPC_FD_REF(fd_or_null, "basicpoll"); |
| 635 | } |
ctiller | 58393c2 | 2015-01-07 14:03:30 -0800 | [diff] [blame] | 636 | } |
Craig Tiller | d14a1a5 | 2015-01-21 15:26:29 -0800 | [diff] [blame] | 637 | |
Craig Tiller | 190d360 | 2015-02-18 09:23:38 -0800 | [diff] [blame] | 638 | #endif /* GPR_POSIX_POLLSET */ |