ctiller | 58393c2 | 2015-01-07 14:03:30 -0800 | [diff] [blame] | 1 | /* |
| 2 | * |
Craig Tiller | 0605995 | 2015-02-18 08:34:56 -0800 | [diff] [blame] | 3 | * Copyright 2015, Google Inc. |
ctiller | 58393c2 | 2015-01-07 14:03:30 -0800 | [diff] [blame] | 4 | * All rights reserved. |
| 5 | * |
| 6 | * Redistribution and use in source and binary forms, with or without |
| 7 | * modification, are permitted provided that the following conditions are |
| 8 | * met: |
| 9 | * |
| 10 | * * Redistributions of source code must retain the above copyright |
| 11 | * notice, this list of conditions and the following disclaimer. |
| 12 | * * Redistributions in binary form must reproduce the above |
| 13 | * copyright notice, this list of conditions and the following disclaimer |
| 14 | * in the documentation and/or other materials provided with the |
| 15 | * distribution. |
| 16 | * * Neither the name of Google Inc. nor the names of its |
| 17 | * contributors may be used to endorse or promote products derived from |
| 18 | * this software without specific prior written permission. |
| 19 | * |
| 20 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS |
| 21 | * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT |
| 22 | * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR |
| 23 | * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT |
| 24 | * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
| 25 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
| 26 | * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
| 27 | * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
| 28 | * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
| 29 | * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
| 30 | * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
| 31 | * |
| 32 | */ |
| 33 | |
Craig Tiller | d14a1a5 | 2015-01-21 15:26:29 -0800 | [diff] [blame] | 34 | #include <grpc/support/port_platform.h> |
| 35 | |
| 36 | #ifdef GPR_POSIX_SOCKET |
| 37 | |
ctiller | 58393c2 | 2015-01-07 14:03:30 -0800 | [diff] [blame] | 38 | #include "src/core/iomgr/fd_posix.h" |
| 39 | |
| 40 | #include <assert.h> |
David Klempner | c6bccc2 | 2015-02-24 17:33:05 -0800 | [diff] [blame] | 41 | #include <sys/socket.h> |
ctiller | 58393c2 | 2015-01-07 14:03:30 -0800 | [diff] [blame] | 42 | #include <unistd.h> |
| 43 | |
ctiller | 58393c2 | 2015-01-07 14:03:30 -0800 | [diff] [blame] | 44 | #include <grpc/support/alloc.h> |
| 45 | #include <grpc/support/log.h> |
| 46 | #include <grpc/support/useful.h> |
| 47 | |
Craig Tiller | f95e37f | 2015-02-18 15:15:29 -0800 | [diff] [blame] | 48 | enum descriptor_state { |
| 49 | NOT_READY = 0, |
| 50 | READY = 1 |
| 51 | }; /* or a pointer to a closure to call */ |
ctiller | 58393c2 | 2015-01-07 14:03:30 -0800 | [diff] [blame] | 52 | |
David Klempner | d178524 | 2015-01-28 17:00:21 -0800 | [diff] [blame] | 53 | /* We need to keep a freelist not because of any concerns of malloc performance |
| 54 | * but instead so that implementations with multiple threads in (for example) |
| 55 | * epoll_wait deal with the race between pollset removal and incoming poll |
| 56 | * notifications. |
| 57 | * |
| 58 | * The problem is that the poller ultimately holds a reference to this |
| 59 | * object, so it is very difficult to know when is safe to free it, at least |
| 60 | * without some expensive synchronization. |
| 61 | * |
| 62 | * If we keep the object freelisted, in the worst case losing this race just |
| 63 | * becomes a spurious read notification on a reused fd. |
| 64 | */ |
| 65 | /* TODO(klempner): We could use some form of polling generation count to know |
| 66 | * when these are safe to free. */ |
| 67 | /* TODO(klempner): Consider disabling freelisting if we don't have multiple |
| 68 | * threads in poll on the same fd */ |
| 69 | /* TODO(klempner): Batch these allocations to reduce fragmentation */ |
| 70 | static grpc_fd *fd_freelist = NULL; |
| 71 | static gpr_mu fd_freelist_mu; |
| 72 | |
| 73 | static void freelist_fd(grpc_fd *fd) { |
David Klempner | d178524 | 2015-01-28 17:00:21 -0800 | [diff] [blame] | 74 | gpr_mu_lock(&fd_freelist_mu); |
| 75 | fd->freelist_next = fd_freelist; |
| 76 | fd_freelist = fd; |
Craig Tiller | b1f220d | 2015-07-01 13:54:28 -0700 | [diff] [blame] | 77 | grpc_iomgr_unregister_object(&fd->iomgr_object); |
David Klempner | d178524 | 2015-01-28 17:00:21 -0800 | [diff] [blame] | 78 | gpr_mu_unlock(&fd_freelist_mu); |
| 79 | } |
| 80 | |
| 81 | static grpc_fd *alloc_fd(int fd) { |
| 82 | grpc_fd *r = NULL; |
| 83 | gpr_mu_lock(&fd_freelist_mu); |
| 84 | if (fd_freelist != NULL) { |
| 85 | r = fd_freelist; |
| 86 | fd_freelist = fd_freelist->freelist_next; |
| 87 | } |
| 88 | gpr_mu_unlock(&fd_freelist_mu); |
| 89 | if (r == NULL) { |
| 90 | r = gpr_malloc(sizeof(grpc_fd)); |
| 91 | gpr_mu_init(&r->set_state_mu); |
| 92 | gpr_mu_init(&r->watcher_mu); |
| 93 | } |
David Garcia Quintas | 5f228f5 | 2015-05-26 19:58:50 -0700 | [diff] [blame] | 94 | |
David Klempner | d178524 | 2015-01-28 17:00:21 -0800 | [diff] [blame] | 95 | gpr_atm_rel_store(&r->refst, 1); |
Craig Tiller | 0fcd53c | 2015-02-18 15:10:53 -0800 | [diff] [blame] | 96 | gpr_atm_rel_store(&r->readst, NOT_READY); |
| 97 | gpr_atm_rel_store(&r->writest, NOT_READY); |
David Klempner | d178524 | 2015-01-28 17:00:21 -0800 | [diff] [blame] | 98 | gpr_atm_rel_store(&r->shutdown, 0); |
| 99 | r->fd = fd; |
Craig Tiller | 8e50fe9 | 2015-05-18 10:45:04 -0700 | [diff] [blame] | 100 | r->inactive_watcher_root.next = r->inactive_watcher_root.prev = |
| 101 | &r->inactive_watcher_root; |
David Klempner | d178524 | 2015-01-28 17:00:21 -0800 | [diff] [blame] | 102 | r->freelist_next = NULL; |
Craig Tiller | 886d7ec | 2015-05-14 16:18:42 -0700 | [diff] [blame] | 103 | r->read_watcher = r->write_watcher = NULL; |
Craig Tiller | 8b6cb8d | 2015-06-26 08:08:35 -0700 | [diff] [blame] | 104 | r->on_done_closure = NULL; |
Craig Tiller | 0613e58 | 2015-07-30 11:55:43 -0700 | [diff] [blame^] | 105 | r->closed = 0; |
David Klempner | d178524 | 2015-01-28 17:00:21 -0800 | [diff] [blame] | 106 | return r; |
| 107 | } |
| 108 | |
| 109 | static void destroy(grpc_fd *fd) { |
| 110 | gpr_mu_destroy(&fd->set_state_mu); |
| 111 | gpr_mu_destroy(&fd->watcher_mu); |
ctiller | 58393c2 | 2015-01-07 14:03:30 -0800 | [diff] [blame] | 112 | gpr_free(fd); |
ctiller | 58393c2 | 2015-01-07 14:03:30 -0800 | [diff] [blame] | 113 | } |
| 114 | |
Craig Tiller | 9ae7697 | 2015-05-31 13:58:24 -0700 | [diff] [blame] | 115 | #ifdef GRPC_FD_REF_COUNT_DEBUG |
| 116 | #define REF_BY(fd, n, reason) ref_by(fd, n, reason, __FILE__, __LINE__) |
| 117 | #define UNREF_BY(fd, n, reason) unref_by(fd, n, reason, __FILE__, __LINE__) |
Craig Tiller | 8674cb1 | 2015-06-05 07:09:25 -0700 | [diff] [blame] | 118 | static void ref_by(grpc_fd *fd, int n, const char *reason, const char *file, |
| 119 | int line) { |
Craig Tiller | 87cc084 | 2015-06-30 08:15:55 -0700 | [diff] [blame] | 120 | gpr_log(GPR_DEBUG, "FD %d %p ref %d %d -> %d [%s; %s:%d]", fd->fd, fd, n, |
Craig Tiller | 4b678bd | 2015-06-02 16:12:24 -0700 | [diff] [blame] | 121 | gpr_atm_no_barrier_load(&fd->refst), |
| 122 | gpr_atm_no_barrier_load(&fd->refst) + n, reason, file, line); |
Craig Tiller | 9ae7697 | 2015-05-31 13:58:24 -0700 | [diff] [blame] | 123 | #else |
| 124 | #define REF_BY(fd, n, reason) ref_by(fd, n) |
| 125 | #define UNREF_BY(fd, n, reason) unref_by(fd, n) |
ctiller | 58393c2 | 2015-01-07 14:03:30 -0800 | [diff] [blame] | 126 | static void ref_by(grpc_fd *fd, int n) { |
Craig Tiller | 9ae7697 | 2015-05-31 13:58:24 -0700 | [diff] [blame] | 127 | #endif |
Craig Tiller | 23139ae | 2015-02-17 15:46:13 -0800 | [diff] [blame] | 128 | GPR_ASSERT(gpr_atm_no_barrier_fetch_add(&fd->refst, n) > 0); |
ctiller | 58393c2 | 2015-01-07 14:03:30 -0800 | [diff] [blame] | 129 | } |
| 130 | |
Craig Tiller | 9ae7697 | 2015-05-31 13:58:24 -0700 | [diff] [blame] | 131 | #ifdef GRPC_FD_REF_COUNT_DEBUG |
Craig Tiller | 8674cb1 | 2015-06-05 07:09:25 -0700 | [diff] [blame] | 132 | static void unref_by(grpc_fd *fd, int n, const char *reason, const char *file, |
| 133 | int line) { |
Craig Tiller | 9ae7697 | 2015-05-31 13:58:24 -0700 | [diff] [blame] | 134 | gpr_atm old; |
Craig Tiller | 8e0b08a | 2015-06-01 17:04:17 -0700 | [diff] [blame] | 135 | gpr_log(GPR_DEBUG, "FD %d %p unref %d %d -> %d [%s; %s:%d]", fd->fd, fd, n, |
Craig Tiller | 4b678bd | 2015-06-02 16:12:24 -0700 | [diff] [blame] | 136 | gpr_atm_no_barrier_load(&fd->refst), |
| 137 | gpr_atm_no_barrier_load(&fd->refst) - n, reason, file, line); |
Craig Tiller | 9ae7697 | 2015-05-31 13:58:24 -0700 | [diff] [blame] | 138 | #else |
ctiller | 58393c2 | 2015-01-07 14:03:30 -0800 | [diff] [blame] | 139 | static void unref_by(grpc_fd *fd, int n) { |
Craig Tiller | 9ae7697 | 2015-05-31 13:58:24 -0700 | [diff] [blame] | 140 | gpr_atm old; |
| 141 | #endif |
| 142 | old = gpr_atm_full_fetch_add(&fd->refst, -n); |
Craig Tiller | 23139ae | 2015-02-17 15:46:13 -0800 | [diff] [blame] | 143 | if (old == n) { |
Craig Tiller | 6f43216 | 2015-06-02 12:51:43 -0700 | [diff] [blame] | 144 | freelist_fd(fd); |
Craig Tiller | 23139ae | 2015-02-17 15:46:13 -0800 | [diff] [blame] | 145 | } else { |
| 146 | GPR_ASSERT(old > n); |
David Klempner | d178524 | 2015-01-28 17:00:21 -0800 | [diff] [blame] | 147 | } |
| 148 | } |
| 149 | |
Craig Tiller | 7d41321 | 2015-02-09 08:00:02 -0800 | [diff] [blame] | 150 | void grpc_fd_global_init(void) { gpr_mu_init(&fd_freelist_mu); } |
David Klempner | d178524 | 2015-01-28 17:00:21 -0800 | [diff] [blame] | 151 | |
| 152 | void grpc_fd_global_shutdown(void) { |
| 153 | while (fd_freelist != NULL) { |
| 154 | grpc_fd *fd = fd_freelist; |
| 155 | fd_freelist = fd_freelist->freelist_next; |
ctiller | 58393c2 | 2015-01-07 14:03:30 -0800 | [diff] [blame] | 156 | destroy(fd); |
| 157 | } |
David Klempner | d178524 | 2015-01-28 17:00:21 -0800 | [diff] [blame] | 158 | gpr_mu_destroy(&fd_freelist_mu); |
ctiller | 58393c2 | 2015-01-07 14:03:30 -0800 | [diff] [blame] | 159 | } |
| 160 | |
Craig Tiller | fa275a9 | 2015-06-01 13:55:54 -0700 | [diff] [blame] | 161 | grpc_fd *grpc_fd_create(int fd, const char *name) { |
David Klempner | d178524 | 2015-01-28 17:00:21 -0800 | [diff] [blame] | 162 | grpc_fd *r = alloc_fd(fd); |
Craig Tiller | fa275a9 | 2015-06-01 13:55:54 -0700 | [diff] [blame] | 163 | grpc_iomgr_register_object(&r->iomgr_object, name); |
ctiller | 58393c2 | 2015-01-07 14:03:30 -0800 | [diff] [blame] | 164 | return r; |
| 165 | } |
| 166 | |
| 167 | int grpc_fd_is_orphaned(grpc_fd *fd) { |
| 168 | return (gpr_atm_acq_load(&fd->refst) & 1) == 0; |
| 169 | } |
| 170 | |
Craig Tiller | 886d7ec | 2015-05-14 16:18:42 -0700 | [diff] [blame] | 171 | static void maybe_wake_one_watcher_locked(grpc_fd *fd) { |
Craig Tiller | 354bf6d | 2015-05-18 10:18:03 -0700 | [diff] [blame] | 172 | if (fd->inactive_watcher_root.next != &fd->inactive_watcher_root) { |
| 173 | grpc_pollset_force_kick(fd->inactive_watcher_root.next->pollset); |
| 174 | } else if (fd->read_watcher) { |
| 175 | grpc_pollset_force_kick(fd->read_watcher->pollset); |
| 176 | } else if (fd->write_watcher) { |
| 177 | grpc_pollset_force_kick(fd->write_watcher->pollset); |
Craig Tiller | 886d7ec | 2015-05-14 16:18:42 -0700 | [diff] [blame] | 178 | } |
| 179 | } |
| 180 | |
| 181 | static void maybe_wake_one_watcher(grpc_fd *fd) { |
ctiller | 58393c2 | 2015-01-07 14:03:30 -0800 | [diff] [blame] | 182 | gpr_mu_lock(&fd->watcher_mu); |
Craig Tiller | 886d7ec | 2015-05-14 16:18:42 -0700 | [diff] [blame] | 183 | maybe_wake_one_watcher_locked(fd); |
| 184 | gpr_mu_unlock(&fd->watcher_mu); |
| 185 | } |
| 186 | |
Craig Tiller | c95de72 | 2015-05-29 08:56:46 -0700 | [diff] [blame] | 187 | static void wake_all_watchers_locked(grpc_fd *fd) { |
Craig Tiller | 886d7ec | 2015-05-14 16:18:42 -0700 | [diff] [blame] | 188 | grpc_fd_watcher *watcher; |
Craig Tiller | 8e50fe9 | 2015-05-18 10:45:04 -0700 | [diff] [blame] | 189 | for (watcher = fd->inactive_watcher_root.next; |
| 190 | watcher != &fd->inactive_watcher_root; watcher = watcher->next) { |
Craig Tiller | 7d41321 | 2015-02-09 08:00:02 -0800 | [diff] [blame] | 191 | grpc_pollset_force_kick(watcher->pollset); |
ctiller | 58393c2 | 2015-01-07 14:03:30 -0800 | [diff] [blame] | 192 | } |
Craig Tiller | 354bf6d | 2015-05-18 10:18:03 -0700 | [diff] [blame] | 193 | if (fd->read_watcher) { |
| 194 | grpc_pollset_force_kick(fd->read_watcher->pollset); |
| 195 | } |
| 196 | if (fd->write_watcher && fd->write_watcher != fd->read_watcher) { |
| 197 | grpc_pollset_force_kick(fd->write_watcher->pollset); |
| 198 | } |
ctiller | 58393c2 | 2015-01-07 14:03:30 -0800 | [diff] [blame] | 199 | } |
| 200 | |
Craig Tiller | 8b6cb8d | 2015-06-26 08:08:35 -0700 | [diff] [blame] | 201 | static int has_watchers(grpc_fd *fd) { |
Craig Tiller | 079a11b | 2015-06-30 10:07:15 -0700 | [diff] [blame] | 202 | return fd->read_watcher != NULL || fd->write_watcher != NULL || |
| 203 | fd->inactive_watcher_root.next != &fd->inactive_watcher_root; |
Craig Tiller | 8b6cb8d | 2015-06-26 08:08:35 -0700 | [diff] [blame] | 204 | } |
| 205 | |
Craig Tiller | 4b678bd | 2015-06-02 16:12:24 -0700 | [diff] [blame] | 206 | void grpc_fd_orphan(grpc_fd *fd, grpc_iomgr_closure *on_done, |
| 207 | const char *reason) { |
Craig Tiller | 0317b3d | 2015-06-01 21:57:03 -0700 | [diff] [blame] | 208 | fd->on_done_closure = on_done; |
David Klempner | c6bccc2 | 2015-02-24 17:33:05 -0800 | [diff] [blame] | 209 | shutdown(fd->fd, SHUT_RDWR); |
Craig Tiller | 4b678bd | 2015-06-02 16:12:24 -0700 | [diff] [blame] | 210 | REF_BY(fd, 1, reason); /* remove active status, but keep referenced */ |
Craig Tiller | 017912d | 2015-05-29 07:05:13 -0700 | [diff] [blame] | 211 | gpr_mu_lock(&fd->watcher_mu); |
Craig Tiller | 8b6cb8d | 2015-06-26 08:08:35 -0700 | [diff] [blame] | 212 | if (!has_watchers(fd)) { |
Craig Tiller | 0613e58 | 2015-07-30 11:55:43 -0700 | [diff] [blame^] | 213 | GPR_ASSERT(!fd->closed); |
| 214 | fd->closed = 1; |
Craig Tiller | ee98d84 | 2015-06-29 08:45:55 -0700 | [diff] [blame] | 215 | close(fd->fd); |
Craig Tiller | 8b6cb8d | 2015-06-26 08:08:35 -0700 | [diff] [blame] | 216 | if (fd->on_done_closure) { |
| 217 | grpc_iomgr_add_callback(fd->on_done_closure); |
| 218 | } |
| 219 | } else { |
| 220 | wake_all_watchers_locked(fd); |
| 221 | } |
Craig Tiller | 017912d | 2015-05-29 07:05:13 -0700 | [diff] [blame] | 222 | gpr_mu_unlock(&fd->watcher_mu); |
Craig Tiller | 4b678bd | 2015-06-02 16:12:24 -0700 | [diff] [blame] | 223 | UNREF_BY(fd, 2, reason); /* drop the reference */ |
ctiller | 58393c2 | 2015-01-07 14:03:30 -0800 | [diff] [blame] | 224 | } |
| 225 | |
| 226 | /* increment refcount by two to avoid changing the orphan bit */ |
Craig Tiller | 9ae7697 | 2015-05-31 13:58:24 -0700 | [diff] [blame] | 227 | #ifdef GRPC_FD_REF_COUNT_DEBUG |
Craig Tiller | 0317b3d | 2015-06-01 21:57:03 -0700 | [diff] [blame] | 228 | void grpc_fd_ref(grpc_fd *fd, const char *reason, const char *file, int line) { |
| 229 | ref_by(fd, 2, reason, file, line); |
Craig Tiller | 9ae7697 | 2015-05-31 13:58:24 -0700 | [diff] [blame] | 230 | } |
ctiller | 58393c2 | 2015-01-07 14:03:30 -0800 | [diff] [blame] | 231 | |
Craig Tiller | 0317b3d | 2015-06-01 21:57:03 -0700 | [diff] [blame] | 232 | void grpc_fd_unref(grpc_fd *fd, const char *reason, const char *file, |
| 233 | int line) { |
| 234 | unref_by(fd, 2, reason, file, line); |
Craig Tiller | 9ae7697 | 2015-05-31 13:58:24 -0700 | [diff] [blame] | 235 | } |
| 236 | #else |
Craig Tiller | 0317b3d | 2015-06-01 21:57:03 -0700 | [diff] [blame] | 237 | void grpc_fd_ref(grpc_fd *fd) { ref_by(fd, 2); } |
Craig Tiller | 9ae7697 | 2015-05-31 13:58:24 -0700 | [diff] [blame] | 238 | |
Craig Tiller | 0317b3d | 2015-06-01 21:57:03 -0700 | [diff] [blame] | 239 | void grpc_fd_unref(grpc_fd *fd) { unref_by(fd, 2); } |
Craig Tiller | 9ae7697 | 2015-05-31 13:58:24 -0700 | [diff] [blame] | 240 | #endif |
ctiller | 58393c2 | 2015-01-07 14:03:30 -0800 | [diff] [blame] | 241 | |
David Garcia Quintas | a30020f | 2015-05-27 19:21:01 -0700 | [diff] [blame] | 242 | static void process_callback(grpc_iomgr_closure *closure, int success, |
Craig Tiller | 8674cb1 | 2015-06-05 07:09:25 -0700 | [diff] [blame] | 243 | int allow_synchronous_callback) { |
ctiller | 58393c2 | 2015-01-07 14:03:30 -0800 | [diff] [blame] | 244 | if (allow_synchronous_callback) { |
David Garcia Quintas | a30020f | 2015-05-27 19:21:01 -0700 | [diff] [blame] | 245 | closure->cb(closure->cb_arg, success); |
ctiller | 58393c2 | 2015-01-07 14:03:30 -0800 | [diff] [blame] | 246 | } else { |
David Garcia Quintas | a30020f | 2015-05-27 19:21:01 -0700 | [diff] [blame] | 247 | grpc_iomgr_add_delayed_callback(closure, success); |
ctiller | 58393c2 | 2015-01-07 14:03:30 -0800 | [diff] [blame] | 248 | } |
| 249 | } |
| 250 | |
David Garcia Quintas | a30020f | 2015-05-27 19:21:01 -0700 | [diff] [blame] | 251 | static void process_callbacks(grpc_iomgr_closure *callbacks, size_t n, |
| 252 | int success, int allow_synchronous_callback) { |
ctiller | 58393c2 | 2015-01-07 14:03:30 -0800 | [diff] [blame] | 253 | size_t i; |
| 254 | for (i = 0; i < n; i++) { |
David Garcia Quintas | a30020f | 2015-05-27 19:21:01 -0700 | [diff] [blame] | 255 | process_callback(callbacks + i, success, allow_synchronous_callback); |
ctiller | 58393c2 | 2015-01-07 14:03:30 -0800 | [diff] [blame] | 256 | } |
| 257 | } |
| 258 | |
Craig Tiller | f95e37f | 2015-02-18 15:15:29 -0800 | [diff] [blame] | 259 | static void notify_on(grpc_fd *fd, gpr_atm *st, grpc_iomgr_closure *closure, |
| 260 | int allow_synchronous_callback) { |
Craig Tiller | 0fcd53c | 2015-02-18 15:10:53 -0800 | [diff] [blame] | 261 | switch (gpr_atm_acq_load(st)) { |
ctiller | 58393c2 | 2015-01-07 14:03:30 -0800 | [diff] [blame] | 262 | case NOT_READY: |
| 263 | /* There is no race if the descriptor is already ready, so we skip |
| 264 | the interlocked op in that case. As long as the app doesn't |
| 265 | try to set the same upcall twice (which it shouldn't) then |
| 266 | oldval should never be anything other than READY or NOT_READY. We |
| 267 | don't |
| 268 | check for user error on the fast path. */ |
Craig Tiller | f95e37f | 2015-02-18 15:15:29 -0800 | [diff] [blame] | 269 | if (gpr_atm_rel_cas(st, NOT_READY, (gpr_intptr)closure)) { |
ctiller | 58393c2 | 2015-01-07 14:03:30 -0800 | [diff] [blame] | 270 | /* swap was successful -- the closure will run after the next |
| 271 | set_ready call. NOTE: we don't have an ABA problem here, |
| 272 | since we should never have concurrent calls to the same |
| 273 | notify_on function. */ |
Craig Tiller | 886d7ec | 2015-05-14 16:18:42 -0700 | [diff] [blame] | 274 | maybe_wake_one_watcher(fd); |
ctiller | 58393c2 | 2015-01-07 14:03:30 -0800 | [diff] [blame] | 275 | return; |
| 276 | } |
| 277 | /* swap was unsuccessful due to an intervening set_ready call. |
| 278 | Fall through to the READY code below */ |
| 279 | case READY: |
David Garcia Quintas | 07997b6 | 2015-05-28 13:51:50 -0700 | [diff] [blame] | 280 | GPR_ASSERT(gpr_atm_no_barrier_load(st) == READY); |
Craig Tiller | 0fcd53c | 2015-02-18 15:10:53 -0800 | [diff] [blame] | 281 | gpr_atm_rel_store(st, NOT_READY); |
David Garcia Quintas | a30020f | 2015-05-27 19:21:01 -0700 | [diff] [blame] | 282 | process_callback(closure, !gpr_atm_acq_load(&fd->shutdown), |
Craig Tiller | 8674cb1 | 2015-06-05 07:09:25 -0700 | [diff] [blame] | 283 | allow_synchronous_callback); |
ctiller | 58393c2 | 2015-01-07 14:03:30 -0800 | [diff] [blame] | 284 | return; |
Craig Tiller | 0fcd53c | 2015-02-18 15:10:53 -0800 | [diff] [blame] | 285 | default: /* WAITING */ |
ctiller | 58393c2 | 2015-01-07 14:03:30 -0800 | [diff] [blame] | 286 | /* upcallptr was set to a different closure. This is an error! */ |
| 287 | gpr_log(GPR_ERROR, |
| 288 | "User called a notify_on function with a previous callback still " |
| 289 | "pending"); |
| 290 | abort(); |
| 291 | } |
| 292 | gpr_log(GPR_ERROR, "Corrupt memory in &st->state"); |
| 293 | abort(); |
| 294 | } |
| 295 | |
David Garcia Quintas | 2738ae8 | 2015-05-28 16:06:48 -0700 | [diff] [blame] | 296 | static void set_ready_locked(gpr_atm *st, grpc_iomgr_closure **callbacks, |
ctiller | 58393c2 | 2015-01-07 14:03:30 -0800 | [diff] [blame] | 297 | size_t *ncallbacks) { |
Craig Tiller | 0fcd53c | 2015-02-18 15:10:53 -0800 | [diff] [blame] | 298 | gpr_intptr state = gpr_atm_acq_load(st); |
ctiller | 58393c2 | 2015-01-07 14:03:30 -0800 | [diff] [blame] | 299 | |
Craig Tiller | 0fcd53c | 2015-02-18 15:10:53 -0800 | [diff] [blame] | 300 | switch (state) { |
| 301 | case READY: |
| 302 | /* duplicate ready, ignore */ |
| 303 | return; |
ctiller | 58393c2 | 2015-01-07 14:03:30 -0800 | [diff] [blame] | 304 | case NOT_READY: |
Craig Tiller | 0fcd53c | 2015-02-18 15:10:53 -0800 | [diff] [blame] | 305 | if (gpr_atm_rel_cas(st, NOT_READY, READY)) { |
ctiller | 58393c2 | 2015-01-07 14:03:30 -0800 | [diff] [blame] | 306 | /* swap was successful -- the closure will run after the next |
| 307 | notify_on call. */ |
| 308 | return; |
| 309 | } |
Craig Tiller | f95e37f | 2015-02-18 15:15:29 -0800 | [diff] [blame] | 310 | /* swap was unsuccessful due to an intervening set_ready call. |
| 311 | Fall through to the WAITING code below */ |
Craig Tiller | 0fcd53c | 2015-02-18 15:10:53 -0800 | [diff] [blame] | 312 | state = gpr_atm_acq_load(st); |
| 313 | default: /* waiting */ |
David Garcia Quintas | 07997b6 | 2015-05-28 13:51:50 -0700 | [diff] [blame] | 314 | GPR_ASSERT(gpr_atm_no_barrier_load(st) != READY && |
| 315 | gpr_atm_no_barrier_load(st) != NOT_READY); |
David Garcia Quintas | 2738ae8 | 2015-05-28 16:06:48 -0700 | [diff] [blame] | 316 | callbacks[(*ncallbacks)++] = (grpc_iomgr_closure *)state; |
Craig Tiller | 0fcd53c | 2015-02-18 15:10:53 -0800 | [diff] [blame] | 317 | gpr_atm_rel_store(st, NOT_READY); |
ctiller | 58393c2 | 2015-01-07 14:03:30 -0800 | [diff] [blame] | 318 | return; |
| 319 | } |
| 320 | } |
| 321 | |
Craig Tiller | 0fcd53c | 2015-02-18 15:10:53 -0800 | [diff] [blame] | 322 | static void set_ready(grpc_fd *fd, gpr_atm *st, |
ctiller | 58393c2 | 2015-01-07 14:03:30 -0800 | [diff] [blame] | 323 | int allow_synchronous_callback) { |
| 324 | /* only one set_ready can be active at once (but there may be a racing |
| 325 | notify_on) */ |
| 326 | int success; |
Craig Tiller | 8674cb1 | 2015-06-05 07:09:25 -0700 | [diff] [blame] | 327 | grpc_iomgr_closure *closure; |
ctiller | 58393c2 | 2015-01-07 14:03:30 -0800 | [diff] [blame] | 328 | size_t ncb = 0; |
David Garcia Quintas | a30020f | 2015-05-27 19:21:01 -0700 | [diff] [blame] | 329 | |
ctiller | 58393c2 | 2015-01-07 14:03:30 -0800 | [diff] [blame] | 330 | gpr_mu_lock(&fd->set_state_mu); |
David Garcia Quintas | a30020f | 2015-05-27 19:21:01 -0700 | [diff] [blame] | 331 | set_ready_locked(st, &closure, &ncb); |
ctiller | 58393c2 | 2015-01-07 14:03:30 -0800 | [diff] [blame] | 332 | gpr_mu_unlock(&fd->set_state_mu); |
| 333 | success = !gpr_atm_acq_load(&fd->shutdown); |
David Garcia Quintas | 07997b6 | 2015-05-28 13:51:50 -0700 | [diff] [blame] | 334 | GPR_ASSERT(ncb <= 1); |
David Garcia Quintas | a30020f | 2015-05-27 19:21:01 -0700 | [diff] [blame] | 335 | if (ncb > 0) { |
David Garcia Quintas | 2738ae8 | 2015-05-28 16:06:48 -0700 | [diff] [blame] | 336 | process_callbacks(closure, ncb, success, allow_synchronous_callback); |
David Garcia Quintas | a30020f | 2015-05-27 19:21:01 -0700 | [diff] [blame] | 337 | } |
ctiller | 58393c2 | 2015-01-07 14:03:30 -0800 | [diff] [blame] | 338 | } |
| 339 | |
| 340 | void grpc_fd_shutdown(grpc_fd *fd) { |
ctiller | 58393c2 | 2015-01-07 14:03:30 -0800 | [diff] [blame] | 341 | size_t ncb = 0; |
| 342 | gpr_mu_lock(&fd->set_state_mu); |
David Klempner | 466423b | 2015-03-11 15:00:46 -0700 | [diff] [blame] | 343 | GPR_ASSERT(!gpr_atm_no_barrier_load(&fd->shutdown)); |
ctiller | 58393c2 | 2015-01-07 14:03:30 -0800 | [diff] [blame] | 344 | gpr_atm_rel_store(&fd->shutdown, 1); |
David Garcia Quintas | 2738ae8 | 2015-05-28 16:06:48 -0700 | [diff] [blame] | 345 | set_ready_locked(&fd->readst, &fd->shutdown_closures[0], &ncb); |
| 346 | set_ready_locked(&fd->writest, &fd->shutdown_closures[0], &ncb); |
ctiller | 58393c2 | 2015-01-07 14:03:30 -0800 | [diff] [blame] | 347 | gpr_mu_unlock(&fd->set_state_mu); |
David Garcia Quintas | 07997b6 | 2015-05-28 13:51:50 -0700 | [diff] [blame] | 348 | GPR_ASSERT(ncb <= 2); |
David Garcia Quintas | 1c762bd | 2015-05-31 17:04:43 -0700 | [diff] [blame] | 349 | process_callbacks(fd->shutdown_closures[0], ncb, 0 /* GPR_FALSE */, |
| 350 | 0 /* GPR_FALSE */); |
ctiller | 58393c2 | 2015-01-07 14:03:30 -0800 | [diff] [blame] | 351 | } |
| 352 | |
Craig Tiller | 0fcd53c | 2015-02-18 15:10:53 -0800 | [diff] [blame] | 353 | void grpc_fd_notify_on_read(grpc_fd *fd, grpc_iomgr_closure *closure) { |
| 354 | notify_on(fd, &fd->readst, closure, 0); |
ctiller | 58393c2 | 2015-01-07 14:03:30 -0800 | [diff] [blame] | 355 | } |
| 356 | |
Craig Tiller | 0fcd53c | 2015-02-18 15:10:53 -0800 | [diff] [blame] | 357 | void grpc_fd_notify_on_write(grpc_fd *fd, grpc_iomgr_closure *closure) { |
| 358 | notify_on(fd, &fd->writest, closure, 0); |
ctiller | 58393c2 | 2015-01-07 14:03:30 -0800 | [diff] [blame] | 359 | } |
| 360 | |
| 361 | gpr_uint32 grpc_fd_begin_poll(grpc_fd *fd, grpc_pollset *pollset, |
Craig Tiller | 7d41321 | 2015-02-09 08:00:02 -0800 | [diff] [blame] | 362 | gpr_uint32 read_mask, gpr_uint32 write_mask, |
| 363 | grpc_fd_watcher *watcher) { |
Craig Tiller | 886d7ec | 2015-05-14 16:18:42 -0700 | [diff] [blame] | 364 | gpr_uint32 mask = 0; |
ctiller | 58393c2 | 2015-01-07 14:03:30 -0800 | [diff] [blame] | 365 | /* keep track of pollers that have requested our events, in case they change |
| 366 | */ |
Craig Tiller | 9ae7697 | 2015-05-31 13:58:24 -0700 | [diff] [blame] | 367 | GRPC_FD_REF(fd, "poll"); |
Craig Tiller | 59ea16f | 2015-02-18 16:18:08 -0800 | [diff] [blame] | 368 | |
ctiller | 58393c2 | 2015-01-07 14:03:30 -0800 | [diff] [blame] | 369 | gpr_mu_lock(&fd->watcher_mu); |
Craig Tiller | 8b6cb8d | 2015-06-26 08:08:35 -0700 | [diff] [blame] | 370 | /* if we are shutdown, then don't add to the watcher set */ |
| 371 | if (gpr_atm_no_barrier_load(&fd->shutdown)) { |
| 372 | watcher->fd = NULL; |
| 373 | watcher->pollset = NULL; |
| 374 | gpr_mu_unlock(&fd->watcher_mu); |
Craig Tiller | df26af5 | 2015-07-01 16:53:09 -0700 | [diff] [blame] | 375 | GRPC_FD_UNREF(fd, "poll"); |
Craig Tiller | 8b6cb8d | 2015-06-26 08:08:35 -0700 | [diff] [blame] | 376 | return 0; |
| 377 | } |
Craig Tiller | 886d7ec | 2015-05-14 16:18:42 -0700 | [diff] [blame] | 378 | /* if there is nobody polling for read, but we need to, then start doing so */ |
Craig Tiller | 2daa88c | 2015-07-01 14:46:42 -0700 | [diff] [blame] | 379 | if (read_mask && !fd->read_watcher && gpr_atm_acq_load(&fd->readst) > READY) { |
Craig Tiller | 886d7ec | 2015-05-14 16:18:42 -0700 | [diff] [blame] | 380 | fd->read_watcher = watcher; |
| 381 | mask |= read_mask; |
| 382 | } |
Craig Tiller | 8e50fe9 | 2015-05-18 10:45:04 -0700 | [diff] [blame] | 383 | /* if there is nobody polling for write, but we need to, then start doing so |
| 384 | */ |
Craig Tiller | 2daa88c | 2015-07-01 14:46:42 -0700 | [diff] [blame] | 385 | if (write_mask && !fd->write_watcher && gpr_atm_acq_load(&fd->writest) > READY) { |
Craig Tiller | 886d7ec | 2015-05-14 16:18:42 -0700 | [diff] [blame] | 386 | fd->write_watcher = watcher; |
| 387 | mask |= write_mask; |
| 388 | } |
| 389 | /* if not polling, remember this watcher in case we need someone to later */ |
| 390 | if (mask == 0) { |
Craig Tiller | 354bf6d | 2015-05-18 10:18:03 -0700 | [diff] [blame] | 391 | watcher->next = &fd->inactive_watcher_root; |
Craig Tiller | 886d7ec | 2015-05-14 16:18:42 -0700 | [diff] [blame] | 392 | watcher->prev = watcher->next->prev; |
| 393 | watcher->next->prev = watcher->prev->next = watcher; |
| 394 | } |
Craig Tiller | 7d41321 | 2015-02-09 08:00:02 -0800 | [diff] [blame] | 395 | watcher->pollset = pollset; |
| 396 | watcher->fd = fd; |
ctiller | 58393c2 | 2015-01-07 14:03:30 -0800 | [diff] [blame] | 397 | gpr_mu_unlock(&fd->watcher_mu); |
| 398 | |
Craig Tiller | 886d7ec | 2015-05-14 16:18:42 -0700 | [diff] [blame] | 399 | return mask; |
ctiller | 58393c2 | 2015-01-07 14:03:30 -0800 | [diff] [blame] | 400 | } |
| 401 | |
Craig Tiller | 886d7ec | 2015-05-14 16:18:42 -0700 | [diff] [blame] | 402 | void grpc_fd_end_poll(grpc_fd_watcher *watcher, int got_read, int got_write) { |
| 403 | int was_polling = 0; |
| 404 | int kick = 0; |
| 405 | grpc_fd *fd = watcher->fd; |
Craig Tiller | 59ea16f | 2015-02-18 16:18:08 -0800 | [diff] [blame] | 406 | |
Craig Tiller | 8b6cb8d | 2015-06-26 08:08:35 -0700 | [diff] [blame] | 407 | if (fd == NULL) { |
| 408 | return; |
| 409 | } |
| 410 | |
Craig Tiller | 886d7ec | 2015-05-14 16:18:42 -0700 | [diff] [blame] | 411 | gpr_mu_lock(&fd->watcher_mu); |
| 412 | if (watcher == fd->read_watcher) { |
Craig Tiller | 236d098 | 2015-05-18 10:26:44 -0700 | [diff] [blame] | 413 | /* remove read watcher, kick if we still need a read */ |
Craig Tiller | 886d7ec | 2015-05-14 16:18:42 -0700 | [diff] [blame] | 414 | was_polling = 1; |
Craig Tiller | 8e50fe9 | 2015-05-18 10:45:04 -0700 | [diff] [blame] | 415 | kick = kick || !got_read; |
Craig Tiller | 886d7ec | 2015-05-14 16:18:42 -0700 | [diff] [blame] | 416 | fd->read_watcher = NULL; |
| 417 | } |
| 418 | if (watcher == fd->write_watcher) { |
Craig Tiller | 236d098 | 2015-05-18 10:26:44 -0700 | [diff] [blame] | 419 | /* remove write watcher, kick if we still need a write */ |
Craig Tiller | 886d7ec | 2015-05-14 16:18:42 -0700 | [diff] [blame] | 420 | was_polling = 1; |
Craig Tiller | 8e50fe9 | 2015-05-18 10:45:04 -0700 | [diff] [blame] | 421 | kick = kick || !got_write; |
Craig Tiller | 886d7ec | 2015-05-14 16:18:42 -0700 | [diff] [blame] | 422 | fd->write_watcher = NULL; |
| 423 | } |
| 424 | if (!was_polling) { |
Craig Tiller | 236d098 | 2015-05-18 10:26:44 -0700 | [diff] [blame] | 425 | /* remove from inactive list */ |
Craig Tiller | 886d7ec | 2015-05-14 16:18:42 -0700 | [diff] [blame] | 426 | watcher->next->prev = watcher->prev; |
| 427 | watcher->prev->next = watcher->next; |
| 428 | } |
| 429 | if (kick) { |
| 430 | maybe_wake_one_watcher_locked(fd); |
| 431 | } |
Craig Tiller | 0613e58 | 2015-07-30 11:55:43 -0700 | [diff] [blame^] | 432 | if (grpc_fd_is_orphaned(fd) && !has_watchers(fd) && !fd->closed) { |
| 433 | fd->closed = 1; |
Craig Tiller | ee98d84 | 2015-06-29 08:45:55 -0700 | [diff] [blame] | 434 | close(fd->fd); |
| 435 | if (fd->on_done_closure != NULL) { |
| 436 | grpc_iomgr_add_callback(fd->on_done_closure); |
| 437 | } |
Craig Tiller | 8b6cb8d | 2015-06-26 08:08:35 -0700 | [diff] [blame] | 438 | } |
Craig Tiller | 886d7ec | 2015-05-14 16:18:42 -0700 | [diff] [blame] | 439 | gpr_mu_unlock(&fd->watcher_mu); |
| 440 | |
Craig Tiller | 9ae7697 | 2015-05-31 13:58:24 -0700 | [diff] [blame] | 441 | GRPC_FD_UNREF(fd, "poll"); |
ctiller | 58393c2 | 2015-01-07 14:03:30 -0800 | [diff] [blame] | 442 | } |
| 443 | |
| 444 | void grpc_fd_become_readable(grpc_fd *fd, int allow_synchronous_callback) { |
| 445 | set_ready(fd, &fd->readst, allow_synchronous_callback); |
| 446 | } |
| 447 | |
| 448 | void grpc_fd_become_writable(grpc_fd *fd, int allow_synchronous_callback) { |
| 449 | set_ready(fd, &fd->writest, allow_synchronous_callback); |
| 450 | } |
Craig Tiller | d14a1a5 | 2015-01-21 15:26:29 -0800 | [diff] [blame] | 451 | |
Craig Tiller | 190d360 | 2015-02-18 09:23:38 -0800 | [diff] [blame] | 452 | #endif |