Start sketching hierarchical turnstile
diff --git a/src/core/lib/iomgr/ev_epoll1_linux.c b/src/core/lib/iomgr/ev_epoll1_linux.c
index 195c3bd..fdd6384 100644
--- a/src/core/lib/iomgr/ev_epoll1_linux.c
+++ b/src/core/lib/iomgr/ev_epoll1_linux.c
@@ -48,6 +48,7 @@
#include <unistd.h>
#include <grpc/support/alloc.h>
+#include <grpc/support/cpu.h>
#include <grpc/support/log.h>
#include <grpc/support/string_util.h>
#include <grpc/support/tls.h>
@@ -113,18 +114,32 @@
gpr_cv cv;
};
+typedef struct pollset_neighbourhood {
+ gpr_mu mu;
+ grpc_pollset *active_root;
+ grpc_pollset *inactive_root;
+ bool seen_inactive;
+ char pad[GPR_CACHELINE_SIZE];
+} pollset_neighbourhood;
+
struct grpc_pollset {
+ gpr_mu mu;
+ pollset_neighbourhood *neighbourhood;
grpc_pollset_worker *root_worker;
bool kicked_without_poller;
-
+ bool seen_inactive;
bool shutting_down; /* Is the pollset shutting down ? */
bool finish_shutdown_called; /* Is the 'finish_shutdown_locked()' called ? */
grpc_closure *shutdown_closure; /* Called after after shutdown is complete */
+
+ grpc_pollset *next;
+ grpc_pollset *prev;
};
/*******************************************************************************
* Pollset-set Declarations
*/
+
struct grpc_pollset_set {};
/*******************************************************************************
@@ -303,6 +318,11 @@
* Pollset Definitions
*/
+GPR_TLS_DECL(g_current_thread_pollset);
+GPR_TLS_DECL(g_current_thread_worker);
+static gpr_atm g_active_poller;
+static pollset_neighbourhood *g_neighbourhoods;
+
/* Return true if first in list */
static bool worker_insert(grpc_pollset_worker **root, pollset_worker_links link,
grpc_pollset_worker *worker) {
@@ -342,15 +362,10 @@
}
}
-GPR_TLS_DECL(g_current_thread_pollset);
-GPR_TLS_DECL(g_current_thread_worker);
-static gpr_mu g_pollset_mu;
-static grpc_pollset_worker *g_root_worker;
-
static grpc_error *pollset_global_init(void) {
- gpr_mu_init(&g_pollset_mu);
gpr_tls_init(&g_current_thread_pollset);
gpr_tls_init(&g_current_thread_worker);
+ gpr_atm_no_barrier_store(&g_active_poller, 0);
global_wakeup_fd.read_fd = -1;
grpc_error *err = grpc_wakeup_fd_init(&global_wakeup_fd);
if (err != GRPC_ERROR_NONE) return err;
@@ -363,14 +378,32 @@
}
static void pollset_global_shutdown(void) {
- gpr_mu_destroy(&g_pollset_mu);
gpr_tls_destroy(&g_current_thread_pollset);
gpr_tls_destroy(&g_current_thread_worker);
if (global_wakeup_fd.read_fd != -1) grpc_wakeup_fd_destroy(&global_wakeup_fd);
}
static void pollset_init(grpc_pollset *pollset, gpr_mu **mu) {
- *mu = &g_pollset_mu;
+ gpr_mu_init(&pollset->mu);
+ *mu = &pollset->mu;
+ pollset->neighbourhood = &g_neighbourhoods[gpr_cpu_current_cpu()];
+ pollset->seen_inactive = true;
+ pollset->next = pollset->prev = pollset;
+}
+
+static void pollset_destroy(grpc_pollset *pollset) {
+ gpr_mu_destroy(&pollset->mu);
+ gpr_mu_lock(&pollset->neighbourhood->mu);
+ pollset->prev->next = pollset->next;
+ pollset->next->prev = pollset->prev;
+ if (pollset == pollset->neighbourhood->active_root) {
+ pollset->neighbourhood->active_root =
+ pollset->next == pollset ? NULL : pollset->next;
+ } else if (pollset == pollset->neighbourhood->inactive_root) {
+ pollset->neighbourhood->inactive_root =
+ pollset->next == pollset ? NULL : pollset->next;
+ }
+ gpr_mu_unlock(&pollset->neighbourhood->mu);
}
static grpc_error *pollset_kick_all(grpc_pollset *pollset) {
@@ -408,8 +441,6 @@
pollset_maybe_finish_shutdown(exec_ctx, pollset);
}
-static void pollset_destroy(grpc_pollset *pollset) {}
-
#define MAX_EPOLL_EVENTS 100
static int poll_deadline_to_millis_timeout(gpr_timespec deadline,