Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 1 | /* |
| 2 | * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved. |
| 3 | * |
| 4 | * Copyright (c) 2011, 2012, Intel Corporation. |
| 5 | * |
| 6 | * Author: Zach Brown <zab@zabbo.net> |
| 7 | * Author: Peter J. Braam <braam@clusterfs.com> |
| 8 | * Author: Phil Schwan <phil@clusterfs.com> |
| 9 | * Author: Eric Barton <eric@bartonsoftware.com> |
| 10 | * |
| 11 | * This file is part of Lustre, http://www.lustre.org |
| 12 | * |
| 13 | * Portals is free software; you can redistribute it and/or |
| 14 | * modify it under the terms of version 2 of the GNU General Public |
| 15 | * License as published by the Free Software Foundation. |
| 16 | * |
| 17 | * Portals is distributed in the hope that it will be useful, |
| 18 | * but WITHOUT ANY WARRANTY; without even the implied warranty of |
| 19 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
| 20 | * GNU General Public License for more details. |
| 21 | * |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 22 | */ |
| 23 | |
John L. Hammond | 12c41f0 | 2015-06-11 15:18:10 -0400 | [diff] [blame] | 24 | #ifndef _SOCKLND_SOCKLND_H_ |
| 25 | #define _SOCKLND_SOCKLND_H_ |
| 26 | |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 27 | #define DEBUG_PORTAL_ALLOC |
| 28 | #define DEBUG_SUBSYSTEM S_LND |
| 29 | |
John L. Hammond | 12c41f0 | 2015-06-11 15:18:10 -0400 | [diff] [blame] | 30 | #include <linux/crc32.h> |
| 31 | #include <linux/errno.h> |
| 32 | #include <linux/if.h> |
| 33 | #include <linux/init.h> |
| 34 | #include <linux/kernel.h> |
| 35 | #include <linux/kmod.h> |
| 36 | #include <linux/list.h> |
| 37 | #include <linux/mm.h> |
| 38 | #include <linux/module.h> |
| 39 | #include <linux/stat.h> |
| 40 | #include <linux/string.h> |
| 41 | #include <linux/syscalls.h> |
| 42 | #include <linux/sysctl.h> |
| 43 | #include <linux/uio.h> |
| 44 | #include <linux/unistd.h> |
Oleg Drokin | a82de7e | 2015-06-24 10:07:45 -0400 | [diff] [blame] | 45 | #include <asm/irq.h> |
John L. Hammond | 12c41f0 | 2015-06-11 15:18:10 -0400 | [diff] [blame] | 46 | #include <net/sock.h> |
| 47 | #include <net/tcp.h> |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 48 | |
Greg Kroah-Hartman | 9fdaf8c | 2014-07-11 20:51:16 -0700 | [diff] [blame] | 49 | #include "../../../include/linux/libcfs/libcfs.h" |
| 50 | #include "../../../include/linux/lnet/lnet.h" |
| 51 | #include "../../../include/linux/lnet/lib-lnet.h" |
| 52 | #include "../../../include/linux/lnet/socklnd.h" |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 53 | |
John L. Hammond | 12c41f0 | 2015-06-11 15:18:10 -0400 | [diff] [blame] | 54 | /* assume one thread for each connection type */ |
| 55 | #define SOCKNAL_NSCHEDS 3 |
| 56 | #define SOCKNAL_NSCHEDS_HIGH (SOCKNAL_NSCHEDS << 1) |
| 57 | |
Mike Shuey | 97d10d0 | 2015-05-19 10:14:37 -0400 | [diff] [blame] | 58 | #define SOCKNAL_PEER_HASH_SIZE 101 /* # peer lists */ |
| 59 | #define SOCKNAL_RESCHED 100 /* # scheduler loops before reschedule */ |
| 60 | #define SOCKNAL_INSANITY_RECONN 5000 /* connd is trying on reconn infinitely */ |
| 61 | #define SOCKNAL_ENOMEM_RETRY CFS_TICK /* jiffies between retries */ |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 62 | |
Mike Shuey | 97d10d0 | 2015-05-19 10:14:37 -0400 | [diff] [blame] | 63 | #define SOCKNAL_SINGLE_FRAG_TX 0 /* disable multi-fragment sends */ |
| 64 | #define SOCKNAL_SINGLE_FRAG_RX 0 /* disable multi-fragment receives */ |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 65 | |
Mike Shuey | 97d10d0 | 2015-05-19 10:14:37 -0400 | [diff] [blame] | 66 | #define SOCKNAL_VERSION_DEBUG 0 /* enable protocol version debugging */ |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 67 | |
James Simmons | 4420cfd | 2016-02-12 12:06:00 -0500 | [diff] [blame] | 68 | /* |
| 69 | * risk kmap deadlock on multi-frag I/O (backs off to single-frag if disabled). |
| 70 | * no risk if we're not running on a CONFIG_HIGHMEM platform. |
| 71 | */ |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 72 | #ifdef CONFIG_HIGHMEM |
| 73 | # define SOCKNAL_RISK_KMAP_DEADLOCK 0 |
| 74 | #else |
| 75 | # define SOCKNAL_RISK_KMAP_DEADLOCK 1 |
| 76 | #endif |
| 77 | |
| 78 | struct ksock_sched_info; |
| 79 | |
| 80 | typedef struct /* per scheduler state */ |
| 81 | { |
Mike Shuey | 97d10d0 | 2015-05-19 10:14:37 -0400 | [diff] [blame] | 82 | spinlock_t kss_lock; /* serialise */ |
| 83 | struct list_head kss_rx_conns; /* conn waiting to be read */ |
| 84 | struct list_head kss_tx_conns; /* conn waiting to be written */ |
| 85 | struct list_head kss_zombie_noop_txs; /* zombie noop tx list */ |
| 86 | wait_queue_head_t kss_waitq; /* where scheduler sleeps */ |
| 87 | int kss_nconns; /* # connections assigned to |
| 88 | * this scheduler */ |
| 89 | struct ksock_sched_info *kss_info; /* owner of it */ |
| 90 | struct page *kss_rx_scratch_pgs[LNET_MAX_IOV]; |
| 91 | struct kvec kss_scratch_iov[LNET_MAX_IOV]; |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 92 | } ksock_sched_t; |
| 93 | |
| 94 | struct ksock_sched_info { |
Mike Shuey | 97d10d0 | 2015-05-19 10:14:37 -0400 | [diff] [blame] | 95 | int ksi_nthreads_max; /* max allowed threads */ |
| 96 | int ksi_nthreads; /* number of threads */ |
| 97 | int ksi_cpt; /* CPT id */ |
| 98 | ksock_sched_t *ksi_scheds; /* array of schedulers */ |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 99 | }; |
| 100 | |
Mike Shuey | 97d10d0 | 2015-05-19 10:14:37 -0400 | [diff] [blame] | 101 | #define KSOCK_CPT_SHIFT 16 |
| 102 | #define KSOCK_THREAD_ID(cpt, sid) (((cpt) << KSOCK_CPT_SHIFT) | (sid)) |
| 103 | #define KSOCK_THREAD_CPT(id) ((id) >> KSOCK_CPT_SHIFT) |
| 104 | #define KSOCK_THREAD_SID(id) ((id) & ((1UL << KSOCK_CPT_SHIFT) - 1)) |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 105 | |
Mike Shuey | 97d10d0 | 2015-05-19 10:14:37 -0400 | [diff] [blame] | 106 | typedef struct /* in-use interface */ |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 107 | { |
| 108 | __u32 ksni_ipaddr; /* interface's IP address */ |
| 109 | __u32 ksni_netmask; /* interface's network mask */ |
| 110 | int ksni_nroutes; /* # routes using (active) */ |
| 111 | int ksni_npeers; /* # peers using (passive) */ |
| 112 | char ksni_name[IFNAMSIZ]; /* interface name */ |
| 113 | } ksock_interface_t; |
| 114 | |
Greg Donald | 75c49d4 | 2014-09-22 09:39:53 -0500 | [diff] [blame] | 115 | typedef struct { |
Mike Shuey | 97d10d0 | 2015-05-19 10:14:37 -0400 | [diff] [blame] | 116 | int *ksnd_timeout; /* "stuck" socket timeout |
| 117 | * (seconds) */ |
| 118 | int *ksnd_nscheds; /* # scheduler threads in each |
| 119 | * pool while starting */ |
| 120 | int *ksnd_nconnds; /* # connection daemons */ |
| 121 | int *ksnd_nconnds_max; /* max # connection daemons */ |
| 122 | int *ksnd_min_reconnectms; /* first connection retry after |
| 123 | * (ms)... */ |
| 124 | int *ksnd_max_reconnectms; /* ...exponentially increasing to |
| 125 | * this */ |
| 126 | int *ksnd_eager_ack; /* make TCP ack eagerly? */ |
| 127 | int *ksnd_typed_conns; /* drive sockets by type? */ |
| 128 | int *ksnd_min_bulk; /* smallest "large" message */ |
| 129 | int *ksnd_tx_buffer_size; /* socket tx buffer size */ |
| 130 | int *ksnd_rx_buffer_size; /* socket rx buffer size */ |
| 131 | int *ksnd_nagle; /* enable NAGLE? */ |
| 132 | int *ksnd_round_robin; /* round robin for multiple |
| 133 | * interfaces */ |
| 134 | int *ksnd_keepalive; /* # secs for sending keepalive |
| 135 | * NOOP */ |
| 136 | int *ksnd_keepalive_idle; /* # idle secs before 1st probe |
| 137 | */ |
| 138 | int *ksnd_keepalive_count; /* # probes */ |
| 139 | int *ksnd_keepalive_intvl; /* time between probes */ |
| 140 | int *ksnd_credits; /* # concurrent sends */ |
| 141 | int *ksnd_peertxcredits; /* # concurrent sends to 1 peer |
| 142 | */ |
| 143 | int *ksnd_peerrtrcredits; /* # per-peer router buffer |
| 144 | * credits */ |
| 145 | int *ksnd_peertimeout; /* seconds to consider peer dead |
| 146 | */ |
| 147 | int *ksnd_enable_csum; /* enable check sum */ |
| 148 | int *ksnd_inject_csum_error; /* set non-zero to inject |
| 149 | * checksum error */ |
| 150 | int *ksnd_nonblk_zcack; /* always send zc-ack on |
| 151 | * non-blocking connection */ |
| 152 | unsigned int *ksnd_zc_min_payload; /* minimum zero copy payload |
| 153 | * size */ |
| 154 | int *ksnd_zc_recv; /* enable ZC receive (for |
| 155 | * Chelsio TOE) */ |
| 156 | int *ksnd_zc_recv_min_nfrags; /* minimum # of fragments to |
| 157 | * enable ZC receive */ |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 158 | } ksock_tunables_t; |
| 159 | |
Greg Donald | 75c49d4 | 2014-09-22 09:39:53 -0500 | [diff] [blame] | 160 | typedef struct { |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 161 | __u64 ksnn_incarnation; /* my epoch */ |
| 162 | spinlock_t ksnn_lock; /* serialise */ |
| 163 | struct list_head ksnn_list; /* chain on global list */ |
| 164 | int ksnn_npeers; /* # peers */ |
| 165 | int ksnn_shutdown; /* shutting down? */ |
| 166 | int ksnn_ninterfaces; /* IP interfaces */ |
| 167 | ksock_interface_t ksnn_interfaces[LNET_MAX_INTERFACES]; |
| 168 | } ksock_net_t; |
| 169 | |
| 170 | /** connd timeout */ |
| 171 | #define SOCKNAL_CONND_TIMEOUT 120 |
| 172 | /** reserved thread for accepting & creating new connd */ |
| 173 | #define SOCKNAL_CONND_RESV 1 |
| 174 | |
Greg Donald | 75c49d4 | 2014-09-22 09:39:53 -0500 | [diff] [blame] | 175 | typedef struct { |
Mike Shuey | 97d10d0 | 2015-05-19 10:14:37 -0400 | [diff] [blame] | 176 | int ksnd_init; /* initialisation state |
| 177 | */ |
| 178 | int ksnd_nnets; /* # networks set up */ |
| 179 | struct list_head ksnd_nets; /* list of nets */ |
| 180 | rwlock_t ksnd_global_lock; /* stabilize peer/conn |
| 181 | * ops */ |
| 182 | struct list_head *ksnd_peers; /* hash table of all my |
| 183 | * known peers */ |
| 184 | int ksnd_peer_hash_size; /* size of ksnd_peers */ |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 185 | |
Mike Shuey | 97d10d0 | 2015-05-19 10:14:37 -0400 | [diff] [blame] | 186 | int ksnd_nthreads; /* # live threads */ |
| 187 | int ksnd_shuttingdown; /* tell threads to exit |
| 188 | */ |
| 189 | struct ksock_sched_info **ksnd_sched_info; /* schedulers info */ |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 190 | |
Mike Shuey | 97d10d0 | 2015-05-19 10:14:37 -0400 | [diff] [blame] | 191 | atomic_t ksnd_nactive_txs; /* #active txs */ |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 192 | |
Mike Shuey | 97d10d0 | 2015-05-19 10:14:37 -0400 | [diff] [blame] | 193 | struct list_head ksnd_deathrow_conns; /* conns to close: |
| 194 | * reaper_lock*/ |
| 195 | struct list_head ksnd_zombie_conns; /* conns to free: |
| 196 | * reaper_lock */ |
| 197 | struct list_head ksnd_enomem_conns; /* conns to retry: |
| 198 | * reaper_lock*/ |
| 199 | wait_queue_head_t ksnd_reaper_waitq; /* reaper sleeps here */ |
| 200 | unsigned long ksnd_reaper_waketime; /* when reaper will wake |
| 201 | */ |
| 202 | spinlock_t ksnd_reaper_lock; /* serialise */ |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 203 | |
Mike Shuey | 97d10d0 | 2015-05-19 10:14:37 -0400 | [diff] [blame] | 204 | int ksnd_enomem_tx; /* test ENOMEM sender */ |
| 205 | int ksnd_stall_tx; /* test sluggish sender |
| 206 | */ |
| 207 | int ksnd_stall_rx; /* test sluggish |
| 208 | * receiver */ |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 209 | |
Mike Shuey | 97d10d0 | 2015-05-19 10:14:37 -0400 | [diff] [blame] | 210 | struct list_head ksnd_connd_connreqs; /* incoming connection |
| 211 | * requests */ |
| 212 | struct list_head ksnd_connd_routes; /* routes waiting to be |
| 213 | * connected */ |
| 214 | wait_queue_head_t ksnd_connd_waitq; /* connds sleep here */ |
| 215 | int ksnd_connd_connecting; /* # connds connecting |
| 216 | */ |
Arnd Bergmann | 74ad578 | 2015-09-27 16:45:21 -0400 | [diff] [blame] | 217 | time64_t ksnd_connd_failed_stamp;/* time stamp of the |
Mike Shuey | 97d10d0 | 2015-05-19 10:14:37 -0400 | [diff] [blame] | 218 | * last failed |
| 219 | * connecting attempt */ |
Arnd Bergmann | 74ad578 | 2015-09-27 16:45:21 -0400 | [diff] [blame] | 220 | time64_t ksnd_connd_starting_stamp;/* time stamp of the |
Mike Shuey | 97d10d0 | 2015-05-19 10:14:37 -0400 | [diff] [blame] | 221 | * last starting connd |
| 222 | */ |
Arnd Bergmann | 74ad578 | 2015-09-27 16:45:21 -0400 | [diff] [blame] | 223 | unsigned ksnd_connd_starting; /* # starting connd */ |
Mike Shuey | 97d10d0 | 2015-05-19 10:14:37 -0400 | [diff] [blame] | 224 | unsigned ksnd_connd_running; /* # running connd */ |
| 225 | spinlock_t ksnd_connd_lock; /* serialise */ |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 226 | |
Mike Shuey | 97d10d0 | 2015-05-19 10:14:37 -0400 | [diff] [blame] | 227 | struct list_head ksnd_idle_noop_txs; /* list head for freed |
| 228 | * noop tx */ |
| 229 | spinlock_t ksnd_tx_lock; /* serialise, g_lock |
| 230 | * unsafe */ |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 231 | |
| 232 | } ksock_nal_data_t; |
| 233 | |
Mike Shuey | 97d10d0 | 2015-05-19 10:14:37 -0400 | [diff] [blame] | 234 | #define SOCKNAL_INIT_NOTHING 0 |
| 235 | #define SOCKNAL_INIT_DATA 1 |
| 236 | #define SOCKNAL_INIT_ALL 2 |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 237 | |
James Simmons | 4420cfd | 2016-02-12 12:06:00 -0500 | [diff] [blame] | 238 | /* |
| 239 | * A packet just assembled for transmission is represented by 1 or more |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 240 | * struct iovec fragments (the first frag contains the portals header), |
| 241 | * followed by 0 or more lnet_kiov_t fragments. |
| 242 | * |
| 243 | * On the receive side, initially 1 struct iovec fragment is posted for |
| 244 | * receive (the header). Once the header has been received, the payload is |
| 245 | * received into either struct iovec or lnet_kiov_t fragments, depending on |
James Simmons | 4420cfd | 2016-02-12 12:06:00 -0500 | [diff] [blame] | 246 | * what the header matched or whether the message needs forwarding. |
| 247 | */ |
Mike Shuey | 97d10d0 | 2015-05-19 10:14:37 -0400 | [diff] [blame] | 248 | struct ksock_conn; /* forward ref */ |
| 249 | struct ksock_peer; /* forward ref */ |
| 250 | struct ksock_route; /* forward ref */ |
| 251 | struct ksock_proto; /* forward ref */ |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 252 | |
Mike Shuey | 97d10d0 | 2015-05-19 10:14:37 -0400 | [diff] [blame] | 253 | typedef struct /* transmit packet */ |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 254 | { |
Mike Shuey | 97d10d0 | 2015-05-19 10:14:37 -0400 | [diff] [blame] | 255 | struct list_head tx_list; /* queue on conn for transmission etc |
| 256 | */ |
| 257 | struct list_head tx_zc_list; /* queue on peer for ZC request */ |
| 258 | atomic_t tx_refcount; /* tx reference count */ |
| 259 | int tx_nob; /* # packet bytes */ |
| 260 | int tx_resid; /* residual bytes */ |
| 261 | int tx_niov; /* # packet iovec frags */ |
| 262 | struct kvec *tx_iov; /* packet iovec frags */ |
| 263 | int tx_nkiov; /* # packet page frags */ |
| 264 | unsigned short tx_zc_aborted; /* aborted ZC request */ |
| 265 | unsigned short tx_zc_capable:1; /* payload is large enough for ZC */ |
| 266 | unsigned short tx_zc_checked:1; /* Have I checked if I should ZC? */ |
| 267 | unsigned short tx_nonblk:1; /* it's a non-blocking ACK */ |
| 268 | lnet_kiov_t *tx_kiov; /* packet page frags */ |
| 269 | struct ksock_conn *tx_conn; /* owning conn */ |
| 270 | lnet_msg_t *tx_lnetmsg; /* lnet message for lnet_finalize() |
| 271 | */ |
| 272 | unsigned long tx_deadline; /* when (in jiffies) tx times out */ |
| 273 | ksock_msg_t tx_msg; /* socklnd message buffer */ |
| 274 | int tx_desc_size; /* size of this descriptor */ |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 275 | union { |
| 276 | struct { |
Mike Shuey | 97d10d0 | 2015-05-19 10:14:37 -0400 | [diff] [blame] | 277 | struct kvec iov; /* virt hdr */ |
| 278 | lnet_kiov_t kiov[0]; /* paged payload */ |
| 279 | } paged; |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 280 | struct { |
Mike Shuey | 97d10d0 | 2015-05-19 10:14:37 -0400 | [diff] [blame] | 281 | struct kvec iov[1]; /* virt hdr + payload */ |
| 282 | } virt; |
| 283 | } tx_frags; |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 284 | } ksock_tx_t; |
| 285 | |
Sandhya Bankar | 04697f7 | 2016-02-28 14:46:56 +0530 | [diff] [blame] | 286 | #define KSOCK_NOOP_TX_SIZE (offsetof(ksock_tx_t, tx_frags.paged.kiov[0])) |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 287 | |
| 288 | /* network zero copy callback descriptor embedded in ksock_tx_t */ |
| 289 | |
James Simmons | 4420cfd | 2016-02-12 12:06:00 -0500 | [diff] [blame] | 290 | /* |
| 291 | * space for the rx frag descriptors; we either read a single contiguous |
| 292 | * header, or up to LNET_MAX_IOV frags of payload of either type. |
| 293 | */ |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 294 | typedef union { |
Al Viro | f351bad | 2014-12-02 17:15:37 +0000 | [diff] [blame] | 295 | struct kvec iov[LNET_MAX_IOV]; |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 296 | lnet_kiov_t kiov[LNET_MAX_IOV]; |
| 297 | } ksock_rxiovspace_t; |
| 298 | |
Mike Shuey | 97d10d0 | 2015-05-19 10:14:37 -0400 | [diff] [blame] | 299 | #define SOCKNAL_RX_KSM_HEADER 1 /* reading ksock message header */ |
| 300 | #define SOCKNAL_RX_LNET_HEADER 2 /* reading lnet message header */ |
| 301 | #define SOCKNAL_RX_PARSE 3 /* Calling lnet_parse() */ |
| 302 | #define SOCKNAL_RX_PARSE_WAIT 4 /* waiting to be told to read the body */ |
| 303 | #define SOCKNAL_RX_LNET_PAYLOAD 5 /* reading lnet payload (to deliver here) */ |
| 304 | #define SOCKNAL_RX_SLOP 6 /* skipping body */ |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 305 | |
Greg Donald | 75c49d4 | 2014-09-22 09:39:53 -0500 | [diff] [blame] | 306 | typedef struct ksock_conn { |
Mike Shuey | 97d10d0 | 2015-05-19 10:14:37 -0400 | [diff] [blame] | 307 | struct ksock_peer *ksnc_peer; /* owning peer */ |
| 308 | struct ksock_route *ksnc_route; /* owning route */ |
| 309 | struct list_head ksnc_list; /* stash on peer's conn list */ |
| 310 | struct socket *ksnc_sock; /* actual socket */ |
| 311 | void *ksnc_saved_data_ready; /* socket's original |
| 312 | * data_ready() callback */ |
| 313 | void *ksnc_saved_write_space; /* socket's original |
| 314 | * write_space() callback */ |
| 315 | atomic_t ksnc_conn_refcount;/* conn refcount */ |
| 316 | atomic_t ksnc_sock_refcount;/* sock refcount */ |
| 317 | ksock_sched_t *ksnc_scheduler; /* who schedules this connection |
| 318 | */ |
| 319 | __u32 ksnc_myipaddr; /* my IP */ |
| 320 | __u32 ksnc_ipaddr; /* peer's IP */ |
| 321 | int ksnc_port; /* peer's port */ |
| 322 | signed int ksnc_type:3; /* type of connection, should be |
| 323 | * signed value */ |
| 324 | unsigned int ksnc_closing:1; /* being shut down */ |
| 325 | unsigned int ksnc_flip:1; /* flip or not, only for V2.x */ |
| 326 | unsigned int ksnc_zc_capable:1; /* enable to ZC */ |
| 327 | struct ksock_proto *ksnc_proto; /* protocol for the connection */ |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 328 | |
| 329 | /* reader */ |
Mike Shuey | 97d10d0 | 2015-05-19 10:14:37 -0400 | [diff] [blame] | 330 | struct list_head ksnc_rx_list; /* where I enq waiting input or a |
| 331 | * forwarding descriptor */ |
| 332 | unsigned long ksnc_rx_deadline; /* when (in jiffies) receive times |
| 333 | * out */ |
| 334 | __u8 ksnc_rx_started; /* started receiving a message */ |
| 335 | __u8 ksnc_rx_ready; /* data ready to read */ |
| 336 | __u8 ksnc_rx_scheduled; /* being progressed */ |
| 337 | __u8 ksnc_rx_state; /* what is being read */ |
| 338 | int ksnc_rx_nob_left; /* # bytes to next hdr/body */ |
| 339 | int ksnc_rx_nob_wanted;/* bytes actually wanted */ |
| 340 | int ksnc_rx_niov; /* # iovec frags */ |
| 341 | struct kvec *ksnc_rx_iov; /* the iovec frags */ |
| 342 | int ksnc_rx_nkiov; /* # page frags */ |
| 343 | lnet_kiov_t *ksnc_rx_kiov; /* the page frags */ |
| 344 | ksock_rxiovspace_t ksnc_rx_iov_space; /* space for frag descriptors */ |
| 345 | __u32 ksnc_rx_csum; /* partial checksum for incoming |
| 346 | * data */ |
| 347 | void *ksnc_cookie; /* rx lnet_finalize passthru arg |
| 348 | */ |
| 349 | ksock_msg_t ksnc_msg; /* incoming message buffer: |
| 350 | * V2.x message takes the |
| 351 | * whole struct |
| 352 | * V1.x message is a bare |
| 353 | * lnet_hdr_t, it's stored in |
| 354 | * ksnc_msg.ksm_u.lnetmsg */ |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 355 | |
| 356 | /* WRITER */ |
Mike Shuey | 97d10d0 | 2015-05-19 10:14:37 -0400 | [diff] [blame] | 357 | struct list_head ksnc_tx_list; /* where I enq waiting for output |
| 358 | * space */ |
| 359 | struct list_head ksnc_tx_queue; /* packets waiting to be sent */ |
| 360 | ksock_tx_t *ksnc_tx_carrier; /* next TX that can carry a LNet |
| 361 | * message or ZC-ACK */ |
| 362 | unsigned long ksnc_tx_deadline; /* when (in jiffies) tx times out |
| 363 | */ |
| 364 | int ksnc_tx_bufnob; /* send buffer marker */ |
| 365 | atomic_t ksnc_tx_nob; /* # bytes queued */ |
| 366 | int ksnc_tx_ready; /* write space */ |
| 367 | int ksnc_tx_scheduled; /* being progressed */ |
| 368 | unsigned long ksnc_tx_last_post; /* time stamp of the last posted |
| 369 | * TX */ |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 370 | } ksock_conn_t; |
| 371 | |
Greg Donald | 75c49d4 | 2014-09-22 09:39:53 -0500 | [diff] [blame] | 372 | typedef struct ksock_route { |
Mike Shuey | 97d10d0 | 2015-05-19 10:14:37 -0400 | [diff] [blame] | 373 | struct list_head ksnr_list; /* chain on peer route list */ |
| 374 | struct list_head ksnr_connd_list; /* chain on ksnr_connd_routes */ |
| 375 | struct ksock_peer *ksnr_peer; /* owning peer */ |
| 376 | atomic_t ksnr_refcount; /* # users */ |
| 377 | unsigned long ksnr_timeout; /* when (in jiffies) reconnection |
| 378 | * can happen next */ |
| 379 | long ksnr_retry_interval; /* how long between retries */ |
| 380 | __u32 ksnr_myipaddr; /* my IP */ |
| 381 | __u32 ksnr_ipaddr; /* IP address to connect to */ |
| 382 | int ksnr_port; /* port to connect to */ |
| 383 | unsigned int ksnr_scheduled:1; /* scheduled for attention */ |
| 384 | unsigned int ksnr_connecting:1; /* connection establishment in |
| 385 | * progress */ |
| 386 | unsigned int ksnr_connected:4; /* connections established by |
| 387 | * type */ |
| 388 | unsigned int ksnr_deleted:1; /* been removed from peer? */ |
| 389 | unsigned int ksnr_share_count; /* created explicitly? */ |
| 390 | int ksnr_conn_count; /* # conns established by this |
| 391 | * route */ |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 392 | } ksock_route_t; |
| 393 | |
Mike Shuey | 97d10d0 | 2015-05-19 10:14:37 -0400 | [diff] [blame] | 394 | #define SOCKNAL_KEEPALIVE_PING 1 /* cookie for keepalive ping */ |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 395 | |
Greg Donald | 75c49d4 | 2014-09-22 09:39:53 -0500 | [diff] [blame] | 396 | typedef struct ksock_peer { |
Mike Shuey | 97d10d0 | 2015-05-19 10:14:37 -0400 | [diff] [blame] | 397 | struct list_head ksnp_list; /* stash on global peer list */ |
| 398 | unsigned long ksnp_last_alive; /* when (in jiffies) I was last |
| 399 | * alive */ |
| 400 | lnet_process_id_t ksnp_id; /* who's on the other end(s) */ |
| 401 | atomic_t ksnp_refcount; /* # users */ |
| 402 | int ksnp_sharecount; /* lconf usage counter */ |
| 403 | int ksnp_closing; /* being closed */ |
| 404 | int ksnp_accepting; /* # passive connections pending |
| 405 | */ |
| 406 | int ksnp_error; /* errno on closing last conn */ |
| 407 | __u64 ksnp_zc_next_cookie; /* ZC completion cookie */ |
| 408 | __u64 ksnp_incarnation; /* latest known peer incarnation |
| 409 | */ |
| 410 | struct ksock_proto *ksnp_proto; /* latest known peer protocol */ |
| 411 | struct list_head ksnp_conns; /* all active connections */ |
| 412 | struct list_head ksnp_routes; /* routes */ |
| 413 | struct list_head ksnp_tx_queue; /* waiting packets */ |
| 414 | spinlock_t ksnp_lock; /* serialize, g_lock unsafe */ |
| 415 | struct list_head ksnp_zc_req_list; /* zero copy requests wait for |
| 416 | * ACK */ |
| 417 | unsigned long ksnp_send_keepalive; /* time to send keepalive */ |
| 418 | lnet_ni_t *ksnp_ni; /* which network */ |
| 419 | int ksnp_n_passive_ips; /* # of... */ |
| 420 | |
| 421 | /* preferred local interfaces */ |
| 422 | __u32 ksnp_passive_ips[LNET_MAX_INTERFACES]; |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 423 | } ksock_peer_t; |
| 424 | |
Greg Donald | 75c49d4 | 2014-09-22 09:39:53 -0500 | [diff] [blame] | 425 | typedef struct ksock_connreq { |
Mike Shuey | 97d10d0 | 2015-05-19 10:14:37 -0400 | [diff] [blame] | 426 | struct list_head ksncr_list; /* stash on ksnd_connd_connreqs */ |
| 427 | lnet_ni_t *ksncr_ni; /* chosen NI */ |
| 428 | struct socket *ksncr_sock; /* accepted socket */ |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 429 | } ksock_connreq_t; |
| 430 | |
| 431 | extern ksock_nal_data_t ksocknal_data; |
| 432 | extern ksock_tunables_t ksocknal_tunables; |
| 433 | |
Mike Shuey | 97d10d0 | 2015-05-19 10:14:37 -0400 | [diff] [blame] | 434 | #define SOCKNAL_MATCH_NO 0 /* TX can't match type of connection */ |
| 435 | #define SOCKNAL_MATCH_YES 1 /* TX matches type of connection */ |
| 436 | #define SOCKNAL_MATCH_MAY 2 /* TX can be sent on the connection, but not |
| 437 | * preferred */ |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 438 | |
Greg Donald | 75c49d4 | 2014-09-22 09:39:53 -0500 | [diff] [blame] | 439 | typedef struct ksock_proto { |
Mike Shuey | 97d10d0 | 2015-05-19 10:14:37 -0400 | [diff] [blame] | 440 | /* version number of protocol */ |
| 441 | int pro_version; |
| 442 | |
| 443 | /* handshake function */ |
| 444 | int (*pro_send_hello)(ksock_conn_t *, ksock_hello_msg_t *); |
| 445 | |
| 446 | /* handshake function */ |
| 447 | int (*pro_recv_hello)(ksock_conn_t *, ksock_hello_msg_t *, int); |
| 448 | |
| 449 | /* message pack */ |
| 450 | void (*pro_pack)(ksock_tx_t *); |
| 451 | |
| 452 | /* message unpack */ |
| 453 | void (*pro_unpack)(ksock_msg_t *); |
| 454 | |
| 455 | /* queue tx on the connection */ |
| 456 | ksock_tx_t *(*pro_queue_tx_msg)(ksock_conn_t *, ksock_tx_t *); |
| 457 | |
| 458 | /* queue ZC ack on the connection */ |
| 459 | int (*pro_queue_tx_zcack)(ksock_conn_t *, ksock_tx_t *, __u64); |
| 460 | |
| 461 | /* handle ZC request */ |
| 462 | int (*pro_handle_zcreq)(ksock_conn_t *, __u64, int); |
| 463 | |
| 464 | /* handle ZC ACK */ |
| 465 | int (*pro_handle_zcack)(ksock_conn_t *, __u64, __u64); |
| 466 | |
James Simmons | 4420cfd | 2016-02-12 12:06:00 -0500 | [diff] [blame] | 467 | /* |
| 468 | * msg type matches the connection type: |
Mike Shuey | 97d10d0 | 2015-05-19 10:14:37 -0400 | [diff] [blame] | 469 | * return value: |
| 470 | * return MATCH_NO : no |
| 471 | * return MATCH_YES : matching type |
James Simmons | 4420cfd | 2016-02-12 12:06:00 -0500 | [diff] [blame] | 472 | * return MATCH_MAY : can be backup |
| 473 | */ |
Mike Shuey | 97d10d0 | 2015-05-19 10:14:37 -0400 | [diff] [blame] | 474 | int (*pro_match_tx)(ksock_conn_t *, ksock_tx_t *, int); |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 475 | } ksock_proto_t; |
| 476 | |
| 477 | extern ksock_proto_t ksocknal_protocol_v1x; |
| 478 | extern ksock_proto_t ksocknal_protocol_v2x; |
| 479 | extern ksock_proto_t ksocknal_protocol_v3x; |
| 480 | |
Mike Shuey | 97d10d0 | 2015-05-19 10:14:37 -0400 | [diff] [blame] | 481 | #define KSOCK_PROTO_V1_MAJOR LNET_PROTO_TCP_VERSION_MAJOR |
| 482 | #define KSOCK_PROTO_V1_MINOR LNET_PROTO_TCP_VERSION_MINOR |
| 483 | #define KSOCK_PROTO_V1 KSOCK_PROTO_V1_MAJOR |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 484 | |
| 485 | #ifndef CPU_MASK_NONE |
| 486 | #define CPU_MASK_NONE 0UL |
| 487 | #endif |
| 488 | |
John L. Hammond | 12c41f0 | 2015-06-11 15:18:10 -0400 | [diff] [blame] | 489 | static inline __u32 ksocknal_csum(__u32 crc, unsigned char const *p, size_t len) |
| 490 | { |
| 491 | #if 1 |
| 492 | return crc32_le(crc, p, len); |
| 493 | #else |
| 494 | while (len-- > 0) |
| 495 | crc = ((crc + 0x100) & ~0xff) | ((crc + *p++) & 0xff) ; |
| 496 | return crc; |
| 497 | #endif |
| 498 | } |
| 499 | |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 500 | static inline int |
| 501 | ksocknal_route_mask(void) |
| 502 | { |
| 503 | if (!*ksocknal_tunables.ksnd_typed_conns) |
| 504 | return (1 << SOCKLND_CONN_ANY); |
| 505 | |
| 506 | return ((1 << SOCKLND_CONN_CONTROL) | |
| 507 | (1 << SOCKLND_CONN_BULK_IN) | |
| 508 | (1 << SOCKLND_CONN_BULK_OUT)); |
| 509 | } |
| 510 | |
| 511 | static inline struct list_head * |
Adrien Descamps | d9dfb48 | 2015-02-25 00:59:25 +0100 | [diff] [blame] | 512 | ksocknal_nid2peerlist(lnet_nid_t nid) |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 513 | { |
| 514 | unsigned int hash = ((unsigned int)nid) % ksocknal_data.ksnd_peer_hash_size; |
| 515 | |
Masaru Nomura | 7139709 | 2014-05-15 18:54:05 +0100 | [diff] [blame] | 516 | return &ksocknal_data.ksnd_peers[hash]; |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 517 | } |
| 518 | |
| 519 | static inline void |
Adrien Descamps | d9dfb48 | 2015-02-25 00:59:25 +0100 | [diff] [blame] | 520 | ksocknal_conn_addref(ksock_conn_t *conn) |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 521 | { |
Adrien Descamps | d9dfb48 | 2015-02-25 00:59:25 +0100 | [diff] [blame] | 522 | LASSERT(atomic_read(&conn->ksnc_conn_refcount) > 0); |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 523 | atomic_inc(&conn->ksnc_conn_refcount); |
| 524 | } |
| 525 | |
Joe Perches | 8150a97 | 2015-08-10 14:51:21 -0700 | [diff] [blame] | 526 | void ksocknal_queue_zombie_conn(ksock_conn_t *conn); |
| 527 | void ksocknal_finalize_zcreq(ksock_conn_t *conn); |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 528 | |
| 529 | static inline void |
Adrien Descamps | d9dfb48 | 2015-02-25 00:59:25 +0100 | [diff] [blame] | 530 | ksocknal_conn_decref(ksock_conn_t *conn) |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 531 | { |
Adrien Descamps | d9dfb48 | 2015-02-25 00:59:25 +0100 | [diff] [blame] | 532 | LASSERT(atomic_read(&conn->ksnc_conn_refcount) > 0); |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 533 | if (atomic_dec_and_test(&conn->ksnc_conn_refcount)) |
| 534 | ksocknal_queue_zombie_conn(conn); |
| 535 | } |
| 536 | |
| 537 | static inline int |
Adrien Descamps | d9dfb48 | 2015-02-25 00:59:25 +0100 | [diff] [blame] | 538 | ksocknal_connsock_addref(ksock_conn_t *conn) |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 539 | { |
Mike Shuey | 97d10d0 | 2015-05-19 10:14:37 -0400 | [diff] [blame] | 540 | int rc = -ESHUTDOWN; |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 541 | |
| 542 | read_lock(&ksocknal_data.ksnd_global_lock); |
| 543 | if (!conn->ksnc_closing) { |
| 544 | LASSERT(atomic_read(&conn->ksnc_sock_refcount) > 0); |
| 545 | atomic_inc(&conn->ksnc_sock_refcount); |
| 546 | rc = 0; |
| 547 | } |
| 548 | read_unlock(&ksocknal_data.ksnd_global_lock); |
| 549 | |
Masaru Nomura | 7139709 | 2014-05-15 18:54:05 +0100 | [diff] [blame] | 550 | return rc; |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 551 | } |
| 552 | |
| 553 | static inline void |
Adrien Descamps | d9dfb48 | 2015-02-25 00:59:25 +0100 | [diff] [blame] | 554 | ksocknal_connsock_decref(ksock_conn_t *conn) |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 555 | { |
Adrien Descamps | d9dfb48 | 2015-02-25 00:59:25 +0100 | [diff] [blame] | 556 | LASSERT(atomic_read(&conn->ksnc_sock_refcount) > 0); |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 557 | if (atomic_dec_and_test(&conn->ksnc_sock_refcount)) { |
Adrien Descamps | d9dfb48 | 2015-02-25 00:59:25 +0100 | [diff] [blame] | 558 | LASSERT(conn->ksnc_closing); |
James Simmons | e52fc91 | 2015-06-08 22:27:08 -0400 | [diff] [blame] | 559 | sock_release(conn->ksnc_sock); |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 560 | conn->ksnc_sock = NULL; |
| 561 | ksocknal_finalize_zcreq(conn); |
| 562 | } |
| 563 | } |
| 564 | |
| 565 | static inline void |
Adrien Descamps | d9dfb48 | 2015-02-25 00:59:25 +0100 | [diff] [blame] | 566 | ksocknal_tx_addref(ksock_tx_t *tx) |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 567 | { |
Adrien Descamps | d9dfb48 | 2015-02-25 00:59:25 +0100 | [diff] [blame] | 568 | LASSERT(atomic_read(&tx->tx_refcount) > 0); |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 569 | atomic_inc(&tx->tx_refcount); |
| 570 | } |
| 571 | |
Joe Perches | 8150a97 | 2015-08-10 14:51:21 -0700 | [diff] [blame] | 572 | void ksocknal_tx_prep(ksock_conn_t *, ksock_tx_t *tx); |
| 573 | void ksocknal_tx_done(lnet_ni_t *ni, ksock_tx_t *tx); |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 574 | |
| 575 | static inline void |
Adrien Descamps | d9dfb48 | 2015-02-25 00:59:25 +0100 | [diff] [blame] | 576 | ksocknal_tx_decref(ksock_tx_t *tx) |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 577 | { |
Adrien Descamps | d9dfb48 | 2015-02-25 00:59:25 +0100 | [diff] [blame] | 578 | LASSERT(atomic_read(&tx->tx_refcount) > 0); |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 579 | if (atomic_dec_and_test(&tx->tx_refcount)) |
| 580 | ksocknal_tx_done(NULL, tx); |
| 581 | } |
| 582 | |
| 583 | static inline void |
Adrien Descamps | d9dfb48 | 2015-02-25 00:59:25 +0100 | [diff] [blame] | 584 | ksocknal_route_addref(ksock_route_t *route) |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 585 | { |
Adrien Descamps | d9dfb48 | 2015-02-25 00:59:25 +0100 | [diff] [blame] | 586 | LASSERT(atomic_read(&route->ksnr_refcount) > 0); |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 587 | atomic_inc(&route->ksnr_refcount); |
| 588 | } |
| 589 | |
Joe Perches | 8150a97 | 2015-08-10 14:51:21 -0700 | [diff] [blame] | 590 | void ksocknal_destroy_route(ksock_route_t *route); |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 591 | |
| 592 | static inline void |
Adrien Descamps | d9dfb48 | 2015-02-25 00:59:25 +0100 | [diff] [blame] | 593 | ksocknal_route_decref(ksock_route_t *route) |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 594 | { |
Adrien Descamps | d9dfb48 | 2015-02-25 00:59:25 +0100 | [diff] [blame] | 595 | LASSERT(atomic_read(&route->ksnr_refcount) > 0); |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 596 | if (atomic_dec_and_test(&route->ksnr_refcount)) |
Adrien Descamps | d9dfb48 | 2015-02-25 00:59:25 +0100 | [diff] [blame] | 597 | ksocknal_destroy_route(route); |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 598 | } |
| 599 | |
| 600 | static inline void |
Adrien Descamps | d9dfb48 | 2015-02-25 00:59:25 +0100 | [diff] [blame] | 601 | ksocknal_peer_addref(ksock_peer_t *peer) |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 602 | { |
Adrien Descamps | d9dfb48 | 2015-02-25 00:59:25 +0100 | [diff] [blame] | 603 | LASSERT(atomic_read(&peer->ksnp_refcount) > 0); |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 604 | atomic_inc(&peer->ksnp_refcount); |
| 605 | } |
| 606 | |
Joe Perches | 8150a97 | 2015-08-10 14:51:21 -0700 | [diff] [blame] | 607 | void ksocknal_destroy_peer(ksock_peer_t *peer); |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 608 | |
| 609 | static inline void |
Adrien Descamps | d9dfb48 | 2015-02-25 00:59:25 +0100 | [diff] [blame] | 610 | ksocknal_peer_decref(ksock_peer_t *peer) |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 611 | { |
Adrien Descamps | d9dfb48 | 2015-02-25 00:59:25 +0100 | [diff] [blame] | 612 | LASSERT(atomic_read(&peer->ksnp_refcount) > 0); |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 613 | if (atomic_dec_and_test(&peer->ksnp_refcount)) |
Adrien Descamps | d9dfb48 | 2015-02-25 00:59:25 +0100 | [diff] [blame] | 614 | ksocknal_destroy_peer(peer); |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 615 | } |
| 616 | |
Adrien Descamps | d9dfb48 | 2015-02-25 00:59:25 +0100 | [diff] [blame] | 617 | int ksocknal_startup(lnet_ni_t *ni); |
| 618 | void ksocknal_shutdown(lnet_ni_t *ni); |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 619 | int ksocknal_ctl(lnet_ni_t *ni, unsigned int cmd, void *arg); |
Adrien Descamps | d9dfb48 | 2015-02-25 00:59:25 +0100 | [diff] [blame] | 620 | int ksocknal_send(lnet_ni_t *ni, void *private, lnet_msg_t *lntmsg); |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 621 | int ksocknal_recv(lnet_ni_t *ni, void *private, lnet_msg_t *lntmsg, |
| 622 | int delayed, unsigned int niov, |
Al Viro | f351bad | 2014-12-02 17:15:37 +0000 | [diff] [blame] | 623 | struct kvec *iov, lnet_kiov_t *kiov, |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 624 | unsigned int offset, unsigned int mlen, unsigned int rlen); |
Greg Kroah-Hartman | e327dc8 | 2014-07-11 23:42:52 -0700 | [diff] [blame] | 625 | int ksocknal_accept(lnet_ni_t *ni, struct socket *sock); |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 626 | |
Joe Perches | 8150a97 | 2015-08-10 14:51:21 -0700 | [diff] [blame] | 627 | int ksocknal_add_peer(lnet_ni_t *ni, lnet_process_id_t id, __u32 ip, int port); |
| 628 | ksock_peer_t *ksocknal_find_peer_locked(lnet_ni_t *ni, lnet_process_id_t id); |
| 629 | ksock_peer_t *ksocknal_find_peer(lnet_ni_t *ni, lnet_process_id_t id); |
| 630 | void ksocknal_peer_failed(ksock_peer_t *peer); |
| 631 | int ksocknal_create_conn(lnet_ni_t *ni, ksock_route_t *route, |
| 632 | struct socket *sock, int type); |
| 633 | void ksocknal_close_conn_locked(ksock_conn_t *conn, int why); |
| 634 | void ksocknal_terminate_conn(ksock_conn_t *conn); |
| 635 | void ksocknal_destroy_conn(ksock_conn_t *conn); |
| 636 | int ksocknal_close_peer_conns_locked(ksock_peer_t *peer, |
| 637 | __u32 ipaddr, int why); |
| 638 | int ksocknal_close_conn_and_siblings(ksock_conn_t *conn, int why); |
| 639 | int ksocknal_close_matching_conns(lnet_process_id_t id, __u32 ipaddr); |
| 640 | ksock_conn_t *ksocknal_find_conn_locked(ksock_peer_t *peer, |
| 641 | ksock_tx_t *tx, int nonblk); |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 642 | |
Joe Perches | 8150a97 | 2015-08-10 14:51:21 -0700 | [diff] [blame] | 643 | int ksocknal_launch_packet(lnet_ni_t *ni, ksock_tx_t *tx, |
| 644 | lnet_process_id_t id); |
| 645 | ksock_tx_t *ksocknal_alloc_tx(int type, int size); |
| 646 | void ksocknal_free_tx(ksock_tx_t *tx); |
| 647 | ksock_tx_t *ksocknal_alloc_tx_noop(__u64 cookie, int nonblk); |
| 648 | void ksocknal_next_tx_carrier(ksock_conn_t *conn); |
| 649 | void ksocknal_queue_tx_locked(ksock_tx_t *tx, ksock_conn_t *conn); |
| 650 | void ksocknal_txlist_done(lnet_ni_t *ni, struct list_head *txlist, int error); |
| 651 | void ksocknal_notify(lnet_ni_t *ni, lnet_nid_t gw_nid, int alive); |
| 652 | void ksocknal_query(struct lnet_ni *ni, lnet_nid_t nid, unsigned long *when); |
| 653 | int ksocknal_thread_start(int (*fn)(void *arg), void *arg, char *name); |
| 654 | void ksocknal_thread_fini(void); |
| 655 | void ksocknal_launch_all_connections_locked(ksock_peer_t *peer); |
| 656 | ksock_route_t *ksocknal_find_connectable_route_locked(ksock_peer_t *peer); |
| 657 | ksock_route_t *ksocknal_find_connecting_route_locked(ksock_peer_t *peer); |
| 658 | int ksocknal_new_packet(ksock_conn_t *conn, int skip); |
| 659 | int ksocknal_scheduler(void *arg); |
| 660 | int ksocknal_connd(void *arg); |
| 661 | int ksocknal_reaper(void *arg); |
| 662 | int ksocknal_send_hello(lnet_ni_t *ni, ksock_conn_t *conn, |
| 663 | lnet_nid_t peer_nid, ksock_hello_msg_t *hello); |
| 664 | int ksocknal_recv_hello(lnet_ni_t *ni, ksock_conn_t *conn, |
| 665 | ksock_hello_msg_t *hello, lnet_process_id_t *id, |
| 666 | __u64 *incarnation); |
| 667 | void ksocknal_read_callback(ksock_conn_t *conn); |
| 668 | void ksocknal_write_callback(ksock_conn_t *conn); |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 669 | |
Joe Perches | 8150a97 | 2015-08-10 14:51:21 -0700 | [diff] [blame] | 670 | int ksocknal_lib_zc_capable(ksock_conn_t *conn); |
| 671 | void ksocknal_lib_save_callback(struct socket *sock, ksock_conn_t *conn); |
| 672 | void ksocknal_lib_set_callback(struct socket *sock, ksock_conn_t *conn); |
| 673 | void ksocknal_lib_reset_callback(struct socket *sock, ksock_conn_t *conn); |
| 674 | void ksocknal_lib_push_conn(ksock_conn_t *conn); |
| 675 | int ksocknal_lib_get_conn_addrs(ksock_conn_t *conn); |
| 676 | int ksocknal_lib_setup_sock(struct socket *so); |
| 677 | int ksocknal_lib_send_iov(ksock_conn_t *conn, ksock_tx_t *tx); |
| 678 | int ksocknal_lib_send_kiov(ksock_conn_t *conn, ksock_tx_t *tx); |
| 679 | void ksocknal_lib_eager_ack(ksock_conn_t *conn); |
| 680 | int ksocknal_lib_recv_iov(ksock_conn_t *conn); |
| 681 | int ksocknal_lib_recv_kiov(ksock_conn_t *conn); |
| 682 | int ksocknal_lib_get_conn_tunables(ksock_conn_t *conn, int *txmem, |
| 683 | int *rxmem, int *nagle); |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 684 | |
Amitoj Kaur Chawla | ee98e44 | 2015-11-06 20:26:52 +0530 | [diff] [blame] | 685 | void ksocknal_read_callback(ksock_conn_t *conn); |
| 686 | void ksocknal_write_callback(ksock_conn_t *conn); |
| 687 | |
Joe Perches | 8150a97 | 2015-08-10 14:51:21 -0700 | [diff] [blame] | 688 | int ksocknal_tunables_init(void); |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 689 | |
Joe Perches | 8150a97 | 2015-08-10 14:51:21 -0700 | [diff] [blame] | 690 | void ksocknal_lib_csum_tx(ksock_tx_t *tx); |
Peng Tao | d7e09d0 | 2013-05-02 16:46:55 +0800 | [diff] [blame] | 691 | |
Joe Perches | 8150a97 | 2015-08-10 14:51:21 -0700 | [diff] [blame] | 692 | int ksocknal_lib_memory_pressure(ksock_conn_t *conn); |
| 693 | int ksocknal_lib_bind_thread_to_cpu(int id); |
John L. Hammond | 12c41f0 | 2015-06-11 15:18:10 -0400 | [diff] [blame] | 694 | |
| 695 | #endif /* _SOCKLND_SOCKLND_H_ */ |