Nicholas Bellinger | b8d26b3 | 2013-03-07 00:56:19 -0800 | [diff] [blame] | 1 | #include <linux/socket.h> |
| 2 | #include <linux/in.h> |
| 3 | #include <linux/in6.h> |
| 4 | #include <rdma/ib_verbs.h> |
| 5 | #include <rdma/rdma_cm.h> |
| 6 | |
Sagi Grimberg | 24f412d | 2014-12-07 13:12:02 +0200 | [diff] [blame^] | 7 | #define DRV_NAME "isert" |
| 8 | #define PFX DRV_NAME ": " |
| 9 | |
| 10 | #define isert_dbg(fmt, arg...) \ |
| 11 | do { \ |
| 12 | if (unlikely(isert_debug_level > 2)) \ |
| 13 | printk(KERN_DEBUG PFX "%s: " fmt,\ |
| 14 | __func__ , ## arg); \ |
| 15 | } while (0) |
| 16 | |
| 17 | #define isert_warn(fmt, arg...) \ |
| 18 | do { \ |
| 19 | if (unlikely(isert_debug_level > 0)) \ |
| 20 | pr_warn(PFX "%s: " fmt, \ |
| 21 | __func__ , ## arg); \ |
| 22 | } while (0) |
| 23 | |
| 24 | #define isert_info(fmt, arg...) \ |
| 25 | do { \ |
| 26 | if (unlikely(isert_debug_level > 1)) \ |
| 27 | pr_info(PFX "%s: " fmt, \ |
| 28 | __func__ , ## arg); \ |
| 29 | } while (0) |
| 30 | |
| 31 | #define isert_err(fmt, arg...) \ |
| 32 | pr_err(PFX "%s: " fmt, __func__ , ## arg) |
| 33 | |
Nicholas Bellinger | b8d26b3 | 2013-03-07 00:56:19 -0800 | [diff] [blame] | 34 | #define ISERT_RDMA_LISTEN_BACKLOG 10 |
Vu Pham | 59464ef | 2013-08-28 23:23:35 +0300 | [diff] [blame] | 35 | #define ISCSI_ISER_SG_TABLESIZE 256 |
Nicholas Bellinger | 9bb4ca6 | 2014-02-27 07:02:48 -0800 | [diff] [blame] | 36 | #define ISER_FASTREG_LI_WRID 0xffffffffffffffffULL |
Sagi Grimberg | bdf20e7 | 2014-12-02 16:57:43 +0200 | [diff] [blame] | 37 | #define ISER_BEACON_WRID 0xfffffffffffffffeULL |
Nicholas Bellinger | b8d26b3 | 2013-03-07 00:56:19 -0800 | [diff] [blame] | 38 | |
| 39 | enum isert_desc_type { |
| 40 | ISCSI_TX_CONTROL, |
| 41 | ISCSI_TX_DATAIN |
| 42 | }; |
| 43 | |
| 44 | enum iser_ib_op_code { |
| 45 | ISER_IB_RECV, |
| 46 | ISER_IB_SEND, |
| 47 | ISER_IB_RDMA_WRITE, |
| 48 | ISER_IB_RDMA_READ, |
| 49 | }; |
| 50 | |
| 51 | enum iser_conn_state { |
| 52 | ISER_CONN_INIT, |
| 53 | ISER_CONN_UP, |
Sagi Grimberg | 128e9cc | 2014-12-02 16:57:20 +0200 | [diff] [blame] | 54 | ISER_CONN_FULL_FEATURE, |
Nicholas Bellinger | b8d26b3 | 2013-03-07 00:56:19 -0800 | [diff] [blame] | 55 | ISER_CONN_TERMINATING, |
| 56 | ISER_CONN_DOWN, |
| 57 | }; |
| 58 | |
| 59 | struct iser_rx_desc { |
| 60 | struct iser_hdr iser_header; |
| 61 | struct iscsi_hdr iscsi_header; |
| 62 | char data[ISER_RECV_DATA_SEG_LEN]; |
| 63 | u64 dma_addr; |
| 64 | struct ib_sge rx_sg; |
| 65 | char pad[ISER_RX_PAD_SIZE]; |
| 66 | } __packed; |
| 67 | |
| 68 | struct iser_tx_desc { |
| 69 | struct iser_hdr iser_header; |
| 70 | struct iscsi_hdr iscsi_header; |
| 71 | enum isert_desc_type type; |
| 72 | u64 dma_addr; |
| 73 | struct ib_sge tx_sg[2]; |
| 74 | int num_sge; |
| 75 | struct isert_cmd *isert_cmd; |
| 76 | struct ib_send_wr send_wr; |
| 77 | } __packed; |
| 78 | |
Sagi Grimberg | d3e125d | 2014-02-19 17:50:23 +0200 | [diff] [blame] | 79 | enum isert_indicator { |
| 80 | ISERT_PROTECTED = 1 << 0, |
| 81 | ISERT_DATA_KEY_VALID = 1 << 1, |
| 82 | ISERT_PROT_KEY_VALID = 1 << 2, |
| 83 | ISERT_SIG_KEY_VALID = 1 << 3, |
| 84 | }; |
| 85 | |
| 86 | struct pi_context { |
| 87 | struct ib_mr *prot_mr; |
| 88 | struct ib_fast_reg_page_list *prot_frpl; |
| 89 | struct ib_mr *sig_mr; |
| 90 | }; |
| 91 | |
Vu Pham | 59464ef | 2013-08-28 23:23:35 +0300 | [diff] [blame] | 92 | struct fast_reg_descriptor { |
Sagi Grimberg | d3e125d | 2014-02-19 17:50:23 +0200 | [diff] [blame] | 93 | struct list_head list; |
| 94 | struct ib_mr *data_mr; |
| 95 | struct ib_fast_reg_page_list *data_frpl; |
| 96 | u8 ind; |
| 97 | struct pi_context *pi_ctx; |
Vu Pham | 59464ef | 2013-08-28 23:23:35 +0300 | [diff] [blame] | 98 | }; |
| 99 | |
Sagi Grimberg | e3d7e4c | 2014-02-19 17:50:22 +0200 | [diff] [blame] | 100 | struct isert_data_buf { |
| 101 | struct scatterlist *sg; |
| 102 | int nents; |
| 103 | u32 sg_off; |
| 104 | u32 len; /* cur_rdma_length */ |
| 105 | u32 offset; |
| 106 | unsigned int dma_nents; |
| 107 | enum dma_data_direction dma_dir; |
| 108 | }; |
| 109 | |
Sagi Grimberg | 570db17 | 2014-12-02 16:57:31 +0200 | [diff] [blame] | 110 | enum { |
| 111 | DATA = 0, |
| 112 | PROT = 1, |
| 113 | SIG = 2, |
| 114 | }; |
| 115 | |
Nicholas Bellinger | b8d26b3 | 2013-03-07 00:56:19 -0800 | [diff] [blame] | 116 | struct isert_rdma_wr { |
| 117 | struct list_head wr_list; |
| 118 | struct isert_cmd *isert_cmd; |
| 119 | enum iser_ib_op_code iser_ib_op; |
| 120 | struct ib_sge *ib_sge; |
Vu Pham | 59464ef | 2013-08-28 23:23:35 +0300 | [diff] [blame] | 121 | struct ib_sge s_ib_sge; |
Nicholas Bellinger | b8d26b3 | 2013-03-07 00:56:19 -0800 | [diff] [blame] | 122 | int send_wr_num; |
| 123 | struct ib_send_wr *send_wr; |
Vu Pham | 59464ef | 2013-08-28 23:23:35 +0300 | [diff] [blame] | 124 | struct ib_send_wr s_send_wr; |
Sagi Grimberg | 570db17 | 2014-12-02 16:57:31 +0200 | [diff] [blame] | 125 | struct ib_sge ib_sg[3]; |
Sagi Grimberg | e3d7e4c | 2014-02-19 17:50:22 +0200 | [diff] [blame] | 126 | struct isert_data_buf data; |
Sagi Grimberg | 9e961ae | 2014-02-19 17:50:25 +0200 | [diff] [blame] | 127 | struct isert_data_buf prot; |
Vu Pham | 59464ef | 2013-08-28 23:23:35 +0300 | [diff] [blame] | 128 | struct fast_reg_descriptor *fr_desc; |
Nicholas Bellinger | b8d26b3 | 2013-03-07 00:56:19 -0800 | [diff] [blame] | 129 | }; |
| 130 | |
| 131 | struct isert_cmd { |
| 132 | uint32_t read_stag; |
| 133 | uint32_t write_stag; |
| 134 | uint64_t read_va; |
| 135 | uint64_t write_va; |
Nicholas Bellinger | dbbc5d1 | 2013-07-03 19:39:37 -0700 | [diff] [blame] | 136 | u64 pdu_buf_dma; |
| 137 | u32 pdu_buf_len; |
Nicholas Bellinger | b8d26b3 | 2013-03-07 00:56:19 -0800 | [diff] [blame] | 138 | u32 read_va_off; |
| 139 | u32 write_va_off; |
| 140 | u32 rdma_wr_num; |
| 141 | struct isert_conn *conn; |
Nicholas Bellinger | d703ce2 | 2013-08-17 14:27:56 -0700 | [diff] [blame] | 142 | struct iscsi_cmd *iscsi_cmd; |
Nicholas Bellinger | b8d26b3 | 2013-03-07 00:56:19 -0800 | [diff] [blame] | 143 | struct iser_tx_desc tx_desc; |
| 144 | struct isert_rdma_wr rdma_wr; |
| 145 | struct work_struct comp_work; |
| 146 | }; |
| 147 | |
| 148 | struct isert_device; |
| 149 | |
| 150 | struct isert_conn { |
| 151 | enum iser_conn_state state; |
Nicholas Bellinger | b8d26b3 | 2013-03-07 00:56:19 -0800 | [diff] [blame] | 152 | int post_recv_buf_count; |
Nicholas Bellinger | b8d26b3 | 2013-03-07 00:56:19 -0800 | [diff] [blame] | 153 | u32 responder_resources; |
| 154 | u32 initiator_depth; |
Sagi Grimberg | 23a548e | 2014-12-02 16:57:35 +0200 | [diff] [blame] | 155 | bool pi_support; |
Nicholas Bellinger | b8d26b3 | 2013-03-07 00:56:19 -0800 | [diff] [blame] | 156 | u32 max_sge; |
| 157 | char *login_buf; |
| 158 | char *login_req_buf; |
| 159 | char *login_rsp_buf; |
| 160 | u64 login_req_dma; |
Sagi Grimberg | 2371e5d | 2014-12-02 16:57:21 +0200 | [diff] [blame] | 161 | int login_req_len; |
Nicholas Bellinger | b8d26b3 | 2013-03-07 00:56:19 -0800 | [diff] [blame] | 162 | u64 login_rsp_dma; |
| 163 | unsigned int conn_rx_desc_head; |
| 164 | struct iser_rx_desc *conn_rx_descs; |
| 165 | struct ib_recv_wr conn_rx_wr[ISERT_MIN_POSTED_RX]; |
| 166 | struct iscsi_conn *conn; |
| 167 | struct list_head conn_accept_node; |
| 168 | struct completion conn_login_comp; |
Sagi Grimberg | 2371e5d | 2014-12-02 16:57:21 +0200 | [diff] [blame] | 169 | struct completion login_req_comp; |
Nicholas Bellinger | b8d26b3 | 2013-03-07 00:56:19 -0800 | [diff] [blame] | 170 | struct iser_tx_desc conn_login_tx_desc; |
| 171 | struct rdma_cm_id *conn_cm_id; |
| 172 | struct ib_pd *conn_pd; |
| 173 | struct ib_mr *conn_mr; |
| 174 | struct ib_qp *conn_qp; |
| 175 | struct isert_device *conn_device; |
Nicholas Bellinger | b2cb964 | 2013-07-03 03:05:37 -0700 | [diff] [blame] | 176 | struct mutex conn_mutex; |
Nicholas Bellinger | defd884 | 2014-02-03 12:54:39 -0800 | [diff] [blame] | 177 | struct completion conn_wait; |
| 178 | struct completion conn_wait_comp_err; |
Nicholas Bellinger | b8d26b3 | 2013-03-07 00:56:19 -0800 | [diff] [blame] | 179 | struct kref conn_kref; |
Sagi Grimberg | a3a5a82 | 2014-01-09 18:40:50 +0200 | [diff] [blame] | 180 | struct list_head conn_fr_pool; |
| 181 | int conn_fr_pool_size; |
| 182 | /* lock to protect fastreg pool */ |
Vu Pham | 59464ef | 2013-08-28 23:23:35 +0300 | [diff] [blame] | 183 | spinlock_t conn_lock; |
Sagi Grimberg | b02efbf | 2014-12-02 16:57:29 +0200 | [diff] [blame] | 184 | struct work_struct release_work; |
Sagi Grimberg | bdf20e7 | 2014-12-02 16:57:43 +0200 | [diff] [blame] | 185 | struct ib_recv_wr beacon; |
Sagi Grimberg | 991bb76 | 2014-12-07 13:12:01 +0200 | [diff] [blame] | 186 | bool logout_posted; |
Nicholas Bellinger | b8d26b3 | 2013-03-07 00:56:19 -0800 | [diff] [blame] | 187 | }; |
| 188 | |
| 189 | #define ISERT_MAX_CQ 64 |
| 190 | |
Sagi Grimberg | 4a295ba | 2014-12-02 16:57:40 +0200 | [diff] [blame] | 191 | /** |
| 192 | * struct isert_comp - iSER completion context |
| 193 | * |
| 194 | * @device: pointer to device handle |
Sagi Grimberg | 6f0fae3 | 2014-12-02 16:57:41 +0200 | [diff] [blame] | 195 | * @cq: completion queue |
Sagi Grimberg | 36ea63b | 2014-12-02 16:57:45 +0200 | [diff] [blame] | 196 | * @wcs: work completion array |
Sagi Grimberg | 4a295ba | 2014-12-02 16:57:40 +0200 | [diff] [blame] | 197 | * @active_qps: Number of active QPs attached |
| 198 | * to completion context |
Sagi Grimberg | 6f0fae3 | 2014-12-02 16:57:41 +0200 | [diff] [blame] | 199 | * @work: completion work handle |
Sagi Grimberg | 4a295ba | 2014-12-02 16:57:40 +0200 | [diff] [blame] | 200 | */ |
| 201 | struct isert_comp { |
Sagi Grimberg | 6f0fae3 | 2014-12-02 16:57:41 +0200 | [diff] [blame] | 202 | struct isert_device *device; |
| 203 | struct ib_cq *cq; |
Sagi Grimberg | 36ea63b | 2014-12-02 16:57:45 +0200 | [diff] [blame] | 204 | struct ib_wc wcs[16]; |
Sagi Grimberg | 4a295ba | 2014-12-02 16:57:40 +0200 | [diff] [blame] | 205 | int active_qps; |
Sagi Grimberg | 6f0fae3 | 2014-12-02 16:57:41 +0200 | [diff] [blame] | 206 | struct work_struct work; |
Nicholas Bellinger | b8d26b3 | 2013-03-07 00:56:19 -0800 | [diff] [blame] | 207 | }; |
| 208 | |
| 209 | struct isert_device { |
Sagi Grimberg | a3a5a82 | 2014-01-09 18:40:50 +0200 | [diff] [blame] | 210 | int use_fastreg; |
Sagi Grimberg | d3e125d | 2014-02-19 17:50:23 +0200 | [diff] [blame] | 211 | bool pi_capable; |
Nicholas Bellinger | b8d26b3 | 2013-03-07 00:56:19 -0800 | [diff] [blame] | 212 | int refcount; |
Nicholas Bellinger | b8d26b3 | 2013-03-07 00:56:19 -0800 | [diff] [blame] | 213 | struct ib_device *ib_device; |
Sagi Grimberg | 4a295ba | 2014-12-02 16:57:40 +0200 | [diff] [blame] | 214 | struct isert_comp *comps; |
| 215 | int comps_used; |
Nicholas Bellinger | b8d26b3 | 2013-03-07 00:56:19 -0800 | [diff] [blame] | 216 | struct list_head dev_node; |
Vu Pham | 59464ef | 2013-08-28 23:23:35 +0300 | [diff] [blame] | 217 | struct ib_device_attr dev_attr; |
Vu Pham | d40945d | 2013-08-28 23:23:34 +0300 | [diff] [blame] | 218 | int (*reg_rdma_mem)(struct iscsi_conn *conn, |
| 219 | struct iscsi_cmd *cmd, |
| 220 | struct isert_rdma_wr *wr); |
| 221 | void (*unreg_rdma_mem)(struct isert_cmd *isert_cmd, |
| 222 | struct isert_conn *isert_conn); |
Nicholas Bellinger | b8d26b3 | 2013-03-07 00:56:19 -0800 | [diff] [blame] | 223 | }; |
| 224 | |
| 225 | struct isert_np { |
Sagi Grimberg | ca6c1d8 | 2014-12-02 16:57:27 +0200 | [diff] [blame] | 226 | struct iscsi_np *np; |
Sagi Grimberg | 531b7bf | 2014-04-29 13:13:45 +0300 | [diff] [blame] | 227 | struct semaphore np_sem; |
Nicholas Bellinger | b8d26b3 | 2013-03-07 00:56:19 -0800 | [diff] [blame] | 228 | struct rdma_cm_id *np_cm_id; |
| 229 | struct mutex np_accept_mutex; |
| 230 | struct list_head np_accept_list; |
| 231 | struct completion np_login_comp; |
| 232 | }; |