Nicholas Bellinger | b8d26b3 | 2013-03-07 00:56:19 -0800 | [diff] [blame] | 1 | #include <linux/socket.h> |
| 2 | #include <linux/in.h> |
| 3 | #include <linux/in6.h> |
| 4 | #include <rdma/ib_verbs.h> |
| 5 | #include <rdma/rdma_cm.h> |
Christoph Hellwig | 38a2d0d4 | 2016-05-03 18:01:13 +0200 | [diff] [blame] | 6 | #include <rdma/rw.h> |
Sagi Grimberg | d3cf81f | 2015-12-09 14:12:03 +0200 | [diff] [blame] | 7 | #include <scsi/iser.h> |
| 8 | |
Nicholas Bellinger | b8d26b3 | 2013-03-07 00:56:19 -0800 | [diff] [blame] | 9 | |
Sagi Grimberg | 24f412d | 2014-12-07 13:12:02 +0200 | [diff] [blame] | 10 | #define DRV_NAME "isert" |
| 11 | #define PFX DRV_NAME ": " |
| 12 | |
| 13 | #define isert_dbg(fmt, arg...) \ |
| 14 | do { \ |
| 15 | if (unlikely(isert_debug_level > 2)) \ |
| 16 | printk(KERN_DEBUG PFX "%s: " fmt,\ |
| 17 | __func__ , ## arg); \ |
| 18 | } while (0) |
| 19 | |
| 20 | #define isert_warn(fmt, arg...) \ |
| 21 | do { \ |
| 22 | if (unlikely(isert_debug_level > 0)) \ |
| 23 | pr_warn(PFX "%s: " fmt, \ |
| 24 | __func__ , ## arg); \ |
| 25 | } while (0) |
| 26 | |
| 27 | #define isert_info(fmt, arg...) \ |
| 28 | do { \ |
| 29 | if (unlikely(isert_debug_level > 1)) \ |
| 30 | pr_info(PFX "%s: " fmt, \ |
| 31 | __func__ , ## arg); \ |
| 32 | } while (0) |
| 33 | |
| 34 | #define isert_err(fmt, arg...) \ |
| 35 | pr_err(PFX "%s: " fmt, __func__ , ## arg) |
| 36 | |
Sagi Grimberg | c649415 | 2015-12-09 14:12:04 +0200 | [diff] [blame] | 37 | /* Constant PDU lengths calculations */ |
| 38 | #define ISER_HEADERS_LEN (sizeof(struct iser_ctrl) + \ |
| 39 | sizeof(struct iscsi_hdr)) |
Christoph Hellwig | ed1083b | 2016-02-24 19:24:04 +0200 | [diff] [blame] | 40 | #define ISER_RX_PAYLOAD_SIZE (ISER_HEADERS_LEN + ISCSI_DEF_MAX_RECV_SEG_LEN) |
Sagi Grimberg | c649415 | 2015-12-09 14:12:04 +0200 | [diff] [blame] | 41 | |
| 42 | /* QP settings */ |
| 43 | /* Maximal bounds on received asynchronous PDUs */ |
| 44 | #define ISERT_MAX_TX_MISC_PDUS 4 /* NOOP_IN(2) , ASYNC_EVENT(2) */ |
| 45 | |
| 46 | #define ISERT_MAX_RX_MISC_PDUS 6 /* |
| 47 | * NOOP_OUT(2), TEXT(1), |
| 48 | * SCSI_TMFUNC(2), LOGOUT(1) |
| 49 | */ |
| 50 | |
| 51 | #define ISCSI_DEF_XMIT_CMDS_MAX 128 /* from libiscsi.h, must be power of 2 */ |
| 52 | |
| 53 | #define ISERT_QP_MAX_RECV_DTOS (ISCSI_DEF_XMIT_CMDS_MAX) |
| 54 | |
| 55 | #define ISERT_MIN_POSTED_RX (ISCSI_DEF_XMIT_CMDS_MAX >> 2) |
| 56 | |
Christoph Hellwig | 38a2d0d4 | 2016-05-03 18:01:13 +0200 | [diff] [blame] | 57 | #define ISERT_QP_MAX_REQ_DTOS (ISCSI_DEF_XMIT_CMDS_MAX + \ |
Sagi Grimberg | c649415 | 2015-12-09 14:12:04 +0200 | [diff] [blame] | 58 | ISERT_MAX_TX_MISC_PDUS + \ |
| 59 | ISERT_MAX_RX_MISC_PDUS) |
| 60 | |
Christoph Hellwig | ed1083b | 2016-02-24 19:24:04 +0200 | [diff] [blame] | 61 | #define ISER_RX_PAD_SIZE (ISCSI_DEF_MAX_RECV_SEG_LEN + 4096 - \ |
Christoph Hellwig | 9679cc5 | 2016-02-24 19:24:06 +0200 | [diff] [blame] | 62 | (ISER_RX_PAYLOAD_SIZE + sizeof(u64) + sizeof(struct ib_sge) + \ |
| 63 | sizeof(struct ib_cqe))) |
Sagi Grimberg | c649415 | 2015-12-09 14:12:04 +0200 | [diff] [blame] | 64 | |
Vu Pham | 59464ef | 2013-08-28 23:23:35 +0300 | [diff] [blame] | 65 | #define ISCSI_ISER_SG_TABLESIZE 256 |
Nicholas Bellinger | b8d26b3 | 2013-03-07 00:56:19 -0800 | [diff] [blame] | 66 | |
| 67 | enum isert_desc_type { |
| 68 | ISCSI_TX_CONTROL, |
| 69 | ISCSI_TX_DATAIN |
| 70 | }; |
| 71 | |
Nicholas Bellinger | b8d26b3 | 2013-03-07 00:56:19 -0800 | [diff] [blame] | 72 | enum iser_conn_state { |
| 73 | ISER_CONN_INIT, |
| 74 | ISER_CONN_UP, |
Jenny Derzhavetz | aea9298 | 2016-02-24 19:23:59 +0200 | [diff] [blame] | 75 | ISER_CONN_BOUND, |
Sagi Grimberg | 128e9cc | 2014-12-02 16:57:20 +0200 | [diff] [blame] | 76 | ISER_CONN_FULL_FEATURE, |
Nicholas Bellinger | b8d26b3 | 2013-03-07 00:56:19 -0800 | [diff] [blame] | 77 | ISER_CONN_TERMINATING, |
| 78 | ISER_CONN_DOWN, |
| 79 | }; |
| 80 | |
| 81 | struct iser_rx_desc { |
Sagi Grimberg | d3cf81f | 2015-12-09 14:12:03 +0200 | [diff] [blame] | 82 | struct iser_ctrl iser_header; |
Nicholas Bellinger | b8d26b3 | 2013-03-07 00:56:19 -0800 | [diff] [blame] | 83 | struct iscsi_hdr iscsi_header; |
Christoph Hellwig | ed1083b | 2016-02-24 19:24:04 +0200 | [diff] [blame] | 84 | char data[ISCSI_DEF_MAX_RECV_SEG_LEN]; |
Nicholas Bellinger | b8d26b3 | 2013-03-07 00:56:19 -0800 | [diff] [blame] | 85 | u64 dma_addr; |
| 86 | struct ib_sge rx_sg; |
Christoph Hellwig | 9679cc5 | 2016-02-24 19:24:06 +0200 | [diff] [blame] | 87 | struct ib_cqe rx_cqe; |
Nicholas Bellinger | b8d26b3 | 2013-03-07 00:56:19 -0800 | [diff] [blame] | 88 | char pad[ISER_RX_PAD_SIZE]; |
| 89 | } __packed; |
| 90 | |
Christoph Hellwig | 9679cc5 | 2016-02-24 19:24:06 +0200 | [diff] [blame] | 91 | static inline struct iser_rx_desc *cqe_to_rx_desc(struct ib_cqe *cqe) |
| 92 | { |
| 93 | return container_of(cqe, struct iser_rx_desc, rx_cqe); |
| 94 | } |
| 95 | |
Nicholas Bellinger | b8d26b3 | 2013-03-07 00:56:19 -0800 | [diff] [blame] | 96 | struct iser_tx_desc { |
Sagi Grimberg | d3cf81f | 2015-12-09 14:12:03 +0200 | [diff] [blame] | 97 | struct iser_ctrl iser_header; |
Nicholas Bellinger | b8d26b3 | 2013-03-07 00:56:19 -0800 | [diff] [blame] | 98 | struct iscsi_hdr iscsi_header; |
| 99 | enum isert_desc_type type; |
| 100 | u64 dma_addr; |
| 101 | struct ib_sge tx_sg[2]; |
Christoph Hellwig | 9679cc5 | 2016-02-24 19:24:06 +0200 | [diff] [blame] | 102 | struct ib_cqe tx_cqe; |
Nicholas Bellinger | b8d26b3 | 2013-03-07 00:56:19 -0800 | [diff] [blame] | 103 | int num_sge; |
Nicholas Bellinger | b8d26b3 | 2013-03-07 00:56:19 -0800 | [diff] [blame] | 104 | struct ib_send_wr send_wr; |
| 105 | } __packed; |
| 106 | |
Christoph Hellwig | 9679cc5 | 2016-02-24 19:24:06 +0200 | [diff] [blame] | 107 | static inline struct iser_tx_desc *cqe_to_tx_desc(struct ib_cqe *cqe) |
| 108 | { |
| 109 | return container_of(cqe, struct iser_tx_desc, tx_cqe); |
| 110 | } |
| 111 | |
Nicholas Bellinger | b8d26b3 | 2013-03-07 00:56:19 -0800 | [diff] [blame] | 112 | struct isert_cmd { |
| 113 | uint32_t read_stag; |
| 114 | uint32_t write_stag; |
| 115 | uint64_t read_va; |
| 116 | uint64_t write_va; |
Jenny Derzhavetz | 422bd0a | 2015-12-09 14:12:06 +0200 | [diff] [blame] | 117 | uint32_t inv_rkey; |
Nicholas Bellinger | dbbc5d1 | 2013-07-03 19:39:37 -0700 | [diff] [blame] | 118 | u64 pdu_buf_dma; |
| 119 | u32 pdu_buf_len; |
Nicholas Bellinger | b8d26b3 | 2013-03-07 00:56:19 -0800 | [diff] [blame] | 120 | struct isert_conn *conn; |
Nicholas Bellinger | d703ce2 | 2013-08-17 14:27:56 -0700 | [diff] [blame] | 121 | struct iscsi_cmd *iscsi_cmd; |
Nicholas Bellinger | b8d26b3 | 2013-03-07 00:56:19 -0800 | [diff] [blame] | 122 | struct iser_tx_desc tx_desc; |
Jenny Derzhavetz | 4366b19 | 2015-09-06 14:52:25 +0300 | [diff] [blame] | 123 | struct iser_rx_desc *rx_desc; |
Christoph Hellwig | 38a2d0d4 | 2016-05-03 18:01:13 +0200 | [diff] [blame] | 124 | struct rdma_rw_ctx rw; |
Nicholas Bellinger | b8d26b3 | 2013-03-07 00:56:19 -0800 | [diff] [blame] | 125 | struct work_struct comp_work; |
Jenny Derzhavetz | 9fd6008 | 2015-09-06 14:52:26 +0300 | [diff] [blame] | 126 | struct scatterlist sg; |
Nicholas Bellinger | b8d26b3 | 2013-03-07 00:56:19 -0800 | [diff] [blame] | 127 | }; |
| 128 | |
Christoph Hellwig | e3416ab | 2016-02-24 19:24:08 +0200 | [diff] [blame] | 129 | static inline struct isert_cmd *tx_desc_to_cmd(struct iser_tx_desc *desc) |
| 130 | { |
| 131 | return container_of(desc, struct isert_cmd, tx_desc); |
| 132 | } |
| 133 | |
Nicholas Bellinger | b8d26b3 | 2013-03-07 00:56:19 -0800 | [diff] [blame] | 134 | struct isert_device; |
| 135 | |
| 136 | struct isert_conn { |
| 137 | enum iser_conn_state state; |
Nicholas Bellinger | b8d26b3 | 2013-03-07 00:56:19 -0800 | [diff] [blame] | 138 | u32 responder_resources; |
| 139 | u32 initiator_depth; |
Sagi Grimberg | 23a548e | 2014-12-02 16:57:35 +0200 | [diff] [blame] | 140 | bool pi_support; |
Christoph Hellwig | 5adabdd | 2016-02-24 19:24:05 +0200 | [diff] [blame] | 141 | struct iser_rx_desc *login_req_buf; |
Nicholas Bellinger | b8d26b3 | 2013-03-07 00:56:19 -0800 | [diff] [blame] | 142 | char *login_rsp_buf; |
| 143 | u64 login_req_dma; |
Sagi Grimberg | 2371e5d | 2014-12-02 16:57:21 +0200 | [diff] [blame] | 144 | int login_req_len; |
Nicholas Bellinger | b8d26b3 | 2013-03-07 00:56:19 -0800 | [diff] [blame] | 145 | u64 login_rsp_dma; |
Sagi Grimberg | dac6ab3 | 2015-03-29 15:52:19 +0300 | [diff] [blame] | 146 | struct iser_rx_desc *rx_descs; |
Jenny Derzhavetz | 4366b19 | 2015-09-06 14:52:25 +0300 | [diff] [blame] | 147 | struct ib_recv_wr rx_wr[ISERT_QP_MAX_RECV_DTOS]; |
Nicholas Bellinger | b8d26b3 | 2013-03-07 00:56:19 -0800 | [diff] [blame] | 148 | struct iscsi_conn *conn; |
Jenny Derzhavetz | bd37922 | 2015-09-06 14:52:24 +0300 | [diff] [blame] | 149 | struct list_head node; |
Sagi Grimberg | dac6ab3 | 2015-03-29 15:52:19 +0300 | [diff] [blame] | 150 | struct completion login_comp; |
Sagi Grimberg | 2371e5d | 2014-12-02 16:57:21 +0200 | [diff] [blame] | 151 | struct completion login_req_comp; |
Sagi Grimberg | dac6ab3 | 2015-03-29 15:52:19 +0300 | [diff] [blame] | 152 | struct iser_tx_desc login_tx_desc; |
| 153 | struct rdma_cm_id *cm_id; |
| 154 | struct ib_qp *qp; |
| 155 | struct isert_device *device; |
| 156 | struct mutex mutex; |
Sagi Grimberg | dac6ab3 | 2015-03-29 15:52:19 +0300 | [diff] [blame] | 157 | struct kref kref; |
Sagi Grimberg | b02efbf | 2014-12-02 16:57:29 +0200 | [diff] [blame] | 158 | struct work_struct release_work; |
Sagi Grimberg | 991bb76 | 2014-12-07 13:12:01 +0200 | [diff] [blame] | 159 | bool logout_posted; |
Jenny Derzhavetz | 422bd0a | 2015-12-09 14:12:06 +0200 | [diff] [blame] | 160 | bool snd_w_inv; |
Nicholas Bellinger | b8d26b3 | 2013-03-07 00:56:19 -0800 | [diff] [blame] | 161 | }; |
| 162 | |
| 163 | #define ISERT_MAX_CQ 64 |
| 164 | |
Sagi Grimberg | 4a295ba | 2014-12-02 16:57:40 +0200 | [diff] [blame] | 165 | /** |
| 166 | * struct isert_comp - iSER completion context |
| 167 | * |
| 168 | * @device: pointer to device handle |
Sagi Grimberg | 6f0fae3 | 2014-12-02 16:57:41 +0200 | [diff] [blame] | 169 | * @cq: completion queue |
Sagi Grimberg | 4a295ba | 2014-12-02 16:57:40 +0200 | [diff] [blame] | 170 | * @active_qps: Number of active QPs attached |
| 171 | * to completion context |
Sagi Grimberg | 4a295ba | 2014-12-02 16:57:40 +0200 | [diff] [blame] | 172 | */ |
| 173 | struct isert_comp { |
Sagi Grimberg | 6f0fae3 | 2014-12-02 16:57:41 +0200 | [diff] [blame] | 174 | struct isert_device *device; |
| 175 | struct ib_cq *cq; |
Sagi Grimberg | 4a295ba | 2014-12-02 16:57:40 +0200 | [diff] [blame] | 176 | int active_qps; |
Nicholas Bellinger | b8d26b3 | 2013-03-07 00:56:19 -0800 | [diff] [blame] | 177 | }; |
| 178 | |
| 179 | struct isert_device { |
Sagi Grimberg | d3e125d | 2014-02-19 17:50:23 +0200 | [diff] [blame] | 180 | bool pi_capable; |
Nicholas Bellinger | b8d26b3 | 2013-03-07 00:56:19 -0800 | [diff] [blame] | 181 | int refcount; |
Nicholas Bellinger | b8d26b3 | 2013-03-07 00:56:19 -0800 | [diff] [blame] | 182 | struct ib_device *ib_device; |
Sagi Grimberg | 67cb394 | 2015-03-29 15:52:05 +0300 | [diff] [blame] | 183 | struct ib_pd *pd; |
Sagi Grimberg | 4a295ba | 2014-12-02 16:57:40 +0200 | [diff] [blame] | 184 | struct isert_comp *comps; |
| 185 | int comps_used; |
Nicholas Bellinger | b8d26b3 | 2013-03-07 00:56:19 -0800 | [diff] [blame] | 186 | struct list_head dev_node; |
Nicholas Bellinger | b8d26b3 | 2013-03-07 00:56:19 -0800 | [diff] [blame] | 187 | }; |
| 188 | |
| 189 | struct isert_np { |
Sagi Grimberg | ca6c1d8 | 2014-12-02 16:57:27 +0200 | [diff] [blame] | 190 | struct iscsi_np *np; |
Jenny Derzhavetz | ed8cb0a | 2015-09-06 14:52:23 +0300 | [diff] [blame] | 191 | struct semaphore sem; |
| 192 | struct rdma_cm_id *cm_id; |
| 193 | struct mutex mutex; |
Jenny Derzhavetz | bd37922 | 2015-09-06 14:52:24 +0300 | [diff] [blame] | 194 | struct list_head accepted; |
| 195 | struct list_head pending; |
Nicholas Bellinger | b8d26b3 | 2013-03-07 00:56:19 -0800 | [diff] [blame] | 196 | }; |