Divy Le Ray | 4d22de3 | 2007-01-18 22:04:14 -0500 | [diff] [blame] | 1 | /* |
Divy Le Ray | 1d68e93 | 2007-01-30 19:44:35 -0800 | [diff] [blame] | 2 | * Copyright (c) 2006-2007 Chelsio, Inc. All rights reserved. |
Divy Le Ray | 4d22de3 | 2007-01-18 22:04:14 -0500 | [diff] [blame] | 3 | * |
| 4 | * This software is available to you under a choice of one of two |
| 5 | * licenses. You may choose to be licensed under the terms of the GNU |
| 6 | * General Public License (GPL) Version 2, available from the file |
| 7 | * COPYING in the main directory of this source tree, or the |
| 8 | * OpenIB.org BSD license below: |
| 9 | * |
| 10 | * Redistribution and use in source and binary forms, with or |
| 11 | * without modification, are permitted provided that the following |
| 12 | * conditions are met: |
| 13 | * |
| 14 | * - Redistributions of source code must retain the above |
| 15 | * copyright notice, this list of conditions and the following |
| 16 | * disclaimer. |
| 17 | * |
| 18 | * - Redistributions in binary form must reproduce the above |
| 19 | * copyright notice, this list of conditions and the following |
| 20 | * disclaimer in the documentation and/or other materials |
| 21 | * provided with the distribution. |
| 22 | * |
| 23 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, |
| 24 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF |
| 25 | * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND |
| 26 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS |
| 27 | * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN |
| 28 | * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN |
| 29 | * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE |
| 30 | * SOFTWARE. |
| 31 | */ |
| 32 | |
| 33 | #include <linux/list.h> |
| 34 | #include <net/neighbour.h> |
| 35 | #include <linux/notifier.h> |
| 36 | #include <asm/atomic.h> |
| 37 | #include <linux/proc_fs.h> |
| 38 | #include <linux/if_vlan.h> |
| 39 | #include <net/netevent.h> |
| 40 | #include <linux/highmem.h> |
| 41 | #include <linux/vmalloc.h> |
| 42 | |
| 43 | #include "common.h" |
| 44 | #include "regs.h" |
| 45 | #include "cxgb3_ioctl.h" |
| 46 | #include "cxgb3_ctl_defs.h" |
| 47 | #include "cxgb3_defs.h" |
| 48 | #include "l2t.h" |
| 49 | #include "firmware_exports.h" |
| 50 | #include "cxgb3_offload.h" |
| 51 | |
| 52 | static LIST_HEAD(client_list); |
| 53 | static LIST_HEAD(ofld_dev_list); |
| 54 | static DEFINE_MUTEX(cxgb3_db_lock); |
| 55 | |
| 56 | static DEFINE_RWLOCK(adapter_list_lock); |
| 57 | static LIST_HEAD(adapter_list); |
| 58 | |
| 59 | static const unsigned int MAX_ATIDS = 64 * 1024; |
Divy Le Ray | c9a6ce5 | 2007-08-21 20:49:26 -0700 | [diff] [blame] | 60 | static const unsigned int ATID_BASE = 0x10000; |
Divy Le Ray | 4d22de3 | 2007-01-18 22:04:14 -0500 | [diff] [blame] | 61 | |
| 62 | static inline int offload_activated(struct t3cdev *tdev) |
| 63 | { |
| 64 | const struct adapter *adapter = tdev2adap(tdev); |
| 65 | |
| 66 | return (test_bit(OFFLOAD_DEVMAP_BIT, &adapter->open_device_map)); |
| 67 | } |
| 68 | |
| 69 | /** |
| 70 | * cxgb3_register_client - register an offload client |
| 71 | * @client: the client |
| 72 | * |
| 73 | * Add the client to the client list, |
| 74 | * and call backs the client for each activated offload device |
| 75 | */ |
| 76 | void cxgb3_register_client(struct cxgb3_client *client) |
| 77 | { |
| 78 | struct t3cdev *tdev; |
| 79 | |
| 80 | mutex_lock(&cxgb3_db_lock); |
| 81 | list_add_tail(&client->client_list, &client_list); |
| 82 | |
| 83 | if (client->add) { |
| 84 | list_for_each_entry(tdev, &ofld_dev_list, ofld_dev_list) { |
| 85 | if (offload_activated(tdev)) |
| 86 | client->add(tdev); |
| 87 | } |
| 88 | } |
| 89 | mutex_unlock(&cxgb3_db_lock); |
| 90 | } |
| 91 | |
| 92 | EXPORT_SYMBOL(cxgb3_register_client); |
| 93 | |
| 94 | /** |
| 95 | * cxgb3_unregister_client - unregister an offload client |
| 96 | * @client: the client |
| 97 | * |
| 98 | * Remove the client to the client list, |
| 99 | * and call backs the client for each activated offload device. |
| 100 | */ |
| 101 | void cxgb3_unregister_client(struct cxgb3_client *client) |
| 102 | { |
| 103 | struct t3cdev *tdev; |
| 104 | |
| 105 | mutex_lock(&cxgb3_db_lock); |
| 106 | list_del(&client->client_list); |
| 107 | |
| 108 | if (client->remove) { |
| 109 | list_for_each_entry(tdev, &ofld_dev_list, ofld_dev_list) { |
| 110 | if (offload_activated(tdev)) |
| 111 | client->remove(tdev); |
| 112 | } |
| 113 | } |
| 114 | mutex_unlock(&cxgb3_db_lock); |
| 115 | } |
| 116 | |
| 117 | EXPORT_SYMBOL(cxgb3_unregister_client); |
| 118 | |
| 119 | /** |
| 120 | * cxgb3_add_clients - activate registered clients for an offload device |
| 121 | * @tdev: the offload device |
| 122 | * |
| 123 | * Call backs all registered clients once a offload device is activated |
| 124 | */ |
| 125 | void cxgb3_add_clients(struct t3cdev *tdev) |
| 126 | { |
| 127 | struct cxgb3_client *client; |
| 128 | |
| 129 | mutex_lock(&cxgb3_db_lock); |
| 130 | list_for_each_entry(client, &client_list, client_list) { |
| 131 | if (client->add) |
| 132 | client->add(tdev); |
| 133 | } |
| 134 | mutex_unlock(&cxgb3_db_lock); |
| 135 | } |
| 136 | |
| 137 | /** |
| 138 | * cxgb3_remove_clients - deactivates registered clients |
| 139 | * for an offload device |
| 140 | * @tdev: the offload device |
| 141 | * |
| 142 | * Call backs all registered clients once a offload device is deactivated |
| 143 | */ |
| 144 | void cxgb3_remove_clients(struct t3cdev *tdev) |
| 145 | { |
| 146 | struct cxgb3_client *client; |
| 147 | |
| 148 | mutex_lock(&cxgb3_db_lock); |
| 149 | list_for_each_entry(client, &client_list, client_list) { |
| 150 | if (client->remove) |
| 151 | client->remove(tdev); |
| 152 | } |
| 153 | mutex_unlock(&cxgb3_db_lock); |
| 154 | } |
| 155 | |
| 156 | static struct net_device *get_iff_from_mac(struct adapter *adapter, |
| 157 | const unsigned char *mac, |
| 158 | unsigned int vlan) |
| 159 | { |
| 160 | int i; |
| 161 | |
| 162 | for_each_port(adapter, i) { |
Dan Aloni | 5c15bde | 2007-03-02 20:44:51 -0800 | [diff] [blame] | 163 | struct vlan_group *grp; |
Divy Le Ray | 4d22de3 | 2007-01-18 22:04:14 -0500 | [diff] [blame] | 164 | struct net_device *dev = adapter->port[i]; |
| 165 | const struct port_info *p = netdev_priv(dev); |
| 166 | |
| 167 | if (!memcmp(dev->dev_addr, mac, ETH_ALEN)) { |
| 168 | if (vlan && vlan != VLAN_VID_MASK) { |
| 169 | grp = p->vlan_grp; |
Dan Aloni | 5c15bde | 2007-03-02 20:44:51 -0800 | [diff] [blame] | 170 | dev = NULL; |
| 171 | if (grp) |
| 172 | dev = vlan_group_get_device(grp, vlan); |
Divy Le Ray | 4d22de3 | 2007-01-18 22:04:14 -0500 | [diff] [blame] | 173 | } else |
| 174 | while (dev->master) |
| 175 | dev = dev->master; |
| 176 | return dev; |
| 177 | } |
| 178 | } |
| 179 | return NULL; |
| 180 | } |
| 181 | |
| 182 | static int cxgb_ulp_iscsi_ctl(struct adapter *adapter, unsigned int req, |
| 183 | void *data) |
| 184 | { |
| 185 | int ret = 0; |
| 186 | struct ulp_iscsi_info *uiip = data; |
| 187 | |
| 188 | switch (req) { |
| 189 | case ULP_ISCSI_GET_PARAMS: |
| 190 | uiip->pdev = adapter->pdev; |
| 191 | uiip->llimit = t3_read_reg(adapter, A_ULPRX_ISCSI_LLIMIT); |
| 192 | uiip->ulimit = t3_read_reg(adapter, A_ULPRX_ISCSI_ULIMIT); |
| 193 | uiip->tagmask = t3_read_reg(adapter, A_ULPRX_ISCSI_TAGMASK); |
| 194 | /* |
| 195 | * On tx, the iscsi pdu has to be <= tx page size and has to |
| 196 | * fit into the Tx PM FIFO. |
| 197 | */ |
| 198 | uiip->max_txsz = min(adapter->params.tp.tx_pg_size, |
| 199 | t3_read_reg(adapter, A_PM1_TX_CFG) >> 17); |
| 200 | /* on rx, the iscsi pdu has to be < rx page size and the |
| 201 | whole pdu + cpl headers has to fit into one sge buffer */ |
| 202 | uiip->max_rxsz = min_t(unsigned int, |
| 203 | adapter->params.tp.rx_pg_size, |
| 204 | (adapter->sge.qs[0].fl[1].buf_size - |
| 205 | sizeof(struct cpl_rx_data) * 2 - |
| 206 | sizeof(struct cpl_rx_data_ddp))); |
| 207 | break; |
| 208 | case ULP_ISCSI_SET_PARAMS: |
| 209 | t3_write_reg(adapter, A_ULPRX_ISCSI_TAGMASK, uiip->tagmask); |
| 210 | break; |
| 211 | default: |
| 212 | ret = -EOPNOTSUPP; |
| 213 | } |
| 214 | return ret; |
| 215 | } |
| 216 | |
| 217 | /* Response queue used for RDMA events. */ |
| 218 | #define ASYNC_NOTIF_RSPQ 0 |
| 219 | |
| 220 | static int cxgb_rdma_ctl(struct adapter *adapter, unsigned int req, void *data) |
| 221 | { |
| 222 | int ret = 0; |
| 223 | |
| 224 | switch (req) { |
Stephen Hemminger | 9265fab | 2007-10-08 16:22:29 -0700 | [diff] [blame] | 225 | case RDMA_GET_PARAMS: { |
| 226 | struct rdma_info *rdma = data; |
Divy Le Ray | 4d22de3 | 2007-01-18 22:04:14 -0500 | [diff] [blame] | 227 | struct pci_dev *pdev = adapter->pdev; |
| 228 | |
Stephen Hemminger | 9265fab | 2007-10-08 16:22:29 -0700 | [diff] [blame] | 229 | rdma->udbell_physbase = pci_resource_start(pdev, 2); |
| 230 | rdma->udbell_len = pci_resource_len(pdev, 2); |
| 231 | rdma->tpt_base = |
Divy Le Ray | 4d22de3 | 2007-01-18 22:04:14 -0500 | [diff] [blame] | 232 | t3_read_reg(adapter, A_ULPTX_TPT_LLIMIT); |
Stephen Hemminger | 9265fab | 2007-10-08 16:22:29 -0700 | [diff] [blame] | 233 | rdma->tpt_top = t3_read_reg(adapter, A_ULPTX_TPT_ULIMIT); |
| 234 | rdma->pbl_base = |
Divy Le Ray | 4d22de3 | 2007-01-18 22:04:14 -0500 | [diff] [blame] | 235 | t3_read_reg(adapter, A_ULPTX_PBL_LLIMIT); |
Stephen Hemminger | 9265fab | 2007-10-08 16:22:29 -0700 | [diff] [blame] | 236 | rdma->pbl_top = t3_read_reg(adapter, A_ULPTX_PBL_ULIMIT); |
| 237 | rdma->rqt_base = t3_read_reg(adapter, A_ULPRX_RQ_LLIMIT); |
| 238 | rdma->rqt_top = t3_read_reg(adapter, A_ULPRX_RQ_ULIMIT); |
| 239 | rdma->kdb_addr = adapter->regs + A_SG_KDOORBELL; |
| 240 | rdma->pdev = pdev; |
Divy Le Ray | 4d22de3 | 2007-01-18 22:04:14 -0500 | [diff] [blame] | 241 | break; |
| 242 | } |
| 243 | case RDMA_CQ_OP:{ |
| 244 | unsigned long flags; |
Stephen Hemminger | 9265fab | 2007-10-08 16:22:29 -0700 | [diff] [blame] | 245 | struct rdma_cq_op *rdma = data; |
Divy Le Ray | 4d22de3 | 2007-01-18 22:04:14 -0500 | [diff] [blame] | 246 | |
| 247 | /* may be called in any context */ |
| 248 | spin_lock_irqsave(&adapter->sge.reg_lock, flags); |
Stephen Hemminger | 9265fab | 2007-10-08 16:22:29 -0700 | [diff] [blame] | 249 | ret = t3_sge_cqcntxt_op(adapter, rdma->id, rdma->op, |
| 250 | rdma->credits); |
Divy Le Ray | 4d22de3 | 2007-01-18 22:04:14 -0500 | [diff] [blame] | 251 | spin_unlock_irqrestore(&adapter->sge.reg_lock, flags); |
| 252 | break; |
| 253 | } |
| 254 | case RDMA_GET_MEM:{ |
| 255 | struct ch_mem_range *t = data; |
| 256 | struct mc7 *mem; |
| 257 | |
| 258 | if ((t->addr & 7) || (t->len & 7)) |
| 259 | return -EINVAL; |
| 260 | if (t->mem_id == MEM_CM) |
| 261 | mem = &adapter->cm; |
| 262 | else if (t->mem_id == MEM_PMRX) |
| 263 | mem = &adapter->pmrx; |
| 264 | else if (t->mem_id == MEM_PMTX) |
| 265 | mem = &adapter->pmtx; |
| 266 | else |
| 267 | return -EINVAL; |
| 268 | |
| 269 | ret = |
| 270 | t3_mc7_bd_read(mem, t->addr / 8, t->len / 8, |
| 271 | (u64 *) t->buf); |
| 272 | if (ret) |
| 273 | return ret; |
| 274 | break; |
| 275 | } |
| 276 | case RDMA_CQ_SETUP:{ |
Stephen Hemminger | 9265fab | 2007-10-08 16:22:29 -0700 | [diff] [blame] | 277 | struct rdma_cq_setup *rdma = data; |
Divy Le Ray | 4d22de3 | 2007-01-18 22:04:14 -0500 | [diff] [blame] | 278 | |
| 279 | spin_lock_irq(&adapter->sge.reg_lock); |
| 280 | ret = |
Stephen Hemminger | 9265fab | 2007-10-08 16:22:29 -0700 | [diff] [blame] | 281 | t3_sge_init_cqcntxt(adapter, rdma->id, |
| 282 | rdma->base_addr, rdma->size, |
Divy Le Ray | 4d22de3 | 2007-01-18 22:04:14 -0500 | [diff] [blame] | 283 | ASYNC_NOTIF_RSPQ, |
Stephen Hemminger | 9265fab | 2007-10-08 16:22:29 -0700 | [diff] [blame] | 284 | rdma->ovfl_mode, rdma->credits, |
| 285 | rdma->credit_thres); |
Divy Le Ray | 4d22de3 | 2007-01-18 22:04:14 -0500 | [diff] [blame] | 286 | spin_unlock_irq(&adapter->sge.reg_lock); |
| 287 | break; |
| 288 | } |
| 289 | case RDMA_CQ_DISABLE: |
| 290 | spin_lock_irq(&adapter->sge.reg_lock); |
| 291 | ret = t3_sge_disable_cqcntxt(adapter, *(unsigned int *)data); |
| 292 | spin_unlock_irq(&adapter->sge.reg_lock); |
| 293 | break; |
| 294 | case RDMA_CTRL_QP_SETUP:{ |
Stephen Hemminger | 9265fab | 2007-10-08 16:22:29 -0700 | [diff] [blame] | 295 | struct rdma_ctrlqp_setup *rdma = data; |
Divy Le Ray | 4d22de3 | 2007-01-18 22:04:14 -0500 | [diff] [blame] | 296 | |
| 297 | spin_lock_irq(&adapter->sge.reg_lock); |
| 298 | ret = t3_sge_init_ecntxt(adapter, FW_RI_SGEEC_START, 0, |
| 299 | SGE_CNTXT_RDMA, |
| 300 | ASYNC_NOTIF_RSPQ, |
Stephen Hemminger | 9265fab | 2007-10-08 16:22:29 -0700 | [diff] [blame] | 301 | rdma->base_addr, rdma->size, |
Divy Le Ray | 4d22de3 | 2007-01-18 22:04:14 -0500 | [diff] [blame] | 302 | FW_RI_TID_START, 1, 0); |
| 303 | spin_unlock_irq(&adapter->sge.reg_lock); |
| 304 | break; |
| 305 | } |
| 306 | default: |
| 307 | ret = -EOPNOTSUPP; |
| 308 | } |
| 309 | return ret; |
| 310 | } |
| 311 | |
| 312 | static int cxgb_offload_ctl(struct t3cdev *tdev, unsigned int req, void *data) |
| 313 | { |
| 314 | struct adapter *adapter = tdev2adap(tdev); |
| 315 | struct tid_range *tid; |
| 316 | struct mtutab *mtup; |
| 317 | struct iff_mac *iffmacp; |
| 318 | struct ddp_params *ddpp; |
| 319 | struct adap_ports *ports; |
Divy Le Ray | e22bb45 | 2007-08-21 20:49:21 -0700 | [diff] [blame] | 320 | struct ofld_page_info *rx_page_info; |
| 321 | struct tp_params *tp = &adapter->params.tp; |
Divy Le Ray | 4d22de3 | 2007-01-18 22:04:14 -0500 | [diff] [blame] | 322 | int i; |
| 323 | |
| 324 | switch (req) { |
| 325 | case GET_MAX_OUTSTANDING_WR: |
| 326 | *(unsigned int *)data = FW_WR_NUM; |
| 327 | break; |
| 328 | case GET_WR_LEN: |
| 329 | *(unsigned int *)data = WR_FLITS; |
| 330 | break; |
| 331 | case GET_TX_MAX_CHUNK: |
| 332 | *(unsigned int *)data = 1 << 20; /* 1MB */ |
| 333 | break; |
| 334 | case GET_TID_RANGE: |
| 335 | tid = data; |
| 336 | tid->num = t3_mc5_size(&adapter->mc5) - |
| 337 | adapter->params.mc5.nroutes - |
| 338 | adapter->params.mc5.nfilters - adapter->params.mc5.nservers; |
| 339 | tid->base = 0; |
| 340 | break; |
| 341 | case GET_STID_RANGE: |
| 342 | tid = data; |
| 343 | tid->num = adapter->params.mc5.nservers; |
| 344 | tid->base = t3_mc5_size(&adapter->mc5) - tid->num - |
| 345 | adapter->params.mc5.nfilters - adapter->params.mc5.nroutes; |
| 346 | break; |
| 347 | case GET_L2T_CAPACITY: |
| 348 | *(unsigned int *)data = 2048; |
| 349 | break; |
| 350 | case GET_MTUS: |
| 351 | mtup = data; |
| 352 | mtup->size = NMTUS; |
| 353 | mtup->mtus = adapter->params.mtus; |
| 354 | break; |
| 355 | case GET_IFF_FROM_MAC: |
| 356 | iffmacp = data; |
| 357 | iffmacp->dev = get_iff_from_mac(adapter, iffmacp->mac_addr, |
| 358 | iffmacp->vlan_tag & |
| 359 | VLAN_VID_MASK); |
| 360 | break; |
| 361 | case GET_DDP_PARAMS: |
| 362 | ddpp = data; |
| 363 | ddpp->llimit = t3_read_reg(adapter, A_ULPRX_TDDP_LLIMIT); |
| 364 | ddpp->ulimit = t3_read_reg(adapter, A_ULPRX_TDDP_ULIMIT); |
| 365 | ddpp->tag_mask = t3_read_reg(adapter, A_ULPRX_TDDP_TAGMASK); |
| 366 | break; |
| 367 | case GET_PORTS: |
| 368 | ports = data; |
| 369 | ports->nports = adapter->params.nports; |
| 370 | for_each_port(adapter, i) |
| 371 | ports->lldevs[i] = adapter->port[i]; |
| 372 | break; |
| 373 | case ULP_ISCSI_GET_PARAMS: |
| 374 | case ULP_ISCSI_SET_PARAMS: |
| 375 | if (!offload_running(adapter)) |
| 376 | return -EAGAIN; |
| 377 | return cxgb_ulp_iscsi_ctl(adapter, req, data); |
| 378 | case RDMA_GET_PARAMS: |
| 379 | case RDMA_CQ_OP: |
| 380 | case RDMA_CQ_SETUP: |
| 381 | case RDMA_CQ_DISABLE: |
| 382 | case RDMA_CTRL_QP_SETUP: |
| 383 | case RDMA_GET_MEM: |
| 384 | if (!offload_running(adapter)) |
| 385 | return -EAGAIN; |
| 386 | return cxgb_rdma_ctl(adapter, req, data); |
Divy Le Ray | e22bb45 | 2007-08-21 20:49:21 -0700 | [diff] [blame] | 387 | case GET_RX_PAGE_INFO: |
| 388 | rx_page_info = data; |
| 389 | rx_page_info->page_size = tp->rx_pg_size; |
| 390 | rx_page_info->num = tp->rx_num_pgs; |
| 391 | break; |
Divy Le Ray | 4d22de3 | 2007-01-18 22:04:14 -0500 | [diff] [blame] | 392 | default: |
| 393 | return -EOPNOTSUPP; |
| 394 | } |
| 395 | return 0; |
| 396 | } |
| 397 | |
| 398 | /* |
| 399 | * Dummy handler for Rx offload packets in case we get an offload packet before |
| 400 | * proper processing is setup. This complains and drops the packet as it isn't |
| 401 | * normal to get offload packets at this stage. |
| 402 | */ |
| 403 | static int rx_offload_blackhole(struct t3cdev *dev, struct sk_buff **skbs, |
| 404 | int n) |
| 405 | { |
| 406 | CH_ERR(tdev2adap(dev), "%d unexpected offload packets, first data %u\n", |
Al Viro | b534497 | 2007-02-09 16:40:10 +0000 | [diff] [blame] | 407 | n, ntohl(*(__be32 *)skbs[0]->data)); |
Divy Le Ray | 4d22de3 | 2007-01-18 22:04:14 -0500 | [diff] [blame] | 408 | while (n--) |
| 409 | dev_kfree_skb_any(skbs[n]); |
| 410 | return 0; |
| 411 | } |
| 412 | |
| 413 | static void dummy_neigh_update(struct t3cdev *dev, struct neighbour *neigh) |
| 414 | { |
| 415 | } |
| 416 | |
| 417 | void cxgb3_set_dummy_ops(struct t3cdev *dev) |
| 418 | { |
| 419 | dev->recv = rx_offload_blackhole; |
| 420 | dev->neigh_update = dummy_neigh_update; |
| 421 | } |
| 422 | |
| 423 | /* |
| 424 | * Free an active-open TID. |
| 425 | */ |
| 426 | void *cxgb3_free_atid(struct t3cdev *tdev, int atid) |
| 427 | { |
| 428 | struct tid_info *t = &(T3C_DATA(tdev))->tid_maps; |
| 429 | union active_open_entry *p = atid2entry(t, atid); |
| 430 | void *ctx = p->t3c_tid.ctx; |
| 431 | |
| 432 | spin_lock_bh(&t->atid_lock); |
| 433 | p->next = t->afree; |
| 434 | t->afree = p; |
| 435 | t->atids_in_use--; |
| 436 | spin_unlock_bh(&t->atid_lock); |
| 437 | |
| 438 | return ctx; |
| 439 | } |
| 440 | |
| 441 | EXPORT_SYMBOL(cxgb3_free_atid); |
| 442 | |
| 443 | /* |
| 444 | * Free a server TID and return it to the free pool. |
| 445 | */ |
| 446 | void cxgb3_free_stid(struct t3cdev *tdev, int stid) |
| 447 | { |
| 448 | struct tid_info *t = &(T3C_DATA(tdev))->tid_maps; |
| 449 | union listen_entry *p = stid2entry(t, stid); |
| 450 | |
| 451 | spin_lock_bh(&t->stid_lock); |
| 452 | p->next = t->sfree; |
| 453 | t->sfree = p; |
| 454 | t->stids_in_use--; |
| 455 | spin_unlock_bh(&t->stid_lock); |
| 456 | } |
| 457 | |
| 458 | EXPORT_SYMBOL(cxgb3_free_stid); |
| 459 | |
| 460 | void cxgb3_insert_tid(struct t3cdev *tdev, struct cxgb3_client *client, |
| 461 | void *ctx, unsigned int tid) |
| 462 | { |
| 463 | struct tid_info *t = &(T3C_DATA(tdev))->tid_maps; |
| 464 | |
| 465 | t->tid_tab[tid].client = client; |
| 466 | t->tid_tab[tid].ctx = ctx; |
| 467 | atomic_inc(&t->tids_in_use); |
| 468 | } |
| 469 | |
| 470 | EXPORT_SYMBOL(cxgb3_insert_tid); |
| 471 | |
| 472 | /* |
| 473 | * Populate a TID_RELEASE WR. The skb must be already propely sized. |
| 474 | */ |
| 475 | static inline void mk_tid_release(struct sk_buff *skb, unsigned int tid) |
| 476 | { |
| 477 | struct cpl_tid_release *req; |
| 478 | |
| 479 | skb->priority = CPL_PRIORITY_SETUP; |
| 480 | req = (struct cpl_tid_release *)__skb_put(skb, sizeof(*req)); |
| 481 | req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD)); |
| 482 | OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_TID_RELEASE, tid)); |
| 483 | } |
| 484 | |
| 485 | static void t3_process_tid_release_list(struct work_struct *work) |
| 486 | { |
| 487 | struct t3c_data *td = container_of(work, struct t3c_data, |
| 488 | tid_release_task); |
| 489 | struct sk_buff *skb; |
| 490 | struct t3cdev *tdev = td->dev; |
Jeff Garzik | 2eab17a | 2007-11-23 21:59:45 -0500 | [diff] [blame^] | 491 | |
Divy Le Ray | 4d22de3 | 2007-01-18 22:04:14 -0500 | [diff] [blame] | 492 | |
| 493 | spin_lock_bh(&td->tid_release_lock); |
| 494 | while (td->tid_release_list) { |
| 495 | struct t3c_tid_entry *p = td->tid_release_list; |
| 496 | |
| 497 | td->tid_release_list = (struct t3c_tid_entry *)p->ctx; |
| 498 | spin_unlock_bh(&td->tid_release_lock); |
| 499 | |
| 500 | skb = alloc_skb(sizeof(struct cpl_tid_release), |
| 501 | GFP_KERNEL | __GFP_NOFAIL); |
| 502 | mk_tid_release(skb, p - td->tid_maps.tid_tab); |
| 503 | cxgb3_ofld_send(tdev, skb); |
| 504 | p->ctx = NULL; |
| 505 | spin_lock_bh(&td->tid_release_lock); |
| 506 | } |
| 507 | spin_unlock_bh(&td->tid_release_lock); |
| 508 | } |
| 509 | |
| 510 | /* use ctx as a next pointer in the tid release list */ |
| 511 | void cxgb3_queue_tid_release(struct t3cdev *tdev, unsigned int tid) |
| 512 | { |
| 513 | struct t3c_data *td = T3C_DATA(tdev); |
| 514 | struct t3c_tid_entry *p = &td->tid_maps.tid_tab[tid]; |
| 515 | |
| 516 | spin_lock_bh(&td->tid_release_lock); |
| 517 | p->ctx = (void *)td->tid_release_list; |
Divy Le Ray | 606fcd0 | 2007-04-17 11:06:30 -0700 | [diff] [blame] | 518 | p->client = NULL; |
Divy Le Ray | 4d22de3 | 2007-01-18 22:04:14 -0500 | [diff] [blame] | 519 | td->tid_release_list = p; |
| 520 | if (!p->ctx) |
| 521 | schedule_work(&td->tid_release_task); |
| 522 | spin_unlock_bh(&td->tid_release_lock); |
| 523 | } |
| 524 | |
| 525 | EXPORT_SYMBOL(cxgb3_queue_tid_release); |
| 526 | |
| 527 | /* |
| 528 | * Remove a tid from the TID table. A client may defer processing its last |
| 529 | * CPL message if it is locked at the time it arrives, and while the message |
| 530 | * sits in the client's backlog the TID may be reused for another connection. |
| 531 | * To handle this we atomically switch the TID association if it still points |
| 532 | * to the original client context. |
| 533 | */ |
| 534 | void cxgb3_remove_tid(struct t3cdev *tdev, void *ctx, unsigned int tid) |
| 535 | { |
| 536 | struct tid_info *t = &(T3C_DATA(tdev))->tid_maps; |
| 537 | |
| 538 | BUG_ON(tid >= t->ntids); |
| 539 | if (tdev->type == T3A) |
| 540 | (void)cmpxchg(&t->tid_tab[tid].ctx, ctx, NULL); |
| 541 | else { |
| 542 | struct sk_buff *skb; |
| 543 | |
| 544 | skb = alloc_skb(sizeof(struct cpl_tid_release), GFP_ATOMIC); |
| 545 | if (likely(skb)) { |
| 546 | mk_tid_release(skb, tid); |
| 547 | cxgb3_ofld_send(tdev, skb); |
| 548 | t->tid_tab[tid].ctx = NULL; |
| 549 | } else |
| 550 | cxgb3_queue_tid_release(tdev, tid); |
| 551 | } |
| 552 | atomic_dec(&t->tids_in_use); |
| 553 | } |
| 554 | |
| 555 | EXPORT_SYMBOL(cxgb3_remove_tid); |
| 556 | |
| 557 | int cxgb3_alloc_atid(struct t3cdev *tdev, struct cxgb3_client *client, |
| 558 | void *ctx) |
| 559 | { |
| 560 | int atid = -1; |
| 561 | struct tid_info *t = &(T3C_DATA(tdev))->tid_maps; |
| 562 | |
| 563 | spin_lock_bh(&t->atid_lock); |
Divy Le Ray | 9f23848 | 2007-03-31 00:23:13 -0700 | [diff] [blame] | 564 | if (t->afree && |
| 565 | t->atids_in_use + atomic_read(&t->tids_in_use) + MC5_MIN_TIDS <= |
| 566 | t->ntids) { |
Divy Le Ray | 4d22de3 | 2007-01-18 22:04:14 -0500 | [diff] [blame] | 567 | union active_open_entry *p = t->afree; |
| 568 | |
| 569 | atid = (p - t->atid_tab) + t->atid_base; |
| 570 | t->afree = p->next; |
| 571 | p->t3c_tid.ctx = ctx; |
| 572 | p->t3c_tid.client = client; |
| 573 | t->atids_in_use++; |
| 574 | } |
| 575 | spin_unlock_bh(&t->atid_lock); |
| 576 | return atid; |
| 577 | } |
| 578 | |
| 579 | EXPORT_SYMBOL(cxgb3_alloc_atid); |
| 580 | |
| 581 | int cxgb3_alloc_stid(struct t3cdev *tdev, struct cxgb3_client *client, |
| 582 | void *ctx) |
| 583 | { |
| 584 | int stid = -1; |
| 585 | struct tid_info *t = &(T3C_DATA(tdev))->tid_maps; |
| 586 | |
| 587 | spin_lock_bh(&t->stid_lock); |
| 588 | if (t->sfree) { |
| 589 | union listen_entry *p = t->sfree; |
| 590 | |
| 591 | stid = (p - t->stid_tab) + t->stid_base; |
| 592 | t->sfree = p->next; |
| 593 | p->t3c_tid.ctx = ctx; |
| 594 | p->t3c_tid.client = client; |
| 595 | t->stids_in_use++; |
| 596 | } |
| 597 | spin_unlock_bh(&t->stid_lock); |
| 598 | return stid; |
| 599 | } |
| 600 | |
| 601 | EXPORT_SYMBOL(cxgb3_alloc_stid); |
| 602 | |
Divy Le Ray | 5fbf816 | 2007-08-29 19:15:47 -0700 | [diff] [blame] | 603 | /* Get the t3cdev associated with a net_device */ |
| 604 | struct t3cdev *dev2t3cdev(struct net_device *dev) |
| 605 | { |
| 606 | const struct port_info *pi = netdev_priv(dev); |
| 607 | |
| 608 | return (struct t3cdev *)pi->adapter; |
| 609 | } |
| 610 | |
| 611 | EXPORT_SYMBOL(dev2t3cdev); |
| 612 | |
Divy Le Ray | 4d22de3 | 2007-01-18 22:04:14 -0500 | [diff] [blame] | 613 | static int do_smt_write_rpl(struct t3cdev *dev, struct sk_buff *skb) |
| 614 | { |
| 615 | struct cpl_smt_write_rpl *rpl = cplhdr(skb); |
| 616 | |
| 617 | if (rpl->status != CPL_ERR_NONE) |
| 618 | printk(KERN_ERR |
| 619 | "Unexpected SMT_WRITE_RPL status %u for entry %u\n", |
| 620 | rpl->status, GET_TID(rpl)); |
| 621 | |
| 622 | return CPL_RET_BUF_DONE; |
| 623 | } |
| 624 | |
| 625 | static int do_l2t_write_rpl(struct t3cdev *dev, struct sk_buff *skb) |
| 626 | { |
| 627 | struct cpl_l2t_write_rpl *rpl = cplhdr(skb); |
| 628 | |
| 629 | if (rpl->status != CPL_ERR_NONE) |
| 630 | printk(KERN_ERR |
| 631 | "Unexpected L2T_WRITE_RPL status %u for entry %u\n", |
| 632 | rpl->status, GET_TID(rpl)); |
| 633 | |
| 634 | return CPL_RET_BUF_DONE; |
| 635 | } |
| 636 | |
| 637 | static int do_act_open_rpl(struct t3cdev *dev, struct sk_buff *skb) |
| 638 | { |
| 639 | struct cpl_act_open_rpl *rpl = cplhdr(skb); |
| 640 | unsigned int atid = G_TID(ntohl(rpl->atid)); |
| 641 | struct t3c_tid_entry *t3c_tid; |
| 642 | |
| 643 | t3c_tid = lookup_atid(&(T3C_DATA(dev))->tid_maps, atid); |
Divy Le Ray | 606fcd0 | 2007-04-17 11:06:30 -0700 | [diff] [blame] | 644 | if (t3c_tid && t3c_tid->ctx && t3c_tid->client && |
| 645 | t3c_tid->client->handlers && |
Divy Le Ray | 4d22de3 | 2007-01-18 22:04:14 -0500 | [diff] [blame] | 646 | t3c_tid->client->handlers[CPL_ACT_OPEN_RPL]) { |
| 647 | return t3c_tid->client->handlers[CPL_ACT_OPEN_RPL] (dev, skb, |
| 648 | t3c_tid-> |
| 649 | ctx); |
| 650 | } else { |
| 651 | printk(KERN_ERR "%s: received clientless CPL command 0x%x\n", |
| 652 | dev->name, CPL_ACT_OPEN_RPL); |
| 653 | return CPL_RET_BUF_DONE | CPL_RET_BAD_MSG; |
| 654 | } |
| 655 | } |
| 656 | |
| 657 | static int do_stid_rpl(struct t3cdev *dev, struct sk_buff *skb) |
| 658 | { |
| 659 | union opcode_tid *p = cplhdr(skb); |
| 660 | unsigned int stid = G_TID(ntohl(p->opcode_tid)); |
| 661 | struct t3c_tid_entry *t3c_tid; |
| 662 | |
| 663 | t3c_tid = lookup_stid(&(T3C_DATA(dev))->tid_maps, stid); |
Divy Le Ray | 606fcd0 | 2007-04-17 11:06:30 -0700 | [diff] [blame] | 664 | if (t3c_tid && t3c_tid->ctx && t3c_tid->client->handlers && |
Divy Le Ray | 4d22de3 | 2007-01-18 22:04:14 -0500 | [diff] [blame] | 665 | t3c_tid->client->handlers[p->opcode]) { |
| 666 | return t3c_tid->client->handlers[p->opcode] (dev, skb, |
| 667 | t3c_tid->ctx); |
| 668 | } else { |
| 669 | printk(KERN_ERR "%s: received clientless CPL command 0x%x\n", |
| 670 | dev->name, p->opcode); |
| 671 | return CPL_RET_BUF_DONE | CPL_RET_BAD_MSG; |
| 672 | } |
| 673 | } |
| 674 | |
| 675 | static int do_hwtid_rpl(struct t3cdev *dev, struct sk_buff *skb) |
| 676 | { |
| 677 | union opcode_tid *p = cplhdr(skb); |
| 678 | unsigned int hwtid = G_TID(ntohl(p->opcode_tid)); |
| 679 | struct t3c_tid_entry *t3c_tid; |
| 680 | |
| 681 | t3c_tid = lookup_tid(&(T3C_DATA(dev))->tid_maps, hwtid); |
Divy Le Ray | 606fcd0 | 2007-04-17 11:06:30 -0700 | [diff] [blame] | 682 | if (t3c_tid && t3c_tid->ctx && t3c_tid->client->handlers && |
Divy Le Ray | 4d22de3 | 2007-01-18 22:04:14 -0500 | [diff] [blame] | 683 | t3c_tid->client->handlers[p->opcode]) { |
| 684 | return t3c_tid->client->handlers[p->opcode] |
| 685 | (dev, skb, t3c_tid->ctx); |
| 686 | } else { |
| 687 | printk(KERN_ERR "%s: received clientless CPL command 0x%x\n", |
| 688 | dev->name, p->opcode); |
| 689 | return CPL_RET_BUF_DONE | CPL_RET_BAD_MSG; |
| 690 | } |
| 691 | } |
| 692 | |
| 693 | static int do_cr(struct t3cdev *dev, struct sk_buff *skb) |
| 694 | { |
| 695 | struct cpl_pass_accept_req *req = cplhdr(skb); |
| 696 | unsigned int stid = G_PASS_OPEN_TID(ntohl(req->tos_tid)); |
Divy Le Ray | c9a6ce5 | 2007-08-21 20:49:26 -0700 | [diff] [blame] | 697 | struct tid_info *t = &(T3C_DATA(dev))->tid_maps; |
Divy Le Ray | 4d22de3 | 2007-01-18 22:04:14 -0500 | [diff] [blame] | 698 | struct t3c_tid_entry *t3c_tid; |
Divy Le Ray | c9a6ce5 | 2007-08-21 20:49:26 -0700 | [diff] [blame] | 699 | unsigned int tid = GET_TID(req); |
Divy Le Ray | 4d22de3 | 2007-01-18 22:04:14 -0500 | [diff] [blame] | 700 | |
Divy Le Ray | c9a6ce5 | 2007-08-21 20:49:26 -0700 | [diff] [blame] | 701 | if (unlikely(tid >= t->ntids)) { |
| 702 | printk("%s: passive open TID %u too large\n", |
| 703 | dev->name, tid); |
| 704 | t3_fatal_err(tdev2adap(dev)); |
| 705 | return CPL_RET_BUF_DONE; |
| 706 | } |
| 707 | |
| 708 | t3c_tid = lookup_stid(t, stid); |
| 709 | if (t3c_tid && t3c_tid->ctx && t3c_tid->client->handlers && |
Divy Le Ray | 4d22de3 | 2007-01-18 22:04:14 -0500 | [diff] [blame] | 710 | t3c_tid->client->handlers[CPL_PASS_ACCEPT_REQ]) { |
| 711 | return t3c_tid->client->handlers[CPL_PASS_ACCEPT_REQ] |
| 712 | (dev, skb, t3c_tid->ctx); |
| 713 | } else { |
| 714 | printk(KERN_ERR "%s: received clientless CPL command 0x%x\n", |
| 715 | dev->name, CPL_PASS_ACCEPT_REQ); |
| 716 | return CPL_RET_BUF_DONE | CPL_RET_BAD_MSG; |
| 717 | } |
| 718 | } |
| 719 | |
Divy Le Ray | 606fcd0 | 2007-04-17 11:06:30 -0700 | [diff] [blame] | 720 | /* |
| 721 | * Returns an sk_buff for a reply CPL message of size len. If the input |
| 722 | * sk_buff has no other users it is trimmed and reused, otherwise a new buffer |
| 723 | * is allocated. The input skb must be of size at least len. Note that this |
| 724 | * operation does not destroy the original skb data even if it decides to reuse |
| 725 | * the buffer. |
| 726 | */ |
| 727 | static struct sk_buff *cxgb3_get_cpl_reply_skb(struct sk_buff *skb, size_t len, |
Al Viro | 1f41bb3 | 2007-07-26 17:35:19 +0100 | [diff] [blame] | 728 | gfp_t gfp) |
Divy Le Ray | 606fcd0 | 2007-04-17 11:06:30 -0700 | [diff] [blame] | 729 | { |
| 730 | if (likely(!skb_cloned(skb))) { |
| 731 | BUG_ON(skb->len < len); |
| 732 | __skb_trim(skb, len); |
| 733 | skb_get(skb); |
| 734 | } else { |
| 735 | skb = alloc_skb(len, gfp); |
| 736 | if (skb) |
| 737 | __skb_put(skb, len); |
| 738 | } |
| 739 | return skb; |
| 740 | } |
| 741 | |
Divy Le Ray | 4d22de3 | 2007-01-18 22:04:14 -0500 | [diff] [blame] | 742 | static int do_abort_req_rss(struct t3cdev *dev, struct sk_buff *skb) |
| 743 | { |
| 744 | union opcode_tid *p = cplhdr(skb); |
| 745 | unsigned int hwtid = G_TID(ntohl(p->opcode_tid)); |
| 746 | struct t3c_tid_entry *t3c_tid; |
| 747 | |
| 748 | t3c_tid = lookup_tid(&(T3C_DATA(dev))->tid_maps, hwtid); |
Divy Le Ray | 606fcd0 | 2007-04-17 11:06:30 -0700 | [diff] [blame] | 749 | if (t3c_tid && t3c_tid->ctx && t3c_tid->client->handlers && |
Divy Le Ray | 4d22de3 | 2007-01-18 22:04:14 -0500 | [diff] [blame] | 750 | t3c_tid->client->handlers[p->opcode]) { |
| 751 | return t3c_tid->client->handlers[p->opcode] |
| 752 | (dev, skb, t3c_tid->ctx); |
| 753 | } else { |
| 754 | struct cpl_abort_req_rss *req = cplhdr(skb); |
| 755 | struct cpl_abort_rpl *rpl; |
Divy Le Ray | 606fcd0 | 2007-04-17 11:06:30 -0700 | [diff] [blame] | 756 | struct sk_buff *reply_skb; |
| 757 | unsigned int tid = GET_TID(req); |
| 758 | u8 cmd = req->status; |
Divy Le Ray | 4d22de3 | 2007-01-18 22:04:14 -0500 | [diff] [blame] | 759 | |
Divy Le Ray | 606fcd0 | 2007-04-17 11:06:30 -0700 | [diff] [blame] | 760 | if (req->status == CPL_ERR_RTX_NEG_ADVICE || |
| 761 | req->status == CPL_ERR_PERSIST_NEG_ADVICE) |
| 762 | goto out; |
| 763 | |
| 764 | reply_skb = cxgb3_get_cpl_reply_skb(skb, |
| 765 | sizeof(struct |
| 766 | cpl_abort_rpl), |
| 767 | GFP_ATOMIC); |
| 768 | |
| 769 | if (!reply_skb) { |
Divy Le Ray | 4d22de3 | 2007-01-18 22:04:14 -0500 | [diff] [blame] | 770 | printk("do_abort_req_rss: couldn't get skb!\n"); |
| 771 | goto out; |
| 772 | } |
Divy Le Ray | 606fcd0 | 2007-04-17 11:06:30 -0700 | [diff] [blame] | 773 | reply_skb->priority = CPL_PRIORITY_DATA; |
| 774 | __skb_put(reply_skb, sizeof(struct cpl_abort_rpl)); |
| 775 | rpl = cplhdr(reply_skb); |
Divy Le Ray | 4d22de3 | 2007-01-18 22:04:14 -0500 | [diff] [blame] | 776 | rpl->wr.wr_hi = |
| 777 | htonl(V_WR_OP(FW_WROPCODE_OFLD_HOST_ABORT_CON_RPL)); |
Divy Le Ray | 606fcd0 | 2007-04-17 11:06:30 -0700 | [diff] [blame] | 778 | rpl->wr.wr_lo = htonl(V_WR_TID(tid)); |
| 779 | OPCODE_TID(rpl) = htonl(MK_OPCODE_TID(CPL_ABORT_RPL, tid)); |
| 780 | rpl->cmd = cmd; |
| 781 | cxgb3_ofld_send(dev, reply_skb); |
Divy Le Ray | 4d22de3 | 2007-01-18 22:04:14 -0500 | [diff] [blame] | 782 | out: |
| 783 | return CPL_RET_BUF_DONE; |
| 784 | } |
| 785 | } |
| 786 | |
| 787 | static int do_act_establish(struct t3cdev *dev, struct sk_buff *skb) |
| 788 | { |
| 789 | struct cpl_act_establish *req = cplhdr(skb); |
| 790 | unsigned int atid = G_PASS_OPEN_TID(ntohl(req->tos_tid)); |
Divy Le Ray | c9a6ce5 | 2007-08-21 20:49:26 -0700 | [diff] [blame] | 791 | struct tid_info *t = &(T3C_DATA(dev))->tid_maps; |
Divy Le Ray | 4d22de3 | 2007-01-18 22:04:14 -0500 | [diff] [blame] | 792 | struct t3c_tid_entry *t3c_tid; |
Divy Le Ray | c9a6ce5 | 2007-08-21 20:49:26 -0700 | [diff] [blame] | 793 | unsigned int tid = GET_TID(req); |
Divy Le Ray | 4d22de3 | 2007-01-18 22:04:14 -0500 | [diff] [blame] | 794 | |
Divy Le Ray | c9a6ce5 | 2007-08-21 20:49:26 -0700 | [diff] [blame] | 795 | if (unlikely(tid >= t->ntids)) { |
| 796 | printk("%s: active establish TID %u too large\n", |
| 797 | dev->name, tid); |
| 798 | t3_fatal_err(tdev2adap(dev)); |
| 799 | return CPL_RET_BUF_DONE; |
| 800 | } |
| 801 | |
| 802 | t3c_tid = lookup_atid(t, atid); |
Divy Le Ray | 606fcd0 | 2007-04-17 11:06:30 -0700 | [diff] [blame] | 803 | if (t3c_tid && t3c_tid->ctx && t3c_tid->client->handlers && |
Divy Le Ray | 4d22de3 | 2007-01-18 22:04:14 -0500 | [diff] [blame] | 804 | t3c_tid->client->handlers[CPL_ACT_ESTABLISH]) { |
| 805 | return t3c_tid->client->handlers[CPL_ACT_ESTABLISH] |
| 806 | (dev, skb, t3c_tid->ctx); |
| 807 | } else { |
| 808 | printk(KERN_ERR "%s: received clientless CPL command 0x%x\n", |
Divy Le Ray | c9a6ce5 | 2007-08-21 20:49:26 -0700 | [diff] [blame] | 809 | dev->name, CPL_ACT_ESTABLISH); |
Divy Le Ray | 4d22de3 | 2007-01-18 22:04:14 -0500 | [diff] [blame] | 810 | return CPL_RET_BUF_DONE | CPL_RET_BAD_MSG; |
| 811 | } |
| 812 | } |
| 813 | |
Divy Le Ray | 4d22de3 | 2007-01-18 22:04:14 -0500 | [diff] [blame] | 814 | static int do_trace(struct t3cdev *dev, struct sk_buff *skb) |
| 815 | { |
| 816 | struct cpl_trace_pkt *p = cplhdr(skb); |
| 817 | |
Al Viro | b534497 | 2007-02-09 16:40:10 +0000 | [diff] [blame] | 818 | skb->protocol = htons(0xffff); |
Divy Le Ray | 4d22de3 | 2007-01-18 22:04:14 -0500 | [diff] [blame] | 819 | skb->dev = dev->lldev; |
| 820 | skb_pull(skb, sizeof(*p)); |
Arnaldo Carvalho de Melo | 459a98e | 2007-03-19 15:30:44 -0700 | [diff] [blame] | 821 | skb_reset_mac_header(skb); |
Divy Le Ray | 4d22de3 | 2007-01-18 22:04:14 -0500 | [diff] [blame] | 822 | netif_receive_skb(skb); |
| 823 | return 0; |
| 824 | } |
| 825 | |
| 826 | static int do_term(struct t3cdev *dev, struct sk_buff *skb) |
| 827 | { |
| 828 | unsigned int hwtid = ntohl(skb->priority) >> 8 & 0xfffff; |
| 829 | unsigned int opcode = G_OPCODE(ntohl(skb->csum)); |
| 830 | struct t3c_tid_entry *t3c_tid; |
| 831 | |
| 832 | t3c_tid = lookup_tid(&(T3C_DATA(dev))->tid_maps, hwtid); |
Divy Le Ray | 606fcd0 | 2007-04-17 11:06:30 -0700 | [diff] [blame] | 833 | if (t3c_tid && t3c_tid->ctx && t3c_tid->client->handlers && |
Divy Le Ray | 4d22de3 | 2007-01-18 22:04:14 -0500 | [diff] [blame] | 834 | t3c_tid->client->handlers[opcode]) { |
| 835 | return t3c_tid->client->handlers[opcode] (dev, skb, |
| 836 | t3c_tid->ctx); |
| 837 | } else { |
| 838 | printk(KERN_ERR "%s: received clientless CPL command 0x%x\n", |
| 839 | dev->name, opcode); |
| 840 | return CPL_RET_BUF_DONE | CPL_RET_BAD_MSG; |
| 841 | } |
| 842 | } |
| 843 | |
| 844 | static int nb_callback(struct notifier_block *self, unsigned long event, |
| 845 | void *ctx) |
| 846 | { |
| 847 | switch (event) { |
| 848 | case (NETEVENT_NEIGH_UPDATE):{ |
| 849 | cxgb_neigh_update((struct neighbour *)ctx); |
| 850 | break; |
| 851 | } |
| 852 | case (NETEVENT_PMTU_UPDATE): |
| 853 | break; |
| 854 | case (NETEVENT_REDIRECT):{ |
| 855 | struct netevent_redirect *nr = ctx; |
| 856 | cxgb_redirect(nr->old, nr->new); |
| 857 | cxgb_neigh_update(nr->new->neighbour); |
| 858 | break; |
| 859 | } |
| 860 | default: |
| 861 | break; |
| 862 | } |
| 863 | return 0; |
| 864 | } |
| 865 | |
| 866 | static struct notifier_block nb = { |
| 867 | .notifier_call = nb_callback |
| 868 | }; |
| 869 | |
| 870 | /* |
| 871 | * Process a received packet with an unknown/unexpected CPL opcode. |
| 872 | */ |
| 873 | static int do_bad_cpl(struct t3cdev *dev, struct sk_buff *skb) |
| 874 | { |
| 875 | printk(KERN_ERR "%s: received bad CPL command 0x%x\n", dev->name, |
| 876 | *skb->data); |
| 877 | return CPL_RET_BUF_DONE | CPL_RET_BAD_MSG; |
| 878 | } |
| 879 | |
| 880 | /* |
| 881 | * Handlers for each CPL opcode |
| 882 | */ |
| 883 | static cpl_handler_func cpl_handlers[NUM_CPL_CMDS]; |
| 884 | |
| 885 | /* |
| 886 | * Add a new handler to the CPL dispatch table. A NULL handler may be supplied |
| 887 | * to unregister an existing handler. |
| 888 | */ |
| 889 | void t3_register_cpl_handler(unsigned int opcode, cpl_handler_func h) |
| 890 | { |
| 891 | if (opcode < NUM_CPL_CMDS) |
| 892 | cpl_handlers[opcode] = h ? h : do_bad_cpl; |
| 893 | else |
| 894 | printk(KERN_ERR "T3C: handler registration for " |
| 895 | "opcode %x failed\n", opcode); |
| 896 | } |
| 897 | |
| 898 | EXPORT_SYMBOL(t3_register_cpl_handler); |
| 899 | |
| 900 | /* |
| 901 | * T3CDEV's receive method. |
| 902 | */ |
| 903 | int process_rx(struct t3cdev *dev, struct sk_buff **skbs, int n) |
| 904 | { |
| 905 | while (n--) { |
| 906 | struct sk_buff *skb = *skbs++; |
| 907 | unsigned int opcode = G_OPCODE(ntohl(skb->csum)); |
| 908 | int ret = cpl_handlers[opcode] (dev, skb); |
| 909 | |
| 910 | #if VALIDATE_TID |
| 911 | if (ret & CPL_RET_UNKNOWN_TID) { |
| 912 | union opcode_tid *p = cplhdr(skb); |
| 913 | |
| 914 | printk(KERN_ERR "%s: CPL message (opcode %u) had " |
| 915 | "unknown TID %u\n", dev->name, opcode, |
| 916 | G_TID(ntohl(p->opcode_tid))); |
| 917 | } |
| 918 | #endif |
| 919 | if (ret & CPL_RET_BUF_DONE) |
| 920 | kfree_skb(skb); |
| 921 | } |
| 922 | return 0; |
| 923 | } |
| 924 | |
| 925 | /* |
| 926 | * Sends an sk_buff to a T3C driver after dealing with any active network taps. |
| 927 | */ |
| 928 | int cxgb3_ofld_send(struct t3cdev *dev, struct sk_buff *skb) |
| 929 | { |
| 930 | int r; |
| 931 | |
| 932 | local_bh_disable(); |
| 933 | r = dev->send(dev, skb); |
| 934 | local_bh_enable(); |
| 935 | return r; |
| 936 | } |
| 937 | |
| 938 | EXPORT_SYMBOL(cxgb3_ofld_send); |
| 939 | |
| 940 | static int is_offloading(struct net_device *dev) |
| 941 | { |
| 942 | struct adapter *adapter; |
| 943 | int i; |
| 944 | |
| 945 | read_lock_bh(&adapter_list_lock); |
| 946 | list_for_each_entry(adapter, &adapter_list, adapter_list) { |
| 947 | for_each_port(adapter, i) { |
| 948 | if (dev == adapter->port[i]) { |
| 949 | read_unlock_bh(&adapter_list_lock); |
| 950 | return 1; |
| 951 | } |
| 952 | } |
| 953 | } |
| 954 | read_unlock_bh(&adapter_list_lock); |
| 955 | return 0; |
| 956 | } |
| 957 | |
| 958 | void cxgb_neigh_update(struct neighbour *neigh) |
| 959 | { |
| 960 | struct net_device *dev = neigh->dev; |
| 961 | |
| 962 | if (dev && (is_offloading(dev))) { |
Divy Le Ray | 5fbf816 | 2007-08-29 19:15:47 -0700 | [diff] [blame] | 963 | struct t3cdev *tdev = dev2t3cdev(dev); |
Divy Le Ray | 4d22de3 | 2007-01-18 22:04:14 -0500 | [diff] [blame] | 964 | |
| 965 | BUG_ON(!tdev); |
| 966 | t3_l2t_update(tdev, neigh); |
| 967 | } |
| 968 | } |
| 969 | |
| 970 | static void set_l2t_ix(struct t3cdev *tdev, u32 tid, struct l2t_entry *e) |
| 971 | { |
| 972 | struct sk_buff *skb; |
| 973 | struct cpl_set_tcb_field *req; |
| 974 | |
| 975 | skb = alloc_skb(sizeof(*req), GFP_ATOMIC); |
| 976 | if (!skb) { |
| 977 | printk(KERN_ERR "%s: cannot allocate skb!\n", __FUNCTION__); |
| 978 | return; |
| 979 | } |
| 980 | skb->priority = CPL_PRIORITY_CONTROL; |
| 981 | req = (struct cpl_set_tcb_field *)skb_put(skb, sizeof(*req)); |
| 982 | req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD)); |
| 983 | OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, tid)); |
| 984 | req->reply = 0; |
| 985 | req->cpu_idx = 0; |
| 986 | req->word = htons(W_TCB_L2T_IX); |
| 987 | req->mask = cpu_to_be64(V_TCB_L2T_IX(M_TCB_L2T_IX)); |
| 988 | req->val = cpu_to_be64(V_TCB_L2T_IX(e->idx)); |
| 989 | tdev->send(tdev, skb); |
| 990 | } |
| 991 | |
| 992 | void cxgb_redirect(struct dst_entry *old, struct dst_entry *new) |
| 993 | { |
| 994 | struct net_device *olddev, *newdev; |
| 995 | struct tid_info *ti; |
| 996 | struct t3cdev *tdev; |
| 997 | u32 tid; |
| 998 | int update_tcb; |
| 999 | struct l2t_entry *e; |
| 1000 | struct t3c_tid_entry *te; |
| 1001 | |
| 1002 | olddev = old->neighbour->dev; |
| 1003 | newdev = new->neighbour->dev; |
| 1004 | if (!is_offloading(olddev)) |
| 1005 | return; |
| 1006 | if (!is_offloading(newdev)) { |
Joe Perches | f07b2e4 | 2007-11-19 17:48:22 -0800 | [diff] [blame] | 1007 | printk(KERN_WARNING "%s: Redirect to non-offload " |
Divy Le Ray | 4d22de3 | 2007-01-18 22:04:14 -0500 | [diff] [blame] | 1008 | "device ignored.\n", __FUNCTION__); |
| 1009 | return; |
| 1010 | } |
Divy Le Ray | 5fbf816 | 2007-08-29 19:15:47 -0700 | [diff] [blame] | 1011 | tdev = dev2t3cdev(olddev); |
Divy Le Ray | 4d22de3 | 2007-01-18 22:04:14 -0500 | [diff] [blame] | 1012 | BUG_ON(!tdev); |
Divy Le Ray | 5fbf816 | 2007-08-29 19:15:47 -0700 | [diff] [blame] | 1013 | if (tdev != dev2t3cdev(newdev)) { |
Divy Le Ray | 4d22de3 | 2007-01-18 22:04:14 -0500 | [diff] [blame] | 1014 | printk(KERN_WARNING "%s: Redirect to different " |
| 1015 | "offload device ignored.\n", __FUNCTION__); |
| 1016 | return; |
| 1017 | } |
| 1018 | |
| 1019 | /* Add new L2T entry */ |
| 1020 | e = t3_l2t_get(tdev, new->neighbour, newdev); |
| 1021 | if (!e) { |
| 1022 | printk(KERN_ERR "%s: couldn't allocate new l2t entry!\n", |
| 1023 | __FUNCTION__); |
| 1024 | return; |
| 1025 | } |
| 1026 | |
| 1027 | /* Walk tid table and notify clients of dst change. */ |
| 1028 | ti = &(T3C_DATA(tdev))->tid_maps; |
| 1029 | for (tid = 0; tid < ti->ntids; tid++) { |
| 1030 | te = lookup_tid(ti, tid); |
| 1031 | BUG_ON(!te); |
Divy Le Ray | 606fcd0 | 2007-04-17 11:06:30 -0700 | [diff] [blame] | 1032 | if (te && te->ctx && te->client && te->client->redirect) { |
Divy Le Ray | 4d22de3 | 2007-01-18 22:04:14 -0500 | [diff] [blame] | 1033 | update_tcb = te->client->redirect(te->ctx, old, new, e); |
| 1034 | if (update_tcb) { |
| 1035 | l2t_hold(L2DATA(tdev), e); |
| 1036 | set_l2t_ix(tdev, tid, e); |
| 1037 | } |
| 1038 | } |
| 1039 | } |
| 1040 | l2t_release(L2DATA(tdev), e); |
| 1041 | } |
| 1042 | |
| 1043 | /* |
| 1044 | * Allocate a chunk of memory using kmalloc or, if that fails, vmalloc. |
| 1045 | * The allocated memory is cleared. |
| 1046 | */ |
| 1047 | void *cxgb_alloc_mem(unsigned long size) |
| 1048 | { |
| 1049 | void *p = kmalloc(size, GFP_KERNEL); |
| 1050 | |
| 1051 | if (!p) |
| 1052 | p = vmalloc(size); |
| 1053 | if (p) |
| 1054 | memset(p, 0, size); |
| 1055 | return p; |
| 1056 | } |
| 1057 | |
| 1058 | /* |
| 1059 | * Free memory allocated through t3_alloc_mem(). |
| 1060 | */ |
| 1061 | void cxgb_free_mem(void *addr) |
| 1062 | { |
| 1063 | unsigned long p = (unsigned long)addr; |
| 1064 | |
| 1065 | if (p >= VMALLOC_START && p < VMALLOC_END) |
| 1066 | vfree(addr); |
| 1067 | else |
| 1068 | kfree(addr); |
| 1069 | } |
| 1070 | |
| 1071 | /* |
| 1072 | * Allocate and initialize the TID tables. Returns 0 on success. |
| 1073 | */ |
| 1074 | static int init_tid_tabs(struct tid_info *t, unsigned int ntids, |
| 1075 | unsigned int natids, unsigned int nstids, |
| 1076 | unsigned int atid_base, unsigned int stid_base) |
| 1077 | { |
| 1078 | unsigned long size = ntids * sizeof(*t->tid_tab) + |
| 1079 | natids * sizeof(*t->atid_tab) + nstids * sizeof(*t->stid_tab); |
| 1080 | |
| 1081 | t->tid_tab = cxgb_alloc_mem(size); |
| 1082 | if (!t->tid_tab) |
| 1083 | return -ENOMEM; |
| 1084 | |
| 1085 | t->stid_tab = (union listen_entry *)&t->tid_tab[ntids]; |
| 1086 | t->atid_tab = (union active_open_entry *)&t->stid_tab[nstids]; |
| 1087 | t->ntids = ntids; |
| 1088 | t->nstids = nstids; |
| 1089 | t->stid_base = stid_base; |
| 1090 | t->sfree = NULL; |
| 1091 | t->natids = natids; |
| 1092 | t->atid_base = atid_base; |
| 1093 | t->afree = NULL; |
| 1094 | t->stids_in_use = t->atids_in_use = 0; |
| 1095 | atomic_set(&t->tids_in_use, 0); |
| 1096 | spin_lock_init(&t->stid_lock); |
| 1097 | spin_lock_init(&t->atid_lock); |
| 1098 | |
| 1099 | /* |
| 1100 | * Setup the free lists for stid_tab and atid_tab. |
| 1101 | */ |
| 1102 | if (nstids) { |
| 1103 | while (--nstids) |
| 1104 | t->stid_tab[nstids - 1].next = &t->stid_tab[nstids]; |
| 1105 | t->sfree = t->stid_tab; |
| 1106 | } |
| 1107 | if (natids) { |
| 1108 | while (--natids) |
| 1109 | t->atid_tab[natids - 1].next = &t->atid_tab[natids]; |
| 1110 | t->afree = t->atid_tab; |
| 1111 | } |
| 1112 | return 0; |
| 1113 | } |
| 1114 | |
| 1115 | static void free_tid_maps(struct tid_info *t) |
| 1116 | { |
| 1117 | cxgb_free_mem(t->tid_tab); |
| 1118 | } |
| 1119 | |
| 1120 | static inline void add_adapter(struct adapter *adap) |
| 1121 | { |
| 1122 | write_lock_bh(&adapter_list_lock); |
| 1123 | list_add_tail(&adap->adapter_list, &adapter_list); |
| 1124 | write_unlock_bh(&adapter_list_lock); |
| 1125 | } |
| 1126 | |
| 1127 | static inline void remove_adapter(struct adapter *adap) |
| 1128 | { |
| 1129 | write_lock_bh(&adapter_list_lock); |
| 1130 | list_del(&adap->adapter_list); |
| 1131 | write_unlock_bh(&adapter_list_lock); |
| 1132 | } |
| 1133 | |
| 1134 | int cxgb3_offload_activate(struct adapter *adapter) |
| 1135 | { |
| 1136 | struct t3cdev *dev = &adapter->tdev; |
| 1137 | int natids, err; |
| 1138 | struct t3c_data *t; |
| 1139 | struct tid_range stid_range, tid_range; |
| 1140 | struct mtutab mtutab; |
| 1141 | unsigned int l2t_capacity; |
| 1142 | |
| 1143 | t = kcalloc(1, sizeof(*t), GFP_KERNEL); |
| 1144 | if (!t) |
| 1145 | return -ENOMEM; |
| 1146 | |
| 1147 | err = -EOPNOTSUPP; |
| 1148 | if (dev->ctl(dev, GET_TX_MAX_CHUNK, &t->tx_max_chunk) < 0 || |
| 1149 | dev->ctl(dev, GET_MAX_OUTSTANDING_WR, &t->max_wrs) < 0 || |
| 1150 | dev->ctl(dev, GET_L2T_CAPACITY, &l2t_capacity) < 0 || |
| 1151 | dev->ctl(dev, GET_MTUS, &mtutab) < 0 || |
| 1152 | dev->ctl(dev, GET_TID_RANGE, &tid_range) < 0 || |
| 1153 | dev->ctl(dev, GET_STID_RANGE, &stid_range) < 0) |
| 1154 | goto out_free; |
| 1155 | |
| 1156 | err = -ENOMEM; |
| 1157 | L2DATA(dev) = t3_init_l2t(l2t_capacity); |
| 1158 | if (!L2DATA(dev)) |
| 1159 | goto out_free; |
| 1160 | |
| 1161 | natids = min(tid_range.num / 2, MAX_ATIDS); |
| 1162 | err = init_tid_tabs(&t->tid_maps, tid_range.num, natids, |
| 1163 | stid_range.num, ATID_BASE, stid_range.base); |
| 1164 | if (err) |
| 1165 | goto out_free_l2t; |
| 1166 | |
| 1167 | t->mtus = mtutab.mtus; |
| 1168 | t->nmtus = mtutab.size; |
| 1169 | |
| 1170 | INIT_WORK(&t->tid_release_task, t3_process_tid_release_list); |
| 1171 | spin_lock_init(&t->tid_release_lock); |
| 1172 | INIT_LIST_HEAD(&t->list_node); |
| 1173 | t->dev = dev; |
| 1174 | |
| 1175 | T3C_DATA(dev) = t; |
| 1176 | dev->recv = process_rx; |
| 1177 | dev->neigh_update = t3_l2t_update; |
| 1178 | |
| 1179 | /* Register netevent handler once */ |
| 1180 | if (list_empty(&adapter_list)) |
| 1181 | register_netevent_notifier(&nb); |
| 1182 | |
| 1183 | add_adapter(adapter); |
| 1184 | return 0; |
| 1185 | |
| 1186 | out_free_l2t: |
| 1187 | t3_free_l2t(L2DATA(dev)); |
| 1188 | L2DATA(dev) = NULL; |
| 1189 | out_free: |
| 1190 | kfree(t); |
| 1191 | return err; |
| 1192 | } |
| 1193 | |
| 1194 | void cxgb3_offload_deactivate(struct adapter *adapter) |
| 1195 | { |
| 1196 | struct t3cdev *tdev = &adapter->tdev; |
| 1197 | struct t3c_data *t = T3C_DATA(tdev); |
| 1198 | |
| 1199 | remove_adapter(adapter); |
| 1200 | if (list_empty(&adapter_list)) |
| 1201 | unregister_netevent_notifier(&nb); |
| 1202 | |
| 1203 | free_tid_maps(&t->tid_maps); |
| 1204 | T3C_DATA(tdev) = NULL; |
| 1205 | t3_free_l2t(L2DATA(tdev)); |
| 1206 | L2DATA(tdev) = NULL; |
| 1207 | kfree(t); |
| 1208 | } |
| 1209 | |
| 1210 | static inline void register_tdev(struct t3cdev *tdev) |
| 1211 | { |
| 1212 | static int unit; |
| 1213 | |
| 1214 | mutex_lock(&cxgb3_db_lock); |
| 1215 | snprintf(tdev->name, sizeof(tdev->name), "ofld_dev%d", unit++); |
| 1216 | list_add_tail(&tdev->ofld_dev_list, &ofld_dev_list); |
| 1217 | mutex_unlock(&cxgb3_db_lock); |
| 1218 | } |
| 1219 | |
| 1220 | static inline void unregister_tdev(struct t3cdev *tdev) |
| 1221 | { |
| 1222 | mutex_lock(&cxgb3_db_lock); |
| 1223 | list_del(&tdev->ofld_dev_list); |
| 1224 | mutex_unlock(&cxgb3_db_lock); |
| 1225 | } |
| 1226 | |
| 1227 | void __devinit cxgb3_adapter_ofld(struct adapter *adapter) |
| 1228 | { |
| 1229 | struct t3cdev *tdev = &adapter->tdev; |
| 1230 | |
| 1231 | INIT_LIST_HEAD(&tdev->ofld_dev_list); |
| 1232 | |
| 1233 | cxgb3_set_dummy_ops(tdev); |
| 1234 | tdev->send = t3_offload_tx; |
| 1235 | tdev->ctl = cxgb_offload_ctl; |
| 1236 | tdev->type = adapter->params.rev == 0 ? T3A : T3B; |
| 1237 | |
| 1238 | register_tdev(tdev); |
| 1239 | } |
| 1240 | |
| 1241 | void __devexit cxgb3_adapter_unofld(struct adapter *adapter) |
| 1242 | { |
| 1243 | struct t3cdev *tdev = &adapter->tdev; |
| 1244 | |
| 1245 | tdev->recv = NULL; |
| 1246 | tdev->neigh_update = NULL; |
| 1247 | |
| 1248 | unregister_tdev(tdev); |
| 1249 | } |
| 1250 | |
| 1251 | void __init cxgb3_offload_init(void) |
| 1252 | { |
| 1253 | int i; |
| 1254 | |
| 1255 | for (i = 0; i < NUM_CPL_CMDS; ++i) |
| 1256 | cpl_handlers[i] = do_bad_cpl; |
| 1257 | |
| 1258 | t3_register_cpl_handler(CPL_SMT_WRITE_RPL, do_smt_write_rpl); |
| 1259 | t3_register_cpl_handler(CPL_L2T_WRITE_RPL, do_l2t_write_rpl); |
| 1260 | t3_register_cpl_handler(CPL_PASS_OPEN_RPL, do_stid_rpl); |
| 1261 | t3_register_cpl_handler(CPL_CLOSE_LISTSRV_RPL, do_stid_rpl); |
| 1262 | t3_register_cpl_handler(CPL_PASS_ACCEPT_REQ, do_cr); |
| 1263 | t3_register_cpl_handler(CPL_PASS_ESTABLISH, do_hwtid_rpl); |
| 1264 | t3_register_cpl_handler(CPL_ABORT_RPL_RSS, do_hwtid_rpl); |
| 1265 | t3_register_cpl_handler(CPL_ABORT_RPL, do_hwtid_rpl); |
| 1266 | t3_register_cpl_handler(CPL_RX_URG_NOTIFY, do_hwtid_rpl); |
| 1267 | t3_register_cpl_handler(CPL_RX_DATA, do_hwtid_rpl); |
| 1268 | t3_register_cpl_handler(CPL_TX_DATA_ACK, do_hwtid_rpl); |
| 1269 | t3_register_cpl_handler(CPL_TX_DMA_ACK, do_hwtid_rpl); |
| 1270 | t3_register_cpl_handler(CPL_ACT_OPEN_RPL, do_act_open_rpl); |
| 1271 | t3_register_cpl_handler(CPL_PEER_CLOSE, do_hwtid_rpl); |
| 1272 | t3_register_cpl_handler(CPL_CLOSE_CON_RPL, do_hwtid_rpl); |
| 1273 | t3_register_cpl_handler(CPL_ABORT_REQ_RSS, do_abort_req_rss); |
| 1274 | t3_register_cpl_handler(CPL_ACT_ESTABLISH, do_act_establish); |
Divy Le Ray | 6cdbd77 | 2007-04-09 20:10:33 -0700 | [diff] [blame] | 1275 | t3_register_cpl_handler(CPL_SET_TCB_RPL, do_hwtid_rpl); |
| 1276 | t3_register_cpl_handler(CPL_GET_TCB_RPL, do_hwtid_rpl); |
Divy Le Ray | 4d22de3 | 2007-01-18 22:04:14 -0500 | [diff] [blame] | 1277 | t3_register_cpl_handler(CPL_RDMA_TERMINATE, do_term); |
| 1278 | t3_register_cpl_handler(CPL_RDMA_EC_STATUS, do_hwtid_rpl); |
| 1279 | t3_register_cpl_handler(CPL_TRACE_PKT, do_trace); |
| 1280 | t3_register_cpl_handler(CPL_RX_DATA_DDP, do_hwtid_rpl); |
| 1281 | t3_register_cpl_handler(CPL_RX_DDP_COMPLETE, do_hwtid_rpl); |
| 1282 | t3_register_cpl_handler(CPL_ISCSI_HDR, do_hwtid_rpl); |
| 1283 | } |