| /* |
| * Copyright(c) 2015, 2016 Intel Corporation. |
| * |
| * This file is provided under a dual BSD/GPLv2 license. When using or |
| * redistributing this file, you may do so under either license. |
| * |
| * GPL LICENSE SUMMARY |
| * |
| * This program is free software; you can redistribute it and/or modify |
| * it under the terms of version 2 of the GNU General Public License as |
| * published by the Free Software Foundation. |
| * |
| * This program is distributed in the hope that it will be useful, but |
| * WITHOUT ANY WARRANTY; without even the implied warranty of |
| * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU |
| * General Public License for more details. |
| * |
| * BSD LICENSE |
| * |
| * Redistribution and use in source and binary forms, with or without |
| * modification, are permitted provided that the following conditions |
| * are met: |
| * |
| * - Redistributions of source code must retain the above copyright |
| * notice, this list of conditions and the following disclaimer. |
| * - Redistributions in binary form must reproduce the above copyright |
| * notice, this list of conditions and the following disclaimer in |
| * the documentation and/or other materials provided with the |
| * distribution. |
| * - Neither the name of Intel Corporation nor the names of its |
| * contributors may be used to endorse or promote products derived |
| * from this software without specific prior written permission. |
| * |
| * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS |
| * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT |
| * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR |
| * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT |
| * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
| * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
| * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
| * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
| * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
| * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
| * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
| * |
| */ |
| |
| #include <linux/err.h> |
| #include <linux/vmalloc.h> |
| #include <linux/hash.h> |
| #include <linux/module.h> |
| #include <linux/seq_file.h> |
| #include <rdma/rdma_vt.h> |
| #include <rdma/rdmavt_qp.h> |
| |
| #include "hfi.h" |
| #include "qp.h" |
| #include "trace.h" |
| #include "verbs_txreq.h" |
| |
| unsigned int hfi1_qp_table_size = 256; |
| module_param_named(qp_table_size, hfi1_qp_table_size, uint, S_IRUGO); |
| MODULE_PARM_DESC(qp_table_size, "QP table size"); |
| |
| static void flush_tx_list(struct rvt_qp *qp); |
| static int iowait_sleep( |
| struct sdma_engine *sde, |
| struct iowait *wait, |
| struct sdma_txreq *stx, |
| unsigned seq); |
| static void iowait_wakeup(struct iowait *wait, int reason); |
| static void iowait_sdma_drained(struct iowait *wait); |
| static void qp_pio_drain(struct rvt_qp *qp); |
| |
| static inline unsigned mk_qpn(struct rvt_qpn_table *qpt, |
| struct rvt_qpn_map *map, unsigned off) |
| { |
| return (map - qpt->map) * RVT_BITS_PER_PAGE + off; |
| } |
| |
| /* |
| * Convert the AETH credit code into the number of credits. |
| */ |
| static const u16 credit_table[31] = { |
| 0, /* 0 */ |
| 1, /* 1 */ |
| 2, /* 2 */ |
| 3, /* 3 */ |
| 4, /* 4 */ |
| 6, /* 5 */ |
| 8, /* 6 */ |
| 12, /* 7 */ |
| 16, /* 8 */ |
| 24, /* 9 */ |
| 32, /* A */ |
| 48, /* B */ |
| 64, /* C */ |
| 96, /* D */ |
| 128, /* E */ |
| 192, /* F */ |
| 256, /* 10 */ |
| 384, /* 11 */ |
| 512, /* 12 */ |
| 768, /* 13 */ |
| 1024, /* 14 */ |
| 1536, /* 15 */ |
| 2048, /* 16 */ |
| 3072, /* 17 */ |
| 4096, /* 18 */ |
| 6144, /* 19 */ |
| 8192, /* 1A */ |
| 12288, /* 1B */ |
| 16384, /* 1C */ |
| 24576, /* 1D */ |
| 32768 /* 1E */ |
| }; |
| |
| static void flush_tx_list(struct rvt_qp *qp) |
| { |
| struct hfi1_qp_priv *priv = qp->priv; |
| |
| while (!list_empty(&priv->s_iowait.tx_head)) { |
| struct sdma_txreq *tx; |
| |
| tx = list_first_entry( |
| &priv->s_iowait.tx_head, |
| struct sdma_txreq, |
| list); |
| list_del_init(&tx->list); |
| hfi1_put_txreq( |
| container_of(tx, struct verbs_txreq, txreq)); |
| } |
| } |
| |
| static void flush_iowait(struct rvt_qp *qp) |
| { |
| struct hfi1_qp_priv *priv = qp->priv; |
| struct hfi1_ibdev *dev = to_idev(qp->ibqp.device); |
| unsigned long flags; |
| |
| write_seqlock_irqsave(&dev->iowait_lock, flags); |
| if (!list_empty(&priv->s_iowait.list)) { |
| list_del_init(&priv->s_iowait.list); |
| if (atomic_dec_and_test(&qp->refcount)) |
| wake_up(&qp->wait); |
| } |
| write_sequnlock_irqrestore(&dev->iowait_lock, flags); |
| } |
| |
| static inline int opa_mtu_enum_to_int(int mtu) |
| { |
| switch (mtu) { |
| case OPA_MTU_8192: return 8192; |
| case OPA_MTU_10240: return 10240; |
| default: return -1; |
| } |
| } |
| |
| /** |
| * This function is what we would push to the core layer if we wanted to be a |
| * "first class citizen". Instead we hide this here and rely on Verbs ULPs |
| * to blindly pass the MTU enum value from the PathRecord to us. |
| * |
| * The actual flag used to determine "8k MTU" will change and is currently |
| * unknown. |
| */ |
| static inline int verbs_mtu_enum_to_int(struct ib_device *dev, enum ib_mtu mtu) |
| { |
| int val; |
| |
| /* Constraining 10KB packets to 8KB packets */ |
| if (mtu == (enum ib_mtu)OPA_MTU_10240) |
| mtu = OPA_MTU_8192; |
| val = opa_mtu_enum_to_int((int)mtu); |
| if (val > 0) |
| return val; |
| return ib_mtu_enum_to_int(mtu); |
| } |
| |
| int hfi1_check_modify_qp(struct rvt_qp *qp, struct ib_qp_attr *attr, |
| int attr_mask, struct ib_udata *udata) |
| { |
| struct ib_qp *ibqp = &qp->ibqp; |
| struct hfi1_ibdev *dev = to_idev(ibqp->device); |
| struct hfi1_devdata *dd = dd_from_dev(dev); |
| u8 sc; |
| |
| if (attr_mask & IB_QP_AV) { |
| sc = ah_to_sc(ibqp->device, &attr->ah_attr); |
| if (sc == 0xf) |
| return -EINVAL; |
| |
| if (!qp_to_sdma_engine(qp, sc) && |
| dd->flags & HFI1_HAS_SEND_DMA) |
| return -EINVAL; |
| |
| if (!qp_to_send_context(qp, sc)) |
| return -EINVAL; |
| } |
| |
| if (attr_mask & IB_QP_ALT_PATH) { |
| sc = ah_to_sc(ibqp->device, &attr->alt_ah_attr); |
| if (sc == 0xf) |
| return -EINVAL; |
| |
| if (!qp_to_sdma_engine(qp, sc) && |
| dd->flags & HFI1_HAS_SEND_DMA) |
| return -EINVAL; |
| |
| if (!qp_to_send_context(qp, sc)) |
| return -EINVAL; |
| } |
| |
| return 0; |
| } |
| |
| void hfi1_modify_qp(struct rvt_qp *qp, struct ib_qp_attr *attr, |
| int attr_mask, struct ib_udata *udata) |
| { |
| struct ib_qp *ibqp = &qp->ibqp; |
| struct hfi1_qp_priv *priv = qp->priv; |
| |
| if (attr_mask & IB_QP_AV) { |
| priv->s_sc = ah_to_sc(ibqp->device, &qp->remote_ah_attr); |
| priv->s_sde = qp_to_sdma_engine(qp, priv->s_sc); |
| priv->s_sendcontext = qp_to_send_context(qp, priv->s_sc); |
| } |
| |
| if (attr_mask & IB_QP_PATH_MIG_STATE && |
| attr->path_mig_state == IB_MIG_MIGRATED && |
| qp->s_mig_state == IB_MIG_ARMED) { |
| qp->s_flags |= RVT_S_AHG_CLEAR; |
| priv->s_sc = ah_to_sc(ibqp->device, &qp->remote_ah_attr); |
| priv->s_sde = qp_to_sdma_engine(qp, priv->s_sc); |
| priv->s_sendcontext = qp_to_send_context(qp, priv->s_sc); |
| } |
| } |
| |
| /** |
| * hfi1_check_send_wqe - validate wqe |
| * @qp - The qp |
| * @wqe - The built wqe |
| * |
| * validate wqe. This is called |
| * prior to inserting the wqe into |
| * the ring but after the wqe has been |
| * setup. |
| * |
| * Returns 0 on success, -EINVAL on failure |
| * |
| */ |
| int hfi1_check_send_wqe(struct rvt_qp *qp, |
| struct rvt_swqe *wqe) |
| { |
| struct hfi1_ibport *ibp = to_iport(qp->ibqp.device, qp->port_num); |
| struct rvt_ah *ah; |
| |
| switch (qp->ibqp.qp_type) { |
| case IB_QPT_RC: |
| case IB_QPT_UC: |
| if (wqe->length > 0x80000000U) |
| return -EINVAL; |
| break; |
| case IB_QPT_SMI: |
| ah = ibah_to_rvtah(wqe->ud_wr.ah); |
| if (wqe->length > (1 << ah->log_pmtu)) |
| return -EINVAL; |
| break; |
| case IB_QPT_GSI: |
| case IB_QPT_UD: |
| ah = ibah_to_rvtah(wqe->ud_wr.ah); |
| if (wqe->length > (1 << ah->log_pmtu)) |
| return -EINVAL; |
| if (ibp->sl_to_sc[ah->attr.sl] == 0xf) |
| return -EINVAL; |
| default: |
| break; |
| } |
| return wqe->length <= piothreshold; |
| } |
| |
| /** |
| * hfi1_compute_aeth - compute the AETH (syndrome + MSN) |
| * @qp: the queue pair to compute the AETH for |
| * |
| * Returns the AETH. |
| */ |
| __be32 hfi1_compute_aeth(struct rvt_qp *qp) |
| { |
| u32 aeth = qp->r_msn & HFI1_MSN_MASK; |
| |
| if (qp->ibqp.srq) { |
| /* |
| * Shared receive queues don't generate credits. |
| * Set the credit field to the invalid value. |
| */ |
| aeth |= HFI1_AETH_CREDIT_INVAL << HFI1_AETH_CREDIT_SHIFT; |
| } else { |
| u32 min, max, x; |
| u32 credits; |
| struct rvt_rwq *wq = qp->r_rq.wq; |
| u32 head; |
| u32 tail; |
| |
| /* sanity check pointers before trusting them */ |
| head = wq->head; |
| if (head >= qp->r_rq.size) |
| head = 0; |
| tail = wq->tail; |
| if (tail >= qp->r_rq.size) |
| tail = 0; |
| /* |
| * Compute the number of credits available (RWQEs). |
| * There is a small chance that the pair of reads are |
| * not atomic, which is OK, since the fuzziness is |
| * resolved as further ACKs go out. |
| */ |
| credits = head - tail; |
| if ((int)credits < 0) |
| credits += qp->r_rq.size; |
| /* |
| * Binary search the credit table to find the code to |
| * use. |
| */ |
| min = 0; |
| max = 31; |
| for (;;) { |
| x = (min + max) / 2; |
| if (credit_table[x] == credits) |
| break; |
| if (credit_table[x] > credits) { |
| max = x; |
| } else { |
| if (min == x) |
| break; |
| min = x; |
| } |
| } |
| aeth |= x << HFI1_AETH_CREDIT_SHIFT; |
| } |
| return cpu_to_be32(aeth); |
| } |
| |
| /** |
| * _hfi1_schedule_send - schedule progress |
| * @qp: the QP |
| * |
| * This schedules qp progress w/o regard to the s_flags. |
| * |
| * It is only used in the post send, which doesn't hold |
| * the s_lock. |
| */ |
| void _hfi1_schedule_send(struct rvt_qp *qp) |
| { |
| struct hfi1_qp_priv *priv = qp->priv; |
| struct hfi1_ibport *ibp = |
| to_iport(qp->ibqp.device, qp->port_num); |
| struct hfi1_pportdata *ppd = ppd_from_ibp(ibp); |
| struct hfi1_devdata *dd = dd_from_ibdev(qp->ibqp.device); |
| |
| iowait_schedule(&priv->s_iowait, ppd->hfi1_wq, |
| priv->s_sde ? |
| priv->s_sde->cpu : |
| cpumask_first(cpumask_of_node(dd->node))); |
| } |
| |
| static void qp_pio_drain(struct rvt_qp *qp) |
| { |
| struct hfi1_ibdev *dev; |
| struct hfi1_qp_priv *priv = qp->priv; |
| |
| if (!priv->s_sendcontext) |
| return; |
| dev = to_idev(qp->ibqp.device); |
| while (iowait_pio_pending(&priv->s_iowait)) { |
| write_seqlock_irq(&dev->iowait_lock); |
| hfi1_sc_wantpiobuf_intr(priv->s_sendcontext, 1); |
| write_sequnlock_irq(&dev->iowait_lock); |
| iowait_pio_drain(&priv->s_iowait); |
| write_seqlock_irq(&dev->iowait_lock); |
| hfi1_sc_wantpiobuf_intr(priv->s_sendcontext, 0); |
| write_sequnlock_irq(&dev->iowait_lock); |
| } |
| } |
| |
| /** |
| * hfi1_schedule_send - schedule progress |
| * @qp: the QP |
| * |
| * This schedules qp progress and caller should hold |
| * the s_lock. |
| */ |
| void hfi1_schedule_send(struct rvt_qp *qp) |
| { |
| if (hfi1_send_ok(qp)) |
| _hfi1_schedule_send(qp); |
| } |
| |
| /** |
| * hfi1_get_credit - flush the send work queue of a QP |
| * @qp: the qp who's send work queue to flush |
| * @aeth: the Acknowledge Extended Transport Header |
| * |
| * The QP s_lock should be held. |
| */ |
| void hfi1_get_credit(struct rvt_qp *qp, u32 aeth) |
| { |
| u32 credit = (aeth >> HFI1_AETH_CREDIT_SHIFT) & HFI1_AETH_CREDIT_MASK; |
| |
| /* |
| * If the credit is invalid, we can send |
| * as many packets as we like. Otherwise, we have to |
| * honor the credit field. |
| */ |
| if (credit == HFI1_AETH_CREDIT_INVAL) { |
| if (!(qp->s_flags & RVT_S_UNLIMITED_CREDIT)) { |
| qp->s_flags |= RVT_S_UNLIMITED_CREDIT; |
| if (qp->s_flags & RVT_S_WAIT_SSN_CREDIT) { |
| qp->s_flags &= ~RVT_S_WAIT_SSN_CREDIT; |
| hfi1_schedule_send(qp); |
| } |
| } |
| } else if (!(qp->s_flags & RVT_S_UNLIMITED_CREDIT)) { |
| /* Compute new LSN (i.e., MSN + credit) */ |
| credit = (aeth + credit_table[credit]) & HFI1_MSN_MASK; |
| if (cmp_msn(credit, qp->s_lsn) > 0) { |
| qp->s_lsn = credit; |
| if (qp->s_flags & RVT_S_WAIT_SSN_CREDIT) { |
| qp->s_flags &= ~RVT_S_WAIT_SSN_CREDIT; |
| hfi1_schedule_send(qp); |
| } |
| } |
| } |
| } |
| |
| void hfi1_qp_wakeup(struct rvt_qp *qp, u32 flag) |
| { |
| unsigned long flags; |
| |
| spin_lock_irqsave(&qp->s_lock, flags); |
| if (qp->s_flags & flag) { |
| qp->s_flags &= ~flag; |
| trace_hfi1_qpwakeup(qp, flag); |
| hfi1_schedule_send(qp); |
| } |
| spin_unlock_irqrestore(&qp->s_lock, flags); |
| /* Notify hfi1_destroy_qp() if it is waiting. */ |
| if (atomic_dec_and_test(&qp->refcount)) |
| wake_up(&qp->wait); |
| } |
| |
| static int iowait_sleep( |
| struct sdma_engine *sde, |
| struct iowait *wait, |
| struct sdma_txreq *stx, |
| unsigned seq) |
| { |
| struct verbs_txreq *tx = container_of(stx, struct verbs_txreq, txreq); |
| struct rvt_qp *qp; |
| struct hfi1_qp_priv *priv; |
| unsigned long flags; |
| int ret = 0; |
| struct hfi1_ibdev *dev; |
| |
| qp = tx->qp; |
| priv = qp->priv; |
| |
| spin_lock_irqsave(&qp->s_lock, flags); |
| if (ib_rvt_state_ops[qp->state] & RVT_PROCESS_RECV_OK) { |
| /* |
| * If we couldn't queue the DMA request, save the info |
| * and try again later rather than destroying the |
| * buffer and undoing the side effects of the copy. |
| */ |
| /* Make a common routine? */ |
| dev = &sde->dd->verbs_dev; |
| list_add_tail(&stx->list, &wait->tx_head); |
| write_seqlock(&dev->iowait_lock); |
| if (sdma_progress(sde, seq, stx)) |
| goto eagain; |
| if (list_empty(&priv->s_iowait.list)) { |
| struct hfi1_ibport *ibp = |
| to_iport(qp->ibqp.device, qp->port_num); |
| |
| ibp->rvp.n_dmawait++; |
| qp->s_flags |= RVT_S_WAIT_DMA_DESC; |
| list_add_tail(&priv->s_iowait.list, &sde->dmawait); |
| trace_hfi1_qpsleep(qp, RVT_S_WAIT_DMA_DESC); |
| atomic_inc(&qp->refcount); |
| } |
| write_sequnlock(&dev->iowait_lock); |
| qp->s_flags &= ~RVT_S_BUSY; |
| spin_unlock_irqrestore(&qp->s_lock, flags); |
| ret = -EBUSY; |
| } else { |
| spin_unlock_irqrestore(&qp->s_lock, flags); |
| hfi1_put_txreq(tx); |
| } |
| return ret; |
| eagain: |
| write_sequnlock(&dev->iowait_lock); |
| spin_unlock_irqrestore(&qp->s_lock, flags); |
| list_del_init(&stx->list); |
| return -EAGAIN; |
| } |
| |
| static void iowait_wakeup(struct iowait *wait, int reason) |
| { |
| struct rvt_qp *qp = iowait_to_qp(wait); |
| |
| WARN_ON(reason != SDMA_AVAIL_REASON); |
| hfi1_qp_wakeup(qp, RVT_S_WAIT_DMA_DESC); |
| } |
| |
| static void iowait_sdma_drained(struct iowait *wait) |
| { |
| struct rvt_qp *qp = iowait_to_qp(wait); |
| |
| /* |
| * This happens when the send engine notes |
| * a QP in the error state and cannot |
| * do the flush work until that QP's |
| * sdma work has finished. |
| */ |
| spin_lock(&qp->s_lock); |
| if (qp->s_flags & RVT_S_WAIT_DMA) { |
| qp->s_flags &= ~RVT_S_WAIT_DMA; |
| hfi1_schedule_send(qp); |
| } |
| spin_unlock(&qp->s_lock); |
| } |
| |
| /** |
| * |
| * qp_to_sdma_engine - map a qp to a send engine |
| * @qp: the QP |
| * @sc5: the 5 bit sc |
| * |
| * Return: |
| * A send engine for the qp or NULL for SMI type qp. |
| */ |
| struct sdma_engine *qp_to_sdma_engine(struct rvt_qp *qp, u8 sc5) |
| { |
| struct hfi1_devdata *dd = dd_from_ibdev(qp->ibqp.device); |
| struct sdma_engine *sde; |
| |
| if (!(dd->flags & HFI1_HAS_SEND_DMA)) |
| return NULL; |
| switch (qp->ibqp.qp_type) { |
| case IB_QPT_SMI: |
| return NULL; |
| default: |
| break; |
| } |
| sde = sdma_select_engine_sc(dd, qp->ibqp.qp_num >> dd->qos_shift, sc5); |
| return sde; |
| } |
| |
| /* |
| * qp_to_send_context - map a qp to a send context |
| * @qp: the QP |
| * @sc5: the 5 bit sc |
| * |
| * Return: |
| * A send context for the qp |
| */ |
| struct send_context *qp_to_send_context(struct rvt_qp *qp, u8 sc5) |
| { |
| struct hfi1_devdata *dd = dd_from_ibdev(qp->ibqp.device); |
| |
| switch (qp->ibqp.qp_type) { |
| case IB_QPT_SMI: |
| /* SMA packets to VL15 */ |
| return dd->vld[15].sc; |
| default: |
| break; |
| } |
| |
| return pio_select_send_context_sc(dd, qp->ibqp.qp_num >> dd->qos_shift, |
| sc5); |
| } |
| |
| struct qp_iter { |
| struct hfi1_ibdev *dev; |
| struct rvt_qp *qp; |
| int specials; |
| int n; |
| }; |
| |
| struct qp_iter *qp_iter_init(struct hfi1_ibdev *dev) |
| { |
| struct qp_iter *iter; |
| |
| iter = kzalloc(sizeof(*iter), GFP_KERNEL); |
| if (!iter) |
| return NULL; |
| |
| iter->dev = dev; |
| iter->specials = dev->rdi.ibdev.phys_port_cnt * 2; |
| if (qp_iter_next(iter)) { |
| kfree(iter); |
| return NULL; |
| } |
| |
| return iter; |
| } |
| |
| int qp_iter_next(struct qp_iter *iter) |
| { |
| struct hfi1_ibdev *dev = iter->dev; |
| int n = iter->n; |
| int ret = 1; |
| struct rvt_qp *pqp = iter->qp; |
| struct rvt_qp *qp; |
| |
| /* |
| * The approach is to consider the special qps |
| * as an additional table entries before the |
| * real hash table. Since the qp code sets |
| * the qp->next hash link to NULL, this works just fine. |
| * |
| * iter->specials is 2 * # ports |
| * |
| * n = 0..iter->specials is the special qp indices |
| * |
| * n = iter->specials..dev->rdi.qp_dev->qp_table_size+iter->specials are |
| * the potential hash bucket entries |
| * |
| */ |
| for (; n < dev->rdi.qp_dev->qp_table_size + iter->specials; n++) { |
| if (pqp) { |
| qp = rcu_dereference(pqp->next); |
| } else { |
| if (n < iter->specials) { |
| struct hfi1_pportdata *ppd; |
| struct hfi1_ibport *ibp; |
| int pidx; |
| |
| pidx = n % dev->rdi.ibdev.phys_port_cnt; |
| ppd = &dd_from_dev(dev)->pport[pidx]; |
| ibp = &ppd->ibport_data; |
| |
| if (!(n & 1)) |
| qp = rcu_dereference(ibp->rvp.qp[0]); |
| else |
| qp = rcu_dereference(ibp->rvp.qp[1]); |
| } else { |
| qp = rcu_dereference( |
| dev->rdi.qp_dev->qp_table[ |
| (n - iter->specials)]); |
| } |
| } |
| pqp = qp; |
| if (qp) { |
| iter->qp = qp; |
| iter->n = n; |
| return 0; |
| } |
| } |
| return ret; |
| } |
| |
| static const char * const qp_type_str[] = { |
| "SMI", "GSI", "RC", "UC", "UD", |
| }; |
| |
| static int qp_idle(struct rvt_qp *qp) |
| { |
| return |
| qp->s_last == qp->s_acked && |
| qp->s_acked == qp->s_cur && |
| qp->s_cur == qp->s_tail && |
| qp->s_tail == qp->s_head; |
| } |
| |
| void qp_iter_print(struct seq_file *s, struct qp_iter *iter) |
| { |
| struct rvt_swqe *wqe; |
| struct rvt_qp *qp = iter->qp; |
| struct hfi1_qp_priv *priv = qp->priv; |
| struct sdma_engine *sde; |
| struct send_context *send_context; |
| |
| sde = qp_to_sdma_engine(qp, priv->s_sc); |
| wqe = rvt_get_swqe_ptr(qp, qp->s_last); |
| send_context = qp_to_send_context(qp, priv->s_sc); |
| seq_printf(s, |
| "N %d %s QP %x R %u %s %u %u %u f=%x %u %u %u %u %u %u PSN %x %x %x %x %x (%u %u %u %u %u %u %u) RQP %x LID %x SL %u MTU %u %u %u %u SDE %p,%u SC %p,%u SCQ %u %u PID %d\n", |
| iter->n, |
| qp_idle(qp) ? "I" : "B", |
| qp->ibqp.qp_num, |
| atomic_read(&qp->refcount), |
| qp_type_str[qp->ibqp.qp_type], |
| qp->state, |
| wqe ? wqe->wr.opcode : 0, |
| qp->s_hdrwords, |
| qp->s_flags, |
| iowait_sdma_pending(&priv->s_iowait), |
| iowait_pio_pending(&priv->s_iowait), |
| !list_empty(&priv->s_iowait.list), |
| qp->timeout, |
| wqe ? wqe->ssn : 0, |
| qp->s_lsn, |
| qp->s_last_psn, |
| qp->s_psn, qp->s_next_psn, |
| qp->s_sending_psn, qp->s_sending_hpsn, |
| qp->s_last, qp->s_acked, qp->s_cur, |
| qp->s_tail, qp->s_head, qp->s_size, |
| qp->s_avail, |
| qp->remote_qpn, |
| qp->remote_ah_attr.dlid, |
| qp->remote_ah_attr.sl, |
| qp->pmtu, |
| qp->s_retry, |
| qp->s_retry_cnt, |
| qp->s_rnr_retry_cnt, |
| sde, |
| sde ? sde->this_idx : 0, |
| send_context, |
| send_context ? send_context->sw_index : 0, |
| ibcq_to_rvtcq(qp->ibqp.send_cq)->queue->head, |
| ibcq_to_rvtcq(qp->ibqp.send_cq)->queue->tail, |
| qp->pid); |
| } |
| |
| void qp_comm_est(struct rvt_qp *qp) |
| { |
| qp->r_flags |= RVT_R_COMM_EST; |
| if (qp->ibqp.event_handler) { |
| struct ib_event ev; |
| |
| ev.device = qp->ibqp.device; |
| ev.element.qp = &qp->ibqp; |
| ev.event = IB_EVENT_COMM_EST; |
| qp->ibqp.event_handler(&ev, qp->ibqp.qp_context); |
| } |
| } |
| |
| void *qp_priv_alloc(struct rvt_dev_info *rdi, struct rvt_qp *qp, |
| gfp_t gfp) |
| { |
| struct hfi1_qp_priv *priv; |
| |
| priv = kzalloc_node(sizeof(*priv), gfp, rdi->dparms.node); |
| if (!priv) |
| return ERR_PTR(-ENOMEM); |
| |
| priv->owner = qp; |
| |
| priv->s_hdr = kzalloc_node(sizeof(*priv->s_hdr), gfp, rdi->dparms.node); |
| if (!priv->s_hdr) { |
| kfree(priv); |
| return ERR_PTR(-ENOMEM); |
| } |
| setup_timer(&priv->s_rnr_timer, hfi1_rc_rnr_retry, (unsigned long)qp); |
| qp->s_timer.function = hfi1_rc_timeout; |
| return priv; |
| } |
| |
| void qp_priv_free(struct rvt_dev_info *rdi, struct rvt_qp *qp) |
| { |
| struct hfi1_qp_priv *priv = qp->priv; |
| |
| kfree(priv->s_hdr); |
| kfree(priv); |
| } |
| |
| unsigned free_all_qps(struct rvt_dev_info *rdi) |
| { |
| struct hfi1_ibdev *verbs_dev = container_of(rdi, |
| struct hfi1_ibdev, |
| rdi); |
| struct hfi1_devdata *dd = container_of(verbs_dev, |
| struct hfi1_devdata, |
| verbs_dev); |
| int n; |
| unsigned qp_inuse = 0; |
| |
| for (n = 0; n < dd->num_pports; n++) { |
| struct hfi1_ibport *ibp = &dd->pport[n].ibport_data; |
| |
| rcu_read_lock(); |
| if (rcu_dereference(ibp->rvp.qp[0])) |
| qp_inuse++; |
| if (rcu_dereference(ibp->rvp.qp[1])) |
| qp_inuse++; |
| rcu_read_unlock(); |
| } |
| |
| return qp_inuse; |
| } |
| |
| void flush_qp_waiters(struct rvt_qp *qp) |
| { |
| flush_iowait(qp); |
| hfi1_stop_rc_timers(qp); |
| } |
| |
| void stop_send_queue(struct rvt_qp *qp) |
| { |
| struct hfi1_qp_priv *priv = qp->priv; |
| |
| cancel_work_sync(&priv->s_iowait.iowork); |
| hfi1_del_timers_sync(qp); |
| } |
| |
| void quiesce_qp(struct rvt_qp *qp) |
| { |
| struct hfi1_qp_priv *priv = qp->priv; |
| |
| iowait_sdma_drain(&priv->s_iowait); |
| qp_pio_drain(qp); |
| flush_tx_list(qp); |
| } |
| |
| void notify_qp_reset(struct rvt_qp *qp) |
| { |
| struct hfi1_qp_priv *priv = qp->priv; |
| |
| iowait_init( |
| &priv->s_iowait, |
| 1, |
| _hfi1_do_send, |
| iowait_sleep, |
| iowait_wakeup, |
| iowait_sdma_drained); |
| priv->r_adefered = 0; |
| clear_ahg(qp); |
| } |
| |
| /* |
| * Switch to alternate path. |
| * The QP s_lock should be held and interrupts disabled. |
| */ |
| void hfi1_migrate_qp(struct rvt_qp *qp) |
| { |
| struct hfi1_qp_priv *priv = qp->priv; |
| struct ib_event ev; |
| |
| qp->s_mig_state = IB_MIG_MIGRATED; |
| qp->remote_ah_attr = qp->alt_ah_attr; |
| qp->port_num = qp->alt_ah_attr.port_num; |
| qp->s_pkey_index = qp->s_alt_pkey_index; |
| qp->s_flags |= RVT_S_AHG_CLEAR; |
| priv->s_sc = ah_to_sc(qp->ibqp.device, &qp->remote_ah_attr); |
| priv->s_sde = qp_to_sdma_engine(qp, priv->s_sc); |
| |
| ev.device = qp->ibqp.device; |
| ev.element.qp = &qp->ibqp; |
| ev.event = IB_EVENT_PATH_MIG; |
| qp->ibqp.event_handler(&ev, qp->ibqp.qp_context); |
| } |
| |
| int mtu_to_path_mtu(u32 mtu) |
| { |
| return mtu_to_enum(mtu, OPA_MTU_8192); |
| } |
| |
| u32 mtu_from_qp(struct rvt_dev_info *rdi, struct rvt_qp *qp, u32 pmtu) |
| { |
| u32 mtu; |
| struct hfi1_ibdev *verbs_dev = container_of(rdi, |
| struct hfi1_ibdev, |
| rdi); |
| struct hfi1_devdata *dd = container_of(verbs_dev, |
| struct hfi1_devdata, |
| verbs_dev); |
| struct hfi1_ibport *ibp; |
| u8 sc, vl; |
| |
| ibp = &dd->pport[qp->port_num - 1].ibport_data; |
| sc = ibp->sl_to_sc[qp->remote_ah_attr.sl]; |
| vl = sc_to_vlt(dd, sc); |
| |
| mtu = verbs_mtu_enum_to_int(qp->ibqp.device, pmtu); |
| if (vl < PER_VL_SEND_CONTEXTS) |
| mtu = min_t(u32, mtu, dd->vld[vl].mtu); |
| return mtu; |
| } |
| |
| int get_pmtu_from_attr(struct rvt_dev_info *rdi, struct rvt_qp *qp, |
| struct ib_qp_attr *attr) |
| { |
| int mtu, pidx = qp->port_num - 1; |
| struct hfi1_ibdev *verbs_dev = container_of(rdi, |
| struct hfi1_ibdev, |
| rdi); |
| struct hfi1_devdata *dd = container_of(verbs_dev, |
| struct hfi1_devdata, |
| verbs_dev); |
| mtu = verbs_mtu_enum_to_int(qp->ibqp.device, attr->path_mtu); |
| if (mtu == -1) |
| return -1; /* values less than 0 are error */ |
| |
| if (mtu > dd->pport[pidx].ibmtu) |
| return mtu_to_enum(dd->pport[pidx].ibmtu, IB_MTU_2048); |
| else |
| return attr->path_mtu; |
| } |
| |
| void notify_error_qp(struct rvt_qp *qp) |
| { |
| struct hfi1_ibdev *dev = to_idev(qp->ibqp.device); |
| struct hfi1_qp_priv *priv = qp->priv; |
| |
| write_seqlock(&dev->iowait_lock); |
| if (!list_empty(&priv->s_iowait.list) && !(qp->s_flags & RVT_S_BUSY)) { |
| qp->s_flags &= ~RVT_S_ANY_WAIT_IO; |
| list_del_init(&priv->s_iowait.list); |
| if (atomic_dec_and_test(&qp->refcount)) |
| wake_up(&qp->wait); |
| } |
| write_sequnlock(&dev->iowait_lock); |
| |
| if (!(qp->s_flags & RVT_S_BUSY)) { |
| qp->s_hdrwords = 0; |
| if (qp->s_rdma_mr) { |
| rvt_put_mr(qp->s_rdma_mr); |
| qp->s_rdma_mr = NULL; |
| } |
| flush_tx_list(qp); |
| } |
| } |
| |
| /** |
| * hfi1_error_port_qps - put a port's RC/UC qps into error state |
| * @ibp: the ibport. |
| * @sl: the service level. |
| * |
| * This function places all RC/UC qps with a given service level into error |
| * state. It is generally called to force upper lay apps to abandon stale qps |
| * after an sl->sc mapping change. |
| */ |
| void hfi1_error_port_qps(struct hfi1_ibport *ibp, u8 sl) |
| { |
| struct rvt_qp *qp = NULL; |
| struct hfi1_pportdata *ppd = ppd_from_ibp(ibp); |
| struct hfi1_ibdev *dev = &ppd->dd->verbs_dev; |
| int n; |
| int lastwqe; |
| struct ib_event ev; |
| |
| rcu_read_lock(); |
| |
| /* Deal only with RC/UC qps that use the given SL. */ |
| for (n = 0; n < dev->rdi.qp_dev->qp_table_size; n++) { |
| for (qp = rcu_dereference(dev->rdi.qp_dev->qp_table[n]); qp; |
| qp = rcu_dereference(qp->next)) { |
| if (qp->port_num == ppd->port && |
| (qp->ibqp.qp_type == IB_QPT_UC || |
| qp->ibqp.qp_type == IB_QPT_RC) && |
| qp->remote_ah_attr.sl == sl && |
| (ib_rvt_state_ops[qp->state] & |
| RVT_POST_SEND_OK)) { |
| spin_lock_irq(&qp->r_lock); |
| spin_lock(&qp->s_hlock); |
| spin_lock(&qp->s_lock); |
| lastwqe = rvt_error_qp(qp, |
| IB_WC_WR_FLUSH_ERR); |
| spin_unlock(&qp->s_lock); |
| spin_unlock(&qp->s_hlock); |
| spin_unlock_irq(&qp->r_lock); |
| if (lastwqe) { |
| ev.device = qp->ibqp.device; |
| ev.element.qp = &qp->ibqp; |
| ev.event = |
| IB_EVENT_QP_LAST_WQE_REACHED; |
| qp->ibqp.event_handler(&ev, |
| qp->ibqp.qp_context); |
| } |
| } |
| } |
| } |
| |
| rcu_read_unlock(); |
| } |