Amir Vadai | e586b3b | 2015-05-28 22:28:46 +0300 | [diff] [blame] | 1 | /* |
| 2 | * Copyright (c) 2015, Mellanox Technologies. All rights reserved. |
| 3 | * |
| 4 | * This software is available to you under a choice of one of two |
| 5 | * licenses. You may choose to be licensed under the terms of the GNU |
| 6 | * General Public License (GPL) Version 2, available from the file |
| 7 | * COPYING in the main directory of this source tree, or the |
| 8 | * OpenIB.org BSD license below: |
| 9 | * |
| 10 | * Redistribution and use in source and binary forms, with or |
| 11 | * without modification, are permitted provided that the following |
| 12 | * conditions are met: |
| 13 | * |
| 14 | * - Redistributions of source code must retain the above |
| 15 | * copyright notice, this list of conditions and the following |
| 16 | * disclaimer. |
| 17 | * |
| 18 | * - Redistributions in binary form must reproduce the above |
| 19 | * copyright notice, this list of conditions and the following |
| 20 | * disclaimer in the documentation and/or other materials |
| 21 | * provided with the distribution. |
| 22 | * |
| 23 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, |
| 24 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF |
| 25 | * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND |
| 26 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS |
| 27 | * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN |
| 28 | * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN |
| 29 | * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE |
| 30 | * SOFTWARE. |
| 31 | */ |
| 32 | |
| 33 | #include "en.h" |
| 34 | |
| 35 | struct mlx5_cqe64 *mlx5e_get_cqe(struct mlx5e_cq *cq) |
| 36 | { |
| 37 | struct mlx5_cqwq *wq = &cq->wq; |
| 38 | u32 ci = mlx5_cqwq_get_ci(wq); |
| 39 | struct mlx5_cqe64 *cqe = mlx5_cqwq_get_wqe(wq, ci); |
| 40 | int cqe_ownership_bit = cqe->op_own & MLX5_CQE_OWNER_MASK; |
| 41 | int sw_ownership_val = mlx5_cqwq_get_wrap_cnt(wq) & 1; |
| 42 | |
| 43 | if (cqe_ownership_bit != sw_ownership_val) |
| 44 | return NULL; |
| 45 | |
Amir Vadai | e586b3b | 2015-05-28 22:28:46 +0300 | [diff] [blame] | 46 | /* ensure cqe content is read after cqe ownership bit */ |
| 47 | rmb(); |
| 48 | |
| 49 | return cqe; |
| 50 | } |
| 51 | |
Tariq Toukan | d3c9bc2 | 2016-04-20 22:02:14 +0300 | [diff] [blame] | 52 | static void mlx5e_poll_ico_cq(struct mlx5e_cq *cq) |
| 53 | { |
Saeed Mahameed | 6e8dd6d | 2016-08-29 01:13:45 +0300 | [diff] [blame] | 54 | struct mlx5e_sq *sq = container_of(cq, struct mlx5e_sq, cq); |
Tariq Toukan | d3c9bc2 | 2016-04-20 22:02:14 +0300 | [diff] [blame] | 55 | struct mlx5_wq_cyc *wq; |
| 56 | struct mlx5_cqe64 *cqe; |
Tariq Toukan | d3c9bc2 | 2016-04-20 22:02:14 +0300 | [diff] [blame] | 57 | u16 sqcc; |
| 58 | |
Saeed Mahameed | 6e8dd6d | 2016-08-29 01:13:45 +0300 | [diff] [blame] | 59 | if (unlikely(test_bit(MLX5E_SQ_STATE_FLUSH, &sq->state))) |
| 60 | return; |
| 61 | |
Tariq Toukan | d3c9bc2 | 2016-04-20 22:02:14 +0300 | [diff] [blame] | 62 | cqe = mlx5e_get_cqe(cq); |
| 63 | if (likely(!cqe)) |
| 64 | return; |
| 65 | |
Tariq Toukan | d3c9bc2 | 2016-04-20 22:02:14 +0300 | [diff] [blame] | 66 | wq = &sq->wq; |
| 67 | |
| 68 | /* sq->cc must be updated only after mlx5_cqwq_update_db_record(), |
| 69 | * otherwise a cq overrun may occur |
| 70 | */ |
| 71 | sqcc = sq->cc; |
| 72 | |
| 73 | do { |
| 74 | u16 ci = be16_to_cpu(cqe->wqe_counter) & wq->sz_m1; |
| 75 | struct mlx5e_ico_wqe_info *icowi = &sq->ico_wqe_info[ci]; |
| 76 | |
| 77 | mlx5_cqwq_pop(&cq->wq); |
| 78 | sqcc += icowi->num_wqebbs; |
| 79 | |
| 80 | if (unlikely((cqe->op_own >> 4) != MLX5_CQE_REQ)) { |
| 81 | WARN_ONCE(true, "mlx5e: Bad OP in ICOSQ CQE: 0x%x\n", |
| 82 | cqe->op_own); |
| 83 | break; |
| 84 | } |
| 85 | |
| 86 | switch (icowi->opcode) { |
| 87 | case MLX5_OPCODE_NOP: |
| 88 | break; |
Tariq Toukan | bc77b24 | 2016-04-20 22:02:15 +0300 | [diff] [blame] | 89 | case MLX5_OPCODE_UMR: |
| 90 | mlx5e_post_rx_fragmented_mpwqe(&sq->channel->rq); |
| 91 | break; |
Tariq Toukan | d3c9bc2 | 2016-04-20 22:02:14 +0300 | [diff] [blame] | 92 | default: |
| 93 | WARN_ONCE(true, |
| 94 | "mlx5e: Bad OPCODE in ICOSQ WQE info: 0x%x\n", |
| 95 | icowi->opcode); |
| 96 | } |
| 97 | |
| 98 | } while ((cqe = mlx5e_get_cqe(cq))); |
| 99 | |
| 100 | mlx5_cqwq_update_db_record(&cq->wq); |
| 101 | |
| 102 | /* ensure cq space is freed before enabling more cqes */ |
| 103 | wmb(); |
| 104 | |
| 105 | sq->cc = sqcc; |
| 106 | } |
| 107 | |
Amir Vadai | e586b3b | 2015-05-28 22:28:46 +0300 | [diff] [blame] | 108 | int mlx5e_napi_poll(struct napi_struct *napi, int budget) |
| 109 | { |
| 110 | struct mlx5e_channel *c = container_of(napi, struct mlx5e_channel, |
| 111 | napi); |
| 112 | bool busy = false; |
Eric Dumazet | 44fb6fb | 2015-11-18 06:30:56 -0800 | [diff] [blame] | 113 | int work_done; |
Amir Vadai | e586b3b | 2015-05-28 22:28:46 +0300 | [diff] [blame] | 114 | int i; |
| 115 | |
| 116 | clear_bit(MLX5E_CHANNEL_NAPI_SCHED, &c->flags); |
| 117 | |
| 118 | for (i = 0; i < c->num_tc; i++) |
Jesper Dangaard Brouer | 8ec736e | 2016-03-11 09:44:17 +0100 | [diff] [blame] | 119 | busy |= mlx5e_poll_tx_cq(&c->sq[i].cq, budget); |
Amir Vadai | e586b3b | 2015-05-28 22:28:46 +0300 | [diff] [blame] | 120 | |
Eric Dumazet | 44fb6fb | 2015-11-18 06:30:56 -0800 | [diff] [blame] | 121 | work_done = mlx5e_poll_rx_cq(&c->rq.cq, budget); |
| 122 | busy |= work_done == budget; |
Tariq Toukan | d3c9bc2 | 2016-04-20 22:02:14 +0300 | [diff] [blame] | 123 | |
| 124 | mlx5e_poll_ico_cq(&c->icosq.cq); |
| 125 | |
Achiad Shochat | e339105 | 2015-06-23 17:14:20 +0300 | [diff] [blame] | 126 | busy |= mlx5e_post_rx_wqes(&c->rq); |
Amir Vadai | e586b3b | 2015-05-28 22:28:46 +0300 | [diff] [blame] | 127 | |
| 128 | if (busy) |
| 129 | return budget; |
| 130 | |
Eric Dumazet | 44fb6fb | 2015-11-18 06:30:56 -0800 | [diff] [blame] | 131 | napi_complete_done(napi, work_done); |
Amir Vadai | e586b3b | 2015-05-28 22:28:46 +0300 | [diff] [blame] | 132 | |
| 133 | /* avoid losing completion event during/after polling cqs */ |
| 134 | if (test_bit(MLX5E_CHANNEL_NAPI_SCHED, &c->flags)) { |
| 135 | napi_schedule(napi); |
Eric Dumazet | 44fb6fb | 2015-11-18 06:30:56 -0800 | [diff] [blame] | 136 | return work_done; |
Amir Vadai | e586b3b | 2015-05-28 22:28:46 +0300 | [diff] [blame] | 137 | } |
| 138 | |
| 139 | for (i = 0; i < c->num_tc; i++) |
| 140 | mlx5e_cq_arm(&c->sq[i].cq); |
Gil Rockah | cb3c7fd | 2016-06-23 17:02:41 +0300 | [diff] [blame] | 141 | |
| 142 | if (test_bit(MLX5E_RQ_STATE_AM, &c->rq.state)) |
| 143 | mlx5e_rx_am(&c->rq); |
| 144 | |
Amir Vadai | e586b3b | 2015-05-28 22:28:46 +0300 | [diff] [blame] | 145 | mlx5e_cq_arm(&c->rq.cq); |
Tariq Toukan | d3c9bc2 | 2016-04-20 22:02:14 +0300 | [diff] [blame] | 146 | mlx5e_cq_arm(&c->icosq.cq); |
Amir Vadai | e586b3b | 2015-05-28 22:28:46 +0300 | [diff] [blame] | 147 | |
Eric Dumazet | 44fb6fb | 2015-11-18 06:30:56 -0800 | [diff] [blame] | 148 | return work_done; |
Amir Vadai | e586b3b | 2015-05-28 22:28:46 +0300 | [diff] [blame] | 149 | } |
| 150 | |
| 151 | void mlx5e_completion_event(struct mlx5_core_cq *mcq) |
| 152 | { |
| 153 | struct mlx5e_cq *cq = container_of(mcq, struct mlx5e_cq, mcq); |
| 154 | |
Gil Rockah | cb3c7fd | 2016-06-23 17:02:41 +0300 | [diff] [blame] | 155 | cq->event_ctr++; |
Amir Vadai | e586b3b | 2015-05-28 22:28:46 +0300 | [diff] [blame] | 156 | set_bit(MLX5E_CHANNEL_NAPI_SCHED, &cq->channel->flags); |
Amir Vadai | e586b3b | 2015-05-28 22:28:46 +0300 | [diff] [blame] | 157 | napi_schedule(cq->napi); |
| 158 | } |
| 159 | |
| 160 | void mlx5e_cq_error_event(struct mlx5_core_cq *mcq, enum mlx5_event event) |
| 161 | { |
| 162 | struct mlx5e_cq *cq = container_of(mcq, struct mlx5e_cq, mcq); |
| 163 | struct mlx5e_channel *c = cq->channel; |
| 164 | struct mlx5e_priv *priv = c->priv; |
| 165 | struct net_device *netdev = priv->netdev; |
| 166 | |
| 167 | netdev_err(netdev, "%s: cqn=0x%.6x event=0x%.2x\n", |
| 168 | __func__, mcq->cqn, event); |
| 169 | } |