Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1 | /* |
| 2 | * This file is provided under a dual BSD/GPLv2 license. When using or |
| 3 | * redistributing this file, you may do so under either license. |
| 4 | * |
| 5 | * GPL LICENSE SUMMARY |
| 6 | * |
| 7 | * Copyright(c) 2012 Intel Corporation. All rights reserved. |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 8 | * Copyright (C) 2015 EMC Corporation. All Rights Reserved. |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 9 | * |
| 10 | * This program is free software; you can redistribute it and/or modify |
| 11 | * it under the terms of version 2 of the GNU General Public License as |
| 12 | * published by the Free Software Foundation. |
| 13 | * |
| 14 | * BSD LICENSE |
| 15 | * |
| 16 | * Copyright(c) 2012 Intel Corporation. All rights reserved. |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 17 | * Copyright (C) 2015 EMC Corporation. All Rights Reserved. |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 18 | * |
| 19 | * Redistribution and use in source and binary forms, with or without |
| 20 | * modification, are permitted provided that the following conditions |
| 21 | * are met: |
| 22 | * |
| 23 | * * Redistributions of source code must retain the above copyright |
| 24 | * notice, this list of conditions and the following disclaimer. |
| 25 | * * Redistributions in binary form must reproduce the above copy |
| 26 | * notice, this list of conditions and the following disclaimer in |
| 27 | * the documentation and/or other materials provided with the |
| 28 | * distribution. |
| 29 | * * Neither the name of Intel Corporation nor the names of its |
| 30 | * contributors may be used to endorse or promote products derived |
| 31 | * from this software without specific prior written permission. |
| 32 | * |
| 33 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS |
| 34 | * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT |
| 35 | * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR |
| 36 | * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT |
| 37 | * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
| 38 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
| 39 | * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
| 40 | * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
| 41 | * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
| 42 | * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
| 43 | * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
| 44 | * |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 45 | * PCIe NTB Transport Linux driver |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 46 | * |
| 47 | * Contact Information: |
| 48 | * Jon Mason <jon.mason@intel.com> |
| 49 | */ |
| 50 | #include <linux/debugfs.h> |
| 51 | #include <linux/delay.h> |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 52 | #include <linux/dmaengine.h> |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 53 | #include <linux/dma-mapping.h> |
| 54 | #include <linux/errno.h> |
| 55 | #include <linux/export.h> |
| 56 | #include <linux/interrupt.h> |
| 57 | #include <linux/module.h> |
| 58 | #include <linux/pci.h> |
| 59 | #include <linux/slab.h> |
| 60 | #include <linux/types.h> |
Dave Jiang | 06917f7 | 2015-05-19 16:45:46 -0400 | [diff] [blame] | 61 | #include <linux/uaccess.h> |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 62 | #include "linux/ntb.h" |
| 63 | #include "linux/ntb_transport.h" |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 64 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 65 | #define NTB_TRANSPORT_VERSION 4 |
| 66 | #define NTB_TRANSPORT_VER "4" |
| 67 | #define NTB_TRANSPORT_NAME "ntb_transport" |
| 68 | #define NTB_TRANSPORT_DESC "Software Queue-Pair Transport over NTB" |
Shyam Sundar S K | b17faba | 2016-12-07 22:37:05 +0530 | [diff] [blame] | 69 | #define NTB_TRANSPORT_MIN_SPADS (MW0_SZ_HIGH + 2) |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 70 | |
| 71 | MODULE_DESCRIPTION(NTB_TRANSPORT_DESC); |
| 72 | MODULE_VERSION(NTB_TRANSPORT_VER); |
| 73 | MODULE_LICENSE("Dual BSD/GPL"); |
| 74 | MODULE_AUTHOR("Intel Corporation"); |
| 75 | |
| 76 | static unsigned long max_mw_size; |
| 77 | module_param(max_mw_size, ulong, 0644); |
| 78 | MODULE_PARM_DESC(max_mw_size, "Limit size of large memory windows"); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 79 | |
Dave Jiang | 9891417 | 2015-06-03 11:29:38 -0400 | [diff] [blame] | 80 | static unsigned int transport_mtu = 0x10000; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 81 | module_param(transport_mtu, uint, 0644); |
| 82 | MODULE_PARM_DESC(transport_mtu, "Maximum size of NTB transport packets"); |
| 83 | |
Jon Mason | 948d3a6 | 2013-04-18 17:07:36 -0700 | [diff] [blame] | 84 | static unsigned char max_num_clients; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 85 | module_param(max_num_clients, byte, 0644); |
| 86 | MODULE_PARM_DESC(max_num_clients, "Maximum number of NTB transport clients"); |
| 87 | |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 88 | static unsigned int copy_bytes = 1024; |
| 89 | module_param(copy_bytes, uint, 0644); |
| 90 | MODULE_PARM_DESC(copy_bytes, "Threshold under which NTB will use the CPU to copy instead of DMA"); |
| 91 | |
Dave Jiang | a41ef05 | 2015-05-19 16:52:04 -0400 | [diff] [blame] | 92 | static bool use_dma; |
| 93 | module_param(use_dma, bool, 0644); |
| 94 | MODULE_PARM_DESC(use_dma, "Use DMA engine to perform large data copy"); |
| 95 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 96 | static struct dentry *nt_debugfs_dir; |
| 97 | |
Serge Semin | 1e53011 | 2016-12-14 02:49:14 +0300 | [diff] [blame] | 98 | /* Only two-ports NTB devices are supported */ |
| 99 | #define PIDX NTB_DEF_PEER_IDX |
| 100 | |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 101 | struct ntb_queue_entry { |
| 102 | /* ntb_queue list reference */ |
| 103 | struct list_head entry; |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 104 | /* pointers to data to be transferred */ |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 105 | void *cb_data; |
| 106 | void *buf; |
| 107 | unsigned int len; |
| 108 | unsigned int flags; |
Dave Jiang | 9cabc26 | 2016-07-20 13:14:07 -0700 | [diff] [blame] | 109 | int retries; |
| 110 | int errors; |
| 111 | unsigned int tx_index; |
Dave Jiang | 7220357 | 2016-07-20 13:14:13 -0700 | [diff] [blame] | 112 | unsigned int rx_index; |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 113 | |
| 114 | struct ntb_transport_qp *qp; |
| 115 | union { |
| 116 | struct ntb_payload_header __iomem *tx_hdr; |
| 117 | struct ntb_payload_header *rx_hdr; |
| 118 | }; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 119 | }; |
| 120 | |
Jon Mason | 793c20e | 2013-01-19 02:02:26 -0700 | [diff] [blame] | 121 | struct ntb_rx_info { |
| 122 | unsigned int entry; |
| 123 | }; |
| 124 | |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 125 | struct ntb_transport_qp { |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 126 | struct ntb_transport_ctx *transport; |
| 127 | struct ntb_dev *ndev; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 128 | void *cb_data; |
Dave Jiang | 569410ca | 2015-07-13 08:07:22 -0400 | [diff] [blame] | 129 | struct dma_chan *tx_dma_chan; |
| 130 | struct dma_chan *rx_dma_chan; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 131 | |
| 132 | bool client_ready; |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 133 | bool link_is_up; |
Dave Jiang | e902133 | 2016-02-23 09:11:36 -0700 | [diff] [blame] | 134 | bool active; |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 135 | |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 136 | u8 qp_num; /* Only 64 QP's are allowed. 0-63 */ |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 137 | u64 qp_bit; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 138 | |
Jon Mason | 7446564 | 2013-01-21 15:28:52 -0700 | [diff] [blame] | 139 | struct ntb_rx_info __iomem *rx_info; |
Jon Mason | 793c20e | 2013-01-19 02:02:26 -0700 | [diff] [blame] | 140 | struct ntb_rx_info *remote_rx_info; |
| 141 | |
Jon Mason | 53ca4fe | 2013-11-26 11:21:50 -0700 | [diff] [blame] | 142 | void (*tx_handler)(struct ntb_transport_qp *qp, void *qp_data, |
| 143 | void *data, int len); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 144 | struct list_head tx_free_q; |
| 145 | spinlock_t ntb_tx_free_q_lock; |
Jon Mason | 7446564 | 2013-01-21 15:28:52 -0700 | [diff] [blame] | 146 | void __iomem *tx_mw; |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 147 | dma_addr_t tx_mw_phys; |
Jon Mason | 793c20e | 2013-01-19 02:02:26 -0700 | [diff] [blame] | 148 | unsigned int tx_index; |
| 149 | unsigned int tx_max_entry; |
Jon Mason | ef114ed | 2013-01-19 02:02:18 -0700 | [diff] [blame] | 150 | unsigned int tx_max_frame; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 151 | |
Jon Mason | 53ca4fe | 2013-11-26 11:21:50 -0700 | [diff] [blame] | 152 | void (*rx_handler)(struct ntb_transport_qp *qp, void *qp_data, |
| 153 | void *data, int len); |
Allen Hubbe | da2e5ae | 2015-07-13 08:07:08 -0400 | [diff] [blame] | 154 | struct list_head rx_post_q; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 155 | struct list_head rx_pend_q; |
| 156 | struct list_head rx_free_q; |
Allen Hubbe | da2e5ae | 2015-07-13 08:07:08 -0400 | [diff] [blame] | 157 | /* ntb_rx_q_lock: synchronize access to rx_XXXX_q */ |
| 158 | spinlock_t ntb_rx_q_lock; |
Jon Mason | 793c20e | 2013-01-19 02:02:26 -0700 | [diff] [blame] | 159 | void *rx_buff; |
| 160 | unsigned int rx_index; |
| 161 | unsigned int rx_max_entry; |
Jon Mason | ef114ed | 2013-01-19 02:02:18 -0700 | [diff] [blame] | 162 | unsigned int rx_max_frame; |
Dave Jiang | a754a8f | 2016-04-08 10:49:06 -0700 | [diff] [blame] | 163 | unsigned int rx_alloc_entry; |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 164 | dma_cookie_t last_cookie; |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 165 | struct tasklet_struct rxc_db_work; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 166 | |
Jon Mason | 53ca4fe | 2013-11-26 11:21:50 -0700 | [diff] [blame] | 167 | void (*event_handler)(void *data, int status); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 168 | struct delayed_work link_work; |
Jon Mason | 7b4f2d3 | 2013-01-19 02:02:19 -0700 | [diff] [blame] | 169 | struct work_struct link_cleanup; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 170 | |
| 171 | struct dentry *debugfs_dir; |
| 172 | struct dentry *debugfs_stats; |
| 173 | |
| 174 | /* Stats */ |
| 175 | u64 rx_bytes; |
| 176 | u64 rx_pkts; |
| 177 | u64 rx_ring_empty; |
| 178 | u64 rx_err_no_buf; |
| 179 | u64 rx_err_oflow; |
| 180 | u64 rx_err_ver; |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 181 | u64 rx_memcpy; |
| 182 | u64 rx_async; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 183 | u64 tx_bytes; |
| 184 | u64 tx_pkts; |
| 185 | u64 tx_ring_full; |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 186 | u64 tx_err_no_buf; |
| 187 | u64 tx_memcpy; |
| 188 | u64 tx_async; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 189 | }; |
| 190 | |
| 191 | struct ntb_transport_mw { |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 192 | phys_addr_t phys_addr; |
| 193 | resource_size_t phys_size; |
| 194 | resource_size_t xlat_align; |
| 195 | resource_size_t xlat_align_size; |
| 196 | void __iomem *vbase; |
| 197 | size_t xlat_size; |
| 198 | size_t buff_size; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 199 | void *virt_addr; |
| 200 | dma_addr_t dma_addr; |
| 201 | }; |
| 202 | |
| 203 | struct ntb_transport_client_dev { |
| 204 | struct list_head entry; |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 205 | struct ntb_transport_ctx *nt; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 206 | struct device dev; |
| 207 | }; |
| 208 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 209 | struct ntb_transport_ctx { |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 210 | struct list_head entry; |
| 211 | struct list_head client_devs; |
| 212 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 213 | struct ntb_dev *ndev; |
| 214 | |
| 215 | struct ntb_transport_mw *mw_vec; |
| 216 | struct ntb_transport_qp *qp_vec; |
| 217 | unsigned int mw_count; |
| 218 | unsigned int qp_count; |
| 219 | u64 qp_bitmap; |
| 220 | u64 qp_bitmap_free; |
| 221 | |
| 222 | bool link_is_up; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 223 | struct delayed_work link_work; |
Jon Mason | 7b4f2d3 | 2013-01-19 02:02:19 -0700 | [diff] [blame] | 224 | struct work_struct link_cleanup; |
Dave Jiang | c8650fd | 2015-07-13 08:07:09 -0400 | [diff] [blame] | 225 | |
| 226 | struct dentry *debugfs_node_dir; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 227 | }; |
| 228 | |
| 229 | enum { |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 230 | DESC_DONE_FLAG = BIT(0), |
| 231 | LINK_DOWN_FLAG = BIT(1), |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 232 | }; |
| 233 | |
| 234 | struct ntb_payload_header { |
Jon Mason | 7446564 | 2013-01-21 15:28:52 -0700 | [diff] [blame] | 235 | unsigned int ver; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 236 | unsigned int len; |
| 237 | unsigned int flags; |
| 238 | }; |
| 239 | |
| 240 | enum { |
| 241 | VERSION = 0, |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 242 | QP_LINKS, |
Jon Mason | 113fc50 | 2013-01-30 11:40:52 -0700 | [diff] [blame] | 243 | NUM_QPS, |
| 244 | NUM_MWS, |
| 245 | MW0_SZ_HIGH, |
| 246 | MW0_SZ_LOW, |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 247 | }; |
| 248 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 249 | #define dev_client_dev(__dev) \ |
| 250 | container_of((__dev), struct ntb_transport_client_dev, dev) |
| 251 | |
| 252 | #define drv_client(__drv) \ |
| 253 | container_of((__drv), struct ntb_transport_client, driver) |
| 254 | |
| 255 | #define QP_TO_MW(nt, qp) ((qp) % nt->mw_count) |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 256 | #define NTB_QP_DEF_NUM_ENTRIES 100 |
| 257 | #define NTB_LINK_DOWN_TIMEOUT 10 |
| 258 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 259 | static void ntb_transport_rxc_db(unsigned long data); |
| 260 | static const struct ntb_ctx_ops ntb_transport_ops; |
| 261 | static struct ntb_client ntb_transport_client; |
Dave Jiang | 9cabc26 | 2016-07-20 13:14:07 -0700 | [diff] [blame] | 262 | static int ntb_async_tx_submit(struct ntb_transport_qp *qp, |
| 263 | struct ntb_queue_entry *entry); |
| 264 | static void ntb_memcpy_tx(struct ntb_queue_entry *entry, void __iomem *offset); |
Dave Jiang | 7220357 | 2016-07-20 13:14:13 -0700 | [diff] [blame] | 265 | static int ntb_async_rx_submit(struct ntb_queue_entry *entry, void *offset); |
| 266 | static void ntb_memcpy_rx(struct ntb_queue_entry *entry, void *offset); |
| 267 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 268 | |
| 269 | static int ntb_transport_bus_match(struct device *dev, |
| 270 | struct device_driver *drv) |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 271 | { |
| 272 | return !strncmp(dev_name(dev), drv->name, strlen(drv->name)); |
| 273 | } |
| 274 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 275 | static int ntb_transport_bus_probe(struct device *dev) |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 276 | { |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 277 | const struct ntb_transport_client *client; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 278 | int rc = -EINVAL; |
| 279 | |
| 280 | get_device(dev); |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 281 | |
| 282 | client = drv_client(dev->driver); |
| 283 | rc = client->probe(dev); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 284 | if (rc) |
| 285 | put_device(dev); |
| 286 | |
| 287 | return rc; |
| 288 | } |
| 289 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 290 | static int ntb_transport_bus_remove(struct device *dev) |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 291 | { |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 292 | const struct ntb_transport_client *client; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 293 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 294 | client = drv_client(dev->driver); |
| 295 | client->remove(dev); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 296 | |
| 297 | put_device(dev); |
| 298 | |
| 299 | return 0; |
| 300 | } |
| 301 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 302 | static struct bus_type ntb_transport_bus = { |
| 303 | .name = "ntb_transport", |
| 304 | .match = ntb_transport_bus_match, |
| 305 | .probe = ntb_transport_bus_probe, |
| 306 | .remove = ntb_transport_bus_remove, |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 307 | }; |
| 308 | |
| 309 | static LIST_HEAD(ntb_transport_list); |
| 310 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 311 | static int ntb_bus_init(struct ntb_transport_ctx *nt) |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 312 | { |
Dave Jiang | 3151000 | 2015-07-13 08:07:19 -0400 | [diff] [blame] | 313 | list_add_tail(&nt->entry, &ntb_transport_list); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 314 | return 0; |
| 315 | } |
| 316 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 317 | static void ntb_bus_remove(struct ntb_transport_ctx *nt) |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 318 | { |
| 319 | struct ntb_transport_client_dev *client_dev, *cd; |
| 320 | |
| 321 | list_for_each_entry_safe(client_dev, cd, &nt->client_devs, entry) { |
| 322 | dev_err(client_dev->dev.parent, "%s still attached to bus, removing\n", |
| 323 | dev_name(&client_dev->dev)); |
| 324 | list_del(&client_dev->entry); |
| 325 | device_unregister(&client_dev->dev); |
| 326 | } |
| 327 | |
| 328 | list_del(&nt->entry); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 329 | } |
| 330 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 331 | static void ntb_transport_client_release(struct device *dev) |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 332 | { |
| 333 | struct ntb_transport_client_dev *client_dev; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 334 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 335 | client_dev = dev_client_dev(dev); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 336 | kfree(client_dev); |
| 337 | } |
| 338 | |
| 339 | /** |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 340 | * ntb_transport_unregister_client_dev - Unregister NTB client device |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 341 | * @device_name: Name of NTB client device |
| 342 | * |
| 343 | * Unregister an NTB client device with the NTB transport layer |
| 344 | */ |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 345 | void ntb_transport_unregister_client_dev(char *device_name) |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 346 | { |
| 347 | struct ntb_transport_client_dev *client, *cd; |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 348 | struct ntb_transport_ctx *nt; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 349 | |
| 350 | list_for_each_entry(nt, &ntb_transport_list, entry) |
| 351 | list_for_each_entry_safe(client, cd, &nt->client_devs, entry) |
| 352 | if (!strncmp(dev_name(&client->dev), device_name, |
| 353 | strlen(device_name))) { |
| 354 | list_del(&client->entry); |
| 355 | device_unregister(&client->dev); |
| 356 | } |
| 357 | } |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 358 | EXPORT_SYMBOL_GPL(ntb_transport_unregister_client_dev); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 359 | |
| 360 | /** |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 361 | * ntb_transport_register_client_dev - Register NTB client device |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 362 | * @device_name: Name of NTB client device |
| 363 | * |
| 364 | * Register an NTB client device with the NTB transport layer |
| 365 | */ |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 366 | int ntb_transport_register_client_dev(char *device_name) |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 367 | { |
| 368 | struct ntb_transport_client_dev *client_dev; |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 369 | struct ntb_transport_ctx *nt; |
Allen Hubbe | 1199aa6 | 2015-05-18 06:20:47 -0400 | [diff] [blame] | 370 | int node; |
Jon Mason | 8b19d45 | 2013-04-26 14:51:57 -0700 | [diff] [blame] | 371 | int rc, i = 0; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 372 | |
Jon Mason | 8222b40 | 2013-01-19 02:02:16 -0700 | [diff] [blame] | 373 | if (list_empty(&ntb_transport_list)) |
| 374 | return -ENODEV; |
| 375 | |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 376 | list_for_each_entry(nt, &ntb_transport_list, entry) { |
| 377 | struct device *dev; |
| 378 | |
Allen Hubbe | 1199aa6 | 2015-05-18 06:20:47 -0400 | [diff] [blame] | 379 | node = dev_to_node(&nt->ndev->dev); |
| 380 | |
| 381 | client_dev = kzalloc_node(sizeof(*client_dev), |
| 382 | GFP_KERNEL, node); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 383 | if (!client_dev) { |
| 384 | rc = -ENOMEM; |
| 385 | goto err; |
| 386 | } |
| 387 | |
| 388 | dev = &client_dev->dev; |
| 389 | |
| 390 | /* setup and register client devices */ |
Jon Mason | 8b19d45 | 2013-04-26 14:51:57 -0700 | [diff] [blame] | 391 | dev_set_name(dev, "%s%d", device_name, i); |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 392 | dev->bus = &ntb_transport_bus; |
| 393 | dev->release = ntb_transport_client_release; |
| 394 | dev->parent = &nt->ndev->dev; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 395 | |
| 396 | rc = device_register(dev); |
| 397 | if (rc) { |
| 398 | kfree(client_dev); |
| 399 | goto err; |
| 400 | } |
| 401 | |
| 402 | list_add_tail(&client_dev->entry, &nt->client_devs); |
Jon Mason | 8b19d45 | 2013-04-26 14:51:57 -0700 | [diff] [blame] | 403 | i++; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 404 | } |
| 405 | |
| 406 | return 0; |
| 407 | |
| 408 | err: |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 409 | ntb_transport_unregister_client_dev(device_name); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 410 | |
| 411 | return rc; |
| 412 | } |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 413 | EXPORT_SYMBOL_GPL(ntb_transport_register_client_dev); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 414 | |
| 415 | /** |
Allen Hubbe | ec110bc | 2015-05-07 06:45:21 -0400 | [diff] [blame] | 416 | * ntb_transport_register_client - Register NTB client driver |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 417 | * @drv: NTB client driver to be registered |
| 418 | * |
| 419 | * Register an NTB client driver with the NTB transport layer |
| 420 | * |
| 421 | * RETURNS: An appropriate -ERRNO error value on error, or zero for success. |
| 422 | */ |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 423 | int ntb_transport_register_client(struct ntb_transport_client *drv) |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 424 | { |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 425 | drv->driver.bus = &ntb_transport_bus; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 426 | |
Jon Mason | 8222b40 | 2013-01-19 02:02:16 -0700 | [diff] [blame] | 427 | if (list_empty(&ntb_transport_list)) |
| 428 | return -ENODEV; |
| 429 | |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 430 | return driver_register(&drv->driver); |
| 431 | } |
Allen Hubbe | ec110bc | 2015-05-07 06:45:21 -0400 | [diff] [blame] | 432 | EXPORT_SYMBOL_GPL(ntb_transport_register_client); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 433 | |
| 434 | /** |
Allen Hubbe | ec110bc | 2015-05-07 06:45:21 -0400 | [diff] [blame] | 435 | * ntb_transport_unregister_client - Unregister NTB client driver |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 436 | * @drv: NTB client driver to be unregistered |
| 437 | * |
| 438 | * Unregister an NTB client driver with the NTB transport layer |
| 439 | * |
| 440 | * RETURNS: An appropriate -ERRNO error value on error, or zero for success. |
| 441 | */ |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 442 | void ntb_transport_unregister_client(struct ntb_transport_client *drv) |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 443 | { |
| 444 | driver_unregister(&drv->driver); |
| 445 | } |
Allen Hubbe | ec110bc | 2015-05-07 06:45:21 -0400 | [diff] [blame] | 446 | EXPORT_SYMBOL_GPL(ntb_transport_unregister_client); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 447 | |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 448 | static ssize_t debugfs_read(struct file *filp, char __user *ubuf, size_t count, |
| 449 | loff_t *offp) |
| 450 | { |
| 451 | struct ntb_transport_qp *qp; |
Jon Mason | d7237e2 | 2013-01-19 02:02:25 -0700 | [diff] [blame] | 452 | char *buf; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 453 | ssize_t ret, out_offset, out_count; |
| 454 | |
Dave Jiang | 260bee9 | 2015-07-13 08:07:11 -0400 | [diff] [blame] | 455 | qp = filp->private_data; |
| 456 | |
| 457 | if (!qp || !qp->link_is_up) |
| 458 | return 0; |
| 459 | |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 460 | out_count = 1000; |
Jon Mason | d7237e2 | 2013-01-19 02:02:25 -0700 | [diff] [blame] | 461 | |
| 462 | buf = kmalloc(out_count, GFP_KERNEL); |
| 463 | if (!buf) |
| 464 | return -ENOMEM; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 465 | |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 466 | out_offset = 0; |
| 467 | out_offset += snprintf(buf + out_offset, out_count - out_offset, |
Dave Jiang | d98ef99 | 2015-07-13 08:07:20 -0400 | [diff] [blame] | 468 | "\nNTB QP stats:\n\n"); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 469 | out_offset += snprintf(buf + out_offset, out_count - out_offset, |
| 470 | "rx_bytes - \t%llu\n", qp->rx_bytes); |
| 471 | out_offset += snprintf(buf + out_offset, out_count - out_offset, |
| 472 | "rx_pkts - \t%llu\n", qp->rx_pkts); |
| 473 | out_offset += snprintf(buf + out_offset, out_count - out_offset, |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 474 | "rx_memcpy - \t%llu\n", qp->rx_memcpy); |
| 475 | out_offset += snprintf(buf + out_offset, out_count - out_offset, |
| 476 | "rx_async - \t%llu\n", qp->rx_async); |
| 477 | out_offset += snprintf(buf + out_offset, out_count - out_offset, |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 478 | "rx_ring_empty - %llu\n", qp->rx_ring_empty); |
| 479 | out_offset += snprintf(buf + out_offset, out_count - out_offset, |
| 480 | "rx_err_no_buf - %llu\n", qp->rx_err_no_buf); |
| 481 | out_offset += snprintf(buf + out_offset, out_count - out_offset, |
| 482 | "rx_err_oflow - \t%llu\n", qp->rx_err_oflow); |
| 483 | out_offset += snprintf(buf + out_offset, out_count - out_offset, |
| 484 | "rx_err_ver - \t%llu\n", qp->rx_err_ver); |
| 485 | out_offset += snprintf(buf + out_offset, out_count - out_offset, |
Dave Jiang | d98ef99 | 2015-07-13 08:07:20 -0400 | [diff] [blame] | 486 | "rx_buff - \t0x%p\n", qp->rx_buff); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 487 | out_offset += snprintf(buf + out_offset, out_count - out_offset, |
Jon Mason | 793c20e | 2013-01-19 02:02:26 -0700 | [diff] [blame] | 488 | "rx_index - \t%u\n", qp->rx_index); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 489 | out_offset += snprintf(buf + out_offset, out_count - out_offset, |
Dave Jiang | a754a8f | 2016-04-08 10:49:06 -0700 | [diff] [blame] | 490 | "rx_max_entry - \t%u\n", qp->rx_max_entry); |
| 491 | out_offset += snprintf(buf + out_offset, out_count - out_offset, |
| 492 | "rx_alloc_entry - \t%u\n\n", qp->rx_alloc_entry); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 493 | |
| 494 | out_offset += snprintf(buf + out_offset, out_count - out_offset, |
| 495 | "tx_bytes - \t%llu\n", qp->tx_bytes); |
| 496 | out_offset += snprintf(buf + out_offset, out_count - out_offset, |
| 497 | "tx_pkts - \t%llu\n", qp->tx_pkts); |
| 498 | out_offset += snprintf(buf + out_offset, out_count - out_offset, |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 499 | "tx_memcpy - \t%llu\n", qp->tx_memcpy); |
| 500 | out_offset += snprintf(buf + out_offset, out_count - out_offset, |
| 501 | "tx_async - \t%llu\n", qp->tx_async); |
| 502 | out_offset += snprintf(buf + out_offset, out_count - out_offset, |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 503 | "tx_ring_full - \t%llu\n", qp->tx_ring_full); |
| 504 | out_offset += snprintf(buf + out_offset, out_count - out_offset, |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 505 | "tx_err_no_buf - %llu\n", qp->tx_err_no_buf); |
| 506 | out_offset += snprintf(buf + out_offset, out_count - out_offset, |
Dave Jiang | d98ef99 | 2015-07-13 08:07:20 -0400 | [diff] [blame] | 507 | "tx_mw - \t0x%p\n", qp->tx_mw); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 508 | out_offset += snprintf(buf + out_offset, out_count - out_offset, |
Dave Jiang | d98ef99 | 2015-07-13 08:07:20 -0400 | [diff] [blame] | 509 | "tx_index (H) - \t%u\n", qp->tx_index); |
| 510 | out_offset += snprintf(buf + out_offset, out_count - out_offset, |
| 511 | "RRI (T) - \t%u\n", |
| 512 | qp->remote_rx_info->entry); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 513 | out_offset += snprintf(buf + out_offset, out_count - out_offset, |
Jon Mason | 793c20e | 2013-01-19 02:02:26 -0700 | [diff] [blame] | 514 | "tx_max_entry - \t%u\n", qp->tx_max_entry); |
Dave Jiang | e74bfee | 2015-07-13 08:07:17 -0400 | [diff] [blame] | 515 | out_offset += snprintf(buf + out_offset, out_count - out_offset, |
Dave Jiang | e74bfee | 2015-07-13 08:07:17 -0400 | [diff] [blame] | 516 | "free tx - \t%u\n", |
| 517 | ntb_transport_tx_free_entry(qp)); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 518 | |
| 519 | out_offset += snprintf(buf + out_offset, out_count - out_offset, |
Dave Jiang | d98ef99 | 2015-07-13 08:07:20 -0400 | [diff] [blame] | 520 | "\n"); |
| 521 | out_offset += snprintf(buf + out_offset, out_count - out_offset, |
Dave Jiang | 569410ca | 2015-07-13 08:07:22 -0400 | [diff] [blame] | 522 | "Using TX DMA - \t%s\n", |
| 523 | qp->tx_dma_chan ? "Yes" : "No"); |
| 524 | out_offset += snprintf(buf + out_offset, out_count - out_offset, |
| 525 | "Using RX DMA - \t%s\n", |
| 526 | qp->rx_dma_chan ? "Yes" : "No"); |
Dave Jiang | d98ef99 | 2015-07-13 08:07:20 -0400 | [diff] [blame] | 527 | out_offset += snprintf(buf + out_offset, out_count - out_offset, |
| 528 | "QP Link - \t%s\n", |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 529 | qp->link_is_up ? "Up" : "Down"); |
Dave Jiang | d98ef99 | 2015-07-13 08:07:20 -0400 | [diff] [blame] | 530 | out_offset += snprintf(buf + out_offset, out_count - out_offset, |
| 531 | "\n"); |
| 532 | |
Jon Mason | d7237e2 | 2013-01-19 02:02:25 -0700 | [diff] [blame] | 533 | if (out_offset > out_count) |
| 534 | out_offset = out_count; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 535 | |
| 536 | ret = simple_read_from_buffer(ubuf, count, offp, buf, out_offset); |
Jon Mason | d7237e2 | 2013-01-19 02:02:25 -0700 | [diff] [blame] | 537 | kfree(buf); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 538 | return ret; |
| 539 | } |
| 540 | |
| 541 | static const struct file_operations ntb_qp_debugfs_stats = { |
| 542 | .owner = THIS_MODULE, |
Jon Mason | d66d7ac | 2013-01-19 02:02:20 -0700 | [diff] [blame] | 543 | .open = simple_open, |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 544 | .read = debugfs_read, |
| 545 | }; |
| 546 | |
| 547 | static void ntb_list_add(spinlock_t *lock, struct list_head *entry, |
| 548 | struct list_head *list) |
| 549 | { |
| 550 | unsigned long flags; |
| 551 | |
| 552 | spin_lock_irqsave(lock, flags); |
| 553 | list_add_tail(entry, list); |
| 554 | spin_unlock_irqrestore(lock, flags); |
| 555 | } |
| 556 | |
| 557 | static struct ntb_queue_entry *ntb_list_rm(spinlock_t *lock, |
Jon Mason | 53ca4fe | 2013-11-26 11:21:50 -0700 | [diff] [blame] | 558 | struct list_head *list) |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 559 | { |
| 560 | struct ntb_queue_entry *entry; |
| 561 | unsigned long flags; |
| 562 | |
| 563 | spin_lock_irqsave(lock, flags); |
| 564 | if (list_empty(list)) { |
| 565 | entry = NULL; |
| 566 | goto out; |
| 567 | } |
| 568 | entry = list_first_entry(list, struct ntb_queue_entry, entry); |
| 569 | list_del(&entry->entry); |
Dave Jiang | e74bfee | 2015-07-13 08:07:17 -0400 | [diff] [blame] | 570 | |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 571 | out: |
| 572 | spin_unlock_irqrestore(lock, flags); |
| 573 | |
| 574 | return entry; |
| 575 | } |
| 576 | |
Allen Hubbe | da2e5ae | 2015-07-13 08:07:08 -0400 | [diff] [blame] | 577 | static struct ntb_queue_entry *ntb_list_mv(spinlock_t *lock, |
| 578 | struct list_head *list, |
| 579 | struct list_head *to_list) |
| 580 | { |
| 581 | struct ntb_queue_entry *entry; |
| 582 | unsigned long flags; |
| 583 | |
| 584 | spin_lock_irqsave(lock, flags); |
| 585 | |
| 586 | if (list_empty(list)) { |
| 587 | entry = NULL; |
| 588 | } else { |
| 589 | entry = list_first_entry(list, struct ntb_queue_entry, entry); |
| 590 | list_move_tail(&entry->entry, to_list); |
| 591 | } |
| 592 | |
| 593 | spin_unlock_irqrestore(lock, flags); |
| 594 | |
| 595 | return entry; |
| 596 | } |
| 597 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 598 | static int ntb_transport_setup_qp_mw(struct ntb_transport_ctx *nt, |
| 599 | unsigned int qp_num) |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 600 | { |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 601 | struct ntb_transport_qp *qp = &nt->qp_vec[qp_num]; |
| 602 | struct ntb_transport_mw *mw; |
Dave Jiang | a754a8f | 2016-04-08 10:49:06 -0700 | [diff] [blame] | 603 | struct ntb_dev *ndev = nt->ndev; |
| 604 | struct ntb_queue_entry *entry; |
Jon Mason | ef114ed | 2013-01-19 02:02:18 -0700 | [diff] [blame] | 605 | unsigned int rx_size, num_qps_mw; |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 606 | unsigned int mw_num, mw_count, qp_count; |
Jon Mason | 793c20e | 2013-01-19 02:02:26 -0700 | [diff] [blame] | 607 | unsigned int i; |
Dave Jiang | a754a8f | 2016-04-08 10:49:06 -0700 | [diff] [blame] | 608 | int node; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 609 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 610 | mw_count = nt->mw_count; |
| 611 | qp_count = nt->qp_count; |
Jon Mason | 948d3a6 | 2013-04-18 17:07:36 -0700 | [diff] [blame] | 612 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 613 | mw_num = QP_TO_MW(nt, qp_num); |
| 614 | mw = &nt->mw_vec[mw_num]; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 615 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 616 | if (!mw->virt_addr) |
| 617 | return -ENOMEM; |
| 618 | |
Logan Gunthorpe | 8e8496e | 2017-06-05 14:00:53 -0600 | [diff] [blame] | 619 | if (mw_num < qp_count % mw_count) |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 620 | num_qps_mw = qp_count / mw_count + 1; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 621 | else |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 622 | num_qps_mw = qp_count / mw_count; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 623 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 624 | rx_size = (unsigned int)mw->xlat_size / num_qps_mw; |
Jon Mason | c92ba3c | 2015-10-04 22:54:22 -0400 | [diff] [blame] | 625 | qp->rx_buff = mw->virt_addr + rx_size * (qp_num / mw_count); |
Jon Mason | 793c20e | 2013-01-19 02:02:26 -0700 | [diff] [blame] | 626 | rx_size -= sizeof(struct ntb_rx_info); |
| 627 | |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 628 | qp->remote_rx_info = qp->rx_buff + rx_size; |
| 629 | |
Jon Mason | c9d534c | 2013-02-01 15:45:16 -0700 | [diff] [blame] | 630 | /* Due to housekeeping, there must be atleast 2 buffs */ |
| 631 | qp->rx_max_frame = min(transport_mtu, rx_size / 2); |
Jon Mason | 793c20e | 2013-01-19 02:02:26 -0700 | [diff] [blame] | 632 | qp->rx_max_entry = rx_size / qp->rx_max_frame; |
| 633 | qp->rx_index = 0; |
| 634 | |
Dave Jiang | a754a8f | 2016-04-08 10:49:06 -0700 | [diff] [blame] | 635 | /* |
| 636 | * Checking to see if we have more entries than the default. |
| 637 | * We should add additional entries if that is the case so we |
| 638 | * can be in sync with the transport frames. |
| 639 | */ |
| 640 | node = dev_to_node(&ndev->dev); |
| 641 | for (i = qp->rx_alloc_entry; i < qp->rx_max_entry; i++) { |
| 642 | entry = kzalloc_node(sizeof(*entry), GFP_ATOMIC, node); |
| 643 | if (!entry) |
| 644 | return -ENOMEM; |
| 645 | |
| 646 | entry->qp = qp; |
| 647 | ntb_list_add(&qp->ntb_rx_q_lock, &entry->entry, |
| 648 | &qp->rx_free_q); |
| 649 | qp->rx_alloc_entry++; |
| 650 | } |
| 651 | |
Jon Mason | c9d534c | 2013-02-01 15:45:16 -0700 | [diff] [blame] | 652 | qp->remote_rx_info->entry = qp->rx_max_entry - 1; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 653 | |
Jon Mason | ef114ed | 2013-01-19 02:02:18 -0700 | [diff] [blame] | 654 | /* setup the hdr offsets with 0's */ |
Jon Mason | 793c20e | 2013-01-19 02:02:26 -0700 | [diff] [blame] | 655 | for (i = 0; i < qp->rx_max_entry; i++) { |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 656 | void *offset = (qp->rx_buff + qp->rx_max_frame * (i + 1) - |
| 657 | sizeof(struct ntb_payload_header)); |
Jon Mason | ef114ed | 2013-01-19 02:02:18 -0700 | [diff] [blame] | 658 | memset(offset, 0, sizeof(struct ntb_payload_header)); |
Jon Mason | 793c20e | 2013-01-19 02:02:26 -0700 | [diff] [blame] | 659 | } |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 660 | |
| 661 | qp->rx_pkts = 0; |
| 662 | qp->tx_pkts = 0; |
Jon Mason | 90f9e93 | 2013-02-01 15:34:35 -0700 | [diff] [blame] | 663 | qp->tx_index = 0; |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 664 | |
| 665 | return 0; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 666 | } |
| 667 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 668 | static void ntb_free_mw(struct ntb_transport_ctx *nt, int num_mw) |
Jon Mason | 113fc50 | 2013-01-30 11:40:52 -0700 | [diff] [blame] | 669 | { |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 670 | struct ntb_transport_mw *mw = &nt->mw_vec[num_mw]; |
| 671 | struct pci_dev *pdev = nt->ndev->pdev; |
Jon Mason | 113fc50 | 2013-01-30 11:40:52 -0700 | [diff] [blame] | 672 | |
| 673 | if (!mw->virt_addr) |
| 674 | return; |
| 675 | |
Serge Semin | 443b9a1 | 2017-01-11 03:11:33 +0300 | [diff] [blame] | 676 | ntb_mw_clear_trans(nt->ndev, PIDX, num_mw); |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 677 | dma_free_coherent(&pdev->dev, mw->buff_size, |
| 678 | mw->virt_addr, mw->dma_addr); |
| 679 | mw->xlat_size = 0; |
| 680 | mw->buff_size = 0; |
Jon Mason | 113fc50 | 2013-01-30 11:40:52 -0700 | [diff] [blame] | 681 | mw->virt_addr = NULL; |
| 682 | } |
| 683 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 684 | static int ntb_set_mw(struct ntb_transport_ctx *nt, int num_mw, |
Allen Hubbe | 8c9edf6 | 2015-07-13 08:07:13 -0400 | [diff] [blame] | 685 | resource_size_t size) |
Jon Mason | b77b263 | 2013-02-01 15:25:37 -0700 | [diff] [blame] | 686 | { |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 687 | struct ntb_transport_mw *mw = &nt->mw_vec[num_mw]; |
| 688 | struct pci_dev *pdev = nt->ndev->pdev; |
Allen Hubbe | 8c9edf6 | 2015-07-13 08:07:13 -0400 | [diff] [blame] | 689 | size_t xlat_size, buff_size; |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 690 | int rc; |
| 691 | |
Allen Hubbe | 8c9edf6 | 2015-07-13 08:07:13 -0400 | [diff] [blame] | 692 | if (!size) |
| 693 | return -EINVAL; |
| 694 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 695 | xlat_size = round_up(size, mw->xlat_align_size); |
| 696 | buff_size = round_up(size, mw->xlat_align); |
Jon Mason | b77b263 | 2013-02-01 15:25:37 -0700 | [diff] [blame] | 697 | |
| 698 | /* No need to re-setup */ |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 699 | if (mw->xlat_size == xlat_size) |
Jon Mason | b77b263 | 2013-02-01 15:25:37 -0700 | [diff] [blame] | 700 | return 0; |
| 701 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 702 | if (mw->buff_size) |
Jon Mason | b77b263 | 2013-02-01 15:25:37 -0700 | [diff] [blame] | 703 | ntb_free_mw(nt, num_mw); |
| 704 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 705 | /* Alloc memory for receiving data. Must be aligned */ |
| 706 | mw->xlat_size = xlat_size; |
| 707 | mw->buff_size = buff_size; |
Jon Mason | b77b263 | 2013-02-01 15:25:37 -0700 | [diff] [blame] | 708 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 709 | mw->virt_addr = dma_alloc_coherent(&pdev->dev, buff_size, |
| 710 | &mw->dma_addr, GFP_KERNEL); |
Jon Mason | b77b263 | 2013-02-01 15:25:37 -0700 | [diff] [blame] | 711 | if (!mw->virt_addr) { |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 712 | mw->xlat_size = 0; |
| 713 | mw->buff_size = 0; |
Allen Hubbe | 8c9edf6 | 2015-07-13 08:07:13 -0400 | [diff] [blame] | 714 | dev_err(&pdev->dev, "Unable to alloc MW buff of size %zu\n", |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 715 | buff_size); |
Jon Mason | b77b263 | 2013-02-01 15:25:37 -0700 | [diff] [blame] | 716 | return -ENOMEM; |
| 717 | } |
| 718 | |
Dave Jiang | 3cc5ba1 | 2014-08-28 13:53:02 -0700 | [diff] [blame] | 719 | /* |
| 720 | * we must ensure that the memory address allocated is BAR size |
| 721 | * aligned in order for the XLAT register to take the value. This |
| 722 | * is a requirement of the hardware. It is recommended to setup CMA |
| 723 | * for BAR sizes equal or greater than 4MB. |
| 724 | */ |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 725 | if (!IS_ALIGNED(mw->dma_addr, mw->xlat_align)) { |
| 726 | dev_err(&pdev->dev, "DMA memory %pad is not aligned\n", |
Dave Jiang | 3cc5ba1 | 2014-08-28 13:53:02 -0700 | [diff] [blame] | 727 | &mw->dma_addr); |
| 728 | ntb_free_mw(nt, num_mw); |
| 729 | return -ENOMEM; |
| 730 | } |
| 731 | |
Jon Mason | b77b263 | 2013-02-01 15:25:37 -0700 | [diff] [blame] | 732 | /* Notify HW the memory location of the receive buffer */ |
Serge Semin | 443b9a1 | 2017-01-11 03:11:33 +0300 | [diff] [blame] | 733 | rc = ntb_mw_set_trans(nt->ndev, PIDX, num_mw, mw->dma_addr, |
| 734 | mw->xlat_size); |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 735 | if (rc) { |
| 736 | dev_err(&pdev->dev, "Unable to set mw%d translation", num_mw); |
| 737 | ntb_free_mw(nt, num_mw); |
| 738 | return -EIO; |
| 739 | } |
Jon Mason | b77b263 | 2013-02-01 15:25:37 -0700 | [diff] [blame] | 740 | |
| 741 | return 0; |
| 742 | } |
| 743 | |
Allen Hubbe | 2849b5d7 | 2015-05-12 08:09:15 -0400 | [diff] [blame] | 744 | static void ntb_qp_link_down_reset(struct ntb_transport_qp *qp) |
| 745 | { |
| 746 | qp->link_is_up = false; |
Dave Jiang | e902133 | 2016-02-23 09:11:36 -0700 | [diff] [blame] | 747 | qp->active = false; |
Allen Hubbe | 2849b5d7 | 2015-05-12 08:09:15 -0400 | [diff] [blame] | 748 | |
| 749 | qp->tx_index = 0; |
| 750 | qp->rx_index = 0; |
| 751 | qp->rx_bytes = 0; |
| 752 | qp->rx_pkts = 0; |
| 753 | qp->rx_ring_empty = 0; |
| 754 | qp->rx_err_no_buf = 0; |
| 755 | qp->rx_err_oflow = 0; |
| 756 | qp->rx_err_ver = 0; |
| 757 | qp->rx_memcpy = 0; |
| 758 | qp->rx_async = 0; |
| 759 | qp->tx_bytes = 0; |
| 760 | qp->tx_pkts = 0; |
| 761 | qp->tx_ring_full = 0; |
| 762 | qp->tx_err_no_buf = 0; |
| 763 | qp->tx_memcpy = 0; |
| 764 | qp->tx_async = 0; |
| 765 | } |
| 766 | |
Jon Mason | fca4d51 | 2013-09-09 13:39:55 -0700 | [diff] [blame] | 767 | static void ntb_qp_link_cleanup(struct ntb_transport_qp *qp) |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 768 | { |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 769 | struct ntb_transport_ctx *nt = qp->transport; |
| 770 | struct pci_dev *pdev = nt->ndev->pdev; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 771 | |
Allen Hubbe | e22e0b9 | 2015-05-12 06:55:44 -0400 | [diff] [blame] | 772 | dev_info(&pdev->dev, "qp %d: Link Cleanup\n", qp->qp_num); |
Allen Hubbe | 2849b5d7 | 2015-05-12 08:09:15 -0400 | [diff] [blame] | 773 | |
| 774 | cancel_delayed_work_sync(&qp->link_work); |
| 775 | ntb_qp_link_down_reset(qp); |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 776 | |
| 777 | if (qp->event_handler) |
| 778 | qp->event_handler(qp->cb_data, qp->link_is_up); |
Jon Mason | fca4d51 | 2013-09-09 13:39:55 -0700 | [diff] [blame] | 779 | } |
| 780 | |
| 781 | static void ntb_qp_link_cleanup_work(struct work_struct *work) |
| 782 | { |
| 783 | struct ntb_transport_qp *qp = container_of(work, |
| 784 | struct ntb_transport_qp, |
| 785 | link_cleanup); |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 786 | struct ntb_transport_ctx *nt = qp->transport; |
Jon Mason | fca4d51 | 2013-09-09 13:39:55 -0700 | [diff] [blame] | 787 | |
| 788 | ntb_qp_link_cleanup(qp); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 789 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 790 | if (nt->link_is_up) |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 791 | schedule_delayed_work(&qp->link_work, |
| 792 | msecs_to_jiffies(NTB_LINK_DOWN_TIMEOUT)); |
| 793 | } |
| 794 | |
Jon Mason | 7b4f2d3 | 2013-01-19 02:02:19 -0700 | [diff] [blame] | 795 | static void ntb_qp_link_down(struct ntb_transport_qp *qp) |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 796 | { |
Jon Mason | 7b4f2d3 | 2013-01-19 02:02:19 -0700 | [diff] [blame] | 797 | schedule_work(&qp->link_cleanup); |
| 798 | } |
| 799 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 800 | static void ntb_transport_link_cleanup(struct ntb_transport_ctx *nt) |
Jon Mason | 7b4f2d3 | 2013-01-19 02:02:19 -0700 | [diff] [blame] | 801 | { |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 802 | struct ntb_transport_qp *qp; |
| 803 | u64 qp_bitmap_alloc; |
Shyam Sundar S K | b17faba | 2016-12-07 22:37:05 +0530 | [diff] [blame] | 804 | unsigned int i, count; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 805 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 806 | qp_bitmap_alloc = nt->qp_bitmap & ~nt->qp_bitmap_free; |
Jon Mason | fca4d51 | 2013-09-09 13:39:55 -0700 | [diff] [blame] | 807 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 808 | /* Pass along the info to any clients */ |
| 809 | for (i = 0; i < nt->qp_count; i++) |
| 810 | if (qp_bitmap_alloc & BIT_ULL(i)) { |
| 811 | qp = &nt->qp_vec[i]; |
| 812 | ntb_qp_link_cleanup(qp); |
| 813 | cancel_work_sync(&qp->link_cleanup); |
| 814 | cancel_delayed_work_sync(&qp->link_work); |
| 815 | } |
| 816 | |
| 817 | if (!nt->link_is_up) |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 818 | cancel_delayed_work_sync(&nt->link_work); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 819 | |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 820 | /* The scratchpad registers keep the values if the remote side |
| 821 | * goes down, blast them now to give them a sane value the next |
| 822 | * time they are accessed |
| 823 | */ |
Shyam Sundar S K | b17faba | 2016-12-07 22:37:05 +0530 | [diff] [blame] | 824 | count = ntb_spad_count(nt->ndev); |
| 825 | for (i = 0; i < count; i++) |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 826 | ntb_spad_write(nt->ndev, i, 0); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 827 | } |
| 828 | |
Jon Mason | fca4d51 | 2013-09-09 13:39:55 -0700 | [diff] [blame] | 829 | static void ntb_transport_link_cleanup_work(struct work_struct *work) |
| 830 | { |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 831 | struct ntb_transport_ctx *nt = |
| 832 | container_of(work, struct ntb_transport_ctx, link_cleanup); |
Jon Mason | fca4d51 | 2013-09-09 13:39:55 -0700 | [diff] [blame] | 833 | |
| 834 | ntb_transport_link_cleanup(nt); |
| 835 | } |
| 836 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 837 | static void ntb_transport_event_callback(void *data) |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 838 | { |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 839 | struct ntb_transport_ctx *nt = data; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 840 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 841 | if (ntb_link_is_up(nt->ndev, NULL, NULL) == 1) |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 842 | schedule_delayed_work(&nt->link_work, 0); |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 843 | else |
Jon Mason | 7b4f2d3 | 2013-01-19 02:02:19 -0700 | [diff] [blame] | 844 | schedule_work(&nt->link_cleanup); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 845 | } |
| 846 | |
| 847 | static void ntb_transport_link_work(struct work_struct *work) |
| 848 | { |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 849 | struct ntb_transport_ctx *nt = |
| 850 | container_of(work, struct ntb_transport_ctx, link_work.work); |
| 851 | struct ntb_dev *ndev = nt->ndev; |
| 852 | struct pci_dev *pdev = ndev->pdev; |
| 853 | resource_size_t size; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 854 | u32 val; |
Dave Jiang | 84f7668 | 2016-02-29 09:35:26 -0700 | [diff] [blame] | 855 | int rc = 0, i, spad; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 856 | |
Jon Mason | 113fc50 | 2013-01-30 11:40:52 -0700 | [diff] [blame] | 857 | /* send the local info, in the opposite order of the way we read it */ |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 858 | for (i = 0; i < nt->mw_count; i++) { |
| 859 | size = nt->mw_vec[i].phys_size; |
Jon Mason | 113fc50 | 2013-01-30 11:40:52 -0700 | [diff] [blame] | 860 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 861 | if (max_mw_size && size > max_mw_size) |
| 862 | size = max_mw_size; |
| 863 | |
| 864 | spad = MW0_SZ_HIGH + (i * 2); |
Serge Semin | d67288a | 2017-01-11 03:13:20 +0300 | [diff] [blame] | 865 | ntb_peer_spad_write(ndev, PIDX, spad, upper_32_bits(size)); |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 866 | |
| 867 | spad = MW0_SZ_LOW + (i * 2); |
Serge Semin | d67288a | 2017-01-11 03:13:20 +0300 | [diff] [blame] | 868 | ntb_peer_spad_write(ndev, PIDX, spad, lower_32_bits(size)); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 869 | } |
| 870 | |
Serge Semin | d67288a | 2017-01-11 03:13:20 +0300 | [diff] [blame] | 871 | ntb_peer_spad_write(ndev, PIDX, NUM_MWS, nt->mw_count); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 872 | |
Serge Semin | d67288a | 2017-01-11 03:13:20 +0300 | [diff] [blame] | 873 | ntb_peer_spad_write(ndev, PIDX, NUM_QPS, nt->qp_count); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 874 | |
Serge Semin | d67288a | 2017-01-11 03:13:20 +0300 | [diff] [blame] | 875 | ntb_peer_spad_write(ndev, PIDX, VERSION, NTB_TRANSPORT_VERSION); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 876 | |
| 877 | /* Query the remote side for its info */ |
Dave Jiang | 0f69a7d | 2015-06-02 03:45:07 -0400 | [diff] [blame] | 878 | val = ntb_spad_read(ndev, VERSION); |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 879 | dev_dbg(&pdev->dev, "Remote version = %d\n", val); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 880 | if (val != NTB_TRANSPORT_VERSION) |
| 881 | goto out; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 882 | |
Dave Jiang | 0f69a7d | 2015-06-02 03:45:07 -0400 | [diff] [blame] | 883 | val = ntb_spad_read(ndev, NUM_QPS); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 884 | dev_dbg(&pdev->dev, "Remote max number of qps = %d\n", val); |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 885 | if (val != nt->qp_count) |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 886 | goto out; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 887 | |
Dave Jiang | 0f69a7d | 2015-06-02 03:45:07 -0400 | [diff] [blame] | 888 | val = ntb_spad_read(ndev, NUM_MWS); |
Jon Mason | 113fc50 | 2013-01-30 11:40:52 -0700 | [diff] [blame] | 889 | dev_dbg(&pdev->dev, "Remote number of mws = %d\n", val); |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 890 | if (val != nt->mw_count) |
| 891 | goto out; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 892 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 893 | for (i = 0; i < nt->mw_count; i++) { |
Jon Mason | 113fc50 | 2013-01-30 11:40:52 -0700 | [diff] [blame] | 894 | u64 val64; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 895 | |
Dave Jiang | 0f69a7d | 2015-06-02 03:45:07 -0400 | [diff] [blame] | 896 | val = ntb_spad_read(ndev, MW0_SZ_HIGH + (i * 2)); |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 897 | val64 = (u64)val << 32; |
Jon Mason | 113fc50 | 2013-01-30 11:40:52 -0700 | [diff] [blame] | 898 | |
Dave Jiang | 0f69a7d | 2015-06-02 03:45:07 -0400 | [diff] [blame] | 899 | val = ntb_spad_read(ndev, MW0_SZ_LOW + (i * 2)); |
Jon Mason | 113fc50 | 2013-01-30 11:40:52 -0700 | [diff] [blame] | 900 | val64 |= val; |
| 901 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 902 | dev_dbg(&pdev->dev, "Remote MW%d size = %#llx\n", i, val64); |
Jon Mason | 113fc50 | 2013-01-30 11:40:52 -0700 | [diff] [blame] | 903 | |
| 904 | rc = ntb_set_mw(nt, i, val64); |
| 905 | if (rc) |
| 906 | goto out1; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 907 | } |
| 908 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 909 | nt->link_is_up = true; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 910 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 911 | for (i = 0; i < nt->qp_count; i++) { |
| 912 | struct ntb_transport_qp *qp = &nt->qp_vec[i]; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 913 | |
| 914 | ntb_transport_setup_qp_mw(nt, i); |
| 915 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 916 | if (qp->client_ready) |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 917 | schedule_delayed_work(&qp->link_work, 0); |
| 918 | } |
| 919 | |
| 920 | return; |
| 921 | |
Jon Mason | 113fc50 | 2013-01-30 11:40:52 -0700 | [diff] [blame] | 922 | out1: |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 923 | for (i = 0; i < nt->mw_count; i++) |
Jon Mason | 113fc50 | 2013-01-30 11:40:52 -0700 | [diff] [blame] | 924 | ntb_free_mw(nt, i); |
Dave Jiang | 84f7668 | 2016-02-29 09:35:26 -0700 | [diff] [blame] | 925 | |
| 926 | /* if there's an actual failure, we should just bail */ |
| 927 | if (rc < 0) { |
| 928 | ntb_link_disable(ndev); |
| 929 | return; |
| 930 | } |
| 931 | |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 932 | out: |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 933 | if (ntb_link_is_up(ndev, NULL, NULL) == 1) |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 934 | schedule_delayed_work(&nt->link_work, |
| 935 | msecs_to_jiffies(NTB_LINK_DOWN_TIMEOUT)); |
| 936 | } |
| 937 | |
| 938 | static void ntb_qp_link_work(struct work_struct *work) |
| 939 | { |
| 940 | struct ntb_transport_qp *qp = container_of(work, |
| 941 | struct ntb_transport_qp, |
| 942 | link_work.work); |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 943 | struct pci_dev *pdev = qp->ndev->pdev; |
| 944 | struct ntb_transport_ctx *nt = qp->transport; |
| 945 | int val; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 946 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 947 | WARN_ON(!nt->link_is_up); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 948 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 949 | val = ntb_spad_read(nt->ndev, QP_LINKS); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 950 | |
Serge Semin | d67288a | 2017-01-11 03:13:20 +0300 | [diff] [blame] | 951 | ntb_peer_spad_write(nt->ndev, PIDX, QP_LINKS, val | BIT(qp->qp_num)); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 952 | |
| 953 | /* query remote spad for qp ready bits */ |
Allen Hubbe | 2876228 | 2015-05-11 10:08:26 -0400 | [diff] [blame] | 954 | dev_dbg_ratelimited(&pdev->dev, "Remote QP link status = %x\n", val); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 955 | |
| 956 | /* See if the remote side is up */ |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 957 | if (val & BIT(qp->qp_num)) { |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 958 | dev_info(&pdev->dev, "qp %d: Link Up\n", qp->qp_num); |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 959 | qp->link_is_up = true; |
Dave Jiang | e902133 | 2016-02-23 09:11:36 -0700 | [diff] [blame] | 960 | qp->active = true; |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 961 | |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 962 | if (qp->event_handler) |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 963 | qp->event_handler(qp->cb_data, qp->link_is_up); |
Allen Hubbe | 8b5a22d | 2015-07-13 08:07:12 -0400 | [diff] [blame] | 964 | |
Dave Jiang | e902133 | 2016-02-23 09:11:36 -0700 | [diff] [blame] | 965 | if (qp->active) |
| 966 | tasklet_schedule(&qp->rxc_db_work); |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 967 | } else if (nt->link_is_up) |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 968 | schedule_delayed_work(&qp->link_work, |
| 969 | msecs_to_jiffies(NTB_LINK_DOWN_TIMEOUT)); |
| 970 | } |
| 971 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 972 | static int ntb_transport_init_queue(struct ntb_transport_ctx *nt, |
Jon Mason | 53ca4fe | 2013-11-26 11:21:50 -0700 | [diff] [blame] | 973 | unsigned int qp_num) |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 974 | { |
| 975 | struct ntb_transport_qp *qp; |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 976 | phys_addr_t mw_base; |
| 977 | resource_size_t mw_size; |
Jon Mason | ef114ed | 2013-01-19 02:02:18 -0700 | [diff] [blame] | 978 | unsigned int num_qps_mw, tx_size; |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 979 | unsigned int mw_num, mw_count, qp_count; |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 980 | u64 qp_offset; |
Jon Mason | 948d3a6 | 2013-04-18 17:07:36 -0700 | [diff] [blame] | 981 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 982 | mw_count = nt->mw_count; |
| 983 | qp_count = nt->qp_count; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 984 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 985 | mw_num = QP_TO_MW(nt, qp_num); |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 986 | |
| 987 | qp = &nt->qp_vec[qp_num]; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 988 | qp->qp_num = qp_num; |
| 989 | qp->transport = nt; |
| 990 | qp->ndev = nt->ndev; |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 991 | qp->client_ready = false; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 992 | qp->event_handler = NULL; |
Allen Hubbe | 2849b5d7 | 2015-05-12 08:09:15 -0400 | [diff] [blame] | 993 | ntb_qp_link_down_reset(qp); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 994 | |
Logan Gunthorpe | 8e8496e | 2017-06-05 14:00:53 -0600 | [diff] [blame] | 995 | if (mw_num < qp_count % mw_count) |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 996 | num_qps_mw = qp_count / mw_count + 1; |
Jon Mason | ef114ed | 2013-01-19 02:02:18 -0700 | [diff] [blame] | 997 | else |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 998 | num_qps_mw = qp_count / mw_count; |
Jon Mason | ef114ed | 2013-01-19 02:02:18 -0700 | [diff] [blame] | 999 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 1000 | mw_base = nt->mw_vec[mw_num].phys_addr; |
| 1001 | mw_size = nt->mw_vec[mw_num].phys_size; |
| 1002 | |
| 1003 | tx_size = (unsigned int)mw_size / num_qps_mw; |
Jon Mason | c92ba3c | 2015-10-04 22:54:22 -0400 | [diff] [blame] | 1004 | qp_offset = tx_size * (qp_num / mw_count); |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 1005 | |
| 1006 | qp->tx_mw = nt->mw_vec[mw_num].vbase + qp_offset; |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 1007 | if (!qp->tx_mw) |
| 1008 | return -EINVAL; |
Jon Mason | 793c20e | 2013-01-19 02:02:26 -0700 | [diff] [blame] | 1009 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 1010 | qp->tx_mw_phys = mw_base + qp_offset; |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 1011 | if (!qp->tx_mw_phys) |
| 1012 | return -EINVAL; |
| 1013 | |
| 1014 | tx_size -= sizeof(struct ntb_rx_info); |
| 1015 | qp->rx_info = qp->tx_mw + tx_size; |
| 1016 | |
Jon Mason | c9d534c | 2013-02-01 15:45:16 -0700 | [diff] [blame] | 1017 | /* Due to housekeeping, there must be atleast 2 buffs */ |
| 1018 | qp->tx_max_frame = min(transport_mtu, tx_size / 2); |
Jon Mason | 793c20e | 2013-01-19 02:02:26 -0700 | [diff] [blame] | 1019 | qp->tx_max_entry = tx_size / qp->tx_max_frame; |
Jon Mason | ef114ed | 2013-01-19 02:02:18 -0700 | [diff] [blame] | 1020 | |
Dave Jiang | c8650fd | 2015-07-13 08:07:09 -0400 | [diff] [blame] | 1021 | if (nt->debugfs_node_dir) { |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1022 | char debugfs_name[4]; |
| 1023 | |
| 1024 | snprintf(debugfs_name, 4, "qp%d", qp_num); |
| 1025 | qp->debugfs_dir = debugfs_create_dir(debugfs_name, |
Dave Jiang | c8650fd | 2015-07-13 08:07:09 -0400 | [diff] [blame] | 1026 | nt->debugfs_node_dir); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1027 | |
| 1028 | qp->debugfs_stats = debugfs_create_file("stats", S_IRUSR, |
| 1029 | qp->debugfs_dir, qp, |
| 1030 | &ntb_qp_debugfs_stats); |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 1031 | } else { |
| 1032 | qp->debugfs_dir = NULL; |
| 1033 | qp->debugfs_stats = NULL; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1034 | } |
| 1035 | |
| 1036 | INIT_DELAYED_WORK(&qp->link_work, ntb_qp_link_work); |
Jon Mason | fca4d51 | 2013-09-09 13:39:55 -0700 | [diff] [blame] | 1037 | INIT_WORK(&qp->link_cleanup, ntb_qp_link_cleanup_work); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1038 | |
Allen Hubbe | da2e5ae | 2015-07-13 08:07:08 -0400 | [diff] [blame] | 1039 | spin_lock_init(&qp->ntb_rx_q_lock); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1040 | spin_lock_init(&qp->ntb_tx_free_q_lock); |
| 1041 | |
Allen Hubbe | da2e5ae | 2015-07-13 08:07:08 -0400 | [diff] [blame] | 1042 | INIT_LIST_HEAD(&qp->rx_post_q); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1043 | INIT_LIST_HEAD(&qp->rx_pend_q); |
| 1044 | INIT_LIST_HEAD(&qp->rx_free_q); |
| 1045 | INIT_LIST_HEAD(&qp->tx_free_q); |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 1046 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 1047 | tasklet_init(&qp->rxc_db_work, ntb_transport_rxc_db, |
| 1048 | (unsigned long)qp); |
| 1049 | |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 1050 | return 0; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1051 | } |
| 1052 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 1053 | static int ntb_transport_probe(struct ntb_client *self, struct ntb_dev *ndev) |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1054 | { |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 1055 | struct ntb_transport_ctx *nt; |
| 1056 | struct ntb_transport_mw *mw; |
Shyam Sundar S K | b17faba | 2016-12-07 22:37:05 +0530 | [diff] [blame] | 1057 | unsigned int mw_count, qp_count, spad_count, max_mw_count_for_spads; |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 1058 | u64 qp_bitmap; |
Allen Hubbe | 1199aa6 | 2015-05-18 06:20:47 -0400 | [diff] [blame] | 1059 | int node; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1060 | int rc, i; |
| 1061 | |
Serge Semin | 443b9a1 | 2017-01-11 03:11:33 +0300 | [diff] [blame] | 1062 | mw_count = ntb_mw_count(ndev, PIDX); |
| 1063 | |
| 1064 | if (!ndev->ops->mw_set_trans) { |
| 1065 | dev_err(&ndev->dev, "Inbound MW based NTB API is required\n"); |
| 1066 | return -EINVAL; |
| 1067 | } |
Logan Gunthorpe | 19645a0 | 2016-06-07 11:20:22 -0600 | [diff] [blame] | 1068 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 1069 | if (ntb_db_is_unsafe(ndev)) |
| 1070 | dev_dbg(&ndev->dev, |
| 1071 | "doorbell is unsafe, proceed anyway...\n"); |
| 1072 | if (ntb_spad_is_unsafe(ndev)) |
| 1073 | dev_dbg(&ndev->dev, |
| 1074 | "scratchpad is unsafe, proceed anyway...\n"); |
| 1075 | |
Serge Semin | 1e53011 | 2016-12-14 02:49:14 +0300 | [diff] [blame] | 1076 | if (ntb_peer_port_count(ndev) != NTB_DEF_PEER_CNT) |
| 1077 | dev_warn(&ndev->dev, "Multi-port NTB devices unsupported\n"); |
| 1078 | |
Allen Hubbe | 1199aa6 | 2015-05-18 06:20:47 -0400 | [diff] [blame] | 1079 | node = dev_to_node(&ndev->dev); |
| 1080 | |
| 1081 | nt = kzalloc_node(sizeof(*nt), GFP_KERNEL, node); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1082 | if (!nt) |
| 1083 | return -ENOMEM; |
| 1084 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 1085 | nt->ndev = ndev; |
Shyam Sundar S K | b17faba | 2016-12-07 22:37:05 +0530 | [diff] [blame] | 1086 | spad_count = ntb_spad_count(ndev); |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 1087 | |
Shyam Sundar S K | b17faba | 2016-12-07 22:37:05 +0530 | [diff] [blame] | 1088 | /* Limit the MW's based on the availability of scratchpads */ |
| 1089 | |
| 1090 | if (spad_count < NTB_TRANSPORT_MIN_SPADS) { |
| 1091 | nt->mw_count = 0; |
| 1092 | rc = -EINVAL; |
| 1093 | goto err; |
| 1094 | } |
| 1095 | |
| 1096 | max_mw_count_for_spads = (spad_count - MW0_SZ_HIGH) / 2; |
| 1097 | nt->mw_count = min(mw_count, max_mw_count_for_spads); |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 1098 | |
Allen Hubbe | 1199aa6 | 2015-05-18 06:20:47 -0400 | [diff] [blame] | 1099 | nt->mw_vec = kzalloc_node(mw_count * sizeof(*nt->mw_vec), |
| 1100 | GFP_KERNEL, node); |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 1101 | if (!nt->mw_vec) { |
| 1102 | rc = -ENOMEM; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1103 | goto err; |
| 1104 | } |
| 1105 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 1106 | for (i = 0; i < mw_count; i++) { |
| 1107 | mw = &nt->mw_vec[i]; |
| 1108 | |
Serge Semin | 443b9a1 | 2017-01-11 03:11:33 +0300 | [diff] [blame] | 1109 | rc = ntb_mw_get_align(ndev, PIDX, i, &mw->xlat_align, |
| 1110 | &mw->xlat_align_size, NULL); |
| 1111 | if (rc) |
| 1112 | goto err1; |
| 1113 | |
| 1114 | rc = ntb_peer_mw_get_addr(ndev, i, &mw->phys_addr, |
| 1115 | &mw->phys_size); |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 1116 | if (rc) |
| 1117 | goto err1; |
| 1118 | |
Dave Jiang | 06917f7 | 2015-05-19 16:45:46 -0400 | [diff] [blame] | 1119 | mw->vbase = ioremap_wc(mw->phys_addr, mw->phys_size); |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 1120 | if (!mw->vbase) { |
| 1121 | rc = -ENOMEM; |
| 1122 | goto err1; |
| 1123 | } |
| 1124 | |
| 1125 | mw->buff_size = 0; |
| 1126 | mw->xlat_size = 0; |
| 1127 | mw->virt_addr = NULL; |
| 1128 | mw->dma_addr = 0; |
Jon Mason | 948d3a6 | 2013-04-18 17:07:36 -0700 | [diff] [blame] | 1129 | } |
| 1130 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 1131 | qp_bitmap = ntb_db_valid_mask(ndev); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1132 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 1133 | qp_count = ilog2(qp_bitmap); |
| 1134 | if (max_num_clients && max_num_clients < qp_count) |
| 1135 | qp_count = max_num_clients; |
Logan Gunthorpe | cb827ee | 2017-06-05 14:00:52 -0600 | [diff] [blame] | 1136 | else if (nt->mw_count < qp_count) |
| 1137 | qp_count = nt->mw_count; |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 1138 | |
| 1139 | qp_bitmap &= BIT_ULL(qp_count) - 1; |
| 1140 | |
| 1141 | nt->qp_count = qp_count; |
| 1142 | nt->qp_bitmap = qp_bitmap; |
| 1143 | nt->qp_bitmap_free = qp_bitmap; |
| 1144 | |
Allen Hubbe | 1199aa6 | 2015-05-18 06:20:47 -0400 | [diff] [blame] | 1145 | nt->qp_vec = kzalloc_node(qp_count * sizeof(*nt->qp_vec), |
| 1146 | GFP_KERNEL, node); |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 1147 | if (!nt->qp_vec) { |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1148 | rc = -ENOMEM; |
Sudip Mukherjee | d4adee0 | 2015-10-03 13:39:41 +0530 | [diff] [blame] | 1149 | goto err1; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1150 | } |
| 1151 | |
Dave Jiang | c8650fd | 2015-07-13 08:07:09 -0400 | [diff] [blame] | 1152 | if (nt_debugfs_dir) { |
| 1153 | nt->debugfs_node_dir = |
| 1154 | debugfs_create_dir(pci_name(ndev->pdev), |
| 1155 | nt_debugfs_dir); |
| 1156 | } |
| 1157 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 1158 | for (i = 0; i < qp_count; i++) { |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 1159 | rc = ntb_transport_init_queue(nt, i); |
| 1160 | if (rc) |
Sudip Mukherjee | d4adee0 | 2015-10-03 13:39:41 +0530 | [diff] [blame] | 1161 | goto err2; |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 1162 | } |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1163 | |
| 1164 | INIT_DELAYED_WORK(&nt->link_work, ntb_transport_link_work); |
Jon Mason | fca4d51 | 2013-09-09 13:39:55 -0700 | [diff] [blame] | 1165 | INIT_WORK(&nt->link_cleanup, ntb_transport_link_cleanup_work); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1166 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 1167 | rc = ntb_set_ctx(ndev, nt, &ntb_transport_ops); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1168 | if (rc) |
Sudip Mukherjee | d4adee0 | 2015-10-03 13:39:41 +0530 | [diff] [blame] | 1169 | goto err2; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1170 | |
| 1171 | INIT_LIST_HEAD(&nt->client_devs); |
| 1172 | rc = ntb_bus_init(nt); |
| 1173 | if (rc) |
Sudip Mukherjee | d4adee0 | 2015-10-03 13:39:41 +0530 | [diff] [blame] | 1174 | goto err3; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1175 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 1176 | nt->link_is_up = false; |
| 1177 | ntb_link_enable(ndev, NTB_SPEED_AUTO, NTB_WIDTH_AUTO); |
| 1178 | ntb_link_event(ndev); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1179 | |
| 1180 | return 0; |
| 1181 | |
Jon Mason | 948d3a6 | 2013-04-18 17:07:36 -0700 | [diff] [blame] | 1182 | err3: |
Sudip Mukherjee | d4adee0 | 2015-10-03 13:39:41 +0530 | [diff] [blame] | 1183 | ntb_clear_ctx(ndev); |
Jon Mason | 948d3a6 | 2013-04-18 17:07:36 -0700 | [diff] [blame] | 1184 | err2: |
Sudip Mukherjee | d4adee0 | 2015-10-03 13:39:41 +0530 | [diff] [blame] | 1185 | kfree(nt->qp_vec); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1186 | err1: |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 1187 | while (i--) { |
| 1188 | mw = &nt->mw_vec[i]; |
| 1189 | iounmap(mw->vbase); |
| 1190 | } |
Sudip Mukherjee | d4adee0 | 2015-10-03 13:39:41 +0530 | [diff] [blame] | 1191 | kfree(nt->mw_vec); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1192 | err: |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1193 | kfree(nt); |
| 1194 | return rc; |
| 1195 | } |
| 1196 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 1197 | static void ntb_transport_free(struct ntb_client *self, struct ntb_dev *ndev) |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1198 | { |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 1199 | struct ntb_transport_ctx *nt = ndev->ctx; |
| 1200 | struct ntb_transport_qp *qp; |
| 1201 | u64 qp_bitmap_alloc; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1202 | int i; |
| 1203 | |
Jon Mason | fca4d51 | 2013-09-09 13:39:55 -0700 | [diff] [blame] | 1204 | ntb_transport_link_cleanup(nt); |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 1205 | cancel_work_sync(&nt->link_cleanup); |
| 1206 | cancel_delayed_work_sync(&nt->link_work); |
| 1207 | |
| 1208 | qp_bitmap_alloc = nt->qp_bitmap & ~nt->qp_bitmap_free; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1209 | |
| 1210 | /* verify that all the qp's are freed */ |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 1211 | for (i = 0; i < nt->qp_count; i++) { |
| 1212 | qp = &nt->qp_vec[i]; |
| 1213 | if (qp_bitmap_alloc & BIT_ULL(i)) |
| 1214 | ntb_transport_free_queue(qp); |
| 1215 | debugfs_remove_recursive(qp->debugfs_dir); |
Jon Mason | 1517a3f | 2013-07-30 15:58:49 -0700 | [diff] [blame] | 1216 | } |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1217 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 1218 | ntb_link_disable(ndev); |
| 1219 | ntb_clear_ctx(ndev); |
| 1220 | |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1221 | ntb_bus_remove(nt); |
| 1222 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 1223 | for (i = nt->mw_count; i--; ) { |
Jon Mason | 113fc50 | 2013-01-30 11:40:52 -0700 | [diff] [blame] | 1224 | ntb_free_mw(nt, i); |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 1225 | iounmap(nt->mw_vec[i].vbase); |
| 1226 | } |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1227 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 1228 | kfree(nt->qp_vec); |
| 1229 | kfree(nt->mw_vec); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1230 | kfree(nt); |
| 1231 | } |
| 1232 | |
Allen Hubbe | da2e5ae | 2015-07-13 08:07:08 -0400 | [diff] [blame] | 1233 | static void ntb_complete_rxc(struct ntb_transport_qp *qp) |
| 1234 | { |
| 1235 | struct ntb_queue_entry *entry; |
| 1236 | void *cb_data; |
| 1237 | unsigned int len; |
| 1238 | unsigned long irqflags; |
| 1239 | |
| 1240 | spin_lock_irqsave(&qp->ntb_rx_q_lock, irqflags); |
| 1241 | |
| 1242 | while (!list_empty(&qp->rx_post_q)) { |
| 1243 | entry = list_first_entry(&qp->rx_post_q, |
| 1244 | struct ntb_queue_entry, entry); |
| 1245 | if (!(entry->flags & DESC_DONE_FLAG)) |
| 1246 | break; |
| 1247 | |
| 1248 | entry->rx_hdr->flags = 0; |
Dave Jiang | 7220357 | 2016-07-20 13:14:13 -0700 | [diff] [blame] | 1249 | iowrite32(entry->rx_index, &qp->rx_info->entry); |
Allen Hubbe | da2e5ae | 2015-07-13 08:07:08 -0400 | [diff] [blame] | 1250 | |
| 1251 | cb_data = entry->cb_data; |
| 1252 | len = entry->len; |
| 1253 | |
| 1254 | list_move_tail(&entry->entry, &qp->rx_free_q); |
| 1255 | |
| 1256 | spin_unlock_irqrestore(&qp->ntb_rx_q_lock, irqflags); |
| 1257 | |
| 1258 | if (qp->rx_handler && qp->client_ready) |
| 1259 | qp->rx_handler(qp, qp->cb_data, cb_data, len); |
| 1260 | |
| 1261 | spin_lock_irqsave(&qp->ntb_rx_q_lock, irqflags); |
| 1262 | } |
| 1263 | |
| 1264 | spin_unlock_irqrestore(&qp->ntb_rx_q_lock, irqflags); |
| 1265 | } |
| 1266 | |
Dave Jiang | 7220357 | 2016-07-20 13:14:13 -0700 | [diff] [blame] | 1267 | static void ntb_rx_copy_callback(void *data, |
| 1268 | const struct dmaengine_result *res) |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1269 | { |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 1270 | struct ntb_queue_entry *entry = data; |
Jon Mason | 448c6fb | 2013-01-19 02:02:27 -0700 | [diff] [blame] | 1271 | |
Dave Jiang | 7220357 | 2016-07-20 13:14:13 -0700 | [diff] [blame] | 1272 | /* we need to check DMA results if we are using DMA */ |
| 1273 | if (res) { |
| 1274 | enum dmaengine_tx_result dma_err = res->result; |
| 1275 | |
| 1276 | switch (dma_err) { |
| 1277 | case DMA_TRANS_READ_FAILED: |
| 1278 | case DMA_TRANS_WRITE_FAILED: |
| 1279 | entry->errors++; |
| 1280 | case DMA_TRANS_ABORTED: |
| 1281 | { |
| 1282 | struct ntb_transport_qp *qp = entry->qp; |
| 1283 | void *offset = qp->rx_buff + qp->rx_max_frame * |
| 1284 | qp->rx_index; |
| 1285 | |
| 1286 | ntb_memcpy_rx(entry, offset); |
| 1287 | qp->rx_memcpy++; |
| 1288 | return; |
| 1289 | } |
| 1290 | |
| 1291 | case DMA_TRANS_NOERROR: |
| 1292 | default: |
| 1293 | break; |
| 1294 | } |
| 1295 | } |
| 1296 | |
Allen Hubbe | da2e5ae | 2015-07-13 08:07:08 -0400 | [diff] [blame] | 1297 | entry->flags |= DESC_DONE_FLAG; |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 1298 | |
Allen Hubbe | da2e5ae | 2015-07-13 08:07:08 -0400 | [diff] [blame] | 1299 | ntb_complete_rxc(entry->qp); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1300 | } |
| 1301 | |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 1302 | static void ntb_memcpy_rx(struct ntb_queue_entry *entry, void *offset) |
| 1303 | { |
| 1304 | void *buf = entry->buf; |
| 1305 | size_t len = entry->len; |
| 1306 | |
| 1307 | memcpy(buf, offset, len); |
| 1308 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 1309 | /* Ensure that the data is fully copied out before clearing the flag */ |
| 1310 | wmb(); |
| 1311 | |
Dave Jiang | 7220357 | 2016-07-20 13:14:13 -0700 | [diff] [blame] | 1312 | ntb_rx_copy_callback(entry, NULL); |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 1313 | } |
| 1314 | |
Dave Jiang | 7220357 | 2016-07-20 13:14:13 -0700 | [diff] [blame] | 1315 | static int ntb_async_rx_submit(struct ntb_queue_entry *entry, void *offset) |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 1316 | { |
| 1317 | struct dma_async_tx_descriptor *txd; |
| 1318 | struct ntb_transport_qp *qp = entry->qp; |
Dave Jiang | 569410ca | 2015-07-13 08:07:22 -0400 | [diff] [blame] | 1319 | struct dma_chan *chan = qp->rx_dma_chan; |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 1320 | struct dma_device *device; |
Allen Hubbe | da2e5ae | 2015-07-13 08:07:08 -0400 | [diff] [blame] | 1321 | size_t pay_off, buff_off, len; |
Bartlomiej Zolnierkiewicz | 6f57fd0 | 2013-10-18 19:35:31 +0200 | [diff] [blame] | 1322 | struct dmaengine_unmap_data *unmap; |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 1323 | dma_cookie_t cookie; |
| 1324 | void *buf = entry->buf; |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 1325 | |
Allen Hubbe | da2e5ae | 2015-07-13 08:07:08 -0400 | [diff] [blame] | 1326 | len = entry->len; |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 1327 | device = chan->device; |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 1328 | pay_off = (size_t)offset & ~PAGE_MASK; |
| 1329 | buff_off = (size_t)buf & ~PAGE_MASK; |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 1330 | |
| 1331 | if (!is_dma_copy_aligned(device, pay_off, buff_off, len)) |
Allen Hubbe | 905921e | 2015-07-13 08:07:21 -0400 | [diff] [blame] | 1332 | goto err; |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 1333 | |
Bartlomiej Zolnierkiewicz | 6f57fd0 | 2013-10-18 19:35:31 +0200 | [diff] [blame] | 1334 | unmap = dmaengine_get_unmap_data(device->dev, 2, GFP_NOWAIT); |
| 1335 | if (!unmap) |
Allen Hubbe | 905921e | 2015-07-13 08:07:21 -0400 | [diff] [blame] | 1336 | goto err; |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 1337 | |
Bartlomiej Zolnierkiewicz | 6f57fd0 | 2013-10-18 19:35:31 +0200 | [diff] [blame] | 1338 | unmap->len = len; |
| 1339 | unmap->addr[0] = dma_map_page(device->dev, virt_to_page(offset), |
| 1340 | pay_off, len, DMA_TO_DEVICE); |
| 1341 | if (dma_mapping_error(device->dev, unmap->addr[0])) |
| 1342 | goto err_get_unmap; |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 1343 | |
Bartlomiej Zolnierkiewicz | 6f57fd0 | 2013-10-18 19:35:31 +0200 | [diff] [blame] | 1344 | unmap->to_cnt = 1; |
| 1345 | |
| 1346 | unmap->addr[1] = dma_map_page(device->dev, virt_to_page(buf), |
| 1347 | buff_off, len, DMA_FROM_DEVICE); |
| 1348 | if (dma_mapping_error(device->dev, unmap->addr[1])) |
| 1349 | goto err_get_unmap; |
| 1350 | |
| 1351 | unmap->from_cnt = 1; |
| 1352 | |
Allen Hubbe | 88931ec | 2017-06-09 18:06:36 -0400 | [diff] [blame] | 1353 | txd = device->device_prep_dma_memcpy(chan, unmap->addr[1], |
| 1354 | unmap->addr[0], len, |
| 1355 | DMA_PREP_INTERRUPT); |
| 1356 | if (!txd) |
Bartlomiej Zolnierkiewicz | 6f57fd0 | 2013-10-18 19:35:31 +0200 | [diff] [blame] | 1357 | goto err_get_unmap; |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 1358 | |
Dave Jiang | 7220357 | 2016-07-20 13:14:13 -0700 | [diff] [blame] | 1359 | txd->callback_result = ntb_rx_copy_callback; |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 1360 | txd->callback_param = entry; |
Bartlomiej Zolnierkiewicz | 6f57fd0 | 2013-10-18 19:35:31 +0200 | [diff] [blame] | 1361 | dma_set_unmap(txd, unmap); |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 1362 | |
| 1363 | cookie = dmaengine_submit(txd); |
| 1364 | if (dma_submit_error(cookie)) |
Bartlomiej Zolnierkiewicz | 6f57fd0 | 2013-10-18 19:35:31 +0200 | [diff] [blame] | 1365 | goto err_set_unmap; |
| 1366 | |
| 1367 | dmaengine_unmap_put(unmap); |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 1368 | |
| 1369 | qp->last_cookie = cookie; |
| 1370 | |
| 1371 | qp->rx_async++; |
| 1372 | |
Dave Jiang | 7220357 | 2016-07-20 13:14:13 -0700 | [diff] [blame] | 1373 | return 0; |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 1374 | |
Bartlomiej Zolnierkiewicz | 6f57fd0 | 2013-10-18 19:35:31 +0200 | [diff] [blame] | 1375 | err_set_unmap: |
| 1376 | dmaengine_unmap_put(unmap); |
| 1377 | err_get_unmap: |
| 1378 | dmaengine_unmap_put(unmap); |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 1379 | err: |
Dave Jiang | 7220357 | 2016-07-20 13:14:13 -0700 | [diff] [blame] | 1380 | return -ENXIO; |
| 1381 | } |
| 1382 | |
| 1383 | static void ntb_async_rx(struct ntb_queue_entry *entry, void *offset) |
| 1384 | { |
| 1385 | struct ntb_transport_qp *qp = entry->qp; |
| 1386 | struct dma_chan *chan = qp->rx_dma_chan; |
| 1387 | int res; |
| 1388 | |
| 1389 | if (!chan) |
| 1390 | goto err; |
| 1391 | |
| 1392 | if (entry->len < copy_bytes) |
| 1393 | goto err; |
| 1394 | |
| 1395 | res = ntb_async_rx_submit(entry, offset); |
| 1396 | if (res < 0) |
| 1397 | goto err; |
| 1398 | |
| 1399 | if (!entry->retries) |
| 1400 | qp->rx_async++; |
| 1401 | |
| 1402 | return; |
| 1403 | |
| 1404 | err: |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 1405 | ntb_memcpy_rx(entry, offset); |
| 1406 | qp->rx_memcpy++; |
| 1407 | } |
| 1408 | |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1409 | static int ntb_process_rxc(struct ntb_transport_qp *qp) |
| 1410 | { |
| 1411 | struct ntb_payload_header *hdr; |
| 1412 | struct ntb_queue_entry *entry; |
| 1413 | void *offset; |
| 1414 | |
Jon Mason | 793c20e | 2013-01-19 02:02:26 -0700 | [diff] [blame] | 1415 | offset = qp->rx_buff + qp->rx_max_frame * qp->rx_index; |
| 1416 | hdr = offset + qp->rx_max_frame - sizeof(struct ntb_payload_header); |
| 1417 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 1418 | dev_dbg(&qp->ndev->pdev->dev, "qp %d: RX ver %u len %d flags %x\n", |
| 1419 | qp->qp_num, hdr->ver, hdr->len, hdr->flags); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1420 | |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1421 | if (!(hdr->flags & DESC_DONE_FLAG)) { |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 1422 | dev_dbg(&qp->ndev->pdev->dev, "done flag not set\n"); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1423 | qp->rx_ring_empty++; |
| 1424 | return -EAGAIN; |
| 1425 | } |
| 1426 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 1427 | if (hdr->flags & LINK_DOWN_FLAG) { |
| 1428 | dev_dbg(&qp->ndev->pdev->dev, "link down flag set\n"); |
| 1429 | ntb_qp_link_down(qp); |
| 1430 | hdr->flags = 0; |
Allen Hubbe | c0900b3 | 2015-05-12 06:24:27 -0400 | [diff] [blame] | 1431 | return -EAGAIN; |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 1432 | } |
| 1433 | |
| 1434 | if (hdr->ver != (u32)qp->rx_pkts) { |
| 1435 | dev_dbg(&qp->ndev->pdev->dev, |
| 1436 | "version mismatch, expected %llu - got %u\n", |
| 1437 | qp->rx_pkts, hdr->ver); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1438 | qp->rx_err_ver++; |
| 1439 | return -EIO; |
| 1440 | } |
| 1441 | |
Allen Hubbe | da2e5ae | 2015-07-13 08:07:08 -0400 | [diff] [blame] | 1442 | entry = ntb_list_mv(&qp->ntb_rx_q_lock, &qp->rx_pend_q, &qp->rx_post_q); |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 1443 | if (!entry) { |
| 1444 | dev_dbg(&qp->ndev->pdev->dev, "no receive buffer\n"); |
| 1445 | qp->rx_err_no_buf++; |
Allen Hubbe | da2e5ae | 2015-07-13 08:07:08 -0400 | [diff] [blame] | 1446 | return -EAGAIN; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1447 | } |
| 1448 | |
Allen Hubbe | da2e5ae | 2015-07-13 08:07:08 -0400 | [diff] [blame] | 1449 | entry->rx_hdr = hdr; |
Dave Jiang | 7220357 | 2016-07-20 13:14:13 -0700 | [diff] [blame] | 1450 | entry->rx_index = qp->rx_index; |
Allen Hubbe | da2e5ae | 2015-07-13 08:07:08 -0400 | [diff] [blame] | 1451 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 1452 | if (hdr->len > entry->len) { |
| 1453 | dev_dbg(&qp->ndev->pdev->dev, |
| 1454 | "receive buffer overflow! Wanted %d got %d\n", |
| 1455 | hdr->len, entry->len); |
| 1456 | qp->rx_err_oflow++; |
| 1457 | |
Allen Hubbe | da2e5ae | 2015-07-13 08:07:08 -0400 | [diff] [blame] | 1458 | entry->len = -EIO; |
| 1459 | entry->flags |= DESC_DONE_FLAG; |
| 1460 | |
| 1461 | ntb_complete_rxc(qp); |
| 1462 | } else { |
| 1463 | dev_dbg(&qp->ndev->pdev->dev, |
| 1464 | "RX OK index %u ver %u size %d into buf size %d\n", |
| 1465 | qp->rx_index, hdr->ver, hdr->len, entry->len); |
| 1466 | |
| 1467 | qp->rx_bytes += hdr->len; |
| 1468 | qp->rx_pkts++; |
| 1469 | |
| 1470 | entry->len = hdr->len; |
| 1471 | |
| 1472 | ntb_async_rx(entry, offset); |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 1473 | } |
| 1474 | |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 1475 | qp->rx_index++; |
| 1476 | qp->rx_index %= qp->rx_max_entry; |
| 1477 | |
| 1478 | return 0; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1479 | } |
| 1480 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 1481 | static void ntb_transport_rxc_db(unsigned long data) |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1482 | { |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 1483 | struct ntb_transport_qp *qp = (void *)data; |
Jon Mason | c336acd | 2013-01-17 15:28:45 -0700 | [diff] [blame] | 1484 | int rc, i; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1485 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 1486 | dev_dbg(&qp->ndev->pdev->dev, "%s: doorbell %d received\n", |
| 1487 | __func__, qp->qp_num); |
Jon Mason | e8aeb60 | 2013-04-18 17:59:44 -0700 | [diff] [blame] | 1488 | |
Jon Mason | c336acd | 2013-01-17 15:28:45 -0700 | [diff] [blame] | 1489 | /* Limit the number of packets processed in a single interrupt to |
| 1490 | * provide fairness to others |
| 1491 | */ |
| 1492 | for (i = 0; i < qp->rx_max_entry; i++) { |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1493 | rc = ntb_process_rxc(qp); |
Jon Mason | c336acd | 2013-01-17 15:28:45 -0700 | [diff] [blame] | 1494 | if (rc) |
| 1495 | break; |
| 1496 | } |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 1497 | |
Dave Jiang | 569410ca | 2015-07-13 08:07:22 -0400 | [diff] [blame] | 1498 | if (i && qp->rx_dma_chan) |
| 1499 | dma_async_issue_pending(qp->rx_dma_chan); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1500 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 1501 | if (i == qp->rx_max_entry) { |
| 1502 | /* there is more work to do */ |
Dave Jiang | e902133 | 2016-02-23 09:11:36 -0700 | [diff] [blame] | 1503 | if (qp->active) |
| 1504 | tasklet_schedule(&qp->rxc_db_work); |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 1505 | } else if (ntb_db_read(qp->ndev) & BIT_ULL(qp->qp_num)) { |
| 1506 | /* the doorbell bit is set: clear it */ |
| 1507 | ntb_db_clear(qp->ndev, BIT_ULL(qp->qp_num)); |
| 1508 | /* ntb_db_read ensures ntb_db_clear write is committed */ |
| 1509 | ntb_db_read(qp->ndev); |
| 1510 | |
| 1511 | /* an interrupt may have arrived between finishing |
| 1512 | * ntb_process_rxc and clearing the doorbell bit: |
| 1513 | * there might be some more work to do. |
| 1514 | */ |
Dave Jiang | e902133 | 2016-02-23 09:11:36 -0700 | [diff] [blame] | 1515 | if (qp->active) |
| 1516 | tasklet_schedule(&qp->rxc_db_work); |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 1517 | } |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1518 | } |
| 1519 | |
Dave Jiang | 9cabc26 | 2016-07-20 13:14:07 -0700 | [diff] [blame] | 1520 | static void ntb_tx_copy_callback(void *data, |
| 1521 | const struct dmaengine_result *res) |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1522 | { |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 1523 | struct ntb_queue_entry *entry = data; |
| 1524 | struct ntb_transport_qp *qp = entry->qp; |
| 1525 | struct ntb_payload_header __iomem *hdr = entry->tx_hdr; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1526 | |
Dave Jiang | 9cabc26 | 2016-07-20 13:14:07 -0700 | [diff] [blame] | 1527 | /* we need to check DMA results if we are using DMA */ |
| 1528 | if (res) { |
| 1529 | enum dmaengine_tx_result dma_err = res->result; |
| 1530 | |
| 1531 | switch (dma_err) { |
| 1532 | case DMA_TRANS_READ_FAILED: |
| 1533 | case DMA_TRANS_WRITE_FAILED: |
| 1534 | entry->errors++; |
| 1535 | case DMA_TRANS_ABORTED: |
| 1536 | { |
| 1537 | void __iomem *offset = |
| 1538 | qp->tx_mw + qp->tx_max_frame * |
| 1539 | entry->tx_index; |
| 1540 | |
| 1541 | /* resubmit via CPU */ |
| 1542 | ntb_memcpy_tx(entry, offset); |
| 1543 | qp->tx_memcpy++; |
| 1544 | return; |
| 1545 | } |
| 1546 | |
| 1547 | case DMA_TRANS_NOERROR: |
| 1548 | default: |
| 1549 | break; |
| 1550 | } |
| 1551 | } |
| 1552 | |
Jon Mason | 7446564 | 2013-01-21 15:28:52 -0700 | [diff] [blame] | 1553 | iowrite32(entry->flags | DESC_DONE_FLAG, &hdr->flags); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1554 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 1555 | ntb_peer_db_set(qp->ndev, BIT_ULL(qp->qp_num)); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1556 | |
| 1557 | /* The entry length can only be zero if the packet is intended to be a |
| 1558 | * "link down" or similar. Since no payload is being sent in these |
| 1559 | * cases, there is nothing to add to the completion queue. |
| 1560 | */ |
| 1561 | if (entry->len > 0) { |
| 1562 | qp->tx_bytes += entry->len; |
| 1563 | |
| 1564 | if (qp->tx_handler) |
| 1565 | qp->tx_handler(qp, qp->cb_data, entry->cb_data, |
| 1566 | entry->len); |
| 1567 | } |
| 1568 | |
| 1569 | ntb_list_add(&qp->ntb_tx_free_q_lock, &entry->entry, &qp->tx_free_q); |
| 1570 | } |
| 1571 | |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 1572 | static void ntb_memcpy_tx(struct ntb_queue_entry *entry, void __iomem *offset) |
| 1573 | { |
Dave Jiang | 06917f7 | 2015-05-19 16:45:46 -0400 | [diff] [blame] | 1574 | #ifdef ARCH_HAS_NOCACHE_UACCESS |
| 1575 | /* |
| 1576 | * Using non-temporal mov to improve performance on non-cached |
| 1577 | * writes, even though we aren't actually copying from user space. |
| 1578 | */ |
| 1579 | __copy_from_user_inatomic_nocache(offset, entry->buf, entry->len); |
| 1580 | #else |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 1581 | memcpy_toio(offset, entry->buf, entry->len); |
Dave Jiang | 06917f7 | 2015-05-19 16:45:46 -0400 | [diff] [blame] | 1582 | #endif |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 1583 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 1584 | /* Ensure that the data is fully copied out before setting the flags */ |
| 1585 | wmb(); |
| 1586 | |
Dave Jiang | 9cabc26 | 2016-07-20 13:14:07 -0700 | [diff] [blame] | 1587 | ntb_tx_copy_callback(entry, NULL); |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 1588 | } |
| 1589 | |
Dave Jiang | 9cabc26 | 2016-07-20 13:14:07 -0700 | [diff] [blame] | 1590 | static int ntb_async_tx_submit(struct ntb_transport_qp *qp, |
| 1591 | struct ntb_queue_entry *entry) |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 1592 | { |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 1593 | struct dma_async_tx_descriptor *txd; |
Dave Jiang | 569410ca | 2015-07-13 08:07:22 -0400 | [diff] [blame] | 1594 | struct dma_chan *chan = qp->tx_dma_chan; |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 1595 | struct dma_device *device; |
Dave Jiang | 9cabc26 | 2016-07-20 13:14:07 -0700 | [diff] [blame] | 1596 | size_t len = entry->len; |
| 1597 | void *buf = entry->buf; |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 1598 | size_t dest_off, buff_off; |
Bartlomiej Zolnierkiewicz | 6f57fd0 | 2013-10-18 19:35:31 +0200 | [diff] [blame] | 1599 | struct dmaengine_unmap_data *unmap; |
| 1600 | dma_addr_t dest; |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 1601 | dma_cookie_t cookie; |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 1602 | |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 1603 | device = chan->device; |
Dave Jiang | 9cabc26 | 2016-07-20 13:14:07 -0700 | [diff] [blame] | 1604 | dest = qp->tx_mw_phys + qp->tx_max_frame * entry->tx_index; |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 1605 | buff_off = (size_t)buf & ~PAGE_MASK; |
| 1606 | dest_off = (size_t)dest & ~PAGE_MASK; |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 1607 | |
| 1608 | if (!is_dma_copy_aligned(device, buff_off, dest_off, len)) |
| 1609 | goto err; |
| 1610 | |
Bartlomiej Zolnierkiewicz | 6f57fd0 | 2013-10-18 19:35:31 +0200 | [diff] [blame] | 1611 | unmap = dmaengine_get_unmap_data(device->dev, 1, GFP_NOWAIT); |
| 1612 | if (!unmap) |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 1613 | goto err; |
| 1614 | |
Bartlomiej Zolnierkiewicz | 6f57fd0 | 2013-10-18 19:35:31 +0200 | [diff] [blame] | 1615 | unmap->len = len; |
| 1616 | unmap->addr[0] = dma_map_page(device->dev, virt_to_page(buf), |
| 1617 | buff_off, len, DMA_TO_DEVICE); |
| 1618 | if (dma_mapping_error(device->dev, unmap->addr[0])) |
| 1619 | goto err_get_unmap; |
| 1620 | |
| 1621 | unmap->to_cnt = 1; |
| 1622 | |
Allen Hubbe | 88931ec | 2017-06-09 18:06:36 -0400 | [diff] [blame] | 1623 | txd = device->device_prep_dma_memcpy(chan, dest, unmap->addr[0], len, |
| 1624 | DMA_PREP_INTERRUPT); |
| 1625 | if (!txd) |
Bartlomiej Zolnierkiewicz | 6f57fd0 | 2013-10-18 19:35:31 +0200 | [diff] [blame] | 1626 | goto err_get_unmap; |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 1627 | |
Dave Jiang | 9cabc26 | 2016-07-20 13:14:07 -0700 | [diff] [blame] | 1628 | txd->callback_result = ntb_tx_copy_callback; |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 1629 | txd->callback_param = entry; |
Bartlomiej Zolnierkiewicz | 6f57fd0 | 2013-10-18 19:35:31 +0200 | [diff] [blame] | 1630 | dma_set_unmap(txd, unmap); |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 1631 | |
| 1632 | cookie = dmaengine_submit(txd); |
| 1633 | if (dma_submit_error(cookie)) |
Bartlomiej Zolnierkiewicz | 6f57fd0 | 2013-10-18 19:35:31 +0200 | [diff] [blame] | 1634 | goto err_set_unmap; |
| 1635 | |
| 1636 | dmaengine_unmap_put(unmap); |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 1637 | |
| 1638 | dma_async_issue_pending(chan); |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 1639 | |
Dave Jiang | 9cabc26 | 2016-07-20 13:14:07 -0700 | [diff] [blame] | 1640 | return 0; |
Bartlomiej Zolnierkiewicz | 6f57fd0 | 2013-10-18 19:35:31 +0200 | [diff] [blame] | 1641 | err_set_unmap: |
| 1642 | dmaengine_unmap_put(unmap); |
| 1643 | err_get_unmap: |
| 1644 | dmaengine_unmap_put(unmap); |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 1645 | err: |
Dave Jiang | 9cabc26 | 2016-07-20 13:14:07 -0700 | [diff] [blame] | 1646 | return -ENXIO; |
| 1647 | } |
| 1648 | |
| 1649 | static void ntb_async_tx(struct ntb_transport_qp *qp, |
| 1650 | struct ntb_queue_entry *entry) |
| 1651 | { |
| 1652 | struct ntb_payload_header __iomem *hdr; |
| 1653 | struct dma_chan *chan = qp->tx_dma_chan; |
| 1654 | void __iomem *offset; |
| 1655 | int res; |
| 1656 | |
| 1657 | entry->tx_index = qp->tx_index; |
| 1658 | offset = qp->tx_mw + qp->tx_max_frame * entry->tx_index; |
| 1659 | hdr = offset + qp->tx_max_frame - sizeof(struct ntb_payload_header); |
| 1660 | entry->tx_hdr = hdr; |
| 1661 | |
| 1662 | iowrite32(entry->len, &hdr->len); |
| 1663 | iowrite32((u32)qp->tx_pkts, &hdr->ver); |
| 1664 | |
| 1665 | if (!chan) |
| 1666 | goto err; |
| 1667 | |
| 1668 | if (entry->len < copy_bytes) |
| 1669 | goto err; |
| 1670 | |
| 1671 | res = ntb_async_tx_submit(qp, entry); |
| 1672 | if (res < 0) |
| 1673 | goto err; |
| 1674 | |
| 1675 | if (!entry->retries) |
| 1676 | qp->tx_async++; |
| 1677 | |
| 1678 | return; |
| 1679 | |
| 1680 | err: |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 1681 | ntb_memcpy_tx(entry, offset); |
| 1682 | qp->tx_memcpy++; |
| 1683 | } |
| 1684 | |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1685 | static int ntb_process_tx(struct ntb_transport_qp *qp, |
| 1686 | struct ntb_queue_entry *entry) |
| 1687 | { |
Jon Mason | 793c20e | 2013-01-19 02:02:26 -0700 | [diff] [blame] | 1688 | if (qp->tx_index == qp->remote_rx_info->entry) { |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1689 | qp->tx_ring_full++; |
| 1690 | return -EAGAIN; |
| 1691 | } |
| 1692 | |
Jon Mason | ef114ed | 2013-01-19 02:02:18 -0700 | [diff] [blame] | 1693 | if (entry->len > qp->tx_max_frame - sizeof(struct ntb_payload_header)) { |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1694 | if (qp->tx_handler) |
Jon Mason | 179f912 | 2015-12-18 13:22:37 -0500 | [diff] [blame] | 1695 | qp->tx_handler(qp, qp->cb_data, NULL, -EIO); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1696 | |
| 1697 | ntb_list_add(&qp->ntb_tx_free_q_lock, &entry->entry, |
| 1698 | &qp->tx_free_q); |
| 1699 | return 0; |
| 1700 | } |
| 1701 | |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 1702 | ntb_async_tx(qp, entry); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1703 | |
Jon Mason | 793c20e | 2013-01-19 02:02:26 -0700 | [diff] [blame] | 1704 | qp->tx_index++; |
| 1705 | qp->tx_index %= qp->tx_max_entry; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1706 | |
| 1707 | qp->tx_pkts++; |
| 1708 | |
| 1709 | return 0; |
| 1710 | } |
| 1711 | |
| 1712 | static void ntb_send_link_down(struct ntb_transport_qp *qp) |
| 1713 | { |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 1714 | struct pci_dev *pdev = qp->ndev->pdev; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1715 | struct ntb_queue_entry *entry; |
| 1716 | int i, rc; |
| 1717 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 1718 | if (!qp->link_is_up) |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1719 | return; |
| 1720 | |
Allen Hubbe | e22e0b9 | 2015-05-12 06:55:44 -0400 | [diff] [blame] | 1721 | dev_info(&pdev->dev, "qp %d: Send Link Down\n", qp->qp_num); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1722 | |
| 1723 | for (i = 0; i < NTB_LINK_DOWN_TIMEOUT; i++) { |
Jon Mason | f766755 | 2013-01-19 02:02:24 -0700 | [diff] [blame] | 1724 | entry = ntb_list_rm(&qp->ntb_tx_free_q_lock, &qp->tx_free_q); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1725 | if (entry) |
| 1726 | break; |
| 1727 | msleep(100); |
| 1728 | } |
| 1729 | |
| 1730 | if (!entry) |
| 1731 | return; |
| 1732 | |
| 1733 | entry->cb_data = NULL; |
| 1734 | entry->buf = NULL; |
| 1735 | entry->len = 0; |
| 1736 | entry->flags = LINK_DOWN_FLAG; |
| 1737 | |
| 1738 | rc = ntb_process_tx(qp, entry); |
| 1739 | if (rc) |
| 1740 | dev_err(&pdev->dev, "ntb: QP%d unable to send linkdown msg\n", |
| 1741 | qp->qp_num); |
Allen Hubbe | 2849b5d7 | 2015-05-12 08:09:15 -0400 | [diff] [blame] | 1742 | |
| 1743 | ntb_qp_link_down_reset(qp); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1744 | } |
| 1745 | |
Allen Hubbe | 1199aa6 | 2015-05-18 06:20:47 -0400 | [diff] [blame] | 1746 | static bool ntb_dma_filter_fn(struct dma_chan *chan, void *node) |
| 1747 | { |
| 1748 | return dev_to_node(&chan->dev->device) == (int)(unsigned long)node; |
| 1749 | } |
| 1750 | |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1751 | /** |
| 1752 | * ntb_transport_create_queue - Create a new NTB transport layer queue |
| 1753 | * @rx_handler: receive callback function |
| 1754 | * @tx_handler: transmit callback function |
| 1755 | * @event_handler: event callback function |
| 1756 | * |
| 1757 | * Create a new NTB transport layer queue and provide the queue with a callback |
| 1758 | * routine for both transmit and receive. The receive callback routine will be |
| 1759 | * used to pass up data when the transport has received it on the queue. The |
| 1760 | * transmit callback routine will be called when the transport has completed the |
| 1761 | * transmission of the data on the queue and the data is ready to be freed. |
| 1762 | * |
| 1763 | * RETURNS: pointer to newly created ntb_queue, NULL on error. |
| 1764 | */ |
| 1765 | struct ntb_transport_qp * |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 1766 | ntb_transport_create_queue(void *data, struct device *client_dev, |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1767 | const struct ntb_queue_handlers *handlers) |
| 1768 | { |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 1769 | struct ntb_dev *ndev; |
| 1770 | struct pci_dev *pdev; |
| 1771 | struct ntb_transport_ctx *nt; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1772 | struct ntb_queue_entry *entry; |
| 1773 | struct ntb_transport_qp *qp; |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 1774 | u64 qp_bit; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1775 | unsigned int free_queue; |
Allen Hubbe | 1199aa6 | 2015-05-18 06:20:47 -0400 | [diff] [blame] | 1776 | dma_cap_mask_t dma_mask; |
| 1777 | int node; |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 1778 | int i; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1779 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 1780 | ndev = dev_ntb(client_dev->parent); |
| 1781 | pdev = ndev->pdev; |
| 1782 | nt = ndev->ctx; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1783 | |
Allen Hubbe | 1199aa6 | 2015-05-18 06:20:47 -0400 | [diff] [blame] | 1784 | node = dev_to_node(&ndev->dev); |
| 1785 | |
Thomas VanSelus | 8fcd095 | 2017-02-13 16:46:26 -0600 | [diff] [blame] | 1786 | free_queue = ffs(nt->qp_bitmap_free); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1787 | if (!free_queue) |
| 1788 | goto err; |
| 1789 | |
| 1790 | /* decrement free_queue to make it zero based */ |
| 1791 | free_queue--; |
| 1792 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 1793 | qp = &nt->qp_vec[free_queue]; |
| 1794 | qp_bit = BIT_ULL(qp->qp_num); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1795 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 1796 | nt->qp_bitmap_free &= ~qp_bit; |
| 1797 | |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1798 | qp->cb_data = data; |
| 1799 | qp->rx_handler = handlers->rx_handler; |
| 1800 | qp->tx_handler = handlers->tx_handler; |
| 1801 | qp->event_handler = handlers->event_handler; |
| 1802 | |
Allen Hubbe | 1199aa6 | 2015-05-18 06:20:47 -0400 | [diff] [blame] | 1803 | dma_cap_zero(dma_mask); |
| 1804 | dma_cap_set(DMA_MEMCPY, dma_mask); |
| 1805 | |
Dave Jiang | a41ef05 | 2015-05-19 16:52:04 -0400 | [diff] [blame] | 1806 | if (use_dma) { |
Dave Jiang | 569410ca | 2015-07-13 08:07:22 -0400 | [diff] [blame] | 1807 | qp->tx_dma_chan = |
| 1808 | dma_request_channel(dma_mask, ntb_dma_filter_fn, |
| 1809 | (void *)(unsigned long)node); |
| 1810 | if (!qp->tx_dma_chan) |
| 1811 | dev_info(&pdev->dev, "Unable to allocate TX DMA channel\n"); |
| 1812 | |
| 1813 | qp->rx_dma_chan = |
| 1814 | dma_request_channel(dma_mask, ntb_dma_filter_fn, |
| 1815 | (void *)(unsigned long)node); |
| 1816 | if (!qp->rx_dma_chan) |
| 1817 | dev_info(&pdev->dev, "Unable to allocate RX DMA channel\n"); |
Dave Jiang | a41ef05 | 2015-05-19 16:52:04 -0400 | [diff] [blame] | 1818 | } else { |
Dave Jiang | 569410ca | 2015-07-13 08:07:22 -0400 | [diff] [blame] | 1819 | qp->tx_dma_chan = NULL; |
| 1820 | qp->rx_dma_chan = NULL; |
Dave Jiang | a41ef05 | 2015-05-19 16:52:04 -0400 | [diff] [blame] | 1821 | } |
Dave Jiang | 569410ca | 2015-07-13 08:07:22 -0400 | [diff] [blame] | 1822 | |
| 1823 | dev_dbg(&pdev->dev, "Using %s memcpy for TX\n", |
| 1824 | qp->tx_dma_chan ? "DMA" : "CPU"); |
| 1825 | |
| 1826 | dev_dbg(&pdev->dev, "Using %s memcpy for RX\n", |
| 1827 | qp->rx_dma_chan ? "DMA" : "CPU"); |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 1828 | |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1829 | for (i = 0; i < NTB_QP_DEF_NUM_ENTRIES; i++) { |
Allen Hubbe | 1199aa6 | 2015-05-18 06:20:47 -0400 | [diff] [blame] | 1830 | entry = kzalloc_node(sizeof(*entry), GFP_ATOMIC, node); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1831 | if (!entry) |
| 1832 | goto err1; |
| 1833 | |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 1834 | entry->qp = qp; |
Allen Hubbe | da2e5ae | 2015-07-13 08:07:08 -0400 | [diff] [blame] | 1835 | ntb_list_add(&qp->ntb_rx_q_lock, &entry->entry, |
Jon Mason | f766755 | 2013-01-19 02:02:24 -0700 | [diff] [blame] | 1836 | &qp->rx_free_q); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1837 | } |
Dave Jiang | a754a8f | 2016-04-08 10:49:06 -0700 | [diff] [blame] | 1838 | qp->rx_alloc_entry = NTB_QP_DEF_NUM_ENTRIES; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1839 | |
Dave Jiang | a754a8f | 2016-04-08 10:49:06 -0700 | [diff] [blame] | 1840 | for (i = 0; i < qp->tx_max_entry; i++) { |
Allen Hubbe | 1199aa6 | 2015-05-18 06:20:47 -0400 | [diff] [blame] | 1841 | entry = kzalloc_node(sizeof(*entry), GFP_ATOMIC, node); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1842 | if (!entry) |
| 1843 | goto err2; |
| 1844 | |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 1845 | entry->qp = qp; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1846 | ntb_list_add(&qp->ntb_tx_free_q_lock, &entry->entry, |
Jon Mason | f766755 | 2013-01-19 02:02:24 -0700 | [diff] [blame] | 1847 | &qp->tx_free_q); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1848 | } |
| 1849 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 1850 | ntb_db_clear(qp->ndev, qp_bit); |
| 1851 | ntb_db_clear_mask(qp->ndev, qp_bit); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1852 | |
| 1853 | dev_info(&pdev->dev, "NTB Transport QP %d created\n", qp->qp_num); |
| 1854 | |
| 1855 | return qp; |
| 1856 | |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1857 | err2: |
Jon Mason | f766755 | 2013-01-19 02:02:24 -0700 | [diff] [blame] | 1858 | while ((entry = ntb_list_rm(&qp->ntb_tx_free_q_lock, &qp->tx_free_q))) |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1859 | kfree(entry); |
| 1860 | err1: |
Dave Jiang | a754a8f | 2016-04-08 10:49:06 -0700 | [diff] [blame] | 1861 | qp->rx_alloc_entry = 0; |
Allen Hubbe | da2e5ae | 2015-07-13 08:07:08 -0400 | [diff] [blame] | 1862 | while ((entry = ntb_list_rm(&qp->ntb_rx_q_lock, &qp->rx_free_q))) |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1863 | kfree(entry); |
Dave Jiang | 569410ca | 2015-07-13 08:07:22 -0400 | [diff] [blame] | 1864 | if (qp->tx_dma_chan) |
| 1865 | dma_release_channel(qp->tx_dma_chan); |
| 1866 | if (qp->rx_dma_chan) |
| 1867 | dma_release_channel(qp->rx_dma_chan); |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 1868 | nt->qp_bitmap_free |= qp_bit; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1869 | err: |
| 1870 | return NULL; |
| 1871 | } |
| 1872 | EXPORT_SYMBOL_GPL(ntb_transport_create_queue); |
| 1873 | |
| 1874 | /** |
| 1875 | * ntb_transport_free_queue - Frees NTB transport queue |
| 1876 | * @qp: NTB queue to be freed |
| 1877 | * |
| 1878 | * Frees NTB transport queue |
| 1879 | */ |
| 1880 | void ntb_transport_free_queue(struct ntb_transport_qp *qp) |
| 1881 | { |
Jon Mason | 186f27f | 2013-01-22 11:35:40 -0700 | [diff] [blame] | 1882 | struct pci_dev *pdev; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1883 | struct ntb_queue_entry *entry; |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 1884 | u64 qp_bit; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1885 | |
| 1886 | if (!qp) |
| 1887 | return; |
| 1888 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 1889 | pdev = qp->ndev->pdev; |
Jon Mason | 186f27f | 2013-01-22 11:35:40 -0700 | [diff] [blame] | 1890 | |
Dave Jiang | e902133 | 2016-02-23 09:11:36 -0700 | [diff] [blame] | 1891 | qp->active = false; |
| 1892 | |
Dave Jiang | 569410ca | 2015-07-13 08:07:22 -0400 | [diff] [blame] | 1893 | if (qp->tx_dma_chan) { |
| 1894 | struct dma_chan *chan = qp->tx_dma_chan; |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 1895 | /* Putting the dma_chan to NULL will force any new traffic to be |
| 1896 | * processed by the CPU instead of the DAM engine |
| 1897 | */ |
Dave Jiang | 569410ca | 2015-07-13 08:07:22 -0400 | [diff] [blame] | 1898 | qp->tx_dma_chan = NULL; |
| 1899 | |
| 1900 | /* Try to be nice and wait for any queued DMA engine |
| 1901 | * transactions to process before smashing it with a rock |
| 1902 | */ |
| 1903 | dma_sync_wait(chan, qp->last_cookie); |
| 1904 | dmaengine_terminate_all(chan); |
| 1905 | dma_release_channel(chan); |
| 1906 | } |
| 1907 | |
| 1908 | if (qp->rx_dma_chan) { |
| 1909 | struct dma_chan *chan = qp->rx_dma_chan; |
| 1910 | /* Putting the dma_chan to NULL will force any new traffic to be |
| 1911 | * processed by the CPU instead of the DAM engine |
| 1912 | */ |
| 1913 | qp->rx_dma_chan = NULL; |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 1914 | |
| 1915 | /* Try to be nice and wait for any queued DMA engine |
| 1916 | * transactions to process before smashing it with a rock |
| 1917 | */ |
| 1918 | dma_sync_wait(chan, qp->last_cookie); |
| 1919 | dmaengine_terminate_all(chan); |
Allen Hubbe | 1199aa6 | 2015-05-18 06:20:47 -0400 | [diff] [blame] | 1920 | dma_release_channel(chan); |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 1921 | } |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1922 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 1923 | qp_bit = BIT_ULL(qp->qp_num); |
| 1924 | |
| 1925 | ntb_db_set_mask(qp->ndev, qp_bit); |
Dave Jiang | e902133 | 2016-02-23 09:11:36 -0700 | [diff] [blame] | 1926 | tasklet_kill(&qp->rxc_db_work); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1927 | |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 1928 | cancel_delayed_work_sync(&qp->link_work); |
| 1929 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 1930 | qp->cb_data = NULL; |
| 1931 | qp->rx_handler = NULL; |
| 1932 | qp->tx_handler = NULL; |
| 1933 | qp->event_handler = NULL; |
| 1934 | |
Allen Hubbe | da2e5ae | 2015-07-13 08:07:08 -0400 | [diff] [blame] | 1935 | while ((entry = ntb_list_rm(&qp->ntb_rx_q_lock, &qp->rx_free_q))) |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1936 | kfree(entry); |
| 1937 | |
Allen Hubbe | da2e5ae | 2015-07-13 08:07:08 -0400 | [diff] [blame] | 1938 | while ((entry = ntb_list_rm(&qp->ntb_rx_q_lock, &qp->rx_pend_q))) { |
| 1939 | dev_warn(&pdev->dev, "Freeing item from non-empty rx_pend_q\n"); |
| 1940 | kfree(entry); |
| 1941 | } |
| 1942 | |
| 1943 | while ((entry = ntb_list_rm(&qp->ntb_rx_q_lock, &qp->rx_post_q))) { |
| 1944 | dev_warn(&pdev->dev, "Freeing item from non-empty rx_post_q\n"); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1945 | kfree(entry); |
| 1946 | } |
| 1947 | |
Jon Mason | f766755 | 2013-01-19 02:02:24 -0700 | [diff] [blame] | 1948 | while ((entry = ntb_list_rm(&qp->ntb_tx_free_q_lock, &qp->tx_free_q))) |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1949 | kfree(entry); |
| 1950 | |
Allen Hubbe | 30a4bb1 | 2015-07-13 08:07:14 -0400 | [diff] [blame] | 1951 | qp->transport->qp_bitmap_free |= qp_bit; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1952 | |
| 1953 | dev_info(&pdev->dev, "NTB Transport QP %d freed\n", qp->qp_num); |
| 1954 | } |
| 1955 | EXPORT_SYMBOL_GPL(ntb_transport_free_queue); |
| 1956 | |
| 1957 | /** |
| 1958 | * ntb_transport_rx_remove - Dequeues enqueued rx packet |
| 1959 | * @qp: NTB queue to be freed |
| 1960 | * @len: pointer to variable to write enqueued buffers length |
| 1961 | * |
| 1962 | * Dequeues unused buffers from receive queue. Should only be used during |
| 1963 | * shutdown of qp. |
| 1964 | * |
| 1965 | * RETURNS: NULL error value on error, or void* for success. |
| 1966 | */ |
| 1967 | void *ntb_transport_rx_remove(struct ntb_transport_qp *qp, unsigned int *len) |
| 1968 | { |
| 1969 | struct ntb_queue_entry *entry; |
| 1970 | void *buf; |
| 1971 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 1972 | if (!qp || qp->client_ready) |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1973 | return NULL; |
| 1974 | |
Allen Hubbe | da2e5ae | 2015-07-13 08:07:08 -0400 | [diff] [blame] | 1975 | entry = ntb_list_rm(&qp->ntb_rx_q_lock, &qp->rx_pend_q); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1976 | if (!entry) |
| 1977 | return NULL; |
| 1978 | |
| 1979 | buf = entry->cb_data; |
| 1980 | *len = entry->len; |
| 1981 | |
Allen Hubbe | da2e5ae | 2015-07-13 08:07:08 -0400 | [diff] [blame] | 1982 | ntb_list_add(&qp->ntb_rx_q_lock, &entry->entry, &qp->rx_free_q); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1983 | |
| 1984 | return buf; |
| 1985 | } |
| 1986 | EXPORT_SYMBOL_GPL(ntb_transport_rx_remove); |
| 1987 | |
| 1988 | /** |
| 1989 | * ntb_transport_rx_enqueue - Enqueue a new NTB queue entry |
| 1990 | * @qp: NTB transport layer queue the entry is to be enqueued on |
| 1991 | * @cb: per buffer pointer for callback function to use |
| 1992 | * @data: pointer to data buffer that incoming packets will be copied into |
| 1993 | * @len: length of the data buffer |
| 1994 | * |
| 1995 | * Enqueue a new receive buffer onto the transport queue into which a NTB |
| 1996 | * payload can be received into. |
| 1997 | * |
| 1998 | * RETURNS: An appropriate -ERRNO error value on error, or zero for success. |
| 1999 | */ |
| 2000 | int ntb_transport_rx_enqueue(struct ntb_transport_qp *qp, void *cb, void *data, |
| 2001 | unsigned int len) |
| 2002 | { |
| 2003 | struct ntb_queue_entry *entry; |
| 2004 | |
| 2005 | if (!qp) |
| 2006 | return -EINVAL; |
| 2007 | |
Allen Hubbe | da2e5ae | 2015-07-13 08:07:08 -0400 | [diff] [blame] | 2008 | entry = ntb_list_rm(&qp->ntb_rx_q_lock, &qp->rx_free_q); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 2009 | if (!entry) |
| 2010 | return -ENOMEM; |
| 2011 | |
| 2012 | entry->cb_data = cb; |
| 2013 | entry->buf = data; |
| 2014 | entry->len = len; |
Allen Hubbe | da2e5ae | 2015-07-13 08:07:08 -0400 | [diff] [blame] | 2015 | entry->flags = 0; |
Dave Jiang | 7220357 | 2016-07-20 13:14:13 -0700 | [diff] [blame] | 2016 | entry->retries = 0; |
| 2017 | entry->errors = 0; |
| 2018 | entry->rx_index = 0; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 2019 | |
Allen Hubbe | da2e5ae | 2015-07-13 08:07:08 -0400 | [diff] [blame] | 2020 | ntb_list_add(&qp->ntb_rx_q_lock, &entry->entry, &qp->rx_pend_q); |
| 2021 | |
Dave Jiang | e902133 | 2016-02-23 09:11:36 -0700 | [diff] [blame] | 2022 | if (qp->active) |
| 2023 | tasklet_schedule(&qp->rxc_db_work); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 2024 | |
| 2025 | return 0; |
| 2026 | } |
| 2027 | EXPORT_SYMBOL_GPL(ntb_transport_rx_enqueue); |
| 2028 | |
| 2029 | /** |
| 2030 | * ntb_transport_tx_enqueue - Enqueue a new NTB queue entry |
| 2031 | * @qp: NTB transport layer queue the entry is to be enqueued on |
| 2032 | * @cb: per buffer pointer for callback function to use |
| 2033 | * @data: pointer to data buffer that will be sent |
| 2034 | * @len: length of the data buffer |
| 2035 | * |
| 2036 | * Enqueue a new transmit buffer onto the transport queue from which a NTB |
Jon Mason | f9a2cf8 | 2013-07-29 16:46:43 -0700 | [diff] [blame] | 2037 | * payload will be transmitted. This assumes that a lock is being held to |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 2038 | * serialize access to the qp. |
| 2039 | * |
| 2040 | * RETURNS: An appropriate -ERRNO error value on error, or zero for success. |
| 2041 | */ |
| 2042 | int ntb_transport_tx_enqueue(struct ntb_transport_qp *qp, void *cb, void *data, |
| 2043 | unsigned int len) |
| 2044 | { |
| 2045 | struct ntb_queue_entry *entry; |
| 2046 | int rc; |
| 2047 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 2048 | if (!qp || !qp->link_is_up || !len) |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 2049 | return -EINVAL; |
| 2050 | |
| 2051 | entry = ntb_list_rm(&qp->ntb_tx_free_q_lock, &qp->tx_free_q); |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 2052 | if (!entry) { |
| 2053 | qp->tx_err_no_buf++; |
Dave Jiang | e74bfee | 2015-07-13 08:07:17 -0400 | [diff] [blame] | 2054 | return -EBUSY; |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 2055 | } |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 2056 | |
| 2057 | entry->cb_data = cb; |
| 2058 | entry->buf = data; |
| 2059 | entry->len = len; |
| 2060 | entry->flags = 0; |
Dave Jiang | 9cabc26 | 2016-07-20 13:14:07 -0700 | [diff] [blame] | 2061 | entry->errors = 0; |
| 2062 | entry->retries = 0; |
| 2063 | entry->tx_index = 0; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 2064 | |
| 2065 | rc = ntb_process_tx(qp, entry); |
| 2066 | if (rc) |
| 2067 | ntb_list_add(&qp->ntb_tx_free_q_lock, &entry->entry, |
| 2068 | &qp->tx_free_q); |
| 2069 | |
| 2070 | return rc; |
| 2071 | } |
| 2072 | EXPORT_SYMBOL_GPL(ntb_transport_tx_enqueue); |
| 2073 | |
| 2074 | /** |
| 2075 | * ntb_transport_link_up - Notify NTB transport of client readiness to use queue |
| 2076 | * @qp: NTB transport layer queue to be enabled |
| 2077 | * |
| 2078 | * Notify NTB transport layer of client readiness to use queue |
| 2079 | */ |
| 2080 | void ntb_transport_link_up(struct ntb_transport_qp *qp) |
| 2081 | { |
| 2082 | if (!qp) |
| 2083 | return; |
| 2084 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 2085 | qp->client_ready = true; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 2086 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 2087 | if (qp->transport->link_is_up) |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 2088 | schedule_delayed_work(&qp->link_work, 0); |
| 2089 | } |
| 2090 | EXPORT_SYMBOL_GPL(ntb_transport_link_up); |
| 2091 | |
| 2092 | /** |
| 2093 | * ntb_transport_link_down - Notify NTB transport to no longer enqueue data |
| 2094 | * @qp: NTB transport layer queue to be disabled |
| 2095 | * |
| 2096 | * Notify NTB transport layer of client's desire to no longer receive data on |
| 2097 | * transport queue specified. It is the client's responsibility to ensure all |
Jon Mason | f9a2cf8 | 2013-07-29 16:46:43 -0700 | [diff] [blame] | 2098 | * entries on queue are purged or otherwise handled appropriately. |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 2099 | */ |
| 2100 | void ntb_transport_link_down(struct ntb_transport_qp *qp) |
| 2101 | { |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 2102 | int val; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 2103 | |
| 2104 | if (!qp) |
| 2105 | return; |
| 2106 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 2107 | qp->client_ready = false; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 2108 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 2109 | val = ntb_spad_read(qp->ndev, QP_LINKS); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 2110 | |
Serge Semin | d67288a | 2017-01-11 03:13:20 +0300 | [diff] [blame] | 2111 | ntb_peer_spad_write(qp->ndev, PIDX, QP_LINKS, val & ~BIT(qp->qp_num)); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 2112 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 2113 | if (qp->link_is_up) |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 2114 | ntb_send_link_down(qp); |
| 2115 | else |
| 2116 | cancel_delayed_work_sync(&qp->link_work); |
| 2117 | } |
| 2118 | EXPORT_SYMBOL_GPL(ntb_transport_link_down); |
| 2119 | |
| 2120 | /** |
| 2121 | * ntb_transport_link_query - Query transport link state |
| 2122 | * @qp: NTB transport layer queue to be queried |
| 2123 | * |
| 2124 | * Query connectivity to the remote system of the NTB transport queue |
| 2125 | * |
| 2126 | * RETURNS: true for link up or false for link down |
| 2127 | */ |
| 2128 | bool ntb_transport_link_query(struct ntb_transport_qp *qp) |
| 2129 | { |
Jon Mason | 186f27f | 2013-01-22 11:35:40 -0700 | [diff] [blame] | 2130 | if (!qp) |
| 2131 | return false; |
| 2132 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 2133 | return qp->link_is_up; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 2134 | } |
| 2135 | EXPORT_SYMBOL_GPL(ntb_transport_link_query); |
| 2136 | |
| 2137 | /** |
| 2138 | * ntb_transport_qp_num - Query the qp number |
| 2139 | * @qp: NTB transport layer queue to be queried |
| 2140 | * |
| 2141 | * Query qp number of the NTB transport queue |
| 2142 | * |
| 2143 | * RETURNS: a zero based number specifying the qp number |
| 2144 | */ |
| 2145 | unsigned char ntb_transport_qp_num(struct ntb_transport_qp *qp) |
| 2146 | { |
Jon Mason | 186f27f | 2013-01-22 11:35:40 -0700 | [diff] [blame] | 2147 | if (!qp) |
| 2148 | return 0; |
| 2149 | |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 2150 | return qp->qp_num; |
| 2151 | } |
| 2152 | EXPORT_SYMBOL_GPL(ntb_transport_qp_num); |
| 2153 | |
| 2154 | /** |
| 2155 | * ntb_transport_max_size - Query the max payload size of a qp |
| 2156 | * @qp: NTB transport layer queue to be queried |
| 2157 | * |
| 2158 | * Query the maximum payload size permissible on the given qp |
| 2159 | * |
| 2160 | * RETURNS: the max payload size of a qp |
| 2161 | */ |
Jon Mason | ef114ed | 2013-01-19 02:02:18 -0700 | [diff] [blame] | 2162 | unsigned int ntb_transport_max_size(struct ntb_transport_qp *qp) |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 2163 | { |
Dave Jiang | 04afde4 | 2015-09-17 13:27:04 -0700 | [diff] [blame] | 2164 | unsigned int max_size; |
Dave Jiang | 569410ca | 2015-07-13 08:07:22 -0400 | [diff] [blame] | 2165 | unsigned int copy_align; |
Dave Jiang | 04afde4 | 2015-09-17 13:27:04 -0700 | [diff] [blame] | 2166 | struct dma_chan *rx_chan, *tx_chan; |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 2167 | |
Jon Mason | 186f27f | 2013-01-22 11:35:40 -0700 | [diff] [blame] | 2168 | if (!qp) |
| 2169 | return 0; |
| 2170 | |
Dave Jiang | 04afde4 | 2015-09-17 13:27:04 -0700 | [diff] [blame] | 2171 | rx_chan = qp->rx_dma_chan; |
| 2172 | tx_chan = qp->tx_dma_chan; |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 2173 | |
Dave Jiang | 04afde4 | 2015-09-17 13:27:04 -0700 | [diff] [blame] | 2174 | copy_align = max(rx_chan ? rx_chan->device->copy_align : 0, |
| 2175 | tx_chan ? tx_chan->device->copy_align : 0); |
Dave Jiang | 569410ca | 2015-07-13 08:07:22 -0400 | [diff] [blame] | 2176 | |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 2177 | /* If DMA engine usage is possible, try to find the max size for that */ |
Dave Jiang | 04afde4 | 2015-09-17 13:27:04 -0700 | [diff] [blame] | 2178 | max_size = qp->tx_max_frame - sizeof(struct ntb_payload_header); |
| 2179 | max_size = round_down(max_size, 1 << copy_align); |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 2180 | |
Dave Jiang | 04afde4 | 2015-09-17 13:27:04 -0700 | [diff] [blame] | 2181 | return max_size; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 2182 | } |
| 2183 | EXPORT_SYMBOL_GPL(ntb_transport_max_size); |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 2184 | |
Dave Jiang | e74bfee | 2015-07-13 08:07:17 -0400 | [diff] [blame] | 2185 | unsigned int ntb_transport_tx_free_entry(struct ntb_transport_qp *qp) |
| 2186 | { |
| 2187 | unsigned int head = qp->tx_index; |
| 2188 | unsigned int tail = qp->remote_rx_info->entry; |
| 2189 | |
| 2190 | return tail > head ? tail - head : qp->tx_max_entry + tail - head; |
| 2191 | } |
| 2192 | EXPORT_SYMBOL_GPL(ntb_transport_tx_free_entry); |
| 2193 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 2194 | static void ntb_transport_doorbell_callback(void *data, int vector) |
| 2195 | { |
| 2196 | struct ntb_transport_ctx *nt = data; |
| 2197 | struct ntb_transport_qp *qp; |
| 2198 | u64 db_bits; |
| 2199 | unsigned int qp_num; |
| 2200 | |
| 2201 | db_bits = (nt->qp_bitmap & ~nt->qp_bitmap_free & |
| 2202 | ntb_db_vector_mask(nt->ndev, vector)); |
| 2203 | |
| 2204 | while (db_bits) { |
| 2205 | qp_num = __ffs(db_bits); |
| 2206 | qp = &nt->qp_vec[qp_num]; |
| 2207 | |
Dave Jiang | e902133 | 2016-02-23 09:11:36 -0700 | [diff] [blame] | 2208 | if (qp->active) |
| 2209 | tasklet_schedule(&qp->rxc_db_work); |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 2210 | |
| 2211 | db_bits &= ~BIT_ULL(qp_num); |
| 2212 | } |
| 2213 | } |
| 2214 | |
| 2215 | static const struct ntb_ctx_ops ntb_transport_ops = { |
| 2216 | .link_event = ntb_transport_event_callback, |
| 2217 | .db_event = ntb_transport_doorbell_callback, |
| 2218 | }; |
| 2219 | |
| 2220 | static struct ntb_client ntb_transport_client = { |
| 2221 | .ops = { |
| 2222 | .probe = ntb_transport_probe, |
| 2223 | .remove = ntb_transport_free, |
| 2224 | }, |
| 2225 | }; |
| 2226 | |
| 2227 | static int __init ntb_transport_init(void) |
| 2228 | { |
| 2229 | int rc; |
| 2230 | |
Dave Jiang | 7eb3878 | 2015-06-15 08:21:33 -0400 | [diff] [blame] | 2231 | pr_info("%s, version %s\n", NTB_TRANSPORT_DESC, NTB_TRANSPORT_VER); |
| 2232 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 2233 | if (debugfs_initialized()) |
| 2234 | nt_debugfs_dir = debugfs_create_dir(KBUILD_MODNAME, NULL); |
| 2235 | |
| 2236 | rc = bus_register(&ntb_transport_bus); |
| 2237 | if (rc) |
| 2238 | goto err_bus; |
| 2239 | |
| 2240 | rc = ntb_register_client(&ntb_transport_client); |
| 2241 | if (rc) |
| 2242 | goto err_client; |
| 2243 | |
| 2244 | return 0; |
| 2245 | |
| 2246 | err_client: |
| 2247 | bus_unregister(&ntb_transport_bus); |
| 2248 | err_bus: |
| 2249 | debugfs_remove_recursive(nt_debugfs_dir); |
| 2250 | return rc; |
| 2251 | } |
| 2252 | module_init(ntb_transport_init); |
| 2253 | |
| 2254 | static void __exit ntb_transport_exit(void) |
| 2255 | { |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 2256 | ntb_unregister_client(&ntb_transport_client); |
| 2257 | bus_unregister(&ntb_transport_bus); |
Allen Hubbe | dd62245 | 2016-12-27 17:57:04 -0500 | [diff] [blame] | 2258 | debugfs_remove_recursive(nt_debugfs_dir); |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 2259 | } |
| 2260 | module_exit(ntb_transport_exit); |