Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1 | /* |
| 2 | * This file is provided under a dual BSD/GPLv2 license. When using or |
| 3 | * redistributing this file, you may do so under either license. |
| 4 | * |
| 5 | * GPL LICENSE SUMMARY |
| 6 | * |
| 7 | * Copyright(c) 2012 Intel Corporation. All rights reserved. |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 8 | * Copyright (C) 2015 EMC Corporation. All Rights Reserved. |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 9 | * |
| 10 | * This program is free software; you can redistribute it and/or modify |
| 11 | * it under the terms of version 2 of the GNU General Public License as |
| 12 | * published by the Free Software Foundation. |
| 13 | * |
| 14 | * BSD LICENSE |
| 15 | * |
| 16 | * Copyright(c) 2012 Intel Corporation. All rights reserved. |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 17 | * Copyright (C) 2015 EMC Corporation. All Rights Reserved. |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 18 | * |
| 19 | * Redistribution and use in source and binary forms, with or without |
| 20 | * modification, are permitted provided that the following conditions |
| 21 | * are met: |
| 22 | * |
| 23 | * * Redistributions of source code must retain the above copyright |
| 24 | * notice, this list of conditions and the following disclaimer. |
| 25 | * * Redistributions in binary form must reproduce the above copy |
| 26 | * notice, this list of conditions and the following disclaimer in |
| 27 | * the documentation and/or other materials provided with the |
| 28 | * distribution. |
| 29 | * * Neither the name of Intel Corporation nor the names of its |
| 30 | * contributors may be used to endorse or promote products derived |
| 31 | * from this software without specific prior written permission. |
| 32 | * |
| 33 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS |
| 34 | * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT |
| 35 | * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR |
| 36 | * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT |
| 37 | * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
| 38 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
| 39 | * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
| 40 | * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
| 41 | * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
| 42 | * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
| 43 | * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
| 44 | * |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 45 | * PCIe NTB Transport Linux driver |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 46 | * |
| 47 | * Contact Information: |
| 48 | * Jon Mason <jon.mason@intel.com> |
| 49 | */ |
| 50 | #include <linux/debugfs.h> |
| 51 | #include <linux/delay.h> |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 52 | #include <linux/dmaengine.h> |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 53 | #include <linux/dma-mapping.h> |
| 54 | #include <linux/errno.h> |
| 55 | #include <linux/export.h> |
| 56 | #include <linux/interrupt.h> |
| 57 | #include <linux/module.h> |
| 58 | #include <linux/pci.h> |
| 59 | #include <linux/slab.h> |
| 60 | #include <linux/types.h> |
Dave Jiang | 06917f7 | 2015-05-19 16:45:46 -0400 | [diff] [blame] | 61 | #include <linux/uaccess.h> |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 62 | #include "linux/ntb.h" |
| 63 | #include "linux/ntb_transport.h" |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 64 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 65 | #define NTB_TRANSPORT_VERSION 4 |
| 66 | #define NTB_TRANSPORT_VER "4" |
| 67 | #define NTB_TRANSPORT_NAME "ntb_transport" |
| 68 | #define NTB_TRANSPORT_DESC "Software Queue-Pair Transport over NTB" |
| 69 | |
| 70 | MODULE_DESCRIPTION(NTB_TRANSPORT_DESC); |
| 71 | MODULE_VERSION(NTB_TRANSPORT_VER); |
| 72 | MODULE_LICENSE("Dual BSD/GPL"); |
| 73 | MODULE_AUTHOR("Intel Corporation"); |
| 74 | |
| 75 | static unsigned long max_mw_size; |
| 76 | module_param(max_mw_size, ulong, 0644); |
| 77 | MODULE_PARM_DESC(max_mw_size, "Limit size of large memory windows"); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 78 | |
Dave Jiang | 9891417 | 2015-06-03 11:29:38 -0400 | [diff] [blame] | 79 | static unsigned int transport_mtu = 0x10000; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 80 | module_param(transport_mtu, uint, 0644); |
| 81 | MODULE_PARM_DESC(transport_mtu, "Maximum size of NTB transport packets"); |
| 82 | |
Jon Mason | 948d3a6 | 2013-04-18 17:07:36 -0700 | [diff] [blame] | 83 | static unsigned char max_num_clients; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 84 | module_param(max_num_clients, byte, 0644); |
| 85 | MODULE_PARM_DESC(max_num_clients, "Maximum number of NTB transport clients"); |
| 86 | |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 87 | static unsigned int copy_bytes = 1024; |
| 88 | module_param(copy_bytes, uint, 0644); |
| 89 | MODULE_PARM_DESC(copy_bytes, "Threshold under which NTB will use the CPU to copy instead of DMA"); |
| 90 | |
Dave Jiang | a41ef05 | 2015-05-19 16:52:04 -0400 | [diff] [blame] | 91 | static bool use_dma; |
| 92 | module_param(use_dma, bool, 0644); |
| 93 | MODULE_PARM_DESC(use_dma, "Use DMA engine to perform large data copy"); |
| 94 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 95 | static struct dentry *nt_debugfs_dir; |
| 96 | |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 97 | struct ntb_queue_entry { |
| 98 | /* ntb_queue list reference */ |
| 99 | struct list_head entry; |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 100 | /* pointers to data to be transferred */ |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 101 | void *cb_data; |
| 102 | void *buf; |
| 103 | unsigned int len; |
| 104 | unsigned int flags; |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 105 | |
| 106 | struct ntb_transport_qp *qp; |
| 107 | union { |
| 108 | struct ntb_payload_header __iomem *tx_hdr; |
| 109 | struct ntb_payload_header *rx_hdr; |
| 110 | }; |
| 111 | unsigned int index; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 112 | }; |
| 113 | |
Jon Mason | 793c20e | 2013-01-19 02:02:26 -0700 | [diff] [blame] | 114 | struct ntb_rx_info { |
| 115 | unsigned int entry; |
| 116 | }; |
| 117 | |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 118 | struct ntb_transport_qp { |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 119 | struct ntb_transport_ctx *transport; |
| 120 | struct ntb_dev *ndev; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 121 | void *cb_data; |
Dave Jiang | 569410ca | 2015-07-13 08:07:22 -0400 | [diff] [blame] | 122 | struct dma_chan *tx_dma_chan; |
| 123 | struct dma_chan *rx_dma_chan; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 124 | |
| 125 | bool client_ready; |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 126 | bool link_is_up; |
| 127 | |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 128 | u8 qp_num; /* Only 64 QP's are allowed. 0-63 */ |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 129 | u64 qp_bit; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 130 | |
Jon Mason | 7446564 | 2013-01-21 15:28:52 -0700 | [diff] [blame] | 131 | struct ntb_rx_info __iomem *rx_info; |
Jon Mason | 793c20e | 2013-01-19 02:02:26 -0700 | [diff] [blame] | 132 | struct ntb_rx_info *remote_rx_info; |
| 133 | |
Jon Mason | 53ca4fe | 2013-11-26 11:21:50 -0700 | [diff] [blame] | 134 | void (*tx_handler)(struct ntb_transport_qp *qp, void *qp_data, |
| 135 | void *data, int len); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 136 | struct list_head tx_free_q; |
| 137 | spinlock_t ntb_tx_free_q_lock; |
Jon Mason | 7446564 | 2013-01-21 15:28:52 -0700 | [diff] [blame] | 138 | void __iomem *tx_mw; |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 139 | dma_addr_t tx_mw_phys; |
Jon Mason | 793c20e | 2013-01-19 02:02:26 -0700 | [diff] [blame] | 140 | unsigned int tx_index; |
| 141 | unsigned int tx_max_entry; |
Jon Mason | ef114ed | 2013-01-19 02:02:18 -0700 | [diff] [blame] | 142 | unsigned int tx_max_frame; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 143 | |
Jon Mason | 53ca4fe | 2013-11-26 11:21:50 -0700 | [diff] [blame] | 144 | void (*rx_handler)(struct ntb_transport_qp *qp, void *qp_data, |
| 145 | void *data, int len); |
Allen Hubbe | da2e5ae | 2015-07-13 08:07:08 -0400 | [diff] [blame] | 146 | struct list_head rx_post_q; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 147 | struct list_head rx_pend_q; |
| 148 | struct list_head rx_free_q; |
Allen Hubbe | da2e5ae | 2015-07-13 08:07:08 -0400 | [diff] [blame] | 149 | /* ntb_rx_q_lock: synchronize access to rx_XXXX_q */ |
| 150 | spinlock_t ntb_rx_q_lock; |
Jon Mason | 793c20e | 2013-01-19 02:02:26 -0700 | [diff] [blame] | 151 | void *rx_buff; |
| 152 | unsigned int rx_index; |
| 153 | unsigned int rx_max_entry; |
Jon Mason | ef114ed | 2013-01-19 02:02:18 -0700 | [diff] [blame] | 154 | unsigned int rx_max_frame; |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 155 | dma_cookie_t last_cookie; |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 156 | struct tasklet_struct rxc_db_work; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 157 | |
Jon Mason | 53ca4fe | 2013-11-26 11:21:50 -0700 | [diff] [blame] | 158 | void (*event_handler)(void *data, int status); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 159 | struct delayed_work link_work; |
Jon Mason | 7b4f2d3 | 2013-01-19 02:02:19 -0700 | [diff] [blame] | 160 | struct work_struct link_cleanup; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 161 | |
| 162 | struct dentry *debugfs_dir; |
| 163 | struct dentry *debugfs_stats; |
| 164 | |
| 165 | /* Stats */ |
| 166 | u64 rx_bytes; |
| 167 | u64 rx_pkts; |
| 168 | u64 rx_ring_empty; |
| 169 | u64 rx_err_no_buf; |
| 170 | u64 rx_err_oflow; |
| 171 | u64 rx_err_ver; |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 172 | u64 rx_memcpy; |
| 173 | u64 rx_async; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 174 | u64 tx_bytes; |
| 175 | u64 tx_pkts; |
| 176 | u64 tx_ring_full; |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 177 | u64 tx_err_no_buf; |
| 178 | u64 tx_memcpy; |
| 179 | u64 tx_async; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 180 | }; |
| 181 | |
| 182 | struct ntb_transport_mw { |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 183 | phys_addr_t phys_addr; |
| 184 | resource_size_t phys_size; |
| 185 | resource_size_t xlat_align; |
| 186 | resource_size_t xlat_align_size; |
| 187 | void __iomem *vbase; |
| 188 | size_t xlat_size; |
| 189 | size_t buff_size; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 190 | void *virt_addr; |
| 191 | dma_addr_t dma_addr; |
| 192 | }; |
| 193 | |
| 194 | struct ntb_transport_client_dev { |
| 195 | struct list_head entry; |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 196 | struct ntb_transport_ctx *nt; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 197 | struct device dev; |
| 198 | }; |
| 199 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 200 | struct ntb_transport_ctx { |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 201 | struct list_head entry; |
| 202 | struct list_head client_devs; |
| 203 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 204 | struct ntb_dev *ndev; |
| 205 | |
| 206 | struct ntb_transport_mw *mw_vec; |
| 207 | struct ntb_transport_qp *qp_vec; |
| 208 | unsigned int mw_count; |
| 209 | unsigned int qp_count; |
| 210 | u64 qp_bitmap; |
| 211 | u64 qp_bitmap_free; |
| 212 | |
| 213 | bool link_is_up; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 214 | struct delayed_work link_work; |
Jon Mason | 7b4f2d3 | 2013-01-19 02:02:19 -0700 | [diff] [blame] | 215 | struct work_struct link_cleanup; |
Dave Jiang | c8650fd | 2015-07-13 08:07:09 -0400 | [diff] [blame] | 216 | |
| 217 | struct dentry *debugfs_node_dir; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 218 | }; |
| 219 | |
| 220 | enum { |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 221 | DESC_DONE_FLAG = BIT(0), |
| 222 | LINK_DOWN_FLAG = BIT(1), |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 223 | }; |
| 224 | |
| 225 | struct ntb_payload_header { |
Jon Mason | 7446564 | 2013-01-21 15:28:52 -0700 | [diff] [blame] | 226 | unsigned int ver; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 227 | unsigned int len; |
| 228 | unsigned int flags; |
| 229 | }; |
| 230 | |
| 231 | enum { |
| 232 | VERSION = 0, |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 233 | QP_LINKS, |
Jon Mason | 113fc50 | 2013-01-30 11:40:52 -0700 | [diff] [blame] | 234 | NUM_QPS, |
| 235 | NUM_MWS, |
| 236 | MW0_SZ_HIGH, |
| 237 | MW0_SZ_LOW, |
| 238 | MW1_SZ_HIGH, |
| 239 | MW1_SZ_LOW, |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 240 | MAX_SPAD, |
| 241 | }; |
| 242 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 243 | #define dev_client_dev(__dev) \ |
| 244 | container_of((__dev), struct ntb_transport_client_dev, dev) |
| 245 | |
| 246 | #define drv_client(__drv) \ |
| 247 | container_of((__drv), struct ntb_transport_client, driver) |
| 248 | |
| 249 | #define QP_TO_MW(nt, qp) ((qp) % nt->mw_count) |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 250 | #define NTB_QP_DEF_NUM_ENTRIES 100 |
| 251 | #define NTB_LINK_DOWN_TIMEOUT 10 |
| 252 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 253 | static void ntb_transport_rxc_db(unsigned long data); |
| 254 | static const struct ntb_ctx_ops ntb_transport_ops; |
| 255 | static struct ntb_client ntb_transport_client; |
| 256 | |
| 257 | static int ntb_transport_bus_match(struct device *dev, |
| 258 | struct device_driver *drv) |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 259 | { |
| 260 | return !strncmp(dev_name(dev), drv->name, strlen(drv->name)); |
| 261 | } |
| 262 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 263 | static int ntb_transport_bus_probe(struct device *dev) |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 264 | { |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 265 | const struct ntb_transport_client *client; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 266 | int rc = -EINVAL; |
| 267 | |
| 268 | get_device(dev); |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 269 | |
| 270 | client = drv_client(dev->driver); |
| 271 | rc = client->probe(dev); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 272 | if (rc) |
| 273 | put_device(dev); |
| 274 | |
| 275 | return rc; |
| 276 | } |
| 277 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 278 | static int ntb_transport_bus_remove(struct device *dev) |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 279 | { |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 280 | const struct ntb_transport_client *client; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 281 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 282 | client = drv_client(dev->driver); |
| 283 | client->remove(dev); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 284 | |
| 285 | put_device(dev); |
| 286 | |
| 287 | return 0; |
| 288 | } |
| 289 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 290 | static struct bus_type ntb_transport_bus = { |
| 291 | .name = "ntb_transport", |
| 292 | .match = ntb_transport_bus_match, |
| 293 | .probe = ntb_transport_bus_probe, |
| 294 | .remove = ntb_transport_bus_remove, |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 295 | }; |
| 296 | |
| 297 | static LIST_HEAD(ntb_transport_list); |
| 298 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 299 | static int ntb_bus_init(struct ntb_transport_ctx *nt) |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 300 | { |
Dave Jiang | 3151000 | 2015-07-13 08:07:19 -0400 | [diff] [blame] | 301 | list_add_tail(&nt->entry, &ntb_transport_list); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 302 | return 0; |
| 303 | } |
| 304 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 305 | static void ntb_bus_remove(struct ntb_transport_ctx *nt) |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 306 | { |
| 307 | struct ntb_transport_client_dev *client_dev, *cd; |
| 308 | |
| 309 | list_for_each_entry_safe(client_dev, cd, &nt->client_devs, entry) { |
| 310 | dev_err(client_dev->dev.parent, "%s still attached to bus, removing\n", |
| 311 | dev_name(&client_dev->dev)); |
| 312 | list_del(&client_dev->entry); |
| 313 | device_unregister(&client_dev->dev); |
| 314 | } |
| 315 | |
| 316 | list_del(&nt->entry); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 317 | } |
| 318 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 319 | static void ntb_transport_client_release(struct device *dev) |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 320 | { |
| 321 | struct ntb_transport_client_dev *client_dev; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 322 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 323 | client_dev = dev_client_dev(dev); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 324 | kfree(client_dev); |
| 325 | } |
| 326 | |
| 327 | /** |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 328 | * ntb_transport_unregister_client_dev - Unregister NTB client device |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 329 | * @device_name: Name of NTB client device |
| 330 | * |
| 331 | * Unregister an NTB client device with the NTB transport layer |
| 332 | */ |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 333 | void ntb_transport_unregister_client_dev(char *device_name) |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 334 | { |
| 335 | struct ntb_transport_client_dev *client, *cd; |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 336 | struct ntb_transport_ctx *nt; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 337 | |
| 338 | list_for_each_entry(nt, &ntb_transport_list, entry) |
| 339 | list_for_each_entry_safe(client, cd, &nt->client_devs, entry) |
| 340 | if (!strncmp(dev_name(&client->dev), device_name, |
| 341 | strlen(device_name))) { |
| 342 | list_del(&client->entry); |
| 343 | device_unregister(&client->dev); |
| 344 | } |
| 345 | } |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 346 | EXPORT_SYMBOL_GPL(ntb_transport_unregister_client_dev); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 347 | |
| 348 | /** |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 349 | * ntb_transport_register_client_dev - Register NTB client device |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 350 | * @device_name: Name of NTB client device |
| 351 | * |
| 352 | * Register an NTB client device with the NTB transport layer |
| 353 | */ |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 354 | int ntb_transport_register_client_dev(char *device_name) |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 355 | { |
| 356 | struct ntb_transport_client_dev *client_dev; |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 357 | struct ntb_transport_ctx *nt; |
Allen Hubbe | 1199aa6 | 2015-05-18 06:20:47 -0400 | [diff] [blame] | 358 | int node; |
Jon Mason | 8b19d45 | 2013-04-26 14:51:57 -0700 | [diff] [blame] | 359 | int rc, i = 0; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 360 | |
Jon Mason | 8222b40 | 2013-01-19 02:02:16 -0700 | [diff] [blame] | 361 | if (list_empty(&ntb_transport_list)) |
| 362 | return -ENODEV; |
| 363 | |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 364 | list_for_each_entry(nt, &ntb_transport_list, entry) { |
| 365 | struct device *dev; |
| 366 | |
Allen Hubbe | 1199aa6 | 2015-05-18 06:20:47 -0400 | [diff] [blame] | 367 | node = dev_to_node(&nt->ndev->dev); |
| 368 | |
| 369 | client_dev = kzalloc_node(sizeof(*client_dev), |
| 370 | GFP_KERNEL, node); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 371 | if (!client_dev) { |
| 372 | rc = -ENOMEM; |
| 373 | goto err; |
| 374 | } |
| 375 | |
| 376 | dev = &client_dev->dev; |
| 377 | |
| 378 | /* setup and register client devices */ |
Jon Mason | 8b19d45 | 2013-04-26 14:51:57 -0700 | [diff] [blame] | 379 | dev_set_name(dev, "%s%d", device_name, i); |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 380 | dev->bus = &ntb_transport_bus; |
| 381 | dev->release = ntb_transport_client_release; |
| 382 | dev->parent = &nt->ndev->dev; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 383 | |
| 384 | rc = device_register(dev); |
| 385 | if (rc) { |
| 386 | kfree(client_dev); |
| 387 | goto err; |
| 388 | } |
| 389 | |
| 390 | list_add_tail(&client_dev->entry, &nt->client_devs); |
Jon Mason | 8b19d45 | 2013-04-26 14:51:57 -0700 | [diff] [blame] | 391 | i++; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 392 | } |
| 393 | |
| 394 | return 0; |
| 395 | |
| 396 | err: |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 397 | ntb_transport_unregister_client_dev(device_name); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 398 | |
| 399 | return rc; |
| 400 | } |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 401 | EXPORT_SYMBOL_GPL(ntb_transport_register_client_dev); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 402 | |
| 403 | /** |
Allen Hubbe | ec110bc | 2015-05-07 06:45:21 -0400 | [diff] [blame] | 404 | * ntb_transport_register_client - Register NTB client driver |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 405 | * @drv: NTB client driver to be registered |
| 406 | * |
| 407 | * Register an NTB client driver with the NTB transport layer |
| 408 | * |
| 409 | * RETURNS: An appropriate -ERRNO error value on error, or zero for success. |
| 410 | */ |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 411 | int ntb_transport_register_client(struct ntb_transport_client *drv) |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 412 | { |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 413 | drv->driver.bus = &ntb_transport_bus; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 414 | |
Jon Mason | 8222b40 | 2013-01-19 02:02:16 -0700 | [diff] [blame] | 415 | if (list_empty(&ntb_transport_list)) |
| 416 | return -ENODEV; |
| 417 | |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 418 | return driver_register(&drv->driver); |
| 419 | } |
Allen Hubbe | ec110bc | 2015-05-07 06:45:21 -0400 | [diff] [blame] | 420 | EXPORT_SYMBOL_GPL(ntb_transport_register_client); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 421 | |
| 422 | /** |
Allen Hubbe | ec110bc | 2015-05-07 06:45:21 -0400 | [diff] [blame] | 423 | * ntb_transport_unregister_client - Unregister NTB client driver |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 424 | * @drv: NTB client driver to be unregistered |
| 425 | * |
| 426 | * Unregister an NTB client driver with the NTB transport layer |
| 427 | * |
| 428 | * RETURNS: An appropriate -ERRNO error value on error, or zero for success. |
| 429 | */ |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 430 | void ntb_transport_unregister_client(struct ntb_transport_client *drv) |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 431 | { |
| 432 | driver_unregister(&drv->driver); |
| 433 | } |
Allen Hubbe | ec110bc | 2015-05-07 06:45:21 -0400 | [diff] [blame] | 434 | EXPORT_SYMBOL_GPL(ntb_transport_unregister_client); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 435 | |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 436 | static ssize_t debugfs_read(struct file *filp, char __user *ubuf, size_t count, |
| 437 | loff_t *offp) |
| 438 | { |
| 439 | struct ntb_transport_qp *qp; |
Jon Mason | d7237e2 | 2013-01-19 02:02:25 -0700 | [diff] [blame] | 440 | char *buf; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 441 | ssize_t ret, out_offset, out_count; |
| 442 | |
Dave Jiang | 260bee9 | 2015-07-13 08:07:11 -0400 | [diff] [blame] | 443 | qp = filp->private_data; |
| 444 | |
| 445 | if (!qp || !qp->link_is_up) |
| 446 | return 0; |
| 447 | |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 448 | out_count = 1000; |
Jon Mason | d7237e2 | 2013-01-19 02:02:25 -0700 | [diff] [blame] | 449 | |
| 450 | buf = kmalloc(out_count, GFP_KERNEL); |
| 451 | if (!buf) |
| 452 | return -ENOMEM; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 453 | |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 454 | out_offset = 0; |
| 455 | out_offset += snprintf(buf + out_offset, out_count - out_offset, |
Dave Jiang | d98ef99 | 2015-07-13 08:07:20 -0400 | [diff] [blame] | 456 | "\nNTB QP stats:\n\n"); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 457 | out_offset += snprintf(buf + out_offset, out_count - out_offset, |
| 458 | "rx_bytes - \t%llu\n", qp->rx_bytes); |
| 459 | out_offset += snprintf(buf + out_offset, out_count - out_offset, |
| 460 | "rx_pkts - \t%llu\n", qp->rx_pkts); |
| 461 | out_offset += snprintf(buf + out_offset, out_count - out_offset, |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 462 | "rx_memcpy - \t%llu\n", qp->rx_memcpy); |
| 463 | out_offset += snprintf(buf + out_offset, out_count - out_offset, |
| 464 | "rx_async - \t%llu\n", qp->rx_async); |
| 465 | out_offset += snprintf(buf + out_offset, out_count - out_offset, |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 466 | "rx_ring_empty - %llu\n", qp->rx_ring_empty); |
| 467 | out_offset += snprintf(buf + out_offset, out_count - out_offset, |
| 468 | "rx_err_no_buf - %llu\n", qp->rx_err_no_buf); |
| 469 | out_offset += snprintf(buf + out_offset, out_count - out_offset, |
| 470 | "rx_err_oflow - \t%llu\n", qp->rx_err_oflow); |
| 471 | out_offset += snprintf(buf + out_offset, out_count - out_offset, |
| 472 | "rx_err_ver - \t%llu\n", qp->rx_err_ver); |
| 473 | out_offset += snprintf(buf + out_offset, out_count - out_offset, |
Dave Jiang | d98ef99 | 2015-07-13 08:07:20 -0400 | [diff] [blame] | 474 | "rx_buff - \t0x%p\n", qp->rx_buff); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 475 | out_offset += snprintf(buf + out_offset, out_count - out_offset, |
Jon Mason | 793c20e | 2013-01-19 02:02:26 -0700 | [diff] [blame] | 476 | "rx_index - \t%u\n", qp->rx_index); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 477 | out_offset += snprintf(buf + out_offset, out_count - out_offset, |
Dave Jiang | d98ef99 | 2015-07-13 08:07:20 -0400 | [diff] [blame] | 478 | "rx_max_entry - \t%u\n\n", qp->rx_max_entry); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 479 | |
| 480 | out_offset += snprintf(buf + out_offset, out_count - out_offset, |
| 481 | "tx_bytes - \t%llu\n", qp->tx_bytes); |
| 482 | out_offset += snprintf(buf + out_offset, out_count - out_offset, |
| 483 | "tx_pkts - \t%llu\n", qp->tx_pkts); |
| 484 | out_offset += snprintf(buf + out_offset, out_count - out_offset, |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 485 | "tx_memcpy - \t%llu\n", qp->tx_memcpy); |
| 486 | out_offset += snprintf(buf + out_offset, out_count - out_offset, |
| 487 | "tx_async - \t%llu\n", qp->tx_async); |
| 488 | out_offset += snprintf(buf + out_offset, out_count - out_offset, |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 489 | "tx_ring_full - \t%llu\n", qp->tx_ring_full); |
| 490 | out_offset += snprintf(buf + out_offset, out_count - out_offset, |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 491 | "tx_err_no_buf - %llu\n", qp->tx_err_no_buf); |
| 492 | out_offset += snprintf(buf + out_offset, out_count - out_offset, |
Dave Jiang | d98ef99 | 2015-07-13 08:07:20 -0400 | [diff] [blame] | 493 | "tx_mw - \t0x%p\n", qp->tx_mw); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 494 | out_offset += snprintf(buf + out_offset, out_count - out_offset, |
Dave Jiang | d98ef99 | 2015-07-13 08:07:20 -0400 | [diff] [blame] | 495 | "tx_index (H) - \t%u\n", qp->tx_index); |
| 496 | out_offset += snprintf(buf + out_offset, out_count - out_offset, |
| 497 | "RRI (T) - \t%u\n", |
| 498 | qp->remote_rx_info->entry); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 499 | out_offset += snprintf(buf + out_offset, out_count - out_offset, |
Jon Mason | 793c20e | 2013-01-19 02:02:26 -0700 | [diff] [blame] | 500 | "tx_max_entry - \t%u\n", qp->tx_max_entry); |
Dave Jiang | e74bfee | 2015-07-13 08:07:17 -0400 | [diff] [blame] | 501 | out_offset += snprintf(buf + out_offset, out_count - out_offset, |
Dave Jiang | e74bfee | 2015-07-13 08:07:17 -0400 | [diff] [blame] | 502 | "free tx - \t%u\n", |
| 503 | ntb_transport_tx_free_entry(qp)); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 504 | |
| 505 | out_offset += snprintf(buf + out_offset, out_count - out_offset, |
Dave Jiang | d98ef99 | 2015-07-13 08:07:20 -0400 | [diff] [blame] | 506 | "\n"); |
| 507 | out_offset += snprintf(buf + out_offset, out_count - out_offset, |
Dave Jiang | 569410ca | 2015-07-13 08:07:22 -0400 | [diff] [blame] | 508 | "Using TX DMA - \t%s\n", |
| 509 | qp->tx_dma_chan ? "Yes" : "No"); |
| 510 | out_offset += snprintf(buf + out_offset, out_count - out_offset, |
| 511 | "Using RX DMA - \t%s\n", |
| 512 | qp->rx_dma_chan ? "Yes" : "No"); |
Dave Jiang | d98ef99 | 2015-07-13 08:07:20 -0400 | [diff] [blame] | 513 | out_offset += snprintf(buf + out_offset, out_count - out_offset, |
| 514 | "QP Link - \t%s\n", |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 515 | qp->link_is_up ? "Up" : "Down"); |
Dave Jiang | d98ef99 | 2015-07-13 08:07:20 -0400 | [diff] [blame] | 516 | out_offset += snprintf(buf + out_offset, out_count - out_offset, |
| 517 | "\n"); |
| 518 | |
Jon Mason | d7237e2 | 2013-01-19 02:02:25 -0700 | [diff] [blame] | 519 | if (out_offset > out_count) |
| 520 | out_offset = out_count; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 521 | |
| 522 | ret = simple_read_from_buffer(ubuf, count, offp, buf, out_offset); |
Jon Mason | d7237e2 | 2013-01-19 02:02:25 -0700 | [diff] [blame] | 523 | kfree(buf); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 524 | return ret; |
| 525 | } |
| 526 | |
| 527 | static const struct file_operations ntb_qp_debugfs_stats = { |
| 528 | .owner = THIS_MODULE, |
Jon Mason | d66d7ac | 2013-01-19 02:02:20 -0700 | [diff] [blame] | 529 | .open = simple_open, |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 530 | .read = debugfs_read, |
| 531 | }; |
| 532 | |
| 533 | static void ntb_list_add(spinlock_t *lock, struct list_head *entry, |
| 534 | struct list_head *list) |
| 535 | { |
| 536 | unsigned long flags; |
| 537 | |
| 538 | spin_lock_irqsave(lock, flags); |
| 539 | list_add_tail(entry, list); |
| 540 | spin_unlock_irqrestore(lock, flags); |
| 541 | } |
| 542 | |
| 543 | static struct ntb_queue_entry *ntb_list_rm(spinlock_t *lock, |
Jon Mason | 53ca4fe | 2013-11-26 11:21:50 -0700 | [diff] [blame] | 544 | struct list_head *list) |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 545 | { |
| 546 | struct ntb_queue_entry *entry; |
| 547 | unsigned long flags; |
| 548 | |
| 549 | spin_lock_irqsave(lock, flags); |
| 550 | if (list_empty(list)) { |
| 551 | entry = NULL; |
| 552 | goto out; |
| 553 | } |
| 554 | entry = list_first_entry(list, struct ntb_queue_entry, entry); |
| 555 | list_del(&entry->entry); |
Dave Jiang | e74bfee | 2015-07-13 08:07:17 -0400 | [diff] [blame] | 556 | |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 557 | out: |
| 558 | spin_unlock_irqrestore(lock, flags); |
| 559 | |
| 560 | return entry; |
| 561 | } |
| 562 | |
Allen Hubbe | da2e5ae | 2015-07-13 08:07:08 -0400 | [diff] [blame] | 563 | static struct ntb_queue_entry *ntb_list_mv(spinlock_t *lock, |
| 564 | struct list_head *list, |
| 565 | struct list_head *to_list) |
| 566 | { |
| 567 | struct ntb_queue_entry *entry; |
| 568 | unsigned long flags; |
| 569 | |
| 570 | spin_lock_irqsave(lock, flags); |
| 571 | |
| 572 | if (list_empty(list)) { |
| 573 | entry = NULL; |
| 574 | } else { |
| 575 | entry = list_first_entry(list, struct ntb_queue_entry, entry); |
| 576 | list_move_tail(&entry->entry, to_list); |
| 577 | } |
| 578 | |
| 579 | spin_unlock_irqrestore(lock, flags); |
| 580 | |
| 581 | return entry; |
| 582 | } |
| 583 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 584 | static int ntb_transport_setup_qp_mw(struct ntb_transport_ctx *nt, |
| 585 | unsigned int qp_num) |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 586 | { |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 587 | struct ntb_transport_qp *qp = &nt->qp_vec[qp_num]; |
| 588 | struct ntb_transport_mw *mw; |
Jon Mason | ef114ed | 2013-01-19 02:02:18 -0700 | [diff] [blame] | 589 | unsigned int rx_size, num_qps_mw; |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 590 | unsigned int mw_num, mw_count, qp_count; |
Jon Mason | 793c20e | 2013-01-19 02:02:26 -0700 | [diff] [blame] | 591 | unsigned int i; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 592 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 593 | mw_count = nt->mw_count; |
| 594 | qp_count = nt->qp_count; |
Jon Mason | 948d3a6 | 2013-04-18 17:07:36 -0700 | [diff] [blame] | 595 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 596 | mw_num = QP_TO_MW(nt, qp_num); |
| 597 | mw = &nt->mw_vec[mw_num]; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 598 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 599 | if (!mw->virt_addr) |
| 600 | return -ENOMEM; |
| 601 | |
| 602 | if (qp_count % mw_count && mw_num + 1 < qp_count / mw_count) |
| 603 | num_qps_mw = qp_count / mw_count + 1; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 604 | else |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 605 | num_qps_mw = qp_count / mw_count; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 606 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 607 | rx_size = (unsigned int)mw->xlat_size / num_qps_mw; |
Jon Mason | c92ba3c | 2015-10-04 22:54:22 -0400 | [diff] [blame] | 608 | qp->rx_buff = mw->virt_addr + rx_size * (qp_num / mw_count); |
Jon Mason | 793c20e | 2013-01-19 02:02:26 -0700 | [diff] [blame] | 609 | rx_size -= sizeof(struct ntb_rx_info); |
| 610 | |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 611 | qp->remote_rx_info = qp->rx_buff + rx_size; |
| 612 | |
Jon Mason | c9d534c | 2013-02-01 15:45:16 -0700 | [diff] [blame] | 613 | /* Due to housekeeping, there must be atleast 2 buffs */ |
| 614 | qp->rx_max_frame = min(transport_mtu, rx_size / 2); |
Jon Mason | 793c20e | 2013-01-19 02:02:26 -0700 | [diff] [blame] | 615 | qp->rx_max_entry = rx_size / qp->rx_max_frame; |
| 616 | qp->rx_index = 0; |
| 617 | |
Jon Mason | c9d534c | 2013-02-01 15:45:16 -0700 | [diff] [blame] | 618 | qp->remote_rx_info->entry = qp->rx_max_entry - 1; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 619 | |
Jon Mason | ef114ed | 2013-01-19 02:02:18 -0700 | [diff] [blame] | 620 | /* setup the hdr offsets with 0's */ |
Jon Mason | 793c20e | 2013-01-19 02:02:26 -0700 | [diff] [blame] | 621 | for (i = 0; i < qp->rx_max_entry; i++) { |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 622 | void *offset = (qp->rx_buff + qp->rx_max_frame * (i + 1) - |
| 623 | sizeof(struct ntb_payload_header)); |
Jon Mason | ef114ed | 2013-01-19 02:02:18 -0700 | [diff] [blame] | 624 | memset(offset, 0, sizeof(struct ntb_payload_header)); |
Jon Mason | 793c20e | 2013-01-19 02:02:26 -0700 | [diff] [blame] | 625 | } |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 626 | |
| 627 | qp->rx_pkts = 0; |
| 628 | qp->tx_pkts = 0; |
Jon Mason | 90f9e93 | 2013-02-01 15:34:35 -0700 | [diff] [blame] | 629 | qp->tx_index = 0; |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 630 | |
| 631 | return 0; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 632 | } |
| 633 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 634 | static void ntb_free_mw(struct ntb_transport_ctx *nt, int num_mw) |
Jon Mason | 113fc50 | 2013-01-30 11:40:52 -0700 | [diff] [blame] | 635 | { |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 636 | struct ntb_transport_mw *mw = &nt->mw_vec[num_mw]; |
| 637 | struct pci_dev *pdev = nt->ndev->pdev; |
Jon Mason | 113fc50 | 2013-01-30 11:40:52 -0700 | [diff] [blame] | 638 | |
| 639 | if (!mw->virt_addr) |
| 640 | return; |
| 641 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 642 | ntb_mw_clear_trans(nt->ndev, num_mw); |
| 643 | dma_free_coherent(&pdev->dev, mw->buff_size, |
| 644 | mw->virt_addr, mw->dma_addr); |
| 645 | mw->xlat_size = 0; |
| 646 | mw->buff_size = 0; |
Jon Mason | 113fc50 | 2013-01-30 11:40:52 -0700 | [diff] [blame] | 647 | mw->virt_addr = NULL; |
| 648 | } |
| 649 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 650 | static int ntb_set_mw(struct ntb_transport_ctx *nt, int num_mw, |
Allen Hubbe | 8c9edf6 | 2015-07-13 08:07:13 -0400 | [diff] [blame] | 651 | resource_size_t size) |
Jon Mason | b77b263 | 2013-02-01 15:25:37 -0700 | [diff] [blame] | 652 | { |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 653 | struct ntb_transport_mw *mw = &nt->mw_vec[num_mw]; |
| 654 | struct pci_dev *pdev = nt->ndev->pdev; |
Allen Hubbe | 8c9edf6 | 2015-07-13 08:07:13 -0400 | [diff] [blame] | 655 | size_t xlat_size, buff_size; |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 656 | int rc; |
| 657 | |
Allen Hubbe | 8c9edf6 | 2015-07-13 08:07:13 -0400 | [diff] [blame] | 658 | if (!size) |
| 659 | return -EINVAL; |
| 660 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 661 | xlat_size = round_up(size, mw->xlat_align_size); |
| 662 | buff_size = round_up(size, mw->xlat_align); |
Jon Mason | b77b263 | 2013-02-01 15:25:37 -0700 | [diff] [blame] | 663 | |
| 664 | /* No need to re-setup */ |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 665 | if (mw->xlat_size == xlat_size) |
Jon Mason | b77b263 | 2013-02-01 15:25:37 -0700 | [diff] [blame] | 666 | return 0; |
| 667 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 668 | if (mw->buff_size) |
Jon Mason | b77b263 | 2013-02-01 15:25:37 -0700 | [diff] [blame] | 669 | ntb_free_mw(nt, num_mw); |
| 670 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 671 | /* Alloc memory for receiving data. Must be aligned */ |
| 672 | mw->xlat_size = xlat_size; |
| 673 | mw->buff_size = buff_size; |
Jon Mason | b77b263 | 2013-02-01 15:25:37 -0700 | [diff] [blame] | 674 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 675 | mw->virt_addr = dma_alloc_coherent(&pdev->dev, buff_size, |
| 676 | &mw->dma_addr, GFP_KERNEL); |
Jon Mason | b77b263 | 2013-02-01 15:25:37 -0700 | [diff] [blame] | 677 | if (!mw->virt_addr) { |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 678 | mw->xlat_size = 0; |
| 679 | mw->buff_size = 0; |
Allen Hubbe | 8c9edf6 | 2015-07-13 08:07:13 -0400 | [diff] [blame] | 680 | dev_err(&pdev->dev, "Unable to alloc MW buff of size %zu\n", |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 681 | buff_size); |
Jon Mason | b77b263 | 2013-02-01 15:25:37 -0700 | [diff] [blame] | 682 | return -ENOMEM; |
| 683 | } |
| 684 | |
Dave Jiang | 3cc5ba1 | 2014-08-28 13:53:02 -0700 | [diff] [blame] | 685 | /* |
| 686 | * we must ensure that the memory address allocated is BAR size |
| 687 | * aligned in order for the XLAT register to take the value. This |
| 688 | * is a requirement of the hardware. It is recommended to setup CMA |
| 689 | * for BAR sizes equal or greater than 4MB. |
| 690 | */ |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 691 | if (!IS_ALIGNED(mw->dma_addr, mw->xlat_align)) { |
| 692 | dev_err(&pdev->dev, "DMA memory %pad is not aligned\n", |
Dave Jiang | 3cc5ba1 | 2014-08-28 13:53:02 -0700 | [diff] [blame] | 693 | &mw->dma_addr); |
| 694 | ntb_free_mw(nt, num_mw); |
| 695 | return -ENOMEM; |
| 696 | } |
| 697 | |
Jon Mason | b77b263 | 2013-02-01 15:25:37 -0700 | [diff] [blame] | 698 | /* Notify HW the memory location of the receive buffer */ |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 699 | rc = ntb_mw_set_trans(nt->ndev, num_mw, mw->dma_addr, mw->xlat_size); |
| 700 | if (rc) { |
| 701 | dev_err(&pdev->dev, "Unable to set mw%d translation", num_mw); |
| 702 | ntb_free_mw(nt, num_mw); |
| 703 | return -EIO; |
| 704 | } |
Jon Mason | b77b263 | 2013-02-01 15:25:37 -0700 | [diff] [blame] | 705 | |
| 706 | return 0; |
| 707 | } |
| 708 | |
Allen Hubbe | 2849b5d7 | 2015-05-12 08:09:15 -0400 | [diff] [blame] | 709 | static void ntb_qp_link_down_reset(struct ntb_transport_qp *qp) |
| 710 | { |
| 711 | qp->link_is_up = false; |
| 712 | |
| 713 | qp->tx_index = 0; |
| 714 | qp->rx_index = 0; |
| 715 | qp->rx_bytes = 0; |
| 716 | qp->rx_pkts = 0; |
| 717 | qp->rx_ring_empty = 0; |
| 718 | qp->rx_err_no_buf = 0; |
| 719 | qp->rx_err_oflow = 0; |
| 720 | qp->rx_err_ver = 0; |
| 721 | qp->rx_memcpy = 0; |
| 722 | qp->rx_async = 0; |
| 723 | qp->tx_bytes = 0; |
| 724 | qp->tx_pkts = 0; |
| 725 | qp->tx_ring_full = 0; |
| 726 | qp->tx_err_no_buf = 0; |
| 727 | qp->tx_memcpy = 0; |
| 728 | qp->tx_async = 0; |
| 729 | } |
| 730 | |
Jon Mason | fca4d51 | 2013-09-09 13:39:55 -0700 | [diff] [blame] | 731 | static void ntb_qp_link_cleanup(struct ntb_transport_qp *qp) |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 732 | { |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 733 | struct ntb_transport_ctx *nt = qp->transport; |
| 734 | struct pci_dev *pdev = nt->ndev->pdev; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 735 | |
Allen Hubbe | e22e0b9 | 2015-05-12 06:55:44 -0400 | [diff] [blame] | 736 | dev_info(&pdev->dev, "qp %d: Link Cleanup\n", qp->qp_num); |
Allen Hubbe | 2849b5d7 | 2015-05-12 08:09:15 -0400 | [diff] [blame] | 737 | |
| 738 | cancel_delayed_work_sync(&qp->link_work); |
| 739 | ntb_qp_link_down_reset(qp); |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 740 | |
| 741 | if (qp->event_handler) |
| 742 | qp->event_handler(qp->cb_data, qp->link_is_up); |
Jon Mason | fca4d51 | 2013-09-09 13:39:55 -0700 | [diff] [blame] | 743 | } |
| 744 | |
| 745 | static void ntb_qp_link_cleanup_work(struct work_struct *work) |
| 746 | { |
| 747 | struct ntb_transport_qp *qp = container_of(work, |
| 748 | struct ntb_transport_qp, |
| 749 | link_cleanup); |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 750 | struct ntb_transport_ctx *nt = qp->transport; |
Jon Mason | fca4d51 | 2013-09-09 13:39:55 -0700 | [diff] [blame] | 751 | |
| 752 | ntb_qp_link_cleanup(qp); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 753 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 754 | if (nt->link_is_up) |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 755 | schedule_delayed_work(&qp->link_work, |
| 756 | msecs_to_jiffies(NTB_LINK_DOWN_TIMEOUT)); |
| 757 | } |
| 758 | |
Jon Mason | 7b4f2d3 | 2013-01-19 02:02:19 -0700 | [diff] [blame] | 759 | static void ntb_qp_link_down(struct ntb_transport_qp *qp) |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 760 | { |
Jon Mason | 7b4f2d3 | 2013-01-19 02:02:19 -0700 | [diff] [blame] | 761 | schedule_work(&qp->link_cleanup); |
| 762 | } |
| 763 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 764 | static void ntb_transport_link_cleanup(struct ntb_transport_ctx *nt) |
Jon Mason | 7b4f2d3 | 2013-01-19 02:02:19 -0700 | [diff] [blame] | 765 | { |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 766 | struct ntb_transport_qp *qp; |
| 767 | u64 qp_bitmap_alloc; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 768 | int i; |
| 769 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 770 | qp_bitmap_alloc = nt->qp_bitmap & ~nt->qp_bitmap_free; |
Jon Mason | fca4d51 | 2013-09-09 13:39:55 -0700 | [diff] [blame] | 771 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 772 | /* Pass along the info to any clients */ |
| 773 | for (i = 0; i < nt->qp_count; i++) |
| 774 | if (qp_bitmap_alloc & BIT_ULL(i)) { |
| 775 | qp = &nt->qp_vec[i]; |
| 776 | ntb_qp_link_cleanup(qp); |
| 777 | cancel_work_sync(&qp->link_cleanup); |
| 778 | cancel_delayed_work_sync(&qp->link_work); |
| 779 | } |
| 780 | |
| 781 | if (!nt->link_is_up) |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 782 | cancel_delayed_work_sync(&nt->link_work); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 783 | |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 784 | /* The scratchpad registers keep the values if the remote side |
| 785 | * goes down, blast them now to give them a sane value the next |
| 786 | * time they are accessed |
| 787 | */ |
| 788 | for (i = 0; i < MAX_SPAD; i++) |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 789 | ntb_spad_write(nt->ndev, i, 0); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 790 | } |
| 791 | |
Jon Mason | fca4d51 | 2013-09-09 13:39:55 -0700 | [diff] [blame] | 792 | static void ntb_transport_link_cleanup_work(struct work_struct *work) |
| 793 | { |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 794 | struct ntb_transport_ctx *nt = |
| 795 | container_of(work, struct ntb_transport_ctx, link_cleanup); |
Jon Mason | fca4d51 | 2013-09-09 13:39:55 -0700 | [diff] [blame] | 796 | |
| 797 | ntb_transport_link_cleanup(nt); |
| 798 | } |
| 799 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 800 | static void ntb_transport_event_callback(void *data) |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 801 | { |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 802 | struct ntb_transport_ctx *nt = data; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 803 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 804 | if (ntb_link_is_up(nt->ndev, NULL, NULL) == 1) |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 805 | schedule_delayed_work(&nt->link_work, 0); |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 806 | else |
Jon Mason | 7b4f2d3 | 2013-01-19 02:02:19 -0700 | [diff] [blame] | 807 | schedule_work(&nt->link_cleanup); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 808 | } |
| 809 | |
| 810 | static void ntb_transport_link_work(struct work_struct *work) |
| 811 | { |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 812 | struct ntb_transport_ctx *nt = |
| 813 | container_of(work, struct ntb_transport_ctx, link_work.work); |
| 814 | struct ntb_dev *ndev = nt->ndev; |
| 815 | struct pci_dev *pdev = ndev->pdev; |
| 816 | resource_size_t size; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 817 | u32 val; |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 818 | int rc, i, spad; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 819 | |
Jon Mason | 113fc50 | 2013-01-30 11:40:52 -0700 | [diff] [blame] | 820 | /* send the local info, in the opposite order of the way we read it */ |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 821 | for (i = 0; i < nt->mw_count; i++) { |
| 822 | size = nt->mw_vec[i].phys_size; |
Jon Mason | 113fc50 | 2013-01-30 11:40:52 -0700 | [diff] [blame] | 823 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 824 | if (max_mw_size && size > max_mw_size) |
| 825 | size = max_mw_size; |
| 826 | |
| 827 | spad = MW0_SZ_HIGH + (i * 2); |
Arnd Bergmann | fdcb4b2 | 2015-10-07 09:59:34 +0200 | [diff] [blame] | 828 | ntb_peer_spad_write(ndev, spad, upper_32_bits(size)); |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 829 | |
| 830 | spad = MW0_SZ_LOW + (i * 2); |
Arnd Bergmann | fdcb4b2 | 2015-10-07 09:59:34 +0200 | [diff] [blame] | 831 | ntb_peer_spad_write(ndev, spad, lower_32_bits(size)); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 832 | } |
| 833 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 834 | ntb_peer_spad_write(ndev, NUM_MWS, nt->mw_count); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 835 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 836 | ntb_peer_spad_write(ndev, NUM_QPS, nt->qp_count); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 837 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 838 | ntb_peer_spad_write(ndev, VERSION, NTB_TRANSPORT_VERSION); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 839 | |
| 840 | /* Query the remote side for its info */ |
Dave Jiang | 0f69a7d | 2015-06-02 03:45:07 -0400 | [diff] [blame] | 841 | val = ntb_spad_read(ndev, VERSION); |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 842 | dev_dbg(&pdev->dev, "Remote version = %d\n", val); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 843 | if (val != NTB_TRANSPORT_VERSION) |
| 844 | goto out; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 845 | |
Dave Jiang | 0f69a7d | 2015-06-02 03:45:07 -0400 | [diff] [blame] | 846 | val = ntb_spad_read(ndev, NUM_QPS); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 847 | dev_dbg(&pdev->dev, "Remote max number of qps = %d\n", val); |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 848 | if (val != nt->qp_count) |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 849 | goto out; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 850 | |
Dave Jiang | 0f69a7d | 2015-06-02 03:45:07 -0400 | [diff] [blame] | 851 | val = ntb_spad_read(ndev, NUM_MWS); |
Jon Mason | 113fc50 | 2013-01-30 11:40:52 -0700 | [diff] [blame] | 852 | dev_dbg(&pdev->dev, "Remote number of mws = %d\n", val); |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 853 | if (val != nt->mw_count) |
| 854 | goto out; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 855 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 856 | for (i = 0; i < nt->mw_count; i++) { |
Jon Mason | 113fc50 | 2013-01-30 11:40:52 -0700 | [diff] [blame] | 857 | u64 val64; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 858 | |
Dave Jiang | 0f69a7d | 2015-06-02 03:45:07 -0400 | [diff] [blame] | 859 | val = ntb_spad_read(ndev, MW0_SZ_HIGH + (i * 2)); |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 860 | val64 = (u64)val << 32; |
Jon Mason | 113fc50 | 2013-01-30 11:40:52 -0700 | [diff] [blame] | 861 | |
Dave Jiang | 0f69a7d | 2015-06-02 03:45:07 -0400 | [diff] [blame] | 862 | val = ntb_spad_read(ndev, MW0_SZ_LOW + (i * 2)); |
Jon Mason | 113fc50 | 2013-01-30 11:40:52 -0700 | [diff] [blame] | 863 | val64 |= val; |
| 864 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 865 | dev_dbg(&pdev->dev, "Remote MW%d size = %#llx\n", i, val64); |
Jon Mason | 113fc50 | 2013-01-30 11:40:52 -0700 | [diff] [blame] | 866 | |
| 867 | rc = ntb_set_mw(nt, i, val64); |
| 868 | if (rc) |
| 869 | goto out1; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 870 | } |
| 871 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 872 | nt->link_is_up = true; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 873 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 874 | for (i = 0; i < nt->qp_count; i++) { |
| 875 | struct ntb_transport_qp *qp = &nt->qp_vec[i]; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 876 | |
| 877 | ntb_transport_setup_qp_mw(nt, i); |
| 878 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 879 | if (qp->client_ready) |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 880 | schedule_delayed_work(&qp->link_work, 0); |
| 881 | } |
| 882 | |
| 883 | return; |
| 884 | |
Jon Mason | 113fc50 | 2013-01-30 11:40:52 -0700 | [diff] [blame] | 885 | out1: |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 886 | for (i = 0; i < nt->mw_count; i++) |
Jon Mason | 113fc50 | 2013-01-30 11:40:52 -0700 | [diff] [blame] | 887 | ntb_free_mw(nt, i); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 888 | out: |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 889 | if (ntb_link_is_up(ndev, NULL, NULL) == 1) |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 890 | schedule_delayed_work(&nt->link_work, |
| 891 | msecs_to_jiffies(NTB_LINK_DOWN_TIMEOUT)); |
| 892 | } |
| 893 | |
| 894 | static void ntb_qp_link_work(struct work_struct *work) |
| 895 | { |
| 896 | struct ntb_transport_qp *qp = container_of(work, |
| 897 | struct ntb_transport_qp, |
| 898 | link_work.work); |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 899 | struct pci_dev *pdev = qp->ndev->pdev; |
| 900 | struct ntb_transport_ctx *nt = qp->transport; |
| 901 | int val; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 902 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 903 | WARN_ON(!nt->link_is_up); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 904 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 905 | val = ntb_spad_read(nt->ndev, QP_LINKS); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 906 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 907 | ntb_peer_spad_write(nt->ndev, QP_LINKS, val | BIT(qp->qp_num)); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 908 | |
| 909 | /* query remote spad for qp ready bits */ |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 910 | ntb_peer_spad_read(nt->ndev, QP_LINKS); |
Allen Hubbe | 2876228 | 2015-05-11 10:08:26 -0400 | [diff] [blame] | 911 | dev_dbg_ratelimited(&pdev->dev, "Remote QP link status = %x\n", val); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 912 | |
| 913 | /* See if the remote side is up */ |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 914 | if (val & BIT(qp->qp_num)) { |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 915 | dev_info(&pdev->dev, "qp %d: Link Up\n", qp->qp_num); |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 916 | qp->link_is_up = true; |
| 917 | |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 918 | if (qp->event_handler) |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 919 | qp->event_handler(qp->cb_data, qp->link_is_up); |
Allen Hubbe | 8b5a22d | 2015-07-13 08:07:12 -0400 | [diff] [blame] | 920 | |
| 921 | tasklet_schedule(&qp->rxc_db_work); |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 922 | } else if (nt->link_is_up) |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 923 | schedule_delayed_work(&qp->link_work, |
| 924 | msecs_to_jiffies(NTB_LINK_DOWN_TIMEOUT)); |
| 925 | } |
| 926 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 927 | static int ntb_transport_init_queue(struct ntb_transport_ctx *nt, |
Jon Mason | 53ca4fe | 2013-11-26 11:21:50 -0700 | [diff] [blame] | 928 | unsigned int qp_num) |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 929 | { |
| 930 | struct ntb_transport_qp *qp; |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 931 | phys_addr_t mw_base; |
| 932 | resource_size_t mw_size; |
Jon Mason | ef114ed | 2013-01-19 02:02:18 -0700 | [diff] [blame] | 933 | unsigned int num_qps_mw, tx_size; |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 934 | unsigned int mw_num, mw_count, qp_count; |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 935 | u64 qp_offset; |
Jon Mason | 948d3a6 | 2013-04-18 17:07:36 -0700 | [diff] [blame] | 936 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 937 | mw_count = nt->mw_count; |
| 938 | qp_count = nt->qp_count; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 939 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 940 | mw_num = QP_TO_MW(nt, qp_num); |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 941 | |
| 942 | qp = &nt->qp_vec[qp_num]; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 943 | qp->qp_num = qp_num; |
| 944 | qp->transport = nt; |
| 945 | qp->ndev = nt->ndev; |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 946 | qp->client_ready = false; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 947 | qp->event_handler = NULL; |
Allen Hubbe | 2849b5d7 | 2015-05-12 08:09:15 -0400 | [diff] [blame] | 948 | ntb_qp_link_down_reset(qp); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 949 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 950 | if (qp_count % mw_count && mw_num + 1 < qp_count / mw_count) |
| 951 | num_qps_mw = qp_count / mw_count + 1; |
Jon Mason | ef114ed | 2013-01-19 02:02:18 -0700 | [diff] [blame] | 952 | else |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 953 | num_qps_mw = qp_count / mw_count; |
Jon Mason | ef114ed | 2013-01-19 02:02:18 -0700 | [diff] [blame] | 954 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 955 | mw_base = nt->mw_vec[mw_num].phys_addr; |
| 956 | mw_size = nt->mw_vec[mw_num].phys_size; |
| 957 | |
| 958 | tx_size = (unsigned int)mw_size / num_qps_mw; |
Jon Mason | c92ba3c | 2015-10-04 22:54:22 -0400 | [diff] [blame] | 959 | qp_offset = tx_size * (qp_num / mw_count); |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 960 | |
| 961 | qp->tx_mw = nt->mw_vec[mw_num].vbase + qp_offset; |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 962 | if (!qp->tx_mw) |
| 963 | return -EINVAL; |
Jon Mason | 793c20e | 2013-01-19 02:02:26 -0700 | [diff] [blame] | 964 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 965 | qp->tx_mw_phys = mw_base + qp_offset; |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 966 | if (!qp->tx_mw_phys) |
| 967 | return -EINVAL; |
| 968 | |
| 969 | tx_size -= sizeof(struct ntb_rx_info); |
| 970 | qp->rx_info = qp->tx_mw + tx_size; |
| 971 | |
Jon Mason | c9d534c | 2013-02-01 15:45:16 -0700 | [diff] [blame] | 972 | /* Due to housekeeping, there must be atleast 2 buffs */ |
| 973 | qp->tx_max_frame = min(transport_mtu, tx_size / 2); |
Jon Mason | 793c20e | 2013-01-19 02:02:26 -0700 | [diff] [blame] | 974 | qp->tx_max_entry = tx_size / qp->tx_max_frame; |
Jon Mason | ef114ed | 2013-01-19 02:02:18 -0700 | [diff] [blame] | 975 | |
Dave Jiang | c8650fd | 2015-07-13 08:07:09 -0400 | [diff] [blame] | 976 | if (nt->debugfs_node_dir) { |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 977 | char debugfs_name[4]; |
| 978 | |
| 979 | snprintf(debugfs_name, 4, "qp%d", qp_num); |
| 980 | qp->debugfs_dir = debugfs_create_dir(debugfs_name, |
Dave Jiang | c8650fd | 2015-07-13 08:07:09 -0400 | [diff] [blame] | 981 | nt->debugfs_node_dir); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 982 | |
| 983 | qp->debugfs_stats = debugfs_create_file("stats", S_IRUSR, |
| 984 | qp->debugfs_dir, qp, |
| 985 | &ntb_qp_debugfs_stats); |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 986 | } else { |
| 987 | qp->debugfs_dir = NULL; |
| 988 | qp->debugfs_stats = NULL; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 989 | } |
| 990 | |
| 991 | INIT_DELAYED_WORK(&qp->link_work, ntb_qp_link_work); |
Jon Mason | fca4d51 | 2013-09-09 13:39:55 -0700 | [diff] [blame] | 992 | INIT_WORK(&qp->link_cleanup, ntb_qp_link_cleanup_work); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 993 | |
Allen Hubbe | da2e5ae | 2015-07-13 08:07:08 -0400 | [diff] [blame] | 994 | spin_lock_init(&qp->ntb_rx_q_lock); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 995 | spin_lock_init(&qp->ntb_tx_free_q_lock); |
| 996 | |
Allen Hubbe | da2e5ae | 2015-07-13 08:07:08 -0400 | [diff] [blame] | 997 | INIT_LIST_HEAD(&qp->rx_post_q); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 998 | INIT_LIST_HEAD(&qp->rx_pend_q); |
| 999 | INIT_LIST_HEAD(&qp->rx_free_q); |
| 1000 | INIT_LIST_HEAD(&qp->tx_free_q); |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 1001 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 1002 | tasklet_init(&qp->rxc_db_work, ntb_transport_rxc_db, |
| 1003 | (unsigned long)qp); |
| 1004 | |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 1005 | return 0; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1006 | } |
| 1007 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 1008 | static int ntb_transport_probe(struct ntb_client *self, struct ntb_dev *ndev) |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1009 | { |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 1010 | struct ntb_transport_ctx *nt; |
| 1011 | struct ntb_transport_mw *mw; |
| 1012 | unsigned int mw_count, qp_count; |
| 1013 | u64 qp_bitmap; |
Allen Hubbe | 1199aa6 | 2015-05-18 06:20:47 -0400 | [diff] [blame] | 1014 | int node; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1015 | int rc, i; |
| 1016 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 1017 | if (ntb_db_is_unsafe(ndev)) |
| 1018 | dev_dbg(&ndev->dev, |
| 1019 | "doorbell is unsafe, proceed anyway...\n"); |
| 1020 | if (ntb_spad_is_unsafe(ndev)) |
| 1021 | dev_dbg(&ndev->dev, |
| 1022 | "scratchpad is unsafe, proceed anyway...\n"); |
| 1023 | |
Allen Hubbe | 1199aa6 | 2015-05-18 06:20:47 -0400 | [diff] [blame] | 1024 | node = dev_to_node(&ndev->dev); |
| 1025 | |
| 1026 | nt = kzalloc_node(sizeof(*nt), GFP_KERNEL, node); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1027 | if (!nt) |
| 1028 | return -ENOMEM; |
| 1029 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 1030 | nt->ndev = ndev; |
| 1031 | |
| 1032 | mw_count = ntb_mw_count(ndev); |
| 1033 | |
| 1034 | nt->mw_count = mw_count; |
| 1035 | |
Allen Hubbe | 1199aa6 | 2015-05-18 06:20:47 -0400 | [diff] [blame] | 1036 | nt->mw_vec = kzalloc_node(mw_count * sizeof(*nt->mw_vec), |
| 1037 | GFP_KERNEL, node); |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 1038 | if (!nt->mw_vec) { |
| 1039 | rc = -ENOMEM; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1040 | goto err; |
| 1041 | } |
| 1042 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 1043 | for (i = 0; i < mw_count; i++) { |
| 1044 | mw = &nt->mw_vec[i]; |
| 1045 | |
| 1046 | rc = ntb_mw_get_range(ndev, i, &mw->phys_addr, &mw->phys_size, |
| 1047 | &mw->xlat_align, &mw->xlat_align_size); |
| 1048 | if (rc) |
| 1049 | goto err1; |
| 1050 | |
Dave Jiang | 06917f7 | 2015-05-19 16:45:46 -0400 | [diff] [blame] | 1051 | mw->vbase = ioremap_wc(mw->phys_addr, mw->phys_size); |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 1052 | if (!mw->vbase) { |
| 1053 | rc = -ENOMEM; |
| 1054 | goto err1; |
| 1055 | } |
| 1056 | |
| 1057 | mw->buff_size = 0; |
| 1058 | mw->xlat_size = 0; |
| 1059 | mw->virt_addr = NULL; |
| 1060 | mw->dma_addr = 0; |
Jon Mason | 948d3a6 | 2013-04-18 17:07:36 -0700 | [diff] [blame] | 1061 | } |
| 1062 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 1063 | qp_bitmap = ntb_db_valid_mask(ndev); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1064 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 1065 | qp_count = ilog2(qp_bitmap); |
| 1066 | if (max_num_clients && max_num_clients < qp_count) |
| 1067 | qp_count = max_num_clients; |
| 1068 | else if (mw_count < qp_count) |
| 1069 | qp_count = mw_count; |
| 1070 | |
| 1071 | qp_bitmap &= BIT_ULL(qp_count) - 1; |
| 1072 | |
| 1073 | nt->qp_count = qp_count; |
| 1074 | nt->qp_bitmap = qp_bitmap; |
| 1075 | nt->qp_bitmap_free = qp_bitmap; |
| 1076 | |
Allen Hubbe | 1199aa6 | 2015-05-18 06:20:47 -0400 | [diff] [blame] | 1077 | nt->qp_vec = kzalloc_node(qp_count * sizeof(*nt->qp_vec), |
| 1078 | GFP_KERNEL, node); |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 1079 | if (!nt->qp_vec) { |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1080 | rc = -ENOMEM; |
Sudip Mukherjee | d4adee0 | 2015-10-03 13:39:41 +0530 | [diff] [blame] | 1081 | goto err1; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1082 | } |
| 1083 | |
Dave Jiang | c8650fd | 2015-07-13 08:07:09 -0400 | [diff] [blame] | 1084 | if (nt_debugfs_dir) { |
| 1085 | nt->debugfs_node_dir = |
| 1086 | debugfs_create_dir(pci_name(ndev->pdev), |
| 1087 | nt_debugfs_dir); |
| 1088 | } |
| 1089 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 1090 | for (i = 0; i < qp_count; i++) { |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 1091 | rc = ntb_transport_init_queue(nt, i); |
| 1092 | if (rc) |
Sudip Mukherjee | d4adee0 | 2015-10-03 13:39:41 +0530 | [diff] [blame] | 1093 | goto err2; |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 1094 | } |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1095 | |
| 1096 | INIT_DELAYED_WORK(&nt->link_work, ntb_transport_link_work); |
Jon Mason | fca4d51 | 2013-09-09 13:39:55 -0700 | [diff] [blame] | 1097 | INIT_WORK(&nt->link_cleanup, ntb_transport_link_cleanup_work); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1098 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 1099 | rc = ntb_set_ctx(ndev, nt, &ntb_transport_ops); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1100 | if (rc) |
Sudip Mukherjee | d4adee0 | 2015-10-03 13:39:41 +0530 | [diff] [blame] | 1101 | goto err2; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1102 | |
| 1103 | INIT_LIST_HEAD(&nt->client_devs); |
| 1104 | rc = ntb_bus_init(nt); |
| 1105 | if (rc) |
Sudip Mukherjee | d4adee0 | 2015-10-03 13:39:41 +0530 | [diff] [blame] | 1106 | goto err3; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1107 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 1108 | nt->link_is_up = false; |
| 1109 | ntb_link_enable(ndev, NTB_SPEED_AUTO, NTB_WIDTH_AUTO); |
| 1110 | ntb_link_event(ndev); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1111 | |
| 1112 | return 0; |
| 1113 | |
Jon Mason | 948d3a6 | 2013-04-18 17:07:36 -0700 | [diff] [blame] | 1114 | err3: |
Sudip Mukherjee | d4adee0 | 2015-10-03 13:39:41 +0530 | [diff] [blame] | 1115 | ntb_clear_ctx(ndev); |
Jon Mason | 948d3a6 | 2013-04-18 17:07:36 -0700 | [diff] [blame] | 1116 | err2: |
Sudip Mukherjee | d4adee0 | 2015-10-03 13:39:41 +0530 | [diff] [blame] | 1117 | kfree(nt->qp_vec); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1118 | err1: |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 1119 | while (i--) { |
| 1120 | mw = &nt->mw_vec[i]; |
| 1121 | iounmap(mw->vbase); |
| 1122 | } |
Sudip Mukherjee | d4adee0 | 2015-10-03 13:39:41 +0530 | [diff] [blame] | 1123 | kfree(nt->mw_vec); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1124 | err: |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1125 | kfree(nt); |
| 1126 | return rc; |
| 1127 | } |
| 1128 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 1129 | static void ntb_transport_free(struct ntb_client *self, struct ntb_dev *ndev) |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1130 | { |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 1131 | struct ntb_transport_ctx *nt = ndev->ctx; |
| 1132 | struct ntb_transport_qp *qp; |
| 1133 | u64 qp_bitmap_alloc; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1134 | int i; |
| 1135 | |
Jon Mason | fca4d51 | 2013-09-09 13:39:55 -0700 | [diff] [blame] | 1136 | ntb_transport_link_cleanup(nt); |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 1137 | cancel_work_sync(&nt->link_cleanup); |
| 1138 | cancel_delayed_work_sync(&nt->link_work); |
| 1139 | |
| 1140 | qp_bitmap_alloc = nt->qp_bitmap & ~nt->qp_bitmap_free; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1141 | |
| 1142 | /* verify that all the qp's are freed */ |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 1143 | for (i = 0; i < nt->qp_count; i++) { |
| 1144 | qp = &nt->qp_vec[i]; |
| 1145 | if (qp_bitmap_alloc & BIT_ULL(i)) |
| 1146 | ntb_transport_free_queue(qp); |
| 1147 | debugfs_remove_recursive(qp->debugfs_dir); |
Jon Mason | 1517a3f | 2013-07-30 15:58:49 -0700 | [diff] [blame] | 1148 | } |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1149 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 1150 | ntb_link_disable(ndev); |
| 1151 | ntb_clear_ctx(ndev); |
| 1152 | |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1153 | ntb_bus_remove(nt); |
| 1154 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 1155 | for (i = nt->mw_count; i--; ) { |
Jon Mason | 113fc50 | 2013-01-30 11:40:52 -0700 | [diff] [blame] | 1156 | ntb_free_mw(nt, i); |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 1157 | iounmap(nt->mw_vec[i].vbase); |
| 1158 | } |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1159 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 1160 | kfree(nt->qp_vec); |
| 1161 | kfree(nt->mw_vec); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1162 | kfree(nt); |
| 1163 | } |
| 1164 | |
Allen Hubbe | da2e5ae | 2015-07-13 08:07:08 -0400 | [diff] [blame] | 1165 | static void ntb_complete_rxc(struct ntb_transport_qp *qp) |
| 1166 | { |
| 1167 | struct ntb_queue_entry *entry; |
| 1168 | void *cb_data; |
| 1169 | unsigned int len; |
| 1170 | unsigned long irqflags; |
| 1171 | |
| 1172 | spin_lock_irqsave(&qp->ntb_rx_q_lock, irqflags); |
| 1173 | |
| 1174 | while (!list_empty(&qp->rx_post_q)) { |
| 1175 | entry = list_first_entry(&qp->rx_post_q, |
| 1176 | struct ntb_queue_entry, entry); |
| 1177 | if (!(entry->flags & DESC_DONE_FLAG)) |
| 1178 | break; |
| 1179 | |
| 1180 | entry->rx_hdr->flags = 0; |
| 1181 | iowrite32(entry->index, &qp->rx_info->entry); |
| 1182 | |
| 1183 | cb_data = entry->cb_data; |
| 1184 | len = entry->len; |
| 1185 | |
| 1186 | list_move_tail(&entry->entry, &qp->rx_free_q); |
| 1187 | |
| 1188 | spin_unlock_irqrestore(&qp->ntb_rx_q_lock, irqflags); |
| 1189 | |
| 1190 | if (qp->rx_handler && qp->client_ready) |
| 1191 | qp->rx_handler(qp, qp->cb_data, cb_data, len); |
| 1192 | |
| 1193 | spin_lock_irqsave(&qp->ntb_rx_q_lock, irqflags); |
| 1194 | } |
| 1195 | |
| 1196 | spin_unlock_irqrestore(&qp->ntb_rx_q_lock, irqflags); |
| 1197 | } |
| 1198 | |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 1199 | static void ntb_rx_copy_callback(void *data) |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1200 | { |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 1201 | struct ntb_queue_entry *entry = data; |
Jon Mason | 448c6fb | 2013-01-19 02:02:27 -0700 | [diff] [blame] | 1202 | |
Allen Hubbe | da2e5ae | 2015-07-13 08:07:08 -0400 | [diff] [blame] | 1203 | entry->flags |= DESC_DONE_FLAG; |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 1204 | |
Allen Hubbe | da2e5ae | 2015-07-13 08:07:08 -0400 | [diff] [blame] | 1205 | ntb_complete_rxc(entry->qp); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1206 | } |
| 1207 | |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 1208 | static void ntb_memcpy_rx(struct ntb_queue_entry *entry, void *offset) |
| 1209 | { |
| 1210 | void *buf = entry->buf; |
| 1211 | size_t len = entry->len; |
| 1212 | |
| 1213 | memcpy(buf, offset, len); |
| 1214 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 1215 | /* Ensure that the data is fully copied out before clearing the flag */ |
| 1216 | wmb(); |
| 1217 | |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 1218 | ntb_rx_copy_callback(entry); |
| 1219 | } |
| 1220 | |
Allen Hubbe | da2e5ae | 2015-07-13 08:07:08 -0400 | [diff] [blame] | 1221 | static void ntb_async_rx(struct ntb_queue_entry *entry, void *offset) |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 1222 | { |
| 1223 | struct dma_async_tx_descriptor *txd; |
| 1224 | struct ntb_transport_qp *qp = entry->qp; |
Dave Jiang | 569410ca | 2015-07-13 08:07:22 -0400 | [diff] [blame] | 1225 | struct dma_chan *chan = qp->rx_dma_chan; |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 1226 | struct dma_device *device; |
Allen Hubbe | da2e5ae | 2015-07-13 08:07:08 -0400 | [diff] [blame] | 1227 | size_t pay_off, buff_off, len; |
Bartlomiej Zolnierkiewicz | 6f57fd0 | 2013-10-18 19:35:31 +0200 | [diff] [blame] | 1228 | struct dmaengine_unmap_data *unmap; |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 1229 | dma_cookie_t cookie; |
| 1230 | void *buf = entry->buf; |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 1231 | |
Allen Hubbe | da2e5ae | 2015-07-13 08:07:08 -0400 | [diff] [blame] | 1232 | len = entry->len; |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 1233 | |
| 1234 | if (!chan) |
| 1235 | goto err; |
| 1236 | |
Jon Mason | 53ca4fe | 2013-11-26 11:21:50 -0700 | [diff] [blame] | 1237 | if (len < copy_bytes) |
Allen Hubbe | 905921e | 2015-07-13 08:07:21 -0400 | [diff] [blame] | 1238 | goto err; |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 1239 | |
| 1240 | device = chan->device; |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 1241 | pay_off = (size_t)offset & ~PAGE_MASK; |
| 1242 | buff_off = (size_t)buf & ~PAGE_MASK; |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 1243 | |
| 1244 | if (!is_dma_copy_aligned(device, pay_off, buff_off, len)) |
Allen Hubbe | 905921e | 2015-07-13 08:07:21 -0400 | [diff] [blame] | 1245 | goto err; |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 1246 | |
Bartlomiej Zolnierkiewicz | 6f57fd0 | 2013-10-18 19:35:31 +0200 | [diff] [blame] | 1247 | unmap = dmaengine_get_unmap_data(device->dev, 2, GFP_NOWAIT); |
| 1248 | if (!unmap) |
Allen Hubbe | 905921e | 2015-07-13 08:07:21 -0400 | [diff] [blame] | 1249 | goto err; |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 1250 | |
Bartlomiej Zolnierkiewicz | 6f57fd0 | 2013-10-18 19:35:31 +0200 | [diff] [blame] | 1251 | unmap->len = len; |
| 1252 | unmap->addr[0] = dma_map_page(device->dev, virt_to_page(offset), |
| 1253 | pay_off, len, DMA_TO_DEVICE); |
| 1254 | if (dma_mapping_error(device->dev, unmap->addr[0])) |
| 1255 | goto err_get_unmap; |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 1256 | |
Bartlomiej Zolnierkiewicz | 6f57fd0 | 2013-10-18 19:35:31 +0200 | [diff] [blame] | 1257 | unmap->to_cnt = 1; |
| 1258 | |
| 1259 | unmap->addr[1] = dma_map_page(device->dev, virt_to_page(buf), |
| 1260 | buff_off, len, DMA_FROM_DEVICE); |
| 1261 | if (dma_mapping_error(device->dev, unmap->addr[1])) |
| 1262 | goto err_get_unmap; |
| 1263 | |
| 1264 | unmap->from_cnt = 1; |
| 1265 | |
Bartlomiej Zolnierkiewicz | 6f57fd0 | 2013-10-18 19:35:31 +0200 | [diff] [blame] | 1266 | txd = device->device_prep_dma_memcpy(chan, unmap->addr[1], |
Bartlomiej Zolnierkiewicz | 0776ae7 | 2013-10-18 19:35:33 +0200 | [diff] [blame] | 1267 | unmap->addr[0], len, |
| 1268 | DMA_PREP_INTERRUPT); |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 1269 | if (!txd) |
Bartlomiej Zolnierkiewicz | 6f57fd0 | 2013-10-18 19:35:31 +0200 | [diff] [blame] | 1270 | goto err_get_unmap; |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 1271 | |
| 1272 | txd->callback = ntb_rx_copy_callback; |
| 1273 | txd->callback_param = entry; |
Bartlomiej Zolnierkiewicz | 6f57fd0 | 2013-10-18 19:35:31 +0200 | [diff] [blame] | 1274 | dma_set_unmap(txd, unmap); |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 1275 | |
| 1276 | cookie = dmaengine_submit(txd); |
| 1277 | if (dma_submit_error(cookie)) |
Bartlomiej Zolnierkiewicz | 6f57fd0 | 2013-10-18 19:35:31 +0200 | [diff] [blame] | 1278 | goto err_set_unmap; |
| 1279 | |
| 1280 | dmaengine_unmap_put(unmap); |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 1281 | |
| 1282 | qp->last_cookie = cookie; |
| 1283 | |
| 1284 | qp->rx_async++; |
| 1285 | |
| 1286 | return; |
| 1287 | |
Bartlomiej Zolnierkiewicz | 6f57fd0 | 2013-10-18 19:35:31 +0200 | [diff] [blame] | 1288 | err_set_unmap: |
| 1289 | dmaengine_unmap_put(unmap); |
| 1290 | err_get_unmap: |
| 1291 | dmaengine_unmap_put(unmap); |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 1292 | err: |
| 1293 | ntb_memcpy_rx(entry, offset); |
| 1294 | qp->rx_memcpy++; |
| 1295 | } |
| 1296 | |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1297 | static int ntb_process_rxc(struct ntb_transport_qp *qp) |
| 1298 | { |
| 1299 | struct ntb_payload_header *hdr; |
| 1300 | struct ntb_queue_entry *entry; |
| 1301 | void *offset; |
| 1302 | |
Jon Mason | 793c20e | 2013-01-19 02:02:26 -0700 | [diff] [blame] | 1303 | offset = qp->rx_buff + qp->rx_max_frame * qp->rx_index; |
| 1304 | hdr = offset + qp->rx_max_frame - sizeof(struct ntb_payload_header); |
| 1305 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 1306 | dev_dbg(&qp->ndev->pdev->dev, "qp %d: RX ver %u len %d flags %x\n", |
| 1307 | qp->qp_num, hdr->ver, hdr->len, hdr->flags); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1308 | |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1309 | if (!(hdr->flags & DESC_DONE_FLAG)) { |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 1310 | dev_dbg(&qp->ndev->pdev->dev, "done flag not set\n"); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1311 | qp->rx_ring_empty++; |
| 1312 | return -EAGAIN; |
| 1313 | } |
| 1314 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 1315 | if (hdr->flags & LINK_DOWN_FLAG) { |
| 1316 | dev_dbg(&qp->ndev->pdev->dev, "link down flag set\n"); |
| 1317 | ntb_qp_link_down(qp); |
| 1318 | hdr->flags = 0; |
Allen Hubbe | c0900b3 | 2015-05-12 06:24:27 -0400 | [diff] [blame] | 1319 | return -EAGAIN; |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 1320 | } |
| 1321 | |
| 1322 | if (hdr->ver != (u32)qp->rx_pkts) { |
| 1323 | dev_dbg(&qp->ndev->pdev->dev, |
| 1324 | "version mismatch, expected %llu - got %u\n", |
| 1325 | qp->rx_pkts, hdr->ver); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1326 | qp->rx_err_ver++; |
| 1327 | return -EIO; |
| 1328 | } |
| 1329 | |
Allen Hubbe | da2e5ae | 2015-07-13 08:07:08 -0400 | [diff] [blame] | 1330 | entry = ntb_list_mv(&qp->ntb_rx_q_lock, &qp->rx_pend_q, &qp->rx_post_q); |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 1331 | if (!entry) { |
| 1332 | dev_dbg(&qp->ndev->pdev->dev, "no receive buffer\n"); |
| 1333 | qp->rx_err_no_buf++; |
Allen Hubbe | da2e5ae | 2015-07-13 08:07:08 -0400 | [diff] [blame] | 1334 | return -EAGAIN; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1335 | } |
| 1336 | |
Allen Hubbe | da2e5ae | 2015-07-13 08:07:08 -0400 | [diff] [blame] | 1337 | entry->rx_hdr = hdr; |
| 1338 | entry->index = qp->rx_index; |
| 1339 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 1340 | if (hdr->len > entry->len) { |
| 1341 | dev_dbg(&qp->ndev->pdev->dev, |
| 1342 | "receive buffer overflow! Wanted %d got %d\n", |
| 1343 | hdr->len, entry->len); |
| 1344 | qp->rx_err_oflow++; |
| 1345 | |
Allen Hubbe | da2e5ae | 2015-07-13 08:07:08 -0400 | [diff] [blame] | 1346 | entry->len = -EIO; |
| 1347 | entry->flags |= DESC_DONE_FLAG; |
| 1348 | |
| 1349 | ntb_complete_rxc(qp); |
| 1350 | } else { |
| 1351 | dev_dbg(&qp->ndev->pdev->dev, |
| 1352 | "RX OK index %u ver %u size %d into buf size %d\n", |
| 1353 | qp->rx_index, hdr->ver, hdr->len, entry->len); |
| 1354 | |
| 1355 | qp->rx_bytes += hdr->len; |
| 1356 | qp->rx_pkts++; |
| 1357 | |
| 1358 | entry->len = hdr->len; |
| 1359 | |
| 1360 | ntb_async_rx(entry, offset); |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 1361 | } |
| 1362 | |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 1363 | qp->rx_index++; |
| 1364 | qp->rx_index %= qp->rx_max_entry; |
| 1365 | |
| 1366 | return 0; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1367 | } |
| 1368 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 1369 | static void ntb_transport_rxc_db(unsigned long data) |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1370 | { |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 1371 | struct ntb_transport_qp *qp = (void *)data; |
Jon Mason | c336acd | 2013-01-17 15:28:45 -0700 | [diff] [blame] | 1372 | int rc, i; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1373 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 1374 | dev_dbg(&qp->ndev->pdev->dev, "%s: doorbell %d received\n", |
| 1375 | __func__, qp->qp_num); |
Jon Mason | e8aeb60 | 2013-04-18 17:59:44 -0700 | [diff] [blame] | 1376 | |
Jon Mason | c336acd | 2013-01-17 15:28:45 -0700 | [diff] [blame] | 1377 | /* Limit the number of packets processed in a single interrupt to |
| 1378 | * provide fairness to others |
| 1379 | */ |
| 1380 | for (i = 0; i < qp->rx_max_entry; i++) { |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1381 | rc = ntb_process_rxc(qp); |
Jon Mason | c336acd | 2013-01-17 15:28:45 -0700 | [diff] [blame] | 1382 | if (rc) |
| 1383 | break; |
| 1384 | } |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 1385 | |
Dave Jiang | 569410ca | 2015-07-13 08:07:22 -0400 | [diff] [blame] | 1386 | if (i && qp->rx_dma_chan) |
| 1387 | dma_async_issue_pending(qp->rx_dma_chan); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1388 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 1389 | if (i == qp->rx_max_entry) { |
| 1390 | /* there is more work to do */ |
| 1391 | tasklet_schedule(&qp->rxc_db_work); |
| 1392 | } else if (ntb_db_read(qp->ndev) & BIT_ULL(qp->qp_num)) { |
| 1393 | /* the doorbell bit is set: clear it */ |
| 1394 | ntb_db_clear(qp->ndev, BIT_ULL(qp->qp_num)); |
| 1395 | /* ntb_db_read ensures ntb_db_clear write is committed */ |
| 1396 | ntb_db_read(qp->ndev); |
| 1397 | |
| 1398 | /* an interrupt may have arrived between finishing |
| 1399 | * ntb_process_rxc and clearing the doorbell bit: |
| 1400 | * there might be some more work to do. |
| 1401 | */ |
| 1402 | tasklet_schedule(&qp->rxc_db_work); |
| 1403 | } |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1404 | } |
| 1405 | |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 1406 | static void ntb_tx_copy_callback(void *data) |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1407 | { |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 1408 | struct ntb_queue_entry *entry = data; |
| 1409 | struct ntb_transport_qp *qp = entry->qp; |
| 1410 | struct ntb_payload_header __iomem *hdr = entry->tx_hdr; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1411 | |
Jon Mason | 7446564 | 2013-01-21 15:28:52 -0700 | [diff] [blame] | 1412 | iowrite32(entry->flags | DESC_DONE_FLAG, &hdr->flags); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1413 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 1414 | ntb_peer_db_set(qp->ndev, BIT_ULL(qp->qp_num)); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1415 | |
| 1416 | /* The entry length can only be zero if the packet is intended to be a |
| 1417 | * "link down" or similar. Since no payload is being sent in these |
| 1418 | * cases, there is nothing to add to the completion queue. |
| 1419 | */ |
| 1420 | if (entry->len > 0) { |
| 1421 | qp->tx_bytes += entry->len; |
| 1422 | |
| 1423 | if (qp->tx_handler) |
| 1424 | qp->tx_handler(qp, qp->cb_data, entry->cb_data, |
| 1425 | entry->len); |
| 1426 | } |
| 1427 | |
| 1428 | ntb_list_add(&qp->ntb_tx_free_q_lock, &entry->entry, &qp->tx_free_q); |
| 1429 | } |
| 1430 | |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 1431 | static void ntb_memcpy_tx(struct ntb_queue_entry *entry, void __iomem *offset) |
| 1432 | { |
Dave Jiang | 06917f7 | 2015-05-19 16:45:46 -0400 | [diff] [blame] | 1433 | #ifdef ARCH_HAS_NOCACHE_UACCESS |
| 1434 | /* |
| 1435 | * Using non-temporal mov to improve performance on non-cached |
| 1436 | * writes, even though we aren't actually copying from user space. |
| 1437 | */ |
| 1438 | __copy_from_user_inatomic_nocache(offset, entry->buf, entry->len); |
| 1439 | #else |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 1440 | memcpy_toio(offset, entry->buf, entry->len); |
Dave Jiang | 06917f7 | 2015-05-19 16:45:46 -0400 | [diff] [blame] | 1441 | #endif |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 1442 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 1443 | /* Ensure that the data is fully copied out before setting the flags */ |
| 1444 | wmb(); |
| 1445 | |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 1446 | ntb_tx_copy_callback(entry); |
| 1447 | } |
| 1448 | |
| 1449 | static void ntb_async_tx(struct ntb_transport_qp *qp, |
| 1450 | struct ntb_queue_entry *entry) |
| 1451 | { |
| 1452 | struct ntb_payload_header __iomem *hdr; |
| 1453 | struct dma_async_tx_descriptor *txd; |
Dave Jiang | 569410ca | 2015-07-13 08:07:22 -0400 | [diff] [blame] | 1454 | struct dma_chan *chan = qp->tx_dma_chan; |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 1455 | struct dma_device *device; |
| 1456 | size_t dest_off, buff_off; |
Bartlomiej Zolnierkiewicz | 6f57fd0 | 2013-10-18 19:35:31 +0200 | [diff] [blame] | 1457 | struct dmaengine_unmap_data *unmap; |
| 1458 | dma_addr_t dest; |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 1459 | dma_cookie_t cookie; |
| 1460 | void __iomem *offset; |
| 1461 | size_t len = entry->len; |
| 1462 | void *buf = entry->buf; |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 1463 | |
| 1464 | offset = qp->tx_mw + qp->tx_max_frame * qp->tx_index; |
| 1465 | hdr = offset + qp->tx_max_frame - sizeof(struct ntb_payload_header); |
| 1466 | entry->tx_hdr = hdr; |
| 1467 | |
| 1468 | iowrite32(entry->len, &hdr->len); |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 1469 | iowrite32((u32)qp->tx_pkts, &hdr->ver); |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 1470 | |
| 1471 | if (!chan) |
| 1472 | goto err; |
| 1473 | |
| 1474 | if (len < copy_bytes) |
| 1475 | goto err; |
| 1476 | |
| 1477 | device = chan->device; |
| 1478 | dest = qp->tx_mw_phys + qp->tx_max_frame * qp->tx_index; |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 1479 | buff_off = (size_t)buf & ~PAGE_MASK; |
| 1480 | dest_off = (size_t)dest & ~PAGE_MASK; |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 1481 | |
| 1482 | if (!is_dma_copy_aligned(device, buff_off, dest_off, len)) |
| 1483 | goto err; |
| 1484 | |
Bartlomiej Zolnierkiewicz | 6f57fd0 | 2013-10-18 19:35:31 +0200 | [diff] [blame] | 1485 | unmap = dmaengine_get_unmap_data(device->dev, 1, GFP_NOWAIT); |
| 1486 | if (!unmap) |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 1487 | goto err; |
| 1488 | |
Bartlomiej Zolnierkiewicz | 6f57fd0 | 2013-10-18 19:35:31 +0200 | [diff] [blame] | 1489 | unmap->len = len; |
| 1490 | unmap->addr[0] = dma_map_page(device->dev, virt_to_page(buf), |
| 1491 | buff_off, len, DMA_TO_DEVICE); |
| 1492 | if (dma_mapping_error(device->dev, unmap->addr[0])) |
| 1493 | goto err_get_unmap; |
| 1494 | |
| 1495 | unmap->to_cnt = 1; |
| 1496 | |
Bartlomiej Zolnierkiewicz | 6f57fd0 | 2013-10-18 19:35:31 +0200 | [diff] [blame] | 1497 | txd = device->device_prep_dma_memcpy(chan, dest, unmap->addr[0], len, |
Bartlomiej Zolnierkiewicz | 0776ae7 | 2013-10-18 19:35:33 +0200 | [diff] [blame] | 1498 | DMA_PREP_INTERRUPT); |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 1499 | if (!txd) |
Bartlomiej Zolnierkiewicz | 6f57fd0 | 2013-10-18 19:35:31 +0200 | [diff] [blame] | 1500 | goto err_get_unmap; |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 1501 | |
| 1502 | txd->callback = ntb_tx_copy_callback; |
| 1503 | txd->callback_param = entry; |
Bartlomiej Zolnierkiewicz | 6f57fd0 | 2013-10-18 19:35:31 +0200 | [diff] [blame] | 1504 | dma_set_unmap(txd, unmap); |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 1505 | |
| 1506 | cookie = dmaengine_submit(txd); |
| 1507 | if (dma_submit_error(cookie)) |
Bartlomiej Zolnierkiewicz | 6f57fd0 | 2013-10-18 19:35:31 +0200 | [diff] [blame] | 1508 | goto err_set_unmap; |
| 1509 | |
| 1510 | dmaengine_unmap_put(unmap); |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 1511 | |
| 1512 | dma_async_issue_pending(chan); |
| 1513 | qp->tx_async++; |
| 1514 | |
| 1515 | return; |
Bartlomiej Zolnierkiewicz | 6f57fd0 | 2013-10-18 19:35:31 +0200 | [diff] [blame] | 1516 | err_set_unmap: |
| 1517 | dmaengine_unmap_put(unmap); |
| 1518 | err_get_unmap: |
| 1519 | dmaengine_unmap_put(unmap); |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 1520 | err: |
| 1521 | ntb_memcpy_tx(entry, offset); |
| 1522 | qp->tx_memcpy++; |
| 1523 | } |
| 1524 | |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1525 | static int ntb_process_tx(struct ntb_transport_qp *qp, |
| 1526 | struct ntb_queue_entry *entry) |
| 1527 | { |
Jon Mason | 793c20e | 2013-01-19 02:02:26 -0700 | [diff] [blame] | 1528 | if (qp->tx_index == qp->remote_rx_info->entry) { |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1529 | qp->tx_ring_full++; |
| 1530 | return -EAGAIN; |
| 1531 | } |
| 1532 | |
Jon Mason | ef114ed | 2013-01-19 02:02:18 -0700 | [diff] [blame] | 1533 | if (entry->len > qp->tx_max_frame - sizeof(struct ntb_payload_header)) { |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1534 | if (qp->tx_handler) |
| 1535 | qp->tx_handler(qp->cb_data, qp, NULL, -EIO); |
| 1536 | |
| 1537 | ntb_list_add(&qp->ntb_tx_free_q_lock, &entry->entry, |
| 1538 | &qp->tx_free_q); |
| 1539 | return 0; |
| 1540 | } |
| 1541 | |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 1542 | ntb_async_tx(qp, entry); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1543 | |
Jon Mason | 793c20e | 2013-01-19 02:02:26 -0700 | [diff] [blame] | 1544 | qp->tx_index++; |
| 1545 | qp->tx_index %= qp->tx_max_entry; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1546 | |
| 1547 | qp->tx_pkts++; |
| 1548 | |
| 1549 | return 0; |
| 1550 | } |
| 1551 | |
| 1552 | static void ntb_send_link_down(struct ntb_transport_qp *qp) |
| 1553 | { |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 1554 | struct pci_dev *pdev = qp->ndev->pdev; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1555 | struct ntb_queue_entry *entry; |
| 1556 | int i, rc; |
| 1557 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 1558 | if (!qp->link_is_up) |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1559 | return; |
| 1560 | |
Allen Hubbe | e22e0b9 | 2015-05-12 06:55:44 -0400 | [diff] [blame] | 1561 | dev_info(&pdev->dev, "qp %d: Send Link Down\n", qp->qp_num); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1562 | |
| 1563 | for (i = 0; i < NTB_LINK_DOWN_TIMEOUT; i++) { |
Jon Mason | f766755 | 2013-01-19 02:02:24 -0700 | [diff] [blame] | 1564 | entry = ntb_list_rm(&qp->ntb_tx_free_q_lock, &qp->tx_free_q); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1565 | if (entry) |
| 1566 | break; |
| 1567 | msleep(100); |
| 1568 | } |
| 1569 | |
| 1570 | if (!entry) |
| 1571 | return; |
| 1572 | |
| 1573 | entry->cb_data = NULL; |
| 1574 | entry->buf = NULL; |
| 1575 | entry->len = 0; |
| 1576 | entry->flags = LINK_DOWN_FLAG; |
| 1577 | |
| 1578 | rc = ntb_process_tx(qp, entry); |
| 1579 | if (rc) |
| 1580 | dev_err(&pdev->dev, "ntb: QP%d unable to send linkdown msg\n", |
| 1581 | qp->qp_num); |
Allen Hubbe | 2849b5d7 | 2015-05-12 08:09:15 -0400 | [diff] [blame] | 1582 | |
| 1583 | ntb_qp_link_down_reset(qp); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1584 | } |
| 1585 | |
Allen Hubbe | 1199aa6 | 2015-05-18 06:20:47 -0400 | [diff] [blame] | 1586 | static bool ntb_dma_filter_fn(struct dma_chan *chan, void *node) |
| 1587 | { |
| 1588 | return dev_to_node(&chan->dev->device) == (int)(unsigned long)node; |
| 1589 | } |
| 1590 | |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1591 | /** |
| 1592 | * ntb_transport_create_queue - Create a new NTB transport layer queue |
| 1593 | * @rx_handler: receive callback function |
| 1594 | * @tx_handler: transmit callback function |
| 1595 | * @event_handler: event callback function |
| 1596 | * |
| 1597 | * Create a new NTB transport layer queue and provide the queue with a callback |
| 1598 | * routine for both transmit and receive. The receive callback routine will be |
| 1599 | * used to pass up data when the transport has received it on the queue. The |
| 1600 | * transmit callback routine will be called when the transport has completed the |
| 1601 | * transmission of the data on the queue and the data is ready to be freed. |
| 1602 | * |
| 1603 | * RETURNS: pointer to newly created ntb_queue, NULL on error. |
| 1604 | */ |
| 1605 | struct ntb_transport_qp * |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 1606 | ntb_transport_create_queue(void *data, struct device *client_dev, |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1607 | const struct ntb_queue_handlers *handlers) |
| 1608 | { |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 1609 | struct ntb_dev *ndev; |
| 1610 | struct pci_dev *pdev; |
| 1611 | struct ntb_transport_ctx *nt; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1612 | struct ntb_queue_entry *entry; |
| 1613 | struct ntb_transport_qp *qp; |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 1614 | u64 qp_bit; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1615 | unsigned int free_queue; |
Allen Hubbe | 1199aa6 | 2015-05-18 06:20:47 -0400 | [diff] [blame] | 1616 | dma_cap_mask_t dma_mask; |
| 1617 | int node; |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 1618 | int i; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1619 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 1620 | ndev = dev_ntb(client_dev->parent); |
| 1621 | pdev = ndev->pdev; |
| 1622 | nt = ndev->ctx; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1623 | |
Allen Hubbe | 1199aa6 | 2015-05-18 06:20:47 -0400 | [diff] [blame] | 1624 | node = dev_to_node(&ndev->dev); |
| 1625 | |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1626 | free_queue = ffs(nt->qp_bitmap); |
| 1627 | if (!free_queue) |
| 1628 | goto err; |
| 1629 | |
| 1630 | /* decrement free_queue to make it zero based */ |
| 1631 | free_queue--; |
| 1632 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 1633 | qp = &nt->qp_vec[free_queue]; |
| 1634 | qp_bit = BIT_ULL(qp->qp_num); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1635 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 1636 | nt->qp_bitmap_free &= ~qp_bit; |
| 1637 | |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1638 | qp->cb_data = data; |
| 1639 | qp->rx_handler = handlers->rx_handler; |
| 1640 | qp->tx_handler = handlers->tx_handler; |
| 1641 | qp->event_handler = handlers->event_handler; |
| 1642 | |
Allen Hubbe | 1199aa6 | 2015-05-18 06:20:47 -0400 | [diff] [blame] | 1643 | dma_cap_zero(dma_mask); |
| 1644 | dma_cap_set(DMA_MEMCPY, dma_mask); |
| 1645 | |
Dave Jiang | a41ef05 | 2015-05-19 16:52:04 -0400 | [diff] [blame] | 1646 | if (use_dma) { |
Dave Jiang | 569410ca | 2015-07-13 08:07:22 -0400 | [diff] [blame] | 1647 | qp->tx_dma_chan = |
| 1648 | dma_request_channel(dma_mask, ntb_dma_filter_fn, |
| 1649 | (void *)(unsigned long)node); |
| 1650 | if (!qp->tx_dma_chan) |
| 1651 | dev_info(&pdev->dev, "Unable to allocate TX DMA channel\n"); |
| 1652 | |
| 1653 | qp->rx_dma_chan = |
| 1654 | dma_request_channel(dma_mask, ntb_dma_filter_fn, |
| 1655 | (void *)(unsigned long)node); |
| 1656 | if (!qp->rx_dma_chan) |
| 1657 | dev_info(&pdev->dev, "Unable to allocate RX DMA channel\n"); |
Dave Jiang | a41ef05 | 2015-05-19 16:52:04 -0400 | [diff] [blame] | 1658 | } else { |
Dave Jiang | 569410ca | 2015-07-13 08:07:22 -0400 | [diff] [blame] | 1659 | qp->tx_dma_chan = NULL; |
| 1660 | qp->rx_dma_chan = NULL; |
Dave Jiang | a41ef05 | 2015-05-19 16:52:04 -0400 | [diff] [blame] | 1661 | } |
Dave Jiang | 569410ca | 2015-07-13 08:07:22 -0400 | [diff] [blame] | 1662 | |
| 1663 | dev_dbg(&pdev->dev, "Using %s memcpy for TX\n", |
| 1664 | qp->tx_dma_chan ? "DMA" : "CPU"); |
| 1665 | |
| 1666 | dev_dbg(&pdev->dev, "Using %s memcpy for RX\n", |
| 1667 | qp->rx_dma_chan ? "DMA" : "CPU"); |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 1668 | |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1669 | for (i = 0; i < NTB_QP_DEF_NUM_ENTRIES; i++) { |
Allen Hubbe | 1199aa6 | 2015-05-18 06:20:47 -0400 | [diff] [blame] | 1670 | entry = kzalloc_node(sizeof(*entry), GFP_ATOMIC, node); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1671 | if (!entry) |
| 1672 | goto err1; |
| 1673 | |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 1674 | entry->qp = qp; |
Allen Hubbe | da2e5ae | 2015-07-13 08:07:08 -0400 | [diff] [blame] | 1675 | ntb_list_add(&qp->ntb_rx_q_lock, &entry->entry, |
Jon Mason | f766755 | 2013-01-19 02:02:24 -0700 | [diff] [blame] | 1676 | &qp->rx_free_q); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1677 | } |
| 1678 | |
| 1679 | for (i = 0; i < NTB_QP_DEF_NUM_ENTRIES; i++) { |
Allen Hubbe | 1199aa6 | 2015-05-18 06:20:47 -0400 | [diff] [blame] | 1680 | entry = kzalloc_node(sizeof(*entry), GFP_ATOMIC, node); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1681 | if (!entry) |
| 1682 | goto err2; |
| 1683 | |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 1684 | entry->qp = qp; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1685 | ntb_list_add(&qp->ntb_tx_free_q_lock, &entry->entry, |
Jon Mason | f766755 | 2013-01-19 02:02:24 -0700 | [diff] [blame] | 1686 | &qp->tx_free_q); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1687 | } |
| 1688 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 1689 | ntb_db_clear(qp->ndev, qp_bit); |
| 1690 | ntb_db_clear_mask(qp->ndev, qp_bit); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1691 | |
| 1692 | dev_info(&pdev->dev, "NTB Transport QP %d created\n", qp->qp_num); |
| 1693 | |
| 1694 | return qp; |
| 1695 | |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1696 | err2: |
Jon Mason | f766755 | 2013-01-19 02:02:24 -0700 | [diff] [blame] | 1697 | while ((entry = ntb_list_rm(&qp->ntb_tx_free_q_lock, &qp->tx_free_q))) |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1698 | kfree(entry); |
| 1699 | err1: |
Allen Hubbe | da2e5ae | 2015-07-13 08:07:08 -0400 | [diff] [blame] | 1700 | while ((entry = ntb_list_rm(&qp->ntb_rx_q_lock, &qp->rx_free_q))) |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1701 | kfree(entry); |
Dave Jiang | 569410ca | 2015-07-13 08:07:22 -0400 | [diff] [blame] | 1702 | if (qp->tx_dma_chan) |
| 1703 | dma_release_channel(qp->tx_dma_chan); |
| 1704 | if (qp->rx_dma_chan) |
| 1705 | dma_release_channel(qp->rx_dma_chan); |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 1706 | nt->qp_bitmap_free |= qp_bit; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1707 | err: |
| 1708 | return NULL; |
| 1709 | } |
| 1710 | EXPORT_SYMBOL_GPL(ntb_transport_create_queue); |
| 1711 | |
| 1712 | /** |
| 1713 | * ntb_transport_free_queue - Frees NTB transport queue |
| 1714 | * @qp: NTB queue to be freed |
| 1715 | * |
| 1716 | * Frees NTB transport queue |
| 1717 | */ |
| 1718 | void ntb_transport_free_queue(struct ntb_transport_qp *qp) |
| 1719 | { |
Jon Mason | 186f27f | 2013-01-22 11:35:40 -0700 | [diff] [blame] | 1720 | struct pci_dev *pdev; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1721 | struct ntb_queue_entry *entry; |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 1722 | u64 qp_bit; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1723 | |
| 1724 | if (!qp) |
| 1725 | return; |
| 1726 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 1727 | pdev = qp->ndev->pdev; |
Jon Mason | 186f27f | 2013-01-22 11:35:40 -0700 | [diff] [blame] | 1728 | |
Dave Jiang | 569410ca | 2015-07-13 08:07:22 -0400 | [diff] [blame] | 1729 | if (qp->tx_dma_chan) { |
| 1730 | struct dma_chan *chan = qp->tx_dma_chan; |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 1731 | /* Putting the dma_chan to NULL will force any new traffic to be |
| 1732 | * processed by the CPU instead of the DAM engine |
| 1733 | */ |
Dave Jiang | 569410ca | 2015-07-13 08:07:22 -0400 | [diff] [blame] | 1734 | qp->tx_dma_chan = NULL; |
| 1735 | |
| 1736 | /* Try to be nice and wait for any queued DMA engine |
| 1737 | * transactions to process before smashing it with a rock |
| 1738 | */ |
| 1739 | dma_sync_wait(chan, qp->last_cookie); |
| 1740 | dmaengine_terminate_all(chan); |
| 1741 | dma_release_channel(chan); |
| 1742 | } |
| 1743 | |
| 1744 | if (qp->rx_dma_chan) { |
| 1745 | struct dma_chan *chan = qp->rx_dma_chan; |
| 1746 | /* Putting the dma_chan to NULL will force any new traffic to be |
| 1747 | * processed by the CPU instead of the DAM engine |
| 1748 | */ |
| 1749 | qp->rx_dma_chan = NULL; |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 1750 | |
| 1751 | /* Try to be nice and wait for any queued DMA engine |
| 1752 | * transactions to process before smashing it with a rock |
| 1753 | */ |
| 1754 | dma_sync_wait(chan, qp->last_cookie); |
| 1755 | dmaengine_terminate_all(chan); |
Allen Hubbe | 1199aa6 | 2015-05-18 06:20:47 -0400 | [diff] [blame] | 1756 | dma_release_channel(chan); |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 1757 | } |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1758 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 1759 | qp_bit = BIT_ULL(qp->qp_num); |
| 1760 | |
| 1761 | ntb_db_set_mask(qp->ndev, qp_bit); |
| 1762 | tasklet_disable(&qp->rxc_db_work); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1763 | |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 1764 | cancel_delayed_work_sync(&qp->link_work); |
| 1765 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 1766 | qp->cb_data = NULL; |
| 1767 | qp->rx_handler = NULL; |
| 1768 | qp->tx_handler = NULL; |
| 1769 | qp->event_handler = NULL; |
| 1770 | |
Allen Hubbe | da2e5ae | 2015-07-13 08:07:08 -0400 | [diff] [blame] | 1771 | while ((entry = ntb_list_rm(&qp->ntb_rx_q_lock, &qp->rx_free_q))) |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1772 | kfree(entry); |
| 1773 | |
Allen Hubbe | da2e5ae | 2015-07-13 08:07:08 -0400 | [diff] [blame] | 1774 | while ((entry = ntb_list_rm(&qp->ntb_rx_q_lock, &qp->rx_pend_q))) { |
| 1775 | dev_warn(&pdev->dev, "Freeing item from non-empty rx_pend_q\n"); |
| 1776 | kfree(entry); |
| 1777 | } |
| 1778 | |
| 1779 | while ((entry = ntb_list_rm(&qp->ntb_rx_q_lock, &qp->rx_post_q))) { |
| 1780 | dev_warn(&pdev->dev, "Freeing item from non-empty rx_post_q\n"); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1781 | kfree(entry); |
| 1782 | } |
| 1783 | |
Jon Mason | f766755 | 2013-01-19 02:02:24 -0700 | [diff] [blame] | 1784 | while ((entry = ntb_list_rm(&qp->ntb_tx_free_q_lock, &qp->tx_free_q))) |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1785 | kfree(entry); |
| 1786 | |
Allen Hubbe | 30a4bb1 | 2015-07-13 08:07:14 -0400 | [diff] [blame] | 1787 | qp->transport->qp_bitmap_free |= qp_bit; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1788 | |
| 1789 | dev_info(&pdev->dev, "NTB Transport QP %d freed\n", qp->qp_num); |
| 1790 | } |
| 1791 | EXPORT_SYMBOL_GPL(ntb_transport_free_queue); |
| 1792 | |
| 1793 | /** |
| 1794 | * ntb_transport_rx_remove - Dequeues enqueued rx packet |
| 1795 | * @qp: NTB queue to be freed |
| 1796 | * @len: pointer to variable to write enqueued buffers length |
| 1797 | * |
| 1798 | * Dequeues unused buffers from receive queue. Should only be used during |
| 1799 | * shutdown of qp. |
| 1800 | * |
| 1801 | * RETURNS: NULL error value on error, or void* for success. |
| 1802 | */ |
| 1803 | void *ntb_transport_rx_remove(struct ntb_transport_qp *qp, unsigned int *len) |
| 1804 | { |
| 1805 | struct ntb_queue_entry *entry; |
| 1806 | void *buf; |
| 1807 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 1808 | if (!qp || qp->client_ready) |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1809 | return NULL; |
| 1810 | |
Allen Hubbe | da2e5ae | 2015-07-13 08:07:08 -0400 | [diff] [blame] | 1811 | entry = ntb_list_rm(&qp->ntb_rx_q_lock, &qp->rx_pend_q); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1812 | if (!entry) |
| 1813 | return NULL; |
| 1814 | |
| 1815 | buf = entry->cb_data; |
| 1816 | *len = entry->len; |
| 1817 | |
Allen Hubbe | da2e5ae | 2015-07-13 08:07:08 -0400 | [diff] [blame] | 1818 | ntb_list_add(&qp->ntb_rx_q_lock, &entry->entry, &qp->rx_free_q); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1819 | |
| 1820 | return buf; |
| 1821 | } |
| 1822 | EXPORT_SYMBOL_GPL(ntb_transport_rx_remove); |
| 1823 | |
| 1824 | /** |
| 1825 | * ntb_transport_rx_enqueue - Enqueue a new NTB queue entry |
| 1826 | * @qp: NTB transport layer queue the entry is to be enqueued on |
| 1827 | * @cb: per buffer pointer for callback function to use |
| 1828 | * @data: pointer to data buffer that incoming packets will be copied into |
| 1829 | * @len: length of the data buffer |
| 1830 | * |
| 1831 | * Enqueue a new receive buffer onto the transport queue into which a NTB |
| 1832 | * payload can be received into. |
| 1833 | * |
| 1834 | * RETURNS: An appropriate -ERRNO error value on error, or zero for success. |
| 1835 | */ |
| 1836 | int ntb_transport_rx_enqueue(struct ntb_transport_qp *qp, void *cb, void *data, |
| 1837 | unsigned int len) |
| 1838 | { |
| 1839 | struct ntb_queue_entry *entry; |
| 1840 | |
| 1841 | if (!qp) |
| 1842 | return -EINVAL; |
| 1843 | |
Allen Hubbe | da2e5ae | 2015-07-13 08:07:08 -0400 | [diff] [blame] | 1844 | entry = ntb_list_rm(&qp->ntb_rx_q_lock, &qp->rx_free_q); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1845 | if (!entry) |
| 1846 | return -ENOMEM; |
| 1847 | |
| 1848 | entry->cb_data = cb; |
| 1849 | entry->buf = data; |
| 1850 | entry->len = len; |
Allen Hubbe | da2e5ae | 2015-07-13 08:07:08 -0400 | [diff] [blame] | 1851 | entry->flags = 0; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1852 | |
Allen Hubbe | da2e5ae | 2015-07-13 08:07:08 -0400 | [diff] [blame] | 1853 | ntb_list_add(&qp->ntb_rx_q_lock, &entry->entry, &qp->rx_pend_q); |
| 1854 | |
| 1855 | tasklet_schedule(&qp->rxc_db_work); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1856 | |
| 1857 | return 0; |
| 1858 | } |
| 1859 | EXPORT_SYMBOL_GPL(ntb_transport_rx_enqueue); |
| 1860 | |
| 1861 | /** |
| 1862 | * ntb_transport_tx_enqueue - Enqueue a new NTB queue entry |
| 1863 | * @qp: NTB transport layer queue the entry is to be enqueued on |
| 1864 | * @cb: per buffer pointer for callback function to use |
| 1865 | * @data: pointer to data buffer that will be sent |
| 1866 | * @len: length of the data buffer |
| 1867 | * |
| 1868 | * Enqueue a new transmit buffer onto the transport queue from which a NTB |
Jon Mason | f9a2cf8 | 2013-07-29 16:46:43 -0700 | [diff] [blame] | 1869 | * payload will be transmitted. This assumes that a lock is being held to |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1870 | * serialize access to the qp. |
| 1871 | * |
| 1872 | * RETURNS: An appropriate -ERRNO error value on error, or zero for success. |
| 1873 | */ |
| 1874 | int ntb_transport_tx_enqueue(struct ntb_transport_qp *qp, void *cb, void *data, |
| 1875 | unsigned int len) |
| 1876 | { |
| 1877 | struct ntb_queue_entry *entry; |
| 1878 | int rc; |
| 1879 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 1880 | if (!qp || !qp->link_is_up || !len) |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1881 | return -EINVAL; |
| 1882 | |
| 1883 | entry = ntb_list_rm(&qp->ntb_tx_free_q_lock, &qp->tx_free_q); |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 1884 | if (!entry) { |
| 1885 | qp->tx_err_no_buf++; |
Dave Jiang | e74bfee | 2015-07-13 08:07:17 -0400 | [diff] [blame] | 1886 | return -EBUSY; |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 1887 | } |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1888 | |
| 1889 | entry->cb_data = cb; |
| 1890 | entry->buf = data; |
| 1891 | entry->len = len; |
| 1892 | entry->flags = 0; |
| 1893 | |
| 1894 | rc = ntb_process_tx(qp, entry); |
| 1895 | if (rc) |
| 1896 | ntb_list_add(&qp->ntb_tx_free_q_lock, &entry->entry, |
| 1897 | &qp->tx_free_q); |
| 1898 | |
| 1899 | return rc; |
| 1900 | } |
| 1901 | EXPORT_SYMBOL_GPL(ntb_transport_tx_enqueue); |
| 1902 | |
| 1903 | /** |
| 1904 | * ntb_transport_link_up - Notify NTB transport of client readiness to use queue |
| 1905 | * @qp: NTB transport layer queue to be enabled |
| 1906 | * |
| 1907 | * Notify NTB transport layer of client readiness to use queue |
| 1908 | */ |
| 1909 | void ntb_transport_link_up(struct ntb_transport_qp *qp) |
| 1910 | { |
| 1911 | if (!qp) |
| 1912 | return; |
| 1913 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 1914 | qp->client_ready = true; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1915 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 1916 | if (qp->transport->link_is_up) |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1917 | schedule_delayed_work(&qp->link_work, 0); |
| 1918 | } |
| 1919 | EXPORT_SYMBOL_GPL(ntb_transport_link_up); |
| 1920 | |
| 1921 | /** |
| 1922 | * ntb_transport_link_down - Notify NTB transport to no longer enqueue data |
| 1923 | * @qp: NTB transport layer queue to be disabled |
| 1924 | * |
| 1925 | * Notify NTB transport layer of client's desire to no longer receive data on |
| 1926 | * transport queue specified. It is the client's responsibility to ensure all |
Jon Mason | f9a2cf8 | 2013-07-29 16:46:43 -0700 | [diff] [blame] | 1927 | * entries on queue are purged or otherwise handled appropriately. |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1928 | */ |
| 1929 | void ntb_transport_link_down(struct ntb_transport_qp *qp) |
| 1930 | { |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 1931 | int val; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1932 | |
| 1933 | if (!qp) |
| 1934 | return; |
| 1935 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 1936 | qp->client_ready = false; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1937 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 1938 | val = ntb_spad_read(qp->ndev, QP_LINKS); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1939 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 1940 | ntb_peer_spad_write(qp->ndev, QP_LINKS, |
| 1941 | val & ~BIT(qp->qp_num)); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1942 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 1943 | if (qp->link_is_up) |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1944 | ntb_send_link_down(qp); |
| 1945 | else |
| 1946 | cancel_delayed_work_sync(&qp->link_work); |
| 1947 | } |
| 1948 | EXPORT_SYMBOL_GPL(ntb_transport_link_down); |
| 1949 | |
| 1950 | /** |
| 1951 | * ntb_transport_link_query - Query transport link state |
| 1952 | * @qp: NTB transport layer queue to be queried |
| 1953 | * |
| 1954 | * Query connectivity to the remote system of the NTB transport queue |
| 1955 | * |
| 1956 | * RETURNS: true for link up or false for link down |
| 1957 | */ |
| 1958 | bool ntb_transport_link_query(struct ntb_transport_qp *qp) |
| 1959 | { |
Jon Mason | 186f27f | 2013-01-22 11:35:40 -0700 | [diff] [blame] | 1960 | if (!qp) |
| 1961 | return false; |
| 1962 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 1963 | return qp->link_is_up; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1964 | } |
| 1965 | EXPORT_SYMBOL_GPL(ntb_transport_link_query); |
| 1966 | |
| 1967 | /** |
| 1968 | * ntb_transport_qp_num - Query the qp number |
| 1969 | * @qp: NTB transport layer queue to be queried |
| 1970 | * |
| 1971 | * Query qp number of the NTB transport queue |
| 1972 | * |
| 1973 | * RETURNS: a zero based number specifying the qp number |
| 1974 | */ |
| 1975 | unsigned char ntb_transport_qp_num(struct ntb_transport_qp *qp) |
| 1976 | { |
Jon Mason | 186f27f | 2013-01-22 11:35:40 -0700 | [diff] [blame] | 1977 | if (!qp) |
| 1978 | return 0; |
| 1979 | |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1980 | return qp->qp_num; |
| 1981 | } |
| 1982 | EXPORT_SYMBOL_GPL(ntb_transport_qp_num); |
| 1983 | |
| 1984 | /** |
| 1985 | * ntb_transport_max_size - Query the max payload size of a qp |
| 1986 | * @qp: NTB transport layer queue to be queried |
| 1987 | * |
| 1988 | * Query the maximum payload size permissible on the given qp |
| 1989 | * |
| 1990 | * RETURNS: the max payload size of a qp |
| 1991 | */ |
Jon Mason | ef114ed | 2013-01-19 02:02:18 -0700 | [diff] [blame] | 1992 | unsigned int ntb_transport_max_size(struct ntb_transport_qp *qp) |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1993 | { |
Dave Jiang | 04afde4 | 2015-09-17 13:27:04 -0700 | [diff] [blame] | 1994 | unsigned int max_size; |
Dave Jiang | 569410ca | 2015-07-13 08:07:22 -0400 | [diff] [blame] | 1995 | unsigned int copy_align; |
Dave Jiang | 04afde4 | 2015-09-17 13:27:04 -0700 | [diff] [blame] | 1996 | struct dma_chan *rx_chan, *tx_chan; |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 1997 | |
Jon Mason | 186f27f | 2013-01-22 11:35:40 -0700 | [diff] [blame] | 1998 | if (!qp) |
| 1999 | return 0; |
| 2000 | |
Dave Jiang | 04afde4 | 2015-09-17 13:27:04 -0700 | [diff] [blame] | 2001 | rx_chan = qp->rx_dma_chan; |
| 2002 | tx_chan = qp->tx_dma_chan; |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 2003 | |
Dave Jiang | 04afde4 | 2015-09-17 13:27:04 -0700 | [diff] [blame] | 2004 | copy_align = max(rx_chan ? rx_chan->device->copy_align : 0, |
| 2005 | tx_chan ? tx_chan->device->copy_align : 0); |
Dave Jiang | 569410ca | 2015-07-13 08:07:22 -0400 | [diff] [blame] | 2006 | |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 2007 | /* If DMA engine usage is possible, try to find the max size for that */ |
Dave Jiang | 04afde4 | 2015-09-17 13:27:04 -0700 | [diff] [blame] | 2008 | max_size = qp->tx_max_frame - sizeof(struct ntb_payload_header); |
| 2009 | max_size = round_down(max_size, 1 << copy_align); |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 2010 | |
Dave Jiang | 04afde4 | 2015-09-17 13:27:04 -0700 | [diff] [blame] | 2011 | return max_size; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 2012 | } |
| 2013 | EXPORT_SYMBOL_GPL(ntb_transport_max_size); |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 2014 | |
Dave Jiang | e74bfee | 2015-07-13 08:07:17 -0400 | [diff] [blame] | 2015 | unsigned int ntb_transport_tx_free_entry(struct ntb_transport_qp *qp) |
| 2016 | { |
| 2017 | unsigned int head = qp->tx_index; |
| 2018 | unsigned int tail = qp->remote_rx_info->entry; |
| 2019 | |
| 2020 | return tail > head ? tail - head : qp->tx_max_entry + tail - head; |
| 2021 | } |
| 2022 | EXPORT_SYMBOL_GPL(ntb_transport_tx_free_entry); |
| 2023 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 2024 | static void ntb_transport_doorbell_callback(void *data, int vector) |
| 2025 | { |
| 2026 | struct ntb_transport_ctx *nt = data; |
| 2027 | struct ntb_transport_qp *qp; |
| 2028 | u64 db_bits; |
| 2029 | unsigned int qp_num; |
| 2030 | |
| 2031 | db_bits = (nt->qp_bitmap & ~nt->qp_bitmap_free & |
| 2032 | ntb_db_vector_mask(nt->ndev, vector)); |
| 2033 | |
| 2034 | while (db_bits) { |
| 2035 | qp_num = __ffs(db_bits); |
| 2036 | qp = &nt->qp_vec[qp_num]; |
| 2037 | |
| 2038 | tasklet_schedule(&qp->rxc_db_work); |
| 2039 | |
| 2040 | db_bits &= ~BIT_ULL(qp_num); |
| 2041 | } |
| 2042 | } |
| 2043 | |
| 2044 | static const struct ntb_ctx_ops ntb_transport_ops = { |
| 2045 | .link_event = ntb_transport_event_callback, |
| 2046 | .db_event = ntb_transport_doorbell_callback, |
| 2047 | }; |
| 2048 | |
| 2049 | static struct ntb_client ntb_transport_client = { |
| 2050 | .ops = { |
| 2051 | .probe = ntb_transport_probe, |
| 2052 | .remove = ntb_transport_free, |
| 2053 | }, |
| 2054 | }; |
| 2055 | |
| 2056 | static int __init ntb_transport_init(void) |
| 2057 | { |
| 2058 | int rc; |
| 2059 | |
Dave Jiang | 7eb3878 | 2015-06-15 08:21:33 -0400 | [diff] [blame] | 2060 | pr_info("%s, version %s\n", NTB_TRANSPORT_DESC, NTB_TRANSPORT_VER); |
| 2061 | |
Allen Hubbe | e26a584 | 2015-04-09 10:33:20 -0400 | [diff] [blame] | 2062 | if (debugfs_initialized()) |
| 2063 | nt_debugfs_dir = debugfs_create_dir(KBUILD_MODNAME, NULL); |
| 2064 | |
| 2065 | rc = bus_register(&ntb_transport_bus); |
| 2066 | if (rc) |
| 2067 | goto err_bus; |
| 2068 | |
| 2069 | rc = ntb_register_client(&ntb_transport_client); |
| 2070 | if (rc) |
| 2071 | goto err_client; |
| 2072 | |
| 2073 | return 0; |
| 2074 | |
| 2075 | err_client: |
| 2076 | bus_unregister(&ntb_transport_bus); |
| 2077 | err_bus: |
| 2078 | debugfs_remove_recursive(nt_debugfs_dir); |
| 2079 | return rc; |
| 2080 | } |
| 2081 | module_init(ntb_transport_init); |
| 2082 | |
| 2083 | static void __exit ntb_transport_exit(void) |
| 2084 | { |
| 2085 | debugfs_remove_recursive(nt_debugfs_dir); |
| 2086 | |
| 2087 | ntb_unregister_client(&ntb_transport_client); |
| 2088 | bus_unregister(&ntb_transport_bus); |
| 2089 | } |
| 2090 | module_exit(ntb_transport_exit); |