Roland Dreier | 225c7b1 | 2007-05-08 18:00:38 -0700 | [diff] [blame] | 1 | /* |
| 2 | * Copyright (c) 2006, 2007 Cisco Systems, Inc. All rights reserved. |
| 3 | * Copyright (c) 2005 Mellanox Technologies. All rights reserved. |
| 4 | * |
| 5 | * This software is available to you under a choice of one of two |
| 6 | * licenses. You may choose to be licensed under the terms of the GNU |
| 7 | * General Public License (GPL) Version 2, available from the file |
| 8 | * COPYING in the main directory of this source tree, or the |
| 9 | * OpenIB.org BSD license below: |
| 10 | * |
| 11 | * Redistribution and use in source and binary forms, with or |
| 12 | * without modification, are permitted provided that the following |
| 13 | * conditions are met: |
| 14 | * |
| 15 | * - Redistributions of source code must retain the above |
| 16 | * copyright notice, this list of conditions and the following |
| 17 | * disclaimer. |
| 18 | * |
| 19 | * - Redistributions in binary form must reproduce the above |
| 20 | * copyright notice, this list of conditions and the following |
| 21 | * disclaimer in the documentation and/or other materials |
| 22 | * provided with the distribution. |
| 23 | * |
| 24 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, |
| 25 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF |
| 26 | * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND |
| 27 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS |
| 28 | * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN |
| 29 | * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN |
| 30 | * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE |
| 31 | * SOFTWARE. |
| 32 | */ |
| 33 | |
Roland Dreier | 225c7b1 | 2007-05-08 18:00:38 -0700 | [diff] [blame] | 34 | #include <linux/errno.h> |
Paul Gortmaker | ee40fa0 | 2011-05-27 16:14:23 -0400 | [diff] [blame] | 35 | #include <linux/export.h> |
Eli Cohen | c1b43dc | 2011-03-22 22:38:41 +0000 | [diff] [blame] | 36 | #include <linux/io-mapping.h> |
Roland Dreier | 225c7b1 | 2007-05-08 18:00:38 -0700 | [diff] [blame] | 37 | |
| 38 | #include <asm/page.h> |
| 39 | |
| 40 | #include "mlx4.h" |
| 41 | #include "icm.h" |
| 42 | |
Eli Cohen | 9ace5e0 | 2011-03-22 22:38:48 +0000 | [diff] [blame] | 43 | enum { |
| 44 | MLX4_NUM_RESERVED_UARS = 8 |
| 45 | }; |
| 46 | |
Roland Dreier | 225c7b1 | 2007-05-08 18:00:38 -0700 | [diff] [blame] | 47 | int mlx4_pd_alloc(struct mlx4_dev *dev, u32 *pdn) |
| 48 | { |
| 49 | struct mlx4_priv *priv = mlx4_priv(dev); |
| 50 | |
| 51 | *pdn = mlx4_bitmap_alloc(&priv->pd_bitmap); |
| 52 | if (*pdn == -1) |
| 53 | return -ENOMEM; |
Marcel Apfelbaum | eb41049 | 2012-01-19 09:45:19 +0000 | [diff] [blame] | 54 | |
Roland Dreier | 225c7b1 | 2007-05-08 18:00:38 -0700 | [diff] [blame] | 55 | return 0; |
| 56 | } |
| 57 | EXPORT_SYMBOL_GPL(mlx4_pd_alloc); |
| 58 | |
| 59 | void mlx4_pd_free(struct mlx4_dev *dev, u32 pdn) |
| 60 | { |
Jack Morgenstein | 7c6d74d | 2013-12-08 16:50:17 +0200 | [diff] [blame] | 61 | mlx4_bitmap_free(&mlx4_priv(dev)->pd_bitmap, pdn, MLX4_USE_RR); |
Roland Dreier | 225c7b1 | 2007-05-08 18:00:38 -0700 | [diff] [blame] | 62 | } |
| 63 | EXPORT_SYMBOL_GPL(mlx4_pd_free); |
| 64 | |
Jack Morgenstein | ba062d5 | 2012-05-15 10:35:03 +0000 | [diff] [blame] | 65 | int __mlx4_xrcd_alloc(struct mlx4_dev *dev, u32 *xrcdn) |
Sean Hefty | 012a8ff | 2011-06-02 09:01:33 -0700 | [diff] [blame] | 66 | { |
| 67 | struct mlx4_priv *priv = mlx4_priv(dev); |
| 68 | |
| 69 | *xrcdn = mlx4_bitmap_alloc(&priv->xrcd_bitmap); |
| 70 | if (*xrcdn == -1) |
| 71 | return -ENOMEM; |
| 72 | |
| 73 | return 0; |
| 74 | } |
Jack Morgenstein | ba062d5 | 2012-05-15 10:35:03 +0000 | [diff] [blame] | 75 | |
| 76 | int mlx4_xrcd_alloc(struct mlx4_dev *dev, u32 *xrcdn) |
| 77 | { |
| 78 | u64 out_param; |
| 79 | int err; |
| 80 | |
| 81 | if (mlx4_is_mfunc(dev)) { |
| 82 | err = mlx4_cmd_imm(dev, 0, &out_param, |
| 83 | RES_XRCD, RES_OP_RESERVE, |
| 84 | MLX4_CMD_ALLOC_RES, |
| 85 | MLX4_CMD_TIME_CLASS_A, MLX4_CMD_WRAPPED); |
| 86 | if (err) |
| 87 | return err; |
| 88 | |
| 89 | *xrcdn = get_param_l(&out_param); |
| 90 | return 0; |
| 91 | } |
| 92 | return __mlx4_xrcd_alloc(dev, xrcdn); |
| 93 | } |
Sean Hefty | 012a8ff | 2011-06-02 09:01:33 -0700 | [diff] [blame] | 94 | EXPORT_SYMBOL_GPL(mlx4_xrcd_alloc); |
| 95 | |
Jack Morgenstein | ba062d5 | 2012-05-15 10:35:03 +0000 | [diff] [blame] | 96 | void __mlx4_xrcd_free(struct mlx4_dev *dev, u32 xrcdn) |
| 97 | { |
Jack Morgenstein | 7c6d74d | 2013-12-08 16:50:17 +0200 | [diff] [blame] | 98 | mlx4_bitmap_free(&mlx4_priv(dev)->xrcd_bitmap, xrcdn, MLX4_USE_RR); |
Jack Morgenstein | ba062d5 | 2012-05-15 10:35:03 +0000 | [diff] [blame] | 99 | } |
| 100 | |
Sean Hefty | 012a8ff | 2011-06-02 09:01:33 -0700 | [diff] [blame] | 101 | void mlx4_xrcd_free(struct mlx4_dev *dev, u32 xrcdn) |
| 102 | { |
Jack Morgenstein | e7dbeba | 2013-03-07 03:46:54 +0000 | [diff] [blame] | 103 | u64 in_param = 0; |
Jack Morgenstein | ba062d5 | 2012-05-15 10:35:03 +0000 | [diff] [blame] | 104 | int err; |
| 105 | |
| 106 | if (mlx4_is_mfunc(dev)) { |
| 107 | set_param_l(&in_param, xrcdn); |
| 108 | err = mlx4_cmd(dev, in_param, RES_XRCD, |
| 109 | RES_OP_RESERVE, MLX4_CMD_FREE_RES, |
| 110 | MLX4_CMD_TIME_CLASS_A, MLX4_CMD_WRAPPED); |
| 111 | if (err) |
| 112 | mlx4_warn(dev, "Failed to release xrcdn %d\n", xrcdn); |
| 113 | } else |
| 114 | __mlx4_xrcd_free(dev, xrcdn); |
Sean Hefty | 012a8ff | 2011-06-02 09:01:33 -0700 | [diff] [blame] | 115 | } |
| 116 | EXPORT_SYMBOL_GPL(mlx4_xrcd_free); |
| 117 | |
Roland Dreier | 3d73c28 | 2007-10-10 15:43:54 -0700 | [diff] [blame] | 118 | int mlx4_init_pd_table(struct mlx4_dev *dev) |
Roland Dreier | 225c7b1 | 2007-05-08 18:00:38 -0700 | [diff] [blame] | 119 | { |
| 120 | struct mlx4_priv *priv = mlx4_priv(dev); |
| 121 | |
| 122 | return mlx4_bitmap_init(&priv->pd_bitmap, dev->caps.num_pds, |
Jack Morgenstein | f5311ac | 2011-12-13 04:12:13 +0000 | [diff] [blame] | 123 | (1 << NOT_MASKED_PD_BITS) - 1, |
| 124 | dev->caps.reserved_pds, 0); |
Roland Dreier | 225c7b1 | 2007-05-08 18:00:38 -0700 | [diff] [blame] | 125 | } |
| 126 | |
| 127 | void mlx4_cleanup_pd_table(struct mlx4_dev *dev) |
| 128 | { |
| 129 | mlx4_bitmap_cleanup(&mlx4_priv(dev)->pd_bitmap); |
| 130 | } |
| 131 | |
Sean Hefty | 012a8ff | 2011-06-02 09:01:33 -0700 | [diff] [blame] | 132 | int mlx4_init_xrcd_table(struct mlx4_dev *dev) |
| 133 | { |
| 134 | struct mlx4_priv *priv = mlx4_priv(dev); |
| 135 | |
| 136 | return mlx4_bitmap_init(&priv->xrcd_bitmap, (1 << 16), |
| 137 | (1 << 16) - 1, dev->caps.reserved_xrcds + 1, 0); |
| 138 | } |
| 139 | |
| 140 | void mlx4_cleanup_xrcd_table(struct mlx4_dev *dev) |
| 141 | { |
| 142 | mlx4_bitmap_cleanup(&mlx4_priv(dev)->xrcd_bitmap); |
| 143 | } |
Roland Dreier | 225c7b1 | 2007-05-08 18:00:38 -0700 | [diff] [blame] | 144 | |
| 145 | int mlx4_uar_alloc(struct mlx4_dev *dev, struct mlx4_uar *uar) |
| 146 | { |
Jack Morgenstein | f5311ac | 2011-12-13 04:12:13 +0000 | [diff] [blame] | 147 | int offset; |
| 148 | |
Roland Dreier | 225c7b1 | 2007-05-08 18:00:38 -0700 | [diff] [blame] | 149 | uar->index = mlx4_bitmap_alloc(&mlx4_priv(dev)->uar_table.bitmap); |
| 150 | if (uar->index == -1) |
| 151 | return -ENOMEM; |
| 152 | |
Jack Morgenstein | f5311ac | 2011-12-13 04:12:13 +0000 | [diff] [blame] | 153 | if (mlx4_is_slave(dev)) |
Yishai Hadas | 872bf2f | 2015-01-25 16:59:35 +0200 | [diff] [blame] | 154 | offset = uar->index % ((int)pci_resource_len(dev->persist->pdev, |
| 155 | 2) / |
Jack Morgenstein | f5311ac | 2011-12-13 04:12:13 +0000 | [diff] [blame] | 156 | dev->caps.uar_page_size); |
| 157 | else |
| 158 | offset = uar->index; |
Yishai Hadas | 872bf2f | 2015-01-25 16:59:35 +0200 | [diff] [blame] | 159 | uar->pfn = (pci_resource_start(dev->persist->pdev, 2) >> PAGE_SHIFT) |
| 160 | + offset; |
Eli Cohen | c1b43dc | 2011-03-22 22:38:41 +0000 | [diff] [blame] | 161 | uar->map = NULL; |
Roland Dreier | 225c7b1 | 2007-05-08 18:00:38 -0700 | [diff] [blame] | 162 | return 0; |
| 163 | } |
| 164 | EXPORT_SYMBOL_GPL(mlx4_uar_alloc); |
| 165 | |
| 166 | void mlx4_uar_free(struct mlx4_dev *dev, struct mlx4_uar *uar) |
| 167 | { |
Jack Morgenstein | 7c6d74d | 2013-12-08 16:50:17 +0200 | [diff] [blame] | 168 | mlx4_bitmap_free(&mlx4_priv(dev)->uar_table.bitmap, uar->index, MLX4_USE_RR); |
Roland Dreier | 225c7b1 | 2007-05-08 18:00:38 -0700 | [diff] [blame] | 169 | } |
| 170 | EXPORT_SYMBOL_GPL(mlx4_uar_free); |
| 171 | |
Eugenia Emantayev | 163561a | 2013-11-07 12:19:54 +0200 | [diff] [blame] | 172 | int mlx4_bf_alloc(struct mlx4_dev *dev, struct mlx4_bf *bf, int node) |
Eli Cohen | c1b43dc | 2011-03-22 22:38:41 +0000 | [diff] [blame] | 173 | { |
| 174 | struct mlx4_priv *priv = mlx4_priv(dev); |
| 175 | struct mlx4_uar *uar; |
| 176 | int err = 0; |
| 177 | int idx; |
| 178 | |
| 179 | if (!priv->bf_mapping) |
| 180 | return -ENOMEM; |
| 181 | |
| 182 | mutex_lock(&priv->bf_mutex); |
| 183 | if (!list_empty(&priv->bf_list)) |
| 184 | uar = list_entry(priv->bf_list.next, struct mlx4_uar, bf_list); |
| 185 | else { |
Eli Cohen | 9ace5e0 | 2011-03-22 22:38:48 +0000 | [diff] [blame] | 186 | if (mlx4_bitmap_avail(&priv->uar_table.bitmap) < MLX4_NUM_RESERVED_UARS) { |
| 187 | err = -ENOMEM; |
| 188 | goto out; |
| 189 | } |
Eugenia Emantayev | 163561a | 2013-11-07 12:19:54 +0200 | [diff] [blame] | 190 | uar = kmalloc_node(sizeof(*uar), GFP_KERNEL, node); |
Eli Cohen | c1b43dc | 2011-03-22 22:38:41 +0000 | [diff] [blame] | 191 | if (!uar) { |
Eugenia Emantayev | 163561a | 2013-11-07 12:19:54 +0200 | [diff] [blame] | 192 | uar = kmalloc(sizeof(*uar), GFP_KERNEL); |
| 193 | if (!uar) { |
| 194 | err = -ENOMEM; |
| 195 | goto out; |
| 196 | } |
Eli Cohen | c1b43dc | 2011-03-22 22:38:41 +0000 | [diff] [blame] | 197 | } |
| 198 | err = mlx4_uar_alloc(dev, uar); |
| 199 | if (err) |
| 200 | goto free_kmalloc; |
| 201 | |
| 202 | uar->map = ioremap(uar->pfn << PAGE_SHIFT, PAGE_SIZE); |
| 203 | if (!uar->map) { |
| 204 | err = -ENOMEM; |
| 205 | goto free_uar; |
| 206 | } |
| 207 | |
| 208 | uar->bf_map = io_mapping_map_wc(priv->bf_mapping, uar->index << PAGE_SHIFT); |
| 209 | if (!uar->bf_map) { |
| 210 | err = -ENOMEM; |
| 211 | goto unamp_uar; |
| 212 | } |
| 213 | uar->free_bf_bmap = 0; |
| 214 | list_add(&uar->bf_list, &priv->bf_list); |
| 215 | } |
| 216 | |
Eli Cohen | c1b43dc | 2011-03-22 22:38:41 +0000 | [diff] [blame] | 217 | idx = ffz(uar->free_bf_bmap); |
| 218 | uar->free_bf_bmap |= 1 << idx; |
| 219 | bf->uar = uar; |
| 220 | bf->offset = 0; |
| 221 | bf->buf_size = dev->caps.bf_reg_size / 2; |
| 222 | bf->reg = uar->bf_map + idx * dev->caps.bf_reg_size; |
| 223 | if (uar->free_bf_bmap == (1 << dev->caps.bf_regs_per_page) - 1) |
| 224 | list_del_init(&uar->bf_list); |
| 225 | |
| 226 | goto out; |
| 227 | |
| 228 | unamp_uar: |
| 229 | bf->uar = NULL; |
| 230 | iounmap(uar->map); |
| 231 | |
| 232 | free_uar: |
| 233 | mlx4_uar_free(dev, uar); |
| 234 | |
| 235 | free_kmalloc: |
| 236 | kfree(uar); |
| 237 | |
| 238 | out: |
| 239 | mutex_unlock(&priv->bf_mutex); |
| 240 | return err; |
| 241 | } |
| 242 | EXPORT_SYMBOL_GPL(mlx4_bf_alloc); |
| 243 | |
| 244 | void mlx4_bf_free(struct mlx4_dev *dev, struct mlx4_bf *bf) |
| 245 | { |
| 246 | struct mlx4_priv *priv = mlx4_priv(dev); |
| 247 | int idx; |
| 248 | |
| 249 | if (!bf->uar || !bf->uar->bf_map) |
| 250 | return; |
| 251 | |
| 252 | mutex_lock(&priv->bf_mutex); |
| 253 | idx = (bf->reg - bf->uar->bf_map) / dev->caps.bf_reg_size; |
| 254 | bf->uar->free_bf_bmap &= ~(1 << idx); |
| 255 | if (!bf->uar->free_bf_bmap) { |
| 256 | if (!list_empty(&bf->uar->bf_list)) |
| 257 | list_del(&bf->uar->bf_list); |
| 258 | |
| 259 | io_mapping_unmap(bf->uar->bf_map); |
| 260 | iounmap(bf->uar->map); |
| 261 | mlx4_uar_free(dev, bf->uar); |
| 262 | kfree(bf->uar); |
| 263 | } else if (list_empty(&bf->uar->bf_list)) |
| 264 | list_add(&bf->uar->bf_list, &priv->bf_list); |
| 265 | |
| 266 | mutex_unlock(&priv->bf_mutex); |
| 267 | } |
| 268 | EXPORT_SYMBOL_GPL(mlx4_bf_free); |
| 269 | |
Roland Dreier | 225c7b1 | 2007-05-08 18:00:38 -0700 | [diff] [blame] | 270 | int mlx4_init_uar_table(struct mlx4_dev *dev) |
| 271 | { |
Huy Nguyen | 85743f1 | 2016-02-17 17:24:26 +0200 | [diff] [blame] | 272 | int num_reserved_uar = mlx4_get_num_reserved_uar(dev); |
| 273 | |
| 274 | mlx4_dbg(dev, "uar_page_shift = %d", dev->uar_page_shift); |
| 275 | mlx4_dbg(dev, "Effective reserved_uars=%d", dev->caps.reserved_uars); |
| 276 | |
| 277 | if (dev->caps.num_uars <= num_reserved_uar) { |
| 278 | mlx4_err( |
| 279 | dev, "Only %d UAR pages (need more than %d)\n", |
| 280 | dev->caps.num_uars, num_reserved_uar); |
Roland Dreier | 7644264 | 2008-07-23 08:12:47 -0700 | [diff] [blame] | 281 | mlx4_err(dev, "Increase firmware log2_uar_bar_megabytes?\n"); |
| 282 | return -ENODEV; |
| 283 | } |
| 284 | |
Roland Dreier | 225c7b1 | 2007-05-08 18:00:38 -0700 | [diff] [blame] | 285 | return mlx4_bitmap_init(&mlx4_priv(dev)->uar_table.bitmap, |
| 286 | dev->caps.num_uars, dev->caps.num_uars - 1, |
Jack Morgenstein | f5311ac | 2011-12-13 04:12:13 +0000 | [diff] [blame] | 287 | dev->caps.reserved_uars, 0); |
Roland Dreier | 225c7b1 | 2007-05-08 18:00:38 -0700 | [diff] [blame] | 288 | } |
| 289 | |
| 290 | void mlx4_cleanup_uar_table(struct mlx4_dev *dev) |
| 291 | { |
| 292 | mlx4_bitmap_cleanup(&mlx4_priv(dev)->uar_table.bitmap); |
| 293 | } |