Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 1 | /* |
Saeed Mahameed | 302bdf6 | 2015-04-02 17:07:29 +0300 | [diff] [blame] | 2 | * Copyright (c) 2013-2015, Mellanox Technologies. All rights reserved. |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 3 | * |
| 4 | * This software is available to you under a choice of one of two |
| 5 | * licenses. You may choose to be licensed under the terms of the GNU |
| 6 | * General Public License (GPL) Version 2, available from the file |
| 7 | * COPYING in the main directory of this source tree, or the |
| 8 | * OpenIB.org BSD license below: |
| 9 | * |
| 10 | * Redistribution and use in source and binary forms, with or |
| 11 | * without modification, are permitted provided that the following |
| 12 | * conditions are met: |
| 13 | * |
| 14 | * - Redistributions of source code must retain the above |
| 15 | * copyright notice, this list of conditions and the following |
| 16 | * disclaimer. |
| 17 | * |
| 18 | * - Redistributions in binary form must reproduce the above |
| 19 | * copyright notice, this list of conditions and the following |
| 20 | * disclaimer in the documentation and/or other materials |
| 21 | * provided with the distribution. |
| 22 | * |
| 23 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, |
| 24 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF |
| 25 | * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND |
| 26 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS |
| 27 | * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN |
| 28 | * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN |
| 29 | * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE |
| 30 | * SOFTWARE. |
| 31 | */ |
| 32 | |
Christoph Hellwig | adec640 | 2015-08-28 09:27:19 +0200 | [diff] [blame] | 33 | #include <linux/highmem.h> |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 34 | #include <linux/kernel.h> |
| 35 | #include <linux/module.h> |
Eli Cohen | fc50db9 | 2015-12-01 18:03:09 +0200 | [diff] [blame] | 36 | #include <linux/delay.h> |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 37 | #include <linux/mlx5/driver.h> |
| 38 | #include <linux/mlx5/cmd.h> |
| 39 | #include "mlx5_core.h" |
| 40 | |
| 41 | enum { |
| 42 | MLX5_PAGES_CANT_GIVE = 0, |
| 43 | MLX5_PAGES_GIVE = 1, |
| 44 | MLX5_PAGES_TAKE = 2 |
| 45 | }; |
| 46 | |
| 47 | struct mlx5_pages_req { |
| 48 | struct mlx5_core_dev *dev; |
Jack Morgenstein | f241e74 | 2014-07-28 23:30:23 +0300 | [diff] [blame] | 49 | u16 func_id; |
Moshe Lazer | 0a324f31 | 2013-08-14 17:46:48 +0300 | [diff] [blame] | 50 | s32 npages; |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 51 | struct work_struct work; |
| 52 | }; |
| 53 | |
| 54 | struct fw_page { |
Eli Cohen | bf0bf77 | 2013-10-23 09:53:19 +0300 | [diff] [blame] | 55 | struct rb_node rb_node; |
| 56 | u64 addr; |
| 57 | struct page *page; |
| 58 | u16 func_id; |
| 59 | unsigned long bitmask; |
| 60 | struct list_head list; |
| 61 | unsigned free_count; |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 62 | }; |
| 63 | |
Eli Cohen | dabed0e | 2013-09-11 16:35:28 +0300 | [diff] [blame] | 64 | enum { |
| 65 | MAX_RECLAIM_TIME_MSECS = 5000, |
Eli Cohen | fc50db9 | 2015-12-01 18:03:09 +0200 | [diff] [blame] | 66 | MAX_RECLAIM_VFS_PAGES_TIME_MSECS = 2 * 1000 * 60, |
Eli Cohen | dabed0e | 2013-09-11 16:35:28 +0300 | [diff] [blame] | 67 | }; |
| 68 | |
Eli Cohen | bf0bf77 | 2013-10-23 09:53:19 +0300 | [diff] [blame] | 69 | enum { |
| 70 | MLX5_MAX_RECLAIM_TIME_MILI = 5000, |
Eli Cohen | 05bdb2a | 2014-01-14 17:45:20 +0200 | [diff] [blame] | 71 | MLX5_NUM_4K_IN_PAGE = PAGE_SIZE / MLX5_ADAPTER_PAGE_SIZE, |
Eli Cohen | bf0bf77 | 2013-10-23 09:53:19 +0300 | [diff] [blame] | 72 | }; |
| 73 | |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 74 | static int insert_page(struct mlx5_core_dev *dev, u64 addr, struct page *page, u16 func_id) |
| 75 | { |
| 76 | struct rb_root *root = &dev->priv.page_root; |
| 77 | struct rb_node **new = &root->rb_node; |
| 78 | struct rb_node *parent = NULL; |
| 79 | struct fw_page *nfp; |
| 80 | struct fw_page *tfp; |
Eli Cohen | bf0bf77 | 2013-10-23 09:53:19 +0300 | [diff] [blame] | 81 | int i; |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 82 | |
| 83 | while (*new) { |
| 84 | parent = *new; |
| 85 | tfp = rb_entry(parent, struct fw_page, rb_node); |
| 86 | if (tfp->addr < addr) |
| 87 | new = &parent->rb_left; |
| 88 | else if (tfp->addr > addr) |
| 89 | new = &parent->rb_right; |
| 90 | else |
| 91 | return -EEXIST; |
| 92 | } |
| 93 | |
Eli Cohen | bf0bf77 | 2013-10-23 09:53:19 +0300 | [diff] [blame] | 94 | nfp = kzalloc(sizeof(*nfp), GFP_KERNEL); |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 95 | if (!nfp) |
| 96 | return -ENOMEM; |
| 97 | |
| 98 | nfp->addr = addr; |
| 99 | nfp->page = page; |
| 100 | nfp->func_id = func_id; |
Eli Cohen | bf0bf77 | 2013-10-23 09:53:19 +0300 | [diff] [blame] | 101 | nfp->free_count = MLX5_NUM_4K_IN_PAGE; |
| 102 | for (i = 0; i < MLX5_NUM_4K_IN_PAGE; i++) |
| 103 | set_bit(i, &nfp->bitmask); |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 104 | |
| 105 | rb_link_node(&nfp->rb_node, parent, new); |
| 106 | rb_insert_color(&nfp->rb_node, root); |
Eli Cohen | bf0bf77 | 2013-10-23 09:53:19 +0300 | [diff] [blame] | 107 | list_add(&nfp->list, &dev->priv.free_list); |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 108 | |
| 109 | return 0; |
| 110 | } |
| 111 | |
Eli Cohen | bf0bf77 | 2013-10-23 09:53:19 +0300 | [diff] [blame] | 112 | static struct fw_page *find_fw_page(struct mlx5_core_dev *dev, u64 addr) |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 113 | { |
| 114 | struct rb_root *root = &dev->priv.page_root; |
| 115 | struct rb_node *tmp = root->rb_node; |
Eli Cohen | bf0bf77 | 2013-10-23 09:53:19 +0300 | [diff] [blame] | 116 | struct fw_page *result = NULL; |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 117 | struct fw_page *tfp; |
| 118 | |
| 119 | while (tmp) { |
| 120 | tfp = rb_entry(tmp, struct fw_page, rb_node); |
| 121 | if (tfp->addr < addr) { |
| 122 | tmp = tmp->rb_left; |
| 123 | } else if (tfp->addr > addr) { |
| 124 | tmp = tmp->rb_right; |
| 125 | } else { |
Eli Cohen | bf0bf77 | 2013-10-23 09:53:19 +0300 | [diff] [blame] | 126 | result = tfp; |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 127 | break; |
| 128 | } |
| 129 | } |
| 130 | |
| 131 | return result; |
| 132 | } |
| 133 | |
| 134 | static int mlx5_cmd_query_pages(struct mlx5_core_dev *dev, u16 *func_id, |
Moshe Lazer | 0a324f31 | 2013-08-14 17:46:48 +0300 | [diff] [blame] | 135 | s32 *npages, int boot) |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 136 | { |
Saeed Mahameed | a533ed5 | 2016-07-17 13:27:25 +0300 | [diff] [blame] | 137 | u32 out[MLX5_ST_SZ_DW(query_pages_out)] = {0}; |
| 138 | u32 in[MLX5_ST_SZ_DW(query_pages_in)] = {0}; |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 139 | int err; |
| 140 | |
Saeed Mahameed | a533ed5 | 2016-07-17 13:27:25 +0300 | [diff] [blame] | 141 | MLX5_SET(query_pages_in, in, opcode, MLX5_CMD_OP_QUERY_PAGES); |
| 142 | MLX5_SET(query_pages_in, in, op_mod, boot ? |
| 143 | MLX5_QUERY_PAGES_IN_OP_MOD_BOOT_PAGES : |
| 144 | MLX5_QUERY_PAGES_IN_OP_MOD_INIT_PAGES); |
Moshe Lazer | 0a324f31 | 2013-08-14 17:46:48 +0300 | [diff] [blame] | 145 | |
Saeed Mahameed | a533ed5 | 2016-07-17 13:27:25 +0300 | [diff] [blame] | 146 | err = mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 147 | if (err) |
| 148 | return err; |
| 149 | |
Saeed Mahameed | a533ed5 | 2016-07-17 13:27:25 +0300 | [diff] [blame] | 150 | *npages = MLX5_GET(query_pages_out, out, num_pages); |
| 151 | *func_id = MLX5_GET(query_pages_out, out, function_id); |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 152 | |
| 153 | return err; |
| 154 | } |
| 155 | |
Eli Cohen | bf0bf77 | 2013-10-23 09:53:19 +0300 | [diff] [blame] | 156 | static int alloc_4k(struct mlx5_core_dev *dev, u64 *addr) |
| 157 | { |
| 158 | struct fw_page *fp; |
| 159 | unsigned n; |
| 160 | |
Dan Carpenter | 24e42754 | 2014-01-14 17:45:11 +0200 | [diff] [blame] | 161 | if (list_empty(&dev->priv.free_list)) |
Eli Cohen | bf0bf77 | 2013-10-23 09:53:19 +0300 | [diff] [blame] | 162 | return -ENOMEM; |
Eli Cohen | bf0bf77 | 2013-10-23 09:53:19 +0300 | [diff] [blame] | 163 | |
| 164 | fp = list_entry(dev->priv.free_list.next, struct fw_page, list); |
| 165 | n = find_first_bit(&fp->bitmask, 8 * sizeof(fp->bitmask)); |
| 166 | if (n >= MLX5_NUM_4K_IN_PAGE) { |
| 167 | mlx5_core_warn(dev, "alloc 4k bug\n"); |
| 168 | return -ENOENT; |
| 169 | } |
| 170 | clear_bit(n, &fp->bitmask); |
| 171 | fp->free_count--; |
| 172 | if (!fp->free_count) |
| 173 | list_del(&fp->list); |
| 174 | |
Eli Cohen | 05bdb2a | 2014-01-14 17:45:20 +0200 | [diff] [blame] | 175 | *addr = fp->addr + n * MLX5_ADAPTER_PAGE_SIZE; |
Eli Cohen | bf0bf77 | 2013-10-23 09:53:19 +0300 | [diff] [blame] | 176 | |
| 177 | return 0; |
| 178 | } |
| 179 | |
Honggang LI | 59d2d18 | 2015-04-15 16:36:15 +0800 | [diff] [blame] | 180 | #define MLX5_U64_4K_PAGE_MASK ((~(u64)0U) << PAGE_SHIFT) |
| 181 | |
Eli Cohen | bf0bf77 | 2013-10-23 09:53:19 +0300 | [diff] [blame] | 182 | static void free_4k(struct mlx5_core_dev *dev, u64 addr) |
| 183 | { |
| 184 | struct fw_page *fwp; |
| 185 | int n; |
| 186 | |
Honggang LI | 59d2d18 | 2015-04-15 16:36:15 +0800 | [diff] [blame] | 187 | fwp = find_fw_page(dev, addr & MLX5_U64_4K_PAGE_MASK); |
Eli Cohen | bf0bf77 | 2013-10-23 09:53:19 +0300 | [diff] [blame] | 188 | if (!fwp) { |
| 189 | mlx5_core_warn(dev, "page not found\n"); |
| 190 | return; |
| 191 | } |
| 192 | |
Honggang LI | 59d2d18 | 2015-04-15 16:36:15 +0800 | [diff] [blame] | 193 | n = (addr & ~MLX5_U64_4K_PAGE_MASK) >> MLX5_ADAPTER_PAGE_SHIFT; |
Eli Cohen | bf0bf77 | 2013-10-23 09:53:19 +0300 | [diff] [blame] | 194 | fwp->free_count++; |
| 195 | set_bit(n, &fwp->bitmask); |
| 196 | if (fwp->free_count == MLX5_NUM_4K_IN_PAGE) { |
| 197 | rb_erase(&fwp->rb_node, &dev->priv.page_root); |
Eli Cohen | 2b136d0 | 2013-10-31 15:26:33 +0200 | [diff] [blame] | 198 | if (fwp->free_count != 1) |
| 199 | list_del(&fwp->list); |
Honggang LI | 59d2d18 | 2015-04-15 16:36:15 +0800 | [diff] [blame] | 200 | dma_unmap_page(&dev->pdev->dev, addr & MLX5_U64_4K_PAGE_MASK, |
| 201 | PAGE_SIZE, DMA_BIDIRECTIONAL); |
Eli Cohen | bf0bf77 | 2013-10-23 09:53:19 +0300 | [diff] [blame] | 202 | __free_page(fwp->page); |
| 203 | kfree(fwp); |
| 204 | } else if (fwp->free_count == 1) { |
| 205 | list_add(&fwp->list, &dev->priv.free_list); |
| 206 | } |
| 207 | } |
| 208 | |
| 209 | static int alloc_system_page(struct mlx5_core_dev *dev, u16 func_id) |
| 210 | { |
| 211 | struct page *page; |
| 212 | u64 addr; |
| 213 | int err; |
Eli Cohen | ad18910 | 2015-04-02 17:07:19 +0300 | [diff] [blame] | 214 | int nid = dev_to_node(&dev->pdev->dev); |
Eli Cohen | bf0bf77 | 2013-10-23 09:53:19 +0300 | [diff] [blame] | 215 | |
Eli Cohen | ad18910 | 2015-04-02 17:07:19 +0300 | [diff] [blame] | 216 | page = alloc_pages_node(nid, GFP_HIGHUSER, 0); |
Eli Cohen | bf0bf77 | 2013-10-23 09:53:19 +0300 | [diff] [blame] | 217 | if (!page) { |
| 218 | mlx5_core_warn(dev, "failed to allocate page\n"); |
| 219 | return -ENOMEM; |
| 220 | } |
| 221 | addr = dma_map_page(&dev->pdev->dev, page, 0, |
| 222 | PAGE_SIZE, DMA_BIDIRECTIONAL); |
| 223 | if (dma_mapping_error(&dev->pdev->dev, addr)) { |
| 224 | mlx5_core_warn(dev, "failed dma mapping page\n"); |
| 225 | err = -ENOMEM; |
| 226 | goto out_alloc; |
| 227 | } |
| 228 | err = insert_page(dev, addr, page, func_id); |
| 229 | if (err) { |
| 230 | mlx5_core_err(dev, "failed to track allocated page\n"); |
| 231 | goto out_mapping; |
| 232 | } |
| 233 | |
| 234 | return 0; |
| 235 | |
| 236 | out_mapping: |
| 237 | dma_unmap_page(&dev->pdev->dev, addr, PAGE_SIZE, DMA_BIDIRECTIONAL); |
| 238 | |
| 239 | out_alloc: |
| 240 | __free_page(page); |
| 241 | |
| 242 | return err; |
| 243 | } |
Eli Cohen | a8ffe63 | 2015-09-25 10:49:13 +0300 | [diff] [blame] | 244 | |
| 245 | static void page_notify_fail(struct mlx5_core_dev *dev, u16 func_id) |
| 246 | { |
Saeed Mahameed | a533ed5 | 2016-07-17 13:27:25 +0300 | [diff] [blame] | 247 | u32 out[MLX5_ST_SZ_DW(manage_pages_out)] = {0}; |
| 248 | u32 in[MLX5_ST_SZ_DW(manage_pages_in)] = {0}; |
Eli Cohen | a8ffe63 | 2015-09-25 10:49:13 +0300 | [diff] [blame] | 249 | int err; |
| 250 | |
Saeed Mahameed | a533ed5 | 2016-07-17 13:27:25 +0300 | [diff] [blame] | 251 | MLX5_SET(manage_pages_in, in, opcode, MLX5_CMD_OP_MANAGE_PAGES); |
| 252 | MLX5_SET(manage_pages_in, in, op_mod, MLX5_PAGES_CANT_GIVE); |
| 253 | MLX5_SET(manage_pages_in, in, function_id, func_id); |
Saeed Mahameed | c4f287c | 2016-07-19 20:17:12 +0300 | [diff] [blame] | 254 | |
Saeed Mahameed | a533ed5 | 2016-07-17 13:27:25 +0300 | [diff] [blame] | 255 | err = mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); |
Eli Cohen | a8ffe63 | 2015-09-25 10:49:13 +0300 | [diff] [blame] | 256 | if (err) |
Saeed Mahameed | a533ed5 | 2016-07-17 13:27:25 +0300 | [diff] [blame] | 257 | mlx5_core_warn(dev, "page notify failed func_id(%d) err(%d)\n", |
| 258 | func_id, err); |
Eli Cohen | a8ffe63 | 2015-09-25 10:49:13 +0300 | [diff] [blame] | 259 | } |
| 260 | |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 261 | static int give_pages(struct mlx5_core_dev *dev, u16 func_id, int npages, |
| 262 | int notify_fail) |
| 263 | { |
Saeed Mahameed | a533ed5 | 2016-07-17 13:27:25 +0300 | [diff] [blame] | 264 | u32 out[MLX5_ST_SZ_DW(manage_pages_out)] = {0}; |
| 265 | int inlen = MLX5_ST_SZ_BYTES(manage_pages_in); |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 266 | u64 addr; |
| 267 | int err; |
Saeed Mahameed | a533ed5 | 2016-07-17 13:27:25 +0300 | [diff] [blame] | 268 | u32 *in; |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 269 | int i; |
| 270 | |
Saeed Mahameed | a533ed5 | 2016-07-17 13:27:25 +0300 | [diff] [blame] | 271 | inlen += npages * MLX5_FLD_SZ_BYTES(manage_pages_in, pas[0]); |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 272 | in = mlx5_vzalloc(inlen); |
| 273 | if (!in) { |
Eli Cohen | a8ffe63 | 2015-09-25 10:49:13 +0300 | [diff] [blame] | 274 | err = -ENOMEM; |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 275 | mlx5_core_warn(dev, "vzalloc failed %d\n", inlen); |
Eli Cohen | a8ffe63 | 2015-09-25 10:49:13 +0300 | [diff] [blame] | 276 | goto out_free; |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 277 | } |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 278 | |
| 279 | for (i = 0; i < npages; i++) { |
Eli Cohen | bf0bf77 | 2013-10-23 09:53:19 +0300 | [diff] [blame] | 280 | retry: |
| 281 | err = alloc_4k(dev, &addr); |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 282 | if (err) { |
Eli Cohen | bf0bf77 | 2013-10-23 09:53:19 +0300 | [diff] [blame] | 283 | if (err == -ENOMEM) |
| 284 | err = alloc_system_page(dev, func_id); |
| 285 | if (err) |
| 286 | goto out_4k; |
| 287 | |
| 288 | goto retry; |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 289 | } |
Saeed Mahameed | a533ed5 | 2016-07-17 13:27:25 +0300 | [diff] [blame] | 290 | MLX5_SET64(manage_pages_in, in, pas[i], addr); |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 291 | } |
| 292 | |
Saeed Mahameed | a533ed5 | 2016-07-17 13:27:25 +0300 | [diff] [blame] | 293 | MLX5_SET(manage_pages_in, in, opcode, MLX5_CMD_OP_MANAGE_PAGES); |
| 294 | MLX5_SET(manage_pages_in, in, op_mod, MLX5_PAGES_GIVE); |
| 295 | MLX5_SET(manage_pages_in, in, function_id, func_id); |
| 296 | MLX5_SET(manage_pages_in, in, input_num_entries, npages); |
| 297 | |
| 298 | err = mlx5_cmd_exec(dev, in, inlen, out, sizeof(out)); |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 299 | if (err) { |
Joe Perches | 1a91de2 | 2014-05-07 12:52:57 -0700 | [diff] [blame] | 300 | mlx5_core_warn(dev, "func_id 0x%x, npages %d, err %d\n", |
| 301 | func_id, npages, err); |
Eli Cohen | a8ffe63 | 2015-09-25 10:49:13 +0300 | [diff] [blame] | 302 | goto out_4k; |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 303 | } |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 304 | |
Eli Cohen | fc50db9 | 2015-12-01 18:03:09 +0200 | [diff] [blame] | 305 | dev->priv.fw_pages += npages; |
| 306 | if (func_id) |
| 307 | dev->priv.vfs_pages += npages; |
| 308 | |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 309 | mlx5_core_dbg(dev, "err %d\n", err); |
| 310 | |
Eli Cohen | a8ffe63 | 2015-09-25 10:49:13 +0300 | [diff] [blame] | 311 | kvfree(in); |
| 312 | return 0; |
Eli Cohen | 952f5f6 | 2013-10-23 09:53:18 +0300 | [diff] [blame] | 313 | |
Eli Cohen | bf0bf77 | 2013-10-23 09:53:19 +0300 | [diff] [blame] | 314 | out_4k: |
| 315 | for (i--; i >= 0; i--) |
Saeed Mahameed | a533ed5 | 2016-07-17 13:27:25 +0300 | [diff] [blame] | 316 | free_4k(dev, MLX5_GET64(manage_pages_in, in, pas[i])); |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 317 | out_free: |
Al Viro | 479163f | 2014-11-20 08:13:57 +0000 | [diff] [blame] | 318 | kvfree(in); |
Eli Cohen | a8ffe63 | 2015-09-25 10:49:13 +0300 | [diff] [blame] | 319 | if (notify_fail) |
| 320 | page_notify_fail(dev, func_id); |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 321 | return err; |
| 322 | } |
| 323 | |
Daniel Jurgens | 5adff6a | 2016-06-30 17:34:40 +0300 | [diff] [blame] | 324 | static int reclaim_pages_cmd(struct mlx5_core_dev *dev, |
Saeed Mahameed | a533ed5 | 2016-07-17 13:27:25 +0300 | [diff] [blame] | 325 | u32 *in, int in_size, u32 *out, int out_size) |
Daniel Jurgens | 5adff6a | 2016-06-30 17:34:40 +0300 | [diff] [blame] | 326 | { |
| 327 | struct fw_page *fwp; |
| 328 | struct rb_node *p; |
Mohamad Haj Yahia | d62292e | 2016-09-09 17:35:17 +0300 | [diff] [blame] | 329 | u32 func_id; |
Daniel Jurgens | 5adff6a | 2016-06-30 17:34:40 +0300 | [diff] [blame] | 330 | u32 npages; |
| 331 | u32 i = 0; |
| 332 | |
Saeed Mahameed | c4f287c | 2016-07-19 20:17:12 +0300 | [diff] [blame] | 333 | if (dev->state != MLX5_DEVICE_STATE_INTERNAL_ERROR) |
| 334 | return mlx5_cmd_exec(dev, in, in_size, out, out_size); |
Saeed Mahameed | a533ed5 | 2016-07-17 13:27:25 +0300 | [diff] [blame] | 335 | |
| 336 | /* No hard feelings, we want our pages back! */ |
| 337 | npages = MLX5_GET(manage_pages_in, in, input_num_entries); |
Mohamad Haj Yahia | d62292e | 2016-09-09 17:35:17 +0300 | [diff] [blame] | 338 | func_id = MLX5_GET(manage_pages_in, in, function_id); |
Daniel Jurgens | 5adff6a | 2016-06-30 17:34:40 +0300 | [diff] [blame] | 339 | |
| 340 | p = rb_first(&dev->priv.page_root); |
| 341 | while (p && i < npages) { |
| 342 | fwp = rb_entry(p, struct fw_page, rb_node); |
Daniel Jurgens | 5adff6a | 2016-06-30 17:34:40 +0300 | [diff] [blame] | 343 | p = rb_next(p); |
Mohamad Haj Yahia | d62292e | 2016-09-09 17:35:17 +0300 | [diff] [blame] | 344 | if (fwp->func_id != func_id) |
| 345 | continue; |
| 346 | |
| 347 | MLX5_SET64(manage_pages_out, out, pas[i], fwp->addr); |
Daniel Jurgens | 5adff6a | 2016-06-30 17:34:40 +0300 | [diff] [blame] | 348 | i++; |
| 349 | } |
| 350 | |
Saeed Mahameed | a533ed5 | 2016-07-17 13:27:25 +0300 | [diff] [blame] | 351 | MLX5_SET(manage_pages_out, out, output_num_entries, i); |
Daniel Jurgens | 5adff6a | 2016-06-30 17:34:40 +0300 | [diff] [blame] | 352 | return 0; |
| 353 | } |
| 354 | |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 355 | static int reclaim_pages(struct mlx5_core_dev *dev, u32 func_id, int npages, |
| 356 | int *nclaimed) |
| 357 | { |
Saeed Mahameed | a533ed5 | 2016-07-17 13:27:25 +0300 | [diff] [blame] | 358 | int outlen = MLX5_ST_SZ_BYTES(manage_pages_out); |
| 359 | u32 in[MLX5_ST_SZ_DW(manage_pages_in)] = {0}; |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 360 | int num_claimed; |
Saeed Mahameed | a533ed5 | 2016-07-17 13:27:25 +0300 | [diff] [blame] | 361 | u32 *out; |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 362 | int err; |
| 363 | int i; |
| 364 | |
Eli Cohen | dabed0e | 2013-09-11 16:35:28 +0300 | [diff] [blame] | 365 | if (nclaimed) |
| 366 | *nclaimed = 0; |
| 367 | |
Saeed Mahameed | a533ed5 | 2016-07-17 13:27:25 +0300 | [diff] [blame] | 368 | outlen += npages * MLX5_FLD_SZ_BYTES(manage_pages_out, pas[0]); |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 369 | out = mlx5_vzalloc(outlen); |
| 370 | if (!out) |
| 371 | return -ENOMEM; |
| 372 | |
Saeed Mahameed | a533ed5 | 2016-07-17 13:27:25 +0300 | [diff] [blame] | 373 | MLX5_SET(manage_pages_in, in, opcode, MLX5_CMD_OP_MANAGE_PAGES); |
| 374 | MLX5_SET(manage_pages_in, in, op_mod, MLX5_PAGES_TAKE); |
| 375 | MLX5_SET(manage_pages_in, in, function_id, func_id); |
| 376 | MLX5_SET(manage_pages_in, in, input_num_entries, npages); |
| 377 | |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 378 | mlx5_core_dbg(dev, "npages %d, outlen %d\n", npages, outlen); |
Saeed Mahameed | a533ed5 | 2016-07-17 13:27:25 +0300 | [diff] [blame] | 379 | err = reclaim_pages_cmd(dev, in, sizeof(in), out, outlen); |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 380 | if (err) { |
Daniel Jurgens | 5adff6a | 2016-06-30 17:34:40 +0300 | [diff] [blame] | 381 | mlx5_core_err(dev, "failed reclaiming pages: err %d\n", err); |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 382 | goto out_free; |
| 383 | } |
| 384 | |
Saeed Mahameed | a533ed5 | 2016-07-17 13:27:25 +0300 | [diff] [blame] | 385 | num_claimed = MLX5_GET(manage_pages_out, out, output_num_entries); |
Eli Cohen | fc50db9 | 2015-12-01 18:03:09 +0200 | [diff] [blame] | 386 | if (num_claimed > npages) { |
| 387 | mlx5_core_warn(dev, "fw returned %d, driver asked %d => corruption\n", |
| 388 | num_claimed, npages); |
| 389 | err = -EINVAL; |
| 390 | goto out_free; |
| 391 | } |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 392 | |
Saeed Mahameed | a533ed5 | 2016-07-17 13:27:25 +0300 | [diff] [blame] | 393 | for (i = 0; i < num_claimed; i++) |
| 394 | free_4k(dev, MLX5_GET64(manage_pages_out, out, pas[i])); |
| 395 | |
Daniel Jurgens | 5adff6a | 2016-06-30 17:34:40 +0300 | [diff] [blame] | 396 | |
| 397 | if (nclaimed) |
| 398 | *nclaimed = num_claimed; |
| 399 | |
Eli Cohen | fc50db9 | 2015-12-01 18:03:09 +0200 | [diff] [blame] | 400 | dev->priv.fw_pages -= num_claimed; |
| 401 | if (func_id) |
| 402 | dev->priv.vfs_pages -= num_claimed; |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 403 | |
| 404 | out_free: |
Al Viro | 479163f | 2014-11-20 08:13:57 +0000 | [diff] [blame] | 405 | kvfree(out); |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 406 | return err; |
| 407 | } |
| 408 | |
| 409 | static void pages_work_handler(struct work_struct *work) |
| 410 | { |
| 411 | struct mlx5_pages_req *req = container_of(work, struct mlx5_pages_req, work); |
| 412 | struct mlx5_core_dev *dev = req->dev; |
| 413 | int err = 0; |
| 414 | |
| 415 | if (req->npages < 0) |
| 416 | err = reclaim_pages(dev, req->func_id, -1 * req->npages, NULL); |
| 417 | else if (req->npages > 0) |
| 418 | err = give_pages(dev, req->func_id, req->npages, 1); |
| 419 | |
| 420 | if (err) |
Joe Perches | 1a91de2 | 2014-05-07 12:52:57 -0700 | [diff] [blame] | 421 | mlx5_core_warn(dev, "%s fail %d\n", |
| 422 | req->npages < 0 ? "reclaim" : "give", err); |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 423 | |
| 424 | kfree(req); |
| 425 | } |
| 426 | |
| 427 | void mlx5_core_req_pages_handler(struct mlx5_core_dev *dev, u16 func_id, |
Moshe Lazer | 0a324f31 | 2013-08-14 17:46:48 +0300 | [diff] [blame] | 428 | s32 npages) |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 429 | { |
| 430 | struct mlx5_pages_req *req; |
| 431 | |
| 432 | req = kzalloc(sizeof(*req), GFP_ATOMIC); |
| 433 | if (!req) { |
| 434 | mlx5_core_warn(dev, "failed to allocate pages request\n"); |
| 435 | return; |
| 436 | } |
| 437 | |
| 438 | req->dev = dev; |
| 439 | req->func_id = func_id; |
| 440 | req->npages = npages; |
| 441 | INIT_WORK(&req->work, pages_work_handler); |
| 442 | queue_work(dev->priv.pg_wq, &req->work); |
| 443 | } |
| 444 | |
Eli Cohen | cd23b14 | 2013-07-18 15:31:08 +0300 | [diff] [blame] | 445 | int mlx5_satisfy_startup_pages(struct mlx5_core_dev *dev, int boot) |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 446 | { |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 447 | u16 uninitialized_var(func_id); |
Moshe Lazer | 0a324f31 | 2013-08-14 17:46:48 +0300 | [diff] [blame] | 448 | s32 uninitialized_var(npages); |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 449 | int err; |
| 450 | |
Moshe Lazer | 0a324f31 | 2013-08-14 17:46:48 +0300 | [diff] [blame] | 451 | err = mlx5_cmd_query_pages(dev, &func_id, &npages, boot); |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 452 | if (err) |
| 453 | return err; |
| 454 | |
Moshe Lazer | 0a324f31 | 2013-08-14 17:46:48 +0300 | [diff] [blame] | 455 | mlx5_core_dbg(dev, "requested %d %s pages for func_id 0x%x\n", |
| 456 | npages, boot ? "boot" : "init", func_id); |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 457 | |
Moshe Lazer | 0a324f31 | 2013-08-14 17:46:48 +0300 | [diff] [blame] | 458 | return give_pages(dev, func_id, npages, 0); |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 459 | } |
| 460 | |
Moshe Lazer | 4e3d677 | 2013-10-23 09:53:21 +0300 | [diff] [blame] | 461 | enum { |
| 462 | MLX5_BLKS_FOR_RECLAIM_PAGES = 12 |
| 463 | }; |
| 464 | |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 465 | static int optimal_reclaimed_pages(void) |
| 466 | { |
| 467 | struct mlx5_cmd_prot_block *block; |
| 468 | struct mlx5_cmd_layout *lay; |
| 469 | int ret; |
| 470 | |
Moshe Lazer | 4e3d677 | 2013-10-23 09:53:21 +0300 | [diff] [blame] | 471 | ret = (sizeof(lay->out) + MLX5_BLKS_FOR_RECLAIM_PAGES * sizeof(block->data) - |
Saeed Mahameed | a533ed5 | 2016-07-17 13:27:25 +0300 | [diff] [blame] | 472 | MLX5_ST_SZ_BYTES(manage_pages_out)) / |
| 473 | MLX5_FLD_SZ_BYTES(manage_pages_out, pas[0]); |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 474 | |
| 475 | return ret; |
| 476 | } |
| 477 | |
| 478 | int mlx5_reclaim_startup_pages(struct mlx5_core_dev *dev) |
| 479 | { |
Eli Cohen | dabed0e | 2013-09-11 16:35:28 +0300 | [diff] [blame] | 480 | unsigned long end = jiffies + msecs_to_jiffies(MAX_RECLAIM_TIME_MSECS); |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 481 | struct fw_page *fwp; |
| 482 | struct rb_node *p; |
Eli Cohen | dabed0e | 2013-09-11 16:35:28 +0300 | [diff] [blame] | 483 | int nclaimed = 0; |
Majd Dibbiny | 89d44f0 | 2015-10-14 17:43:46 +0300 | [diff] [blame] | 484 | int err = 0; |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 485 | |
| 486 | do { |
| 487 | p = rb_first(&dev->priv.page_root); |
| 488 | if (p) { |
| 489 | fwp = rb_entry(p, struct fw_page, rb_node); |
Daniel Jurgens | 5adff6a | 2016-06-30 17:34:40 +0300 | [diff] [blame] | 490 | err = reclaim_pages(dev, fwp->func_id, |
| 491 | optimal_reclaimed_pages(), |
| 492 | &nclaimed); |
| 493 | |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 494 | if (err) { |
Joe Perches | 1a91de2 | 2014-05-07 12:52:57 -0700 | [diff] [blame] | 495 | mlx5_core_warn(dev, "failed reclaiming pages (%d)\n", |
| 496 | err); |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 497 | return err; |
| 498 | } |
Eli Cohen | dabed0e | 2013-09-11 16:35:28 +0300 | [diff] [blame] | 499 | if (nclaimed) |
| 500 | end = jiffies + msecs_to_jiffies(MAX_RECLAIM_TIME_MSECS); |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 501 | } |
| 502 | if (time_after(jiffies, end)) { |
| 503 | mlx5_core_warn(dev, "FW did not return all pages. giving up...\n"); |
| 504 | break; |
| 505 | } |
| 506 | } while (p); |
| 507 | |
Daniel Jurgens | 5adff6a | 2016-06-30 17:34:40 +0300 | [diff] [blame] | 508 | WARN(dev->priv.fw_pages, |
| 509 | "FW pages counter is %d after reclaiming all pages\n", |
| 510 | dev->priv.fw_pages); |
| 511 | WARN(dev->priv.vfs_pages, |
| 512 | "VFs FW pages counter is %d after reclaiming all pages\n", |
| 513 | dev->priv.vfs_pages); |
| 514 | |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 515 | return 0; |
| 516 | } |
| 517 | |
| 518 | void mlx5_pagealloc_init(struct mlx5_core_dev *dev) |
| 519 | { |
| 520 | dev->priv.page_root = RB_ROOT; |
Eli Cohen | bf0bf77 | 2013-10-23 09:53:19 +0300 | [diff] [blame] | 521 | INIT_LIST_HEAD(&dev->priv.free_list); |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 522 | } |
| 523 | |
| 524 | void mlx5_pagealloc_cleanup(struct mlx5_core_dev *dev) |
| 525 | { |
| 526 | /* nothing */ |
| 527 | } |
| 528 | |
| 529 | int mlx5_pagealloc_start(struct mlx5_core_dev *dev) |
| 530 | { |
| 531 | dev->priv.pg_wq = create_singlethread_workqueue("mlx5_page_allocator"); |
| 532 | if (!dev->priv.pg_wq) |
| 533 | return -ENOMEM; |
| 534 | |
| 535 | return 0; |
| 536 | } |
| 537 | |
| 538 | void mlx5_pagealloc_stop(struct mlx5_core_dev *dev) |
| 539 | { |
| 540 | destroy_workqueue(dev->priv.pg_wq); |
| 541 | } |
Eli Cohen | fc50db9 | 2015-12-01 18:03:09 +0200 | [diff] [blame] | 542 | |
| 543 | int mlx5_wait_for_vf_pages(struct mlx5_core_dev *dev) |
| 544 | { |
| 545 | unsigned long end = jiffies + msecs_to_jiffies(MAX_RECLAIM_VFS_PAGES_TIME_MSECS); |
| 546 | int prev_vfs_pages = dev->priv.vfs_pages; |
| 547 | |
Mohamad Haj Yahia | d62292e | 2016-09-09 17:35:17 +0300 | [diff] [blame] | 548 | /* In case of internal error we will free the pages manually later */ |
| 549 | if (dev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR) { |
| 550 | mlx5_core_warn(dev, "Skipping wait for vf pages stage"); |
| 551 | return 0; |
| 552 | } |
| 553 | |
Eli Cohen | fc50db9 | 2015-12-01 18:03:09 +0200 | [diff] [blame] | 554 | mlx5_core_dbg(dev, "Waiting for %d pages from %s\n", prev_vfs_pages, |
| 555 | dev->priv.name); |
| 556 | while (dev->priv.vfs_pages) { |
| 557 | if (time_after(jiffies, end)) { |
| 558 | mlx5_core_warn(dev, "aborting while there are %d pending pages\n", dev->priv.vfs_pages); |
| 559 | return -ETIMEDOUT; |
| 560 | } |
| 561 | if (dev->priv.vfs_pages < prev_vfs_pages) { |
| 562 | end = jiffies + msecs_to_jiffies(MAX_RECLAIM_VFS_PAGES_TIME_MSECS); |
| 563 | prev_vfs_pages = dev->priv.vfs_pages; |
| 564 | } |
| 565 | msleep(50); |
| 566 | } |
| 567 | |
| 568 | mlx5_core_dbg(dev, "All pages received from %s\n", dev->priv.name); |
| 569 | return 0; |
| 570 | } |