Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 1 | /* |
Saeed Mahameed | 302bdf6 | 2015-04-02 17:07:29 +0300 | [diff] [blame] | 2 | * Copyright (c) 2013-2015, Mellanox Technologies. All rights reserved. |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 3 | * |
| 4 | * This software is available to you under a choice of one of two |
| 5 | * licenses. You may choose to be licensed under the terms of the GNU |
| 6 | * General Public License (GPL) Version 2, available from the file |
| 7 | * COPYING in the main directory of this source tree, or the |
| 8 | * OpenIB.org BSD license below: |
| 9 | * |
| 10 | * Redistribution and use in source and binary forms, with or |
| 11 | * without modification, are permitted provided that the following |
| 12 | * conditions are met: |
| 13 | * |
| 14 | * - Redistributions of source code must retain the above |
| 15 | * copyright notice, this list of conditions and the following |
| 16 | * disclaimer. |
| 17 | * |
| 18 | * - Redistributions in binary form must reproduce the above |
| 19 | * copyright notice, this list of conditions and the following |
| 20 | * disclaimer in the documentation and/or other materials |
| 21 | * provided with the distribution. |
| 22 | * |
| 23 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, |
| 24 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF |
| 25 | * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND |
| 26 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS |
| 27 | * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN |
| 28 | * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN |
| 29 | * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE |
| 30 | * SOFTWARE. |
| 31 | */ |
| 32 | |
Christoph Hellwig | adec640 | 2015-08-28 09:27:19 +0200 | [diff] [blame] | 33 | #include <linux/highmem.h> |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 34 | #include <linux/kernel.h> |
| 35 | #include <linux/module.h> |
Eli Cohen | fc50db9 | 2015-12-01 18:03:09 +0200 | [diff] [blame] | 36 | #include <linux/delay.h> |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 37 | #include <linux/mlx5/driver.h> |
| 38 | #include <linux/mlx5/cmd.h> |
| 39 | #include "mlx5_core.h" |
| 40 | |
| 41 | enum { |
| 42 | MLX5_PAGES_CANT_GIVE = 0, |
| 43 | MLX5_PAGES_GIVE = 1, |
| 44 | MLX5_PAGES_TAKE = 2 |
| 45 | }; |
| 46 | |
Moshe Lazer | 0a324f31 | 2013-08-14 17:46:48 +0300 | [diff] [blame] | 47 | enum { |
| 48 | MLX5_BOOT_PAGES = 1, |
| 49 | MLX5_INIT_PAGES = 2, |
| 50 | MLX5_POST_INIT_PAGES = 3 |
| 51 | }; |
| 52 | |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 53 | struct mlx5_pages_req { |
| 54 | struct mlx5_core_dev *dev; |
Jack Morgenstein | f241e74 | 2014-07-28 23:30:23 +0300 | [diff] [blame] | 55 | u16 func_id; |
Moshe Lazer | 0a324f31 | 2013-08-14 17:46:48 +0300 | [diff] [blame] | 56 | s32 npages; |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 57 | struct work_struct work; |
| 58 | }; |
| 59 | |
| 60 | struct fw_page { |
Eli Cohen | bf0bf77 | 2013-10-23 09:53:19 +0300 | [diff] [blame] | 61 | struct rb_node rb_node; |
| 62 | u64 addr; |
| 63 | struct page *page; |
| 64 | u16 func_id; |
| 65 | unsigned long bitmask; |
| 66 | struct list_head list; |
| 67 | unsigned free_count; |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 68 | }; |
| 69 | |
| 70 | struct mlx5_query_pages_inbox { |
| 71 | struct mlx5_inbox_hdr hdr; |
| 72 | u8 rsvd[8]; |
| 73 | }; |
| 74 | |
| 75 | struct mlx5_query_pages_outbox { |
| 76 | struct mlx5_outbox_hdr hdr; |
Moshe Lazer | 0a324f31 | 2013-08-14 17:46:48 +0300 | [diff] [blame] | 77 | __be16 rsvd; |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 78 | __be16 func_id; |
Moshe Lazer | 0a324f31 | 2013-08-14 17:46:48 +0300 | [diff] [blame] | 79 | __be32 num_pages; |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 80 | }; |
| 81 | |
| 82 | struct mlx5_manage_pages_inbox { |
| 83 | struct mlx5_inbox_hdr hdr; |
Moshe Lazer | 0a324f31 | 2013-08-14 17:46:48 +0300 | [diff] [blame] | 84 | __be16 rsvd; |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 85 | __be16 func_id; |
Moshe Lazer | 0a324f31 | 2013-08-14 17:46:48 +0300 | [diff] [blame] | 86 | __be32 num_entries; |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 87 | __be64 pas[0]; |
| 88 | }; |
| 89 | |
| 90 | struct mlx5_manage_pages_outbox { |
| 91 | struct mlx5_outbox_hdr hdr; |
Moshe Lazer | 0a324f31 | 2013-08-14 17:46:48 +0300 | [diff] [blame] | 92 | __be32 num_entries; |
| 93 | u8 rsvd[4]; |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 94 | __be64 pas[0]; |
| 95 | }; |
| 96 | |
Eli Cohen | dabed0e | 2013-09-11 16:35:28 +0300 | [diff] [blame] | 97 | enum { |
| 98 | MAX_RECLAIM_TIME_MSECS = 5000, |
Eli Cohen | fc50db9 | 2015-12-01 18:03:09 +0200 | [diff] [blame] | 99 | MAX_RECLAIM_VFS_PAGES_TIME_MSECS = 2 * 1000 * 60, |
Eli Cohen | dabed0e | 2013-09-11 16:35:28 +0300 | [diff] [blame] | 100 | }; |
| 101 | |
Eli Cohen | bf0bf77 | 2013-10-23 09:53:19 +0300 | [diff] [blame] | 102 | enum { |
| 103 | MLX5_MAX_RECLAIM_TIME_MILI = 5000, |
Eli Cohen | 05bdb2a | 2014-01-14 17:45:20 +0200 | [diff] [blame] | 104 | MLX5_NUM_4K_IN_PAGE = PAGE_SIZE / MLX5_ADAPTER_PAGE_SIZE, |
Eli Cohen | bf0bf77 | 2013-10-23 09:53:19 +0300 | [diff] [blame] | 105 | }; |
| 106 | |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 107 | static int insert_page(struct mlx5_core_dev *dev, u64 addr, struct page *page, u16 func_id) |
| 108 | { |
| 109 | struct rb_root *root = &dev->priv.page_root; |
| 110 | struct rb_node **new = &root->rb_node; |
| 111 | struct rb_node *parent = NULL; |
| 112 | struct fw_page *nfp; |
| 113 | struct fw_page *tfp; |
Eli Cohen | bf0bf77 | 2013-10-23 09:53:19 +0300 | [diff] [blame] | 114 | int i; |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 115 | |
| 116 | while (*new) { |
| 117 | parent = *new; |
| 118 | tfp = rb_entry(parent, struct fw_page, rb_node); |
| 119 | if (tfp->addr < addr) |
| 120 | new = &parent->rb_left; |
| 121 | else if (tfp->addr > addr) |
| 122 | new = &parent->rb_right; |
| 123 | else |
| 124 | return -EEXIST; |
| 125 | } |
| 126 | |
Eli Cohen | bf0bf77 | 2013-10-23 09:53:19 +0300 | [diff] [blame] | 127 | nfp = kzalloc(sizeof(*nfp), GFP_KERNEL); |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 128 | if (!nfp) |
| 129 | return -ENOMEM; |
| 130 | |
| 131 | nfp->addr = addr; |
| 132 | nfp->page = page; |
| 133 | nfp->func_id = func_id; |
Eli Cohen | bf0bf77 | 2013-10-23 09:53:19 +0300 | [diff] [blame] | 134 | nfp->free_count = MLX5_NUM_4K_IN_PAGE; |
| 135 | for (i = 0; i < MLX5_NUM_4K_IN_PAGE; i++) |
| 136 | set_bit(i, &nfp->bitmask); |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 137 | |
| 138 | rb_link_node(&nfp->rb_node, parent, new); |
| 139 | rb_insert_color(&nfp->rb_node, root); |
Eli Cohen | bf0bf77 | 2013-10-23 09:53:19 +0300 | [diff] [blame] | 140 | list_add(&nfp->list, &dev->priv.free_list); |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 141 | |
| 142 | return 0; |
| 143 | } |
| 144 | |
Eli Cohen | bf0bf77 | 2013-10-23 09:53:19 +0300 | [diff] [blame] | 145 | static struct fw_page *find_fw_page(struct mlx5_core_dev *dev, u64 addr) |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 146 | { |
| 147 | struct rb_root *root = &dev->priv.page_root; |
| 148 | struct rb_node *tmp = root->rb_node; |
Eli Cohen | bf0bf77 | 2013-10-23 09:53:19 +0300 | [diff] [blame] | 149 | struct fw_page *result = NULL; |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 150 | struct fw_page *tfp; |
| 151 | |
| 152 | while (tmp) { |
| 153 | tfp = rb_entry(tmp, struct fw_page, rb_node); |
| 154 | if (tfp->addr < addr) { |
| 155 | tmp = tmp->rb_left; |
| 156 | } else if (tfp->addr > addr) { |
| 157 | tmp = tmp->rb_right; |
| 158 | } else { |
Eli Cohen | bf0bf77 | 2013-10-23 09:53:19 +0300 | [diff] [blame] | 159 | result = tfp; |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 160 | break; |
| 161 | } |
| 162 | } |
| 163 | |
| 164 | return result; |
| 165 | } |
| 166 | |
| 167 | static int mlx5_cmd_query_pages(struct mlx5_core_dev *dev, u16 *func_id, |
Moshe Lazer | 0a324f31 | 2013-08-14 17:46:48 +0300 | [diff] [blame] | 168 | s32 *npages, int boot) |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 169 | { |
| 170 | struct mlx5_query_pages_inbox in; |
| 171 | struct mlx5_query_pages_outbox out; |
| 172 | int err; |
| 173 | |
| 174 | memset(&in, 0, sizeof(in)); |
| 175 | memset(&out, 0, sizeof(out)); |
| 176 | in.hdr.opcode = cpu_to_be16(MLX5_CMD_OP_QUERY_PAGES); |
Moshe Lazer | 0a324f31 | 2013-08-14 17:46:48 +0300 | [diff] [blame] | 177 | in.hdr.opmod = boot ? cpu_to_be16(MLX5_BOOT_PAGES) : cpu_to_be16(MLX5_INIT_PAGES); |
| 178 | |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 179 | err = mlx5_cmd_exec(dev, &in, sizeof(in), &out, sizeof(out)); |
| 180 | if (err) |
| 181 | return err; |
| 182 | |
| 183 | if (out.hdr.status) |
| 184 | return mlx5_cmd_status_to_err(&out.hdr); |
| 185 | |
Moshe Lazer | 0a324f31 | 2013-08-14 17:46:48 +0300 | [diff] [blame] | 186 | *npages = be32_to_cpu(out.num_pages); |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 187 | *func_id = be16_to_cpu(out.func_id); |
| 188 | |
| 189 | return err; |
| 190 | } |
| 191 | |
Eli Cohen | bf0bf77 | 2013-10-23 09:53:19 +0300 | [diff] [blame] | 192 | static int alloc_4k(struct mlx5_core_dev *dev, u64 *addr) |
| 193 | { |
| 194 | struct fw_page *fp; |
| 195 | unsigned n; |
| 196 | |
Dan Carpenter | 24e42754 | 2014-01-14 17:45:11 +0200 | [diff] [blame] | 197 | if (list_empty(&dev->priv.free_list)) |
Eli Cohen | bf0bf77 | 2013-10-23 09:53:19 +0300 | [diff] [blame] | 198 | return -ENOMEM; |
Eli Cohen | bf0bf77 | 2013-10-23 09:53:19 +0300 | [diff] [blame] | 199 | |
| 200 | fp = list_entry(dev->priv.free_list.next, struct fw_page, list); |
| 201 | n = find_first_bit(&fp->bitmask, 8 * sizeof(fp->bitmask)); |
| 202 | if (n >= MLX5_NUM_4K_IN_PAGE) { |
| 203 | mlx5_core_warn(dev, "alloc 4k bug\n"); |
| 204 | return -ENOENT; |
| 205 | } |
| 206 | clear_bit(n, &fp->bitmask); |
| 207 | fp->free_count--; |
| 208 | if (!fp->free_count) |
| 209 | list_del(&fp->list); |
| 210 | |
Eli Cohen | 05bdb2a | 2014-01-14 17:45:20 +0200 | [diff] [blame] | 211 | *addr = fp->addr + n * MLX5_ADAPTER_PAGE_SIZE; |
Eli Cohen | bf0bf77 | 2013-10-23 09:53:19 +0300 | [diff] [blame] | 212 | |
| 213 | return 0; |
| 214 | } |
| 215 | |
Honggang LI | 59d2d18 | 2015-04-15 16:36:15 +0800 | [diff] [blame] | 216 | #define MLX5_U64_4K_PAGE_MASK ((~(u64)0U) << PAGE_SHIFT) |
| 217 | |
Eli Cohen | bf0bf77 | 2013-10-23 09:53:19 +0300 | [diff] [blame] | 218 | static void free_4k(struct mlx5_core_dev *dev, u64 addr) |
| 219 | { |
| 220 | struct fw_page *fwp; |
| 221 | int n; |
| 222 | |
Honggang LI | 59d2d18 | 2015-04-15 16:36:15 +0800 | [diff] [blame] | 223 | fwp = find_fw_page(dev, addr & MLX5_U64_4K_PAGE_MASK); |
Eli Cohen | bf0bf77 | 2013-10-23 09:53:19 +0300 | [diff] [blame] | 224 | if (!fwp) { |
| 225 | mlx5_core_warn(dev, "page not found\n"); |
| 226 | return; |
| 227 | } |
| 228 | |
Honggang LI | 59d2d18 | 2015-04-15 16:36:15 +0800 | [diff] [blame] | 229 | n = (addr & ~MLX5_U64_4K_PAGE_MASK) >> MLX5_ADAPTER_PAGE_SHIFT; |
Eli Cohen | bf0bf77 | 2013-10-23 09:53:19 +0300 | [diff] [blame] | 230 | fwp->free_count++; |
| 231 | set_bit(n, &fwp->bitmask); |
| 232 | if (fwp->free_count == MLX5_NUM_4K_IN_PAGE) { |
| 233 | rb_erase(&fwp->rb_node, &dev->priv.page_root); |
Eli Cohen | 2b136d0 | 2013-10-31 15:26:33 +0200 | [diff] [blame] | 234 | if (fwp->free_count != 1) |
| 235 | list_del(&fwp->list); |
Honggang LI | 59d2d18 | 2015-04-15 16:36:15 +0800 | [diff] [blame] | 236 | dma_unmap_page(&dev->pdev->dev, addr & MLX5_U64_4K_PAGE_MASK, |
| 237 | PAGE_SIZE, DMA_BIDIRECTIONAL); |
Eli Cohen | bf0bf77 | 2013-10-23 09:53:19 +0300 | [diff] [blame] | 238 | __free_page(fwp->page); |
| 239 | kfree(fwp); |
| 240 | } else if (fwp->free_count == 1) { |
| 241 | list_add(&fwp->list, &dev->priv.free_list); |
| 242 | } |
| 243 | } |
| 244 | |
| 245 | static int alloc_system_page(struct mlx5_core_dev *dev, u16 func_id) |
| 246 | { |
| 247 | struct page *page; |
| 248 | u64 addr; |
| 249 | int err; |
Eli Cohen | ad18910 | 2015-04-02 17:07:19 +0300 | [diff] [blame] | 250 | int nid = dev_to_node(&dev->pdev->dev); |
Eli Cohen | bf0bf77 | 2013-10-23 09:53:19 +0300 | [diff] [blame] | 251 | |
Eli Cohen | ad18910 | 2015-04-02 17:07:19 +0300 | [diff] [blame] | 252 | page = alloc_pages_node(nid, GFP_HIGHUSER, 0); |
Eli Cohen | bf0bf77 | 2013-10-23 09:53:19 +0300 | [diff] [blame] | 253 | if (!page) { |
| 254 | mlx5_core_warn(dev, "failed to allocate page\n"); |
| 255 | return -ENOMEM; |
| 256 | } |
| 257 | addr = dma_map_page(&dev->pdev->dev, page, 0, |
| 258 | PAGE_SIZE, DMA_BIDIRECTIONAL); |
| 259 | if (dma_mapping_error(&dev->pdev->dev, addr)) { |
| 260 | mlx5_core_warn(dev, "failed dma mapping page\n"); |
| 261 | err = -ENOMEM; |
| 262 | goto out_alloc; |
| 263 | } |
| 264 | err = insert_page(dev, addr, page, func_id); |
| 265 | if (err) { |
| 266 | mlx5_core_err(dev, "failed to track allocated page\n"); |
| 267 | goto out_mapping; |
| 268 | } |
| 269 | |
| 270 | return 0; |
| 271 | |
| 272 | out_mapping: |
| 273 | dma_unmap_page(&dev->pdev->dev, addr, PAGE_SIZE, DMA_BIDIRECTIONAL); |
| 274 | |
| 275 | out_alloc: |
| 276 | __free_page(page); |
| 277 | |
| 278 | return err; |
| 279 | } |
Eli Cohen | a8ffe63 | 2015-09-25 10:49:13 +0300 | [diff] [blame] | 280 | |
| 281 | static void page_notify_fail(struct mlx5_core_dev *dev, u16 func_id) |
| 282 | { |
| 283 | struct mlx5_manage_pages_inbox *in; |
| 284 | struct mlx5_manage_pages_outbox out; |
| 285 | int err; |
| 286 | |
| 287 | in = kzalloc(sizeof(*in), GFP_KERNEL); |
| 288 | if (!in) |
| 289 | return; |
| 290 | |
| 291 | memset(&out, 0, sizeof(out)); |
| 292 | in->hdr.opcode = cpu_to_be16(MLX5_CMD_OP_MANAGE_PAGES); |
| 293 | in->hdr.opmod = cpu_to_be16(MLX5_PAGES_CANT_GIVE); |
| 294 | in->func_id = cpu_to_be16(func_id); |
| 295 | err = mlx5_cmd_exec(dev, in, sizeof(*in), &out, sizeof(out)); |
| 296 | if (!err) |
| 297 | err = mlx5_cmd_status_to_err(&out.hdr); |
| 298 | |
| 299 | if (err) |
| 300 | mlx5_core_warn(dev, "page notify failed\n"); |
| 301 | |
| 302 | kfree(in); |
| 303 | } |
| 304 | |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 305 | static int give_pages(struct mlx5_core_dev *dev, u16 func_id, int npages, |
| 306 | int notify_fail) |
| 307 | { |
| 308 | struct mlx5_manage_pages_inbox *in; |
| 309 | struct mlx5_manage_pages_outbox out; |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 310 | int inlen; |
| 311 | u64 addr; |
| 312 | int err; |
| 313 | int i; |
| 314 | |
| 315 | inlen = sizeof(*in) + npages * sizeof(in->pas[0]); |
| 316 | in = mlx5_vzalloc(inlen); |
| 317 | if (!in) { |
Eli Cohen | a8ffe63 | 2015-09-25 10:49:13 +0300 | [diff] [blame] | 318 | err = -ENOMEM; |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 319 | mlx5_core_warn(dev, "vzalloc failed %d\n", inlen); |
Eli Cohen | a8ffe63 | 2015-09-25 10:49:13 +0300 | [diff] [blame] | 320 | goto out_free; |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 321 | } |
| 322 | memset(&out, 0, sizeof(out)); |
| 323 | |
| 324 | for (i = 0; i < npages; i++) { |
Eli Cohen | bf0bf77 | 2013-10-23 09:53:19 +0300 | [diff] [blame] | 325 | retry: |
| 326 | err = alloc_4k(dev, &addr); |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 327 | if (err) { |
Eli Cohen | bf0bf77 | 2013-10-23 09:53:19 +0300 | [diff] [blame] | 328 | if (err == -ENOMEM) |
| 329 | err = alloc_system_page(dev, func_id); |
| 330 | if (err) |
| 331 | goto out_4k; |
| 332 | |
| 333 | goto retry; |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 334 | } |
| 335 | in->pas[i] = cpu_to_be64(addr); |
| 336 | } |
| 337 | |
| 338 | in->hdr.opcode = cpu_to_be16(MLX5_CMD_OP_MANAGE_PAGES); |
| 339 | in->hdr.opmod = cpu_to_be16(MLX5_PAGES_GIVE); |
| 340 | in->func_id = cpu_to_be16(func_id); |
Moshe Lazer | 0a324f31 | 2013-08-14 17:46:48 +0300 | [diff] [blame] | 341 | in->num_entries = cpu_to_be32(npages); |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 342 | err = mlx5_cmd_exec(dev, in, inlen, &out, sizeof(out)); |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 343 | if (err) { |
Joe Perches | 1a91de2 | 2014-05-07 12:52:57 -0700 | [diff] [blame] | 344 | mlx5_core_warn(dev, "func_id 0x%x, npages %d, err %d\n", |
| 345 | func_id, npages, err); |
Eli Cohen | a8ffe63 | 2015-09-25 10:49:13 +0300 | [diff] [blame] | 346 | goto out_4k; |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 347 | } |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 348 | |
Eli Cohen | a8ffe63 | 2015-09-25 10:49:13 +0300 | [diff] [blame] | 349 | err = mlx5_cmd_status_to_err(&out.hdr); |
| 350 | if (err) { |
| 351 | mlx5_core_warn(dev, "func_id 0x%x, npages %d, status %d\n", |
| 352 | func_id, npages, out.hdr.status); |
| 353 | goto out_4k; |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 354 | } |
| 355 | |
Eli Cohen | fc50db9 | 2015-12-01 18:03:09 +0200 | [diff] [blame] | 356 | dev->priv.fw_pages += npages; |
| 357 | if (func_id) |
| 358 | dev->priv.vfs_pages += npages; |
| 359 | |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 360 | mlx5_core_dbg(dev, "err %d\n", err); |
| 361 | |
Eli Cohen | a8ffe63 | 2015-09-25 10:49:13 +0300 | [diff] [blame] | 362 | kvfree(in); |
| 363 | return 0; |
Eli Cohen | 952f5f6 | 2013-10-23 09:53:18 +0300 | [diff] [blame] | 364 | |
Eli Cohen | bf0bf77 | 2013-10-23 09:53:19 +0300 | [diff] [blame] | 365 | out_4k: |
| 366 | for (i--; i >= 0; i--) |
| 367 | free_4k(dev, be64_to_cpu(in->pas[i])); |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 368 | out_free: |
Al Viro | 479163f | 2014-11-20 08:13:57 +0000 | [diff] [blame] | 369 | kvfree(in); |
Eli Cohen | a8ffe63 | 2015-09-25 10:49:13 +0300 | [diff] [blame] | 370 | if (notify_fail) |
| 371 | page_notify_fail(dev, func_id); |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 372 | return err; |
| 373 | } |
| 374 | |
Daniel Jurgens | 5adff6a | 2016-06-30 17:34:40 +0300 | [diff] [blame] | 375 | static int reclaim_pages_cmd(struct mlx5_core_dev *dev, |
| 376 | struct mlx5_manage_pages_inbox *in, int in_size, |
| 377 | struct mlx5_manage_pages_outbox *out, int out_size) |
| 378 | { |
| 379 | struct fw_page *fwp; |
| 380 | struct rb_node *p; |
| 381 | u32 npages; |
| 382 | u32 i = 0; |
| 383 | |
| 384 | if (dev->state != MLX5_DEVICE_STATE_INTERNAL_ERROR) |
| 385 | return mlx5_cmd_exec_check_status(dev, (u32 *)in, in_size, |
| 386 | (u32 *)out, out_size); |
| 387 | |
| 388 | npages = be32_to_cpu(in->num_entries); |
| 389 | |
| 390 | p = rb_first(&dev->priv.page_root); |
| 391 | while (p && i < npages) { |
| 392 | fwp = rb_entry(p, struct fw_page, rb_node); |
| 393 | out->pas[i] = cpu_to_be64(fwp->addr); |
| 394 | p = rb_next(p); |
| 395 | i++; |
| 396 | } |
| 397 | |
| 398 | out->num_entries = cpu_to_be32(i); |
| 399 | return 0; |
| 400 | } |
| 401 | |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 402 | static int reclaim_pages(struct mlx5_core_dev *dev, u32 func_id, int npages, |
| 403 | int *nclaimed) |
| 404 | { |
| 405 | struct mlx5_manage_pages_inbox in; |
| 406 | struct mlx5_manage_pages_outbox *out; |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 407 | int num_claimed; |
| 408 | int outlen; |
| 409 | u64 addr; |
| 410 | int err; |
| 411 | int i; |
| 412 | |
Eli Cohen | dabed0e | 2013-09-11 16:35:28 +0300 | [diff] [blame] | 413 | if (nclaimed) |
| 414 | *nclaimed = 0; |
| 415 | |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 416 | memset(&in, 0, sizeof(in)); |
| 417 | outlen = sizeof(*out) + npages * sizeof(out->pas[0]); |
| 418 | out = mlx5_vzalloc(outlen); |
| 419 | if (!out) |
| 420 | return -ENOMEM; |
| 421 | |
| 422 | in.hdr.opcode = cpu_to_be16(MLX5_CMD_OP_MANAGE_PAGES); |
| 423 | in.hdr.opmod = cpu_to_be16(MLX5_PAGES_TAKE); |
| 424 | in.func_id = cpu_to_be16(func_id); |
Moshe Lazer | 0a324f31 | 2013-08-14 17:46:48 +0300 | [diff] [blame] | 425 | in.num_entries = cpu_to_be32(npages); |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 426 | mlx5_core_dbg(dev, "npages %d, outlen %d\n", npages, outlen); |
Daniel Jurgens | 5adff6a | 2016-06-30 17:34:40 +0300 | [diff] [blame] | 427 | err = reclaim_pages_cmd(dev, &in, sizeof(in), out, outlen); |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 428 | if (err) { |
Daniel Jurgens | 5adff6a | 2016-06-30 17:34:40 +0300 | [diff] [blame] | 429 | mlx5_core_err(dev, "failed reclaiming pages: err %d\n", err); |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 430 | goto out_free; |
| 431 | } |
| 432 | |
Moshe Lazer | 0a324f31 | 2013-08-14 17:46:48 +0300 | [diff] [blame] | 433 | num_claimed = be32_to_cpu(out->num_entries); |
Eli Cohen | fc50db9 | 2015-12-01 18:03:09 +0200 | [diff] [blame] | 434 | if (num_claimed > npages) { |
| 435 | mlx5_core_warn(dev, "fw returned %d, driver asked %d => corruption\n", |
| 436 | num_claimed, npages); |
| 437 | err = -EINVAL; |
| 438 | goto out_free; |
| 439 | } |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 440 | |
| 441 | for (i = 0; i < num_claimed; i++) { |
| 442 | addr = be64_to_cpu(out->pas[i]); |
Eli Cohen | bf0bf77 | 2013-10-23 09:53:19 +0300 | [diff] [blame] | 443 | free_4k(dev, addr); |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 444 | } |
Daniel Jurgens | 5adff6a | 2016-06-30 17:34:40 +0300 | [diff] [blame] | 445 | |
| 446 | if (nclaimed) |
| 447 | *nclaimed = num_claimed; |
| 448 | |
Eli Cohen | fc50db9 | 2015-12-01 18:03:09 +0200 | [diff] [blame] | 449 | dev->priv.fw_pages -= num_claimed; |
| 450 | if (func_id) |
| 451 | dev->priv.vfs_pages -= num_claimed; |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 452 | |
| 453 | out_free: |
Al Viro | 479163f | 2014-11-20 08:13:57 +0000 | [diff] [blame] | 454 | kvfree(out); |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 455 | return err; |
| 456 | } |
| 457 | |
| 458 | static void pages_work_handler(struct work_struct *work) |
| 459 | { |
| 460 | struct mlx5_pages_req *req = container_of(work, struct mlx5_pages_req, work); |
| 461 | struct mlx5_core_dev *dev = req->dev; |
| 462 | int err = 0; |
| 463 | |
| 464 | if (req->npages < 0) |
| 465 | err = reclaim_pages(dev, req->func_id, -1 * req->npages, NULL); |
| 466 | else if (req->npages > 0) |
| 467 | err = give_pages(dev, req->func_id, req->npages, 1); |
| 468 | |
| 469 | if (err) |
Joe Perches | 1a91de2 | 2014-05-07 12:52:57 -0700 | [diff] [blame] | 470 | mlx5_core_warn(dev, "%s fail %d\n", |
| 471 | req->npages < 0 ? "reclaim" : "give", err); |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 472 | |
| 473 | kfree(req); |
| 474 | } |
| 475 | |
| 476 | void mlx5_core_req_pages_handler(struct mlx5_core_dev *dev, u16 func_id, |
Moshe Lazer | 0a324f31 | 2013-08-14 17:46:48 +0300 | [diff] [blame] | 477 | s32 npages) |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 478 | { |
| 479 | struct mlx5_pages_req *req; |
| 480 | |
| 481 | req = kzalloc(sizeof(*req), GFP_ATOMIC); |
| 482 | if (!req) { |
| 483 | mlx5_core_warn(dev, "failed to allocate pages request\n"); |
| 484 | return; |
| 485 | } |
| 486 | |
| 487 | req->dev = dev; |
| 488 | req->func_id = func_id; |
| 489 | req->npages = npages; |
| 490 | INIT_WORK(&req->work, pages_work_handler); |
| 491 | queue_work(dev->priv.pg_wq, &req->work); |
| 492 | } |
| 493 | |
Eli Cohen | cd23b14 | 2013-07-18 15:31:08 +0300 | [diff] [blame] | 494 | int mlx5_satisfy_startup_pages(struct mlx5_core_dev *dev, int boot) |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 495 | { |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 496 | u16 uninitialized_var(func_id); |
Moshe Lazer | 0a324f31 | 2013-08-14 17:46:48 +0300 | [diff] [blame] | 497 | s32 uninitialized_var(npages); |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 498 | int err; |
| 499 | |
Moshe Lazer | 0a324f31 | 2013-08-14 17:46:48 +0300 | [diff] [blame] | 500 | err = mlx5_cmd_query_pages(dev, &func_id, &npages, boot); |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 501 | if (err) |
| 502 | return err; |
| 503 | |
Moshe Lazer | 0a324f31 | 2013-08-14 17:46:48 +0300 | [diff] [blame] | 504 | mlx5_core_dbg(dev, "requested %d %s pages for func_id 0x%x\n", |
| 505 | npages, boot ? "boot" : "init", func_id); |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 506 | |
Moshe Lazer | 0a324f31 | 2013-08-14 17:46:48 +0300 | [diff] [blame] | 507 | return give_pages(dev, func_id, npages, 0); |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 508 | } |
| 509 | |
Moshe Lazer | 4e3d677 | 2013-10-23 09:53:21 +0300 | [diff] [blame] | 510 | enum { |
| 511 | MLX5_BLKS_FOR_RECLAIM_PAGES = 12 |
| 512 | }; |
| 513 | |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 514 | static int optimal_reclaimed_pages(void) |
| 515 | { |
| 516 | struct mlx5_cmd_prot_block *block; |
| 517 | struct mlx5_cmd_layout *lay; |
| 518 | int ret; |
| 519 | |
Moshe Lazer | 4e3d677 | 2013-10-23 09:53:21 +0300 | [diff] [blame] | 520 | ret = (sizeof(lay->out) + MLX5_BLKS_FOR_RECLAIM_PAGES * sizeof(block->data) - |
| 521 | sizeof(struct mlx5_manage_pages_outbox)) / |
| 522 | FIELD_SIZEOF(struct mlx5_manage_pages_outbox, pas[0]); |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 523 | |
| 524 | return ret; |
| 525 | } |
| 526 | |
| 527 | int mlx5_reclaim_startup_pages(struct mlx5_core_dev *dev) |
| 528 | { |
Eli Cohen | dabed0e | 2013-09-11 16:35:28 +0300 | [diff] [blame] | 529 | unsigned long end = jiffies + msecs_to_jiffies(MAX_RECLAIM_TIME_MSECS); |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 530 | struct fw_page *fwp; |
| 531 | struct rb_node *p; |
Eli Cohen | dabed0e | 2013-09-11 16:35:28 +0300 | [diff] [blame] | 532 | int nclaimed = 0; |
Majd Dibbiny | 89d44f0 | 2015-10-14 17:43:46 +0300 | [diff] [blame] | 533 | int err = 0; |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 534 | |
| 535 | do { |
| 536 | p = rb_first(&dev->priv.page_root); |
| 537 | if (p) { |
| 538 | fwp = rb_entry(p, struct fw_page, rb_node); |
Daniel Jurgens | 5adff6a | 2016-06-30 17:34:40 +0300 | [diff] [blame] | 539 | err = reclaim_pages(dev, fwp->func_id, |
| 540 | optimal_reclaimed_pages(), |
| 541 | &nclaimed); |
| 542 | |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 543 | if (err) { |
Joe Perches | 1a91de2 | 2014-05-07 12:52:57 -0700 | [diff] [blame] | 544 | mlx5_core_warn(dev, "failed reclaiming pages (%d)\n", |
| 545 | err); |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 546 | return err; |
| 547 | } |
Eli Cohen | dabed0e | 2013-09-11 16:35:28 +0300 | [diff] [blame] | 548 | if (nclaimed) |
| 549 | end = jiffies + msecs_to_jiffies(MAX_RECLAIM_TIME_MSECS); |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 550 | } |
| 551 | if (time_after(jiffies, end)) { |
| 552 | mlx5_core_warn(dev, "FW did not return all pages. giving up...\n"); |
| 553 | break; |
| 554 | } |
| 555 | } while (p); |
| 556 | |
Daniel Jurgens | 5adff6a | 2016-06-30 17:34:40 +0300 | [diff] [blame] | 557 | WARN(dev->priv.fw_pages, |
| 558 | "FW pages counter is %d after reclaiming all pages\n", |
| 559 | dev->priv.fw_pages); |
| 560 | WARN(dev->priv.vfs_pages, |
| 561 | "VFs FW pages counter is %d after reclaiming all pages\n", |
| 562 | dev->priv.vfs_pages); |
| 563 | |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 564 | return 0; |
| 565 | } |
| 566 | |
| 567 | void mlx5_pagealloc_init(struct mlx5_core_dev *dev) |
| 568 | { |
| 569 | dev->priv.page_root = RB_ROOT; |
Eli Cohen | bf0bf77 | 2013-10-23 09:53:19 +0300 | [diff] [blame] | 570 | INIT_LIST_HEAD(&dev->priv.free_list); |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 571 | } |
| 572 | |
| 573 | void mlx5_pagealloc_cleanup(struct mlx5_core_dev *dev) |
| 574 | { |
| 575 | /* nothing */ |
| 576 | } |
| 577 | |
| 578 | int mlx5_pagealloc_start(struct mlx5_core_dev *dev) |
| 579 | { |
| 580 | dev->priv.pg_wq = create_singlethread_workqueue("mlx5_page_allocator"); |
| 581 | if (!dev->priv.pg_wq) |
| 582 | return -ENOMEM; |
| 583 | |
| 584 | return 0; |
| 585 | } |
| 586 | |
| 587 | void mlx5_pagealloc_stop(struct mlx5_core_dev *dev) |
| 588 | { |
| 589 | destroy_workqueue(dev->priv.pg_wq); |
| 590 | } |
Eli Cohen | fc50db9 | 2015-12-01 18:03:09 +0200 | [diff] [blame] | 591 | |
| 592 | int mlx5_wait_for_vf_pages(struct mlx5_core_dev *dev) |
| 593 | { |
| 594 | unsigned long end = jiffies + msecs_to_jiffies(MAX_RECLAIM_VFS_PAGES_TIME_MSECS); |
| 595 | int prev_vfs_pages = dev->priv.vfs_pages; |
| 596 | |
| 597 | mlx5_core_dbg(dev, "Waiting for %d pages from %s\n", prev_vfs_pages, |
| 598 | dev->priv.name); |
| 599 | while (dev->priv.vfs_pages) { |
| 600 | if (time_after(jiffies, end)) { |
| 601 | mlx5_core_warn(dev, "aborting while there are %d pending pages\n", dev->priv.vfs_pages); |
| 602 | return -ETIMEDOUT; |
| 603 | } |
| 604 | if (dev->priv.vfs_pages < prev_vfs_pages) { |
| 605 | end = jiffies + msecs_to_jiffies(MAX_RECLAIM_VFS_PAGES_TIME_MSECS); |
| 606 | prev_vfs_pages = dev->priv.vfs_pages; |
| 607 | } |
| 608 | msleep(50); |
| 609 | } |
| 610 | |
| 611 | mlx5_core_dbg(dev, "All pages received from %s\n", dev->priv.name); |
| 612 | return 0; |
| 613 | } |