Maor Gottlieb | de8575e | 2015-12-10 17:12:40 +0200 | [diff] [blame] | 1 | /* |
| 2 | * Copyright (c) 2015, Mellanox Technologies. All rights reserved. |
| 3 | * |
| 4 | * This software is available to you under a choice of one of two |
| 5 | * licenses. You may choose to be licensed under the terms of the GNU |
| 6 | * General Public License (GPL) Version 2, available from the file |
| 7 | * COPYING in the main directory of this source tree, or the |
| 8 | * OpenIB.org BSD license below: |
| 9 | * |
| 10 | * Redistribution and use in source and binary forms, with or |
| 11 | * without modification, are permitted provided that the following |
| 12 | * conditions are met: |
| 13 | * |
| 14 | * - Redistributions of source code must retain the above |
| 15 | * copyright notice, this list of conditions and the following |
| 16 | * disclaimer. |
| 17 | * |
| 18 | * - Redistributions in binary form must reproduce the above |
| 19 | * copyright notice, this list of conditions and the following |
| 20 | * disclaimer in the documentation and/or other materials |
| 21 | * provided with the distribution. |
| 22 | * |
| 23 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, |
| 24 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF |
| 25 | * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND |
| 26 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS |
| 27 | * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN |
| 28 | * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN |
| 29 | * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE |
| 30 | * SOFTWARE. |
| 31 | */ |
| 32 | |
| 33 | #include <linux/mutex.h> |
| 34 | #include <linux/mlx5/driver.h> |
| 35 | |
| 36 | #include "mlx5_core.h" |
| 37 | #include "fs_core.h" |
Maor Gottlieb | 0c56b97 | 2015-12-10 17:12:42 +0200 | [diff] [blame] | 38 | #include "fs_cmd.h" |
| 39 | |
Maor Gottlieb | 2530236 | 2015-12-10 17:12:43 +0200 | [diff] [blame] | 40 | #define INIT_TREE_NODE_ARRAY_SIZE(...) (sizeof((struct init_tree_node[]){__VA_ARGS__}) /\ |
| 41 | sizeof(struct init_tree_node)) |
| 42 | |
Maor Gottlieb | 4cbdd30 | 2016-01-11 10:26:04 +0200 | [diff] [blame] | 43 | #define ADD_PRIO(num_prios_val, min_level_val, max_ft_val, caps_val,\ |
Maor Gottlieb | 8d40d16 | 2016-01-11 10:26:03 +0200 | [diff] [blame] | 44 | ...) {.type = FS_TYPE_PRIO,\ |
Maor Gottlieb | 2530236 | 2015-12-10 17:12:43 +0200 | [diff] [blame] | 45 | .min_ft_level = min_level_val,\ |
Maor Gottlieb | 2530236 | 2015-12-10 17:12:43 +0200 | [diff] [blame] | 46 | .max_ft = max_ft_val,\ |
Maor Gottlieb | 4cbdd30 | 2016-01-11 10:26:04 +0200 | [diff] [blame] | 47 | .num_leaf_prios = num_prios_val,\ |
Maor Gottlieb | 8d40d16 | 2016-01-11 10:26:03 +0200 | [diff] [blame] | 48 | .caps = caps_val,\ |
Maor Gottlieb | 2530236 | 2015-12-10 17:12:43 +0200 | [diff] [blame] | 49 | .children = (struct init_tree_node[]) {__VA_ARGS__},\ |
| 50 | .ar_size = INIT_TREE_NODE_ARRAY_SIZE(__VA_ARGS__) \ |
| 51 | } |
| 52 | |
Maor Gottlieb | 4cbdd30 | 2016-01-11 10:26:04 +0200 | [diff] [blame] | 53 | #define ADD_MULTIPLE_PRIO(num_prios_val, max_ft_val, ...)\ |
| 54 | ADD_PRIO(num_prios_val, 0, max_ft_val, {},\ |
| 55 | __VA_ARGS__)\ |
Maor Gottlieb | 2530236 | 2015-12-10 17:12:43 +0200 | [diff] [blame] | 56 | |
| 57 | #define ADD_NS(...) {.type = FS_TYPE_NAMESPACE,\ |
| 58 | .children = (struct init_tree_node[]) {__VA_ARGS__},\ |
| 59 | .ar_size = INIT_TREE_NODE_ARRAY_SIZE(__VA_ARGS__) \ |
| 60 | } |
| 61 | |
Maor Gottlieb | 8d40d16 | 2016-01-11 10:26:03 +0200 | [diff] [blame] | 62 | #define INIT_CAPS_ARRAY_SIZE(...) (sizeof((long[]){__VA_ARGS__}) /\ |
| 63 | sizeof(long)) |
| 64 | |
| 65 | #define FS_CAP(cap) (__mlx5_bit_off(flow_table_nic_cap, cap)) |
| 66 | |
| 67 | #define FS_REQUIRED_CAPS(...) {.arr_sz = INIT_CAPS_ARRAY_SIZE(__VA_ARGS__), \ |
| 68 | .caps = (long[]) {__VA_ARGS__} } |
| 69 | |
Maor Gottlieb | 4cbdd30 | 2016-01-11 10:26:04 +0200 | [diff] [blame] | 70 | #define LEFTOVERS_MAX_FT 1 |
| 71 | #define LEFTOVERS_NUM_PRIOS 1 |
| 72 | #define BY_PASS_PRIO_MAX_FT 1 |
| 73 | #define BY_PASS_MIN_LEVEL (KENREL_MIN_LEVEL + MLX5_BY_PASS_NUM_PRIOS +\ |
| 74 | LEFTOVERS_MAX_FT) |
| 75 | |
Maor Gottlieb | 2530236 | 2015-12-10 17:12:43 +0200 | [diff] [blame] | 76 | #define KERNEL_MAX_FT 2 |
Maor Gottlieb | 4cbdd30 | 2016-01-11 10:26:04 +0200 | [diff] [blame] | 77 | #define KERNEL_NUM_PRIOS 1 |
Maor Gottlieb | 2530236 | 2015-12-10 17:12:43 +0200 | [diff] [blame] | 78 | #define KENREL_MIN_LEVEL 2 |
Maor Gottlieb | 8d40d16 | 2016-01-11 10:26:03 +0200 | [diff] [blame] | 79 | |
| 80 | struct node_caps { |
| 81 | size_t arr_sz; |
| 82 | long *caps; |
| 83 | }; |
Maor Gottlieb | 2530236 | 2015-12-10 17:12:43 +0200 | [diff] [blame] | 84 | static struct init_tree_node { |
| 85 | enum fs_node_type type; |
| 86 | struct init_tree_node *children; |
| 87 | int ar_size; |
Maor Gottlieb | 8d40d16 | 2016-01-11 10:26:03 +0200 | [diff] [blame] | 88 | struct node_caps caps; |
Maor Gottlieb | 2530236 | 2015-12-10 17:12:43 +0200 | [diff] [blame] | 89 | int min_ft_level; |
Maor Gottlieb | 4cbdd30 | 2016-01-11 10:26:04 +0200 | [diff] [blame] | 90 | int num_leaf_prios; |
Maor Gottlieb | 2530236 | 2015-12-10 17:12:43 +0200 | [diff] [blame] | 91 | int prio; |
| 92 | int max_ft; |
Maor Gottlieb | 2530236 | 2015-12-10 17:12:43 +0200 | [diff] [blame] | 93 | } root_fs = { |
| 94 | .type = FS_TYPE_NAMESPACE, |
Maor Gottlieb | 4cbdd30 | 2016-01-11 10:26:04 +0200 | [diff] [blame] | 95 | .ar_size = 3, |
Maor Gottlieb | 2530236 | 2015-12-10 17:12:43 +0200 | [diff] [blame] | 96 | .children = (struct init_tree_node[]) { |
Maor Gottlieb | 4cbdd30 | 2016-01-11 10:26:04 +0200 | [diff] [blame] | 97 | ADD_PRIO(0, BY_PASS_MIN_LEVEL, 0, |
| 98 | FS_REQUIRED_CAPS(FS_CAP(flow_table_properties_nic_receive.flow_modify_en), |
| 99 | FS_CAP(flow_table_properties_nic_receive.modify_root), |
| 100 | FS_CAP(flow_table_properties_nic_receive.identified_miss_table_mode), |
| 101 | FS_CAP(flow_table_properties_nic_receive.flow_table_modify)), |
| 102 | ADD_NS(ADD_MULTIPLE_PRIO(MLX5_BY_PASS_NUM_PRIOS, BY_PASS_PRIO_MAX_FT))), |
| 103 | ADD_PRIO(0, KENREL_MIN_LEVEL, 0, {}, |
| 104 | ADD_NS(ADD_MULTIPLE_PRIO(KERNEL_NUM_PRIOS, KERNEL_MAX_FT))), |
| 105 | ADD_PRIO(0, BY_PASS_MIN_LEVEL, 0, |
| 106 | FS_REQUIRED_CAPS(FS_CAP(flow_table_properties_nic_receive.flow_modify_en), |
| 107 | FS_CAP(flow_table_properties_nic_receive.modify_root), |
| 108 | FS_CAP(flow_table_properties_nic_receive.identified_miss_table_mode), |
| 109 | FS_CAP(flow_table_properties_nic_receive.flow_table_modify)), |
| 110 | ADD_NS(ADD_MULTIPLE_PRIO(LEFTOVERS_NUM_PRIOS, LEFTOVERS_MAX_FT))), |
Maor Gottlieb | 2530236 | 2015-12-10 17:12:43 +0200 | [diff] [blame] | 111 | } |
| 112 | }; |
| 113 | |
Maor Gottlieb | f0d22d1 | 2016-01-11 10:25:57 +0200 | [diff] [blame] | 114 | enum fs_i_mutex_lock_class { |
| 115 | FS_MUTEX_GRANDPARENT, |
| 116 | FS_MUTEX_PARENT, |
| 117 | FS_MUTEX_CHILD |
| 118 | }; |
| 119 | |
Maor Gottlieb | 0c56b97 | 2015-12-10 17:12:42 +0200 | [diff] [blame] | 120 | static void del_rule(struct fs_node *node); |
| 121 | static void del_flow_table(struct fs_node *node); |
| 122 | static void del_flow_group(struct fs_node *node); |
| 123 | static void del_fte(struct fs_node *node); |
Maor Gottlieb | de8575e | 2015-12-10 17:12:40 +0200 | [diff] [blame] | 124 | |
| 125 | static void tree_init_node(struct fs_node *node, |
| 126 | unsigned int refcount, |
| 127 | void (*remove_func)(struct fs_node *)) |
| 128 | { |
| 129 | atomic_set(&node->refcount, refcount); |
| 130 | INIT_LIST_HEAD(&node->list); |
| 131 | INIT_LIST_HEAD(&node->children); |
| 132 | mutex_init(&node->lock); |
| 133 | node->remove_func = remove_func; |
| 134 | } |
| 135 | |
| 136 | static void tree_add_node(struct fs_node *node, struct fs_node *parent) |
| 137 | { |
| 138 | if (parent) |
| 139 | atomic_inc(&parent->refcount); |
| 140 | node->parent = parent; |
| 141 | |
| 142 | /* Parent is the root */ |
| 143 | if (!parent) |
| 144 | node->root = node; |
| 145 | else |
| 146 | node->root = parent->root; |
| 147 | } |
| 148 | |
| 149 | static void tree_get_node(struct fs_node *node) |
| 150 | { |
| 151 | atomic_inc(&node->refcount); |
| 152 | } |
| 153 | |
Maor Gottlieb | f0d22d1 | 2016-01-11 10:25:57 +0200 | [diff] [blame] | 154 | static void nested_lock_ref_node(struct fs_node *node, |
| 155 | enum fs_i_mutex_lock_class class) |
Maor Gottlieb | de8575e | 2015-12-10 17:12:40 +0200 | [diff] [blame] | 156 | { |
| 157 | if (node) { |
Maor Gottlieb | f0d22d1 | 2016-01-11 10:25:57 +0200 | [diff] [blame] | 158 | mutex_lock_nested(&node->lock, class); |
Maor Gottlieb | de8575e | 2015-12-10 17:12:40 +0200 | [diff] [blame] | 159 | atomic_inc(&node->refcount); |
| 160 | } |
| 161 | } |
| 162 | |
| 163 | static void lock_ref_node(struct fs_node *node) |
| 164 | { |
| 165 | if (node) { |
| 166 | mutex_lock(&node->lock); |
| 167 | atomic_inc(&node->refcount); |
| 168 | } |
| 169 | } |
| 170 | |
| 171 | static void unlock_ref_node(struct fs_node *node) |
| 172 | { |
| 173 | if (node) { |
| 174 | atomic_dec(&node->refcount); |
| 175 | mutex_unlock(&node->lock); |
| 176 | } |
| 177 | } |
| 178 | |
| 179 | static void tree_put_node(struct fs_node *node) |
| 180 | { |
| 181 | struct fs_node *parent_node = node->parent; |
| 182 | |
| 183 | lock_ref_node(parent_node); |
| 184 | if (atomic_dec_and_test(&node->refcount)) { |
| 185 | if (parent_node) |
| 186 | list_del_init(&node->list); |
| 187 | if (node->remove_func) |
| 188 | node->remove_func(node); |
| 189 | kfree(node); |
| 190 | node = NULL; |
| 191 | } |
| 192 | unlock_ref_node(parent_node); |
| 193 | if (!node && parent_node) |
| 194 | tree_put_node(parent_node); |
| 195 | } |
| 196 | |
| 197 | static int tree_remove_node(struct fs_node *node) |
| 198 | { |
| 199 | if (atomic_read(&node->refcount) > 1) |
| 200 | return -EPERM; |
| 201 | tree_put_node(node); |
| 202 | return 0; |
| 203 | } |
Maor Gottlieb | 5e1626c | 2015-12-10 17:12:41 +0200 | [diff] [blame] | 204 | |
| 205 | static struct fs_prio *find_prio(struct mlx5_flow_namespace *ns, |
| 206 | unsigned int prio) |
| 207 | { |
| 208 | struct fs_prio *iter_prio; |
| 209 | |
| 210 | fs_for_each_prio(iter_prio, ns) { |
| 211 | if (iter_prio->prio == prio) |
| 212 | return iter_prio; |
| 213 | } |
| 214 | |
| 215 | return NULL; |
| 216 | } |
| 217 | |
| 218 | static unsigned int find_next_free_level(struct fs_prio *prio) |
| 219 | { |
| 220 | if (!list_empty(&prio->node.children)) { |
| 221 | struct mlx5_flow_table *ft; |
| 222 | |
| 223 | ft = list_last_entry(&prio->node.children, |
| 224 | struct mlx5_flow_table, |
| 225 | node.list); |
| 226 | return ft->level + 1; |
| 227 | } |
| 228 | return prio->start_level; |
| 229 | } |
| 230 | |
| 231 | static bool masked_memcmp(void *mask, void *val1, void *val2, size_t size) |
| 232 | { |
| 233 | unsigned int i; |
| 234 | |
| 235 | for (i = 0; i < size; i++, mask++, val1++, val2++) |
| 236 | if ((*((u8 *)val1) & (*(u8 *)mask)) != |
| 237 | ((*(u8 *)val2) & (*(u8 *)mask))) |
| 238 | return false; |
| 239 | |
| 240 | return true; |
| 241 | } |
| 242 | |
| 243 | static bool compare_match_value(struct mlx5_flow_group_mask *mask, |
| 244 | void *fte_param1, void *fte_param2) |
| 245 | { |
| 246 | if (mask->match_criteria_enable & |
| 247 | 1 << MLX5_CREATE_FLOW_GROUP_IN_MATCH_CRITERIA_ENABLE_OUTER_HEADERS) { |
| 248 | void *fte_match1 = MLX5_ADDR_OF(fte_match_param, |
| 249 | fte_param1, outer_headers); |
| 250 | void *fte_match2 = MLX5_ADDR_OF(fte_match_param, |
| 251 | fte_param2, outer_headers); |
| 252 | void *fte_mask = MLX5_ADDR_OF(fte_match_param, |
| 253 | mask->match_criteria, outer_headers); |
| 254 | |
| 255 | if (!masked_memcmp(fte_mask, fte_match1, fte_match2, |
| 256 | MLX5_ST_SZ_BYTES(fte_match_set_lyr_2_4))) |
| 257 | return false; |
| 258 | } |
| 259 | |
| 260 | if (mask->match_criteria_enable & |
| 261 | 1 << MLX5_CREATE_FLOW_GROUP_IN_MATCH_CRITERIA_ENABLE_MISC_PARAMETERS) { |
| 262 | void *fte_match1 = MLX5_ADDR_OF(fte_match_param, |
| 263 | fte_param1, misc_parameters); |
| 264 | void *fte_match2 = MLX5_ADDR_OF(fte_match_param, |
| 265 | fte_param2, misc_parameters); |
| 266 | void *fte_mask = MLX5_ADDR_OF(fte_match_param, |
| 267 | mask->match_criteria, misc_parameters); |
| 268 | |
| 269 | if (!masked_memcmp(fte_mask, fte_match1, fte_match2, |
| 270 | MLX5_ST_SZ_BYTES(fte_match_set_misc))) |
| 271 | return false; |
| 272 | } |
| 273 | |
| 274 | if (mask->match_criteria_enable & |
| 275 | 1 << MLX5_CREATE_FLOW_GROUP_IN_MATCH_CRITERIA_ENABLE_INNER_HEADERS) { |
| 276 | void *fte_match1 = MLX5_ADDR_OF(fte_match_param, |
| 277 | fte_param1, inner_headers); |
| 278 | void *fte_match2 = MLX5_ADDR_OF(fte_match_param, |
| 279 | fte_param2, inner_headers); |
| 280 | void *fte_mask = MLX5_ADDR_OF(fte_match_param, |
| 281 | mask->match_criteria, inner_headers); |
| 282 | |
| 283 | if (!masked_memcmp(fte_mask, fte_match1, fte_match2, |
| 284 | MLX5_ST_SZ_BYTES(fte_match_set_lyr_2_4))) |
| 285 | return false; |
| 286 | } |
| 287 | return true; |
| 288 | } |
| 289 | |
| 290 | static bool compare_match_criteria(u8 match_criteria_enable1, |
| 291 | u8 match_criteria_enable2, |
| 292 | void *mask1, void *mask2) |
| 293 | { |
| 294 | return match_criteria_enable1 == match_criteria_enable2 && |
| 295 | !memcmp(mask1, mask2, MLX5_ST_SZ_BYTES(fte_match_param)); |
| 296 | } |
Maor Gottlieb | 0c56b97 | 2015-12-10 17:12:42 +0200 | [diff] [blame] | 297 | |
| 298 | static struct mlx5_flow_root_namespace *find_root(struct fs_node *node) |
| 299 | { |
| 300 | struct fs_node *root; |
| 301 | struct mlx5_flow_namespace *ns; |
| 302 | |
| 303 | root = node->root; |
| 304 | |
| 305 | if (WARN_ON(root->type != FS_TYPE_NAMESPACE)) { |
| 306 | pr_warn("mlx5: flow steering node is not in tree or garbaged\n"); |
| 307 | return NULL; |
| 308 | } |
| 309 | |
| 310 | ns = container_of(root, struct mlx5_flow_namespace, node); |
| 311 | return container_of(ns, struct mlx5_flow_root_namespace, ns); |
| 312 | } |
| 313 | |
| 314 | static inline struct mlx5_core_dev *get_dev(struct fs_node *node) |
| 315 | { |
| 316 | struct mlx5_flow_root_namespace *root = find_root(node); |
| 317 | |
| 318 | if (root) |
| 319 | return root->dev; |
| 320 | return NULL; |
| 321 | } |
| 322 | |
| 323 | static void del_flow_table(struct fs_node *node) |
| 324 | { |
| 325 | struct mlx5_flow_table *ft; |
| 326 | struct mlx5_core_dev *dev; |
| 327 | struct fs_prio *prio; |
| 328 | int err; |
| 329 | |
| 330 | fs_get_obj(ft, node); |
| 331 | dev = get_dev(&ft->node); |
| 332 | |
| 333 | err = mlx5_cmd_destroy_flow_table(dev, ft); |
| 334 | if (err) |
| 335 | pr_warn("flow steering can't destroy ft\n"); |
| 336 | fs_get_obj(prio, ft->node.parent); |
| 337 | prio->num_ft--; |
| 338 | } |
| 339 | |
| 340 | static void del_rule(struct fs_node *node) |
| 341 | { |
| 342 | struct mlx5_flow_rule *rule; |
| 343 | struct mlx5_flow_table *ft; |
| 344 | struct mlx5_flow_group *fg; |
| 345 | struct fs_fte *fte; |
| 346 | u32 *match_value; |
| 347 | struct mlx5_core_dev *dev = get_dev(node); |
| 348 | int match_len = MLX5_ST_SZ_BYTES(fte_match_param); |
| 349 | int err; |
| 350 | |
| 351 | match_value = mlx5_vzalloc(match_len); |
| 352 | if (!match_value) { |
| 353 | pr_warn("failed to allocate inbox\n"); |
| 354 | return; |
| 355 | } |
| 356 | |
| 357 | fs_get_obj(rule, node); |
| 358 | fs_get_obj(fte, rule->node.parent); |
| 359 | fs_get_obj(fg, fte->node.parent); |
| 360 | memcpy(match_value, fte->val, sizeof(fte->val)); |
| 361 | fs_get_obj(ft, fg->node.parent); |
| 362 | list_del(&rule->node.list); |
| 363 | fte->dests_size--; |
| 364 | if (fte->dests_size) { |
| 365 | err = mlx5_cmd_update_fte(dev, ft, |
| 366 | fg->id, fte); |
| 367 | if (err) |
| 368 | pr_warn("%s can't del rule fg id=%d fte_index=%d\n", |
| 369 | __func__, fg->id, fte->index); |
| 370 | } |
| 371 | kvfree(match_value); |
| 372 | } |
| 373 | |
| 374 | static void del_fte(struct fs_node *node) |
| 375 | { |
| 376 | struct mlx5_flow_table *ft; |
| 377 | struct mlx5_flow_group *fg; |
| 378 | struct mlx5_core_dev *dev; |
| 379 | struct fs_fte *fte; |
| 380 | int err; |
| 381 | |
| 382 | fs_get_obj(fte, node); |
| 383 | fs_get_obj(fg, fte->node.parent); |
| 384 | fs_get_obj(ft, fg->node.parent); |
| 385 | |
| 386 | dev = get_dev(&ft->node); |
| 387 | err = mlx5_cmd_delete_fte(dev, ft, |
| 388 | fte->index); |
| 389 | if (err) |
| 390 | pr_warn("flow steering can't delete fte in index %d of flow group id %d\n", |
| 391 | fte->index, fg->id); |
| 392 | |
| 393 | fte->status = 0; |
| 394 | fg->num_ftes--; |
| 395 | } |
| 396 | |
| 397 | static void del_flow_group(struct fs_node *node) |
| 398 | { |
| 399 | struct mlx5_flow_group *fg; |
| 400 | struct mlx5_flow_table *ft; |
| 401 | struct mlx5_core_dev *dev; |
| 402 | |
| 403 | fs_get_obj(fg, node); |
| 404 | fs_get_obj(ft, fg->node.parent); |
| 405 | dev = get_dev(&ft->node); |
| 406 | |
| 407 | if (mlx5_cmd_destroy_flow_group(dev, ft, fg->id)) |
| 408 | pr_warn("flow steering can't destroy fg %d of ft %d\n", |
| 409 | fg->id, ft->id); |
| 410 | } |
| 411 | |
| 412 | static struct fs_fte *alloc_fte(u8 action, |
| 413 | u32 flow_tag, |
| 414 | u32 *match_value, |
| 415 | unsigned int index) |
| 416 | { |
| 417 | struct fs_fte *fte; |
| 418 | |
| 419 | fte = kzalloc(sizeof(*fte), GFP_KERNEL); |
| 420 | if (!fte) |
| 421 | return ERR_PTR(-ENOMEM); |
| 422 | |
| 423 | memcpy(fte->val, match_value, sizeof(fte->val)); |
| 424 | fte->node.type = FS_TYPE_FLOW_ENTRY; |
| 425 | fte->flow_tag = flow_tag; |
| 426 | fte->index = index; |
| 427 | fte->action = action; |
| 428 | |
| 429 | return fte; |
| 430 | } |
| 431 | |
| 432 | static struct mlx5_flow_group *alloc_flow_group(u32 *create_fg_in) |
| 433 | { |
| 434 | struct mlx5_flow_group *fg; |
| 435 | void *match_criteria = MLX5_ADDR_OF(create_flow_group_in, |
| 436 | create_fg_in, match_criteria); |
| 437 | u8 match_criteria_enable = MLX5_GET(create_flow_group_in, |
| 438 | create_fg_in, |
| 439 | match_criteria_enable); |
| 440 | fg = kzalloc(sizeof(*fg), GFP_KERNEL); |
| 441 | if (!fg) |
| 442 | return ERR_PTR(-ENOMEM); |
| 443 | |
| 444 | fg->mask.match_criteria_enable = match_criteria_enable; |
| 445 | memcpy(&fg->mask.match_criteria, match_criteria, |
| 446 | sizeof(fg->mask.match_criteria)); |
| 447 | fg->node.type = FS_TYPE_FLOW_GROUP; |
| 448 | fg->start_index = MLX5_GET(create_flow_group_in, create_fg_in, |
| 449 | start_flow_index); |
| 450 | fg->max_ftes = MLX5_GET(create_flow_group_in, create_fg_in, |
| 451 | end_flow_index) - fg->start_index + 1; |
| 452 | return fg; |
| 453 | } |
| 454 | |
| 455 | static struct mlx5_flow_table *alloc_flow_table(int level, int max_fte, |
| 456 | enum fs_flow_table_type table_type) |
| 457 | { |
| 458 | struct mlx5_flow_table *ft; |
| 459 | |
| 460 | ft = kzalloc(sizeof(*ft), GFP_KERNEL); |
| 461 | if (!ft) |
| 462 | return NULL; |
| 463 | |
| 464 | ft->level = level; |
| 465 | ft->node.type = FS_TYPE_FLOW_TABLE; |
| 466 | ft->type = table_type; |
| 467 | ft->max_fte = max_fte; |
| 468 | |
| 469 | return ft; |
| 470 | } |
| 471 | |
Maor Gottlieb | fdb6896 | 2016-01-11 10:25:58 +0200 | [diff] [blame] | 472 | /* If reverse is false, then we search for the first flow table in the |
| 473 | * root sub-tree from start(closest from right), else we search for the |
| 474 | * last flow table in the root sub-tree till start(closest from left). |
| 475 | */ |
| 476 | static struct mlx5_flow_table *find_closest_ft_recursive(struct fs_node *root, |
| 477 | struct list_head *start, |
| 478 | bool reverse) |
| 479 | { |
| 480 | #define list_advance_entry(pos, reverse) \ |
| 481 | ((reverse) ? list_prev_entry(pos, list) : list_next_entry(pos, list)) |
| 482 | |
| 483 | #define list_for_each_advance_continue(pos, head, reverse) \ |
| 484 | for (pos = list_advance_entry(pos, reverse); \ |
| 485 | &pos->list != (head); \ |
| 486 | pos = list_advance_entry(pos, reverse)) |
| 487 | |
| 488 | struct fs_node *iter = list_entry(start, struct fs_node, list); |
| 489 | struct mlx5_flow_table *ft = NULL; |
| 490 | |
| 491 | if (!root) |
| 492 | return NULL; |
| 493 | |
| 494 | list_for_each_advance_continue(iter, &root->children, reverse) { |
| 495 | if (iter->type == FS_TYPE_FLOW_TABLE) { |
| 496 | fs_get_obj(ft, iter); |
| 497 | return ft; |
| 498 | } |
| 499 | ft = find_closest_ft_recursive(iter, &iter->children, reverse); |
| 500 | if (ft) |
| 501 | return ft; |
| 502 | } |
| 503 | |
| 504 | return ft; |
| 505 | } |
| 506 | |
| 507 | /* If reverse if false then return the first flow table in next priority of |
| 508 | * prio in the tree, else return the last flow table in the previous priority |
| 509 | * of prio in the tree. |
| 510 | */ |
| 511 | static struct mlx5_flow_table *find_closest_ft(struct fs_prio *prio, bool reverse) |
| 512 | { |
| 513 | struct mlx5_flow_table *ft = NULL; |
| 514 | struct fs_node *curr_node; |
| 515 | struct fs_node *parent; |
| 516 | |
| 517 | parent = prio->node.parent; |
| 518 | curr_node = &prio->node; |
| 519 | while (!ft && parent) { |
| 520 | ft = find_closest_ft_recursive(parent, &curr_node->list, reverse); |
| 521 | curr_node = parent; |
| 522 | parent = curr_node->parent; |
| 523 | } |
| 524 | return ft; |
| 525 | } |
| 526 | |
| 527 | /* Assuming all the tree is locked by mutex chain lock */ |
| 528 | static struct mlx5_flow_table *find_next_chained_ft(struct fs_prio *prio) |
| 529 | { |
| 530 | return find_closest_ft(prio, false); |
| 531 | } |
| 532 | |
| 533 | /* Assuming all the tree is locked by mutex chain lock */ |
| 534 | static struct mlx5_flow_table *find_prev_chained_ft(struct fs_prio *prio) |
| 535 | { |
| 536 | return find_closest_ft(prio, true); |
| 537 | } |
| 538 | |
Maor Gottlieb | f90edfd | 2016-01-11 10:26:01 +0200 | [diff] [blame] | 539 | static int connect_fts_in_prio(struct mlx5_core_dev *dev, |
| 540 | struct fs_prio *prio, |
| 541 | struct mlx5_flow_table *ft) |
| 542 | { |
| 543 | struct mlx5_flow_table *iter; |
| 544 | int i = 0; |
| 545 | int err; |
| 546 | |
| 547 | fs_for_each_ft(iter, prio) { |
| 548 | i++; |
| 549 | err = mlx5_cmd_modify_flow_table(dev, |
| 550 | iter, |
| 551 | ft); |
| 552 | if (err) { |
| 553 | mlx5_core_warn(dev, "Failed to modify flow table %d\n", |
| 554 | iter->id); |
| 555 | /* The driver is out of sync with the FW */ |
| 556 | if (i > 1) |
| 557 | WARN_ON(true); |
| 558 | return err; |
| 559 | } |
| 560 | } |
| 561 | return 0; |
| 562 | } |
| 563 | |
| 564 | /* Connect flow tables from previous priority of prio to ft */ |
| 565 | static int connect_prev_fts(struct mlx5_core_dev *dev, |
| 566 | struct mlx5_flow_table *ft, |
| 567 | struct fs_prio *prio) |
| 568 | { |
| 569 | struct mlx5_flow_table *prev_ft; |
| 570 | |
| 571 | prev_ft = find_prev_chained_ft(prio); |
| 572 | if (prev_ft) { |
| 573 | struct fs_prio *prev_prio; |
| 574 | |
| 575 | fs_get_obj(prev_prio, prev_ft->node.parent); |
| 576 | return connect_fts_in_prio(dev, prev_prio, ft); |
| 577 | } |
| 578 | return 0; |
| 579 | } |
| 580 | |
Maor Gottlieb | 2cc43b4 | 2016-01-11 10:25:59 +0200 | [diff] [blame] | 581 | static int update_root_ft_create(struct mlx5_flow_table *ft, struct fs_prio |
| 582 | *prio) |
| 583 | { |
| 584 | struct mlx5_flow_root_namespace *root = find_root(&prio->node); |
| 585 | int min_level = INT_MAX; |
| 586 | int err; |
| 587 | |
| 588 | if (root->root_ft) |
| 589 | min_level = root->root_ft->level; |
| 590 | |
| 591 | if (ft->level >= min_level) |
| 592 | return 0; |
| 593 | |
| 594 | err = mlx5_cmd_update_root_ft(root->dev, ft); |
| 595 | if (err) |
| 596 | mlx5_core_warn(root->dev, "Update root flow table of id=%u failed\n", |
| 597 | ft->id); |
| 598 | else |
| 599 | root->root_ft = ft; |
| 600 | |
| 601 | return err; |
| 602 | } |
| 603 | |
Maor Gottlieb | f90edfd | 2016-01-11 10:26:01 +0200 | [diff] [blame] | 604 | static int connect_flow_table(struct mlx5_core_dev *dev, struct mlx5_flow_table *ft, |
| 605 | struct fs_prio *prio) |
| 606 | { |
| 607 | int err = 0; |
| 608 | |
| 609 | /* Connect_prev_fts and update_root_ft_create are mutually exclusive */ |
| 610 | |
| 611 | if (list_empty(&prio->node.children)) { |
| 612 | err = connect_prev_fts(dev, ft, prio); |
| 613 | if (err) |
| 614 | return err; |
| 615 | } |
| 616 | |
| 617 | if (MLX5_CAP_FLOWTABLE(dev, |
| 618 | flow_table_properties_nic_receive.modify_root)) |
| 619 | err = update_root_ft_create(ft, prio); |
| 620 | return err; |
| 621 | } |
| 622 | |
Maor Gottlieb | 86d722a | 2015-12-10 17:12:44 +0200 | [diff] [blame] | 623 | struct mlx5_flow_table *mlx5_create_flow_table(struct mlx5_flow_namespace *ns, |
| 624 | int prio, |
| 625 | int max_fte) |
Maor Gottlieb | 0c56b97 | 2015-12-10 17:12:42 +0200 | [diff] [blame] | 626 | { |
Maor Gottlieb | f90edfd | 2016-01-11 10:26:01 +0200 | [diff] [blame] | 627 | struct mlx5_flow_table *next_ft = NULL; |
Maor Gottlieb | 0c56b97 | 2015-12-10 17:12:42 +0200 | [diff] [blame] | 628 | struct mlx5_flow_table *ft; |
| 629 | int err; |
| 630 | int log_table_sz; |
| 631 | struct mlx5_flow_root_namespace *root = |
| 632 | find_root(&ns->node); |
| 633 | struct fs_prio *fs_prio = NULL; |
| 634 | |
| 635 | if (!root) { |
| 636 | pr_err("mlx5: flow steering failed to find root of namespace\n"); |
| 637 | return ERR_PTR(-ENODEV); |
| 638 | } |
| 639 | |
Maor Gottlieb | 2cc43b4 | 2016-01-11 10:25:59 +0200 | [diff] [blame] | 640 | mutex_lock(&root->chain_lock); |
Maor Gottlieb | 0c56b97 | 2015-12-10 17:12:42 +0200 | [diff] [blame] | 641 | fs_prio = find_prio(ns, prio); |
Maor Gottlieb | 2cc43b4 | 2016-01-11 10:25:59 +0200 | [diff] [blame] | 642 | if (!fs_prio) { |
| 643 | err = -EINVAL; |
| 644 | goto unlock_root; |
| 645 | } |
Maor Gottlieb | 0c56b97 | 2015-12-10 17:12:42 +0200 | [diff] [blame] | 646 | if (fs_prio->num_ft == fs_prio->max_ft) { |
| 647 | err = -ENOSPC; |
Maor Gottlieb | 2cc43b4 | 2016-01-11 10:25:59 +0200 | [diff] [blame] | 648 | goto unlock_root; |
Maor Gottlieb | 0c56b97 | 2015-12-10 17:12:42 +0200 | [diff] [blame] | 649 | } |
| 650 | |
| 651 | ft = alloc_flow_table(find_next_free_level(fs_prio), |
| 652 | roundup_pow_of_two(max_fte), |
| 653 | root->table_type); |
| 654 | if (!ft) { |
| 655 | err = -ENOMEM; |
Maor Gottlieb | 2cc43b4 | 2016-01-11 10:25:59 +0200 | [diff] [blame] | 656 | goto unlock_root; |
Maor Gottlieb | 0c56b97 | 2015-12-10 17:12:42 +0200 | [diff] [blame] | 657 | } |
| 658 | |
| 659 | tree_init_node(&ft->node, 1, del_flow_table); |
| 660 | log_table_sz = ilog2(ft->max_fte); |
Maor Gottlieb | f90edfd | 2016-01-11 10:26:01 +0200 | [diff] [blame] | 661 | next_ft = find_next_chained_ft(fs_prio); |
Maor Gottlieb | 0c56b97 | 2015-12-10 17:12:42 +0200 | [diff] [blame] | 662 | err = mlx5_cmd_create_flow_table(root->dev, ft->type, ft->level, |
Maor Gottlieb | f90edfd | 2016-01-11 10:26:01 +0200 | [diff] [blame] | 663 | log_table_sz, next_ft, &ft->id); |
Maor Gottlieb | 0c56b97 | 2015-12-10 17:12:42 +0200 | [diff] [blame] | 664 | if (err) |
| 665 | goto free_ft; |
| 666 | |
Maor Gottlieb | f90edfd | 2016-01-11 10:26:01 +0200 | [diff] [blame] | 667 | err = connect_flow_table(root->dev, ft, fs_prio); |
| 668 | if (err) |
| 669 | goto destroy_ft; |
Maor Gottlieb | 2cc43b4 | 2016-01-11 10:25:59 +0200 | [diff] [blame] | 670 | lock_ref_node(&fs_prio->node); |
Maor Gottlieb | 0c56b97 | 2015-12-10 17:12:42 +0200 | [diff] [blame] | 671 | tree_add_node(&ft->node, &fs_prio->node); |
| 672 | list_add_tail(&ft->node.list, &fs_prio->node.children); |
| 673 | fs_prio->num_ft++; |
| 674 | unlock_ref_node(&fs_prio->node); |
Maor Gottlieb | 2cc43b4 | 2016-01-11 10:25:59 +0200 | [diff] [blame] | 675 | mutex_unlock(&root->chain_lock); |
Maor Gottlieb | 0c56b97 | 2015-12-10 17:12:42 +0200 | [diff] [blame] | 676 | return ft; |
Maor Gottlieb | 2cc43b4 | 2016-01-11 10:25:59 +0200 | [diff] [blame] | 677 | destroy_ft: |
| 678 | mlx5_cmd_destroy_flow_table(root->dev, ft); |
Maor Gottlieb | 0c56b97 | 2015-12-10 17:12:42 +0200 | [diff] [blame] | 679 | free_ft: |
| 680 | kfree(ft); |
Maor Gottlieb | 2cc43b4 | 2016-01-11 10:25:59 +0200 | [diff] [blame] | 681 | unlock_root: |
| 682 | mutex_unlock(&root->chain_lock); |
Maor Gottlieb | 0c56b97 | 2015-12-10 17:12:42 +0200 | [diff] [blame] | 683 | return ERR_PTR(err); |
| 684 | } |
| 685 | |
Maor Gottlieb | f0d22d1 | 2016-01-11 10:25:57 +0200 | [diff] [blame] | 686 | struct mlx5_flow_table *mlx5_create_auto_grouped_flow_table(struct mlx5_flow_namespace *ns, |
| 687 | int prio, |
| 688 | int num_flow_table_entries, |
| 689 | int max_num_groups) |
| 690 | { |
| 691 | struct mlx5_flow_table *ft; |
| 692 | |
| 693 | if (max_num_groups > num_flow_table_entries) |
| 694 | return ERR_PTR(-EINVAL); |
| 695 | |
| 696 | ft = mlx5_create_flow_table(ns, prio, num_flow_table_entries); |
| 697 | if (IS_ERR(ft)) |
| 698 | return ft; |
| 699 | |
| 700 | ft->autogroup.active = true; |
| 701 | ft->autogroup.required_groups = max_num_groups; |
| 702 | |
| 703 | return ft; |
| 704 | } |
Maor Gottlieb | b217ea2 | 2016-01-11 10:26:06 +0200 | [diff] [blame] | 705 | EXPORT_SYMBOL(mlx5_create_auto_grouped_flow_table); |
Maor Gottlieb | f0d22d1 | 2016-01-11 10:25:57 +0200 | [diff] [blame] | 706 | |
| 707 | /* Flow table should be locked */ |
| 708 | static struct mlx5_flow_group *create_flow_group_common(struct mlx5_flow_table *ft, |
| 709 | u32 *fg_in, |
| 710 | struct list_head |
| 711 | *prev_fg, |
| 712 | bool is_auto_fg) |
Maor Gottlieb | 0c56b97 | 2015-12-10 17:12:42 +0200 | [diff] [blame] | 713 | { |
| 714 | struct mlx5_flow_group *fg; |
| 715 | struct mlx5_core_dev *dev = get_dev(&ft->node); |
| 716 | int err; |
| 717 | |
| 718 | if (!dev) |
| 719 | return ERR_PTR(-ENODEV); |
| 720 | |
| 721 | fg = alloc_flow_group(fg_in); |
| 722 | if (IS_ERR(fg)) |
| 723 | return fg; |
| 724 | |
Maor Gottlieb | 0c56b97 | 2015-12-10 17:12:42 +0200 | [diff] [blame] | 725 | err = mlx5_cmd_create_flow_group(dev, ft, fg_in, &fg->id); |
| 726 | if (err) { |
| 727 | kfree(fg); |
Maor Gottlieb | 0c56b97 | 2015-12-10 17:12:42 +0200 | [diff] [blame] | 728 | return ERR_PTR(err); |
| 729 | } |
Maor Gottlieb | f0d22d1 | 2016-01-11 10:25:57 +0200 | [diff] [blame] | 730 | |
| 731 | if (ft->autogroup.active) |
| 732 | ft->autogroup.num_groups++; |
Maor Gottlieb | 0c56b97 | 2015-12-10 17:12:42 +0200 | [diff] [blame] | 733 | /* Add node to tree */ |
Maor Gottlieb | f0d22d1 | 2016-01-11 10:25:57 +0200 | [diff] [blame] | 734 | tree_init_node(&fg->node, !is_auto_fg, del_flow_group); |
Maor Gottlieb | 0c56b97 | 2015-12-10 17:12:42 +0200 | [diff] [blame] | 735 | tree_add_node(&fg->node, &ft->node); |
| 736 | /* Add node to group list */ |
| 737 | list_add(&fg->node.list, ft->node.children.prev); |
Maor Gottlieb | f0d22d1 | 2016-01-11 10:25:57 +0200 | [diff] [blame] | 738 | |
| 739 | return fg; |
| 740 | } |
| 741 | |
| 742 | struct mlx5_flow_group *mlx5_create_flow_group(struct mlx5_flow_table *ft, |
| 743 | u32 *fg_in) |
| 744 | { |
| 745 | struct mlx5_flow_group *fg; |
| 746 | |
| 747 | if (ft->autogroup.active) |
| 748 | return ERR_PTR(-EPERM); |
| 749 | |
| 750 | lock_ref_node(&ft->node); |
| 751 | fg = create_flow_group_common(ft, fg_in, &ft->node.children, false); |
Maor Gottlieb | 0c56b97 | 2015-12-10 17:12:42 +0200 | [diff] [blame] | 752 | unlock_ref_node(&ft->node); |
| 753 | |
| 754 | return fg; |
| 755 | } |
| 756 | |
| 757 | static struct mlx5_flow_rule *alloc_rule(struct mlx5_flow_destination *dest) |
| 758 | { |
| 759 | struct mlx5_flow_rule *rule; |
| 760 | |
| 761 | rule = kzalloc(sizeof(*rule), GFP_KERNEL); |
| 762 | if (!rule) |
| 763 | return NULL; |
| 764 | |
| 765 | rule->node.type = FS_TYPE_FLOW_DEST; |
| 766 | memcpy(&rule->dest_attr, dest, sizeof(*dest)); |
| 767 | |
| 768 | return rule; |
| 769 | } |
| 770 | |
| 771 | /* fte should not be deleted while calling this function */ |
| 772 | static struct mlx5_flow_rule *add_rule_fte(struct fs_fte *fte, |
| 773 | struct mlx5_flow_group *fg, |
| 774 | struct mlx5_flow_destination *dest) |
| 775 | { |
| 776 | struct mlx5_flow_table *ft; |
| 777 | struct mlx5_flow_rule *rule; |
| 778 | int err; |
| 779 | |
| 780 | rule = alloc_rule(dest); |
| 781 | if (!rule) |
| 782 | return ERR_PTR(-ENOMEM); |
| 783 | |
| 784 | fs_get_obj(ft, fg->node.parent); |
| 785 | /* Add dest to dests list- added as first element after the head */ |
| 786 | tree_init_node(&rule->node, 1, del_rule); |
| 787 | list_add_tail(&rule->node.list, &fte->node.children); |
| 788 | fte->dests_size++; |
| 789 | if (fte->dests_size == 1) |
| 790 | err = mlx5_cmd_create_fte(get_dev(&ft->node), |
| 791 | ft, fg->id, fte); |
| 792 | else |
| 793 | err = mlx5_cmd_update_fte(get_dev(&ft->node), |
| 794 | ft, fg->id, fte); |
| 795 | if (err) |
| 796 | goto free_rule; |
| 797 | |
| 798 | fte->status |= FS_FTE_STATUS_EXISTING; |
| 799 | |
| 800 | return rule; |
| 801 | |
| 802 | free_rule: |
| 803 | list_del(&rule->node.list); |
| 804 | kfree(rule); |
| 805 | fte->dests_size--; |
| 806 | return ERR_PTR(err); |
| 807 | } |
| 808 | |
| 809 | /* Assumed fg is locked */ |
| 810 | static unsigned int get_free_fte_index(struct mlx5_flow_group *fg, |
| 811 | struct list_head **prev) |
| 812 | { |
| 813 | struct fs_fte *fte; |
| 814 | unsigned int start = fg->start_index; |
| 815 | |
| 816 | if (prev) |
| 817 | *prev = &fg->node.children; |
| 818 | |
| 819 | /* assumed list is sorted by index */ |
| 820 | fs_for_each_fte(fte, fg) { |
| 821 | if (fte->index != start) |
| 822 | return start; |
| 823 | start++; |
| 824 | if (prev) |
| 825 | *prev = &fte->node.list; |
| 826 | } |
| 827 | |
| 828 | return start; |
| 829 | } |
| 830 | |
| 831 | /* prev is output, prev->next = new_fte */ |
| 832 | static struct fs_fte *create_fte(struct mlx5_flow_group *fg, |
| 833 | u32 *match_value, |
| 834 | u8 action, |
| 835 | u32 flow_tag, |
| 836 | struct list_head **prev) |
| 837 | { |
| 838 | struct fs_fte *fte; |
| 839 | int index; |
| 840 | |
| 841 | index = get_free_fte_index(fg, prev); |
| 842 | fte = alloc_fte(action, flow_tag, match_value, index); |
| 843 | if (IS_ERR(fte)) |
| 844 | return fte; |
| 845 | |
| 846 | return fte; |
| 847 | } |
| 848 | |
Maor Gottlieb | f0d22d1 | 2016-01-11 10:25:57 +0200 | [diff] [blame] | 849 | static struct mlx5_flow_group *create_autogroup(struct mlx5_flow_table *ft, |
| 850 | u8 match_criteria_enable, |
| 851 | u32 *match_criteria) |
| 852 | { |
| 853 | int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); |
| 854 | struct list_head *prev = &ft->node.children; |
| 855 | unsigned int candidate_index = 0; |
| 856 | struct mlx5_flow_group *fg; |
| 857 | void *match_criteria_addr; |
| 858 | unsigned int group_size = 0; |
| 859 | u32 *in; |
| 860 | |
| 861 | if (!ft->autogroup.active) |
| 862 | return ERR_PTR(-ENOENT); |
| 863 | |
| 864 | in = mlx5_vzalloc(inlen); |
| 865 | if (!in) |
| 866 | return ERR_PTR(-ENOMEM); |
| 867 | |
| 868 | if (ft->autogroup.num_groups < ft->autogroup.required_groups) |
| 869 | /* We save place for flow groups in addition to max types */ |
| 870 | group_size = ft->max_fte / (ft->autogroup.required_groups + 1); |
| 871 | |
| 872 | /* ft->max_fte == ft->autogroup.max_types */ |
| 873 | if (group_size == 0) |
| 874 | group_size = 1; |
| 875 | |
| 876 | /* sorted by start_index */ |
| 877 | fs_for_each_fg(fg, ft) { |
| 878 | if (candidate_index + group_size > fg->start_index) |
| 879 | candidate_index = fg->start_index + fg->max_ftes; |
| 880 | else |
| 881 | break; |
| 882 | prev = &fg->node.list; |
| 883 | } |
| 884 | |
| 885 | if (candidate_index + group_size > ft->max_fte) { |
| 886 | fg = ERR_PTR(-ENOSPC); |
| 887 | goto out; |
| 888 | } |
| 889 | |
| 890 | MLX5_SET(create_flow_group_in, in, match_criteria_enable, |
| 891 | match_criteria_enable); |
| 892 | MLX5_SET(create_flow_group_in, in, start_flow_index, candidate_index); |
| 893 | MLX5_SET(create_flow_group_in, in, end_flow_index, candidate_index + |
| 894 | group_size - 1); |
| 895 | match_criteria_addr = MLX5_ADDR_OF(create_flow_group_in, |
| 896 | in, match_criteria); |
| 897 | memcpy(match_criteria_addr, match_criteria, |
| 898 | MLX5_ST_SZ_BYTES(fte_match_param)); |
| 899 | |
| 900 | fg = create_flow_group_common(ft, in, prev, true); |
| 901 | out: |
| 902 | kvfree(in); |
| 903 | return fg; |
| 904 | } |
| 905 | |
Maor Gottlieb | 0c56b97 | 2015-12-10 17:12:42 +0200 | [diff] [blame] | 906 | static struct mlx5_flow_rule *add_rule_fg(struct mlx5_flow_group *fg, |
| 907 | u32 *match_value, |
| 908 | u8 action, |
| 909 | u32 flow_tag, |
| 910 | struct mlx5_flow_destination *dest) |
| 911 | { |
| 912 | struct fs_fte *fte; |
| 913 | struct mlx5_flow_rule *rule; |
| 914 | struct mlx5_flow_table *ft; |
| 915 | struct list_head *prev; |
| 916 | |
Maor Gottlieb | f0d22d1 | 2016-01-11 10:25:57 +0200 | [diff] [blame] | 917 | nested_lock_ref_node(&fg->node, FS_MUTEX_PARENT); |
Maor Gottlieb | 0c56b97 | 2015-12-10 17:12:42 +0200 | [diff] [blame] | 918 | fs_for_each_fte(fte, fg) { |
Maor Gottlieb | f0d22d1 | 2016-01-11 10:25:57 +0200 | [diff] [blame] | 919 | nested_lock_ref_node(&fte->node, FS_MUTEX_CHILD); |
Maor Gottlieb | 0c56b97 | 2015-12-10 17:12:42 +0200 | [diff] [blame] | 920 | if (compare_match_value(&fg->mask, match_value, &fte->val) && |
| 921 | action == fte->action && flow_tag == fte->flow_tag) { |
| 922 | rule = add_rule_fte(fte, fg, dest); |
| 923 | unlock_ref_node(&fte->node); |
| 924 | if (IS_ERR(rule)) |
| 925 | goto unlock_fg; |
| 926 | else |
| 927 | goto add_rule; |
| 928 | } |
| 929 | unlock_ref_node(&fte->node); |
| 930 | } |
| 931 | fs_get_obj(ft, fg->node.parent); |
| 932 | if (fg->num_ftes >= fg->max_ftes) { |
| 933 | rule = ERR_PTR(-ENOSPC); |
| 934 | goto unlock_fg; |
| 935 | } |
| 936 | |
| 937 | fte = create_fte(fg, match_value, action, flow_tag, &prev); |
| 938 | if (IS_ERR(fte)) { |
| 939 | rule = (void *)fte; |
| 940 | goto unlock_fg; |
| 941 | } |
| 942 | tree_init_node(&fte->node, 0, del_fte); |
| 943 | rule = add_rule_fte(fte, fg, dest); |
| 944 | if (IS_ERR(rule)) { |
| 945 | kfree(fte); |
| 946 | goto unlock_fg; |
| 947 | } |
| 948 | |
| 949 | fg->num_ftes++; |
| 950 | |
| 951 | tree_add_node(&fte->node, &fg->node); |
| 952 | list_add(&fte->node.list, prev); |
| 953 | add_rule: |
| 954 | tree_add_node(&rule->node, &fte->node); |
| 955 | unlock_fg: |
| 956 | unlock_ref_node(&fg->node); |
| 957 | return rule; |
| 958 | } |
| 959 | |
Maor Gottlieb | f0d22d1 | 2016-01-11 10:25:57 +0200 | [diff] [blame] | 960 | static struct mlx5_flow_rule *add_rule_to_auto_fg(struct mlx5_flow_table *ft, |
| 961 | u8 match_criteria_enable, |
| 962 | u32 *match_criteria, |
| 963 | u32 *match_value, |
| 964 | u8 action, |
| 965 | u32 flow_tag, |
| 966 | struct mlx5_flow_destination *dest) |
| 967 | { |
| 968 | struct mlx5_flow_rule *rule; |
| 969 | struct mlx5_flow_group *g; |
| 970 | |
| 971 | g = create_autogroup(ft, match_criteria_enable, match_criteria); |
| 972 | if (IS_ERR(g)) |
| 973 | return (void *)g; |
| 974 | |
| 975 | rule = add_rule_fg(g, match_value, |
| 976 | action, flow_tag, dest); |
| 977 | if (IS_ERR(rule)) { |
| 978 | /* Remove assumes refcount > 0 and autogroup creates a group |
| 979 | * with a refcount = 0. |
| 980 | */ |
| 981 | tree_get_node(&g->node); |
| 982 | tree_remove_node(&g->node); |
| 983 | } |
| 984 | return rule; |
| 985 | } |
| 986 | |
Maor Gottlieb | 86d722a | 2015-12-10 17:12:44 +0200 | [diff] [blame] | 987 | struct mlx5_flow_rule * |
Maor Gottlieb | 0c56b97 | 2015-12-10 17:12:42 +0200 | [diff] [blame] | 988 | mlx5_add_flow_rule(struct mlx5_flow_table *ft, |
| 989 | u8 match_criteria_enable, |
| 990 | u32 *match_criteria, |
| 991 | u32 *match_value, |
| 992 | u32 action, |
| 993 | u32 flow_tag, |
| 994 | struct mlx5_flow_destination *dest) |
| 995 | { |
| 996 | struct mlx5_flow_group *g; |
Maor Gottlieb | f0d22d1 | 2016-01-11 10:25:57 +0200 | [diff] [blame] | 997 | struct mlx5_flow_rule *rule; |
Maor Gottlieb | 0c56b97 | 2015-12-10 17:12:42 +0200 | [diff] [blame] | 998 | |
Maor Gottlieb | f0d22d1 | 2016-01-11 10:25:57 +0200 | [diff] [blame] | 999 | nested_lock_ref_node(&ft->node, FS_MUTEX_GRANDPARENT); |
Maor Gottlieb | 0c56b97 | 2015-12-10 17:12:42 +0200 | [diff] [blame] | 1000 | fs_for_each_fg(g, ft) |
| 1001 | if (compare_match_criteria(g->mask.match_criteria_enable, |
| 1002 | match_criteria_enable, |
| 1003 | g->mask.match_criteria, |
| 1004 | match_criteria)) { |
Maor Gottlieb | 0c56b97 | 2015-12-10 17:12:42 +0200 | [diff] [blame] | 1005 | rule = add_rule_fg(g, match_value, |
| 1006 | action, flow_tag, dest); |
Maor Gottlieb | f0d22d1 | 2016-01-11 10:25:57 +0200 | [diff] [blame] | 1007 | if (!IS_ERR(rule) || PTR_ERR(rule) != -ENOSPC) |
| 1008 | goto unlock; |
Maor Gottlieb | 0c56b97 | 2015-12-10 17:12:42 +0200 | [diff] [blame] | 1009 | } |
Maor Gottlieb | f0d22d1 | 2016-01-11 10:25:57 +0200 | [diff] [blame] | 1010 | |
| 1011 | rule = add_rule_to_auto_fg(ft, match_criteria_enable, match_criteria, |
| 1012 | match_value, action, flow_tag, dest); |
| 1013 | unlock: |
Maor Gottlieb | 0c56b97 | 2015-12-10 17:12:42 +0200 | [diff] [blame] | 1014 | unlock_ref_node(&ft->node); |
Maor Gottlieb | 0c56b97 | 2015-12-10 17:12:42 +0200 | [diff] [blame] | 1015 | return rule; |
| 1016 | } |
Maor Gottlieb | b217ea2 | 2016-01-11 10:26:06 +0200 | [diff] [blame] | 1017 | EXPORT_SYMBOL(mlx5_add_flow_rule); |
Maor Gottlieb | 0c56b97 | 2015-12-10 17:12:42 +0200 | [diff] [blame] | 1018 | |
Maor Gottlieb | 86d722a | 2015-12-10 17:12:44 +0200 | [diff] [blame] | 1019 | void mlx5_del_flow_rule(struct mlx5_flow_rule *rule) |
Maor Gottlieb | 0c56b97 | 2015-12-10 17:12:42 +0200 | [diff] [blame] | 1020 | { |
| 1021 | tree_remove_node(&rule->node); |
| 1022 | } |
Maor Gottlieb | b217ea2 | 2016-01-11 10:26:06 +0200 | [diff] [blame] | 1023 | EXPORT_SYMBOL(mlx5_del_flow_rule); |
Maor Gottlieb | 0c56b97 | 2015-12-10 17:12:42 +0200 | [diff] [blame] | 1024 | |
Maor Gottlieb | 2cc43b4 | 2016-01-11 10:25:59 +0200 | [diff] [blame] | 1025 | /* Assuming prio->node.children(flow tables) is sorted by level */ |
| 1026 | static struct mlx5_flow_table *find_next_ft(struct mlx5_flow_table *ft) |
| 1027 | { |
| 1028 | struct fs_prio *prio; |
| 1029 | |
| 1030 | fs_get_obj(prio, ft->node.parent); |
| 1031 | |
| 1032 | if (!list_is_last(&ft->node.list, &prio->node.children)) |
| 1033 | return list_next_entry(ft, node.list); |
| 1034 | return find_next_chained_ft(prio); |
| 1035 | } |
| 1036 | |
| 1037 | static int update_root_ft_destroy(struct mlx5_flow_table *ft) |
| 1038 | { |
| 1039 | struct mlx5_flow_root_namespace *root = find_root(&ft->node); |
| 1040 | struct mlx5_flow_table *new_root_ft = NULL; |
| 1041 | |
| 1042 | if (root->root_ft != ft) |
| 1043 | return 0; |
| 1044 | |
| 1045 | new_root_ft = find_next_ft(ft); |
| 1046 | if (new_root_ft) { |
| 1047 | int err = mlx5_cmd_update_root_ft(root->dev, new_root_ft); |
| 1048 | |
| 1049 | if (err) { |
| 1050 | mlx5_core_warn(root->dev, "Update root flow table of id=%u failed\n", |
| 1051 | ft->id); |
| 1052 | return err; |
| 1053 | } |
| 1054 | root->root_ft = new_root_ft; |
| 1055 | } |
| 1056 | return 0; |
| 1057 | } |
| 1058 | |
Maor Gottlieb | f90edfd | 2016-01-11 10:26:01 +0200 | [diff] [blame] | 1059 | /* Connect flow table from previous priority to |
| 1060 | * the next flow table. |
| 1061 | */ |
| 1062 | static int disconnect_flow_table(struct mlx5_flow_table *ft) |
| 1063 | { |
| 1064 | struct mlx5_core_dev *dev = get_dev(&ft->node); |
| 1065 | struct mlx5_flow_table *next_ft; |
| 1066 | struct fs_prio *prio; |
| 1067 | int err = 0; |
| 1068 | |
| 1069 | err = update_root_ft_destroy(ft); |
| 1070 | if (err) |
| 1071 | return err; |
| 1072 | |
| 1073 | fs_get_obj(prio, ft->node.parent); |
| 1074 | if (!(list_first_entry(&prio->node.children, |
| 1075 | struct mlx5_flow_table, |
| 1076 | node.list) == ft)) |
| 1077 | return 0; |
| 1078 | |
| 1079 | next_ft = find_next_chained_ft(prio); |
| 1080 | err = connect_prev_fts(dev, next_ft, prio); |
| 1081 | if (err) |
| 1082 | mlx5_core_warn(dev, "Failed to disconnect flow table %d\n", |
| 1083 | ft->id); |
| 1084 | return err; |
| 1085 | } |
| 1086 | |
Maor Gottlieb | 86d722a | 2015-12-10 17:12:44 +0200 | [diff] [blame] | 1087 | int mlx5_destroy_flow_table(struct mlx5_flow_table *ft) |
Maor Gottlieb | 0c56b97 | 2015-12-10 17:12:42 +0200 | [diff] [blame] | 1088 | { |
Maor Gottlieb | 2cc43b4 | 2016-01-11 10:25:59 +0200 | [diff] [blame] | 1089 | struct mlx5_flow_root_namespace *root = find_root(&ft->node); |
| 1090 | int err = 0; |
| 1091 | |
| 1092 | mutex_lock(&root->chain_lock); |
Maor Gottlieb | f90edfd | 2016-01-11 10:26:01 +0200 | [diff] [blame] | 1093 | err = disconnect_flow_table(ft); |
Maor Gottlieb | 2cc43b4 | 2016-01-11 10:25:59 +0200 | [diff] [blame] | 1094 | if (err) { |
| 1095 | mutex_unlock(&root->chain_lock); |
| 1096 | return err; |
| 1097 | } |
Maor Gottlieb | 0c56b97 | 2015-12-10 17:12:42 +0200 | [diff] [blame] | 1098 | if (tree_remove_node(&ft->node)) |
| 1099 | mlx5_core_warn(get_dev(&ft->node), "Flow table %d wasn't destroyed, refcount > 1\n", |
| 1100 | ft->id); |
Maor Gottlieb | 2cc43b4 | 2016-01-11 10:25:59 +0200 | [diff] [blame] | 1101 | mutex_unlock(&root->chain_lock); |
Maor Gottlieb | 0c56b97 | 2015-12-10 17:12:42 +0200 | [diff] [blame] | 1102 | |
Maor Gottlieb | 2cc43b4 | 2016-01-11 10:25:59 +0200 | [diff] [blame] | 1103 | return err; |
Maor Gottlieb | 0c56b97 | 2015-12-10 17:12:42 +0200 | [diff] [blame] | 1104 | } |
Maor Gottlieb | b217ea2 | 2016-01-11 10:26:06 +0200 | [diff] [blame] | 1105 | EXPORT_SYMBOL(mlx5_destroy_flow_table); |
Maor Gottlieb | 0c56b97 | 2015-12-10 17:12:42 +0200 | [diff] [blame] | 1106 | |
Maor Gottlieb | 86d722a | 2015-12-10 17:12:44 +0200 | [diff] [blame] | 1107 | void mlx5_destroy_flow_group(struct mlx5_flow_group *fg) |
Maor Gottlieb | 0c56b97 | 2015-12-10 17:12:42 +0200 | [diff] [blame] | 1108 | { |
| 1109 | if (tree_remove_node(&fg->node)) |
| 1110 | mlx5_core_warn(get_dev(&fg->node), "Flow group %d wasn't destroyed, refcount > 1\n", |
| 1111 | fg->id); |
| 1112 | } |
Maor Gottlieb | 2530236 | 2015-12-10 17:12:43 +0200 | [diff] [blame] | 1113 | |
Maor Gottlieb | 86d722a | 2015-12-10 17:12:44 +0200 | [diff] [blame] | 1114 | struct mlx5_flow_namespace *mlx5_get_flow_namespace(struct mlx5_core_dev *dev, |
| 1115 | enum mlx5_flow_namespace_type type) |
Maor Gottlieb | 2530236 | 2015-12-10 17:12:43 +0200 | [diff] [blame] | 1116 | { |
| 1117 | struct mlx5_flow_root_namespace *root_ns = dev->priv.root_ns; |
| 1118 | int prio; |
| 1119 | static struct fs_prio *fs_prio; |
| 1120 | struct mlx5_flow_namespace *ns; |
| 1121 | |
| 1122 | if (!root_ns) |
| 1123 | return NULL; |
| 1124 | |
| 1125 | switch (type) { |
Maor Gottlieb | 4cbdd30 | 2016-01-11 10:26:04 +0200 | [diff] [blame] | 1126 | case MLX5_FLOW_NAMESPACE_BYPASS: |
Maor Gottlieb | 2530236 | 2015-12-10 17:12:43 +0200 | [diff] [blame] | 1127 | case MLX5_FLOW_NAMESPACE_KERNEL: |
Maor Gottlieb | 4cbdd30 | 2016-01-11 10:26:04 +0200 | [diff] [blame] | 1128 | case MLX5_FLOW_NAMESPACE_LEFTOVERS: |
| 1129 | prio = type; |
Maor Gottlieb | 2530236 | 2015-12-10 17:12:43 +0200 | [diff] [blame] | 1130 | break; |
| 1131 | case MLX5_FLOW_NAMESPACE_FDB: |
| 1132 | if (dev->priv.fdb_root_ns) |
| 1133 | return &dev->priv.fdb_root_ns->ns; |
| 1134 | else |
| 1135 | return NULL; |
| 1136 | default: |
| 1137 | return NULL; |
| 1138 | } |
| 1139 | |
| 1140 | fs_prio = find_prio(&root_ns->ns, prio); |
| 1141 | if (!fs_prio) |
| 1142 | return NULL; |
| 1143 | |
| 1144 | ns = list_first_entry(&fs_prio->node.children, |
| 1145 | typeof(*ns), |
| 1146 | node.list); |
| 1147 | |
| 1148 | return ns; |
| 1149 | } |
Maor Gottlieb | b217ea2 | 2016-01-11 10:26:06 +0200 | [diff] [blame] | 1150 | EXPORT_SYMBOL(mlx5_get_flow_namespace); |
Maor Gottlieb | 2530236 | 2015-12-10 17:12:43 +0200 | [diff] [blame] | 1151 | |
| 1152 | static struct fs_prio *fs_create_prio(struct mlx5_flow_namespace *ns, |
Maor Gottlieb | 655227e | 2016-01-11 10:26:02 +0200 | [diff] [blame] | 1153 | unsigned prio, int max_ft) |
Maor Gottlieb | 2530236 | 2015-12-10 17:12:43 +0200 | [diff] [blame] | 1154 | { |
| 1155 | struct fs_prio *fs_prio; |
| 1156 | |
| 1157 | fs_prio = kzalloc(sizeof(*fs_prio), GFP_KERNEL); |
| 1158 | if (!fs_prio) |
| 1159 | return ERR_PTR(-ENOMEM); |
| 1160 | |
| 1161 | fs_prio->node.type = FS_TYPE_PRIO; |
| 1162 | tree_init_node(&fs_prio->node, 1, NULL); |
| 1163 | tree_add_node(&fs_prio->node, &ns->node); |
| 1164 | fs_prio->max_ft = max_ft; |
| 1165 | fs_prio->prio = prio; |
Maor Gottlieb | 2530236 | 2015-12-10 17:12:43 +0200 | [diff] [blame] | 1166 | list_add_tail(&fs_prio->node.list, &ns->node.children); |
| 1167 | |
| 1168 | return fs_prio; |
| 1169 | } |
| 1170 | |
| 1171 | static struct mlx5_flow_namespace *fs_init_namespace(struct mlx5_flow_namespace |
| 1172 | *ns) |
| 1173 | { |
| 1174 | ns->node.type = FS_TYPE_NAMESPACE; |
| 1175 | |
| 1176 | return ns; |
| 1177 | } |
| 1178 | |
| 1179 | static struct mlx5_flow_namespace *fs_create_namespace(struct fs_prio *prio) |
| 1180 | { |
| 1181 | struct mlx5_flow_namespace *ns; |
| 1182 | |
| 1183 | ns = kzalloc(sizeof(*ns), GFP_KERNEL); |
| 1184 | if (!ns) |
| 1185 | return ERR_PTR(-ENOMEM); |
| 1186 | |
| 1187 | fs_init_namespace(ns); |
| 1188 | tree_init_node(&ns->node, 1, NULL); |
| 1189 | tree_add_node(&ns->node, &prio->node); |
| 1190 | list_add_tail(&ns->node.list, &prio->node.children); |
| 1191 | |
| 1192 | return ns; |
| 1193 | } |
| 1194 | |
Maor Gottlieb | 4cbdd30 | 2016-01-11 10:26:04 +0200 | [diff] [blame] | 1195 | static int create_leaf_prios(struct mlx5_flow_namespace *ns, struct init_tree_node |
| 1196 | *prio_metadata) |
| 1197 | { |
| 1198 | struct fs_prio *fs_prio; |
| 1199 | int i; |
| 1200 | |
| 1201 | for (i = 0; i < prio_metadata->num_leaf_prios; i++) { |
| 1202 | fs_prio = fs_create_prio(ns, i, prio_metadata->max_ft); |
| 1203 | if (IS_ERR(fs_prio)) |
| 1204 | return PTR_ERR(fs_prio); |
| 1205 | } |
| 1206 | return 0; |
| 1207 | } |
| 1208 | |
Maor Gottlieb | 8d40d16 | 2016-01-11 10:26:03 +0200 | [diff] [blame] | 1209 | #define FLOW_TABLE_BIT_SZ 1 |
| 1210 | #define GET_FLOW_TABLE_CAP(dev, offset) \ |
| 1211 | ((be32_to_cpu(*((__be32 *)(dev->hca_caps_cur[MLX5_CAP_FLOW_TABLE]) + \ |
| 1212 | offset / 32)) >> \ |
| 1213 | (32 - FLOW_TABLE_BIT_SZ - (offset & 0x1f))) & FLOW_TABLE_BIT_SZ) |
| 1214 | static bool has_required_caps(struct mlx5_core_dev *dev, struct node_caps *caps) |
| 1215 | { |
| 1216 | int i; |
| 1217 | |
| 1218 | for (i = 0; i < caps->arr_sz; i++) { |
| 1219 | if (!GET_FLOW_TABLE_CAP(dev, caps->caps[i])) |
| 1220 | return false; |
| 1221 | } |
| 1222 | return true; |
| 1223 | } |
| 1224 | |
| 1225 | static int init_root_tree_recursive(struct mlx5_core_dev *dev, |
| 1226 | struct init_tree_node *init_node, |
Maor Gottlieb | 2530236 | 2015-12-10 17:12:43 +0200 | [diff] [blame] | 1227 | struct fs_node *fs_parent_node, |
| 1228 | struct init_tree_node *init_parent_node, |
| 1229 | int index) |
| 1230 | { |
Maor Gottlieb | 8d40d16 | 2016-01-11 10:26:03 +0200 | [diff] [blame] | 1231 | int max_ft_level = MLX5_CAP_FLOWTABLE(dev, |
| 1232 | flow_table_properties_nic_receive. |
| 1233 | max_ft_level); |
Maor Gottlieb | 2530236 | 2015-12-10 17:12:43 +0200 | [diff] [blame] | 1234 | struct mlx5_flow_namespace *fs_ns; |
| 1235 | struct fs_prio *fs_prio; |
| 1236 | struct fs_node *base; |
| 1237 | int i; |
| 1238 | int err; |
| 1239 | |
| 1240 | if (init_node->type == FS_TYPE_PRIO) { |
Maor Gottlieb | 8d40d16 | 2016-01-11 10:26:03 +0200 | [diff] [blame] | 1241 | if ((init_node->min_ft_level > max_ft_level) || |
| 1242 | !has_required_caps(dev, &init_node->caps)) |
| 1243 | return 0; |
Maor Gottlieb | 2530236 | 2015-12-10 17:12:43 +0200 | [diff] [blame] | 1244 | |
| 1245 | fs_get_obj(fs_ns, fs_parent_node); |
Maor Gottlieb | 4cbdd30 | 2016-01-11 10:26:04 +0200 | [diff] [blame] | 1246 | if (init_node->num_leaf_prios) |
| 1247 | return create_leaf_prios(fs_ns, init_node); |
Maor Gottlieb | 655227e | 2016-01-11 10:26:02 +0200 | [diff] [blame] | 1248 | fs_prio = fs_create_prio(fs_ns, index, init_node->max_ft); |
Maor Gottlieb | 2530236 | 2015-12-10 17:12:43 +0200 | [diff] [blame] | 1249 | if (IS_ERR(fs_prio)) |
| 1250 | return PTR_ERR(fs_prio); |
| 1251 | base = &fs_prio->node; |
| 1252 | } else if (init_node->type == FS_TYPE_NAMESPACE) { |
| 1253 | fs_get_obj(fs_prio, fs_parent_node); |
| 1254 | fs_ns = fs_create_namespace(fs_prio); |
| 1255 | if (IS_ERR(fs_ns)) |
| 1256 | return PTR_ERR(fs_ns); |
| 1257 | base = &fs_ns->node; |
| 1258 | } else { |
| 1259 | return -EINVAL; |
| 1260 | } |
| 1261 | for (i = 0; i < init_node->ar_size; i++) { |
Maor Gottlieb | 8d40d16 | 2016-01-11 10:26:03 +0200 | [diff] [blame] | 1262 | err = init_root_tree_recursive(dev, &init_node->children[i], |
| 1263 | base, init_node, i); |
Maor Gottlieb | 2530236 | 2015-12-10 17:12:43 +0200 | [diff] [blame] | 1264 | if (err) |
| 1265 | return err; |
| 1266 | } |
| 1267 | |
| 1268 | return 0; |
| 1269 | } |
| 1270 | |
Maor Gottlieb | 8d40d16 | 2016-01-11 10:26:03 +0200 | [diff] [blame] | 1271 | static int init_root_tree(struct mlx5_core_dev *dev, |
| 1272 | struct init_tree_node *init_node, |
Maor Gottlieb | 2530236 | 2015-12-10 17:12:43 +0200 | [diff] [blame] | 1273 | struct fs_node *fs_parent_node) |
| 1274 | { |
| 1275 | int i; |
| 1276 | struct mlx5_flow_namespace *fs_ns; |
| 1277 | int err; |
| 1278 | |
| 1279 | fs_get_obj(fs_ns, fs_parent_node); |
| 1280 | for (i = 0; i < init_node->ar_size; i++) { |
Maor Gottlieb | 8d40d16 | 2016-01-11 10:26:03 +0200 | [diff] [blame] | 1281 | err = init_root_tree_recursive(dev, &init_node->children[i], |
Maor Gottlieb | 2530236 | 2015-12-10 17:12:43 +0200 | [diff] [blame] | 1282 | &fs_ns->node, |
| 1283 | init_node, i); |
| 1284 | if (err) |
| 1285 | return err; |
| 1286 | } |
| 1287 | return 0; |
| 1288 | } |
| 1289 | |
| 1290 | static struct mlx5_flow_root_namespace *create_root_ns(struct mlx5_core_dev *dev, |
| 1291 | enum fs_flow_table_type |
| 1292 | table_type) |
| 1293 | { |
| 1294 | struct mlx5_flow_root_namespace *root_ns; |
| 1295 | struct mlx5_flow_namespace *ns; |
| 1296 | |
Maor Gottlieb | 86d722a | 2015-12-10 17:12:44 +0200 | [diff] [blame] | 1297 | /* Create the root namespace */ |
Maor Gottlieb | 2530236 | 2015-12-10 17:12:43 +0200 | [diff] [blame] | 1298 | root_ns = mlx5_vzalloc(sizeof(*root_ns)); |
| 1299 | if (!root_ns) |
| 1300 | return NULL; |
| 1301 | |
| 1302 | root_ns->dev = dev; |
| 1303 | root_ns->table_type = table_type; |
| 1304 | |
| 1305 | ns = &root_ns->ns; |
| 1306 | fs_init_namespace(ns); |
Maor Gottlieb | 2cc43b4 | 2016-01-11 10:25:59 +0200 | [diff] [blame] | 1307 | mutex_init(&root_ns->chain_lock); |
Maor Gottlieb | 2530236 | 2015-12-10 17:12:43 +0200 | [diff] [blame] | 1308 | tree_init_node(&ns->node, 1, NULL); |
| 1309 | tree_add_node(&ns->node, NULL); |
| 1310 | |
| 1311 | return root_ns; |
| 1312 | } |
| 1313 | |
Maor Gottlieb | 655227e | 2016-01-11 10:26:02 +0200 | [diff] [blame] | 1314 | static void set_prio_attrs_in_prio(struct fs_prio *prio, int acc_level); |
| 1315 | |
| 1316 | static int set_prio_attrs_in_ns(struct mlx5_flow_namespace *ns, int acc_level) |
| 1317 | { |
| 1318 | struct fs_prio *prio; |
| 1319 | |
| 1320 | fs_for_each_prio(prio, ns) { |
| 1321 | /* This updates prio start_level and max_ft */ |
| 1322 | set_prio_attrs_in_prio(prio, acc_level); |
| 1323 | acc_level += prio->max_ft; |
| 1324 | } |
| 1325 | return acc_level; |
| 1326 | } |
| 1327 | |
| 1328 | static void set_prio_attrs_in_prio(struct fs_prio *prio, int acc_level) |
| 1329 | { |
| 1330 | struct mlx5_flow_namespace *ns; |
| 1331 | int acc_level_ns = acc_level; |
| 1332 | |
| 1333 | prio->start_level = acc_level; |
| 1334 | fs_for_each_ns(ns, prio) |
| 1335 | /* This updates start_level and max_ft of ns's priority descendants */ |
| 1336 | acc_level_ns = set_prio_attrs_in_ns(ns, acc_level); |
| 1337 | if (!prio->max_ft) |
| 1338 | prio->max_ft = acc_level_ns - prio->start_level; |
| 1339 | WARN_ON(prio->max_ft < acc_level_ns - prio->start_level); |
| 1340 | } |
| 1341 | |
| 1342 | static void set_prio_attrs(struct mlx5_flow_root_namespace *root_ns) |
| 1343 | { |
| 1344 | struct mlx5_flow_namespace *ns = &root_ns->ns; |
| 1345 | struct fs_prio *prio; |
| 1346 | int start_level = 0; |
| 1347 | |
| 1348 | fs_for_each_prio(prio, ns) { |
| 1349 | set_prio_attrs_in_prio(prio, start_level); |
| 1350 | start_level += prio->max_ft; |
| 1351 | } |
| 1352 | } |
| 1353 | |
Maor Gottlieb | 2530236 | 2015-12-10 17:12:43 +0200 | [diff] [blame] | 1354 | static int init_root_ns(struct mlx5_core_dev *dev) |
| 1355 | { |
Maor Gottlieb | 2530236 | 2015-12-10 17:12:43 +0200 | [diff] [blame] | 1356 | |
| 1357 | dev->priv.root_ns = create_root_ns(dev, FS_FT_NIC_RX); |
| 1358 | if (IS_ERR_OR_NULL(dev->priv.root_ns)) |
| 1359 | goto cleanup; |
| 1360 | |
Maor Gottlieb | 8d40d16 | 2016-01-11 10:26:03 +0200 | [diff] [blame] | 1361 | if (init_root_tree(dev, &root_fs, &dev->priv.root_ns->ns.node)) |
Maor Gottlieb | 2530236 | 2015-12-10 17:12:43 +0200 | [diff] [blame] | 1362 | goto cleanup; |
| 1363 | |
Maor Gottlieb | 655227e | 2016-01-11 10:26:02 +0200 | [diff] [blame] | 1364 | set_prio_attrs(dev->priv.root_ns); |
| 1365 | |
Maor Gottlieb | 2530236 | 2015-12-10 17:12:43 +0200 | [diff] [blame] | 1366 | return 0; |
| 1367 | |
| 1368 | cleanup: |
| 1369 | mlx5_cleanup_fs(dev); |
| 1370 | return -ENOMEM; |
| 1371 | } |
| 1372 | |
| 1373 | static void cleanup_single_prio_root_ns(struct mlx5_core_dev *dev, |
| 1374 | struct mlx5_flow_root_namespace *root_ns) |
| 1375 | { |
| 1376 | struct fs_node *prio; |
| 1377 | |
| 1378 | if (!root_ns) |
| 1379 | return; |
| 1380 | |
| 1381 | if (!list_empty(&root_ns->ns.node.children)) { |
| 1382 | prio = list_first_entry(&root_ns->ns.node.children, |
| 1383 | struct fs_node, |
| 1384 | list); |
| 1385 | if (tree_remove_node(prio)) |
| 1386 | mlx5_core_warn(dev, |
| 1387 | "Flow steering priority wasn't destroyed, refcount > 1\n"); |
| 1388 | } |
| 1389 | if (tree_remove_node(&root_ns->ns.node)) |
| 1390 | mlx5_core_warn(dev, |
| 1391 | "Flow steering namespace wasn't destroyed, refcount > 1\n"); |
| 1392 | root_ns = NULL; |
| 1393 | } |
| 1394 | |
| 1395 | static void cleanup_root_ns(struct mlx5_core_dev *dev) |
| 1396 | { |
| 1397 | struct mlx5_flow_root_namespace *root_ns = dev->priv.root_ns; |
| 1398 | struct fs_prio *iter_prio; |
| 1399 | |
| 1400 | if (!MLX5_CAP_GEN(dev, nic_flow_table)) |
| 1401 | return; |
| 1402 | |
| 1403 | if (!root_ns) |
| 1404 | return; |
| 1405 | |
| 1406 | /* stage 1 */ |
| 1407 | fs_for_each_prio(iter_prio, &root_ns->ns) { |
| 1408 | struct fs_node *node; |
| 1409 | struct mlx5_flow_namespace *iter_ns; |
| 1410 | |
| 1411 | fs_for_each_ns_or_ft(node, iter_prio) { |
| 1412 | if (node->type == FS_TYPE_FLOW_TABLE) |
| 1413 | continue; |
| 1414 | fs_get_obj(iter_ns, node); |
| 1415 | while (!list_empty(&iter_ns->node.children)) { |
| 1416 | struct fs_prio *obj_iter_prio2; |
| 1417 | struct fs_node *iter_prio2 = |
| 1418 | list_first_entry(&iter_ns->node.children, |
| 1419 | struct fs_node, |
| 1420 | list); |
| 1421 | |
| 1422 | fs_get_obj(obj_iter_prio2, iter_prio2); |
| 1423 | if (tree_remove_node(iter_prio2)) { |
| 1424 | mlx5_core_warn(dev, |
| 1425 | "Priority %d wasn't destroyed, refcount > 1\n", |
| 1426 | obj_iter_prio2->prio); |
| 1427 | return; |
| 1428 | } |
| 1429 | } |
| 1430 | } |
| 1431 | } |
| 1432 | |
| 1433 | /* stage 2 */ |
| 1434 | fs_for_each_prio(iter_prio, &root_ns->ns) { |
| 1435 | while (!list_empty(&iter_prio->node.children)) { |
| 1436 | struct fs_node *iter_ns = |
| 1437 | list_first_entry(&iter_prio->node.children, |
| 1438 | struct fs_node, |
| 1439 | list); |
| 1440 | if (tree_remove_node(iter_ns)) { |
| 1441 | mlx5_core_warn(dev, |
| 1442 | "Namespace wasn't destroyed, refcount > 1\n"); |
| 1443 | return; |
| 1444 | } |
| 1445 | } |
| 1446 | } |
| 1447 | |
| 1448 | /* stage 3 */ |
| 1449 | while (!list_empty(&root_ns->ns.node.children)) { |
| 1450 | struct fs_prio *obj_prio_node; |
| 1451 | struct fs_node *prio_node = |
| 1452 | list_first_entry(&root_ns->ns.node.children, |
| 1453 | struct fs_node, |
| 1454 | list); |
| 1455 | |
| 1456 | fs_get_obj(obj_prio_node, prio_node); |
| 1457 | if (tree_remove_node(prio_node)) { |
| 1458 | mlx5_core_warn(dev, |
| 1459 | "Priority %d wasn't destroyed, refcount > 1\n", |
| 1460 | obj_prio_node->prio); |
| 1461 | return; |
| 1462 | } |
| 1463 | } |
| 1464 | |
| 1465 | if (tree_remove_node(&root_ns->ns.node)) { |
| 1466 | mlx5_core_warn(dev, |
| 1467 | "root namespace wasn't destroyed, refcount > 1\n"); |
| 1468 | return; |
| 1469 | } |
| 1470 | |
| 1471 | dev->priv.root_ns = NULL; |
| 1472 | } |
| 1473 | |
| 1474 | void mlx5_cleanup_fs(struct mlx5_core_dev *dev) |
| 1475 | { |
| 1476 | cleanup_root_ns(dev); |
| 1477 | cleanup_single_prio_root_ns(dev, dev->priv.fdb_root_ns); |
| 1478 | } |
| 1479 | |
| 1480 | static int init_fdb_root_ns(struct mlx5_core_dev *dev) |
| 1481 | { |
| 1482 | struct fs_prio *prio; |
| 1483 | |
| 1484 | dev->priv.fdb_root_ns = create_root_ns(dev, FS_FT_FDB); |
| 1485 | if (!dev->priv.fdb_root_ns) |
| 1486 | return -ENOMEM; |
| 1487 | |
Maor Gottlieb | 86d722a | 2015-12-10 17:12:44 +0200 | [diff] [blame] | 1488 | /* Create single prio */ |
Maor Gottlieb | 655227e | 2016-01-11 10:26:02 +0200 | [diff] [blame] | 1489 | prio = fs_create_prio(&dev->priv.fdb_root_ns->ns, 0, 1); |
Maor Gottlieb | 2530236 | 2015-12-10 17:12:43 +0200 | [diff] [blame] | 1490 | if (IS_ERR(prio)) { |
| 1491 | cleanup_single_prio_root_ns(dev, dev->priv.fdb_root_ns); |
| 1492 | return PTR_ERR(prio); |
| 1493 | } else { |
| 1494 | return 0; |
| 1495 | } |
| 1496 | } |
| 1497 | |
| 1498 | int mlx5_init_fs(struct mlx5_core_dev *dev) |
| 1499 | { |
| 1500 | int err = 0; |
| 1501 | |
| 1502 | if (MLX5_CAP_GEN(dev, nic_flow_table)) { |
| 1503 | err = init_root_ns(dev); |
| 1504 | if (err) |
| 1505 | return err; |
| 1506 | } |
| 1507 | if (MLX5_CAP_GEN(dev, eswitch_flow_table)) { |
| 1508 | err = init_fdb_root_ns(dev); |
| 1509 | if (err) |
| 1510 | cleanup_root_ns(dev); |
| 1511 | } |
| 1512 | |
| 1513 | return err; |
| 1514 | } |