Maor Gottlieb | 1cabe6b | 2016-04-29 01:36:40 +0300 | [diff] [blame] | 1 | /* |
| 2 | * Copyright (c) 2016, Mellanox Technologies. All rights reserved. |
| 3 | * |
| 4 | * This software is available to you under a choice of one of two |
| 5 | * licenses. You may choose to be licensed under the terms of the GNU |
| 6 | * General Public License (GPL) Version 2, available from the file |
| 7 | * COPYING in the main directory of this source tree, or the |
| 8 | * OpenIB.org BSD license below: |
| 9 | * |
| 10 | * Redistribution and use in source and binary forms, with or |
| 11 | * without modification, are permitted provided that the following |
| 12 | * conditions are met: |
| 13 | * |
| 14 | * - Redistributions of source code must retain the above |
| 15 | * copyright notice, this list of conditions and the following |
| 16 | * disclaimer. |
| 17 | * |
| 18 | * - Redistributions in binary form must reproduce the above |
| 19 | * copyright notice, this list of conditions and the following |
| 20 | * disclaimer in the documentation and/or other materials |
| 21 | * provided with the distribution. |
| 22 | * |
| 23 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, |
| 24 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF |
| 25 | * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND |
| 26 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS |
| 27 | * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN |
| 28 | * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN |
| 29 | * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE |
| 30 | * SOFTWARE. |
| 31 | */ |
| 32 | |
Maor Gottlieb | fbc4a69 | 2016-05-03 17:13:53 +0300 | [diff] [blame] | 33 | #ifdef CONFIG_RFS_ACCEL |
| 34 | |
Maor Gottlieb | 18c908e | 2016-04-29 01:36:41 +0300 | [diff] [blame] | 35 | #include <linux/hash.h> |
Maor Gottlieb | 1cabe6b | 2016-04-29 01:36:40 +0300 | [diff] [blame] | 36 | #include <linux/mlx5/fs.h> |
Maor Gottlieb | 18c908e | 2016-04-29 01:36:41 +0300 | [diff] [blame] | 37 | #include <linux/ip.h> |
| 38 | #include <linux/ipv6.h> |
| 39 | #include "en.h" |
| 40 | |
| 41 | struct arfs_tuple { |
| 42 | __be16 etype; |
| 43 | u8 ip_proto; |
| 44 | union { |
| 45 | __be32 src_ipv4; |
| 46 | struct in6_addr src_ipv6; |
| 47 | }; |
| 48 | union { |
| 49 | __be32 dst_ipv4; |
| 50 | struct in6_addr dst_ipv6; |
| 51 | }; |
| 52 | __be16 src_port; |
| 53 | __be16 dst_port; |
| 54 | }; |
| 55 | |
| 56 | struct arfs_rule { |
| 57 | struct mlx5e_priv *priv; |
| 58 | struct work_struct arfs_work; |
| 59 | struct mlx5_flow_rule *rule; |
| 60 | struct hlist_node hlist; |
| 61 | int rxq; |
| 62 | /* Flow ID passed to ndo_rx_flow_steer */ |
| 63 | int flow_id; |
| 64 | /* Filter ID returned by ndo_rx_flow_steer */ |
| 65 | int filter_id; |
| 66 | struct arfs_tuple tuple; |
| 67 | }; |
| 68 | |
| 69 | #define mlx5e_for_each_arfs_rule(hn, tmp, arfs_tables, i, j) \ |
| 70 | for (i = 0; i < ARFS_NUM_TYPES; i++) \ |
| 71 | mlx5e_for_each_hash_arfs_rule(hn, tmp, arfs_tables[i].rules_hash, j) |
| 72 | |
| 73 | #define mlx5e_for_each_hash_arfs_rule(hn, tmp, hash, j) \ |
| 74 | for (j = 0; j < ARFS_HASH_SIZE; j++) \ |
| 75 | hlist_for_each_entry_safe(hn, tmp, &hash[j], hlist) |
Maor Gottlieb | 1cabe6b | 2016-04-29 01:36:40 +0300 | [diff] [blame] | 76 | |
Maor Gottlieb | 45bf454a | 2016-04-29 01:36:42 +0300 | [diff] [blame] | 77 | static enum mlx5e_traffic_types arfs_get_tt(enum arfs_type type) |
| 78 | { |
| 79 | switch (type) { |
| 80 | case ARFS_IPV4_TCP: |
| 81 | return MLX5E_TT_IPV4_TCP; |
| 82 | case ARFS_IPV4_UDP: |
| 83 | return MLX5E_TT_IPV4_UDP; |
| 84 | case ARFS_IPV6_TCP: |
| 85 | return MLX5E_TT_IPV6_TCP; |
| 86 | case ARFS_IPV6_UDP: |
| 87 | return MLX5E_TT_IPV6_UDP; |
| 88 | default: |
| 89 | return -EINVAL; |
| 90 | } |
| 91 | } |
| 92 | |
| 93 | static int arfs_disable(struct mlx5e_priv *priv) |
| 94 | { |
| 95 | struct mlx5_flow_destination dest; |
Hadar Hen Zion | 724b2aa | 2016-07-01 14:51:05 +0300 | [diff] [blame] | 96 | struct mlx5e_tir *tir = priv->indir_tir; |
Maor Gottlieb | 45bf454a | 2016-04-29 01:36:42 +0300 | [diff] [blame] | 97 | int err = 0; |
| 98 | int tt; |
| 99 | int i; |
| 100 | |
| 101 | dest.type = MLX5_FLOW_DESTINATION_TYPE_TIR; |
| 102 | for (i = 0; i < ARFS_NUM_TYPES; i++) { |
Hadar Hen Zion | 724b2aa | 2016-07-01 14:51:05 +0300 | [diff] [blame] | 103 | dest.tir_num = tir[i].tirn; |
Maor Gottlieb | 45bf454a | 2016-04-29 01:36:42 +0300 | [diff] [blame] | 104 | tt = arfs_get_tt(i); |
| 105 | /* Modify ttc rules destination to bypass the aRFS tables*/ |
| 106 | err = mlx5_modify_rule_destination(priv->fs.ttc.rules[tt], |
| 107 | &dest); |
| 108 | if (err) { |
| 109 | netdev_err(priv->netdev, |
| 110 | "%s: modify ttc destination failed\n", |
| 111 | __func__); |
| 112 | return err; |
| 113 | } |
| 114 | } |
| 115 | return 0; |
| 116 | } |
| 117 | |
| 118 | static void arfs_del_rules(struct mlx5e_priv *priv); |
| 119 | |
| 120 | int mlx5e_arfs_disable(struct mlx5e_priv *priv) |
| 121 | { |
| 122 | arfs_del_rules(priv); |
| 123 | |
| 124 | return arfs_disable(priv); |
| 125 | } |
| 126 | |
| 127 | int mlx5e_arfs_enable(struct mlx5e_priv *priv) |
| 128 | { |
| 129 | struct mlx5_flow_destination dest; |
| 130 | int err = 0; |
| 131 | int tt; |
| 132 | int i; |
| 133 | |
| 134 | dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; |
| 135 | for (i = 0; i < ARFS_NUM_TYPES; i++) { |
| 136 | dest.ft = priv->fs.arfs.arfs_tables[i].ft.t; |
| 137 | tt = arfs_get_tt(i); |
| 138 | /* Modify ttc rules destination to point on the aRFS FTs */ |
| 139 | err = mlx5_modify_rule_destination(priv->fs.ttc.rules[tt], |
| 140 | &dest); |
| 141 | if (err) { |
| 142 | netdev_err(priv->netdev, |
| 143 | "%s: modify ttc destination failed err=%d\n", |
| 144 | __func__, err); |
| 145 | arfs_disable(priv); |
| 146 | return err; |
| 147 | } |
| 148 | } |
| 149 | return 0; |
| 150 | } |
| 151 | |
Maor Gottlieb | 1cabe6b | 2016-04-29 01:36:40 +0300 | [diff] [blame] | 152 | static void arfs_destroy_table(struct arfs_table *arfs_t) |
| 153 | { |
| 154 | mlx5_del_flow_rule(arfs_t->default_rule); |
| 155 | mlx5e_destroy_flow_table(&arfs_t->ft); |
| 156 | } |
| 157 | |
| 158 | void mlx5e_arfs_destroy_tables(struct mlx5e_priv *priv) |
| 159 | { |
| 160 | int i; |
| 161 | |
| 162 | if (!(priv->netdev->hw_features & NETIF_F_NTUPLE)) |
| 163 | return; |
Maor Gottlieb | 18c908e | 2016-04-29 01:36:41 +0300 | [diff] [blame] | 164 | |
| 165 | arfs_del_rules(priv); |
| 166 | destroy_workqueue(priv->fs.arfs.wq); |
Maor Gottlieb | 1cabe6b | 2016-04-29 01:36:40 +0300 | [diff] [blame] | 167 | for (i = 0; i < ARFS_NUM_TYPES; i++) { |
| 168 | if (!IS_ERR_OR_NULL(priv->fs.arfs.arfs_tables[i].ft.t)) |
| 169 | arfs_destroy_table(&priv->fs.arfs.arfs_tables[i]); |
| 170 | } |
| 171 | } |
| 172 | |
| 173 | static int arfs_add_default_rule(struct mlx5e_priv *priv, |
| 174 | enum arfs_type type) |
| 175 | { |
| 176 | struct arfs_table *arfs_t = &priv->fs.arfs.arfs_tables[type]; |
| 177 | struct mlx5_flow_destination dest; |
Hadar Hen Zion | 724b2aa | 2016-07-01 14:51:05 +0300 | [diff] [blame] | 178 | struct mlx5e_tir *tir = priv->indir_tir; |
Maor Gottlieb | c5bb173 | 2016-07-04 17:23:05 +0300 | [diff] [blame] | 179 | struct mlx5_flow_spec *spec; |
Maor Gottlieb | 1cabe6b | 2016-04-29 01:36:40 +0300 | [diff] [blame] | 180 | int err = 0; |
| 181 | |
Maor Gottlieb | c5bb173 | 2016-07-04 17:23:05 +0300 | [diff] [blame] | 182 | spec = mlx5_vzalloc(sizeof(*spec)); |
| 183 | if (!spec) { |
Maor Gottlieb | 1cabe6b | 2016-04-29 01:36:40 +0300 | [diff] [blame] | 184 | netdev_err(priv->netdev, "%s: alloc failed\n", __func__); |
| 185 | err = -ENOMEM; |
| 186 | goto out; |
| 187 | } |
| 188 | |
| 189 | dest.type = MLX5_FLOW_DESTINATION_TYPE_TIR; |
| 190 | switch (type) { |
| 191 | case ARFS_IPV4_TCP: |
Hadar Hen Zion | 724b2aa | 2016-07-01 14:51:05 +0300 | [diff] [blame] | 192 | dest.tir_num = tir[MLX5E_TT_IPV4_TCP].tirn; |
Maor Gottlieb | 1cabe6b | 2016-04-29 01:36:40 +0300 | [diff] [blame] | 193 | break; |
| 194 | case ARFS_IPV4_UDP: |
Hadar Hen Zion | 724b2aa | 2016-07-01 14:51:05 +0300 | [diff] [blame] | 195 | dest.tir_num = tir[MLX5E_TT_IPV4_UDP].tirn; |
Maor Gottlieb | 1cabe6b | 2016-04-29 01:36:40 +0300 | [diff] [blame] | 196 | break; |
| 197 | case ARFS_IPV6_TCP: |
Hadar Hen Zion | 724b2aa | 2016-07-01 14:51:05 +0300 | [diff] [blame] | 198 | dest.tir_num = tir[MLX5E_TT_IPV6_TCP].tirn; |
Maor Gottlieb | 1cabe6b | 2016-04-29 01:36:40 +0300 | [diff] [blame] | 199 | break; |
| 200 | case ARFS_IPV6_UDP: |
Hadar Hen Zion | 724b2aa | 2016-07-01 14:51:05 +0300 | [diff] [blame] | 201 | dest.tir_num = tir[MLX5E_TT_IPV6_UDP].tirn; |
Maor Gottlieb | 1cabe6b | 2016-04-29 01:36:40 +0300 | [diff] [blame] | 202 | break; |
| 203 | default: |
| 204 | err = -EINVAL; |
| 205 | goto out; |
| 206 | } |
| 207 | |
Maor Gottlieb | c5bb173 | 2016-07-04 17:23:05 +0300 | [diff] [blame] | 208 | arfs_t->default_rule = mlx5_add_flow_rule(arfs_t->ft.t, spec, |
Maor Gottlieb | 1cabe6b | 2016-04-29 01:36:40 +0300 | [diff] [blame] | 209 | MLX5_FLOW_CONTEXT_ACTION_FWD_DEST, |
| 210 | MLX5_FS_DEFAULT_FLOW_TAG, |
| 211 | &dest); |
| 212 | if (IS_ERR(arfs_t->default_rule)) { |
| 213 | err = PTR_ERR(arfs_t->default_rule); |
| 214 | arfs_t->default_rule = NULL; |
| 215 | netdev_err(priv->netdev, "%s: add rule failed, arfs type=%d\n", |
| 216 | __func__, type); |
| 217 | } |
| 218 | out: |
Maor Gottlieb | c5bb173 | 2016-07-04 17:23:05 +0300 | [diff] [blame] | 219 | kvfree(spec); |
Maor Gottlieb | 1cabe6b | 2016-04-29 01:36:40 +0300 | [diff] [blame] | 220 | return err; |
| 221 | } |
| 222 | |
| 223 | #define MLX5E_ARFS_NUM_GROUPS 2 |
| 224 | #define MLX5E_ARFS_GROUP1_SIZE BIT(12) |
| 225 | #define MLX5E_ARFS_GROUP2_SIZE BIT(0) |
| 226 | #define MLX5E_ARFS_TABLE_SIZE (MLX5E_ARFS_GROUP1_SIZE +\ |
| 227 | MLX5E_ARFS_GROUP2_SIZE) |
| 228 | static int arfs_create_groups(struct mlx5e_flow_table *ft, |
| 229 | enum arfs_type type) |
| 230 | { |
| 231 | int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); |
| 232 | void *outer_headers_c; |
| 233 | int ix = 0; |
| 234 | u32 *in; |
| 235 | int err; |
| 236 | u8 *mc; |
| 237 | |
| 238 | ft->g = kcalloc(MLX5E_ARFS_NUM_GROUPS, |
| 239 | sizeof(*ft->g), GFP_KERNEL); |
| 240 | in = mlx5_vzalloc(inlen); |
| 241 | if (!in || !ft->g) { |
| 242 | kvfree(ft->g); |
| 243 | kvfree(in); |
| 244 | return -ENOMEM; |
| 245 | } |
| 246 | |
| 247 | mc = MLX5_ADDR_OF(create_flow_group_in, in, match_criteria); |
| 248 | outer_headers_c = MLX5_ADDR_OF(fte_match_param, mc, |
| 249 | outer_headers); |
| 250 | MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, ethertype); |
| 251 | switch (type) { |
| 252 | case ARFS_IPV4_TCP: |
| 253 | case ARFS_IPV6_TCP: |
| 254 | MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, tcp_dport); |
| 255 | MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, tcp_sport); |
| 256 | break; |
| 257 | case ARFS_IPV4_UDP: |
| 258 | case ARFS_IPV6_UDP: |
| 259 | MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, udp_dport); |
| 260 | MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, udp_sport); |
| 261 | break; |
| 262 | default: |
| 263 | err = -EINVAL; |
| 264 | goto out; |
| 265 | } |
| 266 | |
| 267 | switch (type) { |
| 268 | case ARFS_IPV4_TCP: |
| 269 | case ARFS_IPV4_UDP: |
| 270 | MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, |
| 271 | src_ipv4_src_ipv6.ipv4_layout.ipv4); |
| 272 | MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, |
| 273 | dst_ipv4_dst_ipv6.ipv4_layout.ipv4); |
| 274 | break; |
| 275 | case ARFS_IPV6_TCP: |
| 276 | case ARFS_IPV6_UDP: |
| 277 | memset(MLX5_ADDR_OF(fte_match_set_lyr_2_4, outer_headers_c, |
| 278 | src_ipv4_src_ipv6.ipv6_layout.ipv6), |
| 279 | 0xff, 16); |
| 280 | memset(MLX5_ADDR_OF(fte_match_set_lyr_2_4, outer_headers_c, |
| 281 | dst_ipv4_dst_ipv6.ipv6_layout.ipv6), |
| 282 | 0xff, 16); |
| 283 | break; |
| 284 | default: |
| 285 | err = -EINVAL; |
| 286 | goto out; |
| 287 | } |
| 288 | |
| 289 | MLX5_SET_CFG(in, match_criteria_enable, MLX5_MATCH_OUTER_HEADERS); |
| 290 | MLX5_SET_CFG(in, start_flow_index, ix); |
| 291 | ix += MLX5E_ARFS_GROUP1_SIZE; |
| 292 | MLX5_SET_CFG(in, end_flow_index, ix - 1); |
| 293 | ft->g[ft->num_groups] = mlx5_create_flow_group(ft->t, in); |
| 294 | if (IS_ERR(ft->g[ft->num_groups])) |
| 295 | goto err; |
| 296 | ft->num_groups++; |
| 297 | |
| 298 | memset(in, 0, inlen); |
| 299 | MLX5_SET_CFG(in, start_flow_index, ix); |
| 300 | ix += MLX5E_ARFS_GROUP2_SIZE; |
| 301 | MLX5_SET_CFG(in, end_flow_index, ix - 1); |
| 302 | ft->g[ft->num_groups] = mlx5_create_flow_group(ft->t, in); |
| 303 | if (IS_ERR(ft->g[ft->num_groups])) |
| 304 | goto err; |
| 305 | ft->num_groups++; |
| 306 | |
| 307 | kvfree(in); |
| 308 | return 0; |
| 309 | |
| 310 | err: |
| 311 | err = PTR_ERR(ft->g[ft->num_groups]); |
| 312 | ft->g[ft->num_groups] = NULL; |
| 313 | out: |
| 314 | kvfree(in); |
| 315 | |
| 316 | return err; |
| 317 | } |
| 318 | |
| 319 | static int arfs_create_table(struct mlx5e_priv *priv, |
| 320 | enum arfs_type type) |
| 321 | { |
| 322 | struct mlx5e_arfs_tables *arfs = &priv->fs.arfs; |
| 323 | struct mlx5e_flow_table *ft = &arfs->arfs_tables[type].ft; |
| 324 | int err; |
| 325 | |
| 326 | ft->t = mlx5_create_flow_table(priv->fs.ns, MLX5E_NIC_PRIO, |
| 327 | MLX5E_ARFS_TABLE_SIZE, MLX5E_ARFS_FT_LEVEL); |
| 328 | if (IS_ERR(ft->t)) { |
| 329 | err = PTR_ERR(ft->t); |
| 330 | ft->t = NULL; |
| 331 | return err; |
| 332 | } |
| 333 | |
| 334 | err = arfs_create_groups(ft, type); |
| 335 | if (err) |
| 336 | goto err; |
| 337 | |
| 338 | err = arfs_add_default_rule(priv, type); |
| 339 | if (err) |
| 340 | goto err; |
| 341 | |
| 342 | return 0; |
| 343 | err: |
| 344 | mlx5e_destroy_flow_table(ft); |
| 345 | return err; |
| 346 | } |
| 347 | |
| 348 | int mlx5e_arfs_create_tables(struct mlx5e_priv *priv) |
| 349 | { |
| 350 | int err = 0; |
| 351 | int i; |
| 352 | |
| 353 | if (!(priv->netdev->hw_features & NETIF_F_NTUPLE)) |
| 354 | return 0; |
| 355 | |
Maor Gottlieb | 18c908e | 2016-04-29 01:36:41 +0300 | [diff] [blame] | 356 | spin_lock_init(&priv->fs.arfs.arfs_lock); |
| 357 | INIT_LIST_HEAD(&priv->fs.arfs.rules); |
| 358 | priv->fs.arfs.wq = create_singlethread_workqueue("mlx5e_arfs"); |
| 359 | if (!priv->fs.arfs.wq) |
| 360 | return -ENOMEM; |
| 361 | |
Maor Gottlieb | 1cabe6b | 2016-04-29 01:36:40 +0300 | [diff] [blame] | 362 | for (i = 0; i < ARFS_NUM_TYPES; i++) { |
| 363 | err = arfs_create_table(priv, i); |
| 364 | if (err) |
| 365 | goto err; |
| 366 | } |
| 367 | return 0; |
| 368 | err: |
| 369 | mlx5e_arfs_destroy_tables(priv); |
| 370 | return err; |
| 371 | } |
Maor Gottlieb | 18c908e | 2016-04-29 01:36:41 +0300 | [diff] [blame] | 372 | |
| 373 | #define MLX5E_ARFS_EXPIRY_QUOTA 60 |
| 374 | |
| 375 | static void arfs_may_expire_flow(struct mlx5e_priv *priv) |
| 376 | { |
| 377 | struct arfs_rule *arfs_rule; |
| 378 | struct hlist_node *htmp; |
| 379 | int quota = 0; |
| 380 | int i; |
| 381 | int j; |
| 382 | |
| 383 | HLIST_HEAD(del_list); |
| 384 | spin_lock_bh(&priv->fs.arfs.arfs_lock); |
| 385 | mlx5e_for_each_arfs_rule(arfs_rule, htmp, priv->fs.arfs.arfs_tables, i, j) { |
Maor Gottlieb | 18c908e | 2016-04-29 01:36:41 +0300 | [diff] [blame] | 386 | if (!work_pending(&arfs_rule->arfs_work) && |
| 387 | rps_may_expire_flow(priv->netdev, |
| 388 | arfs_rule->rxq, arfs_rule->flow_id, |
| 389 | arfs_rule->filter_id)) { |
| 390 | hlist_del_init(&arfs_rule->hlist); |
| 391 | hlist_add_head(&arfs_rule->hlist, &del_list); |
Eran Ben Elisha | b7e37ad | 2018-07-08 13:08:55 +0300 | [diff] [blame] | 392 | if (quota++ > MLX5E_ARFS_EXPIRY_QUOTA) |
| 393 | break; |
Maor Gottlieb | 18c908e | 2016-04-29 01:36:41 +0300 | [diff] [blame] | 394 | } |
| 395 | } |
| 396 | spin_unlock_bh(&priv->fs.arfs.arfs_lock); |
| 397 | hlist_for_each_entry_safe(arfs_rule, htmp, &del_list, hlist) { |
| 398 | if (arfs_rule->rule) |
| 399 | mlx5_del_flow_rule(arfs_rule->rule); |
| 400 | hlist_del(&arfs_rule->hlist); |
| 401 | kfree(arfs_rule); |
| 402 | } |
| 403 | } |
| 404 | |
| 405 | static void arfs_del_rules(struct mlx5e_priv *priv) |
| 406 | { |
| 407 | struct hlist_node *htmp; |
| 408 | struct arfs_rule *rule; |
| 409 | int i; |
| 410 | int j; |
| 411 | |
| 412 | HLIST_HEAD(del_list); |
| 413 | spin_lock_bh(&priv->fs.arfs.arfs_lock); |
| 414 | mlx5e_for_each_arfs_rule(rule, htmp, priv->fs.arfs.arfs_tables, i, j) { |
| 415 | hlist_del_init(&rule->hlist); |
| 416 | hlist_add_head(&rule->hlist, &del_list); |
| 417 | } |
| 418 | spin_unlock_bh(&priv->fs.arfs.arfs_lock); |
| 419 | |
| 420 | hlist_for_each_entry_safe(rule, htmp, &del_list, hlist) { |
| 421 | cancel_work_sync(&rule->arfs_work); |
| 422 | if (rule->rule) |
| 423 | mlx5_del_flow_rule(rule->rule); |
| 424 | hlist_del(&rule->hlist); |
| 425 | kfree(rule); |
| 426 | } |
| 427 | } |
| 428 | |
| 429 | static struct hlist_head * |
| 430 | arfs_hash_bucket(struct arfs_table *arfs_t, __be16 src_port, |
| 431 | __be16 dst_port) |
| 432 | { |
| 433 | unsigned long l; |
| 434 | int bucket_idx; |
| 435 | |
| 436 | l = (__force unsigned long)src_port | |
| 437 | ((__force unsigned long)dst_port << 2); |
| 438 | |
| 439 | bucket_idx = hash_long(l, ARFS_HASH_SHIFT); |
| 440 | |
| 441 | return &arfs_t->rules_hash[bucket_idx]; |
| 442 | } |
| 443 | |
| 444 | static u8 arfs_get_ip_proto(const struct sk_buff *skb) |
| 445 | { |
| 446 | return (skb->protocol == htons(ETH_P_IP)) ? |
| 447 | ip_hdr(skb)->protocol : ipv6_hdr(skb)->nexthdr; |
| 448 | } |
| 449 | |
| 450 | static struct arfs_table *arfs_get_table(struct mlx5e_arfs_tables *arfs, |
| 451 | u8 ip_proto, __be16 etype) |
| 452 | { |
| 453 | if (etype == htons(ETH_P_IP) && ip_proto == IPPROTO_TCP) |
| 454 | return &arfs->arfs_tables[ARFS_IPV4_TCP]; |
| 455 | if (etype == htons(ETH_P_IP) && ip_proto == IPPROTO_UDP) |
| 456 | return &arfs->arfs_tables[ARFS_IPV4_UDP]; |
| 457 | if (etype == htons(ETH_P_IPV6) && ip_proto == IPPROTO_TCP) |
| 458 | return &arfs->arfs_tables[ARFS_IPV6_TCP]; |
| 459 | if (etype == htons(ETH_P_IPV6) && ip_proto == IPPROTO_UDP) |
| 460 | return &arfs->arfs_tables[ARFS_IPV6_UDP]; |
| 461 | |
| 462 | return NULL; |
| 463 | } |
| 464 | |
| 465 | static struct mlx5_flow_rule *arfs_add_rule(struct mlx5e_priv *priv, |
| 466 | struct arfs_rule *arfs_rule) |
| 467 | { |
| 468 | struct mlx5e_arfs_tables *arfs = &priv->fs.arfs; |
| 469 | struct arfs_tuple *tuple = &arfs_rule->tuple; |
| 470 | struct mlx5_flow_rule *rule = NULL; |
| 471 | struct mlx5_flow_destination dest; |
| 472 | struct arfs_table *arfs_table; |
Maor Gottlieb | c5bb173 | 2016-07-04 17:23:05 +0300 | [diff] [blame] | 473 | struct mlx5_flow_spec *spec; |
Maor Gottlieb | 18c908e | 2016-04-29 01:36:41 +0300 | [diff] [blame] | 474 | struct mlx5_flow_table *ft; |
Maor Gottlieb | 18c908e | 2016-04-29 01:36:41 +0300 | [diff] [blame] | 475 | int err = 0; |
| 476 | |
Maor Gottlieb | c5bb173 | 2016-07-04 17:23:05 +0300 | [diff] [blame] | 477 | spec = mlx5_vzalloc(sizeof(*spec)); |
| 478 | if (!spec) { |
Maor Gottlieb | 18c908e | 2016-04-29 01:36:41 +0300 | [diff] [blame] | 479 | netdev_err(priv->netdev, "%s: alloc failed\n", __func__); |
| 480 | err = -ENOMEM; |
| 481 | goto out; |
| 482 | } |
Maor Gottlieb | c5bb173 | 2016-07-04 17:23:05 +0300 | [diff] [blame] | 483 | spec->match_criteria_enable = MLX5_MATCH_OUTER_HEADERS; |
| 484 | MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, |
Maor Gottlieb | 18c908e | 2016-04-29 01:36:41 +0300 | [diff] [blame] | 485 | outer_headers.ethertype); |
Maor Gottlieb | c5bb173 | 2016-07-04 17:23:05 +0300 | [diff] [blame] | 486 | MLX5_SET(fte_match_param, spec->match_value, outer_headers.ethertype, |
Maor Gottlieb | 18c908e | 2016-04-29 01:36:41 +0300 | [diff] [blame] | 487 | ntohs(tuple->etype)); |
| 488 | arfs_table = arfs_get_table(arfs, tuple->ip_proto, tuple->etype); |
| 489 | if (!arfs_table) { |
| 490 | err = -EINVAL; |
| 491 | goto out; |
| 492 | } |
| 493 | |
| 494 | ft = arfs_table->ft.t; |
| 495 | if (tuple->ip_proto == IPPROTO_TCP) { |
Maor Gottlieb | c5bb173 | 2016-07-04 17:23:05 +0300 | [diff] [blame] | 496 | MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, |
Maor Gottlieb | 18c908e | 2016-04-29 01:36:41 +0300 | [diff] [blame] | 497 | outer_headers.tcp_dport); |
Maor Gottlieb | c5bb173 | 2016-07-04 17:23:05 +0300 | [diff] [blame] | 498 | MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, |
Maor Gottlieb | 18c908e | 2016-04-29 01:36:41 +0300 | [diff] [blame] | 499 | outer_headers.tcp_sport); |
Maor Gottlieb | c5bb173 | 2016-07-04 17:23:05 +0300 | [diff] [blame] | 500 | MLX5_SET(fte_match_param, spec->match_value, outer_headers.tcp_dport, |
Maor Gottlieb | 18c908e | 2016-04-29 01:36:41 +0300 | [diff] [blame] | 501 | ntohs(tuple->dst_port)); |
Maor Gottlieb | c5bb173 | 2016-07-04 17:23:05 +0300 | [diff] [blame] | 502 | MLX5_SET(fte_match_param, spec->match_value, outer_headers.tcp_sport, |
Maor Gottlieb | 18c908e | 2016-04-29 01:36:41 +0300 | [diff] [blame] | 503 | ntohs(tuple->src_port)); |
| 504 | } else { |
Maor Gottlieb | c5bb173 | 2016-07-04 17:23:05 +0300 | [diff] [blame] | 505 | MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, |
Maor Gottlieb | 18c908e | 2016-04-29 01:36:41 +0300 | [diff] [blame] | 506 | outer_headers.udp_dport); |
Maor Gottlieb | c5bb173 | 2016-07-04 17:23:05 +0300 | [diff] [blame] | 507 | MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, |
Maor Gottlieb | 18c908e | 2016-04-29 01:36:41 +0300 | [diff] [blame] | 508 | outer_headers.udp_sport); |
Maor Gottlieb | c5bb173 | 2016-07-04 17:23:05 +0300 | [diff] [blame] | 509 | MLX5_SET(fte_match_param, spec->match_value, outer_headers.udp_dport, |
Maor Gottlieb | 18c908e | 2016-04-29 01:36:41 +0300 | [diff] [blame] | 510 | ntohs(tuple->dst_port)); |
Maor Gottlieb | c5bb173 | 2016-07-04 17:23:05 +0300 | [diff] [blame] | 511 | MLX5_SET(fte_match_param, spec->match_value, outer_headers.udp_sport, |
Maor Gottlieb | 18c908e | 2016-04-29 01:36:41 +0300 | [diff] [blame] | 512 | ntohs(tuple->src_port)); |
| 513 | } |
| 514 | if (tuple->etype == htons(ETH_P_IP)) { |
Maor Gottlieb | c5bb173 | 2016-07-04 17:23:05 +0300 | [diff] [blame] | 515 | memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value, |
Maor Gottlieb | 18c908e | 2016-04-29 01:36:41 +0300 | [diff] [blame] | 516 | outer_headers.src_ipv4_src_ipv6.ipv4_layout.ipv4), |
| 517 | &tuple->src_ipv4, |
| 518 | 4); |
Maor Gottlieb | c5bb173 | 2016-07-04 17:23:05 +0300 | [diff] [blame] | 519 | memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value, |
Maor Gottlieb | 18c908e | 2016-04-29 01:36:41 +0300 | [diff] [blame] | 520 | outer_headers.dst_ipv4_dst_ipv6.ipv4_layout.ipv4), |
| 521 | &tuple->dst_ipv4, |
| 522 | 4); |
Maor Gottlieb | c5bb173 | 2016-07-04 17:23:05 +0300 | [diff] [blame] | 523 | MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, |
Maor Gottlieb | 18c908e | 2016-04-29 01:36:41 +0300 | [diff] [blame] | 524 | outer_headers.src_ipv4_src_ipv6.ipv4_layout.ipv4); |
Maor Gottlieb | c5bb173 | 2016-07-04 17:23:05 +0300 | [diff] [blame] | 525 | MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, |
Maor Gottlieb | 18c908e | 2016-04-29 01:36:41 +0300 | [diff] [blame] | 526 | outer_headers.dst_ipv4_dst_ipv6.ipv4_layout.ipv4); |
| 527 | } else { |
Maor Gottlieb | c5bb173 | 2016-07-04 17:23:05 +0300 | [diff] [blame] | 528 | memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value, |
Maor Gottlieb | 18c908e | 2016-04-29 01:36:41 +0300 | [diff] [blame] | 529 | outer_headers.src_ipv4_src_ipv6.ipv6_layout.ipv6), |
| 530 | &tuple->src_ipv6, |
| 531 | 16); |
Maor Gottlieb | c5bb173 | 2016-07-04 17:23:05 +0300 | [diff] [blame] | 532 | memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value, |
Maor Gottlieb | 18c908e | 2016-04-29 01:36:41 +0300 | [diff] [blame] | 533 | outer_headers.dst_ipv4_dst_ipv6.ipv6_layout.ipv6), |
| 534 | &tuple->dst_ipv6, |
| 535 | 16); |
Maor Gottlieb | c5bb173 | 2016-07-04 17:23:05 +0300 | [diff] [blame] | 536 | memset(MLX5_ADDR_OF(fte_match_param, spec->match_criteria, |
Maor Gottlieb | 18c908e | 2016-04-29 01:36:41 +0300 | [diff] [blame] | 537 | outer_headers.src_ipv4_src_ipv6.ipv6_layout.ipv6), |
| 538 | 0xff, |
| 539 | 16); |
Maor Gottlieb | c5bb173 | 2016-07-04 17:23:05 +0300 | [diff] [blame] | 540 | memset(MLX5_ADDR_OF(fte_match_param, spec->match_criteria, |
Maor Gottlieb | 18c908e | 2016-04-29 01:36:41 +0300 | [diff] [blame] | 541 | outer_headers.dst_ipv4_dst_ipv6.ipv6_layout.ipv6), |
| 542 | 0xff, |
| 543 | 16); |
| 544 | } |
| 545 | dest.type = MLX5_FLOW_DESTINATION_TYPE_TIR; |
| 546 | dest.tir_num = priv->direct_tir[arfs_rule->rxq].tirn; |
Maor Gottlieb | c5bb173 | 2016-07-04 17:23:05 +0300 | [diff] [blame] | 547 | rule = mlx5_add_flow_rule(ft, spec, MLX5_FLOW_CONTEXT_ACTION_FWD_DEST, |
Maor Gottlieb | 18c908e | 2016-04-29 01:36:41 +0300 | [diff] [blame] | 548 | MLX5_FS_DEFAULT_FLOW_TAG, |
| 549 | &dest); |
| 550 | if (IS_ERR(rule)) { |
| 551 | err = PTR_ERR(rule); |
| 552 | netdev_err(priv->netdev, "%s: add rule(filter id=%d, rq idx=%d) failed, err=%d\n", |
| 553 | __func__, arfs_rule->filter_id, arfs_rule->rxq, err); |
| 554 | } |
| 555 | |
| 556 | out: |
Maor Gottlieb | c5bb173 | 2016-07-04 17:23:05 +0300 | [diff] [blame] | 557 | kvfree(spec); |
Maor Gottlieb | 18c908e | 2016-04-29 01:36:41 +0300 | [diff] [blame] | 558 | return err ? ERR_PTR(err) : rule; |
| 559 | } |
| 560 | |
| 561 | static void arfs_modify_rule_rq(struct mlx5e_priv *priv, |
| 562 | struct mlx5_flow_rule *rule, u16 rxq) |
| 563 | { |
| 564 | struct mlx5_flow_destination dst; |
| 565 | int err = 0; |
| 566 | |
| 567 | dst.type = MLX5_FLOW_DESTINATION_TYPE_TIR; |
| 568 | dst.tir_num = priv->direct_tir[rxq].tirn; |
| 569 | err = mlx5_modify_rule_destination(rule, &dst); |
| 570 | if (err) |
| 571 | netdev_warn(priv->netdev, |
| 572 | "Failed to modfiy aRFS rule destination to rq=%d\n", rxq); |
| 573 | } |
| 574 | |
| 575 | static void arfs_handle_work(struct work_struct *work) |
| 576 | { |
| 577 | struct arfs_rule *arfs_rule = container_of(work, |
| 578 | struct arfs_rule, |
| 579 | arfs_work); |
| 580 | struct mlx5e_priv *priv = arfs_rule->priv; |
| 581 | struct mlx5_flow_rule *rule; |
| 582 | |
| 583 | mutex_lock(&priv->state_lock); |
| 584 | if (!test_bit(MLX5E_STATE_OPENED, &priv->state)) { |
| 585 | spin_lock_bh(&priv->fs.arfs.arfs_lock); |
| 586 | hlist_del(&arfs_rule->hlist); |
| 587 | spin_unlock_bh(&priv->fs.arfs.arfs_lock); |
| 588 | |
| 589 | mutex_unlock(&priv->state_lock); |
| 590 | kfree(arfs_rule); |
| 591 | goto out; |
| 592 | } |
| 593 | mutex_unlock(&priv->state_lock); |
| 594 | |
| 595 | if (!arfs_rule->rule) { |
| 596 | rule = arfs_add_rule(priv, arfs_rule); |
| 597 | if (IS_ERR(rule)) |
| 598 | goto out; |
| 599 | arfs_rule->rule = rule; |
| 600 | } else { |
| 601 | arfs_modify_rule_rq(priv, arfs_rule->rule, |
| 602 | arfs_rule->rxq); |
| 603 | } |
| 604 | out: |
| 605 | arfs_may_expire_flow(priv); |
| 606 | } |
| 607 | |
| 608 | /* return L4 destination port from ip4/6 packets */ |
| 609 | static __be16 arfs_get_dst_port(const struct sk_buff *skb) |
| 610 | { |
| 611 | char *transport_header; |
| 612 | |
| 613 | transport_header = skb_transport_header(skb); |
| 614 | if (arfs_get_ip_proto(skb) == IPPROTO_TCP) |
| 615 | return ((struct tcphdr *)transport_header)->dest; |
| 616 | return ((struct udphdr *)transport_header)->dest; |
| 617 | } |
| 618 | |
| 619 | /* return L4 source port from ip4/6 packets */ |
| 620 | static __be16 arfs_get_src_port(const struct sk_buff *skb) |
| 621 | { |
| 622 | char *transport_header; |
| 623 | |
| 624 | transport_header = skb_transport_header(skb); |
| 625 | if (arfs_get_ip_proto(skb) == IPPROTO_TCP) |
| 626 | return ((struct tcphdr *)transport_header)->source; |
| 627 | return ((struct udphdr *)transport_header)->source; |
| 628 | } |
| 629 | |
| 630 | static struct arfs_rule *arfs_alloc_rule(struct mlx5e_priv *priv, |
| 631 | struct arfs_table *arfs_t, |
| 632 | const struct sk_buff *skb, |
| 633 | u16 rxq, u32 flow_id) |
| 634 | { |
| 635 | struct arfs_rule *rule; |
| 636 | struct arfs_tuple *tuple; |
| 637 | |
| 638 | rule = kzalloc(sizeof(*rule), GFP_ATOMIC); |
| 639 | if (!rule) |
| 640 | return NULL; |
| 641 | |
| 642 | rule->priv = priv; |
| 643 | rule->rxq = rxq; |
| 644 | INIT_WORK(&rule->arfs_work, arfs_handle_work); |
| 645 | |
| 646 | tuple = &rule->tuple; |
| 647 | tuple->etype = skb->protocol; |
| 648 | if (tuple->etype == htons(ETH_P_IP)) { |
| 649 | tuple->src_ipv4 = ip_hdr(skb)->saddr; |
| 650 | tuple->dst_ipv4 = ip_hdr(skb)->daddr; |
| 651 | } else { |
| 652 | memcpy(&tuple->src_ipv6, &ipv6_hdr(skb)->saddr, |
| 653 | sizeof(struct in6_addr)); |
| 654 | memcpy(&tuple->dst_ipv6, &ipv6_hdr(skb)->daddr, |
| 655 | sizeof(struct in6_addr)); |
| 656 | } |
| 657 | tuple->ip_proto = arfs_get_ip_proto(skb); |
| 658 | tuple->src_port = arfs_get_src_port(skb); |
| 659 | tuple->dst_port = arfs_get_dst_port(skb); |
| 660 | |
| 661 | rule->flow_id = flow_id; |
| 662 | rule->filter_id = priv->fs.arfs.last_filter_id++ % RPS_NO_FILTER; |
| 663 | |
| 664 | hlist_add_head(&rule->hlist, |
| 665 | arfs_hash_bucket(arfs_t, tuple->src_port, |
| 666 | tuple->dst_port)); |
| 667 | return rule; |
| 668 | } |
| 669 | |
| 670 | static bool arfs_cmp_ips(struct arfs_tuple *tuple, |
| 671 | const struct sk_buff *skb) |
| 672 | { |
| 673 | if (tuple->etype == htons(ETH_P_IP) && |
| 674 | tuple->src_ipv4 == ip_hdr(skb)->saddr && |
| 675 | tuple->dst_ipv4 == ip_hdr(skb)->daddr) |
| 676 | return true; |
| 677 | if (tuple->etype == htons(ETH_P_IPV6) && |
| 678 | (!memcmp(&tuple->src_ipv6, &ipv6_hdr(skb)->saddr, |
| 679 | sizeof(struct in6_addr))) && |
| 680 | (!memcmp(&tuple->dst_ipv6, &ipv6_hdr(skb)->daddr, |
| 681 | sizeof(struct in6_addr)))) |
| 682 | return true; |
| 683 | return false; |
| 684 | } |
| 685 | |
| 686 | static struct arfs_rule *arfs_find_rule(struct arfs_table *arfs_t, |
| 687 | const struct sk_buff *skb) |
| 688 | { |
| 689 | struct arfs_rule *arfs_rule; |
| 690 | struct hlist_head *head; |
| 691 | __be16 src_port = arfs_get_src_port(skb); |
| 692 | __be16 dst_port = arfs_get_dst_port(skb); |
| 693 | |
| 694 | head = arfs_hash_bucket(arfs_t, src_port, dst_port); |
| 695 | hlist_for_each_entry(arfs_rule, head, hlist) { |
| 696 | if (arfs_rule->tuple.src_port == src_port && |
| 697 | arfs_rule->tuple.dst_port == dst_port && |
| 698 | arfs_cmp_ips(&arfs_rule->tuple, skb)) { |
| 699 | return arfs_rule; |
| 700 | } |
| 701 | } |
| 702 | |
| 703 | return NULL; |
| 704 | } |
| 705 | |
| 706 | int mlx5e_rx_flow_steer(struct net_device *dev, const struct sk_buff *skb, |
| 707 | u16 rxq_index, u32 flow_id) |
| 708 | { |
| 709 | struct mlx5e_priv *priv = netdev_priv(dev); |
| 710 | struct mlx5e_arfs_tables *arfs = &priv->fs.arfs; |
| 711 | struct arfs_table *arfs_t; |
| 712 | struct arfs_rule *arfs_rule; |
| 713 | |
| 714 | if (skb->protocol != htons(ETH_P_IP) && |
| 715 | skb->protocol != htons(ETH_P_IPV6)) |
| 716 | return -EPROTONOSUPPORT; |
| 717 | |
Eran Ben Elisha | d9d5801 | 2018-07-08 14:52:12 +0300 | [diff] [blame] | 718 | if (skb->encapsulation) |
| 719 | return -EPROTONOSUPPORT; |
| 720 | |
Maor Gottlieb | 18c908e | 2016-04-29 01:36:41 +0300 | [diff] [blame] | 721 | arfs_t = arfs_get_table(arfs, arfs_get_ip_proto(skb), skb->protocol); |
| 722 | if (!arfs_t) |
| 723 | return -EPROTONOSUPPORT; |
| 724 | |
| 725 | spin_lock_bh(&arfs->arfs_lock); |
| 726 | arfs_rule = arfs_find_rule(arfs_t, skb); |
| 727 | if (arfs_rule) { |
| 728 | if (arfs_rule->rxq == rxq_index) { |
| 729 | spin_unlock_bh(&arfs->arfs_lock); |
| 730 | return arfs_rule->filter_id; |
| 731 | } |
| 732 | arfs_rule->rxq = rxq_index; |
| 733 | } else { |
| 734 | arfs_rule = arfs_alloc_rule(priv, arfs_t, skb, |
| 735 | rxq_index, flow_id); |
| 736 | if (!arfs_rule) { |
| 737 | spin_unlock_bh(&arfs->arfs_lock); |
| 738 | return -ENOMEM; |
| 739 | } |
| 740 | } |
| 741 | queue_work(priv->fs.arfs.wq, &arfs_rule->arfs_work); |
| 742 | spin_unlock_bh(&arfs->arfs_lock); |
| 743 | return arfs_rule->filter_id; |
| 744 | } |
Maor Gottlieb | fbc4a69 | 2016-05-03 17:13:53 +0300 | [diff] [blame] | 745 | #endif |