Pablo Neira Ayuso | a3c90f7a | 2018-01-07 01:04:26 +0100 | [diff] [blame] | 1 | #include <linux/kernel.h> |
| 2 | #include <linux/module.h> |
| 3 | #include <linux/init.h> |
| 4 | #include <linux/netlink.h> |
| 5 | #include <linux/netfilter.h> |
| 6 | #include <linux/workqueue.h> |
| 7 | #include <linux/spinlock.h> |
| 8 | #include <linux/netfilter/nf_tables.h> |
| 9 | #include <net/ip.h> /* for ipv4 options. */ |
| 10 | #include <net/netfilter/nf_tables.h> |
| 11 | #include <net/netfilter/nf_tables_core.h> |
| 12 | #include <net/netfilter/nf_conntrack_core.h> |
| 13 | #include <linux/netfilter/nf_conntrack_common.h> |
| 14 | #include <net/netfilter/nf_flow_table.h> |
| 15 | |
| 16 | struct nft_flow_offload { |
| 17 | struct nft_flowtable *flowtable; |
| 18 | }; |
| 19 | |
| 20 | static int nft_flow_route(const struct nft_pktinfo *pkt, |
| 21 | const struct nf_conn *ct, |
| 22 | struct nf_flow_route *route, |
| 23 | enum ip_conntrack_dir dir) |
| 24 | { |
| 25 | struct dst_entry *this_dst = skb_dst(pkt->skb); |
| 26 | struct dst_entry *other_dst = NULL; |
| 27 | struct flowi fl; |
| 28 | |
| 29 | memset(&fl, 0, sizeof(fl)); |
| 30 | switch (nft_pf(pkt)) { |
| 31 | case NFPROTO_IPV4: |
wenxu | 535be46 | 2019-01-09 10:40:11 +0800 | [diff] [blame] | 32 | fl.u.ip4.daddr = ct->tuplehash[dir].tuple.src.u3.ip; |
wenxu | 6d26c37 | 2019-01-10 14:51:35 +0800 | [diff] [blame] | 33 | fl.u.ip4.flowi4_oif = nft_in(pkt)->ifindex; |
Pablo Neira Ayuso | a3c90f7a | 2018-01-07 01:04:26 +0100 | [diff] [blame] | 34 | break; |
| 35 | case NFPROTO_IPV6: |
wenxu | 535be46 | 2019-01-09 10:40:11 +0800 | [diff] [blame] | 36 | fl.u.ip6.daddr = ct->tuplehash[dir].tuple.src.u3.in6; |
wenxu | 6d26c37 | 2019-01-10 14:51:35 +0800 | [diff] [blame] | 37 | fl.u.ip6.flowi6_oif = nft_in(pkt)->ifindex; |
Pablo Neira Ayuso | a3c90f7a | 2018-01-07 01:04:26 +0100 | [diff] [blame] | 38 | break; |
| 39 | } |
| 40 | |
| 41 | nf_route(nft_net(pkt), &other_dst, &fl, false, nft_pf(pkt)); |
| 42 | if (!other_dst) |
| 43 | return -ENOENT; |
| 44 | |
| 45 | route->tuple[dir].dst = this_dst; |
Pablo Neira Ayuso | a3c90f7a | 2018-01-07 01:04:26 +0100 | [diff] [blame] | 46 | route->tuple[!dir].dst = other_dst; |
Pablo Neira Ayuso | a3c90f7a | 2018-01-07 01:04:26 +0100 | [diff] [blame] | 47 | |
| 48 | return 0; |
| 49 | } |
| 50 | |
Florian Westphal | c549680 | 2019-05-21 13:24:33 +0200 | [diff] [blame] | 51 | static bool nft_flow_offload_skip(struct sk_buff *skb, int family) |
Pablo Neira Ayuso | a3c90f7a | 2018-01-07 01:04:26 +0100 | [diff] [blame] | 52 | { |
Pablo Neira Ayuso | a3c90f7a | 2018-01-07 01:04:26 +0100 | [diff] [blame] | 53 | if (skb_sec_path(skb)) |
| 54 | return true; |
| 55 | |
Florian Westphal | c549680 | 2019-05-21 13:24:33 +0200 | [diff] [blame] | 56 | if (family == NFPROTO_IPV4) { |
| 57 | const struct ip_options *opt; |
| 58 | |
| 59 | opt = &(IPCB(skb)->opt); |
| 60 | |
| 61 | if (unlikely(opt->optlen)) |
| 62 | return true; |
| 63 | } |
| 64 | |
Pablo Neira Ayuso | a3c90f7a | 2018-01-07 01:04:26 +0100 | [diff] [blame] | 65 | return false; |
| 66 | } |
| 67 | |
| 68 | static void nft_flow_offload_eval(const struct nft_expr *expr, |
| 69 | struct nft_regs *regs, |
| 70 | const struct nft_pktinfo *pkt) |
| 71 | { |
| 72 | struct nft_flow_offload *priv = nft_expr_priv(expr); |
| 73 | struct nf_flowtable *flowtable = &priv->flowtable->data; |
Pablo Neira Ayuso | a54fa5d | 2019-08-13 17:41:13 +0200 | [diff] [blame] | 74 | struct tcphdr _tcph, *tcph = NULL; |
Pablo Neira Ayuso | a3c90f7a | 2018-01-07 01:04:26 +0100 | [diff] [blame] | 75 | enum ip_conntrack_info ctinfo; |
| 76 | struct nf_flow_route route; |
| 77 | struct flow_offload *flow; |
| 78 | enum ip_conntrack_dir dir; |
| 79 | struct nf_conn *ct; |
| 80 | int ret; |
| 81 | |
Florian Westphal | c549680 | 2019-05-21 13:24:33 +0200 | [diff] [blame] | 82 | if (nft_flow_offload_skip(pkt->skb, nft_pf(pkt))) |
Pablo Neira Ayuso | a3c90f7a | 2018-01-07 01:04:26 +0100 | [diff] [blame] | 83 | goto out; |
| 84 | |
| 85 | ct = nf_ct_get(pkt->skb, &ctinfo); |
| 86 | if (!ct) |
| 87 | goto out; |
| 88 | |
| 89 | switch (ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple.dst.protonum) { |
| 90 | case IPPROTO_TCP: |
Pablo Neira Ayuso | a54fa5d | 2019-08-13 17:41:13 +0200 | [diff] [blame] | 91 | tcph = skb_header_pointer(pkt->skb, pkt->xt.thoff, |
| 92 | sizeof(_tcph), &_tcph); |
| 93 | if (unlikely(!tcph || tcph->fin || tcph->rst)) |
| 94 | goto out; |
Florian Westphal | 48f611e | 2019-05-21 13:24:31 +0200 | [diff] [blame] | 95 | break; |
Pablo Neira Ayuso | a3c90f7a | 2018-01-07 01:04:26 +0100 | [diff] [blame] | 96 | case IPPROTO_UDP: |
| 97 | break; |
| 98 | default: |
| 99 | goto out; |
| 100 | } |
| 101 | |
Florian Westphal | 041c181 | 2019-05-21 13:24:32 +0200 | [diff] [blame] | 102 | if (nf_ct_ext_exist(ct, NF_CT_EXT_HELPER) || |
| 103 | ct->status & IPS_SEQ_ADJUST) |
Pablo Neira Ayuso | a3c90f7a | 2018-01-07 01:04:26 +0100 | [diff] [blame] | 104 | goto out; |
| 105 | |
| 106 | if (ctinfo == IP_CT_NEW || |
| 107 | ctinfo == IP_CT_RELATED) |
| 108 | goto out; |
| 109 | |
| 110 | if (test_and_set_bit(IPS_OFFLOAD_BIT, &ct->status)) |
| 111 | goto out; |
| 112 | |
| 113 | dir = CTINFO2DIR(ctinfo); |
| 114 | if (nft_flow_route(pkt, ct, &route, dir) < 0) |
| 115 | goto err_flow_route; |
| 116 | |
| 117 | flow = flow_offload_alloc(ct, &route); |
| 118 | if (!flow) |
| 119 | goto err_flow_alloc; |
| 120 | |
Pablo Neira Ayuso | a54fa5d | 2019-08-13 17:41:13 +0200 | [diff] [blame] | 121 | if (tcph) { |
Florian Westphal | 48f611e | 2019-05-21 13:24:31 +0200 | [diff] [blame] | 122 | ct->proto.tcp.seen[0].flags |= IP_CT_TCP_FLAG_BE_LIBERAL; |
| 123 | ct->proto.tcp.seen[1].flags |= IP_CT_TCP_FLAG_BE_LIBERAL; |
| 124 | } |
| 125 | |
Pablo Neira Ayuso | a3c90f7a | 2018-01-07 01:04:26 +0100 | [diff] [blame] | 126 | ret = flow_offload_add(flowtable, flow); |
| 127 | if (ret < 0) |
| 128 | goto err_flow_add; |
| 129 | |
Taehee Yoo | 028b3d8 | 2019-04-30 01:55:29 +0900 | [diff] [blame] | 130 | dst_release(route.tuple[!dir].dst); |
Pablo Neira Ayuso | a3c90f7a | 2018-01-07 01:04:26 +0100 | [diff] [blame] | 131 | return; |
| 132 | |
| 133 | err_flow_add: |
| 134 | flow_offload_free(flow); |
| 135 | err_flow_alloc: |
| 136 | dst_release(route.tuple[!dir].dst); |
| 137 | err_flow_route: |
| 138 | clear_bit(IPS_OFFLOAD_BIT, &ct->status); |
| 139 | out: |
| 140 | regs->verdict.code = NFT_BREAK; |
| 141 | } |
| 142 | |
| 143 | static int nft_flow_offload_validate(const struct nft_ctx *ctx, |
| 144 | const struct nft_expr *expr, |
| 145 | const struct nft_data **data) |
| 146 | { |
| 147 | unsigned int hook_mask = (1 << NF_INET_FORWARD); |
| 148 | |
| 149 | return nft_chain_validate_hooks(ctx->chain, hook_mask); |
| 150 | } |
| 151 | |
Pablo Neira Ayuso | a02c676 | 2019-08-16 11:23:58 +0200 | [diff] [blame] | 152 | static const struct nla_policy nft_flow_offload_policy[NFTA_FLOW_MAX + 1] = { |
| 153 | [NFTA_FLOW_TABLE_NAME] = { .type = NLA_STRING, |
| 154 | .len = NFT_NAME_MAXLEN - 1 }, |
| 155 | }; |
| 156 | |
Pablo Neira Ayuso | a3c90f7a | 2018-01-07 01:04:26 +0100 | [diff] [blame] | 157 | static int nft_flow_offload_init(const struct nft_ctx *ctx, |
| 158 | const struct nft_expr *expr, |
| 159 | const struct nlattr * const tb[]) |
| 160 | { |
| 161 | struct nft_flow_offload *priv = nft_expr_priv(expr); |
| 162 | u8 genmask = nft_genmask_next(ctx->net); |
| 163 | struct nft_flowtable *flowtable; |
| 164 | |
| 165 | if (!tb[NFTA_FLOW_TABLE_NAME]) |
| 166 | return -EINVAL; |
| 167 | |
Pablo Neira Ayuso | cac20fc | 2018-03-28 12:06:51 +0200 | [diff] [blame] | 168 | flowtable = nft_flowtable_lookup(ctx->table, tb[NFTA_FLOW_TABLE_NAME], |
| 169 | genmask); |
Pablo Neira Ayuso | a3c90f7a | 2018-01-07 01:04:26 +0100 | [diff] [blame] | 170 | if (IS_ERR(flowtable)) |
| 171 | return PTR_ERR(flowtable); |
| 172 | |
| 173 | priv->flowtable = flowtable; |
| 174 | flowtable->use++; |
| 175 | |
Pablo Neira Ayuso | 36596da | 2018-01-09 02:38:03 +0100 | [diff] [blame] | 176 | return nf_ct_netns_get(ctx->net, ctx->family); |
Pablo Neira Ayuso | a3c90f7a | 2018-01-07 01:04:26 +0100 | [diff] [blame] | 177 | } |
| 178 | |
| 179 | static void nft_flow_offload_destroy(const struct nft_ctx *ctx, |
| 180 | const struct nft_expr *expr) |
| 181 | { |
| 182 | struct nft_flow_offload *priv = nft_expr_priv(expr); |
| 183 | |
| 184 | priv->flowtable->use--; |
Pablo Neira Ayuso | 36596da | 2018-01-09 02:38:03 +0100 | [diff] [blame] | 185 | nf_ct_netns_put(ctx->net, ctx->family); |
Pablo Neira Ayuso | a3c90f7a | 2018-01-07 01:04:26 +0100 | [diff] [blame] | 186 | } |
| 187 | |
| 188 | static int nft_flow_offload_dump(struct sk_buff *skb, const struct nft_expr *expr) |
| 189 | { |
| 190 | struct nft_flow_offload *priv = nft_expr_priv(expr); |
| 191 | |
| 192 | if (nla_put_string(skb, NFTA_FLOW_TABLE_NAME, priv->flowtable->name)) |
| 193 | goto nla_put_failure; |
| 194 | |
| 195 | return 0; |
| 196 | |
| 197 | nla_put_failure: |
| 198 | return -1; |
| 199 | } |
| 200 | |
| 201 | static struct nft_expr_type nft_flow_offload_type; |
| 202 | static const struct nft_expr_ops nft_flow_offload_ops = { |
| 203 | .type = &nft_flow_offload_type, |
| 204 | .size = NFT_EXPR_SIZE(sizeof(struct nft_flow_offload)), |
| 205 | .eval = nft_flow_offload_eval, |
| 206 | .init = nft_flow_offload_init, |
| 207 | .destroy = nft_flow_offload_destroy, |
| 208 | .validate = nft_flow_offload_validate, |
| 209 | .dump = nft_flow_offload_dump, |
| 210 | }; |
| 211 | |
| 212 | static struct nft_expr_type nft_flow_offload_type __read_mostly = { |
| 213 | .name = "flow_offload", |
| 214 | .ops = &nft_flow_offload_ops, |
Pablo Neira Ayuso | a02c676 | 2019-08-16 11:23:58 +0200 | [diff] [blame] | 215 | .policy = nft_flow_offload_policy, |
Pablo Neira Ayuso | a3c90f7a | 2018-01-07 01:04:26 +0100 | [diff] [blame] | 216 | .maxattr = NFTA_FLOW_MAX, |
| 217 | .owner = THIS_MODULE, |
| 218 | }; |
| 219 | |
Pablo Neira Ayuso | a3c90f7a | 2018-01-07 01:04:26 +0100 | [diff] [blame] | 220 | static int flow_offload_netdev_event(struct notifier_block *this, |
| 221 | unsigned long event, void *ptr) |
| 222 | { |
| 223 | struct net_device *dev = netdev_notifier_info_to_dev(ptr); |
| 224 | |
| 225 | if (event != NETDEV_DOWN) |
| 226 | return NOTIFY_DONE; |
| 227 | |
Pablo Neira Ayuso | c0ea1bc | 2018-02-06 13:22:44 +0100 | [diff] [blame] | 228 | nf_flow_table_cleanup(dev_net(dev), dev); |
Pablo Neira Ayuso | a3c90f7a | 2018-01-07 01:04:26 +0100 | [diff] [blame] | 229 | |
| 230 | return NOTIFY_DONE; |
| 231 | } |
| 232 | |
| 233 | static struct notifier_block flow_offload_netdev_notifier = { |
| 234 | .notifier_call = flow_offload_netdev_event, |
| 235 | }; |
| 236 | |
| 237 | static int __init nft_flow_offload_module_init(void) |
| 238 | { |
| 239 | int err; |
| 240 | |
Taehee Yoo | 18218f8 | 2018-11-22 19:59:46 +0900 | [diff] [blame] | 241 | err = register_netdevice_notifier(&flow_offload_netdev_notifier); |
| 242 | if (err) |
| 243 | goto err; |
Pablo Neira Ayuso | a3c90f7a | 2018-01-07 01:04:26 +0100 | [diff] [blame] | 244 | |
| 245 | err = nft_register_expr(&nft_flow_offload_type); |
| 246 | if (err < 0) |
| 247 | goto register_expr; |
| 248 | |
| 249 | return 0; |
| 250 | |
| 251 | register_expr: |
| 252 | unregister_netdevice_notifier(&flow_offload_netdev_notifier); |
Taehee Yoo | 18218f8 | 2018-11-22 19:59:46 +0900 | [diff] [blame] | 253 | err: |
Pablo Neira Ayuso | a3c90f7a | 2018-01-07 01:04:26 +0100 | [diff] [blame] | 254 | return err; |
| 255 | } |
| 256 | |
| 257 | static void __exit nft_flow_offload_module_exit(void) |
| 258 | { |
Pablo Neira Ayuso | a3c90f7a | 2018-01-07 01:04:26 +0100 | [diff] [blame] | 259 | nft_unregister_expr(&nft_flow_offload_type); |
| 260 | unregister_netdevice_notifier(&flow_offload_netdev_notifier); |
Pablo Neira Ayuso | a3c90f7a | 2018-01-07 01:04:26 +0100 | [diff] [blame] | 261 | } |
| 262 | |
| 263 | module_init(nft_flow_offload_module_init); |
| 264 | module_exit(nft_flow_offload_module_exit); |
| 265 | |
| 266 | MODULE_LICENSE("GPL"); |
| 267 | MODULE_AUTHOR("Pablo Neira Ayuso <pablo@netfilter.org>"); |
| 268 | MODULE_ALIAS_NFT_EXPR("flow_offload"); |