Tom Herbert | 2346155 | 2014-09-17 12:25:56 -0700 | [diff] [blame] | 1 | #include <linux/module.h> |
| 2 | #include <linux/errno.h> |
| 3 | #include <linux/socket.h> |
| 4 | #include <linux/skbuff.h> |
| 5 | #include <linux/ip.h> |
| 6 | #include <linux/udp.h> |
| 7 | #include <linux/types.h> |
| 8 | #include <linux/kernel.h> |
| 9 | #include <net/genetlink.h> |
Tom Herbert | 37dd024 | 2014-10-03 15:48:09 -0700 | [diff] [blame] | 10 | #include <net/gue.h> |
Tom Herbert | 2346155 | 2014-09-17 12:25:56 -0700 | [diff] [blame] | 11 | #include <net/ip.h> |
Tom Herbert | afe9332 | 2014-09-17 12:25:57 -0700 | [diff] [blame] | 12 | #include <net/protocol.h> |
Tom Herbert | 2346155 | 2014-09-17 12:25:56 -0700 | [diff] [blame] | 13 | #include <net/udp.h> |
| 14 | #include <net/udp_tunnel.h> |
| 15 | #include <net/xfrm.h> |
| 16 | #include <uapi/linux/fou.h> |
| 17 | #include <uapi/linux/genetlink.h> |
| 18 | |
Tom Herbert | 2346155 | 2014-09-17 12:25:56 -0700 | [diff] [blame] | 19 | struct fou { |
| 20 | struct socket *sock; |
| 21 | u8 protocol; |
Tom Herbert | fe881ef | 2015-02-10 16:30:33 -0800 | [diff] [blame] | 22 | u8 flags; |
WANG Cong | 4cbcdf2 | 2015-04-10 12:00:28 -0700 | [diff] [blame] | 23 | __be16 port; |
Tom Herbert | afe9332 | 2014-09-17 12:25:57 -0700 | [diff] [blame] | 24 | struct udp_offload udp_offloads; |
Tom Herbert | 2346155 | 2014-09-17 12:25:56 -0700 | [diff] [blame] | 25 | struct list_head list; |
| 26 | }; |
| 27 | |
Tom Herbert | fe881ef | 2015-02-10 16:30:33 -0800 | [diff] [blame] | 28 | #define FOU_F_REMCSUM_NOPARTIAL BIT(0) |
| 29 | |
Tom Herbert | 2346155 | 2014-09-17 12:25:56 -0700 | [diff] [blame] | 30 | struct fou_cfg { |
Tom Herbert | 37dd024 | 2014-10-03 15:48:09 -0700 | [diff] [blame] | 31 | u16 type; |
Tom Herbert | 2346155 | 2014-09-17 12:25:56 -0700 | [diff] [blame] | 32 | u8 protocol; |
Tom Herbert | fe881ef | 2015-02-10 16:30:33 -0800 | [diff] [blame] | 33 | u8 flags; |
Tom Herbert | 2346155 | 2014-09-17 12:25:56 -0700 | [diff] [blame] | 34 | struct udp_port_cfg udp_config; |
| 35 | }; |
| 36 | |
WANG Cong | 02d793c | 2015-04-10 12:00:29 -0700 | [diff] [blame^] | 37 | static unsigned int fou_net_id; |
| 38 | |
| 39 | struct fou_net { |
| 40 | struct list_head fou_list; |
| 41 | struct mutex fou_lock; |
| 42 | }; |
| 43 | |
Tom Herbert | 2346155 | 2014-09-17 12:25:56 -0700 | [diff] [blame] | 44 | static inline struct fou *fou_from_sock(struct sock *sk) |
| 45 | { |
| 46 | return sk->sk_user_data; |
| 47 | } |
| 48 | |
Tom Herbert | 5024c33 | 2014-11-04 09:06:53 -0800 | [diff] [blame] | 49 | static void fou_recv_pull(struct sk_buff *skb, size_t len) |
Tom Herbert | 2346155 | 2014-09-17 12:25:56 -0700 | [diff] [blame] | 50 | { |
| 51 | struct iphdr *iph = ip_hdr(skb); |
| 52 | |
| 53 | /* Remove 'len' bytes from the packet (UDP header and |
Tom Herbert | 5024c33 | 2014-11-04 09:06:53 -0800 | [diff] [blame] | 54 | * FOU header if present). |
Tom Herbert | 2346155 | 2014-09-17 12:25:56 -0700 | [diff] [blame] | 55 | */ |
| 56 | iph->tot_len = htons(ntohs(iph->tot_len) - len); |
| 57 | __skb_pull(skb, len); |
| 58 | skb_postpull_rcsum(skb, udp_hdr(skb), len); |
| 59 | skb_reset_transport_header(skb); |
Tom Herbert | 2346155 | 2014-09-17 12:25:56 -0700 | [diff] [blame] | 60 | } |
| 61 | |
| 62 | static int fou_udp_recv(struct sock *sk, struct sk_buff *skb) |
| 63 | { |
| 64 | struct fou *fou = fou_from_sock(sk); |
| 65 | |
| 66 | if (!fou) |
| 67 | return 1; |
| 68 | |
Tom Herbert | 5024c33 | 2014-11-04 09:06:53 -0800 | [diff] [blame] | 69 | fou_recv_pull(skb, sizeof(struct udphdr)); |
| 70 | |
| 71 | return -fou->protocol; |
| 72 | } |
| 73 | |
Tom Herbert | a8d31c1 | 2014-11-04 09:06:57 -0800 | [diff] [blame] | 74 | static struct guehdr *gue_remcsum(struct sk_buff *skb, struct guehdr *guehdr, |
Tom Herbert | fe881ef | 2015-02-10 16:30:33 -0800 | [diff] [blame] | 75 | void *data, size_t hdrlen, u8 ipproto, |
| 76 | bool nopartial) |
Tom Herbert | a8d31c1 | 2014-11-04 09:06:57 -0800 | [diff] [blame] | 77 | { |
| 78 | __be16 *pd = data; |
Tom Herbert | 4fd671d | 2014-11-25 11:21:20 -0800 | [diff] [blame] | 79 | size_t start = ntohs(pd[0]); |
| 80 | size_t offset = ntohs(pd[1]); |
| 81 | size_t plen = hdrlen + max_t(size_t, offset + sizeof(u16), start); |
Tom Herbert | a8d31c1 | 2014-11-04 09:06:57 -0800 | [diff] [blame] | 82 | |
Tom Herbert | a8d31c1 | 2014-11-04 09:06:57 -0800 | [diff] [blame] | 83 | if (!pskb_may_pull(skb, plen)) |
| 84 | return NULL; |
| 85 | guehdr = (struct guehdr *)&udp_hdr(skb)[1]; |
| 86 | |
Tom Herbert | fe881ef | 2015-02-10 16:30:33 -0800 | [diff] [blame] | 87 | skb_remcsum_process(skb, (void *)guehdr + hdrlen, |
| 88 | start, offset, nopartial); |
Tom Herbert | a8d31c1 | 2014-11-04 09:06:57 -0800 | [diff] [blame] | 89 | |
| 90 | return guehdr; |
| 91 | } |
| 92 | |
Tom Herbert | 5024c33 | 2014-11-04 09:06:53 -0800 | [diff] [blame] | 93 | static int gue_control_message(struct sk_buff *skb, struct guehdr *guehdr) |
| 94 | { |
| 95 | /* No support yet */ |
| 96 | kfree_skb(skb); |
| 97 | return 0; |
Tom Herbert | 2346155 | 2014-09-17 12:25:56 -0700 | [diff] [blame] | 98 | } |
| 99 | |
Tom Herbert | 37dd024 | 2014-10-03 15:48:09 -0700 | [diff] [blame] | 100 | static int gue_udp_recv(struct sock *sk, struct sk_buff *skb) |
| 101 | { |
| 102 | struct fou *fou = fou_from_sock(sk); |
Tom Herbert | 5024c33 | 2014-11-04 09:06:53 -0800 | [diff] [blame] | 103 | size_t len, optlen, hdrlen; |
Tom Herbert | 37dd024 | 2014-10-03 15:48:09 -0700 | [diff] [blame] | 104 | struct guehdr *guehdr; |
Tom Herbert | 5024c33 | 2014-11-04 09:06:53 -0800 | [diff] [blame] | 105 | void *data; |
Tom Herbert | a8d31c1 | 2014-11-04 09:06:57 -0800 | [diff] [blame] | 106 | u16 doffset = 0; |
Tom Herbert | 37dd024 | 2014-10-03 15:48:09 -0700 | [diff] [blame] | 107 | |
| 108 | if (!fou) |
| 109 | return 1; |
| 110 | |
| 111 | len = sizeof(struct udphdr) + sizeof(struct guehdr); |
| 112 | if (!pskb_may_pull(skb, len)) |
| 113 | goto drop; |
| 114 | |
Tom Herbert | 5024c33 | 2014-11-04 09:06:53 -0800 | [diff] [blame] | 115 | guehdr = (struct guehdr *)&udp_hdr(skb)[1]; |
Tom Herbert | 37dd024 | 2014-10-03 15:48:09 -0700 | [diff] [blame] | 116 | |
Tom Herbert | 5024c33 | 2014-11-04 09:06:53 -0800 | [diff] [blame] | 117 | optlen = guehdr->hlen << 2; |
| 118 | len += optlen; |
| 119 | |
Tom Herbert | 37dd024 | 2014-10-03 15:48:09 -0700 | [diff] [blame] | 120 | if (!pskb_may_pull(skb, len)) |
| 121 | goto drop; |
| 122 | |
Tom Herbert | 5024c33 | 2014-11-04 09:06:53 -0800 | [diff] [blame] | 123 | /* guehdr may change after pull */ |
| 124 | guehdr = (struct guehdr *)&udp_hdr(skb)[1]; |
Li RongQing | d8f00d2 | 2014-10-17 16:53:47 +0800 | [diff] [blame] | 125 | |
Tom Herbert | 5024c33 | 2014-11-04 09:06:53 -0800 | [diff] [blame] | 126 | hdrlen = sizeof(struct guehdr) + optlen; |
| 127 | |
| 128 | if (guehdr->version != 0 || validate_gue_flags(guehdr, optlen)) |
Tom Herbert | 37dd024 | 2014-10-03 15:48:09 -0700 | [diff] [blame] | 129 | goto drop; |
| 130 | |
Tom Herbert | a8d31c1 | 2014-11-04 09:06:57 -0800 | [diff] [blame] | 131 | hdrlen = sizeof(struct guehdr) + optlen; |
| 132 | |
| 133 | ip_hdr(skb)->tot_len = htons(ntohs(ip_hdr(skb)->tot_len) - len); |
| 134 | |
Tom Herbert | a8d31c1 | 2014-11-04 09:06:57 -0800 | [diff] [blame] | 135 | /* Pull csum through the guehdr now . This can be used if |
| 136 | * there is a remote checksum offload. |
| 137 | */ |
| 138 | skb_postpull_rcsum(skb, udp_hdr(skb), len); |
Tom Herbert | 5024c33 | 2014-11-04 09:06:53 -0800 | [diff] [blame] | 139 | |
| 140 | data = &guehdr[1]; |
| 141 | |
| 142 | if (guehdr->flags & GUE_FLAG_PRIV) { |
Tom Herbert | a8d31c1 | 2014-11-04 09:06:57 -0800 | [diff] [blame] | 143 | __be32 flags = *(__be32 *)(data + doffset); |
Tom Herbert | 5024c33 | 2014-11-04 09:06:53 -0800 | [diff] [blame] | 144 | |
Tom Herbert | a8d31c1 | 2014-11-04 09:06:57 -0800 | [diff] [blame] | 145 | doffset += GUE_LEN_PRIV; |
| 146 | |
| 147 | if (flags & GUE_PFLAG_REMCSUM) { |
| 148 | guehdr = gue_remcsum(skb, guehdr, data + doffset, |
Tom Herbert | fe881ef | 2015-02-10 16:30:33 -0800 | [diff] [blame] | 149 | hdrlen, guehdr->proto_ctype, |
| 150 | !!(fou->flags & |
| 151 | FOU_F_REMCSUM_NOPARTIAL)); |
Tom Herbert | a8d31c1 | 2014-11-04 09:06:57 -0800 | [diff] [blame] | 152 | if (!guehdr) |
| 153 | goto drop; |
| 154 | |
| 155 | data = &guehdr[1]; |
| 156 | |
| 157 | doffset += GUE_PLEN_REMCSUM; |
| 158 | } |
Tom Herbert | 37dd024 | 2014-10-03 15:48:09 -0700 | [diff] [blame] | 159 | } |
| 160 | |
Tom Herbert | 5024c33 | 2014-11-04 09:06:53 -0800 | [diff] [blame] | 161 | if (unlikely(guehdr->control)) |
| 162 | return gue_control_message(skb, guehdr); |
| 163 | |
Tom Herbert | 4fd671d | 2014-11-25 11:21:20 -0800 | [diff] [blame] | 164 | __skb_pull(skb, sizeof(struct udphdr) + hdrlen); |
Tom Herbert | a8d31c1 | 2014-11-04 09:06:57 -0800 | [diff] [blame] | 165 | skb_reset_transport_header(skb); |
| 166 | |
Tom Herbert | 5024c33 | 2014-11-04 09:06:53 -0800 | [diff] [blame] | 167 | return -guehdr->proto_ctype; |
| 168 | |
Tom Herbert | 37dd024 | 2014-10-03 15:48:09 -0700 | [diff] [blame] | 169 | drop: |
| 170 | kfree_skb(skb); |
| 171 | return 0; |
| 172 | } |
| 173 | |
Tom Herbert | afe9332 | 2014-09-17 12:25:57 -0700 | [diff] [blame] | 174 | static struct sk_buff **fou_gro_receive(struct sk_buff **head, |
Tom Herbert | a2b12f3 | 2015-01-12 17:00:37 -0800 | [diff] [blame] | 175 | struct sk_buff *skb, |
| 176 | struct udp_offload *uoff) |
Tom Herbert | afe9332 | 2014-09-17 12:25:57 -0700 | [diff] [blame] | 177 | { |
| 178 | const struct net_offload *ops; |
| 179 | struct sk_buff **pp = NULL; |
| 180 | u8 proto = NAPI_GRO_CB(skb)->proto; |
Tom Herbert | efc98d0 | 2014-10-03 15:48:08 -0700 | [diff] [blame] | 181 | const struct net_offload **offloads; |
Tom Herbert | afe9332 | 2014-09-17 12:25:57 -0700 | [diff] [blame] | 182 | |
| 183 | rcu_read_lock(); |
Tom Herbert | efc98d0 | 2014-10-03 15:48:08 -0700 | [diff] [blame] | 184 | offloads = NAPI_GRO_CB(skb)->is_ipv6 ? inet6_offloads : inet_offloads; |
Tom Herbert | afe9332 | 2014-09-17 12:25:57 -0700 | [diff] [blame] | 185 | ops = rcu_dereference(offloads[proto]); |
| 186 | if (!ops || !ops->callbacks.gro_receive) |
| 187 | goto out_unlock; |
| 188 | |
| 189 | pp = ops->callbacks.gro_receive(head, skb); |
| 190 | |
| 191 | out_unlock: |
| 192 | rcu_read_unlock(); |
| 193 | |
| 194 | return pp; |
| 195 | } |
| 196 | |
Tom Herbert | a2b12f3 | 2015-01-12 17:00:37 -0800 | [diff] [blame] | 197 | static int fou_gro_complete(struct sk_buff *skb, int nhoff, |
| 198 | struct udp_offload *uoff) |
Tom Herbert | afe9332 | 2014-09-17 12:25:57 -0700 | [diff] [blame] | 199 | { |
| 200 | const struct net_offload *ops; |
| 201 | u8 proto = NAPI_GRO_CB(skb)->proto; |
| 202 | int err = -ENOSYS; |
Tom Herbert | efc98d0 | 2014-10-03 15:48:08 -0700 | [diff] [blame] | 203 | const struct net_offload **offloads; |
Tom Herbert | afe9332 | 2014-09-17 12:25:57 -0700 | [diff] [blame] | 204 | |
Jesse Gross | cfdf1e1 | 2014-11-10 11:45:13 -0800 | [diff] [blame] | 205 | udp_tunnel_gro_complete(skb, nhoff); |
| 206 | |
Tom Herbert | afe9332 | 2014-09-17 12:25:57 -0700 | [diff] [blame] | 207 | rcu_read_lock(); |
Tom Herbert | efc98d0 | 2014-10-03 15:48:08 -0700 | [diff] [blame] | 208 | offloads = NAPI_GRO_CB(skb)->is_ipv6 ? inet6_offloads : inet_offloads; |
Tom Herbert | afe9332 | 2014-09-17 12:25:57 -0700 | [diff] [blame] | 209 | ops = rcu_dereference(offloads[proto]); |
| 210 | if (WARN_ON(!ops || !ops->callbacks.gro_complete)) |
| 211 | goto out_unlock; |
| 212 | |
| 213 | err = ops->callbacks.gro_complete(skb, nhoff); |
| 214 | |
| 215 | out_unlock: |
| 216 | rcu_read_unlock(); |
| 217 | |
| 218 | return err; |
| 219 | } |
| 220 | |
Tom Herbert | a8d31c1 | 2014-11-04 09:06:57 -0800 | [diff] [blame] | 221 | static struct guehdr *gue_gro_remcsum(struct sk_buff *skb, unsigned int off, |
| 222 | struct guehdr *guehdr, void *data, |
Tom Herbert | 26c4f7d | 2015-02-10 16:30:27 -0800 | [diff] [blame] | 223 | size_t hdrlen, u8 ipproto, |
Tom Herbert | fe881ef | 2015-02-10 16:30:33 -0800 | [diff] [blame] | 224 | struct gro_remcsum *grc, bool nopartial) |
Tom Herbert | a8d31c1 | 2014-11-04 09:06:57 -0800 | [diff] [blame] | 225 | { |
| 226 | __be16 *pd = data; |
Tom Herbert | 4fd671d | 2014-11-25 11:21:20 -0800 | [diff] [blame] | 227 | size_t start = ntohs(pd[0]); |
| 228 | size_t offset = ntohs(pd[1]); |
| 229 | size_t plen = hdrlen + max_t(size_t, offset + sizeof(u16), start); |
Tom Herbert | a8d31c1 | 2014-11-04 09:06:57 -0800 | [diff] [blame] | 230 | |
| 231 | if (skb->remcsum_offload) |
Tom Herbert | 26c4f7d | 2015-02-10 16:30:27 -0800 | [diff] [blame] | 232 | return NULL; |
Tom Herbert | a8d31c1 | 2014-11-04 09:06:57 -0800 | [diff] [blame] | 233 | |
Tom Herbert | 4fd671d | 2014-11-25 11:21:20 -0800 | [diff] [blame] | 234 | if (!NAPI_GRO_CB(skb)->csum_valid) |
Tom Herbert | a8d31c1 | 2014-11-04 09:06:57 -0800 | [diff] [blame] | 235 | return NULL; |
| 236 | |
Tom Herbert | a8d31c1 | 2014-11-04 09:06:57 -0800 | [diff] [blame] | 237 | /* Pull checksum that will be written */ |
| 238 | if (skb_gro_header_hard(skb, off + plen)) { |
| 239 | guehdr = skb_gro_header_slow(skb, off + plen, off); |
| 240 | if (!guehdr) |
| 241 | return NULL; |
| 242 | } |
| 243 | |
Tom Herbert | 26c4f7d | 2015-02-10 16:30:27 -0800 | [diff] [blame] | 244 | skb_gro_remcsum_process(skb, (void *)guehdr + hdrlen, |
Tom Herbert | fe881ef | 2015-02-10 16:30:33 -0800 | [diff] [blame] | 245 | start, offset, grc, nopartial); |
Tom Herbert | a8d31c1 | 2014-11-04 09:06:57 -0800 | [diff] [blame] | 246 | |
| 247 | skb->remcsum_offload = 1; |
| 248 | |
| 249 | return guehdr; |
| 250 | } |
| 251 | |
Tom Herbert | 37dd024 | 2014-10-03 15:48:09 -0700 | [diff] [blame] | 252 | static struct sk_buff **gue_gro_receive(struct sk_buff **head, |
Tom Herbert | a2b12f3 | 2015-01-12 17:00:37 -0800 | [diff] [blame] | 253 | struct sk_buff *skb, |
| 254 | struct udp_offload *uoff) |
Tom Herbert | 37dd024 | 2014-10-03 15:48:09 -0700 | [diff] [blame] | 255 | { |
| 256 | const struct net_offload **offloads; |
| 257 | const struct net_offload *ops; |
| 258 | struct sk_buff **pp = NULL; |
| 259 | struct sk_buff *p; |
Tom Herbert | 37dd024 | 2014-10-03 15:48:09 -0700 | [diff] [blame] | 260 | struct guehdr *guehdr; |
Tom Herbert | 5024c33 | 2014-11-04 09:06:53 -0800 | [diff] [blame] | 261 | size_t len, optlen, hdrlen, off; |
| 262 | void *data; |
Tom Herbert | a8d31c1 | 2014-11-04 09:06:57 -0800 | [diff] [blame] | 263 | u16 doffset = 0; |
Tom Herbert | 37dd024 | 2014-10-03 15:48:09 -0700 | [diff] [blame] | 264 | int flush = 1; |
Tom Herbert | fe881ef | 2015-02-10 16:30:33 -0800 | [diff] [blame] | 265 | struct fou *fou = container_of(uoff, struct fou, udp_offloads); |
Tom Herbert | 26c4f7d | 2015-02-10 16:30:27 -0800 | [diff] [blame] | 266 | struct gro_remcsum grc; |
| 267 | |
| 268 | skb_gro_remcsum_init(&grc); |
Tom Herbert | 37dd024 | 2014-10-03 15:48:09 -0700 | [diff] [blame] | 269 | |
| 270 | off = skb_gro_offset(skb); |
Tom Herbert | 5024c33 | 2014-11-04 09:06:53 -0800 | [diff] [blame] | 271 | len = off + sizeof(*guehdr); |
| 272 | |
Tom Herbert | 37dd024 | 2014-10-03 15:48:09 -0700 | [diff] [blame] | 273 | guehdr = skb_gro_header_fast(skb, off); |
Tom Herbert | 5024c33 | 2014-11-04 09:06:53 -0800 | [diff] [blame] | 274 | if (skb_gro_header_hard(skb, len)) { |
| 275 | guehdr = skb_gro_header_slow(skb, len, off); |
Tom Herbert | 37dd024 | 2014-10-03 15:48:09 -0700 | [diff] [blame] | 276 | if (unlikely(!guehdr)) |
| 277 | goto out; |
| 278 | } |
| 279 | |
Tom Herbert | 5024c33 | 2014-11-04 09:06:53 -0800 | [diff] [blame] | 280 | optlen = guehdr->hlen << 2; |
| 281 | len += optlen; |
Tom Herbert | 37dd024 | 2014-10-03 15:48:09 -0700 | [diff] [blame] | 282 | |
Tom Herbert | 5024c33 | 2014-11-04 09:06:53 -0800 | [diff] [blame] | 283 | if (skb_gro_header_hard(skb, len)) { |
| 284 | guehdr = skb_gro_header_slow(skb, len, off); |
Tom Herbert | 37dd024 | 2014-10-03 15:48:09 -0700 | [diff] [blame] | 285 | if (unlikely(!guehdr)) |
Tom Herbert | 5024c33 | 2014-11-04 09:06:53 -0800 | [diff] [blame] | 286 | goto out; |
| 287 | } |
| 288 | |
| 289 | if (unlikely(guehdr->control) || guehdr->version != 0 || |
| 290 | validate_gue_flags(guehdr, optlen)) |
| 291 | goto out; |
| 292 | |
| 293 | hdrlen = sizeof(*guehdr) + optlen; |
| 294 | |
Tom Herbert | a8d31c1 | 2014-11-04 09:06:57 -0800 | [diff] [blame] | 295 | /* Adjust NAPI_GRO_CB(skb)->csum to account for guehdr, |
| 296 | * this is needed if there is a remote checkcsum offload. |
| 297 | */ |
Tom Herbert | 5024c33 | 2014-11-04 09:06:53 -0800 | [diff] [blame] | 298 | skb_gro_postpull_rcsum(skb, guehdr, hdrlen); |
| 299 | |
| 300 | data = &guehdr[1]; |
| 301 | |
| 302 | if (guehdr->flags & GUE_FLAG_PRIV) { |
Tom Herbert | a8d31c1 | 2014-11-04 09:06:57 -0800 | [diff] [blame] | 303 | __be32 flags = *(__be32 *)(data + doffset); |
Tom Herbert | 5024c33 | 2014-11-04 09:06:53 -0800 | [diff] [blame] | 304 | |
Tom Herbert | a8d31c1 | 2014-11-04 09:06:57 -0800 | [diff] [blame] | 305 | doffset += GUE_LEN_PRIV; |
| 306 | |
| 307 | if (flags & GUE_PFLAG_REMCSUM) { |
| 308 | guehdr = gue_gro_remcsum(skb, off, guehdr, |
| 309 | data + doffset, hdrlen, |
Tom Herbert | fe881ef | 2015-02-10 16:30:33 -0800 | [diff] [blame] | 310 | guehdr->proto_ctype, &grc, |
| 311 | !!(fou->flags & |
| 312 | FOU_F_REMCSUM_NOPARTIAL)); |
Tom Herbert | a8d31c1 | 2014-11-04 09:06:57 -0800 | [diff] [blame] | 313 | if (!guehdr) |
| 314 | goto out; |
| 315 | |
| 316 | data = &guehdr[1]; |
| 317 | |
| 318 | doffset += GUE_PLEN_REMCSUM; |
| 319 | } |
Tom Herbert | 37dd024 | 2014-10-03 15:48:09 -0700 | [diff] [blame] | 320 | } |
| 321 | |
Tom Herbert | a8d31c1 | 2014-11-04 09:06:57 -0800 | [diff] [blame] | 322 | skb_gro_pull(skb, hdrlen); |
| 323 | |
Tom Herbert | 37dd024 | 2014-10-03 15:48:09 -0700 | [diff] [blame] | 324 | flush = 0; |
| 325 | |
| 326 | for (p = *head; p; p = p->next) { |
| 327 | const struct guehdr *guehdr2; |
| 328 | |
| 329 | if (!NAPI_GRO_CB(p)->same_flow) |
| 330 | continue; |
| 331 | |
| 332 | guehdr2 = (struct guehdr *)(p->data + off); |
| 333 | |
| 334 | /* Compare base GUE header to be equal (covers |
Tom Herbert | 5024c33 | 2014-11-04 09:06:53 -0800 | [diff] [blame] | 335 | * hlen, version, proto_ctype, and flags. |
Tom Herbert | 37dd024 | 2014-10-03 15:48:09 -0700 | [diff] [blame] | 336 | */ |
| 337 | if (guehdr->word != guehdr2->word) { |
| 338 | NAPI_GRO_CB(p)->same_flow = 0; |
| 339 | continue; |
| 340 | } |
| 341 | |
| 342 | /* Compare optional fields are the same. */ |
| 343 | if (guehdr->hlen && memcmp(&guehdr[1], &guehdr2[1], |
| 344 | guehdr->hlen << 2)) { |
| 345 | NAPI_GRO_CB(p)->same_flow = 0; |
| 346 | continue; |
| 347 | } |
| 348 | } |
| 349 | |
Tom Herbert | 5024c33 | 2014-11-04 09:06:53 -0800 | [diff] [blame] | 350 | rcu_read_lock(); |
| 351 | offloads = NAPI_GRO_CB(skb)->is_ipv6 ? inet6_offloads : inet_offloads; |
| 352 | ops = rcu_dereference(offloads[guehdr->proto_ctype]); |
| 353 | if (WARN_ON(!ops || !ops->callbacks.gro_receive)) |
| 354 | goto out_unlock; |
Tom Herbert | 37dd024 | 2014-10-03 15:48:09 -0700 | [diff] [blame] | 355 | |
| 356 | pp = ops->callbacks.gro_receive(head, skb); |
| 357 | |
| 358 | out_unlock: |
| 359 | rcu_read_unlock(); |
| 360 | out: |
| 361 | NAPI_GRO_CB(skb)->flush |= flush; |
Tom Herbert | 26c4f7d | 2015-02-10 16:30:27 -0800 | [diff] [blame] | 362 | skb_gro_remcsum_cleanup(skb, &grc); |
Tom Herbert | 37dd024 | 2014-10-03 15:48:09 -0700 | [diff] [blame] | 363 | |
| 364 | return pp; |
| 365 | } |
| 366 | |
Tom Herbert | a2b12f3 | 2015-01-12 17:00:37 -0800 | [diff] [blame] | 367 | static int gue_gro_complete(struct sk_buff *skb, int nhoff, |
| 368 | struct udp_offload *uoff) |
Tom Herbert | 37dd024 | 2014-10-03 15:48:09 -0700 | [diff] [blame] | 369 | { |
| 370 | const struct net_offload **offloads; |
| 371 | struct guehdr *guehdr = (struct guehdr *)(skb->data + nhoff); |
| 372 | const struct net_offload *ops; |
| 373 | unsigned int guehlen; |
| 374 | u8 proto; |
| 375 | int err = -ENOENT; |
| 376 | |
Tom Herbert | 5024c33 | 2014-11-04 09:06:53 -0800 | [diff] [blame] | 377 | proto = guehdr->proto_ctype; |
Tom Herbert | 37dd024 | 2014-10-03 15:48:09 -0700 | [diff] [blame] | 378 | |
| 379 | guehlen = sizeof(*guehdr) + (guehdr->hlen << 2); |
| 380 | |
| 381 | rcu_read_lock(); |
| 382 | offloads = NAPI_GRO_CB(skb)->is_ipv6 ? inet6_offloads : inet_offloads; |
| 383 | ops = rcu_dereference(offloads[proto]); |
| 384 | if (WARN_ON(!ops || !ops->callbacks.gro_complete)) |
| 385 | goto out_unlock; |
| 386 | |
| 387 | err = ops->callbacks.gro_complete(skb, nhoff + guehlen); |
| 388 | |
| 389 | out_unlock: |
| 390 | rcu_read_unlock(); |
| 391 | return err; |
| 392 | } |
| 393 | |
WANG Cong | 02d793c | 2015-04-10 12:00:29 -0700 | [diff] [blame^] | 394 | static int fou_add_to_port_list(struct net *net, struct fou *fou) |
Tom Herbert | 2346155 | 2014-09-17 12:25:56 -0700 | [diff] [blame] | 395 | { |
WANG Cong | 02d793c | 2015-04-10 12:00:29 -0700 | [diff] [blame^] | 396 | struct fou_net *fn = net_generic(net, fou_net_id); |
Tom Herbert | 2346155 | 2014-09-17 12:25:56 -0700 | [diff] [blame] | 397 | struct fou *fout; |
| 398 | |
WANG Cong | 02d793c | 2015-04-10 12:00:29 -0700 | [diff] [blame^] | 399 | mutex_lock(&fn->fou_lock); |
| 400 | list_for_each_entry(fout, &fn->fou_list, list) { |
Tom Herbert | 2346155 | 2014-09-17 12:25:56 -0700 | [diff] [blame] | 401 | if (fou->port == fout->port) { |
WANG Cong | 02d793c | 2015-04-10 12:00:29 -0700 | [diff] [blame^] | 402 | mutex_unlock(&fn->fou_lock); |
Tom Herbert | 2346155 | 2014-09-17 12:25:56 -0700 | [diff] [blame] | 403 | return -EALREADY; |
| 404 | } |
| 405 | } |
| 406 | |
WANG Cong | 02d793c | 2015-04-10 12:00:29 -0700 | [diff] [blame^] | 407 | list_add(&fou->list, &fn->fou_list); |
| 408 | mutex_unlock(&fn->fou_lock); |
Tom Herbert | 2346155 | 2014-09-17 12:25:56 -0700 | [diff] [blame] | 409 | |
| 410 | return 0; |
| 411 | } |
| 412 | |
| 413 | static void fou_release(struct fou *fou) |
| 414 | { |
| 415 | struct socket *sock = fou->sock; |
| 416 | struct sock *sk = sock->sk; |
| 417 | |
WANG Cong | 9272f04 | 2015-04-10 12:00:26 -0700 | [diff] [blame] | 418 | if (sk->sk_family == AF_INET) |
| 419 | udp_del_offload(&fou->udp_offloads); |
Tom Herbert | 2346155 | 2014-09-17 12:25:56 -0700 | [diff] [blame] | 420 | list_del(&fou->list); |
WANG Cong | 02d793c | 2015-04-10 12:00:29 -0700 | [diff] [blame^] | 421 | udp_tunnel_sock_release(sock); |
Tom Herbert | 2346155 | 2014-09-17 12:25:56 -0700 | [diff] [blame] | 422 | |
| 423 | kfree(fou); |
| 424 | } |
| 425 | |
Tom Herbert | 37dd024 | 2014-10-03 15:48:09 -0700 | [diff] [blame] | 426 | static int fou_encap_init(struct sock *sk, struct fou *fou, struct fou_cfg *cfg) |
| 427 | { |
| 428 | udp_sk(sk)->encap_rcv = fou_udp_recv; |
| 429 | fou->protocol = cfg->protocol; |
| 430 | fou->udp_offloads.callbacks.gro_receive = fou_gro_receive; |
| 431 | fou->udp_offloads.callbacks.gro_complete = fou_gro_complete; |
| 432 | fou->udp_offloads.port = cfg->udp_config.local_udp_port; |
| 433 | fou->udp_offloads.ipproto = cfg->protocol; |
| 434 | |
| 435 | return 0; |
| 436 | } |
| 437 | |
| 438 | static int gue_encap_init(struct sock *sk, struct fou *fou, struct fou_cfg *cfg) |
| 439 | { |
| 440 | udp_sk(sk)->encap_rcv = gue_udp_recv; |
| 441 | fou->udp_offloads.callbacks.gro_receive = gue_gro_receive; |
| 442 | fou->udp_offloads.callbacks.gro_complete = gue_gro_complete; |
| 443 | fou->udp_offloads.port = cfg->udp_config.local_udp_port; |
| 444 | |
| 445 | return 0; |
| 446 | } |
| 447 | |
Tom Herbert | 2346155 | 2014-09-17 12:25:56 -0700 | [diff] [blame] | 448 | static int fou_create(struct net *net, struct fou_cfg *cfg, |
| 449 | struct socket **sockp) |
| 450 | { |
Tom Herbert | 2346155 | 2014-09-17 12:25:56 -0700 | [diff] [blame] | 451 | struct socket *sock = NULL; |
WANG Cong | 02d793c | 2015-04-10 12:00:29 -0700 | [diff] [blame^] | 452 | struct fou *fou = NULL; |
Tom Herbert | 2346155 | 2014-09-17 12:25:56 -0700 | [diff] [blame] | 453 | struct sock *sk; |
WANG Cong | 02d793c | 2015-04-10 12:00:29 -0700 | [diff] [blame^] | 454 | int err; |
Tom Herbert | 2346155 | 2014-09-17 12:25:56 -0700 | [diff] [blame] | 455 | |
| 456 | /* Open UDP socket */ |
| 457 | err = udp_sock_create(net, &cfg->udp_config, &sock); |
| 458 | if (err < 0) |
| 459 | goto error; |
| 460 | |
| 461 | /* Allocate FOU port structure */ |
| 462 | fou = kzalloc(sizeof(*fou), GFP_KERNEL); |
| 463 | if (!fou) { |
| 464 | err = -ENOMEM; |
| 465 | goto error; |
| 466 | } |
| 467 | |
| 468 | sk = sock->sk; |
| 469 | |
Tom Herbert | fe881ef | 2015-02-10 16:30:33 -0800 | [diff] [blame] | 470 | fou->flags = cfg->flags; |
Tom Herbert | 37dd024 | 2014-10-03 15:48:09 -0700 | [diff] [blame] | 471 | fou->port = cfg->udp_config.local_udp_port; |
| 472 | |
| 473 | /* Initial for fou type */ |
| 474 | switch (cfg->type) { |
| 475 | case FOU_ENCAP_DIRECT: |
| 476 | err = fou_encap_init(sk, fou, cfg); |
| 477 | if (err) |
| 478 | goto error; |
| 479 | break; |
| 480 | case FOU_ENCAP_GUE: |
| 481 | err = gue_encap_init(sk, fou, cfg); |
| 482 | if (err) |
| 483 | goto error; |
| 484 | break; |
| 485 | default: |
| 486 | err = -EINVAL; |
| 487 | goto error; |
| 488 | } |
Tom Herbert | 2346155 | 2014-09-17 12:25:56 -0700 | [diff] [blame] | 489 | |
| 490 | udp_sk(sk)->encap_type = 1; |
| 491 | udp_encap_enable(); |
| 492 | |
| 493 | sk->sk_user_data = fou; |
| 494 | fou->sock = sock; |
| 495 | |
Tom Herbert | 224d019 | 2015-01-05 13:56:14 -0800 | [diff] [blame] | 496 | inet_inc_convert_csum(sk); |
Tom Herbert | 2346155 | 2014-09-17 12:25:56 -0700 | [diff] [blame] | 497 | |
| 498 | sk->sk_allocation = GFP_ATOMIC; |
| 499 | |
Tom Herbert | afe9332 | 2014-09-17 12:25:57 -0700 | [diff] [blame] | 500 | if (cfg->udp_config.family == AF_INET) { |
| 501 | err = udp_add_offload(&fou->udp_offloads); |
| 502 | if (err) |
| 503 | goto error; |
| 504 | } |
| 505 | |
WANG Cong | 02d793c | 2015-04-10 12:00:29 -0700 | [diff] [blame^] | 506 | err = fou_add_to_port_list(net, fou); |
Tom Herbert | 2346155 | 2014-09-17 12:25:56 -0700 | [diff] [blame] | 507 | if (err) |
| 508 | goto error; |
| 509 | |
| 510 | if (sockp) |
| 511 | *sockp = sock; |
| 512 | |
| 513 | return 0; |
| 514 | |
| 515 | error: |
| 516 | kfree(fou); |
| 517 | if (sock) |
WANG Cong | 02d793c | 2015-04-10 12:00:29 -0700 | [diff] [blame^] | 518 | udp_tunnel_sock_release(sock); |
Tom Herbert | 2346155 | 2014-09-17 12:25:56 -0700 | [diff] [blame] | 519 | |
| 520 | return err; |
| 521 | } |
| 522 | |
| 523 | static int fou_destroy(struct net *net, struct fou_cfg *cfg) |
| 524 | { |
WANG Cong | 02d793c | 2015-04-10 12:00:29 -0700 | [diff] [blame^] | 525 | struct fou_net *fn = net_generic(net, fou_net_id); |
WANG Cong | 4cbcdf2 | 2015-04-10 12:00:28 -0700 | [diff] [blame] | 526 | __be16 port = cfg->udp_config.local_udp_port; |
Tom Herbert | 2346155 | 2014-09-17 12:25:56 -0700 | [diff] [blame] | 527 | int err = -EINVAL; |
WANG Cong | 02d793c | 2015-04-10 12:00:29 -0700 | [diff] [blame^] | 528 | struct fou *fou; |
Tom Herbert | 2346155 | 2014-09-17 12:25:56 -0700 | [diff] [blame] | 529 | |
WANG Cong | 02d793c | 2015-04-10 12:00:29 -0700 | [diff] [blame^] | 530 | mutex_lock(&fn->fou_lock); |
| 531 | list_for_each_entry(fou, &fn->fou_list, list) { |
Tom Herbert | 2346155 | 2014-09-17 12:25:56 -0700 | [diff] [blame] | 532 | if (fou->port == port) { |
Tom Herbert | 2346155 | 2014-09-17 12:25:56 -0700 | [diff] [blame] | 533 | fou_release(fou); |
| 534 | err = 0; |
| 535 | break; |
| 536 | } |
| 537 | } |
WANG Cong | 02d793c | 2015-04-10 12:00:29 -0700 | [diff] [blame^] | 538 | mutex_unlock(&fn->fou_lock); |
Tom Herbert | 2346155 | 2014-09-17 12:25:56 -0700 | [diff] [blame] | 539 | |
| 540 | return err; |
| 541 | } |
| 542 | |
| 543 | static struct genl_family fou_nl_family = { |
| 544 | .id = GENL_ID_GENERATE, |
| 545 | .hdrsize = 0, |
| 546 | .name = FOU_GENL_NAME, |
| 547 | .version = FOU_GENL_VERSION, |
| 548 | .maxattr = FOU_ATTR_MAX, |
| 549 | .netnsok = true, |
| 550 | }; |
| 551 | |
| 552 | static struct nla_policy fou_nl_policy[FOU_ATTR_MAX + 1] = { |
| 553 | [FOU_ATTR_PORT] = { .type = NLA_U16, }, |
| 554 | [FOU_ATTR_AF] = { .type = NLA_U8, }, |
| 555 | [FOU_ATTR_IPPROTO] = { .type = NLA_U8, }, |
Tom Herbert | 37dd024 | 2014-10-03 15:48:09 -0700 | [diff] [blame] | 556 | [FOU_ATTR_TYPE] = { .type = NLA_U8, }, |
Tom Herbert | fe881ef | 2015-02-10 16:30:33 -0800 | [diff] [blame] | 557 | [FOU_ATTR_REMCSUM_NOPARTIAL] = { .type = NLA_FLAG, }, |
Tom Herbert | 2346155 | 2014-09-17 12:25:56 -0700 | [diff] [blame] | 558 | }; |
| 559 | |
| 560 | static int parse_nl_config(struct genl_info *info, |
| 561 | struct fou_cfg *cfg) |
| 562 | { |
| 563 | memset(cfg, 0, sizeof(*cfg)); |
| 564 | |
| 565 | cfg->udp_config.family = AF_INET; |
| 566 | |
| 567 | if (info->attrs[FOU_ATTR_AF]) { |
| 568 | u8 family = nla_get_u8(info->attrs[FOU_ATTR_AF]); |
| 569 | |
| 570 | if (family != AF_INET && family != AF_INET6) |
| 571 | return -EINVAL; |
| 572 | |
| 573 | cfg->udp_config.family = family; |
| 574 | } |
| 575 | |
| 576 | if (info->attrs[FOU_ATTR_PORT]) { |
WANG Cong | 4cbcdf2 | 2015-04-10 12:00:28 -0700 | [diff] [blame] | 577 | __be16 port = nla_get_be16(info->attrs[FOU_ATTR_PORT]); |
Tom Herbert | 2346155 | 2014-09-17 12:25:56 -0700 | [diff] [blame] | 578 | |
| 579 | cfg->udp_config.local_udp_port = port; |
| 580 | } |
| 581 | |
| 582 | if (info->attrs[FOU_ATTR_IPPROTO]) |
| 583 | cfg->protocol = nla_get_u8(info->attrs[FOU_ATTR_IPPROTO]); |
| 584 | |
Tom Herbert | 37dd024 | 2014-10-03 15:48:09 -0700 | [diff] [blame] | 585 | if (info->attrs[FOU_ATTR_TYPE]) |
| 586 | cfg->type = nla_get_u8(info->attrs[FOU_ATTR_TYPE]); |
| 587 | |
Tom Herbert | fe881ef | 2015-02-10 16:30:33 -0800 | [diff] [blame] | 588 | if (info->attrs[FOU_ATTR_REMCSUM_NOPARTIAL]) |
| 589 | cfg->flags |= FOU_F_REMCSUM_NOPARTIAL; |
| 590 | |
Tom Herbert | 2346155 | 2014-09-17 12:25:56 -0700 | [diff] [blame] | 591 | return 0; |
| 592 | } |
| 593 | |
| 594 | static int fou_nl_cmd_add_port(struct sk_buff *skb, struct genl_info *info) |
| 595 | { |
WANG Cong | 02d793c | 2015-04-10 12:00:29 -0700 | [diff] [blame^] | 596 | struct net *net = genl_info_net(info); |
Tom Herbert | 2346155 | 2014-09-17 12:25:56 -0700 | [diff] [blame] | 597 | struct fou_cfg cfg; |
| 598 | int err; |
| 599 | |
| 600 | err = parse_nl_config(info, &cfg); |
| 601 | if (err) |
| 602 | return err; |
| 603 | |
WANG Cong | 02d793c | 2015-04-10 12:00:29 -0700 | [diff] [blame^] | 604 | return fou_create(net, &cfg, NULL); |
Tom Herbert | 2346155 | 2014-09-17 12:25:56 -0700 | [diff] [blame] | 605 | } |
| 606 | |
| 607 | static int fou_nl_cmd_rm_port(struct sk_buff *skb, struct genl_info *info) |
| 608 | { |
WANG Cong | 02d793c | 2015-04-10 12:00:29 -0700 | [diff] [blame^] | 609 | struct net *net = genl_info_net(info); |
Tom Herbert | 2346155 | 2014-09-17 12:25:56 -0700 | [diff] [blame] | 610 | struct fou_cfg cfg; |
WANG Cong | 6727063 | 2015-04-10 12:00:27 -0700 | [diff] [blame] | 611 | int err; |
Tom Herbert | 2346155 | 2014-09-17 12:25:56 -0700 | [diff] [blame] | 612 | |
WANG Cong | 6727063 | 2015-04-10 12:00:27 -0700 | [diff] [blame] | 613 | err = parse_nl_config(info, &cfg); |
| 614 | if (err) |
| 615 | return err; |
Tom Herbert | 2346155 | 2014-09-17 12:25:56 -0700 | [diff] [blame] | 616 | |
WANG Cong | 02d793c | 2015-04-10 12:00:29 -0700 | [diff] [blame^] | 617 | return fou_destroy(net, &cfg); |
Tom Herbert | 2346155 | 2014-09-17 12:25:56 -0700 | [diff] [blame] | 618 | } |
| 619 | |
| 620 | static const struct genl_ops fou_nl_ops[] = { |
| 621 | { |
| 622 | .cmd = FOU_CMD_ADD, |
| 623 | .doit = fou_nl_cmd_add_port, |
| 624 | .policy = fou_nl_policy, |
| 625 | .flags = GENL_ADMIN_PERM, |
| 626 | }, |
| 627 | { |
| 628 | .cmd = FOU_CMD_DEL, |
| 629 | .doit = fou_nl_cmd_rm_port, |
| 630 | .policy = fou_nl_policy, |
| 631 | .flags = GENL_ADMIN_PERM, |
| 632 | }, |
| 633 | }; |
| 634 | |
Tom Herbert | a8c5f90 | 2014-11-12 11:54:09 -0800 | [diff] [blame] | 635 | size_t fou_encap_hlen(struct ip_tunnel_encap *e) |
| 636 | { |
| 637 | return sizeof(struct udphdr); |
| 638 | } |
| 639 | EXPORT_SYMBOL(fou_encap_hlen); |
| 640 | |
| 641 | size_t gue_encap_hlen(struct ip_tunnel_encap *e) |
| 642 | { |
| 643 | size_t len; |
| 644 | bool need_priv = false; |
| 645 | |
| 646 | len = sizeof(struct udphdr) + sizeof(struct guehdr); |
| 647 | |
| 648 | if (e->flags & TUNNEL_ENCAP_FLAG_REMCSUM) { |
| 649 | len += GUE_PLEN_REMCSUM; |
| 650 | need_priv = true; |
| 651 | } |
| 652 | |
| 653 | len += need_priv ? GUE_LEN_PRIV : 0; |
| 654 | |
| 655 | return len; |
| 656 | } |
| 657 | EXPORT_SYMBOL(gue_encap_hlen); |
| 658 | |
Tom Herbert | 63487ba | 2014-11-04 09:06:51 -0800 | [diff] [blame] | 659 | static void fou_build_udp(struct sk_buff *skb, struct ip_tunnel_encap *e, |
| 660 | struct flowi4 *fl4, u8 *protocol, __be16 sport) |
| 661 | { |
| 662 | struct udphdr *uh; |
| 663 | |
| 664 | skb_push(skb, sizeof(struct udphdr)); |
| 665 | skb_reset_transport_header(skb); |
| 666 | |
| 667 | uh = udp_hdr(skb); |
| 668 | |
| 669 | uh->dest = e->dport; |
| 670 | uh->source = sport; |
| 671 | uh->len = htons(skb->len); |
| 672 | uh->check = 0; |
| 673 | udp_set_csum(!(e->flags & TUNNEL_ENCAP_FLAG_CSUM), skb, |
| 674 | fl4->saddr, fl4->daddr, skb->len); |
| 675 | |
| 676 | *protocol = IPPROTO_UDP; |
| 677 | } |
| 678 | |
| 679 | int fou_build_header(struct sk_buff *skb, struct ip_tunnel_encap *e, |
| 680 | u8 *protocol, struct flowi4 *fl4) |
| 681 | { |
| 682 | bool csum = !!(e->flags & TUNNEL_ENCAP_FLAG_CSUM); |
| 683 | int type = csum ? SKB_GSO_UDP_TUNNEL_CSUM : SKB_GSO_UDP_TUNNEL; |
| 684 | __be16 sport; |
| 685 | |
| 686 | skb = iptunnel_handle_offloads(skb, csum, type); |
| 687 | |
| 688 | if (IS_ERR(skb)) |
| 689 | return PTR_ERR(skb); |
| 690 | |
| 691 | sport = e->sport ? : udp_flow_src_port(dev_net(skb->dev), |
| 692 | skb, 0, 0, false); |
| 693 | fou_build_udp(skb, e, fl4, protocol, sport); |
| 694 | |
| 695 | return 0; |
| 696 | } |
| 697 | EXPORT_SYMBOL(fou_build_header); |
| 698 | |
| 699 | int gue_build_header(struct sk_buff *skb, struct ip_tunnel_encap *e, |
| 700 | u8 *protocol, struct flowi4 *fl4) |
| 701 | { |
| 702 | bool csum = !!(e->flags & TUNNEL_ENCAP_FLAG_CSUM); |
| 703 | int type = csum ? SKB_GSO_UDP_TUNNEL_CSUM : SKB_GSO_UDP_TUNNEL; |
| 704 | struct guehdr *guehdr; |
Tom Herbert | b17f709 | 2014-11-04 09:06:56 -0800 | [diff] [blame] | 705 | size_t hdrlen, optlen = 0; |
Tom Herbert | 63487ba | 2014-11-04 09:06:51 -0800 | [diff] [blame] | 706 | __be16 sport; |
Tom Herbert | 5024c33 | 2014-11-04 09:06:53 -0800 | [diff] [blame] | 707 | void *data; |
| 708 | bool need_priv = false; |
| 709 | |
Tom Herbert | b17f709 | 2014-11-04 09:06:56 -0800 | [diff] [blame] | 710 | if ((e->flags & TUNNEL_ENCAP_FLAG_REMCSUM) && |
| 711 | skb->ip_summed == CHECKSUM_PARTIAL) { |
| 712 | csum = false; |
| 713 | optlen += GUE_PLEN_REMCSUM; |
| 714 | type |= SKB_GSO_TUNNEL_REMCSUM; |
| 715 | need_priv = true; |
| 716 | } |
| 717 | |
Tom Herbert | 5024c33 | 2014-11-04 09:06:53 -0800 | [diff] [blame] | 718 | optlen += need_priv ? GUE_LEN_PRIV : 0; |
Tom Herbert | 63487ba | 2014-11-04 09:06:51 -0800 | [diff] [blame] | 719 | |
| 720 | skb = iptunnel_handle_offloads(skb, csum, type); |
| 721 | |
| 722 | if (IS_ERR(skb)) |
| 723 | return PTR_ERR(skb); |
| 724 | |
| 725 | /* Get source port (based on flow hash) before skb_push */ |
| 726 | sport = e->sport ? : udp_flow_src_port(dev_net(skb->dev), |
| 727 | skb, 0, 0, false); |
| 728 | |
Tom Herbert | b17f709 | 2014-11-04 09:06:56 -0800 | [diff] [blame] | 729 | hdrlen = sizeof(struct guehdr) + optlen; |
| 730 | |
| 731 | skb_push(skb, hdrlen); |
Tom Herbert | 63487ba | 2014-11-04 09:06:51 -0800 | [diff] [blame] | 732 | |
| 733 | guehdr = (struct guehdr *)skb->data; |
| 734 | |
Tom Herbert | 5024c33 | 2014-11-04 09:06:53 -0800 | [diff] [blame] | 735 | guehdr->control = 0; |
Tom Herbert | 63487ba | 2014-11-04 09:06:51 -0800 | [diff] [blame] | 736 | guehdr->version = 0; |
Tom Herbert | 5024c33 | 2014-11-04 09:06:53 -0800 | [diff] [blame] | 737 | guehdr->hlen = optlen >> 2; |
Tom Herbert | 63487ba | 2014-11-04 09:06:51 -0800 | [diff] [blame] | 738 | guehdr->flags = 0; |
Tom Herbert | 5024c33 | 2014-11-04 09:06:53 -0800 | [diff] [blame] | 739 | guehdr->proto_ctype = *protocol; |
| 740 | |
| 741 | data = &guehdr[1]; |
| 742 | |
| 743 | if (need_priv) { |
| 744 | __be32 *flags = data; |
| 745 | |
| 746 | guehdr->flags |= GUE_FLAG_PRIV; |
| 747 | *flags = 0; |
| 748 | data += GUE_LEN_PRIV; |
| 749 | |
Tom Herbert | b17f709 | 2014-11-04 09:06:56 -0800 | [diff] [blame] | 750 | if (type & SKB_GSO_TUNNEL_REMCSUM) { |
| 751 | u16 csum_start = skb_checksum_start_offset(skb); |
| 752 | __be16 *pd = data; |
| 753 | |
| 754 | if (csum_start < hdrlen) |
| 755 | return -EINVAL; |
| 756 | |
| 757 | csum_start -= hdrlen; |
| 758 | pd[0] = htons(csum_start); |
| 759 | pd[1] = htons(csum_start + skb->csum_offset); |
| 760 | |
| 761 | if (!skb_is_gso(skb)) { |
| 762 | skb->ip_summed = CHECKSUM_NONE; |
| 763 | skb->encapsulation = 0; |
| 764 | } |
| 765 | |
| 766 | *flags |= GUE_PFLAG_REMCSUM; |
| 767 | data += GUE_PLEN_REMCSUM; |
| 768 | } |
| 769 | |
Tom Herbert | 5024c33 | 2014-11-04 09:06:53 -0800 | [diff] [blame] | 770 | } |
Tom Herbert | 63487ba | 2014-11-04 09:06:51 -0800 | [diff] [blame] | 771 | |
| 772 | fou_build_udp(skb, e, fl4, protocol, sport); |
| 773 | |
| 774 | return 0; |
| 775 | } |
| 776 | EXPORT_SYMBOL(gue_build_header); |
| 777 | |
Tom Herbert | a8c5f90 | 2014-11-12 11:54:09 -0800 | [diff] [blame] | 778 | #ifdef CONFIG_NET_FOU_IP_TUNNELS |
| 779 | |
| 780 | static const struct ip_tunnel_encap_ops __read_mostly fou_iptun_ops = { |
| 781 | .encap_hlen = fou_encap_hlen, |
| 782 | .build_header = fou_build_header, |
| 783 | }; |
| 784 | |
| 785 | static const struct ip_tunnel_encap_ops __read_mostly gue_iptun_ops = { |
| 786 | .encap_hlen = gue_encap_hlen, |
| 787 | .build_header = gue_build_header, |
| 788 | }; |
| 789 | |
| 790 | static int ip_tunnel_encap_add_fou_ops(void) |
| 791 | { |
| 792 | int ret; |
| 793 | |
| 794 | ret = ip_tunnel_encap_add_ops(&fou_iptun_ops, TUNNEL_ENCAP_FOU); |
| 795 | if (ret < 0) { |
| 796 | pr_err("can't add fou ops\n"); |
| 797 | return ret; |
| 798 | } |
| 799 | |
| 800 | ret = ip_tunnel_encap_add_ops(&gue_iptun_ops, TUNNEL_ENCAP_GUE); |
| 801 | if (ret < 0) { |
| 802 | pr_err("can't add gue ops\n"); |
| 803 | ip_tunnel_encap_del_ops(&fou_iptun_ops, TUNNEL_ENCAP_FOU); |
| 804 | return ret; |
| 805 | } |
| 806 | |
| 807 | return 0; |
| 808 | } |
| 809 | |
| 810 | static void ip_tunnel_encap_del_fou_ops(void) |
| 811 | { |
| 812 | ip_tunnel_encap_del_ops(&fou_iptun_ops, TUNNEL_ENCAP_FOU); |
| 813 | ip_tunnel_encap_del_ops(&gue_iptun_ops, TUNNEL_ENCAP_GUE); |
| 814 | } |
| 815 | |
| 816 | #else |
| 817 | |
| 818 | static int ip_tunnel_encap_add_fou_ops(void) |
| 819 | { |
| 820 | return 0; |
| 821 | } |
| 822 | |
Thomas Graf | 882288c | 2014-11-13 12:48:21 +0100 | [diff] [blame] | 823 | static void ip_tunnel_encap_del_fou_ops(void) |
Tom Herbert | a8c5f90 | 2014-11-12 11:54:09 -0800 | [diff] [blame] | 824 | { |
| 825 | } |
| 826 | |
| 827 | #endif |
| 828 | |
WANG Cong | 02d793c | 2015-04-10 12:00:29 -0700 | [diff] [blame^] | 829 | static __net_init int fou_init_net(struct net *net) |
| 830 | { |
| 831 | struct fou_net *fn = net_generic(net, fou_net_id); |
| 832 | |
| 833 | INIT_LIST_HEAD(&fn->fou_list); |
| 834 | mutex_init(&fn->fou_lock); |
| 835 | return 0; |
| 836 | } |
| 837 | |
| 838 | static __net_exit void fou_exit_net(struct net *net) |
| 839 | { |
| 840 | struct fou_net *fn = net_generic(net, fou_net_id); |
| 841 | struct fou *fou, *next; |
| 842 | |
| 843 | /* Close all the FOU sockets */ |
| 844 | mutex_lock(&fn->fou_lock); |
| 845 | list_for_each_entry_safe(fou, next, &fn->fou_list, list) |
| 846 | fou_release(fou); |
| 847 | mutex_unlock(&fn->fou_lock); |
| 848 | } |
| 849 | |
| 850 | static struct pernet_operations fou_net_ops = { |
| 851 | .init = fou_init_net, |
| 852 | .exit = fou_exit_net, |
| 853 | .id = &fou_net_id, |
| 854 | .size = sizeof(struct fou_net), |
| 855 | }; |
| 856 | |
Tom Herbert | 2346155 | 2014-09-17 12:25:56 -0700 | [diff] [blame] | 857 | static int __init fou_init(void) |
| 858 | { |
| 859 | int ret; |
| 860 | |
WANG Cong | 02d793c | 2015-04-10 12:00:29 -0700 | [diff] [blame^] | 861 | ret = register_pernet_device(&fou_net_ops); |
| 862 | if (ret) |
Tom Herbert | a8c5f90 | 2014-11-12 11:54:09 -0800 | [diff] [blame] | 863 | goto exit; |
| 864 | |
WANG Cong | 02d793c | 2015-04-10 12:00:29 -0700 | [diff] [blame^] | 865 | ret = genl_register_family_with_ops(&fou_nl_family, |
| 866 | fou_nl_ops); |
Tom Herbert | a8c5f90 | 2014-11-12 11:54:09 -0800 | [diff] [blame] | 867 | if (ret < 0) |
WANG Cong | 02d793c | 2015-04-10 12:00:29 -0700 | [diff] [blame^] | 868 | goto unregister; |
Tom Herbert | a8c5f90 | 2014-11-12 11:54:09 -0800 | [diff] [blame] | 869 | |
WANG Cong | 02d793c | 2015-04-10 12:00:29 -0700 | [diff] [blame^] | 870 | ret = ip_tunnel_encap_add_fou_ops(); |
| 871 | if (ret == 0) |
| 872 | return 0; |
| 873 | |
| 874 | genl_unregister_family(&fou_nl_family); |
| 875 | unregister: |
| 876 | unregister_pernet_device(&fou_net_ops); |
Tom Herbert | a8c5f90 | 2014-11-12 11:54:09 -0800 | [diff] [blame] | 877 | exit: |
Tom Herbert | 2346155 | 2014-09-17 12:25:56 -0700 | [diff] [blame] | 878 | return ret; |
| 879 | } |
| 880 | |
| 881 | static void __exit fou_fini(void) |
| 882 | { |
Tom Herbert | a8c5f90 | 2014-11-12 11:54:09 -0800 | [diff] [blame] | 883 | ip_tunnel_encap_del_fou_ops(); |
Tom Herbert | 2346155 | 2014-09-17 12:25:56 -0700 | [diff] [blame] | 884 | genl_unregister_family(&fou_nl_family); |
WANG Cong | 02d793c | 2015-04-10 12:00:29 -0700 | [diff] [blame^] | 885 | unregister_pernet_device(&fou_net_ops); |
Tom Herbert | 2346155 | 2014-09-17 12:25:56 -0700 | [diff] [blame] | 886 | } |
| 887 | |
| 888 | module_init(fou_init); |
| 889 | module_exit(fou_fini); |
| 890 | MODULE_AUTHOR("Tom Herbert <therbert@google.com>"); |
| 891 | MODULE_LICENSE("GPL"); |