blob: a774711a88b9de0a11a00df68462c6b824f929f1 [file] [log] [blame]
Daniel Borkmannda5bab02013-06-08 12:56:03 +02001/*
2 * IPV4 GSO/GRO offload support
3 * Linux INET implementation
4 *
5 * This program is free software; you can redistribute it and/or
6 * modify it under the terms of the GNU General Public License
7 * as published by the Free Software Foundation; either version
8 * 2 of the License, or (at your option) any later version.
9 *
10 * UDPv4 GSO support
11 */
12
13#include <linux/skbuff.h>
14#include <net/udp.h>
15#include <net/protocol.h>
16
Or Gerlitzb582ef02014-01-20 13:59:19 +020017static DEFINE_SPINLOCK(udp_offload_lock);
Shlomo Pongratza1d0cd82014-01-22 15:23:29 +020018static struct udp_offload_priv __rcu *udp_offload_base __read_mostly;
Or Gerlitzb582ef02014-01-20 13:59:19 +020019
Shlomo Pongratza664a4f2014-02-02 15:42:10 +020020#define udp_deref_protected(X) rcu_dereference_protected(X, lockdep_is_held(&udp_offload_lock))
21
Or Gerlitzb582ef02014-01-20 13:59:19 +020022struct udp_offload_priv {
23 struct udp_offload *offload;
24 struct rcu_head rcu;
25 struct udp_offload_priv __rcu *next;
26};
27
Tom Herbert8bce6d72014-09-29 20:22:29 -070028static struct sk_buff *__skb_udp_tunnel_segment(struct sk_buff *skb,
29 netdev_features_t features,
30 struct sk_buff *(*gso_inner_segment)(struct sk_buff *skb,
31 netdev_features_t features),
Tom Herbert4bcb8772014-11-04 09:06:52 -080032 __be16 new_protocol, bool is_ipv6)
Tom Herbert155e0102014-07-13 19:49:56 -070033{
34 struct sk_buff *segs = ERR_PTR(-EINVAL);
35 u16 mac_offset = skb->mac_header;
36 int mac_len = skb->mac_len;
37 int tnl_hlen = skb_inner_mac_header(skb) - skb_transport_header(skb);
38 __be16 protocol = skb->protocol;
39 netdev_features_t enc_features;
40 int udp_offset, outer_hlen;
41 unsigned int oldlen;
Tom Herbert4bcb8772014-11-04 09:06:52 -080042 bool need_csum = !!(skb_shinfo(skb)->gso_type &
43 SKB_GSO_UDP_TUNNEL_CSUM);
44 bool offload_csum = false, dont_encap = need_csum;
Tom Herbert155e0102014-07-13 19:49:56 -070045
46 oldlen = (u16)~skb->len;
47
48 if (unlikely(!pskb_may_pull(skb, tnl_hlen)))
49 goto out;
50
51 skb->encapsulation = 0;
52 __skb_pull(skb, tnl_hlen);
53 skb_reset_mac_header(skb);
54 skb_set_network_header(skb, skb_inner_network_offset(skb));
55 skb->mac_len = skb_inner_network_offset(skb);
Tom Herbert8bce6d72014-09-29 20:22:29 -070056 skb->protocol = new_protocol;
Tom Herbert4bcb8772014-11-04 09:06:52 -080057 skb->encap_hdr_csum = need_csum;
Tom Herbert155e0102014-07-13 19:49:56 -070058
Tom Herbert4bcb8772014-11-04 09:06:52 -080059 /* Try to offload checksum if possible */
60 offload_csum = !!(need_csum &&
61 (skb->dev->features &
62 (is_ipv6 ? NETIF_F_V6_CSUM : NETIF_F_V4_CSUM)));
Tom Herbert155e0102014-07-13 19:49:56 -070063
64 /* segment inner packet. */
Florian Westphal1e16aa32014-10-20 13:49:16 +020065 enc_features = skb->dev->hw_enc_features & features;
Tom Herbert8bce6d72014-09-29 20:22:29 -070066 segs = gso_inner_segment(skb, enc_features);
Himangi Saraogi27446442014-07-27 12:38:38 +053067 if (IS_ERR_OR_NULL(segs)) {
Tom Herbert155e0102014-07-13 19:49:56 -070068 skb_gso_error_unwind(skb, protocol, tnl_hlen, mac_offset,
69 mac_len);
70 goto out;
71 }
72
73 outer_hlen = skb_tnl_header_len(skb);
74 udp_offset = outer_hlen - tnl_hlen;
75 skb = segs;
76 do {
77 struct udphdr *uh;
78 int len;
Tom Herbert4bcb8772014-11-04 09:06:52 -080079 __be32 delta;
Tom Herbert155e0102014-07-13 19:49:56 -070080
Tom Herbert4bcb8772014-11-04 09:06:52 -080081 if (dont_encap) {
82 skb->encapsulation = 0;
83 skb->ip_summed = CHECKSUM_NONE;
84 } else {
85 /* Only set up inner headers if we might be offloading
86 * inner checksum.
87 */
88 skb_reset_inner_headers(skb);
89 skb->encapsulation = 1;
90 }
Tom Herbert155e0102014-07-13 19:49:56 -070091
92 skb->mac_len = mac_len;
Tom Herbert4bcb8772014-11-04 09:06:52 -080093 skb->protocol = protocol;
Tom Herbert155e0102014-07-13 19:49:56 -070094
95 skb_push(skb, outer_hlen);
96 skb_reset_mac_header(skb);
97 skb_set_network_header(skb, mac_len);
98 skb_set_transport_header(skb, udp_offset);
99 len = skb->len - udp_offset;
100 uh = udp_hdr(skb);
101 uh->len = htons(len);
102
Tom Herbert4bcb8772014-11-04 09:06:52 -0800103 if (!need_csum)
104 continue;
Tom Herbert155e0102014-07-13 19:49:56 -0700105
Tom Herbert4bcb8772014-11-04 09:06:52 -0800106 delta = htonl(oldlen + len);
107
108 uh->check = ~csum_fold((__force __wsum)
109 ((__force u32)uh->check +
110 (__force u32)delta));
111
112 if (offload_csum) {
113 skb->ip_summed = CHECKSUM_PARTIAL;
114 skb->csum_start = skb_transport_header(skb) - skb->head;
115 skb->csum_offset = offsetof(struct udphdr, check);
116 } else {
Tom Herbert155e0102014-07-13 19:49:56 -0700117 uh->check = gso_make_checksum(skb, ~uh->check);
118
119 if (uh->check == 0)
120 uh->check = CSUM_MANGLED_0;
121 }
Tom Herbert155e0102014-07-13 19:49:56 -0700122 } while ((skb = skb->next));
123out:
124 return segs;
125}
126
Tom Herbert8bce6d72014-09-29 20:22:29 -0700127struct sk_buff *skb_udp_tunnel_segment(struct sk_buff *skb,
128 netdev_features_t features,
129 bool is_ipv6)
130{
131 __be16 protocol = skb->protocol;
132 const struct net_offload **offloads;
133 const struct net_offload *ops;
134 struct sk_buff *segs = ERR_PTR(-EINVAL);
135 struct sk_buff *(*gso_inner_segment)(struct sk_buff *skb,
136 netdev_features_t features);
137
138 rcu_read_lock();
139
140 switch (skb->inner_protocol_type) {
141 case ENCAP_TYPE_ETHER:
142 protocol = skb->inner_protocol;
143 gso_inner_segment = skb_mac_gso_segment;
144 break;
145 case ENCAP_TYPE_IPPROTO:
146 offloads = is_ipv6 ? inet6_offloads : inet_offloads;
147 ops = rcu_dereference(offloads[skb->inner_ipproto]);
148 if (!ops || !ops->callbacks.gso_segment)
149 goto out_unlock;
150 gso_inner_segment = ops->callbacks.gso_segment;
151 break;
152 default:
153 goto out_unlock;
154 }
155
156 segs = __skb_udp_tunnel_segment(skb, features, gso_inner_segment,
Tom Herbert4bcb8772014-11-04 09:06:52 -0800157 protocol, is_ipv6);
Tom Herbert8bce6d72014-09-29 20:22:29 -0700158
159out_unlock:
160 rcu_read_unlock();
161
162 return segs;
163}
164
Daniel Borkmannda5bab02013-06-08 12:56:03 +0200165static struct sk_buff *udp4_ufo_fragment(struct sk_buff *skb,
166 netdev_features_t features)
167{
168 struct sk_buff *segs = ERR_PTR(-EINVAL);
169 unsigned int mss;
Wei-Chun Chao7a7ffba2013-12-26 13:10:22 -0800170 __wsum csum;
Tom Herbertf71470b2014-09-20 14:52:29 -0700171 struct udphdr *uh;
172 struct iphdr *iph;
Wei-Chun Chao7a7ffba2013-12-26 13:10:22 -0800173
174 if (skb->encapsulation &&
Tom Herbert0f4f4ff2014-06-04 17:20:16 -0700175 (skb_shinfo(skb)->gso_type &
176 (SKB_GSO_UDP_TUNNEL|SKB_GSO_UDP_TUNNEL_CSUM))) {
Tom Herbert8bce6d72014-09-29 20:22:29 -0700177 segs = skb_udp_tunnel_segment(skb, features, false);
Wei-Chun Chao7a7ffba2013-12-26 13:10:22 -0800178 goto out;
179 }
Daniel Borkmannda5bab02013-06-08 12:56:03 +0200180
Tom Herbertf71470b2014-09-20 14:52:29 -0700181 if (!pskb_may_pull(skb, sizeof(struct udphdr)))
182 goto out;
183
Daniel Borkmannda5bab02013-06-08 12:56:03 +0200184 mss = skb_shinfo(skb)->gso_size;
185 if (unlikely(skb->len <= mss))
186 goto out;
187
188 if (skb_gso_ok(skb, features | NETIF_F_GSO_ROBUST)) {
189 /* Packet is from an untrusted source, reset gso_segs. */
190 int type = skb_shinfo(skb)->gso_type;
191
192 if (unlikely(type & ~(SKB_GSO_UDP | SKB_GSO_DODGY |
193 SKB_GSO_UDP_TUNNEL |
Tom Herbert0f4f4ff2014-06-04 17:20:16 -0700194 SKB_GSO_UDP_TUNNEL_CSUM |
Eric Dumazetcb32f512013-10-19 11:42:57 -0700195 SKB_GSO_IPIP |
Tom Herbert4749c092014-06-04 17:20:23 -0700196 SKB_GSO_GRE | SKB_GSO_GRE_CSUM |
197 SKB_GSO_MPLS) ||
Daniel Borkmannda5bab02013-06-08 12:56:03 +0200198 !(type & (SKB_GSO_UDP))))
199 goto out;
200
201 skb_shinfo(skb)->gso_segs = DIV_ROUND_UP(skb->len, mss);
202
203 segs = NULL;
204 goto out;
205 }
206
Wei-Chun Chao7a7ffba2013-12-26 13:10:22 -0800207 /* Do software UFO. Complete and fill in the UDP checksum as
208 * HW cannot do checksum of UDP packets sent as multiple
209 * IP fragments.
210 */
Tom Herbertf71470b2014-09-20 14:52:29 -0700211
212 uh = udp_hdr(skb);
213 iph = ip_hdr(skb);
214
215 uh->check = 0;
216 csum = skb_checksum(skb, 0, skb->len, 0);
217 uh->check = udp_v4_check(skb->len, iph->saddr, iph->daddr, csum);
218 if (uh->check == 0)
219 uh->check = CSUM_MANGLED_0;
220
Wei-Chun Chao7a7ffba2013-12-26 13:10:22 -0800221 skb->ip_summed = CHECKSUM_NONE;
222
Daniel Borkmannda5bab02013-06-08 12:56:03 +0200223 /* Fragment the skb. IP headers of the fragments are updated in
224 * inet_gso_segment()
225 */
Wei-Chun Chao7a7ffba2013-12-26 13:10:22 -0800226 segs = skb_segment(skb, features);
Daniel Borkmannda5bab02013-06-08 12:56:03 +0200227out:
228 return segs;
229}
230
Or Gerlitzb582ef02014-01-20 13:59:19 +0200231int udp_add_offload(struct udp_offload *uo)
232{
Or Gerlitzb5aaab12014-01-29 18:08:59 +0200233 struct udp_offload_priv *new_offload = kzalloc(sizeof(*new_offload), GFP_ATOMIC);
Or Gerlitzb582ef02014-01-20 13:59:19 +0200234
235 if (!new_offload)
236 return -ENOMEM;
237
238 new_offload->offload = uo;
239
240 spin_lock(&udp_offload_lock);
Shlomo Pongratza664a4f2014-02-02 15:42:10 +0200241 new_offload->next = udp_offload_base;
242 rcu_assign_pointer(udp_offload_base, new_offload);
Or Gerlitzb582ef02014-01-20 13:59:19 +0200243 spin_unlock(&udp_offload_lock);
244
245 return 0;
246}
247EXPORT_SYMBOL(udp_add_offload);
248
249static void udp_offload_free_routine(struct rcu_head *head)
250{
251 struct udp_offload_priv *ou_priv = container_of(head, struct udp_offload_priv, rcu);
252 kfree(ou_priv);
253}
254
255void udp_del_offload(struct udp_offload *uo)
256{
257 struct udp_offload_priv __rcu **head = &udp_offload_base;
258 struct udp_offload_priv *uo_priv;
259
260 spin_lock(&udp_offload_lock);
261
Shlomo Pongratza664a4f2014-02-02 15:42:10 +0200262 uo_priv = udp_deref_protected(*head);
Or Gerlitzb582ef02014-01-20 13:59:19 +0200263 for (; uo_priv != NULL;
Shlomo Pongratza664a4f2014-02-02 15:42:10 +0200264 uo_priv = udp_deref_protected(*head)) {
Or Gerlitzb582ef02014-01-20 13:59:19 +0200265 if (uo_priv->offload == uo) {
Shlomo Pongratza664a4f2014-02-02 15:42:10 +0200266 rcu_assign_pointer(*head,
267 udp_deref_protected(uo_priv->next));
Or Gerlitzb582ef02014-01-20 13:59:19 +0200268 goto unlock;
269 }
270 head = &uo_priv->next;
271 }
Shlomo Pongratza1d0cd82014-01-22 15:23:29 +0200272 pr_warn("udp_del_offload: didn't find offload for port %d\n", ntohs(uo->port));
Or Gerlitzb582ef02014-01-20 13:59:19 +0200273unlock:
274 spin_unlock(&udp_offload_lock);
275 if (uo_priv != NULL)
276 call_rcu(&uo_priv->rcu, udp_offload_free_routine);
277}
278EXPORT_SYMBOL(udp_del_offload);
279
Tom Herbert57c67ff2014-08-22 13:34:44 -0700280struct sk_buff **udp_gro_receive(struct sk_buff **head, struct sk_buff *skb,
281 struct udphdr *uh)
Or Gerlitzb582ef02014-01-20 13:59:19 +0200282{
283 struct udp_offload_priv *uo_priv;
284 struct sk_buff *p, **pp = NULL;
Tom Herbert57c67ff2014-08-22 13:34:44 -0700285 struct udphdr *uh2;
286 unsigned int off = skb_gro_offset(skb);
Or Gerlitzb582ef02014-01-20 13:59:19 +0200287 int flush = 1;
288
289 if (NAPI_GRO_CB(skb)->udp_mark ||
Tom Herbert662880f2014-08-27 21:26:56 -0700290 (skb->ip_summed != CHECKSUM_PARTIAL &&
291 NAPI_GRO_CB(skb)->csum_cnt == 0 &&
292 !NAPI_GRO_CB(skb)->csum_valid))
Or Gerlitzb582ef02014-01-20 13:59:19 +0200293 goto out;
294
295 /* mark that this skb passed once through the udp gro layer */
296 NAPI_GRO_CB(skb)->udp_mark = 1;
Or Gerlitzb582ef02014-01-20 13:59:19 +0200297
298 rcu_read_lock();
299 uo_priv = rcu_dereference(udp_offload_base);
300 for (; uo_priv != NULL; uo_priv = rcu_dereference(uo_priv->next)) {
301 if (uo_priv->offload->port == uh->dest &&
302 uo_priv->offload->callbacks.gro_receive)
303 goto unflush;
304 }
305 goto out_unlock;
306
307unflush:
308 flush = 0;
309
310 for (p = *head; p; p = p->next) {
311 if (!NAPI_GRO_CB(p)->same_flow)
312 continue;
313
314 uh2 = (struct udphdr *)(p->data + off);
Tom Herbert57c67ff2014-08-22 13:34:44 -0700315
316 /* Match ports and either checksums are either both zero
317 * or nonzero.
318 */
319 if ((*(u32 *)&uh->source != *(u32 *)&uh2->source) ||
320 (!uh->check ^ !uh2->check)) {
Or Gerlitzb582ef02014-01-20 13:59:19 +0200321 NAPI_GRO_CB(p)->same_flow = 0;
322 continue;
323 }
324 }
325
326 skb_gro_pull(skb, sizeof(struct udphdr)); /* pull encapsulating udp header */
Tom Herbert6bae1d42014-06-10 18:54:26 -0700327 skb_gro_postpull_rcsum(skb, uh, sizeof(struct udphdr));
Tom Herbertafe93322014-09-17 12:25:57 -0700328 NAPI_GRO_CB(skb)->proto = uo_priv->offload->ipproto;
Or Gerlitzb582ef02014-01-20 13:59:19 +0200329 pp = uo_priv->offload->callbacks.gro_receive(head, skb);
330
331out_unlock:
332 rcu_read_unlock();
333out:
334 NAPI_GRO_CB(skb)->flush |= flush;
335 return pp;
336}
337
Tom Herbert57c67ff2014-08-22 13:34:44 -0700338static struct sk_buff **udp4_gro_receive(struct sk_buff **head,
339 struct sk_buff *skb)
340{
341 struct udphdr *uh = udp_gro_udphdr(skb);
342
Tom Herbert2abb7cd2014-08-31 15:12:43 -0700343 if (unlikely(!uh))
344 goto flush;
Tom Herbert57c67ff2014-08-22 13:34:44 -0700345
Tom Herbert2abb7cd2014-08-31 15:12:43 -0700346 /* Don't bother verifying checksum if we're going to flush anyway. */
Scott Wood2d8f7e22014-09-10 21:23:18 -0500347 if (NAPI_GRO_CB(skb)->flush)
Tom Herbert2abb7cd2014-08-31 15:12:43 -0700348 goto skip;
349
350 if (skb_gro_checksum_validate_zero_check(skb, IPPROTO_UDP, uh->check,
351 inet_gro_compute_pseudo))
352 goto flush;
353 else if (uh->check)
354 skb_gro_checksum_try_convert(skb, IPPROTO_UDP, uh->check,
355 inet_gro_compute_pseudo);
356skip:
Tom Herbertefc98d02014-10-03 15:48:08 -0700357 NAPI_GRO_CB(skb)->is_ipv6 = 0;
Tom Herbert57c67ff2014-08-22 13:34:44 -0700358 return udp_gro_receive(head, skb, uh);
Tom Herbert2abb7cd2014-08-31 15:12:43 -0700359
360flush:
361 NAPI_GRO_CB(skb)->flush = 1;
362 return NULL;
Tom Herbert57c67ff2014-08-22 13:34:44 -0700363}
364
365int udp_gro_complete(struct sk_buff *skb, int nhoff)
Or Gerlitzb582ef02014-01-20 13:59:19 +0200366{
367 struct udp_offload_priv *uo_priv;
368 __be16 newlen = htons(skb->len - nhoff);
369 struct udphdr *uh = (struct udphdr *)(skb->data + nhoff);
370 int err = -ENOSYS;
371
372 uh->len = newlen;
373
374 rcu_read_lock();
375
376 uo_priv = rcu_dereference(udp_offload_base);
377 for (; uo_priv != NULL; uo_priv = rcu_dereference(uo_priv->next)) {
378 if (uo_priv->offload->port == uh->dest &&
379 uo_priv->offload->callbacks.gro_complete)
380 break;
381 }
382
Tom Herbertafe93322014-09-17 12:25:57 -0700383 if (uo_priv != NULL) {
384 NAPI_GRO_CB(skb)->proto = uo_priv->offload->ipproto;
Or Gerlitzb582ef02014-01-20 13:59:19 +0200385 err = uo_priv->offload->callbacks.gro_complete(skb, nhoff + sizeof(struct udphdr));
Tom Herbertafe93322014-09-17 12:25:57 -0700386 }
Or Gerlitzb582ef02014-01-20 13:59:19 +0200387
388 rcu_read_unlock();
389 return err;
390}
391
Eric Dumazet72bb17b2014-09-09 08:29:12 -0700392static int udp4_gro_complete(struct sk_buff *skb, int nhoff)
Tom Herbert57c67ff2014-08-22 13:34:44 -0700393{
394 const struct iphdr *iph = ip_hdr(skb);
395 struct udphdr *uh = (struct udphdr *)(skb->data + nhoff);
396
397 if (uh->check)
398 uh->check = ~udp_v4_check(skb->len - nhoff, iph->saddr,
399 iph->daddr, 0);
400
401 return udp_gro_complete(skb, nhoff);
402}
403
Daniel Borkmannda5bab02013-06-08 12:56:03 +0200404static const struct net_offload udpv4_offload = {
405 .callbacks = {
Daniel Borkmannda5bab02013-06-08 12:56:03 +0200406 .gso_segment = udp4_ufo_fragment,
Tom Herbert57c67ff2014-08-22 13:34:44 -0700407 .gro_receive = udp4_gro_receive,
408 .gro_complete = udp4_gro_complete,
Daniel Borkmannda5bab02013-06-08 12:56:03 +0200409 },
410};
411
412int __init udpv4_offload_init(void)
413{
414 return inet_add_offload(&udpv4_offload, IPPROTO_UDP);
415}