Daniel Borkmann | da5bab0 | 2013-06-08 12:56:03 +0200 | [diff] [blame] | 1 | /* |
| 2 | * IPV4 GSO/GRO offload support |
| 3 | * Linux INET implementation |
| 4 | * |
| 5 | * This program is free software; you can redistribute it and/or |
| 6 | * modify it under the terms of the GNU General Public License |
| 7 | * as published by the Free Software Foundation; either version |
| 8 | * 2 of the License, or (at your option) any later version. |
| 9 | * |
| 10 | * UDPv4 GSO support |
| 11 | */ |
| 12 | |
| 13 | #include <linux/skbuff.h> |
| 14 | #include <net/udp.h> |
| 15 | #include <net/protocol.h> |
| 16 | |
Or Gerlitz | b582ef0 | 2014-01-20 13:59:19 +0200 | [diff] [blame] | 17 | static DEFINE_SPINLOCK(udp_offload_lock); |
Shlomo Pongratz | a1d0cd8 | 2014-01-22 15:23:29 +0200 | [diff] [blame] | 18 | static struct udp_offload_priv __rcu *udp_offload_base __read_mostly; |
Or Gerlitz | b582ef0 | 2014-01-20 13:59:19 +0200 | [diff] [blame] | 19 | |
Shlomo Pongratz | a664a4f | 2014-02-02 15:42:10 +0200 | [diff] [blame] | 20 | #define udp_deref_protected(X) rcu_dereference_protected(X, lockdep_is_held(&udp_offload_lock)) |
| 21 | |
Or Gerlitz | b582ef0 | 2014-01-20 13:59:19 +0200 | [diff] [blame] | 22 | struct udp_offload_priv { |
| 23 | struct udp_offload *offload; |
| 24 | struct rcu_head rcu; |
| 25 | struct udp_offload_priv __rcu *next; |
| 26 | }; |
| 27 | |
Tom Herbert | 8bce6d7 | 2014-09-29 20:22:29 -0700 | [diff] [blame] | 28 | static struct sk_buff *__skb_udp_tunnel_segment(struct sk_buff *skb, |
| 29 | netdev_features_t features, |
| 30 | struct sk_buff *(*gso_inner_segment)(struct sk_buff *skb, |
| 31 | netdev_features_t features), |
| 32 | __be16 new_protocol) |
Tom Herbert | 155e010 | 2014-07-13 19:49:56 -0700 | [diff] [blame] | 33 | { |
| 34 | struct sk_buff *segs = ERR_PTR(-EINVAL); |
| 35 | u16 mac_offset = skb->mac_header; |
| 36 | int mac_len = skb->mac_len; |
| 37 | int tnl_hlen = skb_inner_mac_header(skb) - skb_transport_header(skb); |
| 38 | __be16 protocol = skb->protocol; |
| 39 | netdev_features_t enc_features; |
| 40 | int udp_offset, outer_hlen; |
| 41 | unsigned int oldlen; |
| 42 | bool need_csum; |
| 43 | |
| 44 | oldlen = (u16)~skb->len; |
| 45 | |
| 46 | if (unlikely(!pskb_may_pull(skb, tnl_hlen))) |
| 47 | goto out; |
| 48 | |
| 49 | skb->encapsulation = 0; |
| 50 | __skb_pull(skb, tnl_hlen); |
| 51 | skb_reset_mac_header(skb); |
| 52 | skb_set_network_header(skb, skb_inner_network_offset(skb)); |
| 53 | skb->mac_len = skb_inner_network_offset(skb); |
Tom Herbert | 8bce6d7 | 2014-09-29 20:22:29 -0700 | [diff] [blame] | 54 | skb->protocol = new_protocol; |
Tom Herbert | 155e010 | 2014-07-13 19:49:56 -0700 | [diff] [blame] | 55 | |
| 56 | need_csum = !!(skb_shinfo(skb)->gso_type & SKB_GSO_UDP_TUNNEL_CSUM); |
| 57 | if (need_csum) |
| 58 | skb->encap_hdr_csum = 1; |
| 59 | |
| 60 | /* segment inner packet. */ |
| 61 | enc_features = skb->dev->hw_enc_features & netif_skb_features(skb); |
Tom Herbert | 8bce6d7 | 2014-09-29 20:22:29 -0700 | [diff] [blame] | 62 | segs = gso_inner_segment(skb, enc_features); |
Himangi Saraogi | 2744644 | 2014-07-27 12:38:38 +0530 | [diff] [blame] | 63 | if (IS_ERR_OR_NULL(segs)) { |
Tom Herbert | 155e010 | 2014-07-13 19:49:56 -0700 | [diff] [blame] | 64 | skb_gso_error_unwind(skb, protocol, tnl_hlen, mac_offset, |
| 65 | mac_len); |
| 66 | goto out; |
| 67 | } |
| 68 | |
| 69 | outer_hlen = skb_tnl_header_len(skb); |
| 70 | udp_offset = outer_hlen - tnl_hlen; |
| 71 | skb = segs; |
| 72 | do { |
| 73 | struct udphdr *uh; |
| 74 | int len; |
| 75 | |
| 76 | skb_reset_inner_headers(skb); |
| 77 | skb->encapsulation = 1; |
| 78 | |
| 79 | skb->mac_len = mac_len; |
| 80 | |
| 81 | skb_push(skb, outer_hlen); |
| 82 | skb_reset_mac_header(skb); |
| 83 | skb_set_network_header(skb, mac_len); |
| 84 | skb_set_transport_header(skb, udp_offset); |
| 85 | len = skb->len - udp_offset; |
| 86 | uh = udp_hdr(skb); |
| 87 | uh->len = htons(len); |
| 88 | |
| 89 | if (need_csum) { |
| 90 | __be32 delta = htonl(oldlen + len); |
| 91 | |
| 92 | uh->check = ~csum_fold((__force __wsum) |
| 93 | ((__force u32)uh->check + |
| 94 | (__force u32)delta)); |
| 95 | uh->check = gso_make_checksum(skb, ~uh->check); |
| 96 | |
| 97 | if (uh->check == 0) |
| 98 | uh->check = CSUM_MANGLED_0; |
| 99 | } |
| 100 | |
| 101 | skb->protocol = protocol; |
| 102 | } while ((skb = skb->next)); |
| 103 | out: |
| 104 | return segs; |
| 105 | } |
| 106 | |
Tom Herbert | 8bce6d7 | 2014-09-29 20:22:29 -0700 | [diff] [blame] | 107 | struct sk_buff *skb_udp_tunnel_segment(struct sk_buff *skb, |
| 108 | netdev_features_t features, |
| 109 | bool is_ipv6) |
| 110 | { |
| 111 | __be16 protocol = skb->protocol; |
| 112 | const struct net_offload **offloads; |
| 113 | const struct net_offload *ops; |
| 114 | struct sk_buff *segs = ERR_PTR(-EINVAL); |
| 115 | struct sk_buff *(*gso_inner_segment)(struct sk_buff *skb, |
| 116 | netdev_features_t features); |
| 117 | |
| 118 | rcu_read_lock(); |
| 119 | |
| 120 | switch (skb->inner_protocol_type) { |
| 121 | case ENCAP_TYPE_ETHER: |
| 122 | protocol = skb->inner_protocol; |
| 123 | gso_inner_segment = skb_mac_gso_segment; |
| 124 | break; |
| 125 | case ENCAP_TYPE_IPPROTO: |
| 126 | offloads = is_ipv6 ? inet6_offloads : inet_offloads; |
| 127 | ops = rcu_dereference(offloads[skb->inner_ipproto]); |
| 128 | if (!ops || !ops->callbacks.gso_segment) |
| 129 | goto out_unlock; |
| 130 | gso_inner_segment = ops->callbacks.gso_segment; |
| 131 | break; |
| 132 | default: |
| 133 | goto out_unlock; |
| 134 | } |
| 135 | |
| 136 | segs = __skb_udp_tunnel_segment(skb, features, gso_inner_segment, |
| 137 | protocol); |
| 138 | |
| 139 | out_unlock: |
| 140 | rcu_read_unlock(); |
| 141 | |
| 142 | return segs; |
| 143 | } |
| 144 | |
Daniel Borkmann | da5bab0 | 2013-06-08 12:56:03 +0200 | [diff] [blame] | 145 | static struct sk_buff *udp4_ufo_fragment(struct sk_buff *skb, |
| 146 | netdev_features_t features) |
| 147 | { |
| 148 | struct sk_buff *segs = ERR_PTR(-EINVAL); |
| 149 | unsigned int mss; |
Wei-Chun Chao | 7a7ffba | 2013-12-26 13:10:22 -0800 | [diff] [blame] | 150 | __wsum csum; |
Tom Herbert | f71470b | 2014-09-20 14:52:29 -0700 | [diff] [blame] | 151 | struct udphdr *uh; |
| 152 | struct iphdr *iph; |
Wei-Chun Chao | 7a7ffba | 2013-12-26 13:10:22 -0800 | [diff] [blame] | 153 | |
| 154 | if (skb->encapsulation && |
Tom Herbert | 0f4f4ff | 2014-06-04 17:20:16 -0700 | [diff] [blame] | 155 | (skb_shinfo(skb)->gso_type & |
| 156 | (SKB_GSO_UDP_TUNNEL|SKB_GSO_UDP_TUNNEL_CSUM))) { |
Tom Herbert | 8bce6d7 | 2014-09-29 20:22:29 -0700 | [diff] [blame] | 157 | segs = skb_udp_tunnel_segment(skb, features, false); |
Wei-Chun Chao | 7a7ffba | 2013-12-26 13:10:22 -0800 | [diff] [blame] | 158 | goto out; |
| 159 | } |
Daniel Borkmann | da5bab0 | 2013-06-08 12:56:03 +0200 | [diff] [blame] | 160 | |
Tom Herbert | f71470b | 2014-09-20 14:52:29 -0700 | [diff] [blame] | 161 | if (!pskb_may_pull(skb, sizeof(struct udphdr))) |
| 162 | goto out; |
| 163 | |
Daniel Borkmann | da5bab0 | 2013-06-08 12:56:03 +0200 | [diff] [blame] | 164 | mss = skb_shinfo(skb)->gso_size; |
| 165 | if (unlikely(skb->len <= mss)) |
| 166 | goto out; |
| 167 | |
| 168 | if (skb_gso_ok(skb, features | NETIF_F_GSO_ROBUST)) { |
| 169 | /* Packet is from an untrusted source, reset gso_segs. */ |
| 170 | int type = skb_shinfo(skb)->gso_type; |
| 171 | |
| 172 | if (unlikely(type & ~(SKB_GSO_UDP | SKB_GSO_DODGY | |
| 173 | SKB_GSO_UDP_TUNNEL | |
Tom Herbert | 0f4f4ff | 2014-06-04 17:20:16 -0700 | [diff] [blame] | 174 | SKB_GSO_UDP_TUNNEL_CSUM | |
Eric Dumazet | cb32f51 | 2013-10-19 11:42:57 -0700 | [diff] [blame] | 175 | SKB_GSO_IPIP | |
Tom Herbert | 4749c09 | 2014-06-04 17:20:23 -0700 | [diff] [blame] | 176 | SKB_GSO_GRE | SKB_GSO_GRE_CSUM | |
| 177 | SKB_GSO_MPLS) || |
Daniel Borkmann | da5bab0 | 2013-06-08 12:56:03 +0200 | [diff] [blame] | 178 | !(type & (SKB_GSO_UDP)))) |
| 179 | goto out; |
| 180 | |
| 181 | skb_shinfo(skb)->gso_segs = DIV_ROUND_UP(skb->len, mss); |
| 182 | |
| 183 | segs = NULL; |
| 184 | goto out; |
| 185 | } |
| 186 | |
Wei-Chun Chao | 7a7ffba | 2013-12-26 13:10:22 -0800 | [diff] [blame] | 187 | /* Do software UFO. Complete and fill in the UDP checksum as |
| 188 | * HW cannot do checksum of UDP packets sent as multiple |
| 189 | * IP fragments. |
| 190 | */ |
Tom Herbert | f71470b | 2014-09-20 14:52:29 -0700 | [diff] [blame] | 191 | |
| 192 | uh = udp_hdr(skb); |
| 193 | iph = ip_hdr(skb); |
| 194 | |
| 195 | uh->check = 0; |
| 196 | csum = skb_checksum(skb, 0, skb->len, 0); |
| 197 | uh->check = udp_v4_check(skb->len, iph->saddr, iph->daddr, csum); |
| 198 | if (uh->check == 0) |
| 199 | uh->check = CSUM_MANGLED_0; |
| 200 | |
Wei-Chun Chao | 7a7ffba | 2013-12-26 13:10:22 -0800 | [diff] [blame] | 201 | skb->ip_summed = CHECKSUM_NONE; |
| 202 | |
Daniel Borkmann | da5bab0 | 2013-06-08 12:56:03 +0200 | [diff] [blame] | 203 | /* Fragment the skb. IP headers of the fragments are updated in |
| 204 | * inet_gso_segment() |
| 205 | */ |
Wei-Chun Chao | 7a7ffba | 2013-12-26 13:10:22 -0800 | [diff] [blame] | 206 | segs = skb_segment(skb, features); |
Daniel Borkmann | da5bab0 | 2013-06-08 12:56:03 +0200 | [diff] [blame] | 207 | out: |
| 208 | return segs; |
| 209 | } |
| 210 | |
Or Gerlitz | b582ef0 | 2014-01-20 13:59:19 +0200 | [diff] [blame] | 211 | int udp_add_offload(struct udp_offload *uo) |
| 212 | { |
Or Gerlitz | b5aaab1 | 2014-01-29 18:08:59 +0200 | [diff] [blame] | 213 | struct udp_offload_priv *new_offload = kzalloc(sizeof(*new_offload), GFP_ATOMIC); |
Or Gerlitz | b582ef0 | 2014-01-20 13:59:19 +0200 | [diff] [blame] | 214 | |
| 215 | if (!new_offload) |
| 216 | return -ENOMEM; |
| 217 | |
| 218 | new_offload->offload = uo; |
| 219 | |
| 220 | spin_lock(&udp_offload_lock); |
Shlomo Pongratz | a664a4f | 2014-02-02 15:42:10 +0200 | [diff] [blame] | 221 | new_offload->next = udp_offload_base; |
| 222 | rcu_assign_pointer(udp_offload_base, new_offload); |
Or Gerlitz | b582ef0 | 2014-01-20 13:59:19 +0200 | [diff] [blame] | 223 | spin_unlock(&udp_offload_lock); |
| 224 | |
| 225 | return 0; |
| 226 | } |
| 227 | EXPORT_SYMBOL(udp_add_offload); |
| 228 | |
| 229 | static void udp_offload_free_routine(struct rcu_head *head) |
| 230 | { |
| 231 | struct udp_offload_priv *ou_priv = container_of(head, struct udp_offload_priv, rcu); |
| 232 | kfree(ou_priv); |
| 233 | } |
| 234 | |
| 235 | void udp_del_offload(struct udp_offload *uo) |
| 236 | { |
| 237 | struct udp_offload_priv __rcu **head = &udp_offload_base; |
| 238 | struct udp_offload_priv *uo_priv; |
| 239 | |
| 240 | spin_lock(&udp_offload_lock); |
| 241 | |
Shlomo Pongratz | a664a4f | 2014-02-02 15:42:10 +0200 | [diff] [blame] | 242 | uo_priv = udp_deref_protected(*head); |
Or Gerlitz | b582ef0 | 2014-01-20 13:59:19 +0200 | [diff] [blame] | 243 | for (; uo_priv != NULL; |
Shlomo Pongratz | a664a4f | 2014-02-02 15:42:10 +0200 | [diff] [blame] | 244 | uo_priv = udp_deref_protected(*head)) { |
Or Gerlitz | b582ef0 | 2014-01-20 13:59:19 +0200 | [diff] [blame] | 245 | if (uo_priv->offload == uo) { |
Shlomo Pongratz | a664a4f | 2014-02-02 15:42:10 +0200 | [diff] [blame] | 246 | rcu_assign_pointer(*head, |
| 247 | udp_deref_protected(uo_priv->next)); |
Or Gerlitz | b582ef0 | 2014-01-20 13:59:19 +0200 | [diff] [blame] | 248 | goto unlock; |
| 249 | } |
| 250 | head = &uo_priv->next; |
| 251 | } |
Shlomo Pongratz | a1d0cd8 | 2014-01-22 15:23:29 +0200 | [diff] [blame] | 252 | pr_warn("udp_del_offload: didn't find offload for port %d\n", ntohs(uo->port)); |
Or Gerlitz | b582ef0 | 2014-01-20 13:59:19 +0200 | [diff] [blame] | 253 | unlock: |
| 254 | spin_unlock(&udp_offload_lock); |
| 255 | if (uo_priv != NULL) |
| 256 | call_rcu(&uo_priv->rcu, udp_offload_free_routine); |
| 257 | } |
| 258 | EXPORT_SYMBOL(udp_del_offload); |
| 259 | |
Tom Herbert | 57c67ff | 2014-08-22 13:34:44 -0700 | [diff] [blame] | 260 | struct sk_buff **udp_gro_receive(struct sk_buff **head, struct sk_buff *skb, |
| 261 | struct udphdr *uh) |
Or Gerlitz | b582ef0 | 2014-01-20 13:59:19 +0200 | [diff] [blame] | 262 | { |
| 263 | struct udp_offload_priv *uo_priv; |
| 264 | struct sk_buff *p, **pp = NULL; |
Tom Herbert | 57c67ff | 2014-08-22 13:34:44 -0700 | [diff] [blame] | 265 | struct udphdr *uh2; |
| 266 | unsigned int off = skb_gro_offset(skb); |
Or Gerlitz | b582ef0 | 2014-01-20 13:59:19 +0200 | [diff] [blame] | 267 | int flush = 1; |
| 268 | |
| 269 | if (NAPI_GRO_CB(skb)->udp_mark || |
Tom Herbert | 662880f | 2014-08-27 21:26:56 -0700 | [diff] [blame] | 270 | (skb->ip_summed != CHECKSUM_PARTIAL && |
| 271 | NAPI_GRO_CB(skb)->csum_cnt == 0 && |
| 272 | !NAPI_GRO_CB(skb)->csum_valid)) |
Or Gerlitz | b582ef0 | 2014-01-20 13:59:19 +0200 | [diff] [blame] | 273 | goto out; |
| 274 | |
| 275 | /* mark that this skb passed once through the udp gro layer */ |
| 276 | NAPI_GRO_CB(skb)->udp_mark = 1; |
Or Gerlitz | b582ef0 | 2014-01-20 13:59:19 +0200 | [diff] [blame] | 277 | |
| 278 | rcu_read_lock(); |
| 279 | uo_priv = rcu_dereference(udp_offload_base); |
| 280 | for (; uo_priv != NULL; uo_priv = rcu_dereference(uo_priv->next)) { |
| 281 | if (uo_priv->offload->port == uh->dest && |
| 282 | uo_priv->offload->callbacks.gro_receive) |
| 283 | goto unflush; |
| 284 | } |
| 285 | goto out_unlock; |
| 286 | |
| 287 | unflush: |
| 288 | flush = 0; |
| 289 | |
| 290 | for (p = *head; p; p = p->next) { |
| 291 | if (!NAPI_GRO_CB(p)->same_flow) |
| 292 | continue; |
| 293 | |
| 294 | uh2 = (struct udphdr *)(p->data + off); |
Tom Herbert | 57c67ff | 2014-08-22 13:34:44 -0700 | [diff] [blame] | 295 | |
| 296 | /* Match ports and either checksums are either both zero |
| 297 | * or nonzero. |
| 298 | */ |
| 299 | if ((*(u32 *)&uh->source != *(u32 *)&uh2->source) || |
| 300 | (!uh->check ^ !uh2->check)) { |
Or Gerlitz | b582ef0 | 2014-01-20 13:59:19 +0200 | [diff] [blame] | 301 | NAPI_GRO_CB(p)->same_flow = 0; |
| 302 | continue; |
| 303 | } |
| 304 | } |
| 305 | |
| 306 | skb_gro_pull(skb, sizeof(struct udphdr)); /* pull encapsulating udp header */ |
Tom Herbert | 6bae1d4 | 2014-06-10 18:54:26 -0700 | [diff] [blame] | 307 | skb_gro_postpull_rcsum(skb, uh, sizeof(struct udphdr)); |
Tom Herbert | afe9332 | 2014-09-17 12:25:57 -0700 | [diff] [blame] | 308 | NAPI_GRO_CB(skb)->proto = uo_priv->offload->ipproto; |
Or Gerlitz | b582ef0 | 2014-01-20 13:59:19 +0200 | [diff] [blame] | 309 | pp = uo_priv->offload->callbacks.gro_receive(head, skb); |
| 310 | |
| 311 | out_unlock: |
| 312 | rcu_read_unlock(); |
| 313 | out: |
| 314 | NAPI_GRO_CB(skb)->flush |= flush; |
| 315 | return pp; |
| 316 | } |
| 317 | |
Tom Herbert | 57c67ff | 2014-08-22 13:34:44 -0700 | [diff] [blame] | 318 | static struct sk_buff **udp4_gro_receive(struct sk_buff **head, |
| 319 | struct sk_buff *skb) |
| 320 | { |
| 321 | struct udphdr *uh = udp_gro_udphdr(skb); |
| 322 | |
Tom Herbert | 2abb7cd | 2014-08-31 15:12:43 -0700 | [diff] [blame] | 323 | if (unlikely(!uh)) |
| 324 | goto flush; |
Tom Herbert | 57c67ff | 2014-08-22 13:34:44 -0700 | [diff] [blame] | 325 | |
Tom Herbert | 2abb7cd | 2014-08-31 15:12:43 -0700 | [diff] [blame] | 326 | /* Don't bother verifying checksum if we're going to flush anyway. */ |
Scott Wood | 2d8f7e2 | 2014-09-10 21:23:18 -0500 | [diff] [blame] | 327 | if (NAPI_GRO_CB(skb)->flush) |
Tom Herbert | 2abb7cd | 2014-08-31 15:12:43 -0700 | [diff] [blame] | 328 | goto skip; |
| 329 | |
| 330 | if (skb_gro_checksum_validate_zero_check(skb, IPPROTO_UDP, uh->check, |
| 331 | inet_gro_compute_pseudo)) |
| 332 | goto flush; |
| 333 | else if (uh->check) |
| 334 | skb_gro_checksum_try_convert(skb, IPPROTO_UDP, uh->check, |
| 335 | inet_gro_compute_pseudo); |
| 336 | skip: |
Tom Herbert | efc98d0 | 2014-10-03 15:48:08 -0700 | [diff] [blame] | 337 | NAPI_GRO_CB(skb)->is_ipv6 = 0; |
Tom Herbert | 57c67ff | 2014-08-22 13:34:44 -0700 | [diff] [blame] | 338 | return udp_gro_receive(head, skb, uh); |
Tom Herbert | 2abb7cd | 2014-08-31 15:12:43 -0700 | [diff] [blame] | 339 | |
| 340 | flush: |
| 341 | NAPI_GRO_CB(skb)->flush = 1; |
| 342 | return NULL; |
Tom Herbert | 57c67ff | 2014-08-22 13:34:44 -0700 | [diff] [blame] | 343 | } |
| 344 | |
| 345 | int udp_gro_complete(struct sk_buff *skb, int nhoff) |
Or Gerlitz | b582ef0 | 2014-01-20 13:59:19 +0200 | [diff] [blame] | 346 | { |
| 347 | struct udp_offload_priv *uo_priv; |
| 348 | __be16 newlen = htons(skb->len - nhoff); |
| 349 | struct udphdr *uh = (struct udphdr *)(skb->data + nhoff); |
| 350 | int err = -ENOSYS; |
| 351 | |
| 352 | uh->len = newlen; |
| 353 | |
| 354 | rcu_read_lock(); |
| 355 | |
| 356 | uo_priv = rcu_dereference(udp_offload_base); |
| 357 | for (; uo_priv != NULL; uo_priv = rcu_dereference(uo_priv->next)) { |
| 358 | if (uo_priv->offload->port == uh->dest && |
| 359 | uo_priv->offload->callbacks.gro_complete) |
| 360 | break; |
| 361 | } |
| 362 | |
Tom Herbert | afe9332 | 2014-09-17 12:25:57 -0700 | [diff] [blame] | 363 | if (uo_priv != NULL) { |
| 364 | NAPI_GRO_CB(skb)->proto = uo_priv->offload->ipproto; |
Or Gerlitz | b582ef0 | 2014-01-20 13:59:19 +0200 | [diff] [blame] | 365 | err = uo_priv->offload->callbacks.gro_complete(skb, nhoff + sizeof(struct udphdr)); |
Tom Herbert | afe9332 | 2014-09-17 12:25:57 -0700 | [diff] [blame] | 366 | } |
Or Gerlitz | b582ef0 | 2014-01-20 13:59:19 +0200 | [diff] [blame] | 367 | |
| 368 | rcu_read_unlock(); |
| 369 | return err; |
| 370 | } |
| 371 | |
Eric Dumazet | 72bb17b | 2014-09-09 08:29:12 -0700 | [diff] [blame] | 372 | static int udp4_gro_complete(struct sk_buff *skb, int nhoff) |
Tom Herbert | 57c67ff | 2014-08-22 13:34:44 -0700 | [diff] [blame] | 373 | { |
| 374 | const struct iphdr *iph = ip_hdr(skb); |
| 375 | struct udphdr *uh = (struct udphdr *)(skb->data + nhoff); |
| 376 | |
| 377 | if (uh->check) |
| 378 | uh->check = ~udp_v4_check(skb->len - nhoff, iph->saddr, |
| 379 | iph->daddr, 0); |
| 380 | |
| 381 | return udp_gro_complete(skb, nhoff); |
| 382 | } |
| 383 | |
Daniel Borkmann | da5bab0 | 2013-06-08 12:56:03 +0200 | [diff] [blame] | 384 | static const struct net_offload udpv4_offload = { |
| 385 | .callbacks = { |
Daniel Borkmann | da5bab0 | 2013-06-08 12:56:03 +0200 | [diff] [blame] | 386 | .gso_segment = udp4_ufo_fragment, |
Tom Herbert | 57c67ff | 2014-08-22 13:34:44 -0700 | [diff] [blame] | 387 | .gro_receive = udp4_gro_receive, |
| 388 | .gro_complete = udp4_gro_complete, |
Daniel Borkmann | da5bab0 | 2013-06-08 12:56:03 +0200 | [diff] [blame] | 389 | }, |
| 390 | }; |
| 391 | |
| 392 | int __init udpv4_offload_init(void) |
| 393 | { |
| 394 | return inet_add_offload(&udpv4_offload, IPPROTO_UDP); |
| 395 | } |