blob: 5aa46d4b44efb99702ccd89005528f20ae422a0e [file] [log] [blame]
Daniel Borkmannc50cd352013-07-01 19:24:00 +02001/*
2 * IPV4 GSO/GRO offload support
3 * Linux INET implementation
4 *
5 * This program is free software; you can redistribute it and/or
6 * modify it under the terms of the GNU General Public License
7 * as published by the Free Software Foundation; either version
8 * 2 of the License, or (at your option) any later version.
9 *
10 * GRE GSO support
11 */
12
13#include <linux/skbuff.h>
Paul Gortmakercf172282014-01-15 11:19:55 -050014#include <linux/init.h>
Daniel Borkmannc50cd352013-07-01 19:24:00 +020015#include <net/protocol.h>
16#include <net/gre.h>
17
Daniel Borkmannc50cd352013-07-01 19:24:00 +020018static struct sk_buff *gre_gso_segment(struct sk_buff *skb,
19 netdev_features_t features)
20{
21 struct sk_buff *segs = ERR_PTR(-EINVAL);
22 netdev_features_t enc_features;
Neal Cardwellb884b1a2014-01-09 20:47:17 -050023 int ghl;
Daniel Borkmannc50cd352013-07-01 19:24:00 +020024 struct gre_base_hdr *greh;
Wei-Chun Chao7a7ffba2013-12-26 13:10:22 -080025 u16 mac_offset = skb->mac_header;
Daniel Borkmannc50cd352013-07-01 19:24:00 +020026 int mac_len = skb->mac_len;
27 __be16 protocol = skb->protocol;
28 int tnl_hlen;
29 bool csum;
30
31 if (unlikely(skb_shinfo(skb)->gso_type &
32 ~(SKB_GSO_TCPV4 |
33 SKB_GSO_TCPV6 |
34 SKB_GSO_UDP |
35 SKB_GSO_DODGY |
36 SKB_GSO_TCP_ECN |
Eric Dumazetcb32f512013-10-19 11:42:57 -070037 SKB_GSO_GRE |
Tom Herbert4749c092014-06-04 17:20:23 -070038 SKB_GSO_GRE_CSUM |
Eric Dumazetcb32f512013-10-19 11:42:57 -070039 SKB_GSO_IPIP)))
Daniel Borkmannc50cd352013-07-01 19:24:00 +020040 goto out;
41
Tom Herbert53e50392014-09-20 14:52:30 -070042 if (!skb->encapsulation)
43 goto out;
44
Daniel Borkmannc50cd352013-07-01 19:24:00 +020045 if (unlikely(!pskb_may_pull(skb, sizeof(*greh))))
46 goto out;
47
48 greh = (struct gre_base_hdr *)skb_transport_header(skb);
49
Tom Herbert14051f02014-10-30 08:40:56 -070050 ghl = skb_inner_mac_header(skb) - skb_transport_header(skb);
Neal Cardwellb884b1a2014-01-09 20:47:17 -050051 if (unlikely(ghl < sizeof(*greh)))
52 goto out;
53
54 csum = !!(greh->flags & GRE_CSUM);
Tom Herbert4749c092014-06-04 17:20:23 -070055 if (csum)
56 skb->encap_hdr_csum = 1;
Daniel Borkmannc50cd352013-07-01 19:24:00 +020057
58 /* setup inner skb. */
59 skb->protocol = greh->protocol;
60 skb->encapsulation = 0;
61
Li RongQingb4e3cef2014-10-18 17:26:04 +080062 if (unlikely(!pskb_may_pull(skb, ghl)))
63 goto out;
64
Daniel Borkmannc50cd352013-07-01 19:24:00 +020065 __skb_pull(skb, ghl);
66 skb_reset_mac_header(skb);
67 skb_set_network_header(skb, skb_inner_network_offset(skb));
68 skb->mac_len = skb_inner_network_offset(skb);
69
70 /* segment inner packet. */
Florian Westphal1e16aa32014-10-20 13:49:16 +020071 enc_features = skb->dev->hw_enc_features & features;
Daniel Borkmannc50cd352013-07-01 19:24:00 +020072 segs = skb_mac_gso_segment(skb, enc_features);
Himangi Saraogi5a8dbf02014-07-27 12:36:51 +053073 if (IS_ERR_OR_NULL(segs)) {
Wei-Chun Chao7a7ffba2013-12-26 13:10:22 -080074 skb_gso_error_unwind(skb, protocol, ghl, mac_offset, mac_len);
Daniel Borkmannc50cd352013-07-01 19:24:00 +020075 goto out;
Wei-Chun Chao7a7ffba2013-12-26 13:10:22 -080076 }
Daniel Borkmannc50cd352013-07-01 19:24:00 +020077
78 skb = segs;
79 tnl_hlen = skb_tnl_header_len(skb);
80 do {
81 __skb_push(skb, ghl);
82 if (csum) {
83 __be32 *pcsum;
84
85 if (skb_has_shared_frag(skb)) {
86 int err;
87
88 err = __skb_linearize(skb);
89 if (err) {
David S. Miller0c1072a2013-07-03 14:50:41 -070090 kfree_skb_list(segs);
Daniel Borkmannc50cd352013-07-01 19:24:00 +020091 segs = ERR_PTR(err);
92 goto out;
93 }
94 }
95
Tom Herbert4749c092014-06-04 17:20:23 -070096 skb_reset_transport_header(skb);
97
98 greh = (struct gre_base_hdr *)
99 skb_transport_header(skb);
Daniel Borkmannc50cd352013-07-01 19:24:00 +0200100 pcsum = (__be32 *)(greh + 1);
101 *pcsum = 0;
Tom Herbert4749c092014-06-04 17:20:23 -0700102 *(__sum16 *)pcsum = gso_make_checksum(skb, 0);
Daniel Borkmannc50cd352013-07-01 19:24:00 +0200103 }
104 __skb_push(skb, tnl_hlen - ghl);
105
Alexander Duyckcdbaa0b2013-07-10 17:05:06 -0700106 skb_reset_inner_headers(skb);
107 skb->encapsulation = 1;
108
Daniel Borkmannc50cd352013-07-01 19:24:00 +0200109 skb_reset_mac_header(skb);
110 skb_set_network_header(skb, mac_len);
111 skb->mac_len = mac_len;
112 skb->protocol = protocol;
113 } while ((skb = skb->next));
114out:
115 return segs;
116}
117
Jerry Chubf5a7552014-01-07 10:23:19 -0800118static struct sk_buff **gre_gro_receive(struct sk_buff **head,
119 struct sk_buff *skb)
120{
121 struct sk_buff **pp = NULL;
122 struct sk_buff *p;
123 const struct gre_base_hdr *greh;
124 unsigned int hlen, grehlen;
125 unsigned int off;
126 int flush = 1;
127 struct packet_offload *ptype;
128 __be16 type;
129
130 off = skb_gro_offset(skb);
131 hlen = off + sizeof(*greh);
132 greh = skb_gro_header_fast(skb, off);
133 if (skb_gro_header_hard(skb, hlen)) {
134 greh = skb_gro_header_slow(skb, hlen, off);
135 if (unlikely(!greh))
136 goto out;
137 }
138
139 /* Only support version 0 and K (key), C (csum) flags. Note that
140 * although the support for the S (seq#) flag can be added easily
141 * for GRO, this is problematic for GSO hence can not be enabled
142 * here because a GRO pkt may end up in the forwarding path, thus
143 * requiring GSO support to break it up correctly.
144 */
145 if ((greh->flags & ~(GRE_KEY|GRE_CSUM)) != 0)
146 goto out;
147
148 type = greh->protocol;
149
150 rcu_read_lock();
151 ptype = gro_find_receive_by_type(type);
Ian Morris51456b22015-04-03 09:17:26 +0100152 if (!ptype)
Jerry Chubf5a7552014-01-07 10:23:19 -0800153 goto out_unlock;
154
155 grehlen = GRE_HEADER_SECTION;
156
157 if (greh->flags & GRE_KEY)
158 grehlen += GRE_HEADER_SECTION;
159
160 if (greh->flags & GRE_CSUM)
161 grehlen += GRE_HEADER_SECTION;
162
163 hlen = off + grehlen;
164 if (skb_gro_header_hard(skb, hlen)) {
165 greh = skb_gro_header_slow(skb, hlen, off);
166 if (unlikely(!greh))
167 goto out_unlock;
168 }
Jerry Chubf5a7552014-01-07 10:23:19 -0800169
Tom Herbert758f75d2014-08-22 13:34:22 -0700170 /* Don't bother verifying checksum if we're going to flush anyway. */
Tom Herbert884d3382014-08-31 15:12:44 -0700171 if ((greh->flags & GRE_CSUM) && !NAPI_GRO_CB(skb)->flush) {
172 if (skb_gro_checksum_simple_validate(skb))
Jerry Chubf5a7552014-01-07 10:23:19 -0800173 goto out_unlock;
Tom Herbert758f75d2014-08-22 13:34:22 -0700174
Tom Herbert884d3382014-08-31 15:12:44 -0700175 skb_gro_checksum_try_convert(skb, IPPROTO_GRE, 0,
176 null_compute_pseudo);
177 }
178
Jerry Chubf5a7552014-01-07 10:23:19 -0800179 flush = 0;
180
181 for (p = *head; p; p = p->next) {
182 const struct gre_base_hdr *greh2;
183
184 if (!NAPI_GRO_CB(p)->same_flow)
185 continue;
186
187 /* The following checks are needed to ensure only pkts
188 * from the same tunnel are considered for aggregation.
189 * The criteria for "the same tunnel" includes:
190 * 1) same version (we only support version 0 here)
191 * 2) same protocol (we only support ETH_P_IP for now)
192 * 3) same set of flags
193 * 4) same key if the key field is present.
194 */
195 greh2 = (struct gre_base_hdr *)(p->data + off);
196
197 if (greh2->flags != greh->flags ||
198 greh2->protocol != greh->protocol) {
199 NAPI_GRO_CB(p)->same_flow = 0;
200 continue;
201 }
202 if (greh->flags & GRE_KEY) {
203 /* compare keys */
204 if (*(__be32 *)(greh2+1) != *(__be32 *)(greh+1)) {
205 NAPI_GRO_CB(p)->same_flow = 0;
206 continue;
207 }
208 }
209 }
210
211 skb_gro_pull(skb, grehlen);
212
213 /* Adjusted NAPI_GRO_CB(skb)->csum after skb_gro_pull()*/
214 skb_gro_postpull_rcsum(skb, greh, grehlen);
215
216 pp = ptype->callbacks.gro_receive(head, skb);
217
218out_unlock:
219 rcu_read_unlock();
220out:
221 NAPI_GRO_CB(skb)->flush |= flush;
222
223 return pp;
224}
225
Wei Yongjund10dbad2014-01-09 22:22:05 +0800226static int gre_gro_complete(struct sk_buff *skb, int nhoff)
Jerry Chubf5a7552014-01-07 10:23:19 -0800227{
228 struct gre_base_hdr *greh = (struct gre_base_hdr *)(skb->data + nhoff);
229 struct packet_offload *ptype;
230 unsigned int grehlen = sizeof(*greh);
231 int err = -ENOENT;
232 __be16 type;
233
Jerry Chuc3caf112014-07-14 15:54:46 -0700234 skb->encapsulation = 1;
235 skb_shinfo(skb)->gso_type = SKB_GSO_GRE;
236
Jerry Chubf5a7552014-01-07 10:23:19 -0800237 type = greh->protocol;
238 if (greh->flags & GRE_KEY)
239 grehlen += GRE_HEADER_SECTION;
240
241 if (greh->flags & GRE_CSUM)
242 grehlen += GRE_HEADER_SECTION;
243
244 rcu_read_lock();
245 ptype = gro_find_complete_by_type(type);
Ian Morris00db4122015-04-03 09:17:27 +0100246 if (ptype)
Jerry Chubf5a7552014-01-07 10:23:19 -0800247 err = ptype->callbacks.gro_complete(skb, nhoff + grehlen);
248
249 rcu_read_unlock();
Tom Herbert6fb2a752014-11-29 09:59:45 -0800250
251 skb_set_inner_mac_header(skb, nhoff + grehlen);
252
Jerry Chubf5a7552014-01-07 10:23:19 -0800253 return err;
254}
255
Daniel Borkmannc50cd352013-07-01 19:24:00 +0200256static const struct net_offload gre_offload = {
257 .callbacks = {
Daniel Borkmannc50cd352013-07-01 19:24:00 +0200258 .gso_segment = gre_gso_segment,
Jerry Chubf5a7552014-01-07 10:23:19 -0800259 .gro_receive = gre_gro_receive,
260 .gro_complete = gre_gro_complete,
Daniel Borkmannc50cd352013-07-01 19:24:00 +0200261 },
262};
263
Eric Dumazet438e38f2014-01-06 14:03:07 -0800264static int __init gre_offload_init(void)
Daniel Borkmannc50cd352013-07-01 19:24:00 +0200265{
266 return inet_add_offload(&gre_offload, IPPROTO_GRE);
267}
Paul Gortmakercf172282014-01-15 11:19:55 -0500268device_initcall(gre_offload_init);