blob: d6427d9185125c34140f5306b98a692b255a6619 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * INET An implementation of the TCP/IP protocol suite for the LINUX
3 * operating system. INET is implemented using the BSD Socket
4 * interface as the means of communication with the user level.
5 *
6 * The Internet Protocol (IP) output module.
7 *
8 * Version: $Id: ip_output.c,v 1.100 2002/02/01 22:01:03 davem Exp $
9 *
Jesper Juhl02c30a82005-05-05 16:16:16 -070010 * Authors: Ross Biro
Linus Torvalds1da177e2005-04-16 15:20:36 -070011 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
12 * Donald Becker, <becker@super.org>
13 * Alan Cox, <Alan.Cox@linux.org>
14 * Richard Underwood
15 * Stefan Becker, <stefanb@yello.ping.de>
16 * Jorge Cwik, <jorge@laser.satlink.net>
17 * Arnt Gulbrandsen, <agulbra@nvg.unit.no>
18 * Hirokazu Takahashi, <taka@valinux.co.jp>
19 *
20 * See ip_input.c for original log
21 *
22 * Fixes:
23 * Alan Cox : Missing nonblock feature in ip_build_xmit.
24 * Mike Kilburn : htons() missing in ip_build_xmit.
YOSHIFUJI Hideakie905a9e2007-02-09 23:24:47 +090025 * Bradford Johnson: Fix faulty handling of some frames when
Linus Torvalds1da177e2005-04-16 15:20:36 -070026 * no route is found.
27 * Alexander Demenshin: Missing sk/skb free in ip_queue_xmit
28 * (in case if packet not accepted by
29 * output firewall rules)
30 * Mike McLagan : Routing by source
31 * Alexey Kuznetsov: use new route cache
32 * Andi Kleen: Fix broken PMTU recovery and remove
33 * some redundant tests.
34 * Vitaly E. Lavrov : Transparent proxy revived after year coma.
35 * Andi Kleen : Replace ip_reply with ip_send_reply.
YOSHIFUJI Hideakie905a9e2007-02-09 23:24:47 +090036 * Andi Kleen : Split fast and slow ip_build_xmit path
37 * for decreased register pressure on x86
38 * and more readibility.
Linus Torvalds1da177e2005-04-16 15:20:36 -070039 * Marc Boucher : When call_out_firewall returns FW_QUEUE,
40 * silently drop skb instead of failing with -EPERM.
41 * Detlev Wengorz : Copy protocol for fragments.
42 * Hirokazu Takahashi: HW checksumming for outgoing UDP
43 * datagrams.
44 * Hirokazu Takahashi: sendfile() on UDP works now.
45 */
46
47#include <asm/uaccess.h>
48#include <asm/system.h>
49#include <linux/module.h>
50#include <linux/types.h>
51#include <linux/kernel.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070052#include <linux/mm.h>
53#include <linux/string.h>
54#include <linux/errno.h>
Al Viroa1f8e7f2006-10-19 16:08:53 -040055#include <linux/highmem.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070056
57#include <linux/socket.h>
58#include <linux/sockios.h>
59#include <linux/in.h>
60#include <linux/inet.h>
61#include <linux/netdevice.h>
62#include <linux/etherdevice.h>
63#include <linux/proc_fs.h>
64#include <linux/stat.h>
65#include <linux/init.h>
66
67#include <net/snmp.h>
68#include <net/ip.h>
69#include <net/protocol.h>
70#include <net/route.h>
Patrick McHardycfacb052006-01-08 22:36:54 -080071#include <net/xfrm.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070072#include <linux/skbuff.h>
73#include <net/sock.h>
74#include <net/arp.h>
75#include <net/icmp.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070076#include <net/checksum.h>
77#include <net/inetpeer.h>
78#include <net/checksum.h>
79#include <linux/igmp.h>
80#include <linux/netfilter_ipv4.h>
81#include <linux/netfilter_bridge.h>
82#include <linux/mroute.h>
83#include <linux/netlink.h>
Arnaldo Carvalho de Melo6cbb0df2005-08-09 19:49:02 -070084#include <linux/tcp.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070085
Brian Haleyab32ea52006-09-22 14:15:41 -070086int sysctl_ip_default_ttl __read_mostly = IPDEFTTL;
Linus Torvalds1da177e2005-04-16 15:20:36 -070087
88/* Generate a checksum for an outgoing IP datagram. */
89__inline__ void ip_send_check(struct iphdr *iph)
90{
91 iph->check = 0;
92 iph->check = ip_fast_csum((unsigned char *)iph, iph->ihl);
93}
94
95/* dev_loopback_xmit for use with netfilter. */
96static int ip_dev_loopback_xmit(struct sk_buff *newskb)
97{
Arnaldo Carvalho de Melo459a98e2007-03-19 15:30:44 -070098 skb_reset_mac_header(newskb);
Arnaldo Carvalho de Melobbe735e2007-03-10 22:16:10 -030099 __skb_pull(newskb, skb_network_offset(newskb));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700100 newskb->pkt_type = PACKET_LOOPBACK;
101 newskb->ip_summed = CHECKSUM_UNNECESSARY;
102 BUG_TRAP(newskb->dst);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700103 netif_rx(newskb);
104 return 0;
105}
106
107static inline int ip_select_ttl(struct inet_sock *inet, struct dst_entry *dst)
108{
109 int ttl = inet->uc_ttl;
110
111 if (ttl < 0)
112 ttl = dst_metric(dst, RTAX_HOPLIMIT);
113 return ttl;
114}
115
YOSHIFUJI Hideakie905a9e2007-02-09 23:24:47 +0900116/*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700117 * Add an ip header to a skbuff and send it out.
118 *
119 */
120int ip_build_and_send_pkt(struct sk_buff *skb, struct sock *sk,
Al Viro13d8eaa2006-09-26 22:27:30 -0700121 __be32 saddr, __be32 daddr, struct ip_options *opt)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700122{
123 struct inet_sock *inet = inet_sk(sk);
124 struct rtable *rt = (struct rtable *)skb->dst;
125 struct iphdr *iph;
126
127 /* Build the IP header. */
Arnaldo Carvalho de Melo8856dfa2007-03-10 19:40:39 -0300128 skb_push(skb, sizeof(struct iphdr) + (opt ? opt->optlen : 0));
129 skb_reset_network_header(skb);
Arnaldo Carvalho de Meloeddc9ec2007-04-20 22:47:35 -0700130 iph = ip_hdr(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700131 iph->version = 4;
132 iph->ihl = 5;
133 iph->tos = inet->tos;
134 if (ip_dont_fragment(sk, &rt->u.dst))
135 iph->frag_off = htons(IP_DF);
136 else
137 iph->frag_off = 0;
138 iph->ttl = ip_select_ttl(inet, &rt->u.dst);
139 iph->daddr = rt->rt_dst;
140 iph->saddr = rt->rt_src;
141 iph->protocol = sk->sk_protocol;
142 iph->tot_len = htons(skb->len);
143 ip_select_ident(iph, &rt->u.dst, sk);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700144
145 if (opt && opt->optlen) {
146 iph->ihl += opt->optlen>>2;
147 ip_options_build(skb, opt, daddr, rt, 0);
148 }
149 ip_send_check(iph);
150
151 skb->priority = sk->sk_priority;
152
153 /* Send it out. */
154 return NF_HOOK(PF_INET, NF_IP_LOCAL_OUT, skb, NULL, rt->u.dst.dev,
155 dst_output);
156}
157
Arnaldo Carvalho de Melod8c97a92005-08-09 20:12:12 -0700158EXPORT_SYMBOL_GPL(ip_build_and_send_pkt);
159
Linus Torvalds1da177e2005-04-16 15:20:36 -0700160static inline int ip_finish_output2(struct sk_buff *skb)
161{
162 struct dst_entry *dst = skb->dst;
Mitsuru Chinen80787eb2007-04-30 00:48:20 -0700163 struct rtable *rt = (struct rtable *)dst;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700164 struct net_device *dev = dst->dev;
165 int hh_len = LL_RESERVED_SPACE(dev);
166
Mitsuru Chinen80787eb2007-04-30 00:48:20 -0700167 if (rt->rt_type == RTN_MULTICAST)
168 IP_INC_STATS(IPSTATS_MIB_OUTMCASTPKTS);
169 else if (rt->rt_type == RTN_BROADCAST)
170 IP_INC_STATS(IPSTATS_MIB_OUTBCASTPKTS);
171
Linus Torvalds1da177e2005-04-16 15:20:36 -0700172 /* Be paranoid, rather than too clever. */
173 if (unlikely(skb_headroom(skb) < hh_len && dev->hard_header)) {
174 struct sk_buff *skb2;
175
176 skb2 = skb_realloc_headroom(skb, LL_RESERVED_SPACE(dev));
177 if (skb2 == NULL) {
178 kfree_skb(skb);
179 return -ENOMEM;
180 }
181 if (skb->sk)
182 skb_set_owner_w(skb2, skb->sk);
183 kfree_skb(skb);
184 skb = skb2;
185 }
186
Stephen Hemminger3644f0c2006-12-07 15:08:17 -0800187 if (dst->hh)
188 return neigh_hh_output(dst->hh, skb);
189 else if (dst->neighbour)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700190 return dst->neighbour->output(skb);
191
192 if (net_ratelimit())
193 printk(KERN_DEBUG "ip_finish_output2: No header cache and no neighbour!\n");
194 kfree_skb(skb);
195 return -EINVAL;
196}
197
John Heffner628a5c52007-04-20 15:53:27 -0700198static inline int ip_skb_dst_mtu(struct sk_buff *skb)
199{
200 struct inet_sock *inet = skb->sk ? inet_sk(skb->sk) : NULL;
201
202 return (inet && inet->pmtudisc == IP_PMTUDISC_PROBE) ?
203 skb->dst->dev->mtu : dst_mtu(skb->dst);
204}
205
Thomas Graf33d043d2005-08-20 17:27:34 -0700206static inline int ip_finish_output(struct sk_buff *skb)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700207{
Patrick McHardy5c901da2006-01-06 23:05:36 -0800208#if defined(CONFIG_NETFILTER) && defined(CONFIG_XFRM)
209 /* Policy lookup after SNAT yielded a new policy */
Patrick McHardy48d5cad2006-02-15 15:10:22 -0800210 if (skb->dst->xfrm != NULL) {
211 IPCB(skb)->flags |= IPSKB_REROUTED;
212 return dst_output(skb);
213 }
Patrick McHardy5c901da2006-01-06 23:05:36 -0800214#endif
John Heffner628a5c52007-04-20 15:53:27 -0700215 if (skb->len > ip_skb_dst_mtu(skb) && !skb_is_gso(skb))
Patrick McHardy1bd9bef2006-01-05 12:20:59 -0800216 return ip_fragment(skb, ip_finish_output2);
217 else
218 return ip_finish_output2(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700219}
220
221int ip_mc_output(struct sk_buff *skb)
222{
223 struct sock *sk = skb->sk;
224 struct rtable *rt = (struct rtable*)skb->dst;
225 struct net_device *dev = rt->u.dst.dev;
226
227 /*
228 * If the indicated interface is up and running, send the packet.
229 */
230 IP_INC_STATS(IPSTATS_MIB_OUTREQUESTS);
231
232 skb->dev = dev;
233 skb->protocol = htons(ETH_P_IP);
234
235 /*
236 * Multicasts are looped back for other local users
237 */
238
239 if (rt->rt_flags&RTCF_MULTICAST) {
240 if ((!sk || inet_sk(sk)->mc_loop)
241#ifdef CONFIG_IP_MROUTE
242 /* Small optimization: do not loopback not local frames,
243 which returned after forwarding; they will be dropped
244 by ip_mr_input in any case.
245 Note, that local frames are looped back to be delivered
246 to local recipients.
247
248 This check is duplicated in ip_mr_input at the moment.
249 */
250 && ((rt->rt_flags&RTCF_LOCAL) || !(IPCB(skb)->flags&IPSKB_FORWARDED))
251#endif
252 ) {
253 struct sk_buff *newskb = skb_clone(skb, GFP_ATOMIC);
254 if (newskb)
255 NF_HOOK(PF_INET, NF_IP_POST_ROUTING, newskb, NULL,
YOSHIFUJI Hideakie905a9e2007-02-09 23:24:47 +0900256 newskb->dev,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700257 ip_dev_loopback_xmit);
258 }
259
260 /* Multicasts with ttl 0 must not go beyond the host */
261
Arnaldo Carvalho de Meloeddc9ec2007-04-20 22:47:35 -0700262 if (ip_hdr(skb)->ttl == 0) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700263 kfree_skb(skb);
264 return 0;
265 }
266 }
267
268 if (rt->rt_flags&RTCF_BROADCAST) {
269 struct sk_buff *newskb = skb_clone(skb, GFP_ATOMIC);
270 if (newskb)
271 NF_HOOK(PF_INET, NF_IP_POST_ROUTING, newskb, NULL,
272 newskb->dev, ip_dev_loopback_xmit);
273 }
274
Patrick McHardy48d5cad2006-02-15 15:10:22 -0800275 return NF_HOOK_COND(PF_INET, NF_IP_POST_ROUTING, skb, NULL, skb->dev,
276 ip_finish_output,
277 !(IPCB(skb)->flags & IPSKB_REROUTED));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700278}
279
280int ip_output(struct sk_buff *skb)
281{
Patrick McHardy1bd9bef2006-01-05 12:20:59 -0800282 struct net_device *dev = skb->dst->dev;
283
Linus Torvalds1da177e2005-04-16 15:20:36 -0700284 IP_INC_STATS(IPSTATS_MIB_OUTREQUESTS);
285
Patrick McHardy1bd9bef2006-01-05 12:20:59 -0800286 skb->dev = dev;
287 skb->protocol = htons(ETH_P_IP);
288
Patrick McHardy48d5cad2006-02-15 15:10:22 -0800289 return NF_HOOK_COND(PF_INET, NF_IP_POST_ROUTING, skb, NULL, dev,
YOSHIFUJI Hideakie905a9e2007-02-09 23:24:47 +0900290 ip_finish_output,
Patrick McHardy48d5cad2006-02-15 15:10:22 -0800291 !(IPCB(skb)->flags & IPSKB_REROUTED));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700292}
293
David S. Millere89862f2007-01-26 01:04:55 -0800294int ip_queue_xmit(struct sk_buff *skb, int ipfragok)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700295{
David S. Millere89862f2007-01-26 01:04:55 -0800296 struct sock *sk = skb->sk;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700297 struct inet_sock *inet = inet_sk(sk);
298 struct ip_options *opt = inet->opt;
299 struct rtable *rt;
300 struct iphdr *iph;
301
302 /* Skip all of this if the packet is already routed,
303 * f.e. by something like SCTP.
304 */
305 rt = (struct rtable *) skb->dst;
306 if (rt != NULL)
307 goto packet_routed;
308
309 /* Make sure we can route this packet. */
310 rt = (struct rtable *)__sk_dst_check(sk, 0);
311 if (rt == NULL) {
Al Viro3ca3c682006-09-27 18:28:07 -0700312 __be32 daddr;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700313
314 /* Use correct destination address if we have options. */
315 daddr = inet->daddr;
316 if(opt && opt->srr)
317 daddr = opt->faddr;
318
319 {
320 struct flowi fl = { .oif = sk->sk_bound_dev_if,
321 .nl_u = { .ip4_u =
322 { .daddr = daddr,
323 .saddr = inet->saddr,
324 .tos = RT_CONN_FLAGS(sk) } },
325 .proto = sk->sk_protocol,
326 .uli_u = { .ports =
327 { .sport = inet->sport,
328 .dport = inet->dport } } };
329
330 /* If this fails, retransmit mechanism of transport layer will
331 * keep trying until route appears or the connection times
332 * itself out.
333 */
Venkat Yekkiralabeb8d132006-08-04 23:12:42 -0700334 security_sk_classify_flow(sk, &fl);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700335 if (ip_route_output_flow(&rt, &fl, sk, 0))
336 goto no_route;
337 }
Arnaldo Carvalho de Melo6cbb0df2005-08-09 19:49:02 -0700338 sk_setup_caps(sk, &rt->u.dst);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700339 }
340 skb->dst = dst_clone(&rt->u.dst);
341
342packet_routed:
343 if (opt && opt->is_strictroute && rt->rt_dst != rt->rt_gateway)
344 goto no_route;
345
346 /* OK, we know where to send it, allocate and build IP header. */
Arnaldo Carvalho de Melo8856dfa2007-03-10 19:40:39 -0300347 skb_push(skb, sizeof(struct iphdr) + (opt ? opt->optlen : 0));
348 skb_reset_network_header(skb);
Arnaldo Carvalho de Meloeddc9ec2007-04-20 22:47:35 -0700349 iph = ip_hdr(skb);
Al Viro714e85b2006-11-14 20:51:49 -0800350 *((__be16 *)iph) = htons((4 << 12) | (5 << 8) | (inet->tos & 0xff));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700351 iph->tot_len = htons(skb->len);
352 if (ip_dont_fragment(sk, &rt->u.dst) && !ipfragok)
353 iph->frag_off = htons(IP_DF);
354 else
355 iph->frag_off = 0;
356 iph->ttl = ip_select_ttl(inet, &rt->u.dst);
357 iph->protocol = sk->sk_protocol;
358 iph->saddr = rt->rt_src;
359 iph->daddr = rt->rt_dst;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700360 /* Transport layer set skb->h.foo itself. */
361
362 if (opt && opt->optlen) {
363 iph->ihl += opt->optlen >> 2;
364 ip_options_build(skb, opt, inet->daddr, rt, 0);
365 }
366
Herbert Xu89f5f0a2005-11-08 09:41:56 -0800367 ip_select_ident_more(iph, &rt->u.dst, sk,
Herbert Xu79671682006-06-22 02:40:14 -0700368 (skb_shinfo(skb)->gso_segs ?: 1) - 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700369
370 /* Add an IP checksum. */
371 ip_send_check(iph);
372
373 skb->priority = sk->sk_priority;
374
375 return NF_HOOK(PF_INET, NF_IP_LOCAL_OUT, skb, NULL, rt->u.dst.dev,
376 dst_output);
377
378no_route:
379 IP_INC_STATS(IPSTATS_MIB_OUTNOROUTES);
380 kfree_skb(skb);
381 return -EHOSTUNREACH;
382}
383
384
385static void ip_copy_metadata(struct sk_buff *to, struct sk_buff *from)
386{
387 to->pkt_type = from->pkt_type;
388 to->priority = from->priority;
389 to->protocol = from->protocol;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700390 dst_release(to->dst);
391 to->dst = dst_clone(from->dst);
392 to->dev = from->dev;
Thomas Graf82e91ff2006-11-09 15:19:14 -0800393 to->mark = from->mark;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700394
395 /* Copy the flags to each fragment. */
396 IPCB(to)->flags = IPCB(from)->flags;
397
398#ifdef CONFIG_NET_SCHED
399 to->tc_index = from->tc_index;
400#endif
Yasuyuki Kozakaie7ac05f2007-03-14 16:44:01 -0700401 nf_copy(to, from);
Julian Anastasovc98d80e2005-10-22 13:39:21 +0300402#if defined(CONFIG_IP_VS) || defined(CONFIG_IP_VS_MODULE)
403 to->ipvs_property = from->ipvs_property;
404#endif
James Morris984bc162006-06-09 00:29:17 -0700405 skb_copy_secmark(to, from);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700406}
407
408/*
409 * This IP datagram is too large to be sent in one piece. Break it up into
410 * smaller pieces (each of size equal to IP header plus
411 * a block of the data of the original IP data part) that will yet fit in a
412 * single device frame, and queue such a frame for sending.
413 */
414
Patrick McHardy2e2f7ae2006-04-04 13:42:35 -0700415int ip_fragment(struct sk_buff *skb, int (*output)(struct sk_buff*))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700416{
417 struct iphdr *iph;
418 int raw = 0;
419 int ptr;
420 struct net_device *dev;
421 struct sk_buff *skb2;
Stephen Hemminger9bcfcaf2006-08-29 17:48:57 -0700422 unsigned int mtu, hlen, left, len, ll_rs, pad;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700423 int offset;
Alexey Dobriyan76ab6082006-01-06 13:24:29 -0800424 __be16 not_last_frag;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700425 struct rtable *rt = (struct rtable*)skb->dst;
426 int err = 0;
427
428 dev = rt->u.dst.dev;
429
430 /*
431 * Point into the IP datagram header.
432 */
433
Arnaldo Carvalho de Meloeddc9ec2007-04-20 22:47:35 -0700434 iph = ip_hdr(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700435
436 if (unlikely((iph->frag_off & htons(IP_DF)) && !skb->local_df)) {
Wei Dong0668b472006-08-31 15:24:48 -0700437 IP_INC_STATS(IPSTATS_MIB_FRAGFAILS);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700438 icmp_send(skb, ICMP_DEST_UNREACH, ICMP_FRAG_NEEDED,
John Heffner628a5c52007-04-20 15:53:27 -0700439 htonl(ip_skb_dst_mtu(skb)));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700440 kfree_skb(skb);
441 return -EMSGSIZE;
442 }
443
444 /*
445 * Setup starting values.
446 */
447
448 hlen = iph->ihl * 4;
449 mtu = dst_mtu(&rt->u.dst) - hlen; /* Size of data space */
Herbert Xu89cee8b2005-12-13 23:14:27 -0800450 IPCB(skb)->flags |= IPSKB_FRAG_COMPLETE;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700451
452 /* When frag_list is given, use it. First, check its validity:
453 * some transformers could create wrong frag_list or break existing
454 * one, it is not prohibited. In this case fall back to copying.
455 *
456 * LATER: this step can be merged to real generation of fragments,
457 * we can switch to copy when see the first bad fragment.
458 */
459 if (skb_shinfo(skb)->frag_list) {
460 struct sk_buff *frag;
461 int first_len = skb_pagelen(skb);
462
463 if (first_len - hlen > mtu ||
464 ((first_len - hlen) & 7) ||
465 (iph->frag_off & htons(IP_MF|IP_OFFSET)) ||
466 skb_cloned(skb))
467 goto slow_path;
468
469 for (frag = skb_shinfo(skb)->frag_list; frag; frag = frag->next) {
470 /* Correct geometry. */
471 if (frag->len > mtu ||
472 ((frag->len & 7) && frag->next) ||
473 skb_headroom(frag) < hlen)
474 goto slow_path;
475
476 /* Partially cloned skb? */
477 if (skb_shared(frag))
478 goto slow_path;
Herbert Xu2fdba6b2005-05-18 22:52:33 -0700479
480 BUG_ON(frag->sk);
481 if (skb->sk) {
482 sock_hold(skb->sk);
483 frag->sk = skb->sk;
484 frag->destructor = sock_wfree;
485 skb->truesize -= frag->truesize;
486 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700487 }
488
489 /* Everything is OK. Generate! */
490
491 err = 0;
492 offset = 0;
493 frag = skb_shinfo(skb)->frag_list;
494 skb_shinfo(skb)->frag_list = NULL;
495 skb->data_len = first_len - skb_headlen(skb);
496 skb->len = first_len;
497 iph->tot_len = htons(first_len);
498 iph->frag_off = htons(IP_MF);
499 ip_send_check(iph);
500
501 for (;;) {
502 /* Prepare header of the next frame,
503 * before previous one went down. */
504 if (frag) {
505 frag->ip_summed = CHECKSUM_NONE;
Arnaldo Carvalho de Melobadff6d2007-03-13 13:06:52 -0300506 skb_reset_transport_header(frag);
Arnaldo Carvalho de Meloe2d1bca2007-04-10 20:46:21 -0700507 __skb_push(frag, hlen);
508 skb_reset_network_header(frag);
Arnaldo Carvalho de Melod56f90a2007-04-10 20:50:43 -0700509 memcpy(skb_network_header(frag), iph, hlen);
Arnaldo Carvalho de Meloeddc9ec2007-04-20 22:47:35 -0700510 iph = ip_hdr(frag);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700511 iph->tot_len = htons(frag->len);
512 ip_copy_metadata(frag, skb);
513 if (offset == 0)
514 ip_options_fragment(frag);
515 offset += skb->len - hlen;
516 iph->frag_off = htons(offset>>3);
517 if (frag->next != NULL)
518 iph->frag_off |= htons(IP_MF);
519 /* Ready, complete checksum */
520 ip_send_check(iph);
521 }
522
523 err = output(skb);
524
Wei Dongdafee492006-08-02 13:41:21 -0700525 if (!err)
526 IP_INC_STATS(IPSTATS_MIB_FRAGCREATES);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700527 if (err || !frag)
528 break;
529
530 skb = frag;
531 frag = skb->next;
532 skb->next = NULL;
533 }
534
535 if (err == 0) {
536 IP_INC_STATS(IPSTATS_MIB_FRAGOKS);
537 return 0;
538 }
539
540 while (frag) {
541 skb = frag->next;
542 kfree_skb(frag);
543 frag = skb;
544 }
545 IP_INC_STATS(IPSTATS_MIB_FRAGFAILS);
546 return err;
547 }
548
549slow_path:
550 left = skb->len - hlen; /* Space per frame */
551 ptr = raw + hlen; /* Where to start from */
552
Linus Torvalds1da177e2005-04-16 15:20:36 -0700553 /* for bridged IP traffic encapsulated inside f.e. a vlan header,
Stephen Hemminger9bcfcaf2006-08-29 17:48:57 -0700554 * we need to make room for the encapsulating header
555 */
556 pad = nf_bridge_pad(skb);
557 ll_rs = LL_RESERVED_SPACE_EXTRA(rt->u.dst.dev, pad);
558 mtu -= pad;
559
Linus Torvalds1da177e2005-04-16 15:20:36 -0700560 /*
561 * Fragment the datagram.
562 */
563
564 offset = (ntohs(iph->frag_off) & IP_OFFSET) << 3;
565 not_last_frag = iph->frag_off & htons(IP_MF);
566
567 /*
568 * Keep copying data until we run out.
569 */
570
Stephen Hemminger132adf52007-03-08 20:44:43 -0800571 while (left > 0) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700572 len = left;
573 /* IF: it doesn't fit, use 'mtu' - the data space left */
574 if (len > mtu)
575 len = mtu;
576 /* IF: we are not sending upto and including the packet end
577 then align the next start on an eight byte boundary */
578 if (len < left) {
579 len &= ~7;
580 }
581 /*
582 * Allocate buffer.
583 */
584
585 if ((skb2 = alloc_skb(len+hlen+ll_rs, GFP_ATOMIC)) == NULL) {
Patrick McHardy64ce2072005-08-09 20:50:53 -0700586 NETDEBUG(KERN_INFO "IP: frag: no memory for new fragment!\n");
Linus Torvalds1da177e2005-04-16 15:20:36 -0700587 err = -ENOMEM;
588 goto fail;
589 }
590
591 /*
592 * Set up data on packet
593 */
594
595 ip_copy_metadata(skb2, skb);
596 skb_reserve(skb2, ll_rs);
597 skb_put(skb2, len + hlen);
Arnaldo Carvalho de Meloc1d2bbe2007-04-10 20:45:18 -0700598 skb_reset_network_header(skb2);
Arnaldo Carvalho de Melob0e380b2007-04-10 21:21:55 -0700599 skb2->transport_header = skb2->network_header + hlen;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700600
601 /*
602 * Charge the memory for the fragment to any owner
603 * it might possess
604 */
605
606 if (skb->sk)
607 skb_set_owner_w(skb2, skb->sk);
608
609 /*
610 * Copy the packet header into the new buffer.
611 */
612
Arnaldo Carvalho de Melod626f622007-03-27 18:55:52 -0300613 skb_copy_from_linear_data(skb, skb_network_header(skb2), hlen);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700614
615 /*
616 * Copy a block of the IP datagram.
617 */
Arnaldo Carvalho de Melobff9b612007-03-16 17:19:57 -0300618 if (skb_copy_bits(skb, ptr, skb_transport_header(skb2), len))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700619 BUG();
620 left -= len;
621
622 /*
623 * Fill in the new header fields.
624 */
Arnaldo Carvalho de Meloeddc9ec2007-04-20 22:47:35 -0700625 iph = ip_hdr(skb2);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700626 iph->frag_off = htons((offset >> 3));
627
628 /* ANK: dirty, but effective trick. Upgrade options only if
629 * the segment to be fragmented was THE FIRST (otherwise,
630 * options are already fixed) and make it ONCE
631 * on the initial skb, so that all the following fragments
632 * will inherit fixed options.
633 */
634 if (offset == 0)
635 ip_options_fragment(skb);
636
637 /*
638 * Added AC : If we are fragmenting a fragment that's not the
639 * last fragment then keep MF on each bit
640 */
641 if (left > 0 || not_last_frag)
642 iph->frag_off |= htons(IP_MF);
643 ptr += len;
644 offset += len;
645
646 /*
647 * Put this fragment into the sending queue.
648 */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700649 iph->tot_len = htons(len + hlen);
650
651 ip_send_check(iph);
652
653 err = output(skb2);
654 if (err)
655 goto fail;
Wei Dongdafee492006-08-02 13:41:21 -0700656
657 IP_INC_STATS(IPSTATS_MIB_FRAGCREATES);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700658 }
659 kfree_skb(skb);
660 IP_INC_STATS(IPSTATS_MIB_FRAGOKS);
661 return err;
662
663fail:
YOSHIFUJI Hideakie905a9e2007-02-09 23:24:47 +0900664 kfree_skb(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700665 IP_INC_STATS(IPSTATS_MIB_FRAGFAILS);
666 return err;
667}
668
Patrick McHardy2e2f7ae2006-04-04 13:42:35 -0700669EXPORT_SYMBOL(ip_fragment);
670
Linus Torvalds1da177e2005-04-16 15:20:36 -0700671int
672ip_generic_getfrag(void *from, char *to, int offset, int len, int odd, struct sk_buff *skb)
673{
674 struct iovec *iov = from;
675
Patrick McHardy84fa7932006-08-29 16:44:56 -0700676 if (skb->ip_summed == CHECKSUM_PARTIAL) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700677 if (memcpy_fromiovecend(to, iov, offset, len) < 0)
678 return -EFAULT;
679 } else {
Al Viro44bb9362006-11-14 21:36:14 -0800680 __wsum csum = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700681 if (csum_partial_copy_fromiovecend(to, iov, offset, len, &csum) < 0)
682 return -EFAULT;
683 skb->csum = csum_block_add(skb->csum, csum, odd);
684 }
685 return 0;
686}
687
Al Viro44bb9362006-11-14 21:36:14 -0800688static inline __wsum
Linus Torvalds1da177e2005-04-16 15:20:36 -0700689csum_page(struct page *page, int offset, int copy)
690{
691 char *kaddr;
Al Viro44bb9362006-11-14 21:36:14 -0800692 __wsum csum;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700693 kaddr = kmap(page);
694 csum = csum_partial(kaddr + offset, copy, 0);
695 kunmap(page);
696 return csum;
697}
698
Adrian Bunk4b30b1c2005-11-29 16:27:20 -0800699static inline int ip_ufo_append_data(struct sock *sk,
Ananda Rajue89e9cf2005-10-18 15:46:41 -0700700 int getfrag(void *from, char *to, int offset, int len,
701 int odd, struct sk_buff *skb),
702 void *from, int length, int hh_len, int fragheaderlen,
703 int transhdrlen, int mtu,unsigned int flags)
704{
705 struct sk_buff *skb;
706 int err;
707
708 /* There is support for UDP fragmentation offload by network
709 * device, so create one single skb packet containing complete
710 * udp datagram
711 */
712 if ((skb = skb_peek_tail(&sk->sk_write_queue)) == NULL) {
713 skb = sock_alloc_send_skb(sk,
714 hh_len + fragheaderlen + transhdrlen + 20,
715 (flags & MSG_DONTWAIT), &err);
716
717 if (skb == NULL)
718 return err;
719
720 /* reserve space for Hardware header */
721 skb_reserve(skb, hh_len);
722
723 /* create space for UDP/IP header */
724 skb_put(skb,fragheaderlen + transhdrlen);
725
726 /* initialize network header pointer */
Arnaldo Carvalho de Meloc1d2bbe2007-04-10 20:45:18 -0700727 skb_reset_network_header(skb);
Ananda Rajue89e9cf2005-10-18 15:46:41 -0700728
729 /* initialize protocol header pointer */
Arnaldo Carvalho de Melob0e380b2007-04-10 21:21:55 -0700730 skb->transport_header = skb->network_header + fragheaderlen;
Ananda Rajue89e9cf2005-10-18 15:46:41 -0700731
Patrick McHardy84fa7932006-08-29 16:44:56 -0700732 skb->ip_summed = CHECKSUM_PARTIAL;
Ananda Rajue89e9cf2005-10-18 15:46:41 -0700733 skb->csum = 0;
734 sk->sk_sndmsg_off = 0;
735 }
736
737 err = skb_append_datato_frags(sk,skb, getfrag, from,
738 (length - transhdrlen));
739 if (!err) {
740 /* specify the length of each IP datagram fragment*/
Herbert Xu79671682006-06-22 02:40:14 -0700741 skb_shinfo(skb)->gso_size = mtu - fragheaderlen;
Herbert Xuf83ef8c2006-06-30 13:37:03 -0700742 skb_shinfo(skb)->gso_type = SKB_GSO_UDP;
Ananda Rajue89e9cf2005-10-18 15:46:41 -0700743 __skb_queue_tail(&sk->sk_write_queue, skb);
744
745 return 0;
746 }
747 /* There is not enough support do UFO ,
748 * so follow normal path
749 */
750 kfree_skb(skb);
751 return err;
752}
753
Linus Torvalds1da177e2005-04-16 15:20:36 -0700754/*
755 * ip_append_data() and ip_append_page() can make one large IP datagram
756 * from many pieces of data. Each pieces will be holded on the socket
757 * until ip_push_pending_frames() is called. Each piece can be a page
758 * or non-page data.
YOSHIFUJI Hideakie905a9e2007-02-09 23:24:47 +0900759 *
Linus Torvalds1da177e2005-04-16 15:20:36 -0700760 * Not only UDP, other transport protocols - e.g. raw sockets - can use
761 * this interface potentially.
762 *
763 * LATER: length must be adjusted by pad at tail, when it is required.
764 */
765int ip_append_data(struct sock *sk,
766 int getfrag(void *from, char *to, int offset, int len,
767 int odd, struct sk_buff *skb),
768 void *from, int length, int transhdrlen,
769 struct ipcm_cookie *ipc, struct rtable *rt,
770 unsigned int flags)
771{
772 struct inet_sock *inet = inet_sk(sk);
773 struct sk_buff *skb;
774
775 struct ip_options *opt = NULL;
776 int hh_len;
777 int exthdrlen;
778 int mtu;
779 int copy;
780 int err;
781 int offset = 0;
782 unsigned int maxfraglen, fragheaderlen;
783 int csummode = CHECKSUM_NONE;
784
785 if (flags&MSG_PROBE)
786 return 0;
787
788 if (skb_queue_empty(&sk->sk_write_queue)) {
789 /*
790 * setup for corking.
791 */
792 opt = ipc->opt;
793 if (opt) {
794 if (inet->cork.opt == NULL) {
795 inet->cork.opt = kmalloc(sizeof(struct ip_options) + 40, sk->sk_allocation);
796 if (unlikely(inet->cork.opt == NULL))
797 return -ENOBUFS;
798 }
799 memcpy(inet->cork.opt, opt, sizeof(struct ip_options)+opt->optlen);
800 inet->cork.flags |= IPCORK_OPT;
801 inet->cork.addr = ipc->addr;
802 }
803 dst_hold(&rt->u.dst);
John Heffner628a5c52007-04-20 15:53:27 -0700804 inet->cork.fragsize = mtu = inet->pmtudisc == IP_PMTUDISC_PROBE ?
805 rt->u.dst.dev->mtu :
806 dst_mtu(rt->u.dst.path);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700807 inet->cork.rt = rt;
808 inet->cork.length = 0;
809 sk->sk_sndmsg_page = NULL;
810 sk->sk_sndmsg_off = 0;
811 if ((exthdrlen = rt->u.dst.header_len) != 0) {
812 length += exthdrlen;
813 transhdrlen += exthdrlen;
814 }
815 } else {
816 rt = inet->cork.rt;
817 if (inet->cork.flags & IPCORK_OPT)
818 opt = inet->cork.opt;
819
820 transhdrlen = 0;
821 exthdrlen = 0;
822 mtu = inet->cork.fragsize;
823 }
824 hh_len = LL_RESERVED_SPACE(rt->u.dst.dev);
825
826 fragheaderlen = sizeof(struct iphdr) + (opt ? opt->optlen : 0);
827 maxfraglen = ((mtu - fragheaderlen) & ~7) + fragheaderlen;
828
829 if (inet->cork.length + length > 0xFFFF - fragheaderlen) {
830 ip_local_error(sk, EMSGSIZE, rt->rt_dst, inet->dport, mtu-exthdrlen);
831 return -EMSGSIZE;
832 }
833
834 /*
835 * transhdrlen > 0 means that this is the first fragment and we wish
836 * it won't be fragmented in the future.
837 */
838 if (transhdrlen &&
839 length + fragheaderlen <= mtu &&
Herbert Xu8648b302006-06-17 22:06:05 -0700840 rt->u.dst.dev->features & NETIF_F_ALL_CSUM &&
Linus Torvalds1da177e2005-04-16 15:20:36 -0700841 !exthdrlen)
Patrick McHardy84fa7932006-08-29 16:44:56 -0700842 csummode = CHECKSUM_PARTIAL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700843
844 inet->cork.length += length;
Ananda Rajue89e9cf2005-10-18 15:46:41 -0700845 if (((length > mtu) && (sk->sk_protocol == IPPROTO_UDP)) &&
846 (rt->u.dst.dev->features & NETIF_F_UFO)) {
847
Patrick McHardybaa829d2006-03-12 20:35:12 -0800848 err = ip_ufo_append_data(sk, getfrag, from, length, hh_len,
849 fragheaderlen, transhdrlen, mtu,
850 flags);
851 if (err)
Ananda Rajue89e9cf2005-10-18 15:46:41 -0700852 goto error;
Ananda Rajue89e9cf2005-10-18 15:46:41 -0700853 return 0;
854 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700855
856 /* So, what's going on in the loop below?
857 *
858 * We use calculated fragment length to generate chained skb,
859 * each of segments is IP fragment ready for sending to network after
860 * adding appropriate IP header.
861 */
862
863 if ((skb = skb_peek_tail(&sk->sk_write_queue)) == NULL)
864 goto alloc_new_skb;
865
866 while (length > 0) {
867 /* Check if the remaining data fits into current packet. */
868 copy = mtu - skb->len;
869 if (copy < length)
870 copy = maxfraglen - skb->len;
871 if (copy <= 0) {
872 char *data;
873 unsigned int datalen;
874 unsigned int fraglen;
875 unsigned int fraggap;
876 unsigned int alloclen;
877 struct sk_buff *skb_prev;
878alloc_new_skb:
879 skb_prev = skb;
880 if (skb_prev)
881 fraggap = skb_prev->len - maxfraglen;
882 else
883 fraggap = 0;
884
885 /*
886 * If remaining data exceeds the mtu,
887 * we know we need more fragment(s).
888 */
889 datalen = length + fraggap;
890 if (datalen > mtu - fragheaderlen)
891 datalen = maxfraglen - fragheaderlen;
892 fraglen = datalen + fragheaderlen;
893
YOSHIFUJI Hideakie905a9e2007-02-09 23:24:47 +0900894 if ((flags & MSG_MORE) &&
Linus Torvalds1da177e2005-04-16 15:20:36 -0700895 !(rt->u.dst.dev->features&NETIF_F_SG))
896 alloclen = mtu;
897 else
898 alloclen = datalen + fragheaderlen;
899
900 /* The last fragment gets additional space at tail.
901 * Note, with MSG_MORE we overallocate on fragments,
902 * because we have no idea what fragment will be
903 * the last.
904 */
Zach Brown3d9dd752006-04-14 16:04:18 -0700905 if (datalen == length + fraggap)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700906 alloclen += rt->u.dst.trailer_len;
907
908 if (transhdrlen) {
YOSHIFUJI Hideakie905a9e2007-02-09 23:24:47 +0900909 skb = sock_alloc_send_skb(sk,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700910 alloclen + hh_len + 15,
911 (flags & MSG_DONTWAIT), &err);
912 } else {
913 skb = NULL;
914 if (atomic_read(&sk->sk_wmem_alloc) <=
915 2 * sk->sk_sndbuf)
YOSHIFUJI Hideakie905a9e2007-02-09 23:24:47 +0900916 skb = sock_wmalloc(sk,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700917 alloclen + hh_len + 15, 1,
918 sk->sk_allocation);
919 if (unlikely(skb == NULL))
920 err = -ENOBUFS;
921 }
922 if (skb == NULL)
923 goto error;
924
925 /*
926 * Fill in the control structures
927 */
928 skb->ip_summed = csummode;
929 skb->csum = 0;
930 skb_reserve(skb, hh_len);
931
932 /*
933 * Find where to start putting bytes.
934 */
935 data = skb_put(skb, fraglen);
Arnaldo Carvalho de Meloc14d2452007-03-11 22:39:41 -0300936 skb_set_network_header(skb, exthdrlen);
Arnaldo Carvalho de Melob0e380b2007-04-10 21:21:55 -0700937 skb->transport_header = (skb->network_header +
938 fragheaderlen);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700939 data += fragheaderlen;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700940
941 if (fraggap) {
942 skb->csum = skb_copy_and_csum_bits(
943 skb_prev, maxfraglen,
944 data + transhdrlen, fraggap, 0);
945 skb_prev->csum = csum_sub(skb_prev->csum,
946 skb->csum);
947 data += fraggap;
Herbert Xue9fa4f72006-08-13 20:12:58 -0700948 pskb_trim_unique(skb_prev, maxfraglen);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700949 }
950
951 copy = datalen - transhdrlen - fraggap;
952 if (copy > 0 && getfrag(from, data + transhdrlen, offset, copy, fraggap, skb) < 0) {
953 err = -EFAULT;
954 kfree_skb(skb);
955 goto error;
956 }
957
958 offset += copy;
959 length -= datalen - fraggap;
960 transhdrlen = 0;
961 exthdrlen = 0;
962 csummode = CHECKSUM_NONE;
963
964 /*
965 * Put the packet on the pending queue.
966 */
967 __skb_queue_tail(&sk->sk_write_queue, skb);
968 continue;
969 }
970
971 if (copy > length)
972 copy = length;
973
974 if (!(rt->u.dst.dev->features&NETIF_F_SG)) {
975 unsigned int off;
976
977 off = skb->len;
YOSHIFUJI Hideakie905a9e2007-02-09 23:24:47 +0900978 if (getfrag(from, skb_put(skb, copy),
Linus Torvalds1da177e2005-04-16 15:20:36 -0700979 offset, copy, off, skb) < 0) {
980 __skb_trim(skb, off);
981 err = -EFAULT;
982 goto error;
983 }
984 } else {
985 int i = skb_shinfo(skb)->nr_frags;
986 skb_frag_t *frag = &skb_shinfo(skb)->frags[i-1];
987 struct page *page = sk->sk_sndmsg_page;
988 int off = sk->sk_sndmsg_off;
989 unsigned int left;
990
991 if (page && (left = PAGE_SIZE - off) > 0) {
992 if (copy >= left)
993 copy = left;
994 if (page != frag->page) {
995 if (i == MAX_SKB_FRAGS) {
996 err = -EMSGSIZE;
997 goto error;
998 }
999 get_page(page);
YOSHIFUJI Hideakie905a9e2007-02-09 23:24:47 +09001000 skb_fill_page_desc(skb, i, page, sk->sk_sndmsg_off, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001001 frag = &skb_shinfo(skb)->frags[i];
1002 }
1003 } else if (i < MAX_SKB_FRAGS) {
1004 if (copy > PAGE_SIZE)
1005 copy = PAGE_SIZE;
1006 page = alloc_pages(sk->sk_allocation, 0);
1007 if (page == NULL) {
1008 err = -ENOMEM;
1009 goto error;
1010 }
1011 sk->sk_sndmsg_page = page;
1012 sk->sk_sndmsg_off = 0;
1013
1014 skb_fill_page_desc(skb, i, page, 0, 0);
1015 frag = &skb_shinfo(skb)->frags[i];
1016 skb->truesize += PAGE_SIZE;
1017 atomic_add(PAGE_SIZE, &sk->sk_wmem_alloc);
1018 } else {
1019 err = -EMSGSIZE;
1020 goto error;
1021 }
1022 if (getfrag(from, page_address(frag->page)+frag->page_offset+frag->size, offset, copy, skb->len, skb) < 0) {
1023 err = -EFAULT;
1024 goto error;
1025 }
1026 sk->sk_sndmsg_off += copy;
1027 frag->size += copy;
1028 skb->len += copy;
1029 skb->data_len += copy;
1030 }
1031 offset += copy;
1032 length -= copy;
1033 }
1034
1035 return 0;
1036
1037error:
1038 inet->cork.length -= length;
1039 IP_INC_STATS(IPSTATS_MIB_OUTDISCARDS);
YOSHIFUJI Hideakie905a9e2007-02-09 23:24:47 +09001040 return err;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001041}
1042
1043ssize_t ip_append_page(struct sock *sk, struct page *page,
1044 int offset, size_t size, int flags)
1045{
1046 struct inet_sock *inet = inet_sk(sk);
1047 struct sk_buff *skb;
1048 struct rtable *rt;
1049 struct ip_options *opt = NULL;
1050 int hh_len;
1051 int mtu;
1052 int len;
1053 int err;
1054 unsigned int maxfraglen, fragheaderlen, fraggap;
1055
1056 if (inet->hdrincl)
1057 return -EPERM;
1058
1059 if (flags&MSG_PROBE)
1060 return 0;
1061
1062 if (skb_queue_empty(&sk->sk_write_queue))
1063 return -EINVAL;
1064
1065 rt = inet->cork.rt;
1066 if (inet->cork.flags & IPCORK_OPT)
1067 opt = inet->cork.opt;
1068
1069 if (!(rt->u.dst.dev->features&NETIF_F_SG))
1070 return -EOPNOTSUPP;
1071
1072 hh_len = LL_RESERVED_SPACE(rt->u.dst.dev);
1073 mtu = inet->cork.fragsize;
1074
1075 fragheaderlen = sizeof(struct iphdr) + (opt ? opt->optlen : 0);
1076 maxfraglen = ((mtu - fragheaderlen) & ~7) + fragheaderlen;
1077
1078 if (inet->cork.length + size > 0xFFFF - fragheaderlen) {
1079 ip_local_error(sk, EMSGSIZE, rt->rt_dst, inet->dport, mtu);
1080 return -EMSGSIZE;
1081 }
1082
1083 if ((skb = skb_peek_tail(&sk->sk_write_queue)) == NULL)
1084 return -EINVAL;
1085
1086 inet->cork.length += size;
Ananda Rajue89e9cf2005-10-18 15:46:41 -07001087 if ((sk->sk_protocol == IPPROTO_UDP) &&
Herbert Xu79671682006-06-22 02:40:14 -07001088 (rt->u.dst.dev->features & NETIF_F_UFO)) {
1089 skb_shinfo(skb)->gso_size = mtu - fragheaderlen;
Herbert Xuf83ef8c2006-06-30 13:37:03 -07001090 skb_shinfo(skb)->gso_type = SKB_GSO_UDP;
Herbert Xu79671682006-06-22 02:40:14 -07001091 }
Ananda Rajue89e9cf2005-10-18 15:46:41 -07001092
Linus Torvalds1da177e2005-04-16 15:20:36 -07001093
1094 while (size > 0) {
1095 int i;
1096
Herbert Xu89114af2006-07-08 13:34:32 -07001097 if (skb_is_gso(skb))
Ananda Rajue89e9cf2005-10-18 15:46:41 -07001098 len = size;
1099 else {
1100
1101 /* Check if the remaining data fits into current packet. */
1102 len = mtu - skb->len;
1103 if (len < size)
1104 len = maxfraglen - skb->len;
1105 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001106 if (len <= 0) {
1107 struct sk_buff *skb_prev;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001108 int alloclen;
1109
1110 skb_prev = skb;
Jayachandran C0d0d2bb2005-10-13 11:43:02 -07001111 fraggap = skb_prev->len - maxfraglen;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001112
1113 alloclen = fragheaderlen + hh_len + fraggap + 15;
1114 skb = sock_wmalloc(sk, alloclen, 1, sk->sk_allocation);
1115 if (unlikely(!skb)) {
1116 err = -ENOBUFS;
1117 goto error;
1118 }
1119
1120 /*
1121 * Fill in the control structures
1122 */
1123 skb->ip_summed = CHECKSUM_NONE;
1124 skb->csum = 0;
1125 skb_reserve(skb, hh_len);
1126
1127 /*
1128 * Find where to start putting bytes.
1129 */
Arnaldo Carvalho de Melo967b05f2007-03-13 13:51:52 -03001130 skb_put(skb, fragheaderlen + fraggap);
Arnaldo Carvalho de Melo2ca9e6f2007-03-10 19:15:25 -03001131 skb_reset_network_header(skb);
Arnaldo Carvalho de Melob0e380b2007-04-10 21:21:55 -07001132 skb->transport_header = (skb->network_header +
1133 fragheaderlen);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001134 if (fraggap) {
Arnaldo Carvalho de Melo967b05f2007-03-13 13:51:52 -03001135 skb->csum = skb_copy_and_csum_bits(skb_prev,
1136 maxfraglen,
Arnaldo Carvalho de Melo9c702202007-04-25 18:04:18 -07001137 skb_transport_header(skb),
Arnaldo Carvalho de Melo967b05f2007-03-13 13:51:52 -03001138 fraggap, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001139 skb_prev->csum = csum_sub(skb_prev->csum,
1140 skb->csum);
Herbert Xue9fa4f72006-08-13 20:12:58 -07001141 pskb_trim_unique(skb_prev, maxfraglen);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001142 }
1143
1144 /*
1145 * Put the packet on the pending queue.
1146 */
1147 __skb_queue_tail(&sk->sk_write_queue, skb);
1148 continue;
1149 }
1150
1151 i = skb_shinfo(skb)->nr_frags;
1152 if (len > size)
1153 len = size;
1154 if (skb_can_coalesce(skb, i, page, offset)) {
1155 skb_shinfo(skb)->frags[i-1].size += len;
1156 } else if (i < MAX_SKB_FRAGS) {
1157 get_page(page);
1158 skb_fill_page_desc(skb, i, page, offset, len);
1159 } else {
1160 err = -EMSGSIZE;
1161 goto error;
1162 }
1163
1164 if (skb->ip_summed == CHECKSUM_NONE) {
Al Viro44bb9362006-11-14 21:36:14 -08001165 __wsum csum;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001166 csum = csum_page(page, offset, len);
1167 skb->csum = csum_block_add(skb->csum, csum, skb->len);
1168 }
1169
1170 skb->len += len;
1171 skb->data_len += len;
1172 offset += len;
1173 size -= len;
1174 }
1175 return 0;
1176
1177error:
1178 inet->cork.length -= size;
1179 IP_INC_STATS(IPSTATS_MIB_OUTDISCARDS);
1180 return err;
1181}
1182
1183/*
1184 * Combined all pending IP fragments on the socket as one IP datagram
1185 * and push them out.
1186 */
1187int ip_push_pending_frames(struct sock *sk)
1188{
1189 struct sk_buff *skb, *tmp_skb;
1190 struct sk_buff **tail_skb;
1191 struct inet_sock *inet = inet_sk(sk);
1192 struct ip_options *opt = NULL;
1193 struct rtable *rt = inet->cork.rt;
1194 struct iphdr *iph;
Alexey Dobriyan76ab6082006-01-06 13:24:29 -08001195 __be16 df = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001196 __u8 ttl;
1197 int err = 0;
1198
1199 if ((skb = __skb_dequeue(&sk->sk_write_queue)) == NULL)
1200 goto out;
1201 tail_skb = &(skb_shinfo(skb)->frag_list);
1202
1203 /* move skb->data to ip header from ext header */
Arnaldo Carvalho de Melod56f90a2007-04-10 20:50:43 -07001204 if (skb->data < skb_network_header(skb))
Arnaldo Carvalho de Melobbe735e2007-03-10 22:16:10 -03001205 __skb_pull(skb, skb_network_offset(skb));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001206 while ((tmp_skb = __skb_dequeue(&sk->sk_write_queue)) != NULL) {
Arnaldo Carvalho de Melocfe1fc72007-03-16 17:26:39 -03001207 __skb_pull(tmp_skb, skb_network_header_len(skb));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001208 *tail_skb = tmp_skb;
1209 tail_skb = &(tmp_skb->next);
1210 skb->len += tmp_skb->len;
1211 skb->data_len += tmp_skb->len;
1212 skb->truesize += tmp_skb->truesize;
1213 __sock_put(tmp_skb->sk);
1214 tmp_skb->destructor = NULL;
1215 tmp_skb->sk = NULL;
1216 }
1217
1218 /* Unless user demanded real pmtu discovery (IP_PMTUDISC_DO), we allow
1219 * to fragment the frame generated here. No matter, what transforms
1220 * how transforms change size of the packet, it will come out.
1221 */
John Heffner628a5c52007-04-20 15:53:27 -07001222 if (inet->pmtudisc < IP_PMTUDISC_DO)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001223 skb->local_df = 1;
1224
1225 /* DF bit is set when we want to see DF on outgoing frames.
1226 * If local_df is set too, we still allow to fragment this frame
1227 * locally. */
John Heffner628a5c52007-04-20 15:53:27 -07001228 if (inet->pmtudisc >= IP_PMTUDISC_DO ||
Linus Torvalds1da177e2005-04-16 15:20:36 -07001229 (skb->len <= dst_mtu(&rt->u.dst) &&
1230 ip_dont_fragment(sk, &rt->u.dst)))
1231 df = htons(IP_DF);
1232
1233 if (inet->cork.flags & IPCORK_OPT)
1234 opt = inet->cork.opt;
1235
1236 if (rt->rt_type == RTN_MULTICAST)
1237 ttl = inet->mc_ttl;
1238 else
1239 ttl = ip_select_ttl(inet, &rt->u.dst);
1240
1241 iph = (struct iphdr *)skb->data;
1242 iph->version = 4;
1243 iph->ihl = 5;
1244 if (opt) {
1245 iph->ihl += opt->optlen>>2;
1246 ip_options_build(skb, opt, inet->cork.addr, rt, 0);
1247 }
1248 iph->tos = inet->tos;
1249 iph->tot_len = htons(skb->len);
1250 iph->frag_off = df;
Alexey Kuznetsov1a55d572006-03-22 14:27:59 -08001251 ip_select_ident(iph, &rt->u.dst, sk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001252 iph->ttl = ttl;
1253 iph->protocol = sk->sk_protocol;
1254 iph->saddr = rt->rt_src;
1255 iph->daddr = rt->rt_dst;
1256 ip_send_check(iph);
1257
1258 skb->priority = sk->sk_priority;
1259 skb->dst = dst_clone(&rt->u.dst);
1260
1261 /* Netfilter gets whole the not fragmented skb. */
YOSHIFUJI Hideakie905a9e2007-02-09 23:24:47 +09001262 err = NF_HOOK(PF_INET, NF_IP_LOCAL_OUT, skb, NULL,
Linus Torvalds1da177e2005-04-16 15:20:36 -07001263 skb->dst->dev, dst_output);
1264 if (err) {
1265 if (err > 0)
1266 err = inet->recverr ? net_xmit_errno(err) : 0;
1267 if (err)
1268 goto error;
1269 }
1270
1271out:
1272 inet->cork.flags &= ~IPCORK_OPT;
Jesper Juhla51482b2005-11-08 09:41:34 -08001273 kfree(inet->cork.opt);
1274 inet->cork.opt = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001275 if (inet->cork.rt) {
1276 ip_rt_put(inet->cork.rt);
1277 inet->cork.rt = NULL;
1278 }
1279 return err;
1280
1281error:
1282 IP_INC_STATS(IPSTATS_MIB_OUTDISCARDS);
1283 goto out;
1284}
1285
1286/*
1287 * Throw away all pending data on the socket.
1288 */
1289void ip_flush_pending_frames(struct sock *sk)
1290{
1291 struct inet_sock *inet = inet_sk(sk);
1292 struct sk_buff *skb;
1293
1294 while ((skb = __skb_dequeue_tail(&sk->sk_write_queue)) != NULL)
1295 kfree_skb(skb);
1296
1297 inet->cork.flags &= ~IPCORK_OPT;
Jesper Juhla51482b2005-11-08 09:41:34 -08001298 kfree(inet->cork.opt);
1299 inet->cork.opt = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001300 if (inet->cork.rt) {
1301 ip_rt_put(inet->cork.rt);
1302 inet->cork.rt = NULL;
1303 }
1304}
1305
1306
1307/*
1308 * Fetch data from kernel space and fill in checksum if needed.
1309 */
YOSHIFUJI Hideakie905a9e2007-02-09 23:24:47 +09001310static int ip_reply_glue_bits(void *dptr, char *to, int offset,
Linus Torvalds1da177e2005-04-16 15:20:36 -07001311 int len, int odd, struct sk_buff *skb)
1312{
Al Viro50842052006-11-14 21:36:34 -08001313 __wsum csum;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001314
1315 csum = csum_partial_copy_nocheck(dptr+offset, to, len, 0);
1316 skb->csum = csum_block_add(skb->csum, csum, odd);
YOSHIFUJI Hideakie905a9e2007-02-09 23:24:47 +09001317 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001318}
1319
YOSHIFUJI Hideakie905a9e2007-02-09 23:24:47 +09001320/*
Linus Torvalds1da177e2005-04-16 15:20:36 -07001321 * Generic function to send a packet as reply to another packet.
1322 * Used to send TCP resets so far. ICMP should use this function too.
1323 *
YOSHIFUJI Hideakie905a9e2007-02-09 23:24:47 +09001324 * Should run single threaded per socket because it uses the sock
Linus Torvalds1da177e2005-04-16 15:20:36 -07001325 * structure to pass arguments.
1326 *
1327 * LATER: switch from ip_build_xmit to ip_append_*
1328 */
1329void ip_send_reply(struct sock *sk, struct sk_buff *skb, struct ip_reply_arg *arg,
1330 unsigned int len)
1331{
1332 struct inet_sock *inet = inet_sk(sk);
1333 struct {
1334 struct ip_options opt;
1335 char data[40];
1336 } replyopts;
1337 struct ipcm_cookie ipc;
Al Viro3ca3c682006-09-27 18:28:07 -07001338 __be32 daddr;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001339 struct rtable *rt = (struct rtable*)skb->dst;
1340
1341 if (ip_options_echo(&replyopts.opt, skb))
1342 return;
1343
1344 daddr = ipc.addr = rt->rt_src;
1345 ipc.opt = NULL;
1346
1347 if (replyopts.opt.optlen) {
1348 ipc.opt = &replyopts.opt;
1349
1350 if (ipc.opt->srr)
1351 daddr = replyopts.opt.faddr;
1352 }
1353
1354 {
1355 struct flowi fl = { .nl_u = { .ip4_u =
1356 { .daddr = daddr,
1357 .saddr = rt->rt_spec_dst,
Arnaldo Carvalho de Meloeddc9ec2007-04-20 22:47:35 -07001358 .tos = RT_TOS(ip_hdr(skb)->tos) } },
Linus Torvalds1da177e2005-04-16 15:20:36 -07001359 /* Not quite clean, but right. */
1360 .uli_u = { .ports =
Arnaldo Carvalho de Meloaa8223c2007-04-10 21:04:22 -07001361 { .sport = tcp_hdr(skb)->dest,
1362 .dport = tcp_hdr(skb)->source } },
Linus Torvalds1da177e2005-04-16 15:20:36 -07001363 .proto = sk->sk_protocol };
Venkat Yekkiralabeb8d132006-08-04 23:12:42 -07001364 security_skb_classify_flow(skb, &fl);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001365 if (ip_route_output_key(&rt, &fl))
1366 return;
1367 }
1368
1369 /* And let IP do all the hard work.
1370
1371 This chunk is not reenterable, hence spinlock.
1372 Note that it uses the fact, that this function is called
1373 with locally disabled BH and that sk cannot be already spinlocked.
1374 */
1375 bh_lock_sock(sk);
Arnaldo Carvalho de Meloeddc9ec2007-04-20 22:47:35 -07001376 inet->tos = ip_hdr(skb)->tos;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001377 sk->sk_priority = skb->priority;
Arnaldo Carvalho de Meloeddc9ec2007-04-20 22:47:35 -07001378 sk->sk_protocol = ip_hdr(skb)->protocol;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001379 ip_append_data(sk, ip_reply_glue_bits, arg->iov->iov_base, len, 0,
1380 &ipc, rt, MSG_DONTWAIT);
1381 if ((skb = skb_peek(&sk->sk_write_queue)) != NULL) {
1382 if (arg->csumoffset >= 0)
Arnaldo Carvalho de Melo9c702202007-04-25 18:04:18 -07001383 *((__sum16 *)skb_transport_header(skb) +
1384 arg->csumoffset) = csum_fold(csum_add(skb->csum,
1385 arg->csum));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001386 skb->ip_summed = CHECKSUM_NONE;
1387 ip_push_pending_frames(sk);
1388 }
1389
1390 bh_unlock_sock(sk);
1391
1392 ip_rt_put(rt);
1393}
1394
Linus Torvalds1da177e2005-04-16 15:20:36 -07001395void __init ip_init(void)
1396{
Linus Torvalds1da177e2005-04-16 15:20:36 -07001397 ip_rt_init();
1398 inet_initpeers();
1399
1400#if defined(CONFIG_IP_MULTICAST) && defined(CONFIG_PROC_FS)
1401 igmp_mc_proc_init();
1402#endif
1403}
1404
Linus Torvalds1da177e2005-04-16 15:20:36 -07001405EXPORT_SYMBOL(ip_generic_getfrag);
1406EXPORT_SYMBOL(ip_queue_xmit);
1407EXPORT_SYMBOL(ip_send_check);