blob: 70778d48aa7b19ce9531f5b96beb48fb2896e425 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * INET An implementation of the TCP/IP protocol suite for the LINUX
3 * operating system. INET is implemented using the BSD Socket
4 * interface as the means of communication with the user level.
5 *
6 * The Internet Protocol (IP) output module.
7 *
Jesper Juhl02c30a82005-05-05 16:16:16 -07008 * Authors: Ross Biro
Linus Torvalds1da177e2005-04-16 15:20:36 -07009 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
10 * Donald Becker, <becker@super.org>
11 * Alan Cox, <Alan.Cox@linux.org>
12 * Richard Underwood
13 * Stefan Becker, <stefanb@yello.ping.de>
14 * Jorge Cwik, <jorge@laser.satlink.net>
15 * Arnt Gulbrandsen, <agulbra@nvg.unit.no>
16 * Hirokazu Takahashi, <taka@valinux.co.jp>
17 *
18 * See ip_input.c for original log
19 *
20 * Fixes:
21 * Alan Cox : Missing nonblock feature in ip_build_xmit.
22 * Mike Kilburn : htons() missing in ip_build_xmit.
YOSHIFUJI Hideakie905a9e2007-02-09 23:24:47 +090023 * Bradford Johnson: Fix faulty handling of some frames when
Linus Torvalds1da177e2005-04-16 15:20:36 -070024 * no route is found.
25 * Alexander Demenshin: Missing sk/skb free in ip_queue_xmit
26 * (in case if packet not accepted by
27 * output firewall rules)
28 * Mike McLagan : Routing by source
29 * Alexey Kuznetsov: use new route cache
30 * Andi Kleen: Fix broken PMTU recovery and remove
31 * some redundant tests.
32 * Vitaly E. Lavrov : Transparent proxy revived after year coma.
33 * Andi Kleen : Replace ip_reply with ip_send_reply.
YOSHIFUJI Hideakie905a9e2007-02-09 23:24:47 +090034 * Andi Kleen : Split fast and slow ip_build_xmit path
35 * for decreased register pressure on x86
36 * and more readibility.
Linus Torvalds1da177e2005-04-16 15:20:36 -070037 * Marc Boucher : When call_out_firewall returns FW_QUEUE,
38 * silently drop skb instead of failing with -EPERM.
39 * Detlev Wengorz : Copy protocol for fragments.
40 * Hirokazu Takahashi: HW checksumming for outgoing UDP
41 * datagrams.
42 * Hirokazu Takahashi: sendfile() on UDP works now.
43 */
44
45#include <asm/uaccess.h>
46#include <asm/system.h>
47#include <linux/module.h>
48#include <linux/types.h>
49#include <linux/kernel.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070050#include <linux/mm.h>
51#include <linux/string.h>
52#include <linux/errno.h>
Al Viroa1f8e7f72006-10-19 16:08:53 -040053#include <linux/highmem.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090054#include <linux/slab.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070055
56#include <linux/socket.h>
57#include <linux/sockios.h>
58#include <linux/in.h>
59#include <linux/inet.h>
60#include <linux/netdevice.h>
61#include <linux/etherdevice.h>
62#include <linux/proc_fs.h>
63#include <linux/stat.h>
64#include <linux/init.h>
65
66#include <net/snmp.h>
67#include <net/ip.h>
68#include <net/protocol.h>
69#include <net/route.h>
Patrick McHardycfacb052006-01-08 22:36:54 -080070#include <net/xfrm.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070071#include <linux/skbuff.h>
72#include <net/sock.h>
73#include <net/arp.h>
74#include <net/icmp.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070075#include <net/checksum.h>
76#include <net/inetpeer.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070077#include <linux/igmp.h>
78#include <linux/netfilter_ipv4.h>
79#include <linux/netfilter_bridge.h>
80#include <linux/mroute.h>
81#include <linux/netlink.h>
Arnaldo Carvalho de Melo6cbb0df2005-08-09 19:49:02 -070082#include <linux/tcp.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070083
Brian Haleyab32ea52006-09-22 14:15:41 -070084int sysctl_ip_default_ttl __read_mostly = IPDEFTTL;
David S. Miller323e1262010-12-12 21:55:08 -080085EXPORT_SYMBOL(sysctl_ip_default_ttl);
Linus Torvalds1da177e2005-04-16 15:20:36 -070086
87/* Generate a checksum for an outgoing IP datagram. */
88__inline__ void ip_send_check(struct iphdr *iph)
89{
90 iph->check = 0;
91 iph->check = ip_fast_csum((unsigned char *)iph, iph->ihl);
92}
Eric Dumazet4bc2f182010-07-09 21:22:10 +000093EXPORT_SYMBOL(ip_send_check);
Linus Torvalds1da177e2005-04-16 15:20:36 -070094
Herbert Xuc439cb22008-01-11 19:14:00 -080095int __ip_local_out(struct sk_buff *skb)
96{
97 struct iphdr *iph = ip_hdr(skb);
98
99 iph->tot_len = htons(skb->len);
100 ip_send_check(iph);
Jan Engelhardt9bbc7682010-03-23 04:07:29 +0100101 return nf_hook(NFPROTO_IPV4, NF_INET_LOCAL_OUT, skb, NULL,
102 skb_dst(skb)->dev, dst_output);
Herbert Xuc439cb22008-01-11 19:14:00 -0800103}
104
105int ip_local_out(struct sk_buff *skb)
106{
107 int err;
108
109 err = __ip_local_out(skb);
110 if (likely(err == 1))
111 err = dst_output(skb);
112
113 return err;
114}
115EXPORT_SYMBOL_GPL(ip_local_out);
116
Linus Torvalds1da177e2005-04-16 15:20:36 -0700117/* dev_loopback_xmit for use with netfilter. */
118static int ip_dev_loopback_xmit(struct sk_buff *newskb)
119{
Arnaldo Carvalho de Melo459a98e2007-03-19 15:30:44 -0700120 skb_reset_mac_header(newskb);
Arnaldo Carvalho de Melobbe735e2007-03-10 22:16:10 -0300121 __skb_pull(newskb, skb_network_offset(newskb));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700122 newskb->pkt_type = PACKET_LOOPBACK;
123 newskb->ip_summed = CHECKSUM_UNNECESSARY;
Eric Dumazetadf30902009-06-02 05:19:30 +0000124 WARN_ON(!skb_dst(newskb));
Eric Dumazete30b38c2010-04-15 09:13:03 +0000125 netif_rx_ni(newskb);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700126 return 0;
127}
128
129static inline int ip_select_ttl(struct inet_sock *inet, struct dst_entry *dst)
130{
131 int ttl = inet->uc_ttl;
132
133 if (ttl < 0)
David S. Miller323e1262010-12-12 21:55:08 -0800134 ttl = ip4_dst_hoplimit(dst);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700135 return ttl;
136}
137
YOSHIFUJI Hideakie905a9e2007-02-09 23:24:47 +0900138/*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700139 * Add an ip header to a skbuff and send it out.
140 *
141 */
142int ip_build_and_send_pkt(struct sk_buff *skb, struct sock *sk,
Eric Dumazetf6d8bd02011-04-21 09:45:37 +0000143 __be32 saddr, __be32 daddr, struct ip_options_rcu *opt)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700144{
145 struct inet_sock *inet = inet_sk(sk);
Eric Dumazet511c3f92009-06-02 05:14:27 +0000146 struct rtable *rt = skb_rtable(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700147 struct iphdr *iph;
148
149 /* Build the IP header. */
Eric Dumazetf6d8bd02011-04-21 09:45:37 +0000150 skb_push(skb, sizeof(struct iphdr) + (opt ? opt->opt.optlen : 0));
Arnaldo Carvalho de Melo8856dfa2007-03-10 19:40:39 -0300151 skb_reset_network_header(skb);
Arnaldo Carvalho de Meloeddc9ec2007-04-20 22:47:35 -0700152 iph = ip_hdr(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700153 iph->version = 4;
154 iph->ihl = 5;
155 iph->tos = inet->tos;
Changli Gaod8d1f302010-06-10 23:31:35 -0700156 if (ip_dont_fragment(sk, &rt->dst))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700157 iph->frag_off = htons(IP_DF);
158 else
159 iph->frag_off = 0;
Changli Gaod8d1f302010-06-10 23:31:35 -0700160 iph->ttl = ip_select_ttl(inet, &rt->dst);
David S. Millerdd927a22011-05-04 12:03:30 -0700161 iph->daddr = (opt && opt->opt.srr ? opt->opt.faddr : daddr);
162 iph->saddr = saddr;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700163 iph->protocol = sk->sk_protocol;
Changli Gaod8d1f302010-06-10 23:31:35 -0700164 ip_select_ident(iph, &rt->dst, sk);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700165
Eric Dumazetf6d8bd02011-04-21 09:45:37 +0000166 if (opt && opt->opt.optlen) {
167 iph->ihl += opt->opt.optlen>>2;
168 ip_options_build(skb, &opt->opt, daddr, rt, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700169 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700170
171 skb->priority = sk->sk_priority;
Laszlo Attila Toth4a19ec52008-01-30 19:08:16 -0800172 skb->mark = sk->sk_mark;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700173
174 /* Send it out. */
Herbert Xuc439cb22008-01-11 19:14:00 -0800175 return ip_local_out(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700176}
Arnaldo Carvalho de Melod8c97a92005-08-09 20:12:12 -0700177EXPORT_SYMBOL_GPL(ip_build_and_send_pkt);
178
Linus Torvalds1da177e2005-04-16 15:20:36 -0700179static inline int ip_finish_output2(struct sk_buff *skb)
180{
Eric Dumazetadf30902009-06-02 05:19:30 +0000181 struct dst_entry *dst = skb_dst(skb);
Mitsuru Chinen80787eb2007-04-30 00:48:20 -0700182 struct rtable *rt = (struct rtable *)dst;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700183 struct net_device *dev = dst->dev;
Chuck Leverc2636b42007-10-23 21:07:32 -0700184 unsigned int hh_len = LL_RESERVED_SPACE(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700185
Neil Hormanedf391f2009-04-27 02:45:02 -0700186 if (rt->rt_type == RTN_MULTICAST) {
187 IP_UPD_PO_STATS(dev_net(dev), IPSTATS_MIB_OUTMCAST, skb->len);
188 } else if (rt->rt_type == RTN_BROADCAST)
189 IP_UPD_PO_STATS(dev_net(dev), IPSTATS_MIB_OUTBCAST, skb->len);
Mitsuru Chinen80787eb2007-04-30 00:48:20 -0700190
Linus Torvalds1da177e2005-04-16 15:20:36 -0700191 /* Be paranoid, rather than too clever. */
Stephen Hemminger3b04ddd2007-10-09 01:40:57 -0700192 if (unlikely(skb_headroom(skb) < hh_len && dev->header_ops)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700193 struct sk_buff *skb2;
194
195 skb2 = skb_realloc_headroom(skb, LL_RESERVED_SPACE(dev));
196 if (skb2 == NULL) {
197 kfree_skb(skb);
198 return -ENOMEM;
199 }
200 if (skb->sk)
201 skb_set_owner_w(skb2, skb->sk);
202 kfree_skb(skb);
203 skb = skb2;
204 }
205
Stephen Hemminger3644f0c2006-12-07 15:08:17 -0800206 if (dst->hh)
207 return neigh_hh_output(dst->hh, skb);
208 else if (dst->neighbour)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700209 return dst->neighbour->output(skb);
210
211 if (net_ratelimit())
212 printk(KERN_DEBUG "ip_finish_output2: No header cache and no neighbour!\n");
213 kfree_skb(skb);
214 return -EINVAL;
215}
216
John Heffner628a5c52007-04-20 15:53:27 -0700217static inline int ip_skb_dst_mtu(struct sk_buff *skb)
218{
219 struct inet_sock *inet = skb->sk ? inet_sk(skb->sk) : NULL;
220
221 return (inet && inet->pmtudisc == IP_PMTUDISC_PROBE) ?
Eric Dumazetadf30902009-06-02 05:19:30 +0000222 skb_dst(skb)->dev->mtu : dst_mtu(skb_dst(skb));
John Heffner628a5c52007-04-20 15:53:27 -0700223}
224
Patrick McHardy861d0482007-10-15 01:48:39 -0700225static int ip_finish_output(struct sk_buff *skb)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700226{
Patrick McHardy5c901da2006-01-06 23:05:36 -0800227#if defined(CONFIG_NETFILTER) && defined(CONFIG_XFRM)
228 /* Policy lookup after SNAT yielded a new policy */
Eric Dumazetadf30902009-06-02 05:19:30 +0000229 if (skb_dst(skb)->xfrm != NULL) {
Patrick McHardy48d5cad2006-02-15 15:10:22 -0800230 IPCB(skb)->flags |= IPSKB_REROUTED;
231 return dst_output(skb);
232 }
Patrick McHardy5c901da2006-01-06 23:05:36 -0800233#endif
John Heffner628a5c52007-04-20 15:53:27 -0700234 if (skb->len > ip_skb_dst_mtu(skb) && !skb_is_gso(skb))
Patrick McHardy1bd9bef2006-01-05 12:20:59 -0800235 return ip_fragment(skb, ip_finish_output2);
236 else
237 return ip_finish_output2(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700238}
239
240int ip_mc_output(struct sk_buff *skb)
241{
242 struct sock *sk = skb->sk;
Eric Dumazet511c3f92009-06-02 05:14:27 +0000243 struct rtable *rt = skb_rtable(skb);
Changli Gaod8d1f302010-06-10 23:31:35 -0700244 struct net_device *dev = rt->dst.dev;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700245
246 /*
247 * If the indicated interface is up and running, send the packet.
248 */
Neil Hormanedf391f2009-04-27 02:45:02 -0700249 IP_UPD_PO_STATS(dev_net(dev), IPSTATS_MIB_OUT, skb->len);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700250
251 skb->dev = dev;
252 skb->protocol = htons(ETH_P_IP);
253
254 /*
255 * Multicasts are looped back for other local users
256 */
257
258 if (rt->rt_flags&RTCF_MULTICAST) {
Octavian Purdila7ad68482010-01-06 20:37:01 -0800259 if (sk_mc_loop(sk)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700260#ifdef CONFIG_IP_MROUTE
261 /* Small optimization: do not loopback not local frames,
262 which returned after forwarding; they will be dropped
263 by ip_mr_input in any case.
264 Note, that local frames are looped back to be delivered
265 to local recipients.
266
267 This check is duplicated in ip_mr_input at the moment.
268 */
Joe Perches9d4fb272009-11-23 10:41:23 -0800269 &&
270 ((rt->rt_flags & RTCF_LOCAL) ||
271 !(IPCB(skb)->flags & IPSKB_FORWARDED))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700272#endif
Joe Perches9d4fb272009-11-23 10:41:23 -0800273 ) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700274 struct sk_buff *newskb = skb_clone(skb, GFP_ATOMIC);
275 if (newskb)
Jan Engelhardt9bbc7682010-03-23 04:07:29 +0100276 NF_HOOK(NFPROTO_IPV4, NF_INET_POST_ROUTING,
277 newskb, NULL, newskb->dev,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700278 ip_dev_loopback_xmit);
279 }
280
281 /* Multicasts with ttl 0 must not go beyond the host */
282
Arnaldo Carvalho de Meloeddc9ec2007-04-20 22:47:35 -0700283 if (ip_hdr(skb)->ttl == 0) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700284 kfree_skb(skb);
285 return 0;
286 }
287 }
288
289 if (rt->rt_flags&RTCF_BROADCAST) {
290 struct sk_buff *newskb = skb_clone(skb, GFP_ATOMIC);
291 if (newskb)
Jan Engelhardt9bbc7682010-03-23 04:07:29 +0100292 NF_HOOK(NFPROTO_IPV4, NF_INET_POST_ROUTING, newskb,
293 NULL, newskb->dev, ip_dev_loopback_xmit);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700294 }
295
Jan Engelhardt9bbc7682010-03-23 04:07:29 +0100296 return NF_HOOK_COND(NFPROTO_IPV4, NF_INET_POST_ROUTING, skb, NULL,
297 skb->dev, ip_finish_output,
Patrick McHardy48d5cad2006-02-15 15:10:22 -0800298 !(IPCB(skb)->flags & IPSKB_REROUTED));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700299}
300
301int ip_output(struct sk_buff *skb)
302{
Eric Dumazetadf30902009-06-02 05:19:30 +0000303 struct net_device *dev = skb_dst(skb)->dev;
Patrick McHardy1bd9bef2006-01-05 12:20:59 -0800304
Neil Hormanedf391f2009-04-27 02:45:02 -0700305 IP_UPD_PO_STATS(dev_net(dev), IPSTATS_MIB_OUT, skb->len);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700306
Patrick McHardy1bd9bef2006-01-05 12:20:59 -0800307 skb->dev = dev;
308 skb->protocol = htons(ETH_P_IP);
309
Jan Engelhardt9bbc7682010-03-23 04:07:29 +0100310 return NF_HOOK_COND(NFPROTO_IPV4, NF_INET_POST_ROUTING, skb, NULL, dev,
YOSHIFUJI Hideakie905a9e2007-02-09 23:24:47 +0900311 ip_finish_output,
Patrick McHardy48d5cad2006-02-15 15:10:22 -0800312 !(IPCB(skb)->flags & IPSKB_REROUTED));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700313}
314
David S. Millerd9d8da82011-05-06 22:23:20 -0700315int ip_queue_xmit(struct sk_buff *skb, struct flowi *fl)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700316{
David S. Millere89862f2007-01-26 01:04:55 -0800317 struct sock *sk = skb->sk;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700318 struct inet_sock *inet = inet_sk(sk);
Eric Dumazetf6d8bd02011-04-21 09:45:37 +0000319 struct ip_options_rcu *inet_opt;
David S. Millerb57ae012011-05-06 16:24:06 -0700320 struct flowi4 *fl4;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700321 struct rtable *rt;
322 struct iphdr *iph;
Eric Dumazetab6e3fe2010-05-10 11:31:49 +0000323 int res;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700324
325 /* Skip all of this if the packet is already routed,
326 * f.e. by something like SCTP.
327 */
Eric Dumazetab6e3fe2010-05-10 11:31:49 +0000328 rcu_read_lock();
Eric Dumazetf6d8bd02011-04-21 09:45:37 +0000329 inet_opt = rcu_dereference(inet->inet_opt);
David S. Millerea4fc0d2011-05-06 22:30:20 -0700330 fl4 = &fl->u.ip4;
Eric Dumazet511c3f92009-06-02 05:14:27 +0000331 rt = skb_rtable(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700332 if (rt != NULL)
333 goto packet_routed;
334
335 /* Make sure we can route this packet. */
336 rt = (struct rtable *)__sk_dst_check(sk, 0);
337 if (rt == NULL) {
Al Viro3ca3c682006-09-27 18:28:07 -0700338 __be32 daddr;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700339
340 /* Use correct destination address if we have options. */
Eric Dumazetc720c7e2009-10-15 06:30:45 +0000341 daddr = inet->inet_daddr;
Eric Dumazetf6d8bd02011-04-21 09:45:37 +0000342 if (inet_opt && inet_opt->opt.srr)
343 daddr = inet_opt->opt.faddr;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700344
David S. Miller78fbfd82011-03-12 00:00:52 -0500345 /* If this fails, retransmit mechanism of transport layer will
346 * keep trying until route appears or the connection times
347 * itself out.
348 */
David S. Millerb57ae012011-05-06 16:24:06 -0700349 rt = ip_route_output_ports(sock_net(sk), fl4, sk,
David S. Miller78fbfd82011-03-12 00:00:52 -0500350 daddr, inet->inet_saddr,
351 inet->inet_dport,
352 inet->inet_sport,
353 sk->sk_protocol,
354 RT_CONN_FLAGS(sk),
355 sk->sk_bound_dev_if);
356 if (IS_ERR(rt))
357 goto no_route;
Changli Gaod8d1f302010-06-10 23:31:35 -0700358 sk_setup_caps(sk, &rt->dst);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700359 }
Changli Gaod8d1f302010-06-10 23:31:35 -0700360 skb_dst_set_noref(skb, &rt->dst);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700361
362packet_routed:
David S. Millerea4fc0d2011-05-06 22:30:20 -0700363 if (inet_opt && inet_opt->opt.is_strictroute && fl4->daddr != rt->rt_gateway)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700364 goto no_route;
365
366 /* OK, we know where to send it, allocate and build IP header. */
Eric Dumazetf6d8bd02011-04-21 09:45:37 +0000367 skb_push(skb, sizeof(struct iphdr) + (inet_opt ? inet_opt->opt.optlen : 0));
Arnaldo Carvalho de Melo8856dfa2007-03-10 19:40:39 -0300368 skb_reset_network_header(skb);
Arnaldo Carvalho de Meloeddc9ec2007-04-20 22:47:35 -0700369 iph = ip_hdr(skb);
Al Viro714e85b2006-11-14 20:51:49 -0800370 *((__be16 *)iph) = htons((4 << 12) | (5 << 8) | (inet->tos & 0xff));
Changli Gaod8d1f302010-06-10 23:31:35 -0700371 if (ip_dont_fragment(sk, &rt->dst) && !skb->local_df)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700372 iph->frag_off = htons(IP_DF);
373 else
374 iph->frag_off = 0;
Changli Gaod8d1f302010-06-10 23:31:35 -0700375 iph->ttl = ip_select_ttl(inet, &rt->dst);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700376 iph->protocol = sk->sk_protocol;
David S. Millerea4fc0d2011-05-06 22:30:20 -0700377 iph->saddr = fl4->saddr;
378 iph->daddr = fl4->daddr;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700379 /* Transport layer set skb->h.foo itself. */
380
Eric Dumazetf6d8bd02011-04-21 09:45:37 +0000381 if (inet_opt && inet_opt->opt.optlen) {
382 iph->ihl += inet_opt->opt.optlen >> 2;
383 ip_options_build(skb, &inet_opt->opt, inet->inet_daddr, rt, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700384 }
385
Changli Gaod8d1f302010-06-10 23:31:35 -0700386 ip_select_ident_more(iph, &rt->dst, sk,
Herbert Xu79671682006-06-22 02:40:14 -0700387 (skb_shinfo(skb)->gso_segs ?: 1) - 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700388
Linus Torvalds1da177e2005-04-16 15:20:36 -0700389 skb->priority = sk->sk_priority;
Laszlo Attila Toth4a19ec52008-01-30 19:08:16 -0800390 skb->mark = sk->sk_mark;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700391
Eric Dumazetab6e3fe2010-05-10 11:31:49 +0000392 res = ip_local_out(skb);
393 rcu_read_unlock();
394 return res;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700395
396no_route:
Eric Dumazetab6e3fe2010-05-10 11:31:49 +0000397 rcu_read_unlock();
Pavel Emelyanov5e38e272008-07-16 20:19:49 -0700398 IP_INC_STATS(sock_net(sk), IPSTATS_MIB_OUTNOROUTES);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700399 kfree_skb(skb);
400 return -EHOSTUNREACH;
401}
Eric Dumazet4bc2f182010-07-09 21:22:10 +0000402EXPORT_SYMBOL(ip_queue_xmit);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700403
404
405static void ip_copy_metadata(struct sk_buff *to, struct sk_buff *from)
406{
407 to->pkt_type = from->pkt_type;
408 to->priority = from->priority;
409 to->protocol = from->protocol;
Eric Dumazetadf30902009-06-02 05:19:30 +0000410 skb_dst_drop(to);
Eric Dumazetfe76cda2010-07-01 23:48:22 +0000411 skb_dst_copy(to, from);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700412 to->dev = from->dev;
Thomas Graf82e91ff2006-11-09 15:19:14 -0800413 to->mark = from->mark;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700414
415 /* Copy the flags to each fragment. */
416 IPCB(to)->flags = IPCB(from)->flags;
417
418#ifdef CONFIG_NET_SCHED
419 to->tc_index = from->tc_index;
420#endif
Yasuyuki Kozakaie7ac05f2007-03-14 16:44:01 -0700421 nf_copy(to, from);
Jozsef Kadlecsikba9dda32007-07-07 22:21:23 -0700422#if defined(CONFIG_NETFILTER_XT_TARGET_TRACE) || \
423 defined(CONFIG_NETFILTER_XT_TARGET_TRACE_MODULE)
424 to->nf_trace = from->nf_trace;
425#endif
Julian Anastasovc98d80e2005-10-22 13:39:21 +0300426#if defined(CONFIG_IP_VS) || defined(CONFIG_IP_VS_MODULE)
427 to->ipvs_property = from->ipvs_property;
428#endif
James Morris984bc162006-06-09 00:29:17 -0700429 skb_copy_secmark(to, from);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700430}
431
432/*
433 * This IP datagram is too large to be sent in one piece. Break it up into
434 * smaller pieces (each of size equal to IP header plus
435 * a block of the data of the original IP data part) that will yet fit in a
436 * single device frame, and queue such a frame for sending.
437 */
438
Jianjun Kongd93191002008-11-03 00:23:42 -0800439int ip_fragment(struct sk_buff *skb, int (*output)(struct sk_buff *))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700440{
441 struct iphdr *iph;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700442 int ptr;
443 struct net_device *dev;
444 struct sk_buff *skb2;
Changli Gaoc893b802010-07-31 13:25:08 +0000445 unsigned int mtu, hlen, left, len, ll_rs;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700446 int offset;
Alexey Dobriyan76ab6082006-01-06 13:24:29 -0800447 __be16 not_last_frag;
Eric Dumazet511c3f92009-06-02 05:14:27 +0000448 struct rtable *rt = skb_rtable(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700449 int err = 0;
450
Changli Gaod8d1f302010-06-10 23:31:35 -0700451 dev = rt->dst.dev;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700452
453 /*
454 * Point into the IP datagram header.
455 */
456
Arnaldo Carvalho de Meloeddc9ec2007-04-20 22:47:35 -0700457 iph = ip_hdr(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700458
459 if (unlikely((iph->frag_off & htons(IP_DF)) && !skb->local_df)) {
Pavel Emelyanov5e38e272008-07-16 20:19:49 -0700460 IP_INC_STATS(dev_net(dev), IPSTATS_MIB_FRAGFAILS);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700461 icmp_send(skb, ICMP_DEST_UNREACH, ICMP_FRAG_NEEDED,
John Heffner628a5c52007-04-20 15:53:27 -0700462 htonl(ip_skb_dst_mtu(skb)));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700463 kfree_skb(skb);
464 return -EMSGSIZE;
465 }
466
467 /*
468 * Setup starting values.
469 */
470
471 hlen = iph->ihl * 4;
Changli Gaod8d1f302010-06-10 23:31:35 -0700472 mtu = dst_mtu(&rt->dst) - hlen; /* Size of data space */
Bart De Schuymer6c79bf02010-04-20 16:22:01 +0200473#ifdef CONFIG_BRIDGE_NETFILTER
474 if (skb->nf_bridge)
475 mtu -= nf_bridge_mtu_reduction(skb);
476#endif
Herbert Xu89cee8b2005-12-13 23:14:27 -0800477 IPCB(skb)->flags |= IPSKB_FRAG_COMPLETE;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700478
479 /* When frag_list is given, use it. First, check its validity:
480 * some transformers could create wrong frag_list or break existing
481 * one, it is not prohibited. In this case fall back to copying.
482 *
483 * LATER: this step can be merged to real generation of fragments,
484 * we can switch to copy when see the first bad fragment.
485 */
David S. Miller21dc3302010-08-23 00:13:46 -0700486 if (skb_has_frag_list(skb)) {
Eric Dumazet3d130082010-09-21 08:47:45 +0000487 struct sk_buff *frag, *frag2;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700488 int first_len = skb_pagelen(skb);
489
490 if (first_len - hlen > mtu ||
491 ((first_len - hlen) & 7) ||
492 (iph->frag_off & htons(IP_MF|IP_OFFSET)) ||
493 skb_cloned(skb))
494 goto slow_path;
495
David S. Millerd7fcf1a2009-06-09 00:19:37 -0700496 skb_walk_frags(skb, frag) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700497 /* Correct geometry. */
498 if (frag->len > mtu ||
499 ((frag->len & 7) && frag->next) ||
500 skb_headroom(frag) < hlen)
Eric Dumazet3d130082010-09-21 08:47:45 +0000501 goto slow_path_clean;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700502
503 /* Partially cloned skb? */
504 if (skb_shared(frag))
Eric Dumazet3d130082010-09-21 08:47:45 +0000505 goto slow_path_clean;
Herbert Xu2fdba6b2005-05-18 22:52:33 -0700506
507 BUG_ON(frag->sk);
508 if (skb->sk) {
Herbert Xu2fdba6b2005-05-18 22:52:33 -0700509 frag->sk = skb->sk;
510 frag->destructor = sock_wfree;
Herbert Xu2fdba6b2005-05-18 22:52:33 -0700511 }
Eric Dumazet3d130082010-09-21 08:47:45 +0000512 skb->truesize -= frag->truesize;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700513 }
514
515 /* Everything is OK. Generate! */
516
517 err = 0;
518 offset = 0;
519 frag = skb_shinfo(skb)->frag_list;
David S. Millerd7fcf1a2009-06-09 00:19:37 -0700520 skb_frag_list_init(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700521 skb->data_len = first_len - skb_headlen(skb);
522 skb->len = first_len;
523 iph->tot_len = htons(first_len);
524 iph->frag_off = htons(IP_MF);
525 ip_send_check(iph);
526
527 for (;;) {
528 /* Prepare header of the next frame,
529 * before previous one went down. */
530 if (frag) {
531 frag->ip_summed = CHECKSUM_NONE;
Arnaldo Carvalho de Melobadff6d2007-03-13 13:06:52 -0300532 skb_reset_transport_header(frag);
Arnaldo Carvalho de Meloe2d1bca2007-04-10 20:46:21 -0700533 __skb_push(frag, hlen);
534 skb_reset_network_header(frag);
Arnaldo Carvalho de Melod56f90a2007-04-10 20:50:43 -0700535 memcpy(skb_network_header(frag), iph, hlen);
Arnaldo Carvalho de Meloeddc9ec2007-04-20 22:47:35 -0700536 iph = ip_hdr(frag);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700537 iph->tot_len = htons(frag->len);
538 ip_copy_metadata(frag, skb);
539 if (offset == 0)
540 ip_options_fragment(frag);
541 offset += skb->len - hlen;
542 iph->frag_off = htons(offset>>3);
543 if (frag->next != NULL)
544 iph->frag_off |= htons(IP_MF);
545 /* Ready, complete checksum */
546 ip_send_check(iph);
547 }
548
549 err = output(skb);
550
Wei Dongdafee492006-08-02 13:41:21 -0700551 if (!err)
Pavel Emelyanov5e38e272008-07-16 20:19:49 -0700552 IP_INC_STATS(dev_net(dev), IPSTATS_MIB_FRAGCREATES);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700553 if (err || !frag)
554 break;
555
556 skb = frag;
557 frag = skb->next;
558 skb->next = NULL;
559 }
560
561 if (err == 0) {
Pavel Emelyanov5e38e272008-07-16 20:19:49 -0700562 IP_INC_STATS(dev_net(dev), IPSTATS_MIB_FRAGOKS);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700563 return 0;
564 }
565
566 while (frag) {
567 skb = frag->next;
568 kfree_skb(frag);
569 frag = skb;
570 }
Pavel Emelyanov5e38e272008-07-16 20:19:49 -0700571 IP_INC_STATS(dev_net(dev), IPSTATS_MIB_FRAGFAILS);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700572 return err;
Eric Dumazet3d130082010-09-21 08:47:45 +0000573
574slow_path_clean:
575 skb_walk_frags(skb, frag2) {
576 if (frag2 == frag)
577 break;
578 frag2->sk = NULL;
579 frag2->destructor = NULL;
580 skb->truesize += frag2->truesize;
581 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700582 }
583
584slow_path:
585 left = skb->len - hlen; /* Space per frame */
George Kadianakis49085bd2010-07-06 11:44:12 +0000586 ptr = hlen; /* Where to start from */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700587
Linus Torvalds1da177e2005-04-16 15:20:36 -0700588 /* for bridged IP traffic encapsulated inside f.e. a vlan header,
Stephen Hemminger9bcfcaf2006-08-29 17:48:57 -0700589 * we need to make room for the encapsulating header
590 */
Changli Gaoc893b802010-07-31 13:25:08 +0000591 ll_rs = LL_RESERVED_SPACE_EXTRA(rt->dst.dev, nf_bridge_pad(skb));
Stephen Hemminger9bcfcaf2006-08-29 17:48:57 -0700592
Linus Torvalds1da177e2005-04-16 15:20:36 -0700593 /*
594 * Fragment the datagram.
595 */
596
597 offset = (ntohs(iph->frag_off) & IP_OFFSET) << 3;
598 not_last_frag = iph->frag_off & htons(IP_MF);
599
600 /*
601 * Keep copying data until we run out.
602 */
603
Stephen Hemminger132adf52007-03-08 20:44:43 -0800604 while (left > 0) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700605 len = left;
606 /* IF: it doesn't fit, use 'mtu' - the data space left */
607 if (len > mtu)
608 len = mtu;
Lucas De Marchi25985ed2011-03-30 22:57:33 -0300609 /* IF: we are not sending up to and including the packet end
Linus Torvalds1da177e2005-04-16 15:20:36 -0700610 then align the next start on an eight byte boundary */
611 if (len < left) {
612 len &= ~7;
613 }
614 /*
615 * Allocate buffer.
616 */
617
618 if ((skb2 = alloc_skb(len+hlen+ll_rs, GFP_ATOMIC)) == NULL) {
Patrick McHardy64ce2072005-08-09 20:50:53 -0700619 NETDEBUG(KERN_INFO "IP: frag: no memory for new fragment!\n");
Linus Torvalds1da177e2005-04-16 15:20:36 -0700620 err = -ENOMEM;
621 goto fail;
622 }
623
624 /*
625 * Set up data on packet
626 */
627
628 ip_copy_metadata(skb2, skb);
629 skb_reserve(skb2, ll_rs);
630 skb_put(skb2, len + hlen);
Arnaldo Carvalho de Meloc1d2bbe2007-04-10 20:45:18 -0700631 skb_reset_network_header(skb2);
Arnaldo Carvalho de Melob0e380b2007-04-10 21:21:55 -0700632 skb2->transport_header = skb2->network_header + hlen;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700633
634 /*
635 * Charge the memory for the fragment to any owner
636 * it might possess
637 */
638
639 if (skb->sk)
640 skb_set_owner_w(skb2, skb->sk);
641
642 /*
643 * Copy the packet header into the new buffer.
644 */
645
Arnaldo Carvalho de Melod626f622007-03-27 18:55:52 -0300646 skb_copy_from_linear_data(skb, skb_network_header(skb2), hlen);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700647
648 /*
649 * Copy a block of the IP datagram.
650 */
Arnaldo Carvalho de Melobff9b612007-03-16 17:19:57 -0300651 if (skb_copy_bits(skb, ptr, skb_transport_header(skb2), len))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700652 BUG();
653 left -= len;
654
655 /*
656 * Fill in the new header fields.
657 */
Arnaldo Carvalho de Meloeddc9ec2007-04-20 22:47:35 -0700658 iph = ip_hdr(skb2);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700659 iph->frag_off = htons((offset >> 3));
660
661 /* ANK: dirty, but effective trick. Upgrade options only if
662 * the segment to be fragmented was THE FIRST (otherwise,
663 * options are already fixed) and make it ONCE
664 * on the initial skb, so that all the following fragments
665 * will inherit fixed options.
666 */
667 if (offset == 0)
668 ip_options_fragment(skb);
669
670 /*
671 * Added AC : If we are fragmenting a fragment that's not the
672 * last fragment then keep MF on each bit
673 */
674 if (left > 0 || not_last_frag)
675 iph->frag_off |= htons(IP_MF);
676 ptr += len;
677 offset += len;
678
679 /*
680 * Put this fragment into the sending queue.
681 */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700682 iph->tot_len = htons(len + hlen);
683
684 ip_send_check(iph);
685
686 err = output(skb2);
687 if (err)
688 goto fail;
Wei Dongdafee492006-08-02 13:41:21 -0700689
Pavel Emelyanov5e38e272008-07-16 20:19:49 -0700690 IP_INC_STATS(dev_net(dev), IPSTATS_MIB_FRAGCREATES);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700691 }
692 kfree_skb(skb);
Pavel Emelyanov5e38e272008-07-16 20:19:49 -0700693 IP_INC_STATS(dev_net(dev), IPSTATS_MIB_FRAGOKS);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700694 return err;
695
696fail:
YOSHIFUJI Hideakie905a9e2007-02-09 23:24:47 +0900697 kfree_skb(skb);
Pavel Emelyanov5e38e272008-07-16 20:19:49 -0700698 IP_INC_STATS(dev_net(dev), IPSTATS_MIB_FRAGFAILS);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700699 return err;
700}
Patrick McHardy2e2f7ae2006-04-04 13:42:35 -0700701EXPORT_SYMBOL(ip_fragment);
702
Linus Torvalds1da177e2005-04-16 15:20:36 -0700703int
704ip_generic_getfrag(void *from, char *to, int offset, int len, int odd, struct sk_buff *skb)
705{
706 struct iovec *iov = from;
707
Patrick McHardy84fa7932006-08-29 16:44:56 -0700708 if (skb->ip_summed == CHECKSUM_PARTIAL) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700709 if (memcpy_fromiovecend(to, iov, offset, len) < 0)
710 return -EFAULT;
711 } else {
Al Viro44bb9362006-11-14 21:36:14 -0800712 __wsum csum = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700713 if (csum_partial_copy_fromiovecend(to, iov, offset, len, &csum) < 0)
714 return -EFAULT;
715 skb->csum = csum_block_add(skb->csum, csum, odd);
716 }
717 return 0;
718}
Eric Dumazet4bc2f182010-07-09 21:22:10 +0000719EXPORT_SYMBOL(ip_generic_getfrag);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700720
Al Viro44bb9362006-11-14 21:36:14 -0800721static inline __wsum
Linus Torvalds1da177e2005-04-16 15:20:36 -0700722csum_page(struct page *page, int offset, int copy)
723{
724 char *kaddr;
Al Viro44bb9362006-11-14 21:36:14 -0800725 __wsum csum;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700726 kaddr = kmap(page);
727 csum = csum_partial(kaddr + offset, copy, 0);
728 kunmap(page);
729 return csum;
730}
731
Adrian Bunk4b30b1c2005-11-29 16:27:20 -0800732static inline int ip_ufo_append_data(struct sock *sk,
Herbert Xu1470ddf2011-03-01 02:36:47 +0000733 struct sk_buff_head *queue,
Ananda Rajue89e9cf2005-10-18 15:46:41 -0700734 int getfrag(void *from, char *to, int offset, int len,
735 int odd, struct sk_buff *skb),
736 void *from, int length, int hh_len, int fragheaderlen,
Jianjun Kongd93191002008-11-03 00:23:42 -0800737 int transhdrlen, int mtu, unsigned int flags)
Ananda Rajue89e9cf2005-10-18 15:46:41 -0700738{
739 struct sk_buff *skb;
740 int err;
741
742 /* There is support for UDP fragmentation offload by network
743 * device, so create one single skb packet containing complete
744 * udp datagram
745 */
Herbert Xu1470ddf2011-03-01 02:36:47 +0000746 if ((skb = skb_peek_tail(queue)) == NULL) {
Ananda Rajue89e9cf2005-10-18 15:46:41 -0700747 skb = sock_alloc_send_skb(sk,
748 hh_len + fragheaderlen + transhdrlen + 20,
749 (flags & MSG_DONTWAIT), &err);
750
751 if (skb == NULL)
752 return err;
753
754 /* reserve space for Hardware header */
755 skb_reserve(skb, hh_len);
756
757 /* create space for UDP/IP header */
Jianjun Kongd93191002008-11-03 00:23:42 -0800758 skb_put(skb, fragheaderlen + transhdrlen);
Ananda Rajue89e9cf2005-10-18 15:46:41 -0700759
760 /* initialize network header pointer */
Arnaldo Carvalho de Meloc1d2bbe2007-04-10 20:45:18 -0700761 skb_reset_network_header(skb);
Ananda Rajue89e9cf2005-10-18 15:46:41 -0700762
763 /* initialize protocol header pointer */
Arnaldo Carvalho de Melob0e380b2007-04-10 21:21:55 -0700764 skb->transport_header = skb->network_header + fragheaderlen;
Ananda Rajue89e9cf2005-10-18 15:46:41 -0700765
Patrick McHardy84fa7932006-08-29 16:44:56 -0700766 skb->ip_summed = CHECKSUM_PARTIAL;
Ananda Rajue89e9cf2005-10-18 15:46:41 -0700767 skb->csum = 0;
Ananda Rajue89e9cf2005-10-18 15:46:41 -0700768
Kostya Bbe9164e2008-04-29 22:36:30 -0700769 /* specify the length of each IP datagram fragment */
Herbert Xu79671682006-06-22 02:40:14 -0700770 skb_shinfo(skb)->gso_size = mtu - fragheaderlen;
Herbert Xuf83ef8c2006-06-30 13:37:03 -0700771 skb_shinfo(skb)->gso_type = SKB_GSO_UDP;
Herbert Xu1470ddf2011-03-01 02:36:47 +0000772 __skb_queue_tail(queue, skb);
Ananda Rajue89e9cf2005-10-18 15:46:41 -0700773 }
Kostya Bbe9164e2008-04-29 22:36:30 -0700774
775 return skb_append_datato_frags(sk, skb, getfrag, from,
776 (length - transhdrlen));
Ananda Rajue89e9cf2005-10-18 15:46:41 -0700777}
778
David S. Millerf5fca602011-05-08 17:24:10 -0700779static int __ip_append_data(struct sock *sk,
780 struct flowi4 *fl4,
781 struct sk_buff_head *queue,
Herbert Xu1470ddf2011-03-01 02:36:47 +0000782 struct inet_cork *cork,
783 int getfrag(void *from, char *to, int offset,
784 int len, int odd, struct sk_buff *skb),
785 void *from, int length, int transhdrlen,
786 unsigned int flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700787{
788 struct inet_sock *inet = inet_sk(sk);
789 struct sk_buff *skb;
790
Herbert Xu07df5292011-03-01 23:00:58 -0800791 struct ip_options *opt = cork->opt;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700792 int hh_len;
793 int exthdrlen;
794 int mtu;
795 int copy;
796 int err;
797 int offset = 0;
798 unsigned int maxfraglen, fragheaderlen;
799 int csummode = CHECKSUM_NONE;
Herbert Xu1470ddf2011-03-01 02:36:47 +0000800 struct rtable *rt = (struct rtable *)cork->dst;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700801
Herbert Xu1470ddf2011-03-01 02:36:47 +0000802 exthdrlen = transhdrlen ? rt->dst.header_len : 0;
803 length += exthdrlen;
804 transhdrlen += exthdrlen;
Herbert Xu07df5292011-03-01 23:00:58 -0800805 mtu = cork->fragsize;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700806
Changli Gaod8d1f302010-06-10 23:31:35 -0700807 hh_len = LL_RESERVED_SPACE(rt->dst.dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700808
809 fragheaderlen = sizeof(struct iphdr) + (opt ? opt->optlen : 0);
810 maxfraglen = ((mtu - fragheaderlen) & ~7) + fragheaderlen;
811
Herbert Xu1470ddf2011-03-01 02:36:47 +0000812 if (cork->length + length > 0xFFFF - fragheaderlen) {
David S. Millerf5fca602011-05-08 17:24:10 -0700813 ip_local_error(sk, EMSGSIZE, fl4->daddr, inet->inet_dport,
Eric Dumazetc720c7e2009-10-15 06:30:45 +0000814 mtu-exthdrlen);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700815 return -EMSGSIZE;
816 }
817
818 /*
819 * transhdrlen > 0 means that this is the first fragment and we wish
820 * it won't be fragmented in the future.
821 */
822 if (transhdrlen &&
823 length + fragheaderlen <= mtu &&
Changli Gaod8d1f302010-06-10 23:31:35 -0700824 rt->dst.dev->features & NETIF_F_V4_CSUM &&
Linus Torvalds1da177e2005-04-16 15:20:36 -0700825 !exthdrlen)
Patrick McHardy84fa7932006-08-29 16:44:56 -0700826 csummode = CHECKSUM_PARTIAL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700827
Herbert Xu1470ddf2011-03-01 02:36:47 +0000828 skb = skb_peek_tail(queue);
Herbert Xu26cde9f2010-06-15 01:52:25 +0000829
Herbert Xu1470ddf2011-03-01 02:36:47 +0000830 cork->length += length;
Herbert Xu26cde9f2010-06-15 01:52:25 +0000831 if (((length > mtu) || (skb && skb_is_gso(skb))) &&
Kostya Bbe9164e2008-04-29 22:36:30 -0700832 (sk->sk_protocol == IPPROTO_UDP) &&
Changli Gaod8d1f302010-06-10 23:31:35 -0700833 (rt->dst.dev->features & NETIF_F_UFO)) {
Herbert Xu1470ddf2011-03-01 02:36:47 +0000834 err = ip_ufo_append_data(sk, queue, getfrag, from, length,
835 hh_len, fragheaderlen, transhdrlen,
836 mtu, flags);
Patrick McHardybaa829d2006-03-12 20:35:12 -0800837 if (err)
Ananda Rajue89e9cf2005-10-18 15:46:41 -0700838 goto error;
Ananda Rajue89e9cf2005-10-18 15:46:41 -0700839 return 0;
840 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700841
842 /* So, what's going on in the loop below?
843 *
844 * We use calculated fragment length to generate chained skb,
845 * each of segments is IP fragment ready for sending to network after
846 * adding appropriate IP header.
847 */
848
Herbert Xu26cde9f2010-06-15 01:52:25 +0000849 if (!skb)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700850 goto alloc_new_skb;
851
852 while (length > 0) {
853 /* Check if the remaining data fits into current packet. */
854 copy = mtu - skb->len;
855 if (copy < length)
856 copy = maxfraglen - skb->len;
857 if (copy <= 0) {
858 char *data;
859 unsigned int datalen;
860 unsigned int fraglen;
861 unsigned int fraggap;
862 unsigned int alloclen;
863 struct sk_buff *skb_prev;
864alloc_new_skb:
865 skb_prev = skb;
866 if (skb_prev)
867 fraggap = skb_prev->len - maxfraglen;
868 else
869 fraggap = 0;
870
871 /*
872 * If remaining data exceeds the mtu,
873 * we know we need more fragment(s).
874 */
875 datalen = length + fraggap;
876 if (datalen > mtu - fragheaderlen)
877 datalen = maxfraglen - fragheaderlen;
878 fraglen = datalen + fragheaderlen;
879
YOSHIFUJI Hideakie905a9e2007-02-09 23:24:47 +0900880 if ((flags & MSG_MORE) &&
Changli Gaod8d1f302010-06-10 23:31:35 -0700881 !(rt->dst.dev->features&NETIF_F_SG))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700882 alloclen = mtu;
883 else
Eric Dumazet59104f02010-09-20 20:16:27 +0000884 alloclen = fraglen;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700885
886 /* The last fragment gets additional space at tail.
887 * Note, with MSG_MORE we overallocate on fragments,
888 * because we have no idea what fragment will be
889 * the last.
890 */
Eric Dumazet59104f02010-09-20 20:16:27 +0000891 if (datalen == length + fraggap) {
Changli Gaod8d1f302010-06-10 23:31:35 -0700892 alloclen += rt->dst.trailer_len;
Eric Dumazet59104f02010-09-20 20:16:27 +0000893 /* make sure mtu is not reached */
894 if (datalen > mtu - fragheaderlen - rt->dst.trailer_len)
895 datalen -= ALIGN(rt->dst.trailer_len, 8);
896 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700897 if (transhdrlen) {
YOSHIFUJI Hideakie905a9e2007-02-09 23:24:47 +0900898 skb = sock_alloc_send_skb(sk,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700899 alloclen + hh_len + 15,
900 (flags & MSG_DONTWAIT), &err);
901 } else {
902 skb = NULL;
903 if (atomic_read(&sk->sk_wmem_alloc) <=
904 2 * sk->sk_sndbuf)
YOSHIFUJI Hideakie905a9e2007-02-09 23:24:47 +0900905 skb = sock_wmalloc(sk,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700906 alloclen + hh_len + 15, 1,
907 sk->sk_allocation);
908 if (unlikely(skb == NULL))
909 err = -ENOBUFS;
Patrick Ohly51f31ca2009-02-12 05:03:39 +0000910 else
911 /* only the initial fragment is
912 time stamped */
Herbert Xu1470ddf2011-03-01 02:36:47 +0000913 cork->tx_flags = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700914 }
915 if (skb == NULL)
916 goto error;
917
918 /*
919 * Fill in the control structures
920 */
921 skb->ip_summed = csummode;
922 skb->csum = 0;
923 skb_reserve(skb, hh_len);
Herbert Xu1470ddf2011-03-01 02:36:47 +0000924 skb_shinfo(skb)->tx_flags = cork->tx_flags;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700925
926 /*
927 * Find where to start putting bytes.
928 */
929 data = skb_put(skb, fraglen);
Arnaldo Carvalho de Meloc14d2452007-03-11 22:39:41 -0300930 skb_set_network_header(skb, exthdrlen);
Arnaldo Carvalho de Melob0e380b2007-04-10 21:21:55 -0700931 skb->transport_header = (skb->network_header +
932 fragheaderlen);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700933 data += fragheaderlen;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700934
935 if (fraggap) {
936 skb->csum = skb_copy_and_csum_bits(
937 skb_prev, maxfraglen,
938 data + transhdrlen, fraggap, 0);
939 skb_prev->csum = csum_sub(skb_prev->csum,
940 skb->csum);
941 data += fraggap;
Herbert Xue9fa4f72006-08-13 20:12:58 -0700942 pskb_trim_unique(skb_prev, maxfraglen);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700943 }
944
945 copy = datalen - transhdrlen - fraggap;
946 if (copy > 0 && getfrag(from, data + transhdrlen, offset, copy, fraggap, skb) < 0) {
947 err = -EFAULT;
948 kfree_skb(skb);
949 goto error;
950 }
951
952 offset += copy;
953 length -= datalen - fraggap;
954 transhdrlen = 0;
955 exthdrlen = 0;
956 csummode = CHECKSUM_NONE;
957
958 /*
959 * Put the packet on the pending queue.
960 */
Herbert Xu1470ddf2011-03-01 02:36:47 +0000961 __skb_queue_tail(queue, skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700962 continue;
963 }
964
965 if (copy > length)
966 copy = length;
967
Changli Gaod8d1f302010-06-10 23:31:35 -0700968 if (!(rt->dst.dev->features&NETIF_F_SG)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700969 unsigned int off;
970
971 off = skb->len;
YOSHIFUJI Hideakie905a9e2007-02-09 23:24:47 +0900972 if (getfrag(from, skb_put(skb, copy),
Linus Torvalds1da177e2005-04-16 15:20:36 -0700973 offset, copy, off, skb) < 0) {
974 __skb_trim(skb, off);
975 err = -EFAULT;
976 goto error;
977 }
978 } else {
979 int i = skb_shinfo(skb)->nr_frags;
980 skb_frag_t *frag = &skb_shinfo(skb)->frags[i-1];
Herbert Xu1470ddf2011-03-01 02:36:47 +0000981 struct page *page = cork->page;
982 int off = cork->off;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700983 unsigned int left;
984
985 if (page && (left = PAGE_SIZE - off) > 0) {
986 if (copy >= left)
987 copy = left;
988 if (page != frag->page) {
989 if (i == MAX_SKB_FRAGS) {
990 err = -EMSGSIZE;
991 goto error;
992 }
993 get_page(page);
Herbert Xu1470ddf2011-03-01 02:36:47 +0000994 skb_fill_page_desc(skb, i, page, off, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700995 frag = &skb_shinfo(skb)->frags[i];
996 }
997 } else if (i < MAX_SKB_FRAGS) {
998 if (copy > PAGE_SIZE)
999 copy = PAGE_SIZE;
1000 page = alloc_pages(sk->sk_allocation, 0);
1001 if (page == NULL) {
1002 err = -ENOMEM;
1003 goto error;
1004 }
Herbert Xu1470ddf2011-03-01 02:36:47 +00001005 cork->page = page;
1006 cork->off = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001007
1008 skb_fill_page_desc(skb, i, page, 0, 0);
1009 frag = &skb_shinfo(skb)->frags[i];
Linus Torvalds1da177e2005-04-16 15:20:36 -07001010 } else {
1011 err = -EMSGSIZE;
1012 goto error;
1013 }
1014 if (getfrag(from, page_address(frag->page)+frag->page_offset+frag->size, offset, copy, skb->len, skb) < 0) {
1015 err = -EFAULT;
1016 goto error;
1017 }
Herbert Xu1470ddf2011-03-01 02:36:47 +00001018 cork->off += copy;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001019 frag->size += copy;
1020 skb->len += copy;
1021 skb->data_len += copy;
Herbert Xuf945fa72008-01-22 22:39:26 -08001022 skb->truesize += copy;
1023 atomic_add(copy, &sk->sk_wmem_alloc);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001024 }
1025 offset += copy;
1026 length -= copy;
1027 }
1028
1029 return 0;
1030
1031error:
Herbert Xu1470ddf2011-03-01 02:36:47 +00001032 cork->length -= length;
Pavel Emelyanov5e38e272008-07-16 20:19:49 -07001033 IP_INC_STATS(sock_net(sk), IPSTATS_MIB_OUTDISCARDS);
YOSHIFUJI Hideakie905a9e2007-02-09 23:24:47 +09001034 return err;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001035}
1036
Herbert Xu1470ddf2011-03-01 02:36:47 +00001037static int ip_setup_cork(struct sock *sk, struct inet_cork *cork,
1038 struct ipcm_cookie *ipc, struct rtable **rtp)
1039{
1040 struct inet_sock *inet = inet_sk(sk);
Eric Dumazetf6d8bd02011-04-21 09:45:37 +00001041 struct ip_options_rcu *opt;
Herbert Xu1470ddf2011-03-01 02:36:47 +00001042 struct rtable *rt;
1043
1044 /*
1045 * setup for corking.
1046 */
1047 opt = ipc->opt;
1048 if (opt) {
1049 if (cork->opt == NULL) {
1050 cork->opt = kmalloc(sizeof(struct ip_options) + 40,
1051 sk->sk_allocation);
1052 if (unlikely(cork->opt == NULL))
1053 return -ENOBUFS;
1054 }
Eric Dumazetf6d8bd02011-04-21 09:45:37 +00001055 memcpy(cork->opt, &opt->opt, sizeof(struct ip_options) + opt->opt.optlen);
Herbert Xu1470ddf2011-03-01 02:36:47 +00001056 cork->flags |= IPCORK_OPT;
1057 cork->addr = ipc->addr;
1058 }
1059 rt = *rtp;
1060 if (unlikely(!rt))
1061 return -EFAULT;
1062 /*
1063 * We steal reference to this route, caller should not release it
1064 */
1065 *rtp = NULL;
1066 cork->fragsize = inet->pmtudisc == IP_PMTUDISC_PROBE ?
1067 rt->dst.dev->mtu : dst_mtu(rt->dst.path);
1068 cork->dst = &rt->dst;
1069 cork->length = 0;
1070 cork->tx_flags = ipc->tx_flags;
1071 cork->page = NULL;
1072 cork->off = 0;
1073
1074 return 0;
1075}
1076
1077/*
1078 * ip_append_data() and ip_append_page() can make one large IP datagram
1079 * from many pieces of data. Each pieces will be holded on the socket
1080 * until ip_push_pending_frames() is called. Each piece can be a page
1081 * or non-page data.
1082 *
1083 * Not only UDP, other transport protocols - e.g. raw sockets - can use
1084 * this interface potentially.
1085 *
1086 * LATER: length must be adjusted by pad at tail, when it is required.
1087 */
David S. Millerf5fca602011-05-08 17:24:10 -07001088int ip_append_data(struct sock *sk, struct flowi4 *fl4,
Herbert Xu1470ddf2011-03-01 02:36:47 +00001089 int getfrag(void *from, char *to, int offset, int len,
1090 int odd, struct sk_buff *skb),
1091 void *from, int length, int transhdrlen,
1092 struct ipcm_cookie *ipc, struct rtable **rtp,
1093 unsigned int flags)
1094{
1095 struct inet_sock *inet = inet_sk(sk);
1096 int err;
1097
1098 if (flags&MSG_PROBE)
1099 return 0;
1100
1101 if (skb_queue_empty(&sk->sk_write_queue)) {
David S. Millerbdc712b2011-05-06 15:02:07 -07001102 err = ip_setup_cork(sk, &inet->cork.base, ipc, rtp);
Herbert Xu1470ddf2011-03-01 02:36:47 +00001103 if (err)
1104 return err;
1105 } else {
1106 transhdrlen = 0;
1107 }
1108
David S. Millerf5fca602011-05-08 17:24:10 -07001109 return __ip_append_data(sk, fl4, &sk->sk_write_queue, &inet->cork.base, getfrag,
Herbert Xu1470ddf2011-03-01 02:36:47 +00001110 from, length, transhdrlen, flags);
1111}
1112
David S. Millerf5fca602011-05-08 17:24:10 -07001113ssize_t ip_append_page(struct sock *sk, struct flowi4 *fl4, struct page *page,
Linus Torvalds1da177e2005-04-16 15:20:36 -07001114 int offset, size_t size, int flags)
1115{
1116 struct inet_sock *inet = inet_sk(sk);
1117 struct sk_buff *skb;
1118 struct rtable *rt;
1119 struct ip_options *opt = NULL;
David S. Millerbdc712b2011-05-06 15:02:07 -07001120 struct inet_cork *cork;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001121 int hh_len;
1122 int mtu;
1123 int len;
1124 int err;
1125 unsigned int maxfraglen, fragheaderlen, fraggap;
1126
1127 if (inet->hdrincl)
1128 return -EPERM;
1129
1130 if (flags&MSG_PROBE)
1131 return 0;
1132
1133 if (skb_queue_empty(&sk->sk_write_queue))
1134 return -EINVAL;
1135
David S. Millerbdc712b2011-05-06 15:02:07 -07001136 cork = &inet->cork.base;
1137 rt = (struct rtable *)cork->dst;
1138 if (cork->flags & IPCORK_OPT)
1139 opt = cork->opt;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001140
Changli Gaod8d1f302010-06-10 23:31:35 -07001141 if (!(rt->dst.dev->features&NETIF_F_SG))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001142 return -EOPNOTSUPP;
1143
Changli Gaod8d1f302010-06-10 23:31:35 -07001144 hh_len = LL_RESERVED_SPACE(rt->dst.dev);
David S. Millerbdc712b2011-05-06 15:02:07 -07001145 mtu = cork->fragsize;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001146
1147 fragheaderlen = sizeof(struct iphdr) + (opt ? opt->optlen : 0);
1148 maxfraglen = ((mtu - fragheaderlen) & ~7) + fragheaderlen;
1149
David S. Millerbdc712b2011-05-06 15:02:07 -07001150 if (cork->length + size > 0xFFFF - fragheaderlen) {
David S. Millerf5fca602011-05-08 17:24:10 -07001151 ip_local_error(sk, EMSGSIZE, fl4->daddr, inet->inet_dport, mtu);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001152 return -EMSGSIZE;
1153 }
1154
1155 if ((skb = skb_peek_tail(&sk->sk_write_queue)) == NULL)
1156 return -EINVAL;
1157
David S. Millerbdc712b2011-05-06 15:02:07 -07001158 cork->length += size;
Herbert Xu26cde9f2010-06-15 01:52:25 +00001159 if ((size + skb->len > mtu) &&
1160 (sk->sk_protocol == IPPROTO_UDP) &&
Changli Gaod8d1f302010-06-10 23:31:35 -07001161 (rt->dst.dev->features & NETIF_F_UFO)) {
Herbert Xu79671682006-06-22 02:40:14 -07001162 skb_shinfo(skb)->gso_size = mtu - fragheaderlen;
Herbert Xuf83ef8c2006-06-30 13:37:03 -07001163 skb_shinfo(skb)->gso_type = SKB_GSO_UDP;
Herbert Xu79671682006-06-22 02:40:14 -07001164 }
Ananda Rajue89e9cf2005-10-18 15:46:41 -07001165
Linus Torvalds1da177e2005-04-16 15:20:36 -07001166
1167 while (size > 0) {
1168 int i;
1169
Herbert Xu89114af2006-07-08 13:34:32 -07001170 if (skb_is_gso(skb))
Ananda Rajue89e9cf2005-10-18 15:46:41 -07001171 len = size;
1172 else {
1173
1174 /* Check if the remaining data fits into current packet. */
1175 len = mtu - skb->len;
1176 if (len < size)
1177 len = maxfraglen - skb->len;
1178 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001179 if (len <= 0) {
1180 struct sk_buff *skb_prev;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001181 int alloclen;
1182
1183 skb_prev = skb;
Jayachandran C0d0d2bb2005-10-13 11:43:02 -07001184 fraggap = skb_prev->len - maxfraglen;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001185
1186 alloclen = fragheaderlen + hh_len + fraggap + 15;
1187 skb = sock_wmalloc(sk, alloclen, 1, sk->sk_allocation);
1188 if (unlikely(!skb)) {
1189 err = -ENOBUFS;
1190 goto error;
1191 }
1192
1193 /*
1194 * Fill in the control structures
1195 */
1196 skb->ip_summed = CHECKSUM_NONE;
1197 skb->csum = 0;
1198 skb_reserve(skb, hh_len);
1199
1200 /*
1201 * Find where to start putting bytes.
1202 */
Arnaldo Carvalho de Melo967b05f2007-03-13 13:51:52 -03001203 skb_put(skb, fragheaderlen + fraggap);
Arnaldo Carvalho de Melo2ca9e6f2007-03-10 19:15:25 -03001204 skb_reset_network_header(skb);
Arnaldo Carvalho de Melob0e380b2007-04-10 21:21:55 -07001205 skb->transport_header = (skb->network_header +
1206 fragheaderlen);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001207 if (fraggap) {
Arnaldo Carvalho de Melo967b05f2007-03-13 13:51:52 -03001208 skb->csum = skb_copy_and_csum_bits(skb_prev,
1209 maxfraglen,
Arnaldo Carvalho de Melo9c702202007-04-25 18:04:18 -07001210 skb_transport_header(skb),
Arnaldo Carvalho de Melo967b05f2007-03-13 13:51:52 -03001211 fraggap, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001212 skb_prev->csum = csum_sub(skb_prev->csum,
1213 skb->csum);
Herbert Xue9fa4f72006-08-13 20:12:58 -07001214 pskb_trim_unique(skb_prev, maxfraglen);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001215 }
1216
1217 /*
1218 * Put the packet on the pending queue.
1219 */
1220 __skb_queue_tail(&sk->sk_write_queue, skb);
1221 continue;
1222 }
1223
1224 i = skb_shinfo(skb)->nr_frags;
1225 if (len > size)
1226 len = size;
1227 if (skb_can_coalesce(skb, i, page, offset)) {
1228 skb_shinfo(skb)->frags[i-1].size += len;
1229 } else if (i < MAX_SKB_FRAGS) {
1230 get_page(page);
1231 skb_fill_page_desc(skb, i, page, offset, len);
1232 } else {
1233 err = -EMSGSIZE;
1234 goto error;
1235 }
1236
1237 if (skb->ip_summed == CHECKSUM_NONE) {
Al Viro44bb9362006-11-14 21:36:14 -08001238 __wsum csum;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001239 csum = csum_page(page, offset, len);
1240 skb->csum = csum_block_add(skb->csum, csum, skb->len);
1241 }
1242
1243 skb->len += len;
1244 skb->data_len += len;
David S. Miller1e34a112008-01-22 23:44:31 -08001245 skb->truesize += len;
1246 atomic_add(len, &sk->sk_wmem_alloc);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001247 offset += len;
1248 size -= len;
1249 }
1250 return 0;
1251
1252error:
David S. Millerbdc712b2011-05-06 15:02:07 -07001253 cork->length -= size;
Pavel Emelyanov5e38e272008-07-16 20:19:49 -07001254 IP_INC_STATS(sock_net(sk), IPSTATS_MIB_OUTDISCARDS);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001255 return err;
1256}
1257
Herbert Xu1470ddf2011-03-01 02:36:47 +00001258static void ip_cork_release(struct inet_cork *cork)
Pavel Emelyanov429f08e2007-11-05 21:03:24 -08001259{
Herbert Xu1470ddf2011-03-01 02:36:47 +00001260 cork->flags &= ~IPCORK_OPT;
1261 kfree(cork->opt);
1262 cork->opt = NULL;
1263 dst_release(cork->dst);
1264 cork->dst = NULL;
Pavel Emelyanov429f08e2007-11-05 21:03:24 -08001265}
1266
Linus Torvalds1da177e2005-04-16 15:20:36 -07001267/*
1268 * Combined all pending IP fragments on the socket as one IP datagram
1269 * and push them out.
1270 */
Herbert Xu1c32c5a2011-03-01 02:36:47 +00001271struct sk_buff *__ip_make_skb(struct sock *sk,
David S. Miller77968b72011-05-08 17:12:19 -07001272 struct flowi4 *fl4,
Herbert Xu1c32c5a2011-03-01 02:36:47 +00001273 struct sk_buff_head *queue,
1274 struct inet_cork *cork)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001275{
1276 struct sk_buff *skb, *tmp_skb;
1277 struct sk_buff **tail_skb;
1278 struct inet_sock *inet = inet_sk(sk);
Pavel Emelyanov0388b002008-07-14 23:00:43 -07001279 struct net *net = sock_net(sk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001280 struct ip_options *opt = NULL;
Herbert Xu1470ddf2011-03-01 02:36:47 +00001281 struct rtable *rt = (struct rtable *)cork->dst;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001282 struct iphdr *iph;
Alexey Dobriyan76ab6082006-01-06 13:24:29 -08001283 __be16 df = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001284 __u8 ttl;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001285
Herbert Xu1470ddf2011-03-01 02:36:47 +00001286 if ((skb = __skb_dequeue(queue)) == NULL)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001287 goto out;
1288 tail_skb = &(skb_shinfo(skb)->frag_list);
1289
1290 /* move skb->data to ip header from ext header */
Arnaldo Carvalho de Melod56f90a2007-04-10 20:50:43 -07001291 if (skb->data < skb_network_header(skb))
Arnaldo Carvalho de Melobbe735e2007-03-10 22:16:10 -03001292 __skb_pull(skb, skb_network_offset(skb));
Herbert Xu1470ddf2011-03-01 02:36:47 +00001293 while ((tmp_skb = __skb_dequeue(queue)) != NULL) {
Arnaldo Carvalho de Melocfe1fc72007-03-16 17:26:39 -03001294 __skb_pull(tmp_skb, skb_network_header_len(skb));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001295 *tail_skb = tmp_skb;
1296 tail_skb = &(tmp_skb->next);
1297 skb->len += tmp_skb->len;
1298 skb->data_len += tmp_skb->len;
1299 skb->truesize += tmp_skb->truesize;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001300 tmp_skb->destructor = NULL;
1301 tmp_skb->sk = NULL;
1302 }
1303
1304 /* Unless user demanded real pmtu discovery (IP_PMTUDISC_DO), we allow
1305 * to fragment the frame generated here. No matter, what transforms
1306 * how transforms change size of the packet, it will come out.
1307 */
John Heffner628a5c52007-04-20 15:53:27 -07001308 if (inet->pmtudisc < IP_PMTUDISC_DO)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001309 skb->local_df = 1;
1310
1311 /* DF bit is set when we want to see DF on outgoing frames.
1312 * If local_df is set too, we still allow to fragment this frame
1313 * locally. */
John Heffner628a5c52007-04-20 15:53:27 -07001314 if (inet->pmtudisc >= IP_PMTUDISC_DO ||
Changli Gaod8d1f302010-06-10 23:31:35 -07001315 (skb->len <= dst_mtu(&rt->dst) &&
1316 ip_dont_fragment(sk, &rt->dst)))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001317 df = htons(IP_DF);
1318
Herbert Xu1470ddf2011-03-01 02:36:47 +00001319 if (cork->flags & IPCORK_OPT)
1320 opt = cork->opt;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001321
1322 if (rt->rt_type == RTN_MULTICAST)
1323 ttl = inet->mc_ttl;
1324 else
Changli Gaod8d1f302010-06-10 23:31:35 -07001325 ttl = ip_select_ttl(inet, &rt->dst);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001326
1327 iph = (struct iphdr *)skb->data;
1328 iph->version = 4;
1329 iph->ihl = 5;
1330 if (opt) {
1331 iph->ihl += opt->optlen>>2;
Herbert Xu1470ddf2011-03-01 02:36:47 +00001332 ip_options_build(skb, opt, cork->addr, rt, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001333 }
1334 iph->tos = inet->tos;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001335 iph->frag_off = df;
Changli Gaod8d1f302010-06-10 23:31:35 -07001336 ip_select_ident(iph, &rt->dst, sk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001337 iph->ttl = ttl;
1338 iph->protocol = sk->sk_protocol;
David S. Miller77968b72011-05-08 17:12:19 -07001339 iph->saddr = fl4->saddr;
1340 iph->daddr = fl4->daddr;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001341
1342 skb->priority = sk->sk_priority;
Laszlo Attila Toth4a19ec52008-01-30 19:08:16 -08001343 skb->mark = sk->sk_mark;
Eric Dumazeta21bba92008-11-24 16:07:50 -08001344 /*
1345 * Steal rt from cork.dst to avoid a pair of atomic_inc/atomic_dec
1346 * on dst refcount
1347 */
Herbert Xu1470ddf2011-03-01 02:36:47 +00001348 cork->dst = NULL;
Changli Gaod8d1f302010-06-10 23:31:35 -07001349 skb_dst_set(skb, &rt->dst);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001350
David L Stevens96793b42007-09-17 09:57:33 -07001351 if (iph->protocol == IPPROTO_ICMP)
Pavel Emelyanov0388b002008-07-14 23:00:43 -07001352 icmp_out_count(net, ((struct icmphdr *)
David L Stevens96793b42007-09-17 09:57:33 -07001353 skb_transport_header(skb))->type);
1354
Herbert Xu1c32c5a2011-03-01 02:36:47 +00001355 ip_cork_release(cork);
1356out:
1357 return skb;
1358}
1359
1360int ip_send_skb(struct sk_buff *skb)
1361{
1362 struct net *net = sock_net(skb->sk);
1363 int err;
1364
Herbert Xuc439cb22008-01-11 19:14:00 -08001365 err = ip_local_out(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001366 if (err) {
1367 if (err > 0)
Eric Dumazet6ce9e7b2009-09-02 18:05:33 -07001368 err = net_xmit_errno(err);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001369 if (err)
Herbert Xu1c32c5a2011-03-01 02:36:47 +00001370 IP_INC_STATS(net, IPSTATS_MIB_OUTDISCARDS);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001371 }
1372
Linus Torvalds1da177e2005-04-16 15:20:36 -07001373 return err;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001374}
1375
David S. Miller77968b72011-05-08 17:12:19 -07001376int ip_push_pending_frames(struct sock *sk, struct flowi4 *fl4)
Herbert Xu1470ddf2011-03-01 02:36:47 +00001377{
Herbert Xu1c32c5a2011-03-01 02:36:47 +00001378 struct sk_buff *skb;
1379
David S. Miller77968b72011-05-08 17:12:19 -07001380 skb = ip_finish_skb(sk, fl4);
Herbert Xu1c32c5a2011-03-01 02:36:47 +00001381 if (!skb)
1382 return 0;
1383
1384 /* Netfilter gets whole the not fragmented skb. */
1385 return ip_send_skb(skb);
Herbert Xu1470ddf2011-03-01 02:36:47 +00001386}
1387
Linus Torvalds1da177e2005-04-16 15:20:36 -07001388/*
1389 * Throw away all pending data on the socket.
1390 */
Herbert Xu1470ddf2011-03-01 02:36:47 +00001391static void __ip_flush_pending_frames(struct sock *sk,
1392 struct sk_buff_head *queue,
1393 struct inet_cork *cork)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001394{
Linus Torvalds1da177e2005-04-16 15:20:36 -07001395 struct sk_buff *skb;
1396
Herbert Xu1470ddf2011-03-01 02:36:47 +00001397 while ((skb = __skb_dequeue_tail(queue)) != NULL)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001398 kfree_skb(skb);
1399
Herbert Xu1470ddf2011-03-01 02:36:47 +00001400 ip_cork_release(cork);
1401}
1402
1403void ip_flush_pending_frames(struct sock *sk)
1404{
David S. Millerbdc712b2011-05-06 15:02:07 -07001405 __ip_flush_pending_frames(sk, &sk->sk_write_queue, &inet_sk(sk)->cork.base);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001406}
1407
Herbert Xu1c32c5a2011-03-01 02:36:47 +00001408struct sk_buff *ip_make_skb(struct sock *sk,
David S. Miller77968b72011-05-08 17:12:19 -07001409 struct flowi4 *fl4,
Herbert Xu1c32c5a2011-03-01 02:36:47 +00001410 int getfrag(void *from, char *to, int offset,
1411 int len, int odd, struct sk_buff *skb),
1412 void *from, int length, int transhdrlen,
1413 struct ipcm_cookie *ipc, struct rtable **rtp,
1414 unsigned int flags)
1415{
David S. Millerb80d7222011-05-06 15:06:01 -07001416 struct inet_cork cork;
Herbert Xu1c32c5a2011-03-01 02:36:47 +00001417 struct sk_buff_head queue;
1418 int err;
1419
1420 if (flags & MSG_PROBE)
1421 return NULL;
1422
1423 __skb_queue_head_init(&queue);
1424
David S. Millerb80d7222011-05-06 15:06:01 -07001425 cork.flags = 0;
1426 cork.addr = 0;
David S. Miller70652722011-05-06 16:01:15 -07001427 cork.opt = NULL;
Herbert Xu1c32c5a2011-03-01 02:36:47 +00001428 err = ip_setup_cork(sk, &cork, ipc, rtp);
1429 if (err)
1430 return ERR_PTR(err);
1431
David S. Millerf5fca602011-05-08 17:24:10 -07001432 err = __ip_append_data(sk, fl4, &queue, &cork, getfrag,
Herbert Xu1c32c5a2011-03-01 02:36:47 +00001433 from, length, transhdrlen, flags);
1434 if (err) {
1435 __ip_flush_pending_frames(sk, &queue, &cork);
1436 return ERR_PTR(err);
1437 }
1438
David S. Miller77968b72011-05-08 17:12:19 -07001439 return __ip_make_skb(sk, fl4, &queue, &cork);
Herbert Xu1c32c5a2011-03-01 02:36:47 +00001440}
Linus Torvalds1da177e2005-04-16 15:20:36 -07001441
1442/*
1443 * Fetch data from kernel space and fill in checksum if needed.
1444 */
YOSHIFUJI Hideakie905a9e2007-02-09 23:24:47 +09001445static int ip_reply_glue_bits(void *dptr, char *to, int offset,
Linus Torvalds1da177e2005-04-16 15:20:36 -07001446 int len, int odd, struct sk_buff *skb)
1447{
Al Viro50842052006-11-14 21:36:34 -08001448 __wsum csum;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001449
1450 csum = csum_partial_copy_nocheck(dptr+offset, to, len, 0);
1451 skb->csum = csum_block_add(skb->csum, csum, odd);
YOSHIFUJI Hideakie905a9e2007-02-09 23:24:47 +09001452 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001453}
1454
YOSHIFUJI Hideakie905a9e2007-02-09 23:24:47 +09001455/*
Linus Torvalds1da177e2005-04-16 15:20:36 -07001456 * Generic function to send a packet as reply to another packet.
1457 * Used to send TCP resets so far. ICMP should use this function too.
1458 *
YOSHIFUJI Hideakie905a9e2007-02-09 23:24:47 +09001459 * Should run single threaded per socket because it uses the sock
Linus Torvalds1da177e2005-04-16 15:20:36 -07001460 * structure to pass arguments.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001461 */
David S. Miller0a5ebb82011-05-09 13:22:43 -07001462void ip_send_reply(struct sock *sk, struct sk_buff *skb, __be32 daddr,
1463 struct ip_reply_arg *arg, unsigned int len)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001464{
1465 struct inet_sock *inet = inet_sk(sk);
Eric Dumazetf6d8bd02011-04-21 09:45:37 +00001466 struct ip_options_data replyopts;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001467 struct ipcm_cookie ipc;
David S. Miller77968b72011-05-08 17:12:19 -07001468 struct flowi4 fl4;
Eric Dumazet511c3f92009-06-02 05:14:27 +00001469 struct rtable *rt = skb_rtable(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001470
Eric Dumazetf6d8bd02011-04-21 09:45:37 +00001471 if (ip_options_echo(&replyopts.opt.opt, skb))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001472 return;
1473
David S. Miller0a5ebb82011-05-09 13:22:43 -07001474 ipc.addr = daddr;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001475 ipc.opt = NULL;
Oliver Hartkopp2244d072010-08-17 08:59:14 +00001476 ipc.tx_flags = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001477
Eric Dumazetf6d8bd02011-04-21 09:45:37 +00001478 if (replyopts.opt.opt.optlen) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001479 ipc.opt = &replyopts.opt;
1480
Eric Dumazetf6d8bd02011-04-21 09:45:37 +00001481 if (replyopts.opt.opt.srr)
1482 daddr = replyopts.opt.opt.faddr;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001483 }
1484
David S. Miller77968b72011-05-08 17:12:19 -07001485 flowi4_init_output(&fl4, arg->bound_dev_if, 0,
1486 RT_TOS(ip_hdr(skb)->tos),
1487 RT_SCOPE_UNIVERSE, sk->sk_protocol,
1488 ip_reply_arg_flowi_flags(arg),
1489 daddr, rt->rt_spec_dst,
1490 tcp_hdr(skb)->source, tcp_hdr(skb)->dest);
1491 security_skb_classify_flow(skb, flowi4_to_flowi(&fl4));
1492 rt = ip_route_output_key(sock_net(sk), &fl4);
1493 if (IS_ERR(rt))
1494 return;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001495
1496 /* And let IP do all the hard work.
1497
1498 This chunk is not reenterable, hence spinlock.
1499 Note that it uses the fact, that this function is called
1500 with locally disabled BH and that sk cannot be already spinlocked.
1501 */
1502 bh_lock_sock(sk);
Arnaldo Carvalho de Meloeddc9ec2007-04-20 22:47:35 -07001503 inet->tos = ip_hdr(skb)->tos;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001504 sk->sk_priority = skb->priority;
Arnaldo Carvalho de Meloeddc9ec2007-04-20 22:47:35 -07001505 sk->sk_protocol = ip_hdr(skb)->protocol;
Patrick McHardyf0e48db2007-06-04 21:32:46 -07001506 sk->sk_bound_dev_if = arg->bound_dev_if;
David S. Millerf5fca602011-05-08 17:24:10 -07001507 ip_append_data(sk, &fl4, ip_reply_glue_bits, arg->iov->iov_base, len, 0,
Eric Dumazet2e77d892008-11-24 15:52:46 -08001508 &ipc, &rt, MSG_DONTWAIT);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001509 if ((skb = skb_peek(&sk->sk_write_queue)) != NULL) {
1510 if (arg->csumoffset >= 0)
Arnaldo Carvalho de Melo9c702202007-04-25 18:04:18 -07001511 *((__sum16 *)skb_transport_header(skb) +
1512 arg->csumoffset) = csum_fold(csum_add(skb->csum,
1513 arg->csum));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001514 skb->ip_summed = CHECKSUM_NONE;
David S. Miller77968b72011-05-08 17:12:19 -07001515 ip_push_pending_frames(sk, &fl4);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001516 }
1517
1518 bh_unlock_sock(sk);
1519
1520 ip_rt_put(rt);
1521}
1522
Linus Torvalds1da177e2005-04-16 15:20:36 -07001523void __init ip_init(void)
1524{
Linus Torvalds1da177e2005-04-16 15:20:36 -07001525 ip_rt_init();
1526 inet_initpeers();
1527
1528#if defined(CONFIG_IP_MULTICAST) && defined(CONFIG_PROC_FS)
1529 igmp_mc_proc_init();
1530#endif
1531}