blob: 0f3185a662c3ce7f43f393385fb594a14b57694c [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * INET An implementation of the TCP/IP protocol suite for the LINUX
3 * operating system. INET is implemented using the BSD Socket
4 * interface as the means of communication with the user level.
5 *
6 * The Internet Protocol (IP) output module.
7 *
Jesper Juhl02c30a82005-05-05 16:16:16 -07008 * Authors: Ross Biro
Linus Torvalds1da177e2005-04-16 15:20:36 -07009 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
10 * Donald Becker, <becker@super.org>
11 * Alan Cox, <Alan.Cox@linux.org>
12 * Richard Underwood
13 * Stefan Becker, <stefanb@yello.ping.de>
14 * Jorge Cwik, <jorge@laser.satlink.net>
15 * Arnt Gulbrandsen, <agulbra@nvg.unit.no>
16 * Hirokazu Takahashi, <taka@valinux.co.jp>
17 *
18 * See ip_input.c for original log
19 *
20 * Fixes:
21 * Alan Cox : Missing nonblock feature in ip_build_xmit.
22 * Mike Kilburn : htons() missing in ip_build_xmit.
YOSHIFUJI Hideakie905a9e2007-02-09 23:24:47 +090023 * Bradford Johnson: Fix faulty handling of some frames when
Linus Torvalds1da177e2005-04-16 15:20:36 -070024 * no route is found.
25 * Alexander Demenshin: Missing sk/skb free in ip_queue_xmit
26 * (in case if packet not accepted by
27 * output firewall rules)
28 * Mike McLagan : Routing by source
29 * Alexey Kuznetsov: use new route cache
30 * Andi Kleen: Fix broken PMTU recovery and remove
31 * some redundant tests.
32 * Vitaly E. Lavrov : Transparent proxy revived after year coma.
33 * Andi Kleen : Replace ip_reply with ip_send_reply.
YOSHIFUJI Hideakie905a9e2007-02-09 23:24:47 +090034 * Andi Kleen : Split fast and slow ip_build_xmit path
35 * for decreased register pressure on x86
36 * and more readibility.
Linus Torvalds1da177e2005-04-16 15:20:36 -070037 * Marc Boucher : When call_out_firewall returns FW_QUEUE,
38 * silently drop skb instead of failing with -EPERM.
39 * Detlev Wengorz : Copy protocol for fragments.
40 * Hirokazu Takahashi: HW checksumming for outgoing UDP
41 * datagrams.
42 * Hirokazu Takahashi: sendfile() on UDP works now.
43 */
44
45#include <asm/uaccess.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070046#include <linux/module.h>
47#include <linux/types.h>
48#include <linux/kernel.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070049#include <linux/mm.h>
50#include <linux/string.h>
51#include <linux/errno.h>
Al Viroa1f8e7f72006-10-19 16:08:53 -040052#include <linux/highmem.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090053#include <linux/slab.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070054
55#include <linux/socket.h>
56#include <linux/sockios.h>
57#include <linux/in.h>
58#include <linux/inet.h>
59#include <linux/netdevice.h>
60#include <linux/etherdevice.h>
61#include <linux/proc_fs.h>
62#include <linux/stat.h>
63#include <linux/init.h>
64
65#include <net/snmp.h>
66#include <net/ip.h>
67#include <net/protocol.h>
68#include <net/route.h>
Patrick McHardycfacb052006-01-08 22:36:54 -080069#include <net/xfrm.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070070#include <linux/skbuff.h>
71#include <net/sock.h>
72#include <net/arp.h>
73#include <net/icmp.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070074#include <net/checksum.h>
75#include <net/inetpeer.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070076#include <linux/igmp.h>
77#include <linux/netfilter_ipv4.h>
78#include <linux/netfilter_bridge.h>
79#include <linux/mroute.h>
80#include <linux/netlink.h>
Arnaldo Carvalho de Melo6cbb0df2005-08-09 19:49:02 -070081#include <linux/tcp.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070082
Brian Haleyab32ea52006-09-22 14:15:41 -070083int sysctl_ip_default_ttl __read_mostly = IPDEFTTL;
David S. Miller323e1262010-12-12 21:55:08 -080084EXPORT_SYMBOL(sysctl_ip_default_ttl);
Linus Torvalds1da177e2005-04-16 15:20:36 -070085
86/* Generate a checksum for an outgoing IP datagram. */
87__inline__ void ip_send_check(struct iphdr *iph)
88{
89 iph->check = 0;
90 iph->check = ip_fast_csum((unsigned char *)iph, iph->ihl);
91}
Eric Dumazet4bc2f182010-07-09 21:22:10 +000092EXPORT_SYMBOL(ip_send_check);
Linus Torvalds1da177e2005-04-16 15:20:36 -070093
Herbert Xuc439cb22008-01-11 19:14:00 -080094int __ip_local_out(struct sk_buff *skb)
95{
96 struct iphdr *iph = ip_hdr(skb);
97
98 iph->tot_len = htons(skb->len);
99 ip_send_check(iph);
Jan Engelhardt9bbc7682010-03-23 04:07:29 +0100100 return nf_hook(NFPROTO_IPV4, NF_INET_LOCAL_OUT, skb, NULL,
101 skb_dst(skb)->dev, dst_output);
Herbert Xuc439cb22008-01-11 19:14:00 -0800102}
103
104int ip_local_out(struct sk_buff *skb)
105{
106 int err;
107
108 err = __ip_local_out(skb);
109 if (likely(err == 1))
110 err = dst_output(skb);
111
112 return err;
113}
114EXPORT_SYMBOL_GPL(ip_local_out);
115
Linus Torvalds1da177e2005-04-16 15:20:36 -0700116static inline int ip_select_ttl(struct inet_sock *inet, struct dst_entry *dst)
117{
118 int ttl = inet->uc_ttl;
119
120 if (ttl < 0)
David S. Miller323e1262010-12-12 21:55:08 -0800121 ttl = ip4_dst_hoplimit(dst);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700122 return ttl;
123}
124
YOSHIFUJI Hideakie905a9e2007-02-09 23:24:47 +0900125/*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700126 * Add an ip header to a skbuff and send it out.
127 *
128 */
129int ip_build_and_send_pkt(struct sk_buff *skb, struct sock *sk,
Eric Dumazetf6d8bd02011-04-21 09:45:37 +0000130 __be32 saddr, __be32 daddr, struct ip_options_rcu *opt)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700131{
132 struct inet_sock *inet = inet_sk(sk);
Eric Dumazet511c3f92009-06-02 05:14:27 +0000133 struct rtable *rt = skb_rtable(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700134 struct iphdr *iph;
135
136 /* Build the IP header. */
Eric Dumazetf6d8bd02011-04-21 09:45:37 +0000137 skb_push(skb, sizeof(struct iphdr) + (opt ? opt->opt.optlen : 0));
Arnaldo Carvalho de Melo8856dfa2007-03-10 19:40:39 -0300138 skb_reset_network_header(skb);
Arnaldo Carvalho de Meloeddc9ec2007-04-20 22:47:35 -0700139 iph = ip_hdr(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700140 iph->version = 4;
141 iph->ihl = 5;
142 iph->tos = inet->tos;
Changli Gaod8d1f302010-06-10 23:31:35 -0700143 if (ip_dont_fragment(sk, &rt->dst))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700144 iph->frag_off = htons(IP_DF);
145 else
146 iph->frag_off = 0;
Changli Gaod8d1f302010-06-10 23:31:35 -0700147 iph->ttl = ip_select_ttl(inet, &rt->dst);
David S. Millerdd927a22011-05-04 12:03:30 -0700148 iph->daddr = (opt && opt->opt.srr ? opt->opt.faddr : daddr);
149 iph->saddr = saddr;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700150 iph->protocol = sk->sk_protocol;
Changli Gaod8d1f302010-06-10 23:31:35 -0700151 ip_select_ident(iph, &rt->dst, sk);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700152
Eric Dumazetf6d8bd02011-04-21 09:45:37 +0000153 if (opt && opt->opt.optlen) {
154 iph->ihl += opt->opt.optlen>>2;
155 ip_options_build(skb, &opt->opt, daddr, rt, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700156 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700157
158 skb->priority = sk->sk_priority;
Laszlo Attila Toth4a19ec52008-01-30 19:08:16 -0800159 skb->mark = sk->sk_mark;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700160
161 /* Send it out. */
Herbert Xuc439cb22008-01-11 19:14:00 -0800162 return ip_local_out(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700163}
Arnaldo Carvalho de Melod8c97a92005-08-09 20:12:12 -0700164EXPORT_SYMBOL_GPL(ip_build_and_send_pkt);
165
Linus Torvalds1da177e2005-04-16 15:20:36 -0700166static inline int ip_finish_output2(struct sk_buff *skb)
167{
Eric Dumazetadf30902009-06-02 05:19:30 +0000168 struct dst_entry *dst = skb_dst(skb);
Mitsuru Chinen80787eb2007-04-30 00:48:20 -0700169 struct rtable *rt = (struct rtable *)dst;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700170 struct net_device *dev = dst->dev;
Chuck Leverc2636b42007-10-23 21:07:32 -0700171 unsigned int hh_len = LL_RESERVED_SPACE(dev);
David S. Millerf6b72b62011-07-14 07:53:20 -0700172 struct neighbour *neigh;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700173
Neil Hormanedf391f2009-04-27 02:45:02 -0700174 if (rt->rt_type == RTN_MULTICAST) {
175 IP_UPD_PO_STATS(dev_net(dev), IPSTATS_MIB_OUTMCAST, skb->len);
176 } else if (rt->rt_type == RTN_BROADCAST)
177 IP_UPD_PO_STATS(dev_net(dev), IPSTATS_MIB_OUTBCAST, skb->len);
Mitsuru Chinen80787eb2007-04-30 00:48:20 -0700178
Linus Torvalds1da177e2005-04-16 15:20:36 -0700179 /* Be paranoid, rather than too clever. */
Stephen Hemminger3b04ddd2007-10-09 01:40:57 -0700180 if (unlikely(skb_headroom(skb) < hh_len && dev->header_ops)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700181 struct sk_buff *skb2;
182
183 skb2 = skb_realloc_headroom(skb, LL_RESERVED_SPACE(dev));
184 if (skb2 == NULL) {
185 kfree_skb(skb);
186 return -ENOMEM;
187 }
188 if (skb->sk)
189 skb_set_owner_w(skb2, skb->sk);
Eric Dumazet5d0ba552012-06-04 01:17:19 +0000190 consume_skb(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700191 skb = skb2;
192 }
193
Eric Dumazetf2c31e32011-07-29 19:00:53 +0000194 rcu_read_lock();
David Miller27217452011-12-02 16:52:08 +0000195 neigh = dst_get_neighbour_noref(dst);
Eric Dumazetf2c31e32011-07-29 19:00:53 +0000196 if (neigh) {
197 int res = neigh_output(neigh, skb);
198
199 rcu_read_unlock();
200 return res;
201 }
202 rcu_read_unlock();
David S. Miller05e3aa02011-07-16 17:26:00 -0700203
Joe Perchese87cc472012-05-13 21:56:26 +0000204 net_dbg_ratelimited("%s: No header cache and no neighbour!\n",
205 __func__);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700206 kfree_skb(skb);
207 return -EINVAL;
208}
209
John Heffner628a5c52007-04-20 15:53:27 -0700210static inline int ip_skb_dst_mtu(struct sk_buff *skb)
211{
212 struct inet_sock *inet = skb->sk ? inet_sk(skb->sk) : NULL;
213
214 return (inet && inet->pmtudisc == IP_PMTUDISC_PROBE) ?
Eric Dumazetadf30902009-06-02 05:19:30 +0000215 skb_dst(skb)->dev->mtu : dst_mtu(skb_dst(skb));
John Heffner628a5c52007-04-20 15:53:27 -0700216}
217
Patrick McHardy861d0482007-10-15 01:48:39 -0700218static int ip_finish_output(struct sk_buff *skb)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700219{
Patrick McHardy5c901da2006-01-06 23:05:36 -0800220#if defined(CONFIG_NETFILTER) && defined(CONFIG_XFRM)
221 /* Policy lookup after SNAT yielded a new policy */
Eric Dumazetadf30902009-06-02 05:19:30 +0000222 if (skb_dst(skb)->xfrm != NULL) {
Patrick McHardy48d5cad2006-02-15 15:10:22 -0800223 IPCB(skb)->flags |= IPSKB_REROUTED;
224 return dst_output(skb);
225 }
Patrick McHardy5c901da2006-01-06 23:05:36 -0800226#endif
John Heffner628a5c52007-04-20 15:53:27 -0700227 if (skb->len > ip_skb_dst_mtu(skb) && !skb_is_gso(skb))
Patrick McHardy1bd9bef2006-01-05 12:20:59 -0800228 return ip_fragment(skb, ip_finish_output2);
229 else
230 return ip_finish_output2(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700231}
232
233int ip_mc_output(struct sk_buff *skb)
234{
235 struct sock *sk = skb->sk;
Eric Dumazet511c3f92009-06-02 05:14:27 +0000236 struct rtable *rt = skb_rtable(skb);
Changli Gaod8d1f302010-06-10 23:31:35 -0700237 struct net_device *dev = rt->dst.dev;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700238
239 /*
240 * If the indicated interface is up and running, send the packet.
241 */
Neil Hormanedf391f2009-04-27 02:45:02 -0700242 IP_UPD_PO_STATS(dev_net(dev), IPSTATS_MIB_OUT, skb->len);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700243
244 skb->dev = dev;
245 skb->protocol = htons(ETH_P_IP);
246
247 /*
248 * Multicasts are looped back for other local users
249 */
250
251 if (rt->rt_flags&RTCF_MULTICAST) {
Octavian Purdila7ad68482010-01-06 20:37:01 -0800252 if (sk_mc_loop(sk)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700253#ifdef CONFIG_IP_MROUTE
254 /* Small optimization: do not loopback not local frames,
255 which returned after forwarding; they will be dropped
256 by ip_mr_input in any case.
257 Note, that local frames are looped back to be delivered
258 to local recipients.
259
260 This check is duplicated in ip_mr_input at the moment.
261 */
Joe Perches9d4fb272009-11-23 10:41:23 -0800262 &&
263 ((rt->rt_flags & RTCF_LOCAL) ||
264 !(IPCB(skb)->flags & IPSKB_FORWARDED))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700265#endif
Joe Perches9d4fb272009-11-23 10:41:23 -0800266 ) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700267 struct sk_buff *newskb = skb_clone(skb, GFP_ATOMIC);
268 if (newskb)
Jan Engelhardt9bbc7682010-03-23 04:07:29 +0100269 NF_HOOK(NFPROTO_IPV4, NF_INET_POST_ROUTING,
270 newskb, NULL, newskb->dev,
Michel Machado95603e22012-06-12 10:16:35 +0000271 dev_loopback_xmit);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700272 }
273
274 /* Multicasts with ttl 0 must not go beyond the host */
275
Arnaldo Carvalho de Meloeddc9ec2007-04-20 22:47:35 -0700276 if (ip_hdr(skb)->ttl == 0) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700277 kfree_skb(skb);
278 return 0;
279 }
280 }
281
282 if (rt->rt_flags&RTCF_BROADCAST) {
283 struct sk_buff *newskb = skb_clone(skb, GFP_ATOMIC);
284 if (newskb)
Jan Engelhardt9bbc7682010-03-23 04:07:29 +0100285 NF_HOOK(NFPROTO_IPV4, NF_INET_POST_ROUTING, newskb,
Michel Machado95603e22012-06-12 10:16:35 +0000286 NULL, newskb->dev, dev_loopback_xmit);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700287 }
288
Jan Engelhardt9bbc7682010-03-23 04:07:29 +0100289 return NF_HOOK_COND(NFPROTO_IPV4, NF_INET_POST_ROUTING, skb, NULL,
290 skb->dev, ip_finish_output,
Patrick McHardy48d5cad2006-02-15 15:10:22 -0800291 !(IPCB(skb)->flags & IPSKB_REROUTED));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700292}
293
294int ip_output(struct sk_buff *skb)
295{
Eric Dumazetadf30902009-06-02 05:19:30 +0000296 struct net_device *dev = skb_dst(skb)->dev;
Patrick McHardy1bd9bef2006-01-05 12:20:59 -0800297
Neil Hormanedf391f2009-04-27 02:45:02 -0700298 IP_UPD_PO_STATS(dev_net(dev), IPSTATS_MIB_OUT, skb->len);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700299
Patrick McHardy1bd9bef2006-01-05 12:20:59 -0800300 skb->dev = dev;
301 skb->protocol = htons(ETH_P_IP);
302
Jan Engelhardt9bbc7682010-03-23 04:07:29 +0100303 return NF_HOOK_COND(NFPROTO_IPV4, NF_INET_POST_ROUTING, skb, NULL, dev,
YOSHIFUJI Hideakie905a9e2007-02-09 23:24:47 +0900304 ip_finish_output,
Patrick McHardy48d5cad2006-02-15 15:10:22 -0800305 !(IPCB(skb)->flags & IPSKB_REROUTED));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700306}
307
Eric Dumazet84f93072011-11-30 19:00:53 +0000308/*
309 * copy saddr and daddr, possibly using 64bit load/stores
310 * Equivalent to :
311 * iph->saddr = fl4->saddr;
312 * iph->daddr = fl4->daddr;
313 */
314static void ip_copy_addrs(struct iphdr *iph, const struct flowi4 *fl4)
315{
316 BUILD_BUG_ON(offsetof(typeof(*fl4), daddr) !=
317 offsetof(typeof(*fl4), saddr) + sizeof(fl4->saddr));
318 memcpy(&iph->saddr, &fl4->saddr,
319 sizeof(fl4->saddr) + sizeof(fl4->daddr));
320}
321
David S. Millerd9d8da82011-05-06 22:23:20 -0700322int ip_queue_xmit(struct sk_buff *skb, struct flowi *fl)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700323{
David S. Millere89862f2007-01-26 01:04:55 -0800324 struct sock *sk = skb->sk;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700325 struct inet_sock *inet = inet_sk(sk);
Eric Dumazetf6d8bd02011-04-21 09:45:37 +0000326 struct ip_options_rcu *inet_opt;
David S. Millerb57ae012011-05-06 16:24:06 -0700327 struct flowi4 *fl4;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700328 struct rtable *rt;
329 struct iphdr *iph;
Eric Dumazetab6e3fe2010-05-10 11:31:49 +0000330 int res;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700331
332 /* Skip all of this if the packet is already routed,
333 * f.e. by something like SCTP.
334 */
Eric Dumazetab6e3fe2010-05-10 11:31:49 +0000335 rcu_read_lock();
Eric Dumazetf6d8bd02011-04-21 09:45:37 +0000336 inet_opt = rcu_dereference(inet->inet_opt);
David S. Millerea4fc0d2011-05-06 22:30:20 -0700337 fl4 = &fl->u.ip4;
Eric Dumazet511c3f92009-06-02 05:14:27 +0000338 rt = skb_rtable(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700339 if (rt != NULL)
340 goto packet_routed;
341
342 /* Make sure we can route this packet. */
343 rt = (struct rtable *)__sk_dst_check(sk, 0);
344 if (rt == NULL) {
Al Viro3ca3c682006-09-27 18:28:07 -0700345 __be32 daddr;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700346
347 /* Use correct destination address if we have options. */
Eric Dumazetc720c7e2009-10-15 06:30:45 +0000348 daddr = inet->inet_daddr;
Eric Dumazetf6d8bd02011-04-21 09:45:37 +0000349 if (inet_opt && inet_opt->opt.srr)
350 daddr = inet_opt->opt.faddr;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700351
David S. Miller78fbfd82011-03-12 00:00:52 -0500352 /* If this fails, retransmit mechanism of transport layer will
353 * keep trying until route appears or the connection times
354 * itself out.
355 */
David S. Millerb57ae012011-05-06 16:24:06 -0700356 rt = ip_route_output_ports(sock_net(sk), fl4, sk,
David S. Miller78fbfd82011-03-12 00:00:52 -0500357 daddr, inet->inet_saddr,
358 inet->inet_dport,
359 inet->inet_sport,
360 sk->sk_protocol,
361 RT_CONN_FLAGS(sk),
362 sk->sk_bound_dev_if);
363 if (IS_ERR(rt))
364 goto no_route;
Changli Gaod8d1f302010-06-10 23:31:35 -0700365 sk_setup_caps(sk, &rt->dst);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700366 }
Changli Gaod8d1f302010-06-10 23:31:35 -0700367 skb_dst_set_noref(skb, &rt->dst);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700368
369packet_routed:
David S. Millerea4fc0d2011-05-06 22:30:20 -0700370 if (inet_opt && inet_opt->opt.is_strictroute && fl4->daddr != rt->rt_gateway)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700371 goto no_route;
372
373 /* OK, we know where to send it, allocate and build IP header. */
Eric Dumazetf6d8bd02011-04-21 09:45:37 +0000374 skb_push(skb, sizeof(struct iphdr) + (inet_opt ? inet_opt->opt.optlen : 0));
Arnaldo Carvalho de Melo8856dfa2007-03-10 19:40:39 -0300375 skb_reset_network_header(skb);
Arnaldo Carvalho de Meloeddc9ec2007-04-20 22:47:35 -0700376 iph = ip_hdr(skb);
Al Viro714e85b2006-11-14 20:51:49 -0800377 *((__be16 *)iph) = htons((4 << 12) | (5 << 8) | (inet->tos & 0xff));
Changli Gaod8d1f302010-06-10 23:31:35 -0700378 if (ip_dont_fragment(sk, &rt->dst) && !skb->local_df)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700379 iph->frag_off = htons(IP_DF);
380 else
381 iph->frag_off = 0;
Changli Gaod8d1f302010-06-10 23:31:35 -0700382 iph->ttl = ip_select_ttl(inet, &rt->dst);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700383 iph->protocol = sk->sk_protocol;
Eric Dumazet84f93072011-11-30 19:00:53 +0000384 ip_copy_addrs(iph, fl4);
385
Linus Torvalds1da177e2005-04-16 15:20:36 -0700386 /* Transport layer set skb->h.foo itself. */
387
Eric Dumazetf6d8bd02011-04-21 09:45:37 +0000388 if (inet_opt && inet_opt->opt.optlen) {
389 iph->ihl += inet_opt->opt.optlen >> 2;
390 ip_options_build(skb, &inet_opt->opt, inet->inet_daddr, rt, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700391 }
392
Changli Gaod8d1f302010-06-10 23:31:35 -0700393 ip_select_ident_more(iph, &rt->dst, sk,
Herbert Xu79671682006-06-22 02:40:14 -0700394 (skb_shinfo(skb)->gso_segs ?: 1) - 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700395
Linus Torvalds1da177e2005-04-16 15:20:36 -0700396 skb->priority = sk->sk_priority;
Laszlo Attila Toth4a19ec52008-01-30 19:08:16 -0800397 skb->mark = sk->sk_mark;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700398
Eric Dumazetab6e3fe2010-05-10 11:31:49 +0000399 res = ip_local_out(skb);
400 rcu_read_unlock();
401 return res;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700402
403no_route:
Eric Dumazetab6e3fe2010-05-10 11:31:49 +0000404 rcu_read_unlock();
Pavel Emelyanov5e38e272008-07-16 20:19:49 -0700405 IP_INC_STATS(sock_net(sk), IPSTATS_MIB_OUTNOROUTES);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700406 kfree_skb(skb);
407 return -EHOSTUNREACH;
408}
Eric Dumazet4bc2f182010-07-09 21:22:10 +0000409EXPORT_SYMBOL(ip_queue_xmit);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700410
411
412static void ip_copy_metadata(struct sk_buff *to, struct sk_buff *from)
413{
414 to->pkt_type = from->pkt_type;
415 to->priority = from->priority;
416 to->protocol = from->protocol;
Eric Dumazetadf30902009-06-02 05:19:30 +0000417 skb_dst_drop(to);
Eric Dumazetfe76cda2010-07-01 23:48:22 +0000418 skb_dst_copy(to, from);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700419 to->dev = from->dev;
Thomas Graf82e91ff2006-11-09 15:19:14 -0800420 to->mark = from->mark;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700421
422 /* Copy the flags to each fragment. */
423 IPCB(to)->flags = IPCB(from)->flags;
424
425#ifdef CONFIG_NET_SCHED
426 to->tc_index = from->tc_index;
427#endif
Yasuyuki Kozakaie7ac05f2007-03-14 16:44:01 -0700428 nf_copy(to, from);
Jozsef Kadlecsikba9dda32007-07-07 22:21:23 -0700429#if defined(CONFIG_NETFILTER_XT_TARGET_TRACE) || \
430 defined(CONFIG_NETFILTER_XT_TARGET_TRACE_MODULE)
431 to->nf_trace = from->nf_trace;
432#endif
Julian Anastasovc98d80e2005-10-22 13:39:21 +0300433#if defined(CONFIG_IP_VS) || defined(CONFIG_IP_VS_MODULE)
434 to->ipvs_property = from->ipvs_property;
435#endif
James Morris984bc162006-06-09 00:29:17 -0700436 skb_copy_secmark(to, from);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700437}
438
439/*
440 * This IP datagram is too large to be sent in one piece. Break it up into
441 * smaller pieces (each of size equal to IP header plus
442 * a block of the data of the original IP data part) that will yet fit in a
443 * single device frame, and queue such a frame for sending.
444 */
445
Jianjun Kongd93191002008-11-03 00:23:42 -0800446int ip_fragment(struct sk_buff *skb, int (*output)(struct sk_buff *))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700447{
448 struct iphdr *iph;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700449 int ptr;
450 struct net_device *dev;
451 struct sk_buff *skb2;
Changli Gaoc893b802010-07-31 13:25:08 +0000452 unsigned int mtu, hlen, left, len, ll_rs;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700453 int offset;
Alexey Dobriyan76ab6082006-01-06 13:24:29 -0800454 __be16 not_last_frag;
Eric Dumazet511c3f92009-06-02 05:14:27 +0000455 struct rtable *rt = skb_rtable(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700456 int err = 0;
457
Changli Gaod8d1f302010-06-10 23:31:35 -0700458 dev = rt->dst.dev;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700459
460 /*
461 * Point into the IP datagram header.
462 */
463
Arnaldo Carvalho de Meloeddc9ec2007-04-20 22:47:35 -0700464 iph = ip_hdr(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700465
466 if (unlikely((iph->frag_off & htons(IP_DF)) && !skb->local_df)) {
Pavel Emelyanov5e38e272008-07-16 20:19:49 -0700467 IP_INC_STATS(dev_net(dev), IPSTATS_MIB_FRAGFAILS);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700468 icmp_send(skb, ICMP_DEST_UNREACH, ICMP_FRAG_NEEDED,
John Heffner628a5c52007-04-20 15:53:27 -0700469 htonl(ip_skb_dst_mtu(skb)));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700470 kfree_skb(skb);
471 return -EMSGSIZE;
472 }
473
474 /*
475 * Setup starting values.
476 */
477
478 hlen = iph->ihl * 4;
Changli Gaod8d1f302010-06-10 23:31:35 -0700479 mtu = dst_mtu(&rt->dst) - hlen; /* Size of data space */
Bart De Schuymer6c79bf02010-04-20 16:22:01 +0200480#ifdef CONFIG_BRIDGE_NETFILTER
481 if (skb->nf_bridge)
482 mtu -= nf_bridge_mtu_reduction(skb);
483#endif
Herbert Xu89cee8b2005-12-13 23:14:27 -0800484 IPCB(skb)->flags |= IPSKB_FRAG_COMPLETE;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700485
486 /* When frag_list is given, use it. First, check its validity:
487 * some transformers could create wrong frag_list or break existing
488 * one, it is not prohibited. In this case fall back to copying.
489 *
490 * LATER: this step can be merged to real generation of fragments,
491 * we can switch to copy when see the first bad fragment.
492 */
David S. Miller21dc3302010-08-23 00:13:46 -0700493 if (skb_has_frag_list(skb)) {
Eric Dumazet3d130082010-09-21 08:47:45 +0000494 struct sk_buff *frag, *frag2;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700495 int first_len = skb_pagelen(skb);
496
497 if (first_len - hlen > mtu ||
498 ((first_len - hlen) & 7) ||
Paul Gortmaker56f8a752011-06-21 20:33:34 -0700499 ip_is_fragment(iph) ||
Linus Torvalds1da177e2005-04-16 15:20:36 -0700500 skb_cloned(skb))
501 goto slow_path;
502
David S. Millerd7fcf1a2009-06-09 00:19:37 -0700503 skb_walk_frags(skb, frag) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700504 /* Correct geometry. */
505 if (frag->len > mtu ||
506 ((frag->len & 7) && frag->next) ||
507 skb_headroom(frag) < hlen)
Eric Dumazet3d130082010-09-21 08:47:45 +0000508 goto slow_path_clean;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700509
510 /* Partially cloned skb? */
511 if (skb_shared(frag))
Eric Dumazet3d130082010-09-21 08:47:45 +0000512 goto slow_path_clean;
Herbert Xu2fdba6b2005-05-18 22:52:33 -0700513
514 BUG_ON(frag->sk);
515 if (skb->sk) {
Herbert Xu2fdba6b2005-05-18 22:52:33 -0700516 frag->sk = skb->sk;
517 frag->destructor = sock_wfree;
Herbert Xu2fdba6b2005-05-18 22:52:33 -0700518 }
Eric Dumazet3d130082010-09-21 08:47:45 +0000519 skb->truesize -= frag->truesize;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700520 }
521
522 /* Everything is OK. Generate! */
523
524 err = 0;
525 offset = 0;
526 frag = skb_shinfo(skb)->frag_list;
David S. Millerd7fcf1a2009-06-09 00:19:37 -0700527 skb_frag_list_init(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700528 skb->data_len = first_len - skb_headlen(skb);
529 skb->len = first_len;
530 iph->tot_len = htons(first_len);
531 iph->frag_off = htons(IP_MF);
532 ip_send_check(iph);
533
534 for (;;) {
535 /* Prepare header of the next frame,
536 * before previous one went down. */
537 if (frag) {
538 frag->ip_summed = CHECKSUM_NONE;
Arnaldo Carvalho de Melobadff6d2007-03-13 13:06:52 -0300539 skb_reset_transport_header(frag);
Arnaldo Carvalho de Meloe2d1bca2007-04-10 20:46:21 -0700540 __skb_push(frag, hlen);
541 skb_reset_network_header(frag);
Arnaldo Carvalho de Melod56f90a2007-04-10 20:50:43 -0700542 memcpy(skb_network_header(frag), iph, hlen);
Arnaldo Carvalho de Meloeddc9ec2007-04-20 22:47:35 -0700543 iph = ip_hdr(frag);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700544 iph->tot_len = htons(frag->len);
545 ip_copy_metadata(frag, skb);
546 if (offset == 0)
547 ip_options_fragment(frag);
548 offset += skb->len - hlen;
549 iph->frag_off = htons(offset>>3);
550 if (frag->next != NULL)
551 iph->frag_off |= htons(IP_MF);
552 /* Ready, complete checksum */
553 ip_send_check(iph);
554 }
555
556 err = output(skb);
557
Wei Dongdafee492006-08-02 13:41:21 -0700558 if (!err)
Pavel Emelyanov5e38e272008-07-16 20:19:49 -0700559 IP_INC_STATS(dev_net(dev), IPSTATS_MIB_FRAGCREATES);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700560 if (err || !frag)
561 break;
562
563 skb = frag;
564 frag = skb->next;
565 skb->next = NULL;
566 }
567
568 if (err == 0) {
Pavel Emelyanov5e38e272008-07-16 20:19:49 -0700569 IP_INC_STATS(dev_net(dev), IPSTATS_MIB_FRAGOKS);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700570 return 0;
571 }
572
573 while (frag) {
574 skb = frag->next;
575 kfree_skb(frag);
576 frag = skb;
577 }
Pavel Emelyanov5e38e272008-07-16 20:19:49 -0700578 IP_INC_STATS(dev_net(dev), IPSTATS_MIB_FRAGFAILS);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700579 return err;
Eric Dumazet3d130082010-09-21 08:47:45 +0000580
581slow_path_clean:
582 skb_walk_frags(skb, frag2) {
583 if (frag2 == frag)
584 break;
585 frag2->sk = NULL;
586 frag2->destructor = NULL;
587 skb->truesize += frag2->truesize;
588 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700589 }
590
591slow_path:
592 left = skb->len - hlen; /* Space per frame */
George Kadianakis49085bd2010-07-06 11:44:12 +0000593 ptr = hlen; /* Where to start from */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700594
Linus Torvalds1da177e2005-04-16 15:20:36 -0700595 /* for bridged IP traffic encapsulated inside f.e. a vlan header,
Stephen Hemminger9bcfcaf2006-08-29 17:48:57 -0700596 * we need to make room for the encapsulating header
597 */
Changli Gaoc893b802010-07-31 13:25:08 +0000598 ll_rs = LL_RESERVED_SPACE_EXTRA(rt->dst.dev, nf_bridge_pad(skb));
Stephen Hemminger9bcfcaf2006-08-29 17:48:57 -0700599
Linus Torvalds1da177e2005-04-16 15:20:36 -0700600 /*
601 * Fragment the datagram.
602 */
603
604 offset = (ntohs(iph->frag_off) & IP_OFFSET) << 3;
605 not_last_frag = iph->frag_off & htons(IP_MF);
606
607 /*
608 * Keep copying data until we run out.
609 */
610
Stephen Hemminger132adf52007-03-08 20:44:43 -0800611 while (left > 0) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700612 len = left;
613 /* IF: it doesn't fit, use 'mtu' - the data space left */
614 if (len > mtu)
615 len = mtu;
Lucas De Marchi25985ed2011-03-30 22:57:33 -0300616 /* IF: we are not sending up to and including the packet end
Linus Torvalds1da177e2005-04-16 15:20:36 -0700617 then align the next start on an eight byte boundary */
618 if (len < left) {
619 len &= ~7;
620 }
621 /*
622 * Allocate buffer.
623 */
624
625 if ((skb2 = alloc_skb(len+hlen+ll_rs, GFP_ATOMIC)) == NULL) {
Patrick McHardy64ce2072005-08-09 20:50:53 -0700626 NETDEBUG(KERN_INFO "IP: frag: no memory for new fragment!\n");
Linus Torvalds1da177e2005-04-16 15:20:36 -0700627 err = -ENOMEM;
628 goto fail;
629 }
630
631 /*
632 * Set up data on packet
633 */
634
635 ip_copy_metadata(skb2, skb);
636 skb_reserve(skb2, ll_rs);
637 skb_put(skb2, len + hlen);
Arnaldo Carvalho de Meloc1d2bbe2007-04-10 20:45:18 -0700638 skb_reset_network_header(skb2);
Arnaldo Carvalho de Melob0e380b2007-04-10 21:21:55 -0700639 skb2->transport_header = skb2->network_header + hlen;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700640
641 /*
642 * Charge the memory for the fragment to any owner
643 * it might possess
644 */
645
646 if (skb->sk)
647 skb_set_owner_w(skb2, skb->sk);
648
649 /*
650 * Copy the packet header into the new buffer.
651 */
652
Arnaldo Carvalho de Melod626f622007-03-27 18:55:52 -0300653 skb_copy_from_linear_data(skb, skb_network_header(skb2), hlen);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700654
655 /*
656 * Copy a block of the IP datagram.
657 */
Arnaldo Carvalho de Melobff9b612007-03-16 17:19:57 -0300658 if (skb_copy_bits(skb, ptr, skb_transport_header(skb2), len))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700659 BUG();
660 left -= len;
661
662 /*
663 * Fill in the new header fields.
664 */
Arnaldo Carvalho de Meloeddc9ec2007-04-20 22:47:35 -0700665 iph = ip_hdr(skb2);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700666 iph->frag_off = htons((offset >> 3));
667
668 /* ANK: dirty, but effective trick. Upgrade options only if
669 * the segment to be fragmented was THE FIRST (otherwise,
670 * options are already fixed) and make it ONCE
671 * on the initial skb, so that all the following fragments
672 * will inherit fixed options.
673 */
674 if (offset == 0)
675 ip_options_fragment(skb);
676
677 /*
678 * Added AC : If we are fragmenting a fragment that's not the
679 * last fragment then keep MF on each bit
680 */
681 if (left > 0 || not_last_frag)
682 iph->frag_off |= htons(IP_MF);
683 ptr += len;
684 offset += len;
685
686 /*
687 * Put this fragment into the sending queue.
688 */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700689 iph->tot_len = htons(len + hlen);
690
691 ip_send_check(iph);
692
693 err = output(skb2);
694 if (err)
695 goto fail;
Wei Dongdafee492006-08-02 13:41:21 -0700696
Pavel Emelyanov5e38e272008-07-16 20:19:49 -0700697 IP_INC_STATS(dev_net(dev), IPSTATS_MIB_FRAGCREATES);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700698 }
Eric Dumazet5d0ba552012-06-04 01:17:19 +0000699 consume_skb(skb);
Pavel Emelyanov5e38e272008-07-16 20:19:49 -0700700 IP_INC_STATS(dev_net(dev), IPSTATS_MIB_FRAGOKS);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700701 return err;
702
703fail:
YOSHIFUJI Hideakie905a9e2007-02-09 23:24:47 +0900704 kfree_skb(skb);
Pavel Emelyanov5e38e272008-07-16 20:19:49 -0700705 IP_INC_STATS(dev_net(dev), IPSTATS_MIB_FRAGFAILS);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700706 return err;
707}
Patrick McHardy2e2f7ae2006-04-04 13:42:35 -0700708EXPORT_SYMBOL(ip_fragment);
709
Linus Torvalds1da177e2005-04-16 15:20:36 -0700710int
711ip_generic_getfrag(void *from, char *to, int offset, int len, int odd, struct sk_buff *skb)
712{
713 struct iovec *iov = from;
714
Patrick McHardy84fa7932006-08-29 16:44:56 -0700715 if (skb->ip_summed == CHECKSUM_PARTIAL) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700716 if (memcpy_fromiovecend(to, iov, offset, len) < 0)
717 return -EFAULT;
718 } else {
Al Viro44bb9362006-11-14 21:36:14 -0800719 __wsum csum = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700720 if (csum_partial_copy_fromiovecend(to, iov, offset, len, &csum) < 0)
721 return -EFAULT;
722 skb->csum = csum_block_add(skb->csum, csum, odd);
723 }
724 return 0;
725}
Eric Dumazet4bc2f182010-07-09 21:22:10 +0000726EXPORT_SYMBOL(ip_generic_getfrag);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700727
Al Viro44bb9362006-11-14 21:36:14 -0800728static inline __wsum
Linus Torvalds1da177e2005-04-16 15:20:36 -0700729csum_page(struct page *page, int offset, int copy)
730{
731 char *kaddr;
Al Viro44bb9362006-11-14 21:36:14 -0800732 __wsum csum;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700733 kaddr = kmap(page);
734 csum = csum_partial(kaddr + offset, copy, 0);
735 kunmap(page);
736 return csum;
737}
738
Adrian Bunk4b30b1c2005-11-29 16:27:20 -0800739static inline int ip_ufo_append_data(struct sock *sk,
Herbert Xu1470ddf2011-03-01 02:36:47 +0000740 struct sk_buff_head *queue,
Ananda Rajue89e9cf2005-10-18 15:46:41 -0700741 int getfrag(void *from, char *to, int offset, int len,
742 int odd, struct sk_buff *skb),
743 void *from, int length, int hh_len, int fragheaderlen,
Bill Sommerfeldd9be4f72011-07-19 15:22:33 +0000744 int transhdrlen, int maxfraglen, unsigned int flags)
Ananda Rajue89e9cf2005-10-18 15:46:41 -0700745{
746 struct sk_buff *skb;
747 int err;
748
749 /* There is support for UDP fragmentation offload by network
750 * device, so create one single skb packet containing complete
751 * udp datagram
752 */
Herbert Xu1470ddf2011-03-01 02:36:47 +0000753 if ((skb = skb_peek_tail(queue)) == NULL) {
Ananda Rajue89e9cf2005-10-18 15:46:41 -0700754 skb = sock_alloc_send_skb(sk,
755 hh_len + fragheaderlen + transhdrlen + 20,
756 (flags & MSG_DONTWAIT), &err);
757
758 if (skb == NULL)
759 return err;
760
761 /* reserve space for Hardware header */
762 skb_reserve(skb, hh_len);
763
764 /* create space for UDP/IP header */
Jianjun Kongd93191002008-11-03 00:23:42 -0800765 skb_put(skb, fragheaderlen + transhdrlen);
Ananda Rajue89e9cf2005-10-18 15:46:41 -0700766
767 /* initialize network header pointer */
Arnaldo Carvalho de Meloc1d2bbe2007-04-10 20:45:18 -0700768 skb_reset_network_header(skb);
Ananda Rajue89e9cf2005-10-18 15:46:41 -0700769
770 /* initialize protocol header pointer */
Arnaldo Carvalho de Melob0e380b2007-04-10 21:21:55 -0700771 skb->transport_header = skb->network_header + fragheaderlen;
Ananda Rajue89e9cf2005-10-18 15:46:41 -0700772
Patrick McHardy84fa7932006-08-29 16:44:56 -0700773 skb->ip_summed = CHECKSUM_PARTIAL;
Ananda Rajue89e9cf2005-10-18 15:46:41 -0700774 skb->csum = 0;
Ananda Rajue89e9cf2005-10-18 15:46:41 -0700775
Kostya Bbe9164e2008-04-29 22:36:30 -0700776 /* specify the length of each IP datagram fragment */
Bill Sommerfeldd9be4f72011-07-19 15:22:33 +0000777 skb_shinfo(skb)->gso_size = maxfraglen - fragheaderlen;
Herbert Xuf83ef8c2006-06-30 13:37:03 -0700778 skb_shinfo(skb)->gso_type = SKB_GSO_UDP;
Herbert Xu1470ddf2011-03-01 02:36:47 +0000779 __skb_queue_tail(queue, skb);
Ananda Rajue89e9cf2005-10-18 15:46:41 -0700780 }
Kostya Bbe9164e2008-04-29 22:36:30 -0700781
782 return skb_append_datato_frags(sk, skb, getfrag, from,
783 (length - transhdrlen));
Ananda Rajue89e9cf2005-10-18 15:46:41 -0700784}
785
David S. Millerf5fca602011-05-08 17:24:10 -0700786static int __ip_append_data(struct sock *sk,
787 struct flowi4 *fl4,
788 struct sk_buff_head *queue,
Herbert Xu1470ddf2011-03-01 02:36:47 +0000789 struct inet_cork *cork,
790 int getfrag(void *from, char *to, int offset,
791 int len, int odd, struct sk_buff *skb),
792 void *from, int length, int transhdrlen,
793 unsigned int flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700794{
795 struct inet_sock *inet = inet_sk(sk);
796 struct sk_buff *skb;
797
Herbert Xu07df5292011-03-01 23:00:58 -0800798 struct ip_options *opt = cork->opt;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700799 int hh_len;
800 int exthdrlen;
801 int mtu;
802 int copy;
803 int err;
804 int offset = 0;
805 unsigned int maxfraglen, fragheaderlen;
806 int csummode = CHECKSUM_NONE;
Herbert Xu1470ddf2011-03-01 02:36:47 +0000807 struct rtable *rt = (struct rtable *)cork->dst;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700808
Steffen Klassert96d73032011-06-05 20:48:47 +0000809 skb = skb_peek_tail(queue);
810
811 exthdrlen = !skb ? rt->dst.header_len : 0;
Herbert Xu07df5292011-03-01 23:00:58 -0800812 mtu = cork->fragsize;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700813
Changli Gaod8d1f302010-06-10 23:31:35 -0700814 hh_len = LL_RESERVED_SPACE(rt->dst.dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700815
816 fragheaderlen = sizeof(struct iphdr) + (opt ? opt->optlen : 0);
817 maxfraglen = ((mtu - fragheaderlen) & ~7) + fragheaderlen;
818
Herbert Xu1470ddf2011-03-01 02:36:47 +0000819 if (cork->length + length > 0xFFFF - fragheaderlen) {
David S. Millerf5fca602011-05-08 17:24:10 -0700820 ip_local_error(sk, EMSGSIZE, fl4->daddr, inet->inet_dport,
Eric Dumazetc720c7e2009-10-15 06:30:45 +0000821 mtu-exthdrlen);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700822 return -EMSGSIZE;
823 }
824
825 /*
826 * transhdrlen > 0 means that this is the first fragment and we wish
827 * it won't be fragmented in the future.
828 */
829 if (transhdrlen &&
830 length + fragheaderlen <= mtu &&
Changli Gaod8d1f302010-06-10 23:31:35 -0700831 rt->dst.dev->features & NETIF_F_V4_CSUM &&
Linus Torvalds1da177e2005-04-16 15:20:36 -0700832 !exthdrlen)
Patrick McHardy84fa7932006-08-29 16:44:56 -0700833 csummode = CHECKSUM_PARTIAL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700834
Herbert Xu1470ddf2011-03-01 02:36:47 +0000835 cork->length += length;
Herbert Xu26cde9f2010-06-15 01:52:25 +0000836 if (((length > mtu) || (skb && skb_is_gso(skb))) &&
Kostya Bbe9164e2008-04-29 22:36:30 -0700837 (sk->sk_protocol == IPPROTO_UDP) &&
Steffen Klassertc1460662011-06-29 23:19:32 +0000838 (rt->dst.dev->features & NETIF_F_UFO) && !rt->dst.header_len) {
Herbert Xu1470ddf2011-03-01 02:36:47 +0000839 err = ip_ufo_append_data(sk, queue, getfrag, from, length,
840 hh_len, fragheaderlen, transhdrlen,
Bill Sommerfeldd9be4f72011-07-19 15:22:33 +0000841 maxfraglen, flags);
Patrick McHardybaa829d2006-03-12 20:35:12 -0800842 if (err)
Ananda Rajue89e9cf2005-10-18 15:46:41 -0700843 goto error;
Ananda Rajue89e9cf2005-10-18 15:46:41 -0700844 return 0;
845 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700846
847 /* So, what's going on in the loop below?
848 *
849 * We use calculated fragment length to generate chained skb,
850 * each of segments is IP fragment ready for sending to network after
851 * adding appropriate IP header.
852 */
853
Herbert Xu26cde9f2010-06-15 01:52:25 +0000854 if (!skb)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700855 goto alloc_new_skb;
856
857 while (length > 0) {
858 /* Check if the remaining data fits into current packet. */
859 copy = mtu - skb->len;
860 if (copy < length)
861 copy = maxfraglen - skb->len;
862 if (copy <= 0) {
863 char *data;
864 unsigned int datalen;
865 unsigned int fraglen;
866 unsigned int fraggap;
867 unsigned int alloclen;
868 struct sk_buff *skb_prev;
869alloc_new_skb:
870 skb_prev = skb;
871 if (skb_prev)
872 fraggap = skb_prev->len - maxfraglen;
873 else
874 fraggap = 0;
875
876 /*
877 * If remaining data exceeds the mtu,
878 * we know we need more fragment(s).
879 */
880 datalen = length + fraggap;
881 if (datalen > mtu - fragheaderlen)
882 datalen = maxfraglen - fragheaderlen;
883 fraglen = datalen + fragheaderlen;
884
YOSHIFUJI Hideakie905a9e2007-02-09 23:24:47 +0900885 if ((flags & MSG_MORE) &&
Changli Gaod8d1f302010-06-10 23:31:35 -0700886 !(rt->dst.dev->features&NETIF_F_SG))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700887 alloclen = mtu;
888 else
Eric Dumazet59104f02010-09-20 20:16:27 +0000889 alloclen = fraglen;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700890
Steffen Klassert353e5c92011-06-22 01:05:37 +0000891 alloclen += exthdrlen;
892
Linus Torvalds1da177e2005-04-16 15:20:36 -0700893 /* The last fragment gets additional space at tail.
894 * Note, with MSG_MORE we overallocate on fragments,
895 * because we have no idea what fragment will be
896 * the last.
897 */
Steffen Klassert33f99dc2011-06-22 01:04:37 +0000898 if (datalen == length + fraggap)
Changli Gaod8d1f302010-06-10 23:31:35 -0700899 alloclen += rt->dst.trailer_len;
Steffen Klassert33f99dc2011-06-22 01:04:37 +0000900
Linus Torvalds1da177e2005-04-16 15:20:36 -0700901 if (transhdrlen) {
YOSHIFUJI Hideakie905a9e2007-02-09 23:24:47 +0900902 skb = sock_alloc_send_skb(sk,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700903 alloclen + hh_len + 15,
904 (flags & MSG_DONTWAIT), &err);
905 } else {
906 skb = NULL;
907 if (atomic_read(&sk->sk_wmem_alloc) <=
908 2 * sk->sk_sndbuf)
YOSHIFUJI Hideakie905a9e2007-02-09 23:24:47 +0900909 skb = sock_wmalloc(sk,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700910 alloclen + hh_len + 15, 1,
911 sk->sk_allocation);
912 if (unlikely(skb == NULL))
913 err = -ENOBUFS;
Patrick Ohly51f31ca2009-02-12 05:03:39 +0000914 else
915 /* only the initial fragment is
916 time stamped */
Herbert Xu1470ddf2011-03-01 02:36:47 +0000917 cork->tx_flags = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700918 }
919 if (skb == NULL)
920 goto error;
921
922 /*
923 * Fill in the control structures
924 */
925 skb->ip_summed = csummode;
926 skb->csum = 0;
927 skb_reserve(skb, hh_len);
Herbert Xu1470ddf2011-03-01 02:36:47 +0000928 skb_shinfo(skb)->tx_flags = cork->tx_flags;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700929
930 /*
931 * Find where to start putting bytes.
932 */
Steffen Klassert353e5c92011-06-22 01:05:37 +0000933 data = skb_put(skb, fraglen + exthdrlen);
Arnaldo Carvalho de Meloc14d2452007-03-11 22:39:41 -0300934 skb_set_network_header(skb, exthdrlen);
Arnaldo Carvalho de Melob0e380b2007-04-10 21:21:55 -0700935 skb->transport_header = (skb->network_header +
936 fragheaderlen);
Steffen Klassert353e5c92011-06-22 01:05:37 +0000937 data += fragheaderlen + exthdrlen;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700938
939 if (fraggap) {
940 skb->csum = skb_copy_and_csum_bits(
941 skb_prev, maxfraglen,
942 data + transhdrlen, fraggap, 0);
943 skb_prev->csum = csum_sub(skb_prev->csum,
944 skb->csum);
945 data += fraggap;
Herbert Xue9fa4f72006-08-13 20:12:58 -0700946 pskb_trim_unique(skb_prev, maxfraglen);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700947 }
948
949 copy = datalen - transhdrlen - fraggap;
950 if (copy > 0 && getfrag(from, data + transhdrlen, offset, copy, fraggap, skb) < 0) {
951 err = -EFAULT;
952 kfree_skb(skb);
953 goto error;
954 }
955
956 offset += copy;
957 length -= datalen - fraggap;
958 transhdrlen = 0;
959 exthdrlen = 0;
960 csummode = CHECKSUM_NONE;
961
962 /*
963 * Put the packet on the pending queue.
964 */
Herbert Xu1470ddf2011-03-01 02:36:47 +0000965 __skb_queue_tail(queue, skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700966 continue;
967 }
968
969 if (copy > length)
970 copy = length;
971
Changli Gaod8d1f302010-06-10 23:31:35 -0700972 if (!(rt->dst.dev->features&NETIF_F_SG)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700973 unsigned int off;
974
975 off = skb->len;
YOSHIFUJI Hideakie905a9e2007-02-09 23:24:47 +0900976 if (getfrag(from, skb_put(skb, copy),
Linus Torvalds1da177e2005-04-16 15:20:36 -0700977 offset, copy, off, skb) < 0) {
978 __skb_trim(skb, off);
979 err = -EFAULT;
980 goto error;
981 }
982 } else {
983 int i = skb_shinfo(skb)->nr_frags;
984 skb_frag_t *frag = &skb_shinfo(skb)->frags[i-1];
Herbert Xu1470ddf2011-03-01 02:36:47 +0000985 struct page *page = cork->page;
986 int off = cork->off;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700987 unsigned int left;
988
989 if (page && (left = PAGE_SIZE - off) > 0) {
990 if (copy >= left)
991 copy = left;
Ian Campbellaff65da2011-08-22 23:44:59 +0000992 if (page != skb_frag_page(frag)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700993 if (i == MAX_SKB_FRAGS) {
994 err = -EMSGSIZE;
995 goto error;
996 }
Herbert Xu1470ddf2011-03-01 02:36:47 +0000997 skb_fill_page_desc(skb, i, page, off, 0);
Ian Campbellaff65da2011-08-22 23:44:59 +0000998 skb_frag_ref(skb, i);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700999 frag = &skb_shinfo(skb)->frags[i];
1000 }
1001 } else if (i < MAX_SKB_FRAGS) {
1002 if (copy > PAGE_SIZE)
1003 copy = PAGE_SIZE;
1004 page = alloc_pages(sk->sk_allocation, 0);
1005 if (page == NULL) {
1006 err = -ENOMEM;
1007 goto error;
1008 }
Herbert Xu1470ddf2011-03-01 02:36:47 +00001009 cork->page = page;
1010 cork->off = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001011
1012 skb_fill_page_desc(skb, i, page, 0, 0);
1013 frag = &skb_shinfo(skb)->frags[i];
Linus Torvalds1da177e2005-04-16 15:20:36 -07001014 } else {
1015 err = -EMSGSIZE;
1016 goto error;
1017 }
Eric Dumazet9e903e02011-10-18 21:00:24 +00001018 if (getfrag(from, skb_frag_address(frag)+skb_frag_size(frag),
Ian Campbellaff65da2011-08-22 23:44:59 +00001019 offset, copy, skb->len, skb) < 0) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001020 err = -EFAULT;
1021 goto error;
1022 }
Herbert Xu1470ddf2011-03-01 02:36:47 +00001023 cork->off += copy;
Eric Dumazet9e903e02011-10-18 21:00:24 +00001024 skb_frag_size_add(frag, copy);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001025 skb->len += copy;
1026 skb->data_len += copy;
Herbert Xuf945fa72008-01-22 22:39:26 -08001027 skb->truesize += copy;
1028 atomic_add(copy, &sk->sk_wmem_alloc);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001029 }
1030 offset += copy;
1031 length -= copy;
1032 }
1033
1034 return 0;
1035
1036error:
Herbert Xu1470ddf2011-03-01 02:36:47 +00001037 cork->length -= length;
Pavel Emelyanov5e38e272008-07-16 20:19:49 -07001038 IP_INC_STATS(sock_net(sk), IPSTATS_MIB_OUTDISCARDS);
YOSHIFUJI Hideakie905a9e2007-02-09 23:24:47 +09001039 return err;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001040}
1041
Herbert Xu1470ddf2011-03-01 02:36:47 +00001042static int ip_setup_cork(struct sock *sk, struct inet_cork *cork,
1043 struct ipcm_cookie *ipc, struct rtable **rtp)
1044{
1045 struct inet_sock *inet = inet_sk(sk);
Eric Dumazetf6d8bd02011-04-21 09:45:37 +00001046 struct ip_options_rcu *opt;
Herbert Xu1470ddf2011-03-01 02:36:47 +00001047 struct rtable *rt;
1048
1049 /*
1050 * setup for corking.
1051 */
1052 opt = ipc->opt;
1053 if (opt) {
1054 if (cork->opt == NULL) {
1055 cork->opt = kmalloc(sizeof(struct ip_options) + 40,
1056 sk->sk_allocation);
1057 if (unlikely(cork->opt == NULL))
1058 return -ENOBUFS;
1059 }
Eric Dumazetf6d8bd02011-04-21 09:45:37 +00001060 memcpy(cork->opt, &opt->opt, sizeof(struct ip_options) + opt->opt.optlen);
Herbert Xu1470ddf2011-03-01 02:36:47 +00001061 cork->flags |= IPCORK_OPT;
1062 cork->addr = ipc->addr;
1063 }
1064 rt = *rtp;
1065 if (unlikely(!rt))
1066 return -EFAULT;
1067 /*
1068 * We steal reference to this route, caller should not release it
1069 */
1070 *rtp = NULL;
1071 cork->fragsize = inet->pmtudisc == IP_PMTUDISC_PROBE ?
Steffen Klassert353e5c92011-06-22 01:05:37 +00001072 rt->dst.dev->mtu : dst_mtu(&rt->dst);
Herbert Xu1470ddf2011-03-01 02:36:47 +00001073 cork->dst = &rt->dst;
1074 cork->length = 0;
1075 cork->tx_flags = ipc->tx_flags;
1076 cork->page = NULL;
1077 cork->off = 0;
1078
1079 return 0;
1080}
1081
1082/*
1083 * ip_append_data() and ip_append_page() can make one large IP datagram
1084 * from many pieces of data. Each pieces will be holded on the socket
1085 * until ip_push_pending_frames() is called. Each piece can be a page
1086 * or non-page data.
1087 *
1088 * Not only UDP, other transport protocols - e.g. raw sockets - can use
1089 * this interface potentially.
1090 *
1091 * LATER: length must be adjusted by pad at tail, when it is required.
1092 */
David S. Millerf5fca602011-05-08 17:24:10 -07001093int ip_append_data(struct sock *sk, struct flowi4 *fl4,
Herbert Xu1470ddf2011-03-01 02:36:47 +00001094 int getfrag(void *from, char *to, int offset, int len,
1095 int odd, struct sk_buff *skb),
1096 void *from, int length, int transhdrlen,
1097 struct ipcm_cookie *ipc, struct rtable **rtp,
1098 unsigned int flags)
1099{
1100 struct inet_sock *inet = inet_sk(sk);
1101 int err;
1102
1103 if (flags&MSG_PROBE)
1104 return 0;
1105
1106 if (skb_queue_empty(&sk->sk_write_queue)) {
David S. Millerbdc712b2011-05-06 15:02:07 -07001107 err = ip_setup_cork(sk, &inet->cork.base, ipc, rtp);
Herbert Xu1470ddf2011-03-01 02:36:47 +00001108 if (err)
1109 return err;
1110 } else {
1111 transhdrlen = 0;
1112 }
1113
David S. Millerf5fca602011-05-08 17:24:10 -07001114 return __ip_append_data(sk, fl4, &sk->sk_write_queue, &inet->cork.base, getfrag,
Herbert Xu1470ddf2011-03-01 02:36:47 +00001115 from, length, transhdrlen, flags);
1116}
1117
David S. Millerf5fca602011-05-08 17:24:10 -07001118ssize_t ip_append_page(struct sock *sk, struct flowi4 *fl4, struct page *page,
Linus Torvalds1da177e2005-04-16 15:20:36 -07001119 int offset, size_t size, int flags)
1120{
1121 struct inet_sock *inet = inet_sk(sk);
1122 struct sk_buff *skb;
1123 struct rtable *rt;
1124 struct ip_options *opt = NULL;
David S. Millerbdc712b2011-05-06 15:02:07 -07001125 struct inet_cork *cork;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001126 int hh_len;
1127 int mtu;
1128 int len;
1129 int err;
1130 unsigned int maxfraglen, fragheaderlen, fraggap;
1131
1132 if (inet->hdrincl)
1133 return -EPERM;
1134
1135 if (flags&MSG_PROBE)
1136 return 0;
1137
1138 if (skb_queue_empty(&sk->sk_write_queue))
1139 return -EINVAL;
1140
David S. Millerbdc712b2011-05-06 15:02:07 -07001141 cork = &inet->cork.base;
1142 rt = (struct rtable *)cork->dst;
1143 if (cork->flags & IPCORK_OPT)
1144 opt = cork->opt;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001145
Changli Gaod8d1f302010-06-10 23:31:35 -07001146 if (!(rt->dst.dev->features&NETIF_F_SG))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001147 return -EOPNOTSUPP;
1148
Changli Gaod8d1f302010-06-10 23:31:35 -07001149 hh_len = LL_RESERVED_SPACE(rt->dst.dev);
David S. Millerbdc712b2011-05-06 15:02:07 -07001150 mtu = cork->fragsize;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001151
1152 fragheaderlen = sizeof(struct iphdr) + (opt ? opt->optlen : 0);
1153 maxfraglen = ((mtu - fragheaderlen) & ~7) + fragheaderlen;
1154
David S. Millerbdc712b2011-05-06 15:02:07 -07001155 if (cork->length + size > 0xFFFF - fragheaderlen) {
David S. Millerf5fca602011-05-08 17:24:10 -07001156 ip_local_error(sk, EMSGSIZE, fl4->daddr, inet->inet_dport, mtu);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001157 return -EMSGSIZE;
1158 }
1159
1160 if ((skb = skb_peek_tail(&sk->sk_write_queue)) == NULL)
1161 return -EINVAL;
1162
David S. Millerbdc712b2011-05-06 15:02:07 -07001163 cork->length += size;
Herbert Xu26cde9f2010-06-15 01:52:25 +00001164 if ((size + skb->len > mtu) &&
1165 (sk->sk_protocol == IPPROTO_UDP) &&
Changli Gaod8d1f302010-06-10 23:31:35 -07001166 (rt->dst.dev->features & NETIF_F_UFO)) {
Herbert Xu79671682006-06-22 02:40:14 -07001167 skb_shinfo(skb)->gso_size = mtu - fragheaderlen;
Herbert Xuf83ef8c2006-06-30 13:37:03 -07001168 skb_shinfo(skb)->gso_type = SKB_GSO_UDP;
Herbert Xu79671682006-06-22 02:40:14 -07001169 }
Ananda Rajue89e9cf2005-10-18 15:46:41 -07001170
Linus Torvalds1da177e2005-04-16 15:20:36 -07001171
1172 while (size > 0) {
1173 int i;
1174
Herbert Xu89114af2006-07-08 13:34:32 -07001175 if (skb_is_gso(skb))
Ananda Rajue89e9cf2005-10-18 15:46:41 -07001176 len = size;
1177 else {
1178
1179 /* Check if the remaining data fits into current packet. */
1180 len = mtu - skb->len;
1181 if (len < size)
1182 len = maxfraglen - skb->len;
1183 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001184 if (len <= 0) {
1185 struct sk_buff *skb_prev;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001186 int alloclen;
1187
1188 skb_prev = skb;
Jayachandran C0d0d2bb2005-10-13 11:43:02 -07001189 fraggap = skb_prev->len - maxfraglen;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001190
1191 alloclen = fragheaderlen + hh_len + fraggap + 15;
1192 skb = sock_wmalloc(sk, alloclen, 1, sk->sk_allocation);
1193 if (unlikely(!skb)) {
1194 err = -ENOBUFS;
1195 goto error;
1196 }
1197
1198 /*
1199 * Fill in the control structures
1200 */
1201 skb->ip_summed = CHECKSUM_NONE;
1202 skb->csum = 0;
1203 skb_reserve(skb, hh_len);
1204
1205 /*
1206 * Find where to start putting bytes.
1207 */
Arnaldo Carvalho de Melo967b05f2007-03-13 13:51:52 -03001208 skb_put(skb, fragheaderlen + fraggap);
Arnaldo Carvalho de Melo2ca9e6f2007-03-10 19:15:25 -03001209 skb_reset_network_header(skb);
Arnaldo Carvalho de Melob0e380b2007-04-10 21:21:55 -07001210 skb->transport_header = (skb->network_header +
1211 fragheaderlen);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001212 if (fraggap) {
Arnaldo Carvalho de Melo967b05f2007-03-13 13:51:52 -03001213 skb->csum = skb_copy_and_csum_bits(skb_prev,
1214 maxfraglen,
Arnaldo Carvalho de Melo9c702202007-04-25 18:04:18 -07001215 skb_transport_header(skb),
Arnaldo Carvalho de Melo967b05f2007-03-13 13:51:52 -03001216 fraggap, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001217 skb_prev->csum = csum_sub(skb_prev->csum,
1218 skb->csum);
Herbert Xue9fa4f72006-08-13 20:12:58 -07001219 pskb_trim_unique(skb_prev, maxfraglen);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001220 }
1221
1222 /*
1223 * Put the packet on the pending queue.
1224 */
1225 __skb_queue_tail(&sk->sk_write_queue, skb);
1226 continue;
1227 }
1228
1229 i = skb_shinfo(skb)->nr_frags;
1230 if (len > size)
1231 len = size;
1232 if (skb_can_coalesce(skb, i, page, offset)) {
Eric Dumazet9e903e02011-10-18 21:00:24 +00001233 skb_frag_size_add(&skb_shinfo(skb)->frags[i-1], len);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001234 } else if (i < MAX_SKB_FRAGS) {
1235 get_page(page);
1236 skb_fill_page_desc(skb, i, page, offset, len);
1237 } else {
1238 err = -EMSGSIZE;
1239 goto error;
1240 }
1241
1242 if (skb->ip_summed == CHECKSUM_NONE) {
Al Viro44bb9362006-11-14 21:36:14 -08001243 __wsum csum;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001244 csum = csum_page(page, offset, len);
1245 skb->csum = csum_block_add(skb->csum, csum, skb->len);
1246 }
1247
1248 skb->len += len;
1249 skb->data_len += len;
David S. Miller1e34a112008-01-22 23:44:31 -08001250 skb->truesize += len;
1251 atomic_add(len, &sk->sk_wmem_alloc);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001252 offset += len;
1253 size -= len;
1254 }
1255 return 0;
1256
1257error:
David S. Millerbdc712b2011-05-06 15:02:07 -07001258 cork->length -= size;
Pavel Emelyanov5e38e272008-07-16 20:19:49 -07001259 IP_INC_STATS(sock_net(sk), IPSTATS_MIB_OUTDISCARDS);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001260 return err;
1261}
1262
Herbert Xu1470ddf2011-03-01 02:36:47 +00001263static void ip_cork_release(struct inet_cork *cork)
Pavel Emelyanov429f08e2007-11-05 21:03:24 -08001264{
Herbert Xu1470ddf2011-03-01 02:36:47 +00001265 cork->flags &= ~IPCORK_OPT;
1266 kfree(cork->opt);
1267 cork->opt = NULL;
1268 dst_release(cork->dst);
1269 cork->dst = NULL;
Pavel Emelyanov429f08e2007-11-05 21:03:24 -08001270}
1271
Linus Torvalds1da177e2005-04-16 15:20:36 -07001272/*
1273 * Combined all pending IP fragments on the socket as one IP datagram
1274 * and push them out.
1275 */
Herbert Xu1c32c5a2011-03-01 02:36:47 +00001276struct sk_buff *__ip_make_skb(struct sock *sk,
David S. Miller77968b72011-05-08 17:12:19 -07001277 struct flowi4 *fl4,
Herbert Xu1c32c5a2011-03-01 02:36:47 +00001278 struct sk_buff_head *queue,
1279 struct inet_cork *cork)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001280{
1281 struct sk_buff *skb, *tmp_skb;
1282 struct sk_buff **tail_skb;
1283 struct inet_sock *inet = inet_sk(sk);
Pavel Emelyanov0388b002008-07-14 23:00:43 -07001284 struct net *net = sock_net(sk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001285 struct ip_options *opt = NULL;
Herbert Xu1470ddf2011-03-01 02:36:47 +00001286 struct rtable *rt = (struct rtable *)cork->dst;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001287 struct iphdr *iph;
Alexey Dobriyan76ab6082006-01-06 13:24:29 -08001288 __be16 df = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001289 __u8 ttl;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001290
Herbert Xu1470ddf2011-03-01 02:36:47 +00001291 if ((skb = __skb_dequeue(queue)) == NULL)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001292 goto out;
1293 tail_skb = &(skb_shinfo(skb)->frag_list);
1294
1295 /* move skb->data to ip header from ext header */
Arnaldo Carvalho de Melod56f90a2007-04-10 20:50:43 -07001296 if (skb->data < skb_network_header(skb))
Arnaldo Carvalho de Melobbe735e2007-03-10 22:16:10 -03001297 __skb_pull(skb, skb_network_offset(skb));
Herbert Xu1470ddf2011-03-01 02:36:47 +00001298 while ((tmp_skb = __skb_dequeue(queue)) != NULL) {
Arnaldo Carvalho de Melocfe1fc72007-03-16 17:26:39 -03001299 __skb_pull(tmp_skb, skb_network_header_len(skb));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001300 *tail_skb = tmp_skb;
1301 tail_skb = &(tmp_skb->next);
1302 skb->len += tmp_skb->len;
1303 skb->data_len += tmp_skb->len;
1304 skb->truesize += tmp_skb->truesize;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001305 tmp_skb->destructor = NULL;
1306 tmp_skb->sk = NULL;
1307 }
1308
1309 /* Unless user demanded real pmtu discovery (IP_PMTUDISC_DO), we allow
1310 * to fragment the frame generated here. No matter, what transforms
1311 * how transforms change size of the packet, it will come out.
1312 */
John Heffner628a5c52007-04-20 15:53:27 -07001313 if (inet->pmtudisc < IP_PMTUDISC_DO)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001314 skb->local_df = 1;
1315
1316 /* DF bit is set when we want to see DF on outgoing frames.
1317 * If local_df is set too, we still allow to fragment this frame
1318 * locally. */
John Heffner628a5c52007-04-20 15:53:27 -07001319 if (inet->pmtudisc >= IP_PMTUDISC_DO ||
Changli Gaod8d1f302010-06-10 23:31:35 -07001320 (skb->len <= dst_mtu(&rt->dst) &&
1321 ip_dont_fragment(sk, &rt->dst)))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001322 df = htons(IP_DF);
1323
Herbert Xu1470ddf2011-03-01 02:36:47 +00001324 if (cork->flags & IPCORK_OPT)
1325 opt = cork->opt;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001326
1327 if (rt->rt_type == RTN_MULTICAST)
1328 ttl = inet->mc_ttl;
1329 else
Changli Gaod8d1f302010-06-10 23:31:35 -07001330 ttl = ip_select_ttl(inet, &rt->dst);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001331
1332 iph = (struct iphdr *)skb->data;
1333 iph->version = 4;
1334 iph->ihl = 5;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001335 iph->tos = inet->tos;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001336 iph->frag_off = df;
Changli Gaod8d1f302010-06-10 23:31:35 -07001337 ip_select_ident(iph, &rt->dst, sk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001338 iph->ttl = ttl;
1339 iph->protocol = sk->sk_protocol;
Eric Dumazet84f93072011-11-30 19:00:53 +00001340 ip_copy_addrs(iph, fl4);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001341
David S. Miller22f728f2011-05-13 17:21:27 -04001342 if (opt) {
1343 iph->ihl += opt->optlen>>2;
1344 ip_options_build(skb, opt, cork->addr, rt, 0);
1345 }
1346
Linus Torvalds1da177e2005-04-16 15:20:36 -07001347 skb->priority = sk->sk_priority;
Laszlo Attila Toth4a19ec52008-01-30 19:08:16 -08001348 skb->mark = sk->sk_mark;
Eric Dumazeta21bba92008-11-24 16:07:50 -08001349 /*
1350 * Steal rt from cork.dst to avoid a pair of atomic_inc/atomic_dec
1351 * on dst refcount
1352 */
Herbert Xu1470ddf2011-03-01 02:36:47 +00001353 cork->dst = NULL;
Changli Gaod8d1f302010-06-10 23:31:35 -07001354 skb_dst_set(skb, &rt->dst);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001355
David L Stevens96793b42007-09-17 09:57:33 -07001356 if (iph->protocol == IPPROTO_ICMP)
Pavel Emelyanov0388b002008-07-14 23:00:43 -07001357 icmp_out_count(net, ((struct icmphdr *)
David L Stevens96793b42007-09-17 09:57:33 -07001358 skb_transport_header(skb))->type);
1359
Herbert Xu1c32c5a2011-03-01 02:36:47 +00001360 ip_cork_release(cork);
1361out:
1362 return skb;
1363}
1364
1365int ip_send_skb(struct sk_buff *skb)
1366{
1367 struct net *net = sock_net(skb->sk);
1368 int err;
1369
Herbert Xuc439cb22008-01-11 19:14:00 -08001370 err = ip_local_out(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001371 if (err) {
1372 if (err > 0)
Eric Dumazet6ce9e7b2009-09-02 18:05:33 -07001373 err = net_xmit_errno(err);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001374 if (err)
Herbert Xu1c32c5a2011-03-01 02:36:47 +00001375 IP_INC_STATS(net, IPSTATS_MIB_OUTDISCARDS);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001376 }
1377
Linus Torvalds1da177e2005-04-16 15:20:36 -07001378 return err;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001379}
1380
David S. Miller77968b72011-05-08 17:12:19 -07001381int ip_push_pending_frames(struct sock *sk, struct flowi4 *fl4)
Herbert Xu1470ddf2011-03-01 02:36:47 +00001382{
Herbert Xu1c32c5a2011-03-01 02:36:47 +00001383 struct sk_buff *skb;
1384
David S. Miller77968b72011-05-08 17:12:19 -07001385 skb = ip_finish_skb(sk, fl4);
Herbert Xu1c32c5a2011-03-01 02:36:47 +00001386 if (!skb)
1387 return 0;
1388
1389 /* Netfilter gets whole the not fragmented skb. */
1390 return ip_send_skb(skb);
Herbert Xu1470ddf2011-03-01 02:36:47 +00001391}
1392
Linus Torvalds1da177e2005-04-16 15:20:36 -07001393/*
1394 * Throw away all pending data on the socket.
1395 */
Herbert Xu1470ddf2011-03-01 02:36:47 +00001396static void __ip_flush_pending_frames(struct sock *sk,
1397 struct sk_buff_head *queue,
1398 struct inet_cork *cork)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001399{
Linus Torvalds1da177e2005-04-16 15:20:36 -07001400 struct sk_buff *skb;
1401
Herbert Xu1470ddf2011-03-01 02:36:47 +00001402 while ((skb = __skb_dequeue_tail(queue)) != NULL)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001403 kfree_skb(skb);
1404
Herbert Xu1470ddf2011-03-01 02:36:47 +00001405 ip_cork_release(cork);
1406}
1407
1408void ip_flush_pending_frames(struct sock *sk)
1409{
David S. Millerbdc712b2011-05-06 15:02:07 -07001410 __ip_flush_pending_frames(sk, &sk->sk_write_queue, &inet_sk(sk)->cork.base);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001411}
1412
Herbert Xu1c32c5a2011-03-01 02:36:47 +00001413struct sk_buff *ip_make_skb(struct sock *sk,
David S. Miller77968b72011-05-08 17:12:19 -07001414 struct flowi4 *fl4,
Herbert Xu1c32c5a2011-03-01 02:36:47 +00001415 int getfrag(void *from, char *to, int offset,
1416 int len, int odd, struct sk_buff *skb),
1417 void *from, int length, int transhdrlen,
1418 struct ipcm_cookie *ipc, struct rtable **rtp,
1419 unsigned int flags)
1420{
David S. Millerb80d7222011-05-06 15:06:01 -07001421 struct inet_cork cork;
Herbert Xu1c32c5a2011-03-01 02:36:47 +00001422 struct sk_buff_head queue;
1423 int err;
1424
1425 if (flags & MSG_PROBE)
1426 return NULL;
1427
1428 __skb_queue_head_init(&queue);
1429
David S. Millerb80d7222011-05-06 15:06:01 -07001430 cork.flags = 0;
1431 cork.addr = 0;
David S. Miller70652722011-05-06 16:01:15 -07001432 cork.opt = NULL;
Herbert Xu1c32c5a2011-03-01 02:36:47 +00001433 err = ip_setup_cork(sk, &cork, ipc, rtp);
1434 if (err)
1435 return ERR_PTR(err);
1436
David S. Millerf5fca602011-05-08 17:24:10 -07001437 err = __ip_append_data(sk, fl4, &queue, &cork, getfrag,
Herbert Xu1c32c5a2011-03-01 02:36:47 +00001438 from, length, transhdrlen, flags);
1439 if (err) {
1440 __ip_flush_pending_frames(sk, &queue, &cork);
1441 return ERR_PTR(err);
1442 }
1443
David S. Miller77968b72011-05-08 17:12:19 -07001444 return __ip_make_skb(sk, fl4, &queue, &cork);
Herbert Xu1c32c5a2011-03-01 02:36:47 +00001445}
Linus Torvalds1da177e2005-04-16 15:20:36 -07001446
1447/*
1448 * Fetch data from kernel space and fill in checksum if needed.
1449 */
YOSHIFUJI Hideakie905a9e2007-02-09 23:24:47 +09001450static int ip_reply_glue_bits(void *dptr, char *to, int offset,
Linus Torvalds1da177e2005-04-16 15:20:36 -07001451 int len, int odd, struct sk_buff *skb)
1452{
Al Viro50842052006-11-14 21:36:34 -08001453 __wsum csum;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001454
1455 csum = csum_partial_copy_nocheck(dptr+offset, to, len, 0);
1456 skb->csum = csum_block_add(skb->csum, csum, odd);
YOSHIFUJI Hideakie905a9e2007-02-09 23:24:47 +09001457 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001458}
1459
YOSHIFUJI Hideakie905a9e2007-02-09 23:24:47 +09001460/*
Linus Torvalds1da177e2005-04-16 15:20:36 -07001461 * Generic function to send a packet as reply to another packet.
1462 * Used to send TCP resets so far. ICMP should use this function too.
1463 *
YOSHIFUJI Hideakie905a9e2007-02-09 23:24:47 +09001464 * Should run single threaded per socket because it uses the sock
Linus Torvalds1da177e2005-04-16 15:20:36 -07001465 * structure to pass arguments.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001466 */
David S. Miller0a5ebb82011-05-09 13:22:43 -07001467void ip_send_reply(struct sock *sk, struct sk_buff *skb, __be32 daddr,
Eric Dumazet66b13d92011-10-24 03:06:21 -04001468 const struct ip_reply_arg *arg, unsigned int len)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001469{
1470 struct inet_sock *inet = inet_sk(sk);
Eric Dumazetf6d8bd02011-04-21 09:45:37 +00001471 struct ip_options_data replyopts;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001472 struct ipcm_cookie ipc;
David S. Miller77968b72011-05-08 17:12:19 -07001473 struct flowi4 fl4;
Eric Dumazet511c3f92009-06-02 05:14:27 +00001474 struct rtable *rt = skb_rtable(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001475
Eric Dumazetf6d8bd02011-04-21 09:45:37 +00001476 if (ip_options_echo(&replyopts.opt.opt, skb))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001477 return;
1478
David S. Miller0a5ebb82011-05-09 13:22:43 -07001479 ipc.addr = daddr;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001480 ipc.opt = NULL;
Oliver Hartkopp2244d072010-08-17 08:59:14 +00001481 ipc.tx_flags = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001482
Eric Dumazetf6d8bd02011-04-21 09:45:37 +00001483 if (replyopts.opt.opt.optlen) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001484 ipc.opt = &replyopts.opt;
1485
Eric Dumazetf6d8bd02011-04-21 09:45:37 +00001486 if (replyopts.opt.opt.srr)
1487 daddr = replyopts.opt.opt.faddr;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001488 }
1489
David S. Miller77968b72011-05-08 17:12:19 -07001490 flowi4_init_output(&fl4, arg->bound_dev_if, 0,
Eric Dumazet66b13d92011-10-24 03:06:21 -04001491 RT_TOS(arg->tos),
David S. Miller77968b72011-05-08 17:12:19 -07001492 RT_SCOPE_UNIVERSE, sk->sk_protocol,
1493 ip_reply_arg_flowi_flags(arg),
1494 daddr, rt->rt_spec_dst,
1495 tcp_hdr(skb)->source, tcp_hdr(skb)->dest);
1496 security_skb_classify_flow(skb, flowi4_to_flowi(&fl4));
1497 rt = ip_route_output_key(sock_net(sk), &fl4);
1498 if (IS_ERR(rt))
1499 return;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001500
1501 /* And let IP do all the hard work.
1502
1503 This chunk is not reenterable, hence spinlock.
1504 Note that it uses the fact, that this function is called
1505 with locally disabled BH and that sk cannot be already spinlocked.
1506 */
1507 bh_lock_sock(sk);
Eric Dumazet66b13d92011-10-24 03:06:21 -04001508 inet->tos = arg->tos;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001509 sk->sk_priority = skb->priority;
Arnaldo Carvalho de Meloeddc9ec2007-04-20 22:47:35 -07001510 sk->sk_protocol = ip_hdr(skb)->protocol;
Patrick McHardyf0e48db2007-06-04 21:32:46 -07001511 sk->sk_bound_dev_if = arg->bound_dev_if;
David S. Millerf5fca602011-05-08 17:24:10 -07001512 ip_append_data(sk, &fl4, ip_reply_glue_bits, arg->iov->iov_base, len, 0,
Eric Dumazet2e77d892008-11-24 15:52:46 -08001513 &ipc, &rt, MSG_DONTWAIT);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001514 if ((skb = skb_peek(&sk->sk_write_queue)) != NULL) {
1515 if (arg->csumoffset >= 0)
Arnaldo Carvalho de Melo9c702202007-04-25 18:04:18 -07001516 *((__sum16 *)skb_transport_header(skb) +
1517 arg->csumoffset) = csum_fold(csum_add(skb->csum,
1518 arg->csum));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001519 skb->ip_summed = CHECKSUM_NONE;
David S. Miller77968b72011-05-08 17:12:19 -07001520 ip_push_pending_frames(sk, &fl4);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001521 }
1522
1523 bh_unlock_sock(sk);
1524
1525 ip_rt_put(rt);
1526}
1527
Linus Torvalds1da177e2005-04-16 15:20:36 -07001528void __init ip_init(void)
1529{
Linus Torvalds1da177e2005-04-16 15:20:36 -07001530 ip_rt_init();
1531 inet_initpeers();
1532
1533#if defined(CONFIG_IP_MULTICAST) && defined(CONFIG_PROC_FS)
1534 igmp_mc_proc_init();
1535#endif
1536}