blob: 870b5182ddd8a88c71cd7b4399779a850372cdfa [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * INET An implementation of the TCP/IP protocol suite for the LINUX
3 * operating system. INET is implemented using the BSD Socket
4 * interface as the means of communication with the user level.
5 *
6 * ROUTE - implementation of the IP router.
7 *
Jesper Juhl02c30a82005-05-05 16:16:16 -07008 * Authors: Ross Biro
Linus Torvalds1da177e2005-04-16 15:20:36 -07009 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
10 * Alan Cox, <gw4pts@gw4pts.ampr.org>
11 * Linus Torvalds, <Linus.Torvalds@helsinki.fi>
12 * Alexey Kuznetsov, <kuznet@ms2.inr.ac.ru>
13 *
14 * Fixes:
15 * Alan Cox : Verify area fixes.
16 * Alan Cox : cli() protects routing changes
17 * Rui Oliveira : ICMP routing table updates
18 * (rco@di.uminho.pt) Routing table insertion and update
19 * Linus Torvalds : Rewrote bits to be sensible
20 * Alan Cox : Added BSD route gw semantics
YOSHIFUJI Hideakie905a9e2007-02-09 23:24:47 +090021 * Alan Cox : Super /proc >4K
Linus Torvalds1da177e2005-04-16 15:20:36 -070022 * Alan Cox : MTU in route table
23 * Alan Cox : MSS actually. Also added the window
24 * clamper.
25 * Sam Lantinga : Fixed route matching in rt_del()
26 * Alan Cox : Routing cache support.
27 * Alan Cox : Removed compatibility cruft.
28 * Alan Cox : RTF_REJECT support.
29 * Alan Cox : TCP irtt support.
30 * Jonathan Naylor : Added Metric support.
31 * Miquel van Smoorenburg : BSD API fixes.
32 * Miquel van Smoorenburg : Metrics.
33 * Alan Cox : Use __u32 properly
34 * Alan Cox : Aligned routing errors more closely with BSD
35 * our system is still very different.
36 * Alan Cox : Faster /proc handling
37 * Alexey Kuznetsov : Massive rework to support tree based routing,
38 * routing caches and better behaviour.
YOSHIFUJI Hideakie905a9e2007-02-09 23:24:47 +090039 *
Linus Torvalds1da177e2005-04-16 15:20:36 -070040 * Olaf Erb : irtt wasn't being copied right.
41 * Bjorn Ekwall : Kerneld route support.
42 * Alan Cox : Multicast fixed (I hope)
43 * Pavel Krauz : Limited broadcast fixed
44 * Mike McLagan : Routing by source
45 * Alexey Kuznetsov : End of old history. Split to fib.c and
46 * route.c and rewritten from scratch.
47 * Andi Kleen : Load-limit warning messages.
48 * Vitaly E. Lavrov : Transparent proxy revived after year coma.
49 * Vitaly E. Lavrov : Race condition in ip_route_input_slow.
50 * Tobias Ringstrom : Uninitialized res.type in ip_route_output_slow.
51 * Vladimir V. Ivanov : IP rule info (flowid) is really useful.
52 * Marc Boucher : routing by fwmark
53 * Robert Olsson : Added rt_cache statistics
54 * Arnaldo C. Melo : Convert proc stuff to seq_file
Eric Dumazetbb1d23b2005-07-05 15:00:32 -070055 * Eric Dumazet : hashed spinlocks and rt_check_expire() fixes.
Ilia Sotnikovcef26852006-03-25 01:38:55 -080056 * Ilia Sotnikov : Ignore TOS on PMTUD and Redirect
57 * Ilia Sotnikov : Removed TOS from hash calculations
Linus Torvalds1da177e2005-04-16 15:20:36 -070058 *
59 * This program is free software; you can redistribute it and/or
60 * modify it under the terms of the GNU General Public License
61 * as published by the Free Software Foundation; either version
62 * 2 of the License, or (at your option) any later version.
63 */
64
Linus Torvalds1da177e2005-04-16 15:20:36 -070065#include <linux/module.h>
66#include <asm/uaccess.h>
67#include <asm/system.h>
68#include <linux/bitops.h>
69#include <linux/types.h>
70#include <linux/kernel.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070071#include <linux/mm.h>
Eric Dumazet424c4b72005-07-05 14:58:19 -070072#include <linux/bootmem.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070073#include <linux/string.h>
74#include <linux/socket.h>
75#include <linux/sockios.h>
76#include <linux/errno.h>
77#include <linux/in.h>
78#include <linux/inet.h>
79#include <linux/netdevice.h>
80#include <linux/proc_fs.h>
81#include <linux/init.h>
Eric Dumazet39c90ec2007-09-15 10:55:54 -070082#include <linux/workqueue.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070083#include <linux/skbuff.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070084#include <linux/inetdevice.h>
85#include <linux/igmp.h>
86#include <linux/pkt_sched.h>
87#include <linux/mroute.h>
88#include <linux/netfilter_ipv4.h>
89#include <linux/random.h>
90#include <linux/jhash.h>
91#include <linux/rcupdate.h>
92#include <linux/times.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090093#include <linux/slab.h>
Herbert Xu352e5122007-11-13 21:34:06 -080094#include <net/dst.h>
Eric W. Biederman457c4cb2007-09-12 12:01:34 +020095#include <net/net_namespace.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070096#include <net/protocol.h>
97#include <net/ip.h>
98#include <net/route.h>
99#include <net/inetpeer.h>
100#include <net/sock.h>
101#include <net/ip_fib.h>
102#include <net/arp.h>
103#include <net/tcp.h>
104#include <net/icmp.h>
105#include <net/xfrm.h>
Tom Tucker8d717402006-07-30 20:43:36 -0700106#include <net/netevent.h>
Thomas Graf63f34442007-03-22 11:55:17 -0700107#include <net/rtnetlink.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700108#ifdef CONFIG_SYSCTL
109#include <linux/sysctl.h>
110#endif
111
David S. Miller68a5e3d2011-03-11 20:07:33 -0500112#define RT_FL_TOS(oldflp4) \
113 ((u32)(oldflp4->flowi4_tos & (IPTOS_RT_MASK | RTO_ONLINK)))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700114
115#define IP_MAX_MTU 0xFFF0
116
117#define RT_GC_TIMEOUT (300*HZ)
118
Linus Torvalds1da177e2005-04-16 15:20:36 -0700119static int ip_rt_max_size;
Stephen Hemminger817bc4d2008-03-22 17:43:59 -0700120static int ip_rt_gc_timeout __read_mostly = RT_GC_TIMEOUT;
121static int ip_rt_gc_interval __read_mostly = 60 * HZ;
122static int ip_rt_gc_min_interval __read_mostly = HZ / 2;
123static int ip_rt_redirect_number __read_mostly = 9;
124static int ip_rt_redirect_load __read_mostly = HZ / 50;
125static int ip_rt_redirect_silence __read_mostly = ((HZ / 50) << (9 + 1));
126static int ip_rt_error_cost __read_mostly = HZ;
127static int ip_rt_error_burst __read_mostly = 5 * HZ;
128static int ip_rt_gc_elasticity __read_mostly = 8;
129static int ip_rt_mtu_expires __read_mostly = 10 * 60 * HZ;
130static int ip_rt_min_pmtu __read_mostly = 512 + 20 + 20;
131static int ip_rt_min_advmss __read_mostly = 256;
Neil Horman1080d702008-10-27 12:28:25 -0700132static int rt_chain_length_max __read_mostly = 20;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700133
Linus Torvalds1da177e2005-04-16 15:20:36 -0700134/*
135 * Interface to generic destination cache.
136 */
137
138static struct dst_entry *ipv4_dst_check(struct dst_entry *dst, u32 cookie);
David S. Miller0dbaee32010-12-13 12:52:14 -0800139static unsigned int ipv4_default_advmss(const struct dst_entry *dst);
David S. Millerd33e4552010-12-14 13:01:14 -0800140static unsigned int ipv4_default_mtu(const struct dst_entry *dst);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700141static void ipv4_dst_destroy(struct dst_entry *dst);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700142static struct dst_entry *ipv4_negative_advice(struct dst_entry *dst);
143static void ipv4_link_failure(struct sk_buff *skb);
144static void ip_rt_update_pmtu(struct dst_entry *dst, u32 mtu);
Daniel Lezcano569d3642008-01-18 03:56:57 -0800145static int rt_garbage_collect(struct dst_ops *ops);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700146
Eric Dumazet72cdd1d2010-11-11 07:14:07 +0000147static void ipv4_dst_ifdown(struct dst_entry *dst, struct net_device *dev,
148 int how)
149{
150}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700151
David S. Miller62fa8a82011-01-26 20:51:05 -0800152static u32 *ipv4_cow_metrics(struct dst_entry *dst, unsigned long old)
153{
David S. Miller06582542011-01-27 14:58:42 -0800154 struct rtable *rt = (struct rtable *) dst;
155 struct inet_peer *peer;
156 u32 *p = NULL;
David S. Miller62fa8a82011-01-26 20:51:05 -0800157
David S. Miller06582542011-01-27 14:58:42 -0800158 if (!rt->peer)
159 rt_bind_peer(rt, 1);
160
161 peer = rt->peer;
162 if (peer) {
David S. Miller62fa8a82011-01-26 20:51:05 -0800163 u32 *old_p = __DST_METRICS_PTR(old);
164 unsigned long prev, new;
165
David S. Miller06582542011-01-27 14:58:42 -0800166 p = peer->metrics;
167 if (inet_metrics_new(peer))
168 memcpy(p, old_p, sizeof(u32) * RTAX_MAX);
David S. Miller62fa8a82011-01-26 20:51:05 -0800169
170 new = (unsigned long) p;
171 prev = cmpxchg(&dst->_metrics, old, new);
172
173 if (prev != old) {
David S. Miller62fa8a82011-01-26 20:51:05 -0800174 p = __DST_METRICS_PTR(prev);
175 if (prev & DST_METRICS_READ_ONLY)
176 p = NULL;
177 } else {
David S. Miller62fa8a82011-01-26 20:51:05 -0800178 if (rt->fi) {
179 fib_info_put(rt->fi);
180 rt->fi = NULL;
181 }
182 }
183 }
184 return p;
185}
186
Linus Torvalds1da177e2005-04-16 15:20:36 -0700187static struct dst_ops ipv4_dst_ops = {
188 .family = AF_INET,
Harvey Harrison09640e62009-02-01 00:45:17 -0800189 .protocol = cpu_to_be16(ETH_P_IP),
Linus Torvalds1da177e2005-04-16 15:20:36 -0700190 .gc = rt_garbage_collect,
191 .check = ipv4_dst_check,
David S. Miller0dbaee32010-12-13 12:52:14 -0800192 .default_advmss = ipv4_default_advmss,
David S. Millerd33e4552010-12-14 13:01:14 -0800193 .default_mtu = ipv4_default_mtu,
David S. Miller62fa8a82011-01-26 20:51:05 -0800194 .cow_metrics = ipv4_cow_metrics,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700195 .destroy = ipv4_dst_destroy,
196 .ifdown = ipv4_dst_ifdown,
197 .negative_advice = ipv4_negative_advice,
198 .link_failure = ipv4_link_failure,
199 .update_pmtu = ip_rt_update_pmtu,
Herbert Xu1ac06e02008-05-20 14:32:14 -0700200 .local_out = __ip_local_out,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700201};
202
203#define ECN_OR_COST(class) TC_PRIO_##class
204
Philippe De Muyter4839c522007-07-09 15:32:57 -0700205const __u8 ip_tos2prio[16] = {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700206 TC_PRIO_BESTEFFORT,
Dan Siemon4a2b9c32011-03-15 13:56:07 +0000207 ECN_OR_COST(BESTEFFORT),
Linus Torvalds1da177e2005-04-16 15:20:36 -0700208 TC_PRIO_BESTEFFORT,
209 ECN_OR_COST(BESTEFFORT),
210 TC_PRIO_BULK,
211 ECN_OR_COST(BULK),
212 TC_PRIO_BULK,
213 ECN_OR_COST(BULK),
214 TC_PRIO_INTERACTIVE,
215 ECN_OR_COST(INTERACTIVE),
216 TC_PRIO_INTERACTIVE,
217 ECN_OR_COST(INTERACTIVE),
218 TC_PRIO_INTERACTIVE_BULK,
219 ECN_OR_COST(INTERACTIVE_BULK),
220 TC_PRIO_INTERACTIVE_BULK,
221 ECN_OR_COST(INTERACTIVE_BULK)
222};
223
224
225/*
226 * Route cache.
227 */
228
229/* The locking scheme is rather straight forward:
230 *
231 * 1) Read-Copy Update protects the buckets of the central route hash.
232 * 2) Only writers remove entries, and they hold the lock
233 * as they look at rtable reference counts.
234 * 3) Only readers acquire references to rtable entries,
235 * they do so with atomic increments and with the
236 * lock held.
237 */
238
239struct rt_hash_bucket {
Eric Dumazet1c317202010-10-25 21:02:07 +0000240 struct rtable __rcu *chain;
Eric Dumazet22c047c2005-07-05 14:55:24 -0700241};
Neil Horman1080d702008-10-27 12:28:25 -0700242
Ingo Molnar8a25d5d2006-07-03 00:24:54 -0700243#if defined(CONFIG_SMP) || defined(CONFIG_DEBUG_SPINLOCK) || \
244 defined(CONFIG_PROVE_LOCKING)
Eric Dumazet22c047c2005-07-05 14:55:24 -0700245/*
246 * Instead of using one spinlock for each rt_hash_bucket, we use a table of spinlocks
247 * The size of this table is a power of two and depends on the number of CPUS.
Ingo Molnar62051202006-07-03 00:24:59 -0700248 * (on lockdep we have a quite big spinlock_t, so keep the size down there)
Eric Dumazet22c047c2005-07-05 14:55:24 -0700249 */
Ingo Molnar62051202006-07-03 00:24:59 -0700250#ifdef CONFIG_LOCKDEP
251# define RT_HASH_LOCK_SZ 256
Eric Dumazet22c047c2005-07-05 14:55:24 -0700252#else
Ingo Molnar62051202006-07-03 00:24:59 -0700253# if NR_CPUS >= 32
254# define RT_HASH_LOCK_SZ 4096
255# elif NR_CPUS >= 16
256# define RT_HASH_LOCK_SZ 2048
257# elif NR_CPUS >= 8
258# define RT_HASH_LOCK_SZ 1024
259# elif NR_CPUS >= 4
260# define RT_HASH_LOCK_SZ 512
261# else
262# define RT_HASH_LOCK_SZ 256
263# endif
Eric Dumazet22c047c2005-07-05 14:55:24 -0700264#endif
265
266static spinlock_t *rt_hash_locks;
267# define rt_hash_lock_addr(slot) &rt_hash_locks[(slot) & (RT_HASH_LOCK_SZ - 1)]
Pavel Emelyanov1ff1cc22007-12-05 21:15:05 -0800268
269static __init void rt_hash_lock_init(void)
270{
271 int i;
272
273 rt_hash_locks = kmalloc(sizeof(spinlock_t) * RT_HASH_LOCK_SZ,
274 GFP_KERNEL);
275 if (!rt_hash_locks)
276 panic("IP: failed to allocate rt_hash_locks\n");
277
278 for (i = 0; i < RT_HASH_LOCK_SZ; i++)
279 spin_lock_init(&rt_hash_locks[i]);
280}
Eric Dumazet22c047c2005-07-05 14:55:24 -0700281#else
282# define rt_hash_lock_addr(slot) NULL
Pavel Emelyanov1ff1cc22007-12-05 21:15:05 -0800283
284static inline void rt_hash_lock_init(void)
285{
286}
Eric Dumazet22c047c2005-07-05 14:55:24 -0700287#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -0700288
Stephen Hemminger817bc4d2008-03-22 17:43:59 -0700289static struct rt_hash_bucket *rt_hash_table __read_mostly;
290static unsigned rt_hash_mask __read_mostly;
291static unsigned int rt_hash_log __read_mostly;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700292
Eric Dumazet2f970d82006-01-17 02:54:36 -0800293static DEFINE_PER_CPU(struct rt_cache_stat, rt_cache_stat);
Eric Dumazet27f39c73e2010-05-19 22:07:23 +0000294#define RT_CACHE_STAT_INC(field) __this_cpu_inc(rt_cache_stat.field)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700295
Denis V. Lunevb00180d2008-07-05 19:04:09 -0700296static inline unsigned int rt_hash(__be32 daddr, __be32 saddr, int idx,
Eric Dumazet0eae88f2010-04-20 19:06:52 -0700297 int genid)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700298{
Eric Dumazet0eae88f2010-04-20 19:06:52 -0700299 return jhash_3words((__force u32)daddr, (__force u32)saddr,
Denis V. Lunevb00180d2008-07-05 19:04:09 -0700300 idx, genid)
Eric Dumazet29e75252008-01-31 17:05:09 -0800301 & rt_hash_mask;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700302}
303
Denis V. Luneve84f84f2008-07-05 19:04:32 -0700304static inline int rt_genid(struct net *net)
305{
306 return atomic_read(&net->ipv4.rt_genid);
307}
308
Linus Torvalds1da177e2005-04-16 15:20:36 -0700309#ifdef CONFIG_PROC_FS
310struct rt_cache_iter_state {
Denis V. Luneva75e9362008-02-28 20:50:55 -0800311 struct seq_net_private p;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700312 int bucket;
Eric Dumazet29e75252008-01-31 17:05:09 -0800313 int genid;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700314};
315
YOSHIFUJI Hideaki12188542008-03-26 02:36:06 +0900316static struct rtable *rt_cache_get_first(struct seq_file *seq)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700317{
YOSHIFUJI Hideaki12188542008-03-26 02:36:06 +0900318 struct rt_cache_iter_state *st = seq->private;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700319 struct rtable *r = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700320
321 for (st->bucket = rt_hash_mask; st->bucket >= 0; --st->bucket) {
Eric Dumazet1c317202010-10-25 21:02:07 +0000322 if (!rcu_dereference_raw(rt_hash_table[st->bucket].chain))
Eric Dumazeta6272662008-08-28 01:11:25 -0700323 continue;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700324 rcu_read_lock_bh();
Paul E. McKenneya898def2010-02-22 17:04:49 -0800325 r = rcu_dereference_bh(rt_hash_table[st->bucket].chain);
Eric Dumazet29e75252008-01-31 17:05:09 -0800326 while (r) {
Changli Gaod8d1f302010-06-10 23:31:35 -0700327 if (dev_net(r->dst.dev) == seq_file_net(seq) &&
Denis V. Luneva75e9362008-02-28 20:50:55 -0800328 r->rt_genid == st->genid)
Eric Dumazet29e75252008-01-31 17:05:09 -0800329 return r;
Changli Gaod8d1f302010-06-10 23:31:35 -0700330 r = rcu_dereference_bh(r->dst.rt_next);
Eric Dumazet29e75252008-01-31 17:05:09 -0800331 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700332 rcu_read_unlock_bh();
333 }
Eric Dumazet29e75252008-01-31 17:05:09 -0800334 return r;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700335}
336
YOSHIFUJI Hideaki12188542008-03-26 02:36:06 +0900337static struct rtable *__rt_cache_get_next(struct seq_file *seq,
Denis V. Lunev642d6312008-02-28 20:50:33 -0800338 struct rtable *r)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700339{
YOSHIFUJI Hideaki12188542008-03-26 02:36:06 +0900340 struct rt_cache_iter_state *st = seq->private;
Eric Dumazeta6272662008-08-28 01:11:25 -0700341
Eric Dumazet1c317202010-10-25 21:02:07 +0000342 r = rcu_dereference_bh(r->dst.rt_next);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700343 while (!r) {
344 rcu_read_unlock_bh();
Eric Dumazeta6272662008-08-28 01:11:25 -0700345 do {
346 if (--st->bucket < 0)
347 return NULL;
Eric Dumazet1c317202010-10-25 21:02:07 +0000348 } while (!rcu_dereference_raw(rt_hash_table[st->bucket].chain));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700349 rcu_read_lock_bh();
Eric Dumazet1c317202010-10-25 21:02:07 +0000350 r = rcu_dereference_bh(rt_hash_table[st->bucket].chain);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700351 }
Eric Dumazet1c317202010-10-25 21:02:07 +0000352 return r;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700353}
354
YOSHIFUJI Hideaki12188542008-03-26 02:36:06 +0900355static struct rtable *rt_cache_get_next(struct seq_file *seq,
Denis V. Lunev642d6312008-02-28 20:50:33 -0800356 struct rtable *r)
357{
YOSHIFUJI Hideaki12188542008-03-26 02:36:06 +0900358 struct rt_cache_iter_state *st = seq->private;
359 while ((r = __rt_cache_get_next(seq, r)) != NULL) {
Changli Gaod8d1f302010-06-10 23:31:35 -0700360 if (dev_net(r->dst.dev) != seq_file_net(seq))
Denis V. Luneva75e9362008-02-28 20:50:55 -0800361 continue;
Denis V. Lunev642d6312008-02-28 20:50:33 -0800362 if (r->rt_genid == st->genid)
363 break;
364 }
365 return r;
366}
367
YOSHIFUJI Hideaki12188542008-03-26 02:36:06 +0900368static struct rtable *rt_cache_get_idx(struct seq_file *seq, loff_t pos)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700369{
YOSHIFUJI Hideaki12188542008-03-26 02:36:06 +0900370 struct rtable *r = rt_cache_get_first(seq);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700371
372 if (r)
YOSHIFUJI Hideaki12188542008-03-26 02:36:06 +0900373 while (pos && (r = rt_cache_get_next(seq, r)))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700374 --pos;
375 return pos ? NULL : r;
376}
377
378static void *rt_cache_seq_start(struct seq_file *seq, loff_t *pos)
379{
Eric Dumazet29e75252008-01-31 17:05:09 -0800380 struct rt_cache_iter_state *st = seq->private;
Eric Dumazet29e75252008-01-31 17:05:09 -0800381 if (*pos)
YOSHIFUJI Hideaki12188542008-03-26 02:36:06 +0900382 return rt_cache_get_idx(seq, *pos - 1);
Denis V. Luneve84f84f2008-07-05 19:04:32 -0700383 st->genid = rt_genid(seq_file_net(seq));
Eric Dumazet29e75252008-01-31 17:05:09 -0800384 return SEQ_START_TOKEN;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700385}
386
387static void *rt_cache_seq_next(struct seq_file *seq, void *v, loff_t *pos)
388{
Eric Dumazet29e75252008-01-31 17:05:09 -0800389 struct rtable *r;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700390
391 if (v == SEQ_START_TOKEN)
YOSHIFUJI Hideaki12188542008-03-26 02:36:06 +0900392 r = rt_cache_get_first(seq);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700393 else
YOSHIFUJI Hideaki12188542008-03-26 02:36:06 +0900394 r = rt_cache_get_next(seq, v);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700395 ++*pos;
396 return r;
397}
398
399static void rt_cache_seq_stop(struct seq_file *seq, void *v)
400{
401 if (v && v != SEQ_START_TOKEN)
402 rcu_read_unlock_bh();
403}
404
405static int rt_cache_seq_show(struct seq_file *seq, void *v)
406{
407 if (v == SEQ_START_TOKEN)
408 seq_printf(seq, "%-127s\n",
409 "Iface\tDestination\tGateway \tFlags\t\tRefCnt\tUse\t"
410 "Metric\tSource\t\tMTU\tWindow\tIRTT\tTOS\tHHRef\t"
411 "HHUptod\tSpecDst");
412 else {
413 struct rtable *r = v;
Pavel Emelyanov5e659e42008-04-24 01:02:16 -0700414 int len;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700415
Eric Dumazet0eae88f2010-04-20 19:06:52 -0700416 seq_printf(seq, "%s\t%08X\t%08X\t%8X\t%d\t%u\t%d\t"
417 "%08X\t%d\t%u\t%u\t%02X\t%d\t%1d\t%08X%n",
Changli Gaod8d1f302010-06-10 23:31:35 -0700418 r->dst.dev ? r->dst.dev->name : "*",
Eric Dumazet0eae88f2010-04-20 19:06:52 -0700419 (__force u32)r->rt_dst,
420 (__force u32)r->rt_gateway,
Changli Gaod8d1f302010-06-10 23:31:35 -0700421 r->rt_flags, atomic_read(&r->dst.__refcnt),
422 r->dst.__use, 0, (__force u32)r->rt_src,
David S. Miller0dbaee32010-12-13 12:52:14 -0800423 dst_metric_advmss(&r->dst) + 40,
Changli Gaod8d1f302010-06-10 23:31:35 -0700424 dst_metric(&r->dst, RTAX_WINDOW),
425 (int)((dst_metric(&r->dst, RTAX_RTT) >> 3) +
426 dst_metric(&r->dst, RTAX_RTTVAR)),
David S. Miller5e2b61f2011-03-04 21:47:09 -0800427 r->rt_tos,
Changli Gaod8d1f302010-06-10 23:31:35 -0700428 r->dst.hh ? atomic_read(&r->dst.hh->hh_refcnt) : -1,
429 r->dst.hh ? (r->dst.hh->hh_output ==
Linus Torvalds1da177e2005-04-16 15:20:36 -0700430 dev_queue_xmit) : 0,
Pavel Emelyanov5e659e42008-04-24 01:02:16 -0700431 r->rt_spec_dst, &len);
432
433 seq_printf(seq, "%*s\n", 127 - len, "");
YOSHIFUJI Hideakie905a9e2007-02-09 23:24:47 +0900434 }
435 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700436}
437
Stephen Hemmingerf6908082007-03-12 14:34:29 -0700438static const struct seq_operations rt_cache_seq_ops = {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700439 .start = rt_cache_seq_start,
440 .next = rt_cache_seq_next,
441 .stop = rt_cache_seq_stop,
442 .show = rt_cache_seq_show,
443};
444
445static int rt_cache_seq_open(struct inode *inode, struct file *file)
446{
Denis V. Luneva75e9362008-02-28 20:50:55 -0800447 return seq_open_net(inode, file, &rt_cache_seq_ops,
Pavel Emelyanovcf7732e2007-10-10 02:29:29 -0700448 sizeof(struct rt_cache_iter_state));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700449}
450
Arjan van de Ven9a321442007-02-12 00:55:35 -0800451static const struct file_operations rt_cache_seq_fops = {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700452 .owner = THIS_MODULE,
453 .open = rt_cache_seq_open,
454 .read = seq_read,
455 .llseek = seq_lseek,
Denis V. Luneva75e9362008-02-28 20:50:55 -0800456 .release = seq_release_net,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700457};
458
459
460static void *rt_cpu_seq_start(struct seq_file *seq, loff_t *pos)
461{
462 int cpu;
463
464 if (*pos == 0)
465 return SEQ_START_TOKEN;
466
Rusty Russell0f23174a2008-12-29 12:23:42 +0000467 for (cpu = *pos-1; cpu < nr_cpu_ids; ++cpu) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700468 if (!cpu_possible(cpu))
469 continue;
470 *pos = cpu+1;
Eric Dumazet2f970d82006-01-17 02:54:36 -0800471 return &per_cpu(rt_cache_stat, cpu);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700472 }
473 return NULL;
474}
475
476static void *rt_cpu_seq_next(struct seq_file *seq, void *v, loff_t *pos)
477{
478 int cpu;
479
Rusty Russell0f23174a2008-12-29 12:23:42 +0000480 for (cpu = *pos; cpu < nr_cpu_ids; ++cpu) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700481 if (!cpu_possible(cpu))
482 continue;
483 *pos = cpu+1;
Eric Dumazet2f970d82006-01-17 02:54:36 -0800484 return &per_cpu(rt_cache_stat, cpu);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700485 }
486 return NULL;
YOSHIFUJI Hideakie905a9e2007-02-09 23:24:47 +0900487
Linus Torvalds1da177e2005-04-16 15:20:36 -0700488}
489
490static void rt_cpu_seq_stop(struct seq_file *seq, void *v)
491{
492
493}
494
495static int rt_cpu_seq_show(struct seq_file *seq, void *v)
496{
497 struct rt_cache_stat *st = v;
498
499 if (v == SEQ_START_TOKEN) {
Olaf Rempel5bec0032005-04-28 12:16:08 -0700500 seq_printf(seq, "entries in_hit in_slow_tot in_slow_mc in_no_route in_brd in_martian_dst in_martian_src out_hit out_slow_tot out_slow_mc gc_total gc_ignored gc_goal_miss gc_dst_overflow in_hlist_search out_hlist_search\n");
Linus Torvalds1da177e2005-04-16 15:20:36 -0700501 return 0;
502 }
YOSHIFUJI Hideakie905a9e2007-02-09 23:24:47 +0900503
Linus Torvalds1da177e2005-04-16 15:20:36 -0700504 seq_printf(seq,"%08x %08x %08x %08x %08x %08x %08x %08x "
505 " %08x %08x %08x %08x %08x %08x %08x %08x %08x \n",
Eric Dumazetfc66f952010-10-08 06:37:34 +0000506 dst_entries_get_slow(&ipv4_dst_ops),
Linus Torvalds1da177e2005-04-16 15:20:36 -0700507 st->in_hit,
508 st->in_slow_tot,
509 st->in_slow_mc,
510 st->in_no_route,
511 st->in_brd,
512 st->in_martian_dst,
513 st->in_martian_src,
514
515 st->out_hit,
516 st->out_slow_tot,
YOSHIFUJI Hideakie905a9e2007-02-09 23:24:47 +0900517 st->out_slow_mc,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700518
519 st->gc_total,
520 st->gc_ignored,
521 st->gc_goal_miss,
522 st->gc_dst_overflow,
523 st->in_hlist_search,
524 st->out_hlist_search
525 );
526 return 0;
527}
528
Stephen Hemmingerf6908082007-03-12 14:34:29 -0700529static const struct seq_operations rt_cpu_seq_ops = {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700530 .start = rt_cpu_seq_start,
531 .next = rt_cpu_seq_next,
532 .stop = rt_cpu_seq_stop,
533 .show = rt_cpu_seq_show,
534};
535
536
537static int rt_cpu_seq_open(struct inode *inode, struct file *file)
538{
539 return seq_open(file, &rt_cpu_seq_ops);
540}
541
Arjan van de Ven9a321442007-02-12 00:55:35 -0800542static const struct file_operations rt_cpu_seq_fops = {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700543 .owner = THIS_MODULE,
544 .open = rt_cpu_seq_open,
545 .read = seq_read,
546 .llseek = seq_lseek,
547 .release = seq_release,
548};
549
Patrick McHardyc7066f72011-01-14 13:36:42 +0100550#ifdef CONFIG_IP_ROUTE_CLASSID
Alexey Dobriyana661c412009-11-25 15:40:35 -0800551static int rt_acct_proc_show(struct seq_file *m, void *v)
Pavel Emelyanov78c686e2007-12-05 21:13:48 -0800552{
Alexey Dobriyana661c412009-11-25 15:40:35 -0800553 struct ip_rt_acct *dst, *src;
554 unsigned int i, j;
Pavel Emelyanov78c686e2007-12-05 21:13:48 -0800555
Alexey Dobriyana661c412009-11-25 15:40:35 -0800556 dst = kcalloc(256, sizeof(struct ip_rt_acct), GFP_KERNEL);
557 if (!dst)
558 return -ENOMEM;
Pavel Emelyanov78c686e2007-12-05 21:13:48 -0800559
Alexey Dobriyana661c412009-11-25 15:40:35 -0800560 for_each_possible_cpu(i) {
561 src = (struct ip_rt_acct *)per_cpu_ptr(ip_rt_acct, i);
562 for (j = 0; j < 256; j++) {
563 dst[j].o_bytes += src[j].o_bytes;
564 dst[j].o_packets += src[j].o_packets;
565 dst[j].i_bytes += src[j].i_bytes;
566 dst[j].i_packets += src[j].i_packets;
Pavel Emelyanov78c686e2007-12-05 21:13:48 -0800567 }
568 }
Alexey Dobriyana661c412009-11-25 15:40:35 -0800569
570 seq_write(m, dst, 256 * sizeof(struct ip_rt_acct));
571 kfree(dst);
572 return 0;
Pavel Emelyanov78c686e2007-12-05 21:13:48 -0800573}
Alexey Dobriyana661c412009-11-25 15:40:35 -0800574
575static int rt_acct_proc_open(struct inode *inode, struct file *file)
576{
577 return single_open(file, rt_acct_proc_show, NULL);
578}
579
580static const struct file_operations rt_acct_proc_fops = {
581 .owner = THIS_MODULE,
582 .open = rt_acct_proc_open,
583 .read = seq_read,
584 .llseek = seq_lseek,
585 .release = single_release,
586};
Pavel Emelyanov78c686e2007-12-05 21:13:48 -0800587#endif
Pavel Emelyanov107f1632007-12-05 21:14:28 -0800588
Denis V. Lunev73b38712008-02-28 20:51:18 -0800589static int __net_init ip_rt_do_proc_init(struct net *net)
Pavel Emelyanov107f1632007-12-05 21:14:28 -0800590{
591 struct proc_dir_entry *pde;
592
593 pde = proc_net_fops_create(net, "rt_cache", S_IRUGO,
594 &rt_cache_seq_fops);
595 if (!pde)
596 goto err1;
597
Wang Chen77020722008-02-28 14:14:25 -0800598 pde = proc_create("rt_cache", S_IRUGO,
599 net->proc_net_stat, &rt_cpu_seq_fops);
Pavel Emelyanov107f1632007-12-05 21:14:28 -0800600 if (!pde)
601 goto err2;
602
Patrick McHardyc7066f72011-01-14 13:36:42 +0100603#ifdef CONFIG_IP_ROUTE_CLASSID
Alexey Dobriyana661c412009-11-25 15:40:35 -0800604 pde = proc_create("rt_acct", 0, net->proc_net, &rt_acct_proc_fops);
Pavel Emelyanov107f1632007-12-05 21:14:28 -0800605 if (!pde)
606 goto err3;
607#endif
608 return 0;
609
Patrick McHardyc7066f72011-01-14 13:36:42 +0100610#ifdef CONFIG_IP_ROUTE_CLASSID
Pavel Emelyanov107f1632007-12-05 21:14:28 -0800611err3:
612 remove_proc_entry("rt_cache", net->proc_net_stat);
613#endif
614err2:
615 remove_proc_entry("rt_cache", net->proc_net);
616err1:
617 return -ENOMEM;
618}
Denis V. Lunev73b38712008-02-28 20:51:18 -0800619
620static void __net_exit ip_rt_do_proc_exit(struct net *net)
621{
622 remove_proc_entry("rt_cache", net->proc_net_stat);
623 remove_proc_entry("rt_cache", net->proc_net);
Patrick McHardyc7066f72011-01-14 13:36:42 +0100624#ifdef CONFIG_IP_ROUTE_CLASSID
Denis V. Lunev73b38712008-02-28 20:51:18 -0800625 remove_proc_entry("rt_acct", net->proc_net);
Alexey Dobriyan0a931ac2010-01-17 03:32:50 +0000626#endif
Denis V. Lunev73b38712008-02-28 20:51:18 -0800627}
628
629static struct pernet_operations ip_rt_proc_ops __net_initdata = {
630 .init = ip_rt_do_proc_init,
631 .exit = ip_rt_do_proc_exit,
632};
633
634static int __init ip_rt_proc_init(void)
635{
636 return register_pernet_subsys(&ip_rt_proc_ops);
637}
638
Pavel Emelyanov107f1632007-12-05 21:14:28 -0800639#else
Denis V. Lunev73b38712008-02-28 20:51:18 -0800640static inline int ip_rt_proc_init(void)
Pavel Emelyanov107f1632007-12-05 21:14:28 -0800641{
642 return 0;
643}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700644#endif /* CONFIG_PROC_FS */
YOSHIFUJI Hideakie905a9e2007-02-09 23:24:47 +0900645
Stephen Hemminger5969f712008-04-10 01:52:09 -0700646static inline void rt_free(struct rtable *rt)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700647{
Changli Gaod8d1f302010-06-10 23:31:35 -0700648 call_rcu_bh(&rt->dst.rcu_head, dst_rcu_free);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700649}
650
Stephen Hemminger5969f712008-04-10 01:52:09 -0700651static inline void rt_drop(struct rtable *rt)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700652{
Linus Torvalds1da177e2005-04-16 15:20:36 -0700653 ip_rt_put(rt);
Changli Gaod8d1f302010-06-10 23:31:35 -0700654 call_rcu_bh(&rt->dst.rcu_head, dst_rcu_free);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700655}
656
Stephen Hemminger5969f712008-04-10 01:52:09 -0700657static inline int rt_fast_clean(struct rtable *rth)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700658{
659 /* Kill broadcast/multicast entries very aggresively, if they
660 collide in hash table with more useful entries */
661 return (rth->rt_flags & (RTCF_BROADCAST | RTCF_MULTICAST)) &&
David S. Millerc7537962010-11-11 17:07:48 -0800662 rt_is_input_route(rth) && rth->dst.rt_next;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700663}
664
Stephen Hemminger5969f712008-04-10 01:52:09 -0700665static inline int rt_valuable(struct rtable *rth)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700666{
667 return (rth->rt_flags & (RTCF_REDIRECTED | RTCF_NOTIFY)) ||
David S. Miller2c8cec52011-02-09 20:42:07 -0800668 (rth->peer && rth->peer->pmtu_expires);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700669}
670
671static int rt_may_expire(struct rtable *rth, unsigned long tmo1, unsigned long tmo2)
672{
673 unsigned long age;
674 int ret = 0;
675
Changli Gaod8d1f302010-06-10 23:31:35 -0700676 if (atomic_read(&rth->dst.__refcnt))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700677 goto out;
678
Changli Gaod8d1f302010-06-10 23:31:35 -0700679 age = jiffies - rth->dst.lastuse;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700680 if ((age <= tmo1 && !rt_fast_clean(rth)) ||
681 (age <= tmo2 && rt_valuable(rth)))
682 goto out;
683 ret = 1;
684out: return ret;
685}
686
687/* Bits of score are:
688 * 31: very valuable
689 * 30: not quite useless
690 * 29..0: usage counter
691 */
692static inline u32 rt_score(struct rtable *rt)
693{
Changli Gaod8d1f302010-06-10 23:31:35 -0700694 u32 score = jiffies - rt->dst.lastuse;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700695
696 score = ~score & ~(3<<30);
697
698 if (rt_valuable(rt))
699 score |= (1<<31);
700
David S. Millerc7537962010-11-11 17:07:48 -0800701 if (rt_is_output_route(rt) ||
Linus Torvalds1da177e2005-04-16 15:20:36 -0700702 !(rt->rt_flags & (RTCF_BROADCAST|RTCF_MULTICAST|RTCF_LOCAL)))
703 score |= (1<<30);
704
705 return score;
706}
707
Neil Horman1080d702008-10-27 12:28:25 -0700708static inline bool rt_caching(const struct net *net)
709{
710 return net->ipv4.current_rt_cache_rebuild_count <=
711 net->ipv4.sysctl_rt_cache_rebuild_count;
712}
713
David S. Miller5e2b61f2011-03-04 21:47:09 -0800714static inline bool compare_hash_inputs(const struct rtable *rt1,
715 const struct rtable *rt2)
Neil Horman1080d702008-10-27 12:28:25 -0700716{
David S. Miller5e2b61f2011-03-04 21:47:09 -0800717 return ((((__force u32)rt1->rt_key_dst ^ (__force u32)rt2->rt_key_dst) |
718 ((__force u32)rt1->rt_key_src ^ (__force u32)rt2->rt_key_src) |
719 (rt1->rt_iif ^ rt2->rt_iif)) == 0);
Neil Horman1080d702008-10-27 12:28:25 -0700720}
721
David S. Miller5e2b61f2011-03-04 21:47:09 -0800722static inline int compare_keys(struct rtable *rt1, struct rtable *rt2)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700723{
David S. Miller5e2b61f2011-03-04 21:47:09 -0800724 return (((__force u32)rt1->rt_key_dst ^ (__force u32)rt2->rt_key_dst) |
725 ((__force u32)rt1->rt_key_src ^ (__force u32)rt2->rt_key_src) |
726 (rt1->rt_mark ^ rt2->rt_mark) |
727 (rt1->rt_tos ^ rt2->rt_tos) |
728 (rt1->rt_oif ^ rt2->rt_oif) |
729 (rt1->rt_iif ^ rt2->rt_iif)) == 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700730}
731
Denis V. Lunevb5921912008-01-22 23:50:25 -0800732static inline int compare_netns(struct rtable *rt1, struct rtable *rt2)
733{
Changli Gaod8d1f302010-06-10 23:31:35 -0700734 return net_eq(dev_net(rt1->dst.dev), dev_net(rt2->dst.dev));
Denis V. Lunevb5921912008-01-22 23:50:25 -0800735}
736
Denis V. Luneve84f84f2008-07-05 19:04:32 -0700737static inline int rt_is_expired(struct rtable *rth)
738{
Changli Gaod8d1f302010-06-10 23:31:35 -0700739 return rth->rt_genid != rt_genid(dev_net(rth->dst.dev));
Denis V. Luneve84f84f2008-07-05 19:04:32 -0700740}
741
Eric Dumazetbeb659b2007-11-19 22:43:37 -0800742/*
743 * Perform a full scan of hash table and free all entries.
744 * Can be called by a softirq or a process.
745 * In the later case, we want to be reschedule if necessary
746 */
David S. Miller6561a3b2010-12-19 21:11:20 -0800747static void rt_do_flush(struct net *net, int process_context)
Eric Dumazetbeb659b2007-11-19 22:43:37 -0800748{
749 unsigned int i;
750 struct rtable *rth, *next;
751
752 for (i = 0; i <= rt_hash_mask; i++) {
David S. Miller6561a3b2010-12-19 21:11:20 -0800753 struct rtable __rcu **pprev;
754 struct rtable *list;
755
Eric Dumazetbeb659b2007-11-19 22:43:37 -0800756 if (process_context && need_resched())
757 cond_resched();
Eric Dumazet1c317202010-10-25 21:02:07 +0000758 rth = rcu_dereference_raw(rt_hash_table[i].chain);
Eric Dumazetbeb659b2007-11-19 22:43:37 -0800759 if (!rth)
760 continue;
761
762 spin_lock_bh(rt_hash_lock_addr(i));
Denis V. Lunev32cb5b42008-07-05 19:06:12 -0700763
David S. Miller6561a3b2010-12-19 21:11:20 -0800764 list = NULL;
765 pprev = &rt_hash_table[i].chain;
766 rth = rcu_dereference_protected(*pprev,
Eric Dumazet1c317202010-10-25 21:02:07 +0000767 lockdep_is_held(rt_hash_lock_addr(i)));
Denis V. Lunev32cb5b42008-07-05 19:06:12 -0700768
David S. Miller6561a3b2010-12-19 21:11:20 -0800769 while (rth) {
770 next = rcu_dereference_protected(rth->dst.rt_next,
771 lockdep_is_held(rt_hash_lock_addr(i)));
Denis V. Lunev32cb5b42008-07-05 19:06:12 -0700772
David S. Miller6561a3b2010-12-19 21:11:20 -0800773 if (!net ||
774 net_eq(dev_net(rth->dst.dev), net)) {
775 rcu_assign_pointer(*pprev, next);
776 rcu_assign_pointer(rth->dst.rt_next, list);
777 list = rth;
Denis V. Lunev32cb5b42008-07-05 19:06:12 -0700778 } else {
David S. Miller6561a3b2010-12-19 21:11:20 -0800779 pprev = &rth->dst.rt_next;
Denis V. Lunev32cb5b42008-07-05 19:06:12 -0700780 }
David S. Miller6561a3b2010-12-19 21:11:20 -0800781 rth = next;
Denis V. Lunev32cb5b42008-07-05 19:06:12 -0700782 }
David S. Miller6561a3b2010-12-19 21:11:20 -0800783
Eric Dumazetbeb659b2007-11-19 22:43:37 -0800784 spin_unlock_bh(rt_hash_lock_addr(i));
785
David S. Miller6561a3b2010-12-19 21:11:20 -0800786 for (; list; list = next) {
787 next = rcu_dereference_protected(list->dst.rt_next, 1);
788 rt_free(list);
Eric Dumazetbeb659b2007-11-19 22:43:37 -0800789 }
790 }
791}
792
Neil Horman1080d702008-10-27 12:28:25 -0700793/*
794 * While freeing expired entries, we compute average chain length
795 * and standard deviation, using fixed-point arithmetic.
796 * This to have an estimation of rt_chain_length_max
797 * rt_chain_length_max = max(elasticity, AVG + 4*SD)
798 * We use 3 bits for frational part, and 29 (or 61) for magnitude.
799 */
800
801#define FRACT_BITS 3
802#define ONE (1UL << FRACT_BITS)
803
Eric Dumazet98376382010-03-08 03:20:00 +0000804/*
805 * Given a hash chain and an item in this hash chain,
806 * find if a previous entry has the same hash_inputs
807 * (but differs on tos, mark or oif)
808 * Returns 0 if an alias is found.
809 * Returns ONE if rth has no alias before itself.
810 */
811static int has_noalias(const struct rtable *head, const struct rtable *rth)
812{
813 const struct rtable *aux = head;
814
815 while (aux != rth) {
David S. Miller5e2b61f2011-03-04 21:47:09 -0800816 if (compare_hash_inputs(aux, rth))
Eric Dumazet98376382010-03-08 03:20:00 +0000817 return 0;
Eric Dumazet1c317202010-10-25 21:02:07 +0000818 aux = rcu_dereference_protected(aux->dst.rt_next, 1);
Eric Dumazet98376382010-03-08 03:20:00 +0000819 }
820 return ONE;
821}
822
Eric Dumazet29e75252008-01-31 17:05:09 -0800823/*
824 * Pertubation of rt_genid by a small quantity [1..256]
825 * Using 8 bits of shuffling ensure we can call rt_cache_invalidate()
826 * many times (2^24) without giving recent rt_genid.
827 * Jenkins hash is strong enough that litle changes of rt_genid are OK.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700828 */
Denis V. Lunev86c657f2008-07-05 19:03:31 -0700829static void rt_cache_invalidate(struct net *net)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700830{
Eric Dumazet29e75252008-01-31 17:05:09 -0800831 unsigned char shuffle;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700832
Eric Dumazet29e75252008-01-31 17:05:09 -0800833 get_random_bytes(&shuffle, sizeof(shuffle));
Denis V. Luneve84f84f2008-07-05 19:04:32 -0700834 atomic_add(shuffle + 1U, &net->ipv4.rt_genid);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700835}
836
Eric Dumazetbeb659b2007-11-19 22:43:37 -0800837/*
Eric Dumazet29e75252008-01-31 17:05:09 -0800838 * delay < 0 : invalidate cache (fast : entries will be deleted later)
839 * delay >= 0 : invalidate & flush cache (can be long)
840 */
Denis V. Lunev76e6ebf2008-07-05 19:00:44 -0700841void rt_cache_flush(struct net *net, int delay)
Eric Dumazet29e75252008-01-31 17:05:09 -0800842{
Denis V. Lunev86c657f2008-07-05 19:03:31 -0700843 rt_cache_invalidate(net);
Eric Dumazet29e75252008-01-31 17:05:09 -0800844 if (delay >= 0)
David S. Miller6561a3b2010-12-19 21:11:20 -0800845 rt_do_flush(net, !in_softirq());
Eric Dumazet29e75252008-01-31 17:05:09 -0800846}
847
Eric W. Biedermana5ee1552009-11-29 15:45:58 +0000848/* Flush previous cache invalidated entries from the cache */
David S. Miller6561a3b2010-12-19 21:11:20 -0800849void rt_cache_flush_batch(struct net *net)
Eric W. Biedermana5ee1552009-11-29 15:45:58 +0000850{
David S. Miller6561a3b2010-12-19 21:11:20 -0800851 rt_do_flush(net, !in_softirq());
Eric W. Biedermana5ee1552009-11-29 15:45:58 +0000852}
853
Neil Horman1080d702008-10-27 12:28:25 -0700854static void rt_emergency_hash_rebuild(struct net *net)
855{
Neil Horman3ee94372010-05-08 01:57:52 -0700856 if (net_ratelimit())
Neil Horman1080d702008-10-27 12:28:25 -0700857 printk(KERN_WARNING "Route hash chain too long!\n");
Neil Horman3ee94372010-05-08 01:57:52 -0700858 rt_cache_invalidate(net);
Neil Horman1080d702008-10-27 12:28:25 -0700859}
860
Linus Torvalds1da177e2005-04-16 15:20:36 -0700861/*
862 Short description of GC goals.
863
864 We want to build algorithm, which will keep routing cache
865 at some equilibrium point, when number of aged off entries
866 is kept approximately equal to newly generated ones.
867
868 Current expiration strength is variable "expire".
869 We try to adjust it dynamically, so that if networking
870 is idle expires is large enough to keep enough of warm entries,
871 and when load increases it reduces to limit cache size.
872 */
873
Daniel Lezcano569d3642008-01-18 03:56:57 -0800874static int rt_garbage_collect(struct dst_ops *ops)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700875{
876 static unsigned long expire = RT_GC_TIMEOUT;
877 static unsigned long last_gc;
878 static int rover;
879 static int equilibrium;
Eric Dumazet1c317202010-10-25 21:02:07 +0000880 struct rtable *rth;
881 struct rtable __rcu **rthp;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700882 unsigned long now = jiffies;
883 int goal;
Eric Dumazetfc66f952010-10-08 06:37:34 +0000884 int entries = dst_entries_get_fast(&ipv4_dst_ops);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700885
886 /*
887 * Garbage collection is pretty expensive,
888 * do not make it too frequently.
889 */
890
891 RT_CACHE_STAT_INC(gc_total);
892
893 if (now - last_gc < ip_rt_gc_min_interval &&
Eric Dumazetfc66f952010-10-08 06:37:34 +0000894 entries < ip_rt_max_size) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700895 RT_CACHE_STAT_INC(gc_ignored);
896 goto out;
897 }
898
Eric Dumazetfc66f952010-10-08 06:37:34 +0000899 entries = dst_entries_get_slow(&ipv4_dst_ops);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700900 /* Calculate number of entries, which we want to expire now. */
Eric Dumazetfc66f952010-10-08 06:37:34 +0000901 goal = entries - (ip_rt_gc_elasticity << rt_hash_log);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700902 if (goal <= 0) {
903 if (equilibrium < ipv4_dst_ops.gc_thresh)
904 equilibrium = ipv4_dst_ops.gc_thresh;
Eric Dumazetfc66f952010-10-08 06:37:34 +0000905 goal = entries - equilibrium;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700906 if (goal > 0) {
Eric Dumazetb790ced2007-12-21 01:49:07 -0800907 equilibrium += min_t(unsigned int, goal >> 1, rt_hash_mask + 1);
Eric Dumazetfc66f952010-10-08 06:37:34 +0000908 goal = entries - equilibrium;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700909 }
910 } else {
911 /* We are in dangerous area. Try to reduce cache really
912 * aggressively.
913 */
Eric Dumazetb790ced2007-12-21 01:49:07 -0800914 goal = max_t(unsigned int, goal >> 1, rt_hash_mask + 1);
Eric Dumazetfc66f952010-10-08 06:37:34 +0000915 equilibrium = entries - goal;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700916 }
917
918 if (now - last_gc >= ip_rt_gc_min_interval)
919 last_gc = now;
920
921 if (goal <= 0) {
922 equilibrium += goal;
923 goto work_done;
924 }
925
926 do {
927 int i, k;
928
929 for (i = rt_hash_mask, k = rover; i >= 0; i--) {
930 unsigned long tmo = expire;
931
932 k = (k + 1) & rt_hash_mask;
933 rthp = &rt_hash_table[k].chain;
Eric Dumazet22c047c2005-07-05 14:55:24 -0700934 spin_lock_bh(rt_hash_lock_addr(k));
Eric Dumazet1c317202010-10-25 21:02:07 +0000935 while ((rth = rcu_dereference_protected(*rthp,
936 lockdep_is_held(rt_hash_lock_addr(k)))) != NULL) {
Denis V. Luneve84f84f2008-07-05 19:04:32 -0700937 if (!rt_is_expired(rth) &&
Eric Dumazet29e75252008-01-31 17:05:09 -0800938 !rt_may_expire(rth, tmo, expire)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700939 tmo >>= 1;
Changli Gaod8d1f302010-06-10 23:31:35 -0700940 rthp = &rth->dst.rt_next;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700941 continue;
942 }
Changli Gaod8d1f302010-06-10 23:31:35 -0700943 *rthp = rth->dst.rt_next;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700944 rt_free(rth);
945 goal--;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700946 }
Eric Dumazet22c047c2005-07-05 14:55:24 -0700947 spin_unlock_bh(rt_hash_lock_addr(k));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700948 if (goal <= 0)
949 break;
950 }
951 rover = k;
952
953 if (goal <= 0)
954 goto work_done;
955
956 /* Goal is not achieved. We stop process if:
957
958 - if expire reduced to zero. Otherwise, expire is halfed.
959 - if table is not full.
960 - if we are called from interrupt.
961 - jiffies check is just fallback/debug loop breaker.
962 We will not spin here for long time in any case.
963 */
964
965 RT_CACHE_STAT_INC(gc_goal_miss);
966
967 if (expire == 0)
968 break;
969
970 expire >>= 1;
971#if RT_CACHE_DEBUG >= 2
972 printk(KERN_DEBUG "expire>> %u %d %d %d\n", expire,
Eric Dumazetfc66f952010-10-08 06:37:34 +0000973 dst_entries_get_fast(&ipv4_dst_ops), goal, i);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700974#endif
975
Eric Dumazetfc66f952010-10-08 06:37:34 +0000976 if (dst_entries_get_fast(&ipv4_dst_ops) < ip_rt_max_size)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700977 goto out;
978 } while (!in_softirq() && time_before_eq(jiffies, now));
979
Eric Dumazetfc66f952010-10-08 06:37:34 +0000980 if (dst_entries_get_fast(&ipv4_dst_ops) < ip_rt_max_size)
981 goto out;
982 if (dst_entries_get_slow(&ipv4_dst_ops) < ip_rt_max_size)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700983 goto out;
984 if (net_ratelimit())
985 printk(KERN_WARNING "dst cache overflow\n");
986 RT_CACHE_STAT_INC(gc_dst_overflow);
987 return 1;
988
989work_done:
990 expire += ip_rt_gc_min_interval;
991 if (expire > ip_rt_gc_timeout ||
Eric Dumazetfc66f952010-10-08 06:37:34 +0000992 dst_entries_get_fast(&ipv4_dst_ops) < ipv4_dst_ops.gc_thresh ||
993 dst_entries_get_slow(&ipv4_dst_ops) < ipv4_dst_ops.gc_thresh)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700994 expire = ip_rt_gc_timeout;
995#if RT_CACHE_DEBUG >= 2
996 printk(KERN_DEBUG "expire++ %u %d %d %d\n", expire,
Eric Dumazetfc66f952010-10-08 06:37:34 +0000997 dst_entries_get_fast(&ipv4_dst_ops), goal, rover);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700998#endif
999out: return 0;
1000}
1001
Eric Dumazet98376382010-03-08 03:20:00 +00001002/*
1003 * Returns number of entries in a hash chain that have different hash_inputs
1004 */
1005static int slow_chain_length(const struct rtable *head)
1006{
1007 int length = 0;
1008 const struct rtable *rth = head;
1009
1010 while (rth) {
1011 length += has_noalias(head, rth);
Eric Dumazet1c317202010-10-25 21:02:07 +00001012 rth = rcu_dereference_protected(rth->dst.rt_next, 1);
Eric Dumazet98376382010-03-08 03:20:00 +00001013 }
1014 return length >> FRACT_BITS;
1015}
1016
David S. Millerb23dd4f2011-03-02 14:31:35 -08001017static struct rtable *rt_intern_hash(unsigned hash, struct rtable *rt,
1018 struct sk_buff *skb, int ifindex)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001019{
Eric Dumazet1c317202010-10-25 21:02:07 +00001020 struct rtable *rth, *cand;
1021 struct rtable __rcu **rthp, **candp;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001022 unsigned long now;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001023 u32 min_score;
1024 int chain_length;
1025 int attempts = !in_softirq();
1026
1027restart:
1028 chain_length = 0;
1029 min_score = ~(u32)0;
1030 cand = NULL;
1031 candp = NULL;
1032 now = jiffies;
1033
Changli Gaod8d1f302010-06-10 23:31:35 -07001034 if (!rt_caching(dev_net(rt->dst.dev))) {
Neil Horman73e42892009-06-20 01:15:16 -07001035 /*
1036 * If we're not caching, just tell the caller we
1037 * were successful and don't touch the route. The
1038 * caller hold the sole reference to the cache entry, and
1039 * it will be released when the caller is done with it.
1040 * If we drop it here, the callers have no way to resolve routes
1041 * when we're not caching. Instead, just point *rp at rt, so
1042 * the caller gets a single use out of the route
Neil Hormanb6280b42009-06-22 10:18:53 +00001043 * Note that we do rt_free on this new route entry, so that
1044 * once its refcount hits zero, we are still able to reap it
1045 * (Thanks Alexey)
Eric Dumazet27b75c92010-10-15 05:44:11 +00001046 * Note: To avoid expensive rcu stuff for this uncached dst,
1047 * we set DST_NOCACHE so that dst_release() can free dst without
1048 * waiting a grace period.
Neil Horman73e42892009-06-20 01:15:16 -07001049 */
Neil Hormanb6280b42009-06-22 10:18:53 +00001050
Eric Dumazetc7d44262010-10-03 22:17:54 -07001051 rt->dst.flags |= DST_NOCACHE;
David S. Millerc7537962010-11-11 17:07:48 -08001052 if (rt->rt_type == RTN_UNICAST || rt_is_output_route(rt)) {
Changli Gaod8d1f302010-06-10 23:31:35 -07001053 int err = arp_bind_neighbour(&rt->dst);
Neil Hormanb6280b42009-06-22 10:18:53 +00001054 if (err) {
1055 if (net_ratelimit())
1056 printk(KERN_WARNING
1057 "Neighbour table failure & not caching routes.\n");
Eric Dumazet27b75c92010-10-15 05:44:11 +00001058 ip_rt_put(rt);
David S. Millerb23dd4f2011-03-02 14:31:35 -08001059 return ERR_PTR(err);
Neil Hormanb6280b42009-06-22 10:18:53 +00001060 }
1061 }
1062
Neil Hormanb6280b42009-06-22 10:18:53 +00001063 goto skip_hashing;
Neil Horman1080d702008-10-27 12:28:25 -07001064 }
1065
Linus Torvalds1da177e2005-04-16 15:20:36 -07001066 rthp = &rt_hash_table[hash].chain;
1067
Eric Dumazet22c047c2005-07-05 14:55:24 -07001068 spin_lock_bh(rt_hash_lock_addr(hash));
Eric Dumazet1c317202010-10-25 21:02:07 +00001069 while ((rth = rcu_dereference_protected(*rthp,
1070 lockdep_is_held(rt_hash_lock_addr(hash)))) != NULL) {
Denis V. Luneve84f84f2008-07-05 19:04:32 -07001071 if (rt_is_expired(rth)) {
Changli Gaod8d1f302010-06-10 23:31:35 -07001072 *rthp = rth->dst.rt_next;
Eric Dumazet29e75252008-01-31 17:05:09 -08001073 rt_free(rth);
1074 continue;
1075 }
David S. Miller5e2b61f2011-03-04 21:47:09 -08001076 if (compare_keys(rth, rt) && compare_netns(rth, rt)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001077 /* Put it first */
Changli Gaod8d1f302010-06-10 23:31:35 -07001078 *rthp = rth->dst.rt_next;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001079 /*
1080 * Since lookup is lockfree, the deletion
1081 * must be visible to another weakly ordered CPU before
1082 * the insertion at the start of the hash chain.
1083 */
Changli Gaod8d1f302010-06-10 23:31:35 -07001084 rcu_assign_pointer(rth->dst.rt_next,
Linus Torvalds1da177e2005-04-16 15:20:36 -07001085 rt_hash_table[hash].chain);
1086 /*
1087 * Since lookup is lockfree, the update writes
1088 * must be ordered for consistency on SMP.
1089 */
1090 rcu_assign_pointer(rt_hash_table[hash].chain, rth);
1091
Changli Gaod8d1f302010-06-10 23:31:35 -07001092 dst_use(&rth->dst, now);
Eric Dumazet22c047c2005-07-05 14:55:24 -07001093 spin_unlock_bh(rt_hash_lock_addr(hash));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001094
1095 rt_drop(rt);
David S. Millerb23dd4f2011-03-02 14:31:35 -08001096 if (skb)
Changli Gaod8d1f302010-06-10 23:31:35 -07001097 skb_dst_set(skb, &rth->dst);
David S. Millerb23dd4f2011-03-02 14:31:35 -08001098 return rth;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001099 }
1100
Changli Gaod8d1f302010-06-10 23:31:35 -07001101 if (!atomic_read(&rth->dst.__refcnt)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001102 u32 score = rt_score(rth);
1103
1104 if (score <= min_score) {
1105 cand = rth;
1106 candp = rthp;
1107 min_score = score;
1108 }
1109 }
1110
1111 chain_length++;
1112
Changli Gaod8d1f302010-06-10 23:31:35 -07001113 rthp = &rth->dst.rt_next;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001114 }
1115
1116 if (cand) {
1117 /* ip_rt_gc_elasticity used to be average length of chain
1118 * length, when exceeded gc becomes really aggressive.
1119 *
1120 * The second limit is less certain. At the moment it allows
1121 * only 2 entries per bucket. We will see.
1122 */
1123 if (chain_length > ip_rt_gc_elasticity) {
Changli Gaod8d1f302010-06-10 23:31:35 -07001124 *candp = cand->dst.rt_next;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001125 rt_free(cand);
1126 }
Neil Horman1080d702008-10-27 12:28:25 -07001127 } else {
Eric Dumazet98376382010-03-08 03:20:00 +00001128 if (chain_length > rt_chain_length_max &&
1129 slow_chain_length(rt_hash_table[hash].chain) > rt_chain_length_max) {
Changli Gaod8d1f302010-06-10 23:31:35 -07001130 struct net *net = dev_net(rt->dst.dev);
Neil Horman1080d702008-10-27 12:28:25 -07001131 int num = ++net->ipv4.current_rt_cache_rebuild_count;
Pavel Emelyanovb35ecb52010-03-24 07:43:17 +00001132 if (!rt_caching(net)) {
Neil Horman1080d702008-10-27 12:28:25 -07001133 printk(KERN_WARNING "%s: %d rebuilds is over limit, route caching disabled\n",
Changli Gaod8d1f302010-06-10 23:31:35 -07001134 rt->dst.dev->name, num);
Neil Horman1080d702008-10-27 12:28:25 -07001135 }
Pavel Emelyanovb35ecb52010-03-24 07:43:17 +00001136 rt_emergency_hash_rebuild(net);
Pavel Emelyanov6a2bad72010-03-24 21:51:22 +00001137 spin_unlock_bh(rt_hash_lock_addr(hash));
1138
David S. Miller5e2b61f2011-03-04 21:47:09 -08001139 hash = rt_hash(rt->rt_key_dst, rt->rt_key_src,
Pavel Emelyanov6a2bad72010-03-24 21:51:22 +00001140 ifindex, rt_genid(net));
1141 goto restart;
Neil Horman1080d702008-10-27 12:28:25 -07001142 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001143 }
1144
1145 /* Try to bind route to arp only if it is output
1146 route or unicast forwarding path.
1147 */
David S. Millerc7537962010-11-11 17:07:48 -08001148 if (rt->rt_type == RTN_UNICAST || rt_is_output_route(rt)) {
Changli Gaod8d1f302010-06-10 23:31:35 -07001149 int err = arp_bind_neighbour(&rt->dst);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001150 if (err) {
Eric Dumazet22c047c2005-07-05 14:55:24 -07001151 spin_unlock_bh(rt_hash_lock_addr(hash));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001152
1153 if (err != -ENOBUFS) {
1154 rt_drop(rt);
David S. Millerb23dd4f2011-03-02 14:31:35 -08001155 return ERR_PTR(err);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001156 }
1157
1158 /* Neighbour tables are full and nothing
1159 can be released. Try to shrink route cache,
1160 it is most likely it holds some neighbour records.
1161 */
1162 if (attempts-- > 0) {
1163 int saved_elasticity = ip_rt_gc_elasticity;
1164 int saved_int = ip_rt_gc_min_interval;
1165 ip_rt_gc_elasticity = 1;
1166 ip_rt_gc_min_interval = 0;
Daniel Lezcano569d3642008-01-18 03:56:57 -08001167 rt_garbage_collect(&ipv4_dst_ops);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001168 ip_rt_gc_min_interval = saved_int;
1169 ip_rt_gc_elasticity = saved_elasticity;
1170 goto restart;
1171 }
1172
1173 if (net_ratelimit())
Ulrich Weber7e1b33e2010-09-27 15:02:18 -07001174 printk(KERN_WARNING "ipv4: Neighbour table overflow.\n");
Linus Torvalds1da177e2005-04-16 15:20:36 -07001175 rt_drop(rt);
David S. Millerb23dd4f2011-03-02 14:31:35 -08001176 return ERR_PTR(-ENOBUFS);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001177 }
1178 }
1179
Changli Gaod8d1f302010-06-10 23:31:35 -07001180 rt->dst.rt_next = rt_hash_table[hash].chain;
Neil Horman1080d702008-10-27 12:28:25 -07001181
Linus Torvalds1da177e2005-04-16 15:20:36 -07001182#if RT_CACHE_DEBUG >= 2
Changli Gaod8d1f302010-06-10 23:31:35 -07001183 if (rt->dst.rt_next) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001184 struct rtable *trt;
Neil Hormanb6280b42009-06-22 10:18:53 +00001185 printk(KERN_DEBUG "rt_cache @%02x: %pI4",
1186 hash, &rt->rt_dst);
Changli Gaod8d1f302010-06-10 23:31:35 -07001187 for (trt = rt->dst.rt_next; trt; trt = trt->dst.rt_next)
Harvey Harrison673d57e2008-10-31 00:53:57 -07001188 printk(" . %pI4", &trt->rt_dst);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001189 printk("\n");
1190 }
1191#endif
Eric Dumazet00269b52008-10-16 14:18:29 -07001192 /*
1193 * Since lookup is lockfree, we must make sure
1194 * previous writes to rt are comitted to memory
1195 * before making rt visible to other CPUS.
1196 */
Eric Dumazet1ddbcb02009-05-19 20:14:28 +00001197 rcu_assign_pointer(rt_hash_table[hash].chain, rt);
Neil Horman1080d702008-10-27 12:28:25 -07001198
Eric Dumazet22c047c2005-07-05 14:55:24 -07001199 spin_unlock_bh(rt_hash_lock_addr(hash));
Neil Horman73e42892009-06-20 01:15:16 -07001200
Neil Hormanb6280b42009-06-22 10:18:53 +00001201skip_hashing:
David S. Millerb23dd4f2011-03-02 14:31:35 -08001202 if (skb)
Changli Gaod8d1f302010-06-10 23:31:35 -07001203 skb_dst_set(skb, &rt->dst);
David S. Millerb23dd4f2011-03-02 14:31:35 -08001204 return rt;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001205}
1206
David S. Miller6431cbc2011-02-07 20:38:06 -08001207static atomic_t __rt_peer_genid = ATOMIC_INIT(0);
1208
1209static u32 rt_peer_genid(void)
1210{
1211 return atomic_read(&__rt_peer_genid);
1212}
1213
Linus Torvalds1da177e2005-04-16 15:20:36 -07001214void rt_bind_peer(struct rtable *rt, int create)
1215{
Linus Torvalds1da177e2005-04-16 15:20:36 -07001216 struct inet_peer *peer;
1217
David S. Millerb534ecf2010-11-30 11:54:19 -08001218 peer = inet_getpeer_v4(rt->rt_dst, create);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001219
Eric Dumazet49e8ab02010-08-19 06:10:45 +00001220 if (peer && cmpxchg(&rt->peer, NULL, peer) != NULL)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001221 inet_putpeer(peer);
David S. Miller6431cbc2011-02-07 20:38:06 -08001222 else
1223 rt->rt_peer_genid = rt_peer_genid();
Linus Torvalds1da177e2005-04-16 15:20:36 -07001224}
1225
1226/*
1227 * Peer allocation may fail only in serious out-of-memory conditions. However
1228 * we still can generate some output.
1229 * Random ID selection looks a bit dangerous because we have no chances to
1230 * select ID being unique in a reasonable period of time.
1231 * But broken packet identifier may be better than no packet at all.
1232 */
1233static void ip_select_fb_ident(struct iphdr *iph)
1234{
1235 static DEFINE_SPINLOCK(ip_fb_id_lock);
1236 static u32 ip_fallback_id;
1237 u32 salt;
1238
1239 spin_lock_bh(&ip_fb_id_lock);
Al Viroe4485152006-09-26 22:15:01 -07001240 salt = secure_ip_id((__force __be32)ip_fallback_id ^ iph->daddr);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001241 iph->id = htons(salt & 0xFFFF);
1242 ip_fallback_id = salt;
1243 spin_unlock_bh(&ip_fb_id_lock);
1244}
1245
1246void __ip_select_ident(struct iphdr *iph, struct dst_entry *dst, int more)
1247{
1248 struct rtable *rt = (struct rtable *) dst;
1249
1250 if (rt) {
1251 if (rt->peer == NULL)
1252 rt_bind_peer(rt, 1);
1253
1254 /* If peer is attached to destination, it is never detached,
1255 so that we need not to grab a lock to dereference it.
1256 */
1257 if (rt->peer) {
1258 iph->id = htons(inet_getid(rt->peer, more));
1259 return;
1260 }
1261 } else
YOSHIFUJI Hideakie905a9e2007-02-09 23:24:47 +09001262 printk(KERN_DEBUG "rt_bind_peer(0) @%p\n",
Stephen Hemminger9c2b3322005-04-19 22:39:42 -07001263 __builtin_return_address(0));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001264
1265 ip_select_fb_ident(iph);
1266}
Eric Dumazet4bc2f182010-07-09 21:22:10 +00001267EXPORT_SYMBOL(__ip_select_ident);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001268
1269static void rt_del(unsigned hash, struct rtable *rt)
1270{
Eric Dumazet1c317202010-10-25 21:02:07 +00001271 struct rtable __rcu **rthp;
1272 struct rtable *aux;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001273
Eric Dumazet29e75252008-01-31 17:05:09 -08001274 rthp = &rt_hash_table[hash].chain;
Eric Dumazet22c047c2005-07-05 14:55:24 -07001275 spin_lock_bh(rt_hash_lock_addr(hash));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001276 ip_rt_put(rt);
Eric Dumazet1c317202010-10-25 21:02:07 +00001277 while ((aux = rcu_dereference_protected(*rthp,
1278 lockdep_is_held(rt_hash_lock_addr(hash)))) != NULL) {
Denis V. Luneve84f84f2008-07-05 19:04:32 -07001279 if (aux == rt || rt_is_expired(aux)) {
Changli Gaod8d1f302010-06-10 23:31:35 -07001280 *rthp = aux->dst.rt_next;
Eric Dumazet29e75252008-01-31 17:05:09 -08001281 rt_free(aux);
1282 continue;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001283 }
Changli Gaod8d1f302010-06-10 23:31:35 -07001284 rthp = &aux->dst.rt_next;
Eric Dumazet29e75252008-01-31 17:05:09 -08001285 }
Eric Dumazet22c047c2005-07-05 14:55:24 -07001286 spin_unlock_bh(rt_hash_lock_addr(hash));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001287}
1288
Eric Dumazeted7865a42010-06-07 21:49:44 -07001289/* called in rcu_read_lock() section */
Al Virof7655222006-09-26 21:25:43 -07001290void ip_rt_redirect(__be32 old_gw, __be32 daddr, __be32 new_gw,
1291 __be32 saddr, struct net_device *dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001292{
Eric Dumazeted7865a42010-06-07 21:49:44 -07001293 struct in_device *in_dev = __in_dev_get_rcu(dev);
David S. Millerf39925d2011-02-09 22:00:16 -08001294 struct inet_peer *peer;
Denis V. Lunev317805b2008-02-28 20:50:06 -08001295 struct net *net;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001296
Linus Torvalds1da177e2005-04-16 15:20:36 -07001297 if (!in_dev)
1298 return;
1299
YOSHIFUJI Hideakic346dca2008-03-25 21:47:49 +09001300 net = dev_net(dev);
Joe Perches9d4fb272009-11-23 10:41:23 -08001301 if (new_gw == old_gw || !IN_DEV_RX_REDIRECTS(in_dev) ||
1302 ipv4_is_multicast(new_gw) || ipv4_is_lbcast(new_gw) ||
1303 ipv4_is_zeronet(new_gw))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001304 goto reject_redirect;
1305
1306 if (!IN_DEV_SHARED_MEDIA(in_dev)) {
1307 if (!inet_addr_onlink(in_dev, new_gw, old_gw))
1308 goto reject_redirect;
1309 if (IN_DEV_SEC_REDIRECTS(in_dev) && ip_fib_check_default(new_gw, dev))
1310 goto reject_redirect;
1311 } else {
Denis V. Lunev317805b2008-02-28 20:50:06 -08001312 if (inet_addr_type(net, new_gw) != RTN_UNICAST)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001313 goto reject_redirect;
1314 }
1315
David S. Millerf39925d2011-02-09 22:00:16 -08001316 peer = inet_getpeer_v4(daddr, 1);
1317 if (peer) {
1318 peer->redirect_learned.a4 = new_gw;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001319
David S. Millerf39925d2011-02-09 22:00:16 -08001320 inet_putpeer(peer);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001321
David S. Millerf39925d2011-02-09 22:00:16 -08001322 atomic_inc(&__rt_peer_genid);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001323 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001324 return;
1325
1326reject_redirect:
1327#ifdef CONFIG_IP_ROUTE_VERBOSE
1328 if (IN_DEV_LOG_MARTIANS(in_dev) && net_ratelimit())
Harvey Harrison673d57e2008-10-31 00:53:57 -07001329 printk(KERN_INFO "Redirect from %pI4 on %s about %pI4 ignored.\n"
1330 " Advised path = %pI4 -> %pI4\n",
1331 &old_gw, dev->name, &new_gw,
1332 &saddr, &daddr);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001333#endif
Eric Dumazeted7865a42010-06-07 21:49:44 -07001334 ;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001335}
1336
1337static struct dst_entry *ipv4_negative_advice(struct dst_entry *dst)
1338{
Eric Dumazetee6b9672008-03-05 18:30:47 -08001339 struct rtable *rt = (struct rtable *)dst;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001340 struct dst_entry *ret = dst;
1341
1342 if (rt) {
Timo Teräsd11a4dc2010-03-18 23:20:20 +00001343 if (dst->obsolete > 0) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001344 ip_rt_put(rt);
1345 ret = NULL;
David S. Miller2c8cec52011-02-09 20:42:07 -08001346 } else if (rt->rt_flags & RTCF_REDIRECTED) {
David S. Miller5e2b61f2011-03-04 21:47:09 -08001347 unsigned hash = rt_hash(rt->rt_key_dst, rt->rt_key_src,
1348 rt->rt_oif,
Denis V. Luneve84f84f2008-07-05 19:04:32 -07001349 rt_genid(dev_net(dst->dev)));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001350#if RT_CACHE_DEBUG >= 1
Harvey Harrison673d57e2008-10-31 00:53:57 -07001351 printk(KERN_DEBUG "ipv4_negative_advice: redirect to %pI4/%02x dropped\n",
David S. Miller5e2b61f2011-03-04 21:47:09 -08001352 &rt->rt_dst, rt->rt_tos);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001353#endif
1354 rt_del(hash, rt);
1355 ret = NULL;
David S. Miller2c8cec52011-02-09 20:42:07 -08001356 } else if (rt->peer &&
1357 rt->peer->pmtu_expires &&
1358 time_after_eq(jiffies, rt->peer->pmtu_expires)) {
1359 unsigned long orig = rt->peer->pmtu_expires;
1360
1361 if (cmpxchg(&rt->peer->pmtu_expires, orig, 0) == orig)
1362 dst_metric_set(dst, RTAX_MTU,
1363 rt->peer->pmtu_orig);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001364 }
1365 }
1366 return ret;
1367}
1368
1369/*
1370 * Algorithm:
1371 * 1. The first ip_rt_redirect_number redirects are sent
1372 * with exponential backoff, then we stop sending them at all,
1373 * assuming that the host ignores our redirects.
1374 * 2. If we did not see packets requiring redirects
1375 * during ip_rt_redirect_silence, we assume that the host
1376 * forgot redirected route and start to send redirects again.
1377 *
1378 * This algorithm is much cheaper and more intelligent than dumb load limiting
1379 * in icmp.c.
1380 *
1381 * NOTE. Do not forget to inhibit load limiting for redirects (redundant)
1382 * and "frag. need" (breaks PMTU discovery) in icmp.c.
1383 */
1384
1385void ip_rt_send_redirect(struct sk_buff *skb)
1386{
Eric Dumazet511c3f92009-06-02 05:14:27 +00001387 struct rtable *rt = skb_rtable(skb);
Eric Dumazet30038fc2009-08-28 23:52:01 -07001388 struct in_device *in_dev;
David S. Miller92d86822011-02-04 15:55:25 -08001389 struct inet_peer *peer;
Eric Dumazet30038fc2009-08-28 23:52:01 -07001390 int log_martians;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001391
Eric Dumazet30038fc2009-08-28 23:52:01 -07001392 rcu_read_lock();
Changli Gaod8d1f302010-06-10 23:31:35 -07001393 in_dev = __in_dev_get_rcu(rt->dst.dev);
Eric Dumazet30038fc2009-08-28 23:52:01 -07001394 if (!in_dev || !IN_DEV_TX_REDIRECTS(in_dev)) {
1395 rcu_read_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07001396 return;
Eric Dumazet30038fc2009-08-28 23:52:01 -07001397 }
1398 log_martians = IN_DEV_LOG_MARTIANS(in_dev);
1399 rcu_read_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07001400
David S. Miller92d86822011-02-04 15:55:25 -08001401 if (!rt->peer)
1402 rt_bind_peer(rt, 1);
1403 peer = rt->peer;
1404 if (!peer) {
1405 icmp_send(skb, ICMP_REDIRECT, ICMP_REDIR_HOST, rt->rt_gateway);
1406 return;
1407 }
1408
Linus Torvalds1da177e2005-04-16 15:20:36 -07001409 /* No redirected packets during ip_rt_redirect_silence;
1410 * reset the algorithm.
1411 */
David S. Miller92d86822011-02-04 15:55:25 -08001412 if (time_after(jiffies, peer->rate_last + ip_rt_redirect_silence))
1413 peer->rate_tokens = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001414
1415 /* Too many ignored redirects; do not send anything
Changli Gaod8d1f302010-06-10 23:31:35 -07001416 * set dst.rate_last to the last seen redirected packet.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001417 */
David S. Miller92d86822011-02-04 15:55:25 -08001418 if (peer->rate_tokens >= ip_rt_redirect_number) {
1419 peer->rate_last = jiffies;
Eric Dumazet30038fc2009-08-28 23:52:01 -07001420 return;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001421 }
1422
1423 /* Check for load limit; set rate_last to the latest sent
1424 * redirect.
1425 */
David S. Miller92d86822011-02-04 15:55:25 -08001426 if (peer->rate_tokens == 0 ||
Li Yewang14fb8a72006-12-18 00:26:35 -08001427 time_after(jiffies,
David S. Miller92d86822011-02-04 15:55:25 -08001428 (peer->rate_last +
1429 (ip_rt_redirect_load << peer->rate_tokens)))) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001430 icmp_send(skb, ICMP_REDIRECT, ICMP_REDIR_HOST, rt->rt_gateway);
David S. Miller92d86822011-02-04 15:55:25 -08001431 peer->rate_last = jiffies;
1432 ++peer->rate_tokens;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001433#ifdef CONFIG_IP_ROUTE_VERBOSE
Eric Dumazet30038fc2009-08-28 23:52:01 -07001434 if (log_martians &&
David S. Miller92d86822011-02-04 15:55:25 -08001435 peer->rate_tokens == ip_rt_redirect_number &&
Linus Torvalds1da177e2005-04-16 15:20:36 -07001436 net_ratelimit())
Harvey Harrison673d57e2008-10-31 00:53:57 -07001437 printk(KERN_WARNING "host %pI4/if%d ignores redirects for %pI4 to %pI4.\n",
1438 &rt->rt_src, rt->rt_iif,
1439 &rt->rt_dst, &rt->rt_gateway);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001440#endif
1441 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001442}
1443
1444static int ip_error(struct sk_buff *skb)
1445{
Eric Dumazet511c3f92009-06-02 05:14:27 +00001446 struct rtable *rt = skb_rtable(skb);
David S. Miller92d86822011-02-04 15:55:25 -08001447 struct inet_peer *peer;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001448 unsigned long now;
David S. Miller92d86822011-02-04 15:55:25 -08001449 bool send;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001450 int code;
1451
Changli Gaod8d1f302010-06-10 23:31:35 -07001452 switch (rt->dst.error) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001453 case EINVAL:
1454 default:
1455 goto out;
1456 case EHOSTUNREACH:
1457 code = ICMP_HOST_UNREACH;
1458 break;
1459 case ENETUNREACH:
1460 code = ICMP_NET_UNREACH;
Changli Gaod8d1f302010-06-10 23:31:35 -07001461 IP_INC_STATS_BH(dev_net(rt->dst.dev),
Pavel Emelyanov7c73a6f2008-07-16 20:20:11 -07001462 IPSTATS_MIB_INNOROUTES);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001463 break;
1464 case EACCES:
1465 code = ICMP_PKT_FILTERED;
1466 break;
1467 }
1468
David S. Miller92d86822011-02-04 15:55:25 -08001469 if (!rt->peer)
1470 rt_bind_peer(rt, 1);
1471 peer = rt->peer;
1472
1473 send = true;
1474 if (peer) {
1475 now = jiffies;
1476 peer->rate_tokens += now - peer->rate_last;
1477 if (peer->rate_tokens > ip_rt_error_burst)
1478 peer->rate_tokens = ip_rt_error_burst;
1479 peer->rate_last = now;
1480 if (peer->rate_tokens >= ip_rt_error_cost)
1481 peer->rate_tokens -= ip_rt_error_cost;
1482 else
1483 send = false;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001484 }
David S. Miller92d86822011-02-04 15:55:25 -08001485 if (send)
1486 icmp_send(skb, ICMP_DEST_UNREACH, code, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001487
1488out: kfree_skb(skb);
1489 return 0;
YOSHIFUJI Hideakie905a9e2007-02-09 23:24:47 +09001490}
Linus Torvalds1da177e2005-04-16 15:20:36 -07001491
1492/*
1493 * The last two values are not from the RFC but
1494 * are needed for AMPRnet AX.25 paths.
1495 */
1496
Arjan van de Ven9b5b5cf2005-11-29 16:21:38 -08001497static const unsigned short mtu_plateau[] =
Linus Torvalds1da177e2005-04-16 15:20:36 -07001498{32000, 17914, 8166, 4352, 2002, 1492, 576, 296, 216, 128 };
1499
Stephen Hemminger5969f712008-04-10 01:52:09 -07001500static inline unsigned short guess_mtu(unsigned short old_mtu)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001501{
1502 int i;
YOSHIFUJI Hideakie905a9e2007-02-09 23:24:47 +09001503
Linus Torvalds1da177e2005-04-16 15:20:36 -07001504 for (i = 0; i < ARRAY_SIZE(mtu_plateau); i++)
1505 if (old_mtu > mtu_plateau[i])
1506 return mtu_plateau[i];
1507 return 68;
1508}
1509
Denis V. Lunevb5921912008-01-22 23:50:25 -08001510unsigned short ip_rt_frag_needed(struct net *net, struct iphdr *iph,
Timo Teras0010e462008-04-29 03:32:25 -07001511 unsigned short new_mtu,
1512 struct net_device *dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001513{
Linus Torvalds1da177e2005-04-16 15:20:36 -07001514 unsigned short old_mtu = ntohs(iph->tot_len);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001515 unsigned short est_mtu = 0;
David S. Miller2c8cec52011-02-09 20:42:07 -08001516 struct inet_peer *peer;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001517
David S. Miller2c8cec52011-02-09 20:42:07 -08001518 peer = inet_getpeer_v4(iph->daddr, 1);
1519 if (peer) {
1520 unsigned short mtu = new_mtu;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001521
David S. Miller2c8cec52011-02-09 20:42:07 -08001522 if (new_mtu < 68 || new_mtu >= old_mtu) {
1523 /* BSD 4.2 derived systems incorrectly adjust
1524 * tot_len by the IP header length, and report
1525 * a zero MTU in the ICMP message.
1526 */
1527 if (mtu == 0 &&
1528 old_mtu >= 68 + (iph->ihl << 2))
1529 old_mtu -= iph->ihl << 2;
1530 mtu = guess_mtu(old_mtu);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001531 }
David S. Miller2c8cec52011-02-09 20:42:07 -08001532
1533 if (mtu < ip_rt_min_pmtu)
1534 mtu = ip_rt_min_pmtu;
1535 if (!peer->pmtu_expires || mtu < peer->pmtu_learned) {
Hiroaki SHIMODA46af3182011-03-09 20:09:58 +00001536 unsigned long pmtu_expires;
1537
1538 pmtu_expires = jiffies + ip_rt_mtu_expires;
1539 if (!pmtu_expires)
1540 pmtu_expires = 1UL;
1541
David S. Miller2c8cec52011-02-09 20:42:07 -08001542 est_mtu = mtu;
1543 peer->pmtu_learned = mtu;
Hiroaki SHIMODA46af3182011-03-09 20:09:58 +00001544 peer->pmtu_expires = pmtu_expires;
David S. Miller2c8cec52011-02-09 20:42:07 -08001545 }
1546
1547 inet_putpeer(peer);
1548
1549 atomic_inc(&__rt_peer_genid);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001550 }
1551 return est_mtu ? : new_mtu;
1552}
1553
David S. Miller2c8cec52011-02-09 20:42:07 -08001554static void check_peer_pmtu(struct dst_entry *dst, struct inet_peer *peer)
1555{
1556 unsigned long expires = peer->pmtu_expires;
1557
Hiroaki SHIMODA46af3182011-03-09 20:09:58 +00001558 if (time_before(jiffies, expires)) {
David S. Miller2c8cec52011-02-09 20:42:07 -08001559 u32 orig_dst_mtu = dst_mtu(dst);
1560 if (peer->pmtu_learned < orig_dst_mtu) {
1561 if (!peer->pmtu_orig)
1562 peer->pmtu_orig = dst_metric_raw(dst, RTAX_MTU);
1563 dst_metric_set(dst, RTAX_MTU, peer->pmtu_learned);
1564 }
1565 } else if (cmpxchg(&peer->pmtu_expires, expires, 0) == expires)
1566 dst_metric_set(dst, RTAX_MTU, peer->pmtu_orig);
1567}
1568
Linus Torvalds1da177e2005-04-16 15:20:36 -07001569static void ip_rt_update_pmtu(struct dst_entry *dst, u32 mtu)
1570{
David S. Miller2c8cec52011-02-09 20:42:07 -08001571 struct rtable *rt = (struct rtable *) dst;
1572 struct inet_peer *peer;
1573
1574 dst_confirm(dst);
1575
1576 if (!rt->peer)
1577 rt_bind_peer(rt, 1);
1578 peer = rt->peer;
1579 if (peer) {
1580 if (mtu < ip_rt_min_pmtu)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001581 mtu = ip_rt_min_pmtu;
David S. Miller2c8cec52011-02-09 20:42:07 -08001582 if (!peer->pmtu_expires || mtu < peer->pmtu_learned) {
Hiroaki SHIMODA46af3182011-03-09 20:09:58 +00001583 unsigned long pmtu_expires;
1584
1585 pmtu_expires = jiffies + ip_rt_mtu_expires;
1586 if (!pmtu_expires)
1587 pmtu_expires = 1UL;
1588
David S. Miller2c8cec52011-02-09 20:42:07 -08001589 peer->pmtu_learned = mtu;
Hiroaki SHIMODA46af3182011-03-09 20:09:58 +00001590 peer->pmtu_expires = pmtu_expires;
David S. Miller2c8cec52011-02-09 20:42:07 -08001591
1592 atomic_inc(&__rt_peer_genid);
1593 rt->rt_peer_genid = rt_peer_genid();
Linus Torvalds1da177e2005-04-16 15:20:36 -07001594 }
Hiroaki SHIMODA46af3182011-03-09 20:09:58 +00001595 check_peer_pmtu(dst, peer);
1596
David S. Miller2c8cec52011-02-09 20:42:07 -08001597 inet_putpeer(peer);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001598 }
1599}
1600
David S. Millerf39925d2011-02-09 22:00:16 -08001601static int check_peer_redir(struct dst_entry *dst, struct inet_peer *peer)
1602{
1603 struct rtable *rt = (struct rtable *) dst;
1604 __be32 orig_gw = rt->rt_gateway;
1605
1606 dst_confirm(&rt->dst);
1607
1608 neigh_release(rt->dst.neighbour);
1609 rt->dst.neighbour = NULL;
1610
1611 rt->rt_gateway = peer->redirect_learned.a4;
1612 if (arp_bind_neighbour(&rt->dst) ||
1613 !(rt->dst.neighbour->nud_state & NUD_VALID)) {
1614 if (rt->dst.neighbour)
1615 neigh_event_send(rt->dst.neighbour, NULL);
1616 rt->rt_gateway = orig_gw;
1617 return -EAGAIN;
1618 } else {
1619 rt->rt_flags |= RTCF_REDIRECTED;
1620 call_netevent_notifiers(NETEVENT_NEIGH_UPDATE,
1621 rt->dst.neighbour);
1622 }
1623 return 0;
1624}
1625
Linus Torvalds1da177e2005-04-16 15:20:36 -07001626static struct dst_entry *ipv4_dst_check(struct dst_entry *dst, u32 cookie)
1627{
David S. Miller6431cbc2011-02-07 20:38:06 -08001628 struct rtable *rt = (struct rtable *) dst;
1629
1630 if (rt_is_expired(rt))
Timo Teräsd11a4dc2010-03-18 23:20:20 +00001631 return NULL;
David S. Miller6431cbc2011-02-07 20:38:06 -08001632 if (rt->rt_peer_genid != rt_peer_genid()) {
David S. Miller2c8cec52011-02-09 20:42:07 -08001633 struct inet_peer *peer;
1634
David S. Miller6431cbc2011-02-07 20:38:06 -08001635 if (!rt->peer)
1636 rt_bind_peer(rt, 0);
1637
David S. Miller2c8cec52011-02-09 20:42:07 -08001638 peer = rt->peer;
1639 if (peer && peer->pmtu_expires)
1640 check_peer_pmtu(dst, peer);
1641
David S. Millerf39925d2011-02-09 22:00:16 -08001642 if (peer && peer->redirect_learned.a4 &&
1643 peer->redirect_learned.a4 != rt->rt_gateway) {
1644 if (check_peer_redir(dst, peer))
1645 return NULL;
1646 }
1647
David S. Miller6431cbc2011-02-07 20:38:06 -08001648 rt->rt_peer_genid = rt_peer_genid();
1649 }
Timo Teräsd11a4dc2010-03-18 23:20:20 +00001650 return dst;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001651}
1652
1653static void ipv4_dst_destroy(struct dst_entry *dst)
1654{
1655 struct rtable *rt = (struct rtable *) dst;
1656 struct inet_peer *peer = rt->peer;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001657
David S. Miller62fa8a82011-01-26 20:51:05 -08001658 if (rt->fi) {
1659 fib_info_put(rt->fi);
1660 rt->fi = NULL;
1661 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001662 if (peer) {
1663 rt->peer = NULL;
1664 inet_putpeer(peer);
1665 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001666}
1667
Linus Torvalds1da177e2005-04-16 15:20:36 -07001668
1669static void ipv4_link_failure(struct sk_buff *skb)
1670{
1671 struct rtable *rt;
1672
1673 icmp_send(skb, ICMP_DEST_UNREACH, ICMP_HOST_UNREACH, 0);
1674
Eric Dumazet511c3f92009-06-02 05:14:27 +00001675 rt = skb_rtable(skb);
David S. Miller2c8cec52011-02-09 20:42:07 -08001676 if (rt &&
1677 rt->peer &&
1678 rt->peer->pmtu_expires) {
1679 unsigned long orig = rt->peer->pmtu_expires;
1680
1681 if (cmpxchg(&rt->peer->pmtu_expires, orig, 0) == orig)
1682 dst_metric_set(&rt->dst, RTAX_MTU, rt->peer->pmtu_orig);
1683 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001684}
1685
1686static int ip_rt_bug(struct sk_buff *skb)
1687{
Harvey Harrison673d57e2008-10-31 00:53:57 -07001688 printk(KERN_DEBUG "ip_rt_bug: %pI4 -> %pI4, %s\n",
1689 &ip_hdr(skb)->saddr, &ip_hdr(skb)->daddr,
Linus Torvalds1da177e2005-04-16 15:20:36 -07001690 skb->dev ? skb->dev->name : "?");
1691 kfree_skb(skb);
1692 return 0;
1693}
1694
1695/*
1696 We do not cache source address of outgoing interface,
1697 because it is used only by IP RR, TS and SRR options,
1698 so that it out of fast path.
1699
1700 BTW remember: "addr" is allowed to be not aligned
1701 in IP options!
1702 */
1703
1704void ip_rt_get_source(u8 *addr, struct rtable *rt)
1705{
Al Viroa61ced52006-09-26 21:27:54 -07001706 __be32 src;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001707 struct fib_result res;
1708
David S. Millerc7537962010-11-11 17:07:48 -08001709 if (rt_is_output_route(rt))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001710 src = rt->rt_src;
Eric Dumazetebc0ffa2010-10-05 10:41:36 +00001711 else {
David S. Miller68a5e3d2011-03-11 20:07:33 -05001712 struct flowi4 fl4 = {
1713 .daddr = rt->rt_key_dst,
1714 .saddr = rt->rt_key_src,
1715 .flowi4_tos = rt->rt_tos,
1716 .flowi4_oif = rt->rt_oif,
1717 .flowi4_iif = rt->rt_iif,
1718 .flowi4_mark = rt->rt_mark,
David S. Miller5e2b61f2011-03-04 21:47:09 -08001719 };
1720
Eric Dumazetebc0ffa2010-10-05 10:41:36 +00001721 rcu_read_lock();
David S. Miller68a5e3d2011-03-11 20:07:33 -05001722 if (fib_lookup(dev_net(rt->dst.dev), &fl4, &res) == 0)
Eric Dumazetebc0ffa2010-10-05 10:41:36 +00001723 src = FIB_RES_PREFSRC(res);
1724 else
1725 src = inet_select_addr(rt->dst.dev, rt->rt_gateway,
Linus Torvalds1da177e2005-04-16 15:20:36 -07001726 RT_SCOPE_UNIVERSE);
Eric Dumazetebc0ffa2010-10-05 10:41:36 +00001727 rcu_read_unlock();
1728 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001729 memcpy(addr, &src, 4);
1730}
1731
Patrick McHardyc7066f72011-01-14 13:36:42 +01001732#ifdef CONFIG_IP_ROUTE_CLASSID
Linus Torvalds1da177e2005-04-16 15:20:36 -07001733static void set_class_tag(struct rtable *rt, u32 tag)
1734{
Changli Gaod8d1f302010-06-10 23:31:35 -07001735 if (!(rt->dst.tclassid & 0xFFFF))
1736 rt->dst.tclassid |= tag & 0xFFFF;
1737 if (!(rt->dst.tclassid & 0xFFFF0000))
1738 rt->dst.tclassid |= tag & 0xFFFF0000;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001739}
1740#endif
1741
David S. Miller0dbaee32010-12-13 12:52:14 -08001742static unsigned int ipv4_default_advmss(const struct dst_entry *dst)
1743{
1744 unsigned int advmss = dst_metric_raw(dst, RTAX_ADVMSS);
1745
1746 if (advmss == 0) {
1747 advmss = max_t(unsigned int, dst->dev->mtu - 40,
1748 ip_rt_min_advmss);
1749 if (advmss > 65535 - 40)
1750 advmss = 65535 - 40;
1751 }
1752 return advmss;
1753}
1754
David S. Millerd33e4552010-12-14 13:01:14 -08001755static unsigned int ipv4_default_mtu(const struct dst_entry *dst)
1756{
1757 unsigned int mtu = dst->dev->mtu;
1758
1759 if (unlikely(dst_metric_locked(dst, RTAX_MTU))) {
1760 const struct rtable *rt = (const struct rtable *) dst;
1761
1762 if (rt->rt_gateway != rt->rt_dst && mtu > 576)
1763 mtu = 576;
1764 }
1765
1766 if (mtu > IP_MAX_MTU)
1767 mtu = IP_MAX_MTU;
1768
1769 return mtu;
1770}
1771
David S. Miller68a5e3d2011-03-11 20:07:33 -05001772static void rt_init_metrics(struct rtable *rt, const struct flowi4 *oldflp4,
David S. Miller5e2b61f2011-03-04 21:47:09 -08001773 struct fib_info *fi)
David S. Millera4daad62011-01-27 22:01:53 -08001774{
David S. Miller0131ba42011-02-04 14:37:30 -08001775 struct inet_peer *peer;
1776 int create = 0;
1777
1778 /* If a peer entry exists for this destination, we must hook
1779 * it up in order to get at cached metrics.
1780 */
David S. Miller68a5e3d2011-03-11 20:07:33 -05001781 if (oldflp4 && (oldflp4->flowi4_flags & FLOWI_FLAG_PRECOW_METRICS))
David S. Miller0131ba42011-02-04 14:37:30 -08001782 create = 1;
1783
David S. Miller3c0afdc2011-03-04 21:26:07 -08001784 rt->peer = peer = inet_getpeer_v4(rt->rt_dst, create);
David S. Miller0131ba42011-02-04 14:37:30 -08001785 if (peer) {
David S. Miller3c0afdc2011-03-04 21:26:07 -08001786 rt->rt_peer_genid = rt_peer_genid();
David S. Miller0131ba42011-02-04 14:37:30 -08001787 if (inet_metrics_new(peer))
1788 memcpy(peer->metrics, fi->fib_metrics,
1789 sizeof(u32) * RTAX_MAX);
1790 dst_init_metrics(&rt->dst, peer->metrics, false);
David S. Miller2c8cec52011-02-09 20:42:07 -08001791
1792 if (peer->pmtu_expires)
1793 check_peer_pmtu(&rt->dst, peer);
David S. Millerf39925d2011-02-09 22:00:16 -08001794 if (peer->redirect_learned.a4 &&
1795 peer->redirect_learned.a4 != rt->rt_gateway) {
1796 rt->rt_gateway = peer->redirect_learned.a4;
1797 rt->rt_flags |= RTCF_REDIRECTED;
1798 }
David S. Miller0131ba42011-02-04 14:37:30 -08001799 } else {
David S. Millerb8dad612011-01-28 14:07:16 -08001800 if (fi->fib_metrics != (u32 *) dst_default_metrics) {
1801 rt->fi = fi;
1802 atomic_inc(&fi->fib_clntref);
1803 }
David S. Millera4daad62011-01-27 22:01:53 -08001804 dst_init_metrics(&rt->dst, fi->fib_metrics, true);
David S. Millera4daad62011-01-27 22:01:53 -08001805 }
1806}
1807
David S. Miller68a5e3d2011-03-11 20:07:33 -05001808static void rt_set_nexthop(struct rtable *rt, const struct flowi4 *oldflp4,
David S. Miller5e2b61f2011-03-04 21:47:09 -08001809 const struct fib_result *res,
David S. Miller982721f2011-02-16 21:44:24 -08001810 struct fib_info *fi, u16 type, u32 itag)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001811{
David S. Millerdefb3512010-12-08 21:16:57 -08001812 struct dst_entry *dst = &rt->dst;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001813
1814 if (fi) {
1815 if (FIB_RES_GW(*res) &&
1816 FIB_RES_NH(*res).nh_scope == RT_SCOPE_LINK)
1817 rt->rt_gateway = FIB_RES_GW(*res);
David S. Miller68a5e3d2011-03-11 20:07:33 -05001818 rt_init_metrics(rt, oldflp4, fi);
Patrick McHardyc7066f72011-01-14 13:36:42 +01001819#ifdef CONFIG_IP_ROUTE_CLASSID
David S. Millerdefb3512010-12-08 21:16:57 -08001820 dst->tclassid = FIB_RES_NH(*res).nh_tclassid;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001821#endif
David S. Millerd33e4552010-12-14 13:01:14 -08001822 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001823
David S. Millerdefb3512010-12-08 21:16:57 -08001824 if (dst_mtu(dst) > IP_MAX_MTU)
1825 dst_metric_set(dst, RTAX_MTU, IP_MAX_MTU);
David S. Miller0dbaee32010-12-13 12:52:14 -08001826 if (dst_metric_raw(dst, RTAX_ADVMSS) > 65535 - 40)
David S. Millerdefb3512010-12-08 21:16:57 -08001827 dst_metric_set(dst, RTAX_ADVMSS, 65535 - 40);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001828
Patrick McHardyc7066f72011-01-14 13:36:42 +01001829#ifdef CONFIG_IP_ROUTE_CLASSID
Linus Torvalds1da177e2005-04-16 15:20:36 -07001830#ifdef CONFIG_IP_MULTIPLE_TABLES
1831 set_class_tag(rt, fib_rules_tclass(res));
1832#endif
1833 set_class_tag(rt, itag);
1834#endif
David S. Miller982721f2011-02-16 21:44:24 -08001835 rt->rt_type = type;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001836}
1837
David S. Miller0c4dcd52011-02-17 15:42:37 -08001838static struct rtable *rt_dst_alloc(bool nopolicy, bool noxfrm)
1839{
David S. Miller3c7bd1a2011-02-16 14:08:44 -08001840 struct rtable *rt = dst_alloc(&ipv4_dst_ops, 1);
David S. Miller0c4dcd52011-02-17 15:42:37 -08001841 if (rt) {
1842 rt->dst.obsolete = -1;
1843
David S. Miller0c4dcd52011-02-17 15:42:37 -08001844 rt->dst.flags = DST_HOST |
1845 (nopolicy ? DST_NOPOLICY : 0) |
1846 (noxfrm ? DST_NOXFRM : 0);
1847 }
1848 return rt;
1849}
1850
Eric Dumazet96d36222010-06-02 19:21:31 +00001851/* called in rcu_read_lock() section */
Al Viro9e12bb22006-09-26 21:25:20 -07001852static int ip_route_input_mc(struct sk_buff *skb, __be32 daddr, __be32 saddr,
Linus Torvalds1da177e2005-04-16 15:20:36 -07001853 u8 tos, struct net_device *dev, int our)
1854{
Eric Dumazet96d36222010-06-02 19:21:31 +00001855 unsigned int hash;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001856 struct rtable *rth;
Al Viroa61ced52006-09-26 21:27:54 -07001857 __be32 spec_dst;
Eric Dumazet96d36222010-06-02 19:21:31 +00001858 struct in_device *in_dev = __in_dev_get_rcu(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001859 u32 itag = 0;
Eric Dumazetb5f7e752010-06-02 12:05:27 +00001860 int err;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001861
1862 /* Primary sanity checks. */
1863
1864 if (in_dev == NULL)
1865 return -EINVAL;
1866
Jan Engelhardt1e637c72008-01-21 03:18:08 -08001867 if (ipv4_is_multicast(saddr) || ipv4_is_lbcast(saddr) ||
Joe Perchesf97c1e02007-12-16 13:45:43 -08001868 ipv4_is_loopback(saddr) || skb->protocol != htons(ETH_P_IP))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001869 goto e_inval;
1870
Joe Perchesf97c1e02007-12-16 13:45:43 -08001871 if (ipv4_is_zeronet(saddr)) {
1872 if (!ipv4_is_local_multicast(daddr))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001873 goto e_inval;
1874 spec_dst = inet_select_addr(dev, 0, RT_SCOPE_LINK);
Eric Dumazetb5f7e752010-06-02 12:05:27 +00001875 } else {
1876 err = fib_validate_source(saddr, 0, tos, 0, dev, &spec_dst,
1877 &itag, 0);
1878 if (err < 0)
1879 goto e_err;
1880 }
David S. Miller0c4dcd52011-02-17 15:42:37 -08001881 rth = rt_dst_alloc(IN_DEV_CONF_GET(in_dev, NOPOLICY), false);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001882 if (!rth)
1883 goto e_nobufs;
1884
Changli Gaod8d1f302010-06-10 23:31:35 -07001885 rth->dst.output = ip_rt_bug;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001886
David S. Miller5e2b61f2011-03-04 21:47:09 -08001887 rth->rt_key_dst = daddr;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001888 rth->rt_dst = daddr;
David S. Miller5e2b61f2011-03-04 21:47:09 -08001889 rth->rt_tos = tos;
1890 rth->rt_mark = skb->mark;
1891 rth->rt_key_src = saddr;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001892 rth->rt_src = saddr;
Patrick McHardyc7066f72011-01-14 13:36:42 +01001893#ifdef CONFIG_IP_ROUTE_CLASSID
Changli Gaod8d1f302010-06-10 23:31:35 -07001894 rth->dst.tclassid = itag;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001895#endif
David S. Miller5e2b61f2011-03-04 21:47:09 -08001896 rth->rt_iif = dev->ifindex;
Changli Gaod8d1f302010-06-10 23:31:35 -07001897 rth->dst.dev = init_net.loopback_dev;
1898 dev_hold(rth->dst.dev);
David S. Miller5e2b61f2011-03-04 21:47:09 -08001899 rth->rt_oif = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001900 rth->rt_gateway = daddr;
1901 rth->rt_spec_dst= spec_dst;
Denis V. Luneve84f84f2008-07-05 19:04:32 -07001902 rth->rt_genid = rt_genid(dev_net(dev));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001903 rth->rt_flags = RTCF_MULTICAST;
Eric Dumazet29e75252008-01-31 17:05:09 -08001904 rth->rt_type = RTN_MULTICAST;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001905 if (our) {
Changli Gaod8d1f302010-06-10 23:31:35 -07001906 rth->dst.input= ip_local_deliver;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001907 rth->rt_flags |= RTCF_LOCAL;
1908 }
1909
1910#ifdef CONFIG_IP_MROUTE
Joe Perchesf97c1e02007-12-16 13:45:43 -08001911 if (!ipv4_is_local_multicast(daddr) && IN_DEV_MFORWARD(in_dev))
Changli Gaod8d1f302010-06-10 23:31:35 -07001912 rth->dst.input = ip_mr_input;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001913#endif
1914 RT_CACHE_STAT_INC(in_slow_mc);
1915
Denis V. Luneve84f84f2008-07-05 19:04:32 -07001916 hash = rt_hash(daddr, saddr, dev->ifindex, rt_genid(dev_net(dev)));
David S. Millerb23dd4f2011-03-02 14:31:35 -08001917 rth = rt_intern_hash(hash, rth, skb, dev->ifindex);
1918 err = 0;
1919 if (IS_ERR(rth))
1920 err = PTR_ERR(rth);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001921
1922e_nobufs:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001923 return -ENOBUFS;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001924e_inval:
Eric Dumazet96d36222010-06-02 19:21:31 +00001925 return -EINVAL;
Eric Dumazetb5f7e752010-06-02 12:05:27 +00001926e_err:
Eric Dumazetb5f7e752010-06-02 12:05:27 +00001927 return err;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001928}
1929
1930
1931static void ip_handle_martian_source(struct net_device *dev,
1932 struct in_device *in_dev,
1933 struct sk_buff *skb,
Al Viro9e12bb22006-09-26 21:25:20 -07001934 __be32 daddr,
1935 __be32 saddr)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001936{
1937 RT_CACHE_STAT_INC(in_martian_src);
1938#ifdef CONFIG_IP_ROUTE_VERBOSE
1939 if (IN_DEV_LOG_MARTIANS(in_dev) && net_ratelimit()) {
1940 /*
1941 * RFC1812 recommendation, if source is martian,
1942 * the only hint is MAC header.
1943 */
Harvey Harrison673d57e2008-10-31 00:53:57 -07001944 printk(KERN_WARNING "martian source %pI4 from %pI4, on dev %s\n",
1945 &daddr, &saddr, dev->name);
Arnaldo Carvalho de Melo98e399f2007-03-19 15:33:04 -07001946 if (dev->hard_header_len && skb_mac_header_was_set(skb)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001947 int i;
Arnaldo Carvalho de Melo98e399f2007-03-19 15:33:04 -07001948 const unsigned char *p = skb_mac_header(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001949 printk(KERN_WARNING "ll header: ");
1950 for (i = 0; i < dev->hard_header_len; i++, p++) {
1951 printk("%02x", *p);
1952 if (i < (dev->hard_header_len - 1))
1953 printk(":");
1954 }
1955 printk("\n");
1956 }
1957 }
1958#endif
1959}
1960
Eric Dumazet47360222010-06-03 04:13:21 +00001961/* called in rcu_read_lock() section */
Stephen Hemminger5969f712008-04-10 01:52:09 -07001962static int __mkroute_input(struct sk_buff *skb,
David S. Miller982721f2011-02-16 21:44:24 -08001963 const struct fib_result *res,
Stephen Hemminger5969f712008-04-10 01:52:09 -07001964 struct in_device *in_dev,
1965 __be32 daddr, __be32 saddr, u32 tos,
1966 struct rtable **result)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001967{
Linus Torvalds1da177e2005-04-16 15:20:36 -07001968 struct rtable *rth;
1969 int err;
1970 struct in_device *out_dev;
Eric Dumazet47360222010-06-03 04:13:21 +00001971 unsigned int flags = 0;
Al Virod9c9df82006-09-26 21:28:14 -07001972 __be32 spec_dst;
1973 u32 itag;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001974
1975 /* get a working reference to the output device */
Eric Dumazet47360222010-06-03 04:13:21 +00001976 out_dev = __in_dev_get_rcu(FIB_RES_DEV(*res));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001977 if (out_dev == NULL) {
1978 if (net_ratelimit())
1979 printk(KERN_CRIT "Bug in ip_route_input" \
1980 "_slow(). Please, report\n");
1981 return -EINVAL;
1982 }
1983
1984
YOSHIFUJI Hideakie905a9e2007-02-09 23:24:47 +09001985 err = fib_validate_source(saddr, daddr, tos, FIB_RES_OIF(*res),
jamalb0c110c2009-10-18 02:12:33 +00001986 in_dev->dev, &spec_dst, &itag, skb->mark);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001987 if (err < 0) {
YOSHIFUJI Hideakie905a9e2007-02-09 23:24:47 +09001988 ip_handle_martian_source(in_dev->dev, in_dev, skb, daddr,
Linus Torvalds1da177e2005-04-16 15:20:36 -07001989 saddr);
YOSHIFUJI Hideakie905a9e2007-02-09 23:24:47 +09001990
Linus Torvalds1da177e2005-04-16 15:20:36 -07001991 goto cleanup;
1992 }
1993
1994 if (err)
1995 flags |= RTCF_DIRECTSRC;
1996
Thomas Graf51b77ca2008-06-03 16:36:01 -07001997 if (out_dev == in_dev && err &&
Linus Torvalds1da177e2005-04-16 15:20:36 -07001998 (IN_DEV_SHARED_MEDIA(out_dev) ||
1999 inet_addr_onlink(out_dev, saddr, FIB_RES_GW(*res))))
2000 flags |= RTCF_DOREDIRECT;
2001
2002 if (skb->protocol != htons(ETH_P_IP)) {
2003 /* Not IP (i.e. ARP). Do not create route, if it is
2004 * invalid for proxy arp. DNAT routes are always valid.
Jesper Dangaard Brouer65324142010-01-05 05:50:47 +00002005 *
2006 * Proxy arp feature have been extended to allow, ARP
2007 * replies back to the same interface, to support
2008 * Private VLAN switch technologies. See arp.c.
Linus Torvalds1da177e2005-04-16 15:20:36 -07002009 */
Jesper Dangaard Brouer65324142010-01-05 05:50:47 +00002010 if (out_dev == in_dev &&
2011 IN_DEV_PROXY_ARP_PVLAN(in_dev) == 0) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07002012 err = -EINVAL;
2013 goto cleanup;
2014 }
2015 }
2016
David S. Miller0c4dcd52011-02-17 15:42:37 -08002017 rth = rt_dst_alloc(IN_DEV_CONF_GET(in_dev, NOPOLICY),
2018 IN_DEV_CONF_GET(out_dev, NOXFRM));
Linus Torvalds1da177e2005-04-16 15:20:36 -07002019 if (!rth) {
2020 err = -ENOBUFS;
2021 goto cleanup;
2022 }
2023
David S. Miller5e2b61f2011-03-04 21:47:09 -08002024 rth->rt_key_dst = daddr;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002025 rth->rt_dst = daddr;
David S. Miller5e2b61f2011-03-04 21:47:09 -08002026 rth->rt_tos = tos;
2027 rth->rt_mark = skb->mark;
2028 rth->rt_key_src = saddr;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002029 rth->rt_src = saddr;
2030 rth->rt_gateway = daddr;
David S. Miller5e2b61f2011-03-04 21:47:09 -08002031 rth->rt_iif = in_dev->dev->ifindex;
Changli Gaod8d1f302010-06-10 23:31:35 -07002032 rth->dst.dev = (out_dev)->dev;
2033 dev_hold(rth->dst.dev);
David S. Miller5e2b61f2011-03-04 21:47:09 -08002034 rth->rt_oif = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002035 rth->rt_spec_dst= spec_dst;
2036
Changli Gaod8d1f302010-06-10 23:31:35 -07002037 rth->dst.input = ip_forward;
2038 rth->dst.output = ip_output;
2039 rth->rt_genid = rt_genid(dev_net(rth->dst.dev));
Linus Torvalds1da177e2005-04-16 15:20:36 -07002040
David S. Miller5e2b61f2011-03-04 21:47:09 -08002041 rt_set_nexthop(rth, NULL, res, res->fi, res->type, itag);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002042
2043 rth->rt_flags = flags;
2044
2045 *result = rth;
2046 err = 0;
2047 cleanup:
Linus Torvalds1da177e2005-04-16 15:20:36 -07002048 return err;
YOSHIFUJI Hideakie905a9e2007-02-09 23:24:47 +09002049}
Linus Torvalds1da177e2005-04-16 15:20:36 -07002050
Stephen Hemminger5969f712008-04-10 01:52:09 -07002051static int ip_mkroute_input(struct sk_buff *skb,
2052 struct fib_result *res,
David S. Miller68a5e3d2011-03-11 20:07:33 -05002053 const struct flowi4 *fl4,
Stephen Hemminger5969f712008-04-10 01:52:09 -07002054 struct in_device *in_dev,
2055 __be32 daddr, __be32 saddr, u32 tos)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002056{
Chuck Short7abaa272005-06-22 22:10:23 -07002057 struct rtable* rth = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002058 int err;
2059 unsigned hash;
2060
2061#ifdef CONFIG_IP_ROUTE_MULTIPATH
David S. Millerff3fccb2011-03-10 16:23:24 -08002062 if (res->fi && res->fi->fib_nhs > 1)
David S. Miller1b7fe5932011-03-10 17:01:16 -08002063 fib_select_multipath(res);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002064#endif
2065
2066 /* create a routing cache entry */
2067 err = __mkroute_input(skb, res, in_dev, daddr, saddr, tos, &rth);
2068 if (err)
2069 return err;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002070
2071 /* put it into the cache */
David S. Miller68a5e3d2011-03-11 20:07:33 -05002072 hash = rt_hash(daddr, saddr, fl4->flowi4_iif,
Changli Gaod8d1f302010-06-10 23:31:35 -07002073 rt_genid(dev_net(rth->dst.dev)));
David S. Miller68a5e3d2011-03-11 20:07:33 -05002074 rth = rt_intern_hash(hash, rth, skb, fl4->flowi4_iif);
David S. Millerb23dd4f2011-03-02 14:31:35 -08002075 if (IS_ERR(rth))
2076 return PTR_ERR(rth);
2077 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002078}
2079
Linus Torvalds1da177e2005-04-16 15:20:36 -07002080/*
2081 * NOTE. We drop all the packets that has local source
2082 * addresses, because every properly looped back packet
2083 * must have correct destination already attached by output routine.
2084 *
2085 * Such approach solves two big problems:
2086 * 1. Not simplex devices are handled properly.
2087 * 2. IP spoofing attempts are filtered with 100% of guarantee.
Eric Dumazetebc0ffa2010-10-05 10:41:36 +00002088 * called with rcu_read_lock()
Linus Torvalds1da177e2005-04-16 15:20:36 -07002089 */
2090
Al Viro9e12bb22006-09-26 21:25:20 -07002091static int ip_route_input_slow(struct sk_buff *skb, __be32 daddr, __be32 saddr,
Linus Torvalds1da177e2005-04-16 15:20:36 -07002092 u8 tos, struct net_device *dev)
2093{
2094 struct fib_result res;
Eric Dumazet96d36222010-06-02 19:21:31 +00002095 struct in_device *in_dev = __in_dev_get_rcu(dev);
David S. Miller68a5e3d2011-03-11 20:07:33 -05002096 struct flowi4 fl4;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002097 unsigned flags = 0;
2098 u32 itag = 0;
2099 struct rtable * rth;
2100 unsigned hash;
Al Viro9e12bb22006-09-26 21:25:20 -07002101 __be32 spec_dst;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002102 int err = -EINVAL;
YOSHIFUJI Hideakic346dca2008-03-25 21:47:49 +09002103 struct net * net = dev_net(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002104
2105 /* IP on this device is disabled. */
2106
2107 if (!in_dev)
2108 goto out;
2109
2110 /* Check for the most weird martians, which can be not detected
2111 by fib_lookup.
2112 */
2113
Jan Engelhardt1e637c72008-01-21 03:18:08 -08002114 if (ipv4_is_multicast(saddr) || ipv4_is_lbcast(saddr) ||
Joe Perchesf97c1e02007-12-16 13:45:43 -08002115 ipv4_is_loopback(saddr))
Linus Torvalds1da177e2005-04-16 15:20:36 -07002116 goto martian_source;
2117
Andy Walls27a954b2010-10-17 15:11:22 +00002118 if (ipv4_is_lbcast(daddr) || (saddr == 0 && daddr == 0))
Linus Torvalds1da177e2005-04-16 15:20:36 -07002119 goto brd_input;
2120
2121 /* Accept zero addresses only to limited broadcast;
2122 * I even do not know to fix it or not. Waiting for complains :-)
2123 */
Joe Perchesf97c1e02007-12-16 13:45:43 -08002124 if (ipv4_is_zeronet(saddr))
Linus Torvalds1da177e2005-04-16 15:20:36 -07002125 goto martian_source;
2126
Andy Walls27a954b2010-10-17 15:11:22 +00002127 if (ipv4_is_zeronet(daddr) || ipv4_is_loopback(daddr))
Linus Torvalds1da177e2005-04-16 15:20:36 -07002128 goto martian_destination;
2129
2130 /*
2131 * Now we are ready to route packet.
2132 */
David S. Miller68a5e3d2011-03-11 20:07:33 -05002133 fl4.flowi4_oif = 0;
2134 fl4.flowi4_iif = dev->ifindex;
2135 fl4.flowi4_mark = skb->mark;
2136 fl4.flowi4_tos = tos;
2137 fl4.flowi4_scope = RT_SCOPE_UNIVERSE;
2138 fl4.daddr = daddr;
2139 fl4.saddr = saddr;
2140 err = fib_lookup(net, &fl4, &res);
Eric Dumazetebc0ffa2010-10-05 10:41:36 +00002141 if (err != 0) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07002142 if (!IN_DEV_FORWARD(in_dev))
Dietmar Eggemann2c2910a2005-06-28 13:06:23 -07002143 goto e_hostunreach;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002144 goto no_route;
2145 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002146
2147 RT_CACHE_STAT_INC(in_slow_tot);
2148
2149 if (res.type == RTN_BROADCAST)
2150 goto brd_input;
2151
2152 if (res.type == RTN_LOCAL) {
Eric Dumazetb5f7e752010-06-02 12:05:27 +00002153 err = fib_validate_source(saddr, daddr, tos,
Eric Dumazetebc0ffa2010-10-05 10:41:36 +00002154 net->loopback_dev->ifindex,
2155 dev, &spec_dst, &itag, skb->mark);
Eric Dumazetb5f7e752010-06-02 12:05:27 +00002156 if (err < 0)
2157 goto martian_source_keep_err;
2158 if (err)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002159 flags |= RTCF_DIRECTSRC;
2160 spec_dst = daddr;
2161 goto local_input;
2162 }
2163
2164 if (!IN_DEV_FORWARD(in_dev))
Dietmar Eggemann2c2910a2005-06-28 13:06:23 -07002165 goto e_hostunreach;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002166 if (res.type != RTN_UNICAST)
2167 goto martian_destination;
2168
David S. Miller68a5e3d2011-03-11 20:07:33 -05002169 err = ip_mkroute_input(skb, &res, &fl4, in_dev, daddr, saddr, tos);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002170out: return err;
2171
2172brd_input:
2173 if (skb->protocol != htons(ETH_P_IP))
2174 goto e_inval;
2175
Joe Perchesf97c1e02007-12-16 13:45:43 -08002176 if (ipv4_is_zeronet(saddr))
Linus Torvalds1da177e2005-04-16 15:20:36 -07002177 spec_dst = inet_select_addr(dev, 0, RT_SCOPE_LINK);
2178 else {
2179 err = fib_validate_source(saddr, 0, tos, 0, dev, &spec_dst,
jamalb0c110c2009-10-18 02:12:33 +00002180 &itag, skb->mark);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002181 if (err < 0)
Eric Dumazetb5f7e752010-06-02 12:05:27 +00002182 goto martian_source_keep_err;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002183 if (err)
2184 flags |= RTCF_DIRECTSRC;
2185 }
2186 flags |= RTCF_BROADCAST;
2187 res.type = RTN_BROADCAST;
2188 RT_CACHE_STAT_INC(in_brd);
2189
2190local_input:
David S. Miller0c4dcd52011-02-17 15:42:37 -08002191 rth = rt_dst_alloc(IN_DEV_CONF_GET(in_dev, NOPOLICY), false);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002192 if (!rth)
2193 goto e_nobufs;
2194
Changli Gaod8d1f302010-06-10 23:31:35 -07002195 rth->dst.output= ip_rt_bug;
Denis V. Luneve84f84f2008-07-05 19:04:32 -07002196 rth->rt_genid = rt_genid(net);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002197
David S. Miller5e2b61f2011-03-04 21:47:09 -08002198 rth->rt_key_dst = daddr;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002199 rth->rt_dst = daddr;
David S. Miller5e2b61f2011-03-04 21:47:09 -08002200 rth->rt_tos = tos;
2201 rth->rt_mark = skb->mark;
2202 rth->rt_key_src = saddr;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002203 rth->rt_src = saddr;
Patrick McHardyc7066f72011-01-14 13:36:42 +01002204#ifdef CONFIG_IP_ROUTE_CLASSID
Changli Gaod8d1f302010-06-10 23:31:35 -07002205 rth->dst.tclassid = itag;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002206#endif
David S. Miller5e2b61f2011-03-04 21:47:09 -08002207 rth->rt_iif = dev->ifindex;
Changli Gaod8d1f302010-06-10 23:31:35 -07002208 rth->dst.dev = net->loopback_dev;
2209 dev_hold(rth->dst.dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002210 rth->rt_gateway = daddr;
2211 rth->rt_spec_dst= spec_dst;
Changli Gaod8d1f302010-06-10 23:31:35 -07002212 rth->dst.input= ip_local_deliver;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002213 rth->rt_flags = flags|RTCF_LOCAL;
2214 if (res.type == RTN_UNREACHABLE) {
Changli Gaod8d1f302010-06-10 23:31:35 -07002215 rth->dst.input= ip_error;
2216 rth->dst.error= -err;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002217 rth->rt_flags &= ~RTCF_LOCAL;
2218 }
2219 rth->rt_type = res.type;
David S. Miller68a5e3d2011-03-11 20:07:33 -05002220 hash = rt_hash(daddr, saddr, fl4.flowi4_iif, rt_genid(net));
2221 rth = rt_intern_hash(hash, rth, skb, fl4.flowi4_iif);
David S. Millerb23dd4f2011-03-02 14:31:35 -08002222 err = 0;
2223 if (IS_ERR(rth))
2224 err = PTR_ERR(rth);
Eric Dumazetebc0ffa2010-10-05 10:41:36 +00002225 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002226
2227no_route:
2228 RT_CACHE_STAT_INC(in_no_route);
2229 spec_dst = inet_select_addr(dev, 0, RT_SCOPE_UNIVERSE);
2230 res.type = RTN_UNREACHABLE;
Mitsuru Chinen7f538782007-12-07 01:07:24 -08002231 if (err == -ESRCH)
2232 err = -ENETUNREACH;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002233 goto local_input;
2234
2235 /*
2236 * Do not cache martian addresses: they should be logged (RFC1812)
2237 */
2238martian_destination:
2239 RT_CACHE_STAT_INC(in_martian_dst);
2240#ifdef CONFIG_IP_ROUTE_VERBOSE
2241 if (IN_DEV_LOG_MARTIANS(in_dev) && net_ratelimit())
Harvey Harrison673d57e2008-10-31 00:53:57 -07002242 printk(KERN_WARNING "martian destination %pI4 from %pI4, dev %s\n",
2243 &daddr, &saddr, dev->name);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002244#endif
Dietmar Eggemann2c2910a2005-06-28 13:06:23 -07002245
2246e_hostunreach:
YOSHIFUJI Hideakie905a9e2007-02-09 23:24:47 +09002247 err = -EHOSTUNREACH;
Eric Dumazetebc0ffa2010-10-05 10:41:36 +00002248 goto out;
Dietmar Eggemann2c2910a2005-06-28 13:06:23 -07002249
Linus Torvalds1da177e2005-04-16 15:20:36 -07002250e_inval:
2251 err = -EINVAL;
Eric Dumazetebc0ffa2010-10-05 10:41:36 +00002252 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002253
2254e_nobufs:
2255 err = -ENOBUFS;
Eric Dumazetebc0ffa2010-10-05 10:41:36 +00002256 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002257
2258martian_source:
Eric Dumazetb5f7e752010-06-02 12:05:27 +00002259 err = -EINVAL;
2260martian_source_keep_err:
Linus Torvalds1da177e2005-04-16 15:20:36 -07002261 ip_handle_martian_source(dev, in_dev, skb, daddr, saddr);
Eric Dumazetebc0ffa2010-10-05 10:41:36 +00002262 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002263}
2264
Eric Dumazet407eadd2010-05-10 11:32:55 +00002265int ip_route_input_common(struct sk_buff *skb, __be32 daddr, __be32 saddr,
2266 u8 tos, struct net_device *dev, bool noref)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002267{
2268 struct rtable * rth;
2269 unsigned hash;
2270 int iif = dev->ifindex;
Denis V. Lunevb5921912008-01-22 23:50:25 -08002271 struct net *net;
Eric Dumazet96d36222010-06-02 19:21:31 +00002272 int res;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002273
YOSHIFUJI Hideakic346dca2008-03-25 21:47:49 +09002274 net = dev_net(dev);
Neil Horman1080d702008-10-27 12:28:25 -07002275
Eric Dumazet96d36222010-06-02 19:21:31 +00002276 rcu_read_lock();
2277
Neil Horman1080d702008-10-27 12:28:25 -07002278 if (!rt_caching(net))
2279 goto skip_cache;
2280
Linus Torvalds1da177e2005-04-16 15:20:36 -07002281 tos &= IPTOS_RT_MASK;
Denis V. Luneve84f84f2008-07-05 19:04:32 -07002282 hash = rt_hash(daddr, saddr, iif, rt_genid(net));
Linus Torvalds1da177e2005-04-16 15:20:36 -07002283
Linus Torvalds1da177e2005-04-16 15:20:36 -07002284 for (rth = rcu_dereference(rt_hash_table[hash].chain); rth;
Changli Gaod8d1f302010-06-10 23:31:35 -07002285 rth = rcu_dereference(rth->dst.rt_next)) {
David S. Miller5e2b61f2011-03-04 21:47:09 -08002286 if ((((__force u32)rth->rt_key_dst ^ (__force u32)daddr) |
2287 ((__force u32)rth->rt_key_src ^ (__force u32)saddr) |
2288 (rth->rt_iif ^ iif) |
2289 rth->rt_oif |
2290 (rth->rt_tos ^ tos)) == 0 &&
2291 rth->rt_mark == skb->mark &&
Changli Gaod8d1f302010-06-10 23:31:35 -07002292 net_eq(dev_net(rth->dst.dev), net) &&
Denis V. Luneve84f84f2008-07-05 19:04:32 -07002293 !rt_is_expired(rth)) {
Eric Dumazet407eadd2010-05-10 11:32:55 +00002294 if (noref) {
Changli Gaod8d1f302010-06-10 23:31:35 -07002295 dst_use_noref(&rth->dst, jiffies);
2296 skb_dst_set_noref(skb, &rth->dst);
Eric Dumazet407eadd2010-05-10 11:32:55 +00002297 } else {
Changli Gaod8d1f302010-06-10 23:31:35 -07002298 dst_use(&rth->dst, jiffies);
2299 skb_dst_set(skb, &rth->dst);
Eric Dumazet407eadd2010-05-10 11:32:55 +00002300 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002301 RT_CACHE_STAT_INC(in_hit);
2302 rcu_read_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07002303 return 0;
2304 }
2305 RT_CACHE_STAT_INC(in_hlist_search);
2306 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002307
Neil Horman1080d702008-10-27 12:28:25 -07002308skip_cache:
Linus Torvalds1da177e2005-04-16 15:20:36 -07002309 /* Multicast recognition logic is moved from route cache to here.
2310 The problem was that too many Ethernet cards have broken/missing
2311 hardware multicast filters :-( As result the host on multicasting
2312 network acquires a lot of useless route cache entries, sort of
2313 SDR messages from all the world. Now we try to get rid of them.
2314 Really, provided software IP multicast filter is organized
2315 reasonably (at least, hashed), it does not result in a slowdown
2316 comparing with route cache reject entries.
2317 Note, that multicast routers are not affected, because
2318 route cache entry is created eventually.
2319 */
Joe Perchesf97c1e02007-12-16 13:45:43 -08002320 if (ipv4_is_multicast(daddr)) {
Eric Dumazet96d36222010-06-02 19:21:31 +00002321 struct in_device *in_dev = __in_dev_get_rcu(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002322
Eric Dumazet96d36222010-06-02 19:21:31 +00002323 if (in_dev) {
David S. Millerdbdd9a52011-03-10 16:34:38 -08002324 int our = ip_check_mc_rcu(in_dev, daddr, saddr,
2325 ip_hdr(skb)->protocol);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002326 if (our
2327#ifdef CONFIG_IP_MROUTE
Joe Perches9d4fb272009-11-23 10:41:23 -08002328 ||
2329 (!ipv4_is_local_multicast(daddr) &&
2330 IN_DEV_MFORWARD(in_dev))
Linus Torvalds1da177e2005-04-16 15:20:36 -07002331#endif
Joe Perches9d4fb272009-11-23 10:41:23 -08002332 ) {
Eric Dumazet96d36222010-06-02 19:21:31 +00002333 int res = ip_route_input_mc(skb, daddr, saddr,
2334 tos, dev, our);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002335 rcu_read_unlock();
Eric Dumazet96d36222010-06-02 19:21:31 +00002336 return res;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002337 }
2338 }
2339 rcu_read_unlock();
2340 return -EINVAL;
2341 }
Eric Dumazet96d36222010-06-02 19:21:31 +00002342 res = ip_route_input_slow(skb, daddr, saddr, tos, dev);
2343 rcu_read_unlock();
2344 return res;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002345}
Eric Dumazet407eadd2010-05-10 11:32:55 +00002346EXPORT_SYMBOL(ip_route_input_common);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002347
Eric Dumazetebc0ffa2010-10-05 10:41:36 +00002348/* called with rcu_read_lock() */
David S. Miller982721f2011-02-16 21:44:24 -08002349static struct rtable *__mkroute_output(const struct fib_result *res,
David S. Miller68a5e3d2011-03-11 20:07:33 -05002350 const struct flowi4 *fl4,
2351 const struct flowi4 *oldflp4,
David S. Miller5ada5522011-02-17 15:29:00 -08002352 struct net_device *dev_out,
2353 unsigned int flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002354{
David S. Miller982721f2011-02-16 21:44:24 -08002355 struct fib_info *fi = res->fi;
David S. Miller68a5e3d2011-03-11 20:07:33 -05002356 u32 tos = RT_FL_TOS(oldflp4);
David S. Miller5ada5522011-02-17 15:29:00 -08002357 struct in_device *in_dev;
David S. Miller982721f2011-02-16 21:44:24 -08002358 u16 type = res->type;
David S. Miller5ada5522011-02-17 15:29:00 -08002359 struct rtable *rth;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002360
David S. Miller68a5e3d2011-03-11 20:07:33 -05002361 if (ipv4_is_loopback(fl4->saddr) && !(dev_out->flags & IFF_LOOPBACK))
David S. Miller5ada5522011-02-17 15:29:00 -08002362 return ERR_PTR(-EINVAL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002363
David S. Miller68a5e3d2011-03-11 20:07:33 -05002364 if (ipv4_is_lbcast(fl4->daddr))
David S. Miller982721f2011-02-16 21:44:24 -08002365 type = RTN_BROADCAST;
David S. Miller68a5e3d2011-03-11 20:07:33 -05002366 else if (ipv4_is_multicast(fl4->daddr))
David S. Miller982721f2011-02-16 21:44:24 -08002367 type = RTN_MULTICAST;
David S. Miller68a5e3d2011-03-11 20:07:33 -05002368 else if (ipv4_is_zeronet(fl4->daddr))
David S. Miller5ada5522011-02-17 15:29:00 -08002369 return ERR_PTR(-EINVAL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002370
2371 if (dev_out->flags & IFF_LOOPBACK)
2372 flags |= RTCF_LOCAL;
2373
Eric Dumazetdd28d1a2010-09-29 11:53:50 +00002374 in_dev = __in_dev_get_rcu(dev_out);
Eric Dumazetebc0ffa2010-10-05 10:41:36 +00002375 if (!in_dev)
David S. Miller5ada5522011-02-17 15:29:00 -08002376 return ERR_PTR(-EINVAL);
Eric Dumazetebc0ffa2010-10-05 10:41:36 +00002377
David S. Miller982721f2011-02-16 21:44:24 -08002378 if (type == RTN_BROADCAST) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07002379 flags |= RTCF_BROADCAST | RTCF_LOCAL;
David S. Miller982721f2011-02-16 21:44:24 -08002380 fi = NULL;
2381 } else if (type == RTN_MULTICAST) {
Eric Dumazetdd28d1a2010-09-29 11:53:50 +00002382 flags |= RTCF_MULTICAST | RTCF_LOCAL;
David S. Miller68a5e3d2011-03-11 20:07:33 -05002383 if (!ip_check_mc_rcu(in_dev, oldflp4->daddr, oldflp4->saddr,
2384 oldflp4->flowi4_proto))
Linus Torvalds1da177e2005-04-16 15:20:36 -07002385 flags &= ~RTCF_LOCAL;
2386 /* If multicast route do not exist use
Eric Dumazetdd28d1a2010-09-29 11:53:50 +00002387 * default one, but do not gateway in this case.
2388 * Yes, it is hack.
Linus Torvalds1da177e2005-04-16 15:20:36 -07002389 */
David S. Miller982721f2011-02-16 21:44:24 -08002390 if (fi && res->prefixlen < 4)
2391 fi = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002392 }
2393
David S. Miller0c4dcd52011-02-17 15:42:37 -08002394 rth = rt_dst_alloc(IN_DEV_CONF_GET(in_dev, NOPOLICY),
2395 IN_DEV_CONF_GET(in_dev, NOXFRM));
Dimitris Michailidis8391d072010-10-07 14:48:38 +00002396 if (!rth)
David S. Miller5ada5522011-02-17 15:29:00 -08002397 return ERR_PTR(-ENOBUFS);
Dimitris Michailidis8391d072010-10-07 14:48:38 +00002398
David S. Miller68a5e3d2011-03-11 20:07:33 -05002399 rth->rt_key_dst = oldflp4->daddr;
David S. Miller5e2b61f2011-03-04 21:47:09 -08002400 rth->rt_tos = tos;
David S. Miller68a5e3d2011-03-11 20:07:33 -05002401 rth->rt_key_src = oldflp4->saddr;
2402 rth->rt_oif = oldflp4->flowi4_oif;
2403 rth->rt_mark = oldflp4->flowi4_mark;
2404 rth->rt_dst = fl4->daddr;
2405 rth->rt_src = fl4->saddr;
David S. Miller1018b5c2011-03-04 21:35:25 -08002406 rth->rt_iif = 0;
YOSHIFUJI Hideakie905a9e2007-02-09 23:24:47 +09002407 /* get references to the devices that are to be hold by the routing
Linus Torvalds1da177e2005-04-16 15:20:36 -07002408 cache entry */
Changli Gaod8d1f302010-06-10 23:31:35 -07002409 rth->dst.dev = dev_out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002410 dev_hold(dev_out);
David S. Miller68a5e3d2011-03-11 20:07:33 -05002411 rth->rt_gateway = fl4->daddr;
2412 rth->rt_spec_dst= fl4->saddr;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002413
Changli Gaod8d1f302010-06-10 23:31:35 -07002414 rth->dst.output=ip_output;
Denis V. Luneve84f84f2008-07-05 19:04:32 -07002415 rth->rt_genid = rt_genid(dev_net(dev_out));
Linus Torvalds1da177e2005-04-16 15:20:36 -07002416
2417 RT_CACHE_STAT_INC(out_slow_tot);
2418
2419 if (flags & RTCF_LOCAL) {
Changli Gaod8d1f302010-06-10 23:31:35 -07002420 rth->dst.input = ip_local_deliver;
David S. Miller68a5e3d2011-03-11 20:07:33 -05002421 rth->rt_spec_dst = fl4->daddr;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002422 }
2423 if (flags & (RTCF_BROADCAST | RTCF_MULTICAST)) {
David S. Miller68a5e3d2011-03-11 20:07:33 -05002424 rth->rt_spec_dst = fl4->saddr;
YOSHIFUJI Hideakie905a9e2007-02-09 23:24:47 +09002425 if (flags & RTCF_LOCAL &&
Linus Torvalds1da177e2005-04-16 15:20:36 -07002426 !(dev_out->flags & IFF_LOOPBACK)) {
Changli Gaod8d1f302010-06-10 23:31:35 -07002427 rth->dst.output = ip_mc_output;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002428 RT_CACHE_STAT_INC(out_slow_mc);
2429 }
2430#ifdef CONFIG_IP_MROUTE
David S. Miller982721f2011-02-16 21:44:24 -08002431 if (type == RTN_MULTICAST) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07002432 if (IN_DEV_MFORWARD(in_dev) &&
David S. Miller68a5e3d2011-03-11 20:07:33 -05002433 !ipv4_is_local_multicast(oldflp4->daddr)) {
Changli Gaod8d1f302010-06-10 23:31:35 -07002434 rth->dst.input = ip_mr_input;
2435 rth->dst.output = ip_mc_output;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002436 }
2437 }
2438#endif
2439 }
2440
David S. Miller68a5e3d2011-03-11 20:07:33 -05002441 rt_set_nexthop(rth, oldflp4, res, fi, type, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002442
2443 rth->rt_flags = flags;
David S. Miller5ada5522011-02-17 15:29:00 -08002444 return rth;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002445}
2446
Linus Torvalds1da177e2005-04-16 15:20:36 -07002447/*
2448 * Major route resolver routine.
Eric Dumazet0197aa32010-09-30 03:33:58 +00002449 * called with rcu_read_lock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07002450 */
2451
David S. Millerb23dd4f2011-03-02 14:31:35 -08002452static struct rtable *ip_route_output_slow(struct net *net,
David S. Miller68a5e3d2011-03-11 20:07:33 -05002453 const struct flowi4 *oldflp4)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002454{
David S. Miller68a5e3d2011-03-11 20:07:33 -05002455 u32 tos = RT_FL_TOS(oldflp4);
2456 struct flowi4 fl4;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002457 struct fib_result res;
Eric Dumazet0197aa32010-09-30 03:33:58 +00002458 unsigned int flags = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002459 struct net_device *dev_out = NULL;
David S. Miller5ada5522011-02-17 15:29:00 -08002460 struct rtable *rth;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002461
2462 res.fi = NULL;
2463#ifdef CONFIG_IP_MULTIPLE_TABLES
2464 res.r = NULL;
2465#endif
2466
David S. Miller68a5e3d2011-03-11 20:07:33 -05002467 fl4.flowi4_oif = oldflp4->flowi4_oif;
2468 fl4.flowi4_iif = net->loopback_dev->ifindex;
2469 fl4.flowi4_mark = oldflp4->flowi4_mark;
2470 fl4.daddr = oldflp4->daddr;
2471 fl4.saddr = oldflp4->saddr;
2472 fl4.flowi4_tos = tos & IPTOS_RT_MASK;
2473 fl4.flowi4_scope = ((tos & RTO_ONLINK) ?
David S. Miller44713b62011-03-04 21:24:47 -08002474 RT_SCOPE_LINK : RT_SCOPE_UNIVERSE);
2475
David S. Miller010c2702011-02-17 15:37:09 -08002476 rcu_read_lock();
David S. Miller68a5e3d2011-03-11 20:07:33 -05002477 if (oldflp4->saddr) {
David S. Millerb23dd4f2011-03-02 14:31:35 -08002478 rth = ERR_PTR(-EINVAL);
David S. Miller68a5e3d2011-03-11 20:07:33 -05002479 if (ipv4_is_multicast(oldflp4->saddr) ||
2480 ipv4_is_lbcast(oldflp4->saddr) ||
2481 ipv4_is_zeronet(oldflp4->saddr))
Linus Torvalds1da177e2005-04-16 15:20:36 -07002482 goto out;
2483
Linus Torvalds1da177e2005-04-16 15:20:36 -07002484 /* I removed check for oif == dev_out->oif here.
2485 It was wrong for two reasons:
Denis V. Lunev1ab35272008-01-22 22:04:30 -08002486 1. ip_dev_find(net, saddr) can return wrong iface, if saddr
2487 is assigned to multiple interfaces.
Linus Torvalds1da177e2005-04-16 15:20:36 -07002488 2. Moreover, we are allowed to send packets with saddr
2489 of another iface. --ANK
2490 */
2491
David S. Miller68a5e3d2011-03-11 20:07:33 -05002492 if (oldflp4->flowi4_oif == 0 &&
2493 (ipv4_is_multicast(oldflp4->daddr) ||
2494 ipv4_is_lbcast(oldflp4->daddr))) {
Julian Anastasova210d012008-10-01 07:28:28 -07002495 /* It is equivalent to inet_addr_type(saddr) == RTN_LOCAL */
David S. Miller68a5e3d2011-03-11 20:07:33 -05002496 dev_out = __ip_dev_find(net, oldflp4->saddr, false);
Julian Anastasova210d012008-10-01 07:28:28 -07002497 if (dev_out == NULL)
2498 goto out;
2499
Linus Torvalds1da177e2005-04-16 15:20:36 -07002500 /* Special hack: user can direct multicasts
2501 and limited broadcast via necessary interface
2502 without fiddling with IP_MULTICAST_IF or IP_PKTINFO.
2503 This hack is not just for fun, it allows
2504 vic,vat and friends to work.
2505 They bind socket to loopback, set ttl to zero
2506 and expect that it will work.
2507 From the viewpoint of routing cache they are broken,
2508 because we are not allowed to build multicast path
2509 with loopback source addr (look, routing cache
2510 cannot know, that ttl is zero, so that packet
2511 will not leave this host and route is valid).
2512 Luckily, this hack is good workaround.
2513 */
2514
David S. Miller68a5e3d2011-03-11 20:07:33 -05002515 fl4.flowi4_oif = dev_out->ifindex;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002516 goto make_route;
2517 }
Julian Anastasova210d012008-10-01 07:28:28 -07002518
David S. Miller68a5e3d2011-03-11 20:07:33 -05002519 if (!(oldflp4->flowi4_flags & FLOWI_FLAG_ANYSRC)) {
Julian Anastasova210d012008-10-01 07:28:28 -07002520 /* It is equivalent to inet_addr_type(saddr) == RTN_LOCAL */
David S. Miller68a5e3d2011-03-11 20:07:33 -05002521 if (!__ip_dev_find(net, oldflp4->saddr, false))
Julian Anastasova210d012008-10-01 07:28:28 -07002522 goto out;
Julian Anastasova210d012008-10-01 07:28:28 -07002523 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002524 }
2525
2526
David S. Miller68a5e3d2011-03-11 20:07:33 -05002527 if (oldflp4->flowi4_oif) {
2528 dev_out = dev_get_by_index_rcu(net, oldflp4->flowi4_oif);
David S. Millerb23dd4f2011-03-02 14:31:35 -08002529 rth = ERR_PTR(-ENODEV);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002530 if (dev_out == NULL)
2531 goto out;
Herbert Xue5ed6392005-10-03 14:35:55 -07002532
2533 /* RACE: Check return value of inet_select_addr instead. */
Eric Dumazetfc75fc82010-12-22 04:39:39 +00002534 if (!(dev_out->flags & IFF_UP) || !__in_dev_get_rcu(dev_out)) {
David S. Millerb23dd4f2011-03-02 14:31:35 -08002535 rth = ERR_PTR(-ENETUNREACH);
Eric Dumazetfc75fc82010-12-22 04:39:39 +00002536 goto out;
2537 }
David S. Miller68a5e3d2011-03-11 20:07:33 -05002538 if (ipv4_is_local_multicast(oldflp4->daddr) ||
2539 ipv4_is_lbcast(oldflp4->daddr)) {
2540 if (!fl4.saddr)
2541 fl4.saddr = inet_select_addr(dev_out, 0,
2542 RT_SCOPE_LINK);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002543 goto make_route;
2544 }
David S. Miller68a5e3d2011-03-11 20:07:33 -05002545 if (!fl4.saddr) {
2546 if (ipv4_is_multicast(oldflp4->daddr))
2547 fl4.saddr = inet_select_addr(dev_out, 0,
2548 fl4.flowi4_scope);
2549 else if (!oldflp4->daddr)
2550 fl4.saddr = inet_select_addr(dev_out, 0,
2551 RT_SCOPE_HOST);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002552 }
2553 }
2554
David S. Miller68a5e3d2011-03-11 20:07:33 -05002555 if (!fl4.daddr) {
2556 fl4.daddr = fl4.saddr;
2557 if (!fl4.daddr)
2558 fl4.daddr = fl4.saddr = htonl(INADDR_LOOPBACK);
Denis V. Lunevb40afd02008-01-22 22:06:19 -08002559 dev_out = net->loopback_dev;
David S. Miller68a5e3d2011-03-11 20:07:33 -05002560 fl4.flowi4_oif = net->loopback_dev->ifindex;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002561 res.type = RTN_LOCAL;
2562 flags |= RTCF_LOCAL;
2563 goto make_route;
2564 }
2565
David S. Miller68a5e3d2011-03-11 20:07:33 -05002566 if (fib_lookup(net, &fl4, &res)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07002567 res.fi = NULL;
David S. Miller68a5e3d2011-03-11 20:07:33 -05002568 if (oldflp4->flowi4_oif) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07002569 /* Apparently, routing tables are wrong. Assume,
2570 that the destination is on link.
2571
2572 WHY? DW.
2573 Because we are allowed to send to iface
2574 even if it has NO routes and NO assigned
2575 addresses. When oif is specified, routing
2576 tables are looked up with only one purpose:
2577 to catch if destination is gatewayed, rather than
2578 direct. Moreover, if MSG_DONTROUTE is set,
2579 we send packet, ignoring both routing tables
2580 and ifaddr state. --ANK
2581
2582
2583 We could make it even if oif is unknown,
2584 likely IPv6, but we do not.
2585 */
2586
David S. Miller68a5e3d2011-03-11 20:07:33 -05002587 if (fl4.saddr == 0)
2588 fl4.saddr = inet_select_addr(dev_out, 0,
2589 RT_SCOPE_LINK);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002590 res.type = RTN_UNICAST;
2591 goto make_route;
2592 }
David S. Millerb23dd4f2011-03-02 14:31:35 -08002593 rth = ERR_PTR(-ENETUNREACH);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002594 goto out;
2595 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002596
2597 if (res.type == RTN_LOCAL) {
David S. Miller68a5e3d2011-03-11 20:07:33 -05002598 if (!fl4.saddr) {
Joel Sing9fc3bbb2011-01-03 20:24:20 +00002599 if (res.fi->fib_prefsrc)
David S. Miller68a5e3d2011-03-11 20:07:33 -05002600 fl4.saddr = res.fi->fib_prefsrc;
Joel Sing9fc3bbb2011-01-03 20:24:20 +00002601 else
David S. Miller68a5e3d2011-03-11 20:07:33 -05002602 fl4.saddr = fl4.daddr;
Joel Sing9fc3bbb2011-01-03 20:24:20 +00002603 }
Denis V. Lunevb40afd02008-01-22 22:06:19 -08002604 dev_out = net->loopback_dev;
David S. Miller68a5e3d2011-03-11 20:07:33 -05002605 fl4.flowi4_oif = dev_out->ifindex;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002606 res.fi = NULL;
2607 flags |= RTCF_LOCAL;
2608 goto make_route;
2609 }
2610
2611#ifdef CONFIG_IP_ROUTE_MULTIPATH
David S. Miller68a5e3d2011-03-11 20:07:33 -05002612 if (res.fi->fib_nhs > 1 && fl4.flowi4_oif == 0)
David S. Miller1b7fe5932011-03-10 17:01:16 -08002613 fib_select_multipath(&res);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002614 else
2615#endif
David S. Miller68a5e3d2011-03-11 20:07:33 -05002616 if (!res.prefixlen && res.type == RTN_UNICAST && !fl4.flowi4_oif)
David S. Miller0c838ff2011-01-31 16:16:50 -08002617 fib_select_default(&res);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002618
David S. Miller68a5e3d2011-03-11 20:07:33 -05002619 if (!fl4.saddr)
2620 fl4.saddr = FIB_RES_PREFSRC(res);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002621
Linus Torvalds1da177e2005-04-16 15:20:36 -07002622 dev_out = FIB_RES_DEV(res);
David S. Miller68a5e3d2011-03-11 20:07:33 -05002623 fl4.flowi4_oif = dev_out->ifindex;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002624
2625
2626make_route:
David S. Miller68a5e3d2011-03-11 20:07:33 -05002627 rth = __mkroute_output(&res, &fl4, oldflp4, dev_out, flags);
David S. Millerb23dd4f2011-03-02 14:31:35 -08002628 if (!IS_ERR(rth)) {
David S. Miller5ada5522011-02-17 15:29:00 -08002629 unsigned int hash;
2630
David S. Miller68a5e3d2011-03-11 20:07:33 -05002631 hash = rt_hash(oldflp4->daddr, oldflp4->saddr, oldflp4->flowi4_oif,
David S. Miller5ada5522011-02-17 15:29:00 -08002632 rt_genid(dev_net(dev_out)));
David S. Miller68a5e3d2011-03-11 20:07:33 -05002633 rth = rt_intern_hash(hash, rth, NULL, oldflp4->flowi4_oif);
David S. Miller5ada5522011-02-17 15:29:00 -08002634 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002635
David S. Miller010c2702011-02-17 15:37:09 -08002636out:
2637 rcu_read_unlock();
David S. Millerb23dd4f2011-03-02 14:31:35 -08002638 return rth;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002639}
2640
David S. Miller9d6ec932011-03-12 01:12:47 -05002641struct rtable *__ip_route_output_key(struct net *net, const struct flowi4 *flp4)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002642{
Linus Torvalds1da177e2005-04-16 15:20:36 -07002643 struct rtable *rth;
David S. Miller010c2702011-02-17 15:37:09 -08002644 unsigned int hash;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002645
Neil Horman1080d702008-10-27 12:28:25 -07002646 if (!rt_caching(net))
2647 goto slow_output;
2648
David S. Miller9d6ec932011-03-12 01:12:47 -05002649 hash = rt_hash(flp4->daddr, flp4->saddr, flp4->flowi4_oif, rt_genid(net));
Linus Torvalds1da177e2005-04-16 15:20:36 -07002650
2651 rcu_read_lock_bh();
Paul E. McKenneya898def2010-02-22 17:04:49 -08002652 for (rth = rcu_dereference_bh(rt_hash_table[hash].chain); rth;
Changli Gaod8d1f302010-06-10 23:31:35 -07002653 rth = rcu_dereference_bh(rth->dst.rt_next)) {
David S. Miller9d6ec932011-03-12 01:12:47 -05002654 if (rth->rt_key_dst == flp4->daddr &&
2655 rth->rt_key_src == flp4->saddr &&
David S. Millerc7537962010-11-11 17:07:48 -08002656 rt_is_output_route(rth) &&
David S. Miller9d6ec932011-03-12 01:12:47 -05002657 rth->rt_oif == flp4->flowi4_oif &&
2658 rth->rt_mark == flp4->flowi4_mark &&
2659 !((rth->rt_tos ^ flp4->flowi4_tos) &
Denis V. Lunevb5921912008-01-22 23:50:25 -08002660 (IPTOS_RT_MASK | RTO_ONLINK)) &&
Changli Gaod8d1f302010-06-10 23:31:35 -07002661 net_eq(dev_net(rth->dst.dev), net) &&
Denis V. Luneve84f84f2008-07-05 19:04:32 -07002662 !rt_is_expired(rth)) {
Changli Gaod8d1f302010-06-10 23:31:35 -07002663 dst_use(&rth->dst, jiffies);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002664 RT_CACHE_STAT_INC(out_hit);
2665 rcu_read_unlock_bh();
David S. Millerb23dd4f2011-03-02 14:31:35 -08002666 return rth;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002667 }
2668 RT_CACHE_STAT_INC(out_hlist_search);
2669 }
2670 rcu_read_unlock_bh();
2671
Neil Horman1080d702008-10-27 12:28:25 -07002672slow_output:
David S. Miller9d6ec932011-03-12 01:12:47 -05002673 return ip_route_output_slow(net, flp4);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002674}
Arnaldo Carvalho de Melod8c97a92005-08-09 20:12:12 -07002675EXPORT_SYMBOL_GPL(__ip_route_output_key);
2676
Jianzhao Wangae2688d2010-09-08 14:35:43 -07002677static struct dst_entry *ipv4_blackhole_dst_check(struct dst_entry *dst, u32 cookie)
2678{
2679 return NULL;
2680}
2681
Roland Dreierec831ea2011-01-31 13:16:00 -08002682static unsigned int ipv4_blackhole_default_mtu(const struct dst_entry *dst)
2683{
2684 return 0;
2685}
2686
David S. Miller14e50e52007-05-24 18:17:54 -07002687static void ipv4_rt_blackhole_update_pmtu(struct dst_entry *dst, u32 mtu)
2688{
2689}
2690
2691static struct dst_ops ipv4_dst_blackhole_ops = {
2692 .family = AF_INET,
Harvey Harrison09640e62009-02-01 00:45:17 -08002693 .protocol = cpu_to_be16(ETH_P_IP),
David S. Miller14e50e52007-05-24 18:17:54 -07002694 .destroy = ipv4_dst_destroy,
Jianzhao Wangae2688d2010-09-08 14:35:43 -07002695 .check = ipv4_blackhole_dst_check,
Roland Dreierec831ea2011-01-31 13:16:00 -08002696 .default_mtu = ipv4_blackhole_default_mtu,
Eric Dumazet214f45c2011-02-18 11:39:01 -08002697 .default_advmss = ipv4_default_advmss,
David S. Miller14e50e52007-05-24 18:17:54 -07002698 .update_pmtu = ipv4_rt_blackhole_update_pmtu,
David S. Miller14e50e52007-05-24 18:17:54 -07002699};
2700
David S. Miller2774c132011-03-01 14:59:04 -08002701struct dst_entry *ipv4_blackhole_route(struct net *net, struct dst_entry *dst_orig)
David S. Miller14e50e52007-05-24 18:17:54 -07002702{
David S. Miller2774c132011-03-01 14:59:04 -08002703 struct rtable *rt = dst_alloc(&ipv4_dst_blackhole_ops, 1);
2704 struct rtable *ort = (struct rtable *) dst_orig;
David S. Miller14e50e52007-05-24 18:17:54 -07002705
2706 if (rt) {
Changli Gaod8d1f302010-06-10 23:31:35 -07002707 struct dst_entry *new = &rt->dst;
David S. Miller14e50e52007-05-24 18:17:54 -07002708
David S. Miller14e50e52007-05-24 18:17:54 -07002709 new->__use = 1;
Herbert Xu352e5122007-11-13 21:34:06 -08002710 new->input = dst_discard;
2711 new->output = dst_discard;
David S. Millerdefb3512010-12-08 21:16:57 -08002712 dst_copy_metrics(new, &ort->dst);
David S. Miller14e50e52007-05-24 18:17:54 -07002713
Changli Gaod8d1f302010-06-10 23:31:35 -07002714 new->dev = ort->dst.dev;
David S. Miller14e50e52007-05-24 18:17:54 -07002715 if (new->dev)
2716 dev_hold(new->dev);
2717
David S. Miller5e2b61f2011-03-04 21:47:09 -08002718 rt->rt_key_dst = ort->rt_key_dst;
2719 rt->rt_key_src = ort->rt_key_src;
2720 rt->rt_tos = ort->rt_tos;
2721 rt->rt_iif = ort->rt_iif;
2722 rt->rt_oif = ort->rt_oif;
2723 rt->rt_mark = ort->rt_mark;
David S. Miller14e50e52007-05-24 18:17:54 -07002724
Denis V. Luneve84f84f2008-07-05 19:04:32 -07002725 rt->rt_genid = rt_genid(net);
David S. Miller14e50e52007-05-24 18:17:54 -07002726 rt->rt_flags = ort->rt_flags;
2727 rt->rt_type = ort->rt_type;
2728 rt->rt_dst = ort->rt_dst;
2729 rt->rt_src = ort->rt_src;
2730 rt->rt_iif = ort->rt_iif;
2731 rt->rt_gateway = ort->rt_gateway;
2732 rt->rt_spec_dst = ort->rt_spec_dst;
2733 rt->peer = ort->peer;
2734 if (rt->peer)
2735 atomic_inc(&rt->peer->refcnt);
David S. Miller62fa8a82011-01-26 20:51:05 -08002736 rt->fi = ort->fi;
2737 if (rt->fi)
2738 atomic_inc(&rt->fi->fib_clntref);
David S. Miller14e50e52007-05-24 18:17:54 -07002739
2740 dst_free(new);
2741 }
2742
David S. Miller2774c132011-03-01 14:59:04 -08002743 dst_release(dst_orig);
2744
2745 return rt ? &rt->dst : ERR_PTR(-ENOMEM);
David S. Miller14e50e52007-05-24 18:17:54 -07002746}
2747
David S. Miller9d6ec932011-03-12 01:12:47 -05002748struct rtable *ip_route_output_flow(struct net *net, struct flowi4 *flp4,
David S. Millerb23dd4f2011-03-02 14:31:35 -08002749 struct sock *sk)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002750{
David S. Miller9d6ec932011-03-12 01:12:47 -05002751 struct rtable *rt = __ip_route_output_key(net, flp4);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002752
David S. Millerb23dd4f2011-03-02 14:31:35 -08002753 if (IS_ERR(rt))
2754 return rt;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002755
David S. Miller9d6ec932011-03-12 01:12:47 -05002756 if (flp4->flowi4_proto) {
2757 if (!flp4->saddr)
2758 flp4->saddr = rt->rt_src;
2759 if (!flp4->daddr)
2760 flp4->daddr = rt->rt_dst;
2761 rt = (struct rtable *) xfrm_lookup(net, &rt->dst,
2762 flowi4_to_flowi(flp4),
2763 sk, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002764 }
2765
David S. Millerb23dd4f2011-03-02 14:31:35 -08002766 return rt;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002767}
Arnaldo Carvalho de Melod8c97a92005-08-09 20:12:12 -07002768EXPORT_SYMBOL_GPL(ip_route_output_flow);
2769
Benjamin Thery4feb88e2009-01-22 04:56:23 +00002770static int rt_fill_info(struct net *net,
2771 struct sk_buff *skb, u32 pid, u32 seq, int event,
Jamal Hadi Salimb6544c02005-06-18 22:54:12 -07002772 int nowait, unsigned int flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002773{
Eric Dumazet511c3f92009-06-02 05:14:27 +00002774 struct rtable *rt = skb_rtable(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002775 struct rtmsg *r;
Thomas Grafbe403ea2006-08-17 18:15:17 -07002776 struct nlmsghdr *nlh;
Thomas Grafe3703b32006-11-27 09:27:07 -08002777 long expires;
2778 u32 id = 0, ts = 0, tsage = 0, error;
Thomas Grafbe403ea2006-08-17 18:15:17 -07002779
2780 nlh = nlmsg_put(skb, pid, seq, event, sizeof(*r), flags);
2781 if (nlh == NULL)
Patrick McHardy26932562007-01-31 23:16:40 -08002782 return -EMSGSIZE;
Thomas Grafbe403ea2006-08-17 18:15:17 -07002783
2784 r = nlmsg_data(nlh);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002785 r->rtm_family = AF_INET;
2786 r->rtm_dst_len = 32;
2787 r->rtm_src_len = 0;
David S. Miller5e2b61f2011-03-04 21:47:09 -08002788 r->rtm_tos = rt->rt_tos;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002789 r->rtm_table = RT_TABLE_MAIN;
Thomas Grafbe403ea2006-08-17 18:15:17 -07002790 NLA_PUT_U32(skb, RTA_TABLE, RT_TABLE_MAIN);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002791 r->rtm_type = rt->rt_type;
2792 r->rtm_scope = RT_SCOPE_UNIVERSE;
2793 r->rtm_protocol = RTPROT_UNSPEC;
2794 r->rtm_flags = (rt->rt_flags & ~0xFFFF) | RTM_F_CLONED;
2795 if (rt->rt_flags & RTCF_NOTIFY)
2796 r->rtm_flags |= RTM_F_NOTIFY;
Thomas Grafbe403ea2006-08-17 18:15:17 -07002797
Al Viro17fb2c62006-09-26 22:15:25 -07002798 NLA_PUT_BE32(skb, RTA_DST, rt->rt_dst);
Thomas Grafbe403ea2006-08-17 18:15:17 -07002799
David S. Miller5e2b61f2011-03-04 21:47:09 -08002800 if (rt->rt_key_src) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07002801 r->rtm_src_len = 32;
David S. Miller5e2b61f2011-03-04 21:47:09 -08002802 NLA_PUT_BE32(skb, RTA_SRC, rt->rt_key_src);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002803 }
Changli Gaod8d1f302010-06-10 23:31:35 -07002804 if (rt->dst.dev)
2805 NLA_PUT_U32(skb, RTA_OIF, rt->dst.dev->ifindex);
Patrick McHardyc7066f72011-01-14 13:36:42 +01002806#ifdef CONFIG_IP_ROUTE_CLASSID
Changli Gaod8d1f302010-06-10 23:31:35 -07002807 if (rt->dst.tclassid)
2808 NLA_PUT_U32(skb, RTA_FLOW, rt->dst.tclassid);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002809#endif
David S. Millerc7537962010-11-11 17:07:48 -08002810 if (rt_is_input_route(rt))
Al Viro17fb2c62006-09-26 22:15:25 -07002811 NLA_PUT_BE32(skb, RTA_PREFSRC, rt->rt_spec_dst);
David S. Miller5e2b61f2011-03-04 21:47:09 -08002812 else if (rt->rt_src != rt->rt_key_src)
Al Viro17fb2c62006-09-26 22:15:25 -07002813 NLA_PUT_BE32(skb, RTA_PREFSRC, rt->rt_src);
Thomas Grafbe403ea2006-08-17 18:15:17 -07002814
Linus Torvalds1da177e2005-04-16 15:20:36 -07002815 if (rt->rt_dst != rt->rt_gateway)
Al Viro17fb2c62006-09-26 22:15:25 -07002816 NLA_PUT_BE32(skb, RTA_GATEWAY, rt->rt_gateway);
Thomas Grafbe403ea2006-08-17 18:15:17 -07002817
David S. Millerdefb3512010-12-08 21:16:57 -08002818 if (rtnetlink_put_metrics(skb, dst_metrics_ptr(&rt->dst)) < 0)
Thomas Grafbe403ea2006-08-17 18:15:17 -07002819 goto nla_put_failure;
2820
David S. Miller5e2b61f2011-03-04 21:47:09 -08002821 if (rt->rt_mark)
2822 NLA_PUT_BE32(skb, RTA_MARK, rt->rt_mark);
Eric Dumazet963bfee2010-07-20 22:03:14 +00002823
Changli Gaod8d1f302010-06-10 23:31:35 -07002824 error = rt->dst.error;
David S. Miller2c8cec52011-02-09 20:42:07 -08002825 expires = (rt->peer && rt->peer->pmtu_expires) ?
2826 rt->peer->pmtu_expires - jiffies : 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002827 if (rt->peer) {
Eric Dumazet317fe0e2010-06-16 04:52:13 +00002828 inet_peer_refcheck(rt->peer);
Eric Dumazet2c1409a2009-11-12 09:33:09 +00002829 id = atomic_read(&rt->peer->ip_id_count) & 0xffff;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002830 if (rt->peer->tcp_ts_stamp) {
Thomas Grafe3703b32006-11-27 09:27:07 -08002831 ts = rt->peer->tcp_ts;
James Morris9d729f72007-03-04 16:12:44 -08002832 tsage = get_seconds() - rt->peer->tcp_ts_stamp;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002833 }
2834 }
Thomas Grafbe403ea2006-08-17 18:15:17 -07002835
David S. Millerc7537962010-11-11 17:07:48 -08002836 if (rt_is_input_route(rt)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07002837#ifdef CONFIG_IP_MROUTE
Al Viroe4485152006-09-26 22:15:01 -07002838 __be32 dst = rt->rt_dst;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002839
Joe Perchesf97c1e02007-12-16 13:45:43 -08002840 if (ipv4_is_multicast(dst) && !ipv4_is_local_multicast(dst) &&
Benjamin Thery4feb88e2009-01-22 04:56:23 +00002841 IPV4_DEVCONF_ALL(net, MC_FORWARDING)) {
2842 int err = ipmr_get_route(net, skb, r, nowait);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002843 if (err <= 0) {
2844 if (!nowait) {
2845 if (err == 0)
2846 return 0;
Thomas Grafbe403ea2006-08-17 18:15:17 -07002847 goto nla_put_failure;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002848 } else {
2849 if (err == -EMSGSIZE)
Thomas Grafbe403ea2006-08-17 18:15:17 -07002850 goto nla_put_failure;
Thomas Grafe3703b32006-11-27 09:27:07 -08002851 error = err;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002852 }
2853 }
2854 } else
2855#endif
David S. Miller5e2b61f2011-03-04 21:47:09 -08002856 NLA_PUT_U32(skb, RTA_IIF, rt->rt_iif);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002857 }
2858
Changli Gaod8d1f302010-06-10 23:31:35 -07002859 if (rtnl_put_cacheinfo(skb, &rt->dst, id, ts, tsage,
Thomas Grafe3703b32006-11-27 09:27:07 -08002860 expires, error) < 0)
2861 goto nla_put_failure;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002862
Thomas Grafbe403ea2006-08-17 18:15:17 -07002863 return nlmsg_end(skb, nlh);
2864
2865nla_put_failure:
Patrick McHardy26932562007-01-31 23:16:40 -08002866 nlmsg_cancel(skb, nlh);
2867 return -EMSGSIZE;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002868}
2869
Thomas Graf63f34442007-03-22 11:55:17 -07002870static int inet_rtm_getroute(struct sk_buff *in_skb, struct nlmsghdr* nlh, void *arg)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002871{
YOSHIFUJI Hideaki3b1e0a62008-03-26 02:26:21 +09002872 struct net *net = sock_net(in_skb->sk);
Thomas Grafd889ce32006-08-17 18:15:44 -07002873 struct rtmsg *rtm;
2874 struct nlattr *tb[RTA_MAX+1];
Linus Torvalds1da177e2005-04-16 15:20:36 -07002875 struct rtable *rt = NULL;
Al Viro9e12bb22006-09-26 21:25:20 -07002876 __be32 dst = 0;
2877 __be32 src = 0;
2878 u32 iif;
Thomas Grafd889ce32006-08-17 18:15:44 -07002879 int err;
Eric Dumazet963bfee2010-07-20 22:03:14 +00002880 int mark;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002881 struct sk_buff *skb;
2882
Thomas Grafd889ce32006-08-17 18:15:44 -07002883 err = nlmsg_parse(nlh, sizeof(*rtm), tb, RTA_MAX, rtm_ipv4_policy);
2884 if (err < 0)
2885 goto errout;
2886
2887 rtm = nlmsg_data(nlh);
2888
Linus Torvalds1da177e2005-04-16 15:20:36 -07002889 skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL);
Thomas Grafd889ce32006-08-17 18:15:44 -07002890 if (skb == NULL) {
2891 err = -ENOBUFS;
2892 goto errout;
2893 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002894
2895 /* Reserve room for dummy headers, this skb can pass
2896 through good chunk of routing engine.
2897 */
Arnaldo Carvalho de Melo459a98e2007-03-19 15:30:44 -07002898 skb_reset_mac_header(skb);
Arnaldo Carvalho de Meloc1d2bbe2007-04-10 20:45:18 -07002899 skb_reset_network_header(skb);
Stephen Hemmingerd2c962b2006-04-17 17:27:11 -07002900
2901 /* Bugfix: need to give ip_route_input enough of an IP header to not gag. */
Arnaldo Carvalho de Meloeddc9ec2007-04-20 22:47:35 -07002902 ip_hdr(skb)->protocol = IPPROTO_ICMP;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002903 skb_reserve(skb, MAX_HEADER + sizeof(struct iphdr));
2904
Al Viro17fb2c62006-09-26 22:15:25 -07002905 src = tb[RTA_SRC] ? nla_get_be32(tb[RTA_SRC]) : 0;
2906 dst = tb[RTA_DST] ? nla_get_be32(tb[RTA_DST]) : 0;
Thomas Grafd889ce32006-08-17 18:15:44 -07002907 iif = tb[RTA_IIF] ? nla_get_u32(tb[RTA_IIF]) : 0;
Eric Dumazet963bfee2010-07-20 22:03:14 +00002908 mark = tb[RTA_MARK] ? nla_get_u32(tb[RTA_MARK]) : 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002909
2910 if (iif) {
Thomas Grafd889ce32006-08-17 18:15:44 -07002911 struct net_device *dev;
2912
Denis V. Lunev19375042008-02-28 20:52:04 -08002913 dev = __dev_get_by_index(net, iif);
Thomas Grafd889ce32006-08-17 18:15:44 -07002914 if (dev == NULL) {
2915 err = -ENODEV;
2916 goto errout_free;
2917 }
2918
Linus Torvalds1da177e2005-04-16 15:20:36 -07002919 skb->protocol = htons(ETH_P_IP);
2920 skb->dev = dev;
Eric Dumazet963bfee2010-07-20 22:03:14 +00002921 skb->mark = mark;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002922 local_bh_disable();
2923 err = ip_route_input(skb, dst, src, rtm->rtm_tos, dev);
2924 local_bh_enable();
Thomas Grafd889ce32006-08-17 18:15:44 -07002925
Eric Dumazet511c3f92009-06-02 05:14:27 +00002926 rt = skb_rtable(skb);
Changli Gaod8d1f302010-06-10 23:31:35 -07002927 if (err == 0 && rt->dst.error)
2928 err = -rt->dst.error;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002929 } else {
David S. Miller68a5e3d2011-03-11 20:07:33 -05002930 struct flowi4 fl4 = {
2931 .daddr = dst,
2932 .saddr = src,
2933 .flowi4_tos = rtm->rtm_tos,
2934 .flowi4_oif = tb[RTA_OIF] ? nla_get_u32(tb[RTA_OIF]) : 0,
2935 .flowi4_mark = mark,
Thomas Grafd889ce32006-08-17 18:15:44 -07002936 };
David S. Miller9d6ec932011-03-12 01:12:47 -05002937 rt = ip_route_output_key(net, &fl4);
David S. Millerb23dd4f2011-03-02 14:31:35 -08002938
2939 err = 0;
2940 if (IS_ERR(rt))
2941 err = PTR_ERR(rt);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002942 }
Thomas Grafd889ce32006-08-17 18:15:44 -07002943
Linus Torvalds1da177e2005-04-16 15:20:36 -07002944 if (err)
Thomas Grafd889ce32006-08-17 18:15:44 -07002945 goto errout_free;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002946
Changli Gaod8d1f302010-06-10 23:31:35 -07002947 skb_dst_set(skb, &rt->dst);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002948 if (rtm->rtm_flags & RTM_F_NOTIFY)
2949 rt->rt_flags |= RTCF_NOTIFY;
2950
Benjamin Thery4feb88e2009-01-22 04:56:23 +00002951 err = rt_fill_info(net, skb, NETLINK_CB(in_skb).pid, nlh->nlmsg_seq,
Denis V. Lunev19375042008-02-28 20:52:04 -08002952 RTM_NEWROUTE, 0, 0);
Thomas Grafd889ce32006-08-17 18:15:44 -07002953 if (err <= 0)
2954 goto errout_free;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002955
Denis V. Lunev19375042008-02-28 20:52:04 -08002956 err = rtnl_unicast(skb, net, NETLINK_CB(in_skb).pid);
Thomas Grafd889ce32006-08-17 18:15:44 -07002957errout:
Thomas Graf2942e902006-08-15 00:30:25 -07002958 return err;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002959
Thomas Grafd889ce32006-08-17 18:15:44 -07002960errout_free:
Linus Torvalds1da177e2005-04-16 15:20:36 -07002961 kfree_skb(skb);
Thomas Grafd889ce32006-08-17 18:15:44 -07002962 goto errout;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002963}
2964
2965int ip_rt_dump(struct sk_buff *skb, struct netlink_callback *cb)
2966{
2967 struct rtable *rt;
2968 int h, s_h;
2969 int idx, s_idx;
Denis V. Lunev19375042008-02-28 20:52:04 -08002970 struct net *net;
2971
YOSHIFUJI Hideaki3b1e0a62008-03-26 02:26:21 +09002972 net = sock_net(skb->sk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002973
2974 s_h = cb->args[0];
Eric Dumazetd8c92832008-01-07 21:52:14 -08002975 if (s_h < 0)
2976 s_h = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002977 s_idx = idx = cb->args[1];
Eric Dumazeta6272662008-08-28 01:11:25 -07002978 for (h = s_h; h <= rt_hash_mask; h++, s_idx = 0) {
2979 if (!rt_hash_table[h].chain)
2980 continue;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002981 rcu_read_lock_bh();
Paul E. McKenneya898def2010-02-22 17:04:49 -08002982 for (rt = rcu_dereference_bh(rt_hash_table[h].chain), idx = 0; rt;
Changli Gaod8d1f302010-06-10 23:31:35 -07002983 rt = rcu_dereference_bh(rt->dst.rt_next), idx++) {
2984 if (!net_eq(dev_net(rt->dst.dev), net) || idx < s_idx)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002985 continue;
Denis V. Luneve84f84f2008-07-05 19:04:32 -07002986 if (rt_is_expired(rt))
Eric Dumazet29e75252008-01-31 17:05:09 -08002987 continue;
Changli Gaod8d1f302010-06-10 23:31:35 -07002988 skb_dst_set_noref(skb, &rt->dst);
Benjamin Thery4feb88e2009-01-22 04:56:23 +00002989 if (rt_fill_info(net, skb, NETLINK_CB(cb->skb).pid,
YOSHIFUJI Hideakie905a9e2007-02-09 23:24:47 +09002990 cb->nlh->nlmsg_seq, RTM_NEWROUTE,
Jamal Hadi Salimb6544c02005-06-18 22:54:12 -07002991 1, NLM_F_MULTI) <= 0) {
Eric Dumazetadf30902009-06-02 05:19:30 +00002992 skb_dst_drop(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002993 rcu_read_unlock_bh();
2994 goto done;
2995 }
Eric Dumazetadf30902009-06-02 05:19:30 +00002996 skb_dst_drop(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002997 }
2998 rcu_read_unlock_bh();
2999 }
3000
3001done:
3002 cb->args[0] = h;
3003 cb->args[1] = idx;
3004 return skb->len;
3005}
3006
3007void ip_rt_multicast_event(struct in_device *in_dev)
3008{
Denis V. Lunev76e6ebf2008-07-05 19:00:44 -07003009 rt_cache_flush(dev_net(in_dev->dev), 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003010}
3011
3012#ifdef CONFIG_SYSCTL
Denis V. Lunev81c684d2008-07-08 03:05:28 -07003013static int ipv4_sysctl_rtcache_flush(ctl_table *__ctl, int write,
Alexey Dobriyan8d65af72009-09-23 15:57:19 -07003014 void __user *buffer,
Linus Torvalds1da177e2005-04-16 15:20:36 -07003015 size_t *lenp, loff_t *ppos)
3016{
3017 if (write) {
Denis V. Lunev639e1042008-07-05 19:02:06 -07003018 int flush_delay;
Denis V. Lunev81c684d2008-07-08 03:05:28 -07003019 ctl_table ctl;
Denis V. Lunev39a23e72008-07-05 19:02:33 -07003020 struct net *net;
Denis V. Lunev639e1042008-07-05 19:02:06 -07003021
Denis V. Lunev81c684d2008-07-08 03:05:28 -07003022 memcpy(&ctl, __ctl, sizeof(ctl));
3023 ctl.data = &flush_delay;
Alexey Dobriyan8d65af72009-09-23 15:57:19 -07003024 proc_dointvec(&ctl, write, buffer, lenp, ppos);
Denis V. Lunev639e1042008-07-05 19:02:06 -07003025
Denis V. Lunev81c684d2008-07-08 03:05:28 -07003026 net = (struct net *)__ctl->extra1;
Denis V. Lunev39a23e72008-07-05 19:02:33 -07003027 rt_cache_flush(net, flush_delay);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003028 return 0;
YOSHIFUJI Hideakie905a9e2007-02-09 23:24:47 +09003029 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07003030
3031 return -EINVAL;
3032}
3033
Al Viroeeb61f72008-07-27 08:59:33 +01003034static ctl_table ipv4_route_table[] = {
YOSHIFUJI Hideakie905a9e2007-02-09 23:24:47 +09003035 {
Linus Torvalds1da177e2005-04-16 15:20:36 -07003036 .procname = "gc_thresh",
3037 .data = &ipv4_dst_ops.gc_thresh,
3038 .maxlen = sizeof(int),
3039 .mode = 0644,
Alexey Dobriyan6d9f2392008-11-03 18:21:05 -08003040 .proc_handler = proc_dointvec,
Linus Torvalds1da177e2005-04-16 15:20:36 -07003041 },
3042 {
Linus Torvalds1da177e2005-04-16 15:20:36 -07003043 .procname = "max_size",
3044 .data = &ip_rt_max_size,
3045 .maxlen = sizeof(int),
3046 .mode = 0644,
Alexey Dobriyan6d9f2392008-11-03 18:21:05 -08003047 .proc_handler = proc_dointvec,
Linus Torvalds1da177e2005-04-16 15:20:36 -07003048 },
3049 {
3050 /* Deprecated. Use gc_min_interval_ms */
YOSHIFUJI Hideakie905a9e2007-02-09 23:24:47 +09003051
Linus Torvalds1da177e2005-04-16 15:20:36 -07003052 .procname = "gc_min_interval",
3053 .data = &ip_rt_gc_min_interval,
3054 .maxlen = sizeof(int),
3055 .mode = 0644,
Alexey Dobriyan6d9f2392008-11-03 18:21:05 -08003056 .proc_handler = proc_dointvec_jiffies,
Linus Torvalds1da177e2005-04-16 15:20:36 -07003057 },
3058 {
Linus Torvalds1da177e2005-04-16 15:20:36 -07003059 .procname = "gc_min_interval_ms",
3060 .data = &ip_rt_gc_min_interval,
3061 .maxlen = sizeof(int),
3062 .mode = 0644,
Alexey Dobriyan6d9f2392008-11-03 18:21:05 -08003063 .proc_handler = proc_dointvec_ms_jiffies,
Linus Torvalds1da177e2005-04-16 15:20:36 -07003064 },
3065 {
Linus Torvalds1da177e2005-04-16 15:20:36 -07003066 .procname = "gc_timeout",
3067 .data = &ip_rt_gc_timeout,
3068 .maxlen = sizeof(int),
3069 .mode = 0644,
Alexey Dobriyan6d9f2392008-11-03 18:21:05 -08003070 .proc_handler = proc_dointvec_jiffies,
Linus Torvalds1da177e2005-04-16 15:20:36 -07003071 },
3072 {
Linus Torvalds1da177e2005-04-16 15:20:36 -07003073 .procname = "gc_interval",
3074 .data = &ip_rt_gc_interval,
3075 .maxlen = sizeof(int),
3076 .mode = 0644,
Alexey Dobriyan6d9f2392008-11-03 18:21:05 -08003077 .proc_handler = proc_dointvec_jiffies,
Linus Torvalds1da177e2005-04-16 15:20:36 -07003078 },
3079 {
Linus Torvalds1da177e2005-04-16 15:20:36 -07003080 .procname = "redirect_load",
3081 .data = &ip_rt_redirect_load,
3082 .maxlen = sizeof(int),
3083 .mode = 0644,
Alexey Dobriyan6d9f2392008-11-03 18:21:05 -08003084 .proc_handler = proc_dointvec,
Linus Torvalds1da177e2005-04-16 15:20:36 -07003085 },
3086 {
Linus Torvalds1da177e2005-04-16 15:20:36 -07003087 .procname = "redirect_number",
3088 .data = &ip_rt_redirect_number,
3089 .maxlen = sizeof(int),
3090 .mode = 0644,
Alexey Dobriyan6d9f2392008-11-03 18:21:05 -08003091 .proc_handler = proc_dointvec,
Linus Torvalds1da177e2005-04-16 15:20:36 -07003092 },
3093 {
Linus Torvalds1da177e2005-04-16 15:20:36 -07003094 .procname = "redirect_silence",
3095 .data = &ip_rt_redirect_silence,
3096 .maxlen = sizeof(int),
3097 .mode = 0644,
Alexey Dobriyan6d9f2392008-11-03 18:21:05 -08003098 .proc_handler = proc_dointvec,
Linus Torvalds1da177e2005-04-16 15:20:36 -07003099 },
3100 {
Linus Torvalds1da177e2005-04-16 15:20:36 -07003101 .procname = "error_cost",
3102 .data = &ip_rt_error_cost,
3103 .maxlen = sizeof(int),
3104 .mode = 0644,
Alexey Dobriyan6d9f2392008-11-03 18:21:05 -08003105 .proc_handler = proc_dointvec,
Linus Torvalds1da177e2005-04-16 15:20:36 -07003106 },
3107 {
Linus Torvalds1da177e2005-04-16 15:20:36 -07003108 .procname = "error_burst",
3109 .data = &ip_rt_error_burst,
3110 .maxlen = sizeof(int),
3111 .mode = 0644,
Alexey Dobriyan6d9f2392008-11-03 18:21:05 -08003112 .proc_handler = proc_dointvec,
Linus Torvalds1da177e2005-04-16 15:20:36 -07003113 },
3114 {
Linus Torvalds1da177e2005-04-16 15:20:36 -07003115 .procname = "gc_elasticity",
3116 .data = &ip_rt_gc_elasticity,
3117 .maxlen = sizeof(int),
3118 .mode = 0644,
Alexey Dobriyan6d9f2392008-11-03 18:21:05 -08003119 .proc_handler = proc_dointvec,
Linus Torvalds1da177e2005-04-16 15:20:36 -07003120 },
3121 {
Linus Torvalds1da177e2005-04-16 15:20:36 -07003122 .procname = "mtu_expires",
3123 .data = &ip_rt_mtu_expires,
3124 .maxlen = sizeof(int),
3125 .mode = 0644,
Alexey Dobriyan6d9f2392008-11-03 18:21:05 -08003126 .proc_handler = proc_dointvec_jiffies,
Linus Torvalds1da177e2005-04-16 15:20:36 -07003127 },
3128 {
Linus Torvalds1da177e2005-04-16 15:20:36 -07003129 .procname = "min_pmtu",
3130 .data = &ip_rt_min_pmtu,
3131 .maxlen = sizeof(int),
3132 .mode = 0644,
Alexey Dobriyan6d9f2392008-11-03 18:21:05 -08003133 .proc_handler = proc_dointvec,
Linus Torvalds1da177e2005-04-16 15:20:36 -07003134 },
3135 {
Linus Torvalds1da177e2005-04-16 15:20:36 -07003136 .procname = "min_adv_mss",
3137 .data = &ip_rt_min_advmss,
3138 .maxlen = sizeof(int),
3139 .mode = 0644,
Alexey Dobriyan6d9f2392008-11-03 18:21:05 -08003140 .proc_handler = proc_dointvec,
Linus Torvalds1da177e2005-04-16 15:20:36 -07003141 },
Eric W. Biedermanf8572d82009-11-05 13:32:03 -08003142 { }
Linus Torvalds1da177e2005-04-16 15:20:36 -07003143};
Denis V. Lunev39a23e72008-07-05 19:02:33 -07003144
Al Viro2f4520d2008-08-25 15:17:44 -07003145static struct ctl_table empty[1];
3146
3147static struct ctl_table ipv4_skeleton[] =
3148{
Eric W. Biedermanf8572d82009-11-05 13:32:03 -08003149 { .procname = "route",
Hugh Dickinsd994af02008-08-27 02:35:18 -07003150 .mode = 0555, .child = ipv4_route_table},
Eric W. Biedermanf8572d82009-11-05 13:32:03 -08003151 { .procname = "neigh",
Hugh Dickinsd994af02008-08-27 02:35:18 -07003152 .mode = 0555, .child = empty},
Al Viro2f4520d2008-08-25 15:17:44 -07003153 { }
Denis V. Lunev39a23e72008-07-05 19:02:33 -07003154};
3155
Al Viro2f4520d2008-08-25 15:17:44 -07003156static __net_initdata struct ctl_path ipv4_path[] = {
Eric W. Biedermanf8572d82009-11-05 13:32:03 -08003157 { .procname = "net", },
3158 { .procname = "ipv4", },
Al Viro2f4520d2008-08-25 15:17:44 -07003159 { },
3160};
Denis V. Lunev39a23e72008-07-05 19:02:33 -07003161
3162static struct ctl_table ipv4_route_flush_table[] = {
3163 {
Denis V. Lunev39a23e72008-07-05 19:02:33 -07003164 .procname = "flush",
3165 .maxlen = sizeof(int),
3166 .mode = 0200,
Alexey Dobriyan6d9f2392008-11-03 18:21:05 -08003167 .proc_handler = ipv4_sysctl_rtcache_flush,
Denis V. Lunev39a23e72008-07-05 19:02:33 -07003168 },
Eric W. Biedermanf8572d82009-11-05 13:32:03 -08003169 { },
Denis V. Lunev39a23e72008-07-05 19:02:33 -07003170};
3171
Al Viro2f4520d2008-08-25 15:17:44 -07003172static __net_initdata struct ctl_path ipv4_route_path[] = {
Eric W. Biedermanf8572d82009-11-05 13:32:03 -08003173 { .procname = "net", },
3174 { .procname = "ipv4", },
3175 { .procname = "route", },
Al Viro2f4520d2008-08-25 15:17:44 -07003176 { },
3177};
3178
Denis V. Lunev39a23e72008-07-05 19:02:33 -07003179static __net_init int sysctl_route_net_init(struct net *net)
3180{
3181 struct ctl_table *tbl;
3182
3183 tbl = ipv4_route_flush_table;
Octavian Purdila09ad9bc2009-11-25 15:14:13 -08003184 if (!net_eq(net, &init_net)) {
Denis V. Lunev39a23e72008-07-05 19:02:33 -07003185 tbl = kmemdup(tbl, sizeof(ipv4_route_flush_table), GFP_KERNEL);
3186 if (tbl == NULL)
3187 goto err_dup;
3188 }
3189 tbl[0].extra1 = net;
3190
3191 net->ipv4.route_hdr =
3192 register_net_sysctl_table(net, ipv4_route_path, tbl);
3193 if (net->ipv4.route_hdr == NULL)
3194 goto err_reg;
3195 return 0;
3196
3197err_reg:
3198 if (tbl != ipv4_route_flush_table)
3199 kfree(tbl);
3200err_dup:
3201 return -ENOMEM;
3202}
3203
3204static __net_exit void sysctl_route_net_exit(struct net *net)
3205{
3206 struct ctl_table *tbl;
3207
3208 tbl = net->ipv4.route_hdr->ctl_table_arg;
3209 unregister_net_sysctl_table(net->ipv4.route_hdr);
3210 BUG_ON(tbl == ipv4_route_flush_table);
3211 kfree(tbl);
3212}
3213
3214static __net_initdata struct pernet_operations sysctl_route_ops = {
3215 .init = sysctl_route_net_init,
3216 .exit = sysctl_route_net_exit,
3217};
Linus Torvalds1da177e2005-04-16 15:20:36 -07003218#endif
3219
Neil Horman3ee94372010-05-08 01:57:52 -07003220static __net_init int rt_genid_init(struct net *net)
Denis V. Lunev9f5e97e2008-07-05 19:02:59 -07003221{
Neil Horman3ee94372010-05-08 01:57:52 -07003222 get_random_bytes(&net->ipv4.rt_genid,
3223 sizeof(net->ipv4.rt_genid));
Denis V. Lunev9f5e97e2008-07-05 19:02:59 -07003224 return 0;
3225}
3226
Neil Horman3ee94372010-05-08 01:57:52 -07003227static __net_initdata struct pernet_operations rt_genid_ops = {
3228 .init = rt_genid_init,
Denis V. Lunev9f5e97e2008-07-05 19:02:59 -07003229};
3230
3231
Patrick McHardyc7066f72011-01-14 13:36:42 +01003232#ifdef CONFIG_IP_ROUTE_CLASSID
Tejun Heo7d720c32010-02-16 15:20:26 +00003233struct ip_rt_acct __percpu *ip_rt_acct __read_mostly;
Patrick McHardyc7066f72011-01-14 13:36:42 +01003234#endif /* CONFIG_IP_ROUTE_CLASSID */
Linus Torvalds1da177e2005-04-16 15:20:36 -07003235
3236static __initdata unsigned long rhash_entries;
3237static int __init set_rhash_entries(char *str)
3238{
3239 if (!str)
3240 return 0;
3241 rhash_entries = simple_strtoul(str, &str, 0);
3242 return 1;
3243}
3244__setup("rhash_entries=", set_rhash_entries);
3245
3246int __init ip_rt_init(void)
3247{
Eric Dumazet424c4b72005-07-05 14:58:19 -07003248 int rc = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003249
Patrick McHardyc7066f72011-01-14 13:36:42 +01003250#ifdef CONFIG_IP_ROUTE_CLASSID
Ingo Molnar0dcec8c2009-02-25 14:07:33 +01003251 ip_rt_acct = __alloc_percpu(256 * sizeof(struct ip_rt_acct), __alignof__(struct ip_rt_acct));
Linus Torvalds1da177e2005-04-16 15:20:36 -07003252 if (!ip_rt_acct)
3253 panic("IP: failed to allocate ip_rt_acct\n");
Linus Torvalds1da177e2005-04-16 15:20:36 -07003254#endif
3255
Alexey Dobriyane5d679f332006-08-26 19:25:52 -07003256 ipv4_dst_ops.kmem_cachep =
3257 kmem_cache_create("ip_dst_cache", sizeof(struct rtable), 0,
Paul Mundt20c2df82007-07-20 10:11:58 +09003258 SLAB_HWCACHE_ALIGN|SLAB_PANIC, NULL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003259
David S. Miller14e50e52007-05-24 18:17:54 -07003260 ipv4_dst_blackhole_ops.kmem_cachep = ipv4_dst_ops.kmem_cachep;
3261
Eric Dumazetfc66f952010-10-08 06:37:34 +00003262 if (dst_entries_init(&ipv4_dst_ops) < 0)
3263 panic("IP: failed to allocate ipv4_dst_ops counter\n");
3264
3265 if (dst_entries_init(&ipv4_dst_blackhole_ops) < 0)
3266 panic("IP: failed to allocate ipv4_dst_blackhole_ops counter\n");
3267
Eric Dumazet424c4b72005-07-05 14:58:19 -07003268 rt_hash_table = (struct rt_hash_bucket *)
3269 alloc_large_system_hash("IP route cache",
3270 sizeof(struct rt_hash_bucket),
3271 rhash_entries,
Jan Beulich44813742009-09-21 17:03:05 -07003272 (totalram_pages >= 128 * 1024) ?
Mike Stroyan18955cf2005-11-29 16:12:55 -08003273 15 : 17,
Kirill Korotaev8d1502d2006-08-07 20:44:22 -07003274 0,
Eric Dumazet424c4b72005-07-05 14:58:19 -07003275 &rt_hash_log,
3276 &rt_hash_mask,
Anton Blanchardc9503e02009-04-27 05:42:24 -07003277 rhash_entries ? 0 : 512 * 1024);
Eric Dumazet22c047c2005-07-05 14:55:24 -07003278 memset(rt_hash_table, 0, (rt_hash_mask + 1) * sizeof(struct rt_hash_bucket));
3279 rt_hash_lock_init();
Linus Torvalds1da177e2005-04-16 15:20:36 -07003280
3281 ipv4_dst_ops.gc_thresh = (rt_hash_mask + 1);
3282 ip_rt_max_size = (rt_hash_mask + 1) * 16;
3283
Linus Torvalds1da177e2005-04-16 15:20:36 -07003284 devinet_init();
3285 ip_fib_init();
3286
Denis V. Lunev73b38712008-02-28 20:51:18 -08003287 if (ip_rt_proc_init())
Pavel Emelyanov107f1632007-12-05 21:14:28 -08003288 printk(KERN_ERR "Unable to create route proc files\n");
Linus Torvalds1da177e2005-04-16 15:20:36 -07003289#ifdef CONFIG_XFRM
3290 xfrm_init();
Neil Hormana33bc5c2009-07-30 18:52:15 -07003291 xfrm4_init(ip_rt_max_size);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003292#endif
Thomas Graf63f34442007-03-22 11:55:17 -07003293 rtnl_register(PF_INET, RTM_GETROUTE, inet_rtm_getroute, NULL);
3294
Denis V. Lunev39a23e72008-07-05 19:02:33 -07003295#ifdef CONFIG_SYSCTL
3296 register_pernet_subsys(&sysctl_route_ops);
3297#endif
Neil Horman3ee94372010-05-08 01:57:52 -07003298 register_pernet_subsys(&rt_genid_ops);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003299 return rc;
3300}
3301
Al Viroa1bc6eb2008-07-30 06:32:52 -04003302#ifdef CONFIG_SYSCTL
Al Viroeeb61f72008-07-27 08:59:33 +01003303/*
3304 * We really need to sanitize the damn ipv4 init order, then all
3305 * this nonsense will go away.
3306 */
3307void __init ip_static_sysctl_init(void)
3308{
Al Viro2f4520d2008-08-25 15:17:44 -07003309 register_sysctl_paths(ipv4_path, ipv4_skeleton);
Al Viroeeb61f72008-07-27 08:59:33 +01003310}
Al Viroa1bc6eb2008-07-30 06:32:52 -04003311#endif