blob: 7880af97020885a3ae1b1fdbfbddbeaf9c4d2bbc [file] [log] [blame]
Arnaldo Carvalho de Melo77d8bf92005-08-09 20:00:51 -07001/*
2 * INET An implementation of the TCP/IP protocol suite for the LINUX
3 * operating system. INET is implemented using the BSD Socket
4 * interface as the means of communication with the user level.
5 *
6 * Generic INET transport hashtables
7 *
8 * Authors: Lotsa people, from code originally in tcp
9 *
10 * This program is free software; you can redistribute it and/or
11 * modify it under the terms of the GNU General Public License
12 * as published by the Free Software Foundation; either version
13 * 2 of the License, or (at your option) any later version.
14 */
15
Arnaldo Carvalho de Melo2d8c4ce2005-08-09 20:07:13 -070016#include <linux/module.h>
Arnaldo Carvalho de Meloa7f5e7f2005-12-13 23:25:31 -080017#include <linux/random.h>
Arnaldo Carvalho de Melof3f05f72005-08-09 20:08:09 -070018#include <linux/sched.h>
Arnaldo Carvalho de Melo77d8bf92005-08-09 20:00:51 -070019#include <linux/slab.h>
Arnaldo Carvalho de Melof3f05f72005-08-09 20:08:09 -070020#include <linux/wait.h>
Arnaldo Carvalho de Melo77d8bf92005-08-09 20:00:51 -070021
Arnaldo Carvalho de Melo463c84b2005-08-09 20:10:42 -070022#include <net/inet_connection_sock.h>
Arnaldo Carvalho de Melo77d8bf92005-08-09 20:00:51 -070023#include <net/inet_hashtables.h>
David S. Miller6e5714e2011-08-03 20:50:44 -070024#include <net/secure_seq.h>
Arnaldo Carvalho de Meloa7f5e7f2005-12-13 23:25:31 -080025#include <net/ip.h>
Arnaldo Carvalho de Melo77d8bf92005-08-09 20:00:51 -070026
27/*
28 * Allocate and initialize a new local port bind bucket.
29 * The bindhash mutex for snum's hash chain must be held here.
30 */
Christoph Lametere18b8902006-12-06 20:33:20 -080031struct inet_bind_bucket *inet_bind_bucket_create(struct kmem_cache *cachep,
Pavel Emelyanov941b1d22008-01-31 05:05:50 -080032 struct net *net,
Arnaldo Carvalho de Melo77d8bf92005-08-09 20:00:51 -070033 struct inet_bind_hashbucket *head,
34 const unsigned short snum)
35{
Christoph Lameter54e6ecb2006-12-06 20:33:16 -080036 struct inet_bind_bucket *tb = kmem_cache_alloc(cachep, GFP_ATOMIC);
Arnaldo Carvalho de Melo77d8bf92005-08-09 20:00:51 -070037
38 if (tb != NULL) {
Eric Dumazet7a9546e2008-11-12 00:54:20 -080039 write_pnet(&tb->ib_net, hold_net(net));
Arnaldo Carvalho de Melo77d8bf92005-08-09 20:00:51 -070040 tb->port = snum;
41 tb->fastreuse = 0;
Evgeniy Polyakova9d8f912009-01-19 16:46:02 -080042 tb->num_owners = 0;
Arnaldo Carvalho de Melo77d8bf92005-08-09 20:00:51 -070043 INIT_HLIST_HEAD(&tb->owners);
44 hlist_add_head(&tb->node, &head->chain);
45 }
46 return tb;
47}
48
Arnaldo Carvalho de Melo77d8bf92005-08-09 20:00:51 -070049/*
50 * Caller must hold hashbucket lock for this tb with local BH disabled
51 */
Christoph Lametere18b8902006-12-06 20:33:20 -080052void inet_bind_bucket_destroy(struct kmem_cache *cachep, struct inet_bind_bucket *tb)
Arnaldo Carvalho de Melo77d8bf92005-08-09 20:00:51 -070053{
54 if (hlist_empty(&tb->owners)) {
55 __hlist_del(&tb->node);
Eric Dumazet7a9546e2008-11-12 00:54:20 -080056 release_net(ib_net(tb));
Arnaldo Carvalho de Melo77d8bf92005-08-09 20:00:51 -070057 kmem_cache_free(cachep, tb);
58 }
59}
Arnaldo Carvalho de Melo2d8c4ce2005-08-09 20:07:13 -070060
61void inet_bind_hash(struct sock *sk, struct inet_bind_bucket *tb,
62 const unsigned short snum)
63{
Evgeniy Polyakova9d8f912009-01-19 16:46:02 -080064 struct inet_hashinfo *hashinfo = sk->sk_prot->h.hashinfo;
65
Eric Dumazet24dd1fa2009-02-01 12:31:33 -080066 atomic_inc(&hashinfo->bsockets);
Evgeniy Polyakova9d8f912009-01-19 16:46:02 -080067
Eric Dumazetc720c7e2009-10-15 06:30:45 +000068 inet_sk(sk)->inet_num = snum;
Arnaldo Carvalho de Melo2d8c4ce2005-08-09 20:07:13 -070069 sk_add_bind_node(sk, &tb->owners);
Evgeniy Polyakova9d8f912009-01-19 16:46:02 -080070 tb->num_owners++;
Arnaldo Carvalho de Melo463c84b2005-08-09 20:10:42 -070071 inet_csk(sk)->icsk_bind_hash = tb;
Arnaldo Carvalho de Melo2d8c4ce2005-08-09 20:07:13 -070072}
73
Arnaldo Carvalho de Melo2d8c4ce2005-08-09 20:07:13 -070074/*
75 * Get rid of any references to a local port held by the given sock.
76 */
Arnaldo Carvalho de Meloab1e0a12008-02-03 04:06:04 -080077static void __inet_put_port(struct sock *sk)
Arnaldo Carvalho de Melo2d8c4ce2005-08-09 20:07:13 -070078{
Pavel Emelyanov39d8cda2008-03-22 16:50:58 -070079 struct inet_hashinfo *hashinfo = sk->sk_prot->h.hashinfo;
Eric Dumazetc720c7e2009-10-15 06:30:45 +000080 const int bhash = inet_bhashfn(sock_net(sk), inet_sk(sk)->inet_num,
Pavel Emelyanov7f635ab2008-06-16 17:12:49 -070081 hashinfo->bhash_size);
Arnaldo Carvalho de Melo2d8c4ce2005-08-09 20:07:13 -070082 struct inet_bind_hashbucket *head = &hashinfo->bhash[bhash];
83 struct inet_bind_bucket *tb;
84
Eric Dumazet24dd1fa2009-02-01 12:31:33 -080085 atomic_dec(&hashinfo->bsockets);
Evgeniy Polyakova9d8f912009-01-19 16:46:02 -080086
Arnaldo Carvalho de Melo2d8c4ce2005-08-09 20:07:13 -070087 spin_lock(&head->lock);
Arnaldo Carvalho de Melo463c84b2005-08-09 20:10:42 -070088 tb = inet_csk(sk)->icsk_bind_hash;
Arnaldo Carvalho de Melo2d8c4ce2005-08-09 20:07:13 -070089 __sk_del_bind_node(sk);
Evgeniy Polyakova9d8f912009-01-19 16:46:02 -080090 tb->num_owners--;
Arnaldo Carvalho de Melo463c84b2005-08-09 20:10:42 -070091 inet_csk(sk)->icsk_bind_hash = NULL;
Eric Dumazetc720c7e2009-10-15 06:30:45 +000092 inet_sk(sk)->inet_num = 0;
Arnaldo Carvalho de Melo2d8c4ce2005-08-09 20:07:13 -070093 inet_bind_bucket_destroy(hashinfo->bind_bucket_cachep, tb);
94 spin_unlock(&head->lock);
95}
96
Arnaldo Carvalho de Meloab1e0a12008-02-03 04:06:04 -080097void inet_put_port(struct sock *sk)
Arnaldo Carvalho de Melo2d8c4ce2005-08-09 20:07:13 -070098{
99 local_bh_disable();
Arnaldo Carvalho de Meloab1e0a12008-02-03 04:06:04 -0800100 __inet_put_port(sk);
Arnaldo Carvalho de Melo2d8c4ce2005-08-09 20:07:13 -0700101 local_bh_enable();
102}
Arnaldo Carvalho de Melo2d8c4ce2005-08-09 20:07:13 -0700103EXPORT_SYMBOL(inet_put_port);
Arnaldo Carvalho de Melof3f05f72005-08-09 20:08:09 -0700104
Balazs Scheidler093d2822010-10-21 13:06:43 +0200105int __inet_inherit_port(struct sock *sk, struct sock *child)
Pavel Emelyanov53083772008-04-17 23:18:15 -0700106{
107 struct inet_hashinfo *table = sk->sk_prot->h.hashinfo;
Balazs Scheidler093d2822010-10-21 13:06:43 +0200108 unsigned short port = inet_sk(child)->inet_num;
109 const int bhash = inet_bhashfn(sock_net(sk), port,
Pavel Emelyanov7f635ab2008-06-16 17:12:49 -0700110 table->bhash_size);
Pavel Emelyanov53083772008-04-17 23:18:15 -0700111 struct inet_bind_hashbucket *head = &table->bhash[bhash];
112 struct inet_bind_bucket *tb;
113
114 spin_lock(&head->lock);
115 tb = inet_csk(sk)->icsk_bind_hash;
Balazs Scheidler093d2822010-10-21 13:06:43 +0200116 if (tb->port != port) {
117 /* NOTE: using tproxy and redirecting skbs to a proxy
118 * on a different listener port breaks the assumption
119 * that the listener socket's icsk_bind_hash is the same
120 * as that of the child socket. We have to look up or
121 * create a new bind bucket for the child here. */
122 struct hlist_node *node;
123 inet_bind_bucket_for_each(tb, node, &head->chain) {
124 if (net_eq(ib_net(tb), sock_net(sk)) &&
125 tb->port == port)
126 break;
127 }
128 if (!node) {
129 tb = inet_bind_bucket_create(table->bind_bucket_cachep,
130 sock_net(sk), head, port);
131 if (!tb) {
132 spin_unlock(&head->lock);
133 return -ENOMEM;
134 }
135 }
136 }
Nagendra Tomarb4ff3c92010-11-26 14:26:27 +0000137 inet_bind_hash(child, tb, port);
Pavel Emelyanov53083772008-04-17 23:18:15 -0700138 spin_unlock(&head->lock);
Balazs Scheidler093d2822010-10-21 13:06:43 +0200139
140 return 0;
Pavel Emelyanov53083772008-04-17 23:18:15 -0700141}
Pavel Emelyanov53083772008-04-17 23:18:15 -0700142EXPORT_SYMBOL_GPL(__inet_inherit_port);
143
Eric Dumazetc25eb3b2008-11-23 17:22:55 -0800144static inline int compute_score(struct sock *sk, struct net *net,
145 const unsigned short hnum, const __be32 daddr,
146 const int dif)
147{
148 int score = -1;
149 struct inet_sock *inet = inet_sk(sk);
150
Eric Dumazetc720c7e2009-10-15 06:30:45 +0000151 if (net_eq(sock_net(sk), net) && inet->inet_num == hnum &&
Eric Dumazetc25eb3b2008-11-23 17:22:55 -0800152 !ipv6_only_sock(sk)) {
Eric Dumazetc720c7e2009-10-15 06:30:45 +0000153 __be32 rcv_saddr = inet->inet_rcv_saddr;
Eric Dumazetc25eb3b2008-11-23 17:22:55 -0800154 score = sk->sk_family == PF_INET ? 1 : 0;
155 if (rcv_saddr) {
156 if (rcv_saddr != daddr)
157 return -1;
158 score += 2;
159 }
160 if (sk->sk_bound_dev_if) {
161 if (sk->sk_bound_dev_if != dif)
162 return -1;
163 score += 2;
164 }
165 }
166 return score;
167}
168
Arnaldo Carvalho de Melof3f05f72005-08-09 20:08:09 -0700169/*
Arnaldo Carvalho de Melo33b62232005-08-09 20:09:06 -0700170 * Don't inline this cruft. Here are some nice properties to exploit here. The
171 * BSD API does not allow a listening sock to specify the remote port nor the
172 * remote address for the connection. So always assume those are both
173 * wildcarded during the search since they can never be otherwise.
174 */
Arnaldo Carvalho de Melo33b62232005-08-09 20:09:06 -0700175
Arnaldo Carvalho de Melo33b62232005-08-09 20:09:06 -0700176
Pavel Emelyanovc67499c2008-01-31 05:06:40 -0800177struct sock *__inet_lookup_listener(struct net *net,
178 struct inet_hashinfo *hashinfo,
Al Virofb99c842006-09-27 18:43:33 -0700179 const __be32 daddr, const unsigned short hnum,
Herbert Xu8f4910692006-08-09 15:47:12 -0700180 const int dif)
Herbert Xu99a92ff2006-08-08 02:18:10 -0700181{
Eric Dumazetc25eb3b2008-11-23 17:22:55 -0800182 struct sock *sk, *result;
183 struct hlist_nulls_node *node;
184 unsigned int hash = inet_lhashfn(net, hnum);
185 struct inet_listen_hashbucket *ilb = &hashinfo->listening_hash[hash];
186 int score, hiscore;
Herbert Xu99a92ff2006-08-08 02:18:10 -0700187
Eric Dumazetc25eb3b2008-11-23 17:22:55 -0800188 rcu_read_lock();
189begin:
190 result = NULL;
191 hiscore = -1;
192 sk_nulls_for_each_rcu(sk, node, &ilb->head) {
193 score = compute_score(sk, net, hnum, daddr, dif);
194 if (score > hiscore) {
195 result = sk;
196 hiscore = score;
197 }
Herbert Xu99a92ff2006-08-08 02:18:10 -0700198 }
Eric Dumazetc25eb3b2008-11-23 17:22:55 -0800199 /*
200 * if the nulls value we got at the end of this lookup is
201 * not the expected one, we must restart lookup.
202 * We probably met an item that was moved to another chain.
203 */
204 if (get_nulls_value(node) != hash + LISTENING_NULLS_BASE)
205 goto begin;
206 if (result) {
207 if (unlikely(!atomic_inc_not_zero(&result->sk_refcnt)))
208 result = NULL;
209 else if (unlikely(compute_score(result, net, hnum, daddr,
210 dif) < hiscore)) {
211 sock_put(result);
212 goto begin;
213 }
Herbert Xu99a92ff2006-08-08 02:18:10 -0700214 }
Eric Dumazetc25eb3b2008-11-23 17:22:55 -0800215 rcu_read_unlock();
216 return result;
Herbert Xu99a92ff2006-08-08 02:18:10 -0700217}
Herbert Xu8f4910692006-08-09 15:47:12 -0700218EXPORT_SYMBOL_GPL(__inet_lookup_listener);
Arnaldo Carvalho de Meloa7f5e7f2005-12-13 23:25:31 -0800219
Daniel Baluta5e73ea12012-04-15 01:34:41 +0000220struct sock *__inet_lookup_established(struct net *net,
Pavel Emelyanovc67499c2008-01-31 05:06:40 -0800221 struct inet_hashinfo *hashinfo,
Pavel Emelyanov77a5ba52007-12-20 15:32:17 -0800222 const __be32 saddr, const __be16 sport,
223 const __be32 daddr, const u16 hnum,
224 const int dif)
225{
226 INET_ADDR_COOKIE(acookie, saddr, daddr)
227 const __portpair ports = INET_COMBINED_PORTS(sport, hnum);
228 struct sock *sk;
Eric Dumazet3ab5aee2008-11-16 19:40:17 -0800229 const struct hlist_nulls_node *node;
Pavel Emelyanov77a5ba52007-12-20 15:32:17 -0800230 /* Optimize here for direct hit, only listening connections can
231 * have wildcards anyways.
232 */
Pavel Emelyanov9f26b3a2008-06-16 17:13:27 -0700233 unsigned int hash = inet_ehashfn(net, daddr, hnum, saddr, sport);
Eric Dumazetf373b532009-10-09 00:16:19 +0000234 unsigned int slot = hash & hashinfo->ehash_mask;
Eric Dumazet3ab5aee2008-11-16 19:40:17 -0800235 struct inet_ehash_bucket *head = &hashinfo->ehash[slot];
Pavel Emelyanov77a5ba52007-12-20 15:32:17 -0800236
Eric Dumazet3ab5aee2008-11-16 19:40:17 -0800237 rcu_read_lock();
238begin:
239 sk_nulls_for_each_rcu(sk, node, &head->chain) {
Pavel Emelyanovc67499c2008-01-31 05:06:40 -0800240 if (INET_MATCH(sk, net, hash, acookie,
Eric Dumazet3ab5aee2008-11-16 19:40:17 -0800241 saddr, daddr, ports, dif)) {
242 if (unlikely(!atomic_inc_not_zero(&sk->sk_refcnt)))
243 goto begintw;
244 if (unlikely(!INET_MATCH(sk, net, hash, acookie,
245 saddr, daddr, ports, dif))) {
246 sock_put(sk);
247 goto begin;
248 }
249 goto out;
250 }
Pavel Emelyanov77a5ba52007-12-20 15:32:17 -0800251 }
Eric Dumazet3ab5aee2008-11-16 19:40:17 -0800252 /*
253 * if the nulls value we got at the end of this lookup is
254 * not the expected one, we must restart lookup.
255 * We probably met an item that was moved to another chain.
256 */
257 if (get_nulls_value(node) != slot)
258 goto begin;
Pavel Emelyanov77a5ba52007-12-20 15:32:17 -0800259
Eric Dumazet3ab5aee2008-11-16 19:40:17 -0800260begintw:
Pavel Emelyanov77a5ba52007-12-20 15:32:17 -0800261 /* Must check for a TIME_WAIT'er before going to listener hash. */
Eric Dumazet3ab5aee2008-11-16 19:40:17 -0800262 sk_nulls_for_each_rcu(sk, node, &head->twchain) {
Pavel Emelyanovc67499c2008-01-31 05:06:40 -0800263 if (INET_TW_MATCH(sk, net, hash, acookie,
Eric Dumazet3ab5aee2008-11-16 19:40:17 -0800264 saddr, daddr, ports, dif)) {
265 if (unlikely(!atomic_inc_not_zero(&sk->sk_refcnt))) {
266 sk = NULL;
267 goto out;
268 }
269 if (unlikely(!INET_TW_MATCH(sk, net, hash, acookie,
270 saddr, daddr, ports, dif))) {
271 sock_put(sk);
272 goto begintw;
273 }
274 goto out;
275 }
Pavel Emelyanov77a5ba52007-12-20 15:32:17 -0800276 }
Eric Dumazet3ab5aee2008-11-16 19:40:17 -0800277 /*
278 * if the nulls value we got at the end of this lookup is
279 * not the expected one, we must restart lookup.
280 * We probably met an item that was moved to another chain.
281 */
282 if (get_nulls_value(node) != slot)
283 goto begintw;
Pavel Emelyanov77a5ba52007-12-20 15:32:17 -0800284 sk = NULL;
285out:
Eric Dumazet3ab5aee2008-11-16 19:40:17 -0800286 rcu_read_unlock();
Pavel Emelyanov77a5ba52007-12-20 15:32:17 -0800287 return sk;
Pavel Emelyanov77a5ba52007-12-20 15:32:17 -0800288}
289EXPORT_SYMBOL_GPL(__inet_lookup_established);
290
Arnaldo Carvalho de Meloa7f5e7f2005-12-13 23:25:31 -0800291/* called with local bh disabled */
292static int __inet_check_established(struct inet_timewait_death_row *death_row,
293 struct sock *sk, __u16 lport,
294 struct inet_timewait_sock **twp)
295{
296 struct inet_hashinfo *hinfo = death_row->hashinfo;
297 struct inet_sock *inet = inet_sk(sk);
Eric Dumazetc720c7e2009-10-15 06:30:45 +0000298 __be32 daddr = inet->inet_rcv_saddr;
299 __be32 saddr = inet->inet_daddr;
Arnaldo Carvalho de Meloa7f5e7f2005-12-13 23:25:31 -0800300 int dif = sk->sk_bound_dev_if;
301 INET_ADDR_COOKIE(acookie, saddr, daddr)
Eric Dumazetc720c7e2009-10-15 06:30:45 +0000302 const __portpair ports = INET_COMBINED_PORTS(inet->inet_dport, lport);
Pavel Emelyanov9f26b3a2008-06-16 17:13:27 -0700303 struct net *net = sock_net(sk);
Eric Dumazetc720c7e2009-10-15 06:30:45 +0000304 unsigned int hash = inet_ehashfn(net, daddr, lport,
305 saddr, inet->inet_dport);
Arnaldo Carvalho de Meloa7f5e7f2005-12-13 23:25:31 -0800306 struct inet_ehash_bucket *head = inet_ehash_bucket(hinfo, hash);
Eric Dumazet9db66bd2008-11-20 20:39:09 -0800307 spinlock_t *lock = inet_ehash_lockp(hinfo, hash);
Arnaldo Carvalho de Meloa7f5e7f2005-12-13 23:25:31 -0800308 struct sock *sk2;
Eric Dumazet3ab5aee2008-11-16 19:40:17 -0800309 const struct hlist_nulls_node *node;
Arnaldo Carvalho de Meloa7f5e7f2005-12-13 23:25:31 -0800310 struct inet_timewait_sock *tw;
Eric Dumazet13475a32009-12-02 22:31:19 +0000311 int twrefcnt = 0;
Arnaldo Carvalho de Meloa7f5e7f2005-12-13 23:25:31 -0800312
Eric Dumazet9db66bd2008-11-20 20:39:09 -0800313 spin_lock(lock);
Arnaldo Carvalho de Meloa7f5e7f2005-12-13 23:25:31 -0800314
315 /* Check TIME-WAIT sockets first. */
Eric Dumazet3ab5aee2008-11-16 19:40:17 -0800316 sk_nulls_for_each(sk2, node, &head->twchain) {
Arnaldo Carvalho de Meloa7f5e7f2005-12-13 23:25:31 -0800317 tw = inet_twsk(sk2);
318
Pavel Emelyanovc67499c2008-01-31 05:06:40 -0800319 if (INET_TW_MATCH(sk2, net, hash, acookie,
320 saddr, daddr, ports, dif)) {
Arnaldo Carvalho de Meloa7f5e7f2005-12-13 23:25:31 -0800321 if (twsk_unique(sk, sk2, twp))
322 goto unique;
323 else
324 goto not_unique;
325 }
326 }
327 tw = NULL;
328
329 /* And established part... */
Eric Dumazet3ab5aee2008-11-16 19:40:17 -0800330 sk_nulls_for_each(sk2, node, &head->chain) {
Pavel Emelyanovc67499c2008-01-31 05:06:40 -0800331 if (INET_MATCH(sk2, net, hash, acookie,
332 saddr, daddr, ports, dif))
Arnaldo Carvalho de Meloa7f5e7f2005-12-13 23:25:31 -0800333 goto not_unique;
334 }
335
336unique:
337 /* Must record num and sport now. Otherwise we will see
338 * in hash table socket with a funny identity. */
Eric Dumazetc720c7e2009-10-15 06:30:45 +0000339 inet->inet_num = lport;
340 inet->inet_sport = htons(lport);
Arnaldo Carvalho de Meloa7f5e7f2005-12-13 23:25:31 -0800341 sk->sk_hash = hash;
Ilpo Järvinen547b7922008-07-25 21:43:18 -0700342 WARN_ON(!sk_unhashed(sk));
Eric Dumazet3ab5aee2008-11-16 19:40:17 -0800343 __sk_nulls_add_node_rcu(sk, &head->chain);
Eric Dumazet13475a32009-12-02 22:31:19 +0000344 if (tw) {
345 twrefcnt = inet_twsk_unhash(tw);
346 NET_INC_STATS_BH(net, LINUX_MIB_TIMEWAITRECYCLED);
347 }
Eric Dumazet9db66bd2008-11-20 20:39:09 -0800348 spin_unlock(lock);
Eric Dumazet13475a32009-12-02 22:31:19 +0000349 if (twrefcnt)
350 inet_twsk_put(tw);
Pavel Emelyanovc29a0bc2008-03-31 19:41:46 -0700351 sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1);
Arnaldo Carvalho de Meloa7f5e7f2005-12-13 23:25:31 -0800352
353 if (twp) {
354 *twp = tw;
Arnaldo Carvalho de Meloa7f5e7f2005-12-13 23:25:31 -0800355 } else if (tw) {
356 /* Silly. Should hash-dance instead... */
357 inet_twsk_deschedule(tw, death_row);
Arnaldo Carvalho de Meloa7f5e7f2005-12-13 23:25:31 -0800358
359 inet_twsk_put(tw);
360 }
Arnaldo Carvalho de Meloa7f5e7f2005-12-13 23:25:31 -0800361 return 0;
362
363not_unique:
Eric Dumazet9db66bd2008-11-20 20:39:09 -0800364 spin_unlock(lock);
Arnaldo Carvalho de Meloa7f5e7f2005-12-13 23:25:31 -0800365 return -EADDRNOTAVAIL;
366}
367
368static inline u32 inet_sk_port_offset(const struct sock *sk)
369{
370 const struct inet_sock *inet = inet_sk(sk);
Eric Dumazetc720c7e2009-10-15 06:30:45 +0000371 return secure_ipv4_port_ephemeral(inet->inet_rcv_saddr,
372 inet->inet_daddr,
373 inet->inet_dport);
Arnaldo Carvalho de Meloa7f5e7f2005-12-13 23:25:31 -0800374}
375
Eric Dumazet9327f702009-12-04 03:46:54 +0000376int __inet_hash_nolisten(struct sock *sk, struct inet_timewait_sock *tw)
Pavel Emelyanov152da812007-12-20 15:31:33 -0800377{
Pavel Emelyanov39d8cda2008-03-22 16:50:58 -0700378 struct inet_hashinfo *hashinfo = sk->sk_prot->h.hashinfo;
Eric Dumazet3ab5aee2008-11-16 19:40:17 -0800379 struct hlist_nulls_head *list;
Eric Dumazet9db66bd2008-11-20 20:39:09 -0800380 spinlock_t *lock;
Pavel Emelyanov152da812007-12-20 15:31:33 -0800381 struct inet_ehash_bucket *head;
Eric Dumazet9327f702009-12-04 03:46:54 +0000382 int twrefcnt = 0;
Pavel Emelyanov152da812007-12-20 15:31:33 -0800383
Ilpo Järvinen547b7922008-07-25 21:43:18 -0700384 WARN_ON(!sk_unhashed(sk));
Pavel Emelyanov152da812007-12-20 15:31:33 -0800385
386 sk->sk_hash = inet_sk_ehashfn(sk);
387 head = inet_ehash_bucket(hashinfo, sk->sk_hash);
388 list = &head->chain;
389 lock = inet_ehash_lockp(hashinfo, sk->sk_hash);
390
Eric Dumazet9db66bd2008-11-20 20:39:09 -0800391 spin_lock(lock);
Eric Dumazet3ab5aee2008-11-16 19:40:17 -0800392 __sk_nulls_add_node_rcu(sk, list);
Eric Dumazet9327f702009-12-04 03:46:54 +0000393 if (tw) {
394 WARN_ON(sk->sk_hash != tw->tw_hash);
395 twrefcnt = inet_twsk_unhash(tw);
396 }
Eric Dumazet9db66bd2008-11-20 20:39:09 -0800397 spin_unlock(lock);
Pavel Emelyanovc29a0bc2008-03-31 19:41:46 -0700398 sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1);
Eric Dumazet9327f702009-12-04 03:46:54 +0000399 return twrefcnt;
Pavel Emelyanov152da812007-12-20 15:31:33 -0800400}
401EXPORT_SYMBOL_GPL(__inet_hash_nolisten);
402
Arnaldo Carvalho de Meloab1e0a12008-02-03 04:06:04 -0800403static void __inet_hash(struct sock *sk)
Pavel Emelyanov152da812007-12-20 15:31:33 -0800404{
Pavel Emelyanov39d8cda2008-03-22 16:50:58 -0700405 struct inet_hashinfo *hashinfo = sk->sk_prot->h.hashinfo;
Eric Dumazet5caea4e2008-11-20 00:40:07 -0800406 struct inet_listen_hashbucket *ilb;
Pavel Emelyanov152da812007-12-20 15:31:33 -0800407
408 if (sk->sk_state != TCP_LISTEN) {
Eric Dumazet9327f702009-12-04 03:46:54 +0000409 __inet_hash_nolisten(sk, NULL);
Pavel Emelyanov152da812007-12-20 15:31:33 -0800410 return;
411 }
412
Ilpo Järvinen547b7922008-07-25 21:43:18 -0700413 WARN_ON(!sk_unhashed(sk));
Eric Dumazet5caea4e2008-11-20 00:40:07 -0800414 ilb = &hashinfo->listening_hash[inet_sk_listen_hashfn(sk)];
Pavel Emelyanov152da812007-12-20 15:31:33 -0800415
Eric Dumazet5caea4e2008-11-20 00:40:07 -0800416 spin_lock(&ilb->lock);
Eric Dumazetc25eb3b2008-11-23 17:22:55 -0800417 __sk_nulls_add_node_rcu(sk, &ilb->head);
Pavel Emelyanovc29a0bc2008-03-31 19:41:46 -0700418 sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1);
Eric Dumazet5caea4e2008-11-20 00:40:07 -0800419 spin_unlock(&ilb->lock);
Pavel Emelyanov152da812007-12-20 15:31:33 -0800420}
Arnaldo Carvalho de Meloab1e0a12008-02-03 04:06:04 -0800421
422void inet_hash(struct sock *sk)
423{
424 if (sk->sk_state != TCP_CLOSE) {
425 local_bh_disable();
426 __inet_hash(sk);
427 local_bh_enable();
428 }
429}
430EXPORT_SYMBOL_GPL(inet_hash);
431
432void inet_unhash(struct sock *sk)
433{
Pavel Emelyanov39d8cda2008-03-22 16:50:58 -0700434 struct inet_hashinfo *hashinfo = sk->sk_prot->h.hashinfo;
Eric Dumazetc25eb3b2008-11-23 17:22:55 -0800435 spinlock_t *lock;
436 int done;
Arnaldo Carvalho de Meloab1e0a12008-02-03 04:06:04 -0800437
438 if (sk_unhashed(sk))
Eric Dumazet5caea4e2008-11-20 00:40:07 -0800439 return;
Arnaldo Carvalho de Meloab1e0a12008-02-03 04:06:04 -0800440
Eric Dumazetc25eb3b2008-11-23 17:22:55 -0800441 if (sk->sk_state == TCP_LISTEN)
442 lock = &hashinfo->listening_hash[inet_sk_listen_hashfn(sk)].lock;
443 else
444 lock = inet_ehash_lockp(hashinfo, sk->sk_hash);
Eric Dumazet5caea4e2008-11-20 00:40:07 -0800445
Eric Dumazetc25eb3b2008-11-23 17:22:55 -0800446 spin_lock_bh(lock);
447 done =__sk_nulls_del_node_init_rcu(sk);
Eric Dumazetc25eb3b2008-11-23 17:22:55 -0800448 if (done)
449 sock_prot_inuse_add(sock_net(sk), sk->sk_prot, -1);
Eric Dumazet920de802008-11-24 00:09:29 -0800450 spin_unlock_bh(lock);
Arnaldo Carvalho de Meloab1e0a12008-02-03 04:06:04 -0800451}
452EXPORT_SYMBOL_GPL(inet_unhash);
Pavel Emelyanov152da812007-12-20 15:31:33 -0800453
Pavel Emelyanov5ee31fc2008-01-31 05:04:45 -0800454int __inet_hash_connect(struct inet_timewait_death_row *death_row,
Pavel Emelyanov5d8c0aa2008-02-05 03:14:44 -0800455 struct sock *sk, u32 port_offset,
Pavel Emelyanov5ee31fc2008-01-31 05:04:45 -0800456 int (*check_established)(struct inet_timewait_death_row *,
457 struct sock *, __u16, struct inet_timewait_sock **),
Eric Dumazet9327f702009-12-04 03:46:54 +0000458 int (*hash)(struct sock *sk, struct inet_timewait_sock *twp))
Arnaldo Carvalho de Meloa7f5e7f2005-12-13 23:25:31 -0800459{
460 struct inet_hashinfo *hinfo = death_row->hashinfo;
Eric Dumazetc720c7e2009-10-15 06:30:45 +0000461 const unsigned short snum = inet_sk(sk)->inet_num;
YOSHIFUJI Hideakie905a9e2007-02-09 23:24:47 +0900462 struct inet_bind_hashbucket *head;
463 struct inet_bind_bucket *tb;
Arnaldo Carvalho de Meloa7f5e7f2005-12-13 23:25:31 -0800464 int ret;
YOSHIFUJI Hideaki3b1e0a62008-03-26 02:26:21 +0900465 struct net *net = sock_net(sk);
Eric Dumazet9327f702009-12-04 03:46:54 +0000466 int twrefcnt = 1;
Arnaldo Carvalho de Meloa7f5e7f2005-12-13 23:25:31 -0800467
YOSHIFUJI Hideakie905a9e2007-02-09 23:24:47 +0900468 if (!snum) {
Stephen Hemminger227b60f2007-10-10 17:30:46 -0700469 int i, remaining, low, high, port;
Arnaldo Carvalho de Meloa7f5e7f2005-12-13 23:25:31 -0800470 static u32 hint;
Pavel Emelyanov5d8c0aa2008-02-05 03:14:44 -0800471 u32 offset = hint + port_offset;
Arnaldo Carvalho de Meloa7f5e7f2005-12-13 23:25:31 -0800472 struct hlist_node *node;
YOSHIFUJI Hideakie905a9e2007-02-09 23:24:47 +0900473 struct inet_timewait_sock *tw = NULL;
Arnaldo Carvalho de Meloa7f5e7f2005-12-13 23:25:31 -0800474
Stephen Hemminger227b60f2007-10-10 17:30:46 -0700475 inet_get_local_port_range(&low, &high);
Anton Arapova25de532007-10-18 22:00:17 -0700476 remaining = (high - low) + 1;
Stephen Hemminger227b60f2007-10-10 17:30:46 -0700477
YOSHIFUJI Hideakie905a9e2007-02-09 23:24:47 +0900478 local_bh_disable();
Stephen Hemminger227b60f2007-10-10 17:30:46 -0700479 for (i = 1; i <= remaining; i++) {
480 port = low + (i + offset) % remaining;
Amerigo Wange3826f12010-05-05 00:27:06 +0000481 if (inet_is_reserved_local_port(port))
482 continue;
Pavel Emelyanov7f635ab2008-06-16 17:12:49 -0700483 head = &hinfo->bhash[inet_bhashfn(net, port,
484 hinfo->bhash_size)];
YOSHIFUJI Hideakie905a9e2007-02-09 23:24:47 +0900485 spin_lock(&head->lock);
Arnaldo Carvalho de Meloa7f5e7f2005-12-13 23:25:31 -0800486
YOSHIFUJI Hideakie905a9e2007-02-09 23:24:47 +0900487 /* Does not bother with rcv_saddr checks,
488 * because the established check is already
489 * unique enough.
490 */
Arnaldo Carvalho de Meloa7f5e7f2005-12-13 23:25:31 -0800491 inet_bind_bucket_for_each(tb, node, &head->chain) {
Octavian Purdila09ad9bc2009-11-25 15:14:13 -0800492 if (net_eq(ib_net(tb), net) &&
493 tb->port == port) {
YOSHIFUJI Hideakie905a9e2007-02-09 23:24:47 +0900494 if (tb->fastreuse >= 0)
495 goto next_port;
Evgeniy Polyakova9d8f912009-01-19 16:46:02 -0800496 WARN_ON(hlist_empty(&tb->owners));
Pavel Emelyanov5ee31fc2008-01-31 05:04:45 -0800497 if (!check_established(death_row, sk,
498 port, &tw))
YOSHIFUJI Hideakie905a9e2007-02-09 23:24:47 +0900499 goto ok;
500 goto next_port;
501 }
502 }
Arnaldo Carvalho de Meloa7f5e7f2005-12-13 23:25:31 -0800503
Pavel Emelyanov941b1d22008-01-31 05:05:50 -0800504 tb = inet_bind_bucket_create(hinfo->bind_bucket_cachep,
505 net, head, port);
YOSHIFUJI Hideakie905a9e2007-02-09 23:24:47 +0900506 if (!tb) {
507 spin_unlock(&head->lock);
508 break;
509 }
510 tb->fastreuse = -1;
511 goto ok;
Arnaldo Carvalho de Meloa7f5e7f2005-12-13 23:25:31 -0800512
YOSHIFUJI Hideakie905a9e2007-02-09 23:24:47 +0900513 next_port:
514 spin_unlock(&head->lock);
515 }
516 local_bh_enable();
Arnaldo Carvalho de Meloa7f5e7f2005-12-13 23:25:31 -0800517
YOSHIFUJI Hideakie905a9e2007-02-09 23:24:47 +0900518 return -EADDRNOTAVAIL;
Arnaldo Carvalho de Meloa7f5e7f2005-12-13 23:25:31 -0800519
520ok:
521 hint += i;
522
YOSHIFUJI Hideakie905a9e2007-02-09 23:24:47 +0900523 /* Head lock still held and bh's disabled */
524 inet_bind_hash(sk, tb, port);
Arnaldo Carvalho de Meloa7f5e7f2005-12-13 23:25:31 -0800525 if (sk_unhashed(sk)) {
Eric Dumazetc720c7e2009-10-15 06:30:45 +0000526 inet_sk(sk)->inet_sport = htons(port);
Eric Dumazet9327f702009-12-04 03:46:54 +0000527 twrefcnt += hash(sk, tw);
YOSHIFUJI Hideakie905a9e2007-02-09 23:24:47 +0900528 }
Eric Dumazet3cdaeda2009-12-04 03:47:42 +0000529 if (tw)
530 twrefcnt += inet_twsk_bind_unhash(tw, hinfo);
YOSHIFUJI Hideakie905a9e2007-02-09 23:24:47 +0900531 spin_unlock(&head->lock);
Arnaldo Carvalho de Meloa7f5e7f2005-12-13 23:25:31 -0800532
YOSHIFUJI Hideakie905a9e2007-02-09 23:24:47 +0900533 if (tw) {
534 inet_twsk_deschedule(tw, death_row);
Eric Dumazet9327f702009-12-04 03:46:54 +0000535 while (twrefcnt) {
536 twrefcnt--;
537 inet_twsk_put(tw);
538 }
YOSHIFUJI Hideakie905a9e2007-02-09 23:24:47 +0900539 }
Arnaldo Carvalho de Meloa7f5e7f2005-12-13 23:25:31 -0800540
541 ret = 0;
542 goto out;
YOSHIFUJI Hideakie905a9e2007-02-09 23:24:47 +0900543 }
Arnaldo Carvalho de Meloa7f5e7f2005-12-13 23:25:31 -0800544
Pavel Emelyanov7f635ab2008-06-16 17:12:49 -0700545 head = &hinfo->bhash[inet_bhashfn(net, snum, hinfo->bhash_size)];
YOSHIFUJI Hideakie905a9e2007-02-09 23:24:47 +0900546 tb = inet_csk(sk)->icsk_bind_hash;
Arnaldo Carvalho de Meloa7f5e7f2005-12-13 23:25:31 -0800547 spin_lock_bh(&head->lock);
548 if (sk_head(&tb->owners) == sk && !sk->sk_bind_node.next) {
Eric Dumazet9327f702009-12-04 03:46:54 +0000549 hash(sk, NULL);
Arnaldo Carvalho de Meloa7f5e7f2005-12-13 23:25:31 -0800550 spin_unlock_bh(&head->lock);
551 return 0;
552 } else {
553 spin_unlock(&head->lock);
554 /* No definite answer... Walk to established hash table */
Pavel Emelyanov5ee31fc2008-01-31 05:04:45 -0800555 ret = check_established(death_row, sk, snum, NULL);
Arnaldo Carvalho de Meloa7f5e7f2005-12-13 23:25:31 -0800556out:
557 local_bh_enable();
558 return ret;
559 }
560}
Pavel Emelyanov5ee31fc2008-01-31 05:04:45 -0800561
562/*
563 * Bind a port for a connect operation and hash it.
564 */
565int inet_hash_connect(struct inet_timewait_death_row *death_row,
566 struct sock *sk)
567{
Pavel Emelyanov5d8c0aa2008-02-05 03:14:44 -0800568 return __inet_hash_connect(death_row, sk, inet_sk_port_offset(sk),
Pavel Emelyanov5ee31fc2008-01-31 05:04:45 -0800569 __inet_check_established, __inet_hash_nolisten);
570}
Arnaldo Carvalho de Meloa7f5e7f2005-12-13 23:25:31 -0800571EXPORT_SYMBOL_GPL(inet_hash_connect);
Eric Dumazet5caea4e2008-11-20 00:40:07 -0800572
573void inet_hashinfo_init(struct inet_hashinfo *h)
574{
575 int i;
576
Eric Dumazet24dd1fa2009-02-01 12:31:33 -0800577 atomic_set(&h->bsockets, 0);
Eric Dumazetc25eb3b2008-11-23 17:22:55 -0800578 for (i = 0; i < INET_LHTABLE_SIZE; i++) {
Eric Dumazet5caea4e2008-11-20 00:40:07 -0800579 spin_lock_init(&h->listening_hash[i].lock);
Eric Dumazetc25eb3b2008-11-23 17:22:55 -0800580 INIT_HLIST_NULLS_HEAD(&h->listening_hash[i].head,
581 i + LISTENING_NULLS_BASE);
582 }
Eric Dumazet5caea4e2008-11-20 00:40:07 -0800583}
Eric Dumazet5caea4e2008-11-20 00:40:07 -0800584EXPORT_SYMBOL_GPL(inet_hashinfo_init);