blob: 625cc5f64c949fb2483c18e82d398ba6c2b0236d [file] [log] [blame]
Arnaldo Carvalho de Melo77d8bf92005-08-09 20:00:51 -07001/*
2 * INET An implementation of the TCP/IP protocol suite for the LINUX
3 * operating system. INET is implemented using the BSD Socket
4 * interface as the means of communication with the user level.
5 *
6 * Generic INET transport hashtables
7 *
8 * Authors: Lotsa people, from code originally in tcp
9 *
10 * This program is free software; you can redistribute it and/or
11 * modify it under the terms of the GNU General Public License
12 * as published by the Free Software Foundation; either version
13 * 2 of the License, or (at your option) any later version.
14 */
15
Arnaldo Carvalho de Melo2d8c4ce2005-08-09 20:07:13 -070016#include <linux/module.h>
Arnaldo Carvalho de Meloa7f5e7f2005-12-13 23:25:31 -080017#include <linux/random.h>
Arnaldo Carvalho de Melof3f05f72005-08-09 20:08:09 -070018#include <linux/sched.h>
Arnaldo Carvalho de Melo77d8bf92005-08-09 20:00:51 -070019#include <linux/slab.h>
Arnaldo Carvalho de Melof3f05f72005-08-09 20:08:09 -070020#include <linux/wait.h>
Arnaldo Carvalho de Melo77d8bf92005-08-09 20:00:51 -070021
Arnaldo Carvalho de Melo463c84b2005-08-09 20:10:42 -070022#include <net/inet_connection_sock.h>
Arnaldo Carvalho de Melo77d8bf92005-08-09 20:00:51 -070023#include <net/inet_hashtables.h>
Arnaldo Carvalho de Meloa7f5e7f2005-12-13 23:25:31 -080024#include <net/ip.h>
Arnaldo Carvalho de Melo77d8bf92005-08-09 20:00:51 -070025
26/*
27 * Allocate and initialize a new local port bind bucket.
28 * The bindhash mutex for snum's hash chain must be held here.
29 */
Christoph Lametere18b8902006-12-06 20:33:20 -080030struct inet_bind_bucket *inet_bind_bucket_create(struct kmem_cache *cachep,
Pavel Emelyanov941b1d22008-01-31 05:05:50 -080031 struct net *net,
Arnaldo Carvalho de Melo77d8bf92005-08-09 20:00:51 -070032 struct inet_bind_hashbucket *head,
33 const unsigned short snum)
34{
Christoph Lameter54e6ecb2006-12-06 20:33:16 -080035 struct inet_bind_bucket *tb = kmem_cache_alloc(cachep, GFP_ATOMIC);
Arnaldo Carvalho de Melo77d8bf92005-08-09 20:00:51 -070036
37 if (tb != NULL) {
Eric Dumazet7a9546e2008-11-12 00:54:20 -080038 write_pnet(&tb->ib_net, hold_net(net));
Arnaldo Carvalho de Melo77d8bf92005-08-09 20:00:51 -070039 tb->port = snum;
40 tb->fastreuse = 0;
Evgeniy Polyakova9d8f912009-01-19 16:46:02 -080041 tb->num_owners = 0;
Arnaldo Carvalho de Melo77d8bf92005-08-09 20:00:51 -070042 INIT_HLIST_HEAD(&tb->owners);
43 hlist_add_head(&tb->node, &head->chain);
44 }
45 return tb;
46}
47
Arnaldo Carvalho de Melo77d8bf92005-08-09 20:00:51 -070048/*
49 * Caller must hold hashbucket lock for this tb with local BH disabled
50 */
Christoph Lametere18b8902006-12-06 20:33:20 -080051void inet_bind_bucket_destroy(struct kmem_cache *cachep, struct inet_bind_bucket *tb)
Arnaldo Carvalho de Melo77d8bf92005-08-09 20:00:51 -070052{
53 if (hlist_empty(&tb->owners)) {
54 __hlist_del(&tb->node);
Eric Dumazet7a9546e2008-11-12 00:54:20 -080055 release_net(ib_net(tb));
Arnaldo Carvalho de Melo77d8bf92005-08-09 20:00:51 -070056 kmem_cache_free(cachep, tb);
57 }
58}
Arnaldo Carvalho de Melo2d8c4ce2005-08-09 20:07:13 -070059
60void inet_bind_hash(struct sock *sk, struct inet_bind_bucket *tb,
61 const unsigned short snum)
62{
Evgeniy Polyakova9d8f912009-01-19 16:46:02 -080063 struct inet_hashinfo *hashinfo = sk->sk_prot->h.hashinfo;
64
Eric Dumazet24dd1fa2009-02-01 12:31:33 -080065 atomic_inc(&hashinfo->bsockets);
Evgeniy Polyakova9d8f912009-01-19 16:46:02 -080066
Arnaldo Carvalho de Melo463c84b2005-08-09 20:10:42 -070067 inet_sk(sk)->num = snum;
Arnaldo Carvalho de Melo2d8c4ce2005-08-09 20:07:13 -070068 sk_add_bind_node(sk, &tb->owners);
Evgeniy Polyakova9d8f912009-01-19 16:46:02 -080069 tb->num_owners++;
Arnaldo Carvalho de Melo463c84b2005-08-09 20:10:42 -070070 inet_csk(sk)->icsk_bind_hash = tb;
Arnaldo Carvalho de Melo2d8c4ce2005-08-09 20:07:13 -070071}
72
Arnaldo Carvalho de Melo2d8c4ce2005-08-09 20:07:13 -070073/*
74 * Get rid of any references to a local port held by the given sock.
75 */
Arnaldo Carvalho de Meloab1e0a12008-02-03 04:06:04 -080076static void __inet_put_port(struct sock *sk)
Arnaldo Carvalho de Melo2d8c4ce2005-08-09 20:07:13 -070077{
Pavel Emelyanov39d8cda2008-03-22 16:50:58 -070078 struct inet_hashinfo *hashinfo = sk->sk_prot->h.hashinfo;
Pavel Emelyanov7f635ab2008-06-16 17:12:49 -070079 const int bhash = inet_bhashfn(sock_net(sk), inet_sk(sk)->num,
80 hashinfo->bhash_size);
Arnaldo Carvalho de Melo2d8c4ce2005-08-09 20:07:13 -070081 struct inet_bind_hashbucket *head = &hashinfo->bhash[bhash];
82 struct inet_bind_bucket *tb;
83
Eric Dumazet24dd1fa2009-02-01 12:31:33 -080084 atomic_dec(&hashinfo->bsockets);
Evgeniy Polyakova9d8f912009-01-19 16:46:02 -080085
Arnaldo Carvalho de Melo2d8c4ce2005-08-09 20:07:13 -070086 spin_lock(&head->lock);
Arnaldo Carvalho de Melo463c84b2005-08-09 20:10:42 -070087 tb = inet_csk(sk)->icsk_bind_hash;
Arnaldo Carvalho de Melo2d8c4ce2005-08-09 20:07:13 -070088 __sk_del_bind_node(sk);
Evgeniy Polyakova9d8f912009-01-19 16:46:02 -080089 tb->num_owners--;
Arnaldo Carvalho de Melo463c84b2005-08-09 20:10:42 -070090 inet_csk(sk)->icsk_bind_hash = NULL;
91 inet_sk(sk)->num = 0;
Arnaldo Carvalho de Melo2d8c4ce2005-08-09 20:07:13 -070092 inet_bind_bucket_destroy(hashinfo->bind_bucket_cachep, tb);
93 spin_unlock(&head->lock);
94}
95
Arnaldo Carvalho de Meloab1e0a12008-02-03 04:06:04 -080096void inet_put_port(struct sock *sk)
Arnaldo Carvalho de Melo2d8c4ce2005-08-09 20:07:13 -070097{
98 local_bh_disable();
Arnaldo Carvalho de Meloab1e0a12008-02-03 04:06:04 -080099 __inet_put_port(sk);
Arnaldo Carvalho de Melo2d8c4ce2005-08-09 20:07:13 -0700100 local_bh_enable();
101}
102
103EXPORT_SYMBOL(inet_put_port);
Arnaldo Carvalho de Melof3f05f72005-08-09 20:08:09 -0700104
Pavel Emelyanov53083772008-04-17 23:18:15 -0700105void __inet_inherit_port(struct sock *sk, struct sock *child)
106{
107 struct inet_hashinfo *table = sk->sk_prot->h.hashinfo;
Pavel Emelyanov7f635ab2008-06-16 17:12:49 -0700108 const int bhash = inet_bhashfn(sock_net(sk), inet_sk(child)->num,
109 table->bhash_size);
Pavel Emelyanov53083772008-04-17 23:18:15 -0700110 struct inet_bind_hashbucket *head = &table->bhash[bhash];
111 struct inet_bind_bucket *tb;
112
113 spin_lock(&head->lock);
114 tb = inet_csk(sk)->icsk_bind_hash;
115 sk_add_bind_node(child, &tb->owners);
116 inet_csk(child)->icsk_bind_hash = tb;
117 spin_unlock(&head->lock);
118}
119
120EXPORT_SYMBOL_GPL(__inet_inherit_port);
121
Eric Dumazetc25eb3b2008-11-23 17:22:55 -0800122static inline int compute_score(struct sock *sk, struct net *net,
123 const unsigned short hnum, const __be32 daddr,
124 const int dif)
125{
126 int score = -1;
127 struct inet_sock *inet = inet_sk(sk);
128
129 if (net_eq(sock_net(sk), net) && inet->num == hnum &&
130 !ipv6_only_sock(sk)) {
131 __be32 rcv_saddr = inet->rcv_saddr;
132 score = sk->sk_family == PF_INET ? 1 : 0;
133 if (rcv_saddr) {
134 if (rcv_saddr != daddr)
135 return -1;
136 score += 2;
137 }
138 if (sk->sk_bound_dev_if) {
139 if (sk->sk_bound_dev_if != dif)
140 return -1;
141 score += 2;
142 }
143 }
144 return score;
145}
146
Arnaldo Carvalho de Melof3f05f72005-08-09 20:08:09 -0700147/*
Arnaldo Carvalho de Melo33b62232005-08-09 20:09:06 -0700148 * Don't inline this cruft. Here are some nice properties to exploit here. The
149 * BSD API does not allow a listening sock to specify the remote port nor the
150 * remote address for the connection. So always assume those are both
151 * wildcarded during the search since they can never be otherwise.
152 */
Arnaldo Carvalho de Melo33b62232005-08-09 20:09:06 -0700153
Arnaldo Carvalho de Melo33b62232005-08-09 20:09:06 -0700154
Pavel Emelyanovc67499c2008-01-31 05:06:40 -0800155struct sock *__inet_lookup_listener(struct net *net,
156 struct inet_hashinfo *hashinfo,
Al Virofb99c842006-09-27 18:43:33 -0700157 const __be32 daddr, const unsigned short hnum,
Herbert Xu8f491062006-08-09 15:47:12 -0700158 const int dif)
Herbert Xu99a92ff2006-08-08 02:18:10 -0700159{
Eric Dumazetc25eb3b2008-11-23 17:22:55 -0800160 struct sock *sk, *result;
161 struct hlist_nulls_node *node;
162 unsigned int hash = inet_lhashfn(net, hnum);
163 struct inet_listen_hashbucket *ilb = &hashinfo->listening_hash[hash];
164 int score, hiscore;
Herbert Xu99a92ff2006-08-08 02:18:10 -0700165
Eric Dumazetc25eb3b2008-11-23 17:22:55 -0800166 rcu_read_lock();
167begin:
168 result = NULL;
169 hiscore = -1;
170 sk_nulls_for_each_rcu(sk, node, &ilb->head) {
171 score = compute_score(sk, net, hnum, daddr, dif);
172 if (score > hiscore) {
173 result = sk;
174 hiscore = score;
175 }
Herbert Xu99a92ff2006-08-08 02:18:10 -0700176 }
Eric Dumazetc25eb3b2008-11-23 17:22:55 -0800177 /*
178 * if the nulls value we got at the end of this lookup is
179 * not the expected one, we must restart lookup.
180 * We probably met an item that was moved to another chain.
181 */
182 if (get_nulls_value(node) != hash + LISTENING_NULLS_BASE)
183 goto begin;
184 if (result) {
185 if (unlikely(!atomic_inc_not_zero(&result->sk_refcnt)))
186 result = NULL;
187 else if (unlikely(compute_score(result, net, hnum, daddr,
188 dif) < hiscore)) {
189 sock_put(result);
190 goto begin;
191 }
Herbert Xu99a92ff2006-08-08 02:18:10 -0700192 }
Eric Dumazetc25eb3b2008-11-23 17:22:55 -0800193 rcu_read_unlock();
194 return result;
Herbert Xu99a92ff2006-08-08 02:18:10 -0700195}
Herbert Xu8f491062006-08-09 15:47:12 -0700196EXPORT_SYMBOL_GPL(__inet_lookup_listener);
Arnaldo Carvalho de Meloa7f5e7f2005-12-13 23:25:31 -0800197
Pavel Emelyanovc67499c2008-01-31 05:06:40 -0800198struct sock * __inet_lookup_established(struct net *net,
199 struct inet_hashinfo *hashinfo,
Pavel Emelyanov77a5ba52007-12-20 15:32:17 -0800200 const __be32 saddr, const __be16 sport,
201 const __be32 daddr, const u16 hnum,
202 const int dif)
203{
204 INET_ADDR_COOKIE(acookie, saddr, daddr)
205 const __portpair ports = INET_COMBINED_PORTS(sport, hnum);
206 struct sock *sk;
Eric Dumazet3ab5aee2008-11-16 19:40:17 -0800207 const struct hlist_nulls_node *node;
Pavel Emelyanov77a5ba52007-12-20 15:32:17 -0800208 /* Optimize here for direct hit, only listening connections can
209 * have wildcards anyways.
210 */
Pavel Emelyanov9f26b3a2008-06-16 17:13:27 -0700211 unsigned int hash = inet_ehashfn(net, daddr, hnum, saddr, sport);
Eric Dumazet3ab5aee2008-11-16 19:40:17 -0800212 unsigned int slot = hash & (hashinfo->ehash_size - 1);
213 struct inet_ehash_bucket *head = &hashinfo->ehash[slot];
Pavel Emelyanov77a5ba52007-12-20 15:32:17 -0800214
Eric Dumazet3ab5aee2008-11-16 19:40:17 -0800215 rcu_read_lock();
216begin:
217 sk_nulls_for_each_rcu(sk, node, &head->chain) {
Pavel Emelyanovc67499c2008-01-31 05:06:40 -0800218 if (INET_MATCH(sk, net, hash, acookie,
Eric Dumazet3ab5aee2008-11-16 19:40:17 -0800219 saddr, daddr, ports, dif)) {
220 if (unlikely(!atomic_inc_not_zero(&sk->sk_refcnt)))
221 goto begintw;
222 if (unlikely(!INET_MATCH(sk, net, hash, acookie,
223 saddr, daddr, ports, dif))) {
224 sock_put(sk);
225 goto begin;
226 }
227 goto out;
228 }
Pavel Emelyanov77a5ba52007-12-20 15:32:17 -0800229 }
Eric Dumazet3ab5aee2008-11-16 19:40:17 -0800230 /*
231 * if the nulls value we got at the end of this lookup is
232 * not the expected one, we must restart lookup.
233 * We probably met an item that was moved to another chain.
234 */
235 if (get_nulls_value(node) != slot)
236 goto begin;
Pavel Emelyanov77a5ba52007-12-20 15:32:17 -0800237
Eric Dumazet3ab5aee2008-11-16 19:40:17 -0800238begintw:
Pavel Emelyanov77a5ba52007-12-20 15:32:17 -0800239 /* Must check for a TIME_WAIT'er before going to listener hash. */
Eric Dumazet3ab5aee2008-11-16 19:40:17 -0800240 sk_nulls_for_each_rcu(sk, node, &head->twchain) {
Pavel Emelyanovc67499c2008-01-31 05:06:40 -0800241 if (INET_TW_MATCH(sk, net, hash, acookie,
Eric Dumazet3ab5aee2008-11-16 19:40:17 -0800242 saddr, daddr, ports, dif)) {
243 if (unlikely(!atomic_inc_not_zero(&sk->sk_refcnt))) {
244 sk = NULL;
245 goto out;
246 }
247 if (unlikely(!INET_TW_MATCH(sk, net, hash, acookie,
248 saddr, daddr, ports, dif))) {
249 sock_put(sk);
250 goto begintw;
251 }
252 goto out;
253 }
Pavel Emelyanov77a5ba52007-12-20 15:32:17 -0800254 }
Eric Dumazet3ab5aee2008-11-16 19:40:17 -0800255 /*
256 * if the nulls value we got at the end of this lookup is
257 * not the expected one, we must restart lookup.
258 * We probably met an item that was moved to another chain.
259 */
260 if (get_nulls_value(node) != slot)
261 goto begintw;
Pavel Emelyanov77a5ba52007-12-20 15:32:17 -0800262 sk = NULL;
263out:
Eric Dumazet3ab5aee2008-11-16 19:40:17 -0800264 rcu_read_unlock();
Pavel Emelyanov77a5ba52007-12-20 15:32:17 -0800265 return sk;
Pavel Emelyanov77a5ba52007-12-20 15:32:17 -0800266}
267EXPORT_SYMBOL_GPL(__inet_lookup_established);
268
Arnaldo Carvalho de Meloa7f5e7f2005-12-13 23:25:31 -0800269/* called with local bh disabled */
270static int __inet_check_established(struct inet_timewait_death_row *death_row,
271 struct sock *sk, __u16 lport,
272 struct inet_timewait_sock **twp)
273{
274 struct inet_hashinfo *hinfo = death_row->hashinfo;
275 struct inet_sock *inet = inet_sk(sk);
Al Virofb99c842006-09-27 18:43:33 -0700276 __be32 daddr = inet->rcv_saddr;
277 __be32 saddr = inet->daddr;
Arnaldo Carvalho de Meloa7f5e7f2005-12-13 23:25:31 -0800278 int dif = sk->sk_bound_dev_if;
279 INET_ADDR_COOKIE(acookie, saddr, daddr)
Al Viro4f765d82006-09-27 18:43:07 -0700280 const __portpair ports = INET_COMBINED_PORTS(inet->dport, lport);
Pavel Emelyanov9f26b3a2008-06-16 17:13:27 -0700281 struct net *net = sock_net(sk);
282 unsigned int hash = inet_ehashfn(net, daddr, lport, saddr, inet->dport);
Arnaldo Carvalho de Meloa7f5e7f2005-12-13 23:25:31 -0800283 struct inet_ehash_bucket *head = inet_ehash_bucket(hinfo, hash);
Eric Dumazet9db66bd2008-11-20 20:39:09 -0800284 spinlock_t *lock = inet_ehash_lockp(hinfo, hash);
Arnaldo Carvalho de Meloa7f5e7f2005-12-13 23:25:31 -0800285 struct sock *sk2;
Eric Dumazet3ab5aee2008-11-16 19:40:17 -0800286 const struct hlist_nulls_node *node;
Arnaldo Carvalho de Meloa7f5e7f2005-12-13 23:25:31 -0800287 struct inet_timewait_sock *tw;
288
Eric Dumazet9db66bd2008-11-20 20:39:09 -0800289 spin_lock(lock);
Arnaldo Carvalho de Meloa7f5e7f2005-12-13 23:25:31 -0800290
291 /* Check TIME-WAIT sockets first. */
Eric Dumazet3ab5aee2008-11-16 19:40:17 -0800292 sk_nulls_for_each(sk2, node, &head->twchain) {
Arnaldo Carvalho de Meloa7f5e7f2005-12-13 23:25:31 -0800293 tw = inet_twsk(sk2);
294
Pavel Emelyanovc67499c2008-01-31 05:06:40 -0800295 if (INET_TW_MATCH(sk2, net, hash, acookie,
296 saddr, daddr, ports, dif)) {
Arnaldo Carvalho de Meloa7f5e7f2005-12-13 23:25:31 -0800297 if (twsk_unique(sk, sk2, twp))
298 goto unique;
299 else
300 goto not_unique;
301 }
302 }
303 tw = NULL;
304
305 /* And established part... */
Eric Dumazet3ab5aee2008-11-16 19:40:17 -0800306 sk_nulls_for_each(sk2, node, &head->chain) {
Pavel Emelyanovc67499c2008-01-31 05:06:40 -0800307 if (INET_MATCH(sk2, net, hash, acookie,
308 saddr, daddr, ports, dif))
Arnaldo Carvalho de Meloa7f5e7f2005-12-13 23:25:31 -0800309 goto not_unique;
310 }
311
312unique:
313 /* Must record num and sport now. Otherwise we will see
314 * in hash table socket with a funny identity. */
315 inet->num = lport;
316 inet->sport = htons(lport);
317 sk->sk_hash = hash;
Ilpo Järvinen547b7922008-07-25 21:43:18 -0700318 WARN_ON(!sk_unhashed(sk));
Eric Dumazet3ab5aee2008-11-16 19:40:17 -0800319 __sk_nulls_add_node_rcu(sk, &head->chain);
Eric Dumazet9db66bd2008-11-20 20:39:09 -0800320 spin_unlock(lock);
Pavel Emelyanovc29a0bc2008-03-31 19:41:46 -0700321 sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1);
Arnaldo Carvalho de Meloa7f5e7f2005-12-13 23:25:31 -0800322
323 if (twp) {
324 *twp = tw;
Pavel Emelyanovde0744a2008-07-16 20:31:16 -0700325 NET_INC_STATS_BH(net, LINUX_MIB_TIMEWAITRECYCLED);
Arnaldo Carvalho de Meloa7f5e7f2005-12-13 23:25:31 -0800326 } else if (tw) {
327 /* Silly. Should hash-dance instead... */
328 inet_twsk_deschedule(tw, death_row);
Pavel Emelyanovde0744a2008-07-16 20:31:16 -0700329 NET_INC_STATS_BH(net, LINUX_MIB_TIMEWAITRECYCLED);
Arnaldo Carvalho de Meloa7f5e7f2005-12-13 23:25:31 -0800330
331 inet_twsk_put(tw);
332 }
333
334 return 0;
335
336not_unique:
Eric Dumazet9db66bd2008-11-20 20:39:09 -0800337 spin_unlock(lock);
Arnaldo Carvalho de Meloa7f5e7f2005-12-13 23:25:31 -0800338 return -EADDRNOTAVAIL;
339}
340
341static inline u32 inet_sk_port_offset(const struct sock *sk)
342{
343 const struct inet_sock *inet = inet_sk(sk);
YOSHIFUJI Hideakie905a9e2007-02-09 23:24:47 +0900344 return secure_ipv4_port_ephemeral(inet->rcv_saddr, inet->daddr,
Arnaldo Carvalho de Meloa7f5e7f2005-12-13 23:25:31 -0800345 inet->dport);
346}
347
Arnaldo Carvalho de Meloab1e0a12008-02-03 04:06:04 -0800348void __inet_hash_nolisten(struct sock *sk)
Pavel Emelyanov152da812007-12-20 15:31:33 -0800349{
Pavel Emelyanov39d8cda2008-03-22 16:50:58 -0700350 struct inet_hashinfo *hashinfo = sk->sk_prot->h.hashinfo;
Eric Dumazet3ab5aee2008-11-16 19:40:17 -0800351 struct hlist_nulls_head *list;
Eric Dumazet9db66bd2008-11-20 20:39:09 -0800352 spinlock_t *lock;
Pavel Emelyanov152da812007-12-20 15:31:33 -0800353 struct inet_ehash_bucket *head;
354
Ilpo Järvinen547b7922008-07-25 21:43:18 -0700355 WARN_ON(!sk_unhashed(sk));
Pavel Emelyanov152da812007-12-20 15:31:33 -0800356
357 sk->sk_hash = inet_sk_ehashfn(sk);
358 head = inet_ehash_bucket(hashinfo, sk->sk_hash);
359 list = &head->chain;
360 lock = inet_ehash_lockp(hashinfo, sk->sk_hash);
361
Eric Dumazet9db66bd2008-11-20 20:39:09 -0800362 spin_lock(lock);
Eric Dumazet3ab5aee2008-11-16 19:40:17 -0800363 __sk_nulls_add_node_rcu(sk, list);
Eric Dumazet9db66bd2008-11-20 20:39:09 -0800364 spin_unlock(lock);
Pavel Emelyanovc29a0bc2008-03-31 19:41:46 -0700365 sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1);
Pavel Emelyanov152da812007-12-20 15:31:33 -0800366}
367EXPORT_SYMBOL_GPL(__inet_hash_nolisten);
368
Arnaldo Carvalho de Meloab1e0a12008-02-03 04:06:04 -0800369static void __inet_hash(struct sock *sk)
Pavel Emelyanov152da812007-12-20 15:31:33 -0800370{
Pavel Emelyanov39d8cda2008-03-22 16:50:58 -0700371 struct inet_hashinfo *hashinfo = sk->sk_prot->h.hashinfo;
Eric Dumazet5caea4e2008-11-20 00:40:07 -0800372 struct inet_listen_hashbucket *ilb;
Pavel Emelyanov152da812007-12-20 15:31:33 -0800373
374 if (sk->sk_state != TCP_LISTEN) {
Arnaldo Carvalho de Meloab1e0a12008-02-03 04:06:04 -0800375 __inet_hash_nolisten(sk);
Pavel Emelyanov152da812007-12-20 15:31:33 -0800376 return;
377 }
378
Ilpo Järvinen547b7922008-07-25 21:43:18 -0700379 WARN_ON(!sk_unhashed(sk));
Eric Dumazet5caea4e2008-11-20 00:40:07 -0800380 ilb = &hashinfo->listening_hash[inet_sk_listen_hashfn(sk)];
Pavel Emelyanov152da812007-12-20 15:31:33 -0800381
Eric Dumazet5caea4e2008-11-20 00:40:07 -0800382 spin_lock(&ilb->lock);
Eric Dumazetc25eb3b2008-11-23 17:22:55 -0800383 __sk_nulls_add_node_rcu(sk, &ilb->head);
Pavel Emelyanovc29a0bc2008-03-31 19:41:46 -0700384 sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1);
Eric Dumazet5caea4e2008-11-20 00:40:07 -0800385 spin_unlock(&ilb->lock);
Pavel Emelyanov152da812007-12-20 15:31:33 -0800386}
Arnaldo Carvalho de Meloab1e0a12008-02-03 04:06:04 -0800387
388void inet_hash(struct sock *sk)
389{
390 if (sk->sk_state != TCP_CLOSE) {
391 local_bh_disable();
392 __inet_hash(sk);
393 local_bh_enable();
394 }
395}
396EXPORT_SYMBOL_GPL(inet_hash);
397
398void inet_unhash(struct sock *sk)
399{
Pavel Emelyanov39d8cda2008-03-22 16:50:58 -0700400 struct inet_hashinfo *hashinfo = sk->sk_prot->h.hashinfo;
Eric Dumazetc25eb3b2008-11-23 17:22:55 -0800401 spinlock_t *lock;
402 int done;
Arnaldo Carvalho de Meloab1e0a12008-02-03 04:06:04 -0800403
404 if (sk_unhashed(sk))
Eric Dumazet5caea4e2008-11-20 00:40:07 -0800405 return;
Arnaldo Carvalho de Meloab1e0a12008-02-03 04:06:04 -0800406
Eric Dumazetc25eb3b2008-11-23 17:22:55 -0800407 if (sk->sk_state == TCP_LISTEN)
408 lock = &hashinfo->listening_hash[inet_sk_listen_hashfn(sk)].lock;
409 else
410 lock = inet_ehash_lockp(hashinfo, sk->sk_hash);
Eric Dumazet5caea4e2008-11-20 00:40:07 -0800411
Eric Dumazetc25eb3b2008-11-23 17:22:55 -0800412 spin_lock_bh(lock);
413 done =__sk_nulls_del_node_init_rcu(sk);
Eric Dumazetc25eb3b2008-11-23 17:22:55 -0800414 if (done)
415 sock_prot_inuse_add(sock_net(sk), sk->sk_prot, -1);
Eric Dumazet920de802008-11-24 00:09:29 -0800416 spin_unlock_bh(lock);
Arnaldo Carvalho de Meloab1e0a12008-02-03 04:06:04 -0800417}
418EXPORT_SYMBOL_GPL(inet_unhash);
Pavel Emelyanov152da812007-12-20 15:31:33 -0800419
Pavel Emelyanov5ee31fc2008-01-31 05:04:45 -0800420int __inet_hash_connect(struct inet_timewait_death_row *death_row,
Pavel Emelyanov5d8c0aa2008-02-05 03:14:44 -0800421 struct sock *sk, u32 port_offset,
Pavel Emelyanov5ee31fc2008-01-31 05:04:45 -0800422 int (*check_established)(struct inet_timewait_death_row *,
423 struct sock *, __u16, struct inet_timewait_sock **),
Arnaldo Carvalho de Meloab1e0a12008-02-03 04:06:04 -0800424 void (*hash)(struct sock *sk))
Arnaldo Carvalho de Meloa7f5e7f2005-12-13 23:25:31 -0800425{
426 struct inet_hashinfo *hinfo = death_row->hashinfo;
427 const unsigned short snum = inet_sk(sk)->num;
YOSHIFUJI Hideakie905a9e2007-02-09 23:24:47 +0900428 struct inet_bind_hashbucket *head;
429 struct inet_bind_bucket *tb;
Arnaldo Carvalho de Meloa7f5e7f2005-12-13 23:25:31 -0800430 int ret;
YOSHIFUJI Hideaki3b1e0a62008-03-26 02:26:21 +0900431 struct net *net = sock_net(sk);
Arnaldo Carvalho de Meloa7f5e7f2005-12-13 23:25:31 -0800432
YOSHIFUJI Hideakie905a9e2007-02-09 23:24:47 +0900433 if (!snum) {
Stephen Hemminger227b60f2007-10-10 17:30:46 -0700434 int i, remaining, low, high, port;
Arnaldo Carvalho de Meloa7f5e7f2005-12-13 23:25:31 -0800435 static u32 hint;
Pavel Emelyanov5d8c0aa2008-02-05 03:14:44 -0800436 u32 offset = hint + port_offset;
Arnaldo Carvalho de Meloa7f5e7f2005-12-13 23:25:31 -0800437 struct hlist_node *node;
YOSHIFUJI Hideakie905a9e2007-02-09 23:24:47 +0900438 struct inet_timewait_sock *tw = NULL;
Arnaldo Carvalho de Meloa7f5e7f2005-12-13 23:25:31 -0800439
Stephen Hemminger227b60f2007-10-10 17:30:46 -0700440 inet_get_local_port_range(&low, &high);
Anton Arapova25de532007-10-18 22:00:17 -0700441 remaining = (high - low) + 1;
Stephen Hemminger227b60f2007-10-10 17:30:46 -0700442
YOSHIFUJI Hideakie905a9e2007-02-09 23:24:47 +0900443 local_bh_disable();
Stephen Hemminger227b60f2007-10-10 17:30:46 -0700444 for (i = 1; i <= remaining; i++) {
445 port = low + (i + offset) % remaining;
Pavel Emelyanov7f635ab2008-06-16 17:12:49 -0700446 head = &hinfo->bhash[inet_bhashfn(net, port,
447 hinfo->bhash_size)];
YOSHIFUJI Hideakie905a9e2007-02-09 23:24:47 +0900448 spin_lock(&head->lock);
Arnaldo Carvalho de Meloa7f5e7f2005-12-13 23:25:31 -0800449
YOSHIFUJI Hideakie905a9e2007-02-09 23:24:47 +0900450 /* Does not bother with rcv_saddr checks,
451 * because the established check is already
452 * unique enough.
453 */
Arnaldo Carvalho de Meloa7f5e7f2005-12-13 23:25:31 -0800454 inet_bind_bucket_for_each(tb, node, &head->chain) {
Eric Dumazet7a9546e2008-11-12 00:54:20 -0800455 if (ib_net(tb) == net && tb->port == port) {
YOSHIFUJI Hideakie905a9e2007-02-09 23:24:47 +0900456 if (tb->fastreuse >= 0)
457 goto next_port;
Evgeniy Polyakova9d8f912009-01-19 16:46:02 -0800458 WARN_ON(hlist_empty(&tb->owners));
Pavel Emelyanov5ee31fc2008-01-31 05:04:45 -0800459 if (!check_established(death_row, sk,
460 port, &tw))
YOSHIFUJI Hideakie905a9e2007-02-09 23:24:47 +0900461 goto ok;
462 goto next_port;
463 }
464 }
Arnaldo Carvalho de Meloa7f5e7f2005-12-13 23:25:31 -0800465
Pavel Emelyanov941b1d22008-01-31 05:05:50 -0800466 tb = inet_bind_bucket_create(hinfo->bind_bucket_cachep,
467 net, head, port);
YOSHIFUJI Hideakie905a9e2007-02-09 23:24:47 +0900468 if (!tb) {
469 spin_unlock(&head->lock);
470 break;
471 }
472 tb->fastreuse = -1;
473 goto ok;
Arnaldo Carvalho de Meloa7f5e7f2005-12-13 23:25:31 -0800474
YOSHIFUJI Hideakie905a9e2007-02-09 23:24:47 +0900475 next_port:
476 spin_unlock(&head->lock);
477 }
478 local_bh_enable();
Arnaldo Carvalho de Meloa7f5e7f2005-12-13 23:25:31 -0800479
YOSHIFUJI Hideakie905a9e2007-02-09 23:24:47 +0900480 return -EADDRNOTAVAIL;
Arnaldo Carvalho de Meloa7f5e7f2005-12-13 23:25:31 -0800481
482ok:
483 hint += i;
484
YOSHIFUJI Hideakie905a9e2007-02-09 23:24:47 +0900485 /* Head lock still held and bh's disabled */
486 inet_bind_hash(sk, tb, port);
Arnaldo Carvalho de Meloa7f5e7f2005-12-13 23:25:31 -0800487 if (sk_unhashed(sk)) {
YOSHIFUJI Hideakie905a9e2007-02-09 23:24:47 +0900488 inet_sk(sk)->sport = htons(port);
Arnaldo Carvalho de Meloab1e0a12008-02-03 04:06:04 -0800489 hash(sk);
YOSHIFUJI Hideakie905a9e2007-02-09 23:24:47 +0900490 }
491 spin_unlock(&head->lock);
Arnaldo Carvalho de Meloa7f5e7f2005-12-13 23:25:31 -0800492
YOSHIFUJI Hideakie905a9e2007-02-09 23:24:47 +0900493 if (tw) {
494 inet_twsk_deschedule(tw, death_row);
495 inet_twsk_put(tw);
496 }
Arnaldo Carvalho de Meloa7f5e7f2005-12-13 23:25:31 -0800497
498 ret = 0;
499 goto out;
YOSHIFUJI Hideakie905a9e2007-02-09 23:24:47 +0900500 }
Arnaldo Carvalho de Meloa7f5e7f2005-12-13 23:25:31 -0800501
Pavel Emelyanov7f635ab2008-06-16 17:12:49 -0700502 head = &hinfo->bhash[inet_bhashfn(net, snum, hinfo->bhash_size)];
YOSHIFUJI Hideakie905a9e2007-02-09 23:24:47 +0900503 tb = inet_csk(sk)->icsk_bind_hash;
Arnaldo Carvalho de Meloa7f5e7f2005-12-13 23:25:31 -0800504 spin_lock_bh(&head->lock);
505 if (sk_head(&tb->owners) == sk && !sk->sk_bind_node.next) {
Arnaldo Carvalho de Meloab1e0a12008-02-03 04:06:04 -0800506 hash(sk);
Arnaldo Carvalho de Meloa7f5e7f2005-12-13 23:25:31 -0800507 spin_unlock_bh(&head->lock);
508 return 0;
509 } else {
510 spin_unlock(&head->lock);
511 /* No definite answer... Walk to established hash table */
Pavel Emelyanov5ee31fc2008-01-31 05:04:45 -0800512 ret = check_established(death_row, sk, snum, NULL);
Arnaldo Carvalho de Meloa7f5e7f2005-12-13 23:25:31 -0800513out:
514 local_bh_enable();
515 return ret;
516 }
517}
Pavel Emelyanov5ee31fc2008-01-31 05:04:45 -0800518
519/*
520 * Bind a port for a connect operation and hash it.
521 */
522int inet_hash_connect(struct inet_timewait_death_row *death_row,
523 struct sock *sk)
524{
Pavel Emelyanov5d8c0aa2008-02-05 03:14:44 -0800525 return __inet_hash_connect(death_row, sk, inet_sk_port_offset(sk),
Pavel Emelyanov5ee31fc2008-01-31 05:04:45 -0800526 __inet_check_established, __inet_hash_nolisten);
527}
Arnaldo Carvalho de Meloa7f5e7f2005-12-13 23:25:31 -0800528
529EXPORT_SYMBOL_GPL(inet_hash_connect);
Eric Dumazet5caea4e2008-11-20 00:40:07 -0800530
531void inet_hashinfo_init(struct inet_hashinfo *h)
532{
533 int i;
534
Eric Dumazet24dd1fa2009-02-01 12:31:33 -0800535 atomic_set(&h->bsockets, 0);
Eric Dumazetc25eb3b2008-11-23 17:22:55 -0800536 for (i = 0; i < INET_LHTABLE_SIZE; i++) {
Eric Dumazet5caea4e2008-11-20 00:40:07 -0800537 spin_lock_init(&h->listening_hash[i].lock);
Eric Dumazetc25eb3b2008-11-23 17:22:55 -0800538 INIT_HLIST_NULLS_HEAD(&h->listening_hash[i].head,
539 i + LISTENING_NULLS_BASE);
540 }
Eric Dumazet5caea4e2008-11-20 00:40:07 -0800541}
542
543EXPORT_SYMBOL_GPL(inet_hashinfo_init);