blob: 9111a4e221557173df0ce08e95632ee059d00b61 [file] [log] [blame]
Arnaldo Carvalho de Melo77d8bf92005-08-09 20:00:51 -07001/*
2 * INET An implementation of the TCP/IP protocol suite for the LINUX
3 * operating system. INET is implemented using the BSD Socket
4 * interface as the means of communication with the user level.
5 *
6 * Generic INET transport hashtables
7 *
8 * Authors: Lotsa people, from code originally in tcp
9 *
10 * This program is free software; you can redistribute it and/or
11 * modify it under the terms of the GNU General Public License
12 * as published by the Free Software Foundation; either version
13 * 2 of the License, or (at your option) any later version.
14 */
15
Arnaldo Carvalho de Melo2d8c4ce2005-08-09 20:07:13 -070016#include <linux/module.h>
Arnaldo Carvalho de Meloa7f5e7f2005-12-13 23:25:31 -080017#include <linux/random.h>
Arnaldo Carvalho de Melof3f05f72005-08-09 20:08:09 -070018#include <linux/sched.h>
Arnaldo Carvalho de Melo77d8bf92005-08-09 20:00:51 -070019#include <linux/slab.h>
Arnaldo Carvalho de Melof3f05f72005-08-09 20:08:09 -070020#include <linux/wait.h>
Arnaldo Carvalho de Melo77d8bf92005-08-09 20:00:51 -070021
Arnaldo Carvalho de Melo463c84b2005-08-09 20:10:42 -070022#include <net/inet_connection_sock.h>
Arnaldo Carvalho de Melo77d8bf92005-08-09 20:00:51 -070023#include <net/inet_hashtables.h>
David S. Miller6e5714e2011-08-03 20:50:44 -070024#include <net/secure_seq.h>
Arnaldo Carvalho de Meloa7f5e7f2005-12-13 23:25:31 -080025#include <net/ip.h>
Arnaldo Carvalho de Melo77d8bf92005-08-09 20:00:51 -070026
Hannes Frederic Sowa65cd8032013-10-19 21:48:51 +020027static unsigned int inet_ehashfn(struct net *net, const __be32 laddr,
28 const __u16 lport, const __be32 faddr,
29 const __be16 fport)
30{
Hannes Frederic Sowa1bbdcee2013-10-19 21:48:57 +020031 static u32 inet_ehash_secret __read_mostly;
32
33 net_get_random_once(&inet_ehash_secret, sizeof(inet_ehash_secret));
34
Hannes Frederic Sowa65cd8032013-10-19 21:48:51 +020035 return __inet_ehashfn(laddr, lport, faddr, fport,
36 inet_ehash_secret + net_hash_mix(net));
37}
38
39
40static unsigned int inet_sk_ehashfn(const struct sock *sk)
41{
42 const struct inet_sock *inet = inet_sk(sk);
43 const __be32 laddr = inet->inet_rcv_saddr;
44 const __u16 lport = inet->inet_num;
45 const __be32 faddr = inet->inet_daddr;
46 const __be16 fport = inet->inet_dport;
47 struct net *net = sock_net(sk);
48
49 return inet_ehashfn(net, laddr, lport, faddr, fport);
50}
51
Arnaldo Carvalho de Melo77d8bf92005-08-09 20:00:51 -070052/*
53 * Allocate and initialize a new local port bind bucket.
54 * The bindhash mutex for snum's hash chain must be held here.
55 */
Christoph Lametere18b8902006-12-06 20:33:20 -080056struct inet_bind_bucket *inet_bind_bucket_create(struct kmem_cache *cachep,
Pavel Emelyanov941b1d22008-01-31 05:05:50 -080057 struct net *net,
Arnaldo Carvalho de Melo77d8bf92005-08-09 20:00:51 -070058 struct inet_bind_hashbucket *head,
59 const unsigned short snum)
60{
Christoph Lameter54e6ecb2006-12-06 20:33:16 -080061 struct inet_bind_bucket *tb = kmem_cache_alloc(cachep, GFP_ATOMIC);
Arnaldo Carvalho de Melo77d8bf92005-08-09 20:00:51 -070062
63 if (tb != NULL) {
Eric Dumazet7a9546e2008-11-12 00:54:20 -080064 write_pnet(&tb->ib_net, hold_net(net));
Arnaldo Carvalho de Melo77d8bf92005-08-09 20:00:51 -070065 tb->port = snum;
66 tb->fastreuse = 0;
Tom Herbertda5e3632013-01-22 09:50:24 +000067 tb->fastreuseport = 0;
Evgeniy Polyakova9d8f912009-01-19 16:46:02 -080068 tb->num_owners = 0;
Arnaldo Carvalho de Melo77d8bf92005-08-09 20:00:51 -070069 INIT_HLIST_HEAD(&tb->owners);
70 hlist_add_head(&tb->node, &head->chain);
71 }
72 return tb;
73}
74
Arnaldo Carvalho de Melo77d8bf92005-08-09 20:00:51 -070075/*
76 * Caller must hold hashbucket lock for this tb with local BH disabled
77 */
Christoph Lametere18b8902006-12-06 20:33:20 -080078void inet_bind_bucket_destroy(struct kmem_cache *cachep, struct inet_bind_bucket *tb)
Arnaldo Carvalho de Melo77d8bf92005-08-09 20:00:51 -070079{
80 if (hlist_empty(&tb->owners)) {
81 __hlist_del(&tb->node);
Eric Dumazet7a9546e2008-11-12 00:54:20 -080082 release_net(ib_net(tb));
Arnaldo Carvalho de Melo77d8bf92005-08-09 20:00:51 -070083 kmem_cache_free(cachep, tb);
84 }
85}
Arnaldo Carvalho de Melo2d8c4ce2005-08-09 20:07:13 -070086
87void inet_bind_hash(struct sock *sk, struct inet_bind_bucket *tb,
88 const unsigned short snum)
89{
Evgeniy Polyakova9d8f912009-01-19 16:46:02 -080090 struct inet_hashinfo *hashinfo = sk->sk_prot->h.hashinfo;
91
Eric Dumazet24dd1fa2009-02-01 12:31:33 -080092 atomic_inc(&hashinfo->bsockets);
Evgeniy Polyakova9d8f912009-01-19 16:46:02 -080093
Eric Dumazetc720c7e2009-10-15 06:30:45 +000094 inet_sk(sk)->inet_num = snum;
Arnaldo Carvalho de Melo2d8c4ce2005-08-09 20:07:13 -070095 sk_add_bind_node(sk, &tb->owners);
Evgeniy Polyakova9d8f912009-01-19 16:46:02 -080096 tb->num_owners++;
Arnaldo Carvalho de Melo463c84b2005-08-09 20:10:42 -070097 inet_csk(sk)->icsk_bind_hash = tb;
Arnaldo Carvalho de Melo2d8c4ce2005-08-09 20:07:13 -070098}
99
Arnaldo Carvalho de Melo2d8c4ce2005-08-09 20:07:13 -0700100/*
101 * Get rid of any references to a local port held by the given sock.
102 */
Arnaldo Carvalho de Meloab1e0a12008-02-03 04:06:04 -0800103static void __inet_put_port(struct sock *sk)
Arnaldo Carvalho de Melo2d8c4ce2005-08-09 20:07:13 -0700104{
Pavel Emelyanov39d8cda2008-03-22 16:50:58 -0700105 struct inet_hashinfo *hashinfo = sk->sk_prot->h.hashinfo;
Eric Dumazetc720c7e2009-10-15 06:30:45 +0000106 const int bhash = inet_bhashfn(sock_net(sk), inet_sk(sk)->inet_num,
Pavel Emelyanov7f635ab2008-06-16 17:12:49 -0700107 hashinfo->bhash_size);
Arnaldo Carvalho de Melo2d8c4ce2005-08-09 20:07:13 -0700108 struct inet_bind_hashbucket *head = &hashinfo->bhash[bhash];
109 struct inet_bind_bucket *tb;
110
Eric Dumazet24dd1fa2009-02-01 12:31:33 -0800111 atomic_dec(&hashinfo->bsockets);
Evgeniy Polyakova9d8f912009-01-19 16:46:02 -0800112
Arnaldo Carvalho de Melo2d8c4ce2005-08-09 20:07:13 -0700113 spin_lock(&head->lock);
Arnaldo Carvalho de Melo463c84b2005-08-09 20:10:42 -0700114 tb = inet_csk(sk)->icsk_bind_hash;
Arnaldo Carvalho de Melo2d8c4ce2005-08-09 20:07:13 -0700115 __sk_del_bind_node(sk);
Evgeniy Polyakova9d8f912009-01-19 16:46:02 -0800116 tb->num_owners--;
Arnaldo Carvalho de Melo463c84b2005-08-09 20:10:42 -0700117 inet_csk(sk)->icsk_bind_hash = NULL;
Eric Dumazetc720c7e2009-10-15 06:30:45 +0000118 inet_sk(sk)->inet_num = 0;
Arnaldo Carvalho de Melo2d8c4ce2005-08-09 20:07:13 -0700119 inet_bind_bucket_destroy(hashinfo->bind_bucket_cachep, tb);
120 spin_unlock(&head->lock);
121}
122
Arnaldo Carvalho de Meloab1e0a12008-02-03 04:06:04 -0800123void inet_put_port(struct sock *sk)
Arnaldo Carvalho de Melo2d8c4ce2005-08-09 20:07:13 -0700124{
125 local_bh_disable();
Arnaldo Carvalho de Meloab1e0a12008-02-03 04:06:04 -0800126 __inet_put_port(sk);
Arnaldo Carvalho de Melo2d8c4ce2005-08-09 20:07:13 -0700127 local_bh_enable();
128}
Arnaldo Carvalho de Melo2d8c4ce2005-08-09 20:07:13 -0700129EXPORT_SYMBOL(inet_put_port);
Arnaldo Carvalho de Melof3f05f72005-08-09 20:08:09 -0700130
Balazs Scheidler093d2822010-10-21 13:06:43 +0200131int __inet_inherit_port(struct sock *sk, struct sock *child)
Pavel Emelyanov53083772008-04-17 23:18:15 -0700132{
133 struct inet_hashinfo *table = sk->sk_prot->h.hashinfo;
Balazs Scheidler093d2822010-10-21 13:06:43 +0200134 unsigned short port = inet_sk(child)->inet_num;
135 const int bhash = inet_bhashfn(sock_net(sk), port,
Pavel Emelyanov7f635ab2008-06-16 17:12:49 -0700136 table->bhash_size);
Pavel Emelyanov53083772008-04-17 23:18:15 -0700137 struct inet_bind_hashbucket *head = &table->bhash[bhash];
138 struct inet_bind_bucket *tb;
139
140 spin_lock(&head->lock);
141 tb = inet_csk(sk)->icsk_bind_hash;
Balazs Scheidler093d2822010-10-21 13:06:43 +0200142 if (tb->port != port) {
143 /* NOTE: using tproxy and redirecting skbs to a proxy
144 * on a different listener port breaks the assumption
145 * that the listener socket's icsk_bind_hash is the same
146 * as that of the child socket. We have to look up or
147 * create a new bind bucket for the child here. */
Sasha Levinb67bfe02013-02-27 17:06:00 -0800148 inet_bind_bucket_for_each(tb, &head->chain) {
Balazs Scheidler093d2822010-10-21 13:06:43 +0200149 if (net_eq(ib_net(tb), sock_net(sk)) &&
150 tb->port == port)
151 break;
152 }
Sasha Levinb67bfe02013-02-27 17:06:00 -0800153 if (!tb) {
Balazs Scheidler093d2822010-10-21 13:06:43 +0200154 tb = inet_bind_bucket_create(table->bind_bucket_cachep,
155 sock_net(sk), head, port);
156 if (!tb) {
157 spin_unlock(&head->lock);
158 return -ENOMEM;
159 }
160 }
161 }
Nagendra Tomarb4ff3c92010-11-26 14:26:27 +0000162 inet_bind_hash(child, tb, port);
Pavel Emelyanov53083772008-04-17 23:18:15 -0700163 spin_unlock(&head->lock);
Balazs Scheidler093d2822010-10-21 13:06:43 +0200164
165 return 0;
Pavel Emelyanov53083772008-04-17 23:18:15 -0700166}
Pavel Emelyanov53083772008-04-17 23:18:15 -0700167EXPORT_SYMBOL_GPL(__inet_inherit_port);
168
Eric Dumazetc25eb3b2008-11-23 17:22:55 -0800169static inline int compute_score(struct sock *sk, struct net *net,
170 const unsigned short hnum, const __be32 daddr,
171 const int dif)
172{
173 int score = -1;
174 struct inet_sock *inet = inet_sk(sk);
175
Eric Dumazetc720c7e2009-10-15 06:30:45 +0000176 if (net_eq(sock_net(sk), net) && inet->inet_num == hnum &&
Eric Dumazetc25eb3b2008-11-23 17:22:55 -0800177 !ipv6_only_sock(sk)) {
Eric Dumazetc720c7e2009-10-15 06:30:45 +0000178 __be32 rcv_saddr = inet->inet_rcv_saddr;
Tom Herbertda5e3632013-01-22 09:50:24 +0000179 score = sk->sk_family == PF_INET ? 2 : 1;
Eric Dumazetc25eb3b2008-11-23 17:22:55 -0800180 if (rcv_saddr) {
181 if (rcv_saddr != daddr)
182 return -1;
Tom Herbertda5e3632013-01-22 09:50:24 +0000183 score += 4;
Eric Dumazetc25eb3b2008-11-23 17:22:55 -0800184 }
185 if (sk->sk_bound_dev_if) {
186 if (sk->sk_bound_dev_if != dif)
187 return -1;
Tom Herbertda5e3632013-01-22 09:50:24 +0000188 score += 4;
Eric Dumazetc25eb3b2008-11-23 17:22:55 -0800189 }
190 }
191 return score;
192}
193
Arnaldo Carvalho de Melof3f05f72005-08-09 20:08:09 -0700194/*
Arnaldo Carvalho de Melo33b62232005-08-09 20:09:06 -0700195 * Don't inline this cruft. Here are some nice properties to exploit here. The
196 * BSD API does not allow a listening sock to specify the remote port nor the
197 * remote address for the connection. So always assume those are both
198 * wildcarded during the search since they can never be otherwise.
199 */
Arnaldo Carvalho de Melo33b62232005-08-09 20:09:06 -0700200
Arnaldo Carvalho de Melo33b62232005-08-09 20:09:06 -0700201
Pavel Emelyanovc67499c2008-01-31 05:06:40 -0800202struct sock *__inet_lookup_listener(struct net *net,
203 struct inet_hashinfo *hashinfo,
Tom Herbertda5e3632013-01-22 09:50:24 +0000204 const __be32 saddr, __be16 sport,
Al Virofb99c842006-09-27 18:43:33 -0700205 const __be32 daddr, const unsigned short hnum,
Herbert Xu8f4910692006-08-09 15:47:12 -0700206 const int dif)
Herbert Xu99a92ff2006-08-08 02:18:10 -0700207{
Eric Dumazetc25eb3b2008-11-23 17:22:55 -0800208 struct sock *sk, *result;
209 struct hlist_nulls_node *node;
210 unsigned int hash = inet_lhashfn(net, hnum);
211 struct inet_listen_hashbucket *ilb = &hashinfo->listening_hash[hash];
Tom Herbertda5e3632013-01-22 09:50:24 +0000212 int score, hiscore, matches = 0, reuseport = 0;
213 u32 phash = 0;
Herbert Xu99a92ff2006-08-08 02:18:10 -0700214
Eric Dumazetc25eb3b2008-11-23 17:22:55 -0800215 rcu_read_lock();
216begin:
217 result = NULL;
Tom Herbertda5e3632013-01-22 09:50:24 +0000218 hiscore = 0;
Eric Dumazetc25eb3b2008-11-23 17:22:55 -0800219 sk_nulls_for_each_rcu(sk, node, &ilb->head) {
220 score = compute_score(sk, net, hnum, daddr, dif);
221 if (score > hiscore) {
222 result = sk;
223 hiscore = score;
Tom Herbertda5e3632013-01-22 09:50:24 +0000224 reuseport = sk->sk_reuseport;
225 if (reuseport) {
226 phash = inet_ehashfn(net, daddr, hnum,
227 saddr, sport);
228 matches = 1;
229 }
230 } else if (score == hiscore && reuseport) {
231 matches++;
Daniel Borkmann8fc54f62014-08-23 20:58:54 +0200232 if (reciprocal_scale(phash, matches) == 0)
Tom Herbertda5e3632013-01-22 09:50:24 +0000233 result = sk;
234 phash = next_pseudo_random32(phash);
Eric Dumazetc25eb3b2008-11-23 17:22:55 -0800235 }
Herbert Xu99a92ff2006-08-08 02:18:10 -0700236 }
Eric Dumazetc25eb3b2008-11-23 17:22:55 -0800237 /*
238 * if the nulls value we got at the end of this lookup is
239 * not the expected one, we must restart lookup.
240 * We probably met an item that was moved to another chain.
241 */
242 if (get_nulls_value(node) != hash + LISTENING_NULLS_BASE)
243 goto begin;
244 if (result) {
245 if (unlikely(!atomic_inc_not_zero(&result->sk_refcnt)))
246 result = NULL;
247 else if (unlikely(compute_score(result, net, hnum, daddr,
248 dif) < hiscore)) {
249 sock_put(result);
250 goto begin;
251 }
Herbert Xu99a92ff2006-08-08 02:18:10 -0700252 }
Eric Dumazetc25eb3b2008-11-23 17:22:55 -0800253 rcu_read_unlock();
254 return result;
Herbert Xu99a92ff2006-08-08 02:18:10 -0700255}
Herbert Xu8f4910692006-08-09 15:47:12 -0700256EXPORT_SYMBOL_GPL(__inet_lookup_listener);
Arnaldo Carvalho de Meloa7f5e7f2005-12-13 23:25:31 -0800257
Eric Dumazet05dbc7b2013-10-03 00:22:02 -0700258/* All sockets share common refcount, but have different destructors */
259void sock_gen_put(struct sock *sk)
260{
261 if (!atomic_dec_and_test(&sk->sk_refcnt))
262 return;
263
264 if (sk->sk_state == TCP_TIME_WAIT)
265 inet_twsk_free(inet_twsk(sk));
266 else
267 sk_free(sk);
268}
269EXPORT_SYMBOL_GPL(sock_gen_put);
270
Daniel Baluta5e73ea12012-04-15 01:34:41 +0000271struct sock *__inet_lookup_established(struct net *net,
Pavel Emelyanovc67499c2008-01-31 05:06:40 -0800272 struct inet_hashinfo *hashinfo,
Pavel Emelyanov77a5ba52007-12-20 15:32:17 -0800273 const __be32 saddr, const __be16 sport,
274 const __be32 daddr, const u16 hnum,
275 const int dif)
276{
Joe Perchesc7228312014-05-13 20:30:07 -0700277 INET_ADDR_COOKIE(acookie, saddr, daddr);
Pavel Emelyanov77a5ba52007-12-20 15:32:17 -0800278 const __portpair ports = INET_COMBINED_PORTS(sport, hnum);
279 struct sock *sk;
Eric Dumazet3ab5aee2008-11-16 19:40:17 -0800280 const struct hlist_nulls_node *node;
Pavel Emelyanov77a5ba52007-12-20 15:32:17 -0800281 /* Optimize here for direct hit, only listening connections can
282 * have wildcards anyways.
283 */
Pavel Emelyanov9f26b3a2008-06-16 17:13:27 -0700284 unsigned int hash = inet_ehashfn(net, daddr, hnum, saddr, sport);
Eric Dumazetf373b532009-10-09 00:16:19 +0000285 unsigned int slot = hash & hashinfo->ehash_mask;
Eric Dumazet3ab5aee2008-11-16 19:40:17 -0800286 struct inet_ehash_bucket *head = &hashinfo->ehash[slot];
Pavel Emelyanov77a5ba52007-12-20 15:32:17 -0800287
Eric Dumazet3ab5aee2008-11-16 19:40:17 -0800288 rcu_read_lock();
289begin:
290 sk_nulls_for_each_rcu(sk, node, &head->chain) {
Eric Dumazetce43b032012-11-30 09:49:27 +0000291 if (sk->sk_hash != hash)
292 continue;
293 if (likely(INET_MATCH(sk, net, acookie,
294 saddr, daddr, ports, dif))) {
Eric Dumazet3ab5aee2008-11-16 19:40:17 -0800295 if (unlikely(!atomic_inc_not_zero(&sk->sk_refcnt)))
Eric Dumazet05dbc7b2013-10-03 00:22:02 -0700296 goto out;
Eric Dumazetce43b032012-11-30 09:49:27 +0000297 if (unlikely(!INET_MATCH(sk, net, acookie,
298 saddr, daddr, ports, dif))) {
Eric Dumazet05dbc7b2013-10-03 00:22:02 -0700299 sock_gen_put(sk);
Eric Dumazet3ab5aee2008-11-16 19:40:17 -0800300 goto begin;
301 }
Eric Dumazet05dbc7b2013-10-03 00:22:02 -0700302 goto found;
Eric Dumazet3ab5aee2008-11-16 19:40:17 -0800303 }
Pavel Emelyanov77a5ba52007-12-20 15:32:17 -0800304 }
Eric Dumazet3ab5aee2008-11-16 19:40:17 -0800305 /*
306 * if the nulls value we got at the end of this lookup is
307 * not the expected one, we must restart lookup.
308 * We probably met an item that was moved to another chain.
309 */
310 if (get_nulls_value(node) != slot)
311 goto begin;
Pavel Emelyanov77a5ba52007-12-20 15:32:17 -0800312out:
Eric Dumazet05dbc7b2013-10-03 00:22:02 -0700313 sk = NULL;
314found:
Eric Dumazet3ab5aee2008-11-16 19:40:17 -0800315 rcu_read_unlock();
Pavel Emelyanov77a5ba52007-12-20 15:32:17 -0800316 return sk;
Pavel Emelyanov77a5ba52007-12-20 15:32:17 -0800317}
318EXPORT_SYMBOL_GPL(__inet_lookup_established);
319
Arnaldo Carvalho de Meloa7f5e7f2005-12-13 23:25:31 -0800320/* called with local bh disabled */
321static int __inet_check_established(struct inet_timewait_death_row *death_row,
322 struct sock *sk, __u16 lport,
323 struct inet_timewait_sock **twp)
324{
325 struct inet_hashinfo *hinfo = death_row->hashinfo;
326 struct inet_sock *inet = inet_sk(sk);
Eric Dumazetc720c7e2009-10-15 06:30:45 +0000327 __be32 daddr = inet->inet_rcv_saddr;
328 __be32 saddr = inet->inet_daddr;
Arnaldo Carvalho de Meloa7f5e7f2005-12-13 23:25:31 -0800329 int dif = sk->sk_bound_dev_if;
Joe Perchesc7228312014-05-13 20:30:07 -0700330 INET_ADDR_COOKIE(acookie, saddr, daddr);
Eric Dumazetc720c7e2009-10-15 06:30:45 +0000331 const __portpair ports = INET_COMBINED_PORTS(inet->inet_dport, lport);
Pavel Emelyanov9f26b3a2008-06-16 17:13:27 -0700332 struct net *net = sock_net(sk);
Eric Dumazetc720c7e2009-10-15 06:30:45 +0000333 unsigned int hash = inet_ehashfn(net, daddr, lport,
334 saddr, inet->inet_dport);
Arnaldo Carvalho de Meloa7f5e7f2005-12-13 23:25:31 -0800335 struct inet_ehash_bucket *head = inet_ehash_bucket(hinfo, hash);
Eric Dumazet9db66bd2008-11-20 20:39:09 -0800336 spinlock_t *lock = inet_ehash_lockp(hinfo, hash);
Arnaldo Carvalho de Meloa7f5e7f2005-12-13 23:25:31 -0800337 struct sock *sk2;
Eric Dumazet3ab5aee2008-11-16 19:40:17 -0800338 const struct hlist_nulls_node *node;
Eric Dumazet05dbc7b2013-10-03 00:22:02 -0700339 struct inet_timewait_sock *tw = NULL;
Eric Dumazet13475a32009-12-02 22:31:19 +0000340 int twrefcnt = 0;
Arnaldo Carvalho de Meloa7f5e7f2005-12-13 23:25:31 -0800341
Eric Dumazet9db66bd2008-11-20 20:39:09 -0800342 spin_lock(lock);
Arnaldo Carvalho de Meloa7f5e7f2005-12-13 23:25:31 -0800343
Eric Dumazet3ab5aee2008-11-16 19:40:17 -0800344 sk_nulls_for_each(sk2, node, &head->chain) {
Eric Dumazetce43b032012-11-30 09:49:27 +0000345 if (sk2->sk_hash != hash)
346 continue;
Eric Dumazet05dbc7b2013-10-03 00:22:02 -0700347
Eric Dumazetce43b032012-11-30 09:49:27 +0000348 if (likely(INET_MATCH(sk2, net, acookie,
Eric Dumazet05dbc7b2013-10-03 00:22:02 -0700349 saddr, daddr, ports, dif))) {
350 if (sk2->sk_state == TCP_TIME_WAIT) {
351 tw = inet_twsk(sk2);
352 if (twsk_unique(sk, sk2, twp))
353 break;
354 }
Arnaldo Carvalho de Meloa7f5e7f2005-12-13 23:25:31 -0800355 goto not_unique;
Eric Dumazet05dbc7b2013-10-03 00:22:02 -0700356 }
Arnaldo Carvalho de Meloa7f5e7f2005-12-13 23:25:31 -0800357 }
358
Arnaldo Carvalho de Meloa7f5e7f2005-12-13 23:25:31 -0800359 /* Must record num and sport now. Otherwise we will see
Eric Dumazet05dbc7b2013-10-03 00:22:02 -0700360 * in hash table socket with a funny identity.
361 */
Eric Dumazetc720c7e2009-10-15 06:30:45 +0000362 inet->inet_num = lport;
363 inet->inet_sport = htons(lport);
Arnaldo Carvalho de Meloa7f5e7f2005-12-13 23:25:31 -0800364 sk->sk_hash = hash;
Ilpo Järvinen547b7922008-07-25 21:43:18 -0700365 WARN_ON(!sk_unhashed(sk));
Eric Dumazet3ab5aee2008-11-16 19:40:17 -0800366 __sk_nulls_add_node_rcu(sk, &head->chain);
Eric Dumazet13475a32009-12-02 22:31:19 +0000367 if (tw) {
368 twrefcnt = inet_twsk_unhash(tw);
369 NET_INC_STATS_BH(net, LINUX_MIB_TIMEWAITRECYCLED);
370 }
Eric Dumazet9db66bd2008-11-20 20:39:09 -0800371 spin_unlock(lock);
Eric Dumazet13475a32009-12-02 22:31:19 +0000372 if (twrefcnt)
373 inet_twsk_put(tw);
Pavel Emelyanovc29a0bc2008-03-31 19:41:46 -0700374 sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1);
Arnaldo Carvalho de Meloa7f5e7f2005-12-13 23:25:31 -0800375
376 if (twp) {
377 *twp = tw;
Arnaldo Carvalho de Meloa7f5e7f2005-12-13 23:25:31 -0800378 } else if (tw) {
379 /* Silly. Should hash-dance instead... */
380 inet_twsk_deschedule(tw, death_row);
Arnaldo Carvalho de Meloa7f5e7f2005-12-13 23:25:31 -0800381
382 inet_twsk_put(tw);
383 }
Arnaldo Carvalho de Meloa7f5e7f2005-12-13 23:25:31 -0800384 return 0;
385
386not_unique:
Eric Dumazet9db66bd2008-11-20 20:39:09 -0800387 spin_unlock(lock);
Arnaldo Carvalho de Meloa7f5e7f2005-12-13 23:25:31 -0800388 return -EADDRNOTAVAIL;
389}
390
391static inline u32 inet_sk_port_offset(const struct sock *sk)
392{
393 const struct inet_sock *inet = inet_sk(sk);
Eric Dumazetc720c7e2009-10-15 06:30:45 +0000394 return secure_ipv4_port_ephemeral(inet->inet_rcv_saddr,
395 inet->inet_daddr,
396 inet->inet_dport);
Arnaldo Carvalho de Meloa7f5e7f2005-12-13 23:25:31 -0800397}
398
Eric Dumazet9327f702009-12-04 03:46:54 +0000399int __inet_hash_nolisten(struct sock *sk, struct inet_timewait_sock *tw)
Pavel Emelyanov152da812007-12-20 15:31:33 -0800400{
Pavel Emelyanov39d8cda2008-03-22 16:50:58 -0700401 struct inet_hashinfo *hashinfo = sk->sk_prot->h.hashinfo;
Eric Dumazet3ab5aee2008-11-16 19:40:17 -0800402 struct hlist_nulls_head *list;
Eric Dumazet9db66bd2008-11-20 20:39:09 -0800403 spinlock_t *lock;
Pavel Emelyanov152da812007-12-20 15:31:33 -0800404 struct inet_ehash_bucket *head;
Eric Dumazet9327f702009-12-04 03:46:54 +0000405 int twrefcnt = 0;
Pavel Emelyanov152da812007-12-20 15:31:33 -0800406
Ilpo Järvinen547b7922008-07-25 21:43:18 -0700407 WARN_ON(!sk_unhashed(sk));
Pavel Emelyanov152da812007-12-20 15:31:33 -0800408
409 sk->sk_hash = inet_sk_ehashfn(sk);
410 head = inet_ehash_bucket(hashinfo, sk->sk_hash);
411 list = &head->chain;
412 lock = inet_ehash_lockp(hashinfo, sk->sk_hash);
413
Eric Dumazet9db66bd2008-11-20 20:39:09 -0800414 spin_lock(lock);
Eric Dumazet3ab5aee2008-11-16 19:40:17 -0800415 __sk_nulls_add_node_rcu(sk, list);
Eric Dumazet9327f702009-12-04 03:46:54 +0000416 if (tw) {
417 WARN_ON(sk->sk_hash != tw->tw_hash);
418 twrefcnt = inet_twsk_unhash(tw);
419 }
Eric Dumazet9db66bd2008-11-20 20:39:09 -0800420 spin_unlock(lock);
Pavel Emelyanovc29a0bc2008-03-31 19:41:46 -0700421 sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1);
Eric Dumazet9327f702009-12-04 03:46:54 +0000422 return twrefcnt;
Pavel Emelyanov152da812007-12-20 15:31:33 -0800423}
424EXPORT_SYMBOL_GPL(__inet_hash_nolisten);
425
Arnaldo Carvalho de Meloab1e0a12008-02-03 04:06:04 -0800426static void __inet_hash(struct sock *sk)
Pavel Emelyanov152da812007-12-20 15:31:33 -0800427{
Pavel Emelyanov39d8cda2008-03-22 16:50:58 -0700428 struct inet_hashinfo *hashinfo = sk->sk_prot->h.hashinfo;
Eric Dumazet5caea4e2008-11-20 00:40:07 -0800429 struct inet_listen_hashbucket *ilb;
Pavel Emelyanov152da812007-12-20 15:31:33 -0800430
431 if (sk->sk_state != TCP_LISTEN) {
Eric Dumazet9327f702009-12-04 03:46:54 +0000432 __inet_hash_nolisten(sk, NULL);
Pavel Emelyanov152da812007-12-20 15:31:33 -0800433 return;
434 }
435
Ilpo Järvinen547b7922008-07-25 21:43:18 -0700436 WARN_ON(!sk_unhashed(sk));
Eric Dumazet5caea4e2008-11-20 00:40:07 -0800437 ilb = &hashinfo->listening_hash[inet_sk_listen_hashfn(sk)];
Pavel Emelyanov152da812007-12-20 15:31:33 -0800438
Eric Dumazet5caea4e2008-11-20 00:40:07 -0800439 spin_lock(&ilb->lock);
Eric Dumazetc25eb3b2008-11-23 17:22:55 -0800440 __sk_nulls_add_node_rcu(sk, &ilb->head);
Pavel Emelyanovc29a0bc2008-03-31 19:41:46 -0700441 sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1);
Eric Dumazet5caea4e2008-11-20 00:40:07 -0800442 spin_unlock(&ilb->lock);
Pavel Emelyanov152da812007-12-20 15:31:33 -0800443}
Arnaldo Carvalho de Meloab1e0a12008-02-03 04:06:04 -0800444
445void inet_hash(struct sock *sk)
446{
447 if (sk->sk_state != TCP_CLOSE) {
448 local_bh_disable();
449 __inet_hash(sk);
450 local_bh_enable();
451 }
452}
453EXPORT_SYMBOL_GPL(inet_hash);
454
455void inet_unhash(struct sock *sk)
456{
Pavel Emelyanov39d8cda2008-03-22 16:50:58 -0700457 struct inet_hashinfo *hashinfo = sk->sk_prot->h.hashinfo;
Eric Dumazetc25eb3b2008-11-23 17:22:55 -0800458 spinlock_t *lock;
459 int done;
Arnaldo Carvalho de Meloab1e0a12008-02-03 04:06:04 -0800460
461 if (sk_unhashed(sk))
Eric Dumazet5caea4e2008-11-20 00:40:07 -0800462 return;
Arnaldo Carvalho de Meloab1e0a12008-02-03 04:06:04 -0800463
Eric Dumazetc25eb3b2008-11-23 17:22:55 -0800464 if (sk->sk_state == TCP_LISTEN)
465 lock = &hashinfo->listening_hash[inet_sk_listen_hashfn(sk)].lock;
466 else
467 lock = inet_ehash_lockp(hashinfo, sk->sk_hash);
Eric Dumazet5caea4e2008-11-20 00:40:07 -0800468
Eric Dumazetc25eb3b2008-11-23 17:22:55 -0800469 spin_lock_bh(lock);
Camelia Groza3b8ccd42013-07-11 09:55:51 +0300470 done = __sk_nulls_del_node_init_rcu(sk);
Eric Dumazetc25eb3b2008-11-23 17:22:55 -0800471 if (done)
472 sock_prot_inuse_add(sock_net(sk), sk->sk_prot, -1);
Eric Dumazet920de802008-11-24 00:09:29 -0800473 spin_unlock_bh(lock);
Arnaldo Carvalho de Meloab1e0a12008-02-03 04:06:04 -0800474}
475EXPORT_SYMBOL_GPL(inet_unhash);
Pavel Emelyanov152da812007-12-20 15:31:33 -0800476
Pavel Emelyanov5ee31fc2008-01-31 05:04:45 -0800477int __inet_hash_connect(struct inet_timewait_death_row *death_row,
Pavel Emelyanov5d8c0aa2008-02-05 03:14:44 -0800478 struct sock *sk, u32 port_offset,
Pavel Emelyanov5ee31fc2008-01-31 05:04:45 -0800479 int (*check_established)(struct inet_timewait_death_row *,
480 struct sock *, __u16, struct inet_timewait_sock **),
Eric Dumazet9327f702009-12-04 03:46:54 +0000481 int (*hash)(struct sock *sk, struct inet_timewait_sock *twp))
Arnaldo Carvalho de Meloa7f5e7f2005-12-13 23:25:31 -0800482{
483 struct inet_hashinfo *hinfo = death_row->hashinfo;
Eric Dumazetc720c7e2009-10-15 06:30:45 +0000484 const unsigned short snum = inet_sk(sk)->inet_num;
YOSHIFUJI Hideakie905a9e2007-02-09 23:24:47 +0900485 struct inet_bind_hashbucket *head;
486 struct inet_bind_bucket *tb;
Arnaldo Carvalho de Meloa7f5e7f2005-12-13 23:25:31 -0800487 int ret;
YOSHIFUJI Hideaki3b1e0a62008-03-26 02:26:21 +0900488 struct net *net = sock_net(sk);
Eric Dumazet9327f702009-12-04 03:46:54 +0000489 int twrefcnt = 1;
Arnaldo Carvalho de Meloa7f5e7f2005-12-13 23:25:31 -0800490
YOSHIFUJI Hideakie905a9e2007-02-09 23:24:47 +0900491 if (!snum) {
Stephen Hemminger227b60f2007-10-10 17:30:46 -0700492 int i, remaining, low, high, port;
Arnaldo Carvalho de Meloa7f5e7f2005-12-13 23:25:31 -0800493 static u32 hint;
Pavel Emelyanov5d8c0aa2008-02-05 03:14:44 -0800494 u32 offset = hint + port_offset;
YOSHIFUJI Hideakie905a9e2007-02-09 23:24:47 +0900495 struct inet_timewait_sock *tw = NULL;
Arnaldo Carvalho de Meloa7f5e7f2005-12-13 23:25:31 -0800496
Eric W. Biederman0bbf87d2013-09-28 14:10:59 -0700497 inet_get_local_port_range(net, &low, &high);
Anton Arapova25de532007-10-18 22:00:17 -0700498 remaining = (high - low) + 1;
Stephen Hemminger227b60f2007-10-10 17:30:46 -0700499
YOSHIFUJI Hideakie905a9e2007-02-09 23:24:47 +0900500 local_bh_disable();
Stephen Hemminger227b60f2007-10-10 17:30:46 -0700501 for (i = 1; i <= remaining; i++) {
502 port = low + (i + offset) % remaining;
WANG Cong122ff242014-05-12 16:04:53 -0700503 if (inet_is_local_reserved_port(net, port))
Amerigo Wange3826f12010-05-05 00:27:06 +0000504 continue;
Pavel Emelyanov7f635ab2008-06-16 17:12:49 -0700505 head = &hinfo->bhash[inet_bhashfn(net, port,
506 hinfo->bhash_size)];
YOSHIFUJI Hideakie905a9e2007-02-09 23:24:47 +0900507 spin_lock(&head->lock);
Arnaldo Carvalho de Meloa7f5e7f2005-12-13 23:25:31 -0800508
YOSHIFUJI Hideakie905a9e2007-02-09 23:24:47 +0900509 /* Does not bother with rcv_saddr checks,
510 * because the established check is already
511 * unique enough.
512 */
Sasha Levinb67bfe02013-02-27 17:06:00 -0800513 inet_bind_bucket_for_each(tb, &head->chain) {
Octavian Purdila09ad9bc2009-11-25 15:14:13 -0800514 if (net_eq(ib_net(tb), net) &&
515 tb->port == port) {
Tom Herbertda5e3632013-01-22 09:50:24 +0000516 if (tb->fastreuse >= 0 ||
517 tb->fastreuseport >= 0)
YOSHIFUJI Hideakie905a9e2007-02-09 23:24:47 +0900518 goto next_port;
Evgeniy Polyakova9d8f912009-01-19 16:46:02 -0800519 WARN_ON(hlist_empty(&tb->owners));
Pavel Emelyanov5ee31fc2008-01-31 05:04:45 -0800520 if (!check_established(death_row, sk,
521 port, &tw))
YOSHIFUJI Hideakie905a9e2007-02-09 23:24:47 +0900522 goto ok;
523 goto next_port;
524 }
525 }
Arnaldo Carvalho de Meloa7f5e7f2005-12-13 23:25:31 -0800526
Pavel Emelyanov941b1d22008-01-31 05:05:50 -0800527 tb = inet_bind_bucket_create(hinfo->bind_bucket_cachep,
528 net, head, port);
YOSHIFUJI Hideakie905a9e2007-02-09 23:24:47 +0900529 if (!tb) {
530 spin_unlock(&head->lock);
531 break;
532 }
533 tb->fastreuse = -1;
Tom Herbertda5e3632013-01-22 09:50:24 +0000534 tb->fastreuseport = -1;
YOSHIFUJI Hideakie905a9e2007-02-09 23:24:47 +0900535 goto ok;
Arnaldo Carvalho de Meloa7f5e7f2005-12-13 23:25:31 -0800536
YOSHIFUJI Hideakie905a9e2007-02-09 23:24:47 +0900537 next_port:
538 spin_unlock(&head->lock);
539 }
540 local_bh_enable();
Arnaldo Carvalho de Meloa7f5e7f2005-12-13 23:25:31 -0800541
YOSHIFUJI Hideakie905a9e2007-02-09 23:24:47 +0900542 return -EADDRNOTAVAIL;
Arnaldo Carvalho de Meloa7f5e7f2005-12-13 23:25:31 -0800543
544ok:
545 hint += i;
546
YOSHIFUJI Hideakie905a9e2007-02-09 23:24:47 +0900547 /* Head lock still held and bh's disabled */
548 inet_bind_hash(sk, tb, port);
Arnaldo Carvalho de Meloa7f5e7f2005-12-13 23:25:31 -0800549 if (sk_unhashed(sk)) {
Eric Dumazetc720c7e2009-10-15 06:30:45 +0000550 inet_sk(sk)->inet_sport = htons(port);
Eric Dumazet9327f702009-12-04 03:46:54 +0000551 twrefcnt += hash(sk, tw);
YOSHIFUJI Hideakie905a9e2007-02-09 23:24:47 +0900552 }
Eric Dumazet3cdaeda2009-12-04 03:47:42 +0000553 if (tw)
554 twrefcnt += inet_twsk_bind_unhash(tw, hinfo);
YOSHIFUJI Hideakie905a9e2007-02-09 23:24:47 +0900555 spin_unlock(&head->lock);
Arnaldo Carvalho de Meloa7f5e7f2005-12-13 23:25:31 -0800556
YOSHIFUJI Hideakie905a9e2007-02-09 23:24:47 +0900557 if (tw) {
558 inet_twsk_deschedule(tw, death_row);
Eric Dumazet9327f702009-12-04 03:46:54 +0000559 while (twrefcnt) {
560 twrefcnt--;
561 inet_twsk_put(tw);
562 }
YOSHIFUJI Hideakie905a9e2007-02-09 23:24:47 +0900563 }
Arnaldo Carvalho de Meloa7f5e7f2005-12-13 23:25:31 -0800564
565 ret = 0;
566 goto out;
YOSHIFUJI Hideakie905a9e2007-02-09 23:24:47 +0900567 }
Arnaldo Carvalho de Meloa7f5e7f2005-12-13 23:25:31 -0800568
Pavel Emelyanov7f635ab2008-06-16 17:12:49 -0700569 head = &hinfo->bhash[inet_bhashfn(net, snum, hinfo->bhash_size)];
YOSHIFUJI Hideakie905a9e2007-02-09 23:24:47 +0900570 tb = inet_csk(sk)->icsk_bind_hash;
Arnaldo Carvalho de Meloa7f5e7f2005-12-13 23:25:31 -0800571 spin_lock_bh(&head->lock);
572 if (sk_head(&tb->owners) == sk && !sk->sk_bind_node.next) {
Eric Dumazet9327f702009-12-04 03:46:54 +0000573 hash(sk, NULL);
Arnaldo Carvalho de Meloa7f5e7f2005-12-13 23:25:31 -0800574 spin_unlock_bh(&head->lock);
575 return 0;
576 } else {
577 spin_unlock(&head->lock);
578 /* No definite answer... Walk to established hash table */
Pavel Emelyanov5ee31fc2008-01-31 05:04:45 -0800579 ret = check_established(death_row, sk, snum, NULL);
Arnaldo Carvalho de Meloa7f5e7f2005-12-13 23:25:31 -0800580out:
581 local_bh_enable();
582 return ret;
583 }
584}
Pavel Emelyanov5ee31fc2008-01-31 05:04:45 -0800585
586/*
587 * Bind a port for a connect operation and hash it.
588 */
589int inet_hash_connect(struct inet_timewait_death_row *death_row,
590 struct sock *sk)
591{
Pavel Emelyanov5d8c0aa2008-02-05 03:14:44 -0800592 return __inet_hash_connect(death_row, sk, inet_sk_port_offset(sk),
Pavel Emelyanov5ee31fc2008-01-31 05:04:45 -0800593 __inet_check_established, __inet_hash_nolisten);
594}
Arnaldo Carvalho de Meloa7f5e7f2005-12-13 23:25:31 -0800595EXPORT_SYMBOL_GPL(inet_hash_connect);
Eric Dumazet5caea4e2008-11-20 00:40:07 -0800596
597void inet_hashinfo_init(struct inet_hashinfo *h)
598{
599 int i;
600
Eric Dumazet24dd1fa2009-02-01 12:31:33 -0800601 atomic_set(&h->bsockets, 0);
Eric Dumazetc25eb3b2008-11-23 17:22:55 -0800602 for (i = 0; i < INET_LHTABLE_SIZE; i++) {
Eric Dumazet5caea4e2008-11-20 00:40:07 -0800603 spin_lock_init(&h->listening_hash[i].lock);
Eric Dumazetc25eb3b2008-11-23 17:22:55 -0800604 INIT_HLIST_NULLS_HEAD(&h->listening_hash[i].head,
605 i + LISTENING_NULLS_BASE);
606 }
Eric Dumazet5caea4e2008-11-20 00:40:07 -0800607}
Eric Dumazet5caea4e2008-11-20 00:40:07 -0800608EXPORT_SYMBOL_GPL(inet_hashinfo_init);