blob: 00cf28ad45650c801c90c37fb571acb7d1615183 [file] [log] [blame]
Arnaldo Carvalho de Melo5324a042005-08-12 09:26:18 -03001/*
2 * INET An implementation of the TCP/IP protocol suite for the LINUX
3 * operating system. INET is implemented using the BSD Socket
4 * interface as the means of communication with the user level.
5 *
6 * Generic INET6 transport hashtables
7 *
Arnaldo Carvalho de Melod8313f52005-12-13 23:25:44 -08008 * Authors: Lotsa people, from code originally in tcp, generalised here
Ian Morris67ba4152014-08-24 21:53:10 +01009 * by Arnaldo Carvalho de Melo <acme@mandriva.com>
Arnaldo Carvalho de Melo5324a042005-08-12 09:26:18 -030010 *
11 * This program is free software; you can redistribute it and/or
12 * modify it under the terms of the GNU General Public License
13 * as published by the Free Software Foundation; either version
14 * 2 of the License, or (at your option) any later version.
15 */
16
Arnaldo Carvalho de Melo5324a042005-08-12 09:26:18 -030017#include <linux/module.h>
Arnaldo Carvalho de Melod8313f52005-12-13 23:25:44 -080018#include <linux/random.h>
Arnaldo Carvalho de Melo5324a042005-08-12 09:26:18 -030019
Craig Gallekc125e802016-02-10 11:50:40 -050020#include <net/addrconf.h>
Arnaldo Carvalho de Melo5324a042005-08-12 09:26:18 -030021#include <net/inet_connection_sock.h>
22#include <net/inet_hashtables.h>
23#include <net/inet6_hashtables.h>
David S. Miller6e5714e2011-08-03 20:50:44 -070024#include <net/secure_seq.h>
Arnaldo Carvalho de Melod8313f52005-12-13 23:25:44 -080025#include <net/ip.h>
Craig Gallekc125e802016-02-10 11:50:40 -050026#include <net/sock_reuseport.h>
Arnaldo Carvalho de Melo5324a042005-08-12 09:26:18 -030027
Eric Dumazetd1e559d2015-03-18 14:05:35 -070028u32 inet6_ehashfn(const struct net *net,
29 const struct in6_addr *laddr, const u16 lport,
30 const struct in6_addr *faddr, const __be16 fport)
Hannes Frederic Sowab50026b2013-10-19 21:48:52 +020031{
Hannes Frederic Sowa1bbdcee2013-10-19 21:48:57 +020032 static u32 inet6_ehash_secret __read_mostly;
33 static u32 ipv6_hash_secret __read_mostly;
34
35 u32 lhash, fhash;
36
37 net_get_random_once(&inet6_ehash_secret, sizeof(inet6_ehash_secret));
38 net_get_random_once(&ipv6_hash_secret, sizeof(ipv6_hash_secret));
39
40 lhash = (__force u32)laddr->s6_addr32[3];
41 fhash = __ipv6_addr_jhash(faddr, ipv6_hash_secret);
42
Hannes Frederic Sowab50026b2013-10-19 21:48:52 +020043 return __inet6_ehashfn(lhash, lport, fhash, fport,
Hannes Frederic Sowa1bbdcee2013-10-19 21:48:57 +020044 inet6_ehash_secret + net_hash_mix(net));
Hannes Frederic Sowab50026b2013-10-19 21:48:52 +020045}
46
Denis Vlasenkob1a7ffc2006-04-09 22:48:59 -070047/*
48 * Sockets in TCP_CLOSE state are _always_ taken out of the hash, so
49 * we need not check it for TCP lookups anymore, thanks Alexey. -DaveM
50 *
51 * The sockhash lock must be held as a reader here.
52 */
Pavel Emelyanovd86e0da2008-01-31 05:07:21 -080053struct sock *__inet6_lookup_established(struct net *net,
54 struct inet_hashinfo *hashinfo,
Denis Vlasenkob1a7ffc2006-04-09 22:48:59 -070055 const struct in6_addr *saddr,
Al Virod2ecd9c2006-11-08 00:20:00 -080056 const __be16 sport,
Denis Vlasenkob1a7ffc2006-04-09 22:48:59 -070057 const struct in6_addr *daddr,
58 const u16 hnum,
59 const int dif)
60{
61 struct sock *sk;
Eric Dumazet3ab5aee2008-11-16 19:40:17 -080062 const struct hlist_nulls_node *node;
Al Viro4f765d82006-09-27 18:43:07 -070063 const __portpair ports = INET_COMBINED_PORTS(sport, hnum);
Denis Vlasenkob1a7ffc2006-04-09 22:48:59 -070064 /* Optimize here for direct hit, only listening connections can
65 * have wildcards anyways.
66 */
Pavel Emelyanov33de0142008-06-16 17:13:48 -070067 unsigned int hash = inet6_ehashfn(net, daddr, hnum, saddr, sport);
Eric Dumazetf373b532009-10-09 00:16:19 +000068 unsigned int slot = hash & hashinfo->ehash_mask;
Eric Dumazet3ab5aee2008-11-16 19:40:17 -080069 struct inet_ehash_bucket *head = &hashinfo->ehash[slot];
Denis Vlasenkob1a7ffc2006-04-09 22:48:59 -070070
Eric Dumazet3ab5aee2008-11-16 19:40:17 -080071
Eric Dumazet3ab5aee2008-11-16 19:40:17 -080072begin:
73 sk_nulls_for_each_rcu(sk, node, &head->chain) {
Eric Dumazetce43b032012-11-30 09:49:27 +000074 if (sk->sk_hash != hash)
75 continue;
Eric Dumazetefe42082013-10-03 15:42:29 -070076 if (!INET6_MATCH(sk, net, saddr, daddr, ports, dif))
77 continue;
Eric Dumazet05dbc7b2013-10-03 00:22:02 -070078 if (unlikely(!atomic_inc_not_zero(&sk->sk_refcnt)))
79 goto out;
80
Eric Dumazetefe42082013-10-03 15:42:29 -070081 if (unlikely(!INET6_MATCH(sk, net, saddr, daddr, ports, dif))) {
82 sock_gen_put(sk);
83 goto begin;
Eric Dumazet3ab5aee2008-11-16 19:40:17 -080084 }
Eric Dumazetefe42082013-10-03 15:42:29 -070085 goto found;
Denis Vlasenkob1a7ffc2006-04-09 22:48:59 -070086 }
Eric Dumazet3ab5aee2008-11-16 19:40:17 -080087 if (get_nulls_value(node) != slot)
88 goto begin;
Eric Dumazet3ab5aee2008-11-16 19:40:17 -080089out:
Eric Dumazet05dbc7b2013-10-03 00:22:02 -070090 sk = NULL;
91found:
Denis Vlasenkob1a7ffc2006-04-09 22:48:59 -070092 return sk;
93}
94EXPORT_SYMBOL(__inet6_lookup_established);
95
Jesper Juhl42b16b32011-01-17 00:09:38 +010096static inline int compute_score(struct sock *sk, struct net *net,
Eric Dumazetc25eb3b2008-11-23 17:22:55 -080097 const unsigned short hnum,
98 const struct in6_addr *daddr,
99 const int dif)
100{
101 int score = -1;
102
Eric Dumazetc720c7e2009-10-15 06:30:45 +0000103 if (net_eq(sock_net(sk), net) && inet_sk(sk)->inet_num == hnum &&
Eric Dumazetc25eb3b2008-11-23 17:22:55 -0800104 sk->sk_family == PF_INET6) {
Eric Dumazetc25eb3b2008-11-23 17:22:55 -0800105
106 score = 1;
Eric Dumazetefe42082013-10-03 15:42:29 -0700107 if (!ipv6_addr_any(&sk->sk_v6_rcv_saddr)) {
108 if (!ipv6_addr_equal(&sk->sk_v6_rcv_saddr, daddr))
Eric Dumazetc25eb3b2008-11-23 17:22:55 -0800109 return -1;
110 score++;
111 }
112 if (sk->sk_bound_dev_if) {
113 if (sk->sk_bound_dev_if != dif)
114 return -1;
115 score++;
116 }
Eric Dumazet70da2682015-10-08 19:33:21 -0700117 if (sk->sk_incoming_cpu == raw_smp_processor_id())
118 score++;
Eric Dumazetc25eb3b2008-11-23 17:22:55 -0800119 }
120 return score;
121}
122
Eric Dumazet3b24d852016-04-01 08:52:17 -0700123/* called with rcu_read_lock() */
Pavel Emelyanovd86e0da2008-01-31 05:07:21 -0800124struct sock *inet6_lookup_listener(struct net *net,
Craig Galleka5836362016-02-10 11:50:38 -0500125 struct inet_hashinfo *hashinfo,
126 struct sk_buff *skb, int doff,
127 const struct in6_addr *saddr,
Tom Herbert5ba24952013-01-22 09:50:39 +0000128 const __be16 sport, const struct in6_addr *daddr,
Pavel Emelyanovd86e0da2008-01-31 05:07:21 -0800129 const unsigned short hnum, const int dif)
Arnaldo Carvalho de Melo5324a042005-08-12 09:26:18 -0300130{
Eric Dumazetc25eb3b2008-11-23 17:22:55 -0800131 unsigned int hash = inet_lhashfn(net, hnum);
132 struct inet_listen_hashbucket *ilb = &hashinfo->listening_hash[hash];
Eric Dumazet3b24d852016-04-01 08:52:17 -0700133 int score, hiscore = 0, matches = 0, reuseport = 0;
134 struct sock *sk, *result = NULL;
135 u32 phash = 0;
Arnaldo Carvalho de Melo5324a042005-08-12 09:26:18 -0300136
Eric Dumazet3b24d852016-04-01 08:52:17 -0700137 sk_for_each(sk, &ilb->head) {
Eric Dumazetc25eb3b2008-11-23 17:22:55 -0800138 score = compute_score(sk, net, hnum, daddr, dif);
139 if (score > hiscore) {
Eric Dumazet03c5b532016-04-09 08:01:13 -0700140 reuseport = sk->sk_reuseport;
Tom Herbert5ba24952013-01-22 09:50:39 +0000141 if (reuseport) {
142 phash = inet6_ehashfn(net, daddr, hnum,
143 saddr, sport);
Eric Dumazet3b24d852016-04-01 08:52:17 -0700144 result = reuseport_select_sock(sk, phash,
145 skb, doff);
146 if (result)
147 return result;
Tom Herbert5ba24952013-01-22 09:50:39 +0000148 matches = 1;
149 }
Eric Dumazet3b24d852016-04-01 08:52:17 -0700150 result = sk;
Eric Dumazet03c5b532016-04-09 08:01:13 -0700151 hiscore = score;
Tom Herbert5ba24952013-01-22 09:50:39 +0000152 } else if (score == hiscore && reuseport) {
153 matches++;
Daniel Borkmann8fc54f62014-08-23 20:58:54 +0200154 if (reciprocal_scale(phash, matches) == 0)
Tom Herbert5ba24952013-01-22 09:50:39 +0000155 result = sk;
156 phash = next_pseudo_random32(phash);
Arnaldo Carvalho de Melo5324a042005-08-12 09:26:18 -0300157 }
158 }
Arnaldo Carvalho de Melo5324a042005-08-12 09:26:18 -0300159 return result;
160}
Arnaldo Carvalho de Melo5324a042005-08-12 09:26:18 -0300161EXPORT_SYMBOL_GPL(inet6_lookup_listener);
162
Pavel Emelyanovd86e0da2008-01-31 05:07:21 -0800163struct sock *inet6_lookup(struct net *net, struct inet_hashinfo *hashinfo,
Craig Galleka5836362016-02-10 11:50:38 -0500164 struct sk_buff *skb, int doff,
Al Virod2ecd9c2006-11-08 00:20:00 -0800165 const struct in6_addr *saddr, const __be16 sport,
166 const struct in6_addr *daddr, const __be16 dport,
Arnaldo Carvalho de Melo5324a042005-08-12 09:26:18 -0300167 const int dif)
168{
169 struct sock *sk;
Eric Dumazet3b24d852016-04-01 08:52:17 -0700170 bool refcounted;
Arnaldo Carvalho de Melo5324a042005-08-12 09:26:18 -0300171
Craig Galleka5836362016-02-10 11:50:38 -0500172 sk = __inet6_lookup(net, hashinfo, skb, doff, saddr, sport, daddr,
Eric Dumazet3b24d852016-04-01 08:52:17 -0700173 ntohs(dport), dif, &refcounted);
174 if (sk && !refcounted && !atomic_inc_not_zero(&sk->sk_refcnt))
175 sk = NULL;
Arnaldo Carvalho de Melo5324a042005-08-12 09:26:18 -0300176 return sk;
177}
Arnaldo Carvalho de Melo5324a042005-08-12 09:26:18 -0300178EXPORT_SYMBOL_GPL(inet6_lookup);
Arnaldo Carvalho de Melod8313f52005-12-13 23:25:44 -0800179
180static int __inet6_check_established(struct inet_timewait_death_row *death_row,
181 struct sock *sk, const __u16 lport,
182 struct inet_timewait_sock **twp)
183{
184 struct inet_hashinfo *hinfo = death_row->hashinfo;
Herbert Xu3759fa92006-03-13 14:26:12 -0800185 struct inet_sock *inet = inet_sk(sk);
Eric Dumazetefe42082013-10-03 15:42:29 -0700186 const struct in6_addr *daddr = &sk->sk_v6_rcv_saddr;
187 const struct in6_addr *saddr = &sk->sk_v6_daddr;
Arnaldo Carvalho de Melod8313f52005-12-13 23:25:44 -0800188 const int dif = sk->sk_bound_dev_if;
Eric Dumazetc720c7e2009-10-15 06:30:45 +0000189 const __portpair ports = INET_COMBINED_PORTS(inet->inet_dport, lport);
Pavel Emelyanov33de0142008-06-16 17:13:48 -0700190 struct net *net = sock_net(sk);
191 const unsigned int hash = inet6_ehashfn(net, daddr, lport, saddr,
Eric Dumazetc720c7e2009-10-15 06:30:45 +0000192 inet->inet_dport);
Arnaldo Carvalho de Melod8313f52005-12-13 23:25:44 -0800193 struct inet_ehash_bucket *head = inet_ehash_bucket(hinfo, hash);
Eric Dumazet9db66bd2008-11-20 20:39:09 -0800194 spinlock_t *lock = inet_ehash_lockp(hinfo, hash);
Arnaldo Carvalho de Melod8313f52005-12-13 23:25:44 -0800195 struct sock *sk2;
Eric Dumazet3ab5aee2008-11-16 19:40:17 -0800196 const struct hlist_nulls_node *node;
Eric Dumazet05dbc7b2013-10-03 00:22:02 -0700197 struct inet_timewait_sock *tw = NULL;
Arnaldo Carvalho de Melod8313f52005-12-13 23:25:44 -0800198
Eric Dumazet9db66bd2008-11-20 20:39:09 -0800199 spin_lock(lock);
Arnaldo Carvalho de Melod8313f52005-12-13 23:25:44 -0800200
Eric Dumazet3ab5aee2008-11-16 19:40:17 -0800201 sk_nulls_for_each(sk2, node, &head->chain) {
Eric Dumazetce43b032012-11-30 09:49:27 +0000202 if (sk2->sk_hash != hash)
203 continue;
Eric Dumazet05dbc7b2013-10-03 00:22:02 -0700204
Eric Dumazetefe42082013-10-03 15:42:29 -0700205 if (likely(INET6_MATCH(sk2, net, saddr, daddr, ports, dif))) {
206 if (sk2->sk_state == TCP_TIME_WAIT) {
Eric Dumazet05dbc7b2013-10-03 00:22:02 -0700207 tw = inet_twsk(sk2);
208 if (twsk_unique(sk, sk2, twp))
Eric Dumazetefe42082013-10-03 15:42:29 -0700209 break;
Eric Dumazet05dbc7b2013-10-03 00:22:02 -0700210 }
Arnaldo Carvalho de Melod8313f52005-12-13 23:25:44 -0800211 goto not_unique;
Eric Dumazetefe42082013-10-03 15:42:29 -0700212 }
Arnaldo Carvalho de Melod8313f52005-12-13 23:25:44 -0800213 }
214
Herbert Xu3759fa92006-03-13 14:26:12 -0800215 /* Must record num and sport now. Otherwise we will see
Eric Dumazetefe42082013-10-03 15:42:29 -0700216 * in hash table socket with a funny identity.
217 */
Eric Dumazetc720c7e2009-10-15 06:30:45 +0000218 inet->inet_num = lport;
219 inet->inet_sport = htons(lport);
Eric Dumazet13475a32009-12-02 22:31:19 +0000220 sk->sk_hash = hash;
Ilpo Järvinen547b7922008-07-25 21:43:18 -0700221 WARN_ON(!sk_unhashed(sk));
Eric Dumazet3ab5aee2008-11-16 19:40:17 -0800222 __sk_nulls_add_node_rcu(sk, &head->chain);
Eric Dumazet13475a32009-12-02 22:31:19 +0000223 if (tw) {
Eric Dumazetfc01538f2015-07-08 14:28:29 -0700224 sk_nulls_del_node_init_rcu((struct sock *)tw);
Eric Dumazet02a1d6e2016-04-27 16:44:39 -0700225 __NET_INC_STATS(net, LINUX_MIB_TIMEWAITRECYCLED);
Eric Dumazet13475a32009-12-02 22:31:19 +0000226 }
Eric Dumazet9db66bd2008-11-20 20:39:09 -0800227 spin_unlock(lock);
Pavel Emelyanovc29a0bc2008-03-31 19:41:46 -0700228 sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1);
Arnaldo Carvalho de Melod8313f52005-12-13 23:25:44 -0800229
Eric Dumazet13475a32009-12-02 22:31:19 +0000230 if (twp) {
Arnaldo Carvalho de Melod8313f52005-12-13 23:25:44 -0800231 *twp = tw;
Eric Dumazet13475a32009-12-02 22:31:19 +0000232 } else if (tw) {
Arnaldo Carvalho de Melod8313f52005-12-13 23:25:44 -0800233 /* Silly. Should hash-dance instead... */
Eric Dumazetdbe7faa2015-07-08 14:28:30 -0700234 inet_twsk_deschedule_put(tw);
Arnaldo Carvalho de Melod8313f52005-12-13 23:25:44 -0800235 }
236 return 0;
237
238not_unique:
Eric Dumazet9db66bd2008-11-20 20:39:09 -0800239 spin_unlock(lock);
Arnaldo Carvalho de Melod8313f52005-12-13 23:25:44 -0800240 return -EADDRNOTAVAIL;
241}
242
Eric Dumazete2baad92015-05-27 10:46:02 -0700243static u32 inet6_sk_port_offset(const struct sock *sk)
Arnaldo Carvalho de Melod8313f52005-12-13 23:25:44 -0800244{
245 const struct inet_sock *inet = inet_sk(sk);
Eric Dumazetefe42082013-10-03 15:42:29 -0700246
247 return secure_ipv6_port_ephemeral(sk->sk_v6_rcv_saddr.s6_addr32,
248 sk->sk_v6_daddr.s6_addr32,
Eric Dumazetc720c7e2009-10-15 06:30:45 +0000249 inet->inet_dport);
Arnaldo Carvalho de Melod8313f52005-12-13 23:25:44 -0800250}
251
252int inet6_hash_connect(struct inet_timewait_death_row *death_row,
253 struct sock *sk)
254{
Eric Dumazete2baad92015-05-27 10:46:02 -0700255 u32 port_offset = 0;
256
257 if (!inet_sk(sk)->inet_num)
258 port_offset = inet6_sk_port_offset(sk);
259 return __inet_hash_connect(death_row, sk, port_offset,
Eric Dumazetb4d64442015-03-18 14:05:37 -0700260 __inet6_check_established);
Arnaldo Carvalho de Melod8313f52005-12-13 23:25:44 -0800261}
Arnaldo Carvalho de Melod8313f52005-12-13 23:25:44 -0800262EXPORT_SYMBOL_GPL(inet6_hash_connect);
Craig Gallek496611d2016-02-10 11:50:36 -0500263
264int inet6_hash(struct sock *sk)
265{
266 if (sk->sk_state != TCP_CLOSE) {
267 local_bh_disable();
Craig Gallekc125e802016-02-10 11:50:40 -0500268 __inet_hash(sk, NULL, ipv6_rcv_saddr_equal);
Craig Gallek496611d2016-02-10 11:50:36 -0500269 local_bh_enable();
270 }
271
272 return 0;
273}
274EXPORT_SYMBOL_GPL(inet6_hash);
275
276/* match_wildcard == true: IPV6_ADDR_ANY equals to any IPv6 addresses if IPv6
277 * only, and any IPv4 addresses if not IPv6 only
278 * match_wildcard == false: addresses must be exactly the same, i.e.
279 * IPV6_ADDR_ANY only equals to IPV6_ADDR_ANY,
280 * and 0.0.0.0 equals to 0.0.0.0 only
281 */
282int ipv6_rcv_saddr_equal(const struct sock *sk, const struct sock *sk2,
283 bool match_wildcard)
284{
285 const struct in6_addr *sk2_rcv_saddr6 = inet6_rcv_saddr(sk2);
286 int sk2_ipv6only = inet_v6_ipv6only(sk2);
287 int addr_type = ipv6_addr_type(&sk->sk_v6_rcv_saddr);
288 int addr_type2 = sk2_rcv_saddr6 ? ipv6_addr_type(sk2_rcv_saddr6) : IPV6_ADDR_MAPPED;
289
290 /* if both are mapped, treat as IPv4 */
291 if (addr_type == IPV6_ADDR_MAPPED && addr_type2 == IPV6_ADDR_MAPPED) {
292 if (!sk2_ipv6only) {
293 if (sk->sk_rcv_saddr == sk2->sk_rcv_saddr)
294 return 1;
295 if (!sk->sk_rcv_saddr || !sk2->sk_rcv_saddr)
296 return match_wildcard;
297 }
298 return 0;
299 }
300
301 if (addr_type == IPV6_ADDR_ANY && addr_type2 == IPV6_ADDR_ANY)
302 return 1;
303
304 if (addr_type2 == IPV6_ADDR_ANY && match_wildcard &&
305 !(sk2_ipv6only && addr_type == IPV6_ADDR_MAPPED))
306 return 1;
307
308 if (addr_type == IPV6_ADDR_ANY && match_wildcard &&
309 !(ipv6_only_sock(sk) && addr_type2 == IPV6_ADDR_MAPPED))
310 return 1;
311
312 if (sk2_rcv_saddr6 &&
313 ipv6_addr_equal(&sk->sk_v6_rcv_saddr, sk2_rcv_saddr6))
314 return 1;
315
316 return 0;
317}
318EXPORT_SYMBOL_GPL(ipv6_rcv_saddr_equal);