blob: c7f7c53353691070813eeb7ece86d1826f408fd5 [file] [log] [blame]
Arnaldo Carvalho de Melo3f421ba2005-08-09 20:11:08 -07001/*
2 * INET An implementation of the TCP/IP protocol suite for the LINUX
3 * operating system. INET is implemented using the BSD Socket
4 * interface as the means of communication with the user level.
5 *
6 * Support for INET connection oriented protocols.
7 *
8 * Authors: See the TCP sources
9 *
10 * This program is free software; you can redistribute it and/or
11 * modify it under the terms of the GNU General Public License
12 * as published by the Free Software Foundation; either version
13 * 2 of the License, or(at your option) any later version.
14 */
15
Arnaldo Carvalho de Melo3f421ba2005-08-09 20:11:08 -070016#include <linux/module.h>
17#include <linux/jhash.h>
18
19#include <net/inet_connection_sock.h>
20#include <net/inet_hashtables.h>
21#include <net/inet_timewait_sock.h>
22#include <net/ip.h>
23#include <net/route.h>
24#include <net/tcp_states.h>
Arnaldo Carvalho de Meloa019d6f2005-08-09 20:15:09 -070025#include <net/xfrm.h>
Eric Dumazetfa76ce732015-03-19 19:04:20 -070026#include <net/tcp.h>
Craig Gallekc125e802016-02-10 11:50:40 -050027#include <net/sock_reuseport.h>
Arnaldo Carvalho de Melo3f421ba2005-08-09 20:11:08 -070028
29#ifdef INET_CSK_DEBUG
30const char inet_csk_timer_bug_msg[] = "inet_csk BUG: unknown timer value\n";
31EXPORT_SYMBOL(inet_csk_timer_bug_msg);
32#endif
33
Josef Bacikfe38d2a2017-01-17 07:51:01 -080034#if IS_ENABLED(CONFIG_IPV6)
35/* match_wildcard == true: IPV6_ADDR_ANY equals to any IPv6 addresses if IPv6
36 * only, and any IPv4 addresses if not IPv6 only
37 * match_wildcard == false: addresses must be exactly the same, i.e.
38 * IPV6_ADDR_ANY only equals to IPV6_ADDR_ANY,
39 * and 0.0.0.0 equals to 0.0.0.0 only
40 */
Josef Bacik637bc8b2017-01-17 07:51:06 -080041static int ipv6_rcv_saddr_equal(const struct in6_addr *sk1_rcv_saddr6,
42 const struct in6_addr *sk2_rcv_saddr6,
43 __be32 sk1_rcv_saddr, __be32 sk2_rcv_saddr,
44 bool sk1_ipv6only, bool sk2_ipv6only,
Josef Bacikfe38d2a2017-01-17 07:51:01 -080045 bool match_wildcard)
46{
Josef Bacik637bc8b2017-01-17 07:51:06 -080047 int addr_type = ipv6_addr_type(sk1_rcv_saddr6);
Josef Bacikfe38d2a2017-01-17 07:51:01 -080048 int addr_type2 = sk2_rcv_saddr6 ? ipv6_addr_type(sk2_rcv_saddr6) : IPV6_ADDR_MAPPED;
49
50 /* if both are mapped, treat as IPv4 */
51 if (addr_type == IPV6_ADDR_MAPPED && addr_type2 == IPV6_ADDR_MAPPED) {
52 if (!sk2_ipv6only) {
Josef Bacik637bc8b2017-01-17 07:51:06 -080053 if (sk1_rcv_saddr == sk2_rcv_saddr)
Josef Bacikfe38d2a2017-01-17 07:51:01 -080054 return 1;
Josef Bacik637bc8b2017-01-17 07:51:06 -080055 if (!sk1_rcv_saddr || !sk2_rcv_saddr)
Josef Bacikfe38d2a2017-01-17 07:51:01 -080056 return match_wildcard;
57 }
58 return 0;
59 }
60
61 if (addr_type == IPV6_ADDR_ANY && addr_type2 == IPV6_ADDR_ANY)
62 return 1;
63
64 if (addr_type2 == IPV6_ADDR_ANY && match_wildcard &&
65 !(sk2_ipv6only && addr_type == IPV6_ADDR_MAPPED))
66 return 1;
67
68 if (addr_type == IPV6_ADDR_ANY && match_wildcard &&
Josef Bacik637bc8b2017-01-17 07:51:06 -080069 !(sk1_ipv6only && addr_type2 == IPV6_ADDR_MAPPED))
Josef Bacikfe38d2a2017-01-17 07:51:01 -080070 return 1;
71
72 if (sk2_rcv_saddr6 &&
Josef Bacik637bc8b2017-01-17 07:51:06 -080073 ipv6_addr_equal(sk1_rcv_saddr6, sk2_rcv_saddr6))
Josef Bacikfe38d2a2017-01-17 07:51:01 -080074 return 1;
75
76 return 0;
77}
78#endif
79
80/* match_wildcard == true: 0.0.0.0 equals to any IPv4 addresses
81 * match_wildcard == false: addresses must be exactly the same, i.e.
82 * 0.0.0.0 only equals to 0.0.0.0
83 */
Josef Bacik637bc8b2017-01-17 07:51:06 -080084static int ipv4_rcv_saddr_equal(__be32 sk1_rcv_saddr, __be32 sk2_rcv_saddr,
85 bool sk2_ipv6only, bool match_wildcard)
Josef Bacikfe38d2a2017-01-17 07:51:01 -080086{
Josef Bacik637bc8b2017-01-17 07:51:06 -080087 if (!sk2_ipv6only) {
88 if (sk1_rcv_saddr == sk2_rcv_saddr)
Josef Bacikfe38d2a2017-01-17 07:51:01 -080089 return 1;
Josef Bacik637bc8b2017-01-17 07:51:06 -080090 if (!sk1_rcv_saddr || !sk2_rcv_saddr)
Josef Bacikfe38d2a2017-01-17 07:51:01 -080091 return match_wildcard;
92 }
93 return 0;
94}
95
96int inet_rcv_saddr_equal(const struct sock *sk, const struct sock *sk2,
97 bool match_wildcard)
98{
99#if IS_ENABLED(CONFIG_IPV6)
100 if (sk->sk_family == AF_INET6)
Josef Bacik637bc8b2017-01-17 07:51:06 -0800101 return ipv6_rcv_saddr_equal(&sk->sk_v6_rcv_saddr,
102 &sk2->sk_v6_rcv_saddr,
103 sk->sk_rcv_saddr,
104 sk2->sk_rcv_saddr,
105 ipv6_only_sock(sk),
106 ipv6_only_sock(sk2),
107 match_wildcard);
Josef Bacikfe38d2a2017-01-17 07:51:01 -0800108#endif
Josef Bacik637bc8b2017-01-17 07:51:06 -0800109 return ipv4_rcv_saddr_equal(sk->sk_rcv_saddr, sk2->sk_rcv_saddr,
110 ipv6_only_sock(sk2), match_wildcard);
Josef Bacikfe38d2a2017-01-17 07:51:01 -0800111}
112EXPORT_SYMBOL(inet_rcv_saddr_equal);
113
Eric W. Biederman0bbf87d2013-09-28 14:10:59 -0700114void inet_get_local_port_range(struct net *net, int *low, int *high)
Stephen Hemminger227b60f2007-10-10 17:30:46 -0700115{
Eric Dumazet95c96172012-04-15 05:58:06 +0000116 unsigned int seq;
117
Stephen Hemminger227b60f2007-10-10 17:30:46 -0700118 do {
Cong Wangc9d8f1a2014-05-06 11:02:49 -0700119 seq = read_seqbegin(&net->ipv4.ip_local_ports.lock);
Stephen Hemminger227b60f2007-10-10 17:30:46 -0700120
Cong Wangc9d8f1a2014-05-06 11:02:49 -0700121 *low = net->ipv4.ip_local_ports.range[0];
122 *high = net->ipv4.ip_local_ports.range[1];
123 } while (read_seqretry(&net->ipv4.ip_local_ports.lock, seq));
Stephen Hemminger227b60f2007-10-10 17:30:46 -0700124}
125EXPORT_SYMBOL(inet_get_local_port_range);
Arnaldo Carvalho de Melo3f421ba2005-08-09 20:11:08 -0700126
Josef Bacikaa078842017-01-17 07:51:02 -0800127static int inet_csk_bind_conflict(const struct sock *sk,
128 const struct inet_bind_bucket *tb,
129 bool relax, bool reuseport_ok)
Arnaldo Carvalho de Melo3f421ba2005-08-09 20:11:08 -0700130{
Arnaldo Carvalho de Melo3f421ba2005-08-09 20:11:08 -0700131 struct sock *sk2;
Tom Herbert0643ee42016-12-14 16:54:16 -0800132 bool reuse = sk->sk_reuse;
133 bool reuseport = !!sk->sk_reuseport && reuseport_ok;
Tom Herbertda5e3632013-01-22 09:50:24 +0000134 kuid_t uid = sock_i_uid((struct sock *)sk);
Arnaldo Carvalho de Melo3f421ba2005-08-09 20:11:08 -0700135
Pavel Emelyanov7477fd2e2008-04-14 02:42:27 -0700136 /*
137 * Unlike other sk lookup places we do not check
138 * for sk_net here, since _all_ the socks listed
139 * in tb->owners list belong to the same net - the
140 * one this bucket belongs to.
141 */
142
Sasha Levinb67bfe02013-02-27 17:06:00 -0800143 sk_for_each_bound(sk2, &tb->owners) {
Arnaldo Carvalho de Melo3f421ba2005-08-09 20:11:08 -0700144 if (sk != sk2 &&
Arnaldo Carvalho de Melo3f421ba2005-08-09 20:11:08 -0700145 (!sk->sk_bound_dev_if ||
146 !sk2->sk_bound_dev_if ||
147 sk->sk_bound_dev_if == sk2->sk_bound_dev_if)) {
Tom Herbertda5e3632013-01-22 09:50:24 +0000148 if ((!reuse || !sk2->sk_reuse ||
149 sk2->sk_state == TCP_LISTEN) &&
150 (!reuseport || !sk2->sk_reuseport ||
Craig Gallekc125e802016-02-10 11:50:40 -0500151 rcu_access_pointer(sk->sk_reuseport_cb) ||
152 (sk2->sk_state != TCP_TIME_WAIT &&
Tom Herbertda5e3632013-01-22 09:50:24 +0000153 !uid_eq(uid, sock_i_uid(sk2))))) {
Josef Bacikaa078842017-01-17 07:51:02 -0800154 if (inet_rcv_saddr_equal(sk, sk2, true))
Arnaldo Carvalho de Melo3f421ba2005-08-09 20:11:08 -0700155 break;
David S. Miller8d238b22010-04-28 11:25:59 -0700156 }
Alex Copotaacd9282012-04-12 22:21:45 +0000157 if (!relax && reuse && sk2->sk_reuse &&
158 sk2->sk_state != TCP_LISTEN) {
Josef Bacikaa078842017-01-17 07:51:02 -0800159 if (inet_rcv_saddr_equal(sk, sk2, true))
Alex Copotaacd9282012-04-12 22:21:45 +0000160 break;
161 }
Arnaldo Carvalho de Melo3f421ba2005-08-09 20:11:08 -0700162 }
163 }
Sasha Levinb67bfe02013-02-27 17:06:00 -0800164 return sk2 != NULL;
Arnaldo Carvalho de Melo3f421ba2005-08-09 20:11:08 -0700165}
Arnaldo Carvalho de Melo971af182005-12-13 23:14:47 -0800166
Josef Bacik289141b2017-01-17 07:51:05 -0800167/*
168 * Find an open port number for the socket. Returns with the
169 * inet_bind_hashbucket lock held.
Arnaldo Carvalho de Melo3f421ba2005-08-09 20:11:08 -0700170 */
Josef Bacik289141b2017-01-17 07:51:05 -0800171static struct inet_bind_hashbucket *
172inet_csk_find_open_port(struct sock *sk, struct inet_bind_bucket **tb_ret, int *port_ret)
Arnaldo Carvalho de Melo3f421ba2005-08-09 20:11:08 -0700173{
Eric Dumazetea8add22016-02-11 16:28:50 -0800174 struct inet_hashinfo *hinfo = sk->sk_prot->h.hashinfo;
Josef Bacik289141b2017-01-17 07:51:05 -0800175 int port = 0;
Arnaldo Carvalho de Melo3f421ba2005-08-09 20:11:08 -0700176 struct inet_bind_hashbucket *head;
YOSHIFUJI Hideaki3b1e0a62008-03-26 02:26:21 +0900177 struct net *net = sock_net(sk);
Eric Dumazetea8add22016-02-11 16:28:50 -0800178 int i, low, high, attempt_half;
179 struct inet_bind_bucket *tb;
Eric Dumazetea8add22016-02-11 16:28:50 -0800180 u32 remaining, offset;
Arnaldo Carvalho de Melo3f421ba2005-08-09 20:11:08 -0700181
Eric Dumazetea8add22016-02-11 16:28:50 -0800182 attempt_half = (sk->sk_reuse == SK_CAN_REUSE) ? 1 : 0;
183other_half_scan:
184 inet_get_local_port_range(net, &low, &high);
185 high++; /* [32768, 60999] -> [32768, 61000[ */
186 if (high - low < 4)
187 attempt_half = 0;
188 if (attempt_half) {
189 int half = low + (((high - low) >> 2) << 1);
190
191 if (attempt_half == 1)
192 high = half;
193 else
194 low = half;
195 }
196 remaining = high - low;
197 if (likely(remaining > 1))
198 remaining &= ~1U;
199
200 offset = prandom_u32() % remaining;
201 /* __inet_hash_connect() favors ports having @low parity
202 * We do the opposite to not pollute connect() users.
203 */
204 offset |= 1U;
Eric Dumazetea8add22016-02-11 16:28:50 -0800205
206other_parity_scan:
207 port = low + offset;
208 for (i = 0; i < remaining; i += 2, port += 2) {
209 if (unlikely(port >= high))
210 port -= remaining;
211 if (inet_is_local_reserved_port(net, port))
212 continue;
213 head = &hinfo->bhash[inet_bhashfn(net, port,
214 hinfo->bhash_size)];
215 spin_lock_bh(&head->lock);
216 inet_bind_bucket_for_each(tb, &head->chain)
217 if (net_eq(ib_net(tb), net) && tb->port == port) {
Josef Bacik289141b2017-01-17 07:51:05 -0800218 if (!inet_csk_bind_conflict(sk, tb, false, false))
Josef Bacik6cd66612017-01-17 07:51:04 -0800219 goto success;
Eric Dumazetea8add22016-02-11 16:28:50 -0800220 goto next_port;
221 }
Josef Bacik289141b2017-01-17 07:51:05 -0800222 tb = NULL;
223 goto success;
Eric Dumazetea8add22016-02-11 16:28:50 -0800224next_port:
225 spin_unlock_bh(&head->lock);
226 cond_resched();
227 }
228
Eric Dumazetea8add22016-02-11 16:28:50 -0800229 offset--;
230 if (!(offset & 1))
231 goto other_parity_scan;
232
233 if (attempt_half == 1) {
234 /* OK we now try the upper half of the range */
235 attempt_half = 2;
236 goto other_half_scan;
237 }
Josef Bacik289141b2017-01-17 07:51:05 -0800238 return NULL;
239success:
240 *port_ret = port;
241 *tb_ret = tb;
242 return head;
243}
Eric Dumazetea8add22016-02-11 16:28:50 -0800244
Josef Bacik637bc8b2017-01-17 07:51:06 -0800245static inline int sk_reuseport_match(struct inet_bind_bucket *tb,
246 struct sock *sk)
247{
248 kuid_t uid = sock_i_uid(sk);
249
250 if (tb->fastreuseport <= 0)
251 return 0;
252 if (!sk->sk_reuseport)
253 return 0;
254 if (rcu_access_pointer(sk->sk_reuseport_cb))
255 return 0;
256 if (!uid_eq(tb->fastuid, uid))
257 return 0;
258 /* We only need to check the rcv_saddr if this tb was once marked
259 * without fastreuseport and then was reset, as we can only know that
260 * the fast_*rcv_saddr doesn't have any conflicts with the socks on the
261 * owners list.
262 */
263 if (tb->fastreuseport == FASTREUSEPORT_ANY)
264 return 1;
265#if IS_ENABLED(CONFIG_IPV6)
266 if (tb->fast_sk_family == AF_INET6)
267 return ipv6_rcv_saddr_equal(&tb->fast_v6_rcv_saddr,
268 &sk->sk_v6_rcv_saddr,
269 tb->fast_rcv_saddr,
270 sk->sk_rcv_saddr,
271 tb->fast_ipv6_only,
272 ipv6_only_sock(sk), true);
273#endif
274 return ipv4_rcv_saddr_equal(tb->fast_rcv_saddr, sk->sk_rcv_saddr,
275 ipv6_only_sock(sk), true);
276}
277
Josef Bacik289141b2017-01-17 07:51:05 -0800278/* Obtain a reference to a local port for the given sock,
279 * if snum is zero it means select any available local port.
280 * We try to allocate an odd port (and leave even ports for connect())
281 */
282int inet_csk_get_port(struct sock *sk, unsigned short snum)
283{
284 bool reuse = sk->sk_reuse && sk->sk_state != TCP_LISTEN;
285 struct inet_hashinfo *hinfo = sk->sk_prot->h.hashinfo;
286 int ret = 1, port = snum;
287 struct inet_bind_hashbucket *head;
288 struct net *net = sock_net(sk);
289 struct inet_bind_bucket *tb = NULL;
290 kuid_t uid = sock_i_uid(sk);
291
292 if (!port) {
293 head = inet_csk_find_open_port(sk, &tb, &port);
294 if (!head)
295 return ret;
296 if (!tb)
297 goto tb_not_found;
298 goto success;
299 }
300 head = &hinfo->bhash[inet_bhashfn(net, port,
301 hinfo->bhash_size)];
302 spin_lock_bh(&head->lock);
303 inet_bind_bucket_for_each(tb, &head->chain)
304 if (net_eq(ib_net(tb), net) && tb->port == port)
305 goto tb_found;
Eric Dumazetea8add22016-02-11 16:28:50 -0800306tb_not_found:
307 tb = inet_bind_bucket_create(hinfo->bind_bucket_cachep,
308 net, head, port);
309 if (!tb)
310 goto fail_unlock;
Arnaldo Carvalho de Melo3f421ba2005-08-09 20:11:08 -0700311tb_found:
312 if (!hlist_empty(&tb->owners)) {
Pavel Emelyanov4a17fd52012-04-19 03:39:36 +0000313 if (sk->sk_reuse == SK_FORCE_REUSE)
314 goto success;
315
Josef Bacikb9470c22017-01-17 07:51:03 -0800316 if ((tb->fastreuse > 0 && reuse) ||
Josef Bacik637bc8b2017-01-17 07:51:06 -0800317 sk_reuseport_match(tb, sk))
Arnaldo Carvalho de Melo3f421ba2005-08-09 20:11:08 -0700318 goto success;
Josef Bacik289141b2017-01-17 07:51:05 -0800319 if (inet_csk_bind_conflict(sk, tb, true, true))
Eric Dumazetea8add22016-02-11 16:28:50 -0800320 goto fail_unlock;
Josef Bacik6cd66612017-01-17 07:51:04 -0800321 }
322success:
323 if (!hlist_empty(&tb->owners)) {
Eric Dumazetea8add22016-02-11 16:28:50 -0800324 tb->fastreuse = reuse;
Tom Herbertda5e3632013-01-22 09:50:24 +0000325 if (sk->sk_reuseport) {
Josef Bacik637bc8b2017-01-17 07:51:06 -0800326 tb->fastreuseport = FASTREUSEPORT_ANY;
Tom Herbertda5e3632013-01-22 09:50:24 +0000327 tb->fastuid = uid;
Josef Bacik637bc8b2017-01-17 07:51:06 -0800328 tb->fast_rcv_saddr = sk->sk_rcv_saddr;
329 tb->fast_ipv6_only = ipv6_only_sock(sk);
330#if IS_ENABLED(CONFIG_IPV6)
331 tb->fast_v6_rcv_saddr = sk->sk_v6_rcv_saddr;
332#endif
Eric Dumazetea8add22016-02-11 16:28:50 -0800333 } else {
Tom Herbertda5e3632013-01-22 09:50:24 +0000334 tb->fastreuseport = 0;
Eric Dumazetea8add22016-02-11 16:28:50 -0800335 }
Josef Bacik6cd66612017-01-17 07:51:04 -0800336 } else {
337 if (!reuse)
338 tb->fastreuse = 0;
Josef Bacik637bc8b2017-01-17 07:51:06 -0800339 if (sk->sk_reuseport) {
340 /* We didn't match or we don't have fastreuseport set on
341 * the tb, but we have sk_reuseport set on this socket
342 * and we know that there are no bind conflicts with
343 * this socket in this tb, so reset our tb's reuseport
344 * settings so that any subsequent sockets that match
345 * our current socket will be put on the fast path.
346 *
347 * If we reset we need to set FASTREUSEPORT_STRICT so we
348 * do extra checking for all subsequent sk_reuseport
349 * socks.
350 */
351 if (!sk_reuseport_match(tb, sk)) {
352 tb->fastreuseport = FASTREUSEPORT_STRICT;
353 tb->fastuid = uid;
354 tb->fast_rcv_saddr = sk->sk_rcv_saddr;
355 tb->fast_ipv6_only = ipv6_only_sock(sk);
356#if IS_ENABLED(CONFIG_IPV6)
357 tb->fast_v6_rcv_saddr = sk->sk_v6_rcv_saddr;
358#endif
359 }
360 } else {
Josef Bacik6cd66612017-01-17 07:51:04 -0800361 tb->fastreuseport = 0;
Josef Bacik637bc8b2017-01-17 07:51:06 -0800362 }
Tom Herbertda5e3632013-01-22 09:50:24 +0000363 }
Arnaldo Carvalho de Melo3f421ba2005-08-09 20:11:08 -0700364 if (!inet_csk(sk)->icsk_bind_hash)
Eric Dumazetea8add22016-02-11 16:28:50 -0800365 inet_bind_hash(sk, tb, port);
Ilpo Järvinen547b7922008-07-25 21:43:18 -0700366 WARN_ON(inet_csk(sk)->icsk_bind_hash != tb);
YOSHIFUJI Hideakie905a9e2007-02-09 23:24:47 +0900367 ret = 0;
Arnaldo Carvalho de Melo3f421ba2005-08-09 20:11:08 -0700368
369fail_unlock:
Eric Dumazetea8add22016-02-11 16:28:50 -0800370 spin_unlock_bh(&head->lock);
Arnaldo Carvalho de Melo3f421ba2005-08-09 20:11:08 -0700371 return ret;
372}
Arnaldo Carvalho de Melo3f421ba2005-08-09 20:11:08 -0700373EXPORT_SYMBOL_GPL(inet_csk_get_port);
374
375/*
376 * Wait for an incoming connection, avoid race conditions. This must be called
377 * with the socket locked.
378 */
379static int inet_csk_wait_for_connect(struct sock *sk, long timeo)
380{
381 struct inet_connection_sock *icsk = inet_csk(sk);
382 DEFINE_WAIT(wait);
383 int err;
384
385 /*
386 * True wake-one mechanism for incoming connections: only
387 * one process gets woken up, not the 'whole herd'.
388 * Since we do not 'race & poll' for established sockets
389 * anymore, the common case will execute the loop only once.
390 *
391 * Subtle issue: "add_wait_queue_exclusive()" will be added
392 * after any current non-exclusive waiters, and we know that
393 * it will always _stay_ after any new non-exclusive waiters
394 * because all non-exclusive waiters are added at the
395 * beginning of the wait-queue. As such, it's ok to "drop"
396 * our exclusiveness temporarily when we get woken up without
397 * having to remove and re-insert us on the wait queue.
398 */
399 for (;;) {
Eric Dumazetaa395142010-04-20 13:03:51 +0000400 prepare_to_wait_exclusive(sk_sleep(sk), &wait,
Arnaldo Carvalho de Melo3f421ba2005-08-09 20:11:08 -0700401 TASK_INTERRUPTIBLE);
402 release_sock(sk);
403 if (reqsk_queue_empty(&icsk->icsk_accept_queue))
404 timeo = schedule_timeout(timeo);
Eric Dumazetcb7cf8a2015-03-16 12:19:24 -0700405 sched_annotate_sleep();
Arnaldo Carvalho de Melo3f421ba2005-08-09 20:11:08 -0700406 lock_sock(sk);
407 err = 0;
408 if (!reqsk_queue_empty(&icsk->icsk_accept_queue))
409 break;
410 err = -EINVAL;
411 if (sk->sk_state != TCP_LISTEN)
412 break;
413 err = sock_intr_errno(timeo);
414 if (signal_pending(current))
415 break;
416 err = -EAGAIN;
417 if (!timeo)
418 break;
419 }
Eric Dumazetaa395142010-04-20 13:03:51 +0000420 finish_wait(sk_sleep(sk), &wait);
Arnaldo Carvalho de Melo3f421ba2005-08-09 20:11:08 -0700421 return err;
422}
423
424/*
425 * This will accept the next outstanding connection.
426 */
427struct sock *inet_csk_accept(struct sock *sk, int flags, int *err)
428{
429 struct inet_connection_sock *icsk = inet_csk(sk);
Jerry Chu83368862012-08-31 12:29:12 +0000430 struct request_sock_queue *queue = &icsk->icsk_accept_queue;
Jerry Chu83368862012-08-31 12:29:12 +0000431 struct request_sock *req;
Eric Dumazete3d95ad2015-03-17 18:32:30 -0700432 struct sock *newsk;
Arnaldo Carvalho de Melo3f421ba2005-08-09 20:11:08 -0700433 int error;
434
435 lock_sock(sk);
436
437 /* We need to make sure that this socket is listening,
438 * and that it has something pending.
439 */
440 error = -EINVAL;
441 if (sk->sk_state != TCP_LISTEN)
442 goto out_err;
443
444 /* Find already established connection */
Jerry Chu83368862012-08-31 12:29:12 +0000445 if (reqsk_queue_empty(queue)) {
Arnaldo Carvalho de Melo3f421ba2005-08-09 20:11:08 -0700446 long timeo = sock_rcvtimeo(sk, flags & O_NONBLOCK);
447
448 /* If this is a non blocking socket don't sleep */
449 error = -EAGAIN;
450 if (!timeo)
451 goto out_err;
452
453 error = inet_csk_wait_for_connect(sk, timeo);
454 if (error)
455 goto out_err;
456 }
Eric Dumazetfff1f302015-10-02 11:43:23 -0700457 req = reqsk_queue_remove(queue, sk);
Jerry Chu83368862012-08-31 12:29:12 +0000458 newsk = req->sk;
Arnaldo Carvalho de Melo3f421ba2005-08-09 20:11:08 -0700459
Eric Dumazete3d95ad2015-03-17 18:32:30 -0700460 if (sk->sk_protocol == IPPROTO_TCP &&
Eric Dumazet0536fcc2015-09-29 07:42:52 -0700461 tcp_rsk(req)->tfo_listener) {
462 spin_lock_bh(&queue->fastopenq.lock);
Eric Dumazet9439ce02015-03-17 18:32:29 -0700463 if (tcp_rsk(req)->tfo_listener) {
Jerry Chu83368862012-08-31 12:29:12 +0000464 /* We are still waiting for the final ACK from 3WHS
465 * so can't free req now. Instead, we set req->sk to
466 * NULL to signify that the child socket is taken
467 * so reqsk_fastopen_remove() will free the req
468 * when 3WHS finishes (or is aborted).
469 */
470 req->sk = NULL;
471 req = NULL;
472 }
Eric Dumazet0536fcc2015-09-29 07:42:52 -0700473 spin_unlock_bh(&queue->fastopenq.lock);
Jerry Chu83368862012-08-31 12:29:12 +0000474 }
Arnaldo Carvalho de Melo3f421ba2005-08-09 20:11:08 -0700475out:
476 release_sock(sk);
Jerry Chu83368862012-08-31 12:29:12 +0000477 if (req)
Eric Dumazet13854e52015-03-15 21:12:16 -0700478 reqsk_put(req);
Arnaldo Carvalho de Melo3f421ba2005-08-09 20:11:08 -0700479 return newsk;
480out_err:
481 newsk = NULL;
Jerry Chu83368862012-08-31 12:29:12 +0000482 req = NULL;
Arnaldo Carvalho de Melo3f421ba2005-08-09 20:11:08 -0700483 *err = error;
484 goto out;
485}
Arnaldo Carvalho de Melo3f421ba2005-08-09 20:11:08 -0700486EXPORT_SYMBOL(inet_csk_accept);
487
488/*
489 * Using different timers for retransmit, delayed acks and probes
YOSHIFUJI Hideakie905a9e2007-02-09 23:24:47 +0900490 * We may wish use just one timer maintaining a list of expire jiffies
Arnaldo Carvalho de Melo3f421ba2005-08-09 20:11:08 -0700491 * to optimize.
492 */
493void inet_csk_init_xmit_timers(struct sock *sk,
494 void (*retransmit_handler)(unsigned long),
495 void (*delack_handler)(unsigned long),
496 void (*keepalive_handler)(unsigned long))
497{
498 struct inet_connection_sock *icsk = inet_csk(sk);
499
Pavel Emelyanovb24b8a22008-01-23 21:20:07 -0800500 setup_timer(&icsk->icsk_retransmit_timer, retransmit_handler,
501 (unsigned long)sk);
502 setup_timer(&icsk->icsk_delack_timer, delack_handler,
503 (unsigned long)sk);
504 setup_timer(&sk->sk_timer, keepalive_handler, (unsigned long)sk);
Arnaldo Carvalho de Melo3f421ba2005-08-09 20:11:08 -0700505 icsk->icsk_pending = icsk->icsk_ack.pending = 0;
506}
Arnaldo Carvalho de Melo3f421ba2005-08-09 20:11:08 -0700507EXPORT_SYMBOL(inet_csk_init_xmit_timers);
508
509void inet_csk_clear_xmit_timers(struct sock *sk)
510{
511 struct inet_connection_sock *icsk = inet_csk(sk);
512
513 icsk->icsk_pending = icsk->icsk_ack.pending = icsk->icsk_ack.blocked = 0;
514
515 sk_stop_timer(sk, &icsk->icsk_retransmit_timer);
516 sk_stop_timer(sk, &icsk->icsk_delack_timer);
517 sk_stop_timer(sk, &sk->sk_timer);
518}
Arnaldo Carvalho de Melo3f421ba2005-08-09 20:11:08 -0700519EXPORT_SYMBOL(inet_csk_clear_xmit_timers);
520
521void inet_csk_delete_keepalive_timer(struct sock *sk)
522{
523 sk_stop_timer(sk, &sk->sk_timer);
524}
Arnaldo Carvalho de Melo3f421ba2005-08-09 20:11:08 -0700525EXPORT_SYMBOL(inet_csk_delete_keepalive_timer);
526
527void inet_csk_reset_keepalive_timer(struct sock *sk, unsigned long len)
528{
529 sk_reset_timer(sk, &sk->sk_timer, jiffies + len);
530}
Arnaldo Carvalho de Melo3f421ba2005-08-09 20:11:08 -0700531EXPORT_SYMBOL(inet_csk_reset_keepalive_timer);
532
Eric Dumazete5895bc2015-09-25 07:39:11 -0700533struct dst_entry *inet_csk_route_req(const struct sock *sk,
David S. Miller6bd023f2011-05-18 18:32:03 -0400534 struct flowi4 *fl4,
David S. Millerba3f7f02012-07-17 14:02:46 -0700535 const struct request_sock *req)
Arnaldo Carvalho de Melo3f421ba2005-08-09 20:11:08 -0700536{
Arnaldo Carvalho de Melo3f421ba2005-08-09 20:11:08 -0700537 const struct inet_request_sock *ireq = inet_rsk(req);
Eric Dumazet8b929ab2015-03-22 10:22:20 -0700538 struct net *net = read_pnet(&ireq->ireq_net);
539 struct ip_options_rcu *opt = ireq->opt;
540 struct rtable *rt;
Arnaldo Carvalho de Melo3f421ba2005-08-09 20:11:08 -0700541
Eric Dumazet8b929ab2015-03-22 10:22:20 -0700542 flowi4_init_output(fl4, ireq->ir_iif, ireq->ir_mark,
David S. Millere79d9bc2011-03-31 04:53:20 -0700543 RT_CONN_FLAGS(sk), RT_SCOPE_UNIVERSE,
Eric Dumazet8b929ab2015-03-22 10:22:20 -0700544 sk->sk_protocol, inet_sk_flowi_flags(sk),
Eric Dumazet634fb9792013-10-09 15:21:29 -0700545 (opt && opt->opt.srr) ? opt->opt.faddr : ireq->ir_rmt_addr,
Eric Dumazet8b929ab2015-03-22 10:22:20 -0700546 ireq->ir_loc_addr, ireq->ir_rmt_port,
Lorenzo Colittie2d118a2016-11-04 02:23:43 +0900547 htons(ireq->ir_num), sk->sk_uid);
David S. Miller6bd023f2011-05-18 18:32:03 -0400548 security_req_classify_flow(req, flowi4_to_flowi(fl4));
549 rt = ip_route_output_flow(net, fl4, sk);
David S. Millerb23dd4f2011-03-02 14:31:35 -0800550 if (IS_ERR(rt))
Ilpo Järvinen857a6e02008-12-14 23:13:08 -0800551 goto no_route;
Julian Anastasov155e8332012-10-08 11:41:18 +0000552 if (opt && opt->opt.is_strictroute && rt->rt_uses_gateway)
Ilpo Järvinen857a6e02008-12-14 23:13:08 -0800553 goto route_err;
Changli Gaod8d1f302010-06-10 23:31:35 -0700554 return &rt->dst;
Ilpo Järvinen857a6e02008-12-14 23:13:08 -0800555
556route_err:
557 ip_rt_put(rt);
558no_route:
Eric Dumazetb45386e2016-04-27 16:44:35 -0700559 __IP_INC_STATS(net, IPSTATS_MIB_OUTNOROUTES);
Ilpo Järvinen857a6e02008-12-14 23:13:08 -0800560 return NULL;
Arnaldo Carvalho de Melo3f421ba2005-08-09 20:11:08 -0700561}
Arnaldo Carvalho de Melo3f421ba2005-08-09 20:11:08 -0700562EXPORT_SYMBOL_GPL(inet_csk_route_req);
563
Eric Dumazeta2432c42015-09-29 07:42:43 -0700564struct dst_entry *inet_csk_route_child_sock(const struct sock *sk,
David S. Miller77357a92011-05-08 14:34:22 -0700565 struct sock *newsk,
566 const struct request_sock *req)
567{
568 const struct inet_request_sock *ireq = inet_rsk(req);
Eric Dumazet8b929ab2015-03-22 10:22:20 -0700569 struct net *net = read_pnet(&ireq->ireq_net);
David S. Miller77357a92011-05-08 14:34:22 -0700570 struct inet_sock *newinet = inet_sk(newsk);
Christoph Paasch1a7b27c2012-08-20 02:52:09 +0000571 struct ip_options_rcu *opt;
David S. Miller77357a92011-05-08 14:34:22 -0700572 struct flowi4 *fl4;
573 struct rtable *rt;
574
575 fl4 = &newinet->cork.fl.u.ip4;
Christoph Paasch1a7b27c2012-08-20 02:52:09 +0000576
577 rcu_read_lock();
578 opt = rcu_dereference(newinet->inet_opt);
Eric Dumazet8b929ab2015-03-22 10:22:20 -0700579 flowi4_init_output(fl4, ireq->ir_iif, ireq->ir_mark,
David S. Miller77357a92011-05-08 14:34:22 -0700580 RT_CONN_FLAGS(sk), RT_SCOPE_UNIVERSE,
581 sk->sk_protocol, inet_sk_flowi_flags(sk),
Eric Dumazet634fb9792013-10-09 15:21:29 -0700582 (opt && opt->opt.srr) ? opt->opt.faddr : ireq->ir_rmt_addr,
Eric Dumazet8b929ab2015-03-22 10:22:20 -0700583 ireq->ir_loc_addr, ireq->ir_rmt_port,
Lorenzo Colittie2d118a2016-11-04 02:23:43 +0900584 htons(ireq->ir_num), sk->sk_uid);
David S. Miller77357a92011-05-08 14:34:22 -0700585 security_req_classify_flow(req, flowi4_to_flowi(fl4));
586 rt = ip_route_output_flow(net, fl4, sk);
587 if (IS_ERR(rt))
588 goto no_route;
Julian Anastasov155e8332012-10-08 11:41:18 +0000589 if (opt && opt->opt.is_strictroute && rt->rt_uses_gateway)
David S. Miller77357a92011-05-08 14:34:22 -0700590 goto route_err;
Christoph Paasch1a7b27c2012-08-20 02:52:09 +0000591 rcu_read_unlock();
David S. Miller77357a92011-05-08 14:34:22 -0700592 return &rt->dst;
593
594route_err:
595 ip_rt_put(rt);
596no_route:
Christoph Paasch1a7b27c2012-08-20 02:52:09 +0000597 rcu_read_unlock();
Eric Dumazetb45386e2016-04-27 16:44:35 -0700598 __IP_INC_STATS(net, IPSTATS_MIB_OUTNOROUTES);
David S. Miller77357a92011-05-08 14:34:22 -0700599 return NULL;
600}
601EXPORT_SYMBOL_GPL(inet_csk_route_child_sock);
602
Eric Dumazetdfd56b82011-12-10 09:48:31 +0000603#if IS_ENABLED(CONFIG_IPV6)
Arnaldo Carvalho de Melo3f421ba2005-08-09 20:11:08 -0700604#define AF_INET_FAMILY(fam) ((fam) == AF_INET)
605#else
Eric Dumazetfa76ce732015-03-19 19:04:20 -0700606#define AF_INET_FAMILY(fam) true
Arnaldo Carvalho de Melo3f421ba2005-08-09 20:11:08 -0700607#endif
608
Julian Anastasov0c3d79b2009-10-19 10:03:58 +0000609/* Decide when to expire the request and when to resend SYN-ACK */
610static inline void syn_ack_recalc(struct request_sock *req, const int thresh,
611 const int max_retries,
612 const u8 rskq_defer_accept,
613 int *expire, int *resend)
614{
615 if (!rskq_defer_accept) {
Eric Dumazete6c022a2012-10-27 23:16:46 +0000616 *expire = req->num_timeout >= thresh;
Julian Anastasov0c3d79b2009-10-19 10:03:58 +0000617 *resend = 1;
618 return;
619 }
Eric Dumazete6c022a2012-10-27 23:16:46 +0000620 *expire = req->num_timeout >= thresh &&
621 (!inet_rsk(req)->acked || req->num_timeout >= max_retries);
Julian Anastasov0c3d79b2009-10-19 10:03:58 +0000622 /*
623 * Do not resend while waiting for data after ACK,
624 * start to resend on end of deferring period to give
625 * last chance for data or ACK to create established socket.
626 */
627 *resend = !inet_rsk(req)->acked ||
Eric Dumazete6c022a2012-10-27 23:16:46 +0000628 req->num_timeout >= rskq_defer_accept - 1;
Julian Anastasov0c3d79b2009-10-19 10:03:58 +0000629}
630
Eric Dumazet1b70e972015-09-25 07:39:24 -0700631int inet_rtx_syn_ack(const struct sock *parent, struct request_sock *req)
Eric Dumazete6c022a2012-10-27 23:16:46 +0000632{
Christoph Paasch1a2c6182013-03-17 08:23:34 +0000633 int err = req->rsk_ops->rtx_syn_ack(parent, req);
Eric Dumazete6c022a2012-10-27 23:16:46 +0000634
635 if (!err)
636 req->num_retrans++;
637 return err;
638}
639EXPORT_SYMBOL(inet_rtx_syn_ack);
640
Eric Dumazet079096f2015-10-02 11:43:32 -0700641/* return true if req was found in the ehash table */
Eric Dumazetb357a362015-04-23 18:03:44 -0700642static bool reqsk_queue_unlink(struct request_sock_queue *queue,
643 struct request_sock *req)
644{
Eric Dumazet079096f2015-10-02 11:43:32 -0700645 struct inet_hashinfo *hashinfo = req_to_sk(req)->sk_prot->h.hashinfo;
Eric Dumazet5e0724d2015-10-22 08:20:46 -0700646 bool found = false;
Eric Dumazetb357a362015-04-23 18:03:44 -0700647
Eric Dumazet5e0724d2015-10-22 08:20:46 -0700648 if (sk_hashed(req_to_sk(req))) {
649 spinlock_t *lock = inet_ehash_lockp(hashinfo, req->rsk_hash);
Eric Dumazetb357a362015-04-23 18:03:44 -0700650
Eric Dumazet5e0724d2015-10-22 08:20:46 -0700651 spin_lock(lock);
652 found = __sk_nulls_del_node_init_rcu(req_to_sk(req));
653 spin_unlock(lock);
654 }
Eric Dumazet83fccfc2015-08-13 15:44:51 -0700655 if (timer_pending(&req->rsk_timer) && del_timer_sync(&req->rsk_timer))
Eric Dumazetb357a362015-04-23 18:03:44 -0700656 reqsk_put(req);
657 return found;
658}
659
660void inet_csk_reqsk_queue_drop(struct sock *sk, struct request_sock *req)
661{
662 if (reqsk_queue_unlink(&inet_csk(sk)->icsk_accept_queue, req)) {
663 reqsk_queue_removed(&inet_csk(sk)->icsk_accept_queue, req);
664 reqsk_put(req);
665 }
666}
667EXPORT_SYMBOL(inet_csk_reqsk_queue_drop);
668
Eric Dumazetf03f2e12015-10-14 11:16:27 -0700669void inet_csk_reqsk_queue_drop_and_put(struct sock *sk, struct request_sock *req)
670{
671 inet_csk_reqsk_queue_drop(sk, req);
672 reqsk_put(req);
673}
674EXPORT_SYMBOL(inet_csk_reqsk_queue_drop_and_put);
675
Eric Dumazetfa76ce732015-03-19 19:04:20 -0700676static void reqsk_timer_handler(unsigned long data)
Arnaldo Carvalho de Meloa019d6f2005-08-09 20:15:09 -0700677{
Eric Dumazetfa76ce732015-03-19 19:04:20 -0700678 struct request_sock *req = (struct request_sock *)data;
679 struct sock *sk_listener = req->rsk_listener;
Nikolay Borisov7c083ec2016-02-03 09:46:50 +0200680 struct net *net = sock_net(sk_listener);
Eric Dumazetfa76ce732015-03-19 19:04:20 -0700681 struct inet_connection_sock *icsk = inet_csk(sk_listener);
Arnaldo Carvalho de Meloa019d6f2005-08-09 20:15:09 -0700682 struct request_sock_queue *queue = &icsk->icsk_accept_queue;
Eric Dumazet2b41fab2015-03-22 10:22:18 -0700683 int qlen, expire = 0, resend = 0;
Eric Dumazetfa76ce732015-03-19 19:04:20 -0700684 int max_retries, thresh;
Eric Dumazet2b41fab2015-03-22 10:22:18 -0700685 u8 defer_accept;
Arnaldo Carvalho de Meloa019d6f2005-08-09 20:15:09 -0700686
Eric Dumazet00fd38d2015-11-12 08:43:18 -0800687 if (sk_state_load(sk_listener) != TCP_LISTEN)
Eric Dumazet079096f2015-10-02 11:43:32 -0700688 goto drop;
Arnaldo Carvalho de Meloa019d6f2005-08-09 20:15:09 -0700689
Nikolay Borisov7c083ec2016-02-03 09:46:50 +0200690 max_retries = icsk->icsk_syn_retries ? : net->ipv4.sysctl_tcp_synack_retries;
Eric Dumazetfa76ce732015-03-19 19:04:20 -0700691 thresh = max_retries;
Arnaldo Carvalho de Meloa019d6f2005-08-09 20:15:09 -0700692 /* Normally all the openreqs are young and become mature
693 * (i.e. converted to established socket) for first timeout.
Eric Dumazetfd4f2ce2012-04-12 19:48:40 +0000694 * If synack was not acknowledged for 1 second, it means
Arnaldo Carvalho de Meloa019d6f2005-08-09 20:15:09 -0700695 * one of the following things: synack was lost, ack was lost,
696 * rtt is high or nobody planned to ack (i.e. synflood).
697 * When server is a bit loaded, queue is populated with old
698 * open requests, reducing effective size of queue.
699 * When server is well loaded, queue size reduces to zero
700 * after several minutes of work. It is not synflood,
701 * it is normal operation. The solution is pruning
702 * too old entries overriding normal timeout, when
703 * situation becomes dangerous.
704 *
705 * Essentially, we reserve half of room for young
706 * embrions; and abort old ones without pity, if old
707 * ones are about to clog our table.
708 */
Eric Dumazetaac065c2015-10-02 11:43:24 -0700709 qlen = reqsk_queue_len(queue);
Eric Dumazetacb4a6b2015-10-06 14:49:58 -0700710 if ((qlen << 1) > max(8U, sk_listener->sk_max_ack_backlog)) {
Eric Dumazetaac065c2015-10-02 11:43:24 -0700711 int young = reqsk_queue_len_young(queue) << 1;
Arnaldo Carvalho de Meloa019d6f2005-08-09 20:15:09 -0700712
713 while (thresh > 2) {
Eric Dumazet2b41fab2015-03-22 10:22:18 -0700714 if (qlen < young)
Arnaldo Carvalho de Meloa019d6f2005-08-09 20:15:09 -0700715 break;
716 thresh--;
717 young <<= 1;
718 }
719 }
Eric Dumazet2b41fab2015-03-22 10:22:18 -0700720 defer_accept = READ_ONCE(queue->rskq_defer_accept);
721 if (defer_accept)
722 max_retries = defer_accept;
723 syn_ack_recalc(req, thresh, max_retries, defer_accept,
Eric Dumazetfa76ce732015-03-19 19:04:20 -0700724 &expire, &resend);
Eric Dumazet42cb80a2015-03-22 10:22:19 -0700725 req->rsk_ops->syn_ack_timeout(req);
Eric Dumazetfa76ce732015-03-19 19:04:20 -0700726 if (!expire &&
727 (!resend ||
728 !inet_rtx_syn_ack(sk_listener, req) ||
729 inet_rsk(req)->acked)) {
730 unsigned long timeo;
David S. Millerec0a1962008-06-12 16:31:35 -0700731
Eric Dumazetfa76ce732015-03-19 19:04:20 -0700732 if (req->num_timeout++ == 0)
Eric Dumazetaac065c2015-10-02 11:43:24 -0700733 atomic_dec(&queue->young);
Eric Dumazetfa76ce732015-03-19 19:04:20 -0700734 timeo = min(TCP_TIMEOUT_INIT << req->num_timeout, TCP_RTO_MAX);
Thomas Gleixnerf3438bc2016-07-04 09:50:23 +0000735 mod_timer(&req->rsk_timer, jiffies + timeo);
Eric Dumazetfa76ce732015-03-19 19:04:20 -0700736 return;
737 }
Eric Dumazet079096f2015-10-02 11:43:32 -0700738drop:
Eric Dumazetf03f2e12015-10-14 11:16:27 -0700739 inet_csk_reqsk_queue_drop_and_put(sk_listener, req);
Arnaldo Carvalho de Meloa019d6f2005-08-09 20:15:09 -0700740}
Eric Dumazetfa76ce732015-03-19 19:04:20 -0700741
Eric Dumazet079096f2015-10-02 11:43:32 -0700742static void reqsk_queue_hash_req(struct request_sock *req,
743 unsigned long timeout)
Eric Dumazetfa76ce732015-03-19 19:04:20 -0700744{
Eric Dumazetfa76ce732015-03-19 19:04:20 -0700745 req->num_retrans = 0;
746 req->num_timeout = 0;
747 req->sk = NULL;
748
Thomas Gleixnerf3438bc2016-07-04 09:50:23 +0000749 setup_pinned_timer(&req->rsk_timer, reqsk_timer_handler,
750 (unsigned long)req);
751 mod_timer(&req->rsk_timer, jiffies + timeout);
Eric Dumazet29c68522015-09-19 09:48:04 -0700752
Eric Dumazet079096f2015-10-02 11:43:32 -0700753 inet_ehash_insert(req_to_sk(req), NULL);
Eric Dumazetfa76ce732015-03-19 19:04:20 -0700754 /* before letting lookups find us, make sure all req fields
755 * are committed to memory and refcnt initialized.
756 */
757 smp_wmb();
Eric Dumazetca6fb062015-10-02 11:43:35 -0700758 atomic_set(&req->rsk_refcnt, 2 + 1);
Eric Dumazetfa76ce732015-03-19 19:04:20 -0700759}
Eric Dumazet079096f2015-10-02 11:43:32 -0700760
761void inet_csk_reqsk_queue_hash_add(struct sock *sk, struct request_sock *req,
762 unsigned long timeout)
763{
764 reqsk_queue_hash_req(req, timeout);
765 inet_csk_reqsk_queue_added(sk);
766}
767EXPORT_SYMBOL_GPL(inet_csk_reqsk_queue_hash_add);
Arnaldo Carvalho de Meloa019d6f2005-08-09 20:15:09 -0700768
Eric Dumazete56c57d2011-11-08 17:07:07 -0500769/**
770 * inet_csk_clone_lock - clone an inet socket, and lock its clone
771 * @sk: the socket to clone
772 * @req: request_sock
773 * @priority: for allocation (%GFP_KERNEL, %GFP_ATOMIC, etc)
774 *
775 * Caller must unlock socket even in error path (bh_unlock_sock(newsk))
776 */
777struct sock *inet_csk_clone_lock(const struct sock *sk,
778 const struct request_sock *req,
779 const gfp_t priority)
Arnaldo Carvalho de Melo9f1d2602005-08-09 20:11:24 -0700780{
Eric Dumazete56c57d2011-11-08 17:07:07 -0500781 struct sock *newsk = sk_clone_lock(sk, priority);
Arnaldo Carvalho de Melo9f1d2602005-08-09 20:11:24 -0700782
Ian Morris00db4122015-04-03 09:17:27 +0100783 if (newsk) {
Arnaldo Carvalho de Melo9f1d2602005-08-09 20:11:24 -0700784 struct inet_connection_sock *newicsk = inet_csk(newsk);
785
786 newsk->sk_state = TCP_SYN_RECV;
787 newicsk->icsk_bind_hash = NULL;
788
Eric Dumazet634fb9792013-10-09 15:21:29 -0700789 inet_sk(newsk)->inet_dport = inet_rsk(req)->ir_rmt_port;
Eric Dumazetb44084c2013-10-10 00:04:37 -0700790 inet_sk(newsk)->inet_num = inet_rsk(req)->ir_num;
791 inet_sk(newsk)->inet_sport = htons(inet_rsk(req)->ir_num);
Arnaldo Carvalho de Melo9f1d2602005-08-09 20:11:24 -0700792 newsk->sk_write_space = sk_stream_write_space;
793
Eric Dumazet85017862016-04-06 22:07:34 -0700794 /* listeners have SOCK_RCU_FREE, not the children */
795 sock_reset_flag(newsk, SOCK_RCU_FREE);
796
Lorenzo Colitti84f39b02014-05-13 10:17:35 -0700797 newsk->sk_mark = inet_rsk(req)->ir_mark;
Eric Dumazet33cf7c92015-03-11 18:53:14 -0700798 atomic64_set(&newsk->sk_cookie,
799 atomic64_read(&inet_rsk(req)->ir_cookie));
Lorenzo Colitti84f39b02014-05-13 10:17:35 -0700800
Arnaldo Carvalho de Melo9f1d2602005-08-09 20:11:24 -0700801 newicsk->icsk_retransmits = 0;
Arnaldo Carvalho de Melo6687e982005-08-10 04:03:31 -0300802 newicsk->icsk_backoff = 0;
803 newicsk->icsk_probes_out = 0;
Arnaldo Carvalho de Melo9f1d2602005-08-09 20:11:24 -0700804
805 /* Deinitialize accept_queue to trap illegal accesses. */
806 memset(&newicsk->icsk_accept_queue, 0, sizeof(newicsk->icsk_accept_queue));
Venkat Yekkirala4237c752006-07-24 23:32:50 -0700807
808 security_inet_csk_clone(newsk, req);
Arnaldo Carvalho de Melo9f1d2602005-08-09 20:11:24 -0700809 }
810 return newsk;
811}
Eric Dumazete56c57d2011-11-08 17:07:07 -0500812EXPORT_SYMBOL_GPL(inet_csk_clone_lock);
Arnaldo Carvalho de Meloa019d6f2005-08-09 20:15:09 -0700813
814/*
815 * At this point, there should be no process reference to this
816 * socket, and thus no user references at all. Therefore we
817 * can assume the socket waitqueue is inactive and nobody will
818 * try to jump onto it.
819 */
820void inet_csk_destroy_sock(struct sock *sk)
821{
Ilpo Järvinen547b7922008-07-25 21:43:18 -0700822 WARN_ON(sk->sk_state != TCP_CLOSE);
823 WARN_ON(!sock_flag(sk, SOCK_DEAD));
Arnaldo Carvalho de Meloa019d6f2005-08-09 20:15:09 -0700824
825 /* It cannot be in hash table! */
Ilpo Järvinen547b7922008-07-25 21:43:18 -0700826 WARN_ON(!sk_unhashed(sk));
Arnaldo Carvalho de Meloa019d6f2005-08-09 20:15:09 -0700827
Eric Dumazetc720c7e2009-10-15 06:30:45 +0000828 /* If it has not 0 inet_sk(sk)->inet_num, it must be bound */
829 WARN_ON(inet_sk(sk)->inet_num && !inet_csk(sk)->icsk_bind_hash);
Arnaldo Carvalho de Meloa019d6f2005-08-09 20:15:09 -0700830
831 sk->sk_prot->destroy(sk);
832
833 sk_stream_kill_queues(sk);
834
835 xfrm_sk_free_policy(sk);
836
837 sk_refcnt_debug_release(sk);
838
Eric Dumazetdd24c002008-11-25 21:17:14 -0800839 percpu_counter_dec(sk->sk_prot->orphan_count);
Eric Dumazetc2a2efb2017-01-20 05:06:08 -0800840
Arnaldo Carvalho de Meloa019d6f2005-08-09 20:15:09 -0700841 sock_put(sk);
842}
Arnaldo Carvalho de Meloa019d6f2005-08-09 20:15:09 -0700843EXPORT_SYMBOL(inet_csk_destroy_sock);
844
Christoph Paasche337e242012-12-14 04:07:58 +0000845/* This function allows to force a closure of a socket after the call to
846 * tcp/dccp_create_openreq_child().
847 */
848void inet_csk_prepare_forced_close(struct sock *sk)
Christoph Paaschc10cb5f2013-03-07 02:34:33 +0000849 __releases(&sk->sk_lock.slock)
Christoph Paasche337e242012-12-14 04:07:58 +0000850{
851 /* sk_clone_lock locked the socket and set refcnt to 2 */
852 bh_unlock_sock(sk);
853 sock_put(sk);
854
855 /* The below has to be done to allow calling inet_csk_destroy_sock */
856 sock_set_flag(sk, SOCK_DEAD);
857 percpu_counter_inc(sk->sk_prot->orphan_count);
858 inet_sk(sk)->inet_num = 0;
859}
860EXPORT_SYMBOL(inet_csk_prepare_forced_close);
861
Eric Dumazetf985c652015-10-14 06:16:49 -0700862int inet_csk_listen_start(struct sock *sk, int backlog)
Arnaldo Carvalho de Meloa019d6f2005-08-09 20:15:09 -0700863{
Arnaldo Carvalho de Meloa019d6f2005-08-09 20:15:09 -0700864 struct inet_connection_sock *icsk = inet_csk(sk);
Eric Dumazet10cbc8f2015-10-02 11:43:36 -0700865 struct inet_sock *inet = inet_sk(sk);
Craig Gallek086c6532016-02-10 11:50:35 -0500866 int err = -EADDRINUSE;
Arnaldo Carvalho de Meloa019d6f2005-08-09 20:15:09 -0700867
Eric Dumazetef547f22015-10-02 11:43:37 -0700868 reqsk_queue_alloc(&icsk->icsk_accept_queue);
Arnaldo Carvalho de Meloa019d6f2005-08-09 20:15:09 -0700869
Eric Dumazetf985c652015-10-14 06:16:49 -0700870 sk->sk_max_ack_backlog = backlog;
Arnaldo Carvalho de Meloa019d6f2005-08-09 20:15:09 -0700871 sk->sk_ack_backlog = 0;
872 inet_csk_delack_init(sk);
873
874 /* There is race window here: we announce ourselves listening,
875 * but this transition is still not validated by get_port().
876 * It is OK, because this socket enters to hash table only
877 * after validation is complete.
878 */
Eric Dumazet00fd38d2015-11-12 08:43:18 -0800879 sk_state_store(sk, TCP_LISTEN);
Eric Dumazetc720c7e2009-10-15 06:30:45 +0000880 if (!sk->sk_prot->get_port(sk, inet->inet_num)) {
881 inet->inet_sport = htons(inet->inet_num);
Arnaldo Carvalho de Meloa019d6f2005-08-09 20:15:09 -0700882
883 sk_dst_reset(sk);
Craig Gallek086c6532016-02-10 11:50:35 -0500884 err = sk->sk_prot->hash(sk);
Arnaldo Carvalho de Meloa019d6f2005-08-09 20:15:09 -0700885
Craig Gallek086c6532016-02-10 11:50:35 -0500886 if (likely(!err))
887 return 0;
Arnaldo Carvalho de Meloa019d6f2005-08-09 20:15:09 -0700888 }
889
890 sk->sk_state = TCP_CLOSE;
Craig Gallek086c6532016-02-10 11:50:35 -0500891 return err;
Arnaldo Carvalho de Meloa019d6f2005-08-09 20:15:09 -0700892}
Arnaldo Carvalho de Meloa019d6f2005-08-09 20:15:09 -0700893EXPORT_SYMBOL_GPL(inet_csk_listen_start);
894
Eric Dumazetebb516a2015-10-14 11:16:28 -0700895static void inet_child_forget(struct sock *sk, struct request_sock *req,
896 struct sock *child)
897{
898 sk->sk_prot->disconnect(child, O_NONBLOCK);
899
900 sock_orphan(child);
901
902 percpu_counter_inc(sk->sk_prot->orphan_count);
903
904 if (sk->sk_protocol == IPPROTO_TCP && tcp_rsk(req)->tfo_listener) {
905 BUG_ON(tcp_sk(child)->fastopen_rsk != req);
906 BUG_ON(sk != req->rsk_listener);
907
908 /* Paranoid, to prevent race condition if
909 * an inbound pkt destined for child is
910 * blocked by sock lock in tcp_v4_rcv().
911 * Also to satisfy an assertion in
912 * tcp_v4_destroy_sock().
913 */
914 tcp_sk(child)->fastopen_rsk = NULL;
915 }
916 inet_csk_destroy_sock(child);
917 reqsk_put(req);
918}
919
Eric Dumazet77166822016-02-18 05:39:18 -0800920struct sock *inet_csk_reqsk_queue_add(struct sock *sk,
921 struct request_sock *req,
922 struct sock *child)
Eric Dumazetebb516a2015-10-14 11:16:28 -0700923{
924 struct request_sock_queue *queue = &inet_csk(sk)->icsk_accept_queue;
925
926 spin_lock(&queue->rskq_lock);
927 if (unlikely(sk->sk_state != TCP_LISTEN)) {
928 inet_child_forget(sk, req, child);
Eric Dumazet77166822016-02-18 05:39:18 -0800929 child = NULL;
Eric Dumazetebb516a2015-10-14 11:16:28 -0700930 } else {
931 req->sk = child;
932 req->dl_next = NULL;
933 if (queue->rskq_accept_head == NULL)
934 queue->rskq_accept_head = req;
935 else
936 queue->rskq_accept_tail->dl_next = req;
937 queue->rskq_accept_tail = req;
938 sk_acceptq_added(sk);
939 }
940 spin_unlock(&queue->rskq_lock);
Eric Dumazet77166822016-02-18 05:39:18 -0800941 return child;
Eric Dumazetebb516a2015-10-14 11:16:28 -0700942}
943EXPORT_SYMBOL(inet_csk_reqsk_queue_add);
944
Eric Dumazet5e0724d2015-10-22 08:20:46 -0700945struct sock *inet_csk_complete_hashdance(struct sock *sk, struct sock *child,
946 struct request_sock *req, bool own_req)
947{
948 if (own_req) {
949 inet_csk_reqsk_queue_drop(sk, req);
950 reqsk_queue_removed(&inet_csk(sk)->icsk_accept_queue, req);
Eric Dumazet77166822016-02-18 05:39:18 -0800951 if (inet_csk_reqsk_queue_add(sk, req, child))
952 return child;
Eric Dumazet5e0724d2015-10-22 08:20:46 -0700953 }
954 /* Too bad, another child took ownership of the request, undo. */
955 bh_unlock_sock(child);
956 sock_put(child);
957 return NULL;
958}
959EXPORT_SYMBOL(inet_csk_complete_hashdance);
960
Arnaldo Carvalho de Meloa019d6f2005-08-09 20:15:09 -0700961/*
962 * This routine closes sockets which have been at least partially
963 * opened, but not yet accepted.
964 */
965void inet_csk_listen_stop(struct sock *sk)
966{
967 struct inet_connection_sock *icsk = inet_csk(sk);
Jerry Chu83368862012-08-31 12:29:12 +0000968 struct request_sock_queue *queue = &icsk->icsk_accept_queue;
Eric Dumazetfff1f302015-10-02 11:43:23 -0700969 struct request_sock *next, *req;
Arnaldo Carvalho de Meloa019d6f2005-08-09 20:15:09 -0700970
971 /* Following specs, it would be better either to send FIN
972 * (and enter FIN-WAIT-1, it is normal close)
973 * or to send active reset (abort).
974 * Certainly, it is pretty dangerous while synflood, but it is
975 * bad justification for our negligence 8)
976 * To be honest, we are not able to make either
977 * of the variants now. --ANK
978 */
Eric Dumazetfff1f302015-10-02 11:43:23 -0700979 while ((req = reqsk_queue_remove(queue, sk)) != NULL) {
Arnaldo Carvalho de Meloa019d6f2005-08-09 20:15:09 -0700980 struct sock *child = req->sk;
981
Arnaldo Carvalho de Meloa019d6f2005-08-09 20:15:09 -0700982 local_bh_disable();
983 bh_lock_sock(child);
Ilpo Järvinen547b7922008-07-25 21:43:18 -0700984 WARN_ON(sock_owned_by_user(child));
Arnaldo Carvalho de Meloa019d6f2005-08-09 20:15:09 -0700985 sock_hold(child);
986
Eric Dumazetebb516a2015-10-14 11:16:28 -0700987 inet_child_forget(sk, req, child);
Arnaldo Carvalho de Meloa019d6f2005-08-09 20:15:09 -0700988 bh_unlock_sock(child);
989 local_bh_enable();
990 sock_put(child);
991
Eric Dumazet92d6f172015-10-02 11:43:38 -0700992 cond_resched();
Arnaldo Carvalho de Meloa019d6f2005-08-09 20:15:09 -0700993 }
Eric Dumazet0536fcc2015-09-29 07:42:52 -0700994 if (queue->fastopenq.rskq_rst_head) {
Jerry Chu83368862012-08-31 12:29:12 +0000995 /* Free all the reqs queued in rskq_rst_head. */
Eric Dumazet0536fcc2015-09-29 07:42:52 -0700996 spin_lock_bh(&queue->fastopenq.lock);
Eric Dumazetfff1f302015-10-02 11:43:23 -0700997 req = queue->fastopenq.rskq_rst_head;
Eric Dumazet0536fcc2015-09-29 07:42:52 -0700998 queue->fastopenq.rskq_rst_head = NULL;
999 spin_unlock_bh(&queue->fastopenq.lock);
Eric Dumazetfff1f302015-10-02 11:43:23 -07001000 while (req != NULL) {
1001 next = req->dl_next;
Eric Dumazet13854e52015-03-15 21:12:16 -07001002 reqsk_put(req);
Eric Dumazetfff1f302015-10-02 11:43:23 -07001003 req = next;
Jerry Chu83368862012-08-31 12:29:12 +00001004 }
1005 }
Eric Dumazetebb516a2015-10-14 11:16:28 -07001006 WARN_ON_ONCE(sk->sk_ack_backlog);
Arnaldo Carvalho de Meloa019d6f2005-08-09 20:15:09 -07001007}
Arnaldo Carvalho de Meloa019d6f2005-08-09 20:15:09 -07001008EXPORT_SYMBOL_GPL(inet_csk_listen_stop);
Arnaldo Carvalho de Meloaf05dc92005-12-13 23:16:04 -08001009
1010void inet_csk_addr2sockaddr(struct sock *sk, struct sockaddr *uaddr)
1011{
1012 struct sockaddr_in *sin = (struct sockaddr_in *)uaddr;
1013 const struct inet_sock *inet = inet_sk(sk);
1014
1015 sin->sin_family = AF_INET;
Eric Dumazetc720c7e2009-10-15 06:30:45 +00001016 sin->sin_addr.s_addr = inet->inet_daddr;
1017 sin->sin_port = inet->inet_dport;
Arnaldo Carvalho de Meloaf05dc92005-12-13 23:16:04 -08001018}
Arnaldo Carvalho de Meloaf05dc92005-12-13 23:16:04 -08001019EXPORT_SYMBOL_GPL(inet_csk_addr2sockaddr);
Arnaldo Carvalho de Meloc4d93902006-03-20 22:01:03 -08001020
Arnaldo Carvalho de Melodec73ff2006-03-20 22:46:16 -08001021#ifdef CONFIG_COMPAT
1022int inet_csk_compat_getsockopt(struct sock *sk, int level, int optname,
1023 char __user *optval, int __user *optlen)
1024{
David S. Millerdbeff122006-03-20 22:52:32 -08001025 const struct inet_connection_sock *icsk = inet_csk(sk);
Arnaldo Carvalho de Melodec73ff2006-03-20 22:46:16 -08001026
Ian Morris00db4122015-04-03 09:17:27 +01001027 if (icsk->icsk_af_ops->compat_getsockopt)
Arnaldo Carvalho de Melodec73ff2006-03-20 22:46:16 -08001028 return icsk->icsk_af_ops->compat_getsockopt(sk, level, optname,
1029 optval, optlen);
1030 return icsk->icsk_af_ops->getsockopt(sk, level, optname,
1031 optval, optlen);
1032}
Arnaldo Carvalho de Melodec73ff2006-03-20 22:46:16 -08001033EXPORT_SYMBOL_GPL(inet_csk_compat_getsockopt);
1034
1035int inet_csk_compat_setsockopt(struct sock *sk, int level, int optname,
David S. Millerb7058842009-09-30 16:12:20 -07001036 char __user *optval, unsigned int optlen)
Arnaldo Carvalho de Melodec73ff2006-03-20 22:46:16 -08001037{
David S. Millerdbeff122006-03-20 22:52:32 -08001038 const struct inet_connection_sock *icsk = inet_csk(sk);
Arnaldo Carvalho de Melodec73ff2006-03-20 22:46:16 -08001039
Ian Morris00db4122015-04-03 09:17:27 +01001040 if (icsk->icsk_af_ops->compat_setsockopt)
Arnaldo Carvalho de Melodec73ff2006-03-20 22:46:16 -08001041 return icsk->icsk_af_ops->compat_setsockopt(sk, level, optname,
1042 optval, optlen);
1043 return icsk->icsk_af_ops->setsockopt(sk, level, optname,
1044 optval, optlen);
1045}
Arnaldo Carvalho de Melodec73ff2006-03-20 22:46:16 -08001046EXPORT_SYMBOL_GPL(inet_csk_compat_setsockopt);
1047#endif
David S. Miller80d0a692012-07-16 03:28:06 -07001048
1049static struct dst_entry *inet_csk_rebuild_route(struct sock *sk, struct flowi *fl)
1050{
Eric Dumazet5abf7f72012-07-17 22:42:13 +02001051 const struct inet_sock *inet = inet_sk(sk);
1052 const struct ip_options_rcu *inet_opt;
David S. Miller80d0a692012-07-16 03:28:06 -07001053 __be32 daddr = inet->inet_daddr;
1054 struct flowi4 *fl4;
1055 struct rtable *rt;
1056
1057 rcu_read_lock();
1058 inet_opt = rcu_dereference(inet->inet_opt);
1059 if (inet_opt && inet_opt->opt.srr)
1060 daddr = inet_opt->opt.faddr;
1061 fl4 = &fl->u.ip4;
1062 rt = ip_route_output_ports(sock_net(sk), fl4, sk, daddr,
1063 inet->inet_saddr, inet->inet_dport,
1064 inet->inet_sport, sk->sk_protocol,
1065 RT_CONN_FLAGS(sk), sk->sk_bound_dev_if);
1066 if (IS_ERR(rt))
1067 rt = NULL;
1068 if (rt)
1069 sk_setup_caps(sk, &rt->dst);
1070 rcu_read_unlock();
1071
1072 return &rt->dst;
1073}
1074
1075struct dst_entry *inet_csk_update_pmtu(struct sock *sk, u32 mtu)
1076{
1077 struct dst_entry *dst = __sk_dst_check(sk, 0);
1078 struct inet_sock *inet = inet_sk(sk);
1079
1080 if (!dst) {
1081 dst = inet_csk_rebuild_route(sk, &inet->cork.fl);
1082 if (!dst)
1083 goto out;
1084 }
David S. Miller6700c272012-07-17 03:29:28 -07001085 dst->ops->update_pmtu(dst, sk, NULL, mtu);
David S. Miller80d0a692012-07-16 03:28:06 -07001086
1087 dst = __sk_dst_check(sk, 0);
1088 if (!dst)
1089 dst = inet_csk_rebuild_route(sk, &inet->cork.fl);
1090out:
1091 return dst;
1092}
1093EXPORT_SYMBOL_GPL(inet_csk_update_pmtu);