blob: 2065816748066986f0356df168c2d76fe2d53d85 [file] [log] [blame]
Arnaldo Carvalho de Meloe48c4142005-08-09 20:09:46 -07001/*
2 * INET An implementation of the TCP/IP protocol suite for the LINUX
3 * operating system. INET is implemented using the BSD Socket
4 * interface as the means of communication with the user level.
5 *
6 * Generic TIME_WAIT sockets functions
7 *
8 * From code orinally in TCP
9 */
10
Ilpo Järvinen172589c2007-08-28 15:50:33 -070011#include <linux/kernel.h>
Vegard Nossum9e337b02008-10-18 17:37:51 +020012#include <linux/kmemcheck.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090013#include <linux/slab.h>
Paul Gortmaker3a9a2312011-05-27 09:12:25 -040014#include <linux/module.h>
Arnaldo Carvalho de Meloe48c4142005-08-09 20:09:46 -070015#include <net/inet_hashtables.h>
16#include <net/inet_timewait_sock.h>
Arnaldo Carvalho de Melo696ab2d2005-08-09 20:45:03 -070017#include <net/ip.h>
Arnaldo Carvalho de Meloe48c4142005-08-09 20:09:46 -070018
Eric Dumazet13475a32009-12-02 22:31:19 +000019
Eric Dumazet2a8875e2009-12-08 20:19:53 -080020/**
Eric Dumazet2a8875e2009-12-08 20:19:53 -080021 * inet_twsk_bind_unhash - unhash a timewait socket from bind hash
22 * @tw: timewait socket
23 * @hashinfo: hashinfo pointer
24 *
25 * unhash a timewait socket from bind hash, if hashed.
26 * bind hash lock must be held by caller.
27 * Returns 1 if caller should call inet_twsk_put() after lock release.
Eric Dumazet3cdaeda2009-12-04 03:47:42 +000028 */
Eric Dumazetfc01538f2015-07-08 14:28:29 -070029void inet_twsk_bind_unhash(struct inet_timewait_sock *tw,
Eric Dumazet3cdaeda2009-12-04 03:47:42 +000030 struct inet_hashinfo *hashinfo)
31{
32 struct inet_bind_bucket *tb = tw->tw_tb;
33
34 if (!tb)
Eric Dumazetfc01538f2015-07-08 14:28:29 -070035 return;
Eric Dumazet3cdaeda2009-12-04 03:47:42 +000036
37 __hlist_del(&tw->tw_bind_node);
38 tw->tw_tb = NULL;
39 inet_bind_bucket_destroy(hashinfo->bind_bucket_cachep, tb);
Eric Dumazetfc01538f2015-07-08 14:28:29 -070040 __sock_put((struct sock *)tw);
Eric Dumazet3cdaeda2009-12-04 03:47:42 +000041}
42
Arnaldo Carvalho de Meloe48c4142005-08-09 20:09:46 -070043/* Must be called with locally disabled BHs. */
Eric Dumazet789f5582015-04-12 18:51:09 -070044static void inet_twsk_kill(struct inet_timewait_sock *tw)
Arnaldo Carvalho de Meloe48c4142005-08-09 20:09:46 -070045{
Eric Dumazet789f5582015-04-12 18:51:09 -070046 struct inet_hashinfo *hashinfo = tw->tw_dr->hashinfo;
Eric Dumazet9db66bd2008-11-20 20:39:09 -080047 spinlock_t *lock = inet_ehash_lockp(hashinfo, tw->tw_hash);
Eric Dumazetfc01538f2015-07-08 14:28:29 -070048 struct inet_bind_hashbucket *bhead;
Arnaldo Carvalho de Meloe48c4142005-08-09 20:09:46 -070049
Eric Dumazet9db66bd2008-11-20 20:39:09 -080050 spin_lock(lock);
Eric Dumazetfc01538f2015-07-08 14:28:29 -070051 sk_nulls_del_node_init_rcu((struct sock *)tw);
Eric Dumazet9db66bd2008-11-20 20:39:09 -080052 spin_unlock(lock);
Arnaldo Carvalho de Meloe48c4142005-08-09 20:09:46 -070053
54 /* Disassociate with bind bucket. */
Pavel Emelyanov7f635ab2008-06-16 17:12:49 -070055 bhead = &hashinfo->bhash[inet_bhashfn(twsk_net(tw), tw->tw_num,
56 hashinfo->bhash_size)];
Eric Dumazet3cdaeda2009-12-04 03:47:42 +000057
Arnaldo Carvalho de Meloe48c4142005-08-09 20:09:46 -070058 spin_lock(&bhead->lock);
Eric Dumazetfc01538f2015-07-08 14:28:29 -070059 inet_twsk_bind_unhash(tw, hashinfo);
Arnaldo Carvalho de Meloe48c4142005-08-09 20:09:46 -070060 spin_unlock(&bhead->lock);
Eric Dumazet3cdaeda2009-12-04 03:47:42 +000061
Eric Dumazet789f5582015-04-12 18:51:09 -070062 atomic_dec(&tw->tw_dr->tw_count);
63 inet_twsk_put(tw);
Arnaldo Carvalho de Meloe48c4142005-08-09 20:09:46 -070064}
65
Eric Dumazet05dbc7b2013-10-03 00:22:02 -070066void inet_twsk_free(struct inet_timewait_sock *tw)
Arnaldo Carvalho de Melo4dbc8ef2009-05-06 16:50:52 -070067{
68 struct module *owner = tw->tw_prot->owner;
69 twsk_destructor((struct sock *)tw);
70#ifdef SOCK_REFCNT_DEBUG
71 pr_debug("%s timewait_sock %p released\n", tw->tw_prot->name, tw);
72#endif
Arnaldo Carvalho de Melo4dbc8ef2009-05-06 16:50:52 -070073 kmem_cache_free(tw->tw_prot->twsk_prot->twsk_slab, tw);
74 module_put(owner);
75}
76
Pavel Emelyanov7054fb92007-12-20 15:32:54 -080077void inet_twsk_put(struct inet_timewait_sock *tw)
78{
Arnaldo Carvalho de Melo4dbc8ef2009-05-06 16:50:52 -070079 if (atomic_dec_and_test(&tw->tw_refcnt))
80 inet_twsk_free(tw);
Pavel Emelyanov7054fb92007-12-20 15:32:54 -080081}
82EXPORT_SYMBOL_GPL(inet_twsk_put);
83
Eric Dumazet05dbc7b2013-10-03 00:22:02 -070084static void inet_twsk_add_node_rcu(struct inet_timewait_sock *tw,
85 struct hlist_nulls_head *list)
86{
87 hlist_nulls_add_head_rcu(&tw->tw_node, list);
88}
89
90static void inet_twsk_add_bind_node(struct inet_timewait_sock *tw,
91 struct hlist_head *list)
92{
93 hlist_add_head(&tw->tw_bind_node, list);
94}
95
Arnaldo Carvalho de Meloe48c4142005-08-09 20:09:46 -070096/*
Eric Dumazet614bdd42016-05-03 17:10:50 -070097 * Enter the time wait state.
Arnaldo Carvalho de Meloe48c4142005-08-09 20:09:46 -070098 * Essentially we whip up a timewait bucket, copy the relevant info into it
99 * from the SK, and mess with hash chains and list linkage.
100 */
101void __inet_twsk_hashdance(struct inet_timewait_sock *tw, struct sock *sk,
102 struct inet_hashinfo *hashinfo)
103{
104 const struct inet_sock *inet = inet_sk(sk);
Arnaldo Carvalho de Melo463c84b2005-08-09 20:10:42 -0700105 const struct inet_connection_sock *icsk = inet_csk(sk);
Eric Dumazet81c3d542005-10-03 14:13:38 -0700106 struct inet_ehash_bucket *ehead = inet_ehash_bucket(hashinfo, sk->sk_hash);
Eric Dumazet9db66bd2008-11-20 20:39:09 -0800107 spinlock_t *lock = inet_ehash_lockp(hashinfo, sk->sk_hash);
Arnaldo Carvalho de Meloe48c4142005-08-09 20:09:46 -0700108 struct inet_bind_hashbucket *bhead;
109 /* Step 1: Put TW into bind hash. Original socket stays there too.
110 Note, that any socket with inet->num != 0 MUST be bound in
111 binding cache, even if it is closed.
112 */
Eric Dumazetc720c7e2009-10-15 06:30:45 +0000113 bhead = &hashinfo->bhash[inet_bhashfn(twsk_net(tw), inet->inet_num,
Pavel Emelyanov7f635ab2008-06-16 17:12:49 -0700114 hashinfo->bhash_size)];
Eric Dumazet614bdd42016-05-03 17:10:50 -0700115 spin_lock_bh(&bhead->lock);
Arnaldo Carvalho de Melo463c84b2005-08-09 20:10:42 -0700116 tw->tw_tb = icsk->icsk_bind_hash;
Ilpo Järvinen547b7922008-07-25 21:43:18 -0700117 WARN_ON(!icsk->icsk_bind_hash);
Arnaldo Carvalho de Meloe48c4142005-08-09 20:09:46 -0700118 inet_twsk_add_bind_node(tw, &tw->tw_tb->owners);
119 spin_unlock(&bhead->lock);
120
Eric Dumazet9db66bd2008-11-20 20:39:09 -0800121 spin_lock(lock);
Arnaldo Carvalho de Meloe48c4142005-08-09 20:09:46 -0700122
Eric Dumazet3ab5aee2008-11-16 19:40:17 -0800123 /*
Eric Dumazet05dbc7b2013-10-03 00:22:02 -0700124 * Step 2: Hash TW into tcp ehash chain.
125 * Notes :
Eric Dumazeted2e9232015-09-19 09:08:34 -0700126 * - tw_refcnt is set to 4 because :
Eric Dumazet05dbc7b2013-10-03 00:22:02 -0700127 * - We have one reference from bhash chain.
128 * - We have one reference from ehash chain.
Eric Dumazeted2e9232015-09-19 09:08:34 -0700129 * - We have one reference from timer.
130 * - One reference for ourself (our caller will release it).
Eric Dumazet05dbc7b2013-10-03 00:22:02 -0700131 * We can use atomic_set() because prior spin_lock()/spin_unlock()
132 * committed into memory all tw fields.
Eric Dumazet3ab5aee2008-11-16 19:40:17 -0800133 */
Eric Dumazeted2e9232015-09-19 09:08:34 -0700134 atomic_set(&tw->tw_refcnt, 4);
Eric Dumazet05dbc7b2013-10-03 00:22:02 -0700135 inet_twsk_add_node_rcu(tw, &ehead->chain);
Eric Dumazet3ab5aee2008-11-16 19:40:17 -0800136
Eric Dumazet05dbc7b2013-10-03 00:22:02 -0700137 /* Step 3: Remove SK from hash chain */
Eric Dumazet3ab5aee2008-11-16 19:40:17 -0800138 if (__sk_nulls_del_node_init_rcu(sk))
139 sock_prot_inuse_add(sock_net(sk), sk->sk_prot, -1);
Arnaldo Carvalho de Meloe48c4142005-08-09 20:09:46 -0700140
Eric Dumazet614bdd42016-05-03 17:10:50 -0700141 spin_unlock_bh(lock);
Arnaldo Carvalho de Meloe48c4142005-08-09 20:09:46 -0700142}
Arnaldo Carvalho de Melo696ab2d2005-08-09 20:45:03 -0700143EXPORT_SYMBOL_GPL(__inet_twsk_hashdance);
144
Eric Dumazet216f8bb2015-05-12 06:22:56 -0700145static void tw_timer_handler(unsigned long data)
Arnaldo Carvalho de Meloc6762702005-08-09 20:09:59 -0700146{
Eric Dumazet789f5582015-04-12 18:51:09 -0700147 struct inet_timewait_sock *tw = (struct inet_timewait_sock *)data;
148
149 if (tw->tw_kill)
Eric Dumazet02a1d6e2016-04-27 16:44:39 -0700150 __NET_INC_STATS(twsk_net(tw), LINUX_MIB_TIMEWAITKILLED);
Eric Dumazet789f5582015-04-12 18:51:09 -0700151 else
Eric Dumazet02a1d6e2016-04-27 16:44:39 -0700152 __NET_INC_STATS(twsk_net(tw), LINUX_MIB_TIMEWAITED);
Eric Dumazet789f5582015-04-12 18:51:09 -0700153 inet_twsk_kill(tw);
154}
155
156struct inet_timewait_sock *inet_twsk_alloc(const struct sock *sk,
157 struct inet_timewait_death_row *dr,
158 const int state)
159{
160 struct inet_timewait_sock *tw;
161
162 if (atomic_read(&dr->tw_count) >= dr->sysctl_max_tw_buckets)
163 return NULL;
164
165 tw = kmem_cache_alloc(sk->sk_prot_creator->twsk_prot->twsk_slab,
166 GFP_ATOMIC);
Ian Morris00db4122015-04-03 09:17:27 +0100167 if (tw) {
Arnaldo Carvalho de Meloc6762702005-08-09 20:09:59 -0700168 const struct inet_sock *inet = inet_sk(sk);
169
Vegard Nossum9e337b02008-10-18 17:37:51 +0200170 kmemcheck_annotate_bitfield(tw, flags);
171
Eric Dumazet789f5582015-04-12 18:51:09 -0700172 tw->tw_dr = dr;
Arnaldo Carvalho de Meloc6762702005-08-09 20:09:59 -0700173 /* Give us an identity. */
Eric Dumazetc720c7e2009-10-15 06:30:45 +0000174 tw->tw_daddr = inet->inet_daddr;
175 tw->tw_rcv_saddr = inet->inet_rcv_saddr;
Arnaldo Carvalho de Meloc6762702005-08-09 20:09:59 -0700176 tw->tw_bound_dev_if = sk->sk_bound_dev_if;
Eric Dumazet66b13d92011-10-24 03:06:21 -0400177 tw->tw_tos = inet->tos;
Eric Dumazetc720c7e2009-10-15 06:30:45 +0000178 tw->tw_num = inet->inet_num;
Arnaldo Carvalho de Meloc6762702005-08-09 20:09:59 -0700179 tw->tw_state = TCP_TIME_WAIT;
180 tw->tw_substate = state;
Eric Dumazetc720c7e2009-10-15 06:30:45 +0000181 tw->tw_sport = inet->inet_sport;
182 tw->tw_dport = inet->inet_dport;
Arnaldo Carvalho de Meloc6762702005-08-09 20:09:59 -0700183 tw->tw_family = sk->sk_family;
184 tw->tw_reuse = sk->sk_reuse;
Eric Dumazet81c3d542005-10-03 14:13:38 -0700185 tw->tw_hash = sk->sk_hash;
Arnaldo Carvalho de Meloc6762702005-08-09 20:09:59 -0700186 tw->tw_ipv6only = 0;
KOVACS Krisztianf5715ae2008-10-01 07:30:02 -0700187 tw->tw_transparent = inet->transparent;
Arnaldo Carvalho de Meloc6762702005-08-09 20:09:59 -0700188 tw->tw_prot = sk->sk_prot_creator;
Eric Dumazet33cf7c92015-03-11 18:53:14 -0700189 atomic64_set(&tw->tw_cookie, atomic64_read(&sk->sk_cookie));
Eric W. Biedermanefd7ef12015-03-11 23:04:08 -0500190 twsk_net_set(tw, sock_net(sk));
Eric Dumazet789f5582015-04-12 18:51:09 -0700191 setup_timer(&tw->tw_timer, tw_timer_handler, (unsigned long)tw);
Eric Dumazet47e1c322009-12-03 00:49:01 +0000192 /*
193 * Because we use RCU lookups, we should not set tw_refcnt
194 * to a non null value before everything is setup for this
195 * timewait socket.
196 */
197 atomic_set(&tw->tw_refcnt, 0);
Eric Dumazet789f5582015-04-12 18:51:09 -0700198
Arnaldo Carvalho de Meloeeb2b852005-10-10 21:25:23 -0700199 __module_get(tw->tw_prot->owner);
Arnaldo Carvalho de Meloc6762702005-08-09 20:09:59 -0700200 }
201
202 return tw;
203}
Arnaldo Carvalho de Melo696ab2d2005-08-09 20:45:03 -0700204EXPORT_SYMBOL_GPL(inet_twsk_alloc);
205
Arnaldo Carvalho de Melo696ab2d2005-08-09 20:45:03 -0700206/* These are always called from BH context. See callers in
207 * tcp_input.c to verify this.
208 */
209
Eric Dumazetdbe7faa2015-07-08 14:28:30 -0700210/* This is for handling early-kills of TIME_WAIT sockets.
211 * Warning : consume reference.
212 * Caller should not access tw anymore.
213 */
214void inet_twsk_deschedule_put(struct inet_timewait_sock *tw)
Arnaldo Carvalho de Melo696ab2d2005-08-09 20:45:03 -0700215{
Eric Dumazet789f5582015-04-12 18:51:09 -0700216 if (del_timer_sync(&tw->tw_timer))
217 inet_twsk_kill(tw);
Eric Dumazetdbe7faa2015-07-08 14:28:30 -0700218 inet_twsk_put(tw);
Arnaldo Carvalho de Melo696ab2d2005-08-09 20:45:03 -0700219}
Eric Dumazetdbe7faa2015-07-08 14:28:30 -0700220EXPORT_SYMBOL(inet_twsk_deschedule_put);
Arnaldo Carvalho de Melo696ab2d2005-08-09 20:45:03 -0700221
Eric Dumazeted2e9232015-09-19 09:08:34 -0700222void __inet_twsk_schedule(struct inet_timewait_sock *tw, int timeo, bool rearm)
Arnaldo Carvalho de Melo696ab2d2005-08-09 20:45:03 -0700223{
Arnaldo Carvalho de Melo696ab2d2005-08-09 20:45:03 -0700224 /* timeout := RTO * 3.5
225 *
226 * 3.5 = 1+2+0.5 to wait for two retransmits.
227 *
228 * RATIONALE: if FIN arrived and we entered TIME-WAIT state,
229 * our ACK acking that FIN can be lost. If N subsequent retransmitted
230 * FINs (or previous seqments) are lost (probability of such event
231 * is p^(N+1), where p is probability to lose single packet and
232 * time to detect the loss is about RTO*(2^N - 1) with exponential
233 * backoff). Normal timewait length is calculated so, that we
234 * waited at least for one retransmitted FIN (maximal RTO is 120sec).
235 * [ BTW Linux. following BSD, violates this requirement waiting
236 * only for 60sec, we should wait at least for 240 secs.
237 * Well, 240 consumes too much of resources 8)
238 * ]
239 * This interval is not reduced to catch old duplicate and
240 * responces to our wandering segments living for two MSLs.
241 * However, if we use PAWS to detect
242 * old duplicates, we can reduce the interval to bounds required
243 * by RTO, rather than MSL. So, if peer understands PAWS, we
244 * kill tw bucket after 3.5*RTO (it is important that this number
245 * is greater than TS tick!) and detect old duplicates with help
246 * of PAWS.
247 */
Arnaldo Carvalho de Melo696ab2d2005-08-09 20:45:03 -0700248
Eric Dumazet789f5582015-04-12 18:51:09 -0700249 tw->tw_kill = timeo <= 4*HZ;
Eric Dumazeted2e9232015-09-19 09:08:34 -0700250 if (!rearm) {
251 BUG_ON(mod_timer_pinned(&tw->tw_timer, jiffies + timeo));
Eric Dumazet789f5582015-04-12 18:51:09 -0700252 atomic_inc(&tw->tw_dr->tw_count);
Eric Dumazeted2e9232015-09-19 09:08:34 -0700253 } else {
254 mod_timer_pending(&tw->tw_timer, jiffies + timeo);
Arnaldo Carvalho de Melo696ab2d2005-08-09 20:45:03 -0700255 }
Arnaldo Carvalho de Melo696ab2d2005-08-09 20:45:03 -0700256}
Eric Dumazeted2e9232015-09-19 09:08:34 -0700257EXPORT_SYMBOL_GPL(__inet_twsk_schedule);
Arnaldo Carvalho de Melo696ab2d2005-08-09 20:45:03 -0700258
Eric W. Biedermanb099ce22009-12-03 02:29:09 +0000259void inet_twsk_purge(struct inet_hashinfo *hashinfo,
Daniel Lezcanod3154922008-09-08 13:17:27 -0700260 struct inet_timewait_death_row *twdr, int family)
261{
262 struct inet_timewait_sock *tw;
263 struct sock *sk;
Eric Dumazet3ab5aee2008-11-16 19:40:17 -0800264 struct hlist_nulls_node *node;
Eric W. Biederman575f4cd2009-12-03 02:29:08 +0000265 unsigned int slot;
Daniel Lezcanod3154922008-09-08 13:17:27 -0700266
Eric W. Biederman575f4cd2009-12-03 02:29:08 +0000267 for (slot = 0; slot <= hashinfo->ehash_mask; slot++) {
268 struct inet_ehash_bucket *head = &hashinfo->ehash[slot];
269restart_rcu:
Eric Dumazet738e6d32015-03-18 17:40:51 -0700270 cond_resched();
Eric W. Biederman575f4cd2009-12-03 02:29:08 +0000271 rcu_read_lock();
Daniel Lezcanod3154922008-09-08 13:17:27 -0700272restart:
Eric Dumazet05dbc7b2013-10-03 00:22:02 -0700273 sk_nulls_for_each_rcu(sk, node, &head->chain) {
274 if (sk->sk_state != TCP_TIME_WAIT)
275 continue;
Daniel Lezcanod3154922008-09-08 13:17:27 -0700276 tw = inet_twsk(sk);
Eric W. Biedermanb099ce22009-12-03 02:29:09 +0000277 if ((tw->tw_family != family) ||
278 atomic_read(&twsk_net(tw)->count))
Daniel Lezcanod3154922008-09-08 13:17:27 -0700279 continue;
280
Eric W. Biederman575f4cd2009-12-03 02:29:08 +0000281 if (unlikely(!atomic_inc_not_zero(&tw->tw_refcnt)))
282 continue;
283
Eric W. Biedermanb099ce22009-12-03 02:29:09 +0000284 if (unlikely((tw->tw_family != family) ||
285 atomic_read(&twsk_net(tw)->count))) {
Eric W. Biederman575f4cd2009-12-03 02:29:08 +0000286 inet_twsk_put(tw);
287 goto restart;
288 }
289
290 rcu_read_unlock();
Eric Dumazet91035f02011-02-18 22:35:56 +0000291 local_bh_disable();
Eric Dumazetdbe7faa2015-07-08 14:28:30 -0700292 inet_twsk_deschedule_put(tw);
Eric Dumazet91035f02011-02-18 22:35:56 +0000293 local_bh_enable();
Eric W. Biederman575f4cd2009-12-03 02:29:08 +0000294 goto restart_rcu;
Daniel Lezcanod3154922008-09-08 13:17:27 -0700295 }
Eric W. Biederman575f4cd2009-12-03 02:29:08 +0000296 /* If the nulls value we got at the end of this lookup is
297 * not the expected one, we must restart lookup.
298 * We probably met an item that was moved to another chain.
299 */
300 if (get_nulls_value(node) != slot)
301 goto restart;
302 rcu_read_unlock();
Daniel Lezcanod3154922008-09-08 13:17:27 -0700303 }
Daniel Lezcanod3154922008-09-08 13:17:27 -0700304}
305EXPORT_SYMBOL_GPL(inet_twsk_purge);