blob: 8554d0ea1719b7621667026a1641cb7da20ca2d4 [file] [log] [blame]
Arnaldo Carvalho de Meloe48c4142005-08-09 20:09:46 -07001/*
2 * INET An implementation of the TCP/IP protocol suite for the LINUX
3 * operating system. INET is implemented using the BSD Socket
4 * interface as the means of communication with the user level.
5 *
6 * Generic TIME_WAIT sockets functions
7 *
8 * From code orinally in TCP
9 */
10
Ilpo Järvinen172589c2007-08-28 15:50:33 -070011#include <linux/kernel.h>
Arnaldo Carvalho de Meloe48c4142005-08-09 20:09:46 -070012#include <net/inet_hashtables.h>
13#include <net/inet_timewait_sock.h>
Arnaldo Carvalho de Melo696ab2d2005-08-09 20:45:03 -070014#include <net/ip.h>
Arnaldo Carvalho de Meloe48c4142005-08-09 20:09:46 -070015
16/* Must be called with locally disabled BHs. */
Adrian Bunkacd159b2007-07-14 19:00:59 -070017static void __inet_twsk_kill(struct inet_timewait_sock *tw,
18 struct inet_hashinfo *hashinfo)
Arnaldo Carvalho de Meloe48c4142005-08-09 20:09:46 -070019{
20 struct inet_bind_hashbucket *bhead;
21 struct inet_bind_bucket *tb;
22 /* Unlink from established hashes. */
Eric Dumazet9db66bd2008-11-20 20:39:09 -080023 spinlock_t *lock = inet_ehash_lockp(hashinfo, tw->tw_hash);
Arnaldo Carvalho de Meloe48c4142005-08-09 20:09:46 -070024
Eric Dumazet9db66bd2008-11-20 20:39:09 -080025 spin_lock(lock);
Eric Dumazet3ab5aee2008-11-16 19:40:17 -080026 if (hlist_nulls_unhashed(&tw->tw_node)) {
Eric Dumazet9db66bd2008-11-20 20:39:09 -080027 spin_unlock(lock);
Arnaldo Carvalho de Meloe48c4142005-08-09 20:09:46 -070028 return;
29 }
Eric Dumazet3ab5aee2008-11-16 19:40:17 -080030 hlist_nulls_del_rcu(&tw->tw_node);
31 sk_nulls_node_init(&tw->tw_node);
Eric Dumazet9db66bd2008-11-20 20:39:09 -080032 spin_unlock(lock);
Arnaldo Carvalho de Meloe48c4142005-08-09 20:09:46 -070033
34 /* Disassociate with bind bucket. */
Pavel Emelyanov7f635ab2008-06-16 17:12:49 -070035 bhead = &hashinfo->bhash[inet_bhashfn(twsk_net(tw), tw->tw_num,
36 hashinfo->bhash_size)];
Arnaldo Carvalho de Meloe48c4142005-08-09 20:09:46 -070037 spin_lock(&bhead->lock);
38 tb = tw->tw_tb;
39 __hlist_del(&tw->tw_bind_node);
40 tw->tw_tb = NULL;
41 inet_bind_bucket_destroy(hashinfo->bind_bucket_cachep, tb);
42 spin_unlock(&bhead->lock);
43#ifdef SOCK_REFCNT_DEBUG
44 if (atomic_read(&tw->tw_refcnt) != 1) {
45 printk(KERN_DEBUG "%s timewait_sock %p refcnt=%d\n",
46 tw->tw_prot->name, tw, atomic_read(&tw->tw_refcnt));
47 }
48#endif
49 inet_twsk_put(tw);
50}
51
Pavel Emelyanov7054fb92007-12-20 15:32:54 -080052void inet_twsk_put(struct inet_timewait_sock *tw)
53{
54 if (atomic_dec_and_test(&tw->tw_refcnt)) {
55 struct module *owner = tw->tw_prot->owner;
56 twsk_destructor((struct sock *)tw);
57#ifdef SOCK_REFCNT_DEBUG
58 printk(KERN_DEBUG "%s timewait_sock %p released\n",
59 tw->tw_prot->name, tw);
60#endif
Denis V. Lunevcd5342d2008-04-16 02:00:28 -070061 release_net(twsk_net(tw));
Pavel Emelyanov7054fb92007-12-20 15:32:54 -080062 kmem_cache_free(tw->tw_prot->twsk_prot->twsk_slab, tw);
63 module_put(owner);
64 }
65}
66EXPORT_SYMBOL_GPL(inet_twsk_put);
67
Arnaldo Carvalho de Meloe48c4142005-08-09 20:09:46 -070068/*
69 * Enter the time wait state. This is called with locally disabled BH.
70 * Essentially we whip up a timewait bucket, copy the relevant info into it
71 * from the SK, and mess with hash chains and list linkage.
72 */
73void __inet_twsk_hashdance(struct inet_timewait_sock *tw, struct sock *sk,
74 struct inet_hashinfo *hashinfo)
75{
76 const struct inet_sock *inet = inet_sk(sk);
Arnaldo Carvalho de Melo463c84b2005-08-09 20:10:42 -070077 const struct inet_connection_sock *icsk = inet_csk(sk);
Eric Dumazet81c3d542005-10-03 14:13:38 -070078 struct inet_ehash_bucket *ehead = inet_ehash_bucket(hashinfo, sk->sk_hash);
Eric Dumazet9db66bd2008-11-20 20:39:09 -080079 spinlock_t *lock = inet_ehash_lockp(hashinfo, sk->sk_hash);
Arnaldo Carvalho de Meloe48c4142005-08-09 20:09:46 -070080 struct inet_bind_hashbucket *bhead;
81 /* Step 1: Put TW into bind hash. Original socket stays there too.
82 Note, that any socket with inet->num != 0 MUST be bound in
83 binding cache, even if it is closed.
84 */
Pavel Emelyanov7f635ab2008-06-16 17:12:49 -070085 bhead = &hashinfo->bhash[inet_bhashfn(twsk_net(tw), inet->num,
86 hashinfo->bhash_size)];
Arnaldo Carvalho de Meloe48c4142005-08-09 20:09:46 -070087 spin_lock(&bhead->lock);
Arnaldo Carvalho de Melo463c84b2005-08-09 20:10:42 -070088 tw->tw_tb = icsk->icsk_bind_hash;
Ilpo Järvinen547b7922008-07-25 21:43:18 -070089 WARN_ON(!icsk->icsk_bind_hash);
Arnaldo Carvalho de Meloe48c4142005-08-09 20:09:46 -070090 inet_twsk_add_bind_node(tw, &tw->tw_tb->owners);
91 spin_unlock(&bhead->lock);
92
Eric Dumazet9db66bd2008-11-20 20:39:09 -080093 spin_lock(lock);
Arnaldo Carvalho de Meloe48c4142005-08-09 20:09:46 -070094
Eric Dumazet3ab5aee2008-11-16 19:40:17 -080095 /*
96 * Step 2: Hash TW into TIMEWAIT chain.
97 * Should be done before removing sk from established chain
98 * because readers are lockless and search established first.
99 */
Arnaldo Carvalho de Meloe48c4142005-08-09 20:09:46 -0700100 atomic_inc(&tw->tw_refcnt);
Eric Dumazet3ab5aee2008-11-16 19:40:17 -0800101 inet_twsk_add_node_rcu(tw, &ehead->twchain);
102
103 /* Step 3: Remove SK from established hash. */
104 if (__sk_nulls_del_node_init_rcu(sk))
105 sock_prot_inuse_add(sock_net(sk), sk->sk_prot, -1);
Arnaldo Carvalho de Meloe48c4142005-08-09 20:09:46 -0700106
Eric Dumazet9db66bd2008-11-20 20:39:09 -0800107 spin_unlock(lock);
Arnaldo Carvalho de Meloe48c4142005-08-09 20:09:46 -0700108}
Arnaldo Carvalho de Meloc6762702005-08-09 20:09:59 -0700109
Arnaldo Carvalho de Melo696ab2d2005-08-09 20:45:03 -0700110EXPORT_SYMBOL_GPL(__inet_twsk_hashdance);
111
Arnaldo Carvalho de Meloc6762702005-08-09 20:09:59 -0700112struct inet_timewait_sock *inet_twsk_alloc(const struct sock *sk, const int state)
113{
Arnaldo Carvalho de Melo6d6ee432005-12-13 23:25:19 -0800114 struct inet_timewait_sock *tw =
115 kmem_cache_alloc(sk->sk_prot_creator->twsk_prot->twsk_slab,
Christoph Lameter54e6ecb2006-12-06 20:33:16 -0800116 GFP_ATOMIC);
Arnaldo Carvalho de Meloc6762702005-08-09 20:09:59 -0700117 if (tw != NULL) {
118 const struct inet_sock *inet = inet_sk(sk);
119
120 /* Give us an identity. */
121 tw->tw_daddr = inet->daddr;
122 tw->tw_rcv_saddr = inet->rcv_saddr;
123 tw->tw_bound_dev_if = sk->sk_bound_dev_if;
124 tw->tw_num = inet->num;
125 tw->tw_state = TCP_TIME_WAIT;
126 tw->tw_substate = state;
127 tw->tw_sport = inet->sport;
128 tw->tw_dport = inet->dport;
129 tw->tw_family = sk->sk_family;
130 tw->tw_reuse = sk->sk_reuse;
Eric Dumazet81c3d542005-10-03 14:13:38 -0700131 tw->tw_hash = sk->sk_hash;
Arnaldo Carvalho de Meloc6762702005-08-09 20:09:59 -0700132 tw->tw_ipv6only = 0;
KOVACS Krisztianf5715ae2008-10-01 07:30:02 -0700133 tw->tw_transparent = inet->transparent;
Arnaldo Carvalho de Meloc6762702005-08-09 20:09:59 -0700134 tw->tw_prot = sk->sk_prot_creator;
Denis V. Lunevcd5342d2008-04-16 02:00:28 -0700135 twsk_net_set(tw, hold_net(sock_net(sk)));
Arnaldo Carvalho de Meloc6762702005-08-09 20:09:59 -0700136 atomic_set(&tw->tw_refcnt, 1);
137 inet_twsk_dead_node_init(tw);
Arnaldo Carvalho de Meloeeb2b852005-10-10 21:25:23 -0700138 __module_get(tw->tw_prot->owner);
Arnaldo Carvalho de Meloc6762702005-08-09 20:09:59 -0700139 }
140
141 return tw;
142}
Arnaldo Carvalho de Melo696ab2d2005-08-09 20:45:03 -0700143
144EXPORT_SYMBOL_GPL(inet_twsk_alloc);
145
146/* Returns non-zero if quota exceeded. */
147static int inet_twdr_do_twkill_work(struct inet_timewait_death_row *twdr,
148 const int slot)
149{
150 struct inet_timewait_sock *tw;
151 struct hlist_node *node;
152 unsigned int killed;
153 int ret;
154
155 /* NOTE: compare this to previous version where lock
156 * was released after detaching chain. It was racy,
157 * because tw buckets are scheduled in not serialized context
158 * in 2.3 (with netfilter), and with softnet it is common, because
159 * soft irqs are not sequenced.
160 */
161 killed = 0;
162 ret = 0;
163rescan:
164 inet_twsk_for_each_inmate(tw, node, &twdr->cells[slot]) {
165 __inet_twsk_del_dead_node(tw);
166 spin_unlock(&twdr->death_lock);
167 __inet_twsk_kill(tw, twdr->hashinfo);
Pavel Emelyanovf2bf4152008-07-16 20:32:25 -0700168#ifdef CONFIG_NET_NS
169 NET_INC_STATS_BH(twsk_net(tw), LINUX_MIB_TIMEWAITED);
170#endif
Arnaldo Carvalho de Melo696ab2d2005-08-09 20:45:03 -0700171 inet_twsk_put(tw);
172 killed++;
173 spin_lock(&twdr->death_lock);
174 if (killed > INET_TWDR_TWKILL_QUOTA) {
175 ret = 1;
176 break;
177 }
178
179 /* While we dropped twdr->death_lock, another cpu may have
180 * killed off the next TW bucket in the list, therefore
181 * do a fresh re-read of the hlist head node with the
182 * lock reacquired. We still use the hlist traversal
183 * macro in order to get the prefetches.
184 */
185 goto rescan;
186 }
187
188 twdr->tw_count -= killed;
Pavel Emelyanovf2bf4152008-07-16 20:32:25 -0700189#ifndef CONFIG_NET_NS
190 NET_ADD_STATS_BH(&init_net, LINUX_MIB_TIMEWAITED, killed);
191#endif
Arnaldo Carvalho de Melo696ab2d2005-08-09 20:45:03 -0700192 return ret;
193}
194
195void inet_twdr_hangman(unsigned long data)
196{
197 struct inet_timewait_death_row *twdr;
198 int unsigned need_timer;
199
200 twdr = (struct inet_timewait_death_row *)data;
201 spin_lock(&twdr->death_lock);
202
203 if (twdr->tw_count == 0)
204 goto out;
205
206 need_timer = 0;
207 if (inet_twdr_do_twkill_work(twdr, twdr->slot)) {
208 twdr->thread_slots |= (1 << twdr->slot);
Arnaldo Carvalho de Melo696ab2d2005-08-09 20:45:03 -0700209 schedule_work(&twdr->twkill_work);
210 need_timer = 1;
211 } else {
212 /* We purged the entire slot, anything left? */
213 if (twdr->tw_count)
214 need_timer = 1;
215 }
216 twdr->slot = ((twdr->slot + 1) & (INET_TWDR_TWKILL_SLOTS - 1));
217 if (need_timer)
218 mod_timer(&twdr->tw_timer, jiffies + twdr->period);
219out:
220 spin_unlock(&twdr->death_lock);
221}
222
223EXPORT_SYMBOL_GPL(inet_twdr_hangman);
224
David Howells65f27f32006-11-22 14:55:48 +0000225void inet_twdr_twkill_work(struct work_struct *work)
Arnaldo Carvalho de Melo696ab2d2005-08-09 20:45:03 -0700226{
David Howells65f27f32006-11-22 14:55:48 +0000227 struct inet_timewait_death_row *twdr =
228 container_of(work, struct inet_timewait_death_row, twkill_work);
Arnaldo Carvalho de Melo696ab2d2005-08-09 20:45:03 -0700229 int i;
230
Pavel Emelyanov95c93822007-12-11 02:12:36 -0800231 BUILD_BUG_ON((INET_TWDR_TWKILL_SLOTS - 1) >
232 (sizeof(twdr->thread_slots) * 8));
Arnaldo Carvalho de Melo696ab2d2005-08-09 20:45:03 -0700233
234 while (twdr->thread_slots) {
235 spin_lock_bh(&twdr->death_lock);
236 for (i = 0; i < INET_TWDR_TWKILL_SLOTS; i++) {
237 if (!(twdr->thread_slots & (1 << i)))
238 continue;
239
240 while (inet_twdr_do_twkill_work(twdr, i) != 0) {
241 if (need_resched()) {
242 spin_unlock_bh(&twdr->death_lock);
243 schedule();
244 spin_lock_bh(&twdr->death_lock);
245 }
246 }
247
248 twdr->thread_slots &= ~(1 << i);
249 }
250 spin_unlock_bh(&twdr->death_lock);
251 }
252}
253
254EXPORT_SYMBOL_GPL(inet_twdr_twkill_work);
255
256/* These are always called from BH context. See callers in
257 * tcp_input.c to verify this.
258 */
259
260/* This is for handling early-kills of TIME_WAIT sockets. */
261void inet_twsk_deschedule(struct inet_timewait_sock *tw,
262 struct inet_timewait_death_row *twdr)
263{
264 spin_lock(&twdr->death_lock);
265 if (inet_twsk_del_dead_node(tw)) {
266 inet_twsk_put(tw);
267 if (--twdr->tw_count == 0)
268 del_timer(&twdr->tw_timer);
269 }
270 spin_unlock(&twdr->death_lock);
271 __inet_twsk_kill(tw, twdr->hashinfo);
272}
273
274EXPORT_SYMBOL(inet_twsk_deschedule);
275
276void inet_twsk_schedule(struct inet_timewait_sock *tw,
277 struct inet_timewait_death_row *twdr,
278 const int timeo, const int timewait_len)
279{
280 struct hlist_head *list;
281 int slot;
282
283 /* timeout := RTO * 3.5
284 *
285 * 3.5 = 1+2+0.5 to wait for two retransmits.
286 *
287 * RATIONALE: if FIN arrived and we entered TIME-WAIT state,
288 * our ACK acking that FIN can be lost. If N subsequent retransmitted
289 * FINs (or previous seqments) are lost (probability of such event
290 * is p^(N+1), where p is probability to lose single packet and
291 * time to detect the loss is about RTO*(2^N - 1) with exponential
292 * backoff). Normal timewait length is calculated so, that we
293 * waited at least for one retransmitted FIN (maximal RTO is 120sec).
294 * [ BTW Linux. following BSD, violates this requirement waiting
295 * only for 60sec, we should wait at least for 240 secs.
296 * Well, 240 consumes too much of resources 8)
297 * ]
298 * This interval is not reduced to catch old duplicate and
299 * responces to our wandering segments living for two MSLs.
300 * However, if we use PAWS to detect
301 * old duplicates, we can reduce the interval to bounds required
302 * by RTO, rather than MSL. So, if peer understands PAWS, we
303 * kill tw bucket after 3.5*RTO (it is important that this number
304 * is greater than TS tick!) and detect old duplicates with help
305 * of PAWS.
306 */
307 slot = (timeo + (1 << INET_TWDR_RECYCLE_TICK) - 1) >> INET_TWDR_RECYCLE_TICK;
308
309 spin_lock(&twdr->death_lock);
310
311 /* Unlink it, if it was scheduled */
312 if (inet_twsk_del_dead_node(tw))
313 twdr->tw_count--;
314 else
315 atomic_inc(&tw->tw_refcnt);
316
317 if (slot >= INET_TWDR_RECYCLE_SLOTS) {
318 /* Schedule to slow timer */
319 if (timeo >= timewait_len) {
320 slot = INET_TWDR_TWKILL_SLOTS - 1;
321 } else {
Ilpo Järvinen172589c2007-08-28 15:50:33 -0700322 slot = DIV_ROUND_UP(timeo, twdr->period);
Arnaldo Carvalho de Melo696ab2d2005-08-09 20:45:03 -0700323 if (slot >= INET_TWDR_TWKILL_SLOTS)
324 slot = INET_TWDR_TWKILL_SLOTS - 1;
325 }
326 tw->tw_ttd = jiffies + timeo;
327 slot = (twdr->slot + slot) & (INET_TWDR_TWKILL_SLOTS - 1);
328 list = &twdr->cells[slot];
329 } else {
330 tw->tw_ttd = jiffies + (slot << INET_TWDR_RECYCLE_TICK);
331
332 if (twdr->twcal_hand < 0) {
333 twdr->twcal_hand = 0;
334 twdr->twcal_jiffie = jiffies;
335 twdr->twcal_timer.expires = twdr->twcal_jiffie +
336 (slot << INET_TWDR_RECYCLE_TICK);
337 add_timer(&twdr->twcal_timer);
338 } else {
339 if (time_after(twdr->twcal_timer.expires,
340 jiffies + (slot << INET_TWDR_RECYCLE_TICK)))
341 mod_timer(&twdr->twcal_timer,
342 jiffies + (slot << INET_TWDR_RECYCLE_TICK));
343 slot = (twdr->twcal_hand + slot) & (INET_TWDR_RECYCLE_SLOTS - 1);
344 }
345 list = &twdr->twcal_row[slot];
346 }
347
348 hlist_add_head(&tw->tw_death_node, list);
349
350 if (twdr->tw_count++ == 0)
351 mod_timer(&twdr->tw_timer, jiffies + twdr->period);
352 spin_unlock(&twdr->death_lock);
353}
354
355EXPORT_SYMBOL_GPL(inet_twsk_schedule);
356
357void inet_twdr_twcal_tick(unsigned long data)
358{
359 struct inet_timewait_death_row *twdr;
360 int n, slot;
361 unsigned long j;
362 unsigned long now = jiffies;
363 int killed = 0;
364 int adv = 0;
365
366 twdr = (struct inet_timewait_death_row *)data;
367
368 spin_lock(&twdr->death_lock);
369 if (twdr->twcal_hand < 0)
370 goto out;
371
372 slot = twdr->twcal_hand;
373 j = twdr->twcal_jiffie;
374
375 for (n = 0; n < INET_TWDR_RECYCLE_SLOTS; n++) {
376 if (time_before_eq(j, now)) {
377 struct hlist_node *node, *safe;
378 struct inet_timewait_sock *tw;
379
380 inet_twsk_for_each_inmate_safe(tw, node, safe,
381 &twdr->twcal_row[slot]) {
382 __inet_twsk_del_dead_node(tw);
383 __inet_twsk_kill(tw, twdr->hashinfo);
Pavel Emelyanovf2bf4152008-07-16 20:32:25 -0700384#ifdef CONFIG_NET_NS
385 NET_INC_STATS_BH(twsk_net(tw), LINUX_MIB_TIMEWAITKILLED);
386#endif
Arnaldo Carvalho de Melo696ab2d2005-08-09 20:45:03 -0700387 inet_twsk_put(tw);
388 killed++;
389 }
390 } else {
391 if (!adv) {
392 adv = 1;
393 twdr->twcal_jiffie = j;
394 twdr->twcal_hand = slot;
395 }
396
397 if (!hlist_empty(&twdr->twcal_row[slot])) {
398 mod_timer(&twdr->twcal_timer, j);
399 goto out;
400 }
401 }
402 j += 1 << INET_TWDR_RECYCLE_TICK;
403 slot = (slot + 1) & (INET_TWDR_RECYCLE_SLOTS - 1);
404 }
405 twdr->twcal_hand = -1;
406
407out:
408 if ((twdr->tw_count -= killed) == 0)
409 del_timer(&twdr->tw_timer);
Pavel Emelyanovf2bf4152008-07-16 20:32:25 -0700410#ifndef CONFIG_NET_NS
411 NET_ADD_STATS_BH(&init_net, LINUX_MIB_TIMEWAITKILLED, killed);
412#endif
Arnaldo Carvalho de Melo696ab2d2005-08-09 20:45:03 -0700413 spin_unlock(&twdr->death_lock);
414}
415
416EXPORT_SYMBOL_GPL(inet_twdr_twcal_tick);
Daniel Lezcanod3154922008-09-08 13:17:27 -0700417
418void inet_twsk_purge(struct net *net, struct inet_hashinfo *hashinfo,
419 struct inet_timewait_death_row *twdr, int family)
420{
421 struct inet_timewait_sock *tw;
422 struct sock *sk;
Eric Dumazet3ab5aee2008-11-16 19:40:17 -0800423 struct hlist_nulls_node *node;
Daniel Lezcanod3154922008-09-08 13:17:27 -0700424 int h;
425
426 local_bh_disable();
427 for (h = 0; h < (hashinfo->ehash_size); h++) {
428 struct inet_ehash_bucket *head =
429 inet_ehash_bucket(hashinfo, h);
Eric Dumazet9db66bd2008-11-20 20:39:09 -0800430 spinlock_t *lock = inet_ehash_lockp(hashinfo, h);
Daniel Lezcanod3154922008-09-08 13:17:27 -0700431restart:
Eric Dumazet9db66bd2008-11-20 20:39:09 -0800432 spin_lock(lock);
Eric Dumazet3ab5aee2008-11-16 19:40:17 -0800433 sk_nulls_for_each(sk, node, &head->twchain) {
Daniel Lezcanod3154922008-09-08 13:17:27 -0700434
435 tw = inet_twsk(sk);
436 if (!net_eq(twsk_net(tw), net) ||
437 tw->tw_family != family)
438 continue;
439
440 atomic_inc(&tw->tw_refcnt);
Eric Dumazet9db66bd2008-11-20 20:39:09 -0800441 spin_unlock(lock);
Daniel Lezcanod3154922008-09-08 13:17:27 -0700442 inet_twsk_deschedule(tw, twdr);
443 inet_twsk_put(tw);
444
445 goto restart;
446 }
Eric Dumazet9db66bd2008-11-20 20:39:09 -0800447 spin_unlock(lock);
Daniel Lezcanod3154922008-09-08 13:17:27 -0700448 }
449 local_bh_enable();
450}
451EXPORT_SYMBOL_GPL(inet_twsk_purge);