blob: afd9120c2fc41b9fbf0ac2caba469eff39b95e87 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * linux/kernel/softirq.c
3 *
4 * Copyright (C) 1992 Linus Torvalds
5 *
Pavel Machekb10db7f2008-01-30 13:30:00 +01006 * Distribute under GPLv2.
7 *
8 * Rewritten. Old one was good in 2.2, but in 2.3 it was immoral. --ANK (990903)
Linus Torvalds1da177e2005-04-16 15:20:36 -07009 */
10
11#include <linux/module.h>
12#include <linux/kernel_stat.h>
13#include <linux/interrupt.h>
14#include <linux/init.h>
15#include <linux/mm.h>
16#include <linux/notifier.h>
17#include <linux/percpu.h>
18#include <linux/cpu.h>
Rafael J. Wysocki83144182007-07-17 04:03:35 -070019#include <linux/freezer.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070020#include <linux/kthread.h>
21#include <linux/rcupdate.h>
Andrew Morton78eef012006-03-22 00:08:16 -080022#include <linux/smp.h>
Thomas Gleixner79bf2bb2007-02-16 01:28:03 -080023#include <linux/tick.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070024
25#include <asm/irq.h>
26/*
27 - No shared variables, all the data are CPU local.
28 - If a softirq needs serialization, let it serialize itself
29 by its own spinlocks.
30 - Even if softirq is serialized, only local cpu is marked for
31 execution. Hence, we get something sort of weak cpu binding.
32 Though it is still not clear, will it result in better locality
33 or will not.
34
35 Examples:
36 - NET RX softirq. It is multithreaded and does not require
37 any global serialization.
38 - NET TX softirq. It kicks software netdevice queues, hence
39 it is logically serialized per device, but this serialization
40 is invisible to common code.
41 - Tasklets: serialized wrt itself.
42 */
43
44#ifndef __ARCH_IRQ_STAT
45irq_cpustat_t irq_stat[NR_CPUS] ____cacheline_aligned;
46EXPORT_SYMBOL(irq_stat);
47#endif
48
49static struct softirq_action softirq_vec[32] __cacheline_aligned_in_smp;
50
51static DEFINE_PER_CPU(struct task_struct *, ksoftirqd);
52
53/*
54 * we cannot loop indefinitely here to avoid userspace starvation,
55 * but we also don't want to introduce a worst case 1/HZ latency
56 * to the pending events, so lets the scheduler to balance
57 * the softirq load for us.
58 */
59static inline void wakeup_softirqd(void)
60{
61 /* Interrupts are disabled: no need to stop preemption */
62 struct task_struct *tsk = __get_cpu_var(ksoftirqd);
63
64 if (tsk && tsk->state != TASK_RUNNING)
65 wake_up_process(tsk);
66}
67
68/*
Ingo Molnarde30a2b2006-07-03 00:24:42 -070069 * This one is for softirq.c-internal use,
70 * where hardirqs are disabled legitimately:
71 */
Tim Chen3c829c32006-07-30 03:04:02 -070072#ifdef CONFIG_TRACE_IRQFLAGS
Ingo Molnarde30a2b2006-07-03 00:24:42 -070073static void __local_bh_disable(unsigned long ip)
74{
75 unsigned long flags;
76
77 WARN_ON_ONCE(in_irq());
78
79 raw_local_irq_save(flags);
80 add_preempt_count(SOFTIRQ_OFFSET);
81 /*
82 * Were softirqs turned off above:
83 */
84 if (softirq_count() == SOFTIRQ_OFFSET)
85 trace_softirqs_off(ip);
86 raw_local_irq_restore(flags);
87}
Tim Chen3c829c32006-07-30 03:04:02 -070088#else /* !CONFIG_TRACE_IRQFLAGS */
89static inline void __local_bh_disable(unsigned long ip)
90{
91 add_preempt_count(SOFTIRQ_OFFSET);
92 barrier();
93}
94#endif /* CONFIG_TRACE_IRQFLAGS */
Ingo Molnarde30a2b2006-07-03 00:24:42 -070095
96void local_bh_disable(void)
97{
98 __local_bh_disable((unsigned long)__builtin_return_address(0));
99}
100
101EXPORT_SYMBOL(local_bh_disable);
102
103void __local_bh_enable(void)
104{
105 WARN_ON_ONCE(in_irq());
106
107 /*
108 * softirqs should never be enabled by __local_bh_enable(),
109 * it always nests inside local_bh_enable() sections:
110 */
111 WARN_ON_ONCE(softirq_count() == SOFTIRQ_OFFSET);
112
113 sub_preempt_count(SOFTIRQ_OFFSET);
114}
115EXPORT_SYMBOL_GPL(__local_bh_enable);
116
117/*
118 * Special-case - softirqs can safely be enabled in
119 * cond_resched_softirq(), or by __do_softirq(),
120 * without processing still-pending softirqs:
121 */
122void _local_bh_enable(void)
123{
124 WARN_ON_ONCE(in_irq());
125 WARN_ON_ONCE(!irqs_disabled());
126
127 if (softirq_count() == SOFTIRQ_OFFSET)
128 trace_softirqs_on((unsigned long)__builtin_return_address(0));
129 sub_preempt_count(SOFTIRQ_OFFSET);
130}
131
132EXPORT_SYMBOL(_local_bh_enable);
133
134void local_bh_enable(void)
135{
Tim Chen3c829c32006-07-30 03:04:02 -0700136#ifdef CONFIG_TRACE_IRQFLAGS
Ingo Molnarde30a2b2006-07-03 00:24:42 -0700137 unsigned long flags;
138
139 WARN_ON_ONCE(in_irq());
Tim Chen3c829c32006-07-30 03:04:02 -0700140#endif
Ingo Molnarde30a2b2006-07-03 00:24:42 -0700141 WARN_ON_ONCE(irqs_disabled());
142
Tim Chen3c829c32006-07-30 03:04:02 -0700143#ifdef CONFIG_TRACE_IRQFLAGS
Ingo Molnarde30a2b2006-07-03 00:24:42 -0700144 local_irq_save(flags);
Tim Chen3c829c32006-07-30 03:04:02 -0700145#endif
Ingo Molnarde30a2b2006-07-03 00:24:42 -0700146 /*
147 * Are softirqs going to be turned on now:
148 */
149 if (softirq_count() == SOFTIRQ_OFFSET)
150 trace_softirqs_on((unsigned long)__builtin_return_address(0));
151 /*
152 * Keep preemption disabled until we are done with
153 * softirq processing:
154 */
155 sub_preempt_count(SOFTIRQ_OFFSET - 1);
156
157 if (unlikely(!in_interrupt() && local_softirq_pending()))
158 do_softirq();
159
160 dec_preempt_count();
Tim Chen3c829c32006-07-30 03:04:02 -0700161#ifdef CONFIG_TRACE_IRQFLAGS
Ingo Molnarde30a2b2006-07-03 00:24:42 -0700162 local_irq_restore(flags);
Tim Chen3c829c32006-07-30 03:04:02 -0700163#endif
Ingo Molnarde30a2b2006-07-03 00:24:42 -0700164 preempt_check_resched();
165}
166EXPORT_SYMBOL(local_bh_enable);
167
168void local_bh_enable_ip(unsigned long ip)
169{
Tim Chen3c829c32006-07-30 03:04:02 -0700170#ifdef CONFIG_TRACE_IRQFLAGS
Ingo Molnarde30a2b2006-07-03 00:24:42 -0700171 unsigned long flags;
172
173 WARN_ON_ONCE(in_irq());
174
175 local_irq_save(flags);
Tim Chen3c829c32006-07-30 03:04:02 -0700176#endif
Ingo Molnarde30a2b2006-07-03 00:24:42 -0700177 /*
178 * Are softirqs going to be turned on now:
179 */
180 if (softirq_count() == SOFTIRQ_OFFSET)
181 trace_softirqs_on(ip);
182 /*
183 * Keep preemption disabled until we are done with
184 * softirq processing:
185 */
186 sub_preempt_count(SOFTIRQ_OFFSET - 1);
187
188 if (unlikely(!in_interrupt() && local_softirq_pending()))
189 do_softirq();
190
191 dec_preempt_count();
Tim Chen3c829c32006-07-30 03:04:02 -0700192#ifdef CONFIG_TRACE_IRQFLAGS
Ingo Molnarde30a2b2006-07-03 00:24:42 -0700193 local_irq_restore(flags);
Tim Chen3c829c32006-07-30 03:04:02 -0700194#endif
Ingo Molnarde30a2b2006-07-03 00:24:42 -0700195 preempt_check_resched();
196}
197EXPORT_SYMBOL(local_bh_enable_ip);
198
199/*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700200 * We restart softirq processing MAX_SOFTIRQ_RESTART times,
201 * and we fall back to softirqd after that.
202 *
203 * This number has been established via experimentation.
204 * The two things to balance is latency against fairness -
205 * we want to handle softirqs as soon as possible, but they
206 * should not be able to lock up the box.
207 */
208#define MAX_SOFTIRQ_RESTART 10
209
210asmlinkage void __do_softirq(void)
211{
212 struct softirq_action *h;
213 __u32 pending;
214 int max_restart = MAX_SOFTIRQ_RESTART;
215 int cpu;
216
217 pending = local_softirq_pending();
Paul Mackerras829035fd2006-07-03 00:25:40 -0700218 account_system_vtime(current);
219
Ingo Molnarde30a2b2006-07-03 00:24:42 -0700220 __local_bh_disable((unsigned long)__builtin_return_address(0));
221 trace_softirq_enter();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700222
Linus Torvalds1da177e2005-04-16 15:20:36 -0700223 cpu = smp_processor_id();
224restart:
225 /* Reset the pending bitmask before enabling irqs */
Andi Kleen3f744782005-09-12 18:49:24 +0200226 set_softirq_pending(0);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700227
Andrew Mortonc70f5d62005-07-30 10:22:49 -0700228 local_irq_enable();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700229
230 h = softirq_vec;
231
232 do {
233 if (pending & 1) {
234 h->action(h);
235 rcu_bh_qsctr_inc(cpu);
236 }
237 h++;
238 pending >>= 1;
239 } while (pending);
240
Andrew Mortonc70f5d62005-07-30 10:22:49 -0700241 local_irq_disable();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700242
243 pending = local_softirq_pending();
244 if (pending && --max_restart)
245 goto restart;
246
247 if (pending)
248 wakeup_softirqd();
249
Ingo Molnarde30a2b2006-07-03 00:24:42 -0700250 trace_softirq_exit();
Paul Mackerras829035fd2006-07-03 00:25:40 -0700251
252 account_system_vtime(current);
Ingo Molnarde30a2b2006-07-03 00:24:42 -0700253 _local_bh_enable();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700254}
255
256#ifndef __ARCH_HAS_DO_SOFTIRQ
257
258asmlinkage void do_softirq(void)
259{
260 __u32 pending;
261 unsigned long flags;
262
263 if (in_interrupt())
264 return;
265
266 local_irq_save(flags);
267
268 pending = local_softirq_pending();
269
270 if (pending)
271 __do_softirq();
272
273 local_irq_restore(flags);
274}
275
Linus Torvalds1da177e2005-04-16 15:20:36 -0700276#endif
277
Ingo Molnardde4b2b2007-02-16 01:27:45 -0800278/*
279 * Enter an interrupt context.
280 */
281void irq_enter(void)
282{
Venki Pallipadi6378ddb2008-01-30 13:30:04 +0100283#ifdef CONFIG_NO_HZ
284 int cpu = smp_processor_id();
285 if (idle_cpu(cpu) && !in_interrupt())
286 tick_nohz_stop_idle(cpu);
287#endif
Thomas Gleixner79bf2bb2007-02-16 01:28:03 -0800288 __irq_enter();
289#ifdef CONFIG_NO_HZ
Venki Pallipadi6378ddb2008-01-30 13:30:04 +0100290 if (idle_cpu(cpu))
Thomas Gleixner79bf2bb2007-02-16 01:28:03 -0800291 tick_nohz_update_jiffies();
292#endif
Ingo Molnardde4b2b2007-02-16 01:27:45 -0800293}
294
Linus Torvalds1da177e2005-04-16 15:20:36 -0700295#ifdef __ARCH_IRQ_EXIT_IRQS_DISABLED
296# define invoke_softirq() __do_softirq()
297#else
298# define invoke_softirq() do_softirq()
299#endif
300
301/*
302 * Exit an interrupt context. Process softirqs if needed and possible:
303 */
304void irq_exit(void)
305{
306 account_system_vtime(current);
Ingo Molnarde30a2b2006-07-03 00:24:42 -0700307 trace_hardirq_exit();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700308 sub_preempt_count(IRQ_EXIT_OFFSET);
309 if (!in_interrupt() && local_softirq_pending())
310 invoke_softirq();
Thomas Gleixner79bf2bb2007-02-16 01:28:03 -0800311
312#ifdef CONFIG_NO_HZ
313 /* Make sure that timer wheel updates are propagated */
314 if (!in_interrupt() && idle_cpu(smp_processor_id()) && !need_resched())
315 tick_nohz_stop_sched_tick();
Steven Rostedt2232c2d2008-02-29 18:46:50 +0100316 rcu_irq_exit();
Thomas Gleixner79bf2bb2007-02-16 01:28:03 -0800317#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -0700318 preempt_enable_no_resched();
319}
320
321/*
322 * This function must run with irqs disabled!
323 */
Harvey Harrison7ad5b3a2008-02-08 04:19:53 -0800324inline void raise_softirq_irqoff(unsigned int nr)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700325{
326 __raise_softirq_irqoff(nr);
327
328 /*
329 * If we're in an interrupt or softirq, we're done
330 * (this also catches softirq-disabled code). We will
331 * actually run the softirq once we return from
332 * the irq or softirq.
333 *
334 * Otherwise we wake up ksoftirqd to make sure we
335 * schedule the softirq soon.
336 */
337 if (!in_interrupt())
338 wakeup_softirqd();
339}
340
Harvey Harrison7ad5b3a2008-02-08 04:19:53 -0800341void raise_softirq(unsigned int nr)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700342{
343 unsigned long flags;
344
345 local_irq_save(flags);
346 raise_softirq_irqoff(nr);
347 local_irq_restore(flags);
348}
349
350void open_softirq(int nr, void (*action)(struct softirq_action*), void *data)
351{
352 softirq_vec[nr].data = data;
353 softirq_vec[nr].action = action;
354}
355
Linus Torvalds1da177e2005-04-16 15:20:36 -0700356/* Tasklets */
357struct tasklet_head
358{
Olof Johansson48f20a92008-03-04 15:23:25 -0800359 struct tasklet_struct *head;
360 struct tasklet_struct **tail;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700361};
362
363/* Some compilers disobey section attribute on statics when not
364 initialized -- RR */
365static DEFINE_PER_CPU(struct tasklet_head, tasklet_vec) = { NULL };
366static DEFINE_PER_CPU(struct tasklet_head, tasklet_hi_vec) = { NULL };
367
Harvey Harrison7ad5b3a2008-02-08 04:19:53 -0800368void __tasklet_schedule(struct tasklet_struct *t)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700369{
370 unsigned long flags;
371
372 local_irq_save(flags);
Olof Johansson48f20a92008-03-04 15:23:25 -0800373 t->next = NULL;
374 *__get_cpu_var(tasklet_vec).tail = t;
375 __get_cpu_var(tasklet_vec).tail = &(t->next);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700376 raise_softirq_irqoff(TASKLET_SOFTIRQ);
377 local_irq_restore(flags);
378}
379
380EXPORT_SYMBOL(__tasklet_schedule);
381
Harvey Harrison7ad5b3a2008-02-08 04:19:53 -0800382void __tasklet_hi_schedule(struct tasklet_struct *t)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700383{
384 unsigned long flags;
385
386 local_irq_save(flags);
Olof Johansson48f20a92008-03-04 15:23:25 -0800387 t->next = NULL;
388 *__get_cpu_var(tasklet_hi_vec).tail = t;
389 __get_cpu_var(tasklet_hi_vec).tail = &(t->next);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700390 raise_softirq_irqoff(HI_SOFTIRQ);
391 local_irq_restore(flags);
392}
393
394EXPORT_SYMBOL(__tasklet_hi_schedule);
395
396static void tasklet_action(struct softirq_action *a)
397{
398 struct tasklet_struct *list;
399
400 local_irq_disable();
Olof Johansson48f20a92008-03-04 15:23:25 -0800401 list = __get_cpu_var(tasklet_vec).head;
402 __get_cpu_var(tasklet_vec).head = NULL;
403 __get_cpu_var(tasklet_vec).tail = &__get_cpu_var(tasklet_vec).head;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700404 local_irq_enable();
405
406 while (list) {
407 struct tasklet_struct *t = list;
408
409 list = list->next;
410
411 if (tasklet_trylock(t)) {
412 if (!atomic_read(&t->count)) {
413 if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
414 BUG();
415 t->func(t->data);
416 tasklet_unlock(t);
417 continue;
418 }
419 tasklet_unlock(t);
420 }
421
422 local_irq_disable();
Olof Johansson48f20a92008-03-04 15:23:25 -0800423 t->next = NULL;
424 *__get_cpu_var(tasklet_vec).tail = t;
425 __get_cpu_var(tasklet_vec).tail = &(t->next);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700426 __raise_softirq_irqoff(TASKLET_SOFTIRQ);
427 local_irq_enable();
428 }
429}
430
431static void tasklet_hi_action(struct softirq_action *a)
432{
433 struct tasklet_struct *list;
434
435 local_irq_disable();
Olof Johansson48f20a92008-03-04 15:23:25 -0800436 list = __get_cpu_var(tasklet_hi_vec).head;
437 __get_cpu_var(tasklet_hi_vec).head = NULL;
438 __get_cpu_var(tasklet_hi_vec).tail = &__get_cpu_var(tasklet_hi_vec).head;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700439 local_irq_enable();
440
441 while (list) {
442 struct tasklet_struct *t = list;
443
444 list = list->next;
445
446 if (tasklet_trylock(t)) {
447 if (!atomic_read(&t->count)) {
448 if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
449 BUG();
450 t->func(t->data);
451 tasklet_unlock(t);
452 continue;
453 }
454 tasklet_unlock(t);
455 }
456
457 local_irq_disable();
Olof Johansson48f20a92008-03-04 15:23:25 -0800458 t->next = NULL;
459 *__get_cpu_var(tasklet_hi_vec).tail = t;
460 __get_cpu_var(tasklet_hi_vec).tail = &(t->next);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700461 __raise_softirq_irqoff(HI_SOFTIRQ);
462 local_irq_enable();
463 }
464}
465
466
467void tasklet_init(struct tasklet_struct *t,
468 void (*func)(unsigned long), unsigned long data)
469{
470 t->next = NULL;
471 t->state = 0;
472 atomic_set(&t->count, 0);
473 t->func = func;
474 t->data = data;
475}
476
477EXPORT_SYMBOL(tasklet_init);
478
479void tasklet_kill(struct tasklet_struct *t)
480{
481 if (in_interrupt())
482 printk("Attempt to kill tasklet from interrupt\n");
483
484 while (test_and_set_bit(TASKLET_STATE_SCHED, &t->state)) {
485 do
486 yield();
487 while (test_bit(TASKLET_STATE_SCHED, &t->state));
488 }
489 tasklet_unlock_wait(t);
490 clear_bit(TASKLET_STATE_SCHED, &t->state);
491}
492
493EXPORT_SYMBOL(tasklet_kill);
494
495void __init softirq_init(void)
496{
Olof Johansson48f20a92008-03-04 15:23:25 -0800497 int cpu;
498
499 for_each_possible_cpu(cpu) {
500 per_cpu(tasklet_vec, cpu).tail =
501 &per_cpu(tasklet_vec, cpu).head;
502 per_cpu(tasklet_hi_vec, cpu).tail =
503 &per_cpu(tasklet_hi_vec, cpu).head;
504 }
505
Linus Torvalds1da177e2005-04-16 15:20:36 -0700506 open_softirq(TASKLET_SOFTIRQ, tasklet_action, NULL);
507 open_softirq(HI_SOFTIRQ, tasklet_hi_action, NULL);
508}
509
510static int ksoftirqd(void * __bind_cpu)
511{
Linus Torvalds1da177e2005-04-16 15:20:36 -0700512 set_current_state(TASK_INTERRUPTIBLE);
513
514 while (!kthread_should_stop()) {
515 preempt_disable();
516 if (!local_softirq_pending()) {
517 preempt_enable_no_resched();
518 schedule();
519 preempt_disable();
520 }
521
522 __set_current_state(TASK_RUNNING);
523
524 while (local_softirq_pending()) {
525 /* Preempt disable stops cpu going offline.
526 If already offline, we'll be on wrong CPU:
527 don't process */
528 if (cpu_is_offline((long)__bind_cpu))
529 goto wait_to_die;
530 do_softirq();
531 preempt_enable_no_resched();
532 cond_resched();
533 preempt_disable();
534 }
535 preempt_enable();
536 set_current_state(TASK_INTERRUPTIBLE);
537 }
538 __set_current_state(TASK_RUNNING);
539 return 0;
540
541wait_to_die:
542 preempt_enable();
543 /* Wait for kthread_stop */
544 set_current_state(TASK_INTERRUPTIBLE);
545 while (!kthread_should_stop()) {
546 schedule();
547 set_current_state(TASK_INTERRUPTIBLE);
548 }
549 __set_current_state(TASK_RUNNING);
550 return 0;
551}
552
553#ifdef CONFIG_HOTPLUG_CPU
554/*
555 * tasklet_kill_immediate is called to remove a tasklet which can already be
556 * scheduled for execution on @cpu.
557 *
558 * Unlike tasklet_kill, this function removes the tasklet
559 * _immediately_, even if the tasklet is in TASKLET_STATE_SCHED state.
560 *
561 * When this function is called, @cpu must be in the CPU_DEAD state.
562 */
563void tasklet_kill_immediate(struct tasklet_struct *t, unsigned int cpu)
564{
565 struct tasklet_struct **i;
566
567 BUG_ON(cpu_online(cpu));
568 BUG_ON(test_bit(TASKLET_STATE_RUN, &t->state));
569
570 if (!test_bit(TASKLET_STATE_SCHED, &t->state))
571 return;
572
573 /* CPU is dead, so no lock needed. */
Olof Johansson48f20a92008-03-04 15:23:25 -0800574 for (i = &per_cpu(tasklet_vec, cpu).head; *i; i = &(*i)->next) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700575 if (*i == t) {
576 *i = t->next;
Olof Johansson48f20a92008-03-04 15:23:25 -0800577 /* If this was the tail element, move the tail ptr */
578 if (*i == NULL)
579 per_cpu(tasklet_vec, cpu).tail = i;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700580 return;
581 }
582 }
583 BUG();
584}
585
586static void takeover_tasklets(unsigned int cpu)
587{
Linus Torvalds1da177e2005-04-16 15:20:36 -0700588 /* CPU is dead, so no lock needed. */
589 local_irq_disable();
590
591 /* Find end, append list for that CPU. */
Christian Borntraegere5e41722008-05-01 04:34:23 -0700592 if (&per_cpu(tasklet_vec, cpu).head != per_cpu(tasklet_vec, cpu).tail) {
593 *(__get_cpu_var(tasklet_vec).tail) = per_cpu(tasklet_vec, cpu).head;
594 __get_cpu_var(tasklet_vec).tail = per_cpu(tasklet_vec, cpu).tail;
595 per_cpu(tasklet_vec, cpu).head = NULL;
596 per_cpu(tasklet_vec, cpu).tail = &per_cpu(tasklet_vec, cpu).head;
597 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700598 raise_softirq_irqoff(TASKLET_SOFTIRQ);
599
Christian Borntraegere5e41722008-05-01 04:34:23 -0700600 if (&per_cpu(tasklet_hi_vec, cpu).head != per_cpu(tasklet_hi_vec, cpu).tail) {
601 *__get_cpu_var(tasklet_hi_vec).tail = per_cpu(tasklet_hi_vec, cpu).head;
602 __get_cpu_var(tasklet_hi_vec).tail = per_cpu(tasklet_hi_vec, cpu).tail;
603 per_cpu(tasklet_hi_vec, cpu).head = NULL;
604 per_cpu(tasklet_hi_vec, cpu).tail = &per_cpu(tasklet_hi_vec, cpu).head;
605 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700606 raise_softirq_irqoff(HI_SOFTIRQ);
607
608 local_irq_enable();
609}
610#endif /* CONFIG_HOTPLUG_CPU */
611
Chandra Seetharaman8c78f302006-07-30 03:03:35 -0700612static int __cpuinit cpu_callback(struct notifier_block *nfb,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700613 unsigned long action,
614 void *hcpu)
615{
616 int hotcpu = (unsigned long)hcpu;
617 struct task_struct *p;
618
619 switch (action) {
620 case CPU_UP_PREPARE:
Rafael J. Wysocki8bb78442007-05-09 02:35:10 -0700621 case CPU_UP_PREPARE_FROZEN:
Linus Torvalds1da177e2005-04-16 15:20:36 -0700622 p = kthread_create(ksoftirqd, hcpu, "ksoftirqd/%d", hotcpu);
623 if (IS_ERR(p)) {
624 printk("ksoftirqd for %i failed\n", hotcpu);
625 return NOTIFY_BAD;
626 }
627 kthread_bind(p, hotcpu);
628 per_cpu(ksoftirqd, hotcpu) = p;
629 break;
630 case CPU_ONLINE:
Rafael J. Wysocki8bb78442007-05-09 02:35:10 -0700631 case CPU_ONLINE_FROZEN:
Linus Torvalds1da177e2005-04-16 15:20:36 -0700632 wake_up_process(per_cpu(ksoftirqd, hotcpu));
633 break;
634#ifdef CONFIG_HOTPLUG_CPU
635 case CPU_UP_CANCELED:
Rafael J. Wysocki8bb78442007-05-09 02:35:10 -0700636 case CPU_UP_CANCELED_FROZEN:
Heiko Carstensfc75cdf2006-06-25 05:49:10 -0700637 if (!per_cpu(ksoftirqd, hotcpu))
638 break;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700639 /* Unbind so it can run. Fall thru. */
Heiko Carstensa4c4af72005-11-07 00:58:38 -0800640 kthread_bind(per_cpu(ksoftirqd, hotcpu),
641 any_online_cpu(cpu_online_map));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700642 case CPU_DEAD:
Satoru Takeuchi1c6b4aa2007-07-15 23:39:48 -0700643 case CPU_DEAD_FROZEN: {
644 struct sched_param param = { .sched_priority = MAX_RT_PRIO-1 };
645
Linus Torvalds1da177e2005-04-16 15:20:36 -0700646 p = per_cpu(ksoftirqd, hotcpu);
647 per_cpu(ksoftirqd, hotcpu) = NULL;
Rusty Russell961ccdd2008-06-23 13:55:38 +1000648 sched_setscheduler_nocheck(p, SCHED_FIFO, &param);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700649 kthread_stop(p);
650 takeover_tasklets(hotcpu);
651 break;
Satoru Takeuchi1c6b4aa2007-07-15 23:39:48 -0700652 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700653#endif /* CONFIG_HOTPLUG_CPU */
654 }
655 return NOTIFY_OK;
656}
657
Chandra Seetharaman8c78f302006-07-30 03:03:35 -0700658static struct notifier_block __cpuinitdata cpu_nfb = {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700659 .notifier_call = cpu_callback
660};
661
662__init int spawn_ksoftirqd(void)
663{
664 void *cpu = (void *)(long)smp_processor_id();
Akinobu Mita07dccf32006-09-29 02:00:22 -0700665 int err = cpu_callback(&cpu_nfb, CPU_UP_PREPARE, cpu);
666
667 BUG_ON(err == NOTIFY_BAD);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700668 cpu_callback(&cpu_nfb, CPU_ONLINE, cpu);
669 register_cpu_notifier(&cpu_nfb);
670 return 0;
671}
Andrew Morton78eef012006-03-22 00:08:16 -0800672
673#ifdef CONFIG_SMP
674/*
675 * Call a function on all processors
676 */
677int on_each_cpu(void (*func) (void *info), void *info, int retry, int wait)
678{
679 int ret = 0;
680
681 preempt_disable();
682 ret = smp_call_function(func, info, retry, wait);
683 local_irq_disable();
684 func(info);
685 local_irq_enable();
686 preempt_enable();
687 return ret;
688}
689EXPORT_SYMBOL(on_each_cpu);
690#endif