blob: 4689ef210a13c057a9c44880b1442437cdbd06af [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001#ifndef __LINUX_PREEMPT_H
2#define __LINUX_PREEMPT_H
3
4/*
5 * include/linux/preempt.h - macros for accessing and manipulating
6 * preempt_count (used for kernel preemption, interrupt count, etc.)
7 */
8
Linus Torvalds1da177e2005-04-16 15:20:36 -07009#include <linux/linkage.h>
Avi Kivitye107be32007-07-26 13:40:43 +020010#include <linux/list.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070011
Peter Zijlstraf27dde82013-08-14 14:55:31 +020012/*
Frederic Weisbecker92cf2112015-05-12 16:41:46 +020013 * We put the hardirq and softirq counter into the preemption
14 * counter. The bitmask has the following meaning:
15 *
16 * - bits 0-7 are the preemption count (max preemption depth: 256)
17 * - bits 8-15 are the softirq count (max # of softirqs: 256)
18 *
19 * The hardirq count could in theory be the same as the number of
20 * interrupts in the system, but we run all interrupt handlers with
21 * interrupts disabled, so we cannot have nesting interrupts. Though
22 * there are a few palaeontologic drivers which reenable interrupts in
23 * the handler, so we need more than one bit here.
24 *
Frederic Weisbecker2e10e712015-05-12 16:41:47 +020025 * PREEMPT_MASK: 0x000000ff
26 * SOFTIRQ_MASK: 0x0000ff00
27 * HARDIRQ_MASK: 0x000f0000
28 * NMI_MASK: 0x00100000
29 * PREEMPT_ACTIVE: 0x00200000
30 * PREEMPT_NEED_RESCHED: 0x80000000
Frederic Weisbecker92cf2112015-05-12 16:41:46 +020031 */
32#define PREEMPT_BITS 8
33#define SOFTIRQ_BITS 8
34#define HARDIRQ_BITS 4
35#define NMI_BITS 1
36
37#define PREEMPT_SHIFT 0
38#define SOFTIRQ_SHIFT (PREEMPT_SHIFT + PREEMPT_BITS)
39#define HARDIRQ_SHIFT (SOFTIRQ_SHIFT + SOFTIRQ_BITS)
40#define NMI_SHIFT (HARDIRQ_SHIFT + HARDIRQ_BITS)
41
42#define __IRQ_MASK(x) ((1UL << (x))-1)
43
44#define PREEMPT_MASK (__IRQ_MASK(PREEMPT_BITS) << PREEMPT_SHIFT)
45#define SOFTIRQ_MASK (__IRQ_MASK(SOFTIRQ_BITS) << SOFTIRQ_SHIFT)
46#define HARDIRQ_MASK (__IRQ_MASK(HARDIRQ_BITS) << HARDIRQ_SHIFT)
47#define NMI_MASK (__IRQ_MASK(NMI_BITS) << NMI_SHIFT)
48
49#define PREEMPT_OFFSET (1UL << PREEMPT_SHIFT)
50#define SOFTIRQ_OFFSET (1UL << SOFTIRQ_SHIFT)
51#define HARDIRQ_OFFSET (1UL << HARDIRQ_SHIFT)
52#define NMI_OFFSET (1UL << NMI_SHIFT)
53
54#define SOFTIRQ_DISABLE_OFFSET (2 * SOFTIRQ_OFFSET)
55
56#define PREEMPT_ACTIVE_BITS 1
57#define PREEMPT_ACTIVE_SHIFT (NMI_SHIFT + NMI_BITS)
58#define PREEMPT_ACTIVE (__IRQ_MASK(PREEMPT_ACTIVE_BITS) << PREEMPT_ACTIVE_SHIFT)
59
Frederic Weisbecker2e10e712015-05-12 16:41:47 +020060/* We use the MSB mostly because its available */
61#define PREEMPT_NEED_RESCHED 0x80000000
62
63/* preempt_count() and related functions, depends on PREEMPT_NEED_RESCHED */
64#include <asm/preempt.h>
65
Frederic Weisbecker92cf2112015-05-12 16:41:46 +020066#define hardirq_count() (preempt_count() & HARDIRQ_MASK)
67#define softirq_count() (preempt_count() & SOFTIRQ_MASK)
68#define irq_count() (preempt_count() & (HARDIRQ_MASK | SOFTIRQ_MASK \
69 | NMI_MASK))
70
71/*
72 * Are we doing bottom half or hardware interrupt processing?
73 * Are we in a softirq context? Interrupt context?
74 * in_softirq - Are we currently processing softirq or have bh disabled?
75 * in_serving_softirq - Are we currently processing softirq?
76 */
77#define in_irq() (hardirq_count())
78#define in_softirq() (softirq_count())
79#define in_interrupt() (irq_count())
80#define in_serving_softirq() (softirq_count() & SOFTIRQ_OFFSET)
81
82/*
83 * Are we in NMI context?
84 */
85#define in_nmi() (preempt_count() & NMI_MASK)
86
87#if defined(CONFIG_PREEMPT_COUNT)
Frederic Weisbecker90b62b52015-05-12 16:41:48 +020088# define PREEMPT_DISABLE_OFFSET 1
Frederic Weisbecker92cf2112015-05-12 16:41:46 +020089#else
Frederic Weisbecker90b62b52015-05-12 16:41:48 +020090# define PREEMPT_DISABLE_OFFSET 0
Frederic Weisbecker92cf2112015-05-12 16:41:46 +020091#endif
92
93/*
94 * The preempt_count offset needed for things like:
95 *
96 * spin_lock_bh()
97 *
98 * Which need to disable both preemption (CONFIG_PREEMPT_COUNT) and
99 * softirqs, such that unlock sequences of:
100 *
101 * spin_unlock();
102 * local_bh_enable();
103 *
104 * Work as expected.
105 */
Frederic Weisbecker90b62b52015-05-12 16:41:48 +0200106#define SOFTIRQ_LOCK_OFFSET (SOFTIRQ_DISABLE_OFFSET + PREEMPT_DISABLE_OFFSET)
Frederic Weisbecker92cf2112015-05-12 16:41:46 +0200107
108/*
109 * Are we running in atomic context? WARNING: this macro cannot
110 * always detect atomic context; in particular, it cannot know about
111 * held spinlocks in non-preemptible kernels. Thus it should not be
112 * used in the general case to determine whether sleeping is possible.
113 * Do not use in_atomic() in driver code.
114 */
115#define in_atomic() ((preempt_count() & ~PREEMPT_ACTIVE) != 0)
116
117/*
118 * Check whether we were atomic before we did preempt_disable():
119 * (used by the scheduler, *after* releasing the kernel lock)
120 */
121#define in_atomic_preempt_off() \
Frederic Weisbecker90b62b52015-05-12 16:41:48 +0200122 ((preempt_count() & ~PREEMPT_ACTIVE) != PREEMPT_DISABLE_OFFSET)
Frederic Weisbecker92cf2112015-05-12 16:41:46 +0200123
Steven Rostedt6cd8a4b2008-05-12 21:20:42 +0200124#if defined(CONFIG_DEBUG_PREEMPT) || defined(CONFIG_PREEMPT_TRACER)
Peter Zijlstrabdb43802013-09-10 12:15:23 +0200125extern void preempt_count_add(int val);
126extern void preempt_count_sub(int val);
127#define preempt_count_dec_and_test() ({ preempt_count_sub(1); should_resched(); })
Linus Torvalds1da177e2005-04-16 15:20:36 -0700128#else
Peter Zijlstrabdb43802013-09-10 12:15:23 +0200129#define preempt_count_add(val) __preempt_count_add(val)
130#define preempt_count_sub(val) __preempt_count_sub(val)
131#define preempt_count_dec_and_test() __preempt_count_dec_and_test()
Linus Torvalds1da177e2005-04-16 15:20:36 -0700132#endif
133
Peter Zijlstrabdb43802013-09-10 12:15:23 +0200134#define __preempt_count_inc() __preempt_count_add(1)
135#define __preempt_count_dec() __preempt_count_sub(1)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700136
Peter Zijlstrabdb43802013-09-10 12:15:23 +0200137#define preempt_count_inc() preempt_count_add(1)
138#define preempt_count_dec() preempt_count_sub(1)
Frederic Weisbeckerbdd4e852011-06-08 01:13:27 +0200139
140#ifdef CONFIG_PREEMPT_COUNT
141
Linus Torvalds1da177e2005-04-16 15:20:36 -0700142#define preempt_disable() \
143do { \
Peter Zijlstrabdb43802013-09-10 12:15:23 +0200144 preempt_count_inc(); \
Linus Torvalds1da177e2005-04-16 15:20:36 -0700145 barrier(); \
146} while (0)
147
Thomas Gleixnerba74c142011-03-21 13:32:17 +0100148#define sched_preempt_enable_no_resched() \
Linus Torvalds1da177e2005-04-16 15:20:36 -0700149do { \
150 barrier(); \
Peter Zijlstrabdb43802013-09-10 12:15:23 +0200151 preempt_count_dec(); \
Linus Torvalds1da177e2005-04-16 15:20:36 -0700152} while (0)
153
Peter Zijlstrabdb43802013-09-10 12:15:23 +0200154#define preempt_enable_no_resched() sched_preempt_enable_no_resched()
Thomas Gleixnerba74c142011-03-21 13:32:17 +0100155
Frederic Weisbecker2e10e712015-05-12 16:41:47 +0200156#define preemptible() (preempt_count() == 0 && !irqs_disabled())
157
Peter Zijlstrabdb43802013-09-10 12:15:23 +0200158#ifdef CONFIG_PREEMPT
Linus Torvalds1da177e2005-04-16 15:20:36 -0700159#define preempt_enable() \
160do { \
Peter Zijlstrabdb43802013-09-10 12:15:23 +0200161 barrier(); \
162 if (unlikely(preempt_count_dec_and_test())) \
Peter Zijlstra1a338ac2013-08-14 14:51:00 +0200163 __preempt_schedule(); \
Linus Torvalds1da177e2005-04-16 15:20:36 -0700164} while (0)
165
Peter Zijlstrabdb43802013-09-10 12:15:23 +0200166#define preempt_check_resched() \
167do { \
168 if (should_resched()) \
Peter Zijlstra1a338ac2013-08-14 14:51:00 +0200169 __preempt_schedule(); \
Peter Zijlstrabdb43802013-09-10 12:15:23 +0200170} while (0)
171
172#else
Peter Zijlstra62b94a02013-11-20 16:52:19 +0100173#define preempt_enable() \
174do { \
175 barrier(); \
176 preempt_count_dec(); \
177} while (0)
Peter Zijlstrabdb43802013-09-10 12:15:23 +0200178#define preempt_check_resched() do { } while (0)
179#endif
Steven Rostedt50282522008-05-12 21:20:41 +0200180
181#define preempt_disable_notrace() \
182do { \
Peter Zijlstrabdb43802013-09-10 12:15:23 +0200183 __preempt_count_inc(); \
Steven Rostedt50282522008-05-12 21:20:41 +0200184 barrier(); \
185} while (0)
186
187#define preempt_enable_no_resched_notrace() \
188do { \
189 barrier(); \
Peter Zijlstrabdb43802013-09-10 12:15:23 +0200190 __preempt_count_dec(); \
Steven Rostedt50282522008-05-12 21:20:41 +0200191} while (0)
192
Peter Zijlstrabdb43802013-09-10 12:15:23 +0200193#ifdef CONFIG_PREEMPT
194
Peter Zijlstra1a338ac2013-08-14 14:51:00 +0200195#ifndef CONFIG_CONTEXT_TRACKING
196#define __preempt_schedule_context() __preempt_schedule()
Peter Zijlstrabdb43802013-09-10 12:15:23 +0200197#endif
198
Steven Rostedt50282522008-05-12 21:20:41 +0200199#define preempt_enable_notrace() \
200do { \
Peter Zijlstrabdb43802013-09-10 12:15:23 +0200201 barrier(); \
202 if (unlikely(__preempt_count_dec_and_test())) \
Peter Zijlstra1a338ac2013-08-14 14:51:00 +0200203 __preempt_schedule_context(); \
Steven Rostedt50282522008-05-12 21:20:41 +0200204} while (0)
Peter Zijlstrabdb43802013-09-10 12:15:23 +0200205#else
Peter Zijlstra62b94a02013-11-20 16:52:19 +0100206#define preempt_enable_notrace() \
207do { \
208 barrier(); \
209 __preempt_count_dec(); \
210} while (0)
Peter Zijlstrabdb43802013-09-10 12:15:23 +0200211#endif
Steven Rostedt50282522008-05-12 21:20:41 +0200212
Frederic Weisbeckerbdd4e852011-06-08 01:13:27 +0200213#else /* !CONFIG_PREEMPT_COUNT */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700214
Linus Torvalds386afc92013-04-09 10:48:33 -0700215/*
216 * Even if we don't have any preemption, we need preempt disable/enable
217 * to be barriers, so that we don't have things like get_user/put_user
218 * that can cause faults and scheduling migrate into our preempt-protected
219 * region.
220 */
Peter Zijlstrabdb43802013-09-10 12:15:23 +0200221#define preempt_disable() barrier()
Linus Torvalds386afc92013-04-09 10:48:33 -0700222#define sched_preempt_enable_no_resched() barrier()
Peter Zijlstrabdb43802013-09-10 12:15:23 +0200223#define preempt_enable_no_resched() barrier()
224#define preempt_enable() barrier()
225#define preempt_check_resched() do { } while (0)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700226
Linus Torvalds386afc92013-04-09 10:48:33 -0700227#define preempt_disable_notrace() barrier()
228#define preempt_enable_no_resched_notrace() barrier()
229#define preempt_enable_notrace() barrier()
Frederic Weisbecker2e10e712015-05-12 16:41:47 +0200230#define preemptible() 0
Steven Rostedt50282522008-05-12 21:20:41 +0200231
Frederic Weisbeckerbdd4e852011-06-08 01:13:27 +0200232#endif /* CONFIG_PREEMPT_COUNT */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700233
Peter Zijlstra62b94a02013-11-20 16:52:19 +0100234#ifdef MODULE
235/*
236 * Modules have no business playing preemption tricks.
237 */
238#undef sched_preempt_enable_no_resched
239#undef preempt_enable_no_resched
240#undef preempt_enable_no_resched_notrace
241#undef preempt_check_resched
242#endif
243
Peter Zijlstra8cb75e02013-11-20 12:22:37 +0100244#define preempt_set_need_resched() \
245do { \
246 set_preempt_need_resched(); \
247} while (0)
248#define preempt_fold_need_resched() \
249do { \
250 if (tif_need_resched()) \
251 set_preempt_need_resched(); \
252} while (0)
Peter Zijlstra8cb75e02013-11-20 12:22:37 +0100253
Avi Kivitye107be32007-07-26 13:40:43 +0200254#ifdef CONFIG_PREEMPT_NOTIFIERS
255
256struct preempt_notifier;
257
258/**
259 * preempt_ops - notifiers called when a task is preempted and rescheduled
260 * @sched_in: we're about to be rescheduled:
261 * notifier: struct preempt_notifier for the task being scheduled
262 * cpu: cpu we're scheduled on
263 * @sched_out: we've just been preempted
264 * notifier: struct preempt_notifier for the task being preempted
265 * next: the task that's kicking us out
Tejun Heo8592e642009-12-02 12:56:46 +0900266 *
267 * Please note that sched_in and out are called under different
268 * contexts. sched_out is called with rq lock held and irq disabled
269 * while sched_in is called without rq lock and irq enabled. This
270 * difference is intentional and depended upon by its users.
Avi Kivitye107be32007-07-26 13:40:43 +0200271 */
272struct preempt_ops {
273 void (*sched_in)(struct preempt_notifier *notifier, int cpu);
274 void (*sched_out)(struct preempt_notifier *notifier,
275 struct task_struct *next);
276};
277
278/**
279 * preempt_notifier - key for installing preemption notifiers
280 * @link: internal use
281 * @ops: defines the notifier functions to be called
282 *
283 * Usually used in conjunction with container_of().
284 */
285struct preempt_notifier {
286 struct hlist_node link;
287 struct preempt_ops *ops;
288};
289
290void preempt_notifier_register(struct preempt_notifier *notifier);
291void preempt_notifier_unregister(struct preempt_notifier *notifier);
292
293static inline void preempt_notifier_init(struct preempt_notifier *notifier,
294 struct preempt_ops *ops)
295{
296 INIT_HLIST_NODE(&notifier->link);
297 notifier->ops = ops;
298}
299
300#endif
301
Linus Torvalds1da177e2005-04-16 15:20:36 -0700302#endif /* __LINUX_PREEMPT_H */