Ingo Molnar | 23f78d4a | 2006-06-27 02:54:53 -0700 | [diff] [blame] | 1 | /* |
| 2 | * RT Mutexes: blocking mutual exclusion locks with PI support |
| 3 | * |
| 4 | * started by Ingo Molnar and Thomas Gleixner: |
| 5 | * |
| 6 | * Copyright (C) 2004-2006 Red Hat, Inc., Ingo Molnar <mingo@redhat.com> |
| 7 | * Copyright (C) 2006, Timesys Corp., Thomas Gleixner <tglx@timesys.com> |
| 8 | * |
| 9 | * This file contains the private data structure and API definitions. |
| 10 | */ |
| 11 | |
| 12 | #ifndef __KERNEL_RTMUTEX_COMMON_H |
| 13 | #define __KERNEL_RTMUTEX_COMMON_H |
| 14 | |
| 15 | #include <linux/rtmutex.h> |
| 16 | |
| 17 | /* |
Ingo Molnar | 23f78d4a | 2006-06-27 02:54:53 -0700 | [diff] [blame] | 18 | * This is the control structure for tasks blocked on a rt_mutex, |
| 19 | * which is allocated on the kernel stack on of the blocked task. |
| 20 | * |
Peter Zijlstra | fb00aca | 2013-11-07 14:43:43 +0100 | [diff] [blame] | 21 | * @tree_entry: pi node to enqueue into the mutex waiters tree |
| 22 | * @pi_tree_entry: pi node to enqueue into the mutex owner waiters tree |
Ingo Molnar | 23f78d4a | 2006-06-27 02:54:53 -0700 | [diff] [blame] | 23 | * @task: task reference to the blocked task |
| 24 | */ |
| 25 | struct rt_mutex_waiter { |
Peter Zijlstra | fb00aca | 2013-11-07 14:43:43 +0100 | [diff] [blame] | 26 | struct rb_node tree_entry; |
| 27 | struct rb_node pi_tree_entry; |
Ingo Molnar | 23f78d4a | 2006-06-27 02:54:53 -0700 | [diff] [blame] | 28 | struct task_struct *task; |
| 29 | struct rt_mutex *lock; |
| 30 | #ifdef CONFIG_DEBUG_RT_MUTEXES |
| 31 | unsigned long ip; |
Pavel Emelyanov | 48d13e4 | 2008-02-08 04:21:53 -0800 | [diff] [blame] | 32 | struct pid *deadlock_task_pid; |
Ingo Molnar | 23f78d4a | 2006-06-27 02:54:53 -0700 | [diff] [blame] | 33 | struct rt_mutex *deadlock_lock; |
| 34 | #endif |
Dario Faggioli | 2d3d891 | 2013-11-07 14:43:44 +0100 | [diff] [blame] | 35 | int prio; |
Peter Zijlstra | 0e6661b | 2017-03-23 15:56:13 +0100 | [diff] [blame] | 36 | u64 deadline; |
Ingo Molnar | 23f78d4a | 2006-06-27 02:54:53 -0700 | [diff] [blame] | 37 | }; |
| 38 | |
| 39 | /* |
Peter Zijlstra | fb00aca | 2013-11-07 14:43:43 +0100 | [diff] [blame] | 40 | * Various helpers to access the waiters-tree: |
Ingo Molnar | 23f78d4a | 2006-06-27 02:54:53 -0700 | [diff] [blame] | 41 | */ |
| 42 | static inline int rt_mutex_has_waiters(struct rt_mutex *lock) |
| 43 | { |
Peter Zijlstra | fb00aca | 2013-11-07 14:43:43 +0100 | [diff] [blame] | 44 | return !RB_EMPTY_ROOT(&lock->waiters); |
Ingo Molnar | 23f78d4a | 2006-06-27 02:54:53 -0700 | [diff] [blame] | 45 | } |
| 46 | |
| 47 | static inline struct rt_mutex_waiter * |
| 48 | rt_mutex_top_waiter(struct rt_mutex *lock) |
| 49 | { |
| 50 | struct rt_mutex_waiter *w; |
| 51 | |
Peter Zijlstra | fb00aca | 2013-11-07 14:43:43 +0100 | [diff] [blame] | 52 | w = rb_entry(lock->waiters_leftmost, struct rt_mutex_waiter, |
| 53 | tree_entry); |
Ingo Molnar | 23f78d4a | 2006-06-27 02:54:53 -0700 | [diff] [blame] | 54 | BUG_ON(w->lock != lock); |
| 55 | |
| 56 | return w; |
| 57 | } |
| 58 | |
| 59 | static inline int task_has_pi_waiters(struct task_struct *p) |
| 60 | { |
Peter Zijlstra | fb00aca | 2013-11-07 14:43:43 +0100 | [diff] [blame] | 61 | return !RB_EMPTY_ROOT(&p->pi_waiters); |
Ingo Molnar | 23f78d4a | 2006-06-27 02:54:53 -0700 | [diff] [blame] | 62 | } |
| 63 | |
| 64 | static inline struct rt_mutex_waiter * |
| 65 | task_top_pi_waiter(struct task_struct *p) |
| 66 | { |
Peter Zijlstra | fb00aca | 2013-11-07 14:43:43 +0100 | [diff] [blame] | 67 | return rb_entry(p->pi_waiters_leftmost, struct rt_mutex_waiter, |
| 68 | pi_tree_entry); |
Ingo Molnar | 23f78d4a | 2006-06-27 02:54:53 -0700 | [diff] [blame] | 69 | } |
| 70 | |
| 71 | /* |
| 72 | * lock->owner state tracking: |
| 73 | */ |
Lai Jiangshan | 8161239 | 2011-01-14 17:09:41 +0800 | [diff] [blame] | 74 | #define RT_MUTEX_HAS_WAITERS 1UL |
| 75 | #define RT_MUTEX_OWNER_MASKALL 1UL |
Ingo Molnar | 23f78d4a | 2006-06-27 02:54:53 -0700 | [diff] [blame] | 76 | |
| 77 | static inline struct task_struct *rt_mutex_owner(struct rt_mutex *lock) |
| 78 | { |
Thomas Gleixner | 1be5d4f | 2016-11-30 21:04:42 +0000 | [diff] [blame] | 79 | unsigned long owner = (unsigned long) READ_ONCE(lock->owner); |
| 80 | |
| 81 | return (struct task_struct *) (owner & ~RT_MUTEX_OWNER_MASKALL); |
Ingo Molnar | 23f78d4a | 2006-06-27 02:54:53 -0700 | [diff] [blame] | 82 | } |
| 83 | |
Ingo Molnar | c87e283 | 2006-06-27 02:54:58 -0700 | [diff] [blame] | 84 | /* |
Thomas Gleixner | 8930ed8 | 2014-05-22 03:25:47 +0000 | [diff] [blame] | 85 | * Constants for rt mutex functions which have a selectable deadlock |
| 86 | * detection. |
| 87 | * |
| 88 | * RT_MUTEX_MIN_CHAINWALK: Stops the lock chain walk when there are |
| 89 | * no further PI adjustments to be made. |
| 90 | * |
| 91 | * RT_MUTEX_FULL_CHAINWALK: Invoke deadlock detection with a full |
| 92 | * walk of the lock chain. |
| 93 | */ |
| 94 | enum rtmutex_chainwalk { |
| 95 | RT_MUTEX_MIN_CHAINWALK, |
| 96 | RT_MUTEX_FULL_CHAINWALK, |
| 97 | }; |
| 98 | |
| 99 | /* |
Ingo Molnar | c87e283 | 2006-06-27 02:54:58 -0700 | [diff] [blame] | 100 | * PI-futex support (proxy locking functions, etc.): |
| 101 | */ |
| 102 | extern struct task_struct *rt_mutex_next_owner(struct rt_mutex *lock); |
| 103 | extern void rt_mutex_init_proxy_locked(struct rt_mutex *lock, |
| 104 | struct task_struct *proxy_owner); |
| 105 | extern void rt_mutex_proxy_unlock(struct rt_mutex *lock, |
| 106 | struct task_struct *proxy_owner); |
Darren Hart | 8dac456 | 2009-04-03 13:40:12 -0700 | [diff] [blame] | 107 | extern int rt_mutex_start_proxy_lock(struct rt_mutex *lock, |
| 108 | struct rt_mutex_waiter *waiter, |
Thomas Gleixner | c051b21 | 2014-05-22 03:25:50 +0000 | [diff] [blame] | 109 | struct task_struct *task); |
Peter Zijlstra | ce81355 | 2017-03-22 11:35:57 +0100 | [diff] [blame] | 110 | extern int rt_mutex_wait_proxy_lock(struct rt_mutex *lock, |
| 111 | struct hrtimer_sleeper *to, |
| 112 | struct rt_mutex_waiter *waiter); |
| 113 | extern bool rt_mutex_cleanup_proxy_lock(struct rt_mutex *lock, |
| 114 | struct rt_mutex_waiter *waiter); |
Thomas Gleixner | c051b21 | 2014-05-22 03:25:50 +0000 | [diff] [blame] | 115 | extern int rt_mutex_timed_futex_lock(struct rt_mutex *l, struct hrtimer_sleeper *to); |
Sebastian Andrzej Siewior | 802ab58 | 2015-06-17 10:33:50 +0200 | [diff] [blame] | 116 | extern bool rt_mutex_futex_unlock(struct rt_mutex *lock, |
| 117 | struct wake_q_head *wqh); |
| 118 | extern void rt_mutex_adjust_prio(struct task_struct *task); |
Thomas Gleixner | 36cf3b5 | 2007-07-15 23:41:20 -0700 | [diff] [blame] | 119 | |
| 120 | #ifdef CONFIG_DEBUG_RT_MUTEXES |
| 121 | # include "rtmutex-debug.h" |
| 122 | #else |
| 123 | # include "rtmutex.h" |
| 124 | #endif |
| 125 | |
Ingo Molnar | 23f78d4a | 2006-06-27 02:54:53 -0700 | [diff] [blame] | 126 | #endif |