blob: f2aa5de6f6c204f709a96f4f15fd6809744c797d [file] [log] [blame]
Ingo Molnarfb1c8f92005-09-10 00:25:56 -07001/*
2 * Copyright 2005, Red Hat, Inc., Ingo Molnar
3 * Released under the General Public License (GPL).
4 *
5 * This file contains the spinlock/rwlock implementations for
6 * DEBUG_SPINLOCK.
7 */
8
Ingo Molnarfb1c8f92005-09-10 00:25:56 -07009#include <linux/spinlock.h>
Andrew Mortonbb81a092006-12-07 02:14:01 +010010#include <linux/nmi.h>
Ingo Molnarfb1c8f92005-09-10 00:25:56 -070011#include <linux/interrupt.h>
Ingo Molnar9a11b49a2006-07-03 00:24:33 -070012#include <linux/debug_locks.h>
Ingo Molnarfb1c8f92005-09-10 00:25:56 -070013#include <linux/delay.h>
Paul Gortmaker8bc3bcc2011-11-16 21:29:17 -050014#include <linux/export.h>
Ingo Molnarfb1c8f92005-09-10 00:25:56 -070015
Thomas Gleixnerc2f21ce2009-12-02 20:02:59 +010016void __raw_spin_lock_init(raw_spinlock_t *lock, const char *name,
17 struct lock_class_key *key)
Ingo Molnar8a25d5d2006-07-03 00:24:54 -070018{
19#ifdef CONFIG_DEBUG_LOCK_ALLOC
20 /*
21 * Make sure we are not reinitializing a held lock:
22 */
23 debug_check_no_locks_freed((void *)lock, sizeof(*lock));
Peter Zijlstra4dfbb9d2006-10-11 01:45:14 -040024 lockdep_init_map(&lock->dep_map, name, key, 0);
Ingo Molnar8a25d5d2006-07-03 00:24:54 -070025#endif
Thomas Gleixneredc35bd2009-12-03 12:38:57 +010026 lock->raw_lock = (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
Ingo Molnar8a25d5d2006-07-03 00:24:54 -070027 lock->magic = SPINLOCK_MAGIC;
28 lock->owner = SPINLOCK_OWNER_INIT;
29 lock->owner_cpu = -1;
30}
31
Thomas Gleixnerc2f21ce2009-12-02 20:02:59 +010032EXPORT_SYMBOL(__raw_spin_lock_init);
Ingo Molnar8a25d5d2006-07-03 00:24:54 -070033
34void __rwlock_init(rwlock_t *lock, const char *name,
35 struct lock_class_key *key)
36{
37#ifdef CONFIG_DEBUG_LOCK_ALLOC
38 /*
39 * Make sure we are not reinitializing a held lock:
40 */
41 debug_check_no_locks_freed((void *)lock, sizeof(*lock));
Peter Zijlstra4dfbb9d2006-10-11 01:45:14 -040042 lockdep_init_map(&lock->dep_map, name, key, 0);
Ingo Molnar8a25d5d2006-07-03 00:24:54 -070043#endif
Thomas Gleixnerfb3a6bb2009-12-03 20:01:19 +010044 lock->raw_lock = (arch_rwlock_t) __ARCH_RW_LOCK_UNLOCKED;
Ingo Molnar8a25d5d2006-07-03 00:24:54 -070045 lock->magic = RWLOCK_MAGIC;
46 lock->owner = SPINLOCK_OWNER_INIT;
47 lock->owner_cpu = -1;
48}
49
50EXPORT_SYMBOL(__rwlock_init);
51
Akinobu Mita4e101b02011-10-31 17:12:29 -070052static void spin_dump(raw_spinlock_t *lock, const char *msg)
Ingo Molnarfb1c8f92005-09-10 00:25:56 -070053{
Ingo Molnarfb1c8f92005-09-10 00:25:56 -070054 struct task_struct *owner = NULL;
55
Ingo Molnar9a11b49a2006-07-03 00:24:33 -070056 if (lock->owner && lock->owner != SPINLOCK_OWNER_INIT)
57 owner = lock->owner;
58 printk(KERN_EMERG "BUG: spinlock %s on CPU#%d, %s/%d\n",
59 msg, raw_smp_processor_id(),
Pavel Emelyanovba25f9d2007-10-18 23:40:40 -070060 current->comm, task_pid_nr(current));
Stephen Boyd9e684932012-07-25 18:30:45 -070061 printk(KERN_EMERG " lock: %pS, .magic: %08x, .owner: %s/%d, "
Ingo Molnar9a11b49a2006-07-03 00:24:33 -070062 ".owner_cpu: %d\n",
63 lock, lock->magic,
64 owner ? owner->comm : "<none>",
Pavel Emelyanovba25f9d2007-10-18 23:40:40 -070065 owner ? task_pid_nr(owner) : -1,
Ingo Molnar9a11b49a2006-07-03 00:24:33 -070066 lock->owner_cpu);
67 dump_stack();
Ingo Molnarfb1c8f92005-09-10 00:25:56 -070068}
69
Akinobu Mita4e101b02011-10-31 17:12:29 -070070static void spin_bug(raw_spinlock_t *lock, const char *msg)
71{
72 if (!debug_locks_off())
73 return;
74
75 spin_dump(lock, msg);
76}
77
Ingo Molnarfb1c8f92005-09-10 00:25:56 -070078#define SPIN_BUG_ON(cond, lock, msg) if (unlikely(cond)) spin_bug(lock, msg)
79
Ingo Molnar9a11b49a2006-07-03 00:24:33 -070080static inline void
Thomas Gleixnerc2f21ce2009-12-02 20:02:59 +010081debug_spin_lock_before(raw_spinlock_t *lock)
Ingo Molnarfb1c8f92005-09-10 00:25:56 -070082{
83 SPIN_BUG_ON(lock->magic != SPINLOCK_MAGIC, lock, "bad magic");
84 SPIN_BUG_ON(lock->owner == current, lock, "recursion");
85 SPIN_BUG_ON(lock->owner_cpu == raw_smp_processor_id(),
86 lock, "cpu recursion");
87}
88
Thomas Gleixnerc2f21ce2009-12-02 20:02:59 +010089static inline void debug_spin_lock_after(raw_spinlock_t *lock)
Ingo Molnarfb1c8f92005-09-10 00:25:56 -070090{
91 lock->owner_cpu = raw_smp_processor_id();
92 lock->owner = current;
93}
94
Thomas Gleixnerc2f21ce2009-12-02 20:02:59 +010095static inline void debug_spin_unlock(raw_spinlock_t *lock)
Ingo Molnarfb1c8f92005-09-10 00:25:56 -070096{
97 SPIN_BUG_ON(lock->magic != SPINLOCK_MAGIC, lock, "bad magic");
Thomas Gleixnerc2f21ce2009-12-02 20:02:59 +010098 SPIN_BUG_ON(!raw_spin_is_locked(lock), lock, "already unlocked");
Ingo Molnarfb1c8f92005-09-10 00:25:56 -070099 SPIN_BUG_ON(lock->owner != current, lock, "wrong owner");
100 SPIN_BUG_ON(lock->owner_cpu != raw_smp_processor_id(),
101 lock, "wrong CPU");
102 lock->owner = SPINLOCK_OWNER_INIT;
103 lock->owner_cpu = -1;
104}
105
Thomas Gleixnerc2f21ce2009-12-02 20:02:59 +0100106static void __spin_lock_debug(raw_spinlock_t *lock)
Ingo Molnarfb1c8f92005-09-10 00:25:56 -0700107{
Ingo Molnarfb1c8f92005-09-10 00:25:56 -0700108 u64 i;
Vikram Mulukutla03747622012-08-02 13:18:15 -0700109 u64 loops = (loops_per_jiffy * HZ) >> 4;
Ingo Molnarfb1c8f92005-09-10 00:25:56 -0700110
Vikram Mulukutla03747622012-08-02 13:18:15 -0700111 for (i = 0; i < loops; i++) {
112 if (arch_spin_trylock(&lock->raw_lock))
113 return;
114 __delay(1);
Ingo Molnarfb1c8f92005-09-10 00:25:56 -0700115 }
Vikram Mulukutla03747622012-08-02 13:18:15 -0700116 /* lockup suspected: */
117 spin_dump(lock, "lockup");
118#ifdef CONFIG_SMP
119 trigger_all_cpu_backtrace();
120#endif
121
122 /*
123 * The trylock above was causing a livelock. Give the lower level arch
124 * specific lock code a chance to acquire the lock. We have already
125 * printed a warning/backtrace at this point. The non-debug arch
126 * specific code might actually succeed in acquiring the lock. If it is
127 * not successful, the end-result is the same - there is no forward
128 * progress.
129 */
130 arch_spin_lock(&lock->raw_lock);
Ingo Molnarfb1c8f92005-09-10 00:25:56 -0700131}
132
Thomas Gleixner9828ea92009-12-03 20:55:53 +0100133void do_raw_spin_lock(raw_spinlock_t *lock)
Ingo Molnarfb1c8f92005-09-10 00:25:56 -0700134{
135 debug_spin_lock_before(lock);
Thomas Gleixner0199c4e2009-12-02 20:01:25 +0100136 if (unlikely(!arch_spin_trylock(&lock->raw_lock)))
Ingo Molnarfb1c8f92005-09-10 00:25:56 -0700137 __spin_lock_debug(lock);
138 debug_spin_lock_after(lock);
139}
140
Thomas Gleixner9828ea92009-12-03 20:55:53 +0100141int do_raw_spin_trylock(raw_spinlock_t *lock)
Ingo Molnarfb1c8f92005-09-10 00:25:56 -0700142{
Thomas Gleixner0199c4e2009-12-02 20:01:25 +0100143 int ret = arch_spin_trylock(&lock->raw_lock);
Ingo Molnarfb1c8f92005-09-10 00:25:56 -0700144
145 if (ret)
146 debug_spin_lock_after(lock);
147#ifndef CONFIG_SMP
148 /*
149 * Must not happen on UP:
150 */
151 SPIN_BUG_ON(!ret, lock, "trylock failure on UP");
152#endif
153 return ret;
154}
155
Thomas Gleixner9828ea92009-12-03 20:55:53 +0100156void do_raw_spin_unlock(raw_spinlock_t *lock)
Ingo Molnarfb1c8f92005-09-10 00:25:56 -0700157{
158 debug_spin_unlock(lock);
Thomas Gleixner0199c4e2009-12-02 20:01:25 +0100159 arch_spin_unlock(&lock->raw_lock);
Ingo Molnarfb1c8f92005-09-10 00:25:56 -0700160}
161
162static void rwlock_bug(rwlock_t *lock, const char *msg)
163{
Ingo Molnar9a11b49a2006-07-03 00:24:33 -0700164 if (!debug_locks_off())
165 return;
Ingo Molnarfb1c8f92005-09-10 00:25:56 -0700166
Ingo Molnar9a11b49a2006-07-03 00:24:33 -0700167 printk(KERN_EMERG "BUG: rwlock %s on CPU#%d, %s/%d, %p\n",
168 msg, raw_smp_processor_id(), current->comm,
Pavel Emelyanovba25f9d2007-10-18 23:40:40 -0700169 task_pid_nr(current), lock);
Ingo Molnar9a11b49a2006-07-03 00:24:33 -0700170 dump_stack();
Ingo Molnarfb1c8f92005-09-10 00:25:56 -0700171}
172
173#define RWLOCK_BUG_ON(cond, lock, msg) if (unlikely(cond)) rwlock_bug(lock, msg)
174
Andrew Morton72f0b4e2006-08-05 12:13:47 -0700175#if 0 /* __write_lock_debug() can lock up - maybe this can too? */
Ingo Molnarfb1c8f92005-09-10 00:25:56 -0700176static void __read_lock_debug(rwlock_t *lock)
177{
Ingo Molnarfb1c8f92005-09-10 00:25:56 -0700178 u64 i;
Chuck Ebbertc22f0082006-09-29 01:59:14 -0700179 u64 loops = loops_per_jiffy * HZ;
180 int print_once = 1;
Ingo Molnarfb1c8f92005-09-10 00:25:56 -0700181
182 for (;;) {
Chuck Ebbertc22f0082006-09-29 01:59:14 -0700183 for (i = 0; i < loops; i++) {
Thomas Gleixnere5931942009-12-03 20:08:46 +0100184 if (arch_read_trylock(&lock->raw_lock))
Ingo Molnarfb1c8f92005-09-10 00:25:56 -0700185 return;
Ingo Molnare0a60292006-02-07 12:58:54 -0800186 __delay(1);
Ingo Molnarfb1c8f92005-09-10 00:25:56 -0700187 }
188 /* lockup suspected: */
189 if (print_once) {
190 print_once = 0;
Dave Jones51989b92006-01-09 20:51:32 -0800191 printk(KERN_EMERG "BUG: read-lock lockup on CPU#%d, "
192 "%s/%d, %p\n",
Ingo Molnarbb44f112005-12-20 11:54:17 +0100193 raw_smp_processor_id(), current->comm,
194 current->pid, lock);
Ingo Molnarfb1c8f92005-09-10 00:25:56 -0700195 dump_stack();
196 }
197 }
198}
Andrew Morton72f0b4e2006-08-05 12:13:47 -0700199#endif
Ingo Molnarfb1c8f92005-09-10 00:25:56 -0700200
Thomas Gleixner9828ea92009-12-03 20:55:53 +0100201void do_raw_read_lock(rwlock_t *lock)
Ingo Molnarfb1c8f92005-09-10 00:25:56 -0700202{
203 RWLOCK_BUG_ON(lock->magic != RWLOCK_MAGIC, lock, "bad magic");
Thomas Gleixnere5931942009-12-03 20:08:46 +0100204 arch_read_lock(&lock->raw_lock);
Ingo Molnarfb1c8f92005-09-10 00:25:56 -0700205}
206
Thomas Gleixner9828ea92009-12-03 20:55:53 +0100207int do_raw_read_trylock(rwlock_t *lock)
Ingo Molnarfb1c8f92005-09-10 00:25:56 -0700208{
Thomas Gleixnere5931942009-12-03 20:08:46 +0100209 int ret = arch_read_trylock(&lock->raw_lock);
Ingo Molnarfb1c8f92005-09-10 00:25:56 -0700210
211#ifndef CONFIG_SMP
212 /*
213 * Must not happen on UP:
214 */
215 RWLOCK_BUG_ON(!ret, lock, "trylock failure on UP");
216#endif
217 return ret;
218}
219
Thomas Gleixner9828ea92009-12-03 20:55:53 +0100220void do_raw_read_unlock(rwlock_t *lock)
Ingo Molnarfb1c8f92005-09-10 00:25:56 -0700221{
222 RWLOCK_BUG_ON(lock->magic != RWLOCK_MAGIC, lock, "bad magic");
Thomas Gleixnere5931942009-12-03 20:08:46 +0100223 arch_read_unlock(&lock->raw_lock);
Ingo Molnarfb1c8f92005-09-10 00:25:56 -0700224}
225
226static inline void debug_write_lock_before(rwlock_t *lock)
227{
228 RWLOCK_BUG_ON(lock->magic != RWLOCK_MAGIC, lock, "bad magic");
229 RWLOCK_BUG_ON(lock->owner == current, lock, "recursion");
230 RWLOCK_BUG_ON(lock->owner_cpu == raw_smp_processor_id(),
231 lock, "cpu recursion");
232}
233
234static inline void debug_write_lock_after(rwlock_t *lock)
235{
236 lock->owner_cpu = raw_smp_processor_id();
237 lock->owner = current;
238}
239
240static inline void debug_write_unlock(rwlock_t *lock)
241{
242 RWLOCK_BUG_ON(lock->magic != RWLOCK_MAGIC, lock, "bad magic");
243 RWLOCK_BUG_ON(lock->owner != current, lock, "wrong owner");
244 RWLOCK_BUG_ON(lock->owner_cpu != raw_smp_processor_id(),
245 lock, "wrong CPU");
246 lock->owner = SPINLOCK_OWNER_INIT;
247 lock->owner_cpu = -1;
248}
249
Andrew Morton72f0b4e2006-08-05 12:13:47 -0700250#if 0 /* This can cause lockups */
Ingo Molnarfb1c8f92005-09-10 00:25:56 -0700251static void __write_lock_debug(rwlock_t *lock)
252{
Ingo Molnarfb1c8f92005-09-10 00:25:56 -0700253 u64 i;
Chuck Ebbertc22f0082006-09-29 01:59:14 -0700254 u64 loops = loops_per_jiffy * HZ;
255 int print_once = 1;
Ingo Molnarfb1c8f92005-09-10 00:25:56 -0700256
257 for (;;) {
Chuck Ebbertc22f0082006-09-29 01:59:14 -0700258 for (i = 0; i < loops; i++) {
Thomas Gleixnere5931942009-12-03 20:08:46 +0100259 if (arch_write_trylock(&lock->raw_lock))
Ingo Molnarfb1c8f92005-09-10 00:25:56 -0700260 return;
Ingo Molnare0a60292006-02-07 12:58:54 -0800261 __delay(1);
Ingo Molnarfb1c8f92005-09-10 00:25:56 -0700262 }
263 /* lockup suspected: */
264 if (print_once) {
265 print_once = 0;
Dave Jones51989b92006-01-09 20:51:32 -0800266 printk(KERN_EMERG "BUG: write-lock lockup on CPU#%d, "
267 "%s/%d, %p\n",
Ingo Molnarbb44f112005-12-20 11:54:17 +0100268 raw_smp_processor_id(), current->comm,
269 current->pid, lock);
Ingo Molnarfb1c8f92005-09-10 00:25:56 -0700270 dump_stack();
271 }
272 }
273}
Andrew Morton72f0b4e2006-08-05 12:13:47 -0700274#endif
Ingo Molnarfb1c8f92005-09-10 00:25:56 -0700275
Thomas Gleixner9828ea92009-12-03 20:55:53 +0100276void do_raw_write_lock(rwlock_t *lock)
Ingo Molnarfb1c8f92005-09-10 00:25:56 -0700277{
278 debug_write_lock_before(lock);
Thomas Gleixnere5931942009-12-03 20:08:46 +0100279 arch_write_lock(&lock->raw_lock);
Ingo Molnarfb1c8f92005-09-10 00:25:56 -0700280 debug_write_lock_after(lock);
281}
282
Thomas Gleixner9828ea92009-12-03 20:55:53 +0100283int do_raw_write_trylock(rwlock_t *lock)
Ingo Molnarfb1c8f92005-09-10 00:25:56 -0700284{
Thomas Gleixnere5931942009-12-03 20:08:46 +0100285 int ret = arch_write_trylock(&lock->raw_lock);
Ingo Molnarfb1c8f92005-09-10 00:25:56 -0700286
287 if (ret)
288 debug_write_lock_after(lock);
289#ifndef CONFIG_SMP
290 /*
291 * Must not happen on UP:
292 */
293 RWLOCK_BUG_ON(!ret, lock, "trylock failure on UP");
294#endif
295 return ret;
296}
297
Thomas Gleixner9828ea92009-12-03 20:55:53 +0100298void do_raw_write_unlock(rwlock_t *lock)
Ingo Molnarfb1c8f92005-09-10 00:25:56 -0700299{
300 debug_write_unlock(lock);
Thomas Gleixnere5931942009-12-03 20:08:46 +0100301 arch_write_unlock(&lock->raw_lock);
Ingo Molnarfb1c8f92005-09-10 00:25:56 -0700302}