Paul E. McKenney | 0af3fe1 | 2014-02-04 15:51:41 -0800 | [diff] [blame] | 1 | /* |
| 2 | * Module-based torture test facility for locking |
| 3 | * |
| 4 | * This program is free software; you can redistribute it and/or modify |
| 5 | * it under the terms of the GNU General Public License as published by |
| 6 | * the Free Software Foundation; either version 2 of the License, or |
| 7 | * (at your option) any later version. |
| 8 | * |
| 9 | * This program is distributed in the hope that it will be useful, |
| 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of |
| 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
| 12 | * GNU General Public License for more details. |
| 13 | * |
| 14 | * You should have received a copy of the GNU General Public License |
| 15 | * along with this program; if not, you can access it online at |
| 16 | * http://www.gnu.org/licenses/gpl-2.0.html. |
| 17 | * |
| 18 | * Copyright (C) IBM Corporation, 2014 |
| 19 | * |
| 20 | * Author: Paul E. McKenney <paulmck@us.ibm.com> |
| 21 | * Based on kernel/rcu/torture.c. |
| 22 | */ |
| 23 | #include <linux/types.h> |
| 24 | #include <linux/kernel.h> |
| 25 | #include <linux/init.h> |
| 26 | #include <linux/module.h> |
| 27 | #include <linux/kthread.h> |
| 28 | #include <linux/err.h> |
| 29 | #include <linux/spinlock.h> |
Davidlohr Bueso | 42ddc75 | 2014-09-11 20:40:18 -0700 | [diff] [blame] | 30 | #include <linux/mutex.h> |
Paul E. McKenney | 0af3fe1 | 2014-02-04 15:51:41 -0800 | [diff] [blame] | 31 | #include <linux/smp.h> |
| 32 | #include <linux/interrupt.h> |
| 33 | #include <linux/sched.h> |
| 34 | #include <linux/atomic.h> |
| 35 | #include <linux/bitops.h> |
| 36 | #include <linux/completion.h> |
| 37 | #include <linux/moduleparam.h> |
| 38 | #include <linux/percpu.h> |
| 39 | #include <linux/notifier.h> |
| 40 | #include <linux/reboot.h> |
| 41 | #include <linux/freezer.h> |
| 42 | #include <linux/cpu.h> |
| 43 | #include <linux/delay.h> |
| 44 | #include <linux/stat.h> |
| 45 | #include <linux/slab.h> |
| 46 | #include <linux/trace_clock.h> |
| 47 | #include <asm/byteorder.h> |
| 48 | #include <linux/torture.h> |
| 49 | |
| 50 | MODULE_LICENSE("GPL"); |
| 51 | MODULE_AUTHOR("Paul E. McKenney <paulmck@us.ibm.com>"); |
| 52 | |
| 53 | torture_param(int, nwriters_stress, -1, |
| 54 | "Number of write-locking stress-test threads"); |
Davidlohr Bueso | 4f6332c | 2014-09-11 21:40:41 -0700 | [diff] [blame^] | 55 | torture_param(int, nreaders_stress, -1, |
| 56 | "Number of read-locking stress-test threads"); |
Paul E. McKenney | 0af3fe1 | 2014-02-04 15:51:41 -0800 | [diff] [blame] | 57 | torture_param(int, onoff_holdoff, 0, "Time after boot before CPU hotplugs (s)"); |
| 58 | torture_param(int, onoff_interval, 0, |
| 59 | "Time between CPU hotplugs (s), 0=disable"); |
| 60 | torture_param(int, shuffle_interval, 3, |
| 61 | "Number of jiffies between shuffles, 0=disable"); |
| 62 | torture_param(int, shutdown_secs, 0, "Shutdown time (j), <= zero to disable."); |
| 63 | torture_param(int, stat_interval, 60, |
| 64 | "Number of seconds between stats printk()s"); |
| 65 | torture_param(int, stutter, 5, "Number of jiffies to run/halt test, 0=disable"); |
| 66 | torture_param(bool, verbose, true, |
| 67 | "Enable verbose debugging printk()s"); |
| 68 | |
Davidlohr Bueso | f095bfc | 2014-09-11 20:40:19 -0700 | [diff] [blame] | 69 | static bool debug_lock = false; |
Paul E. McKenney | 0af3fe1 | 2014-02-04 15:51:41 -0800 | [diff] [blame] | 70 | static char *torture_type = "spin_lock"; |
| 71 | module_param(torture_type, charp, 0444); |
| 72 | MODULE_PARM_DESC(torture_type, |
Davidlohr Bueso | 42ddc75 | 2014-09-11 20:40:18 -0700 | [diff] [blame] | 73 | "Type of lock to torture (spin_lock, spin_lock_irq, mutex_lock, ...)"); |
Paul E. McKenney | 0af3fe1 | 2014-02-04 15:51:41 -0800 | [diff] [blame] | 74 | |
| 75 | static atomic_t n_lock_torture_errors; |
| 76 | |
| 77 | static struct task_struct *stats_task; |
| 78 | static struct task_struct **writer_tasks; |
Davidlohr Bueso | 4f6332c | 2014-09-11 21:40:41 -0700 | [diff] [blame^] | 79 | static struct task_struct **reader_tasks; |
Paul E. McKenney | 0af3fe1 | 2014-02-04 15:51:41 -0800 | [diff] [blame] | 80 | |
| 81 | static int nrealwriters_stress; |
| 82 | static bool lock_is_write_held; |
Davidlohr Bueso | 4f6332c | 2014-09-11 21:40:41 -0700 | [diff] [blame^] | 83 | static int nrealreaders_stress; |
| 84 | static bool lock_is_read_held; |
Paul E. McKenney | 0af3fe1 | 2014-02-04 15:51:41 -0800 | [diff] [blame] | 85 | |
Davidlohr Bueso | 1e6757a | 2014-09-11 20:40:20 -0700 | [diff] [blame] | 86 | struct lock_stress_stats { |
| 87 | long n_lock_fail; |
| 88 | long n_lock_acquired; |
Paul E. McKenney | 0af3fe1 | 2014-02-04 15:51:41 -0800 | [diff] [blame] | 89 | }; |
Davidlohr Bueso | 1e6757a | 2014-09-11 20:40:20 -0700 | [diff] [blame] | 90 | static struct lock_stress_stats *lwsa; /* writer statistics */ |
Davidlohr Bueso | 4f6332c | 2014-09-11 21:40:41 -0700 | [diff] [blame^] | 91 | static struct lock_stress_stats *lrsa; /* reader statistics */ |
Paul E. McKenney | 0af3fe1 | 2014-02-04 15:51:41 -0800 | [diff] [blame] | 92 | |
Paul E. McKenney | d065eac | 2014-04-04 17:17:35 -0700 | [diff] [blame] | 93 | #if defined(MODULE) |
Paul E. McKenney | 0af3fe1 | 2014-02-04 15:51:41 -0800 | [diff] [blame] | 94 | #define LOCKTORTURE_RUNNABLE_INIT 1 |
| 95 | #else |
| 96 | #define LOCKTORTURE_RUNNABLE_INIT 0 |
| 97 | #endif |
Davidlohr Bueso | 23a8e5c | 2014-09-11 20:40:16 -0700 | [diff] [blame] | 98 | int torture_runnable = LOCKTORTURE_RUNNABLE_INIT; |
| 99 | module_param(torture_runnable, int, 0444); |
| 100 | MODULE_PARM_DESC(torture_runnable, "Start locktorture at module init"); |
Paul E. McKenney | 0af3fe1 | 2014-02-04 15:51:41 -0800 | [diff] [blame] | 101 | |
| 102 | /* Forward reference. */ |
| 103 | static void lock_torture_cleanup(void); |
| 104 | |
| 105 | /* |
| 106 | * Operations vector for selecting different types of tests. |
| 107 | */ |
| 108 | struct lock_torture_ops { |
| 109 | void (*init)(void); |
| 110 | int (*writelock)(void); |
| 111 | void (*write_delay)(struct torture_random_state *trsp); |
| 112 | void (*writeunlock)(void); |
Davidlohr Bueso | 4f6332c | 2014-09-11 21:40:41 -0700 | [diff] [blame^] | 113 | int (*readlock)(void); |
| 114 | void (*read_delay)(struct torture_random_state *trsp); |
| 115 | void (*readunlock)(void); |
Paul E. McKenney | 0af3fe1 | 2014-02-04 15:51:41 -0800 | [diff] [blame] | 116 | unsigned long flags; |
| 117 | const char *name; |
| 118 | }; |
| 119 | |
| 120 | static struct lock_torture_ops *cur_ops; |
| 121 | |
| 122 | /* |
| 123 | * Definitions for lock torture testing. |
| 124 | */ |
| 125 | |
Paul E. McKenney | e086481 | 2014-02-11 08:05:07 -0800 | [diff] [blame] | 126 | static int torture_lock_busted_write_lock(void) |
| 127 | { |
| 128 | return 0; /* BUGGY, do not use in real life!!! */ |
| 129 | } |
| 130 | |
| 131 | static void torture_lock_busted_write_delay(struct torture_random_state *trsp) |
| 132 | { |
| 133 | const unsigned long longdelay_us = 100; |
| 134 | |
| 135 | /* We want a long delay occasionally to force massive contention. */ |
| 136 | if (!(torture_random(trsp) % |
| 137 | (nrealwriters_stress * 2000 * longdelay_us))) |
| 138 | mdelay(longdelay_us); |
| 139 | #ifdef CONFIG_PREEMPT |
| 140 | if (!(torture_random(trsp) % (nrealwriters_stress * 20000))) |
| 141 | preempt_schedule(); /* Allow test to be preempted. */ |
| 142 | #endif |
| 143 | } |
| 144 | |
| 145 | static void torture_lock_busted_write_unlock(void) |
| 146 | { |
| 147 | /* BUGGY, do not use in real life!!! */ |
| 148 | } |
| 149 | |
| 150 | static struct lock_torture_ops lock_busted_ops = { |
| 151 | .writelock = torture_lock_busted_write_lock, |
| 152 | .write_delay = torture_lock_busted_write_delay, |
| 153 | .writeunlock = torture_lock_busted_write_unlock, |
Davidlohr Bueso | 4f6332c | 2014-09-11 21:40:41 -0700 | [diff] [blame^] | 154 | .readlock = NULL, |
| 155 | .read_delay = NULL, |
| 156 | .readunlock = NULL, |
Paul E. McKenney | e086481 | 2014-02-11 08:05:07 -0800 | [diff] [blame] | 157 | .name = "lock_busted" |
| 158 | }; |
| 159 | |
Paul E. McKenney | 0af3fe1 | 2014-02-04 15:51:41 -0800 | [diff] [blame] | 160 | static DEFINE_SPINLOCK(torture_spinlock); |
| 161 | |
| 162 | static int torture_spin_lock_write_lock(void) __acquires(torture_spinlock) |
| 163 | { |
| 164 | spin_lock(&torture_spinlock); |
| 165 | return 0; |
| 166 | } |
| 167 | |
| 168 | static void torture_spin_lock_write_delay(struct torture_random_state *trsp) |
| 169 | { |
| 170 | const unsigned long shortdelay_us = 2; |
| 171 | const unsigned long longdelay_us = 100; |
| 172 | |
| 173 | /* We want a short delay mostly to emulate likely code, and |
| 174 | * we want a long delay occasionally to force massive contention. |
| 175 | */ |
| 176 | if (!(torture_random(trsp) % |
| 177 | (nrealwriters_stress * 2000 * longdelay_us))) |
| 178 | mdelay(longdelay_us); |
| 179 | if (!(torture_random(trsp) % |
| 180 | (nrealwriters_stress * 2 * shortdelay_us))) |
| 181 | udelay(shortdelay_us); |
| 182 | #ifdef CONFIG_PREEMPT |
| 183 | if (!(torture_random(trsp) % (nrealwriters_stress * 20000))) |
| 184 | preempt_schedule(); /* Allow test to be preempted. */ |
| 185 | #endif |
| 186 | } |
| 187 | |
| 188 | static void torture_spin_lock_write_unlock(void) __releases(torture_spinlock) |
| 189 | { |
| 190 | spin_unlock(&torture_spinlock); |
| 191 | } |
| 192 | |
| 193 | static struct lock_torture_ops spin_lock_ops = { |
| 194 | .writelock = torture_spin_lock_write_lock, |
| 195 | .write_delay = torture_spin_lock_write_delay, |
| 196 | .writeunlock = torture_spin_lock_write_unlock, |
Davidlohr Bueso | 4f6332c | 2014-09-11 21:40:41 -0700 | [diff] [blame^] | 197 | .readlock = NULL, |
| 198 | .read_delay = NULL, |
| 199 | .readunlock = NULL, |
Paul E. McKenney | 0af3fe1 | 2014-02-04 15:51:41 -0800 | [diff] [blame] | 200 | .name = "spin_lock" |
| 201 | }; |
| 202 | |
| 203 | static int torture_spin_lock_write_lock_irq(void) |
| 204 | __acquires(torture_spinlock_irq) |
| 205 | { |
| 206 | unsigned long flags; |
| 207 | |
| 208 | spin_lock_irqsave(&torture_spinlock, flags); |
| 209 | cur_ops->flags = flags; |
| 210 | return 0; |
| 211 | } |
| 212 | |
| 213 | static void torture_lock_spin_write_unlock_irq(void) |
| 214 | __releases(torture_spinlock) |
| 215 | { |
| 216 | spin_unlock_irqrestore(&torture_spinlock, cur_ops->flags); |
| 217 | } |
| 218 | |
| 219 | static struct lock_torture_ops spin_lock_irq_ops = { |
| 220 | .writelock = torture_spin_lock_write_lock_irq, |
| 221 | .write_delay = torture_spin_lock_write_delay, |
| 222 | .writeunlock = torture_lock_spin_write_unlock_irq, |
Davidlohr Bueso | 4f6332c | 2014-09-11 21:40:41 -0700 | [diff] [blame^] | 223 | .readlock = NULL, |
| 224 | .read_delay = NULL, |
| 225 | .readunlock = NULL, |
Paul E. McKenney | 0af3fe1 | 2014-02-04 15:51:41 -0800 | [diff] [blame] | 226 | .name = "spin_lock_irq" |
| 227 | }; |
| 228 | |
Davidlohr Bueso | 42ddc75 | 2014-09-11 20:40:18 -0700 | [diff] [blame] | 229 | static DEFINE_MUTEX(torture_mutex); |
| 230 | |
| 231 | static int torture_mutex_lock(void) __acquires(torture_mutex) |
| 232 | { |
| 233 | mutex_lock(&torture_mutex); |
| 234 | return 0; |
| 235 | } |
| 236 | |
| 237 | static void torture_mutex_delay(struct torture_random_state *trsp) |
| 238 | { |
| 239 | const unsigned long longdelay_ms = 100; |
| 240 | |
| 241 | /* We want a long delay occasionally to force massive contention. */ |
| 242 | if (!(torture_random(trsp) % |
| 243 | (nrealwriters_stress * 2000 * longdelay_ms))) |
| 244 | mdelay(longdelay_ms * 5); |
| 245 | else |
| 246 | mdelay(longdelay_ms / 5); |
| 247 | #ifdef CONFIG_PREEMPT |
| 248 | if (!(torture_random(trsp) % (nrealwriters_stress * 20000))) |
| 249 | preempt_schedule(); /* Allow test to be preempted. */ |
| 250 | #endif |
| 251 | } |
| 252 | |
| 253 | static void torture_mutex_unlock(void) __releases(torture_mutex) |
| 254 | { |
| 255 | mutex_unlock(&torture_mutex); |
| 256 | } |
| 257 | |
| 258 | static struct lock_torture_ops mutex_lock_ops = { |
| 259 | .writelock = torture_mutex_lock, |
| 260 | .write_delay = torture_mutex_delay, |
| 261 | .writeunlock = torture_mutex_unlock, |
Davidlohr Bueso | 4f6332c | 2014-09-11 21:40:41 -0700 | [diff] [blame^] | 262 | .readlock = NULL, |
| 263 | .read_delay = NULL, |
| 264 | .readunlock = NULL, |
Davidlohr Bueso | 42ddc75 | 2014-09-11 20:40:18 -0700 | [diff] [blame] | 265 | .name = "mutex_lock" |
| 266 | }; |
| 267 | |
Paul E. McKenney | 0af3fe1 | 2014-02-04 15:51:41 -0800 | [diff] [blame] | 268 | /* |
| 269 | * Lock torture writer kthread. Repeatedly acquires and releases |
| 270 | * the lock, checking for duplicate acquisitions. |
| 271 | */ |
| 272 | static int lock_torture_writer(void *arg) |
| 273 | { |
Davidlohr Bueso | 1e6757a | 2014-09-11 20:40:20 -0700 | [diff] [blame] | 274 | struct lock_stress_stats *lwsp = arg; |
Paul E. McKenney | 0af3fe1 | 2014-02-04 15:51:41 -0800 | [diff] [blame] | 275 | static DEFINE_TORTURE_RANDOM(rand); |
| 276 | |
| 277 | VERBOSE_TOROUT_STRING("lock_torture_writer task started"); |
Dongsheng Yang | 8698a74 | 2014-03-11 18:09:12 +0800 | [diff] [blame] | 278 | set_user_nice(current, MAX_NICE); |
Paul E. McKenney | 0af3fe1 | 2014-02-04 15:51:41 -0800 | [diff] [blame] | 279 | |
| 280 | do { |
Paul E. McKenney | da601c6 | 2014-02-26 12:14:51 -0800 | [diff] [blame] | 281 | if ((torture_random(&rand) & 0xfffff) == 0) |
| 282 | schedule_timeout_uninterruptible(1); |
Paul E. McKenney | 0af3fe1 | 2014-02-04 15:51:41 -0800 | [diff] [blame] | 283 | cur_ops->writelock(); |
| 284 | if (WARN_ON_ONCE(lock_is_write_held)) |
Davidlohr Bueso | 1e6757a | 2014-09-11 20:40:20 -0700 | [diff] [blame] | 285 | lwsp->n_lock_fail++; |
Paul E. McKenney | 0af3fe1 | 2014-02-04 15:51:41 -0800 | [diff] [blame] | 286 | lock_is_write_held = 1; |
Davidlohr Bueso | 1e6757a | 2014-09-11 20:40:20 -0700 | [diff] [blame] | 287 | lwsp->n_lock_acquired++; |
Paul E. McKenney | 0af3fe1 | 2014-02-04 15:51:41 -0800 | [diff] [blame] | 288 | cur_ops->write_delay(&rand); |
| 289 | lock_is_write_held = 0; |
| 290 | cur_ops->writeunlock(); |
| 291 | stutter_wait("lock_torture_writer"); |
| 292 | } while (!torture_must_stop()); |
| 293 | torture_kthread_stopping("lock_torture_writer"); |
| 294 | return 0; |
| 295 | } |
| 296 | |
| 297 | /* |
Davidlohr Bueso | 4f6332c | 2014-09-11 21:40:41 -0700 | [diff] [blame^] | 298 | * Lock torture reader kthread. Repeatedly acquires and releases |
| 299 | * the reader lock. |
| 300 | */ |
| 301 | static int lock_torture_reader(void *arg) |
| 302 | { |
| 303 | struct lock_stress_stats *lrsp = arg; |
| 304 | static DEFINE_TORTURE_RANDOM(rand); |
| 305 | |
| 306 | VERBOSE_TOROUT_STRING("lock_torture_reader task started"); |
| 307 | set_user_nice(current, MAX_NICE); |
| 308 | |
| 309 | do { |
| 310 | if ((torture_random(&rand) & 0xfffff) == 0) |
| 311 | schedule_timeout_uninterruptible(1); |
| 312 | cur_ops->readlock(); |
| 313 | lock_is_read_held = 1; |
| 314 | lrsp->n_lock_acquired++; |
| 315 | cur_ops->read_delay(&rand); |
| 316 | lock_is_read_held = 0; |
| 317 | cur_ops->readunlock(); |
| 318 | stutter_wait("lock_torture_reader"); |
| 319 | } while (!torture_must_stop()); |
| 320 | torture_kthread_stopping("lock_torture_reader"); |
| 321 | return 0; |
| 322 | } |
| 323 | |
| 324 | /* |
Paul E. McKenney | 0af3fe1 | 2014-02-04 15:51:41 -0800 | [diff] [blame] | 325 | * Create an lock-torture-statistics message in the specified buffer. |
| 326 | */ |
Davidlohr Bueso | 4f6332c | 2014-09-11 21:40:41 -0700 | [diff] [blame^] | 327 | static void __torture_print_stats(char *page, |
| 328 | struct lock_stress_stats *statp, bool write) |
Paul E. McKenney | 0af3fe1 | 2014-02-04 15:51:41 -0800 | [diff] [blame] | 329 | { |
| 330 | bool fail = 0; |
Davidlohr Bueso | 4f6332c | 2014-09-11 21:40:41 -0700 | [diff] [blame^] | 331 | int i, n_stress; |
Paul E. McKenney | 0af3fe1 | 2014-02-04 15:51:41 -0800 | [diff] [blame] | 332 | long max = 0; |
Davidlohr Bueso | 4f6332c | 2014-09-11 21:40:41 -0700 | [diff] [blame^] | 333 | long min = statp[0].n_lock_acquired; |
Paul E. McKenney | 0af3fe1 | 2014-02-04 15:51:41 -0800 | [diff] [blame] | 334 | long long sum = 0; |
| 335 | |
Davidlohr Bueso | 4f6332c | 2014-09-11 21:40:41 -0700 | [diff] [blame^] | 336 | n_stress = write ? nrealwriters_stress : nrealreaders_stress; |
| 337 | for (i = 0; i < n_stress; i++) { |
| 338 | if (statp[i].n_lock_fail) |
Paul E. McKenney | 0af3fe1 | 2014-02-04 15:51:41 -0800 | [diff] [blame] | 339 | fail = true; |
Davidlohr Bueso | 4f6332c | 2014-09-11 21:40:41 -0700 | [diff] [blame^] | 340 | sum += statp[i].n_lock_acquired; |
| 341 | if (max < statp[i].n_lock_fail) |
| 342 | max = statp[i].n_lock_fail; |
| 343 | if (min > statp[i].n_lock_fail) |
| 344 | min = statp[i].n_lock_fail; |
Paul E. McKenney | 0af3fe1 | 2014-02-04 15:51:41 -0800 | [diff] [blame] | 345 | } |
Paul E. McKenney | 0af3fe1 | 2014-02-04 15:51:41 -0800 | [diff] [blame] | 346 | page += sprintf(page, |
Davidlohr Bueso | 4f6332c | 2014-09-11 21:40:41 -0700 | [diff] [blame^] | 347 | "%s: Total: %lld Max/Min: %ld/%ld %s Fail: %d %s\n", |
| 348 | write ? "Writes" : "Reads ", |
Paul E. McKenney | 0af3fe1 | 2014-02-04 15:51:41 -0800 | [diff] [blame] | 349 | sum, max, min, max / 2 > min ? "???" : "", |
| 350 | fail, fail ? "!!!" : ""); |
| 351 | if (fail) |
| 352 | atomic_inc(&n_lock_torture_errors); |
| 353 | } |
| 354 | |
| 355 | /* |
| 356 | * Print torture statistics. Caller must ensure that there is only one |
| 357 | * call to this function at a given time!!! This is normally accomplished |
| 358 | * by relying on the module system to only have one copy of the module |
| 359 | * loaded, and then by giving the lock_torture_stats kthread full control |
| 360 | * (or the init/cleanup functions when lock_torture_stats thread is not |
| 361 | * running). |
| 362 | */ |
| 363 | static void lock_torture_stats_print(void) |
| 364 | { |
| 365 | int size = nrealwriters_stress * 200 + 8192; |
| 366 | char *buf; |
| 367 | |
Davidlohr Bueso | 4f6332c | 2014-09-11 21:40:41 -0700 | [diff] [blame^] | 368 | if (cur_ops->readlock) |
| 369 | size += nrealreaders_stress * 200 + 8192; |
| 370 | |
Paul E. McKenney | 0af3fe1 | 2014-02-04 15:51:41 -0800 | [diff] [blame] | 371 | buf = kmalloc(size, GFP_KERNEL); |
| 372 | if (!buf) { |
| 373 | pr_err("lock_torture_stats_print: Out of memory, need: %d", |
| 374 | size); |
| 375 | return; |
| 376 | } |
Davidlohr Bueso | 4f6332c | 2014-09-11 21:40:41 -0700 | [diff] [blame^] | 377 | |
| 378 | __torture_print_stats(buf, lwsa, true); |
Paul E. McKenney | 0af3fe1 | 2014-02-04 15:51:41 -0800 | [diff] [blame] | 379 | pr_alert("%s", buf); |
| 380 | kfree(buf); |
Davidlohr Bueso | 4f6332c | 2014-09-11 21:40:41 -0700 | [diff] [blame^] | 381 | |
| 382 | if (cur_ops->readlock) { |
| 383 | buf = kmalloc(size, GFP_KERNEL); |
| 384 | if (!buf) { |
| 385 | pr_err("lock_torture_stats_print: Out of memory, need: %d", |
| 386 | size); |
| 387 | return; |
| 388 | } |
| 389 | |
| 390 | __torture_print_stats(buf, lrsa, false); |
| 391 | pr_alert("%s", buf); |
| 392 | kfree(buf); |
| 393 | } |
Paul E. McKenney | 0af3fe1 | 2014-02-04 15:51:41 -0800 | [diff] [blame] | 394 | } |
| 395 | |
| 396 | /* |
| 397 | * Periodically prints torture statistics, if periodic statistics printing |
| 398 | * was specified via the stat_interval module parameter. |
| 399 | * |
| 400 | * No need to worry about fullstop here, since this one doesn't reference |
| 401 | * volatile state or register callbacks. |
| 402 | */ |
| 403 | static int lock_torture_stats(void *arg) |
| 404 | { |
| 405 | VERBOSE_TOROUT_STRING("lock_torture_stats task started"); |
| 406 | do { |
| 407 | schedule_timeout_interruptible(stat_interval * HZ); |
| 408 | lock_torture_stats_print(); |
| 409 | torture_shutdown_absorb("lock_torture_stats"); |
| 410 | } while (!torture_must_stop()); |
| 411 | torture_kthread_stopping("lock_torture_stats"); |
| 412 | return 0; |
| 413 | } |
| 414 | |
| 415 | static inline void |
| 416 | lock_torture_print_module_parms(struct lock_torture_ops *cur_ops, |
| 417 | const char *tag) |
| 418 | { |
| 419 | pr_alert("%s" TORTURE_FLAG |
Davidlohr Bueso | 4f6332c | 2014-09-11 21:40:41 -0700 | [diff] [blame^] | 420 | "--- %s%s: nwriters_stress=%d nreaders_stress=%d stat_interval=%d verbose=%d shuffle_interval=%d stutter=%d shutdown_secs=%d onoff_interval=%d onoff_holdoff=%d\n", |
Davidlohr Bueso | f095bfc | 2014-09-11 20:40:19 -0700 | [diff] [blame] | 421 | torture_type, tag, debug_lock ? " [debug]": "", |
Davidlohr Bueso | 4f6332c | 2014-09-11 21:40:41 -0700 | [diff] [blame^] | 422 | nrealwriters_stress, nrealreaders_stress, stat_interval, |
| 423 | verbose, shuffle_interval, stutter, shutdown_secs, |
Paul E. McKenney | 0af3fe1 | 2014-02-04 15:51:41 -0800 | [diff] [blame] | 424 | onoff_interval, onoff_holdoff); |
| 425 | } |
| 426 | |
| 427 | static void lock_torture_cleanup(void) |
| 428 | { |
| 429 | int i; |
| 430 | |
Davidlohr Bueso | d36a7a0 | 2014-09-11 20:40:21 -0700 | [diff] [blame] | 431 | if (torture_cleanup_begin()) |
Paul E. McKenney | 0af3fe1 | 2014-02-04 15:51:41 -0800 | [diff] [blame] | 432 | return; |
| 433 | |
| 434 | if (writer_tasks) { |
| 435 | for (i = 0; i < nrealwriters_stress; i++) |
| 436 | torture_stop_kthread(lock_torture_writer, |
| 437 | writer_tasks[i]); |
| 438 | kfree(writer_tasks); |
| 439 | writer_tasks = NULL; |
| 440 | } |
| 441 | |
Davidlohr Bueso | 4f6332c | 2014-09-11 21:40:41 -0700 | [diff] [blame^] | 442 | if (reader_tasks) { |
| 443 | for (i = 0; i < nrealreaders_stress; i++) |
| 444 | torture_stop_kthread(lock_torture_reader, |
| 445 | reader_tasks[i]); |
| 446 | kfree(reader_tasks); |
| 447 | reader_tasks = NULL; |
| 448 | } |
| 449 | |
Paul E. McKenney | 0af3fe1 | 2014-02-04 15:51:41 -0800 | [diff] [blame] | 450 | torture_stop_kthread(lock_torture_stats, stats_task); |
| 451 | lock_torture_stats_print(); /* -After- the stats thread is stopped! */ |
| 452 | |
| 453 | if (atomic_read(&n_lock_torture_errors)) |
| 454 | lock_torture_print_module_parms(cur_ops, |
| 455 | "End of test: FAILURE"); |
| 456 | else if (torture_onoff_failures()) |
| 457 | lock_torture_print_module_parms(cur_ops, |
| 458 | "End of test: LOCK_HOTPLUG"); |
| 459 | else |
| 460 | lock_torture_print_module_parms(cur_ops, |
| 461 | "End of test: SUCCESS"); |
Davidlohr Bueso | d36a7a0 | 2014-09-11 20:40:21 -0700 | [diff] [blame] | 462 | torture_cleanup_end(); |
Paul E. McKenney | 0af3fe1 | 2014-02-04 15:51:41 -0800 | [diff] [blame] | 463 | } |
| 464 | |
| 465 | static int __init lock_torture_init(void) |
| 466 | { |
Davidlohr Bueso | 4f6332c | 2014-09-11 21:40:41 -0700 | [diff] [blame^] | 467 | int i, j; |
Paul E. McKenney | 0af3fe1 | 2014-02-04 15:51:41 -0800 | [diff] [blame] | 468 | int firsterr = 0; |
| 469 | static struct lock_torture_ops *torture_ops[] = { |
Davidlohr Bueso | 42ddc75 | 2014-09-11 20:40:18 -0700 | [diff] [blame] | 470 | &lock_busted_ops, &spin_lock_ops, &spin_lock_irq_ops, &mutex_lock_ops, |
Paul E. McKenney | 0af3fe1 | 2014-02-04 15:51:41 -0800 | [diff] [blame] | 471 | }; |
| 472 | |
Davidlohr Bueso | 23a8e5c | 2014-09-11 20:40:16 -0700 | [diff] [blame] | 473 | if (!torture_init_begin(torture_type, verbose, &torture_runnable)) |
Paul E. McKenney | 5228084 | 2014-04-07 09:14:11 -0700 | [diff] [blame] | 474 | return -EBUSY; |
Paul E. McKenney | 0af3fe1 | 2014-02-04 15:51:41 -0800 | [diff] [blame] | 475 | |
| 476 | /* Process args and tell the world that the torturer is on the job. */ |
| 477 | for (i = 0; i < ARRAY_SIZE(torture_ops); i++) { |
| 478 | cur_ops = torture_ops[i]; |
| 479 | if (strcmp(torture_type, cur_ops->name) == 0) |
| 480 | break; |
| 481 | } |
| 482 | if (i == ARRAY_SIZE(torture_ops)) { |
| 483 | pr_alert("lock-torture: invalid torture type: \"%s\"\n", |
| 484 | torture_type); |
| 485 | pr_alert("lock-torture types:"); |
| 486 | for (i = 0; i < ARRAY_SIZE(torture_ops); i++) |
| 487 | pr_alert(" %s", torture_ops[i]->name); |
| 488 | pr_alert("\n"); |
| 489 | torture_init_end(); |
| 490 | return -EINVAL; |
| 491 | } |
| 492 | if (cur_ops->init) |
| 493 | cur_ops->init(); /* no "goto unwind" prior to this point!!! */ |
| 494 | |
| 495 | if (nwriters_stress >= 0) |
| 496 | nrealwriters_stress = nwriters_stress; |
| 497 | else |
| 498 | nrealwriters_stress = 2 * num_online_cpus(); |
Davidlohr Bueso | f095bfc | 2014-09-11 20:40:19 -0700 | [diff] [blame] | 499 | |
| 500 | #ifdef CONFIG_DEBUG_MUTEXES |
| 501 | if (strncmp(torture_type, "mutex", 5) == 0) |
| 502 | debug_lock = true; |
| 503 | #endif |
| 504 | #ifdef CONFIG_DEBUG_SPINLOCK |
| 505 | if (strncmp(torture_type, "spin", 4) == 0) |
| 506 | debug_lock = true; |
| 507 | #endif |
Paul E. McKenney | 0af3fe1 | 2014-02-04 15:51:41 -0800 | [diff] [blame] | 508 | |
| 509 | /* Initialize the statistics so that each run gets its own numbers. */ |
| 510 | |
| 511 | lock_is_write_held = 0; |
| 512 | lwsa = kmalloc(sizeof(*lwsa) * nrealwriters_stress, GFP_KERNEL); |
| 513 | if (lwsa == NULL) { |
| 514 | VERBOSE_TOROUT_STRING("lwsa: Out of memory"); |
| 515 | firsterr = -ENOMEM; |
| 516 | goto unwind; |
| 517 | } |
| 518 | for (i = 0; i < nrealwriters_stress; i++) { |
Davidlohr Bueso | 1e6757a | 2014-09-11 20:40:20 -0700 | [diff] [blame] | 519 | lwsa[i].n_lock_fail = 0; |
| 520 | lwsa[i].n_lock_acquired = 0; |
Paul E. McKenney | 0af3fe1 | 2014-02-04 15:51:41 -0800 | [diff] [blame] | 521 | } |
| 522 | |
Davidlohr Bueso | 4f6332c | 2014-09-11 21:40:41 -0700 | [diff] [blame^] | 523 | if (cur_ops->readlock) { |
| 524 | if (nreaders_stress >= 0) |
| 525 | nrealreaders_stress = nreaders_stress; |
| 526 | else { |
| 527 | /* |
| 528 | * By default distribute evenly the number of |
| 529 | * readers and writers. We still run the same number |
| 530 | * of threads as the writer-only locks default. |
| 531 | */ |
| 532 | if (nwriters_stress < 0) /* user doesn't care */ |
| 533 | nrealwriters_stress = num_online_cpus(); |
| 534 | nrealreaders_stress = nrealwriters_stress; |
| 535 | } |
Paul E. McKenney | 0af3fe1 | 2014-02-04 15:51:41 -0800 | [diff] [blame] | 536 | |
Davidlohr Bueso | 4f6332c | 2014-09-11 21:40:41 -0700 | [diff] [blame^] | 537 | lock_is_read_held = 0; |
| 538 | lrsa = kmalloc(sizeof(*lrsa) * nrealreaders_stress, GFP_KERNEL); |
| 539 | if (lrsa == NULL) { |
| 540 | VERBOSE_TOROUT_STRING("lrsa: Out of memory"); |
| 541 | firsterr = -ENOMEM; |
| 542 | kfree(lwsa); |
| 543 | goto unwind; |
| 544 | } |
| 545 | |
| 546 | for (i = 0; i < nrealreaders_stress; i++) { |
| 547 | lrsa[i].n_lock_fail = 0; |
| 548 | lrsa[i].n_lock_acquired = 0; |
| 549 | } |
| 550 | } |
| 551 | lock_torture_print_module_parms(cur_ops, "Start of test"); |
| 552 | |
| 553 | /* Prepare torture context. */ |
Paul E. McKenney | 0af3fe1 | 2014-02-04 15:51:41 -0800 | [diff] [blame] | 554 | if (onoff_interval > 0) { |
| 555 | firsterr = torture_onoff_init(onoff_holdoff * HZ, |
| 556 | onoff_interval * HZ); |
| 557 | if (firsterr) |
| 558 | goto unwind; |
| 559 | } |
| 560 | if (shuffle_interval > 0) { |
| 561 | firsterr = torture_shuffle_init(shuffle_interval); |
| 562 | if (firsterr) |
| 563 | goto unwind; |
| 564 | } |
| 565 | if (shutdown_secs > 0) { |
| 566 | firsterr = torture_shutdown_init(shutdown_secs, |
| 567 | lock_torture_cleanup); |
| 568 | if (firsterr) |
| 569 | goto unwind; |
| 570 | } |
| 571 | if (stutter > 0) { |
| 572 | firsterr = torture_stutter_init(stutter); |
| 573 | if (firsterr) |
| 574 | goto unwind; |
| 575 | } |
| 576 | |
| 577 | writer_tasks = kzalloc(nrealwriters_stress * sizeof(writer_tasks[0]), |
| 578 | GFP_KERNEL); |
| 579 | if (writer_tasks == NULL) { |
| 580 | VERBOSE_TOROUT_ERRSTRING("writer_tasks: Out of memory"); |
| 581 | firsterr = -ENOMEM; |
| 582 | goto unwind; |
| 583 | } |
Davidlohr Bueso | 4f6332c | 2014-09-11 21:40:41 -0700 | [diff] [blame^] | 584 | |
| 585 | if (cur_ops->readlock) { |
| 586 | reader_tasks = kzalloc(nrealreaders_stress * sizeof(reader_tasks[0]), |
| 587 | GFP_KERNEL); |
| 588 | if (reader_tasks == NULL) { |
| 589 | VERBOSE_TOROUT_ERRSTRING("reader_tasks: Out of memory"); |
| 590 | firsterr = -ENOMEM; |
| 591 | goto unwind; |
| 592 | } |
| 593 | } |
| 594 | |
| 595 | /* |
| 596 | * Create the kthreads and start torturing (oh, those poor little locks). |
| 597 | * |
| 598 | * TODO: Note that we interleave writers with readers, giving writers a |
| 599 | * slight advantage, by creating its kthread first. This can be modified |
| 600 | * for very specific needs, or even let the user choose the policy, if |
| 601 | * ever wanted. |
| 602 | */ |
| 603 | for (i = 0, j = 0; i < nrealwriters_stress || |
| 604 | j < nrealreaders_stress; i++, j++) { |
| 605 | if (i >= nrealwriters_stress) |
| 606 | goto create_reader; |
| 607 | |
| 608 | /* Create writer. */ |
Paul E. McKenney | 0af3fe1 | 2014-02-04 15:51:41 -0800 | [diff] [blame] | 609 | firsterr = torture_create_kthread(lock_torture_writer, &lwsa[i], |
| 610 | writer_tasks[i]); |
| 611 | if (firsterr) |
| 612 | goto unwind; |
Davidlohr Bueso | 4f6332c | 2014-09-11 21:40:41 -0700 | [diff] [blame^] | 613 | |
| 614 | create_reader: |
| 615 | if (cur_ops->readlock == NULL || (j >= nrealreaders_stress)) |
| 616 | continue; |
| 617 | /* Create reader. */ |
| 618 | firsterr = torture_create_kthread(lock_torture_reader, &lrsa[j], |
| 619 | reader_tasks[j]); |
| 620 | if (firsterr) |
| 621 | goto unwind; |
Paul E. McKenney | 0af3fe1 | 2014-02-04 15:51:41 -0800 | [diff] [blame] | 622 | } |
| 623 | if (stat_interval > 0) { |
| 624 | firsterr = torture_create_kthread(lock_torture_stats, NULL, |
| 625 | stats_task); |
| 626 | if (firsterr) |
| 627 | goto unwind; |
| 628 | } |
| 629 | torture_init_end(); |
| 630 | return 0; |
| 631 | |
| 632 | unwind: |
| 633 | torture_init_end(); |
| 634 | lock_torture_cleanup(); |
| 635 | return firsterr; |
| 636 | } |
| 637 | |
| 638 | module_init(lock_torture_init); |
| 639 | module_exit(lock_torture_cleanup); |