blob: 2592f215a905a63e721b692aa06866f857a0d04b [file] [log] [blame]
Jens Axboe2056a782006-03-23 20:00:26 +01001/*
2 * Copyright (C) 2006 Jens Axboe <axboe@suse.de>
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as
6 * published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope that it will be useful,
9 * but WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 * GNU General Public License for more details.
12 *
13 * You should have received a copy of the GNU General Public License
14 * along with this program; if not, write to the Free Software
15 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
16 *
17 */
Jens Axboe2056a782006-03-23 20:00:26 +010018#include <linux/kernel.h>
19#include <linux/blkdev.h>
20#include <linux/blktrace_api.h>
21#include <linux/percpu.h>
22#include <linux/init.h>
23#include <linux/mutex.h>
24#include <linux/debugfs.h>
25#include <asm/uaccess.h>
26
27static DEFINE_PER_CPU(unsigned long long, blk_trace_cpu_offset) = { 0, };
28static unsigned int blktrace_seq __read_mostly = 1;
29
30/*
31 * Send out a notify for this process, if we haven't done so since a trace
32 * started
33 */
34static void trace_note_tsk(struct blk_trace *bt, struct task_struct *tsk)
35{
36 struct blk_io_trace *t;
37
38 t = relay_reserve(bt->rchan, sizeof(*t) + sizeof(tsk->comm));
39 if (t) {
40 t->magic = BLK_IO_TRACE_MAGIC | BLK_IO_TRACE_VERSION;
41 t->device = bt->dev;
42 t->action = BLK_TC_ACT(BLK_TC_NOTIFY);
43 t->pid = tsk->pid;
44 t->cpu = smp_processor_id();
45 t->pdu_len = sizeof(tsk->comm);
46 memcpy((void *) t + sizeof(*t), tsk->comm, t->pdu_len);
47 tsk->btrace_seq = blktrace_seq;
48 }
49}
50
51static int act_log_check(struct blk_trace *bt, u32 what, sector_t sector,
52 pid_t pid)
53{
54 if (((bt->act_mask << BLK_TC_SHIFT) & what) == 0)
55 return 1;
56 if (sector < bt->start_lba || sector > bt->end_lba)
57 return 1;
58 if (bt->pid && pid != bt->pid)
59 return 1;
60
61 return 0;
62}
63
64/*
65 * Data direction bit lookup
66 */
67static u32 ddir_act[2] __read_mostly = { BLK_TC_ACT(BLK_TC_READ), BLK_TC_ACT(BLK_TC_WRITE) };
68
69/*
70 * Bio action bits of interest
71 */
Jens Axboe7457e6e2006-07-23 02:12:01 +020072static u32 bio_act[9] __read_mostly = { 0, BLK_TC_ACT(BLK_TC_BARRIER), BLK_TC_ACT(BLK_TC_SYNC), 0, BLK_TC_ACT(BLK_TC_AHEAD), 0, 0, 0, BLK_TC_ACT(BLK_TC_META) };
Jens Axboe2056a782006-03-23 20:00:26 +010073
74/*
75 * More could be added as needed, taking care to increment the decrementer
76 * to get correct indexing
77 */
78#define trace_barrier_bit(rw) \
79 (((rw) & (1 << BIO_RW_BARRIER)) >> (BIO_RW_BARRIER - 0))
80#define trace_sync_bit(rw) \
81 (((rw) & (1 << BIO_RW_SYNC)) >> (BIO_RW_SYNC - 1))
Nathan Scott40359cc2006-07-06 10:03:28 +020082#define trace_ahead_bit(rw) \
Milton Millerad01b1c2006-07-25 15:04:13 +020083 (((rw) & (1 << BIO_RW_AHEAD)) << (2 - BIO_RW_AHEAD))
Jens Axboe7457e6e2006-07-23 02:12:01 +020084#define trace_meta_bit(rw) \
85 (((rw) & (1 << BIO_RW_META)) >> (BIO_RW_META - 3))
Jens Axboe2056a782006-03-23 20:00:26 +010086
87/*
88 * The worker for the various blk_add_trace*() types. Fills out a
89 * blk_io_trace structure and places it in a per-cpu subbuffer.
90 */
91void __blk_add_trace(struct blk_trace *bt, sector_t sector, int bytes,
92 int rw, u32 what, int error, int pdu_len, void *pdu_data)
93{
94 struct task_struct *tsk = current;
95 struct blk_io_trace *t;
96 unsigned long flags;
97 unsigned long *sequence;
98 pid_t pid;
99 int cpu;
100
101 if (unlikely(bt->trace_state != Blktrace_running))
102 return;
103
104 what |= ddir_act[rw & WRITE];
105 what |= bio_act[trace_barrier_bit(rw)];
106 what |= bio_act[trace_sync_bit(rw)];
Nathan Scott40359cc2006-07-06 10:03:28 +0200107 what |= bio_act[trace_ahead_bit(rw)];
Jens Axboe7457e6e2006-07-23 02:12:01 +0200108 what |= bio_act[trace_meta_bit(rw)];
Jens Axboe2056a782006-03-23 20:00:26 +0100109
110 pid = tsk->pid;
111 if (unlikely(act_log_check(bt, what, sector, pid)))
112 return;
113
114 /*
115 * A word about the locking here - we disable interrupts to reserve
116 * some space in the relay per-cpu buffer, to prevent an irq
117 * from coming in and stepping on our toes. Once reserved, it's
118 * enough to get preemption disabled to prevent read of this data
119 * before we are through filling it. get_cpu()/put_cpu() does this
120 * for us
121 */
122 local_irq_save(flags);
123
124 if (unlikely(tsk->btrace_seq != blktrace_seq))
125 trace_note_tsk(bt, tsk);
126
127 t = relay_reserve(bt->rchan, sizeof(*t) + pdu_len);
128 if (t) {
129 cpu = smp_processor_id();
130 sequence = per_cpu_ptr(bt->sequence, cpu);
131
132 t->magic = BLK_IO_TRACE_MAGIC | BLK_IO_TRACE_VERSION;
133 t->sequence = ++(*sequence);
134 t->time = sched_clock() - per_cpu(blk_trace_cpu_offset, cpu);
135 t->sector = sector;
136 t->bytes = bytes;
137 t->action = what;
138 t->pid = pid;
139 t->device = bt->dev;
140 t->cpu = cpu;
141 t->error = error;
142 t->pdu_len = pdu_len;
143
144 if (pdu_len)
145 memcpy((void *) t + sizeof(*t), pdu_data, pdu_len);
146 }
147
148 local_irq_restore(flags);
149}
150
151EXPORT_SYMBOL_GPL(__blk_add_trace);
152
153static struct dentry *blk_tree_root;
154static struct mutex blk_tree_mutex;
155static unsigned int root_users;
156
157static inline void blk_remove_root(void)
158{
159 if (blk_tree_root) {
160 debugfs_remove(blk_tree_root);
161 blk_tree_root = NULL;
162 }
163}
164
165static void blk_remove_tree(struct dentry *dir)
166{
167 mutex_lock(&blk_tree_mutex);
168 debugfs_remove(dir);
169 if (--root_users == 0)
170 blk_remove_root();
171 mutex_unlock(&blk_tree_mutex);
172}
173
174static struct dentry *blk_create_tree(const char *blk_name)
175{
176 struct dentry *dir = NULL;
177
178 mutex_lock(&blk_tree_mutex);
179
180 if (!blk_tree_root) {
181 blk_tree_root = debugfs_create_dir("block", NULL);
182 if (!blk_tree_root)
183 goto err;
184 }
185
186 dir = debugfs_create_dir(blk_name, blk_tree_root);
187 if (dir)
188 root_users++;
189 else
190 blk_remove_root();
191
192err:
193 mutex_unlock(&blk_tree_mutex);
194 return dir;
195}
196
197static void blk_trace_cleanup(struct blk_trace *bt)
198{
199 relay_close(bt->rchan);
200 debugfs_remove(bt->dropped_file);
201 blk_remove_tree(bt->dir);
202 free_percpu(bt->sequence);
203 kfree(bt);
204}
205
206static int blk_trace_remove(request_queue_t *q)
207{
208 struct blk_trace *bt;
209
210 bt = xchg(&q->blk_trace, NULL);
211 if (!bt)
212 return -EINVAL;
213
214 if (bt->trace_state == Blktrace_setup ||
215 bt->trace_state == Blktrace_stopped)
216 blk_trace_cleanup(bt);
217
218 return 0;
219}
220
221static int blk_dropped_open(struct inode *inode, struct file *filp)
222{
Theodore Ts'o8e18e292006-09-27 01:50:46 -0700223 filp->private_data = inode->i_private;
Jens Axboe2056a782006-03-23 20:00:26 +0100224
225 return 0;
226}
227
228static ssize_t blk_dropped_read(struct file *filp, char __user *buffer,
229 size_t count, loff_t *ppos)
230{
231 struct blk_trace *bt = filp->private_data;
232 char buf[16];
233
234 snprintf(buf, sizeof(buf), "%u\n", atomic_read(&bt->dropped));
235
236 return simple_read_from_buffer(buffer, count, ppos, buf, strlen(buf));
237}
238
239static struct file_operations blk_dropped_fops = {
240 .owner = THIS_MODULE,
241 .open = blk_dropped_open,
242 .read = blk_dropped_read,
243};
244
245/*
246 * Keep track of how many times we encountered a full subbuffer, to aid
247 * the user space app in telling how many lost events there were.
248 */
249static int blk_subbuf_start_callback(struct rchan_buf *buf, void *subbuf,
250 void *prev_subbuf, size_t prev_padding)
251{
252 struct blk_trace *bt;
253
254 if (!relay_buf_full(buf))
255 return 1;
256
257 bt = buf->chan->private_data;
258 atomic_inc(&bt->dropped);
259 return 0;
260}
261
262static int blk_remove_buf_file_callback(struct dentry *dentry)
263{
264 debugfs_remove(dentry);
265 return 0;
266}
267
268static struct dentry *blk_create_buf_file_callback(const char *filename,
269 struct dentry *parent,
270 int mode,
271 struct rchan_buf *buf,
272 int *is_global)
273{
274 return debugfs_create_file(filename, mode, parent, buf,
275 &relay_file_operations);
276}
277
278static struct rchan_callbacks blk_relay_callbacks = {
279 .subbuf_start = blk_subbuf_start_callback,
280 .create_buf_file = blk_create_buf_file_callback,
281 .remove_buf_file = blk_remove_buf_file_callback,
282};
283
284/*
285 * Setup everything required to start tracing
286 */
287static int blk_trace_setup(request_queue_t *q, struct block_device *bdev,
288 char __user *arg)
289{
290 struct blk_user_trace_setup buts;
291 struct blk_trace *old_bt, *bt = NULL;
292 struct dentry *dir = NULL;
293 char b[BDEVNAME_SIZE];
294 int ret, i;
295
296 if (copy_from_user(&buts, arg, sizeof(buts)))
297 return -EFAULT;
298
299 if (!buts.buf_size || !buts.buf_nr)
300 return -EINVAL;
301
302 strcpy(buts.name, bdevname(bdev, b));
303
304 /*
305 * some device names have larger paths - convert the slashes
306 * to underscores for this to work as expected
307 */
308 for (i = 0; i < strlen(buts.name); i++)
309 if (buts.name[i] == '/')
310 buts.name[i] = '_';
311
312 if (copy_to_user(arg, &buts, sizeof(buts)))
313 return -EFAULT;
314
315 ret = -ENOMEM;
316 bt = kzalloc(sizeof(*bt), GFP_KERNEL);
317 if (!bt)
318 goto err;
319
320 bt->sequence = alloc_percpu(unsigned long);
321 if (!bt->sequence)
322 goto err;
323
324 ret = -ENOENT;
325 dir = blk_create_tree(buts.name);
326 if (!dir)
327 goto err;
328
329 bt->dir = dir;
330 bt->dev = bdev->bd_dev;
331 atomic_set(&bt->dropped, 0);
332
333 ret = -EIO;
334 bt->dropped_file = debugfs_create_file("dropped", 0444, dir, bt, &blk_dropped_fops);
335 if (!bt->dropped_file)
336 goto err;
337
338 bt->rchan = relay_open("trace", dir, buts.buf_size, buts.buf_nr, &blk_relay_callbacks);
339 if (!bt->rchan)
340 goto err;
341 bt->rchan->private_data = bt;
342
343 bt->act_mask = buts.act_mask;
344 if (!bt->act_mask)
345 bt->act_mask = (u16) -1;
346
347 bt->start_lba = buts.start_lba;
348 bt->end_lba = buts.end_lba;
349 if (!bt->end_lba)
350 bt->end_lba = -1ULL;
351
352 bt->pid = buts.pid;
353 bt->trace_state = Blktrace_setup;
354
355 ret = -EBUSY;
356 old_bt = xchg(&q->blk_trace, bt);
357 if (old_bt) {
358 (void) xchg(&q->blk_trace, old_bt);
359 goto err;
360 }
361
362 return 0;
363err:
364 if (dir)
365 blk_remove_tree(dir);
366 if (bt) {
367 if (bt->dropped_file)
368 debugfs_remove(bt->dropped_file);
369 if (bt->sequence)
370 free_percpu(bt->sequence);
371 if (bt->rchan)
372 relay_close(bt->rchan);
373 kfree(bt);
374 }
375 return ret;
376}
377
378static int blk_trace_startstop(request_queue_t *q, int start)
379{
380 struct blk_trace *bt;
381 int ret;
382
383 if ((bt = q->blk_trace) == NULL)
384 return -EINVAL;
385
386 /*
387 * For starting a trace, we can transition from a setup or stopped
388 * trace. For stopping a trace, the state must be running
389 */
390 ret = -EINVAL;
391 if (start) {
392 if (bt->trace_state == Blktrace_setup ||
393 bt->trace_state == Blktrace_stopped) {
394 blktrace_seq++;
395 smp_mb();
396 bt->trace_state = Blktrace_running;
397 ret = 0;
398 }
399 } else {
400 if (bt->trace_state == Blktrace_running) {
401 bt->trace_state = Blktrace_stopped;
402 relay_flush(bt->rchan);
403 ret = 0;
404 }
405 }
406
407 return ret;
408}
409
410/**
411 * blk_trace_ioctl: - handle the ioctls associated with tracing
412 * @bdev: the block device
413 * @cmd: the ioctl cmd
414 * @arg: the argument data, if any
415 *
416 **/
417int blk_trace_ioctl(struct block_device *bdev, unsigned cmd, char __user *arg)
418{
419 request_queue_t *q;
420 int ret, start = 0;
421
422 q = bdev_get_queue(bdev);
423 if (!q)
424 return -ENXIO;
425
426 mutex_lock(&bdev->bd_mutex);
427
428 switch (cmd) {
429 case BLKTRACESETUP:
430 ret = blk_trace_setup(q, bdev, arg);
431 break;
432 case BLKTRACESTART:
433 start = 1;
434 case BLKTRACESTOP:
435 ret = blk_trace_startstop(q, start);
436 break;
437 case BLKTRACETEARDOWN:
438 ret = blk_trace_remove(q);
439 break;
440 default:
441 ret = -ENOTTY;
442 break;
443 }
444
445 mutex_unlock(&bdev->bd_mutex);
446 return ret;
447}
448
449/**
450 * blk_trace_shutdown: - stop and cleanup trace structures
451 * @q: the request queue associated with the device
452 *
453 **/
454void blk_trace_shutdown(request_queue_t *q)
455{
Alexey Dobriyan6c5c9342006-09-29 01:59:40 -0700456 if (q->blk_trace) {
457 blk_trace_startstop(q, 0);
458 blk_trace_remove(q);
459 }
Jens Axboe2056a782006-03-23 20:00:26 +0100460}
461
462/*
463 * Average offset over two calls to sched_clock() with a gettimeofday()
464 * in the middle
465 */
466static void blk_check_time(unsigned long long *t)
467{
468 unsigned long long a, b;
469 struct timeval tv;
470
471 a = sched_clock();
472 do_gettimeofday(&tv);
473 b = sched_clock();
474
475 *t = tv.tv_sec * 1000000000 + tv.tv_usec * 1000;
476 *t -= (a + b) / 2;
477}
478
479static void blk_trace_check_cpu_time(void *data)
480{
481 unsigned long long *t;
482 int cpu = get_cpu();
483
484 t = &per_cpu(blk_trace_cpu_offset, cpu);
485
486 /*
487 * Just call it twice, hopefully the second call will be cache hot
488 * and a little more precise
489 */
490 blk_check_time(t);
491 blk_check_time(t);
492
493 put_cpu();
494}
495
496/*
497 * Call blk_trace_check_cpu_time() on each CPU to calibrate our inter-CPU
498 * timings
499 */
500static void blk_trace_calibrate_offsets(void)
501{
502 unsigned long flags;
503
504 smp_call_function(blk_trace_check_cpu_time, NULL, 1, 1);
505 local_irq_save(flags);
506 blk_trace_check_cpu_time(NULL);
507 local_irq_restore(flags);
508}
509
510static void blk_trace_set_ht_offsets(void)
511{
512#if defined(CONFIG_SCHED_SMT)
513 int cpu, i;
514
515 /*
516 * now make sure HT siblings have the same time offset
517 */
518 preempt_disable();
519 for_each_online_cpu(cpu) {
520 unsigned long long *cpu_off, *sibling_off;
521
522 for_each_cpu_mask(i, cpu_sibling_map[cpu]) {
523 if (i == cpu)
524 continue;
525
526 cpu_off = &per_cpu(blk_trace_cpu_offset, cpu);
527 sibling_off = &per_cpu(blk_trace_cpu_offset, i);
528 *sibling_off = *cpu_off;
529 }
530 }
531 preempt_enable();
532#endif
533}
534
535static __init int blk_trace_init(void)
536{
537 mutex_init(&blk_tree_mutex);
538 blk_trace_calibrate_offsets();
539 blk_trace_set_ht_offsets();
540
541 return 0;
542}
543
544module_init(blk_trace_init);
545