Brendan Gregg | 38cef48 | 2016-01-15 17:26:30 -0800 | [diff] [blame] | 1 | #!/usr/bin/python |
| 2 | # |
Andrew Birchall | 1f202e7 | 2016-05-05 10:56:40 -0700 | [diff] [blame] | 3 | # offcputime Summarize off-CPU time by stack trace |
Brendan Gregg | 38cef48 | 2016-01-15 17:26:30 -0800 | [diff] [blame] | 4 | # For Linux, uses BCC, eBPF. |
| 5 | # |
Andrew Birchall | 1f202e7 | 2016-05-05 10:56:40 -0700 | [diff] [blame] | 6 | # USAGE: offcputime [-h] [-p PID | -u | -k] [-U | -K] [-f] [duration] |
Brendan Gregg | 38cef48 | 2016-01-15 17:26:30 -0800 | [diff] [blame] | 7 | # |
Brendan Gregg | 38cef48 | 2016-01-15 17:26:30 -0800 | [diff] [blame] | 8 | # Copyright 2016 Netflix, Inc. |
| 9 | # Licensed under the Apache License, Version 2.0 (the "License") |
| 10 | # |
| 11 | # 13-Jan-2016 Brendan Gregg Created this. |
| 12 | |
| 13 | from __future__ import print_function |
| 14 | from bcc import BPF |
Andrew Birchall | ee7e5b4 | 2016-05-03 16:54:00 -0700 | [diff] [blame] | 15 | from sys import stderr |
Brendan Gregg | 38cef48 | 2016-01-15 17:26:30 -0800 | [diff] [blame] | 16 | from time import sleep, strftime |
| 17 | import argparse |
Teng Qin | 0b11d22 | 2016-07-18 13:21:10 -0700 | [diff] [blame] | 18 | import errno |
Brendan Gregg | 38cef48 | 2016-01-15 17:26:30 -0800 | [diff] [blame] | 19 | import signal |
| 20 | |
Andrew Birchall | 47d871f | 2016-05-11 18:31:49 -0700 | [diff] [blame] | 21 | # arg validation |
| 22 | def positive_int(val): |
| 23 | try: |
| 24 | ival = int(val) |
| 25 | except ValueError: |
| 26 | raise argparse.ArgumentTypeError("must be an integer") |
| 27 | |
| 28 | if ival < 0: |
| 29 | raise argparse.ArgumentTypeError("must be positive") |
| 30 | return ival |
| 31 | |
| 32 | def positive_nonzero_int(val): |
| 33 | ival = positive_int(val) |
| 34 | if ival == 0: |
| 35 | raise argparse.ArgumentTypeError("must be nonzero") |
| 36 | return ival |
| 37 | |
Teng Qin | ea72805 | 2018-04-13 16:48:09 -0700 | [diff] [blame] | 38 | def stack_id_err(stack_id): |
| 39 | # -EFAULT in get_stackid normally means the stack-trace is not availible, |
| 40 | # Such as getting kernel stack trace in userspace code |
| 41 | return (stack_id < 0) and (stack_id != -errno.EFAULT) |
| 42 | |
Brendan Gregg | 38cef48 | 2016-01-15 17:26:30 -0800 | [diff] [blame] | 43 | # arguments |
| 44 | examples = """examples: |
| 45 | ./offcputime # trace off-CPU stack time until Ctrl-C |
| 46 | ./offcputime 5 # trace for 5 seconds only |
| 47 | ./offcputime -f 5 # 5 seconds, and output in folded format |
Sasha Goldshtein | f41ae86 | 2016-10-19 01:14:30 +0300 | [diff] [blame] | 48 | ./offcputime -m 1000 # trace only events that last more than 1000 usec |
| 49 | ./offcputime -M 10000 # trace only events that last less than 10000 usec |
Andrew Birchall | 582b5dd | 2016-05-04 16:03:34 -0700 | [diff] [blame] | 50 | ./offcputime -p 185 # only trace threads for PID 185 |
Mark Drayton | 66bf2e8 | 2016-07-31 22:47:07 +0100 | [diff] [blame] | 51 | ./offcputime -t 188 # only trace thread 188 |
Andrew Birchall | 582b5dd | 2016-05-04 16:03:34 -0700 | [diff] [blame] | 52 | ./offcputime -u # only trace user threads (no kernel) |
| 53 | ./offcputime -k # only trace kernel threads (no user) |
Andrew Birchall | 7f0a6f8 | 2016-05-24 01:44:41 -0700 | [diff] [blame] | 54 | ./offcputime -U # only show user space stacks (no kernel) |
| 55 | ./offcputime -K # only show kernel space stacks (no user) |
Brendan Gregg | 38cef48 | 2016-01-15 17:26:30 -0800 | [diff] [blame] | 56 | """ |
| 57 | parser = argparse.ArgumentParser( |
Andrew Birchall | 7f0a6f8 | 2016-05-24 01:44:41 -0700 | [diff] [blame] | 58 | description="Summarize off-CPU time by stack trace", |
Brendan Gregg | 38cef48 | 2016-01-15 17:26:30 -0800 | [diff] [blame] | 59 | formatter_class=argparse.RawDescriptionHelpFormatter, |
| 60 | epilog=examples) |
Andrew Birchall | 47d871f | 2016-05-11 18:31:49 -0700 | [diff] [blame] | 61 | thread_group = parser.add_mutually_exclusive_group() |
Mark Drayton | 66bf2e8 | 2016-07-31 22:47:07 +0100 | [diff] [blame] | 62 | # Note: this script provides --pid and --tid flags but their arguments are |
| 63 | # referred to internally using kernel nomenclature: TGID and PID. |
| 64 | thread_group.add_argument("-p", "--pid", metavar="PID", dest="tgid", |
| 65 | help="trace this PID only", type=positive_int) |
| 66 | thread_group.add_argument("-t", "--tid", metavar="TID", dest="pid", |
| 67 | help="trace this TID only", type=positive_int) |
Andrew Birchall | 582b5dd | 2016-05-04 16:03:34 -0700 | [diff] [blame] | 68 | thread_group.add_argument("-u", "--user-threads-only", action="store_true", |
| 69 | help="user threads only (no kernel threads)") |
Andrew Birchall | 7f0a6f8 | 2016-05-24 01:44:41 -0700 | [diff] [blame] | 70 | thread_group.add_argument("-k", "--kernel-threads-only", action="store_true", |
| 71 | help="kernel threads only (no user threads)") |
Andrew Birchall | 1f202e7 | 2016-05-05 10:56:40 -0700 | [diff] [blame] | 72 | stack_group = parser.add_mutually_exclusive_group() |
| 73 | stack_group.add_argument("-U", "--user-stacks-only", action="store_true", |
Andrew Birchall | 7f0a6f8 | 2016-05-24 01:44:41 -0700 | [diff] [blame] | 74 | help="show stacks from user space only (no kernel space stacks)") |
Andrew Birchall | 1f202e7 | 2016-05-05 10:56:40 -0700 | [diff] [blame] | 75 | stack_group.add_argument("-K", "--kernel-stacks-only", action="store_true", |
Andrew Birchall | 7f0a6f8 | 2016-05-24 01:44:41 -0700 | [diff] [blame] | 76 | help="show stacks from kernel space only (no user space stacks)") |
Evgeny Vereshchagin | 4509f09 | 2016-06-08 06:33:54 +1000 | [diff] [blame] | 77 | parser.add_argument("-d", "--delimited", action="store_true", |
| 78 | help="insert delimiter between kernel/user stacks") |
Brendan Gregg | 38cef48 | 2016-01-15 17:26:30 -0800 | [diff] [blame] | 79 | parser.add_argument("-f", "--folded", action="store_true", |
| 80 | help="output folded format") |
Andrew Birchall | 47d871f | 2016-05-11 18:31:49 -0700 | [diff] [blame] | 81 | parser.add_argument("--stack-storage-size", default=1024, |
| 82 | type=positive_nonzero_int, |
Mark Drayton | 66bf2e8 | 2016-07-31 22:47:07 +0100 | [diff] [blame] | 83 | help="the number of unique stack traces that can be stored and " |
| 84 | "displayed (default 1024)") |
Brendan Gregg | 38cef48 | 2016-01-15 17:26:30 -0800 | [diff] [blame] | 85 | parser.add_argument("duration", nargs="?", default=99999999, |
Andrew Birchall | 47d871f | 2016-05-11 18:31:49 -0700 | [diff] [blame] | 86 | type=positive_nonzero_int, |
Brendan Gregg | 38cef48 | 2016-01-15 17:26:30 -0800 | [diff] [blame] | 87 | help="duration of trace, in seconds") |
Glauber Costa | 5246458 | 2016-09-26 12:59:32 -0700 | [diff] [blame] | 88 | parser.add_argument("-m", "--min-block-time", default=1, |
| 89 | type=positive_nonzero_int, |
Sasha Goldshtein | f41ae86 | 2016-10-19 01:14:30 +0300 | [diff] [blame] | 90 | help="the amount of time in microseconds over which we " + |
| 91 | "store traces (default 1)") |
| 92 | parser.add_argument("-M", "--max-block-time", default=(1 << 64) - 1, |
Glauber Costa | 5246458 | 2016-09-26 12:59:32 -0700 | [diff] [blame] | 93 | type=positive_nonzero_int, |
Sasha Goldshtein | f41ae86 | 2016-10-19 01:14:30 +0300 | [diff] [blame] | 94 | help="the amount of time in microseconds under which we " + |
| 95 | "store traces (default U64_MAX)") |
Brendan Gregg | 4870819 | 2017-08-03 11:49:27 -0700 | [diff] [blame] | 96 | parser.add_argument("--state", type=positive_int, |
Brendan Gregg | 843340d | 2017-07-31 18:39:30 -0700 | [diff] [blame] | 97 | help="filter on this thread state bitmask (eg, 2 == TASK_UNINTERRUPTIBLE" + |
| 98 | ") see include/linux/sched.h") |
Nathan Scott | cf0792f | 2018-02-02 16:56:50 +1100 | [diff] [blame] | 99 | parser.add_argument("--ebpf", action="store_true", |
| 100 | help=argparse.SUPPRESS) |
Brendan Gregg | 38cef48 | 2016-01-15 17:26:30 -0800 | [diff] [blame] | 101 | args = parser.parse_args() |
Mark Drayton | 66bf2e8 | 2016-07-31 22:47:07 +0100 | [diff] [blame] | 102 | if args.pid and args.tgid: |
| 103 | parser.error("specify only one of -p and -t") |
Brendan Gregg | 38cef48 | 2016-01-15 17:26:30 -0800 | [diff] [blame] | 104 | folded = args.folded |
| 105 | duration = int(args.duration) |
Brendan Gregg | 843340d | 2017-07-31 18:39:30 -0700 | [diff] [blame] | 106 | debug = 0 |
Brendan Gregg | 38cef48 | 2016-01-15 17:26:30 -0800 | [diff] [blame] | 107 | |
| 108 | # signal handler |
| 109 | def signal_ignore(signal, frame): |
| 110 | print() |
| 111 | |
Brendan Gregg | d364d04 | 2016-01-19 17:12:52 -0800 | [diff] [blame] | 112 | # define BPF program |
Brendan Gregg | 38cef48 | 2016-01-15 17:26:30 -0800 | [diff] [blame] | 113 | bpf_text = """ |
| 114 | #include <uapi/linux/ptrace.h> |
| 115 | #include <linux/sched.h> |
| 116 | |
Glauber Costa | 5246458 | 2016-09-26 12:59:32 -0700 | [diff] [blame] | 117 | #define MINBLOCK_US MINBLOCK_US_VALUEULL |
| 118 | #define MAXBLOCK_US MAXBLOCK_US_VALUEULL |
Brendan Gregg | 38cef48 | 2016-01-15 17:26:30 -0800 | [diff] [blame] | 119 | |
| 120 | struct key_t { |
Andrew Birchall | 1f202e7 | 2016-05-05 10:56:40 -0700 | [diff] [blame] | 121 | u32 pid; |
Mark Drayton | 66bf2e8 | 2016-07-31 22:47:07 +0100 | [diff] [blame] | 122 | u32 tgid; |
Andrew Birchall | 1f202e7 | 2016-05-05 10:56:40 -0700 | [diff] [blame] | 123 | int user_stack_id; |
| 124 | int kernel_stack_id; |
Brendan Gregg | 38cef48 | 2016-01-15 17:26:30 -0800 | [diff] [blame] | 125 | char name[TASK_COMM_LEN]; |
Brendan Gregg | 38cef48 | 2016-01-15 17:26:30 -0800 | [diff] [blame] | 126 | }; |
| 127 | BPF_HASH(counts, struct key_t); |
| 128 | BPF_HASH(start, u32); |
Song Liu | 67ae605 | 2018-02-01 14:59:24 -0800 | [diff] [blame] | 129 | BPF_STACK_TRACE(stack_traces, STACK_STORAGE_SIZE); |
Brendan Gregg | 38cef48 | 2016-01-15 17:26:30 -0800 | [diff] [blame] | 130 | |
Brendan Gregg | d364d04 | 2016-01-19 17:12:52 -0800 | [diff] [blame] | 131 | int oncpu(struct pt_regs *ctx, struct task_struct *prev) { |
Evgeny Vereshchagin | 9858ca5 | 2016-05-27 06:13:52 +0000 | [diff] [blame] | 132 | u32 pid = prev->pid; |
Mark Drayton | 66bf2e8 | 2016-07-31 22:47:07 +0100 | [diff] [blame] | 133 | u32 tgid = prev->tgid; |
Brendan Gregg | d364d04 | 2016-01-19 17:12:52 -0800 | [diff] [blame] | 134 | u64 ts, *tsp; |
Brendan Gregg | 38cef48 | 2016-01-15 17:26:30 -0800 | [diff] [blame] | 135 | |
Brendan Gregg | d364d04 | 2016-01-19 17:12:52 -0800 | [diff] [blame] | 136 | // record previous thread sleep time |
Brendan Gregg | 843340d | 2017-07-31 18:39:30 -0700 | [diff] [blame] | 137 | if ((THREAD_FILTER) && (STATE_FILTER)) { |
Brendan Gregg | d364d04 | 2016-01-19 17:12:52 -0800 | [diff] [blame] | 138 | ts = bpf_ktime_get_ns(); |
| 139 | start.update(&pid, &ts); |
| 140 | } |
Brendan Gregg | 38cef48 | 2016-01-15 17:26:30 -0800 | [diff] [blame] | 141 | |
Andrew Birchall | 1f202e7 | 2016-05-05 10:56:40 -0700 | [diff] [blame] | 142 | // get the current thread's start time |
Brendan Gregg | d364d04 | 2016-01-19 17:12:52 -0800 | [diff] [blame] | 143 | pid = bpf_get_current_pid_tgid(); |
Mark Drayton | 66bf2e8 | 2016-07-31 22:47:07 +0100 | [diff] [blame] | 144 | tgid = bpf_get_current_pid_tgid() >> 32; |
Brendan Gregg | 38cef48 | 2016-01-15 17:26:30 -0800 | [diff] [blame] | 145 | tsp = start.lookup(&pid); |
Andrew Birchall | 1f202e7 | 2016-05-05 10:56:40 -0700 | [diff] [blame] | 146 | if (tsp == 0) { |
Brendan Gregg | f747114 | 2016-01-19 14:40:41 -0800 | [diff] [blame] | 147 | return 0; // missed start or filtered |
Andrew Birchall | 1f202e7 | 2016-05-05 10:56:40 -0700 | [diff] [blame] | 148 | } |
| 149 | |
| 150 | // calculate current thread's delta time |
Brendan Gregg | f747114 | 2016-01-19 14:40:41 -0800 | [diff] [blame] | 151 | u64 delta = bpf_ktime_get_ns() - *tsp; |
Brendan Gregg | 38cef48 | 2016-01-15 17:26:30 -0800 | [diff] [blame] | 152 | start.delete(&pid); |
| 153 | delta = delta / 1000; |
Glauber Costa | 5246458 | 2016-09-26 12:59:32 -0700 | [diff] [blame] | 154 | if ((delta < MINBLOCK_US) || (delta > MAXBLOCK_US)) { |
Brendan Gregg | 38cef48 | 2016-01-15 17:26:30 -0800 | [diff] [blame] | 155 | return 0; |
Andrew Birchall | 1f202e7 | 2016-05-05 10:56:40 -0700 | [diff] [blame] | 156 | } |
Brendan Gregg | 38cef48 | 2016-01-15 17:26:30 -0800 | [diff] [blame] | 157 | |
Brendan Gregg | f747114 | 2016-01-19 14:40:41 -0800 | [diff] [blame] | 158 | // create map key |
Brendan Gregg | f747114 | 2016-01-19 14:40:41 -0800 | [diff] [blame] | 159 | struct key_t key = {}; |
Vicent Marti | e82fb1b | 2016-03-25 17:21:44 +0100 | [diff] [blame] | 160 | |
Andrew Birchall | 1f202e7 | 2016-05-05 10:56:40 -0700 | [diff] [blame] | 161 | key.pid = pid; |
Mark Drayton | 66bf2e8 | 2016-07-31 22:47:07 +0100 | [diff] [blame] | 162 | key.tgid = tgid; |
Andrew Birchall | 1f202e7 | 2016-05-05 10:56:40 -0700 | [diff] [blame] | 163 | key.user_stack_id = USER_STACK_GET; |
| 164 | key.kernel_stack_id = KERNEL_STACK_GET; |
Brendan Gregg | 38cef48 | 2016-01-15 17:26:30 -0800 | [diff] [blame] | 165 | bpf_get_current_comm(&key.name, sizeof(key.name)); |
Brendan Gregg | 38cef48 | 2016-01-15 17:26:30 -0800 | [diff] [blame] | 166 | |
Javier Honduvilla Coto | 64bf965 | 2018-08-01 06:50:19 +0200 | [diff] [blame] | 167 | counts.increment(key, delta); |
Brendan Gregg | 38cef48 | 2016-01-15 17:26:30 -0800 | [diff] [blame] | 168 | return 0; |
| 169 | } |
| 170 | """ |
Andrew Birchall | 47d871f | 2016-05-11 18:31:49 -0700 | [diff] [blame] | 171 | |
| 172 | # set thread filter |
Andrew Birchall | 582b5dd | 2016-05-04 16:03:34 -0700 | [diff] [blame] | 173 | thread_context = "" |
Mark Drayton | 66bf2e8 | 2016-07-31 22:47:07 +0100 | [diff] [blame] | 174 | if args.tgid is not None: |
| 175 | thread_context = "PID %d" % args.tgid |
| 176 | thread_filter = 'tgid == %d' % args.tgid |
| 177 | elif args.pid is not None: |
| 178 | thread_context = "TID %d" % args.pid |
| 179 | thread_filter = 'pid == %d' % args.pid |
Andrew Birchall | 582b5dd | 2016-05-04 16:03:34 -0700 | [diff] [blame] | 180 | elif args.user_threads_only: |
| 181 | thread_context = "user threads" |
| 182 | thread_filter = '!(prev->flags & PF_KTHREAD)' |
| 183 | elif args.kernel_threads_only: |
| 184 | thread_context = "kernel threads" |
| 185 | thread_filter = 'prev->flags & PF_KTHREAD' |
Brendan Gregg | 38cef48 | 2016-01-15 17:26:30 -0800 | [diff] [blame] | 186 | else: |
Andrew Birchall | 582b5dd | 2016-05-04 16:03:34 -0700 | [diff] [blame] | 187 | thread_context = "all threads" |
| 188 | thread_filter = '1' |
Brendan Gregg | 843340d | 2017-07-31 18:39:30 -0700 | [diff] [blame] | 189 | if args.state == 0: |
| 190 | state_filter = 'prev->state == 0' |
Brendan Gregg | 4870819 | 2017-08-03 11:49:27 -0700 | [diff] [blame] | 191 | elif args.state: |
Brendan Gregg | 843340d | 2017-07-31 18:39:30 -0700 | [diff] [blame] | 192 | # these states are sometimes bitmask checked |
| 193 | state_filter = 'prev->state & %d' % args.state |
| 194 | else: |
| 195 | state_filter = '1' |
Andrew Birchall | 582b5dd | 2016-05-04 16:03:34 -0700 | [diff] [blame] | 196 | bpf_text = bpf_text.replace('THREAD_FILTER', thread_filter) |
Brendan Gregg | 843340d | 2017-07-31 18:39:30 -0700 | [diff] [blame] | 197 | bpf_text = bpf_text.replace('STATE_FILTER', state_filter) |
Andrew Birchall | 47d871f | 2016-05-11 18:31:49 -0700 | [diff] [blame] | 198 | |
| 199 | # set stack storage size |
| 200 | bpf_text = bpf_text.replace('STACK_STORAGE_SIZE', str(args.stack_storage_size)) |
Glauber Costa | 5246458 | 2016-09-26 12:59:32 -0700 | [diff] [blame] | 201 | bpf_text = bpf_text.replace('MINBLOCK_US_VALUE', str(args.min_block_time)) |
| 202 | bpf_text = bpf_text.replace('MAXBLOCK_US_VALUE', str(args.max_block_time)) |
Brendan Gregg | d364d04 | 2016-01-19 17:12:52 -0800 | [diff] [blame] | 203 | |
Andrew Birchall | 1f202e7 | 2016-05-05 10:56:40 -0700 | [diff] [blame] | 204 | # handle stack args |
Teng Qin | ea72805 | 2018-04-13 16:48:09 -0700 | [diff] [blame] | 205 | kernel_stack_get = "stack_traces.get_stackid(ctx, 0)" |
| 206 | user_stack_get = "stack_traces.get_stackid(ctx, BPF_F_USER_STACK)" |
Andrew Birchall | 1f202e7 | 2016-05-05 10:56:40 -0700 | [diff] [blame] | 207 | stack_context = "" |
| 208 | if args.user_stacks_only: |
| 209 | stack_context = "user" |
| 210 | kernel_stack_get = "-1" |
| 211 | elif args.kernel_stacks_only: |
| 212 | stack_context = "kernel" |
| 213 | user_stack_get = "-1" |
| 214 | else: |
| 215 | stack_context = "user + kernel" |
| 216 | bpf_text = bpf_text.replace('USER_STACK_GET', user_stack_get) |
| 217 | bpf_text = bpf_text.replace('KERNEL_STACK_GET', kernel_stack_get) |
| 218 | |
Sasha Goldshtein | f41ae86 | 2016-10-19 01:14:30 +0300 | [diff] [blame] | 219 | need_delimiter = args.delimited and not (args.kernel_stacks_only or |
| 220 | args.user_stacks_only) |
Evgeny Vereshchagin | 4509f09 | 2016-06-08 06:33:54 +1000 | [diff] [blame] | 221 | |
Andrew Birchall | 1f202e7 | 2016-05-05 10:56:40 -0700 | [diff] [blame] | 222 | # check for an edge case; the code below will handle this case correctly |
| 223 | # but ultimately nothing will be displayed |
| 224 | if args.kernel_threads_only and args.user_stacks_only: |
Sasha Goldshtein | f41ae86 | 2016-10-19 01:14:30 +0300 | [diff] [blame] | 225 | print("ERROR: Displaying user stacks for kernel threads " + |
| 226 | "doesn't make sense.", file=stderr) |
Andrew Birchall | 1f202e7 | 2016-05-05 10:56:40 -0700 | [diff] [blame] | 227 | exit(1) |
| 228 | |
Nathan Scott | cf0792f | 2018-02-02 16:56:50 +1100 | [diff] [blame] | 229 | if debug or args.ebpf: |
Brendan Gregg | 843340d | 2017-07-31 18:39:30 -0700 | [diff] [blame] | 230 | print(bpf_text) |
Nathan Scott | cf0792f | 2018-02-02 16:56:50 +1100 | [diff] [blame] | 231 | if args.ebpf: |
| 232 | exit() |
Brendan Gregg | 843340d | 2017-07-31 18:39:30 -0700 | [diff] [blame] | 233 | |
Brendan Gregg | d364d04 | 2016-01-19 17:12:52 -0800 | [diff] [blame] | 234 | # initialize BPF |
Brendan Gregg | 38cef48 | 2016-01-15 17:26:30 -0800 | [diff] [blame] | 235 | b = BPF(text=bpf_text) |
Brendan Gregg | 38cef48 | 2016-01-15 17:26:30 -0800 | [diff] [blame] | 236 | b.attach_kprobe(event="finish_task_switch", fn_name="oncpu") |
| 237 | matched = b.num_open_kprobes() |
| 238 | if matched == 0: |
Andrew Birchall | 47d871f | 2016-05-11 18:31:49 -0700 | [diff] [blame] | 239 | print("error: 0 functions traced. Exiting.", file=stderr) |
| 240 | exit(1) |
Brendan Gregg | 38cef48 | 2016-01-15 17:26:30 -0800 | [diff] [blame] | 241 | |
| 242 | # header |
| 243 | if not folded: |
Andrew Birchall | 1f202e7 | 2016-05-05 10:56:40 -0700 | [diff] [blame] | 244 | print("Tracing off-CPU time (us) of %s by %s stack" % |
| 245 | (thread_context, stack_context), end="") |
Brendan Gregg | 38cef48 | 2016-01-15 17:26:30 -0800 | [diff] [blame] | 246 | if duration < 99999999: |
| 247 | print(" for %d secs." % duration) |
| 248 | else: |
| 249 | print("... Hit Ctrl-C to end.") |
| 250 | |
Andrew Birchall | ee7e5b4 | 2016-05-03 16:54:00 -0700 | [diff] [blame] | 251 | try: |
| 252 | sleep(duration) |
| 253 | except KeyboardInterrupt: |
| 254 | # as cleanup can take many seconds, trap Ctrl-C: |
| 255 | signal.signal(signal.SIGINT, signal_ignore) |
Brendan Gregg | 38cef48 | 2016-01-15 17:26:30 -0800 | [diff] [blame] | 256 | |
Andrew Birchall | ee7e5b4 | 2016-05-03 16:54:00 -0700 | [diff] [blame] | 257 | if not folded: |
| 258 | print() |
Brendan Gregg | 38cef48 | 2016-01-15 17:26:30 -0800 | [diff] [blame] | 259 | |
Andrew Birchall | ee7e5b4 | 2016-05-03 16:54:00 -0700 | [diff] [blame] | 260 | missing_stacks = 0 |
Andrew Birchall | 47d871f | 2016-05-11 18:31:49 -0700 | [diff] [blame] | 261 | has_enomem = False |
Andrew Birchall | ee7e5b4 | 2016-05-03 16:54:00 -0700 | [diff] [blame] | 262 | counts = b.get_table("counts") |
| 263 | stack_traces = b.get_table("stack_traces") |
| 264 | for k, v in sorted(counts.items(), key=lambda counts: counts[1].value): |
Teng Qin | ea72805 | 2018-04-13 16:48:09 -0700 | [diff] [blame] | 265 | # handle get_stackid errors |
| 266 | if not args.user_stacks_only and stack_id_err(k.kernel_stack_id): |
Andrew Birchall | ee7e5b4 | 2016-05-03 16:54:00 -0700 | [diff] [blame] | 267 | missing_stacks += 1 |
Teng Qin | ea72805 | 2018-04-13 16:48:09 -0700 | [diff] [blame] | 268 | has_enomem = has_enomem or k.kernel_stack_id == -errno.ENOMEM |
| 269 | if not args.kernel_stacks_only and stack_id_err(k.user_stack_id): |
| 270 | missing_stacks += 1 |
| 271 | has_enomem = has_enomem or k.user_stack_id == -errno.ENOMEM |
Andrew Birchall | ee7e5b4 | 2016-05-03 16:54:00 -0700 | [diff] [blame] | 272 | |
Mark Drayton | 66bf2e8 | 2016-07-31 22:47:07 +0100 | [diff] [blame] | 273 | # user stacks will be symbolized by tgid, not pid, to avoid the overhead |
| 274 | # of one symbol resolver per thread |
Andrew Birchall | 1f202e7 | 2016-05-05 10:56:40 -0700 | [diff] [blame] | 275 | user_stack = [] if k.user_stack_id < 0 else \ |
| 276 | stack_traces.walk(k.user_stack_id) |
| 277 | kernel_stack = [] if k.kernel_stack_id < 0 else \ |
| 278 | stack_traces.walk(k.kernel_stack_id) |
Andrew Birchall | ee7e5b4 | 2016-05-03 16:54:00 -0700 | [diff] [blame] | 279 | |
| 280 | if folded: |
| 281 | # print folded stack output |
Evgeny Vereshchagin | ff39d0c | 2016-06-07 18:00:01 +1000 | [diff] [blame] | 282 | user_stack = list(user_stack) |
| 283 | kernel_stack = list(kernel_stack) |
Teng Qin | ea72805 | 2018-04-13 16:48:09 -0700 | [diff] [blame] | 284 | line = [k.name.decode()] |
| 285 | # if we failed to get the stack is, such as due to no space (-ENOMEM) or |
| 286 | # hash collision (-EEXIST), we still print a placeholder for consistency |
| 287 | if not args.kernel_stacks_only: |
| 288 | if stack_id_err(k.user_stack_id): |
| 289 | line.append("[Missed User Stack]") |
| 290 | else: |
| 291 | line.extend([b.sym(addr, k.tgid) for addr in reversed(user_stack)]) |
| 292 | if not args.user_stacks_only: |
Teng Qin | e778db0 | 2018-04-24 16:11:49 -0700 | [diff] [blame] | 293 | line.extend(["-"] if (need_delimiter and k.kernel_stack_id >= 0 and k.user_stack_id >= 0) else []) |
Teng Qin | ea72805 | 2018-04-13 16:48:09 -0700 | [diff] [blame] | 294 | if stack_id_err(k.kernel_stack_id): |
| 295 | line.append("[Missed Kernel Stack]") |
| 296 | else: |
| 297 | line.extend([b.ksym(addr) for addr in reversed(kernel_stack)]) |
Andrew Birchall | ee7e5b4 | 2016-05-03 16:54:00 -0700 | [diff] [blame] | 298 | print("%s %d" % (";".join(line), v.value)) |
| 299 | else: |
| 300 | # print default multi-line stack output |
Teng Qin | ea72805 | 2018-04-13 16:48:09 -0700 | [diff] [blame] | 301 | if not args.user_stacks_only: |
| 302 | if stack_id_err(k.kernel_stack_id): |
| 303 | print(" [Missed Kernel Stack]") |
| 304 | else: |
| 305 | for addr in kernel_stack: |
| 306 | print(" %s" % b.ksym(addr)) |
| 307 | if not args.kernel_stacks_only: |
Teng Qin | e778db0 | 2018-04-24 16:11:49 -0700 | [diff] [blame] | 308 | if need_delimiter and k.user_stack_id >= 0 and k.kernel_stack_id >= 0: |
Teng Qin | ea72805 | 2018-04-13 16:48:09 -0700 | [diff] [blame] | 309 | print(" --") |
| 310 | if stack_id_err(k.user_stack_id): |
| 311 | print(" [Missed User Stack]") |
| 312 | else: |
| 313 | for addr in user_stack: |
| 314 | print(" %s" % b.sym(addr, k.tgid)) |
Rafael F | 78948e4 | 2017-03-26 14:54:25 +0200 | [diff] [blame] | 315 | print(" %-16s %s (%d)" % ("-", k.name.decode(), k.pid)) |
Andrew Birchall | ee7e5b4 | 2016-05-03 16:54:00 -0700 | [diff] [blame] | 316 | print(" %d\n" % v.value) |
| 317 | |
| 318 | if missing_stacks > 0: |
Andrew Birchall | 47d871f | 2016-05-11 18:31:49 -0700 | [diff] [blame] | 319 | enomem_str = "" if not has_enomem else \ |
| 320 | " Consider increasing --stack-storage-size." |
Teng Qin | ea72805 | 2018-04-13 16:48:09 -0700 | [diff] [blame] | 321 | print("WARNING: %d stack traces lost and could not be displayed.%s" % |
Andrew Birchall | 47d871f | 2016-05-11 18:31:49 -0700 | [diff] [blame] | 322 | (missing_stacks, enomem_str), |
Andrew Birchall | ee7e5b4 | 2016-05-03 16:54:00 -0700 | [diff] [blame] | 323 | file=stderr) |