Sasha Goldshtein | 07175d0 | 2016-10-06 01:11:55 +0300 | [diff] [blame] | 1 | #!/usr/bin/env python |
Brendan Gregg | 38cef48 | 2016-01-15 17:26:30 -0800 | [diff] [blame] | 2 | # |
Sasha Goldshtein | 07175d0 | 2016-10-06 01:11:55 +0300 | [diff] [blame] | 3 | # stackcount Count events and their stack traces. |
Brendan Gregg | 38cef48 | 2016-01-15 17:26:30 -0800 | [diff] [blame] | 4 | # For Linux, uses BCC, eBPF. |
| 5 | # |
Brendan Gregg | 5888989 | 2017-09-03 12:08:52 -0700 | [diff] [blame] | 6 | # USAGE: stackcount.py [-h] [-p PID] [-i INTERVAL] [-D DURATION] [-T] [-r] [-s] |
| 7 | # [-P] [-K] [-U] [-v] [-d] [-f] [--debug] |
Brendan Gregg | 38cef48 | 2016-01-15 17:26:30 -0800 | [diff] [blame] | 8 | # |
| 9 | # The pattern is a string with optional '*' wildcards, similar to file |
| 10 | # globbing. If you'd prefer to use regular expressions, use the -r option. |
| 11 | # |
Brendan Gregg | 38cef48 | 2016-01-15 17:26:30 -0800 | [diff] [blame] | 12 | # Copyright 2016 Netflix, Inc. |
| 13 | # Licensed under the Apache License, Version 2.0 (the "License") |
| 14 | # |
Sasha Goldshtein | 07175d0 | 2016-10-06 01:11:55 +0300 | [diff] [blame] | 15 | # 12-Jan-2016 Brendan Gregg Created this. |
| 16 | # 09-Jul-2016 Sasha Goldshtein Generalized for uprobes and tracepoints. |
Brendan Gregg | 38cef48 | 2016-01-15 17:26:30 -0800 | [diff] [blame] | 17 | |
| 18 | from __future__ import print_function |
Sasha Goldshtein | 07175d0 | 2016-10-06 01:11:55 +0300 | [diff] [blame] | 19 | from bcc import BPF, USDT |
Brendan Gregg | 38cef48 | 2016-01-15 17:26:30 -0800 | [diff] [blame] | 20 | from time import sleep, strftime |
| 21 | import argparse |
Sasha Goldshtein | 07175d0 | 2016-10-06 01:11:55 +0300 | [diff] [blame] | 22 | import re |
Brendan Gregg | 38cef48 | 2016-01-15 17:26:30 -0800 | [diff] [blame] | 23 | import signal |
Sasha Goldshtein | 07175d0 | 2016-10-06 01:11:55 +0300 | [diff] [blame] | 24 | import sys |
| 25 | import traceback |
Brendan Gregg | 38cef48 | 2016-01-15 17:26:30 -0800 | [diff] [blame] | 26 | |
Sasha Goldshtein | 07175d0 | 2016-10-06 01:11:55 +0300 | [diff] [blame] | 27 | debug = False |
Brendan Gregg | 38cef48 | 2016-01-15 17:26:30 -0800 | [diff] [blame] | 28 | |
Sasha Goldshtein | 07175d0 | 2016-10-06 01:11:55 +0300 | [diff] [blame] | 29 | class Probe(object): |
Brendan Gregg | 5888989 | 2017-09-03 12:08:52 -0700 | [diff] [blame] | 30 | def __init__(self, pattern, kernel_stack, user_stack, use_regex=False, |
| 31 | pid=None, per_pid=False): |
Sasha Goldshtein | 07175d0 | 2016-10-06 01:11:55 +0300 | [diff] [blame] | 32 | """Init a new probe. |
Brendan Gregg | 38cef48 | 2016-01-15 17:26:30 -0800 | [diff] [blame] | 33 | |
Sasha Goldshtein | 07175d0 | 2016-10-06 01:11:55 +0300 | [diff] [blame] | 34 | Init the probe from the pattern provided by the user. The supported |
| 35 | patterns mimic the 'trace' and 'argdist' tools, but are simpler because |
| 36 | we don't have to distinguish between probes and retprobes. |
Brendan Gregg | 38cef48 | 2016-01-15 17:26:30 -0800 | [diff] [blame] | 37 | |
Sasha Goldshtein | 07175d0 | 2016-10-06 01:11:55 +0300 | [diff] [blame] | 38 | func -- probe a kernel function |
| 39 | lib:func -- probe a user-space function in the library 'lib' |
| 40 | p::func -- same thing as 'func' |
| 41 | p:lib:func -- same thing as 'lib:func' |
| 42 | t:cat:event -- probe a kernel tracepoint |
| 43 | u:lib:probe -- probe a USDT tracepoint |
| 44 | """ |
Brendan Gregg | 5888989 | 2017-09-03 12:08:52 -0700 | [diff] [blame] | 45 | self.kernel_stack = kernel_stack |
| 46 | self.user_stack = user_stack |
Sasha Goldshtein | 07175d0 | 2016-10-06 01:11:55 +0300 | [diff] [blame] | 47 | parts = pattern.split(':') |
| 48 | if len(parts) == 1: |
| 49 | parts = ["p", "", parts[0]] |
| 50 | elif len(parts) == 2: |
| 51 | parts = ["p", parts[0], parts[1]] |
| 52 | elif len(parts) == 3: |
| 53 | if parts[0] == "t": |
| 54 | parts = ["t", "", "%s:%s" % tuple(parts[1:])] |
| 55 | if parts[0] not in ["p", "t", "u"]: |
| 56 | raise Exception("Type must be 'p', 't', or 'u', but got %s" % |
| 57 | parts[0]) |
| 58 | else: |
| 59 | raise Exception("Too many ':'-separated components in pattern %s" % |
| 60 | pattern) |
Brendan Gregg | 38cef48 | 2016-01-15 17:26:30 -0800 | [diff] [blame] | 61 | |
Sasha Goldshtein | 07175d0 | 2016-10-06 01:11:55 +0300 | [diff] [blame] | 62 | (self.type, self.library, self.pattern) = parts |
| 63 | if not use_regex: |
| 64 | self.pattern = self.pattern.replace('*', '.*') |
| 65 | self.pattern = '^' + self.pattern + '$' |
| 66 | |
| 67 | if (self.type == "p" and self.library) or self.type == "u": |
| 68 | libpath = BPF.find_library(self.library) |
| 69 | if libpath is None: |
| 70 | # This might be an executable (e.g. 'bash') |
| 71 | libpath = BPF.find_exe(self.library) |
| 72 | if libpath is None or len(libpath) == 0: |
| 73 | raise Exception("unable to find library %s" % self.library) |
| 74 | self.library = libpath |
| 75 | |
| 76 | self.pid = pid |
| 77 | self.per_pid = per_pid |
| 78 | self.matched = 0 |
| 79 | |
| 80 | def is_kernel_probe(self): |
| 81 | return self.type == "t" or (self.type == "p" and self.library == "") |
| 82 | |
| 83 | def attach(self): |
| 84 | if self.type == "p": |
| 85 | if self.library: |
| 86 | self.bpf.attach_uprobe(name=self.library, |
| 87 | sym_re=self.pattern, |
| 88 | fn_name="trace_count", |
| 89 | pid=self.pid or -1) |
| 90 | self.matched = self.bpf.num_open_uprobes() |
| 91 | else: |
| 92 | self.bpf.attach_kprobe(event_re=self.pattern, |
Teng Qin | fd24405 | 2017-12-15 15:53:53 -0800 | [diff] [blame] | 93 | fn_name="trace_count") |
Sasha Goldshtein | 07175d0 | 2016-10-06 01:11:55 +0300 | [diff] [blame] | 94 | self.matched = self.bpf.num_open_kprobes() |
| 95 | elif self.type == "t": |
| 96 | self.bpf.attach_tracepoint(tp_re=self.pattern, |
Teng Qin | fd24405 | 2017-12-15 15:53:53 -0800 | [diff] [blame] | 97 | fn_name="trace_count") |
Sasha Goldshtein | 07175d0 | 2016-10-06 01:11:55 +0300 | [diff] [blame] | 98 | self.matched = self.bpf.num_open_tracepoints() |
| 99 | elif self.type == "u": |
Sasha Goldshtein | b778ccd | 2016-10-08 06:59:37 -0700 | [diff] [blame] | 100 | pass # Nothing to do -- attach already happened in `load` |
Sasha Goldshtein | 07175d0 | 2016-10-06 01:11:55 +0300 | [diff] [blame] | 101 | |
| 102 | if self.matched == 0: |
Sasha Goldshtein | b778ccd | 2016-10-08 06:59:37 -0700 | [diff] [blame] | 103 | raise Exception("No functions matched by pattern %s" % |
| 104 | self.pattern) |
Sasha Goldshtein | 07175d0 | 2016-10-06 01:11:55 +0300 | [diff] [blame] | 105 | |
| 106 | def load(self): |
Brendan Gregg | 5888989 | 2017-09-03 12:08:52 -0700 | [diff] [blame] | 107 | ctx_name = "ctx" |
| 108 | stack_trace = "" |
| 109 | if self.user_stack: |
| 110 | stack_trace += """ |
| 111 | key.user_stack_id = stack_traces.get_stackid( |
| 112 | %s, BPF_F_REUSE_STACKID | BPF_F_USER_STACK |
| 113 | );""" % (ctx_name) |
| 114 | else: |
| 115 | stack_trace += "key.user_stack_id = -1;" |
| 116 | if self.kernel_stack: |
| 117 | stack_trace += """ |
| 118 | key.kernel_stack_id = stack_traces.get_stackid( |
| 119 | %s, BPF_F_REUSE_STACKID |
| 120 | );""" % (ctx_name) |
| 121 | else: |
| 122 | stack_trace += "key.kernel_stack_id = -1;" |
| 123 | |
Sasha Goldshtein | 07175d0 | 2016-10-06 01:11:55 +0300 | [diff] [blame] | 124 | trace_count_text = """ |
| 125 | int trace_count(void *ctx) { |
Brendan Gregg | 38cef48 | 2016-01-15 17:26:30 -0800 | [diff] [blame] | 126 | FILTER |
Sasha Goldshtein | 07175d0 | 2016-10-06 01:11:55 +0300 | [diff] [blame] | 127 | struct key_t key = {}; |
Brendan Gregg | 5888989 | 2017-09-03 12:08:52 -0700 | [diff] [blame] | 128 | key.tgid = GET_TGID; |
| 129 | STORE_COMM |
| 130 | %s |
Vicent Marti | b4ebed0 | 2016-03-27 18:39:18 +0200 | [diff] [blame] | 131 | u64 zero = 0; |
| 132 | u64 *val = counts.lookup_or_init(&key, &zero); |
Brendan Gregg | 38cef48 | 2016-01-15 17:26:30 -0800 | [diff] [blame] | 133 | (*val)++; |
| 134 | return 0; |
| 135 | } |
Sasha Goldshtein | 07175d0 | 2016-10-06 01:11:55 +0300 | [diff] [blame] | 136 | """ |
Brendan Gregg | 5888989 | 2017-09-03 12:08:52 -0700 | [diff] [blame] | 137 | trace_count_text = trace_count_text % (stack_trace) |
| 138 | |
Sasha Goldshtein | 07175d0 | 2016-10-06 01:11:55 +0300 | [diff] [blame] | 139 | bpf_text = """#include <uapi/linux/ptrace.h> |
Brendan Gregg | 5888989 | 2017-09-03 12:08:52 -0700 | [diff] [blame] | 140 | #include <linux/sched.h> |
Brendan Gregg | 38cef48 | 2016-01-15 17:26:30 -0800 | [diff] [blame] | 141 | |
Sasha Goldshtein | 07175d0 | 2016-10-06 01:11:55 +0300 | [diff] [blame] | 142 | struct key_t { |
Brendan Gregg | 5888989 | 2017-09-03 12:08:52 -0700 | [diff] [blame] | 143 | // no pid (thread ID) so that we do not needlessly split this key |
| 144 | u32 tgid; |
| 145 | int kernel_stack_id; |
| 146 | int user_stack_id; |
| 147 | char name[TASK_COMM_LEN]; |
Sasha Goldshtein | 07175d0 | 2016-10-06 01:11:55 +0300 | [diff] [blame] | 148 | }; |
Brendan Gregg | 38cef48 | 2016-01-15 17:26:30 -0800 | [diff] [blame] | 149 | |
Sasha Goldshtein | 07175d0 | 2016-10-06 01:11:55 +0300 | [diff] [blame] | 150 | BPF_HASH(counts, struct key_t); |
| 151 | BPF_STACK_TRACE(stack_traces, 1024); |
Sasha Goldshtein | 07175d0 | 2016-10-06 01:11:55 +0300 | [diff] [blame] | 152 | """ |
| 153 | |
| 154 | # We really mean the tgid from the kernel's perspective, which is in |
| 155 | # the top 32 bits of bpf_get_current_pid_tgid(). |
| 156 | if self.is_kernel_probe() and self.pid: |
| 157 | trace_count_text = trace_count_text.replace('FILTER', |
| 158 | ('u32 pid; pid = bpf_get_current_pid_tgid() >> 32; ' + |
| 159 | 'if (pid != %d) { return 0; }') % (self.pid)) |
| 160 | else: |
| 161 | trace_count_text = trace_count_text.replace('FILTER', '') |
| 162 | |
| 163 | # We need per-pid statistics when tracing a user-space process, because |
| 164 | # the meaning of the symbols depends on the pid. We also need them if |
Brendan Gregg | 5888989 | 2017-09-03 12:08:52 -0700 | [diff] [blame] | 165 | # per-pid statistics were requested with -P, or for user stacks. |
| 166 | if self.per_pid or not self.is_kernel_probe() or self.user_stack: |
| 167 | trace_count_text = trace_count_text.replace('GET_TGID', |
Sasha Goldshtein | 07175d0 | 2016-10-06 01:11:55 +0300 | [diff] [blame] | 168 | 'bpf_get_current_pid_tgid() >> 32') |
Brendan Gregg | 5888989 | 2017-09-03 12:08:52 -0700 | [diff] [blame] | 169 | trace_count_text = trace_count_text.replace('STORE_COMM', |
| 170 | 'bpf_get_current_comm(&key.name, sizeof(key.name));') |
Sasha Goldshtein | 07175d0 | 2016-10-06 01:11:55 +0300 | [diff] [blame] | 171 | else: |
Brendan Gregg | 5888989 | 2017-09-03 12:08:52 -0700 | [diff] [blame] | 172 | # kernel stacks only. skip splitting on PID so these aggregate |
| 173 | # together, and don't store the process name. |
Sasha Goldshtein | b778ccd | 2016-10-08 06:59:37 -0700 | [diff] [blame] | 174 | trace_count_text = trace_count_text.replace( |
Brendan Gregg | 5888989 | 2017-09-03 12:08:52 -0700 | [diff] [blame] | 175 | 'GET_TGID', '0xffffffff') |
| 176 | trace_count_text = trace_count_text.replace('STORE_COMM', '') |
Sasha Goldshtein | 07175d0 | 2016-10-06 01:11:55 +0300 | [diff] [blame] | 177 | |
| 178 | self.usdt = None |
| 179 | if self.type == "u": |
| 180 | self.usdt = USDT(path=self.library, pid=self.pid) |
| 181 | for probe in self.usdt.enumerate_probes(): |
| 182 | if not self.pid and (probe.bin_path != self.library): |
| 183 | continue |
| 184 | if re.match(self.pattern, probe.name): |
| 185 | # This hack is required because the bpf_usdt_readarg |
| 186 | # functions generated need different function names for |
| 187 | # each attached probe. If we just stick to trace_count, |
| 188 | # we'd get multiple bpf_usdt_readarg helpers with the same |
| 189 | # name when enabling more than one USDT probe. |
| 190 | new_func = "trace_count_%d" % self.matched |
| 191 | bpf_text += trace_count_text.replace( |
| 192 | "trace_count", new_func) |
| 193 | self.usdt.enable_probe(probe.name, new_func) |
| 194 | self.matched += 1 |
| 195 | if debug: |
| 196 | print(self.usdt.get_text()) |
| 197 | else: |
| 198 | bpf_text += trace_count_text |
| 199 | |
| 200 | if debug: |
| 201 | print(bpf_text) |
Sasha Goldshtein | b778ccd | 2016-10-08 06:59:37 -0700 | [diff] [blame] | 202 | self.bpf = BPF(text=bpf_text, |
| 203 | usdt_contexts=[self.usdt] if self.usdt else []) |
Sasha Goldshtein | 07175d0 | 2016-10-06 01:11:55 +0300 | [diff] [blame] | 204 | |
| 205 | class Tool(object): |
| 206 | def __init__(self): |
| 207 | examples = """examples: |
Sasha Goldshtein | b778ccd | 2016-10-08 06:59:37 -0700 | [diff] [blame] | 208 | ./stackcount submit_bio # count kernel stack traces for submit_bio |
Brendan Gregg | 5888989 | 2017-09-03 12:08:52 -0700 | [diff] [blame] | 209 | ./stackcount -d ip_output # include a user/kernel stack delimiter |
Sasha Goldshtein | b778ccd | 2016-10-08 06:59:37 -0700 | [diff] [blame] | 210 | ./stackcount -s ip_output # show symbol offsets |
| 211 | ./stackcount -sv ip_output # show offsets and raw addresses (verbose) |
| 212 | ./stackcount 'tcp_send*' # count stacks for funcs matching tcp_send* |
| 213 | ./stackcount -r '^tcp_send.*' # same as above, using regular expressions |
| 214 | ./stackcount -Ti 5 ip_output # output every 5 seconds, with timestamps |
| 215 | ./stackcount -p 185 ip_output # count ip_output stacks for PID 185 only |
| 216 | ./stackcount -p 185 c:malloc # count stacks for malloc in PID 185 |
| 217 | ./stackcount t:sched:sched_fork # count stacks for sched_fork tracepoint |
| 218 | ./stackcount -p 185 u:node:* # count stacks for all USDT probes in node |
Brendan Gregg | 5888989 | 2017-09-03 12:08:52 -0700 | [diff] [blame] | 219 | ./stackcount -K t:sched:sched_switch # kernel stacks only |
| 220 | ./stackcount -U t:sched:sched_switch # user stacks only |
Sasha Goldshtein | 07175d0 | 2016-10-06 01:11:55 +0300 | [diff] [blame] | 221 | """ |
| 222 | parser = argparse.ArgumentParser( |
| 223 | description="Count events and their stack traces", |
| 224 | formatter_class=argparse.RawDescriptionHelpFormatter, |
| 225 | epilog=examples) |
| 226 | parser.add_argument("-p", "--pid", type=int, |
| 227 | help="trace this PID only") |
Brendan Gregg | 5888989 | 2017-09-03 12:08:52 -0700 | [diff] [blame] | 228 | parser.add_argument("-i", "--interval", |
Sasha Goldshtein | 07175d0 | 2016-10-06 01:11:55 +0300 | [diff] [blame] | 229 | help="summary interval, seconds") |
Brendan Gregg | 5888989 | 2017-09-03 12:08:52 -0700 | [diff] [blame] | 230 | parser.add_argument("-D", "--duration", |
| 231 | help="total duration of trace, seconds") |
Sasha Goldshtein | 07175d0 | 2016-10-06 01:11:55 +0300 | [diff] [blame] | 232 | parser.add_argument("-T", "--timestamp", action="store_true", |
| 233 | help="include timestamp on output") |
| 234 | parser.add_argument("-r", "--regexp", action="store_true", |
| 235 | help="use regular expressions. Default is \"*\" wildcards only.") |
| 236 | parser.add_argument("-s", "--offset", action="store_true", |
| 237 | help="show address offsets") |
| 238 | parser.add_argument("-P", "--perpid", action="store_true", |
| 239 | help="display stacks separately for each process") |
Brendan Gregg | 5888989 | 2017-09-03 12:08:52 -0700 | [diff] [blame] | 240 | parser.add_argument("-K", "--kernel-stacks-only", |
| 241 | action="store_true", help="kernel stack only", default=False) |
| 242 | parser.add_argument("-U", "--user-stacks-only", |
| 243 | action="store_true", help="user stack only", default=False) |
Sasha Goldshtein | 07175d0 | 2016-10-06 01:11:55 +0300 | [diff] [blame] | 244 | parser.add_argument("-v", "--verbose", action="store_true", |
| 245 | help="show raw addresses") |
Brendan Gregg | 5888989 | 2017-09-03 12:08:52 -0700 | [diff] [blame] | 246 | parser.add_argument("-d", "--delimited", action="store_true", |
| 247 | help="insert delimiter between kernel/user stacks") |
| 248 | parser.add_argument("-f", "--folded", action="store_true", |
| 249 | help="output folded format") |
| 250 | parser.add_argument("--debug", action="store_true", |
Sasha Goldshtein | 07175d0 | 2016-10-06 01:11:55 +0300 | [diff] [blame] | 251 | help="print BPF program before starting (for debugging purposes)") |
| 252 | parser.add_argument("pattern", |
| 253 | help="search expression for events") |
| 254 | self.args = parser.parse_args() |
| 255 | global debug |
| 256 | debug = self.args.debug |
Sasha Goldshtein | 07175d0 | 2016-10-06 01:11:55 +0300 | [diff] [blame] | 257 | |
Brendan Gregg | 5888989 | 2017-09-03 12:08:52 -0700 | [diff] [blame] | 258 | if self.args.duration and not self.args.interval: |
| 259 | self.args.interval = self.args.duration |
| 260 | if not self.args.interval: |
| 261 | self.args.interval = 99999999 |
| 262 | |
| 263 | if self.args.kernel_stacks_only and self.args.user_stacks_only: |
| 264 | print("ERROR: -K and -U are mutually exclusive. If you want " + |
| 265 | "both stacks, that is the default.") |
| 266 | exit() |
| 267 | if not self.args.kernel_stacks_only and not self.args.user_stacks_only: |
| 268 | self.kernel_stack = True |
| 269 | self.user_stack = True |
| 270 | else: |
| 271 | self.kernel_stack = self.args.kernel_stacks_only |
| 272 | self.user_stack = self.args.user_stacks_only |
| 273 | |
| 274 | self.probe = Probe(self.args.pattern, |
| 275 | self.kernel_stack, self.user_stack, |
| 276 | self.args.regexp, self.args.pid, self.args.perpid) |
| 277 | self.need_delimiter = self.args.delimited and not ( |
| 278 | self.args.kernel_stacks_only or self.args.user_stacks_only) |
| 279 | |
| 280 | def _print_kframe(self, addr): |
| 281 | print(" ", end="") |
| 282 | if self.args.verbose: |
| 283 | print("%-16x " % addr, end="") |
| 284 | if self.args.offset: |
| 285 | print("%s" % self.probe.bpf.ksym(addr, show_offset=True)) |
| 286 | else: |
| 287 | print("%s" % self.probe.bpf.ksym(addr)) |
| 288 | |
| 289 | def _print_uframe(self, addr, pid): |
Sasha Goldshtein | 07175d0 | 2016-10-06 01:11:55 +0300 | [diff] [blame] | 290 | print(" ", end="") |
| 291 | if self.args.verbose: |
| 292 | print("%-16x " % addr, end="") |
| 293 | if self.args.offset: |
Sasha Goldshtein | 0155385 | 2017-02-09 03:58:09 -0500 | [diff] [blame] | 294 | print("%s" % self.probe.bpf.sym(addr, pid, show_offset=True)) |
Sasha Goldshtein | 07175d0 | 2016-10-06 01:11:55 +0300 | [diff] [blame] | 295 | else: |
| 296 | print("%s" % self.probe.bpf.sym(addr, pid)) |
| 297 | |
| 298 | @staticmethod |
| 299 | def _signal_ignore(signal, frame): |
| 300 | print() |
| 301 | |
Brendan Gregg | 5888989 | 2017-09-03 12:08:52 -0700 | [diff] [blame] | 302 | def _print_comm(self, comm, pid): |
| 303 | print(" %s [%d]" % (comm, pid)) |
Sasha Goldshtein | 07175d0 | 2016-10-06 01:11:55 +0300 | [diff] [blame] | 304 | |
| 305 | def run(self): |
| 306 | self.probe.load() |
| 307 | self.probe.attach() |
Brendan Gregg | 5888989 | 2017-09-03 12:08:52 -0700 | [diff] [blame] | 308 | if not self.args.folded: |
| 309 | print("Tracing %d functions for \"%s\"... Hit Ctrl-C to end." % |
| 310 | (self.probe.matched, self.args.pattern)) |
| 311 | b = self.probe.bpf |
Sasha Goldshtein | 07175d0 | 2016-10-06 01:11:55 +0300 | [diff] [blame] | 312 | exiting = 0 if self.args.interval else 1 |
Brendan Gregg | 5888989 | 2017-09-03 12:08:52 -0700 | [diff] [blame] | 313 | seconds = 0 |
Sasha Goldshtein | 07175d0 | 2016-10-06 01:11:55 +0300 | [diff] [blame] | 314 | while True: |
| 315 | try: |
| 316 | sleep(int(self.args.interval)) |
Brendan Gregg | 5888989 | 2017-09-03 12:08:52 -0700 | [diff] [blame] | 317 | seconds += int(self.args.interval) |
Sasha Goldshtein | 07175d0 | 2016-10-06 01:11:55 +0300 | [diff] [blame] | 318 | except KeyboardInterrupt: |
| 319 | exiting = 1 |
| 320 | # as cleanup can take many seconds, trap Ctrl-C: |
| 321 | signal.signal(signal.SIGINT, Tool._signal_ignore) |
Brendan Gregg | 5888989 | 2017-09-03 12:08:52 -0700 | [diff] [blame] | 322 | if self.args.duration and seconds >= int(self.args.duration): |
| 323 | exiting = 1 |
Sasha Goldshtein | 07175d0 | 2016-10-06 01:11:55 +0300 | [diff] [blame] | 324 | |
Brendan Gregg | 5888989 | 2017-09-03 12:08:52 -0700 | [diff] [blame] | 325 | if not self.args.folded: |
| 326 | print() |
Sasha Goldshtein | 07175d0 | 2016-10-06 01:11:55 +0300 | [diff] [blame] | 327 | if self.args.timestamp: |
| 328 | print("%-8s\n" % strftime("%H:%M:%S"), end="") |
| 329 | |
| 330 | counts = self.probe.bpf["counts"] |
| 331 | stack_traces = self.probe.bpf["stack_traces"] |
| 332 | self.comm_cache = {} |
| 333 | for k, v in sorted(counts.items(), |
| 334 | key=lambda counts: counts[1].value): |
Brendan Gregg | 5888989 | 2017-09-03 12:08:52 -0700 | [diff] [blame] | 335 | user_stack = [] if k.user_stack_id < 0 else \ |
| 336 | stack_traces.walk(k.user_stack_id) |
| 337 | kernel_stack = [] if k.kernel_stack_id < 0 else \ |
| 338 | stack_traces.walk(k.kernel_stack_id) |
| 339 | |
| 340 | if self.args.folded: |
| 341 | # print folded stack output |
| 342 | user_stack = list(user_stack) |
| 343 | kernel_stack = list(kernel_stack) |
| 344 | line = [k.name.decode()] + \ |
| 345 | [b.sym(addr, k.tgid) for addr in |
| 346 | reversed(user_stack)] + \ |
| 347 | (self.need_delimiter and ["-"] or []) + \ |
| 348 | [b.ksym(addr) for addr in reversed(kernel_stack)] |
| 349 | print("%s %d" % (";".join(line), v.value)) |
| 350 | else: |
| 351 | # print multi-line stack output |
| 352 | for addr in kernel_stack: |
| 353 | self._print_kframe(addr) |
| 354 | if self.need_delimiter: |
| 355 | print(" --") |
| 356 | for addr in user_stack: |
| 357 | self._print_uframe(addr, k.tgid) |
| 358 | if not self.args.pid and k.tgid != 0xffffffff: |
| 359 | self._print_comm(k.name, k.tgid) |
| 360 | print(" %d\n" % v.value) |
Sasha Goldshtein | 07175d0 | 2016-10-06 01:11:55 +0300 | [diff] [blame] | 361 | counts.clear() |
| 362 | |
| 363 | if exiting: |
Brendan Gregg | 5888989 | 2017-09-03 12:08:52 -0700 | [diff] [blame] | 364 | if not self.args.folded: |
| 365 | print("Detaching...") |
Sasha Goldshtein | 07175d0 | 2016-10-06 01:11:55 +0300 | [diff] [blame] | 366 | exit() |
| 367 | |
| 368 | if __name__ == "__main__": |
Brendan Gregg | 38cef48 | 2016-01-15 17:26:30 -0800 | [diff] [blame] | 369 | try: |
Sasha Goldshtein | 07175d0 | 2016-10-06 01:11:55 +0300 | [diff] [blame] | 370 | Tool().run() |
| 371 | except Exception: |
| 372 | if debug: |
| 373 | traceback.print_exc() |
| 374 | elif sys.exc_info()[0] is not SystemExit: |
| 375 | print(sys.exc_info()[1]) |