Brendan Gregg | 3e55ae2 | 2015-09-10 12:11:35 -0700 | [diff] [blame] | 1 | #!/usr/bin/python |
Alexei Starovoitov | bdf0773 | 2016-01-14 10:09:20 -0800 | [diff] [blame] | 2 | # @lint-avoid-python-3-compatibility-imports |
Brendan Gregg | 3e55ae2 | 2015-09-10 12:11:35 -0700 | [diff] [blame] | 3 | # |
Sasha Goldshtein | ff3b9f3 | 2016-10-08 07:01:21 -0700 | [diff] [blame] | 4 | # funccount Count functions, tracepoints, and USDT probes. |
| 5 | # For Linux, uses BCC, eBPF. |
Brendan Gregg | 3e55ae2 | 2015-09-10 12:11:35 -0700 | [diff] [blame] | 6 | # |
Brendan Gregg | b03d9eb | 2017-08-23 15:00:30 -0700 | [diff] [blame] | 7 | # USAGE: funccount [-h] [-p PID] [-i INTERVAL] [-d DURATION] [-T] [-r] pattern |
Brendan Gregg | 3e55ae2 | 2015-09-10 12:11:35 -0700 | [diff] [blame] | 8 | # |
Sasha Goldshtein | ff3b9f3 | 2016-10-08 07:01:21 -0700 | [diff] [blame] | 9 | # The pattern is a string with optional '*' wildcards, similar to file |
| 10 | # globbing. If you'd prefer to use regular expressions, use the -r option. |
Brendan Gregg | 3e55ae2 | 2015-09-10 12:11:35 -0700 | [diff] [blame] | 11 | # |
| 12 | # Copyright (c) 2015 Brendan Gregg. |
| 13 | # Licensed under the Apache License, Version 2.0 (the "License") |
| 14 | # |
Sasha Goldshtein | ff3b9f3 | 2016-10-08 07:01:21 -0700 | [diff] [blame] | 15 | # 09-Sep-2015 Brendan Gregg Created this. |
| 16 | # 18-Oct-2016 Sasha Goldshtein Generalized for uprobes, tracepoints, USDT. |
Brendan Gregg | 3e55ae2 | 2015-09-10 12:11:35 -0700 | [diff] [blame] | 17 | |
| 18 | from __future__ import print_function |
Sasha Goldshtein | ff3b9f3 | 2016-10-08 07:01:21 -0700 | [diff] [blame] | 19 | from bcc import BPF, USDT |
Brendan Gregg | 3e55ae2 | 2015-09-10 12:11:35 -0700 | [diff] [blame] | 20 | from time import sleep, strftime |
| 21 | import argparse |
Junli Ou | ec615af | 2016-08-06 11:43:20 +0800 | [diff] [blame] | 22 | import os |
Sasha Goldshtein | ff3b9f3 | 2016-10-08 07:01:21 -0700 | [diff] [blame] | 23 | import re |
| 24 | import signal |
| 25 | import sys |
| 26 | import traceback |
Brendan Gregg | 3e55ae2 | 2015-09-10 12:11:35 -0700 | [diff] [blame] | 27 | |
Sasha Goldshtein | ff3b9f3 | 2016-10-08 07:01:21 -0700 | [diff] [blame] | 28 | debug = False |
Brendan Gregg | 3e55ae2 | 2015-09-10 12:11:35 -0700 | [diff] [blame] | 29 | |
Sasha Goldshtein | 3ba14ef | 2016-10-20 02:46:37 +0100 | [diff] [blame] | 30 | def verify_limit(num): |
| 31 | probe_limit = 1000 |
| 32 | if num > probe_limit: |
| 33 | raise Exception("maximum of %d probes allowed, attempted %d" % |
| 34 | (probe_limit, num)) |
| 35 | |
Sasha Goldshtein | ff3b9f3 | 2016-10-08 07:01:21 -0700 | [diff] [blame] | 36 | class Probe(object): |
| 37 | def __init__(self, pattern, use_regex=False, pid=None): |
| 38 | """Init a new probe. |
Brendan Gregg | 3e55ae2 | 2015-09-10 12:11:35 -0700 | [diff] [blame] | 39 | |
Sasha Goldshtein | ff3b9f3 | 2016-10-08 07:01:21 -0700 | [diff] [blame] | 40 | Init the probe from the pattern provided by the user. The supported |
| 41 | patterns mimic the 'trace' and 'argdist' tools, but are simpler because |
| 42 | we don't have to distinguish between probes and retprobes. |
Brendan Gregg | 3e55ae2 | 2015-09-10 12:11:35 -0700 | [diff] [blame] | 43 | |
Sasha Goldshtein | ff3b9f3 | 2016-10-08 07:01:21 -0700 | [diff] [blame] | 44 | func -- probe a kernel function |
| 45 | lib:func -- probe a user-space function in the library 'lib' |
Brendan Gregg | b6035b6 | 2017-01-10 17:36:07 -0800 | [diff] [blame] | 46 | /path:func -- probe a user-space function in binary '/path' |
Sasha Goldshtein | ff3b9f3 | 2016-10-08 07:01:21 -0700 | [diff] [blame] | 47 | p::func -- same thing as 'func' |
| 48 | p:lib:func -- same thing as 'lib:func' |
| 49 | t:cat:event -- probe a kernel tracepoint |
| 50 | u:lib:probe -- probe a USDT tracepoint |
| 51 | """ |
| 52 | parts = pattern.split(':') |
| 53 | if len(parts) == 1: |
| 54 | parts = ["p", "", parts[0]] |
| 55 | elif len(parts) == 2: |
| 56 | parts = ["p", parts[0], parts[1]] |
| 57 | elif len(parts) == 3: |
| 58 | if parts[0] == "t": |
| 59 | parts = ["t", "", "%s:%s" % tuple(parts[1:])] |
| 60 | if parts[0] not in ["p", "t", "u"]: |
| 61 | raise Exception("Type must be 'p', 't', or 'u', but got %s" % |
| 62 | parts[0]) |
| 63 | else: |
| 64 | raise Exception("Too many ':'-separated components in pattern %s" % |
| 65 | pattern) |
Brendan Gregg | 3e55ae2 | 2015-09-10 12:11:35 -0700 | [diff] [blame] | 66 | |
Sasha Goldshtein | ff3b9f3 | 2016-10-08 07:01:21 -0700 | [diff] [blame] | 67 | (self.type, self.library, self.pattern) = parts |
| 68 | if not use_regex: |
| 69 | self.pattern = self.pattern.replace('*', '.*') |
| 70 | self.pattern = '^' + self.pattern + '$' |
| 71 | |
| 72 | if (self.type == "p" and self.library) or self.type == "u": |
| 73 | libpath = BPF.find_library(self.library) |
| 74 | if libpath is None: |
| 75 | # This might be an executable (e.g. 'bash') |
| 76 | libpath = BPF.find_exe(self.library) |
| 77 | if libpath is None or len(libpath) == 0: |
| 78 | raise Exception("unable to find library %s" % self.library) |
| 79 | self.library = libpath |
| 80 | |
| 81 | self.pid = pid |
| 82 | self.matched = 0 |
| 83 | self.trace_functions = {} # map location number to function name |
| 84 | |
| 85 | def is_kernel_probe(self): |
| 86 | return self.type == "t" or (self.type == "p" and self.library == "") |
| 87 | |
| 88 | def attach(self): |
| 89 | if self.type == "p" and not self.library: |
| 90 | for index, function in self.trace_functions.items(): |
| 91 | self.bpf.attach_kprobe( |
| 92 | event=function, |
Teng Qin | fd24405 | 2017-12-15 15:53:53 -0800 | [diff] [blame] | 93 | fn_name="trace_count_%d" % index) |
Sasha Goldshtein | ff3b9f3 | 2016-10-08 07:01:21 -0700 | [diff] [blame] | 94 | elif self.type == "p" and self.library: |
| 95 | for index, function in self.trace_functions.items(): |
| 96 | self.bpf.attach_uprobe( |
| 97 | name=self.library, |
| 98 | sym=function, |
| 99 | fn_name="trace_count_%d" % index, |
| 100 | pid=self.pid or -1) |
| 101 | elif self.type == "t": |
| 102 | for index, function in self.trace_functions.items(): |
| 103 | self.bpf.attach_tracepoint( |
| 104 | tp=function, |
Teng Qin | fd24405 | 2017-12-15 15:53:53 -0800 | [diff] [blame] | 105 | fn_name="trace_count_%d" % index) |
Sasha Goldshtein | ff3b9f3 | 2016-10-08 07:01:21 -0700 | [diff] [blame] | 106 | elif self.type == "u": |
| 107 | pass # Nothing to do -- attach already happened in `load` |
| 108 | |
Sasha Goldshtein | ff3b9f3 | 2016-10-08 07:01:21 -0700 | [diff] [blame] | 109 | def _add_function(self, template, probe_name): |
| 110 | new_func = "trace_count_%d" % self.matched |
| 111 | text = template.replace("PROBE_FUNCTION", new_func) |
| 112 | text = text.replace("LOCATION", str(self.matched)) |
| 113 | self.trace_functions[self.matched] = probe_name |
| 114 | self.matched += 1 |
| 115 | return text |
| 116 | |
| 117 | def _generate_functions(self, template): |
| 118 | self.usdt = None |
| 119 | text = "" |
| 120 | if self.type == "p" and not self.library: |
Sasha Goldshtein | 3ba14ef | 2016-10-20 02:46:37 +0100 | [diff] [blame] | 121 | functions = BPF.get_kprobe_functions(self.pattern) |
| 122 | verify_limit(len(functions)) |
| 123 | for function in functions: |
Sasha Goldshtein | ff3b9f3 | 2016-10-08 07:01:21 -0700 | [diff] [blame] | 124 | text += self._add_function(template, function) |
| 125 | elif self.type == "p" and self.library: |
| 126 | # uprobes are tricky because the same function may have multiple |
| 127 | # addresses, and the same address may be mapped to multiple |
| 128 | # functions. We aren't allowed to create more than one uprobe |
| 129 | # per address, so track unique addresses and ignore functions that |
| 130 | # map to an address that we've already seen. Also ignore functions |
| 131 | # that may repeat multiple times with different addresses. |
| 132 | addresses, functions = (set(), set()) |
Sasha Goldshtein | 3ba14ef | 2016-10-20 02:46:37 +0100 | [diff] [blame] | 133 | functions_and_addresses = BPF.get_user_functions_and_addresses( |
| 134 | self.library, self.pattern) |
| 135 | verify_limit(len(functions_and_addresses)) |
| 136 | for function, address in functions_and_addresses: |
Sasha Goldshtein | ff3b9f3 | 2016-10-08 07:01:21 -0700 | [diff] [blame] | 137 | if address in addresses or function in functions: |
| 138 | continue |
| 139 | addresses.add(address) |
| 140 | functions.add(function) |
| 141 | text += self._add_function(template, function) |
| 142 | elif self.type == "t": |
Sasha Goldshtein | 3ba14ef | 2016-10-20 02:46:37 +0100 | [diff] [blame] | 143 | tracepoints = BPF.get_tracepoints(self.pattern) |
| 144 | verify_limit(len(tracepoints)) |
| 145 | for tracepoint in tracepoints: |
Sasha Goldshtein | ff3b9f3 | 2016-10-08 07:01:21 -0700 | [diff] [blame] | 146 | text += self._add_function(template, tracepoint) |
| 147 | elif self.type == "u": |
| 148 | self.usdt = USDT(path=self.library, pid=self.pid) |
Sasha Goldshtein | 3ba14ef | 2016-10-20 02:46:37 +0100 | [diff] [blame] | 149 | matches = [] |
Sasha Goldshtein | ff3b9f3 | 2016-10-08 07:01:21 -0700 | [diff] [blame] | 150 | for probe in self.usdt.enumerate_probes(): |
| 151 | if not self.pid and (probe.bin_path != self.library): |
| 152 | continue |
| 153 | if re.match(self.pattern, probe.name): |
Sasha Goldshtein | 3ba14ef | 2016-10-20 02:46:37 +0100 | [diff] [blame] | 154 | matches.append(probe.name) |
| 155 | verify_limit(len(matches)) |
| 156 | for match in matches: |
| 157 | new_func = "trace_count_%d" % self.matched |
| 158 | text += self._add_function(template, match) |
| 159 | self.usdt.enable_probe(match, new_func) |
Sasha Goldshtein | ff3b9f3 | 2016-10-08 07:01:21 -0700 | [diff] [blame] | 160 | if debug: |
| 161 | print(self.usdt.get_text()) |
| 162 | return text |
| 163 | |
| 164 | def load(self): |
| 165 | trace_count_text = """ |
| 166 | int PROBE_FUNCTION(void *ctx) { |
Alexei Starovoitov | bdf0773 | 2016-01-14 10:09:20 -0800 | [diff] [blame] | 167 | FILTER |
Sasha Goldshtein | dd7ec5a | 2016-10-25 07:18:24 -0700 | [diff] [blame] | 168 | int loc = LOCATION; |
| 169 | u64 *val = counts.lookup(&loc); |
| 170 | if (!val) { |
| 171 | return 0; // Should never happen, # of locations is known |
| 172 | } |
Sasha Goldshtein | 4e9289b | 2016-10-20 16:19:12 -0700 | [diff] [blame] | 173 | (*val)++; |
Alexei Starovoitov | bdf0773 | 2016-01-14 10:09:20 -0800 | [diff] [blame] | 174 | return 0; |
Brendan Gregg | 3e55ae2 | 2015-09-10 12:11:35 -0700 | [diff] [blame] | 175 | } |
Sasha Goldshtein | ff3b9f3 | 2016-10-08 07:01:21 -0700 | [diff] [blame] | 176 | """ |
| 177 | bpf_text = """#include <uapi/linux/ptrace.h> |
Brenden Blanco | 9572883 | 2016-02-23 13:13:57 -0800 | [diff] [blame] | 178 | |
Teng Qin | 7a3e5bc | 2017-03-29 13:39:17 -0700 | [diff] [blame] | 179 | BPF_ARRAY(counts, u64, NUMLOCATIONS); |
Sasha Goldshtein | ff3b9f3 | 2016-10-08 07:01:21 -0700 | [diff] [blame] | 180 | """ |
Brendan Gregg | 3e55ae2 | 2015-09-10 12:11:35 -0700 | [diff] [blame] | 181 | |
Sasha Goldshtein | ff3b9f3 | 2016-10-08 07:01:21 -0700 | [diff] [blame] | 182 | # We really mean the tgid from the kernel's perspective, which is in |
| 183 | # the top 32 bits of bpf_get_current_pid_tgid(). |
| 184 | if self.pid: |
| 185 | trace_count_text = trace_count_text.replace('FILTER', |
| 186 | """u32 pid = bpf_get_current_pid_tgid() >> 32; |
| 187 | if (pid != %d) { return 0; }""" % self.pid) |
| 188 | else: |
| 189 | trace_count_text = trace_count_text.replace('FILTER', '') |
Brendan Gregg | 3e55ae2 | 2015-09-10 12:11:35 -0700 | [diff] [blame] | 190 | |
Sasha Goldshtein | ff3b9f3 | 2016-10-08 07:01:21 -0700 | [diff] [blame] | 191 | bpf_text += self._generate_functions(trace_count_text) |
Sasha Goldshtein | dd7ec5a | 2016-10-25 07:18:24 -0700 | [diff] [blame] | 192 | bpf_text = bpf_text.replace("NUMLOCATIONS", |
| 193 | str(len(self.trace_functions))) |
Sasha Goldshtein | ff3b9f3 | 2016-10-08 07:01:21 -0700 | [diff] [blame] | 194 | if debug: |
| 195 | print(bpf_text) |
| 196 | |
Sasha Goldshtein | 8e6109b | 2016-10-28 23:45:08 +0300 | [diff] [blame] | 197 | if self.matched == 0: |
| 198 | raise Exception("No functions matched by pattern %s" % |
| 199 | self.pattern) |
| 200 | |
Sasha Goldshtein | ff3b9f3 | 2016-10-08 07:01:21 -0700 | [diff] [blame] | 201 | self.bpf = BPF(text=bpf_text, |
| 202 | usdt_contexts=[self.usdt] if self.usdt else []) |
Sasha Goldshtein | dd7ec5a | 2016-10-25 07:18:24 -0700 | [diff] [blame] | 203 | self.clear() # Initialize all array items to zero |
| 204 | |
| 205 | def counts(self): |
| 206 | return self.bpf["counts"] |
| 207 | |
| 208 | def clear(self): |
| 209 | counts = self.bpf["counts"] |
| 210 | for location, _ in list(self.trace_functions.items()): |
| 211 | counts[counts.Key(location)] = counts.Leaf() |
Sasha Goldshtein | ff3b9f3 | 2016-10-08 07:01:21 -0700 | [diff] [blame] | 212 | |
Sasha Goldshtein | ff3b9f3 | 2016-10-08 07:01:21 -0700 | [diff] [blame] | 213 | class Tool(object): |
| 214 | def __init__(self): |
| 215 | examples = """examples: |
| 216 | ./funccount 'vfs_*' # count kernel fns starting with "vfs" |
| 217 | ./funccount -r '^vfs.*' # same as above, using regular expressions |
| 218 | ./funccount -Ti 5 'vfs_*' # output every 5 seconds, with timestamps |
Brendan Gregg | b03d9eb | 2017-08-23 15:00:30 -0700 | [diff] [blame] | 219 | ./funccount -d 10 'vfs_*' # trace for 10 seconds only |
Sasha Goldshtein | ff3b9f3 | 2016-10-08 07:01:21 -0700 | [diff] [blame] | 220 | ./funccount -p 185 'vfs_*' # count vfs calls for PID 181 only |
| 221 | ./funccount t:sched:sched_fork # count calls to the sched_fork tracepoint |
Brendan Gregg | b6035b6 | 2017-01-10 17:36:07 -0800 | [diff] [blame] | 222 | ./funccount -p 185 u:node:gc* # count all GC USDT probes in node, PID 185 |
Sasha Goldshtein | ff3b9f3 | 2016-10-08 07:01:21 -0700 | [diff] [blame] | 223 | ./funccount c:malloc # count all malloc() calls in libc |
Brendan Gregg | b6035b6 | 2017-01-10 17:36:07 -0800 | [diff] [blame] | 224 | ./funccount go:os.* # count all "os.*" calls in libgo |
| 225 | ./funccount -p 185 go:os.* # count all "os.*" calls in libgo, PID 185 |
| 226 | ./funccount ./test:read* # count "read*" calls in the ./test binary |
Sasha Goldshtein | ff3b9f3 | 2016-10-08 07:01:21 -0700 | [diff] [blame] | 227 | """ |
| 228 | parser = argparse.ArgumentParser( |
| 229 | description="Count functions, tracepoints, and USDT probes", |
| 230 | formatter_class=argparse.RawDescriptionHelpFormatter, |
| 231 | epilog=examples) |
| 232 | parser.add_argument("-p", "--pid", type=int, |
| 233 | help="trace this PID only") |
Brendan Gregg | b03d9eb | 2017-08-23 15:00:30 -0700 | [diff] [blame] | 234 | parser.add_argument("-i", "--interval", |
Sasha Goldshtein | ff3b9f3 | 2016-10-08 07:01:21 -0700 | [diff] [blame] | 235 | help="summary interval, seconds") |
Brendan Gregg | b03d9eb | 2017-08-23 15:00:30 -0700 | [diff] [blame] | 236 | parser.add_argument("-d", "--duration", |
| 237 | help="total duration of trace, seconds") |
Sasha Goldshtein | ff3b9f3 | 2016-10-08 07:01:21 -0700 | [diff] [blame] | 238 | parser.add_argument("-T", "--timestamp", action="store_true", |
| 239 | help="include timestamp on output") |
| 240 | parser.add_argument("-r", "--regexp", action="store_true", |
| 241 | help="use regular expressions. Default is \"*\" wildcards only.") |
Brendan Gregg | b03d9eb | 2017-08-23 15:00:30 -0700 | [diff] [blame] | 242 | parser.add_argument("-D", "--debug", action="store_true", |
Sasha Goldshtein | ff3b9f3 | 2016-10-08 07:01:21 -0700 | [diff] [blame] | 243 | help="print BPF program before starting (for debugging purposes)") |
| 244 | parser.add_argument("pattern", |
| 245 | help="search expression for events") |
| 246 | self.args = parser.parse_args() |
| 247 | global debug |
| 248 | debug = self.args.debug |
| 249 | self.probe = Probe(self.args.pattern, self.args.regexp, self.args.pid) |
Brendan Gregg | b03d9eb | 2017-08-23 15:00:30 -0700 | [diff] [blame] | 250 | if self.args.duration and not self.args.interval: |
| 251 | self.args.interval = self.args.duration |
| 252 | if not self.args.interval: |
| 253 | self.args.interval = 99999999 |
Sasha Goldshtein | ff3b9f3 | 2016-10-08 07:01:21 -0700 | [diff] [blame] | 254 | |
| 255 | @staticmethod |
| 256 | def _signal_ignore(signal, frame): |
| 257 | print() |
| 258 | |
| 259 | def run(self): |
| 260 | self.probe.load() |
| 261 | self.probe.attach() |
| 262 | print("Tracing %d functions for \"%s\"... Hit Ctrl-C to end." % |
| 263 | (self.probe.matched, self.args.pattern)) |
| 264 | exiting = 0 if self.args.interval else 1 |
Brendan Gregg | b03d9eb | 2017-08-23 15:00:30 -0700 | [diff] [blame] | 265 | seconds = 0 |
Sasha Goldshtein | ff3b9f3 | 2016-10-08 07:01:21 -0700 | [diff] [blame] | 266 | while True: |
| 267 | try: |
| 268 | sleep(int(self.args.interval)) |
Brendan Gregg | b03d9eb | 2017-08-23 15:00:30 -0700 | [diff] [blame] | 269 | seconds += int(self.args.interval) |
Sasha Goldshtein | ff3b9f3 | 2016-10-08 07:01:21 -0700 | [diff] [blame] | 270 | except KeyboardInterrupt: |
| 271 | exiting = 1 |
| 272 | # as cleanup can take many seconds, trap Ctrl-C: |
| 273 | signal.signal(signal.SIGINT, Tool._signal_ignore) |
Brendan Gregg | b03d9eb | 2017-08-23 15:00:30 -0700 | [diff] [blame] | 274 | if self.args.duration and seconds >= int(self.args.duration): |
| 275 | exiting = 1 |
Sasha Goldshtein | ff3b9f3 | 2016-10-08 07:01:21 -0700 | [diff] [blame] | 276 | |
| 277 | print() |
| 278 | if self.args.timestamp: |
| 279 | print("%-8s\n" % strftime("%H:%M:%S"), end="") |
| 280 | |
| 281 | print("%-36s %8s" % ("FUNC", "COUNT")) |
Sasha Goldshtein | dd7ec5a | 2016-10-25 07:18:24 -0700 | [diff] [blame] | 282 | counts = self.probe.counts() |
Sasha Goldshtein | ff3b9f3 | 2016-10-08 07:01:21 -0700 | [diff] [blame] | 283 | for k, v in sorted(counts.items(), |
| 284 | key=lambda counts: counts[1].value): |
| 285 | if v.value == 0: |
| 286 | continue |
| 287 | print("%-36s %8d" % |
| 288 | (self.probe.trace_functions[k.value], v.value)) |
Sasha Goldshtein | ff3b9f3 | 2016-10-08 07:01:21 -0700 | [diff] [blame] | 289 | |
| 290 | if exiting: |
| 291 | print("Detaching...") |
| 292 | exit() |
Sasha Goldshtein | dd7ec5a | 2016-10-25 07:18:24 -0700 | [diff] [blame] | 293 | else: |
| 294 | self.probe.clear() |
Sasha Goldshtein | ff3b9f3 | 2016-10-08 07:01:21 -0700 | [diff] [blame] | 295 | |
| 296 | if __name__ == "__main__": |
Alexei Starovoitov | bdf0773 | 2016-01-14 10:09:20 -0800 | [diff] [blame] | 297 | try: |
Sasha Goldshtein | ff3b9f3 | 2016-10-08 07:01:21 -0700 | [diff] [blame] | 298 | Tool().run() |
| 299 | except Exception: |
| 300 | if debug: |
| 301 | traceback.print_exc() |
| 302 | elif sys.exc_info()[0] is not SystemExit: |
| 303 | print(sys.exc_info()[1]) |