Sasha Goldshtein | 38847f0 | 2016-02-22 02:19:24 -0800 | [diff] [blame] | 1 | #!/usr/bin/env python |
| 2 | # |
| 3 | # trace Trace a function and print a trace message based on its |
| 4 | # parameters, with an optional filter. |
| 5 | # |
| 6 | # USAGE: trace [-h] [-p PID] [-v] [-Z STRING_SIZE] [-S] [-M MAX_EVENTS] [-o] |
| 7 | # probe [probe ...] |
Sasha Goldshtein | fd60d55 | 2016-03-01 12:15:34 -0800 | [diff] [blame] | 8 | # |
Sasha Goldshtein | 38847f0 | 2016-02-22 02:19:24 -0800 | [diff] [blame] | 9 | # Licensed under the Apache License, Version 2.0 (the "License") |
| 10 | # Copyright (C) 2016 Sasha Goldshtein. |
| 11 | |
Teng Qin | 9b04a6f | 2016-07-31 10:17:07 -0700 | [diff] [blame] | 12 | from bcc import BPF, Tracepoint, Perf, USDT |
Teng Qin | 6b0ed37 | 2016-09-29 21:30:13 -0700 | [diff] [blame] | 13 | from functools import partial |
Sasha Goldshtein | 38847f0 | 2016-02-22 02:19:24 -0800 | [diff] [blame] | 14 | from time import sleep, strftime |
| 15 | import argparse |
| 16 | import re |
| 17 | import ctypes as ct |
| 18 | import os |
| 19 | import traceback |
| 20 | import sys |
| 21 | |
| 22 | class Time(object): |
| 23 | # BPF timestamps come from the monotonic clock. To be able to filter |
| 24 | # and compare them from Python, we need to invoke clock_gettime. |
| 25 | # Adapted from http://stackoverflow.com/a/1205762 |
| 26 | CLOCK_MONOTONIC_RAW = 4 # see <linux/time.h> |
| 27 | |
| 28 | class timespec(ct.Structure): |
| 29 | _fields_ = [ |
| 30 | ('tv_sec', ct.c_long), |
| 31 | ('tv_nsec', ct.c_long) |
| 32 | ] |
| 33 | |
| 34 | librt = ct.CDLL('librt.so.1', use_errno=True) |
| 35 | clock_gettime = librt.clock_gettime |
| 36 | clock_gettime.argtypes = [ct.c_int, ct.POINTER(timespec)] |
| 37 | |
| 38 | @staticmethod |
| 39 | def monotonic_time(): |
| 40 | t = Time.timespec() |
| 41 | if Time.clock_gettime( |
| 42 | Time.CLOCK_MONOTONIC_RAW, ct.pointer(t)) != 0: |
| 43 | errno_ = ct.get_errno() |
| 44 | raise OSError(errno_, os.strerror(errno_)) |
| 45 | return t.tv_sec * 1e9 + t.tv_nsec |
| 46 | |
| 47 | class Probe(object): |
| 48 | probe_count = 0 |
| 49 | max_events = None |
| 50 | event_count = 0 |
| 51 | first_ts = 0 |
| 52 | use_localtime = True |
Sasha Goldshtein | 3e39a08 | 2016-03-24 08:39:47 -0700 | [diff] [blame] | 53 | pid = -1 |
Sasha Goldshtein | 38847f0 | 2016-02-22 02:19:24 -0800 | [diff] [blame] | 54 | |
| 55 | @classmethod |
| 56 | def configure(cls, args): |
| 57 | cls.max_events = args.max_events |
| 58 | cls.use_localtime = not args.offset |
| 59 | cls.first_ts = Time.monotonic_time() |
Sasha Goldshtein | 3e39a08 | 2016-03-24 08:39:47 -0700 | [diff] [blame] | 60 | cls.pid = args.pid or -1 |
Sasha Goldshtein | 38847f0 | 2016-02-22 02:19:24 -0800 | [diff] [blame] | 61 | |
Teng Qin | 6b0ed37 | 2016-09-29 21:30:13 -0700 | [diff] [blame] | 62 | def __init__(self, probe, string_size, kernel_stack, user_stack): |
Sasha Goldshtein | 69e361a | 2016-09-27 19:40:00 +0300 | [diff] [blame] | 63 | self.usdt = None |
Sasha Goldshtein | 38847f0 | 2016-02-22 02:19:24 -0800 | [diff] [blame] | 64 | self.raw_probe = probe |
| 65 | self.string_size = string_size |
Teng Qin | 6b0ed37 | 2016-09-29 21:30:13 -0700 | [diff] [blame] | 66 | self.kernel_stack = kernel_stack |
| 67 | self.user_stack = user_stack |
Sasha Goldshtein | 38847f0 | 2016-02-22 02:19:24 -0800 | [diff] [blame] | 68 | Probe.probe_count += 1 |
| 69 | self._parse_probe() |
| 70 | self.probe_num = Probe.probe_count |
| 71 | self.probe_name = "probe_%s_%d" % \ |
Sasha Goldshtein | 3e39a08 | 2016-03-24 08:39:47 -0700 | [diff] [blame] | 72 | (self._display_function(), self.probe_num) |
Sasha Goldshtein | 38847f0 | 2016-02-22 02:19:24 -0800 | [diff] [blame] | 73 | |
| 74 | def __str__(self): |
Sasha Goldshtein | 3e39a08 | 2016-03-24 08:39:47 -0700 | [diff] [blame] | 75 | return "%s:%s:%s FLT=%s ACT=%s/%s" % (self.probe_type, |
| 76 | self.library, self._display_function(), self.filter, |
Sasha Goldshtein | 38847f0 | 2016-02-22 02:19:24 -0800 | [diff] [blame] | 77 | self.types, self.values) |
| 78 | |
| 79 | def is_default_action(self): |
| 80 | return self.python_format == "" |
| 81 | |
| 82 | def _bail(self, error): |
Sasha Goldshtein | 3e39a08 | 2016-03-24 08:39:47 -0700 | [diff] [blame] | 83 | raise ValueError("error in probe '%s': %s" % |
Sasha Goldshtein | 38847f0 | 2016-02-22 02:19:24 -0800 | [diff] [blame] | 84 | (self.raw_probe, error)) |
| 85 | |
| 86 | def _parse_probe(self): |
| 87 | text = self.raw_probe |
| 88 | |
| 89 | # Everything until the first space is the probe specifier |
| 90 | first_space = text.find(' ') |
| 91 | spec = text[:first_space] if first_space >= 0 else text |
| 92 | self._parse_spec(spec) |
| 93 | if first_space >= 0: |
| 94 | text = text[first_space:].lstrip() |
| 95 | else: |
| 96 | text = "" |
| 97 | |
| 98 | # If we now have a (, wait for the balanced closing ) and that |
| 99 | # will be the predicate |
| 100 | self.filter = None |
| 101 | if len(text) > 0 and text[0] == "(": |
| 102 | balance = 1 |
| 103 | for i in range(1, len(text)): |
| 104 | if text[i] == "(": |
| 105 | balance += 1 |
| 106 | if text[i] == ")": |
| 107 | balance -= 1 |
| 108 | if balance == 0: |
| 109 | self._parse_filter(text[:i+1]) |
| 110 | text = text[i+1:] |
| 111 | break |
| 112 | if self.filter is None: |
| 113 | self._bail("unmatched end of predicate") |
| 114 | |
| 115 | if self.filter is None: |
| 116 | self.filter = "1" |
| 117 | |
| 118 | # The remainder of the text is the printf action |
| 119 | self._parse_action(text.lstrip()) |
| 120 | |
| 121 | def _parse_spec(self, spec): |
| 122 | parts = spec.split(":") |
| 123 | # Two special cases: 'func' means 'p::func', 'lib:func' means |
| 124 | # 'p:lib:func'. Other combinations need to provide an empty |
| 125 | # value between delimiters, e.g. 'r::func' for a kretprobe on |
| 126 | # the function func. |
| 127 | if len(parts) == 1: |
| 128 | parts = ["p", "", parts[0]] |
| 129 | elif len(parts) == 2: |
| 130 | parts = ["p", parts[0], parts[1]] |
| 131 | if len(parts[0]) == 0: |
| 132 | self.probe_type = "p" |
Sasha Goldshtein | 3e39a08 | 2016-03-24 08:39:47 -0700 | [diff] [blame] | 133 | elif parts[0] in ["p", "r", "t", "u"]: |
Sasha Goldshtein | 38847f0 | 2016-02-22 02:19:24 -0800 | [diff] [blame] | 134 | self.probe_type = parts[0] |
| 135 | else: |
Sasha Goldshtein | 3e39a08 | 2016-03-24 08:39:47 -0700 | [diff] [blame] | 136 | self._bail("probe type must be '', 'p', 't', 'r', " + |
| 137 | "or 'u', but got '%s'" % parts[0]) |
Sasha Goldshtein | fd60d55 | 2016-03-01 12:15:34 -0800 | [diff] [blame] | 138 | if self.probe_type == "t": |
| 139 | self.tp_category = parts[1] |
| 140 | self.tp_event = parts[2] |
Sasha Goldshtein | c08c431 | 2016-03-21 03:52:09 -0700 | [diff] [blame] | 141 | self.tp = Tracepoint.enable_tracepoint( |
| 142 | self.tp_category, self.tp_event) |
Sasha Goldshtein | fd60d55 | 2016-03-01 12:15:34 -0800 | [diff] [blame] | 143 | self.library = "" # kernel |
| 144 | self.function = "perf_trace_%s" % self.tp_event |
Sasha Goldshtein | 3e39a08 | 2016-03-24 08:39:47 -0700 | [diff] [blame] | 145 | elif self.probe_type == "u": |
| 146 | self.library = parts[1] |
| 147 | self.usdt_name = parts[2] |
| 148 | self.function = "" # no function, just address |
| 149 | # We will discover the USDT provider by matching on |
| 150 | # the USDT name in the specified library |
| 151 | self._find_usdt_probe() |
Sasha Goldshtein | fd60d55 | 2016-03-01 12:15:34 -0800 | [diff] [blame] | 152 | else: |
| 153 | self.library = parts[1] |
| 154 | self.function = parts[2] |
Sasha Goldshtein | 38847f0 | 2016-02-22 02:19:24 -0800 | [diff] [blame] | 155 | |
Sasha Goldshtein | 3e39a08 | 2016-03-24 08:39:47 -0700 | [diff] [blame] | 156 | def _find_usdt_probe(self): |
Sasha Goldshtein | 69e361a | 2016-09-27 19:40:00 +0300 | [diff] [blame] | 157 | self.usdt = USDT(path=self.library, pid=Probe.pid) |
| 158 | for probe in self.usdt.enumerate_probes(): |
Sasha Goldshtein | 3e39a08 | 2016-03-24 08:39:47 -0700 | [diff] [blame] | 159 | if probe.name == self.usdt_name: |
Sasha Goldshtein | 69e361a | 2016-09-27 19:40:00 +0300 | [diff] [blame] | 160 | return # Found it, will enable later |
Sasha Goldshtein | 3e39a08 | 2016-03-24 08:39:47 -0700 | [diff] [blame] | 161 | self._bail("unrecognized USDT probe %s" % self.usdt_name) |
| 162 | |
Sasha Goldshtein | 38847f0 | 2016-02-22 02:19:24 -0800 | [diff] [blame] | 163 | def _parse_filter(self, filt): |
| 164 | self.filter = self._replace_args(filt) |
| 165 | |
| 166 | def _parse_types(self, fmt): |
| 167 | for match in re.finditer( |
| 168 | r'[^%]%(s|u|d|llu|lld|hu|hd|x|llx|c)', fmt): |
| 169 | self.types.append(match.group(1)) |
| 170 | fmt = re.sub(r'([^%]%)(u|d|llu|lld|hu|hd)', r'\1d', fmt) |
| 171 | fmt = re.sub(r'([^%]%)(x|llx)', r'\1x', fmt) |
| 172 | self.python_format = fmt.strip('"') |
| 173 | |
| 174 | def _parse_action(self, action): |
| 175 | self.values = [] |
| 176 | self.types = [] |
| 177 | self.python_format = "" |
| 178 | if len(action) == 0: |
| 179 | return |
| 180 | |
Sasha Goldshtein | fd60d55 | 2016-03-01 12:15:34 -0800 | [diff] [blame] | 181 | action = action.strip() |
| 182 | match = re.search(r'(\".*\"),?(.*)', action) |
| 183 | if match is None: |
| 184 | self._bail("expected format string in \"s") |
| 185 | |
| 186 | self.raw_format = match.group(1) |
Sasha Goldshtein | 38847f0 | 2016-02-22 02:19:24 -0800 | [diff] [blame] | 187 | self._parse_types(self.raw_format) |
Sasha Goldshtein | fd60d55 | 2016-03-01 12:15:34 -0800 | [diff] [blame] | 188 | for part in match.group(2).split(','): |
Sasha Goldshtein | 38847f0 | 2016-02-22 02:19:24 -0800 | [diff] [blame] | 189 | part = self._replace_args(part) |
Sasha Goldshtein | fd60d55 | 2016-03-01 12:15:34 -0800 | [diff] [blame] | 190 | if len(part) > 0: |
| 191 | self.values.append(part) |
Sasha Goldshtein | 38847f0 | 2016-02-22 02:19:24 -0800 | [diff] [blame] | 192 | |
| 193 | aliases = { |
Naveen N. Rao | 4afa96a | 2016-05-03 14:54:21 +0530 | [diff] [blame] | 194 | "retval": "PT_REGS_RC(ctx)", |
| 195 | "arg1": "PT_REGS_PARM1(ctx)", |
| 196 | "arg2": "PT_REGS_PARM2(ctx)", |
| 197 | "arg3": "PT_REGS_PARM3(ctx)", |
| 198 | "arg4": "PT_REGS_PARM4(ctx)", |
| 199 | "arg5": "PT_REGS_PARM5(ctx)", |
| 200 | "arg6": "PT_REGS_PARM6(ctx)", |
Sasha Goldshtein | 38847f0 | 2016-02-22 02:19:24 -0800 | [diff] [blame] | 201 | "$uid": "(unsigned)(bpf_get_current_uid_gid() & 0xffffffff)", |
| 202 | "$gid": "(unsigned)(bpf_get_current_uid_gid() >> 32)", |
| 203 | "$pid": "(unsigned)(bpf_get_current_pid_tgid() & 0xffffffff)", |
| 204 | "$tgid": "(unsigned)(bpf_get_current_pid_tgid() >> 32)", |
| 205 | "$cpu": "bpf_get_smp_processor_id()" |
| 206 | } |
| 207 | |
| 208 | def _replace_args(self, expr): |
| 209 | for alias, replacement in Probe.aliases.items(): |
Sasha Goldshtein | 3e39a08 | 2016-03-24 08:39:47 -0700 | [diff] [blame] | 210 | # For USDT probes, we replace argN values with the |
Sasha Goldshtein | 69e361a | 2016-09-27 19:40:00 +0300 | [diff] [blame] | 211 | # actual arguments for that probe obtained using special |
| 212 | # bpf_readarg_N macros emitted at BPF construction. |
Sasha Goldshtein | 3e39a08 | 2016-03-24 08:39:47 -0700 | [diff] [blame] | 213 | if alias.startswith("arg") and self.probe_type == "u": |
| 214 | continue |
Sasha Goldshtein | 38847f0 | 2016-02-22 02:19:24 -0800 | [diff] [blame] | 215 | expr = expr.replace(alias, replacement) |
| 216 | return expr |
| 217 | |
| 218 | p_type = { "u": ct.c_uint, "d": ct.c_int, |
| 219 | "llu": ct.c_ulonglong, "lld": ct.c_longlong, |
| 220 | "hu": ct.c_ushort, "hd": ct.c_short, |
| 221 | "x": ct.c_uint, "llx": ct.c_ulonglong, |
| 222 | "c": ct.c_ubyte } |
| 223 | |
| 224 | def _generate_python_field_decl(self, idx, fields): |
| 225 | field_type = self.types[idx] |
| 226 | if field_type == "s": |
| 227 | ptype = ct.c_char * self.string_size |
| 228 | else: |
| 229 | ptype = Probe.p_type[field_type] |
| 230 | fields.append(("v%d" % idx, ptype)) |
| 231 | |
| 232 | def _generate_python_data_decl(self): |
| 233 | self.python_struct_name = "%s_%d_Data" % \ |
Sasha Goldshtein | 3e39a08 | 2016-03-24 08:39:47 -0700 | [diff] [blame] | 234 | (self._display_function(), self.probe_num) |
Sasha Goldshtein | 38847f0 | 2016-02-22 02:19:24 -0800 | [diff] [blame] | 235 | fields = [ |
| 236 | ("timestamp_ns", ct.c_ulonglong), |
| 237 | ("pid", ct.c_uint), |
| 238 | ("comm", ct.c_char * 16) # TASK_COMM_LEN |
| 239 | ] |
| 240 | for i in range(0, len(self.types)): |
| 241 | self._generate_python_field_decl(i, fields) |
Teng Qin | 6b0ed37 | 2016-09-29 21:30:13 -0700 | [diff] [blame] | 242 | if self.kernel_stack: |
| 243 | fields.append(("kernel_stack_id", ct.c_int)) |
| 244 | if self.user_stack: |
| 245 | fields.append(("user_stack_id", ct.c_int)) |
Sasha Goldshtein | 38847f0 | 2016-02-22 02:19:24 -0800 | [diff] [blame] | 246 | return type(self.python_struct_name, (ct.Structure,), |
| 247 | dict(_fields_=fields)) |
| 248 | |
| 249 | c_type = { "u": "unsigned int", "d": "int", |
| 250 | "llu": "unsigned long long", "lld": "long long", |
| 251 | "hu": "unsigned short", "hd": "short", |
| 252 | "x": "unsigned int", "llx": "unsigned long long", |
| 253 | "c": "char" } |
| 254 | fmt_types = c_type.keys() |
| 255 | |
| 256 | def _generate_field_decl(self, idx): |
| 257 | field_type = self.types[idx] |
| 258 | if field_type == "s": |
| 259 | return "char v%d[%d];\n" % (idx, self.string_size) |
| 260 | if field_type in Probe.fmt_types: |
| 261 | return "%s v%d;\n" % (Probe.c_type[field_type], idx) |
| 262 | self._bail("unrecognized format specifier %s" % field_type) |
| 263 | |
| 264 | def _generate_data_decl(self): |
| 265 | # The BPF program will populate values into the struct |
| 266 | # according to the format string, and the Python program will |
| 267 | # construct the final display string. |
| 268 | self.events_name = "%s_events" % self.probe_name |
| 269 | self.struct_name = "%s_data_t" % self.probe_name |
Teng Qin | 6b0ed37 | 2016-09-29 21:30:13 -0700 | [diff] [blame] | 270 | self.stacks_name = "%s_stacks" % self.probe_name |
| 271 | stack_table = "BPF_STACK_TRACE(%s, 1024);" % self.stacks_name \ |
| 272 | if (self.kernel_stack or self.user_stack) else "" |
Sasha Goldshtein | 38847f0 | 2016-02-22 02:19:24 -0800 | [diff] [blame] | 273 | data_fields = "" |
| 274 | for i, field_type in enumerate(self.types): |
| 275 | data_fields += " " + \ |
| 276 | self._generate_field_decl(i) |
| 277 | |
Teng Qin | 6b0ed37 | 2016-09-29 21:30:13 -0700 | [diff] [blame] | 278 | kernel_stack_str = " int kernel_stack_id;" \ |
| 279 | if self.kernel_stack else "" |
| 280 | user_stack_str = " int user_stack_id;" \ |
| 281 | if self.user_stack else "" |
| 282 | |
Sasha Goldshtein | 38847f0 | 2016-02-22 02:19:24 -0800 | [diff] [blame] | 283 | text = """ |
| 284 | struct %s |
| 285 | { |
| 286 | u64 timestamp_ns; |
| 287 | u32 pid; |
| 288 | char comm[TASK_COMM_LEN]; |
| 289 | %s |
Teng Qin | 6b0ed37 | 2016-09-29 21:30:13 -0700 | [diff] [blame] | 290 | %s |
| 291 | %s |
Sasha Goldshtein | 38847f0 | 2016-02-22 02:19:24 -0800 | [diff] [blame] | 292 | }; |
| 293 | |
| 294 | BPF_PERF_OUTPUT(%s); |
Teng Qin | 6b0ed37 | 2016-09-29 21:30:13 -0700 | [diff] [blame] | 295 | %s |
Sasha Goldshtein | 38847f0 | 2016-02-22 02:19:24 -0800 | [diff] [blame] | 296 | """ |
Teng Qin | 6b0ed37 | 2016-09-29 21:30:13 -0700 | [diff] [blame] | 297 | return text % (self.struct_name, data_fields, |
| 298 | kernel_stack_str, user_stack_str, |
| 299 | self.events_name, stack_table) |
Sasha Goldshtein | 38847f0 | 2016-02-22 02:19:24 -0800 | [diff] [blame] | 300 | |
| 301 | def _generate_field_assign(self, idx): |
| 302 | field_type = self.types[idx] |
Sasha Goldshtein | 69e361a | 2016-09-27 19:40:00 +0300 | [diff] [blame] | 303 | expr = self.values[idx].strip() |
| 304 | text = "" |
| 305 | if self.probe_type == "u" and expr[0:3] == "arg": |
| 306 | text = (" u64 %s;\n" + |
| 307 | " bpf_usdt_readarg(%s, ctx, &%s);\n") % \ |
| 308 | (expr, expr[3], expr) |
| 309 | |
Sasha Goldshtein | 38847f0 | 2016-02-22 02:19:24 -0800 | [diff] [blame] | 310 | if field_type == "s": |
Sasha Goldshtein | 69e361a | 2016-09-27 19:40:00 +0300 | [diff] [blame] | 311 | return text + """ |
Sasha Goldshtein | 38847f0 | 2016-02-22 02:19:24 -0800 | [diff] [blame] | 312 | if (%s != 0) { |
| 313 | bpf_probe_read(&__data.v%d, sizeof(__data.v%d), (void *)%s); |
| 314 | } |
| 315 | """ % (expr, idx, idx, expr) |
Sasha Goldshtein | 38847f0 | 2016-02-22 02:19:24 -0800 | [diff] [blame] | 316 | if field_type in Probe.fmt_types: |
Sasha Goldshtein | 69e361a | 2016-09-27 19:40:00 +0300 | [diff] [blame] | 317 | return text + " __data.v%d = (%s)%s;\n" % \ |
Sasha Goldshtein | 38847f0 | 2016-02-22 02:19:24 -0800 | [diff] [blame] | 318 | (idx, Probe.c_type[field_type], expr) |
| 319 | self._bail("unrecognized field type %s" % field_type) |
| 320 | |
Teng Qin | 0615bff | 2016-09-28 08:19:40 -0700 | [diff] [blame] | 321 | def _generate_usdt_filter_read(self): |
| 322 | text = "" |
| 323 | if self.probe_type == "u": |
| 324 | for arg, _ in Probe.aliases.items(): |
| 325 | if not (arg.startswith("arg") and (arg in self.filter)): |
| 326 | continue |
| 327 | arg_index = int(arg.replace("arg", "")) |
| 328 | arg_ctype = self.usdt.get_probe_arg_ctype( |
| 329 | self.usdt_name, arg_index) |
| 330 | if not arg_ctype: |
| 331 | self._bail("Unable to determine type of {} " |
| 332 | "in the filter".format(arg)) |
| 333 | text += """ |
| 334 | {} {}_filter; |
| 335 | bpf_usdt_readarg({}, ctx, &{}_filter); |
| 336 | """.format(arg_ctype, arg, arg_index, arg) |
| 337 | self.filter = self.filter.replace( |
| 338 | arg, "{}_filter".format(arg)) |
| 339 | return text |
| 340 | |
Sasha Goldshtein | 3e39a08 | 2016-03-24 08:39:47 -0700 | [diff] [blame] | 341 | def generate_program(self, include_self): |
Sasha Goldshtein | 38847f0 | 2016-02-22 02:19:24 -0800 | [diff] [blame] | 342 | data_decl = self._generate_data_decl() |
Sasha Goldshtein | 38847f0 | 2016-02-22 02:19:24 -0800 | [diff] [blame] | 343 | # kprobes don't have built-in pid filters, so we have to add |
| 344 | # it to the function body: |
Sasha Goldshtein | 3e39a08 | 2016-03-24 08:39:47 -0700 | [diff] [blame] | 345 | if len(self.library) == 0 and Probe.pid != -1: |
Sasha Goldshtein | 38847f0 | 2016-02-22 02:19:24 -0800 | [diff] [blame] | 346 | pid_filter = """ |
| 347 | u32 __pid = bpf_get_current_pid_tgid(); |
| 348 | if (__pid != %d) { return 0; } |
Sasha Goldshtein | de34c25 | 2016-06-30 12:16:39 +0300 | [diff] [blame] | 349 | """ % Probe.pid |
Sasha Goldshtein | 38847f0 | 2016-02-22 02:19:24 -0800 | [diff] [blame] | 350 | elif not include_self: |
| 351 | pid_filter = """ |
| 352 | u32 __pid = bpf_get_current_pid_tgid(); |
| 353 | if (__pid == %d) { return 0; } |
| 354 | """ % os.getpid() |
| 355 | else: |
| 356 | pid_filter = "" |
| 357 | |
Sasha Goldshtein | 3e39a08 | 2016-03-24 08:39:47 -0700 | [diff] [blame] | 358 | prefix = "" |
Sasha Goldshtein | 3e39a08 | 2016-03-24 08:39:47 -0700 | [diff] [blame] | 359 | signature = "struct pt_regs *ctx" |
| 360 | if self.probe_type == "t": |
| 361 | data_decl += self.tp.generate_struct() |
| 362 | prefix = self.tp.generate_get_struct() |
Sasha Goldshtein | 3e39a08 | 2016-03-24 08:39:47 -0700 | [diff] [blame] | 363 | |
Sasha Goldshtein | 38847f0 | 2016-02-22 02:19:24 -0800 | [diff] [blame] | 364 | data_fields = "" |
| 365 | for i, expr in enumerate(self.values): |
| 366 | data_fields += self._generate_field_assign(i) |
| 367 | |
Teng Qin | 6b0ed37 | 2016-09-29 21:30:13 -0700 | [diff] [blame] | 368 | stack_trace = "" |
| 369 | if self.user_stack: |
| 370 | stack_trace += """ |
| 371 | __data.user_stack_id = %s.get_stackid( |
| 372 | ctx, BPF_F_REUSE_STACKID | BPF_F_USER_STACK |
| 373 | );""" % self.stacks_name |
| 374 | if self.kernel_stack: |
| 375 | stack_trace += """ |
| 376 | __data.kernel_stack_id = %s.get_stackid( |
| 377 | ctx, BPF_F_REUSE_STACKID |
| 378 | );""" % self.stacks_name |
| 379 | |
Sasha Goldshtein | 38847f0 | 2016-02-22 02:19:24 -0800 | [diff] [blame] | 380 | text = """ |
Sasha Goldshtein | 69e361a | 2016-09-27 19:40:00 +0300 | [diff] [blame] | 381 | int %s(%s) |
Sasha Goldshtein | 38847f0 | 2016-02-22 02:19:24 -0800 | [diff] [blame] | 382 | { |
| 383 | %s |
Sasha Goldshtein | fd60d55 | 2016-03-01 12:15:34 -0800 | [diff] [blame] | 384 | %s |
Teng Qin | 0615bff | 2016-09-28 08:19:40 -0700 | [diff] [blame] | 385 | %s |
Sasha Goldshtein | 38847f0 | 2016-02-22 02:19:24 -0800 | [diff] [blame] | 386 | if (!(%s)) return 0; |
| 387 | |
| 388 | struct %s __data = {0}; |
| 389 | __data.timestamp_ns = bpf_ktime_get_ns(); |
| 390 | __data.pid = bpf_get_current_pid_tgid(); |
| 391 | bpf_get_current_comm(&__data.comm, sizeof(__data.comm)); |
| 392 | %s |
Teng Qin | 6b0ed37 | 2016-09-29 21:30:13 -0700 | [diff] [blame] | 393 | %s |
Sasha Goldshtein | 38847f0 | 2016-02-22 02:19:24 -0800 | [diff] [blame] | 394 | %s.perf_submit(ctx, &__data, sizeof(__data)); |
| 395 | return 0; |
| 396 | } |
| 397 | """ |
Sasha Goldshtein | 69e361a | 2016-09-27 19:40:00 +0300 | [diff] [blame] | 398 | text = text % (self.probe_name, signature, |
Teng Qin | 0615bff | 2016-09-28 08:19:40 -0700 | [diff] [blame] | 399 | pid_filter, prefix, |
| 400 | self._generate_usdt_filter_read(), self.filter, |
Teng Qin | 6b0ed37 | 2016-09-29 21:30:13 -0700 | [diff] [blame] | 401 | self.struct_name, data_fields, |
| 402 | stack_trace, self.events_name) |
Sasha Goldshtein | 3e39a08 | 2016-03-24 08:39:47 -0700 | [diff] [blame] | 403 | |
Sasha Goldshtein | 38847f0 | 2016-02-22 02:19:24 -0800 | [diff] [blame] | 404 | return data_decl + "\n" + text |
| 405 | |
| 406 | @classmethod |
| 407 | def _time_off_str(cls, timestamp_ns): |
| 408 | return "%.6f" % (1e-9 * (timestamp_ns - cls.first_ts)) |
| 409 | |
Sasha Goldshtein | fd60d55 | 2016-03-01 12:15:34 -0800 | [diff] [blame] | 410 | def _display_function(self): |
Sasha Goldshtein | 3e39a08 | 2016-03-24 08:39:47 -0700 | [diff] [blame] | 411 | if self.probe_type == 'p' or self.probe_type == 'r': |
Sasha Goldshtein | fd60d55 | 2016-03-01 12:15:34 -0800 | [diff] [blame] | 412 | return self.function |
Sasha Goldshtein | 3e39a08 | 2016-03-24 08:39:47 -0700 | [diff] [blame] | 413 | elif self.probe_type == 'u': |
| 414 | return self.usdt_name |
| 415 | else: # self.probe_type == 't' |
| 416 | return self.tp_event |
Sasha Goldshtein | fd60d55 | 2016-03-01 12:15:34 -0800 | [diff] [blame] | 417 | |
Teng Qin | 6b0ed37 | 2016-09-29 21:30:13 -0700 | [diff] [blame] | 418 | def print_stack(self, bpf, stack_id, pid): |
| 419 | if stack_id < 0: |
| 420 | print(" %d" % stack_id) |
| 421 | return |
| 422 | |
| 423 | stack = list(bpf.get_table(self.stacks_name).walk(stack_id)) |
| 424 | for addr in stack: |
| 425 | print(" %016x %s" % (addr, bpf.sym(addr, pid))) |
| 426 | |
| 427 | def print_event(self, bpf, cpu, data, size): |
Sasha Goldshtein | 38847f0 | 2016-02-22 02:19:24 -0800 | [diff] [blame] | 428 | # Cast as the generated structure type and display |
| 429 | # according to the format string in the probe. |
| 430 | event = ct.cast(data, ct.POINTER(self.python_struct)).contents |
| 431 | values = map(lambda i: getattr(event, "v%d" % i), |
| 432 | range(0, len(self.values))) |
| 433 | msg = self.python_format % tuple(values) |
| 434 | time = strftime("%H:%M:%S") if Probe.use_localtime else \ |
| 435 | Probe._time_off_str(event.timestamp_ns) |
| 436 | print("%-8s %-6d %-12s %-16s %s" % \ |
Sasha Goldshtein | fd60d55 | 2016-03-01 12:15:34 -0800 | [diff] [blame] | 437 | (time[:8], event.pid, event.comm[:12], |
| 438 | self._display_function(), msg)) |
Sasha Goldshtein | 38847f0 | 2016-02-22 02:19:24 -0800 | [diff] [blame] | 439 | |
Teng Qin | 6b0ed37 | 2016-09-29 21:30:13 -0700 | [diff] [blame] | 440 | if self.user_stack: |
| 441 | print(" User Stack Trace:") |
| 442 | self.print_stack(bpf, event.user_stack_id, event.pid) |
| 443 | if self.kernel_stack: |
| 444 | print(" Kernel Stack Trace:") |
| 445 | self.print_stack(bpf, event.kernel_stack_id, -1) |
| 446 | if self.user_stack or self.kernel_stack: |
| 447 | print("") |
| 448 | |
Sasha Goldshtein | 38847f0 | 2016-02-22 02:19:24 -0800 | [diff] [blame] | 449 | Probe.event_count += 1 |
| 450 | if Probe.max_events is not None and \ |
| 451 | Probe.event_count >= Probe.max_events: |
| 452 | exit() |
| 453 | |
| 454 | def attach(self, bpf, verbose): |
| 455 | if len(self.library) == 0: |
| 456 | self._attach_k(bpf) |
| 457 | else: |
| 458 | self._attach_u(bpf) |
| 459 | self.python_struct = self._generate_python_data_decl() |
Teng Qin | 6b0ed37 | 2016-09-29 21:30:13 -0700 | [diff] [blame] | 460 | callback = partial(self.print_event, bpf) |
| 461 | bpf[self.events_name].open_perf_buffer(callback) |
Sasha Goldshtein | 38847f0 | 2016-02-22 02:19:24 -0800 | [diff] [blame] | 462 | |
| 463 | def _attach_k(self, bpf): |
| 464 | if self.probe_type == "r": |
| 465 | bpf.attach_kretprobe(event=self.function, |
| 466 | fn_name=self.probe_name) |
Sasha Goldshtein | fd60d55 | 2016-03-01 12:15:34 -0800 | [diff] [blame] | 467 | elif self.probe_type == "p" or self.probe_type == "t": |
Sasha Goldshtein | 38847f0 | 2016-02-22 02:19:24 -0800 | [diff] [blame] | 468 | bpf.attach_kprobe(event=self.function, |
| 469 | fn_name=self.probe_name) |
| 470 | |
| 471 | def _attach_u(self, bpf): |
| 472 | libpath = BPF.find_library(self.library) |
| 473 | if libpath is None: |
| 474 | # This might be an executable (e.g. 'bash') |
Teng Qin | 9b04a6f | 2016-07-31 10:17:07 -0700 | [diff] [blame] | 475 | libpath = BPF._find_exe(self.library) |
Sasha Goldshtein | 38847f0 | 2016-02-22 02:19:24 -0800 | [diff] [blame] | 476 | if libpath is None or len(libpath) == 0: |
| 477 | self._bail("unable to find library %s" % self.library) |
| 478 | |
Sasha Goldshtein | 3e39a08 | 2016-03-24 08:39:47 -0700 | [diff] [blame] | 479 | if self.probe_type == "u": |
Sasha Goldshtein | 69e361a | 2016-09-27 19:40:00 +0300 | [diff] [blame] | 480 | pass # Was already enabled by the BPF constructor |
Sasha Goldshtein | 3e39a08 | 2016-03-24 08:39:47 -0700 | [diff] [blame] | 481 | elif self.probe_type == "r": |
Sasha Goldshtein | 38847f0 | 2016-02-22 02:19:24 -0800 | [diff] [blame] | 482 | bpf.attach_uretprobe(name=libpath, |
| 483 | sym=self.function, |
| 484 | fn_name=self.probe_name, |
Sasha Goldshtein | 3e39a08 | 2016-03-24 08:39:47 -0700 | [diff] [blame] | 485 | pid=Probe.pid) |
Sasha Goldshtein | 38847f0 | 2016-02-22 02:19:24 -0800 | [diff] [blame] | 486 | else: |
| 487 | bpf.attach_uprobe(name=libpath, |
| 488 | sym=self.function, |
| 489 | fn_name=self.probe_name, |
Sasha Goldshtein | 3e39a08 | 2016-03-24 08:39:47 -0700 | [diff] [blame] | 490 | pid=Probe.pid) |
Sasha Goldshtein | 38847f0 | 2016-02-22 02:19:24 -0800 | [diff] [blame] | 491 | |
| 492 | class Tool(object): |
| 493 | examples = """ |
| 494 | EXAMPLES: |
| 495 | |
Sasha Goldshtein | 8acd015 | 2016-02-22 02:25:03 -0800 | [diff] [blame] | 496 | trace do_sys_open |
Sasha Goldshtein | 38847f0 | 2016-02-22 02:19:24 -0800 | [diff] [blame] | 497 | Trace the open syscall and print a default trace message when entered |
Sasha Goldshtein | 8acd015 | 2016-02-22 02:25:03 -0800 | [diff] [blame] | 498 | trace 'do_sys_open "%s", arg2' |
Sasha Goldshtein | 38847f0 | 2016-02-22 02:19:24 -0800 | [diff] [blame] | 499 | Trace the open syscall and print the filename being opened |
Sasha Goldshtein | 8acd015 | 2016-02-22 02:25:03 -0800 | [diff] [blame] | 500 | trace 'sys_read (arg3 > 20000) "read %d bytes", arg3' |
Sasha Goldshtein | 38847f0 | 2016-02-22 02:19:24 -0800 | [diff] [blame] | 501 | Trace the read syscall and print a message for reads >20000 bytes |
| 502 | trace 'r::do_sys_return "%llx", retval' |
| 503 | Trace the return from the open syscall and print the return value |
Sasha Goldshtein | 8acd015 | 2016-02-22 02:25:03 -0800 | [diff] [blame] | 504 | trace 'c:open (arg2 == 42) "%s %d", arg1, arg2' |
Sasha Goldshtein | 38847f0 | 2016-02-22 02:19:24 -0800 | [diff] [blame] | 505 | Trace the open() call from libc only if the flags (arg2) argument is 42 |
Sasha Goldshtein | 8acd015 | 2016-02-22 02:25:03 -0800 | [diff] [blame] | 506 | trace 'c:malloc "size = %d", arg1' |
Sasha Goldshtein | 38847f0 | 2016-02-22 02:19:24 -0800 | [diff] [blame] | 507 | Trace malloc calls and print the size being allocated |
Sasha Goldshtein | 8acd015 | 2016-02-22 02:25:03 -0800 | [diff] [blame] | 508 | trace 'p:c:write (arg1 == 1) "writing %d bytes to STDOUT", arg3' |
| 509 | Trace the write() call from libc to monitor writes to STDOUT |
| 510 | trace 'r::__kmalloc (retval == 0) "kmalloc failed!" |
| 511 | Trace returns from __kmalloc which returned a null pointer |
Sasha Goldshtein | 38847f0 | 2016-02-22 02:19:24 -0800 | [diff] [blame] | 512 | trace 'r:c:malloc (retval) "allocated = %p", retval |
| 513 | Trace returns from malloc and print non-NULL allocated buffers |
Sasha Goldshtein | fd60d55 | 2016-03-01 12:15:34 -0800 | [diff] [blame] | 514 | trace 't:block:block_rq_complete "sectors=%d", tp.nr_sector' |
| 515 | Trace the block_rq_complete kernel tracepoint and print # of tx sectors |
Sasha Goldshtein | 3e39a08 | 2016-03-24 08:39:47 -0700 | [diff] [blame] | 516 | trace 'u:pthread:pthread_create (arg4 != 0)' |
| 517 | Trace the USDT probe pthread_create when its 4th argument is non-zero |
Sasha Goldshtein | 38847f0 | 2016-02-22 02:19:24 -0800 | [diff] [blame] | 518 | """ |
| 519 | |
| 520 | def __init__(self): |
| 521 | parser = argparse.ArgumentParser(description= |
| 522 | "Attach to functions and print trace messages.", |
| 523 | formatter_class=argparse.RawDescriptionHelpFormatter, |
| 524 | epilog=Tool.examples) |
| 525 | parser.add_argument("-p", "--pid", type=int, |
| 526 | help="id of the process to trace (optional)") |
| 527 | parser.add_argument("-v", "--verbose", action="store_true", |
| 528 | help="print resulting BPF program code before executing") |
| 529 | parser.add_argument("-Z", "--string-size", type=int, |
| 530 | default=80, help="maximum size to read from strings") |
| 531 | parser.add_argument("-S", "--include-self", action="store_true", |
| 532 | help="do not filter trace's own pid from the trace") |
| 533 | parser.add_argument("-M", "--max-events", type=int, |
| 534 | help="number of events to print before quitting") |
| 535 | parser.add_argument("-o", "--offset", action="store_true", |
| 536 | help="use relative time from first traced message") |
Teng Qin | 6b0ed37 | 2016-09-29 21:30:13 -0700 | [diff] [blame] | 537 | parser.add_argument("-K", "--kernel-stack", action="store_true", |
| 538 | help="output kernel stack trace") |
| 539 | parser.add_argument("-U", "--user_stack", action="store_true", |
| 540 | help="output user stack trace") |
Sasha Goldshtein | 38847f0 | 2016-02-22 02:19:24 -0800 | [diff] [blame] | 541 | parser.add_argument(metavar="probe", dest="probes", nargs="+", |
| 542 | help="probe specifier (see examples)") |
| 543 | self.args = parser.parse_args() |
| 544 | |
| 545 | def _create_probes(self): |
| 546 | Probe.configure(self.args) |
| 547 | self.probes = [] |
| 548 | for probe_spec in self.args.probes: |
| 549 | self.probes.append(Probe( |
Teng Qin | 6b0ed37 | 2016-09-29 21:30:13 -0700 | [diff] [blame] | 550 | probe_spec, self.args.string_size, |
| 551 | self.args.kernel_stack, self.args.user_stack)) |
Sasha Goldshtein | 38847f0 | 2016-02-22 02:19:24 -0800 | [diff] [blame] | 552 | |
| 553 | def _generate_program(self): |
| 554 | self.program = """ |
| 555 | #include <linux/ptrace.h> |
| 556 | #include <linux/sched.h> /* For TASK_COMM_LEN */ |
| 557 | |
| 558 | """ |
Sasha Goldshtein | b950d6f | 2016-03-21 04:06:15 -0700 | [diff] [blame] | 559 | self.program += BPF.generate_auto_includes( |
Sasha Goldshtein | fd60d55 | 2016-03-01 12:15:34 -0800 | [diff] [blame] | 560 | map(lambda p: p.raw_probe, self.probes)) |
| 561 | self.program += Tracepoint.generate_decl() |
| 562 | self.program += Tracepoint.generate_entry_probe() |
Sasha Goldshtein | 38847f0 | 2016-02-22 02:19:24 -0800 | [diff] [blame] | 563 | for probe in self.probes: |
| 564 | self.program += probe.generate_program( |
Sasha Goldshtein | 3e39a08 | 2016-03-24 08:39:47 -0700 | [diff] [blame] | 565 | self.args.include_self) |
Sasha Goldshtein | 38847f0 | 2016-02-22 02:19:24 -0800 | [diff] [blame] | 566 | |
| 567 | if self.args.verbose: |
| 568 | print(self.program) |
| 569 | |
| 570 | def _attach_probes(self): |
Sasha Goldshtein | 69e361a | 2016-09-27 19:40:00 +0300 | [diff] [blame] | 571 | usdt_contexts = [] |
| 572 | for probe in self.probes: |
| 573 | if probe.usdt: |
| 574 | # USDT probes must be enabled before the BPF object |
| 575 | # is initialized, because that's where the actual |
| 576 | # uprobe is being attached. |
| 577 | probe.usdt.enable_probe( |
| 578 | probe.usdt_name, probe.probe_name) |
| 579 | usdt_contexts.append(probe.usdt) |
| 580 | self.bpf = BPF(text=self.program, usdt_contexts=usdt_contexts) |
Sasha Goldshtein | fd60d55 | 2016-03-01 12:15:34 -0800 | [diff] [blame] | 581 | Tracepoint.attach(self.bpf) |
Sasha Goldshtein | 38847f0 | 2016-02-22 02:19:24 -0800 | [diff] [blame] | 582 | for probe in self.probes: |
| 583 | if self.args.verbose: |
| 584 | print(probe) |
| 585 | probe.attach(self.bpf, self.args.verbose) |
| 586 | |
| 587 | def _main_loop(self): |
| 588 | all_probes_trivial = all(map(Probe.is_default_action, |
| 589 | self.probes)) |
| 590 | |
| 591 | # Print header |
| 592 | print("%-8s %-6s %-12s %-16s %s" % \ |
| 593 | ("TIME", "PID", "COMM", "FUNC", |
| 594 | "-" if not all_probes_trivial else "")) |
| 595 | |
| 596 | while True: |
| 597 | self.bpf.kprobe_poll() |
| 598 | |
| 599 | def run(self): |
| 600 | try: |
| 601 | self._create_probes() |
| 602 | self._generate_program() |
| 603 | self._attach_probes() |
| 604 | self._main_loop() |
| 605 | except: |
| 606 | if self.args.verbose: |
| 607 | traceback.print_exc() |
Brenden Blanco | de14f4f | 2016-04-08 15:52:55 -0700 | [diff] [blame] | 608 | elif sys.exc_info()[0] is not SystemExit: |
| 609 | print(sys.exc_info()[1]) |
Sasha Goldshtein | 38847f0 | 2016-02-22 02:19:24 -0800 | [diff] [blame] | 610 | |
| 611 | if __name__ == "__main__": |
| 612 | Tool().run() |