argdist, trace, and tplist support for USDT probes
These tools now support USDT probes with the 'u:provider:probe' syntax.
Probes in a library or process can be listed with 'tplist -l LIB' or 'tplist -p PID'.
Probe arguments are also parsed and available in both argdist and trace as arg1,
arg2, etc., regardless of the probe attach location.
The same USDT probe can be used at multiple locations, which means the attach infra-
structure must probe all these locations. argdist and trace register thunk probes
at each location, which call a central probe function (which is static inline) with
the location id (__loc_id). The central probe function checks the location id to
determine how the arguments should be retrieved -- this is location-dependent.
Finally, some USDT probes must be enabled first by writing a value to a memory
location (this is called a "semaphore"). This value is per-process, so we require a
process id for this kind of probes.
Along with trace and argdist tool support, this commit also introduces new classes
in the bcc module: ProcStat handles pid-wrap detection, whereas USDTReader,
USDTProbe, USDTProbeLocation, and USDTArgument are the shared USDT-related
infrastructure that enables enumeration, attachment, and argument retrieval for
USDT probes.
diff --git a/tools/argdist.py b/tools/argdist.py
index 8f8327d..d3c5239 100755
--- a/tools/argdist.py
+++ b/tools/argdist.py
@@ -12,7 +12,7 @@
# Licensed under the Apache License, Version 2.0 (the "License")
# Copyright (C) 2016 Sasha Goldshtein.
-from bcc import BPF, Tracepoint, Perf
+from bcc import BPF, Tracepoint, Perf, USDTReader
from time import sleep, strftime
import argparse
import re
@@ -20,27 +20,14 @@
import os
import sys
-class Specifier(object):
- probe_text = """
-DATA_DECL
-
-int PROBENAME(struct pt_regs *ctx SIGNATURE)
-{
- PREFIX
- PID_FILTER
- if (!(FILTER)) return 0;
- KEY_EXPR
- COLLECT
- return 0;
-}
-"""
+class Probe(object):
next_probe_index = 0
aliases = { "$PID": "bpf_get_current_pid_tgid()" }
def _substitute_aliases(self, expr):
if expr is None:
return expr
- for alias, subst in Specifier.aliases.items():
+ for alias, subst in Probe.aliases.items():
expr = expr.replace(alias, subst)
return expr
@@ -57,7 +44,9 @@
param_name = param[index+1:].strip()
self.param_types[param_name] = param_type
- entry_probe_text = """
+ def _generate_entry(self):
+ self.entry_probe_func = self.probe_func_name + "_entry"
+ text = """
int PROBENAME(struct pt_regs *ctx SIGNATURE)
{
u32 pid = bpf_get_current_pid_tgid();
@@ -66,10 +55,6 @@
return 0;
}
"""
-
- def _generate_entry(self):
- self.entry_probe_func = self.probe_func_name + "_entry"
- text = self.entry_probe_text
text = text.replace("PROBENAME", self.entry_probe_func)
text = text.replace("SIGNATURE",
"" if len(self.signature) == 0 else ", " + self.signature)
@@ -173,8 +158,8 @@
"function signature must be specified")
if len(parts) > 6:
self._bail("extraneous ':'-separated parts detected")
- if parts[0] not in ["r", "p", "t"]:
- self._bail("probe type must be 'p', 'r', or 't', " +
+ if parts[0] not in ["r", "p", "t", "u"]:
+ self._bail("probe type must be 'p', 'r', 't', or 'u' " +
"but got '%s'" % parts[0])
if re.match(r"\w+\(.*\)", parts[2]) is None:
self._bail(("function signature '%s' has an invalid " +
@@ -191,6 +176,7 @@
self.exprs = exprs.split(',')
def __init__(self, type, specifier, pid):
+ self.pid = pid
self.raw_spec = specifier
self._validate_specifier()
@@ -210,6 +196,10 @@
self.tp = Tracepoint.enable_tracepoint(
self.tp_category, self.tp_event)
self.function = "perf_trace_" + self.function
+ elif self.probe_type == "u":
+ self.library = parts[1]
+ self._find_usdt_probe()
+ self._enable_usdt_probe()
else:
self.library = parts[1]
self.is_user = len(self.library) > 0
@@ -244,12 +234,32 @@
self.entry_probe_required = self.probe_type == "r" and \
(any(map(check, self.exprs)) or check(self.filter))
- self.pid = pid
self.probe_func_name = "%s_probe%d" % \
- (self.function, Specifier.next_probe_index)
+ (self.function, Probe.next_probe_index)
self.probe_hash_name = "%s_hash%d" % \
- (self.function, Specifier.next_probe_index)
- Specifier.next_probe_index += 1
+ (self.function, Probe.next_probe_index)
+ Probe.next_probe_index += 1
+
+ def _enable_usdt_probe(self):
+ if self.usdt.need_enable():
+ if self.pid is None:
+ self._bail("probe needs pid to enable")
+ self.usdt.enable(self.pid)
+
+ def _disable_usdt_probe(self):
+ if self.probe_type == "u" and self.usdt.need_enable():
+ self.usdt.disable(self.pid)
+
+ def close(self):
+ self._disable_usdt_probe()
+
+ def _find_usdt_probe(self):
+ reader = USDTReader(bin_path=self.library)
+ for probe in reader.probes:
+ if probe.name == self.function:
+ self.usdt = probe
+ return
+ self._bail("unrecognized USDT probe %s" % self.function)
def _substitute_exprs(self):
def repl(expr):
@@ -270,8 +280,8 @@
def _generate_field_assignment(self, i):
if self._is_string(self.expr_types[i]):
- return " bpf_probe_read(" + \
- "&__key.v%d.s, sizeof(__key.v%d.s), %s);\n" % \
+ return (" bpf_probe_read(&__key.v%d.s," +
+ " sizeof(__key.v%d.s), (void *)%s);\n") % \
(i, i, self.exprs[i])
else:
return " __key.v%d = %s;\n" % (i, self.exprs[i])
@@ -318,10 +328,25 @@
def generate_text(self):
program = ""
+ probe_text = """
+DATA_DECL
+
+QUALIFIER int PROBENAME(struct pt_regs *ctx SIGNATURE)
+{
+ PID_FILTER
+ PREFIX
+ if (!(FILTER)) return 0;
+ KEY_EXPR
+ COLLECT
+ return 0;
+}
+"""
+ prefix = ""
+ qualifier = ""
+ signature = ""
# If any entry arguments are probed in a ret probe, we need
# to generate an entry probe to collect them
- prefix = ""
if self.entry_probe_required:
program += self._generate_entry_probe()
prefix += self._generate_retprobe_prefix()
@@ -329,18 +354,19 @@
# value we collected when entering the function:
self._replace_entry_exprs()
- # If this is a tracepoint probe, generate a local variable
- # that enables access to the tracepoint structure and also
- # the structure definition itself
if self.probe_type == "t":
program += self.tp.generate_struct()
prefix += self.tp.generate_get_struct()
+ elif self.probe_type == "u":
+ qualifier = "static inline"
+ signature = ", int __loc_id"
+ prefix += self.usdt.generate_usdt_cases()
+ elif self.probe_type == "p" and len(self.signature) > 0:
+ # Only entry uprobes/kprobes can have user-specified
+ # signatures. Other probes force it to ().
+ signature = ", " + self.signature
- program += self.probe_text.replace("PROBENAME",
- self.probe_func_name)
- signature = "" if len(self.signature) == 0 \
- or self.probe_type == "r" \
- else ", " + self.signature
+ program += probe_text.replace("PROBENAME", self.probe_func_name)
program = program.replace("SIGNATURE", signature)
program = program.replace("PID_FILTER",
self._generate_pid_filter())
@@ -354,34 +380,56 @@
"1" if len(self.filter) == 0 else self.filter)
program = program.replace("COLLECT", collect)
program = program.replace("PREFIX", prefix)
+ program = program.replace("QUALIFIER", qualifier)
+
+ if self.probe_type == "u":
+ self.usdt_thunk_names = []
+ program += self.usdt.generate_usdt_thunks(
+ self.probe_func_name, self.usdt_thunk_names)
+
return program
+ def _attach_u(self):
+ libpath = BPF.find_library(self.library)
+ if libpath is None:
+ with os.popen(("which --skip-alias %s " +
+ "2>/dev/null") % self.library) as w:
+ libpath = w.read().strip()
+ if libpath is None or len(libpath) == 0:
+ self._bail("unable to find library %s" %
+ self.library)
+
+ if self.probe_type == "u":
+ for i, location in enumerate(self.usdt.locations):
+ self.bpf.attach_uprobe(name=libpath,
+ addr=location.address,
+ fn_name=self.usdt_thunk_names[i],
+ pid=self.pid or -1)
+ elif self.probe_type == "r":
+ self.bpf.attach_uretprobe(name=libpath,
+ sym=self.function,
+ fn_name=self.probe_func_name,
+ pid=self.pid or -1)
+ else:
+ self.bpf.attach_uprobe(name=libpath,
+ sym=self.function,
+ fn_name=self.probe_func_name,
+ pid=self.pid or -1)
+
+ def _attach_k(self):
+ if self.probe_type == "r" or self.probe_type == "t":
+ self.bpf.attach_kretprobe(event=self.function,
+ fn_name=self.probe_func_name)
+ else:
+ self.bpf.attach_kprobe(event=self.function,
+ fn_name=self.probe_func_name)
+
def attach(self, bpf):
self.bpf = bpf
- uprobes_start = len(BPF.open_uprobes())
- kprobes_start = len(BPF.open_kprobes())
if self.is_user:
- if self.probe_type == "r":
- bpf.attach_uretprobe(name=self.library,
- sym=self.function,
- fn_name=self.probe_func_name,
- pid=self.pid or -1)
- else:
- bpf.attach_uprobe(name=self.library,
- sym=self.function,
- fn_name=self.probe_func_name,
- pid=self.pid or -1)
- if len(BPF.open_uprobes()) != uprobes_start + 1:
- self._bail("error attaching probe")
+ self._attach_u()
else:
- if self.probe_type == "r" or self.probe_type == "t":
- bpf.attach_kretprobe(event=self.function,
- fn_name=self.probe_func_name)
- else:
- bpf.attach_kprobe(event=self.function,
- fn_name=self.probe_func_name)
- if len(BPF.open_kprobes()) != kprobes_start + 1:
- self._bail("error attaching probe")
+ self._attach_k()
if self.entry_probe_required:
self._attach_entry_probe()
@@ -397,7 +445,7 @@
expr = self.exprs[i].replace(
"(bpf_ktime_get_ns() - *____latency_val)", "$latency")
# Replace alias values back with the alias name
- for alias, subst in Specifier.aliases.items():
+ for alias, subst in Probe.aliases.items():
expr = expr.replace(subst, alias)
# Replace retval expression with $retval
expr = expr.replace("ctx->ax", "$retval")
@@ -445,12 +493,16 @@
if not self.is_default_expr else "retval")
data.print_log2_hist(val_type=label)
+ def __str__(self):
+ return self.label or self.raw_spec
+
class Tool(object):
examples = """
Probe specifier syntax:
- {p,r,t}:{[library],category}:function(signature)[:type[,type...]:expr[,expr...][:filter]][#label]
+ {p,r,t,u}:{[library],category}:function(signature)[:type[,type...]:expr[,expr...][:filter]][#label]
Where:
- p,r,t -- probe at function entry, function exit, or kernel tracepoint
+ p,r,t,u -- probe at function entry, function exit, kernel tracepoint,
+ or USDT probe
in exit probes: can use $retval, $entry(param), $latency
library -- the library that contains the function
(leave empty for kernel functions)
@@ -509,6 +561,10 @@
argdist -C 't:irq:irq_handler_entry():int:tp.irq'
Aggregate interrupts by interrupt request (IRQ)
+argdist -C 'u:pthread:pthread_start():u64:arg2' -p 1337
+ Print frequency of function addresses used as a pthread start function,
+ relying on the USDT pthread_start probe in process 1337
+
argdist -H \\
'p:c:sleep(u32 seconds):u32:seconds' \\
'p:c:nanosleep(struct timespec *req):long:req->tv_nsec'
@@ -552,15 +608,15 @@
help="additional header files to include in the BPF program")
self.args = parser.parse_args()
- def _create_specifiers(self):
- self.specifiers = []
+ def _create_probes(self):
+ self.probes = []
for specifier in (self.args.countspecifier or []):
- self.specifiers.append(Specifier(
+ self.probes.append(Probe(
"freq", specifier, self.args.pid))
for histspecifier in (self.args.histspecifier or []):
- self.specifiers.append(
- Specifier("hist", histspecifier, self.args.pid))
- if len(self.specifiers) == 0:
+ self.probes.append(
+ Probe("hist", histspecifier, self.args.pid))
+ if len(self.probes) == 0:
print("at least one specifier is required")
exit()
@@ -573,19 +629,19 @@
for include in (self.args.include or []):
bpf_source += "#include <%s>\n" % include
bpf_source += BPF.generate_auto_includes(
- map(lambda s: s.raw_spec, self.specifiers))
+ map(lambda p: p.raw_spec, self.probes))
bpf_source += Tracepoint.generate_decl()
bpf_source += Tracepoint.generate_entry_probe()
- for specifier in self.specifiers:
- bpf_source += specifier.generate_text()
+ for probe in self.probes:
+ bpf_source += probe.generate_text()
if self.args.verbose:
print(bpf_source)
self.bpf = BPF(text=bpf_source)
def _attach(self):
Tracepoint.attach(self.bpf)
- for specifier in self.specifiers:
- specifier.attach(self.bpf)
+ for probe in self.probes:
+ probe.attach(self.bpf)
if self.args.verbose:
print("open uprobes: %s" % BPF.open_uprobes())
print("open kprobes: %s" % BPF.open_kprobes())
@@ -598,16 +654,22 @@
except KeyboardInterrupt:
exit()
print("[%s]" % strftime("%H:%M:%S"))
- for specifier in self.specifiers:
- specifier.display(self.args.top)
+ for probe in self.probes:
+ probe.display(self.args.top)
count_so_far += 1
if self.args.count is not None and \
count_so_far >= self.args.count:
exit()
+ def _close_probes(self):
+ for probe in self.probes:
+ probe.close()
+ if self.args.verbose:
+ print("closed probe: " + str(probe))
+
def run(self):
try:
- self._create_specifiers()
+ self._create_probes()
self._generate_program()
self._attach()
self._main_loop()
@@ -616,6 +678,7 @@
traceback.print_exc()
elif sys.exc_type is not SystemExit:
print(sys.exc_value)
+ self._close_probes()
if __name__ == "__main__":
Tool().run()
diff --git a/tools/argdist_example.txt b/tools/argdist_example.txt
index acd4549..d851337 100644
--- a/tools/argdist_example.txt
+++ b/tools/argdist_example.txt
@@ -332,9 +332,10 @@
additional header files to include in the BPF program
Probe specifier syntax:
- {p,r,t}:{[library],category}:function(signature)[:type[,type...]:expr[,expr...][:filter]][#label]
+ {p,r,t,u}:{[library],category}:function(signature)[:type[,type...]:expr[,expr...][:filter]][#label]
Where:
- p,r,t -- probe at function entry, function exit, or kernel tracepoint
+ p,r,t,u -- probe at function entry, function exit, kernel tracepoint,
+ or USDT probe
in exit probes: can use $retval, $entry(param), $latency
library -- the library that contains the function
(leave empty for kernel functions)
@@ -392,6 +393,10 @@
argdist -C 't:irq:irq_handler_entry():int:tp.irq'
Aggregate interrupts by interrupt request (IRQ)
+argdist -C 'u:pthread:pthread_start():u64:arg2' -p 1337
+ Print frequency of function addresses used as a pthread start function,
+ relying on the USDT pthread_start probe in process 1337
+
argdist -H \
'p:c:sleep(u32 seconds):u32:seconds' \
'p:c:nanosleep(struct timespec *req):long:req->tv_nsec'
diff --git a/tools/tplist.py b/tools/tplist.py
index fead0a1..abb011d 100755
--- a/tools/tplist.py
+++ b/tools/tplist.py
@@ -1,27 +1,34 @@
#!/usr/bin/env python
#
-# tplist Display kernel tracepoints and their formats.
+# tplist Display kernel tracepoints or USDT probes and their formats.
#
-# USAGE: tplist [-v] [tracepoint]
+# USAGE: tplist [-p PID] [-l LIB] [-v] [filter]
#
# Licensed under the Apache License, Version 2.0 (the "License")
# Copyright (C) 2016 Sasha Goldshtein.
import argparse
import fnmatch
-import re
import os
+import re
+import sys
+
+from bcc import USDTReader
trace_root = "/sys/kernel/debug/tracing"
event_root = os.path.join(trace_root, "events")
parser = argparse.ArgumentParser(description=
- "Display kernel tracepoints and their formats.",
+ "Display kernel tracepoints or USDT probes and their formats.",
formatter_class=argparse.RawDescriptionHelpFormatter)
+parser.add_argument("-p", "--pid", type=int, default=-1, help=
+ "List USDT probes in the specified process")
+parser.add_argument("-l", "--lib", default="", help=
+ "List USDT probes in the specified library or executable")
parser.add_argument("-v", dest="variables", action="store_true", help=
- "Print the format (available variables) for each tracepoint")
-parser.add_argument(dest="tracepoint", nargs="?",
- help="The tracepoint name to print (wildcards allowed)")
+ "Print the format (available variables)")
+parser.add_argument(dest="filter", nargs="?", help=
+ "A filter that specifies which probes/tracepoints to print")
args = parser.parse_args()
def print_tpoint_format(category, event):
@@ -42,12 +49,12 @@
def print_tpoint(category, event):
tpoint = "%s:%s" % (category, event)
- if not args.tracepoint or fnmatch.fnmatch(tpoint, args.tracepoint):
+ if not args.filter or fnmatch.fnmatch(tpoint, args.filter):
print(tpoint)
if args.variables:
print_tpoint_format(category, event)
-def print_all():
+def print_tracepoints():
for category in os.listdir(event_root):
cat_dir = os.path.join(event_root, category)
if not os.path.isdir(cat_dir):
@@ -57,5 +64,28 @@
if os.path.isdir(evt_dir):
print_tpoint(category, event)
+def print_usdt(pid, lib):
+ reader = USDTReader(bin_path=lib, pid=pid)
+ probes_seen = []
+ for probe in reader.probes:
+ probe_name = "%s:%s" % (probe.provider, probe.name)
+ if not args.filter or fnmatch.fnmatch(probe_name, args.filter):
+ if probe_name in probes_seen:
+ continue
+ probes_seen.append(probe_name)
+ if args.variables:
+ print(probe.display_verbose())
+ else:
+ print("%s %s:%s" % (probe.bin_path,
+ probe.provider, probe.name))
+
if __name__ == "__main__":
- print_all()
+ try:
+ if args.pid != -1 or args.lib != "":
+ print_usdt(args.pid, args.lib)
+ else:
+ print_tracepoints()
+ except:
+ if sys.exc_type is not SystemExit:
+ print(sys.exc_value)
+
diff --git a/tools/tplist_example.txt b/tools/tplist_example.txt
new file mode 100644
index 0000000..dfa13e2
--- /dev/null
+++ b/tools/tplist_example.txt
@@ -0,0 +1,113 @@
+Demonstrations of tplist.
+
+
+tplist displays kernel tracepoints and USDT probes, including their
+format. It can be used to discover probe points for use with the trace
+and argdist tools. Kernel tracepoints are scattered around the kernel
+and provide valuable static tracing on block and network I/O, scheduling,
+power events, and many other subjects. USDT probes are placed in libraries
+(such as libc) and executables (such as node) and provide static tracing
+information that can (optionally) be turned on and off at runtime.
+
+For example, suppose you want to discover which USDT probes a particular
+executable contains. Just run tplist on that executable (or library):
+
+$ tplist -l basic_usdt
+/home/vagrant/basic_usdt basic_usdt:start_main
+/home/vagrant/basic_usdt basic_usdt:loop_iter
+/home/vagrant/basic_usdt basic_usdt:end_main
+
+The loop_iter probe sounds interesting. What are the locations of that
+probe, and which variables are available?
+
+$ tplist '*loop_iter' -l basic_usdt -v
+/home/vagrant/basic_usdt basic_usdt:loop_iter [sema 0x601036]
+ location 0x400550 raw args: -4@$42 8@%rax
+ 4 signed bytes @ constant 42
+ 8 unsigned bytes @ register %rax
+ location 0x40056f raw args: 8@-8(%rbp) 8@%rax
+ 8 unsigned bytes @ -8(%rbp)
+ 8 unsigned bytes @ register %rax
+
+This output indicates that the loop_iter probe is used in two locations
+in the basic_usdt executable. The first location passes a constant value,
+42, to the probe. The second location passes a variable value located at
+an offset from the %rbp register. Don't worry -- you don't have to trace
+the register values yourself. The argdist and trace tools understand the
+probe format and can print out the arguments automatically -- you can
+refer to them as arg1, arg2, and so on.
+
+Try to explore with some common libraries on your system and see if they
+contain UDST probes. Here are two examples you might find interesting:
+
+$ tplist -l pthread # list probes in libpthread
+/lib64/libpthread.so.0 libpthread:pthread_start
+/lib64/libpthread.so.0 libpthread:pthread_create
+/lib64/libpthread.so.0 libpthread:pthread_join
+/lib64/libpthread.so.0 libpthread:pthread_join_ret
+/lib64/libpthread.so.0 libpthread:mutex_init
+... more output truncated
+
+$ tplist -l c # list probes in libc
+/lib64/libc.so.6 libc:setjmp
+/lib64/libc.so.6 libc:longjmp
+/lib64/libc.so.6 libc:longjmp_target
+/lib64/libc.so.6 libc:memory_arena_reuse_free_list
+/lib64/libc.so.6 libc:memory_heap_new
+... more output truncated
+
+tplist also understands kernel tracepoints, and can list their format
+as well. For example, let's look for all block I/O-related tracepoints:
+
+# tplist 'block*'
+block:block_touch_buffer
+block:block_dirty_buffer
+block:block_rq_abort
+block:block_rq_requeue
+block:block_rq_complete
+block:block_rq_insert
+block:block_rq_issue
+block:block_bio_bounce
+block:block_bio_complete
+block:block_bio_backmerge
+block:block_bio_frontmerge
+block:block_bio_queue
+block:block_getrq
+block:block_sleeprq
+block:block_plug
+block:block_unplug
+block:block_split
+block:block_bio_remap
+block:block_rq_remap
+
+The block:block_rq_complete tracepoints sounds interesting. Let's print
+its format to see what we can trace with argdist and trace:
+
+$ tplist -v block:block_rq_complete
+block:block_rq_complete
+ dev_t dev;
+ sector_t sector;
+ unsigned int nr_sector;
+ int errors;
+ char rwbs[8];
+
+The dev, sector, nr_sector, etc. variables can now all be used in probes
+you specify with argdist or trace.
+
+
+USAGE message:
+
+$ tplist -h
+usage: tplist.py [-h] [-p PID] [-l LIB] [-v] [filter]
+
+Display kernel tracepoints or USDT probes and their formats.
+
+positional arguments:
+ filter A filter that specifies which probes/tracepoints to print
+
+optional arguments:
+ -h, --help show this help message and exit
+ -p PID, --pid PID List USDT probes in the specified process
+ -l LIB, --lib LIB List USDT probes in the specified library or executable
+ -v Print the format (available variables)
+
diff --git a/tools/trace.py b/tools/trace.py
index 4aac067..c5ec39c 100755
--- a/tools/trace.py
+++ b/tools/trace.py
@@ -9,7 +9,7 @@
# Licensed under the Apache License, Version 2.0 (the "License")
# Copyright (C) 2016 Sasha Goldshtein.
-from bcc import BPF, Tracepoint, Perf
+from bcc import BPF, Tracepoint, Perf, USDTReader
from time import sleep, strftime
import argparse
import re
@@ -49,12 +49,14 @@
event_count = 0
first_ts = 0
use_localtime = True
+ pid = -1
@classmethod
def configure(cls, args):
cls.max_events = args.max_events
cls.use_localtime = not args.offset
cls.first_ts = Time.monotonic_time()
+ cls.pid = args.pid or -1
def __init__(self, probe, string_size):
self.raw_probe = probe
@@ -63,18 +65,18 @@
self._parse_probe()
self.probe_num = Probe.probe_count
self.probe_name = "probe_%s_%d" % \
- (self.function, self.probe_num)
+ (self._display_function(), self.probe_num)
def __str__(self):
- return "%s:%s`%s FLT=%s ACT=%s/%s" % (self.probe_type,
- self.library, self.function, self.filter,
+ return "%s:%s:%s FLT=%s ACT=%s/%s" % (self.probe_type,
+ self.library, self._display_function(), self.filter,
self.types, self.values)
def is_default_action(self):
return self.python_format == ""
def _bail(self, error):
- raise ValueError("error parsing probe '%s': %s" %
+ raise ValueError("error in probe '%s': %s" %
(self.raw_probe, error))
def _parse_probe(self):
@@ -124,11 +126,11 @@
parts = ["p", parts[0], parts[1]]
if len(parts[0]) == 0:
self.probe_type = "p"
- elif parts[0] in ["p", "r", "t"]:
+ elif parts[0] in ["p", "r", "t", "u"]:
self.probe_type = parts[0]
else:
- self._bail("expected '', 'p', 't', or 'r', got '%s'" %
- parts[0])
+ self._bail("probe type must be '', 'p', 't', 'r', " +
+ "or 'u', but got '%s'" % parts[0])
if self.probe_type == "t":
self.tp_category = parts[1]
self.tp_event = parts[2]
@@ -136,10 +138,39 @@
self.tp_category, self.tp_event)
self.library = "" # kernel
self.function = "perf_trace_%s" % self.tp_event
+ elif self.probe_type == "u":
+ self.library = parts[1]
+ self.usdt_name = parts[2]
+ self.function = "" # no function, just address
+ # We will discover the USDT provider by matching on
+ # the USDT name in the specified library
+ self._find_usdt_probe()
+ self._enable_usdt_probe()
else:
self.library = parts[1]
self.function = parts[2]
+ def _enable_usdt_probe(self):
+ if self.usdt.need_enable():
+ if Probe.pid == -1:
+ self._bail("probe needs pid to enable")
+ self.usdt.enable(Probe.pid)
+
+ def _disable_usdt_probe(self):
+ if self.probe_type == "u" and self.usdt.need_enable():
+ self.usdt.disable(Probe.pid)
+
+ def close(self):
+ self._disable_usdt_probe()
+
+ def _find_usdt_probe(self):
+ reader = USDTReader(bin_path=self.library)
+ for probe in reader.probes:
+ if probe.name == self.usdt_name:
+ self.usdt = probe
+ return
+ self._bail("unrecognized USDT probe %s" % self.usdt_name)
+
def _parse_filter(self, filt):
self.filter = self._replace_args(filt)
@@ -187,6 +218,10 @@
def _replace_args(self, expr):
for alias, replacement in Probe.aliases.items():
+ # For USDT probes, we replace argN values with the
+ # actual arguments for that probe.
+ if alias.startswith("arg") and self.probe_type == "u":
+ continue
expr = expr.replace(alias, replacement)
return expr
@@ -206,7 +241,7 @@
def _generate_python_data_decl(self):
self.python_struct_name = "%s_%d_Data" % \
- (self.function, self.probe_num)
+ (self._display_function(), self.probe_num)
fields = [
("timestamp_ns", ct.c_ulonglong),
("pid", ct.c_uint),
@@ -266,21 +301,16 @@
bpf_probe_read(&__data.v%d, sizeof(__data.v%d), (void *)%s);
}
""" % (expr, idx, idx, expr)
- # return ("bpf_probe_read(&__data.v%d, " + \
- # "sizeof(__data.v%d), (char*)%s);\n") % (idx, idx, expr)
- # return ("__builtin_memcpy(&__data.v%d, (void *)%s, " + \
- # "sizeof(__data.v%d));\n") % (idx, expr, idx)
if field_type in Probe.fmt_types:
return " __data.v%d = (%s)%s;\n" % \
(idx, Probe.c_type[field_type], expr)
self._bail("unrecognized field type %s" % field_type)
- def generate_program(self, pid, include_self):
+ def generate_program(self, include_self):
data_decl = self._generate_data_decl()
- self.pid = pid
# kprobes don't have built-in pid filters, so we have to add
# it to the function body:
- if len(self.library) == 0 and pid != -1:
+ if len(self.library) == 0 and Probe.pid != -1:
pid_filter = """
u32 __pid = bpf_get_current_pid_tgid();
if (__pid != %d) { return 0; }
@@ -293,17 +323,23 @@
else:
pid_filter = ""
+ prefix = ""
+ qualifier = ""
+ signature = "struct pt_regs *ctx"
+ if self.probe_type == "t":
+ data_decl += self.tp.generate_struct()
+ prefix = self.tp.generate_get_struct()
+ elif self.probe_type == "u":
+ signature += ", int __loc_id"
+ prefix = self.usdt.generate_usdt_cases()
+ qualifier = "static inline"
+
data_fields = ""
for i, expr in enumerate(self.values):
data_fields += self._generate_field_assign(i)
- prefix = ""
- if self.probe_type == "t":
- data_decl += self.tp.generate_struct()
- prefix = self.tp.generate_get_struct()
-
text = """
-int %s(struct pt_regs *ctx)
+%s int %s(%s)
{
%s
%s
@@ -318,9 +354,14 @@
return 0;
}
"""
- text = text % (self.probe_name, pid_filter, prefix,
- self.filter, self.struct_name,
- data_fields, self.events_name)
+ text = text % (qualifier, self.probe_name, signature,
+ pid_filter, prefix, self.filter,
+ self.struct_name, data_fields, self.events_name)
+
+ if self.probe_type == "u":
+ self.usdt_thunk_names = []
+ text += self.usdt.generate_usdt_thunks(
+ self.probe_name, self.usdt_thunk_names)
return data_decl + "\n" + text
@@ -329,10 +370,12 @@
return "%.6f" % (1e-9 * (timestamp_ns - cls.first_ts))
def _display_function(self):
- if self.probe_type != 't':
+ if self.probe_type == 'p' or self.probe_type == 'r':
return self.function
- else:
- return self.function.replace("perf_trace_", "")
+ elif self.probe_type == 'u':
+ return self.usdt_name
+ else: # self.probe_type == 't'
+ return self.tp_event
def print_event(self, cpu, data, size):
# Cast as the generated structure type and display
@@ -361,39 +404,40 @@
bpf[self.events_name].open_perf_buffer(self.print_event)
def _attach_k(self, bpf):
- kprobes_start = len(BPF.open_kprobes())
if self.probe_type == "r":
bpf.attach_kretprobe(event=self.function,
fn_name=self.probe_name)
elif self.probe_type == "p" or self.probe_type == "t":
bpf.attach_kprobe(event=self.function,
fn_name=self.probe_name)
- if len(BPF.open_kprobes()) != kprobes_start + 1:
- self._bail("error attaching probe")
def _attach_u(self, bpf):
libpath = BPF.find_library(self.library)
if libpath is None:
# This might be an executable (e.g. 'bash')
- with os.popen("/usr/bin/which %s 2>/dev/null" %
- self.library) as w:
+ with os.popen(
+ "/usr/bin/which --skip-alias %s 2>/dev/null" %
+ self.library) as w:
libpath = w.read().strip()
if libpath is None or len(libpath) == 0:
self._bail("unable to find library %s" % self.library)
- uprobes_start = len(BPF.open_uprobes())
- if self.probe_type == "r":
+ if self.probe_type == "u":
+ for i, location in enumerate(self.usdt.locations):
+ bpf.attach_uprobe(name=libpath,
+ addr=location.address,
+ fn_name=self.usdt_thunk_names[i],
+ pid=Probe.pid)
+ elif self.probe_type == "r":
bpf.attach_uretprobe(name=libpath,
sym=self.function,
fn_name=self.probe_name,
- pid=self.pid)
+ pid=Probe.pid)
else:
bpf.attach_uprobe(name=libpath,
sym=self.function,
fn_name=self.probe_name,
- pid=self.pid)
- if len(BPF.open_uprobes()) != uprobes_start + 1:
- self._bail("error attaching probe")
+ pid=Probe.pid)
class Tool(object):
examples = """
@@ -419,6 +463,8 @@
Trace returns from malloc and print non-NULL allocated buffers
trace 't:block:block_rq_complete "sectors=%d", tp.nr_sector'
Trace the block_rq_complete kernel tracepoint and print # of tx sectors
+trace 'u:pthread:pthread_create (arg4 != 0)'
+ Trace the USDT probe pthread_create when its 4th argument is non-zero
"""
def __init__(self):
@@ -461,7 +507,7 @@
self.program += Tracepoint.generate_entry_probe()
for probe in self.probes:
self.program += probe.generate_program(
- self.args.pid or -1, self.args.include_self)
+ self.args.include_self)
if self.args.verbose:
print(self.program)
@@ -486,6 +532,12 @@
while True:
self.bpf.kprobe_poll()
+ def _close_probes(self):
+ for probe in self.probes:
+ probe.close()
+ if self.args.verbose:
+ print("closed probe: " + str(probe))
+
def run(self):
try:
self._create_probes()
@@ -497,6 +549,7 @@
traceback.print_exc()
elif sys.exc_type is not SystemExit:
print(sys.exc_value)
+ self._close_probes()
if __name__ == "__main__":
Tool().run()
diff --git a/tools/trace_example.txt b/tools/trace_example.txt
index 98831ab..dce72b9 100644
--- a/tools/trace_example.txt
+++ b/tools/trace_example.txt
@@ -171,4 +171,6 @@
Trace returns from malloc and print non-NULL allocated buffers
trace 't:block:block_rq_complete "sectors=%d", tp.nr_sector'
Trace the block_rq_complete kernel tracepoint and print # of tx sectors
+trace 'u:pthread:pthread_create (arg4 != 0)'
+ Trace the USDT probe pthread_create when its 4th argument is non-zero