Sasha Goldshtein | 4f1ea67 | 2016-02-07 01:57:42 -0800 | [diff] [blame] | 1 | #!/usr/bin/env python |
Sasha Goldshtein | 5045964 | 2016-02-10 08:35:20 -0800 | [diff] [blame] | 2 | # |
Sasha Goldshtein | 29e37d9 | 2016-02-14 06:56:07 -0800 | [diff] [blame^] | 3 | # memleak Trace and display outstanding allocations to detect |
Sasha Goldshtein | 5045964 | 2016-02-10 08:35:20 -0800 | [diff] [blame] | 4 | # memory leaks in user-mode processes and the kernel. |
| 5 | # |
Sasha Goldshtein | 29e37d9 | 2016-02-14 06:56:07 -0800 | [diff] [blame^] | 6 | # USAGE: memleak [-h] [-p PID] [-t] [-a] [-o OLDER] [-c COMMAND] |
Sasha Goldshtein | 5045964 | 2016-02-10 08:35:20 -0800 | [diff] [blame] | 7 | # [-s SAMPLE_RATE] [-d STACK_DEPTH] [-T TOP] [-z MIN_SIZE] |
| 8 | # [-Z MAX_SIZE] |
| 9 | # [interval] [count] |
| 10 | # |
Sasha Goldshtein | 43fa041 | 2016-02-10 22:17:26 -0800 | [diff] [blame] | 11 | # Licensed under the Apache License, Version 2.0 (the "License") |
Sasha Goldshtein | 5045964 | 2016-02-10 08:35:20 -0800 | [diff] [blame] | 12 | # Copyright (C) 2016 Sasha Goldshtein. |
Sasha Goldshtein | 4f1ea67 | 2016-02-07 01:57:42 -0800 | [diff] [blame] | 13 | |
| 14 | from bcc import BPF |
| 15 | from time import sleep |
Sasha Goldshtein | c8148c8 | 2016-02-09 11:15:41 -0800 | [diff] [blame] | 16 | from datetime import datetime |
Sasha Goldshtein | 4f1ea67 | 2016-02-07 01:57:42 -0800 | [diff] [blame] | 17 | import argparse |
| 18 | import subprocess |
Sasha Goldshtein | cfce311 | 2016-02-07 11:09:36 -0800 | [diff] [blame] | 19 | import ctypes |
| 20 | import os |
Sasha Goldshtein | 4f1ea67 | 2016-02-07 01:57:42 -0800 | [diff] [blame] | 21 | |
Sasha Goldshtein | a7cc6c2 | 2016-02-07 12:03:54 -0800 | [diff] [blame] | 22 | class Time(object): |
Sasha Goldshtein | 33522d7 | 2016-02-08 03:39:44 -0800 | [diff] [blame] | 23 | # BPF timestamps come from the monotonic clock. To be able to filter |
| 24 | # and compare them from Python, we need to invoke clock_gettime. |
| 25 | # Adapted from http://stackoverflow.com/a/1205762 |
| 26 | CLOCK_MONOTONIC_RAW = 4 # see <linux/time.h> |
Sasha Goldshtein | a7cc6c2 | 2016-02-07 12:03:54 -0800 | [diff] [blame] | 27 | |
Sasha Goldshtein | 33522d7 | 2016-02-08 03:39:44 -0800 | [diff] [blame] | 28 | class timespec(ctypes.Structure): |
| 29 | _fields_ = [ |
| 30 | ('tv_sec', ctypes.c_long), |
| 31 | ('tv_nsec', ctypes.c_long) |
| 32 | ] |
Sasha Goldshtein | a7cc6c2 | 2016-02-07 12:03:54 -0800 | [diff] [blame] | 33 | |
Sasha Goldshtein | 33522d7 | 2016-02-08 03:39:44 -0800 | [diff] [blame] | 34 | librt = ctypes.CDLL('librt.so.1', use_errno=True) |
| 35 | clock_gettime = librt.clock_gettime |
| 36 | clock_gettime.argtypes = [ctypes.c_int, ctypes.POINTER(timespec)] |
Sasha Goldshtein | a7cc6c2 | 2016-02-07 12:03:54 -0800 | [diff] [blame] | 37 | |
Sasha Goldshtein | 33522d7 | 2016-02-08 03:39:44 -0800 | [diff] [blame] | 38 | @staticmethod |
| 39 | def monotonic_time(): |
| 40 | t = Time.timespec() |
| 41 | if Time.clock_gettime( |
| 42 | Time.CLOCK_MONOTONIC_RAW, ctypes.pointer(t)) != 0: |
| 43 | errno_ = ctypes.get_errno() |
| 44 | raise OSError(errno_, os.strerror(errno_)) |
| 45 | return t.tv_sec * 1e9 + t.tv_nsec |
Sasha Goldshtein | a7cc6c2 | 2016-02-07 12:03:54 -0800 | [diff] [blame] | 46 | |
Sasha Goldshtein | 2922861 | 2016-02-07 12:20:19 -0800 | [diff] [blame] | 47 | class StackDecoder(object): |
Sasha Goldshtein | 33522d7 | 2016-02-08 03:39:44 -0800 | [diff] [blame] | 48 | def __init__(self, pid, bpf): |
| 49 | self.pid = pid |
| 50 | self.bpf = bpf |
| 51 | self.ranges_cache = {} |
| 52 | self.refresh_code_ranges() |
Sasha Goldshtein | 2922861 | 2016-02-07 12:20:19 -0800 | [diff] [blame] | 53 | |
Sasha Goldshtein | 33522d7 | 2016-02-08 03:39:44 -0800 | [diff] [blame] | 54 | def refresh_code_ranges(self): |
| 55 | if self.pid == -1: |
| 56 | return |
| 57 | self.code_ranges = self._get_code_ranges() |
Sasha Goldshtein | 2922861 | 2016-02-07 12:20:19 -0800 | [diff] [blame] | 58 | |
Sasha Goldshtein | 33522d7 | 2016-02-08 03:39:44 -0800 | [diff] [blame] | 59 | @staticmethod |
| 60 | def _is_binary_segment(parts): |
| 61 | return len(parts) == 6 and \ |
Sasha Goldshtein | 521ab4f | 2016-02-08 05:48:31 -0800 | [diff] [blame] | 62 | parts[5][0] != '[' and 'x' in parts[1] |
Sasha Goldshtein | 2922861 | 2016-02-07 12:20:19 -0800 | [diff] [blame] | 63 | |
Sasha Goldshtein | 33522d7 | 2016-02-08 03:39:44 -0800 | [diff] [blame] | 64 | def _get_code_ranges(self): |
| 65 | ranges = {} |
| 66 | raw_ranges = open("/proc/%d/maps" % self.pid).readlines() |
| 67 | # A typical line from /proc/PID/maps looks like this: |
| 68 | # 7f21b6635000-7f21b67eb000 r-xp ... /usr/lib64/libc-2.21.so |
| 69 | # We are looking for executable segments that have a .so file |
| 70 | # or the main executable. The first two lines are the range of |
| 71 | # that memory segment, which we index by binary name. |
| 72 | for raw_range in raw_ranges: |
| 73 | parts = raw_range.split() |
| 74 | if not StackDecoder._is_binary_segment(parts): |
| 75 | continue |
| 76 | binary = parts[5] |
| 77 | range_parts = parts[0].split('-') |
| 78 | addr_range = (int(range_parts[0], 16), |
| 79 | int(range_parts[1], 16)) |
| 80 | ranges[binary] = addr_range |
| 81 | return ranges |
Sasha Goldshtein | 2922861 | 2016-02-07 12:20:19 -0800 | [diff] [blame] | 82 | |
Sasha Goldshtein | 33522d7 | 2016-02-08 03:39:44 -0800 | [diff] [blame] | 83 | @staticmethod |
| 84 | def _is_function_symbol(parts): |
| 85 | return len(parts) == 6 and parts[3] == ".text" \ |
| 86 | and parts[2] == "F" |
Sasha Goldshtein | 2922861 | 2016-02-07 12:20:19 -0800 | [diff] [blame] | 87 | |
Sasha Goldshtein | 33522d7 | 2016-02-08 03:39:44 -0800 | [diff] [blame] | 88 | def _get_sym_ranges(self, binary): |
| 89 | if binary in self.ranges_cache: |
| 90 | return self.ranges_cache[binary] |
| 91 | sym_ranges = {} |
| 92 | raw_symbols = run_command_get_output("objdump -t %s" % binary) |
| 93 | for raw_symbol in raw_symbols: |
| 94 | # A typical line from objdump -t looks like this: |
| 95 | # 00000000004007f5 g F .text 000000000000010e main |
| 96 | # We only care about functions in the .text segment. |
| 97 | # The first number is the start address, and the second |
| 98 | # number is the length. |
| 99 | parts = raw_symbol.split() |
| 100 | if not StackDecoder._is_function_symbol(parts): |
| 101 | continue |
| 102 | sym_start = int(parts[0], 16) |
| 103 | sym_len = int(parts[4], 16) |
| 104 | sym_name = parts[5] |
| 105 | sym_ranges[sym_name] = (sym_start, sym_len) |
| 106 | self.ranges_cache[binary] = sym_ranges |
| 107 | return sym_ranges |
Sasha Goldshtein | 2922861 | 2016-02-07 12:20:19 -0800 | [diff] [blame] | 108 | |
Sasha Goldshtein | 33522d7 | 2016-02-08 03:39:44 -0800 | [diff] [blame] | 109 | def _decode_sym(self, binary, offset): |
| 110 | sym_ranges = self._get_sym_ranges(binary) |
| 111 | # Find the symbol that contains the specified offset. |
| 112 | # There might not be one. |
| 113 | for name, (start, length) in sym_ranges.items(): |
| 114 | if offset >= start and offset <= (start + length): |
| 115 | return "%s+0x%x" % (name, offset - start) |
| 116 | return "%x" % offset |
| 117 | |
| 118 | def _decode_addr(self, addr): |
| 119 | code_ranges = self._get_code_ranges() |
| 120 | # Find the binary that contains the specified address. |
| 121 | # For .so files, look at the relative address; for the main |
| 122 | # executable, look at the absolute address. |
| 123 | for binary, (start, end) in code_ranges.items(): |
| 124 | if addr >= start and addr <= end: |
| 125 | offset = addr - start \ |
| 126 | if binary.endswith(".so") else addr |
| 127 | return "%s [%s]" % (self._decode_sym(binary, |
| 128 | offset), binary) |
| 129 | return "%x" % addr |
| 130 | |
| 131 | def decode_stack(self, info, is_kernel_trace): |
| 132 | stack = "" |
| 133 | if info.num_frames <= 0: |
| 134 | return "???" |
| 135 | for i in range(0, info.num_frames): |
| 136 | addr = info.callstack[i] |
| 137 | if is_kernel_trace: |
| 138 | stack += " %s [kernel] (%x) ;" % \ |
| 139 | (self.bpf.ksym(addr), addr) |
| 140 | else: |
| 141 | # At some point, we hope to have native BPF |
| 142 | # user-mode symbol decoding, but for now we |
| 143 | # have to use our own. |
| 144 | stack += " %s (%x) ;" % \ |
| 145 | (self._decode_addr(addr), addr) |
| 146 | return stack |
Sasha Goldshtein | 2922861 | 2016-02-07 12:20:19 -0800 | [diff] [blame] | 147 | |
Sasha Goldshtein | 751fce5 | 2016-02-08 02:57:02 -0800 | [diff] [blame] | 148 | def run_command_get_output(command): |
Sasha Goldshtein | 33522d7 | 2016-02-08 03:39:44 -0800 | [diff] [blame] | 149 | p = subprocess.Popen(command.split(), |
| 150 | stdout=subprocess.PIPE, stderr=subprocess.STDOUT) |
| 151 | return iter(p.stdout.readline, b'') |
Sasha Goldshtein | 2922861 | 2016-02-07 12:20:19 -0800 | [diff] [blame] | 152 | |
Sasha Goldshtein | 751fce5 | 2016-02-08 02:57:02 -0800 | [diff] [blame] | 153 | def run_command_get_pid(command): |
Sasha Goldshtein | 33522d7 | 2016-02-08 03:39:44 -0800 | [diff] [blame] | 154 | p = subprocess.Popen(command.split()) |
| 155 | return p.pid |
Sasha Goldshtein | 751fce5 | 2016-02-08 02:57:02 -0800 | [diff] [blame] | 156 | |
Sasha Goldshtein | 4f1ea67 | 2016-02-07 01:57:42 -0800 | [diff] [blame] | 157 | examples = """ |
| 158 | EXAMPLES: |
| 159 | |
Sasha Goldshtein | 29e37d9 | 2016-02-14 06:56:07 -0800 | [diff] [blame^] | 160 | ./memleak -p $(pidof allocs) |
Sasha Goldshtein | 33522d7 | 2016-02-08 03:39:44 -0800 | [diff] [blame] | 161 | Trace allocations and display a summary of "leaked" (outstanding) |
| 162 | allocations every 5 seconds |
Sasha Goldshtein | 29e37d9 | 2016-02-14 06:56:07 -0800 | [diff] [blame^] | 163 | ./memleak -p $(pidof allocs) -t |
Sasha Goldshtein | 33522d7 | 2016-02-08 03:39:44 -0800 | [diff] [blame] | 164 | Trace allocations and display each individual call to malloc/free |
Sasha Goldshtein | 29e37d9 | 2016-02-14 06:56:07 -0800 | [diff] [blame^] | 165 | ./memleak -ap $(pidof allocs) 10 |
Sasha Goldshtein | 33522d7 | 2016-02-08 03:39:44 -0800 | [diff] [blame] | 166 | Trace allocations and display allocated addresses, sizes, and stacks |
| 167 | every 10 seconds for outstanding allocations |
Sasha Goldshtein | 29e37d9 | 2016-02-14 06:56:07 -0800 | [diff] [blame^] | 168 | ./memleak -c "./allocs" |
Sasha Goldshtein | 33522d7 | 2016-02-08 03:39:44 -0800 | [diff] [blame] | 169 | Run the specified command and trace its allocations |
Sasha Goldshtein | 29e37d9 | 2016-02-14 06:56:07 -0800 | [diff] [blame^] | 170 | ./memleak |
Sasha Goldshtein | 33522d7 | 2016-02-08 03:39:44 -0800 | [diff] [blame] | 171 | Trace allocations in kernel mode and display a summary of outstanding |
| 172 | allocations every 5 seconds |
Sasha Goldshtein | 29e37d9 | 2016-02-14 06:56:07 -0800 | [diff] [blame^] | 173 | ./memleak -o 60000 |
Sasha Goldshtein | 33522d7 | 2016-02-08 03:39:44 -0800 | [diff] [blame] | 174 | Trace allocations in kernel mode and display a summary of outstanding |
| 175 | allocations that are at least one minute (60 seconds) old |
Sasha Goldshtein | 29e37d9 | 2016-02-14 06:56:07 -0800 | [diff] [blame^] | 176 | ./memleak -s 5 |
Sasha Goldshtein | 521ab4f | 2016-02-08 05:48:31 -0800 | [diff] [blame] | 177 | Trace roughly every 5th allocation, to reduce overhead |
Sasha Goldshtein | 4f1ea67 | 2016-02-07 01:57:42 -0800 | [diff] [blame] | 178 | """ |
| 179 | |
| 180 | description = """ |
| 181 | Trace outstanding memory allocations that weren't freed. |
| 182 | Supports both user-mode allocations made with malloc/free and kernel-mode |
| 183 | allocations made with kmalloc/kfree. |
| 184 | """ |
| 185 | |
Sasha Goldshtein | a7cc6c2 | 2016-02-07 12:03:54 -0800 | [diff] [blame] | 186 | parser = argparse.ArgumentParser(description=description, |
Sasha Goldshtein | 33522d7 | 2016-02-08 03:39:44 -0800 | [diff] [blame] | 187 | formatter_class=argparse.RawDescriptionHelpFormatter, |
| 188 | epilog=examples) |
Sasha Goldshtein | d2241f4 | 2016-02-09 06:23:10 -0800 | [diff] [blame] | 189 | parser.add_argument("-p", "--pid", type=int, default=-1, |
Sasha Goldshtein | 33522d7 | 2016-02-08 03:39:44 -0800 | [diff] [blame] | 190 | help="the PID to trace; if not specified, trace kernel allocs") |
Sasha Goldshtein | a7cc6c2 | 2016-02-07 12:03:54 -0800 | [diff] [blame] | 191 | parser.add_argument("-t", "--trace", action="store_true", |
Sasha Goldshtein | 33522d7 | 2016-02-08 03:39:44 -0800 | [diff] [blame] | 192 | help="print trace messages for each alloc/free call") |
Sasha Goldshtein | 40e55ba | 2016-02-09 05:53:48 -0800 | [diff] [blame] | 193 | parser.add_argument("interval", nargs="?", default=5, type=int, |
Sasha Goldshtein | 33522d7 | 2016-02-08 03:39:44 -0800 | [diff] [blame] | 194 | help="interval in seconds to print outstanding allocations") |
Sasha Goldshtein | 40e55ba | 2016-02-09 05:53:48 -0800 | [diff] [blame] | 195 | parser.add_argument("count", nargs="?", type=int, |
| 196 | help="number of times to print the report before exiting") |
Sasha Goldshtein | a7cc6c2 | 2016-02-07 12:03:54 -0800 | [diff] [blame] | 197 | parser.add_argument("-a", "--show-allocs", default=False, action="store_true", |
Sasha Goldshtein | 33522d7 | 2016-02-08 03:39:44 -0800 | [diff] [blame] | 198 | help="show allocation addresses and sizes as well as call stacks") |
Sasha Goldshtein | 40e55ba | 2016-02-09 05:53:48 -0800 | [diff] [blame] | 199 | parser.add_argument("-o", "--older", default=500, type=int, |
Sasha Goldshtein | 33522d7 | 2016-02-08 03:39:44 -0800 | [diff] [blame] | 200 | help="prune allocations younger than this age in milliseconds") |
Sasha Goldshtein | 2922861 | 2016-02-07 12:20:19 -0800 | [diff] [blame] | 201 | parser.add_argument("-c", "--command", |
Sasha Goldshtein | 33522d7 | 2016-02-08 03:39:44 -0800 | [diff] [blame] | 202 | help="execute and trace the specified command") |
Sasha Goldshtein | 40e55ba | 2016-02-09 05:53:48 -0800 | [diff] [blame] | 203 | parser.add_argument("-s", "--sample-rate", default=1, type=int, |
Sasha Goldshtein | 521ab4f | 2016-02-08 05:48:31 -0800 | [diff] [blame] | 204 | help="sample every N-th allocation to decrease the overhead") |
Sasha Goldshtein | dcee30d | 2016-02-09 06:24:33 -0800 | [diff] [blame] | 205 | parser.add_argument("-d", "--stack-depth", default=10, type=int, |
Sasha Goldshtein | d2241f4 | 2016-02-09 06:23:10 -0800 | [diff] [blame] | 206 | help="maximum stack depth to capture") |
Sasha Goldshtein | c8148c8 | 2016-02-09 11:15:41 -0800 | [diff] [blame] | 207 | parser.add_argument("-T", "--top", type=int, default=10, |
| 208 | help="display only this many top allocating stacks (by size)") |
Sasha Goldshtein | 5045964 | 2016-02-10 08:35:20 -0800 | [diff] [blame] | 209 | parser.add_argument("-z", "--min-size", type=int, |
| 210 | help="capture only allocations larger than this size") |
| 211 | parser.add_argument("-Z", "--max-size", type=int, |
| 212 | help="capture only allocations smaller than this size") |
Sasha Goldshtein | 4f1ea67 | 2016-02-07 01:57:42 -0800 | [diff] [blame] | 213 | |
| 214 | args = parser.parse_args() |
| 215 | |
Sasha Goldshtein | d2241f4 | 2016-02-09 06:23:10 -0800 | [diff] [blame] | 216 | pid = args.pid |
Sasha Goldshtein | 2922861 | 2016-02-07 12:20:19 -0800 | [diff] [blame] | 217 | command = args.command |
| 218 | kernel_trace = (pid == -1 and command is None) |
Sasha Goldshtein | 4f1ea67 | 2016-02-07 01:57:42 -0800 | [diff] [blame] | 219 | trace_all = args.trace |
Sasha Goldshtein | 40e55ba | 2016-02-09 05:53:48 -0800 | [diff] [blame] | 220 | interval = args.interval |
| 221 | min_age_ns = 1e6 * args.older |
Sasha Goldshtein | 521ab4f | 2016-02-08 05:48:31 -0800 | [diff] [blame] | 222 | sample_every_n = args.sample_rate |
Sasha Goldshtein | 40e55ba | 2016-02-09 05:53:48 -0800 | [diff] [blame] | 223 | num_prints = args.count |
Sasha Goldshtein | d2241f4 | 2016-02-09 06:23:10 -0800 | [diff] [blame] | 224 | max_stack_size = args.stack_depth + 2 |
Sasha Goldshtein | c8148c8 | 2016-02-09 11:15:41 -0800 | [diff] [blame] | 225 | top_stacks = args.top |
Sasha Goldshtein | 5045964 | 2016-02-10 08:35:20 -0800 | [diff] [blame] | 226 | min_size = args.min_size |
| 227 | max_size = args.max_size |
| 228 | |
| 229 | if min_size is not None and max_size is not None and min_size > max_size: |
| 230 | print("min_size (-z) can't be greater than max_size (-Z)") |
| 231 | exit(1) |
Sasha Goldshtein | 4f1ea67 | 2016-02-07 01:57:42 -0800 | [diff] [blame] | 232 | |
Sasha Goldshtein | 33522d7 | 2016-02-08 03:39:44 -0800 | [diff] [blame] | 233 | if command is not None: |
| 234 | print("Executing '%s' and tracing the resulting process." % command) |
| 235 | pid = run_command_get_pid(command) |
Sasha Goldshtein | 2922861 | 2016-02-07 12:20:19 -0800 | [diff] [blame] | 236 | |
Sasha Goldshtein | 43fa041 | 2016-02-10 22:17:26 -0800 | [diff] [blame] | 237 | bpf_source = """ |
| 238 | #include <uapi/linux/ptrace.h> |
| 239 | |
| 240 | struct alloc_info_t { |
| 241 | u64 size; |
| 242 | u64 timestamp_ns; |
| 243 | int num_frames; |
| 244 | u64 callstack[MAX_STACK_SIZE]; |
| 245 | }; |
| 246 | |
| 247 | BPF_HASH(sizes, u64); |
| 248 | BPF_HASH(allocs, u64, struct alloc_info_t); |
| 249 | |
| 250 | // Adapted from https://github.com/iovisor/bcc/tools/offcputime.py |
| 251 | static u64 get_frame(u64 *bp) { |
| 252 | if (*bp) { |
| 253 | // The following stack walker is x86_64 specific |
| 254 | u64 ret = 0; |
| 255 | if (bpf_probe_read(&ret, sizeof(ret), (void *)(*bp+8))) |
| 256 | return 0; |
| 257 | if (bpf_probe_read(bp, sizeof(*bp), (void *)*bp)) |
| 258 | *bp = 0; |
| 259 | return ret; |
| 260 | } |
| 261 | return 0; |
| 262 | } |
| 263 | static int grab_stack(struct pt_regs *ctx, struct alloc_info_t *info) |
| 264 | { |
| 265 | int depth = 0; |
| 266 | u64 bp = ctx->bp; |
| 267 | GRAB_ONE_FRAME |
| 268 | return depth; |
| 269 | } |
| 270 | |
| 271 | int alloc_enter(struct pt_regs *ctx, size_t size) |
| 272 | { |
| 273 | SIZE_FILTER |
| 274 | if (SAMPLE_EVERY_N > 1) { |
| 275 | u64 ts = bpf_ktime_get_ns(); |
| 276 | if (ts % SAMPLE_EVERY_N != 0) |
| 277 | return 0; |
| 278 | } |
| 279 | |
| 280 | u64 pid = bpf_get_current_pid_tgid(); |
| 281 | u64 size64 = size; |
| 282 | sizes.update(&pid, &size64); |
| 283 | |
| 284 | if (SHOULD_PRINT) |
| 285 | bpf_trace_printk("alloc entered, size = %u\\n", size); |
| 286 | return 0; |
| 287 | } |
| 288 | |
| 289 | int alloc_exit(struct pt_regs *ctx) |
| 290 | { |
| 291 | u64 address = ctx->ax; |
| 292 | u64 pid = bpf_get_current_pid_tgid(); |
| 293 | u64* size64 = sizes.lookup(&pid); |
| 294 | struct alloc_info_t info = {0}; |
| 295 | |
| 296 | if (size64 == 0) |
| 297 | return 0; // missed alloc entry |
| 298 | |
| 299 | info.size = *size64; |
| 300 | sizes.delete(&pid); |
| 301 | |
| 302 | info.timestamp_ns = bpf_ktime_get_ns(); |
| 303 | info.num_frames = grab_stack(ctx, &info) - 2; |
| 304 | allocs.update(&address, &info); |
| 305 | |
| 306 | if (SHOULD_PRINT) { |
| 307 | bpf_trace_printk("alloc exited, size = %lu, result = %lx, frames = %d\\n", |
| 308 | info.size, address, info.num_frames); |
| 309 | } |
| 310 | return 0; |
| 311 | } |
| 312 | |
| 313 | int free_enter(struct pt_regs *ctx, void *address) |
| 314 | { |
| 315 | u64 addr = (u64)address; |
| 316 | struct alloc_info_t *info = allocs.lookup(&addr); |
| 317 | if (info == 0) |
| 318 | return 0; |
| 319 | |
| 320 | allocs.delete(&addr); |
| 321 | |
| 322 | if (SHOULD_PRINT) { |
| 323 | bpf_trace_printk("free entered, address = %lx, size = %lu\\n", |
| 324 | address, info->size); |
| 325 | } |
| 326 | return 0; |
| 327 | } |
| 328 | """ |
Sasha Goldshtein | 4f1ea67 | 2016-02-07 01:57:42 -0800 | [diff] [blame] | 329 | bpf_source = bpf_source.replace("SHOULD_PRINT", "1" if trace_all else "0") |
Sasha Goldshtein | 521ab4f | 2016-02-08 05:48:31 -0800 | [diff] [blame] | 330 | bpf_source = bpf_source.replace("SAMPLE_EVERY_N", str(sample_every_n)) |
Sasha Goldshtein | d2241f4 | 2016-02-09 06:23:10 -0800 | [diff] [blame] | 331 | bpf_source = bpf_source.replace("GRAB_ONE_FRAME", max_stack_size * |
| 332 | "\tif (!(info->callstack[depth++] = get_frame(&bp))) return depth;\n") |
| 333 | bpf_source = bpf_source.replace("MAX_STACK_SIZE", str(max_stack_size)) |
Sasha Goldshtein | 5045964 | 2016-02-10 08:35:20 -0800 | [diff] [blame] | 334 | |
| 335 | size_filter = "" |
| 336 | if min_size is not None and max_size is not None: |
| 337 | size_filter = "if (size < %d || size > %d) return 0;" % \ |
| 338 | (min_size, max_size) |
| 339 | elif min_size is not None: |
| 340 | size_filter = "if (size < %d) return 0;" % min_size |
| 341 | elif max_size is not None: |
| 342 | size_filter = "if (size > %d) return 0;" % max_size |
| 343 | bpf_source = bpf_source.replace("SIZE_FILTER", size_filter) |
| 344 | |
Sasha Goldshtein | 4f1ea67 | 2016-02-07 01:57:42 -0800 | [diff] [blame] | 345 | bpf_program = BPF(text=bpf_source) |
| 346 | |
| 347 | if not kernel_trace: |
Sasha Goldshtein | 33522d7 | 2016-02-08 03:39:44 -0800 | [diff] [blame] | 348 | print("Attaching to malloc and free in pid %d, Ctrl+C to quit." % pid) |
| 349 | bpf_program.attach_uprobe(name="c", sym="malloc", |
| 350 | fn_name="alloc_enter", pid=pid) |
| 351 | bpf_program.attach_uretprobe(name="c", sym="malloc", |
| 352 | fn_name="alloc_exit", pid=pid) |
| 353 | bpf_program.attach_uprobe(name="c", sym="free", |
| 354 | fn_name="free_enter", pid=pid) |
Sasha Goldshtein | 4f1ea67 | 2016-02-07 01:57:42 -0800 | [diff] [blame] | 355 | else: |
Sasha Goldshtein | 33522d7 | 2016-02-08 03:39:44 -0800 | [diff] [blame] | 356 | print("Attaching to kmalloc and kfree, Ctrl+C to quit.") |
| 357 | bpf_program.attach_kprobe(event="__kmalloc", fn_name="alloc_enter") |
| 358 | bpf_program.attach_kretprobe(event="__kmalloc", fn_name="alloc_exit") |
| 359 | bpf_program.attach_kprobe(event="kfree", fn_name="free_enter") |
Sasha Goldshtein | 4f1ea67 | 2016-02-07 01:57:42 -0800 | [diff] [blame] | 360 | |
Sasha Goldshtein | 2922861 | 2016-02-07 12:20:19 -0800 | [diff] [blame] | 361 | decoder = StackDecoder(pid, bpf_program) |
Sasha Goldshtein | 4f1ea67 | 2016-02-07 01:57:42 -0800 | [diff] [blame] | 362 | |
| 363 | def print_outstanding(): |
Sasha Goldshtein | 33522d7 | 2016-02-08 03:39:44 -0800 | [diff] [blame] | 364 | stacks = {} |
Sasha Goldshtein | c8148c8 | 2016-02-09 11:15:41 -0800 | [diff] [blame] | 365 | print("[%s] Top %d stacks with outstanding allocations:" % |
| 366 | (datetime.now().strftime("%H:%M:%S"), top_stacks)) |
Sasha Goldshtein | 33522d7 | 2016-02-08 03:39:44 -0800 | [diff] [blame] | 367 | allocs = bpf_program.get_table("allocs") |
| 368 | for address, info in sorted(allocs.items(), key=lambda a: a[1].size): |
| 369 | if Time.monotonic_time() - min_age_ns < info.timestamp_ns: |
| 370 | continue |
| 371 | stack = decoder.decode_stack(info, kernel_trace) |
| 372 | if stack in stacks: |
| 373 | stacks[stack] = (stacks[stack][0] + 1, |
| 374 | stacks[stack][1] + info.size) |
| 375 | else: |
| 376 | stacks[stack] = (1, info.size) |
| 377 | if args.show_allocs: |
| 378 | print("\taddr = %x size = %s" % |
| 379 | (address.value, info.size)) |
Sasha Goldshtein | c8148c8 | 2016-02-09 11:15:41 -0800 | [diff] [blame] | 380 | to_show = sorted(stacks.items(), key=lambda s: s[1][1])[-top_stacks:] |
| 381 | for stack, (count, size) in to_show: |
Sasha Goldshtein | 33522d7 | 2016-02-08 03:39:44 -0800 | [diff] [blame] | 382 | print("\t%d bytes in %d allocations from stack\n\t\t%s" % |
| 383 | (size, count, stack.replace(";", "\n\t\t"))) |
Sasha Goldshtein | 4f1ea67 | 2016-02-07 01:57:42 -0800 | [diff] [blame] | 384 | |
Sasha Goldshtein | 40e55ba | 2016-02-09 05:53:48 -0800 | [diff] [blame] | 385 | count_so_far = 0 |
Sasha Goldshtein | 4f1ea67 | 2016-02-07 01:57:42 -0800 | [diff] [blame] | 386 | while True: |
Sasha Goldshtein | 33522d7 | 2016-02-08 03:39:44 -0800 | [diff] [blame] | 387 | if trace_all: |
| 388 | print bpf_program.trace_fields() |
| 389 | else: |
| 390 | try: |
| 391 | sleep(interval) |
| 392 | except KeyboardInterrupt: |
| 393 | exit() |
| 394 | decoder.refresh_code_ranges() |
| 395 | print_outstanding() |
Sasha Goldshtein | 40e55ba | 2016-02-09 05:53:48 -0800 | [diff] [blame] | 396 | count_so_far += 1 |
| 397 | if num_prints is not None and count_so_far >= num_prints: |
| 398 | exit() |