Alexey Ivanov | 777e802 | 2019-01-03 13:46:38 -0800 | [diff] [blame] | 1 | #!/usr/bin/env python |
Brendan Gregg | 60393ea | 2016-10-04 15:18:11 -0700 | [diff] [blame] | 2 | # @lint-avoid-python-3-compatibility-imports |
| 3 | # |
| 4 | # tcptop Summarize TCP send/recv throughput by host. |
| 5 | # For Linux, uses BCC, eBPF. Embedded C. |
| 6 | # |
| 7 | # USAGE: tcptop [-h] [-C] [-S] [-p PID] [interval [count]] |
| 8 | # |
| 9 | # This uses dynamic tracing of kernel functions, and will need to be updated |
| 10 | # to match kernel changes. |
| 11 | # |
| 12 | # WARNING: This traces all send/receives at the TCP level, and while it |
| 13 | # summarizes data in-kernel to reduce overhead, there may still be some |
| 14 | # overhead at high TCP send/receive rates (eg, ~13% of one CPU at 100k TCP |
| 15 | # events/sec. This is not the same as packet rate: funccount can be used to |
| 16 | # count the kprobes below to find out the TCP rate). Test in a lab environment |
| 17 | # first. If your send/receive rate is low (eg, <1k/sec) then the overhead is |
| 18 | # expected to be negligible. |
| 19 | # |
| 20 | # ToDo: Fit output to screen size (top X only) in default (not -C) mode. |
| 21 | # |
| 22 | # Copyright 2016 Netflix, Inc. |
| 23 | # Licensed under the Apache License, Version 2.0 (the "License") |
| 24 | # |
| 25 | # 02-Sep-2016 Brendan Gregg Created this. |
| 26 | |
| 27 | from __future__ import print_function |
| 28 | from bcc import BPF |
| 29 | import argparse |
| 30 | from socket import inet_ntop, AF_INET, AF_INET6 |
| 31 | from struct import pack |
| 32 | from time import sleep, strftime |
| 33 | from subprocess import call |
| 34 | import ctypes as ct |
Andreas Gerstmayr | c64f487 | 2018-07-06 14:59:07 +0200 | [diff] [blame] | 35 | from collections import namedtuple, defaultdict |
Brendan Gregg | 60393ea | 2016-10-04 15:18:11 -0700 | [diff] [blame] | 36 | |
| 37 | # arguments |
Benjamin Poirier | 8e86b9e | 2017-07-27 16:07:06 -0700 | [diff] [blame] | 38 | def range_check(string): |
| 39 | value = int(string) |
| 40 | if value < 1: |
| 41 | msg = "value must be stricly positive, got %d" % (value,) |
| 42 | raise argparse.ArgumentTypeError(msg) |
| 43 | return value |
| 44 | |
Brendan Gregg | 60393ea | 2016-10-04 15:18:11 -0700 | [diff] [blame] | 45 | examples = """examples: |
| 46 | ./tcptop # trace TCP send/recv by host |
| 47 | ./tcptop -C # don't clear the screen |
| 48 | ./tcptop -p 181 # only trace PID 181 |
| 49 | """ |
| 50 | parser = argparse.ArgumentParser( |
| 51 | description="Summarize TCP send/recv throughput by host", |
| 52 | formatter_class=argparse.RawDescriptionHelpFormatter, |
| 53 | epilog=examples) |
| 54 | parser.add_argument("-C", "--noclear", action="store_true", |
| 55 | help="don't clear the screen") |
| 56 | parser.add_argument("-S", "--nosummary", action="store_true", |
| 57 | help="skip system summary line") |
| 58 | parser.add_argument("-p", "--pid", |
| 59 | help="trace this PID only") |
Benjamin Poirier | 8e86b9e | 2017-07-27 16:07:06 -0700 | [diff] [blame] | 60 | parser.add_argument("interval", nargs="?", default=1, type=range_check, |
Brendan Gregg | 60393ea | 2016-10-04 15:18:11 -0700 | [diff] [blame] | 61 | help="output interval, in seconds (default 1)") |
Benjamin Poirier | 8e86b9e | 2017-07-27 16:07:06 -0700 | [diff] [blame] | 62 | parser.add_argument("count", nargs="?", default=-1, type=range_check, |
Brendan Gregg | 60393ea | 2016-10-04 15:18:11 -0700 | [diff] [blame] | 63 | help="number of outputs") |
Nathan Scott | cf0792f | 2018-02-02 16:56:50 +1100 | [diff] [blame] | 64 | parser.add_argument("--ebpf", action="store_true", |
| 65 | help=argparse.SUPPRESS) |
Brendan Gregg | 60393ea | 2016-10-04 15:18:11 -0700 | [diff] [blame] | 66 | args = parser.parse_args() |
Brendan Gregg | 60393ea | 2016-10-04 15:18:11 -0700 | [diff] [blame] | 67 | debug = 0 |
| 68 | |
| 69 | # linux stats |
| 70 | loadavg = "/proc/loadavg" |
| 71 | |
| 72 | # define BPF program |
| 73 | bpf_text = """ |
| 74 | #include <uapi/linux/ptrace.h> |
| 75 | #include <net/sock.h> |
| 76 | #include <bcc/proto.h> |
| 77 | |
| 78 | struct ipv4_key_t { |
| 79 | u32 pid; |
| 80 | u32 saddr; |
| 81 | u32 daddr; |
| 82 | u16 lport; |
| 83 | u16 dport; |
| 84 | }; |
| 85 | BPF_HASH(ipv4_send_bytes, struct ipv4_key_t); |
| 86 | BPF_HASH(ipv4_recv_bytes, struct ipv4_key_t); |
| 87 | |
| 88 | struct ipv6_key_t { |
| 89 | u32 pid; |
Marko Myllynen | bfbf17e | 2018-09-11 21:49:58 +0300 | [diff] [blame] | 90 | unsigned __int128 saddr; |
| 91 | unsigned __int128 daddr; |
Brendan Gregg | 60393ea | 2016-10-04 15:18:11 -0700 | [diff] [blame] | 92 | u16 lport; |
| 93 | u16 dport; |
| 94 | }; |
| 95 | BPF_HASH(ipv6_send_bytes, struct ipv6_key_t); |
| 96 | BPF_HASH(ipv6_recv_bytes, struct ipv6_key_t); |
| 97 | |
| 98 | int kprobe__tcp_sendmsg(struct pt_regs *ctx, struct sock *sk, |
| 99 | struct msghdr *msg, size_t size) |
| 100 | { |
| 101 | u32 pid = bpf_get_current_pid_tgid(); |
| 102 | FILTER |
| 103 | u16 dport = 0, family = sk->__sk_common.skc_family; |
Brendan Gregg | 60393ea | 2016-10-04 15:18:11 -0700 | [diff] [blame] | 104 | |
| 105 | if (family == AF_INET) { |
| 106 | struct ipv4_key_t ipv4_key = {.pid = pid}; |
| 107 | ipv4_key.saddr = sk->__sk_common.skc_rcv_saddr; |
| 108 | ipv4_key.daddr = sk->__sk_common.skc_daddr; |
| 109 | ipv4_key.lport = sk->__sk_common.skc_num; |
| 110 | dport = sk->__sk_common.skc_dport; |
| 111 | ipv4_key.dport = ntohs(dport); |
Javier Honduvilla Coto | 64bf965 | 2018-08-01 06:50:19 +0200 | [diff] [blame] | 112 | ipv4_send_bytes.increment(ipv4_key, size); |
Brendan Gregg | 60393ea | 2016-10-04 15:18:11 -0700 | [diff] [blame] | 113 | |
| 114 | } else if (family == AF_INET6) { |
| 115 | struct ipv6_key_t ipv6_key = {.pid = pid}; |
Marko Myllynen | bfbf17e | 2018-09-11 21:49:58 +0300 | [diff] [blame] | 116 | __builtin_memcpy(&ipv6_key.saddr, |
| 117 | sk->__sk_common.skc_v6_rcv_saddr.in6_u.u6_addr32, sizeof(ipv6_key.saddr)); |
| 118 | __builtin_memcpy(&ipv6_key.daddr, |
| 119 | sk->__sk_common.skc_v6_daddr.in6_u.u6_addr32, sizeof(ipv6_key.daddr)); |
Brendan Gregg | 60393ea | 2016-10-04 15:18:11 -0700 | [diff] [blame] | 120 | ipv6_key.lport = sk->__sk_common.skc_num; |
| 121 | dport = sk->__sk_common.skc_dport; |
| 122 | ipv6_key.dport = ntohs(dport); |
Javier Honduvilla Coto | 64bf965 | 2018-08-01 06:50:19 +0200 | [diff] [blame] | 123 | ipv6_send_bytes.increment(ipv6_key, size); |
Brendan Gregg | 60393ea | 2016-10-04 15:18:11 -0700 | [diff] [blame] | 124 | } |
| 125 | // else drop |
| 126 | |
| 127 | return 0; |
| 128 | } |
| 129 | |
| 130 | /* |
| 131 | * tcp_recvmsg() would be obvious to trace, but is less suitable because: |
| 132 | * - we'd need to trace both entry and return, to have both sock and size |
| 133 | * - misses tcp_read_sock() traffic |
| 134 | * we'd much prefer tracepoints once they are available. |
| 135 | */ |
| 136 | int kprobe__tcp_cleanup_rbuf(struct pt_regs *ctx, struct sock *sk, int copied) |
| 137 | { |
| 138 | u32 pid = bpf_get_current_pid_tgid(); |
| 139 | FILTER |
| 140 | u16 dport = 0, family = sk->__sk_common.skc_family; |
| 141 | u64 *val, zero = 0; |
| 142 | |
Benjamin Poirier | 81ad054 | 2017-07-28 13:25:14 -0700 | [diff] [blame] | 143 | if (copied <= 0) |
Paul Chaignon | 6d9b1b2 | 2017-10-07 11:06:41 +0200 | [diff] [blame] | 144 | return 0; |
Benjamin Poirier | 81ad054 | 2017-07-28 13:25:14 -0700 | [diff] [blame] | 145 | |
Brendan Gregg | 60393ea | 2016-10-04 15:18:11 -0700 | [diff] [blame] | 146 | if (family == AF_INET) { |
| 147 | struct ipv4_key_t ipv4_key = {.pid = pid}; |
| 148 | ipv4_key.saddr = sk->__sk_common.skc_rcv_saddr; |
| 149 | ipv4_key.daddr = sk->__sk_common.skc_daddr; |
| 150 | ipv4_key.lport = sk->__sk_common.skc_num; |
| 151 | dport = sk->__sk_common.skc_dport; |
| 152 | ipv4_key.dport = ntohs(dport); |
Javier Honduvilla Coto | 64bf965 | 2018-08-01 06:50:19 +0200 | [diff] [blame] | 153 | ipv4_recv_bytes.increment(ipv4_key, copied); |
| 154 | |
Brendan Gregg | 60393ea | 2016-10-04 15:18:11 -0700 | [diff] [blame] | 155 | } else if (family == AF_INET6) { |
| 156 | struct ipv6_key_t ipv6_key = {.pid = pid}; |
Marko Myllynen | bfbf17e | 2018-09-11 21:49:58 +0300 | [diff] [blame] | 157 | __builtin_memcpy(&ipv6_key.saddr, |
| 158 | sk->__sk_common.skc_v6_rcv_saddr.in6_u.u6_addr32, sizeof(ipv6_key.saddr)); |
| 159 | __builtin_memcpy(&ipv6_key.daddr, |
| 160 | sk->__sk_common.skc_v6_daddr.in6_u.u6_addr32, sizeof(ipv6_key.daddr)); |
Brendan Gregg | 60393ea | 2016-10-04 15:18:11 -0700 | [diff] [blame] | 161 | ipv6_key.lport = sk->__sk_common.skc_num; |
| 162 | dport = sk->__sk_common.skc_dport; |
| 163 | ipv6_key.dport = ntohs(dport); |
Javier Honduvilla Coto | 64bf965 | 2018-08-01 06:50:19 +0200 | [diff] [blame] | 164 | ipv6_recv_bytes.increment(ipv6_key, copied); |
Brendan Gregg | 60393ea | 2016-10-04 15:18:11 -0700 | [diff] [blame] | 165 | } |
| 166 | // else drop |
| 167 | |
| 168 | return 0; |
| 169 | } |
| 170 | """ |
| 171 | |
| 172 | # code substitutions |
| 173 | if args.pid: |
| 174 | bpf_text = bpf_text.replace('FILTER', |
| 175 | 'if (pid != %s) { return 0; }' % args.pid) |
| 176 | else: |
| 177 | bpf_text = bpf_text.replace('FILTER', '') |
Nathan Scott | cf0792f | 2018-02-02 16:56:50 +1100 | [diff] [blame] | 178 | if debug or args.ebpf: |
Brendan Gregg | 60393ea | 2016-10-04 15:18:11 -0700 | [diff] [blame] | 179 | print(bpf_text) |
Nathan Scott | cf0792f | 2018-02-02 16:56:50 +1100 | [diff] [blame] | 180 | if args.ebpf: |
| 181 | exit() |
Brendan Gregg | 60393ea | 2016-10-04 15:18:11 -0700 | [diff] [blame] | 182 | |
Andreas Gerstmayr | c64f487 | 2018-07-06 14:59:07 +0200 | [diff] [blame] | 183 | TCPSessionKey = namedtuple('TCPSession', ['pid', 'laddr', 'lport', 'daddr', 'dport']) |
| 184 | |
Brendan Gregg | 60393ea | 2016-10-04 15:18:11 -0700 | [diff] [blame] | 185 | def pid_to_comm(pid): |
| 186 | try: |
| 187 | comm = open("/proc/%d/comm" % pid, "r").read().rstrip() |
| 188 | return comm |
| 189 | except IOError: |
| 190 | return str(pid) |
| 191 | |
Andreas Gerstmayr | c64f487 | 2018-07-06 14:59:07 +0200 | [diff] [blame] | 192 | def get_ipv4_session_key(k): |
| 193 | return TCPSessionKey(pid=k.pid, |
| 194 | laddr=inet_ntop(AF_INET, pack("I", k.saddr)), |
| 195 | lport=k.lport, |
| 196 | daddr=inet_ntop(AF_INET, pack("I", k.daddr)), |
| 197 | dport=k.dport) |
| 198 | |
| 199 | def get_ipv6_session_key(k): |
| 200 | return TCPSessionKey(pid=k.pid, |
Marko Myllynen | bfbf17e | 2018-09-11 21:49:58 +0300 | [diff] [blame] | 201 | laddr=inet_ntop(AF_INET6, k.saddr), |
Andreas Gerstmayr | c64f487 | 2018-07-06 14:59:07 +0200 | [diff] [blame] | 202 | lport=k.lport, |
Marko Myllynen | bfbf17e | 2018-09-11 21:49:58 +0300 | [diff] [blame] | 203 | daddr=inet_ntop(AF_INET6, k.daddr), |
Andreas Gerstmayr | c64f487 | 2018-07-06 14:59:07 +0200 | [diff] [blame] | 204 | dport=k.dport) |
| 205 | |
Brendan Gregg | 60393ea | 2016-10-04 15:18:11 -0700 | [diff] [blame] | 206 | # initialize BPF |
| 207 | b = BPF(text=bpf_text) |
| 208 | |
| 209 | ipv4_send_bytes = b["ipv4_send_bytes"] |
| 210 | ipv4_recv_bytes = b["ipv4_recv_bytes"] |
| 211 | ipv6_send_bytes = b["ipv6_send_bytes"] |
| 212 | ipv6_recv_bytes = b["ipv6_recv_bytes"] |
| 213 | |
| 214 | print('Tracing... Output every %s secs. Hit Ctrl-C to end' % args.interval) |
| 215 | |
| 216 | # output |
Benjamin Poirier | 8e86b9e | 2017-07-27 16:07:06 -0700 | [diff] [blame] | 217 | i = 0 |
| 218 | exiting = False |
| 219 | while i != args.count and not exiting: |
Brendan Gregg | 60393ea | 2016-10-04 15:18:11 -0700 | [diff] [blame] | 220 | try: |
Benjamin Poirier | 8e86b9e | 2017-07-27 16:07:06 -0700 | [diff] [blame] | 221 | sleep(args.interval) |
Brendan Gregg | 60393ea | 2016-10-04 15:18:11 -0700 | [diff] [blame] | 222 | except KeyboardInterrupt: |
Benjamin Poirier | 8e86b9e | 2017-07-27 16:07:06 -0700 | [diff] [blame] | 223 | exiting = True |
Brendan Gregg | 60393ea | 2016-10-04 15:18:11 -0700 | [diff] [blame] | 224 | |
| 225 | # header |
| 226 | if args.noclear: |
| 227 | print() |
| 228 | else: |
| 229 | call("clear") |
| 230 | if not args.nosummary: |
| 231 | with open(loadavg) as stats: |
| 232 | print("%-8s loadavg: %s" % (strftime("%H:%M:%S"), stats.read())) |
| 233 | |
Andreas Gerstmayr | c64f487 | 2018-07-06 14:59:07 +0200 | [diff] [blame] | 234 | # IPv4: build dict of all seen keys |
| 235 | ipv4_throughput = defaultdict(lambda: [0, 0]) |
Brendan Gregg | 60393ea | 2016-10-04 15:18:11 -0700 | [diff] [blame] | 236 | for k, v in ipv4_send_bytes.items(): |
Andreas Gerstmayr | c64f487 | 2018-07-06 14:59:07 +0200 | [diff] [blame] | 237 | key = get_ipv4_session_key(k) |
| 238 | ipv4_throughput[key][0] = v.value |
| 239 | ipv4_send_bytes.clear() |
Brendan Gregg | 60393ea | 2016-10-04 15:18:11 -0700 | [diff] [blame] | 240 | |
Andreas Gerstmayr | c64f487 | 2018-07-06 14:59:07 +0200 | [diff] [blame] | 241 | for k, v in ipv4_recv_bytes.items(): |
| 242 | key = get_ipv4_session_key(k) |
| 243 | ipv4_throughput[key][1] = v.value |
| 244 | ipv4_recv_bytes.clear() |
| 245 | |
| 246 | if ipv4_throughput: |
Brendan Gregg | 60393ea | 2016-10-04 15:18:11 -0700 | [diff] [blame] | 247 | print("%-6s %-12s %-21s %-21s %6s %6s" % ("PID", "COMM", |
| 248 | "LADDR", "RADDR", "RX_KB", "TX_KB")) |
| 249 | |
| 250 | # output |
Andreas Gerstmayr | c64f487 | 2018-07-06 14:59:07 +0200 | [diff] [blame] | 251 | for k, (send_bytes, recv_bytes) in sorted(ipv4_throughput.items(), |
| 252 | key=lambda kv: sum(kv[1]), |
| 253 | reverse=True): |
Brendan Gregg | 60393ea | 2016-10-04 15:18:11 -0700 | [diff] [blame] | 254 | print("%-6d %-12.12s %-21s %-21s %6d %6d" % (k.pid, |
| 255 | pid_to_comm(k.pid), |
Andreas Gerstmayr | c64f487 | 2018-07-06 14:59:07 +0200 | [diff] [blame] | 256 | k.laddr + ":" + str(k.lport), |
| 257 | k.daddr + ":" + str(k.dport), |
| 258 | int(recv_bytes / 1024), int(send_bytes / 1024))) |
Brendan Gregg | 60393ea | 2016-10-04 15:18:11 -0700 | [diff] [blame] | 259 | |
| 260 | # IPv6: build dict of all seen keys |
Andreas Gerstmayr | c64f487 | 2018-07-06 14:59:07 +0200 | [diff] [blame] | 261 | ipv6_throughput = defaultdict(lambda: [0, 0]) |
Brendan Gregg | 60393ea | 2016-10-04 15:18:11 -0700 | [diff] [blame] | 262 | for k, v in ipv6_send_bytes.items(): |
Andreas Gerstmayr | c64f487 | 2018-07-06 14:59:07 +0200 | [diff] [blame] | 263 | key = get_ipv6_session_key(k) |
| 264 | ipv6_throughput[key][0] = v.value |
| 265 | ipv6_send_bytes.clear() |
Brendan Gregg | 60393ea | 2016-10-04 15:18:11 -0700 | [diff] [blame] | 266 | |
Andreas Gerstmayr | c64f487 | 2018-07-06 14:59:07 +0200 | [diff] [blame] | 267 | for k, v in ipv6_recv_bytes.items(): |
| 268 | key = get_ipv6_session_key(k) |
| 269 | ipv6_throughput[key][1] = v.value |
| 270 | ipv6_recv_bytes.clear() |
| 271 | |
| 272 | if ipv6_throughput: |
Brendan Gregg | 60393ea | 2016-10-04 15:18:11 -0700 | [diff] [blame] | 273 | # more than 80 chars, sadly. |
| 274 | print("\n%-6s %-12s %-32s %-32s %6s %6s" % ("PID", "COMM", |
| 275 | "LADDR6", "RADDR6", "RX_KB", "TX_KB")) |
| 276 | |
| 277 | # output |
Andreas Gerstmayr | c64f487 | 2018-07-06 14:59:07 +0200 | [diff] [blame] | 278 | for k, (send_bytes, recv_bytes) in sorted(ipv6_throughput.items(), |
| 279 | key=lambda kv: sum(kv[1]), |
| 280 | reverse=True): |
Brendan Gregg | 60393ea | 2016-10-04 15:18:11 -0700 | [diff] [blame] | 281 | print("%-6d %-12.12s %-32s %-32s %6d %6d" % (k.pid, |
| 282 | pid_to_comm(k.pid), |
Andreas Gerstmayr | c64f487 | 2018-07-06 14:59:07 +0200 | [diff] [blame] | 283 | k.laddr + ":" + str(k.lport), |
| 284 | k.daddr + ":" + str(k.dport), |
| 285 | int(recv_bytes / 1024), int(send_bytes / 1024))) |
Brendan Gregg | 60393ea | 2016-10-04 15:18:11 -0700 | [diff] [blame] | 286 | |
Benjamin Poirier | 8e86b9e | 2017-07-27 16:07:06 -0700 | [diff] [blame] | 287 | i += 1 |