add tcplife (#773)
diff --git a/README.md b/README.md
index f94e837..14f725f 100644
--- a/README.md
+++ b/README.md
@@ -121,6 +121,7 @@
- tools/[tcpaccept](tools/tcpaccept.py): Trace TCP passive connections (accept()). [Examples](tools/tcpaccept_example.txt).
- tools/[tcpconnect](tools/tcpconnect.py): Trace TCP active connections (connect()). [Examples](tools/tcpconnect_example.txt).
- tools/[tcpconnlat](tools/tcpconnlat.py): Trace TCP active connection latency (connect()). [Examples](tools/tcpconnlat_example.txt).
+- tools/[tcplife](tools/tcplife.py): Trace TCP sessions and summarize lifespan. [Examples](tools/tcplife_example.txt).
- tools/[tcpretrans](tools/tcpretrans.py): Trace TCP retransmits and TLPs. [Examples](tools/tcpretrans_example.txt).
- tools/[tcptop](tools/tcptop.py): Summarize TCP send/recv throughput by host. Top for TCP. [Examples](tools/tcptop_example.txt).
- tools/[tplist](tools/tplist.py): Display kernel tracepoints or USDT probes and their formats. [Examples](tools/tplist_example.txt).
diff --git a/man/man8/tcplife.8 b/man/man8/tcplife.8
new file mode 100644
index 0000000..059caa9
--- /dev/null
+++ b/man/man8/tcplife.8
@@ -0,0 +1,130 @@
+.TH tcplife 8 "2016-10-19" "USER COMMANDS"
+.SH NAME
+tcplife \- Trace TCP sessions and summarize lifespan. Uses Linux eBPF/bcc.
+.SH SYNOPSIS
+.B tcplife [\-h] [\-T] [\-t] [\-w] [\-s] [\-p PID] [\-D PORTS] [\-L PORTS]
+.SH DESCRIPTION
+This tool traces TCP sessions that open and close while tracing, and prints
+a line of output to summarize each one. This includes the IP addresses, ports,
+duration, and throughput for the session. This is useful for workload
+characterisation and flow accounting: identifying what connections are
+happening, with the bytes transferred.
+
+This tool works by using kernel dynamic tracing, and will need to be updated
+if the kernel implementation changes. Only TCP state changes are traced, so
+it is expected that the overhead of this tool is much lower than typical
+send/receive tracing.
+
+Since this uses BPF, only the root user can use this tool.
+.SH REQUIREMENTS
+CONFIG_BPF and bcc.
+.SH OPTIONS
+.TP
+\-h
+Print usage message.
+.TP
+\-s
+Comma separated values output (parseable).
+.TP
+\-t
+Include a timestamp column (seconds).
+.TP
+\-T
+Include a time column (HH:MM:SS).
+.TP
+\-w
+Wide column output (fits IPv6 addresses).
+.TP
+\-p PID
+Trace this process ID only (filtered in-kernel).
+.TP
+\-L PORTS
+Comma-separated list of local ports to trace (filtered in-kernel).
+.TP
+\-D PORTS
+Comma-separated list of destination ports to trace (filtered in-kernel).
+.SH EXAMPLES
+.TP
+Trace all TCP sessions, and summarize lifespan and throughput:
+#
+.B tcplife
+.TP
+Include a timestamp column, and wide column output:
+#
+.B tcplife \-tw
+.TP
+Trace PID 181 only:
+#
+.B tcplife \-p 181
+.TP
+Trace connections to local ports 80 and 81 only:
+#
+.B tcplife \-L 80,81
+.TP
+Trace connections to remote port 80 only:
+#
+.B tcplife \-D 80
+.SH FIELDS
+.TP
+TIME
+Time of the call, in HH:MM:SS format.
+.TP
+TIME(s)
+Time of the call, in seconds.
+.TP
+PID
+Process ID
+.TP
+COMM
+Process name
+.TP
+IP
+IP address family (4 or 6)
+.TP
+LADDR
+Local IP address.
+.TP
+DADDR
+Remote IP address.
+.TP
+LPORT
+Local port.
+.TP
+DPORT
+Destination port.
+.TP
+TX_KB
+Total transmitted Kbytes.
+.TP
+RX_KB
+Total received Kbytes.
+.TP
+MS
+Lifespan of the session, in milliseconds.
+.SH OVERHEAD
+This traces the kernel TCP set state function, which should be called much
+less often than send/receive tracing, and therefore have lower overhead. The
+overhead of the tool is relative to the rate of new TCP sessions: if this is
+high, over 10,000 per second, then there may be noticable overhead just to
+print out 10k lines of formatted output per second.
+
+You can find out the rate of new TCP sessions using "sar \-n TCP 1", and
+adding the active/s and passive/s columns.
+
+As always, test and understand this tools overhead for your types of
+workloads before production use.
+.SH SOURCE
+This is from bcc.
+.IP
+https://github.com/iovisor/bcc
+.PP
+Also look in the bcc distribution for a companion _examples.txt file containing
+example usage, output, and commentary for this tool.
+.SH OS
+Linux
+.SH STABILITY
+Unstable - in development.
+.SH AUTHOR
+Brendan Gregg
+.SH SEE ALSO
+tcpaccept(8), tcpconnect(8), tcptop(8)
diff --git a/tools/tcplife.py b/tools/tcplife.py
new file mode 100755
index 0000000..4bf2ca6
--- /dev/null
+++ b/tools/tcplife.py
@@ -0,0 +1,356 @@
+#!/usr/bin/python
+# @lint-avoid-python-3-compatibility-imports
+#
+# tcplife Trace the lifespan of TCP sessions and summarize.
+# For Linux, uses BCC, BPF. Embedded C.
+#
+# USAGE: tcplife [-h] [-C] [-S] [-p PID] [interval [count]]
+#
+# This uses dynamic tracing of kernel functions, and will need to be updated
+# to match kernel changes.
+#
+# While throughput counters are emitted, they are fetched in a low-overhead
+# manner: reading members of the tcp_info struct on TCP close. ie, we do not
+# trace send/receive.
+#
+# Copyright 2016 Netflix, Inc.
+# Licensed under the Apache License, Version 2.0 (the "License")
+#
+# IDEA: Julia Evans
+#
+# 18-Oct-2016 Brendan Gregg Created this.
+
+from __future__ import print_function
+from bcc import BPF
+import argparse
+from socket import inet_ntop, ntohs, AF_INET, AF_INET6
+from struct import pack
+import ctypes as ct
+from time import strftime
+
+# arguments
+examples = """examples:
+ ./tcplife # trace all TCP connect()s
+ ./tcplife -t # include time column (HH:MM:SS)
+ ./tcplife -w # wider colums (fit IPv6)
+ ./tcplife -stT # csv output, with times & timestamps
+ ./tcplife -p 181 # only trace PID 181
+ ./tcplife -L 80 # only trace local port 80
+ ./tcplife -L 80,81 # only trace local ports 80 and 81
+ ./tcplife -D 80 # only trace remote port 80
+"""
+parser = argparse.ArgumentParser(
+ description="Trace the lifespan of TCP sessions and summarize",
+ formatter_class=argparse.RawDescriptionHelpFormatter,
+ epilog=examples)
+parser.add_argument("-T", "--time", action="store_true",
+ help="include time column on output (HH:MM:SS)")
+parser.add_argument("-t", "--timestamp", action="store_true",
+ help="include timestamp on output (seconds)")
+parser.add_argument("-w", "--wide", action="store_true",
+ help="wide column output (fits IPv6 addresses)")
+parser.add_argument("-s", "--csv", action="store_true",
+ help="comma seperated values output")
+parser.add_argument("-p", "--pid",
+ help="trace this PID only")
+parser.add_argument("-L", "--localport",
+ help="comma-separated list of local ports to trace.")
+parser.add_argument("-D", "--remoteport",
+ help="comma-separated list of remote ports to trace.")
+args = parser.parse_args()
+debug = 0
+
+# define BPF program
+bpf_text = """
+#include <uapi/linux/ptrace.h>
+#define KBUILD_MODNAME "foo"
+#include <linux/tcp.h>
+#include <net/sock.h>
+#include <bcc/proto.h>
+
+BPF_HASH(birth, struct sock *, u64);
+
+// separate data structs for ipv4 and ipv6
+struct ipv4_data_t {
+ // XXX: switch some to u32's when supported
+ u64 ts_us;
+ u64 pid;
+ u64 saddr;
+ u64 daddr;
+ u64 ports;
+ u64 rx_b;
+ u64 tx_b;
+ u64 span_us;
+ char task[TASK_COMM_LEN];
+};
+BPF_PERF_OUTPUT(ipv4_events);
+
+struct ipv6_data_t {
+ u64 ts_us;
+ u64 pid;
+ unsigned __int128 saddr;
+ unsigned __int128 daddr;
+ u64 ports;
+ u64 rx_b;
+ u64 tx_b;
+ u64 span_us;
+ char task[TASK_COMM_LEN];
+};
+BPF_PERF_OUTPUT(ipv6_events);
+
+struct id_t {
+ u32 pid;
+ char task[TASK_COMM_LEN];
+};
+BPF_HASH(whoami, struct sock *, struct id_t);
+
+int kprobe__tcp_set_state(struct pt_regs *ctx, struct sock *sk, int state)
+{
+ u32 pid = bpf_get_current_pid_tgid() >> 32;
+
+ // lport is either used in a filter here, or later
+ u16 lport = sk->__sk_common.skc_num;
+ FILTER_LPORT
+
+ // dport is either used in a filter here, or later
+ u16 dport = sk->__sk_common.skc_dport;
+ FILTER_DPORT
+
+ /*
+ * This tool includes PID and comm context. It's best effort, and may
+ * be wrong in some situations. It currently works like this:
+ * - active connections: cache PID & comm on TCP_SYN_SENT
+ * - passive connections: read PID & comm on TCP_LAST_ACK
+ */
+
+ // record PID & comm on SYN_SENT
+ if (state == TCP_SYN_SENT) {
+ FILTER_PID
+ struct id_t me = {.pid = pid};
+ bpf_get_current_comm(&me.task, sizeof(me.task));
+ whoami.update(&sk, &me);
+ }
+
+ // capture birth time
+ if (state < TCP_FIN_WAIT1) {
+ // matching just ESTABLISHED may be sufficient, provided no code-path
+ // sets ESTABLISHED without a tcp_set_state() call. Until we know
+ // that for sure, match all early states to increase chances a
+ // timestamp is set.
+ u64 ts = bpf_ktime_get_ns();
+ birth.update(&sk, &ts);
+ return 0;
+ }
+
+ // fetch possible cached data
+ struct id_t *mep;
+ mep = whoami.lookup(&sk);
+
+ // passive connection closing; wait until TCP_LAST_ACK
+ if (mep == 0 && state != TCP_LAST_ACK)
+ return 0;
+
+ if (state == TCP_LAST_ACK) {
+ FILTER_PID
+ }
+
+ // calculate lifespan
+ u64 *tsp, delta_us;
+ tsp = birth.lookup(&sk);
+ if (tsp == 0) {
+ return 0; // missed create
+ }
+ delta_us = (bpf_ktime_get_ns() - *tsp) / 1000;
+
+ // get throughput stats. see tcp_get_info().
+ u64 rx_b = 0, tx_b = 0, sport = 0;
+ struct tcp_sock *tp = (struct tcp_sock *)sk;
+ rx_b = tp->bytes_received;
+ tx_b = tp->bytes_acked;
+
+ u16 family = sk->__sk_common.skc_family;
+
+ if (family == AF_INET) {
+ struct ipv4_data_t data4 = {.span_us = delta_us,
+ .rx_b = rx_b, .tx_b = tx_b};
+ data4.ts_us = bpf_ktime_get_ns() / 1000;
+ data4.saddr = sk->__sk_common.skc_rcv_saddr;
+ data4.daddr = sk->__sk_common.skc_daddr;
+ // a workaround until data4 compiles with separate lport/dport
+ data4.ports = ntohs(dport) + ((0ULL + lport) << 32);
+ if (mep == 0) {
+ data4.pid = pid;
+ bpf_get_current_comm(&data4.task, sizeof(data4.task));
+ bpf_trace_printk("state %d 0pid %d\\n", state, pid);
+ } else {
+ data4.pid = mep->pid;
+ bpf_trace_printk("state %d mpid %d\\n", state, mep->pid);
+ bpf_probe_read(&data4.task, sizeof(data4.task), (void *)mep->task);
+ }
+ ipv4_events.perf_submit(ctx, &data4, sizeof(data4));
+
+ } else /* 6 */ {
+ struct ipv6_data_t data6 = {.span_us = delta_us,
+ .rx_b = rx_b, .tx_b = tx_b};
+ data6.ts_us = bpf_ktime_get_ns() / 1000;
+ bpf_probe_read(&data6.saddr, sizeof(data6.saddr),
+ sk->__sk_common.skc_v6_rcv_saddr.in6_u.u6_addr32);
+ bpf_probe_read(&data6.daddr, sizeof(data6.daddr),
+ sk->__sk_common.skc_v6_daddr.in6_u.u6_addr32);
+ // a workaround until data6 compiles with separate lport/dport
+ data6.ports = ntohs(dport) + ((0ULL + lport) << 32);
+ if (mep == 0) {
+ data6.pid = pid;
+ bpf_get_current_comm(&data6.task, sizeof(data6.task));
+ } else {
+ data6.pid = mep->pid;
+ bpf_probe_read(&data6.task, sizeof(data6.task), (void *)mep->task);
+ }
+ ipv6_events.perf_submit(ctx, &data6, sizeof(data6));
+ }
+
+ birth.delete(&sk);
+ if (mep != 0)
+ whoami.delete(&sk);
+
+ return 0;
+}
+"""
+
+# code substitutions
+if args.pid:
+ bpf_text = bpf_text.replace('FILTER_PID',
+ 'if (pid != %s) { return 0; }' % args.pid)
+if args.remoteport:
+ dports = [int(dport) for dport in args.remoteport.split(',')]
+ dports_if = ' && '.join(['dport != %d' % ntohs(dport) for dport in dports])
+ bpf_text = bpf_text.replace('FILTER_DPORT',
+ 'if (%s) { birth.delete(&sk); return 0; }' % dports_if)
+if args.localport:
+ lports = [int(lport) for lport in args.localport.split(',')]
+ lports_if = ' && '.join(['lport != %d' % lport for lport in lports])
+ bpf_text = bpf_text.replace('FILTER_LPORT',
+ 'if (%s) { birth.delete(&sk); return 0; }' % lports_if)
+bpf_text = bpf_text.replace('FILTER_PID', '')
+bpf_text = bpf_text.replace('FILTER_DPORT', '')
+bpf_text = bpf_text.replace('FILTER_LPORT', '')
+
+if debug:
+ print(bpf_text)
+
+# event data
+TASK_COMM_LEN = 16 # linux/sched.h
+
+class Data_ipv4(ct.Structure):
+ _fields_ = [
+ ("ts_us", ct.c_ulonglong),
+ ("pid", ct.c_ulonglong),
+ ("saddr", ct.c_ulonglong),
+ ("daddr", ct.c_ulonglong),
+ ("ports", ct.c_ulonglong),
+ ("rx_b", ct.c_ulonglong),
+ ("tx_b", ct.c_ulonglong),
+ ("span_us", ct.c_ulonglong),
+ ("task", ct.c_char * TASK_COMM_LEN)
+ ]
+
+class Data_ipv6(ct.Structure):
+ _fields_ = [
+ ("ts_us", ct.c_ulonglong),
+ ("pid", ct.c_ulonglong),
+ ("saddr", (ct.c_ulonglong * 2)),
+ ("daddr", (ct.c_ulonglong * 2)),
+ ("ports", ct.c_ulonglong),
+ ("rx_b", ct.c_ulonglong),
+ ("tx_b", ct.c_ulonglong),
+ ("span_us", ct.c_ulonglong),
+ ("task", ct.c_char * TASK_COMM_LEN)
+ ]
+
+#
+# Setup output formats
+#
+# Don't change the default output (next 2 lines): this fits in 80 chars. I
+# know it doesn't have NS or UIDs etc. I know. If you really, really, really
+# need to add columns, columns that solve real actual problems, I'd start by
+# adding an extended mode (-x) to included those columns.
+#
+header_string = "%-5s %-10.10s %s%-15s %-5s %-15s %-5s %5s %5s %s"
+format_string = "%-5d %-10.10s %s%-15s %-5d %-15s %-5d %5d %5d %.2f"
+if args.wide:
+ header_string = "%-5s %-16.16s %-2s %-26s %-5s %-26s %-5s %6s %6s %s"
+ format_string = "%-5d %-16.16s %-2s %-26s %-5s %-26s %-5d %6d %6d %.2f"
+if args.csv:
+ header_string = "%s,%s,%s,%s,%s,%s,%s,%s,%s,%s"
+ format_string = "%d,%s,%s,%s,%s,%s,%d,%d,%d,%.2f"
+
+# process event
+def print_ipv4_event(cpu, data, size):
+ event = ct.cast(data, ct.POINTER(Data_ipv4)).contents
+ global start_ts
+ if args.time:
+ if args.csv:
+ print("%s," % strftime("%H:%M:%S"), end="")
+ else:
+ print("%-8s " % strftime("%H:%M:%S"), end="")
+ if args.timestamp:
+ if start_ts == 0:
+ start_ts = event.ts_us
+ delta_s = (float(event.ts_us) - start_ts) / 1000000
+ if args.csv:
+ print("%.6f," % delta_s, end="")
+ else:
+ print("%-9.6f " % delta_s, end="")
+ print(format_string % (event.pid, event.task,
+ "4" if args.wide or args.csv else "",
+ inet_ntop(AF_INET, pack("I", event.saddr)), event.ports >> 32,
+ inet_ntop(AF_INET, pack("I", event.daddr)), event.ports & 0xffffffff,
+ event.tx_b / 1024, event.rx_b / 1024, float(event.span_us) / 1000))
+
+def print_ipv6_event(cpu, data, size):
+ event = ct.cast(data, ct.POINTER(Data_ipv6)).contents
+ global start_ts
+ if args.time:
+ if args.csv:
+ print("%s," % strftime("%H:%M:%S"), end="")
+ else:
+ print("%-8s " % strftime("%H:%M:%S"), end="")
+ if args.timestamp:
+ if start_ts == 0:
+ start_ts = event.ts_us
+ delta_s = (float(event.ts_us) - start_ts) / 1000000
+ if args.csv:
+ print("%.6f," % delta_s, end="")
+ else:
+ print("%-9.6f " % delta_s, end="")
+ print(format_string % (event.pid, event.task,
+ "6" if args.wide or args.csv else "",
+ inet_ntop(AF_INET6, event.saddr), event.ports >> 32,
+ inet_ntop(AF_INET6, event.daddr), event.ports & 0xffffffff,
+ event.tx_b / 1024, event.rx_b / 1024, float(event.span_us) / 1000))
+
+# initialize BPF
+b = BPF(text=bpf_text)
+
+# header
+if args.time:
+ if args.csv:
+ print("%s," % ("TIME"), end="")
+ else:
+ print("%-8s " % ("TIME"), end="")
+if args.timestamp:
+ if args.csv:
+ print("%s," % ("TIME(s)"), end="")
+ else:
+ print("%-9s " % ("TIME(s)"), end="")
+print(header_string % ("PID", "COMM",
+ "IP" if args.wide or args.csv else "", "LADDR",
+ "LPORT", "RADDR", "RPORT", "TX_KB", "RX_KB", "MS"))
+
+start_ts = 0
+
+# read events
+b["ipv4_events"].open_perf_buffer(print_ipv4_event)
+b["ipv6_events"].open_perf_buffer(print_ipv6_event)
+while 1:
+ b.kprobe_poll()
diff --git a/tools/tcplife_example.txt b/tools/tcplife_example.txt
new file mode 100644
index 0000000..457fe78
--- /dev/null
+++ b/tools/tcplife_example.txt
@@ -0,0 +1,135 @@
+Demonstrations of tcplife, the Linux BPF/bcc version.
+
+
+tcplife summarizes TCP sessions that open and close while tracing. For example:
+
+# ./tcplife
+PID COMM LADDR LPORT RADDR RPORT TX_KB RX_KB MS
+22597 recordProg 127.0.0.1 46644 127.0.0.1 28527 0 0 0.23
+3277 redis-serv 127.0.0.1 28527 127.0.0.1 46644 0 0 0.28
+22598 curl 100.66.3.172 61620 52.205.89.26 80 0 1 91.79
+22604 curl 100.66.3.172 44400 52.204.43.121 80 0 1 121.38
+22624 recordProg 127.0.0.1 46648 127.0.0.1 28527 0 0 0.22
+3277 redis-serv 127.0.0.1 28527 127.0.0.1 46648 0 0 0.27
+22647 recordProg 127.0.0.1 46650 127.0.0.1 28527 0 0 0.21
+3277 redis-serv 127.0.0.1 28527 127.0.0.1 46650 0 0 0.26
+[...]
+
+This caught a program, "recordProg" making a few short-lived TCP connections
+to "redis-serv", lasting about 0.25 milliseconds each connection. A couple of
+"curl" sessions were also traced, connecting to port 80, and lasting 91 and 121
+milliseconds.
+
+This tool is useful for workload characterisation and flow accounting:
+identifying what connections are happening, with the bytes transferred.
+
+
+Process names are truncated to 10 characters. By using the wide option, -w,
+the column width becomes 16 characters. The IP address columns are also wider
+to fit IPv6 addresses:
+
+# ./tcplife -w
+PID COMM IP LADDR LPORT RADDR RPORT TX_KB RX_KB MS
+26315 recordProgramSt 4 127.0.0.1 44188 127.0.0.1 28527 0 0 0.21
+3277 redis-server 4 127.0.0.1 28527 127.0.0.1 44188 0 0 0.26
+26320 ssh 6 fe80::8a3:9dff:fed5:6b19 22440 fe80::8a3:9dff:fed5:6b19 22 1 1 457.52
+26321 sshd 6 fe80::8a3:9dff:fed5:6b19 22 fe80::8a3:9dff:fed5:6b19 22440 1 1 458.69
+26341 recordProgramSt 4 127.0.0.1 44192 127.0.0.1 28527 0 0 0.27
+3277 redis-server 4 127.0.0.1 28527 127.0.0.1 44192 0 0 0.32
+
+
+In this example, I uploaded a 10 Mbyte file to the server, and then downloaded
+it again, using scp:
+
+# ./tcplife
+PID COMM LADDR LPORT RADDR RPORT TX_KB RX_KB MS
+7715 recordProg 127.0.0.1 50894 127.0.0.1 28527 0 0 0.25
+3277 redis-serv 127.0.0.1 28527 127.0.0.1 50894 0 0 0.30
+7619 sshd 100.66.3.172 22 100.127.64.230 63033 5 10255 3066.79
+7770 recordProg 127.0.0.1 50896 127.0.0.1 28527 0 0 0.20
+3277 redis-serv 127.0.0.1 28527 127.0.0.1 50896 0 0 0.24
+7793 recordProg 127.0.0.1 50898 127.0.0.1 28527 0 0 0.23
+3277 redis-serv 127.0.0.1 28527 127.0.0.1 50898 0 0 0.27
+7847 recordProg 127.0.0.1 50900 127.0.0.1 28527 0 0 0.24
+3277 redis-serv 127.0.0.1 28527 127.0.0.1 50900 0 0 0.29
+7870 recordProg 127.0.0.1 50902 127.0.0.1 28527 0 0 0.29
+3277 redis-serv 127.0.0.1 28527 127.0.0.1 50902 0 0 0.30
+7798 sshd 100.66.3.172 22 100.127.64.230 64925 10265 6 2176.15
+[...]
+
+You can see the 10 Mbytes received by sshd, and then later transmitted. Looks
+like receive was slower (3.07 seconds) than transmit (2.18 seconds).
+
+
+Timestamps can be added with -t:
+
+# ./tcplife -t
+TIME(s) PID COMM LADDR LPORT RADDR RPORT TX_KB RX_KB MS
+0.000000 5973 recordProg 127.0.0.1 47986 127.0.0.1 28527 0 0 0.25
+0.000059 3277 redis-serv 127.0.0.1 28527 127.0.0.1 47986 0 0 0.29
+1.022454 5996 recordProg 127.0.0.1 47988 127.0.0.1 28527 0 0 0.23
+1.022513 3277 redis-serv 127.0.0.1 28527 127.0.0.1 47988 0 0 0.27
+2.044868 6019 recordProg 127.0.0.1 47990 127.0.0.1 28527 0 0 0.24
+2.044924 3277 redis-serv 127.0.0.1 28527 127.0.0.1 47990 0 0 0.28
+3.069136 6042 recordProg 127.0.0.1 47992 127.0.0.1 28527 0 0 0.22
+3.069204 3277 redis-serv 127.0.0.1 28527 127.0.0.1 47992 0 0 0.28
+
+This shows that the recordProg process was connecting once per second.
+
+There's also a -T for HH:MM:SS formatted times.
+
+
+There's a comma separated values mode, -s. Here it is with both -t and -T
+timestamps:
+
+# ./tcplife -stT
+TIME,TIME(s),PID,COMM,IP,LADDR,LPORT,RADDR,RPORT,TX_KB,RX_KB,MS
+23:39:38,0.000000,7335,recordProgramSt,4,127.0.0.1,48098,127.0.0.1,28527,0,0,0.26
+23:39:38,0.000064,3277,redis-server,4,127.0.0.1,28527,127.0.0.1,48098,0,0,0.32
+23:39:39,1.025078,7358,recordProgramSt,4,127.0.0.1,48100,127.0.0.1,28527,0,0,0.25
+23:39:39,1.025141,3277,redis-server,4,127.0.0.1,28527,127.0.0.1,48100,0,0,0.30
+23:39:41,2.040949,7381,recordProgramSt,4,127.0.0.1,48102,127.0.0.1,28527,0,0,0.24
+23:39:41,2.041011,3277,redis-server,4,127.0.0.1,28527,127.0.0.1,48102,0,0,0.29
+23:39:42,3.067848,7404,recordProgramSt,4,127.0.0.1,48104,127.0.0.1,28527,0,0,0.30
+23:39:42,3.067914,3277,redis-server,4,127.0.0.1,28527,127.0.0.1,48104,0,0,0.35
+[...]
+
+
+There are options for filtering on local and remote ports. Here is filtering
+on local ports 22 and 80:
+
+# ./tcplife.py -L 22,80
+PID COMM LADDR LPORT RADDR RPORT TX_KB RX_KB MS
+8301 sshd 100.66.3.172 22 100.127.64.230 58671 3 3 1448.52
+[...]
+
+
+USAGE:
+
+# ./tcplife.py -h
+usage: tcplife.py [-h] [-T] [-t] [-w] [-s] [-p PID] [-L LOCALPORT]
+ [-D REMOTEPORT]
+
+Trace the lifespan of TCP sessions and summarize
+
+optional arguments:
+ -h, --help show this help message and exit
+ -T, --time include time column on output (HH:MM:SS)
+ -t, --timestamp include timestamp on output (seconds)
+ -w, --wide wide column output (fits IPv6 addresses)
+ -s, --csv comma seperated values output
+ -p PID, --pid PID trace this PID only
+ -L LOCALPORT, --localport LOCALPORT
+ comma-separated list of local ports to trace.
+ -D REMOTEPORT, --remoteport REMOTEPORT
+ comma-separated list of remote ports to trace.
+
+examples:
+ ./tcplife # trace all TCP connect()s
+ ./tcplife -t # include time column (HH:MM:SS)
+ ./tcplife -w # wider colums (fit IPv6)
+ ./tcplife -stT # csv output, with times & timestamps
+ ./tcplife -p 181 # only trace PID 181
+ ./tcplife -L 80 # only trace local port 80
+ ./tcplife -L 80,81 # only trace local ports 80 and 81
+ ./tcplife -D 80 # only trace remote port 80