Brendan Gregg | 860b649 | 2015-10-20 15:52:23 -0700 | [diff] [blame] | 1 | #!/usr/bin/python |
Alexei Starovoitov | bdf0773 | 2016-01-14 10:09:20 -0800 | [diff] [blame] | 2 | # @lint-avoid-python-3-compatibility-imports |
Brendan Gregg | 860b649 | 2015-10-20 15:52:23 -0700 | [diff] [blame] | 3 | # |
Alexei Starovoitov | bdf0773 | 2016-01-14 10:09:20 -0800 | [diff] [blame] | 4 | # hardirqs Summarize hard IRQ (interrupt) event time. |
| 5 | # For Linux, uses BCC, eBPF. |
Brendan Gregg | 860b649 | 2015-10-20 15:52:23 -0700 | [diff] [blame] | 6 | # |
Brendan Gregg | c32b845 | 2017-11-26 23:38:32 -0800 | [diff] [blame] | 7 | # USAGE: hardirqs [-h] [-T] [-N] [-C] [-d] [interval] [outputs] |
Brendan Gregg | 860b649 | 2015-10-20 15:52:23 -0700 | [diff] [blame] | 8 | # |
| 9 | # Thanks Amer Ather for help understanding irq behavior. |
| 10 | # |
| 11 | # Copyright (c) 2015 Brendan Gregg. |
| 12 | # Licensed under the Apache License, Version 2.0 (the "License") |
| 13 | # |
Alexei Starovoitov | bdf0773 | 2016-01-14 10:09:20 -0800 | [diff] [blame] | 14 | # 19-Oct-2015 Brendan Gregg Created this. |
Brendan Gregg | 860b649 | 2015-10-20 15:52:23 -0700 | [diff] [blame] | 15 | |
| 16 | from __future__ import print_function |
| 17 | from bcc import BPF |
| 18 | from time import sleep, strftime |
| 19 | import argparse |
| 20 | |
Sasha Goldshtein | f41ae86 | 2016-10-19 01:14:30 +0300 | [diff] [blame] | 21 | # arguments |
Brendan Gregg | 860b649 | 2015-10-20 15:52:23 -0700 | [diff] [blame] | 22 | examples = """examples: |
| 23 | ./hardirqs # sum hard irq event time |
| 24 | ./hardirqs -d # show hard irq event time as histograms |
| 25 | ./hardirqs 1 10 # print 1 second summaries, 10 times |
| 26 | ./hardirqs -NT 1 # 1s summaries, nanoseconds, and timestamps |
| 27 | """ |
| 28 | parser = argparse.ArgumentParser( |
Alexei Starovoitov | bdf0773 | 2016-01-14 10:09:20 -0800 | [diff] [blame] | 29 | description="Summarize hard irq event time as histograms", |
| 30 | formatter_class=argparse.RawDescriptionHelpFormatter, |
| 31 | epilog=examples) |
Brendan Gregg | 860b649 | 2015-10-20 15:52:23 -0700 | [diff] [blame] | 32 | parser.add_argument("-T", "--timestamp", action="store_true", |
Alexei Starovoitov | bdf0773 | 2016-01-14 10:09:20 -0800 | [diff] [blame] | 33 | help="include timestamp on output") |
Brendan Gregg | 860b649 | 2015-10-20 15:52:23 -0700 | [diff] [blame] | 34 | parser.add_argument("-N", "--nanoseconds", action="store_true", |
Alexei Starovoitov | bdf0773 | 2016-01-14 10:09:20 -0800 | [diff] [blame] | 35 | help="output in nanoseconds") |
Brendan Gregg | c32b845 | 2017-11-26 23:38:32 -0800 | [diff] [blame] | 36 | parser.add_argument("-C", "--count", action="store_true", |
| 37 | help="show event counts instead of timing") |
Brendan Gregg | 860b649 | 2015-10-20 15:52:23 -0700 | [diff] [blame] | 38 | parser.add_argument("-d", "--dist", action="store_true", |
Alexei Starovoitov | bdf0773 | 2016-01-14 10:09:20 -0800 | [diff] [blame] | 39 | help="show distributions as histograms") |
Brendan Gregg | 860b649 | 2015-10-20 15:52:23 -0700 | [diff] [blame] | 40 | parser.add_argument("interval", nargs="?", default=99999999, |
Alexei Starovoitov | bdf0773 | 2016-01-14 10:09:20 -0800 | [diff] [blame] | 41 | help="output interval, in seconds") |
Brendan Gregg | c32b845 | 2017-11-26 23:38:32 -0800 | [diff] [blame] | 42 | parser.add_argument("outputs", nargs="?", default=99999999, |
Alexei Starovoitov | bdf0773 | 2016-01-14 10:09:20 -0800 | [diff] [blame] | 43 | help="number of outputs") |
Nathan Scott | cf0792f | 2018-02-02 16:56:50 +1100 | [diff] [blame] | 44 | parser.add_argument("--ebpf", action="store_true", |
| 45 | help=argparse.SUPPRESS) |
Brendan Gregg | 860b649 | 2015-10-20 15:52:23 -0700 | [diff] [blame] | 46 | args = parser.parse_args() |
Brendan Gregg | c32b845 | 2017-11-26 23:38:32 -0800 | [diff] [blame] | 47 | countdown = int(args.outputs) |
| 48 | if args.count and (args.dist or args.nanoseconds): |
| 49 | print("The --count option can't be used with time-based options") |
| 50 | exit() |
| 51 | if args.count: |
| 52 | factor = 1 |
| 53 | label = "count" |
| 54 | elif args.nanoseconds: |
Alexei Starovoitov | bdf0773 | 2016-01-14 10:09:20 -0800 | [diff] [blame] | 55 | factor = 1 |
| 56 | label = "nsecs" |
Brendan Gregg | 860b649 | 2015-10-20 15:52:23 -0700 | [diff] [blame] | 57 | else: |
Alexei Starovoitov | bdf0773 | 2016-01-14 10:09:20 -0800 | [diff] [blame] | 58 | factor = 1000 |
| 59 | label = "usecs" |
Brendan Gregg | 860b649 | 2015-10-20 15:52:23 -0700 | [diff] [blame] | 60 | debug = 0 |
| 61 | |
Sasha Goldshtein | f41ae86 | 2016-10-19 01:14:30 +0300 | [diff] [blame] | 62 | # define BPF program |
Brendan Gregg | 860b649 | 2015-10-20 15:52:23 -0700 | [diff] [blame] | 63 | bpf_text = """ |
| 64 | #include <uapi/linux/ptrace.h> |
| 65 | #include <linux/irq.h> |
| 66 | #include <linux/irqdesc.h> |
| 67 | #include <linux/interrupt.h> |
| 68 | |
| 69 | typedef struct irq_key { |
Alexei Starovoitov | bdf0773 | 2016-01-14 10:09:20 -0800 | [diff] [blame] | 70 | char name[32]; |
| 71 | u64 slot; |
Brendan Gregg | 860b649 | 2015-10-20 15:52:23 -0700 | [diff] [blame] | 72 | } irq_key_t; |
| 73 | BPF_HASH(start, u32); |
| 74 | BPF_HASH(irqdesc, u32, struct irq_desc *); |
| 75 | BPF_HISTOGRAM(dist, irq_key_t); |
| 76 | |
Brendan Gregg | c32b845 | 2017-11-26 23:38:32 -0800 | [diff] [blame] | 77 | // count IRQ |
| 78 | int count_only(struct pt_regs *ctx, struct irq_desc *desc) |
| 79 | { |
| 80 | u32 pid = bpf_get_current_pid_tgid(); |
| 81 | |
| 82 | struct irqaction *action = desc->action; |
| 83 | char *name = (char *)action->name; |
| 84 | |
| 85 | irq_key_t key = {.slot = 0 /* ignore */}; |
| 86 | bpf_probe_read(&key.name, sizeof(key.name), name); |
Javier Honduvilla Coto | 64bf965 | 2018-08-01 06:50:19 +0200 | [diff] [blame] | 87 | dist.increment(key); |
Brendan Gregg | c32b845 | 2017-11-26 23:38:32 -0800 | [diff] [blame] | 88 | |
| 89 | return 0; |
| 90 | } |
| 91 | |
Brendan Gregg | 860b649 | 2015-10-20 15:52:23 -0700 | [diff] [blame] | 92 | // time IRQ |
| 93 | int trace_start(struct pt_regs *ctx, struct irq_desc *desc) |
| 94 | { |
Alexei Starovoitov | bdf0773 | 2016-01-14 10:09:20 -0800 | [diff] [blame] | 95 | u32 pid = bpf_get_current_pid_tgid(); |
| 96 | u64 ts = bpf_ktime_get_ns(); |
| 97 | start.update(&pid, &ts); |
| 98 | irqdesc.update(&pid, &desc); |
| 99 | return 0; |
Brendan Gregg | 860b649 | 2015-10-20 15:52:23 -0700 | [diff] [blame] | 100 | } |
| 101 | |
| 102 | int trace_completion(struct pt_regs *ctx) |
| 103 | { |
Alexei Starovoitov | bdf0773 | 2016-01-14 10:09:20 -0800 | [diff] [blame] | 104 | u64 *tsp, delta; |
| 105 | struct irq_desc **descp; |
| 106 | u32 pid = bpf_get_current_pid_tgid(); |
Brendan Gregg | 860b649 | 2015-10-20 15:52:23 -0700 | [diff] [blame] | 107 | |
Alexei Starovoitov | bdf0773 | 2016-01-14 10:09:20 -0800 | [diff] [blame] | 108 | // fetch timestamp and calculate delta |
| 109 | tsp = start.lookup(&pid); |
| 110 | descp = irqdesc.lookup(&pid); |
| 111 | if (tsp == 0 || descp == 0) { |
| 112 | return 0; // missed start |
| 113 | } |
Paul Chaignon | eae0acf | 2017-08-05 23:04:41 +0200 | [diff] [blame] | 114 | struct irq_desc *desc = *descp; |
| 115 | struct irqaction *action = desc->action; |
| 116 | char *name = (char *)action->name; |
Alexei Starovoitov | bdf0773 | 2016-01-14 10:09:20 -0800 | [diff] [blame] | 117 | delta = bpf_ktime_get_ns() - *tsp; |
Brendan Gregg | 860b649 | 2015-10-20 15:52:23 -0700 | [diff] [blame] | 118 | |
Alexei Starovoitov | bdf0773 | 2016-01-14 10:09:20 -0800 | [diff] [blame] | 119 | // store as sum or histogram |
| 120 | STORE |
Brendan Gregg | 860b649 | 2015-10-20 15:52:23 -0700 | [diff] [blame] | 121 | |
Alexei Starovoitov | bdf0773 | 2016-01-14 10:09:20 -0800 | [diff] [blame] | 122 | start.delete(&pid); |
| 123 | irqdesc.delete(&pid); |
| 124 | return 0; |
Brendan Gregg | 860b649 | 2015-10-20 15:52:23 -0700 | [diff] [blame] | 125 | } |
| 126 | """ |
| 127 | |
Sasha Goldshtein | f41ae86 | 2016-10-19 01:14:30 +0300 | [diff] [blame] | 128 | # code substitutions |
Brendan Gregg | 860b649 | 2015-10-20 15:52:23 -0700 | [diff] [blame] | 129 | if args.dist: |
Alexei Starovoitov | bdf0773 | 2016-01-14 10:09:20 -0800 | [diff] [blame] | 130 | bpf_text = bpf_text.replace('STORE', |
Kirill Smelkov | f2d125e | 2017-09-25 11:23:03 +0300 | [diff] [blame] | 131 | 'irq_key_t key = {.slot = bpf_log2l(delta / %d)};' % factor + |
Alexei Starovoitov | bdf0773 | 2016-01-14 10:09:20 -0800 | [diff] [blame] | 132 | 'bpf_probe_read(&key.name, sizeof(key.name), name);' + |
| 133 | 'dist.increment(key);') |
Brendan Gregg | 860b649 | 2015-10-20 15:52:23 -0700 | [diff] [blame] | 134 | else: |
Alexei Starovoitov | bdf0773 | 2016-01-14 10:09:20 -0800 | [diff] [blame] | 135 | bpf_text = bpf_text.replace('STORE', |
| 136 | 'irq_key_t key = {.slot = 0 /* ignore */};' + |
| 137 | 'bpf_probe_read(&key.name, sizeof(key.name), name);' + |
Javier Honduvilla Coto | 64bf965 | 2018-08-01 06:50:19 +0200 | [diff] [blame] | 138 | 'dist.increment(key, delta);') |
Nathan Scott | cf0792f | 2018-02-02 16:56:50 +1100 | [diff] [blame] | 139 | if debug or args.ebpf: |
Alexei Starovoitov | bdf0773 | 2016-01-14 10:09:20 -0800 | [diff] [blame] | 140 | print(bpf_text) |
Nathan Scott | cf0792f | 2018-02-02 16:56:50 +1100 | [diff] [blame] | 141 | if args.ebpf: |
| 142 | exit() |
Brendan Gregg | 860b649 | 2015-10-20 15:52:23 -0700 | [diff] [blame] | 143 | |
Sasha Goldshtein | f41ae86 | 2016-10-19 01:14:30 +0300 | [diff] [blame] | 144 | # load BPF program |
Brendan Gregg | 860b649 | 2015-10-20 15:52:23 -0700 | [diff] [blame] | 145 | b = BPF(text=bpf_text) |
| 146 | |
| 147 | # these should really use irq:irq_handler_entry/exit tracepoints: |
Brendan Gregg | c32b845 | 2017-11-26 23:38:32 -0800 | [diff] [blame] | 148 | if args.count: |
| 149 | b.attach_kprobe(event="handle_irq_event_percpu", fn_name="count_only") |
| 150 | print("Tracing hard irq events... Hit Ctrl-C to end.") |
| 151 | else: |
| 152 | b.attach_kprobe(event="handle_irq_event_percpu", fn_name="trace_start") |
| 153 | b.attach_kretprobe(event="handle_irq_event_percpu", |
| 154 | fn_name="trace_completion") |
| 155 | print("Tracing hard irq event time... Hit Ctrl-C to end.") |
Brendan Gregg | 860b649 | 2015-10-20 15:52:23 -0700 | [diff] [blame] | 156 | |
Sasha Goldshtein | f41ae86 | 2016-10-19 01:14:30 +0300 | [diff] [blame] | 157 | # output |
Brendan Gregg | 860b649 | 2015-10-20 15:52:23 -0700 | [diff] [blame] | 158 | exiting = 0 if args.interval else 1 |
| 159 | dist = b.get_table("dist") |
| 160 | while (1): |
Alexei Starovoitov | bdf0773 | 2016-01-14 10:09:20 -0800 | [diff] [blame] | 161 | try: |
| 162 | sleep(int(args.interval)) |
| 163 | except KeyboardInterrupt: |
| 164 | exiting = 1 |
Brendan Gregg | 860b649 | 2015-10-20 15:52:23 -0700 | [diff] [blame] | 165 | |
Alexei Starovoitov | bdf0773 | 2016-01-14 10:09:20 -0800 | [diff] [blame] | 166 | print() |
| 167 | if args.timestamp: |
| 168 | print("%-8s\n" % strftime("%H:%M:%S"), end="") |
Brendan Gregg | 860b649 | 2015-10-20 15:52:23 -0700 | [diff] [blame] | 169 | |
Alexei Starovoitov | bdf0773 | 2016-01-14 10:09:20 -0800 | [diff] [blame] | 170 | if args.dist: |
| 171 | dist.print_log2_hist(label, "hardirq") |
| 172 | else: |
| 173 | print("%-26s %11s" % ("HARDIRQ", "TOTAL_" + label)) |
| 174 | for k, v in sorted(dist.items(), key=lambda dist: dist[1].value): |
Rafael F | 78948e4 | 2017-03-26 14:54:25 +0200 | [diff] [blame] | 175 | print("%-26s %11d" % (k.name.decode(), v.value / factor)) |
Alexei Starovoitov | bdf0773 | 2016-01-14 10:09:20 -0800 | [diff] [blame] | 176 | dist.clear() |
Brendan Gregg | 860b649 | 2015-10-20 15:52:23 -0700 | [diff] [blame] | 177 | |
Alexei Starovoitov | bdf0773 | 2016-01-14 10:09:20 -0800 | [diff] [blame] | 178 | countdown -= 1 |
| 179 | if exiting or countdown == 0: |
| 180 | exit() |