Steven Rostedt | 81d68a9 | 2008-05-12 21:20:42 +0200 | [diff] [blame] | 1 | /* |
Wenji Huang | 73d8b8b | 2009-02-17 01:10:02 -0500 | [diff] [blame] | 2 | * trace irqs off critical timings |
Steven Rostedt | 81d68a9 | 2008-05-12 21:20:42 +0200 | [diff] [blame] | 3 | * |
| 4 | * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com> |
| 5 | * Copyright (C) 2008 Ingo Molnar <mingo@redhat.com> |
| 6 | * |
| 7 | * From code in the latency_tracer, that is: |
| 8 | * |
| 9 | * Copyright (C) 2004-2006 Ingo Molnar |
Nadia Yvette Chambers | 6d49e35 | 2012-12-06 10:39:54 +0100 | [diff] [blame] | 10 | * Copyright (C) 2004 Nadia Yvette Chambers |
Steven Rostedt | 81d68a9 | 2008-05-12 21:20:42 +0200 | [diff] [blame] | 11 | */ |
| 12 | #include <linux/kallsyms.h> |
Steven Rostedt | 81d68a9 | 2008-05-12 21:20:42 +0200 | [diff] [blame] | 13 | #include <linux/uaccess.h> |
| 14 | #include <linux/module.h> |
| 15 | #include <linux/ftrace.h> |
Steven Rostedt | 81d68a9 | 2008-05-12 21:20:42 +0200 | [diff] [blame] | 16 | |
| 17 | #include "trace.h" |
| 18 | |
| 19 | static struct trace_array *irqsoff_trace __read_mostly; |
| 20 | static int tracer_enabled __read_mostly; |
| 21 | |
Steven Rostedt | 6cd8a4b | 2008-05-12 21:20:42 +0200 | [diff] [blame] | 22 | static DEFINE_PER_CPU(int, tracing_cpu); |
| 23 | |
Thomas Gleixner | 5389f6f | 2009-07-25 17:13:33 +0200 | [diff] [blame] | 24 | static DEFINE_RAW_SPINLOCK(max_trace_lock); |
Steven Rostedt | 89b2f97 | 2008-05-12 21:20:44 +0200 | [diff] [blame] | 25 | |
Steven Rostedt | 6cd8a4b | 2008-05-12 21:20:42 +0200 | [diff] [blame] | 26 | enum { |
| 27 | TRACER_IRQS_OFF = (1 << 1), |
| 28 | TRACER_PREEMPT_OFF = (1 << 2), |
| 29 | }; |
| 30 | |
| 31 | static int trace_type __read_mostly; |
| 32 | |
Steven Rostedt (Red Hat) | 613f04a | 2013-03-14 15:03:53 -0400 | [diff] [blame] | 33 | static int save_flags; |
Steven Rostedt | e9d25fe | 2009-03-04 22:15:30 -0500 | [diff] [blame] | 34 | |
Jiri Olsa | 62b915f | 2010-04-02 19:01:22 +0200 | [diff] [blame] | 35 | static void stop_irqsoff_tracer(struct trace_array *tr, int graph); |
| 36 | static int start_irqsoff_tracer(struct trace_array *tr, int graph); |
| 37 | |
Steven Rostedt | 6cd8a4b | 2008-05-12 21:20:42 +0200 | [diff] [blame] | 38 | #ifdef CONFIG_PREEMPT_TRACER |
Ingo Molnar | e309b41 | 2008-05-12 21:20:51 +0200 | [diff] [blame] | 39 | static inline int |
Steven Rostedt | 6cd8a4b | 2008-05-12 21:20:42 +0200 | [diff] [blame] | 40 | preempt_trace(void) |
| 41 | { |
| 42 | return ((trace_type & TRACER_PREEMPT_OFF) && preempt_count()); |
| 43 | } |
| 44 | #else |
| 45 | # define preempt_trace() (0) |
| 46 | #endif |
| 47 | |
| 48 | #ifdef CONFIG_IRQSOFF_TRACER |
Ingo Molnar | e309b41 | 2008-05-12 21:20:51 +0200 | [diff] [blame] | 49 | static inline int |
Steven Rostedt | 6cd8a4b | 2008-05-12 21:20:42 +0200 | [diff] [blame] | 50 | irq_trace(void) |
| 51 | { |
| 52 | return ((trace_type & TRACER_IRQS_OFF) && |
| 53 | irqs_disabled()); |
| 54 | } |
| 55 | #else |
| 56 | # define irq_trace() (0) |
| 57 | #endif |
| 58 | |
Jiri Olsa | 62b915f | 2010-04-02 19:01:22 +0200 | [diff] [blame] | 59 | #ifdef CONFIG_FUNCTION_GRAPH_TRACER |
Steven Rostedt (Red Hat) | 0390558 | 2015-09-28 15:37:49 -0400 | [diff] [blame] | 60 | static int irqsoff_display_graph(struct trace_array *tr, int set); |
Steven Rostedt (Red Hat) | 983f938 | 2015-09-30 09:42:05 -0400 | [diff] [blame] | 61 | # define is_graph(tr) ((tr)->trace_flags & TRACE_ITER_DISPLAY_GRAPH) |
Steven Rostedt (Red Hat) | 0390558 | 2015-09-28 15:37:49 -0400 | [diff] [blame] | 62 | #else |
| 63 | static inline int irqsoff_display_graph(struct trace_array *tr, int set) |
| 64 | { |
| 65 | return -EINVAL; |
| 66 | } |
Steven Rostedt (Red Hat) | 983f938 | 2015-09-30 09:42:05 -0400 | [diff] [blame] | 67 | # define is_graph(tr) false |
Jiri Olsa | 62b915f | 2010-04-02 19:01:22 +0200 | [diff] [blame] | 68 | #endif |
Jiri Olsa | 62b915f | 2010-04-02 19:01:22 +0200 | [diff] [blame] | 69 | |
Steven Rostedt | 81d68a9 | 2008-05-12 21:20:42 +0200 | [diff] [blame] | 70 | /* |
| 71 | * Sequence count - we record it when starting a measurement and |
| 72 | * skip the latency if the sequence has changed - some other section |
| 73 | * did a maximum and could disturb our measurement with serial console |
| 74 | * printouts, etc. Truly coinciding maximum latencies should be rare |
Lucas De Marchi | 25985ed | 2011-03-30 22:57:33 -0300 | [diff] [blame] | 75 | * and what happens together happens separately as well, so this doesn't |
Steven Rostedt | 81d68a9 | 2008-05-12 21:20:42 +0200 | [diff] [blame] | 76 | * decrease the validity of the maximum found: |
| 77 | */ |
| 78 | static __cacheline_aligned_in_smp unsigned long max_sequence; |
| 79 | |
Steven Rostedt | 606576c | 2008-10-06 19:06:12 -0400 | [diff] [blame] | 80 | #ifdef CONFIG_FUNCTION_TRACER |
Steven Rostedt | 81d68a9 | 2008-05-12 21:20:42 +0200 | [diff] [blame] | 81 | /* |
Steven Rostedt | 5e6d2b9 | 2010-10-05 19:41:43 -0400 | [diff] [blame] | 82 | * Prologue for the preempt and irqs off function tracers. |
| 83 | * |
| 84 | * Returns 1 if it is OK to continue, and data->disabled is |
| 85 | * incremented. |
| 86 | * 0 if the trace is to be ignored, and data->disabled |
| 87 | * is kept the same. |
| 88 | * |
| 89 | * Note, this function is also used outside this ifdef but |
| 90 | * inside the #ifdef of the function graph tracer below. |
| 91 | * This is OK, since the function graph tracer is |
| 92 | * dependent on the function tracer. |
Steven Rostedt | 81d68a9 | 2008-05-12 21:20:42 +0200 | [diff] [blame] | 93 | */ |
Steven Rostedt | 5e6d2b9 | 2010-10-05 19:41:43 -0400 | [diff] [blame] | 94 | static int func_prolog_dec(struct trace_array *tr, |
| 95 | struct trace_array_cpu **data, |
| 96 | unsigned long *flags) |
Steven Rostedt | 81d68a9 | 2008-05-12 21:20:42 +0200 | [diff] [blame] | 97 | { |
Steven Rostedt | 81d68a9 | 2008-05-12 21:20:42 +0200 | [diff] [blame] | 98 | long disabled; |
| 99 | int cpu; |
| 100 | |
Steven Rostedt | 361943a | 2008-05-12 21:20:44 +0200 | [diff] [blame] | 101 | /* |
| 102 | * Does not matter if we preempt. We test the flags |
| 103 | * afterward, to see if irqs are disabled or not. |
| 104 | * If we preempt and get a false positive, the flags |
| 105 | * test will fail. |
| 106 | */ |
| 107 | cpu = raw_smp_processor_id(); |
| 108 | if (likely(!per_cpu(tracing_cpu, cpu))) |
Steven Rostedt | 5e6d2b9 | 2010-10-05 19:41:43 -0400 | [diff] [blame] | 109 | return 0; |
Steven Rostedt | 81d68a9 | 2008-05-12 21:20:42 +0200 | [diff] [blame] | 110 | |
Steven Rostedt | 5e6d2b9 | 2010-10-05 19:41:43 -0400 | [diff] [blame] | 111 | local_save_flags(*flags); |
Steven Rostedt (Red Hat) | cb86e05 | 2016-03-18 12:27:43 -0400 | [diff] [blame] | 112 | /* |
| 113 | * Slight chance to get a false positive on tracing_cpu, |
| 114 | * although I'm starting to think there isn't a chance. |
| 115 | * Leave this for now just to be paranoid. |
| 116 | */ |
| 117 | if (!irqs_disabled_flags(*flags) && !preempt_count()) |
Steven Rostedt | 5e6d2b9 | 2010-10-05 19:41:43 -0400 | [diff] [blame] | 118 | return 0; |
Steven Rostedt | 81d68a9 | 2008-05-12 21:20:42 +0200 | [diff] [blame] | 119 | |
Steven Rostedt (Red Hat) | 12883ef | 2013-03-05 09:24:35 -0500 | [diff] [blame] | 120 | *data = per_cpu_ptr(tr->trace_buffer.data, cpu); |
Steven Rostedt | 5e6d2b9 | 2010-10-05 19:41:43 -0400 | [diff] [blame] | 121 | disabled = atomic_inc_return(&(*data)->disabled); |
Steven Rostedt | 81d68a9 | 2008-05-12 21:20:42 +0200 | [diff] [blame] | 122 | |
| 123 | if (likely(disabled == 1)) |
Steven Rostedt | 5e6d2b9 | 2010-10-05 19:41:43 -0400 | [diff] [blame] | 124 | return 1; |
| 125 | |
| 126 | atomic_dec(&(*data)->disabled); |
| 127 | |
| 128 | return 0; |
| 129 | } |
| 130 | |
| 131 | /* |
| 132 | * irqsoff uses its own tracer function to keep the overhead down: |
| 133 | */ |
| 134 | static void |
Steven Rostedt | 2f5f6ad | 2011-08-08 16:57:47 -0400 | [diff] [blame] | 135 | irqsoff_tracer_call(unsigned long ip, unsigned long parent_ip, |
Steven Rostedt | a1e2e31 | 2011-08-09 12:50:46 -0400 | [diff] [blame] | 136 | struct ftrace_ops *op, struct pt_regs *pt_regs) |
Steven Rostedt | 5e6d2b9 | 2010-10-05 19:41:43 -0400 | [diff] [blame] | 137 | { |
| 138 | struct trace_array *tr = irqsoff_trace; |
| 139 | struct trace_array_cpu *data; |
| 140 | unsigned long flags; |
| 141 | |
| 142 | if (!func_prolog_dec(tr, &data, &flags)) |
| 143 | return; |
| 144 | |
| 145 | trace_function(tr, ip, parent_ip, flags, preempt_count()); |
Steven Rostedt | 81d68a9 | 2008-05-12 21:20:42 +0200 | [diff] [blame] | 146 | |
| 147 | atomic_dec(&data->disabled); |
| 148 | } |
Steven Rostedt | 606576c | 2008-10-06 19:06:12 -0400 | [diff] [blame] | 149 | #endif /* CONFIG_FUNCTION_TRACER */ |
Steven Rostedt | 81d68a9 | 2008-05-12 21:20:42 +0200 | [diff] [blame] | 150 | |
Jiri Olsa | 62b915f | 2010-04-02 19:01:22 +0200 | [diff] [blame] | 151 | #ifdef CONFIG_FUNCTION_GRAPH_TRACER |
Steven Rostedt (Red Hat) | 0390558 | 2015-09-28 15:37:49 -0400 | [diff] [blame] | 152 | static int irqsoff_display_graph(struct trace_array *tr, int set) |
Jiri Olsa | 62b915f | 2010-04-02 19:01:22 +0200 | [diff] [blame] | 153 | { |
| 154 | int cpu; |
| 155 | |
Steven Rostedt (Red Hat) | 983f938 | 2015-09-30 09:42:05 -0400 | [diff] [blame] | 156 | if (!(is_graph(tr) ^ set)) |
Jiri Olsa | 62b915f | 2010-04-02 19:01:22 +0200 | [diff] [blame] | 157 | return 0; |
| 158 | |
| 159 | stop_irqsoff_tracer(irqsoff_trace, !set); |
| 160 | |
| 161 | for_each_possible_cpu(cpu) |
| 162 | per_cpu(tracing_cpu, cpu) = 0; |
| 163 | |
Steven Rostedt (Red Hat) | 6d9b3fa | 2014-01-14 11:28:38 -0500 | [diff] [blame] | 164 | tr->max_latency = 0; |
Steven Rostedt (Red Hat) | 12883ef | 2013-03-05 09:24:35 -0500 | [diff] [blame] | 165 | tracing_reset_online_cpus(&irqsoff_trace->trace_buffer); |
Jiri Olsa | 62b915f | 2010-04-02 19:01:22 +0200 | [diff] [blame] | 166 | |
| 167 | return start_irqsoff_tracer(irqsoff_trace, set); |
| 168 | } |
| 169 | |
| 170 | static int irqsoff_graph_entry(struct ftrace_graph_ent *trace) |
| 171 | { |
| 172 | struct trace_array *tr = irqsoff_trace; |
| 173 | struct trace_array_cpu *data; |
| 174 | unsigned long flags; |
Jiri Olsa | 62b915f | 2010-04-02 19:01:22 +0200 | [diff] [blame] | 175 | int ret; |
Jiri Olsa | 62b915f | 2010-04-02 19:01:22 +0200 | [diff] [blame] | 176 | int pc; |
| 177 | |
Steven Rostedt | 5e6d2b9 | 2010-10-05 19:41:43 -0400 | [diff] [blame] | 178 | if (!func_prolog_dec(tr, &data, &flags)) |
Jiri Olsa | 62b915f | 2010-04-02 19:01:22 +0200 | [diff] [blame] | 179 | return 0; |
| 180 | |
Steven Rostedt | 5e6d2b9 | 2010-10-05 19:41:43 -0400 | [diff] [blame] | 181 | pc = preempt_count(); |
| 182 | ret = __trace_graph_entry(tr, trace, flags, pc); |
Jiri Olsa | 62b915f | 2010-04-02 19:01:22 +0200 | [diff] [blame] | 183 | atomic_dec(&data->disabled); |
Steven Rostedt | 5e6d2b9 | 2010-10-05 19:41:43 -0400 | [diff] [blame] | 184 | |
Jiri Olsa | 62b915f | 2010-04-02 19:01:22 +0200 | [diff] [blame] | 185 | return ret; |
| 186 | } |
| 187 | |
| 188 | static void irqsoff_graph_return(struct ftrace_graph_ret *trace) |
| 189 | { |
| 190 | struct trace_array *tr = irqsoff_trace; |
| 191 | struct trace_array_cpu *data; |
| 192 | unsigned long flags; |
Jiri Olsa | 62b915f | 2010-04-02 19:01:22 +0200 | [diff] [blame] | 193 | int pc; |
| 194 | |
Steven Rostedt | 5e6d2b9 | 2010-10-05 19:41:43 -0400 | [diff] [blame] | 195 | if (!func_prolog_dec(tr, &data, &flags)) |
Jiri Olsa | 62b915f | 2010-04-02 19:01:22 +0200 | [diff] [blame] | 196 | return; |
| 197 | |
Steven Rostedt | 5e6d2b9 | 2010-10-05 19:41:43 -0400 | [diff] [blame] | 198 | pc = preempt_count(); |
| 199 | __trace_graph_return(tr, trace, flags, pc); |
Jiri Olsa | 62b915f | 2010-04-02 19:01:22 +0200 | [diff] [blame] | 200 | atomic_dec(&data->disabled); |
| 201 | } |
| 202 | |
| 203 | static void irqsoff_trace_open(struct trace_iterator *iter) |
| 204 | { |
Steven Rostedt (Red Hat) | 983f938 | 2015-09-30 09:42:05 -0400 | [diff] [blame] | 205 | if (is_graph(iter->tr)) |
Jiri Olsa | 62b915f | 2010-04-02 19:01:22 +0200 | [diff] [blame] | 206 | graph_trace_open(iter); |
| 207 | |
| 208 | } |
| 209 | |
| 210 | static void irqsoff_trace_close(struct trace_iterator *iter) |
| 211 | { |
| 212 | if (iter->private) |
| 213 | graph_trace_close(iter); |
| 214 | } |
| 215 | |
| 216 | #define GRAPH_TRACER_FLAGS (TRACE_GRAPH_PRINT_CPU | \ |
Jiri Olsa | 321e68b | 2011-06-03 16:58:47 +0200 | [diff] [blame] | 217 | TRACE_GRAPH_PRINT_PROC | \ |
| 218 | TRACE_GRAPH_PRINT_ABS_TIME | \ |
| 219 | TRACE_GRAPH_PRINT_DURATION) |
Jiri Olsa | 62b915f | 2010-04-02 19:01:22 +0200 | [diff] [blame] | 220 | |
| 221 | static enum print_line_t irqsoff_print_line(struct trace_iterator *iter) |
| 222 | { |
Jiri Olsa | 62b915f | 2010-04-02 19:01:22 +0200 | [diff] [blame] | 223 | /* |
| 224 | * In graph mode call the graph tracer output function, |
| 225 | * otherwise go with the TRACE_FN event handler |
| 226 | */ |
Steven Rostedt (Red Hat) | 983f938 | 2015-09-30 09:42:05 -0400 | [diff] [blame] | 227 | if (is_graph(iter->tr)) |
Jiri Olsa | 0a77262 | 2010-09-23 14:00:52 +0200 | [diff] [blame] | 228 | return print_graph_function_flags(iter, GRAPH_TRACER_FLAGS); |
Jiri Olsa | 62b915f | 2010-04-02 19:01:22 +0200 | [diff] [blame] | 229 | |
| 230 | return TRACE_TYPE_UNHANDLED; |
| 231 | } |
| 232 | |
| 233 | static void irqsoff_print_header(struct seq_file *s) |
| 234 | { |
Steven Rostedt (Red Hat) | 983f938 | 2015-09-30 09:42:05 -0400 | [diff] [blame] | 235 | struct trace_array *tr = irqsoff_trace; |
| 236 | |
| 237 | if (is_graph(tr)) |
Jiri Olsa | 0a77262 | 2010-09-23 14:00:52 +0200 | [diff] [blame] | 238 | print_graph_headers_flags(s, GRAPH_TRACER_FLAGS); |
| 239 | else |
Jiri Olsa | 62b915f | 2010-04-02 19:01:22 +0200 | [diff] [blame] | 240 | trace_default_header(s); |
| 241 | } |
| 242 | |
| 243 | static void |
Jiri Olsa | 62b915f | 2010-04-02 19:01:22 +0200 | [diff] [blame] | 244 | __trace_function(struct trace_array *tr, |
| 245 | unsigned long ip, unsigned long parent_ip, |
| 246 | unsigned long flags, int pc) |
| 247 | { |
Steven Rostedt (Red Hat) | 983f938 | 2015-09-30 09:42:05 -0400 | [diff] [blame] | 248 | if (is_graph(tr)) |
Jiri Olsa | 0a77262 | 2010-09-23 14:00:52 +0200 | [diff] [blame] | 249 | trace_graph_function(tr, ip, parent_ip, flags, pc); |
| 250 | else |
Jiri Olsa | 62b915f | 2010-04-02 19:01:22 +0200 | [diff] [blame] | 251 | trace_function(tr, ip, parent_ip, flags, pc); |
Jiri Olsa | 62b915f | 2010-04-02 19:01:22 +0200 | [diff] [blame] | 252 | } |
| 253 | |
| 254 | #else |
| 255 | #define __trace_function trace_function |
| 256 | |
Steven Rostedt (Red Hat) | 8179e8a | 2015-09-29 10:24:56 -0400 | [diff] [blame] | 257 | #ifdef CONFIG_FUNCTION_TRACER |
Jiri Olsa | 62b915f | 2010-04-02 19:01:22 +0200 | [diff] [blame] | 258 | static int irqsoff_graph_entry(struct ftrace_graph_ent *trace) |
| 259 | { |
| 260 | return -1; |
| 261 | } |
Steven Rostedt (Red Hat) | 8179e8a | 2015-09-29 10:24:56 -0400 | [diff] [blame] | 262 | #endif |
Jiri Olsa | 62b915f | 2010-04-02 19:01:22 +0200 | [diff] [blame] | 263 | |
| 264 | static enum print_line_t irqsoff_print_line(struct trace_iterator *iter) |
| 265 | { |
| 266 | return TRACE_TYPE_UNHANDLED; |
| 267 | } |
| 268 | |
Jiri Olsa | 62b915f | 2010-04-02 19:01:22 +0200 | [diff] [blame] | 269 | static void irqsoff_trace_open(struct trace_iterator *iter) { } |
| 270 | static void irqsoff_trace_close(struct trace_iterator *iter) { } |
Jiri Olsa | 7e9a49e | 2011-11-07 16:08:49 +0100 | [diff] [blame] | 271 | |
| 272 | #ifdef CONFIG_FUNCTION_TRACER |
Steven Rostedt (Red Hat) | 8179e8a | 2015-09-29 10:24:56 -0400 | [diff] [blame] | 273 | static void irqsoff_graph_return(struct ftrace_graph_ret *trace) { } |
Jiri Olsa | 7e9a49e | 2011-11-07 16:08:49 +0100 | [diff] [blame] | 274 | static void irqsoff_print_header(struct seq_file *s) |
| 275 | { |
| 276 | trace_default_header(s); |
| 277 | } |
| 278 | #else |
| 279 | static void irqsoff_print_header(struct seq_file *s) |
| 280 | { |
| 281 | trace_latency_header(s); |
| 282 | } |
| 283 | #endif /* CONFIG_FUNCTION_TRACER */ |
Jiri Olsa | 62b915f | 2010-04-02 19:01:22 +0200 | [diff] [blame] | 284 | #endif /* CONFIG_FUNCTION_GRAPH_TRACER */ |
| 285 | |
Steven Rostedt | 81d68a9 | 2008-05-12 21:20:42 +0200 | [diff] [blame] | 286 | /* |
| 287 | * Should this new latency be reported/recorded? |
| 288 | */ |
Yaowei Bai | 7985182 | 2015-09-29 22:43:30 +0800 | [diff] [blame] | 289 | static bool report_latency(struct trace_array *tr, cycle_t delta) |
Steven Rostedt | 81d68a9 | 2008-05-12 21:20:42 +0200 | [diff] [blame] | 290 | { |
| 291 | if (tracing_thresh) { |
| 292 | if (delta < tracing_thresh) |
Yaowei Bai | 7985182 | 2015-09-29 22:43:30 +0800 | [diff] [blame] | 293 | return false; |
Steven Rostedt | 81d68a9 | 2008-05-12 21:20:42 +0200 | [diff] [blame] | 294 | } else { |
Steven Rostedt (Red Hat) | 6d9b3fa | 2014-01-14 11:28:38 -0500 | [diff] [blame] | 295 | if (delta <= tr->max_latency) |
Yaowei Bai | 7985182 | 2015-09-29 22:43:30 +0800 | [diff] [blame] | 296 | return false; |
Steven Rostedt | 81d68a9 | 2008-05-12 21:20:42 +0200 | [diff] [blame] | 297 | } |
Yaowei Bai | 7985182 | 2015-09-29 22:43:30 +0800 | [diff] [blame] | 298 | return true; |
Steven Rostedt | 81d68a9 | 2008-05-12 21:20:42 +0200 | [diff] [blame] | 299 | } |
| 300 | |
Ingo Molnar | e309b41 | 2008-05-12 21:20:51 +0200 | [diff] [blame] | 301 | static void |
Steven Rostedt | 81d68a9 | 2008-05-12 21:20:42 +0200 | [diff] [blame] | 302 | check_critical_timing(struct trace_array *tr, |
| 303 | struct trace_array_cpu *data, |
| 304 | unsigned long parent_ip, |
| 305 | int cpu) |
| 306 | { |
Steven Rostedt | 89b2f97 | 2008-05-12 21:20:44 +0200 | [diff] [blame] | 307 | cycle_t T0, T1, delta; |
Steven Rostedt | 81d68a9 | 2008-05-12 21:20:42 +0200 | [diff] [blame] | 308 | unsigned long flags; |
Steven Rostedt | 3869705 | 2008-10-01 13:14:09 -0400 | [diff] [blame] | 309 | int pc; |
Steven Rostedt | 81d68a9 | 2008-05-12 21:20:42 +0200 | [diff] [blame] | 310 | |
Steven Rostedt | 81d68a9 | 2008-05-12 21:20:42 +0200 | [diff] [blame] | 311 | T0 = data->preempt_timestamp; |
Ingo Molnar | 750ed1a | 2008-05-12 21:20:46 +0200 | [diff] [blame] | 312 | T1 = ftrace_now(cpu); |
Steven Rostedt | 81d68a9 | 2008-05-12 21:20:42 +0200 | [diff] [blame] | 313 | delta = T1-T0; |
| 314 | |
| 315 | local_save_flags(flags); |
| 316 | |
Steven Rostedt | 6450c1d | 2008-10-02 19:23:04 -0400 | [diff] [blame] | 317 | pc = preempt_count(); |
| 318 | |
Steven Rostedt (Red Hat) | 6d9b3fa | 2014-01-14 11:28:38 -0500 | [diff] [blame] | 319 | if (!report_latency(tr, delta)) |
Steven Rostedt | 81d68a9 | 2008-05-12 21:20:42 +0200 | [diff] [blame] | 320 | goto out; |
| 321 | |
Thomas Gleixner | 5389f6f | 2009-07-25 17:13:33 +0200 | [diff] [blame] | 322 | raw_spin_lock_irqsave(&max_trace_lock, flags); |
Steven Rostedt | 81d68a9 | 2008-05-12 21:20:42 +0200 | [diff] [blame] | 323 | |
Steven Rostedt | 89b2f97 | 2008-05-12 21:20:44 +0200 | [diff] [blame] | 324 | /* check if we are still the max latency */ |
Steven Rostedt (Red Hat) | 6d9b3fa | 2014-01-14 11:28:38 -0500 | [diff] [blame] | 325 | if (!report_latency(tr, delta)) |
Steven Rostedt | 89b2f97 | 2008-05-12 21:20:44 +0200 | [diff] [blame] | 326 | goto out_unlock; |
| 327 | |
Jiri Olsa | 62b915f | 2010-04-02 19:01:22 +0200 | [diff] [blame] | 328 | __trace_function(tr, CALLER_ADDR0, parent_ip, flags, pc); |
Steven Rostedt | cc51a0f | 2009-12-11 11:54:51 -0500 | [diff] [blame] | 329 | /* Skip 5 functions to get to the irq/preempt enable function */ |
| 330 | __trace_stack(tr, flags, 5, pc); |
Steven Rostedt | 81d68a9 | 2008-05-12 21:20:42 +0200 | [diff] [blame] | 331 | |
Steven Rostedt | 81d68a9 | 2008-05-12 21:20:42 +0200 | [diff] [blame] | 332 | if (data->critical_sequence != max_sequence) |
Steven Rostedt | 89b2f97 | 2008-05-12 21:20:44 +0200 | [diff] [blame] | 333 | goto out_unlock; |
Steven Rostedt | 81d68a9 | 2008-05-12 21:20:42 +0200 | [diff] [blame] | 334 | |
Steven Rostedt | 81d68a9 | 2008-05-12 21:20:42 +0200 | [diff] [blame] | 335 | data->critical_end = parent_ip; |
| 336 | |
Carsten Emde | b5130b1 | 2009-09-13 01:43:07 +0200 | [diff] [blame] | 337 | if (likely(!is_tracing_stopped())) { |
Steven Rostedt (Red Hat) | 6d9b3fa | 2014-01-14 11:28:38 -0500 | [diff] [blame] | 338 | tr->max_latency = delta; |
Carsten Emde | b5130b1 | 2009-09-13 01:43:07 +0200 | [diff] [blame] | 339 | update_max_tr_single(tr, current, cpu); |
| 340 | } |
Steven Rostedt | 81d68a9 | 2008-05-12 21:20:42 +0200 | [diff] [blame] | 341 | |
Steven Rostedt | 81d68a9 | 2008-05-12 21:20:42 +0200 | [diff] [blame] | 342 | max_sequence++; |
| 343 | |
Steven Rostedt | 89b2f97 | 2008-05-12 21:20:44 +0200 | [diff] [blame] | 344 | out_unlock: |
Thomas Gleixner | 5389f6f | 2009-07-25 17:13:33 +0200 | [diff] [blame] | 345 | raw_spin_unlock_irqrestore(&max_trace_lock, flags); |
Steven Rostedt | 89b2f97 | 2008-05-12 21:20:44 +0200 | [diff] [blame] | 346 | |
Steven Rostedt | 81d68a9 | 2008-05-12 21:20:42 +0200 | [diff] [blame] | 347 | out: |
| 348 | data->critical_sequence = max_sequence; |
Ingo Molnar | 750ed1a | 2008-05-12 21:20:46 +0200 | [diff] [blame] | 349 | data->preempt_timestamp = ftrace_now(cpu); |
Jiri Olsa | 62b915f | 2010-04-02 19:01:22 +0200 | [diff] [blame] | 350 | __trace_function(tr, CALLER_ADDR0, parent_ip, flags, pc); |
Steven Rostedt | 81d68a9 | 2008-05-12 21:20:42 +0200 | [diff] [blame] | 351 | } |
| 352 | |
Ingo Molnar | e309b41 | 2008-05-12 21:20:51 +0200 | [diff] [blame] | 353 | static inline void |
Steven Rostedt | 81d68a9 | 2008-05-12 21:20:42 +0200 | [diff] [blame] | 354 | start_critical_timing(unsigned long ip, unsigned long parent_ip) |
| 355 | { |
| 356 | int cpu; |
| 357 | struct trace_array *tr = irqsoff_trace; |
| 358 | struct trace_array_cpu *data; |
| 359 | unsigned long flags; |
| 360 | |
Steven Rostedt (Red Hat) | 10246fa | 2013-07-01 15:58:24 -0400 | [diff] [blame] | 361 | if (!tracer_enabled || !tracing_is_enabled()) |
Steven Rostedt | 81d68a9 | 2008-05-12 21:20:42 +0200 | [diff] [blame] | 362 | return; |
| 363 | |
Steven Rostedt | c5f888c | 2008-05-12 21:20:55 +0200 | [diff] [blame] | 364 | cpu = raw_smp_processor_id(); |
| 365 | |
| 366 | if (per_cpu(tracing_cpu, cpu)) |
Steven Rostedt | 6cd8a4b | 2008-05-12 21:20:42 +0200 | [diff] [blame] | 367 | return; |
| 368 | |
Steven Rostedt (Red Hat) | 12883ef | 2013-03-05 09:24:35 -0500 | [diff] [blame] | 369 | data = per_cpu_ptr(tr->trace_buffer.data, cpu); |
Steven Rostedt | 81d68a9 | 2008-05-12 21:20:42 +0200 | [diff] [blame] | 370 | |
Steven Rostedt | c5f888c | 2008-05-12 21:20:55 +0200 | [diff] [blame] | 371 | if (unlikely(!data) || atomic_read(&data->disabled)) |
Steven Rostedt | 81d68a9 | 2008-05-12 21:20:42 +0200 | [diff] [blame] | 372 | return; |
| 373 | |
| 374 | atomic_inc(&data->disabled); |
| 375 | |
| 376 | data->critical_sequence = max_sequence; |
Ingo Molnar | 750ed1a | 2008-05-12 21:20:46 +0200 | [diff] [blame] | 377 | data->preempt_timestamp = ftrace_now(cpu); |
Steven Rostedt | 6cd8a4b | 2008-05-12 21:20:42 +0200 | [diff] [blame] | 378 | data->critical_start = parent_ip ? : ip; |
Steven Rostedt | 81d68a9 | 2008-05-12 21:20:42 +0200 | [diff] [blame] | 379 | |
| 380 | local_save_flags(flags); |
Steven Rostedt | 6cd8a4b | 2008-05-12 21:20:42 +0200 | [diff] [blame] | 381 | |
Jiri Olsa | 62b915f | 2010-04-02 19:01:22 +0200 | [diff] [blame] | 382 | __trace_function(tr, ip, parent_ip, flags, preempt_count()); |
Steven Rostedt | 81d68a9 | 2008-05-12 21:20:42 +0200 | [diff] [blame] | 383 | |
Steven Rostedt | c5f888c | 2008-05-12 21:20:55 +0200 | [diff] [blame] | 384 | per_cpu(tracing_cpu, cpu) = 1; |
Steven Rostedt | 6cd8a4b | 2008-05-12 21:20:42 +0200 | [diff] [blame] | 385 | |
Steven Rostedt | 81d68a9 | 2008-05-12 21:20:42 +0200 | [diff] [blame] | 386 | atomic_dec(&data->disabled); |
| 387 | } |
| 388 | |
Ingo Molnar | e309b41 | 2008-05-12 21:20:51 +0200 | [diff] [blame] | 389 | static inline void |
Steven Rostedt | 81d68a9 | 2008-05-12 21:20:42 +0200 | [diff] [blame] | 390 | stop_critical_timing(unsigned long ip, unsigned long parent_ip) |
| 391 | { |
| 392 | int cpu; |
| 393 | struct trace_array *tr = irqsoff_trace; |
| 394 | struct trace_array_cpu *data; |
| 395 | unsigned long flags; |
| 396 | |
Steven Rostedt | c5f888c | 2008-05-12 21:20:55 +0200 | [diff] [blame] | 397 | cpu = raw_smp_processor_id(); |
Steven Rostedt | 6cd8a4b | 2008-05-12 21:20:42 +0200 | [diff] [blame] | 398 | /* Always clear the tracing cpu on stopping the trace */ |
Steven Rostedt | c5f888c | 2008-05-12 21:20:55 +0200 | [diff] [blame] | 399 | if (unlikely(per_cpu(tracing_cpu, cpu))) |
| 400 | per_cpu(tracing_cpu, cpu) = 0; |
Steven Rostedt | 6cd8a4b | 2008-05-12 21:20:42 +0200 | [diff] [blame] | 401 | else |
| 402 | return; |
| 403 | |
Steven Rostedt (Red Hat) | 10246fa | 2013-07-01 15:58:24 -0400 | [diff] [blame] | 404 | if (!tracer_enabled || !tracing_is_enabled()) |
Steven Rostedt | 81d68a9 | 2008-05-12 21:20:42 +0200 | [diff] [blame] | 405 | return; |
| 406 | |
Steven Rostedt (Red Hat) | 12883ef | 2013-03-05 09:24:35 -0500 | [diff] [blame] | 407 | data = per_cpu_ptr(tr->trace_buffer.data, cpu); |
Steven Rostedt | 81d68a9 | 2008-05-12 21:20:42 +0200 | [diff] [blame] | 408 | |
Steven Rostedt | 3928a8a | 2008-09-29 23:02:41 -0400 | [diff] [blame] | 409 | if (unlikely(!data) || |
Steven Rostedt | 81d68a9 | 2008-05-12 21:20:42 +0200 | [diff] [blame] | 410 | !data->critical_start || atomic_read(&data->disabled)) |
| 411 | return; |
| 412 | |
| 413 | atomic_inc(&data->disabled); |
Steven Rostedt | c5f888c | 2008-05-12 21:20:55 +0200 | [diff] [blame] | 414 | |
Steven Rostedt | 81d68a9 | 2008-05-12 21:20:42 +0200 | [diff] [blame] | 415 | local_save_flags(flags); |
Jiri Olsa | 62b915f | 2010-04-02 19:01:22 +0200 | [diff] [blame] | 416 | __trace_function(tr, ip, parent_ip, flags, preempt_count()); |
Steven Rostedt | 6cd8a4b | 2008-05-12 21:20:42 +0200 | [diff] [blame] | 417 | check_critical_timing(tr, data, parent_ip ? : ip, cpu); |
Steven Rostedt | 81d68a9 | 2008-05-12 21:20:42 +0200 | [diff] [blame] | 418 | data->critical_start = 0; |
| 419 | atomic_dec(&data->disabled); |
| 420 | } |
| 421 | |
Steven Rostedt | 6cd8a4b | 2008-05-12 21:20:42 +0200 | [diff] [blame] | 422 | /* start and stop critical timings used to for stoppage (in idle) */ |
Ingo Molnar | e309b41 | 2008-05-12 21:20:51 +0200 | [diff] [blame] | 423 | void start_critical_timings(void) |
Steven Rostedt | 81d68a9 | 2008-05-12 21:20:42 +0200 | [diff] [blame] | 424 | { |
Steven Rostedt | 6cd8a4b | 2008-05-12 21:20:42 +0200 | [diff] [blame] | 425 | if (preempt_trace() || irq_trace()) |
Steven Rostedt | 81d68a9 | 2008-05-12 21:20:42 +0200 | [diff] [blame] | 426 | start_critical_timing(CALLER_ADDR0, CALLER_ADDR1); |
| 427 | } |
Ingo Molnar | 1fe3710 | 2008-07-26 15:09:47 +0200 | [diff] [blame] | 428 | EXPORT_SYMBOL_GPL(start_critical_timings); |
Steven Rostedt | 81d68a9 | 2008-05-12 21:20:42 +0200 | [diff] [blame] | 429 | |
Ingo Molnar | e309b41 | 2008-05-12 21:20:51 +0200 | [diff] [blame] | 430 | void stop_critical_timings(void) |
Steven Rostedt | 81d68a9 | 2008-05-12 21:20:42 +0200 | [diff] [blame] | 431 | { |
Steven Rostedt | 6cd8a4b | 2008-05-12 21:20:42 +0200 | [diff] [blame] | 432 | if (preempt_trace() || irq_trace()) |
Steven Rostedt | 81d68a9 | 2008-05-12 21:20:42 +0200 | [diff] [blame] | 433 | stop_critical_timing(CALLER_ADDR0, CALLER_ADDR1); |
| 434 | } |
Ingo Molnar | 1fe3710 | 2008-07-26 15:09:47 +0200 | [diff] [blame] | 435 | EXPORT_SYMBOL_GPL(stop_critical_timings); |
Steven Rostedt | 81d68a9 | 2008-05-12 21:20:42 +0200 | [diff] [blame] | 436 | |
Steven Rostedt | 6cd8a4b | 2008-05-12 21:20:42 +0200 | [diff] [blame] | 437 | #ifdef CONFIG_IRQSOFF_TRACER |
Steven Rostedt | 81d68a9 | 2008-05-12 21:20:42 +0200 | [diff] [blame] | 438 | #ifdef CONFIG_PROVE_LOCKING |
Ingo Molnar | e309b41 | 2008-05-12 21:20:51 +0200 | [diff] [blame] | 439 | void time_hardirqs_on(unsigned long a0, unsigned long a1) |
Steven Rostedt | 81d68a9 | 2008-05-12 21:20:42 +0200 | [diff] [blame] | 440 | { |
Steven Rostedt | 6cd8a4b | 2008-05-12 21:20:42 +0200 | [diff] [blame] | 441 | if (!preempt_trace() && irq_trace()) |
Steven Rostedt | 81d68a9 | 2008-05-12 21:20:42 +0200 | [diff] [blame] | 442 | stop_critical_timing(a0, a1); |
| 443 | } |
| 444 | |
Ingo Molnar | e309b41 | 2008-05-12 21:20:51 +0200 | [diff] [blame] | 445 | void time_hardirqs_off(unsigned long a0, unsigned long a1) |
Steven Rostedt | 81d68a9 | 2008-05-12 21:20:42 +0200 | [diff] [blame] | 446 | { |
Steven Rostedt | 6cd8a4b | 2008-05-12 21:20:42 +0200 | [diff] [blame] | 447 | if (!preempt_trace() && irq_trace()) |
Steven Rostedt | 81d68a9 | 2008-05-12 21:20:42 +0200 | [diff] [blame] | 448 | start_critical_timing(a0, a1); |
| 449 | } |
| 450 | |
| 451 | #else /* !CONFIG_PROVE_LOCKING */ |
| 452 | |
| 453 | /* |
| 454 | * Stubs: |
| 455 | */ |
| 456 | |
Steven Rostedt | 81d68a9 | 2008-05-12 21:20:42 +0200 | [diff] [blame] | 457 | void trace_softirqs_on(unsigned long ip) |
| 458 | { |
| 459 | } |
| 460 | |
| 461 | void trace_softirqs_off(unsigned long ip) |
| 462 | { |
| 463 | } |
| 464 | |
Ingo Molnar | e309b41 | 2008-05-12 21:20:51 +0200 | [diff] [blame] | 465 | inline void print_irqtrace_events(struct task_struct *curr) |
Steven Rostedt | 81d68a9 | 2008-05-12 21:20:42 +0200 | [diff] [blame] | 466 | { |
| 467 | } |
| 468 | |
| 469 | /* |
| 470 | * We are only interested in hardirq on/off events: |
| 471 | */ |
Ingo Molnar | e309b41 | 2008-05-12 21:20:51 +0200 | [diff] [blame] | 472 | void trace_hardirqs_on(void) |
Steven Rostedt | 81d68a9 | 2008-05-12 21:20:42 +0200 | [diff] [blame] | 473 | { |
Steven Rostedt | 6cd8a4b | 2008-05-12 21:20:42 +0200 | [diff] [blame] | 474 | if (!preempt_trace() && irq_trace()) |
Steven Rostedt | 81d68a9 | 2008-05-12 21:20:42 +0200 | [diff] [blame] | 475 | stop_critical_timing(CALLER_ADDR0, CALLER_ADDR1); |
| 476 | } |
| 477 | EXPORT_SYMBOL(trace_hardirqs_on); |
| 478 | |
Ingo Molnar | e309b41 | 2008-05-12 21:20:51 +0200 | [diff] [blame] | 479 | void trace_hardirqs_off(void) |
Steven Rostedt | 81d68a9 | 2008-05-12 21:20:42 +0200 | [diff] [blame] | 480 | { |
Steven Rostedt | 6cd8a4b | 2008-05-12 21:20:42 +0200 | [diff] [blame] | 481 | if (!preempt_trace() && irq_trace()) |
Steven Rostedt | 81d68a9 | 2008-05-12 21:20:42 +0200 | [diff] [blame] | 482 | start_critical_timing(CALLER_ADDR0, CALLER_ADDR1); |
| 483 | } |
| 484 | EXPORT_SYMBOL(trace_hardirqs_off); |
| 485 | |
Andi Kleen | 285c00a | 2014-02-08 08:52:08 +0100 | [diff] [blame] | 486 | __visible void trace_hardirqs_on_caller(unsigned long caller_addr) |
Steven Rostedt | 81d68a9 | 2008-05-12 21:20:42 +0200 | [diff] [blame] | 487 | { |
Steven Rostedt | 6cd8a4b | 2008-05-12 21:20:42 +0200 | [diff] [blame] | 488 | if (!preempt_trace() && irq_trace()) |
Steven Rostedt | 81d68a9 | 2008-05-12 21:20:42 +0200 | [diff] [blame] | 489 | stop_critical_timing(CALLER_ADDR0, caller_addr); |
| 490 | } |
| 491 | EXPORT_SYMBOL(trace_hardirqs_on_caller); |
| 492 | |
Andi Kleen | 285c00a | 2014-02-08 08:52:08 +0100 | [diff] [blame] | 493 | __visible void trace_hardirqs_off_caller(unsigned long caller_addr) |
Steven Rostedt | 81d68a9 | 2008-05-12 21:20:42 +0200 | [diff] [blame] | 494 | { |
Steven Rostedt | 6cd8a4b | 2008-05-12 21:20:42 +0200 | [diff] [blame] | 495 | if (!preempt_trace() && irq_trace()) |
Steven Rostedt | 81d68a9 | 2008-05-12 21:20:42 +0200 | [diff] [blame] | 496 | start_critical_timing(CALLER_ADDR0, caller_addr); |
| 497 | } |
| 498 | EXPORT_SYMBOL(trace_hardirqs_off_caller); |
| 499 | |
| 500 | #endif /* CONFIG_PROVE_LOCKING */ |
Steven Rostedt | 6cd8a4b | 2008-05-12 21:20:42 +0200 | [diff] [blame] | 501 | #endif /* CONFIG_IRQSOFF_TRACER */ |
| 502 | |
| 503 | #ifdef CONFIG_PREEMPT_TRACER |
Ingo Molnar | e309b41 | 2008-05-12 21:20:51 +0200 | [diff] [blame] | 504 | void trace_preempt_on(unsigned long a0, unsigned long a1) |
Steven Rostedt | 6cd8a4b | 2008-05-12 21:20:42 +0200 | [diff] [blame] | 505 | { |
Steven Rostedt | e36de1d | 2011-09-22 11:11:51 -0400 | [diff] [blame] | 506 | if (preempt_trace() && !irq_trace()) |
Steven Rostedt | 1e01cb0 | 2008-07-15 09:53:37 -0400 | [diff] [blame] | 507 | stop_critical_timing(a0, a1); |
Steven Rostedt | 6cd8a4b | 2008-05-12 21:20:42 +0200 | [diff] [blame] | 508 | } |
| 509 | |
Ingo Molnar | e309b41 | 2008-05-12 21:20:51 +0200 | [diff] [blame] | 510 | void trace_preempt_off(unsigned long a0, unsigned long a1) |
Steven Rostedt | 6cd8a4b | 2008-05-12 21:20:42 +0200 | [diff] [blame] | 511 | { |
Steven Rostedt | e36de1d | 2011-09-22 11:11:51 -0400 | [diff] [blame] | 512 | if (preempt_trace() && !irq_trace()) |
Steven Rostedt | 1e01cb0 | 2008-07-15 09:53:37 -0400 | [diff] [blame] | 513 | start_critical_timing(a0, a1); |
Steven Rostedt | 6cd8a4b | 2008-05-12 21:20:42 +0200 | [diff] [blame] | 514 | } |
| 515 | #endif /* CONFIG_PREEMPT_TRACER */ |
Steven Rostedt | 81d68a9 | 2008-05-12 21:20:42 +0200 | [diff] [blame] | 516 | |
Steven Rostedt (Red Hat) | 8179e8a | 2015-09-29 10:24:56 -0400 | [diff] [blame] | 517 | #ifdef CONFIG_FUNCTION_TRACER |
| 518 | static bool function_enabled; |
| 519 | |
Steven Rostedt (Red Hat) | 4104d32 | 2014-01-10 17:01:58 -0500 | [diff] [blame] | 520 | static int register_irqsoff_function(struct trace_array *tr, int graph, int set) |
Steven Rostedt | 81d68a9 | 2008-05-12 21:20:42 +0200 | [diff] [blame] | 521 | { |
Steven Rostedt (Red Hat) | 328df47 | 2013-03-14 12:10:40 -0400 | [diff] [blame] | 522 | int ret; |
Jiri Olsa | 62b915f | 2010-04-02 19:01:22 +0200 | [diff] [blame] | 523 | |
Steven Rostedt (Red Hat) | 328df47 | 2013-03-14 12:10:40 -0400 | [diff] [blame] | 524 | /* 'set' is set if TRACE_ITER_FUNCTION is about to be set */ |
Steven Rostedt (Red Hat) | 983f938 | 2015-09-30 09:42:05 -0400 | [diff] [blame] | 525 | if (function_enabled || (!set && !(tr->trace_flags & TRACE_ITER_FUNCTION))) |
Steven Rostedt (Red Hat) | 328df47 | 2013-03-14 12:10:40 -0400 | [diff] [blame] | 526 | return 0; |
| 527 | |
| 528 | if (graph) |
Jiri Olsa | 62b915f | 2010-04-02 19:01:22 +0200 | [diff] [blame] | 529 | ret = register_ftrace_graph(&irqsoff_graph_return, |
| 530 | &irqsoff_graph_entry); |
Steven Rostedt (Red Hat) | 328df47 | 2013-03-14 12:10:40 -0400 | [diff] [blame] | 531 | else |
Steven Rostedt (Red Hat) | 4104d32 | 2014-01-10 17:01:58 -0500 | [diff] [blame] | 532 | ret = register_ftrace_function(tr->ops); |
Steven Rostedt (Red Hat) | 328df47 | 2013-03-14 12:10:40 -0400 | [diff] [blame] | 533 | |
| 534 | if (!ret) |
| 535 | function_enabled = true; |
| 536 | |
| 537 | return ret; |
| 538 | } |
| 539 | |
Steven Rostedt (Red Hat) | 4104d32 | 2014-01-10 17:01:58 -0500 | [diff] [blame] | 540 | static void unregister_irqsoff_function(struct trace_array *tr, int graph) |
Steven Rostedt (Red Hat) | 328df47 | 2013-03-14 12:10:40 -0400 | [diff] [blame] | 541 | { |
| 542 | if (!function_enabled) |
| 543 | return; |
| 544 | |
| 545 | if (graph) |
| 546 | unregister_ftrace_graph(); |
| 547 | else |
Steven Rostedt (Red Hat) | 4104d32 | 2014-01-10 17:01:58 -0500 | [diff] [blame] | 548 | unregister_ftrace_function(tr->ops); |
Steven Rostedt (Red Hat) | 328df47 | 2013-03-14 12:10:40 -0400 | [diff] [blame] | 549 | |
| 550 | function_enabled = false; |
| 551 | } |
| 552 | |
Steven Rostedt (Red Hat) | 8179e8a | 2015-09-29 10:24:56 -0400 | [diff] [blame] | 553 | static int irqsoff_function_set(struct trace_array *tr, u32 mask, int set) |
Steven Rostedt (Red Hat) | 328df47 | 2013-03-14 12:10:40 -0400 | [diff] [blame] | 554 | { |
Steven Rostedt (Red Hat) | 8179e8a | 2015-09-29 10:24:56 -0400 | [diff] [blame] | 555 | if (!(mask & TRACE_ITER_FUNCTION)) |
| 556 | return 0; |
| 557 | |
Steven Rostedt (Red Hat) | 328df47 | 2013-03-14 12:10:40 -0400 | [diff] [blame] | 558 | if (set) |
Steven Rostedt (Red Hat) | 983f938 | 2015-09-30 09:42:05 -0400 | [diff] [blame] | 559 | register_irqsoff_function(tr, is_graph(tr), 1); |
Steven Rostedt (Red Hat) | 328df47 | 2013-03-14 12:10:40 -0400 | [diff] [blame] | 560 | else |
Steven Rostedt (Red Hat) | 983f938 | 2015-09-30 09:42:05 -0400 | [diff] [blame] | 561 | unregister_irqsoff_function(tr, is_graph(tr)); |
Steven Rostedt (Red Hat) | 8179e8a | 2015-09-29 10:24:56 -0400 | [diff] [blame] | 562 | return 1; |
| 563 | } |
| 564 | #else |
| 565 | static int register_irqsoff_function(struct trace_array *tr, int graph, int set) |
| 566 | { |
Steven Rostedt (Red Hat) | 0390558 | 2015-09-28 15:37:49 -0400 | [diff] [blame] | 567 | return 0; |
Steven Rostedt (Red Hat) | 328df47 | 2013-03-14 12:10:40 -0400 | [diff] [blame] | 568 | } |
Steven Rostedt (Red Hat) | 8179e8a | 2015-09-29 10:24:56 -0400 | [diff] [blame] | 569 | static void unregister_irqsoff_function(struct trace_array *tr, int graph) { } |
| 570 | static inline int irqsoff_function_set(struct trace_array *tr, u32 mask, int set) |
| 571 | { |
| 572 | return 0; |
| 573 | } |
| 574 | #endif /* CONFIG_FUNCTION_TRACER */ |
Steven Rostedt (Red Hat) | 328df47 | 2013-03-14 12:10:40 -0400 | [diff] [blame] | 575 | |
Steven Rostedt (Red Hat) | bf6065b | 2014-01-10 17:51:01 -0500 | [diff] [blame] | 576 | static int irqsoff_flag_changed(struct trace_array *tr, u32 mask, int set) |
Steven Rostedt (Red Hat) | 328df47 | 2013-03-14 12:10:40 -0400 | [diff] [blame] | 577 | { |
Steven Rostedt (Red Hat) | bf6065b | 2014-01-10 17:51:01 -0500 | [diff] [blame] | 578 | struct tracer *tracer = tr->current_trace; |
| 579 | |
Steven Rostedt (Red Hat) | 8179e8a | 2015-09-29 10:24:56 -0400 | [diff] [blame] | 580 | if (irqsoff_function_set(tr, mask, set)) |
| 581 | return 0; |
Steven Rostedt (Red Hat) | 0390558 | 2015-09-28 15:37:49 -0400 | [diff] [blame] | 582 | |
Steven Rostedt (Red Hat) | 729358d | 2015-09-29 10:15:10 -0400 | [diff] [blame] | 583 | #ifdef CONFIG_FUNCTION_GRAPH_TRACER |
Steven Rostedt (Red Hat) | 0390558 | 2015-09-28 15:37:49 -0400 | [diff] [blame] | 584 | if (mask & TRACE_ITER_DISPLAY_GRAPH) |
| 585 | return irqsoff_display_graph(tr, set); |
Steven Rostedt (Red Hat) | 729358d | 2015-09-29 10:15:10 -0400 | [diff] [blame] | 586 | #endif |
Steven Rostedt (Red Hat) | 328df47 | 2013-03-14 12:10:40 -0400 | [diff] [blame] | 587 | |
| 588 | return trace_keep_overwrite(tracer, mask, set); |
| 589 | } |
| 590 | |
| 591 | static int start_irqsoff_tracer(struct trace_array *tr, int graph) |
| 592 | { |
| 593 | int ret; |
| 594 | |
Steven Rostedt (Red Hat) | 4104d32 | 2014-01-10 17:01:58 -0500 | [diff] [blame] | 595 | ret = register_irqsoff_function(tr, graph, 0); |
Jiri Olsa | 62b915f | 2010-04-02 19:01:22 +0200 | [diff] [blame] | 596 | |
| 597 | if (!ret && tracing_is_enabled()) |
Steven Rostedt | 9036990 | 2008-11-05 16:05:44 -0500 | [diff] [blame] | 598 | tracer_enabled = 1; |
Steven Rostedt | 94523e8 | 2009-01-22 11:18:06 -0500 | [diff] [blame] | 599 | else |
Steven Rostedt | 9036990 | 2008-11-05 16:05:44 -0500 | [diff] [blame] | 600 | tracer_enabled = 0; |
Jiri Olsa | 62b915f | 2010-04-02 19:01:22 +0200 | [diff] [blame] | 601 | |
| 602 | return ret; |
Steven Rostedt | 81d68a9 | 2008-05-12 21:20:42 +0200 | [diff] [blame] | 603 | } |
| 604 | |
Jiri Olsa | 62b915f | 2010-04-02 19:01:22 +0200 | [diff] [blame] | 605 | static void stop_irqsoff_tracer(struct trace_array *tr, int graph) |
Steven Rostedt | 81d68a9 | 2008-05-12 21:20:42 +0200 | [diff] [blame] | 606 | { |
Steven Rostedt | 81d68a9 | 2008-05-12 21:20:42 +0200 | [diff] [blame] | 607 | tracer_enabled = 0; |
Jiri Olsa | 62b915f | 2010-04-02 19:01:22 +0200 | [diff] [blame] | 608 | |
Steven Rostedt (Red Hat) | 4104d32 | 2014-01-10 17:01:58 -0500 | [diff] [blame] | 609 | unregister_irqsoff_function(tr, graph); |
Steven Rostedt | 81d68a9 | 2008-05-12 21:20:42 +0200 | [diff] [blame] | 610 | } |
| 611 | |
Steven Rostedt (Red Hat) | 02f2f76 | 2014-01-14 09:32:58 -0500 | [diff] [blame] | 612 | static bool irqsoff_busy; |
| 613 | |
| 614 | static int __irqsoff_tracer_init(struct trace_array *tr) |
Steven Rostedt | 81d68a9 | 2008-05-12 21:20:42 +0200 | [diff] [blame] | 615 | { |
Steven Rostedt (Red Hat) | 02f2f76 | 2014-01-14 09:32:58 -0500 | [diff] [blame] | 616 | if (irqsoff_busy) |
| 617 | return -EBUSY; |
| 618 | |
Steven Rostedt (Red Hat) | 983f938 | 2015-09-30 09:42:05 -0400 | [diff] [blame] | 619 | save_flags = tr->trace_flags; |
Steven Rostedt (Red Hat) | 613f04a | 2013-03-14 15:03:53 -0400 | [diff] [blame] | 620 | |
| 621 | /* non overwrite screws up the latency tracers */ |
Steven Rostedt | 2b6080f | 2012-05-11 13:29:49 -0400 | [diff] [blame] | 622 | set_tracer_flag(tr, TRACE_ITER_OVERWRITE, 1); |
| 623 | set_tracer_flag(tr, TRACE_ITER_LATENCY_FMT, 1); |
Steven Rostedt | e9d25fe | 2009-03-04 22:15:30 -0500 | [diff] [blame] | 624 | |
Steven Rostedt (Red Hat) | 6d9b3fa | 2014-01-14 11:28:38 -0500 | [diff] [blame] | 625 | tr->max_latency = 0; |
Steven Rostedt | 81d68a9 | 2008-05-12 21:20:42 +0200 | [diff] [blame] | 626 | irqsoff_trace = tr; |
Steven Rostedt | c5f888c | 2008-05-12 21:20:55 +0200 | [diff] [blame] | 627 | /* make sure that the tracer is visible */ |
Steven Rostedt | 81d68a9 | 2008-05-12 21:20:42 +0200 | [diff] [blame] | 628 | smp_wmb(); |
Jiri Olsa | 62b915f | 2010-04-02 19:01:22 +0200 | [diff] [blame] | 629 | |
Steven Rostedt (Red Hat) | 4104d32 | 2014-01-10 17:01:58 -0500 | [diff] [blame] | 630 | ftrace_init_array_ops(tr, irqsoff_tracer_call); |
| 631 | |
| 632 | /* Only toplevel instance supports graph tracing */ |
| 633 | if (start_irqsoff_tracer(tr, (tr->flags & TRACE_ARRAY_FL_GLOBAL && |
Steven Rostedt (Red Hat) | 983f938 | 2015-09-30 09:42:05 -0400 | [diff] [blame] | 634 | is_graph(tr)))) |
Jiri Olsa | 62b915f | 2010-04-02 19:01:22 +0200 | [diff] [blame] | 635 | printk(KERN_ERR "failed to start irqsoff tracer\n"); |
Steven Rostedt (Red Hat) | 02f2f76 | 2014-01-14 09:32:58 -0500 | [diff] [blame] | 636 | |
| 637 | irqsoff_busy = true; |
| 638 | return 0; |
Steven Rostedt | 81d68a9 | 2008-05-12 21:20:42 +0200 | [diff] [blame] | 639 | } |
| 640 | |
| 641 | static void irqsoff_tracer_reset(struct trace_array *tr) |
| 642 | { |
Steven Rostedt (Red Hat) | 613f04a | 2013-03-14 15:03:53 -0400 | [diff] [blame] | 643 | int lat_flag = save_flags & TRACE_ITER_LATENCY_FMT; |
| 644 | int overwrite_flag = save_flags & TRACE_ITER_OVERWRITE; |
| 645 | |
Steven Rostedt (Red Hat) | 983f938 | 2015-09-30 09:42:05 -0400 | [diff] [blame] | 646 | stop_irqsoff_tracer(tr, is_graph(tr)); |
Steven Rostedt | e9d25fe | 2009-03-04 22:15:30 -0500 | [diff] [blame] | 647 | |
Steven Rostedt | 2b6080f | 2012-05-11 13:29:49 -0400 | [diff] [blame] | 648 | set_tracer_flag(tr, TRACE_ITER_LATENCY_FMT, lat_flag); |
| 649 | set_tracer_flag(tr, TRACE_ITER_OVERWRITE, overwrite_flag); |
Steven Rostedt (Red Hat) | 4104d32 | 2014-01-10 17:01:58 -0500 | [diff] [blame] | 650 | ftrace_reset_array_ops(tr); |
Steven Rostedt (Red Hat) | 02f2f76 | 2014-01-14 09:32:58 -0500 | [diff] [blame] | 651 | |
| 652 | irqsoff_busy = false; |
Steven Rostedt | 81d68a9 | 2008-05-12 21:20:42 +0200 | [diff] [blame] | 653 | } |
| 654 | |
Steven Rostedt | 9036990 | 2008-11-05 16:05:44 -0500 | [diff] [blame] | 655 | static void irqsoff_tracer_start(struct trace_array *tr) |
| 656 | { |
Steven Rostedt | 9036990 | 2008-11-05 16:05:44 -0500 | [diff] [blame] | 657 | tracer_enabled = 1; |
Steven Rostedt | 9036990 | 2008-11-05 16:05:44 -0500 | [diff] [blame] | 658 | } |
| 659 | |
| 660 | static void irqsoff_tracer_stop(struct trace_array *tr) |
| 661 | { |
| 662 | tracer_enabled = 0; |
Steven Rostedt | 81d68a9 | 2008-05-12 21:20:42 +0200 | [diff] [blame] | 663 | } |
| 664 | |
Steven Rostedt | 6cd8a4b | 2008-05-12 21:20:42 +0200 | [diff] [blame] | 665 | #ifdef CONFIG_IRQSOFF_TRACER |
Frederic Weisbecker | 1c80025 | 2008-11-16 05:57:26 +0100 | [diff] [blame] | 666 | static int irqsoff_tracer_init(struct trace_array *tr) |
Steven Rostedt | 6cd8a4b | 2008-05-12 21:20:42 +0200 | [diff] [blame] | 667 | { |
| 668 | trace_type = TRACER_IRQS_OFF; |
| 669 | |
Steven Rostedt (Red Hat) | 02f2f76 | 2014-01-14 09:32:58 -0500 | [diff] [blame] | 670 | return __irqsoff_tracer_init(tr); |
Steven Rostedt | 6cd8a4b | 2008-05-12 21:20:42 +0200 | [diff] [blame] | 671 | } |
Steven Rostedt | 81d68a9 | 2008-05-12 21:20:42 +0200 | [diff] [blame] | 672 | static struct tracer irqsoff_tracer __read_mostly = |
| 673 | { |
| 674 | .name = "irqsoff", |
| 675 | .init = irqsoff_tracer_init, |
| 676 | .reset = irqsoff_tracer_reset, |
Steven Rostedt | 9036990 | 2008-11-05 16:05:44 -0500 | [diff] [blame] | 677 | .start = irqsoff_tracer_start, |
| 678 | .stop = irqsoff_tracer_stop, |
Hiraku Toyooka | f43c738 | 2012-10-02 17:27:10 +0900 | [diff] [blame] | 679 | .print_max = true, |
Jiri Olsa | 62b915f | 2010-04-02 19:01:22 +0200 | [diff] [blame] | 680 | .print_header = irqsoff_print_header, |
| 681 | .print_line = irqsoff_print_line, |
Steven Rostedt (Red Hat) | 328df47 | 2013-03-14 12:10:40 -0400 | [diff] [blame] | 682 | .flag_changed = irqsoff_flag_changed, |
Steven Rostedt | 60a1177 | 2008-05-12 21:20:44 +0200 | [diff] [blame] | 683 | #ifdef CONFIG_FTRACE_SELFTEST |
| 684 | .selftest = trace_selftest_startup_irqsoff, |
| 685 | #endif |
Jiri Olsa | 62b915f | 2010-04-02 19:01:22 +0200 | [diff] [blame] | 686 | .open = irqsoff_trace_open, |
| 687 | .close = irqsoff_trace_close, |
Steven Rostedt (Red Hat) | 02f2f76 | 2014-01-14 09:32:58 -0500 | [diff] [blame] | 688 | .allow_instances = true, |
Hiraku Toyooka | f43c738 | 2012-10-02 17:27:10 +0900 | [diff] [blame] | 689 | .use_max_tr = true, |
Steven Rostedt | 81d68a9 | 2008-05-12 21:20:42 +0200 | [diff] [blame] | 690 | }; |
Steven Rostedt | 6cd8a4b | 2008-05-12 21:20:42 +0200 | [diff] [blame] | 691 | # define register_irqsoff(trace) register_tracer(&trace) |
| 692 | #else |
| 693 | # define register_irqsoff(trace) do { } while (0) |
| 694 | #endif |
| 695 | |
| 696 | #ifdef CONFIG_PREEMPT_TRACER |
Frederic Weisbecker | 1c80025 | 2008-11-16 05:57:26 +0100 | [diff] [blame] | 697 | static int preemptoff_tracer_init(struct trace_array *tr) |
Steven Rostedt | 6cd8a4b | 2008-05-12 21:20:42 +0200 | [diff] [blame] | 698 | { |
| 699 | trace_type = TRACER_PREEMPT_OFF; |
| 700 | |
Steven Rostedt (Red Hat) | 02f2f76 | 2014-01-14 09:32:58 -0500 | [diff] [blame] | 701 | return __irqsoff_tracer_init(tr); |
Steven Rostedt | 6cd8a4b | 2008-05-12 21:20:42 +0200 | [diff] [blame] | 702 | } |
| 703 | |
| 704 | static struct tracer preemptoff_tracer __read_mostly = |
| 705 | { |
| 706 | .name = "preemptoff", |
| 707 | .init = preemptoff_tracer_init, |
| 708 | .reset = irqsoff_tracer_reset, |
Steven Rostedt | 9036990 | 2008-11-05 16:05:44 -0500 | [diff] [blame] | 709 | .start = irqsoff_tracer_start, |
| 710 | .stop = irqsoff_tracer_stop, |
Hiraku Toyooka | f43c738 | 2012-10-02 17:27:10 +0900 | [diff] [blame] | 711 | .print_max = true, |
Jiri Olsa | 62b915f | 2010-04-02 19:01:22 +0200 | [diff] [blame] | 712 | .print_header = irqsoff_print_header, |
| 713 | .print_line = irqsoff_print_line, |
Steven Rostedt (Red Hat) | 328df47 | 2013-03-14 12:10:40 -0400 | [diff] [blame] | 714 | .flag_changed = irqsoff_flag_changed, |
Steven Rostedt | 60a1177 | 2008-05-12 21:20:44 +0200 | [diff] [blame] | 715 | #ifdef CONFIG_FTRACE_SELFTEST |
| 716 | .selftest = trace_selftest_startup_preemptoff, |
| 717 | #endif |
Jiri Olsa | 62b915f | 2010-04-02 19:01:22 +0200 | [diff] [blame] | 718 | .open = irqsoff_trace_open, |
| 719 | .close = irqsoff_trace_close, |
Steven Rostedt (Red Hat) | 02f2f76 | 2014-01-14 09:32:58 -0500 | [diff] [blame] | 720 | .allow_instances = true, |
Hiraku Toyooka | f43c738 | 2012-10-02 17:27:10 +0900 | [diff] [blame] | 721 | .use_max_tr = true, |
Steven Rostedt | 6cd8a4b | 2008-05-12 21:20:42 +0200 | [diff] [blame] | 722 | }; |
| 723 | # define register_preemptoff(trace) register_tracer(&trace) |
| 724 | #else |
| 725 | # define register_preemptoff(trace) do { } while (0) |
| 726 | #endif |
| 727 | |
| 728 | #if defined(CONFIG_IRQSOFF_TRACER) && \ |
| 729 | defined(CONFIG_PREEMPT_TRACER) |
| 730 | |
Frederic Weisbecker | 1c80025 | 2008-11-16 05:57:26 +0100 | [diff] [blame] | 731 | static int preemptirqsoff_tracer_init(struct trace_array *tr) |
Steven Rostedt | 6cd8a4b | 2008-05-12 21:20:42 +0200 | [diff] [blame] | 732 | { |
| 733 | trace_type = TRACER_IRQS_OFF | TRACER_PREEMPT_OFF; |
| 734 | |
Steven Rostedt (Red Hat) | 02f2f76 | 2014-01-14 09:32:58 -0500 | [diff] [blame] | 735 | return __irqsoff_tracer_init(tr); |
Steven Rostedt | 6cd8a4b | 2008-05-12 21:20:42 +0200 | [diff] [blame] | 736 | } |
| 737 | |
| 738 | static struct tracer preemptirqsoff_tracer __read_mostly = |
| 739 | { |
| 740 | .name = "preemptirqsoff", |
| 741 | .init = preemptirqsoff_tracer_init, |
| 742 | .reset = irqsoff_tracer_reset, |
Steven Rostedt | 9036990 | 2008-11-05 16:05:44 -0500 | [diff] [blame] | 743 | .start = irqsoff_tracer_start, |
| 744 | .stop = irqsoff_tracer_stop, |
Hiraku Toyooka | f43c738 | 2012-10-02 17:27:10 +0900 | [diff] [blame] | 745 | .print_max = true, |
Jiri Olsa | 62b915f | 2010-04-02 19:01:22 +0200 | [diff] [blame] | 746 | .print_header = irqsoff_print_header, |
| 747 | .print_line = irqsoff_print_line, |
Steven Rostedt (Red Hat) | 328df47 | 2013-03-14 12:10:40 -0400 | [diff] [blame] | 748 | .flag_changed = irqsoff_flag_changed, |
Steven Rostedt | 60a1177 | 2008-05-12 21:20:44 +0200 | [diff] [blame] | 749 | #ifdef CONFIG_FTRACE_SELFTEST |
| 750 | .selftest = trace_selftest_startup_preemptirqsoff, |
| 751 | #endif |
Jiri Olsa | 62b915f | 2010-04-02 19:01:22 +0200 | [diff] [blame] | 752 | .open = irqsoff_trace_open, |
| 753 | .close = irqsoff_trace_close, |
Steven Rostedt (Red Hat) | 02f2f76 | 2014-01-14 09:32:58 -0500 | [diff] [blame] | 754 | .allow_instances = true, |
Hiraku Toyooka | f43c738 | 2012-10-02 17:27:10 +0900 | [diff] [blame] | 755 | .use_max_tr = true, |
Steven Rostedt | 6cd8a4b | 2008-05-12 21:20:42 +0200 | [diff] [blame] | 756 | }; |
| 757 | |
| 758 | # define register_preemptirqsoff(trace) register_tracer(&trace) |
| 759 | #else |
| 760 | # define register_preemptirqsoff(trace) do { } while (0) |
| 761 | #endif |
Steven Rostedt | 81d68a9 | 2008-05-12 21:20:42 +0200 | [diff] [blame] | 762 | |
| 763 | __init static int init_irqsoff_tracer(void) |
| 764 | { |
Steven Rostedt | 6cd8a4b | 2008-05-12 21:20:42 +0200 | [diff] [blame] | 765 | register_irqsoff(irqsoff_tracer); |
| 766 | register_preemptoff(preemptoff_tracer); |
| 767 | register_preemptirqsoff(preemptirqsoff_tracer); |
Steven Rostedt | 81d68a9 | 2008-05-12 21:20:42 +0200 | [diff] [blame] | 768 | |
| 769 | return 0; |
| 770 | } |
Steven Rostedt | 6f41567 | 2012-10-05 12:13:07 -0400 | [diff] [blame] | 771 | core_initcall(init_irqsoff_tracer); |