blob: 138229439a93f3c8527f3da1cd319779fe287fe7 [file] [log] [blame]
Ingo Molnar0a02ad92009-09-11 12:12:54 +02001#include "builtin.h"
Ingo Molnarb1ffe8f2009-09-11 12:12:54 +02002#include "perf.h"
Ingo Molnar0a02ad92009-09-11 12:12:54 +02003
4#include "util/util.h"
Arnaldo Carvalho de Meloee29be62011-11-28 17:57:40 -02005#include "util/evlist.h"
Ingo Molnar0a02ad92009-09-11 12:12:54 +02006#include "util/cache.h"
Arnaldo Carvalho de Meloe3f42602011-11-16 17:02:54 -02007#include "util/evsel.h"
Ingo Molnar0a02ad92009-09-11 12:12:54 +02008#include "util/symbol.h"
9#include "util/thread.h"
10#include "util/header.h"
Arnaldo Carvalho de Melo94c744b2009-12-11 21:24:02 -020011#include "util/session.h"
Arnaldo Carvalho de Melo45694aa2011-11-28 08:30:20 -020012#include "util/tool.h"
Ingo Molnar0a02ad92009-09-11 12:12:54 +020013
14#include "util/parse-options.h"
Ingo Molnarb1ffe8f2009-09-11 12:12:54 +020015#include "util/trace-event.h"
Ingo Molnar0a02ad92009-09-11 12:12:54 +020016
Ingo Molnar0a02ad92009-09-11 12:12:54 +020017#include "util/debug.h"
18
Ingo Molnarb1ffe8f2009-09-11 12:12:54 +020019#include <sys/prctl.h>
Markus Trippelsdorf7b78f132012-04-04 10:45:27 +020020#include <sys/resource.h>
Ingo Molnar0a02ad92009-09-11 12:12:54 +020021
Ingo Molnarb1ffe8f2009-09-11 12:12:54 +020022#include <semaphore.h>
23#include <pthread.h>
24#include <math.h>
Frederic Weisbecker419ab0d2009-09-12 03:59:01 +020025
Ingo Molnarb1ffe8f2009-09-11 12:12:54 +020026#define PR_SET_NAME 15 /* Set process name */
27#define MAX_CPUS 4096
Ingo Molnarb1ffe8f2009-09-11 12:12:54 +020028#define COMM_LEN 20
29#define SYM_LEN 129
Ingo Molnarb1ffe8f2009-09-11 12:12:54 +020030#define MAX_PID 65536
Ingo Molnarec156762009-09-11 12:12:54 +020031
mingo39aeb522009-09-14 20:04:48 +020032struct sched_atom;
Ingo Molnarec156762009-09-11 12:12:54 +020033
34struct task_desc {
35 unsigned long nr;
36 unsigned long pid;
37 char comm[COMM_LEN];
38
39 unsigned long nr_events;
40 unsigned long curr_event;
mingo39aeb522009-09-14 20:04:48 +020041 struct sched_atom **atoms;
Ingo Molnarec156762009-09-11 12:12:54 +020042
43 pthread_t thread;
44 sem_t sleep_sem;
45
46 sem_t ready_for_work;
47 sem_t work_done_sem;
48
Ingo Molnarb1ffe8f2009-09-11 12:12:54 +020049 u64 cpu_usage;
Ingo Molnarec156762009-09-11 12:12:54 +020050};
51
52enum sched_event_type {
53 SCHED_EVENT_RUN,
54 SCHED_EVENT_SLEEP,
55 SCHED_EVENT_WAKEUP,
Mike Galbraith55ffb7a2009-10-10 14:46:04 +020056 SCHED_EVENT_MIGRATION,
Ingo Molnarec156762009-09-11 12:12:54 +020057};
58
mingo39aeb522009-09-14 20:04:48 +020059struct sched_atom {
Ingo Molnarec156762009-09-11 12:12:54 +020060 enum sched_event_type type;
Arnaldo Carvalho de Meloeed05fe2010-04-05 12:53:45 -030061 int specific_wait;
Ingo Molnarb1ffe8f2009-09-11 12:12:54 +020062 u64 timestamp;
63 u64 duration;
Ingo Molnarec156762009-09-11 12:12:54 +020064 unsigned long nr;
Ingo Molnarec156762009-09-11 12:12:54 +020065 sem_t *wait_sem;
66 struct task_desc *wakee;
67};
68
Ingo Molnarb1ffe8f2009-09-11 12:12:54 +020069#define TASK_STATE_TO_CHAR_STR "RSDTtZX"
70
71enum thread_state {
72 THREAD_SLEEPING = 0,
73 THREAD_WAIT_CPU,
74 THREAD_SCHED_IN,
75 THREAD_IGNORE
76};
77
78struct work_atom {
79 struct list_head list;
80 enum thread_state state;
Frederic Weisbeckeraa1ab9d2009-09-14 03:01:12 +020081 u64 sched_out_time;
Ingo Molnarb1ffe8f2009-09-11 12:12:54 +020082 u64 wake_up_time;
83 u64 sched_in_time;
84 u64 runtime;
85};
86
mingo39aeb522009-09-14 20:04:48 +020087struct work_atoms {
88 struct list_head work_list;
Ingo Molnarb1ffe8f2009-09-11 12:12:54 +020089 struct thread *thread;
90 struct rb_node node;
91 u64 max_lat;
Frederic Weisbecker3786310a2009-12-09 21:40:08 +010092 u64 max_lat_at;
Ingo Molnarb1ffe8f2009-09-11 12:12:54 +020093 u64 total_lat;
94 u64 nb_atoms;
95 u64 total_runtime;
96};
97
mingo39aeb522009-09-14 20:04:48 +020098typedef int (*sort_fn_t)(struct work_atoms *, struct work_atoms *);
Ingo Molnarb1ffe8f2009-09-11 12:12:54 +020099
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -0300100struct perf_sched;
101
102struct trace_sched_handler {
Arnaldo Carvalho de Melo9ec3f4e2012-09-11 19:29:17 -0300103 int (*switch_event)(struct perf_sched *sched, struct perf_evsel *evsel,
104 struct perf_sample *sample, struct machine *machine);
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -0300105
Arnaldo Carvalho de Melo9ec3f4e2012-09-11 19:29:17 -0300106 int (*runtime_event)(struct perf_sched *sched, struct perf_evsel *evsel,
107 struct perf_sample *sample, struct machine *machine);
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -0300108
Arnaldo Carvalho de Melo9ec3f4e2012-09-11 19:29:17 -0300109 int (*wakeup_event)(struct perf_sched *sched, struct perf_evsel *evsel,
110 struct perf_sample *sample, struct machine *machine);
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -0300111
Arnaldo Carvalho de Melo9ec3f4e2012-09-11 19:29:17 -0300112 int (*fork_event)(struct perf_sched *sched, struct perf_evsel *evsel,
113 struct perf_sample *sample);
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -0300114
115 int (*migrate_task_event)(struct perf_sched *sched,
Arnaldo Carvalho de Melo9ec3f4e2012-09-11 19:29:17 -0300116 struct perf_evsel *evsel,
117 struct perf_sample *sample,
118 struct machine *machine);
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -0300119};
120
121struct perf_sched {
122 struct perf_tool tool;
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -0300123 const char *sort_order;
124 unsigned long nr_tasks;
125 struct task_desc *pid_to_task[MAX_PID];
126 struct task_desc **tasks;
127 const struct trace_sched_handler *tp_handler;
128 pthread_mutex_t start_work_mutex;
129 pthread_mutex_t work_done_wait_mutex;
130 int profile_cpu;
131/*
132 * Track the current task - that way we can know whether there's any
133 * weird events, such as a task being switched away that is not current.
134 */
135 int max_cpu;
136 u32 curr_pid[MAX_CPUS];
137 struct thread *curr_thread[MAX_CPUS];
138 char next_shortname1;
139 char next_shortname2;
140 unsigned int replay_repeat;
141 unsigned long nr_run_events;
142 unsigned long nr_sleep_events;
143 unsigned long nr_wakeup_events;
144 unsigned long nr_sleep_corrections;
145 unsigned long nr_run_events_optimized;
146 unsigned long targetless_wakeups;
147 unsigned long multitarget_wakeups;
148 unsigned long nr_runs;
149 unsigned long nr_timestamps;
150 unsigned long nr_unordered_timestamps;
151 unsigned long nr_state_machine_bugs;
152 unsigned long nr_context_switch_bugs;
153 unsigned long nr_events;
154 unsigned long nr_lost_chunks;
155 unsigned long nr_lost_events;
156 u64 run_measurement_overhead;
157 u64 sleep_measurement_overhead;
158 u64 start_time;
159 u64 cpu_usage;
160 u64 runavg_cpu_usage;
161 u64 parent_cpu_usage;
162 u64 runavg_parent_cpu_usage;
163 u64 sum_runtime;
164 u64 sum_fluct;
165 u64 run_avg;
166 u64 all_runtime;
167 u64 all_count;
168 u64 cpu_last_switched[MAX_CPUS];
169 struct rb_root atom_root, sorted_atom_root;
170 struct list_head sort_list, cmp_pid;
171};
Ingo Molnarb1ffe8f2009-09-11 12:12:54 +0200172
173static u64 get_nsecs(void)
174{
175 struct timespec ts;
176
177 clock_gettime(CLOCK_MONOTONIC, &ts);
178
179 return ts.tv_sec * 1000000000ULL + ts.tv_nsec;
180}
181
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -0300182static void burn_nsecs(struct perf_sched *sched, u64 nsecs)
Ingo Molnarb1ffe8f2009-09-11 12:12:54 +0200183{
184 u64 T0 = get_nsecs(), T1;
185
186 do {
187 T1 = get_nsecs();
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -0300188 } while (T1 + sched->run_measurement_overhead < T0 + nsecs);
Ingo Molnarb1ffe8f2009-09-11 12:12:54 +0200189}
190
191static void sleep_nsecs(u64 nsecs)
192{
193 struct timespec ts;
194
195 ts.tv_nsec = nsecs % 999999999;
196 ts.tv_sec = nsecs / 999999999;
197
198 nanosleep(&ts, NULL);
199}
200
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -0300201static void calibrate_run_measurement_overhead(struct perf_sched *sched)
Ingo Molnarb1ffe8f2009-09-11 12:12:54 +0200202{
203 u64 T0, T1, delta, min_delta = 1000000000ULL;
204 int i;
205
206 for (i = 0; i < 10; i++) {
207 T0 = get_nsecs();
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -0300208 burn_nsecs(sched, 0);
Ingo Molnarb1ffe8f2009-09-11 12:12:54 +0200209 T1 = get_nsecs();
210 delta = T1-T0;
211 min_delta = min(min_delta, delta);
212 }
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -0300213 sched->run_measurement_overhead = min_delta;
Ingo Molnarb1ffe8f2009-09-11 12:12:54 +0200214
Arnaldo Carvalho de Melo9486aa32011-01-22 20:37:02 -0200215 printf("run measurement overhead: %" PRIu64 " nsecs\n", min_delta);
Ingo Molnarb1ffe8f2009-09-11 12:12:54 +0200216}
217
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -0300218static void calibrate_sleep_measurement_overhead(struct perf_sched *sched)
Ingo Molnarb1ffe8f2009-09-11 12:12:54 +0200219{
220 u64 T0, T1, delta, min_delta = 1000000000ULL;
221 int i;
222
223 for (i = 0; i < 10; i++) {
224 T0 = get_nsecs();
225 sleep_nsecs(10000);
226 T1 = get_nsecs();
227 delta = T1-T0;
228 min_delta = min(min_delta, delta);
229 }
230 min_delta -= 10000;
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -0300231 sched->sleep_measurement_overhead = min_delta;
Ingo Molnarb1ffe8f2009-09-11 12:12:54 +0200232
Arnaldo Carvalho de Melo9486aa32011-01-22 20:37:02 -0200233 printf("sleep measurement overhead: %" PRIu64 " nsecs\n", min_delta);
Ingo Molnarb1ffe8f2009-09-11 12:12:54 +0200234}
235
mingo39aeb522009-09-14 20:04:48 +0200236static struct sched_atom *
Ingo Molnarb1ffe8f2009-09-11 12:12:54 +0200237get_new_event(struct task_desc *task, u64 timestamp)
Ingo Molnarec156762009-09-11 12:12:54 +0200238{
Arnaldo Carvalho de Melo36479482009-11-24 12:05:16 -0200239 struct sched_atom *event = zalloc(sizeof(*event));
Ingo Molnarec156762009-09-11 12:12:54 +0200240 unsigned long idx = task->nr_events;
241 size_t size;
242
243 event->timestamp = timestamp;
244 event->nr = idx;
245
246 task->nr_events++;
mingo39aeb522009-09-14 20:04:48 +0200247 size = sizeof(struct sched_atom *) * task->nr_events;
248 task->atoms = realloc(task->atoms, size);
249 BUG_ON(!task->atoms);
Ingo Molnarec156762009-09-11 12:12:54 +0200250
mingo39aeb522009-09-14 20:04:48 +0200251 task->atoms[idx] = event;
Ingo Molnarec156762009-09-11 12:12:54 +0200252
253 return event;
254}
255
mingo39aeb522009-09-14 20:04:48 +0200256static struct sched_atom *last_event(struct task_desc *task)
Ingo Molnarec156762009-09-11 12:12:54 +0200257{
258 if (!task->nr_events)
259 return NULL;
260
mingo39aeb522009-09-14 20:04:48 +0200261 return task->atoms[task->nr_events - 1];
Ingo Molnarec156762009-09-11 12:12:54 +0200262}
263
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -0300264static void add_sched_event_run(struct perf_sched *sched, struct task_desc *task,
265 u64 timestamp, u64 duration)
Ingo Molnarec156762009-09-11 12:12:54 +0200266{
mingo39aeb522009-09-14 20:04:48 +0200267 struct sched_atom *event, *curr_event = last_event(task);
Ingo Molnarec156762009-09-11 12:12:54 +0200268
269 /*
Ingo Molnarfbf94822009-09-11 12:12:54 +0200270 * optimize an existing RUN event by merging this one
271 * to it:
272 */
Ingo Molnarec156762009-09-11 12:12:54 +0200273 if (curr_event && curr_event->type == SCHED_EVENT_RUN) {
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -0300274 sched->nr_run_events_optimized++;
Ingo Molnarec156762009-09-11 12:12:54 +0200275 curr_event->duration += duration;
276 return;
277 }
278
279 event = get_new_event(task, timestamp);
280
281 event->type = SCHED_EVENT_RUN;
282 event->duration = duration;
283
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -0300284 sched->nr_run_events++;
Ingo Molnarec156762009-09-11 12:12:54 +0200285}
286
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -0300287static void add_sched_event_wakeup(struct perf_sched *sched, struct task_desc *task,
288 u64 timestamp, struct task_desc *wakee)
Ingo Molnarec156762009-09-11 12:12:54 +0200289{
mingo39aeb522009-09-14 20:04:48 +0200290 struct sched_atom *event, *wakee_event;
Ingo Molnarec156762009-09-11 12:12:54 +0200291
292 event = get_new_event(task, timestamp);
293 event->type = SCHED_EVENT_WAKEUP;
294 event->wakee = wakee;
295
296 wakee_event = last_event(wakee);
297 if (!wakee_event || wakee_event->type != SCHED_EVENT_SLEEP) {
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -0300298 sched->targetless_wakeups++;
Ingo Molnarec156762009-09-11 12:12:54 +0200299 return;
300 }
301 if (wakee_event->wait_sem) {
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -0300302 sched->multitarget_wakeups++;
Ingo Molnarec156762009-09-11 12:12:54 +0200303 return;
304 }
305
Arnaldo Carvalho de Melo36479482009-11-24 12:05:16 -0200306 wakee_event->wait_sem = zalloc(sizeof(*wakee_event->wait_sem));
Ingo Molnarec156762009-09-11 12:12:54 +0200307 sem_init(wakee_event->wait_sem, 0, 0);
308 wakee_event->specific_wait = 1;
309 event->wait_sem = wakee_event->wait_sem;
310
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -0300311 sched->nr_wakeup_events++;
Ingo Molnarec156762009-09-11 12:12:54 +0200312}
313
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -0300314static void add_sched_event_sleep(struct perf_sched *sched, struct task_desc *task,
315 u64 timestamp, u64 task_state __maybe_unused)
Ingo Molnarec156762009-09-11 12:12:54 +0200316{
mingo39aeb522009-09-14 20:04:48 +0200317 struct sched_atom *event = get_new_event(task, timestamp);
Ingo Molnarec156762009-09-11 12:12:54 +0200318
319 event->type = SCHED_EVENT_SLEEP;
320
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -0300321 sched->nr_sleep_events++;
Ingo Molnarec156762009-09-11 12:12:54 +0200322}
323
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -0300324static struct task_desc *register_pid(struct perf_sched *sched,
325 unsigned long pid, const char *comm)
Ingo Molnarec156762009-09-11 12:12:54 +0200326{
327 struct task_desc *task;
328
329 BUG_ON(pid >= MAX_PID);
330
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -0300331 task = sched->pid_to_task[pid];
Ingo Molnarec156762009-09-11 12:12:54 +0200332
333 if (task)
334 return task;
335
Arnaldo Carvalho de Melo36479482009-11-24 12:05:16 -0200336 task = zalloc(sizeof(*task));
Ingo Molnarec156762009-09-11 12:12:54 +0200337 task->pid = pid;
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -0300338 task->nr = sched->nr_tasks;
Ingo Molnarec156762009-09-11 12:12:54 +0200339 strcpy(task->comm, comm);
340 /*
341 * every task starts in sleeping state - this gets ignored
342 * if there's no wakeup pointing to this sleep state:
343 */
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -0300344 add_sched_event_sleep(sched, task, 0, 0);
Ingo Molnarec156762009-09-11 12:12:54 +0200345
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -0300346 sched->pid_to_task[pid] = task;
347 sched->nr_tasks++;
348 sched->tasks = realloc(sched->tasks, sched->nr_tasks * sizeof(struct task_task *));
349 BUG_ON(!sched->tasks);
350 sched->tasks[task->nr] = task;
Ingo Molnarec156762009-09-11 12:12:54 +0200351
Ingo Molnarad236fd2009-09-11 12:12:54 +0200352 if (verbose)
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -0300353 printf("registered task #%ld, PID %ld (%s)\n", sched->nr_tasks, pid, comm);
Ingo Molnarec156762009-09-11 12:12:54 +0200354
355 return task;
356}
357
358
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -0300359static void print_task_traces(struct perf_sched *sched)
Ingo Molnarec156762009-09-11 12:12:54 +0200360{
361 struct task_desc *task;
362 unsigned long i;
363
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -0300364 for (i = 0; i < sched->nr_tasks; i++) {
365 task = sched->tasks[i];
Ingo Molnarad236fd2009-09-11 12:12:54 +0200366 printf("task %6ld (%20s:%10ld), nr_events: %ld\n",
Ingo Molnarec156762009-09-11 12:12:54 +0200367 task->nr, task->comm, task->pid, task->nr_events);
368 }
369}
370
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -0300371static void add_cross_task_wakeups(struct perf_sched *sched)
Ingo Molnarec156762009-09-11 12:12:54 +0200372{
373 struct task_desc *task1, *task2;
374 unsigned long i, j;
375
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -0300376 for (i = 0; i < sched->nr_tasks; i++) {
377 task1 = sched->tasks[i];
Ingo Molnarec156762009-09-11 12:12:54 +0200378 j = i + 1;
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -0300379 if (j == sched->nr_tasks)
Ingo Molnarec156762009-09-11 12:12:54 +0200380 j = 0;
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -0300381 task2 = sched->tasks[j];
382 add_sched_event_wakeup(sched, task1, 0, task2);
Ingo Molnarec156762009-09-11 12:12:54 +0200383 }
384}
385
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -0300386static void perf_sched__process_event(struct perf_sched *sched,
387 struct sched_atom *atom)
Ingo Molnarec156762009-09-11 12:12:54 +0200388{
389 int ret = 0;
Ingo Molnarec156762009-09-11 12:12:54 +0200390
mingo39aeb522009-09-14 20:04:48 +0200391 switch (atom->type) {
Ingo Molnarec156762009-09-11 12:12:54 +0200392 case SCHED_EVENT_RUN:
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -0300393 burn_nsecs(sched, atom->duration);
Ingo Molnarec156762009-09-11 12:12:54 +0200394 break;
395 case SCHED_EVENT_SLEEP:
mingo39aeb522009-09-14 20:04:48 +0200396 if (atom->wait_sem)
397 ret = sem_wait(atom->wait_sem);
Ingo Molnarec156762009-09-11 12:12:54 +0200398 BUG_ON(ret);
399 break;
400 case SCHED_EVENT_WAKEUP:
mingo39aeb522009-09-14 20:04:48 +0200401 if (atom->wait_sem)
402 ret = sem_post(atom->wait_sem);
Ingo Molnarec156762009-09-11 12:12:54 +0200403 BUG_ON(ret);
404 break;
Mike Galbraith55ffb7a2009-10-10 14:46:04 +0200405 case SCHED_EVENT_MIGRATION:
406 break;
Ingo Molnarec156762009-09-11 12:12:54 +0200407 default:
408 BUG_ON(1);
409 }
410}
411
Ingo Molnarb1ffe8f2009-09-11 12:12:54 +0200412static u64 get_cpu_usage_nsec_parent(void)
Ingo Molnarec156762009-09-11 12:12:54 +0200413{
414 struct rusage ru;
Ingo Molnarb1ffe8f2009-09-11 12:12:54 +0200415 u64 sum;
Ingo Molnarec156762009-09-11 12:12:54 +0200416 int err;
417
418 err = getrusage(RUSAGE_SELF, &ru);
419 BUG_ON(err);
420
421 sum = ru.ru_utime.tv_sec*1e9 + ru.ru_utime.tv_usec*1e3;
422 sum += ru.ru_stime.tv_sec*1e9 + ru.ru_stime.tv_usec*1e3;
423
424 return sum;
425}
426
Xiao Guangrongc0c9e722009-12-09 17:51:30 +0800427static int self_open_counters(void)
Ingo Molnarec156762009-09-11 12:12:54 +0200428{
Xiao Guangrongc0c9e722009-12-09 17:51:30 +0800429 struct perf_event_attr attr;
430 int fd;
431
432 memset(&attr, 0, sizeof(attr));
433
434 attr.type = PERF_TYPE_SOFTWARE;
435 attr.config = PERF_COUNT_SW_TASK_CLOCK;
436
437 fd = sys_perf_event_open(&attr, 0, -1, -1, 0);
438
439 if (fd < 0)
Namhyung Kim60b7d142012-09-12 11:11:06 +0900440 pr_err("Error: sys_perf_event_open() syscall returned "
441 "with %d (%s)\n", fd, strerror(errno));
Xiao Guangrongc0c9e722009-12-09 17:51:30 +0800442 return fd;
443}
444
445static u64 get_cpu_usage_nsec_self(int fd)
446{
447 u64 runtime;
Ingo Molnarec156762009-09-11 12:12:54 +0200448 int ret;
449
Xiao Guangrongc0c9e722009-12-09 17:51:30 +0800450 ret = read(fd, &runtime, sizeof(runtime));
451 BUG_ON(ret != sizeof(runtime));
Ingo Molnarec156762009-09-11 12:12:54 +0200452
Xiao Guangrongc0c9e722009-12-09 17:51:30 +0800453 return runtime;
Ingo Molnarec156762009-09-11 12:12:54 +0200454}
455
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -0300456struct sched_thread_parms {
457 struct task_desc *task;
458 struct perf_sched *sched;
459};
460
Ingo Molnarec156762009-09-11 12:12:54 +0200461static void *thread_func(void *ctx)
462{
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -0300463 struct sched_thread_parms *parms = ctx;
464 struct task_desc *this_task = parms->task;
465 struct perf_sched *sched = parms->sched;
Ingo Molnarb1ffe8f2009-09-11 12:12:54 +0200466 u64 cpu_usage_0, cpu_usage_1;
Ingo Molnarec156762009-09-11 12:12:54 +0200467 unsigned long i, ret;
468 char comm2[22];
Xiao Guangrongc0c9e722009-12-09 17:51:30 +0800469 int fd;
Ingo Molnarec156762009-09-11 12:12:54 +0200470
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -0300471 free(parms);
472
Ingo Molnarec156762009-09-11 12:12:54 +0200473 sprintf(comm2, ":%s", this_task->comm);
474 prctl(PR_SET_NAME, comm2);
Xiao Guangrongc0c9e722009-12-09 17:51:30 +0800475 fd = self_open_counters();
Arnaldo Carvalho de Meloa116e052012-09-08 22:53:06 -0300476 if (fd < 0)
477 return NULL;
Ingo Molnarec156762009-09-11 12:12:54 +0200478again:
479 ret = sem_post(&this_task->ready_for_work);
480 BUG_ON(ret);
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -0300481 ret = pthread_mutex_lock(&sched->start_work_mutex);
Ingo Molnarec156762009-09-11 12:12:54 +0200482 BUG_ON(ret);
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -0300483 ret = pthread_mutex_unlock(&sched->start_work_mutex);
Ingo Molnarec156762009-09-11 12:12:54 +0200484 BUG_ON(ret);
Ingo Molnarec156762009-09-11 12:12:54 +0200485
Xiao Guangrongc0c9e722009-12-09 17:51:30 +0800486 cpu_usage_0 = get_cpu_usage_nsec_self(fd);
Ingo Molnarec156762009-09-11 12:12:54 +0200487
488 for (i = 0; i < this_task->nr_events; i++) {
489 this_task->curr_event = i;
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -0300490 perf_sched__process_event(sched, this_task->atoms[i]);
Ingo Molnarec156762009-09-11 12:12:54 +0200491 }
492
Xiao Guangrongc0c9e722009-12-09 17:51:30 +0800493 cpu_usage_1 = get_cpu_usage_nsec_self(fd);
Ingo Molnarec156762009-09-11 12:12:54 +0200494 this_task->cpu_usage = cpu_usage_1 - cpu_usage_0;
Ingo Molnarec156762009-09-11 12:12:54 +0200495 ret = sem_post(&this_task->work_done_sem);
496 BUG_ON(ret);
Ingo Molnarec156762009-09-11 12:12:54 +0200497
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -0300498 ret = pthread_mutex_lock(&sched->work_done_wait_mutex);
Ingo Molnarec156762009-09-11 12:12:54 +0200499 BUG_ON(ret);
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -0300500 ret = pthread_mutex_unlock(&sched->work_done_wait_mutex);
Ingo Molnarec156762009-09-11 12:12:54 +0200501 BUG_ON(ret);
Ingo Molnarec156762009-09-11 12:12:54 +0200502
503 goto again;
504}
505
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -0300506static void create_tasks(struct perf_sched *sched)
Ingo Molnarec156762009-09-11 12:12:54 +0200507{
508 struct task_desc *task;
509 pthread_attr_t attr;
510 unsigned long i;
511 int err;
512
513 err = pthread_attr_init(&attr);
514 BUG_ON(err);
Jiri Pirko12f7e032011-01-10 14:14:23 -0200515 err = pthread_attr_setstacksize(&attr,
516 (size_t) max(16 * 1024, PTHREAD_STACK_MIN));
Ingo Molnarec156762009-09-11 12:12:54 +0200517 BUG_ON(err);
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -0300518 err = pthread_mutex_lock(&sched->start_work_mutex);
Ingo Molnarec156762009-09-11 12:12:54 +0200519 BUG_ON(err);
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -0300520 err = pthread_mutex_lock(&sched->work_done_wait_mutex);
Ingo Molnarec156762009-09-11 12:12:54 +0200521 BUG_ON(err);
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -0300522 for (i = 0; i < sched->nr_tasks; i++) {
523 struct sched_thread_parms *parms = malloc(sizeof(*parms));
524 BUG_ON(parms == NULL);
525 parms->task = task = sched->tasks[i];
526 parms->sched = sched;
Ingo Molnarec156762009-09-11 12:12:54 +0200527 sem_init(&task->sleep_sem, 0, 0);
528 sem_init(&task->ready_for_work, 0, 0);
529 sem_init(&task->work_done_sem, 0, 0);
530 task->curr_event = 0;
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -0300531 err = pthread_create(&task->thread, &attr, thread_func, parms);
Ingo Molnarec156762009-09-11 12:12:54 +0200532 BUG_ON(err);
533 }
534}
535
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -0300536static void wait_for_tasks(struct perf_sched *sched)
Ingo Molnarec156762009-09-11 12:12:54 +0200537{
Ingo Molnarb1ffe8f2009-09-11 12:12:54 +0200538 u64 cpu_usage_0, cpu_usage_1;
Ingo Molnarec156762009-09-11 12:12:54 +0200539 struct task_desc *task;
540 unsigned long i, ret;
541
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -0300542 sched->start_time = get_nsecs();
543 sched->cpu_usage = 0;
544 pthread_mutex_unlock(&sched->work_done_wait_mutex);
Ingo Molnarec156762009-09-11 12:12:54 +0200545
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -0300546 for (i = 0; i < sched->nr_tasks; i++) {
547 task = sched->tasks[i];
Ingo Molnarec156762009-09-11 12:12:54 +0200548 ret = sem_wait(&task->ready_for_work);
549 BUG_ON(ret);
550 sem_init(&task->ready_for_work, 0, 0);
551 }
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -0300552 ret = pthread_mutex_lock(&sched->work_done_wait_mutex);
Ingo Molnarec156762009-09-11 12:12:54 +0200553 BUG_ON(ret);
554
555 cpu_usage_0 = get_cpu_usage_nsec_parent();
556
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -0300557 pthread_mutex_unlock(&sched->start_work_mutex);
Ingo Molnarec156762009-09-11 12:12:54 +0200558
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -0300559 for (i = 0; i < sched->nr_tasks; i++) {
560 task = sched->tasks[i];
Ingo Molnarec156762009-09-11 12:12:54 +0200561 ret = sem_wait(&task->work_done_sem);
562 BUG_ON(ret);
563 sem_init(&task->work_done_sem, 0, 0);
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -0300564 sched->cpu_usage += task->cpu_usage;
Ingo Molnarec156762009-09-11 12:12:54 +0200565 task->cpu_usage = 0;
566 }
567
568 cpu_usage_1 = get_cpu_usage_nsec_parent();
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -0300569 if (!sched->runavg_cpu_usage)
570 sched->runavg_cpu_usage = sched->cpu_usage;
571 sched->runavg_cpu_usage = (sched->runavg_cpu_usage * 9 + sched->cpu_usage) / 10;
Ingo Molnarec156762009-09-11 12:12:54 +0200572
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -0300573 sched->parent_cpu_usage = cpu_usage_1 - cpu_usage_0;
574 if (!sched->runavg_parent_cpu_usage)
575 sched->runavg_parent_cpu_usage = sched->parent_cpu_usage;
576 sched->runavg_parent_cpu_usage = (sched->runavg_parent_cpu_usage * 9 +
577 sched->parent_cpu_usage)/10;
Ingo Molnarec156762009-09-11 12:12:54 +0200578
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -0300579 ret = pthread_mutex_lock(&sched->start_work_mutex);
Ingo Molnarec156762009-09-11 12:12:54 +0200580 BUG_ON(ret);
581
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -0300582 for (i = 0; i < sched->nr_tasks; i++) {
583 task = sched->tasks[i];
Ingo Molnarec156762009-09-11 12:12:54 +0200584 sem_init(&task->sleep_sem, 0, 0);
585 task->curr_event = 0;
586 }
587}
588
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -0300589static void run_one_test(struct perf_sched *sched)
Ingo Molnarec156762009-09-11 12:12:54 +0200590{
Kyle McMartinfb7d0b32011-01-24 11:13:04 -0500591 u64 T0, T1, delta, avg_delta, fluct;
Ingo Molnarec156762009-09-11 12:12:54 +0200592
593 T0 = get_nsecs();
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -0300594 wait_for_tasks(sched);
Ingo Molnarec156762009-09-11 12:12:54 +0200595 T1 = get_nsecs();
596
597 delta = T1 - T0;
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -0300598 sched->sum_runtime += delta;
599 sched->nr_runs++;
Ingo Molnarec156762009-09-11 12:12:54 +0200600
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -0300601 avg_delta = sched->sum_runtime / sched->nr_runs;
Ingo Molnarec156762009-09-11 12:12:54 +0200602 if (delta < avg_delta)
603 fluct = avg_delta - delta;
604 else
605 fluct = delta - avg_delta;
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -0300606 sched->sum_fluct += fluct;
607 if (!sched->run_avg)
608 sched->run_avg = delta;
609 sched->run_avg = (sched->run_avg * 9 + delta) / 10;
Ingo Molnarec156762009-09-11 12:12:54 +0200610
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -0300611 printf("#%-3ld: %0.3f, ", sched->nr_runs, (double)delta / 1000000.0);
Ingo Molnarec156762009-09-11 12:12:54 +0200612
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -0300613 printf("ravg: %0.2f, ", (double)sched->run_avg / 1e6);
Ingo Molnarec156762009-09-11 12:12:54 +0200614
Ingo Molnarad236fd2009-09-11 12:12:54 +0200615 printf("cpu: %0.2f / %0.2f",
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -0300616 (double)sched->cpu_usage / 1e6, (double)sched->runavg_cpu_usage / 1e6);
Ingo Molnarec156762009-09-11 12:12:54 +0200617
618#if 0
619 /*
Ingo Molnarfbf94822009-09-11 12:12:54 +0200620 * rusage statistics done by the parent, these are less
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -0300621 * accurate than the sched->sum_exec_runtime based statistics:
Ingo Molnarfbf94822009-09-11 12:12:54 +0200622 */
Ingo Molnarad236fd2009-09-11 12:12:54 +0200623 printf(" [%0.2f / %0.2f]",
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -0300624 (double)sched->parent_cpu_usage/1e6,
625 (double)sched->runavg_parent_cpu_usage/1e6);
Ingo Molnarec156762009-09-11 12:12:54 +0200626#endif
627
Ingo Molnarad236fd2009-09-11 12:12:54 +0200628 printf("\n");
Ingo Molnarec156762009-09-11 12:12:54 +0200629
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -0300630 if (sched->nr_sleep_corrections)
631 printf(" (%ld sleep corrections)\n", sched->nr_sleep_corrections);
632 sched->nr_sleep_corrections = 0;
Ingo Molnarec156762009-09-11 12:12:54 +0200633}
634
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -0300635static void test_calibrations(struct perf_sched *sched)
Ingo Molnarec156762009-09-11 12:12:54 +0200636{
Ingo Molnarb1ffe8f2009-09-11 12:12:54 +0200637 u64 T0, T1;
Ingo Molnarec156762009-09-11 12:12:54 +0200638
639 T0 = get_nsecs();
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -0300640 burn_nsecs(sched, 1e6);
Ingo Molnarec156762009-09-11 12:12:54 +0200641 T1 = get_nsecs();
642
Arnaldo Carvalho de Melo9486aa32011-01-22 20:37:02 -0200643 printf("the run test took %" PRIu64 " nsecs\n", T1 - T0);
Ingo Molnarec156762009-09-11 12:12:54 +0200644
645 T0 = get_nsecs();
646 sleep_nsecs(1e6);
647 T1 = get_nsecs();
648
Arnaldo Carvalho de Melo9486aa32011-01-22 20:37:02 -0200649 printf("the sleep test took %" PRIu64 " nsecs\n", T1 - T0);
Ingo Molnarec156762009-09-11 12:12:54 +0200650}
651
Arnaldo Carvalho de Meloa116e052012-09-08 22:53:06 -0300652static int
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -0300653replay_wakeup_event(struct perf_sched *sched,
Arnaldo Carvalho de Melo9ec3f4e2012-09-11 19:29:17 -0300654 struct perf_evsel *evsel, struct perf_sample *sample,
655 struct machine *machine __maybe_unused)
Ingo Molnarec156762009-09-11 12:12:54 +0200656{
Arnaldo Carvalho de Melo9ec3f4e2012-09-11 19:29:17 -0300657 const char *comm = perf_evsel__strval(evsel, sample, "comm");
658 const u32 pid = perf_evsel__intval(evsel, sample, "pid");
Frederic Weisbecker419ab0d2009-09-12 03:59:01 +0200659 struct task_desc *waker, *wakee;
660
661 if (verbose) {
Arnaldo Carvalho de Melo2b7fcbc2012-09-11 19:29:17 -0300662 printf("sched_wakeup event %p\n", evsel);
Frederic Weisbecker419ab0d2009-09-12 03:59:01 +0200663
Arnaldo Carvalho de Melo9ec3f4e2012-09-11 19:29:17 -0300664 printf(" ... pid %d woke up %s/%d\n", sample->tid, comm, pid);
Frederic Weisbecker419ab0d2009-09-12 03:59:01 +0200665 }
666
Arnaldo Carvalho de Melo2b7fcbc2012-09-11 19:29:17 -0300667 waker = register_pid(sched, sample->tid, "<unknown>");
Arnaldo Carvalho de Melo9ec3f4e2012-09-11 19:29:17 -0300668 wakee = register_pid(sched, pid, comm);
Frederic Weisbecker419ab0d2009-09-12 03:59:01 +0200669
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -0300670 add_sched_event_wakeup(sched, waker, sample->time, wakee);
Arnaldo Carvalho de Meloa116e052012-09-08 22:53:06 -0300671 return 0;
Frederic Weisbecker419ab0d2009-09-12 03:59:01 +0200672}
673
Arnaldo Carvalho de Melo9ec3f4e2012-09-11 19:29:17 -0300674static int replay_switch_event(struct perf_sched *sched,
675 struct perf_evsel *evsel,
676 struct perf_sample *sample,
677 struct machine *machine __maybe_unused)
Frederic Weisbecker419ab0d2009-09-12 03:59:01 +0200678{
Arnaldo Carvalho de Melo9ec3f4e2012-09-11 19:29:17 -0300679 const char *prev_comm = perf_evsel__strval(evsel, sample, "prev_comm"),
680 *next_comm = perf_evsel__strval(evsel, sample, "next_comm");
681 const u32 prev_pid = perf_evsel__intval(evsel, sample, "prev_pid"),
682 next_pid = perf_evsel__intval(evsel, sample, "next_pid");
683 const u64 prev_state = perf_evsel__intval(evsel, sample, "prev_state");
Irina Tirdea1d037ca2012-09-11 01:15:03 +0300684 struct task_desc *prev, __maybe_unused *next;
Arnaldo Carvalho de Melo7f7f8d02012-08-07 11:33:42 -0300685 u64 timestamp0, timestamp = sample->time;
686 int cpu = sample->cpu;
Ingo Molnarfbf94822009-09-11 12:12:54 +0200687 s64 delta;
688
Ingo Molnarad236fd2009-09-11 12:12:54 +0200689 if (verbose)
Arnaldo Carvalho de Melo2b7fcbc2012-09-11 19:29:17 -0300690 printf("sched_switch event %p\n", evsel);
Ingo Molnarad236fd2009-09-11 12:12:54 +0200691
Ingo Molnarfbf94822009-09-11 12:12:54 +0200692 if (cpu >= MAX_CPUS || cpu < 0)
Arnaldo Carvalho de Meloa116e052012-09-08 22:53:06 -0300693 return 0;
Ingo Molnarfbf94822009-09-11 12:12:54 +0200694
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -0300695 timestamp0 = sched->cpu_last_switched[cpu];
Ingo Molnarfbf94822009-09-11 12:12:54 +0200696 if (timestamp0)
697 delta = timestamp - timestamp0;
698 else
699 delta = 0;
700
Arnaldo Carvalho de Meloa116e052012-09-08 22:53:06 -0300701 if (delta < 0) {
Namhyung Kim60b7d142012-09-12 11:11:06 +0900702 pr_err("hm, delta: %" PRIu64 " < 0 ?\n", delta);
Arnaldo Carvalho de Meloa116e052012-09-08 22:53:06 -0300703 return -1;
704 }
Ingo Molnarfbf94822009-09-11 12:12:54 +0200705
Arnaldo Carvalho de Melo9ec3f4e2012-09-11 19:29:17 -0300706 pr_debug(" ... switch from %s/%d to %s/%d [ran %" PRIu64 " nsecs]\n",
707 prev_comm, prev_pid, next_comm, next_pid, delta);
Ingo Molnarfbf94822009-09-11 12:12:54 +0200708
Arnaldo Carvalho de Melo9ec3f4e2012-09-11 19:29:17 -0300709 prev = register_pid(sched, prev_pid, prev_comm);
710 next = register_pid(sched, next_pid, next_comm);
Ingo Molnarfbf94822009-09-11 12:12:54 +0200711
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -0300712 sched->cpu_last_switched[cpu] = timestamp;
Ingo Molnarfbf94822009-09-11 12:12:54 +0200713
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -0300714 add_sched_event_run(sched, prev, timestamp, delta);
Arnaldo Carvalho de Melo9ec3f4e2012-09-11 19:29:17 -0300715 add_sched_event_sleep(sched, prev, timestamp, prev_state);
Arnaldo Carvalho de Meloa116e052012-09-08 22:53:06 -0300716
717 return 0;
Ingo Molnarfbf94822009-09-11 12:12:54 +0200718}
719
Arnaldo Carvalho de Melo9ec3f4e2012-09-11 19:29:17 -0300720static int replay_fork_event(struct perf_sched *sched, struct perf_evsel *evsel,
721 struct perf_sample *sample)
Frederic Weisbecker419ab0d2009-09-12 03:59:01 +0200722{
Arnaldo Carvalho de Melo9ec3f4e2012-09-11 19:29:17 -0300723 const char *parent_comm = perf_evsel__strval(evsel, sample, "parent_comm"),
724 *child_comm = perf_evsel__strval(evsel, sample, "child_comm");
725 const u32 parent_pid = perf_evsel__intval(evsel, sample, "parent_pid"),
726 child_pid = perf_evsel__intval(evsel, sample, "child_pid");
727
Frederic Weisbecker419ab0d2009-09-12 03:59:01 +0200728 if (verbose) {
Arnaldo Carvalho de Melo2b7fcbc2012-09-11 19:29:17 -0300729 printf("sched_fork event %p\n", evsel);
Arnaldo Carvalho de Melo9ec3f4e2012-09-11 19:29:17 -0300730 printf("... parent: %s/%d\n", parent_comm, parent_pid);
731 printf("... child: %s/%d\n", child_comm, child_pid);
Frederic Weisbecker419ab0d2009-09-12 03:59:01 +0200732 }
Arnaldo Carvalho de Melo9ec3f4e2012-09-11 19:29:17 -0300733
734 register_pid(sched, parent_pid, parent_comm);
735 register_pid(sched, child_pid, child_comm);
Arnaldo Carvalho de Meloa116e052012-09-08 22:53:06 -0300736 return 0;
Frederic Weisbecker419ab0d2009-09-12 03:59:01 +0200737}
738
Ingo Molnarb1ffe8f2009-09-11 12:12:54 +0200739struct sort_dimension {
740 const char *name;
Ingo Molnarb5fae122009-09-11 12:12:54 +0200741 sort_fn_t cmp;
Ingo Molnarb1ffe8f2009-09-11 12:12:54 +0200742 struct list_head list;
743};
744
Frederic Weisbeckerdaa1d7a2009-09-13 03:36:29 +0200745static int
mingo39aeb522009-09-14 20:04:48 +0200746thread_lat_cmp(struct list_head *list, struct work_atoms *l, struct work_atoms *r)
Frederic Weisbeckerdaa1d7a2009-09-13 03:36:29 +0200747{
748 struct sort_dimension *sort;
749 int ret = 0;
750
Ingo Molnarb5fae122009-09-11 12:12:54 +0200751 BUG_ON(list_empty(list));
752
Frederic Weisbeckerdaa1d7a2009-09-13 03:36:29 +0200753 list_for_each_entry(sort, list, list) {
754 ret = sort->cmp(l, r);
755 if (ret)
756 return ret;
757 }
758
759 return ret;
760}
761
mingo39aeb522009-09-14 20:04:48 +0200762static struct work_atoms *
Ingo Molnarb5fae122009-09-11 12:12:54 +0200763thread_atoms_search(struct rb_root *root, struct thread *thread,
764 struct list_head *sort_list)
765{
766 struct rb_node *node = root->rb_node;
mingo39aeb522009-09-14 20:04:48 +0200767 struct work_atoms key = { .thread = thread };
Ingo Molnarb5fae122009-09-11 12:12:54 +0200768
769 while (node) {
mingo39aeb522009-09-14 20:04:48 +0200770 struct work_atoms *atoms;
Ingo Molnarb5fae122009-09-11 12:12:54 +0200771 int cmp;
772
mingo39aeb522009-09-14 20:04:48 +0200773 atoms = container_of(node, struct work_atoms, node);
Ingo Molnarb5fae122009-09-11 12:12:54 +0200774
775 cmp = thread_lat_cmp(sort_list, &key, atoms);
776 if (cmp > 0)
777 node = node->rb_left;
778 else if (cmp < 0)
779 node = node->rb_right;
780 else {
781 BUG_ON(thread != atoms->thread);
782 return atoms;
783 }
784 }
785 return NULL;
786}
787
Frederic Weisbeckercdce9d72009-09-12 08:06:14 +0200788static void
mingo39aeb522009-09-14 20:04:48 +0200789__thread_latency_insert(struct rb_root *root, struct work_atoms *data,
Frederic Weisbeckerdaa1d7a2009-09-13 03:36:29 +0200790 struct list_head *sort_list)
Frederic Weisbeckercdce9d72009-09-12 08:06:14 +0200791{
792 struct rb_node **new = &(root->rb_node), *parent = NULL;
793
794 while (*new) {
mingo39aeb522009-09-14 20:04:48 +0200795 struct work_atoms *this;
Frederic Weisbeckerdaa1d7a2009-09-13 03:36:29 +0200796 int cmp;
Frederic Weisbeckercdce9d72009-09-12 08:06:14 +0200797
mingo39aeb522009-09-14 20:04:48 +0200798 this = container_of(*new, struct work_atoms, node);
Frederic Weisbeckercdce9d72009-09-12 08:06:14 +0200799 parent = *new;
Frederic Weisbeckerdaa1d7a2009-09-13 03:36:29 +0200800
801 cmp = thread_lat_cmp(sort_list, data, this);
802
803 if (cmp > 0)
Frederic Weisbeckercdce9d72009-09-12 08:06:14 +0200804 new = &((*new)->rb_left);
Frederic Weisbeckercdce9d72009-09-12 08:06:14 +0200805 else
Frederic Weisbeckerdaa1d7a2009-09-13 03:36:29 +0200806 new = &((*new)->rb_right);
Frederic Weisbeckercdce9d72009-09-12 08:06:14 +0200807 }
808
809 rb_link_node(&data->node, parent, new);
810 rb_insert_color(&data->node, root);
811}
812
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -0300813static int thread_atoms_insert(struct perf_sched *sched, struct thread *thread)
Frederic Weisbeckercdce9d72009-09-12 08:06:14 +0200814{
Arnaldo Carvalho de Melo36479482009-11-24 12:05:16 -0200815 struct work_atoms *atoms = zalloc(sizeof(*atoms));
Arnaldo Carvalho de Meloa116e052012-09-08 22:53:06 -0300816 if (!atoms) {
817 pr_err("No memory at %s\n", __func__);
818 return -1;
819 }
Frederic Weisbeckercdce9d72009-09-12 08:06:14 +0200820
Frederic Weisbecker17562202009-09-12 23:11:32 +0200821 atoms->thread = thread;
mingo39aeb522009-09-14 20:04:48 +0200822 INIT_LIST_HEAD(&atoms->work_list);
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -0300823 __thread_latency_insert(&sched->atom_root, atoms, &sched->cmp_pid);
Arnaldo Carvalho de Meloa116e052012-09-08 22:53:06 -0300824 return 0;
Frederic Weisbeckercdce9d72009-09-12 08:06:14 +0200825}
826
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -0300827static int latency_fork_event(struct perf_sched *sched __maybe_unused,
Arnaldo Carvalho de Melo9ec3f4e2012-09-11 19:29:17 -0300828 struct perf_evsel *evsel __maybe_unused,
829 struct perf_sample *sample __maybe_unused)
Frederic Weisbeckercdce9d72009-09-12 08:06:14 +0200830{
831 /* should insert the newcomer */
Arnaldo Carvalho de Meloa116e052012-09-08 22:53:06 -0300832 return 0;
Frederic Weisbeckercdce9d72009-09-12 08:06:14 +0200833}
834
Arnaldo Carvalho de Melo9ec3f4e2012-09-11 19:29:17 -0300835static char sched_out_state(u64 prev_state)
Frederic Weisbeckercdce9d72009-09-12 08:06:14 +0200836{
837 const char *str = TASK_STATE_TO_CHAR_STR;
838
Arnaldo Carvalho de Melo9ec3f4e2012-09-11 19:29:17 -0300839 return str[prev_state];
Frederic Weisbeckercdce9d72009-09-12 08:06:14 +0200840}
841
Arnaldo Carvalho de Meloa116e052012-09-08 22:53:06 -0300842static int
mingo39aeb522009-09-14 20:04:48 +0200843add_sched_out_event(struct work_atoms *atoms,
844 char run_state,
845 u64 timestamp)
Frederic Weisbeckercdce9d72009-09-12 08:06:14 +0200846{
Arnaldo Carvalho de Melo36479482009-11-24 12:05:16 -0200847 struct work_atom *atom = zalloc(sizeof(*atom));
Arnaldo Carvalho de Meloa116e052012-09-08 22:53:06 -0300848 if (!atom) {
849 pr_err("Non memory at %s", __func__);
850 return -1;
851 }
Frederic Weisbeckercdce9d72009-09-12 08:06:14 +0200852
Frederic Weisbeckeraa1ab9d2009-09-14 03:01:12 +0200853 atom->sched_out_time = timestamp;
854
mingo39aeb522009-09-14 20:04:48 +0200855 if (run_state == 'R') {
Ingo Molnarb1ffe8f2009-09-11 12:12:54 +0200856 atom->state = THREAD_WAIT_CPU;
Frederic Weisbeckeraa1ab9d2009-09-14 03:01:12 +0200857 atom->wake_up_time = atom->sched_out_time;
Frederic Weisbeckerc6ced612009-09-13 00:46:19 +0200858 }
859
mingo39aeb522009-09-14 20:04:48 +0200860 list_add_tail(&atom->list, &atoms->work_list);
Arnaldo Carvalho de Meloa116e052012-09-08 22:53:06 -0300861 return 0;
Frederic Weisbeckercdce9d72009-09-12 08:06:14 +0200862}
863
864static void
Irina Tirdea1d037ca2012-09-11 01:15:03 +0300865add_runtime_event(struct work_atoms *atoms, u64 delta,
866 u64 timestamp __maybe_unused)
mingo39aeb522009-09-14 20:04:48 +0200867{
868 struct work_atom *atom;
869
870 BUG_ON(list_empty(&atoms->work_list));
871
872 atom = list_entry(atoms->work_list.prev, struct work_atom, list);
873
874 atom->runtime += delta;
875 atoms->total_runtime += delta;
876}
877
878static void
879add_sched_in_event(struct work_atoms *atoms, u64 timestamp)
Frederic Weisbeckercdce9d72009-09-12 08:06:14 +0200880{
Ingo Molnarb1ffe8f2009-09-11 12:12:54 +0200881 struct work_atom *atom;
Frederic Weisbecker66685672009-09-13 01:56:25 +0200882 u64 delta;
Frederic Weisbeckercdce9d72009-09-12 08:06:14 +0200883
mingo39aeb522009-09-14 20:04:48 +0200884 if (list_empty(&atoms->work_list))
Frederic Weisbeckercdce9d72009-09-12 08:06:14 +0200885 return;
886
mingo39aeb522009-09-14 20:04:48 +0200887 atom = list_entry(atoms->work_list.prev, struct work_atom, list);
Frederic Weisbeckercdce9d72009-09-12 08:06:14 +0200888
Ingo Molnarb1ffe8f2009-09-11 12:12:54 +0200889 if (atom->state != THREAD_WAIT_CPU)
Frederic Weisbeckercdce9d72009-09-12 08:06:14 +0200890 return;
891
Ingo Molnarb1ffe8f2009-09-11 12:12:54 +0200892 if (timestamp < atom->wake_up_time) {
893 atom->state = THREAD_IGNORE;
Frederic Weisbeckercdce9d72009-09-12 08:06:14 +0200894 return;
895 }
896
Ingo Molnarb1ffe8f2009-09-11 12:12:54 +0200897 atom->state = THREAD_SCHED_IN;
898 atom->sched_in_time = timestamp;
Frederic Weisbecker66685672009-09-13 01:56:25 +0200899
Ingo Molnarb1ffe8f2009-09-11 12:12:54 +0200900 delta = atom->sched_in_time - atom->wake_up_time;
Frederic Weisbecker66685672009-09-13 01:56:25 +0200901 atoms->total_lat += delta;
Frederic Weisbecker3786310a2009-12-09 21:40:08 +0100902 if (delta > atoms->max_lat) {
Frederic Weisbecker66685672009-09-13 01:56:25 +0200903 atoms->max_lat = delta;
Frederic Weisbecker3786310a2009-12-09 21:40:08 +0100904 atoms->max_lat_at = timestamp;
905 }
Frederic Weisbecker66685672009-09-13 01:56:25 +0200906 atoms->nb_atoms++;
Frederic Weisbeckercdce9d72009-09-12 08:06:14 +0200907}
908
Arnaldo Carvalho de Melo9ec3f4e2012-09-11 19:29:17 -0300909static int latency_switch_event(struct perf_sched *sched,
910 struct perf_evsel *evsel,
911 struct perf_sample *sample,
912 struct machine *machine)
Frederic Weisbeckercdce9d72009-09-12 08:06:14 +0200913{
Arnaldo Carvalho de Melo9ec3f4e2012-09-11 19:29:17 -0300914 const u32 prev_pid = perf_evsel__intval(evsel, sample, "prev_pid"),
915 next_pid = perf_evsel__intval(evsel, sample, "next_pid");
916 const u64 prev_state = perf_evsel__intval(evsel, sample, "prev_state");
mingo39aeb522009-09-14 20:04:48 +0200917 struct work_atoms *out_events, *in_events;
Frederic Weisbeckercdce9d72009-09-12 08:06:14 +0200918 struct thread *sched_out, *sched_in;
Arnaldo Carvalho de Melo7f7f8d02012-08-07 11:33:42 -0300919 u64 timestamp0, timestamp = sample->time;
920 int cpu = sample->cpu;
Ingo Molnarea92ed52009-09-12 10:08:34 +0200921 s64 delta;
922
mingo39aeb522009-09-14 20:04:48 +0200923 BUG_ON(cpu >= MAX_CPUS || cpu < 0);
Ingo Molnarea92ed52009-09-12 10:08:34 +0200924
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -0300925 timestamp0 = sched->cpu_last_switched[cpu];
926 sched->cpu_last_switched[cpu] = timestamp;
Ingo Molnarea92ed52009-09-12 10:08:34 +0200927 if (timestamp0)
928 delta = timestamp - timestamp0;
929 else
930 delta = 0;
931
Arnaldo Carvalho de Meloa116e052012-09-08 22:53:06 -0300932 if (delta < 0) {
933 pr_err("hm, delta: %" PRIu64 " < 0 ?\n", delta);
934 return -1;
935 }
Frederic Weisbeckercdce9d72009-09-12 08:06:14 +0200936
Arnaldo Carvalho de Melo9ec3f4e2012-09-11 19:29:17 -0300937 sched_out = machine__findnew_thread(machine, prev_pid);
938 sched_in = machine__findnew_thread(machine, next_pid);
Frederic Weisbeckercdce9d72009-09-12 08:06:14 +0200939
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -0300940 out_events = thread_atoms_search(&sched->atom_root, sched_out, &sched->cmp_pid);
mingo39aeb522009-09-14 20:04:48 +0200941 if (!out_events) {
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -0300942 if (thread_atoms_insert(sched, sched_out))
Arnaldo Carvalho de Meloa116e052012-09-08 22:53:06 -0300943 return -1;
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -0300944 out_events = thread_atoms_search(&sched->atom_root, sched_out, &sched->cmp_pid);
Arnaldo Carvalho de Meloa116e052012-09-08 22:53:06 -0300945 if (!out_events) {
946 pr_err("out-event: Internal tree error");
947 return -1;
948 }
mingo39aeb522009-09-14 20:04:48 +0200949 }
Arnaldo Carvalho de Melo9ec3f4e2012-09-11 19:29:17 -0300950 if (add_sched_out_event(out_events, sched_out_state(prev_state), timestamp))
Arnaldo Carvalho de Meloa116e052012-09-08 22:53:06 -0300951 return -1;
mingo39aeb522009-09-14 20:04:48 +0200952
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -0300953 in_events = thread_atoms_search(&sched->atom_root, sched_in, &sched->cmp_pid);
mingo39aeb522009-09-14 20:04:48 +0200954 if (!in_events) {
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -0300955 if (thread_atoms_insert(sched, sched_in))
Arnaldo Carvalho de Meloa116e052012-09-08 22:53:06 -0300956 return -1;
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -0300957 in_events = thread_atoms_search(&sched->atom_root, sched_in, &sched->cmp_pid);
Arnaldo Carvalho de Meloa116e052012-09-08 22:53:06 -0300958 if (!in_events) {
959 pr_err("in-event: Internal tree error");
960 return -1;
961 }
mingo39aeb522009-09-14 20:04:48 +0200962 /*
963 * Take came in we have not heard about yet,
964 * add in an initial atom in runnable state:
965 */
Arnaldo Carvalho de Meloa116e052012-09-08 22:53:06 -0300966 if (add_sched_out_event(in_events, 'R', timestamp))
967 return -1;
mingo39aeb522009-09-14 20:04:48 +0200968 }
969 add_sched_in_event(in_events, timestamp);
Arnaldo Carvalho de Meloa116e052012-09-08 22:53:06 -0300970
971 return 0;
mingo39aeb522009-09-14 20:04:48 +0200972}
973
Arnaldo Carvalho de Melo9ec3f4e2012-09-11 19:29:17 -0300974static int latency_runtime_event(struct perf_sched *sched,
975 struct perf_evsel *evsel,
976 struct perf_sample *sample,
977 struct machine *machine)
mingo39aeb522009-09-14 20:04:48 +0200978{
Arnaldo Carvalho de Melo9ec3f4e2012-09-11 19:29:17 -0300979 const u32 pid = perf_evsel__intval(evsel, sample, "pid");
980 const u64 runtime = perf_evsel__intval(evsel, sample, "runtime");
981 struct thread *thread = machine__findnew_thread(machine, pid);
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -0300982 struct work_atoms *atoms = thread_atoms_search(&sched->atom_root, thread, &sched->cmp_pid);
Arnaldo Carvalho de Melo7f7f8d02012-08-07 11:33:42 -0300983 u64 timestamp = sample->time;
984 int cpu = sample->cpu;
mingo39aeb522009-09-14 20:04:48 +0200985
986 BUG_ON(cpu >= MAX_CPUS || cpu < 0);
mingo39aeb522009-09-14 20:04:48 +0200987 if (!atoms) {
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -0300988 if (thread_atoms_insert(sched, thread))
Arnaldo Carvalho de Meloa116e052012-09-08 22:53:06 -0300989 return -1;
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -0300990 atoms = thread_atoms_search(&sched->atom_root, thread, &sched->cmp_pid);
Arnaldo Carvalho de Meloa116e052012-09-08 22:53:06 -0300991 if (!atoms) {
Namhyung Kim60b7d142012-09-12 11:11:06 +0900992 pr_err("in-event: Internal tree error");
Arnaldo Carvalho de Meloa116e052012-09-08 22:53:06 -0300993 return -1;
994 }
995 if (add_sched_out_event(atoms, 'R', timestamp))
996 return -1;
Frederic Weisbeckercdce9d72009-09-12 08:06:14 +0200997 }
998
Arnaldo Carvalho de Melo9ec3f4e2012-09-11 19:29:17 -0300999 add_runtime_event(atoms, runtime, timestamp);
Arnaldo Carvalho de Meloa116e052012-09-08 22:53:06 -03001000 return 0;
Frederic Weisbeckercdce9d72009-09-12 08:06:14 +02001001}
1002
Arnaldo Carvalho de Melo9ec3f4e2012-09-11 19:29:17 -03001003static int latency_wakeup_event(struct perf_sched *sched,
1004 struct perf_evsel *evsel,
1005 struct perf_sample *sample,
1006 struct machine *machine)
Frederic Weisbeckercdce9d72009-09-12 08:06:14 +02001007{
Arnaldo Carvalho de Melo9ec3f4e2012-09-11 19:29:17 -03001008 const u32 pid = perf_evsel__intval(evsel, sample, "pid"),
1009 success = perf_evsel__intval(evsel, sample, "success");
mingo39aeb522009-09-14 20:04:48 +02001010 struct work_atoms *atoms;
Ingo Molnarb1ffe8f2009-09-11 12:12:54 +02001011 struct work_atom *atom;
Frederic Weisbeckercdce9d72009-09-12 08:06:14 +02001012 struct thread *wakee;
Arnaldo Carvalho de Melo7f7f8d02012-08-07 11:33:42 -03001013 u64 timestamp = sample->time;
Frederic Weisbeckercdce9d72009-09-12 08:06:14 +02001014
1015 /* Note for later, it may be interesting to observe the failing cases */
Arnaldo Carvalho de Melo9ec3f4e2012-09-11 19:29:17 -03001016 if (!success)
Arnaldo Carvalho de Meloa116e052012-09-08 22:53:06 -03001017 return 0;
Frederic Weisbeckercdce9d72009-09-12 08:06:14 +02001018
Arnaldo Carvalho de Melo9ec3f4e2012-09-11 19:29:17 -03001019 wakee = machine__findnew_thread(machine, pid);
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -03001020 atoms = thread_atoms_search(&sched->atom_root, wakee, &sched->cmp_pid);
Frederic Weisbecker17562202009-09-12 23:11:32 +02001021 if (!atoms) {
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -03001022 if (thread_atoms_insert(sched, wakee))
Arnaldo Carvalho de Meloa116e052012-09-08 22:53:06 -03001023 return -1;
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -03001024 atoms = thread_atoms_search(&sched->atom_root, wakee, &sched->cmp_pid);
Arnaldo Carvalho de Meloa116e052012-09-08 22:53:06 -03001025 if (!atoms) {
Namhyung Kim60b7d142012-09-12 11:11:06 +09001026 pr_err("wakeup-event: Internal tree error");
Arnaldo Carvalho de Meloa116e052012-09-08 22:53:06 -03001027 return -1;
1028 }
1029 if (add_sched_out_event(atoms, 'S', timestamp))
1030 return -1;
Frederic Weisbeckercdce9d72009-09-12 08:06:14 +02001031 }
1032
mingo39aeb522009-09-14 20:04:48 +02001033 BUG_ON(list_empty(&atoms->work_list));
Frederic Weisbeckercdce9d72009-09-12 08:06:14 +02001034
mingo39aeb522009-09-14 20:04:48 +02001035 atom = list_entry(atoms->work_list.prev, struct work_atom, list);
Frederic Weisbeckercdce9d72009-09-12 08:06:14 +02001036
Mike Galbraith55ffb7a2009-10-10 14:46:04 +02001037 /*
1038 * You WILL be missing events if you've recorded only
1039 * one CPU, or are only looking at only one, so don't
1040 * make useless noise.
1041 */
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -03001042 if (sched->profile_cpu == -1 && atom->state != THREAD_SLEEPING)
1043 sched->nr_state_machine_bugs++;
Frederic Weisbeckercdce9d72009-09-12 08:06:14 +02001044
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -03001045 sched->nr_timestamps++;
Ingo Molnarea57c4f2009-09-13 18:15:54 +02001046 if (atom->sched_out_time > timestamp) {
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -03001047 sched->nr_unordered_timestamps++;
Arnaldo Carvalho de Meloa116e052012-09-08 22:53:06 -03001048 return 0;
Ingo Molnarea57c4f2009-09-13 18:15:54 +02001049 }
Frederic Weisbeckeraa1ab9d2009-09-14 03:01:12 +02001050
Ingo Molnarb1ffe8f2009-09-11 12:12:54 +02001051 atom->state = THREAD_WAIT_CPU;
1052 atom->wake_up_time = timestamp;
Arnaldo Carvalho de Meloa116e052012-09-08 22:53:06 -03001053 return 0;
Frederic Weisbeckercdce9d72009-09-12 08:06:14 +02001054}
1055
Arnaldo Carvalho de Melo9ec3f4e2012-09-11 19:29:17 -03001056static int latency_migrate_task_event(struct perf_sched *sched,
1057 struct perf_evsel *evsel,
1058 struct perf_sample *sample,
1059 struct machine *machine)
Mike Galbraith55ffb7a2009-10-10 14:46:04 +02001060{
Arnaldo Carvalho de Melo9ec3f4e2012-09-11 19:29:17 -03001061 const u32 pid = perf_evsel__intval(evsel, sample, "pid");
Arnaldo Carvalho de Melo7f7f8d02012-08-07 11:33:42 -03001062 u64 timestamp = sample->time;
Mike Galbraith55ffb7a2009-10-10 14:46:04 +02001063 struct work_atoms *atoms;
1064 struct work_atom *atom;
1065 struct thread *migrant;
1066
1067 /*
1068 * Only need to worry about migration when profiling one CPU.
1069 */
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -03001070 if (sched->profile_cpu == -1)
Arnaldo Carvalho de Meloa116e052012-09-08 22:53:06 -03001071 return 0;
Mike Galbraith55ffb7a2009-10-10 14:46:04 +02001072
Arnaldo Carvalho de Melo9ec3f4e2012-09-11 19:29:17 -03001073 migrant = machine__findnew_thread(machine, pid);
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -03001074 atoms = thread_atoms_search(&sched->atom_root, migrant, &sched->cmp_pid);
Mike Galbraith55ffb7a2009-10-10 14:46:04 +02001075 if (!atoms) {
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -03001076 if (thread_atoms_insert(sched, migrant))
Arnaldo Carvalho de Meloa116e052012-09-08 22:53:06 -03001077 return -1;
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -03001078 register_pid(sched, migrant->pid, migrant->comm);
1079 atoms = thread_atoms_search(&sched->atom_root, migrant, &sched->cmp_pid);
Arnaldo Carvalho de Meloa116e052012-09-08 22:53:06 -03001080 if (!atoms) {
Namhyung Kim60b7d142012-09-12 11:11:06 +09001081 pr_err("migration-event: Internal tree error");
Arnaldo Carvalho de Meloa116e052012-09-08 22:53:06 -03001082 return -1;
1083 }
1084 if (add_sched_out_event(atoms, 'R', timestamp))
1085 return -1;
Mike Galbraith55ffb7a2009-10-10 14:46:04 +02001086 }
1087
1088 BUG_ON(list_empty(&atoms->work_list));
1089
1090 atom = list_entry(atoms->work_list.prev, struct work_atom, list);
1091 atom->sched_in_time = atom->sched_out_time = atom->wake_up_time = timestamp;
1092
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -03001093 sched->nr_timestamps++;
Mike Galbraith55ffb7a2009-10-10 14:46:04 +02001094
1095 if (atom->sched_out_time > timestamp)
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -03001096 sched->nr_unordered_timestamps++;
Arnaldo Carvalho de Meloa116e052012-09-08 22:53:06 -03001097
1098 return 0;
Mike Galbraith55ffb7a2009-10-10 14:46:04 +02001099}
1100
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -03001101static void output_lat_thread(struct perf_sched *sched, struct work_atoms *work_list)
Frederic Weisbeckercdce9d72009-09-12 08:06:14 +02001102{
Frederic Weisbeckercdce9d72009-09-12 08:06:14 +02001103 int i;
1104 int ret;
Frederic Weisbecker66685672009-09-13 01:56:25 +02001105 u64 avg;
Frederic Weisbeckercdce9d72009-09-12 08:06:14 +02001106
mingo39aeb522009-09-14 20:04:48 +02001107 if (!work_list->nb_atoms)
Frederic Weisbeckercdce9d72009-09-12 08:06:14 +02001108 return;
Ingo Molnarea57c4f2009-09-13 18:15:54 +02001109 /*
1110 * Ignore idle threads:
1111 */
Ingo Molnar80ed0982009-09-16 14:12:36 +02001112 if (!strcmp(work_list->thread->comm, "swapper"))
Ingo Molnarea57c4f2009-09-13 18:15:54 +02001113 return;
Frederic Weisbeckercdce9d72009-09-12 08:06:14 +02001114
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -03001115 sched->all_runtime += work_list->total_runtime;
1116 sched->all_count += work_list->nb_atoms;
Frederic Weisbecker66685672009-09-13 01:56:25 +02001117
Ingo Molnar80ed0982009-09-16 14:12:36 +02001118 ret = printf(" %s:%d ", work_list->thread->comm, work_list->thread->pid);
Frederic Weisbeckercdce9d72009-09-12 08:06:14 +02001119
mingo08f69e62009-09-14 18:30:44 +02001120 for (i = 0; i < 24 - ret; i++)
Frederic Weisbeckercdce9d72009-09-12 08:06:14 +02001121 printf(" ");
1122
mingo39aeb522009-09-14 20:04:48 +02001123 avg = work_list->total_lat / work_list->nb_atoms;
Frederic Weisbeckercdce9d72009-09-12 08:06:14 +02001124
Arnaldo Carvalho de Melo9486aa32011-01-22 20:37:02 -02001125 printf("|%11.3f ms |%9" PRIu64 " | avg:%9.3f ms | max:%9.3f ms | max at: %9.6f s\n",
mingo39aeb522009-09-14 20:04:48 +02001126 (double)work_list->total_runtime / 1e6,
1127 work_list->nb_atoms, (double)avg / 1e6,
Frederic Weisbecker3786310a2009-12-09 21:40:08 +01001128 (double)work_list->max_lat / 1e6,
1129 (double)work_list->max_lat_at / 1e9);
Frederic Weisbeckercdce9d72009-09-12 08:06:14 +02001130}
1131
mingo39aeb522009-09-14 20:04:48 +02001132static int pid_cmp(struct work_atoms *l, struct work_atoms *r)
Frederic Weisbeckerdaa1d7a2009-09-13 03:36:29 +02001133{
Frederic Weisbeckerdaa1d7a2009-09-13 03:36:29 +02001134 if (l->thread->pid < r->thread->pid)
1135 return -1;
1136 if (l->thread->pid > r->thread->pid)
1137 return 1;
1138
1139 return 0;
1140}
1141
mingo39aeb522009-09-14 20:04:48 +02001142static int avg_cmp(struct work_atoms *l, struct work_atoms *r)
Frederic Weisbeckerdaa1d7a2009-09-13 03:36:29 +02001143{
1144 u64 avgl, avgr;
1145
1146 if (!l->nb_atoms)
1147 return -1;
1148
1149 if (!r->nb_atoms)
1150 return 1;
1151
1152 avgl = l->total_lat / l->nb_atoms;
1153 avgr = r->total_lat / r->nb_atoms;
1154
1155 if (avgl < avgr)
1156 return -1;
1157 if (avgl > avgr)
1158 return 1;
1159
1160 return 0;
1161}
1162
mingo39aeb522009-09-14 20:04:48 +02001163static int max_cmp(struct work_atoms *l, struct work_atoms *r)
Frederic Weisbeckerdaa1d7a2009-09-13 03:36:29 +02001164{
1165 if (l->max_lat < r->max_lat)
1166 return -1;
1167 if (l->max_lat > r->max_lat)
1168 return 1;
1169
1170 return 0;
1171}
1172
mingo39aeb522009-09-14 20:04:48 +02001173static int switch_cmp(struct work_atoms *l, struct work_atoms *r)
Frederic Weisbeckerdaa1d7a2009-09-13 03:36:29 +02001174{
1175 if (l->nb_atoms < r->nb_atoms)
1176 return -1;
1177 if (l->nb_atoms > r->nb_atoms)
1178 return 1;
1179
1180 return 0;
1181}
1182
mingo39aeb522009-09-14 20:04:48 +02001183static int runtime_cmp(struct work_atoms *l, struct work_atoms *r)
Frederic Weisbeckerdaa1d7a2009-09-13 03:36:29 +02001184{
1185 if (l->total_runtime < r->total_runtime)
1186 return -1;
1187 if (l->total_runtime > r->total_runtime)
1188 return 1;
1189
1190 return 0;
1191}
1192
Randy Dunlapcbef79a2009-10-05 13:17:29 -07001193static int sort_dimension__add(const char *tok, struct list_head *list)
Frederic Weisbeckerdaa1d7a2009-09-13 03:36:29 +02001194{
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -03001195 size_t i;
1196 static struct sort_dimension avg_sort_dimension = {
1197 .name = "avg",
1198 .cmp = avg_cmp,
1199 };
1200 static struct sort_dimension max_sort_dimension = {
1201 .name = "max",
1202 .cmp = max_cmp,
1203 };
1204 static struct sort_dimension pid_sort_dimension = {
1205 .name = "pid",
1206 .cmp = pid_cmp,
1207 };
1208 static struct sort_dimension runtime_sort_dimension = {
1209 .name = "runtime",
1210 .cmp = runtime_cmp,
1211 };
1212 static struct sort_dimension switch_sort_dimension = {
1213 .name = "switch",
1214 .cmp = switch_cmp,
1215 };
1216 struct sort_dimension *available_sorts[] = {
1217 &pid_sort_dimension,
1218 &avg_sort_dimension,
1219 &max_sort_dimension,
1220 &switch_sort_dimension,
1221 &runtime_sort_dimension,
1222 };
Frederic Weisbeckerdaa1d7a2009-09-13 03:36:29 +02001223
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -03001224 for (i = 0; i < ARRAY_SIZE(available_sorts); i++) {
Frederic Weisbeckerdaa1d7a2009-09-13 03:36:29 +02001225 if (!strcmp(available_sorts[i]->name, tok)) {
1226 list_add_tail(&available_sorts[i]->list, list);
1227
1228 return 0;
1229 }
1230 }
1231
1232 return -1;
1233}
1234
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -03001235static void perf_sched__sort_lat(struct perf_sched *sched)
Frederic Weisbeckerdaa1d7a2009-09-13 03:36:29 +02001236{
1237 struct rb_node *node;
1238
1239 for (;;) {
mingo39aeb522009-09-14 20:04:48 +02001240 struct work_atoms *data;
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -03001241 node = rb_first(&sched->atom_root);
Frederic Weisbeckerdaa1d7a2009-09-13 03:36:29 +02001242 if (!node)
1243 break;
1244
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -03001245 rb_erase(node, &sched->atom_root);
mingo39aeb522009-09-14 20:04:48 +02001246 data = rb_entry(node, struct work_atoms, node);
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -03001247 __thread_latency_insert(&sched->sorted_atom_root, data, &sched->sort_list);
Frederic Weisbeckerdaa1d7a2009-09-13 03:36:29 +02001248 }
1249}
1250
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -03001251static int process_sched_wakeup_event(struct perf_tool *tool,
Arnaldo Carvalho de Melo2b7fcbc2012-09-11 19:29:17 -03001252 struct perf_evsel *evsel,
Irina Tirdea1d037ca2012-09-11 01:15:03 +03001253 struct perf_sample *sample,
Arnaldo Carvalho de Melo4218e672012-09-11 13:18:47 -03001254 struct machine *machine)
Frederic Weisbecker419ab0d2009-09-12 03:59:01 +02001255{
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -03001256 struct perf_sched *sched = container_of(tool, struct perf_sched, tool);
Frederic Weisbecker419ab0d2009-09-12 03:59:01 +02001257
Arnaldo Carvalho de Melo9ec3f4e2012-09-11 19:29:17 -03001258 if (sched->tp_handler->wakeup_event)
1259 return sched->tp_handler->wakeup_event(sched, evsel, sample, machine);
Frederic Weisbecker419ab0d2009-09-12 03:59:01 +02001260
Arnaldo Carvalho de Melo2b7fcbc2012-09-11 19:29:17 -03001261 return 0;
Frederic Weisbecker419ab0d2009-09-12 03:59:01 +02001262}
1263
Arnaldo Carvalho de Melo9ec3f4e2012-09-11 19:29:17 -03001264static int map_switch_event(struct perf_sched *sched, struct perf_evsel *evsel,
1265 struct perf_sample *sample, struct machine *machine)
Ingo Molnar0ec04e12009-09-16 17:40:48 +02001266{
Arnaldo Carvalho de Melo9ec3f4e2012-09-11 19:29:17 -03001267 const u32 prev_pid = perf_evsel__intval(evsel, sample, "prev_pid"),
1268 next_pid = perf_evsel__intval(evsel, sample, "next_pid");
Irina Tirdea1d037ca2012-09-11 01:15:03 +03001269 struct thread *sched_out __maybe_unused, *sched_in;
Ingo Molnar0ec04e12009-09-16 17:40:48 +02001270 int new_shortname;
Arnaldo Carvalho de Melo7f7f8d02012-08-07 11:33:42 -03001271 u64 timestamp0, timestamp = sample->time;
Ingo Molnar0ec04e12009-09-16 17:40:48 +02001272 s64 delta;
Arnaldo Carvalho de Melo7f7f8d02012-08-07 11:33:42 -03001273 int cpu, this_cpu = sample->cpu;
Ingo Molnar0ec04e12009-09-16 17:40:48 +02001274
1275 BUG_ON(this_cpu >= MAX_CPUS || this_cpu < 0);
1276
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -03001277 if (this_cpu > sched->max_cpu)
1278 sched->max_cpu = this_cpu;
Ingo Molnar0ec04e12009-09-16 17:40:48 +02001279
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -03001280 timestamp0 = sched->cpu_last_switched[this_cpu];
1281 sched->cpu_last_switched[this_cpu] = timestamp;
Ingo Molnar0ec04e12009-09-16 17:40:48 +02001282 if (timestamp0)
1283 delta = timestamp - timestamp0;
1284 else
1285 delta = 0;
1286
Arnaldo Carvalho de Meloa116e052012-09-08 22:53:06 -03001287 if (delta < 0) {
Namhyung Kim60b7d142012-09-12 11:11:06 +09001288 pr_err("hm, delta: %" PRIu64 " < 0 ?\n", delta);
Arnaldo Carvalho de Meloa116e052012-09-08 22:53:06 -03001289 return -1;
1290 }
Ingo Molnar0ec04e12009-09-16 17:40:48 +02001291
Arnaldo Carvalho de Melo9ec3f4e2012-09-11 19:29:17 -03001292 sched_out = machine__findnew_thread(machine, prev_pid);
1293 sched_in = machine__findnew_thread(machine, next_pid);
Ingo Molnar0ec04e12009-09-16 17:40:48 +02001294
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -03001295 sched->curr_thread[this_cpu] = sched_in;
Ingo Molnar0ec04e12009-09-16 17:40:48 +02001296
1297 printf(" ");
1298
1299 new_shortname = 0;
1300 if (!sched_in->shortname[0]) {
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -03001301 sched_in->shortname[0] = sched->next_shortname1;
1302 sched_in->shortname[1] = sched->next_shortname2;
Ingo Molnar0ec04e12009-09-16 17:40:48 +02001303
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -03001304 if (sched->next_shortname1 < 'Z') {
1305 sched->next_shortname1++;
Ingo Molnar0ec04e12009-09-16 17:40:48 +02001306 } else {
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -03001307 sched->next_shortname1='A';
1308 if (sched->next_shortname2 < '9') {
1309 sched->next_shortname2++;
Ingo Molnar0ec04e12009-09-16 17:40:48 +02001310 } else {
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -03001311 sched->next_shortname2='0';
Ingo Molnar0ec04e12009-09-16 17:40:48 +02001312 }
1313 }
1314 new_shortname = 1;
1315 }
1316
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -03001317 for (cpu = 0; cpu <= sched->max_cpu; cpu++) {
Ingo Molnar0ec04e12009-09-16 17:40:48 +02001318 if (cpu != this_cpu)
1319 printf(" ");
1320 else
1321 printf("*");
1322
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -03001323 if (sched->curr_thread[cpu]) {
1324 if (sched->curr_thread[cpu]->pid)
1325 printf("%2s ", sched->curr_thread[cpu]->shortname);
Ingo Molnar0ec04e12009-09-16 17:40:48 +02001326 else
1327 printf(". ");
1328 } else
1329 printf(" ");
1330 }
1331
1332 printf(" %12.6f secs ", (double)timestamp/1e9);
1333 if (new_shortname) {
1334 printf("%s => %s:%d\n",
1335 sched_in->shortname, sched_in->comm, sched_in->pid);
1336 } else {
1337 printf("\n");
1338 }
Arnaldo Carvalho de Meloa116e052012-09-08 22:53:06 -03001339
1340 return 0;
Ingo Molnar0ec04e12009-09-16 17:40:48 +02001341}
1342
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -03001343static int process_sched_switch_event(struct perf_tool *tool,
Arnaldo Carvalho de Melo2b7fcbc2012-09-11 19:29:17 -03001344 struct perf_evsel *evsel,
Irina Tirdea1d037ca2012-09-11 01:15:03 +03001345 struct perf_sample *sample,
Arnaldo Carvalho de Melo4218e672012-09-11 13:18:47 -03001346 struct machine *machine)
Frederic Weisbecker419ab0d2009-09-12 03:59:01 +02001347{
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -03001348 struct perf_sched *sched = container_of(tool, struct perf_sched, tool);
Arnaldo Carvalho de Meloa116e052012-09-08 22:53:06 -03001349 int this_cpu = sample->cpu, err = 0;
Arnaldo Carvalho de Melo2b7fcbc2012-09-11 19:29:17 -03001350 u32 prev_pid = perf_evsel__intval(evsel, sample, "prev_pid"),
1351 next_pid = perf_evsel__intval(evsel, sample, "next_pid");
Frederic Weisbecker419ab0d2009-09-12 03:59:01 +02001352
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -03001353 if (sched->curr_pid[this_cpu] != (u32)-1) {
Ingo Molnarc8a37752009-09-16 14:07:00 +02001354 /*
1355 * Are we trying to switch away a PID that is
1356 * not current?
1357 */
Arnaldo Carvalho de Melo2b7fcbc2012-09-11 19:29:17 -03001358 if (sched->curr_pid[this_cpu] != prev_pid)
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -03001359 sched->nr_context_switch_bugs++;
Ingo Molnarc8a37752009-09-16 14:07:00 +02001360 }
Ingo Molnarc8a37752009-09-16 14:07:00 +02001361
Arnaldo Carvalho de Melo9ec3f4e2012-09-11 19:29:17 -03001362 if (sched->tp_handler->switch_event)
1363 err = sched->tp_handler->switch_event(sched, evsel, sample, machine);
Arnaldo Carvalho de Melo2b7fcbc2012-09-11 19:29:17 -03001364
1365 sched->curr_pid[this_cpu] = next_pid;
Arnaldo Carvalho de Meloa116e052012-09-08 22:53:06 -03001366 return err;
Frederic Weisbecker419ab0d2009-09-12 03:59:01 +02001367}
1368
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -03001369static int process_sched_runtime_event(struct perf_tool *tool,
Arnaldo Carvalho de Melo2b7fcbc2012-09-11 19:29:17 -03001370 struct perf_evsel *evsel,
Irina Tirdea1d037ca2012-09-11 01:15:03 +03001371 struct perf_sample *sample,
Arnaldo Carvalho de Melo4218e672012-09-11 13:18:47 -03001372 struct machine *machine)
mingo39aeb522009-09-14 20:04:48 +02001373{
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -03001374 struct perf_sched *sched = container_of(tool, struct perf_sched, tool);
mingo39aeb522009-09-14 20:04:48 +02001375
Arnaldo Carvalho de Melo9ec3f4e2012-09-11 19:29:17 -03001376 if (sched->tp_handler->runtime_event)
1377 return sched->tp_handler->runtime_event(sched, evsel, sample, machine);
Arnaldo Carvalho de Meloa116e052012-09-08 22:53:06 -03001378
1379 return 0;
Ingo Molnarec156762009-09-11 12:12:54 +02001380}
1381
Arnaldo Carvalho de Melo2b7fcbc2012-09-11 19:29:17 -03001382static int process_sched_fork_event(struct perf_tool *tool,
1383 struct perf_evsel *evsel,
1384 struct perf_sample *sample,
1385 struct machine *machine __maybe_unused)
1386{
1387 struct perf_sched *sched = container_of(tool, struct perf_sched, tool);
1388
Arnaldo Carvalho de Melo9ec3f4e2012-09-11 19:29:17 -03001389 if (sched->tp_handler->fork_event)
1390 return sched->tp_handler->fork_event(sched, evsel, sample);
Arnaldo Carvalho de Melo2b7fcbc2012-09-11 19:29:17 -03001391
1392 return 0;
1393}
1394
1395static int process_sched_exit_event(struct perf_tool *tool __maybe_unused,
1396 struct perf_evsel *evsel,
1397 struct perf_sample *sample __maybe_unused,
1398 struct machine *machine __maybe_unused)
1399{
1400 pr_debug("sched_exit event %p\n", evsel);
1401 return 0;
1402}
1403
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -03001404static int process_sched_migrate_task_event(struct perf_tool *tool,
Arnaldo Carvalho de Melo2b7fcbc2012-09-11 19:29:17 -03001405 struct perf_evsel *evsel,
Irina Tirdea1d037ca2012-09-11 01:15:03 +03001406 struct perf_sample *sample,
Arnaldo Carvalho de Melo4218e672012-09-11 13:18:47 -03001407 struct machine *machine)
Mike Galbraith55ffb7a2009-10-10 14:46:04 +02001408{
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -03001409 struct perf_sched *sched = container_of(tool, struct perf_sched, tool);
Mike Galbraith55ffb7a2009-10-10 14:46:04 +02001410
Arnaldo Carvalho de Melo9ec3f4e2012-09-11 19:29:17 -03001411 if (sched->tp_handler->migrate_task_event)
1412 return sched->tp_handler->migrate_task_event(sched, evsel, sample, machine);
Mike Galbraith55ffb7a2009-10-10 14:46:04 +02001413
Arnaldo Carvalho de Melo2b7fcbc2012-09-11 19:29:17 -03001414 return 0;
Mike Galbraith55ffb7a2009-10-10 14:46:04 +02001415}
1416
Arnaldo Carvalho de Meloa116e052012-09-08 22:53:06 -03001417typedef int (*tracepoint_handler)(struct perf_tool *tool,
Arnaldo Carvalho de Melo2b7fcbc2012-09-11 19:29:17 -03001418 struct perf_evsel *evsel,
Arnaldo Carvalho de Meloa116e052012-09-08 22:53:06 -03001419 struct perf_sample *sample,
Arnaldo Carvalho de Melo4218e672012-09-11 13:18:47 -03001420 struct machine *machine);
Arnaldo Carvalho de Meloee29be62011-11-28 17:57:40 -02001421
Irina Tirdea1d037ca2012-09-11 01:15:03 +03001422static int perf_sched__process_tracepoint_sample(struct perf_tool *tool __maybe_unused,
1423 union perf_event *event __maybe_unused,
Arnaldo Carvalho de Meloee29be62011-11-28 17:57:40 -02001424 struct perf_sample *sample,
1425 struct perf_evsel *evsel,
1426 struct machine *machine)
Ingo Molnarec156762009-09-11 12:12:54 +02001427{
Arnaldo Carvalho de Melo73ee3b22012-10-01 13:58:17 -03001428 struct thread *thread = machine__findnew_thread(machine, sample->tid);
Arnaldo Carvalho de Meloa116e052012-09-08 22:53:06 -03001429 int err = 0;
Ingo Molnarec156762009-09-11 12:12:54 +02001430
Ingo Molnar0a02ad92009-09-11 12:12:54 +02001431 if (thread == NULL) {
Arnaldo Carvalho de Meloee29be62011-11-28 17:57:40 -02001432 pr_debug("problem processing %s event, skipping it.\n",
Arnaldo Carvalho de Melo22c8b842012-06-12 13:55:13 -03001433 perf_evsel__name(evsel));
Ingo Molnar0a02ad92009-09-11 12:12:54 +02001434 return -1;
1435 }
1436
Arnaldo Carvalho de Meloee29be62011-11-28 17:57:40 -02001437 evsel->hists.stats.total_period += sample->period;
1438 hists__inc_nr_events(&evsel->hists, PERF_RECORD_SAMPLE);
Julia Lawallf39cdf22009-10-17 08:43:17 +02001439
Arnaldo Carvalho de Meloee29be62011-11-28 17:57:40 -02001440 if (evsel->handler.func != NULL) {
1441 tracepoint_handler f = evsel->handler.func;
Arnaldo Carvalho de Melo2b7fcbc2012-09-11 19:29:17 -03001442 err = f(tool, evsel, sample, machine);
Arnaldo Carvalho de Meloee29be62011-11-28 17:57:40 -02001443 }
Ingo Molnar0a02ad92009-09-11 12:12:54 +02001444
Arnaldo Carvalho de Meloa116e052012-09-08 22:53:06 -03001445 return err;
Ingo Molnar0a02ad92009-09-11 12:12:54 +02001446}
1447
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -03001448static int perf_sched__read_events(struct perf_sched *sched, bool destroy,
1449 struct perf_session **psession)
Ingo Molnar0a02ad92009-09-11 12:12:54 +02001450{
Arnaldo Carvalho de Meloee29be62011-11-28 17:57:40 -02001451 const struct perf_evsel_str_handler handlers[] = {
1452 { "sched:sched_switch", process_sched_switch_event, },
1453 { "sched:sched_stat_runtime", process_sched_runtime_event, },
1454 { "sched:sched_wakeup", process_sched_wakeup_event, },
1455 { "sched:sched_wakeup_new", process_sched_wakeup_event, },
1456 { "sched:sched_process_fork", process_sched_fork_event, },
1457 { "sched:sched_process_exit", process_sched_exit_event, },
1458 { "sched:sched_migrate_task", process_sched_migrate_task_event, },
1459 };
Arnaldo Carvalho de Meloda378962012-06-27 13:08:42 -03001460 struct perf_session *session;
1461
Feng Tang70cb4e92012-10-30 11:56:02 +08001462 session = perf_session__new(input_name, O_RDONLY, 0, false, &sched->tool);
Arnaldo Carvalho de Meloa116e052012-09-08 22:53:06 -03001463 if (session == NULL) {
1464 pr_debug("No Memory for session\n");
1465 return -1;
1466 }
Arnaldo Carvalho de Melo94c744b2009-12-11 21:24:02 -02001467
Arnaldo Carvalho de Meloa116e052012-09-08 22:53:06 -03001468 if (perf_session__set_tracepoints_handlers(session, handlers))
1469 goto out_delete;
Arnaldo Carvalho de Meloee29be62011-11-28 17:57:40 -02001470
Arnaldo Carvalho de Melocee75ac2010-05-14 13:16:55 -03001471 if (perf_session__has_traces(session, "record -R")) {
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -03001472 int err = perf_session__process_events(session, &sched->tool);
Arnaldo Carvalho de Meloa116e052012-09-08 22:53:06 -03001473 if (err) {
1474 pr_err("Failed to process events, error %d", err);
1475 goto out_delete;
1476 }
Jiri Olsa4c09baf2011-08-08 23:03:34 +02001477
Arnaldo Carvalho de Melo28a6b6a2012-12-18 16:24:46 -03001478 sched->nr_events = session->stats.nr_events[0];
1479 sched->nr_lost_events = session->stats.total_lost;
1480 sched->nr_lost_chunks = session->stats.nr_events[PERF_RECORD_LOST];
Arnaldo Carvalho de Melocee75ac2010-05-14 13:16:55 -03001481 }
Arnaldo Carvalho de Melod549c7692009-12-27 21:37:02 -02001482
Jiri Olsa4c09baf2011-08-08 23:03:34 +02001483 if (destroy)
1484 perf_session__delete(session);
1485
1486 if (psession)
1487 *psession = session;
Arnaldo Carvalho de Meloa116e052012-09-08 22:53:06 -03001488
1489 return 0;
1490
1491out_delete:
1492 perf_session__delete(session);
1493 return -1;
Ingo Molnar0a02ad92009-09-11 12:12:54 +02001494}
1495
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -03001496static void print_bad_events(struct perf_sched *sched)
Ingo Molnar0ec04e12009-09-16 17:40:48 +02001497{
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -03001498 if (sched->nr_unordered_timestamps && sched->nr_timestamps) {
Ingo Molnar0ec04e12009-09-16 17:40:48 +02001499 printf(" INFO: %.3f%% unordered timestamps (%ld out of %ld)\n",
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -03001500 (double)sched->nr_unordered_timestamps/(double)sched->nr_timestamps*100.0,
1501 sched->nr_unordered_timestamps, sched->nr_timestamps);
Ingo Molnar0ec04e12009-09-16 17:40:48 +02001502 }
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -03001503 if (sched->nr_lost_events && sched->nr_events) {
Ingo Molnar0ec04e12009-09-16 17:40:48 +02001504 printf(" INFO: %.3f%% lost events (%ld out of %ld, in %ld chunks)\n",
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -03001505 (double)sched->nr_lost_events/(double)sched->nr_events * 100.0,
1506 sched->nr_lost_events, sched->nr_events, sched->nr_lost_chunks);
Ingo Molnar0ec04e12009-09-16 17:40:48 +02001507 }
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -03001508 if (sched->nr_state_machine_bugs && sched->nr_timestamps) {
Ingo Molnar0ec04e12009-09-16 17:40:48 +02001509 printf(" INFO: %.3f%% state machine bugs (%ld out of %ld)",
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -03001510 (double)sched->nr_state_machine_bugs/(double)sched->nr_timestamps*100.0,
1511 sched->nr_state_machine_bugs, sched->nr_timestamps);
1512 if (sched->nr_lost_events)
Ingo Molnar0ec04e12009-09-16 17:40:48 +02001513 printf(" (due to lost events?)");
1514 printf("\n");
1515 }
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -03001516 if (sched->nr_context_switch_bugs && sched->nr_timestamps) {
Ingo Molnar0ec04e12009-09-16 17:40:48 +02001517 printf(" INFO: %.3f%% context switch bugs (%ld out of %ld)",
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -03001518 (double)sched->nr_context_switch_bugs/(double)sched->nr_timestamps*100.0,
1519 sched->nr_context_switch_bugs, sched->nr_timestamps);
1520 if (sched->nr_lost_events)
Ingo Molnar0ec04e12009-09-16 17:40:48 +02001521 printf(" (due to lost events?)");
1522 printf("\n");
1523 }
1524}
1525
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -03001526static int perf_sched__lat(struct perf_sched *sched)
Ingo Molnar0ec04e12009-09-16 17:40:48 +02001527{
1528 struct rb_node *next;
Jiri Olsa4c09baf2011-08-08 23:03:34 +02001529 struct perf_session *session;
Ingo Molnar0ec04e12009-09-16 17:40:48 +02001530
1531 setup_pager();
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -03001532 if (perf_sched__read_events(sched, false, &session))
Arnaldo Carvalho de Meloa116e052012-09-08 22:53:06 -03001533 return -1;
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -03001534 perf_sched__sort_lat(sched);
Ingo Molnar0ec04e12009-09-16 17:40:48 +02001535
Frederic Weisbecker3786310a2009-12-09 21:40:08 +01001536 printf("\n ---------------------------------------------------------------------------------------------------------------\n");
1537 printf(" Task | Runtime ms | Switches | Average delay ms | Maximum delay ms | Maximum delay at |\n");
1538 printf(" ---------------------------------------------------------------------------------------------------------------\n");
Ingo Molnar0ec04e12009-09-16 17:40:48 +02001539
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -03001540 next = rb_first(&sched->sorted_atom_root);
Ingo Molnar0ec04e12009-09-16 17:40:48 +02001541
1542 while (next) {
1543 struct work_atoms *work_list;
1544
1545 work_list = rb_entry(next, struct work_atoms, node);
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -03001546 output_lat_thread(sched, work_list);
Ingo Molnar0ec04e12009-09-16 17:40:48 +02001547 next = rb_next(next);
1548 }
1549
1550 printf(" -----------------------------------------------------------------------------------------\n");
Arnaldo Carvalho de Melo9486aa32011-01-22 20:37:02 -02001551 printf(" TOTAL: |%11.3f ms |%9" PRIu64 " |\n",
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -03001552 (double)sched->all_runtime / 1e6, sched->all_count);
Ingo Molnar0ec04e12009-09-16 17:40:48 +02001553
1554 printf(" ---------------------------------------------------\n");
1555
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -03001556 print_bad_events(sched);
Ingo Molnar0ec04e12009-09-16 17:40:48 +02001557 printf("\n");
1558
Jiri Olsa4c09baf2011-08-08 23:03:34 +02001559 perf_session__delete(session);
Arnaldo Carvalho de Meloa116e052012-09-08 22:53:06 -03001560 return 0;
Ingo Molnar0ec04e12009-09-16 17:40:48 +02001561}
1562
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -03001563static int perf_sched__map(struct perf_sched *sched)
Ingo Molnar0ec04e12009-09-16 17:40:48 +02001564{
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -03001565 sched->max_cpu = sysconf(_SC_NPROCESSORS_CONF);
Ingo Molnar40749d02009-09-17 18:24:55 +02001566
Ingo Molnar0ec04e12009-09-16 17:40:48 +02001567 setup_pager();
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -03001568 if (perf_sched__read_events(sched, true, NULL))
Arnaldo Carvalho de Meloa116e052012-09-08 22:53:06 -03001569 return -1;
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -03001570 print_bad_events(sched);
Arnaldo Carvalho de Meloa116e052012-09-08 22:53:06 -03001571 return 0;
Ingo Molnar0ec04e12009-09-16 17:40:48 +02001572}
1573
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -03001574static int perf_sched__replay(struct perf_sched *sched)
Ingo Molnar0ec04e12009-09-16 17:40:48 +02001575{
1576 unsigned long i;
1577
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -03001578 calibrate_run_measurement_overhead(sched);
1579 calibrate_sleep_measurement_overhead(sched);
Ingo Molnar0ec04e12009-09-16 17:40:48 +02001580
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -03001581 test_calibrations(sched);
Ingo Molnar0ec04e12009-09-16 17:40:48 +02001582
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -03001583 if (perf_sched__read_events(sched, true, NULL))
Arnaldo Carvalho de Meloa116e052012-09-08 22:53:06 -03001584 return -1;
Ingo Molnar0ec04e12009-09-16 17:40:48 +02001585
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -03001586 printf("nr_run_events: %ld\n", sched->nr_run_events);
1587 printf("nr_sleep_events: %ld\n", sched->nr_sleep_events);
1588 printf("nr_wakeup_events: %ld\n", sched->nr_wakeup_events);
Ingo Molnar0ec04e12009-09-16 17:40:48 +02001589
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -03001590 if (sched->targetless_wakeups)
1591 printf("target-less wakeups: %ld\n", sched->targetless_wakeups);
1592 if (sched->multitarget_wakeups)
1593 printf("multi-target wakeups: %ld\n", sched->multitarget_wakeups);
1594 if (sched->nr_run_events_optimized)
Ingo Molnar0ec04e12009-09-16 17:40:48 +02001595 printf("run atoms optimized: %ld\n",
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -03001596 sched->nr_run_events_optimized);
Ingo Molnar0ec04e12009-09-16 17:40:48 +02001597
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -03001598 print_task_traces(sched);
1599 add_cross_task_wakeups(sched);
Ingo Molnar0ec04e12009-09-16 17:40:48 +02001600
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -03001601 create_tasks(sched);
Ingo Molnar0ec04e12009-09-16 17:40:48 +02001602 printf("------------------------------------------------------------\n");
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -03001603 for (i = 0; i < sched->replay_repeat; i++)
1604 run_one_test(sched);
Arnaldo Carvalho de Meloa116e052012-09-08 22:53:06 -03001605
1606 return 0;
Ingo Molnar0ec04e12009-09-16 17:40:48 +02001607}
1608
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -03001609static void setup_sorting(struct perf_sched *sched, const struct option *options,
1610 const char * const usage_msg[])
Frederic Weisbeckerdaa1d7a2009-09-13 03:36:29 +02001611{
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -03001612 char *tmp, *tok, *str = strdup(sched->sort_order);
Frederic Weisbeckerdaa1d7a2009-09-13 03:36:29 +02001613
1614 for (tok = strtok_r(str, ", ", &tmp);
1615 tok; tok = strtok_r(NULL, ", ", &tmp)) {
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -03001616 if (sort_dimension__add(tok, &sched->sort_list) < 0) {
Frederic Weisbeckerdaa1d7a2009-09-13 03:36:29 +02001617 error("Unknown --sort key: `%s'", tok);
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -03001618 usage_with_options(usage_msg, options);
Frederic Weisbeckerdaa1d7a2009-09-13 03:36:29 +02001619 }
1620 }
1621
1622 free(str);
1623
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -03001624 sort_dimension__add("pid", &sched->cmp_pid);
Frederic Weisbeckerdaa1d7a2009-09-13 03:36:29 +02001625}
1626
Ingo Molnar1fc35b22009-09-13 09:44:29 +02001627static int __cmd_record(int argc, const char **argv)
1628{
1629 unsigned int rec_argc, i, j;
1630 const char **rec_argv;
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -03001631 const char * const record_args[] = {
1632 "record",
1633 "-a",
1634 "-R",
1635 "-f",
1636 "-m", "1024",
1637 "-c", "1",
1638 "-e", "sched:sched_switch",
1639 "-e", "sched:sched_stat_wait",
1640 "-e", "sched:sched_stat_sleep",
1641 "-e", "sched:sched_stat_iowait",
1642 "-e", "sched:sched_stat_runtime",
1643 "-e", "sched:sched_process_exit",
1644 "-e", "sched:sched_process_fork",
1645 "-e", "sched:sched_wakeup",
1646 "-e", "sched:sched_migrate_task",
1647 };
Ingo Molnar1fc35b22009-09-13 09:44:29 +02001648
1649 rec_argc = ARRAY_SIZE(record_args) + argc - 1;
1650 rec_argv = calloc(rec_argc + 1, sizeof(char *));
1651
Arnaldo Carvalho de Meloe462dc52011-01-10 10:48:47 -02001652 if (rec_argv == NULL)
Chris Samuelce47dc52010-11-13 13:35:06 +11001653 return -ENOMEM;
1654
Ingo Molnar1fc35b22009-09-13 09:44:29 +02001655 for (i = 0; i < ARRAY_SIZE(record_args); i++)
1656 rec_argv[i] = strdup(record_args[i]);
1657
1658 for (j = 1; j < (unsigned int)argc; j++, i++)
1659 rec_argv[i] = argv[j];
1660
1661 BUG_ON(i != rec_argc);
1662
1663 return cmd_record(i, rec_argv, NULL);
1664}
1665
Irina Tirdea1d037ca2012-09-11 01:15:03 +03001666int cmd_sched(int argc, const char **argv, const char *prefix __maybe_unused)
Ingo Molnar0a02ad92009-09-11 12:12:54 +02001667{
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -03001668 const char default_sort_order[] = "avg, max, switch, runtime";
1669 struct perf_sched sched = {
1670 .tool = {
1671 .sample = perf_sched__process_tracepoint_sample,
1672 .comm = perf_event__process_comm,
1673 .lost = perf_event__process_lost,
Arnaldo Carvalho de Melo04395392012-10-06 15:53:41 -03001674 .exit = perf_event__process_exit,
Arnaldo Carvalho de Melof62d3f02012-10-06 15:44:59 -03001675 .fork = perf_event__process_fork,
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -03001676 .ordered_samples = true,
1677 },
1678 .cmp_pid = LIST_HEAD_INIT(sched.cmp_pid),
1679 .sort_list = LIST_HEAD_INIT(sched.sort_list),
1680 .start_work_mutex = PTHREAD_MUTEX_INITIALIZER,
1681 .work_done_wait_mutex = PTHREAD_MUTEX_INITIALIZER,
1682 .curr_pid = { [0 ... MAX_CPUS - 1] = -1 },
1683 .sort_order = default_sort_order,
1684 .replay_repeat = 10,
1685 .profile_cpu = -1,
1686 .next_shortname1 = 'A',
1687 .next_shortname2 = '0',
1688 };
1689 const struct option latency_options[] = {
1690 OPT_STRING('s', "sort", &sched.sort_order, "key[,key2...]",
1691 "sort by key(s): runtime, switch, avg, max"),
1692 OPT_INCR('v', "verbose", &verbose,
1693 "be more verbose (show symbol address, etc)"),
1694 OPT_INTEGER('C', "CPU", &sched.profile_cpu,
1695 "CPU to profile on"),
1696 OPT_BOOLEAN('D', "dump-raw-trace", &dump_trace,
1697 "dump raw trace in ASCII"),
1698 OPT_END()
1699 };
1700 const struct option replay_options[] = {
1701 OPT_UINTEGER('r', "repeat", &sched.replay_repeat,
1702 "repeat the workload replay N times (-1: infinite)"),
1703 OPT_INCR('v', "verbose", &verbose,
1704 "be more verbose (show symbol address, etc)"),
1705 OPT_BOOLEAN('D', "dump-raw-trace", &dump_trace,
1706 "dump raw trace in ASCII"),
1707 OPT_END()
1708 };
1709 const struct option sched_options[] = {
Feng Tang70cb4e92012-10-30 11:56:02 +08001710 OPT_STRING('i', "input", &input_name, "file",
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -03001711 "input file name"),
1712 OPT_INCR('v', "verbose", &verbose,
1713 "be more verbose (show symbol address, etc)"),
1714 OPT_BOOLEAN('D', "dump-raw-trace", &dump_trace,
1715 "dump raw trace in ASCII"),
1716 OPT_END()
1717 };
1718 const char * const latency_usage[] = {
1719 "perf sched latency [<options>]",
1720 NULL
1721 };
1722 const char * const replay_usage[] = {
1723 "perf sched replay [<options>]",
1724 NULL
1725 };
1726 const char * const sched_usage[] = {
1727 "perf sched [<options>] {record|latency|map|replay|script}",
1728 NULL
1729 };
1730 struct trace_sched_handler lat_ops = {
1731 .wakeup_event = latency_wakeup_event,
1732 .switch_event = latency_switch_event,
1733 .runtime_event = latency_runtime_event,
1734 .fork_event = latency_fork_event,
1735 .migrate_task_event = latency_migrate_task_event,
1736 };
1737 struct trace_sched_handler map_ops = {
1738 .switch_event = map_switch_event,
1739 };
1740 struct trace_sched_handler replay_ops = {
1741 .wakeup_event = replay_wakeup_event,
1742 .switch_event = replay_switch_event,
1743 .fork_event = replay_fork_event,
1744 };
1745
Ingo Molnarf2858d82009-09-11 12:12:54 +02001746 argc = parse_options(argc, argv, sched_options, sched_usage,
1747 PARSE_OPT_STOP_AT_NON_OPTION);
1748 if (!argc)
1749 usage_with_options(sched_usage, sched_options);
1750
Xiao Guangrongc0777c52009-12-07 12:04:49 +08001751 /*
Ingo Molnar133dc4c2010-11-16 18:45:39 +01001752 * Aliased to 'perf script' for now:
Xiao Guangrongc0777c52009-12-07 12:04:49 +08001753 */
Ingo Molnar133dc4c2010-11-16 18:45:39 +01001754 if (!strcmp(argv[0], "script"))
1755 return cmd_script(argc, argv, prefix);
Xiao Guangrongc0777c52009-12-07 12:04:49 +08001756
Arnaldo Carvalho de Melo75be6cf2009-12-15 20:04:39 -02001757 symbol__init();
Ingo Molnar1fc35b22009-09-13 09:44:29 +02001758 if (!strncmp(argv[0], "rec", 3)) {
1759 return __cmd_record(argc, argv);
1760 } else if (!strncmp(argv[0], "lat", 3)) {
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -03001761 sched.tp_handler = &lat_ops;
Ingo Molnarf2858d82009-09-11 12:12:54 +02001762 if (argc > 1) {
1763 argc = parse_options(argc, argv, latency_options, latency_usage, 0);
1764 if (argc)
1765 usage_with_options(latency_usage, latency_options);
Ingo Molnarf2858d82009-09-11 12:12:54 +02001766 }
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -03001767 setup_sorting(&sched, latency_options, latency_usage);
1768 return perf_sched__lat(&sched);
Ingo Molnar0ec04e12009-09-16 17:40:48 +02001769 } else if (!strcmp(argv[0], "map")) {
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -03001770 sched.tp_handler = &map_ops;
1771 setup_sorting(&sched, latency_options, latency_usage);
1772 return perf_sched__map(&sched);
Ingo Molnarf2858d82009-09-11 12:12:54 +02001773 } else if (!strncmp(argv[0], "rep", 3)) {
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -03001774 sched.tp_handler = &replay_ops;
Ingo Molnarf2858d82009-09-11 12:12:54 +02001775 if (argc) {
1776 argc = parse_options(argc, argv, replay_options, replay_usage, 0);
1777 if (argc)
1778 usage_with_options(replay_usage, replay_options);
1779 }
Arnaldo Carvalho de Melo0e9b07e2012-09-11 17:29:27 -03001780 return perf_sched__replay(&sched);
Ingo Molnarf2858d82009-09-11 12:12:54 +02001781 } else {
1782 usage_with_options(sched_usage, sched_options);
Ingo Molnar0a02ad92009-09-11 12:12:54 +02001783 }
1784
Ingo Molnarec156762009-09-11 12:12:54 +02001785 return 0;
Ingo Molnar0a02ad92009-09-11 12:12:54 +02001786}