blob: d969bb13a25281b875e9c4726d7a3fb51ff663ac [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/**
2 * @file buffer_sync.c
3 *
Robert Richterae735e92008-12-25 17:26:07 +01004 * @remark Copyright 2002-2009 OProfile authors
Linus Torvalds1da177e2005-04-16 15:20:36 -07005 * @remark Read the file COPYING
6 *
7 * @author John Levon <levon@movementarian.org>
Barry Kasindorf345c2572008-07-22 21:08:54 +02008 * @author Barry Kasindorf
Robert Richterae735e92008-12-25 17:26:07 +01009 * @author Robert Richter <robert.richter@amd.com>
Linus Torvalds1da177e2005-04-16 15:20:36 -070010 *
11 * This is the core of the buffer management. Each
12 * CPU buffer is processed and entered into the
13 * global event buffer. Such processing is necessary
14 * in several circumstances, mentioned below.
15 *
16 * The processing does the job of converting the
17 * transitory EIP value into a persistent dentry/offset
18 * value that the profiler can record at its leisure.
19 *
20 * See fs/dcookies.c for a description of the dentry/offset
21 * objects.
22 */
23
24#include <linux/mm.h>
25#include <linux/workqueue.h>
26#include <linux/notifier.h>
27#include <linux/dcookies.h>
28#include <linux/profile.h>
29#include <linux/module.h>
30#include <linux/fs.h>
Bob Nelson14748552007-07-20 21:39:53 +020031#include <linux/oprofile.h>
Alexey Dobriyane8edc6e2007-05-21 01:22:52 +040032#include <linux/sched.h>
Bob Nelson14748552007-07-20 21:39:53 +020033
Linus Torvalds1da177e2005-04-16 15:20:36 -070034#include "oprofile_stats.h"
35#include "event_buffer.h"
36#include "cpu_buffer.h"
37#include "buffer_sync.h"
Robert Richter73185e02008-07-22 21:08:51 +020038
Linus Torvalds1da177e2005-04-16 15:20:36 -070039static LIST_HEAD(dying_tasks);
40static LIST_HEAD(dead_tasks);
41static cpumask_t marked_cpus = CPU_MASK_NONE;
42static DEFINE_SPINLOCK(task_mortuary);
43static void process_task_mortuary(void);
44
Linus Torvalds1da177e2005-04-16 15:20:36 -070045/* Take ownership of the task struct and place it on the
46 * list for processing. Only after two full buffer syncs
47 * does the task eventually get freed, because by then
48 * we are sure we will not reference it again.
Paul E. McKenney4369ef32006-01-08 01:01:35 -080049 * Can be invoked from softirq via RCU callback due to
50 * call_rcu() of the task struct, hence the _irqsave.
Linus Torvalds1da177e2005-04-16 15:20:36 -070051 */
Robert Richter73185e02008-07-22 21:08:51 +020052static int
53task_free_notify(struct notifier_block *self, unsigned long val, void *data)
Linus Torvalds1da177e2005-04-16 15:20:36 -070054{
Paul E. McKenney4369ef32006-01-08 01:01:35 -080055 unsigned long flags;
Robert Richter73185e02008-07-22 21:08:51 +020056 struct task_struct *task = data;
Paul E. McKenney4369ef32006-01-08 01:01:35 -080057 spin_lock_irqsave(&task_mortuary, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -070058 list_add(&task->tasks, &dying_tasks);
Paul E. McKenney4369ef32006-01-08 01:01:35 -080059 spin_unlock_irqrestore(&task_mortuary, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -070060 return NOTIFY_OK;
61}
62
63
64/* The task is on its way out. A sync of the buffer means we can catch
65 * any remaining samples for this task.
66 */
Robert Richter73185e02008-07-22 21:08:51 +020067static int
68task_exit_notify(struct notifier_block *self, unsigned long val, void *data)
Linus Torvalds1da177e2005-04-16 15:20:36 -070069{
70 /* To avoid latency problems, we only process the current CPU,
71 * hoping that most samples for the task are on this CPU
72 */
Ingo Molnar39c715b2005-06-21 17:14:34 -070073 sync_buffer(raw_smp_processor_id());
Robert Richter73185e02008-07-22 21:08:51 +020074 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -070075}
76
77
78/* The task is about to try a do_munmap(). We peek at what it's going to
79 * do, and if it's an executable region, process the samples first, so
80 * we don't lose any. This does not have to be exact, it's a QoI issue
81 * only.
82 */
Robert Richter73185e02008-07-22 21:08:51 +020083static int
84munmap_notify(struct notifier_block *self, unsigned long val, void *data)
Linus Torvalds1da177e2005-04-16 15:20:36 -070085{
86 unsigned long addr = (unsigned long)data;
Robert Richter73185e02008-07-22 21:08:51 +020087 struct mm_struct *mm = current->mm;
88 struct vm_area_struct *mpnt;
Linus Torvalds1da177e2005-04-16 15:20:36 -070089
90 down_read(&mm->mmap_sem);
91
92 mpnt = find_vma(mm, addr);
93 if (mpnt && mpnt->vm_file && (mpnt->vm_flags & VM_EXEC)) {
94 up_read(&mm->mmap_sem);
95 /* To avoid latency problems, we only process the current CPU,
96 * hoping that most samples for the task are on this CPU
97 */
Ingo Molnar39c715b2005-06-21 17:14:34 -070098 sync_buffer(raw_smp_processor_id());
Linus Torvalds1da177e2005-04-16 15:20:36 -070099 return 0;
100 }
101
102 up_read(&mm->mmap_sem);
103 return 0;
104}
105
Robert Richter73185e02008-07-22 21:08:51 +0200106
Linus Torvalds1da177e2005-04-16 15:20:36 -0700107/* We need to be told about new modules so we don't attribute to a previously
108 * loaded module, or drop the samples on the floor.
109 */
Robert Richter73185e02008-07-22 21:08:51 +0200110static int
111module_load_notify(struct notifier_block *self, unsigned long val, void *data)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700112{
113#ifdef CONFIG_MODULES
114 if (val != MODULE_STATE_COMING)
115 return 0;
116
117 /* FIXME: should we process all CPU buffers ? */
Markus Armbruster59cc1852006-06-25 05:47:33 -0700118 mutex_lock(&buffer_mutex);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700119 add_event_entry(ESCAPE_CODE);
120 add_event_entry(MODULE_LOADED_CODE);
Markus Armbruster59cc1852006-06-25 05:47:33 -0700121 mutex_unlock(&buffer_mutex);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700122#endif
123 return 0;
124}
125
Robert Richter73185e02008-07-22 21:08:51 +0200126
Linus Torvalds1da177e2005-04-16 15:20:36 -0700127static struct notifier_block task_free_nb = {
128 .notifier_call = task_free_notify,
129};
130
131static struct notifier_block task_exit_nb = {
132 .notifier_call = task_exit_notify,
133};
134
135static struct notifier_block munmap_nb = {
136 .notifier_call = munmap_notify,
137};
138
139static struct notifier_block module_load_nb = {
140 .notifier_call = module_load_notify,
141};
142
Robert Richter73185e02008-07-22 21:08:51 +0200143
Linus Torvalds1da177e2005-04-16 15:20:36 -0700144static void end_sync(void)
145{
146 end_cpu_work();
147 /* make sure we don't leak task structs */
148 process_task_mortuary();
149 process_task_mortuary();
150}
151
152
153int sync_start(void)
154{
155 int err;
156
157 start_cpu_work();
158
159 err = task_handoff_register(&task_free_nb);
160 if (err)
161 goto out1;
162 err = profile_event_register(PROFILE_TASK_EXIT, &task_exit_nb);
163 if (err)
164 goto out2;
165 err = profile_event_register(PROFILE_MUNMAP, &munmap_nb);
166 if (err)
167 goto out3;
168 err = register_module_notifier(&module_load_nb);
169 if (err)
170 goto out4;
171
172out:
173 return err;
174out4:
175 profile_event_unregister(PROFILE_MUNMAP, &munmap_nb);
176out3:
177 profile_event_unregister(PROFILE_TASK_EXIT, &task_exit_nb);
178out2:
179 task_handoff_unregister(&task_free_nb);
180out1:
181 end_sync();
182 goto out;
183}
184
185
186void sync_stop(void)
187{
188 unregister_module_notifier(&module_load_nb);
189 profile_event_unregister(PROFILE_MUNMAP, &munmap_nb);
190 profile_event_unregister(PROFILE_TASK_EXIT, &task_exit_nb);
191 task_handoff_unregister(&task_free_nb);
192 end_sync();
193}
194
Jan Blunck448678a2008-02-14 19:38:36 -0800195
Linus Torvalds1da177e2005-04-16 15:20:36 -0700196/* Optimisation. We can manage without taking the dcookie sem
197 * because we cannot reach this code without at least one
198 * dcookie user still being registered (namely, the reader
199 * of the event buffer). */
Jan Blunck448678a2008-02-14 19:38:36 -0800200static inline unsigned long fast_get_dcookie(struct path *path)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700201{
202 unsigned long cookie;
Jan Blunck448678a2008-02-14 19:38:36 -0800203
204 if (path->dentry->d_cookie)
205 return (unsigned long)path->dentry;
206 get_dcookie(path, &cookie);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700207 return cookie;
208}
209
Jan Blunck448678a2008-02-14 19:38:36 -0800210
Linus Torvalds1da177e2005-04-16 15:20:36 -0700211/* Look up the dcookie for the task's first VM_EXECUTABLE mapping,
212 * which corresponds loosely to "application name". This is
213 * not strictly necessary but allows oprofile to associate
214 * shared-library samples with particular applications
215 */
Robert Richter73185e02008-07-22 21:08:51 +0200216static unsigned long get_exec_dcookie(struct mm_struct *mm)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700217{
John Levon0c0a4002005-06-23 22:02:47 -0700218 unsigned long cookie = NO_COOKIE;
Robert Richter73185e02008-07-22 21:08:51 +0200219 struct vm_area_struct *vma;
220
Linus Torvalds1da177e2005-04-16 15:20:36 -0700221 if (!mm)
222 goto out;
Robert Richter73185e02008-07-22 21:08:51 +0200223
Linus Torvalds1da177e2005-04-16 15:20:36 -0700224 for (vma = mm->mmap; vma; vma = vma->vm_next) {
225 if (!vma->vm_file)
226 continue;
227 if (!(vma->vm_flags & VM_EXECUTABLE))
228 continue;
Jan Blunck448678a2008-02-14 19:38:36 -0800229 cookie = fast_get_dcookie(&vma->vm_file->f_path);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700230 break;
231 }
232
233out:
234 return cookie;
235}
236
237
238/* Convert the EIP value of a sample into a persistent dentry/offset
239 * pair that can then be added to the global event buffer. We make
240 * sure to do this lookup before a mm->mmap modification happens so
241 * we don't lose track.
242 */
Robert Richter73185e02008-07-22 21:08:51 +0200243static unsigned long
244lookup_dcookie(struct mm_struct *mm, unsigned long addr, off_t *offset)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700245{
John Levon0c0a4002005-06-23 22:02:47 -0700246 unsigned long cookie = NO_COOKIE;
Robert Richter73185e02008-07-22 21:08:51 +0200247 struct vm_area_struct *vma;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700248
249 for (vma = find_vma(mm, addr); vma; vma = vma->vm_next) {
Robert Richter73185e02008-07-22 21:08:51 +0200250
Linus Torvalds1da177e2005-04-16 15:20:36 -0700251 if (addr < vma->vm_start || addr >= vma->vm_end)
252 continue;
253
John Levon0c0a4002005-06-23 22:02:47 -0700254 if (vma->vm_file) {
Jan Blunck448678a2008-02-14 19:38:36 -0800255 cookie = fast_get_dcookie(&vma->vm_file->f_path);
John Levon0c0a4002005-06-23 22:02:47 -0700256 *offset = (vma->vm_pgoff << PAGE_SHIFT) + addr -
257 vma->vm_start;
258 } else {
259 /* must be an anonymous map */
260 *offset = addr;
261 }
262
Linus Torvalds1da177e2005-04-16 15:20:36 -0700263 break;
264 }
265
John Levon0c0a4002005-06-23 22:02:47 -0700266 if (!vma)
267 cookie = INVALID_COOKIE;
268
Linus Torvalds1da177e2005-04-16 15:20:36 -0700269 return cookie;
270}
271
John Levon0c0a4002005-06-23 22:02:47 -0700272static unsigned long last_cookie = INVALID_COOKIE;
Robert Richter73185e02008-07-22 21:08:51 +0200273
Linus Torvalds1da177e2005-04-16 15:20:36 -0700274static void add_cpu_switch(int i)
275{
276 add_event_entry(ESCAPE_CODE);
277 add_event_entry(CPU_SWITCH_CODE);
278 add_event_entry(i);
John Levon0c0a4002005-06-23 22:02:47 -0700279 last_cookie = INVALID_COOKIE;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700280}
281
282static void add_kernel_ctx_switch(unsigned int in_kernel)
283{
284 add_event_entry(ESCAPE_CODE);
285 if (in_kernel)
Robert Richter73185e02008-07-22 21:08:51 +0200286 add_event_entry(KERNEL_ENTER_SWITCH_CODE);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700287 else
Robert Richter73185e02008-07-22 21:08:51 +0200288 add_event_entry(KERNEL_EXIT_SWITCH_CODE);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700289}
Robert Richter73185e02008-07-22 21:08:51 +0200290
Linus Torvalds1da177e2005-04-16 15:20:36 -0700291static void
Robert Richter73185e02008-07-22 21:08:51 +0200292add_user_ctx_switch(struct task_struct const *task, unsigned long cookie)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700293{
294 add_event_entry(ESCAPE_CODE);
Robert Richter73185e02008-07-22 21:08:51 +0200295 add_event_entry(CTX_SWITCH_CODE);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700296 add_event_entry(task->pid);
297 add_event_entry(cookie);
298 /* Another code for daemon back-compat */
299 add_event_entry(ESCAPE_CODE);
300 add_event_entry(CTX_TGID_CODE);
301 add_event_entry(task->tgid);
302}
303
Robert Richter73185e02008-07-22 21:08:51 +0200304
Linus Torvalds1da177e2005-04-16 15:20:36 -0700305static void add_cookie_switch(unsigned long cookie)
306{
307 add_event_entry(ESCAPE_CODE);
308 add_event_entry(COOKIE_SWITCH_CODE);
309 add_event_entry(cookie);
310}
311
Robert Richter73185e02008-07-22 21:08:51 +0200312
Linus Torvalds1da177e2005-04-16 15:20:36 -0700313static void add_trace_begin(void)
314{
315 add_event_entry(ESCAPE_CODE);
316 add_event_entry(TRACE_BEGIN_CODE);
317}
318
Robert Richter852402c2008-07-22 21:09:06 +0200319#ifdef CONFIG_OPROFILE_IBS
320
Barry Kasindorf345c2572008-07-22 21:08:54 +0200321#define IBS_FETCH_CODE_SIZE 2
322#define IBS_OP_CODE_SIZE 5
Barry Kasindorf345c2572008-07-22 21:08:54 +0200323
324/*
325 * Add IBS fetch and op entries to event buffer
326 */
Robert Richter6dad8282008-12-09 01:21:32 +0100327static void add_ibs_begin(int cpu, int code, struct mm_struct *mm)
Barry Kasindorf345c2572008-07-22 21:08:54 +0200328{
Robert Richterd358e752009-01-05 13:14:04 +0100329 unsigned long pc;
Barry Kasindorf345c2572008-07-22 21:08:54 +0200330 int i, count;
Robert Richterd358e752009-01-05 13:14:04 +0100331 unsigned long cookie = 0;
Barry Kasindorf345c2572008-07-22 21:08:54 +0200332 off_t offset;
Robert Richter2d87b142008-12-30 04:10:46 +0100333 struct op_entry entry;
Robert Richter6dad8282008-12-09 01:21:32 +0100334 struct op_sample *sample;
Barry Kasindorf345c2572008-07-22 21:08:54 +0200335
Robert Richter2d87b142008-12-30 04:10:46 +0100336 sample = op_cpu_buffer_read_entry(&entry, cpu);
Robert Richter6dad8282008-12-09 01:21:32 +0100337 if (!sample)
Robert Richterdbe6e282008-12-16 11:01:18 +0100338 return;
Robert Richterd358e752009-01-05 13:14:04 +0100339 pc = sample->eip;
Barry Kasindorf345c2572008-07-22 21:08:54 +0200340
341#ifdef __LP64__
Robert Richterd358e752009-01-05 13:14:04 +0100342 pc += sample->event << 32;
Barry Kasindorf345c2572008-07-22 21:08:54 +0200343#endif
344
345 if (mm) {
Robert Richterd358e752009-01-05 13:14:04 +0100346 cookie = lookup_dcookie(mm, pc, &offset);
Barry Kasindorf345c2572008-07-22 21:08:54 +0200347
Robert Richterd358e752009-01-05 13:14:04 +0100348 if (cookie == NO_COOKIE)
349 offset = pc;
350 if (cookie == INVALID_COOKIE) {
Barry Kasindorf345c2572008-07-22 21:08:54 +0200351 atomic_inc(&oprofile_stats.sample_lost_no_mapping);
Robert Richterd358e752009-01-05 13:14:04 +0100352 offset = pc;
Barry Kasindorf345c2572008-07-22 21:08:54 +0200353 }
Robert Richterd358e752009-01-05 13:14:04 +0100354 if (cookie != last_cookie) {
355 add_cookie_switch(cookie);
356 last_cookie = cookie;
Barry Kasindorf345c2572008-07-22 21:08:54 +0200357 }
358 } else
Robert Richterd358e752009-01-05 13:14:04 +0100359 offset = pc;
Barry Kasindorf345c2572008-07-22 21:08:54 +0200360
361 add_event_entry(ESCAPE_CODE);
362 add_event_entry(code);
363 add_event_entry(offset); /* Offset from Dcookie */
364
365 /* we send the Dcookie offset, but send the raw Linear Add also*/
Robert Richter6dad8282008-12-09 01:21:32 +0100366 add_event_entry(sample->eip);
367 add_event_entry(sample->event);
Barry Kasindorf345c2572008-07-22 21:08:54 +0200368
369 if (code == IBS_FETCH_CODE)
370 count = IBS_FETCH_CODE_SIZE; /*IBS FETCH is 2 int64s*/
371 else
372 count = IBS_OP_CODE_SIZE; /*IBS OP is 5 int64s*/
373
374 for (i = 0; i < count; i++) {
Robert Richter2d87b142008-12-30 04:10:46 +0100375 sample = op_cpu_buffer_read_entry(&entry, cpu);
Robert Richter6dad8282008-12-09 01:21:32 +0100376 if (!sample)
Robert Richterdbe6e282008-12-16 11:01:18 +0100377 return;
Robert Richter6dad8282008-12-09 01:21:32 +0100378 add_event_entry(sample->eip);
379 add_event_entry(sample->event);
Barry Kasindorf345c2572008-07-22 21:08:54 +0200380 }
Robert Richter6dad8282008-12-09 01:21:32 +0100381
382 return;
Barry Kasindorf345c2572008-07-22 21:08:54 +0200383}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700384
Robert Richter852402c2008-07-22 21:09:06 +0200385#endif
386
Robert Richter6368a1f2008-12-29 18:44:21 +0100387static inline void add_sample_entry(unsigned long offset, unsigned long event)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700388{
389 add_event_entry(offset);
390 add_event_entry(event);
391}
392
393
Robert Richter9741b302008-12-18 19:44:20 +0100394/*
395 * Add a sample to the global event buffer. If possible the
396 * sample is converted into a persistent dentry/offset pair
397 * for later lookup from userspace. Return 0 on failure.
398 */
399static int
400add_sample(struct mm_struct *mm, struct op_sample *s, int in_kernel)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700401{
402 unsigned long cookie;
403 off_t offset;
Robert Richter73185e02008-07-22 21:08:51 +0200404
Robert Richter9741b302008-12-18 19:44:20 +0100405 if (in_kernel) {
406 add_sample_entry(s->eip, s->event);
407 return 1;
408 }
409
410 /* add userspace sample */
411
412 if (!mm) {
413 atomic_inc(&oprofile_stats.sample_lost_no_mm);
414 return 0;
415 }
416
Robert Richter73185e02008-07-22 21:08:51 +0200417 cookie = lookup_dcookie(mm, s->eip, &offset);
418
John Levon0c0a4002005-06-23 22:02:47 -0700419 if (cookie == INVALID_COOKIE) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700420 atomic_inc(&oprofile_stats.sample_lost_no_mapping);
421 return 0;
422 }
423
424 if (cookie != last_cookie) {
425 add_cookie_switch(cookie);
426 last_cookie = cookie;
427 }
428
429 add_sample_entry(offset, s->event);
430
431 return 1;
432}
433
Robert Richter73185e02008-07-22 21:08:51 +0200434
Robert Richter73185e02008-07-22 21:08:51 +0200435static void release_mm(struct mm_struct *mm)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700436{
437 if (!mm)
438 return;
439 up_read(&mm->mmap_sem);
440 mmput(mm);
441}
442
443
Robert Richter73185e02008-07-22 21:08:51 +0200444static struct mm_struct *take_tasks_mm(struct task_struct *task)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700445{
Robert Richter73185e02008-07-22 21:08:51 +0200446 struct mm_struct *mm = get_task_mm(task);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700447 if (mm)
448 down_read(&mm->mmap_sem);
449 return mm;
450}
451
452
453static inline int is_code(unsigned long val)
454{
455 return val == ESCAPE_CODE;
456}
Robert Richter73185e02008-07-22 21:08:51 +0200457
Linus Torvalds1da177e2005-04-16 15:20:36 -0700458
Linus Torvalds1da177e2005-04-16 15:20:36 -0700459/* Move tasks along towards death. Any tasks on dead_tasks
460 * will definitely have no remaining references in any
461 * CPU buffers at this point, because we use two lists,
462 * and to have reached the list, it must have gone through
463 * one full sync already.
464 */
465static void process_task_mortuary(void)
466{
Paul E. McKenney4369ef32006-01-08 01:01:35 -0800467 unsigned long flags;
468 LIST_HEAD(local_dead_tasks);
Robert Richter73185e02008-07-22 21:08:51 +0200469 struct task_struct *task;
470 struct task_struct *ttask;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700471
Paul E. McKenney4369ef32006-01-08 01:01:35 -0800472 spin_lock_irqsave(&task_mortuary, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700473
Paul E. McKenney4369ef32006-01-08 01:01:35 -0800474 list_splice_init(&dead_tasks, &local_dead_tasks);
475 list_splice_init(&dying_tasks, &dead_tasks);
476
477 spin_unlock_irqrestore(&task_mortuary, flags);
478
479 list_for_each_entry_safe(task, ttask, &local_dead_tasks, tasks) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700480 list_del(&task->tasks);
481 free_task(task);
482 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700483}
484
485
486static void mark_done(int cpu)
487{
488 int i;
489
490 cpu_set(cpu, marked_cpus);
491
492 for_each_online_cpu(i) {
493 if (!cpu_isset(i, marked_cpus))
494 return;
495 }
496
497 /* All CPUs have been processed at least once,
498 * we can process the mortuary once
499 */
500 process_task_mortuary();
501
502 cpus_clear(marked_cpus);
503}
504
505
506/* FIXME: this is not sufficient if we implement syscall barrier backtrace
507 * traversal, the code switch to sb_sample_start at first kernel enter/exit
508 * switch so we need a fifth state and some special handling in sync_buffer()
509 */
510typedef enum {
511 sb_bt_ignore = -2,
512 sb_buffer_start,
513 sb_bt_start,
514 sb_sample_start,
515} sync_buffer_state;
516
517/* Sync one of the CPU's buffers into the global event buffer.
518 * Here we need to go through each batch of samples punctuated
519 * by context switch notes, taking the task's mmap_sem and doing
520 * lookup in task->mm->mmap to convert EIP into dcookie/offset
521 * value.
522 */
523void sync_buffer(int cpu)
524{
Linus Torvalds1da177e2005-04-16 15:20:36 -0700525 struct mm_struct *mm = NULL;
Robert Richterfd7826d2008-09-26 17:50:31 -0400526 struct mm_struct *oldmm;
Robert Richter73185e02008-07-22 21:08:51 +0200527 struct task_struct *new;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700528 unsigned long cookie = 0;
529 int in_kernel = 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700530 sync_buffer_state state = sb_buffer_start;
Barry Kasindorf9b1f2612008-07-15 00:10:36 +0200531 unsigned int i;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700532 unsigned long available;
Robert Richterae735e92008-12-25 17:26:07 +0100533 unsigned long flags;
Robert Richter2d87b142008-12-30 04:10:46 +0100534 struct op_entry entry;
535 struct op_sample *sample;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700536
Markus Armbruster59cc1852006-06-25 05:47:33 -0700537 mutex_lock(&buffer_mutex);
Robert Richter73185e02008-07-22 21:08:51 +0200538
Linus Torvalds1da177e2005-04-16 15:20:36 -0700539 add_cpu_switch(cpu);
540
Robert Richter6d2c53f2008-12-24 16:53:53 +0100541 op_cpu_buffer_reset(cpu);
542 available = op_cpu_buffer_entries(cpu);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700543
544 for (i = 0; i < available; ++i) {
Robert Richter2d87b142008-12-30 04:10:46 +0100545 sample = op_cpu_buffer_read_entry(&entry, cpu);
546 if (!sample)
Robert Richter6dad8282008-12-09 01:21:32 +0100547 break;
Robert Richter73185e02008-07-22 21:08:51 +0200548
Robert Richter2d87b142008-12-30 04:10:46 +0100549 if (is_code(sample->eip)) {
Robert Richterae735e92008-12-25 17:26:07 +0100550 flags = sample->event;
551 if (flags & TRACE_BEGIN) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700552 state = sb_bt_start;
553 add_trace_begin();
Robert Richterae735e92008-12-25 17:26:07 +0100554 }
555 if (flags & KERNEL_CTX_SWITCH) {
556 /* kernel/userspace switch */
557 in_kernel = flags & IS_KERNEL;
558 if (state == sb_buffer_start)
559 state = sb_sample_start;
560 add_kernel_ctx_switch(flags & IS_KERNEL);
561 }
562 if (flags & USER_CTX_SWITCH) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700563 /* userspace context switch */
Robert Richterfd7826d2008-09-26 17:50:31 -0400564 oldmm = mm;
Robert Richterae735e92008-12-25 17:26:07 +0100565 new = (struct task_struct *)sample->data[0];
Linus Torvalds1da177e2005-04-16 15:20:36 -0700566 release_mm(oldmm);
567 mm = take_tasks_mm(new);
568 if (mm != oldmm)
569 cookie = get_exec_dcookie(mm);
570 add_user_ctx_switch(new, cookie);
571 }
Robert Richterae735e92008-12-25 17:26:07 +0100572#ifdef CONFIG_OPROFILE_IBS
573 if (flags & IBS_FETCH_BEGIN)
574 add_ibs_begin(cpu, IBS_FETCH_CODE, mm);
575 if (flags & IBS_OP_BEGIN)
576 add_ibs_begin(cpu, IBS_OP_CODE, mm);
577#endif
Robert Richter317f33b2008-12-18 19:44:20 +0100578 continue;
579 }
580
581 if (state < sb_bt_start)
582 /* ignore sample */
583 continue;
584
Robert Richter2d87b142008-12-30 04:10:46 +0100585 if (add_sample(mm, sample, in_kernel))
Robert Richter317f33b2008-12-18 19:44:20 +0100586 continue;
587
588 /* ignore backtraces if failed to add a sample */
589 if (state == sb_bt_start) {
590 state = sb_bt_ignore;
591 atomic_inc(&oprofile_stats.bt_lost_no_mapping);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700592 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700593 }
594 release_mm(mm);
595
596 mark_done(cpu);
597
Markus Armbruster59cc1852006-06-25 05:47:33 -0700598 mutex_unlock(&buffer_mutex);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700599}
Carl Lovea5598ca2008-10-14 23:37:01 +0000600
601/* The function can be used to add a buffer worth of data directly to
602 * the kernel buffer. The buffer is assumed to be a circular buffer.
603 * Take the entries from index start and end at index end, wrapping
604 * at max_entries.
605 */
606void oprofile_put_buff(unsigned long *buf, unsigned int start,
607 unsigned int stop, unsigned int max)
608{
609 int i;
610
611 i = start;
612
613 mutex_lock(&buffer_mutex);
614 while (i != stop) {
615 add_event_entry(buf[i++]);
616
617 if (i >= max)
618 i = 0;
619 }
620
621 mutex_unlock(&buffer_mutex);
622}
623