blob: 486e84ccb1f92545ec0d4f74aaa153abf0ff8049 [file] [log] [blame]
Thomas Gleixner0793a612008-12-04 20:12:29 +01001/*
Ingo Molnar57c0c152009-09-21 12:20:38 +02002 * Performance events:
Thomas Gleixner0793a612008-12-04 20:12:29 +01003 *
Ingo Molnara3084442009-06-11 14:44:26 +02004 * Copyright (C) 2008-2009, Thomas Gleixner <tglx@linutronix.de>
Ingo Molnare7e7ee22011-05-04 08:42:29 +02005 * Copyright (C) 2008-2011, Red Hat, Inc., Ingo Molnar
6 * Copyright (C) 2008-2011, Red Hat, Inc., Peter Zijlstra
Thomas Gleixner0793a612008-12-04 20:12:29 +01007 *
Ingo Molnar57c0c152009-09-21 12:20:38 +02008 * Data type definitions, declarations, prototypes.
Thomas Gleixner0793a612008-12-04 20:12:29 +01009 *
Ingo Molnara3084442009-06-11 14:44:26 +020010 * Started by: Thomas Gleixner and Ingo Molnar
Thomas Gleixner0793a612008-12-04 20:12:29 +010011 *
Ingo Molnar57c0c152009-09-21 12:20:38 +020012 * For licencing details see kernel-base/COPYING
Thomas Gleixner0793a612008-12-04 20:12:29 +010013 */
Ingo Molnarcdd6c482009-09-21 12:02:48 +020014#ifndef _LINUX_PERF_EVENT_H
15#define _LINUX_PERF_EVENT_H
Thomas Gleixner0793a612008-12-04 20:12:29 +010016
David Howells607ca462012-10-13 10:46:48 +010017#include <uapi/linux/perf_event.h>
Peter Zijlstraa4be7c22009-08-19 11:18:27 +020018
Paul Mackerrasd859e292009-01-17 18:10:22 +110019/*
Paul Mackerrasf3dfd262009-02-26 22:43:46 +110020 * Kernel-internal data types and definitions:
Ingo Molnar9f66a382008-12-10 12:33:23 +010021 */
22
Ingo Molnarcdd6c482009-09-21 12:02:48 +020023#ifdef CONFIG_PERF_EVENTS
24# include <asm/perf_event.h>
Peter Zijlstra7be79232010-06-09 11:57:23 +020025# include <asm/local64.h>
Paul Mackerrasf3dfd262009-02-26 22:43:46 +110026#endif
27
Zhang, Yanmin39447b32010-04-19 13:32:41 +080028struct perf_guest_info_callbacks {
Ingo Molnare7e7ee22011-05-04 08:42:29 +020029 int (*is_in_guest)(void);
30 int (*is_user_mode)(void);
31 unsigned long (*get_guest_ip)(void);
Zhang, Yanmin39447b32010-04-19 13:32:41 +080032};
33
Arnd Bergmann2ff6cfd2009-12-07 17:12:58 +010034#ifdef CONFIG_HAVE_HW_BREAKPOINT
35#include <asm/hw_breakpoint.h>
36#endif
37
Paul Mackerrasf3dfd262009-02-26 22:43:46 +110038#include <linux/list.h>
39#include <linux/mutex.h>
40#include <linux/rculist.h>
41#include <linux/rcupdate.h>
42#include <linux/spinlock.h>
Peter Zijlstrad6d020e2009-03-13 12:21:35 +010043#include <linux/hrtimer.h>
Peter Zijlstra3c446b3d2009-04-06 11:45:01 +020044#include <linux/fs.h>
Peter Zijlstra709e50c2009-06-02 14:13:15 +020045#include <linux/pid_namespace.h>
Peter Zijlstra906010b2009-09-21 16:08:49 +020046#include <linux/workqueue.h>
Frederic Weisbecker5331d7b2010-03-04 21:15:56 +010047#include <linux/ftrace.h>
Peter Zijlstra85cfabb2010-03-11 13:06:56 +010048#include <linux/cpu.h>
Peter Zijlstrae360adb2010-10-14 14:01:34 +080049#include <linux/irq_work.h>
Ingo Molnarc5905af2012-02-24 08:31:31 +010050#include <linux/static_key.h>
Andrew Jones851cf6e2013-08-09 19:51:57 +053051#include <linux/jump_label_ratelimit.h>
Arun Sharma600634972011-07-26 16:09:06 -070052#include <linux/atomic.h>
Jiri Olsa641cc932012-03-15 20:09:14 +010053#include <linux/sysfs.h>
Jiri Olsa40189942012-08-07 15:20:37 +020054#include <linux/perf_regs.h>
Jiri Olsafadfe7b2014-08-01 14:33:02 +020055#include <linux/workqueue.h>
Peter Zijlstrafa588152010-05-18 10:54:20 +020056#include <asm/local.h>
Paul Mackerrasf3dfd262009-02-26 22:43:46 +110057
Peter Zijlstraf9188e02009-06-18 22:20:52 +020058struct perf_callchain_entry {
59 __u64 nr;
60 __u64 ip[PERF_MAX_STACK_DEPTH];
61};
62
Frederic Weisbecker3a43ce62009-08-08 04:26:37 +020063struct perf_raw_record {
64 u32 size;
65 void *data;
Frederic Weisbeckerf413cdb2009-08-07 01:25:54 +020066};
67
Stephane Eranianbce38cd2012-02-09 23:20:51 +010068/*
Stephane Eranianbce38cd2012-02-09 23:20:51 +010069 * branch stack layout:
70 * nr: number of taken branches stored in entries[]
71 *
72 * Note that nr can vary from sample to sample
73 * branches (to, from) are stored from most recent
74 * to least recent, i.e., entries[0] contains the most
75 * recent branch.
76 */
Peter Zijlstracaff2be2010-03-03 12:02:30 +010077struct perf_branch_stack {
78 __u64 nr;
79 struct perf_branch_entry entries[0];
80};
81
Stephane Eranian60e23642014-09-24 13:48:37 +020082struct perf_regs {
Jiri Olsa40189942012-08-07 15:20:37 +020083 __u64 abi;
84 struct pt_regs *regs;
85};
86
Paul Mackerrasf3dfd262009-02-26 22:43:46 +110087struct task_struct;
88
Stephane Eranianefc9f052011-06-06 16:57:03 +020089/*
90 * extra PMU register associated with an event
91 */
92struct hw_perf_event_extra {
93 u64 config; /* register value */
94 unsigned int reg; /* register address or index */
95 int alloc; /* extra register already allocated */
96 int idx; /* index in shared_regs->regs[] */
97};
98
Andrew Hunter43b457802013-05-23 11:07:03 -070099struct event_constraint;
100
Thomas Gleixner0793a612008-12-04 20:12:29 +0100101/**
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200102 * struct hw_perf_event - performance event hardware details:
Thomas Gleixner0793a612008-12-04 20:12:29 +0100103 */
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200104struct hw_perf_event {
105#ifdef CONFIG_PERF_EVENTS
Peter Zijlstrad6d020e2009-03-13 12:21:35 +0100106 union {
107 struct { /* hardware */
Ingo Molnara3084442009-06-11 14:44:26 +0200108 u64 config;
Stephane Eranian447a1942010-02-01 14:50:01 +0200109 u64 last_tag;
Ingo Molnara3084442009-06-11 14:44:26 +0200110 unsigned long config_base;
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200111 unsigned long event_base;
Vince Weaverc48b6052012-03-01 17:28:14 -0500112 int event_base_rdpmc;
Ingo Molnara3084442009-06-11 14:44:26 +0200113 int idx;
Stephane Eranian447a1942010-02-01 14:50:01 +0200114 int last_cpu;
Stephane Eranian9fac2cf2013-01-24 16:10:27 +0100115 int flags;
Stephane Eranianbce38cd2012-02-09 23:20:51 +0100116
Stephane Eranianefc9f052011-06-06 16:57:03 +0200117 struct hw_perf_event_extra extra_reg;
Stephane Eranianbce38cd2012-02-09 23:20:51 +0100118 struct hw_perf_event_extra branch_reg;
Andrew Hunter43b457802013-05-23 11:07:03 -0700119
120 struct event_constraint *constraint;
Peter Zijlstrad6d020e2009-03-13 12:21:35 +0100121 };
Soeren Sandmann721a6692009-09-15 14:33:08 +0200122 struct { /* software */
Ingo Molnara3084442009-06-11 14:44:26 +0200123 struct hrtimer hrtimer;
Peter Zijlstrad6d020e2009-03-13 12:21:35 +0100124 };
Oleg Nesterovf22c1bb2013-02-02 16:27:52 +0100125 struct { /* tracepoint */
126 struct task_struct *tp_target;
127 /* for tp_event->class */
128 struct list_head tp_list;
129 };
Frederic Weisbecker24f1e32c2009-09-09 19:22:48 +0200130#ifdef CONFIG_HAVE_HW_BREAKPOINT
Frederic Weisbecker45a73372010-06-23 23:00:37 +0200131 struct { /* breakpoint */
Peter Zijlstrad580ff82010-10-14 17:43:23 +0200132 /*
133 * Crufty hack to avoid the chicken and egg
134 * problem hw_breakpoint has with context
135 * creation and event initalization.
136 */
137 struct task_struct *bp_target;
Oleg Nesterovf22c1bb2013-02-02 16:27:52 +0100138 struct arch_hw_breakpoint info;
139 struct list_head bp_list;
Frederic Weisbecker45a73372010-06-23 23:00:37 +0200140 };
Frederic Weisbecker24f1e32c2009-09-09 19:22:48 +0200141#endif
Peter Zijlstrad6d020e2009-03-13 12:21:35 +0100142 };
Peter Zijlstraa4eaf7f2010-06-16 14:37:10 +0200143 int state;
Peter Zijlstrae7850592010-05-21 14:43:08 +0200144 local64_t prev_count;
Peter Zijlstrab23f3322009-06-02 15:13:03 +0200145 u64 sample_period;
Peter Zijlstra9e350de2009-06-10 21:34:59 +0200146 u64 last_period;
Peter Zijlstrae7850592010-05-21 14:43:08 +0200147 local64_t period_left;
Stephane Eraniane050e3f2012-01-26 17:03:19 +0100148 u64 interrupts_seq;
Peter Zijlstra60db5e02009-05-15 15:19:28 +0200149 u64 interrupts;
Peter Zijlstra6a24ed6c2009-06-05 18:01:29 +0200150
Peter Zijlstraabd50712010-01-26 18:50:16 +0100151 u64 freq_time_stamp;
152 u64 freq_count_stamp;
Ingo Molnaree060942008-12-13 09:00:03 +0100153#endif
Thomas Gleixner0793a612008-12-04 20:12:29 +0100154};
155
Peter Zijlstraa4eaf7f2010-06-16 14:37:10 +0200156/*
157 * hw_perf_event::state flags
158 */
159#define PERF_HES_STOPPED 0x01 /* the counter is stopped */
160#define PERF_HES_UPTODATE 0x02 /* event->count up-to-date */
161#define PERF_HES_ARCH 0x04
162
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200163struct perf_event;
Ingo Molnar621a01e2008-12-11 12:46:46 +0100164
Peter Zijlstra8d2cacb2010-05-25 17:49:05 +0200165/*
166 * Common implementation detail of pmu::{start,commit,cancel}_txn
167 */
168#define PERF_EVENT_TXN 0x1
Lin Ming6bde9b62010-04-23 13:56:00 +0800169
Ingo Molnar621a01e2008-12-11 12:46:46 +0100170/**
Vince Weaver53b25332014-05-16 17:12:12 -0400171 * pmu::capabilities flags
172 */
173#define PERF_PMU_CAP_NO_INTERRUPT 0x01
174
175/**
Robert Richter4aeb0b42009-04-29 12:47:03 +0200176 * struct pmu - generic performance monitoring unit
Ingo Molnar621a01e2008-12-11 12:46:46 +0100177 */
Robert Richter4aeb0b42009-04-29 12:47:03 +0200178struct pmu {
Peter Zijlstrab0a873e2010-06-11 13:35:08 +0200179 struct list_head entry;
180
Yan, Zhengc464c762014-03-18 16:56:41 +0800181 struct module *module;
Peter Zijlstraabe43402010-11-17 23:17:37 +0100182 struct device *dev;
Peter Zijlstra0c9d42e2011-11-20 23:30:47 +0100183 const struct attribute_group **attr_groups;
Mischa Jonker03d8e802013-06-04 11:45:48 +0200184 const char *name;
Peter Zijlstra2e80a822010-11-17 23:17:36 +0100185 int type;
186
Vince Weaver53b25332014-05-16 17:12:12 -0400187 /*
188 * various common per-pmu feature flags
189 */
190 int capabilities;
191
Peter Zijlstra108b02c2010-09-06 14:32:03 +0200192 int * __percpu pmu_disable_count;
193 struct perf_cpu_context * __percpu pmu_cpu_context;
Peter Zijlstra8dc85d52010-09-02 16:50:03 +0200194 int task_ctx_nr;
Stephane Eranian62b85632013-04-03 14:21:34 +0200195 int hrtimer_interval_ms;
Peter Zijlstra33696fc2010-06-14 08:49:00 +0200196
Peter Zijlstraa4eaf7f2010-06-16 14:37:10 +0200197 /*
198 * Fully disable/enable this PMU, can be used to protect from the PMI
199 * as well as for lazy/batch writing of the MSRs.
200 */
Peter Zijlstraad5133b2010-06-15 12:22:39 +0200201 void (*pmu_enable) (struct pmu *pmu); /* optional */
202 void (*pmu_disable) (struct pmu *pmu); /* optional */
Peter Zijlstra33696fc2010-06-14 08:49:00 +0200203
Peter Zijlstrab0a873e2010-06-11 13:35:08 +0200204 /*
Peter Zijlstraa4eaf7f2010-06-16 14:37:10 +0200205 * Try and initialize the event for this PMU.
Peter Zijlstra24cd7f52010-06-11 17:32:03 +0200206 * Should return -ENOENT when the @event doesn't match this PMU.
Peter Zijlstrab0a873e2010-06-11 13:35:08 +0200207 */
208 int (*event_init) (struct perf_event *event);
209
Peter Zijlstraa4eaf7f2010-06-16 14:37:10 +0200210#define PERF_EF_START 0x01 /* start the counter when adding */
211#define PERF_EF_RELOAD 0x02 /* reload the counter when starting */
212#define PERF_EF_UPDATE 0x04 /* update the counter when stopping */
213
214 /*
215 * Adds/Removes a counter to/from the PMU, can be done inside
216 * a transaction, see the ->*_txn() methods.
217 */
218 int (*add) (struct perf_event *event, int flags);
219 void (*del) (struct perf_event *event, int flags);
220
221 /*
222 * Starts/Stops a counter present on the PMU. The PMI handler
223 * should stop the counter when perf_event_overflow() returns
224 * !0. ->start() will be used to continue.
225 */
226 void (*start) (struct perf_event *event, int flags);
227 void (*stop) (struct perf_event *event, int flags);
228
229 /*
230 * Updates the counter value of the event.
231 */
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200232 void (*read) (struct perf_event *event);
Lin Ming6bde9b62010-04-23 13:56:00 +0800233
234 /*
Peter Zijlstra24cd7f52010-06-11 17:32:03 +0200235 * Group events scheduling is treated as a transaction, add
236 * group events as a whole and perform one schedulability test.
237 * If the test fails, roll back the whole group
Peter Zijlstraa4eaf7f2010-06-16 14:37:10 +0200238 *
239 * Start the transaction, after this ->add() doesn't need to
Peter Zijlstra24cd7f52010-06-11 17:32:03 +0200240 * do schedulability tests.
Lin Ming6bde9b62010-04-23 13:56:00 +0800241 */
Ingo Molnare7e7ee22011-05-04 08:42:29 +0200242 void (*start_txn) (struct pmu *pmu); /* optional */
Peter Zijlstra8d2cacb2010-05-25 17:49:05 +0200243 /*
Peter Zijlstraa4eaf7f2010-06-16 14:37:10 +0200244 * If ->start_txn() disabled the ->add() schedulability test
Peter Zijlstra8d2cacb2010-05-25 17:49:05 +0200245 * then ->commit_txn() is required to perform one. On success
246 * the transaction is closed. On error the transaction is kept
247 * open until ->cancel_txn() is called.
248 */
Ingo Molnare7e7ee22011-05-04 08:42:29 +0200249 int (*commit_txn) (struct pmu *pmu); /* optional */
Peter Zijlstra8d2cacb2010-05-25 17:49:05 +0200250 /*
Peter Zijlstraa4eaf7f2010-06-16 14:37:10 +0200251 * Will cancel the transaction, assumes ->del() is called
Lucas De Marchi25985ed2011-03-30 22:57:33 -0300252 * for each successful ->add() during the transaction.
Peter Zijlstra8d2cacb2010-05-25 17:49:05 +0200253 */
Ingo Molnare7e7ee22011-05-04 08:42:29 +0200254 void (*cancel_txn) (struct pmu *pmu); /* optional */
Peter Zijlstra35edc2a2011-11-20 20:36:02 +0100255
256 /*
257 * Will return the value for perf_event_mmap_page::index for this event,
258 * if no implementation is provided it will default to: event->hw.idx + 1.
259 */
260 int (*event_idx) (struct perf_event *event); /*optional */
Stephane Eraniand010b332012-02-09 23:21:00 +0100261
262 /*
263 * flush branch stack on context-switches (needed in cpu-wide mode)
264 */
265 void (*flush_branch_stack) (void);
Ingo Molnar621a01e2008-12-11 12:46:46 +0100266};
267
Thomas Gleixner0793a612008-12-04 20:12:29 +0100268/**
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200269 * enum perf_event_active_state - the states of a event
Ingo Molnar6a930702008-12-11 15:17:03 +0100270 */
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200271enum perf_event_active_state {
Jiri Olsa179033b2014-08-07 11:48:26 -0400272 PERF_EVENT_STATE_EXIT = -3,
Ingo Molnar57c0c152009-09-21 12:20:38 +0200273 PERF_EVENT_STATE_ERROR = -2,
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200274 PERF_EVENT_STATE_OFF = -1,
275 PERF_EVENT_STATE_INACTIVE = 0,
Ingo Molnar57c0c152009-09-21 12:20:38 +0200276 PERF_EVENT_STATE_ACTIVE = 1,
Ingo Molnar6a930702008-12-11 15:17:03 +0100277};
278
Ingo Molnar9b51f662008-12-12 13:49:45 +0100279struct file;
Peter Zijlstra453f19e2009-11-20 22:19:43 +0100280struct perf_sample_data;
281
Peter Zijlstraa8b0ca12011-06-27 14:41:57 +0200282typedef void (*perf_overflow_handler_t)(struct perf_event *,
Frederic Weisbeckerb326e952009-12-05 09:44:31 +0100283 struct perf_sample_data *,
284 struct pt_regs *regs);
285
Frederic Weisbeckerd6f962b2010-01-10 01:25:51 +0100286enum perf_group_flag {
Ingo Molnare7e7ee22011-05-04 08:42:29 +0200287 PERF_GROUP_SOFTWARE = 0x1,
Frederic Weisbeckerd6f962b2010-01-10 01:25:51 +0100288};
289
Ingo Molnare7e7ee22011-05-04 08:42:29 +0200290#define SWEVENT_HLIST_BITS 8
291#define SWEVENT_HLIST_SIZE (1 << SWEVENT_HLIST_BITS)
Frederic Weisbecker76e1d902010-04-05 15:35:57 +0200292
293struct swevent_hlist {
Ingo Molnare7e7ee22011-05-04 08:42:29 +0200294 struct hlist_head heads[SWEVENT_HLIST_SIZE];
295 struct rcu_head rcu_head;
Frederic Weisbecker76e1d902010-04-05 15:35:57 +0200296};
297
Peter Zijlstra8a495422010-05-27 15:47:49 +0200298#define PERF_ATTACH_CONTEXT 0x01
299#define PERF_ATTACH_GROUP 0x02
Peter Zijlstrad580ff82010-10-14 17:43:23 +0200300#define PERF_ATTACH_TASK 0x04
Peter Zijlstra8a495422010-05-27 15:47:49 +0200301
Li Zefan877c6852013-03-05 11:38:08 +0800302struct perf_cgroup;
Frederic Weisbecker76369132011-05-19 19:55:04 +0200303struct ring_buffer;
304
Ingo Molnar6a930702008-12-11 15:17:03 +0100305/**
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200306 * struct perf_event - performance event kernel representation:
Thomas Gleixner0793a612008-12-04 20:12:29 +0100307 */
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200308struct perf_event {
309#ifdef CONFIG_PERF_EVENTS
Peter Zijlstra98861672013-10-03 16:02:23 +0200310 /*
311 * entry onto perf_event_context::event_list;
312 * modifications require ctx->lock
313 * RCU safe iterations.
314 */
Peter Zijlstra592903c2009-03-13 12:21:36 +0100315 struct list_head event_entry;
Peter Zijlstra98861672013-10-03 16:02:23 +0200316
317 /*
318 * XXX: group_entry and sibling_list should be mutually exclusive;
319 * either you're a sibling on a group, or you're the group leader.
320 * Rework the code to always use the same list element.
321 *
322 * Locked for modification by both ctx->mutex and ctx->lock; holding
323 * either sufficies for read.
324 */
325 struct list_head group_entry;
Ingo Molnar04289bb2008-12-11 08:38:42 +0100326 struct list_head sibling_list;
Peter Zijlstra98861672013-10-03 16:02:23 +0200327
328 /*
329 * We need storage to track the entries in perf_pmu_migrate_context; we
330 * cannot use the event_entry because of RCU and we want to keep the
331 * group in tact which avoids us using the other two entries.
332 */
333 struct list_head migrate_entry;
334
Stephane Eranianf3ae75d2014-01-08 11:15:52 +0100335 struct hlist_node hlist_entry;
336 struct list_head active_entry;
Ingo Molnar0127c3e2009-05-25 22:03:26 +0200337 int nr_siblings;
Frederic Weisbeckerd6f962b2010-01-10 01:25:51 +0100338 int group_flags;
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200339 struct perf_event *group_leader;
Peter Zijlstraa4eaf7f2010-06-16 14:37:10 +0200340 struct pmu *pmu;
Ingo Molnar04289bb2008-12-11 08:38:42 +0100341
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200342 enum perf_event_active_state state;
Peter Zijlstra8a495422010-05-27 15:47:49 +0200343 unsigned int attach_state;
Peter Zijlstrae7850592010-05-21 14:43:08 +0200344 local64_t count;
Peter Zijlstraa6e6dea2010-05-21 14:27:58 +0200345 atomic64_t child_count;
Ingo Molnaree060942008-12-13 09:00:03 +0100346
Paul Mackerras53cfbf52009-03-25 22:46:58 +1100347 /*
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200348 * These are the total time in nanoseconds that the event
Paul Mackerras53cfbf52009-03-25 22:46:58 +1100349 * has been enabled (i.e. eligible to run, and the task has
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200350 * been scheduled in, if this is a per-task event)
Paul Mackerras53cfbf52009-03-25 22:46:58 +1100351 * and running (scheduled onto the CPU), respectively.
352 *
353 * They are computed from tstamp_enabled, tstamp_running and
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200354 * tstamp_stopped when the event is in INACTIVE or ACTIVE state.
Paul Mackerras53cfbf52009-03-25 22:46:58 +1100355 */
356 u64 total_time_enabled;
357 u64 total_time_running;
358
359 /*
360 * These are timestamps used for computing total_time_enabled
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200361 * and total_time_running when the event is in INACTIVE or
Paul Mackerras53cfbf52009-03-25 22:46:58 +1100362 * ACTIVE state, measured in nanoseconds from an arbitrary point
363 * in time.
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200364 * tstamp_enabled: the notional time when the event was enabled
365 * tstamp_running: the notional time when the event was scheduled on
Paul Mackerras53cfbf52009-03-25 22:46:58 +1100366 * tstamp_stopped: in INACTIVE state, the notional time when the
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200367 * event was scheduled off.
Paul Mackerras53cfbf52009-03-25 22:46:58 +1100368 */
369 u64 tstamp_enabled;
370 u64 tstamp_running;
371 u64 tstamp_stopped;
372
Stephane Eranianeed01522010-10-26 16:08:01 +0200373 /*
374 * timestamp shadows the actual context timing but it can
375 * be safely used in NMI interrupt context. It reflects the
376 * context time as it was when the event was last scheduled in.
377 *
378 * ctx_time already accounts for ctx->timestamp. Therefore to
379 * compute ctx_time for a sample, simply add perf_clock().
380 */
381 u64 shadow_ctx_time;
382
Frederic Weisbecker24f1e32c2009-09-09 19:22:48 +0200383 struct perf_event_attr attr;
Arnaldo Carvalho de Meloc320c7b2010-10-20 12:50:11 -0200384 u16 header_size;
Arnaldo Carvalho de Melo6844c092010-12-03 16:36:35 -0200385 u16 id_header_size;
Arnaldo Carvalho de Meloc320c7b2010-10-20 12:50:11 -0200386 u16 read_size;
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200387 struct hw_perf_event hw;
Thomas Gleixner0793a612008-12-04 20:12:29 +0100388
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200389 struct perf_event_context *ctx;
Al Viroa6fa9412012-08-20 14:59:25 +0100390 atomic_long_t refcount;
Thomas Gleixner0793a612008-12-04 20:12:29 +0100391
392 /*
Paul Mackerras53cfbf52009-03-25 22:46:58 +1100393 * These accumulate total time (in nanoseconds) that children
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200394 * events have been enabled and running, respectively.
Paul Mackerras53cfbf52009-03-25 22:46:58 +1100395 */
396 atomic64_t child_total_time_enabled;
397 atomic64_t child_total_time_running;
398
399 /*
Paul Mackerrasd859e292009-01-17 18:10:22 +1100400 * Protect attach/detach and child_list:
Thomas Gleixner0793a612008-12-04 20:12:29 +0100401 */
Peter Zijlstrafccc7142009-05-23 18:28:56 +0200402 struct mutex child_mutex;
403 struct list_head child_list;
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200404 struct perf_event *parent;
Thomas Gleixner0793a612008-12-04 20:12:29 +0100405
406 int oncpu;
407 int cpu;
408
Peter Zijlstra082ff5a2009-05-23 18:29:00 +0200409 struct list_head owner_entry;
410 struct task_struct *owner;
411
Peter Zijlstra7b732a72009-03-23 18:22:10 +0100412 /* mmap bits */
413 struct mutex mmap_mutex;
414 atomic_t mmap_count;
Peter Zijlstra26cb63a2013-05-28 10:55:48 +0200415
Frederic Weisbecker76369132011-05-19 19:55:04 +0200416 struct ring_buffer *rb;
Peter Zijlstra10c6db12011-11-26 02:47:31 +0100417 struct list_head rb_entry;
Peter Zijlstrab69cf532014-03-14 10:50:33 +0100418 unsigned long rcu_batches;
419 int rcu_pending;
Paul Mackerras37d81822009-03-23 18:22:08 +0100420
Peter Zijlstra7b732a72009-03-23 18:22:10 +0100421 /* poll related */
Thomas Gleixner0793a612008-12-04 20:12:29 +0100422 wait_queue_head_t waitq;
Peter Zijlstra3c446b3d2009-04-06 11:45:01 +0200423 struct fasync_struct *fasync;
Peter Zijlstra79f14642009-04-06 11:45:07 +0200424
425 /* delayed work for NMIs and such */
426 int pending_wakeup;
Peter Zijlstra4c9e2542009-04-06 11:45:09 +0200427 int pending_kill;
Peter Zijlstra79f14642009-04-06 11:45:07 +0200428 int pending_disable;
Peter Zijlstrae360adb2010-10-14 14:01:34 +0800429 struct irq_work pending;
Peter Zijlstra592903c2009-03-13 12:21:36 +0100430
Peter Zijlstra79f14642009-04-06 11:45:07 +0200431 atomic_t event_limit;
432
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200433 void (*destroy)(struct perf_event *);
Peter Zijlstra592903c2009-03-13 12:21:36 +0100434 struct rcu_head rcu_head;
Peter Zijlstra709e50c2009-06-02 14:13:15 +0200435
436 struct pid_namespace *ns;
Peter Zijlstra8e5799b2009-06-02 15:08:15 +0200437 u64 id;
Li Zefan6fb29152009-10-15 11:21:42 +0800438
Frederic Weisbeckerb326e952009-12-05 09:44:31 +0100439 perf_overflow_handler_t overflow_handler;
Avi Kivity4dc0da82011-06-29 18:42:35 +0300440 void *overflow_handler_context;
Peter Zijlstra453f19e2009-11-20 22:19:43 +0100441
Li Zefan07b139c2009-12-21 14:27:35 +0800442#ifdef CONFIG_EVENT_TRACING
Peter Zijlstra1c024eca2010-05-19 14:02:22 +0200443 struct ftrace_event_call *tp_event;
Li Zefan6fb29152009-10-15 11:21:42 +0800444 struct event_filter *filter;
Jiri Olsaced39002012-02-15 15:51:52 +0100445#ifdef CONFIG_FUNCTION_TRACER
446 struct ftrace_ops ftrace_ops;
447#endif
Ingo Molnaree060942008-12-13 09:00:03 +0100448#endif
Li Zefan6fb29152009-10-15 11:21:42 +0800449
Stephane Eraniane5d13672011-02-14 11:20:01 +0200450#ifdef CONFIG_CGROUP_PERF
451 struct perf_cgroup *cgrp; /* cgroup event is attach to */
452 int cgrp_defer_enabled;
453#endif
454
Li Zefan6fb29152009-10-15 11:21:42 +0800455#endif /* CONFIG_PERF_EVENTS */
Thomas Gleixner0793a612008-12-04 20:12:29 +0100456};
457
Peter Zijlstrab04243e2010-09-17 11:28:48 +0200458enum perf_event_context_type {
459 task_context,
460 cpu_context,
461};
462
Thomas Gleixner0793a612008-12-04 20:12:29 +0100463/**
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200464 * struct perf_event_context - event context structure
Thomas Gleixner0793a612008-12-04 20:12:29 +0100465 *
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200466 * Used as a container for task events and CPU events as well:
Thomas Gleixner0793a612008-12-04 20:12:29 +0100467 */
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200468struct perf_event_context {
Peter Zijlstra108b02c2010-09-06 14:32:03 +0200469 struct pmu *pmu;
Richard Kennedyee643c42011-03-07 15:46:59 +0000470 enum perf_event_context_type type;
Thomas Gleixner0793a612008-12-04 20:12:29 +0100471 /*
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200472 * Protect the states of the events in the list,
Paul Mackerrasd859e292009-01-17 18:10:22 +1100473 * nr_active, and the list:
Thomas Gleixner0793a612008-12-04 20:12:29 +0100474 */
Thomas Gleixnere625cce12009-11-17 18:02:06 +0100475 raw_spinlock_t lock;
Paul Mackerrasd859e292009-01-17 18:10:22 +1100476 /*
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200477 * Protect the list of events. Locking either mutex or lock
Paul Mackerrasd859e292009-01-17 18:10:22 +1100478 * is sufficient to ensure the list doesn't change; to change
479 * the list you need to lock both the mutex and the spinlock.
480 */
Ingo Molnara3084442009-06-11 14:44:26 +0200481 struct mutex mutex;
Ingo Molnar04289bb2008-12-11 08:38:42 +0100482
Frederic Weisbecker889ff012010-01-09 20:04:47 +0100483 struct list_head pinned_groups;
484 struct list_head flexible_groups;
Ingo Molnara3084442009-06-11 14:44:26 +0200485 struct list_head event_list;
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200486 int nr_events;
Ingo Molnara3084442009-06-11 14:44:26 +0200487 int nr_active;
488 int is_active;
Peter Zijlstrabfbd3382009-06-24 21:11:59 +0200489 int nr_stat;
Peter Zijlstra0f5a2602011-11-16 14:38:16 +0100490 int nr_freq;
Thomas Gleixnerdddd3372010-11-24 10:05:55 +0100491 int rotate_disable;
Ingo Molnara3084442009-06-11 14:44:26 +0200492 atomic_t refcount;
493 struct task_struct *task;
Paul Mackerras53cfbf52009-03-25 22:46:58 +1100494
495 /*
Peter Zijlstra4af49982009-04-06 11:45:10 +0200496 * Context clock, runs when context enabled.
Paul Mackerras53cfbf52009-03-25 22:46:58 +1100497 */
Ingo Molnara3084442009-06-11 14:44:26 +0200498 u64 time;
499 u64 timestamp;
Paul Mackerras564c2b22009-05-22 14:27:22 +1000500
501 /*
502 * These fields let us detect when two contexts have both
503 * been cloned (inherited) from a common ancestor.
504 */
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200505 struct perf_event_context *parent_ctx;
Ingo Molnara3084442009-06-11 14:44:26 +0200506 u64 parent_gen;
507 u64 generation;
508 int pin_count;
Stephane Eraniand010b332012-02-09 23:21:00 +0100509 int nr_cgroups; /* cgroup evts */
510 int nr_branch_stack; /* branch_stack evt */
Richard Kennedy28009ce2011-06-07 16:33:38 +0100511 struct rcu_head rcu_head;
Jiri Olsafadfe7b2014-08-01 14:33:02 +0200512
513 struct delayed_work orphans_remove;
514 bool orphans_remove_sched;
Thomas Gleixner0793a612008-12-04 20:12:29 +0100515};
516
Frederic Weisbecker7ae07ea2010-08-14 20:45:13 +0200517/*
518 * Number of contexts where an event can trigger:
Ingo Molnare7e7ee22011-05-04 08:42:29 +0200519 * task, softirq, hardirq, nmi.
Frederic Weisbecker7ae07ea2010-08-14 20:45:13 +0200520 */
521#define PERF_NR_CONTEXTS 4
522
Thomas Gleixner0793a612008-12-04 20:12:29 +0100523/**
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200524 * struct perf_event_cpu_context - per cpu event context structure
Thomas Gleixner0793a612008-12-04 20:12:29 +0100525 */
526struct perf_cpu_context {
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200527 struct perf_event_context ctx;
528 struct perf_event_context *task_ctx;
Thomas Gleixner0793a612008-12-04 20:12:29 +0100529 int active_oncpu;
Paul Mackerras3b6f9e52009-01-14 21:00:30 +1100530 int exclusive;
Stephane Eranian9e630202013-04-03 14:21:33 +0200531 struct hrtimer hrtimer;
532 ktime_t hrtimer_interval;
Peter Zijlstrae9d2b062010-09-17 11:28:50 +0200533 struct list_head rotation_list;
Peter Zijlstra3f1f3322012-10-02 15:38:52 +0200534 struct pmu *unique_pmu;
Stephane Eraniane5d13672011-02-14 11:20:01 +0200535 struct perf_cgroup *cgrp;
Thomas Gleixner0793a612008-12-04 20:12:29 +0100536};
537
Markus Metzger5622f292009-09-15 13:00:23 +0200538struct perf_output_handle {
Ingo Molnar57c0c152009-09-21 12:20:38 +0200539 struct perf_event *event;
Frederic Weisbecker76369132011-05-19 19:55:04 +0200540 struct ring_buffer *rb;
Peter Zijlstra6d1acfd2010-05-18 11:12:48 +0200541 unsigned long wakeup;
Peter Zijlstra5d967a82010-05-20 16:46:39 +0200542 unsigned long size;
543 void *addr;
544 int page;
Markus Metzger5622f292009-09-15 13:00:23 +0200545};
546
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200547#ifdef CONFIG_PERF_EVENTS
Robert Richter829b42d2009-04-29 12:46:59 +0200548
Mischa Jonker03d8e802013-06-04 11:45:48 +0200549extern int perf_pmu_register(struct pmu *pmu, const char *name, int type);
Peter Zijlstrab0a873e2010-06-11 13:35:08 +0200550extern void perf_pmu_unregister(struct pmu *pmu);
Ingo Molnar621a01e2008-12-11 12:46:46 +0100551
Matt Fleming3bf101b2010-09-27 20:22:24 +0100552extern int perf_num_counters(void);
Matt Fleming84c79912010-10-03 21:41:13 +0100553extern const char *perf_pmu_name(void);
Jiri Olsaab0cce52012-05-23 13:13:02 +0200554extern void __perf_event_task_sched_in(struct task_struct *prev,
555 struct task_struct *task);
556extern void __perf_event_task_sched_out(struct task_struct *prev,
557 struct task_struct *next);
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200558extern int perf_event_init_task(struct task_struct *child);
559extern void perf_event_exit_task(struct task_struct *child);
560extern void perf_event_free_task(struct task_struct *task);
Peter Zijlstra4e231c72010-09-09 21:01:59 +0200561extern void perf_event_delayed_put(struct task_struct *task);
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200562extern void perf_event_print_debug(void);
Peter Zijlstra33696fc2010-06-14 08:49:00 +0200563extern void perf_pmu_disable(struct pmu *pmu);
564extern void perf_pmu_enable(struct pmu *pmu);
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200565extern int perf_event_task_disable(void);
566extern int perf_event_task_enable(void);
Avi Kivity26ca5c12011-06-29 18:42:37 +0300567extern int perf_event_refresh(struct perf_event *event, int refresh);
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200568extern void perf_event_update_userpage(struct perf_event *event);
Arjan van de Venfb0459d2009-09-25 12:25:56 +0200569extern int perf_event_release_kernel(struct perf_event *event);
570extern struct perf_event *
571perf_event_create_kernel_counter(struct perf_event_attr *attr,
572 int cpu,
Matt Helsley38a81da2010-09-13 13:01:20 -0700573 struct task_struct *task,
Avi Kivity4dc0da82011-06-29 18:42:35 +0300574 perf_overflow_handler_t callback,
575 void *context);
Yan, Zheng0cda4c02012-06-15 14:31:33 +0800576extern void perf_pmu_migrate_context(struct pmu *pmu,
577 int src_cpu, int dst_cpu);
Peter Zijlstra59ed4462009-11-20 22:19:55 +0100578extern u64 perf_event_read_value(struct perf_event *event,
579 u64 *enabled, u64 *running);
Ingo Molnar5c92d122008-12-11 13:21:10 +0100580
Stephane Eraniand010b332012-02-09 23:21:00 +0100581
Peter Zijlstradf1a1322009-06-10 21:02:22 +0200582struct perf_sample_data {
Peter Zijlstra25657112014-09-24 13:48:42 +0200583 /*
584 * Fields set by perf_sample_data_init(), group so as to
585 * minimize the cachelines touched.
586 */
587 u64 addr;
588 struct perf_raw_record *raw;
589 struct perf_branch_stack *br_stack;
590 u64 period;
591 u64 weight;
592 u64 txn;
593 union perf_mem_data_src data_src;
Markus Metzger5622f292009-09-15 13:00:23 +0200594
Peter Zijlstra25657112014-09-24 13:48:42 +0200595 /*
596 * The other fields, optionally {set,used} by
597 * perf_{prepare,output}_sample().
598 */
599 u64 type;
Markus Metzger5622f292009-09-15 13:00:23 +0200600 u64 ip;
601 struct {
602 u32 pid;
603 u32 tid;
604 } tid_entry;
605 u64 time;
Markus Metzger5622f292009-09-15 13:00:23 +0200606 u64 id;
607 u64 stream_id;
608 struct {
609 u32 cpu;
610 u32 reserved;
611 } cpu_entry;
Markus Metzger5622f292009-09-15 13:00:23 +0200612 struct perf_callchain_entry *callchain;
Stephane Eranian60e23642014-09-24 13:48:37 +0200613 struct perf_regs regs_user;
614 struct perf_regs regs_intr;
Jiri Olsac5ebced2012-08-07 15:20:40 +0200615 u64 stack_user_size;
Peter Zijlstra25657112014-09-24 13:48:42 +0200616} ____cacheline_aligned;
Peter Zijlstradf1a1322009-06-10 21:02:22 +0200617
Stephane Eranian770eee12014-08-11 21:27:12 +0200618/* default value for data source */
619#define PERF_MEM_NA (PERF_MEM_S(OP, NA) |\
620 PERF_MEM_S(LVL, NA) |\
621 PERF_MEM_S(SNOOP, NA) |\
622 PERF_MEM_S(LOCK, NA) |\
623 PERF_MEM_S(TLB, NA))
624
Robert Richterfd0d0002012-04-02 20:19:08 +0200625static inline void perf_sample_data_init(struct perf_sample_data *data,
626 u64 addr, u64 period)
Peter Zijlstradc1d6282010-03-03 15:55:04 +0100627{
Robert Richterfd0d0002012-04-02 20:19:08 +0200628 /* remaining struct members initialized in perf_prepare_sample() */
Peter Zijlstradc1d6282010-03-03 15:55:04 +0100629 data->addr = addr;
630 data->raw = NULL;
Stephane Eranianbce38cd2012-02-09 23:20:51 +0100631 data->br_stack = NULL;
Jiri Olsa40189942012-08-07 15:20:37 +0200632 data->period = period;
Andi Kleenc3feedf2013-01-24 16:10:28 +0100633 data->weight = 0;
Stephane Eranian770eee12014-08-11 21:27:12 +0200634 data->data_src.val = PERF_MEM_NA;
Andi Kleenfdfbbd02013-09-20 07:40:39 -0700635 data->txn = 0;
Peter Zijlstradc1d6282010-03-03 15:55:04 +0100636}
637
Markus Metzger5622f292009-09-15 13:00:23 +0200638extern void perf_output_sample(struct perf_output_handle *handle,
639 struct perf_event_header *header,
640 struct perf_sample_data *data,
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200641 struct perf_event *event);
Markus Metzger5622f292009-09-15 13:00:23 +0200642extern void perf_prepare_sample(struct perf_event_header *header,
643 struct perf_sample_data *data,
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200644 struct perf_event *event,
Markus Metzger5622f292009-09-15 13:00:23 +0200645 struct pt_regs *regs);
646
Peter Zijlstraa8b0ca12011-06-27 14:41:57 +0200647extern int perf_event_overflow(struct perf_event *event,
Markus Metzger5622f292009-09-15 13:00:23 +0200648 struct perf_sample_data *data,
649 struct pt_regs *regs);
Peter Zijlstradf1a1322009-06-10 21:02:22 +0200650
Franck Bui-Huu6c7e5502010-11-23 16:21:43 +0100651static inline bool is_sampling_event(struct perf_event *event)
652{
653 return event->attr.sample_period != 0;
654}
655
Paul Mackerras3b6f9e52009-01-14 21:00:30 +1100656/*
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200657 * Return 1 for a software event, 0 for a hardware event
Paul Mackerras3b6f9e52009-01-14 21:00:30 +1100658 */
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200659static inline int is_software_event(struct perf_event *event)
Paul Mackerras3b6f9e52009-01-14 21:00:30 +1100660{
Peter Zijlstra89a1e182010-09-07 17:34:50 +0200661 return event->pmu->task_ctx_nr == perf_sw_context;
Paul Mackerras3b6f9e52009-01-14 21:00:30 +1100662}
663
Ingo Molnarc5905af2012-02-24 08:31:31 +0100664extern struct static_key perf_swevent_enabled[PERF_COUNT_SW_MAX];
Peter Zijlstraf29ac752009-06-19 18:27:26 +0200665
Peter Zijlstraa8b0ca12011-06-27 14:41:57 +0200666extern void __perf_sw_event(u32, u64, struct pt_regs *, u64);
Peter Zijlstraf29ac752009-06-19 18:27:26 +0200667
Frederic Weisbeckerb0f82b82010-05-20 07:47:21 +0200668#ifndef perf_arch_fetch_caller_regs
Ingo Molnare7e7ee22011-05-04 08:42:29 +0200669static inline void perf_arch_fetch_caller_regs(struct pt_regs *regs, unsigned long ip) { }
Frederic Weisbeckerb0f82b82010-05-20 07:47:21 +0200670#endif
Frederic Weisbecker5331d7b2010-03-04 21:15:56 +0100671
672/*
673 * Take a snapshot of the regs. Skip ip and frame pointer to
674 * the nth caller. We only need a few of the regs:
675 * - ip for PERF_SAMPLE_IP
676 * - cs for user_mode() tests
677 * - bp for callchains
678 * - eflags, for future purposes, just in case
679 */
Frederic Weisbeckerb0f82b82010-05-20 07:47:21 +0200680static inline void perf_fetch_caller_regs(struct pt_regs *regs)
Frederic Weisbecker5331d7b2010-03-04 21:15:56 +0100681{
Frederic Weisbecker5331d7b2010-03-04 21:15:56 +0100682 memset(regs, 0, sizeof(*regs));
683
Frederic Weisbeckerb0f82b82010-05-20 07:47:21 +0200684 perf_arch_fetch_caller_regs(regs, CALLER_ADDR0);
Frederic Weisbecker5331d7b2010-03-04 21:15:56 +0100685}
686
Peter Zijlstra7e54a5a2010-10-14 22:32:45 +0200687static __always_inline void
Peter Zijlstraa8b0ca12011-06-27 14:41:57 +0200688perf_sw_event(u32 event_id, u64 nr, struct pt_regs *regs, u64 addr)
Frederic Weisbeckere49a5bd2010-03-22 19:40:03 +0100689{
Peter Zijlstra7e54a5a2010-10-14 22:32:45 +0200690 struct pt_regs hot_regs;
Frederic Weisbeckere49a5bd2010-03-22 19:40:03 +0100691
Ingo Molnarc5905af2012-02-24 08:31:31 +0100692 if (static_key_false(&perf_swevent_enabled[event_id])) {
Jason Barond430d3d2011-03-16 17:29:47 -0400693 if (!regs) {
694 perf_fetch_caller_regs(&hot_regs);
695 regs = &hot_regs;
696 }
Peter Zijlstraa8b0ca12011-06-27 14:41:57 +0200697 __perf_sw_event(event_id, nr, regs, addr);
Frederic Weisbeckere49a5bd2010-03-22 19:40:03 +0100698 }
699}
700
Ingo Molnarc5905af2012-02-24 08:31:31 +0100701extern struct static_key_deferred perf_sched_events;
Peter Zijlstraee6dcfa2010-11-26 13:49:04 +0100702
Jiri Olsaab0cce52012-05-23 13:13:02 +0200703static inline void perf_event_task_sched_in(struct task_struct *prev,
Stephane Eraniana8d757e2011-08-25 15:58:03 +0200704 struct task_struct *task)
Peter Zijlstraee6dcfa2010-11-26 13:49:04 +0100705{
Jiri Olsaab0cce52012-05-23 13:13:02 +0200706 if (static_key_false(&perf_sched_events.key))
707 __perf_event_task_sched_in(prev, task);
708}
709
710static inline void perf_event_task_sched_out(struct task_struct *prev,
711 struct task_struct *next)
712{
Peter Zijlstraa8b0ca12011-06-27 14:41:57 +0200713 perf_sw_event(PERF_COUNT_SW_CONTEXT_SWITCHES, 1, NULL, 0);
Peter Zijlstraee6dcfa2010-11-26 13:49:04 +0100714
Ingo Molnarc5905af2012-02-24 08:31:31 +0100715 if (static_key_false(&perf_sched_events.key))
Jiri Olsaab0cce52012-05-23 13:13:02 +0200716 __perf_event_task_sched_out(prev, next);
Peter Zijlstraee6dcfa2010-11-26 13:49:04 +0100717}
718
Eric B Munson3af9e852010-05-18 15:30:49 +0100719extern void perf_event_mmap(struct vm_area_struct *vma);
Zhang, Yanmin39447b32010-04-19 13:32:41 +0800720extern struct perf_guest_info_callbacks *perf_guest_cbs;
Zhang, Yanmindcf46b92010-04-20 10:13:58 +0800721extern int perf_register_guest_info_callbacks(struct perf_guest_info_callbacks *callbacks);
722extern int perf_unregister_guest_info_callbacks(struct perf_guest_info_callbacks *callbacks);
Zhang, Yanmin39447b32010-04-19 13:32:41 +0800723
Peter Zijlstrae041e322014-05-21 17:32:19 +0200724extern void perf_event_exec(void);
Adrian Hunter82b89772014-05-28 11:45:04 +0300725extern void perf_event_comm(struct task_struct *tsk, bool exec);
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200726extern void perf_event_fork(struct task_struct *tsk);
Peter Zijlstra8d1b2d92009-04-08 15:01:30 +0200727
Frederic Weisbecker56962b42010-06-30 23:03:51 +0200728/* Callchains */
729DECLARE_PER_CPU(struct perf_callchain_entry, perf_callchain_entry);
730
Ingo Molnare7e7ee22011-05-04 08:42:29 +0200731extern void perf_callchain_user(struct perf_callchain_entry *entry, struct pt_regs *regs);
732extern void perf_callchain_kernel(struct perf_callchain_entry *entry, struct pt_regs *regs);
Frederic Weisbecker56962b42010-06-30 23:03:51 +0200733
Ingo Molnare7e7ee22011-05-04 08:42:29 +0200734static inline void perf_callchain_store(struct perf_callchain_entry *entry, u64 ip)
Frederic Weisbecker70791ce2010-06-29 19:34:05 +0200735{
736 if (entry->nr < PERF_MAX_STACK_DEPTH)
737 entry->ip[entry->nr++] = ip;
738}
Peter Zijlstra394ee072009-03-30 19:07:14 +0200739
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200740extern int sysctl_perf_event_paranoid;
741extern int sysctl_perf_event_mlock;
742extern int sysctl_perf_event_sample_rate;
Dave Hansen14c63f12013-06-21 08:51:36 -0700743extern int sysctl_perf_cpu_time_max_percent;
744
745extern void perf_sample_event_took(u64 sample_len_ns);
Peter Zijlstra1ccd1542009-04-09 10:53:45 +0200746
Peter Zijlstra163ec432011-02-16 11:22:34 +0100747extern int perf_proc_update_handler(struct ctl_table *table, int write,
748 void __user *buffer, size_t *lenp,
749 loff_t *ppos);
Dave Hansen14c63f12013-06-21 08:51:36 -0700750extern int perf_cpu_time_max_percent_handler(struct ctl_table *table, int write,
751 void __user *buffer, size_t *lenp,
752 loff_t *ppos);
753
Peter Zijlstra163ec432011-02-16 11:22:34 +0100754
Peter Zijlstra320ebf02010-03-02 12:35:37 +0100755static inline bool perf_paranoid_tracepoint_raw(void)
756{
757 return sysctl_perf_event_paranoid > -1;
758}
759
760static inline bool perf_paranoid_cpu(void)
761{
762 return sysctl_perf_event_paranoid > 0;
763}
764
765static inline bool perf_paranoid_kernel(void)
766{
767 return sysctl_perf_event_paranoid > 1;
768}
769
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200770extern void perf_event_init(void);
Peter Zijlstra1c024eca2010-05-19 14:02:22 +0200771extern void perf_tp_event(u64 addr, u64 count, void *record,
772 int entry_size, struct pt_regs *regs,
Andrew Vagine6dab5f2012-07-11 18:14:58 +0400773 struct hlist_head *head, int rctx,
774 struct task_struct *task);
Frederic Weisbecker24f1e32c2009-09-09 19:22:48 +0200775extern void perf_bp_event(struct perf_event *event, void *data);
Ingo Molnar0d905bc2009-05-04 19:13:30 +0200776
Paul Mackerras9d23a902009-05-14 21:48:08 +1000777#ifndef perf_misc_flags
Ingo Molnare7e7ee22011-05-04 08:42:29 +0200778# define perf_misc_flags(regs) \
779 (user_mode(regs) ? PERF_RECORD_MISC_USER : PERF_RECORD_MISC_KERNEL)
780# define perf_instruction_pointer(regs) instruction_pointer(regs)
Paul Mackerras9d23a902009-05-14 21:48:08 +1000781#endif
782
Stephane Eranianbce38cd2012-02-09 23:20:51 +0100783static inline bool has_branch_stack(struct perf_event *event)
784{
785 return event->attr.sample_type & PERF_SAMPLE_BRANCH_STACK;
786}
787
Markus Metzger5622f292009-09-15 13:00:23 +0200788extern int perf_output_begin(struct perf_output_handle *handle,
Peter Zijlstraa7ac67e2011-06-27 16:47:16 +0200789 struct perf_event *event, unsigned int size);
Markus Metzger5622f292009-09-15 13:00:23 +0200790extern void perf_output_end(struct perf_output_handle *handle);
Frederic Weisbecker91d77532012-08-07 15:20:38 +0200791extern unsigned int perf_output_copy(struct perf_output_handle *handle,
Markus Metzger5622f292009-09-15 13:00:23 +0200792 const void *buf, unsigned int len);
Jiri Olsa5685e0f2012-08-07 15:20:39 +0200793extern unsigned int perf_output_skip(struct perf_output_handle *handle,
794 unsigned int len);
Peter Zijlstra4ed7c922009-11-23 11:37:29 +0100795extern int perf_swevent_get_recursion_context(void);
796extern void perf_swevent_put_recursion_context(int rctx);
Jiri Olsaab573842013-05-01 17:25:44 +0200797extern u64 perf_swevent_set_period(struct perf_event *event);
Frederic Weisbecker44234ad2009-12-09 09:25:48 +0100798extern void perf_event_enable(struct perf_event *event);
799extern void perf_event_disable(struct perf_event *event);
K.Prasad500ad2d2012-08-02 13:46:35 +0530800extern int __perf_event_disable(void *info);
Peter Zijlstrae9d2b062010-09-17 11:28:50 +0200801extern void perf_event_task_tick(void);
Peter Zijlstrae041e322014-05-21 17:32:19 +0200802#else /* !CONFIG_PERF_EVENTS: */
Thomas Gleixner0793a612008-12-04 20:12:29 +0100803static inline void
Jiri Olsaab0cce52012-05-23 13:13:02 +0200804perf_event_task_sched_in(struct task_struct *prev,
805 struct task_struct *task) { }
806static inline void
807perf_event_task_sched_out(struct task_struct *prev,
808 struct task_struct *next) { }
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200809static inline int perf_event_init_task(struct task_struct *child) { return 0; }
810static inline void perf_event_exit_task(struct task_struct *child) { }
811static inline void perf_event_free_task(struct task_struct *task) { }
Peter Zijlstra4e231c72010-09-09 21:01:59 +0200812static inline void perf_event_delayed_put(struct task_struct *task) { }
Ingo Molnar57c0c152009-09-21 12:20:38 +0200813static inline void perf_event_print_debug(void) { }
Ingo Molnar57c0c152009-09-21 12:20:38 +0200814static inline int perf_event_task_disable(void) { return -EINVAL; }
815static inline int perf_event_task_enable(void) { return -EINVAL; }
Avi Kivity26ca5c12011-06-29 18:42:37 +0300816static inline int perf_event_refresh(struct perf_event *event, int refresh)
817{
818 return -EINVAL;
819}
Peter Zijlstra15dbf272009-03-13 12:21:32 +0100820
Peter Zijlstra925d5192009-03-30 19:07:02 +0200821static inline void
Peter Zijlstraa8b0ca12011-06-27 14:41:57 +0200822perf_sw_event(u32 event_id, u64 nr, struct pt_regs *regs, u64 addr) { }
Frederic Weisbecker24f1e32c2009-09-09 19:22:48 +0200823static inline void
Ingo Molnar184f4122010-01-27 08:39:39 +0100824perf_bp_event(struct perf_event *event, void *data) { }
Peter Zijlstra0a4a9392009-03-30 19:07:05 +0200825
Zhang, Yanmin39447b32010-04-19 13:32:41 +0800826static inline int perf_register_guest_info_callbacks
Ingo Molnare7e7ee22011-05-04 08:42:29 +0200827(struct perf_guest_info_callbacks *callbacks) { return 0; }
Zhang, Yanmin39447b32010-04-19 13:32:41 +0800828static inline int perf_unregister_guest_info_callbacks
Ingo Molnare7e7ee22011-05-04 08:42:29 +0200829(struct perf_guest_info_callbacks *callbacks) { return 0; }
Zhang, Yanmin39447b32010-04-19 13:32:41 +0800830
Ingo Molnar57c0c152009-09-21 12:20:38 +0200831static inline void perf_event_mmap(struct vm_area_struct *vma) { }
Peter Zijlstrae041e322014-05-21 17:32:19 +0200832static inline void perf_event_exec(void) { }
Adrian Hunter82b89772014-05-28 11:45:04 +0300833static inline void perf_event_comm(struct task_struct *tsk, bool exec) { }
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200834static inline void perf_event_fork(struct task_struct *tsk) { }
835static inline void perf_event_init(void) { }
Ingo Molnar184f4122010-01-27 08:39:39 +0100836static inline int perf_swevent_get_recursion_context(void) { return -1; }
Peter Zijlstra4ed7c922009-11-23 11:37:29 +0100837static inline void perf_swevent_put_recursion_context(int rctx) { }
Jiri Olsaab573842013-05-01 17:25:44 +0200838static inline u64 perf_swevent_set_period(struct perf_event *event) { return 0; }
Frederic Weisbecker44234ad2009-12-09 09:25:48 +0100839static inline void perf_event_enable(struct perf_event *event) { }
840static inline void perf_event_disable(struct perf_event *event) { }
K.Prasad500ad2d2012-08-02 13:46:35 +0530841static inline int __perf_event_disable(void *info) { return -1; }
Peter Zijlstrae9d2b062010-09-17 11:28:50 +0200842static inline void perf_event_task_tick(void) { }
Thomas Gleixner0793a612008-12-04 20:12:29 +0100843#endif
844
Frederic Weisbecker026249e2013-04-20 15:58:34 +0200845#if defined(CONFIG_PERF_EVENTS) && defined(CONFIG_NO_HZ_FULL)
846extern bool perf_event_can_stop_tick(void);
847#else
848static inline bool perf_event_can_stop_tick(void) { return true; }
849#endif
850
David Rientjes6c4d3bc2013-03-17 15:49:10 -0700851#if defined(CONFIG_PERF_EVENTS) && defined(CONFIG_CPU_SUP_INTEL)
852extern void perf_restore_debug_store(void);
853#else
Stephane Eranian1d9d8632013-03-15 14:26:07 +0100854static inline void perf_restore_debug_store(void) { }
Thomas Gleixner0793a612008-12-04 20:12:29 +0100855#endif
856
Ingo Molnare7e7ee22011-05-04 08:42:29 +0200857#define perf_output_put(handle, x) perf_output_copy((handle), &(x), sizeof(x))
Markus Metzger5622f292009-09-15 13:00:23 +0200858
Peter Zijlstra3f6da392010-03-05 13:01:18 +0100859/*
Viresh Kumar0a0fca92013-06-04 13:10:24 +0530860 * This has to have a higher priority than migration_notifier in sched/core.c.
Peter Zijlstra3f6da392010-03-05 13:01:18 +0100861 */
Ingo Molnare7e7ee22011-05-04 08:42:29 +0200862#define perf_cpu_notifier(fn) \
863do { \
Paul Gortmaker0db06282013-06-19 14:53:51 -0400864 static struct notifier_block fn##_nb = \
Ingo Molnare7e7ee22011-05-04 08:42:29 +0200865 { .notifier_call = fn, .priority = CPU_PRI_PERF }; \
Srivatsa S. Bhatc13d38e2012-10-16 13:28:17 +0530866 unsigned long cpu = smp_processor_id(); \
Srivatsa S. Bhat6760bca2012-10-16 13:28:10 +0530867 unsigned long flags; \
Srivatsa S. Bhatf0bdb5e2014-03-11 02:04:39 +0530868 \
869 cpu_notifier_register_begin(); \
Ingo Molnare7e7ee22011-05-04 08:42:29 +0200870 fn(&fn##_nb, (unsigned long)CPU_UP_PREPARE, \
Srivatsa S. Bhatc13d38e2012-10-16 13:28:17 +0530871 (void *)(unsigned long)cpu); \
Srivatsa S. Bhat6760bca2012-10-16 13:28:10 +0530872 local_irq_save(flags); \
Ingo Molnare7e7ee22011-05-04 08:42:29 +0200873 fn(&fn##_nb, (unsigned long)CPU_STARTING, \
Srivatsa S. Bhatc13d38e2012-10-16 13:28:17 +0530874 (void *)(unsigned long)cpu); \
Srivatsa S. Bhat6760bca2012-10-16 13:28:10 +0530875 local_irq_restore(flags); \
Ingo Molnare7e7ee22011-05-04 08:42:29 +0200876 fn(&fn##_nb, (unsigned long)CPU_ONLINE, \
Srivatsa S. Bhatc13d38e2012-10-16 13:28:17 +0530877 (void *)(unsigned long)cpu); \
Srivatsa S. Bhatf0bdb5e2014-03-11 02:04:39 +0530878 __register_cpu_notifier(&fn##_nb); \
879 cpu_notifier_register_done(); \
Peter Zijlstra3f6da392010-03-05 13:01:18 +0100880} while (0)
881
Srivatsa S. Bhatf0bdb5e2014-03-11 02:04:39 +0530882/*
883 * Bare-bones version of perf_cpu_notifier(), which doesn't invoke the
884 * callback for already online CPUs.
885 */
886#define __perf_cpu_notifier(fn) \
887do { \
888 static struct notifier_block fn##_nb = \
889 { .notifier_call = fn, .priority = CPU_PRI_PERF }; \
890 \
891 __register_cpu_notifier(&fn##_nb); \
892} while (0)
Jiri Olsa641cc932012-03-15 20:09:14 +0100893
Sukadev Bhattiprolu26639602013-01-22 22:24:23 -0800894struct perf_pmu_events_attr {
895 struct device_attribute attr;
896 u64 id;
Stephane Eranian3a54aaa2013-01-24 16:10:26 +0100897 const char *event_str;
Sukadev Bhattiprolu26639602013-01-22 22:24:23 -0800898};
899
900#define PMU_EVENT_ATTR(_name, _var, _id, _show) \
901static struct perf_pmu_events_attr _var = { \
902 .attr = __ATTR(_name, 0444, _show, NULL), \
903 .id = _id, \
904};
905
Jiri Olsa641cc932012-03-15 20:09:14 +0100906#define PMU_FORMAT_ATTR(_name, _format) \
907static ssize_t \
908_name##_show(struct device *dev, \
909 struct device_attribute *attr, \
910 char *page) \
911{ \
912 BUILD_BUG_ON(sizeof(_format) >= PAGE_SIZE); \
913 return sprintf(page, _format "\n"); \
914} \
915 \
916static struct device_attribute format_attr_##_name = __ATTR_RO(_name)
917
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200918#endif /* _LINUX_PERF_EVENT_H */