blob: 2b621982938d4653436b3d5cfe7578c1c71e6bb9 [file] [log] [blame]
Thomas Gleixner0793a612008-12-04 20:12:29 +01001/*
Ingo Molnar57c0c152009-09-21 12:20:38 +02002 * Performance events:
Thomas Gleixner0793a612008-12-04 20:12:29 +01003 *
Ingo Molnara3084442009-06-11 14:44:26 +02004 * Copyright (C) 2008-2009, Thomas Gleixner <tglx@linutronix.de>
Ingo Molnare7e7ee22011-05-04 08:42:29 +02005 * Copyright (C) 2008-2011, Red Hat, Inc., Ingo Molnar
6 * Copyright (C) 2008-2011, Red Hat, Inc., Peter Zijlstra
Thomas Gleixner0793a612008-12-04 20:12:29 +01007 *
Ingo Molnar57c0c152009-09-21 12:20:38 +02008 * Data type definitions, declarations, prototypes.
Thomas Gleixner0793a612008-12-04 20:12:29 +01009 *
Ingo Molnara3084442009-06-11 14:44:26 +020010 * Started by: Thomas Gleixner and Ingo Molnar
Thomas Gleixner0793a612008-12-04 20:12:29 +010011 *
Ingo Molnar57c0c152009-09-21 12:20:38 +020012 * For licencing details see kernel-base/COPYING
Thomas Gleixner0793a612008-12-04 20:12:29 +010013 */
Ingo Molnarcdd6c482009-09-21 12:02:48 +020014#ifndef _LINUX_PERF_EVENT_H
15#define _LINUX_PERF_EVENT_H
Thomas Gleixner0793a612008-12-04 20:12:29 +010016
David Howells607ca462012-10-13 10:46:48 +010017#include <uapi/linux/perf_event.h>
Peter Zijlstraa4be7c22009-08-19 11:18:27 +020018
Paul Mackerrasd859e292009-01-17 18:10:22 +110019/*
Paul Mackerrasf3dfd262009-02-26 22:43:46 +110020 * Kernel-internal data types and definitions:
Ingo Molnar9f66a382008-12-10 12:33:23 +010021 */
22
Ingo Molnarcdd6c482009-09-21 12:02:48 +020023#ifdef CONFIG_PERF_EVENTS
24# include <asm/perf_event.h>
Peter Zijlstra7be79232010-06-09 11:57:23 +020025# include <asm/local64.h>
Paul Mackerrasf3dfd262009-02-26 22:43:46 +110026#endif
27
Zhang, Yanmin39447b32010-04-19 13:32:41 +080028struct perf_guest_info_callbacks {
Ingo Molnare7e7ee22011-05-04 08:42:29 +020029 int (*is_in_guest)(void);
30 int (*is_user_mode)(void);
31 unsigned long (*get_guest_ip)(void);
Zhang, Yanmin39447b32010-04-19 13:32:41 +080032};
33
Arnd Bergmann2ff6cfd2009-12-07 17:12:58 +010034#ifdef CONFIG_HAVE_HW_BREAKPOINT
35#include <asm/hw_breakpoint.h>
36#endif
37
Paul Mackerrasf3dfd262009-02-26 22:43:46 +110038#include <linux/list.h>
39#include <linux/mutex.h>
40#include <linux/rculist.h>
41#include <linux/rcupdate.h>
42#include <linux/spinlock.h>
Peter Zijlstrad6d020e2009-03-13 12:21:35 +010043#include <linux/hrtimer.h>
Peter Zijlstra3c446b3d2009-04-06 11:45:01 +020044#include <linux/fs.h>
Peter Zijlstra709e50c2009-06-02 14:13:15 +020045#include <linux/pid_namespace.h>
Peter Zijlstra906010b2009-09-21 16:08:49 +020046#include <linux/workqueue.h>
Frederic Weisbecker5331d7b2010-03-04 21:15:56 +010047#include <linux/ftrace.h>
Peter Zijlstra85cfabb2010-03-11 13:06:56 +010048#include <linux/cpu.h>
Peter Zijlstrae360adb2010-10-14 14:01:34 +080049#include <linux/irq_work.h>
Ingo Molnarc5905af2012-02-24 08:31:31 +010050#include <linux/static_key.h>
Andrew Jones851cf6e2013-08-09 19:51:57 +053051#include <linux/jump_label_ratelimit.h>
Arun Sharma600634972011-07-26 16:09:06 -070052#include <linux/atomic.h>
Jiri Olsa641cc932012-03-15 20:09:14 +010053#include <linux/sysfs.h>
Jiri Olsa40189942012-08-07 15:20:37 +020054#include <linux/perf_regs.h>
Jiri Olsafadfe7b2014-08-01 14:33:02 +020055#include <linux/workqueue.h>
Peter Zijlstrafa588152010-05-18 10:54:20 +020056#include <asm/local.h>
Paul Mackerrasf3dfd262009-02-26 22:43:46 +110057
Peter Zijlstraf9188e02009-06-18 22:20:52 +020058struct perf_callchain_entry {
59 __u64 nr;
60 __u64 ip[PERF_MAX_STACK_DEPTH];
61};
62
Frederic Weisbecker3a43ce62009-08-08 04:26:37 +020063struct perf_raw_record {
64 u32 size;
65 void *data;
Frederic Weisbeckerf413cdb2009-08-07 01:25:54 +020066};
67
Stephane Eranianbce38cd2012-02-09 23:20:51 +010068/*
Stephane Eranianbce38cd2012-02-09 23:20:51 +010069 * branch stack layout:
70 * nr: number of taken branches stored in entries[]
71 *
72 * Note that nr can vary from sample to sample
73 * branches (to, from) are stored from most recent
74 * to least recent, i.e., entries[0] contains the most
75 * recent branch.
76 */
Peter Zijlstracaff2be2010-03-03 12:02:30 +010077struct perf_branch_stack {
78 __u64 nr;
79 struct perf_branch_entry entries[0];
80};
81
Paul Mackerrasf3dfd262009-02-26 22:43:46 +110082struct task_struct;
83
Stephane Eranianefc9f052011-06-06 16:57:03 +020084/*
85 * extra PMU register associated with an event
86 */
87struct hw_perf_event_extra {
88 u64 config; /* register value */
89 unsigned int reg; /* register address or index */
90 int alloc; /* extra register already allocated */
91 int idx; /* index in shared_regs->regs[] */
92};
93
Andrew Hunter43b457802013-05-23 11:07:03 -070094struct event_constraint;
95
Thomas Gleixner0793a612008-12-04 20:12:29 +010096/**
Ingo Molnarcdd6c482009-09-21 12:02:48 +020097 * struct hw_perf_event - performance event hardware details:
Thomas Gleixner0793a612008-12-04 20:12:29 +010098 */
Ingo Molnarcdd6c482009-09-21 12:02:48 +020099struct hw_perf_event {
100#ifdef CONFIG_PERF_EVENTS
Peter Zijlstrad6d020e2009-03-13 12:21:35 +0100101 union {
102 struct { /* hardware */
Ingo Molnara3084442009-06-11 14:44:26 +0200103 u64 config;
Stephane Eranian447a1942010-02-01 14:50:01 +0200104 u64 last_tag;
Ingo Molnara3084442009-06-11 14:44:26 +0200105 unsigned long config_base;
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200106 unsigned long event_base;
Vince Weaverc48b6052012-03-01 17:28:14 -0500107 int event_base_rdpmc;
Ingo Molnara3084442009-06-11 14:44:26 +0200108 int idx;
Stephane Eranian447a1942010-02-01 14:50:01 +0200109 int last_cpu;
Stephane Eranian9fac2cf2013-01-24 16:10:27 +0100110 int flags;
Stephane Eranianbce38cd2012-02-09 23:20:51 +0100111
Stephane Eranianefc9f052011-06-06 16:57:03 +0200112 struct hw_perf_event_extra extra_reg;
Stephane Eranianbce38cd2012-02-09 23:20:51 +0100113 struct hw_perf_event_extra branch_reg;
Andrew Hunter43b457802013-05-23 11:07:03 -0700114
115 struct event_constraint *constraint;
Peter Zijlstrad6d020e2009-03-13 12:21:35 +0100116 };
Soeren Sandmann721a6692009-09-15 14:33:08 +0200117 struct { /* software */
Ingo Molnara3084442009-06-11 14:44:26 +0200118 struct hrtimer hrtimer;
Peter Zijlstrad6d020e2009-03-13 12:21:35 +0100119 };
Oleg Nesterovf22c1bb2013-02-02 16:27:52 +0100120 struct { /* tracepoint */
121 struct task_struct *tp_target;
122 /* for tp_event->class */
123 struct list_head tp_list;
124 };
Frederic Weisbecker24f1e32c2009-09-09 19:22:48 +0200125#ifdef CONFIG_HAVE_HW_BREAKPOINT
Frederic Weisbecker45a73372010-06-23 23:00:37 +0200126 struct { /* breakpoint */
Peter Zijlstrad580ff82010-10-14 17:43:23 +0200127 /*
128 * Crufty hack to avoid the chicken and egg
129 * problem hw_breakpoint has with context
130 * creation and event initalization.
131 */
132 struct task_struct *bp_target;
Oleg Nesterovf22c1bb2013-02-02 16:27:52 +0100133 struct arch_hw_breakpoint info;
134 struct list_head bp_list;
Frederic Weisbecker45a73372010-06-23 23:00:37 +0200135 };
Frederic Weisbecker24f1e32c2009-09-09 19:22:48 +0200136#endif
Peter Zijlstrad6d020e2009-03-13 12:21:35 +0100137 };
Peter Zijlstraa4eaf7f2010-06-16 14:37:10 +0200138 int state;
Peter Zijlstrae7850592010-05-21 14:43:08 +0200139 local64_t prev_count;
Peter Zijlstrab23f3322009-06-02 15:13:03 +0200140 u64 sample_period;
Peter Zijlstra9e350de2009-06-10 21:34:59 +0200141 u64 last_period;
Peter Zijlstrae7850592010-05-21 14:43:08 +0200142 local64_t period_left;
Stephane Eraniane050e3f2012-01-26 17:03:19 +0100143 u64 interrupts_seq;
Peter Zijlstra60db5e02009-05-15 15:19:28 +0200144 u64 interrupts;
Peter Zijlstra6a24ed6c2009-06-05 18:01:29 +0200145
Peter Zijlstraabd50712010-01-26 18:50:16 +0100146 u64 freq_time_stamp;
147 u64 freq_count_stamp;
Ingo Molnaree060942008-12-13 09:00:03 +0100148#endif
Thomas Gleixner0793a612008-12-04 20:12:29 +0100149};
150
Peter Zijlstraa4eaf7f2010-06-16 14:37:10 +0200151/*
152 * hw_perf_event::state flags
153 */
154#define PERF_HES_STOPPED 0x01 /* the counter is stopped */
155#define PERF_HES_UPTODATE 0x02 /* event->count up-to-date */
156#define PERF_HES_ARCH 0x04
157
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200158struct perf_event;
Ingo Molnar621a01e2008-12-11 12:46:46 +0100159
Peter Zijlstra8d2cacb2010-05-25 17:49:05 +0200160/*
161 * Common implementation detail of pmu::{start,commit,cancel}_txn
162 */
163#define PERF_EVENT_TXN 0x1
Lin Ming6bde9b62010-04-23 13:56:00 +0800164
Ingo Molnar621a01e2008-12-11 12:46:46 +0100165/**
Vince Weaver53b25332014-05-16 17:12:12 -0400166 * pmu::capabilities flags
167 */
168#define PERF_PMU_CAP_NO_INTERRUPT 0x01
169
170/**
Robert Richter4aeb0b42009-04-29 12:47:03 +0200171 * struct pmu - generic performance monitoring unit
Ingo Molnar621a01e2008-12-11 12:46:46 +0100172 */
Robert Richter4aeb0b42009-04-29 12:47:03 +0200173struct pmu {
Peter Zijlstrab0a873e2010-06-11 13:35:08 +0200174 struct list_head entry;
175
Yan, Zhengc464c762014-03-18 16:56:41 +0800176 struct module *module;
Peter Zijlstraabe43402010-11-17 23:17:37 +0100177 struct device *dev;
Peter Zijlstra0c9d42e2011-11-20 23:30:47 +0100178 const struct attribute_group **attr_groups;
Mischa Jonker03d8e802013-06-04 11:45:48 +0200179 const char *name;
Peter Zijlstra2e80a822010-11-17 23:17:36 +0100180 int type;
181
Vince Weaver53b25332014-05-16 17:12:12 -0400182 /*
183 * various common per-pmu feature flags
184 */
185 int capabilities;
186
Peter Zijlstra108b02c2010-09-06 14:32:03 +0200187 int * __percpu pmu_disable_count;
188 struct perf_cpu_context * __percpu pmu_cpu_context;
Peter Zijlstra8dc85d52010-09-02 16:50:03 +0200189 int task_ctx_nr;
Stephane Eranian62b85632013-04-03 14:21:34 +0200190 int hrtimer_interval_ms;
Peter Zijlstra33696fc2010-06-14 08:49:00 +0200191
Peter Zijlstraa4eaf7f2010-06-16 14:37:10 +0200192 /*
193 * Fully disable/enable this PMU, can be used to protect from the PMI
194 * as well as for lazy/batch writing of the MSRs.
195 */
Peter Zijlstraad5133b2010-06-15 12:22:39 +0200196 void (*pmu_enable) (struct pmu *pmu); /* optional */
197 void (*pmu_disable) (struct pmu *pmu); /* optional */
Peter Zijlstra33696fc2010-06-14 08:49:00 +0200198
Peter Zijlstrab0a873e2010-06-11 13:35:08 +0200199 /*
Peter Zijlstraa4eaf7f2010-06-16 14:37:10 +0200200 * Try and initialize the event for this PMU.
Peter Zijlstra24cd7f52010-06-11 17:32:03 +0200201 * Should return -ENOENT when the @event doesn't match this PMU.
Peter Zijlstrab0a873e2010-06-11 13:35:08 +0200202 */
203 int (*event_init) (struct perf_event *event);
204
Andy Lutomirski1e0fb9e2014-10-24 15:58:10 -0700205 /*
206 * Notification that the event was mapped or unmapped. Called
207 * in the context of the mapping task.
208 */
209 void (*event_mapped) (struct perf_event *event); /*optional*/
210 void (*event_unmapped) (struct perf_event *event); /*optional*/
211
Peter Zijlstraa4eaf7f2010-06-16 14:37:10 +0200212#define PERF_EF_START 0x01 /* start the counter when adding */
213#define PERF_EF_RELOAD 0x02 /* reload the counter when starting */
214#define PERF_EF_UPDATE 0x04 /* update the counter when stopping */
215
216 /*
217 * Adds/Removes a counter to/from the PMU, can be done inside
218 * a transaction, see the ->*_txn() methods.
219 */
220 int (*add) (struct perf_event *event, int flags);
221 void (*del) (struct perf_event *event, int flags);
222
223 /*
224 * Starts/Stops a counter present on the PMU. The PMI handler
225 * should stop the counter when perf_event_overflow() returns
226 * !0. ->start() will be used to continue.
227 */
228 void (*start) (struct perf_event *event, int flags);
229 void (*stop) (struct perf_event *event, int flags);
230
231 /*
232 * Updates the counter value of the event.
233 */
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200234 void (*read) (struct perf_event *event);
Lin Ming6bde9b62010-04-23 13:56:00 +0800235
236 /*
Peter Zijlstra24cd7f52010-06-11 17:32:03 +0200237 * Group events scheduling is treated as a transaction, add
238 * group events as a whole and perform one schedulability test.
239 * If the test fails, roll back the whole group
Peter Zijlstraa4eaf7f2010-06-16 14:37:10 +0200240 *
241 * Start the transaction, after this ->add() doesn't need to
Peter Zijlstra24cd7f52010-06-11 17:32:03 +0200242 * do schedulability tests.
Lin Ming6bde9b62010-04-23 13:56:00 +0800243 */
Ingo Molnare7e7ee22011-05-04 08:42:29 +0200244 void (*start_txn) (struct pmu *pmu); /* optional */
Peter Zijlstra8d2cacb2010-05-25 17:49:05 +0200245 /*
Peter Zijlstraa4eaf7f2010-06-16 14:37:10 +0200246 * If ->start_txn() disabled the ->add() schedulability test
Peter Zijlstra8d2cacb2010-05-25 17:49:05 +0200247 * then ->commit_txn() is required to perform one. On success
248 * the transaction is closed. On error the transaction is kept
249 * open until ->cancel_txn() is called.
250 */
Ingo Molnare7e7ee22011-05-04 08:42:29 +0200251 int (*commit_txn) (struct pmu *pmu); /* optional */
Peter Zijlstra8d2cacb2010-05-25 17:49:05 +0200252 /*
Peter Zijlstraa4eaf7f2010-06-16 14:37:10 +0200253 * Will cancel the transaction, assumes ->del() is called
Lucas De Marchi25985ed2011-03-30 22:57:33 -0300254 * for each successful ->add() during the transaction.
Peter Zijlstra8d2cacb2010-05-25 17:49:05 +0200255 */
Ingo Molnare7e7ee22011-05-04 08:42:29 +0200256 void (*cancel_txn) (struct pmu *pmu); /* optional */
Peter Zijlstra35edc2a2011-11-20 20:36:02 +0100257
258 /*
259 * Will return the value for perf_event_mmap_page::index for this event,
260 * if no implementation is provided it will default to: event->hw.idx + 1.
261 */
262 int (*event_idx) (struct perf_event *event); /*optional */
Stephane Eraniand010b332012-02-09 23:21:00 +0100263
264 /*
265 * flush branch stack on context-switches (needed in cpu-wide mode)
266 */
267 void (*flush_branch_stack) (void);
Ingo Molnar621a01e2008-12-11 12:46:46 +0100268};
269
Thomas Gleixner0793a612008-12-04 20:12:29 +0100270/**
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200271 * enum perf_event_active_state - the states of a event
Ingo Molnar6a930702008-12-11 15:17:03 +0100272 */
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200273enum perf_event_active_state {
Jiri Olsa179033b2014-08-07 11:48:26 -0400274 PERF_EVENT_STATE_EXIT = -3,
Ingo Molnar57c0c152009-09-21 12:20:38 +0200275 PERF_EVENT_STATE_ERROR = -2,
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200276 PERF_EVENT_STATE_OFF = -1,
277 PERF_EVENT_STATE_INACTIVE = 0,
Ingo Molnar57c0c152009-09-21 12:20:38 +0200278 PERF_EVENT_STATE_ACTIVE = 1,
Ingo Molnar6a930702008-12-11 15:17:03 +0100279};
280
Ingo Molnar9b51f662008-12-12 13:49:45 +0100281struct file;
Peter Zijlstra453f19e2009-11-20 22:19:43 +0100282struct perf_sample_data;
283
Peter Zijlstraa8b0ca12011-06-27 14:41:57 +0200284typedef void (*perf_overflow_handler_t)(struct perf_event *,
Frederic Weisbeckerb326e952009-12-05 09:44:31 +0100285 struct perf_sample_data *,
286 struct pt_regs *regs);
287
Frederic Weisbeckerd6f962b2010-01-10 01:25:51 +0100288enum perf_group_flag {
Ingo Molnare7e7ee22011-05-04 08:42:29 +0200289 PERF_GROUP_SOFTWARE = 0x1,
Frederic Weisbeckerd6f962b2010-01-10 01:25:51 +0100290};
291
Ingo Molnare7e7ee22011-05-04 08:42:29 +0200292#define SWEVENT_HLIST_BITS 8
293#define SWEVENT_HLIST_SIZE (1 << SWEVENT_HLIST_BITS)
Frederic Weisbecker76e1d902010-04-05 15:35:57 +0200294
295struct swevent_hlist {
Ingo Molnare7e7ee22011-05-04 08:42:29 +0200296 struct hlist_head heads[SWEVENT_HLIST_SIZE];
297 struct rcu_head rcu_head;
Frederic Weisbecker76e1d902010-04-05 15:35:57 +0200298};
299
Peter Zijlstra8a495422010-05-27 15:47:49 +0200300#define PERF_ATTACH_CONTEXT 0x01
301#define PERF_ATTACH_GROUP 0x02
Peter Zijlstrad580ff82010-10-14 17:43:23 +0200302#define PERF_ATTACH_TASK 0x04
Peter Zijlstra8a495422010-05-27 15:47:49 +0200303
Li Zefan877c6852013-03-05 11:38:08 +0800304struct perf_cgroup;
Frederic Weisbecker76369132011-05-19 19:55:04 +0200305struct ring_buffer;
306
Ingo Molnar6a930702008-12-11 15:17:03 +0100307/**
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200308 * struct perf_event - performance event kernel representation:
Thomas Gleixner0793a612008-12-04 20:12:29 +0100309 */
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200310struct perf_event {
311#ifdef CONFIG_PERF_EVENTS
Peter Zijlstra98861672013-10-03 16:02:23 +0200312 /*
313 * entry onto perf_event_context::event_list;
314 * modifications require ctx->lock
315 * RCU safe iterations.
316 */
Peter Zijlstra592903c2009-03-13 12:21:36 +0100317 struct list_head event_entry;
Peter Zijlstra98861672013-10-03 16:02:23 +0200318
319 /*
320 * XXX: group_entry and sibling_list should be mutually exclusive;
321 * either you're a sibling on a group, or you're the group leader.
322 * Rework the code to always use the same list element.
323 *
324 * Locked for modification by both ctx->mutex and ctx->lock; holding
325 * either sufficies for read.
326 */
327 struct list_head group_entry;
Ingo Molnar04289bb2008-12-11 08:38:42 +0100328 struct list_head sibling_list;
Peter Zijlstra98861672013-10-03 16:02:23 +0200329
330 /*
331 * We need storage to track the entries in perf_pmu_migrate_context; we
332 * cannot use the event_entry because of RCU and we want to keep the
333 * group in tact which avoids us using the other two entries.
334 */
335 struct list_head migrate_entry;
336
Stephane Eranianf3ae75d2014-01-08 11:15:52 +0100337 struct hlist_node hlist_entry;
338 struct list_head active_entry;
Ingo Molnar0127c3e2009-05-25 22:03:26 +0200339 int nr_siblings;
Frederic Weisbeckerd6f962b2010-01-10 01:25:51 +0100340 int group_flags;
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200341 struct perf_event *group_leader;
Peter Zijlstraa4eaf7f2010-06-16 14:37:10 +0200342 struct pmu *pmu;
Ingo Molnar04289bb2008-12-11 08:38:42 +0100343
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200344 enum perf_event_active_state state;
Peter Zijlstra8a495422010-05-27 15:47:49 +0200345 unsigned int attach_state;
Peter Zijlstrae7850592010-05-21 14:43:08 +0200346 local64_t count;
Peter Zijlstraa6e6dea2010-05-21 14:27:58 +0200347 atomic64_t child_count;
Ingo Molnaree060942008-12-13 09:00:03 +0100348
Paul Mackerras53cfbf52009-03-25 22:46:58 +1100349 /*
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200350 * These are the total time in nanoseconds that the event
Paul Mackerras53cfbf52009-03-25 22:46:58 +1100351 * has been enabled (i.e. eligible to run, and the task has
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200352 * been scheduled in, if this is a per-task event)
Paul Mackerras53cfbf52009-03-25 22:46:58 +1100353 * and running (scheduled onto the CPU), respectively.
354 *
355 * They are computed from tstamp_enabled, tstamp_running and
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200356 * tstamp_stopped when the event is in INACTIVE or ACTIVE state.
Paul Mackerras53cfbf52009-03-25 22:46:58 +1100357 */
358 u64 total_time_enabled;
359 u64 total_time_running;
360
361 /*
362 * These are timestamps used for computing total_time_enabled
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200363 * and total_time_running when the event is in INACTIVE or
Paul Mackerras53cfbf52009-03-25 22:46:58 +1100364 * ACTIVE state, measured in nanoseconds from an arbitrary point
365 * in time.
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200366 * tstamp_enabled: the notional time when the event was enabled
367 * tstamp_running: the notional time when the event was scheduled on
Paul Mackerras53cfbf52009-03-25 22:46:58 +1100368 * tstamp_stopped: in INACTIVE state, the notional time when the
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200369 * event was scheduled off.
Paul Mackerras53cfbf52009-03-25 22:46:58 +1100370 */
371 u64 tstamp_enabled;
372 u64 tstamp_running;
373 u64 tstamp_stopped;
374
Stephane Eranianeed01522010-10-26 16:08:01 +0200375 /*
376 * timestamp shadows the actual context timing but it can
377 * be safely used in NMI interrupt context. It reflects the
378 * context time as it was when the event was last scheduled in.
379 *
380 * ctx_time already accounts for ctx->timestamp. Therefore to
381 * compute ctx_time for a sample, simply add perf_clock().
382 */
383 u64 shadow_ctx_time;
384
Frederic Weisbecker24f1e32c2009-09-09 19:22:48 +0200385 struct perf_event_attr attr;
Arnaldo Carvalho de Meloc320c7b2010-10-20 12:50:11 -0200386 u16 header_size;
Arnaldo Carvalho de Melo6844c092010-12-03 16:36:35 -0200387 u16 id_header_size;
Arnaldo Carvalho de Meloc320c7b2010-10-20 12:50:11 -0200388 u16 read_size;
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200389 struct hw_perf_event hw;
Thomas Gleixner0793a612008-12-04 20:12:29 +0100390
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200391 struct perf_event_context *ctx;
Al Viroa6fa9412012-08-20 14:59:25 +0100392 atomic_long_t refcount;
Thomas Gleixner0793a612008-12-04 20:12:29 +0100393
394 /*
Paul Mackerras53cfbf52009-03-25 22:46:58 +1100395 * These accumulate total time (in nanoseconds) that children
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200396 * events have been enabled and running, respectively.
Paul Mackerras53cfbf52009-03-25 22:46:58 +1100397 */
398 atomic64_t child_total_time_enabled;
399 atomic64_t child_total_time_running;
400
401 /*
Paul Mackerrasd859e292009-01-17 18:10:22 +1100402 * Protect attach/detach and child_list:
Thomas Gleixner0793a612008-12-04 20:12:29 +0100403 */
Peter Zijlstrafccc7142009-05-23 18:28:56 +0200404 struct mutex child_mutex;
405 struct list_head child_list;
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200406 struct perf_event *parent;
Thomas Gleixner0793a612008-12-04 20:12:29 +0100407
408 int oncpu;
409 int cpu;
410
Peter Zijlstra082ff5a2009-05-23 18:29:00 +0200411 struct list_head owner_entry;
412 struct task_struct *owner;
413
Peter Zijlstra7b732a72009-03-23 18:22:10 +0100414 /* mmap bits */
415 struct mutex mmap_mutex;
416 atomic_t mmap_count;
Peter Zijlstra26cb63a2013-05-28 10:55:48 +0200417
Frederic Weisbecker76369132011-05-19 19:55:04 +0200418 struct ring_buffer *rb;
Peter Zijlstra10c6db12011-11-26 02:47:31 +0100419 struct list_head rb_entry;
Peter Zijlstrab69cf532014-03-14 10:50:33 +0100420 unsigned long rcu_batches;
421 int rcu_pending;
Paul Mackerras37d81822009-03-23 18:22:08 +0100422
Peter Zijlstra7b732a72009-03-23 18:22:10 +0100423 /* poll related */
Thomas Gleixner0793a612008-12-04 20:12:29 +0100424 wait_queue_head_t waitq;
Peter Zijlstra3c446b3d2009-04-06 11:45:01 +0200425 struct fasync_struct *fasync;
Peter Zijlstra79f14642009-04-06 11:45:07 +0200426
427 /* delayed work for NMIs and such */
428 int pending_wakeup;
Peter Zijlstra4c9e2542009-04-06 11:45:09 +0200429 int pending_kill;
Peter Zijlstra79f14642009-04-06 11:45:07 +0200430 int pending_disable;
Peter Zijlstrae360adb2010-10-14 14:01:34 +0800431 struct irq_work pending;
Peter Zijlstra592903c2009-03-13 12:21:36 +0100432
Peter Zijlstra79f14642009-04-06 11:45:07 +0200433 atomic_t event_limit;
434
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200435 void (*destroy)(struct perf_event *);
Peter Zijlstra592903c2009-03-13 12:21:36 +0100436 struct rcu_head rcu_head;
Peter Zijlstra709e50c2009-06-02 14:13:15 +0200437
438 struct pid_namespace *ns;
Peter Zijlstra8e5799b2009-06-02 15:08:15 +0200439 u64 id;
Li Zefan6fb29152009-10-15 11:21:42 +0800440
Frederic Weisbeckerb326e952009-12-05 09:44:31 +0100441 perf_overflow_handler_t overflow_handler;
Avi Kivity4dc0da82011-06-29 18:42:35 +0300442 void *overflow_handler_context;
Peter Zijlstra453f19e2009-11-20 22:19:43 +0100443
Li Zefan07b139c2009-12-21 14:27:35 +0800444#ifdef CONFIG_EVENT_TRACING
Peter Zijlstra1c024eca2010-05-19 14:02:22 +0200445 struct ftrace_event_call *tp_event;
Li Zefan6fb29152009-10-15 11:21:42 +0800446 struct event_filter *filter;
Jiri Olsaced39002012-02-15 15:51:52 +0100447#ifdef CONFIG_FUNCTION_TRACER
448 struct ftrace_ops ftrace_ops;
449#endif
Ingo Molnaree060942008-12-13 09:00:03 +0100450#endif
Li Zefan6fb29152009-10-15 11:21:42 +0800451
Stephane Eraniane5d13672011-02-14 11:20:01 +0200452#ifdef CONFIG_CGROUP_PERF
453 struct perf_cgroup *cgrp; /* cgroup event is attach to */
454 int cgrp_defer_enabled;
455#endif
456
Li Zefan6fb29152009-10-15 11:21:42 +0800457#endif /* CONFIG_PERF_EVENTS */
Thomas Gleixner0793a612008-12-04 20:12:29 +0100458};
459
460/**
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200461 * struct perf_event_context - event context structure
Thomas Gleixner0793a612008-12-04 20:12:29 +0100462 *
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200463 * Used as a container for task events and CPU events as well:
Thomas Gleixner0793a612008-12-04 20:12:29 +0100464 */
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200465struct perf_event_context {
Peter Zijlstra108b02c2010-09-06 14:32:03 +0200466 struct pmu *pmu;
Thomas Gleixner0793a612008-12-04 20:12:29 +0100467 /*
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200468 * Protect the states of the events in the list,
Paul Mackerrasd859e292009-01-17 18:10:22 +1100469 * nr_active, and the list:
Thomas Gleixner0793a612008-12-04 20:12:29 +0100470 */
Thomas Gleixnere625cce12009-11-17 18:02:06 +0100471 raw_spinlock_t lock;
Paul Mackerrasd859e292009-01-17 18:10:22 +1100472 /*
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200473 * Protect the list of events. Locking either mutex or lock
Paul Mackerrasd859e292009-01-17 18:10:22 +1100474 * is sufficient to ensure the list doesn't change; to change
475 * the list you need to lock both the mutex and the spinlock.
476 */
Ingo Molnara3084442009-06-11 14:44:26 +0200477 struct mutex mutex;
Ingo Molnar04289bb2008-12-11 08:38:42 +0100478
Mark Rutland2fde4f92015-01-07 15:01:54 +0000479 struct list_head active_ctx_list;
Frederic Weisbecker889ff012010-01-09 20:04:47 +0100480 struct list_head pinned_groups;
481 struct list_head flexible_groups;
Ingo Molnara3084442009-06-11 14:44:26 +0200482 struct list_head event_list;
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200483 int nr_events;
Ingo Molnara3084442009-06-11 14:44:26 +0200484 int nr_active;
485 int is_active;
Peter Zijlstrabfbd3382009-06-24 21:11:59 +0200486 int nr_stat;
Peter Zijlstra0f5a2602011-11-16 14:38:16 +0100487 int nr_freq;
Thomas Gleixnerdddd3372010-11-24 10:05:55 +0100488 int rotate_disable;
Ingo Molnara3084442009-06-11 14:44:26 +0200489 atomic_t refcount;
490 struct task_struct *task;
Paul Mackerras53cfbf52009-03-25 22:46:58 +1100491
492 /*
Peter Zijlstra4af49982009-04-06 11:45:10 +0200493 * Context clock, runs when context enabled.
Paul Mackerras53cfbf52009-03-25 22:46:58 +1100494 */
Ingo Molnara3084442009-06-11 14:44:26 +0200495 u64 time;
496 u64 timestamp;
Paul Mackerras564c2b22009-05-22 14:27:22 +1000497
498 /*
499 * These fields let us detect when two contexts have both
500 * been cloned (inherited) from a common ancestor.
501 */
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200502 struct perf_event_context *parent_ctx;
Ingo Molnara3084442009-06-11 14:44:26 +0200503 u64 parent_gen;
504 u64 generation;
505 int pin_count;
Stephane Eraniand010b332012-02-09 23:21:00 +0100506 int nr_cgroups; /* cgroup evts */
507 int nr_branch_stack; /* branch_stack evt */
Richard Kennedy28009ce2011-06-07 16:33:38 +0100508 struct rcu_head rcu_head;
Jiri Olsafadfe7b2014-08-01 14:33:02 +0200509
510 struct delayed_work orphans_remove;
511 bool orphans_remove_sched;
Thomas Gleixner0793a612008-12-04 20:12:29 +0100512};
513
Frederic Weisbecker7ae07ea2010-08-14 20:45:13 +0200514/*
515 * Number of contexts where an event can trigger:
Ingo Molnare7e7ee22011-05-04 08:42:29 +0200516 * task, softirq, hardirq, nmi.
Frederic Weisbecker7ae07ea2010-08-14 20:45:13 +0200517 */
518#define PERF_NR_CONTEXTS 4
519
Thomas Gleixner0793a612008-12-04 20:12:29 +0100520/**
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200521 * struct perf_event_cpu_context - per cpu event context structure
Thomas Gleixner0793a612008-12-04 20:12:29 +0100522 */
523struct perf_cpu_context {
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200524 struct perf_event_context ctx;
525 struct perf_event_context *task_ctx;
Thomas Gleixner0793a612008-12-04 20:12:29 +0100526 int active_oncpu;
Paul Mackerras3b6f9e52009-01-14 21:00:30 +1100527 int exclusive;
Stephane Eranian9e630202013-04-03 14:21:33 +0200528 struct hrtimer hrtimer;
529 ktime_t hrtimer_interval;
Peter Zijlstra3f1f3322012-10-02 15:38:52 +0200530 struct pmu *unique_pmu;
Stephane Eraniane5d13672011-02-14 11:20:01 +0200531 struct perf_cgroup *cgrp;
Thomas Gleixner0793a612008-12-04 20:12:29 +0100532};
533
Markus Metzger5622f292009-09-15 13:00:23 +0200534struct perf_output_handle {
Ingo Molnar57c0c152009-09-21 12:20:38 +0200535 struct perf_event *event;
Frederic Weisbecker76369132011-05-19 19:55:04 +0200536 struct ring_buffer *rb;
Peter Zijlstra6d1acfd2010-05-18 11:12:48 +0200537 unsigned long wakeup;
Peter Zijlstra5d967a82010-05-20 16:46:39 +0200538 unsigned long size;
539 void *addr;
540 int page;
Markus Metzger5622f292009-09-15 13:00:23 +0200541};
542
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200543#ifdef CONFIG_PERF_EVENTS
Robert Richter829b42d2009-04-29 12:46:59 +0200544
Mischa Jonker03d8e802013-06-04 11:45:48 +0200545extern int perf_pmu_register(struct pmu *pmu, const char *name, int type);
Peter Zijlstrab0a873e2010-06-11 13:35:08 +0200546extern void perf_pmu_unregister(struct pmu *pmu);
Ingo Molnar621a01e2008-12-11 12:46:46 +0100547
Matt Fleming3bf101b2010-09-27 20:22:24 +0100548extern int perf_num_counters(void);
Matt Fleming84c79912010-10-03 21:41:13 +0100549extern const char *perf_pmu_name(void);
Jiri Olsaab0cce52012-05-23 13:13:02 +0200550extern void __perf_event_task_sched_in(struct task_struct *prev,
551 struct task_struct *task);
552extern void __perf_event_task_sched_out(struct task_struct *prev,
553 struct task_struct *next);
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200554extern int perf_event_init_task(struct task_struct *child);
555extern void perf_event_exit_task(struct task_struct *child);
556extern void perf_event_free_task(struct task_struct *task);
Peter Zijlstra4e231c72010-09-09 21:01:59 +0200557extern void perf_event_delayed_put(struct task_struct *task);
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200558extern void perf_event_print_debug(void);
Peter Zijlstra33696fc2010-06-14 08:49:00 +0200559extern void perf_pmu_disable(struct pmu *pmu);
560extern void perf_pmu_enable(struct pmu *pmu);
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200561extern int perf_event_task_disable(void);
562extern int perf_event_task_enable(void);
Avi Kivity26ca5c12011-06-29 18:42:37 +0300563extern int perf_event_refresh(struct perf_event *event, int refresh);
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200564extern void perf_event_update_userpage(struct perf_event *event);
Arjan van de Venfb0459d2009-09-25 12:25:56 +0200565extern int perf_event_release_kernel(struct perf_event *event);
566extern struct perf_event *
567perf_event_create_kernel_counter(struct perf_event_attr *attr,
568 int cpu,
Matt Helsley38a81da2010-09-13 13:01:20 -0700569 struct task_struct *task,
Avi Kivity4dc0da82011-06-29 18:42:35 +0300570 perf_overflow_handler_t callback,
571 void *context);
Yan, Zheng0cda4c02012-06-15 14:31:33 +0800572extern void perf_pmu_migrate_context(struct pmu *pmu,
573 int src_cpu, int dst_cpu);
Peter Zijlstra59ed4462009-11-20 22:19:55 +0100574extern u64 perf_event_read_value(struct perf_event *event,
575 u64 *enabled, u64 *running);
Ingo Molnar5c92d122008-12-11 13:21:10 +0100576
Stephane Eraniand010b332012-02-09 23:21:00 +0100577
Peter Zijlstradf1a1322009-06-10 21:02:22 +0200578struct perf_sample_data {
Peter Zijlstra25657112014-09-24 13:48:42 +0200579 /*
580 * Fields set by perf_sample_data_init(), group so as to
581 * minimize the cachelines touched.
582 */
583 u64 addr;
584 struct perf_raw_record *raw;
585 struct perf_branch_stack *br_stack;
586 u64 period;
587 u64 weight;
588 u64 txn;
589 union perf_mem_data_src data_src;
Markus Metzger5622f292009-09-15 13:00:23 +0200590
Peter Zijlstra25657112014-09-24 13:48:42 +0200591 /*
592 * The other fields, optionally {set,used} by
593 * perf_{prepare,output}_sample().
594 */
595 u64 type;
Markus Metzger5622f292009-09-15 13:00:23 +0200596 u64 ip;
597 struct {
598 u32 pid;
599 u32 tid;
600 } tid_entry;
601 u64 time;
Markus Metzger5622f292009-09-15 13:00:23 +0200602 u64 id;
603 u64 stream_id;
604 struct {
605 u32 cpu;
606 u32 reserved;
607 } cpu_entry;
Markus Metzger5622f292009-09-15 13:00:23 +0200608 struct perf_callchain_entry *callchain;
Andy Lutomirski88a7c262015-01-04 10:36:19 -0800609
610 /*
611 * regs_user may point to task_pt_regs or to regs_user_copy, depending
612 * on arch details.
613 */
Stephane Eranian60e23642014-09-24 13:48:37 +0200614 struct perf_regs regs_user;
Andy Lutomirski88a7c262015-01-04 10:36:19 -0800615 struct pt_regs regs_user_copy;
616
Stephane Eranian60e23642014-09-24 13:48:37 +0200617 struct perf_regs regs_intr;
Jiri Olsac5ebced2012-08-07 15:20:40 +0200618 u64 stack_user_size;
Peter Zijlstra25657112014-09-24 13:48:42 +0200619} ____cacheline_aligned;
Peter Zijlstradf1a1322009-06-10 21:02:22 +0200620
Stephane Eranian770eee12014-08-11 21:27:12 +0200621/* default value for data source */
622#define PERF_MEM_NA (PERF_MEM_S(OP, NA) |\
623 PERF_MEM_S(LVL, NA) |\
624 PERF_MEM_S(SNOOP, NA) |\
625 PERF_MEM_S(LOCK, NA) |\
626 PERF_MEM_S(TLB, NA))
627
Robert Richterfd0d0002012-04-02 20:19:08 +0200628static inline void perf_sample_data_init(struct perf_sample_data *data,
629 u64 addr, u64 period)
Peter Zijlstradc1d6282010-03-03 15:55:04 +0100630{
Robert Richterfd0d0002012-04-02 20:19:08 +0200631 /* remaining struct members initialized in perf_prepare_sample() */
Peter Zijlstradc1d6282010-03-03 15:55:04 +0100632 data->addr = addr;
633 data->raw = NULL;
Stephane Eranianbce38cd2012-02-09 23:20:51 +0100634 data->br_stack = NULL;
Jiri Olsa40189942012-08-07 15:20:37 +0200635 data->period = period;
Andi Kleenc3feedf2013-01-24 16:10:28 +0100636 data->weight = 0;
Stephane Eranian770eee12014-08-11 21:27:12 +0200637 data->data_src.val = PERF_MEM_NA;
Andi Kleenfdfbbd02013-09-20 07:40:39 -0700638 data->txn = 0;
Peter Zijlstradc1d6282010-03-03 15:55:04 +0100639}
640
Markus Metzger5622f292009-09-15 13:00:23 +0200641extern void perf_output_sample(struct perf_output_handle *handle,
642 struct perf_event_header *header,
643 struct perf_sample_data *data,
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200644 struct perf_event *event);
Markus Metzger5622f292009-09-15 13:00:23 +0200645extern void perf_prepare_sample(struct perf_event_header *header,
646 struct perf_sample_data *data,
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200647 struct perf_event *event,
Markus Metzger5622f292009-09-15 13:00:23 +0200648 struct pt_regs *regs);
649
Peter Zijlstraa8b0ca12011-06-27 14:41:57 +0200650extern int perf_event_overflow(struct perf_event *event,
Markus Metzger5622f292009-09-15 13:00:23 +0200651 struct perf_sample_data *data,
652 struct pt_regs *regs);
Peter Zijlstradf1a1322009-06-10 21:02:22 +0200653
Franck Bui-Huu6c7e5502010-11-23 16:21:43 +0100654static inline bool is_sampling_event(struct perf_event *event)
655{
656 return event->attr.sample_period != 0;
657}
658
Paul Mackerras3b6f9e52009-01-14 21:00:30 +1100659/*
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200660 * Return 1 for a software event, 0 for a hardware event
Paul Mackerras3b6f9e52009-01-14 21:00:30 +1100661 */
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200662static inline int is_software_event(struct perf_event *event)
Paul Mackerras3b6f9e52009-01-14 21:00:30 +1100663{
Peter Zijlstra89a1e182010-09-07 17:34:50 +0200664 return event->pmu->task_ctx_nr == perf_sw_context;
Paul Mackerras3b6f9e52009-01-14 21:00:30 +1100665}
666
Ingo Molnarc5905af2012-02-24 08:31:31 +0100667extern struct static_key perf_swevent_enabled[PERF_COUNT_SW_MAX];
Peter Zijlstraf29ac752009-06-19 18:27:26 +0200668
Peter Zijlstra (Intel)86038c52014-12-16 12:47:34 +0100669extern void ___perf_sw_event(u32, u64, struct pt_regs *, u64);
Peter Zijlstraa8b0ca12011-06-27 14:41:57 +0200670extern void __perf_sw_event(u32, u64, struct pt_regs *, u64);
Peter Zijlstraf29ac752009-06-19 18:27:26 +0200671
Frederic Weisbeckerb0f82b82010-05-20 07:47:21 +0200672#ifndef perf_arch_fetch_caller_regs
Ingo Molnare7e7ee22011-05-04 08:42:29 +0200673static inline void perf_arch_fetch_caller_regs(struct pt_regs *regs, unsigned long ip) { }
Frederic Weisbeckerb0f82b82010-05-20 07:47:21 +0200674#endif
Frederic Weisbecker5331d7b2010-03-04 21:15:56 +0100675
676/*
677 * Take a snapshot of the regs. Skip ip and frame pointer to
678 * the nth caller. We only need a few of the regs:
679 * - ip for PERF_SAMPLE_IP
680 * - cs for user_mode() tests
681 * - bp for callchains
682 * - eflags, for future purposes, just in case
683 */
Frederic Weisbeckerb0f82b82010-05-20 07:47:21 +0200684static inline void perf_fetch_caller_regs(struct pt_regs *regs)
Frederic Weisbecker5331d7b2010-03-04 21:15:56 +0100685{
Frederic Weisbecker5331d7b2010-03-04 21:15:56 +0100686 memset(regs, 0, sizeof(*regs));
687
Frederic Weisbeckerb0f82b82010-05-20 07:47:21 +0200688 perf_arch_fetch_caller_regs(regs, CALLER_ADDR0);
Frederic Weisbecker5331d7b2010-03-04 21:15:56 +0100689}
690
Peter Zijlstra7e54a5a2010-10-14 22:32:45 +0200691static __always_inline void
Peter Zijlstraa8b0ca12011-06-27 14:41:57 +0200692perf_sw_event(u32 event_id, u64 nr, struct pt_regs *regs, u64 addr)
Frederic Weisbeckere49a5bd2010-03-22 19:40:03 +0100693{
Peter Zijlstra (Intel)86038c52014-12-16 12:47:34 +0100694 if (static_key_false(&perf_swevent_enabled[event_id]))
Peter Zijlstraa8b0ca12011-06-27 14:41:57 +0200695 __perf_sw_event(event_id, nr, regs, addr);
Peter Zijlstra (Intel)86038c52014-12-16 12:47:34 +0100696}
697
698DECLARE_PER_CPU(struct pt_regs, __perf_regs[4]);
699
700/*
701 * 'Special' version for the scheduler, it hard assumes no recursion,
702 * which is guaranteed by us not actually scheduling inside other swevents
703 * because those disable preemption.
704 */
705static __always_inline void
706perf_sw_event_sched(u32 event_id, u64 nr, u64 addr)
707{
708 if (static_key_false(&perf_swevent_enabled[event_id])) {
709 struct pt_regs *regs = this_cpu_ptr(&__perf_regs[0]);
710
711 perf_fetch_caller_regs(regs);
712 ___perf_sw_event(event_id, nr, regs, addr);
Frederic Weisbeckere49a5bd2010-03-22 19:40:03 +0100713 }
714}
715
Ingo Molnarc5905af2012-02-24 08:31:31 +0100716extern struct static_key_deferred perf_sched_events;
Peter Zijlstraee6dcfa2010-11-26 13:49:04 +0100717
Jiri Olsaab0cce52012-05-23 13:13:02 +0200718static inline void perf_event_task_sched_in(struct task_struct *prev,
Stephane Eraniana8d757e2011-08-25 15:58:03 +0200719 struct task_struct *task)
Peter Zijlstraee6dcfa2010-11-26 13:49:04 +0100720{
Jiri Olsaab0cce52012-05-23 13:13:02 +0200721 if (static_key_false(&perf_sched_events.key))
722 __perf_event_task_sched_in(prev, task);
723}
724
725static inline void perf_event_task_sched_out(struct task_struct *prev,
726 struct task_struct *next)
727{
Peter Zijlstra (Intel)86038c52014-12-16 12:47:34 +0100728 perf_sw_event_sched(PERF_COUNT_SW_CONTEXT_SWITCHES, 1, 0);
Peter Zijlstraee6dcfa2010-11-26 13:49:04 +0100729
Ingo Molnarc5905af2012-02-24 08:31:31 +0100730 if (static_key_false(&perf_sched_events.key))
Jiri Olsaab0cce52012-05-23 13:13:02 +0200731 __perf_event_task_sched_out(prev, next);
Peter Zijlstraee6dcfa2010-11-26 13:49:04 +0100732}
733
Eric B Munson3af9e852010-05-18 15:30:49 +0100734extern void perf_event_mmap(struct vm_area_struct *vma);
Zhang, Yanmin39447b32010-04-19 13:32:41 +0800735extern struct perf_guest_info_callbacks *perf_guest_cbs;
Zhang, Yanmindcf46b92010-04-20 10:13:58 +0800736extern int perf_register_guest_info_callbacks(struct perf_guest_info_callbacks *callbacks);
737extern int perf_unregister_guest_info_callbacks(struct perf_guest_info_callbacks *callbacks);
Zhang, Yanmin39447b32010-04-19 13:32:41 +0800738
Peter Zijlstrae041e322014-05-21 17:32:19 +0200739extern void perf_event_exec(void);
Adrian Hunter82b89772014-05-28 11:45:04 +0300740extern void perf_event_comm(struct task_struct *tsk, bool exec);
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200741extern void perf_event_fork(struct task_struct *tsk);
Peter Zijlstra8d1b2d92009-04-08 15:01:30 +0200742
Frederic Weisbecker56962b42010-06-30 23:03:51 +0200743/* Callchains */
744DECLARE_PER_CPU(struct perf_callchain_entry, perf_callchain_entry);
745
Ingo Molnare7e7ee22011-05-04 08:42:29 +0200746extern void perf_callchain_user(struct perf_callchain_entry *entry, struct pt_regs *regs);
747extern void perf_callchain_kernel(struct perf_callchain_entry *entry, struct pt_regs *regs);
Frederic Weisbecker56962b42010-06-30 23:03:51 +0200748
Ingo Molnare7e7ee22011-05-04 08:42:29 +0200749static inline void perf_callchain_store(struct perf_callchain_entry *entry, u64 ip)
Frederic Weisbecker70791ce2010-06-29 19:34:05 +0200750{
751 if (entry->nr < PERF_MAX_STACK_DEPTH)
752 entry->ip[entry->nr++] = ip;
753}
Peter Zijlstra394ee072009-03-30 19:07:14 +0200754
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200755extern int sysctl_perf_event_paranoid;
756extern int sysctl_perf_event_mlock;
757extern int sysctl_perf_event_sample_rate;
Dave Hansen14c63f12013-06-21 08:51:36 -0700758extern int sysctl_perf_cpu_time_max_percent;
759
760extern void perf_sample_event_took(u64 sample_len_ns);
Peter Zijlstra1ccd1542009-04-09 10:53:45 +0200761
Peter Zijlstra163ec432011-02-16 11:22:34 +0100762extern int perf_proc_update_handler(struct ctl_table *table, int write,
763 void __user *buffer, size_t *lenp,
764 loff_t *ppos);
Dave Hansen14c63f12013-06-21 08:51:36 -0700765extern int perf_cpu_time_max_percent_handler(struct ctl_table *table, int write,
766 void __user *buffer, size_t *lenp,
767 loff_t *ppos);
768
Peter Zijlstra163ec432011-02-16 11:22:34 +0100769
Peter Zijlstra320ebf02010-03-02 12:35:37 +0100770static inline bool perf_paranoid_tracepoint_raw(void)
771{
772 return sysctl_perf_event_paranoid > -1;
773}
774
775static inline bool perf_paranoid_cpu(void)
776{
777 return sysctl_perf_event_paranoid > 0;
778}
779
780static inline bool perf_paranoid_kernel(void)
781{
782 return sysctl_perf_event_paranoid > 1;
783}
784
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200785extern void perf_event_init(void);
Peter Zijlstra1c024eca2010-05-19 14:02:22 +0200786extern void perf_tp_event(u64 addr, u64 count, void *record,
787 int entry_size, struct pt_regs *regs,
Andrew Vagine6dab5f2012-07-11 18:14:58 +0400788 struct hlist_head *head, int rctx,
789 struct task_struct *task);
Frederic Weisbecker24f1e32c2009-09-09 19:22:48 +0200790extern void perf_bp_event(struct perf_event *event, void *data);
Ingo Molnar0d905bc2009-05-04 19:13:30 +0200791
Paul Mackerras9d23a902009-05-14 21:48:08 +1000792#ifndef perf_misc_flags
Ingo Molnare7e7ee22011-05-04 08:42:29 +0200793# define perf_misc_flags(regs) \
794 (user_mode(regs) ? PERF_RECORD_MISC_USER : PERF_RECORD_MISC_KERNEL)
795# define perf_instruction_pointer(regs) instruction_pointer(regs)
Paul Mackerras9d23a902009-05-14 21:48:08 +1000796#endif
797
Stephane Eranianbce38cd2012-02-09 23:20:51 +0100798static inline bool has_branch_stack(struct perf_event *event)
799{
800 return event->attr.sample_type & PERF_SAMPLE_BRANCH_STACK;
801}
802
Markus Metzger5622f292009-09-15 13:00:23 +0200803extern int perf_output_begin(struct perf_output_handle *handle,
Peter Zijlstraa7ac67e2011-06-27 16:47:16 +0200804 struct perf_event *event, unsigned int size);
Markus Metzger5622f292009-09-15 13:00:23 +0200805extern void perf_output_end(struct perf_output_handle *handle);
Frederic Weisbecker91d77532012-08-07 15:20:38 +0200806extern unsigned int perf_output_copy(struct perf_output_handle *handle,
Markus Metzger5622f292009-09-15 13:00:23 +0200807 const void *buf, unsigned int len);
Jiri Olsa5685e0f2012-08-07 15:20:39 +0200808extern unsigned int perf_output_skip(struct perf_output_handle *handle,
809 unsigned int len);
Peter Zijlstra4ed7c922009-11-23 11:37:29 +0100810extern int perf_swevent_get_recursion_context(void);
811extern void perf_swevent_put_recursion_context(int rctx);
Jiri Olsaab573842013-05-01 17:25:44 +0200812extern u64 perf_swevent_set_period(struct perf_event *event);
Frederic Weisbecker44234ad2009-12-09 09:25:48 +0100813extern void perf_event_enable(struct perf_event *event);
814extern void perf_event_disable(struct perf_event *event);
K.Prasad500ad2d2012-08-02 13:46:35 +0530815extern int __perf_event_disable(void *info);
Peter Zijlstrae9d2b062010-09-17 11:28:50 +0200816extern void perf_event_task_tick(void);
Peter Zijlstrae041e322014-05-21 17:32:19 +0200817#else /* !CONFIG_PERF_EVENTS: */
Thomas Gleixner0793a612008-12-04 20:12:29 +0100818static inline void
Jiri Olsaab0cce52012-05-23 13:13:02 +0200819perf_event_task_sched_in(struct task_struct *prev,
820 struct task_struct *task) { }
821static inline void
822perf_event_task_sched_out(struct task_struct *prev,
823 struct task_struct *next) { }
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200824static inline int perf_event_init_task(struct task_struct *child) { return 0; }
825static inline void perf_event_exit_task(struct task_struct *child) { }
826static inline void perf_event_free_task(struct task_struct *task) { }
Peter Zijlstra4e231c72010-09-09 21:01:59 +0200827static inline void perf_event_delayed_put(struct task_struct *task) { }
Ingo Molnar57c0c152009-09-21 12:20:38 +0200828static inline void perf_event_print_debug(void) { }
Ingo Molnar57c0c152009-09-21 12:20:38 +0200829static inline int perf_event_task_disable(void) { return -EINVAL; }
830static inline int perf_event_task_enable(void) { return -EINVAL; }
Avi Kivity26ca5c12011-06-29 18:42:37 +0300831static inline int perf_event_refresh(struct perf_event *event, int refresh)
832{
833 return -EINVAL;
834}
Peter Zijlstra15dbf272009-03-13 12:21:32 +0100835
Peter Zijlstra925d5192009-03-30 19:07:02 +0200836static inline void
Peter Zijlstraa8b0ca12011-06-27 14:41:57 +0200837perf_sw_event(u32 event_id, u64 nr, struct pt_regs *regs, u64 addr) { }
Frederic Weisbecker24f1e32c2009-09-09 19:22:48 +0200838static inline void
Peter Zijlstra (Intel)86038c52014-12-16 12:47:34 +0100839perf_sw_event_sched(u32 event_id, u64 nr, u64 addr) { }
840static inline void
Ingo Molnar184f4122010-01-27 08:39:39 +0100841perf_bp_event(struct perf_event *event, void *data) { }
Peter Zijlstra0a4a9392009-03-30 19:07:05 +0200842
Zhang, Yanmin39447b32010-04-19 13:32:41 +0800843static inline int perf_register_guest_info_callbacks
Ingo Molnare7e7ee22011-05-04 08:42:29 +0200844(struct perf_guest_info_callbacks *callbacks) { return 0; }
Zhang, Yanmin39447b32010-04-19 13:32:41 +0800845static inline int perf_unregister_guest_info_callbacks
Ingo Molnare7e7ee22011-05-04 08:42:29 +0200846(struct perf_guest_info_callbacks *callbacks) { return 0; }
Zhang, Yanmin39447b32010-04-19 13:32:41 +0800847
Ingo Molnar57c0c152009-09-21 12:20:38 +0200848static inline void perf_event_mmap(struct vm_area_struct *vma) { }
Peter Zijlstrae041e322014-05-21 17:32:19 +0200849static inline void perf_event_exec(void) { }
Adrian Hunter82b89772014-05-28 11:45:04 +0300850static inline void perf_event_comm(struct task_struct *tsk, bool exec) { }
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200851static inline void perf_event_fork(struct task_struct *tsk) { }
852static inline void perf_event_init(void) { }
Ingo Molnar184f4122010-01-27 08:39:39 +0100853static inline int perf_swevent_get_recursion_context(void) { return -1; }
Peter Zijlstra4ed7c922009-11-23 11:37:29 +0100854static inline void perf_swevent_put_recursion_context(int rctx) { }
Jiri Olsaab573842013-05-01 17:25:44 +0200855static inline u64 perf_swevent_set_period(struct perf_event *event) { return 0; }
Frederic Weisbecker44234ad2009-12-09 09:25:48 +0100856static inline void perf_event_enable(struct perf_event *event) { }
857static inline void perf_event_disable(struct perf_event *event) { }
K.Prasad500ad2d2012-08-02 13:46:35 +0530858static inline int __perf_event_disable(void *info) { return -1; }
Peter Zijlstrae9d2b062010-09-17 11:28:50 +0200859static inline void perf_event_task_tick(void) { }
Thomas Gleixner0793a612008-12-04 20:12:29 +0100860#endif
861
Frederic Weisbecker026249e2013-04-20 15:58:34 +0200862#if defined(CONFIG_PERF_EVENTS) && defined(CONFIG_NO_HZ_FULL)
863extern bool perf_event_can_stop_tick(void);
864#else
865static inline bool perf_event_can_stop_tick(void) { return true; }
866#endif
867
David Rientjes6c4d3bc2013-03-17 15:49:10 -0700868#if defined(CONFIG_PERF_EVENTS) && defined(CONFIG_CPU_SUP_INTEL)
869extern void perf_restore_debug_store(void);
870#else
Stephane Eranian1d9d8632013-03-15 14:26:07 +0100871static inline void perf_restore_debug_store(void) { }
Thomas Gleixner0793a612008-12-04 20:12:29 +0100872#endif
873
Ingo Molnare7e7ee22011-05-04 08:42:29 +0200874#define perf_output_put(handle, x) perf_output_copy((handle), &(x), sizeof(x))
Markus Metzger5622f292009-09-15 13:00:23 +0200875
Peter Zijlstra3f6da392010-03-05 13:01:18 +0100876/*
Viresh Kumar0a0fca92013-06-04 13:10:24 +0530877 * This has to have a higher priority than migration_notifier in sched/core.c.
Peter Zijlstra3f6da392010-03-05 13:01:18 +0100878 */
Ingo Molnare7e7ee22011-05-04 08:42:29 +0200879#define perf_cpu_notifier(fn) \
880do { \
Paul Gortmaker0db06282013-06-19 14:53:51 -0400881 static struct notifier_block fn##_nb = \
Ingo Molnare7e7ee22011-05-04 08:42:29 +0200882 { .notifier_call = fn, .priority = CPU_PRI_PERF }; \
Srivatsa S. Bhatc13d38e2012-10-16 13:28:17 +0530883 unsigned long cpu = smp_processor_id(); \
Srivatsa S. Bhat6760bca2012-10-16 13:28:10 +0530884 unsigned long flags; \
Srivatsa S. Bhatf0bdb5e2014-03-11 02:04:39 +0530885 \
886 cpu_notifier_register_begin(); \
Ingo Molnare7e7ee22011-05-04 08:42:29 +0200887 fn(&fn##_nb, (unsigned long)CPU_UP_PREPARE, \
Srivatsa S. Bhatc13d38e2012-10-16 13:28:17 +0530888 (void *)(unsigned long)cpu); \
Srivatsa S. Bhat6760bca2012-10-16 13:28:10 +0530889 local_irq_save(flags); \
Ingo Molnare7e7ee22011-05-04 08:42:29 +0200890 fn(&fn##_nb, (unsigned long)CPU_STARTING, \
Srivatsa S. Bhatc13d38e2012-10-16 13:28:17 +0530891 (void *)(unsigned long)cpu); \
Srivatsa S. Bhat6760bca2012-10-16 13:28:10 +0530892 local_irq_restore(flags); \
Ingo Molnare7e7ee22011-05-04 08:42:29 +0200893 fn(&fn##_nb, (unsigned long)CPU_ONLINE, \
Srivatsa S. Bhatc13d38e2012-10-16 13:28:17 +0530894 (void *)(unsigned long)cpu); \
Srivatsa S. Bhatf0bdb5e2014-03-11 02:04:39 +0530895 __register_cpu_notifier(&fn##_nb); \
896 cpu_notifier_register_done(); \
Peter Zijlstra3f6da392010-03-05 13:01:18 +0100897} while (0)
898
Srivatsa S. Bhatf0bdb5e2014-03-11 02:04:39 +0530899/*
900 * Bare-bones version of perf_cpu_notifier(), which doesn't invoke the
901 * callback for already online CPUs.
902 */
903#define __perf_cpu_notifier(fn) \
904do { \
905 static struct notifier_block fn##_nb = \
906 { .notifier_call = fn, .priority = CPU_PRI_PERF }; \
907 \
908 __register_cpu_notifier(&fn##_nb); \
909} while (0)
Jiri Olsa641cc932012-03-15 20:09:14 +0100910
Sukadev Bhattiprolu26639602013-01-22 22:24:23 -0800911struct perf_pmu_events_attr {
912 struct device_attribute attr;
913 u64 id;
Stephane Eranian3a54aaa2013-01-24 16:10:26 +0100914 const char *event_str;
Sukadev Bhattiprolu26639602013-01-22 22:24:23 -0800915};
916
Cody P Schaferfd979c02015-01-30 13:45:57 -0800917ssize_t perf_event_sysfs_show(struct device *dev, struct device_attribute *attr,
918 char *page);
919
Sukadev Bhattiprolu26639602013-01-22 22:24:23 -0800920#define PMU_EVENT_ATTR(_name, _var, _id, _show) \
921static struct perf_pmu_events_attr _var = { \
922 .attr = __ATTR(_name, 0444, _show, NULL), \
923 .id = _id, \
924};
925
Cody P Schaferf0405b82015-01-30 13:45:58 -0800926#define PMU_EVENT_ATTR_STRING(_name, _var, _str) \
927static struct perf_pmu_events_attr _var = { \
928 .attr = __ATTR(_name, 0444, perf_event_sysfs_show, NULL), \
929 .id = 0, \
930 .event_str = _str, \
931};
932
Jiri Olsa641cc932012-03-15 20:09:14 +0100933#define PMU_FORMAT_ATTR(_name, _format) \
934static ssize_t \
935_name##_show(struct device *dev, \
936 struct device_attribute *attr, \
937 char *page) \
938{ \
939 BUILD_BUG_ON(sizeof(_format) >= PAGE_SIZE); \
940 return sprintf(page, _format "\n"); \
941} \
942 \
943static struct device_attribute format_attr_##_name = __ATTR_RO(_name)
944
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200945#endif /* _LINUX_PERF_EVENT_H */