Kevin Winchester | de0428a | 2011-08-30 20:41:05 -0300 | [diff] [blame] | 1 | #include <linux/perf_event.h> |
| 2 | #include <linux/types.h> |
| 3 | |
| 4 | #include <asm/perf_event.h> |
| 5 | #include <asm/msr.h> |
Stephane Eranian | 3e702ff | 2012-02-09 23:20:58 +0100 | [diff] [blame] | 6 | #include <asm/insn.h> |
Kevin Winchester | de0428a | 2011-08-30 20:41:05 -0300 | [diff] [blame] | 7 | |
Borislav Petkov | 27f6d22 | 2016-02-10 10:55:23 +0100 | [diff] [blame] | 8 | #include "../perf_event.h" |
Peter Zijlstra | caff2be | 2010-03-03 12:02:30 +0100 | [diff] [blame] | 9 | |
| 10 | enum { |
| 11 | LBR_FORMAT_32 = 0x00, |
| 12 | LBR_FORMAT_LIP = 0x01, |
| 13 | LBR_FORMAT_EIP = 0x02, |
| 14 | LBR_FORMAT_EIP_FLAGS = 0x03, |
Andi Kleen | 135c561 | 2013-06-17 17:36:51 -0700 | [diff] [blame] | 15 | LBR_FORMAT_EIP_FLAGS2 = 0x04, |
Andi Kleen | 50eab8f | 2015-05-10 12:22:43 -0700 | [diff] [blame] | 16 | LBR_FORMAT_INFO = 0x05, |
Kan Liang | 8b92c3a | 2016-04-15 00:42:47 -0700 | [diff] [blame] | 17 | LBR_FORMAT_TIME = 0x06, |
| 18 | LBR_FORMAT_MAX_KNOWN = LBR_FORMAT_TIME, |
Andi Kleen | 135c561 | 2013-06-17 17:36:51 -0700 | [diff] [blame] | 19 | }; |
| 20 | |
| 21 | static enum { |
| 22 | LBR_EIP_FLAGS = 1, |
| 23 | LBR_TSX = 2, |
| 24 | } lbr_desc[LBR_FORMAT_MAX_KNOWN + 1] = { |
| 25 | [LBR_FORMAT_EIP_FLAGS] = LBR_EIP_FLAGS, |
| 26 | [LBR_FORMAT_EIP_FLAGS2] = LBR_EIP_FLAGS | LBR_TSX, |
Peter Zijlstra | caff2be | 2010-03-03 12:02:30 +0100 | [diff] [blame] | 27 | }; |
| 28 | |
| 29 | /* |
Stephane Eranian | c5cc2cd | 2012-02-09 23:20:55 +0100 | [diff] [blame] | 30 | * Intel LBR_SELECT bits |
| 31 | * Intel Vol3a, April 2011, Section 16.7 Table 16-10 |
| 32 | * |
| 33 | * Hardware branch filter (not available on all CPUs) |
| 34 | */ |
| 35 | #define LBR_KERNEL_BIT 0 /* do not capture at ring0 */ |
| 36 | #define LBR_USER_BIT 1 /* do not capture at ring > 0 */ |
| 37 | #define LBR_JCC_BIT 2 /* do not capture conditional branches */ |
| 38 | #define LBR_REL_CALL_BIT 3 /* do not capture relative calls */ |
| 39 | #define LBR_IND_CALL_BIT 4 /* do not capture indirect calls */ |
| 40 | #define LBR_RETURN_BIT 5 /* do not capture near returns */ |
| 41 | #define LBR_IND_JMP_BIT 6 /* do not capture indirect jumps */ |
| 42 | #define LBR_REL_JMP_BIT 7 /* do not capture relative jumps */ |
| 43 | #define LBR_FAR_BIT 8 /* do not capture far branches */ |
Yan, Zheng | e9d7f7c | 2014-11-04 21:56:00 -0500 | [diff] [blame] | 44 | #define LBR_CALL_STACK_BIT 9 /* enable call stack */ |
Stephane Eranian | c5cc2cd | 2012-02-09 23:20:55 +0100 | [diff] [blame] | 45 | |
Andi Kleen | b16a5b5 | 2015-10-20 11:46:34 -0700 | [diff] [blame] | 46 | /* |
| 47 | * Following bit only exists in Linux; we mask it out before writing it to |
| 48 | * the actual MSR. But it helps the constraint perf code to understand |
| 49 | * that this is a separate configuration. |
| 50 | */ |
| 51 | #define LBR_NO_INFO_BIT 63 /* don't read LBR_INFO. */ |
| 52 | |
Stephane Eranian | c5cc2cd | 2012-02-09 23:20:55 +0100 | [diff] [blame] | 53 | #define LBR_KERNEL (1 << LBR_KERNEL_BIT) |
| 54 | #define LBR_USER (1 << LBR_USER_BIT) |
| 55 | #define LBR_JCC (1 << LBR_JCC_BIT) |
| 56 | #define LBR_REL_CALL (1 << LBR_REL_CALL_BIT) |
| 57 | #define LBR_IND_CALL (1 << LBR_IND_CALL_BIT) |
| 58 | #define LBR_RETURN (1 << LBR_RETURN_BIT) |
| 59 | #define LBR_REL_JMP (1 << LBR_REL_JMP_BIT) |
| 60 | #define LBR_IND_JMP (1 << LBR_IND_JMP_BIT) |
| 61 | #define LBR_FAR (1 << LBR_FAR_BIT) |
Yan, Zheng | e9d7f7c | 2014-11-04 21:56:00 -0500 | [diff] [blame] | 62 | #define LBR_CALL_STACK (1 << LBR_CALL_STACK_BIT) |
Andi Kleen | b16a5b5 | 2015-10-20 11:46:34 -0700 | [diff] [blame] | 63 | #define LBR_NO_INFO (1ULL << LBR_NO_INFO_BIT) |
Stephane Eranian | c5cc2cd | 2012-02-09 23:20:55 +0100 | [diff] [blame] | 64 | |
| 65 | #define LBR_PLM (LBR_KERNEL | LBR_USER) |
| 66 | |
| 67 | #define LBR_SEL_MASK 0x1ff /* valid bits in LBR_SELECT */ |
| 68 | #define LBR_NOT_SUPP -1 /* LBR filter not supported */ |
| 69 | #define LBR_IGN 0 /* ignored */ |
| 70 | |
| 71 | #define LBR_ANY \ |
| 72 | (LBR_JCC |\ |
| 73 | LBR_REL_CALL |\ |
| 74 | LBR_IND_CALL |\ |
| 75 | LBR_RETURN |\ |
| 76 | LBR_REL_JMP |\ |
| 77 | LBR_IND_JMP |\ |
| 78 | LBR_FAR) |
| 79 | |
| 80 | #define LBR_FROM_FLAG_MISPRED (1ULL << 63) |
Andi Kleen | 135c561 | 2013-06-17 17:36:51 -0700 | [diff] [blame] | 81 | #define LBR_FROM_FLAG_IN_TX (1ULL << 62) |
| 82 | #define LBR_FROM_FLAG_ABORT (1ULL << 61) |
Stephane Eranian | c5cc2cd | 2012-02-09 23:20:55 +0100 | [diff] [blame] | 83 | |
| 84 | /* |
Stephane Eranian | 3e702ff | 2012-02-09 23:20:58 +0100 | [diff] [blame] | 85 | * x86control flow change classification |
| 86 | * x86control flow changes include branches, interrupts, traps, faults |
| 87 | */ |
| 88 | enum { |
Yan, Zheng | e9d7f7c | 2014-11-04 21:56:00 -0500 | [diff] [blame] | 89 | X86_BR_NONE = 0, /* unknown */ |
Stephane Eranian | 3e702ff | 2012-02-09 23:20:58 +0100 | [diff] [blame] | 90 | |
Yan, Zheng | e9d7f7c | 2014-11-04 21:56:00 -0500 | [diff] [blame] | 91 | X86_BR_USER = 1 << 0, /* branch target is user */ |
| 92 | X86_BR_KERNEL = 1 << 1, /* branch target is kernel */ |
Stephane Eranian | 3e702ff | 2012-02-09 23:20:58 +0100 | [diff] [blame] | 93 | |
Yan, Zheng | e9d7f7c | 2014-11-04 21:56:00 -0500 | [diff] [blame] | 94 | X86_BR_CALL = 1 << 2, /* call */ |
| 95 | X86_BR_RET = 1 << 3, /* return */ |
| 96 | X86_BR_SYSCALL = 1 << 4, /* syscall */ |
| 97 | X86_BR_SYSRET = 1 << 5, /* syscall return */ |
| 98 | X86_BR_INT = 1 << 6, /* sw interrupt */ |
| 99 | X86_BR_IRET = 1 << 7, /* return from interrupt */ |
| 100 | X86_BR_JCC = 1 << 8, /* conditional */ |
| 101 | X86_BR_JMP = 1 << 9, /* jump */ |
| 102 | X86_BR_IRQ = 1 << 10,/* hw interrupt or trap or fault */ |
| 103 | X86_BR_IND_CALL = 1 << 11,/* indirect calls */ |
| 104 | X86_BR_ABORT = 1 << 12,/* transaction abort */ |
| 105 | X86_BR_IN_TX = 1 << 13,/* in transaction */ |
| 106 | X86_BR_NO_TX = 1 << 14,/* not in transaction */ |
Yan, Zheng | aa54ae9 | 2014-11-04 21:56:11 -0500 | [diff] [blame] | 107 | X86_BR_ZERO_CALL = 1 << 15,/* zero length call */ |
| 108 | X86_BR_CALL_STACK = 1 << 16,/* call stack */ |
Stephane Eranian | 7b74cfb | 2015-05-14 23:09:59 +0200 | [diff] [blame] | 109 | X86_BR_IND_JMP = 1 << 17,/* indirect jump */ |
Stephane Eranian | 3e702ff | 2012-02-09 23:20:58 +0100 | [diff] [blame] | 110 | }; |
| 111 | |
| 112 | #define X86_BR_PLM (X86_BR_USER | X86_BR_KERNEL) |
Andi Kleen | 135c561 | 2013-06-17 17:36:51 -0700 | [diff] [blame] | 113 | #define X86_BR_ANYTX (X86_BR_NO_TX | X86_BR_IN_TX) |
Stephane Eranian | 3e702ff | 2012-02-09 23:20:58 +0100 | [diff] [blame] | 114 | |
| 115 | #define X86_BR_ANY \ |
| 116 | (X86_BR_CALL |\ |
| 117 | X86_BR_RET |\ |
| 118 | X86_BR_SYSCALL |\ |
| 119 | X86_BR_SYSRET |\ |
| 120 | X86_BR_INT |\ |
| 121 | X86_BR_IRET |\ |
| 122 | X86_BR_JCC |\ |
| 123 | X86_BR_JMP |\ |
| 124 | X86_BR_IRQ |\ |
Andi Kleen | 135c561 | 2013-06-17 17:36:51 -0700 | [diff] [blame] | 125 | X86_BR_ABORT |\ |
Yan, Zheng | aa54ae9 | 2014-11-04 21:56:11 -0500 | [diff] [blame] | 126 | X86_BR_IND_CALL |\ |
Stephane Eranian | 7b74cfb | 2015-05-14 23:09:59 +0200 | [diff] [blame] | 127 | X86_BR_IND_JMP |\ |
Yan, Zheng | aa54ae9 | 2014-11-04 21:56:11 -0500 | [diff] [blame] | 128 | X86_BR_ZERO_CALL) |
Stephane Eranian | 3e702ff | 2012-02-09 23:20:58 +0100 | [diff] [blame] | 129 | |
| 130 | #define X86_BR_ALL (X86_BR_PLM | X86_BR_ANY) |
| 131 | |
| 132 | #define X86_BR_ANY_CALL \ |
| 133 | (X86_BR_CALL |\ |
| 134 | X86_BR_IND_CALL |\ |
Yan, Zheng | aa54ae9 | 2014-11-04 21:56:11 -0500 | [diff] [blame] | 135 | X86_BR_ZERO_CALL |\ |
Stephane Eranian | 3e702ff | 2012-02-09 23:20:58 +0100 | [diff] [blame] | 136 | X86_BR_SYSCALL |\ |
| 137 | X86_BR_IRQ |\ |
| 138 | X86_BR_INT) |
| 139 | |
| 140 | static void intel_pmu_lbr_filter(struct cpu_hw_events *cpuc); |
| 141 | |
| 142 | /* |
Peter Zijlstra | caff2be | 2010-03-03 12:02:30 +0100 | [diff] [blame] | 143 | * We only support LBR implementations that have FREEZE_LBRS_ON_PMI |
| 144 | * otherwise it becomes near impossible to get a reliable stack. |
| 145 | */ |
| 146 | |
Andi Kleen | 1a78d93 | 2015-03-20 10:11:23 -0700 | [diff] [blame] | 147 | static void __intel_pmu_lbr_enable(bool pmi) |
Peter Zijlstra | caff2be | 2010-03-03 12:02:30 +0100 | [diff] [blame] | 148 | { |
Christoph Lameter | 89cbc76 | 2014-08-17 12:30:40 -0500 | [diff] [blame] | 149 | struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); |
Andi Kleen | cd1f11d | 2015-03-20 10:11:24 -0700 | [diff] [blame] | 150 | u64 debugctl, lbr_select = 0, orig_debugctl; |
Stephane Eranian | 60ce0fb | 2012-02-09 23:20:57 +0100 | [diff] [blame] | 151 | |
Andi Kleen | 1a78d93 | 2015-03-20 10:11:23 -0700 | [diff] [blame] | 152 | /* |
Andi Kleen | 425507f | 2015-05-10 12:22:46 -0700 | [diff] [blame] | 153 | * No need to unfreeze manually, as v4 can do that as part |
| 154 | * of the GLOBAL_STATUS ack. |
| 155 | */ |
| 156 | if (pmi && x86_pmu.version >= 4) |
| 157 | return; |
| 158 | |
| 159 | /* |
Andi Kleen | 1a78d93 | 2015-03-20 10:11:23 -0700 | [diff] [blame] | 160 | * No need to reprogram LBR_SELECT in a PMI, as it |
| 161 | * did not change. |
| 162 | */ |
Kan Liang | 96f3eda | 2015-09-14 10:14:07 -0400 | [diff] [blame] | 163 | if (cpuc->lbr_sel) |
Andi Kleen | b16a5b5 | 2015-10-20 11:46:34 -0700 | [diff] [blame] | 164 | lbr_select = cpuc->lbr_sel->config & x86_pmu.lbr_sel_mask; |
Stephane Eranian | 6fc2e83 | 2015-12-03 23:33:17 +0100 | [diff] [blame] | 165 | if (!pmi && cpuc->lbr_sel) |
Yan, Zheng | 2c70d00 | 2014-11-04 21:56:10 -0500 | [diff] [blame] | 166 | wrmsrl(MSR_LBR_SELECT, lbr_select); |
Peter Zijlstra | caff2be | 2010-03-03 12:02:30 +0100 | [diff] [blame] | 167 | |
| 168 | rdmsrl(MSR_IA32_DEBUGCTLMSR, debugctl); |
Andi Kleen | cd1f11d | 2015-03-20 10:11:24 -0700 | [diff] [blame] | 169 | orig_debugctl = debugctl; |
Yan, Zheng | 2c70d00 | 2014-11-04 21:56:10 -0500 | [diff] [blame] | 170 | debugctl |= DEBUGCTLMSR_LBR; |
| 171 | /* |
| 172 | * LBR callstack does not work well with FREEZE_LBRS_ON_PMI. |
| 173 | * If FREEZE_LBRS_ON_PMI is set, PMI near call/return instructions |
| 174 | * may cause superfluous increase/decrease of LBR_TOS. |
| 175 | */ |
| 176 | if (!(lbr_select & LBR_CALL_STACK)) |
| 177 | debugctl |= DEBUGCTLMSR_FREEZE_LBRS_ON_PMI; |
Andi Kleen | cd1f11d | 2015-03-20 10:11:24 -0700 | [diff] [blame] | 178 | if (orig_debugctl != debugctl) |
| 179 | wrmsrl(MSR_IA32_DEBUGCTLMSR, debugctl); |
Peter Zijlstra | caff2be | 2010-03-03 12:02:30 +0100 | [diff] [blame] | 180 | } |
| 181 | |
| 182 | static void __intel_pmu_lbr_disable(void) |
| 183 | { |
| 184 | u64 debugctl; |
| 185 | |
| 186 | rdmsrl(MSR_IA32_DEBUGCTLMSR, debugctl); |
Peter Zijlstra | 7c5ecaf | 2010-03-25 14:51:49 +0100 | [diff] [blame] | 187 | debugctl &= ~(DEBUGCTLMSR_LBR | DEBUGCTLMSR_FREEZE_LBRS_ON_PMI); |
Peter Zijlstra | caff2be | 2010-03-03 12:02:30 +0100 | [diff] [blame] | 188 | wrmsrl(MSR_IA32_DEBUGCTLMSR, debugctl); |
| 189 | } |
| 190 | |
| 191 | static void intel_pmu_lbr_reset_32(void) |
| 192 | { |
| 193 | int i; |
| 194 | |
| 195 | for (i = 0; i < x86_pmu.lbr_nr; i++) |
| 196 | wrmsrl(x86_pmu.lbr_from + i, 0); |
| 197 | } |
| 198 | |
| 199 | static void intel_pmu_lbr_reset_64(void) |
| 200 | { |
| 201 | int i; |
| 202 | |
| 203 | for (i = 0; i < x86_pmu.lbr_nr; i++) { |
| 204 | wrmsrl(x86_pmu.lbr_from + i, 0); |
| 205 | wrmsrl(x86_pmu.lbr_to + i, 0); |
Andi Kleen | 50eab8f | 2015-05-10 12:22:43 -0700 | [diff] [blame] | 206 | if (x86_pmu.intel_cap.lbr_format == LBR_FORMAT_INFO) |
| 207 | wrmsrl(MSR_LBR_INFO_0 + i, 0); |
Peter Zijlstra | caff2be | 2010-03-03 12:02:30 +0100 | [diff] [blame] | 208 | } |
| 209 | } |
| 210 | |
Kevin Winchester | de0428a | 2011-08-30 20:41:05 -0300 | [diff] [blame] | 211 | void intel_pmu_lbr_reset(void) |
Peter Zijlstra | caff2be | 2010-03-03 12:02:30 +0100 | [diff] [blame] | 212 | { |
Peter Zijlstra | 74846d3 | 2010-03-05 13:49:35 +0100 | [diff] [blame] | 213 | if (!x86_pmu.lbr_nr) |
| 214 | return; |
| 215 | |
Peter Zijlstra | 8db909a | 2010-03-03 17:07:40 +0100 | [diff] [blame] | 216 | if (x86_pmu.intel_cap.lbr_format == LBR_FORMAT_32) |
Peter Zijlstra | caff2be | 2010-03-03 12:02:30 +0100 | [diff] [blame] | 217 | intel_pmu_lbr_reset_32(); |
| 218 | else |
| 219 | intel_pmu_lbr_reset_64(); |
| 220 | } |
| 221 | |
Yan, Zheng | 76cb2c6 | 2014-11-04 21:56:05 -0500 | [diff] [blame] | 222 | /* |
| 223 | * TOS = most recently recorded branch |
| 224 | */ |
| 225 | static inline u64 intel_pmu_lbr_tos(void) |
| 226 | { |
| 227 | u64 tos; |
| 228 | |
| 229 | rdmsrl(x86_pmu.lbr_tos, tos); |
| 230 | return tos; |
| 231 | } |
| 232 | |
| 233 | enum { |
| 234 | LBR_NONE, |
| 235 | LBR_VALID, |
| 236 | }; |
| 237 | |
| 238 | static void __intel_pmu_lbr_restore(struct x86_perf_task_context *task_ctx) |
| 239 | { |
| 240 | int i; |
| 241 | unsigned lbr_idx, mask; |
| 242 | u64 tos; |
| 243 | |
| 244 | if (task_ctx->lbr_callstack_users == 0 || |
| 245 | task_ctx->lbr_stack_state == LBR_NONE) { |
| 246 | intel_pmu_lbr_reset(); |
| 247 | return; |
| 248 | } |
| 249 | |
| 250 | mask = x86_pmu.lbr_nr - 1; |
Andi Kleen | b28ae95 | 2015-10-20 11:46:33 -0700 | [diff] [blame] | 251 | tos = task_ctx->tos; |
Andi Kleen | 90405aa | 2015-05-27 21:13:18 -0700 | [diff] [blame] | 252 | for (i = 0; i < tos; i++) { |
Yan, Zheng | 76cb2c6 | 2014-11-04 21:56:05 -0500 | [diff] [blame] | 253 | lbr_idx = (tos - i) & mask; |
| 254 | wrmsrl(x86_pmu.lbr_from + lbr_idx, task_ctx->lbr_from[i]); |
| 255 | wrmsrl(x86_pmu.lbr_to + lbr_idx, task_ctx->lbr_to[i]); |
Andi Kleen | 50eab8f | 2015-05-10 12:22:43 -0700 | [diff] [blame] | 256 | if (x86_pmu.intel_cap.lbr_format == LBR_FORMAT_INFO) |
Andi Kleen | e057336 | 2015-05-27 21:13:17 -0700 | [diff] [blame] | 257 | wrmsrl(MSR_LBR_INFO_0 + lbr_idx, task_ctx->lbr_info[i]); |
Yan, Zheng | 76cb2c6 | 2014-11-04 21:56:05 -0500 | [diff] [blame] | 258 | } |
Andi Kleen | b28ae95 | 2015-10-20 11:46:33 -0700 | [diff] [blame] | 259 | wrmsrl(x86_pmu.lbr_tos, tos); |
Yan, Zheng | 76cb2c6 | 2014-11-04 21:56:05 -0500 | [diff] [blame] | 260 | task_ctx->lbr_stack_state = LBR_NONE; |
| 261 | } |
| 262 | |
| 263 | static void __intel_pmu_lbr_save(struct x86_perf_task_context *task_ctx) |
| 264 | { |
| 265 | int i; |
| 266 | unsigned lbr_idx, mask; |
| 267 | u64 tos; |
| 268 | |
| 269 | if (task_ctx->lbr_callstack_users == 0) { |
| 270 | task_ctx->lbr_stack_state = LBR_NONE; |
| 271 | return; |
| 272 | } |
| 273 | |
| 274 | mask = x86_pmu.lbr_nr - 1; |
| 275 | tos = intel_pmu_lbr_tos(); |
Andi Kleen | 90405aa | 2015-05-27 21:13:18 -0700 | [diff] [blame] | 276 | for (i = 0; i < tos; i++) { |
Yan, Zheng | 76cb2c6 | 2014-11-04 21:56:05 -0500 | [diff] [blame] | 277 | lbr_idx = (tos - i) & mask; |
| 278 | rdmsrl(x86_pmu.lbr_from + lbr_idx, task_ctx->lbr_from[i]); |
| 279 | rdmsrl(x86_pmu.lbr_to + lbr_idx, task_ctx->lbr_to[i]); |
Andi Kleen | 50eab8f | 2015-05-10 12:22:43 -0700 | [diff] [blame] | 280 | if (x86_pmu.intel_cap.lbr_format == LBR_FORMAT_INFO) |
Andi Kleen | e057336 | 2015-05-27 21:13:17 -0700 | [diff] [blame] | 281 | rdmsrl(MSR_LBR_INFO_0 + lbr_idx, task_ctx->lbr_info[i]); |
Yan, Zheng | 76cb2c6 | 2014-11-04 21:56:05 -0500 | [diff] [blame] | 282 | } |
Andi Kleen | b28ae95 | 2015-10-20 11:46:33 -0700 | [diff] [blame] | 283 | task_ctx->tos = tos; |
Yan, Zheng | 76cb2c6 | 2014-11-04 21:56:05 -0500 | [diff] [blame] | 284 | task_ctx->lbr_stack_state = LBR_VALID; |
| 285 | } |
| 286 | |
Yan, Zheng | 2a0ad3b | 2014-11-04 21:55:59 -0500 | [diff] [blame] | 287 | void intel_pmu_lbr_sched_task(struct perf_event_context *ctx, bool sched_in) |
| 288 | { |
| 289 | struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); |
Yan, Zheng | 76cb2c6 | 2014-11-04 21:56:05 -0500 | [diff] [blame] | 290 | struct x86_perf_task_context *task_ctx; |
Yan, Zheng | 2a0ad3b | 2014-11-04 21:55:59 -0500 | [diff] [blame] | 291 | |
Yan, Zheng | 2a0ad3b | 2014-11-04 21:55:59 -0500 | [diff] [blame] | 292 | /* |
Yan, Zheng | 76cb2c6 | 2014-11-04 21:56:05 -0500 | [diff] [blame] | 293 | * If LBR callstack feature is enabled and the stack was saved when |
| 294 | * the task was scheduled out, restore the stack. Otherwise flush |
| 295 | * the LBR stack. |
| 296 | */ |
| 297 | task_ctx = ctx ? ctx->task_ctx_data : NULL; |
| 298 | if (task_ctx) { |
| 299 | if (sched_in) { |
| 300 | __intel_pmu_lbr_restore(task_ctx); |
| 301 | cpuc->lbr_context = ctx; |
| 302 | } else { |
| 303 | __intel_pmu_lbr_save(task_ctx); |
| 304 | } |
| 305 | return; |
| 306 | } |
| 307 | |
| 308 | /* |
Yan, Zheng | 2a0ad3b | 2014-11-04 21:55:59 -0500 | [diff] [blame] | 309 | * When sampling the branck stack in system-wide, it may be |
| 310 | * necessary to flush the stack on context switch. This happens |
| 311 | * when the branch stack does not tag its entries with the pid |
| 312 | * of the current task. Otherwise it becomes impossible to |
| 313 | * associate a branch entry with a task. This ambiguity is more |
| 314 | * likely to appear when the branch stack supports priv level |
| 315 | * filtering and the user sets it to monitor only at the user |
| 316 | * level (which could be a useful measurement in system-wide |
| 317 | * mode). In that case, the risk is high of having a branch |
| 318 | * stack with branch from multiple tasks. |
| 319 | */ |
| 320 | if (sched_in) { |
| 321 | intel_pmu_lbr_reset(); |
| 322 | cpuc->lbr_context = ctx; |
| 323 | } |
| 324 | } |
| 325 | |
Yan, Zheng | 63f0c1d | 2014-11-04 21:56:04 -0500 | [diff] [blame] | 326 | static inline bool branch_user_callstack(unsigned br_sel) |
| 327 | { |
| 328 | return (br_sel & X86_BR_USER) && (br_sel & X86_BR_CALL_STACK); |
| 329 | } |
| 330 | |
Kevin Winchester | de0428a | 2011-08-30 20:41:05 -0300 | [diff] [blame] | 331 | void intel_pmu_lbr_enable(struct perf_event *event) |
Peter Zijlstra | caff2be | 2010-03-03 12:02:30 +0100 | [diff] [blame] | 332 | { |
Christoph Lameter | 89cbc76 | 2014-08-17 12:30:40 -0500 | [diff] [blame] | 333 | struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); |
Yan, Zheng | 63f0c1d | 2014-11-04 21:56:04 -0500 | [diff] [blame] | 334 | struct x86_perf_task_context *task_ctx; |
Peter Zijlstra | caff2be | 2010-03-03 12:02:30 +0100 | [diff] [blame] | 335 | |
| 336 | if (!x86_pmu.lbr_nr) |
| 337 | return; |
| 338 | |
Peter Zijlstra | caff2be | 2010-03-03 12:02:30 +0100 | [diff] [blame] | 339 | /* |
Peter Zijlstra | b83a46e | 2010-03-08 13:51:12 +0100 | [diff] [blame] | 340 | * Reset the LBR stack if we changed task context to |
| 341 | * avoid data leaks. |
Peter Zijlstra | caff2be | 2010-03-03 12:02:30 +0100 | [diff] [blame] | 342 | */ |
Peter Zijlstra | b83a46e | 2010-03-08 13:51:12 +0100 | [diff] [blame] | 343 | if (event->ctx->task && cpuc->lbr_context != event->ctx) { |
Peter Zijlstra | caff2be | 2010-03-03 12:02:30 +0100 | [diff] [blame] | 344 | intel_pmu_lbr_reset(); |
| 345 | cpuc->lbr_context = event->ctx; |
| 346 | } |
Stephane Eranian | 3e702ff | 2012-02-09 23:20:58 +0100 | [diff] [blame] | 347 | cpuc->br_sel = event->hw.branch_reg.reg; |
Peter Zijlstra | caff2be | 2010-03-03 12:02:30 +0100 | [diff] [blame] | 348 | |
Yan, Zheng | 63f0c1d | 2014-11-04 21:56:04 -0500 | [diff] [blame] | 349 | if (branch_user_callstack(cpuc->br_sel) && event->ctx && |
| 350 | event->ctx->task_ctx_data) { |
| 351 | task_ctx = event->ctx->task_ctx_data; |
| 352 | task_ctx->lbr_callstack_users++; |
| 353 | } |
| 354 | |
Peter Zijlstra | caff2be | 2010-03-03 12:02:30 +0100 | [diff] [blame] | 355 | cpuc->lbr_users++; |
Yan, Zheng | 2a0ad3b | 2014-11-04 21:55:59 -0500 | [diff] [blame] | 356 | perf_sched_cb_inc(event->ctx->pmu); |
Peter Zijlstra | caff2be | 2010-03-03 12:02:30 +0100 | [diff] [blame] | 357 | } |
| 358 | |
Kevin Winchester | de0428a | 2011-08-30 20:41:05 -0300 | [diff] [blame] | 359 | void intel_pmu_lbr_disable(struct perf_event *event) |
Peter Zijlstra | caff2be | 2010-03-03 12:02:30 +0100 | [diff] [blame] | 360 | { |
Christoph Lameter | 89cbc76 | 2014-08-17 12:30:40 -0500 | [diff] [blame] | 361 | struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); |
Yan, Zheng | 63f0c1d | 2014-11-04 21:56:04 -0500 | [diff] [blame] | 362 | struct x86_perf_task_context *task_ctx; |
Peter Zijlstra | caff2be | 2010-03-03 12:02:30 +0100 | [diff] [blame] | 363 | |
| 364 | if (!x86_pmu.lbr_nr) |
| 365 | return; |
| 366 | |
Yan, Zheng | 63f0c1d | 2014-11-04 21:56:04 -0500 | [diff] [blame] | 367 | if (branch_user_callstack(cpuc->br_sel) && event->ctx && |
| 368 | event->ctx->task_ctx_data) { |
| 369 | task_ctx = event->ctx->task_ctx_data; |
| 370 | task_ctx->lbr_callstack_users--; |
| 371 | } |
| 372 | |
Peter Zijlstra | caff2be | 2010-03-03 12:02:30 +0100 | [diff] [blame] | 373 | cpuc->lbr_users--; |
Peter Zijlstra | b83a46e | 2010-03-08 13:51:12 +0100 | [diff] [blame] | 374 | WARN_ON_ONCE(cpuc->lbr_users < 0); |
Yan, Zheng | 2a0ad3b | 2014-11-04 21:55:59 -0500 | [diff] [blame] | 375 | perf_sched_cb_dec(event->ctx->pmu); |
Peter Zijlstra | 2df202b | 2010-03-06 13:48:54 +0100 | [diff] [blame] | 376 | |
Stephane Eranian | 60ce0fb | 2012-02-09 23:20:57 +0100 | [diff] [blame] | 377 | if (cpuc->enabled && !cpuc->lbr_users) { |
Peter Zijlstra | 2df202b | 2010-03-06 13:48:54 +0100 | [diff] [blame] | 378 | __intel_pmu_lbr_disable(); |
Stephane Eranian | 60ce0fb | 2012-02-09 23:20:57 +0100 | [diff] [blame] | 379 | /* avoid stale pointer */ |
| 380 | cpuc->lbr_context = NULL; |
| 381 | } |
Peter Zijlstra | caff2be | 2010-03-03 12:02:30 +0100 | [diff] [blame] | 382 | } |
| 383 | |
Andi Kleen | 1a78d93 | 2015-03-20 10:11:23 -0700 | [diff] [blame] | 384 | void intel_pmu_lbr_enable_all(bool pmi) |
Peter Zijlstra | caff2be | 2010-03-03 12:02:30 +0100 | [diff] [blame] | 385 | { |
Christoph Lameter | 89cbc76 | 2014-08-17 12:30:40 -0500 | [diff] [blame] | 386 | struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); |
Peter Zijlstra | caff2be | 2010-03-03 12:02:30 +0100 | [diff] [blame] | 387 | |
| 388 | if (cpuc->lbr_users) |
Andi Kleen | 1a78d93 | 2015-03-20 10:11:23 -0700 | [diff] [blame] | 389 | __intel_pmu_lbr_enable(pmi); |
Peter Zijlstra | caff2be | 2010-03-03 12:02:30 +0100 | [diff] [blame] | 390 | } |
| 391 | |
Kevin Winchester | de0428a | 2011-08-30 20:41:05 -0300 | [diff] [blame] | 392 | void intel_pmu_lbr_disable_all(void) |
Peter Zijlstra | caff2be | 2010-03-03 12:02:30 +0100 | [diff] [blame] | 393 | { |
Christoph Lameter | 89cbc76 | 2014-08-17 12:30:40 -0500 | [diff] [blame] | 394 | struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); |
Peter Zijlstra | caff2be | 2010-03-03 12:02:30 +0100 | [diff] [blame] | 395 | |
| 396 | if (cpuc->lbr_users) |
| 397 | __intel_pmu_lbr_disable(); |
| 398 | } |
| 399 | |
Peter Zijlstra | caff2be | 2010-03-03 12:02:30 +0100 | [diff] [blame] | 400 | static void intel_pmu_lbr_read_32(struct cpu_hw_events *cpuc) |
| 401 | { |
| 402 | unsigned long mask = x86_pmu.lbr_nr - 1; |
| 403 | u64 tos = intel_pmu_lbr_tos(); |
| 404 | int i; |
| 405 | |
Peter Zijlstra | 63fb3f9 | 2010-03-09 11:51:02 +0100 | [diff] [blame] | 406 | for (i = 0; i < x86_pmu.lbr_nr; i++) { |
Peter Zijlstra | caff2be | 2010-03-03 12:02:30 +0100 | [diff] [blame] | 407 | unsigned long lbr_idx = (tos - i) & mask; |
| 408 | union { |
| 409 | struct { |
| 410 | u32 from; |
| 411 | u32 to; |
| 412 | }; |
| 413 | u64 lbr; |
| 414 | } msr_lastbranch; |
| 415 | |
| 416 | rdmsrl(x86_pmu.lbr_from + lbr_idx, msr_lastbranch.lbr); |
| 417 | |
Stephane Eranian | bce38cd | 2012-02-09 23:20:51 +0100 | [diff] [blame] | 418 | cpuc->lbr_entries[i].from = msr_lastbranch.from; |
| 419 | cpuc->lbr_entries[i].to = msr_lastbranch.to; |
| 420 | cpuc->lbr_entries[i].mispred = 0; |
| 421 | cpuc->lbr_entries[i].predicted = 0; |
| 422 | cpuc->lbr_entries[i].reserved = 0; |
Peter Zijlstra | caff2be | 2010-03-03 12:02:30 +0100 | [diff] [blame] | 423 | } |
| 424 | cpuc->lbr_stack.nr = i; |
| 425 | } |
| 426 | |
Peter Zijlstra | caff2be | 2010-03-03 12:02:30 +0100 | [diff] [blame] | 427 | /* |
| 428 | * Due to lack of segmentation in Linux the effective address (offset) |
| 429 | * is the same as the linear address, allowing us to merge the LIP and EIP |
| 430 | * LBR formats. |
| 431 | */ |
| 432 | static void intel_pmu_lbr_read_64(struct cpu_hw_events *cpuc) |
| 433 | { |
Stephane Eranian | 6fc2e83 | 2015-12-03 23:33:17 +0100 | [diff] [blame] | 434 | bool need_info = false; |
Peter Zijlstra | caff2be | 2010-03-03 12:02:30 +0100 | [diff] [blame] | 435 | unsigned long mask = x86_pmu.lbr_nr - 1; |
Peter Zijlstra | 8db909a | 2010-03-03 17:07:40 +0100 | [diff] [blame] | 436 | int lbr_format = x86_pmu.intel_cap.lbr_format; |
Peter Zijlstra | caff2be | 2010-03-03 12:02:30 +0100 | [diff] [blame] | 437 | u64 tos = intel_pmu_lbr_tos(); |
| 438 | int i; |
Andi Kleen | b7af41a | 2013-09-20 07:40:44 -0700 | [diff] [blame] | 439 | int out = 0; |
Andi Kleen | 90405aa | 2015-05-27 21:13:18 -0700 | [diff] [blame] | 440 | int num = x86_pmu.lbr_nr; |
Peter Zijlstra | caff2be | 2010-03-03 12:02:30 +0100 | [diff] [blame] | 441 | |
Stephane Eranian | 6fc2e83 | 2015-12-03 23:33:17 +0100 | [diff] [blame] | 442 | if (cpuc->lbr_sel) { |
| 443 | need_info = !(cpuc->lbr_sel->config & LBR_NO_INFO); |
| 444 | if (cpuc->lbr_sel->config & LBR_CALL_STACK) |
| 445 | num = tos; |
| 446 | } |
Andi Kleen | 90405aa | 2015-05-27 21:13:18 -0700 | [diff] [blame] | 447 | |
| 448 | for (i = 0; i < num; i++) { |
Peter Zijlstra | caff2be | 2010-03-03 12:02:30 +0100 | [diff] [blame] | 449 | unsigned long lbr_idx = (tos - i) & mask; |
Andi Kleen | 135c561 | 2013-06-17 17:36:51 -0700 | [diff] [blame] | 450 | u64 from, to, mis = 0, pred = 0, in_tx = 0, abort = 0; |
| 451 | int skip = 0; |
Andi Kleen | 50eab8f | 2015-05-10 12:22:43 -0700 | [diff] [blame] | 452 | u16 cycles = 0; |
Andi Kleen | 135c561 | 2013-06-17 17:36:51 -0700 | [diff] [blame] | 453 | int lbr_flags = lbr_desc[lbr_format]; |
Peter Zijlstra | caff2be | 2010-03-03 12:02:30 +0100 | [diff] [blame] | 454 | |
| 455 | rdmsrl(x86_pmu.lbr_from + lbr_idx, from); |
| 456 | rdmsrl(x86_pmu.lbr_to + lbr_idx, to); |
| 457 | |
Andi Kleen | b16a5b5 | 2015-10-20 11:46:34 -0700 | [diff] [blame] | 458 | if (lbr_format == LBR_FORMAT_INFO && need_info) { |
Andi Kleen | 50eab8f | 2015-05-10 12:22:43 -0700 | [diff] [blame] | 459 | u64 info; |
| 460 | |
| 461 | rdmsrl(MSR_LBR_INFO_0 + lbr_idx, info); |
| 462 | mis = !!(info & LBR_INFO_MISPRED); |
| 463 | pred = !mis; |
| 464 | in_tx = !!(info & LBR_INFO_IN_TX); |
| 465 | abort = !!(info & LBR_INFO_ABORT); |
| 466 | cycles = (info & LBR_INFO_CYCLES); |
| 467 | } |
Kan Liang | 8b92c3a | 2016-04-15 00:42:47 -0700 | [diff] [blame] | 468 | |
| 469 | if (lbr_format == LBR_FORMAT_TIME) { |
| 470 | mis = !!(from & LBR_FROM_FLAG_MISPRED); |
| 471 | pred = !mis; |
| 472 | skip = 1; |
| 473 | cycles = ((to >> 48) & LBR_INFO_CYCLES); |
| 474 | |
| 475 | to = (u64)((((s64)to) << 16) >> 16); |
| 476 | } |
| 477 | |
Andi Kleen | 135c561 | 2013-06-17 17:36:51 -0700 | [diff] [blame] | 478 | if (lbr_flags & LBR_EIP_FLAGS) { |
Stephane Eranian | bce38cd | 2012-02-09 23:20:51 +0100 | [diff] [blame] | 479 | mis = !!(from & LBR_FROM_FLAG_MISPRED); |
| 480 | pred = !mis; |
Andi Kleen | 135c561 | 2013-06-17 17:36:51 -0700 | [diff] [blame] | 481 | skip = 1; |
Peter Zijlstra | caff2be | 2010-03-03 12:02:30 +0100 | [diff] [blame] | 482 | } |
Andi Kleen | 135c561 | 2013-06-17 17:36:51 -0700 | [diff] [blame] | 483 | if (lbr_flags & LBR_TSX) { |
| 484 | in_tx = !!(from & LBR_FROM_FLAG_IN_TX); |
| 485 | abort = !!(from & LBR_FROM_FLAG_ABORT); |
| 486 | skip = 3; |
| 487 | } |
| 488 | from = (u64)((((s64)from) << skip) >> skip); |
Peter Zijlstra | caff2be | 2010-03-03 12:02:30 +0100 | [diff] [blame] | 489 | |
Andi Kleen | b7af41a | 2013-09-20 07:40:44 -0700 | [diff] [blame] | 490 | /* |
| 491 | * Some CPUs report duplicated abort records, |
| 492 | * with the second entry not having an abort bit set. |
| 493 | * Skip them here. This loop runs backwards, |
| 494 | * so we need to undo the previous record. |
| 495 | * If the abort just happened outside the window |
| 496 | * the extra entry cannot be removed. |
| 497 | */ |
| 498 | if (abort && x86_pmu.lbr_double_abort && out > 0) |
| 499 | out--; |
| 500 | |
| 501 | cpuc->lbr_entries[out].from = from; |
| 502 | cpuc->lbr_entries[out].to = to; |
| 503 | cpuc->lbr_entries[out].mispred = mis; |
| 504 | cpuc->lbr_entries[out].predicted = pred; |
| 505 | cpuc->lbr_entries[out].in_tx = in_tx; |
| 506 | cpuc->lbr_entries[out].abort = abort; |
Andi Kleen | 50eab8f | 2015-05-10 12:22:43 -0700 | [diff] [blame] | 507 | cpuc->lbr_entries[out].cycles = cycles; |
Andi Kleen | b7af41a | 2013-09-20 07:40:44 -0700 | [diff] [blame] | 508 | cpuc->lbr_entries[out].reserved = 0; |
| 509 | out++; |
Peter Zijlstra | caff2be | 2010-03-03 12:02:30 +0100 | [diff] [blame] | 510 | } |
Andi Kleen | b7af41a | 2013-09-20 07:40:44 -0700 | [diff] [blame] | 511 | cpuc->lbr_stack.nr = out; |
Peter Zijlstra | caff2be | 2010-03-03 12:02:30 +0100 | [diff] [blame] | 512 | } |
| 513 | |
Kevin Winchester | de0428a | 2011-08-30 20:41:05 -0300 | [diff] [blame] | 514 | void intel_pmu_lbr_read(void) |
Peter Zijlstra | caff2be | 2010-03-03 12:02:30 +0100 | [diff] [blame] | 515 | { |
Christoph Lameter | 89cbc76 | 2014-08-17 12:30:40 -0500 | [diff] [blame] | 516 | struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); |
Peter Zijlstra | caff2be | 2010-03-03 12:02:30 +0100 | [diff] [blame] | 517 | |
| 518 | if (!cpuc->lbr_users) |
| 519 | return; |
| 520 | |
Peter Zijlstra | 8db909a | 2010-03-03 17:07:40 +0100 | [diff] [blame] | 521 | if (x86_pmu.intel_cap.lbr_format == LBR_FORMAT_32) |
Peter Zijlstra | caff2be | 2010-03-03 12:02:30 +0100 | [diff] [blame] | 522 | intel_pmu_lbr_read_32(cpuc); |
| 523 | else |
| 524 | intel_pmu_lbr_read_64(cpuc); |
Stephane Eranian | 3e702ff | 2012-02-09 23:20:58 +0100 | [diff] [blame] | 525 | |
| 526 | intel_pmu_lbr_filter(cpuc); |
| 527 | } |
| 528 | |
| 529 | /* |
| 530 | * SW filter is used: |
| 531 | * - in case there is no HW filter |
| 532 | * - in case the HW filter has errata or limitations |
| 533 | */ |
Yan, Zheng | e9d7f7c | 2014-11-04 21:56:00 -0500 | [diff] [blame] | 534 | static int intel_pmu_setup_sw_lbr_filter(struct perf_event *event) |
Stephane Eranian | 3e702ff | 2012-02-09 23:20:58 +0100 | [diff] [blame] | 535 | { |
| 536 | u64 br_type = event->attr.branch_sample_type; |
| 537 | int mask = 0; |
| 538 | |
| 539 | if (br_type & PERF_SAMPLE_BRANCH_USER) |
| 540 | mask |= X86_BR_USER; |
| 541 | |
Stephane Eranian | 2b923c8 | 2013-05-21 12:53:37 +0200 | [diff] [blame] | 542 | if (br_type & PERF_SAMPLE_BRANCH_KERNEL) |
Stephane Eranian | 3e702ff | 2012-02-09 23:20:58 +0100 | [diff] [blame] | 543 | mask |= X86_BR_KERNEL; |
| 544 | |
| 545 | /* we ignore BRANCH_HV here */ |
| 546 | |
| 547 | if (br_type & PERF_SAMPLE_BRANCH_ANY) |
| 548 | mask |= X86_BR_ANY; |
| 549 | |
| 550 | if (br_type & PERF_SAMPLE_BRANCH_ANY_CALL) |
| 551 | mask |= X86_BR_ANY_CALL; |
| 552 | |
| 553 | if (br_type & PERF_SAMPLE_BRANCH_ANY_RETURN) |
| 554 | mask |= X86_BR_RET | X86_BR_IRET | X86_BR_SYSRET; |
| 555 | |
| 556 | if (br_type & PERF_SAMPLE_BRANCH_IND_CALL) |
| 557 | mask |= X86_BR_IND_CALL; |
Andi Kleen | 135c561 | 2013-06-17 17:36:51 -0700 | [diff] [blame] | 558 | |
| 559 | if (br_type & PERF_SAMPLE_BRANCH_ABORT_TX) |
| 560 | mask |= X86_BR_ABORT; |
| 561 | |
| 562 | if (br_type & PERF_SAMPLE_BRANCH_IN_TX) |
| 563 | mask |= X86_BR_IN_TX; |
| 564 | |
| 565 | if (br_type & PERF_SAMPLE_BRANCH_NO_TX) |
| 566 | mask |= X86_BR_NO_TX; |
| 567 | |
Anshuman Khandual | 3754891 | 2014-05-22 12:50:09 +0530 | [diff] [blame] | 568 | if (br_type & PERF_SAMPLE_BRANCH_COND) |
| 569 | mask |= X86_BR_JCC; |
| 570 | |
Yan, Zheng | e9d7f7c | 2014-11-04 21:56:00 -0500 | [diff] [blame] | 571 | if (br_type & PERF_SAMPLE_BRANCH_CALL_STACK) { |
| 572 | if (!x86_pmu_has_lbr_callstack()) |
| 573 | return -EOPNOTSUPP; |
| 574 | if (mask & ~(X86_BR_USER | X86_BR_KERNEL)) |
| 575 | return -EINVAL; |
| 576 | mask |= X86_BR_CALL | X86_BR_IND_CALL | X86_BR_RET | |
| 577 | X86_BR_CALL_STACK; |
| 578 | } |
| 579 | |
Stephane Eranian | 7b74cfb | 2015-05-14 23:09:59 +0200 | [diff] [blame] | 580 | if (br_type & PERF_SAMPLE_BRANCH_IND_JUMP) |
| 581 | mask |= X86_BR_IND_JMP; |
| 582 | |
Stephane Eranian | d892819 | 2015-10-13 09:09:09 +0200 | [diff] [blame] | 583 | if (br_type & PERF_SAMPLE_BRANCH_CALL) |
| 584 | mask |= X86_BR_CALL | X86_BR_ZERO_CALL; |
Stephane Eranian | 3e702ff | 2012-02-09 23:20:58 +0100 | [diff] [blame] | 585 | /* |
| 586 | * stash actual user request into reg, it may |
| 587 | * be used by fixup code for some CPU |
| 588 | */ |
| 589 | event->hw.branch_reg.reg = mask; |
Yan, Zheng | e9d7f7c | 2014-11-04 21:56:00 -0500 | [diff] [blame] | 590 | return 0; |
Peter Zijlstra | caff2be | 2010-03-03 12:02:30 +0100 | [diff] [blame] | 591 | } |
| 592 | |
Stephane Eranian | c5cc2cd | 2012-02-09 23:20:55 +0100 | [diff] [blame] | 593 | /* |
Stephane Eranian | 60ce0fb | 2012-02-09 23:20:57 +0100 | [diff] [blame] | 594 | * setup the HW LBR filter |
| 595 | * Used only when available, may not be enough to disambiguate |
| 596 | * all branches, may need the help of the SW filter |
| 597 | */ |
| 598 | static int intel_pmu_setup_hw_lbr_filter(struct perf_event *event) |
| 599 | { |
| 600 | struct hw_perf_event_extra *reg; |
| 601 | u64 br_type = event->attr.branch_sample_type; |
Yan, Zheng | 27ac905 | 2014-11-04 21:55:57 -0500 | [diff] [blame] | 602 | u64 mask = 0, v; |
| 603 | int i; |
Stephane Eranian | 60ce0fb | 2012-02-09 23:20:57 +0100 | [diff] [blame] | 604 | |
Peter Zijlstra | 2c44b19 | 2014-11-05 10:36:45 +0100 | [diff] [blame] | 605 | for (i = 0; i < PERF_SAMPLE_BRANCH_MAX_SHIFT; i++) { |
Yan, Zheng | 27ac905 | 2014-11-04 21:55:57 -0500 | [diff] [blame] | 606 | if (!(br_type & (1ULL << i))) |
Stephane Eranian | 60ce0fb | 2012-02-09 23:20:57 +0100 | [diff] [blame] | 607 | continue; |
| 608 | |
Yan, Zheng | 27ac905 | 2014-11-04 21:55:57 -0500 | [diff] [blame] | 609 | v = x86_pmu.lbr_sel_map[i]; |
Stephane Eranian | 60ce0fb | 2012-02-09 23:20:57 +0100 | [diff] [blame] | 610 | if (v == LBR_NOT_SUPP) |
| 611 | return -EOPNOTSUPP; |
Stephane Eranian | 60ce0fb | 2012-02-09 23:20:57 +0100 | [diff] [blame] | 612 | |
Stephane Eranian | 3e702ff | 2012-02-09 23:20:58 +0100 | [diff] [blame] | 613 | if (v != LBR_IGN) |
| 614 | mask |= v; |
Stephane Eranian | 60ce0fb | 2012-02-09 23:20:57 +0100 | [diff] [blame] | 615 | } |
Andi Kleen | b16a5b5 | 2015-10-20 11:46:34 -0700 | [diff] [blame] | 616 | |
Stephane Eranian | 60ce0fb | 2012-02-09 23:20:57 +0100 | [diff] [blame] | 617 | reg = &event->hw.branch_reg; |
| 618 | reg->idx = EXTRA_REG_LBR; |
| 619 | |
Yan, Zheng | e9d7f7c | 2014-11-04 21:56:00 -0500 | [diff] [blame] | 620 | /* |
| 621 | * The first 9 bits (LBR_SEL_MASK) in LBR_SELECT operate |
| 622 | * in suppress mode. So LBR_SELECT should be set to |
| 623 | * (~mask & LBR_SEL_MASK) | (mask & ~LBR_SEL_MASK) |
| 624 | */ |
| 625 | reg->config = mask ^ x86_pmu.lbr_sel_mask; |
Stephane Eranian | 60ce0fb | 2012-02-09 23:20:57 +0100 | [diff] [blame] | 626 | |
Andi Kleen | b16a5b5 | 2015-10-20 11:46:34 -0700 | [diff] [blame] | 627 | if ((br_type & PERF_SAMPLE_BRANCH_NO_CYCLES) && |
| 628 | (br_type & PERF_SAMPLE_BRANCH_NO_FLAGS) && |
| 629 | (x86_pmu.intel_cap.lbr_format == LBR_FORMAT_INFO)) |
| 630 | reg->config |= LBR_NO_INFO; |
| 631 | |
Stephane Eranian | 60ce0fb | 2012-02-09 23:20:57 +0100 | [diff] [blame] | 632 | return 0; |
| 633 | } |
| 634 | |
Stephane Eranian | 60ce0fb | 2012-02-09 23:20:57 +0100 | [diff] [blame] | 635 | int intel_pmu_setup_lbr_filter(struct perf_event *event) |
| 636 | { |
Stephane Eranian | 3e702ff | 2012-02-09 23:20:58 +0100 | [diff] [blame] | 637 | int ret = 0; |
Stephane Eranian | 60ce0fb | 2012-02-09 23:20:57 +0100 | [diff] [blame] | 638 | |
| 639 | /* |
| 640 | * no LBR on this PMU |
| 641 | */ |
| 642 | if (!x86_pmu.lbr_nr) |
| 643 | return -EOPNOTSUPP; |
| 644 | |
| 645 | /* |
Stephane Eranian | 3e702ff | 2012-02-09 23:20:58 +0100 | [diff] [blame] | 646 | * setup SW LBR filter |
Stephane Eranian | 60ce0fb | 2012-02-09 23:20:57 +0100 | [diff] [blame] | 647 | */ |
Yan, Zheng | e9d7f7c | 2014-11-04 21:56:00 -0500 | [diff] [blame] | 648 | ret = intel_pmu_setup_sw_lbr_filter(event); |
| 649 | if (ret) |
| 650 | return ret; |
Stephane Eranian | 3e702ff | 2012-02-09 23:20:58 +0100 | [diff] [blame] | 651 | |
| 652 | /* |
| 653 | * setup HW LBR filter, if any |
| 654 | */ |
| 655 | if (x86_pmu.lbr_sel_map) |
| 656 | ret = intel_pmu_setup_hw_lbr_filter(event); |
| 657 | |
| 658 | return ret; |
| 659 | } |
| 660 | |
| 661 | /* |
| 662 | * return the type of control flow change at address "from" |
Adam Buchbinder | 6a6256f | 2016-02-23 15:34:30 -0800 | [diff] [blame] | 663 | * instruction is not necessarily a branch (in case of interrupt). |
Stephane Eranian | 3e702ff | 2012-02-09 23:20:58 +0100 | [diff] [blame] | 664 | * |
| 665 | * The branch type returned also includes the priv level of the |
| 666 | * target of the control flow change (X86_BR_USER, X86_BR_KERNEL). |
| 667 | * |
| 668 | * If a branch type is unknown OR the instruction cannot be |
| 669 | * decoded (e.g., text page not present), then X86_BR_NONE is |
| 670 | * returned. |
| 671 | */ |
Andi Kleen | 135c561 | 2013-06-17 17:36:51 -0700 | [diff] [blame] | 672 | static int branch_type(unsigned long from, unsigned long to, int abort) |
Stephane Eranian | 3e702ff | 2012-02-09 23:20:58 +0100 | [diff] [blame] | 673 | { |
| 674 | struct insn insn; |
| 675 | void *addr; |
Dave Hansen | 6ba48ff | 2014-11-14 07:39:57 -0800 | [diff] [blame] | 676 | int bytes_read, bytes_left; |
Stephane Eranian | 3e702ff | 2012-02-09 23:20:58 +0100 | [diff] [blame] | 677 | int ret = X86_BR_NONE; |
| 678 | int ext, to_plm, from_plm; |
| 679 | u8 buf[MAX_INSN_SIZE]; |
| 680 | int is64 = 0; |
| 681 | |
| 682 | to_plm = kernel_ip(to) ? X86_BR_KERNEL : X86_BR_USER; |
| 683 | from_plm = kernel_ip(from) ? X86_BR_KERNEL : X86_BR_USER; |
| 684 | |
| 685 | /* |
| 686 | * maybe zero if lbr did not fill up after a reset by the time |
| 687 | * we get a PMU interrupt |
| 688 | */ |
| 689 | if (from == 0 || to == 0) |
| 690 | return X86_BR_NONE; |
| 691 | |
Andi Kleen | 135c561 | 2013-06-17 17:36:51 -0700 | [diff] [blame] | 692 | if (abort) |
| 693 | return X86_BR_ABORT | to_plm; |
| 694 | |
Stephane Eranian | 3e702ff | 2012-02-09 23:20:58 +0100 | [diff] [blame] | 695 | if (from_plm == X86_BR_USER) { |
| 696 | /* |
| 697 | * can happen if measuring at the user level only |
| 698 | * and we interrupt in a kernel thread, e.g., idle. |
| 699 | */ |
| 700 | if (!current->mm) |
| 701 | return X86_BR_NONE; |
| 702 | |
| 703 | /* may fail if text not present */ |
Dave Hansen | 6ba48ff | 2014-11-14 07:39:57 -0800 | [diff] [blame] | 704 | bytes_left = copy_from_user_nmi(buf, (void __user *)from, |
| 705 | MAX_INSN_SIZE); |
| 706 | bytes_read = MAX_INSN_SIZE - bytes_left; |
| 707 | if (!bytes_read) |
Stephane Eranian | 3e702ff | 2012-02-09 23:20:58 +0100 | [diff] [blame] | 708 | return X86_BR_NONE; |
| 709 | |
| 710 | addr = buf; |
Peter Zijlstra | 6e15eb3 | 2013-05-03 14:11:24 +0200 | [diff] [blame] | 711 | } else { |
| 712 | /* |
| 713 | * The LBR logs any address in the IP, even if the IP just |
| 714 | * faulted. This means userspace can control the from address. |
| 715 | * Ensure we don't blindy read any address by validating it is |
| 716 | * a known text address. |
| 717 | */ |
Dave Hansen | 6ba48ff | 2014-11-14 07:39:57 -0800 | [diff] [blame] | 718 | if (kernel_text_address(from)) { |
Peter Zijlstra | 6e15eb3 | 2013-05-03 14:11:24 +0200 | [diff] [blame] | 719 | addr = (void *)from; |
Dave Hansen | 6ba48ff | 2014-11-14 07:39:57 -0800 | [diff] [blame] | 720 | /* |
| 721 | * Assume we can get the maximum possible size |
| 722 | * when grabbing kernel data. This is not |
| 723 | * _strictly_ true since we could possibly be |
| 724 | * executing up next to a memory hole, but |
| 725 | * it is very unlikely to be a problem. |
| 726 | */ |
| 727 | bytes_read = MAX_INSN_SIZE; |
| 728 | } else { |
Peter Zijlstra | 6e15eb3 | 2013-05-03 14:11:24 +0200 | [diff] [blame] | 729 | return X86_BR_NONE; |
Dave Hansen | 6ba48ff | 2014-11-14 07:39:57 -0800 | [diff] [blame] | 730 | } |
Peter Zijlstra | 6e15eb3 | 2013-05-03 14:11:24 +0200 | [diff] [blame] | 731 | } |
Stephane Eranian | 3e702ff | 2012-02-09 23:20:58 +0100 | [diff] [blame] | 732 | |
| 733 | /* |
| 734 | * decoder needs to know the ABI especially |
| 735 | * on 64-bit systems running 32-bit apps |
| 736 | */ |
| 737 | #ifdef CONFIG_X86_64 |
| 738 | is64 = kernel_ip((unsigned long)addr) || !test_thread_flag(TIF_IA32); |
| 739 | #endif |
Dave Hansen | 6ba48ff | 2014-11-14 07:39:57 -0800 | [diff] [blame] | 740 | insn_init(&insn, addr, bytes_read, is64); |
Stephane Eranian | 3e702ff | 2012-02-09 23:20:58 +0100 | [diff] [blame] | 741 | insn_get_opcode(&insn); |
Dave Hansen | 6ba48ff | 2014-11-14 07:39:57 -0800 | [diff] [blame] | 742 | if (!insn.opcode.got) |
| 743 | return X86_BR_ABORT; |
Stephane Eranian | 3e702ff | 2012-02-09 23:20:58 +0100 | [diff] [blame] | 744 | |
| 745 | switch (insn.opcode.bytes[0]) { |
| 746 | case 0xf: |
| 747 | switch (insn.opcode.bytes[1]) { |
| 748 | case 0x05: /* syscall */ |
| 749 | case 0x34: /* sysenter */ |
| 750 | ret = X86_BR_SYSCALL; |
| 751 | break; |
| 752 | case 0x07: /* sysret */ |
| 753 | case 0x35: /* sysexit */ |
| 754 | ret = X86_BR_SYSRET; |
| 755 | break; |
| 756 | case 0x80 ... 0x8f: /* conditional */ |
| 757 | ret = X86_BR_JCC; |
| 758 | break; |
| 759 | default: |
| 760 | ret = X86_BR_NONE; |
| 761 | } |
| 762 | break; |
| 763 | case 0x70 ... 0x7f: /* conditional */ |
| 764 | ret = X86_BR_JCC; |
| 765 | break; |
| 766 | case 0xc2: /* near ret */ |
| 767 | case 0xc3: /* near ret */ |
| 768 | case 0xca: /* far ret */ |
| 769 | case 0xcb: /* far ret */ |
| 770 | ret = X86_BR_RET; |
| 771 | break; |
| 772 | case 0xcf: /* iret */ |
| 773 | ret = X86_BR_IRET; |
| 774 | break; |
| 775 | case 0xcc ... 0xce: /* int */ |
| 776 | ret = X86_BR_INT; |
| 777 | break; |
| 778 | case 0xe8: /* call near rel */ |
Yan, Zheng | aa54ae9 | 2014-11-04 21:56:11 -0500 | [diff] [blame] | 779 | insn_get_immediate(&insn); |
| 780 | if (insn.immediate1.value == 0) { |
| 781 | /* zero length call */ |
| 782 | ret = X86_BR_ZERO_CALL; |
| 783 | break; |
| 784 | } |
Stephane Eranian | 3e702ff | 2012-02-09 23:20:58 +0100 | [diff] [blame] | 785 | case 0x9a: /* call far absolute */ |
| 786 | ret = X86_BR_CALL; |
| 787 | break; |
| 788 | case 0xe0 ... 0xe3: /* loop jmp */ |
| 789 | ret = X86_BR_JCC; |
| 790 | break; |
| 791 | case 0xe9 ... 0xeb: /* jmp */ |
| 792 | ret = X86_BR_JMP; |
| 793 | break; |
| 794 | case 0xff: /* call near absolute, call far absolute ind */ |
| 795 | insn_get_modrm(&insn); |
| 796 | ext = (insn.modrm.bytes[0] >> 3) & 0x7; |
| 797 | switch (ext) { |
| 798 | case 2: /* near ind call */ |
| 799 | case 3: /* far ind call */ |
| 800 | ret = X86_BR_IND_CALL; |
| 801 | break; |
| 802 | case 4: |
| 803 | case 5: |
Stephane Eranian | 7b74cfb | 2015-05-14 23:09:59 +0200 | [diff] [blame] | 804 | ret = X86_BR_IND_JMP; |
Stephane Eranian | 3e702ff | 2012-02-09 23:20:58 +0100 | [diff] [blame] | 805 | break; |
| 806 | } |
| 807 | break; |
| 808 | default: |
| 809 | ret = X86_BR_NONE; |
Stephane Eranian | 60ce0fb | 2012-02-09 23:20:57 +0100 | [diff] [blame] | 810 | } |
| 811 | /* |
Stephane Eranian | 3e702ff | 2012-02-09 23:20:58 +0100 | [diff] [blame] | 812 | * interrupts, traps, faults (and thus ring transition) may |
| 813 | * occur on any instructions. Thus, to classify them correctly, |
| 814 | * we need to first look at the from and to priv levels. If they |
| 815 | * are different and to is in the kernel, then it indicates |
| 816 | * a ring transition. If the from instruction is not a ring |
| 817 | * transition instr (syscall, systenter, int), then it means |
| 818 | * it was a irq, trap or fault. |
| 819 | * |
| 820 | * we have no way of detecting kernel to kernel faults. |
Stephane Eranian | 60ce0fb | 2012-02-09 23:20:57 +0100 | [diff] [blame] | 821 | */ |
Stephane Eranian | 3e702ff | 2012-02-09 23:20:58 +0100 | [diff] [blame] | 822 | if (from_plm == X86_BR_USER && to_plm == X86_BR_KERNEL |
| 823 | && ret != X86_BR_SYSCALL && ret != X86_BR_INT) |
| 824 | ret = X86_BR_IRQ; |
Stephane Eranian | 60ce0fb | 2012-02-09 23:20:57 +0100 | [diff] [blame] | 825 | |
Stephane Eranian | 3e702ff | 2012-02-09 23:20:58 +0100 | [diff] [blame] | 826 | /* |
| 827 | * branch priv level determined by target as |
| 828 | * is done by HW when LBR_SELECT is implemented |
| 829 | */ |
| 830 | if (ret != X86_BR_NONE) |
| 831 | ret |= to_plm; |
| 832 | |
| 833 | return ret; |
| 834 | } |
| 835 | |
| 836 | /* |
| 837 | * implement actual branch filter based on user demand. |
| 838 | * Hardware may not exactly satisfy that request, thus |
| 839 | * we need to inspect opcodes. Mismatched branches are |
| 840 | * discarded. Therefore, the number of branches returned |
| 841 | * in PERF_SAMPLE_BRANCH_STACK sample may vary. |
| 842 | */ |
| 843 | static void |
| 844 | intel_pmu_lbr_filter(struct cpu_hw_events *cpuc) |
| 845 | { |
| 846 | u64 from, to; |
| 847 | int br_sel = cpuc->br_sel; |
| 848 | int i, j, type; |
| 849 | bool compress = false; |
| 850 | |
| 851 | /* if sampling all branches, then nothing to filter */ |
| 852 | if ((br_sel & X86_BR_ALL) == X86_BR_ALL) |
| 853 | return; |
| 854 | |
| 855 | for (i = 0; i < cpuc->lbr_stack.nr; i++) { |
| 856 | |
| 857 | from = cpuc->lbr_entries[i].from; |
| 858 | to = cpuc->lbr_entries[i].to; |
| 859 | |
Andi Kleen | 135c561 | 2013-06-17 17:36:51 -0700 | [diff] [blame] | 860 | type = branch_type(from, to, cpuc->lbr_entries[i].abort); |
| 861 | if (type != X86_BR_NONE && (br_sel & X86_BR_ANYTX)) { |
| 862 | if (cpuc->lbr_entries[i].in_tx) |
| 863 | type |= X86_BR_IN_TX; |
| 864 | else |
| 865 | type |= X86_BR_NO_TX; |
| 866 | } |
Stephane Eranian | 3e702ff | 2012-02-09 23:20:58 +0100 | [diff] [blame] | 867 | |
| 868 | /* if type does not correspond, then discard */ |
| 869 | if (type == X86_BR_NONE || (br_sel & type) != type) { |
| 870 | cpuc->lbr_entries[i].from = 0; |
| 871 | compress = true; |
| 872 | } |
| 873 | } |
| 874 | |
| 875 | if (!compress) |
| 876 | return; |
| 877 | |
| 878 | /* remove all entries with from=0 */ |
| 879 | for (i = 0; i < cpuc->lbr_stack.nr; ) { |
| 880 | if (!cpuc->lbr_entries[i].from) { |
| 881 | j = i; |
| 882 | while (++j < cpuc->lbr_stack.nr) |
| 883 | cpuc->lbr_entries[j-1] = cpuc->lbr_entries[j]; |
| 884 | cpuc->lbr_stack.nr--; |
| 885 | if (!cpuc->lbr_entries[i].from) |
| 886 | continue; |
| 887 | } |
| 888 | i++; |
| 889 | } |
Stephane Eranian | 60ce0fb | 2012-02-09 23:20:57 +0100 | [diff] [blame] | 890 | } |
| 891 | |
| 892 | /* |
Stephane Eranian | c5cc2cd | 2012-02-09 23:20:55 +0100 | [diff] [blame] | 893 | * Map interface branch filters onto LBR filters |
| 894 | */ |
Peter Zijlstra | 2c44b19 | 2014-11-05 10:36:45 +0100 | [diff] [blame] | 895 | static const int nhm_lbr_sel_map[PERF_SAMPLE_BRANCH_MAX_SHIFT] = { |
Yan, Zheng | 27ac905 | 2014-11-04 21:55:57 -0500 | [diff] [blame] | 896 | [PERF_SAMPLE_BRANCH_ANY_SHIFT] = LBR_ANY, |
| 897 | [PERF_SAMPLE_BRANCH_USER_SHIFT] = LBR_USER, |
| 898 | [PERF_SAMPLE_BRANCH_KERNEL_SHIFT] = LBR_KERNEL, |
| 899 | [PERF_SAMPLE_BRANCH_HV_SHIFT] = LBR_IGN, |
| 900 | [PERF_SAMPLE_BRANCH_ANY_RETURN_SHIFT] = LBR_RETURN | LBR_REL_JMP |
| 901 | | LBR_IND_JMP | LBR_FAR, |
Stephane Eranian | c5cc2cd | 2012-02-09 23:20:55 +0100 | [diff] [blame] | 902 | /* |
| 903 | * NHM/WSM erratum: must include REL_JMP+IND_JMP to get CALL branches |
| 904 | */ |
Yan, Zheng | 27ac905 | 2014-11-04 21:55:57 -0500 | [diff] [blame] | 905 | [PERF_SAMPLE_BRANCH_ANY_CALL_SHIFT] = |
Stephane Eranian | c5cc2cd | 2012-02-09 23:20:55 +0100 | [diff] [blame] | 906 | LBR_REL_CALL | LBR_IND_CALL | LBR_REL_JMP | LBR_IND_JMP | LBR_FAR, |
| 907 | /* |
| 908 | * NHM/WSM erratum: must include IND_JMP to capture IND_CALL |
| 909 | */ |
Yan, Zheng | 27ac905 | 2014-11-04 21:55:57 -0500 | [diff] [blame] | 910 | [PERF_SAMPLE_BRANCH_IND_CALL_SHIFT] = LBR_IND_CALL | LBR_IND_JMP, |
| 911 | [PERF_SAMPLE_BRANCH_COND_SHIFT] = LBR_JCC, |
Stephane Eranian | 7b74cfb | 2015-05-14 23:09:59 +0200 | [diff] [blame] | 912 | [PERF_SAMPLE_BRANCH_IND_JUMP_SHIFT] = LBR_IND_JMP, |
Stephane Eranian | c5cc2cd | 2012-02-09 23:20:55 +0100 | [diff] [blame] | 913 | }; |
| 914 | |
Peter Zijlstra | 2c44b19 | 2014-11-05 10:36:45 +0100 | [diff] [blame] | 915 | static const int snb_lbr_sel_map[PERF_SAMPLE_BRANCH_MAX_SHIFT] = { |
Yan, Zheng | 27ac905 | 2014-11-04 21:55:57 -0500 | [diff] [blame] | 916 | [PERF_SAMPLE_BRANCH_ANY_SHIFT] = LBR_ANY, |
| 917 | [PERF_SAMPLE_BRANCH_USER_SHIFT] = LBR_USER, |
| 918 | [PERF_SAMPLE_BRANCH_KERNEL_SHIFT] = LBR_KERNEL, |
| 919 | [PERF_SAMPLE_BRANCH_HV_SHIFT] = LBR_IGN, |
| 920 | [PERF_SAMPLE_BRANCH_ANY_RETURN_SHIFT] = LBR_RETURN | LBR_FAR, |
| 921 | [PERF_SAMPLE_BRANCH_ANY_CALL_SHIFT] = LBR_REL_CALL | LBR_IND_CALL |
| 922 | | LBR_FAR, |
| 923 | [PERF_SAMPLE_BRANCH_IND_CALL_SHIFT] = LBR_IND_CALL, |
| 924 | [PERF_SAMPLE_BRANCH_COND_SHIFT] = LBR_JCC, |
Stephane Eranian | 7b74cfb | 2015-05-14 23:09:59 +0200 | [diff] [blame] | 925 | [PERF_SAMPLE_BRANCH_IND_JUMP_SHIFT] = LBR_IND_JMP, |
Stephane Eranian | d892819 | 2015-10-13 09:09:09 +0200 | [diff] [blame] | 926 | [PERF_SAMPLE_BRANCH_CALL_SHIFT] = LBR_REL_CALL, |
Stephane Eranian | c5cc2cd | 2012-02-09 23:20:55 +0100 | [diff] [blame] | 927 | }; |
| 928 | |
Peter Zijlstra | 2c44b19 | 2014-11-05 10:36:45 +0100 | [diff] [blame] | 929 | static const int hsw_lbr_sel_map[PERF_SAMPLE_BRANCH_MAX_SHIFT] = { |
Yan, Zheng | e9d7f7c | 2014-11-04 21:56:00 -0500 | [diff] [blame] | 930 | [PERF_SAMPLE_BRANCH_ANY_SHIFT] = LBR_ANY, |
| 931 | [PERF_SAMPLE_BRANCH_USER_SHIFT] = LBR_USER, |
| 932 | [PERF_SAMPLE_BRANCH_KERNEL_SHIFT] = LBR_KERNEL, |
| 933 | [PERF_SAMPLE_BRANCH_HV_SHIFT] = LBR_IGN, |
| 934 | [PERF_SAMPLE_BRANCH_ANY_RETURN_SHIFT] = LBR_RETURN | LBR_FAR, |
| 935 | [PERF_SAMPLE_BRANCH_ANY_CALL_SHIFT] = LBR_REL_CALL | LBR_IND_CALL |
| 936 | | LBR_FAR, |
| 937 | [PERF_SAMPLE_BRANCH_IND_CALL_SHIFT] = LBR_IND_CALL, |
| 938 | [PERF_SAMPLE_BRANCH_COND_SHIFT] = LBR_JCC, |
| 939 | [PERF_SAMPLE_BRANCH_CALL_STACK_SHIFT] = LBR_REL_CALL | LBR_IND_CALL |
| 940 | | LBR_RETURN | LBR_CALL_STACK, |
Stephane Eranian | 7b74cfb | 2015-05-14 23:09:59 +0200 | [diff] [blame] | 941 | [PERF_SAMPLE_BRANCH_IND_JUMP_SHIFT] = LBR_IND_JMP, |
Stephane Eranian | d892819 | 2015-10-13 09:09:09 +0200 | [diff] [blame] | 942 | [PERF_SAMPLE_BRANCH_CALL_SHIFT] = LBR_REL_CALL, |
Yan, Zheng | e9d7f7c | 2014-11-04 21:56:00 -0500 | [diff] [blame] | 943 | }; |
| 944 | |
Stephane Eranian | c5cc2cd | 2012-02-09 23:20:55 +0100 | [diff] [blame] | 945 | /* core */ |
Mathias Krause | 066ce64 | 2014-08-26 18:49:45 +0200 | [diff] [blame] | 946 | void __init intel_pmu_lbr_init_core(void) |
Peter Zijlstra | caff2be | 2010-03-03 12:02:30 +0100 | [diff] [blame] | 947 | { |
Peter Zijlstra | caff2be | 2010-03-03 12:02:30 +0100 | [diff] [blame] | 948 | x86_pmu.lbr_nr = 4; |
Stephane Eranian | 225ce53 | 2012-02-09 23:20:52 +0100 | [diff] [blame] | 949 | x86_pmu.lbr_tos = MSR_LBR_TOS; |
| 950 | x86_pmu.lbr_from = MSR_LBR_CORE_FROM; |
| 951 | x86_pmu.lbr_to = MSR_LBR_CORE_TO; |
Stephane Eranian | c5cc2cd | 2012-02-09 23:20:55 +0100 | [diff] [blame] | 952 | |
Stephane Eranian | 3e702ff | 2012-02-09 23:20:58 +0100 | [diff] [blame] | 953 | /* |
| 954 | * SW branch filter usage: |
| 955 | * - compensate for lack of HW filter |
| 956 | */ |
Stephane Eranian | c5cc2cd | 2012-02-09 23:20:55 +0100 | [diff] [blame] | 957 | pr_cont("4-deep LBR, "); |
Peter Zijlstra | caff2be | 2010-03-03 12:02:30 +0100 | [diff] [blame] | 958 | } |
| 959 | |
Stephane Eranian | c5cc2cd | 2012-02-09 23:20:55 +0100 | [diff] [blame] | 960 | /* nehalem/westmere */ |
Mathias Krause | 066ce64 | 2014-08-26 18:49:45 +0200 | [diff] [blame] | 961 | void __init intel_pmu_lbr_init_nhm(void) |
Peter Zijlstra | caff2be | 2010-03-03 12:02:30 +0100 | [diff] [blame] | 962 | { |
Peter Zijlstra | caff2be | 2010-03-03 12:02:30 +0100 | [diff] [blame] | 963 | x86_pmu.lbr_nr = 16; |
Stephane Eranian | 225ce53 | 2012-02-09 23:20:52 +0100 | [diff] [blame] | 964 | x86_pmu.lbr_tos = MSR_LBR_TOS; |
| 965 | x86_pmu.lbr_from = MSR_LBR_NHM_FROM; |
| 966 | x86_pmu.lbr_to = MSR_LBR_NHM_TO; |
Stephane Eranian | c5cc2cd | 2012-02-09 23:20:55 +0100 | [diff] [blame] | 967 | |
| 968 | x86_pmu.lbr_sel_mask = LBR_SEL_MASK; |
| 969 | x86_pmu.lbr_sel_map = nhm_lbr_sel_map; |
| 970 | |
Stephane Eranian | 3e702ff | 2012-02-09 23:20:58 +0100 | [diff] [blame] | 971 | /* |
| 972 | * SW branch filter usage: |
| 973 | * - workaround LBR_SEL errata (see above) |
| 974 | * - support syscall, sysret capture. |
| 975 | * That requires LBR_FAR but that means far |
| 976 | * jmp need to be filtered out |
| 977 | */ |
Stephane Eranian | c5cc2cd | 2012-02-09 23:20:55 +0100 | [diff] [blame] | 978 | pr_cont("16-deep LBR, "); |
Peter Zijlstra | caff2be | 2010-03-03 12:02:30 +0100 | [diff] [blame] | 979 | } |
| 980 | |
Stephane Eranian | c5cc2cd | 2012-02-09 23:20:55 +0100 | [diff] [blame] | 981 | /* sandy bridge */ |
Mathias Krause | 066ce64 | 2014-08-26 18:49:45 +0200 | [diff] [blame] | 982 | void __init intel_pmu_lbr_init_snb(void) |
Stephane Eranian | c5cc2cd | 2012-02-09 23:20:55 +0100 | [diff] [blame] | 983 | { |
| 984 | x86_pmu.lbr_nr = 16; |
| 985 | x86_pmu.lbr_tos = MSR_LBR_TOS; |
| 986 | x86_pmu.lbr_from = MSR_LBR_NHM_FROM; |
| 987 | x86_pmu.lbr_to = MSR_LBR_NHM_TO; |
| 988 | |
| 989 | x86_pmu.lbr_sel_mask = LBR_SEL_MASK; |
| 990 | x86_pmu.lbr_sel_map = snb_lbr_sel_map; |
| 991 | |
Stephane Eranian | 3e702ff | 2012-02-09 23:20:58 +0100 | [diff] [blame] | 992 | /* |
| 993 | * SW branch filter usage: |
| 994 | * - support syscall, sysret capture. |
| 995 | * That requires LBR_FAR but that means far |
| 996 | * jmp need to be filtered out |
| 997 | */ |
Stephane Eranian | c5cc2cd | 2012-02-09 23:20:55 +0100 | [diff] [blame] | 998 | pr_cont("16-deep LBR, "); |
| 999 | } |
| 1000 | |
Yan, Zheng | e9d7f7c | 2014-11-04 21:56:00 -0500 | [diff] [blame] | 1001 | /* haswell */ |
| 1002 | void intel_pmu_lbr_init_hsw(void) |
| 1003 | { |
| 1004 | x86_pmu.lbr_nr = 16; |
| 1005 | x86_pmu.lbr_tos = MSR_LBR_TOS; |
| 1006 | x86_pmu.lbr_from = MSR_LBR_NHM_FROM; |
| 1007 | x86_pmu.lbr_to = MSR_LBR_NHM_TO; |
| 1008 | |
| 1009 | x86_pmu.lbr_sel_mask = LBR_SEL_MASK; |
| 1010 | x86_pmu.lbr_sel_map = hsw_lbr_sel_map; |
| 1011 | |
| 1012 | pr_cont("16-deep LBR, "); |
| 1013 | } |
| 1014 | |
Andi Kleen | 9a92e16 | 2015-05-10 12:22:44 -0700 | [diff] [blame] | 1015 | /* skylake */ |
| 1016 | __init void intel_pmu_lbr_init_skl(void) |
| 1017 | { |
| 1018 | x86_pmu.lbr_nr = 32; |
| 1019 | x86_pmu.lbr_tos = MSR_LBR_TOS; |
| 1020 | x86_pmu.lbr_from = MSR_LBR_NHM_FROM; |
| 1021 | x86_pmu.lbr_to = MSR_LBR_NHM_TO; |
| 1022 | |
| 1023 | x86_pmu.lbr_sel_mask = LBR_SEL_MASK; |
| 1024 | x86_pmu.lbr_sel_map = hsw_lbr_sel_map; |
| 1025 | |
| 1026 | /* |
| 1027 | * SW branch filter usage: |
| 1028 | * - support syscall, sysret capture. |
| 1029 | * That requires LBR_FAR but that means far |
| 1030 | * jmp need to be filtered out |
| 1031 | */ |
| 1032 | pr_cont("32-deep LBR, "); |
| 1033 | } |
| 1034 | |
Stephane Eranian | c5cc2cd | 2012-02-09 23:20:55 +0100 | [diff] [blame] | 1035 | /* atom */ |
Mathias Krause | 066ce64 | 2014-08-26 18:49:45 +0200 | [diff] [blame] | 1036 | void __init intel_pmu_lbr_init_atom(void) |
Peter Zijlstra | caff2be | 2010-03-03 12:02:30 +0100 | [diff] [blame] | 1037 | { |
Stephane Eranian | 88c9a65 | 2012-02-09 23:20:56 +0100 | [diff] [blame] | 1038 | /* |
| 1039 | * only models starting at stepping 10 seems |
| 1040 | * to have an operational LBR which can freeze |
| 1041 | * on PMU interrupt |
| 1042 | */ |
Stephane Eranian | 3ec18cd | 2012-08-20 11:24:21 +0200 | [diff] [blame] | 1043 | if (boot_cpu_data.x86_model == 28 |
| 1044 | && boot_cpu_data.x86_mask < 10) { |
Stephane Eranian | 88c9a65 | 2012-02-09 23:20:56 +0100 | [diff] [blame] | 1045 | pr_cont("LBR disabled due to erratum"); |
| 1046 | return; |
| 1047 | } |
| 1048 | |
Peter Zijlstra | caff2be | 2010-03-03 12:02:30 +0100 | [diff] [blame] | 1049 | x86_pmu.lbr_nr = 8; |
Stephane Eranian | 225ce53 | 2012-02-09 23:20:52 +0100 | [diff] [blame] | 1050 | x86_pmu.lbr_tos = MSR_LBR_TOS; |
| 1051 | x86_pmu.lbr_from = MSR_LBR_CORE_FROM; |
| 1052 | x86_pmu.lbr_to = MSR_LBR_CORE_TO; |
Stephane Eranian | c5cc2cd | 2012-02-09 23:20:55 +0100 | [diff] [blame] | 1053 | |
Stephane Eranian | 3e702ff | 2012-02-09 23:20:58 +0100 | [diff] [blame] | 1054 | /* |
| 1055 | * SW branch filter usage: |
| 1056 | * - compensate for lack of HW filter |
| 1057 | */ |
Stephane Eranian | c5cc2cd | 2012-02-09 23:20:55 +0100 | [diff] [blame] | 1058 | pr_cont("8-deep LBR, "); |
Peter Zijlstra | caff2be | 2010-03-03 12:02:30 +0100 | [diff] [blame] | 1059 | } |
Harish Chegondi | 1e7b939 | 2015-12-07 14:28:18 -0800 | [diff] [blame] | 1060 | |
Kan Liang | f21d5ad | 2016-04-15 00:53:45 -0700 | [diff] [blame^] | 1061 | /* slm */ |
| 1062 | void __init intel_pmu_lbr_init_slm(void) |
| 1063 | { |
| 1064 | x86_pmu.lbr_nr = 8; |
| 1065 | x86_pmu.lbr_tos = MSR_LBR_TOS; |
| 1066 | x86_pmu.lbr_from = MSR_LBR_CORE_FROM; |
| 1067 | x86_pmu.lbr_to = MSR_LBR_CORE_TO; |
| 1068 | |
| 1069 | x86_pmu.lbr_sel_mask = LBR_SEL_MASK; |
| 1070 | x86_pmu.lbr_sel_map = nhm_lbr_sel_map; |
| 1071 | |
| 1072 | /* |
| 1073 | * SW branch filter usage: |
| 1074 | * - compensate for lack of HW filter |
| 1075 | */ |
| 1076 | pr_cont("8-deep LBR, "); |
| 1077 | } |
| 1078 | |
Harish Chegondi | 1e7b939 | 2015-12-07 14:28:18 -0800 | [diff] [blame] | 1079 | /* Knights Landing */ |
| 1080 | void intel_pmu_lbr_init_knl(void) |
| 1081 | { |
| 1082 | x86_pmu.lbr_nr = 8; |
| 1083 | x86_pmu.lbr_tos = MSR_LBR_TOS; |
| 1084 | x86_pmu.lbr_from = MSR_LBR_NHM_FROM; |
| 1085 | x86_pmu.lbr_to = MSR_LBR_NHM_TO; |
| 1086 | |
| 1087 | x86_pmu.lbr_sel_mask = LBR_SEL_MASK; |
| 1088 | x86_pmu.lbr_sel_map = snb_lbr_sel_map; |
| 1089 | |
| 1090 | pr_cont("8-deep LBR, "); |
| 1091 | } |