Paolo Ciarrocchi | 8b45b72 | 2008-02-19 23:43:25 +0100 | [diff] [blame] | 1 | /* |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 2 | * @file op_model_ppro.h |
Andi Kleen | b991702 | 2008-08-18 14:50:31 +0200 | [diff] [blame] | 3 | * Family 6 perfmon and architectural perfmon MSR operations |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 4 | * |
| 5 | * @remark Copyright 2002 OProfile authors |
Andi Kleen | b991702 | 2008-08-18 14:50:31 +0200 | [diff] [blame] | 6 | * @remark Copyright 2008 Intel Corporation |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 7 | * @remark Read the file COPYING |
| 8 | * |
| 9 | * @author John Levon |
| 10 | * @author Philippe Elie |
| 11 | * @author Graydon Hoare |
Andi Kleen | b991702 | 2008-08-18 14:50:31 +0200 | [diff] [blame] | 12 | * @author Andi Kleen |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 13 | */ |
| 14 | |
| 15 | #include <linux/oprofile.h> |
Andi Kleen | b991702 | 2008-08-18 14:50:31 +0200 | [diff] [blame] | 16 | #include <linux/slab.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 17 | #include <asm/ptrace.h> |
| 18 | #include <asm/msr.h> |
| 19 | #include <asm/apic.h> |
Don Zickus | 3e4ff11 | 2006-06-26 13:57:01 +0200 | [diff] [blame] | 20 | #include <asm/nmi.h> |
Andi Kleen | b991702 | 2008-08-18 14:50:31 +0200 | [diff] [blame] | 21 | #include <asm/intel_arch_perfmon.h> |
Paolo Ciarrocchi | 8b45b72 | 2008-02-19 23:43:25 +0100 | [diff] [blame] | 22 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 23 | #include "op_x86_model.h" |
| 24 | #include "op_counter.h" |
| 25 | |
Andi Kleen | b991702 | 2008-08-18 14:50:31 +0200 | [diff] [blame] | 26 | static int num_counters = 2; |
| 27 | static int counter_width = 32; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 28 | |
Paolo Ciarrocchi | 8b45b72 | 2008-02-19 23:43:25 +0100 | [diff] [blame] | 29 | #define CTR_IS_RESERVED(msrs, c) (msrs->counters[(c)].addr ? 1 : 0) |
Andi Kleen | 7c64ade | 2008-11-07 14:02:49 +0100 | [diff] [blame^] | 30 | #define CTR_OVERFLOWED(n) (!((n) & (1ULL<<(counter_width-1)))) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 31 | |
Paolo Ciarrocchi | 8b45b72 | 2008-02-19 23:43:25 +0100 | [diff] [blame] | 32 | #define CTRL_IS_RESERVED(msrs, c) (msrs->controls[(c)].addr ? 1 : 0) |
| 33 | #define CTRL_READ(l, h, msrs, c) do {rdmsr((msrs->controls[(c)].addr), (l), (h)); } while (0) |
| 34 | #define CTRL_WRITE(l, h, msrs, c) do {wrmsr((msrs->controls[(c)].addr), (l), (h)); } while (0) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 35 | #define CTRL_SET_ACTIVE(n) (n |= (1<<22)) |
| 36 | #define CTRL_SET_INACTIVE(n) (n &= ~(1<<22)) |
| 37 | #define CTRL_CLEAR(x) (x &= (1<<21)) |
| 38 | #define CTRL_SET_ENABLE(val) (val |= 1<<20) |
Paolo Ciarrocchi | 8b45b72 | 2008-02-19 23:43:25 +0100 | [diff] [blame] | 39 | #define CTRL_SET_USR(val, u) (val |= ((u & 1) << 16)) |
| 40 | #define CTRL_SET_KERN(val, k) (val |= ((k & 1) << 17)) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 41 | #define CTRL_SET_UM(val, m) (val |= (m << 8)) |
| 42 | #define CTRL_SET_EVENT(val, e) (val |= e) |
| 43 | |
Andi Kleen | b991702 | 2008-08-18 14:50:31 +0200 | [diff] [blame] | 44 | static u64 *reset_value; |
Paolo Ciarrocchi | 8b45b72 | 2008-02-19 23:43:25 +0100 | [diff] [blame] | 45 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 46 | static void ppro_fill_in_addresses(struct op_msrs * const msrs) |
| 47 | { |
Don Zickus | cb9c448 | 2006-09-26 10:52:26 +0200 | [diff] [blame] | 48 | int i; |
| 49 | |
Andi Kleen | b991702 | 2008-08-18 14:50:31 +0200 | [diff] [blame] | 50 | for (i = 0; i < num_counters; i++) { |
Don Zickus | cb9c448 | 2006-09-26 10:52:26 +0200 | [diff] [blame] | 51 | if (reserve_perfctr_nmi(MSR_P6_PERFCTR0 + i)) |
| 52 | msrs->counters[i].addr = MSR_P6_PERFCTR0 + i; |
| 53 | else |
| 54 | msrs->counters[i].addr = 0; |
| 55 | } |
Paolo Ciarrocchi | 8b45b72 | 2008-02-19 23:43:25 +0100 | [diff] [blame] | 56 | |
Andi Kleen | b991702 | 2008-08-18 14:50:31 +0200 | [diff] [blame] | 57 | for (i = 0; i < num_counters; i++) { |
Don Zickus | cb9c448 | 2006-09-26 10:52:26 +0200 | [diff] [blame] | 58 | if (reserve_evntsel_nmi(MSR_P6_EVNTSEL0 + i)) |
| 59 | msrs->controls[i].addr = MSR_P6_EVNTSEL0 + i; |
| 60 | else |
| 61 | msrs->controls[i].addr = 0; |
| 62 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 63 | } |
| 64 | |
| 65 | |
| 66 | static void ppro_setup_ctrs(struct op_msrs const * const msrs) |
| 67 | { |
| 68 | unsigned int low, high; |
| 69 | int i; |
| 70 | |
Andi Kleen | b991702 | 2008-08-18 14:50:31 +0200 | [diff] [blame] | 71 | if (!reset_value) { |
| 72 | reset_value = kmalloc(sizeof(unsigned) * num_counters, |
| 73 | GFP_ATOMIC); |
| 74 | if (!reset_value) |
| 75 | return; |
| 76 | } |
| 77 | |
| 78 | if (cpu_has_arch_perfmon) { |
| 79 | union cpuid10_eax eax; |
| 80 | eax.full = cpuid_eax(0xa); |
| 81 | if (counter_width < eax.split.bit_width) |
| 82 | counter_width = eax.split.bit_width; |
| 83 | } |
| 84 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 85 | /* clear all counters */ |
Andi Kleen | b991702 | 2008-08-18 14:50:31 +0200 | [diff] [blame] | 86 | for (i = 0 ; i < num_counters; ++i) { |
Paolo Ciarrocchi | 8b45b72 | 2008-02-19 23:43:25 +0100 | [diff] [blame] | 87 | if (unlikely(!CTRL_IS_RESERVED(msrs, i))) |
Don Zickus | cb9c448 | 2006-09-26 10:52:26 +0200 | [diff] [blame] | 88 | continue; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 89 | CTRL_READ(low, high, msrs, i); |
| 90 | CTRL_CLEAR(low); |
| 91 | CTRL_WRITE(low, high, msrs, i); |
| 92 | } |
Paolo Ciarrocchi | 8b45b72 | 2008-02-19 23:43:25 +0100 | [diff] [blame] | 93 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 94 | /* avoid a false detection of ctr overflows in NMI handler */ |
Andi Kleen | b991702 | 2008-08-18 14:50:31 +0200 | [diff] [blame] | 95 | for (i = 0; i < num_counters; ++i) { |
Paolo Ciarrocchi | 8b45b72 | 2008-02-19 23:43:25 +0100 | [diff] [blame] | 96 | if (unlikely(!CTR_IS_RESERVED(msrs, i))) |
Don Zickus | cb9c448 | 2006-09-26 10:52:26 +0200 | [diff] [blame] | 97 | continue; |
Andi Kleen | b991702 | 2008-08-18 14:50:31 +0200 | [diff] [blame] | 98 | wrmsrl(msrs->counters[i].addr, -1LL); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 99 | } |
| 100 | |
| 101 | /* enable active counters */ |
Andi Kleen | b991702 | 2008-08-18 14:50:31 +0200 | [diff] [blame] | 102 | for (i = 0; i < num_counters; ++i) { |
Paolo Ciarrocchi | 8b45b72 | 2008-02-19 23:43:25 +0100 | [diff] [blame] | 103 | if ((counter_config[i].enabled) && (CTR_IS_RESERVED(msrs, i))) { |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 104 | reset_value[i] = counter_config[i].count; |
| 105 | |
Andi Kleen | b991702 | 2008-08-18 14:50:31 +0200 | [diff] [blame] | 106 | wrmsrl(msrs->counters[i].addr, -reset_value[i]); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 107 | |
| 108 | CTRL_READ(low, high, msrs, i); |
| 109 | CTRL_CLEAR(low); |
| 110 | CTRL_SET_ENABLE(low); |
| 111 | CTRL_SET_USR(low, counter_config[i].user); |
| 112 | CTRL_SET_KERN(low, counter_config[i].kernel); |
| 113 | CTRL_SET_UM(low, counter_config[i].unit_mask); |
| 114 | CTRL_SET_EVENT(low, counter_config[i].event); |
| 115 | CTRL_WRITE(low, high, msrs, i); |
Don Zickus | cb9c448 | 2006-09-26 10:52:26 +0200 | [diff] [blame] | 116 | } else { |
| 117 | reset_value[i] = 0; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 118 | } |
| 119 | } |
| 120 | } |
| 121 | |
Paolo Ciarrocchi | 8b45b72 | 2008-02-19 23:43:25 +0100 | [diff] [blame] | 122 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 123 | static int ppro_check_ctrs(struct pt_regs * const regs, |
| 124 | struct op_msrs const * const msrs) |
| 125 | { |
Andi Kleen | 7c64ade | 2008-11-07 14:02:49 +0100 | [diff] [blame^] | 126 | u64 val; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 127 | int i; |
Paolo Ciarrocchi | 8b45b72 | 2008-02-19 23:43:25 +0100 | [diff] [blame] | 128 | |
Andi Kleen | b991702 | 2008-08-18 14:50:31 +0200 | [diff] [blame] | 129 | for (i = 0 ; i < num_counters; ++i) { |
Don Zickus | cb9c448 | 2006-09-26 10:52:26 +0200 | [diff] [blame] | 130 | if (!reset_value[i]) |
| 131 | continue; |
Andi Kleen | 7c64ade | 2008-11-07 14:02:49 +0100 | [diff] [blame^] | 132 | rdmsrl(msrs->counters[i].addr, val); |
| 133 | if (CTR_OVERFLOWED(val)) { |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 134 | oprofile_add_sample(regs, i); |
Andi Kleen | b991702 | 2008-08-18 14:50:31 +0200 | [diff] [blame] | 135 | wrmsrl(msrs->counters[i].addr, -reset_value[i]); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 136 | } |
| 137 | } |
| 138 | |
| 139 | /* Only P6 based Pentium M need to re-unmask the apic vector but it |
| 140 | * doesn't hurt other P6 variant */ |
| 141 | apic_write(APIC_LVTPC, apic_read(APIC_LVTPC) & ~APIC_LVT_MASKED); |
| 142 | |
| 143 | /* We can't work out if we really handled an interrupt. We |
| 144 | * might have caught a *second* counter just after overflowing |
| 145 | * the interrupt for this counter then arrives |
| 146 | * and we don't find a counter that's overflowed, so we |
| 147 | * would return 0 and get dazed + confused. Instead we always |
| 148 | * assume we found an overflow. This sucks. |
| 149 | */ |
| 150 | return 1; |
| 151 | } |
| 152 | |
Paolo Ciarrocchi | 8b45b72 | 2008-02-19 23:43:25 +0100 | [diff] [blame] | 153 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 154 | static void ppro_start(struct op_msrs const * const msrs) |
| 155 | { |
Paolo Ciarrocchi | 8b45b72 | 2008-02-19 23:43:25 +0100 | [diff] [blame] | 156 | unsigned int low, high; |
Arun Sharma | 6b77df0 | 2006-09-29 02:00:01 -0700 | [diff] [blame] | 157 | int i; |
Don Zickus | cb9c448 | 2006-09-26 10:52:26 +0200 | [diff] [blame] | 158 | |
Andi Kleen | b991702 | 2008-08-18 14:50:31 +0200 | [diff] [blame] | 159 | for (i = 0; i < num_counters; ++i) { |
Arun Sharma | 6b77df0 | 2006-09-29 02:00:01 -0700 | [diff] [blame] | 160 | if (reset_value[i]) { |
| 161 | CTRL_READ(low, high, msrs, i); |
| 162 | CTRL_SET_ACTIVE(low); |
| 163 | CTRL_WRITE(low, high, msrs, i); |
| 164 | } |
Don Zickus | cb9c448 | 2006-09-26 10:52:26 +0200 | [diff] [blame] | 165 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 166 | } |
| 167 | |
| 168 | |
| 169 | static void ppro_stop(struct op_msrs const * const msrs) |
| 170 | { |
Paolo Ciarrocchi | 8b45b72 | 2008-02-19 23:43:25 +0100 | [diff] [blame] | 171 | unsigned int low, high; |
Arun Sharma | 6b77df0 | 2006-09-29 02:00:01 -0700 | [diff] [blame] | 172 | int i; |
Don Zickus | cb9c448 | 2006-09-26 10:52:26 +0200 | [diff] [blame] | 173 | |
Andi Kleen | b991702 | 2008-08-18 14:50:31 +0200 | [diff] [blame] | 174 | for (i = 0; i < num_counters; ++i) { |
Arun Sharma | 6b77df0 | 2006-09-29 02:00:01 -0700 | [diff] [blame] | 175 | if (!reset_value[i]) |
| 176 | continue; |
| 177 | CTRL_READ(low, high, msrs, i); |
Don Zickus | cb9c448 | 2006-09-26 10:52:26 +0200 | [diff] [blame] | 178 | CTRL_SET_INACTIVE(low); |
Arun Sharma | 6b77df0 | 2006-09-29 02:00:01 -0700 | [diff] [blame] | 179 | CTRL_WRITE(low, high, msrs, i); |
Don Zickus | cb9c448 | 2006-09-26 10:52:26 +0200 | [diff] [blame] | 180 | } |
| 181 | } |
| 182 | |
| 183 | static void ppro_shutdown(struct op_msrs const * const msrs) |
| 184 | { |
| 185 | int i; |
| 186 | |
Andi Kleen | b991702 | 2008-08-18 14:50:31 +0200 | [diff] [blame] | 187 | for (i = 0 ; i < num_counters ; ++i) { |
Paolo Ciarrocchi | 8b45b72 | 2008-02-19 23:43:25 +0100 | [diff] [blame] | 188 | if (CTR_IS_RESERVED(msrs, i)) |
Don Zickus | cb9c448 | 2006-09-26 10:52:26 +0200 | [diff] [blame] | 189 | release_perfctr_nmi(MSR_P6_PERFCTR0 + i); |
| 190 | } |
Andi Kleen | b991702 | 2008-08-18 14:50:31 +0200 | [diff] [blame] | 191 | for (i = 0 ; i < num_counters ; ++i) { |
Paolo Ciarrocchi | 8b45b72 | 2008-02-19 23:43:25 +0100 | [diff] [blame] | 192 | if (CTRL_IS_RESERVED(msrs, i)) |
Don Zickus | cb9c448 | 2006-09-26 10:52:26 +0200 | [diff] [blame] | 193 | release_evntsel_nmi(MSR_P6_EVNTSEL0 + i); |
| 194 | } |
Andi Kleen | b991702 | 2008-08-18 14:50:31 +0200 | [diff] [blame] | 195 | if (reset_value) { |
| 196 | kfree(reset_value); |
| 197 | reset_value = NULL; |
| 198 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 199 | } |
| 200 | |
| 201 | |
Andi Kleen | 5951290 | 2008-09-29 22:23:33 +0200 | [diff] [blame] | 202 | struct op_x86_model_spec op_ppro_spec = { |
Robert Richter | 5a28939 | 2008-10-15 22:19:41 +0200 | [diff] [blame] | 203 | .num_counters = 2, /* can be overriden */ |
| 204 | .num_controls = 2, /* dito */ |
Robert Richter | c92960f | 2008-09-05 17:12:36 +0200 | [diff] [blame] | 205 | .fill_in_addresses = &ppro_fill_in_addresses, |
| 206 | .setup_ctrs = &ppro_setup_ctrs, |
| 207 | .check_ctrs = &ppro_check_ctrs, |
| 208 | .start = &ppro_start, |
| 209 | .stop = &ppro_stop, |
| 210 | .shutdown = &ppro_shutdown |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 211 | }; |
Andi Kleen | b991702 | 2008-08-18 14:50:31 +0200 | [diff] [blame] | 212 | |
| 213 | /* |
| 214 | * Architectural performance monitoring. |
| 215 | * |
| 216 | * Newer Intel CPUs (Core1+) have support for architectural |
| 217 | * events described in CPUID 0xA. See the IA32 SDM Vol3b.18 for details. |
| 218 | * The advantage of this is that it can be done without knowing about |
| 219 | * the specific CPU. |
| 220 | */ |
| 221 | |
| 222 | void arch_perfmon_setup_counters(void) |
| 223 | { |
| 224 | union cpuid10_eax eax; |
| 225 | |
| 226 | eax.full = cpuid_eax(0xa); |
| 227 | |
| 228 | /* Workaround for BIOS bugs in 6/15. Taken from perfmon2 */ |
| 229 | if (eax.split.version_id == 0 && current_cpu_data.x86 == 6 && |
| 230 | current_cpu_data.x86_model == 15) { |
| 231 | eax.split.version_id = 2; |
| 232 | eax.split.num_counters = 2; |
| 233 | eax.split.bit_width = 40; |
| 234 | } |
| 235 | |
| 236 | num_counters = eax.split.num_counters; |
| 237 | |
| 238 | op_arch_perfmon_spec.num_counters = num_counters; |
| 239 | op_arch_perfmon_spec.num_controls = num_counters; |
Andi Kleen | 5951290 | 2008-09-29 22:23:33 +0200 | [diff] [blame] | 240 | op_ppro_spec.num_counters = num_counters; |
| 241 | op_ppro_spec.num_controls = num_counters; |
Andi Kleen | b991702 | 2008-08-18 14:50:31 +0200 | [diff] [blame] | 242 | } |
| 243 | |
| 244 | struct op_x86_model_spec op_arch_perfmon_spec = { |
| 245 | /* num_counters/num_controls filled in at runtime */ |
Robert Richter | 5a28939 | 2008-10-15 22:19:41 +0200 | [diff] [blame] | 246 | .fill_in_addresses = &ppro_fill_in_addresses, |
Andi Kleen | b991702 | 2008-08-18 14:50:31 +0200 | [diff] [blame] | 247 | /* user space does the cpuid check for available events */ |
Robert Richter | 5a28939 | 2008-10-15 22:19:41 +0200 | [diff] [blame] | 248 | .setup_ctrs = &ppro_setup_ctrs, |
| 249 | .check_ctrs = &ppro_check_ctrs, |
| 250 | .start = &ppro_start, |
| 251 | .stop = &ppro_stop, |
| 252 | .shutdown = &ppro_shutdown |
Andi Kleen | b991702 | 2008-08-18 14:50:31 +0200 | [diff] [blame] | 253 | }; |