Paolo Ciarrocchi | 8b45b72 | 2008-02-19 23:43:25 +0100 | [diff] [blame] | 1 | /* |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 2 | * @file op_model_ppro.h |
Andi Kleen | b991702 | 2008-08-18 14:50:31 +0200 | [diff] [blame] | 3 | * Family 6 perfmon and architectural perfmon MSR operations |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 4 | * |
| 5 | * @remark Copyright 2002 OProfile authors |
Andi Kleen | b991702 | 2008-08-18 14:50:31 +0200 | [diff] [blame] | 6 | * @remark Copyright 2008 Intel Corporation |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 7 | * @remark Read the file COPYING |
| 8 | * |
| 9 | * @author John Levon |
| 10 | * @author Philippe Elie |
| 11 | * @author Graydon Hoare |
Andi Kleen | b991702 | 2008-08-18 14:50:31 +0200 | [diff] [blame] | 12 | * @author Andi Kleen |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 13 | */ |
| 14 | |
| 15 | #include <linux/oprofile.h> |
Andi Kleen | b991702 | 2008-08-18 14:50:31 +0200 | [diff] [blame] | 16 | #include <linux/slab.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 17 | #include <asm/ptrace.h> |
| 18 | #include <asm/msr.h> |
| 19 | #include <asm/apic.h> |
Don Zickus | 3e4ff11 | 2006-06-26 13:57:01 +0200 | [diff] [blame] | 20 | #include <asm/nmi.h> |
Andi Kleen | b991702 | 2008-08-18 14:50:31 +0200 | [diff] [blame] | 21 | #include <asm/intel_arch_perfmon.h> |
Paolo Ciarrocchi | 8b45b72 | 2008-02-19 23:43:25 +0100 | [diff] [blame] | 22 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 23 | #include "op_x86_model.h" |
| 24 | #include "op_counter.h" |
| 25 | |
Andi Kleen | b991702 | 2008-08-18 14:50:31 +0200 | [diff] [blame] | 26 | static int num_counters = 2; |
| 27 | static int counter_width = 32; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 28 | |
Paolo Ciarrocchi | 8b45b72 | 2008-02-19 23:43:25 +0100 | [diff] [blame] | 29 | #define CTR_IS_RESERVED(msrs, c) (msrs->counters[(c)].addr ? 1 : 0) |
| 30 | #define CTR_READ(l, h, msrs, c) do {rdmsr(msrs->counters[(c)].addr, (l), (h)); } while (0) |
Andi Kleen | b991702 | 2008-08-18 14:50:31 +0200 | [diff] [blame] | 31 | #define CTR_OVERFLOWED(n) (!((n) & (1U<<(counter_width-1)))) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 32 | |
Paolo Ciarrocchi | 8b45b72 | 2008-02-19 23:43:25 +0100 | [diff] [blame] | 33 | #define CTRL_IS_RESERVED(msrs, c) (msrs->controls[(c)].addr ? 1 : 0) |
| 34 | #define CTRL_READ(l, h, msrs, c) do {rdmsr((msrs->controls[(c)].addr), (l), (h)); } while (0) |
| 35 | #define CTRL_WRITE(l, h, msrs, c) do {wrmsr((msrs->controls[(c)].addr), (l), (h)); } while (0) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 36 | #define CTRL_SET_ACTIVE(n) (n |= (1<<22)) |
| 37 | #define CTRL_SET_INACTIVE(n) (n &= ~(1<<22)) |
| 38 | #define CTRL_CLEAR(x) (x &= (1<<21)) |
| 39 | #define CTRL_SET_ENABLE(val) (val |= 1<<20) |
Paolo Ciarrocchi | 8b45b72 | 2008-02-19 23:43:25 +0100 | [diff] [blame] | 40 | #define CTRL_SET_USR(val, u) (val |= ((u & 1) << 16)) |
| 41 | #define CTRL_SET_KERN(val, k) (val |= ((k & 1) << 17)) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 42 | #define CTRL_SET_UM(val, m) (val |= (m << 8)) |
| 43 | #define CTRL_SET_EVENT(val, e) (val |= e) |
| 44 | |
Andi Kleen | b991702 | 2008-08-18 14:50:31 +0200 | [diff] [blame] | 45 | static u64 *reset_value; |
Paolo Ciarrocchi | 8b45b72 | 2008-02-19 23:43:25 +0100 | [diff] [blame] | 46 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 47 | static void ppro_fill_in_addresses(struct op_msrs * const msrs) |
| 48 | { |
Don Zickus | cb9c448 | 2006-09-26 10:52:26 +0200 | [diff] [blame] | 49 | int i; |
| 50 | |
Andi Kleen | b991702 | 2008-08-18 14:50:31 +0200 | [diff] [blame] | 51 | for (i = 0; i < num_counters; i++) { |
Don Zickus | cb9c448 | 2006-09-26 10:52:26 +0200 | [diff] [blame] | 52 | if (reserve_perfctr_nmi(MSR_P6_PERFCTR0 + i)) |
| 53 | msrs->counters[i].addr = MSR_P6_PERFCTR0 + i; |
| 54 | else |
| 55 | msrs->counters[i].addr = 0; |
| 56 | } |
Paolo Ciarrocchi | 8b45b72 | 2008-02-19 23:43:25 +0100 | [diff] [blame] | 57 | |
Andi Kleen | b991702 | 2008-08-18 14:50:31 +0200 | [diff] [blame] | 58 | for (i = 0; i < num_counters; i++) { |
Don Zickus | cb9c448 | 2006-09-26 10:52:26 +0200 | [diff] [blame] | 59 | if (reserve_evntsel_nmi(MSR_P6_EVNTSEL0 + i)) |
| 60 | msrs->controls[i].addr = MSR_P6_EVNTSEL0 + i; |
| 61 | else |
| 62 | msrs->controls[i].addr = 0; |
| 63 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 64 | } |
| 65 | |
| 66 | |
| 67 | static void ppro_setup_ctrs(struct op_msrs const * const msrs) |
| 68 | { |
| 69 | unsigned int low, high; |
| 70 | int i; |
| 71 | |
Andi Kleen | b991702 | 2008-08-18 14:50:31 +0200 | [diff] [blame] | 72 | if (!reset_value) { |
| 73 | reset_value = kmalloc(sizeof(unsigned) * num_counters, |
| 74 | GFP_ATOMIC); |
| 75 | if (!reset_value) |
| 76 | return; |
| 77 | } |
| 78 | |
| 79 | if (cpu_has_arch_perfmon) { |
| 80 | union cpuid10_eax eax; |
| 81 | eax.full = cpuid_eax(0xa); |
| 82 | if (counter_width < eax.split.bit_width) |
| 83 | counter_width = eax.split.bit_width; |
| 84 | } |
| 85 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 86 | /* clear all counters */ |
Andi Kleen | b991702 | 2008-08-18 14:50:31 +0200 | [diff] [blame] | 87 | for (i = 0 ; i < num_counters; ++i) { |
Paolo Ciarrocchi | 8b45b72 | 2008-02-19 23:43:25 +0100 | [diff] [blame] | 88 | if (unlikely(!CTRL_IS_RESERVED(msrs, i))) |
Don Zickus | cb9c448 | 2006-09-26 10:52:26 +0200 | [diff] [blame] | 89 | continue; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 90 | CTRL_READ(low, high, msrs, i); |
| 91 | CTRL_CLEAR(low); |
| 92 | CTRL_WRITE(low, high, msrs, i); |
| 93 | } |
Paolo Ciarrocchi | 8b45b72 | 2008-02-19 23:43:25 +0100 | [diff] [blame] | 94 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 95 | /* avoid a false detection of ctr overflows in NMI handler */ |
Andi Kleen | b991702 | 2008-08-18 14:50:31 +0200 | [diff] [blame] | 96 | for (i = 0; i < num_counters; ++i) { |
Paolo Ciarrocchi | 8b45b72 | 2008-02-19 23:43:25 +0100 | [diff] [blame] | 97 | if (unlikely(!CTR_IS_RESERVED(msrs, i))) |
Don Zickus | cb9c448 | 2006-09-26 10:52:26 +0200 | [diff] [blame] | 98 | continue; |
Andi Kleen | b991702 | 2008-08-18 14:50:31 +0200 | [diff] [blame] | 99 | wrmsrl(msrs->counters[i].addr, -1LL); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 100 | } |
| 101 | |
| 102 | /* enable active counters */ |
Andi Kleen | b991702 | 2008-08-18 14:50:31 +0200 | [diff] [blame] | 103 | for (i = 0; i < num_counters; ++i) { |
Paolo Ciarrocchi | 8b45b72 | 2008-02-19 23:43:25 +0100 | [diff] [blame] | 104 | if ((counter_config[i].enabled) && (CTR_IS_RESERVED(msrs, i))) { |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 105 | reset_value[i] = counter_config[i].count; |
| 106 | |
Andi Kleen | b991702 | 2008-08-18 14:50:31 +0200 | [diff] [blame] | 107 | wrmsrl(msrs->counters[i].addr, -reset_value[i]); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 108 | |
| 109 | CTRL_READ(low, high, msrs, i); |
| 110 | CTRL_CLEAR(low); |
| 111 | CTRL_SET_ENABLE(low); |
| 112 | CTRL_SET_USR(low, counter_config[i].user); |
| 113 | CTRL_SET_KERN(low, counter_config[i].kernel); |
| 114 | CTRL_SET_UM(low, counter_config[i].unit_mask); |
| 115 | CTRL_SET_EVENT(low, counter_config[i].event); |
| 116 | CTRL_WRITE(low, high, msrs, i); |
Don Zickus | cb9c448 | 2006-09-26 10:52:26 +0200 | [diff] [blame] | 117 | } else { |
| 118 | reset_value[i] = 0; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 119 | } |
| 120 | } |
| 121 | } |
| 122 | |
Paolo Ciarrocchi | 8b45b72 | 2008-02-19 23:43:25 +0100 | [diff] [blame] | 123 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 124 | static int ppro_check_ctrs(struct pt_regs * const regs, |
| 125 | struct op_msrs const * const msrs) |
| 126 | { |
| 127 | unsigned int low, high; |
| 128 | int i; |
Paolo Ciarrocchi | 8b45b72 | 2008-02-19 23:43:25 +0100 | [diff] [blame] | 129 | |
Andi Kleen | b991702 | 2008-08-18 14:50:31 +0200 | [diff] [blame] | 130 | for (i = 0 ; i < num_counters; ++i) { |
Don Zickus | cb9c448 | 2006-09-26 10:52:26 +0200 | [diff] [blame] | 131 | if (!reset_value[i]) |
| 132 | continue; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 133 | CTR_READ(low, high, msrs, i); |
| 134 | if (CTR_OVERFLOWED(low)) { |
| 135 | oprofile_add_sample(regs, i); |
Andi Kleen | b991702 | 2008-08-18 14:50:31 +0200 | [diff] [blame] | 136 | wrmsrl(msrs->counters[i].addr, -reset_value[i]); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 137 | } |
| 138 | } |
| 139 | |
| 140 | /* Only P6 based Pentium M need to re-unmask the apic vector but it |
| 141 | * doesn't hurt other P6 variant */ |
| 142 | apic_write(APIC_LVTPC, apic_read(APIC_LVTPC) & ~APIC_LVT_MASKED); |
| 143 | |
| 144 | /* We can't work out if we really handled an interrupt. We |
| 145 | * might have caught a *second* counter just after overflowing |
| 146 | * the interrupt for this counter then arrives |
| 147 | * and we don't find a counter that's overflowed, so we |
| 148 | * would return 0 and get dazed + confused. Instead we always |
| 149 | * assume we found an overflow. This sucks. |
| 150 | */ |
| 151 | return 1; |
| 152 | } |
| 153 | |
Paolo Ciarrocchi | 8b45b72 | 2008-02-19 23:43:25 +0100 | [diff] [blame] | 154 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 155 | static void ppro_start(struct op_msrs const * const msrs) |
| 156 | { |
Paolo Ciarrocchi | 8b45b72 | 2008-02-19 23:43:25 +0100 | [diff] [blame] | 157 | unsigned int low, high; |
Arun Sharma | 6b77df0 | 2006-09-29 02:00:01 -0700 | [diff] [blame] | 158 | int i; |
Don Zickus | cb9c448 | 2006-09-26 10:52:26 +0200 | [diff] [blame] | 159 | |
Andi Kleen | b991702 | 2008-08-18 14:50:31 +0200 | [diff] [blame] | 160 | for (i = 0; i < num_counters; ++i) { |
Arun Sharma | 6b77df0 | 2006-09-29 02:00:01 -0700 | [diff] [blame] | 161 | if (reset_value[i]) { |
| 162 | CTRL_READ(low, high, msrs, i); |
| 163 | CTRL_SET_ACTIVE(low); |
| 164 | CTRL_WRITE(low, high, msrs, i); |
| 165 | } |
Don Zickus | cb9c448 | 2006-09-26 10:52:26 +0200 | [diff] [blame] | 166 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 167 | } |
| 168 | |
| 169 | |
| 170 | static void ppro_stop(struct op_msrs const * const msrs) |
| 171 | { |
Paolo Ciarrocchi | 8b45b72 | 2008-02-19 23:43:25 +0100 | [diff] [blame] | 172 | unsigned int low, high; |
Arun Sharma | 6b77df0 | 2006-09-29 02:00:01 -0700 | [diff] [blame] | 173 | int i; |
Don Zickus | cb9c448 | 2006-09-26 10:52:26 +0200 | [diff] [blame] | 174 | |
Andi Kleen | b991702 | 2008-08-18 14:50:31 +0200 | [diff] [blame] | 175 | for (i = 0; i < num_counters; ++i) { |
Arun Sharma | 6b77df0 | 2006-09-29 02:00:01 -0700 | [diff] [blame] | 176 | if (!reset_value[i]) |
| 177 | continue; |
| 178 | CTRL_READ(low, high, msrs, i); |
Don Zickus | cb9c448 | 2006-09-26 10:52:26 +0200 | [diff] [blame] | 179 | CTRL_SET_INACTIVE(low); |
Arun Sharma | 6b77df0 | 2006-09-29 02:00:01 -0700 | [diff] [blame] | 180 | CTRL_WRITE(low, high, msrs, i); |
Don Zickus | cb9c448 | 2006-09-26 10:52:26 +0200 | [diff] [blame] | 181 | } |
| 182 | } |
| 183 | |
| 184 | static void ppro_shutdown(struct op_msrs const * const msrs) |
| 185 | { |
| 186 | int i; |
| 187 | |
Andi Kleen | b991702 | 2008-08-18 14:50:31 +0200 | [diff] [blame] | 188 | for (i = 0 ; i < num_counters ; ++i) { |
Paolo Ciarrocchi | 8b45b72 | 2008-02-19 23:43:25 +0100 | [diff] [blame] | 189 | if (CTR_IS_RESERVED(msrs, i)) |
Don Zickus | cb9c448 | 2006-09-26 10:52:26 +0200 | [diff] [blame] | 190 | release_perfctr_nmi(MSR_P6_PERFCTR0 + i); |
| 191 | } |
Andi Kleen | b991702 | 2008-08-18 14:50:31 +0200 | [diff] [blame] | 192 | for (i = 0 ; i < num_counters ; ++i) { |
Paolo Ciarrocchi | 8b45b72 | 2008-02-19 23:43:25 +0100 | [diff] [blame] | 193 | if (CTRL_IS_RESERVED(msrs, i)) |
Don Zickus | cb9c448 | 2006-09-26 10:52:26 +0200 | [diff] [blame] | 194 | release_evntsel_nmi(MSR_P6_EVNTSEL0 + i); |
| 195 | } |
Andi Kleen | b991702 | 2008-08-18 14:50:31 +0200 | [diff] [blame] | 196 | if (reset_value) { |
| 197 | kfree(reset_value); |
| 198 | reset_value = NULL; |
| 199 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 200 | } |
| 201 | |
| 202 | |
Andi Kleen | 5951290 | 2008-09-29 22:23:33 +0200 | [diff] [blame] | 203 | struct op_x86_model_spec op_ppro_spec = { |
Robert Richter | 5a28939 | 2008-10-15 22:19:41 +0200 | [diff] [blame] | 204 | .num_counters = 2, /* can be overriden */ |
| 205 | .num_controls = 2, /* dito */ |
Robert Richter | c92960f | 2008-09-05 17:12:36 +0200 | [diff] [blame] | 206 | .fill_in_addresses = &ppro_fill_in_addresses, |
| 207 | .setup_ctrs = &ppro_setup_ctrs, |
| 208 | .check_ctrs = &ppro_check_ctrs, |
| 209 | .start = &ppro_start, |
| 210 | .stop = &ppro_stop, |
| 211 | .shutdown = &ppro_shutdown |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 212 | }; |
Andi Kleen | b991702 | 2008-08-18 14:50:31 +0200 | [diff] [blame] | 213 | |
| 214 | /* |
| 215 | * Architectural performance monitoring. |
| 216 | * |
| 217 | * Newer Intel CPUs (Core1+) have support for architectural |
| 218 | * events described in CPUID 0xA. See the IA32 SDM Vol3b.18 for details. |
| 219 | * The advantage of this is that it can be done without knowing about |
| 220 | * the specific CPU. |
| 221 | */ |
| 222 | |
| 223 | void arch_perfmon_setup_counters(void) |
| 224 | { |
| 225 | union cpuid10_eax eax; |
| 226 | |
| 227 | eax.full = cpuid_eax(0xa); |
| 228 | |
| 229 | /* Workaround for BIOS bugs in 6/15. Taken from perfmon2 */ |
| 230 | if (eax.split.version_id == 0 && current_cpu_data.x86 == 6 && |
| 231 | current_cpu_data.x86_model == 15) { |
| 232 | eax.split.version_id = 2; |
| 233 | eax.split.num_counters = 2; |
| 234 | eax.split.bit_width = 40; |
| 235 | } |
| 236 | |
| 237 | num_counters = eax.split.num_counters; |
| 238 | |
| 239 | op_arch_perfmon_spec.num_counters = num_counters; |
| 240 | op_arch_perfmon_spec.num_controls = num_counters; |
Andi Kleen | 5951290 | 2008-09-29 22:23:33 +0200 | [diff] [blame] | 241 | op_ppro_spec.num_counters = num_counters; |
| 242 | op_ppro_spec.num_controls = num_counters; |
Andi Kleen | b991702 | 2008-08-18 14:50:31 +0200 | [diff] [blame] | 243 | } |
| 244 | |
| 245 | struct op_x86_model_spec op_arch_perfmon_spec = { |
| 246 | /* num_counters/num_controls filled in at runtime */ |
Robert Richter | 5a28939 | 2008-10-15 22:19:41 +0200 | [diff] [blame] | 247 | .fill_in_addresses = &ppro_fill_in_addresses, |
Andi Kleen | b991702 | 2008-08-18 14:50:31 +0200 | [diff] [blame] | 248 | /* user space does the cpuid check for available events */ |
Robert Richter | 5a28939 | 2008-10-15 22:19:41 +0200 | [diff] [blame] | 249 | .setup_ctrs = &ppro_setup_ctrs, |
| 250 | .check_ctrs = &ppro_check_ctrs, |
| 251 | .start = &ppro_start, |
| 252 | .stop = &ppro_stop, |
| 253 | .shutdown = &ppro_shutdown |
Andi Kleen | b991702 | 2008-08-18 14:50:31 +0200 | [diff] [blame] | 254 | }; |