blob: fb0082ab40a7ea5a452c67ad612b33a8dddba76d [file] [log] [blame]
Will Deacon478fcb22012-03-05 11:49:33 +00001/*
2 * HW_breakpoint: a unified kernel/user-space hardware breakpoint facility,
3 * using the CPU's debug registers.
4 *
5 * Copyright (C) 2012 ARM Limited
6 * Author: Will Deacon <will.deacon@arm.com>
7 *
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License version 2 as
10 * published by the Free Software Foundation.
11 *
12 * This program is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 * GNU General Public License for more details.
16 *
17 * You should have received a copy of the GNU General Public License
18 * along with this program. If not, see <http://www.gnu.org/licenses/>.
19 */
20
21#define pr_fmt(fmt) "hw-breakpoint: " fmt
22
AKASHI Takahirofd92d4a2014-04-30 10:51:32 +010023#include <linux/compat.h>
Lorenzo Pieralisi60fc6942013-08-05 15:20:35 +010024#include <linux/cpu_pm.h>
Will Deacon478fcb22012-03-05 11:49:33 +000025#include <linux/errno.h>
26#include <linux/hw_breakpoint.h>
Pratyush Anand44b53f62016-07-08 12:35:49 -040027#include <linux/kprobes.h>
Will Deacon478fcb22012-03-05 11:49:33 +000028#include <linux/perf_event.h>
29#include <linux/ptrace.h>
30#include <linux/smp.h>
31
Catalin Marinascb50ce32015-10-12 12:10:53 +010032#include <asm/compat.h>
Will Deacon478fcb22012-03-05 11:49:33 +000033#include <asm/current.h>
34#include <asm/debug-monitors.h>
35#include <asm/hw_breakpoint.h>
Will Deacon478fcb22012-03-05 11:49:33 +000036#include <asm/traps.h>
37#include <asm/cputype.h>
38#include <asm/system_misc.h>
Kristina Martsenko1d61ccb2017-06-06 20:14:09 +010039#include <asm/uaccess.h>
Will Deacon478fcb22012-03-05 11:49:33 +000040
41/* Breakpoint currently in use for each BRP. */
42static DEFINE_PER_CPU(struct perf_event *, bp_on_reg[ARM_MAX_BRP]);
43
44/* Watchpoint currently in use for each WRP. */
45static DEFINE_PER_CPU(struct perf_event *, wp_on_reg[ARM_MAX_WRP]);
46
47/* Currently stepping a per-CPU kernel breakpoint. */
48static DEFINE_PER_CPU(int, stepping_kernel_bp);
49
50/* Number of BRP/WRP registers on this CPU. */
51static int core_num_brps;
52static int core_num_wrps;
53
Will Deacon478fcb22012-03-05 11:49:33 +000054int hw_breakpoint_slots(int type)
55{
56 /*
57 * We can be called early, so don't rely on
58 * our static variables being initialised.
59 */
60 switch (type) {
61 case TYPE_INST:
62 return get_num_brps();
63 case TYPE_DATA:
64 return get_num_wrps();
65 default:
66 pr_warning("unknown slot type: %d\n", type);
67 return 0;
68 }
69}
70
71#define READ_WB_REG_CASE(OFF, N, REG, VAL) \
72 case (OFF + N): \
73 AARCH64_DBG_READ(N, REG, VAL); \
74 break
75
76#define WRITE_WB_REG_CASE(OFF, N, REG, VAL) \
77 case (OFF + N): \
78 AARCH64_DBG_WRITE(N, REG, VAL); \
79 break
80
81#define GEN_READ_WB_REG_CASES(OFF, REG, VAL) \
82 READ_WB_REG_CASE(OFF, 0, REG, VAL); \
83 READ_WB_REG_CASE(OFF, 1, REG, VAL); \
84 READ_WB_REG_CASE(OFF, 2, REG, VAL); \
85 READ_WB_REG_CASE(OFF, 3, REG, VAL); \
86 READ_WB_REG_CASE(OFF, 4, REG, VAL); \
87 READ_WB_REG_CASE(OFF, 5, REG, VAL); \
88 READ_WB_REG_CASE(OFF, 6, REG, VAL); \
89 READ_WB_REG_CASE(OFF, 7, REG, VAL); \
90 READ_WB_REG_CASE(OFF, 8, REG, VAL); \
91 READ_WB_REG_CASE(OFF, 9, REG, VAL); \
92 READ_WB_REG_CASE(OFF, 10, REG, VAL); \
93 READ_WB_REG_CASE(OFF, 11, REG, VAL); \
94 READ_WB_REG_CASE(OFF, 12, REG, VAL); \
95 READ_WB_REG_CASE(OFF, 13, REG, VAL); \
96 READ_WB_REG_CASE(OFF, 14, REG, VAL); \
97 READ_WB_REG_CASE(OFF, 15, REG, VAL)
98
99#define GEN_WRITE_WB_REG_CASES(OFF, REG, VAL) \
100 WRITE_WB_REG_CASE(OFF, 0, REG, VAL); \
101 WRITE_WB_REG_CASE(OFF, 1, REG, VAL); \
102 WRITE_WB_REG_CASE(OFF, 2, REG, VAL); \
103 WRITE_WB_REG_CASE(OFF, 3, REG, VAL); \
104 WRITE_WB_REG_CASE(OFF, 4, REG, VAL); \
105 WRITE_WB_REG_CASE(OFF, 5, REG, VAL); \
106 WRITE_WB_REG_CASE(OFF, 6, REG, VAL); \
107 WRITE_WB_REG_CASE(OFF, 7, REG, VAL); \
108 WRITE_WB_REG_CASE(OFF, 8, REG, VAL); \
109 WRITE_WB_REG_CASE(OFF, 9, REG, VAL); \
110 WRITE_WB_REG_CASE(OFF, 10, REG, VAL); \
111 WRITE_WB_REG_CASE(OFF, 11, REG, VAL); \
112 WRITE_WB_REG_CASE(OFF, 12, REG, VAL); \
113 WRITE_WB_REG_CASE(OFF, 13, REG, VAL); \
114 WRITE_WB_REG_CASE(OFF, 14, REG, VAL); \
115 WRITE_WB_REG_CASE(OFF, 15, REG, VAL)
116
117static u64 read_wb_reg(int reg, int n)
118{
119 u64 val = 0;
120
121 switch (reg + n) {
122 GEN_READ_WB_REG_CASES(AARCH64_DBG_REG_BVR, AARCH64_DBG_REG_NAME_BVR, val);
123 GEN_READ_WB_REG_CASES(AARCH64_DBG_REG_BCR, AARCH64_DBG_REG_NAME_BCR, val);
124 GEN_READ_WB_REG_CASES(AARCH64_DBG_REG_WVR, AARCH64_DBG_REG_NAME_WVR, val);
125 GEN_READ_WB_REG_CASES(AARCH64_DBG_REG_WCR, AARCH64_DBG_REG_NAME_WCR, val);
126 default:
127 pr_warning("attempt to read from unknown breakpoint register %d\n", n);
128 }
129
130 return val;
131}
Pratyush Anand44b53f62016-07-08 12:35:49 -0400132NOKPROBE_SYMBOL(read_wb_reg);
Will Deacon478fcb22012-03-05 11:49:33 +0000133
134static void write_wb_reg(int reg, int n, u64 val)
135{
136 switch (reg + n) {
137 GEN_WRITE_WB_REG_CASES(AARCH64_DBG_REG_BVR, AARCH64_DBG_REG_NAME_BVR, val);
138 GEN_WRITE_WB_REG_CASES(AARCH64_DBG_REG_BCR, AARCH64_DBG_REG_NAME_BCR, val);
139 GEN_WRITE_WB_REG_CASES(AARCH64_DBG_REG_WVR, AARCH64_DBG_REG_NAME_WVR, val);
140 GEN_WRITE_WB_REG_CASES(AARCH64_DBG_REG_WCR, AARCH64_DBG_REG_NAME_WCR, val);
141 default:
142 pr_warning("attempt to write to unknown breakpoint register %d\n", n);
143 }
144 isb();
145}
Pratyush Anand44b53f62016-07-08 12:35:49 -0400146NOKPROBE_SYMBOL(write_wb_reg);
Will Deacon478fcb22012-03-05 11:49:33 +0000147
148/*
149 * Convert a breakpoint privilege level to the corresponding exception
150 * level.
151 */
Will Deacon6f883d12015-07-27 18:36:54 +0100152static enum dbg_active_el debug_exception_level(int privilege)
Will Deacon478fcb22012-03-05 11:49:33 +0000153{
154 switch (privilege) {
155 case AARCH64_BREAKPOINT_EL0:
156 return DBG_ACTIVE_EL0;
157 case AARCH64_BREAKPOINT_EL1:
158 return DBG_ACTIVE_EL1;
159 default:
160 pr_warning("invalid breakpoint privilege level %d\n", privilege);
161 return -EINVAL;
162 }
163}
Pratyush Anand44b53f62016-07-08 12:35:49 -0400164NOKPROBE_SYMBOL(debug_exception_level);
Will Deacon478fcb22012-03-05 11:49:33 +0000165
Lorenzo Pieralisi2f043042013-08-13 10:45:19 +0100166enum hw_breakpoint_ops {
167 HW_BREAKPOINT_INSTALL,
Lorenzo Pieralisi60fc6942013-08-05 15:20:35 +0100168 HW_BREAKPOINT_UNINSTALL,
169 HW_BREAKPOINT_RESTORE
Lorenzo Pieralisi2f043042013-08-13 10:45:19 +0100170};
171
Will Deacon8f48c062015-10-07 11:37:36 +0100172static int is_compat_bp(struct perf_event *bp)
173{
174 struct task_struct *tsk = bp->hw.target;
175
176 /*
177 * tsk can be NULL for per-cpu (non-ptrace) breakpoints.
178 * In this case, use the native interface, since we don't have
179 * the notion of a "compat CPU" and could end up relying on
180 * deprecated behaviour if we use unaligned watchpoints in
181 * AArch64 state.
182 */
183 return tsk && is_compat_thread(task_thread_info(tsk));
184}
185
Lorenzo Pieralisi2f043042013-08-13 10:45:19 +0100186/**
187 * hw_breakpoint_slot_setup - Find and setup a perf slot according to
188 * operations
189 *
190 * @slots: pointer to array of slots
191 * @max_slots: max number of slots
192 * @bp: perf_event to setup
193 * @ops: operation to be carried out on the slot
194 *
195 * Return:
196 * slot index on success
197 * -ENOSPC if no slot is available/matches
198 * -EINVAL on wrong operations parameter
Will Deacon478fcb22012-03-05 11:49:33 +0000199 */
Lorenzo Pieralisi2f043042013-08-13 10:45:19 +0100200static int hw_breakpoint_slot_setup(struct perf_event **slots, int max_slots,
201 struct perf_event *bp,
202 enum hw_breakpoint_ops ops)
203{
204 int i;
205 struct perf_event **slot;
206
207 for (i = 0; i < max_slots; ++i) {
208 slot = &slots[i];
209 switch (ops) {
210 case HW_BREAKPOINT_INSTALL:
211 if (!*slot) {
212 *slot = bp;
213 return i;
214 }
215 break;
216 case HW_BREAKPOINT_UNINSTALL:
217 if (*slot == bp) {
218 *slot = NULL;
219 return i;
220 }
221 break;
Lorenzo Pieralisi60fc6942013-08-05 15:20:35 +0100222 case HW_BREAKPOINT_RESTORE:
223 if (*slot == bp)
224 return i;
225 break;
Lorenzo Pieralisi2f043042013-08-13 10:45:19 +0100226 default:
227 pr_warn_once("Unhandled hw breakpoint ops %d\n", ops);
228 return -EINVAL;
229 }
230 }
231 return -ENOSPC;
232}
233
234static int hw_breakpoint_control(struct perf_event *bp,
235 enum hw_breakpoint_ops ops)
Will Deacon478fcb22012-03-05 11:49:33 +0000236{
237 struct arch_hw_breakpoint *info = counter_arch_bp(bp);
Lorenzo Pieralisi2f043042013-08-13 10:45:19 +0100238 struct perf_event **slots;
Will Deacon478fcb22012-03-05 11:49:33 +0000239 struct debug_info *debug_info = &current->thread.debug;
240 int i, max_slots, ctrl_reg, val_reg, reg_enable;
Will Deacon6f883d12015-07-27 18:36:54 +0100241 enum dbg_active_el dbg_el = debug_exception_level(info->ctrl.privilege);
Will Deacon478fcb22012-03-05 11:49:33 +0000242 u32 ctrl;
243
244 if (info->ctrl.type == ARM_BREAKPOINT_EXECUTE) {
245 /* Breakpoint */
246 ctrl_reg = AARCH64_DBG_REG_BCR;
247 val_reg = AARCH64_DBG_REG_BVR;
Christoph Lameter1436c1a2013-10-21 13:17:08 +0100248 slots = this_cpu_ptr(bp_on_reg);
Will Deacon478fcb22012-03-05 11:49:33 +0000249 max_slots = core_num_brps;
250 reg_enable = !debug_info->bps_disabled;
251 } else {
252 /* Watchpoint */
253 ctrl_reg = AARCH64_DBG_REG_WCR;
254 val_reg = AARCH64_DBG_REG_WVR;
Christoph Lameter1436c1a2013-10-21 13:17:08 +0100255 slots = this_cpu_ptr(wp_on_reg);
Will Deacon478fcb22012-03-05 11:49:33 +0000256 max_slots = core_num_wrps;
257 reg_enable = !debug_info->wps_disabled;
258 }
259
Lorenzo Pieralisi2f043042013-08-13 10:45:19 +0100260 i = hw_breakpoint_slot_setup(slots, max_slots, bp, ops);
Will Deacon478fcb22012-03-05 11:49:33 +0000261
Lorenzo Pieralisi2f043042013-08-13 10:45:19 +0100262 if (WARN_ONCE(i < 0, "Can't find any breakpoint slot"))
263 return i;
264
265 switch (ops) {
266 case HW_BREAKPOINT_INSTALL:
267 /*
268 * Ensure debug monitors are enabled at the correct exception
269 * level.
270 */
271 enable_debug_monitors(dbg_el);
Lorenzo Pieralisi60fc6942013-08-05 15:20:35 +0100272 /* Fall through */
273 case HW_BREAKPOINT_RESTORE:
Lorenzo Pieralisi2f043042013-08-13 10:45:19 +0100274 /* Setup the address register. */
275 write_wb_reg(val_reg, i, info->address);
276
277 /* Setup the control register. */
278 ctrl = encode_ctrl_reg(info->ctrl);
279 write_wb_reg(ctrl_reg, i,
280 reg_enable ? ctrl | 0x1 : ctrl & ~0x1);
281 break;
282 case HW_BREAKPOINT_UNINSTALL:
283 /* Reset the control register. */
284 write_wb_reg(ctrl_reg, i, 0);
285
286 /*
287 * Release the debug monitors for the correct exception
288 * level.
289 */
290 disable_debug_monitors(dbg_el);
291 break;
Will Deacon478fcb22012-03-05 11:49:33 +0000292 }
293
Will Deacon478fcb22012-03-05 11:49:33 +0000294 return 0;
295}
296
Lorenzo Pieralisi2f043042013-08-13 10:45:19 +0100297/*
298 * Install a perf counter breakpoint.
299 */
300int arch_install_hw_breakpoint(struct perf_event *bp)
301{
302 return hw_breakpoint_control(bp, HW_BREAKPOINT_INSTALL);
303}
304
Will Deacon478fcb22012-03-05 11:49:33 +0000305void arch_uninstall_hw_breakpoint(struct perf_event *bp)
306{
Lorenzo Pieralisi2f043042013-08-13 10:45:19 +0100307 hw_breakpoint_control(bp, HW_BREAKPOINT_UNINSTALL);
Will Deacon478fcb22012-03-05 11:49:33 +0000308}
309
310static int get_hbp_len(u8 hbp_len)
311{
312 unsigned int len_in_bytes = 0;
313
314 switch (hbp_len) {
315 case ARM_BREAKPOINT_LEN_1:
316 len_in_bytes = 1;
317 break;
318 case ARM_BREAKPOINT_LEN_2:
319 len_in_bytes = 2;
320 break;
Pratyush Anandd32793b2016-11-14 19:32:45 +0530321 case ARM_BREAKPOINT_LEN_3:
322 len_in_bytes = 3;
323 break;
Will Deacon478fcb22012-03-05 11:49:33 +0000324 case ARM_BREAKPOINT_LEN_4:
325 len_in_bytes = 4;
326 break;
Pratyush Anandd32793b2016-11-14 19:32:45 +0530327 case ARM_BREAKPOINT_LEN_5:
328 len_in_bytes = 5;
329 break;
330 case ARM_BREAKPOINT_LEN_6:
331 len_in_bytes = 6;
332 break;
333 case ARM_BREAKPOINT_LEN_7:
334 len_in_bytes = 7;
335 break;
Will Deacon478fcb22012-03-05 11:49:33 +0000336 case ARM_BREAKPOINT_LEN_8:
337 len_in_bytes = 8;
338 break;
339 }
340
341 return len_in_bytes;
342}
343
344/*
345 * Check whether bp virtual address is in kernel space.
346 */
347int arch_check_bp_in_kernelspace(struct perf_event *bp)
348{
349 unsigned int len;
350 unsigned long va;
351 struct arch_hw_breakpoint *info = counter_arch_bp(bp);
352
353 va = info->address;
354 len = get_hbp_len(info->ctrl.len);
355
356 return (va >= TASK_SIZE) && ((va + len - 1) >= TASK_SIZE);
357}
358
359/*
360 * Extract generic type and length encodings from an arch_hw_breakpoint_ctrl.
361 * Hopefully this will disappear when ptrace can bypass the conversion
362 * to generic breakpoint descriptions.
363 */
364int arch_bp_generic_fields(struct arch_hw_breakpoint_ctrl ctrl,
Pratyush Anand67de4de2016-11-14 19:32:43 +0530365 int *gen_len, int *gen_type, int *offset)
Will Deacon478fcb22012-03-05 11:49:33 +0000366{
367 /* Type */
368 switch (ctrl.type) {
369 case ARM_BREAKPOINT_EXECUTE:
370 *gen_type = HW_BREAKPOINT_X;
371 break;
372 case ARM_BREAKPOINT_LOAD:
373 *gen_type = HW_BREAKPOINT_R;
374 break;
375 case ARM_BREAKPOINT_STORE:
376 *gen_type = HW_BREAKPOINT_W;
377 break;
378 case ARM_BREAKPOINT_LOAD | ARM_BREAKPOINT_STORE:
379 *gen_type = HW_BREAKPOINT_RW;
380 break;
381 default:
382 return -EINVAL;
383 }
384
Pratyush Anand67de4de2016-11-14 19:32:43 +0530385 if (!ctrl.len)
386 return -EINVAL;
387 *offset = __ffs(ctrl.len);
388
Will Deacon478fcb22012-03-05 11:49:33 +0000389 /* Len */
Pratyush Anand67de4de2016-11-14 19:32:43 +0530390 switch (ctrl.len >> *offset) {
Will Deacon478fcb22012-03-05 11:49:33 +0000391 case ARM_BREAKPOINT_LEN_1:
392 *gen_len = HW_BREAKPOINT_LEN_1;
393 break;
394 case ARM_BREAKPOINT_LEN_2:
395 *gen_len = HW_BREAKPOINT_LEN_2;
396 break;
Pratyush Anandd32793b2016-11-14 19:32:45 +0530397 case ARM_BREAKPOINT_LEN_3:
398 *gen_len = HW_BREAKPOINT_LEN_3;
399 break;
Will Deacon478fcb22012-03-05 11:49:33 +0000400 case ARM_BREAKPOINT_LEN_4:
401 *gen_len = HW_BREAKPOINT_LEN_4;
402 break;
Pratyush Anandd32793b2016-11-14 19:32:45 +0530403 case ARM_BREAKPOINT_LEN_5:
404 *gen_len = HW_BREAKPOINT_LEN_5;
405 break;
406 case ARM_BREAKPOINT_LEN_6:
407 *gen_len = HW_BREAKPOINT_LEN_6;
408 break;
409 case ARM_BREAKPOINT_LEN_7:
410 *gen_len = HW_BREAKPOINT_LEN_7;
411 break;
Will Deacon478fcb22012-03-05 11:49:33 +0000412 case ARM_BREAKPOINT_LEN_8:
413 *gen_len = HW_BREAKPOINT_LEN_8;
414 break;
415 default:
416 return -EINVAL;
417 }
418
419 return 0;
420}
421
422/*
423 * Construct an arch_hw_breakpoint from a perf_event.
424 */
425static int arch_build_bp_info(struct perf_event *bp)
426{
427 struct arch_hw_breakpoint *info = counter_arch_bp(bp);
428
429 /* Type */
430 switch (bp->attr.bp_type) {
431 case HW_BREAKPOINT_X:
432 info->ctrl.type = ARM_BREAKPOINT_EXECUTE;
433 break;
434 case HW_BREAKPOINT_R:
435 info->ctrl.type = ARM_BREAKPOINT_LOAD;
436 break;
437 case HW_BREAKPOINT_W:
438 info->ctrl.type = ARM_BREAKPOINT_STORE;
439 break;
440 case HW_BREAKPOINT_RW:
441 info->ctrl.type = ARM_BREAKPOINT_LOAD | ARM_BREAKPOINT_STORE;
442 break;
443 default:
444 return -EINVAL;
445 }
446
447 /* Len */
448 switch (bp->attr.bp_len) {
449 case HW_BREAKPOINT_LEN_1:
450 info->ctrl.len = ARM_BREAKPOINT_LEN_1;
451 break;
452 case HW_BREAKPOINT_LEN_2:
453 info->ctrl.len = ARM_BREAKPOINT_LEN_2;
454 break;
Pratyush Anandd32793b2016-11-14 19:32:45 +0530455 case HW_BREAKPOINT_LEN_3:
456 info->ctrl.len = ARM_BREAKPOINT_LEN_3;
457 break;
Will Deacon478fcb22012-03-05 11:49:33 +0000458 case HW_BREAKPOINT_LEN_4:
459 info->ctrl.len = ARM_BREAKPOINT_LEN_4;
460 break;
Pratyush Anandd32793b2016-11-14 19:32:45 +0530461 case HW_BREAKPOINT_LEN_5:
462 info->ctrl.len = ARM_BREAKPOINT_LEN_5;
463 break;
464 case HW_BREAKPOINT_LEN_6:
465 info->ctrl.len = ARM_BREAKPOINT_LEN_6;
466 break;
467 case HW_BREAKPOINT_LEN_7:
468 info->ctrl.len = ARM_BREAKPOINT_LEN_7;
469 break;
Will Deacon478fcb22012-03-05 11:49:33 +0000470 case HW_BREAKPOINT_LEN_8:
471 info->ctrl.len = ARM_BREAKPOINT_LEN_8;
472 break;
473 default:
474 return -EINVAL;
475 }
476
477 /*
478 * On AArch64, we only permit breakpoints of length 4, whereas
479 * AArch32 also requires breakpoints of length 2 for Thumb.
480 * Watchpoints can be of length 1, 2, 4 or 8 bytes.
481 */
482 if (info->ctrl.type == ARM_BREAKPOINT_EXECUTE) {
Will Deacon8f48c062015-10-07 11:37:36 +0100483 if (is_compat_bp(bp)) {
Will Deacon478fcb22012-03-05 11:49:33 +0000484 if (info->ctrl.len != ARM_BREAKPOINT_LEN_2 &&
485 info->ctrl.len != ARM_BREAKPOINT_LEN_4)
486 return -EINVAL;
487 } else if (info->ctrl.len != ARM_BREAKPOINT_LEN_4) {
488 /*
489 * FIXME: Some tools (I'm looking at you perf) assume
490 * that breakpoints should be sizeof(long). This
491 * is nonsense. For now, we fix up the parameter
492 * but we should probably return -EINVAL instead.
493 */
494 info->ctrl.len = ARM_BREAKPOINT_LEN_4;
495 }
496 }
497
498 /* Address */
499 info->address = bp->attr.bp_addr;
500
501 /*
502 * Privilege
503 * Note that we disallow combined EL0/EL1 breakpoints because
504 * that would complicate the stepping code.
505 */
506 if (arch_check_bp_in_kernelspace(bp))
507 info->ctrl.privilege = AARCH64_BREAKPOINT_EL1;
508 else
509 info->ctrl.privilege = AARCH64_BREAKPOINT_EL0;
510
511 /* Enabled? */
512 info->ctrl.enabled = !bp->attr.disabled;
513
514 return 0;
515}
516
517/*
518 * Validate the arch-specific HW Breakpoint register settings.
519 */
520int arch_validate_hwbkpt_settings(struct perf_event *bp)
521{
522 struct arch_hw_breakpoint *info = counter_arch_bp(bp);
523 int ret;
524 u64 alignment_mask, offset;
525
526 /* Build the arch_hw_breakpoint. */
527 ret = arch_build_bp_info(bp);
528 if (ret)
529 return ret;
530
531 /*
532 * Check address alignment.
533 * We don't do any clever alignment correction for watchpoints
534 * because using 64-bit unaligned addresses is deprecated for
535 * AArch64.
536 *
537 * AArch32 tasks expect some simple alignment fixups, so emulate
538 * that here.
539 */
Will Deacon8f48c062015-10-07 11:37:36 +0100540 if (is_compat_bp(bp)) {
Will Deacon478fcb22012-03-05 11:49:33 +0000541 if (info->ctrl.len == ARM_BREAKPOINT_LEN_8)
542 alignment_mask = 0x7;
543 else
544 alignment_mask = 0x3;
545 offset = info->address & alignment_mask;
546 switch (offset) {
547 case 0:
548 /* Aligned */
549 break;
550 case 1:
551 /* Allow single byte watchpoint. */
552 if (info->ctrl.len == ARM_BREAKPOINT_LEN_1)
553 break;
554 case 2:
555 /* Allow halfword watchpoints and breakpoints. */
556 if (info->ctrl.len == ARM_BREAKPOINT_LEN_2)
557 break;
558 default:
559 return -EINVAL;
560 }
Will Deacon478fcb22012-03-05 11:49:33 +0000561 } else {
562 if (info->ctrl.type == ARM_BREAKPOINT_EXECUTE)
563 alignment_mask = 0x3;
564 else
565 alignment_mask = 0x7;
Pratyush Anand67de4de2016-11-14 19:32:43 +0530566 offset = info->address & alignment_mask;
Will Deacon478fcb22012-03-05 11:49:33 +0000567 }
568
Pratyush Anand67de4de2016-11-14 19:32:43 +0530569 info->address &= ~alignment_mask;
570 info->ctrl.len <<= offset;
571
Will Deacon478fcb22012-03-05 11:49:33 +0000572 /*
573 * Disallow per-task kernel breakpoints since these would
574 * complicate the stepping code.
575 */
Peter Zijlstra50f16a82015-03-05 22:10:19 +0100576 if (info->ctrl.privilege == AARCH64_BREAKPOINT_EL1 && bp->hw.target)
Will Deacon478fcb22012-03-05 11:49:33 +0000577 return -EINVAL;
578
579 return 0;
580}
581
582/*
583 * Enable/disable all of the breakpoints active at the specified
584 * exception level at the register level.
585 * This is used when single-stepping after a breakpoint exception.
586 */
Will Deacon6f883d12015-07-27 18:36:54 +0100587static void toggle_bp_registers(int reg, enum dbg_active_el el, int enable)
Will Deacon478fcb22012-03-05 11:49:33 +0000588{
589 int i, max_slots, privilege;
590 u32 ctrl;
591 struct perf_event **slots;
592
593 switch (reg) {
594 case AARCH64_DBG_REG_BCR:
Christoph Lameter1436c1a2013-10-21 13:17:08 +0100595 slots = this_cpu_ptr(bp_on_reg);
Will Deacon478fcb22012-03-05 11:49:33 +0000596 max_slots = core_num_brps;
597 break;
598 case AARCH64_DBG_REG_WCR:
Christoph Lameter1436c1a2013-10-21 13:17:08 +0100599 slots = this_cpu_ptr(wp_on_reg);
Will Deacon478fcb22012-03-05 11:49:33 +0000600 max_slots = core_num_wrps;
601 break;
602 default:
603 return;
604 }
605
606 for (i = 0; i < max_slots; ++i) {
607 if (!slots[i])
608 continue;
609
610 privilege = counter_arch_bp(slots[i])->ctrl.privilege;
611 if (debug_exception_level(privilege) != el)
612 continue;
613
614 ctrl = read_wb_reg(reg, i);
615 if (enable)
616 ctrl |= 0x1;
617 else
618 ctrl &= ~0x1;
619 write_wb_reg(reg, i, ctrl);
620 }
621}
Pratyush Anand44b53f62016-07-08 12:35:49 -0400622NOKPROBE_SYMBOL(toggle_bp_registers);
Will Deacon478fcb22012-03-05 11:49:33 +0000623
624/*
625 * Debug exception handlers.
626 */
627static int breakpoint_handler(unsigned long unused, unsigned int esr,
628 struct pt_regs *regs)
629{
630 int i, step = 0, *kernel_step;
631 u32 ctrl_reg;
632 u64 addr, val;
633 struct perf_event *bp, **slots;
634 struct debug_info *debug_info;
635 struct arch_hw_breakpoint_ctrl ctrl;
636
Christoph Lameter1436c1a2013-10-21 13:17:08 +0100637 slots = this_cpu_ptr(bp_on_reg);
Will Deacon478fcb22012-03-05 11:49:33 +0000638 addr = instruction_pointer(regs);
639 debug_info = &current->thread.debug;
640
641 for (i = 0; i < core_num_brps; ++i) {
642 rcu_read_lock();
643
644 bp = slots[i];
645
646 if (bp == NULL)
647 goto unlock;
648
649 /* Check if the breakpoint value matches. */
650 val = read_wb_reg(AARCH64_DBG_REG_BVR, i);
651 if (val != (addr & ~0x3))
652 goto unlock;
653
654 /* Possible match, check the byte address select to confirm. */
655 ctrl_reg = read_wb_reg(AARCH64_DBG_REG_BCR, i);
656 decode_ctrl_reg(ctrl_reg, &ctrl);
657 if (!((1 << (addr & 0x3)) & ctrl.len))
658 goto unlock;
659
660 counter_arch_bp(bp)->trigger = addr;
661 perf_bp_event(bp, regs);
662
663 /* Do we need to handle the stepping? */
Wang Nan18794452016-03-28 06:41:30 +0000664 if (is_default_overflow_handler(bp))
Will Deacon478fcb22012-03-05 11:49:33 +0000665 step = 1;
666unlock:
667 rcu_read_unlock();
668 }
669
670 if (!step)
671 return 0;
672
673 if (user_mode(regs)) {
674 debug_info->bps_disabled = 1;
675 toggle_bp_registers(AARCH64_DBG_REG_BCR, DBG_ACTIVE_EL0, 0);
676
677 /* If we're already stepping a watchpoint, just return. */
678 if (debug_info->wps_disabled)
679 return 0;
680
681 if (test_thread_flag(TIF_SINGLESTEP))
682 debug_info->suspended_step = 1;
683 else
684 user_enable_single_step(current);
685 } else {
686 toggle_bp_registers(AARCH64_DBG_REG_BCR, DBG_ACTIVE_EL1, 0);
Christoph Lameter1436c1a2013-10-21 13:17:08 +0100687 kernel_step = this_cpu_ptr(&stepping_kernel_bp);
Will Deacon478fcb22012-03-05 11:49:33 +0000688
689 if (*kernel_step != ARM_KERNEL_STEP_NONE)
690 return 0;
691
692 if (kernel_active_single_step()) {
693 *kernel_step = ARM_KERNEL_STEP_SUSPEND;
694 } else {
695 *kernel_step = ARM_KERNEL_STEP_ACTIVE;
696 kernel_enable_single_step(regs);
697 }
698 }
699
700 return 0;
701}
Pratyush Anand44b53f62016-07-08 12:35:49 -0400702NOKPROBE_SYMBOL(breakpoint_handler);
Will Deacon478fcb22012-03-05 11:49:33 +0000703
Pavel Labath32649b02016-11-14 19:32:44 +0530704/*
705 * Arm64 hardware does not always report a watchpoint hit address that matches
706 * one of the watchpoints set. It can also report an address "near" the
707 * watchpoint if a single instruction access both watched and unwatched
708 * addresses. There is no straight-forward way, short of disassembling the
709 * offending instruction, to map that address back to the watchpoint. This
710 * function computes the distance of the memory access from the watchpoint as a
711 * heuristic for the likelyhood that a given access triggered the watchpoint.
712 *
713 * See Section D2.10.5 "Determining the memory location that caused a Watchpoint
714 * exception" of ARMv8 Architecture Reference Manual for details.
715 *
716 * The function returns the distance of the address from the bytes watched by
717 * the watchpoint. In case of an exact match, it returns 0.
718 */
719static u64 get_distance_from_watchpoint(unsigned long addr, u64 val,
720 struct arch_hw_breakpoint_ctrl *ctrl)
721{
722 u64 wp_low, wp_high;
723 u32 lens, lene;
724
725 lens = __ffs(ctrl->len);
726 lene = __fls(ctrl->len);
727
728 wp_low = val + lens;
729 wp_high = val + lene;
730 if (addr < wp_low)
731 return wp_low - addr;
732 else if (addr > wp_high)
733 return addr - wp_high;
734 else
735 return 0;
736}
737
Will Deacon478fcb22012-03-05 11:49:33 +0000738static int watchpoint_handler(unsigned long addr, unsigned int esr,
739 struct pt_regs *regs)
740{
Pavel Labath32649b02016-11-14 19:32:44 +0530741 int i, step = 0, *kernel_step, access, closest_match = 0;
742 u64 min_dist = -1, dist;
743 u32 ctrl_reg;
Pratyush Anand67de4de2016-11-14 19:32:43 +0530744 u64 val;
Will Deacon478fcb22012-03-05 11:49:33 +0000745 struct perf_event *wp, **slots;
746 struct debug_info *debug_info;
747 struct arch_hw_breakpoint *info;
748 struct arch_hw_breakpoint_ctrl ctrl;
749
Christoph Lameter1436c1a2013-10-21 13:17:08 +0100750 slots = this_cpu_ptr(wp_on_reg);
Will Deacon478fcb22012-03-05 11:49:33 +0000751 debug_info = &current->thread.debug;
752
Pavel Labath32649b02016-11-14 19:32:44 +0530753 /*
754 * Find all watchpoints that match the reported address. If no exact
755 * match is found. Attribute the hit to the closest watchpoint.
756 */
757 rcu_read_lock();
Will Deacon478fcb22012-03-05 11:49:33 +0000758 for (i = 0; i < core_num_wrps; ++i) {
Will Deacon478fcb22012-03-05 11:49:33 +0000759 wp = slots[i];
Will Deacon478fcb22012-03-05 11:49:33 +0000760 if (wp == NULL)
Pavel Labath32649b02016-11-14 19:32:44 +0530761 continue;
Will Deacon478fcb22012-03-05 11:49:33 +0000762
763 /*
764 * Check that the access type matches.
765 * 0 => load, otherwise => store
766 */
767 access = (esr & AARCH64_ESR_ACCESS_MASK) ? HW_BREAKPOINT_W :
768 HW_BREAKPOINT_R;
769 if (!(access & hw_breakpoint_type(wp)))
Pavel Labath32649b02016-11-14 19:32:44 +0530770 continue;
Will Deacon478fcb22012-03-05 11:49:33 +0000771
Pavel Labath32649b02016-11-14 19:32:44 +0530772 /* Check if the watchpoint value and byte select match. */
773 val = read_wb_reg(AARCH64_DBG_REG_WVR, i);
774 ctrl_reg = read_wb_reg(AARCH64_DBG_REG_WCR, i);
775 decode_ctrl_reg(ctrl_reg, &ctrl);
776 dist = get_distance_from_watchpoint(addr, val, &ctrl);
777 if (dist < min_dist) {
778 min_dist = dist;
779 closest_match = i;
780 }
781 /* Is this an exact match? */
782 if (dist != 0)
783 continue;
784
785 info = counter_arch_bp(wp);
Will Deacon478fcb22012-03-05 11:49:33 +0000786 info->trigger = addr;
787 perf_bp_event(wp, regs);
788
789 /* Do we need to handle the stepping? */
Wang Nan18794452016-03-28 06:41:30 +0000790 if (is_default_overflow_handler(wp))
Will Deacon478fcb22012-03-05 11:49:33 +0000791 step = 1;
Will Deacon478fcb22012-03-05 11:49:33 +0000792 }
Pavel Labath32649b02016-11-14 19:32:44 +0530793 if (min_dist > 0 && min_dist != -1) {
794 /* No exact match found. */
795 wp = slots[closest_match];
796 info = counter_arch_bp(wp);
797 info->trigger = addr;
798 perf_bp_event(wp, regs);
799
800 /* Do we need to handle the stepping? */
801 if (is_default_overflow_handler(wp))
802 step = 1;
803 }
804 rcu_read_unlock();
Will Deacon478fcb22012-03-05 11:49:33 +0000805
806 if (!step)
807 return 0;
808
809 /*
810 * We always disable EL0 watchpoints because the kernel can
811 * cause these to fire via an unprivileged access.
812 */
813 toggle_bp_registers(AARCH64_DBG_REG_WCR, DBG_ACTIVE_EL0, 0);
814
815 if (user_mode(regs)) {
816 debug_info->wps_disabled = 1;
817
818 /* If we're already stepping a breakpoint, just return. */
819 if (debug_info->bps_disabled)
820 return 0;
821
822 if (test_thread_flag(TIF_SINGLESTEP))
823 debug_info->suspended_step = 1;
824 else
825 user_enable_single_step(current);
826 } else {
827 toggle_bp_registers(AARCH64_DBG_REG_WCR, DBG_ACTIVE_EL1, 0);
Christoph Lameter1436c1a2013-10-21 13:17:08 +0100828 kernel_step = this_cpu_ptr(&stepping_kernel_bp);
Will Deacon478fcb22012-03-05 11:49:33 +0000829
830 if (*kernel_step != ARM_KERNEL_STEP_NONE)
831 return 0;
832
833 if (kernel_active_single_step()) {
834 *kernel_step = ARM_KERNEL_STEP_SUSPEND;
835 } else {
836 *kernel_step = ARM_KERNEL_STEP_ACTIVE;
837 kernel_enable_single_step(regs);
838 }
839 }
840
841 return 0;
842}
Pratyush Anand44b53f62016-07-08 12:35:49 -0400843NOKPROBE_SYMBOL(watchpoint_handler);
Will Deacon478fcb22012-03-05 11:49:33 +0000844
845/*
846 * Handle single-step exception.
847 */
848int reinstall_suspended_bps(struct pt_regs *regs)
849{
850 struct debug_info *debug_info = &current->thread.debug;
851 int handled_exception = 0, *kernel_step;
852
Christoph Lameter1436c1a2013-10-21 13:17:08 +0100853 kernel_step = this_cpu_ptr(&stepping_kernel_bp);
Will Deacon478fcb22012-03-05 11:49:33 +0000854
855 /*
856 * Called from single-step exception handler.
857 * Return 0 if execution can resume, 1 if a SIGTRAP should be
858 * reported.
859 */
860 if (user_mode(regs)) {
861 if (debug_info->bps_disabled) {
862 debug_info->bps_disabled = 0;
863 toggle_bp_registers(AARCH64_DBG_REG_BCR, DBG_ACTIVE_EL0, 1);
864 handled_exception = 1;
865 }
866
867 if (debug_info->wps_disabled) {
868 debug_info->wps_disabled = 0;
869 toggle_bp_registers(AARCH64_DBG_REG_WCR, DBG_ACTIVE_EL0, 1);
870 handled_exception = 1;
871 }
872
873 if (handled_exception) {
874 if (debug_info->suspended_step) {
875 debug_info->suspended_step = 0;
876 /* Allow exception handling to fall-through. */
877 handled_exception = 0;
878 } else {
879 user_disable_single_step(current);
880 }
881 }
882 } else if (*kernel_step != ARM_KERNEL_STEP_NONE) {
883 toggle_bp_registers(AARCH64_DBG_REG_BCR, DBG_ACTIVE_EL1, 1);
884 toggle_bp_registers(AARCH64_DBG_REG_WCR, DBG_ACTIVE_EL1, 1);
885
886 if (!debug_info->wps_disabled)
887 toggle_bp_registers(AARCH64_DBG_REG_WCR, DBG_ACTIVE_EL0, 1);
888
889 if (*kernel_step != ARM_KERNEL_STEP_SUSPEND) {
890 kernel_disable_single_step();
891 handled_exception = 1;
892 } else {
893 handled_exception = 0;
894 }
895
896 *kernel_step = ARM_KERNEL_STEP_NONE;
897 }
898
899 return !handled_exception;
900}
Pratyush Anand44b53f62016-07-08 12:35:49 -0400901NOKPROBE_SYMBOL(reinstall_suspended_bps);
Will Deacon478fcb22012-03-05 11:49:33 +0000902
903/*
904 * Context-switcher for restoring suspended breakpoints.
905 */
906void hw_breakpoint_thread_switch(struct task_struct *next)
907{
908 /*
909 * current next
910 * disabled: 0 0 => The usual case, NOTIFY_DONE
911 * 0 1 => Disable the registers
912 * 1 0 => Enable the registers
913 * 1 1 => NOTIFY_DONE. per-task bps will
914 * get taken care of by perf.
915 */
916
917 struct debug_info *current_debug_info, *next_debug_info;
918
919 current_debug_info = &current->thread.debug;
920 next_debug_info = &next->thread.debug;
921
922 /* Update breakpoints. */
923 if (current_debug_info->bps_disabled != next_debug_info->bps_disabled)
924 toggle_bp_registers(AARCH64_DBG_REG_BCR,
925 DBG_ACTIVE_EL0,
926 !next_debug_info->bps_disabled);
927
928 /* Update watchpoints. */
929 if (current_debug_info->wps_disabled != next_debug_info->wps_disabled)
930 toggle_bp_registers(AARCH64_DBG_REG_WCR,
931 DBG_ACTIVE_EL0,
932 !next_debug_info->wps_disabled);
933}
934
935/*
936 * CPU initialisation.
937 */
Will Deacond7a83d12016-08-15 18:55:11 +0100938static int hw_breakpoint_reset(unsigned int cpu)
Will Deacon478fcb22012-03-05 11:49:33 +0000939{
940 int i;
Lorenzo Pieralisi60fc6942013-08-05 15:20:35 +0100941 struct perf_event **slots;
942 /*
943 * When a CPU goes through cold-boot, it does not have any installed
944 * slot, so it is safe to share the same function for restoring and
945 * resetting breakpoints; when a CPU is hotplugged in, it goes
946 * through the slots, which are all empty, hence it just resets control
947 * and value for debug registers.
948 * When this function is triggered on warm-boot through a CPU PM
949 * notifier some slots might be initialized; if so they are
950 * reprogrammed according to the debug slots content.
951 */
952 for (slots = this_cpu_ptr(bp_on_reg), i = 0; i < core_num_brps; ++i) {
953 if (slots[i]) {
954 hw_breakpoint_control(slots[i], HW_BREAKPOINT_RESTORE);
955 } else {
956 write_wb_reg(AARCH64_DBG_REG_BCR, i, 0UL);
957 write_wb_reg(AARCH64_DBG_REG_BVR, i, 0UL);
958 }
Will Deacon478fcb22012-03-05 11:49:33 +0000959 }
960
Lorenzo Pieralisi60fc6942013-08-05 15:20:35 +0100961 for (slots = this_cpu_ptr(wp_on_reg), i = 0; i < core_num_wrps; ++i) {
962 if (slots[i]) {
963 hw_breakpoint_control(slots[i], HW_BREAKPOINT_RESTORE);
964 } else {
965 write_wb_reg(AARCH64_DBG_REG_WCR, i, 0UL);
966 write_wb_reg(AARCH64_DBG_REG_WVR, i, 0UL);
967 }
Will Deacon478fcb22012-03-05 11:49:33 +0000968 }
Will Deacon478fcb22012-03-05 11:49:33 +0000969
Will Deacond7a83d12016-08-15 18:55:11 +0100970 return 0;
Will Deacon478fcb22012-03-05 11:49:33 +0000971}
972
Lorenzo Pieralisiaf3cfdb2015-01-26 18:33:44 +0000973#ifdef CONFIG_CPU_PM
Will Deacond7a83d12016-08-15 18:55:11 +0100974extern void cpu_suspend_set_dbg_restorer(int (*hw_bp_restore)(unsigned int));
Lorenzo Pieralisi60fc6942013-08-05 15:20:35 +0100975#else
Will Deacond7a83d12016-08-15 18:55:11 +0100976static inline void cpu_suspend_set_dbg_restorer(int (*hw_bp_restore)(unsigned int))
Lorenzo Pieralisi60fc6942013-08-05 15:20:35 +0100977{
978}
979#endif
980
Will Deacon478fcb22012-03-05 11:49:33 +0000981/*
982 * One-time initialisation.
983 */
984static int __init arch_hw_breakpoint_init(void)
985{
Will Deacond7a83d12016-08-15 18:55:11 +0100986 int ret;
987
Will Deacon478fcb22012-03-05 11:49:33 +0000988 core_num_brps = get_num_brps();
989 core_num_wrps = get_num_wrps();
990
991 pr_info("found %d breakpoint and %d watchpoint registers.\n",
992 core_num_brps, core_num_wrps);
993
Will Deacon478fcb22012-03-05 11:49:33 +0000994 /* Register debug fault handlers. */
995 hook_debug_fault_code(DBG_ESR_EVT_HWBP, breakpoint_handler, SIGTRAP,
996 TRAP_HWBKPT, "hw-breakpoint handler");
997 hook_debug_fault_code(DBG_ESR_EVT_HWWP, watchpoint_handler, SIGTRAP,
998 TRAP_HWBKPT, "hw-watchpoint handler");
999
Will Deacond7a83d12016-08-15 18:55:11 +01001000 /*
1001 * Reset the breakpoint resources. We assume that a halting
1002 * debugger will leave the world in a nice state for us.
1003 */
1004 ret = cpuhp_setup_state(CPUHP_AP_PERF_ARM_HW_BREAKPOINT_STARTING,
1005 "CPUHP_AP_PERF_ARM_HW_BREAKPOINT_STARTING",
1006 hw_breakpoint_reset, NULL);
1007 if (ret)
1008 pr_err("failed to register CPU hotplug notifier: %d\n", ret);
Srivatsa S. Bhat3d0dc642014-03-11 02:09:08 +05301009
Lorenzo Pieralisi65c021b2014-01-10 13:15:05 +00001010 /* Register cpu_suspend hw breakpoint restore hook */
1011 cpu_suspend_set_dbg_restorer(hw_breakpoint_reset);
Will Deacon478fcb22012-03-05 11:49:33 +00001012
Will Deacond7a83d12016-08-15 18:55:11 +01001013 return ret;
Will Deacon478fcb22012-03-05 11:49:33 +00001014}
1015arch_initcall(arch_hw_breakpoint_init);
1016
1017void hw_breakpoint_pmu_read(struct perf_event *bp)
1018{
1019}
1020
1021/*
1022 * Dummy function to register with die_notifier.
1023 */
1024int hw_breakpoint_exceptions_notify(struct notifier_block *unused,
1025 unsigned long val, void *data)
1026{
1027 return NOTIFY_DONE;
1028}