Lorenzo Pieralisi | de818bd | 2015-11-17 11:50:51 +0000 | [diff] [blame] | 1 | #include <linux/ftrace.h> |
Lorenzo Pieralisi | fb4a960 | 2014-01-24 10:56:19 +0000 | [diff] [blame] | 2 | #include <linux/percpu.h> |
Lorenzo Pieralisi | 9532252 | 2013-07-22 12:22:13 +0100 | [diff] [blame] | 3 | #include <linux/slab.h> |
James Morse | d085441 | 2016-10-18 11:27:48 +0100 | [diff] [blame] | 4 | #include <asm/alternative.h> |
Lorenzo Pieralisi | 9532252 | 2013-07-22 12:22:13 +0100 | [diff] [blame] | 5 | #include <asm/cacheflush.h> |
James Morse | d085441 | 2016-10-18 11:27:48 +0100 | [diff] [blame] | 6 | #include <asm/cpufeature.h> |
Lorenzo Pieralisi | 9532252 | 2013-07-22 12:22:13 +0100 | [diff] [blame] | 7 | #include <asm/debug-monitors.h> |
James Morse | d085441 | 2016-10-18 11:27:48 +0100 | [diff] [blame] | 8 | #include <asm/exec.h> |
Lorenzo Pieralisi | 9532252 | 2013-07-22 12:22:13 +0100 | [diff] [blame] | 9 | #include <asm/pgtable.h> |
| 10 | #include <asm/memory.h> |
Lorenzo Pieralisi | f43c271 | 2014-12-19 17:03:47 +0000 | [diff] [blame] | 11 | #include <asm/mmu_context.h> |
Lorenzo Pieralisi | 9532252 | 2013-07-22 12:22:13 +0100 | [diff] [blame] | 12 | #include <asm/smp_plat.h> |
| 13 | #include <asm/suspend.h> |
| 14 | #include <asm/tlbflush.h> |
| 15 | |
Lorenzo Pieralisi | 9532252 | 2013-07-22 12:22:13 +0100 | [diff] [blame] | 16 | /* |
James Morse | cabe1c8 | 2016-04-27 17:47:07 +0100 | [diff] [blame] | 17 | * This is allocated by cpu_suspend_init(), and used to store a pointer to |
| 18 | * the 'struct sleep_stack_data' the contains a particular CPUs state. |
Lorenzo Pieralisi | 9532252 | 2013-07-22 12:22:13 +0100 | [diff] [blame] | 19 | */ |
James Morse | cabe1c8 | 2016-04-27 17:47:07 +0100 | [diff] [blame] | 20 | unsigned long *sleep_save_stash; |
Lorenzo Pieralisi | 9532252 | 2013-07-22 12:22:13 +0100 | [diff] [blame] | 21 | |
Lorenzo Pieralisi | 65c021b | 2014-01-10 13:15:05 +0000 | [diff] [blame] | 22 | /* |
| 23 | * This hook is provided so that cpu_suspend code can restore HW |
| 24 | * breakpoints as early as possible in the resume path, before reenabling |
| 25 | * debug exceptions. Code cannot be run from a CPU PM notifier since by the |
| 26 | * time the notifier runs debug exceptions might have been enabled already, |
| 27 | * with HW breakpoints registers content still in an unknown state. |
| 28 | */ |
Will Deacon | d7a83d1 | 2016-08-15 18:55:11 +0100 | [diff] [blame] | 29 | static int (*hw_breakpoint_restore)(unsigned int); |
| 30 | void __init cpu_suspend_set_dbg_restorer(int (*hw_bp_restore)(unsigned int)) |
Lorenzo Pieralisi | 65c021b | 2014-01-10 13:15:05 +0000 | [diff] [blame] | 31 | { |
| 32 | /* Prevent multiple restore hook initializations */ |
| 33 | if (WARN_ON(hw_breakpoint_restore)) |
| 34 | return; |
| 35 | hw_breakpoint_restore = hw_bp_restore; |
| 36 | } |
| 37 | |
James Morse | adc9b2d | 2016-04-27 17:47:06 +0100 | [diff] [blame] | 38 | void notrace __cpu_suspend_exit(void) |
| 39 | { |
Will Deacon | d7a83d1 | 2016-08-15 18:55:11 +0100 | [diff] [blame] | 40 | unsigned int cpu = smp_processor_id(); |
| 41 | |
James Morse | adc9b2d | 2016-04-27 17:47:06 +0100 | [diff] [blame] | 42 | /* |
| 43 | * We are resuming from reset with the idmap active in TTBR0_EL1. |
| 44 | * We must uninstall the idmap and restore the expected MMU |
| 45 | * state before we can possibly return to userspace. |
| 46 | */ |
| 47 | cpu_uninstall_idmap(); |
| 48 | |
| 49 | /* |
| 50 | * Restore per-cpu offset before any kernel |
| 51 | * subsystem relying on it has a chance to run. |
| 52 | */ |
Will Deacon | d7a83d1 | 2016-08-15 18:55:11 +0100 | [diff] [blame] | 53 | set_my_cpu_offset(per_cpu_offset(cpu)); |
James Morse | adc9b2d | 2016-04-27 17:47:06 +0100 | [diff] [blame] | 54 | |
| 55 | /* |
James Morse | d085441 | 2016-10-18 11:27:48 +0100 | [diff] [blame] | 56 | * PSTATE was not saved over suspend/resume, re-enable any detected |
| 57 | * features that might not have been set correctly. |
| 58 | */ |
| 59 | asm(ALTERNATIVE("nop", SET_PSTATE_PAN(1), ARM64_HAS_PAN, |
| 60 | CONFIG_ARM64_PAN)); |
| 61 | uao_thread_switch(current); |
| 62 | |
| 63 | /* |
James Morse | adc9b2d | 2016-04-27 17:47:06 +0100 | [diff] [blame] | 64 | * Restore HW breakpoint registers to sane values |
| 65 | * before debug exceptions are possibly reenabled |
| 66 | * through local_dbg_restore. |
| 67 | */ |
| 68 | if (hw_breakpoint_restore) |
Will Deacon | d7a83d1 | 2016-08-15 18:55:11 +0100 | [diff] [blame] | 69 | hw_breakpoint_restore(cpu); |
Marc Zyngier | d8fbc84 | 2018-07-20 10:56:28 +0100 | [diff] [blame] | 70 | |
| 71 | /* |
| 72 | * On resume, firmware implementing dynamic mitigation will |
| 73 | * have turned the mitigation on. If the user has forcefully |
| 74 | * disabled it, make sure their wishes are obeyed. |
| 75 | */ |
| 76 | if (arm64_get_ssbd_state() == ARM64_SSBD_FORCE_DISABLE) |
| 77 | arm64_set_ssbd_mitigation(false); |
James Morse | adc9b2d | 2016-04-27 17:47:06 +0100 | [diff] [blame] | 78 | } |
| 79 | |
Lorenzo Pieralisi | 714f599 | 2014-08-07 14:54:50 +0100 | [diff] [blame] | 80 | /* |
Sudeep Holla | af391b1 | 2015-06-18 15:41:32 +0100 | [diff] [blame] | 81 | * cpu_suspend |
Lorenzo Pieralisi | 714f599 | 2014-08-07 14:54:50 +0100 | [diff] [blame] | 82 | * |
| 83 | * arg: argument to pass to the finisher function |
| 84 | * fn: finisher function pointer |
| 85 | * |
| 86 | */ |
Sudeep Holla | af391b1 | 2015-06-18 15:41:32 +0100 | [diff] [blame] | 87 | int cpu_suspend(unsigned long arg, int (*fn)(unsigned long)) |
Lorenzo Pieralisi | 714f599 | 2014-08-07 14:54:50 +0100 | [diff] [blame] | 88 | { |
James Morse | adc9b2d | 2016-04-27 17:47:06 +0100 | [diff] [blame] | 89 | int ret = 0; |
Lorenzo Pieralisi | 714f599 | 2014-08-07 14:54:50 +0100 | [diff] [blame] | 90 | unsigned long flags; |
James Morse | adc9b2d | 2016-04-27 17:47:06 +0100 | [diff] [blame] | 91 | struct sleep_stack_data state; |
Lorenzo Pieralisi | 9532252 | 2013-07-22 12:22:13 +0100 | [diff] [blame] | 92 | |
| 93 | /* |
| 94 | * From this point debug exceptions are disabled to prevent |
| 95 | * updates to mdscr register (saved and restored along with |
| 96 | * general purpose registers) from kernel debuggers. |
| 97 | */ |
| 98 | local_dbg_save(flags); |
| 99 | |
| 100 | /* |
Lorenzo Pieralisi | de818bd | 2015-11-17 11:50:51 +0000 | [diff] [blame] | 101 | * Function graph tracer state gets incosistent when the kernel |
| 102 | * calls functions that never return (aka suspend finishers) hence |
| 103 | * disable graph tracing during their execution. |
| 104 | */ |
| 105 | pause_graph_tracing(); |
| 106 | |
James Morse | adc9b2d | 2016-04-27 17:47:06 +0100 | [diff] [blame] | 107 | if (__cpu_suspend_enter(&state)) { |
| 108 | /* Call the suspend finisher */ |
| 109 | ret = fn(arg); |
Lorenzo Pieralisi | fb4a960 | 2014-01-24 10:56:19 +0000 | [diff] [blame] | 110 | |
| 111 | /* |
James Morse | adc9b2d | 2016-04-27 17:47:06 +0100 | [diff] [blame] | 112 | * Never gets here, unless the suspend finisher fails. |
| 113 | * Successful cpu_suspend() should return from cpu_resume(), |
| 114 | * returning through this code path is considered an error |
| 115 | * If the return value is set to 0 force ret = -EOPNOTSUPP |
| 116 | * to make sure a proper error condition is propagated |
Lorenzo Pieralisi | fb4a960 | 2014-01-24 10:56:19 +0000 | [diff] [blame] | 117 | */ |
James Morse | adc9b2d | 2016-04-27 17:47:06 +0100 | [diff] [blame] | 118 | if (!ret) |
| 119 | ret = -EOPNOTSUPP; |
| 120 | } else { |
| 121 | __cpu_suspend_exit(); |
Lorenzo Pieralisi | 9532252 | 2013-07-22 12:22:13 +0100 | [diff] [blame] | 122 | } |
| 123 | |
Lorenzo Pieralisi | de818bd | 2015-11-17 11:50:51 +0000 | [diff] [blame] | 124 | unpause_graph_tracing(); |
| 125 | |
Lorenzo Pieralisi | 9532252 | 2013-07-22 12:22:13 +0100 | [diff] [blame] | 126 | /* |
| 127 | * Restore pstate flags. OS lock and mdscr have been already |
| 128 | * restored, so from this point onwards, debugging is fully |
| 129 | * renabled if it was enabled when core started shutdown. |
| 130 | */ |
| 131 | local_dbg_restore(flags); |
| 132 | |
| 133 | return ret; |
| 134 | } |
| 135 | |
Lorenzo Pieralisi | 18ab7db | 2014-07-17 18:19:20 +0100 | [diff] [blame] | 136 | static int __init cpu_suspend_init(void) |
Lorenzo Pieralisi | 9532252 | 2013-07-22 12:22:13 +0100 | [diff] [blame] | 137 | { |
Lorenzo Pieralisi | 9532252 | 2013-07-22 12:22:13 +0100 | [diff] [blame] | 138 | /* ctx_ptr is an array of physical addresses */ |
James Morse | cabe1c8 | 2016-04-27 17:47:07 +0100 | [diff] [blame] | 139 | sleep_save_stash = kcalloc(mpidr_hash_size(), sizeof(*sleep_save_stash), |
| 140 | GFP_KERNEL); |
Lorenzo Pieralisi | 9532252 | 2013-07-22 12:22:13 +0100 | [diff] [blame] | 141 | |
James Morse | cabe1c8 | 2016-04-27 17:47:07 +0100 | [diff] [blame] | 142 | if (WARN_ON(!sleep_save_stash)) |
Lorenzo Pieralisi | 9532252 | 2013-07-22 12:22:13 +0100 | [diff] [blame] | 143 | return -ENOMEM; |
| 144 | |
Lorenzo Pieralisi | 9532252 | 2013-07-22 12:22:13 +0100 | [diff] [blame] | 145 | return 0; |
| 146 | } |
| 147 | early_initcall(cpu_suspend_init); |