Ard Biesheuvel | 3be1a5c | 2014-03-04 01:10:04 +0000 | [diff] [blame] | 1 | /* |
| 2 | * Copyright (C) 2014 Linaro Ltd. <ard.biesheuvel@linaro.org> |
| 3 | * |
| 4 | * This program is free software; you can redistribute it and/or modify |
| 5 | * it under the terms of the GNU General Public License version 2 as |
| 6 | * published by the Free Software Foundation. |
| 7 | */ |
| 8 | |
| 9 | #ifndef __ASM_CPUFEATURE_H |
| 10 | #define __ASM_CPUFEATURE_H |
| 11 | |
Catalin Marinas | 272d01b | 2016-11-03 18:34:34 +0000 | [diff] [blame] | 12 | #include <asm/cpucaps.h> |
Ard Biesheuvel | 3be1a5c | 2014-03-04 01:10:04 +0000 | [diff] [blame] | 13 | #include <asm/hwcap.h> |
Suzuki K. Poulose | cdcf817 | 2015-10-19 14:24:42 +0100 | [diff] [blame] | 14 | #include <asm/sysreg.h> |
Ard Biesheuvel | 3be1a5c | 2014-03-04 01:10:04 +0000 | [diff] [blame] | 15 | |
| 16 | /* |
| 17 | * In the arm64 world (as in the ARM world), elf_hwcap is used both internally |
| 18 | * in the kernel and for user space to keep track of which optional features |
| 19 | * are supported by the current system. So let's map feature 'x' to HWCAP_x. |
| 20 | * Note that HWCAP_x constants are bit fields so we need to take the log. |
| 21 | */ |
| 22 | |
| 23 | #define MAX_CPU_FEATURES (8 * sizeof(elf_hwcap)) |
| 24 | #define cpu_feature(x) ilog2(HWCAP_ ## x) |
| 25 | |
Andre Przywara | 301bcfa | 2014-11-14 15:54:10 +0000 | [diff] [blame] | 26 | #ifndef __ASSEMBLY__ |
Andre Przywara | 930da09 | 2014-11-14 15:54:07 +0000 | [diff] [blame] | 27 | |
Suzuki K Poulose | fe64d7d | 2016-11-08 13:56:20 +0000 | [diff] [blame] | 28 | #include <linux/bug.h> |
| 29 | #include <linux/jump_label.h> |
Will Deacon | 144e969 | 2015-04-30 18:55:50 +0100 | [diff] [blame] | 30 | #include <linux/kernel.h> |
| 31 | |
Suzuki K. Poulose | 3c739b5 | 2015-10-19 14:24:45 +0100 | [diff] [blame] | 32 | /* CPU feature register tracking */ |
| 33 | enum ftr_type { |
Will Deacon | 3c5dbb9 | 2019-08-05 18:13:55 +0100 | [diff] [blame] | 34 | FTR_EXACT, /* Use a predefined safe value */ |
| 35 | FTR_LOWER_SAFE, /* Smaller value is safe */ |
| 36 | FTR_HIGHER_SAFE, /* Bigger value is safe */ |
| 37 | FTR_HIGHER_OR_ZERO_SAFE, /* Bigger value is safe, but 0 is biggest */ |
Suzuki K. Poulose | 3c739b5 | 2015-10-19 14:24:45 +0100 | [diff] [blame] | 38 | }; |
| 39 | |
| 40 | #define FTR_STRICT true /* SANITY check strict matching required */ |
| 41 | #define FTR_NONSTRICT false /* SANITY check ignored */ |
| 42 | |
Suzuki K. Poulose | 4f0a606 | 2015-11-18 17:08:57 +0000 | [diff] [blame] | 43 | #define FTR_SIGNED true /* Value should be treated as signed */ |
| 44 | #define FTR_UNSIGNED false /* Value should be treated as unsigned */ |
| 45 | |
Suzuki K. Poulose | 3c739b5 | 2015-10-19 14:24:45 +0100 | [diff] [blame] | 46 | struct arm64_ftr_bits { |
Suzuki K. Poulose | 4f0a606 | 2015-11-18 17:08:57 +0000 | [diff] [blame] | 47 | bool sign; /* Value is signed ? */ |
| 48 | bool strict; /* CPU Sanity check: strict matching required ? */ |
Suzuki K. Poulose | 3c739b5 | 2015-10-19 14:24:45 +0100 | [diff] [blame] | 49 | enum ftr_type type; |
| 50 | u8 shift; |
| 51 | u8 width; |
Suzuki K Poulose | ee7bc63 | 2016-09-09 14:07:08 +0100 | [diff] [blame] | 52 | s64 safe_val; /* safe value for FTR_EXACT features */ |
Suzuki K. Poulose | 3c739b5 | 2015-10-19 14:24:45 +0100 | [diff] [blame] | 53 | }; |
| 54 | |
| 55 | /* |
| 56 | * @arm64_ftr_reg - Feature register |
| 57 | * @strict_mask Bits which should match across all CPUs for sanity. |
| 58 | * @sys_val Safe value across the CPUs (system view) |
| 59 | */ |
| 60 | struct arm64_ftr_reg { |
Ard Biesheuvel | 5e49d73 | 2016-08-31 11:31:08 +0100 | [diff] [blame] | 61 | const char *name; |
| 62 | u64 strict_mask; |
| 63 | u64 sys_val; |
| 64 | const struct arm64_ftr_bits *ftr_bits; |
Suzuki K. Poulose | 3c739b5 | 2015-10-19 14:24:45 +0100 | [diff] [blame] | 65 | }; |
| 66 | |
Ard Biesheuvel | 675b056 | 2016-08-31 11:31:10 +0100 | [diff] [blame] | 67 | extern struct arm64_ftr_reg arm64_ftr_reg_ctrel0; |
| 68 | |
Suzuki K Poulose | 92406f0 | 2016-04-22 12:25:31 +0100 | [diff] [blame] | 69 | /* scope of capability check */ |
| 70 | enum { |
| 71 | SCOPE_SYSTEM, |
| 72 | SCOPE_LOCAL_CPU, |
| 73 | }; |
| 74 | |
Marc Zyngier | 359b706 | 2015-03-27 13:09:23 +0000 | [diff] [blame] | 75 | struct arm64_cpu_capabilities { |
| 76 | const char *desc; |
| 77 | u16 capability; |
Suzuki K Poulose | 92406f0 | 2016-04-22 12:25:31 +0100 | [diff] [blame] | 78 | int def_scope; /* default scope */ |
| 79 | bool (*matches)(const struct arm64_cpu_capabilities *caps, int scope); |
James Morse | 2a6dcb2 | 2016-10-18 11:27:46 +0100 | [diff] [blame] | 80 | int (*enable)(void *); /* Called on all active CPUs */ |
Marc Zyngier | 359b706 | 2015-03-27 13:09:23 +0000 | [diff] [blame] | 81 | union { |
| 82 | struct { /* To be used for erratum handling only */ |
| 83 | u32 midr_model; |
| 84 | u32 midr_range_min, midr_range_max; |
| 85 | }; |
Marc Zyngier | 94a9e04 | 2015-06-12 12:06:36 +0100 | [diff] [blame] | 86 | |
| 87 | struct { /* Feature register checking */ |
Suzuki K. Poulose | da8d02d | 2015-10-19 14:24:51 +0100 | [diff] [blame] | 88 | u32 sys_reg; |
Suzuki K Poulose | ff96f7b | 2016-01-26 10:58:15 +0000 | [diff] [blame] | 89 | u8 field_pos; |
| 90 | u8 min_field_value; |
| 91 | u8 hwcap_type; |
| 92 | bool sign; |
Suzuki K. Poulose | 37b01d53 | 2015-10-19 14:24:52 +0100 | [diff] [blame] | 93 | unsigned long hwcap; |
Marc Zyngier | 94a9e04 | 2015-06-12 12:06:36 +0100 | [diff] [blame] | 94 | }; |
Marc Zyngier | 359b706 | 2015-03-27 13:09:23 +0000 | [diff] [blame] | 95 | }; |
| 96 | }; |
| 97 | |
Fabio Estevam | 06f9eb8 | 2014-12-04 01:17:01 +0000 | [diff] [blame] | 98 | extern DECLARE_BITMAP(cpu_hwcaps, ARM64_NCAPS); |
Catalin Marinas | efd9e03 | 2016-09-05 18:25:48 +0100 | [diff] [blame] | 99 | extern struct static_key_false cpu_hwcap_keys[ARM64_NCAPS]; |
Mark Rutland | b1d5708 | 2017-05-16 15:18:05 +0100 | [diff] [blame] | 100 | extern struct static_key_false arm64_const_caps_ready; |
Andre Przywara | 930da09 | 2014-11-14 15:54:07 +0000 | [diff] [blame] | 101 | |
Marc Zyngier | e3661b1 | 2016-04-22 12:25:32 +0100 | [diff] [blame] | 102 | bool this_cpu_has_cap(unsigned int cap); |
| 103 | |
Ard Biesheuvel | 3be1a5c | 2014-03-04 01:10:04 +0000 | [diff] [blame] | 104 | static inline bool cpu_have_feature(unsigned int num) |
| 105 | { |
| 106 | return elf_hwcap & (1UL << num); |
| 107 | } |
| 108 | |
Suzuki K Poulose | fe64d7d | 2016-11-08 13:56:20 +0000 | [diff] [blame] | 109 | /* System capability check for constant caps */ |
Mark Rutland | b1d5708 | 2017-05-16 15:18:05 +0100 | [diff] [blame] | 110 | static inline bool __cpus_have_const_cap(int num) |
Suzuki K Poulose | fe64d7d | 2016-11-08 13:56:20 +0000 | [diff] [blame] | 111 | { |
| 112 | if (num >= ARM64_NCAPS) |
| 113 | return false; |
| 114 | return static_branch_unlikely(&cpu_hwcap_keys[num]); |
| 115 | } |
| 116 | |
Andre Przywara | 930da09 | 2014-11-14 15:54:07 +0000 | [diff] [blame] | 117 | static inline bool cpus_have_cap(unsigned int num) |
| 118 | { |
Fabio Estevam | 06f9eb8 | 2014-12-04 01:17:01 +0000 | [diff] [blame] | 119 | if (num >= ARM64_NCAPS) |
Andre Przywara | 930da09 | 2014-11-14 15:54:07 +0000 | [diff] [blame] | 120 | return false; |
Suzuki K Poulose | fe64d7d | 2016-11-08 13:56:20 +0000 | [diff] [blame] | 121 | return test_bit(num, cpu_hwcaps); |
Andre Przywara | 930da09 | 2014-11-14 15:54:07 +0000 | [diff] [blame] | 122 | } |
| 123 | |
Mark Rutland | b1d5708 | 2017-05-16 15:18:05 +0100 | [diff] [blame] | 124 | static inline bool cpus_have_const_cap(int num) |
| 125 | { |
| 126 | if (static_branch_likely(&arm64_const_caps_ready)) |
| 127 | return __cpus_have_const_cap(num); |
| 128 | else |
| 129 | return cpus_have_cap(num); |
| 130 | } |
| 131 | |
Andre Przywara | 930da09 | 2014-11-14 15:54:07 +0000 | [diff] [blame] | 132 | static inline void cpus_set_cap(unsigned int num) |
| 133 | { |
Catalin Marinas | efd9e03 | 2016-09-05 18:25:48 +0100 | [diff] [blame] | 134 | if (num >= ARM64_NCAPS) { |
Andre Przywara | 930da09 | 2014-11-14 15:54:07 +0000 | [diff] [blame] | 135 | pr_warn("Attempt to set an illegal CPU capability (%d >= %d)\n", |
Fabio Estevam | 06f9eb8 | 2014-12-04 01:17:01 +0000 | [diff] [blame] | 136 | num, ARM64_NCAPS); |
Catalin Marinas | efd9e03 | 2016-09-05 18:25:48 +0100 | [diff] [blame] | 137 | } else { |
Andre Przywara | 930da09 | 2014-11-14 15:54:07 +0000 | [diff] [blame] | 138 | __set_bit(num, cpu_hwcaps); |
Catalin Marinas | efd9e03 | 2016-09-05 18:25:48 +0100 | [diff] [blame] | 139 | } |
Andre Przywara | 930da09 | 2014-11-14 15:54:07 +0000 | [diff] [blame] | 140 | } |
| 141 | |
Suzuki K. Poulose | ce98a67 | 2015-10-19 14:24:44 +0100 | [diff] [blame] | 142 | static inline int __attribute_const__ |
Suzuki K Poulose | 28c5dcb | 2016-01-26 10:58:16 +0000 | [diff] [blame] | 143 | cpuid_feature_extract_signed_field_width(u64 features, int field, int width) |
James Morse | 79b0e09 | 2015-07-21 13:23:26 +0100 | [diff] [blame] | 144 | { |
Suzuki K. Poulose | ce98a67 | 2015-10-19 14:24:44 +0100 | [diff] [blame] | 145 | return (s64)(features << (64 - width - field)) >> (64 - width); |
James Morse | 79b0e09 | 2015-07-21 13:23:26 +0100 | [diff] [blame] | 146 | } |
| 147 | |
Suzuki K. Poulose | ce98a67 | 2015-10-19 14:24:44 +0100 | [diff] [blame] | 148 | static inline int __attribute_const__ |
Suzuki K Poulose | 28c5dcb | 2016-01-26 10:58:16 +0000 | [diff] [blame] | 149 | cpuid_feature_extract_signed_field(u64 features, int field) |
Suzuki K. Poulose | ce98a67 | 2015-10-19 14:24:44 +0100 | [diff] [blame] | 150 | { |
Suzuki K Poulose | 28c5dcb | 2016-01-26 10:58:16 +0000 | [diff] [blame] | 151 | return cpuid_feature_extract_signed_field_width(features, field, 4); |
James Morse | 79b0e09 | 2015-07-21 13:23:26 +0100 | [diff] [blame] | 152 | } |
James Morse | 79b0e09 | 2015-07-21 13:23:26 +0100 | [diff] [blame] | 153 | |
Suzuki K. Poulose | d211827 | 2015-11-18 17:08:56 +0000 | [diff] [blame] | 154 | static inline unsigned int __attribute_const__ |
| 155 | cpuid_feature_extract_unsigned_field_width(u64 features, int field, int width) |
| 156 | { |
| 157 | return (u64)(features << (64 - width - field)) >> (64 - width); |
| 158 | } |
| 159 | |
| 160 | static inline unsigned int __attribute_const__ |
| 161 | cpuid_feature_extract_unsigned_field(u64 features, int field) |
| 162 | { |
| 163 | return cpuid_feature_extract_unsigned_field_width(features, field, 4); |
| 164 | } |
| 165 | |
Ard Biesheuvel | 5e49d73 | 2016-08-31 11:31:08 +0100 | [diff] [blame] | 166 | static inline u64 arm64_ftr_mask(const struct arm64_ftr_bits *ftrp) |
Suzuki K. Poulose | 3c739b5 | 2015-10-19 14:24:45 +0100 | [diff] [blame] | 167 | { |
| 168 | return (u64)GENMASK(ftrp->shift + ftrp->width - 1, ftrp->shift); |
| 169 | } |
| 170 | |
Suzuki K Poulose | 28c5dcb | 2016-01-26 10:58:16 +0000 | [diff] [blame] | 171 | static inline int __attribute_const__ |
| 172 | cpuid_feature_extract_field(u64 features, int field, bool sign) |
| 173 | { |
| 174 | return (sign) ? |
| 175 | cpuid_feature_extract_signed_field(features, field) : |
| 176 | cpuid_feature_extract_unsigned_field(features, field); |
| 177 | } |
| 178 | |
Ard Biesheuvel | 5e49d73 | 2016-08-31 11:31:08 +0100 | [diff] [blame] | 179 | static inline s64 arm64_ftr_value(const struct arm64_ftr_bits *ftrp, u64 val) |
Suzuki K. Poulose | 3c739b5 | 2015-10-19 14:24:45 +0100 | [diff] [blame] | 180 | { |
Suzuki K Poulose | 28c5dcb | 2016-01-26 10:58:16 +0000 | [diff] [blame] | 181 | return (s64)cpuid_feature_extract_field(val, ftrp->shift, ftrp->sign); |
Suzuki K. Poulose | 3c739b5 | 2015-10-19 14:24:45 +0100 | [diff] [blame] | 182 | } |
| 183 | |
Suzuki K. Poulose | cdcf817 | 2015-10-19 14:24:42 +0100 | [diff] [blame] | 184 | static inline bool id_aa64mmfr0_mixed_endian_el0(u64 mmfr0) |
| 185 | { |
Suzuki K Poulose | 28c5dcb | 2016-01-26 10:58:16 +0000 | [diff] [blame] | 186 | return cpuid_feature_extract_unsigned_field(mmfr0, ID_AA64MMFR0_BIGENDEL_SHIFT) == 0x1 || |
| 187 | cpuid_feature_extract_unsigned_field(mmfr0, ID_AA64MMFR0_BIGENDEL0_SHIFT) == 0x1; |
Suzuki K. Poulose | cdcf817 | 2015-10-19 14:24:42 +0100 | [diff] [blame] | 188 | } |
| 189 | |
Suzuki K Poulose | c80aba8 | 2016-04-18 10:28:34 +0100 | [diff] [blame] | 190 | static inline bool id_aa64pfr0_32bit_el0(u64 pfr0) |
| 191 | { |
| 192 | u32 val = cpuid_feature_extract_unsigned_field(pfr0, ID_AA64PFR0_EL0_SHIFT); |
| 193 | |
| 194 | return val == ID_AA64PFR0_EL0_32BIT_64BIT; |
| 195 | } |
| 196 | |
Suzuki K. Poulose | 3a75578 | 2015-10-19 14:24:39 +0100 | [diff] [blame] | 197 | void __init setup_cpu_features(void); |
Andre Przywara | e116a37 | 2014-11-14 15:54:09 +0000 | [diff] [blame] | 198 | |
Suzuki K. Poulose | ce8b602 | 2015-10-19 14:24:49 +0100 | [diff] [blame] | 199 | void update_cpu_capabilities(const struct arm64_cpu_capabilities *caps, |
Ard Biesheuvel | 3be1a5c | 2014-03-04 01:10:04 +0000 | [diff] [blame] | 200 | const char *info); |
Andre Przywara | 8e23185 | 2016-06-28 18:07:30 +0100 | [diff] [blame] | 201 | void enable_cpu_capabilities(const struct arm64_cpu_capabilities *caps); |
Suzuki K Poulose | c47a190 | 2016-09-09 14:07:10 +0100 | [diff] [blame] | 202 | void check_local_cpu_capabilities(void); |
| 203 | |
Suzuki K Poulose | 89ba264 | 2016-09-09 14:07:09 +0100 | [diff] [blame] | 204 | void update_cpu_errata_workarounds(void); |
Andre Przywara | 8e23185 | 2016-06-28 18:07:30 +0100 | [diff] [blame] | 205 | void __init enable_errata_workarounds(void); |
Suzuki K Poulose | 89ba264 | 2016-09-09 14:07:09 +0100 | [diff] [blame] | 206 | void verify_local_cpu_errata_workarounds(void); |
Ard Biesheuvel | 3be1a5c | 2014-03-04 01:10:04 +0000 | [diff] [blame] | 207 | |
Suzuki K. Poulose | b3f1537 | 2015-10-19 14:24:47 +0100 | [diff] [blame] | 208 | u64 read_system_reg(u32 id); |
| 209 | |
Suzuki K. Poulose | c1e8656 | 2015-10-19 14:24:48 +0100 | [diff] [blame] | 210 | static inline bool cpu_supports_mixed_endian_el0(void) |
| 211 | { |
| 212 | return id_aa64mmfr0_mixed_endian_el0(read_cpuid(ID_AA64MMFR0_EL1)); |
| 213 | } |
| 214 | |
Suzuki K Poulose | 042446a | 2016-04-18 10:28:36 +0100 | [diff] [blame] | 215 | static inline bool system_supports_32bit_el0(void) |
| 216 | { |
Suzuki K Poulose | fe64d7d | 2016-11-08 13:56:20 +0000 | [diff] [blame] | 217 | return cpus_have_const_cap(ARM64_HAS_32BIT_EL0); |
Suzuki K Poulose | 042446a | 2016-04-18 10:28:36 +0100 | [diff] [blame] | 218 | } |
| 219 | |
Suzuki K. Poulose | c1e8656 | 2015-10-19 14:24:48 +0100 | [diff] [blame] | 220 | static inline bool system_supports_mixed_endian_el0(void) |
| 221 | { |
| 222 | return id_aa64mmfr0_mixed_endian_el0(read_system_reg(SYS_ID_AA64MMFR0_EL1)); |
| 223 | } |
Ard Biesheuvel | 3be1a5c | 2014-03-04 01:10:04 +0000 | [diff] [blame] | 224 | |
Marc Zyngier | 3a64e6a | 2018-07-20 10:56:25 +0100 | [diff] [blame] | 225 | #define ARM64_SSBD_UNKNOWN -1 |
| 226 | #define ARM64_SSBD_FORCE_DISABLE 0 |
| 227 | #define ARM64_SSBD_KERNEL 1 |
| 228 | #define ARM64_SSBD_FORCE_ENABLE 2 |
| 229 | #define ARM64_SSBD_MITIGATED 3 |
| 230 | |
Marc Zyngier | 242bff3 | 2018-07-20 10:56:26 +0100 | [diff] [blame] | 231 | static inline int arm64_get_ssbd_state(void) |
| 232 | { |
| 233 | #ifdef CONFIG_ARM64_SSBD |
| 234 | extern int ssbd_state; |
| 235 | return ssbd_state; |
| 236 | #else |
| 237 | return ARM64_SSBD_UNKNOWN; |
| 238 | #endif |
| 239 | } |
| 240 | |
Marc Zyngier | d8fbc84 | 2018-07-20 10:56:28 +0100 | [diff] [blame] | 241 | #ifdef CONFIG_ARM64_SSBD |
| 242 | void arm64_set_ssbd_mitigation(bool state); |
| 243 | #else |
| 244 | static inline void arm64_set_ssbd_mitigation(bool state) {} |
| 245 | #endif |
| 246 | |
Ard Biesheuvel | 3be1a5c | 2014-03-04 01:10:04 +0000 | [diff] [blame] | 247 | #endif /* __ASSEMBLY__ */ |
| 248 | |
| 249 | #endif |