Ard Biesheuvel | 3be1a5c | 2014-03-04 01:10:04 +0000 | [diff] [blame] | 1 | /* |
| 2 | * Copyright (C) 2014 Linaro Ltd. <ard.biesheuvel@linaro.org> |
| 3 | * |
| 4 | * This program is free software; you can redistribute it and/or modify |
| 5 | * it under the terms of the GNU General Public License version 2 as |
| 6 | * published by the Free Software Foundation. |
| 7 | */ |
| 8 | |
| 9 | #ifndef __ASM_CPUFEATURE_H |
| 10 | #define __ASM_CPUFEATURE_H |
| 11 | |
Catalin Marinas | efd9e03 | 2016-09-05 18:25:48 +0100 | [diff] [blame] | 12 | #include <linux/jump_label.h> |
| 13 | |
Ard Biesheuvel | 3be1a5c | 2014-03-04 01:10:04 +0000 | [diff] [blame] | 14 | #include <asm/hwcap.h> |
Suzuki K. Poulose | cdcf817 | 2015-10-19 14:24:42 +0100 | [diff] [blame] | 15 | #include <asm/sysreg.h> |
Ard Biesheuvel | 3be1a5c | 2014-03-04 01:10:04 +0000 | [diff] [blame] | 16 | |
| 17 | /* |
| 18 | * In the arm64 world (as in the ARM world), elf_hwcap is used both internally |
| 19 | * in the kernel and for user space to keep track of which optional features |
| 20 | * are supported by the current system. So let's map feature 'x' to HWCAP_x. |
| 21 | * Note that HWCAP_x constants are bit fields so we need to take the log. |
| 22 | */ |
| 23 | |
| 24 | #define MAX_CPU_FEATURES (8 * sizeof(elf_hwcap)) |
| 25 | #define cpu_feature(x) ilog2(HWCAP_ ## x) |
| 26 | |
Andre Przywara | 5afaa1f | 2014-11-14 15:54:11 +0000 | [diff] [blame] | 27 | #define ARM64_WORKAROUND_CLEAN_CACHE 0 |
| 28 | #define ARM64_WORKAROUND_DEVICE_LOAD_ACQUIRE 1 |
Will Deacon | 905e8c5 | 2015-03-23 19:07:02 +0000 | [diff] [blame] | 29 | #define ARM64_WORKAROUND_845719 2 |
Marc Zyngier | 94a9e04 | 2015-06-12 12:06:36 +0100 | [diff] [blame] | 30 | #define ARM64_HAS_SYSREG_GIC_CPUIF 3 |
James Morse | 338d4f4 | 2015-07-22 19:05:54 +0100 | [diff] [blame] | 31 | #define ARM64_HAS_PAN 4 |
Will Deacon | c739dc8 | 2015-07-27 14:11:55 +0100 | [diff] [blame] | 32 | #define ARM64_HAS_LSE_ATOMICS 5 |
Robert Richter | 6d4e11c | 2015-09-21 22:58:35 +0200 | [diff] [blame] | 33 | #define ARM64_WORKAROUND_CAVIUM_23154 6 |
Marc Zyngier | 498cd5c | 2015-11-16 10:28:18 +0000 | [diff] [blame] | 34 | #define ARM64_WORKAROUND_834220 7 |
Will Deacon | d5370f7 | 2016-02-02 12:46:24 +0000 | [diff] [blame] | 35 | #define ARM64_HAS_NO_HW_PREFETCH 8 |
James Morse | 57f4959 | 2016-02-05 14:58:48 +0000 | [diff] [blame] | 36 | #define ARM64_HAS_UAO 9 |
James Morse | 7054419 | 2016-02-05 14:58:50 +0000 | [diff] [blame] | 37 | #define ARM64_ALT_PAN_NOT_UAO 10 |
Marc Zyngier | d88701b | 2015-01-29 11:24:05 +0000 | [diff] [blame] | 38 | #define ARM64_HAS_VIRT_HOST_EXTN 11 |
Andrew Pinski | 104a0c0 | 2016-02-24 17:44:57 -0800 | [diff] [blame] | 39 | #define ARM64_WORKAROUND_CAVIUM_27456 12 |
Suzuki K Poulose | 042446a | 2016-04-18 10:28:36 +0100 | [diff] [blame] | 40 | #define ARM64_HAS_32BIT_EL0 13 |
Marc Zyngier | 853c3b2 | 2016-06-30 18:40:38 +0100 | [diff] [blame] | 41 | #define ARM64_HYP_OFFSET_LOW 14 |
Suzuki K Poulose | 116c81f | 2016-09-09 14:07:16 +0100 | [diff] [blame] | 42 | #define ARM64_MISMATCHED_CACHE_LINE_SIZE 15 |
Andre Przywara | 301bcfa | 2014-11-14 15:54:10 +0000 | [diff] [blame] | 43 | |
Suzuki K Poulose | 116c81f | 2016-09-09 14:07:16 +0100 | [diff] [blame] | 44 | #define ARM64_NCAPS 16 |
Andre Przywara | 301bcfa | 2014-11-14 15:54:10 +0000 | [diff] [blame] | 45 | |
| 46 | #ifndef __ASSEMBLY__ |
Andre Przywara | 930da09 | 2014-11-14 15:54:07 +0000 | [diff] [blame] | 47 | |
Will Deacon | 144e969 | 2015-04-30 18:55:50 +0100 | [diff] [blame] | 48 | #include <linux/kernel.h> |
| 49 | |
Suzuki K. Poulose | 3c739b5 | 2015-10-19 14:24:45 +0100 | [diff] [blame] | 50 | /* CPU feature register tracking */ |
| 51 | enum ftr_type { |
| 52 | FTR_EXACT, /* Use a predefined safe value */ |
| 53 | FTR_LOWER_SAFE, /* Smaller value is safe */ |
| 54 | FTR_HIGHER_SAFE,/* Bigger value is safe */ |
| 55 | }; |
| 56 | |
| 57 | #define FTR_STRICT true /* SANITY check strict matching required */ |
| 58 | #define FTR_NONSTRICT false /* SANITY check ignored */ |
| 59 | |
Suzuki K. Poulose | 4f0a606 | 2015-11-18 17:08:57 +0000 | [diff] [blame] | 60 | #define FTR_SIGNED true /* Value should be treated as signed */ |
| 61 | #define FTR_UNSIGNED false /* Value should be treated as unsigned */ |
| 62 | |
Suzuki K. Poulose | 3c739b5 | 2015-10-19 14:24:45 +0100 | [diff] [blame] | 63 | struct arm64_ftr_bits { |
Suzuki K. Poulose | 4f0a606 | 2015-11-18 17:08:57 +0000 | [diff] [blame] | 64 | bool sign; /* Value is signed ? */ |
| 65 | bool strict; /* CPU Sanity check: strict matching required ? */ |
Suzuki K. Poulose | 3c739b5 | 2015-10-19 14:24:45 +0100 | [diff] [blame] | 66 | enum ftr_type type; |
| 67 | u8 shift; |
| 68 | u8 width; |
Suzuki K Poulose | ee7bc63 | 2016-09-09 14:07:08 +0100 | [diff] [blame] | 69 | s64 safe_val; /* safe value for FTR_EXACT features */ |
Suzuki K. Poulose | 3c739b5 | 2015-10-19 14:24:45 +0100 | [diff] [blame] | 70 | }; |
| 71 | |
| 72 | /* |
| 73 | * @arm64_ftr_reg - Feature register |
| 74 | * @strict_mask Bits which should match across all CPUs for sanity. |
| 75 | * @sys_val Safe value across the CPUs (system view) |
| 76 | */ |
| 77 | struct arm64_ftr_reg { |
Ard Biesheuvel | 5e49d73 | 2016-08-31 11:31:08 +0100 | [diff] [blame] | 78 | const char *name; |
| 79 | u64 strict_mask; |
| 80 | u64 sys_val; |
| 81 | const struct arm64_ftr_bits *ftr_bits; |
Suzuki K. Poulose | 3c739b5 | 2015-10-19 14:24:45 +0100 | [diff] [blame] | 82 | }; |
| 83 | |
Ard Biesheuvel | 675b056 | 2016-08-31 11:31:10 +0100 | [diff] [blame] | 84 | extern struct arm64_ftr_reg arm64_ftr_reg_ctrel0; |
| 85 | |
Suzuki K Poulose | 92406f0 | 2016-04-22 12:25:31 +0100 | [diff] [blame] | 86 | /* scope of capability check */ |
| 87 | enum { |
| 88 | SCOPE_SYSTEM, |
| 89 | SCOPE_LOCAL_CPU, |
| 90 | }; |
| 91 | |
Marc Zyngier | 359b706 | 2015-03-27 13:09:23 +0000 | [diff] [blame] | 92 | struct arm64_cpu_capabilities { |
| 93 | const char *desc; |
| 94 | u16 capability; |
Suzuki K Poulose | 92406f0 | 2016-04-22 12:25:31 +0100 | [diff] [blame] | 95 | int def_scope; /* default scope */ |
| 96 | bool (*matches)(const struct arm64_cpu_capabilities *caps, int scope); |
James Morse | 2a6dcb2 | 2016-10-18 11:27:46 +0100 | [diff] [blame^] | 97 | int (*enable)(void *); /* Called on all active CPUs */ |
Marc Zyngier | 359b706 | 2015-03-27 13:09:23 +0000 | [diff] [blame] | 98 | union { |
| 99 | struct { /* To be used for erratum handling only */ |
| 100 | u32 midr_model; |
| 101 | u32 midr_range_min, midr_range_max; |
| 102 | }; |
Marc Zyngier | 94a9e04 | 2015-06-12 12:06:36 +0100 | [diff] [blame] | 103 | |
| 104 | struct { /* Feature register checking */ |
Suzuki K. Poulose | da8d02d | 2015-10-19 14:24:51 +0100 | [diff] [blame] | 105 | u32 sys_reg; |
Suzuki K Poulose | ff96f7b | 2016-01-26 10:58:15 +0000 | [diff] [blame] | 106 | u8 field_pos; |
| 107 | u8 min_field_value; |
| 108 | u8 hwcap_type; |
| 109 | bool sign; |
Suzuki K. Poulose | 37b01d53 | 2015-10-19 14:24:52 +0100 | [diff] [blame] | 110 | unsigned long hwcap; |
Marc Zyngier | 94a9e04 | 2015-06-12 12:06:36 +0100 | [diff] [blame] | 111 | }; |
Marc Zyngier | 359b706 | 2015-03-27 13:09:23 +0000 | [diff] [blame] | 112 | }; |
| 113 | }; |
| 114 | |
Fabio Estevam | 06f9eb8 | 2014-12-04 01:17:01 +0000 | [diff] [blame] | 115 | extern DECLARE_BITMAP(cpu_hwcaps, ARM64_NCAPS); |
Catalin Marinas | efd9e03 | 2016-09-05 18:25:48 +0100 | [diff] [blame] | 116 | extern struct static_key_false cpu_hwcap_keys[ARM64_NCAPS]; |
Andre Przywara | 930da09 | 2014-11-14 15:54:07 +0000 | [diff] [blame] | 117 | |
Marc Zyngier | e3661b1 | 2016-04-22 12:25:32 +0100 | [diff] [blame] | 118 | bool this_cpu_has_cap(unsigned int cap); |
| 119 | |
Ard Biesheuvel | 3be1a5c | 2014-03-04 01:10:04 +0000 | [diff] [blame] | 120 | static inline bool cpu_have_feature(unsigned int num) |
| 121 | { |
| 122 | return elf_hwcap & (1UL << num); |
| 123 | } |
| 124 | |
Andre Przywara | 930da09 | 2014-11-14 15:54:07 +0000 | [diff] [blame] | 125 | static inline bool cpus_have_cap(unsigned int num) |
| 126 | { |
Fabio Estevam | 06f9eb8 | 2014-12-04 01:17:01 +0000 | [diff] [blame] | 127 | if (num >= ARM64_NCAPS) |
Andre Przywara | 930da09 | 2014-11-14 15:54:07 +0000 | [diff] [blame] | 128 | return false; |
Catalin Marinas | efd9e03 | 2016-09-05 18:25:48 +0100 | [diff] [blame] | 129 | if (__builtin_constant_p(num)) |
| 130 | return static_branch_unlikely(&cpu_hwcap_keys[num]); |
| 131 | else |
| 132 | return test_bit(num, cpu_hwcaps); |
Andre Przywara | 930da09 | 2014-11-14 15:54:07 +0000 | [diff] [blame] | 133 | } |
| 134 | |
| 135 | static inline void cpus_set_cap(unsigned int num) |
| 136 | { |
Catalin Marinas | efd9e03 | 2016-09-05 18:25:48 +0100 | [diff] [blame] | 137 | if (num >= ARM64_NCAPS) { |
Andre Przywara | 930da09 | 2014-11-14 15:54:07 +0000 | [diff] [blame] | 138 | pr_warn("Attempt to set an illegal CPU capability (%d >= %d)\n", |
Fabio Estevam | 06f9eb8 | 2014-12-04 01:17:01 +0000 | [diff] [blame] | 139 | num, ARM64_NCAPS); |
Catalin Marinas | efd9e03 | 2016-09-05 18:25:48 +0100 | [diff] [blame] | 140 | } else { |
Andre Przywara | 930da09 | 2014-11-14 15:54:07 +0000 | [diff] [blame] | 141 | __set_bit(num, cpu_hwcaps); |
Catalin Marinas | efd9e03 | 2016-09-05 18:25:48 +0100 | [diff] [blame] | 142 | static_branch_enable(&cpu_hwcap_keys[num]); |
| 143 | } |
Andre Przywara | 930da09 | 2014-11-14 15:54:07 +0000 | [diff] [blame] | 144 | } |
| 145 | |
Suzuki K. Poulose | ce98a67 | 2015-10-19 14:24:44 +0100 | [diff] [blame] | 146 | static inline int __attribute_const__ |
Suzuki K Poulose | 28c5dcb | 2016-01-26 10:58:16 +0000 | [diff] [blame] | 147 | cpuid_feature_extract_signed_field_width(u64 features, int field, int width) |
James Morse | 79b0e09 | 2015-07-21 13:23:26 +0100 | [diff] [blame] | 148 | { |
Suzuki K. Poulose | ce98a67 | 2015-10-19 14:24:44 +0100 | [diff] [blame] | 149 | return (s64)(features << (64 - width - field)) >> (64 - width); |
James Morse | 79b0e09 | 2015-07-21 13:23:26 +0100 | [diff] [blame] | 150 | } |
| 151 | |
Suzuki K. Poulose | ce98a67 | 2015-10-19 14:24:44 +0100 | [diff] [blame] | 152 | static inline int __attribute_const__ |
Suzuki K Poulose | 28c5dcb | 2016-01-26 10:58:16 +0000 | [diff] [blame] | 153 | cpuid_feature_extract_signed_field(u64 features, int field) |
Suzuki K. Poulose | ce98a67 | 2015-10-19 14:24:44 +0100 | [diff] [blame] | 154 | { |
Suzuki K Poulose | 28c5dcb | 2016-01-26 10:58:16 +0000 | [diff] [blame] | 155 | return cpuid_feature_extract_signed_field_width(features, field, 4); |
James Morse | 79b0e09 | 2015-07-21 13:23:26 +0100 | [diff] [blame] | 156 | } |
James Morse | 79b0e09 | 2015-07-21 13:23:26 +0100 | [diff] [blame] | 157 | |
Suzuki K. Poulose | d211827 | 2015-11-18 17:08:56 +0000 | [diff] [blame] | 158 | static inline unsigned int __attribute_const__ |
| 159 | cpuid_feature_extract_unsigned_field_width(u64 features, int field, int width) |
| 160 | { |
| 161 | return (u64)(features << (64 - width - field)) >> (64 - width); |
| 162 | } |
| 163 | |
| 164 | static inline unsigned int __attribute_const__ |
| 165 | cpuid_feature_extract_unsigned_field(u64 features, int field) |
| 166 | { |
| 167 | return cpuid_feature_extract_unsigned_field_width(features, field, 4); |
| 168 | } |
| 169 | |
Ard Biesheuvel | 5e49d73 | 2016-08-31 11:31:08 +0100 | [diff] [blame] | 170 | static inline u64 arm64_ftr_mask(const struct arm64_ftr_bits *ftrp) |
Suzuki K. Poulose | 3c739b5 | 2015-10-19 14:24:45 +0100 | [diff] [blame] | 171 | { |
| 172 | return (u64)GENMASK(ftrp->shift + ftrp->width - 1, ftrp->shift); |
| 173 | } |
| 174 | |
Suzuki K Poulose | 28c5dcb | 2016-01-26 10:58:16 +0000 | [diff] [blame] | 175 | static inline int __attribute_const__ |
| 176 | cpuid_feature_extract_field(u64 features, int field, bool sign) |
| 177 | { |
| 178 | return (sign) ? |
| 179 | cpuid_feature_extract_signed_field(features, field) : |
| 180 | cpuid_feature_extract_unsigned_field(features, field); |
| 181 | } |
| 182 | |
Ard Biesheuvel | 5e49d73 | 2016-08-31 11:31:08 +0100 | [diff] [blame] | 183 | static inline s64 arm64_ftr_value(const struct arm64_ftr_bits *ftrp, u64 val) |
Suzuki K. Poulose | 3c739b5 | 2015-10-19 14:24:45 +0100 | [diff] [blame] | 184 | { |
Suzuki K Poulose | 28c5dcb | 2016-01-26 10:58:16 +0000 | [diff] [blame] | 185 | return (s64)cpuid_feature_extract_field(val, ftrp->shift, ftrp->sign); |
Suzuki K. Poulose | 3c739b5 | 2015-10-19 14:24:45 +0100 | [diff] [blame] | 186 | } |
| 187 | |
Suzuki K. Poulose | cdcf817 | 2015-10-19 14:24:42 +0100 | [diff] [blame] | 188 | static inline bool id_aa64mmfr0_mixed_endian_el0(u64 mmfr0) |
| 189 | { |
Suzuki K Poulose | 28c5dcb | 2016-01-26 10:58:16 +0000 | [diff] [blame] | 190 | return cpuid_feature_extract_unsigned_field(mmfr0, ID_AA64MMFR0_BIGENDEL_SHIFT) == 0x1 || |
| 191 | cpuid_feature_extract_unsigned_field(mmfr0, ID_AA64MMFR0_BIGENDEL0_SHIFT) == 0x1; |
Suzuki K. Poulose | cdcf817 | 2015-10-19 14:24:42 +0100 | [diff] [blame] | 192 | } |
| 193 | |
Suzuki K Poulose | c80aba8 | 2016-04-18 10:28:34 +0100 | [diff] [blame] | 194 | static inline bool id_aa64pfr0_32bit_el0(u64 pfr0) |
| 195 | { |
| 196 | u32 val = cpuid_feature_extract_unsigned_field(pfr0, ID_AA64PFR0_EL0_SHIFT); |
| 197 | |
| 198 | return val == ID_AA64PFR0_EL0_32BIT_64BIT; |
| 199 | } |
| 200 | |
Suzuki K. Poulose | 3a75578 | 2015-10-19 14:24:39 +0100 | [diff] [blame] | 201 | void __init setup_cpu_features(void); |
Andre Przywara | e116a37 | 2014-11-14 15:54:09 +0000 | [diff] [blame] | 202 | |
Suzuki K. Poulose | ce8b602 | 2015-10-19 14:24:49 +0100 | [diff] [blame] | 203 | void update_cpu_capabilities(const struct arm64_cpu_capabilities *caps, |
Ard Biesheuvel | 3be1a5c | 2014-03-04 01:10:04 +0000 | [diff] [blame] | 204 | const char *info); |
Andre Przywara | 8e23185 | 2016-06-28 18:07:30 +0100 | [diff] [blame] | 205 | void enable_cpu_capabilities(const struct arm64_cpu_capabilities *caps); |
Suzuki K Poulose | c47a190 | 2016-09-09 14:07:10 +0100 | [diff] [blame] | 206 | void check_local_cpu_capabilities(void); |
| 207 | |
Suzuki K Poulose | 89ba264 | 2016-09-09 14:07:09 +0100 | [diff] [blame] | 208 | void update_cpu_errata_workarounds(void); |
Andre Przywara | 8e23185 | 2016-06-28 18:07:30 +0100 | [diff] [blame] | 209 | void __init enable_errata_workarounds(void); |
Suzuki K Poulose | 89ba264 | 2016-09-09 14:07:09 +0100 | [diff] [blame] | 210 | void verify_local_cpu_errata_workarounds(void); |
Ard Biesheuvel | 3be1a5c | 2014-03-04 01:10:04 +0000 | [diff] [blame] | 211 | |
Suzuki K. Poulose | b3f1537 | 2015-10-19 14:24:47 +0100 | [diff] [blame] | 212 | u64 read_system_reg(u32 id); |
| 213 | |
Suzuki K. Poulose | c1e8656 | 2015-10-19 14:24:48 +0100 | [diff] [blame] | 214 | static inline bool cpu_supports_mixed_endian_el0(void) |
| 215 | { |
| 216 | return id_aa64mmfr0_mixed_endian_el0(read_cpuid(ID_AA64MMFR0_EL1)); |
| 217 | } |
| 218 | |
Suzuki K Poulose | 042446a | 2016-04-18 10:28:36 +0100 | [diff] [blame] | 219 | static inline bool system_supports_32bit_el0(void) |
| 220 | { |
| 221 | return cpus_have_cap(ARM64_HAS_32BIT_EL0); |
| 222 | } |
| 223 | |
Suzuki K. Poulose | c1e8656 | 2015-10-19 14:24:48 +0100 | [diff] [blame] | 224 | static inline bool system_supports_mixed_endian_el0(void) |
| 225 | { |
| 226 | return id_aa64mmfr0_mixed_endian_el0(read_system_reg(SYS_ID_AA64MMFR0_EL1)); |
| 227 | } |
Ard Biesheuvel | 3be1a5c | 2014-03-04 01:10:04 +0000 | [diff] [blame] | 228 | |
| 229 | #endif /* __ASSEMBLY__ */ |
| 230 | |
| 231 | #endif |