Zhang Xiantao | 1d737c8 | 2007-12-14 09:35:10 +0800 | [diff] [blame] | 1 | #ifndef __KVM_X86_MMU_H |
| 2 | #define __KVM_X86_MMU_H |
| 3 | |
Avi Kivity | edf8841 | 2007-12-16 11:02:48 +0200 | [diff] [blame] | 4 | #include <linux/kvm_host.h> |
Avi Kivity | fc78f51 | 2009-12-07 12:16:48 +0200 | [diff] [blame] | 5 | #include "kvm_cache_regs.h" |
Zhang Xiantao | 1d737c8 | 2007-12-14 09:35:10 +0800 | [diff] [blame] | 6 | |
Sheng Yang | 8c6d6ad | 2008-04-25 10:17:08 +0800 | [diff] [blame] | 7 | #define PT64_PT_BITS 9 |
| 8 | #define PT64_ENT_PER_PAGE (1 << PT64_PT_BITS) |
| 9 | #define PT32_PT_BITS 10 |
| 10 | #define PT32_ENT_PER_PAGE (1 << PT32_PT_BITS) |
| 11 | |
| 12 | #define PT_WRITABLE_SHIFT 1 |
| 13 | |
| 14 | #define PT_PRESENT_MASK (1ULL << 0) |
| 15 | #define PT_WRITABLE_MASK (1ULL << PT_WRITABLE_SHIFT) |
| 16 | #define PT_USER_MASK (1ULL << 2) |
| 17 | #define PT_PWT_MASK (1ULL << 3) |
| 18 | #define PT_PCD_MASK (1ULL << 4) |
Avi Kivity | 1b7fcd3 | 2008-05-15 13:51:35 +0300 | [diff] [blame] | 19 | #define PT_ACCESSED_SHIFT 5 |
| 20 | #define PT_ACCESSED_MASK (1ULL << PT_ACCESSED_SHIFT) |
Avi Kivity | 8ea667f | 2012-09-12 13:44:53 +0300 | [diff] [blame] | 21 | #define PT_DIRTY_SHIFT 6 |
| 22 | #define PT_DIRTY_MASK (1ULL << PT_DIRTY_SHIFT) |
Avi Kivity | 6fd01b7 | 2012-09-12 20:46:56 +0300 | [diff] [blame] | 23 | #define PT_PAGE_SIZE_SHIFT 7 |
| 24 | #define PT_PAGE_SIZE_MASK (1ULL << PT_PAGE_SIZE_SHIFT) |
Sheng Yang | 8c6d6ad | 2008-04-25 10:17:08 +0800 | [diff] [blame] | 25 | #define PT_PAT_MASK (1ULL << 7) |
| 26 | #define PT_GLOBAL_MASK (1ULL << 8) |
| 27 | #define PT64_NX_SHIFT 63 |
| 28 | #define PT64_NX_MASK (1ULL << PT64_NX_SHIFT) |
| 29 | |
| 30 | #define PT_PAT_SHIFT 7 |
| 31 | #define PT_DIR_PAT_SHIFT 12 |
| 32 | #define PT_DIR_PAT_MASK (1ULL << PT_DIR_PAT_SHIFT) |
| 33 | |
| 34 | #define PT32_DIR_PSE36_SIZE 4 |
| 35 | #define PT32_DIR_PSE36_SHIFT 13 |
| 36 | #define PT32_DIR_PSE36_MASK \ |
| 37 | (((1ULL << PT32_DIR_PSE36_SIZE) - 1) << PT32_DIR_PSE36_SHIFT) |
| 38 | |
| 39 | #define PT64_ROOT_LEVEL 4 |
| 40 | #define PT32_ROOT_LEVEL 2 |
| 41 | #define PT32E_ROOT_LEVEL 3 |
| 42 | |
Sheng Yang | c9c5417 | 2010-01-05 19:02:26 +0800 | [diff] [blame] | 43 | #define PT_PDPE_LEVEL 3 |
| 44 | #define PT_DIRECTORY_LEVEL 2 |
| 45 | #define PT_PAGE_TABLE_LEVEL 1 |
Xiao Guangrong | 8a3d08f | 2015-05-13 14:42:21 +0800 | [diff] [blame] | 46 | #define PT_MAX_HUGEPAGE_LEVEL (PT_PAGE_TABLE_LEVEL + KVM_NR_PAGE_SIZES - 1) |
Sheng Yang | c9c5417 | 2010-01-05 19:02:26 +0800 | [diff] [blame] | 47 | |
Tiejun Chen | d143148 | 2014-09-01 18:44:04 +0800 | [diff] [blame] | 48 | static inline u64 rsvd_bits(int s, int e) |
| 49 | { |
| 50 | return ((1ULL << (e - s + 1)) - 1) << s; |
| 51 | } |
| 52 | |
Xiao Guangrong | ce88dec | 2011-07-12 03:33:44 +0800 | [diff] [blame] | 53 | void kvm_mmu_set_mmio_spte_mask(u64 mmio_mask); |
Xiao Guangrong | b37fbea | 2013-06-07 16:51:25 +0800 | [diff] [blame] | 54 | |
Xiao Guangrong | c258b62 | 2015-08-05 12:04:24 +0800 | [diff] [blame] | 55 | void |
| 56 | reset_shadow_zero_bits_mask(struct kvm_vcpu *vcpu, struct kvm_mmu *context); |
| 57 | |
Xiao Guangrong | b37fbea | 2013-06-07 16:51:25 +0800 | [diff] [blame] | 58 | /* |
Paolo Bonzini | 450869d | 2015-11-04 13:41:21 +0100 | [diff] [blame] | 59 | * Return values of handle_mmio_page_fault: |
Xiao Guangrong | b37fbea | 2013-06-07 16:51:25 +0800 | [diff] [blame] | 60 | * RET_MMIO_PF_EMULATE: it is a real mmio page fault, emulate the instruction |
Xiao Guangrong | f8f5594 | 2013-06-07 16:51:26 +0800 | [diff] [blame] | 61 | * directly. |
| 62 | * RET_MMIO_PF_INVALID: invalid spte is detected then let the real page |
| 63 | * fault path update the mmio spte. |
Xiao Guangrong | b37fbea | 2013-06-07 16:51:25 +0800 | [diff] [blame] | 64 | * RET_MMIO_PF_RETRY: let CPU fault again on the address. |
Paolo Bonzini | 450869d | 2015-11-04 13:41:21 +0100 | [diff] [blame] | 65 | * RET_MMIO_PF_BUG: a bug was detected (and a WARN was printed). |
Xiao Guangrong | b37fbea | 2013-06-07 16:51:25 +0800 | [diff] [blame] | 66 | */ |
| 67 | enum { |
| 68 | RET_MMIO_PF_EMULATE = 1, |
Xiao Guangrong | f8f5594 | 2013-06-07 16:51:26 +0800 | [diff] [blame] | 69 | RET_MMIO_PF_INVALID = 2, |
Xiao Guangrong | b37fbea | 2013-06-07 16:51:25 +0800 | [diff] [blame] | 70 | RET_MMIO_PF_RETRY = 0, |
| 71 | RET_MMIO_PF_BUG = -1 |
| 72 | }; |
| 73 | |
Paolo Bonzini | 450869d | 2015-11-04 13:41:21 +0100 | [diff] [blame] | 74 | int handle_mmio_page_fault(struct kvm_vcpu *vcpu, u64 addr, bool direct); |
Paolo Bonzini | ad896af | 2013-10-02 16:56:14 +0200 | [diff] [blame] | 75 | void kvm_init_shadow_mmu(struct kvm_vcpu *vcpu); |
| 76 | void kvm_init_shadow_ept_mmu(struct kvm_vcpu *vcpu, bool execonly); |
Marcelo Tosatti | 94d8b05 | 2009-06-11 12:07:42 -0300 | [diff] [blame] | 77 | |
Dave Hansen | e0df7b9 | 2010-08-19 18:11:05 -0700 | [diff] [blame] | 78 | static inline unsigned int kvm_mmu_available_pages(struct kvm *kvm) |
| 79 | { |
Marcelo Tosatti | 5d21881 | 2013-03-12 22:36:43 -0300 | [diff] [blame] | 80 | if (kvm->arch.n_max_mmu_pages > kvm->arch.n_used_mmu_pages) |
| 81 | return kvm->arch.n_max_mmu_pages - |
| 82 | kvm->arch.n_used_mmu_pages; |
| 83 | |
| 84 | return 0; |
Dave Hansen | e0df7b9 | 2010-08-19 18:11:05 -0700 | [diff] [blame] | 85 | } |
| 86 | |
Zhang Xiantao | 1d737c8 | 2007-12-14 09:35:10 +0800 | [diff] [blame] | 87 | static inline int kvm_mmu_reload(struct kvm_vcpu *vcpu) |
| 88 | { |
| 89 | if (likely(vcpu->arch.mmu.root_hpa != INVALID_PAGE)) |
| 90 | return 0; |
| 91 | |
| 92 | return kvm_mmu_load(vcpu); |
| 93 | } |
| 94 | |
Avi Kivity | 43a3795 | 2009-06-10 14:12:05 +0300 | [diff] [blame] | 95 | static inline int is_present_gpte(unsigned long pte) |
Dong, Eddie | 20c466b | 2009-03-31 23:03:45 +0800 | [diff] [blame] | 96 | { |
| 97 | return pte & PT_PRESENT_MASK; |
| 98 | } |
| 99 | |
Xiao Guangrong | 198c74f | 2014-04-17 17:06:16 +0800 | [diff] [blame] | 100 | /* |
| 101 | * Currently, we have two sorts of write-protection, a) the first one |
| 102 | * write-protects guest page to sync the guest modification, b) another one is |
| 103 | * used to sync dirty bitmap when we do KVM_GET_DIRTY_LOG. The differences |
| 104 | * between these two sorts are: |
| 105 | * 1) the first case clears SPTE_MMU_WRITEABLE bit. |
| 106 | * 2) the first case requires flushing tlb immediately avoiding corrupting |
| 107 | * shadow page table between all vcpus so it should be in the protection of |
| 108 | * mmu-lock. And the another case does not need to flush tlb until returning |
| 109 | * the dirty bitmap to userspace since it only write-protects the page |
| 110 | * logged in the bitmap, that means the page in the dirty bitmap is not |
| 111 | * missed, so it can flush tlb out of mmu-lock. |
| 112 | * |
| 113 | * So, there is the problem: the first case can meet the corrupted tlb caused |
| 114 | * by another case which write-protects pages but without flush tlb |
| 115 | * immediately. In order to making the first case be aware this problem we let |
| 116 | * it flush tlb if we try to write-protect a spte whose SPTE_MMU_WRITEABLE bit |
| 117 | * is set, it works since another case never touches SPTE_MMU_WRITEABLE bit. |
| 118 | * |
| 119 | * Anyway, whenever a spte is updated (only permission and status bits are |
| 120 | * changed) we need to check whether the spte with SPTE_MMU_WRITEABLE becomes |
| 121 | * readonly, if that happens, we need to flush tlb. Fortunately, |
| 122 | * mmu_spte_update() has already handled it perfectly. |
| 123 | * |
| 124 | * The rules to use SPTE_MMU_WRITEABLE and PT_WRITABLE_MASK: |
| 125 | * - if we want to see if it has writable tlb entry or if the spte can be |
| 126 | * writable on the mmu mapping, check SPTE_MMU_WRITEABLE, this is the most |
| 127 | * case, otherwise |
| 128 | * - if we fix page fault on the spte or do write-protection by dirty logging, |
| 129 | * check PT_WRITABLE_MASK. |
| 130 | * |
| 131 | * TODO: introduce APIs to split these two cases. |
| 132 | */ |
Xiao Guangrong | bebb106 | 2011-07-12 03:23:20 +0800 | [diff] [blame] | 133 | static inline int is_writable_pte(unsigned long pte) |
| 134 | { |
| 135 | return pte & PT_WRITABLE_MASK; |
| 136 | } |
| 137 | |
| 138 | static inline bool is_write_protection(struct kvm_vcpu *vcpu) |
| 139 | { |
| 140 | return kvm_read_cr0_bits(vcpu, X86_CR0_WP); |
| 141 | } |
| 142 | |
Avi Kivity | 97d64b7 | 2012-09-12 14:52:00 +0300 | [diff] [blame] | 143 | /* |
| 144 | * Will a fault with a given page-fault error code (pfec) cause a permission |
| 145 | * fault with the given access (in ACC_* format)? |
| 146 | */ |
Feng Wu | 97ec8c0 | 2014-04-01 17:46:34 +0800 | [diff] [blame] | 147 | static inline bool permission_fault(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu, |
| 148 | unsigned pte_access, unsigned pfec) |
Xiao Guangrong | bebb106 | 2011-07-12 03:23:20 +0800 | [diff] [blame] | 149 | { |
Feng Wu | 97ec8c0 | 2014-04-01 17:46:34 +0800 | [diff] [blame] | 150 | int cpl = kvm_x86_ops->get_cpl(vcpu); |
| 151 | unsigned long rflags = kvm_x86_ops->get_rflags(vcpu); |
| 152 | |
| 153 | /* |
| 154 | * If CPL < 3, SMAP prevention are disabled if EFLAGS.AC = 1. |
| 155 | * |
| 156 | * If CPL = 3, SMAP applies to all supervisor-mode data accesses |
| 157 | * (these are implicit supervisor accesses) regardless of the value |
| 158 | * of EFLAGS.AC. |
| 159 | * |
| 160 | * This computes (cpl < 3) && (rflags & X86_EFLAGS_AC), leaving |
| 161 | * the result in X86_EFLAGS_AC. We then insert it in place of |
| 162 | * the PFERR_RSVD_MASK bit; this bit will always be zero in pfec, |
| 163 | * but it will be one in index if SMAP checks are being overridden. |
| 164 | * It is important to keep this branchless. |
| 165 | */ |
| 166 | unsigned long smap = (cpl - 3) & (rflags & X86_EFLAGS_AC); |
| 167 | int index = (pfec >> 1) + |
| 168 | (smap >> (X86_EFLAGS_AC_BIT - PFERR_RSVD_BIT + 1)); |
| 169 | |
Xiao Guangrong | ceee7df | 2015-05-07 16:20:15 +0800 | [diff] [blame] | 170 | WARN_ON(pfec & PFERR_RSVD_MASK); |
| 171 | |
Feng Wu | 97ec8c0 | 2014-04-01 17:46:34 +0800 | [diff] [blame] | 172 | return (mmu->permissions[index] >> pte_access) & 1; |
Xiao Guangrong | bebb106 | 2011-07-12 03:23:20 +0800 | [diff] [blame] | 173 | } |
Avi Kivity | 97d64b7 | 2012-09-12 14:52:00 +0300 | [diff] [blame] | 174 | |
Xiao Guangrong | 5304b8d | 2013-05-31 08:36:22 +0800 | [diff] [blame] | 175 | void kvm_mmu_invalidate_zap_all_pages(struct kvm *kvm); |
Xiao Guangrong | efdfe53 | 2015-05-13 14:42:27 +0800 | [diff] [blame] | 176 | void kvm_zap_gfn_range(struct kvm *kvm, gfn_t gfn_start, gfn_t gfn_end); |
Xiao Guangrong | 547ffae | 2016-02-24 17:51:07 +0800 | [diff] [blame] | 177 | |
| 178 | void kvm_mmu_gfn_disallow_lpage(struct kvm_memory_slot *slot, gfn_t gfn); |
| 179 | void kvm_mmu_gfn_allow_lpage(struct kvm_memory_slot *slot, gfn_t gfn); |
Xiao Guangrong | aeecee2 | 2016-02-24 17:51:08 +0800 | [diff] [blame^] | 180 | bool kvm_mmu_slot_gfn_write_protect(struct kvm *kvm, |
| 181 | struct kvm_memory_slot *slot, u64 gfn); |
Zhang Xiantao | 1d737c8 | 2007-12-14 09:35:10 +0800 | [diff] [blame] | 182 | #endif |