Marc Zyngier | 59529f6 | 2015-11-30 13:09:53 +0000 | [diff] [blame] | 1 | /* |
| 2 | * This program is free software; you can redistribute it and/or modify |
| 3 | * it under the terms of the GNU General Public License version 2 as |
| 4 | * published by the Free Software Foundation. |
| 5 | * |
| 6 | * This program is distributed in the hope that it will be useful, |
| 7 | * but WITHOUT ANY WARRANTY; without even the implied warranty of |
| 8 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
| 9 | * GNU General Public License for more details. |
| 10 | * |
| 11 | * You should have received a copy of the GNU General Public License |
| 12 | * along with this program. If not, see <http://www.gnu.org/licenses/>. |
| 13 | */ |
| 14 | |
| 15 | #include <linux/irqchip/arm-gic-v3.h> |
| 16 | #include <linux/kvm.h> |
| 17 | #include <linux/kvm_host.h> |
Eric Auger | 9097773 | 2015-12-01 15:02:35 +0100 | [diff] [blame] | 18 | #include <kvm/arm_vgic.h> |
| 19 | #include <asm/kvm_mmu.h> |
| 20 | #include <asm/kvm_asm.h> |
Marc Zyngier | 59529f6 | 2015-11-30 13:09:53 +0000 | [diff] [blame] | 21 | |
| 22 | #include "vgic.h" |
| 23 | |
| 24 | void vgic_v3_process_maintenance(struct kvm_vcpu *vcpu) |
| 25 | { |
| 26 | struct vgic_v3_cpu_if *cpuif = &vcpu->arch.vgic_cpu.vgic_v3; |
| 27 | u32 model = vcpu->kvm->arch.vgic.vgic_model; |
| 28 | |
| 29 | if (cpuif->vgic_misr & ICH_MISR_EOI) { |
| 30 | unsigned long eisr_bmap = cpuif->vgic_eisr; |
| 31 | int lr; |
| 32 | |
| 33 | for_each_set_bit(lr, &eisr_bmap, kvm_vgic_global_state.nr_lr) { |
| 34 | u32 intid; |
| 35 | u64 val = cpuif->vgic_lr[lr]; |
| 36 | |
| 37 | if (model == KVM_DEV_TYPE_ARM_VGIC_V3) |
| 38 | intid = val & ICH_LR_VIRTUAL_ID_MASK; |
| 39 | else |
| 40 | intid = val & GICH_LR_VIRTUALID; |
| 41 | |
| 42 | WARN_ON(cpuif->vgic_lr[lr] & ICH_LR_STATE); |
| 43 | |
| 44 | kvm_notify_acked_irq(vcpu->kvm, 0, |
| 45 | intid - VGIC_NR_PRIVATE_IRQS); |
| 46 | } |
| 47 | |
| 48 | /* |
| 49 | * In the next iterations of the vcpu loop, if we sync |
| 50 | * the vgic state after flushing it, but before |
| 51 | * entering the guest (this happens for pending |
| 52 | * signals and vmid rollovers), then make sure we |
| 53 | * don't pick up any old maintenance interrupts here. |
| 54 | */ |
| 55 | cpuif->vgic_eisr = 0; |
| 56 | } |
| 57 | |
| 58 | cpuif->vgic_hcr &= ~ICH_HCR_UIE; |
| 59 | } |
| 60 | |
| 61 | void vgic_v3_set_underflow(struct kvm_vcpu *vcpu) |
| 62 | { |
| 63 | struct vgic_v3_cpu_if *cpuif = &vcpu->arch.vgic_cpu.vgic_v3; |
| 64 | |
| 65 | cpuif->vgic_hcr |= ICH_HCR_UIE; |
| 66 | } |
| 67 | |
| 68 | void vgic_v3_fold_lr_state(struct kvm_vcpu *vcpu) |
| 69 | { |
| 70 | struct vgic_v3_cpu_if *cpuif = &vcpu->arch.vgic_cpu.vgic_v3; |
| 71 | u32 model = vcpu->kvm->arch.vgic.vgic_model; |
| 72 | int lr; |
| 73 | |
| 74 | for (lr = 0; lr < vcpu->arch.vgic_cpu.used_lrs; lr++) { |
| 75 | u64 val = cpuif->vgic_lr[lr]; |
| 76 | u32 intid; |
| 77 | struct vgic_irq *irq; |
| 78 | |
| 79 | if (model == KVM_DEV_TYPE_ARM_VGIC_V3) |
| 80 | intid = val & ICH_LR_VIRTUAL_ID_MASK; |
| 81 | else |
| 82 | intid = val & GICH_LR_VIRTUALID; |
| 83 | irq = vgic_get_irq(vcpu->kvm, vcpu, intid); |
Andre Przywara | 3802411 | 2016-07-15 12:43:33 +0100 | [diff] [blame] | 84 | if (!irq) /* An LPI could have been unmapped. */ |
| 85 | continue; |
Marc Zyngier | 59529f6 | 2015-11-30 13:09:53 +0000 | [diff] [blame] | 86 | |
| 87 | spin_lock(&irq->irq_lock); |
| 88 | |
| 89 | /* Always preserve the active bit */ |
| 90 | irq->active = !!(val & ICH_LR_ACTIVE_BIT); |
| 91 | |
| 92 | /* Edge is the only case where we preserve the pending bit */ |
| 93 | if (irq->config == VGIC_CONFIG_EDGE && |
| 94 | (val & ICH_LR_PENDING_BIT)) { |
| 95 | irq->pending = true; |
| 96 | |
| 97 | if (vgic_irq_is_sgi(intid) && |
| 98 | model == KVM_DEV_TYPE_ARM_VGIC_V2) { |
| 99 | u32 cpuid = val & GICH_LR_PHYSID_CPUID; |
| 100 | |
| 101 | cpuid >>= GICH_LR_PHYSID_CPUID_SHIFT; |
| 102 | irq->source |= (1 << cpuid); |
| 103 | } |
| 104 | } |
| 105 | |
Marc Zyngier | 637d122 | 2016-05-25 15:26:36 +0100 | [diff] [blame] | 106 | /* |
| 107 | * Clear soft pending state when level irqs have been acked. |
| 108 | * Always regenerate the pending state. |
| 109 | */ |
| 110 | if (irq->config == VGIC_CONFIG_LEVEL) { |
| 111 | if (!(val & ICH_LR_PENDING_BIT)) |
| 112 | irq->soft_pending = false; |
| 113 | |
| 114 | irq->pending = irq->line_level || irq->soft_pending; |
Marc Zyngier | 59529f6 | 2015-11-30 13:09:53 +0000 | [diff] [blame] | 115 | } |
| 116 | |
| 117 | spin_unlock(&irq->irq_lock); |
Andre Przywara | 5dd4b92 | 2016-07-15 12:43:27 +0100 | [diff] [blame] | 118 | vgic_put_irq(vcpu->kvm, irq); |
Marc Zyngier | 59529f6 | 2015-11-30 13:09:53 +0000 | [diff] [blame] | 119 | } |
| 120 | } |
| 121 | |
| 122 | /* Requires the irq to be locked already */ |
| 123 | void vgic_v3_populate_lr(struct kvm_vcpu *vcpu, struct vgic_irq *irq, int lr) |
| 124 | { |
| 125 | u32 model = vcpu->kvm->arch.vgic.vgic_model; |
| 126 | u64 val = irq->intid; |
| 127 | |
| 128 | if (irq->pending) { |
| 129 | val |= ICH_LR_PENDING_BIT; |
| 130 | |
| 131 | if (irq->config == VGIC_CONFIG_EDGE) |
| 132 | irq->pending = false; |
| 133 | |
| 134 | if (vgic_irq_is_sgi(irq->intid) && |
| 135 | model == KVM_DEV_TYPE_ARM_VGIC_V2) { |
| 136 | u32 src = ffs(irq->source); |
| 137 | |
| 138 | BUG_ON(!src); |
| 139 | val |= (src - 1) << GICH_LR_PHYSID_CPUID_SHIFT; |
| 140 | irq->source &= ~(1 << (src - 1)); |
| 141 | if (irq->source) |
| 142 | irq->pending = true; |
| 143 | } |
| 144 | } |
| 145 | |
| 146 | if (irq->active) |
| 147 | val |= ICH_LR_ACTIVE_BIT; |
| 148 | |
| 149 | if (irq->hw) { |
| 150 | val |= ICH_LR_HW; |
| 151 | val |= ((u64)irq->hwintid) << ICH_LR_PHYS_ID_SHIFT; |
| 152 | } else { |
| 153 | if (irq->config == VGIC_CONFIG_LEVEL) |
| 154 | val |= ICH_LR_EOI; |
| 155 | } |
| 156 | |
| 157 | /* |
| 158 | * We currently only support Group1 interrupts, which is a |
| 159 | * known defect. This needs to be addressed at some point. |
| 160 | */ |
| 161 | if (model == KVM_DEV_TYPE_ARM_VGIC_V3) |
| 162 | val |= ICH_LR_GROUP; |
| 163 | |
| 164 | val |= (u64)irq->priority << ICH_LR_PRIORITY_SHIFT; |
| 165 | |
| 166 | vcpu->arch.vgic_cpu.vgic_v3.vgic_lr[lr] = val; |
| 167 | } |
| 168 | |
| 169 | void vgic_v3_clear_lr(struct kvm_vcpu *vcpu, int lr) |
| 170 | { |
| 171 | vcpu->arch.vgic_cpu.vgic_v3.vgic_lr[lr] = 0; |
| 172 | } |
Andre Przywara | e4823a7 | 2015-12-03 11:47:37 +0000 | [diff] [blame] | 173 | |
| 174 | void vgic_v3_set_vmcr(struct kvm_vcpu *vcpu, struct vgic_vmcr *vmcrp) |
| 175 | { |
| 176 | u32 vmcr; |
| 177 | |
| 178 | vmcr = (vmcrp->ctlr << ICH_VMCR_CTLR_SHIFT) & ICH_VMCR_CTLR_MASK; |
| 179 | vmcr |= (vmcrp->abpr << ICH_VMCR_BPR1_SHIFT) & ICH_VMCR_BPR1_MASK; |
| 180 | vmcr |= (vmcrp->bpr << ICH_VMCR_BPR0_SHIFT) & ICH_VMCR_BPR0_MASK; |
| 181 | vmcr |= (vmcrp->pmr << ICH_VMCR_PMR_SHIFT) & ICH_VMCR_PMR_MASK; |
| 182 | |
| 183 | vcpu->arch.vgic_cpu.vgic_v3.vgic_vmcr = vmcr; |
| 184 | } |
| 185 | |
| 186 | void vgic_v3_get_vmcr(struct kvm_vcpu *vcpu, struct vgic_vmcr *vmcrp) |
| 187 | { |
| 188 | u32 vmcr = vcpu->arch.vgic_cpu.vgic_v3.vgic_vmcr; |
| 189 | |
| 190 | vmcrp->ctlr = (vmcr & ICH_VMCR_CTLR_MASK) >> ICH_VMCR_CTLR_SHIFT; |
| 191 | vmcrp->abpr = (vmcr & ICH_VMCR_BPR1_MASK) >> ICH_VMCR_BPR1_SHIFT; |
| 192 | vmcrp->bpr = (vmcr & ICH_VMCR_BPR0_MASK) >> ICH_VMCR_BPR0_SHIFT; |
| 193 | vmcrp->pmr = (vmcr & ICH_VMCR_PMR_MASK) >> ICH_VMCR_PMR_SHIFT; |
| 194 | } |
Eric Auger | 9097773 | 2015-12-01 15:02:35 +0100 | [diff] [blame] | 195 | |
Andre Przywara | 0aa1de5 | 2016-07-15 12:43:29 +0100 | [diff] [blame] | 196 | #define INITIAL_PENDBASER_VALUE \ |
| 197 | (GIC_BASER_CACHEABILITY(GICR_PENDBASER, INNER, RaWb) | \ |
| 198 | GIC_BASER_CACHEABILITY(GICR_PENDBASER, OUTER, SameAsInner) | \ |
| 199 | GIC_BASER_SHAREABILITY(GICR_PENDBASER, InnerShareable)) |
| 200 | |
Eric Auger | ad275b8b | 2015-12-21 18:09:38 +0100 | [diff] [blame] | 201 | void vgic_v3_enable(struct kvm_vcpu *vcpu) |
| 202 | { |
Eric Auger | f7b6985 | 2015-12-02 10:30:13 +0100 | [diff] [blame] | 203 | struct vgic_v3_cpu_if *vgic_v3 = &vcpu->arch.vgic_cpu.vgic_v3; |
| 204 | |
| 205 | /* |
| 206 | * By forcing VMCR to zero, the GIC will restore the binary |
| 207 | * points to their reset values. Anything else resets to zero |
| 208 | * anyway. |
| 209 | */ |
| 210 | vgic_v3->vgic_vmcr = 0; |
| 211 | vgic_v3->vgic_elrsr = ~0; |
| 212 | |
| 213 | /* |
| 214 | * If we are emulating a GICv3, we do it in an non-GICv2-compatible |
| 215 | * way, so we force SRE to 1 to demonstrate this to the guest. |
| 216 | * This goes with the spec allowing the value to be RAO/WI. |
| 217 | */ |
Andre Przywara | 0aa1de5 | 2016-07-15 12:43:29 +0100 | [diff] [blame] | 218 | if (vcpu->kvm->arch.vgic.vgic_model == KVM_DEV_TYPE_ARM_VGIC_V3) { |
Eric Auger | f7b6985 | 2015-12-02 10:30:13 +0100 | [diff] [blame] | 219 | vgic_v3->vgic_sre = ICC_SRE_EL1_SRE; |
Andre Przywara | 0aa1de5 | 2016-07-15 12:43:29 +0100 | [diff] [blame] | 220 | vcpu->arch.vgic_cpu.pendbaser = INITIAL_PENDBASER_VALUE; |
| 221 | } else { |
Eric Auger | f7b6985 | 2015-12-02 10:30:13 +0100 | [diff] [blame] | 222 | vgic_v3->vgic_sre = 0; |
Andre Przywara | 0aa1de5 | 2016-07-15 12:43:29 +0100 | [diff] [blame] | 223 | } |
Eric Auger | f7b6985 | 2015-12-02 10:30:13 +0100 | [diff] [blame] | 224 | |
| 225 | /* Get the show on the road... */ |
| 226 | vgic_v3->vgic_hcr = ICH_HCR_EN; |
Eric Auger | ad275b8b | 2015-12-21 18:09:38 +0100 | [diff] [blame] | 227 | } |
| 228 | |
Eric Auger | b0442ee | 2015-12-21 15:04:42 +0100 | [diff] [blame] | 229 | /* check for overlapping regions and for regions crossing the end of memory */ |
| 230 | static bool vgic_v3_check_base(struct kvm *kvm) |
| 231 | { |
| 232 | struct vgic_dist *d = &kvm->arch.vgic; |
| 233 | gpa_t redist_size = KVM_VGIC_V3_REDIST_SIZE; |
| 234 | |
| 235 | redist_size *= atomic_read(&kvm->online_vcpus); |
| 236 | |
| 237 | if (d->vgic_dist_base + KVM_VGIC_V3_DIST_SIZE < d->vgic_dist_base) |
| 238 | return false; |
| 239 | if (d->vgic_redist_base + redist_size < d->vgic_redist_base) |
| 240 | return false; |
| 241 | |
| 242 | if (d->vgic_dist_base + KVM_VGIC_V3_DIST_SIZE <= d->vgic_redist_base) |
| 243 | return true; |
| 244 | if (d->vgic_redist_base + redist_size <= d->vgic_dist_base) |
| 245 | return true; |
| 246 | |
| 247 | return false; |
| 248 | } |
| 249 | |
| 250 | int vgic_v3_map_resources(struct kvm *kvm) |
| 251 | { |
| 252 | int ret = 0; |
| 253 | struct vgic_dist *dist = &kvm->arch.vgic; |
| 254 | |
| 255 | if (vgic_ready(kvm)) |
| 256 | goto out; |
| 257 | |
| 258 | if (IS_VGIC_ADDR_UNDEF(dist->vgic_dist_base) || |
| 259 | IS_VGIC_ADDR_UNDEF(dist->vgic_redist_base)) { |
| 260 | kvm_err("Need to set vgic distributor addresses first\n"); |
| 261 | ret = -ENXIO; |
| 262 | goto out; |
| 263 | } |
| 264 | |
| 265 | if (!vgic_v3_check_base(kvm)) { |
| 266 | kvm_err("VGIC redist and dist frames overlap\n"); |
| 267 | ret = -EINVAL; |
| 268 | goto out; |
| 269 | } |
| 270 | |
| 271 | /* |
| 272 | * For a VGICv3 we require the userland to explicitly initialize |
| 273 | * the VGIC before we need to use it. |
| 274 | */ |
| 275 | if (!vgic_initialized(kvm)) { |
| 276 | ret = -EBUSY; |
| 277 | goto out; |
| 278 | } |
| 279 | |
| 280 | ret = vgic_register_dist_iodev(kvm, dist->vgic_dist_base, VGIC_V3); |
| 281 | if (ret) { |
| 282 | kvm_err("Unable to register VGICv3 dist MMIO regions\n"); |
| 283 | goto out; |
| 284 | } |
| 285 | |
| 286 | ret = vgic_register_redist_iodevs(kvm, dist->vgic_redist_base); |
| 287 | if (ret) { |
| 288 | kvm_err("Unable to register VGICv3 redist MMIO regions\n"); |
| 289 | goto out; |
| 290 | } |
| 291 | |
| 292 | dist->ready = true; |
| 293 | |
| 294 | out: |
| 295 | if (ret) |
| 296 | kvm_vgic_destroy(kvm); |
| 297 | return ret; |
| 298 | } |
| 299 | |
Eric Auger | 9097773 | 2015-12-01 15:02:35 +0100 | [diff] [blame] | 300 | /** |
| 301 | * vgic_v3_probe - probe for a GICv3 compatible interrupt controller in DT |
| 302 | * @node: pointer to the DT node |
| 303 | * |
| 304 | * Returns 0 if a GICv3 has been found, returns an error code otherwise |
| 305 | */ |
| 306 | int vgic_v3_probe(const struct gic_kvm_info *info) |
| 307 | { |
| 308 | u32 ich_vtr_el2 = kvm_call_hyp(__vgic_v3_get_ich_vtr_el2); |
Andre Przywara | 42c8870 | 2016-07-15 12:43:23 +0100 | [diff] [blame] | 309 | int ret; |
Eric Auger | 9097773 | 2015-12-01 15:02:35 +0100 | [diff] [blame] | 310 | |
| 311 | /* |
| 312 | * The ListRegs field is 5 bits, but there is a architectural |
| 313 | * maximum of 16 list registers. Just ignore bit 4... |
| 314 | */ |
| 315 | kvm_vgic_global_state.nr_lr = (ich_vtr_el2 & 0xf) + 1; |
| 316 | kvm_vgic_global_state.can_emulate_gicv2 = false; |
| 317 | |
| 318 | if (!info->vcpu.start) { |
| 319 | kvm_info("GICv3: no GICV resource entry\n"); |
| 320 | kvm_vgic_global_state.vcpu_base = 0; |
| 321 | } else if (!PAGE_ALIGNED(info->vcpu.start)) { |
| 322 | pr_warn("GICV physical address 0x%llx not page aligned\n", |
| 323 | (unsigned long long)info->vcpu.start); |
| 324 | kvm_vgic_global_state.vcpu_base = 0; |
| 325 | } else if (!PAGE_ALIGNED(resource_size(&info->vcpu))) { |
| 326 | pr_warn("GICV size 0x%llx not a multiple of page size 0x%lx\n", |
| 327 | (unsigned long long)resource_size(&info->vcpu), |
| 328 | PAGE_SIZE); |
| 329 | kvm_vgic_global_state.vcpu_base = 0; |
| 330 | } else { |
| 331 | kvm_vgic_global_state.vcpu_base = info->vcpu.start; |
| 332 | kvm_vgic_global_state.can_emulate_gicv2 = true; |
Andre Przywara | 42c8870 | 2016-07-15 12:43:23 +0100 | [diff] [blame] | 333 | ret = kvm_register_vgic_device(KVM_DEV_TYPE_ARM_VGIC_V2); |
| 334 | if (ret) { |
| 335 | kvm_err("Cannot register GICv2 KVM device.\n"); |
| 336 | return ret; |
| 337 | } |
Eric Auger | 9097773 | 2015-12-01 15:02:35 +0100 | [diff] [blame] | 338 | kvm_info("vgic-v2@%llx\n", info->vcpu.start); |
| 339 | } |
Andre Przywara | 42c8870 | 2016-07-15 12:43:23 +0100 | [diff] [blame] | 340 | ret = kvm_register_vgic_device(KVM_DEV_TYPE_ARM_VGIC_V3); |
| 341 | if (ret) { |
| 342 | kvm_err("Cannot register GICv3 KVM device.\n"); |
| 343 | kvm_unregister_device_ops(KVM_DEV_TYPE_ARM_VGIC_V2); |
| 344 | return ret; |
| 345 | } |
| 346 | |
Eric Auger | 9097773 | 2015-12-01 15:02:35 +0100 | [diff] [blame] | 347 | if (kvm_vgic_global_state.vcpu_base == 0) |
| 348 | kvm_info("disabling GICv2 emulation\n"); |
Eric Auger | 9097773 | 2015-12-01 15:02:35 +0100 | [diff] [blame] | 349 | |
| 350 | kvm_vgic_global_state.vctrl_base = NULL; |
| 351 | kvm_vgic_global_state.type = VGIC_V3; |
| 352 | kvm_vgic_global_state.max_gic_vcpus = VGIC_V3_MAX_CPUS; |
| 353 | |
| 354 | return 0; |
| 355 | } |