iommu/io-pgtable-fast: Prove correctness of TLB maintenance
A common software error when it comes to page table code is missing TLB
maintenance. Add some checks to the io-pgtable-fast code to detect when
an address that might be stale in the TLB is being re-used. This can be
accomplished by writing a "stale TLB" flag value to the reserved bits of
the PTE during unmap and then removing the flag value when the TLBs are
invalidated (by sweeping the entire page table). That way, whenever we
map we can know that there might be a stale TLB in the location being
mapped into if it contains the "stale TLB" flag value.
CRs-Fixed: 997751
Change-Id: Icf9c1e41977cb71e8b137190adb3b4a201c339da
Signed-off-by: Mitchel Humpherys <mitchelh@codeaurora.org>
Signed-off-by: Patrick Daly <pdaly@codeaurora.org>
diff --git a/drivers/iommu/dma-mapping-fast.c b/drivers/iommu/dma-mapping-fast.c
index 2d36ee3..8b2be30 100644
--- a/drivers/iommu/dma-mapping-fast.c
+++ b/drivers/iommu/dma-mapping-fast.c
@@ -80,6 +80,7 @@
}
static dma_addr_t __fast_smmu_alloc_iova(struct dma_fast_smmu_mapping *mapping,
+ unsigned long attrs,
size_t size)
{
unsigned long bit, prev_search_start, nbits = size >> FAST_PAGE_SHIFT;
@@ -114,8 +115,11 @@
__bit_covered_stale(mapping->upcoming_stale_bit,
prev_search_start,
bit + nbits - 1)) {
+ bool skip_sync = (attrs & DMA_ATTR_SKIP_CPU_SYNC);
+
iommu_tlbiall(mapping->domain);
mapping->have_stale_tlbs = false;
+ av8l_fast_clear_stale_ptes(mapping->pgtbl_pmds, skip_sync);
}
return (bit << FAST_PAGE_SHIFT) + mapping->base;
@@ -287,7 +291,7 @@
spin_lock_irqsave(&mapping->lock, flags);
- iova = __fast_smmu_alloc_iova(mapping, len);
+ iova = __fast_smmu_alloc_iova(mapping, attrs, len);
if (unlikely(iova == DMA_ERROR_CODE))
goto fail;
@@ -429,7 +433,7 @@
}
spin_lock_irqsave(&mapping->lock, flags);
- dma_addr = __fast_smmu_alloc_iova(mapping, size);
+ dma_addr = __fast_smmu_alloc_iova(mapping, attrs, size);
if (dma_addr == DMA_ERROR_CODE) {
dev_err(dev, "no iova\n");
spin_unlock_irqrestore(&mapping->lock, flags);
@@ -521,6 +525,39 @@
return dma_addr == DMA_ERROR_CODE;
}
+static void __fast_smmu_mapped_over_stale(struct dma_fast_smmu_mapping *fast,
+ void *data)
+{
+ av8l_fast_iopte *ptep = data;
+ dma_addr_t iova;
+ unsigned long bitmap_idx;
+
+ bitmap_idx = (unsigned long)(ptep - fast->pgtbl_pmds);
+ iova = bitmap_idx << FAST_PAGE_SHIFT;
+ dev_err(fast->dev, "Mapped over stale tlb at %pa\n", &iova);
+ dev_err(fast->dev, "bitmap (failure at idx %lu):\n", bitmap_idx);
+ dev_err(fast->dev, "ptep: %p pmds: %p diff: %lu\n", ptep,
+ fast->pgtbl_pmds, ptep - fast->pgtbl_pmds);
+ print_hex_dump(KERN_ERR, "bmap: ", DUMP_PREFIX_ADDRESS,
+ 32, 8, fast->bitmap, fast->bitmap_size, false);
+}
+
+static int fast_smmu_notify(struct notifier_block *self,
+ unsigned long action, void *data)
+{
+ struct dma_fast_smmu_mapping *fast = container_of(
+ self, struct dma_fast_smmu_mapping, notifier);
+
+ switch (action) {
+ case MAPPED_OVER_STALE_TLB:
+ __fast_smmu_mapped_over_stale(fast, data);
+ return NOTIFY_OK;
+ default:
+ WARN(1, "Unhandled notifier action");
+ return NOTIFY_DONE;
+ }
+}
+
static const struct dma_map_ops fast_smmu_dma_ops = {
.alloc = fast_smmu_alloc,
.free = fast_smmu_free,
@@ -618,6 +655,9 @@
}
mapping->fast->pgtbl_pmds = info.pmds;
+ mapping->fast->notifier.notifier_call = fast_smmu_notify;
+ av8l_register_notify(&mapping->fast->notifier);
+
dev->archdata.mapping = mapping;
set_dma_ops(dev, &fast_smmu_dma_ops);