iommu/io-pgtable-arm: Centralise sync points
With all current users now opted in to DMA API operations, make the
iommu_dev pointer mandatory, rendering the flush_pgtable callback
redundant for cache maintenance. However, since the DMA calls could be
nops in the case of a coherent IOMMU, we still need to ensure the page
table updates are fully synchronised against a subsequent page table
walk. In the unmap path, the TLB sync will usually need to do this
anyway, so just cement that requirement; in the map path which may
consist solely of cacheable memory writes (in the coherent case),
insert an appropriate barrier at the end of the operation, and obviate
the need to call flush_pgtable on every individual update for
synchronisation.
Signed-off-by: Robin Murphy <robin.murphy@arm.com>
[will: slight clarification to tlb_sync comment]
Signed-off-by: Will Deacon <will.deacon@arm.com>
diff --git a/drivers/iommu/io-pgtable-arm.c b/drivers/iommu/io-pgtable-arm.c
index 28cca8a..0617687 100644
--- a/drivers/iommu/io-pgtable-arm.c
+++ b/drivers/iommu/io-pgtable-arm.c
@@ -26,6 +26,8 @@
#include <linux/slab.h>
#include <linux/types.h>
+#include <asm/barrier.h>
+
#include "io-pgtable.h"
#define ARM_LPAE_MAX_ADDR_BITS 48
@@ -215,7 +217,7 @@
if (!pages)
return NULL;
- if (dev) {
+ if (!selftest_running) {
dma = dma_map_single(dev, pages, size, DMA_TO_DEVICE);
if (dma_mapping_error(dev, dma))
goto out_free;
@@ -243,24 +245,22 @@
{
struct device *dev = cfg->iommu_dev;
- if (dev)
+ if (!selftest_running)
dma_unmap_single(dev, __arm_lpae_dma_addr(dev, pages),
size, DMA_TO_DEVICE);
free_pages_exact(pages, size);
}
static void __arm_lpae_set_pte(arm_lpae_iopte *ptep, arm_lpae_iopte pte,
- struct io_pgtable_cfg *cfg, void *cookie)
+ struct io_pgtable_cfg *cfg)
{
struct device *dev = cfg->iommu_dev;
*ptep = pte;
- if (dev)
+ if (!selftest_running)
dma_sync_single_for_device(dev, __arm_lpae_dma_addr(dev, ptep),
sizeof(pte), DMA_TO_DEVICE);
- else if (cfg->tlb->flush_pgtable)
- cfg->tlb->flush_pgtable(ptep, sizeof(pte), cookie);
}
static int arm_lpae_init_pte(struct arm_lpae_io_pgtable *data,
@@ -288,7 +288,7 @@
pte |= ARM_LPAE_PTE_AF | ARM_LPAE_PTE_SH_IS;
pte |= pfn_to_iopte(paddr >> data->pg_shift, data);
- __arm_lpae_set_pte(ptep, pte, cfg, data->iop.cookie);
+ __arm_lpae_set_pte(ptep, pte, cfg);
return 0;
}
@@ -297,7 +297,6 @@
int lvl, arm_lpae_iopte *ptep)
{
arm_lpae_iopte *cptep, pte;
- void *cookie = data->iop.cookie;
size_t block_size = ARM_LPAE_BLOCK_SIZE(lvl, data);
struct io_pgtable_cfg *cfg = &data->iop.cfg;
@@ -323,7 +322,7 @@
pte = __pa(cptep) | ARM_LPAE_PTE_TYPE_TABLE;
if (cfg->quirks & IO_PGTABLE_QUIRK_ARM_NS)
pte |= ARM_LPAE_PTE_NSTABLE;
- __arm_lpae_set_pte(ptep, pte, cfg, cookie);
+ __arm_lpae_set_pte(ptep, pte, cfg);
} else {
cptep = iopte_deref(pte, data);
}
@@ -370,7 +369,7 @@
{
struct arm_lpae_io_pgtable *data = io_pgtable_ops_to_data(ops);
arm_lpae_iopte *ptep = data->pgd;
- int lvl = ARM_LPAE_START_LVL(data);
+ int ret, lvl = ARM_LPAE_START_LVL(data);
arm_lpae_iopte prot;
/* If no access, then nothing to do */
@@ -378,7 +377,14 @@
return 0;
prot = arm_lpae_prot_to_pte(data, iommu_prot);
- return __arm_lpae_map(data, iova, paddr, size, prot, lvl, ptep);
+ ret = __arm_lpae_map(data, iova, paddr, size, prot, lvl, ptep);
+ /*
+ * Synchronise all PTE updates for the new mapping before there's
+ * a chance for anything to kick off a table walk for the new iova.
+ */
+ wmb();
+
+ return ret;
}
static void __arm_lpae_free_pgtable(struct arm_lpae_io_pgtable *data, int lvl,
@@ -428,7 +434,6 @@
phys_addr_t blk_paddr;
arm_lpae_iopte table = 0;
struct io_pgtable_cfg *cfg = &data->iop.cfg;
- void *cookie = data->iop.cookie;
blk_start = iova & ~(blk_size - 1);
blk_end = blk_start + blk_size;
@@ -454,9 +459,9 @@
}
}
- __arm_lpae_set_pte(ptep, table, cfg, cookie);
+ __arm_lpae_set_pte(ptep, table, cfg);
iova &= ~(blk_size - 1);
- cfg->tlb->tlb_add_flush(iova, blk_size, true, cookie);
+ cfg->tlb->tlb_add_flush(iova, blk_size, true, data->iop.cookie);
return size;
}
@@ -478,7 +483,7 @@
/* If the size matches this level, we're in the right place */
if (size == blk_size) {
- __arm_lpae_set_pte(ptep, 0, &data->iop.cfg, cookie);
+ __arm_lpae_set_pte(ptep, 0, &data->iop.cfg);
if (!iopte_leaf(pte, lvl)) {
/* Also flush any partial walks */
@@ -703,8 +708,8 @@
if (!data->pgd)
goto out_free_data;
- if (cfg->tlb->flush_pgtable)
- cfg->tlb->flush_pgtable(data->pgd, data->pgd_size, cookie);
+ /* Ensure the empty pgd is visible before any actual TTBR write */
+ wmb();
/* TTBRs */
cfg->arm_lpae_s1_cfg.ttbr[0] = virt_to_phys(data->pgd);
@@ -792,8 +797,8 @@
if (!data->pgd)
goto out_free_data;
- if (cfg->tlb->flush_pgtable)
- cfg->tlb->flush_pgtable(data->pgd, data->pgd_size, cookie);
+ /* Ensure the empty pgd is visible before any actual TTBR write */
+ wmb();
/* VTTBR */
cfg->arm_lpae_s2_cfg.vttbr = virt_to_phys(data->pgd);