x86, pmem: clarify that ARCH_HAS_PMEM_API implies PMEM mapped WB
Given that a write-back (WB) mapping plus non-temporal stores is
expected to be the most efficient way to access PMEM, update the
definition of ARCH_HAS_PMEM_API to imply arch support for
WB-mapped-PMEM. This is needed as a pre-requisite for adding PMEM to
the direct map and mapping it with struct page.
The above clarification for X86_64 means that memcpy_to_pmem() is
permitted to use the non-temporal arch_memcpy_to_pmem() rather than
needlessly fall back to default_memcpy_to_pmem() when the pcommit
instruction is not available. When arch_memcpy_to_pmem() is not
guaranteed to flush writes out of cache, i.e. on older X86_32
implementations where non-temporal stores may just dirty cache,
ARCH_HAS_PMEM_API is simply disabled.
The default fall back for persistent memory handling remains. Namely,
map it with the WT (write-through) cache-type and hope for the best.
arch_has_pmem_api() is updated to only indicate whether the arch
provides the proper helpers to meet the minimum "writes are visible
outside the cache hierarchy after memcpy_to_pmem() + wmb_pmem()". Code
that cares whether wmb_pmem() actually flushes writes to pmem must now
call arch_has_wmb_pmem() directly.
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Ingo Molnar <mingo@redhat.com>
Cc: "H. Peter Anvin" <hpa@zytor.com>
Reviewed-by: Ross Zwisler <ross.zwisler@linux.intel.com>
[hch: set ARCH_HAS_PMEM_API=n on x86_32]
Reviewed-by: Christoph Hellwig <hch@lst.de>
[toshi: x86_32 compile fixes]
Signed-off-by: Toshi Kani <toshi.kani@hp.com>
Signed-off-by: Dan Williams <dan.j.williams@intel.com>
diff --git a/include/linux/pmem.h b/include/linux/pmem.h
index a9d84bf..85f810b3 100644
--- a/include/linux/pmem.h
+++ b/include/linux/pmem.h
@@ -17,18 +17,25 @@
#include <linux/uio.h>
#ifdef CONFIG_ARCH_HAS_PMEM_API
+#define ARCH_MEMREMAP_PMEM MEMREMAP_WB
#include <asm/pmem.h>
#else
+#define ARCH_MEMREMAP_PMEM MEMREMAP_WT
+/*
+ * These are simply here to enable compilation, all call sites gate
+ * calling these symbols with arch_has_pmem_api() and redirect to the
+ * implementation in asm/pmem.h.
+ */
+static inline bool __arch_has_wmb_pmem(void)
+{
+ return false;
+}
+
static inline void arch_wmb_pmem(void)
{
BUG();
}
-static inline bool arch_has_wmb_pmem(void)
-{
- return false;
-}
-
static inline void arch_memcpy_to_pmem(void __pmem *dst, const void *src,
size_t n)
{
@@ -53,7 +60,6 @@
* implementations for arch_memcpy_to_pmem(), arch_wmb_pmem(),
* arch_copy_from_iter_pmem(), arch_clear_pmem() and arch_has_wmb_pmem().
*/
-
static inline void memcpy_from_pmem(void *dst, void __pmem const *src, size_t size)
{
memcpy(dst, (void __force const *) src, size);
@@ -64,8 +70,13 @@
devm_memunmap(dev, (void __force *) addr);
}
+static inline bool arch_has_pmem_api(void)
+{
+ return IS_ENABLED(CONFIG_ARCH_HAS_PMEM_API);
+}
+
/**
- * arch_has_pmem_api - true if wmb_pmem() ensures durability
+ * arch_has_wmb_pmem - true if wmb_pmem() ensures durability
*
* For a given cpu implementation within an architecture it is possible
* that wmb_pmem() resolves to a nop. In the case this returns
@@ -73,9 +84,9 @@
* fall back to a different data consistency model, or otherwise notify
* the user.
*/
-static inline bool arch_has_pmem_api(void)
+static inline bool arch_has_wmb_pmem(void)
{
- return IS_ENABLED(CONFIG_ARCH_HAS_PMEM_API) && arch_has_wmb_pmem();
+ return arch_has_pmem_api() && __arch_has_wmb_pmem();
}
/*
@@ -120,13 +131,8 @@
static inline void __pmem *memremap_pmem(struct device *dev,
resource_size_t offset, unsigned long size)
{
-#ifdef ARCH_MEMREMAP_PMEM
return (void __pmem *) devm_memremap(dev, offset, size,
ARCH_MEMREMAP_PMEM);
-#else
- return (void __pmem *) devm_memremap(dev, offset, size,
- MEMREMAP_WT);
-#endif
}
/**
@@ -158,8 +164,10 @@
*/
static inline void wmb_pmem(void)
{
- if (arch_has_pmem_api())
+ if (arch_has_wmb_pmem())
arch_wmb_pmem();
+ else
+ wmb();
}
/**