David Howells | ae3a197 | 2012-03-28 18:30:02 +0100 | [diff] [blame] | 1 | /* |
| 2 | * Copyright (C) 1999 Cort Dougan <cort@cs.nmt.edu> |
| 3 | */ |
| 4 | #ifndef _ASM_POWERPC_BARRIER_H |
| 5 | #define _ASM_POWERPC_BARRIER_H |
| 6 | |
| 7 | /* |
| 8 | * Memory barrier. |
| 9 | * The sync instruction guarantees that all memory accesses initiated |
| 10 | * by this processor have been performed (with respect to all other |
| 11 | * mechanisms that access memory). The eieio instruction is a barrier |
| 12 | * providing an ordering (separately) for (a) cacheable stores and (b) |
| 13 | * loads and stores to non-cacheable memory (e.g. I/O devices). |
| 14 | * |
| 15 | * mb() prevents loads and stores being reordered across this point. |
| 16 | * rmb() prevents loads being reordered across this point. |
| 17 | * wmb() prevents stores being reordered across this point. |
| 18 | * read_barrier_depends() prevents data-dependent loads being reordered |
| 19 | * across this point (nop on PPC). |
| 20 | * |
| 21 | * *mb() variants without smp_ prefix must order all types of memory |
| 22 | * operations with one another. sync is the only instruction sufficient |
| 23 | * to do this. |
| 24 | * |
| 25 | * For the smp_ barriers, ordering is for cacheable memory operations |
| 26 | * only. We have to use the sync instruction for smp_mb(), since lwsync |
| 27 | * doesn't order loads with respect to previous stores. Lwsync can be |
| 28 | * used for smp_rmb() and smp_wmb(). |
| 29 | * |
| 30 | * However, on CPUs that don't support lwsync, lwsync actually maps to a |
| 31 | * heavy-weight sync, so smp_wmb() can be a lighter-weight eieio. |
| 32 | */ |
| 33 | #define mb() __asm__ __volatile__ ("sync" : : : "memory") |
| 34 | #define rmb() __asm__ __volatile__ ("sync" : : : "memory") |
| 35 | #define wmb() __asm__ __volatile__ ("sync" : : : "memory") |
David Howells | ae3a197 | 2012-03-28 18:30:02 +0100 | [diff] [blame] | 36 | |
Nicholas Piggin | 59f404e | 2018-03-22 20:41:46 +1000 | [diff] [blame] | 37 | /* The sub-arch has lwsync */ |
| 38 | #if defined(__powerpc64__) || defined(CONFIG_PPC_E500MC) |
David Howells | ae3a197 | 2012-03-28 18:30:02 +0100 | [diff] [blame] | 39 | # define SMPWMB LWSYNC |
| 40 | #else |
| 41 | # define SMPWMB eieio |
| 42 | #endif |
| 43 | |
Peter Zijlstra | 47933ad | 2013-11-06 14:57:36 +0100 | [diff] [blame] | 44 | #define __lwsync() __asm__ __volatile__ (stringify_in_c(LWSYNC) : : :"memory") |
Alexander Duyck | 1077fa3 | 2014-12-11 15:02:06 -0800 | [diff] [blame] | 45 | #define dma_rmb() __lwsync() |
| 46 | #define dma_wmb() __asm__ __volatile__ (stringify_in_c(SMPWMB) : : :"memory") |
| 47 | |
Michael S. Tsirkin | 003472a | 2015-12-27 15:04:42 +0200 | [diff] [blame] | 48 | #define __smp_lwsync() __lwsync() |
Peter Zijlstra | 47933ad | 2013-11-06 14:57:36 +0100 | [diff] [blame] | 49 | |
Michael S. Tsirkin | 003472a | 2015-12-27 15:04:42 +0200 | [diff] [blame] | 50 | #define __smp_mb() mb() |
| 51 | #define __smp_rmb() __lwsync() |
| 52 | #define __smp_wmb() __asm__ __volatile__ (stringify_in_c(SMPWMB) : : :"memory") |
David Howells | ae3a197 | 2012-03-28 18:30:02 +0100 | [diff] [blame] | 53 | |
| 54 | /* |
| 55 | * This is a barrier which prevents following instructions from being |
| 56 | * started until the value of the argument x is known. For example, if |
| 57 | * x is a variable loaded from memory, this prevents following |
| 58 | * instructions from being executed until the load has been performed. |
| 59 | */ |
| 60 | #define data_barrier(x) \ |
| 61 | asm volatile("twi 0,%0,0; isync" : : "r" (x) : "memory"); |
| 62 | |
Michael S. Tsirkin | 003472a | 2015-12-27 15:04:42 +0200 | [diff] [blame] | 63 | #define __smp_store_release(p, v) \ |
Peter Zijlstra | 47933ad | 2013-11-06 14:57:36 +0100 | [diff] [blame] | 64 | do { \ |
| 65 | compiletime_assert_atomic_type(*p); \ |
Michael S. Tsirkin | 003472a | 2015-12-27 15:04:42 +0200 | [diff] [blame] | 66 | __smp_lwsync(); \ |
Andrey Konovalov | 76695af | 2015-08-02 17:11:04 +0200 | [diff] [blame] | 67 | WRITE_ONCE(*p, v); \ |
Peter Zijlstra | 47933ad | 2013-11-06 14:57:36 +0100 | [diff] [blame] | 68 | } while (0) |
| 69 | |
Michael S. Tsirkin | 003472a | 2015-12-27 15:04:42 +0200 | [diff] [blame] | 70 | #define __smp_load_acquire(p) \ |
Peter Zijlstra | 47933ad | 2013-11-06 14:57:36 +0100 | [diff] [blame] | 71 | ({ \ |
Andrey Konovalov | 76695af | 2015-08-02 17:11:04 +0200 | [diff] [blame] | 72 | typeof(*p) ___p1 = READ_ONCE(*p); \ |
Peter Zijlstra | 47933ad | 2013-11-06 14:57:36 +0100 | [diff] [blame] | 73 | compiletime_assert_atomic_type(*p); \ |
Michael S. Tsirkin | 003472a | 2015-12-27 15:04:42 +0200 | [diff] [blame] | 74 | __smp_lwsync(); \ |
Peter Zijlstra | 47933ad | 2013-11-06 14:57:36 +0100 | [diff] [blame] | 75 | ___p1; \ |
| 76 | }) |
| 77 | |
Paul E. McKenney | a76ff68 | 2015-04-01 08:19:59 -0700 | [diff] [blame] | 78 | #define smp_mb__before_spinlock() smp_mb() |
Peter Zijlstra | c645073 | 2014-03-13 19:00:35 +0100 | [diff] [blame] | 79 | |
Diana Craciun | a299c07 | 2019-04-11 21:46:11 +1000 | [diff] [blame] | 80 | #ifdef CONFIG_PPC_BOOK3S_64 |
| 81 | #define NOSPEC_BARRIER_SLOT nop |
| 82 | #elif defined(CONFIG_PPC_FSL_BOOK3E) |
| 83 | #define NOSPEC_BARRIER_SLOT nop; nop |
| 84 | #endif |
| 85 | |
Michael Ellerman | 2f8703f | 2019-04-11 21:46:08 +1000 | [diff] [blame] | 86 | #ifdef CONFIG_PPC_BARRIER_NOSPEC |
Michal Suchanek | 4314e77 | 2019-04-11 21:45:57 +1000 | [diff] [blame] | 87 | /* |
| 88 | * Prevent execution of subsequent instructions until preceding branches have |
| 89 | * been fully resolved and are no longer executing speculatively. |
| 90 | */ |
Diana Craciun | a299c07 | 2019-04-11 21:46:11 +1000 | [diff] [blame] | 91 | #define barrier_nospec_asm NOSPEC_BARRIER_FIXUP_SECTION; NOSPEC_BARRIER_SLOT |
Michal Suchanek | 4314e77 | 2019-04-11 21:45:57 +1000 | [diff] [blame] | 92 | |
| 93 | // This also acts as a compiler barrier due to the memory clobber. |
| 94 | #define barrier_nospec() asm (stringify_in_c(barrier_nospec_asm) ::: "memory") |
| 95 | |
Michael Ellerman | 2f8703f | 2019-04-11 21:46:08 +1000 | [diff] [blame] | 96 | #else /* !CONFIG_PPC_BARRIER_NOSPEC */ |
Michal Suchanek | 4314e77 | 2019-04-11 21:45:57 +1000 | [diff] [blame] | 97 | #define barrier_nospec_asm |
| 98 | #define barrier_nospec() |
Michael Ellerman | 2f8703f | 2019-04-11 21:46:08 +1000 | [diff] [blame] | 99 | #endif /* CONFIG_PPC_BARRIER_NOSPEC */ |
Michal Suchanek | 4314e77 | 2019-04-11 21:45:57 +1000 | [diff] [blame] | 100 | |
Michael S. Tsirkin | fbd7ec0 | 2015-12-21 09:22:18 +0200 | [diff] [blame] | 101 | #include <asm-generic/barrier.h> |
| 102 | |
David Howells | ae3a197 | 2012-03-28 18:30:02 +0100 | [diff] [blame] | 103 | #endif /* _ASM_POWERPC_BARRIER_H */ |