Catalin Marinas | 10b663a | 2012-03-05 11:49:34 +0000 | [diff] [blame] | 1 | /* |
| 2 | * Based on arch/arm/include/asm/barrier.h |
| 3 | * |
| 4 | * Copyright (C) 2012 ARM Ltd. |
| 5 | * |
| 6 | * This program is free software; you can redistribute it and/or modify |
| 7 | * it under the terms of the GNU General Public License version 2 as |
| 8 | * published by the Free Software Foundation. |
| 9 | * |
| 10 | * This program is distributed in the hope that it will be useful, |
| 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of |
| 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
| 13 | * GNU General Public License for more details. |
| 14 | * |
| 15 | * You should have received a copy of the GNU General Public License |
| 16 | * along with this program. If not, see <http://www.gnu.org/licenses/>. |
| 17 | */ |
| 18 | #ifndef __ASM_BARRIER_H |
| 19 | #define __ASM_BARRIER_H |
| 20 | |
| 21 | #ifndef __ASSEMBLY__ |
| 22 | |
Will Deacon | f99a250 | 2016-09-06 16:40:23 +0100 | [diff] [blame] | 23 | #define __nops(n) ".rept " #n "\nnop\n.endr\n" |
| 24 | #define nops(n) asm volatile(__nops(n)) |
| 25 | |
Catalin Marinas | 10b663a | 2012-03-05 11:49:34 +0000 | [diff] [blame] | 26 | #define sev() asm volatile("sev" : : : "memory") |
| 27 | #define wfe() asm volatile("wfe" : : : "memory") |
| 28 | #define wfi() asm volatile("wfi" : : : "memory") |
| 29 | |
| 30 | #define isb() asm volatile("isb" : : : "memory") |
Will Deacon | 493e687 | 2014-05-02 16:24:11 +0100 | [diff] [blame] | 31 | #define dmb(opt) asm volatile("dmb " #opt : : : "memory") |
| 32 | #define dsb(opt) asm volatile("dsb " #opt : : : "memory") |
Catalin Marinas | 10b663a | 2012-03-05 11:49:34 +0000 | [diff] [blame] | 33 | |
Will Deacon | 98f7685 | 2014-05-02 16:24:10 +0100 | [diff] [blame] | 34 | #define mb() dsb(sy) |
Will Deacon | 493e687 | 2014-05-02 16:24:11 +0100 | [diff] [blame] | 35 | #define rmb() dsb(ld) |
| 36 | #define wmb() dsb(st) |
Catalin Marinas | 10b663a | 2012-03-05 11:49:34 +0000 | [diff] [blame] | 37 | |
Alexander Duyck | 1077fa3 | 2014-12-11 15:02:06 -0800 | [diff] [blame] | 38 | #define dma_rmb() dmb(oshld) |
| 39 | #define dma_wmb() dmb(oshst) |
| 40 | |
Michael S. Tsirkin | fd072df | 2015-12-27 15:04:42 +0200 | [diff] [blame] | 41 | #define __smp_mb() dmb(ish) |
| 42 | #define __smp_rmb() dmb(ishld) |
| 43 | #define __smp_wmb() dmb(ishst) |
Peter Zijlstra | 47933ad | 2013-11-06 14:57:36 +0100 | [diff] [blame] | 44 | |
Michael S. Tsirkin | fd072df | 2015-12-27 15:04:42 +0200 | [diff] [blame] | 45 | #define __smp_store_release(p, v) \ |
Peter Zijlstra | 47933ad | 2013-11-06 14:57:36 +0100 | [diff] [blame] | 46 | do { \ |
| 47 | compiletime_assert_atomic_type(*p); \ |
| 48 | switch (sizeof(*p)) { \ |
Andre Przywara | 878a84d | 2015-04-20 11:14:19 +0100 | [diff] [blame] | 49 | case 1: \ |
| 50 | asm volatile ("stlrb %w1, %0" \ |
| 51 | : "=Q" (*p) : "r" (v) : "memory"); \ |
| 52 | break; \ |
| 53 | case 2: \ |
| 54 | asm volatile ("stlrh %w1, %0" \ |
| 55 | : "=Q" (*p) : "r" (v) : "memory"); \ |
| 56 | break; \ |
Peter Zijlstra | 47933ad | 2013-11-06 14:57:36 +0100 | [diff] [blame] | 57 | case 4: \ |
| 58 | asm volatile ("stlr %w1, %0" \ |
| 59 | : "=Q" (*p) : "r" (v) : "memory"); \ |
| 60 | break; \ |
| 61 | case 8: \ |
| 62 | asm volatile ("stlr %1, %0" \ |
| 63 | : "=Q" (*p) : "r" (v) : "memory"); \ |
| 64 | break; \ |
| 65 | } \ |
| 66 | } while (0) |
| 67 | |
Michael S. Tsirkin | fd072df | 2015-12-27 15:04:42 +0200 | [diff] [blame] | 68 | #define __smp_load_acquire(p) \ |
Peter Zijlstra | 47933ad | 2013-11-06 14:57:36 +0100 | [diff] [blame] | 69 | ({ \ |
Will Deacon | c139aa6 | 2015-11-18 10:13:08 +0000 | [diff] [blame] | 70 | union { typeof(*p) __val; char __c[1]; } __u; \ |
Peter Zijlstra | 47933ad | 2013-11-06 14:57:36 +0100 | [diff] [blame] | 71 | compiletime_assert_atomic_type(*p); \ |
| 72 | switch (sizeof(*p)) { \ |
Andre Przywara | 878a84d | 2015-04-20 11:14:19 +0100 | [diff] [blame] | 73 | case 1: \ |
| 74 | asm volatile ("ldarb %w0, %1" \ |
Will Deacon | c139aa6 | 2015-11-18 10:13:08 +0000 | [diff] [blame] | 75 | : "=r" (*(__u8 *)__u.__c) \ |
| 76 | : "Q" (*p) : "memory"); \ |
Andre Przywara | 878a84d | 2015-04-20 11:14:19 +0100 | [diff] [blame] | 77 | break; \ |
| 78 | case 2: \ |
| 79 | asm volatile ("ldarh %w0, %1" \ |
Will Deacon | c139aa6 | 2015-11-18 10:13:08 +0000 | [diff] [blame] | 80 | : "=r" (*(__u16 *)__u.__c) \ |
| 81 | : "Q" (*p) : "memory"); \ |
Andre Przywara | 878a84d | 2015-04-20 11:14:19 +0100 | [diff] [blame] | 82 | break; \ |
Peter Zijlstra | 47933ad | 2013-11-06 14:57:36 +0100 | [diff] [blame] | 83 | case 4: \ |
| 84 | asm volatile ("ldar %w0, %1" \ |
Will Deacon | c139aa6 | 2015-11-18 10:13:08 +0000 | [diff] [blame] | 85 | : "=r" (*(__u32 *)__u.__c) \ |
| 86 | : "Q" (*p) : "memory"); \ |
Peter Zijlstra | 47933ad | 2013-11-06 14:57:36 +0100 | [diff] [blame] | 87 | break; \ |
| 88 | case 8: \ |
| 89 | asm volatile ("ldar %0, %1" \ |
Will Deacon | c139aa6 | 2015-11-18 10:13:08 +0000 | [diff] [blame] | 90 | : "=r" (*(__u64 *)__u.__c) \ |
| 91 | : "Q" (*p) : "memory"); \ |
Peter Zijlstra | 47933ad | 2013-11-06 14:57:36 +0100 | [diff] [blame] | 92 | break; \ |
| 93 | } \ |
Will Deacon | c139aa6 | 2015-11-18 10:13:08 +0000 | [diff] [blame] | 94 | __u.__val; \ |
Peter Zijlstra | 47933ad | 2013-11-06 14:57:36 +0100 | [diff] [blame] | 95 | }) |
| 96 | |
Will Deacon | 03e3c2b | 2016-06-27 18:43:54 +0100 | [diff] [blame] | 97 | #define smp_cond_load_acquire(ptr, cond_expr) \ |
| 98 | ({ \ |
| 99 | typeof(ptr) __PTR = (ptr); \ |
| 100 | typeof(*ptr) VAL; \ |
| 101 | for (;;) { \ |
| 102 | VAL = smp_load_acquire(__PTR); \ |
| 103 | if (cond_expr) \ |
| 104 | break; \ |
| 105 | __cmpwait_relaxed(__PTR, VAL); \ |
| 106 | } \ |
| 107 | VAL; \ |
| 108 | }) |
| 109 | |
Michael S. Tsirkin | 90ff6a1 | 2015-12-21 09:22:18 +0200 | [diff] [blame] | 110 | #include <asm-generic/barrier.h> |
Peter Zijlstra | 8715466 | 2014-03-13 19:00:37 +0100 | [diff] [blame] | 111 | |
Catalin Marinas | 10b663a | 2012-03-05 11:49:34 +0000 | [diff] [blame] | 112 | #endif /* __ASSEMBLY__ */ |
| 113 | |
| 114 | #endif /* __ASM_BARRIER_H */ |