H. Peter Anvin | 1965aae | 2008-10-22 22:26:29 -0700 | [diff] [blame] | 1 | #ifndef _ASM_X86_UACCESS_H |
| 2 | #define _ASM_X86_UACCESS_H |
Glauber Costa | ca23386 | 2008-06-13 14:39:25 -0300 | [diff] [blame] | 3 | /* |
| 4 | * User space memory access functions |
| 5 | */ |
| 6 | #include <linux/errno.h> |
| 7 | #include <linux/compiler.h> |
Andrey Ryabinin | 1771c6e | 2016-05-20 16:59:31 -0700 | [diff] [blame] | 8 | #include <linux/kasan-checks.h> |
Glauber Costa | ca23386 | 2008-06-13 14:39:25 -0300 | [diff] [blame] | 9 | #include <linux/thread_info.h> |
Glauber Costa | ca23386 | 2008-06-13 14:39:25 -0300 | [diff] [blame] | 10 | #include <linux/string.h> |
Thomas Garnier | dbee746 | 2017-06-14 18:12:01 -0700 | [diff] [blame] | 11 | #include <linux/sched.h> |
Glauber Costa | ca23386 | 2008-06-13 14:39:25 -0300 | [diff] [blame] | 12 | #include <asm/asm.h> |
| 13 | #include <asm/page.h> |
H. Peter Anvin | 63bcff2 | 2012-09-21 12:43:12 -0700 | [diff] [blame] | 14 | #include <asm/smap.h> |
Al Viro | 45caf47 | 2016-09-05 11:32:44 -0400 | [diff] [blame] | 15 | #include <asm/extable.h> |
Glauber Costa | ca23386 | 2008-06-13 14:39:25 -0300 | [diff] [blame] | 16 | |
| 17 | #define VERIFY_READ 0 |
| 18 | #define VERIFY_WRITE 1 |
| 19 | |
| 20 | /* |
| 21 | * The fs value determines whether argument validity checking should be |
| 22 | * performed or not. If get_fs() == USER_DS, checking is performed, with |
| 23 | * get_fs() == KERNEL_DS, checking is bypassed. |
| 24 | * |
| 25 | * For historical reasons, these macros are grossly misnamed. |
| 26 | */ |
| 27 | |
| 28 | #define MAKE_MM_SEG(s) ((mm_segment_t) { (s) }) |
| 29 | |
| 30 | #define KERNEL_DS MAKE_MM_SEG(-1UL) |
Linus Torvalds | 9063c61 | 2009-06-20 15:40:00 -0700 | [diff] [blame] | 31 | #define USER_DS MAKE_MM_SEG(TASK_SIZE_MAX) |
Glauber Costa | ca23386 | 2008-06-13 14:39:25 -0300 | [diff] [blame] | 32 | |
| 33 | #define get_ds() (KERNEL_DS) |
Andy Lutomirski | 13d4ea0 | 2016-07-14 13:22:57 -0700 | [diff] [blame] | 34 | #define get_fs() (current->thread.addr_limit) |
Thomas Garnier | dbee746 | 2017-06-14 18:12:01 -0700 | [diff] [blame] | 35 | static inline void set_fs(mm_segment_t fs) |
| 36 | { |
| 37 | current->thread.addr_limit = fs; |
| 38 | /* On user-mode return, check fs is correct */ |
| 39 | set_thread_flag(TIF_FSCHECK); |
| 40 | } |
Glauber Costa | ca23386 | 2008-06-13 14:39:25 -0300 | [diff] [blame] | 41 | |
| 42 | #define segment_eq(a, b) ((a).seg == (b).seg) |
| 43 | |
Andy Lutomirski | 13d4ea0 | 2016-07-14 13:22:57 -0700 | [diff] [blame] | 44 | #define user_addr_max() (current->thread.addr_limit.seg) |
Arun Sharma | bc6ca7b | 2012-04-20 15:41:35 -0700 | [diff] [blame] | 45 | #define __addr_ok(addr) \ |
| 46 | ((unsigned long __force)(addr) < user_addr_max()) |
Glauber Costa | 002ca16 | 2008-06-25 11:08:51 -0300 | [diff] [blame] | 47 | |
Glauber Costa | ca23386 | 2008-06-13 14:39:25 -0300 | [diff] [blame] | 48 | /* |
| 49 | * Test whether a block of memory is a valid user space address. |
| 50 | * Returns 0 if the range is valid, nonzero otherwise. |
Glauber Costa | ca23386 | 2008-06-13 14:39:25 -0300 | [diff] [blame] | 51 | */ |
H. Peter Anvin | a740576 | 2013-12-27 16:52:47 -0800 | [diff] [blame] | 52 | static inline bool __chk_range_not_ok(unsigned long addr, unsigned long size, unsigned long limit) |
Linus Torvalds | c5fe5d8 | 2013-12-27 15:30:58 -0800 | [diff] [blame] | 53 | { |
| 54 | /* |
| 55 | * If we have used "sizeof()" for the size, |
| 56 | * we know it won't overflow the limit (but |
| 57 | * it might overflow the 'addr', so it's |
| 58 | * important to subtract the size from the |
| 59 | * limit, not add it to the address). |
| 60 | */ |
| 61 | if (__builtin_constant_p(size)) |
Andy Lutomirski | 7e0f51c | 2015-10-05 17:47:50 -0700 | [diff] [blame] | 62 | return unlikely(addr > limit - size); |
Linus Torvalds | c5fe5d8 | 2013-12-27 15:30:58 -0800 | [diff] [blame] | 63 | |
| 64 | /* Arbitrary sizes? Be careful about overflow */ |
| 65 | addr += size; |
Andy Lutomirski | 7e0f51c | 2015-10-05 17:47:50 -0700 | [diff] [blame] | 66 | if (unlikely(addr < size)) |
H. Peter Anvin | a740576 | 2013-12-27 16:52:47 -0800 | [diff] [blame] | 67 | return true; |
Andy Lutomirski | 7e0f51c | 2015-10-05 17:47:50 -0700 | [diff] [blame] | 68 | return unlikely(addr > limit); |
Linus Torvalds | c5fe5d8 | 2013-12-27 15:30:58 -0800 | [diff] [blame] | 69 | } |
Glauber Costa | ca23386 | 2008-06-13 14:39:25 -0300 | [diff] [blame] | 70 | |
Arun Sharma | bc6ca7b | 2012-04-20 15:41:35 -0700 | [diff] [blame] | 71 | #define __range_not_ok(addr, size, limit) \ |
Glauber Costa | ca23386 | 2008-06-13 14:39:25 -0300 | [diff] [blame] | 72 | ({ \ |
Glauber Costa | ca23386 | 2008-06-13 14:39:25 -0300 | [diff] [blame] | 73 | __chk_user_ptr(addr); \ |
Linus Torvalds | c5fe5d8 | 2013-12-27 15:30:58 -0800 | [diff] [blame] | 74 | __chk_range_not_ok((unsigned long __force)(addr), size, limit); \ |
Glauber Costa | ca23386 | 2008-06-13 14:39:25 -0300 | [diff] [blame] | 75 | }) |
| 76 | |
Peter Zijlstra | 2715f68 | 2016-11-22 10:57:15 +0100 | [diff] [blame] | 77 | #ifdef CONFIG_DEBUG_ATOMIC_SLEEP |
| 78 | # define WARN_ON_IN_IRQ() WARN_ON_ONCE(!in_task()) |
| 79 | #else |
| 80 | # define WARN_ON_IN_IRQ() |
| 81 | #endif |
| 82 | |
Glauber Costa | ca23386 | 2008-06-13 14:39:25 -0300 | [diff] [blame] | 83 | /** |
| 84 | * access_ok: - Checks if a user space pointer is valid |
| 85 | * @type: Type of access: %VERIFY_READ or %VERIFY_WRITE. Note that |
| 86 | * %VERIFY_WRITE is a superset of %VERIFY_READ - if it is safe |
| 87 | * to write to a block, it is always safe to read from it. |
| 88 | * @addr: User space pointer to start of block to check |
| 89 | * @size: Size of block to check |
| 90 | * |
David Hildenbrand | b3c395e | 2015-05-11 17:52:08 +0200 | [diff] [blame] | 91 | * Context: User context only. This function may sleep if pagefaults are |
| 92 | * enabled. |
Glauber Costa | ca23386 | 2008-06-13 14:39:25 -0300 | [diff] [blame] | 93 | * |
| 94 | * Checks if a pointer to a block of memory in user space is valid. |
| 95 | * |
| 96 | * Returns true (nonzero) if the memory block may be valid, false (zero) |
| 97 | * if it is definitely invalid. |
| 98 | * |
| 99 | * Note that, depending on architecture, this function probably just |
| 100 | * checks that the pointer is in the user space range - after calling |
| 101 | * this function, memory access functions may still return -EFAULT. |
| 102 | */ |
Peter Zijlstra | 2715f68 | 2016-11-22 10:57:15 +0100 | [diff] [blame] | 103 | #define access_ok(type, addr, size) \ |
| 104 | ({ \ |
| 105 | WARN_ON_IN_IRQ(); \ |
| 106 | likely(!__range_not_ok(addr, size, user_addr_max())); \ |
| 107 | }) |
Glauber Costa | ca23386 | 2008-06-13 14:39:25 -0300 | [diff] [blame] | 108 | |
| 109 | /* |
Glauber Costa | ca23386 | 2008-06-13 14:39:25 -0300 | [diff] [blame] | 110 | * These are the main single-value transfer routines. They automatically |
| 111 | * use the right size if we just have the right pointer type. |
| 112 | * |
| 113 | * This gets kind of ugly. We want to return _two_ values in "get_user()" |
| 114 | * and yet we don't want to do any pointers, because that is too much |
| 115 | * of a performance impact. Thus we have a few rather ugly macros here, |
| 116 | * and hide all the ugliness from the user. |
| 117 | * |
| 118 | * The "__xxx" versions of the user access functions are versions that |
| 119 | * do not verify the address space, that must have been done previously |
| 120 | * with a separate "access_ok()" call (this is used when we do multiple |
| 121 | * accesses to the same area of user memory). |
| 122 | */ |
| 123 | |
| 124 | extern int __get_user_1(void); |
| 125 | extern int __get_user_2(void); |
| 126 | extern int __get_user_4(void); |
| 127 | extern int __get_user_8(void); |
| 128 | extern int __get_user_bad(void); |
| 129 | |
Linus Torvalds | 11f1a4b | 2015-12-17 09:45:09 -0800 | [diff] [blame] | 130 | #define __uaccess_begin() stac() |
| 131 | #define __uaccess_end() clac() |
Dan Williams | e06d7bf | 2018-01-29 17:02:39 -0800 | [diff] [blame] | 132 | #define __uaccess_begin_nospec() \ |
| 133 | ({ \ |
| 134 | stac(); \ |
| 135 | barrier_nospec(); \ |
| 136 | }) |
Linus Torvalds | 11f1a4b | 2015-12-17 09:45:09 -0800 | [diff] [blame] | 137 | |
H. Peter Anvin | 3578baa | 2013-02-12 11:47:31 -0800 | [diff] [blame] | 138 | /* |
| 139 | * This is a type: either unsigned long, if the argument fits into |
| 140 | * that type, or otherwise unsigned long long. |
| 141 | */ |
| 142 | #define __inttype(x) \ |
| 143 | __typeof__(__builtin_choose_expr(sizeof(x) > sizeof(0UL), 0ULL, 0UL)) |
Glauber Costa | 865e5b7 | 2008-06-25 11:05:11 -0300 | [diff] [blame] | 144 | |
| 145 | /** |
| 146 | * get_user: - Get a simple variable from user space. |
| 147 | * @x: Variable to store result. |
| 148 | * @ptr: Source address, in user space. |
| 149 | * |
David Hildenbrand | b3c395e | 2015-05-11 17:52:08 +0200 | [diff] [blame] | 150 | * Context: User context only. This function may sleep if pagefaults are |
| 151 | * enabled. |
Glauber Costa | 865e5b7 | 2008-06-25 11:05:11 -0300 | [diff] [blame] | 152 | * |
| 153 | * This macro copies a single simple variable from user space to kernel |
| 154 | * space. It supports simple types like char and int, but not larger |
| 155 | * data types like structures or arrays. |
| 156 | * |
| 157 | * @ptr must have pointer-to-simple-variable type, and the result of |
| 158 | * dereferencing @ptr must be assignable to @x without a cast. |
| 159 | * |
| 160 | * Returns zero on success, or -EFAULT on error. |
| 161 | * On error, the variable @x is set to zero. |
H. Peter Anvin | ff52c3b | 2013-02-12 15:37:02 -0800 | [diff] [blame] | 162 | */ |
| 163 | /* |
H. Peter Anvin | 3578baa | 2013-02-12 11:47:31 -0800 | [diff] [blame] | 164 | * Careful: we have to cast the result to the type of the pointer |
| 165 | * for sign reasons. |
H. Peter Anvin | ff52c3b | 2013-02-12 15:37:02 -0800 | [diff] [blame] | 166 | * |
H. Peter Anvin | f69fa9a | 2013-08-29 13:34:50 -0700 | [diff] [blame] | 167 | * The use of _ASM_DX as the register specifier is a bit of a |
H. Peter Anvin | ff52c3b | 2013-02-12 15:37:02 -0800 | [diff] [blame] | 168 | * simplification, as gcc only cares about it as the starting point |
| 169 | * and not size: for a 64-bit value it will use %ecx:%edx on 32 bits |
| 170 | * (%ecx being the next register in gcc's x86 register sequence), and |
| 171 | * %rdx on 64 bits. |
H. Peter Anvin | f69fa9a | 2013-08-29 13:34:50 -0700 | [diff] [blame] | 172 | * |
| 173 | * Clang/LLVM cares about the size of the register, but still wants |
| 174 | * the base register for something that ends up being a pair. |
Glauber Costa | 865e5b7 | 2008-06-25 11:05:11 -0300 | [diff] [blame] | 175 | */ |
Glauber Costa | 865e5b7 | 2008-06-25 11:05:11 -0300 | [diff] [blame] | 176 | #define get_user(x, ptr) \ |
| 177 | ({ \ |
| 178 | int __ret_gu; \ |
Jan-Simon Möller | bdfc017 | 2013-08-29 21:13:05 +0200 | [diff] [blame] | 179 | register __inttype(*(ptr)) __val_gu asm("%"_ASM_DX); \ |
Glauber Costa | 865e5b7 | 2008-06-25 11:05:11 -0300 | [diff] [blame] | 180 | __chk_user_ptr(ptr); \ |
Ingo Molnar | d1a7618 | 2008-10-28 16:54:49 +0100 | [diff] [blame] | 181 | might_fault(); \ |
Chris J Arges | f05058c | 2016-01-21 16:49:25 -0600 | [diff] [blame] | 182 | asm volatile("call __get_user_%P4" \ |
Josh Poimboeuf | d5ea93e | 2017-09-20 16:24:33 -0500 | [diff] [blame] | 183 | : "=a" (__ret_gu), "=r" (__val_gu), \ |
| 184 | ASM_CALL_CONSTRAINT \ |
H. Peter Anvin | 3578baa | 2013-02-12 11:47:31 -0800 | [diff] [blame] | 185 | : "0" (ptr), "i" (sizeof(*(ptr)))); \ |
Michael S. Tsirkin | e182c57 | 2014-12-12 01:56:04 +0200 | [diff] [blame] | 186 | (x) = (__force __typeof__(*(ptr))) __val_gu; \ |
Andy Lutomirski | a76cf66 | 2015-10-05 17:47:49 -0700 | [diff] [blame] | 187 | __builtin_expect(__ret_gu, 0); \ |
Glauber Costa | 865e5b7 | 2008-06-25 11:05:11 -0300 | [diff] [blame] | 188 | }) |
| 189 | |
Glauber Costa | e30a44f | 2008-06-25 13:17:43 -0300 | [diff] [blame] | 190 | #define __put_user_x(size, x, ptr, __ret_pu) \ |
| 191 | asm volatile("call __put_user_" #size : "=a" (__ret_pu) \ |
Hiroshi Shimamoto | 4d5d783 | 2009-01-19 16:34:26 -0800 | [diff] [blame] | 192 | : "0" ((typeof(*(ptr)))(x)), "c" (ptr) : "ebx") |
Glauber Costa | e30a44f | 2008-06-25 13:17:43 -0300 | [diff] [blame] | 193 | |
| 194 | |
| 195 | |
Glauber Costa | dc70ddf | 2008-06-25 11:48:29 -0300 | [diff] [blame] | 196 | #ifdef CONFIG_X86_32 |
Hiroshi Shimamoto | 18114f6 | 2009-01-30 18:16:46 -0800 | [diff] [blame] | 197 | #define __put_user_asm_u64(x, addr, err, errret) \ |
Linus Torvalds | 11f1a4b | 2015-12-17 09:45:09 -0800 | [diff] [blame] | 198 | asm volatile("\n" \ |
H. Peter Anvin | 63bcff2 | 2012-09-21 12:43:12 -0700 | [diff] [blame] | 199 | "1: movl %%eax,0(%2)\n" \ |
Glauber Costa | dc70ddf | 2008-06-25 11:48:29 -0300 | [diff] [blame] | 200 | "2: movl %%edx,4(%2)\n" \ |
Linus Torvalds | 11f1a4b | 2015-12-17 09:45:09 -0800 | [diff] [blame] | 201 | "3:" \ |
Glauber Costa | dc70ddf | 2008-06-25 11:48:29 -0300 | [diff] [blame] | 202 | ".section .fixup,\"ax\"\n" \ |
| 203 | "4: movl %3,%0\n" \ |
| 204 | " jmp 3b\n" \ |
| 205 | ".previous\n" \ |
| 206 | _ASM_EXTABLE(1b, 4b) \ |
| 207 | _ASM_EXTABLE(2b, 4b) \ |
| 208 | : "=r" (err) \ |
Hiroshi Shimamoto | 18114f6 | 2009-01-30 18:16:46 -0800 | [diff] [blame] | 209 | : "A" (x), "r" (addr), "i" (errret), "0" (err)) |
Glauber Costa | e30a44f | 2008-06-25 13:17:43 -0300 | [diff] [blame] | 210 | |
Hiroshi Shimamoto | fe40c0a | 2009-01-23 15:49:41 -0800 | [diff] [blame] | 211 | #define __put_user_asm_ex_u64(x, addr) \ |
Linus Torvalds | 11f1a4b | 2015-12-17 09:45:09 -0800 | [diff] [blame] | 212 | asm volatile("\n" \ |
H. Peter Anvin | 63bcff2 | 2012-09-21 12:43:12 -0700 | [diff] [blame] | 213 | "1: movl %%eax,0(%1)\n" \ |
Hiroshi Shimamoto | fe40c0a | 2009-01-23 15:49:41 -0800 | [diff] [blame] | 214 | "2: movl %%edx,4(%1)\n" \ |
Linus Torvalds | 11f1a4b | 2015-12-17 09:45:09 -0800 | [diff] [blame] | 215 | "3:" \ |
H. Peter Anvin | 535c0c3 | 2012-04-20 16:57:35 -0700 | [diff] [blame] | 216 | _ASM_EXTABLE_EX(1b, 2b) \ |
| 217 | _ASM_EXTABLE_EX(2b, 3b) \ |
Hiroshi Shimamoto | fe40c0a | 2009-01-23 15:49:41 -0800 | [diff] [blame] | 218 | : : "A" (x), "r" (addr)) |
| 219 | |
Glauber Costa | e30a44f | 2008-06-25 13:17:43 -0300 | [diff] [blame] | 220 | #define __put_user_x8(x, ptr, __ret_pu) \ |
| 221 | asm volatile("call __put_user_8" : "=a" (__ret_pu) \ |
| 222 | : "A" ((typeof(*(ptr)))(x)), "c" (ptr) : "ebx") |
Glauber Costa | dc70ddf | 2008-06-25 11:48:29 -0300 | [diff] [blame] | 223 | #else |
Hiroshi Shimamoto | 18114f6 | 2009-01-30 18:16:46 -0800 | [diff] [blame] | 224 | #define __put_user_asm_u64(x, ptr, retval, errret) \ |
H. Peter Anvin | ebe119c | 2009-07-20 23:27:39 -0700 | [diff] [blame] | 225 | __put_user_asm(x, ptr, retval, "q", "", "er", errret) |
Hiroshi Shimamoto | fe40c0a | 2009-01-23 15:49:41 -0800 | [diff] [blame] | 226 | #define __put_user_asm_ex_u64(x, addr) \ |
H. Peter Anvin | ebe119c | 2009-07-20 23:27:39 -0700 | [diff] [blame] | 227 | __put_user_asm_ex(x, addr, "q", "", "er") |
Glauber Costa | e30a44f | 2008-06-25 13:17:43 -0300 | [diff] [blame] | 228 | #define __put_user_x8(x, ptr, __ret_pu) __put_user_x(8, x, ptr, __ret_pu) |
Glauber Costa | dc70ddf | 2008-06-25 11:48:29 -0300 | [diff] [blame] | 229 | #endif |
| 230 | |
Glauber Costa | e30a44f | 2008-06-25 13:17:43 -0300 | [diff] [blame] | 231 | extern void __put_user_bad(void); |
| 232 | |
| 233 | /* |
| 234 | * Strange magic calling convention: pointer in %ecx, |
| 235 | * value in %eax(:%edx), return value in %eax. clobbers %rbx |
| 236 | */ |
| 237 | extern void __put_user_1(void); |
| 238 | extern void __put_user_2(void); |
| 239 | extern void __put_user_4(void); |
| 240 | extern void __put_user_8(void); |
| 241 | |
Glauber Costa | e30a44f | 2008-06-25 13:17:43 -0300 | [diff] [blame] | 242 | /** |
| 243 | * put_user: - Write a simple value into user space. |
| 244 | * @x: Value to copy to user space. |
| 245 | * @ptr: Destination address, in user space. |
| 246 | * |
David Hildenbrand | b3c395e | 2015-05-11 17:52:08 +0200 | [diff] [blame] | 247 | * Context: User context only. This function may sleep if pagefaults are |
| 248 | * enabled. |
Glauber Costa | e30a44f | 2008-06-25 13:17:43 -0300 | [diff] [blame] | 249 | * |
| 250 | * This macro copies a single simple value from kernel space to user |
| 251 | * space. It supports simple types like char and int, but not larger |
| 252 | * data types like structures or arrays. |
| 253 | * |
| 254 | * @ptr must have pointer-to-simple-variable type, and @x must be assignable |
| 255 | * to the result of dereferencing @ptr. |
| 256 | * |
| 257 | * Returns zero on success, or -EFAULT on error. |
| 258 | */ |
| 259 | #define put_user(x, ptr) \ |
| 260 | ({ \ |
| 261 | int __ret_pu; \ |
| 262 | __typeof__(*(ptr)) __pu_val; \ |
| 263 | __chk_user_ptr(ptr); \ |
Ingo Molnar | d1a7618 | 2008-10-28 16:54:49 +0100 | [diff] [blame] | 264 | might_fault(); \ |
Glauber Costa | e30a44f | 2008-06-25 13:17:43 -0300 | [diff] [blame] | 265 | __pu_val = x; \ |
| 266 | switch (sizeof(*(ptr))) { \ |
| 267 | case 1: \ |
| 268 | __put_user_x(1, __pu_val, ptr, __ret_pu); \ |
| 269 | break; \ |
| 270 | case 2: \ |
| 271 | __put_user_x(2, __pu_val, ptr, __ret_pu); \ |
| 272 | break; \ |
| 273 | case 4: \ |
| 274 | __put_user_x(4, __pu_val, ptr, __ret_pu); \ |
| 275 | break; \ |
| 276 | case 8: \ |
| 277 | __put_user_x8(__pu_val, ptr, __ret_pu); \ |
| 278 | break; \ |
| 279 | default: \ |
| 280 | __put_user_x(X, __pu_val, ptr, __ret_pu); \ |
| 281 | break; \ |
| 282 | } \ |
Andy Lutomirski | a76cf66 | 2015-10-05 17:47:49 -0700 | [diff] [blame] | 283 | __builtin_expect(__ret_pu, 0); \ |
Glauber Costa | e30a44f | 2008-06-25 13:17:43 -0300 | [diff] [blame] | 284 | }) |
| 285 | |
Glauber Costa | dc70ddf | 2008-06-25 11:48:29 -0300 | [diff] [blame] | 286 | #define __put_user_size(x, ptr, size, retval, errret) \ |
| 287 | do { \ |
| 288 | retval = 0; \ |
| 289 | __chk_user_ptr(ptr); \ |
| 290 | switch (size) { \ |
| 291 | case 1: \ |
| 292 | __put_user_asm(x, ptr, retval, "b", "b", "iq", errret); \ |
| 293 | break; \ |
| 294 | case 2: \ |
| 295 | __put_user_asm(x, ptr, retval, "w", "w", "ir", errret); \ |
| 296 | break; \ |
| 297 | case 4: \ |
Hiroshi Shimamoto | 4d5d783 | 2009-01-19 16:34:26 -0800 | [diff] [blame] | 298 | __put_user_asm(x, ptr, retval, "l", "k", "ir", errret); \ |
Glauber Costa | dc70ddf | 2008-06-25 11:48:29 -0300 | [diff] [blame] | 299 | break; \ |
| 300 | case 8: \ |
Hiroshi Shimamoto | 18114f6 | 2009-01-30 18:16:46 -0800 | [diff] [blame] | 301 | __put_user_asm_u64((__typeof__(*ptr))(x), ptr, retval, \ |
| 302 | errret); \ |
Glauber Costa | dc70ddf | 2008-06-25 11:48:29 -0300 | [diff] [blame] | 303 | break; \ |
| 304 | default: \ |
| 305 | __put_user_bad(); \ |
| 306 | } \ |
| 307 | } while (0) |
| 308 | |
Linus Torvalds | 11f1a4b | 2015-12-17 09:45:09 -0800 | [diff] [blame] | 309 | /* |
| 310 | * This doesn't do __uaccess_begin/end - the exception handling |
| 311 | * around it must do that. |
| 312 | */ |
Hiroshi Shimamoto | fe40c0a | 2009-01-23 15:49:41 -0800 | [diff] [blame] | 313 | #define __put_user_size_ex(x, ptr, size) \ |
| 314 | do { \ |
| 315 | __chk_user_ptr(ptr); \ |
| 316 | switch (size) { \ |
| 317 | case 1: \ |
| 318 | __put_user_asm_ex(x, ptr, "b", "b", "iq"); \ |
| 319 | break; \ |
| 320 | case 2: \ |
| 321 | __put_user_asm_ex(x, ptr, "w", "w", "ir"); \ |
| 322 | break; \ |
| 323 | case 4: \ |
| 324 | __put_user_asm_ex(x, ptr, "l", "k", "ir"); \ |
| 325 | break; \ |
| 326 | case 8: \ |
| 327 | __put_user_asm_ex_u64((__typeof__(*ptr))(x), ptr); \ |
| 328 | break; \ |
| 329 | default: \ |
| 330 | __put_user_bad(); \ |
| 331 | } \ |
| 332 | } while (0) |
| 333 | |
Glauber Costa | 3f16822 | 2008-06-25 12:48:47 -0300 | [diff] [blame] | 334 | #ifdef CONFIG_X86_32 |
Benjamin LaHaise | b2f6803 | 2016-03-09 15:05:56 -0500 | [diff] [blame] | 335 | #define __get_user_asm_u64(x, ptr, retval, errret) \ |
| 336 | ({ \ |
| 337 | __typeof__(ptr) __ptr = (ptr); \ |
Linus Torvalds | ae382ca | 2017-05-21 18:26:54 -0700 | [diff] [blame] | 338 | asm volatile("\n" \ |
Benjamin LaHaise | b2f6803 | 2016-03-09 15:05:56 -0500 | [diff] [blame] | 339 | "1: movl %2,%%eax\n" \ |
| 340 | "2: movl %3,%%edx\n" \ |
Linus Torvalds | ae382ca | 2017-05-21 18:26:54 -0700 | [diff] [blame] | 341 | "3:\n" \ |
Benjamin LaHaise | b2f6803 | 2016-03-09 15:05:56 -0500 | [diff] [blame] | 342 | ".section .fixup,\"ax\"\n" \ |
| 343 | "4: mov %4,%0\n" \ |
| 344 | " xorl %%eax,%%eax\n" \ |
| 345 | " xorl %%edx,%%edx\n" \ |
| 346 | " jmp 3b\n" \ |
| 347 | ".previous\n" \ |
| 348 | _ASM_EXTABLE(1b, 4b) \ |
| 349 | _ASM_EXTABLE(2b, 4b) \ |
Linus Torvalds | ae382ca | 2017-05-21 18:26:54 -0700 | [diff] [blame] | 350 | : "=r" (retval), "=&A"(x) \ |
Benjamin LaHaise | b2f6803 | 2016-03-09 15:05:56 -0500 | [diff] [blame] | 351 | : "m" (__m(__ptr)), "m" __m(((u32 *)(__ptr)) + 1), \ |
| 352 | "i" (errret), "0" (retval)); \ |
| 353 | }) |
| 354 | |
Hiroshi Shimamoto | fe40c0a | 2009-01-23 15:49:41 -0800 | [diff] [blame] | 355 | #define __get_user_asm_ex_u64(x, ptr) (x) = __get_user_bad() |
Glauber Costa | 3f16822 | 2008-06-25 12:48:47 -0300 | [diff] [blame] | 356 | #else |
| 357 | #define __get_user_asm_u64(x, ptr, retval, errret) \ |
| 358 | __get_user_asm(x, ptr, retval, "q", "", "=r", errret) |
Hiroshi Shimamoto | fe40c0a | 2009-01-23 15:49:41 -0800 | [diff] [blame] | 359 | #define __get_user_asm_ex_u64(x, ptr) \ |
| 360 | __get_user_asm_ex(x, ptr, "q", "", "=r") |
Glauber Costa | 3f16822 | 2008-06-25 12:48:47 -0300 | [diff] [blame] | 361 | #endif |
| 362 | |
| 363 | #define __get_user_size(x, ptr, size, retval, errret) \ |
| 364 | do { \ |
| 365 | retval = 0; \ |
| 366 | __chk_user_ptr(ptr); \ |
| 367 | switch (size) { \ |
| 368 | case 1: \ |
| 369 | __get_user_asm(x, ptr, retval, "b", "b", "=q", errret); \ |
| 370 | break; \ |
| 371 | case 2: \ |
| 372 | __get_user_asm(x, ptr, retval, "w", "w", "=r", errret); \ |
| 373 | break; \ |
| 374 | case 4: \ |
| 375 | __get_user_asm(x, ptr, retval, "l", "k", "=r", errret); \ |
| 376 | break; \ |
| 377 | case 8: \ |
| 378 | __get_user_asm_u64(x, ptr, retval, errret); \ |
| 379 | break; \ |
| 380 | default: \ |
| 381 | (x) = __get_user_bad(); \ |
| 382 | } \ |
| 383 | } while (0) |
| 384 | |
| 385 | #define __get_user_asm(x, addr, err, itype, rtype, ltype, errret) \ |
Linus Torvalds | 11f1a4b | 2015-12-17 09:45:09 -0800 | [diff] [blame] | 386 | asm volatile("\n" \ |
H. Peter Anvin | 63bcff2 | 2012-09-21 12:43:12 -0700 | [diff] [blame] | 387 | "1: mov"itype" %2,%"rtype"1\n" \ |
Linus Torvalds | 11f1a4b | 2015-12-17 09:45:09 -0800 | [diff] [blame] | 388 | "2:\n" \ |
Glauber Costa | 3f16822 | 2008-06-25 12:48:47 -0300 | [diff] [blame] | 389 | ".section .fixup,\"ax\"\n" \ |
| 390 | "3: mov %3,%0\n" \ |
| 391 | " xor"itype" %"rtype"1,%"rtype"1\n" \ |
| 392 | " jmp 2b\n" \ |
| 393 | ".previous\n" \ |
| 394 | _ASM_EXTABLE(1b, 3b) \ |
| 395 | : "=r" (err), ltype(x) \ |
| 396 | : "m" (__m(addr)), "i" (errret), "0" (err)) |
| 397 | |
Linus Torvalds | 11f1a4b | 2015-12-17 09:45:09 -0800 | [diff] [blame] | 398 | /* |
| 399 | * This doesn't do __uaccess_begin/end - the exception handling |
| 400 | * around it must do that. |
| 401 | */ |
Hiroshi Shimamoto | fe40c0a | 2009-01-23 15:49:41 -0800 | [diff] [blame] | 402 | #define __get_user_size_ex(x, ptr, size) \ |
| 403 | do { \ |
| 404 | __chk_user_ptr(ptr); \ |
| 405 | switch (size) { \ |
| 406 | case 1: \ |
| 407 | __get_user_asm_ex(x, ptr, "b", "b", "=q"); \ |
| 408 | break; \ |
| 409 | case 2: \ |
| 410 | __get_user_asm_ex(x, ptr, "w", "w", "=r"); \ |
| 411 | break; \ |
| 412 | case 4: \ |
| 413 | __get_user_asm_ex(x, ptr, "l", "k", "=r"); \ |
| 414 | break; \ |
| 415 | case 8: \ |
| 416 | __get_user_asm_ex_u64(x, ptr); \ |
| 417 | break; \ |
| 418 | default: \ |
| 419 | (x) = __get_user_bad(); \ |
| 420 | } \ |
| 421 | } while (0) |
| 422 | |
| 423 | #define __get_user_asm_ex(x, addr, itype, rtype, ltype) \ |
H. Peter Anvin | 5e88353 | 2012-09-21 12:43:15 -0700 | [diff] [blame] | 424 | asm volatile("1: mov"itype" %1,%"rtype"0\n" \ |
| 425 | "2:\n" \ |
Al Viro | 1c109fa | 2016-09-15 02:35:29 +0100 | [diff] [blame] | 426 | ".section .fixup,\"ax\"\n" \ |
| 427 | "3:xor"itype" %"rtype"0,%"rtype"0\n" \ |
| 428 | " jmp 2b\n" \ |
| 429 | ".previous\n" \ |
| 430 | _ASM_EXTABLE_EX(1b, 3b) \ |
Hiroshi Shimamoto | fe40c0a | 2009-01-23 15:49:41 -0800 | [diff] [blame] | 431 | : ltype(x) : "m" (__m(addr))) |
| 432 | |
Glauber Costa | dc70ddf | 2008-06-25 11:48:29 -0300 | [diff] [blame] | 433 | #define __put_user_nocheck(x, ptr, size) \ |
| 434 | ({ \ |
Hiroshi Shimamoto | 16855f8 | 2008-12-08 19:18:38 -0800 | [diff] [blame] | 435 | int __pu_err; \ |
Linus Torvalds | 11f1a4b | 2015-12-17 09:45:09 -0800 | [diff] [blame] | 436 | __uaccess_begin(); \ |
Glauber Costa | dc70ddf | 2008-06-25 11:48:29 -0300 | [diff] [blame] | 437 | __put_user_size((x), (ptr), (size), __pu_err, -EFAULT); \ |
Linus Torvalds | 11f1a4b | 2015-12-17 09:45:09 -0800 | [diff] [blame] | 438 | __uaccess_end(); \ |
Andy Lutomirski | a76cf66 | 2015-10-05 17:47:49 -0700 | [diff] [blame] | 439 | __builtin_expect(__pu_err, 0); \ |
Glauber Costa | dc70ddf | 2008-06-25 11:48:29 -0300 | [diff] [blame] | 440 | }) |
| 441 | |
Glauber Costa | 3f16822 | 2008-06-25 12:48:47 -0300 | [diff] [blame] | 442 | #define __get_user_nocheck(x, ptr, size) \ |
| 443 | ({ \ |
Hiroshi Shimamoto | 16855f8 | 2008-12-08 19:18:38 -0800 | [diff] [blame] | 444 | int __gu_err; \ |
Benjamin LaHaise | b2f6803 | 2016-03-09 15:05:56 -0500 | [diff] [blame] | 445 | __inttype(*(ptr)) __gu_val; \ |
Dan Williams | 065eae4 | 2018-01-29 17:02:49 -0800 | [diff] [blame] | 446 | __uaccess_begin_nospec(); \ |
Glauber Costa | 3f16822 | 2008-06-25 12:48:47 -0300 | [diff] [blame] | 447 | __get_user_size(__gu_val, (ptr), (size), __gu_err, -EFAULT); \ |
Linus Torvalds | 11f1a4b | 2015-12-17 09:45:09 -0800 | [diff] [blame] | 448 | __uaccess_end(); \ |
Glauber Costa | 3f16822 | 2008-06-25 12:48:47 -0300 | [diff] [blame] | 449 | (x) = (__force __typeof__(*(ptr)))__gu_val; \ |
Andy Lutomirski | a76cf66 | 2015-10-05 17:47:49 -0700 | [diff] [blame] | 450 | __builtin_expect(__gu_err, 0); \ |
Glauber Costa | 3f16822 | 2008-06-25 12:48:47 -0300 | [diff] [blame] | 451 | }) |
Glauber Costa | dc70ddf | 2008-06-25 11:48:29 -0300 | [diff] [blame] | 452 | |
| 453 | /* FIXME: this hack is definitely wrong -AK */ |
| 454 | struct __large_struct { unsigned long buf[100]; }; |
| 455 | #define __m(x) (*(struct __large_struct __user *)(x)) |
| 456 | |
| 457 | /* |
| 458 | * Tell gcc we read from memory instead of writing: this is because |
| 459 | * we do not write to any memory gcc knows about, so there are no |
| 460 | * aliasing issues. |
| 461 | */ |
| 462 | #define __put_user_asm(x, addr, err, itype, rtype, ltype, errret) \ |
Linus Torvalds | 11f1a4b | 2015-12-17 09:45:09 -0800 | [diff] [blame] | 463 | asm volatile("\n" \ |
H. Peter Anvin | 63bcff2 | 2012-09-21 12:43:12 -0700 | [diff] [blame] | 464 | "1: mov"itype" %"rtype"1,%2\n" \ |
Linus Torvalds | 11f1a4b | 2015-12-17 09:45:09 -0800 | [diff] [blame] | 465 | "2:\n" \ |
Glauber Costa | dc70ddf | 2008-06-25 11:48:29 -0300 | [diff] [blame] | 466 | ".section .fixup,\"ax\"\n" \ |
| 467 | "3: mov %3,%0\n" \ |
| 468 | " jmp 2b\n" \ |
| 469 | ".previous\n" \ |
| 470 | _ASM_EXTABLE(1b, 3b) \ |
| 471 | : "=r"(err) \ |
| 472 | : ltype(x), "m" (__m(addr)), "i" (errret), "0" (err)) |
Hiroshi Shimamoto | fe40c0a | 2009-01-23 15:49:41 -0800 | [diff] [blame] | 473 | |
| 474 | #define __put_user_asm_ex(x, addr, itype, rtype, ltype) \ |
H. Peter Anvin | 5e88353 | 2012-09-21 12:43:15 -0700 | [diff] [blame] | 475 | asm volatile("1: mov"itype" %"rtype"0,%1\n" \ |
| 476 | "2:\n" \ |
H. Peter Anvin | 535c0c3 | 2012-04-20 16:57:35 -0700 | [diff] [blame] | 477 | _ASM_EXTABLE_EX(1b, 2b) \ |
Hiroshi Shimamoto | fe40c0a | 2009-01-23 15:49:41 -0800 | [diff] [blame] | 478 | : : ltype(x), "m" (__m(addr))) |
| 479 | |
| 480 | /* |
| 481 | * uaccess_try and catch |
| 482 | */ |
| 483 | #define uaccess_try do { \ |
Andy Lutomirski | dfa9a94 | 2016-07-14 13:22:56 -0700 | [diff] [blame] | 484 | current->thread.uaccess_err = 0; \ |
Linus Torvalds | 11f1a4b | 2015-12-17 09:45:09 -0800 | [diff] [blame] | 485 | __uaccess_begin(); \ |
Hiroshi Shimamoto | fe40c0a | 2009-01-23 15:49:41 -0800 | [diff] [blame] | 486 | barrier(); |
| 487 | |
Dan Williams | e06d7bf | 2018-01-29 17:02:39 -0800 | [diff] [blame] | 488 | #define uaccess_try_nospec do { \ |
| 489 | current->thread.uaccess_err = 0; \ |
| 490 | __uaccess_begin_nospec(); \ |
| 491 | |
Hiroshi Shimamoto | fe40c0a | 2009-01-23 15:49:41 -0800 | [diff] [blame] | 492 | #define uaccess_catch(err) \ |
Linus Torvalds | 11f1a4b | 2015-12-17 09:45:09 -0800 | [diff] [blame] | 493 | __uaccess_end(); \ |
Andy Lutomirski | dfa9a94 | 2016-07-14 13:22:56 -0700 | [diff] [blame] | 494 | (err) |= (current->thread.uaccess_err ? -EFAULT : 0); \ |
Hiroshi Shimamoto | fe40c0a | 2009-01-23 15:49:41 -0800 | [diff] [blame] | 495 | } while (0) |
| 496 | |
Glauber Costa | 8cb834e | 2008-06-25 14:43:30 -0300 | [diff] [blame] | 497 | /** |
| 498 | * __get_user: - Get a simple variable from user space, with less checking. |
| 499 | * @x: Variable to store result. |
| 500 | * @ptr: Source address, in user space. |
| 501 | * |
David Hildenbrand | b3c395e | 2015-05-11 17:52:08 +0200 | [diff] [blame] | 502 | * Context: User context only. This function may sleep if pagefaults are |
| 503 | * enabled. |
Glauber Costa | 8cb834e | 2008-06-25 14:43:30 -0300 | [diff] [blame] | 504 | * |
| 505 | * This macro copies a single simple variable from user space to kernel |
| 506 | * space. It supports simple types like char and int, but not larger |
| 507 | * data types like structures or arrays. |
| 508 | * |
| 509 | * @ptr must have pointer-to-simple-variable type, and the result of |
| 510 | * dereferencing @ptr must be assignable to @x without a cast. |
| 511 | * |
| 512 | * Caller must check the pointer with access_ok() before calling this |
| 513 | * function. |
| 514 | * |
| 515 | * Returns zero on success, or -EFAULT on error. |
| 516 | * On error, the variable @x is set to zero. |
| 517 | */ |
Glauber Costa | dc70ddf | 2008-06-25 11:48:29 -0300 | [diff] [blame] | 518 | |
Glauber Costa | 8cb834e | 2008-06-25 14:43:30 -0300 | [diff] [blame] | 519 | #define __get_user(x, ptr) \ |
| 520 | __get_user_nocheck((x), (ptr), sizeof(*(ptr))) |
Hiroshi Shimamoto | fe40c0a | 2009-01-23 15:49:41 -0800 | [diff] [blame] | 521 | |
Glauber Costa | 8cb834e | 2008-06-25 14:43:30 -0300 | [diff] [blame] | 522 | /** |
| 523 | * __put_user: - Write a simple value into user space, with less checking. |
| 524 | * @x: Value to copy to user space. |
| 525 | * @ptr: Destination address, in user space. |
| 526 | * |
David Hildenbrand | b3c395e | 2015-05-11 17:52:08 +0200 | [diff] [blame] | 527 | * Context: User context only. This function may sleep if pagefaults are |
| 528 | * enabled. |
Glauber Costa | 8cb834e | 2008-06-25 14:43:30 -0300 | [diff] [blame] | 529 | * |
| 530 | * This macro copies a single simple value from kernel space to user |
| 531 | * space. It supports simple types like char and int, but not larger |
| 532 | * data types like structures or arrays. |
| 533 | * |
| 534 | * @ptr must have pointer-to-simple-variable type, and @x must be assignable |
| 535 | * to the result of dereferencing @ptr. |
| 536 | * |
| 537 | * Caller must check the pointer with access_ok() before calling this |
| 538 | * function. |
| 539 | * |
| 540 | * Returns zero on success, or -EFAULT on error. |
| 541 | */ |
| 542 | |
| 543 | #define __put_user(x, ptr) \ |
| 544 | __put_user_nocheck((__typeof__(*(ptr)))(x), (ptr), sizeof(*(ptr))) |
| 545 | |
| 546 | #define __get_user_unaligned __get_user |
| 547 | #define __put_user_unaligned __put_user |
Glauber Costa | 865e5b7 | 2008-06-25 11:05:11 -0300 | [diff] [blame] | 548 | |
Glauber Costa | 8bc7de0c | 2008-06-25 14:53:41 -0300 | [diff] [blame] | 549 | /* |
Hiroshi Shimamoto | fe40c0a | 2009-01-23 15:49:41 -0800 | [diff] [blame] | 550 | * {get|put}_user_try and catch |
| 551 | * |
| 552 | * get_user_try { |
| 553 | * get_user_ex(...); |
| 554 | * } get_user_catch(err) |
| 555 | */ |
Dan Williams | 065eae4 | 2018-01-29 17:02:49 -0800 | [diff] [blame] | 556 | #define get_user_try uaccess_try_nospec |
Hiroshi Shimamoto | fe40c0a | 2009-01-23 15:49:41 -0800 | [diff] [blame] | 557 | #define get_user_catch(err) uaccess_catch(err) |
Hiroshi Shimamoto | fe40c0a | 2009-01-23 15:49:41 -0800 | [diff] [blame] | 558 | |
| 559 | #define get_user_ex(x, ptr) do { \ |
| 560 | unsigned long __gue_val; \ |
| 561 | __get_user_size_ex((__gue_val), (ptr), (sizeof(*(ptr)))); \ |
| 562 | (x) = (__force __typeof__(*(ptr)))__gue_val; \ |
| 563 | } while (0) |
| 564 | |
Hiroshi Shimamoto | 019a136 | 2009-01-29 11:49:18 -0800 | [diff] [blame] | 565 | #define put_user_try uaccess_try |
| 566 | #define put_user_catch(err) uaccess_catch(err) |
| 567 | |
Hiroshi Shimamoto | fe40c0a | 2009-01-23 15:49:41 -0800 | [diff] [blame] | 568 | #define put_user_ex(x, ptr) \ |
| 569 | __put_user_size_ex((__typeof__(*(ptr)))(x), (ptr), sizeof(*(ptr))) |
| 570 | |
Robert Richter | 1ac2e6c | 2011-06-07 11:49:55 +0200 | [diff] [blame] | 571 | extern unsigned long |
| 572 | copy_from_user_nmi(void *to, const void __user *from, unsigned long n); |
Linus Torvalds | 92ae03f | 2012-04-06 14:32:32 -0700 | [diff] [blame] | 573 | extern __must_check long |
| 574 | strncpy_from_user(char *dst, const char __user *src, long count); |
Robert Richter | 1ac2e6c | 2011-06-07 11:49:55 +0200 | [diff] [blame] | 575 | |
Linus Torvalds | 5723aa9 | 2012-05-26 11:09:53 -0700 | [diff] [blame] | 576 | extern __must_check long strlen_user(const char __user *str); |
| 577 | extern __must_check long strnlen_user(const char __user *str, long n); |
| 578 | |
H. Peter Anvin | a052858 | 2012-09-21 12:43:11 -0700 | [diff] [blame] | 579 | unsigned long __must_check clear_user(void __user *mem, unsigned long len); |
| 580 | unsigned long __must_check __clear_user(void __user *mem, unsigned long len); |
| 581 | |
Qiaowei Ren | f09174c | 2013-12-14 14:25:02 +0800 | [diff] [blame] | 582 | extern void __cmpxchg_wrong_size(void) |
| 583 | __compiletime_error("Bad argument size for cmpxchg"); |
| 584 | |
| 585 | #define __user_atomic_cmpxchg_inatomic(uval, ptr, old, new, size) \ |
| 586 | ({ \ |
| 587 | int __ret = 0; \ |
| 588 | __typeof__(ptr) __uval = (uval); \ |
| 589 | __typeof__(*(ptr)) __old = (old); \ |
| 590 | __typeof__(*(ptr)) __new = (new); \ |
Dan Williams | 065eae4 | 2018-01-29 17:02:49 -0800 | [diff] [blame] | 591 | __uaccess_begin_nospec(); \ |
Qiaowei Ren | f09174c | 2013-12-14 14:25:02 +0800 | [diff] [blame] | 592 | switch (size) { \ |
| 593 | case 1: \ |
| 594 | { \ |
Linus Torvalds | 11f1a4b | 2015-12-17 09:45:09 -0800 | [diff] [blame] | 595 | asm volatile("\n" \ |
Qiaowei Ren | f09174c | 2013-12-14 14:25:02 +0800 | [diff] [blame] | 596 | "1:\t" LOCK_PREFIX "cmpxchgb %4, %2\n" \ |
Linus Torvalds | 11f1a4b | 2015-12-17 09:45:09 -0800 | [diff] [blame] | 597 | "2:\n" \ |
Qiaowei Ren | f09174c | 2013-12-14 14:25:02 +0800 | [diff] [blame] | 598 | "\t.section .fixup, \"ax\"\n" \ |
| 599 | "3:\tmov %3, %0\n" \ |
| 600 | "\tjmp 2b\n" \ |
| 601 | "\t.previous\n" \ |
| 602 | _ASM_EXTABLE(1b, 3b) \ |
| 603 | : "+r" (__ret), "=a" (__old), "+m" (*(ptr)) \ |
| 604 | : "i" (-EFAULT), "q" (__new), "1" (__old) \ |
| 605 | : "memory" \ |
| 606 | ); \ |
| 607 | break; \ |
| 608 | } \ |
| 609 | case 2: \ |
| 610 | { \ |
Linus Torvalds | 11f1a4b | 2015-12-17 09:45:09 -0800 | [diff] [blame] | 611 | asm volatile("\n" \ |
Qiaowei Ren | f09174c | 2013-12-14 14:25:02 +0800 | [diff] [blame] | 612 | "1:\t" LOCK_PREFIX "cmpxchgw %4, %2\n" \ |
Linus Torvalds | 11f1a4b | 2015-12-17 09:45:09 -0800 | [diff] [blame] | 613 | "2:\n" \ |
Qiaowei Ren | f09174c | 2013-12-14 14:25:02 +0800 | [diff] [blame] | 614 | "\t.section .fixup, \"ax\"\n" \ |
| 615 | "3:\tmov %3, %0\n" \ |
| 616 | "\tjmp 2b\n" \ |
| 617 | "\t.previous\n" \ |
| 618 | _ASM_EXTABLE(1b, 3b) \ |
| 619 | : "+r" (__ret), "=a" (__old), "+m" (*(ptr)) \ |
| 620 | : "i" (-EFAULT), "r" (__new), "1" (__old) \ |
| 621 | : "memory" \ |
| 622 | ); \ |
| 623 | break; \ |
| 624 | } \ |
| 625 | case 4: \ |
| 626 | { \ |
Linus Torvalds | 11f1a4b | 2015-12-17 09:45:09 -0800 | [diff] [blame] | 627 | asm volatile("\n" \ |
Qiaowei Ren | f09174c | 2013-12-14 14:25:02 +0800 | [diff] [blame] | 628 | "1:\t" LOCK_PREFIX "cmpxchgl %4, %2\n" \ |
Linus Torvalds | 11f1a4b | 2015-12-17 09:45:09 -0800 | [diff] [blame] | 629 | "2:\n" \ |
Qiaowei Ren | f09174c | 2013-12-14 14:25:02 +0800 | [diff] [blame] | 630 | "\t.section .fixup, \"ax\"\n" \ |
| 631 | "3:\tmov %3, %0\n" \ |
| 632 | "\tjmp 2b\n" \ |
| 633 | "\t.previous\n" \ |
| 634 | _ASM_EXTABLE(1b, 3b) \ |
| 635 | : "+r" (__ret), "=a" (__old), "+m" (*(ptr)) \ |
| 636 | : "i" (-EFAULT), "r" (__new), "1" (__old) \ |
| 637 | : "memory" \ |
| 638 | ); \ |
| 639 | break; \ |
| 640 | } \ |
| 641 | case 8: \ |
| 642 | { \ |
| 643 | if (!IS_ENABLED(CONFIG_X86_64)) \ |
| 644 | __cmpxchg_wrong_size(); \ |
| 645 | \ |
Linus Torvalds | 11f1a4b | 2015-12-17 09:45:09 -0800 | [diff] [blame] | 646 | asm volatile("\n" \ |
Qiaowei Ren | f09174c | 2013-12-14 14:25:02 +0800 | [diff] [blame] | 647 | "1:\t" LOCK_PREFIX "cmpxchgq %4, %2\n" \ |
Linus Torvalds | 11f1a4b | 2015-12-17 09:45:09 -0800 | [diff] [blame] | 648 | "2:\n" \ |
Qiaowei Ren | f09174c | 2013-12-14 14:25:02 +0800 | [diff] [blame] | 649 | "\t.section .fixup, \"ax\"\n" \ |
| 650 | "3:\tmov %3, %0\n" \ |
| 651 | "\tjmp 2b\n" \ |
| 652 | "\t.previous\n" \ |
| 653 | _ASM_EXTABLE(1b, 3b) \ |
| 654 | : "+r" (__ret), "=a" (__old), "+m" (*(ptr)) \ |
| 655 | : "i" (-EFAULT), "r" (__new), "1" (__old) \ |
| 656 | : "memory" \ |
| 657 | ); \ |
| 658 | break; \ |
| 659 | } \ |
| 660 | default: \ |
| 661 | __cmpxchg_wrong_size(); \ |
| 662 | } \ |
Linus Torvalds | 11f1a4b | 2015-12-17 09:45:09 -0800 | [diff] [blame] | 663 | __uaccess_end(); \ |
Qiaowei Ren | f09174c | 2013-12-14 14:25:02 +0800 | [diff] [blame] | 664 | *__uval = __old; \ |
| 665 | __ret; \ |
| 666 | }) |
| 667 | |
| 668 | #define user_atomic_cmpxchg_inatomic(uval, ptr, old, new) \ |
| 669 | ({ \ |
| 670 | access_ok(VERIFY_WRITE, (ptr), sizeof(*(ptr))) ? \ |
| 671 | __user_atomic_cmpxchg_inatomic((uval), (ptr), \ |
| 672 | (old), (new), sizeof(*(ptr))) : \ |
| 673 | -EFAULT; \ |
| 674 | }) |
| 675 | |
Hiroshi Shimamoto | fe40c0a | 2009-01-23 15:49:41 -0800 | [diff] [blame] | 676 | /* |
Glauber Costa | 8bc7de0c | 2008-06-25 14:53:41 -0300 | [diff] [blame] | 677 | * movsl can be slow when source and dest are not both 8-byte aligned |
| 678 | */ |
| 679 | #ifdef CONFIG_X86_INTEL_USERCOPY |
| 680 | extern struct movsl_mask { |
| 681 | int mask; |
| 682 | } ____cacheline_aligned_in_smp movsl_mask; |
| 683 | #endif |
| 684 | |
Glauber Costa | 22cac16 | 2008-06-25 14:56:53 -0300 | [diff] [blame] | 685 | #define ARCH_HAS_NOCACHE_UACCESS 1 |
| 686 | |
Thomas Gleixner | 96a388d | 2007-10-11 11:20:03 +0200 | [diff] [blame] | 687 | #ifdef CONFIG_X86_32 |
David Howells | a1ce392 | 2012-10-02 18:01:25 +0100 | [diff] [blame] | 688 | # include <asm/uaccess_32.h> |
Thomas Gleixner | 96a388d | 2007-10-11 11:20:03 +0200 | [diff] [blame] | 689 | #else |
David Howells | a1ce392 | 2012-10-02 18:01:25 +0100 | [diff] [blame] | 690 | # include <asm/uaccess_64.h> |
Thomas Gleixner | 96a388d | 2007-10-11 11:20:03 +0200 | [diff] [blame] | 691 | #endif |
Glauber Costa | ca23386 | 2008-06-13 14:39:25 -0300 | [diff] [blame] | 692 | |
Jan Beulich | 3df7b41 | 2013-10-21 09:43:57 +0100 | [diff] [blame] | 693 | unsigned long __must_check _copy_from_user(void *to, const void __user *from, |
| 694 | unsigned n); |
Jan Beulich | 7a3d9b0 | 2013-10-21 09:44:37 +0100 | [diff] [blame] | 695 | unsigned long __must_check _copy_to_user(void __user *to, const void *from, |
| 696 | unsigned n); |
Jan Beulich | 3df7b41 | 2013-10-21 09:43:57 +0100 | [diff] [blame] | 697 | |
Josh Poimboeuf | 0d025d2 | 2016-08-30 08:04:16 -0500 | [diff] [blame] | 698 | extern void __compiletime_error("usercopy buffer size is too small") |
| 699 | __bad_copy_user(void); |
Jan Beulich | 3df7b41 | 2013-10-21 09:43:57 +0100 | [diff] [blame] | 700 | |
Josh Poimboeuf | 0d025d2 | 2016-08-30 08:04:16 -0500 | [diff] [blame] | 701 | static inline void copy_user_overflow(int size, unsigned long count) |
Jan Beulich | 3df7b41 | 2013-10-21 09:43:57 +0100 | [diff] [blame] | 702 | { |
| 703 | WARN(1, "Buffer overflow detected (%d < %lu)!\n", size, count); |
| 704 | } |
| 705 | |
Kees Cook | e697100 | 2016-09-06 11:56:01 -0700 | [diff] [blame] | 706 | static __always_inline unsigned long __must_check |
Jan Beulich | 3df7b41 | 2013-10-21 09:43:57 +0100 | [diff] [blame] | 707 | copy_from_user(void *to, const void __user *from, unsigned long n) |
| 708 | { |
| 709 | int sz = __compiletime_object_size(to); |
| 710 | |
| 711 | might_fault(); |
| 712 | |
Andrey Ryabinin | 1771c6e | 2016-05-20 16:59:31 -0700 | [diff] [blame] | 713 | kasan_check_write(to, n); |
| 714 | |
Kees Cook | 5b710f3 | 2016-06-23 15:04:01 -0700 | [diff] [blame] | 715 | if (likely(sz < 0 || sz >= n)) { |
| 716 | check_object_size(to, n, false); |
Jan Beulich | 3df7b41 | 2013-10-21 09:43:57 +0100 | [diff] [blame] | 717 | n = _copy_from_user(to, from, n); |
Josh Poimboeuf | 0d025d2 | 2016-08-30 08:04:16 -0500 | [diff] [blame] | 718 | } else if (!__builtin_constant_p(n)) |
| 719 | copy_user_overflow(sz, n); |
Jan Beulich | 3df7b41 | 2013-10-21 09:43:57 +0100 | [diff] [blame] | 720 | else |
Josh Poimboeuf | 0d025d2 | 2016-08-30 08:04:16 -0500 | [diff] [blame] | 721 | __bad_copy_user(); |
Jan Beulich | 3df7b41 | 2013-10-21 09:43:57 +0100 | [diff] [blame] | 722 | |
| 723 | return n; |
| 724 | } |
| 725 | |
Kees Cook | e697100 | 2016-09-06 11:56:01 -0700 | [diff] [blame] | 726 | static __always_inline unsigned long __must_check |
Jan Beulich | 7a3d9b0 | 2013-10-21 09:44:37 +0100 | [diff] [blame] | 727 | copy_to_user(void __user *to, const void *from, unsigned long n) |
| 728 | { |
| 729 | int sz = __compiletime_object_size(from); |
| 730 | |
Andrey Ryabinin | 1771c6e | 2016-05-20 16:59:31 -0700 | [diff] [blame] | 731 | kasan_check_read(from, n); |
| 732 | |
Jan Beulich | 7a3d9b0 | 2013-10-21 09:44:37 +0100 | [diff] [blame] | 733 | might_fault(); |
| 734 | |
Kees Cook | 5b710f3 | 2016-06-23 15:04:01 -0700 | [diff] [blame] | 735 | if (likely(sz < 0 || sz >= n)) { |
| 736 | check_object_size(from, n, true); |
Jan Beulich | 7a3d9b0 | 2013-10-21 09:44:37 +0100 | [diff] [blame] | 737 | n = _copy_to_user(to, from, n); |
Josh Poimboeuf | 0d025d2 | 2016-08-30 08:04:16 -0500 | [diff] [blame] | 738 | } else if (!__builtin_constant_p(n)) |
| 739 | copy_user_overflow(sz, n); |
Jan Beulich | 7a3d9b0 | 2013-10-21 09:44:37 +0100 | [diff] [blame] | 740 | else |
Josh Poimboeuf | 0d025d2 | 2016-08-30 08:04:16 -0500 | [diff] [blame] | 741 | __bad_copy_user(); |
Jan Beulich | 7a3d9b0 | 2013-10-21 09:44:37 +0100 | [diff] [blame] | 742 | |
| 743 | return n; |
| 744 | } |
| 745 | |
Andi Kleen | 10013eb | 2015-10-22 15:07:20 -0700 | [diff] [blame] | 746 | /* |
| 747 | * We rely on the nested NMI work to allow atomic faults from the NMI path; the |
| 748 | * nested NMI paths are careful to preserve CR2. |
| 749 | * |
| 750 | * Caller must use pagefault_enable/disable, or run in interrupt context, |
| 751 | * and also do a uaccess_ok() check |
| 752 | */ |
| 753 | #define __copy_from_user_nmi __copy_from_user_inatomic |
| 754 | |
Linus Torvalds | 404a474 | 2016-01-21 13:02:41 -0800 | [diff] [blame] | 755 | /* |
Linus Torvalds | 5b24a7a | 2015-12-17 09:57:27 -0800 | [diff] [blame] | 756 | * The "unsafe" user accesses aren't really "unsafe", but the naming |
| 757 | * is a big fat warning: you have to not only do the access_ok() |
| 758 | * checking before using them, but you have to surround them with the |
| 759 | * user_access_begin/end() pair. |
| 760 | */ |
| 761 | #define user_access_begin() __uaccess_begin() |
| 762 | #define user_access_end() __uaccess_end() |
| 763 | |
Linus Torvalds | 1bd4403 | 2016-08-08 13:02:01 -0700 | [diff] [blame] | 764 | #define unsafe_put_user(x, ptr, err_label) \ |
| 765 | do { \ |
Linus Torvalds | 5b24a7a | 2015-12-17 09:57:27 -0800 | [diff] [blame] | 766 | int __pu_err; \ |
| 767 | __put_user_size((x), (ptr), sizeof(*(ptr)), __pu_err, -EFAULT); \ |
Linus Torvalds | 1bd4403 | 2016-08-08 13:02:01 -0700 | [diff] [blame] | 768 | if (unlikely(__pu_err)) goto err_label; \ |
| 769 | } while (0) |
Linus Torvalds | 5b24a7a | 2015-12-17 09:57:27 -0800 | [diff] [blame] | 770 | |
Linus Torvalds | 1bd4403 | 2016-08-08 13:02:01 -0700 | [diff] [blame] | 771 | #define unsafe_get_user(x, ptr, err_label) \ |
| 772 | do { \ |
Linus Torvalds | 5b24a7a | 2015-12-17 09:57:27 -0800 | [diff] [blame] | 773 | int __gu_err; \ |
| 774 | unsigned long __gu_val; \ |
| 775 | __get_user_size(__gu_val, (ptr), sizeof(*(ptr)), __gu_err, -EFAULT); \ |
| 776 | (x) = (__force __typeof__(*(ptr)))__gu_val; \ |
Linus Torvalds | 1bd4403 | 2016-08-08 13:02:01 -0700 | [diff] [blame] | 777 | if (unlikely(__gu_err)) goto err_label; \ |
| 778 | } while (0) |
Linus Torvalds | 5b24a7a | 2015-12-17 09:57:27 -0800 | [diff] [blame] | 779 | |
H. Peter Anvin | 1965aae | 2008-10-22 22:26:29 -0700 | [diff] [blame] | 780 | #endif /* _ASM_X86_UACCESS_H */ |
Nick Piggin | 8174c43 | 2008-07-25 19:45:24 -0700 | [diff] [blame] | 781 | |