Paolo Ciarrocchi | 3f50dbc | 2008-02-29 12:50:56 +0100 | [diff] [blame] | 1 | /* |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 2 | * User address space access functions. |
| 3 | * The non inlined parts of asm-i386/uaccess.h are here. |
| 4 | * |
| 5 | * Copyright 1997 Andi Kleen <ak@muc.de> |
| 6 | * Copyright 1997 Linus Torvalds |
| 7 | */ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 8 | #include <linux/mm.h> |
| 9 | #include <linux/highmem.h> |
| 10 | #include <linux/blkdev.h> |
| 11 | #include <linux/module.h> |
Andrew Morton | 3fcfab1 | 2006-10-19 23:28:16 -0700 | [diff] [blame] | 12 | #include <linux/backing-dev.h> |
Thomas Gleixner | b6a8b31 | 2007-04-02 14:25:31 +0200 | [diff] [blame] | 13 | #include <linux/interrupt.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 14 | #include <asm/uaccess.h> |
| 15 | #include <asm/mmx.h> |
H. Peter Anvin | 9c67512 | 2012-04-20 12:19:52 -0700 | [diff] [blame] | 16 | #include <asm/asm.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 17 | |
Thomas Petazzoni | 8bfcb39 | 2008-08-18 12:33:20 +0200 | [diff] [blame] | 18 | #ifdef CONFIG_X86_INTEL_USERCOPY |
| 19 | /* |
| 20 | * Alignment at which movsl is preferred for bulk memory copies. |
| 21 | */ |
| 22 | struct movsl_mask movsl_mask __read_mostly; |
| 23 | #endif |
| 24 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 25 | static inline int __movsl_is_ok(unsigned long a1, unsigned long a2, unsigned long n) |
| 26 | { |
| 27 | #ifdef CONFIG_X86_INTEL_USERCOPY |
| 28 | if (n >= 64 && ((a1 ^ a2) & movsl_mask.mask)) |
| 29 | return 0; |
| 30 | #endif |
| 31 | return 1; |
| 32 | } |
Paolo Ciarrocchi | 3f50dbc | 2008-02-29 12:50:56 +0100 | [diff] [blame] | 33 | #define movsl_is_ok(a1, a2, n) \ |
| 34 | __movsl_is_ok((unsigned long)(a1), (unsigned long)(a2), (n)) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 35 | |
| 36 | /* |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 37 | * Zero Userspace |
| 38 | */ |
| 39 | |
| 40 | #define __do_clear_user(addr,size) \ |
| 41 | do { \ |
| 42 | int __d0; \ |
Nick Piggin | 3ee1afa | 2008-09-10 13:37:17 +0200 | [diff] [blame] | 43 | might_fault(); \ |
Paolo Ciarrocchi | 3f50dbc | 2008-02-29 12:50:56 +0100 | [diff] [blame] | 44 | __asm__ __volatile__( \ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 45 | "0: rep; stosl\n" \ |
| 46 | " movl %2,%0\n" \ |
| 47 | "1: rep; stosb\n" \ |
| 48 | "2:\n" \ |
| 49 | ".section .fixup,\"ax\"\n" \ |
| 50 | "3: lea 0(%2,%0,4),%0\n" \ |
| 51 | " jmp 2b\n" \ |
| 52 | ".previous\n" \ |
H. Peter Anvin | 28777441 | 2008-02-04 16:47:57 +0100 | [diff] [blame] | 53 | _ASM_EXTABLE(0b,3b) \ |
| 54 | _ASM_EXTABLE(1b,2b) \ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 55 | : "=&c"(size), "=&D" (__d0) \ |
| 56 | : "r"(size & 3), "0"(size / 4), "1"(addr), "a"(0)); \ |
| 57 | } while (0) |
| 58 | |
| 59 | /** |
| 60 | * clear_user: - Zero a block of memory in user space. |
| 61 | * @to: Destination address, in user space. |
| 62 | * @n: Number of bytes to zero. |
| 63 | * |
| 64 | * Zero a block of memory in user space. |
| 65 | * |
| 66 | * Returns number of bytes that could not be cleared. |
| 67 | * On success, this will be zero. |
| 68 | */ |
| 69 | unsigned long |
| 70 | clear_user(void __user *to, unsigned long n) |
| 71 | { |
Ingo Molnar | 1d18ef4 | 2008-09-11 20:53:21 +0200 | [diff] [blame] | 72 | might_fault(); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 73 | if (access_ok(VERIFY_WRITE, to, n)) |
| 74 | __do_clear_user(to, n); |
| 75 | return n; |
| 76 | } |
Alexey Dobriyan | 129f694 | 2005-06-23 00:08:33 -0700 | [diff] [blame] | 77 | EXPORT_SYMBOL(clear_user); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 78 | |
| 79 | /** |
| 80 | * __clear_user: - Zero a block of memory in user space, with less checking. |
| 81 | * @to: Destination address, in user space. |
| 82 | * @n: Number of bytes to zero. |
| 83 | * |
| 84 | * Zero a block of memory in user space. Caller must check |
| 85 | * the specified block with access_ok() before calling this function. |
| 86 | * |
| 87 | * Returns number of bytes that could not be cleared. |
| 88 | * On success, this will be zero. |
| 89 | */ |
| 90 | unsigned long |
| 91 | __clear_user(void __user *to, unsigned long n) |
| 92 | { |
| 93 | __do_clear_user(to, n); |
| 94 | return n; |
| 95 | } |
Alexey Dobriyan | 129f694 | 2005-06-23 00:08:33 -0700 | [diff] [blame] | 96 | EXPORT_SYMBOL(__clear_user); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 97 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 98 | #ifdef CONFIG_X86_INTEL_USERCOPY |
| 99 | static unsigned long |
| 100 | __copy_user_intel(void __user *to, const void *from, unsigned long size) |
| 101 | { |
| 102 | int d0, d1; |
| 103 | __asm__ __volatile__( |
| 104 | " .align 2,0x90\n" |
| 105 | "1: movl 32(%4), %%eax\n" |
| 106 | " cmpl $67, %0\n" |
| 107 | " jbe 3f\n" |
| 108 | "2: movl 64(%4), %%eax\n" |
| 109 | " .align 2,0x90\n" |
| 110 | "3: movl 0(%4), %%eax\n" |
| 111 | "4: movl 4(%4), %%edx\n" |
| 112 | "5: movl %%eax, 0(%3)\n" |
| 113 | "6: movl %%edx, 4(%3)\n" |
| 114 | "7: movl 8(%4), %%eax\n" |
| 115 | "8: movl 12(%4),%%edx\n" |
| 116 | "9: movl %%eax, 8(%3)\n" |
| 117 | "10: movl %%edx, 12(%3)\n" |
| 118 | "11: movl 16(%4), %%eax\n" |
| 119 | "12: movl 20(%4), %%edx\n" |
| 120 | "13: movl %%eax, 16(%3)\n" |
| 121 | "14: movl %%edx, 20(%3)\n" |
| 122 | "15: movl 24(%4), %%eax\n" |
| 123 | "16: movl 28(%4), %%edx\n" |
| 124 | "17: movl %%eax, 24(%3)\n" |
| 125 | "18: movl %%edx, 28(%3)\n" |
| 126 | "19: movl 32(%4), %%eax\n" |
| 127 | "20: movl 36(%4), %%edx\n" |
| 128 | "21: movl %%eax, 32(%3)\n" |
| 129 | "22: movl %%edx, 36(%3)\n" |
| 130 | "23: movl 40(%4), %%eax\n" |
| 131 | "24: movl 44(%4), %%edx\n" |
| 132 | "25: movl %%eax, 40(%3)\n" |
| 133 | "26: movl %%edx, 44(%3)\n" |
| 134 | "27: movl 48(%4), %%eax\n" |
| 135 | "28: movl 52(%4), %%edx\n" |
| 136 | "29: movl %%eax, 48(%3)\n" |
| 137 | "30: movl %%edx, 52(%3)\n" |
| 138 | "31: movl 56(%4), %%eax\n" |
| 139 | "32: movl 60(%4), %%edx\n" |
| 140 | "33: movl %%eax, 56(%3)\n" |
| 141 | "34: movl %%edx, 60(%3)\n" |
| 142 | " addl $-64, %0\n" |
| 143 | " addl $64, %4\n" |
| 144 | " addl $64, %3\n" |
| 145 | " cmpl $63, %0\n" |
| 146 | " ja 1b\n" |
| 147 | "35: movl %0, %%eax\n" |
| 148 | " shrl $2, %0\n" |
| 149 | " andl $3, %%eax\n" |
| 150 | " cld\n" |
| 151 | "99: rep; movsl\n" |
| 152 | "36: movl %%eax, %0\n" |
| 153 | "37: rep; movsb\n" |
| 154 | "100:\n" |
| 155 | ".section .fixup,\"ax\"\n" |
| 156 | "101: lea 0(%%eax,%0,4),%0\n" |
| 157 | " jmp 100b\n" |
| 158 | ".previous\n" |
H. Peter Anvin | 9c67512 | 2012-04-20 12:19:52 -0700 | [diff] [blame] | 159 | _ASM_EXTABLE(1b,100b) |
| 160 | _ASM_EXTABLE(2b,100b) |
| 161 | _ASM_EXTABLE(3b,100b) |
| 162 | _ASM_EXTABLE(4b,100b) |
| 163 | _ASM_EXTABLE(5b,100b) |
| 164 | _ASM_EXTABLE(6b,100b) |
| 165 | _ASM_EXTABLE(7b,100b) |
| 166 | _ASM_EXTABLE(8b,100b) |
| 167 | _ASM_EXTABLE(9b,100b) |
| 168 | _ASM_EXTABLE(10b,100b) |
| 169 | _ASM_EXTABLE(11b,100b) |
| 170 | _ASM_EXTABLE(12b,100b) |
| 171 | _ASM_EXTABLE(13b,100b) |
| 172 | _ASM_EXTABLE(14b,100b) |
| 173 | _ASM_EXTABLE(15b,100b) |
| 174 | _ASM_EXTABLE(16b,100b) |
| 175 | _ASM_EXTABLE(17b,100b) |
| 176 | _ASM_EXTABLE(18b,100b) |
| 177 | _ASM_EXTABLE(19b,100b) |
| 178 | _ASM_EXTABLE(20b,100b) |
| 179 | _ASM_EXTABLE(21b,100b) |
| 180 | _ASM_EXTABLE(22b,100b) |
| 181 | _ASM_EXTABLE(23b,100b) |
| 182 | _ASM_EXTABLE(24b,100b) |
| 183 | _ASM_EXTABLE(25b,100b) |
| 184 | _ASM_EXTABLE(26b,100b) |
| 185 | _ASM_EXTABLE(27b,100b) |
| 186 | _ASM_EXTABLE(28b,100b) |
| 187 | _ASM_EXTABLE(29b,100b) |
| 188 | _ASM_EXTABLE(30b,100b) |
| 189 | _ASM_EXTABLE(31b,100b) |
| 190 | _ASM_EXTABLE(32b,100b) |
| 191 | _ASM_EXTABLE(33b,100b) |
| 192 | _ASM_EXTABLE(34b,100b) |
| 193 | _ASM_EXTABLE(35b,100b) |
| 194 | _ASM_EXTABLE(36b,100b) |
| 195 | _ASM_EXTABLE(37b,100b) |
| 196 | _ASM_EXTABLE(99b,101b) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 197 | : "=&c"(size), "=&D" (d0), "=&S" (d1) |
| 198 | : "1"(to), "2"(from), "0"(size) |
| 199 | : "eax", "edx", "memory"); |
| 200 | return size; |
| 201 | } |
| 202 | |
| 203 | static unsigned long |
| 204 | __copy_user_zeroing_intel(void *to, const void __user *from, unsigned long size) |
| 205 | { |
| 206 | int d0, d1; |
| 207 | __asm__ __volatile__( |
| 208 | " .align 2,0x90\n" |
| 209 | "0: movl 32(%4), %%eax\n" |
Paolo Ciarrocchi | 3f50dbc | 2008-02-29 12:50:56 +0100 | [diff] [blame] | 210 | " cmpl $67, %0\n" |
| 211 | " jbe 2f\n" |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 212 | "1: movl 64(%4), %%eax\n" |
Paolo Ciarrocchi | 3f50dbc | 2008-02-29 12:50:56 +0100 | [diff] [blame] | 213 | " .align 2,0x90\n" |
| 214 | "2: movl 0(%4), %%eax\n" |
| 215 | "21: movl 4(%4), %%edx\n" |
| 216 | " movl %%eax, 0(%3)\n" |
| 217 | " movl %%edx, 4(%3)\n" |
| 218 | "3: movl 8(%4), %%eax\n" |
| 219 | "31: movl 12(%4),%%edx\n" |
| 220 | " movl %%eax, 8(%3)\n" |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 221 | " movl %%edx, 12(%3)\n" |
| 222 | "4: movl 16(%4), %%eax\n" |
| 223 | "41: movl 20(%4), %%edx\n" |
| 224 | " movl %%eax, 16(%3)\n" |
| 225 | " movl %%edx, 20(%3)\n" |
| 226 | "10: movl 24(%4), %%eax\n" |
| 227 | "51: movl 28(%4), %%edx\n" |
| 228 | " movl %%eax, 24(%3)\n" |
| 229 | " movl %%edx, 28(%3)\n" |
| 230 | "11: movl 32(%4), %%eax\n" |
| 231 | "61: movl 36(%4), %%edx\n" |
| 232 | " movl %%eax, 32(%3)\n" |
| 233 | " movl %%edx, 36(%3)\n" |
| 234 | "12: movl 40(%4), %%eax\n" |
| 235 | "71: movl 44(%4), %%edx\n" |
| 236 | " movl %%eax, 40(%3)\n" |
| 237 | " movl %%edx, 44(%3)\n" |
| 238 | "13: movl 48(%4), %%eax\n" |
| 239 | "81: movl 52(%4), %%edx\n" |
| 240 | " movl %%eax, 48(%3)\n" |
| 241 | " movl %%edx, 52(%3)\n" |
| 242 | "14: movl 56(%4), %%eax\n" |
| 243 | "91: movl 60(%4), %%edx\n" |
| 244 | " movl %%eax, 56(%3)\n" |
| 245 | " movl %%edx, 60(%3)\n" |
Paolo Ciarrocchi | 3f50dbc | 2008-02-29 12:50:56 +0100 | [diff] [blame] | 246 | " addl $-64, %0\n" |
| 247 | " addl $64, %4\n" |
| 248 | " addl $64, %3\n" |
| 249 | " cmpl $63, %0\n" |
| 250 | " ja 0b\n" |
| 251 | "5: movl %0, %%eax\n" |
| 252 | " shrl $2, %0\n" |
| 253 | " andl $3, %%eax\n" |
| 254 | " cld\n" |
| 255 | "6: rep; movsl\n" |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 256 | " movl %%eax,%0\n" |
Paolo Ciarrocchi | 3f50dbc | 2008-02-29 12:50:56 +0100 | [diff] [blame] | 257 | "7: rep; movsb\n" |
| 258 | "8:\n" |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 259 | ".section .fixup,\"ax\"\n" |
Paolo Ciarrocchi | 3f50dbc | 2008-02-29 12:50:56 +0100 | [diff] [blame] | 260 | "9: lea 0(%%eax,%0,4),%0\n" |
| 261 | "16: pushl %0\n" |
| 262 | " pushl %%eax\n" |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 263 | " xorl %%eax,%%eax\n" |
Paolo Ciarrocchi | 3f50dbc | 2008-02-29 12:50:56 +0100 | [diff] [blame] | 264 | " rep; stosb\n" |
| 265 | " popl %%eax\n" |
| 266 | " popl %0\n" |
| 267 | " jmp 8b\n" |
| 268 | ".previous\n" |
H. Peter Anvin | 9c67512 | 2012-04-20 12:19:52 -0700 | [diff] [blame] | 269 | _ASM_EXTABLE(0b,16b) |
| 270 | _ASM_EXTABLE(1b,16b) |
| 271 | _ASM_EXTABLE(2b,16b) |
| 272 | _ASM_EXTABLE(21b,16b) |
| 273 | _ASM_EXTABLE(3b,16b) |
| 274 | _ASM_EXTABLE(31b,16b) |
| 275 | _ASM_EXTABLE(4b,16b) |
| 276 | _ASM_EXTABLE(41b,16b) |
| 277 | _ASM_EXTABLE(10b,16b) |
| 278 | _ASM_EXTABLE(51b,16b) |
| 279 | _ASM_EXTABLE(11b,16b) |
| 280 | _ASM_EXTABLE(61b,16b) |
| 281 | _ASM_EXTABLE(12b,16b) |
| 282 | _ASM_EXTABLE(71b,16b) |
| 283 | _ASM_EXTABLE(13b,16b) |
| 284 | _ASM_EXTABLE(81b,16b) |
| 285 | _ASM_EXTABLE(14b,16b) |
| 286 | _ASM_EXTABLE(91b,16b) |
| 287 | _ASM_EXTABLE(6b,9b) |
| 288 | _ASM_EXTABLE(7b,16b) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 289 | : "=&c"(size), "=&D" (d0), "=&S" (d1) |
| 290 | : "1"(to), "2"(from), "0"(size) |
| 291 | : "eax", "edx", "memory"); |
| 292 | return size; |
| 293 | } |
Hiro Yoshioka | c22ce14 | 2006-06-23 02:04:16 -0700 | [diff] [blame] | 294 | |
| 295 | /* |
| 296 | * Non Temporal Hint version of __copy_user_zeroing_intel. It is cache aware. |
| 297 | * hyoshiok@miraclelinux.com |
| 298 | */ |
| 299 | |
| 300 | static unsigned long __copy_user_zeroing_intel_nocache(void *to, |
| 301 | const void __user *from, unsigned long size) |
| 302 | { |
Paolo Ciarrocchi | 3f50dbc | 2008-02-29 12:50:56 +0100 | [diff] [blame] | 303 | int d0, d1; |
Hiro Yoshioka | c22ce14 | 2006-06-23 02:04:16 -0700 | [diff] [blame] | 304 | |
| 305 | __asm__ __volatile__( |
| 306 | " .align 2,0x90\n" |
| 307 | "0: movl 32(%4), %%eax\n" |
| 308 | " cmpl $67, %0\n" |
| 309 | " jbe 2f\n" |
| 310 | "1: movl 64(%4), %%eax\n" |
| 311 | " .align 2,0x90\n" |
| 312 | "2: movl 0(%4), %%eax\n" |
| 313 | "21: movl 4(%4), %%edx\n" |
| 314 | " movnti %%eax, 0(%3)\n" |
| 315 | " movnti %%edx, 4(%3)\n" |
| 316 | "3: movl 8(%4), %%eax\n" |
| 317 | "31: movl 12(%4),%%edx\n" |
| 318 | " movnti %%eax, 8(%3)\n" |
| 319 | " movnti %%edx, 12(%3)\n" |
| 320 | "4: movl 16(%4), %%eax\n" |
| 321 | "41: movl 20(%4), %%edx\n" |
| 322 | " movnti %%eax, 16(%3)\n" |
| 323 | " movnti %%edx, 20(%3)\n" |
| 324 | "10: movl 24(%4), %%eax\n" |
| 325 | "51: movl 28(%4), %%edx\n" |
| 326 | " movnti %%eax, 24(%3)\n" |
| 327 | " movnti %%edx, 28(%3)\n" |
| 328 | "11: movl 32(%4), %%eax\n" |
| 329 | "61: movl 36(%4), %%edx\n" |
| 330 | " movnti %%eax, 32(%3)\n" |
| 331 | " movnti %%edx, 36(%3)\n" |
| 332 | "12: movl 40(%4), %%eax\n" |
| 333 | "71: movl 44(%4), %%edx\n" |
| 334 | " movnti %%eax, 40(%3)\n" |
| 335 | " movnti %%edx, 44(%3)\n" |
| 336 | "13: movl 48(%4), %%eax\n" |
| 337 | "81: movl 52(%4), %%edx\n" |
| 338 | " movnti %%eax, 48(%3)\n" |
| 339 | " movnti %%edx, 52(%3)\n" |
| 340 | "14: movl 56(%4), %%eax\n" |
| 341 | "91: movl 60(%4), %%edx\n" |
| 342 | " movnti %%eax, 56(%3)\n" |
| 343 | " movnti %%edx, 60(%3)\n" |
| 344 | " addl $-64, %0\n" |
| 345 | " addl $64, %4\n" |
| 346 | " addl $64, %3\n" |
| 347 | " cmpl $63, %0\n" |
| 348 | " ja 0b\n" |
| 349 | " sfence \n" |
| 350 | "5: movl %0, %%eax\n" |
| 351 | " shrl $2, %0\n" |
| 352 | " andl $3, %%eax\n" |
| 353 | " cld\n" |
| 354 | "6: rep; movsl\n" |
| 355 | " movl %%eax,%0\n" |
| 356 | "7: rep; movsb\n" |
| 357 | "8:\n" |
| 358 | ".section .fixup,\"ax\"\n" |
| 359 | "9: lea 0(%%eax,%0,4),%0\n" |
| 360 | "16: pushl %0\n" |
| 361 | " pushl %%eax\n" |
| 362 | " xorl %%eax,%%eax\n" |
| 363 | " rep; stosb\n" |
| 364 | " popl %%eax\n" |
| 365 | " popl %0\n" |
| 366 | " jmp 8b\n" |
| 367 | ".previous\n" |
H. Peter Anvin | 9c67512 | 2012-04-20 12:19:52 -0700 | [diff] [blame] | 368 | _ASM_EXTABLE(0b,16b) |
| 369 | _ASM_EXTABLE(1b,16b) |
| 370 | _ASM_EXTABLE(2b,16b) |
| 371 | _ASM_EXTABLE(21b,16b) |
| 372 | _ASM_EXTABLE(3b,16b) |
| 373 | _ASM_EXTABLE(31b,16b) |
| 374 | _ASM_EXTABLE(4b,16b) |
| 375 | _ASM_EXTABLE(41b,16b) |
| 376 | _ASM_EXTABLE(10b,16b) |
| 377 | _ASM_EXTABLE(51b,16b) |
| 378 | _ASM_EXTABLE(11b,16b) |
| 379 | _ASM_EXTABLE(61b,16b) |
| 380 | _ASM_EXTABLE(12b,16b) |
| 381 | _ASM_EXTABLE(71b,16b) |
| 382 | _ASM_EXTABLE(13b,16b) |
| 383 | _ASM_EXTABLE(81b,16b) |
| 384 | _ASM_EXTABLE(14b,16b) |
| 385 | _ASM_EXTABLE(91b,16b) |
| 386 | _ASM_EXTABLE(6b,9b) |
| 387 | _ASM_EXTABLE(7b,16b) |
Hiro Yoshioka | c22ce14 | 2006-06-23 02:04:16 -0700 | [diff] [blame] | 388 | : "=&c"(size), "=&D" (d0), "=&S" (d1) |
| 389 | : "1"(to), "2"(from), "0"(size) |
| 390 | : "eax", "edx", "memory"); |
| 391 | return size; |
| 392 | } |
| 393 | |
NeilBrown | 7c12d81 | 2006-06-25 05:48:02 -0700 | [diff] [blame] | 394 | static unsigned long __copy_user_intel_nocache(void *to, |
| 395 | const void __user *from, unsigned long size) |
| 396 | { |
Paolo Ciarrocchi | 3f50dbc | 2008-02-29 12:50:56 +0100 | [diff] [blame] | 397 | int d0, d1; |
NeilBrown | 7c12d81 | 2006-06-25 05:48:02 -0700 | [diff] [blame] | 398 | |
| 399 | __asm__ __volatile__( |
| 400 | " .align 2,0x90\n" |
| 401 | "0: movl 32(%4), %%eax\n" |
| 402 | " cmpl $67, %0\n" |
| 403 | " jbe 2f\n" |
| 404 | "1: movl 64(%4), %%eax\n" |
| 405 | " .align 2,0x90\n" |
| 406 | "2: movl 0(%4), %%eax\n" |
| 407 | "21: movl 4(%4), %%edx\n" |
| 408 | " movnti %%eax, 0(%3)\n" |
| 409 | " movnti %%edx, 4(%3)\n" |
| 410 | "3: movl 8(%4), %%eax\n" |
| 411 | "31: movl 12(%4),%%edx\n" |
| 412 | " movnti %%eax, 8(%3)\n" |
| 413 | " movnti %%edx, 12(%3)\n" |
| 414 | "4: movl 16(%4), %%eax\n" |
| 415 | "41: movl 20(%4), %%edx\n" |
| 416 | " movnti %%eax, 16(%3)\n" |
| 417 | " movnti %%edx, 20(%3)\n" |
| 418 | "10: movl 24(%4), %%eax\n" |
| 419 | "51: movl 28(%4), %%edx\n" |
| 420 | " movnti %%eax, 24(%3)\n" |
| 421 | " movnti %%edx, 28(%3)\n" |
| 422 | "11: movl 32(%4), %%eax\n" |
| 423 | "61: movl 36(%4), %%edx\n" |
| 424 | " movnti %%eax, 32(%3)\n" |
| 425 | " movnti %%edx, 36(%3)\n" |
| 426 | "12: movl 40(%4), %%eax\n" |
| 427 | "71: movl 44(%4), %%edx\n" |
| 428 | " movnti %%eax, 40(%3)\n" |
| 429 | " movnti %%edx, 44(%3)\n" |
| 430 | "13: movl 48(%4), %%eax\n" |
| 431 | "81: movl 52(%4), %%edx\n" |
| 432 | " movnti %%eax, 48(%3)\n" |
| 433 | " movnti %%edx, 52(%3)\n" |
| 434 | "14: movl 56(%4), %%eax\n" |
| 435 | "91: movl 60(%4), %%edx\n" |
| 436 | " movnti %%eax, 56(%3)\n" |
| 437 | " movnti %%edx, 60(%3)\n" |
| 438 | " addl $-64, %0\n" |
| 439 | " addl $64, %4\n" |
| 440 | " addl $64, %3\n" |
| 441 | " cmpl $63, %0\n" |
| 442 | " ja 0b\n" |
| 443 | " sfence \n" |
| 444 | "5: movl %0, %%eax\n" |
| 445 | " shrl $2, %0\n" |
| 446 | " andl $3, %%eax\n" |
| 447 | " cld\n" |
| 448 | "6: rep; movsl\n" |
| 449 | " movl %%eax,%0\n" |
| 450 | "7: rep; movsb\n" |
| 451 | "8:\n" |
| 452 | ".section .fixup,\"ax\"\n" |
| 453 | "9: lea 0(%%eax,%0,4),%0\n" |
| 454 | "16: jmp 8b\n" |
| 455 | ".previous\n" |
H. Peter Anvin | 9c67512 | 2012-04-20 12:19:52 -0700 | [diff] [blame] | 456 | _ASM_EXTABLE(0b,16b) |
| 457 | _ASM_EXTABLE(1b,16b) |
| 458 | _ASM_EXTABLE(2b,16b) |
| 459 | _ASM_EXTABLE(21b,16b) |
| 460 | _ASM_EXTABLE(3b,16b) |
| 461 | _ASM_EXTABLE(31b,16b) |
| 462 | _ASM_EXTABLE(4b,16b) |
| 463 | _ASM_EXTABLE(41b,16b) |
| 464 | _ASM_EXTABLE(10b,16b) |
| 465 | _ASM_EXTABLE(51b,16b) |
| 466 | _ASM_EXTABLE(11b,16b) |
| 467 | _ASM_EXTABLE(61b,16b) |
| 468 | _ASM_EXTABLE(12b,16b) |
| 469 | _ASM_EXTABLE(71b,16b) |
| 470 | _ASM_EXTABLE(13b,16b) |
| 471 | _ASM_EXTABLE(81b,16b) |
| 472 | _ASM_EXTABLE(14b,16b) |
| 473 | _ASM_EXTABLE(91b,16b) |
| 474 | _ASM_EXTABLE(6b,9b) |
| 475 | _ASM_EXTABLE(7b,16b) |
NeilBrown | 7c12d81 | 2006-06-25 05:48:02 -0700 | [diff] [blame] | 476 | : "=&c"(size), "=&D" (d0), "=&S" (d1) |
| 477 | : "1"(to), "2"(from), "0"(size) |
| 478 | : "eax", "edx", "memory"); |
| 479 | return size; |
| 480 | } |
| 481 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 482 | #else |
Hiro Yoshioka | c22ce14 | 2006-06-23 02:04:16 -0700 | [diff] [blame] | 483 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 484 | /* |
| 485 | * Leave these declared but undefined. They should not be any references to |
| 486 | * them |
| 487 | */ |
Hiro Yoshioka | c22ce14 | 2006-06-23 02:04:16 -0700 | [diff] [blame] | 488 | unsigned long __copy_user_zeroing_intel(void *to, const void __user *from, |
| 489 | unsigned long size); |
| 490 | unsigned long __copy_user_intel(void __user *to, const void *from, |
| 491 | unsigned long size); |
| 492 | unsigned long __copy_user_zeroing_intel_nocache(void *to, |
| 493 | const void __user *from, unsigned long size); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 494 | #endif /* CONFIG_X86_INTEL_USERCOPY */ |
| 495 | |
| 496 | /* Generic arbitrary sized copy. */ |
Paolo Ciarrocchi | 3f50dbc | 2008-02-29 12:50:56 +0100 | [diff] [blame] | 497 | #define __copy_user(to, from, size) \ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 498 | do { \ |
| 499 | int __d0, __d1, __d2; \ |
| 500 | __asm__ __volatile__( \ |
| 501 | " cmp $7,%0\n" \ |
| 502 | " jbe 1f\n" \ |
| 503 | " movl %1,%0\n" \ |
| 504 | " negl %0\n" \ |
| 505 | " andl $7,%0\n" \ |
| 506 | " subl %0,%3\n" \ |
| 507 | "4: rep; movsb\n" \ |
| 508 | " movl %3,%0\n" \ |
| 509 | " shrl $2,%0\n" \ |
| 510 | " andl $3,%3\n" \ |
| 511 | " .align 2,0x90\n" \ |
| 512 | "0: rep; movsl\n" \ |
| 513 | " movl %3,%0\n" \ |
| 514 | "1: rep; movsb\n" \ |
| 515 | "2:\n" \ |
| 516 | ".section .fixup,\"ax\"\n" \ |
| 517 | "5: addl %3,%0\n" \ |
| 518 | " jmp 2b\n" \ |
| 519 | "3: lea 0(%3,%0,4),%0\n" \ |
| 520 | " jmp 2b\n" \ |
| 521 | ".previous\n" \ |
H. Peter Anvin | 9c67512 | 2012-04-20 12:19:52 -0700 | [diff] [blame] | 522 | _ASM_EXTABLE(4b,5b) \ |
| 523 | _ASM_EXTABLE(0b,3b) \ |
| 524 | _ASM_EXTABLE(1b,2b) \ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 525 | : "=&c"(size), "=&D" (__d0), "=&S" (__d1), "=r"(__d2) \ |
| 526 | : "3"(size), "0"(size), "1"(to), "2"(from) \ |
| 527 | : "memory"); \ |
| 528 | } while (0) |
| 529 | |
Paolo Ciarrocchi | 3f50dbc | 2008-02-29 12:50:56 +0100 | [diff] [blame] | 530 | #define __copy_user_zeroing(to, from, size) \ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 531 | do { \ |
| 532 | int __d0, __d1, __d2; \ |
| 533 | __asm__ __volatile__( \ |
| 534 | " cmp $7,%0\n" \ |
| 535 | " jbe 1f\n" \ |
| 536 | " movl %1,%0\n" \ |
| 537 | " negl %0\n" \ |
| 538 | " andl $7,%0\n" \ |
| 539 | " subl %0,%3\n" \ |
| 540 | "4: rep; movsb\n" \ |
| 541 | " movl %3,%0\n" \ |
| 542 | " shrl $2,%0\n" \ |
| 543 | " andl $3,%3\n" \ |
| 544 | " .align 2,0x90\n" \ |
| 545 | "0: rep; movsl\n" \ |
| 546 | " movl %3,%0\n" \ |
| 547 | "1: rep; movsb\n" \ |
| 548 | "2:\n" \ |
| 549 | ".section .fixup,\"ax\"\n" \ |
| 550 | "5: addl %3,%0\n" \ |
| 551 | " jmp 6f\n" \ |
| 552 | "3: lea 0(%3,%0,4),%0\n" \ |
| 553 | "6: pushl %0\n" \ |
| 554 | " pushl %%eax\n" \ |
| 555 | " xorl %%eax,%%eax\n" \ |
| 556 | " rep; stosb\n" \ |
| 557 | " popl %%eax\n" \ |
| 558 | " popl %0\n" \ |
| 559 | " jmp 2b\n" \ |
| 560 | ".previous\n" \ |
H. Peter Anvin | 9c67512 | 2012-04-20 12:19:52 -0700 | [diff] [blame] | 561 | _ASM_EXTABLE(4b,5b) \ |
| 562 | _ASM_EXTABLE(0b,3b) \ |
| 563 | _ASM_EXTABLE(1b,6b) \ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 564 | : "=&c"(size), "=&D" (__d0), "=&S" (__d1), "=r"(__d2) \ |
| 565 | : "3"(size), "0"(size), "1"(to), "2"(from) \ |
| 566 | : "memory"); \ |
| 567 | } while (0) |
| 568 | |
Hiro Yoshioka | c22ce14 | 2006-06-23 02:04:16 -0700 | [diff] [blame] | 569 | unsigned long __copy_to_user_ll(void __user *to, const void *from, |
| 570 | unsigned long n) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 571 | { |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 572 | #ifndef CONFIG_X86_WP_WORKS_OK |
| 573 | if (unlikely(boot_cpu_data.wp_works_ok == 0) && |
Paolo Ciarrocchi | 3f50dbc | 2008-02-29 12:50:56 +0100 | [diff] [blame] | 574 | ((unsigned long)to) < TASK_SIZE) { |
Thomas Gleixner | b6a8b31 | 2007-04-02 14:25:31 +0200 | [diff] [blame] | 575 | /* |
| 576 | * When we are in an atomic section (see |
| 577 | * mm/filemap.c:file_read_actor), return the full |
| 578 | * length to take the slow path. |
| 579 | */ |
| 580 | if (in_atomic()) |
| 581 | return n; |
| 582 | |
Paolo Ciarrocchi | 3f50dbc | 2008-02-29 12:50:56 +0100 | [diff] [blame] | 583 | /* |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 584 | * CPU does not honor the WP bit when writing |
| 585 | * from supervisory mode, and due to preemption or SMP, |
| 586 | * the page tables can change at any time. |
| 587 | * Do it manually. Manfred <manfred@colorfullife.com> |
| 588 | */ |
| 589 | while (n) { |
Paolo Ciarrocchi | 3f50dbc | 2008-02-29 12:50:56 +0100 | [diff] [blame] | 590 | unsigned long offset = ((unsigned long)to)%PAGE_SIZE; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 591 | unsigned long len = PAGE_SIZE - offset; |
| 592 | int retval; |
| 593 | struct page *pg; |
| 594 | void *maddr; |
Paolo Ciarrocchi | 3f50dbc | 2008-02-29 12:50:56 +0100 | [diff] [blame] | 595 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 596 | if (len > n) |
| 597 | len = n; |
| 598 | |
| 599 | survive: |
| 600 | down_read(¤t->mm->mmap_sem); |
| 601 | retval = get_user_pages(current, current->mm, |
Paolo Ciarrocchi | 3f50dbc | 2008-02-29 12:50:56 +0100 | [diff] [blame] | 602 | (unsigned long)to, 1, 1, 0, &pg, NULL); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 603 | |
Serge E. Hallyn | b460cbc | 2007-10-18 23:39:52 -0700 | [diff] [blame] | 604 | if (retval == -ENOMEM && is_global_init(current)) { |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 605 | up_read(¤t->mm->mmap_sem); |
Jens Axboe | 8aa7e84 | 2009-07-09 14:52:32 +0200 | [diff] [blame] | 606 | congestion_wait(BLK_RW_ASYNC, HZ/50); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 607 | goto survive; |
| 608 | } |
| 609 | |
| 610 | if (retval != 1) { |
| 611 | up_read(¤t->mm->mmap_sem); |
Paolo Ciarrocchi | 3f50dbc | 2008-02-29 12:50:56 +0100 | [diff] [blame] | 612 | break; |
| 613 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 614 | |
Cong Wang | 8fd75e1 | 2011-11-25 23:14:17 +0800 | [diff] [blame] | 615 | maddr = kmap_atomic(pg); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 616 | memcpy(maddr + offset, from, len); |
Cong Wang | 8fd75e1 | 2011-11-25 23:14:17 +0800 | [diff] [blame] | 617 | kunmap_atomic(maddr); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 618 | set_page_dirty_lock(pg); |
| 619 | put_page(pg); |
| 620 | up_read(¤t->mm->mmap_sem); |
| 621 | |
| 622 | from += len; |
| 623 | to += len; |
| 624 | n -= len; |
| 625 | } |
| 626 | return n; |
| 627 | } |
| 628 | #endif |
| 629 | if (movsl_is_ok(to, from, n)) |
| 630 | __copy_user(to, from, n); |
| 631 | else |
| 632 | n = __copy_user_intel(to, from, n); |
| 633 | return n; |
| 634 | } |
Alexey Dobriyan | 129f694 | 2005-06-23 00:08:33 -0700 | [diff] [blame] | 635 | EXPORT_SYMBOL(__copy_to_user_ll); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 636 | |
Hiro Yoshioka | c22ce14 | 2006-06-23 02:04:16 -0700 | [diff] [blame] | 637 | unsigned long __copy_from_user_ll(void *to, const void __user *from, |
| 638 | unsigned long n) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 639 | { |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 640 | if (movsl_is_ok(to, from, n)) |
| 641 | __copy_user_zeroing(to, from, n); |
| 642 | else |
| 643 | n = __copy_user_zeroing_intel(to, from, n); |
| 644 | return n; |
| 645 | } |
Alexey Dobriyan | 129f694 | 2005-06-23 00:08:33 -0700 | [diff] [blame] | 646 | EXPORT_SYMBOL(__copy_from_user_ll); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 647 | |
NeilBrown | 7c12d81 | 2006-06-25 05:48:02 -0700 | [diff] [blame] | 648 | unsigned long __copy_from_user_ll_nozero(void *to, const void __user *from, |
| 649 | unsigned long n) |
| 650 | { |
NeilBrown | 7c12d81 | 2006-06-25 05:48:02 -0700 | [diff] [blame] | 651 | if (movsl_is_ok(to, from, n)) |
| 652 | __copy_user(to, from, n); |
| 653 | else |
| 654 | n = __copy_user_intel((void __user *)to, |
| 655 | (const void *)from, n); |
| 656 | return n; |
| 657 | } |
| 658 | EXPORT_SYMBOL(__copy_from_user_ll_nozero); |
| 659 | |
Hiro Yoshioka | c22ce14 | 2006-06-23 02:04:16 -0700 | [diff] [blame] | 660 | unsigned long __copy_from_user_ll_nocache(void *to, const void __user *from, |
| 661 | unsigned long n) |
| 662 | { |
Hiro Yoshioka | c22ce14 | 2006-06-23 02:04:16 -0700 | [diff] [blame] | 663 | #ifdef CONFIG_X86_INTEL_USERCOPY |
Paolo Ciarrocchi | 3f50dbc | 2008-02-29 12:50:56 +0100 | [diff] [blame] | 664 | if (n > 64 && cpu_has_xmm2) |
| 665 | n = __copy_user_zeroing_intel_nocache(to, from, n); |
Hiro Yoshioka | c22ce14 | 2006-06-23 02:04:16 -0700 | [diff] [blame] | 666 | else |
| 667 | __copy_user_zeroing(to, from, n); |
| 668 | #else |
Paolo Ciarrocchi | 3f50dbc | 2008-02-29 12:50:56 +0100 | [diff] [blame] | 669 | __copy_user_zeroing(to, from, n); |
Hiro Yoshioka | c22ce14 | 2006-06-23 02:04:16 -0700 | [diff] [blame] | 670 | #endif |
| 671 | return n; |
| 672 | } |
Andrew Morton | 914c826 | 2008-01-30 23:27:57 +0100 | [diff] [blame] | 673 | EXPORT_SYMBOL(__copy_from_user_ll_nocache); |
Hiro Yoshioka | c22ce14 | 2006-06-23 02:04:16 -0700 | [diff] [blame] | 674 | |
NeilBrown | 7c12d81 | 2006-06-25 05:48:02 -0700 | [diff] [blame] | 675 | unsigned long __copy_from_user_ll_nocache_nozero(void *to, const void __user *from, |
| 676 | unsigned long n) |
| 677 | { |
NeilBrown | 7c12d81 | 2006-06-25 05:48:02 -0700 | [diff] [blame] | 678 | #ifdef CONFIG_X86_INTEL_USERCOPY |
Paolo Ciarrocchi | 3f50dbc | 2008-02-29 12:50:56 +0100 | [diff] [blame] | 679 | if (n > 64 && cpu_has_xmm2) |
| 680 | n = __copy_user_intel_nocache(to, from, n); |
NeilBrown | 7c12d81 | 2006-06-25 05:48:02 -0700 | [diff] [blame] | 681 | else |
| 682 | __copy_user(to, from, n); |
| 683 | #else |
Paolo Ciarrocchi | 3f50dbc | 2008-02-29 12:50:56 +0100 | [diff] [blame] | 684 | __copy_user(to, from, n); |
NeilBrown | 7c12d81 | 2006-06-25 05:48:02 -0700 | [diff] [blame] | 685 | #endif |
| 686 | return n; |
| 687 | } |
Andrew Morton | 914c826 | 2008-01-30 23:27:57 +0100 | [diff] [blame] | 688 | EXPORT_SYMBOL(__copy_from_user_ll_nocache_nozero); |
NeilBrown | 7c12d81 | 2006-06-25 05:48:02 -0700 | [diff] [blame] | 689 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 690 | /** |
| 691 | * copy_to_user: - Copy a block of data into user space. |
| 692 | * @to: Destination address, in user space. |
| 693 | * @from: Source address, in kernel space. |
| 694 | * @n: Number of bytes to copy. |
| 695 | * |
| 696 | * Context: User context only. This function may sleep. |
| 697 | * |
| 698 | * Copy data from kernel space to user space. |
| 699 | * |
| 700 | * Returns number of bytes that could not be copied. |
| 701 | * On success, this will be zero. |
| 702 | */ |
| 703 | unsigned long |
| 704 | copy_to_user(void __user *to, const void *from, unsigned long n) |
| 705 | { |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 706 | if (access_ok(VERIFY_WRITE, to, n)) |
| 707 | n = __copy_to_user(to, from, n); |
| 708 | return n; |
| 709 | } |
| 710 | EXPORT_SYMBOL(copy_to_user); |
| 711 | |
| 712 | /** |
| 713 | * copy_from_user: - Copy a block of data from user space. |
| 714 | * @to: Destination address, in kernel space. |
| 715 | * @from: Source address, in user space. |
| 716 | * @n: Number of bytes to copy. |
| 717 | * |
| 718 | * Context: User context only. This function may sleep. |
| 719 | * |
| 720 | * Copy data from user space to kernel space. |
| 721 | * |
| 722 | * Returns number of bytes that could not be copied. |
| 723 | * On success, this will be zero. |
| 724 | * |
| 725 | * If some data could not be copied, this function will pad the copied |
| 726 | * data to the requested size using zero bytes. |
| 727 | */ |
| 728 | unsigned long |
Arjan van de Ven | 9f0cf4a | 2009-09-26 14:33:01 +0200 | [diff] [blame] | 729 | _copy_from_user(void *to, const void __user *from, unsigned long n) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 730 | { |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 731 | if (access_ok(VERIFY_READ, from, n)) |
| 732 | n = __copy_from_user(to, from, n); |
| 733 | else |
| 734 | memset(to, 0, n); |
| 735 | return n; |
| 736 | } |
Arjan van de Ven | 9f0cf4a | 2009-09-26 14:33:01 +0200 | [diff] [blame] | 737 | EXPORT_SYMBOL(_copy_from_user); |
Arjan van de Ven | 4a31276 | 2009-09-30 13:05:23 +0200 | [diff] [blame] | 738 | |
| 739 | void copy_from_user_overflow(void) |
| 740 | { |
| 741 | WARN(1, "Buffer overflow detected!\n"); |
| 742 | } |
| 743 | EXPORT_SYMBOL(copy_from_user_overflow); |