Gerald Schaefer | d02765d | 2006-09-20 15:59:42 +0200 | [diff] [blame] | 1 | /* |
| 2 | * arch/s390/lib/uaccess_std.c |
| 3 | * |
| 4 | * Standard user space access functions based on mvcp/mvcs and doing |
| 5 | * interesting things in the secondary space mode. |
| 6 | * |
| 7 | * Copyright (C) IBM Corp. 2006 |
| 8 | * Author(s): Martin Schwidefsky (schwidefsky@de.ibm.com), |
| 9 | * Gerald Schaefer (gerald.schaefer@de.ibm.com) |
| 10 | */ |
| 11 | |
| 12 | #include <linux/errno.h> |
| 13 | #include <linux/mm.h> |
Peter Zijlstra | a866374 | 2006-12-06 20:32:20 -0800 | [diff] [blame] | 14 | #include <linux/uaccess.h> |
Gerald Schaefer | d02765d | 2006-09-20 15:59:42 +0200 | [diff] [blame] | 15 | #include <asm/futex.h> |
Heiko Carstens | 2b67fc4 | 2007-02-05 21:16:47 +0100 | [diff] [blame] | 16 | #include "uaccess.h" |
Gerald Schaefer | d02765d | 2006-09-20 15:59:42 +0200 | [diff] [blame] | 17 | |
| 18 | #ifndef __s390x__ |
| 19 | #define AHI "ahi" |
| 20 | #define ALR "alr" |
| 21 | #define CLR "clr" |
| 22 | #define LHI "lhi" |
| 23 | #define SLR "slr" |
| 24 | #else |
| 25 | #define AHI "aghi" |
| 26 | #define ALR "algr" |
| 27 | #define CLR "clgr" |
| 28 | #define LHI "lghi" |
| 29 | #define SLR "slgr" |
| 30 | #endif |
| 31 | |
| 32 | size_t copy_from_user_std(size_t size, const void __user *ptr, void *x) |
| 33 | { |
| 34 | unsigned long tmp1, tmp2; |
| 35 | |
| 36 | tmp1 = -256UL; |
| 37 | asm volatile( |
| 38 | "0: mvcp 0(%0,%2),0(%1),%3\n" |
Gerald Schaefer | af9d2ff | 2009-10-06 10:34:14 +0200 | [diff] [blame] | 39 | "10:jz 8f\n" |
Gerald Schaefer | d02765d | 2006-09-20 15:59:42 +0200 | [diff] [blame] | 40 | "1:"ALR" %0,%3\n" |
| 41 | " la %1,256(%1)\n" |
| 42 | " la %2,256(%2)\n" |
| 43 | "2: mvcp 0(%0,%2),0(%1),%3\n" |
Gerald Schaefer | af9d2ff | 2009-10-06 10:34:14 +0200 | [diff] [blame] | 44 | "11:jnz 1b\n" |
Martin Schwidefsky | 52149ba | 2006-09-28 16:56:03 +0200 | [diff] [blame] | 45 | " j 8f\n" |
Gerald Schaefer | d02765d | 2006-09-20 15:59:42 +0200 | [diff] [blame] | 46 | "3: la %4,255(%1)\n" /* %4 = ptr + 255 */ |
| 47 | " "LHI" %3,-4096\n" |
| 48 | " nr %4,%3\n" /* %4 = (ptr + 255) & -4096 */ |
| 49 | " "SLR" %4,%1\n" |
| 50 | " "CLR" %0,%4\n" /* copy crosses next page boundary? */ |
Martin Schwidefsky | 52149ba | 2006-09-28 16:56:03 +0200 | [diff] [blame] | 51 | " jnh 5f\n" |
Gerald Schaefer | d02765d | 2006-09-20 15:59:42 +0200 | [diff] [blame] | 52 | "4: mvcp 0(%4,%2),0(%1),%3\n" |
Gerald Schaefer | af9d2ff | 2009-10-06 10:34:14 +0200 | [diff] [blame] | 53 | "12:"SLR" %0,%4\n" |
Martin Schwidefsky | 52149ba | 2006-09-28 16:56:03 +0200 | [diff] [blame] | 54 | " "ALR" %2,%4\n" |
| 55 | "5:"LHI" %4,-1\n" |
| 56 | " "ALR" %4,%0\n" /* copy remaining size, subtract 1 */ |
| 57 | " bras %3,7f\n" /* memset loop */ |
| 58 | " xc 0(1,%2),0(%2)\n" |
| 59 | "6: xc 0(256,%2),0(%2)\n" |
| 60 | " la %2,256(%2)\n" |
| 61 | "7:"AHI" %4,-256\n" |
| 62 | " jnm 6b\n" |
| 63 | " ex %4,0(%3)\n" |
| 64 | " j 9f\n" |
| 65 | "8:"SLR" %0,%0\n" |
| 66 | "9: \n" |
| 67 | EX_TABLE(0b,3b) EX_TABLE(2b,3b) EX_TABLE(4b,5b) |
Gerald Schaefer | af9d2ff | 2009-10-06 10:34:14 +0200 | [diff] [blame] | 68 | EX_TABLE(10b,3b) EX_TABLE(11b,3b) EX_TABLE(12b,5b) |
Gerald Schaefer | d02765d | 2006-09-20 15:59:42 +0200 | [diff] [blame] | 69 | : "+a" (size), "+a" (ptr), "+a" (x), "+a" (tmp1), "=a" (tmp2) |
| 70 | : : "cc", "memory"); |
| 71 | return size; |
| 72 | } |
| 73 | |
Heiko Carstens | 2b67fc4 | 2007-02-05 21:16:47 +0100 | [diff] [blame] | 74 | static size_t copy_from_user_std_check(size_t size, const void __user *ptr, |
| 75 | void *x) |
Gerald Schaefer | d02765d | 2006-09-20 15:59:42 +0200 | [diff] [blame] | 76 | { |
Gerald Schaefer | 59f35d5 | 2006-12-04 15:40:45 +0100 | [diff] [blame] | 77 | if (size <= 1024) |
| 78 | return copy_from_user_std(size, ptr, x); |
| 79 | return copy_from_user_pt(size, ptr, x); |
Gerald Schaefer | d02765d | 2006-09-20 15:59:42 +0200 | [diff] [blame] | 80 | } |
| 81 | |
| 82 | size_t copy_to_user_std(size_t size, void __user *ptr, const void *x) |
| 83 | { |
| 84 | unsigned long tmp1, tmp2; |
| 85 | |
| 86 | tmp1 = -256UL; |
| 87 | asm volatile( |
| 88 | "0: mvcs 0(%0,%1),0(%2),%3\n" |
Gerald Schaefer | af9d2ff | 2009-10-06 10:34:14 +0200 | [diff] [blame] | 89 | "7: jz 5f\n" |
Gerald Schaefer | d02765d | 2006-09-20 15:59:42 +0200 | [diff] [blame] | 90 | "1:"ALR" %0,%3\n" |
| 91 | " la %1,256(%1)\n" |
| 92 | " la %2,256(%2)\n" |
| 93 | "2: mvcs 0(%0,%1),0(%2),%3\n" |
Gerald Schaefer | af9d2ff | 2009-10-06 10:34:14 +0200 | [diff] [blame] | 94 | "8: jnz 1b\n" |
Gerald Schaefer | d02765d | 2006-09-20 15:59:42 +0200 | [diff] [blame] | 95 | " j 5f\n" |
| 96 | "3: la %4,255(%1)\n" /* %4 = ptr + 255 */ |
| 97 | " "LHI" %3,-4096\n" |
| 98 | " nr %4,%3\n" /* %4 = (ptr + 255) & -4096 */ |
| 99 | " "SLR" %4,%1\n" |
| 100 | " "CLR" %0,%4\n" /* copy crosses next page boundary? */ |
| 101 | " jnh 6f\n" |
| 102 | "4: mvcs 0(%4,%1),0(%2),%3\n" |
Gerald Schaefer | af9d2ff | 2009-10-06 10:34:14 +0200 | [diff] [blame] | 103 | "9:"SLR" %0,%4\n" |
Gerald Schaefer | d02765d | 2006-09-20 15:59:42 +0200 | [diff] [blame] | 104 | " j 6f\n" |
| 105 | "5:"SLR" %0,%0\n" |
| 106 | "6: \n" |
| 107 | EX_TABLE(0b,3b) EX_TABLE(2b,3b) EX_TABLE(4b,6b) |
Gerald Schaefer | af9d2ff | 2009-10-06 10:34:14 +0200 | [diff] [blame] | 108 | EX_TABLE(7b,3b) EX_TABLE(8b,3b) EX_TABLE(9b,6b) |
Gerald Schaefer | d02765d | 2006-09-20 15:59:42 +0200 | [diff] [blame] | 109 | : "+a" (size), "+a" (ptr), "+a" (x), "+a" (tmp1), "=a" (tmp2) |
| 110 | : : "cc", "memory"); |
| 111 | return size; |
| 112 | } |
| 113 | |
Heiko Carstens | 2b67fc4 | 2007-02-05 21:16:47 +0100 | [diff] [blame] | 114 | static size_t copy_to_user_std_check(size_t size, void __user *ptr, |
| 115 | const void *x) |
Gerald Schaefer | d02765d | 2006-09-20 15:59:42 +0200 | [diff] [blame] | 116 | { |
Gerald Schaefer | 59f35d5 | 2006-12-04 15:40:45 +0100 | [diff] [blame] | 117 | if (size <= 1024) |
| 118 | return copy_to_user_std(size, ptr, x); |
| 119 | return copy_to_user_pt(size, ptr, x); |
Gerald Schaefer | d02765d | 2006-09-20 15:59:42 +0200 | [diff] [blame] | 120 | } |
| 121 | |
Heiko Carstens | 2b67fc4 | 2007-02-05 21:16:47 +0100 | [diff] [blame] | 122 | static size_t copy_in_user_std(size_t size, void __user *to, |
| 123 | const void __user *from) |
Gerald Schaefer | d02765d | 2006-09-20 15:59:42 +0200 | [diff] [blame] | 124 | { |
| 125 | unsigned long tmp1; |
| 126 | |
| 127 | asm volatile( |
Martin Schwidefsky | e4d8269 | 2011-01-31 11:30:05 +0100 | [diff] [blame] | 128 | " sacf 256\n" |
Gerald Schaefer | d02765d | 2006-09-20 15:59:42 +0200 | [diff] [blame] | 129 | " "AHI" %0,-1\n" |
| 130 | " jo 5f\n" |
Gerald Schaefer | d02765d | 2006-09-20 15:59:42 +0200 | [diff] [blame] | 131 | " bras %3,3f\n" |
| 132 | "0:"AHI" %0,257\n" |
| 133 | "1: mvc 0(1,%1),0(%2)\n" |
| 134 | " la %1,1(%1)\n" |
| 135 | " la %2,1(%2)\n" |
| 136 | " "AHI" %0,-1\n" |
| 137 | " jnz 1b\n" |
| 138 | " j 5f\n" |
| 139 | "2: mvc 0(256,%1),0(%2)\n" |
| 140 | " la %1,256(%1)\n" |
| 141 | " la %2,256(%2)\n" |
| 142 | "3:"AHI" %0,-256\n" |
| 143 | " jnm 2b\n" |
| 144 | "4: ex %0,1b-0b(%3)\n" |
Gerald Schaefer | d02765d | 2006-09-20 15:59:42 +0200 | [diff] [blame] | 145 | "5: "SLR" %0,%0\n" |
Martin Schwidefsky | e4d8269 | 2011-01-31 11:30:05 +0100 | [diff] [blame] | 146 | "6: sacf 0\n" |
Gerald Schaefer | d02765d | 2006-09-20 15:59:42 +0200 | [diff] [blame] | 147 | EX_TABLE(1b,6b) EX_TABLE(2b,0b) EX_TABLE(4b,0b) |
| 148 | : "+a" (size), "+a" (to), "+a" (from), "=a" (tmp1) |
| 149 | : : "cc", "memory"); |
| 150 | return size; |
| 151 | } |
| 152 | |
Heiko Carstens | 2b67fc4 | 2007-02-05 21:16:47 +0100 | [diff] [blame] | 153 | static size_t clear_user_std(size_t size, void __user *to) |
Gerald Schaefer | d02765d | 2006-09-20 15:59:42 +0200 | [diff] [blame] | 154 | { |
| 155 | unsigned long tmp1, tmp2; |
| 156 | |
| 157 | asm volatile( |
Martin Schwidefsky | e4d8269 | 2011-01-31 11:30:05 +0100 | [diff] [blame] | 158 | " sacf 256\n" |
Gerald Schaefer | d02765d | 2006-09-20 15:59:42 +0200 | [diff] [blame] | 159 | " "AHI" %0,-1\n" |
| 160 | " jo 5f\n" |
Gerald Schaefer | d02765d | 2006-09-20 15:59:42 +0200 | [diff] [blame] | 161 | " bras %3,3f\n" |
| 162 | " xc 0(1,%1),0(%1)\n" |
| 163 | "0:"AHI" %0,257\n" |
| 164 | " la %2,255(%1)\n" /* %2 = ptr + 255 */ |
| 165 | " srl %2,12\n" |
| 166 | " sll %2,12\n" /* %2 = (ptr + 255) & -4096 */ |
| 167 | " "SLR" %2,%1\n" |
| 168 | " "CLR" %0,%2\n" /* clear crosses next page boundary? */ |
| 169 | " jnh 5f\n" |
| 170 | " "AHI" %2,-1\n" |
| 171 | "1: ex %2,0(%3)\n" |
| 172 | " "AHI" %2,1\n" |
| 173 | " "SLR" %0,%2\n" |
| 174 | " j 5f\n" |
| 175 | "2: xc 0(256,%1),0(%1)\n" |
| 176 | " la %1,256(%1)\n" |
| 177 | "3:"AHI" %0,-256\n" |
| 178 | " jnm 2b\n" |
| 179 | "4: ex %0,0(%3)\n" |
Gerald Schaefer | d02765d | 2006-09-20 15:59:42 +0200 | [diff] [blame] | 180 | "5: "SLR" %0,%0\n" |
Martin Schwidefsky | e4d8269 | 2011-01-31 11:30:05 +0100 | [diff] [blame] | 181 | "6: sacf 0\n" |
Gerald Schaefer | d02765d | 2006-09-20 15:59:42 +0200 | [diff] [blame] | 182 | EX_TABLE(1b,6b) EX_TABLE(2b,0b) EX_TABLE(4b,0b) |
| 183 | : "+a" (size), "+a" (to), "=a" (tmp1), "=a" (tmp2) |
| 184 | : : "cc", "memory"); |
| 185 | return size; |
| 186 | } |
| 187 | |
| 188 | size_t strnlen_user_std(size_t size, const char __user *src) |
| 189 | { |
| 190 | register unsigned long reg0 asm("0") = 0UL; |
| 191 | unsigned long tmp1, tmp2; |
| 192 | |
| 193 | asm volatile( |
| 194 | " la %2,0(%1)\n" |
| 195 | " la %3,0(%0,%1)\n" |
| 196 | " "SLR" %0,%0\n" |
| 197 | " sacf 256\n" |
| 198 | "0: srst %3,%2\n" |
| 199 | " jo 0b\n" |
| 200 | " la %0,1(%3)\n" /* strnlen_user results includes \0 */ |
| 201 | " "SLR" %0,%1\n" |
| 202 | "1: sacf 0\n" |
| 203 | EX_TABLE(0b,1b) |
| 204 | : "+a" (size), "+a" (src), "=a" (tmp1), "=a" (tmp2) |
| 205 | : "d" (reg0) : "cc", "memory"); |
| 206 | return size; |
| 207 | } |
| 208 | |
| 209 | size_t strncpy_from_user_std(size_t size, const char __user *src, char *dst) |
| 210 | { |
| 211 | register unsigned long reg0 asm("0") = 0UL; |
| 212 | unsigned long tmp1, tmp2; |
| 213 | |
| 214 | asm volatile( |
| 215 | " la %3,0(%1)\n" |
| 216 | " la %4,0(%0,%1)\n" |
| 217 | " sacf 256\n" |
| 218 | "0: srst %4,%3\n" |
| 219 | " jo 0b\n" |
| 220 | " sacf 0\n" |
| 221 | " la %0,0(%4)\n" |
| 222 | " jh 1f\n" /* found \0 in string ? */ |
| 223 | " "AHI" %4,1\n" /* include \0 in copy */ |
| 224 | "1:"SLR" %0,%1\n" /* %0 = return length (without \0) */ |
| 225 | " "SLR" %4,%1\n" /* %4 = copy length (including \0) */ |
| 226 | "2: mvcp 0(%4,%2),0(%1),%5\n" |
| 227 | " jz 9f\n" |
| 228 | "3:"AHI" %4,-256\n" |
| 229 | " la %1,256(%1)\n" |
| 230 | " la %2,256(%2)\n" |
| 231 | "4: mvcp 0(%4,%2),0(%1),%5\n" |
| 232 | " jnz 3b\n" |
| 233 | " j 9f\n" |
| 234 | "7: sacf 0\n" |
| 235 | "8:"LHI" %0,%6\n" |
| 236 | "9:\n" |
| 237 | EX_TABLE(0b,7b) EX_TABLE(2b,8b) EX_TABLE(4b,8b) |
| 238 | : "+a" (size), "+a" (src), "+d" (dst), "=a" (tmp1), "=a" (tmp2) |
| 239 | : "d" (reg0), "K" (-EFAULT) : "cc", "memory"); |
| 240 | return size; |
| 241 | } |
| 242 | |
| 243 | #define __futex_atomic_op(insn, ret, oldval, newval, uaddr, oparg) \ |
| 244 | asm volatile( \ |
| 245 | " sacf 256\n" \ |
| 246 | "0: l %1,0(%6)\n" \ |
| 247 | "1:"insn \ |
| 248 | "2: cs %1,%2,0(%6)\n" \ |
| 249 | "3: jl 1b\n" \ |
| 250 | " lhi %0,0\n" \ |
| 251 | "4: sacf 0\n" \ |
| 252 | EX_TABLE(0b,4b) EX_TABLE(2b,4b) EX_TABLE(3b,4b) \ |
| 253 | : "=d" (ret), "=&d" (oldval), "=&d" (newval), \ |
| 254 | "=m" (*uaddr) \ |
| 255 | : "0" (-EFAULT), "d" (oparg), "a" (uaddr), \ |
| 256 | "m" (*uaddr) : "cc"); |
| 257 | |
Michel Lespinasse | 8d7718a | 2011-03-10 18:50:58 -0800 | [diff] [blame] | 258 | int futex_atomic_op_std(int op, u32 __user *uaddr, int oparg, int *old) |
Gerald Schaefer | d02765d | 2006-09-20 15:59:42 +0200 | [diff] [blame] | 259 | { |
| 260 | int oldval = 0, newval, ret; |
| 261 | |
Gerald Schaefer | d02765d | 2006-09-20 15:59:42 +0200 | [diff] [blame] | 262 | switch (op) { |
| 263 | case FUTEX_OP_SET: |
| 264 | __futex_atomic_op("lr %2,%5\n", |
| 265 | ret, oldval, newval, uaddr, oparg); |
| 266 | break; |
| 267 | case FUTEX_OP_ADD: |
| 268 | __futex_atomic_op("lr %2,%1\nar %2,%5\n", |
| 269 | ret, oldval, newval, uaddr, oparg); |
| 270 | break; |
| 271 | case FUTEX_OP_OR: |
| 272 | __futex_atomic_op("lr %2,%1\nor %2,%5\n", |
| 273 | ret, oldval, newval, uaddr, oparg); |
| 274 | break; |
| 275 | case FUTEX_OP_ANDN: |
| 276 | __futex_atomic_op("lr %2,%1\nnr %2,%5\n", |
| 277 | ret, oldval, newval, uaddr, oparg); |
| 278 | break; |
| 279 | case FUTEX_OP_XOR: |
| 280 | __futex_atomic_op("lr %2,%1\nxr %2,%5\n", |
| 281 | ret, oldval, newval, uaddr, oparg); |
| 282 | break; |
| 283 | default: |
| 284 | ret = -ENOSYS; |
| 285 | } |
Gerald Schaefer | d02765d | 2006-09-20 15:59:42 +0200 | [diff] [blame] | 286 | *old = oldval; |
| 287 | return ret; |
| 288 | } |
| 289 | |
Michel Lespinasse | 8d7718a | 2011-03-10 18:50:58 -0800 | [diff] [blame] | 290 | int futex_atomic_cmpxchg_std(u32 *uval, u32 __user *uaddr, |
| 291 | u32 oldval, u32 newval) |
Gerald Schaefer | d02765d | 2006-09-20 15:59:42 +0200 | [diff] [blame] | 292 | { |
| 293 | int ret; |
| 294 | |
| 295 | asm volatile( |
| 296 | " sacf 256\n" |
Heiko Carstens | d5b02b3 | 2008-02-19 15:29:34 +0100 | [diff] [blame] | 297 | "0: cs %1,%4,0(%5)\n" |
Michel Lespinasse | 37a9d91 | 2011-03-10 18:48:51 -0800 | [diff] [blame] | 298 | "1: la %0,0\n" |
Heiko Carstens | d5b02b3 | 2008-02-19 15:29:34 +0100 | [diff] [blame] | 299 | "2: sacf 0\n" |
| 300 | EX_TABLE(0b,2b) EX_TABLE(1b,2b) |
Gerald Schaefer | d02765d | 2006-09-20 15:59:42 +0200 | [diff] [blame] | 301 | : "=d" (ret), "+d" (oldval), "=m" (*uaddr) |
| 302 | : "0" (-EFAULT), "d" (newval), "a" (uaddr), "m" (*uaddr) |
| 303 | : "cc", "memory" ); |
Michel Lespinasse | 37a9d91 | 2011-03-10 18:48:51 -0800 | [diff] [blame] | 304 | *uval = oldval; |
Gerald Schaefer | d02765d | 2006-09-20 15:59:42 +0200 | [diff] [blame] | 305 | return ret; |
| 306 | } |
| 307 | |
| 308 | struct uaccess_ops uaccess_std = { |
Gerald Schaefer | 59f35d5 | 2006-12-04 15:40:45 +0100 | [diff] [blame] | 309 | .copy_from_user = copy_from_user_std_check, |
| 310 | .copy_from_user_small = copy_from_user_std, |
| 311 | .copy_to_user = copy_to_user_std_check, |
| 312 | .copy_to_user_small = copy_to_user_std, |
Gerald Schaefer | d02765d | 2006-09-20 15:59:42 +0200 | [diff] [blame] | 313 | .copy_in_user = copy_in_user_std, |
| 314 | .clear_user = clear_user_std, |
| 315 | .strnlen_user = strnlen_user_std, |
| 316 | .strncpy_from_user = strncpy_from_user_std, |
Heiko Carstens | 2b67fc4 | 2007-02-05 21:16:47 +0100 | [diff] [blame] | 317 | .futex_atomic_op = futex_atomic_op_std, |
| 318 | .futex_atomic_cmpxchg = futex_atomic_cmpxchg_std, |
Gerald Schaefer | d02765d | 2006-09-20 15:59:42 +0200 | [diff] [blame] | 319 | }; |