Vitaly Mayatskikh | ad2fc2c | 2008-07-02 15:53:13 +0200 | [diff] [blame] | 1 | /* |
| 2 | * Copyright 2008 Vitaly Mayatskikh <vmayatsk@redhat.com> |
| 3 | * Copyright 2002 Andi Kleen, SuSE Labs. |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 4 | * Subject to the GNU Public License v2. |
Vitaly Mayatskikh | ad2fc2c | 2008-07-02 15:53:13 +0200 | [diff] [blame] | 5 | * |
| 6 | * Functions to copy from and to user space. |
| 7 | */ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 8 | |
Jan Beulich | 8d379da | 2006-09-26 10:52:32 +0200 | [diff] [blame] | 9 | #include <linux/linkage.h> |
| 10 | #include <asm/dwarf2.h> |
| 11 | |
Andi Kleen | 7bcd3f3 | 2006-02-03 21:51:02 +0100 | [diff] [blame] | 12 | #define FIX_ALIGNMENT 1 |
| 13 | |
Andi Kleen | 3022d73 | 2006-09-26 10:52:39 +0200 | [diff] [blame] | 14 | #include <asm/current.h> |
| 15 | #include <asm/asm-offsets.h> |
| 16 | #include <asm/thread_info.h> |
| 17 | #include <asm/cpufeature.h> |
| 18 | |
| 19 | .macro ALTERNATIVE_JUMP feature,orig,alt |
| 20 | 0: |
| 21 | .byte 0xe9 /* 32bit jump */ |
| 22 | .long \orig-1f /* by default jump to orig */ |
| 23 | 1: |
| 24 | .section .altinstr_replacement,"ax" |
Vitaly Mayatskikh | ad2fc2c | 2008-07-02 15:53:13 +0200 | [diff] [blame] | 25 | 2: .byte 0xe9 /* near jump with 32bit immediate */ |
Andi Kleen | 3022d73 | 2006-09-26 10:52:39 +0200 | [diff] [blame] | 26 | .long \alt-1b /* offset */ /* or alternatively to alt */ |
| 27 | .previous |
| 28 | .section .altinstructions,"a" |
| 29 | .align 8 |
| 30 | .quad 0b |
| 31 | .quad 2b |
H. Peter Anvin | df378cc | 2010-07-13 14:55:11 -0700 | [diff] [blame] | 32 | .word \feature /* when feature is set */ |
Andi Kleen | 3022d73 | 2006-09-26 10:52:39 +0200 | [diff] [blame] | 33 | .byte 5 |
| 34 | .byte 5 |
| 35 | .previous |
| 36 | .endm |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 37 | |
Vitaly Mayatskikh | ad2fc2c | 2008-07-02 15:53:13 +0200 | [diff] [blame] | 38 | .macro ALIGN_DESTINATION |
| 39 | #ifdef FIX_ALIGNMENT |
| 40 | /* check for bad alignment of destination */ |
| 41 | movl %edi,%ecx |
| 42 | andl $7,%ecx |
| 43 | jz 102f /* already aligned */ |
| 44 | subl $8,%ecx |
| 45 | negl %ecx |
| 46 | subl %ecx,%edx |
| 47 | 100: movb (%rsi),%al |
| 48 | 101: movb %al,(%rdi) |
| 49 | incq %rsi |
| 50 | incq %rdi |
| 51 | decl %ecx |
| 52 | jnz 100b |
| 53 | 102: |
| 54 | .section .fixup,"ax" |
Vitaly Mayatskikh | afd962a | 2008-07-30 13:30:14 +0200 | [diff] [blame] | 55 | 103: addl %ecx,%edx /* ecx is zerorest also */ |
Vitaly Mayatskikh | ad2fc2c | 2008-07-02 15:53:13 +0200 | [diff] [blame] | 56 | jmp copy_user_handle_tail |
| 57 | .previous |
| 58 | |
| 59 | .section __ex_table,"a" |
| 60 | .align 8 |
| 61 | .quad 100b,103b |
| 62 | .quad 101b,103b |
| 63 | .previous |
| 64 | #endif |
| 65 | .endm |
| 66 | |
| 67 | /* Standard copy_to_user with segment limit checking */ |
Frederic Weisbecker | 3c93ca0 | 2009-11-16 15:42:18 +0100 | [diff] [blame] | 68 | ENTRY(_copy_to_user) |
Jan Beulich | 8d379da | 2006-09-26 10:52:32 +0200 | [diff] [blame] | 69 | CFI_STARTPROC |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 70 | GET_THREAD_INFO(%rax) |
| 71 | movq %rdi,%rcx |
| 72 | addq %rdx,%rcx |
Vitaly Mayatskikh | ad2fc2c | 2008-07-02 15:53:13 +0200 | [diff] [blame] | 73 | jc bad_to_user |
Glauber Costa | 26ccb8a | 2008-06-24 11:19:35 -0300 | [diff] [blame] | 74 | cmpq TI_addr_limit(%rax),%rcx |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 75 | jae bad_to_user |
Andi Kleen | 3022d73 | 2006-09-26 10:52:39 +0200 | [diff] [blame] | 76 | ALTERNATIVE_JUMP X86_FEATURE_REP_GOOD,copy_user_generic_unrolled,copy_user_generic_string |
Jan Beulich | 8d379da | 2006-09-26 10:52:32 +0200 | [diff] [blame] | 77 | CFI_ENDPROC |
Frederic Weisbecker | 3c93ca0 | 2009-11-16 15:42:18 +0100 | [diff] [blame] | 78 | ENDPROC(_copy_to_user) |
Andi Kleen | 7bcd3f3 | 2006-02-03 21:51:02 +0100 | [diff] [blame] | 79 | |
Vitaly Mayatskikh | ad2fc2c | 2008-07-02 15:53:13 +0200 | [diff] [blame] | 80 | /* Standard copy_from_user with segment limit checking */ |
Arjan van de Ven | 9f0cf4a | 2009-09-26 14:33:01 +0200 | [diff] [blame] | 81 | ENTRY(_copy_from_user) |
Jan Beulich | 8d379da | 2006-09-26 10:52:32 +0200 | [diff] [blame] | 82 | CFI_STARTPROC |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 83 | GET_THREAD_INFO(%rax) |
| 84 | movq %rsi,%rcx |
| 85 | addq %rdx,%rcx |
Vitaly Mayatskikh | ad2fc2c | 2008-07-02 15:53:13 +0200 | [diff] [blame] | 86 | jc bad_from_user |
Glauber Costa | 26ccb8a | 2008-06-24 11:19:35 -0300 | [diff] [blame] | 87 | cmpq TI_addr_limit(%rax),%rcx |
Vitaly Mayatskikh | ad2fc2c | 2008-07-02 15:53:13 +0200 | [diff] [blame] | 88 | jae bad_from_user |
Andi Kleen | 3022d73 | 2006-09-26 10:52:39 +0200 | [diff] [blame] | 89 | ALTERNATIVE_JUMP X86_FEATURE_REP_GOOD,copy_user_generic_unrolled,copy_user_generic_string |
Jan Beulich | 8d379da | 2006-09-26 10:52:32 +0200 | [diff] [blame] | 90 | CFI_ENDPROC |
Arjan van de Ven | 9f0cf4a | 2009-09-26 14:33:01 +0200 | [diff] [blame] | 91 | ENDPROC(_copy_from_user) |
Vitaly Mayatskikh | ad2fc2c | 2008-07-02 15:53:13 +0200 | [diff] [blame] | 92 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 93 | .section .fixup,"ax" |
| 94 | /* must zero dest */ |
Vitaly Mayatskikh | ad2fc2c | 2008-07-02 15:53:13 +0200 | [diff] [blame] | 95 | ENTRY(bad_from_user) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 96 | bad_from_user: |
Jan Beulich | 8d379da | 2006-09-26 10:52:32 +0200 | [diff] [blame] | 97 | CFI_STARTPROC |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 98 | movl %edx,%ecx |
| 99 | xorl %eax,%eax |
| 100 | rep |
| 101 | stosb |
| 102 | bad_to_user: |
Vitaly Mayatskikh | ad2fc2c | 2008-07-02 15:53:13 +0200 | [diff] [blame] | 103 | movl %edx,%eax |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 104 | ret |
Jan Beulich | 8d379da | 2006-09-26 10:52:32 +0200 | [diff] [blame] | 105 | CFI_ENDPROC |
Vitaly Mayatskikh | ad2fc2c | 2008-07-02 15:53:13 +0200 | [diff] [blame] | 106 | ENDPROC(bad_from_user) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 107 | .previous |
Vitaly Mayatskikh | ad2fc2c | 2008-07-02 15:53:13 +0200 | [diff] [blame] | 108 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 109 | /* |
Andi Kleen | 3022d73 | 2006-09-26 10:52:39 +0200 | [diff] [blame] | 110 | * copy_user_generic_unrolled - memory copy with exception handling. |
Vitaly Mayatskikh | ad2fc2c | 2008-07-02 15:53:13 +0200 | [diff] [blame] | 111 | * This version is for CPUs like P4 that don't have efficient micro |
| 112 | * code for rep movsq |
| 113 | * |
| 114 | * Input: |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 115 | * rdi destination |
| 116 | * rsi source |
| 117 | * rdx count |
| 118 | * |
Vitaly Mayatskikh | ad2fc2c | 2008-07-02 15:53:13 +0200 | [diff] [blame] | 119 | * Output: |
Lucas De Marchi | 0d2eb44 | 2011-03-17 16:24:16 -0300 | [diff] [blame^] | 120 | * eax uncopied bytes or 0 if successful. |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 121 | */ |
Andi Kleen | 3022d73 | 2006-09-26 10:52:39 +0200 | [diff] [blame] | 122 | ENTRY(copy_user_generic_unrolled) |
Jan Beulich | 8d379da | 2006-09-26 10:52:32 +0200 | [diff] [blame] | 123 | CFI_STARTPROC |
Vitaly Mayatskikh | ad2fc2c | 2008-07-02 15:53:13 +0200 | [diff] [blame] | 124 | cmpl $8,%edx |
| 125 | jb 20f /* less then 8 bytes, go to byte copy loop */ |
| 126 | ALIGN_DESTINATION |
| 127 | movl %edx,%ecx |
| 128 | andl $63,%edx |
| 129 | shrl $6,%ecx |
| 130 | jz 17f |
| 131 | 1: movq (%rsi),%r8 |
| 132 | 2: movq 1*8(%rsi),%r9 |
| 133 | 3: movq 2*8(%rsi),%r10 |
| 134 | 4: movq 3*8(%rsi),%r11 |
| 135 | 5: movq %r8,(%rdi) |
| 136 | 6: movq %r9,1*8(%rdi) |
| 137 | 7: movq %r10,2*8(%rdi) |
| 138 | 8: movq %r11,3*8(%rdi) |
| 139 | 9: movq 4*8(%rsi),%r8 |
| 140 | 10: movq 5*8(%rsi),%r9 |
| 141 | 11: movq 6*8(%rsi),%r10 |
| 142 | 12: movq 7*8(%rsi),%r11 |
| 143 | 13: movq %r8,4*8(%rdi) |
| 144 | 14: movq %r9,5*8(%rdi) |
| 145 | 15: movq %r10,6*8(%rdi) |
| 146 | 16: movq %r11,7*8(%rdi) |
Andi Kleen | 7bcd3f3 | 2006-02-03 21:51:02 +0100 | [diff] [blame] | 147 | leaq 64(%rsi),%rsi |
| 148 | leaq 64(%rdi),%rdi |
Vitaly Mayatskikh | ad2fc2c | 2008-07-02 15:53:13 +0200 | [diff] [blame] | 149 | decl %ecx |
| 150 | jnz 1b |
| 151 | 17: movl %edx,%ecx |
| 152 | andl $7,%edx |
Andi Kleen | 7bcd3f3 | 2006-02-03 21:51:02 +0100 | [diff] [blame] | 153 | shrl $3,%ecx |
Vitaly Mayatskikh | ad2fc2c | 2008-07-02 15:53:13 +0200 | [diff] [blame] | 154 | jz 20f |
| 155 | 18: movq (%rsi),%r8 |
| 156 | 19: movq %r8,(%rdi) |
Andi Kleen | 7bcd3f3 | 2006-02-03 21:51:02 +0100 | [diff] [blame] | 157 | leaq 8(%rsi),%rsi |
Vitaly Mayatskikh | ad2fc2c | 2008-07-02 15:53:13 +0200 | [diff] [blame] | 158 | leaq 8(%rdi),%rdi |
| 159 | decl %ecx |
| 160 | jnz 18b |
| 161 | 20: andl %edx,%edx |
| 162 | jz 23f |
Andi Kleen | 7bcd3f3 | 2006-02-03 21:51:02 +0100 | [diff] [blame] | 163 | movl %edx,%ecx |
Vitaly Mayatskikh | ad2fc2c | 2008-07-02 15:53:13 +0200 | [diff] [blame] | 164 | 21: movb (%rsi),%al |
| 165 | 22: movb %al,(%rdi) |
Andi Kleen | 7bcd3f3 | 2006-02-03 21:51:02 +0100 | [diff] [blame] | 166 | incq %rsi |
Vitaly Mayatskikh | ad2fc2c | 2008-07-02 15:53:13 +0200 | [diff] [blame] | 167 | incq %rdi |
Andi Kleen | 7bcd3f3 | 2006-02-03 21:51:02 +0100 | [diff] [blame] | 168 | decl %ecx |
Vitaly Mayatskikh | ad2fc2c | 2008-07-02 15:53:13 +0200 | [diff] [blame] | 169 | jnz 21b |
| 170 | 23: xor %eax,%eax |
Andi Kleen | 7bcd3f3 | 2006-02-03 21:51:02 +0100 | [diff] [blame] | 171 | ret |
| 172 | |
Vitaly Mayatskikh | ad2fc2c | 2008-07-02 15:53:13 +0200 | [diff] [blame] | 173 | .section .fixup,"ax" |
| 174 | 30: shll $6,%ecx |
| 175 | addl %ecx,%edx |
| 176 | jmp 60f |
Jeremy Fitzhardinge | 27cb0a7 | 2008-07-10 12:52:52 -0700 | [diff] [blame] | 177 | 40: lea (%rdx,%rcx,8),%rdx |
Vitaly Mayatskikh | ad2fc2c | 2008-07-02 15:53:13 +0200 | [diff] [blame] | 178 | jmp 60f |
| 179 | 50: movl %ecx,%edx |
| 180 | 60: jmp copy_user_handle_tail /* ecx is zerorest also */ |
| 181 | .previous |
Andi Kleen | 7bcd3f3 | 2006-02-03 21:51:02 +0100 | [diff] [blame] | 182 | |
Andi Kleen | 7bcd3f3 | 2006-02-03 21:51:02 +0100 | [diff] [blame] | 183 | .section __ex_table,"a" |
| 184 | .align 8 |
Vitaly Mayatskikh | ad2fc2c | 2008-07-02 15:53:13 +0200 | [diff] [blame] | 185 | .quad 1b,30b |
| 186 | .quad 2b,30b |
| 187 | .quad 3b,30b |
| 188 | .quad 4b,30b |
| 189 | .quad 5b,30b |
| 190 | .quad 6b,30b |
| 191 | .quad 7b,30b |
| 192 | .quad 8b,30b |
| 193 | .quad 9b,30b |
| 194 | .quad 10b,30b |
| 195 | .quad 11b,30b |
| 196 | .quad 12b,30b |
| 197 | .quad 13b,30b |
| 198 | .quad 14b,30b |
| 199 | .quad 15b,30b |
| 200 | .quad 16b,30b |
| 201 | .quad 18b,40b |
| 202 | .quad 19b,40b |
| 203 | .quad 21b,50b |
| 204 | .quad 22b,50b |
Andi Kleen | 7bcd3f3 | 2006-02-03 21:51:02 +0100 | [diff] [blame] | 205 | .previous |
Jan Beulich | 8d379da | 2006-09-26 10:52:32 +0200 | [diff] [blame] | 206 | CFI_ENDPROC |
Vitaly Mayatskikh | ad2fc2c | 2008-07-02 15:53:13 +0200 | [diff] [blame] | 207 | ENDPROC(copy_user_generic_unrolled) |
Jan Beulich | 8d379da | 2006-09-26 10:52:32 +0200 | [diff] [blame] | 208 | |
Vitaly Mayatskikh | ad2fc2c | 2008-07-02 15:53:13 +0200 | [diff] [blame] | 209 | /* Some CPUs run faster using the string copy instructions. |
| 210 | * This is also a lot simpler. Use them when possible. |
| 211 | * |
| 212 | * Only 4GB of copy is supported. This shouldn't be a problem |
| 213 | * because the kernel normally only writes from/to page sized chunks |
| 214 | * even if user space passed a longer buffer. |
| 215 | * And more would be dangerous because both Intel and AMD have |
| 216 | * errata with rep movsq > 4GB. If someone feels the need to fix |
| 217 | * this please consider this. |
| 218 | * |
| 219 | * Input: |
| 220 | * rdi destination |
| 221 | * rsi source |
| 222 | * rdx count |
| 223 | * |
| 224 | * Output: |
| 225 | * eax uncopied bytes or 0 if successful. |
| 226 | */ |
Andi Kleen | 3022d73 | 2006-09-26 10:52:39 +0200 | [diff] [blame] | 227 | ENTRY(copy_user_generic_string) |
Jan Beulich | 8d379da | 2006-09-26 10:52:32 +0200 | [diff] [blame] | 228 | CFI_STARTPROC |
Vitaly Mayatskikh | ad2fc2c | 2008-07-02 15:53:13 +0200 | [diff] [blame] | 229 | andl %edx,%edx |
| 230 | jz 4f |
| 231 | cmpl $8,%edx |
| 232 | jb 2f /* less than 8 bytes, go to byte copy loop */ |
| 233 | ALIGN_DESTINATION |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 234 | movl %edx,%ecx |
| 235 | shrl $3,%ecx |
Vitaly Mayatskikh | ad2fc2c | 2008-07-02 15:53:13 +0200 | [diff] [blame] | 236 | andl $7,%edx |
| 237 | 1: rep |
Andi Kleen | 3022d73 | 2006-09-26 10:52:39 +0200 | [diff] [blame] | 238 | movsq |
Vitaly Mayatskikh | ad2fc2c | 2008-07-02 15:53:13 +0200 | [diff] [blame] | 239 | 2: movl %edx,%ecx |
| 240 | 3: rep |
| 241 | movsb |
| 242 | 4: xorl %eax,%eax |
Andi Kleen | 7bcd3f3 | 2006-02-03 21:51:02 +0100 | [diff] [blame] | 243 | ret |
Andi Kleen | 3022d73 | 2006-09-26 10:52:39 +0200 | [diff] [blame] | 244 | |
Vitaly Mayatskikh | ad2fc2c | 2008-07-02 15:53:13 +0200 | [diff] [blame] | 245 | .section .fixup,"ax" |
Jeremy Fitzhardinge | 27cb0a7 | 2008-07-10 12:52:52 -0700 | [diff] [blame] | 246 | 11: lea (%rdx,%rcx,8),%rcx |
Vitaly Mayatskikh | ad2fc2c | 2008-07-02 15:53:13 +0200 | [diff] [blame] | 247 | 12: movl %ecx,%edx /* ecx is zerorest also */ |
| 248 | jmp copy_user_handle_tail |
| 249 | .previous |
Andi Kleen | 2cbc9ee | 2006-01-11 22:44:45 +0100 | [diff] [blame] | 250 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 251 | .section __ex_table,"a" |
Vitaly Mayatskikh | ad2fc2c | 2008-07-02 15:53:13 +0200 | [diff] [blame] | 252 | .align 8 |
| 253 | .quad 1b,11b |
| 254 | .quad 3b,12b |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 255 | .previous |
Vitaly Mayatskikh | ad2fc2c | 2008-07-02 15:53:13 +0200 | [diff] [blame] | 256 | CFI_ENDPROC |
| 257 | ENDPROC(copy_user_generic_string) |