Vitaly Mayatskikh | ad2fc2c | 2008-07-02 15:53:13 +0200 | [diff] [blame] | 1 | /* |
| 2 | * Copyright 2008 Vitaly Mayatskikh <vmayatsk@redhat.com> |
| 3 | * Copyright 2002 Andi Kleen, SuSE Labs. |
Andi Kleen | 0812a57 | 2007-02-13 13:26:19 +0100 | [diff] [blame] | 4 | * Subject to the GNU Public License v2. |
| 5 | * |
| 6 | * Functions to copy from and to user space. |
| 7 | */ |
| 8 | |
| 9 | #include <linux/linkage.h> |
| 10 | #include <asm/dwarf2.h> |
| 11 | |
| 12 | #define FIX_ALIGNMENT 1 |
| 13 | |
| 14 | #include <asm/current.h> |
| 15 | #include <asm/asm-offsets.h> |
| 16 | #include <asm/thread_info.h> |
H. Peter Anvin | 0d8559f | 2012-04-20 12:19:51 -0700 | [diff] [blame] | 17 | #include <asm/asm.h> |
Andi Kleen | 0812a57 | 2007-02-13 13:26:19 +0100 | [diff] [blame] | 18 | |
Vitaly Mayatskikh | ad2fc2c | 2008-07-02 15:53:13 +0200 | [diff] [blame] | 19 | .macro ALIGN_DESTINATION |
Andi Kleen | 0812a57 | 2007-02-13 13:26:19 +0100 | [diff] [blame] | 20 | #ifdef FIX_ALIGNMENT |
| 21 | /* check for bad alignment of destination */ |
| 22 | movl %edi,%ecx |
| 23 | andl $7,%ecx |
Vitaly Mayatskikh | ad2fc2c | 2008-07-02 15:53:13 +0200 | [diff] [blame] | 24 | jz 102f /* already aligned */ |
| 25 | subl $8,%ecx |
| 26 | negl %ecx |
| 27 | subl %ecx,%edx |
| 28 | 100: movb (%rsi),%al |
| 29 | 101: movb %al,(%rdi) |
Andi Kleen | 0812a57 | 2007-02-13 13:26:19 +0100 | [diff] [blame] | 30 | incq %rsi |
| 31 | incq %rdi |
| 32 | decl %ecx |
Vitaly Mayatskikh | ad2fc2c | 2008-07-02 15:53:13 +0200 | [diff] [blame] | 33 | jnz 100b |
| 34 | 102: |
| 35 | .section .fixup,"ax" |
Vitaly Mayatskikh | afd962a | 2008-07-30 13:30:14 +0200 | [diff] [blame] | 36 | 103: addl %ecx,%edx /* ecx is zerorest also */ |
Vitaly Mayatskikh | ad2fc2c | 2008-07-02 15:53:13 +0200 | [diff] [blame] | 37 | jmp copy_user_handle_tail |
Andi Kleen | 0812a57 | 2007-02-13 13:26:19 +0100 | [diff] [blame] | 38 | .previous |
| 39 | |
H. Peter Anvin | 0d8559f | 2012-04-20 12:19:51 -0700 | [diff] [blame] | 40 | _ASM_EXTABLE(100b,103b) |
| 41 | _ASM_EXTABLE(101b,103b) |
Vitaly Mayatskikh | ad2fc2c | 2008-07-02 15:53:13 +0200 | [diff] [blame] | 42 | #endif |
| 43 | .endm |
Andi Kleen | 0812a57 | 2007-02-13 13:26:19 +0100 | [diff] [blame] | 44 | |
Vitaly Mayatskikh | ad2fc2c | 2008-07-02 15:53:13 +0200 | [diff] [blame] | 45 | /* |
| 46 | * copy_user_nocache - Uncached memory copy with exception handling |
| 47 | * This will force destination/source out of cache for more performance. |
| 48 | */ |
| 49 | ENTRY(__copy_user_nocache) |
| 50 | CFI_STARTPROC |
| 51 | cmpl $8,%edx |
| 52 | jb 20f /* less then 8 bytes, go to byte copy loop */ |
| 53 | ALIGN_DESTINATION |
| 54 | movl %edx,%ecx |
| 55 | andl $63,%edx |
| 56 | shrl $6,%ecx |
| 57 | jz 17f |
| 58 | 1: movq (%rsi),%r8 |
| 59 | 2: movq 1*8(%rsi),%r9 |
| 60 | 3: movq 2*8(%rsi),%r10 |
| 61 | 4: movq 3*8(%rsi),%r11 |
| 62 | 5: movnti %r8,(%rdi) |
| 63 | 6: movnti %r9,1*8(%rdi) |
| 64 | 7: movnti %r10,2*8(%rdi) |
| 65 | 8: movnti %r11,3*8(%rdi) |
| 66 | 9: movq 4*8(%rsi),%r8 |
| 67 | 10: movq 5*8(%rsi),%r9 |
| 68 | 11: movq 6*8(%rsi),%r10 |
| 69 | 12: movq 7*8(%rsi),%r11 |
| 70 | 13: movnti %r8,4*8(%rdi) |
| 71 | 14: movnti %r9,5*8(%rdi) |
| 72 | 15: movnti %r10,6*8(%rdi) |
| 73 | 16: movnti %r11,7*8(%rdi) |
| 74 | leaq 64(%rsi),%rsi |
| 75 | leaq 64(%rdi),%rdi |
| 76 | decl %ecx |
| 77 | jnz 1b |
| 78 | 17: movl %edx,%ecx |
Andi Kleen | 0812a57 | 2007-02-13 13:26:19 +0100 | [diff] [blame] | 79 | andl $7,%edx |
Vitaly Mayatskikh | ad2fc2c | 2008-07-02 15:53:13 +0200 | [diff] [blame] | 80 | shrl $3,%ecx |
| 81 | jz 20f |
| 82 | 18: movq (%rsi),%r8 |
| 83 | 19: movnti %r8,(%rdi) |
| 84 | leaq 8(%rsi),%rsi |
| 85 | leaq 8(%rdi),%rdi |
| 86 | decl %ecx |
| 87 | jnz 18b |
| 88 | 20: andl %edx,%edx |
| 89 | jz 23f |
| 90 | movl %edx,%ecx |
| 91 | 21: movb (%rsi),%al |
| 92 | 22: movb %al,(%rdi) |
| 93 | incq %rsi |
| 94 | incq %rdi |
| 95 | decl %ecx |
| 96 | jnz 21b |
| 97 | 23: xorl %eax,%eax |
| 98 | sfence |
| 99 | ret |
| 100 | |
| 101 | .section .fixup,"ax" |
| 102 | 30: shll $6,%ecx |
Andi Kleen | 0812a57 | 2007-02-13 13:26:19 +0100 | [diff] [blame] | 103 | addl %ecx,%edx |
Vitaly Mayatskikh | ad2fc2c | 2008-07-02 15:53:13 +0200 | [diff] [blame] | 104 | jmp 60f |
Jeremy Fitzhardinge | 27cb0a7 | 2008-07-10 12:52:52 -0700 | [diff] [blame] | 105 | 40: lea (%rdx,%rcx,8),%rdx |
Vitaly Mayatskikh | ad2fc2c | 2008-07-02 15:53:13 +0200 | [diff] [blame] | 106 | jmp 60f |
| 107 | 50: movl %ecx,%edx |
| 108 | 60: sfence |
Vitaly Mayatskikh | ad2fc2c | 2008-07-02 15:53:13 +0200 | [diff] [blame] | 109 | jmp copy_user_handle_tail |
| 110 | .previous |
| 111 | |
H. Peter Anvin | 0d8559f | 2012-04-20 12:19:51 -0700 | [diff] [blame] | 112 | _ASM_EXTABLE(1b,30b) |
| 113 | _ASM_EXTABLE(2b,30b) |
| 114 | _ASM_EXTABLE(3b,30b) |
| 115 | _ASM_EXTABLE(4b,30b) |
| 116 | _ASM_EXTABLE(5b,30b) |
| 117 | _ASM_EXTABLE(6b,30b) |
| 118 | _ASM_EXTABLE(7b,30b) |
| 119 | _ASM_EXTABLE(8b,30b) |
| 120 | _ASM_EXTABLE(9b,30b) |
| 121 | _ASM_EXTABLE(10b,30b) |
| 122 | _ASM_EXTABLE(11b,30b) |
| 123 | _ASM_EXTABLE(12b,30b) |
| 124 | _ASM_EXTABLE(13b,30b) |
| 125 | _ASM_EXTABLE(14b,30b) |
| 126 | _ASM_EXTABLE(15b,30b) |
| 127 | _ASM_EXTABLE(16b,30b) |
| 128 | _ASM_EXTABLE(18b,40b) |
| 129 | _ASM_EXTABLE(19b,40b) |
| 130 | _ASM_EXTABLE(21b,50b) |
| 131 | _ASM_EXTABLE(22b,50b) |
Andi Kleen | 0812a57 | 2007-02-13 13:26:19 +0100 | [diff] [blame] | 132 | CFI_ENDPROC |
| 133 | ENDPROC(__copy_user_nocache) |