Vitaly Mayatskikh | ad2fc2c | 2008-07-02 15:53:13 +0200 | [diff] [blame] | 1 | /* |
| 2 | * Copyright 2008 Vitaly Mayatskikh <vmayatsk@redhat.com> |
| 3 | * Copyright 2002 Andi Kleen, SuSE Labs. |
Andi Kleen | 0812a57 | 2007-02-13 13:26:19 +0100 | [diff] [blame] | 4 | * Subject to the GNU Public License v2. |
| 5 | * |
| 6 | * Functions to copy from and to user space. |
| 7 | */ |
| 8 | |
| 9 | #include <linux/linkage.h> |
| 10 | #include <asm/dwarf2.h> |
| 11 | |
| 12 | #define FIX_ALIGNMENT 1 |
| 13 | |
| 14 | #include <asm/current.h> |
| 15 | #include <asm/asm-offsets.h> |
| 16 | #include <asm/thread_info.h> |
Andi Kleen | 0812a57 | 2007-02-13 13:26:19 +0100 | [diff] [blame] | 17 | |
Vitaly Mayatskikh | ad2fc2c | 2008-07-02 15:53:13 +0200 | [diff] [blame] | 18 | .macro ALIGN_DESTINATION |
Andi Kleen | 0812a57 | 2007-02-13 13:26:19 +0100 | [diff] [blame] | 19 | #ifdef FIX_ALIGNMENT |
| 20 | /* check for bad alignment of destination */ |
| 21 | movl %edi,%ecx |
| 22 | andl $7,%ecx |
Vitaly Mayatskikh | ad2fc2c | 2008-07-02 15:53:13 +0200 | [diff] [blame] | 23 | jz 102f /* already aligned */ |
| 24 | subl $8,%ecx |
| 25 | negl %ecx |
| 26 | subl %ecx,%edx |
| 27 | 100: movb (%rsi),%al |
| 28 | 101: movb %al,(%rdi) |
Andi Kleen | 0812a57 | 2007-02-13 13:26:19 +0100 | [diff] [blame] | 29 | incq %rsi |
| 30 | incq %rdi |
| 31 | decl %ecx |
Vitaly Mayatskikh | ad2fc2c | 2008-07-02 15:53:13 +0200 | [diff] [blame] | 32 | jnz 100b |
| 33 | 102: |
| 34 | .section .fixup,"ax" |
| 35 | 103: addl %r8d,%edx /* ecx is zerorest also */ |
| 36 | jmp copy_user_handle_tail |
Andi Kleen | 0812a57 | 2007-02-13 13:26:19 +0100 | [diff] [blame] | 37 | .previous |
| 38 | |
Vitaly Mayatskikh | ad2fc2c | 2008-07-02 15:53:13 +0200 | [diff] [blame] | 39 | .section __ex_table,"a" |
| 40 | .align 8 |
| 41 | .quad 100b,103b |
| 42 | .quad 101b,103b |
| 43 | .previous |
| 44 | #endif |
| 45 | .endm |
Andi Kleen | 0812a57 | 2007-02-13 13:26:19 +0100 | [diff] [blame] | 46 | |
Vitaly Mayatskikh | ad2fc2c | 2008-07-02 15:53:13 +0200 | [diff] [blame] | 47 | /* |
| 48 | * copy_user_nocache - Uncached memory copy with exception handling |
| 49 | * This will force destination/source out of cache for more performance. |
| 50 | */ |
| 51 | ENTRY(__copy_user_nocache) |
| 52 | CFI_STARTPROC |
| 53 | cmpl $8,%edx |
| 54 | jb 20f /* less then 8 bytes, go to byte copy loop */ |
| 55 | ALIGN_DESTINATION |
| 56 | movl %edx,%ecx |
| 57 | andl $63,%edx |
| 58 | shrl $6,%ecx |
| 59 | jz 17f |
| 60 | 1: movq (%rsi),%r8 |
| 61 | 2: movq 1*8(%rsi),%r9 |
| 62 | 3: movq 2*8(%rsi),%r10 |
| 63 | 4: movq 3*8(%rsi),%r11 |
| 64 | 5: movnti %r8,(%rdi) |
| 65 | 6: movnti %r9,1*8(%rdi) |
| 66 | 7: movnti %r10,2*8(%rdi) |
| 67 | 8: movnti %r11,3*8(%rdi) |
| 68 | 9: movq 4*8(%rsi),%r8 |
| 69 | 10: movq 5*8(%rsi),%r9 |
| 70 | 11: movq 6*8(%rsi),%r10 |
| 71 | 12: movq 7*8(%rsi),%r11 |
| 72 | 13: movnti %r8,4*8(%rdi) |
| 73 | 14: movnti %r9,5*8(%rdi) |
| 74 | 15: movnti %r10,6*8(%rdi) |
| 75 | 16: movnti %r11,7*8(%rdi) |
| 76 | leaq 64(%rsi),%rsi |
| 77 | leaq 64(%rdi),%rdi |
| 78 | decl %ecx |
| 79 | jnz 1b |
| 80 | 17: movl %edx,%ecx |
Andi Kleen | 0812a57 | 2007-02-13 13:26:19 +0100 | [diff] [blame] | 81 | andl $7,%edx |
Vitaly Mayatskikh | ad2fc2c | 2008-07-02 15:53:13 +0200 | [diff] [blame] | 82 | shrl $3,%ecx |
| 83 | jz 20f |
| 84 | 18: movq (%rsi),%r8 |
| 85 | 19: movnti %r8,(%rdi) |
| 86 | leaq 8(%rsi),%rsi |
| 87 | leaq 8(%rdi),%rdi |
| 88 | decl %ecx |
| 89 | jnz 18b |
| 90 | 20: andl %edx,%edx |
| 91 | jz 23f |
| 92 | movl %edx,%ecx |
| 93 | 21: movb (%rsi),%al |
| 94 | 22: movb %al,(%rdi) |
| 95 | incq %rsi |
| 96 | incq %rdi |
| 97 | decl %ecx |
| 98 | jnz 21b |
| 99 | 23: xorl %eax,%eax |
| 100 | sfence |
| 101 | ret |
| 102 | |
| 103 | .section .fixup,"ax" |
| 104 | 30: shll $6,%ecx |
Andi Kleen | 0812a57 | 2007-02-13 13:26:19 +0100 | [diff] [blame] | 105 | addl %ecx,%edx |
Vitaly Mayatskikh | ad2fc2c | 2008-07-02 15:53:13 +0200 | [diff] [blame] | 106 | jmp 60f |
Jeremy Fitzhardinge | 27cb0a7 | 2008-07-10 12:52:52 -0700 | [diff] [blame^] | 107 | 40: lea (%rdx,%rcx,8),%rdx |
Vitaly Mayatskikh | ad2fc2c | 2008-07-02 15:53:13 +0200 | [diff] [blame] | 108 | jmp 60f |
| 109 | 50: movl %ecx,%edx |
| 110 | 60: sfence |
| 111 | movl %r8d,%ecx |
| 112 | jmp copy_user_handle_tail |
| 113 | .previous |
| 114 | |
| 115 | .section __ex_table,"a" |
| 116 | .quad 1b,30b |
| 117 | .quad 2b,30b |
| 118 | .quad 3b,30b |
| 119 | .quad 4b,30b |
| 120 | .quad 5b,30b |
| 121 | .quad 6b,30b |
| 122 | .quad 7b,30b |
| 123 | .quad 8b,30b |
| 124 | .quad 9b,30b |
| 125 | .quad 10b,30b |
| 126 | .quad 11b,30b |
| 127 | .quad 12b,30b |
| 128 | .quad 13b,30b |
| 129 | .quad 14b,30b |
| 130 | .quad 15b,30b |
| 131 | .quad 16b,30b |
| 132 | .quad 18b,40b |
| 133 | .quad 19b,40b |
| 134 | .quad 21b,50b |
| 135 | .quad 22b,50b |
| 136 | .previous |
Andi Kleen | 0812a57 | 2007-02-13 13:26:19 +0100 | [diff] [blame] | 137 | CFI_ENDPROC |
| 138 | ENDPROC(__copy_user_nocache) |