Gitiles
Code Review Sign In
gerrit-public.fairphone.software / kernel / msm-4.9 / 95db505125fb7bc624b7c3b6747bbeaebbffc2e4 / . / arch / x86 / lib / copy_page_64.S
blob: 727a5d46d2fc0cfb6888053a698841fb6e9732f9 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -0700[diff] [blame]1/* Written 2003 by Andi Kleen, based on a kernel by Evandro Menezes */
Dave Jones038b0a62006-10-04 03:38:54 -0400[diff] [blame]2
Jan Beulich8d379da2006-09-26 10:52:32 +0200[diff] [blame]3#include <linux/linkage.h>
4#include <asm/dwarf2.h>
5
6 ALIGN
7copy_page_c:
8 CFI_STARTPROC
9 movl $4096/8,%ecx
10 rep movsq
11 ret
12 CFI_ENDPROC
13ENDPROC(copy_page_c)
14
Linus Torvalds1da177e2005-04-16 15:20:36 -0700[diff] [blame]15/* Don't use streaming store because it's better when the target
16 ends up in cache. */
17
18/* Could vary the prefetch distance based on SMP/UP */
19
Jan Beulich8d379da2006-09-26 10:52:32 +0200[diff] [blame]20ENTRY(copy_page)
21 CFI_STARTPROC
Andi Kleen7bcd3f32006-02-03 21:51:02 +0100[diff] [blame]22 subq $3*8,%rsp
Jan Beulich8d379da2006-09-26 10:52:32 +0200[diff] [blame]23 CFI_ADJUST_CFA_OFFSET 3*8
Andi Kleen7bcd3f32006-02-03 21:51:02 +0100[diff] [blame]24 movq %rbx,(%rsp)
Jan Beulich8d379da2006-09-26 10:52:32 +0200[diff] [blame]25 CFI_REL_OFFSET rbx, 0
Andi Kleen7bcd3f32006-02-03 21:51:02 +0100[diff] [blame]26 movq %r12,1*8(%rsp)
Jan Beulich8d379da2006-09-26 10:52:32 +0200[diff] [blame]27 CFI_REL_OFFSET r12, 1*8
Andi Kleen7bcd3f32006-02-03 21:51:02 +0100[diff] [blame]28 movq %r13,2*8(%rsp)
Jan Beulich8d379da2006-09-26 10:52:32 +0200[diff] [blame]29 CFI_REL_OFFSET r13, 2*8
Andi Kleen7bcd3f32006-02-03 21:51:02 +0100[diff] [blame]30
31 movl $(4096/64)-5,%ecx
32 .p2align 4
33.Loop64:
34 dec %rcx
35
36 movq (%rsi), %rax
37 movq 8 (%rsi), %rbx
38 movq 16 (%rsi), %rdx
39 movq 24 (%rsi), %r8
40 movq 32 (%rsi), %r9
41 movq 40 (%rsi), %r10
42 movq 48 (%rsi), %r11
43 movq 56 (%rsi), %r12
44
45 prefetcht0 5*64(%rsi)
46
47 movq %rax, (%rdi)
48 movq %rbx, 8 (%rdi)
49 movq %rdx, 16 (%rdi)
50 movq %r8, 24 (%rdi)
51 movq %r9, 32 (%rdi)
52 movq %r10, 40 (%rdi)
53 movq %r11, 48 (%rdi)
54 movq %r12, 56 (%rdi)
55
56 leaq 64 (%rsi), %rsi
57 leaq 64 (%rdi), %rdi
58
59 jnz .Loop64
60
61 movl $5,%ecx
62 .p2align 4
63.Loop2:
64 decl %ecx
65
66 movq (%rsi), %rax
67 movq 8 (%rsi), %rbx
68 movq 16 (%rsi), %rdx
69 movq 24 (%rsi), %r8
70 movq 32 (%rsi), %r9
71 movq 40 (%rsi), %r10
72 movq 48 (%rsi), %r11
73 movq 56 (%rsi), %r12
74
75 movq %rax, (%rdi)
76 movq %rbx, 8 (%rdi)
77 movq %rdx, 16 (%rdi)
78 movq %r8, 24 (%rdi)
79 movq %r9, 32 (%rdi)
80 movq %r10, 40 (%rdi)
81 movq %r11, 48 (%rdi)
82 movq %r12, 56 (%rdi)
83
84 leaq 64(%rdi),%rdi
85 leaq 64(%rsi),%rsi
86
87 jnz .Loop2
88
89 movq (%rsp),%rbx
Jan Beulich8d379da2006-09-26 10:52:32 +0200[diff] [blame]90 CFI_RESTORE rbx
Andi Kleen7bcd3f32006-02-03 21:51:02 +0100[diff] [blame]91 movq 1*8(%rsp),%r12
Jan Beulich8d379da2006-09-26 10:52:32 +0200[diff] [blame]92 CFI_RESTORE r12
Andi Kleen7bcd3f32006-02-03 21:51:02 +0100[diff] [blame]93 movq 2*8(%rsp),%r13
Jan Beulich8d379da2006-09-26 10:52:32 +0200[diff] [blame]94 CFI_RESTORE r13
Andi Kleen7bcd3f32006-02-03 21:51:02 +0100[diff] [blame]95 addq $3*8,%rsp
Jan Beulich8d379da2006-09-26 10:52:32 +0200[diff] [blame]96 CFI_ADJUST_CFA_OFFSET -3*8
Andi Kleen7bcd3f32006-02-03 21:51:02 +0100[diff] [blame]97 ret
Jan Beulich8d379da2006-09-26 10:52:32 +0200[diff] [blame]98.Lcopy_page_end:
99 CFI_ENDPROC
100ENDPROC(copy_page)
Andi Kleen7bcd3f32006-02-03 21:51:02 +0100[diff] [blame]101
102 /* Some CPUs run faster using the string copy instructions.
103 It is also a lot simpler. Use this when possible */
104
105#include <asm/cpufeature.h>
106
Jan Beulich8d379da2006-09-26 10:52:32 +0200[diff] [blame]107 .section .altinstr_replacement,"ax"
1081: .byte 0xeb /* jmp <disp8> */
109 .byte (copy_page_c - copy_page) - (2f - 1b) /* offset */
1102:
111 .previous
Andi Kleen7bcd3f32006-02-03 21:51:02 +0100[diff] [blame]112 .section .altinstructions,"a"
113 .align 8
Jan Beulich8d379da2006-09-26 10:52:32 +0200[diff] [blame]114 .quad copy_page
115 .quad 1b
116 .byte X86_FEATURE_REP_GOOD
117 .byte .Lcopy_page_end - copy_page
118 .byte 2b - 1b
Andi Kleen7bcd3f32006-02-03 21:51:02 +0100[diff] [blame]119 .previous