Gitiles
Code Review Sign In
gerrit-public.fairphone.software / kernel / msm-4.9 / 4b8b8a4afa4f794a0029cdf2c13b6789fd38d9de / . / arch / x86 / lib / copy_page_64.S
blob: 176cca67212b7072687f42450ef56018c03ebaaa [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -0700[diff] [blame]1/* Written 2003 by Andi Kleen, based on a kernel by Evandro Menezes */
Dave Jones038b0a62006-10-04 03:38:54 -0400[diff] [blame]2
Jan Beulich8d379da2006-09-26 10:52:32 +0200[diff] [blame]3#include <linux/linkage.h>
4#include <asm/dwarf2.h>
Andy Lutomirski59e97e42011-07-13 09:24:10 -0400[diff] [blame]5#include <asm/alternative-asm.h>
Jan Beulich8d379da2006-09-26 10:52:32 +0200[diff] [blame]6
7 ALIGN
Ma Ling269833b2012-10-18 03:52:45 +0800[diff] [blame]8copy_page_rep:
Jan Beulich8d379da2006-09-26 10:52:32 +0200[diff] [blame]9 CFI_STARTPROC
Ma Ling269833b2012-10-18 03:52:45 +0800[diff] [blame]10 movl $4096/8, %ecx
11 rep movsq
Jan Beulich8d379da2006-09-26 10:52:32 +0200[diff] [blame]12 ret
13 CFI_ENDPROC
Ma Ling269833b2012-10-18 03:52:45 +0800[diff] [blame]14ENDPROC(copy_page_rep)
Jan Beulich8d379da2006-09-26 10:52:32 +0200[diff] [blame]15
Ma Ling269833b2012-10-18 03:52:45 +0800[diff] [blame]16/*
17 * Don't use streaming copy unless the CPU indicates X86_FEATURE_REP_GOOD.
18 * Could vary the prefetch distance based on SMP/UP.
19*/
Linus Torvalds1da177e2005-04-16 15:20:36 -0700[diff] [blame]20
Jan Beulich8d379da2006-09-26 10:52:32 +0200[diff] [blame]21ENTRY(copy_page)
22 CFI_STARTPROC
Ma Ling269833b2012-10-18 03:52:45 +0800[diff] [blame]23 subq $2*8, %rsp
Jan Beulich42693292012-01-05 16:12:25 +0000[diff] [blame]24 CFI_ADJUST_CFA_OFFSET 2*8
Ma Ling269833b2012-10-18 03:52:45 +0800[diff] [blame]25 movq %rbx, (%rsp)
Jan Beulich8d379da2006-09-26 10:52:32 +0200[diff] [blame]26 CFI_REL_OFFSET rbx, 0
Ma Ling269833b2012-10-18 03:52:45 +0800[diff] [blame]27 movq %r12, 1*8(%rsp)
Jan Beulich8d379da2006-09-26 10:52:32 +0200[diff] [blame]28 CFI_REL_OFFSET r12, 1*8
Andi Kleen7bcd3f32006-02-03 21:51:02 +0100[diff] [blame]29
Ma Ling269833b2012-10-18 03:52:45 +0800[diff] [blame]30 movl $(4096/64)-5, %ecx
Andi Kleen7bcd3f32006-02-03 21:51:02 +0100[diff] [blame]31 .p2align 4
32.Loop64:
Ma Ling269833b2012-10-18 03:52:45 +0800[diff] [blame]33 dec %rcx
34 movq 0x8*0(%rsi), %rax
35 movq 0x8*1(%rsi), %rbx
36 movq 0x8*2(%rsi), %rdx
37 movq 0x8*3(%rsi), %r8
38 movq 0x8*4(%rsi), %r9
39 movq 0x8*5(%rsi), %r10
40 movq 0x8*6(%rsi), %r11
41 movq 0x8*7(%rsi), %r12
Andi Kleen7bcd3f32006-02-03 21:51:02 +0100[diff] [blame]42
43 prefetcht0 5*64(%rsi)
44
Ma Ling269833b2012-10-18 03:52:45 +0800[diff] [blame]45 movq %rax, 0x8*0(%rdi)
46 movq %rbx, 0x8*1(%rdi)
47 movq %rdx, 0x8*2(%rdi)
48 movq %r8, 0x8*3(%rdi)
49 movq %r9, 0x8*4(%rdi)
50 movq %r10, 0x8*5(%rdi)
51 movq %r11, 0x8*6(%rdi)
52 movq %r12, 0x8*7(%rdi)
Andi Kleen7bcd3f32006-02-03 21:51:02 +0100[diff] [blame]53
Ma Ling269833b2012-10-18 03:52:45 +0800[diff] [blame]54 leaq 64 (%rsi), %rsi
55 leaq 64 (%rdi), %rdi
Andi Kleen7bcd3f32006-02-03 21:51:02 +0100[diff] [blame]56
Ma Ling269833b2012-10-18 03:52:45 +0800[diff] [blame]57 jnz .Loop64
Andi Kleen7bcd3f32006-02-03 21:51:02 +0100[diff] [blame]58
Ma Ling269833b2012-10-18 03:52:45 +0800[diff] [blame]59 movl $5, %ecx
Andi Kleen7bcd3f32006-02-03 21:51:02 +0100[diff] [blame]60 .p2align 4
61.Loop2:
Ma Ling269833b2012-10-18 03:52:45 +0800[diff] [blame]62 decl %ecx
Andi Kleen7bcd3f32006-02-03 21:51:02 +0100[diff] [blame]63
Ma Ling269833b2012-10-18 03:52:45 +0800[diff] [blame]64 movq 0x8*0(%rsi), %rax
65 movq 0x8*1(%rsi), %rbx
66 movq 0x8*2(%rsi), %rdx
67 movq 0x8*3(%rsi), %r8
68 movq 0x8*4(%rsi), %r9
69 movq 0x8*5(%rsi), %r10
70 movq 0x8*6(%rsi), %r11
71 movq 0x8*7(%rsi), %r12
Andi Kleen7bcd3f32006-02-03 21:51:02 +0100[diff] [blame]72
Ma Ling269833b2012-10-18 03:52:45 +0800[diff] [blame]73 movq %rax, 0x8*0(%rdi)
74 movq %rbx, 0x8*1(%rdi)
75 movq %rdx, 0x8*2(%rdi)
76 movq %r8, 0x8*3(%rdi)
77 movq %r9, 0x8*4(%rdi)
78 movq %r10, 0x8*5(%rdi)
79 movq %r11, 0x8*6(%rdi)
80 movq %r12, 0x8*7(%rdi)
Andi Kleen7bcd3f32006-02-03 21:51:02 +0100[diff] [blame]81
Ma Ling269833b2012-10-18 03:52:45 +0800[diff] [blame]82 leaq 64(%rdi), %rdi
83 leaq 64(%rsi), %rsi
Andi Kleen7bcd3f32006-02-03 21:51:02 +0100[diff] [blame]84 jnz .Loop2
85
Ma Ling269833b2012-10-18 03:52:45 +0800[diff] [blame]86 movq (%rsp), %rbx
Jan Beulich8d379da2006-09-26 10:52:32 +0200[diff] [blame]87 CFI_RESTORE rbx
Ma Ling269833b2012-10-18 03:52:45 +0800[diff] [blame]88 movq 1*8(%rsp), %r12
Jan Beulich8d379da2006-09-26 10:52:32 +0200[diff] [blame]89 CFI_RESTORE r12
Ma Ling269833b2012-10-18 03:52:45 +0800[diff] [blame]90 addq $2*8, %rsp
Jan Beulich42693292012-01-05 16:12:25 +0000[diff] [blame]91 CFI_ADJUST_CFA_OFFSET -2*8
Andi Kleen7bcd3f32006-02-03 21:51:02 +0100[diff] [blame]92 ret
Jan Beulich8d379da2006-09-26 10:52:32 +0200[diff] [blame]93.Lcopy_page_end:
94 CFI_ENDPROC
95ENDPROC(copy_page)
Andi Kleen7bcd3f32006-02-03 21:51:02 +0100[diff] [blame]96
97 /* Some CPUs run faster using the string copy instructions.
98 It is also a lot simpler. Use this when possible */
99
100#include <asm/cpufeature.h>
101
Jan Beulich8d379da2006-09-26 10:52:32 +0200[diff] [blame]102 .section .altinstr_replacement,"ax"
1031: .byte 0xeb /* jmp <disp8> */
Ma Ling269833b2012-10-18 03:52:45 +0800[diff] [blame]104 .byte (copy_page_rep - copy_page) - (2f - 1b) /* offset */
Jan Beulich8d379da2006-09-26 10:52:32 +0200[diff] [blame]1052:
106 .previous
Andi Kleen7bcd3f32006-02-03 21:51:02 +0100[diff] [blame]107 .section .altinstructions,"a"
Andy Lutomirski59e97e42011-07-13 09:24:10 -0400[diff] [blame]108 altinstruction_entry copy_page, 1b, X86_FEATURE_REP_GOOD, \
109 .Lcopy_page_end-copy_page, 2b-1b
Andi Kleen7bcd3f32006-02-03 21:51:02 +0100[diff] [blame]110 .previous