Gitiles
Code Review Sign In
gerrit-public.fairphone.software / kernel / msm-4.9 / cea3235cf578b5e952f5a0cec9bc6c2e862eb697 / . / arch / x86 / lib / memset_64.S
blob: 09d3442696522e19aea8adb9efdbfd051d8d57aa [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -0700[diff] [blame]1/* Copyright 2002 Andi Kleen, SuSE Labs */
Jan Beulich8d379da2006-09-26 10:52:32 +0200[diff] [blame]2
Jan Beulich8d379da2006-09-26 10:52:32 +0200[diff] [blame]3#include <linux/linkage.h>
4#include <asm/dwarf2.h>
5
Linus Torvalds1da177e2005-04-16 15:20:36 -0700[diff] [blame]6/*
7 * ISO C memset - set a memory block to a byte value.
8 *
9 * rdi destination
10 * rsi value (char)
11 * rdx count (bytes)
12 *
13 * rax original destination
14 */
Jan Beulich7269e882009-12-18 16:16:03 +0000[diff] [blame]15 .section .altinstr_replacement, "ax", @progbits
16.Lmemset_c:
Jan Beulich8d379da2006-09-26 10:52:32 +0200[diff] [blame]17 movq %rdi,%r9
18 movl %edx,%r8d
19 andl $7,%r8d
20 movl %edx,%ecx
21 shrl $3,%ecx
22 /* expand byte value */
23 movzbl %sil,%esi
24 movabs $0x0101010101010101,%rax
25 mulq %rsi /* with rax, clobbers rdx */
26 rep stosq
27 movl %r8d,%ecx
28 rep stosb
29 movq %r9,%rax
30 ret
Jan Beulich7269e882009-12-18 16:16:03 +0000[diff] [blame]31.Lmemset_e:
32 .previous
Jan Beulich8d379da2006-09-26 10:52:32 +0200[diff] [blame]33
34ENTRY(memset)
35ENTRY(__memset)
36 CFI_STARTPROC
Andi Kleen7bcd3f32006-02-03 21:51:02 +0100[diff] [blame]37 movq %rdi,%r10
38 movq %rdx,%r11
39
40 /* expand byte value */
41 movzbl %sil,%ecx
42 movabs $0x0101010101010101,%rax
43 mul %rcx /* with rax, clobbers rdx */
44
45 /* align dst */
46 movl %edi,%r9d
47 andl $7,%r9d
48 jnz .Lbad_alignment
Jan Beulich8d379da2006-09-26 10:52:32 +0200[diff] [blame]49 CFI_REMEMBER_STATE
Andi Kleen7bcd3f32006-02-03 21:51:02 +0100[diff] [blame]50.Lafter_bad_alignment:
51
52 movl %r11d,%ecx
53 shrl $6,%ecx
54 jz .Lhandle_tail
55
56 .p2align 4
57.Lloop_64:
58 decl %ecx
59 movq %rax,(%rdi)
60 movq %rax,8(%rdi)
61 movq %rax,16(%rdi)
62 movq %rax,24(%rdi)
63 movq %rax,32(%rdi)
64 movq %rax,40(%rdi)
65 movq %rax,48(%rdi)
66 movq %rax,56(%rdi)
67 leaq 64(%rdi),%rdi
68 jnz .Lloop_64
69
70 /* Handle tail in loops. The loops should be faster than hard
71 to predict jump tables. */
72 .p2align 4
73.Lhandle_tail:
74 movl %r11d,%ecx
75 andl $63&(~7),%ecx
76 jz .Lhandle_7
77 shrl $3,%ecx
78 .p2align 4
79.Lloop_8:
80 decl %ecx
81 movq %rax,(%rdi)
82 leaq 8(%rdi),%rdi
83 jnz .Lloop_8
84
85.Lhandle_7:
86 movl %r11d,%ecx
87 andl $7,%ecx
88 jz .Lende
89 .p2align 4
90.Lloop_1:
91 decl %ecx
92 movb %al,(%rdi)
93 leaq 1(%rdi),%rdi
94 jnz .Lloop_1
95
96.Lende:
97 movq %r10,%rax
98 ret
99
Jan Beulich8d379da2006-09-26 10:52:32 +0200[diff] [blame]100 CFI_RESTORE_STATE
Andi Kleen7bcd3f32006-02-03 21:51:02 +0100[diff] [blame]101.Lbad_alignment:
102 cmpq $7,%r11
103 jbe .Lhandle_7
104 movq %rax,(%rdi) /* unaligned store */
105 movq $8,%r8
106 subq %r9,%r8
107 addq %r8,%rdi
108 subq %r8,%r11
109 jmp .Lafter_bad_alignment
Jan Beulich8d379da2006-09-26 10:52:32 +0200[diff] [blame]110.Lfinal:
111 CFI_ENDPROC
112ENDPROC(memset)
113ENDPROC(__memset)
Andi Kleen7bcd3f32006-02-03 21:51:02 +0100[diff] [blame]114
115 /* Some CPUs run faster using the string instructions.
116 It is also a lot simpler. Use this when possible */
117
118#include <asm/cpufeature.h>
119
120 .section .altinstructions,"a"
121 .align 8
Jan Beulich8d379da2006-09-26 10:52:32 +0200[diff] [blame]122 .quad memset
Jan Beulich7269e882009-12-18 16:16:03 +0000[diff] [blame]123 .quad .Lmemset_c
H. Peter Anvin83a7a2a2010-06-10 00:10:43 +0000[diff] [blame]124 .word X86_FEATURE_REP_GOOD
Jan Beulich8d379da2006-09-26 10:52:32 +0200[diff] [blame]125 .byte .Lfinal - memset
Jan Beulich7269e882009-12-18 16:16:03 +0000[diff] [blame]126 .byte .Lmemset_e - .Lmemset_c
Andi Kleen7bcd3f32006-02-03 21:51:02 +0100[diff] [blame]127 .previous