blob: c7bbae8c590f479f32d6a38db364840f295785d7 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/* bzero.S: Simple prefetching memset, bzero, and clear_user
2 * implementations.
3 *
4 * Copyright (C) 2005 David S. Miller <davem@davemloft.net>
5 */
6
7 .text
8
9 .globl __memset
10 .type __memset, #function
11__memset: /* %o0=buf, %o1=pat, %o2=len */
12
13 .globl memset
14 .type memset, #function
15memset: /* %o0=buf, %o1=pat, %o2=len */
16 and %o1, 0xff, %o3
17 mov %o2, %o1
18 sllx %o3, 8, %g1
19 or %g1, %o3, %o2
20 sllx %o2, 16, %g1
21 or %g1, %o2, %o2
22 sllx %o2, 32, %g1
23 ba,pt %xcc, 1f
24 or %g1, %o2, %o2
25
26 .globl __bzero
27 .type __bzero, #function
28__bzero: /* %o0=buf, %o1=len */
29 clr %o2
301: mov %o0, %o3
31 brz,pn %o1, __bzero_done
32 cmp %o1, 16
33 bl,pn %icc, __bzero_tiny
34 prefetch [%o0 + 0x000], #n_writes
35 andcc %o0, 0x3, %g0
36 be,pt %icc, 2f
371: stb %o2, [%o0 + 0x00]
38 add %o0, 1, %o0
39 andcc %o0, 0x3, %g0
40 bne,pn %icc, 1b
41 sub %o1, 1, %o1
422: andcc %o0, 0x7, %g0
43 be,pt %icc, 3f
44 stw %o2, [%o0 + 0x00]
45 sub %o1, 4, %o1
46 add %o0, 4, %o0
473: and %o1, 0x38, %g1
48 cmp %o1, 0x40
49 andn %o1, 0x3f, %o4
50 bl,pn %icc, 5f
51 and %o1, 0x7, %o1
52 prefetch [%o0 + 0x040], #n_writes
53 prefetch [%o0 + 0x080], #n_writes
54 prefetch [%o0 + 0x0c0], #n_writes
55 prefetch [%o0 + 0x100], #n_writes
56 prefetch [%o0 + 0x140], #n_writes
574: prefetch [%o0 + 0x180], #n_writes
58 stx %o2, [%o0 + 0x00]
59 stx %o2, [%o0 + 0x08]
60 stx %o2, [%o0 + 0x10]
61 stx %o2, [%o0 + 0x18]
62 stx %o2, [%o0 + 0x20]
63 stx %o2, [%o0 + 0x28]
64 stx %o2, [%o0 + 0x30]
65 stx %o2, [%o0 + 0x38]
66 subcc %o4, 0x40, %o4
67 bne,pt %icc, 4b
68 add %o0, 0x40, %o0
69 brz,pn %g1, 6f
70 nop
715: stx %o2, [%o0 + 0x00]
72 subcc %g1, 8, %g1
73 bne,pt %icc, 5b
74 add %o0, 0x8, %o0
756: brz,pt %o1, __bzero_done
76 nop
77__bzero_tiny:
781: stb %o2, [%o0 + 0x00]
79 subcc %o1, 1, %o1
80 bne,pt %icc, 1b
81 add %o0, 1, %o0
82__bzero_done:
83 retl
84 mov %o3, %o0
85 .size __bzero, .-__bzero
86 .size __memset, .-__memset
87 .size memset, .-memset
88
89#define EX_ST(x,y) \
9098: x,y; \
91 .section .fixup; \
92 .align 4; \
9399: retl; \
94 mov %o1, %o0; \
David S. Miller4d000d52006-03-04 23:23:56 -080095 .section __ex_table,"a";\
Linus Torvalds1da177e2005-04-16 15:20:36 -070096 .align 4; \
97 .word 98b, 99b; \
98 .text; \
99 .align 4;
100
David S. Millerc857e3f2006-02-17 10:35:23 -0800101 .globl __clear_user
102 .type __clear_user, #function
103__clear_user: /* %o0=buf, %o1=len */
104 brz,pn %o1, __clear_user_done
Linus Torvalds1da177e2005-04-16 15:20:36 -0700105 cmp %o1, 16
David S. Millerc857e3f2006-02-17 10:35:23 -0800106 bl,pn %icc, __clear_user_tiny
Linus Torvalds1da177e2005-04-16 15:20:36 -0700107 EX_ST(prefetcha [%o0 + 0x00] %asi, #n_writes)
108 andcc %o0, 0x3, %g0
109 be,pt %icc, 2f
1101: EX_ST(stba %g0, [%o0 + 0x00] %asi)
111 add %o0, 1, %o0
112 andcc %o0, 0x3, %g0
113 bne,pn %icc, 1b
114 sub %o1, 1, %o1
1152: andcc %o0, 0x7, %g0
116 be,pt %icc, 3f
117 EX_ST(stwa %g0, [%o0 + 0x00] %asi)
118 sub %o1, 4, %o1
119 add %o0, 4, %o0
1203: and %o1, 0x38, %g1
121 cmp %o1, 0x40
122 andn %o1, 0x3f, %o4
123 bl,pn %icc, 5f
124 and %o1, 0x7, %o1
125 EX_ST(prefetcha [%o0 + 0x040] %asi, #n_writes)
126 EX_ST(prefetcha [%o0 + 0x080] %asi, #n_writes)
127 EX_ST(prefetcha [%o0 + 0x0c0] %asi, #n_writes)
128 EX_ST(prefetcha [%o0 + 0x100] %asi, #n_writes)
129 EX_ST(prefetcha [%o0 + 0x140] %asi, #n_writes)
1304: EX_ST(prefetcha [%o0 + 0x180] %asi, #n_writes)
131 EX_ST(stxa %g0, [%o0 + 0x00] %asi)
132 EX_ST(stxa %g0, [%o0 + 0x08] %asi)
133 EX_ST(stxa %g0, [%o0 + 0x10] %asi)
134 EX_ST(stxa %g0, [%o0 + 0x18] %asi)
135 EX_ST(stxa %g0, [%o0 + 0x20] %asi)
136 EX_ST(stxa %g0, [%o0 + 0x28] %asi)
137 EX_ST(stxa %g0, [%o0 + 0x30] %asi)
138 EX_ST(stxa %g0, [%o0 + 0x38] %asi)
139 subcc %o4, 0x40, %o4
140 bne,pt %icc, 4b
141 add %o0, 0x40, %o0
142 brz,pn %g1, 6f
143 nop
1445: EX_ST(stxa %g0, [%o0 + 0x00] %asi)
145 subcc %g1, 8, %g1
146 bne,pt %icc, 5b
147 add %o0, 0x8, %o0
David S. Millerc857e3f2006-02-17 10:35:23 -08001486: brz,pt %o1, __clear_user_done
Linus Torvalds1da177e2005-04-16 15:20:36 -0700149 nop
David S. Millerc857e3f2006-02-17 10:35:23 -0800150__clear_user_tiny:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001511: EX_ST(stba %g0, [%o0 + 0x00] %asi)
152 subcc %o1, 1, %o1
153 bne,pt %icc, 1b
154 add %o0, 1, %o0
David S. Millerc857e3f2006-02-17 10:35:23 -0800155__clear_user_done:
Linus Torvalds1da177e2005-04-16 15:20:36 -0700156 retl
157 clr %o0
David S. Millerc857e3f2006-02-17 10:35:23 -0800158 .size __clear_user, .-__clear_user