Gitiles
Code Review Sign In
gerrit-public.fairphone.software / kernel / msm / 869512ab5ab93e5e82ad7d4aaf4ed098d23bfc3f / . / arch / sparc64 / lib / csum_copy.S
blob: e566c770a0f6372de5e4e7856ff520a9ac8920ac [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -0700[diff] [blame]1/* csum_copy.S: Checksum+copy code for sparc64
2 *
3 * Copyright (C) 2005 David S. Miller <davem@davemloft.net>
4 */
5
6#ifdef __KERNEL__
7#define GLOBAL_SPARE %g7
8#else
9#define GLOBAL_SPARE %g5
10#endif
11
12#ifndef EX_LD
13#define EX_LD(x) x
14#endif
15
16#ifndef EX_ST
17#define EX_ST(x) x
18#endif
19
20#ifndef EX_RETVAL
21#define EX_RETVAL(x) x
22#endif
23
24#ifndef LOAD
25#define LOAD(type,addr,dest) type [addr], dest
26#endif
27
28#ifndef STORE
29#define STORE(type,src,addr) type src, [addr]
30#endif
31
32#ifndef FUNC_NAME
33#define FUNC_NAME csum_partial_copy_nocheck
34#endif
35
36 .register %g2, #scratch
37 .register %g3, #scratch
38
39 .text
40
4190:
42 /* We checked for zero length already, so there must be
43 * at least one byte.
44 */
45 be,pt %icc, 1f
46 nop
47 EX_LD(LOAD(ldub, %o0 + 0x00, %o4))
48 add %o0, 1, %o0
49 sub %o2, 1, %o2
50 EX_ST(STORE(stb, %o4, %o1 + 0x00))
51 add %o1, 1, %o1
521: andcc %o0, 0x2, %g0
53 be,pn %icc, 80f
54 cmp %o2, 2
55 blu,pn %icc, 60f
56 nop
57 EX_LD(LOAD(lduh, %o0 + 0x00, %o5))
58 add %o0, 2, %o0
59 sub %o2, 2, %o2
60 EX_ST(STORE(sth, %o5, %o1 + 0x00))
61 add %o1, 2, %o1
62 ba,pt %xcc, 80f
63 add %o5, %o4, %o4
64
65 .globl FUNC_NAME
66FUNC_NAME: /* %o0=src, %o1=dst, %o2=len, %o3=sum */
67 LOAD(prefetch, %o0 + 0x000, #n_reads)
68 xor %o0, %o1, %g1
69 clr %o4
70 andcc %g1, 0x3, %g0
71 bne,pn %icc, 95f
72 LOAD(prefetch, %o0 + 0x040, #n_reads)
73
74 brz,pn %o2, 70f
75 andcc %o0, 0x3, %g0
76
77 /* We "remember" whether the lowest bit in the address
78 * was set in GLOBAL_SPARE. Because if it is, we have to swap
79 * upper and lower 8 bit fields of the sum we calculate.
80 */
81 bne,pn %icc, 90b
82 andcc %o0, 0x1, GLOBAL_SPARE
83
8480:
85 LOAD(prefetch, %o0 + 0x080, #n_reads)
86 andncc %o2, 0x3f, %g3
87
88 LOAD(prefetch, %o0 + 0x0c0, #n_reads)
89 sub %o2, %g3, %o2
90 brz,pn %g3, 2f
91 LOAD(prefetch, %o0 + 0x100, #n_reads)
92
93 /* So that we don't need to use the non-pairing
94 * add-with-carry instructions we accumulate 32-bit
95 * values into a 64-bit register. At the end of the
96 * loop we fold it down to 32-bits and so on.
97 */
98 ba,pt %xcc, 1f
99 LOAD(prefetch, %o0 + 0x140, #n_reads)
100
101 .align 32
1021: EX_LD(LOAD(lduw, %o0 + 0x00, %o5))
103 EX_LD(LOAD(lduw, %o0 + 0x04, %g1))
104 EX_LD(LOAD(lduw, %o0 + 0x08, %g2))
105 add %o4, %o5, %o4
106 EX_ST(STORE(stw, %o5, %o1 + 0x00))
107 EX_LD(LOAD(lduw, %o0 + 0x0c, %o5))
108 add %o4, %g1, %o4
109 EX_ST(STORE(stw, %g1, %o1 + 0x04))
110 EX_LD(LOAD(lduw, %o0 + 0x10, %g1))
111 add %o4, %g2, %o4
112 EX_ST(STORE(stw, %g2, %o1 + 0x08))
113 EX_LD(LOAD(lduw, %o0 + 0x14, %g2))
114 add %o4, %o5, %o4
115 EX_ST(STORE(stw, %o5, %o1 + 0x0c))
116 EX_LD(LOAD(lduw, %o0 + 0x18, %o5))
117 add %o4, %g1, %o4
118 EX_ST(STORE(stw, %g1, %o1 + 0x10))
119 EX_LD(LOAD(lduw, %o0 + 0x1c, %g1))
120 add %o4, %g2, %o4
121 EX_ST(STORE(stw, %g2, %o1 + 0x14))
122 EX_LD(LOAD(lduw, %o0 + 0x20, %g2))
123 add %o4, %o5, %o4
124 EX_ST(STORE(stw, %o5, %o1 + 0x18))
125 EX_LD(LOAD(lduw, %o0 + 0x24, %o5))
126 add %o4, %g1, %o4
127 EX_ST(STORE(stw, %g1, %o1 + 0x1c))
128 EX_LD(LOAD(lduw, %o0 + 0x28, %g1))
129 add %o4, %g2, %o4
130 EX_ST(STORE(stw, %g2, %o1 + 0x20))
131 EX_LD(LOAD(lduw, %o0 + 0x2c, %g2))
132 add %o4, %o5, %o4
133 EX_ST(STORE(stw, %o5, %o1 + 0x24))
134 EX_LD(LOAD(lduw, %o0 + 0x30, %o5))
135 add %o4, %g1, %o4
136 EX_ST(STORE(stw, %g1, %o1 + 0x28))
137 EX_LD(LOAD(lduw, %o0 + 0x34, %g1))
138 add %o4, %g2, %o4
139 EX_ST(STORE(stw, %g2, %o1 + 0x2c))
140 EX_LD(LOAD(lduw, %o0 + 0x38, %g2))
141 add %o4, %o5, %o4
142 EX_ST(STORE(stw, %o5, %o1 + 0x30))
143 EX_LD(LOAD(lduw, %o0 + 0x3c, %o5))
144 add %o4, %g1, %o4
145 EX_ST(STORE(stw, %g1, %o1 + 0x34))
146 LOAD(prefetch, %o0 + 0x180, #n_reads)
147 add %o4, %g2, %o4
148 EX_ST(STORE(stw, %g2, %o1 + 0x38))
149 subcc %g3, 0x40, %g3
150 add %o0, 0x40, %o0
151 add %o4, %o5, %o4
152 EX_ST(STORE(stw, %o5, %o1 + 0x3c))
153 bne,pt %icc, 1b
154 add %o1, 0x40, %o1
155
1562: and %o2, 0x3c, %g3
157 brz,pn %g3, 2f
158 sub %o2, %g3, %o2
1591: EX_LD(LOAD(lduw, %o0 + 0x00, %o5))
160 subcc %g3, 0x4, %g3
161 add %o0, 0x4, %o0
162 add %o4, %o5, %o4
163 EX_ST(STORE(stw, %o5, %o1 + 0x00))
164 bne,pt %icc, 1b
165 add %o1, 0x4, %o1
166
1672:
168 /* fold 64-->32 */
169 srlx %o4, 32, %o5
170 srl %o4, 0, %o4
171 add %o4, %o5, %o4
172 srlx %o4, 32, %o5
173 srl %o4, 0, %o4
174 add %o4, %o5, %o4
175
176 /* fold 32-->16 */
177 sethi %hi(0xffff0000), %g1
178 srl %o4, 16, %o5
179 andn %o4, %g1, %g2
180 add %o5, %g2, %o4
181 srl %o4, 16, %o5
182 andn %o4, %g1, %g2
183 add %o5, %g2, %o4
184
18560:
186 /* %o4 has the 16-bit sum we have calculated so-far. */
187 cmp %o2, 2
188 blu,pt %icc, 1f
189 nop
190 EX_LD(LOAD(lduh, %o0 + 0x00, %o5))
191 sub %o2, 2, %o2
192 add %o0, 2, %o0
193 add %o4, %o5, %o4
194 EX_ST(STORE(sth, %o5, %o1 + 0x00))
195 add %o1, 0x2, %o1
1961: brz,pt %o2, 1f
197 nop
198 EX_LD(LOAD(ldub, %o0 + 0x00, %o5))
199 sub %o2, 1, %o2
200 add %o0, 1, %o0
201 EX_ST(STORE(stb, %o5, %o1 + 0x00))
202 sllx %o5, 8, %o5
203 add %o1, 1, %o1
204 add %o4, %o5, %o4
2051:
206 /* fold 32-->16 */
207 sethi %hi(0xffff0000), %g1
208 srl %o4, 16, %o5
209 andn %o4, %g1, %g2
210 add %o5, %g2, %o4
211 srl %o4, 16, %o5
212 andn %o4, %g1, %g2
213 add %o5, %g2, %o4
214
2151: brz,pt GLOBAL_SPARE, 1f
216 nop
217
218 /* We started with an odd byte, byte-swap the result. */
219 srl %o4, 8, %o5
220 and %o4, 0xff, %g1
221 sll %g1, 8, %g1
222 or %o5, %g1, %o4
223
David S. Millerae5de0f2006-06-04 21:32:01 -0700[diff] [blame]2241: addcc %o3, %o4, %o3
225 addc %g0, %o3, %o3
Linus Torvalds1da177e2005-04-16 15:20:36 -0700[diff] [blame]226
22770:
228 retl
David S. Millerae5de0f2006-06-04 21:32:01 -0700[diff] [blame]229 srl %o3, 0, %o0
Linus Torvalds1da177e2005-04-16 15:20:36 -0700[diff] [blame]230
23195: mov 0, GLOBAL_SPARE
232 brlez,pn %o2, 4f
233 andcc %o0, 1, %o5
234 be,a,pt %icc, 1f
235 srl %o2, 1, %g1
236 sub %o2, 1, %o2
237 EX_LD(LOAD(ldub, %o0, GLOBAL_SPARE))
238 add %o0, 1, %o0
239 EX_ST(STORE(stb, GLOBAL_SPARE, %o1))
240 srl %o2, 1, %g1
241 add %o1, 1, %o1
2421: brz,a,pn %g1, 3f
243 andcc %o2, 1, %g0
244 andcc %o0, 2, %g0
245 be,a,pt %icc, 1f
246 srl %g1, 1, %g1
247 EX_LD(LOAD(lduh, %o0, %o4))
248 sub %o2, 2, %o2
249 srl %o4, 8, %g2
250 sub %g1, 1, %g1
251 EX_ST(STORE(stb, %g2, %o1))
252 add %o4, GLOBAL_SPARE, GLOBAL_SPARE
253 EX_ST(STORE(stb, %o4, %o1 + 1))
254 add %o0, 2, %o0
255 srl %g1, 1, %g1
256 add %o1, 2, %o1
2571: brz,a,pn %g1, 2f
258 andcc %o2, 2, %g0
259 EX_LD(LOAD(lduw, %o0, %o4))
2605: srl %o4, 24, %g2
261 srl %o4, 16, %g3
262 EX_ST(STORE(stb, %g2, %o1))
263 srl %o4, 8, %g2
264 EX_ST(STORE(stb, %g3, %o1 + 1))
265 add %o0, 4, %o0
266 EX_ST(STORE(stb, %g2, %o1 + 2))
267 addcc %o4, GLOBAL_SPARE, GLOBAL_SPARE
268 EX_ST(STORE(stb, %o4, %o1 + 3))
269 addc GLOBAL_SPARE, %g0, GLOBAL_SPARE
270 add %o1, 4, %o1
271 subcc %g1, 1, %g1
272 bne,a,pt %icc, 5b
273 EX_LD(LOAD(lduw, %o0, %o4))
274 sll GLOBAL_SPARE, 16, %g2
275 srl GLOBAL_SPARE, 16, GLOBAL_SPARE
276 srl %g2, 16, %g2
277 andcc %o2, 2, %g0
278 add %g2, GLOBAL_SPARE, GLOBAL_SPARE
2792: be,a,pt %icc, 3f
280 andcc %o2, 1, %g0
281 EX_LD(LOAD(lduh, %o0, %o4))
282 andcc %o2, 1, %g0
283 srl %o4, 8, %g2
284 add %o0, 2, %o0
285 EX_ST(STORE(stb, %g2, %o1))
286 add GLOBAL_SPARE, %o4, GLOBAL_SPARE
287 EX_ST(STORE(stb, %o4, %o1 + 1))
288 add %o1, 2, %o1
2893: be,a,pt %icc, 1f
290 sll GLOBAL_SPARE, 16, %o4
291 EX_LD(LOAD(ldub, %o0, %g2))
292 sll %g2, 8, %o4
293 EX_ST(STORE(stb, %g2, %o1))
294 add GLOBAL_SPARE, %o4, GLOBAL_SPARE
295 sll GLOBAL_SPARE, 16, %o4
2961: addcc %o4, GLOBAL_SPARE, GLOBAL_SPARE
297 srl GLOBAL_SPARE, 16, %o4
298 addc %g0, %o4, GLOBAL_SPARE
299 brz,pt %o5, 4f
300 srl GLOBAL_SPARE, 8, %o4
301 and GLOBAL_SPARE, 0xff, %g2
302 and %o4, 0xff, %o4
303 sll %g2, 8, %g2
304 or %g2, %o4, GLOBAL_SPARE
3054: addcc %o3, GLOBAL_SPARE, %o3
306 addc %g0, %o3, %o0
307 retl
308 srl %o0, 0, %o0
309 .size FUNC_NAME, .-FUNC_NAME