blob: 1a431a5cf86ffb8d57f1b3adc417c4985818d055 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 *
3 * Optmized version of the standard do_csum() function
4 *
5 * Return: a 64bit quantity containing the 16bit Internet checksum
6 *
7 * Inputs:
8 * in0: address of buffer to checksum (char *)
9 * in1: length of the buffer (int)
10 *
11 * Copyright (C) 1999, 2001-2002 Hewlett-Packard Co
12 * Stephane Eranian <eranian@hpl.hp.com>
13 *
14 * 02/04/22 Ken Chen <kenneth.w.chen@intel.com>
15 * Data locality study on the checksum buffer.
16 * More optimization cleanup - remove excessive stop bits.
17 * 02/04/08 David Mosberger <davidm@hpl.hp.com>
18 * More cleanup and tuning.
19 * 01/04/18 Jun Nakajima <jun.nakajima@intel.com>
20 * Clean up and optimize and the software pipeline, loading two
21 * back-to-back 8-byte words per loop. Clean up the initialization
22 * for the loop. Support the cases where load latency = 1 or 2.
23 * Set CONFIG_IA64_LOAD_LATENCY to 1 or 2 (default).
24 */
25
26#include <asm/asmmacro.h>
27
28//
29// Theory of operations:
30// The goal is to go as quickly as possible to the point where
31// we can checksum 16 bytes/loop. Before reaching that point we must
32// take care of incorrect alignment of first byte.
33//
34// The code hereafter also takes care of the "tail" part of the buffer
35// before entering the core loop, if any. The checksum is a sum so it
36// allows us to commute operations. So we do the "head" and "tail"
37// first to finish at full speed in the body. Once we get the head and
38// tail values, we feed them into the pipeline, very handy initialization.
39//
40// Of course we deal with the special case where the whole buffer fits
41// into one 8 byte word. In this case we have only one entry in the pipeline.
42//
43// We use a (LOAD_LATENCY+2)-stage pipeline in the loop to account for
44// possible load latency and also to accommodate for head and tail.
45//
46// The end of the function deals with folding the checksum from 64bits
47// down to 16bits taking care of the carry.
48//
49// This version avoids synchronization in the core loop by also using a
50// pipeline for the accumulation of the checksum in resultx[] (x=1,2).
51//
52// wordx[] (x=1,2)
53// |---|
54// | | 0 : new value loaded in pipeline
55// |---|
56// | | - : in transit data
57// |---|
58// | | LOAD_LATENCY : current value to add to checksum
59// |---|
60// | | LOAD_LATENCY+1 : previous value added to checksum
61// |---| (previous iteration)
62//
63// resultx[] (x=1,2)
64// |---|
65// | | 0 : initial value
66// |---|
67// | | LOAD_LATENCY-1 : new checksum
68// |---|
69// | | LOAD_LATENCY : previous value of checksum
70// |---|
71// | | LOAD_LATENCY+1 : final checksum when out of the loop
72// |---|
73//
74//
75// See RFC1071 "Computing the Internet Checksum" for various techniques for
76// calculating the Internet checksum.
77//
78// NOT YET DONE:
79// - Maybe another algorithm which would take care of the folding at the
80// end in a different manner
81// - Work with people more knowledgeable than me on the network stack
82// to figure out if we could not split the function depending on the
83// type of packet or alignment we get. Like the ip_fast_csum() routine
84// where we know we have at least 20bytes worth of data to checksum.
85// - Do a better job of handling small packets.
86// - Note on prefetching: it was found that under various load, i.e. ftp read/write,
87// nfs read/write, the L1 cache hit rate is at 60% and L2 cache hit rate is at 99.8%
88// on the data that buffer points to (partly because the checksum is often preceded by
89// a copy_from_user()). This finding indiate that lfetch will not be beneficial since
90// the data is already in the cache.
91//
92
93#define saved_pfs r11
94#define hmask r16
95#define tmask r17
96#define first1 r18
97#define firstval r19
98#define firstoff r20
99#define last r21
100#define lastval r22
101#define lastoff r23
102#define saved_lc r24
103#define saved_pr r25
104#define tmp1 r26
105#define tmp2 r27
106#define tmp3 r28
107#define carry1 r29
108#define carry2 r30
109#define first2 r31
110
111#define buf in0
112#define len in1
113
114#define LOAD_LATENCY 2 // XXX fix me
115
116#if (LOAD_LATENCY != 1) && (LOAD_LATENCY != 2)
117# error "Only 1 or 2 is supported/tested for LOAD_LATENCY."
118#endif
119
120#define PIPE_DEPTH (LOAD_LATENCY+2)
121#define ELD p[LOAD_LATENCY] // end of load
122#define ELD_1 p[LOAD_LATENCY+1] // and next stage
123
124// unsigned long do_csum(unsigned char *buf,long len)
125
126GLOBAL_ENTRY(do_csum)
127 .prologue
128 .save ar.pfs, saved_pfs
129 alloc saved_pfs=ar.pfs,2,16,0,16
130 .rotr word1[4], word2[4],result1[LOAD_LATENCY+2],result2[LOAD_LATENCY+2]
131 .rotp p[PIPE_DEPTH], pC1[2], pC2[2]
132 mov ret0=r0 // in case we have zero length
133 cmp.lt p0,p6=r0,len // check for zero length or negative (32bit len)
134 ;;
135 add tmp1=buf,len // last byte's address
136 .save pr, saved_pr
137 mov saved_pr=pr // preserve predicates (rotation)
138(p6) br.ret.spnt.many rp // return if zero or negative length
139
140 mov hmask=-1 // initialize head mask
141 tbit.nz p15,p0=buf,0 // is buf an odd address?
142 and first1=-8,buf // 8-byte align down address of first1 element
143
144 and firstoff=7,buf // how many bytes off for first1 element
145 mov tmask=-1 // initialize tail mask
146
147 ;;
148 adds tmp2=-1,tmp1 // last-1
149 and lastoff=7,tmp1 // how many bytes off for last element
150 ;;
151 sub tmp1=8,lastoff // complement to lastoff
152 and last=-8,tmp2 // address of word containing last byte
153 ;;
154 sub tmp3=last,first1 // tmp3=distance from first1 to last
155 .save ar.lc, saved_lc
156 mov saved_lc=ar.lc // save lc
157 cmp.eq p8,p9=last,first1 // everything fits in one word ?
158
159 ld8 firstval=[first1],8 // load, ahead of time, "first1" word
160 and tmp1=7, tmp1 // make sure that if tmp1==8 -> tmp1=0
161 shl tmp2=firstoff,3 // number of bits
162 ;;
163(p9) ld8 lastval=[last] // load, ahead of time, "last" word, if needed
164 shl tmp1=tmp1,3 // number of bits
165(p9) adds tmp3=-8,tmp3 // effectively loaded
166 ;;
167(p8) mov lastval=r0 // we don't need lastval if first1==last
168 shl hmask=hmask,tmp2 // build head mask, mask off [0,first1off[
169 shr.u tmask=tmask,tmp1 // build tail mask, mask off ]8,lastoff]
170 ;;
171 .body
172#define count tmp3
173
174(p8) and hmask=hmask,tmask // apply tail mask to head mask if 1 word only
175(p9) and word2[0]=lastval,tmask // mask last it as appropriate
176 shr.u count=count,3 // how many 8-byte?
177 ;;
178 // If count is odd, finish this 8-byte word so that we can
179 // load two back-to-back 8-byte words per loop thereafter.
180 and word1[0]=firstval,hmask // and mask it as appropriate
181 tbit.nz p10,p11=count,0 // if (count is odd)
182 ;;
183(p8) mov result1[0]=word1[0]
184(p9) add result1[0]=word1[0],word2[0]
185 ;;
186 cmp.ltu p6,p0=result1[0],word1[0] // check the carry
187 cmp.eq.or.andcm p8,p0=0,count // exit if zero 8-byte
188 ;;
189(p6) adds result1[0]=1,result1[0]
190(p8) br.cond.dptk .do_csum_exit // if (within an 8-byte word)
191(p11) br.cond.dptk .do_csum16 // if (count is even)
192
193 // Here count is odd.
194 ld8 word1[1]=[first1],8 // load an 8-byte word
195 cmp.eq p9,p10=1,count // if (count == 1)
196 adds count=-1,count // loaded an 8-byte word
197 ;;
198 add result1[0]=result1[0],word1[1]
199 ;;
200 cmp.ltu p6,p0=result1[0],word1[1]
201 ;;
202(p6) adds result1[0]=1,result1[0]
203(p9) br.cond.sptk .do_csum_exit // if (count == 1) exit
Lucas De Marchi25985ed2011-03-30 22:57:33 -0300204 // Fall through to calculate the checksum, feeding result1[0] as
Linus Torvalds1da177e2005-04-16 15:20:36 -0700205 // the initial value in result1[0].
206 //
207 // Calculate the checksum loading two 8-byte words per loop.
208 //
209.do_csum16:
210 add first2=8,first1
211 shr.u count=count,1 // we do 16 bytes per loop
212 ;;
213 adds count=-1,count
214 mov carry1=r0
215 mov carry2=r0
216 brp.loop.imp 1f,2f
217 ;;
218 mov ar.ec=PIPE_DEPTH
219 mov ar.lc=count // set lc
220 mov pr.rot=1<<16
221 // result1[0] must be initialized in advance.
222 mov result2[0]=r0
223 ;;
224 .align 32
2251:
226(ELD_1) cmp.ltu pC1[0],p0=result1[LOAD_LATENCY],word1[LOAD_LATENCY+1]
227(pC1[1])adds carry1=1,carry1
228(ELD_1) cmp.ltu pC2[0],p0=result2[LOAD_LATENCY],word2[LOAD_LATENCY+1]
229(pC2[1])adds carry2=1,carry2
230(ELD) add result1[LOAD_LATENCY-1]=result1[LOAD_LATENCY],word1[LOAD_LATENCY]
231(ELD) add result2[LOAD_LATENCY-1]=result2[LOAD_LATENCY],word2[LOAD_LATENCY]
2322:
233(p[0]) ld8 word1[0]=[first1],16
234(p[0]) ld8 word2[0]=[first2],16
235 br.ctop.sptk 1b
236 ;;
237 // Since len is a 32-bit value, carry cannot be larger than a 64-bit value.
238(pC1[1])adds carry1=1,carry1 // since we miss the last one
239(pC2[1])adds carry2=1,carry2
240 ;;
241 add result1[LOAD_LATENCY+1]=result1[LOAD_LATENCY+1],carry1
242 add result2[LOAD_LATENCY+1]=result2[LOAD_LATENCY+1],carry2
243 ;;
244 cmp.ltu p6,p0=result1[LOAD_LATENCY+1],carry1
245 cmp.ltu p7,p0=result2[LOAD_LATENCY+1],carry2
246 ;;
247(p6) adds result1[LOAD_LATENCY+1]=1,result1[LOAD_LATENCY+1]
248(p7) adds result2[LOAD_LATENCY+1]=1,result2[LOAD_LATENCY+1]
249 ;;
250 add result1[0]=result1[LOAD_LATENCY+1],result2[LOAD_LATENCY+1]
251 ;;
252 cmp.ltu p6,p0=result1[0],result2[LOAD_LATENCY+1]
253 ;;
254(p6) adds result1[0]=1,result1[0]
255 ;;
256.do_csum_exit:
257 //
258 // now fold 64 into 16 bits taking care of carry
259 // that's not very good because it has lots of sequentiality
260 //
261 mov tmp3=0xffff
262 zxt4 tmp1=result1[0]
263 shr.u tmp2=result1[0],32
264 ;;
265 add result1[0]=tmp1,tmp2
266 ;;
267 and tmp1=result1[0],tmp3
268 shr.u tmp2=result1[0],16
269 ;;
270 add result1[0]=tmp1,tmp2
271 ;;
272 and tmp1=result1[0],tmp3
273 shr.u tmp2=result1[0],16
274 ;;
275 add result1[0]=tmp1,tmp2
276 ;;
277 and tmp1=result1[0],tmp3
278 shr.u tmp2=result1[0],16
279 ;;
280 add ret0=tmp1,tmp2
281 mov pr=saved_pr,0xffffffffffff0000
282 ;;
283 // if buf was odd then swap bytes
284 mov ar.pfs=saved_pfs // restore ar.ec
285(p15) mux1 ret0=ret0,@rev // reverse word
286 ;;
287 mov ar.lc=saved_lc
288(p15) shr.u ret0=ret0,64-16 // + shift back to position = swap bytes
289 br.ret.sptk.many rp
290
291// I (Jun Nakajima) wrote an equivalent code (see below), but it was
292// not much better than the original. So keep the original there so that
293// someone else can challenge.
294//
295// shr.u word1[0]=result1[0],32
296// zxt4 result1[0]=result1[0]
297// ;;
298// add result1[0]=result1[0],word1[0]
299// ;;
300// zxt2 result2[0]=result1[0]
301// extr.u word1[0]=result1[0],16,16
302// shr.u carry1=result1[0],32
303// ;;
304// add result2[0]=result2[0],word1[0]
305// ;;
306// add result2[0]=result2[0],carry1
307// ;;
308// extr.u ret0=result2[0],16,16
309// ;;
310// add ret0=ret0,result2[0]
311// ;;
312// zxt2 ret0=ret0
313// mov ar.pfs=saved_pfs // restore ar.ec
314// mov pr=saved_pr,0xffffffffffff0000
315// ;;
316// // if buf was odd then swap bytes
317// mov ar.lc=saved_lc
318//(p15) mux1 ret0=ret0,@rev // reverse word
319// ;;
320//(p15) shr.u ret0=ret0,64-16 // + shift back to position = swap bytes
321// br.ret.sptk.many rp
322
323END(do_csum)