Gitiles
Code Review Sign In
gerrit-public.fairphone.software / platform / external / boringssl / 9083ef933c7f44d6ee3cd1b7ca91135fdada8e31 / . / mac-x86_64 / crypto / fipsmodule / bsaes-x86_64.S
blob: 195abd3b5cfbf321d3e7b1614d6522fa4bfc10d6 [file] [log] [blame]
Robert Sloan6f79a502017-04-03 09:16:40 -0700[diff] [blame]1#if defined(__x86_64__) && !defined(OPENSSL_NO_ASM)
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]2.text
3
4
5
6
7
8.p2align 6
9_bsaes_encrypt8:
10 leaq L$BS0(%rip),%r11
11
12 movdqa (%rax),%xmm8
13 leaq 16(%rax),%rax
14 movdqa 80(%r11),%xmm7
15 pxor %xmm8,%xmm15
16 pxor %xmm8,%xmm0
17 pxor %xmm8,%xmm1
18 pxor %xmm8,%xmm2
19.byte 102,68,15,56,0,255
20.byte 102,15,56,0,199
21 pxor %xmm8,%xmm3
22 pxor %xmm8,%xmm4
23.byte 102,15,56,0,207
24.byte 102,15,56,0,215
25 pxor %xmm8,%xmm5
26 pxor %xmm8,%xmm6
27.byte 102,15,56,0,223
28.byte 102,15,56,0,231
29.byte 102,15,56,0,239
30.byte 102,15,56,0,247
31_bsaes_encrypt8_bitslice:
32 movdqa 0(%r11),%xmm7
33 movdqa 16(%r11),%xmm8
34 movdqa %xmm5,%xmm9
35 psrlq $1,%xmm5
36 movdqa %xmm3,%xmm10
37 psrlq $1,%xmm3
38 pxor %xmm6,%xmm5
39 pxor %xmm4,%xmm3
40 pand %xmm7,%xmm5
41 pand %xmm7,%xmm3
42 pxor %xmm5,%xmm6
43 psllq $1,%xmm5
44 pxor %xmm3,%xmm4
45 psllq $1,%xmm3
46 pxor %xmm9,%xmm5
47 pxor %xmm10,%xmm3
48 movdqa %xmm1,%xmm9
49 psrlq $1,%xmm1
50 movdqa %xmm15,%xmm10
51 psrlq $1,%xmm15
52 pxor %xmm2,%xmm1
53 pxor %xmm0,%xmm15
54 pand %xmm7,%xmm1
55 pand %xmm7,%xmm15
56 pxor %xmm1,%xmm2
57 psllq $1,%xmm1
58 pxor %xmm15,%xmm0
59 psllq $1,%xmm15
60 pxor %xmm9,%xmm1
61 pxor %xmm10,%xmm15
62 movdqa 32(%r11),%xmm7
63 movdqa %xmm4,%xmm9
64 psrlq $2,%xmm4
65 movdqa %xmm3,%xmm10
66 psrlq $2,%xmm3
67 pxor %xmm6,%xmm4
68 pxor %xmm5,%xmm3
69 pand %xmm8,%xmm4
70 pand %xmm8,%xmm3
71 pxor %xmm4,%xmm6
72 psllq $2,%xmm4
73 pxor %xmm3,%xmm5
74 psllq $2,%xmm3
75 pxor %xmm9,%xmm4
76 pxor %xmm10,%xmm3
77 movdqa %xmm0,%xmm9
78 psrlq $2,%xmm0
79 movdqa %xmm15,%xmm10
80 psrlq $2,%xmm15
81 pxor %xmm2,%xmm0
82 pxor %xmm1,%xmm15
83 pand %xmm8,%xmm0
84 pand %xmm8,%xmm15
85 pxor %xmm0,%xmm2
86 psllq $2,%xmm0
87 pxor %xmm15,%xmm1
88 psllq $2,%xmm15
89 pxor %xmm9,%xmm0
90 pxor %xmm10,%xmm15
91 movdqa %xmm2,%xmm9
92 psrlq $4,%xmm2
93 movdqa %xmm1,%xmm10
94 psrlq $4,%xmm1
95 pxor %xmm6,%xmm2
96 pxor %xmm5,%xmm1
97 pand %xmm7,%xmm2
98 pand %xmm7,%xmm1
99 pxor %xmm2,%xmm6
100 psllq $4,%xmm2
101 pxor %xmm1,%xmm5
102 psllq $4,%xmm1
103 pxor %xmm9,%xmm2
104 pxor %xmm10,%xmm1
105 movdqa %xmm0,%xmm9
106 psrlq $4,%xmm0
107 movdqa %xmm15,%xmm10
108 psrlq $4,%xmm15
109 pxor %xmm4,%xmm0
110 pxor %xmm3,%xmm15
111 pand %xmm7,%xmm0
112 pand %xmm7,%xmm15
113 pxor %xmm0,%xmm4
114 psllq $4,%xmm0
115 pxor %xmm15,%xmm3
116 psllq $4,%xmm15
117 pxor %xmm9,%xmm0
118 pxor %xmm10,%xmm15
119 decl %r10d
120 jmp L$enc_sbox
121.p2align 4
122L$enc_loop:
123 pxor 0(%rax),%xmm15
124 pxor 16(%rax),%xmm0
125 pxor 32(%rax),%xmm1
126 pxor 48(%rax),%xmm2
127.byte 102,68,15,56,0,255
128.byte 102,15,56,0,199
129 pxor 64(%rax),%xmm3
130 pxor 80(%rax),%xmm4
131.byte 102,15,56,0,207
132.byte 102,15,56,0,215
133 pxor 96(%rax),%xmm5
134 pxor 112(%rax),%xmm6
135.byte 102,15,56,0,223
136.byte 102,15,56,0,231
137.byte 102,15,56,0,239
138.byte 102,15,56,0,247
139 leaq 128(%rax),%rax
140L$enc_sbox:
141 pxor %xmm5,%xmm4
142 pxor %xmm0,%xmm1
143 pxor %xmm15,%xmm2
144 pxor %xmm1,%xmm5
145 pxor %xmm15,%xmm4
146
147 pxor %xmm2,%xmm5
148 pxor %xmm6,%xmm2
149 pxor %xmm4,%xmm6
150 pxor %xmm3,%xmm2
151 pxor %xmm4,%xmm3
152 pxor %xmm0,%xmm2
153
154 pxor %xmm6,%xmm1
155 pxor %xmm4,%xmm0
156 movdqa %xmm6,%xmm10
157 movdqa %xmm0,%xmm9
158 movdqa %xmm4,%xmm8
159 movdqa %xmm1,%xmm12
160 movdqa %xmm5,%xmm11
161
162 pxor %xmm3,%xmm10
163 pxor %xmm1,%xmm9
164 pxor %xmm2,%xmm8
165 movdqa %xmm10,%xmm13
166 pxor %xmm3,%xmm12
167 movdqa %xmm9,%xmm7
168 pxor %xmm15,%xmm11
169 movdqa %xmm10,%xmm14
170
171 por %xmm8,%xmm9
172 por %xmm11,%xmm10
173 pxor %xmm7,%xmm14
174 pand %xmm11,%xmm13
175 pxor %xmm8,%xmm11
176 pand %xmm8,%xmm7
177 pand %xmm11,%xmm14
178 movdqa %xmm2,%xmm11
179 pxor %xmm15,%xmm11
180 pand %xmm11,%xmm12
181 pxor %xmm12,%xmm10
182 pxor %xmm12,%xmm9
183 movdqa %xmm6,%xmm12
184 movdqa %xmm4,%xmm11
185 pxor %xmm0,%xmm12
186 pxor %xmm5,%xmm11
187 movdqa %xmm12,%xmm8
188 pand %xmm11,%xmm12
189 por %xmm11,%xmm8
190 pxor %xmm12,%xmm7
191 pxor %xmm14,%xmm10
192 pxor %xmm13,%xmm9
193 pxor %xmm14,%xmm8
194 movdqa %xmm1,%xmm11
195 pxor %xmm13,%xmm7
196 movdqa %xmm3,%xmm12
197 pxor %xmm13,%xmm8
198 movdqa %xmm0,%xmm13
199 pand %xmm2,%xmm11
200 movdqa %xmm6,%xmm14
201 pand %xmm15,%xmm12
202 pand %xmm4,%xmm13
203 por %xmm5,%xmm14
204 pxor %xmm11,%xmm10
205 pxor %xmm12,%xmm9
206 pxor %xmm13,%xmm8
207 pxor %xmm14,%xmm7
208
209
210
211
212
213 movdqa %xmm10,%xmm11
214 pand %xmm8,%xmm10
215 pxor %xmm9,%xmm11
216
217 movdqa %xmm7,%xmm13
218 movdqa %xmm11,%xmm14
219 pxor %xmm10,%xmm13
220 pand %xmm13,%xmm14
221
222 movdqa %xmm8,%xmm12
223 pxor %xmm9,%xmm14
224 pxor %xmm7,%xmm12
225
226 pxor %xmm9,%xmm10
227
228 pand %xmm10,%xmm12
229
230 movdqa %xmm13,%xmm9
231 pxor %xmm7,%xmm12
232
233 pxor %xmm12,%xmm9
234 pxor %xmm12,%xmm8
235
236 pand %xmm7,%xmm9
237
238 pxor %xmm9,%xmm13
239 pxor %xmm9,%xmm8
240
241 pand %xmm14,%xmm13
242
243 pxor %xmm11,%xmm13
244 movdqa %xmm5,%xmm11
245 movdqa %xmm4,%xmm7
246 movdqa %xmm14,%xmm9
247 pxor %xmm13,%xmm9
248 pand %xmm5,%xmm9
249 pxor %xmm4,%xmm5
250 pand %xmm14,%xmm4
251 pand %xmm13,%xmm5
252 pxor %xmm4,%xmm5
253 pxor %xmm9,%xmm4
254 pxor %xmm15,%xmm11
255 pxor %xmm2,%xmm7
256 pxor %xmm12,%xmm14
257 pxor %xmm8,%xmm13
258 movdqa %xmm14,%xmm10
259 movdqa %xmm12,%xmm9
260 pxor %xmm13,%xmm10
261 pxor %xmm8,%xmm9
262 pand %xmm11,%xmm10
263 pand %xmm15,%xmm9
264 pxor %xmm7,%xmm11
265 pxor %xmm2,%xmm15
266 pand %xmm14,%xmm7
267 pand %xmm12,%xmm2
268 pand %xmm13,%xmm11
269 pand %xmm8,%xmm15
270 pxor %xmm11,%xmm7
271 pxor %xmm2,%xmm15
272 pxor %xmm10,%xmm11
273 pxor %xmm9,%xmm2
274 pxor %xmm11,%xmm5
275 pxor %xmm11,%xmm15
276 pxor %xmm7,%xmm4
277 pxor %xmm7,%xmm2
278
279 movdqa %xmm6,%xmm11
280 movdqa %xmm0,%xmm7
281 pxor %xmm3,%xmm11
282 pxor %xmm1,%xmm7
283 movdqa %xmm14,%xmm10
284 movdqa %xmm12,%xmm9
285 pxor %xmm13,%xmm10
286 pxor %xmm8,%xmm9
287 pand %xmm11,%xmm10
288 pand %xmm3,%xmm9
289 pxor %xmm7,%xmm11
290 pxor %xmm1,%xmm3
291 pand %xmm14,%xmm7
292 pand %xmm12,%xmm1
293 pand %xmm13,%xmm11
294 pand %xmm8,%xmm3
295 pxor %xmm11,%xmm7
296 pxor %xmm1,%xmm3
297 pxor %xmm10,%xmm11
298 pxor %xmm9,%xmm1
299 pxor %xmm12,%xmm14
300 pxor %xmm8,%xmm13
301 movdqa %xmm14,%xmm10
302 pxor %xmm13,%xmm10
303 pand %xmm6,%xmm10
304 pxor %xmm0,%xmm6
305 pand %xmm14,%xmm0
306 pand %xmm13,%xmm6
307 pxor %xmm0,%xmm6
308 pxor %xmm10,%xmm0
309 pxor %xmm11,%xmm6
310 pxor %xmm11,%xmm3
311 pxor %xmm7,%xmm0
312 pxor %xmm7,%xmm1
313 pxor %xmm15,%xmm6
314 pxor %xmm5,%xmm0
315 pxor %xmm6,%xmm3
316 pxor %xmm15,%xmm5
317 pxor %xmm0,%xmm15
318
319 pxor %xmm4,%xmm0
320 pxor %xmm1,%xmm4
321 pxor %xmm2,%xmm1
322 pxor %xmm4,%xmm2
323 pxor %xmm4,%xmm3
324
325 pxor %xmm2,%xmm5
326 decl %r10d
327 jl L$enc_done
David Benjamin4969cc92016-04-22 15:02:23 -0400[diff] [blame]328 pshufd $0x93,%xmm15,%xmm7
329 pshufd $0x93,%xmm0,%xmm8
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]330 pxor %xmm7,%xmm15
David Benjamin4969cc92016-04-22 15:02:23 -0400[diff] [blame]331 pshufd $0x93,%xmm3,%xmm9
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]332 pxor %xmm8,%xmm0
David Benjamin4969cc92016-04-22 15:02:23 -0400[diff] [blame]333 pshufd $0x93,%xmm5,%xmm10
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]334 pxor %xmm9,%xmm3
David Benjamin4969cc92016-04-22 15:02:23 -0400[diff] [blame]335 pshufd $0x93,%xmm2,%xmm11
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]336 pxor %xmm10,%xmm5
David Benjamin4969cc92016-04-22 15:02:23 -0400[diff] [blame]337 pshufd $0x93,%xmm6,%xmm12
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]338 pxor %xmm11,%xmm2
David Benjamin4969cc92016-04-22 15:02:23 -0400[diff] [blame]339 pshufd $0x93,%xmm1,%xmm13
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]340 pxor %xmm12,%xmm6
David Benjamin4969cc92016-04-22 15:02:23 -0400[diff] [blame]341 pshufd $0x93,%xmm4,%xmm14
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]342 pxor %xmm13,%xmm1
343 pxor %xmm14,%xmm4
344
345 pxor %xmm15,%xmm8
346 pxor %xmm4,%xmm7
347 pxor %xmm4,%xmm8
David Benjamin4969cc92016-04-22 15:02:23 -0400[diff] [blame]348 pshufd $0x4E,%xmm15,%xmm15
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]349 pxor %xmm0,%xmm9
David Benjamin4969cc92016-04-22 15:02:23 -0400[diff] [blame]350 pshufd $0x4E,%xmm0,%xmm0
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]351 pxor %xmm2,%xmm12
352 pxor %xmm7,%xmm15
353 pxor %xmm6,%xmm13
354 pxor %xmm8,%xmm0
355 pxor %xmm5,%xmm11
David Benjamin4969cc92016-04-22 15:02:23 -0400[diff] [blame]356 pshufd $0x4E,%xmm2,%xmm7
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]357 pxor %xmm1,%xmm14
David Benjamin4969cc92016-04-22 15:02:23 -0400[diff] [blame]358 pshufd $0x4E,%xmm6,%xmm8
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]359 pxor %xmm3,%xmm10
David Benjamin4969cc92016-04-22 15:02:23 -0400[diff] [blame]360 pshufd $0x4E,%xmm5,%xmm2
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]361 pxor %xmm4,%xmm10
David Benjamin4969cc92016-04-22 15:02:23 -0400[diff] [blame]362 pshufd $0x4E,%xmm4,%xmm6
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]363 pxor %xmm4,%xmm11
David Benjamin4969cc92016-04-22 15:02:23 -0400[diff] [blame]364 pshufd $0x4E,%xmm1,%xmm5
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]365 pxor %xmm11,%xmm7
David Benjamin4969cc92016-04-22 15:02:23 -0400[diff] [blame]366 pshufd $0x4E,%xmm3,%xmm1
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]367 pxor %xmm12,%xmm8
368 pxor %xmm10,%xmm2
369 pxor %xmm14,%xmm6
370 pxor %xmm13,%xmm5
371 movdqa %xmm7,%xmm3
372 pxor %xmm9,%xmm1
373 movdqa %xmm8,%xmm4
374 movdqa 48(%r11),%xmm7
375 jnz L$enc_loop
376 movdqa 64(%r11),%xmm7
377 jmp L$enc_loop
378.p2align 4
379L$enc_done:
380 movdqa 0(%r11),%xmm7
381 movdqa 16(%r11),%xmm8
382 movdqa %xmm1,%xmm9
383 psrlq $1,%xmm1
384 movdqa %xmm2,%xmm10
385 psrlq $1,%xmm2
386 pxor %xmm4,%xmm1
387 pxor %xmm6,%xmm2
388 pand %xmm7,%xmm1
389 pand %xmm7,%xmm2
390 pxor %xmm1,%xmm4
391 psllq $1,%xmm1
392 pxor %xmm2,%xmm6
393 psllq $1,%xmm2
394 pxor %xmm9,%xmm1
395 pxor %xmm10,%xmm2
396 movdqa %xmm3,%xmm9
397 psrlq $1,%xmm3
398 movdqa %xmm15,%xmm10
399 psrlq $1,%xmm15
400 pxor %xmm5,%xmm3
401 pxor %xmm0,%xmm15
402 pand %xmm7,%xmm3
403 pand %xmm7,%xmm15
404 pxor %xmm3,%xmm5
405 psllq $1,%xmm3
406 pxor %xmm15,%xmm0
407 psllq $1,%xmm15
408 pxor %xmm9,%xmm3
409 pxor %xmm10,%xmm15
410 movdqa 32(%r11),%xmm7
411 movdqa %xmm6,%xmm9
412 psrlq $2,%xmm6
413 movdqa %xmm2,%xmm10
414 psrlq $2,%xmm2
415 pxor %xmm4,%xmm6
416 pxor %xmm1,%xmm2
417 pand %xmm8,%xmm6
418 pand %xmm8,%xmm2
419 pxor %xmm6,%xmm4
420 psllq $2,%xmm6
421 pxor %xmm2,%xmm1
422 psllq $2,%xmm2
423 pxor %xmm9,%xmm6
424 pxor %xmm10,%xmm2
425 movdqa %xmm0,%xmm9
426 psrlq $2,%xmm0
427 movdqa %xmm15,%xmm10
428 psrlq $2,%xmm15
429 pxor %xmm5,%xmm0
430 pxor %xmm3,%xmm15
431 pand %xmm8,%xmm0
432 pand %xmm8,%xmm15
433 pxor %xmm0,%xmm5
434 psllq $2,%xmm0
435 pxor %xmm15,%xmm3
436 psllq $2,%xmm15
437 pxor %xmm9,%xmm0
438 pxor %xmm10,%xmm15
439 movdqa %xmm5,%xmm9
440 psrlq $4,%xmm5
441 movdqa %xmm3,%xmm10
442 psrlq $4,%xmm3
443 pxor %xmm4,%xmm5
444 pxor %xmm1,%xmm3
445 pand %xmm7,%xmm5
446 pand %xmm7,%xmm3
447 pxor %xmm5,%xmm4
448 psllq $4,%xmm5
449 pxor %xmm3,%xmm1
450 psllq $4,%xmm3
451 pxor %xmm9,%xmm5
452 pxor %xmm10,%xmm3
453 movdqa %xmm0,%xmm9
454 psrlq $4,%xmm0
455 movdqa %xmm15,%xmm10
456 psrlq $4,%xmm15
457 pxor %xmm6,%xmm0
458 pxor %xmm2,%xmm15
459 pand %xmm7,%xmm0
460 pand %xmm7,%xmm15
461 pxor %xmm0,%xmm6
462 psllq $4,%xmm0
463 pxor %xmm15,%xmm2
464 psllq $4,%xmm15
465 pxor %xmm9,%xmm0
466 pxor %xmm10,%xmm15
467 movdqa (%rax),%xmm7
468 pxor %xmm7,%xmm3
469 pxor %xmm7,%xmm5
470 pxor %xmm7,%xmm2
471 pxor %xmm7,%xmm6
472 pxor %xmm7,%xmm1
473 pxor %xmm7,%xmm4
474 pxor %xmm7,%xmm15
475 pxor %xmm7,%xmm0
476 .byte 0xf3,0xc3
477
478
479
480.p2align 6
481_bsaes_decrypt8:
482 leaq L$BS0(%rip),%r11
483
484 movdqa (%rax),%xmm8
485 leaq 16(%rax),%rax
486 movdqa -48(%r11),%xmm7
487 pxor %xmm8,%xmm15
488 pxor %xmm8,%xmm0
489 pxor %xmm8,%xmm1
490 pxor %xmm8,%xmm2
491.byte 102,68,15,56,0,255
492.byte 102,15,56,0,199
493 pxor %xmm8,%xmm3
494 pxor %xmm8,%xmm4
495.byte 102,15,56,0,207
496.byte 102,15,56,0,215
497 pxor %xmm8,%xmm5
498 pxor %xmm8,%xmm6
499.byte 102,15,56,0,223
500.byte 102,15,56,0,231
501.byte 102,15,56,0,239
502.byte 102,15,56,0,247
503 movdqa 0(%r11),%xmm7
504 movdqa 16(%r11),%xmm8
505 movdqa %xmm5,%xmm9
506 psrlq $1,%xmm5
507 movdqa %xmm3,%xmm10
508 psrlq $1,%xmm3
509 pxor %xmm6,%xmm5
510 pxor %xmm4,%xmm3
511 pand %xmm7,%xmm5
512 pand %xmm7,%xmm3
513 pxor %xmm5,%xmm6
514 psllq $1,%xmm5
515 pxor %xmm3,%xmm4
516 psllq $1,%xmm3
517 pxor %xmm9,%xmm5
518 pxor %xmm10,%xmm3
519 movdqa %xmm1,%xmm9
520 psrlq $1,%xmm1
521 movdqa %xmm15,%xmm10
522 psrlq $1,%xmm15
523 pxor %xmm2,%xmm1
524 pxor %xmm0,%xmm15
525 pand %xmm7,%xmm1
526 pand %xmm7,%xmm15
527 pxor %xmm1,%xmm2
528 psllq $1,%xmm1
529 pxor %xmm15,%xmm0
530 psllq $1,%xmm15
531 pxor %xmm9,%xmm1
532 pxor %xmm10,%xmm15
533 movdqa 32(%r11),%xmm7
534 movdqa %xmm4,%xmm9
535 psrlq $2,%xmm4
536 movdqa %xmm3,%xmm10
537 psrlq $2,%xmm3
538 pxor %xmm6,%xmm4
539 pxor %xmm5,%xmm3
540 pand %xmm8,%xmm4
541 pand %xmm8,%xmm3
542 pxor %xmm4,%xmm6
543 psllq $2,%xmm4
544 pxor %xmm3,%xmm5
545 psllq $2,%xmm3
546 pxor %xmm9,%xmm4
547 pxor %xmm10,%xmm3
548 movdqa %xmm0,%xmm9
549 psrlq $2,%xmm0
550 movdqa %xmm15,%xmm10
551 psrlq $2,%xmm15
552 pxor %xmm2,%xmm0
553 pxor %xmm1,%xmm15
554 pand %xmm8,%xmm0
555 pand %xmm8,%xmm15
556 pxor %xmm0,%xmm2
557 psllq $2,%xmm0
558 pxor %xmm15,%xmm1
559 psllq $2,%xmm15
560 pxor %xmm9,%xmm0
561 pxor %xmm10,%xmm15
562 movdqa %xmm2,%xmm9
563 psrlq $4,%xmm2
564 movdqa %xmm1,%xmm10
565 psrlq $4,%xmm1
566 pxor %xmm6,%xmm2
567 pxor %xmm5,%xmm1
568 pand %xmm7,%xmm2
569 pand %xmm7,%xmm1
570 pxor %xmm2,%xmm6
571 psllq $4,%xmm2
572 pxor %xmm1,%xmm5
573 psllq $4,%xmm1
574 pxor %xmm9,%xmm2
575 pxor %xmm10,%xmm1
576 movdqa %xmm0,%xmm9
577 psrlq $4,%xmm0
578 movdqa %xmm15,%xmm10
579 psrlq $4,%xmm15
580 pxor %xmm4,%xmm0
581 pxor %xmm3,%xmm15
582 pand %xmm7,%xmm0
583 pand %xmm7,%xmm15
584 pxor %xmm0,%xmm4
585 psllq $4,%xmm0
586 pxor %xmm15,%xmm3
587 psllq $4,%xmm15
588 pxor %xmm9,%xmm0
589 pxor %xmm10,%xmm15
590 decl %r10d
591 jmp L$dec_sbox
592.p2align 4
593L$dec_loop:
594 pxor 0(%rax),%xmm15
595 pxor 16(%rax),%xmm0
596 pxor 32(%rax),%xmm1
597 pxor 48(%rax),%xmm2
598.byte 102,68,15,56,0,255
599.byte 102,15,56,0,199
600 pxor 64(%rax),%xmm3
601 pxor 80(%rax),%xmm4
602.byte 102,15,56,0,207
603.byte 102,15,56,0,215
604 pxor 96(%rax),%xmm5
605 pxor 112(%rax),%xmm6
606.byte 102,15,56,0,223
607.byte 102,15,56,0,231
608.byte 102,15,56,0,239
609.byte 102,15,56,0,247
610 leaq 128(%rax),%rax
611L$dec_sbox:
612 pxor %xmm3,%xmm2
613
614 pxor %xmm6,%xmm3
615 pxor %xmm6,%xmm1
616 pxor %xmm3,%xmm5
617 pxor %xmm5,%xmm6
618 pxor %xmm6,%xmm0
619
620 pxor %xmm0,%xmm15
621 pxor %xmm4,%xmm1
622 pxor %xmm15,%xmm2
623 pxor %xmm15,%xmm4
624 pxor %xmm2,%xmm0
625 movdqa %xmm2,%xmm10
626 movdqa %xmm6,%xmm9
627 movdqa %xmm0,%xmm8
628 movdqa %xmm3,%xmm12
629 movdqa %xmm4,%xmm11
630
631 pxor %xmm15,%xmm10
632 pxor %xmm3,%xmm9
633 pxor %xmm5,%xmm8
634 movdqa %xmm10,%xmm13
635 pxor %xmm15,%xmm12
636 movdqa %xmm9,%xmm7
637 pxor %xmm1,%xmm11
638 movdqa %xmm10,%xmm14
639
640 por %xmm8,%xmm9
641 por %xmm11,%xmm10
642 pxor %xmm7,%xmm14
643 pand %xmm11,%xmm13
644 pxor %xmm8,%xmm11
645 pand %xmm8,%xmm7
646 pand %xmm11,%xmm14
647 movdqa %xmm5,%xmm11
648 pxor %xmm1,%xmm11
649 pand %xmm11,%xmm12
650 pxor %xmm12,%xmm10
651 pxor %xmm12,%xmm9
652 movdqa %xmm2,%xmm12
653 movdqa %xmm0,%xmm11
654 pxor %xmm6,%xmm12
655 pxor %xmm4,%xmm11
656 movdqa %xmm12,%xmm8
657 pand %xmm11,%xmm12
658 por %xmm11,%xmm8
659 pxor %xmm12,%xmm7
660 pxor %xmm14,%xmm10
661 pxor %xmm13,%xmm9
662 pxor %xmm14,%xmm8
663 movdqa %xmm3,%xmm11
664 pxor %xmm13,%xmm7
665 movdqa %xmm15,%xmm12
666 pxor %xmm13,%xmm8
667 movdqa %xmm6,%xmm13
668 pand %xmm5,%xmm11
669 movdqa %xmm2,%xmm14
670 pand %xmm1,%xmm12
671 pand %xmm0,%xmm13
672 por %xmm4,%xmm14
673 pxor %xmm11,%xmm10
674 pxor %xmm12,%xmm9
675 pxor %xmm13,%xmm8
676 pxor %xmm14,%xmm7
677
678
679
680
681
682 movdqa %xmm10,%xmm11
683 pand %xmm8,%xmm10
684 pxor %xmm9,%xmm11
685
686 movdqa %xmm7,%xmm13
687 movdqa %xmm11,%xmm14
688 pxor %xmm10,%xmm13
689 pand %xmm13,%xmm14
690
691 movdqa %xmm8,%xmm12
692 pxor %xmm9,%xmm14
693 pxor %xmm7,%xmm12
694
695 pxor %xmm9,%xmm10
696
697 pand %xmm10,%xmm12
698
699 movdqa %xmm13,%xmm9
700 pxor %xmm7,%xmm12
701
702 pxor %xmm12,%xmm9
703 pxor %xmm12,%xmm8
704
705 pand %xmm7,%xmm9
706
707 pxor %xmm9,%xmm13
708 pxor %xmm9,%xmm8
709
710 pand %xmm14,%xmm13
711
712 pxor %xmm11,%xmm13
713 movdqa %xmm4,%xmm11
714 movdqa %xmm0,%xmm7
715 movdqa %xmm14,%xmm9
716 pxor %xmm13,%xmm9
717 pand %xmm4,%xmm9
718 pxor %xmm0,%xmm4
719 pand %xmm14,%xmm0
720 pand %xmm13,%xmm4
721 pxor %xmm0,%xmm4
722 pxor %xmm9,%xmm0
723 pxor %xmm1,%xmm11
724 pxor %xmm5,%xmm7
725 pxor %xmm12,%xmm14
726 pxor %xmm8,%xmm13
727 movdqa %xmm14,%xmm10
728 movdqa %xmm12,%xmm9
729 pxor %xmm13,%xmm10
730 pxor %xmm8,%xmm9
731 pand %xmm11,%xmm10
732 pand %xmm1,%xmm9
733 pxor %xmm7,%xmm11
734 pxor %xmm5,%xmm1
735 pand %xmm14,%xmm7
736 pand %xmm12,%xmm5
737 pand %xmm13,%xmm11
738 pand %xmm8,%xmm1
739 pxor %xmm11,%xmm7
740 pxor %xmm5,%xmm1
741 pxor %xmm10,%xmm11
742 pxor %xmm9,%xmm5
743 pxor %xmm11,%xmm4
744 pxor %xmm11,%xmm1
745 pxor %xmm7,%xmm0
746 pxor %xmm7,%xmm5
747
748 movdqa %xmm2,%xmm11
749 movdqa %xmm6,%xmm7
750 pxor %xmm15,%xmm11
751 pxor %xmm3,%xmm7
752 movdqa %xmm14,%xmm10
753 movdqa %xmm12,%xmm9
754 pxor %xmm13,%xmm10
755 pxor %xmm8,%xmm9
756 pand %xmm11,%xmm10
757 pand %xmm15,%xmm9
758 pxor %xmm7,%xmm11
759 pxor %xmm3,%xmm15
760 pand %xmm14,%xmm7
761 pand %xmm12,%xmm3
762 pand %xmm13,%xmm11
763 pand %xmm8,%xmm15
764 pxor %xmm11,%xmm7
765 pxor %xmm3,%xmm15
766 pxor %xmm10,%xmm11
767 pxor %xmm9,%xmm3
768 pxor %xmm12,%xmm14
769 pxor %xmm8,%xmm13
770 movdqa %xmm14,%xmm10
771 pxor %xmm13,%xmm10
772 pand %xmm2,%xmm10
773 pxor %xmm6,%xmm2
774 pand %xmm14,%xmm6
775 pand %xmm13,%xmm2
776 pxor %xmm6,%xmm2
777 pxor %xmm10,%xmm6
778 pxor %xmm11,%xmm2
779 pxor %xmm11,%xmm15
780 pxor %xmm7,%xmm6
781 pxor %xmm7,%xmm3
782 pxor %xmm6,%xmm0
783 pxor %xmm4,%xmm5
784
785 pxor %xmm0,%xmm3
786 pxor %xmm6,%xmm1
787 pxor %xmm6,%xmm4
788 pxor %xmm1,%xmm3
789 pxor %xmm15,%xmm6
790 pxor %xmm4,%xmm3
791 pxor %xmm5,%xmm2
792 pxor %xmm0,%xmm5
793 pxor %xmm3,%xmm2
794
795 pxor %xmm15,%xmm3
796 pxor %xmm2,%xmm6
797 decl %r10d
798 jl L$dec_done
799
David Benjamin4969cc92016-04-22 15:02:23 -0400[diff] [blame]800 pshufd $0x4E,%xmm15,%xmm7
801 pshufd $0x4E,%xmm2,%xmm13
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]802 pxor %xmm15,%xmm7
David Benjamin4969cc92016-04-22 15:02:23 -0400[diff] [blame]803 pshufd $0x4E,%xmm4,%xmm14
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]804 pxor %xmm2,%xmm13
David Benjamin4969cc92016-04-22 15:02:23 -0400[diff] [blame]805 pshufd $0x4E,%xmm0,%xmm8
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]806 pxor %xmm4,%xmm14
David Benjamin4969cc92016-04-22 15:02:23 -0400[diff] [blame]807 pshufd $0x4E,%xmm5,%xmm9
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]808 pxor %xmm0,%xmm8
David Benjamin4969cc92016-04-22 15:02:23 -0400[diff] [blame]809 pshufd $0x4E,%xmm3,%xmm10
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]810 pxor %xmm5,%xmm9
811 pxor %xmm13,%xmm15
812 pxor %xmm13,%xmm0
David Benjamin4969cc92016-04-22 15:02:23 -0400[diff] [blame]813 pshufd $0x4E,%xmm1,%xmm11
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]814 pxor %xmm3,%xmm10
815 pxor %xmm7,%xmm5
816 pxor %xmm8,%xmm3
David Benjamin4969cc92016-04-22 15:02:23 -0400[diff] [blame]817 pshufd $0x4E,%xmm6,%xmm12
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]818 pxor %xmm1,%xmm11
819 pxor %xmm14,%xmm0
820 pxor %xmm9,%xmm1
821 pxor %xmm6,%xmm12
822
823 pxor %xmm14,%xmm5
824 pxor %xmm13,%xmm3
825 pxor %xmm13,%xmm1
826 pxor %xmm10,%xmm6
827 pxor %xmm11,%xmm2
828 pxor %xmm14,%xmm1
829 pxor %xmm14,%xmm6
830 pxor %xmm12,%xmm4
David Benjamin4969cc92016-04-22 15:02:23 -0400[diff] [blame]831 pshufd $0x93,%xmm15,%xmm7
832 pshufd $0x93,%xmm0,%xmm8
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]833 pxor %xmm7,%xmm15
David Benjamin4969cc92016-04-22 15:02:23 -0400[diff] [blame]834 pshufd $0x93,%xmm5,%xmm9
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]835 pxor %xmm8,%xmm0
David Benjamin4969cc92016-04-22 15:02:23 -0400[diff] [blame]836 pshufd $0x93,%xmm3,%xmm10
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]837 pxor %xmm9,%xmm5
David Benjamin4969cc92016-04-22 15:02:23 -0400[diff] [blame]838 pshufd $0x93,%xmm1,%xmm11
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]839 pxor %xmm10,%xmm3
David Benjamin4969cc92016-04-22 15:02:23 -0400[diff] [blame]840 pshufd $0x93,%xmm6,%xmm12
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]841 pxor %xmm11,%xmm1
David Benjamin4969cc92016-04-22 15:02:23 -0400[diff] [blame]842 pshufd $0x93,%xmm2,%xmm13
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]843 pxor %xmm12,%xmm6
David Benjamin4969cc92016-04-22 15:02:23 -0400[diff] [blame]844 pshufd $0x93,%xmm4,%xmm14
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]845 pxor %xmm13,%xmm2
846 pxor %xmm14,%xmm4
847
848 pxor %xmm15,%xmm8
849 pxor %xmm4,%xmm7
850 pxor %xmm4,%xmm8
David Benjamin4969cc92016-04-22 15:02:23 -0400[diff] [blame]851 pshufd $0x4E,%xmm15,%xmm15
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]852 pxor %xmm0,%xmm9
David Benjamin4969cc92016-04-22 15:02:23 -0400[diff] [blame]853 pshufd $0x4E,%xmm0,%xmm0
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]854 pxor %xmm1,%xmm12
855 pxor %xmm7,%xmm15
856 pxor %xmm6,%xmm13
857 pxor %xmm8,%xmm0
858 pxor %xmm3,%xmm11
David Benjamin4969cc92016-04-22 15:02:23 -0400[diff] [blame]859 pshufd $0x4E,%xmm1,%xmm7
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]860 pxor %xmm2,%xmm14
David Benjamin4969cc92016-04-22 15:02:23 -0400[diff] [blame]861 pshufd $0x4E,%xmm6,%xmm8
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]862 pxor %xmm5,%xmm10
David Benjamin4969cc92016-04-22 15:02:23 -0400[diff] [blame]863 pshufd $0x4E,%xmm3,%xmm1
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]864 pxor %xmm4,%xmm10
David Benjamin4969cc92016-04-22 15:02:23 -0400[diff] [blame]865 pshufd $0x4E,%xmm4,%xmm6
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]866 pxor %xmm4,%xmm11
David Benjamin4969cc92016-04-22 15:02:23 -0400[diff] [blame]867 pshufd $0x4E,%xmm2,%xmm3
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]868 pxor %xmm11,%xmm7
David Benjamin4969cc92016-04-22 15:02:23 -0400[diff] [blame]869 pshufd $0x4E,%xmm5,%xmm2
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]870 pxor %xmm12,%xmm8
871 pxor %xmm1,%xmm10
872 pxor %xmm14,%xmm6
873 pxor %xmm3,%xmm13
874 movdqa %xmm7,%xmm3
875 pxor %xmm9,%xmm2
876 movdqa %xmm13,%xmm5
877 movdqa %xmm8,%xmm4
878 movdqa %xmm2,%xmm1
879 movdqa %xmm10,%xmm2
880 movdqa -16(%r11),%xmm7
881 jnz L$dec_loop
882 movdqa -32(%r11),%xmm7
883 jmp L$dec_loop
884.p2align 4
885L$dec_done:
886 movdqa 0(%r11),%xmm7
887 movdqa 16(%r11),%xmm8
888 movdqa %xmm2,%xmm9
889 psrlq $1,%xmm2
890 movdqa %xmm1,%xmm10
891 psrlq $1,%xmm1
892 pxor %xmm4,%xmm2
893 pxor %xmm6,%xmm1
894 pand %xmm7,%xmm2
895 pand %xmm7,%xmm1
896 pxor %xmm2,%xmm4
897 psllq $1,%xmm2
898 pxor %xmm1,%xmm6
899 psllq $1,%xmm1
900 pxor %xmm9,%xmm2
901 pxor %xmm10,%xmm1
902 movdqa %xmm5,%xmm9
903 psrlq $1,%xmm5
904 movdqa %xmm15,%xmm10
905 psrlq $1,%xmm15
906 pxor %xmm3,%xmm5
907 pxor %xmm0,%xmm15
908 pand %xmm7,%xmm5
909 pand %xmm7,%xmm15
910 pxor %xmm5,%xmm3
911 psllq $1,%xmm5
912 pxor %xmm15,%xmm0
913 psllq $1,%xmm15
914 pxor %xmm9,%xmm5
915 pxor %xmm10,%xmm15
916 movdqa 32(%r11),%xmm7
917 movdqa %xmm6,%xmm9
918 psrlq $2,%xmm6
919 movdqa %xmm1,%xmm10
920 psrlq $2,%xmm1
921 pxor %xmm4,%xmm6
922 pxor %xmm2,%xmm1
923 pand %xmm8,%xmm6
924 pand %xmm8,%xmm1
925 pxor %xmm6,%xmm4
926 psllq $2,%xmm6
927 pxor %xmm1,%xmm2
928 psllq $2,%xmm1
929 pxor %xmm9,%xmm6
930 pxor %xmm10,%xmm1
931 movdqa %xmm0,%xmm9
932 psrlq $2,%xmm0
933 movdqa %xmm15,%xmm10
934 psrlq $2,%xmm15
935 pxor %xmm3,%xmm0
936 pxor %xmm5,%xmm15
937 pand %xmm8,%xmm0
938 pand %xmm8,%xmm15
939 pxor %xmm0,%xmm3
940 psllq $2,%xmm0
941 pxor %xmm15,%xmm5
942 psllq $2,%xmm15
943 pxor %xmm9,%xmm0
944 pxor %xmm10,%xmm15
945 movdqa %xmm3,%xmm9
946 psrlq $4,%xmm3
947 movdqa %xmm5,%xmm10
948 psrlq $4,%xmm5
949 pxor %xmm4,%xmm3
950 pxor %xmm2,%xmm5
951 pand %xmm7,%xmm3
952 pand %xmm7,%xmm5
953 pxor %xmm3,%xmm4
954 psllq $4,%xmm3
955 pxor %xmm5,%xmm2
956 psllq $4,%xmm5
957 pxor %xmm9,%xmm3
958 pxor %xmm10,%xmm5
959 movdqa %xmm0,%xmm9
960 psrlq $4,%xmm0
961 movdqa %xmm15,%xmm10
962 psrlq $4,%xmm15
963 pxor %xmm6,%xmm0
964 pxor %xmm1,%xmm15
965 pand %xmm7,%xmm0
966 pand %xmm7,%xmm15
967 pxor %xmm0,%xmm6
968 psllq $4,%xmm0
969 pxor %xmm15,%xmm1
970 psllq $4,%xmm15
971 pxor %xmm9,%xmm0
972 pxor %xmm10,%xmm15
973 movdqa (%rax),%xmm7
974 pxor %xmm7,%xmm5
975 pxor %xmm7,%xmm3
976 pxor %xmm7,%xmm1
977 pxor %xmm7,%xmm6
978 pxor %xmm7,%xmm2
979 pxor %xmm7,%xmm4
980 pxor %xmm7,%xmm15
981 pxor %xmm7,%xmm0
982 .byte 0xf3,0xc3
983
984
985.p2align 4
986_bsaes_key_convert:
987 leaq L$masks(%rip),%r11
988 movdqu (%rcx),%xmm7
989 leaq 16(%rcx),%rcx
990 movdqa 0(%r11),%xmm0
991 movdqa 16(%r11),%xmm1
992 movdqa 32(%r11),%xmm2
993 movdqa 48(%r11),%xmm3
994 movdqa 64(%r11),%xmm4
995 pcmpeqd %xmm5,%xmm5
996
997 movdqu (%rcx),%xmm6
998 movdqa %xmm7,(%rax)
999 leaq 16(%rax),%rax
1000 decl %r10d
1001 jmp L$key_loop
1002.p2align 4
1003L$key_loop:
1004.byte 102,15,56,0,244
1005
1006 movdqa %xmm0,%xmm8
1007 movdqa %xmm1,%xmm9
1008
1009 pand %xmm6,%xmm8
1010 pand %xmm6,%xmm9
1011 movdqa %xmm2,%xmm10
1012 pcmpeqb %xmm0,%xmm8
1013 psllq $4,%xmm0
1014 movdqa %xmm3,%xmm11
1015 pcmpeqb %xmm1,%xmm9
1016 psllq $4,%xmm1
1017
1018 pand %xmm6,%xmm10
1019 pand %xmm6,%xmm11
1020 movdqa %xmm0,%xmm12
1021 pcmpeqb %xmm2,%xmm10
1022 psllq $4,%xmm2
1023 movdqa %xmm1,%xmm13
1024 pcmpeqb %xmm3,%xmm11
1025 psllq $4,%xmm3
1026
1027 movdqa %xmm2,%xmm14
1028 movdqa %xmm3,%xmm15
1029 pxor %xmm5,%xmm8
1030 pxor %xmm5,%xmm9
1031
1032 pand %xmm6,%xmm12
1033 pand %xmm6,%xmm13
1034 movdqa %xmm8,0(%rax)
1035 pcmpeqb %xmm0,%xmm12
1036 psrlq $4,%xmm0
1037 movdqa %xmm9,16(%rax)
1038 pcmpeqb %xmm1,%xmm13
1039 psrlq $4,%xmm1
1040 leaq 16(%rcx),%rcx
1041
1042 pand %xmm6,%xmm14
1043 pand %xmm6,%xmm15
1044 movdqa %xmm10,32(%rax)
1045 pcmpeqb %xmm2,%xmm14
1046 psrlq $4,%xmm2
1047 movdqa %xmm11,48(%rax)
1048 pcmpeqb %xmm3,%xmm15
1049 psrlq $4,%xmm3
1050 movdqu (%rcx),%xmm6
1051
1052 pxor %xmm5,%xmm13
1053 pxor %xmm5,%xmm14
1054 movdqa %xmm12,64(%rax)
1055 movdqa %xmm13,80(%rax)
1056 movdqa %xmm14,96(%rax)
1057 movdqa %xmm15,112(%rax)
1058 leaq 128(%rax),%rax
1059 decl %r10d
1060 jnz L$key_loop
1061
1062 movdqa 80(%r11),%xmm7
1063
1064 .byte 0xf3,0xc3
1065
1066
1067.globl _bsaes_cbc_encrypt
1068.private_extern _bsaes_cbc_encrypt
1069
1070.p2align 4
1071_bsaes_cbc_encrypt:
1072 cmpl $0,%r9d
1073 jne _asm_AES_cbc_encrypt
1074 cmpq $128,%rdx
1075 jb _asm_AES_cbc_encrypt
1076
1077 movq %rsp,%rax
1078L$cbc_dec_prologue:
1079 pushq %rbp
1080 pushq %rbx
1081 pushq %r12
1082 pushq %r13
1083 pushq %r14
1084 pushq %r15
1085 leaq -72(%rsp),%rsp
1086 movq %rsp,%rbp
1087 movl 240(%rcx),%eax
1088 movq %rdi,%r12
1089 movq %rsi,%r13
1090 movq %rdx,%r14
1091 movq %rcx,%r15
1092 movq %r8,%rbx
1093 shrq $4,%r14
1094
1095 movl %eax,%edx
1096 shlq $7,%rax
1097 subq $96,%rax
1098 subq %rax,%rsp
1099
1100 movq %rsp,%rax
1101 movq %r15,%rcx
1102 movl %edx,%r10d
1103 call _bsaes_key_convert
1104 pxor (%rsp),%xmm7
1105 movdqa %xmm6,(%rax)
1106 movdqa %xmm7,(%rsp)
1107
1108 movdqu (%rbx),%xmm14
1109 subq $8,%r14
1110L$cbc_dec_loop:
1111 movdqu 0(%r12),%xmm15
1112 movdqu 16(%r12),%xmm0
1113 movdqu 32(%r12),%xmm1
1114 movdqu 48(%r12),%xmm2
1115 movdqu 64(%r12),%xmm3
1116 movdqu 80(%r12),%xmm4
1117 movq %rsp,%rax
1118 movdqu 96(%r12),%xmm5
1119 movl %edx,%r10d
1120 movdqu 112(%r12),%xmm6
1121 movdqa %xmm14,32(%rbp)
1122
1123 call _bsaes_decrypt8
1124
1125 pxor 32(%rbp),%xmm15
1126 movdqu 0(%r12),%xmm7
1127 movdqu 16(%r12),%xmm8
1128 pxor %xmm7,%xmm0
1129 movdqu 32(%r12),%xmm9
1130 pxor %xmm8,%xmm5
1131 movdqu 48(%r12),%xmm10
1132 pxor %xmm9,%xmm3
1133 movdqu 64(%r12),%xmm11
1134 pxor %xmm10,%xmm1
1135 movdqu 80(%r12),%xmm12
1136 pxor %xmm11,%xmm6
1137 movdqu 96(%r12),%xmm13
1138 pxor %xmm12,%xmm2
1139 movdqu 112(%r12),%xmm14
1140 pxor %xmm13,%xmm4
1141 movdqu %xmm15,0(%r13)
1142 leaq 128(%r12),%r12
1143 movdqu %xmm0,16(%r13)
1144 movdqu %xmm5,32(%r13)
1145 movdqu %xmm3,48(%r13)
1146 movdqu %xmm1,64(%r13)
1147 movdqu %xmm6,80(%r13)
1148 movdqu %xmm2,96(%r13)
1149 movdqu %xmm4,112(%r13)
1150 leaq 128(%r13),%r13
1151 subq $8,%r14
1152 jnc L$cbc_dec_loop
1153
1154 addq $8,%r14
1155 jz L$cbc_dec_done
1156
1157 movdqu 0(%r12),%xmm15
1158 movq %rsp,%rax
1159 movl %edx,%r10d
1160 cmpq $2,%r14
1161 jb L$cbc_dec_one
1162 movdqu 16(%r12),%xmm0
1163 je L$cbc_dec_two
1164 movdqu 32(%r12),%xmm1
1165 cmpq $4,%r14
1166 jb L$cbc_dec_three
1167 movdqu 48(%r12),%xmm2
1168 je L$cbc_dec_four
1169 movdqu 64(%r12),%xmm3
1170 cmpq $6,%r14
1171 jb L$cbc_dec_five
1172 movdqu 80(%r12),%xmm4
1173 je L$cbc_dec_six
1174 movdqu 96(%r12),%xmm5
1175 movdqa %xmm14,32(%rbp)
1176 call _bsaes_decrypt8
1177 pxor 32(%rbp),%xmm15
1178 movdqu 0(%r12),%xmm7
1179 movdqu 16(%r12),%xmm8
1180 pxor %xmm7,%xmm0
1181 movdqu 32(%r12),%xmm9
1182 pxor %xmm8,%xmm5
1183 movdqu 48(%r12),%xmm10
1184 pxor %xmm9,%xmm3
1185 movdqu 64(%r12),%xmm11
1186 pxor %xmm10,%xmm1
1187 movdqu 80(%r12),%xmm12
1188 pxor %xmm11,%xmm6
1189 movdqu 96(%r12),%xmm14
1190 pxor %xmm12,%xmm2
1191 movdqu %xmm15,0(%r13)
1192 movdqu %xmm0,16(%r13)
1193 movdqu %xmm5,32(%r13)
1194 movdqu %xmm3,48(%r13)
1195 movdqu %xmm1,64(%r13)
1196 movdqu %xmm6,80(%r13)
1197 movdqu %xmm2,96(%r13)
1198 jmp L$cbc_dec_done
1199.p2align 4
1200L$cbc_dec_six:
1201 movdqa %xmm14,32(%rbp)
1202 call _bsaes_decrypt8
1203 pxor 32(%rbp),%xmm15
1204 movdqu 0(%r12),%xmm7
1205 movdqu 16(%r12),%xmm8
1206 pxor %xmm7,%xmm0
1207 movdqu 32(%r12),%xmm9
1208 pxor %xmm8,%xmm5
1209 movdqu 48(%r12),%xmm10
1210 pxor %xmm9,%xmm3
1211 movdqu 64(%r12),%xmm11
1212 pxor %xmm10,%xmm1
1213 movdqu 80(%r12),%xmm14
1214 pxor %xmm11,%xmm6
1215 movdqu %xmm15,0(%r13)
1216 movdqu %xmm0,16(%r13)
1217 movdqu %xmm5,32(%r13)
1218 movdqu %xmm3,48(%r13)
1219 movdqu %xmm1,64(%r13)
1220 movdqu %xmm6,80(%r13)
1221 jmp L$cbc_dec_done
1222.p2align 4
1223L$cbc_dec_five:
1224 movdqa %xmm14,32(%rbp)
1225 call _bsaes_decrypt8
1226 pxor 32(%rbp),%xmm15
1227 movdqu 0(%r12),%xmm7
1228 movdqu 16(%r12),%xmm8
1229 pxor %xmm7,%xmm0
1230 movdqu 32(%r12),%xmm9
1231 pxor %xmm8,%xmm5
1232 movdqu 48(%r12),%xmm10
1233 pxor %xmm9,%xmm3
1234 movdqu 64(%r12),%xmm14
1235 pxor %xmm10,%xmm1
1236 movdqu %xmm15,0(%r13)
1237 movdqu %xmm0,16(%r13)
1238 movdqu %xmm5,32(%r13)
1239 movdqu %xmm3,48(%r13)
1240 movdqu %xmm1,64(%r13)
1241 jmp L$cbc_dec_done
1242.p2align 4
1243L$cbc_dec_four:
1244 movdqa %xmm14,32(%rbp)
1245 call _bsaes_decrypt8
1246 pxor 32(%rbp),%xmm15
1247 movdqu 0(%r12),%xmm7
1248 movdqu 16(%r12),%xmm8
1249 pxor %xmm7,%xmm0
1250 movdqu 32(%r12),%xmm9
1251 pxor %xmm8,%xmm5
1252 movdqu 48(%r12),%xmm14
1253 pxor %xmm9,%xmm3
1254 movdqu %xmm15,0(%r13)
1255 movdqu %xmm0,16(%r13)
1256 movdqu %xmm5,32(%r13)
1257 movdqu %xmm3,48(%r13)
1258 jmp L$cbc_dec_done
1259.p2align 4
1260L$cbc_dec_three:
1261 movdqa %xmm14,32(%rbp)
1262 call _bsaes_decrypt8
1263 pxor 32(%rbp),%xmm15
1264 movdqu 0(%r12),%xmm7
1265 movdqu 16(%r12),%xmm8
1266 pxor %xmm7,%xmm0
1267 movdqu 32(%r12),%xmm14
1268 pxor %xmm8,%xmm5
1269 movdqu %xmm15,0(%r13)
1270 movdqu %xmm0,16(%r13)
1271 movdqu %xmm5,32(%r13)
1272 jmp L$cbc_dec_done
1273.p2align 4
1274L$cbc_dec_two:
1275 movdqa %xmm14,32(%rbp)
1276 call _bsaes_decrypt8
1277 pxor 32(%rbp),%xmm15
1278 movdqu 0(%r12),%xmm7
1279 movdqu 16(%r12),%xmm14
1280 pxor %xmm7,%xmm0
1281 movdqu %xmm15,0(%r13)
1282 movdqu %xmm0,16(%r13)
1283 jmp L$cbc_dec_done
1284.p2align 4
1285L$cbc_dec_one:
1286 leaq (%r12),%rdi
1287 leaq 32(%rbp),%rsi
1288 leaq (%r15),%rdx
1289 call _asm_AES_decrypt
1290 pxor 32(%rbp),%xmm14
1291 movdqu %xmm14,(%r13)
1292 movdqa %xmm15,%xmm14
1293
1294L$cbc_dec_done:
1295 movdqu %xmm14,(%rbx)
1296 leaq (%rsp),%rax
1297 pxor %xmm0,%xmm0
1298L$cbc_dec_bzero:
1299 movdqa %xmm0,0(%rax)
1300 movdqa %xmm0,16(%rax)
1301 leaq 32(%rax),%rax
1302 cmpq %rax,%rbp
1303 ja L$cbc_dec_bzero
1304
Robert Sloana94fe052017-02-21 08:49:28 -0800[diff] [blame]1305 leaq 120(%rbp),%rax
1306 movq -48(%rax),%r15
1307 movq -40(%rax),%r14
1308 movq -32(%rax),%r13
1309 movq -24(%rax),%r12
1310 movq -16(%rax),%rbx
1311 movq -8(%rax),%rbp
1312 leaq (%rax),%rsp
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]1313L$cbc_dec_epilogue:
1314 .byte 0xf3,0xc3
1315
1316
1317.globl _bsaes_ctr32_encrypt_blocks
1318.private_extern _bsaes_ctr32_encrypt_blocks
1319
1320.p2align 4
1321_bsaes_ctr32_encrypt_blocks:
1322 movq %rsp,%rax
1323L$ctr_enc_prologue:
1324 pushq %rbp
1325 pushq %rbx
1326 pushq %r12
1327 pushq %r13
1328 pushq %r14
1329 pushq %r15
1330 leaq -72(%rsp),%rsp
1331 movq %rsp,%rbp
1332 movdqu (%r8),%xmm0
1333 movl 240(%rcx),%eax
1334 movq %rdi,%r12
1335 movq %rsi,%r13
1336 movq %rdx,%r14
1337 movq %rcx,%r15
1338 movdqa %xmm0,32(%rbp)
1339 cmpq $8,%rdx
1340 jb L$ctr_enc_short
1341
1342 movl %eax,%ebx
1343 shlq $7,%rax
1344 subq $96,%rax
1345 subq %rax,%rsp
1346
1347 movq %rsp,%rax
1348 movq %r15,%rcx
1349 movl %ebx,%r10d
1350 call _bsaes_key_convert
1351 pxor %xmm6,%xmm7
1352 movdqa %xmm7,(%rax)
1353
1354 movdqa (%rsp),%xmm8
1355 leaq L$ADD1(%rip),%r11
1356 movdqa 32(%rbp),%xmm15
1357 movdqa -32(%r11),%xmm7
1358.byte 102,68,15,56,0,199
1359.byte 102,68,15,56,0,255
1360 movdqa %xmm8,(%rsp)
1361 jmp L$ctr_enc_loop
1362.p2align 4
1363L$ctr_enc_loop:
1364 movdqa %xmm15,32(%rbp)
1365 movdqa %xmm15,%xmm0
1366 movdqa %xmm15,%xmm1
1367 paddd 0(%r11),%xmm0
1368 movdqa %xmm15,%xmm2
1369 paddd 16(%r11),%xmm1
1370 movdqa %xmm15,%xmm3
1371 paddd 32(%r11),%xmm2
1372 movdqa %xmm15,%xmm4
1373 paddd 48(%r11),%xmm3
1374 movdqa %xmm15,%xmm5
1375 paddd 64(%r11),%xmm4
1376 movdqa %xmm15,%xmm6
1377 paddd 80(%r11),%xmm5
1378 paddd 96(%r11),%xmm6
1379
1380
1381
1382 movdqa (%rsp),%xmm8
1383 leaq 16(%rsp),%rax
1384 movdqa -16(%r11),%xmm7
1385 pxor %xmm8,%xmm15
1386 pxor %xmm8,%xmm0
1387 pxor %xmm8,%xmm1
1388 pxor %xmm8,%xmm2
1389.byte 102,68,15,56,0,255
1390.byte 102,15,56,0,199
1391 pxor %xmm8,%xmm3
1392 pxor %xmm8,%xmm4
1393.byte 102,15,56,0,207
1394.byte 102,15,56,0,215
1395 pxor %xmm8,%xmm5
1396 pxor %xmm8,%xmm6
1397.byte 102,15,56,0,223
1398.byte 102,15,56,0,231
1399.byte 102,15,56,0,239
1400.byte 102,15,56,0,247
1401 leaq L$BS0(%rip),%r11
1402 movl %ebx,%r10d
1403
1404 call _bsaes_encrypt8_bitslice
1405
1406 subq $8,%r14
1407 jc L$ctr_enc_loop_done
1408
1409 movdqu 0(%r12),%xmm7
1410 movdqu 16(%r12),%xmm8
1411 movdqu 32(%r12),%xmm9
1412 movdqu 48(%r12),%xmm10
1413 movdqu 64(%r12),%xmm11
1414 movdqu 80(%r12),%xmm12
1415 movdqu 96(%r12),%xmm13
1416 movdqu 112(%r12),%xmm14
1417 leaq 128(%r12),%r12
1418 pxor %xmm15,%xmm7
1419 movdqa 32(%rbp),%xmm15
1420 pxor %xmm8,%xmm0
1421 movdqu %xmm7,0(%r13)
1422 pxor %xmm9,%xmm3
1423 movdqu %xmm0,16(%r13)
1424 pxor %xmm10,%xmm5
1425 movdqu %xmm3,32(%r13)
1426 pxor %xmm11,%xmm2
1427 movdqu %xmm5,48(%r13)
1428 pxor %xmm12,%xmm6
1429 movdqu %xmm2,64(%r13)
1430 pxor %xmm13,%xmm1
1431 movdqu %xmm6,80(%r13)
1432 pxor %xmm14,%xmm4
1433 movdqu %xmm1,96(%r13)
1434 leaq L$ADD1(%rip),%r11
1435 movdqu %xmm4,112(%r13)
1436 leaq 128(%r13),%r13
1437 paddd 112(%r11),%xmm15
1438 jnz L$ctr_enc_loop
1439
1440 jmp L$ctr_enc_done
1441.p2align 4
1442L$ctr_enc_loop_done:
1443 addq $8,%r14
1444 movdqu 0(%r12),%xmm7
1445 pxor %xmm7,%xmm15
1446 movdqu %xmm15,0(%r13)
1447 cmpq $2,%r14
1448 jb L$ctr_enc_done
1449 movdqu 16(%r12),%xmm8
1450 pxor %xmm8,%xmm0
1451 movdqu %xmm0,16(%r13)
1452 je L$ctr_enc_done
1453 movdqu 32(%r12),%xmm9
1454 pxor %xmm9,%xmm3
1455 movdqu %xmm3,32(%r13)
1456 cmpq $4,%r14
1457 jb L$ctr_enc_done
1458 movdqu 48(%r12),%xmm10
1459 pxor %xmm10,%xmm5
1460 movdqu %xmm5,48(%r13)
1461 je L$ctr_enc_done
1462 movdqu 64(%r12),%xmm11
1463 pxor %xmm11,%xmm2
1464 movdqu %xmm2,64(%r13)
1465 cmpq $6,%r14
1466 jb L$ctr_enc_done
1467 movdqu 80(%r12),%xmm12
1468 pxor %xmm12,%xmm6
1469 movdqu %xmm6,80(%r13)
1470 je L$ctr_enc_done
1471 movdqu 96(%r12),%xmm13
1472 pxor %xmm13,%xmm1
1473 movdqu %xmm1,96(%r13)
1474 jmp L$ctr_enc_done
1475
1476.p2align 4
1477L$ctr_enc_short:
1478 leaq 32(%rbp),%rdi
1479 leaq 48(%rbp),%rsi
1480 leaq (%r15),%rdx
1481 call _asm_AES_encrypt
1482 movdqu (%r12),%xmm0
1483 leaq 16(%r12),%r12
1484 movl 44(%rbp),%eax
1485 bswapl %eax
1486 pxor 48(%rbp),%xmm0
1487 incl %eax
1488 movdqu %xmm0,(%r13)
1489 bswapl %eax
1490 leaq 16(%r13),%r13
1491 movl %eax,44(%rsp)
1492 decq %r14
1493 jnz L$ctr_enc_short
1494
1495L$ctr_enc_done:
1496 leaq (%rsp),%rax
1497 pxor %xmm0,%xmm0
1498L$ctr_enc_bzero:
1499 movdqa %xmm0,0(%rax)
1500 movdqa %xmm0,16(%rax)
1501 leaq 32(%rax),%rax
1502 cmpq %rax,%rbp
1503 ja L$ctr_enc_bzero
1504
Robert Sloana94fe052017-02-21 08:49:28 -0800[diff] [blame]1505 leaq 120(%rbp),%rax
1506 movq -48(%rax),%r15
1507 movq -40(%rax),%r14
1508 movq -32(%rax),%r13
1509 movq -24(%rax),%r12
1510 movq -16(%rax),%rbx
1511 movq -8(%rax),%rbp
1512 leaq (%rax),%rsp
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]1513L$ctr_enc_epilogue:
1514 .byte 0xf3,0xc3
1515
1516.globl _bsaes_xts_encrypt
1517.private_extern _bsaes_xts_encrypt
1518
1519.p2align 4
1520_bsaes_xts_encrypt:
1521 movq %rsp,%rax
1522L$xts_enc_prologue:
1523 pushq %rbp
1524 pushq %rbx
1525 pushq %r12
1526 pushq %r13
1527 pushq %r14
1528 pushq %r15
1529 leaq -72(%rsp),%rsp
1530 movq %rsp,%rbp
1531 movq %rdi,%r12
1532 movq %rsi,%r13
1533 movq %rdx,%r14
1534 movq %rcx,%r15
1535
1536 leaq (%r9),%rdi
1537 leaq 32(%rbp),%rsi
1538 leaq (%r8),%rdx
1539 call _asm_AES_encrypt
1540
1541 movl 240(%r15),%eax
1542 movq %r14,%rbx
1543
1544 movl %eax,%edx
1545 shlq $7,%rax
1546 subq $96,%rax
1547 subq %rax,%rsp
1548
1549 movq %rsp,%rax
1550 movq %r15,%rcx
1551 movl %edx,%r10d
1552 call _bsaes_key_convert
1553 pxor %xmm6,%xmm7
1554 movdqa %xmm7,(%rax)
1555
1556 andq $-16,%r14
David Benjamin4969cc92016-04-22 15:02:23 -0400[diff] [blame]1557 subq $0x80,%rsp
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]1558 movdqa 32(%rbp),%xmm6
1559
1560 pxor %xmm14,%xmm14
1561 movdqa L$xts_magic(%rip),%xmm12
1562 pcmpgtd %xmm6,%xmm14
1563
David Benjamin4969cc92016-04-22 15:02:23 -0400[diff] [blame]1564 subq $0x80,%r14
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]1565 jc L$xts_enc_short
1566 jmp L$xts_enc_loop
1567
1568.p2align 4
1569L$xts_enc_loop:
David Benjamin4969cc92016-04-22 15:02:23 -0400[diff] [blame]1570 pshufd $0x13,%xmm14,%xmm13
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]1571 pxor %xmm14,%xmm14
1572 movdqa %xmm6,%xmm15
1573 movdqa %xmm6,0(%rsp)
1574 paddq %xmm6,%xmm6
1575 pand %xmm12,%xmm13
1576 pcmpgtd %xmm6,%xmm14
1577 pxor %xmm13,%xmm6
David Benjamin4969cc92016-04-22 15:02:23 -0400[diff] [blame]1578 pshufd $0x13,%xmm14,%xmm13
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]1579 pxor %xmm14,%xmm14
1580 movdqa %xmm6,%xmm0
1581 movdqa %xmm6,16(%rsp)
1582 paddq %xmm6,%xmm6
1583 pand %xmm12,%xmm13
1584 pcmpgtd %xmm6,%xmm14
1585 pxor %xmm13,%xmm6
1586 movdqu 0(%r12),%xmm7
David Benjamin4969cc92016-04-22 15:02:23 -0400[diff] [blame]1587 pshufd $0x13,%xmm14,%xmm13
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]1588 pxor %xmm14,%xmm14
1589 movdqa %xmm6,%xmm1
1590 movdqa %xmm6,32(%rsp)
1591 paddq %xmm6,%xmm6
1592 pand %xmm12,%xmm13
1593 pcmpgtd %xmm6,%xmm14
1594 pxor %xmm13,%xmm6
1595 movdqu 16(%r12),%xmm8
1596 pxor %xmm7,%xmm15
David Benjamin4969cc92016-04-22 15:02:23 -0400[diff] [blame]1597 pshufd $0x13,%xmm14,%xmm13
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]1598 pxor %xmm14,%xmm14
1599 movdqa %xmm6,%xmm2
1600 movdqa %xmm6,48(%rsp)
1601 paddq %xmm6,%xmm6
1602 pand %xmm12,%xmm13
1603 pcmpgtd %xmm6,%xmm14
1604 pxor %xmm13,%xmm6
1605 movdqu 32(%r12),%xmm9
1606 pxor %xmm8,%xmm0
David Benjamin4969cc92016-04-22 15:02:23 -0400[diff] [blame]1607 pshufd $0x13,%xmm14,%xmm13
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]1608 pxor %xmm14,%xmm14
1609 movdqa %xmm6,%xmm3
1610 movdqa %xmm6,64(%rsp)
1611 paddq %xmm6,%xmm6
1612 pand %xmm12,%xmm13
1613 pcmpgtd %xmm6,%xmm14
1614 pxor %xmm13,%xmm6
1615 movdqu 48(%r12),%xmm10
1616 pxor %xmm9,%xmm1
David Benjamin4969cc92016-04-22 15:02:23 -0400[diff] [blame]1617 pshufd $0x13,%xmm14,%xmm13
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]1618 pxor %xmm14,%xmm14
1619 movdqa %xmm6,%xmm4
1620 movdqa %xmm6,80(%rsp)
1621 paddq %xmm6,%xmm6
1622 pand %xmm12,%xmm13
1623 pcmpgtd %xmm6,%xmm14
1624 pxor %xmm13,%xmm6
1625 movdqu 64(%r12),%xmm11
1626 pxor %xmm10,%xmm2
David Benjamin4969cc92016-04-22 15:02:23 -0400[diff] [blame]1627 pshufd $0x13,%xmm14,%xmm13
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]1628 pxor %xmm14,%xmm14
1629 movdqa %xmm6,%xmm5
1630 movdqa %xmm6,96(%rsp)
1631 paddq %xmm6,%xmm6
1632 pand %xmm12,%xmm13
1633 pcmpgtd %xmm6,%xmm14
1634 pxor %xmm13,%xmm6
1635 movdqu 80(%r12),%xmm12
1636 pxor %xmm11,%xmm3
1637 movdqu 96(%r12),%xmm13
1638 pxor %xmm12,%xmm4
1639 movdqu 112(%r12),%xmm14
1640 leaq 128(%r12),%r12
1641 movdqa %xmm6,112(%rsp)
1642 pxor %xmm13,%xmm5
1643 leaq 128(%rsp),%rax
1644 pxor %xmm14,%xmm6
1645 movl %edx,%r10d
1646
1647 call _bsaes_encrypt8
1648
1649 pxor 0(%rsp),%xmm15
1650 pxor 16(%rsp),%xmm0
1651 movdqu %xmm15,0(%r13)
1652 pxor 32(%rsp),%xmm3
1653 movdqu %xmm0,16(%r13)
1654 pxor 48(%rsp),%xmm5
1655 movdqu %xmm3,32(%r13)
1656 pxor 64(%rsp),%xmm2
1657 movdqu %xmm5,48(%r13)
1658 pxor 80(%rsp),%xmm6
1659 movdqu %xmm2,64(%r13)
1660 pxor 96(%rsp),%xmm1
1661 movdqu %xmm6,80(%r13)
1662 pxor 112(%rsp),%xmm4
1663 movdqu %xmm1,96(%r13)
1664 movdqu %xmm4,112(%r13)
1665 leaq 128(%r13),%r13
1666
1667 movdqa 112(%rsp),%xmm6
1668 pxor %xmm14,%xmm14
1669 movdqa L$xts_magic(%rip),%xmm12
1670 pcmpgtd %xmm6,%xmm14
David Benjamin4969cc92016-04-22 15:02:23 -0400[diff] [blame]1671 pshufd $0x13,%xmm14,%xmm13
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]1672 pxor %xmm14,%xmm14
1673 paddq %xmm6,%xmm6
1674 pand %xmm12,%xmm13
1675 pcmpgtd %xmm6,%xmm14
1676 pxor %xmm13,%xmm6
1677
David Benjamin4969cc92016-04-22 15:02:23 -0400[diff] [blame]1678 subq $0x80,%r14
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]1679 jnc L$xts_enc_loop
1680
1681L$xts_enc_short:
David Benjamin4969cc92016-04-22 15:02:23 -0400[diff] [blame]1682 addq $0x80,%r14
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]1683 jz L$xts_enc_done
David Benjamin4969cc92016-04-22 15:02:23 -0400[diff] [blame]1684 pshufd $0x13,%xmm14,%xmm13
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]1685 pxor %xmm14,%xmm14
1686 movdqa %xmm6,%xmm15
1687 movdqa %xmm6,0(%rsp)
1688 paddq %xmm6,%xmm6
1689 pand %xmm12,%xmm13
1690 pcmpgtd %xmm6,%xmm14
1691 pxor %xmm13,%xmm6
David Benjamin4969cc92016-04-22 15:02:23 -0400[diff] [blame]1692 pshufd $0x13,%xmm14,%xmm13
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]1693 pxor %xmm14,%xmm14
1694 movdqa %xmm6,%xmm0
1695 movdqa %xmm6,16(%rsp)
1696 paddq %xmm6,%xmm6
1697 pand %xmm12,%xmm13
1698 pcmpgtd %xmm6,%xmm14
1699 pxor %xmm13,%xmm6
1700 movdqu 0(%r12),%xmm7
1701 cmpq $16,%r14
1702 je L$xts_enc_1
David Benjamin4969cc92016-04-22 15:02:23 -0400[diff] [blame]1703 pshufd $0x13,%xmm14,%xmm13
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]1704 pxor %xmm14,%xmm14
1705 movdqa %xmm6,%xmm1
1706 movdqa %xmm6,32(%rsp)
1707 paddq %xmm6,%xmm6
1708 pand %xmm12,%xmm13
1709 pcmpgtd %xmm6,%xmm14
1710 pxor %xmm13,%xmm6
1711 movdqu 16(%r12),%xmm8
1712 cmpq $32,%r14
1713 je L$xts_enc_2
1714 pxor %xmm7,%xmm15
David Benjamin4969cc92016-04-22 15:02:23 -0400[diff] [blame]1715 pshufd $0x13,%xmm14,%xmm13
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]1716 pxor %xmm14,%xmm14
1717 movdqa %xmm6,%xmm2
1718 movdqa %xmm6,48(%rsp)
1719 paddq %xmm6,%xmm6
1720 pand %xmm12,%xmm13
1721 pcmpgtd %xmm6,%xmm14
1722 pxor %xmm13,%xmm6
1723 movdqu 32(%r12),%xmm9
1724 cmpq $48,%r14
1725 je L$xts_enc_3
1726 pxor %xmm8,%xmm0
David Benjamin4969cc92016-04-22 15:02:23 -0400[diff] [blame]1727 pshufd $0x13,%xmm14,%xmm13
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]1728 pxor %xmm14,%xmm14
1729 movdqa %xmm6,%xmm3
1730 movdqa %xmm6,64(%rsp)
1731 paddq %xmm6,%xmm6
1732 pand %xmm12,%xmm13
1733 pcmpgtd %xmm6,%xmm14
1734 pxor %xmm13,%xmm6
1735 movdqu 48(%r12),%xmm10
1736 cmpq $64,%r14
1737 je L$xts_enc_4
1738 pxor %xmm9,%xmm1
David Benjamin4969cc92016-04-22 15:02:23 -0400[diff] [blame]1739 pshufd $0x13,%xmm14,%xmm13
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]1740 pxor %xmm14,%xmm14
1741 movdqa %xmm6,%xmm4
1742 movdqa %xmm6,80(%rsp)
1743 paddq %xmm6,%xmm6
1744 pand %xmm12,%xmm13
1745 pcmpgtd %xmm6,%xmm14
1746 pxor %xmm13,%xmm6
1747 movdqu 64(%r12),%xmm11
1748 cmpq $80,%r14
1749 je L$xts_enc_5
1750 pxor %xmm10,%xmm2
David Benjamin4969cc92016-04-22 15:02:23 -0400[diff] [blame]1751 pshufd $0x13,%xmm14,%xmm13
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]1752 pxor %xmm14,%xmm14
1753 movdqa %xmm6,%xmm5
1754 movdqa %xmm6,96(%rsp)
1755 paddq %xmm6,%xmm6
1756 pand %xmm12,%xmm13
1757 pcmpgtd %xmm6,%xmm14
1758 pxor %xmm13,%xmm6
1759 movdqu 80(%r12),%xmm12
1760 cmpq $96,%r14
1761 je L$xts_enc_6
1762 pxor %xmm11,%xmm3
1763 movdqu 96(%r12),%xmm13
1764 pxor %xmm12,%xmm4
1765 movdqa %xmm6,112(%rsp)
1766 leaq 112(%r12),%r12
1767 pxor %xmm13,%xmm5
1768 leaq 128(%rsp),%rax
1769 movl %edx,%r10d
1770
1771 call _bsaes_encrypt8
1772
1773 pxor 0(%rsp),%xmm15
1774 pxor 16(%rsp),%xmm0
1775 movdqu %xmm15,0(%r13)
1776 pxor 32(%rsp),%xmm3
1777 movdqu %xmm0,16(%r13)
1778 pxor 48(%rsp),%xmm5
1779 movdqu %xmm3,32(%r13)
1780 pxor 64(%rsp),%xmm2
1781 movdqu %xmm5,48(%r13)
1782 pxor 80(%rsp),%xmm6
1783 movdqu %xmm2,64(%r13)
1784 pxor 96(%rsp),%xmm1
1785 movdqu %xmm6,80(%r13)
1786 movdqu %xmm1,96(%r13)
1787 leaq 112(%r13),%r13
1788
1789 movdqa 112(%rsp),%xmm6
1790 jmp L$xts_enc_done
1791.p2align 4
1792L$xts_enc_6:
1793 pxor %xmm11,%xmm3
1794 leaq 96(%r12),%r12
1795 pxor %xmm12,%xmm4
1796 leaq 128(%rsp),%rax
1797 movl %edx,%r10d
1798
1799 call _bsaes_encrypt8
1800
1801 pxor 0(%rsp),%xmm15
1802 pxor 16(%rsp),%xmm0
1803 movdqu %xmm15,0(%r13)
1804 pxor 32(%rsp),%xmm3
1805 movdqu %xmm0,16(%r13)
1806 pxor 48(%rsp),%xmm5
1807 movdqu %xmm3,32(%r13)
1808 pxor 64(%rsp),%xmm2
1809 movdqu %xmm5,48(%r13)
1810 pxor 80(%rsp),%xmm6
1811 movdqu %xmm2,64(%r13)
1812 movdqu %xmm6,80(%r13)
1813 leaq 96(%r13),%r13
1814
1815 movdqa 96(%rsp),%xmm6
1816 jmp L$xts_enc_done
1817.p2align 4
1818L$xts_enc_5:
1819 pxor %xmm10,%xmm2
1820 leaq 80(%r12),%r12
1821 pxor %xmm11,%xmm3
1822 leaq 128(%rsp),%rax
1823 movl %edx,%r10d
1824
1825 call _bsaes_encrypt8
1826
1827 pxor 0(%rsp),%xmm15
1828 pxor 16(%rsp),%xmm0
1829 movdqu %xmm15,0(%r13)
1830 pxor 32(%rsp),%xmm3
1831 movdqu %xmm0,16(%r13)
1832 pxor 48(%rsp),%xmm5
1833 movdqu %xmm3,32(%r13)
1834 pxor 64(%rsp),%xmm2
1835 movdqu %xmm5,48(%r13)
1836 movdqu %xmm2,64(%r13)
1837 leaq 80(%r13),%r13
1838
1839 movdqa 80(%rsp),%xmm6
1840 jmp L$xts_enc_done
1841.p2align 4
1842L$xts_enc_4:
1843 pxor %xmm9,%xmm1
1844 leaq 64(%r12),%r12
1845 pxor %xmm10,%xmm2
1846 leaq 128(%rsp),%rax
1847 movl %edx,%r10d
1848
1849 call _bsaes_encrypt8
1850
1851 pxor 0(%rsp),%xmm15
1852 pxor 16(%rsp),%xmm0
1853 movdqu %xmm15,0(%r13)
1854 pxor 32(%rsp),%xmm3
1855 movdqu %xmm0,16(%r13)
1856 pxor 48(%rsp),%xmm5
1857 movdqu %xmm3,32(%r13)
1858 movdqu %xmm5,48(%r13)
1859 leaq 64(%r13),%r13
1860
1861 movdqa 64(%rsp),%xmm6
1862 jmp L$xts_enc_done
1863.p2align 4
1864L$xts_enc_3:
1865 pxor %xmm8,%xmm0
1866 leaq 48(%r12),%r12
1867 pxor %xmm9,%xmm1
1868 leaq 128(%rsp),%rax
1869 movl %edx,%r10d
1870
1871 call _bsaes_encrypt8
1872
1873 pxor 0(%rsp),%xmm15
1874 pxor 16(%rsp),%xmm0
1875 movdqu %xmm15,0(%r13)
1876 pxor 32(%rsp),%xmm3
1877 movdqu %xmm0,16(%r13)
1878 movdqu %xmm3,32(%r13)
1879 leaq 48(%r13),%r13
1880
1881 movdqa 48(%rsp),%xmm6
1882 jmp L$xts_enc_done
1883.p2align 4
1884L$xts_enc_2:
1885 pxor %xmm7,%xmm15
1886 leaq 32(%r12),%r12
1887 pxor %xmm8,%xmm0
1888 leaq 128(%rsp),%rax
1889 movl %edx,%r10d
1890
1891 call _bsaes_encrypt8
1892
1893 pxor 0(%rsp),%xmm15
1894 pxor 16(%rsp),%xmm0
1895 movdqu %xmm15,0(%r13)
1896 movdqu %xmm0,16(%r13)
1897 leaq 32(%r13),%r13
1898
1899 movdqa 32(%rsp),%xmm6
1900 jmp L$xts_enc_done
1901.p2align 4
1902L$xts_enc_1:
1903 pxor %xmm15,%xmm7
1904 leaq 16(%r12),%r12
1905 movdqa %xmm7,32(%rbp)
1906 leaq 32(%rbp),%rdi
1907 leaq 32(%rbp),%rsi
1908 leaq (%r15),%rdx
1909 call _asm_AES_encrypt
1910 pxor 32(%rbp),%xmm15
1911
1912
1913
1914
1915
1916 movdqu %xmm15,0(%r13)
1917 leaq 16(%r13),%r13
1918
1919 movdqa 16(%rsp),%xmm6
1920
1921L$xts_enc_done:
1922 andl $15,%ebx
1923 jz L$xts_enc_ret
1924 movq %r13,%rdx
1925
1926L$xts_enc_steal:
1927 movzbl (%r12),%eax
1928 movzbl -16(%rdx),%ecx
1929 leaq 1(%r12),%r12
1930 movb %al,-16(%rdx)
1931 movb %cl,0(%rdx)
1932 leaq 1(%rdx),%rdx
1933 subl $1,%ebx
1934 jnz L$xts_enc_steal
1935
1936 movdqu -16(%r13),%xmm15
1937 leaq 32(%rbp),%rdi
1938 pxor %xmm6,%xmm15
1939 leaq 32(%rbp),%rsi
1940 movdqa %xmm15,32(%rbp)
1941 leaq (%r15),%rdx
1942 call _asm_AES_encrypt
1943 pxor 32(%rbp),%xmm6
1944 movdqu %xmm6,-16(%r13)
1945
1946L$xts_enc_ret:
1947 leaq (%rsp),%rax
1948 pxor %xmm0,%xmm0
1949L$xts_enc_bzero:
1950 movdqa %xmm0,0(%rax)
1951 movdqa %xmm0,16(%rax)
1952 leaq 32(%rax),%rax
1953 cmpq %rax,%rbp
1954 ja L$xts_enc_bzero
1955
Robert Sloana94fe052017-02-21 08:49:28 -0800[diff] [blame]1956 leaq 120(%rbp),%rax
1957 movq -48(%rax),%r15
1958 movq -40(%rax),%r14
1959 movq -32(%rax),%r13
1960 movq -24(%rax),%r12
1961 movq -16(%rax),%rbx
1962 movq -8(%rax),%rbp
1963 leaq (%rax),%rsp
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]1964L$xts_enc_epilogue:
1965 .byte 0xf3,0xc3
1966
1967
1968.globl _bsaes_xts_decrypt
1969.private_extern _bsaes_xts_decrypt
1970
1971.p2align 4
1972_bsaes_xts_decrypt:
1973 movq %rsp,%rax
1974L$xts_dec_prologue:
1975 pushq %rbp
1976 pushq %rbx
1977 pushq %r12
1978 pushq %r13
1979 pushq %r14
1980 pushq %r15
1981 leaq -72(%rsp),%rsp
1982 movq %rsp,%rbp
1983 movq %rdi,%r12
1984 movq %rsi,%r13
1985 movq %rdx,%r14
1986 movq %rcx,%r15
1987
1988 leaq (%r9),%rdi
1989 leaq 32(%rbp),%rsi
1990 leaq (%r8),%rdx
1991 call _asm_AES_encrypt
1992
1993 movl 240(%r15),%eax
1994 movq %r14,%rbx
1995
1996 movl %eax,%edx
1997 shlq $7,%rax
1998 subq $96,%rax
1999 subq %rax,%rsp
2000
2001 movq %rsp,%rax
2002 movq %r15,%rcx
2003 movl %edx,%r10d
2004 call _bsaes_key_convert
2005 pxor (%rsp),%xmm7
2006 movdqa %xmm6,(%rax)
2007 movdqa %xmm7,(%rsp)
2008
2009 xorl %eax,%eax
2010 andq $-16,%r14
2011 testl $15,%ebx
2012 setnz %al
2013 shlq $4,%rax
2014 subq %rax,%r14
2015
David Benjamin4969cc92016-04-22 15:02:23 -0400[diff] [blame]2016 subq $0x80,%rsp
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]2017 movdqa 32(%rbp),%xmm6
2018
2019 pxor %xmm14,%xmm14
2020 movdqa L$xts_magic(%rip),%xmm12
2021 pcmpgtd %xmm6,%xmm14
2022
David Benjamin4969cc92016-04-22 15:02:23 -0400[diff] [blame]2023 subq $0x80,%r14
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]2024 jc L$xts_dec_short
2025 jmp L$xts_dec_loop
2026
2027.p2align 4
2028L$xts_dec_loop:
David Benjamin4969cc92016-04-22 15:02:23 -0400[diff] [blame]2029 pshufd $0x13,%xmm14,%xmm13
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]2030 pxor %xmm14,%xmm14
2031 movdqa %xmm6,%xmm15
2032 movdqa %xmm6,0(%rsp)
2033 paddq %xmm6,%xmm6
2034 pand %xmm12,%xmm13
2035 pcmpgtd %xmm6,%xmm14
2036 pxor %xmm13,%xmm6
David Benjamin4969cc92016-04-22 15:02:23 -0400[diff] [blame]2037 pshufd $0x13,%xmm14,%xmm13
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]2038 pxor %xmm14,%xmm14
2039 movdqa %xmm6,%xmm0
2040 movdqa %xmm6,16(%rsp)
2041 paddq %xmm6,%xmm6
2042 pand %xmm12,%xmm13
2043 pcmpgtd %xmm6,%xmm14
2044 pxor %xmm13,%xmm6
2045 movdqu 0(%r12),%xmm7
David Benjamin4969cc92016-04-22 15:02:23 -0400[diff] [blame]2046 pshufd $0x13,%xmm14,%xmm13
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]2047 pxor %xmm14,%xmm14
2048 movdqa %xmm6,%xmm1
2049 movdqa %xmm6,32(%rsp)
2050 paddq %xmm6,%xmm6
2051 pand %xmm12,%xmm13
2052 pcmpgtd %xmm6,%xmm14
2053 pxor %xmm13,%xmm6
2054 movdqu 16(%r12),%xmm8
2055 pxor %xmm7,%xmm15
David Benjamin4969cc92016-04-22 15:02:23 -0400[diff] [blame]2056 pshufd $0x13,%xmm14,%xmm13
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]2057 pxor %xmm14,%xmm14
2058 movdqa %xmm6,%xmm2
2059 movdqa %xmm6,48(%rsp)
2060 paddq %xmm6,%xmm6
2061 pand %xmm12,%xmm13
2062 pcmpgtd %xmm6,%xmm14
2063 pxor %xmm13,%xmm6
2064 movdqu 32(%r12),%xmm9
2065 pxor %xmm8,%xmm0
David Benjamin4969cc92016-04-22 15:02:23 -0400[diff] [blame]2066 pshufd $0x13,%xmm14,%xmm13
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]2067 pxor %xmm14,%xmm14
2068 movdqa %xmm6,%xmm3
2069 movdqa %xmm6,64(%rsp)
2070 paddq %xmm6,%xmm6
2071 pand %xmm12,%xmm13
2072 pcmpgtd %xmm6,%xmm14
2073 pxor %xmm13,%xmm6
2074 movdqu 48(%r12),%xmm10
2075 pxor %xmm9,%xmm1
David Benjamin4969cc92016-04-22 15:02:23 -0400[diff] [blame]2076 pshufd $0x13,%xmm14,%xmm13
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]2077 pxor %xmm14,%xmm14
2078 movdqa %xmm6,%xmm4
2079 movdqa %xmm6,80(%rsp)
2080 paddq %xmm6,%xmm6
2081 pand %xmm12,%xmm13
2082 pcmpgtd %xmm6,%xmm14
2083 pxor %xmm13,%xmm6
2084 movdqu 64(%r12),%xmm11
2085 pxor %xmm10,%xmm2
David Benjamin4969cc92016-04-22 15:02:23 -0400[diff] [blame]2086 pshufd $0x13,%xmm14,%xmm13
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]2087 pxor %xmm14,%xmm14
2088 movdqa %xmm6,%xmm5
2089 movdqa %xmm6,96(%rsp)
2090 paddq %xmm6,%xmm6
2091 pand %xmm12,%xmm13
2092 pcmpgtd %xmm6,%xmm14
2093 pxor %xmm13,%xmm6
2094 movdqu 80(%r12),%xmm12
2095 pxor %xmm11,%xmm3
2096 movdqu 96(%r12),%xmm13
2097 pxor %xmm12,%xmm4
2098 movdqu 112(%r12),%xmm14
2099 leaq 128(%r12),%r12
2100 movdqa %xmm6,112(%rsp)
2101 pxor %xmm13,%xmm5
2102 leaq 128(%rsp),%rax
2103 pxor %xmm14,%xmm6
2104 movl %edx,%r10d
2105
2106 call _bsaes_decrypt8
2107
2108 pxor 0(%rsp),%xmm15
2109 pxor 16(%rsp),%xmm0
2110 movdqu %xmm15,0(%r13)
2111 pxor 32(%rsp),%xmm5
2112 movdqu %xmm0,16(%r13)
2113 pxor 48(%rsp),%xmm3
2114 movdqu %xmm5,32(%r13)
2115 pxor 64(%rsp),%xmm1
2116 movdqu %xmm3,48(%r13)
2117 pxor 80(%rsp),%xmm6
2118 movdqu %xmm1,64(%r13)
2119 pxor 96(%rsp),%xmm2
2120 movdqu %xmm6,80(%r13)
2121 pxor 112(%rsp),%xmm4
2122 movdqu %xmm2,96(%r13)
2123 movdqu %xmm4,112(%r13)
2124 leaq 128(%r13),%r13
2125
2126 movdqa 112(%rsp),%xmm6
2127 pxor %xmm14,%xmm14
2128 movdqa L$xts_magic(%rip),%xmm12
2129 pcmpgtd %xmm6,%xmm14
David Benjamin4969cc92016-04-22 15:02:23 -0400[diff] [blame]2130 pshufd $0x13,%xmm14,%xmm13
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]2131 pxor %xmm14,%xmm14
2132 paddq %xmm6,%xmm6
2133 pand %xmm12,%xmm13
2134 pcmpgtd %xmm6,%xmm14
2135 pxor %xmm13,%xmm6
2136
David Benjamin4969cc92016-04-22 15:02:23 -0400[diff] [blame]2137 subq $0x80,%r14
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]2138 jnc L$xts_dec_loop
2139
2140L$xts_dec_short:
David Benjamin4969cc92016-04-22 15:02:23 -0400[diff] [blame]2141 addq $0x80,%r14
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]2142 jz L$xts_dec_done
David Benjamin4969cc92016-04-22 15:02:23 -0400[diff] [blame]2143 pshufd $0x13,%xmm14,%xmm13
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]2144 pxor %xmm14,%xmm14
2145 movdqa %xmm6,%xmm15
2146 movdqa %xmm6,0(%rsp)
2147 paddq %xmm6,%xmm6
2148 pand %xmm12,%xmm13
2149 pcmpgtd %xmm6,%xmm14
2150 pxor %xmm13,%xmm6
David Benjamin4969cc92016-04-22 15:02:23 -0400[diff] [blame]2151 pshufd $0x13,%xmm14,%xmm13
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]2152 pxor %xmm14,%xmm14
2153 movdqa %xmm6,%xmm0
2154 movdqa %xmm6,16(%rsp)
2155 paddq %xmm6,%xmm6
2156 pand %xmm12,%xmm13
2157 pcmpgtd %xmm6,%xmm14
2158 pxor %xmm13,%xmm6
2159 movdqu 0(%r12),%xmm7
2160 cmpq $16,%r14
2161 je L$xts_dec_1
David Benjamin4969cc92016-04-22 15:02:23 -0400[diff] [blame]2162 pshufd $0x13,%xmm14,%xmm13
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]2163 pxor %xmm14,%xmm14
2164 movdqa %xmm6,%xmm1
2165 movdqa %xmm6,32(%rsp)
2166 paddq %xmm6,%xmm6
2167 pand %xmm12,%xmm13
2168 pcmpgtd %xmm6,%xmm14
2169 pxor %xmm13,%xmm6
2170 movdqu 16(%r12),%xmm8
2171 cmpq $32,%r14
2172 je L$xts_dec_2
2173 pxor %xmm7,%xmm15
David Benjamin4969cc92016-04-22 15:02:23 -0400[diff] [blame]2174 pshufd $0x13,%xmm14,%xmm13
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]2175 pxor %xmm14,%xmm14
2176 movdqa %xmm6,%xmm2
2177 movdqa %xmm6,48(%rsp)
2178 paddq %xmm6,%xmm6
2179 pand %xmm12,%xmm13
2180 pcmpgtd %xmm6,%xmm14
2181 pxor %xmm13,%xmm6
2182 movdqu 32(%r12),%xmm9
2183 cmpq $48,%r14
2184 je L$xts_dec_3
2185 pxor %xmm8,%xmm0
David Benjamin4969cc92016-04-22 15:02:23 -0400[diff] [blame]2186 pshufd $0x13,%xmm14,%xmm13
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]2187 pxor %xmm14,%xmm14
2188 movdqa %xmm6,%xmm3
2189 movdqa %xmm6,64(%rsp)
2190 paddq %xmm6,%xmm6
2191 pand %xmm12,%xmm13
2192 pcmpgtd %xmm6,%xmm14
2193 pxor %xmm13,%xmm6
2194 movdqu 48(%r12),%xmm10
2195 cmpq $64,%r14
2196 je L$xts_dec_4
2197 pxor %xmm9,%xmm1
David Benjamin4969cc92016-04-22 15:02:23 -0400[diff] [blame]2198 pshufd $0x13,%xmm14,%xmm13
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]2199 pxor %xmm14,%xmm14
2200 movdqa %xmm6,%xmm4
2201 movdqa %xmm6,80(%rsp)
2202 paddq %xmm6,%xmm6
2203 pand %xmm12,%xmm13
2204 pcmpgtd %xmm6,%xmm14
2205 pxor %xmm13,%xmm6
2206 movdqu 64(%r12),%xmm11
2207 cmpq $80,%r14
2208 je L$xts_dec_5
2209 pxor %xmm10,%xmm2
David Benjamin4969cc92016-04-22 15:02:23 -0400[diff] [blame]2210 pshufd $0x13,%xmm14,%xmm13
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]2211 pxor %xmm14,%xmm14
2212 movdqa %xmm6,%xmm5
2213 movdqa %xmm6,96(%rsp)
2214 paddq %xmm6,%xmm6
2215 pand %xmm12,%xmm13
2216 pcmpgtd %xmm6,%xmm14
2217 pxor %xmm13,%xmm6
2218 movdqu 80(%r12),%xmm12
2219 cmpq $96,%r14
2220 je L$xts_dec_6
2221 pxor %xmm11,%xmm3
2222 movdqu 96(%r12),%xmm13
2223 pxor %xmm12,%xmm4
2224 movdqa %xmm6,112(%rsp)
2225 leaq 112(%r12),%r12
2226 pxor %xmm13,%xmm5
2227 leaq 128(%rsp),%rax
2228 movl %edx,%r10d
2229
2230 call _bsaes_decrypt8
2231
2232 pxor 0(%rsp),%xmm15
2233 pxor 16(%rsp),%xmm0
2234 movdqu %xmm15,0(%r13)
2235 pxor 32(%rsp),%xmm5
2236 movdqu %xmm0,16(%r13)
2237 pxor 48(%rsp),%xmm3
2238 movdqu %xmm5,32(%r13)
2239 pxor 64(%rsp),%xmm1
2240 movdqu %xmm3,48(%r13)
2241 pxor 80(%rsp),%xmm6
2242 movdqu %xmm1,64(%r13)
2243 pxor 96(%rsp),%xmm2
2244 movdqu %xmm6,80(%r13)
2245 movdqu %xmm2,96(%r13)
2246 leaq 112(%r13),%r13
2247
2248 movdqa 112(%rsp),%xmm6
2249 jmp L$xts_dec_done
2250.p2align 4
2251L$xts_dec_6:
2252 pxor %xmm11,%xmm3
2253 leaq 96(%r12),%r12
2254 pxor %xmm12,%xmm4
2255 leaq 128(%rsp),%rax
2256 movl %edx,%r10d
2257
2258 call _bsaes_decrypt8
2259
2260 pxor 0(%rsp),%xmm15
2261 pxor 16(%rsp),%xmm0
2262 movdqu %xmm15,0(%r13)
2263 pxor 32(%rsp),%xmm5
2264 movdqu %xmm0,16(%r13)
2265 pxor 48(%rsp),%xmm3
2266 movdqu %xmm5,32(%r13)
2267 pxor 64(%rsp),%xmm1
2268 movdqu %xmm3,48(%r13)
2269 pxor 80(%rsp),%xmm6
2270 movdqu %xmm1,64(%r13)
2271 movdqu %xmm6,80(%r13)
2272 leaq 96(%r13),%r13
2273
2274 movdqa 96(%rsp),%xmm6
2275 jmp L$xts_dec_done
2276.p2align 4
2277L$xts_dec_5:
2278 pxor %xmm10,%xmm2
2279 leaq 80(%r12),%r12
2280 pxor %xmm11,%xmm3
2281 leaq 128(%rsp),%rax
2282 movl %edx,%r10d
2283
2284 call _bsaes_decrypt8
2285
2286 pxor 0(%rsp),%xmm15
2287 pxor 16(%rsp),%xmm0
2288 movdqu %xmm15,0(%r13)
2289 pxor 32(%rsp),%xmm5
2290 movdqu %xmm0,16(%r13)
2291 pxor 48(%rsp),%xmm3
2292 movdqu %xmm5,32(%r13)
2293 pxor 64(%rsp),%xmm1
2294 movdqu %xmm3,48(%r13)
2295 movdqu %xmm1,64(%r13)
2296 leaq 80(%r13),%r13
2297
2298 movdqa 80(%rsp),%xmm6
2299 jmp L$xts_dec_done
2300.p2align 4
2301L$xts_dec_4:
2302 pxor %xmm9,%xmm1
2303 leaq 64(%r12),%r12
2304 pxor %xmm10,%xmm2
2305 leaq 128(%rsp),%rax
2306 movl %edx,%r10d
2307
2308 call _bsaes_decrypt8
2309
2310 pxor 0(%rsp),%xmm15
2311 pxor 16(%rsp),%xmm0
2312 movdqu %xmm15,0(%r13)
2313 pxor 32(%rsp),%xmm5
2314 movdqu %xmm0,16(%r13)
2315 pxor 48(%rsp),%xmm3
2316 movdqu %xmm5,32(%r13)
2317 movdqu %xmm3,48(%r13)
2318 leaq 64(%r13),%r13
2319
2320 movdqa 64(%rsp),%xmm6
2321 jmp L$xts_dec_done
2322.p2align 4
2323L$xts_dec_3:
2324 pxor %xmm8,%xmm0
2325 leaq 48(%r12),%r12
2326 pxor %xmm9,%xmm1
2327 leaq 128(%rsp),%rax
2328 movl %edx,%r10d
2329
2330 call _bsaes_decrypt8
2331
2332 pxor 0(%rsp),%xmm15
2333 pxor 16(%rsp),%xmm0
2334 movdqu %xmm15,0(%r13)
2335 pxor 32(%rsp),%xmm5
2336 movdqu %xmm0,16(%r13)
2337 movdqu %xmm5,32(%r13)
2338 leaq 48(%r13),%r13
2339
2340 movdqa 48(%rsp),%xmm6
2341 jmp L$xts_dec_done
2342.p2align 4
2343L$xts_dec_2:
2344 pxor %xmm7,%xmm15
2345 leaq 32(%r12),%r12
2346 pxor %xmm8,%xmm0
2347 leaq 128(%rsp),%rax
2348 movl %edx,%r10d
2349
2350 call _bsaes_decrypt8
2351
2352 pxor 0(%rsp),%xmm15
2353 pxor 16(%rsp),%xmm0
2354 movdqu %xmm15,0(%r13)
2355 movdqu %xmm0,16(%r13)
2356 leaq 32(%r13),%r13
2357
2358 movdqa 32(%rsp),%xmm6
2359 jmp L$xts_dec_done
2360.p2align 4
2361L$xts_dec_1:
2362 pxor %xmm15,%xmm7
2363 leaq 16(%r12),%r12
2364 movdqa %xmm7,32(%rbp)
2365 leaq 32(%rbp),%rdi
2366 leaq 32(%rbp),%rsi
2367 leaq (%r15),%rdx
2368 call _asm_AES_decrypt
2369 pxor 32(%rbp),%xmm15
2370
2371
2372
2373
2374
2375 movdqu %xmm15,0(%r13)
2376 leaq 16(%r13),%r13
2377
2378 movdqa 16(%rsp),%xmm6
2379
2380L$xts_dec_done:
2381 andl $15,%ebx
2382 jz L$xts_dec_ret
2383
2384 pxor %xmm14,%xmm14
2385 movdqa L$xts_magic(%rip),%xmm12
2386 pcmpgtd %xmm6,%xmm14
David Benjamin4969cc92016-04-22 15:02:23 -0400[diff] [blame]2387 pshufd $0x13,%xmm14,%xmm13
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]2388 movdqa %xmm6,%xmm5
2389 paddq %xmm6,%xmm6
2390 pand %xmm12,%xmm13
2391 movdqu (%r12),%xmm15
2392 pxor %xmm13,%xmm6
2393
2394 leaq 32(%rbp),%rdi
2395 pxor %xmm6,%xmm15
2396 leaq 32(%rbp),%rsi
2397 movdqa %xmm15,32(%rbp)
2398 leaq (%r15),%rdx
2399 call _asm_AES_decrypt
2400 pxor 32(%rbp),%xmm6
2401 movq %r13,%rdx
2402 movdqu %xmm6,(%r13)
2403
2404L$xts_dec_steal:
2405 movzbl 16(%r12),%eax
2406 movzbl (%rdx),%ecx
2407 leaq 1(%r12),%r12
2408 movb %al,(%rdx)
2409 movb %cl,16(%rdx)
2410 leaq 1(%rdx),%rdx
2411 subl $1,%ebx
2412 jnz L$xts_dec_steal
2413
2414 movdqu (%r13),%xmm15
2415 leaq 32(%rbp),%rdi
2416 pxor %xmm5,%xmm15
2417 leaq 32(%rbp),%rsi
2418 movdqa %xmm15,32(%rbp)
2419 leaq (%r15),%rdx
2420 call _asm_AES_decrypt
2421 pxor 32(%rbp),%xmm5
2422 movdqu %xmm5,(%r13)
2423
2424L$xts_dec_ret:
2425 leaq (%rsp),%rax
2426 pxor %xmm0,%xmm0
2427L$xts_dec_bzero:
2428 movdqa %xmm0,0(%rax)
2429 movdqa %xmm0,16(%rax)
2430 leaq 32(%rax),%rax
2431 cmpq %rax,%rbp
2432 ja L$xts_dec_bzero
2433
Robert Sloana94fe052017-02-21 08:49:28 -0800[diff] [blame]2434 leaq 120(%rbp),%rax
2435 movq -48(%rax),%r15
2436 movq -40(%rax),%r14
2437 movq -32(%rax),%r13
2438 movq -24(%rax),%r12
2439 movq -16(%rax),%rbx
2440 movq -8(%rax),%rbp
2441 leaq (%rax),%rsp
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]2442L$xts_dec_epilogue:
2443 .byte 0xf3,0xc3
2444
2445
2446.p2align 6
2447_bsaes_const:
2448L$M0ISR:
2449.quad 0x0a0e0206070b0f03, 0x0004080c0d010509
2450L$ISRM0:
2451.quad 0x01040b0e0205080f, 0x0306090c00070a0d
2452L$ISR:
2453.quad 0x0504070602010003, 0x0f0e0d0c080b0a09
2454L$BS0:
2455.quad 0x5555555555555555, 0x5555555555555555
2456L$BS1:
2457.quad 0x3333333333333333, 0x3333333333333333
2458L$BS2:
2459.quad 0x0f0f0f0f0f0f0f0f, 0x0f0f0f0f0f0f0f0f
2460L$SR:
2461.quad 0x0504070600030201, 0x0f0e0d0c0a09080b
2462L$SRM0:
2463.quad 0x0304090e00050a0f, 0x01060b0c0207080d
2464L$M0SR:
2465.quad 0x0a0e02060f03070b, 0x0004080c05090d01
2466L$SWPUP:
2467.quad 0x0706050403020100, 0x0c0d0e0f0b0a0908
2468L$SWPUPM0SR:
2469.quad 0x0a0d02060c03070b, 0x0004080f05090e01
2470L$ADD1:
2471.quad 0x0000000000000000, 0x0000000100000000
2472L$ADD2:
2473.quad 0x0000000000000000, 0x0000000200000000
2474L$ADD3:
2475.quad 0x0000000000000000, 0x0000000300000000
2476L$ADD4:
2477.quad 0x0000000000000000, 0x0000000400000000
2478L$ADD5:
2479.quad 0x0000000000000000, 0x0000000500000000
2480L$ADD6:
2481.quad 0x0000000000000000, 0x0000000600000000
2482L$ADD7:
2483.quad 0x0000000000000000, 0x0000000700000000
2484L$ADD8:
2485.quad 0x0000000000000000, 0x0000000800000000
2486L$xts_magic:
2487.long 0x87,0,1,0
2488L$masks:
2489.quad 0x0101010101010101, 0x0101010101010101
2490.quad 0x0202020202020202, 0x0202020202020202
2491.quad 0x0404040404040404, 0x0404040404040404
2492.quad 0x0808080808080808, 0x0808080808080808
2493L$M0:
2494.quad 0x02060a0e03070b0f, 0x0004080c0105090d
2495L$63:
2496.quad 0x6363636363636363, 0x6363636363636363
2497.byte 66,105,116,45,115,108,105,99,101,100,32,65,69,83,32,102,111,114,32,120,56,54,95,54,52,47,83,83,83,69,51,44,32,69,109,105,108,105,97,32,75,195,164,115,112,101,114,44,32,80,101,116,101,114,32,83,99,104,119,97,98,101,44,32,65,110,100,121,32,80,111,108,121,97,107,111,118,0
2498.p2align 6
2499
2500#endif