Gitiles
Code Review Sign In
gerrit-public.fairphone.software / kernel / msm-5.4 / ad3793fc3945173f64d82d05d3ecde41f6c0435c / . / arch / arm64 / crypto / aes-modes.S
blob: 2674d43d1384b87614074a6e02e5c1305545c8bd [file] [log] [blame]
Ard Biesheuvel49788fe2014-03-21 10:19:17 +0100[diff] [blame]1/*
2 * linux/arch/arm64/crypto/aes-modes.S - chaining mode wrappers for AES
3 *
Ard Biesheuvel48606202017-02-03 14:49:37 +0000[diff] [blame]4 * Copyright (C) 2013 - 2017 Linaro Ltd <ard.biesheuvel@linaro.org>
Ard Biesheuvel49788fe2014-03-21 10:19:17 +0100[diff] [blame]5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 as
8 * published by the Free Software Foundation.
9 */
10
11/* included by aes-ce.S and aes-neon.S */
12
13 .text
14 .align 4
15
16/*
17 * There are several ways to instantiate this code:
18 * - no interleave, all inline
19 * - 2-way interleave, 2x calls out of line (-DINTERLEAVE=2)
20 * - 2-way interleave, all inline (-DINTERLEAVE=2 -DINTERLEAVE_INLINE)
21 * - 4-way interleave, 4x calls out of line (-DINTERLEAVE=4)
22 * - 4-way interleave, all inline (-DINTERLEAVE=4 -DINTERLEAVE_INLINE)
23 *
24 * Macros imported by this code:
25 * - enc_prepare - setup NEON registers for encryption
26 * - dec_prepare - setup NEON registers for decryption
27 * - enc_switch_key - change to new key after having prepared for encryption
28 * - encrypt_block - encrypt a single block
29 * - decrypt block - decrypt a single block
30 * - encrypt_block2x - encrypt 2 blocks in parallel (if INTERLEAVE == 2)
31 * - decrypt_block2x - decrypt 2 blocks in parallel (if INTERLEAVE == 2)
32 * - encrypt_block4x - encrypt 4 blocks in parallel (if INTERLEAVE == 4)
33 * - decrypt_block4x - decrypt 4 blocks in parallel (if INTERLEAVE == 4)
34 */
35
36#if defined(INTERLEAVE) && !defined(INTERLEAVE_INLINE)
37#define FRAME_PUSH stp x29, x30, [sp,#-16]! ; mov x29, sp
38#define FRAME_POP ldp x29, x30, [sp],#16
39
40#if INTERLEAVE == 2
41
42aes_encrypt_block2x:
43 encrypt_block2x v0, v1, w3, x2, x6, w7
44 ret
45ENDPROC(aes_encrypt_block2x)
46
47aes_decrypt_block2x:
48 decrypt_block2x v0, v1, w3, x2, x6, w7
49 ret
50ENDPROC(aes_decrypt_block2x)
51
52#elif INTERLEAVE == 4
53
54aes_encrypt_block4x:
55 encrypt_block4x v0, v1, v2, v3, w3, x2, x6, w7
56 ret
57ENDPROC(aes_encrypt_block4x)
58
59aes_decrypt_block4x:
60 decrypt_block4x v0, v1, v2, v3, w3, x2, x6, w7
61 ret
62ENDPROC(aes_decrypt_block4x)
63
64#else
65#error INTERLEAVE should equal 2 or 4
66#endif
67
68 .macro do_encrypt_block2x
69 bl aes_encrypt_block2x
70 .endm
71
72 .macro do_decrypt_block2x
73 bl aes_decrypt_block2x
74 .endm
75
76 .macro do_encrypt_block4x
77 bl aes_encrypt_block4x
78 .endm
79
80 .macro do_decrypt_block4x
81 bl aes_decrypt_block4x
82 .endm
83
84#else
85#define FRAME_PUSH
86#define FRAME_POP
87
88 .macro do_encrypt_block2x
89 encrypt_block2x v0, v1, w3, x2, x6, w7
90 .endm
91
92 .macro do_decrypt_block2x
93 decrypt_block2x v0, v1, w3, x2, x6, w7
94 .endm
95
96 .macro do_encrypt_block4x
97 encrypt_block4x v0, v1, v2, v3, w3, x2, x6, w7
98 .endm
99
100 .macro do_decrypt_block4x
101 decrypt_block4x v0, v1, v2, v3, w3, x2, x6, w7
102 .endm
103
104#endif
105
106 /*
107 * aes_ecb_encrypt(u8 out[], u8 const in[], u8 const rk[], int rounds,
108 * int blocks, int first)
109 * aes_ecb_decrypt(u8 out[], u8 const in[], u8 const rk[], int rounds,
110 * int blocks, int first)
111 */
112
113AES_ENTRY(aes_ecb_encrypt)
114 FRAME_PUSH
115 cbz w5, .LecbencloopNx
116
117 enc_prepare w3, x2, x5
118
119.LecbencloopNx:
120#if INTERLEAVE >= 2
121 subs w4, w4, #INTERLEAVE
122 bmi .Lecbenc1x
123#if INTERLEAVE == 2
124 ld1 {v0.16b-v1.16b}, [x1], #32 /* get 2 pt blocks */
125 do_encrypt_block2x
126 st1 {v0.16b-v1.16b}, [x0], #32
127#else
128 ld1 {v0.16b-v3.16b}, [x1], #64 /* get 4 pt blocks */
129 do_encrypt_block4x
130 st1 {v0.16b-v3.16b}, [x0], #64
131#endif
132 b .LecbencloopNx
133.Lecbenc1x:
134 adds w4, w4, #INTERLEAVE
135 beq .Lecbencout
136#endif
137.Lecbencloop:
138 ld1 {v0.16b}, [x1], #16 /* get next pt block */
139 encrypt_block v0, w3, x2, x5, w6
140 st1 {v0.16b}, [x0], #16
141 subs w4, w4, #1
142 bne .Lecbencloop
143.Lecbencout:
144 FRAME_POP
145 ret
146AES_ENDPROC(aes_ecb_encrypt)
147
148
149AES_ENTRY(aes_ecb_decrypt)
150 FRAME_PUSH
151 cbz w5, .LecbdecloopNx
152
153 dec_prepare w3, x2, x5
154
155.LecbdecloopNx:
156#if INTERLEAVE >= 2
157 subs w4, w4, #INTERLEAVE
158 bmi .Lecbdec1x
159#if INTERLEAVE == 2
160 ld1 {v0.16b-v1.16b}, [x1], #32 /* get 2 ct blocks */
161 do_decrypt_block2x
162 st1 {v0.16b-v1.16b}, [x0], #32
163#else
164 ld1 {v0.16b-v3.16b}, [x1], #64 /* get 4 ct blocks */
165 do_decrypt_block4x
166 st1 {v0.16b-v3.16b}, [x0], #64
167#endif
168 b .LecbdecloopNx
169.Lecbdec1x:
170 adds w4, w4, #INTERLEAVE
171 beq .Lecbdecout
172#endif
173.Lecbdecloop:
174 ld1 {v0.16b}, [x1], #16 /* get next ct block */
175 decrypt_block v0, w3, x2, x5, w6
176 st1 {v0.16b}, [x0], #16
177 subs w4, w4, #1
178 bne .Lecbdecloop
179.Lecbdecout:
180 FRAME_POP
181 ret
182AES_ENDPROC(aes_ecb_decrypt)
183
184
185 /*
186 * aes_cbc_encrypt(u8 out[], u8 const in[], u8 const rk[], int rounds,
187 * int blocks, u8 iv[], int first)
188 * aes_cbc_decrypt(u8 out[], u8 const in[], u8 const rk[], int rounds,
189 * int blocks, u8 iv[], int first)
190 */
191
192AES_ENTRY(aes_cbc_encrypt)
193 cbz w6, .Lcbcencloop
194
195 ld1 {v0.16b}, [x5] /* get iv */
Ard Biesheuvel11e3b722017-01-17 13:46:29 +0000[diff] [blame]196 enc_prepare w3, x2, x6
Ard Biesheuvel49788fe2014-03-21 10:19:17 +0100[diff] [blame]197
198.Lcbcencloop:
199 ld1 {v1.16b}, [x1], #16 /* get next pt block */
200 eor v0.16b, v0.16b, v1.16b /* ..and xor with iv */
Ard Biesheuvel11e3b722017-01-17 13:46:29 +0000[diff] [blame]201 encrypt_block v0, w3, x2, x6, w7
Ard Biesheuvel49788fe2014-03-21 10:19:17 +0100[diff] [blame]202 st1 {v0.16b}, [x0], #16
203 subs w4, w4, #1
204 bne .Lcbcencloop
Ard Biesheuvel11e3b722017-01-17 13:46:29 +0000[diff] [blame]205 st1 {v0.16b}, [x5] /* return iv */
Ard Biesheuvel49788fe2014-03-21 10:19:17 +0100[diff] [blame]206 ret
207AES_ENDPROC(aes_cbc_encrypt)
208
209
210AES_ENTRY(aes_cbc_decrypt)
211 FRAME_PUSH
212 cbz w6, .LcbcdecloopNx
213
214 ld1 {v7.16b}, [x5] /* get iv */
Ard Biesheuvel11e3b722017-01-17 13:46:29 +0000[diff] [blame]215 dec_prepare w3, x2, x6
Ard Biesheuvel49788fe2014-03-21 10:19:17 +0100[diff] [blame]216
217.LcbcdecloopNx:
218#if INTERLEAVE >= 2
219 subs w4, w4, #INTERLEAVE
220 bmi .Lcbcdec1x
221#if INTERLEAVE == 2
222 ld1 {v0.16b-v1.16b}, [x1], #32 /* get 2 ct blocks */
223 mov v2.16b, v0.16b
224 mov v3.16b, v1.16b
225 do_decrypt_block2x
226 eor v0.16b, v0.16b, v7.16b
227 eor v1.16b, v1.16b, v2.16b
228 mov v7.16b, v3.16b
229 st1 {v0.16b-v1.16b}, [x0], #32
230#else
231 ld1 {v0.16b-v3.16b}, [x1], #64 /* get 4 ct blocks */
232 mov v4.16b, v0.16b
233 mov v5.16b, v1.16b
234 mov v6.16b, v2.16b
235 do_decrypt_block4x
236 sub x1, x1, #16
237 eor v0.16b, v0.16b, v7.16b
238 eor v1.16b, v1.16b, v4.16b
239 ld1 {v7.16b}, [x1], #16 /* reload 1 ct block */
240 eor v2.16b, v2.16b, v5.16b
241 eor v3.16b, v3.16b, v6.16b
242 st1 {v0.16b-v3.16b}, [x0], #64
243#endif
244 b .LcbcdecloopNx
245.Lcbcdec1x:
246 adds w4, w4, #INTERLEAVE
247 beq .Lcbcdecout
248#endif
249.Lcbcdecloop:
250 ld1 {v1.16b}, [x1], #16 /* get next ct block */
251 mov v0.16b, v1.16b /* ...and copy to v0 */
Ard Biesheuvel11e3b722017-01-17 13:46:29 +0000[diff] [blame]252 decrypt_block v0, w3, x2, x6, w7
Ard Biesheuvel49788fe2014-03-21 10:19:17 +0100[diff] [blame]253 eor v0.16b, v0.16b, v7.16b /* xor with iv => pt */
254 mov v7.16b, v1.16b /* ct is next iv */
255 st1 {v0.16b}, [x0], #16
256 subs w4, w4, #1
257 bne .Lcbcdecloop
258.Lcbcdecout:
259 FRAME_POP
Ard Biesheuvel11e3b722017-01-17 13:46:29 +0000[diff] [blame]260 st1 {v7.16b}, [x5] /* return iv */
Ard Biesheuvel49788fe2014-03-21 10:19:17 +0100[diff] [blame]261 ret
262AES_ENDPROC(aes_cbc_decrypt)
263
264
265 /*
266 * aes_ctr_encrypt(u8 out[], u8 const in[], u8 const rk[], int rounds,
267 * int blocks, u8 ctr[], int first)
268 */
269
270AES_ENTRY(aes_ctr_encrypt)
271 FRAME_PUSH
Ard Biesheuvel11e3b722017-01-17 13:46:29 +0000[diff] [blame]272 cbz w6, .Lctrnotfirst /* 1st time around? */
Ard Biesheuvel49788fe2014-03-21 10:19:17 +0100[diff] [blame]273 enc_prepare w3, x2, x6
274 ld1 {v4.16b}, [x5]
Ard Biesheuvel11e3b722017-01-17 13:46:29 +0000[diff] [blame]275
276.Lctrnotfirst:
277 umov x8, v4.d[1] /* keep swabbed ctr in reg */
278 rev x8, x8
Ard Biesheuvel49788fe2014-03-21 10:19:17 +0100[diff] [blame]279#if INTERLEAVE >= 2
Ard Biesheuvel11e3b722017-01-17 13:46:29 +0000[diff] [blame]280 cmn w8, w4 /* 32 bit overflow? */
Ard Biesheuvel49788fe2014-03-21 10:19:17 +0100[diff] [blame]281 bcs .Lctrloop
282.LctrloopNx:
283 subs w4, w4, #INTERLEAVE
284 bmi .Lctr1x
285#if INTERLEAVE == 2
286 mov v0.8b, v4.8b
287 mov v1.8b, v4.8b
Ard Biesheuvel11e3b722017-01-17 13:46:29 +0000[diff] [blame]288 rev x7, x8
289 add x8, x8, #1
Ard Biesheuvel49788fe2014-03-21 10:19:17 +0100[diff] [blame]290 ins v0.d[1], x7
Ard Biesheuvel11e3b722017-01-17 13:46:29 +0000[diff] [blame]291 rev x7, x8
292 add x8, x8, #1
Ard Biesheuvel49788fe2014-03-21 10:19:17 +0100[diff] [blame]293 ins v1.d[1], x7
294 ld1 {v2.16b-v3.16b}, [x1], #32 /* get 2 input blocks */
295 do_encrypt_block2x
296 eor v0.16b, v0.16b, v2.16b
297 eor v1.16b, v1.16b, v3.16b
298 st1 {v0.16b-v1.16b}, [x0], #32
299#else
300 ldr q8, =0x30000000200000001 /* addends 1,2,3[,0] */
Ard Biesheuvel11e3b722017-01-17 13:46:29 +0000[diff] [blame]301 dup v7.4s, w8
Ard Biesheuvel49788fe2014-03-21 10:19:17 +0100[diff] [blame]302 mov v0.16b, v4.16b
303 add v7.4s, v7.4s, v8.4s
304 mov v1.16b, v4.16b
305 rev32 v8.16b, v7.16b
306 mov v2.16b, v4.16b
307 mov v3.16b, v4.16b
308 mov v1.s[3], v8.s[0]
309 mov v2.s[3], v8.s[1]
310 mov v3.s[3], v8.s[2]
311 ld1 {v5.16b-v7.16b}, [x1], #48 /* get 3 input blocks */
312 do_encrypt_block4x
313 eor v0.16b, v5.16b, v0.16b
314 ld1 {v5.16b}, [x1], #16 /* get 1 input block */
315 eor v1.16b, v6.16b, v1.16b
316 eor v2.16b, v7.16b, v2.16b
317 eor v3.16b, v5.16b, v3.16b
318 st1 {v0.16b-v3.16b}, [x0], #64
Ard Biesheuvel11e3b722017-01-17 13:46:29 +0000[diff] [blame]319 add x8, x8, #INTERLEAVE
Ard Biesheuvel49788fe2014-03-21 10:19:17 +0100[diff] [blame]320#endif
Ard Biesheuvel11e3b722017-01-17 13:46:29 +0000[diff] [blame]321 rev x7, x8
Ard Biesheuvel49788fe2014-03-21 10:19:17 +0100[diff] [blame]322 ins v4.d[1], x7
Ard Biesheuvel11e3b722017-01-17 13:46:29 +0000[diff] [blame]323 cbz w4, .Lctrout
Ard Biesheuvel49788fe2014-03-21 10:19:17 +0100[diff] [blame]324 b .LctrloopNx
Ard Biesheuvel49788fe2014-03-21 10:19:17 +0100[diff] [blame]325.Lctr1x:
326 adds w4, w4, #INTERLEAVE
327 beq .Lctrout
328#endif
329.Lctrloop:
330 mov v0.16b, v4.16b
331 encrypt_block v0, w3, x2, x6, w7
Ard Biesheuvel11e3b722017-01-17 13:46:29 +0000[diff] [blame]332
333 adds x8, x8, #1 /* increment BE ctr */
334 rev x7, x8
335 ins v4.d[1], x7
336 bcs .Lctrcarry /* overflow? */
337
338.Lctrcarrydone:
Ard Biesheuvel49788fe2014-03-21 10:19:17 +0100[diff] [blame]339 subs w4, w4, #1
Ard Biesheuvelccc5d512017-01-28 23:25:34 +0000[diff] [blame]340 bmi .Lctrtailblock /* blocks <0 means tail block */
Ard Biesheuvel49788fe2014-03-21 10:19:17 +0100[diff] [blame]341 ld1 {v3.16b}, [x1], #16
342 eor v3.16b, v0.16b, v3.16b
343 st1 {v3.16b}, [x0], #16
Ard Biesheuvel11e3b722017-01-17 13:46:29 +0000[diff] [blame]344 bne .Lctrloop
345
346.Lctrout:
347 st1 {v4.16b}, [x5] /* return next CTR value */
348 FRAME_POP
349 ret
350
Ard Biesheuvelccc5d512017-01-28 23:25:34 +0000[diff] [blame]351.Lctrtailblock:
352 st1 {v0.16b}, [x0]
Ard Biesheuvel11e3b722017-01-17 13:46:29 +0000[diff] [blame]353 FRAME_POP
354 ret
355
356.Lctrcarry:
Ard Biesheuvel49788fe2014-03-21 10:19:17 +0100[diff] [blame]357 umov x7, v4.d[0] /* load upper word of ctr */
358 rev x7, x7 /* ... to handle the carry */
359 add x7, x7, #1
360 rev x7, x7
361 ins v4.d[0], x7
Ard Biesheuvel11e3b722017-01-17 13:46:29 +0000[diff] [blame]362 b .Lctrcarrydone
Ard Biesheuvel49788fe2014-03-21 10:19:17 +0100[diff] [blame]363AES_ENDPROC(aes_ctr_encrypt)
364 .ltorg
365
366
367 /*
368 * aes_xts_decrypt(u8 out[], u8 const in[], u8 const rk1[], int rounds,
369 * int blocks, u8 const rk2[], u8 iv[], int first)
370 * aes_xts_decrypt(u8 out[], u8 const in[], u8 const rk1[], int rounds,
371 * int blocks, u8 const rk2[], u8 iv[], int first)
372 */
373
374 .macro next_tweak, out, in, const, tmp
375 sshr \tmp\().2d, \in\().2d, #63
376 and \tmp\().16b, \tmp\().16b, \const\().16b
377 add \out\().2d, \in\().2d, \in\().2d
378 ext \tmp\().16b, \tmp\().16b, \tmp\().16b, #8
379 eor \out\().16b, \out\().16b, \tmp\().16b
380 .endm
381
382.Lxts_mul_x:
Ard Biesheuvelcaf4b9e2016-10-11 19:15:19 +0100[diff] [blame]383CPU_LE( .quad 1, 0x87 )
384CPU_BE( .quad 0x87, 1 )
Ard Biesheuvel49788fe2014-03-21 10:19:17 +0100[diff] [blame]385
386AES_ENTRY(aes_xts_encrypt)
387 FRAME_PUSH
388 cbz w7, .LxtsencloopNx
389
390 ld1 {v4.16b}, [x6]
391 enc_prepare w3, x5, x6
392 encrypt_block v4, w3, x5, x6, w7 /* first tweak */
393 enc_switch_key w3, x2, x6
394 ldr q7, .Lxts_mul_x
395 b .LxtsencNx
396
397.LxtsencloopNx:
398 ldr q7, .Lxts_mul_x
399 next_tweak v4, v4, v7, v8
400.LxtsencNx:
401#if INTERLEAVE >= 2
402 subs w4, w4, #INTERLEAVE
403 bmi .Lxtsenc1x
404#if INTERLEAVE == 2
405 ld1 {v0.16b-v1.16b}, [x1], #32 /* get 2 pt blocks */
406 next_tweak v5, v4, v7, v8
407 eor v0.16b, v0.16b, v4.16b
408 eor v1.16b, v1.16b, v5.16b
409 do_encrypt_block2x
410 eor v0.16b, v0.16b, v4.16b
411 eor v1.16b, v1.16b, v5.16b
412 st1 {v0.16b-v1.16b}, [x0], #32
413 cbz w4, .LxtsencoutNx
414 next_tweak v4, v5, v7, v8
415 b .LxtsencNx
416.LxtsencoutNx:
417 mov v4.16b, v5.16b
418 b .Lxtsencout
419#else
420 ld1 {v0.16b-v3.16b}, [x1], #64 /* get 4 pt blocks */
421 next_tweak v5, v4, v7, v8
422 eor v0.16b, v0.16b, v4.16b
423 next_tweak v6, v5, v7, v8
424 eor v1.16b, v1.16b, v5.16b
425 eor v2.16b, v2.16b, v6.16b
426 next_tweak v7, v6, v7, v8
427 eor v3.16b, v3.16b, v7.16b
428 do_encrypt_block4x
429 eor v3.16b, v3.16b, v7.16b
430 eor v0.16b, v0.16b, v4.16b
431 eor v1.16b, v1.16b, v5.16b
432 eor v2.16b, v2.16b, v6.16b
433 st1 {v0.16b-v3.16b}, [x0], #64
434 mov v4.16b, v7.16b
435 cbz w4, .Lxtsencout
436 b .LxtsencloopNx
437#endif
438.Lxtsenc1x:
439 adds w4, w4, #INTERLEAVE
440 beq .Lxtsencout
441#endif
442.Lxtsencloop:
443 ld1 {v1.16b}, [x1], #16
444 eor v0.16b, v1.16b, v4.16b
445 encrypt_block v0, w3, x2, x6, w7
446 eor v0.16b, v0.16b, v4.16b
447 st1 {v0.16b}, [x0], #16
448 subs w4, w4, #1
449 beq .Lxtsencout
450 next_tweak v4, v4, v7, v8
451 b .Lxtsencloop
452.Lxtsencout:
453 FRAME_POP
454 ret
455AES_ENDPROC(aes_xts_encrypt)
456
457
458AES_ENTRY(aes_xts_decrypt)
459 FRAME_PUSH
460 cbz w7, .LxtsdecloopNx
461
462 ld1 {v4.16b}, [x6]
463 enc_prepare w3, x5, x6
464 encrypt_block v4, w3, x5, x6, w7 /* first tweak */
465 dec_prepare w3, x2, x6
466 ldr q7, .Lxts_mul_x
467 b .LxtsdecNx
468
469.LxtsdecloopNx:
470 ldr q7, .Lxts_mul_x
471 next_tweak v4, v4, v7, v8
472.LxtsdecNx:
473#if INTERLEAVE >= 2
474 subs w4, w4, #INTERLEAVE
475 bmi .Lxtsdec1x
476#if INTERLEAVE == 2
477 ld1 {v0.16b-v1.16b}, [x1], #32 /* get 2 ct blocks */
478 next_tweak v5, v4, v7, v8
479 eor v0.16b, v0.16b, v4.16b
480 eor v1.16b, v1.16b, v5.16b
481 do_decrypt_block2x
482 eor v0.16b, v0.16b, v4.16b
483 eor v1.16b, v1.16b, v5.16b
484 st1 {v0.16b-v1.16b}, [x0], #32
485 cbz w4, .LxtsdecoutNx
486 next_tweak v4, v5, v7, v8
487 b .LxtsdecNx
488.LxtsdecoutNx:
489 mov v4.16b, v5.16b
490 b .Lxtsdecout
491#else
492 ld1 {v0.16b-v3.16b}, [x1], #64 /* get 4 ct blocks */
493 next_tweak v5, v4, v7, v8
494 eor v0.16b, v0.16b, v4.16b
495 next_tweak v6, v5, v7, v8
496 eor v1.16b, v1.16b, v5.16b
497 eor v2.16b, v2.16b, v6.16b
498 next_tweak v7, v6, v7, v8
499 eor v3.16b, v3.16b, v7.16b
500 do_decrypt_block4x
501 eor v3.16b, v3.16b, v7.16b
502 eor v0.16b, v0.16b, v4.16b
503 eor v1.16b, v1.16b, v5.16b
504 eor v2.16b, v2.16b, v6.16b
505 st1 {v0.16b-v3.16b}, [x0], #64
506 mov v4.16b, v7.16b
507 cbz w4, .Lxtsdecout
508 b .LxtsdecloopNx
509#endif
510.Lxtsdec1x:
511 adds w4, w4, #INTERLEAVE
512 beq .Lxtsdecout
513#endif
514.Lxtsdecloop:
515 ld1 {v1.16b}, [x1], #16
516 eor v0.16b, v1.16b, v4.16b
517 decrypt_block v0, w3, x2, x6, w7
518 eor v0.16b, v0.16b, v4.16b
519 st1 {v0.16b}, [x0], #16
520 subs w4, w4, #1
521 beq .Lxtsdecout
522 next_tweak v4, v4, v7, v8
523 b .Lxtsdecloop
524.Lxtsdecout:
525 FRAME_POP
526 ret
527AES_ENDPROC(aes_xts_decrypt)
Ard Biesheuvel48606202017-02-03 14:49:37 +0000[diff] [blame]528
529 /*
530 * aes_mac_update(u8 const in[], u32 const rk[], int rounds,
531 * int blocks, u8 dg[], int enc_before, int enc_after)
532 */
533AES_ENTRY(aes_mac_update)
534 ld1 {v0.16b}, [x4] /* get dg */
535 enc_prepare w2, x1, x7
536 cbnz w5, .Lmacenc
537
538.Lmacloop:
539 cbz w3, .Lmacout
540 ld1 {v1.16b}, [x0], #16 /* get next pt block */
541 eor v0.16b, v0.16b, v1.16b /* ..and xor with dg */
542
543 subs w3, w3, #1
544 csinv x5, x6, xzr, eq
545 cbz w5, .Lmacout
546
547.Lmacenc:
548 encrypt_block v0, w2, x1, x7, w8
549 b .Lmacloop
550
551.Lmacout:
552 st1 {v0.16b}, [x4] /* return dg */
553 ret
554AES_ENDPROC(aes_mac_update)