Gitiles
Code Review Sign In
gerrit-public.fairphone.software / kernel / msm-5.4 / 12387a46bb150f5608de4aa9a90dfdddbf991e3f / . / arch / x86 / crypto / aesni-intel_asm.S
blob: 822846a9eba9280c5cbc142591a6291fe6e6608c [file] [log] [blame]
Huang Ying54b6a1b2009-01-18 16:28:34 +1100[diff] [blame]1/*
2 * Implement AES algorithm in Intel AES-NI instructions.
3 *
4 * The white paper of AES-NI instructions can be downloaded from:
5 * http://softwarecommunity.intel.com/isn/downloads/intelavx/AES-Instructions-Set_WP.pdf
6 *
7 * Copyright (C) 2008, Intel Corp.
8 * Author: Huang Ying <ying.huang@intel.com>
9 * Vinodh Gopal <vinodh.gopal@intel.com>
10 * Kahraman Akdemir
11 *
12 * This program is free software; you can redistribute it and/or modify
13 * it under the terms of the GNU General Public License as published by
14 * the Free Software Foundation; either version 2 of the License, or
15 * (at your option) any later version.
16 */
17
18#include <linux/linkage.h>
Huang Yingb369e522009-11-23 19:54:06 +0800[diff] [blame]19#include <asm/inst.h>
Huang Ying54b6a1b2009-01-18 16:28:34 +1100[diff] [blame]20
21.text
22
23#define STATE1 %xmm0
24#define STATE2 %xmm4
25#define STATE3 %xmm5
26#define STATE4 %xmm6
27#define STATE STATE1
28#define IN1 %xmm1
29#define IN2 %xmm7
30#define IN3 %xmm8
31#define IN4 %xmm9
32#define IN IN1
33#define KEY %xmm2
34#define IV %xmm3
Huang Ying12387a42010-03-10 18:28:55 +0800[diff] [blame^]35#define BSWAP_MASK %xmm10
36#define CTR %xmm11
37#define INC %xmm12
Huang Ying54b6a1b2009-01-18 16:28:34 +1100[diff] [blame]38
39#define KEYP %rdi
40#define OUTP %rsi
41#define INP %rdx
42#define LEN %rcx
43#define IVP %r8
44#define KLEN %r9d
45#define T1 %r10
46#define TKEYP T1
47#define T2 %r11
Huang Ying12387a42010-03-10 18:28:55 +0800[diff] [blame^]48#define TCTR_LOW T2
Huang Ying54b6a1b2009-01-18 16:28:34 +1100[diff] [blame]49
50_key_expansion_128:
51_key_expansion_256a:
52 pshufd $0b11111111, %xmm1, %xmm1
53 shufps $0b00010000, %xmm0, %xmm4
54 pxor %xmm4, %xmm0
55 shufps $0b10001100, %xmm0, %xmm4
56 pxor %xmm4, %xmm0
57 pxor %xmm1, %xmm0
58 movaps %xmm0, (%rcx)
59 add $0x10, %rcx
60 ret
61
62_key_expansion_192a:
63 pshufd $0b01010101, %xmm1, %xmm1
64 shufps $0b00010000, %xmm0, %xmm4
65 pxor %xmm4, %xmm0
66 shufps $0b10001100, %xmm0, %xmm4
67 pxor %xmm4, %xmm0
68 pxor %xmm1, %xmm0
69
70 movaps %xmm2, %xmm5
71 movaps %xmm2, %xmm6
72 pslldq $4, %xmm5
73 pshufd $0b11111111, %xmm0, %xmm3
74 pxor %xmm3, %xmm2
75 pxor %xmm5, %xmm2
76
77 movaps %xmm0, %xmm1
78 shufps $0b01000100, %xmm0, %xmm6
79 movaps %xmm6, (%rcx)
80 shufps $0b01001110, %xmm2, %xmm1
81 movaps %xmm1, 16(%rcx)
82 add $0x20, %rcx
83 ret
84
85_key_expansion_192b:
86 pshufd $0b01010101, %xmm1, %xmm1
87 shufps $0b00010000, %xmm0, %xmm4
88 pxor %xmm4, %xmm0
89 shufps $0b10001100, %xmm0, %xmm4
90 pxor %xmm4, %xmm0
91 pxor %xmm1, %xmm0
92
93 movaps %xmm2, %xmm5
94 pslldq $4, %xmm5
95 pshufd $0b11111111, %xmm0, %xmm3
96 pxor %xmm3, %xmm2
97 pxor %xmm5, %xmm2
98
99 movaps %xmm0, (%rcx)
100 add $0x10, %rcx
101 ret
102
103_key_expansion_256b:
104 pshufd $0b10101010, %xmm1, %xmm1
105 shufps $0b00010000, %xmm2, %xmm4
106 pxor %xmm4, %xmm2
107 shufps $0b10001100, %xmm2, %xmm4
108 pxor %xmm4, %xmm2
109 pxor %xmm1, %xmm2
110 movaps %xmm2, (%rcx)
111 add $0x10, %rcx
112 ret
113
114/*
115 * int aesni_set_key(struct crypto_aes_ctx *ctx, const u8 *in_key,
116 * unsigned int key_len)
117 */
118ENTRY(aesni_set_key)
119 movups (%rsi), %xmm0 # user key (first 16 bytes)
120 movaps %xmm0, (%rdi)
121 lea 0x10(%rdi), %rcx # key addr
122 movl %edx, 480(%rdi)
123 pxor %xmm4, %xmm4 # xmm4 is assumed 0 in _key_expansion_x
124 cmp $24, %dl
125 jb .Lenc_key128
126 je .Lenc_key192
127 movups 0x10(%rsi), %xmm2 # other user key
128 movaps %xmm2, (%rcx)
129 add $0x10, %rcx
Huang Yingb369e522009-11-23 19:54:06 +0800[diff] [blame]130 AESKEYGENASSIST 0x1 %xmm2 %xmm1 # round 1
Huang Ying54b6a1b2009-01-18 16:28:34 +1100[diff] [blame]131 call _key_expansion_256a
Huang Yingb369e522009-11-23 19:54:06 +0800[diff] [blame]132 AESKEYGENASSIST 0x1 %xmm0 %xmm1
Huang Ying54b6a1b2009-01-18 16:28:34 +1100[diff] [blame]133 call _key_expansion_256b
Huang Yingb369e522009-11-23 19:54:06 +0800[diff] [blame]134 AESKEYGENASSIST 0x2 %xmm2 %xmm1 # round 2
Huang Ying54b6a1b2009-01-18 16:28:34 +1100[diff] [blame]135 call _key_expansion_256a
Huang Yingb369e522009-11-23 19:54:06 +0800[diff] [blame]136 AESKEYGENASSIST 0x2 %xmm0 %xmm1
Huang Ying54b6a1b2009-01-18 16:28:34 +1100[diff] [blame]137 call _key_expansion_256b
Huang Yingb369e522009-11-23 19:54:06 +0800[diff] [blame]138 AESKEYGENASSIST 0x4 %xmm2 %xmm1 # round 3
Huang Ying54b6a1b2009-01-18 16:28:34 +1100[diff] [blame]139 call _key_expansion_256a
Huang Yingb369e522009-11-23 19:54:06 +0800[diff] [blame]140 AESKEYGENASSIST 0x4 %xmm0 %xmm1
Huang Ying54b6a1b2009-01-18 16:28:34 +1100[diff] [blame]141 call _key_expansion_256b
Huang Yingb369e522009-11-23 19:54:06 +0800[diff] [blame]142 AESKEYGENASSIST 0x8 %xmm2 %xmm1 # round 4
Huang Ying54b6a1b2009-01-18 16:28:34 +1100[diff] [blame]143 call _key_expansion_256a
Huang Yingb369e522009-11-23 19:54:06 +0800[diff] [blame]144 AESKEYGENASSIST 0x8 %xmm0 %xmm1
Huang Ying54b6a1b2009-01-18 16:28:34 +1100[diff] [blame]145 call _key_expansion_256b
Huang Yingb369e522009-11-23 19:54:06 +0800[diff] [blame]146 AESKEYGENASSIST 0x10 %xmm2 %xmm1 # round 5
Huang Ying54b6a1b2009-01-18 16:28:34 +1100[diff] [blame]147 call _key_expansion_256a
Huang Yingb369e522009-11-23 19:54:06 +0800[diff] [blame]148 AESKEYGENASSIST 0x10 %xmm0 %xmm1
Huang Ying54b6a1b2009-01-18 16:28:34 +1100[diff] [blame]149 call _key_expansion_256b
Huang Yingb369e522009-11-23 19:54:06 +0800[diff] [blame]150 AESKEYGENASSIST 0x20 %xmm2 %xmm1 # round 6
Huang Ying54b6a1b2009-01-18 16:28:34 +1100[diff] [blame]151 call _key_expansion_256a
Huang Yingb369e522009-11-23 19:54:06 +0800[diff] [blame]152 AESKEYGENASSIST 0x20 %xmm0 %xmm1
Huang Ying54b6a1b2009-01-18 16:28:34 +1100[diff] [blame]153 call _key_expansion_256b
Huang Yingb369e522009-11-23 19:54:06 +0800[diff] [blame]154 AESKEYGENASSIST 0x40 %xmm2 %xmm1 # round 7
Huang Ying54b6a1b2009-01-18 16:28:34 +1100[diff] [blame]155 call _key_expansion_256a
156 jmp .Ldec_key
157.Lenc_key192:
158 movq 0x10(%rsi), %xmm2 # other user key
Huang Yingb369e522009-11-23 19:54:06 +0800[diff] [blame]159 AESKEYGENASSIST 0x1 %xmm2 %xmm1 # round 1
Huang Ying54b6a1b2009-01-18 16:28:34 +1100[diff] [blame]160 call _key_expansion_192a
Huang Yingb369e522009-11-23 19:54:06 +0800[diff] [blame]161 AESKEYGENASSIST 0x2 %xmm2 %xmm1 # round 2
Huang Ying54b6a1b2009-01-18 16:28:34 +1100[diff] [blame]162 call _key_expansion_192b
Huang Yingb369e522009-11-23 19:54:06 +0800[diff] [blame]163 AESKEYGENASSIST 0x4 %xmm2 %xmm1 # round 3
Huang Ying54b6a1b2009-01-18 16:28:34 +1100[diff] [blame]164 call _key_expansion_192a
Huang Yingb369e522009-11-23 19:54:06 +0800[diff] [blame]165 AESKEYGENASSIST 0x8 %xmm2 %xmm1 # round 4
Huang Ying54b6a1b2009-01-18 16:28:34 +1100[diff] [blame]166 call _key_expansion_192b
Huang Yingb369e522009-11-23 19:54:06 +0800[diff] [blame]167 AESKEYGENASSIST 0x10 %xmm2 %xmm1 # round 5
Huang Ying54b6a1b2009-01-18 16:28:34 +1100[diff] [blame]168 call _key_expansion_192a
Huang Yingb369e522009-11-23 19:54:06 +0800[diff] [blame]169 AESKEYGENASSIST 0x20 %xmm2 %xmm1 # round 6
Huang Ying54b6a1b2009-01-18 16:28:34 +1100[diff] [blame]170 call _key_expansion_192b
Huang Yingb369e522009-11-23 19:54:06 +0800[diff] [blame]171 AESKEYGENASSIST 0x40 %xmm2 %xmm1 # round 7
Huang Ying54b6a1b2009-01-18 16:28:34 +1100[diff] [blame]172 call _key_expansion_192a
Huang Yingb369e522009-11-23 19:54:06 +0800[diff] [blame]173 AESKEYGENASSIST 0x80 %xmm2 %xmm1 # round 8
Huang Ying54b6a1b2009-01-18 16:28:34 +1100[diff] [blame]174 call _key_expansion_192b
175 jmp .Ldec_key
176.Lenc_key128:
Huang Yingb369e522009-11-23 19:54:06 +0800[diff] [blame]177 AESKEYGENASSIST 0x1 %xmm0 %xmm1 # round 1
Huang Ying54b6a1b2009-01-18 16:28:34 +1100[diff] [blame]178 call _key_expansion_128
Huang Yingb369e522009-11-23 19:54:06 +0800[diff] [blame]179 AESKEYGENASSIST 0x2 %xmm0 %xmm1 # round 2
Huang Ying54b6a1b2009-01-18 16:28:34 +1100[diff] [blame]180 call _key_expansion_128
Huang Yingb369e522009-11-23 19:54:06 +0800[diff] [blame]181 AESKEYGENASSIST 0x4 %xmm0 %xmm1 # round 3
Huang Ying54b6a1b2009-01-18 16:28:34 +1100[diff] [blame]182 call _key_expansion_128
Huang Yingb369e522009-11-23 19:54:06 +0800[diff] [blame]183 AESKEYGENASSIST 0x8 %xmm0 %xmm1 # round 4
Huang Ying54b6a1b2009-01-18 16:28:34 +1100[diff] [blame]184 call _key_expansion_128
Huang Yingb369e522009-11-23 19:54:06 +0800[diff] [blame]185 AESKEYGENASSIST 0x10 %xmm0 %xmm1 # round 5
Huang Ying54b6a1b2009-01-18 16:28:34 +1100[diff] [blame]186 call _key_expansion_128
Huang Yingb369e522009-11-23 19:54:06 +0800[diff] [blame]187 AESKEYGENASSIST 0x20 %xmm0 %xmm1 # round 6
Huang Ying54b6a1b2009-01-18 16:28:34 +1100[diff] [blame]188 call _key_expansion_128
Huang Yingb369e522009-11-23 19:54:06 +0800[diff] [blame]189 AESKEYGENASSIST 0x40 %xmm0 %xmm1 # round 7
Huang Ying54b6a1b2009-01-18 16:28:34 +1100[diff] [blame]190 call _key_expansion_128
Huang Yingb369e522009-11-23 19:54:06 +0800[diff] [blame]191 AESKEYGENASSIST 0x80 %xmm0 %xmm1 # round 8
Huang Ying54b6a1b2009-01-18 16:28:34 +1100[diff] [blame]192 call _key_expansion_128
Huang Yingb369e522009-11-23 19:54:06 +0800[diff] [blame]193 AESKEYGENASSIST 0x1b %xmm0 %xmm1 # round 9
Huang Ying54b6a1b2009-01-18 16:28:34 +1100[diff] [blame]194 call _key_expansion_128
Huang Yingb369e522009-11-23 19:54:06 +0800[diff] [blame]195 AESKEYGENASSIST 0x36 %xmm0 %xmm1 # round 10
Huang Ying54b6a1b2009-01-18 16:28:34 +1100[diff] [blame]196 call _key_expansion_128
197.Ldec_key:
198 sub $0x10, %rcx
199 movaps (%rdi), %xmm0
200 movaps (%rcx), %xmm1
201 movaps %xmm0, 240(%rcx)
202 movaps %xmm1, 240(%rdi)
203 add $0x10, %rdi
204 lea 240-16(%rcx), %rsi
205.align 4
206.Ldec_key_loop:
207 movaps (%rdi), %xmm0
Huang Yingb369e522009-11-23 19:54:06 +0800[diff] [blame]208 AESIMC %xmm0 %xmm1
Huang Ying54b6a1b2009-01-18 16:28:34 +1100[diff] [blame]209 movaps %xmm1, (%rsi)
210 add $0x10, %rdi
211 sub $0x10, %rsi
212 cmp %rcx, %rdi
213 jb .Ldec_key_loop
214 xor %rax, %rax
215 ret
216
217/*
218 * void aesni_enc(struct crypto_aes_ctx *ctx, u8 *dst, const u8 *src)
219 */
220ENTRY(aesni_enc)
221 movl 480(KEYP), KLEN # key length
222 movups (INP), STATE # input
223 call _aesni_enc1
224 movups STATE, (OUTP) # output
225 ret
226
227/*
228 * _aesni_enc1: internal ABI
229 * input:
230 * KEYP: key struct pointer
231 * KLEN: round count
232 * STATE: initial state (input)
233 * output:
234 * STATE: finial state (output)
235 * changed:
236 * KEY
237 * TKEYP (T1)
238 */
239_aesni_enc1:
240 movaps (KEYP), KEY # key
241 mov KEYP, TKEYP
242 pxor KEY, STATE # round 0
243 add $0x30, TKEYP
244 cmp $24, KLEN
245 jb .Lenc128
246 lea 0x20(TKEYP), TKEYP
247 je .Lenc192
248 add $0x20, TKEYP
249 movaps -0x60(TKEYP), KEY
Huang Yingb369e522009-11-23 19:54:06 +0800[diff] [blame]250 AESENC KEY STATE
Huang Ying54b6a1b2009-01-18 16:28:34 +1100[diff] [blame]251 movaps -0x50(TKEYP), KEY
Huang Yingb369e522009-11-23 19:54:06 +0800[diff] [blame]252 AESENC KEY STATE
Huang Ying54b6a1b2009-01-18 16:28:34 +1100[diff] [blame]253.align 4
254.Lenc192:
255 movaps -0x40(TKEYP), KEY
Huang Yingb369e522009-11-23 19:54:06 +0800[diff] [blame]256 AESENC KEY STATE
Huang Ying54b6a1b2009-01-18 16:28:34 +1100[diff] [blame]257 movaps -0x30(TKEYP), KEY
Huang Yingb369e522009-11-23 19:54:06 +0800[diff] [blame]258 AESENC KEY STATE
Huang Ying54b6a1b2009-01-18 16:28:34 +1100[diff] [blame]259.align 4
260.Lenc128:
261 movaps -0x20(TKEYP), KEY
Huang Yingb369e522009-11-23 19:54:06 +0800[diff] [blame]262 AESENC KEY STATE
Huang Ying54b6a1b2009-01-18 16:28:34 +1100[diff] [blame]263 movaps -0x10(TKEYP), KEY
Huang Yingb369e522009-11-23 19:54:06 +0800[diff] [blame]264 AESENC KEY STATE
Huang Ying54b6a1b2009-01-18 16:28:34 +1100[diff] [blame]265 movaps (TKEYP), KEY
Huang Yingb369e522009-11-23 19:54:06 +0800[diff] [blame]266 AESENC KEY STATE
Huang Ying54b6a1b2009-01-18 16:28:34 +1100[diff] [blame]267 movaps 0x10(TKEYP), KEY
Huang Yingb369e522009-11-23 19:54:06 +0800[diff] [blame]268 AESENC KEY STATE
Huang Ying54b6a1b2009-01-18 16:28:34 +1100[diff] [blame]269 movaps 0x20(TKEYP), KEY
Huang Yingb369e522009-11-23 19:54:06 +0800[diff] [blame]270 AESENC KEY STATE
Huang Ying54b6a1b2009-01-18 16:28:34 +1100[diff] [blame]271 movaps 0x30(TKEYP), KEY
Huang Yingb369e522009-11-23 19:54:06 +0800[diff] [blame]272 AESENC KEY STATE
Huang Ying54b6a1b2009-01-18 16:28:34 +1100[diff] [blame]273 movaps 0x40(TKEYP), KEY
Huang Yingb369e522009-11-23 19:54:06 +0800[diff] [blame]274 AESENC KEY STATE
Huang Ying54b6a1b2009-01-18 16:28:34 +1100[diff] [blame]275 movaps 0x50(TKEYP), KEY
Huang Yingb369e522009-11-23 19:54:06 +0800[diff] [blame]276 AESENC KEY STATE
Huang Ying54b6a1b2009-01-18 16:28:34 +1100[diff] [blame]277 movaps 0x60(TKEYP), KEY
Huang Yingb369e522009-11-23 19:54:06 +0800[diff] [blame]278 AESENC KEY STATE
Huang Ying54b6a1b2009-01-18 16:28:34 +1100[diff] [blame]279 movaps 0x70(TKEYP), KEY
Huang Yingb369e522009-11-23 19:54:06 +0800[diff] [blame]280 AESENCLAST KEY STATE
Huang Ying54b6a1b2009-01-18 16:28:34 +1100[diff] [blame]281 ret
282
283/*
284 * _aesni_enc4: internal ABI
285 * input:
286 * KEYP: key struct pointer
287 * KLEN: round count
288 * STATE1: initial state (input)
289 * STATE2
290 * STATE3
291 * STATE4
292 * output:
293 * STATE1: finial state (output)
294 * STATE2
295 * STATE3
296 * STATE4
297 * changed:
298 * KEY
299 * TKEYP (T1)
300 */
301_aesni_enc4:
302 movaps (KEYP), KEY # key
303 mov KEYP, TKEYP
304 pxor KEY, STATE1 # round 0
305 pxor KEY, STATE2
306 pxor KEY, STATE3
307 pxor KEY, STATE4
308 add $0x30, TKEYP
309 cmp $24, KLEN
310 jb .L4enc128
311 lea 0x20(TKEYP), TKEYP
312 je .L4enc192
313 add $0x20, TKEYP
314 movaps -0x60(TKEYP), KEY
Huang Yingb369e522009-11-23 19:54:06 +0800[diff] [blame]315 AESENC KEY STATE1
316 AESENC KEY STATE2
317 AESENC KEY STATE3
318 AESENC KEY STATE4
Huang Ying54b6a1b2009-01-18 16:28:34 +1100[diff] [blame]319 movaps -0x50(TKEYP), KEY
Huang Yingb369e522009-11-23 19:54:06 +0800[diff] [blame]320 AESENC KEY STATE1
321 AESENC KEY STATE2
322 AESENC KEY STATE3
323 AESENC KEY STATE4
Huang Ying54b6a1b2009-01-18 16:28:34 +1100[diff] [blame]324#.align 4
325.L4enc192:
326 movaps -0x40(TKEYP), KEY
Huang Yingb369e522009-11-23 19:54:06 +0800[diff] [blame]327 AESENC KEY STATE1
328 AESENC KEY STATE2
329 AESENC KEY STATE3
330 AESENC KEY STATE4
Huang Ying54b6a1b2009-01-18 16:28:34 +1100[diff] [blame]331 movaps -0x30(TKEYP), KEY
Huang Yingb369e522009-11-23 19:54:06 +0800[diff] [blame]332 AESENC KEY STATE1
333 AESENC KEY STATE2
334 AESENC KEY STATE3
335 AESENC KEY STATE4
Huang Ying54b6a1b2009-01-18 16:28:34 +1100[diff] [blame]336#.align 4
337.L4enc128:
338 movaps -0x20(TKEYP), KEY
Huang Yingb369e522009-11-23 19:54:06 +0800[diff] [blame]339 AESENC KEY STATE1
340 AESENC KEY STATE2
341 AESENC KEY STATE3
342 AESENC KEY STATE4
Huang Ying54b6a1b2009-01-18 16:28:34 +1100[diff] [blame]343 movaps -0x10(TKEYP), KEY
Huang Yingb369e522009-11-23 19:54:06 +0800[diff] [blame]344 AESENC KEY STATE1
345 AESENC KEY STATE2
346 AESENC KEY STATE3
347 AESENC KEY STATE4
Huang Ying54b6a1b2009-01-18 16:28:34 +1100[diff] [blame]348 movaps (TKEYP), KEY
Huang Yingb369e522009-11-23 19:54:06 +0800[diff] [blame]349 AESENC KEY STATE1
350 AESENC KEY STATE2
351 AESENC KEY STATE3
352 AESENC KEY STATE4
Huang Ying54b6a1b2009-01-18 16:28:34 +1100[diff] [blame]353 movaps 0x10(TKEYP), KEY
Huang Yingb369e522009-11-23 19:54:06 +0800[diff] [blame]354 AESENC KEY STATE1
355 AESENC KEY STATE2
356 AESENC KEY STATE3
357 AESENC KEY STATE4
Huang Ying54b6a1b2009-01-18 16:28:34 +1100[diff] [blame]358 movaps 0x20(TKEYP), KEY
Huang Yingb369e522009-11-23 19:54:06 +0800[diff] [blame]359 AESENC KEY STATE1
360 AESENC KEY STATE2
361 AESENC KEY STATE3
362 AESENC KEY STATE4
Huang Ying54b6a1b2009-01-18 16:28:34 +1100[diff] [blame]363 movaps 0x30(TKEYP), KEY
Huang Yingb369e522009-11-23 19:54:06 +0800[diff] [blame]364 AESENC KEY STATE1
365 AESENC KEY STATE2
366 AESENC KEY STATE3
367 AESENC KEY STATE4
Huang Ying54b6a1b2009-01-18 16:28:34 +1100[diff] [blame]368 movaps 0x40(TKEYP), KEY
Huang Yingb369e522009-11-23 19:54:06 +0800[diff] [blame]369 AESENC KEY STATE1
370 AESENC KEY STATE2
371 AESENC KEY STATE3
372 AESENC KEY STATE4
Huang Ying54b6a1b2009-01-18 16:28:34 +1100[diff] [blame]373 movaps 0x50(TKEYP), KEY
Huang Yingb369e522009-11-23 19:54:06 +0800[diff] [blame]374 AESENC KEY STATE1
375 AESENC KEY STATE2
376 AESENC KEY STATE3
377 AESENC KEY STATE4
Huang Ying54b6a1b2009-01-18 16:28:34 +1100[diff] [blame]378 movaps 0x60(TKEYP), KEY
Huang Yingb369e522009-11-23 19:54:06 +0800[diff] [blame]379 AESENC KEY STATE1
380 AESENC KEY STATE2
381 AESENC KEY STATE3
382 AESENC KEY STATE4
Huang Ying54b6a1b2009-01-18 16:28:34 +1100[diff] [blame]383 movaps 0x70(TKEYP), KEY
Huang Yingb369e522009-11-23 19:54:06 +0800[diff] [blame]384 AESENCLAST KEY STATE1 # last round
385 AESENCLAST KEY STATE2
386 AESENCLAST KEY STATE3
387 AESENCLAST KEY STATE4
Huang Ying54b6a1b2009-01-18 16:28:34 +1100[diff] [blame]388 ret
389
390/*
391 * void aesni_dec (struct crypto_aes_ctx *ctx, u8 *dst, const u8 *src)
392 */
393ENTRY(aesni_dec)
394 mov 480(KEYP), KLEN # key length
395 add $240, KEYP
396 movups (INP), STATE # input
397 call _aesni_dec1
398 movups STATE, (OUTP) #output
399 ret
400
401/*
402 * _aesni_dec1: internal ABI
403 * input:
404 * KEYP: key struct pointer
405 * KLEN: key length
406 * STATE: initial state (input)
407 * output:
408 * STATE: finial state (output)
409 * changed:
410 * KEY
411 * TKEYP (T1)
412 */
413_aesni_dec1:
414 movaps (KEYP), KEY # key
415 mov KEYP, TKEYP
416 pxor KEY, STATE # round 0
417 add $0x30, TKEYP
418 cmp $24, KLEN
419 jb .Ldec128
420 lea 0x20(TKEYP), TKEYP
421 je .Ldec192
422 add $0x20, TKEYP
423 movaps -0x60(TKEYP), KEY
Huang Yingb369e522009-11-23 19:54:06 +0800[diff] [blame]424 AESDEC KEY STATE
Huang Ying54b6a1b2009-01-18 16:28:34 +1100[diff] [blame]425 movaps -0x50(TKEYP), KEY
Huang Yingb369e522009-11-23 19:54:06 +0800[diff] [blame]426 AESDEC KEY STATE
Huang Ying54b6a1b2009-01-18 16:28:34 +1100[diff] [blame]427.align 4
428.Ldec192:
429 movaps -0x40(TKEYP), KEY
Huang Yingb369e522009-11-23 19:54:06 +0800[diff] [blame]430 AESDEC KEY STATE
Huang Ying54b6a1b2009-01-18 16:28:34 +1100[diff] [blame]431 movaps -0x30(TKEYP), KEY
Huang Yingb369e522009-11-23 19:54:06 +0800[diff] [blame]432 AESDEC KEY STATE
Huang Ying54b6a1b2009-01-18 16:28:34 +1100[diff] [blame]433.align 4
434.Ldec128:
435 movaps -0x20(TKEYP), KEY
Huang Yingb369e522009-11-23 19:54:06 +0800[diff] [blame]436 AESDEC KEY STATE
Huang Ying54b6a1b2009-01-18 16:28:34 +1100[diff] [blame]437 movaps -0x10(TKEYP), KEY
Huang Yingb369e522009-11-23 19:54:06 +0800[diff] [blame]438 AESDEC KEY STATE
Huang Ying54b6a1b2009-01-18 16:28:34 +1100[diff] [blame]439 movaps (TKEYP), KEY
Huang Yingb369e522009-11-23 19:54:06 +0800[diff] [blame]440 AESDEC KEY STATE
Huang Ying54b6a1b2009-01-18 16:28:34 +1100[diff] [blame]441 movaps 0x10(TKEYP), KEY
Huang Yingb369e522009-11-23 19:54:06 +0800[diff] [blame]442 AESDEC KEY STATE
Huang Ying54b6a1b2009-01-18 16:28:34 +1100[diff] [blame]443 movaps 0x20(TKEYP), KEY
Huang Yingb369e522009-11-23 19:54:06 +0800[diff] [blame]444 AESDEC KEY STATE
Huang Ying54b6a1b2009-01-18 16:28:34 +1100[diff] [blame]445 movaps 0x30(TKEYP), KEY
Huang Yingb369e522009-11-23 19:54:06 +0800[diff] [blame]446 AESDEC KEY STATE
Huang Ying54b6a1b2009-01-18 16:28:34 +1100[diff] [blame]447 movaps 0x40(TKEYP), KEY
Huang Yingb369e522009-11-23 19:54:06 +0800[diff] [blame]448 AESDEC KEY STATE
Huang Ying54b6a1b2009-01-18 16:28:34 +1100[diff] [blame]449 movaps 0x50(TKEYP), KEY
Huang Yingb369e522009-11-23 19:54:06 +0800[diff] [blame]450 AESDEC KEY STATE
Huang Ying54b6a1b2009-01-18 16:28:34 +1100[diff] [blame]451 movaps 0x60(TKEYP), KEY
Huang Yingb369e522009-11-23 19:54:06 +0800[diff] [blame]452 AESDEC KEY STATE
Huang Ying54b6a1b2009-01-18 16:28:34 +1100[diff] [blame]453 movaps 0x70(TKEYP), KEY
Huang Yingb369e522009-11-23 19:54:06 +0800[diff] [blame]454 AESDECLAST KEY STATE
Huang Ying54b6a1b2009-01-18 16:28:34 +1100[diff] [blame]455 ret
456
457/*
458 * _aesni_dec4: internal ABI
459 * input:
460 * KEYP: key struct pointer
461 * KLEN: key length
462 * STATE1: initial state (input)
463 * STATE2
464 * STATE3
465 * STATE4
466 * output:
467 * STATE1: finial state (output)
468 * STATE2
469 * STATE3
470 * STATE4
471 * changed:
472 * KEY
473 * TKEYP (T1)
474 */
475_aesni_dec4:
476 movaps (KEYP), KEY # key
477 mov KEYP, TKEYP
478 pxor KEY, STATE1 # round 0
479 pxor KEY, STATE2
480 pxor KEY, STATE3
481 pxor KEY, STATE4
482 add $0x30, TKEYP
483 cmp $24, KLEN
484 jb .L4dec128
485 lea 0x20(TKEYP), TKEYP
486 je .L4dec192
487 add $0x20, TKEYP
488 movaps -0x60(TKEYP), KEY
Huang Yingb369e522009-11-23 19:54:06 +0800[diff] [blame]489 AESDEC KEY STATE1
490 AESDEC KEY STATE2
491 AESDEC KEY STATE3
492 AESDEC KEY STATE4
Huang Ying54b6a1b2009-01-18 16:28:34 +1100[diff] [blame]493 movaps -0x50(TKEYP), KEY
Huang Yingb369e522009-11-23 19:54:06 +0800[diff] [blame]494 AESDEC KEY STATE1
495 AESDEC KEY STATE2
496 AESDEC KEY STATE3
497 AESDEC KEY STATE4
Huang Ying54b6a1b2009-01-18 16:28:34 +1100[diff] [blame]498.align 4
499.L4dec192:
500 movaps -0x40(TKEYP), KEY
Huang Yingb369e522009-11-23 19:54:06 +0800[diff] [blame]501 AESDEC KEY STATE1
502 AESDEC KEY STATE2
503 AESDEC KEY STATE3
504 AESDEC KEY STATE4
Huang Ying54b6a1b2009-01-18 16:28:34 +1100[diff] [blame]505 movaps -0x30(TKEYP), KEY
Huang Yingb369e522009-11-23 19:54:06 +0800[diff] [blame]506 AESDEC KEY STATE1
507 AESDEC KEY STATE2
508 AESDEC KEY STATE3
509 AESDEC KEY STATE4
Huang Ying54b6a1b2009-01-18 16:28:34 +1100[diff] [blame]510.align 4
511.L4dec128:
512 movaps -0x20(TKEYP), KEY
Huang Yingb369e522009-11-23 19:54:06 +0800[diff] [blame]513 AESDEC KEY STATE1
514 AESDEC KEY STATE2
515 AESDEC KEY STATE3
516 AESDEC KEY STATE4
Huang Ying54b6a1b2009-01-18 16:28:34 +1100[diff] [blame]517 movaps -0x10(TKEYP), KEY
Huang Yingb369e522009-11-23 19:54:06 +0800[diff] [blame]518 AESDEC KEY STATE1
519 AESDEC KEY STATE2
520 AESDEC KEY STATE3
521 AESDEC KEY STATE4
Huang Ying54b6a1b2009-01-18 16:28:34 +1100[diff] [blame]522 movaps (TKEYP), KEY
Huang Yingb369e522009-11-23 19:54:06 +0800[diff] [blame]523 AESDEC KEY STATE1
524 AESDEC KEY STATE2
525 AESDEC KEY STATE3
526 AESDEC KEY STATE4
Huang Ying54b6a1b2009-01-18 16:28:34 +1100[diff] [blame]527 movaps 0x10(TKEYP), KEY
Huang Yingb369e522009-11-23 19:54:06 +0800[diff] [blame]528 AESDEC KEY STATE1
529 AESDEC KEY STATE2
530 AESDEC KEY STATE3
531 AESDEC KEY STATE4
Huang Ying54b6a1b2009-01-18 16:28:34 +1100[diff] [blame]532 movaps 0x20(TKEYP), KEY
Huang Yingb369e522009-11-23 19:54:06 +0800[diff] [blame]533 AESDEC KEY STATE1
534 AESDEC KEY STATE2
535 AESDEC KEY STATE3
536 AESDEC KEY STATE4
Huang Ying54b6a1b2009-01-18 16:28:34 +1100[diff] [blame]537 movaps 0x30(TKEYP), KEY
Huang Yingb369e522009-11-23 19:54:06 +0800[diff] [blame]538 AESDEC KEY STATE1
539 AESDEC KEY STATE2
540 AESDEC KEY STATE3
541 AESDEC KEY STATE4
Huang Ying54b6a1b2009-01-18 16:28:34 +1100[diff] [blame]542 movaps 0x40(TKEYP), KEY
Huang Yingb369e522009-11-23 19:54:06 +0800[diff] [blame]543 AESDEC KEY STATE1
544 AESDEC KEY STATE2
545 AESDEC KEY STATE3
546 AESDEC KEY STATE4
Huang Ying54b6a1b2009-01-18 16:28:34 +1100[diff] [blame]547 movaps 0x50(TKEYP), KEY
Huang Yingb369e522009-11-23 19:54:06 +0800[diff] [blame]548 AESDEC KEY STATE1
549 AESDEC KEY STATE2
550 AESDEC KEY STATE3
551 AESDEC KEY STATE4
Huang Ying54b6a1b2009-01-18 16:28:34 +1100[diff] [blame]552 movaps 0x60(TKEYP), KEY
Huang Yingb369e522009-11-23 19:54:06 +0800[diff] [blame]553 AESDEC KEY STATE1
554 AESDEC KEY STATE2
555 AESDEC KEY STATE3
556 AESDEC KEY STATE4
Huang Ying54b6a1b2009-01-18 16:28:34 +1100[diff] [blame]557 movaps 0x70(TKEYP), KEY
Huang Yingb369e522009-11-23 19:54:06 +0800[diff] [blame]558 AESDECLAST KEY STATE1 # last round
559 AESDECLAST KEY STATE2
560 AESDECLAST KEY STATE3
561 AESDECLAST KEY STATE4
Huang Ying54b6a1b2009-01-18 16:28:34 +1100[diff] [blame]562 ret
563
564/*
565 * void aesni_ecb_enc(struct crypto_aes_ctx *ctx, const u8 *dst, u8 *src,
566 * size_t len)
567 */
568ENTRY(aesni_ecb_enc)
569 test LEN, LEN # check length
570 jz .Lecb_enc_ret
571 mov 480(KEYP), KLEN
572 cmp $16, LEN
573 jb .Lecb_enc_ret
574 cmp $64, LEN
575 jb .Lecb_enc_loop1
576.align 4
577.Lecb_enc_loop4:
578 movups (INP), STATE1
579 movups 0x10(INP), STATE2
580 movups 0x20(INP), STATE3
581 movups 0x30(INP), STATE4
582 call _aesni_enc4
583 movups STATE1, (OUTP)
584 movups STATE2, 0x10(OUTP)
585 movups STATE3, 0x20(OUTP)
586 movups STATE4, 0x30(OUTP)
587 sub $64, LEN
588 add $64, INP
589 add $64, OUTP
590 cmp $64, LEN
591 jge .Lecb_enc_loop4
592 cmp $16, LEN
593 jb .Lecb_enc_ret
594.align 4
595.Lecb_enc_loop1:
596 movups (INP), STATE1
597 call _aesni_enc1
598 movups STATE1, (OUTP)
599 sub $16, LEN
600 add $16, INP
601 add $16, OUTP
602 cmp $16, LEN
603 jge .Lecb_enc_loop1
604.Lecb_enc_ret:
605 ret
606
607/*
608 * void aesni_ecb_dec(struct crypto_aes_ctx *ctx, const u8 *dst, u8 *src,
609 * size_t len);
610 */
611ENTRY(aesni_ecb_dec)
612 test LEN, LEN
613 jz .Lecb_dec_ret
614 mov 480(KEYP), KLEN
615 add $240, KEYP
616 cmp $16, LEN
617 jb .Lecb_dec_ret
618 cmp $64, LEN
619 jb .Lecb_dec_loop1
620.align 4
621.Lecb_dec_loop4:
622 movups (INP), STATE1
623 movups 0x10(INP), STATE2
624 movups 0x20(INP), STATE3
625 movups 0x30(INP), STATE4
626 call _aesni_dec4
627 movups STATE1, (OUTP)
628 movups STATE2, 0x10(OUTP)
629 movups STATE3, 0x20(OUTP)
630 movups STATE4, 0x30(OUTP)
631 sub $64, LEN
632 add $64, INP
633 add $64, OUTP
634 cmp $64, LEN
635 jge .Lecb_dec_loop4
636 cmp $16, LEN
637 jb .Lecb_dec_ret
638.align 4
639.Lecb_dec_loop1:
640 movups (INP), STATE1
641 call _aesni_dec1
642 movups STATE1, (OUTP)
643 sub $16, LEN
644 add $16, INP
645 add $16, OUTP
646 cmp $16, LEN
647 jge .Lecb_dec_loop1
648.Lecb_dec_ret:
649 ret
650
651/*
652 * void aesni_cbc_enc(struct crypto_aes_ctx *ctx, const u8 *dst, u8 *src,
653 * size_t len, u8 *iv)
654 */
655ENTRY(aesni_cbc_enc)
656 cmp $16, LEN
657 jb .Lcbc_enc_ret
658 mov 480(KEYP), KLEN
659 movups (IVP), STATE # load iv as initial state
660.align 4
661.Lcbc_enc_loop:
662 movups (INP), IN # load input
663 pxor IN, STATE
664 call _aesni_enc1
665 movups STATE, (OUTP) # store output
666 sub $16, LEN
667 add $16, INP
668 add $16, OUTP
669 cmp $16, LEN
670 jge .Lcbc_enc_loop
671 movups STATE, (IVP)
672.Lcbc_enc_ret:
673 ret
674
675/*
676 * void aesni_cbc_dec(struct crypto_aes_ctx *ctx, const u8 *dst, u8 *src,
677 * size_t len, u8 *iv)
678 */
679ENTRY(aesni_cbc_dec)
680 cmp $16, LEN
Huang Yinge6efaa02009-06-18 19:33:57 +0800[diff] [blame]681 jb .Lcbc_dec_just_ret
Huang Ying54b6a1b2009-01-18 16:28:34 +1100[diff] [blame]682 mov 480(KEYP), KLEN
683 add $240, KEYP
684 movups (IVP), IV
685 cmp $64, LEN
686 jb .Lcbc_dec_loop1
687.align 4
688.Lcbc_dec_loop4:
689 movups (INP), IN1
690 movaps IN1, STATE1
691 movups 0x10(INP), IN2
692 movaps IN2, STATE2
693 movups 0x20(INP), IN3
694 movaps IN3, STATE3
695 movups 0x30(INP), IN4
696 movaps IN4, STATE4
697 call _aesni_dec4
698 pxor IV, STATE1
699 pxor IN1, STATE2
700 pxor IN2, STATE3
701 pxor IN3, STATE4
702 movaps IN4, IV
703 movups STATE1, (OUTP)
704 movups STATE2, 0x10(OUTP)
705 movups STATE3, 0x20(OUTP)
706 movups STATE4, 0x30(OUTP)
707 sub $64, LEN
708 add $64, INP
709 add $64, OUTP
710 cmp $64, LEN
711 jge .Lcbc_dec_loop4
712 cmp $16, LEN
713 jb .Lcbc_dec_ret
714.align 4
715.Lcbc_dec_loop1:
716 movups (INP), IN
717 movaps IN, STATE
718 call _aesni_dec1
719 pxor IV, STATE
720 movups STATE, (OUTP)
721 movaps IN, IV
722 sub $16, LEN
723 add $16, INP
724 add $16, OUTP
725 cmp $16, LEN
726 jge .Lcbc_dec_loop1
Huang Ying54b6a1b2009-01-18 16:28:34 +1100[diff] [blame]727.Lcbc_dec_ret:
Huang Yinge6efaa02009-06-18 19:33:57 +0800[diff] [blame]728 movups IV, (IVP)
729.Lcbc_dec_just_ret:
Huang Ying54b6a1b2009-01-18 16:28:34 +1100[diff] [blame]730 ret
Huang Ying12387a42010-03-10 18:28:55 +0800[diff] [blame^]731
732.align 16
733.Lbswap_mask:
734 .byte 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0
735
736/*
737 * _aesni_inc_init: internal ABI
738 * setup registers used by _aesni_inc
739 * input:
740 * IV
741 * output:
742 * CTR: == IV, in little endian
743 * TCTR_LOW: == lower qword of CTR
744 * INC: == 1, in little endian
745 * BSWAP_MASK == endian swapping mask
746 */
747_aesni_inc_init:
748 movaps .Lbswap_mask, BSWAP_MASK
749 movaps IV, CTR
750 PSHUFB_XMM BSWAP_MASK CTR
751 mov $1, TCTR_LOW
752 movq TCTR_LOW, INC
753 movq CTR, TCTR_LOW
754 ret
755
756/*
757 * _aesni_inc: internal ABI
758 * Increase IV by 1, IV is in big endian
759 * input:
760 * IV
761 * CTR: == IV, in little endian
762 * TCTR_LOW: == lower qword of CTR
763 * INC: == 1, in little endian
764 * BSWAP_MASK == endian swapping mask
765 * output:
766 * IV: Increase by 1
767 * changed:
768 * CTR: == output IV, in little endian
769 * TCTR_LOW: == lower qword of CTR
770 */
771_aesni_inc:
772 paddq INC, CTR
773 add $1, TCTR_LOW
774 jnc .Linc_low
775 pslldq $8, INC
776 paddq INC, CTR
777 psrldq $8, INC
778.Linc_low:
779 movaps CTR, IV
780 PSHUFB_XMM BSWAP_MASK IV
781 ret
782
783/*
784 * void aesni_ctr_enc(struct crypto_aes_ctx *ctx, const u8 *dst, u8 *src,
785 * size_t len, u8 *iv)
786 */
787ENTRY(aesni_ctr_enc)
788 cmp $16, LEN
789 jb .Lctr_enc_just_ret
790 mov 480(KEYP), KLEN
791 movups (IVP), IV
792 call _aesni_inc_init
793 cmp $64, LEN
794 jb .Lctr_enc_loop1
795.align 4
796.Lctr_enc_loop4:
797 movaps IV, STATE1
798 call _aesni_inc
799 movups (INP), IN1
800 movaps IV, STATE2
801 call _aesni_inc
802 movups 0x10(INP), IN2
803 movaps IV, STATE3
804 call _aesni_inc
805 movups 0x20(INP), IN3
806 movaps IV, STATE4
807 call _aesni_inc
808 movups 0x30(INP), IN4
809 call _aesni_enc4
810 pxor IN1, STATE1
811 movups STATE1, (OUTP)
812 pxor IN2, STATE2
813 movups STATE2, 0x10(OUTP)
814 pxor IN3, STATE3
815 movups STATE3, 0x20(OUTP)
816 pxor IN4, STATE4
817 movups STATE4, 0x30(OUTP)
818 sub $64, LEN
819 add $64, INP
820 add $64, OUTP
821 cmp $64, LEN
822 jge .Lctr_enc_loop4
823 cmp $16, LEN
824 jb .Lctr_enc_ret
825.align 4
826.Lctr_enc_loop1:
827 movaps IV, STATE
828 call _aesni_inc
829 movups (INP), IN
830 call _aesni_enc1
831 pxor IN, STATE
832 movups STATE, (OUTP)
833 sub $16, LEN
834 add $16, INP
835 add $16, OUTP
836 cmp $16, LEN
837 jge .Lctr_enc_loop1
838.Lctr_enc_ret:
839 movups IV, (IVP)
840.Lctr_enc_just_ret:
841 ret