blob: 29071a156cbe131fd58b63abdffd947d7601d325 [file] [log] [blame]
Kim Phillips8e8ec592011-03-13 16:54:26 +08001/*
2 * caam - Freescale FSL CAAM support for crypto API
3 *
4 * Copyright 2008-2011 Freescale Semiconductor, Inc.
5 *
6 * Based on talitos crypto API driver.
7 *
8 * relationship of job descriptors to shared descriptors (SteveC Dec 10 2008):
9 *
10 * --------------- ---------------
11 * | JobDesc #1 |-------------------->| ShareDesc |
12 * | *(packet 1) | | (PDB) |
13 * --------------- |------------->| (hashKey) |
14 * . | | (cipherKey) |
15 * . | |-------->| (operation) |
16 * --------------- | | ---------------
17 * | JobDesc #2 |------| |
18 * | *(packet 2) | |
19 * --------------- |
20 * . |
21 * . |
22 * --------------- |
23 * | JobDesc #3 |------------
24 * | *(packet 3) |
25 * ---------------
26 *
27 * The SharedDesc never changes for a connection unless rekeyed, but
28 * each packet will likely be in a different place. So all we need
29 * to know to process the packet is where the input is, where the
30 * output goes, and what context we want to process with. Context is
31 * in the SharedDesc, packet references in the JobDesc.
32 *
33 * So, a job desc looks like:
34 *
35 * ---------------------
36 * | Header |
37 * | ShareDesc Pointer |
38 * | SEQ_OUT_PTR |
39 * | (output buffer) |
Yuan Kang6ec47332012-06-22 19:48:43 -050040 * | (output length) |
Kim Phillips8e8ec592011-03-13 16:54:26 +080041 * | SEQ_IN_PTR |
42 * | (input buffer) |
Yuan Kang6ec47332012-06-22 19:48:43 -050043 * | (input length) |
Kim Phillips8e8ec592011-03-13 16:54:26 +080044 * ---------------------
45 */
46
47#include "compat.h"
48
49#include "regs.h"
50#include "intern.h"
51#include "desc_constr.h"
52#include "jr.h"
53#include "error.h"
Yuan Kanga299c832012-06-22 19:48:46 -050054#include "sg_sw_sec4.h"
Yuan Kang4c1ec1f2012-06-22 19:48:45 -050055#include "key_gen.h"
Kim Phillips8e8ec592011-03-13 16:54:26 +080056
57/*
58 * crypto alg
59 */
60#define CAAM_CRA_PRIORITY 3000
61/* max key is sum of AES_MAX_KEY_SIZE, max split key size */
62#define CAAM_MAX_KEY_SIZE (AES_MAX_KEY_SIZE + \
Catalin Vasiledaebc462014-10-31 12:45:37 +020063 CTR_RFC3686_NONCE_SIZE + \
Kim Phillips8e8ec592011-03-13 16:54:26 +080064 SHA512_DIGEST_SIZE * 2)
65/* max IV is max of AES_BLOCK_SIZE, DES3_EDE_BLOCK_SIZE */
66#define CAAM_MAX_IV_LENGTH 16
67
Kim Phillips4427b1b2011-05-14 22:08:17 -050068/* length of descriptors text */
Yuan Kang1acebad2011-07-15 11:21:42 +080069#define DESC_AEAD_BASE (4 * CAAM_CMD_SZ)
Horia Geanta4464a7d2014-03-14 17:46:49 +020070#define DESC_AEAD_ENC_LEN (DESC_AEAD_BASE + 15 * CAAM_CMD_SZ)
71#define DESC_AEAD_DEC_LEN (DESC_AEAD_BASE + 18 * CAAM_CMD_SZ)
Yuan Kang1acebad2011-07-15 11:21:42 +080072#define DESC_AEAD_GIVENC_LEN (DESC_AEAD_ENC_LEN + 7 * CAAM_CMD_SZ)
73
Catalin Vasiledaebc462014-10-31 12:45:37 +020074/* Note: Nonce is counted in enckeylen */
75#define DESC_AEAD_CTR_RFC3686_LEN (6 * CAAM_CMD_SZ)
76
Horia Geantaae4a8252014-03-14 17:46:52 +020077#define DESC_AEAD_NULL_BASE (3 * CAAM_CMD_SZ)
78#define DESC_AEAD_NULL_ENC_LEN (DESC_AEAD_NULL_BASE + 14 * CAAM_CMD_SZ)
79#define DESC_AEAD_NULL_DEC_LEN (DESC_AEAD_NULL_BASE + 17 * CAAM_CMD_SZ)
80
Tudor Ambarus3ef8d942014-10-23 16:11:23 +030081#define DESC_GCM_BASE (3 * CAAM_CMD_SZ)
82#define DESC_GCM_ENC_LEN (DESC_GCM_BASE + 23 * CAAM_CMD_SZ)
83#define DESC_GCM_DEC_LEN (DESC_GCM_BASE + 19 * CAAM_CMD_SZ)
84
Tudor Ambarusbac68f22014-10-23 16:14:03 +030085#define DESC_RFC4106_BASE (3 * CAAM_CMD_SZ)
86#define DESC_RFC4106_ENC_LEN (DESC_RFC4106_BASE + 15 * CAAM_CMD_SZ)
87#define DESC_RFC4106_DEC_LEN (DESC_RFC4106_BASE + 14 * CAAM_CMD_SZ)
88#define DESC_RFC4106_GIVENC_LEN (DESC_RFC4106_BASE + 21 * CAAM_CMD_SZ)
89
Tudor Ambarus5d0429a2014-10-30 18:55:07 +020090#define DESC_RFC4543_BASE (3 * CAAM_CMD_SZ)
91#define DESC_RFC4543_ENC_LEN (DESC_RFC4543_BASE + 25 * CAAM_CMD_SZ)
92#define DESC_RFC4543_DEC_LEN (DESC_RFC4543_BASE + 27 * CAAM_CMD_SZ)
93#define DESC_RFC4543_GIVENC_LEN (DESC_RFC4543_BASE + 30 * CAAM_CMD_SZ)
94
Yuan Kangacdca312011-07-15 11:21:42 +080095#define DESC_ABLKCIPHER_BASE (3 * CAAM_CMD_SZ)
96#define DESC_ABLKCIPHER_ENC_LEN (DESC_ABLKCIPHER_BASE + \
97 20 * CAAM_CMD_SZ)
98#define DESC_ABLKCIPHER_DEC_LEN (DESC_ABLKCIPHER_BASE + \
99 15 * CAAM_CMD_SZ)
100
Tudor Ambarus5d0429a2014-10-30 18:55:07 +0200101#define DESC_MAX_USED_BYTES (DESC_RFC4543_GIVENC_LEN + \
Yuan Kang1acebad2011-07-15 11:21:42 +0800102 CAAM_MAX_KEY_SIZE)
103#define DESC_MAX_USED_LEN (DESC_MAX_USED_BYTES / CAAM_CMD_SZ)
Kim Phillips4427b1b2011-05-14 22:08:17 -0500104
Kim Phillips8e8ec592011-03-13 16:54:26 +0800105#ifdef DEBUG
106/* for print_hex_dumps with line references */
Kim Phillips8e8ec592011-03-13 16:54:26 +0800107#define debug(format, arg...) printk(format, arg)
108#else
109#define debug(format, arg...)
110#endif
Ruchika Guptacfc6f112013-10-25 12:01:03 +0530111static struct list_head alg_list;
Kim Phillips8e8ec592011-03-13 16:54:26 +0800112
Yuan Kang1acebad2011-07-15 11:21:42 +0800113/* Set DK bit in class 1 operation if shared */
114static inline void append_dec_op1(u32 *desc, u32 type)
115{
116 u32 *jump_cmd, *uncond_jump_cmd;
117
Horia Geantaa60384d2014-07-11 15:46:58 +0300118 /* DK bit is valid only for AES */
119 if ((type & OP_ALG_ALGSEL_MASK) != OP_ALG_ALGSEL_AES) {
120 append_operation(desc, type | OP_ALG_AS_INITFINAL |
121 OP_ALG_DECRYPT);
122 return;
123 }
124
Yuan Kang1acebad2011-07-15 11:21:42 +0800125 jump_cmd = append_jump(desc, JUMP_TEST_ALL | JUMP_COND_SHRD);
126 append_operation(desc, type | OP_ALG_AS_INITFINAL |
127 OP_ALG_DECRYPT);
128 uncond_jump_cmd = append_jump(desc, JUMP_TEST_ALL);
129 set_jump_tgt_here(desc, jump_cmd);
130 append_operation(desc, type | OP_ALG_AS_INITFINAL |
131 OP_ALG_DECRYPT | OP_ALG_AAI_DK);
132 set_jump_tgt_here(desc, uncond_jump_cmd);
133}
134
135/*
Yuan Kang1acebad2011-07-15 11:21:42 +0800136 * For aead functions, read payload and write payload,
137 * both of which are specified in req->src and req->dst
138 */
139static inline void aead_append_src_dst(u32 *desc, u32 msg_type)
140{
Horia Geantaae4a8252014-03-14 17:46:52 +0200141 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | KEY_VLF);
Yuan Kang1acebad2011-07-15 11:21:42 +0800142 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_BOTH |
143 KEY_VLF | msg_type | FIFOLD_TYPE_LASTBOTH);
Yuan Kang1acebad2011-07-15 11:21:42 +0800144}
145
146/*
147 * For aead encrypt and decrypt, read iv for both classes
148 */
Catalin Vasiledaebc462014-10-31 12:45:37 +0200149static inline void aead_append_ld_iv(u32 *desc, int ivsize, int ivoffset)
Yuan Kang1acebad2011-07-15 11:21:42 +0800150{
Catalin Vasiledaebc462014-10-31 12:45:37 +0200151 append_seq_load(desc, ivsize, LDST_CLASS_1_CCB |
152 LDST_SRCDST_BYTE_CONTEXT |
153 (ivoffset << LDST_OFFSET_SHIFT));
154 append_move(desc, MOVE_SRC_CLASS1CTX | MOVE_DEST_CLASS2INFIFO |
155 (ivoffset << MOVE_OFFSET_SHIFT) | ivsize);
Yuan Kang1acebad2011-07-15 11:21:42 +0800156}
157
158/*
Yuan Kangacdca312011-07-15 11:21:42 +0800159 * For ablkcipher encrypt and decrypt, read from req->src and
160 * write to req->dst
161 */
162static inline void ablkcipher_append_src_dst(u32 *desc)
163{
Kim Phillips70d793c2012-06-22 19:42:35 -0500164 append_math_add(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
165 append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
166 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 |
167 KEY_VLF | FIFOLD_TYPE_MSG | FIFOLD_TYPE_LAST1);
168 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | KEY_VLF);
Yuan Kangacdca312011-07-15 11:21:42 +0800169}
170
171/*
Yuan Kang1acebad2011-07-15 11:21:42 +0800172 * If all data, including src (with assoc and iv) or dst (with iv only) are
173 * contiguous
174 */
175#define GIV_SRC_CONTIG 1
176#define GIV_DST_CONTIG (1 << 1)
177
Kim Phillips8e8ec592011-03-13 16:54:26 +0800178/*
179 * per-session context
180 */
181struct caam_ctx {
182 struct device *jrdev;
Yuan Kang1acebad2011-07-15 11:21:42 +0800183 u32 sh_desc_enc[DESC_MAX_USED_LEN];
184 u32 sh_desc_dec[DESC_MAX_USED_LEN];
185 u32 sh_desc_givenc[DESC_MAX_USED_LEN];
186 dma_addr_t sh_desc_enc_dma;
187 dma_addr_t sh_desc_dec_dma;
188 dma_addr_t sh_desc_givenc_dma;
Kim Phillips8e8ec592011-03-13 16:54:26 +0800189 u32 class1_alg_type;
190 u32 class2_alg_type;
191 u32 alg_op;
Yuan Kang1acebad2011-07-15 11:21:42 +0800192 u8 key[CAAM_MAX_KEY_SIZE];
Yuan Kang885e9e22011-07-15 11:21:41 +0800193 dma_addr_t key_dma;
Kim Phillips8e8ec592011-03-13 16:54:26 +0800194 unsigned int enckeylen;
Kim Phillips8e8ec592011-03-13 16:54:26 +0800195 unsigned int split_key_len;
196 unsigned int split_key_pad_len;
197 unsigned int authsize;
198};
199
Yuan Kang1acebad2011-07-15 11:21:42 +0800200static void append_key_aead(u32 *desc, struct caam_ctx *ctx,
Catalin Vasiledaebc462014-10-31 12:45:37 +0200201 int keys_fit_inline, bool is_rfc3686)
Yuan Kang1acebad2011-07-15 11:21:42 +0800202{
Catalin Vasiledaebc462014-10-31 12:45:37 +0200203 u32 *nonce;
204 unsigned int enckeylen = ctx->enckeylen;
205
206 /*
207 * RFC3686 specific:
208 * | ctx->key = {AUTH_KEY, ENC_KEY, NONCE}
209 * | enckeylen = encryption key size + nonce size
210 */
211 if (is_rfc3686)
212 enckeylen -= CTR_RFC3686_NONCE_SIZE;
213
Yuan Kang1acebad2011-07-15 11:21:42 +0800214 if (keys_fit_inline) {
215 append_key_as_imm(desc, ctx->key, ctx->split_key_pad_len,
216 ctx->split_key_len, CLASS_2 |
217 KEY_DEST_MDHA_SPLIT | KEY_ENC);
218 append_key_as_imm(desc, (void *)ctx->key +
Catalin Vasiledaebc462014-10-31 12:45:37 +0200219 ctx->split_key_pad_len, enckeylen,
220 enckeylen, CLASS_1 | KEY_DEST_CLASS_REG);
Yuan Kang1acebad2011-07-15 11:21:42 +0800221 } else {
222 append_key(desc, ctx->key_dma, ctx->split_key_len, CLASS_2 |
223 KEY_DEST_MDHA_SPLIT | KEY_ENC);
224 append_key(desc, ctx->key_dma + ctx->split_key_pad_len,
Catalin Vasiledaebc462014-10-31 12:45:37 +0200225 enckeylen, CLASS_1 | KEY_DEST_CLASS_REG);
226 }
227
228 /* Load Counter into CONTEXT1 reg */
229 if (is_rfc3686) {
230 nonce = (u32 *)((void *)ctx->key + ctx->split_key_pad_len +
231 enckeylen);
232 append_load_imm_u32(desc, *nonce, LDST_CLASS_IND_CCB |
233 LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM);
234 append_move(desc,
235 MOVE_SRC_OUTFIFO |
236 MOVE_DEST_CLASS1CTX |
237 (16 << MOVE_OFFSET_SHIFT) |
238 (CTR_RFC3686_NONCE_SIZE << MOVE_LEN_SHIFT));
Yuan Kang1acebad2011-07-15 11:21:42 +0800239 }
240}
241
242static void init_sh_desc_key_aead(u32 *desc, struct caam_ctx *ctx,
Catalin Vasiledaebc462014-10-31 12:45:37 +0200243 int keys_fit_inline, bool is_rfc3686)
Yuan Kang1acebad2011-07-15 11:21:42 +0800244{
245 u32 *key_jump_cmd;
246
Catalin Vasiledaebc462014-10-31 12:45:37 +0200247 /* Note: Context registers are saved. */
248 init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX);
Yuan Kang1acebad2011-07-15 11:21:42 +0800249
250 /* Skip if already shared */
251 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
252 JUMP_COND_SHRD);
253
Catalin Vasiledaebc462014-10-31 12:45:37 +0200254 append_key_aead(desc, ctx, keys_fit_inline, is_rfc3686);
Yuan Kang1acebad2011-07-15 11:21:42 +0800255
256 set_jump_tgt_here(desc, key_jump_cmd);
Yuan Kang1acebad2011-07-15 11:21:42 +0800257}
258
Horia Geantaae4a8252014-03-14 17:46:52 +0200259static int aead_null_set_sh_desc(struct crypto_aead *aead)
260{
261 struct aead_tfm *tfm = &aead->base.crt_aead;
262 struct caam_ctx *ctx = crypto_aead_ctx(aead);
263 struct device *jrdev = ctx->jrdev;
264 bool keys_fit_inline = false;
265 u32 *key_jump_cmd, *jump_cmd, *read_move_cmd, *write_move_cmd;
266 u32 *desc;
267
268 /*
269 * Job Descriptor and Shared Descriptors
270 * must all fit into the 64-word Descriptor h/w Buffer
271 */
272 if (DESC_AEAD_NULL_ENC_LEN + DESC_JOB_IO_LEN +
273 ctx->split_key_pad_len <= CAAM_DESC_BYTES_MAX)
274 keys_fit_inline = true;
275
276 /* aead_encrypt shared descriptor */
277 desc = ctx->sh_desc_enc;
278
279 init_sh_desc(desc, HDR_SHARE_SERIAL);
280
281 /* Skip if already shared */
282 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
283 JUMP_COND_SHRD);
284 if (keys_fit_inline)
285 append_key_as_imm(desc, ctx->key, ctx->split_key_pad_len,
286 ctx->split_key_len, CLASS_2 |
287 KEY_DEST_MDHA_SPLIT | KEY_ENC);
288 else
289 append_key(desc, ctx->key_dma, ctx->split_key_len, CLASS_2 |
290 KEY_DEST_MDHA_SPLIT | KEY_ENC);
291 set_jump_tgt_here(desc, key_jump_cmd);
292
293 /* cryptlen = seqoutlen - authsize */
294 append_math_sub_imm_u32(desc, REG3, SEQOUTLEN, IMM, ctx->authsize);
295
296 /*
297 * NULL encryption; IV is zero
298 * assoclen = (assoclen + cryptlen) - cryptlen
299 */
300 append_math_sub(desc, VARSEQINLEN, SEQINLEN, REG3, CAAM_CMD_SZ);
301
302 /* read assoc before reading payload */
303 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG |
304 KEY_VLF);
305
306 /* Prepare to read and write cryptlen bytes */
307 append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
308 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
309
310 /*
311 * MOVE_LEN opcode is not available in all SEC HW revisions,
312 * thus need to do some magic, i.e. self-patch the descriptor
313 * buffer.
314 */
315 read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF |
316 MOVE_DEST_MATH3 |
317 (0x6 << MOVE_LEN_SHIFT));
318 write_move_cmd = append_move(desc, MOVE_SRC_MATH3 |
319 MOVE_DEST_DESCBUF |
320 MOVE_WAITCOMP |
321 (0x8 << MOVE_LEN_SHIFT));
322
323 /* Class 2 operation */
324 append_operation(desc, ctx->class2_alg_type |
325 OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT);
326
327 /* Read and write cryptlen bytes */
328 aead_append_src_dst(desc, FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1);
329
330 set_move_tgt_here(desc, read_move_cmd);
331 set_move_tgt_here(desc, write_move_cmd);
332 append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO);
333 append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO |
334 MOVE_AUX_LS);
335
336 /* Write ICV */
337 append_seq_store(desc, ctx->authsize, LDST_CLASS_2_CCB |
338 LDST_SRCDST_BYTE_CONTEXT);
339
340 ctx->sh_desc_enc_dma = dma_map_single(jrdev, desc,
341 desc_bytes(desc),
342 DMA_TO_DEVICE);
343 if (dma_mapping_error(jrdev, ctx->sh_desc_enc_dma)) {
344 dev_err(jrdev, "unable to map shared descriptor\n");
345 return -ENOMEM;
346 }
347#ifdef DEBUG
348 print_hex_dump(KERN_ERR,
349 "aead null enc shdesc@"__stringify(__LINE__)": ",
350 DUMP_PREFIX_ADDRESS, 16, 4, desc,
351 desc_bytes(desc), 1);
352#endif
353
354 /*
355 * Job Descriptor and Shared Descriptors
356 * must all fit into the 64-word Descriptor h/w Buffer
357 */
Vakul Garg80cd88f2014-05-09 20:34:40 -0500358 keys_fit_inline = false;
Horia Geantaae4a8252014-03-14 17:46:52 +0200359 if (DESC_AEAD_NULL_DEC_LEN + DESC_JOB_IO_LEN +
360 ctx->split_key_pad_len <= CAAM_DESC_BYTES_MAX)
361 keys_fit_inline = true;
362
363 desc = ctx->sh_desc_dec;
364
365 /* aead_decrypt shared descriptor */
366 init_sh_desc(desc, HDR_SHARE_SERIAL);
367
368 /* Skip if already shared */
369 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
370 JUMP_COND_SHRD);
371 if (keys_fit_inline)
372 append_key_as_imm(desc, ctx->key, ctx->split_key_pad_len,
373 ctx->split_key_len, CLASS_2 |
374 KEY_DEST_MDHA_SPLIT | KEY_ENC);
375 else
376 append_key(desc, ctx->key_dma, ctx->split_key_len, CLASS_2 |
377 KEY_DEST_MDHA_SPLIT | KEY_ENC);
378 set_jump_tgt_here(desc, key_jump_cmd);
379
380 /* Class 2 operation */
381 append_operation(desc, ctx->class2_alg_type |
382 OP_ALG_AS_INITFINAL | OP_ALG_DECRYPT | OP_ALG_ICV_ON);
383
384 /* assoclen + cryptlen = seqinlen - ivsize - authsize */
385 append_math_sub_imm_u32(desc, REG3, SEQINLEN, IMM,
386 ctx->authsize + tfm->ivsize);
387 /* assoclen = (assoclen + cryptlen) - cryptlen */
388 append_math_sub(desc, REG2, SEQOUTLEN, REG0, CAAM_CMD_SZ);
389 append_math_sub(desc, VARSEQINLEN, REG3, REG2, CAAM_CMD_SZ);
390
391 /* read assoc before reading payload */
392 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG |
393 KEY_VLF);
394
395 /* Prepare to read and write cryptlen bytes */
396 append_math_add(desc, VARSEQINLEN, ZERO, REG2, CAAM_CMD_SZ);
397 append_math_add(desc, VARSEQOUTLEN, ZERO, REG2, CAAM_CMD_SZ);
398
399 /*
400 * MOVE_LEN opcode is not available in all SEC HW revisions,
401 * thus need to do some magic, i.e. self-patch the descriptor
402 * buffer.
403 */
404 read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF |
405 MOVE_DEST_MATH2 |
406 (0x6 << MOVE_LEN_SHIFT));
407 write_move_cmd = append_move(desc, MOVE_SRC_MATH2 |
408 MOVE_DEST_DESCBUF |
409 MOVE_WAITCOMP |
410 (0x8 << MOVE_LEN_SHIFT));
411
412 /* Read and write cryptlen bytes */
413 aead_append_src_dst(desc, FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1);
414
415 /*
416 * Insert a NOP here, since we need at least 4 instructions between
417 * code patching the descriptor buffer and the location being patched.
418 */
419 jump_cmd = append_jump(desc, JUMP_TEST_ALL);
420 set_jump_tgt_here(desc, jump_cmd);
421
422 set_move_tgt_here(desc, read_move_cmd);
423 set_move_tgt_here(desc, write_move_cmd);
424 append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO);
425 append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO |
426 MOVE_AUX_LS);
427 append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO);
428
429 /* Load ICV */
430 append_seq_fifo_load(desc, ctx->authsize, FIFOLD_CLASS_CLASS2 |
431 FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_ICV);
432
433 ctx->sh_desc_dec_dma = dma_map_single(jrdev, desc,
434 desc_bytes(desc),
435 DMA_TO_DEVICE);
436 if (dma_mapping_error(jrdev, ctx->sh_desc_dec_dma)) {
437 dev_err(jrdev, "unable to map shared descriptor\n");
438 return -ENOMEM;
439 }
440#ifdef DEBUG
441 print_hex_dump(KERN_ERR,
442 "aead null dec shdesc@"__stringify(__LINE__)": ",
443 DUMP_PREFIX_ADDRESS, 16, 4, desc,
444 desc_bytes(desc), 1);
445#endif
446
447 return 0;
448}
449
Yuan Kang1acebad2011-07-15 11:21:42 +0800450static int aead_set_sh_desc(struct crypto_aead *aead)
451{
452 struct aead_tfm *tfm = &aead->base.crt_aead;
453 struct caam_ctx *ctx = crypto_aead_ctx(aead);
Catalin Vasiledaebc462014-10-31 12:45:37 +0200454 struct crypto_tfm *ctfm = crypto_aead_tfm(aead);
455 const char *alg_name = crypto_tfm_alg_name(ctfm);
Yuan Kang1acebad2011-07-15 11:21:42 +0800456 struct device *jrdev = ctx->jrdev;
Catalin Vasiledaebc462014-10-31 12:45:37 +0200457 bool keys_fit_inline;
Yuan Kang1acebad2011-07-15 11:21:42 +0800458 u32 geniv, moveiv;
Catalin Vasiledaebc462014-10-31 12:45:37 +0200459 u32 ctx1_iv_off = 0;
Yuan Kang1acebad2011-07-15 11:21:42 +0800460 u32 *desc;
Catalin Vasiledaebc462014-10-31 12:45:37 +0200461 const bool ctr_mode = ((ctx->class1_alg_type & OP_ALG_AAI_MASK) ==
462 OP_ALG_AAI_CTR_MOD128);
463 const bool is_rfc3686 = (ctr_mode &&
464 (strstr(alg_name, "rfc3686") != NULL));
Yuan Kang1acebad2011-07-15 11:21:42 +0800465
Horia Geantaae4a8252014-03-14 17:46:52 +0200466 if (!ctx->authsize)
Yuan Kang1acebad2011-07-15 11:21:42 +0800467 return 0;
468
Horia Geantaae4a8252014-03-14 17:46:52 +0200469 /* NULL encryption / decryption */
470 if (!ctx->enckeylen)
471 return aead_null_set_sh_desc(aead);
472
Yuan Kang1acebad2011-07-15 11:21:42 +0800473 /*
Catalin Vasiledaebc462014-10-31 12:45:37 +0200474 * AES-CTR needs to load IV in CONTEXT1 reg
475 * at an offset of 128bits (16bytes)
476 * CONTEXT1[255:128] = IV
477 */
478 if (ctr_mode)
479 ctx1_iv_off = 16;
480
481 /*
482 * RFC3686 specific:
483 * CONTEXT1[255:128] = {NONCE, IV, COUNTER}
484 */
485 if (is_rfc3686)
486 ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE;
487
488 /*
Yuan Kang1acebad2011-07-15 11:21:42 +0800489 * Job Descriptor and Shared Descriptors
490 * must all fit into the 64-word Descriptor h/w Buffer
491 */
Catalin Vasiledaebc462014-10-31 12:45:37 +0200492 keys_fit_inline = false;
Yuan Kang1acebad2011-07-15 11:21:42 +0800493 if (DESC_AEAD_ENC_LEN + DESC_JOB_IO_LEN +
Catalin Vasiledaebc462014-10-31 12:45:37 +0200494 ctx->split_key_pad_len + ctx->enckeylen +
495 (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0) <=
Yuan Kang1acebad2011-07-15 11:21:42 +0800496 CAAM_DESC_BYTES_MAX)
Kim Phillips2af8f4a2012-09-07 04:17:03 +0800497 keys_fit_inline = true;
Yuan Kang1acebad2011-07-15 11:21:42 +0800498
499 /* aead_encrypt shared descriptor */
500 desc = ctx->sh_desc_enc;
501
Catalin Vasiledaebc462014-10-31 12:45:37 +0200502 /* Note: Context registers are saved. */
503 init_sh_desc_key_aead(desc, ctx, keys_fit_inline, is_rfc3686);
Yuan Kang1acebad2011-07-15 11:21:42 +0800504
505 /* Class 2 operation */
506 append_operation(desc, ctx->class2_alg_type |
507 OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT);
508
509 /* cryptlen = seqoutlen - authsize */
510 append_math_sub_imm_u32(desc, REG3, SEQOUTLEN, IMM, ctx->authsize);
511
512 /* assoclen + cryptlen = seqinlen - ivsize */
513 append_math_sub_imm_u32(desc, REG2, SEQINLEN, IMM, tfm->ivsize);
514
Horia Geanta4464a7d2014-03-14 17:46:49 +0200515 /* assoclen = (assoclen + cryptlen) - cryptlen */
Yuan Kang1acebad2011-07-15 11:21:42 +0800516 append_math_sub(desc, VARSEQINLEN, REG2, REG3, CAAM_CMD_SZ);
517
518 /* read assoc before reading payload */
519 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG |
520 KEY_VLF);
Catalin Vasiledaebc462014-10-31 12:45:37 +0200521 aead_append_ld_iv(desc, tfm->ivsize, ctx1_iv_off);
522
523 /* Load Counter into CONTEXT1 reg */
524 if (is_rfc3686)
525 append_load_imm_u32(desc, be32_to_cpu(1), LDST_IMM |
526 LDST_CLASS_1_CCB |
527 LDST_SRCDST_BYTE_CONTEXT |
528 ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) <<
529 LDST_OFFSET_SHIFT));
Yuan Kang1acebad2011-07-15 11:21:42 +0800530
531 /* Class 1 operation */
532 append_operation(desc, ctx->class1_alg_type |
533 OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT);
534
535 /* Read and write cryptlen bytes */
536 append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
537 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
538 aead_append_src_dst(desc, FIFOLD_TYPE_MSG1OUT2);
539
540 /* Write ICV */
541 append_seq_store(desc, ctx->authsize, LDST_CLASS_2_CCB |
542 LDST_SRCDST_BYTE_CONTEXT);
543
544 ctx->sh_desc_enc_dma = dma_map_single(jrdev, desc,
545 desc_bytes(desc),
546 DMA_TO_DEVICE);
547 if (dma_mapping_error(jrdev, ctx->sh_desc_enc_dma)) {
548 dev_err(jrdev, "unable to map shared descriptor\n");
549 return -ENOMEM;
550 }
551#ifdef DEBUG
Alex Porosanu514df282013-08-14 18:56:45 +0300552 print_hex_dump(KERN_ERR, "aead enc shdesc@"__stringify(__LINE__)": ",
Yuan Kang1acebad2011-07-15 11:21:42 +0800553 DUMP_PREFIX_ADDRESS, 16, 4, desc,
554 desc_bytes(desc), 1);
555#endif
556
557 /*
558 * Job Descriptor and Shared Descriptors
559 * must all fit into the 64-word Descriptor h/w Buffer
560 */
Vakul Garg80cd88f2014-05-09 20:34:40 -0500561 keys_fit_inline = false;
Yuan Kang1acebad2011-07-15 11:21:42 +0800562 if (DESC_AEAD_DEC_LEN + DESC_JOB_IO_LEN +
Catalin Vasiledaebc462014-10-31 12:45:37 +0200563 ctx->split_key_pad_len + ctx->enckeylen +
564 (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0) <=
Yuan Kang1acebad2011-07-15 11:21:42 +0800565 CAAM_DESC_BYTES_MAX)
Kim Phillips2af8f4a2012-09-07 04:17:03 +0800566 keys_fit_inline = true;
Yuan Kang1acebad2011-07-15 11:21:42 +0800567
Horia Geanta4464a7d2014-03-14 17:46:49 +0200568 /* aead_decrypt shared descriptor */
Yuan Kang1acebad2011-07-15 11:21:42 +0800569 desc = ctx->sh_desc_dec;
570
Catalin Vasiledaebc462014-10-31 12:45:37 +0200571 /* Note: Context registers are saved. */
572 init_sh_desc_key_aead(desc, ctx, keys_fit_inline, is_rfc3686);
Yuan Kang1acebad2011-07-15 11:21:42 +0800573
574 /* Class 2 operation */
575 append_operation(desc, ctx->class2_alg_type |
576 OP_ALG_AS_INITFINAL | OP_ALG_DECRYPT | OP_ALG_ICV_ON);
577
Horia Geanta4464a7d2014-03-14 17:46:49 +0200578 /* assoclen + cryptlen = seqinlen - ivsize - authsize */
Yuan Kang1acebad2011-07-15 11:21:42 +0800579 append_math_sub_imm_u32(desc, REG3, SEQINLEN, IMM,
Horia Geantaae4a8252014-03-14 17:46:52 +0200580 ctx->authsize + tfm->ivsize);
Yuan Kang1acebad2011-07-15 11:21:42 +0800581 /* assoclen = (assoclen + cryptlen) - cryptlen */
582 append_math_sub(desc, REG2, SEQOUTLEN, REG0, CAAM_CMD_SZ);
583 append_math_sub(desc, VARSEQINLEN, REG3, REG2, CAAM_CMD_SZ);
584
585 /* read assoc before reading payload */
586 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG |
587 KEY_VLF);
588
Catalin Vasiledaebc462014-10-31 12:45:37 +0200589 aead_append_ld_iv(desc, tfm->ivsize, ctx1_iv_off);
Yuan Kang1acebad2011-07-15 11:21:42 +0800590
Catalin Vasiledaebc462014-10-31 12:45:37 +0200591 /* Load Counter into CONTEXT1 reg */
592 if (is_rfc3686)
593 append_load_imm_u32(desc, be32_to_cpu(1), LDST_IMM |
594 LDST_CLASS_1_CCB |
595 LDST_SRCDST_BYTE_CONTEXT |
596 ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) <<
597 LDST_OFFSET_SHIFT));
598
599 /* Choose operation */
600 if (ctr_mode)
601 append_operation(desc, ctx->class1_alg_type |
602 OP_ALG_AS_INITFINAL | OP_ALG_DECRYPT);
603 else
604 append_dec_op1(desc, ctx->class1_alg_type);
Yuan Kang1acebad2011-07-15 11:21:42 +0800605
606 /* Read and write cryptlen bytes */
607 append_math_add(desc, VARSEQINLEN, ZERO, REG2, CAAM_CMD_SZ);
608 append_math_add(desc, VARSEQOUTLEN, ZERO, REG2, CAAM_CMD_SZ);
609 aead_append_src_dst(desc, FIFOLD_TYPE_MSG);
610
611 /* Load ICV */
612 append_seq_fifo_load(desc, ctx->authsize, FIFOLD_CLASS_CLASS2 |
613 FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_ICV);
Yuan Kang1acebad2011-07-15 11:21:42 +0800614
615 ctx->sh_desc_dec_dma = dma_map_single(jrdev, desc,
616 desc_bytes(desc),
617 DMA_TO_DEVICE);
618 if (dma_mapping_error(jrdev, ctx->sh_desc_dec_dma)) {
619 dev_err(jrdev, "unable to map shared descriptor\n");
620 return -ENOMEM;
621 }
622#ifdef DEBUG
Alex Porosanu514df282013-08-14 18:56:45 +0300623 print_hex_dump(KERN_ERR, "aead dec shdesc@"__stringify(__LINE__)": ",
Yuan Kang1acebad2011-07-15 11:21:42 +0800624 DUMP_PREFIX_ADDRESS, 16, 4, desc,
625 desc_bytes(desc), 1);
626#endif
627
628 /*
629 * Job Descriptor and Shared Descriptors
630 * must all fit into the 64-word Descriptor h/w Buffer
631 */
Vakul Garg80cd88f2014-05-09 20:34:40 -0500632 keys_fit_inline = false;
Yuan Kang1acebad2011-07-15 11:21:42 +0800633 if (DESC_AEAD_GIVENC_LEN + DESC_JOB_IO_LEN +
Catalin Vasiledaebc462014-10-31 12:45:37 +0200634 ctx->split_key_pad_len + ctx->enckeylen +
635 (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0) <=
Yuan Kang1acebad2011-07-15 11:21:42 +0800636 CAAM_DESC_BYTES_MAX)
Kim Phillips2af8f4a2012-09-07 04:17:03 +0800637 keys_fit_inline = true;
Yuan Kang1acebad2011-07-15 11:21:42 +0800638
639 /* aead_givencrypt shared descriptor */
640 desc = ctx->sh_desc_givenc;
641
Catalin Vasiledaebc462014-10-31 12:45:37 +0200642 /* Note: Context registers are saved. */
643 init_sh_desc_key_aead(desc, ctx, keys_fit_inline, is_rfc3686);
Yuan Kang1acebad2011-07-15 11:21:42 +0800644
645 /* Generate IV */
646 geniv = NFIFOENTRY_STYPE_PAD | NFIFOENTRY_DEST_DECO |
647 NFIFOENTRY_DTYPE_MSG | NFIFOENTRY_LC1 |
648 NFIFOENTRY_PTYPE_RND | (tfm->ivsize << NFIFOENTRY_DLEN_SHIFT);
649 append_load_imm_u32(desc, geniv, LDST_CLASS_IND_CCB |
650 LDST_SRCDST_WORD_INFO_FIFO | LDST_IMM);
651 append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO);
Catalin Vasiledaebc462014-10-31 12:45:37 +0200652 append_move(desc, MOVE_WAITCOMP |
653 MOVE_SRC_INFIFO | MOVE_DEST_CLASS1CTX |
654 (ctx1_iv_off << MOVE_OFFSET_SHIFT) |
655 (tfm->ivsize << MOVE_LEN_SHIFT));
Yuan Kang1acebad2011-07-15 11:21:42 +0800656 append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO);
657
658 /* Copy IV to class 1 context */
Catalin Vasiledaebc462014-10-31 12:45:37 +0200659 append_move(desc, MOVE_SRC_CLASS1CTX | MOVE_DEST_OUTFIFO |
660 (ctx1_iv_off << MOVE_OFFSET_SHIFT) |
661 (tfm->ivsize << MOVE_LEN_SHIFT));
Yuan Kang1acebad2011-07-15 11:21:42 +0800662
663 /* Return to encryption */
664 append_operation(desc, ctx->class2_alg_type |
665 OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT);
666
667 /* ivsize + cryptlen = seqoutlen - authsize */
668 append_math_sub_imm_u32(desc, REG3, SEQOUTLEN, IMM, ctx->authsize);
669
670 /* assoclen = seqinlen - (ivsize + cryptlen) */
671 append_math_sub(desc, VARSEQINLEN, SEQINLEN, REG3, CAAM_CMD_SZ);
672
673 /* read assoc before reading payload */
674 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG |
675 KEY_VLF);
676
Catalin Vasiledaebc462014-10-31 12:45:37 +0200677 /* Copy iv from outfifo to class 2 fifo */
Yuan Kang1acebad2011-07-15 11:21:42 +0800678 moveiv = NFIFOENTRY_STYPE_OFIFO | NFIFOENTRY_DEST_CLASS2 |
679 NFIFOENTRY_DTYPE_MSG | (tfm->ivsize << NFIFOENTRY_DLEN_SHIFT);
680 append_load_imm_u32(desc, moveiv, LDST_CLASS_IND_CCB |
681 LDST_SRCDST_WORD_INFO_FIFO | LDST_IMM);
682 append_load_imm_u32(desc, tfm->ivsize, LDST_CLASS_2_CCB |
683 LDST_SRCDST_WORD_DATASZ_REG | LDST_IMM);
684
Catalin Vasiledaebc462014-10-31 12:45:37 +0200685 /* Load Counter into CONTEXT1 reg */
686 if (is_rfc3686)
687 append_load_imm_u32(desc, be32_to_cpu(1), LDST_IMM |
688 LDST_CLASS_1_CCB |
689 LDST_SRCDST_BYTE_CONTEXT |
690 ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) <<
691 LDST_OFFSET_SHIFT));
692
Yuan Kang1acebad2011-07-15 11:21:42 +0800693 /* Class 1 operation */
694 append_operation(desc, ctx->class1_alg_type |
695 OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT);
696
697 /* Will write ivsize + cryptlen */
698 append_math_add(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
699
700 /* Not need to reload iv */
701 append_seq_fifo_load(desc, tfm->ivsize,
702 FIFOLD_CLASS_SKIP);
703
704 /* Will read cryptlen */
705 append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
706 aead_append_src_dst(desc, FIFOLD_TYPE_MSG1OUT2);
707
708 /* Write ICV */
709 append_seq_store(desc, ctx->authsize, LDST_CLASS_2_CCB |
710 LDST_SRCDST_BYTE_CONTEXT);
711
712 ctx->sh_desc_givenc_dma = dma_map_single(jrdev, desc,
713 desc_bytes(desc),
714 DMA_TO_DEVICE);
715 if (dma_mapping_error(jrdev, ctx->sh_desc_givenc_dma)) {
716 dev_err(jrdev, "unable to map shared descriptor\n");
717 return -ENOMEM;
718 }
719#ifdef DEBUG
Alex Porosanu514df282013-08-14 18:56:45 +0300720 print_hex_dump(KERN_ERR, "aead givenc shdesc@"__stringify(__LINE__)": ",
Yuan Kang1acebad2011-07-15 11:21:42 +0800721 DUMP_PREFIX_ADDRESS, 16, 4, desc,
722 desc_bytes(desc), 1);
723#endif
724
725 return 0;
726}
727
Yuan Kang0e479302011-07-15 11:21:41 +0800728static int aead_setauthsize(struct crypto_aead *authenc,
Kim Phillips8e8ec592011-03-13 16:54:26 +0800729 unsigned int authsize)
730{
731 struct caam_ctx *ctx = crypto_aead_ctx(authenc);
732
733 ctx->authsize = authsize;
Yuan Kang1acebad2011-07-15 11:21:42 +0800734 aead_set_sh_desc(authenc);
Kim Phillips8e8ec592011-03-13 16:54:26 +0800735
736 return 0;
737}
738
Tudor Ambarus3ef8d942014-10-23 16:11:23 +0300739static int gcm_set_sh_desc(struct crypto_aead *aead)
740{
741 struct aead_tfm *tfm = &aead->base.crt_aead;
742 struct caam_ctx *ctx = crypto_aead_ctx(aead);
743 struct device *jrdev = ctx->jrdev;
744 bool keys_fit_inline = false;
745 u32 *key_jump_cmd, *zero_payload_jump_cmd,
746 *zero_assoc_jump_cmd1, *zero_assoc_jump_cmd2;
747 u32 *desc;
748
749 if (!ctx->enckeylen || !ctx->authsize)
750 return 0;
751
752 /*
753 * AES GCM encrypt shared descriptor
754 * Job Descriptor and Shared Descriptor
755 * must fit into the 64-word Descriptor h/w Buffer
756 */
757 if (DESC_GCM_ENC_LEN + DESC_JOB_IO_LEN +
758 ctx->enckeylen <= CAAM_DESC_BYTES_MAX)
759 keys_fit_inline = true;
760
761 desc = ctx->sh_desc_enc;
762
763 init_sh_desc(desc, HDR_SHARE_SERIAL);
764
765 /* skip key loading if they are loaded due to sharing */
766 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
767 JUMP_COND_SHRD | JUMP_COND_SELF);
768 if (keys_fit_inline)
769 append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen,
770 ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG);
771 else
772 append_key(desc, ctx->key_dma, ctx->enckeylen,
773 CLASS_1 | KEY_DEST_CLASS_REG);
774 set_jump_tgt_here(desc, key_jump_cmd);
775
776 /* class 1 operation */
777 append_operation(desc, ctx->class1_alg_type |
778 OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT);
779
780 /* cryptlen = seqoutlen - authsize */
781 append_math_sub_imm_u32(desc, REG3, SEQOUTLEN, IMM, ctx->authsize);
782
783 /* assoclen + cryptlen = seqinlen - ivsize */
784 append_math_sub_imm_u32(desc, REG2, SEQINLEN, IMM, tfm->ivsize);
785
786 /* assoclen = (assoclen + cryptlen) - cryptlen */
787 append_math_sub(desc, REG1, REG2, REG3, CAAM_CMD_SZ);
788
789 /* if cryptlen is ZERO jump to zero-payload commands */
790 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
791 zero_payload_jump_cmd = append_jump(desc, JUMP_TEST_ALL |
792 JUMP_COND_MATH_Z);
793 /* read IV */
794 append_seq_fifo_load(desc, tfm->ivsize, FIFOLD_CLASS_CLASS1 |
795 FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1);
796
797 /* if assoclen is ZERO, skip reading the assoc data */
798 append_math_add(desc, VARSEQINLEN, ZERO, REG1, CAAM_CMD_SZ);
799 zero_assoc_jump_cmd1 = append_jump(desc, JUMP_TEST_ALL |
800 JUMP_COND_MATH_Z);
801
802 /* read assoc data */
803 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
804 FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1);
805 set_jump_tgt_here(desc, zero_assoc_jump_cmd1);
806
807 append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
808
809 /* write encrypted data */
810 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF);
811
812 /* read payload data */
813 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
814 FIFOLD_TYPE_MSG | FIFOLD_TYPE_LAST1);
815
816 /* jump the zero-payload commands */
817 append_jump(desc, JUMP_TEST_ALL | 7);
818
819 /* zero-payload commands */
820 set_jump_tgt_here(desc, zero_payload_jump_cmd);
821
822 /* if assoclen is ZERO, jump to IV reading - is the only input data */
823 append_math_add(desc, VARSEQINLEN, ZERO, REG1, CAAM_CMD_SZ);
824 zero_assoc_jump_cmd2 = append_jump(desc, JUMP_TEST_ALL |
825 JUMP_COND_MATH_Z);
826 /* read IV */
827 append_seq_fifo_load(desc, tfm->ivsize, FIFOLD_CLASS_CLASS1 |
828 FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1);
829
830 /* read assoc data */
831 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
832 FIFOLD_TYPE_AAD | FIFOLD_TYPE_LAST1);
833
834 /* jump to ICV writing */
835 append_jump(desc, JUMP_TEST_ALL | 2);
836
837 /* read IV - is the only input data */
838 set_jump_tgt_here(desc, zero_assoc_jump_cmd2);
839 append_seq_fifo_load(desc, tfm->ivsize, FIFOLD_CLASS_CLASS1 |
840 FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1 |
841 FIFOLD_TYPE_LAST1);
842
843 /* write ICV */
844 append_seq_store(desc, ctx->authsize, LDST_CLASS_1_CCB |
845 LDST_SRCDST_BYTE_CONTEXT);
846
847 ctx->sh_desc_enc_dma = dma_map_single(jrdev, desc,
848 desc_bytes(desc),
849 DMA_TO_DEVICE);
850 if (dma_mapping_error(jrdev, ctx->sh_desc_enc_dma)) {
851 dev_err(jrdev, "unable to map shared descriptor\n");
852 return -ENOMEM;
853 }
854#ifdef DEBUG
855 print_hex_dump(KERN_ERR, "gcm enc shdesc@"__stringify(__LINE__)": ",
856 DUMP_PREFIX_ADDRESS, 16, 4, desc,
857 desc_bytes(desc), 1);
858#endif
859
860 /*
861 * Job Descriptor and Shared Descriptors
862 * must all fit into the 64-word Descriptor h/w Buffer
863 */
864 keys_fit_inline = false;
865 if (DESC_GCM_DEC_LEN + DESC_JOB_IO_LEN +
866 ctx->enckeylen <= CAAM_DESC_BYTES_MAX)
867 keys_fit_inline = true;
868
869 desc = ctx->sh_desc_dec;
870
871 init_sh_desc(desc, HDR_SHARE_SERIAL);
872
873 /* skip key loading if they are loaded due to sharing */
874 key_jump_cmd = append_jump(desc, JUMP_JSL |
875 JUMP_TEST_ALL | JUMP_COND_SHRD |
876 JUMP_COND_SELF);
877 if (keys_fit_inline)
878 append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen,
879 ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG);
880 else
881 append_key(desc, ctx->key_dma, ctx->enckeylen,
882 CLASS_1 | KEY_DEST_CLASS_REG);
883 set_jump_tgt_here(desc, key_jump_cmd);
884
885 /* class 1 operation */
886 append_operation(desc, ctx->class1_alg_type |
887 OP_ALG_AS_INITFINAL | OP_ALG_DECRYPT | OP_ALG_ICV_ON);
888
889 /* assoclen + cryptlen = seqinlen - ivsize - icvsize */
890 append_math_sub_imm_u32(desc, REG3, SEQINLEN, IMM,
891 ctx->authsize + tfm->ivsize);
892
893 /* assoclen = (assoclen + cryptlen) - cryptlen */
894 append_math_sub(desc, REG2, SEQOUTLEN, REG0, CAAM_CMD_SZ);
895 append_math_sub(desc, REG1, REG3, REG2, CAAM_CMD_SZ);
896
897 /* read IV */
898 append_seq_fifo_load(desc, tfm->ivsize, FIFOLD_CLASS_CLASS1 |
899 FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1);
900
901 /* jump to zero-payload command if cryptlen is zero */
902 append_math_add(desc, VARSEQOUTLEN, ZERO, REG2, CAAM_CMD_SZ);
903 zero_payload_jump_cmd = append_jump(desc, JUMP_TEST_ALL |
904 JUMP_COND_MATH_Z);
905
906 append_math_add(desc, VARSEQINLEN, ZERO, REG1, CAAM_CMD_SZ);
907 /* if asoclen is ZERO, skip reading assoc data */
908 zero_assoc_jump_cmd1 = append_jump(desc, JUMP_TEST_ALL |
909 JUMP_COND_MATH_Z);
910 /* read assoc data */
911 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
912 FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1);
913 set_jump_tgt_here(desc, zero_assoc_jump_cmd1);
914
915 append_math_add(desc, VARSEQINLEN, ZERO, REG2, CAAM_CMD_SZ);
916
917 /* store encrypted data */
918 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF);
919
920 /* read payload data */
921 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
922 FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1);
923
924 /* jump the zero-payload commands */
925 append_jump(desc, JUMP_TEST_ALL | 4);
926
927 /* zero-payload command */
928 set_jump_tgt_here(desc, zero_payload_jump_cmd);
929
930 /* if assoclen is ZERO, jump to ICV reading */
931 append_math_add(desc, VARSEQINLEN, ZERO, REG1, CAAM_CMD_SZ);
932 zero_assoc_jump_cmd2 = append_jump(desc, JUMP_TEST_ALL |
933 JUMP_COND_MATH_Z);
934 /* read assoc data */
935 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
936 FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1);
937 set_jump_tgt_here(desc, zero_assoc_jump_cmd2);
938
939 /* read ICV */
940 append_seq_fifo_load(desc, ctx->authsize, FIFOLD_CLASS_CLASS1 |
941 FIFOLD_TYPE_ICV | FIFOLD_TYPE_LAST1);
942
943 ctx->sh_desc_dec_dma = dma_map_single(jrdev, desc,
944 desc_bytes(desc),
945 DMA_TO_DEVICE);
946 if (dma_mapping_error(jrdev, ctx->sh_desc_dec_dma)) {
947 dev_err(jrdev, "unable to map shared descriptor\n");
948 return -ENOMEM;
949 }
950#ifdef DEBUG
951 print_hex_dump(KERN_ERR, "gcm dec shdesc@"__stringify(__LINE__)": ",
952 DUMP_PREFIX_ADDRESS, 16, 4, desc,
953 desc_bytes(desc), 1);
954#endif
955
956 return 0;
957}
958
959static int gcm_setauthsize(struct crypto_aead *authenc, unsigned int authsize)
960{
961 struct caam_ctx *ctx = crypto_aead_ctx(authenc);
962
963 ctx->authsize = authsize;
964 gcm_set_sh_desc(authenc);
965
966 return 0;
967}
968
Tudor Ambarusbac68f22014-10-23 16:14:03 +0300969static int rfc4106_set_sh_desc(struct crypto_aead *aead)
970{
971 struct aead_tfm *tfm = &aead->base.crt_aead;
972 struct caam_ctx *ctx = crypto_aead_ctx(aead);
973 struct device *jrdev = ctx->jrdev;
974 bool keys_fit_inline = false;
975 u32 *key_jump_cmd, *move_cmd, *write_iv_cmd;
976 u32 *desc;
977 u32 geniv;
978
979 if (!ctx->enckeylen || !ctx->authsize)
980 return 0;
981
982 /*
983 * RFC4106 encrypt shared descriptor
984 * Job Descriptor and Shared Descriptor
985 * must fit into the 64-word Descriptor h/w Buffer
986 */
987 if (DESC_RFC4106_ENC_LEN + DESC_JOB_IO_LEN +
988 ctx->enckeylen <= CAAM_DESC_BYTES_MAX)
989 keys_fit_inline = true;
990
991 desc = ctx->sh_desc_enc;
992
993 init_sh_desc(desc, HDR_SHARE_SERIAL);
994
995 /* Skip key loading if it is loaded due to sharing */
996 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
997 JUMP_COND_SHRD);
998 if (keys_fit_inline)
999 append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen,
1000 ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG);
1001 else
1002 append_key(desc, ctx->key_dma, ctx->enckeylen,
1003 CLASS_1 | KEY_DEST_CLASS_REG);
1004 set_jump_tgt_here(desc, key_jump_cmd);
1005
1006 /* Class 1 operation */
1007 append_operation(desc, ctx->class1_alg_type |
1008 OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT);
1009
1010 /* cryptlen = seqoutlen - authsize */
1011 append_math_sub_imm_u32(desc, REG3, SEQOUTLEN, IMM, ctx->authsize);
1012 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
1013
1014 /* assoclen + cryptlen = seqinlen - ivsize */
1015 append_math_sub_imm_u32(desc, REG2, SEQINLEN, IMM, tfm->ivsize);
1016
1017 /* assoclen = (assoclen + cryptlen) - cryptlen */
1018 append_math_sub(desc, VARSEQINLEN, REG2, REG3, CAAM_CMD_SZ);
1019
1020 /* Read Salt */
1021 append_fifo_load_as_imm(desc, (void *)(ctx->key + ctx->enckeylen),
1022 4, FIFOLD_CLASS_CLASS1 | FIFOLD_TYPE_IV);
1023 /* Read AES-GCM-ESP IV */
1024 append_seq_fifo_load(desc, tfm->ivsize, FIFOLD_CLASS_CLASS1 |
1025 FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1);
1026
1027 /* Read assoc data */
1028 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
1029 FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1);
1030
1031 /* Will read cryptlen bytes */
1032 append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
1033
1034 /* Write encrypted data */
1035 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF);
1036
1037 /* Read payload data */
1038 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
1039 FIFOLD_TYPE_MSG | FIFOLD_TYPE_LAST1);
1040
1041 /* Write ICV */
1042 append_seq_store(desc, ctx->authsize, LDST_CLASS_1_CCB |
1043 LDST_SRCDST_BYTE_CONTEXT);
1044
1045 ctx->sh_desc_enc_dma = dma_map_single(jrdev, desc,
1046 desc_bytes(desc),
1047 DMA_TO_DEVICE);
1048 if (dma_mapping_error(jrdev, ctx->sh_desc_enc_dma)) {
1049 dev_err(jrdev, "unable to map shared descriptor\n");
1050 return -ENOMEM;
1051 }
1052#ifdef DEBUG
1053 print_hex_dump(KERN_ERR, "rfc4106 enc shdesc@"__stringify(__LINE__)": ",
1054 DUMP_PREFIX_ADDRESS, 16, 4, desc,
1055 desc_bytes(desc), 1);
1056#endif
1057
1058 /*
1059 * Job Descriptor and Shared Descriptors
1060 * must all fit into the 64-word Descriptor h/w Buffer
1061 */
1062 keys_fit_inline = false;
1063 if (DESC_RFC4106_DEC_LEN + DESC_JOB_IO_LEN +
1064 ctx->enckeylen <= CAAM_DESC_BYTES_MAX)
1065 keys_fit_inline = true;
1066
1067 desc = ctx->sh_desc_dec;
1068
1069 init_sh_desc(desc, HDR_SHARE_SERIAL);
1070
1071 /* Skip key loading if it is loaded due to sharing */
1072 key_jump_cmd = append_jump(desc, JUMP_JSL |
1073 JUMP_TEST_ALL | JUMP_COND_SHRD);
1074 if (keys_fit_inline)
1075 append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen,
1076 ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG);
1077 else
1078 append_key(desc, ctx->key_dma, ctx->enckeylen,
1079 CLASS_1 | KEY_DEST_CLASS_REG);
1080 set_jump_tgt_here(desc, key_jump_cmd);
1081
1082 /* Class 1 operation */
1083 append_operation(desc, ctx->class1_alg_type |
1084 OP_ALG_AS_INITFINAL | OP_ALG_DECRYPT | OP_ALG_ICV_ON);
1085
1086 /* assoclen + cryptlen = seqinlen - ivsize - icvsize */
1087 append_math_sub_imm_u32(desc, REG3, SEQINLEN, IMM,
1088 ctx->authsize + tfm->ivsize);
1089
1090 /* assoclen = (assoclen + cryptlen) - cryptlen */
1091 append_math_sub(desc, REG2, SEQOUTLEN, REG0, CAAM_CMD_SZ);
1092 append_math_sub(desc, VARSEQINLEN, REG3, REG2, CAAM_CMD_SZ);
1093
1094 /* Will write cryptlen bytes */
1095 append_math_sub(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ);
1096
1097 /* Read Salt */
1098 append_fifo_load_as_imm(desc, (void *)(ctx->key + ctx->enckeylen),
1099 4, FIFOLD_CLASS_CLASS1 | FIFOLD_TYPE_IV);
1100 /* Read AES-GCM-ESP IV */
1101 append_seq_fifo_load(desc, tfm->ivsize, FIFOLD_CLASS_CLASS1 |
1102 FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1);
1103
1104 /* Read assoc data */
1105 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
1106 FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1);
1107
1108 /* Will read cryptlen bytes */
1109 append_math_add(desc, VARSEQINLEN, ZERO, REG2, CAAM_CMD_SZ);
1110
1111 /* Store payload data */
1112 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF);
1113
1114 /* Read encrypted data */
1115 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
1116 FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1);
1117
1118 /* Read ICV */
1119 append_seq_fifo_load(desc, ctx->authsize, FIFOLD_CLASS_CLASS1 |
1120 FIFOLD_TYPE_ICV | FIFOLD_TYPE_LAST1);
1121
1122 ctx->sh_desc_dec_dma = dma_map_single(jrdev, desc,
1123 desc_bytes(desc),
1124 DMA_TO_DEVICE);
1125 if (dma_mapping_error(jrdev, ctx->sh_desc_dec_dma)) {
1126 dev_err(jrdev, "unable to map shared descriptor\n");
1127 return -ENOMEM;
1128 }
1129#ifdef DEBUG
1130 print_hex_dump(KERN_ERR, "rfc4106 dec shdesc@"__stringify(__LINE__)": ",
1131 DUMP_PREFIX_ADDRESS, 16, 4, desc,
1132 desc_bytes(desc), 1);
1133#endif
1134
1135 /*
1136 * Job Descriptor and Shared Descriptors
1137 * must all fit into the 64-word Descriptor h/w Buffer
1138 */
1139 keys_fit_inline = false;
1140 if (DESC_RFC4106_GIVENC_LEN + DESC_JOB_IO_LEN +
1141 ctx->split_key_pad_len + ctx->enckeylen <=
1142 CAAM_DESC_BYTES_MAX)
1143 keys_fit_inline = true;
1144
1145 /* rfc4106_givencrypt shared descriptor */
1146 desc = ctx->sh_desc_givenc;
1147
1148 init_sh_desc(desc, HDR_SHARE_SERIAL);
1149
1150 /* Skip key loading if it is loaded due to sharing */
1151 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
1152 JUMP_COND_SHRD);
1153 if (keys_fit_inline)
1154 append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen,
1155 ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG);
1156 else
1157 append_key(desc, ctx->key_dma, ctx->enckeylen,
1158 CLASS_1 | KEY_DEST_CLASS_REG);
1159 set_jump_tgt_here(desc, key_jump_cmd);
1160
1161 /* Generate IV */
1162 geniv = NFIFOENTRY_STYPE_PAD | NFIFOENTRY_DEST_DECO |
1163 NFIFOENTRY_DTYPE_MSG | NFIFOENTRY_LC1 |
1164 NFIFOENTRY_PTYPE_RND | (tfm->ivsize << NFIFOENTRY_DLEN_SHIFT);
1165 append_load_imm_u32(desc, geniv, LDST_CLASS_IND_CCB |
1166 LDST_SRCDST_WORD_INFO_FIFO | LDST_IMM);
1167 append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO);
1168 move_cmd = append_move(desc, MOVE_SRC_INFIFO | MOVE_DEST_DESCBUF |
1169 (tfm->ivsize << MOVE_LEN_SHIFT));
1170 append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO);
1171
1172 /* Copy generated IV to OFIFO */
1173 write_iv_cmd = append_move(desc, MOVE_SRC_DESCBUF | MOVE_DEST_OUTFIFO |
1174 (tfm->ivsize << MOVE_LEN_SHIFT));
1175
1176 /* Class 1 operation */
1177 append_operation(desc, ctx->class1_alg_type |
1178 OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT);
1179
1180 /* ivsize + cryptlen = seqoutlen - authsize */
1181 append_math_sub_imm_u32(desc, REG3, SEQOUTLEN, IMM, ctx->authsize);
1182
1183 /* assoclen = seqinlen - (ivsize + cryptlen) */
1184 append_math_sub(desc, VARSEQINLEN, SEQINLEN, REG3, CAAM_CMD_SZ);
1185
1186 /* Will write ivsize + cryptlen */
1187 append_math_add(desc, VARSEQOUTLEN, REG3, REG0, CAAM_CMD_SZ);
1188
1189 /* Read Salt and generated IV */
1190 append_cmd(desc, CMD_FIFO_LOAD | FIFOLD_CLASS_CLASS1 | FIFOLD_TYPE_IV |
1191 FIFOLD_TYPE_FLUSH1 | IMMEDIATE | 12);
1192 /* Append Salt */
1193 append_data(desc, (void *)(ctx->key + ctx->enckeylen), 4);
1194 set_move_tgt_here(desc, move_cmd);
1195 set_move_tgt_here(desc, write_iv_cmd);
1196 /* Blank commands. Will be overwritten by generated IV. */
1197 append_cmd(desc, 0x00000000);
1198 append_cmd(desc, 0x00000000);
1199 /* End of blank commands */
1200
1201 /* No need to reload iv */
1202 append_seq_fifo_load(desc, tfm->ivsize, FIFOLD_CLASS_SKIP);
1203
1204 /* Read assoc data */
1205 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
1206 FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1);
1207
1208 /* Will read cryptlen */
1209 append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
1210
1211 /* Store generated IV and encrypted data */
1212 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF);
1213
1214 /* Read payload data */
1215 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
1216 FIFOLD_TYPE_MSG | FIFOLD_TYPE_LAST1);
1217
1218 /* Write ICV */
1219 append_seq_store(desc, ctx->authsize, LDST_CLASS_1_CCB |
1220 LDST_SRCDST_BYTE_CONTEXT);
1221
1222 ctx->sh_desc_givenc_dma = dma_map_single(jrdev, desc,
1223 desc_bytes(desc),
1224 DMA_TO_DEVICE);
1225 if (dma_mapping_error(jrdev, ctx->sh_desc_givenc_dma)) {
1226 dev_err(jrdev, "unable to map shared descriptor\n");
1227 return -ENOMEM;
1228 }
1229#ifdef DEBUG
1230 print_hex_dump(KERN_ERR,
1231 "rfc4106 givenc shdesc@"__stringify(__LINE__)": ",
1232 DUMP_PREFIX_ADDRESS, 16, 4, desc,
1233 desc_bytes(desc), 1);
1234#endif
1235
1236 return 0;
1237}
1238
1239static int rfc4106_setauthsize(struct crypto_aead *authenc,
1240 unsigned int authsize)
1241{
1242 struct caam_ctx *ctx = crypto_aead_ctx(authenc);
1243
1244 ctx->authsize = authsize;
1245 rfc4106_set_sh_desc(authenc);
1246
1247 return 0;
1248}
1249
Tudor Ambarus5d0429a2014-10-30 18:55:07 +02001250static int rfc4543_set_sh_desc(struct crypto_aead *aead)
1251{
1252 struct aead_tfm *tfm = &aead->base.crt_aead;
1253 struct caam_ctx *ctx = crypto_aead_ctx(aead);
1254 struct device *jrdev = ctx->jrdev;
1255 bool keys_fit_inline = false;
1256 u32 *key_jump_cmd, *write_iv_cmd, *write_aad_cmd;
1257 u32 *read_move_cmd, *write_move_cmd;
1258 u32 *desc;
1259 u32 geniv;
1260
1261 if (!ctx->enckeylen || !ctx->authsize)
1262 return 0;
1263
1264 /*
1265 * RFC4543 encrypt shared descriptor
1266 * Job Descriptor and Shared Descriptor
1267 * must fit into the 64-word Descriptor h/w Buffer
1268 */
1269 if (DESC_RFC4543_ENC_LEN + DESC_JOB_IO_LEN +
1270 ctx->enckeylen <= CAAM_DESC_BYTES_MAX)
1271 keys_fit_inline = true;
1272
1273 desc = ctx->sh_desc_enc;
1274
1275 init_sh_desc(desc, HDR_SHARE_SERIAL);
1276
1277 /* Skip key loading if it is loaded due to sharing */
1278 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
1279 JUMP_COND_SHRD);
1280 if (keys_fit_inline)
1281 append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen,
1282 ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG);
1283 else
1284 append_key(desc, ctx->key_dma, ctx->enckeylen,
1285 CLASS_1 | KEY_DEST_CLASS_REG);
1286 set_jump_tgt_here(desc, key_jump_cmd);
1287
1288 /* Class 1 operation */
1289 append_operation(desc, ctx->class1_alg_type |
1290 OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT);
1291
1292 /* Load AES-GMAC ESP IV into Math1 register */
1293 append_cmd(desc, CMD_SEQ_LOAD | LDST_SRCDST_WORD_DECO_MATH1 |
1294 LDST_CLASS_DECO | tfm->ivsize);
1295
1296 /* Wait the DMA transaction to finish */
1297 append_jump(desc, JUMP_TEST_ALL | JUMP_COND_CALM |
1298 (1 << JUMP_OFFSET_SHIFT));
1299
1300 /* Overwrite blank immediate AES-GMAC ESP IV data */
1301 write_iv_cmd = append_move(desc, MOVE_SRC_MATH1 | MOVE_DEST_DESCBUF |
1302 (tfm->ivsize << MOVE_LEN_SHIFT));
1303
1304 /* Overwrite blank immediate AAD data */
1305 write_aad_cmd = append_move(desc, MOVE_SRC_MATH1 | MOVE_DEST_DESCBUF |
1306 (tfm->ivsize << MOVE_LEN_SHIFT));
1307
1308 /* cryptlen = seqoutlen - authsize */
1309 append_math_sub_imm_u32(desc, REG3, SEQOUTLEN, IMM, ctx->authsize);
1310
1311 /* assoclen = (seqinlen - ivsize) - cryptlen */
1312 append_math_sub(desc, VARSEQINLEN, SEQINLEN, REG3, CAAM_CMD_SZ);
1313
1314 /* Read Salt and AES-GMAC ESP IV */
1315 append_cmd(desc, CMD_FIFO_LOAD | FIFOLD_CLASS_CLASS1 | IMMEDIATE |
1316 FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1 | (4 + tfm->ivsize));
1317 /* Append Salt */
1318 append_data(desc, (void *)(ctx->key + ctx->enckeylen), 4);
1319 set_move_tgt_here(desc, write_iv_cmd);
1320 /* Blank commands. Will be overwritten by AES-GMAC ESP IV. */
1321 append_cmd(desc, 0x00000000);
1322 append_cmd(desc, 0x00000000);
1323 /* End of blank commands */
1324
1325 /* Read assoc data */
1326 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
1327 FIFOLD_TYPE_AAD);
1328
1329 /* Will read cryptlen bytes */
1330 append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
1331
1332 /* Will write cryptlen bytes */
1333 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
1334
1335 /*
1336 * MOVE_LEN opcode is not available in all SEC HW revisions,
1337 * thus need to do some magic, i.e. self-patch the descriptor
1338 * buffer.
1339 */
1340 read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF | MOVE_DEST_MATH3 |
1341 (0x6 << MOVE_LEN_SHIFT));
1342 write_move_cmd = append_move(desc, MOVE_SRC_MATH3 | MOVE_DEST_DESCBUF |
1343 (0x8 << MOVE_LEN_SHIFT));
1344
1345 /* Authenticate AES-GMAC ESP IV */
1346 append_cmd(desc, CMD_FIFO_LOAD | FIFOLD_CLASS_CLASS1 | IMMEDIATE |
1347 FIFOLD_TYPE_AAD | tfm->ivsize);
1348 set_move_tgt_here(desc, write_aad_cmd);
1349 /* Blank commands. Will be overwritten by AES-GMAC ESP IV. */
1350 append_cmd(desc, 0x00000000);
1351 append_cmd(desc, 0x00000000);
1352 /* End of blank commands */
1353
1354 /* Read and write cryptlen bytes */
1355 aead_append_src_dst(desc, FIFOLD_TYPE_AAD);
1356
1357 set_move_tgt_here(desc, read_move_cmd);
1358 set_move_tgt_here(desc, write_move_cmd);
1359 append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO);
1360 /* Move payload data to OFIFO */
1361 append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO);
1362
1363 /* Write ICV */
1364 append_seq_store(desc, ctx->authsize, LDST_CLASS_1_CCB |
1365 LDST_SRCDST_BYTE_CONTEXT);
1366
1367 ctx->sh_desc_enc_dma = dma_map_single(jrdev, desc,
1368 desc_bytes(desc),
1369 DMA_TO_DEVICE);
1370 if (dma_mapping_error(jrdev, ctx->sh_desc_enc_dma)) {
1371 dev_err(jrdev, "unable to map shared descriptor\n");
1372 return -ENOMEM;
1373 }
1374#ifdef DEBUG
1375 print_hex_dump(KERN_ERR, "rfc4543 enc shdesc@"__stringify(__LINE__)": ",
1376 DUMP_PREFIX_ADDRESS, 16, 4, desc,
1377 desc_bytes(desc), 1);
1378#endif
1379
1380 /*
1381 * Job Descriptor and Shared Descriptors
1382 * must all fit into the 64-word Descriptor h/w Buffer
1383 */
1384 keys_fit_inline = false;
1385 if (DESC_RFC4543_DEC_LEN + DESC_JOB_IO_LEN +
1386 ctx->enckeylen <= CAAM_DESC_BYTES_MAX)
1387 keys_fit_inline = true;
1388
1389 desc = ctx->sh_desc_dec;
1390
1391 init_sh_desc(desc, HDR_SHARE_SERIAL);
1392
1393 /* Skip key loading if it is loaded due to sharing */
1394 key_jump_cmd = append_jump(desc, JUMP_JSL |
1395 JUMP_TEST_ALL | JUMP_COND_SHRD);
1396 if (keys_fit_inline)
1397 append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen,
1398 ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG);
1399 else
1400 append_key(desc, ctx->key_dma, ctx->enckeylen,
1401 CLASS_1 | KEY_DEST_CLASS_REG);
1402 set_jump_tgt_here(desc, key_jump_cmd);
1403
1404 /* Class 1 operation */
1405 append_operation(desc, ctx->class1_alg_type |
1406 OP_ALG_AS_INITFINAL | OP_ALG_DECRYPT | OP_ALG_ICV_ON);
1407
1408 /* Load AES-GMAC ESP IV into Math1 register */
1409 append_cmd(desc, CMD_SEQ_LOAD | LDST_SRCDST_WORD_DECO_MATH1 |
1410 LDST_CLASS_DECO | tfm->ivsize);
1411
1412 /* Wait the DMA transaction to finish */
1413 append_jump(desc, JUMP_TEST_ALL | JUMP_COND_CALM |
1414 (1 << JUMP_OFFSET_SHIFT));
1415
1416 /* assoclen + cryptlen = (seqinlen - ivsize) - icvsize */
1417 append_math_sub_imm_u32(desc, REG3, SEQINLEN, IMM, ctx->authsize);
1418
1419 /* Overwrite blank immediate AES-GMAC ESP IV data */
1420 write_iv_cmd = append_move(desc, MOVE_SRC_MATH1 | MOVE_DEST_DESCBUF |
1421 (tfm->ivsize << MOVE_LEN_SHIFT));
1422
1423 /* Overwrite blank immediate AAD data */
1424 write_aad_cmd = append_move(desc, MOVE_SRC_MATH1 | MOVE_DEST_DESCBUF |
1425 (tfm->ivsize << MOVE_LEN_SHIFT));
1426
1427 /* assoclen = (assoclen + cryptlen) - cryptlen */
1428 append_math_sub(desc, REG2, SEQOUTLEN, REG0, CAAM_CMD_SZ);
1429 append_math_sub(desc, VARSEQINLEN, REG3, REG2, CAAM_CMD_SZ);
1430
1431 /*
1432 * MOVE_LEN opcode is not available in all SEC HW revisions,
1433 * thus need to do some magic, i.e. self-patch the descriptor
1434 * buffer.
1435 */
1436 read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF | MOVE_DEST_MATH3 |
1437 (0x6 << MOVE_LEN_SHIFT));
1438 write_move_cmd = append_move(desc, MOVE_SRC_MATH3 | MOVE_DEST_DESCBUF |
1439 (0x8 << MOVE_LEN_SHIFT));
1440
1441 /* Read Salt and AES-GMAC ESP IV */
1442 append_cmd(desc, CMD_FIFO_LOAD | FIFOLD_CLASS_CLASS1 | IMMEDIATE |
1443 FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1 | (4 + tfm->ivsize));
1444 /* Append Salt */
1445 append_data(desc, (void *)(ctx->key + ctx->enckeylen), 4);
1446 set_move_tgt_here(desc, write_iv_cmd);
1447 /* Blank commands. Will be overwritten by AES-GMAC ESP IV. */
1448 append_cmd(desc, 0x00000000);
1449 append_cmd(desc, 0x00000000);
1450 /* End of blank commands */
1451
1452 /* Read assoc data */
1453 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
1454 FIFOLD_TYPE_AAD);
1455
1456 /* Will read cryptlen bytes */
1457 append_math_add(desc, VARSEQINLEN, ZERO, REG2, CAAM_CMD_SZ);
1458
1459 /* Will write cryptlen bytes */
1460 append_math_add(desc, VARSEQOUTLEN, ZERO, REG2, CAAM_CMD_SZ);
1461
1462 /* Authenticate AES-GMAC ESP IV */
1463 append_cmd(desc, CMD_FIFO_LOAD | FIFOLD_CLASS_CLASS1 | IMMEDIATE |
1464 FIFOLD_TYPE_AAD | tfm->ivsize);
1465 set_move_tgt_here(desc, write_aad_cmd);
1466 /* Blank commands. Will be overwritten by AES-GMAC ESP IV. */
1467 append_cmd(desc, 0x00000000);
1468 append_cmd(desc, 0x00000000);
1469 /* End of blank commands */
1470
1471 /* Store payload data */
1472 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF);
1473
1474 /* In-snoop cryptlen data */
1475 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_BOTH | FIFOLDST_VLF |
1476 FIFOLD_TYPE_AAD | FIFOLD_TYPE_LAST2FLUSH1);
1477
1478 set_move_tgt_here(desc, read_move_cmd);
1479 set_move_tgt_here(desc, write_move_cmd);
1480 append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO);
1481 /* Move payload data to OFIFO */
1482 append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO);
1483 append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO);
1484
1485 /* Read ICV */
1486 append_seq_fifo_load(desc, ctx->authsize, FIFOLD_CLASS_CLASS1 |
1487 FIFOLD_TYPE_ICV | FIFOLD_TYPE_LAST1);
1488
1489 ctx->sh_desc_dec_dma = dma_map_single(jrdev, desc,
1490 desc_bytes(desc),
1491 DMA_TO_DEVICE);
1492 if (dma_mapping_error(jrdev, ctx->sh_desc_dec_dma)) {
1493 dev_err(jrdev, "unable to map shared descriptor\n");
1494 return -ENOMEM;
1495 }
1496#ifdef DEBUG
1497 print_hex_dump(KERN_ERR, "rfc4543 dec shdesc@"__stringify(__LINE__)": ",
1498 DUMP_PREFIX_ADDRESS, 16, 4, desc,
1499 desc_bytes(desc), 1);
1500#endif
1501
1502 /*
1503 * Job Descriptor and Shared Descriptors
1504 * must all fit into the 64-word Descriptor h/w Buffer
1505 */
1506 keys_fit_inline = false;
1507 if (DESC_RFC4543_GIVENC_LEN + DESC_JOB_IO_LEN +
1508 ctx->enckeylen <= CAAM_DESC_BYTES_MAX)
1509 keys_fit_inline = true;
1510
1511 /* rfc4543_givencrypt shared descriptor */
1512 desc = ctx->sh_desc_givenc;
1513
1514 init_sh_desc(desc, HDR_SHARE_SERIAL);
1515
1516 /* Skip key loading if it is loaded due to sharing */
1517 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
1518 JUMP_COND_SHRD);
1519 if (keys_fit_inline)
1520 append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen,
1521 ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG);
1522 else
1523 append_key(desc, ctx->key_dma, ctx->enckeylen,
1524 CLASS_1 | KEY_DEST_CLASS_REG);
1525 set_jump_tgt_here(desc, key_jump_cmd);
1526
1527 /* Generate IV */
1528 geniv = NFIFOENTRY_STYPE_PAD | NFIFOENTRY_DEST_DECO |
1529 NFIFOENTRY_DTYPE_MSG | NFIFOENTRY_LC1 |
1530 NFIFOENTRY_PTYPE_RND | (tfm->ivsize << NFIFOENTRY_DLEN_SHIFT);
1531 append_load_imm_u32(desc, geniv, LDST_CLASS_IND_CCB |
1532 LDST_SRCDST_WORD_INFO_FIFO | LDST_IMM);
1533 append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO);
1534 /* Move generated IV to Math1 register */
1535 append_move(desc, MOVE_SRC_INFIFO | MOVE_DEST_MATH1 |
1536 (tfm->ivsize << MOVE_LEN_SHIFT));
1537 append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO);
1538
1539 /* Overwrite blank immediate AES-GMAC IV data */
1540 write_iv_cmd = append_move(desc, MOVE_SRC_MATH1 | MOVE_DEST_DESCBUF |
1541 (tfm->ivsize << MOVE_LEN_SHIFT));
1542
1543 /* Overwrite blank immediate AAD data */
1544 write_aad_cmd = append_move(desc, MOVE_SRC_MATH1 | MOVE_DEST_DESCBUF |
1545 (tfm->ivsize << MOVE_LEN_SHIFT));
1546
1547 /* Copy generated IV to OFIFO */
1548 append_move(desc, MOVE_SRC_MATH1 | MOVE_DEST_OUTFIFO |
1549 (tfm->ivsize << MOVE_LEN_SHIFT));
1550
1551 /* Class 1 operation */
1552 append_operation(desc, ctx->class1_alg_type |
1553 OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT);
1554
1555 /* ivsize + cryptlen = seqoutlen - authsize */
1556 append_math_sub_imm_u32(desc, REG3, SEQOUTLEN, IMM, ctx->authsize);
1557
1558 /* assoclen = seqinlen - (ivsize + cryptlen) */
1559 append_math_sub(desc, VARSEQINLEN, SEQINLEN, REG3, CAAM_CMD_SZ);
1560
1561 /* Will write ivsize + cryptlen */
1562 append_math_add(desc, VARSEQOUTLEN, REG3, REG0, CAAM_CMD_SZ);
1563
1564 /*
1565 * MOVE_LEN opcode is not available in all SEC HW revisions,
1566 * thus need to do some magic, i.e. self-patch the descriptor
1567 * buffer.
1568 */
1569 read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF | MOVE_DEST_MATH3 |
1570 (0x6 << MOVE_LEN_SHIFT));
1571 write_move_cmd = append_move(desc, MOVE_SRC_MATH3 | MOVE_DEST_DESCBUF |
1572 (0x8 << MOVE_LEN_SHIFT));
1573
1574 /* Read Salt and AES-GMAC generated IV */
1575 append_cmd(desc, CMD_FIFO_LOAD | FIFOLD_CLASS_CLASS1 | IMMEDIATE |
1576 FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1 | (4 + tfm->ivsize));
1577 /* Append Salt */
1578 append_data(desc, (void *)(ctx->key + ctx->enckeylen), 4);
1579 set_move_tgt_here(desc, write_iv_cmd);
1580 /* Blank commands. Will be overwritten by AES-GMAC generated IV. */
1581 append_cmd(desc, 0x00000000);
1582 append_cmd(desc, 0x00000000);
1583 /* End of blank commands */
1584
1585 /* No need to reload iv */
1586 append_seq_fifo_load(desc, tfm->ivsize, FIFOLD_CLASS_SKIP);
1587
1588 /* Read assoc data */
1589 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
1590 FIFOLD_TYPE_AAD);
1591
1592 /* Will read cryptlen */
1593 append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
1594
1595 /* Authenticate AES-GMAC IV */
1596 append_cmd(desc, CMD_FIFO_LOAD | FIFOLD_CLASS_CLASS1 | IMMEDIATE |
1597 FIFOLD_TYPE_AAD | tfm->ivsize);
1598 set_move_tgt_here(desc, write_aad_cmd);
1599 /* Blank commands. Will be overwritten by AES-GMAC IV. */
1600 append_cmd(desc, 0x00000000);
1601 append_cmd(desc, 0x00000000);
1602 /* End of blank commands */
1603
1604 /* Read and write cryptlen bytes */
1605 aead_append_src_dst(desc, FIFOLD_TYPE_AAD);
1606
1607 set_move_tgt_here(desc, read_move_cmd);
1608 set_move_tgt_here(desc, write_move_cmd);
1609 append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO);
1610 /* Move payload data to OFIFO */
1611 append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO);
1612
1613 /* Write ICV */
1614 append_seq_store(desc, ctx->authsize, LDST_CLASS_1_CCB |
1615 LDST_SRCDST_BYTE_CONTEXT);
1616
1617 ctx->sh_desc_givenc_dma = dma_map_single(jrdev, desc,
1618 desc_bytes(desc),
1619 DMA_TO_DEVICE);
1620 if (dma_mapping_error(jrdev, ctx->sh_desc_givenc_dma)) {
1621 dev_err(jrdev, "unable to map shared descriptor\n");
1622 return -ENOMEM;
1623 }
1624#ifdef DEBUG
1625 print_hex_dump(KERN_ERR,
1626 "rfc4543 givenc shdesc@"__stringify(__LINE__)": ",
1627 DUMP_PREFIX_ADDRESS, 16, 4, desc,
1628 desc_bytes(desc), 1);
1629#endif
1630
1631 return 0;
1632}
1633
1634static int rfc4543_setauthsize(struct crypto_aead *authenc,
1635 unsigned int authsize)
1636{
1637 struct caam_ctx *ctx = crypto_aead_ctx(authenc);
1638
1639 ctx->authsize = authsize;
1640 rfc4543_set_sh_desc(authenc);
1641
1642 return 0;
1643}
1644
Yuan Kang4c1ec1f2012-06-22 19:48:45 -05001645static u32 gen_split_aead_key(struct caam_ctx *ctx, const u8 *key_in,
1646 u32 authkeylen)
Kim Phillips8e8ec592011-03-13 16:54:26 +08001647{
Yuan Kang4c1ec1f2012-06-22 19:48:45 -05001648 return gen_split_key(ctx->jrdev, ctx->key, ctx->split_key_len,
1649 ctx->split_key_pad_len, key_in, authkeylen,
1650 ctx->alg_op);
Kim Phillips8e8ec592011-03-13 16:54:26 +08001651}
1652
Yuan Kang0e479302011-07-15 11:21:41 +08001653static int aead_setkey(struct crypto_aead *aead,
Kim Phillips8e8ec592011-03-13 16:54:26 +08001654 const u8 *key, unsigned int keylen)
1655{
1656 /* Sizes for MDHA pads (*not* keys): MD5, SHA1, 224, 256, 384, 512 */
1657 static const u8 mdpadlen[] = { 16, 20, 32, 32, 64, 64 };
1658 struct caam_ctx *ctx = crypto_aead_ctx(aead);
1659 struct device *jrdev = ctx->jrdev;
Horia Geanta4e6e0b22013-12-19 17:27:35 +02001660 struct crypto_authenc_keys keys;
Kim Phillips8e8ec592011-03-13 16:54:26 +08001661 int ret = 0;
1662
Horia Geanta4e6e0b22013-12-19 17:27:35 +02001663 if (crypto_authenc_extractkeys(&keys, key, keylen) != 0)
Kim Phillips8e8ec592011-03-13 16:54:26 +08001664 goto badkey;
1665
1666 /* Pick class 2 key length from algorithm submask */
1667 ctx->split_key_len = mdpadlen[(ctx->alg_op & OP_ALG_ALGSEL_SUBMASK) >>
1668 OP_ALG_ALGSEL_SHIFT] * 2;
1669 ctx->split_key_pad_len = ALIGN(ctx->split_key_len, 16);
1670
Horia Geanta4e6e0b22013-12-19 17:27:35 +02001671 if (ctx->split_key_pad_len + keys.enckeylen > CAAM_MAX_KEY_SIZE)
1672 goto badkey;
1673
Kim Phillips8e8ec592011-03-13 16:54:26 +08001674#ifdef DEBUG
1675 printk(KERN_ERR "keylen %d enckeylen %d authkeylen %d\n",
Horia Geanta4e6e0b22013-12-19 17:27:35 +02001676 keys.authkeylen + keys.enckeylen, keys.enckeylen,
1677 keys.authkeylen);
Kim Phillips8e8ec592011-03-13 16:54:26 +08001678 printk(KERN_ERR "split_key_len %d split_key_pad_len %d\n",
1679 ctx->split_key_len, ctx->split_key_pad_len);
Alex Porosanu514df282013-08-14 18:56:45 +03001680 print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ",
Kim Phillips8e8ec592011-03-13 16:54:26 +08001681 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
1682#endif
Kim Phillips8e8ec592011-03-13 16:54:26 +08001683
Horia Geanta4e6e0b22013-12-19 17:27:35 +02001684 ret = gen_split_aead_key(ctx, keys.authkey, keys.authkeylen);
Kim Phillips8e8ec592011-03-13 16:54:26 +08001685 if (ret) {
Kim Phillips8e8ec592011-03-13 16:54:26 +08001686 goto badkey;
1687 }
1688
1689 /* postpend encryption key to auth split key */
Horia Geanta4e6e0b22013-12-19 17:27:35 +02001690 memcpy(ctx->key + ctx->split_key_pad_len, keys.enckey, keys.enckeylen);
Kim Phillips8e8ec592011-03-13 16:54:26 +08001691
Yuan Kang885e9e22011-07-15 11:21:41 +08001692 ctx->key_dma = dma_map_single(jrdev, ctx->key, ctx->split_key_pad_len +
Horia Geanta4e6e0b22013-12-19 17:27:35 +02001693 keys.enckeylen, DMA_TO_DEVICE);
Yuan Kang885e9e22011-07-15 11:21:41 +08001694 if (dma_mapping_error(jrdev, ctx->key_dma)) {
Kim Phillips8e8ec592011-03-13 16:54:26 +08001695 dev_err(jrdev, "unable to map key i/o memory\n");
Kim Phillips8e8ec592011-03-13 16:54:26 +08001696 return -ENOMEM;
1697 }
1698#ifdef DEBUG
Alex Porosanu514df282013-08-14 18:56:45 +03001699 print_hex_dump(KERN_ERR, "ctx.key@"__stringify(__LINE__)": ",
Kim Phillips8e8ec592011-03-13 16:54:26 +08001700 DUMP_PREFIX_ADDRESS, 16, 4, ctx->key,
Horia Geanta4e6e0b22013-12-19 17:27:35 +02001701 ctx->split_key_pad_len + keys.enckeylen, 1);
Kim Phillips8e8ec592011-03-13 16:54:26 +08001702#endif
1703
Horia Geanta4e6e0b22013-12-19 17:27:35 +02001704 ctx->enckeylen = keys.enckeylen;
Kim Phillips8e8ec592011-03-13 16:54:26 +08001705
Yuan Kang1acebad2011-07-15 11:21:42 +08001706 ret = aead_set_sh_desc(aead);
Kim Phillips8e8ec592011-03-13 16:54:26 +08001707 if (ret) {
Yuan Kang885e9e22011-07-15 11:21:41 +08001708 dma_unmap_single(jrdev, ctx->key_dma, ctx->split_key_pad_len +
Horia Geanta4e6e0b22013-12-19 17:27:35 +02001709 keys.enckeylen, DMA_TO_DEVICE);
Kim Phillips8e8ec592011-03-13 16:54:26 +08001710 }
1711
1712 return ret;
1713badkey:
1714 crypto_aead_set_flags(aead, CRYPTO_TFM_RES_BAD_KEY_LEN);
1715 return -EINVAL;
1716}
1717
Tudor Ambarus3ef8d942014-10-23 16:11:23 +03001718static int gcm_setkey(struct crypto_aead *aead,
1719 const u8 *key, unsigned int keylen)
1720{
1721 struct caam_ctx *ctx = crypto_aead_ctx(aead);
1722 struct device *jrdev = ctx->jrdev;
1723 int ret = 0;
1724
1725#ifdef DEBUG
1726 print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ",
1727 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
1728#endif
1729
1730 memcpy(ctx->key, key, keylen);
1731 ctx->key_dma = dma_map_single(jrdev, ctx->key, keylen,
1732 DMA_TO_DEVICE);
1733 if (dma_mapping_error(jrdev, ctx->key_dma)) {
1734 dev_err(jrdev, "unable to map key i/o memory\n");
1735 return -ENOMEM;
1736 }
1737 ctx->enckeylen = keylen;
1738
1739 ret = gcm_set_sh_desc(aead);
1740 if (ret) {
1741 dma_unmap_single(jrdev, ctx->key_dma, ctx->enckeylen,
1742 DMA_TO_DEVICE);
1743 }
1744
1745 return ret;
1746}
1747
Tudor Ambarusbac68f22014-10-23 16:14:03 +03001748static int rfc4106_setkey(struct crypto_aead *aead,
1749 const u8 *key, unsigned int keylen)
1750{
1751 struct caam_ctx *ctx = crypto_aead_ctx(aead);
1752 struct device *jrdev = ctx->jrdev;
1753 int ret = 0;
1754
1755 if (keylen < 4)
1756 return -EINVAL;
1757
1758#ifdef DEBUG
1759 print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ",
1760 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
1761#endif
1762
1763 memcpy(ctx->key, key, keylen);
1764
1765 /*
1766 * The last four bytes of the key material are used as the salt value
1767 * in the nonce. Update the AES key length.
1768 */
1769 ctx->enckeylen = keylen - 4;
1770
1771 ctx->key_dma = dma_map_single(jrdev, ctx->key, ctx->enckeylen,
1772 DMA_TO_DEVICE);
1773 if (dma_mapping_error(jrdev, ctx->key_dma)) {
1774 dev_err(jrdev, "unable to map key i/o memory\n");
1775 return -ENOMEM;
1776 }
1777
1778 ret = rfc4106_set_sh_desc(aead);
1779 if (ret) {
1780 dma_unmap_single(jrdev, ctx->key_dma, ctx->enckeylen,
1781 DMA_TO_DEVICE);
1782 }
1783
1784 return ret;
1785}
1786
Tudor Ambarus5d0429a2014-10-30 18:55:07 +02001787static int rfc4543_setkey(struct crypto_aead *aead,
1788 const u8 *key, unsigned int keylen)
1789{
1790 struct caam_ctx *ctx = crypto_aead_ctx(aead);
1791 struct device *jrdev = ctx->jrdev;
1792 int ret = 0;
1793
1794 if (keylen < 4)
1795 return -EINVAL;
1796
1797#ifdef DEBUG
1798 print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ",
1799 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
1800#endif
1801
1802 memcpy(ctx->key, key, keylen);
1803
1804 /*
1805 * The last four bytes of the key material are used as the salt value
1806 * in the nonce. Update the AES key length.
1807 */
1808 ctx->enckeylen = keylen - 4;
1809
1810 ctx->key_dma = dma_map_single(jrdev, ctx->key, ctx->enckeylen,
1811 DMA_TO_DEVICE);
1812 if (dma_mapping_error(jrdev, ctx->key_dma)) {
1813 dev_err(jrdev, "unable to map key i/o memory\n");
1814 return -ENOMEM;
1815 }
1816
1817 ret = rfc4543_set_sh_desc(aead);
1818 if (ret) {
1819 dma_unmap_single(jrdev, ctx->key_dma, ctx->enckeylen,
1820 DMA_TO_DEVICE);
1821 }
1822
1823 return ret;
1824}
1825
Yuan Kangacdca312011-07-15 11:21:42 +08001826static int ablkcipher_setkey(struct crypto_ablkcipher *ablkcipher,
1827 const u8 *key, unsigned int keylen)
1828{
1829 struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher);
Catalin Vasilea5f57cf2014-10-31 12:45:36 +02001830 struct ablkcipher_tfm *crt = &ablkcipher->base.crt_ablkcipher;
1831 struct crypto_tfm *tfm = crypto_ablkcipher_tfm(ablkcipher);
1832 const char *alg_name = crypto_tfm_alg_name(tfm);
Yuan Kangacdca312011-07-15 11:21:42 +08001833 struct device *jrdev = ctx->jrdev;
1834 int ret = 0;
Horia Geanta4464a7d2014-03-14 17:46:49 +02001835 u32 *key_jump_cmd;
Yuan Kangacdca312011-07-15 11:21:42 +08001836 u32 *desc;
Catalin Vasilea5f57cf2014-10-31 12:45:36 +02001837 u32 *nonce;
Catalin Vasile7222d1a2014-10-31 12:45:38 +02001838 u32 geniv;
Catalin Vasile2b22f6c2014-10-31 12:45:35 +02001839 u32 ctx1_iv_off = 0;
1840 const bool ctr_mode = ((ctx->class1_alg_type & OP_ALG_AAI_MASK) ==
1841 OP_ALG_AAI_CTR_MOD128);
Catalin Vasilea5f57cf2014-10-31 12:45:36 +02001842 const bool is_rfc3686 = (ctr_mode &&
1843 (strstr(alg_name, "rfc3686") != NULL));
Yuan Kangacdca312011-07-15 11:21:42 +08001844
1845#ifdef DEBUG
Alex Porosanu514df282013-08-14 18:56:45 +03001846 print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ",
Yuan Kangacdca312011-07-15 11:21:42 +08001847 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
1848#endif
Catalin Vasile2b22f6c2014-10-31 12:45:35 +02001849 /*
1850 * AES-CTR needs to load IV in CONTEXT1 reg
1851 * at an offset of 128bits (16bytes)
1852 * CONTEXT1[255:128] = IV
1853 */
1854 if (ctr_mode)
1855 ctx1_iv_off = 16;
Yuan Kangacdca312011-07-15 11:21:42 +08001856
Catalin Vasilea5f57cf2014-10-31 12:45:36 +02001857 /*
1858 * RFC3686 specific:
1859 * | CONTEXT1[255:128] = {NONCE, IV, COUNTER}
1860 * | *key = {KEY, NONCE}
1861 */
1862 if (is_rfc3686) {
1863 ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE;
1864 keylen -= CTR_RFC3686_NONCE_SIZE;
1865 }
1866
Yuan Kangacdca312011-07-15 11:21:42 +08001867 memcpy(ctx->key, key, keylen);
1868 ctx->key_dma = dma_map_single(jrdev, ctx->key, keylen,
1869 DMA_TO_DEVICE);
1870 if (dma_mapping_error(jrdev, ctx->key_dma)) {
1871 dev_err(jrdev, "unable to map key i/o memory\n");
1872 return -ENOMEM;
1873 }
1874 ctx->enckeylen = keylen;
1875
1876 /* ablkcipher_encrypt shared descriptor */
1877 desc = ctx->sh_desc_enc;
Catalin Vasilea5f57cf2014-10-31 12:45:36 +02001878 init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX);
Yuan Kangacdca312011-07-15 11:21:42 +08001879 /* Skip if already shared */
1880 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
1881 JUMP_COND_SHRD);
1882
1883 /* Load class1 key only */
1884 append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen,
1885 ctx->enckeylen, CLASS_1 |
1886 KEY_DEST_CLASS_REG);
1887
Catalin Vasilea5f57cf2014-10-31 12:45:36 +02001888 /* Load nonce into CONTEXT1 reg */
1889 if (is_rfc3686) {
1890 nonce = (u32 *)(key + keylen);
1891 append_load_imm_u32(desc, *nonce, LDST_CLASS_IND_CCB |
1892 LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM);
1893 append_move(desc, MOVE_WAITCOMP |
1894 MOVE_SRC_OUTFIFO |
1895 MOVE_DEST_CLASS1CTX |
1896 (16 << MOVE_OFFSET_SHIFT) |
1897 (CTR_RFC3686_NONCE_SIZE << MOVE_LEN_SHIFT));
1898 }
1899
Yuan Kangacdca312011-07-15 11:21:42 +08001900 set_jump_tgt_here(desc, key_jump_cmd);
1901
Yuan Kangacdca312011-07-15 11:21:42 +08001902 /* Load iv */
Catalin Vasilea5f57cf2014-10-31 12:45:36 +02001903 append_seq_load(desc, crt->ivsize, LDST_SRCDST_BYTE_CONTEXT |
Catalin Vasile2b22f6c2014-10-31 12:45:35 +02001904 LDST_CLASS_1_CCB | (ctx1_iv_off << LDST_OFFSET_SHIFT));
Yuan Kangacdca312011-07-15 11:21:42 +08001905
Catalin Vasilea5f57cf2014-10-31 12:45:36 +02001906 /* Load counter into CONTEXT1 reg */
1907 if (is_rfc3686)
1908 append_load_imm_u32(desc, be32_to_cpu(1), LDST_IMM |
1909 LDST_CLASS_1_CCB |
1910 LDST_SRCDST_BYTE_CONTEXT |
1911 ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) <<
1912 LDST_OFFSET_SHIFT));
1913
Yuan Kangacdca312011-07-15 11:21:42 +08001914 /* Load operation */
1915 append_operation(desc, ctx->class1_alg_type |
1916 OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT);
1917
1918 /* Perform operation */
1919 ablkcipher_append_src_dst(desc);
1920
1921 ctx->sh_desc_enc_dma = dma_map_single(jrdev, desc,
1922 desc_bytes(desc),
1923 DMA_TO_DEVICE);
1924 if (dma_mapping_error(jrdev, ctx->sh_desc_enc_dma)) {
1925 dev_err(jrdev, "unable to map shared descriptor\n");
1926 return -ENOMEM;
1927 }
1928#ifdef DEBUG
Alex Porosanu514df282013-08-14 18:56:45 +03001929 print_hex_dump(KERN_ERR,
1930 "ablkcipher enc shdesc@"__stringify(__LINE__)": ",
Yuan Kangacdca312011-07-15 11:21:42 +08001931 DUMP_PREFIX_ADDRESS, 16, 4, desc,
1932 desc_bytes(desc), 1);
1933#endif
1934 /* ablkcipher_decrypt shared descriptor */
1935 desc = ctx->sh_desc_dec;
1936
Catalin Vasilea5f57cf2014-10-31 12:45:36 +02001937 init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX);
Yuan Kangacdca312011-07-15 11:21:42 +08001938 /* Skip if already shared */
1939 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
1940 JUMP_COND_SHRD);
1941
1942 /* Load class1 key only */
1943 append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen,
1944 ctx->enckeylen, CLASS_1 |
1945 KEY_DEST_CLASS_REG);
1946
Catalin Vasilea5f57cf2014-10-31 12:45:36 +02001947 /* Load nonce into CONTEXT1 reg */
1948 if (is_rfc3686) {
1949 nonce = (u32 *)(key + keylen);
1950 append_load_imm_u32(desc, *nonce, LDST_CLASS_IND_CCB |
1951 LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM);
1952 append_move(desc, MOVE_WAITCOMP |
1953 MOVE_SRC_OUTFIFO |
1954 MOVE_DEST_CLASS1CTX |
1955 (16 << MOVE_OFFSET_SHIFT) |
1956 (CTR_RFC3686_NONCE_SIZE << MOVE_LEN_SHIFT));
1957 }
1958
Yuan Kangacdca312011-07-15 11:21:42 +08001959 set_jump_tgt_here(desc, key_jump_cmd);
Yuan Kangacdca312011-07-15 11:21:42 +08001960
1961 /* load IV */
Catalin Vasilea5f57cf2014-10-31 12:45:36 +02001962 append_seq_load(desc, crt->ivsize, LDST_SRCDST_BYTE_CONTEXT |
Catalin Vasile2b22f6c2014-10-31 12:45:35 +02001963 LDST_CLASS_1_CCB | (ctx1_iv_off << LDST_OFFSET_SHIFT));
Yuan Kangacdca312011-07-15 11:21:42 +08001964
Catalin Vasilea5f57cf2014-10-31 12:45:36 +02001965 /* Load counter into CONTEXT1 reg */
1966 if (is_rfc3686)
1967 append_load_imm_u32(desc, be32_to_cpu(1), LDST_IMM |
1968 LDST_CLASS_1_CCB |
1969 LDST_SRCDST_BYTE_CONTEXT |
1970 ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) <<
1971 LDST_OFFSET_SHIFT));
1972
Yuan Kangacdca312011-07-15 11:21:42 +08001973 /* Choose operation */
Catalin Vasile2b22f6c2014-10-31 12:45:35 +02001974 if (ctr_mode)
1975 append_operation(desc, ctx->class1_alg_type |
1976 OP_ALG_AS_INITFINAL | OP_ALG_DECRYPT);
1977 else
1978 append_dec_op1(desc, ctx->class1_alg_type);
Yuan Kangacdca312011-07-15 11:21:42 +08001979
1980 /* Perform operation */
1981 ablkcipher_append_src_dst(desc);
1982
Yuan Kangacdca312011-07-15 11:21:42 +08001983 ctx->sh_desc_dec_dma = dma_map_single(jrdev, desc,
1984 desc_bytes(desc),
1985 DMA_TO_DEVICE);
Horia Geanta71c65f72014-07-11 15:34:48 +03001986 if (dma_mapping_error(jrdev, ctx->sh_desc_dec_dma)) {
Yuan Kangacdca312011-07-15 11:21:42 +08001987 dev_err(jrdev, "unable to map shared descriptor\n");
1988 return -ENOMEM;
1989 }
1990
1991#ifdef DEBUG
Alex Porosanu514df282013-08-14 18:56:45 +03001992 print_hex_dump(KERN_ERR,
1993 "ablkcipher dec shdesc@"__stringify(__LINE__)": ",
Yuan Kangacdca312011-07-15 11:21:42 +08001994 DUMP_PREFIX_ADDRESS, 16, 4, desc,
1995 desc_bytes(desc), 1);
1996#endif
Catalin Vasile7222d1a2014-10-31 12:45:38 +02001997 /* ablkcipher_givencrypt shared descriptor */
1998 desc = ctx->sh_desc_givenc;
1999
2000 init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX);
2001 /* Skip if already shared */
2002 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
2003 JUMP_COND_SHRD);
2004
2005 /* Load class1 key only */
2006 append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen,
2007 ctx->enckeylen, CLASS_1 |
2008 KEY_DEST_CLASS_REG);
2009
2010 /* Load Nonce into CONTEXT1 reg */
2011 if (is_rfc3686) {
2012 nonce = (u32 *)(key + keylen);
2013 append_load_imm_u32(desc, *nonce, LDST_CLASS_IND_CCB |
2014 LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM);
2015 append_move(desc, MOVE_WAITCOMP |
2016 MOVE_SRC_OUTFIFO |
2017 MOVE_DEST_CLASS1CTX |
2018 (16 << MOVE_OFFSET_SHIFT) |
2019 (CTR_RFC3686_NONCE_SIZE << MOVE_LEN_SHIFT));
2020 }
2021 set_jump_tgt_here(desc, key_jump_cmd);
2022
2023 /* Generate IV */
2024 geniv = NFIFOENTRY_STYPE_PAD | NFIFOENTRY_DEST_DECO |
2025 NFIFOENTRY_DTYPE_MSG | NFIFOENTRY_LC1 |
2026 NFIFOENTRY_PTYPE_RND | (crt->ivsize << NFIFOENTRY_DLEN_SHIFT);
2027 append_load_imm_u32(desc, geniv, LDST_CLASS_IND_CCB |
2028 LDST_SRCDST_WORD_INFO_FIFO | LDST_IMM);
2029 append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO);
2030 append_move(desc, MOVE_WAITCOMP |
2031 MOVE_SRC_INFIFO |
2032 MOVE_DEST_CLASS1CTX |
2033 (crt->ivsize << MOVE_LEN_SHIFT) |
2034 (ctx1_iv_off << MOVE_OFFSET_SHIFT));
2035 append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO);
2036
2037 /* Copy generated IV to memory */
2038 append_seq_store(desc, crt->ivsize,
2039 LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB |
2040 (ctx1_iv_off << LDST_OFFSET_SHIFT));
2041
2042 /* Load Counter into CONTEXT1 reg */
2043 if (is_rfc3686)
2044 append_load_imm_u32(desc, (u32)1, LDST_IMM |
2045 LDST_CLASS_1_CCB |
2046 LDST_SRCDST_BYTE_CONTEXT |
2047 ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) <<
2048 LDST_OFFSET_SHIFT));
2049
2050 if (ctx1_iv_off)
2051 append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | JUMP_COND_NCP |
2052 (1 << JUMP_OFFSET_SHIFT));
2053
2054 /* Load operation */
2055 append_operation(desc, ctx->class1_alg_type |
2056 OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT);
2057
2058 /* Perform operation */
2059 ablkcipher_append_src_dst(desc);
2060
2061 ctx->sh_desc_givenc_dma = dma_map_single(jrdev, desc,
2062 desc_bytes(desc),
2063 DMA_TO_DEVICE);
2064 if (dma_mapping_error(jrdev, ctx->sh_desc_givenc_dma)) {
2065 dev_err(jrdev, "unable to map shared descriptor\n");
2066 return -ENOMEM;
2067 }
2068#ifdef DEBUG
2069 print_hex_dump(KERN_ERR,
2070 "ablkcipher givenc shdesc@" __stringify(__LINE__) ": ",
2071 DUMP_PREFIX_ADDRESS, 16, 4, desc,
2072 desc_bytes(desc), 1);
2073#endif
Yuan Kangacdca312011-07-15 11:21:42 +08002074
2075 return ret;
2076}
2077
Kim Phillips8e8ec592011-03-13 16:54:26 +08002078/*
Yuan Kang1acebad2011-07-15 11:21:42 +08002079 * aead_edesc - s/w-extended aead descriptor
2080 * @assoc_nents: number of segments in associated data (SPI+Seq) scatterlist
Yuan Kang643b39b2012-06-22 19:48:49 -05002081 * @assoc_chained: if source is chained
Kim Phillips8e8ec592011-03-13 16:54:26 +08002082 * @src_nents: number of segments in input scatterlist
Yuan Kang643b39b2012-06-22 19:48:49 -05002083 * @src_chained: if source is chained
Kim Phillips8e8ec592011-03-13 16:54:26 +08002084 * @dst_nents: number of segments in output scatterlist
Yuan Kang643b39b2012-06-22 19:48:49 -05002085 * @dst_chained: if destination is chained
Yuan Kang1acebad2011-07-15 11:21:42 +08002086 * @iv_dma: dma address of iv for checking continuity and link table
Kim Phillips8e8ec592011-03-13 16:54:26 +08002087 * @desc: h/w descriptor (variable length; must not exceed MAX_CAAM_DESCSIZE)
Yuan Kanga299c832012-06-22 19:48:46 -05002088 * @sec4_sg_bytes: length of dma mapped sec4_sg space
2089 * @sec4_sg_dma: bus physical mapped address of h/w link table
Kim Phillips8e8ec592011-03-13 16:54:26 +08002090 * @hw_desc: the h/w job descriptor followed by any referenced link tables
2091 */
Yuan Kang0e479302011-07-15 11:21:41 +08002092struct aead_edesc {
Kim Phillips8e8ec592011-03-13 16:54:26 +08002093 int assoc_nents;
Yuan Kang643b39b2012-06-22 19:48:49 -05002094 bool assoc_chained;
Kim Phillips8e8ec592011-03-13 16:54:26 +08002095 int src_nents;
Yuan Kang643b39b2012-06-22 19:48:49 -05002096 bool src_chained;
Kim Phillips8e8ec592011-03-13 16:54:26 +08002097 int dst_nents;
Yuan Kang643b39b2012-06-22 19:48:49 -05002098 bool dst_chained;
Yuan Kang1acebad2011-07-15 11:21:42 +08002099 dma_addr_t iv_dma;
Yuan Kanga299c832012-06-22 19:48:46 -05002100 int sec4_sg_bytes;
2101 dma_addr_t sec4_sg_dma;
2102 struct sec4_sg_entry *sec4_sg;
Kim Phillips8e8ec592011-03-13 16:54:26 +08002103 u32 hw_desc[0];
2104};
2105
Yuan Kangacdca312011-07-15 11:21:42 +08002106/*
2107 * ablkcipher_edesc - s/w-extended ablkcipher descriptor
2108 * @src_nents: number of segments in input scatterlist
Yuan Kang643b39b2012-06-22 19:48:49 -05002109 * @src_chained: if source is chained
Yuan Kangacdca312011-07-15 11:21:42 +08002110 * @dst_nents: number of segments in output scatterlist
Yuan Kang643b39b2012-06-22 19:48:49 -05002111 * @dst_chained: if destination is chained
Yuan Kangacdca312011-07-15 11:21:42 +08002112 * @iv_dma: dma address of iv for checking continuity and link table
2113 * @desc: h/w descriptor (variable length; must not exceed MAX_CAAM_DESCSIZE)
Yuan Kanga299c832012-06-22 19:48:46 -05002114 * @sec4_sg_bytes: length of dma mapped sec4_sg space
2115 * @sec4_sg_dma: bus physical mapped address of h/w link table
Yuan Kangacdca312011-07-15 11:21:42 +08002116 * @hw_desc: the h/w job descriptor followed by any referenced link tables
2117 */
2118struct ablkcipher_edesc {
2119 int src_nents;
Yuan Kang643b39b2012-06-22 19:48:49 -05002120 bool src_chained;
Yuan Kangacdca312011-07-15 11:21:42 +08002121 int dst_nents;
Yuan Kang643b39b2012-06-22 19:48:49 -05002122 bool dst_chained;
Yuan Kangacdca312011-07-15 11:21:42 +08002123 dma_addr_t iv_dma;
Yuan Kanga299c832012-06-22 19:48:46 -05002124 int sec4_sg_bytes;
2125 dma_addr_t sec4_sg_dma;
2126 struct sec4_sg_entry *sec4_sg;
Yuan Kangacdca312011-07-15 11:21:42 +08002127 u32 hw_desc[0];
2128};
2129
Yuan Kang1acebad2011-07-15 11:21:42 +08002130static void caam_unmap(struct device *dev, struct scatterlist *src,
Yuan Kang643b39b2012-06-22 19:48:49 -05002131 struct scatterlist *dst, int src_nents,
2132 bool src_chained, int dst_nents, bool dst_chained,
Yuan Kanga299c832012-06-22 19:48:46 -05002133 dma_addr_t iv_dma, int ivsize, dma_addr_t sec4_sg_dma,
2134 int sec4_sg_bytes)
Kim Phillips8e8ec592011-03-13 16:54:26 +08002135{
Yuan Kang643b39b2012-06-22 19:48:49 -05002136 if (dst != src) {
2137 dma_unmap_sg_chained(dev, src, src_nents ? : 1, DMA_TO_DEVICE,
2138 src_chained);
2139 dma_unmap_sg_chained(dev, dst, dst_nents ? : 1, DMA_FROM_DEVICE,
2140 dst_chained);
Kim Phillips8e8ec592011-03-13 16:54:26 +08002141 } else {
Yuan Kang643b39b2012-06-22 19:48:49 -05002142 dma_unmap_sg_chained(dev, src, src_nents ? : 1,
2143 DMA_BIDIRECTIONAL, src_chained);
Kim Phillips8e8ec592011-03-13 16:54:26 +08002144 }
2145
Yuan Kang1acebad2011-07-15 11:21:42 +08002146 if (iv_dma)
2147 dma_unmap_single(dev, iv_dma, ivsize, DMA_TO_DEVICE);
Yuan Kanga299c832012-06-22 19:48:46 -05002148 if (sec4_sg_bytes)
2149 dma_unmap_single(dev, sec4_sg_dma, sec4_sg_bytes,
Kim Phillips8e8ec592011-03-13 16:54:26 +08002150 DMA_TO_DEVICE);
2151}
2152
Yuan Kang1acebad2011-07-15 11:21:42 +08002153static void aead_unmap(struct device *dev,
2154 struct aead_edesc *edesc,
2155 struct aead_request *req)
2156{
2157 struct crypto_aead *aead = crypto_aead_reqtfm(req);
2158 int ivsize = crypto_aead_ivsize(aead);
2159
Yuan Kang643b39b2012-06-22 19:48:49 -05002160 dma_unmap_sg_chained(dev, req->assoc, edesc->assoc_nents,
2161 DMA_TO_DEVICE, edesc->assoc_chained);
Yuan Kang1acebad2011-07-15 11:21:42 +08002162
2163 caam_unmap(dev, req->src, req->dst,
Yuan Kang643b39b2012-06-22 19:48:49 -05002164 edesc->src_nents, edesc->src_chained, edesc->dst_nents,
2165 edesc->dst_chained, edesc->iv_dma, ivsize,
2166 edesc->sec4_sg_dma, edesc->sec4_sg_bytes);
Yuan Kang1acebad2011-07-15 11:21:42 +08002167}
2168
Yuan Kangacdca312011-07-15 11:21:42 +08002169static void ablkcipher_unmap(struct device *dev,
2170 struct ablkcipher_edesc *edesc,
2171 struct ablkcipher_request *req)
2172{
2173 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
2174 int ivsize = crypto_ablkcipher_ivsize(ablkcipher);
2175
2176 caam_unmap(dev, req->src, req->dst,
Yuan Kang643b39b2012-06-22 19:48:49 -05002177 edesc->src_nents, edesc->src_chained, edesc->dst_nents,
2178 edesc->dst_chained, edesc->iv_dma, ivsize,
2179 edesc->sec4_sg_dma, edesc->sec4_sg_bytes);
Yuan Kangacdca312011-07-15 11:21:42 +08002180}
2181
Yuan Kang0e479302011-07-15 11:21:41 +08002182static void aead_encrypt_done(struct device *jrdev, u32 *desc, u32 err,
Kim Phillips8e8ec592011-03-13 16:54:26 +08002183 void *context)
2184{
Yuan Kang0e479302011-07-15 11:21:41 +08002185 struct aead_request *req = context;
2186 struct aead_edesc *edesc;
Kim Phillips8e8ec592011-03-13 16:54:26 +08002187#ifdef DEBUG
Yuan Kang0e479302011-07-15 11:21:41 +08002188 struct crypto_aead *aead = crypto_aead_reqtfm(req);
Kim Phillips8e8ec592011-03-13 16:54:26 +08002189 struct caam_ctx *ctx = crypto_aead_ctx(aead);
Yuan Kang1acebad2011-07-15 11:21:42 +08002190 int ivsize = crypto_aead_ivsize(aead);
Kim Phillips8e8ec592011-03-13 16:54:26 +08002191
2192 dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
2193#endif
Yuan Kang1acebad2011-07-15 11:21:42 +08002194
Yuan Kang0e479302011-07-15 11:21:41 +08002195 edesc = (struct aead_edesc *)((char *)desc -
2196 offsetof(struct aead_edesc, hw_desc));
Kim Phillips8e8ec592011-03-13 16:54:26 +08002197
Marek Vasutfa9659c2014-04-24 20:05:12 +02002198 if (err)
2199 caam_jr_strstatus(jrdev, err);
Kim Phillips8e8ec592011-03-13 16:54:26 +08002200
Yuan Kang0e479302011-07-15 11:21:41 +08002201 aead_unmap(jrdev, edesc, req);
Kim Phillips8e8ec592011-03-13 16:54:26 +08002202
2203#ifdef DEBUG
Alex Porosanu514df282013-08-14 18:56:45 +03002204 print_hex_dump(KERN_ERR, "assoc @"__stringify(__LINE__)": ",
Yuan Kang0e479302011-07-15 11:21:41 +08002205 DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->assoc),
2206 req->assoclen , 1);
Alex Porosanu514df282013-08-14 18:56:45 +03002207 print_hex_dump(KERN_ERR, "dstiv @"__stringify(__LINE__)": ",
Yuan Kang0e479302011-07-15 11:21:41 +08002208 DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src) - ivsize,
Kim Phillips8e8ec592011-03-13 16:54:26 +08002209 edesc->src_nents ? 100 : ivsize, 1);
Alex Porosanu514df282013-08-14 18:56:45 +03002210 print_hex_dump(KERN_ERR, "dst @"__stringify(__LINE__)": ",
Yuan Kang0e479302011-07-15 11:21:41 +08002211 DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src),
2212 edesc->src_nents ? 100 : req->cryptlen +
Kim Phillips8e8ec592011-03-13 16:54:26 +08002213 ctx->authsize + 4, 1);
2214#endif
2215
2216 kfree(edesc);
2217
Yuan Kang0e479302011-07-15 11:21:41 +08002218 aead_request_complete(req, err);
Kim Phillips8e8ec592011-03-13 16:54:26 +08002219}
2220
Yuan Kang0e479302011-07-15 11:21:41 +08002221static void aead_decrypt_done(struct device *jrdev, u32 *desc, u32 err,
Kim Phillips8e8ec592011-03-13 16:54:26 +08002222 void *context)
2223{
Yuan Kang0e479302011-07-15 11:21:41 +08002224 struct aead_request *req = context;
2225 struct aead_edesc *edesc;
Kim Phillips8e8ec592011-03-13 16:54:26 +08002226#ifdef DEBUG
Yuan Kang0e479302011-07-15 11:21:41 +08002227 struct crypto_aead *aead = crypto_aead_reqtfm(req);
Kim Phillips8e8ec592011-03-13 16:54:26 +08002228 struct caam_ctx *ctx = crypto_aead_ctx(aead);
Yuan Kang1acebad2011-07-15 11:21:42 +08002229 int ivsize = crypto_aead_ivsize(aead);
Kim Phillips8e8ec592011-03-13 16:54:26 +08002230
2231 dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
2232#endif
Yuan Kang1acebad2011-07-15 11:21:42 +08002233
Yuan Kang0e479302011-07-15 11:21:41 +08002234 edesc = (struct aead_edesc *)((char *)desc -
2235 offsetof(struct aead_edesc, hw_desc));
Kim Phillips8e8ec592011-03-13 16:54:26 +08002236
Yuan Kang1acebad2011-07-15 11:21:42 +08002237#ifdef DEBUG
Alex Porosanu514df282013-08-14 18:56:45 +03002238 print_hex_dump(KERN_ERR, "dstiv @"__stringify(__LINE__)": ",
Yuan Kang1acebad2011-07-15 11:21:42 +08002239 DUMP_PREFIX_ADDRESS, 16, 4, req->iv,
2240 ivsize, 1);
Alex Porosanu514df282013-08-14 18:56:45 +03002241 print_hex_dump(KERN_ERR, "dst @"__stringify(__LINE__)": ",
Yuan Kang1acebad2011-07-15 11:21:42 +08002242 DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->dst),
Horia Geantabbf9c892013-11-28 15:11:16 +02002243 req->cryptlen - ctx->authsize, 1);
Yuan Kang1acebad2011-07-15 11:21:42 +08002244#endif
2245
Marek Vasutfa9659c2014-04-24 20:05:12 +02002246 if (err)
2247 caam_jr_strstatus(jrdev, err);
Kim Phillips8e8ec592011-03-13 16:54:26 +08002248
Yuan Kang0e479302011-07-15 11:21:41 +08002249 aead_unmap(jrdev, edesc, req);
Kim Phillips8e8ec592011-03-13 16:54:26 +08002250
2251 /*
2252 * verify hw auth check passed else return -EBADMSG
2253 */
2254 if ((err & JRSTA_CCBERR_ERRID_MASK) == JRSTA_CCBERR_ERRID_ICVCHK)
2255 err = -EBADMSG;
2256
2257#ifdef DEBUG
Alex Porosanu514df282013-08-14 18:56:45 +03002258 print_hex_dump(KERN_ERR, "iphdrout@"__stringify(__LINE__)": ",
Kim Phillips8e8ec592011-03-13 16:54:26 +08002259 DUMP_PREFIX_ADDRESS, 16, 4,
Yuan Kang0e479302011-07-15 11:21:41 +08002260 ((char *)sg_virt(req->assoc) - sizeof(struct iphdr)),
2261 sizeof(struct iphdr) + req->assoclen +
2262 ((req->cryptlen > 1500) ? 1500 : req->cryptlen) +
Kim Phillips8e8ec592011-03-13 16:54:26 +08002263 ctx->authsize + 36, 1);
Yuan Kanga299c832012-06-22 19:48:46 -05002264 if (!err && edesc->sec4_sg_bytes) {
Yuan Kang0e479302011-07-15 11:21:41 +08002265 struct scatterlist *sg = sg_last(req->src, edesc->src_nents);
Alex Porosanu514df282013-08-14 18:56:45 +03002266 print_hex_dump(KERN_ERR, "sglastout@"__stringify(__LINE__)": ",
Kim Phillips8e8ec592011-03-13 16:54:26 +08002267 DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(sg),
2268 sg->length + ctx->authsize + 16, 1);
2269 }
2270#endif
Yuan Kang1acebad2011-07-15 11:21:42 +08002271
Kim Phillips8e8ec592011-03-13 16:54:26 +08002272 kfree(edesc);
2273
Yuan Kang0e479302011-07-15 11:21:41 +08002274 aead_request_complete(req, err);
Kim Phillips8e8ec592011-03-13 16:54:26 +08002275}
2276
Yuan Kangacdca312011-07-15 11:21:42 +08002277static void ablkcipher_encrypt_done(struct device *jrdev, u32 *desc, u32 err,
2278 void *context)
2279{
2280 struct ablkcipher_request *req = context;
2281 struct ablkcipher_edesc *edesc;
2282#ifdef DEBUG
2283 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
2284 int ivsize = crypto_ablkcipher_ivsize(ablkcipher);
2285
2286 dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
2287#endif
2288
2289 edesc = (struct ablkcipher_edesc *)((char *)desc -
2290 offsetof(struct ablkcipher_edesc, hw_desc));
2291
Marek Vasutfa9659c2014-04-24 20:05:12 +02002292 if (err)
2293 caam_jr_strstatus(jrdev, err);
Yuan Kangacdca312011-07-15 11:21:42 +08002294
2295#ifdef DEBUG
Alex Porosanu514df282013-08-14 18:56:45 +03002296 print_hex_dump(KERN_ERR, "dstiv @"__stringify(__LINE__)": ",
Yuan Kangacdca312011-07-15 11:21:42 +08002297 DUMP_PREFIX_ADDRESS, 16, 4, req->info,
2298 edesc->src_nents > 1 ? 100 : ivsize, 1);
Alex Porosanu514df282013-08-14 18:56:45 +03002299 print_hex_dump(KERN_ERR, "dst @"__stringify(__LINE__)": ",
Yuan Kangacdca312011-07-15 11:21:42 +08002300 DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src),
2301 edesc->dst_nents > 1 ? 100 : req->nbytes, 1);
2302#endif
2303
2304 ablkcipher_unmap(jrdev, edesc, req);
2305 kfree(edesc);
2306
2307 ablkcipher_request_complete(req, err);
2308}
2309
2310static void ablkcipher_decrypt_done(struct device *jrdev, u32 *desc, u32 err,
2311 void *context)
2312{
2313 struct ablkcipher_request *req = context;
2314 struct ablkcipher_edesc *edesc;
2315#ifdef DEBUG
2316 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
2317 int ivsize = crypto_ablkcipher_ivsize(ablkcipher);
2318
2319 dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
2320#endif
2321
2322 edesc = (struct ablkcipher_edesc *)((char *)desc -
2323 offsetof(struct ablkcipher_edesc, hw_desc));
Marek Vasutfa9659c2014-04-24 20:05:12 +02002324 if (err)
2325 caam_jr_strstatus(jrdev, err);
Yuan Kangacdca312011-07-15 11:21:42 +08002326
2327#ifdef DEBUG
Alex Porosanu514df282013-08-14 18:56:45 +03002328 print_hex_dump(KERN_ERR, "dstiv @"__stringify(__LINE__)": ",
Yuan Kangacdca312011-07-15 11:21:42 +08002329 DUMP_PREFIX_ADDRESS, 16, 4, req->info,
2330 ivsize, 1);
Alex Porosanu514df282013-08-14 18:56:45 +03002331 print_hex_dump(KERN_ERR, "dst @"__stringify(__LINE__)": ",
Yuan Kangacdca312011-07-15 11:21:42 +08002332 DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src),
2333 edesc->dst_nents > 1 ? 100 : req->nbytes, 1);
2334#endif
2335
2336 ablkcipher_unmap(jrdev, edesc, req);
2337 kfree(edesc);
2338
2339 ablkcipher_request_complete(req, err);
2340}
2341
Kim Phillips8e8ec592011-03-13 16:54:26 +08002342/*
Yuan Kang1acebad2011-07-15 11:21:42 +08002343 * Fill in aead job descriptor
Kim Phillips8e8ec592011-03-13 16:54:26 +08002344 */
Yuan Kang1acebad2011-07-15 11:21:42 +08002345static void init_aead_job(u32 *sh_desc, dma_addr_t ptr,
2346 struct aead_edesc *edesc,
2347 struct aead_request *req,
2348 bool all_contig, bool encrypt)
Kim Phillips8e8ec592011-03-13 16:54:26 +08002349{
Yuan Kang0e479302011-07-15 11:21:41 +08002350 struct crypto_aead *aead = crypto_aead_reqtfm(req);
Kim Phillips8e8ec592011-03-13 16:54:26 +08002351 struct caam_ctx *ctx = crypto_aead_ctx(aead);
Kim Phillips8e8ec592011-03-13 16:54:26 +08002352 int ivsize = crypto_aead_ivsize(aead);
2353 int authsize = ctx->authsize;
Yuan Kang1acebad2011-07-15 11:21:42 +08002354 u32 *desc = edesc->hw_desc;
2355 u32 out_options = 0, in_options;
2356 dma_addr_t dst_dma, src_dma;
Yuan Kanga299c832012-06-22 19:48:46 -05002357 int len, sec4_sg_index = 0;
Tudor Ambarus3ef8d942014-10-23 16:11:23 +03002358 bool is_gcm = false;
Kim Phillips8e8ec592011-03-13 16:54:26 +08002359
Yuan Kang1acebad2011-07-15 11:21:42 +08002360#ifdef DEBUG
Kim Phillips8e8ec592011-03-13 16:54:26 +08002361 debug("assoclen %d cryptlen %d authsize %d\n",
Yuan Kang0e479302011-07-15 11:21:41 +08002362 req->assoclen, req->cryptlen, authsize);
Alex Porosanu514df282013-08-14 18:56:45 +03002363 print_hex_dump(KERN_ERR, "assoc @"__stringify(__LINE__)": ",
Yuan Kang0e479302011-07-15 11:21:41 +08002364 DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->assoc),
2365 req->assoclen , 1);
Alex Porosanu514df282013-08-14 18:56:45 +03002366 print_hex_dump(KERN_ERR, "presciv@"__stringify(__LINE__)": ",
Yuan Kang1acebad2011-07-15 11:21:42 +08002367 DUMP_PREFIX_ADDRESS, 16, 4, req->iv,
Kim Phillips8e8ec592011-03-13 16:54:26 +08002368 edesc->src_nents ? 100 : ivsize, 1);
Alex Porosanu514df282013-08-14 18:56:45 +03002369 print_hex_dump(KERN_ERR, "src @"__stringify(__LINE__)": ",
Yuan Kang0e479302011-07-15 11:21:41 +08002370 DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src),
Yuan Kang1acebad2011-07-15 11:21:42 +08002371 edesc->src_nents ? 100 : req->cryptlen, 1);
Alex Porosanu514df282013-08-14 18:56:45 +03002372 print_hex_dump(KERN_ERR, "shrdesc@"__stringify(__LINE__)": ",
Kim Phillips8e8ec592011-03-13 16:54:26 +08002373 DUMP_PREFIX_ADDRESS, 16, 4, sh_desc,
2374 desc_bytes(sh_desc), 1);
2375#endif
Yuan Kang1acebad2011-07-15 11:21:42 +08002376
Tudor Ambarus3ef8d942014-10-23 16:11:23 +03002377 if (((ctx->class1_alg_type & OP_ALG_ALGSEL_MASK) ==
2378 OP_ALG_ALGSEL_AES) &&
2379 ((ctx->class1_alg_type & OP_ALG_AAI_MASK) == OP_ALG_AAI_GCM))
2380 is_gcm = true;
2381
Yuan Kang1acebad2011-07-15 11:21:42 +08002382 len = desc_len(sh_desc);
2383 init_job_desc_shared(desc, ptr, len, HDR_SHARE_DEFER | HDR_REVERSE);
2384
2385 if (all_contig) {
Tudor Ambarus3ef8d942014-10-23 16:11:23 +03002386 if (is_gcm)
2387 src_dma = edesc->iv_dma;
2388 else
2389 src_dma = sg_dma_address(req->assoc);
Yuan Kang1acebad2011-07-15 11:21:42 +08002390 in_options = 0;
2391 } else {
Yuan Kanga299c832012-06-22 19:48:46 -05002392 src_dma = edesc->sec4_sg_dma;
2393 sec4_sg_index += (edesc->assoc_nents ? : 1) + 1 +
2394 (edesc->src_nents ? : 1);
Yuan Kang1acebad2011-07-15 11:21:42 +08002395 in_options = LDST_SGF;
2396 }
Horia Geantabbf9c892013-11-28 15:11:16 +02002397
2398 append_seq_in_ptr(desc, src_dma, req->assoclen + ivsize + req->cryptlen,
2399 in_options);
Kim Phillips8e8ec592011-03-13 16:54:26 +08002400
Yuan Kang1acebad2011-07-15 11:21:42 +08002401 if (likely(req->src == req->dst)) {
2402 if (all_contig) {
2403 dst_dma = sg_dma_address(req->src);
2404 } else {
Yuan Kanga299c832012-06-22 19:48:46 -05002405 dst_dma = src_dma + sizeof(struct sec4_sg_entry) *
Yuan Kang1acebad2011-07-15 11:21:42 +08002406 ((edesc->assoc_nents ? : 1) + 1);
2407 out_options = LDST_SGF;
2408 }
Kim Phillips8e8ec592011-03-13 16:54:26 +08002409 } else {
Kim Phillips8e8ec592011-03-13 16:54:26 +08002410 if (!edesc->dst_nents) {
Yuan Kang0e479302011-07-15 11:21:41 +08002411 dst_dma = sg_dma_address(req->dst);
Kim Phillips8e8ec592011-03-13 16:54:26 +08002412 } else {
Yuan Kanga299c832012-06-22 19:48:46 -05002413 dst_dma = edesc->sec4_sg_dma +
2414 sec4_sg_index *
2415 sizeof(struct sec4_sg_entry);
Yuan Kang1acebad2011-07-15 11:21:42 +08002416 out_options = LDST_SGF;
Kim Phillips8e8ec592011-03-13 16:54:26 +08002417 }
2418 }
Kim Phillips8e8ec592011-03-13 16:54:26 +08002419 if (encrypt)
Horia Geantabbf9c892013-11-28 15:11:16 +02002420 append_seq_out_ptr(desc, dst_dma, req->cryptlen + authsize,
2421 out_options);
Kim Phillips8e8ec592011-03-13 16:54:26 +08002422 else
Yuan Kang1acebad2011-07-15 11:21:42 +08002423 append_seq_out_ptr(desc, dst_dma, req->cryptlen - authsize,
2424 out_options);
2425}
2426
2427/*
2428 * Fill in aead givencrypt job descriptor
2429 */
2430static void init_aead_giv_job(u32 *sh_desc, dma_addr_t ptr,
2431 struct aead_edesc *edesc,
2432 struct aead_request *req,
2433 int contig)
2434{
2435 struct crypto_aead *aead = crypto_aead_reqtfm(req);
2436 struct caam_ctx *ctx = crypto_aead_ctx(aead);
2437 int ivsize = crypto_aead_ivsize(aead);
2438 int authsize = ctx->authsize;
2439 u32 *desc = edesc->hw_desc;
2440 u32 out_options = 0, in_options;
2441 dma_addr_t dst_dma, src_dma;
Yuan Kanga299c832012-06-22 19:48:46 -05002442 int len, sec4_sg_index = 0;
Tudor Ambarusbac68f22014-10-23 16:14:03 +03002443 bool is_gcm = false;
Kim Phillips8e8ec592011-03-13 16:54:26 +08002444
2445#ifdef DEBUG
Yuan Kang1acebad2011-07-15 11:21:42 +08002446 debug("assoclen %d cryptlen %d authsize %d\n",
2447 req->assoclen, req->cryptlen, authsize);
Alex Porosanu514df282013-08-14 18:56:45 +03002448 print_hex_dump(KERN_ERR, "assoc @"__stringify(__LINE__)": ",
Yuan Kang1acebad2011-07-15 11:21:42 +08002449 DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->assoc),
2450 req->assoclen , 1);
Alex Porosanu514df282013-08-14 18:56:45 +03002451 print_hex_dump(KERN_ERR, "presciv@"__stringify(__LINE__)": ",
Yuan Kang1acebad2011-07-15 11:21:42 +08002452 DUMP_PREFIX_ADDRESS, 16, 4, req->iv, ivsize, 1);
Alex Porosanu514df282013-08-14 18:56:45 +03002453 print_hex_dump(KERN_ERR, "src @"__stringify(__LINE__)": ",
Yuan Kang1acebad2011-07-15 11:21:42 +08002454 DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src),
2455 edesc->src_nents > 1 ? 100 : req->cryptlen, 1);
Alex Porosanu514df282013-08-14 18:56:45 +03002456 print_hex_dump(KERN_ERR, "shrdesc@"__stringify(__LINE__)": ",
Yuan Kang1acebad2011-07-15 11:21:42 +08002457 DUMP_PREFIX_ADDRESS, 16, 4, sh_desc,
2458 desc_bytes(sh_desc), 1);
Kim Phillips8e8ec592011-03-13 16:54:26 +08002459#endif
2460
Tudor Ambarusbac68f22014-10-23 16:14:03 +03002461 if (((ctx->class1_alg_type & OP_ALG_ALGSEL_MASK) ==
2462 OP_ALG_ALGSEL_AES) &&
2463 ((ctx->class1_alg_type & OP_ALG_AAI_MASK) == OP_ALG_AAI_GCM))
2464 is_gcm = true;
2465
Yuan Kang1acebad2011-07-15 11:21:42 +08002466 len = desc_len(sh_desc);
2467 init_job_desc_shared(desc, ptr, len, HDR_SHARE_DEFER | HDR_REVERSE);
2468
2469 if (contig & GIV_SRC_CONTIG) {
Tudor Ambarusbac68f22014-10-23 16:14:03 +03002470 if (is_gcm)
2471 src_dma = edesc->iv_dma;
2472 else
2473 src_dma = sg_dma_address(req->assoc);
Yuan Kang1acebad2011-07-15 11:21:42 +08002474 in_options = 0;
2475 } else {
Yuan Kanga299c832012-06-22 19:48:46 -05002476 src_dma = edesc->sec4_sg_dma;
2477 sec4_sg_index += edesc->assoc_nents + 1 + edesc->src_nents;
Yuan Kang1acebad2011-07-15 11:21:42 +08002478 in_options = LDST_SGF;
2479 }
Horia Geantabbf9c892013-11-28 15:11:16 +02002480 append_seq_in_ptr(desc, src_dma, req->assoclen + ivsize + req->cryptlen,
2481 in_options);
Yuan Kang1acebad2011-07-15 11:21:42 +08002482
2483 if (contig & GIV_DST_CONTIG) {
2484 dst_dma = edesc->iv_dma;
2485 } else {
2486 if (likely(req->src == req->dst)) {
Yuan Kanga299c832012-06-22 19:48:46 -05002487 dst_dma = src_dma + sizeof(struct sec4_sg_entry) *
Tudor Ambarusbac68f22014-10-23 16:14:03 +03002488 (edesc->assoc_nents +
2489 (is_gcm ? 1 + edesc->src_nents : 0));
Yuan Kang1acebad2011-07-15 11:21:42 +08002490 out_options = LDST_SGF;
2491 } else {
Yuan Kanga299c832012-06-22 19:48:46 -05002492 dst_dma = edesc->sec4_sg_dma +
2493 sec4_sg_index *
2494 sizeof(struct sec4_sg_entry);
Yuan Kang1acebad2011-07-15 11:21:42 +08002495 out_options = LDST_SGF;
2496 }
Kim Phillips8e8ec592011-03-13 16:54:26 +08002497 }
2498
Horia Geantabbf9c892013-11-28 15:11:16 +02002499 append_seq_out_ptr(desc, dst_dma, ivsize + req->cryptlen + authsize,
2500 out_options);
Kim Phillips8e8ec592011-03-13 16:54:26 +08002501}
2502
2503/*
Yuan Kangacdca312011-07-15 11:21:42 +08002504 * Fill in ablkcipher job descriptor
2505 */
2506static void init_ablkcipher_job(u32 *sh_desc, dma_addr_t ptr,
2507 struct ablkcipher_edesc *edesc,
2508 struct ablkcipher_request *req,
2509 bool iv_contig)
2510{
2511 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
2512 int ivsize = crypto_ablkcipher_ivsize(ablkcipher);
2513 u32 *desc = edesc->hw_desc;
2514 u32 out_options = 0, in_options;
2515 dma_addr_t dst_dma, src_dma;
Yuan Kanga299c832012-06-22 19:48:46 -05002516 int len, sec4_sg_index = 0;
Yuan Kangacdca312011-07-15 11:21:42 +08002517
2518#ifdef DEBUG
Alex Porosanu514df282013-08-14 18:56:45 +03002519 print_hex_dump(KERN_ERR, "presciv@"__stringify(__LINE__)": ",
Yuan Kangacdca312011-07-15 11:21:42 +08002520 DUMP_PREFIX_ADDRESS, 16, 4, req->info,
2521 ivsize, 1);
Alex Porosanu514df282013-08-14 18:56:45 +03002522 print_hex_dump(KERN_ERR, "src @"__stringify(__LINE__)": ",
Yuan Kangacdca312011-07-15 11:21:42 +08002523 DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src),
2524 edesc->src_nents ? 100 : req->nbytes, 1);
2525#endif
2526
2527 len = desc_len(sh_desc);
2528 init_job_desc_shared(desc, ptr, len, HDR_SHARE_DEFER | HDR_REVERSE);
2529
2530 if (iv_contig) {
2531 src_dma = edesc->iv_dma;
2532 in_options = 0;
2533 } else {
Yuan Kanga299c832012-06-22 19:48:46 -05002534 src_dma = edesc->sec4_sg_dma;
Cristian Stoica35b82e52015-01-21 11:53:30 +02002535 sec4_sg_index += edesc->src_nents + 1;
Yuan Kangacdca312011-07-15 11:21:42 +08002536 in_options = LDST_SGF;
2537 }
2538 append_seq_in_ptr(desc, src_dma, req->nbytes + ivsize, in_options);
2539
2540 if (likely(req->src == req->dst)) {
2541 if (!edesc->src_nents && iv_contig) {
2542 dst_dma = sg_dma_address(req->src);
2543 } else {
Yuan Kanga299c832012-06-22 19:48:46 -05002544 dst_dma = edesc->sec4_sg_dma +
2545 sizeof(struct sec4_sg_entry);
Yuan Kangacdca312011-07-15 11:21:42 +08002546 out_options = LDST_SGF;
2547 }
2548 } else {
2549 if (!edesc->dst_nents) {
2550 dst_dma = sg_dma_address(req->dst);
2551 } else {
Yuan Kanga299c832012-06-22 19:48:46 -05002552 dst_dma = edesc->sec4_sg_dma +
2553 sec4_sg_index * sizeof(struct sec4_sg_entry);
Yuan Kangacdca312011-07-15 11:21:42 +08002554 out_options = LDST_SGF;
2555 }
2556 }
2557 append_seq_out_ptr(desc, dst_dma, req->nbytes, out_options);
2558}
2559
2560/*
Catalin Vasile7222d1a2014-10-31 12:45:38 +02002561 * Fill in ablkcipher givencrypt job descriptor
2562 */
2563static void init_ablkcipher_giv_job(u32 *sh_desc, dma_addr_t ptr,
2564 struct ablkcipher_edesc *edesc,
2565 struct ablkcipher_request *req,
2566 bool iv_contig)
2567{
2568 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
2569 int ivsize = crypto_ablkcipher_ivsize(ablkcipher);
2570 u32 *desc = edesc->hw_desc;
2571 u32 out_options, in_options;
2572 dma_addr_t dst_dma, src_dma;
2573 int len, sec4_sg_index = 0;
2574
2575#ifdef DEBUG
2576 print_hex_dump(KERN_ERR, "presciv@" __stringify(__LINE__) ": ",
2577 DUMP_PREFIX_ADDRESS, 16, 4, req->info,
2578 ivsize, 1);
2579 print_hex_dump(KERN_ERR, "src @" __stringify(__LINE__) ": ",
2580 DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src),
2581 edesc->src_nents ? 100 : req->nbytes, 1);
2582#endif
2583
2584 len = desc_len(sh_desc);
2585 init_job_desc_shared(desc, ptr, len, HDR_SHARE_DEFER | HDR_REVERSE);
2586
2587 if (!edesc->src_nents) {
2588 src_dma = sg_dma_address(req->src);
2589 in_options = 0;
2590 } else {
2591 src_dma = edesc->sec4_sg_dma;
2592 sec4_sg_index += edesc->src_nents;
2593 in_options = LDST_SGF;
2594 }
2595 append_seq_in_ptr(desc, src_dma, req->nbytes, in_options);
2596
2597 if (iv_contig) {
2598 dst_dma = edesc->iv_dma;
2599 out_options = 0;
2600 } else {
2601 dst_dma = edesc->sec4_sg_dma +
2602 sec4_sg_index * sizeof(struct sec4_sg_entry);
2603 out_options = LDST_SGF;
2604 }
2605 append_seq_out_ptr(desc, dst_dma, req->nbytes + ivsize, out_options);
2606}
2607
2608/*
Yuan Kang1acebad2011-07-15 11:21:42 +08002609 * allocate and map the aead extended descriptor
Kim Phillips8e8ec592011-03-13 16:54:26 +08002610 */
Yuan Kang0e479302011-07-15 11:21:41 +08002611static struct aead_edesc *aead_edesc_alloc(struct aead_request *req,
Horia Geantabbf9c892013-11-28 15:11:16 +02002612 int desc_bytes, bool *all_contig_ptr,
2613 bool encrypt)
Kim Phillips8e8ec592011-03-13 16:54:26 +08002614{
Yuan Kang0e479302011-07-15 11:21:41 +08002615 struct crypto_aead *aead = crypto_aead_reqtfm(req);
Kim Phillips8e8ec592011-03-13 16:54:26 +08002616 struct caam_ctx *ctx = crypto_aead_ctx(aead);
2617 struct device *jrdev = ctx->jrdev;
Yuan Kang1acebad2011-07-15 11:21:42 +08002618 gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG |
2619 CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC;
2620 int assoc_nents, src_nents, dst_nents = 0;
Yuan Kang0e479302011-07-15 11:21:41 +08002621 struct aead_edesc *edesc;
Yuan Kang1acebad2011-07-15 11:21:42 +08002622 dma_addr_t iv_dma = 0;
2623 int sgc;
2624 bool all_contig = true;
Yuan Kang643b39b2012-06-22 19:48:49 -05002625 bool assoc_chained = false, src_chained = false, dst_chained = false;
Yuan Kang1acebad2011-07-15 11:21:42 +08002626 int ivsize = crypto_aead_ivsize(aead);
Yuan Kanga299c832012-06-22 19:48:46 -05002627 int sec4_sg_index, sec4_sg_len = 0, sec4_sg_bytes;
Horia Geantabbf9c892013-11-28 15:11:16 +02002628 unsigned int authsize = ctx->authsize;
Tudor Ambarus3ef8d942014-10-23 16:11:23 +03002629 bool is_gcm = false;
Kim Phillips8e8ec592011-03-13 16:54:26 +08002630
Yuan Kang643b39b2012-06-22 19:48:49 -05002631 assoc_nents = sg_count(req->assoc, req->assoclen, &assoc_chained);
Kim Phillips8e8ec592011-03-13 16:54:26 +08002632
Horia Geantabbf9c892013-11-28 15:11:16 +02002633 if (unlikely(req->dst != req->src)) {
2634 src_nents = sg_count(req->src, req->cryptlen, &src_chained);
2635 dst_nents = sg_count(req->dst,
2636 req->cryptlen +
2637 (encrypt ? authsize : (-authsize)),
2638 &dst_chained);
2639 } else {
2640 src_nents = sg_count(req->src,
2641 req->cryptlen +
2642 (encrypt ? authsize : 0),
2643 &src_chained);
2644 }
Kim Phillips8e8ec592011-03-13 16:54:26 +08002645
Yuan Kang643b39b2012-06-22 19:48:49 -05002646 sgc = dma_map_sg_chained(jrdev, req->assoc, assoc_nents ? : 1,
Horia Geanta286233e2013-05-10 15:08:39 +03002647 DMA_TO_DEVICE, assoc_chained);
Yuan Kang1acebad2011-07-15 11:21:42 +08002648 if (likely(req->src == req->dst)) {
Yuan Kang643b39b2012-06-22 19:48:49 -05002649 sgc = dma_map_sg_chained(jrdev, req->src, src_nents ? : 1,
2650 DMA_BIDIRECTIONAL, src_chained);
Yuan Kang1acebad2011-07-15 11:21:42 +08002651 } else {
Yuan Kang643b39b2012-06-22 19:48:49 -05002652 sgc = dma_map_sg_chained(jrdev, req->src, src_nents ? : 1,
2653 DMA_TO_DEVICE, src_chained);
2654 sgc = dma_map_sg_chained(jrdev, req->dst, dst_nents ? : 1,
2655 DMA_FROM_DEVICE, dst_chained);
Kim Phillips8e8ec592011-03-13 16:54:26 +08002656 }
2657
Yuan Kang1acebad2011-07-15 11:21:42 +08002658 iv_dma = dma_map_single(jrdev, req->iv, ivsize, DMA_TO_DEVICE);
Horia Geantace572082014-07-11 15:34:49 +03002659 if (dma_mapping_error(jrdev, iv_dma)) {
2660 dev_err(jrdev, "unable to map IV\n");
2661 return ERR_PTR(-ENOMEM);
2662 }
2663
Tudor Ambarus3ef8d942014-10-23 16:11:23 +03002664 if (((ctx->class1_alg_type & OP_ALG_ALGSEL_MASK) ==
2665 OP_ALG_ALGSEL_AES) &&
2666 ((ctx->class1_alg_type & OP_ALG_AAI_MASK) == OP_ALG_AAI_GCM))
2667 is_gcm = true;
2668
2669 /*
2670 * Check if data are contiguous.
2671 * GCM expected input sequence: IV, AAD, text
2672 * All other - expected input sequence: AAD, IV, text
2673 */
2674 if (is_gcm)
2675 all_contig = (!assoc_nents &&
2676 iv_dma + ivsize == sg_dma_address(req->assoc) &&
2677 !src_nents && sg_dma_address(req->assoc) +
2678 req->assoclen == sg_dma_address(req->src));
2679 else
2680 all_contig = (!assoc_nents && sg_dma_address(req->assoc) +
2681 req->assoclen == iv_dma && !src_nents &&
2682 iv_dma + ivsize == sg_dma_address(req->src));
2683 if (!all_contig) {
Yuan Kang1acebad2011-07-15 11:21:42 +08002684 assoc_nents = assoc_nents ? : 1;
2685 src_nents = src_nents ? : 1;
Yuan Kanga299c832012-06-22 19:48:46 -05002686 sec4_sg_len = assoc_nents + 1 + src_nents;
Yuan Kang1acebad2011-07-15 11:21:42 +08002687 }
Tudor Ambarus3ef8d942014-10-23 16:11:23 +03002688
Yuan Kanga299c832012-06-22 19:48:46 -05002689 sec4_sg_len += dst_nents;
Yuan Kang1acebad2011-07-15 11:21:42 +08002690
Yuan Kanga299c832012-06-22 19:48:46 -05002691 sec4_sg_bytes = sec4_sg_len * sizeof(struct sec4_sg_entry);
Kim Phillips8e8ec592011-03-13 16:54:26 +08002692
2693 /* allocate space for base edesc and hw desc commands, link tables */
Yuan Kang0e479302011-07-15 11:21:41 +08002694 edesc = kmalloc(sizeof(struct aead_edesc) + desc_bytes +
Yuan Kanga299c832012-06-22 19:48:46 -05002695 sec4_sg_bytes, GFP_DMA | flags);
Kim Phillips8e8ec592011-03-13 16:54:26 +08002696 if (!edesc) {
2697 dev_err(jrdev, "could not allocate extended descriptor\n");
2698 return ERR_PTR(-ENOMEM);
2699 }
2700
2701 edesc->assoc_nents = assoc_nents;
Yuan Kang643b39b2012-06-22 19:48:49 -05002702 edesc->assoc_chained = assoc_chained;
Kim Phillips8e8ec592011-03-13 16:54:26 +08002703 edesc->src_nents = src_nents;
Yuan Kang643b39b2012-06-22 19:48:49 -05002704 edesc->src_chained = src_chained;
Kim Phillips8e8ec592011-03-13 16:54:26 +08002705 edesc->dst_nents = dst_nents;
Yuan Kang643b39b2012-06-22 19:48:49 -05002706 edesc->dst_chained = dst_chained;
Yuan Kang1acebad2011-07-15 11:21:42 +08002707 edesc->iv_dma = iv_dma;
Yuan Kanga299c832012-06-22 19:48:46 -05002708 edesc->sec4_sg_bytes = sec4_sg_bytes;
2709 edesc->sec4_sg = (void *)edesc + sizeof(struct aead_edesc) +
2710 desc_bytes;
Yuan Kang1acebad2011-07-15 11:21:42 +08002711 *all_contig_ptr = all_contig;
2712
Yuan Kanga299c832012-06-22 19:48:46 -05002713 sec4_sg_index = 0;
Yuan Kang1acebad2011-07-15 11:21:42 +08002714 if (!all_contig) {
Tudor Ambarus3ef8d942014-10-23 16:11:23 +03002715 if (!is_gcm) {
2716 sg_to_sec4_sg(req->assoc,
Cristian Stoica35b82e52015-01-21 11:53:30 +02002717 assoc_nents,
Tudor Ambarus3ef8d942014-10-23 16:11:23 +03002718 edesc->sec4_sg +
2719 sec4_sg_index, 0);
Cristian Stoica35b82e52015-01-21 11:53:30 +02002720 sec4_sg_index += assoc_nents;
Tudor Ambarus3ef8d942014-10-23 16:11:23 +03002721 }
2722
Yuan Kanga299c832012-06-22 19:48:46 -05002723 dma_to_sec4_sg_one(edesc->sec4_sg + sec4_sg_index,
Yuan Kang1acebad2011-07-15 11:21:42 +08002724 iv_dma, ivsize, 0);
Yuan Kanga299c832012-06-22 19:48:46 -05002725 sec4_sg_index += 1;
Tudor Ambarus3ef8d942014-10-23 16:11:23 +03002726
2727 if (is_gcm) {
2728 sg_to_sec4_sg(req->assoc,
Cristian Stoica35b82e52015-01-21 11:53:30 +02002729 assoc_nents,
Tudor Ambarus3ef8d942014-10-23 16:11:23 +03002730 edesc->sec4_sg +
2731 sec4_sg_index, 0);
Cristian Stoica35b82e52015-01-21 11:53:30 +02002732 sec4_sg_index += assoc_nents;
Tudor Ambarus3ef8d942014-10-23 16:11:23 +03002733 }
2734
Yuan Kanga299c832012-06-22 19:48:46 -05002735 sg_to_sec4_sg_last(req->src,
Cristian Stoica35b82e52015-01-21 11:53:30 +02002736 src_nents,
Yuan Kanga299c832012-06-22 19:48:46 -05002737 edesc->sec4_sg +
2738 sec4_sg_index, 0);
Cristian Stoica35b82e52015-01-21 11:53:30 +02002739 sec4_sg_index += src_nents;
Yuan Kang1acebad2011-07-15 11:21:42 +08002740 }
2741 if (dst_nents) {
Yuan Kanga299c832012-06-22 19:48:46 -05002742 sg_to_sec4_sg_last(req->dst, dst_nents,
2743 edesc->sec4_sg + sec4_sg_index, 0);
Yuan Kang1acebad2011-07-15 11:21:42 +08002744 }
Ruchika Gupta1da2be32014-06-23 19:50:26 +05302745 edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg,
2746 sec4_sg_bytes, DMA_TO_DEVICE);
Horia Geantace572082014-07-11 15:34:49 +03002747 if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) {
2748 dev_err(jrdev, "unable to map S/G table\n");
2749 return ERR_PTR(-ENOMEM);
2750 }
Kim Phillips8e8ec592011-03-13 16:54:26 +08002751
2752 return edesc;
2753}
2754
Yuan Kang0e479302011-07-15 11:21:41 +08002755static int aead_encrypt(struct aead_request *req)
Kim Phillips8e8ec592011-03-13 16:54:26 +08002756{
Yuan Kang0e479302011-07-15 11:21:41 +08002757 struct aead_edesc *edesc;
Kim Phillips8e8ec592011-03-13 16:54:26 +08002758 struct crypto_aead *aead = crypto_aead_reqtfm(req);
Kim Phillips8e8ec592011-03-13 16:54:26 +08002759 struct caam_ctx *ctx = crypto_aead_ctx(aead);
2760 struct device *jrdev = ctx->jrdev;
Yuan Kang1acebad2011-07-15 11:21:42 +08002761 bool all_contig;
Kim Phillips8e8ec592011-03-13 16:54:26 +08002762 u32 *desc;
Yuan Kang1acebad2011-07-15 11:21:42 +08002763 int ret = 0;
2764
Kim Phillips8e8ec592011-03-13 16:54:26 +08002765 /* allocate extended descriptor */
Yuan Kang1acebad2011-07-15 11:21:42 +08002766 edesc = aead_edesc_alloc(req, DESC_JOB_IO_LEN *
Horia Geantabbf9c892013-11-28 15:11:16 +02002767 CAAM_CMD_SZ, &all_contig, true);
Kim Phillips8e8ec592011-03-13 16:54:26 +08002768 if (IS_ERR(edesc))
2769 return PTR_ERR(edesc);
2770
Yuan Kang1acebad2011-07-15 11:21:42 +08002771 /* Create and submit job descriptor */
2772 init_aead_job(ctx->sh_desc_enc, ctx->sh_desc_enc_dma, edesc, req,
2773 all_contig, true);
2774#ifdef DEBUG
Alex Porosanu514df282013-08-14 18:56:45 +03002775 print_hex_dump(KERN_ERR, "aead jobdesc@"__stringify(__LINE__)": ",
Yuan Kang1acebad2011-07-15 11:21:42 +08002776 DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,
2777 desc_bytes(edesc->hw_desc), 1);
2778#endif
2779
Kim Phillips8e8ec592011-03-13 16:54:26 +08002780 desc = edesc->hw_desc;
Yuan Kang1acebad2011-07-15 11:21:42 +08002781 ret = caam_jr_enqueue(jrdev, desc, aead_encrypt_done, req);
2782 if (!ret) {
2783 ret = -EINPROGRESS;
2784 } else {
2785 aead_unmap(jrdev, edesc, req);
2786 kfree(edesc);
2787 }
Kim Phillips8e8ec592011-03-13 16:54:26 +08002788
Yuan Kang1acebad2011-07-15 11:21:42 +08002789 return ret;
Kim Phillips8e8ec592011-03-13 16:54:26 +08002790}
2791
Yuan Kang0e479302011-07-15 11:21:41 +08002792static int aead_decrypt(struct aead_request *req)
Kim Phillips8e8ec592011-03-13 16:54:26 +08002793{
Yuan Kang1acebad2011-07-15 11:21:42 +08002794 struct aead_edesc *edesc;
Yuan Kang0e479302011-07-15 11:21:41 +08002795 struct crypto_aead *aead = crypto_aead_reqtfm(req);
Yuan Kang0e479302011-07-15 11:21:41 +08002796 struct caam_ctx *ctx = crypto_aead_ctx(aead);
2797 struct device *jrdev = ctx->jrdev;
Yuan Kang1acebad2011-07-15 11:21:42 +08002798 bool all_contig;
Yuan Kang0e479302011-07-15 11:21:41 +08002799 u32 *desc;
Yuan Kang1acebad2011-07-15 11:21:42 +08002800 int ret = 0;
Yuan Kang0e479302011-07-15 11:21:41 +08002801
2802 /* allocate extended descriptor */
Yuan Kang1acebad2011-07-15 11:21:42 +08002803 edesc = aead_edesc_alloc(req, DESC_JOB_IO_LEN *
Horia Geantabbf9c892013-11-28 15:11:16 +02002804 CAAM_CMD_SZ, &all_contig, false);
Yuan Kang0e479302011-07-15 11:21:41 +08002805 if (IS_ERR(edesc))
2806 return PTR_ERR(edesc);
2807
Yuan Kang1acebad2011-07-15 11:21:42 +08002808#ifdef DEBUG
Alex Porosanu514df282013-08-14 18:56:45 +03002809 print_hex_dump(KERN_ERR, "dec src@"__stringify(__LINE__)": ",
Yuan Kang1acebad2011-07-15 11:21:42 +08002810 DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src),
2811 req->cryptlen, 1);
2812#endif
2813
2814 /* Create and submit job descriptor*/
2815 init_aead_job(ctx->sh_desc_dec,
2816 ctx->sh_desc_dec_dma, edesc, req, all_contig, false);
2817#ifdef DEBUG
Alex Porosanu514df282013-08-14 18:56:45 +03002818 print_hex_dump(KERN_ERR, "aead jobdesc@"__stringify(__LINE__)": ",
Yuan Kang1acebad2011-07-15 11:21:42 +08002819 DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,
2820 desc_bytes(edesc->hw_desc), 1);
2821#endif
2822
Yuan Kang0e479302011-07-15 11:21:41 +08002823 desc = edesc->hw_desc;
Yuan Kang1acebad2011-07-15 11:21:42 +08002824 ret = caam_jr_enqueue(jrdev, desc, aead_decrypt_done, req);
2825 if (!ret) {
2826 ret = -EINPROGRESS;
2827 } else {
2828 aead_unmap(jrdev, edesc, req);
2829 kfree(edesc);
2830 }
Yuan Kang0e479302011-07-15 11:21:41 +08002831
Yuan Kang1acebad2011-07-15 11:21:42 +08002832 return ret;
2833}
Yuan Kang0e479302011-07-15 11:21:41 +08002834
Yuan Kang1acebad2011-07-15 11:21:42 +08002835/*
2836 * allocate and map the aead extended descriptor for aead givencrypt
2837 */
2838static struct aead_edesc *aead_giv_edesc_alloc(struct aead_givcrypt_request
2839 *greq, int desc_bytes,
2840 u32 *contig_ptr)
2841{
2842 struct aead_request *req = &greq->areq;
2843 struct crypto_aead *aead = crypto_aead_reqtfm(req);
2844 struct caam_ctx *ctx = crypto_aead_ctx(aead);
2845 struct device *jrdev = ctx->jrdev;
2846 gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG |
2847 CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC;
2848 int assoc_nents, src_nents, dst_nents = 0;
2849 struct aead_edesc *edesc;
2850 dma_addr_t iv_dma = 0;
2851 int sgc;
2852 u32 contig = GIV_SRC_CONTIG | GIV_DST_CONTIG;
2853 int ivsize = crypto_aead_ivsize(aead);
Yuan Kang643b39b2012-06-22 19:48:49 -05002854 bool assoc_chained = false, src_chained = false, dst_chained = false;
Yuan Kanga299c832012-06-22 19:48:46 -05002855 int sec4_sg_index, sec4_sg_len = 0, sec4_sg_bytes;
Tudor Ambarusbac68f22014-10-23 16:14:03 +03002856 bool is_gcm = false;
Yuan Kang0e479302011-07-15 11:21:41 +08002857
Yuan Kang643b39b2012-06-22 19:48:49 -05002858 assoc_nents = sg_count(req->assoc, req->assoclen, &assoc_chained);
2859 src_nents = sg_count(req->src, req->cryptlen, &src_chained);
Yuan Kang0e479302011-07-15 11:21:41 +08002860
Yuan Kang1acebad2011-07-15 11:21:42 +08002861 if (unlikely(req->dst != req->src))
Horia Geantabbf9c892013-11-28 15:11:16 +02002862 dst_nents = sg_count(req->dst, req->cryptlen + ctx->authsize,
2863 &dst_chained);
Yuan Kang1acebad2011-07-15 11:21:42 +08002864
Yuan Kang643b39b2012-06-22 19:48:49 -05002865 sgc = dma_map_sg_chained(jrdev, req->assoc, assoc_nents ? : 1,
Horia Geanta286233e2013-05-10 15:08:39 +03002866 DMA_TO_DEVICE, assoc_chained);
Yuan Kang1acebad2011-07-15 11:21:42 +08002867 if (likely(req->src == req->dst)) {
Yuan Kang643b39b2012-06-22 19:48:49 -05002868 sgc = dma_map_sg_chained(jrdev, req->src, src_nents ? : 1,
2869 DMA_BIDIRECTIONAL, src_chained);
Yuan Kang1acebad2011-07-15 11:21:42 +08002870 } else {
Yuan Kang643b39b2012-06-22 19:48:49 -05002871 sgc = dma_map_sg_chained(jrdev, req->src, src_nents ? : 1,
2872 DMA_TO_DEVICE, src_chained);
2873 sgc = dma_map_sg_chained(jrdev, req->dst, dst_nents ? : 1,
2874 DMA_FROM_DEVICE, dst_chained);
Yuan Kang1acebad2011-07-15 11:21:42 +08002875 }
2876
Yuan Kang1acebad2011-07-15 11:21:42 +08002877 iv_dma = dma_map_single(jrdev, greq->giv, ivsize, DMA_TO_DEVICE);
Horia Geantace572082014-07-11 15:34:49 +03002878 if (dma_mapping_error(jrdev, iv_dma)) {
2879 dev_err(jrdev, "unable to map IV\n");
2880 return ERR_PTR(-ENOMEM);
2881 }
2882
Tudor Ambarusbac68f22014-10-23 16:14:03 +03002883 if (((ctx->class1_alg_type & OP_ALG_ALGSEL_MASK) ==
2884 OP_ALG_ALGSEL_AES) &&
2885 ((ctx->class1_alg_type & OP_ALG_AAI_MASK) == OP_ALG_AAI_GCM))
2886 is_gcm = true;
2887
2888 /*
2889 * Check if data are contiguous.
2890 * GCM expected input sequence: IV, AAD, text
2891 * All other - expected input sequence: AAD, IV, text
2892 */
2893
2894 if (is_gcm) {
2895 if (assoc_nents || iv_dma + ivsize !=
2896 sg_dma_address(req->assoc) || src_nents ||
2897 sg_dma_address(req->assoc) + req->assoclen !=
2898 sg_dma_address(req->src))
2899 contig &= ~GIV_SRC_CONTIG;
2900 } else {
2901 if (assoc_nents ||
2902 sg_dma_address(req->assoc) + req->assoclen != iv_dma ||
2903 src_nents || iv_dma + ivsize != sg_dma_address(req->src))
2904 contig &= ~GIV_SRC_CONTIG;
2905 }
2906
Yuan Kang1acebad2011-07-15 11:21:42 +08002907 if (dst_nents || iv_dma + ivsize != sg_dma_address(req->dst))
2908 contig &= ~GIV_DST_CONTIG;
Tudor Ambarusbac68f22014-10-23 16:14:03 +03002909
Yuan Kang1acebad2011-07-15 11:21:42 +08002910 if (!(contig & GIV_SRC_CONTIG)) {
2911 assoc_nents = assoc_nents ? : 1;
2912 src_nents = src_nents ? : 1;
Yuan Kanga299c832012-06-22 19:48:46 -05002913 sec4_sg_len += assoc_nents + 1 + src_nents;
Tudor Ambarus19167bf2014-10-24 18:13:37 +03002914 if (req->src == req->dst &&
2915 (src_nents || iv_dma + ivsize != sg_dma_address(req->src)))
Yuan Kang1acebad2011-07-15 11:21:42 +08002916 contig &= ~GIV_DST_CONTIG;
2917 }
Tudor Ambarusbac68f22014-10-23 16:14:03 +03002918
2919 /*
2920 * Add new sg entries for GCM output sequence.
2921 * Expected output sequence: IV, encrypted text.
2922 */
2923 if (is_gcm && req->src == req->dst && !(contig & GIV_DST_CONTIG))
2924 sec4_sg_len += 1 + src_nents;
2925
2926 if (unlikely(req->src != req->dst)) {
2927 dst_nents = dst_nents ? : 1;
2928 sec4_sg_len += 1 + dst_nents;
2929 }
Yuan Kang1acebad2011-07-15 11:21:42 +08002930
Yuan Kanga299c832012-06-22 19:48:46 -05002931 sec4_sg_bytes = sec4_sg_len * sizeof(struct sec4_sg_entry);
Yuan Kang1acebad2011-07-15 11:21:42 +08002932
2933 /* allocate space for base edesc and hw desc commands, link tables */
2934 edesc = kmalloc(sizeof(struct aead_edesc) + desc_bytes +
Yuan Kanga299c832012-06-22 19:48:46 -05002935 sec4_sg_bytes, GFP_DMA | flags);
Yuan Kang1acebad2011-07-15 11:21:42 +08002936 if (!edesc) {
2937 dev_err(jrdev, "could not allocate extended descriptor\n");
2938 return ERR_PTR(-ENOMEM);
2939 }
2940
2941 edesc->assoc_nents = assoc_nents;
Yuan Kang643b39b2012-06-22 19:48:49 -05002942 edesc->assoc_chained = assoc_chained;
Yuan Kang1acebad2011-07-15 11:21:42 +08002943 edesc->src_nents = src_nents;
Yuan Kang643b39b2012-06-22 19:48:49 -05002944 edesc->src_chained = src_chained;
Yuan Kang1acebad2011-07-15 11:21:42 +08002945 edesc->dst_nents = dst_nents;
Yuan Kang643b39b2012-06-22 19:48:49 -05002946 edesc->dst_chained = dst_chained;
Yuan Kang1acebad2011-07-15 11:21:42 +08002947 edesc->iv_dma = iv_dma;
Yuan Kanga299c832012-06-22 19:48:46 -05002948 edesc->sec4_sg_bytes = sec4_sg_bytes;
2949 edesc->sec4_sg = (void *)edesc + sizeof(struct aead_edesc) +
2950 desc_bytes;
Yuan Kang1acebad2011-07-15 11:21:42 +08002951 *contig_ptr = contig;
2952
Yuan Kanga299c832012-06-22 19:48:46 -05002953 sec4_sg_index = 0;
Yuan Kang1acebad2011-07-15 11:21:42 +08002954 if (!(contig & GIV_SRC_CONTIG)) {
Tudor Ambarusbac68f22014-10-23 16:14:03 +03002955 if (!is_gcm) {
2956 sg_to_sec4_sg(req->assoc, assoc_nents,
2957 edesc->sec4_sg + sec4_sg_index, 0);
2958 sec4_sg_index += assoc_nents;
2959 }
2960
Yuan Kanga299c832012-06-22 19:48:46 -05002961 dma_to_sec4_sg_one(edesc->sec4_sg + sec4_sg_index,
Yuan Kang1acebad2011-07-15 11:21:42 +08002962 iv_dma, ivsize, 0);
Yuan Kanga299c832012-06-22 19:48:46 -05002963 sec4_sg_index += 1;
Tudor Ambarusbac68f22014-10-23 16:14:03 +03002964
2965 if (is_gcm) {
2966 sg_to_sec4_sg(req->assoc, assoc_nents,
2967 edesc->sec4_sg + sec4_sg_index, 0);
2968 sec4_sg_index += assoc_nents;
2969 }
2970
Yuan Kanga299c832012-06-22 19:48:46 -05002971 sg_to_sec4_sg_last(req->src, src_nents,
2972 edesc->sec4_sg +
2973 sec4_sg_index, 0);
2974 sec4_sg_index += src_nents;
Yuan Kang1acebad2011-07-15 11:21:42 +08002975 }
Tudor Ambarusbac68f22014-10-23 16:14:03 +03002976
2977 if (is_gcm && req->src == req->dst && !(contig & GIV_DST_CONTIG)) {
2978 dma_to_sec4_sg_one(edesc->sec4_sg + sec4_sg_index,
2979 iv_dma, ivsize, 0);
2980 sec4_sg_index += 1;
2981 sg_to_sec4_sg_last(req->src, src_nents,
2982 edesc->sec4_sg + sec4_sg_index, 0);
2983 }
2984
Yuan Kang1acebad2011-07-15 11:21:42 +08002985 if (unlikely(req->src != req->dst && !(contig & GIV_DST_CONTIG))) {
Yuan Kanga299c832012-06-22 19:48:46 -05002986 dma_to_sec4_sg_one(edesc->sec4_sg + sec4_sg_index,
Yuan Kang1acebad2011-07-15 11:21:42 +08002987 iv_dma, ivsize, 0);
Yuan Kanga299c832012-06-22 19:48:46 -05002988 sec4_sg_index += 1;
2989 sg_to_sec4_sg_last(req->dst, dst_nents,
2990 edesc->sec4_sg + sec4_sg_index, 0);
Yuan Kang1acebad2011-07-15 11:21:42 +08002991 }
Ruchika Gupta1da2be32014-06-23 19:50:26 +05302992 edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg,
2993 sec4_sg_bytes, DMA_TO_DEVICE);
Horia Geantace572082014-07-11 15:34:49 +03002994 if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) {
2995 dev_err(jrdev, "unable to map S/G table\n");
2996 return ERR_PTR(-ENOMEM);
2997 }
Yuan Kang1acebad2011-07-15 11:21:42 +08002998
2999 return edesc;
Yuan Kang0e479302011-07-15 11:21:41 +08003000}
3001
3002static int aead_givencrypt(struct aead_givcrypt_request *areq)
3003{
3004 struct aead_request *req = &areq->areq;
3005 struct aead_edesc *edesc;
3006 struct crypto_aead *aead = crypto_aead_reqtfm(req);
Kim Phillips8e8ec592011-03-13 16:54:26 +08003007 struct caam_ctx *ctx = crypto_aead_ctx(aead);
3008 struct device *jrdev = ctx->jrdev;
Yuan Kang1acebad2011-07-15 11:21:42 +08003009 u32 contig;
Kim Phillips8e8ec592011-03-13 16:54:26 +08003010 u32 *desc;
Yuan Kang1acebad2011-07-15 11:21:42 +08003011 int ret = 0;
Kim Phillips8e8ec592011-03-13 16:54:26 +08003012
Kim Phillips8e8ec592011-03-13 16:54:26 +08003013 /* allocate extended descriptor */
Yuan Kang1acebad2011-07-15 11:21:42 +08003014 edesc = aead_giv_edesc_alloc(areq, DESC_JOB_IO_LEN *
3015 CAAM_CMD_SZ, &contig);
3016
Kim Phillips8e8ec592011-03-13 16:54:26 +08003017 if (IS_ERR(edesc))
3018 return PTR_ERR(edesc);
3019
Yuan Kang1acebad2011-07-15 11:21:42 +08003020#ifdef DEBUG
Alex Porosanu514df282013-08-14 18:56:45 +03003021 print_hex_dump(KERN_ERR, "giv src@"__stringify(__LINE__)": ",
Yuan Kang1acebad2011-07-15 11:21:42 +08003022 DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src),
3023 req->cryptlen, 1);
3024#endif
3025
3026 /* Create and submit job descriptor*/
3027 init_aead_giv_job(ctx->sh_desc_givenc,
3028 ctx->sh_desc_givenc_dma, edesc, req, contig);
3029#ifdef DEBUG
Alex Porosanu514df282013-08-14 18:56:45 +03003030 print_hex_dump(KERN_ERR, "aead jobdesc@"__stringify(__LINE__)": ",
Yuan Kang1acebad2011-07-15 11:21:42 +08003031 DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,
3032 desc_bytes(edesc->hw_desc), 1);
3033#endif
3034
Kim Phillips8e8ec592011-03-13 16:54:26 +08003035 desc = edesc->hw_desc;
Yuan Kang1acebad2011-07-15 11:21:42 +08003036 ret = caam_jr_enqueue(jrdev, desc, aead_encrypt_done, req);
3037 if (!ret) {
3038 ret = -EINPROGRESS;
3039 } else {
3040 aead_unmap(jrdev, edesc, req);
3041 kfree(edesc);
3042 }
Kim Phillips8e8ec592011-03-13 16:54:26 +08003043
Yuan Kang1acebad2011-07-15 11:21:42 +08003044 return ret;
Kim Phillips8e8ec592011-03-13 16:54:26 +08003045}
3046
Horia Geantaae4a8252014-03-14 17:46:52 +02003047static int aead_null_givencrypt(struct aead_givcrypt_request *areq)
3048{
3049 return aead_encrypt(&areq->areq);
3050}
3051
Yuan Kangacdca312011-07-15 11:21:42 +08003052/*
3053 * allocate and map the ablkcipher extended descriptor for ablkcipher
3054 */
3055static struct ablkcipher_edesc *ablkcipher_edesc_alloc(struct ablkcipher_request
3056 *req, int desc_bytes,
3057 bool *iv_contig_out)
3058{
3059 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
3060 struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher);
3061 struct device *jrdev = ctx->jrdev;
3062 gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG |
3063 CRYPTO_TFM_REQ_MAY_SLEEP)) ?
3064 GFP_KERNEL : GFP_ATOMIC;
Yuan Kanga299c832012-06-22 19:48:46 -05003065 int src_nents, dst_nents = 0, sec4_sg_bytes;
Yuan Kangacdca312011-07-15 11:21:42 +08003066 struct ablkcipher_edesc *edesc;
3067 dma_addr_t iv_dma = 0;
3068 bool iv_contig = false;
3069 int sgc;
3070 int ivsize = crypto_ablkcipher_ivsize(ablkcipher);
Yuan Kang643b39b2012-06-22 19:48:49 -05003071 bool src_chained = false, dst_chained = false;
Yuan Kanga299c832012-06-22 19:48:46 -05003072 int sec4_sg_index;
Yuan Kangacdca312011-07-15 11:21:42 +08003073
Yuan Kang643b39b2012-06-22 19:48:49 -05003074 src_nents = sg_count(req->src, req->nbytes, &src_chained);
Yuan Kangacdca312011-07-15 11:21:42 +08003075
Yuan Kang643b39b2012-06-22 19:48:49 -05003076 if (req->dst != req->src)
3077 dst_nents = sg_count(req->dst, req->nbytes, &dst_chained);
Yuan Kangacdca312011-07-15 11:21:42 +08003078
3079 if (likely(req->src == req->dst)) {
Yuan Kang643b39b2012-06-22 19:48:49 -05003080 sgc = dma_map_sg_chained(jrdev, req->src, src_nents ? : 1,
3081 DMA_BIDIRECTIONAL, src_chained);
Yuan Kangacdca312011-07-15 11:21:42 +08003082 } else {
Yuan Kang643b39b2012-06-22 19:48:49 -05003083 sgc = dma_map_sg_chained(jrdev, req->src, src_nents ? : 1,
3084 DMA_TO_DEVICE, src_chained);
3085 sgc = dma_map_sg_chained(jrdev, req->dst, dst_nents ? : 1,
3086 DMA_FROM_DEVICE, dst_chained);
Yuan Kangacdca312011-07-15 11:21:42 +08003087 }
3088
Horia Geantace572082014-07-11 15:34:49 +03003089 iv_dma = dma_map_single(jrdev, req->info, ivsize, DMA_TO_DEVICE);
3090 if (dma_mapping_error(jrdev, iv_dma)) {
3091 dev_err(jrdev, "unable to map IV\n");
3092 return ERR_PTR(-ENOMEM);
3093 }
3094
Yuan Kangacdca312011-07-15 11:21:42 +08003095 /*
3096 * Check if iv can be contiguous with source and destination.
3097 * If so, include it. If not, create scatterlist.
3098 */
Yuan Kangacdca312011-07-15 11:21:42 +08003099 if (!src_nents && iv_dma + ivsize == sg_dma_address(req->src))
3100 iv_contig = true;
3101 else
3102 src_nents = src_nents ? : 1;
Yuan Kanga299c832012-06-22 19:48:46 -05003103 sec4_sg_bytes = ((iv_contig ? 0 : 1) + src_nents + dst_nents) *
3104 sizeof(struct sec4_sg_entry);
Yuan Kangacdca312011-07-15 11:21:42 +08003105
3106 /* allocate space for base edesc and hw desc commands, link tables */
3107 edesc = kmalloc(sizeof(struct ablkcipher_edesc) + desc_bytes +
Yuan Kanga299c832012-06-22 19:48:46 -05003108 sec4_sg_bytes, GFP_DMA | flags);
Yuan Kangacdca312011-07-15 11:21:42 +08003109 if (!edesc) {
3110 dev_err(jrdev, "could not allocate extended descriptor\n");
3111 return ERR_PTR(-ENOMEM);
3112 }
3113
3114 edesc->src_nents = src_nents;
Yuan Kang643b39b2012-06-22 19:48:49 -05003115 edesc->src_chained = src_chained;
Yuan Kangacdca312011-07-15 11:21:42 +08003116 edesc->dst_nents = dst_nents;
Yuan Kang643b39b2012-06-22 19:48:49 -05003117 edesc->dst_chained = dst_chained;
Yuan Kanga299c832012-06-22 19:48:46 -05003118 edesc->sec4_sg_bytes = sec4_sg_bytes;
3119 edesc->sec4_sg = (void *)edesc + sizeof(struct ablkcipher_edesc) +
3120 desc_bytes;
Yuan Kangacdca312011-07-15 11:21:42 +08003121
Yuan Kanga299c832012-06-22 19:48:46 -05003122 sec4_sg_index = 0;
Yuan Kangacdca312011-07-15 11:21:42 +08003123 if (!iv_contig) {
Yuan Kanga299c832012-06-22 19:48:46 -05003124 dma_to_sec4_sg_one(edesc->sec4_sg, iv_dma, ivsize, 0);
3125 sg_to_sec4_sg_last(req->src, src_nents,
3126 edesc->sec4_sg + 1, 0);
3127 sec4_sg_index += 1 + src_nents;
Yuan Kangacdca312011-07-15 11:21:42 +08003128 }
3129
Yuan Kang643b39b2012-06-22 19:48:49 -05003130 if (dst_nents) {
Yuan Kanga299c832012-06-22 19:48:46 -05003131 sg_to_sec4_sg_last(req->dst, dst_nents,
3132 edesc->sec4_sg + sec4_sg_index, 0);
Yuan Kangacdca312011-07-15 11:21:42 +08003133 }
3134
Yuan Kanga299c832012-06-22 19:48:46 -05003135 edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg,
3136 sec4_sg_bytes, DMA_TO_DEVICE);
Horia Geantace572082014-07-11 15:34:49 +03003137 if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) {
3138 dev_err(jrdev, "unable to map S/G table\n");
3139 return ERR_PTR(-ENOMEM);
3140 }
3141
Yuan Kangacdca312011-07-15 11:21:42 +08003142 edesc->iv_dma = iv_dma;
3143
3144#ifdef DEBUG
Alex Porosanu514df282013-08-14 18:56:45 +03003145 print_hex_dump(KERN_ERR, "ablkcipher sec4_sg@"__stringify(__LINE__)": ",
Yuan Kanga299c832012-06-22 19:48:46 -05003146 DUMP_PREFIX_ADDRESS, 16, 4, edesc->sec4_sg,
3147 sec4_sg_bytes, 1);
Yuan Kangacdca312011-07-15 11:21:42 +08003148#endif
3149
3150 *iv_contig_out = iv_contig;
3151 return edesc;
3152}
3153
3154static int ablkcipher_encrypt(struct ablkcipher_request *req)
3155{
3156 struct ablkcipher_edesc *edesc;
3157 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
3158 struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher);
3159 struct device *jrdev = ctx->jrdev;
3160 bool iv_contig;
3161 u32 *desc;
3162 int ret = 0;
3163
3164 /* allocate extended descriptor */
3165 edesc = ablkcipher_edesc_alloc(req, DESC_JOB_IO_LEN *
3166 CAAM_CMD_SZ, &iv_contig);
3167 if (IS_ERR(edesc))
3168 return PTR_ERR(edesc);
3169
3170 /* Create and submit job descriptor*/
3171 init_ablkcipher_job(ctx->sh_desc_enc,
3172 ctx->sh_desc_enc_dma, edesc, req, iv_contig);
3173#ifdef DEBUG
Alex Porosanu514df282013-08-14 18:56:45 +03003174 print_hex_dump(KERN_ERR, "ablkcipher jobdesc@"__stringify(__LINE__)": ",
Yuan Kangacdca312011-07-15 11:21:42 +08003175 DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,
3176 desc_bytes(edesc->hw_desc), 1);
3177#endif
3178 desc = edesc->hw_desc;
3179 ret = caam_jr_enqueue(jrdev, desc, ablkcipher_encrypt_done, req);
3180
3181 if (!ret) {
3182 ret = -EINPROGRESS;
3183 } else {
3184 ablkcipher_unmap(jrdev, edesc, req);
3185 kfree(edesc);
3186 }
3187
3188 return ret;
3189}
3190
3191static int ablkcipher_decrypt(struct ablkcipher_request *req)
3192{
3193 struct ablkcipher_edesc *edesc;
3194 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
3195 struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher);
3196 struct device *jrdev = ctx->jrdev;
3197 bool iv_contig;
3198 u32 *desc;
3199 int ret = 0;
3200
3201 /* allocate extended descriptor */
3202 edesc = ablkcipher_edesc_alloc(req, DESC_JOB_IO_LEN *
3203 CAAM_CMD_SZ, &iv_contig);
3204 if (IS_ERR(edesc))
3205 return PTR_ERR(edesc);
3206
3207 /* Create and submit job descriptor*/
3208 init_ablkcipher_job(ctx->sh_desc_dec,
3209 ctx->sh_desc_dec_dma, edesc, req, iv_contig);
3210 desc = edesc->hw_desc;
3211#ifdef DEBUG
Alex Porosanu514df282013-08-14 18:56:45 +03003212 print_hex_dump(KERN_ERR, "ablkcipher jobdesc@"__stringify(__LINE__)": ",
Yuan Kangacdca312011-07-15 11:21:42 +08003213 DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,
3214 desc_bytes(edesc->hw_desc), 1);
3215#endif
3216
3217 ret = caam_jr_enqueue(jrdev, desc, ablkcipher_decrypt_done, req);
3218 if (!ret) {
3219 ret = -EINPROGRESS;
3220 } else {
3221 ablkcipher_unmap(jrdev, edesc, req);
3222 kfree(edesc);
3223 }
3224
3225 return ret;
3226}
3227
Catalin Vasile7222d1a2014-10-31 12:45:38 +02003228/*
3229 * allocate and map the ablkcipher extended descriptor
3230 * for ablkcipher givencrypt
3231 */
3232static struct ablkcipher_edesc *ablkcipher_giv_edesc_alloc(
3233 struct skcipher_givcrypt_request *greq,
3234 int desc_bytes,
3235 bool *iv_contig_out)
3236{
3237 struct ablkcipher_request *req = &greq->creq;
3238 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
3239 struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher);
3240 struct device *jrdev = ctx->jrdev;
3241 gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG |
3242 CRYPTO_TFM_REQ_MAY_SLEEP)) ?
3243 GFP_KERNEL : GFP_ATOMIC;
3244 int src_nents, dst_nents = 0, sec4_sg_bytes;
3245 struct ablkcipher_edesc *edesc;
3246 dma_addr_t iv_dma = 0;
3247 bool iv_contig = false;
3248 int sgc;
3249 int ivsize = crypto_ablkcipher_ivsize(ablkcipher);
3250 bool src_chained = false, dst_chained = false;
3251 int sec4_sg_index;
3252
3253 src_nents = sg_count(req->src, req->nbytes, &src_chained);
3254
3255 if (unlikely(req->dst != req->src))
3256 dst_nents = sg_count(req->dst, req->nbytes, &dst_chained);
3257
3258 if (likely(req->src == req->dst)) {
3259 sgc = dma_map_sg_chained(jrdev, req->src, src_nents ? : 1,
3260 DMA_BIDIRECTIONAL, src_chained);
3261 } else {
3262 sgc = dma_map_sg_chained(jrdev, req->src, src_nents ? : 1,
3263 DMA_TO_DEVICE, src_chained);
3264 sgc = dma_map_sg_chained(jrdev, req->dst, dst_nents ? : 1,
3265 DMA_FROM_DEVICE, dst_chained);
3266 }
3267
3268 /*
3269 * Check if iv can be contiguous with source and destination.
3270 * If so, include it. If not, create scatterlist.
3271 */
3272 iv_dma = dma_map_single(jrdev, greq->giv, ivsize, DMA_TO_DEVICE);
3273 if (dma_mapping_error(jrdev, iv_dma)) {
3274 dev_err(jrdev, "unable to map IV\n");
3275 return ERR_PTR(-ENOMEM);
3276 }
3277
3278 if (!dst_nents && iv_dma + ivsize == sg_dma_address(req->dst))
3279 iv_contig = true;
3280 else
3281 dst_nents = dst_nents ? : 1;
3282 sec4_sg_bytes = ((iv_contig ? 0 : 1) + src_nents + dst_nents) *
3283 sizeof(struct sec4_sg_entry);
3284
3285 /* allocate space for base edesc and hw desc commands, link tables */
3286 edesc = kmalloc(sizeof(*edesc) + desc_bytes +
3287 sec4_sg_bytes, GFP_DMA | flags);
3288 if (!edesc) {
3289 dev_err(jrdev, "could not allocate extended descriptor\n");
3290 return ERR_PTR(-ENOMEM);
3291 }
3292
3293 edesc->src_nents = src_nents;
3294 edesc->src_chained = src_chained;
3295 edesc->dst_nents = dst_nents;
3296 edesc->dst_chained = dst_chained;
3297 edesc->sec4_sg_bytes = sec4_sg_bytes;
3298 edesc->sec4_sg = (void *)edesc + sizeof(struct ablkcipher_edesc) +
3299 desc_bytes;
3300
3301 sec4_sg_index = 0;
3302 if (src_nents) {
3303 sg_to_sec4_sg_last(req->src, src_nents, edesc->sec4_sg, 0);
3304 sec4_sg_index += src_nents;
3305 }
3306
3307 if (!iv_contig) {
3308 dma_to_sec4_sg_one(edesc->sec4_sg + sec4_sg_index,
3309 iv_dma, ivsize, 0);
3310 sec4_sg_index += 1;
3311 sg_to_sec4_sg_last(req->dst, dst_nents,
3312 edesc->sec4_sg + sec4_sg_index, 0);
3313 }
3314
3315 edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg,
3316 sec4_sg_bytes, DMA_TO_DEVICE);
3317 if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) {
3318 dev_err(jrdev, "unable to map S/G table\n");
3319 return ERR_PTR(-ENOMEM);
3320 }
3321 edesc->iv_dma = iv_dma;
3322
3323#ifdef DEBUG
3324 print_hex_dump(KERN_ERR,
3325 "ablkcipher sec4_sg@" __stringify(__LINE__) ": ",
3326 DUMP_PREFIX_ADDRESS, 16, 4, edesc->sec4_sg,
3327 sec4_sg_bytes, 1);
3328#endif
3329
3330 *iv_contig_out = iv_contig;
3331 return edesc;
3332}
3333
3334static int ablkcipher_givencrypt(struct skcipher_givcrypt_request *creq)
3335{
3336 struct ablkcipher_request *req = &creq->creq;
3337 struct ablkcipher_edesc *edesc;
3338 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
3339 struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher);
3340 struct device *jrdev = ctx->jrdev;
3341 bool iv_contig;
3342 u32 *desc;
3343 int ret = 0;
3344
3345 /* allocate extended descriptor */
3346 edesc = ablkcipher_giv_edesc_alloc(creq, DESC_JOB_IO_LEN *
3347 CAAM_CMD_SZ, &iv_contig);
3348 if (IS_ERR(edesc))
3349 return PTR_ERR(edesc);
3350
3351 /* Create and submit job descriptor*/
3352 init_ablkcipher_giv_job(ctx->sh_desc_givenc, ctx->sh_desc_givenc_dma,
3353 edesc, req, iv_contig);
3354#ifdef DEBUG
3355 print_hex_dump(KERN_ERR,
3356 "ablkcipher jobdesc@" __stringify(__LINE__) ": ",
3357 DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,
3358 desc_bytes(edesc->hw_desc), 1);
3359#endif
3360 desc = edesc->hw_desc;
3361 ret = caam_jr_enqueue(jrdev, desc, ablkcipher_encrypt_done, req);
3362
3363 if (!ret) {
3364 ret = -EINPROGRESS;
3365 } else {
3366 ablkcipher_unmap(jrdev, edesc, req);
3367 kfree(edesc);
3368 }
3369
3370 return ret;
3371}
3372
Yuan Kang885e9e22011-07-15 11:21:41 +08003373#define template_aead template_u.aead
Yuan Kangacdca312011-07-15 11:21:42 +08003374#define template_ablkcipher template_u.ablkcipher
Kim Phillips8e8ec592011-03-13 16:54:26 +08003375struct caam_alg_template {
3376 char name[CRYPTO_MAX_ALG_NAME];
3377 char driver_name[CRYPTO_MAX_ALG_NAME];
3378 unsigned int blocksize;
Yuan Kang885e9e22011-07-15 11:21:41 +08003379 u32 type;
3380 union {
3381 struct ablkcipher_alg ablkcipher;
3382 struct aead_alg aead;
3383 struct blkcipher_alg blkcipher;
3384 struct cipher_alg cipher;
3385 struct compress_alg compress;
3386 struct rng_alg rng;
3387 } template_u;
Kim Phillips8e8ec592011-03-13 16:54:26 +08003388 u32 class1_alg_type;
3389 u32 class2_alg_type;
3390 u32 alg_op;
3391};
3392
3393static struct caam_alg_template driver_algs[] = {
Horia Geanta246bbed2013-03-20 16:31:58 +02003394 /* single-pass ipsec_esp descriptor */
Kim Phillips8e8ec592011-03-13 16:54:26 +08003395 {
Horia Geantaae4a8252014-03-14 17:46:52 +02003396 .name = "authenc(hmac(md5),ecb(cipher_null))",
3397 .driver_name = "authenc-hmac-md5-ecb-cipher_null-caam",
3398 .blocksize = NULL_BLOCK_SIZE,
3399 .type = CRYPTO_ALG_TYPE_AEAD,
3400 .template_aead = {
3401 .setkey = aead_setkey,
3402 .setauthsize = aead_setauthsize,
3403 .encrypt = aead_encrypt,
3404 .decrypt = aead_decrypt,
3405 .givencrypt = aead_null_givencrypt,
3406 .geniv = "<built-in>",
3407 .ivsize = NULL_IV_SIZE,
3408 .maxauthsize = MD5_DIGEST_SIZE,
3409 },
3410 .class1_alg_type = 0,
3411 .class2_alg_type = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC_PRECOMP,
3412 .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC,
3413 },
3414 {
3415 .name = "authenc(hmac(sha1),ecb(cipher_null))",
3416 .driver_name = "authenc-hmac-sha1-ecb-cipher_null-caam",
3417 .blocksize = NULL_BLOCK_SIZE,
3418 .type = CRYPTO_ALG_TYPE_AEAD,
3419 .template_aead = {
3420 .setkey = aead_setkey,
3421 .setauthsize = aead_setauthsize,
3422 .encrypt = aead_encrypt,
3423 .decrypt = aead_decrypt,
3424 .givencrypt = aead_null_givencrypt,
3425 .geniv = "<built-in>",
3426 .ivsize = NULL_IV_SIZE,
3427 .maxauthsize = SHA1_DIGEST_SIZE,
3428 },
3429 .class1_alg_type = 0,
3430 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC_PRECOMP,
3431 .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC,
3432 },
3433 {
3434 .name = "authenc(hmac(sha224),ecb(cipher_null))",
3435 .driver_name = "authenc-hmac-sha224-ecb-cipher_null-caam",
3436 .blocksize = NULL_BLOCK_SIZE,
3437 .type = CRYPTO_ALG_TYPE_AEAD,
3438 .template_aead = {
3439 .setkey = aead_setkey,
3440 .setauthsize = aead_setauthsize,
3441 .encrypt = aead_encrypt,
3442 .decrypt = aead_decrypt,
3443 .givencrypt = aead_null_givencrypt,
3444 .geniv = "<built-in>",
3445 .ivsize = NULL_IV_SIZE,
3446 .maxauthsize = SHA224_DIGEST_SIZE,
3447 },
3448 .class1_alg_type = 0,
3449 .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
3450 OP_ALG_AAI_HMAC_PRECOMP,
3451 .alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC,
3452 },
3453 {
3454 .name = "authenc(hmac(sha256),ecb(cipher_null))",
3455 .driver_name = "authenc-hmac-sha256-ecb-cipher_null-caam",
3456 .blocksize = NULL_BLOCK_SIZE,
3457 .type = CRYPTO_ALG_TYPE_AEAD,
3458 .template_aead = {
3459 .setkey = aead_setkey,
3460 .setauthsize = aead_setauthsize,
3461 .encrypt = aead_encrypt,
3462 .decrypt = aead_decrypt,
3463 .givencrypt = aead_null_givencrypt,
3464 .geniv = "<built-in>",
3465 .ivsize = NULL_IV_SIZE,
3466 .maxauthsize = SHA256_DIGEST_SIZE,
3467 },
3468 .class1_alg_type = 0,
3469 .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
3470 OP_ALG_AAI_HMAC_PRECOMP,
3471 .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC,
3472 },
3473 {
3474 .name = "authenc(hmac(sha384),ecb(cipher_null))",
3475 .driver_name = "authenc-hmac-sha384-ecb-cipher_null-caam",
3476 .blocksize = NULL_BLOCK_SIZE,
3477 .type = CRYPTO_ALG_TYPE_AEAD,
3478 .template_aead = {
3479 .setkey = aead_setkey,
3480 .setauthsize = aead_setauthsize,
3481 .encrypt = aead_encrypt,
3482 .decrypt = aead_decrypt,
3483 .givencrypt = aead_null_givencrypt,
3484 .geniv = "<built-in>",
3485 .ivsize = NULL_IV_SIZE,
3486 .maxauthsize = SHA384_DIGEST_SIZE,
3487 },
3488 .class1_alg_type = 0,
3489 .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
3490 OP_ALG_AAI_HMAC_PRECOMP,
3491 .alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC,
3492 },
3493 {
3494 .name = "authenc(hmac(sha512),ecb(cipher_null))",
3495 .driver_name = "authenc-hmac-sha512-ecb-cipher_null-caam",
3496 .blocksize = NULL_BLOCK_SIZE,
3497 .type = CRYPTO_ALG_TYPE_AEAD,
3498 .template_aead = {
3499 .setkey = aead_setkey,
3500 .setauthsize = aead_setauthsize,
3501 .encrypt = aead_encrypt,
3502 .decrypt = aead_decrypt,
3503 .givencrypt = aead_null_givencrypt,
3504 .geniv = "<built-in>",
3505 .ivsize = NULL_IV_SIZE,
3506 .maxauthsize = SHA512_DIGEST_SIZE,
3507 },
3508 .class1_alg_type = 0,
3509 .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
3510 OP_ALG_AAI_HMAC_PRECOMP,
3511 .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC,
3512 },
3513 {
Kim Phillips8b4d43a2011-11-21 16:13:27 +08003514 .name = "authenc(hmac(md5),cbc(aes))",
3515 .driver_name = "authenc-hmac-md5-cbc-aes-caam",
3516 .blocksize = AES_BLOCK_SIZE,
3517 .type = CRYPTO_ALG_TYPE_AEAD,
3518 .template_aead = {
3519 .setkey = aead_setkey,
3520 .setauthsize = aead_setauthsize,
3521 .encrypt = aead_encrypt,
3522 .decrypt = aead_decrypt,
3523 .givencrypt = aead_givencrypt,
3524 .geniv = "<built-in>",
3525 .ivsize = AES_BLOCK_SIZE,
3526 .maxauthsize = MD5_DIGEST_SIZE,
3527 },
3528 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
3529 .class2_alg_type = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC_PRECOMP,
3530 .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC,
3531 },
3532 {
Kim Phillips8e8ec592011-03-13 16:54:26 +08003533 .name = "authenc(hmac(sha1),cbc(aes))",
3534 .driver_name = "authenc-hmac-sha1-cbc-aes-caam",
3535 .blocksize = AES_BLOCK_SIZE,
Yuan Kang885e9e22011-07-15 11:21:41 +08003536 .type = CRYPTO_ALG_TYPE_AEAD,
3537 .template_aead = {
Yuan Kang0e479302011-07-15 11:21:41 +08003538 .setkey = aead_setkey,
3539 .setauthsize = aead_setauthsize,
3540 .encrypt = aead_encrypt,
3541 .decrypt = aead_decrypt,
3542 .givencrypt = aead_givencrypt,
Kim Phillips8e8ec592011-03-13 16:54:26 +08003543 .geniv = "<built-in>",
3544 .ivsize = AES_BLOCK_SIZE,
3545 .maxauthsize = SHA1_DIGEST_SIZE,
3546 },
3547 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
3548 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC_PRECOMP,
3549 .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC,
3550 },
3551 {
Hemant Agrawale863f9cc2012-01-09 18:26:44 -06003552 .name = "authenc(hmac(sha224),cbc(aes))",
3553 .driver_name = "authenc-hmac-sha224-cbc-aes-caam",
3554 .blocksize = AES_BLOCK_SIZE,
Vakul Gargcb7d5662013-03-12 14:09:24 +05303555 .type = CRYPTO_ALG_TYPE_AEAD,
Hemant Agrawale863f9cc2012-01-09 18:26:44 -06003556 .template_aead = {
3557 .setkey = aead_setkey,
3558 .setauthsize = aead_setauthsize,
3559 .encrypt = aead_encrypt,
3560 .decrypt = aead_decrypt,
3561 .givencrypt = aead_givencrypt,
3562 .geniv = "<built-in>",
3563 .ivsize = AES_BLOCK_SIZE,
3564 .maxauthsize = SHA224_DIGEST_SIZE,
3565 },
3566 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
3567 .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
3568 OP_ALG_AAI_HMAC_PRECOMP,
3569 .alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC,
3570 },
3571 {
Kim Phillips8e8ec592011-03-13 16:54:26 +08003572 .name = "authenc(hmac(sha256),cbc(aes))",
3573 .driver_name = "authenc-hmac-sha256-cbc-aes-caam",
3574 .blocksize = AES_BLOCK_SIZE,
Yuan Kang885e9e22011-07-15 11:21:41 +08003575 .type = CRYPTO_ALG_TYPE_AEAD,
3576 .template_aead = {
Yuan Kang0e479302011-07-15 11:21:41 +08003577 .setkey = aead_setkey,
3578 .setauthsize = aead_setauthsize,
3579 .encrypt = aead_encrypt,
3580 .decrypt = aead_decrypt,
3581 .givencrypt = aead_givencrypt,
Kim Phillips8e8ec592011-03-13 16:54:26 +08003582 .geniv = "<built-in>",
3583 .ivsize = AES_BLOCK_SIZE,
3584 .maxauthsize = SHA256_DIGEST_SIZE,
3585 },
3586 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
3587 .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
3588 OP_ALG_AAI_HMAC_PRECOMP,
3589 .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC,
3590 },
3591 {
Hemant Agrawale863f9cc2012-01-09 18:26:44 -06003592 .name = "authenc(hmac(sha384),cbc(aes))",
3593 .driver_name = "authenc-hmac-sha384-cbc-aes-caam",
3594 .blocksize = AES_BLOCK_SIZE,
Vakul Gargcb7d5662013-03-12 14:09:24 +05303595 .type = CRYPTO_ALG_TYPE_AEAD,
Hemant Agrawale863f9cc2012-01-09 18:26:44 -06003596 .template_aead = {
3597 .setkey = aead_setkey,
3598 .setauthsize = aead_setauthsize,
3599 .encrypt = aead_encrypt,
3600 .decrypt = aead_decrypt,
3601 .givencrypt = aead_givencrypt,
3602 .geniv = "<built-in>",
3603 .ivsize = AES_BLOCK_SIZE,
3604 .maxauthsize = SHA384_DIGEST_SIZE,
3605 },
3606 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
3607 .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
3608 OP_ALG_AAI_HMAC_PRECOMP,
3609 .alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC,
3610 },
3611
3612 {
Kim Phillips4427b1b2011-05-14 22:08:17 -05003613 .name = "authenc(hmac(sha512),cbc(aes))",
3614 .driver_name = "authenc-hmac-sha512-cbc-aes-caam",
3615 .blocksize = AES_BLOCK_SIZE,
Yuan Kang885e9e22011-07-15 11:21:41 +08003616 .type = CRYPTO_ALG_TYPE_AEAD,
3617 .template_aead = {
Yuan Kang0e479302011-07-15 11:21:41 +08003618 .setkey = aead_setkey,
3619 .setauthsize = aead_setauthsize,
3620 .encrypt = aead_encrypt,
3621 .decrypt = aead_decrypt,
3622 .givencrypt = aead_givencrypt,
Kim Phillips4427b1b2011-05-14 22:08:17 -05003623 .geniv = "<built-in>",
3624 .ivsize = AES_BLOCK_SIZE,
3625 .maxauthsize = SHA512_DIGEST_SIZE,
3626 },
3627 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
3628 .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
3629 OP_ALG_AAI_HMAC_PRECOMP,
3630 .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC,
3631 },
3632 {
Kim Phillips8b4d43a2011-11-21 16:13:27 +08003633 .name = "authenc(hmac(md5),cbc(des3_ede))",
3634 .driver_name = "authenc-hmac-md5-cbc-des3_ede-caam",
3635 .blocksize = DES3_EDE_BLOCK_SIZE,
3636 .type = CRYPTO_ALG_TYPE_AEAD,
3637 .template_aead = {
3638 .setkey = aead_setkey,
3639 .setauthsize = aead_setauthsize,
3640 .encrypt = aead_encrypt,
3641 .decrypt = aead_decrypt,
3642 .givencrypt = aead_givencrypt,
3643 .geniv = "<built-in>",
3644 .ivsize = DES3_EDE_BLOCK_SIZE,
3645 .maxauthsize = MD5_DIGEST_SIZE,
3646 },
3647 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
3648 .class2_alg_type = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC_PRECOMP,
3649 .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC,
3650 },
3651 {
Kim Phillips8e8ec592011-03-13 16:54:26 +08003652 .name = "authenc(hmac(sha1),cbc(des3_ede))",
3653 .driver_name = "authenc-hmac-sha1-cbc-des3_ede-caam",
3654 .blocksize = DES3_EDE_BLOCK_SIZE,
Yuan Kang885e9e22011-07-15 11:21:41 +08003655 .type = CRYPTO_ALG_TYPE_AEAD,
3656 .template_aead = {
Yuan Kang0e479302011-07-15 11:21:41 +08003657 .setkey = aead_setkey,
3658 .setauthsize = aead_setauthsize,
3659 .encrypt = aead_encrypt,
3660 .decrypt = aead_decrypt,
3661 .givencrypt = aead_givencrypt,
Kim Phillips8e8ec592011-03-13 16:54:26 +08003662 .geniv = "<built-in>",
3663 .ivsize = DES3_EDE_BLOCK_SIZE,
3664 .maxauthsize = SHA1_DIGEST_SIZE,
3665 },
3666 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
3667 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC_PRECOMP,
3668 .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC,
3669 },
3670 {
Hemant Agrawale863f9cc2012-01-09 18:26:44 -06003671 .name = "authenc(hmac(sha224),cbc(des3_ede))",
3672 .driver_name = "authenc-hmac-sha224-cbc-des3_ede-caam",
3673 .blocksize = DES3_EDE_BLOCK_SIZE,
Vakul Gargcb7d5662013-03-12 14:09:24 +05303674 .type = CRYPTO_ALG_TYPE_AEAD,
Hemant Agrawale863f9cc2012-01-09 18:26:44 -06003675 .template_aead = {
3676 .setkey = aead_setkey,
3677 .setauthsize = aead_setauthsize,
3678 .encrypt = aead_encrypt,
3679 .decrypt = aead_decrypt,
3680 .givencrypt = aead_givencrypt,
3681 .geniv = "<built-in>",
3682 .ivsize = DES3_EDE_BLOCK_SIZE,
3683 .maxauthsize = SHA224_DIGEST_SIZE,
3684 },
3685 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
3686 .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
3687 OP_ALG_AAI_HMAC_PRECOMP,
3688 .alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC,
3689 },
3690 {
Kim Phillips8e8ec592011-03-13 16:54:26 +08003691 .name = "authenc(hmac(sha256),cbc(des3_ede))",
3692 .driver_name = "authenc-hmac-sha256-cbc-des3_ede-caam",
3693 .blocksize = DES3_EDE_BLOCK_SIZE,
Yuan Kang885e9e22011-07-15 11:21:41 +08003694 .type = CRYPTO_ALG_TYPE_AEAD,
3695 .template_aead = {
Yuan Kang0e479302011-07-15 11:21:41 +08003696 .setkey = aead_setkey,
3697 .setauthsize = aead_setauthsize,
3698 .encrypt = aead_encrypt,
3699 .decrypt = aead_decrypt,
3700 .givencrypt = aead_givencrypt,
Kim Phillips8e8ec592011-03-13 16:54:26 +08003701 .geniv = "<built-in>",
3702 .ivsize = DES3_EDE_BLOCK_SIZE,
3703 .maxauthsize = SHA256_DIGEST_SIZE,
3704 },
3705 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
3706 .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
3707 OP_ALG_AAI_HMAC_PRECOMP,
3708 .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC,
3709 },
3710 {
Hemant Agrawale863f9cc2012-01-09 18:26:44 -06003711 .name = "authenc(hmac(sha384),cbc(des3_ede))",
3712 .driver_name = "authenc-hmac-sha384-cbc-des3_ede-caam",
3713 .blocksize = DES3_EDE_BLOCK_SIZE,
Vakul Gargcb7d5662013-03-12 14:09:24 +05303714 .type = CRYPTO_ALG_TYPE_AEAD,
Hemant Agrawale863f9cc2012-01-09 18:26:44 -06003715 .template_aead = {
3716 .setkey = aead_setkey,
3717 .setauthsize = aead_setauthsize,
3718 .encrypt = aead_encrypt,
3719 .decrypt = aead_decrypt,
3720 .givencrypt = aead_givencrypt,
3721 .geniv = "<built-in>",
3722 .ivsize = DES3_EDE_BLOCK_SIZE,
3723 .maxauthsize = SHA384_DIGEST_SIZE,
3724 },
3725 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
3726 .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
3727 OP_ALG_AAI_HMAC_PRECOMP,
3728 .alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC,
3729 },
3730 {
Kim Phillips4427b1b2011-05-14 22:08:17 -05003731 .name = "authenc(hmac(sha512),cbc(des3_ede))",
3732 .driver_name = "authenc-hmac-sha512-cbc-des3_ede-caam",
3733 .blocksize = DES3_EDE_BLOCK_SIZE,
Yuan Kang885e9e22011-07-15 11:21:41 +08003734 .type = CRYPTO_ALG_TYPE_AEAD,
3735 .template_aead = {
Yuan Kang0e479302011-07-15 11:21:41 +08003736 .setkey = aead_setkey,
3737 .setauthsize = aead_setauthsize,
3738 .encrypt = aead_encrypt,
3739 .decrypt = aead_decrypt,
3740 .givencrypt = aead_givencrypt,
Kim Phillips4427b1b2011-05-14 22:08:17 -05003741 .geniv = "<built-in>",
3742 .ivsize = DES3_EDE_BLOCK_SIZE,
3743 .maxauthsize = SHA512_DIGEST_SIZE,
3744 },
3745 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
3746 .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
3747 OP_ALG_AAI_HMAC_PRECOMP,
3748 .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC,
3749 },
3750 {
Kim Phillips8b4d43a2011-11-21 16:13:27 +08003751 .name = "authenc(hmac(md5),cbc(des))",
3752 .driver_name = "authenc-hmac-md5-cbc-des-caam",
3753 .blocksize = DES_BLOCK_SIZE,
3754 .type = CRYPTO_ALG_TYPE_AEAD,
3755 .template_aead = {
3756 .setkey = aead_setkey,
3757 .setauthsize = aead_setauthsize,
3758 .encrypt = aead_encrypt,
3759 .decrypt = aead_decrypt,
3760 .givencrypt = aead_givencrypt,
3761 .geniv = "<built-in>",
3762 .ivsize = DES_BLOCK_SIZE,
3763 .maxauthsize = MD5_DIGEST_SIZE,
3764 },
3765 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
3766 .class2_alg_type = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC_PRECOMP,
3767 .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC,
3768 },
3769 {
Kim Phillips8e8ec592011-03-13 16:54:26 +08003770 .name = "authenc(hmac(sha1),cbc(des))",
3771 .driver_name = "authenc-hmac-sha1-cbc-des-caam",
3772 .blocksize = DES_BLOCK_SIZE,
Yuan Kang885e9e22011-07-15 11:21:41 +08003773 .type = CRYPTO_ALG_TYPE_AEAD,
3774 .template_aead = {
Yuan Kang0e479302011-07-15 11:21:41 +08003775 .setkey = aead_setkey,
3776 .setauthsize = aead_setauthsize,
3777 .encrypt = aead_encrypt,
3778 .decrypt = aead_decrypt,
3779 .givencrypt = aead_givencrypt,
Kim Phillips8e8ec592011-03-13 16:54:26 +08003780 .geniv = "<built-in>",
3781 .ivsize = DES_BLOCK_SIZE,
3782 .maxauthsize = SHA1_DIGEST_SIZE,
3783 },
3784 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
3785 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC_PRECOMP,
3786 .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC,
3787 },
3788 {
Hemant Agrawale863f9cc2012-01-09 18:26:44 -06003789 .name = "authenc(hmac(sha224),cbc(des))",
3790 .driver_name = "authenc-hmac-sha224-cbc-des-caam",
3791 .blocksize = DES_BLOCK_SIZE,
Vakul Gargcb7d5662013-03-12 14:09:24 +05303792 .type = CRYPTO_ALG_TYPE_AEAD,
Hemant Agrawale863f9cc2012-01-09 18:26:44 -06003793 .template_aead = {
3794 .setkey = aead_setkey,
3795 .setauthsize = aead_setauthsize,
3796 .encrypt = aead_encrypt,
3797 .decrypt = aead_decrypt,
3798 .givencrypt = aead_givencrypt,
3799 .geniv = "<built-in>",
3800 .ivsize = DES_BLOCK_SIZE,
3801 .maxauthsize = SHA224_DIGEST_SIZE,
3802 },
3803 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
3804 .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
3805 OP_ALG_AAI_HMAC_PRECOMP,
3806 .alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC,
3807 },
3808 {
Kim Phillips8e8ec592011-03-13 16:54:26 +08003809 .name = "authenc(hmac(sha256),cbc(des))",
3810 .driver_name = "authenc-hmac-sha256-cbc-des-caam",
3811 .blocksize = DES_BLOCK_SIZE,
Yuan Kang885e9e22011-07-15 11:21:41 +08003812 .type = CRYPTO_ALG_TYPE_AEAD,
3813 .template_aead = {
Yuan Kang0e479302011-07-15 11:21:41 +08003814 .setkey = aead_setkey,
3815 .setauthsize = aead_setauthsize,
3816 .encrypt = aead_encrypt,
3817 .decrypt = aead_decrypt,
3818 .givencrypt = aead_givencrypt,
Kim Phillips8e8ec592011-03-13 16:54:26 +08003819 .geniv = "<built-in>",
3820 .ivsize = DES_BLOCK_SIZE,
3821 .maxauthsize = SHA256_DIGEST_SIZE,
3822 },
3823 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
3824 .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
3825 OP_ALG_AAI_HMAC_PRECOMP,
3826 .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC,
3827 },
Kim Phillips4427b1b2011-05-14 22:08:17 -05003828 {
Hemant Agrawale863f9cc2012-01-09 18:26:44 -06003829 .name = "authenc(hmac(sha384),cbc(des))",
3830 .driver_name = "authenc-hmac-sha384-cbc-des-caam",
3831 .blocksize = DES_BLOCK_SIZE,
Vakul Gargcb7d5662013-03-12 14:09:24 +05303832 .type = CRYPTO_ALG_TYPE_AEAD,
Hemant Agrawale863f9cc2012-01-09 18:26:44 -06003833 .template_aead = {
3834 .setkey = aead_setkey,
3835 .setauthsize = aead_setauthsize,
3836 .encrypt = aead_encrypt,
3837 .decrypt = aead_decrypt,
3838 .givencrypt = aead_givencrypt,
3839 .geniv = "<built-in>",
3840 .ivsize = DES_BLOCK_SIZE,
3841 .maxauthsize = SHA384_DIGEST_SIZE,
3842 },
3843 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
3844 .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
3845 OP_ALG_AAI_HMAC_PRECOMP,
3846 .alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC,
3847 },
3848 {
Kim Phillips4427b1b2011-05-14 22:08:17 -05003849 .name = "authenc(hmac(sha512),cbc(des))",
3850 .driver_name = "authenc-hmac-sha512-cbc-des-caam",
3851 .blocksize = DES_BLOCK_SIZE,
Yuan Kang885e9e22011-07-15 11:21:41 +08003852 .type = CRYPTO_ALG_TYPE_AEAD,
3853 .template_aead = {
Yuan Kang0e479302011-07-15 11:21:41 +08003854 .setkey = aead_setkey,
3855 .setauthsize = aead_setauthsize,
3856 .encrypt = aead_encrypt,
3857 .decrypt = aead_decrypt,
3858 .givencrypt = aead_givencrypt,
Kim Phillips4427b1b2011-05-14 22:08:17 -05003859 .geniv = "<built-in>",
3860 .ivsize = DES_BLOCK_SIZE,
3861 .maxauthsize = SHA512_DIGEST_SIZE,
3862 },
3863 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
3864 .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
3865 OP_ALG_AAI_HMAC_PRECOMP,
3866 .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC,
3867 },
Tudor Ambarusbac68f22014-10-23 16:14:03 +03003868 {
Catalin Vasiledaebc462014-10-31 12:45:37 +02003869 .name = "authenc(hmac(md5),rfc3686(ctr(aes)))",
3870 .driver_name = "authenc-hmac-md5-rfc3686-ctr-aes-caam",
3871 .blocksize = 1,
3872 .type = CRYPTO_ALG_TYPE_AEAD,
3873 .template_aead = {
3874 .setkey = aead_setkey,
3875 .setauthsize = aead_setauthsize,
3876 .encrypt = aead_encrypt,
3877 .decrypt = aead_decrypt,
3878 .givencrypt = aead_givencrypt,
3879 .geniv = "<built-in>",
3880 .ivsize = CTR_RFC3686_IV_SIZE,
3881 .maxauthsize = MD5_DIGEST_SIZE,
3882 },
3883 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CTR_MOD128,
3884 .class2_alg_type = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC_PRECOMP,
3885 .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC,
3886 },
3887 {
3888 .name = "authenc(hmac(sha1),rfc3686(ctr(aes)))",
3889 .driver_name = "authenc-hmac-sha1-rfc3686-ctr-aes-caam",
3890 .blocksize = 1,
3891 .type = CRYPTO_ALG_TYPE_AEAD,
3892 .template_aead = {
3893 .setkey = aead_setkey,
3894 .setauthsize = aead_setauthsize,
3895 .encrypt = aead_encrypt,
3896 .decrypt = aead_decrypt,
3897 .givencrypt = aead_givencrypt,
3898 .geniv = "<built-in>",
3899 .ivsize = CTR_RFC3686_IV_SIZE,
3900 .maxauthsize = SHA1_DIGEST_SIZE,
3901 },
3902 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CTR_MOD128,
3903 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC_PRECOMP,
3904 .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC,
3905 },
3906 {
3907 .name = "authenc(hmac(sha224),rfc3686(ctr(aes)))",
3908 .driver_name = "authenc-hmac-sha224-rfc3686-ctr-aes-caam",
3909 .blocksize = 1,
3910 .type = CRYPTO_ALG_TYPE_AEAD,
3911 .template_aead = {
3912 .setkey = aead_setkey,
3913 .setauthsize = aead_setauthsize,
3914 .encrypt = aead_encrypt,
3915 .decrypt = aead_decrypt,
3916 .givencrypt = aead_givencrypt,
3917 .geniv = "<built-in>",
3918 .ivsize = CTR_RFC3686_IV_SIZE,
3919 .maxauthsize = SHA224_DIGEST_SIZE,
3920 },
3921 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CTR_MOD128,
3922 .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
3923 OP_ALG_AAI_HMAC_PRECOMP,
3924 .alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC,
3925 },
3926 {
3927 .name = "authenc(hmac(sha256),rfc3686(ctr(aes)))",
3928 .driver_name = "authenc-hmac-sha256-rfc3686-ctr-aes-caam",
3929 .blocksize = 1,
3930 .type = CRYPTO_ALG_TYPE_AEAD,
3931 .template_aead = {
3932 .setkey = aead_setkey,
3933 .setauthsize = aead_setauthsize,
3934 .encrypt = aead_encrypt,
3935 .decrypt = aead_decrypt,
3936 .givencrypt = aead_givencrypt,
3937 .geniv = "<built-in>",
3938 .ivsize = CTR_RFC3686_IV_SIZE,
3939 .maxauthsize = SHA256_DIGEST_SIZE,
3940 },
3941 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CTR_MOD128,
3942 .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
3943 OP_ALG_AAI_HMAC_PRECOMP,
3944 .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC,
3945 },
3946 {
3947 .name = "authenc(hmac(sha384),rfc3686(ctr(aes)))",
3948 .driver_name = "authenc-hmac-sha384-rfc3686-ctr-aes-caam",
3949 .blocksize = 1,
3950 .type = CRYPTO_ALG_TYPE_AEAD,
3951 .template_aead = {
3952 .setkey = aead_setkey,
3953 .setauthsize = aead_setauthsize,
3954 .encrypt = aead_encrypt,
3955 .decrypt = aead_decrypt,
3956 .givencrypt = aead_givencrypt,
3957 .geniv = "<built-in>",
3958 .ivsize = CTR_RFC3686_IV_SIZE,
3959 .maxauthsize = SHA384_DIGEST_SIZE,
3960 },
3961 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CTR_MOD128,
3962 .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
3963 OP_ALG_AAI_HMAC_PRECOMP,
3964 .alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC,
3965 },
3966 {
3967 .name = "authenc(hmac(sha512),rfc3686(ctr(aes)))",
3968 .driver_name = "authenc-hmac-sha512-rfc3686-ctr-aes-caam",
3969 .blocksize = 1,
3970 .type = CRYPTO_ALG_TYPE_AEAD,
3971 .template_aead = {
3972 .setkey = aead_setkey,
3973 .setauthsize = aead_setauthsize,
3974 .encrypt = aead_encrypt,
3975 .decrypt = aead_decrypt,
3976 .givencrypt = aead_givencrypt,
3977 .geniv = "<built-in>",
3978 .ivsize = CTR_RFC3686_IV_SIZE,
3979 .maxauthsize = SHA512_DIGEST_SIZE,
3980 },
3981 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CTR_MOD128,
3982 .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
3983 OP_ALG_AAI_HMAC_PRECOMP,
3984 .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC,
3985 },
3986 {
Tudor Ambarusbac68f22014-10-23 16:14:03 +03003987 .name = "rfc4106(gcm(aes))",
3988 .driver_name = "rfc4106-gcm-aes-caam",
3989 .blocksize = 1,
3990 .type = CRYPTO_ALG_TYPE_AEAD,
3991 .template_aead = {
3992 .setkey = rfc4106_setkey,
3993 .setauthsize = rfc4106_setauthsize,
3994 .encrypt = aead_encrypt,
3995 .decrypt = aead_decrypt,
3996 .givencrypt = aead_givencrypt,
3997 .geniv = "<built-in>",
3998 .ivsize = 8,
3999 .maxauthsize = AES_BLOCK_SIZE,
4000 },
4001 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM,
4002 },
Tudor Ambarus5d0429a2014-10-30 18:55:07 +02004003 {
4004 .name = "rfc4543(gcm(aes))",
4005 .driver_name = "rfc4543-gcm-aes-caam",
4006 .blocksize = 1,
4007 .type = CRYPTO_ALG_TYPE_AEAD,
4008 .template_aead = {
4009 .setkey = rfc4543_setkey,
4010 .setauthsize = rfc4543_setauthsize,
4011 .encrypt = aead_encrypt,
4012 .decrypt = aead_decrypt,
4013 .givencrypt = aead_givencrypt,
4014 .geniv = "<built-in>",
4015 .ivsize = 8,
4016 .maxauthsize = AES_BLOCK_SIZE,
4017 },
4018 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM,
4019 },
Tudor Ambarus3ef8d942014-10-23 16:11:23 +03004020 /* Galois Counter Mode */
4021 {
4022 .name = "gcm(aes)",
4023 .driver_name = "gcm-aes-caam",
4024 .blocksize = 1,
4025 .type = CRYPTO_ALG_TYPE_AEAD,
4026 .template_aead = {
4027 .setkey = gcm_setkey,
4028 .setauthsize = gcm_setauthsize,
4029 .encrypt = aead_encrypt,
4030 .decrypt = aead_decrypt,
4031 .givencrypt = NULL,
4032 .geniv = "<built-in>",
4033 .ivsize = 12,
4034 .maxauthsize = AES_BLOCK_SIZE,
4035 },
4036 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM,
4037 },
Yuan Kangacdca312011-07-15 11:21:42 +08004038 /* ablkcipher descriptor */
4039 {
4040 .name = "cbc(aes)",
4041 .driver_name = "cbc-aes-caam",
4042 .blocksize = AES_BLOCK_SIZE,
Catalin Vasile7222d1a2014-10-31 12:45:38 +02004043 .type = CRYPTO_ALG_TYPE_GIVCIPHER,
Yuan Kangacdca312011-07-15 11:21:42 +08004044 .template_ablkcipher = {
4045 .setkey = ablkcipher_setkey,
4046 .encrypt = ablkcipher_encrypt,
4047 .decrypt = ablkcipher_decrypt,
Catalin Vasile7222d1a2014-10-31 12:45:38 +02004048 .givencrypt = ablkcipher_givencrypt,
4049 .geniv = "<built-in>",
Yuan Kangacdca312011-07-15 11:21:42 +08004050 .min_keysize = AES_MIN_KEY_SIZE,
4051 .max_keysize = AES_MAX_KEY_SIZE,
4052 .ivsize = AES_BLOCK_SIZE,
4053 },
4054 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
4055 },
4056 {
4057 .name = "cbc(des3_ede)",
4058 .driver_name = "cbc-3des-caam",
4059 .blocksize = DES3_EDE_BLOCK_SIZE,
Catalin Vasileff2c3a32014-11-11 16:18:13 +02004060 .type = CRYPTO_ALG_TYPE_GIVCIPHER,
Yuan Kangacdca312011-07-15 11:21:42 +08004061 .template_ablkcipher = {
4062 .setkey = ablkcipher_setkey,
4063 .encrypt = ablkcipher_encrypt,
4064 .decrypt = ablkcipher_decrypt,
Catalin Vasileff2c3a32014-11-11 16:18:13 +02004065 .givencrypt = ablkcipher_givencrypt,
4066 .geniv = "<built-in>",
Yuan Kangacdca312011-07-15 11:21:42 +08004067 .min_keysize = DES3_EDE_KEY_SIZE,
4068 .max_keysize = DES3_EDE_KEY_SIZE,
4069 .ivsize = DES3_EDE_BLOCK_SIZE,
4070 },
4071 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
4072 },
4073 {
4074 .name = "cbc(des)",
4075 .driver_name = "cbc-des-caam",
4076 .blocksize = DES_BLOCK_SIZE,
Catalin Vasileff2c3a32014-11-11 16:18:13 +02004077 .type = CRYPTO_ALG_TYPE_GIVCIPHER,
Yuan Kangacdca312011-07-15 11:21:42 +08004078 .template_ablkcipher = {
4079 .setkey = ablkcipher_setkey,
4080 .encrypt = ablkcipher_encrypt,
4081 .decrypt = ablkcipher_decrypt,
Catalin Vasileff2c3a32014-11-11 16:18:13 +02004082 .givencrypt = ablkcipher_givencrypt,
4083 .geniv = "<built-in>",
Yuan Kangacdca312011-07-15 11:21:42 +08004084 .min_keysize = DES_KEY_SIZE,
4085 .max_keysize = DES_KEY_SIZE,
4086 .ivsize = DES_BLOCK_SIZE,
4087 },
4088 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
Catalin Vasile2b22f6c2014-10-31 12:45:35 +02004089 },
4090 {
4091 .name = "ctr(aes)",
4092 .driver_name = "ctr-aes-caam",
4093 .blocksize = 1,
4094 .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
4095 .template_ablkcipher = {
4096 .setkey = ablkcipher_setkey,
4097 .encrypt = ablkcipher_encrypt,
4098 .decrypt = ablkcipher_decrypt,
4099 .geniv = "chainiv",
4100 .min_keysize = AES_MIN_KEY_SIZE,
4101 .max_keysize = AES_MAX_KEY_SIZE,
4102 .ivsize = AES_BLOCK_SIZE,
4103 },
4104 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CTR_MOD128,
Catalin Vasilea5f57cf2014-10-31 12:45:36 +02004105 },
4106 {
4107 .name = "rfc3686(ctr(aes))",
4108 .driver_name = "rfc3686-ctr-aes-caam",
4109 .blocksize = 1,
Catalin Vasile7222d1a2014-10-31 12:45:38 +02004110 .type = CRYPTO_ALG_TYPE_GIVCIPHER,
Catalin Vasilea5f57cf2014-10-31 12:45:36 +02004111 .template_ablkcipher = {
4112 .setkey = ablkcipher_setkey,
4113 .encrypt = ablkcipher_encrypt,
4114 .decrypt = ablkcipher_decrypt,
Catalin Vasile7222d1a2014-10-31 12:45:38 +02004115 .givencrypt = ablkcipher_givencrypt,
4116 .geniv = "<built-in>",
Catalin Vasilea5f57cf2014-10-31 12:45:36 +02004117 .min_keysize = AES_MIN_KEY_SIZE +
4118 CTR_RFC3686_NONCE_SIZE,
4119 .max_keysize = AES_MAX_KEY_SIZE +
4120 CTR_RFC3686_NONCE_SIZE,
4121 .ivsize = CTR_RFC3686_IV_SIZE,
4122 },
4123 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CTR_MOD128,
Yuan Kangacdca312011-07-15 11:21:42 +08004124 }
Kim Phillips8e8ec592011-03-13 16:54:26 +08004125};
4126
4127struct caam_crypto_alg {
4128 struct list_head entry;
Kim Phillips8e8ec592011-03-13 16:54:26 +08004129 int class1_alg_type;
4130 int class2_alg_type;
4131 int alg_op;
4132 struct crypto_alg crypto_alg;
4133};
4134
4135static int caam_cra_init(struct crypto_tfm *tfm)
4136{
4137 struct crypto_alg *alg = tfm->__crt_alg;
4138 struct caam_crypto_alg *caam_alg =
4139 container_of(alg, struct caam_crypto_alg, crypto_alg);
4140 struct caam_ctx *ctx = crypto_tfm_ctx(tfm);
Kim Phillips8e8ec592011-03-13 16:54:26 +08004141
Ruchika Guptacfc6f112013-10-25 12:01:03 +05304142 ctx->jrdev = caam_jr_alloc();
4143 if (IS_ERR(ctx->jrdev)) {
4144 pr_err("Job Ring Device allocation for transform failed\n");
4145 return PTR_ERR(ctx->jrdev);
4146 }
Kim Phillips8e8ec592011-03-13 16:54:26 +08004147
4148 /* copy descriptor header template value */
4149 ctx->class1_alg_type = OP_TYPE_CLASS1_ALG | caam_alg->class1_alg_type;
4150 ctx->class2_alg_type = OP_TYPE_CLASS2_ALG | caam_alg->class2_alg_type;
4151 ctx->alg_op = OP_TYPE_CLASS2_ALG | caam_alg->alg_op;
4152
4153 return 0;
4154}
4155
4156static void caam_cra_exit(struct crypto_tfm *tfm)
4157{
4158 struct caam_ctx *ctx = crypto_tfm_ctx(tfm);
4159
Yuan Kang1acebad2011-07-15 11:21:42 +08004160 if (ctx->sh_desc_enc_dma &&
4161 !dma_mapping_error(ctx->jrdev, ctx->sh_desc_enc_dma))
4162 dma_unmap_single(ctx->jrdev, ctx->sh_desc_enc_dma,
4163 desc_bytes(ctx->sh_desc_enc), DMA_TO_DEVICE);
4164 if (ctx->sh_desc_dec_dma &&
4165 !dma_mapping_error(ctx->jrdev, ctx->sh_desc_dec_dma))
4166 dma_unmap_single(ctx->jrdev, ctx->sh_desc_dec_dma,
4167 desc_bytes(ctx->sh_desc_dec), DMA_TO_DEVICE);
4168 if (ctx->sh_desc_givenc_dma &&
4169 !dma_mapping_error(ctx->jrdev, ctx->sh_desc_givenc_dma))
4170 dma_unmap_single(ctx->jrdev, ctx->sh_desc_givenc_dma,
4171 desc_bytes(ctx->sh_desc_givenc),
Kim Phillips4427b1b2011-05-14 22:08:17 -05004172 DMA_TO_DEVICE);
Horia Geantaec31eed2014-03-14 17:48:30 +02004173 if (ctx->key_dma &&
4174 !dma_mapping_error(ctx->jrdev, ctx->key_dma))
4175 dma_unmap_single(ctx->jrdev, ctx->key_dma,
4176 ctx->enckeylen + ctx->split_key_pad_len,
4177 DMA_TO_DEVICE);
Ruchika Guptacfc6f112013-10-25 12:01:03 +05304178
4179 caam_jr_free(ctx->jrdev);
Kim Phillips8e8ec592011-03-13 16:54:26 +08004180}
4181
4182static void __exit caam_algapi_exit(void)
4183{
4184
Kim Phillips8e8ec592011-03-13 16:54:26 +08004185 struct caam_crypto_alg *t_alg, *n;
Kim Phillips8e8ec592011-03-13 16:54:26 +08004186
Ruchika Guptacfc6f112013-10-25 12:01:03 +05304187 if (!alg_list.next)
Kim Phillips8e8ec592011-03-13 16:54:26 +08004188 return;
4189
Ruchika Guptacfc6f112013-10-25 12:01:03 +05304190 list_for_each_entry_safe(t_alg, n, &alg_list, entry) {
Kim Phillips8e8ec592011-03-13 16:54:26 +08004191 crypto_unregister_alg(&t_alg->crypto_alg);
4192 list_del(&t_alg->entry);
4193 kfree(t_alg);
4194 }
Kim Phillips8e8ec592011-03-13 16:54:26 +08004195}
4196
Ruchika Guptacfc6f112013-10-25 12:01:03 +05304197static struct caam_crypto_alg *caam_alg_alloc(struct caam_alg_template
Kim Phillips8e8ec592011-03-13 16:54:26 +08004198 *template)
4199{
4200 struct caam_crypto_alg *t_alg;
4201 struct crypto_alg *alg;
4202
4203 t_alg = kzalloc(sizeof(struct caam_crypto_alg), GFP_KERNEL);
4204 if (!t_alg) {
Ruchika Guptacfc6f112013-10-25 12:01:03 +05304205 pr_err("failed to allocate t_alg\n");
Kim Phillips8e8ec592011-03-13 16:54:26 +08004206 return ERR_PTR(-ENOMEM);
4207 }
4208
4209 alg = &t_alg->crypto_alg;
4210
4211 snprintf(alg->cra_name, CRYPTO_MAX_ALG_NAME, "%s", template->name);
4212 snprintf(alg->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s",
4213 template->driver_name);
4214 alg->cra_module = THIS_MODULE;
4215 alg->cra_init = caam_cra_init;
4216 alg->cra_exit = caam_cra_exit;
4217 alg->cra_priority = CAAM_CRA_PRIORITY;
Kim Phillips8e8ec592011-03-13 16:54:26 +08004218 alg->cra_blocksize = template->blocksize;
4219 alg->cra_alignmask = 0;
Kim Phillips8e8ec592011-03-13 16:54:26 +08004220 alg->cra_ctxsize = sizeof(struct caam_ctx);
Nikos Mavrogiannopoulosd912bb72011-11-01 13:39:56 +01004221 alg->cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY |
4222 template->type;
Yuan Kang885e9e22011-07-15 11:21:41 +08004223 switch (template->type) {
Catalin Vasile7222d1a2014-10-31 12:45:38 +02004224 case CRYPTO_ALG_TYPE_GIVCIPHER:
4225 alg->cra_type = &crypto_givcipher_type;
4226 alg->cra_ablkcipher = template->template_ablkcipher;
4227 break;
Yuan Kangacdca312011-07-15 11:21:42 +08004228 case CRYPTO_ALG_TYPE_ABLKCIPHER:
4229 alg->cra_type = &crypto_ablkcipher_type;
4230 alg->cra_ablkcipher = template->template_ablkcipher;
4231 break;
Yuan Kang885e9e22011-07-15 11:21:41 +08004232 case CRYPTO_ALG_TYPE_AEAD:
4233 alg->cra_type = &crypto_aead_type;
4234 alg->cra_aead = template->template_aead;
4235 break;
4236 }
Kim Phillips8e8ec592011-03-13 16:54:26 +08004237
4238 t_alg->class1_alg_type = template->class1_alg_type;
4239 t_alg->class2_alg_type = template->class2_alg_type;
4240 t_alg->alg_op = template->alg_op;
Kim Phillips8e8ec592011-03-13 16:54:26 +08004241
4242 return t_alg;
4243}
4244
4245static int __init caam_algapi_init(void)
4246{
Ruchika Gupta35af6402014-07-07 10:42:12 +05304247 struct device_node *dev_node;
4248 struct platform_device *pdev;
4249 struct device *ctrldev;
4250 void *priv;
Kim Phillips8e8ec592011-03-13 16:54:26 +08004251 int i = 0, err = 0;
4252
Ruchika Gupta35af6402014-07-07 10:42:12 +05304253 dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec-v4.0");
4254 if (!dev_node) {
4255 dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec4.0");
4256 if (!dev_node)
4257 return -ENODEV;
4258 }
4259
4260 pdev = of_find_device_by_node(dev_node);
4261 if (!pdev) {
4262 of_node_put(dev_node);
4263 return -ENODEV;
4264 }
4265
4266 ctrldev = &pdev->dev;
4267 priv = dev_get_drvdata(ctrldev);
4268 of_node_put(dev_node);
4269
4270 /*
4271 * If priv is NULL, it's probably because the caam driver wasn't
4272 * properly initialized (e.g. RNG4 init failed). Thus, bail out here.
4273 */
4274 if (!priv)
4275 return -ENODEV;
4276
4277
Ruchika Guptacfc6f112013-10-25 12:01:03 +05304278 INIT_LIST_HEAD(&alg_list);
Kim Phillips8e8ec592011-03-13 16:54:26 +08004279
4280 /* register crypto algorithms the device supports */
4281 for (i = 0; i < ARRAY_SIZE(driver_algs); i++) {
4282 /* TODO: check if h/w supports alg */
4283 struct caam_crypto_alg *t_alg;
4284
Ruchika Guptacfc6f112013-10-25 12:01:03 +05304285 t_alg = caam_alg_alloc(&driver_algs[i]);
Kim Phillips8e8ec592011-03-13 16:54:26 +08004286 if (IS_ERR(t_alg)) {
4287 err = PTR_ERR(t_alg);
Ruchika Guptacfc6f112013-10-25 12:01:03 +05304288 pr_warn("%s alg allocation failed\n",
4289 driver_algs[i].driver_name);
Kim Phillips8e8ec592011-03-13 16:54:26 +08004290 continue;
4291 }
4292
4293 err = crypto_register_alg(&t_alg->crypto_alg);
4294 if (err) {
Ruchika Guptacfc6f112013-10-25 12:01:03 +05304295 pr_warn("%s alg registration failed\n",
Kim Phillips8e8ec592011-03-13 16:54:26 +08004296 t_alg->crypto_alg.cra_driver_name);
4297 kfree(t_alg);
Horia Geanta246bbed2013-03-20 16:31:58 +02004298 } else
Ruchika Guptacfc6f112013-10-25 12:01:03 +05304299 list_add_tail(&t_alg->entry, &alg_list);
Kim Phillips8e8ec592011-03-13 16:54:26 +08004300 }
Ruchika Guptacfc6f112013-10-25 12:01:03 +05304301 if (!list_empty(&alg_list))
4302 pr_info("caam algorithms registered in /proc/crypto\n");
Kim Phillips8e8ec592011-03-13 16:54:26 +08004303
4304 return err;
4305}
4306
4307module_init(caam_algapi_init);
4308module_exit(caam_algapi_exit);
4309
4310MODULE_LICENSE("GPL");
4311MODULE_DESCRIPTION("FSL CAAM support for crypto API");
4312MODULE_AUTHOR("Freescale Semiconductor - NMG/STC");