blob: b7ee8d30147df02371337a0c48501e9c02f7a9e7 [file] [log] [blame]
Kim Phillips9c4a7962008-06-23 19:50:15 +08001/*
2 * talitos - Freescale Integrated Security Engine (SEC) device driver
3 *
Kim Phillips5228f0f2011-07-15 11:21:38 +08004 * Copyright (c) 2008-2011 Freescale Semiconductor, Inc.
Kim Phillips9c4a7962008-06-23 19:50:15 +08005 *
6 * Scatterlist Crypto API glue code copied from files with the following:
7 * Copyright (c) 2006-2007 Herbert Xu <herbert@gondor.apana.org.au>
8 *
9 * Crypto algorithm registration code copied from hifn driver:
10 * 2007+ Copyright (c) Evgeniy Polyakov <johnpol@2ka.mipt.ru>
11 * All rights reserved.
12 *
13 * This program is free software; you can redistribute it and/or modify
14 * it under the terms of the GNU General Public License as published by
15 * the Free Software Foundation; either version 2 of the License, or
16 * (at your option) any later version.
17 *
18 * This program is distributed in the hope that it will be useful,
19 * but WITHOUT ANY WARRANTY; without even the implied warranty of
20 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
21 * GNU General Public License for more details.
22 *
23 * You should have received a copy of the GNU General Public License
24 * along with this program; if not, write to the Free Software
25 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
26 */
27
28#include <linux/kernel.h>
29#include <linux/module.h>
30#include <linux/mod_devicetable.h>
31#include <linux/device.h>
32#include <linux/interrupt.h>
33#include <linux/crypto.h>
34#include <linux/hw_random.h>
Rob Herring5af50732013-09-17 14:28:33 -050035#include <linux/of_address.h>
36#include <linux/of_irq.h>
Kim Phillips9c4a7962008-06-23 19:50:15 +080037#include <linux/of_platform.h>
38#include <linux/dma-mapping.h>
39#include <linux/io.h>
40#include <linux/spinlock.h>
41#include <linux/rtnetlink.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090042#include <linux/slab.h>
Kim Phillips9c4a7962008-06-23 19:50:15 +080043
44#include <crypto/algapi.h>
45#include <crypto/aes.h>
Lee Nipper3952f172008-07-10 18:29:18 +080046#include <crypto/des.h>
Kim Phillips9c4a7962008-06-23 19:50:15 +080047#include <crypto/sha.h>
Lee Nipper497f2e62010-05-19 19:20:36 +100048#include <crypto/md5.h>
Herbert Xue98014a2015-05-11 17:47:48 +080049#include <crypto/internal/aead.h>
Kim Phillips9c4a7962008-06-23 19:50:15 +080050#include <crypto/authenc.h>
Lee Nipper4de9d0b2009-03-29 15:52:32 +080051#include <crypto/skcipher.h>
Lee Nipperacbf7c622010-05-19 19:19:33 +100052#include <crypto/hash.h>
53#include <crypto/internal/hash.h>
Lee Nipper4de9d0b2009-03-29 15:52:32 +080054#include <crypto/scatterwalk.h>
Kim Phillips9c4a7962008-06-23 19:50:15 +080055
56#include "talitos.h"
57
LEROY Christophe922f9dc2015-04-17 16:32:07 +020058static void to_talitos_ptr(struct talitos_ptr *ptr, dma_addr_t dma_addr,
59 bool is_sec1)
Kim Phillips81eb0242009-08-13 11:51:51 +100060{
LEROY Christopheedc6bd62015-04-17 16:31:53 +020061 ptr->ptr = cpu_to_be32(lower_32_bits(dma_addr));
LEROY Christophe922f9dc2015-04-17 16:32:07 +020062 if (!is_sec1)
63 ptr->eptr = upper_32_bits(dma_addr);
Kim Phillips81eb0242009-08-13 11:51:51 +100064}
65
Horia Geant?340ff602016-04-19 20:33:48 +030066static void copy_talitos_ptr(struct talitos_ptr *dst_ptr,
67 struct talitos_ptr *src_ptr, bool is_sec1)
68{
69 dst_ptr->ptr = src_ptr->ptr;
70 if (!is_sec1)
71 dst_ptr->eptr = src_ptr->eptr;
72}
73
Horia Geant?42e8b0d2015-05-11 20:04:56 +030074static void to_talitos_ptr_len(struct talitos_ptr *ptr, unsigned int len,
LEROY Christophe922f9dc2015-04-17 16:32:07 +020075 bool is_sec1)
LEROY Christophe538caf82015-04-17 16:31:59 +020076{
LEROY Christophe922f9dc2015-04-17 16:32:07 +020077 if (is_sec1) {
78 ptr->res = 0;
79 ptr->len1 = cpu_to_be16(len);
80 } else {
81 ptr->len = cpu_to_be16(len);
82 }
LEROY Christophe538caf82015-04-17 16:31:59 +020083}
84
LEROY Christophe922f9dc2015-04-17 16:32:07 +020085static unsigned short from_talitos_ptr_len(struct talitos_ptr *ptr,
86 bool is_sec1)
LEROY Christophe538caf82015-04-17 16:31:59 +020087{
LEROY Christophe922f9dc2015-04-17 16:32:07 +020088 if (is_sec1)
89 return be16_to_cpu(ptr->len1);
90 else
91 return be16_to_cpu(ptr->len);
LEROY Christophe538caf82015-04-17 16:31:59 +020092}
93
LEROY Christophe922f9dc2015-04-17 16:32:07 +020094static void to_talitos_ptr_extent_clear(struct talitos_ptr *ptr, bool is_sec1)
LEROY Christophe185eb792015-04-17 16:31:55 +020095{
LEROY Christophe922f9dc2015-04-17 16:32:07 +020096 if (!is_sec1)
97 ptr->j_extent = 0;
LEROY Christophe185eb792015-04-17 16:31:55 +020098}
99
Kim Phillips9c4a7962008-06-23 19:50:15 +0800100/*
101 * map virtual single (contiguous) pointer to h/w descriptor pointer
102 */
103static void map_single_talitos_ptr(struct device *dev,
LEROY Christopheedc6bd62015-04-17 16:31:53 +0200104 struct talitos_ptr *ptr,
Horia Geant?42e8b0d2015-05-11 20:04:56 +0300105 unsigned int len, void *data,
Kim Phillips9c4a7962008-06-23 19:50:15 +0800106 enum dma_data_direction dir)
107{
Kim Phillips81eb0242009-08-13 11:51:51 +1000108 dma_addr_t dma_addr = dma_map_single(dev, data, len, dir);
LEROY Christophe922f9dc2015-04-17 16:32:07 +0200109 struct talitos_private *priv = dev_get_drvdata(dev);
110 bool is_sec1 = has_ftr_sec1(priv);
Kim Phillips81eb0242009-08-13 11:51:51 +1000111
LEROY Christophe922f9dc2015-04-17 16:32:07 +0200112 to_talitos_ptr_len(ptr, len, is_sec1);
113 to_talitos_ptr(ptr, dma_addr, is_sec1);
114 to_talitos_ptr_extent_clear(ptr, is_sec1);
Kim Phillips9c4a7962008-06-23 19:50:15 +0800115}
116
117/*
118 * unmap bus single (contiguous) h/w descriptor pointer
119 */
120static void unmap_single_talitos_ptr(struct device *dev,
LEROY Christopheedc6bd62015-04-17 16:31:53 +0200121 struct talitos_ptr *ptr,
Kim Phillips9c4a7962008-06-23 19:50:15 +0800122 enum dma_data_direction dir)
123{
LEROY Christophe922f9dc2015-04-17 16:32:07 +0200124 struct talitos_private *priv = dev_get_drvdata(dev);
125 bool is_sec1 = has_ftr_sec1(priv);
126
LEROY Christopheedc6bd62015-04-17 16:31:53 +0200127 dma_unmap_single(dev, be32_to_cpu(ptr->ptr),
LEROY Christophe922f9dc2015-04-17 16:32:07 +0200128 from_talitos_ptr_len(ptr, is_sec1), dir);
Kim Phillips9c4a7962008-06-23 19:50:15 +0800129}
130
131static int reset_channel(struct device *dev, int ch)
132{
133 struct talitos_private *priv = dev_get_drvdata(dev);
134 unsigned int timeout = TALITOS_TIMEOUT;
LEROY Christophedd3c0982015-04-17 16:32:13 +0200135 bool is_sec1 = has_ftr_sec1(priv);
Kim Phillips9c4a7962008-06-23 19:50:15 +0800136
LEROY Christophedd3c0982015-04-17 16:32:13 +0200137 if (is_sec1) {
138 setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO,
139 TALITOS1_CCCR_LO_RESET);
Kim Phillips9c4a7962008-06-23 19:50:15 +0800140
LEROY Christophedd3c0982015-04-17 16:32:13 +0200141 while ((in_be32(priv->chan[ch].reg + TALITOS_CCCR_LO) &
142 TALITOS1_CCCR_LO_RESET) && --timeout)
143 cpu_relax();
144 } else {
145 setbits32(priv->chan[ch].reg + TALITOS_CCCR,
146 TALITOS2_CCCR_RESET);
147
148 while ((in_be32(priv->chan[ch].reg + TALITOS_CCCR) &
149 TALITOS2_CCCR_RESET) && --timeout)
150 cpu_relax();
151 }
Kim Phillips9c4a7962008-06-23 19:50:15 +0800152
153 if (timeout == 0) {
154 dev_err(dev, "failed to reset channel %d\n", ch);
155 return -EIO;
156 }
157
Kim Phillips81eb0242009-08-13 11:51:51 +1000158 /* set 36-bit addressing, done writeback enable and done IRQ enable */
Kim Phillipsad42d5f2011-11-21 16:13:27 +0800159 setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO, TALITOS_CCCR_LO_EAE |
Kim Phillips81eb0242009-08-13 11:51:51 +1000160 TALITOS_CCCR_LO_CDWE | TALITOS_CCCR_LO_CDIE);
Kim Phillips9c4a7962008-06-23 19:50:15 +0800161
Kim Phillipsfe5720e2008-10-12 20:33:14 +0800162 /* and ICCR writeback, if available */
163 if (priv->features & TALITOS_FTR_HW_AUTH_CHECK)
Kim Phillipsad42d5f2011-11-21 16:13:27 +0800164 setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO,
Kim Phillipsfe5720e2008-10-12 20:33:14 +0800165 TALITOS_CCCR_LO_IWSE);
166
Kim Phillips9c4a7962008-06-23 19:50:15 +0800167 return 0;
168}
169
170static int reset_device(struct device *dev)
171{
172 struct talitos_private *priv = dev_get_drvdata(dev);
173 unsigned int timeout = TALITOS_TIMEOUT;
LEROY Christophedd3c0982015-04-17 16:32:13 +0200174 bool is_sec1 = has_ftr_sec1(priv);
175 u32 mcr = is_sec1 ? TALITOS1_MCR_SWR : TALITOS2_MCR_SWR;
Kim Phillips9c4a7962008-06-23 19:50:15 +0800176
Kim Phillipsc3e337f2011-11-21 16:13:27 +0800177 setbits32(priv->reg + TALITOS_MCR, mcr);
Kim Phillips9c4a7962008-06-23 19:50:15 +0800178
LEROY Christophedd3c0982015-04-17 16:32:13 +0200179 while ((in_be32(priv->reg + TALITOS_MCR) & mcr)
Kim Phillips9c4a7962008-06-23 19:50:15 +0800180 && --timeout)
181 cpu_relax();
182
Kim Phillips2cdba3c2011-12-12 14:59:11 -0600183 if (priv->irq[1]) {
Kim Phillipsc3e337f2011-11-21 16:13:27 +0800184 mcr = TALITOS_MCR_RCA1 | TALITOS_MCR_RCA3;
185 setbits32(priv->reg + TALITOS_MCR, mcr);
186 }
187
Kim Phillips9c4a7962008-06-23 19:50:15 +0800188 if (timeout == 0) {
189 dev_err(dev, "failed to reset device\n");
190 return -EIO;
191 }
192
193 return 0;
194}
195
196/*
197 * Reset and initialize the device
198 */
199static int init_device(struct device *dev)
200{
201 struct talitos_private *priv = dev_get_drvdata(dev);
202 int ch, err;
LEROY Christophedd3c0982015-04-17 16:32:13 +0200203 bool is_sec1 = has_ftr_sec1(priv);
Kim Phillips9c4a7962008-06-23 19:50:15 +0800204
205 /*
206 * Master reset
207 * errata documentation: warning: certain SEC interrupts
208 * are not fully cleared by writing the MCR:SWR bit,
209 * set bit twice to completely reset
210 */
211 err = reset_device(dev);
212 if (err)
213 return err;
214
215 err = reset_device(dev);
216 if (err)
217 return err;
218
219 /* reset channels */
220 for (ch = 0; ch < priv->num_channels; ch++) {
221 err = reset_channel(dev, ch);
222 if (err)
223 return err;
224 }
225
226 /* enable channel done and error interrupts */
LEROY Christophedd3c0982015-04-17 16:32:13 +0200227 if (is_sec1) {
228 clrbits32(priv->reg + TALITOS_IMR, TALITOS1_IMR_INIT);
229 clrbits32(priv->reg + TALITOS_IMR_LO, TALITOS1_IMR_LO_INIT);
230 /* disable parity error check in DEU (erroneous? test vect.) */
231 setbits32(priv->reg_deu + TALITOS_EUICR, TALITOS1_DEUICR_KPE);
232 } else {
233 setbits32(priv->reg + TALITOS_IMR, TALITOS2_IMR_INIT);
234 setbits32(priv->reg + TALITOS_IMR_LO, TALITOS2_IMR_LO_INIT);
235 }
Kim Phillips9c4a7962008-06-23 19:50:15 +0800236
Kim Phillipsfe5720e2008-10-12 20:33:14 +0800237 /* disable integrity check error interrupts (use writeback instead) */
238 if (priv->features & TALITOS_FTR_HW_AUTH_CHECK)
LEROY Christophe5fa7fa12015-04-17 16:32:11 +0200239 setbits32(priv->reg_mdeu + TALITOS_EUICR_LO,
Kim Phillipsfe5720e2008-10-12 20:33:14 +0800240 TALITOS_MDEUICR_LO_ICE);
241
Kim Phillips9c4a7962008-06-23 19:50:15 +0800242 return 0;
243}
244
245/**
246 * talitos_submit - submits a descriptor to the device for processing
247 * @dev: the SEC device to be used
Kim Phillips5228f0f2011-07-15 11:21:38 +0800248 * @ch: the SEC device channel to be used
Kim Phillips9c4a7962008-06-23 19:50:15 +0800249 * @desc: the descriptor to be processed by the device
250 * @callback: whom to call when processing is complete
251 * @context: a handle for use by caller (optional)
252 *
253 * desc must contain valid dma-mapped (bus physical) address pointers.
254 * callback must check err and feedback in descriptor header
255 * for device processing status.
256 */
Horia Geanta865d5062012-07-03 19:16:52 +0300257int talitos_submit(struct device *dev, int ch, struct talitos_desc *desc,
258 void (*callback)(struct device *dev,
259 struct talitos_desc *desc,
260 void *context, int error),
261 void *context)
Kim Phillips9c4a7962008-06-23 19:50:15 +0800262{
263 struct talitos_private *priv = dev_get_drvdata(dev);
264 struct talitos_request *request;
Kim Phillips5228f0f2011-07-15 11:21:38 +0800265 unsigned long flags;
Kim Phillips9c4a7962008-06-23 19:50:15 +0800266 int head;
LEROY Christophe7d607c6a2015-04-17 16:32:09 +0200267 bool is_sec1 = has_ftr_sec1(priv);
Kim Phillips9c4a7962008-06-23 19:50:15 +0800268
Kim Phillips4b9926282009-08-13 11:50:38 +1000269 spin_lock_irqsave(&priv->chan[ch].head_lock, flags);
Kim Phillips9c4a7962008-06-23 19:50:15 +0800270
Kim Phillips4b9926282009-08-13 11:50:38 +1000271 if (!atomic_inc_not_zero(&priv->chan[ch].submit_count)) {
Kim Phillipsec6644d2008-07-17 20:16:40 +0800272 /* h/w fifo is full */
Kim Phillips4b9926282009-08-13 11:50:38 +1000273 spin_unlock_irqrestore(&priv->chan[ch].head_lock, flags);
Kim Phillips9c4a7962008-06-23 19:50:15 +0800274 return -EAGAIN;
275 }
276
Kim Phillips4b9926282009-08-13 11:50:38 +1000277 head = priv->chan[ch].head;
278 request = &priv->chan[ch].fifo[head];
Kim Phillipsec6644d2008-07-17 20:16:40 +0800279
Kim Phillips9c4a7962008-06-23 19:50:15 +0800280 /* map descriptor and save caller data */
LEROY Christophe7d607c6a2015-04-17 16:32:09 +0200281 if (is_sec1) {
282 desc->hdr1 = desc->hdr;
283 desc->next_desc = 0;
284 request->dma_desc = dma_map_single(dev, &desc->hdr1,
285 TALITOS_DESC_SIZE,
286 DMA_BIDIRECTIONAL);
287 } else {
288 request->dma_desc = dma_map_single(dev, desc,
289 TALITOS_DESC_SIZE,
290 DMA_BIDIRECTIONAL);
291 }
Kim Phillips9c4a7962008-06-23 19:50:15 +0800292 request->callback = callback;
293 request->context = context;
294
295 /* increment fifo head */
Kim Phillips4b9926282009-08-13 11:50:38 +1000296 priv->chan[ch].head = (priv->chan[ch].head + 1) & (priv->fifo_len - 1);
Kim Phillips9c4a7962008-06-23 19:50:15 +0800297
298 smp_wmb();
299 request->desc = desc;
300
301 /* GO! */
302 wmb();
Kim Phillipsad42d5f2011-11-21 16:13:27 +0800303 out_be32(priv->chan[ch].reg + TALITOS_FF,
304 upper_32_bits(request->dma_desc));
305 out_be32(priv->chan[ch].reg + TALITOS_FF_LO,
Kim Phillipsa7524472010-09-23 15:56:38 +0800306 lower_32_bits(request->dma_desc));
Kim Phillips9c4a7962008-06-23 19:50:15 +0800307
Kim Phillips4b9926282009-08-13 11:50:38 +1000308 spin_unlock_irqrestore(&priv->chan[ch].head_lock, flags);
Kim Phillips9c4a7962008-06-23 19:50:15 +0800309
310 return -EINPROGRESS;
311}
Horia Geanta865d5062012-07-03 19:16:52 +0300312EXPORT_SYMBOL(talitos_submit);
Kim Phillips9c4a7962008-06-23 19:50:15 +0800313
314/*
315 * process what was done, notify callback of error if not
316 */
317static void flush_channel(struct device *dev, int ch, int error, int reset_ch)
318{
319 struct talitos_private *priv = dev_get_drvdata(dev);
320 struct talitos_request *request, saved_req;
321 unsigned long flags;
322 int tail, status;
LEROY Christophe7d607c6a2015-04-17 16:32:09 +0200323 bool is_sec1 = has_ftr_sec1(priv);
Kim Phillips9c4a7962008-06-23 19:50:15 +0800324
Kim Phillips4b9926282009-08-13 11:50:38 +1000325 spin_lock_irqsave(&priv->chan[ch].tail_lock, flags);
Kim Phillips9c4a7962008-06-23 19:50:15 +0800326
Kim Phillips4b9926282009-08-13 11:50:38 +1000327 tail = priv->chan[ch].tail;
328 while (priv->chan[ch].fifo[tail].desc) {
LEROY Christophe7d607c6a2015-04-17 16:32:09 +0200329 __be32 hdr;
330
Kim Phillips4b9926282009-08-13 11:50:38 +1000331 request = &priv->chan[ch].fifo[tail];
Kim Phillips9c4a7962008-06-23 19:50:15 +0800332
333 /* descriptors with their done bits set don't get the error */
334 rmb();
LEROY Christophe7d607c6a2015-04-17 16:32:09 +0200335 hdr = is_sec1 ? request->desc->hdr1 : request->desc->hdr;
336
337 if ((hdr & DESC_HDR_DONE) == DESC_HDR_DONE)
Kim Phillips9c4a7962008-06-23 19:50:15 +0800338 status = 0;
Lee Nipperca38a812008-12-20 17:09:25 +1100339 else
Kim Phillips9c4a7962008-06-23 19:50:15 +0800340 if (!error)
341 break;
342 else
343 status = error;
344
345 dma_unmap_single(dev, request->dma_desc,
LEROY Christophe7d607c6a2015-04-17 16:32:09 +0200346 TALITOS_DESC_SIZE,
Kim Phillipse938e462009-03-29 15:53:23 +0800347 DMA_BIDIRECTIONAL);
Kim Phillips9c4a7962008-06-23 19:50:15 +0800348
349 /* copy entries so we can call callback outside lock */
350 saved_req.desc = request->desc;
351 saved_req.callback = request->callback;
352 saved_req.context = request->context;
353
354 /* release request entry in fifo */
355 smp_wmb();
356 request->desc = NULL;
357
358 /* increment fifo tail */
Kim Phillips4b9926282009-08-13 11:50:38 +1000359 priv->chan[ch].tail = (tail + 1) & (priv->fifo_len - 1);
Kim Phillips9c4a7962008-06-23 19:50:15 +0800360
Kim Phillips4b9926282009-08-13 11:50:38 +1000361 spin_unlock_irqrestore(&priv->chan[ch].tail_lock, flags);
Kim Phillipsec6644d2008-07-17 20:16:40 +0800362
Kim Phillips4b9926282009-08-13 11:50:38 +1000363 atomic_dec(&priv->chan[ch].submit_count);
Kim Phillipsec6644d2008-07-17 20:16:40 +0800364
Kim Phillips9c4a7962008-06-23 19:50:15 +0800365 saved_req.callback(dev, saved_req.desc, saved_req.context,
366 status);
367 /* channel may resume processing in single desc error case */
368 if (error && !reset_ch && status == error)
369 return;
Kim Phillips4b9926282009-08-13 11:50:38 +1000370 spin_lock_irqsave(&priv->chan[ch].tail_lock, flags);
371 tail = priv->chan[ch].tail;
Kim Phillips9c4a7962008-06-23 19:50:15 +0800372 }
373
Kim Phillips4b9926282009-08-13 11:50:38 +1000374 spin_unlock_irqrestore(&priv->chan[ch].tail_lock, flags);
Kim Phillips9c4a7962008-06-23 19:50:15 +0800375}
376
377/*
378 * process completed requests for channels that have done status
379 */
LEROY Christophedd3c0982015-04-17 16:32:13 +0200380#define DEF_TALITOS1_DONE(name, ch_done_mask) \
381static void talitos1_done_##name(unsigned long data) \
382{ \
383 struct device *dev = (struct device *)data; \
384 struct talitos_private *priv = dev_get_drvdata(dev); \
385 unsigned long flags; \
386 \
387 if (ch_done_mask & 0x10000000) \
388 flush_channel(dev, 0, 0, 0); \
389 if (priv->num_channels == 1) \
390 goto out; \
391 if (ch_done_mask & 0x40000000) \
392 flush_channel(dev, 1, 0, 0); \
393 if (ch_done_mask & 0x00010000) \
394 flush_channel(dev, 2, 0, 0); \
395 if (ch_done_mask & 0x00040000) \
396 flush_channel(dev, 3, 0, 0); \
397 \
398out: \
399 /* At this point, all completed channels have been processed */ \
400 /* Unmask done interrupts for channels completed later on. */ \
401 spin_lock_irqsave(&priv->reg_lock, flags); \
402 clrbits32(priv->reg + TALITOS_IMR, ch_done_mask); \
403 clrbits32(priv->reg + TALITOS_IMR_LO, TALITOS1_IMR_LO_INIT); \
404 spin_unlock_irqrestore(&priv->reg_lock, flags); \
405}
406
407DEF_TALITOS1_DONE(4ch, TALITOS1_ISR_4CHDONE)
408
409#define DEF_TALITOS2_DONE(name, ch_done_mask) \
410static void talitos2_done_##name(unsigned long data) \
Kim Phillipsc3e337f2011-11-21 16:13:27 +0800411{ \
412 struct device *dev = (struct device *)data; \
413 struct talitos_private *priv = dev_get_drvdata(dev); \
Horia Geanta511d63c2012-03-30 17:49:53 +0300414 unsigned long flags; \
Kim Phillipsc3e337f2011-11-21 16:13:27 +0800415 \
416 if (ch_done_mask & 1) \
417 flush_channel(dev, 0, 0, 0); \
418 if (priv->num_channels == 1) \
419 goto out; \
420 if (ch_done_mask & (1 << 2)) \
421 flush_channel(dev, 1, 0, 0); \
422 if (ch_done_mask & (1 << 4)) \
423 flush_channel(dev, 2, 0, 0); \
424 if (ch_done_mask & (1 << 6)) \
425 flush_channel(dev, 3, 0, 0); \
426 \
427out: \
428 /* At this point, all completed channels have been processed */ \
429 /* Unmask done interrupts for channels completed later on. */ \
Horia Geanta511d63c2012-03-30 17:49:53 +0300430 spin_lock_irqsave(&priv->reg_lock, flags); \
Kim Phillipsc3e337f2011-11-21 16:13:27 +0800431 setbits32(priv->reg + TALITOS_IMR, ch_done_mask); \
LEROY Christophedd3c0982015-04-17 16:32:13 +0200432 setbits32(priv->reg + TALITOS_IMR_LO, TALITOS2_IMR_LO_INIT); \
Horia Geanta511d63c2012-03-30 17:49:53 +0300433 spin_unlock_irqrestore(&priv->reg_lock, flags); \
Kim Phillips9c4a7962008-06-23 19:50:15 +0800434}
LEROY Christophedd3c0982015-04-17 16:32:13 +0200435
436DEF_TALITOS2_DONE(4ch, TALITOS2_ISR_4CHDONE)
437DEF_TALITOS2_DONE(ch0_2, TALITOS2_ISR_CH_0_2_DONE)
438DEF_TALITOS2_DONE(ch1_3, TALITOS2_ISR_CH_1_3_DONE)
Kim Phillips9c4a7962008-06-23 19:50:15 +0800439
440/*
441 * locate current (offending) descriptor
442 */
Kim Phillips3e721ae2011-10-21 15:20:28 +0200443static u32 current_desc_hdr(struct device *dev, int ch)
Kim Phillips9c4a7962008-06-23 19:50:15 +0800444{
445 struct talitos_private *priv = dev_get_drvdata(dev);
Horia Geantab62ffd82013-11-13 12:20:37 +0200446 int tail, iter;
Kim Phillips9c4a7962008-06-23 19:50:15 +0800447 dma_addr_t cur_desc;
448
Horia Geantab62ffd82013-11-13 12:20:37 +0200449 cur_desc = ((u64)in_be32(priv->chan[ch].reg + TALITOS_CDPR)) << 32;
450 cur_desc |= in_be32(priv->chan[ch].reg + TALITOS_CDPR_LO);
Kim Phillips9c4a7962008-06-23 19:50:15 +0800451
Horia Geantab62ffd82013-11-13 12:20:37 +0200452 if (!cur_desc) {
453 dev_err(dev, "CDPR is NULL, giving up search for offending descriptor\n");
454 return 0;
455 }
456
457 tail = priv->chan[ch].tail;
458
459 iter = tail;
460 while (priv->chan[ch].fifo[iter].dma_desc != cur_desc) {
461 iter = (iter + 1) & (priv->fifo_len - 1);
462 if (iter == tail) {
Kim Phillips9c4a7962008-06-23 19:50:15 +0800463 dev_err(dev, "couldn't locate current descriptor\n");
Kim Phillips3e721ae2011-10-21 15:20:28 +0200464 return 0;
Kim Phillips9c4a7962008-06-23 19:50:15 +0800465 }
466 }
467
Horia Geantab62ffd82013-11-13 12:20:37 +0200468 return priv->chan[ch].fifo[iter].desc->hdr;
Kim Phillips9c4a7962008-06-23 19:50:15 +0800469}
470
471/*
472 * user diagnostics; report root cause of error based on execution unit status
473 */
Kim Phillips3e721ae2011-10-21 15:20:28 +0200474static void report_eu_error(struct device *dev, int ch, u32 desc_hdr)
Kim Phillips9c4a7962008-06-23 19:50:15 +0800475{
476 struct talitos_private *priv = dev_get_drvdata(dev);
477 int i;
478
Kim Phillips3e721ae2011-10-21 15:20:28 +0200479 if (!desc_hdr)
Kim Phillipsad42d5f2011-11-21 16:13:27 +0800480 desc_hdr = in_be32(priv->chan[ch].reg + TALITOS_DESCBUF);
Kim Phillips3e721ae2011-10-21 15:20:28 +0200481
482 switch (desc_hdr & DESC_HDR_SEL0_MASK) {
Kim Phillips9c4a7962008-06-23 19:50:15 +0800483 case DESC_HDR_SEL0_AFEU:
484 dev_err(dev, "AFEUISR 0x%08x_%08x\n",
LEROY Christophe5fa7fa12015-04-17 16:32:11 +0200485 in_be32(priv->reg_afeu + TALITOS_EUISR),
486 in_be32(priv->reg_afeu + TALITOS_EUISR_LO));
Kim Phillips9c4a7962008-06-23 19:50:15 +0800487 break;
488 case DESC_HDR_SEL0_DEU:
489 dev_err(dev, "DEUISR 0x%08x_%08x\n",
LEROY Christophe5fa7fa12015-04-17 16:32:11 +0200490 in_be32(priv->reg_deu + TALITOS_EUISR),
491 in_be32(priv->reg_deu + TALITOS_EUISR_LO));
Kim Phillips9c4a7962008-06-23 19:50:15 +0800492 break;
493 case DESC_HDR_SEL0_MDEUA:
494 case DESC_HDR_SEL0_MDEUB:
495 dev_err(dev, "MDEUISR 0x%08x_%08x\n",
LEROY Christophe5fa7fa12015-04-17 16:32:11 +0200496 in_be32(priv->reg_mdeu + TALITOS_EUISR),
497 in_be32(priv->reg_mdeu + TALITOS_EUISR_LO));
Kim Phillips9c4a7962008-06-23 19:50:15 +0800498 break;
499 case DESC_HDR_SEL0_RNG:
500 dev_err(dev, "RNGUISR 0x%08x_%08x\n",
LEROY Christophe5fa7fa12015-04-17 16:32:11 +0200501 in_be32(priv->reg_rngu + TALITOS_ISR),
502 in_be32(priv->reg_rngu + TALITOS_ISR_LO));
Kim Phillips9c4a7962008-06-23 19:50:15 +0800503 break;
504 case DESC_HDR_SEL0_PKEU:
505 dev_err(dev, "PKEUISR 0x%08x_%08x\n",
LEROY Christophe5fa7fa12015-04-17 16:32:11 +0200506 in_be32(priv->reg_pkeu + TALITOS_EUISR),
507 in_be32(priv->reg_pkeu + TALITOS_EUISR_LO));
Kim Phillips9c4a7962008-06-23 19:50:15 +0800508 break;
509 case DESC_HDR_SEL0_AESU:
510 dev_err(dev, "AESUISR 0x%08x_%08x\n",
LEROY Christophe5fa7fa12015-04-17 16:32:11 +0200511 in_be32(priv->reg_aesu + TALITOS_EUISR),
512 in_be32(priv->reg_aesu + TALITOS_EUISR_LO));
Kim Phillips9c4a7962008-06-23 19:50:15 +0800513 break;
514 case DESC_HDR_SEL0_CRCU:
515 dev_err(dev, "CRCUISR 0x%08x_%08x\n",
LEROY Christophe5fa7fa12015-04-17 16:32:11 +0200516 in_be32(priv->reg_crcu + TALITOS_EUISR),
517 in_be32(priv->reg_crcu + TALITOS_EUISR_LO));
Kim Phillips9c4a7962008-06-23 19:50:15 +0800518 break;
519 case DESC_HDR_SEL0_KEU:
520 dev_err(dev, "KEUISR 0x%08x_%08x\n",
LEROY Christophe5fa7fa12015-04-17 16:32:11 +0200521 in_be32(priv->reg_pkeu + TALITOS_EUISR),
522 in_be32(priv->reg_pkeu + TALITOS_EUISR_LO));
Kim Phillips9c4a7962008-06-23 19:50:15 +0800523 break;
524 }
525
Kim Phillips3e721ae2011-10-21 15:20:28 +0200526 switch (desc_hdr & DESC_HDR_SEL1_MASK) {
Kim Phillips9c4a7962008-06-23 19:50:15 +0800527 case DESC_HDR_SEL1_MDEUA:
528 case DESC_HDR_SEL1_MDEUB:
529 dev_err(dev, "MDEUISR 0x%08x_%08x\n",
LEROY Christophe5fa7fa12015-04-17 16:32:11 +0200530 in_be32(priv->reg_mdeu + TALITOS_EUISR),
531 in_be32(priv->reg_mdeu + TALITOS_EUISR_LO));
Kim Phillips9c4a7962008-06-23 19:50:15 +0800532 break;
533 case DESC_HDR_SEL1_CRCU:
534 dev_err(dev, "CRCUISR 0x%08x_%08x\n",
LEROY Christophe5fa7fa12015-04-17 16:32:11 +0200535 in_be32(priv->reg_crcu + TALITOS_EUISR),
536 in_be32(priv->reg_crcu + TALITOS_EUISR_LO));
Kim Phillips9c4a7962008-06-23 19:50:15 +0800537 break;
538 }
539
540 for (i = 0; i < 8; i++)
541 dev_err(dev, "DESCBUF 0x%08x_%08x\n",
Kim Phillipsad42d5f2011-11-21 16:13:27 +0800542 in_be32(priv->chan[ch].reg + TALITOS_DESCBUF + 8*i),
543 in_be32(priv->chan[ch].reg + TALITOS_DESCBUF_LO + 8*i));
Kim Phillips9c4a7962008-06-23 19:50:15 +0800544}
545
546/*
547 * recover from error interrupts
548 */
Kim Phillips5e718a02011-12-12 14:59:12 -0600549static void talitos_error(struct device *dev, u32 isr, u32 isr_lo)
Kim Phillips9c4a7962008-06-23 19:50:15 +0800550{
Kim Phillips9c4a7962008-06-23 19:50:15 +0800551 struct talitos_private *priv = dev_get_drvdata(dev);
552 unsigned int timeout = TALITOS_TIMEOUT;
LEROY Christophedd3c0982015-04-17 16:32:13 +0200553 int ch, error, reset_dev = 0;
Horia Geant?42e8b0d2015-05-11 20:04:56 +0300554 u32 v_lo;
LEROY Christophedd3c0982015-04-17 16:32:13 +0200555 bool is_sec1 = has_ftr_sec1(priv);
556 int reset_ch = is_sec1 ? 1 : 0; /* only SEC2 supports continuation */
Kim Phillips9c4a7962008-06-23 19:50:15 +0800557
558 for (ch = 0; ch < priv->num_channels; ch++) {
559 /* skip channels without errors */
LEROY Christophedd3c0982015-04-17 16:32:13 +0200560 if (is_sec1) {
561 /* bits 29, 31, 17, 19 */
562 if (!(isr & (1 << (29 + (ch & 1) * 2 - (ch & 2) * 6))))
563 continue;
564 } else {
565 if (!(isr & (1 << (ch * 2 + 1))))
566 continue;
567 }
Kim Phillips9c4a7962008-06-23 19:50:15 +0800568
569 error = -EINVAL;
570
Kim Phillipsad42d5f2011-11-21 16:13:27 +0800571 v_lo = in_be32(priv->chan[ch].reg + TALITOS_CCPSR_LO);
Kim Phillips9c4a7962008-06-23 19:50:15 +0800572
573 if (v_lo & TALITOS_CCPSR_LO_DOF) {
574 dev_err(dev, "double fetch fifo overflow error\n");
575 error = -EAGAIN;
576 reset_ch = 1;
577 }
578 if (v_lo & TALITOS_CCPSR_LO_SOF) {
579 /* h/w dropped descriptor */
580 dev_err(dev, "single fetch fifo overflow error\n");
581 error = -EAGAIN;
582 }
583 if (v_lo & TALITOS_CCPSR_LO_MDTE)
584 dev_err(dev, "master data transfer error\n");
585 if (v_lo & TALITOS_CCPSR_LO_SGDLZ)
LEROY Christophedd3c0982015-04-17 16:32:13 +0200586 dev_err(dev, is_sec1 ? "pointeur not complete error\n"
587 : "s/g data length zero error\n");
Kim Phillips9c4a7962008-06-23 19:50:15 +0800588 if (v_lo & TALITOS_CCPSR_LO_FPZ)
LEROY Christophedd3c0982015-04-17 16:32:13 +0200589 dev_err(dev, is_sec1 ? "parity error\n"
590 : "fetch pointer zero error\n");
Kim Phillips9c4a7962008-06-23 19:50:15 +0800591 if (v_lo & TALITOS_CCPSR_LO_IDH)
592 dev_err(dev, "illegal descriptor header error\n");
593 if (v_lo & TALITOS_CCPSR_LO_IEU)
LEROY Christophedd3c0982015-04-17 16:32:13 +0200594 dev_err(dev, is_sec1 ? "static assignment error\n"
595 : "invalid exec unit error\n");
Kim Phillips9c4a7962008-06-23 19:50:15 +0800596 if (v_lo & TALITOS_CCPSR_LO_EU)
Kim Phillips3e721ae2011-10-21 15:20:28 +0200597 report_eu_error(dev, ch, current_desc_hdr(dev, ch));
LEROY Christophedd3c0982015-04-17 16:32:13 +0200598 if (!is_sec1) {
599 if (v_lo & TALITOS_CCPSR_LO_GB)
600 dev_err(dev, "gather boundary error\n");
601 if (v_lo & TALITOS_CCPSR_LO_GRL)
602 dev_err(dev, "gather return/length error\n");
603 if (v_lo & TALITOS_CCPSR_LO_SB)
604 dev_err(dev, "scatter boundary error\n");
605 if (v_lo & TALITOS_CCPSR_LO_SRL)
606 dev_err(dev, "scatter return/length error\n");
607 }
Kim Phillips9c4a7962008-06-23 19:50:15 +0800608
609 flush_channel(dev, ch, error, reset_ch);
610
611 if (reset_ch) {
612 reset_channel(dev, ch);
613 } else {
Kim Phillipsad42d5f2011-11-21 16:13:27 +0800614 setbits32(priv->chan[ch].reg + TALITOS_CCCR,
LEROY Christophedd3c0982015-04-17 16:32:13 +0200615 TALITOS2_CCCR_CONT);
Kim Phillipsad42d5f2011-11-21 16:13:27 +0800616 setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO, 0);
617 while ((in_be32(priv->chan[ch].reg + TALITOS_CCCR) &
LEROY Christophedd3c0982015-04-17 16:32:13 +0200618 TALITOS2_CCCR_CONT) && --timeout)
Kim Phillips9c4a7962008-06-23 19:50:15 +0800619 cpu_relax();
620 if (timeout == 0) {
621 dev_err(dev, "failed to restart channel %d\n",
622 ch);
623 reset_dev = 1;
624 }
625 }
626 }
LEROY Christophedd3c0982015-04-17 16:32:13 +0200627 if (reset_dev || (is_sec1 && isr & ~TALITOS1_ISR_4CHERR) ||
628 (!is_sec1 && isr & ~TALITOS2_ISR_4CHERR) || isr_lo) {
629 if (is_sec1 && (isr_lo & TALITOS1_ISR_TEA_ERR))
630 dev_err(dev, "TEA error: ISR 0x%08x_%08x\n",
631 isr, isr_lo);
632 else
633 dev_err(dev, "done overflow, internal time out, or "
634 "rngu error: ISR 0x%08x_%08x\n", isr, isr_lo);
Kim Phillips9c4a7962008-06-23 19:50:15 +0800635
636 /* purge request queues */
637 for (ch = 0; ch < priv->num_channels; ch++)
638 flush_channel(dev, ch, -EIO, 1);
639
640 /* reset and reinitialize the device */
641 init_device(dev);
642 }
643}
644
LEROY Christophedd3c0982015-04-17 16:32:13 +0200645#define DEF_TALITOS1_INTERRUPT(name, ch_done_mask, ch_err_mask, tlet) \
646static irqreturn_t talitos1_interrupt_##name(int irq, void *data) \
647{ \
648 struct device *dev = data; \
649 struct talitos_private *priv = dev_get_drvdata(dev); \
650 u32 isr, isr_lo; \
651 unsigned long flags; \
652 \
653 spin_lock_irqsave(&priv->reg_lock, flags); \
654 isr = in_be32(priv->reg + TALITOS_ISR); \
655 isr_lo = in_be32(priv->reg + TALITOS_ISR_LO); \
656 /* Acknowledge interrupt */ \
657 out_be32(priv->reg + TALITOS_ICR, isr & (ch_done_mask | ch_err_mask)); \
658 out_be32(priv->reg + TALITOS_ICR_LO, isr_lo); \
659 \
660 if (unlikely(isr & ch_err_mask || isr_lo & TALITOS1_IMR_LO_INIT)) { \
661 spin_unlock_irqrestore(&priv->reg_lock, flags); \
662 talitos_error(dev, isr & ch_err_mask, isr_lo); \
663 } \
664 else { \
665 if (likely(isr & ch_done_mask)) { \
666 /* mask further done interrupts. */ \
667 setbits32(priv->reg + TALITOS_IMR, ch_done_mask); \
668 /* done_task will unmask done interrupts at exit */ \
669 tasklet_schedule(&priv->done_task[tlet]); \
670 } \
671 spin_unlock_irqrestore(&priv->reg_lock, flags); \
672 } \
673 \
674 return (isr & (ch_done_mask | ch_err_mask) || isr_lo) ? IRQ_HANDLED : \
675 IRQ_NONE; \
676}
677
678DEF_TALITOS1_INTERRUPT(4ch, TALITOS1_ISR_4CHDONE, TALITOS1_ISR_4CHERR, 0)
679
680#define DEF_TALITOS2_INTERRUPT(name, ch_done_mask, ch_err_mask, tlet) \
681static irqreturn_t talitos2_interrupt_##name(int irq, void *data) \
Kim Phillipsc3e337f2011-11-21 16:13:27 +0800682{ \
683 struct device *dev = data; \
684 struct talitos_private *priv = dev_get_drvdata(dev); \
685 u32 isr, isr_lo; \
Horia Geanta511d63c2012-03-30 17:49:53 +0300686 unsigned long flags; \
Kim Phillipsc3e337f2011-11-21 16:13:27 +0800687 \
Horia Geanta511d63c2012-03-30 17:49:53 +0300688 spin_lock_irqsave(&priv->reg_lock, flags); \
Kim Phillipsc3e337f2011-11-21 16:13:27 +0800689 isr = in_be32(priv->reg + TALITOS_ISR); \
690 isr_lo = in_be32(priv->reg + TALITOS_ISR_LO); \
691 /* Acknowledge interrupt */ \
692 out_be32(priv->reg + TALITOS_ICR, isr & (ch_done_mask | ch_err_mask)); \
693 out_be32(priv->reg + TALITOS_ICR_LO, isr_lo); \
694 \
Horia Geanta511d63c2012-03-30 17:49:53 +0300695 if (unlikely(isr & ch_err_mask || isr_lo)) { \
696 spin_unlock_irqrestore(&priv->reg_lock, flags); \
697 talitos_error(dev, isr & ch_err_mask, isr_lo); \
698 } \
699 else { \
Kim Phillipsc3e337f2011-11-21 16:13:27 +0800700 if (likely(isr & ch_done_mask)) { \
701 /* mask further done interrupts. */ \
702 clrbits32(priv->reg + TALITOS_IMR, ch_done_mask); \
703 /* done_task will unmask done interrupts at exit */ \
704 tasklet_schedule(&priv->done_task[tlet]); \
705 } \
Horia Geanta511d63c2012-03-30 17:49:53 +0300706 spin_unlock_irqrestore(&priv->reg_lock, flags); \
707 } \
Kim Phillipsc3e337f2011-11-21 16:13:27 +0800708 \
709 return (isr & (ch_done_mask | ch_err_mask) || isr_lo) ? IRQ_HANDLED : \
710 IRQ_NONE; \
Kim Phillips9c4a7962008-06-23 19:50:15 +0800711}
LEROY Christophedd3c0982015-04-17 16:32:13 +0200712
713DEF_TALITOS2_INTERRUPT(4ch, TALITOS2_ISR_4CHDONE, TALITOS2_ISR_4CHERR, 0)
714DEF_TALITOS2_INTERRUPT(ch0_2, TALITOS2_ISR_CH_0_2_DONE, TALITOS2_ISR_CH_0_2_ERR,
715 0)
716DEF_TALITOS2_INTERRUPT(ch1_3, TALITOS2_ISR_CH_1_3_DONE, TALITOS2_ISR_CH_1_3_ERR,
717 1)
Kim Phillips9c4a7962008-06-23 19:50:15 +0800718
719/*
720 * hwrng
721 */
722static int talitos_rng_data_present(struct hwrng *rng, int wait)
723{
724 struct device *dev = (struct device *)rng->priv;
725 struct talitos_private *priv = dev_get_drvdata(dev);
726 u32 ofl;
727 int i;
728
729 for (i = 0; i < 20; i++) {
LEROY Christophe5fa7fa12015-04-17 16:32:11 +0200730 ofl = in_be32(priv->reg_rngu + TALITOS_EUSR_LO) &
Kim Phillips9c4a7962008-06-23 19:50:15 +0800731 TALITOS_RNGUSR_LO_OFL;
732 if (ofl || !wait)
733 break;
734 udelay(10);
735 }
736
737 return !!ofl;
738}
739
740static int talitos_rng_data_read(struct hwrng *rng, u32 *data)
741{
742 struct device *dev = (struct device *)rng->priv;
743 struct talitos_private *priv = dev_get_drvdata(dev);
744
745 /* rng fifo requires 64-bit accesses */
LEROY Christophe5fa7fa12015-04-17 16:32:11 +0200746 *data = in_be32(priv->reg_rngu + TALITOS_EU_FIFO);
747 *data = in_be32(priv->reg_rngu + TALITOS_EU_FIFO_LO);
Kim Phillips9c4a7962008-06-23 19:50:15 +0800748
749 return sizeof(u32);
750}
751
752static int talitos_rng_init(struct hwrng *rng)
753{
754 struct device *dev = (struct device *)rng->priv;
755 struct talitos_private *priv = dev_get_drvdata(dev);
756 unsigned int timeout = TALITOS_TIMEOUT;
757
LEROY Christophe5fa7fa12015-04-17 16:32:11 +0200758 setbits32(priv->reg_rngu + TALITOS_EURCR_LO, TALITOS_RNGURCR_LO_SR);
759 while (!(in_be32(priv->reg_rngu + TALITOS_EUSR_LO)
760 & TALITOS_RNGUSR_LO_RD)
Kim Phillips9c4a7962008-06-23 19:50:15 +0800761 && --timeout)
762 cpu_relax();
763 if (timeout == 0) {
764 dev_err(dev, "failed to reset rng hw\n");
765 return -ENODEV;
766 }
767
768 /* start generating */
LEROY Christophe5fa7fa12015-04-17 16:32:11 +0200769 setbits32(priv->reg_rngu + TALITOS_EUDSR_LO, 0);
Kim Phillips9c4a7962008-06-23 19:50:15 +0800770
771 return 0;
772}
773
774static int talitos_register_rng(struct device *dev)
775{
776 struct talitos_private *priv = dev_get_drvdata(dev);
Aaron Sierra35a3bb32015-08-05 16:52:08 -0500777 int err;
Kim Phillips9c4a7962008-06-23 19:50:15 +0800778
779 priv->rng.name = dev_driver_string(dev),
780 priv->rng.init = talitos_rng_init,
781 priv->rng.data_present = talitos_rng_data_present,
782 priv->rng.data_read = talitos_rng_data_read,
783 priv->rng.priv = (unsigned long)dev;
784
Aaron Sierra35a3bb32015-08-05 16:52:08 -0500785 err = hwrng_register(&priv->rng);
786 if (!err)
787 priv->rng_registered = true;
788
789 return err;
Kim Phillips9c4a7962008-06-23 19:50:15 +0800790}
791
792static void talitos_unregister_rng(struct device *dev)
793{
794 struct talitos_private *priv = dev_get_drvdata(dev);
795
Aaron Sierra35a3bb32015-08-05 16:52:08 -0500796 if (!priv->rng_registered)
797 return;
798
Kim Phillips9c4a7962008-06-23 19:50:15 +0800799 hwrng_unregister(&priv->rng);
Aaron Sierra35a3bb32015-08-05 16:52:08 -0500800 priv->rng_registered = false;
Kim Phillips9c4a7962008-06-23 19:50:15 +0800801}
802
803/*
804 * crypto alg
805 */
806#define TALITOS_CRA_PRIORITY 3000
Horia Geanta357fb602012-07-03 19:16:53 +0300807#define TALITOS_MAX_KEY_SIZE 96
Lee Nipper3952f172008-07-10 18:29:18 +0800808#define TALITOS_MAX_IV_LENGTH 16 /* max of AES_BLOCK_SIZE, DES3_EDE_BLOCK_SIZE */
Lee Nipper70bcaca2008-07-03 19:08:46 +0800809
Kim Phillips9c4a7962008-06-23 19:50:15 +0800810struct talitos_ctx {
811 struct device *dev;
Kim Phillips5228f0f2011-07-15 11:21:38 +0800812 int ch;
Kim Phillips9c4a7962008-06-23 19:50:15 +0800813 __be32 desc_hdr_template;
814 u8 key[TALITOS_MAX_KEY_SIZE];
Lee Nipper70bcaca2008-07-03 19:08:46 +0800815 u8 iv[TALITOS_MAX_IV_LENGTH];
Kim Phillips9c4a7962008-06-23 19:50:15 +0800816 unsigned int keylen;
817 unsigned int enckeylen;
818 unsigned int authkeylen;
Kim Phillips9c4a7962008-06-23 19:50:15 +0800819};
820
Lee Nipper497f2e62010-05-19 19:20:36 +1000821#define HASH_MAX_BLOCK_SIZE SHA512_BLOCK_SIZE
822#define TALITOS_MDEU_MAX_CONTEXT_SIZE TALITOS_MDEU_CONTEXT_SIZE_SHA384_SHA512
823
824struct talitos_ahash_req_ctx {
Kim Phillips60f208d2010-05-19 19:21:53 +1000825 u32 hw_context[TALITOS_MDEU_MAX_CONTEXT_SIZE / sizeof(u32)];
Lee Nipper497f2e62010-05-19 19:20:36 +1000826 unsigned int hw_context_size;
827 u8 buf[HASH_MAX_BLOCK_SIZE];
828 u8 bufnext[HASH_MAX_BLOCK_SIZE];
Kim Phillips60f208d2010-05-19 19:21:53 +1000829 unsigned int swinit;
Lee Nipper497f2e62010-05-19 19:20:36 +1000830 unsigned int first;
831 unsigned int last;
832 unsigned int to_hash_later;
Horia Geant?42e8b0d2015-05-11 20:04:56 +0300833 unsigned int nbuf;
Lee Nipper497f2e62010-05-19 19:20:36 +1000834 struct scatterlist bufsl[2];
835 struct scatterlist *psrc;
836};
837
Horia Geant?3639ca82016-04-21 19:24:55 +0300838struct talitos_export_state {
839 u32 hw_context[TALITOS_MDEU_MAX_CONTEXT_SIZE / sizeof(u32)];
840 u8 buf[HASH_MAX_BLOCK_SIZE];
841 unsigned int swinit;
842 unsigned int first;
843 unsigned int last;
844 unsigned int to_hash_later;
845 unsigned int nbuf;
846};
847
Lee Nipper56af8cd2009-03-29 15:50:50 +0800848static int aead_setkey(struct crypto_aead *authenc,
849 const u8 *key, unsigned int keylen)
Kim Phillips9c4a7962008-06-23 19:50:15 +0800850{
851 struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
Mathias Krausec306a982013-10-15 13:49:34 +0200852 struct crypto_authenc_keys keys;
Kim Phillips9c4a7962008-06-23 19:50:15 +0800853
Mathias Krausec306a982013-10-15 13:49:34 +0200854 if (crypto_authenc_extractkeys(&keys, key, keylen) != 0)
Kim Phillips9c4a7962008-06-23 19:50:15 +0800855 goto badkey;
856
Mathias Krausec306a982013-10-15 13:49:34 +0200857 if (keys.authkeylen + keys.enckeylen > TALITOS_MAX_KEY_SIZE)
Kim Phillips9c4a7962008-06-23 19:50:15 +0800858 goto badkey;
859
Mathias Krausec306a982013-10-15 13:49:34 +0200860 memcpy(ctx->key, keys.authkey, keys.authkeylen);
861 memcpy(&ctx->key[keys.authkeylen], keys.enckey, keys.enckeylen);
Kim Phillips9c4a7962008-06-23 19:50:15 +0800862
Mathias Krausec306a982013-10-15 13:49:34 +0200863 ctx->keylen = keys.authkeylen + keys.enckeylen;
864 ctx->enckeylen = keys.enckeylen;
865 ctx->authkeylen = keys.authkeylen;
Kim Phillips9c4a7962008-06-23 19:50:15 +0800866
867 return 0;
868
869badkey:
870 crypto_aead_set_flags(authenc, CRYPTO_TFM_RES_BAD_KEY_LEN);
871 return -EINVAL;
872}
873
874/*
Lee Nipper56af8cd2009-03-29 15:50:50 +0800875 * talitos_edesc - s/w-extended descriptor
Kim Phillips9c4a7962008-06-23 19:50:15 +0800876 * @src_nents: number of segments in input scatterlist
877 * @dst_nents: number of segments in output scatterlist
Herbert Xuaeb4c132015-07-30 17:53:22 +0800878 * @icv_ool: whether ICV is out-of-line
Horia Geanta79fd31d2012-08-02 17:16:40 +0300879 * @iv_dma: dma address of iv for checking continuity and link table
Kim Phillips9c4a7962008-06-23 19:50:15 +0800880 * @dma_len: length of dma mapped link_tbl space
LEROY Christophe6f65f6a2015-04-17 16:32:15 +0200881 * @dma_link_tbl: bus physical address of link_tbl/buf
Kim Phillips9c4a7962008-06-23 19:50:15 +0800882 * @desc: h/w descriptor
LEROY Christophe6f65f6a2015-04-17 16:32:15 +0200883 * @link_tbl: input and output h/w link tables (if {src,dst}_nents > 1) (SEC2)
884 * @buf: input and output buffeur (if {src,dst}_nents > 1) (SEC1)
Kim Phillips9c4a7962008-06-23 19:50:15 +0800885 *
886 * if decrypting (with authcheck), or either one of src_nents or dst_nents
887 * is greater than 1, an integrity check value is concatenated to the end
888 * of link_tbl data
889 */
Lee Nipper56af8cd2009-03-29 15:50:50 +0800890struct talitos_edesc {
Kim Phillips9c4a7962008-06-23 19:50:15 +0800891 int src_nents;
892 int dst_nents;
Herbert Xuaeb4c132015-07-30 17:53:22 +0800893 bool icv_ool;
Horia Geanta79fd31d2012-08-02 17:16:40 +0300894 dma_addr_t iv_dma;
Kim Phillips9c4a7962008-06-23 19:50:15 +0800895 int dma_len;
896 dma_addr_t dma_link_tbl;
897 struct talitos_desc desc;
LEROY Christophe6f65f6a2015-04-17 16:32:15 +0200898 union {
899 struct talitos_ptr link_tbl[0];
900 u8 buf[0];
901 };
Kim Phillips9c4a7962008-06-23 19:50:15 +0800902};
903
Lee Nipper4de9d0b2009-03-29 15:52:32 +0800904static void talitos_sg_unmap(struct device *dev,
905 struct talitos_edesc *edesc,
906 struct scatterlist *src,
907 struct scatterlist *dst)
908{
909 unsigned int src_nents = edesc->src_nents ? : 1;
910 unsigned int dst_nents = edesc->dst_nents ? : 1;
911
912 if (src != dst) {
LABBE Corentinb8a011d2015-09-23 13:55:25 +0200913 dma_unmap_sg(dev, src, src_nents, DMA_TO_DEVICE);
Lee Nipper4de9d0b2009-03-29 15:52:32 +0800914
Lee Nipper497f2e62010-05-19 19:20:36 +1000915 if (dst) {
LABBE Corentinb8a011d2015-09-23 13:55:25 +0200916 dma_unmap_sg(dev, dst, dst_nents, DMA_FROM_DEVICE);
Lee Nipper497f2e62010-05-19 19:20:36 +1000917 }
Lee Nipper4de9d0b2009-03-29 15:52:32 +0800918 } else
LABBE Corentinb8a011d2015-09-23 13:55:25 +0200919 dma_unmap_sg(dev, src, src_nents, DMA_BIDIRECTIONAL);
Lee Nipper4de9d0b2009-03-29 15:52:32 +0800920}
921
Kim Phillips9c4a7962008-06-23 19:50:15 +0800922static void ipsec_esp_unmap(struct device *dev,
Lee Nipper56af8cd2009-03-29 15:50:50 +0800923 struct talitos_edesc *edesc,
Kim Phillips9c4a7962008-06-23 19:50:15 +0800924 struct aead_request *areq)
925{
926 unmap_single_talitos_ptr(dev, &edesc->desc.ptr[6], DMA_FROM_DEVICE);
927 unmap_single_talitos_ptr(dev, &edesc->desc.ptr[3], DMA_TO_DEVICE);
928 unmap_single_talitos_ptr(dev, &edesc->desc.ptr[2], DMA_TO_DEVICE);
929 unmap_single_talitos_ptr(dev, &edesc->desc.ptr[0], DMA_TO_DEVICE);
930
Lee Nipper4de9d0b2009-03-29 15:52:32 +0800931 talitos_sg_unmap(dev, edesc, areq->src, areq->dst);
Kim Phillips9c4a7962008-06-23 19:50:15 +0800932
933 if (edesc->dma_len)
934 dma_unmap_single(dev, edesc->dma_link_tbl, edesc->dma_len,
935 DMA_BIDIRECTIONAL);
936}
937
938/*
939 * ipsec_esp descriptor callbacks
940 */
941static void ipsec_esp_encrypt_done(struct device *dev,
942 struct talitos_desc *desc, void *context,
943 int err)
944{
945 struct aead_request *areq = context;
Kim Phillips9c4a7962008-06-23 19:50:15 +0800946 struct crypto_aead *authenc = crypto_aead_reqtfm(areq);
Herbert Xuaeb4c132015-07-30 17:53:22 +0800947 unsigned int authsize = crypto_aead_authsize(authenc);
Kim Phillips19bbbc62009-03-29 15:53:59 +0800948 struct talitos_edesc *edesc;
Kim Phillips9c4a7962008-06-23 19:50:15 +0800949 struct scatterlist *sg;
950 void *icvdata;
951
Kim Phillips19bbbc62009-03-29 15:53:59 +0800952 edesc = container_of(desc, struct talitos_edesc, desc);
953
Kim Phillips9c4a7962008-06-23 19:50:15 +0800954 ipsec_esp_unmap(dev, edesc, areq);
955
956 /* copy the generated ICV to dst */
Herbert Xuaeb4c132015-07-30 17:53:22 +0800957 if (edesc->icv_ool) {
Kim Phillips9c4a7962008-06-23 19:50:15 +0800958 icvdata = &edesc->link_tbl[edesc->src_nents +
Herbert Xuaeb4c132015-07-30 17:53:22 +0800959 edesc->dst_nents + 2];
Kim Phillips9c4a7962008-06-23 19:50:15 +0800960 sg = sg_last(areq->dst, edesc->dst_nents);
Herbert Xuaeb4c132015-07-30 17:53:22 +0800961 memcpy((char *)sg_virt(sg) + sg->length - authsize,
962 icvdata, authsize);
Kim Phillips9c4a7962008-06-23 19:50:15 +0800963 }
964
965 kfree(edesc);
966
967 aead_request_complete(areq, err);
968}
969
Kim Phillipsfe5720e2008-10-12 20:33:14 +0800970static void ipsec_esp_decrypt_swauth_done(struct device *dev,
Kim Phillipse938e462009-03-29 15:53:23 +0800971 struct talitos_desc *desc,
972 void *context, int err)
Kim Phillips9c4a7962008-06-23 19:50:15 +0800973{
974 struct aead_request *req = context;
Kim Phillips9c4a7962008-06-23 19:50:15 +0800975 struct crypto_aead *authenc = crypto_aead_reqtfm(req);
Herbert Xuaeb4c132015-07-30 17:53:22 +0800976 unsigned int authsize = crypto_aead_authsize(authenc);
Kim Phillips19bbbc62009-03-29 15:53:59 +0800977 struct talitos_edesc *edesc;
Kim Phillips9c4a7962008-06-23 19:50:15 +0800978 struct scatterlist *sg;
Herbert Xuaeb4c132015-07-30 17:53:22 +0800979 char *oicv, *icv;
Kim Phillips9c4a7962008-06-23 19:50:15 +0800980
Kim Phillips19bbbc62009-03-29 15:53:59 +0800981 edesc = container_of(desc, struct talitos_edesc, desc);
982
Kim Phillips9c4a7962008-06-23 19:50:15 +0800983 ipsec_esp_unmap(dev, edesc, req);
984
985 if (!err) {
986 /* auth check */
Kim Phillips9c4a7962008-06-23 19:50:15 +0800987 sg = sg_last(req->dst, edesc->dst_nents ? : 1);
Herbert Xuaeb4c132015-07-30 17:53:22 +0800988 icv = (char *)sg_virt(sg) + sg->length - authsize;
989
990 if (edesc->dma_len) {
991 oicv = (char *)&edesc->link_tbl[edesc->src_nents +
992 edesc->dst_nents + 2];
993 if (edesc->icv_ool)
994 icv = oicv + authsize;
995 } else
996 oicv = (char *)&edesc->link_tbl[0];
997
David Gstir79960942015-11-15 17:14:42 +0100998 err = crypto_memneq(oicv, icv, authsize) ? -EBADMSG : 0;
Kim Phillips9c4a7962008-06-23 19:50:15 +0800999 }
1000
1001 kfree(edesc);
1002
1003 aead_request_complete(req, err);
1004}
1005
Kim Phillipsfe5720e2008-10-12 20:33:14 +08001006static void ipsec_esp_decrypt_hwauth_done(struct device *dev,
Kim Phillipse938e462009-03-29 15:53:23 +08001007 struct talitos_desc *desc,
1008 void *context, int err)
Kim Phillipsfe5720e2008-10-12 20:33:14 +08001009{
1010 struct aead_request *req = context;
Kim Phillips19bbbc62009-03-29 15:53:59 +08001011 struct talitos_edesc *edesc;
1012
1013 edesc = container_of(desc, struct talitos_edesc, desc);
Kim Phillipsfe5720e2008-10-12 20:33:14 +08001014
1015 ipsec_esp_unmap(dev, edesc, req);
1016
1017 /* check ICV auth status */
Kim Phillipse938e462009-03-29 15:53:23 +08001018 if (!err && ((desc->hdr_lo & DESC_HDR_LO_ICCR1_MASK) !=
1019 DESC_HDR_LO_ICCR1_PASS))
1020 err = -EBADMSG;
Kim Phillipsfe5720e2008-10-12 20:33:14 +08001021
1022 kfree(edesc);
1023
1024 aead_request_complete(req, err);
1025}
1026
Kim Phillips9c4a7962008-06-23 19:50:15 +08001027/*
1028 * convert scatterlist to SEC h/w link table format
1029 * stop at cryptlen bytes
1030 */
Herbert Xuaeb4c132015-07-30 17:53:22 +08001031static int sg_to_link_tbl_offset(struct scatterlist *sg, int sg_count,
1032 unsigned int offset, int cryptlen,
1033 struct talitos_ptr *link_tbl_ptr)
Kim Phillips9c4a7962008-06-23 19:50:15 +08001034{
Lee Nipper70bcaca2008-07-03 19:08:46 +08001035 int n_sg = sg_count;
Herbert Xuaeb4c132015-07-30 17:53:22 +08001036 int count = 0;
Lee Nipper70bcaca2008-07-03 19:08:46 +08001037
Herbert Xuaeb4c132015-07-30 17:53:22 +08001038 while (cryptlen && sg && n_sg--) {
1039 unsigned int len = sg_dma_len(sg);
1040
1041 if (offset >= len) {
1042 offset -= len;
1043 goto next;
1044 }
1045
1046 len -= offset;
1047
1048 if (len > cryptlen)
1049 len = cryptlen;
1050
1051 to_talitos_ptr(link_tbl_ptr + count,
1052 sg_dma_address(sg) + offset, 0);
1053 link_tbl_ptr[count].len = cpu_to_be16(len);
1054 link_tbl_ptr[count].j_extent = 0;
1055 count++;
1056 cryptlen -= len;
1057 offset = 0;
1058
1059next:
Cristian Stoica5be4d4c2015-01-20 10:06:16 +02001060 sg = sg_next(sg);
Kim Phillips9c4a7962008-06-23 19:50:15 +08001061 }
1062
Kim Phillips9c4a7962008-06-23 19:50:15 +08001063 /* tag end of link table */
Herbert Xuaeb4c132015-07-30 17:53:22 +08001064 if (count > 0)
1065 link_tbl_ptr[count - 1].j_extent = DESC_PTR_LNKTBL_RETURN;
Lee Nipper70bcaca2008-07-03 19:08:46 +08001066
Herbert Xuaeb4c132015-07-30 17:53:22 +08001067 return count;
1068}
1069
1070static inline int sg_to_link_tbl(struct scatterlist *sg, int sg_count,
1071 int cryptlen,
1072 struct talitos_ptr *link_tbl_ptr)
1073{
1074 return sg_to_link_tbl_offset(sg, sg_count, 0, cryptlen,
1075 link_tbl_ptr);
Kim Phillips9c4a7962008-06-23 19:50:15 +08001076}
1077
1078/*
1079 * fill in and submit ipsec_esp descriptor
1080 */
Lee Nipper56af8cd2009-03-29 15:50:50 +08001081static int ipsec_esp(struct talitos_edesc *edesc, struct aead_request *areq,
Herbert Xuaeb4c132015-07-30 17:53:22 +08001082 void (*callback)(struct device *dev,
1083 struct talitos_desc *desc,
1084 void *context, int error))
Kim Phillips9c4a7962008-06-23 19:50:15 +08001085{
1086 struct crypto_aead *aead = crypto_aead_reqtfm(areq);
Herbert Xuaeb4c132015-07-30 17:53:22 +08001087 unsigned int authsize = crypto_aead_authsize(aead);
Kim Phillips9c4a7962008-06-23 19:50:15 +08001088 struct talitos_ctx *ctx = crypto_aead_ctx(aead);
1089 struct device *dev = ctx->dev;
1090 struct talitos_desc *desc = &edesc->desc;
1091 unsigned int cryptlen = areq->cryptlen;
Kim Phillipse41256f2009-08-13 11:49:06 +10001092 unsigned int ivsize = crypto_aead_ivsize(aead);
Herbert Xuaeb4c132015-07-30 17:53:22 +08001093 int tbl_off = 0;
Kim Phillipsfa86a262008-07-17 20:20:06 +08001094 int sg_count, ret;
Kim Phillipsfe5720e2008-10-12 20:33:14 +08001095 int sg_link_tbl_len;
Kim Phillips9c4a7962008-06-23 19:50:15 +08001096
1097 /* hmac key */
1098 map_single_talitos_ptr(dev, &desc->ptr[0], ctx->authkeylen, &ctx->key,
LEROY Christophea2b35aa2015-04-17 16:31:57 +02001099 DMA_TO_DEVICE);
Horia Geanta79fd31d2012-08-02 17:16:40 +03001100
LABBE Corentinb8a011d2015-09-23 13:55:25 +02001101 sg_count = dma_map_sg(dev, areq->src, edesc->src_nents ?: 1,
1102 (areq->src == areq->dst) ? DMA_BIDIRECTIONAL
1103 : DMA_TO_DEVICE);
Kim Phillips9c4a7962008-06-23 19:50:15 +08001104 /* hmac data */
Herbert Xuaeb4c132015-07-30 17:53:22 +08001105 desc->ptr[1].len = cpu_to_be16(areq->assoclen);
1106 if (sg_count > 1 &&
1107 (ret = sg_to_link_tbl_offset(areq->src, sg_count, 0,
1108 areq->assoclen,
1109 &edesc->link_tbl[tbl_off])) > 1) {
Horia Geanta79fd31d2012-08-02 17:16:40 +03001110 to_talitos_ptr(&desc->ptr[1], edesc->dma_link_tbl + tbl_off *
LEROY Christophe922f9dc2015-04-17 16:32:07 +02001111 sizeof(struct talitos_ptr), 0);
Horia Geanta79fd31d2012-08-02 17:16:40 +03001112 desc->ptr[1].j_extent = DESC_PTR_LNKTBL_JUMP;
1113
Horia Geanta79fd31d2012-08-02 17:16:40 +03001114 dma_sync_single_for_device(dev, edesc->dma_link_tbl,
1115 edesc->dma_len, DMA_BIDIRECTIONAL);
Horia Geant?340ff602016-04-19 20:33:48 +03001116
1117 tbl_off += ret;
Horia Geanta79fd31d2012-08-02 17:16:40 +03001118 } else {
Herbert Xuaeb4c132015-07-30 17:53:22 +08001119 to_talitos_ptr(&desc->ptr[1], sg_dma_address(areq->src), 0);
Horia Geanta79fd31d2012-08-02 17:16:40 +03001120 desc->ptr[1].j_extent = 0;
1121 }
1122
Kim Phillips9c4a7962008-06-23 19:50:15 +08001123 /* cipher iv */
LEROY Christophe922f9dc2015-04-17 16:32:07 +02001124 to_talitos_ptr(&desc->ptr[2], edesc->iv_dma, 0);
Horia Geanta79fd31d2012-08-02 17:16:40 +03001125 desc->ptr[2].len = cpu_to_be16(ivsize);
1126 desc->ptr[2].j_extent = 0;
Kim Phillips9c4a7962008-06-23 19:50:15 +08001127
1128 /* cipher key */
1129 map_single_talitos_ptr(dev, &desc->ptr[3], ctx->enckeylen,
LEROY Christophea2b35aa2015-04-17 16:31:57 +02001130 (char *)&ctx->key + ctx->authkeylen,
Kim Phillips9c4a7962008-06-23 19:50:15 +08001131 DMA_TO_DEVICE);
1132
1133 /*
1134 * cipher in
1135 * map and adjust cipher len to aead request cryptlen.
1136 * extent is bytes of HMAC postpended to ciphertext,
1137 * typically 12 for ipsec
1138 */
1139 desc->ptr[4].len = cpu_to_be16(cryptlen);
1140 desc->ptr[4].j_extent = authsize;
1141
Herbert Xuaeb4c132015-07-30 17:53:22 +08001142 sg_link_tbl_len = cryptlen;
1143 if (edesc->desc.hdr & DESC_HDR_MODE1_MDEU_CICV)
1144 sg_link_tbl_len += authsize;
Kim Phillips9c4a7962008-06-23 19:50:15 +08001145
Horia Geant?340ff602016-04-19 20:33:48 +03001146 if (sg_count == 1) {
1147 to_talitos_ptr(&desc->ptr[4], sg_dma_address(areq->src) +
1148 areq->assoclen, 0);
1149 } else if ((ret = sg_to_link_tbl_offset(areq->src, sg_count,
1150 areq->assoclen, sg_link_tbl_len,
1151 &edesc->link_tbl[tbl_off])) >
1152 1) {
Herbert Xuaeb4c132015-07-30 17:53:22 +08001153 desc->ptr[4].j_extent |= DESC_PTR_LNKTBL_JUMP;
1154 to_talitos_ptr(&desc->ptr[4], edesc->dma_link_tbl +
1155 tbl_off *
1156 sizeof(struct talitos_ptr), 0);
1157 dma_sync_single_for_device(dev, edesc->dma_link_tbl,
1158 edesc->dma_len,
1159 DMA_BIDIRECTIONAL);
Horia Geant?340ff602016-04-19 20:33:48 +03001160 tbl_off += ret;
1161 } else {
1162 copy_talitos_ptr(&desc->ptr[4], &edesc->link_tbl[tbl_off], 0);
1163 }
Kim Phillips9c4a7962008-06-23 19:50:15 +08001164
1165 /* cipher out */
1166 desc->ptr[5].len = cpu_to_be16(cryptlen);
1167 desc->ptr[5].j_extent = authsize;
1168
Kim Phillipse938e462009-03-29 15:53:23 +08001169 if (areq->src != areq->dst)
LABBE Corentinb8a011d2015-09-23 13:55:25 +02001170 sg_count = dma_map_sg(dev, areq->dst, edesc->dst_nents ? : 1,
1171 DMA_FROM_DEVICE);
Kim Phillips9c4a7962008-06-23 19:50:15 +08001172
Herbert Xuaeb4c132015-07-30 17:53:22 +08001173 edesc->icv_ool = false;
1174
Horia Geant?340ff602016-04-19 20:33:48 +03001175 if (sg_count == 1) {
1176 to_talitos_ptr(&desc->ptr[5], sg_dma_address(areq->dst) +
1177 areq->assoclen, 0);
1178 } else if ((sg_count =
1179 sg_to_link_tbl_offset(areq->dst, sg_count,
Herbert Xuaeb4c132015-07-30 17:53:22 +08001180 areq->assoclen, cryptlen,
Horia Geant?340ff602016-04-19 20:33:48 +03001181 &edesc->link_tbl[tbl_off])) > 1) {
Horia Geanta79fd31d2012-08-02 17:16:40 +03001182 struct talitos_ptr *tbl_ptr = &edesc->link_tbl[tbl_off];
Kim Phillips9c4a7962008-06-23 19:50:15 +08001183
Kim Phillips81eb0242009-08-13 11:51:51 +10001184 to_talitos_ptr(&desc->ptr[5], edesc->dma_link_tbl +
LEROY Christophe922f9dc2015-04-17 16:32:07 +02001185 tbl_off * sizeof(struct talitos_ptr), 0);
Kim Phillipsfe5720e2008-10-12 20:33:14 +08001186
Lee Nipperf3c85bc2008-07-30 16:26:57 +08001187 /* Add an entry to the link table for ICV data */
Horia Geanta79fd31d2012-08-02 17:16:40 +03001188 tbl_ptr += sg_count - 1;
1189 tbl_ptr->j_extent = 0;
1190 tbl_ptr++;
1191 tbl_ptr->j_extent = DESC_PTR_LNKTBL_RETURN;
1192 tbl_ptr->len = cpu_to_be16(authsize);
Kim Phillips9c4a7962008-06-23 19:50:15 +08001193
1194 /* icv data follows link tables */
Horia Geanta79fd31d2012-08-02 17:16:40 +03001195 to_talitos_ptr(tbl_ptr, edesc->dma_link_tbl +
Herbert Xuaeb4c132015-07-30 17:53:22 +08001196 (edesc->src_nents + edesc->dst_nents +
1197 2) * sizeof(struct talitos_ptr) +
1198 authsize, 0);
Kim Phillips9c4a7962008-06-23 19:50:15 +08001199 desc->ptr[5].j_extent |= DESC_PTR_LNKTBL_JUMP;
1200 dma_sync_single_for_device(ctx->dev, edesc->dma_link_tbl,
1201 edesc->dma_len, DMA_BIDIRECTIONAL);
Herbert Xuaeb4c132015-07-30 17:53:22 +08001202
1203 edesc->icv_ool = true;
Horia Geant?340ff602016-04-19 20:33:48 +03001204 } else {
1205 copy_talitos_ptr(&desc->ptr[5], &edesc->link_tbl[tbl_off], 0);
1206 }
Kim Phillips9c4a7962008-06-23 19:50:15 +08001207
1208 /* iv out */
LEROY Christophea2b35aa2015-04-17 16:31:57 +02001209 map_single_talitos_ptr(dev, &desc->ptr[6], ivsize, ctx->iv,
Kim Phillips9c4a7962008-06-23 19:50:15 +08001210 DMA_FROM_DEVICE);
1211
Kim Phillips5228f0f2011-07-15 11:21:38 +08001212 ret = talitos_submit(dev, ctx->ch, desc, callback, areq);
Kim Phillipsfa86a262008-07-17 20:20:06 +08001213 if (ret != -EINPROGRESS) {
1214 ipsec_esp_unmap(dev, edesc, areq);
1215 kfree(edesc);
1216 }
1217 return ret;
Kim Phillips9c4a7962008-06-23 19:50:15 +08001218}
1219
Kim Phillips9c4a7962008-06-23 19:50:15 +08001220/*
Lee Nipper56af8cd2009-03-29 15:50:50 +08001221 * allocate and map the extended descriptor
Kim Phillips9c4a7962008-06-23 19:50:15 +08001222 */
Lee Nipper4de9d0b2009-03-29 15:52:32 +08001223static struct talitos_edesc *talitos_edesc_alloc(struct device *dev,
1224 struct scatterlist *src,
1225 struct scatterlist *dst,
Horia Geanta79fd31d2012-08-02 17:16:40 +03001226 u8 *iv,
1227 unsigned int assoclen,
Lee Nipper4de9d0b2009-03-29 15:52:32 +08001228 unsigned int cryptlen,
1229 unsigned int authsize,
Horia Geanta79fd31d2012-08-02 17:16:40 +03001230 unsigned int ivsize,
Lee Nipper4de9d0b2009-03-29 15:52:32 +08001231 int icv_stashing,
Horia Geanta62293a32013-11-28 15:11:17 +02001232 u32 cryptoflags,
1233 bool encrypt)
Kim Phillips9c4a7962008-06-23 19:50:15 +08001234{
Lee Nipper56af8cd2009-03-29 15:50:50 +08001235 struct talitos_edesc *edesc;
Herbert Xuaeb4c132015-07-30 17:53:22 +08001236 int src_nents, dst_nents, alloc_len, dma_len;
Horia Geanta79fd31d2012-08-02 17:16:40 +03001237 dma_addr_t iv_dma = 0;
Lee Nipper4de9d0b2009-03-29 15:52:32 +08001238 gfp_t flags = cryptoflags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL :
Kim Phillips586725f2008-07-17 20:19:18 +08001239 GFP_ATOMIC;
LEROY Christophe6f65f6a2015-04-17 16:32:15 +02001240 struct talitos_private *priv = dev_get_drvdata(dev);
1241 bool is_sec1 = has_ftr_sec1(priv);
1242 int max_len = is_sec1 ? TALITOS1_MAX_DATA_LEN : TALITOS2_MAX_DATA_LEN;
LABBE Corentin8e409fe2015-11-04 21:13:34 +01001243 void *err;
Kim Phillips9c4a7962008-06-23 19:50:15 +08001244
LEROY Christophe6f65f6a2015-04-17 16:32:15 +02001245 if (cryptlen + authsize > max_len) {
Lee Nipper4de9d0b2009-03-29 15:52:32 +08001246 dev_err(dev, "length exceeds h/w max limit\n");
Kim Phillips9c4a7962008-06-23 19:50:15 +08001247 return ERR_PTR(-EINVAL);
1248 }
1249
Horia Geanta935e99a2013-11-19 14:57:49 +02001250 if (ivsize)
Horia Geanta79fd31d2012-08-02 17:16:40 +03001251 iv_dma = dma_map_single(dev, iv, ivsize, DMA_TO_DEVICE);
1252
Horia Geanta62293a32013-11-28 15:11:17 +02001253 if (!dst || dst == src) {
LABBE Corentinb8a011d2015-09-23 13:55:25 +02001254 src_nents = sg_nents_for_len(src,
1255 assoclen + cryptlen + authsize);
LABBE Corentin8e409fe2015-11-04 21:13:34 +01001256 if (src_nents < 0) {
1257 dev_err(dev, "Invalid number of src SG.\n");
1258 err = ERR_PTR(-EINVAL);
1259 goto error_sg;
1260 }
Horia Geanta62293a32013-11-28 15:11:17 +02001261 src_nents = (src_nents == 1) ? 0 : src_nents;
1262 dst_nents = dst ? src_nents : 0;
1263 } else { /* dst && dst != src*/
LABBE Corentinb8a011d2015-09-23 13:55:25 +02001264 src_nents = sg_nents_for_len(src, assoclen + cryptlen +
1265 (encrypt ? 0 : authsize));
LABBE Corentin8e409fe2015-11-04 21:13:34 +01001266 if (src_nents < 0) {
1267 dev_err(dev, "Invalid number of src SG.\n");
1268 err = ERR_PTR(-EINVAL);
1269 goto error_sg;
1270 }
Horia Geanta62293a32013-11-28 15:11:17 +02001271 src_nents = (src_nents == 1) ? 0 : src_nents;
LABBE Corentinb8a011d2015-09-23 13:55:25 +02001272 dst_nents = sg_nents_for_len(dst, assoclen + cryptlen +
1273 (encrypt ? authsize : 0));
LABBE Corentin8e409fe2015-11-04 21:13:34 +01001274 if (dst_nents < 0) {
1275 dev_err(dev, "Invalid number of dst SG.\n");
1276 err = ERR_PTR(-EINVAL);
1277 goto error_sg;
1278 }
Horia Geanta62293a32013-11-28 15:11:17 +02001279 dst_nents = (dst_nents == 1) ? 0 : dst_nents;
Kim Phillips9c4a7962008-06-23 19:50:15 +08001280 }
1281
1282 /*
1283 * allocate space for base edesc plus the link tables,
Herbert Xuaeb4c132015-07-30 17:53:22 +08001284 * allowing for two separate entries for AD and generated ICV (+ 2),
1285 * and space for two sets of ICVs (stashed and generated)
Kim Phillips9c4a7962008-06-23 19:50:15 +08001286 */
Lee Nipper56af8cd2009-03-29 15:50:50 +08001287 alloc_len = sizeof(struct talitos_edesc);
Herbert Xuaeb4c132015-07-30 17:53:22 +08001288 if (src_nents || dst_nents) {
LEROY Christophe6f65f6a2015-04-17 16:32:15 +02001289 if (is_sec1)
Dan Carpenter608f37d2015-05-11 13:10:09 +03001290 dma_len = (src_nents ? cryptlen : 0) +
1291 (dst_nents ? cryptlen : 0);
LEROY Christophe6f65f6a2015-04-17 16:32:15 +02001292 else
Herbert Xuaeb4c132015-07-30 17:53:22 +08001293 dma_len = (src_nents + dst_nents + 2) *
1294 sizeof(struct talitos_ptr) + authsize * 2;
Kim Phillips9c4a7962008-06-23 19:50:15 +08001295 alloc_len += dma_len;
1296 } else {
1297 dma_len = 0;
Lee Nipper4de9d0b2009-03-29 15:52:32 +08001298 alloc_len += icv_stashing ? authsize : 0;
Kim Phillips9c4a7962008-06-23 19:50:15 +08001299 }
1300
Kim Phillips586725f2008-07-17 20:19:18 +08001301 edesc = kmalloc(alloc_len, GFP_DMA | flags);
Kim Phillips9c4a7962008-06-23 19:50:15 +08001302 if (!edesc) {
Lee Nipper4de9d0b2009-03-29 15:52:32 +08001303 dev_err(dev, "could not allocate edescriptor\n");
LABBE Corentin8e409fe2015-11-04 21:13:34 +01001304 err = ERR_PTR(-ENOMEM);
1305 goto error_sg;
Kim Phillips9c4a7962008-06-23 19:50:15 +08001306 }
1307
1308 edesc->src_nents = src_nents;
1309 edesc->dst_nents = dst_nents;
Horia Geanta79fd31d2012-08-02 17:16:40 +03001310 edesc->iv_dma = iv_dma;
Kim Phillips9c4a7962008-06-23 19:50:15 +08001311 edesc->dma_len = dma_len;
Lee Nipper497f2e62010-05-19 19:20:36 +10001312 if (dma_len)
1313 edesc->dma_link_tbl = dma_map_single(dev, &edesc->link_tbl[0],
1314 edesc->dma_len,
1315 DMA_BIDIRECTIONAL);
Kim Phillips9c4a7962008-06-23 19:50:15 +08001316
1317 return edesc;
LABBE Corentin8e409fe2015-11-04 21:13:34 +01001318error_sg:
1319 if (iv_dma)
1320 dma_unmap_single(dev, iv_dma, ivsize, DMA_TO_DEVICE);
1321 return err;
Kim Phillips9c4a7962008-06-23 19:50:15 +08001322}
1323
Horia Geanta79fd31d2012-08-02 17:16:40 +03001324static struct talitos_edesc *aead_edesc_alloc(struct aead_request *areq, u8 *iv,
Horia Geanta62293a32013-11-28 15:11:17 +02001325 int icv_stashing, bool encrypt)
Lee Nipper4de9d0b2009-03-29 15:52:32 +08001326{
1327 struct crypto_aead *authenc = crypto_aead_reqtfm(areq);
Herbert Xuaeb4c132015-07-30 17:53:22 +08001328 unsigned int authsize = crypto_aead_authsize(authenc);
Lee Nipper4de9d0b2009-03-29 15:52:32 +08001329 struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
Horia Geanta79fd31d2012-08-02 17:16:40 +03001330 unsigned int ivsize = crypto_aead_ivsize(authenc);
Lee Nipper4de9d0b2009-03-29 15:52:32 +08001331
Herbert Xuaeb4c132015-07-30 17:53:22 +08001332 return talitos_edesc_alloc(ctx->dev, areq->src, areq->dst,
Horia Geanta79fd31d2012-08-02 17:16:40 +03001333 iv, areq->assoclen, areq->cryptlen,
Herbert Xuaeb4c132015-07-30 17:53:22 +08001334 authsize, ivsize, icv_stashing,
Horia Geanta62293a32013-11-28 15:11:17 +02001335 areq->base.flags, encrypt);
Lee Nipper4de9d0b2009-03-29 15:52:32 +08001336}
1337
Lee Nipper56af8cd2009-03-29 15:50:50 +08001338static int aead_encrypt(struct aead_request *req)
Kim Phillips9c4a7962008-06-23 19:50:15 +08001339{
1340 struct crypto_aead *authenc = crypto_aead_reqtfm(req);
1341 struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
Lee Nipper56af8cd2009-03-29 15:50:50 +08001342 struct talitos_edesc *edesc;
Kim Phillips9c4a7962008-06-23 19:50:15 +08001343
1344 /* allocate extended descriptor */
Horia Geanta62293a32013-11-28 15:11:17 +02001345 edesc = aead_edesc_alloc(req, req->iv, 0, true);
Kim Phillips9c4a7962008-06-23 19:50:15 +08001346 if (IS_ERR(edesc))
1347 return PTR_ERR(edesc);
1348
1349 /* set encrypt */
Lee Nipper70bcaca2008-07-03 19:08:46 +08001350 edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_MODE0_ENCRYPT;
Kim Phillips9c4a7962008-06-23 19:50:15 +08001351
Herbert Xuaeb4c132015-07-30 17:53:22 +08001352 return ipsec_esp(edesc, req, ipsec_esp_encrypt_done);
Kim Phillips9c4a7962008-06-23 19:50:15 +08001353}
1354
Lee Nipper56af8cd2009-03-29 15:50:50 +08001355static int aead_decrypt(struct aead_request *req)
Kim Phillips9c4a7962008-06-23 19:50:15 +08001356{
1357 struct crypto_aead *authenc = crypto_aead_reqtfm(req);
Herbert Xuaeb4c132015-07-30 17:53:22 +08001358 unsigned int authsize = crypto_aead_authsize(authenc);
Kim Phillips9c4a7962008-06-23 19:50:15 +08001359 struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
Kim Phillipsfe5720e2008-10-12 20:33:14 +08001360 struct talitos_private *priv = dev_get_drvdata(ctx->dev);
Lee Nipper56af8cd2009-03-29 15:50:50 +08001361 struct talitos_edesc *edesc;
Kim Phillips9c4a7962008-06-23 19:50:15 +08001362 struct scatterlist *sg;
1363 void *icvdata;
1364
1365 req->cryptlen -= authsize;
1366
1367 /* allocate extended descriptor */
Horia Geanta62293a32013-11-28 15:11:17 +02001368 edesc = aead_edesc_alloc(req, req->iv, 1, false);
Kim Phillips9c4a7962008-06-23 19:50:15 +08001369 if (IS_ERR(edesc))
1370 return PTR_ERR(edesc);
1371
Kim Phillipsfe5720e2008-10-12 20:33:14 +08001372 if ((priv->features & TALITOS_FTR_HW_AUTH_CHECK) &&
Kim Phillipse938e462009-03-29 15:53:23 +08001373 ((!edesc->src_nents && !edesc->dst_nents) ||
1374 priv->features & TALITOS_FTR_SRC_LINK_TBL_LEN_INCLUDES_EXTENT)) {
Kim Phillips9c4a7962008-06-23 19:50:15 +08001375
Kim Phillipsfe5720e2008-10-12 20:33:14 +08001376 /* decrypt and check the ICV */
Kim Phillipse938e462009-03-29 15:53:23 +08001377 edesc->desc.hdr = ctx->desc_hdr_template |
1378 DESC_HDR_DIR_INBOUND |
Kim Phillipsfe5720e2008-10-12 20:33:14 +08001379 DESC_HDR_MODE1_MDEU_CICV;
Kim Phillips9c4a7962008-06-23 19:50:15 +08001380
Kim Phillipsfe5720e2008-10-12 20:33:14 +08001381 /* reset integrity check result bits */
1382 edesc->desc.hdr_lo = 0;
Kim Phillips9c4a7962008-06-23 19:50:15 +08001383
Herbert Xuaeb4c132015-07-30 17:53:22 +08001384 return ipsec_esp(edesc, req, ipsec_esp_decrypt_hwauth_done);
Kim Phillipsfe5720e2008-10-12 20:33:14 +08001385 }
Kim Phillipse938e462009-03-29 15:53:23 +08001386
1387 /* Have to check the ICV with software */
1388 edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_DIR_INBOUND;
1389
1390 /* stash incoming ICV for later cmp with ICV generated by the h/w */
1391 if (edesc->dma_len)
Herbert Xuaeb4c132015-07-30 17:53:22 +08001392 icvdata = (char *)&edesc->link_tbl[edesc->src_nents +
1393 edesc->dst_nents + 2];
Kim Phillipse938e462009-03-29 15:53:23 +08001394 else
1395 icvdata = &edesc->link_tbl[0];
1396
1397 sg = sg_last(req->src, edesc->src_nents ? : 1);
1398
Herbert Xuaeb4c132015-07-30 17:53:22 +08001399 memcpy(icvdata, (char *)sg_virt(sg) + sg->length - authsize, authsize);
Kim Phillipse938e462009-03-29 15:53:23 +08001400
Herbert Xuaeb4c132015-07-30 17:53:22 +08001401 return ipsec_esp(edesc, req, ipsec_esp_decrypt_swauth_done);
Kim Phillips9c4a7962008-06-23 19:50:15 +08001402}
1403
Lee Nipper4de9d0b2009-03-29 15:52:32 +08001404static int ablkcipher_setkey(struct crypto_ablkcipher *cipher,
1405 const u8 *key, unsigned int keylen)
1406{
1407 struct talitos_ctx *ctx = crypto_ablkcipher_ctx(cipher);
Lee Nipper4de9d0b2009-03-29 15:52:32 +08001408
1409 memcpy(&ctx->key, key, keylen);
1410 ctx->keylen = keylen;
1411
1412 return 0;
Lee Nipper4de9d0b2009-03-29 15:52:32 +08001413}
1414
LEROY Christophe032d1972015-04-17 16:31:51 +02001415static void unmap_sg_talitos_ptr(struct device *dev, struct scatterlist *src,
1416 struct scatterlist *dst, unsigned int len,
1417 struct talitos_edesc *edesc)
1418{
LEROY Christophe6f65f6a2015-04-17 16:32:15 +02001419 struct talitos_private *priv = dev_get_drvdata(dev);
1420 bool is_sec1 = has_ftr_sec1(priv);
1421
1422 if (is_sec1) {
1423 if (!edesc->src_nents) {
1424 dma_unmap_sg(dev, src, 1,
1425 dst != src ? DMA_TO_DEVICE
1426 : DMA_BIDIRECTIONAL);
1427 }
1428 if (dst && edesc->dst_nents) {
1429 dma_sync_single_for_device(dev,
1430 edesc->dma_link_tbl + len,
1431 len, DMA_FROM_DEVICE);
1432 sg_copy_from_buffer(dst, edesc->dst_nents ? : 1,
1433 edesc->buf + len, len);
1434 } else if (dst && dst != src) {
1435 dma_unmap_sg(dev, dst, 1, DMA_FROM_DEVICE);
1436 }
1437 } else {
1438 talitos_sg_unmap(dev, edesc, src, dst);
1439 }
LEROY Christophe032d1972015-04-17 16:31:51 +02001440}
1441
Lee Nipper4de9d0b2009-03-29 15:52:32 +08001442static void common_nonsnoop_unmap(struct device *dev,
1443 struct talitos_edesc *edesc,
1444 struct ablkcipher_request *areq)
1445{
1446 unmap_single_talitos_ptr(dev, &edesc->desc.ptr[5], DMA_FROM_DEVICE);
LEROY Christophe032d1972015-04-17 16:31:51 +02001447
1448 unmap_sg_talitos_ptr(dev, areq->src, areq->dst, areq->nbytes, edesc);
Lee Nipper4de9d0b2009-03-29 15:52:32 +08001449 unmap_single_talitos_ptr(dev, &edesc->desc.ptr[2], DMA_TO_DEVICE);
1450 unmap_single_talitos_ptr(dev, &edesc->desc.ptr[1], DMA_TO_DEVICE);
1451
Lee Nipper4de9d0b2009-03-29 15:52:32 +08001452 if (edesc->dma_len)
1453 dma_unmap_single(dev, edesc->dma_link_tbl, edesc->dma_len,
1454 DMA_BIDIRECTIONAL);
1455}
1456
1457static void ablkcipher_done(struct device *dev,
1458 struct talitos_desc *desc, void *context,
1459 int err)
1460{
1461 struct ablkcipher_request *areq = context;
Kim Phillips19bbbc62009-03-29 15:53:59 +08001462 struct talitos_edesc *edesc;
1463
1464 edesc = container_of(desc, struct talitos_edesc, desc);
Lee Nipper4de9d0b2009-03-29 15:52:32 +08001465
1466 common_nonsnoop_unmap(dev, edesc, areq);
1467
1468 kfree(edesc);
1469
1470 areq->base.complete(&areq->base, err);
1471}
1472
LEROY Christophe032d1972015-04-17 16:31:51 +02001473int map_sg_in_talitos_ptr(struct device *dev, struct scatterlist *src,
1474 unsigned int len, struct talitos_edesc *edesc,
1475 enum dma_data_direction dir, struct talitos_ptr *ptr)
1476{
1477 int sg_count;
LEROY Christophe922f9dc2015-04-17 16:32:07 +02001478 struct talitos_private *priv = dev_get_drvdata(dev);
1479 bool is_sec1 = has_ftr_sec1(priv);
LEROY Christophe032d1972015-04-17 16:31:51 +02001480
LEROY Christophe922f9dc2015-04-17 16:32:07 +02001481 to_talitos_ptr_len(ptr, len, is_sec1);
LEROY Christophe032d1972015-04-17 16:31:51 +02001482
LEROY Christophe6f65f6a2015-04-17 16:32:15 +02001483 if (is_sec1) {
1484 sg_count = edesc->src_nents ? : 1;
LEROY Christophe032d1972015-04-17 16:31:51 +02001485
LEROY Christophe6f65f6a2015-04-17 16:32:15 +02001486 if (sg_count == 1) {
1487 dma_map_sg(dev, src, 1, dir);
LEROY Christophe922f9dc2015-04-17 16:32:07 +02001488 to_talitos_ptr(ptr, sg_dma_address(src), is_sec1);
LEROY Christophe6f65f6a2015-04-17 16:32:15 +02001489 } else {
1490 sg_copy_to_buffer(src, sg_count, edesc->buf, len);
1491 to_talitos_ptr(ptr, edesc->dma_link_tbl, is_sec1);
1492 dma_sync_single_for_device(dev, edesc->dma_link_tbl,
1493 len, DMA_TO_DEVICE);
1494 }
1495 } else {
1496 to_talitos_ptr_extent_clear(ptr, is_sec1);
1497
LABBE Corentinb8a011d2015-09-23 13:55:25 +02001498 sg_count = dma_map_sg(dev, src, edesc->src_nents ? : 1, dir);
LEROY Christophe6f65f6a2015-04-17 16:32:15 +02001499
1500 if (sg_count == 1) {
1501 to_talitos_ptr(ptr, sg_dma_address(src), is_sec1);
1502 } else {
1503 sg_count = sg_to_link_tbl(src, sg_count, len,
1504 &edesc->link_tbl[0]);
1505 if (sg_count > 1) {
1506 to_talitos_ptr(ptr, edesc->dma_link_tbl, 0);
1507 ptr->j_extent |= DESC_PTR_LNKTBL_JUMP;
1508 dma_sync_single_for_device(dev,
1509 edesc->dma_link_tbl,
1510 edesc->dma_len,
1511 DMA_BIDIRECTIONAL);
1512 } else {
1513 /* Only one segment now, so no link tbl needed*/
1514 to_talitos_ptr(ptr, sg_dma_address(src),
1515 is_sec1);
1516 }
LEROY Christophe032d1972015-04-17 16:31:51 +02001517 }
1518 }
1519 return sg_count;
1520}
1521
1522void map_sg_out_talitos_ptr(struct device *dev, struct scatterlist *dst,
1523 unsigned int len, struct talitos_edesc *edesc,
1524 enum dma_data_direction dir,
1525 struct talitos_ptr *ptr, int sg_count)
1526{
LEROY Christophe922f9dc2015-04-17 16:32:07 +02001527 struct talitos_private *priv = dev_get_drvdata(dev);
1528 bool is_sec1 = has_ftr_sec1(priv);
1529
LEROY Christophe032d1972015-04-17 16:31:51 +02001530 if (dir != DMA_NONE)
LABBE Corentinb8a011d2015-09-23 13:55:25 +02001531 sg_count = dma_map_sg(dev, dst, edesc->dst_nents ? : 1, dir);
LEROY Christophe032d1972015-04-17 16:31:51 +02001532
LEROY Christophe6f65f6a2015-04-17 16:32:15 +02001533 to_talitos_ptr_len(ptr, len, is_sec1);
LEROY Christophe032d1972015-04-17 16:31:51 +02001534
LEROY Christophe6f65f6a2015-04-17 16:32:15 +02001535 if (is_sec1) {
1536 if (sg_count == 1) {
1537 if (dir != DMA_NONE)
1538 dma_map_sg(dev, dst, 1, dir);
1539 to_talitos_ptr(ptr, sg_dma_address(dst), is_sec1);
1540 } else {
1541 to_talitos_ptr(ptr, edesc->dma_link_tbl + len, is_sec1);
1542 dma_sync_single_for_device(dev,
1543 edesc->dma_link_tbl + len,
1544 len, DMA_FROM_DEVICE);
1545 }
1546 } else {
1547 to_talitos_ptr_extent_clear(ptr, is_sec1);
1548
1549 if (sg_count == 1) {
1550 to_talitos_ptr(ptr, sg_dma_address(dst), is_sec1);
1551 } else {
1552 struct talitos_ptr *link_tbl_ptr =
1553 &edesc->link_tbl[edesc->src_nents + 1];
1554
1555 to_talitos_ptr(ptr, edesc->dma_link_tbl +
1556 (edesc->src_nents + 1) *
1557 sizeof(struct talitos_ptr), 0);
1558 ptr->j_extent |= DESC_PTR_LNKTBL_JUMP;
Horia Geant?42e8b0d2015-05-11 20:04:56 +03001559 sg_to_link_tbl(dst, sg_count, len, link_tbl_ptr);
LEROY Christophe6f65f6a2015-04-17 16:32:15 +02001560 dma_sync_single_for_device(dev, edesc->dma_link_tbl,
1561 edesc->dma_len,
1562 DMA_BIDIRECTIONAL);
1563 }
LEROY Christophe032d1972015-04-17 16:31:51 +02001564 }
1565}
1566
Lee Nipper4de9d0b2009-03-29 15:52:32 +08001567static int common_nonsnoop(struct talitos_edesc *edesc,
1568 struct ablkcipher_request *areq,
Lee Nipper4de9d0b2009-03-29 15:52:32 +08001569 void (*callback) (struct device *dev,
1570 struct talitos_desc *desc,
1571 void *context, int error))
1572{
1573 struct crypto_ablkcipher *cipher = crypto_ablkcipher_reqtfm(areq);
1574 struct talitos_ctx *ctx = crypto_ablkcipher_ctx(cipher);
1575 struct device *dev = ctx->dev;
1576 struct talitos_desc *desc = &edesc->desc;
1577 unsigned int cryptlen = areq->nbytes;
Horia Geanta79fd31d2012-08-02 17:16:40 +03001578 unsigned int ivsize = crypto_ablkcipher_ivsize(cipher);
Lee Nipper4de9d0b2009-03-29 15:52:32 +08001579 int sg_count, ret;
LEROY Christophe922f9dc2015-04-17 16:32:07 +02001580 struct talitos_private *priv = dev_get_drvdata(dev);
1581 bool is_sec1 = has_ftr_sec1(priv);
Lee Nipper4de9d0b2009-03-29 15:52:32 +08001582
1583 /* first DWORD empty */
LEROY Christophe2529bc32015-04-17 16:31:49 +02001584 desc->ptr[0] = zero_entry;
Lee Nipper4de9d0b2009-03-29 15:52:32 +08001585
1586 /* cipher iv */
LEROY Christophe922f9dc2015-04-17 16:32:07 +02001587 to_talitos_ptr(&desc->ptr[1], edesc->iv_dma, is_sec1);
1588 to_talitos_ptr_len(&desc->ptr[1], ivsize, is_sec1);
1589 to_talitos_ptr_extent_clear(&desc->ptr[1], is_sec1);
Lee Nipper4de9d0b2009-03-29 15:52:32 +08001590
1591 /* cipher key */
1592 map_single_talitos_ptr(dev, &desc->ptr[2], ctx->keylen,
LEROY Christophea2b35aa2015-04-17 16:31:57 +02001593 (char *)&ctx->key, DMA_TO_DEVICE);
Lee Nipper4de9d0b2009-03-29 15:52:32 +08001594
1595 /*
1596 * cipher in
1597 */
LEROY Christophe032d1972015-04-17 16:31:51 +02001598 sg_count = map_sg_in_talitos_ptr(dev, areq->src, cryptlen, edesc,
1599 (areq->src == areq->dst) ?
1600 DMA_BIDIRECTIONAL : DMA_TO_DEVICE,
1601 &desc->ptr[3]);
Lee Nipper4de9d0b2009-03-29 15:52:32 +08001602
1603 /* cipher out */
LEROY Christophe032d1972015-04-17 16:31:51 +02001604 map_sg_out_talitos_ptr(dev, areq->dst, cryptlen, edesc,
1605 (areq->src == areq->dst) ? DMA_NONE
1606 : DMA_FROM_DEVICE,
1607 &desc->ptr[4], sg_count);
Lee Nipper4de9d0b2009-03-29 15:52:32 +08001608
1609 /* iv out */
LEROY Christophea2b35aa2015-04-17 16:31:57 +02001610 map_single_talitos_ptr(dev, &desc->ptr[5], ivsize, ctx->iv,
Lee Nipper4de9d0b2009-03-29 15:52:32 +08001611 DMA_FROM_DEVICE);
1612
1613 /* last DWORD empty */
LEROY Christophe2529bc32015-04-17 16:31:49 +02001614 desc->ptr[6] = zero_entry;
Lee Nipper4de9d0b2009-03-29 15:52:32 +08001615
Kim Phillips5228f0f2011-07-15 11:21:38 +08001616 ret = talitos_submit(dev, ctx->ch, desc, callback, areq);
Lee Nipper4de9d0b2009-03-29 15:52:32 +08001617 if (ret != -EINPROGRESS) {
1618 common_nonsnoop_unmap(dev, edesc, areq);
1619 kfree(edesc);
1620 }
1621 return ret;
1622}
1623
Kim Phillipse938e462009-03-29 15:53:23 +08001624static struct talitos_edesc *ablkcipher_edesc_alloc(struct ablkcipher_request *
Horia Geanta62293a32013-11-28 15:11:17 +02001625 areq, bool encrypt)
Lee Nipper4de9d0b2009-03-29 15:52:32 +08001626{
1627 struct crypto_ablkcipher *cipher = crypto_ablkcipher_reqtfm(areq);
1628 struct talitos_ctx *ctx = crypto_ablkcipher_ctx(cipher);
Horia Geanta79fd31d2012-08-02 17:16:40 +03001629 unsigned int ivsize = crypto_ablkcipher_ivsize(cipher);
Lee Nipper4de9d0b2009-03-29 15:52:32 +08001630
Herbert Xuaeb4c132015-07-30 17:53:22 +08001631 return talitos_edesc_alloc(ctx->dev, areq->src, areq->dst,
Horia Geanta79fd31d2012-08-02 17:16:40 +03001632 areq->info, 0, areq->nbytes, 0, ivsize, 0,
Horia Geanta62293a32013-11-28 15:11:17 +02001633 areq->base.flags, encrypt);
Lee Nipper4de9d0b2009-03-29 15:52:32 +08001634}
1635
1636static int ablkcipher_encrypt(struct ablkcipher_request *areq)
1637{
1638 struct crypto_ablkcipher *cipher = crypto_ablkcipher_reqtfm(areq);
1639 struct talitos_ctx *ctx = crypto_ablkcipher_ctx(cipher);
1640 struct talitos_edesc *edesc;
1641
1642 /* allocate extended descriptor */
Horia Geanta62293a32013-11-28 15:11:17 +02001643 edesc = ablkcipher_edesc_alloc(areq, true);
Lee Nipper4de9d0b2009-03-29 15:52:32 +08001644 if (IS_ERR(edesc))
1645 return PTR_ERR(edesc);
1646
1647 /* set encrypt */
1648 edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_MODE0_ENCRYPT;
1649
Kim Phillipsfebec542011-07-15 11:21:39 +08001650 return common_nonsnoop(edesc, areq, ablkcipher_done);
Lee Nipper4de9d0b2009-03-29 15:52:32 +08001651}
1652
1653static int ablkcipher_decrypt(struct ablkcipher_request *areq)
1654{
1655 struct crypto_ablkcipher *cipher = crypto_ablkcipher_reqtfm(areq);
1656 struct talitos_ctx *ctx = crypto_ablkcipher_ctx(cipher);
1657 struct talitos_edesc *edesc;
1658
1659 /* allocate extended descriptor */
Horia Geanta62293a32013-11-28 15:11:17 +02001660 edesc = ablkcipher_edesc_alloc(areq, false);
Lee Nipper4de9d0b2009-03-29 15:52:32 +08001661 if (IS_ERR(edesc))
1662 return PTR_ERR(edesc);
1663
1664 edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_DIR_INBOUND;
1665
Kim Phillipsfebec542011-07-15 11:21:39 +08001666 return common_nonsnoop(edesc, areq, ablkcipher_done);
Lee Nipper4de9d0b2009-03-29 15:52:32 +08001667}
1668
Lee Nipper497f2e62010-05-19 19:20:36 +10001669static void common_nonsnoop_hash_unmap(struct device *dev,
1670 struct talitos_edesc *edesc,
1671 struct ahash_request *areq)
1672{
1673 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
LEROY Christophe922f9dc2015-04-17 16:32:07 +02001674 struct talitos_private *priv = dev_get_drvdata(dev);
1675 bool is_sec1 = has_ftr_sec1(priv);
Lee Nipper497f2e62010-05-19 19:20:36 +10001676
1677 unmap_single_talitos_ptr(dev, &edesc->desc.ptr[5], DMA_FROM_DEVICE);
1678
LEROY Christophe032d1972015-04-17 16:31:51 +02001679 unmap_sg_talitos_ptr(dev, req_ctx->psrc, NULL, 0, edesc);
1680
Lee Nipper497f2e62010-05-19 19:20:36 +10001681 /* When using hashctx-in, must unmap it. */
LEROY Christophe922f9dc2015-04-17 16:32:07 +02001682 if (from_talitos_ptr_len(&edesc->desc.ptr[1], is_sec1))
Lee Nipper497f2e62010-05-19 19:20:36 +10001683 unmap_single_talitos_ptr(dev, &edesc->desc.ptr[1],
1684 DMA_TO_DEVICE);
1685
LEROY Christophe922f9dc2015-04-17 16:32:07 +02001686 if (from_talitos_ptr_len(&edesc->desc.ptr[2], is_sec1))
Lee Nipper497f2e62010-05-19 19:20:36 +10001687 unmap_single_talitos_ptr(dev, &edesc->desc.ptr[2],
1688 DMA_TO_DEVICE);
1689
Lee Nipper497f2e62010-05-19 19:20:36 +10001690 if (edesc->dma_len)
1691 dma_unmap_single(dev, edesc->dma_link_tbl, edesc->dma_len,
1692 DMA_BIDIRECTIONAL);
1693
1694}
1695
1696static void ahash_done(struct device *dev,
1697 struct talitos_desc *desc, void *context,
1698 int err)
1699{
1700 struct ahash_request *areq = context;
1701 struct talitos_edesc *edesc =
1702 container_of(desc, struct talitos_edesc, desc);
1703 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1704
1705 if (!req_ctx->last && req_ctx->to_hash_later) {
1706 /* Position any partial block for next update/final/finup */
1707 memcpy(req_ctx->buf, req_ctx->bufnext, req_ctx->to_hash_later);
Lee Nipper5e833bc2010-06-16 15:29:15 +10001708 req_ctx->nbuf = req_ctx->to_hash_later;
Lee Nipper497f2e62010-05-19 19:20:36 +10001709 }
1710 common_nonsnoop_hash_unmap(dev, edesc, areq);
1711
1712 kfree(edesc);
1713
1714 areq->base.complete(&areq->base, err);
1715}
1716
LEROY Christophe2d029052015-04-17 16:32:18 +02001717/*
1718 * SEC1 doesn't like hashing of 0 sized message, so we do the padding
1719 * ourself and submit a padded block
1720 */
1721void talitos_handle_buggy_hash(struct talitos_ctx *ctx,
1722 struct talitos_edesc *edesc,
1723 struct talitos_ptr *ptr)
1724{
1725 static u8 padded_hash[64] = {
1726 0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
1727 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
1728 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
1729 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
1730 };
1731
1732 pr_err_once("Bug in SEC1, padding ourself\n");
1733 edesc->desc.hdr &= ~DESC_HDR_MODE0_MDEU_PAD;
1734 map_single_talitos_ptr(ctx->dev, ptr, sizeof(padded_hash),
1735 (char *)padded_hash, DMA_TO_DEVICE);
1736}
1737
Lee Nipper497f2e62010-05-19 19:20:36 +10001738static int common_nonsnoop_hash(struct talitos_edesc *edesc,
1739 struct ahash_request *areq, unsigned int length,
1740 void (*callback) (struct device *dev,
1741 struct talitos_desc *desc,
1742 void *context, int error))
1743{
1744 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
1745 struct talitos_ctx *ctx = crypto_ahash_ctx(tfm);
1746 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1747 struct device *dev = ctx->dev;
1748 struct talitos_desc *desc = &edesc->desc;
LEROY Christophe032d1972015-04-17 16:31:51 +02001749 int ret;
LEROY Christophe922f9dc2015-04-17 16:32:07 +02001750 struct talitos_private *priv = dev_get_drvdata(dev);
1751 bool is_sec1 = has_ftr_sec1(priv);
Lee Nipper497f2e62010-05-19 19:20:36 +10001752
1753 /* first DWORD empty */
1754 desc->ptr[0] = zero_entry;
1755
Kim Phillips60f208d2010-05-19 19:21:53 +10001756 /* hash context in */
1757 if (!req_ctx->first || req_ctx->swinit) {
Lee Nipper497f2e62010-05-19 19:20:36 +10001758 map_single_talitos_ptr(dev, &desc->ptr[1],
1759 req_ctx->hw_context_size,
LEROY Christophea2b35aa2015-04-17 16:31:57 +02001760 (char *)req_ctx->hw_context,
Lee Nipper497f2e62010-05-19 19:20:36 +10001761 DMA_TO_DEVICE);
Kim Phillips60f208d2010-05-19 19:21:53 +10001762 req_ctx->swinit = 0;
Lee Nipper497f2e62010-05-19 19:20:36 +10001763 } else {
1764 desc->ptr[1] = zero_entry;
1765 /* Indicate next op is not the first. */
1766 req_ctx->first = 0;
1767 }
1768
1769 /* HMAC key */
1770 if (ctx->keylen)
1771 map_single_talitos_ptr(dev, &desc->ptr[2], ctx->keylen,
LEROY Christophea2b35aa2015-04-17 16:31:57 +02001772 (char *)&ctx->key, DMA_TO_DEVICE);
Lee Nipper497f2e62010-05-19 19:20:36 +10001773 else
1774 desc->ptr[2] = zero_entry;
1775
1776 /*
1777 * data in
1778 */
LEROY Christophe032d1972015-04-17 16:31:51 +02001779 map_sg_in_talitos_ptr(dev, req_ctx->psrc, length, edesc,
1780 DMA_TO_DEVICE, &desc->ptr[3]);
Lee Nipper497f2e62010-05-19 19:20:36 +10001781
1782 /* fifth DWORD empty */
1783 desc->ptr[4] = zero_entry;
1784
1785 /* hash/HMAC out -or- hash context out */
1786 if (req_ctx->last)
1787 map_single_talitos_ptr(dev, &desc->ptr[5],
1788 crypto_ahash_digestsize(tfm),
LEROY Christophea2b35aa2015-04-17 16:31:57 +02001789 areq->result, DMA_FROM_DEVICE);
Lee Nipper497f2e62010-05-19 19:20:36 +10001790 else
1791 map_single_talitos_ptr(dev, &desc->ptr[5],
1792 req_ctx->hw_context_size,
LEROY Christophea2b35aa2015-04-17 16:31:57 +02001793 req_ctx->hw_context, DMA_FROM_DEVICE);
Lee Nipper497f2e62010-05-19 19:20:36 +10001794
1795 /* last DWORD empty */
1796 desc->ptr[6] = zero_entry;
1797
LEROY Christophe2d029052015-04-17 16:32:18 +02001798 if (is_sec1 && from_talitos_ptr_len(&desc->ptr[3], true) == 0)
1799 talitos_handle_buggy_hash(ctx, edesc, &desc->ptr[3]);
1800
Kim Phillips5228f0f2011-07-15 11:21:38 +08001801 ret = talitos_submit(dev, ctx->ch, desc, callback, areq);
Lee Nipper497f2e62010-05-19 19:20:36 +10001802 if (ret != -EINPROGRESS) {
1803 common_nonsnoop_hash_unmap(dev, edesc, areq);
1804 kfree(edesc);
1805 }
1806 return ret;
1807}
1808
1809static struct talitos_edesc *ahash_edesc_alloc(struct ahash_request *areq,
1810 unsigned int nbytes)
1811{
1812 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
1813 struct talitos_ctx *ctx = crypto_ahash_ctx(tfm);
1814 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1815
Herbert Xuaeb4c132015-07-30 17:53:22 +08001816 return talitos_edesc_alloc(ctx->dev, req_ctx->psrc, NULL, NULL, 0,
Horia Geanta62293a32013-11-28 15:11:17 +02001817 nbytes, 0, 0, 0, areq->base.flags, false);
Lee Nipper497f2e62010-05-19 19:20:36 +10001818}
1819
1820static int ahash_init(struct ahash_request *areq)
1821{
1822 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
1823 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1824
1825 /* Initialize the context */
Lee Nipper5e833bc2010-06-16 15:29:15 +10001826 req_ctx->nbuf = 0;
Kim Phillips60f208d2010-05-19 19:21:53 +10001827 req_ctx->first = 1; /* first indicates h/w must init its context */
1828 req_ctx->swinit = 0; /* assume h/w init of context */
Lee Nipper497f2e62010-05-19 19:20:36 +10001829 req_ctx->hw_context_size =
1830 (crypto_ahash_digestsize(tfm) <= SHA256_DIGEST_SIZE)
1831 ? TALITOS_MDEU_CONTEXT_SIZE_MD5_SHA1_SHA256
1832 : TALITOS_MDEU_CONTEXT_SIZE_SHA384_SHA512;
1833
1834 return 0;
1835}
1836
Kim Phillips60f208d2010-05-19 19:21:53 +10001837/*
1838 * on h/w without explicit sha224 support, we initialize h/w context
1839 * manually with sha224 constants, and tell it to run sha256.
1840 */
1841static int ahash_init_sha224_swinit(struct ahash_request *areq)
1842{
1843 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1844
1845 ahash_init(areq);
1846 req_ctx->swinit = 1;/* prevent h/w initting context with sha256 values*/
1847
Kim Phillipsa7524472010-09-23 15:56:38 +08001848 req_ctx->hw_context[0] = SHA224_H0;
1849 req_ctx->hw_context[1] = SHA224_H1;
1850 req_ctx->hw_context[2] = SHA224_H2;
1851 req_ctx->hw_context[3] = SHA224_H3;
1852 req_ctx->hw_context[4] = SHA224_H4;
1853 req_ctx->hw_context[5] = SHA224_H5;
1854 req_ctx->hw_context[6] = SHA224_H6;
1855 req_ctx->hw_context[7] = SHA224_H7;
Kim Phillips60f208d2010-05-19 19:21:53 +10001856
1857 /* init 64-bit count */
1858 req_ctx->hw_context[8] = 0;
1859 req_ctx->hw_context[9] = 0;
1860
1861 return 0;
1862}
1863
Lee Nipper497f2e62010-05-19 19:20:36 +10001864static int ahash_process_req(struct ahash_request *areq, unsigned int nbytes)
1865{
1866 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
1867 struct talitos_ctx *ctx = crypto_ahash_ctx(tfm);
1868 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1869 struct talitos_edesc *edesc;
1870 unsigned int blocksize =
1871 crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm));
1872 unsigned int nbytes_to_hash;
1873 unsigned int to_hash_later;
Lee Nipper5e833bc2010-06-16 15:29:15 +10001874 unsigned int nsg;
LABBE Corentin8e409fe2015-11-04 21:13:34 +01001875 int nents;
Lee Nipper497f2e62010-05-19 19:20:36 +10001876
Lee Nipper5e833bc2010-06-16 15:29:15 +10001877 if (!req_ctx->last && (nbytes + req_ctx->nbuf <= blocksize)) {
1878 /* Buffer up to one whole block */
LABBE Corentin8e409fe2015-11-04 21:13:34 +01001879 nents = sg_nents_for_len(areq->src, nbytes);
1880 if (nents < 0) {
1881 dev_err(ctx->dev, "Invalid number of src SG.\n");
1882 return nents;
1883 }
1884 sg_copy_to_buffer(areq->src, nents,
Lee Nipper5e833bc2010-06-16 15:29:15 +10001885 req_ctx->buf + req_ctx->nbuf, nbytes);
1886 req_ctx->nbuf += nbytes;
Lee Nipper497f2e62010-05-19 19:20:36 +10001887 return 0;
1888 }
1889
Lee Nipper5e833bc2010-06-16 15:29:15 +10001890 /* At least (blocksize + 1) bytes are available to hash */
1891 nbytes_to_hash = nbytes + req_ctx->nbuf;
1892 to_hash_later = nbytes_to_hash & (blocksize - 1);
1893
1894 if (req_ctx->last)
1895 to_hash_later = 0;
1896 else if (to_hash_later)
1897 /* There is a partial block. Hash the full block(s) now */
1898 nbytes_to_hash -= to_hash_later;
1899 else {
1900 /* Keep one block buffered */
1901 nbytes_to_hash -= blocksize;
1902 to_hash_later = blocksize;
1903 }
1904
1905 /* Chain in any previously buffered data */
1906 if (req_ctx->nbuf) {
1907 nsg = (req_ctx->nbuf < nbytes_to_hash) ? 2 : 1;
1908 sg_init_table(req_ctx->bufsl, nsg);
1909 sg_set_buf(req_ctx->bufsl, req_ctx->buf, req_ctx->nbuf);
1910 if (nsg > 1)
Dan Williamsc56f6d12015-08-07 18:15:13 +02001911 sg_chain(req_ctx->bufsl, 2, areq->src);
Lee Nipper497f2e62010-05-19 19:20:36 +10001912 req_ctx->psrc = req_ctx->bufsl;
Lee Nipper5e833bc2010-06-16 15:29:15 +10001913 } else
Lee Nipper497f2e62010-05-19 19:20:36 +10001914 req_ctx->psrc = areq->src;
Lee Nipper497f2e62010-05-19 19:20:36 +10001915
Lee Nipper5e833bc2010-06-16 15:29:15 +10001916 if (to_hash_later) {
LABBE Corentin8e409fe2015-11-04 21:13:34 +01001917 nents = sg_nents_for_len(areq->src, nbytes);
1918 if (nents < 0) {
1919 dev_err(ctx->dev, "Invalid number of src SG.\n");
1920 return nents;
1921 }
Akinobu Mitad0525722013-07-08 16:01:55 -07001922 sg_pcopy_to_buffer(areq->src, nents,
Lee Nipper5e833bc2010-06-16 15:29:15 +10001923 req_ctx->bufnext,
1924 to_hash_later,
1925 nbytes - to_hash_later);
Lee Nipper497f2e62010-05-19 19:20:36 +10001926 }
Lee Nipper5e833bc2010-06-16 15:29:15 +10001927 req_ctx->to_hash_later = to_hash_later;
Lee Nipper497f2e62010-05-19 19:20:36 +10001928
Lee Nipper5e833bc2010-06-16 15:29:15 +10001929 /* Allocate extended descriptor */
Lee Nipper497f2e62010-05-19 19:20:36 +10001930 edesc = ahash_edesc_alloc(areq, nbytes_to_hash);
1931 if (IS_ERR(edesc))
1932 return PTR_ERR(edesc);
1933
1934 edesc->desc.hdr = ctx->desc_hdr_template;
1935
1936 /* On last one, request SEC to pad; otherwise continue */
1937 if (req_ctx->last)
1938 edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_PAD;
1939 else
1940 edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_CONT;
1941
Kim Phillips60f208d2010-05-19 19:21:53 +10001942 /* request SEC to INIT hash. */
1943 if (req_ctx->first && !req_ctx->swinit)
Lee Nipper497f2e62010-05-19 19:20:36 +10001944 edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_INIT;
1945
1946 /* When the tfm context has a keylen, it's an HMAC.
1947 * A first or last (ie. not middle) descriptor must request HMAC.
1948 */
1949 if (ctx->keylen && (req_ctx->first || req_ctx->last))
1950 edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_HMAC;
1951
1952 return common_nonsnoop_hash(edesc, areq, nbytes_to_hash,
1953 ahash_done);
1954}
1955
1956static int ahash_update(struct ahash_request *areq)
1957{
1958 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1959
1960 req_ctx->last = 0;
1961
1962 return ahash_process_req(areq, areq->nbytes);
1963}
1964
1965static int ahash_final(struct ahash_request *areq)
1966{
1967 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1968
1969 req_ctx->last = 1;
1970
1971 return ahash_process_req(areq, 0);
1972}
1973
1974static int ahash_finup(struct ahash_request *areq)
1975{
1976 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1977
1978 req_ctx->last = 1;
1979
1980 return ahash_process_req(areq, areq->nbytes);
1981}
1982
1983static int ahash_digest(struct ahash_request *areq)
1984{
1985 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
Kim Phillips60f208d2010-05-19 19:21:53 +10001986 struct crypto_ahash *ahash = crypto_ahash_reqtfm(areq);
Lee Nipper497f2e62010-05-19 19:20:36 +10001987
Kim Phillips60f208d2010-05-19 19:21:53 +10001988 ahash->init(areq);
Lee Nipper497f2e62010-05-19 19:20:36 +10001989 req_ctx->last = 1;
1990
1991 return ahash_process_req(areq, areq->nbytes);
1992}
1993
Horia Geant?3639ca82016-04-21 19:24:55 +03001994static int ahash_export(struct ahash_request *areq, void *out)
1995{
1996 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1997 struct talitos_export_state *export = out;
1998
1999 memcpy(export->hw_context, req_ctx->hw_context,
2000 req_ctx->hw_context_size);
2001 memcpy(export->buf, req_ctx->buf, req_ctx->nbuf);
2002 export->swinit = req_ctx->swinit;
2003 export->first = req_ctx->first;
2004 export->last = req_ctx->last;
2005 export->to_hash_later = req_ctx->to_hash_later;
2006 export->nbuf = req_ctx->nbuf;
2007
2008 return 0;
2009}
2010
2011static int ahash_import(struct ahash_request *areq, const void *in)
2012{
2013 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
2014 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
2015 const struct talitos_export_state *export = in;
2016
2017 memset(req_ctx, 0, sizeof(*req_ctx));
2018 req_ctx->hw_context_size =
2019 (crypto_ahash_digestsize(tfm) <= SHA256_DIGEST_SIZE)
2020 ? TALITOS_MDEU_CONTEXT_SIZE_MD5_SHA1_SHA256
2021 : TALITOS_MDEU_CONTEXT_SIZE_SHA384_SHA512;
2022 memcpy(req_ctx->hw_context, export->hw_context,
2023 req_ctx->hw_context_size);
2024 memcpy(req_ctx->buf, export->buf, export->nbuf);
2025 req_ctx->swinit = export->swinit;
2026 req_ctx->first = export->first;
2027 req_ctx->last = export->last;
2028 req_ctx->to_hash_later = export->to_hash_later;
2029 req_ctx->nbuf = export->nbuf;
2030
2031 return 0;
2032}
2033
Lee Nipper79b3a412011-11-21 16:13:25 +08002034struct keyhash_result {
2035 struct completion completion;
2036 int err;
2037};
2038
2039static void keyhash_complete(struct crypto_async_request *req, int err)
2040{
2041 struct keyhash_result *res = req->data;
2042
2043 if (err == -EINPROGRESS)
2044 return;
2045
2046 res->err = err;
2047 complete(&res->completion);
2048}
2049
2050static int keyhash(struct crypto_ahash *tfm, const u8 *key, unsigned int keylen,
2051 u8 *hash)
2052{
2053 struct talitos_ctx *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm));
2054
2055 struct scatterlist sg[1];
2056 struct ahash_request *req;
2057 struct keyhash_result hresult;
2058 int ret;
2059
2060 init_completion(&hresult.completion);
2061
2062 req = ahash_request_alloc(tfm, GFP_KERNEL);
2063 if (!req)
2064 return -ENOMEM;
2065
2066 /* Keep tfm keylen == 0 during hash of the long key */
2067 ctx->keylen = 0;
2068 ahash_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
2069 keyhash_complete, &hresult);
2070
2071 sg_init_one(&sg[0], key, keylen);
2072
2073 ahash_request_set_crypt(req, sg, hash, keylen);
2074 ret = crypto_ahash_digest(req);
2075 switch (ret) {
2076 case 0:
2077 break;
2078 case -EINPROGRESS:
2079 case -EBUSY:
2080 ret = wait_for_completion_interruptible(
2081 &hresult.completion);
2082 if (!ret)
2083 ret = hresult.err;
2084 break;
2085 default:
2086 break;
2087 }
2088 ahash_request_free(req);
2089
2090 return ret;
2091}
2092
2093static int ahash_setkey(struct crypto_ahash *tfm, const u8 *key,
2094 unsigned int keylen)
2095{
2096 struct talitos_ctx *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm));
2097 unsigned int blocksize =
2098 crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm));
2099 unsigned int digestsize = crypto_ahash_digestsize(tfm);
2100 unsigned int keysize = keylen;
2101 u8 hash[SHA512_DIGEST_SIZE];
2102 int ret;
2103
2104 if (keylen <= blocksize)
2105 memcpy(ctx->key, key, keysize);
2106 else {
2107 /* Must get the hash of the long key */
2108 ret = keyhash(tfm, key, keylen, hash);
2109
2110 if (ret) {
2111 crypto_ahash_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
2112 return -EINVAL;
2113 }
2114
2115 keysize = digestsize;
2116 memcpy(ctx->key, hash, digestsize);
2117 }
2118
2119 ctx->keylen = keysize;
2120
2121 return 0;
2122}
2123
2124
Kim Phillips9c4a7962008-06-23 19:50:15 +08002125struct talitos_alg_template {
Lee Nipperd5e4aae2010-05-19 19:18:38 +10002126 u32 type;
2127 union {
2128 struct crypto_alg crypto;
Lee Nipperacbf7c622010-05-19 19:19:33 +10002129 struct ahash_alg hash;
Herbert Xuaeb4c132015-07-30 17:53:22 +08002130 struct aead_alg aead;
Lee Nipperd5e4aae2010-05-19 19:18:38 +10002131 } alg;
Kim Phillips9c4a7962008-06-23 19:50:15 +08002132 __be32 desc_hdr_template;
2133};
2134
2135static struct talitos_alg_template driver_algs[] = {
Horia Geanta991155b2013-03-20 16:31:38 +02002136 /* AEAD algorithms. These use a single-pass ipsec_esp descriptor */
Lee Nipperd5e4aae2010-05-19 19:18:38 +10002137 { .type = CRYPTO_ALG_TYPE_AEAD,
Herbert Xuaeb4c132015-07-30 17:53:22 +08002138 .alg.aead = {
2139 .base = {
2140 .cra_name = "authenc(hmac(sha1),cbc(aes))",
2141 .cra_driver_name = "authenc-hmac-sha1-"
2142 "cbc-aes-talitos",
2143 .cra_blocksize = AES_BLOCK_SIZE,
2144 .cra_flags = CRYPTO_ALG_ASYNC,
2145 },
2146 .ivsize = AES_BLOCK_SIZE,
2147 .maxauthsize = SHA1_DIGEST_SIZE,
Lee Nipper56af8cd2009-03-29 15:50:50 +08002148 },
Kim Phillips9c4a7962008-06-23 19:50:15 +08002149 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2150 DESC_HDR_SEL0_AESU |
2151 DESC_HDR_MODE0_AESU_CBC |
2152 DESC_HDR_SEL1_MDEUA |
2153 DESC_HDR_MODE1_MDEU_INIT |
2154 DESC_HDR_MODE1_MDEU_PAD |
2155 DESC_HDR_MODE1_MDEU_SHA1_HMAC,
Lee Nipper70bcaca2008-07-03 19:08:46 +08002156 },
Lee Nipperd5e4aae2010-05-19 19:18:38 +10002157 { .type = CRYPTO_ALG_TYPE_AEAD,
Herbert Xuaeb4c132015-07-30 17:53:22 +08002158 .alg.aead = {
2159 .base = {
2160 .cra_name = "authenc(hmac(sha1),"
2161 "cbc(des3_ede))",
2162 .cra_driver_name = "authenc-hmac-sha1-"
2163 "cbc-3des-talitos",
2164 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2165 .cra_flags = CRYPTO_ALG_ASYNC,
2166 },
2167 .ivsize = DES3_EDE_BLOCK_SIZE,
2168 .maxauthsize = SHA1_DIGEST_SIZE,
Lee Nipper56af8cd2009-03-29 15:50:50 +08002169 },
Lee Nipper70bcaca2008-07-03 19:08:46 +08002170 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2171 DESC_HDR_SEL0_DEU |
2172 DESC_HDR_MODE0_DEU_CBC |
2173 DESC_HDR_MODE0_DEU_3DES |
2174 DESC_HDR_SEL1_MDEUA |
2175 DESC_HDR_MODE1_MDEU_INIT |
2176 DESC_HDR_MODE1_MDEU_PAD |
2177 DESC_HDR_MODE1_MDEU_SHA1_HMAC,
Lee Nipper3952f172008-07-10 18:29:18 +08002178 },
Horia Geanta357fb602012-07-03 19:16:53 +03002179 { .type = CRYPTO_ALG_TYPE_AEAD,
Herbert Xuaeb4c132015-07-30 17:53:22 +08002180 .alg.aead = {
2181 .base = {
2182 .cra_name = "authenc(hmac(sha224),cbc(aes))",
2183 .cra_driver_name = "authenc-hmac-sha224-"
2184 "cbc-aes-talitos",
2185 .cra_blocksize = AES_BLOCK_SIZE,
2186 .cra_flags = CRYPTO_ALG_ASYNC,
2187 },
2188 .ivsize = AES_BLOCK_SIZE,
2189 .maxauthsize = SHA224_DIGEST_SIZE,
Horia Geanta357fb602012-07-03 19:16:53 +03002190 },
2191 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2192 DESC_HDR_SEL0_AESU |
2193 DESC_HDR_MODE0_AESU_CBC |
2194 DESC_HDR_SEL1_MDEUA |
2195 DESC_HDR_MODE1_MDEU_INIT |
2196 DESC_HDR_MODE1_MDEU_PAD |
2197 DESC_HDR_MODE1_MDEU_SHA224_HMAC,
2198 },
2199 { .type = CRYPTO_ALG_TYPE_AEAD,
Herbert Xuaeb4c132015-07-30 17:53:22 +08002200 .alg.aead = {
2201 .base = {
2202 .cra_name = "authenc(hmac(sha224),"
2203 "cbc(des3_ede))",
2204 .cra_driver_name = "authenc-hmac-sha224-"
2205 "cbc-3des-talitos",
2206 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2207 .cra_flags = CRYPTO_ALG_ASYNC,
2208 },
2209 .ivsize = DES3_EDE_BLOCK_SIZE,
2210 .maxauthsize = SHA224_DIGEST_SIZE,
Horia Geanta357fb602012-07-03 19:16:53 +03002211 },
2212 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2213 DESC_HDR_SEL0_DEU |
2214 DESC_HDR_MODE0_DEU_CBC |
2215 DESC_HDR_MODE0_DEU_3DES |
2216 DESC_HDR_SEL1_MDEUA |
2217 DESC_HDR_MODE1_MDEU_INIT |
2218 DESC_HDR_MODE1_MDEU_PAD |
2219 DESC_HDR_MODE1_MDEU_SHA224_HMAC,
2220 },
Lee Nipperd5e4aae2010-05-19 19:18:38 +10002221 { .type = CRYPTO_ALG_TYPE_AEAD,
Herbert Xuaeb4c132015-07-30 17:53:22 +08002222 .alg.aead = {
2223 .base = {
2224 .cra_name = "authenc(hmac(sha256),cbc(aes))",
2225 .cra_driver_name = "authenc-hmac-sha256-"
2226 "cbc-aes-talitos",
2227 .cra_blocksize = AES_BLOCK_SIZE,
2228 .cra_flags = CRYPTO_ALG_ASYNC,
2229 },
2230 .ivsize = AES_BLOCK_SIZE,
2231 .maxauthsize = SHA256_DIGEST_SIZE,
Lee Nipper56af8cd2009-03-29 15:50:50 +08002232 },
Lee Nipper3952f172008-07-10 18:29:18 +08002233 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2234 DESC_HDR_SEL0_AESU |
2235 DESC_HDR_MODE0_AESU_CBC |
2236 DESC_HDR_SEL1_MDEUA |
2237 DESC_HDR_MODE1_MDEU_INIT |
2238 DESC_HDR_MODE1_MDEU_PAD |
2239 DESC_HDR_MODE1_MDEU_SHA256_HMAC,
2240 },
Lee Nipperd5e4aae2010-05-19 19:18:38 +10002241 { .type = CRYPTO_ALG_TYPE_AEAD,
Herbert Xuaeb4c132015-07-30 17:53:22 +08002242 .alg.aead = {
2243 .base = {
2244 .cra_name = "authenc(hmac(sha256),"
2245 "cbc(des3_ede))",
2246 .cra_driver_name = "authenc-hmac-sha256-"
2247 "cbc-3des-talitos",
2248 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2249 .cra_flags = CRYPTO_ALG_ASYNC,
2250 },
2251 .ivsize = DES3_EDE_BLOCK_SIZE,
2252 .maxauthsize = SHA256_DIGEST_SIZE,
Lee Nipper56af8cd2009-03-29 15:50:50 +08002253 },
Lee Nipper3952f172008-07-10 18:29:18 +08002254 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2255 DESC_HDR_SEL0_DEU |
2256 DESC_HDR_MODE0_DEU_CBC |
2257 DESC_HDR_MODE0_DEU_3DES |
2258 DESC_HDR_SEL1_MDEUA |
2259 DESC_HDR_MODE1_MDEU_INIT |
2260 DESC_HDR_MODE1_MDEU_PAD |
2261 DESC_HDR_MODE1_MDEU_SHA256_HMAC,
2262 },
Lee Nipperd5e4aae2010-05-19 19:18:38 +10002263 { .type = CRYPTO_ALG_TYPE_AEAD,
Herbert Xuaeb4c132015-07-30 17:53:22 +08002264 .alg.aead = {
2265 .base = {
2266 .cra_name = "authenc(hmac(sha384),cbc(aes))",
2267 .cra_driver_name = "authenc-hmac-sha384-"
2268 "cbc-aes-talitos",
2269 .cra_blocksize = AES_BLOCK_SIZE,
2270 .cra_flags = CRYPTO_ALG_ASYNC,
2271 },
2272 .ivsize = AES_BLOCK_SIZE,
2273 .maxauthsize = SHA384_DIGEST_SIZE,
Horia Geanta357fb602012-07-03 19:16:53 +03002274 },
2275 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2276 DESC_HDR_SEL0_AESU |
2277 DESC_HDR_MODE0_AESU_CBC |
2278 DESC_HDR_SEL1_MDEUB |
2279 DESC_HDR_MODE1_MDEU_INIT |
2280 DESC_HDR_MODE1_MDEU_PAD |
2281 DESC_HDR_MODE1_MDEUB_SHA384_HMAC,
2282 },
2283 { .type = CRYPTO_ALG_TYPE_AEAD,
Herbert Xuaeb4c132015-07-30 17:53:22 +08002284 .alg.aead = {
2285 .base = {
2286 .cra_name = "authenc(hmac(sha384),"
2287 "cbc(des3_ede))",
2288 .cra_driver_name = "authenc-hmac-sha384-"
2289 "cbc-3des-talitos",
2290 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2291 .cra_flags = CRYPTO_ALG_ASYNC,
2292 },
2293 .ivsize = DES3_EDE_BLOCK_SIZE,
2294 .maxauthsize = SHA384_DIGEST_SIZE,
Horia Geanta357fb602012-07-03 19:16:53 +03002295 },
2296 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2297 DESC_HDR_SEL0_DEU |
2298 DESC_HDR_MODE0_DEU_CBC |
2299 DESC_HDR_MODE0_DEU_3DES |
2300 DESC_HDR_SEL1_MDEUB |
2301 DESC_HDR_MODE1_MDEU_INIT |
2302 DESC_HDR_MODE1_MDEU_PAD |
2303 DESC_HDR_MODE1_MDEUB_SHA384_HMAC,
2304 },
2305 { .type = CRYPTO_ALG_TYPE_AEAD,
Herbert Xuaeb4c132015-07-30 17:53:22 +08002306 .alg.aead = {
2307 .base = {
2308 .cra_name = "authenc(hmac(sha512),cbc(aes))",
2309 .cra_driver_name = "authenc-hmac-sha512-"
2310 "cbc-aes-talitos",
2311 .cra_blocksize = AES_BLOCK_SIZE,
2312 .cra_flags = CRYPTO_ALG_ASYNC,
2313 },
2314 .ivsize = AES_BLOCK_SIZE,
2315 .maxauthsize = SHA512_DIGEST_SIZE,
Horia Geanta357fb602012-07-03 19:16:53 +03002316 },
2317 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2318 DESC_HDR_SEL0_AESU |
2319 DESC_HDR_MODE0_AESU_CBC |
2320 DESC_HDR_SEL1_MDEUB |
2321 DESC_HDR_MODE1_MDEU_INIT |
2322 DESC_HDR_MODE1_MDEU_PAD |
2323 DESC_HDR_MODE1_MDEUB_SHA512_HMAC,
2324 },
2325 { .type = CRYPTO_ALG_TYPE_AEAD,
Herbert Xuaeb4c132015-07-30 17:53:22 +08002326 .alg.aead = {
2327 .base = {
2328 .cra_name = "authenc(hmac(sha512),"
2329 "cbc(des3_ede))",
2330 .cra_driver_name = "authenc-hmac-sha512-"
2331 "cbc-3des-talitos",
2332 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2333 .cra_flags = CRYPTO_ALG_ASYNC,
2334 },
2335 .ivsize = DES3_EDE_BLOCK_SIZE,
2336 .maxauthsize = SHA512_DIGEST_SIZE,
Horia Geanta357fb602012-07-03 19:16:53 +03002337 },
2338 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2339 DESC_HDR_SEL0_DEU |
2340 DESC_HDR_MODE0_DEU_CBC |
2341 DESC_HDR_MODE0_DEU_3DES |
2342 DESC_HDR_SEL1_MDEUB |
2343 DESC_HDR_MODE1_MDEU_INIT |
2344 DESC_HDR_MODE1_MDEU_PAD |
2345 DESC_HDR_MODE1_MDEUB_SHA512_HMAC,
2346 },
2347 { .type = CRYPTO_ALG_TYPE_AEAD,
Herbert Xuaeb4c132015-07-30 17:53:22 +08002348 .alg.aead = {
2349 .base = {
2350 .cra_name = "authenc(hmac(md5),cbc(aes))",
2351 .cra_driver_name = "authenc-hmac-md5-"
2352 "cbc-aes-talitos",
2353 .cra_blocksize = AES_BLOCK_SIZE,
2354 .cra_flags = CRYPTO_ALG_ASYNC,
2355 },
2356 .ivsize = AES_BLOCK_SIZE,
2357 .maxauthsize = MD5_DIGEST_SIZE,
Lee Nipper56af8cd2009-03-29 15:50:50 +08002358 },
Lee Nipper3952f172008-07-10 18:29:18 +08002359 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2360 DESC_HDR_SEL0_AESU |
2361 DESC_HDR_MODE0_AESU_CBC |
2362 DESC_HDR_SEL1_MDEUA |
2363 DESC_HDR_MODE1_MDEU_INIT |
2364 DESC_HDR_MODE1_MDEU_PAD |
2365 DESC_HDR_MODE1_MDEU_MD5_HMAC,
2366 },
Lee Nipperd5e4aae2010-05-19 19:18:38 +10002367 { .type = CRYPTO_ALG_TYPE_AEAD,
Herbert Xuaeb4c132015-07-30 17:53:22 +08002368 .alg.aead = {
2369 .base = {
2370 .cra_name = "authenc(hmac(md5),cbc(des3_ede))",
2371 .cra_driver_name = "authenc-hmac-md5-"
2372 "cbc-3des-talitos",
2373 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2374 .cra_flags = CRYPTO_ALG_ASYNC,
2375 },
2376 .ivsize = DES3_EDE_BLOCK_SIZE,
2377 .maxauthsize = MD5_DIGEST_SIZE,
Lee Nipper56af8cd2009-03-29 15:50:50 +08002378 },
Lee Nipper3952f172008-07-10 18:29:18 +08002379 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2380 DESC_HDR_SEL0_DEU |
2381 DESC_HDR_MODE0_DEU_CBC |
2382 DESC_HDR_MODE0_DEU_3DES |
2383 DESC_HDR_SEL1_MDEUA |
2384 DESC_HDR_MODE1_MDEU_INIT |
2385 DESC_HDR_MODE1_MDEU_PAD |
2386 DESC_HDR_MODE1_MDEU_MD5_HMAC,
Lee Nipper4de9d0b2009-03-29 15:52:32 +08002387 },
2388 /* ABLKCIPHER algorithms. */
Lee Nipperd5e4aae2010-05-19 19:18:38 +10002389 { .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
2390 .alg.crypto = {
LEROY Christophe5e75ae12015-12-01 12:44:15 +01002391 .cra_name = "ecb(aes)",
2392 .cra_driver_name = "ecb-aes-talitos",
2393 .cra_blocksize = AES_BLOCK_SIZE,
2394 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
2395 CRYPTO_ALG_ASYNC,
2396 .cra_ablkcipher = {
2397 .min_keysize = AES_MIN_KEY_SIZE,
2398 .max_keysize = AES_MAX_KEY_SIZE,
2399 .ivsize = AES_BLOCK_SIZE,
2400 }
2401 },
2402 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2403 DESC_HDR_SEL0_AESU,
2404 },
2405 { .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
2406 .alg.crypto = {
Lee Nipper4de9d0b2009-03-29 15:52:32 +08002407 .cra_name = "cbc(aes)",
2408 .cra_driver_name = "cbc-aes-talitos",
2409 .cra_blocksize = AES_BLOCK_SIZE,
2410 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
2411 CRYPTO_ALG_ASYNC,
Lee Nipper4de9d0b2009-03-29 15:52:32 +08002412 .cra_ablkcipher = {
Lee Nipper4de9d0b2009-03-29 15:52:32 +08002413 .min_keysize = AES_MIN_KEY_SIZE,
2414 .max_keysize = AES_MAX_KEY_SIZE,
2415 .ivsize = AES_BLOCK_SIZE,
2416 }
2417 },
2418 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2419 DESC_HDR_SEL0_AESU |
2420 DESC_HDR_MODE0_AESU_CBC,
2421 },
Lee Nipperd5e4aae2010-05-19 19:18:38 +10002422 { .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
2423 .alg.crypto = {
LEROY Christophe5e75ae12015-12-01 12:44:15 +01002424 .cra_name = "ctr(aes)",
2425 .cra_driver_name = "ctr-aes-talitos",
2426 .cra_blocksize = AES_BLOCK_SIZE,
2427 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
2428 CRYPTO_ALG_ASYNC,
2429 .cra_ablkcipher = {
2430 .min_keysize = AES_MIN_KEY_SIZE,
2431 .max_keysize = AES_MAX_KEY_SIZE,
2432 .ivsize = AES_BLOCK_SIZE,
2433 }
2434 },
2435 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2436 DESC_HDR_SEL0_AESU |
2437 DESC_HDR_MODE0_AESU_CTR,
2438 },
2439 { .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
2440 .alg.crypto = {
2441 .cra_name = "ecb(des)",
2442 .cra_driver_name = "ecb-des-talitos",
2443 .cra_blocksize = DES_BLOCK_SIZE,
2444 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
2445 CRYPTO_ALG_ASYNC,
2446 .cra_ablkcipher = {
2447 .min_keysize = DES_KEY_SIZE,
2448 .max_keysize = DES_KEY_SIZE,
2449 .ivsize = DES_BLOCK_SIZE,
2450 }
2451 },
2452 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2453 DESC_HDR_SEL0_DEU,
2454 },
2455 { .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
2456 .alg.crypto = {
2457 .cra_name = "cbc(des)",
2458 .cra_driver_name = "cbc-des-talitos",
2459 .cra_blocksize = DES_BLOCK_SIZE,
2460 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
2461 CRYPTO_ALG_ASYNC,
2462 .cra_ablkcipher = {
2463 .min_keysize = DES_KEY_SIZE,
2464 .max_keysize = DES_KEY_SIZE,
2465 .ivsize = DES_BLOCK_SIZE,
2466 }
2467 },
2468 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2469 DESC_HDR_SEL0_DEU |
2470 DESC_HDR_MODE0_DEU_CBC,
2471 },
2472 { .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
2473 .alg.crypto = {
2474 .cra_name = "ecb(des3_ede)",
2475 .cra_driver_name = "ecb-3des-talitos",
2476 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2477 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
2478 CRYPTO_ALG_ASYNC,
2479 .cra_ablkcipher = {
2480 .min_keysize = DES3_EDE_KEY_SIZE,
2481 .max_keysize = DES3_EDE_KEY_SIZE,
2482 .ivsize = DES3_EDE_BLOCK_SIZE,
2483 }
2484 },
2485 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2486 DESC_HDR_SEL0_DEU |
2487 DESC_HDR_MODE0_DEU_3DES,
2488 },
2489 { .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
2490 .alg.crypto = {
Lee Nipper4de9d0b2009-03-29 15:52:32 +08002491 .cra_name = "cbc(des3_ede)",
2492 .cra_driver_name = "cbc-3des-talitos",
2493 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2494 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
2495 CRYPTO_ALG_ASYNC,
Lee Nipper4de9d0b2009-03-29 15:52:32 +08002496 .cra_ablkcipher = {
Lee Nipper4de9d0b2009-03-29 15:52:32 +08002497 .min_keysize = DES3_EDE_KEY_SIZE,
2498 .max_keysize = DES3_EDE_KEY_SIZE,
2499 .ivsize = DES3_EDE_BLOCK_SIZE,
2500 }
2501 },
2502 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2503 DESC_HDR_SEL0_DEU |
2504 DESC_HDR_MODE0_DEU_CBC |
2505 DESC_HDR_MODE0_DEU_3DES,
Lee Nipper497f2e62010-05-19 19:20:36 +10002506 },
2507 /* AHASH algorithms. */
2508 { .type = CRYPTO_ALG_TYPE_AHASH,
2509 .alg.hash = {
Lee Nipper497f2e62010-05-19 19:20:36 +10002510 .halg.digestsize = MD5_DIGEST_SIZE,
Horia Geant?3639ca82016-04-21 19:24:55 +03002511 .halg.statesize = sizeof(struct talitos_export_state),
Lee Nipper497f2e62010-05-19 19:20:36 +10002512 .halg.base = {
2513 .cra_name = "md5",
2514 .cra_driver_name = "md5-talitos",
Martin Hicksb3988612015-03-03 08:21:34 -05002515 .cra_blocksize = MD5_HMAC_BLOCK_SIZE,
Lee Nipper497f2e62010-05-19 19:20:36 +10002516 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2517 CRYPTO_ALG_ASYNC,
Lee Nipper497f2e62010-05-19 19:20:36 +10002518 }
2519 },
2520 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2521 DESC_HDR_SEL0_MDEUA |
2522 DESC_HDR_MODE0_MDEU_MD5,
2523 },
2524 { .type = CRYPTO_ALG_TYPE_AHASH,
2525 .alg.hash = {
Lee Nipper497f2e62010-05-19 19:20:36 +10002526 .halg.digestsize = SHA1_DIGEST_SIZE,
Horia Geant?3639ca82016-04-21 19:24:55 +03002527 .halg.statesize = sizeof(struct talitos_export_state),
Lee Nipper497f2e62010-05-19 19:20:36 +10002528 .halg.base = {
2529 .cra_name = "sha1",
2530 .cra_driver_name = "sha1-talitos",
2531 .cra_blocksize = SHA1_BLOCK_SIZE,
2532 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2533 CRYPTO_ALG_ASYNC,
Lee Nipper497f2e62010-05-19 19:20:36 +10002534 }
2535 },
2536 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2537 DESC_HDR_SEL0_MDEUA |
2538 DESC_HDR_MODE0_MDEU_SHA1,
2539 },
2540 { .type = CRYPTO_ALG_TYPE_AHASH,
2541 .alg.hash = {
Kim Phillips60f208d2010-05-19 19:21:53 +10002542 .halg.digestsize = SHA224_DIGEST_SIZE,
Horia Geant?3639ca82016-04-21 19:24:55 +03002543 .halg.statesize = sizeof(struct talitos_export_state),
Kim Phillips60f208d2010-05-19 19:21:53 +10002544 .halg.base = {
2545 .cra_name = "sha224",
2546 .cra_driver_name = "sha224-talitos",
2547 .cra_blocksize = SHA224_BLOCK_SIZE,
2548 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2549 CRYPTO_ALG_ASYNC,
Kim Phillips60f208d2010-05-19 19:21:53 +10002550 }
2551 },
2552 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2553 DESC_HDR_SEL0_MDEUA |
2554 DESC_HDR_MODE0_MDEU_SHA224,
2555 },
2556 { .type = CRYPTO_ALG_TYPE_AHASH,
2557 .alg.hash = {
Lee Nipper497f2e62010-05-19 19:20:36 +10002558 .halg.digestsize = SHA256_DIGEST_SIZE,
Horia Geant?3639ca82016-04-21 19:24:55 +03002559 .halg.statesize = sizeof(struct talitos_export_state),
Lee Nipper497f2e62010-05-19 19:20:36 +10002560 .halg.base = {
2561 .cra_name = "sha256",
2562 .cra_driver_name = "sha256-talitos",
2563 .cra_blocksize = SHA256_BLOCK_SIZE,
2564 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2565 CRYPTO_ALG_ASYNC,
Lee Nipper497f2e62010-05-19 19:20:36 +10002566 }
2567 },
2568 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2569 DESC_HDR_SEL0_MDEUA |
2570 DESC_HDR_MODE0_MDEU_SHA256,
2571 },
2572 { .type = CRYPTO_ALG_TYPE_AHASH,
2573 .alg.hash = {
Lee Nipper497f2e62010-05-19 19:20:36 +10002574 .halg.digestsize = SHA384_DIGEST_SIZE,
Horia Geant?3639ca82016-04-21 19:24:55 +03002575 .halg.statesize = sizeof(struct talitos_export_state),
Lee Nipper497f2e62010-05-19 19:20:36 +10002576 .halg.base = {
2577 .cra_name = "sha384",
2578 .cra_driver_name = "sha384-talitos",
2579 .cra_blocksize = SHA384_BLOCK_SIZE,
2580 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2581 CRYPTO_ALG_ASYNC,
Lee Nipper497f2e62010-05-19 19:20:36 +10002582 }
2583 },
2584 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2585 DESC_HDR_SEL0_MDEUB |
2586 DESC_HDR_MODE0_MDEUB_SHA384,
2587 },
2588 { .type = CRYPTO_ALG_TYPE_AHASH,
2589 .alg.hash = {
Lee Nipper497f2e62010-05-19 19:20:36 +10002590 .halg.digestsize = SHA512_DIGEST_SIZE,
Horia Geant?3639ca82016-04-21 19:24:55 +03002591 .halg.statesize = sizeof(struct talitos_export_state),
Lee Nipper497f2e62010-05-19 19:20:36 +10002592 .halg.base = {
2593 .cra_name = "sha512",
2594 .cra_driver_name = "sha512-talitos",
2595 .cra_blocksize = SHA512_BLOCK_SIZE,
2596 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2597 CRYPTO_ALG_ASYNC,
Lee Nipper497f2e62010-05-19 19:20:36 +10002598 }
2599 },
2600 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2601 DESC_HDR_SEL0_MDEUB |
2602 DESC_HDR_MODE0_MDEUB_SHA512,
2603 },
Lee Nipper79b3a412011-11-21 16:13:25 +08002604 { .type = CRYPTO_ALG_TYPE_AHASH,
2605 .alg.hash = {
Lee Nipper79b3a412011-11-21 16:13:25 +08002606 .halg.digestsize = MD5_DIGEST_SIZE,
Horia Geant?3639ca82016-04-21 19:24:55 +03002607 .halg.statesize = sizeof(struct talitos_export_state),
Lee Nipper79b3a412011-11-21 16:13:25 +08002608 .halg.base = {
2609 .cra_name = "hmac(md5)",
2610 .cra_driver_name = "hmac-md5-talitos",
Martin Hicksb3988612015-03-03 08:21:34 -05002611 .cra_blocksize = MD5_HMAC_BLOCK_SIZE,
Lee Nipper79b3a412011-11-21 16:13:25 +08002612 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2613 CRYPTO_ALG_ASYNC,
Lee Nipper79b3a412011-11-21 16:13:25 +08002614 }
2615 },
2616 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2617 DESC_HDR_SEL0_MDEUA |
2618 DESC_HDR_MODE0_MDEU_MD5,
2619 },
2620 { .type = CRYPTO_ALG_TYPE_AHASH,
2621 .alg.hash = {
Lee Nipper79b3a412011-11-21 16:13:25 +08002622 .halg.digestsize = SHA1_DIGEST_SIZE,
Horia Geant?3639ca82016-04-21 19:24:55 +03002623 .halg.statesize = sizeof(struct talitos_export_state),
Lee Nipper79b3a412011-11-21 16:13:25 +08002624 .halg.base = {
2625 .cra_name = "hmac(sha1)",
2626 .cra_driver_name = "hmac-sha1-talitos",
2627 .cra_blocksize = SHA1_BLOCK_SIZE,
2628 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2629 CRYPTO_ALG_ASYNC,
Lee Nipper79b3a412011-11-21 16:13:25 +08002630 }
2631 },
2632 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2633 DESC_HDR_SEL0_MDEUA |
2634 DESC_HDR_MODE0_MDEU_SHA1,
2635 },
2636 { .type = CRYPTO_ALG_TYPE_AHASH,
2637 .alg.hash = {
Lee Nipper79b3a412011-11-21 16:13:25 +08002638 .halg.digestsize = SHA224_DIGEST_SIZE,
Horia Geant?3639ca82016-04-21 19:24:55 +03002639 .halg.statesize = sizeof(struct talitos_export_state),
Lee Nipper79b3a412011-11-21 16:13:25 +08002640 .halg.base = {
2641 .cra_name = "hmac(sha224)",
2642 .cra_driver_name = "hmac-sha224-talitos",
2643 .cra_blocksize = SHA224_BLOCK_SIZE,
2644 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2645 CRYPTO_ALG_ASYNC,
Lee Nipper79b3a412011-11-21 16:13:25 +08002646 }
2647 },
2648 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2649 DESC_HDR_SEL0_MDEUA |
2650 DESC_HDR_MODE0_MDEU_SHA224,
2651 },
2652 { .type = CRYPTO_ALG_TYPE_AHASH,
2653 .alg.hash = {
Lee Nipper79b3a412011-11-21 16:13:25 +08002654 .halg.digestsize = SHA256_DIGEST_SIZE,
Horia Geant?3639ca82016-04-21 19:24:55 +03002655 .halg.statesize = sizeof(struct talitos_export_state),
Lee Nipper79b3a412011-11-21 16:13:25 +08002656 .halg.base = {
2657 .cra_name = "hmac(sha256)",
2658 .cra_driver_name = "hmac-sha256-talitos",
2659 .cra_blocksize = SHA256_BLOCK_SIZE,
2660 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2661 CRYPTO_ALG_ASYNC,
Lee Nipper79b3a412011-11-21 16:13:25 +08002662 }
2663 },
2664 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2665 DESC_HDR_SEL0_MDEUA |
2666 DESC_HDR_MODE0_MDEU_SHA256,
2667 },
2668 { .type = CRYPTO_ALG_TYPE_AHASH,
2669 .alg.hash = {
Lee Nipper79b3a412011-11-21 16:13:25 +08002670 .halg.digestsize = SHA384_DIGEST_SIZE,
Horia Geant?3639ca82016-04-21 19:24:55 +03002671 .halg.statesize = sizeof(struct talitos_export_state),
Lee Nipper79b3a412011-11-21 16:13:25 +08002672 .halg.base = {
2673 .cra_name = "hmac(sha384)",
2674 .cra_driver_name = "hmac-sha384-talitos",
2675 .cra_blocksize = SHA384_BLOCK_SIZE,
2676 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2677 CRYPTO_ALG_ASYNC,
Lee Nipper79b3a412011-11-21 16:13:25 +08002678 }
2679 },
2680 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2681 DESC_HDR_SEL0_MDEUB |
2682 DESC_HDR_MODE0_MDEUB_SHA384,
2683 },
2684 { .type = CRYPTO_ALG_TYPE_AHASH,
2685 .alg.hash = {
Lee Nipper79b3a412011-11-21 16:13:25 +08002686 .halg.digestsize = SHA512_DIGEST_SIZE,
Horia Geant?3639ca82016-04-21 19:24:55 +03002687 .halg.statesize = sizeof(struct talitos_export_state),
Lee Nipper79b3a412011-11-21 16:13:25 +08002688 .halg.base = {
2689 .cra_name = "hmac(sha512)",
2690 .cra_driver_name = "hmac-sha512-talitos",
2691 .cra_blocksize = SHA512_BLOCK_SIZE,
2692 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2693 CRYPTO_ALG_ASYNC,
Lee Nipper79b3a412011-11-21 16:13:25 +08002694 }
2695 },
2696 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2697 DESC_HDR_SEL0_MDEUB |
2698 DESC_HDR_MODE0_MDEUB_SHA512,
2699 }
Kim Phillips9c4a7962008-06-23 19:50:15 +08002700};
2701
2702struct talitos_crypto_alg {
2703 struct list_head entry;
2704 struct device *dev;
Lee Nipperacbf7c622010-05-19 19:19:33 +10002705 struct talitos_alg_template algt;
Kim Phillips9c4a7962008-06-23 19:50:15 +08002706};
2707
Jonas Eymann89d124c2016-04-19 20:33:47 +03002708static int talitos_init_common(struct talitos_ctx *ctx,
2709 struct talitos_crypto_alg *talitos_alg)
Kim Phillips9c4a7962008-06-23 19:50:15 +08002710{
Kim Phillips5228f0f2011-07-15 11:21:38 +08002711 struct talitos_private *priv;
Kim Phillips9c4a7962008-06-23 19:50:15 +08002712
2713 /* update context with ptr to dev */
2714 ctx->dev = talitos_alg->dev;
Kim Phillips19bbbc62009-03-29 15:53:59 +08002715
Kim Phillips5228f0f2011-07-15 11:21:38 +08002716 /* assign SEC channel to tfm in round-robin fashion */
2717 priv = dev_get_drvdata(ctx->dev);
2718 ctx->ch = atomic_inc_return(&priv->last_chan) &
2719 (priv->num_channels - 1);
2720
Kim Phillips9c4a7962008-06-23 19:50:15 +08002721 /* copy descriptor header template value */
Lee Nipperacbf7c622010-05-19 19:19:33 +10002722 ctx->desc_hdr_template = talitos_alg->algt.desc_hdr_template;
Kim Phillips9c4a7962008-06-23 19:50:15 +08002723
Kim Phillips602dba52011-07-15 11:21:39 +08002724 /* select done notification */
2725 ctx->desc_hdr_template |= DESC_HDR_DONE_NOTIFY;
2726
Lee Nipper497f2e62010-05-19 19:20:36 +10002727 return 0;
2728}
2729
Jonas Eymann89d124c2016-04-19 20:33:47 +03002730static int talitos_cra_init(struct crypto_tfm *tfm)
2731{
2732 struct crypto_alg *alg = tfm->__crt_alg;
2733 struct talitos_crypto_alg *talitos_alg;
2734 struct talitos_ctx *ctx = crypto_tfm_ctx(tfm);
2735
2736 if ((alg->cra_flags & CRYPTO_ALG_TYPE_MASK) == CRYPTO_ALG_TYPE_AHASH)
2737 talitos_alg = container_of(__crypto_ahash_alg(alg),
2738 struct talitos_crypto_alg,
2739 algt.alg.hash);
2740 else
2741 talitos_alg = container_of(alg, struct talitos_crypto_alg,
2742 algt.alg.crypto);
2743
2744 return talitos_init_common(ctx, talitos_alg);
2745}
2746
Herbert Xuaeb4c132015-07-30 17:53:22 +08002747static int talitos_cra_init_aead(struct crypto_aead *tfm)
Lee Nipper497f2e62010-05-19 19:20:36 +10002748{
Jonas Eymann89d124c2016-04-19 20:33:47 +03002749 struct aead_alg *alg = crypto_aead_alg(tfm);
2750 struct talitos_crypto_alg *talitos_alg;
2751 struct talitos_ctx *ctx = crypto_aead_ctx(tfm);
2752
2753 talitos_alg = container_of(alg, struct talitos_crypto_alg,
2754 algt.alg.aead);
2755
2756 return talitos_init_common(ctx, talitos_alg);
Kim Phillips9c4a7962008-06-23 19:50:15 +08002757}
2758
Lee Nipper497f2e62010-05-19 19:20:36 +10002759static int talitos_cra_init_ahash(struct crypto_tfm *tfm)
2760{
2761 struct talitos_ctx *ctx = crypto_tfm_ctx(tfm);
2762
2763 talitos_cra_init(tfm);
2764
2765 ctx->keylen = 0;
2766 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
2767 sizeof(struct talitos_ahash_req_ctx));
2768
2769 return 0;
2770}
2771
Kim Phillips9c4a7962008-06-23 19:50:15 +08002772/*
2773 * given the alg's descriptor header template, determine whether descriptor
2774 * type and primary/secondary execution units required match the hw
2775 * capabilities description provided in the device tree node.
2776 */
2777static int hw_supports(struct device *dev, __be32 desc_hdr_template)
2778{
2779 struct talitos_private *priv = dev_get_drvdata(dev);
2780 int ret;
2781
2782 ret = (1 << DESC_TYPE(desc_hdr_template) & priv->desc_types) &&
2783 (1 << PRIMARY_EU(desc_hdr_template) & priv->exec_units);
2784
2785 if (SECONDARY_EU(desc_hdr_template))
2786 ret = ret && (1 << SECONDARY_EU(desc_hdr_template)
2787 & priv->exec_units);
2788
2789 return ret;
2790}
2791
Grant Likely2dc11582010-08-06 09:25:50 -06002792static int talitos_remove(struct platform_device *ofdev)
Kim Phillips9c4a7962008-06-23 19:50:15 +08002793{
2794 struct device *dev = &ofdev->dev;
2795 struct talitos_private *priv = dev_get_drvdata(dev);
2796 struct talitos_crypto_alg *t_alg, *n;
2797 int i;
2798
2799 list_for_each_entry_safe(t_alg, n, &priv->alg_list, entry) {
Lee Nipperacbf7c622010-05-19 19:19:33 +10002800 switch (t_alg->algt.type) {
2801 case CRYPTO_ALG_TYPE_ABLKCIPHER:
Lee Nipperacbf7c622010-05-19 19:19:33 +10002802 break;
Herbert Xuaeb4c132015-07-30 17:53:22 +08002803 case CRYPTO_ALG_TYPE_AEAD:
2804 crypto_unregister_aead(&t_alg->algt.alg.aead);
Lee Nipperacbf7c622010-05-19 19:19:33 +10002805 case CRYPTO_ALG_TYPE_AHASH:
2806 crypto_unregister_ahash(&t_alg->algt.alg.hash);
2807 break;
2808 }
Kim Phillips9c4a7962008-06-23 19:50:15 +08002809 list_del(&t_alg->entry);
2810 kfree(t_alg);
2811 }
2812
2813 if (hw_supports(dev, DESC_HDR_SEL0_RNG))
2814 talitos_unregister_rng(dev);
2815
Aaron Sierra35a3bb32015-08-05 16:52:08 -05002816 for (i = 0; priv->chan && i < priv->num_channels; i++)
Kim Phillips0b798242010-09-23 15:56:08 +08002817 kfree(priv->chan[i].fifo);
Kim Phillips9c4a7962008-06-23 19:50:15 +08002818
Kim Phillips4b9926282009-08-13 11:50:38 +10002819 kfree(priv->chan);
Kim Phillips9c4a7962008-06-23 19:50:15 +08002820
Kim Phillipsc3e337f2011-11-21 16:13:27 +08002821 for (i = 0; i < 2; i++)
Kim Phillips2cdba3c2011-12-12 14:59:11 -06002822 if (priv->irq[i]) {
Kim Phillipsc3e337f2011-11-21 16:13:27 +08002823 free_irq(priv->irq[i], dev);
2824 irq_dispose_mapping(priv->irq[i]);
2825 }
Kim Phillips9c4a7962008-06-23 19:50:15 +08002826
Kim Phillipsc3e337f2011-11-21 16:13:27 +08002827 tasklet_kill(&priv->done_task[0]);
Kim Phillips2cdba3c2011-12-12 14:59:11 -06002828 if (priv->irq[1])
Kim Phillipsc3e337f2011-11-21 16:13:27 +08002829 tasklet_kill(&priv->done_task[1]);
Kim Phillips9c4a7962008-06-23 19:50:15 +08002830
2831 iounmap(priv->reg);
2832
Kim Phillips9c4a7962008-06-23 19:50:15 +08002833 kfree(priv);
2834
2835 return 0;
2836}
2837
2838static struct talitos_crypto_alg *talitos_alg_alloc(struct device *dev,
2839 struct talitos_alg_template
2840 *template)
2841{
Kim Phillips60f208d2010-05-19 19:21:53 +10002842 struct talitos_private *priv = dev_get_drvdata(dev);
Kim Phillips9c4a7962008-06-23 19:50:15 +08002843 struct talitos_crypto_alg *t_alg;
2844 struct crypto_alg *alg;
2845
2846 t_alg = kzalloc(sizeof(struct talitos_crypto_alg), GFP_KERNEL);
2847 if (!t_alg)
2848 return ERR_PTR(-ENOMEM);
2849
Lee Nipperacbf7c622010-05-19 19:19:33 +10002850 t_alg->algt = *template;
2851
2852 switch (t_alg->algt.type) {
2853 case CRYPTO_ALG_TYPE_ABLKCIPHER:
Lee Nipper497f2e62010-05-19 19:20:36 +10002854 alg = &t_alg->algt.alg.crypto;
2855 alg->cra_init = talitos_cra_init;
Kim Phillipsd4cd3282012-08-08 20:32:00 -05002856 alg->cra_type = &crypto_ablkcipher_type;
Kim Phillipsb286e002012-08-08 20:33:34 -05002857 alg->cra_ablkcipher.setkey = ablkcipher_setkey;
2858 alg->cra_ablkcipher.encrypt = ablkcipher_encrypt;
2859 alg->cra_ablkcipher.decrypt = ablkcipher_decrypt;
2860 alg->cra_ablkcipher.geniv = "eseqiv";
Lee Nipper497f2e62010-05-19 19:20:36 +10002861 break;
Lee Nipperacbf7c622010-05-19 19:19:33 +10002862 case CRYPTO_ALG_TYPE_AEAD:
Herbert Xuaeb4c132015-07-30 17:53:22 +08002863 alg = &t_alg->algt.alg.aead.base;
Herbert Xuaeb4c132015-07-30 17:53:22 +08002864 t_alg->algt.alg.aead.init = talitos_cra_init_aead;
2865 t_alg->algt.alg.aead.setkey = aead_setkey;
2866 t_alg->algt.alg.aead.encrypt = aead_encrypt;
2867 t_alg->algt.alg.aead.decrypt = aead_decrypt;
Lee Nipperacbf7c622010-05-19 19:19:33 +10002868 break;
2869 case CRYPTO_ALG_TYPE_AHASH:
2870 alg = &t_alg->algt.alg.hash.halg.base;
Lee Nipper497f2e62010-05-19 19:20:36 +10002871 alg->cra_init = talitos_cra_init_ahash;
Kim Phillipsd4cd3282012-08-08 20:32:00 -05002872 alg->cra_type = &crypto_ahash_type;
Kim Phillipsb286e002012-08-08 20:33:34 -05002873 t_alg->algt.alg.hash.init = ahash_init;
2874 t_alg->algt.alg.hash.update = ahash_update;
2875 t_alg->algt.alg.hash.final = ahash_final;
2876 t_alg->algt.alg.hash.finup = ahash_finup;
2877 t_alg->algt.alg.hash.digest = ahash_digest;
2878 t_alg->algt.alg.hash.setkey = ahash_setkey;
Horia Geant?3639ca82016-04-21 19:24:55 +03002879 t_alg->algt.alg.hash.import = ahash_import;
2880 t_alg->algt.alg.hash.export = ahash_export;
Kim Phillipsb286e002012-08-08 20:33:34 -05002881
Lee Nipper79b3a412011-11-21 16:13:25 +08002882 if (!(priv->features & TALITOS_FTR_HMAC_OK) &&
Kim Phillips0b2730d2011-12-12 14:59:10 -06002883 !strncmp(alg->cra_name, "hmac", 4)) {
2884 kfree(t_alg);
Lee Nipper79b3a412011-11-21 16:13:25 +08002885 return ERR_PTR(-ENOTSUPP);
Kim Phillips0b2730d2011-12-12 14:59:10 -06002886 }
Kim Phillips60f208d2010-05-19 19:21:53 +10002887 if (!(priv->features & TALITOS_FTR_SHA224_HWINIT) &&
Lee Nipper79b3a412011-11-21 16:13:25 +08002888 (!strcmp(alg->cra_name, "sha224") ||
2889 !strcmp(alg->cra_name, "hmac(sha224)"))) {
Kim Phillips60f208d2010-05-19 19:21:53 +10002890 t_alg->algt.alg.hash.init = ahash_init_sha224_swinit;
2891 t_alg->algt.desc_hdr_template =
2892 DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2893 DESC_HDR_SEL0_MDEUA |
2894 DESC_HDR_MODE0_MDEU_SHA256;
2895 }
Lee Nipper497f2e62010-05-19 19:20:36 +10002896 break;
Kim Phillips1d119112010-09-23 15:55:27 +08002897 default:
2898 dev_err(dev, "unknown algorithm type %d\n", t_alg->algt.type);
Horia Geant?5fa7dad2015-05-11 20:03:24 +03002899 kfree(t_alg);
Kim Phillips1d119112010-09-23 15:55:27 +08002900 return ERR_PTR(-EINVAL);
Lee Nipperacbf7c622010-05-19 19:19:33 +10002901 }
Kim Phillips9c4a7962008-06-23 19:50:15 +08002902
Kim Phillips9c4a7962008-06-23 19:50:15 +08002903 alg->cra_module = THIS_MODULE;
Kim Phillips9c4a7962008-06-23 19:50:15 +08002904 alg->cra_priority = TALITOS_CRA_PRIORITY;
Kim Phillips9c4a7962008-06-23 19:50:15 +08002905 alg->cra_alignmask = 0;
Kim Phillips9c4a7962008-06-23 19:50:15 +08002906 alg->cra_ctxsize = sizeof(struct talitos_ctx);
Nikos Mavrogiannopoulosd912bb72011-11-01 13:39:56 +01002907 alg->cra_flags |= CRYPTO_ALG_KERN_DRIVER_ONLY;
Kim Phillips9c4a7962008-06-23 19:50:15 +08002908
Kim Phillips9c4a7962008-06-23 19:50:15 +08002909 t_alg->dev = dev;
2910
2911 return t_alg;
2912}
2913
Kim Phillipsc3e337f2011-11-21 16:13:27 +08002914static int talitos_probe_irq(struct platform_device *ofdev)
2915{
2916 struct device *dev = &ofdev->dev;
2917 struct device_node *np = ofdev->dev.of_node;
2918 struct talitos_private *priv = dev_get_drvdata(dev);
2919 int err;
LEROY Christophedd3c0982015-04-17 16:32:13 +02002920 bool is_sec1 = has_ftr_sec1(priv);
Kim Phillipsc3e337f2011-11-21 16:13:27 +08002921
2922 priv->irq[0] = irq_of_parse_and_map(np, 0);
Kim Phillips2cdba3c2011-12-12 14:59:11 -06002923 if (!priv->irq[0]) {
Kim Phillipsc3e337f2011-11-21 16:13:27 +08002924 dev_err(dev, "failed to map irq\n");
2925 return -EINVAL;
2926 }
LEROY Christophedd3c0982015-04-17 16:32:13 +02002927 if (is_sec1) {
2928 err = request_irq(priv->irq[0], talitos1_interrupt_4ch, 0,
2929 dev_driver_string(dev), dev);
2930 goto primary_out;
2931 }
Kim Phillipsc3e337f2011-11-21 16:13:27 +08002932
2933 priv->irq[1] = irq_of_parse_and_map(np, 1);
2934
2935 /* get the primary irq line */
Kim Phillips2cdba3c2011-12-12 14:59:11 -06002936 if (!priv->irq[1]) {
LEROY Christophedd3c0982015-04-17 16:32:13 +02002937 err = request_irq(priv->irq[0], talitos2_interrupt_4ch, 0,
Kim Phillipsc3e337f2011-11-21 16:13:27 +08002938 dev_driver_string(dev), dev);
2939 goto primary_out;
2940 }
2941
LEROY Christophedd3c0982015-04-17 16:32:13 +02002942 err = request_irq(priv->irq[0], talitos2_interrupt_ch0_2, 0,
Kim Phillipsc3e337f2011-11-21 16:13:27 +08002943 dev_driver_string(dev), dev);
2944 if (err)
2945 goto primary_out;
2946
2947 /* get the secondary irq line */
LEROY Christophedd3c0982015-04-17 16:32:13 +02002948 err = request_irq(priv->irq[1], talitos2_interrupt_ch1_3, 0,
Kim Phillipsc3e337f2011-11-21 16:13:27 +08002949 dev_driver_string(dev), dev);
2950 if (err) {
2951 dev_err(dev, "failed to request secondary irq\n");
2952 irq_dispose_mapping(priv->irq[1]);
Kim Phillips2cdba3c2011-12-12 14:59:11 -06002953 priv->irq[1] = 0;
Kim Phillipsc3e337f2011-11-21 16:13:27 +08002954 }
2955
2956 return err;
2957
2958primary_out:
2959 if (err) {
2960 dev_err(dev, "failed to request primary irq\n");
2961 irq_dispose_mapping(priv->irq[0]);
Kim Phillips2cdba3c2011-12-12 14:59:11 -06002962 priv->irq[0] = 0;
Kim Phillipsc3e337f2011-11-21 16:13:27 +08002963 }
2964
2965 return err;
2966}
2967
Grant Likely1c48a5c2011-02-17 02:43:24 -07002968static int talitos_probe(struct platform_device *ofdev)
Kim Phillips9c4a7962008-06-23 19:50:15 +08002969{
2970 struct device *dev = &ofdev->dev;
Grant Likely61c7a082010-04-13 16:12:29 -07002971 struct device_node *np = ofdev->dev.of_node;
Kim Phillips9c4a7962008-06-23 19:50:15 +08002972 struct talitos_private *priv;
2973 const unsigned int *prop;
2974 int i, err;
LEROY Christophe5fa7fa12015-04-17 16:32:11 +02002975 int stride;
Kim Phillips9c4a7962008-06-23 19:50:15 +08002976
2977 priv = kzalloc(sizeof(struct talitos_private), GFP_KERNEL);
2978 if (!priv)
2979 return -ENOMEM;
2980
Kevin Haof3de9cb2014-01-28 20:17:23 +08002981 INIT_LIST_HEAD(&priv->alg_list);
2982
Kim Phillips9c4a7962008-06-23 19:50:15 +08002983 dev_set_drvdata(dev, priv);
2984
2985 priv->ofdev = ofdev;
2986
Horia Geanta511d63c2012-03-30 17:49:53 +03002987 spin_lock_init(&priv->reg_lock);
2988
Kim Phillips9c4a7962008-06-23 19:50:15 +08002989 priv->reg = of_iomap(np, 0);
2990 if (!priv->reg) {
2991 dev_err(dev, "failed to of_iomap\n");
2992 err = -ENOMEM;
2993 goto err_out;
2994 }
2995
2996 /* get SEC version capabilities from device tree */
2997 prop = of_get_property(np, "fsl,num-channels", NULL);
2998 if (prop)
2999 priv->num_channels = *prop;
3000
3001 prop = of_get_property(np, "fsl,channel-fifo-len", NULL);
3002 if (prop)
3003 priv->chfifo_len = *prop;
3004
3005 prop = of_get_property(np, "fsl,exec-units-mask", NULL);
3006 if (prop)
3007 priv->exec_units = *prop;
3008
3009 prop = of_get_property(np, "fsl,descriptor-types-mask", NULL);
3010 if (prop)
3011 priv->desc_types = *prop;
3012
3013 if (!is_power_of_2(priv->num_channels) || !priv->chfifo_len ||
3014 !priv->exec_units || !priv->desc_types) {
3015 dev_err(dev, "invalid property data in device tree node\n");
3016 err = -EINVAL;
3017 goto err_out;
3018 }
3019
Lee Nipperf3c85bc2008-07-30 16:26:57 +08003020 if (of_device_is_compatible(np, "fsl,sec3.0"))
3021 priv->features |= TALITOS_FTR_SRC_LINK_TBL_LEN_INCLUDES_EXTENT;
3022
Kim Phillipsfe5720e2008-10-12 20:33:14 +08003023 if (of_device_is_compatible(np, "fsl,sec2.1"))
Kim Phillips60f208d2010-05-19 19:21:53 +10003024 priv->features |= TALITOS_FTR_HW_AUTH_CHECK |
Lee Nipper79b3a412011-11-21 16:13:25 +08003025 TALITOS_FTR_SHA224_HWINIT |
3026 TALITOS_FTR_HMAC_OK;
Kim Phillipsfe5720e2008-10-12 20:33:14 +08003027
LEROY Christophe21590882015-04-17 16:32:05 +02003028 if (of_device_is_compatible(np, "fsl,sec1.0"))
3029 priv->features |= TALITOS_FTR_SEC1;
3030
LEROY Christophe5fa7fa12015-04-17 16:32:11 +02003031 if (of_device_is_compatible(np, "fsl,sec1.2")) {
3032 priv->reg_deu = priv->reg + TALITOS12_DEU;
3033 priv->reg_aesu = priv->reg + TALITOS12_AESU;
3034 priv->reg_mdeu = priv->reg + TALITOS12_MDEU;
3035 stride = TALITOS1_CH_STRIDE;
3036 } else if (of_device_is_compatible(np, "fsl,sec1.0")) {
3037 priv->reg_deu = priv->reg + TALITOS10_DEU;
3038 priv->reg_aesu = priv->reg + TALITOS10_AESU;
3039 priv->reg_mdeu = priv->reg + TALITOS10_MDEU;
3040 priv->reg_afeu = priv->reg + TALITOS10_AFEU;
3041 priv->reg_rngu = priv->reg + TALITOS10_RNGU;
3042 priv->reg_pkeu = priv->reg + TALITOS10_PKEU;
3043 stride = TALITOS1_CH_STRIDE;
3044 } else {
3045 priv->reg_deu = priv->reg + TALITOS2_DEU;
3046 priv->reg_aesu = priv->reg + TALITOS2_AESU;
3047 priv->reg_mdeu = priv->reg + TALITOS2_MDEU;
3048 priv->reg_afeu = priv->reg + TALITOS2_AFEU;
3049 priv->reg_rngu = priv->reg + TALITOS2_RNGU;
3050 priv->reg_pkeu = priv->reg + TALITOS2_PKEU;
3051 priv->reg_keu = priv->reg + TALITOS2_KEU;
3052 priv->reg_crcu = priv->reg + TALITOS2_CRCU;
3053 stride = TALITOS2_CH_STRIDE;
3054 }
3055
LEROY Christophedd3c0982015-04-17 16:32:13 +02003056 err = talitos_probe_irq(ofdev);
3057 if (err)
3058 goto err_out;
3059
3060 if (of_device_is_compatible(np, "fsl,sec1.0")) {
3061 tasklet_init(&priv->done_task[0], talitos1_done_4ch,
3062 (unsigned long)dev);
3063 } else {
3064 if (!priv->irq[1]) {
3065 tasklet_init(&priv->done_task[0], talitos2_done_4ch,
3066 (unsigned long)dev);
3067 } else {
3068 tasklet_init(&priv->done_task[0], talitos2_done_ch0_2,
3069 (unsigned long)dev);
3070 tasklet_init(&priv->done_task[1], talitos2_done_ch1_3,
3071 (unsigned long)dev);
3072 }
3073 }
3074
Kim Phillips4b9926282009-08-13 11:50:38 +10003075 priv->chan = kzalloc(sizeof(struct talitos_channel) *
3076 priv->num_channels, GFP_KERNEL);
3077 if (!priv->chan) {
3078 dev_err(dev, "failed to allocate channel management space\n");
Kim Phillips9c4a7962008-06-23 19:50:15 +08003079 err = -ENOMEM;
3080 goto err_out;
3081 }
3082
Martin Hicksf641ddd2015-03-03 08:21:33 -05003083 priv->fifo_len = roundup_pow_of_two(priv->chfifo_len);
3084
Kim Phillipsc3e337f2011-11-21 16:13:27 +08003085 for (i = 0; i < priv->num_channels; i++) {
LEROY Christophe5fa7fa12015-04-17 16:32:11 +02003086 priv->chan[i].reg = priv->reg + stride * (i + 1);
Kim Phillips2cdba3c2011-12-12 14:59:11 -06003087 if (!priv->irq[1] || !(i & 1))
Kim Phillipsc3e337f2011-11-21 16:13:27 +08003088 priv->chan[i].reg += TALITOS_CH_BASE_OFFSET;
Kim Phillipsad42d5f2011-11-21 16:13:27 +08003089
Kim Phillips4b9926282009-08-13 11:50:38 +10003090 spin_lock_init(&priv->chan[i].head_lock);
3091 spin_lock_init(&priv->chan[i].tail_lock);
Kim Phillips9c4a7962008-06-23 19:50:15 +08003092
Kim Phillips4b9926282009-08-13 11:50:38 +10003093 priv->chan[i].fifo = kzalloc(sizeof(struct talitos_request) *
3094 priv->fifo_len, GFP_KERNEL);
3095 if (!priv->chan[i].fifo) {
Kim Phillips9c4a7962008-06-23 19:50:15 +08003096 dev_err(dev, "failed to allocate request fifo %d\n", i);
3097 err = -ENOMEM;
3098 goto err_out;
3099 }
Kim Phillips9c4a7962008-06-23 19:50:15 +08003100
Kim Phillips4b9926282009-08-13 11:50:38 +10003101 atomic_set(&priv->chan[i].submit_count,
3102 -(priv->chfifo_len - 1));
Martin Hicksf641ddd2015-03-03 08:21:33 -05003103 }
Kim Phillips9c4a7962008-06-23 19:50:15 +08003104
Kim Phillips81eb0242009-08-13 11:51:51 +10003105 dma_set_mask(dev, DMA_BIT_MASK(36));
3106
Kim Phillips9c4a7962008-06-23 19:50:15 +08003107 /* reset and initialize the h/w */
3108 err = init_device(dev);
3109 if (err) {
3110 dev_err(dev, "failed to initialize device\n");
3111 goto err_out;
3112 }
3113
3114 /* register the RNG, if available */
3115 if (hw_supports(dev, DESC_HDR_SEL0_RNG)) {
3116 err = talitos_register_rng(dev);
3117 if (err) {
3118 dev_err(dev, "failed to register hwrng: %d\n", err);
3119 goto err_out;
3120 } else
3121 dev_info(dev, "hwrng\n");
3122 }
3123
3124 /* register crypto algorithms the device supports */
Kim Phillips9c4a7962008-06-23 19:50:15 +08003125 for (i = 0; i < ARRAY_SIZE(driver_algs); i++) {
3126 if (hw_supports(dev, driver_algs[i].desc_hdr_template)) {
3127 struct talitos_crypto_alg *t_alg;
Herbert Xuaeb4c132015-07-30 17:53:22 +08003128 struct crypto_alg *alg = NULL;
Kim Phillips9c4a7962008-06-23 19:50:15 +08003129
3130 t_alg = talitos_alg_alloc(dev, &driver_algs[i]);
3131 if (IS_ERR(t_alg)) {
3132 err = PTR_ERR(t_alg);
Kim Phillips0b2730d2011-12-12 14:59:10 -06003133 if (err == -ENOTSUPP)
Lee Nipper79b3a412011-11-21 16:13:25 +08003134 continue;
Kim Phillips9c4a7962008-06-23 19:50:15 +08003135 goto err_out;
3136 }
3137
Lee Nipperacbf7c622010-05-19 19:19:33 +10003138 switch (t_alg->algt.type) {
3139 case CRYPTO_ALG_TYPE_ABLKCIPHER:
Lee Nipperacbf7c622010-05-19 19:19:33 +10003140 err = crypto_register_alg(
3141 &t_alg->algt.alg.crypto);
Herbert Xuaeb4c132015-07-30 17:53:22 +08003142 alg = &t_alg->algt.alg.crypto;
Lee Nipperacbf7c622010-05-19 19:19:33 +10003143 break;
Herbert Xuaeb4c132015-07-30 17:53:22 +08003144
3145 case CRYPTO_ALG_TYPE_AEAD:
3146 err = crypto_register_aead(
3147 &t_alg->algt.alg.aead);
3148 alg = &t_alg->algt.alg.aead.base;
3149 break;
3150
Lee Nipperacbf7c622010-05-19 19:19:33 +10003151 case CRYPTO_ALG_TYPE_AHASH:
3152 err = crypto_register_ahash(
3153 &t_alg->algt.alg.hash);
Herbert Xuaeb4c132015-07-30 17:53:22 +08003154 alg = &t_alg->algt.alg.hash.halg.base;
Lee Nipperacbf7c622010-05-19 19:19:33 +10003155 break;
3156 }
Kim Phillips9c4a7962008-06-23 19:50:15 +08003157 if (err) {
3158 dev_err(dev, "%s alg registration failed\n",
Herbert Xuaeb4c132015-07-30 17:53:22 +08003159 alg->cra_driver_name);
Kim Phillips9c4a7962008-06-23 19:50:15 +08003160 kfree(t_alg);
Horia Geanta991155b2013-03-20 16:31:38 +02003161 } else
Kim Phillips9c4a7962008-06-23 19:50:15 +08003162 list_add_tail(&t_alg->entry, &priv->alg_list);
Kim Phillips9c4a7962008-06-23 19:50:15 +08003163 }
3164 }
Kim Phillips5b859b6e2011-11-21 16:13:26 +08003165 if (!list_empty(&priv->alg_list))
3166 dev_info(dev, "%s algorithms registered in /proc/crypto\n",
3167 (char *)of_get_property(np, "compatible", NULL));
Kim Phillips9c4a7962008-06-23 19:50:15 +08003168
3169 return 0;
3170
3171err_out:
3172 talitos_remove(ofdev);
Kim Phillips9c4a7962008-06-23 19:50:15 +08003173
3174 return err;
3175}
3176
Márton Németh6c3f9752010-01-17 21:54:01 +11003177static const struct of_device_id talitos_match[] = {
LEROY Christophe0635b7d2015-04-17 16:32:20 +02003178#ifdef CONFIG_CRYPTO_DEV_TALITOS1
3179 {
3180 .compatible = "fsl,sec1.0",
3181 },
3182#endif
3183#ifdef CONFIG_CRYPTO_DEV_TALITOS2
Kim Phillips9c4a7962008-06-23 19:50:15 +08003184 {
3185 .compatible = "fsl,sec2.0",
3186 },
LEROY Christophe0635b7d2015-04-17 16:32:20 +02003187#endif
Kim Phillips9c4a7962008-06-23 19:50:15 +08003188 {},
3189};
3190MODULE_DEVICE_TABLE(of, talitos_match);
3191
Grant Likely1c48a5c2011-02-17 02:43:24 -07003192static struct platform_driver talitos_driver = {
Grant Likely40182942010-04-13 16:13:02 -07003193 .driver = {
3194 .name = "talitos",
Grant Likely40182942010-04-13 16:13:02 -07003195 .of_match_table = talitos_match,
3196 },
Kim Phillips9c4a7962008-06-23 19:50:15 +08003197 .probe = talitos_probe,
Al Viro596f1032008-11-22 17:34:24 +00003198 .remove = talitos_remove,
Kim Phillips9c4a7962008-06-23 19:50:15 +08003199};
3200
Axel Lin741e8c22011-11-26 21:26:19 +08003201module_platform_driver(talitos_driver);
Kim Phillips9c4a7962008-06-23 19:50:15 +08003202
3203MODULE_LICENSE("GPL");
3204MODULE_AUTHOR("Kim Phillips <kim.phillips@freescale.com>");
3205MODULE_DESCRIPTION("Freescale integrated security engine (SEC) driver");