Gary R Hook | ea0375a | 2016-03-01 13:49:25 -0600 | [diff] [blame] | 1 | /* |
| 2 | * AMD Cryptographic Coprocessor (CCP) driver |
| 3 | * |
| 4 | * Copyright (C) 2013,2016 Advanced Micro Devices, Inc. |
| 5 | * |
| 6 | * Author: Tom Lendacky <thomas.lendacky@amd.com> |
Gary R Hook | fba8855 | 2016-07-26 19:09:20 -0500 | [diff] [blame] | 7 | * Author: Gary R Hook <gary.hook@amd.com> |
Gary R Hook | ea0375a | 2016-03-01 13:49:25 -0600 | [diff] [blame] | 8 | * |
| 9 | * This program is free software; you can redistribute it and/or modify |
| 10 | * it under the terms of the GNU General Public License version 2 as |
| 11 | * published by the Free Software Foundation. |
| 12 | */ |
| 13 | |
| 14 | #include <linux/module.h> |
| 15 | #include <linux/kernel.h> |
| 16 | #include <linux/pci.h> |
| 17 | #include <linux/kthread.h> |
| 18 | #include <linux/interrupt.h> |
| 19 | #include <linux/ccp.h> |
| 20 | |
| 21 | #include "ccp-dev.h" |
| 22 | |
Gary R Hook | 58a690b | 2016-07-26 19:09:50 -0500 | [diff] [blame] | 23 | static u32 ccp_alloc_ksb(struct ccp_cmd_queue *cmd_q, unsigned int count) |
| 24 | { |
| 25 | int start; |
| 26 | struct ccp_device *ccp = cmd_q->ccp; |
| 27 | |
| 28 | for (;;) { |
| 29 | mutex_lock(&ccp->sb_mutex); |
| 30 | |
| 31 | start = (u32)bitmap_find_next_zero_area(ccp->sb, |
| 32 | ccp->sb_count, |
| 33 | ccp->sb_start, |
| 34 | count, 0); |
| 35 | if (start <= ccp->sb_count) { |
| 36 | bitmap_set(ccp->sb, start, count); |
| 37 | |
| 38 | mutex_unlock(&ccp->sb_mutex); |
| 39 | break; |
| 40 | } |
| 41 | |
| 42 | ccp->sb_avail = 0; |
| 43 | |
| 44 | mutex_unlock(&ccp->sb_mutex); |
| 45 | |
| 46 | /* Wait for KSB entries to become available */ |
| 47 | if (wait_event_interruptible(ccp->sb_queue, ccp->sb_avail)) |
| 48 | return 0; |
| 49 | } |
| 50 | |
| 51 | return KSB_START + start; |
| 52 | } |
| 53 | |
| 54 | static void ccp_free_ksb(struct ccp_cmd_queue *cmd_q, unsigned int start, |
| 55 | unsigned int count) |
| 56 | { |
| 57 | struct ccp_device *ccp = cmd_q->ccp; |
| 58 | |
| 59 | if (!start) |
| 60 | return; |
| 61 | |
| 62 | mutex_lock(&ccp->sb_mutex); |
| 63 | |
| 64 | bitmap_clear(ccp->sb, start - KSB_START, count); |
| 65 | |
| 66 | ccp->sb_avail = 1; |
| 67 | |
| 68 | mutex_unlock(&ccp->sb_mutex); |
| 69 | |
| 70 | wake_up_interruptible_all(&ccp->sb_queue); |
| 71 | } |
| 72 | |
Gary R Hook | bb4e89b | 2016-07-26 19:10:13 -0500 | [diff] [blame] | 73 | static unsigned int ccp_get_free_slots(struct ccp_cmd_queue *cmd_q) |
| 74 | { |
| 75 | return CMD_Q_DEPTH(ioread32(cmd_q->reg_status)); |
| 76 | } |
| 77 | |
Gary R Hook | ea0375a | 2016-03-01 13:49:25 -0600 | [diff] [blame] | 78 | static int ccp_do_cmd(struct ccp_op *op, u32 *cr, unsigned int cr_count) |
| 79 | { |
| 80 | struct ccp_cmd_queue *cmd_q = op->cmd_q; |
| 81 | struct ccp_device *ccp = cmd_q->ccp; |
| 82 | void __iomem *cr_addr; |
| 83 | u32 cr0, cmd; |
| 84 | unsigned int i; |
| 85 | int ret = 0; |
| 86 | |
| 87 | /* We could read a status register to see how many free slots |
| 88 | * are actually available, but reading that register resets it |
| 89 | * and you could lose some error information. |
| 90 | */ |
| 91 | cmd_q->free_slots--; |
| 92 | |
| 93 | cr0 = (cmd_q->id << REQ0_CMD_Q_SHIFT) |
| 94 | | (op->jobid << REQ0_JOBID_SHIFT) |
| 95 | | REQ0_WAIT_FOR_WRITE; |
| 96 | |
| 97 | if (op->soc) |
| 98 | cr0 |= REQ0_STOP_ON_COMPLETE |
| 99 | | REQ0_INT_ON_COMPLETE; |
| 100 | |
| 101 | if (op->ioc || !cmd_q->free_slots) |
| 102 | cr0 |= REQ0_INT_ON_COMPLETE; |
| 103 | |
| 104 | /* Start at CMD_REQ1 */ |
| 105 | cr_addr = ccp->io_regs + CMD_REQ0 + CMD_REQ_INCR; |
| 106 | |
| 107 | mutex_lock(&ccp->req_mutex); |
| 108 | |
| 109 | /* Write CMD_REQ1 through CMD_REQx first */ |
| 110 | for (i = 0; i < cr_count; i++, cr_addr += CMD_REQ_INCR) |
| 111 | iowrite32(*(cr + i), cr_addr); |
| 112 | |
| 113 | /* Tell the CCP to start */ |
| 114 | wmb(); |
| 115 | iowrite32(cr0, ccp->io_regs + CMD_REQ0); |
| 116 | |
| 117 | mutex_unlock(&ccp->req_mutex); |
| 118 | |
| 119 | if (cr0 & REQ0_INT_ON_COMPLETE) { |
| 120 | /* Wait for the job to complete */ |
| 121 | ret = wait_event_interruptible(cmd_q->int_queue, |
| 122 | cmd_q->int_rcvd); |
| 123 | if (ret || cmd_q->cmd_error) { |
| 124 | /* On error delete all related jobs from the queue */ |
| 125 | cmd = (cmd_q->id << DEL_Q_ID_SHIFT) |
| 126 | | op->jobid; |
Gary R Hook | 81422ba | 2016-09-28 11:53:56 -0500 | [diff] [blame] | 127 | if (cmd_q->cmd_error) |
| 128 | ccp_log_error(cmd_q->ccp, |
| 129 | cmd_q->cmd_error); |
Gary R Hook | ea0375a | 2016-03-01 13:49:25 -0600 | [diff] [blame] | 130 | |
| 131 | iowrite32(cmd, ccp->io_regs + DEL_CMD_Q_JOB); |
| 132 | |
| 133 | if (!ret) |
| 134 | ret = -EIO; |
| 135 | } else if (op->soc) { |
| 136 | /* Delete just head job from the queue on SoC */ |
| 137 | cmd = DEL_Q_ACTIVE |
| 138 | | (cmd_q->id << DEL_Q_ID_SHIFT) |
| 139 | | op->jobid; |
| 140 | |
| 141 | iowrite32(cmd, ccp->io_regs + DEL_CMD_Q_JOB); |
| 142 | } |
| 143 | |
| 144 | cmd_q->free_slots = CMD_Q_DEPTH(cmd_q->q_status); |
| 145 | |
| 146 | cmd_q->int_rcvd = 0; |
| 147 | } |
| 148 | |
| 149 | return ret; |
| 150 | } |
| 151 | |
| 152 | static int ccp_perform_aes(struct ccp_op *op) |
| 153 | { |
| 154 | u32 cr[6]; |
| 155 | |
| 156 | /* Fill out the register contents for REQ1 through REQ6 */ |
| 157 | cr[0] = (CCP_ENGINE_AES << REQ1_ENGINE_SHIFT) |
| 158 | | (op->u.aes.type << REQ1_AES_TYPE_SHIFT) |
| 159 | | (op->u.aes.mode << REQ1_AES_MODE_SHIFT) |
| 160 | | (op->u.aes.action << REQ1_AES_ACTION_SHIFT) |
Gary R Hook | 956ee21 | 2016-07-26 19:09:40 -0500 | [diff] [blame] | 161 | | (op->sb_key << REQ1_KEY_KSB_SHIFT); |
Gary R Hook | ea0375a | 2016-03-01 13:49:25 -0600 | [diff] [blame] | 162 | cr[1] = op->src.u.dma.length - 1; |
| 163 | cr[2] = ccp_addr_lo(&op->src.u.dma); |
Gary R Hook | 956ee21 | 2016-07-26 19:09:40 -0500 | [diff] [blame] | 164 | cr[3] = (op->sb_ctx << REQ4_KSB_SHIFT) |
Gary R Hook | ea0375a | 2016-03-01 13:49:25 -0600 | [diff] [blame] | 165 | | (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT) |
| 166 | | ccp_addr_hi(&op->src.u.dma); |
| 167 | cr[4] = ccp_addr_lo(&op->dst.u.dma); |
| 168 | cr[5] = (CCP_MEMTYPE_SYSTEM << REQ6_MEMTYPE_SHIFT) |
| 169 | | ccp_addr_hi(&op->dst.u.dma); |
| 170 | |
| 171 | if (op->u.aes.mode == CCP_AES_MODE_CFB) |
| 172 | cr[0] |= ((0x7f) << REQ1_AES_CFB_SIZE_SHIFT); |
| 173 | |
| 174 | if (op->eom) |
| 175 | cr[0] |= REQ1_EOM; |
| 176 | |
| 177 | if (op->init) |
| 178 | cr[0] |= REQ1_INIT; |
| 179 | |
| 180 | return ccp_do_cmd(op, cr, ARRAY_SIZE(cr)); |
| 181 | } |
| 182 | |
| 183 | static int ccp_perform_xts_aes(struct ccp_op *op) |
| 184 | { |
| 185 | u32 cr[6]; |
| 186 | |
| 187 | /* Fill out the register contents for REQ1 through REQ6 */ |
| 188 | cr[0] = (CCP_ENGINE_XTS_AES_128 << REQ1_ENGINE_SHIFT) |
| 189 | | (op->u.xts.action << REQ1_AES_ACTION_SHIFT) |
| 190 | | (op->u.xts.unit_size << REQ1_XTS_AES_SIZE_SHIFT) |
Gary R Hook | 956ee21 | 2016-07-26 19:09:40 -0500 | [diff] [blame] | 191 | | (op->sb_key << REQ1_KEY_KSB_SHIFT); |
Gary R Hook | ea0375a | 2016-03-01 13:49:25 -0600 | [diff] [blame] | 192 | cr[1] = op->src.u.dma.length - 1; |
| 193 | cr[2] = ccp_addr_lo(&op->src.u.dma); |
Gary R Hook | 956ee21 | 2016-07-26 19:09:40 -0500 | [diff] [blame] | 194 | cr[3] = (op->sb_ctx << REQ4_KSB_SHIFT) |
Gary R Hook | ea0375a | 2016-03-01 13:49:25 -0600 | [diff] [blame] | 195 | | (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT) |
| 196 | | ccp_addr_hi(&op->src.u.dma); |
| 197 | cr[4] = ccp_addr_lo(&op->dst.u.dma); |
| 198 | cr[5] = (CCP_MEMTYPE_SYSTEM << REQ6_MEMTYPE_SHIFT) |
| 199 | | ccp_addr_hi(&op->dst.u.dma); |
| 200 | |
| 201 | if (op->eom) |
| 202 | cr[0] |= REQ1_EOM; |
| 203 | |
| 204 | if (op->init) |
| 205 | cr[0] |= REQ1_INIT; |
| 206 | |
| 207 | return ccp_do_cmd(op, cr, ARRAY_SIZE(cr)); |
| 208 | } |
| 209 | |
| 210 | static int ccp_perform_sha(struct ccp_op *op) |
| 211 | { |
| 212 | u32 cr[6]; |
| 213 | |
| 214 | /* Fill out the register contents for REQ1 through REQ6 */ |
| 215 | cr[0] = (CCP_ENGINE_SHA << REQ1_ENGINE_SHIFT) |
| 216 | | (op->u.sha.type << REQ1_SHA_TYPE_SHIFT) |
| 217 | | REQ1_INIT; |
| 218 | cr[1] = op->src.u.dma.length - 1; |
| 219 | cr[2] = ccp_addr_lo(&op->src.u.dma); |
Gary R Hook | 956ee21 | 2016-07-26 19:09:40 -0500 | [diff] [blame] | 220 | cr[3] = (op->sb_ctx << REQ4_KSB_SHIFT) |
Gary R Hook | ea0375a | 2016-03-01 13:49:25 -0600 | [diff] [blame] | 221 | | (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT) |
| 222 | | ccp_addr_hi(&op->src.u.dma); |
| 223 | |
| 224 | if (op->eom) { |
| 225 | cr[0] |= REQ1_EOM; |
| 226 | cr[4] = lower_32_bits(op->u.sha.msg_bits); |
| 227 | cr[5] = upper_32_bits(op->u.sha.msg_bits); |
| 228 | } else { |
| 229 | cr[4] = 0; |
| 230 | cr[5] = 0; |
| 231 | } |
| 232 | |
| 233 | return ccp_do_cmd(op, cr, ARRAY_SIZE(cr)); |
| 234 | } |
| 235 | |
| 236 | static int ccp_perform_rsa(struct ccp_op *op) |
| 237 | { |
| 238 | u32 cr[6]; |
| 239 | |
| 240 | /* Fill out the register contents for REQ1 through REQ6 */ |
| 241 | cr[0] = (CCP_ENGINE_RSA << REQ1_ENGINE_SHIFT) |
| 242 | | (op->u.rsa.mod_size << REQ1_RSA_MOD_SIZE_SHIFT) |
Gary R Hook | 956ee21 | 2016-07-26 19:09:40 -0500 | [diff] [blame] | 243 | | (op->sb_key << REQ1_KEY_KSB_SHIFT) |
Gary R Hook | ea0375a | 2016-03-01 13:49:25 -0600 | [diff] [blame] | 244 | | REQ1_EOM; |
| 245 | cr[1] = op->u.rsa.input_len - 1; |
| 246 | cr[2] = ccp_addr_lo(&op->src.u.dma); |
Gary R Hook | 956ee21 | 2016-07-26 19:09:40 -0500 | [diff] [blame] | 247 | cr[3] = (op->sb_ctx << REQ4_KSB_SHIFT) |
Gary R Hook | ea0375a | 2016-03-01 13:49:25 -0600 | [diff] [blame] | 248 | | (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT) |
| 249 | | ccp_addr_hi(&op->src.u.dma); |
| 250 | cr[4] = ccp_addr_lo(&op->dst.u.dma); |
| 251 | cr[5] = (CCP_MEMTYPE_SYSTEM << REQ6_MEMTYPE_SHIFT) |
| 252 | | ccp_addr_hi(&op->dst.u.dma); |
| 253 | |
| 254 | return ccp_do_cmd(op, cr, ARRAY_SIZE(cr)); |
| 255 | } |
| 256 | |
| 257 | static int ccp_perform_passthru(struct ccp_op *op) |
| 258 | { |
| 259 | u32 cr[6]; |
| 260 | |
| 261 | /* Fill out the register contents for REQ1 through REQ6 */ |
| 262 | cr[0] = (CCP_ENGINE_PASSTHRU << REQ1_ENGINE_SHIFT) |
| 263 | | (op->u.passthru.bit_mod << REQ1_PT_BW_SHIFT) |
| 264 | | (op->u.passthru.byte_swap << REQ1_PT_BS_SHIFT); |
| 265 | |
| 266 | if (op->src.type == CCP_MEMTYPE_SYSTEM) |
| 267 | cr[1] = op->src.u.dma.length - 1; |
| 268 | else |
| 269 | cr[1] = op->dst.u.dma.length - 1; |
| 270 | |
| 271 | if (op->src.type == CCP_MEMTYPE_SYSTEM) { |
| 272 | cr[2] = ccp_addr_lo(&op->src.u.dma); |
| 273 | cr[3] = (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT) |
| 274 | | ccp_addr_hi(&op->src.u.dma); |
| 275 | |
| 276 | if (op->u.passthru.bit_mod != CCP_PASSTHRU_BITWISE_NOOP) |
Gary R Hook | 956ee21 | 2016-07-26 19:09:40 -0500 | [diff] [blame] | 277 | cr[3] |= (op->sb_key << REQ4_KSB_SHIFT); |
Gary R Hook | ea0375a | 2016-03-01 13:49:25 -0600 | [diff] [blame] | 278 | } else { |
Gary R Hook | 956ee21 | 2016-07-26 19:09:40 -0500 | [diff] [blame] | 279 | cr[2] = op->src.u.sb * CCP_SB_BYTES; |
| 280 | cr[3] = (CCP_MEMTYPE_SB << REQ4_MEMTYPE_SHIFT); |
Gary R Hook | ea0375a | 2016-03-01 13:49:25 -0600 | [diff] [blame] | 281 | } |
| 282 | |
| 283 | if (op->dst.type == CCP_MEMTYPE_SYSTEM) { |
| 284 | cr[4] = ccp_addr_lo(&op->dst.u.dma); |
| 285 | cr[5] = (CCP_MEMTYPE_SYSTEM << REQ6_MEMTYPE_SHIFT) |
| 286 | | ccp_addr_hi(&op->dst.u.dma); |
| 287 | } else { |
Gary R Hook | 956ee21 | 2016-07-26 19:09:40 -0500 | [diff] [blame] | 288 | cr[4] = op->dst.u.sb * CCP_SB_BYTES; |
| 289 | cr[5] = (CCP_MEMTYPE_SB << REQ6_MEMTYPE_SHIFT); |
Gary R Hook | ea0375a | 2016-03-01 13:49:25 -0600 | [diff] [blame] | 290 | } |
| 291 | |
| 292 | if (op->eom) |
| 293 | cr[0] |= REQ1_EOM; |
| 294 | |
| 295 | return ccp_do_cmd(op, cr, ARRAY_SIZE(cr)); |
| 296 | } |
| 297 | |
| 298 | static int ccp_perform_ecc(struct ccp_op *op) |
| 299 | { |
| 300 | u32 cr[6]; |
| 301 | |
| 302 | /* Fill out the register contents for REQ1 through REQ6 */ |
| 303 | cr[0] = REQ1_ECC_AFFINE_CONVERT |
| 304 | | (CCP_ENGINE_ECC << REQ1_ENGINE_SHIFT) |
| 305 | | (op->u.ecc.function << REQ1_ECC_FUNCTION_SHIFT) |
| 306 | | REQ1_EOM; |
| 307 | cr[1] = op->src.u.dma.length - 1; |
| 308 | cr[2] = ccp_addr_lo(&op->src.u.dma); |
| 309 | cr[3] = (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT) |
| 310 | | ccp_addr_hi(&op->src.u.dma); |
| 311 | cr[4] = ccp_addr_lo(&op->dst.u.dma); |
| 312 | cr[5] = (CCP_MEMTYPE_SYSTEM << REQ6_MEMTYPE_SHIFT) |
| 313 | | ccp_addr_hi(&op->dst.u.dma); |
| 314 | |
| 315 | return ccp_do_cmd(op, cr, ARRAY_SIZE(cr)); |
| 316 | } |
| 317 | |
Gary R Hook | a0a2324 | 2017-04-21 10:50:05 -0500 | [diff] [blame] | 318 | static void ccp_disable_queue_interrupts(struct ccp_device *ccp) |
| 319 | { |
| 320 | iowrite32(0x00, ccp->io_regs + IRQ_MASK_REG); |
| 321 | } |
| 322 | |
| 323 | static void ccp_enable_queue_interrupts(struct ccp_device *ccp) |
| 324 | { |
| 325 | iowrite32(ccp->qim, ccp->io_regs + IRQ_MASK_REG); |
| 326 | } |
| 327 | |
| 328 | static void ccp_irq_bh(unsigned long data) |
| 329 | { |
| 330 | struct ccp_device *ccp = (struct ccp_device *)data; |
| 331 | struct ccp_cmd_queue *cmd_q; |
| 332 | u32 q_int, status; |
| 333 | unsigned int i; |
| 334 | |
| 335 | status = ioread32(ccp->io_regs + IRQ_STATUS_REG); |
| 336 | |
| 337 | for (i = 0; i < ccp->cmd_q_count; i++) { |
| 338 | cmd_q = &ccp->cmd_q[i]; |
| 339 | |
| 340 | q_int = status & (cmd_q->int_ok | cmd_q->int_err); |
| 341 | if (q_int) { |
| 342 | cmd_q->int_status = status; |
| 343 | cmd_q->q_status = ioread32(cmd_q->reg_status); |
| 344 | cmd_q->q_int_status = ioread32(cmd_q->reg_int_status); |
| 345 | |
| 346 | /* On error, only save the first error value */ |
| 347 | if ((q_int & cmd_q->int_err) && !cmd_q->cmd_error) |
| 348 | cmd_q->cmd_error = CMD_Q_ERROR(cmd_q->q_status); |
| 349 | |
| 350 | cmd_q->int_rcvd = 1; |
| 351 | |
| 352 | /* Acknowledge the interrupt and wake the kthread */ |
| 353 | iowrite32(q_int, ccp->io_regs + IRQ_STATUS_REG); |
| 354 | wake_up_interruptible(&cmd_q->int_queue); |
| 355 | } |
| 356 | } |
| 357 | ccp_enable_queue_interrupts(ccp); |
| 358 | } |
| 359 | |
| 360 | static irqreturn_t ccp_irq_handler(int irq, void *data) |
| 361 | { |
| 362 | struct device *dev = data; |
| 363 | struct ccp_device *ccp = dev_get_drvdata(dev); |
| 364 | |
| 365 | ccp_disable_queue_interrupts(ccp); |
| 366 | if (ccp->use_tasklet) |
| 367 | tasklet_schedule(&ccp->irq_tasklet); |
| 368 | else |
| 369 | ccp_irq_bh((unsigned long)ccp); |
| 370 | |
| 371 | return IRQ_HANDLED; |
| 372 | } |
| 373 | |
Gary R Hook | ea0375a | 2016-03-01 13:49:25 -0600 | [diff] [blame] | 374 | static int ccp_init(struct ccp_device *ccp) |
| 375 | { |
| 376 | struct device *dev = ccp->dev; |
| 377 | struct ccp_cmd_queue *cmd_q; |
| 378 | struct dma_pool *dma_pool; |
| 379 | char dma_pool_name[MAX_DMAPOOL_NAME_LEN]; |
Gary R Hook | a0a2324 | 2017-04-21 10:50:05 -0500 | [diff] [blame] | 380 | unsigned int qmr, i; |
Gary R Hook | ea0375a | 2016-03-01 13:49:25 -0600 | [diff] [blame] | 381 | int ret; |
| 382 | |
| 383 | /* Find available queues */ |
Gary R Hook | a0a2324 | 2017-04-21 10:50:05 -0500 | [diff] [blame] | 384 | ccp->qim = 0; |
Gary R Hook | ea0375a | 2016-03-01 13:49:25 -0600 | [diff] [blame] | 385 | qmr = ioread32(ccp->io_regs + Q_MASK_REG); |
| 386 | for (i = 0; i < MAX_HW_QUEUES; i++) { |
| 387 | if (!(qmr & (1 << i))) |
| 388 | continue; |
| 389 | |
| 390 | /* Allocate a dma pool for this queue */ |
| 391 | snprintf(dma_pool_name, sizeof(dma_pool_name), "%s_q%d", |
| 392 | ccp->name, i); |
| 393 | dma_pool = dma_pool_create(dma_pool_name, dev, |
| 394 | CCP_DMAPOOL_MAX_SIZE, |
| 395 | CCP_DMAPOOL_ALIGN, 0); |
| 396 | if (!dma_pool) { |
| 397 | dev_err(dev, "unable to allocate dma pool\n"); |
| 398 | ret = -ENOMEM; |
| 399 | goto e_pool; |
| 400 | } |
| 401 | |
| 402 | cmd_q = &ccp->cmd_q[ccp->cmd_q_count]; |
| 403 | ccp->cmd_q_count++; |
| 404 | |
| 405 | cmd_q->ccp = ccp; |
| 406 | cmd_q->id = i; |
| 407 | cmd_q->dma_pool = dma_pool; |
| 408 | |
| 409 | /* Reserve 2 KSB regions for the queue */ |
Gary R Hook | 956ee21 | 2016-07-26 19:09:40 -0500 | [diff] [blame] | 410 | cmd_q->sb_key = KSB_START + ccp->sb_start++; |
| 411 | cmd_q->sb_ctx = KSB_START + ccp->sb_start++; |
| 412 | ccp->sb_count -= 2; |
Gary R Hook | ea0375a | 2016-03-01 13:49:25 -0600 | [diff] [blame] | 413 | |
| 414 | /* Preset some register values and masks that are queue |
| 415 | * number dependent |
| 416 | */ |
| 417 | cmd_q->reg_status = ccp->io_regs + CMD_Q_STATUS_BASE + |
| 418 | (CMD_Q_STATUS_INCR * i); |
| 419 | cmd_q->reg_int_status = ccp->io_regs + CMD_Q_INT_STATUS_BASE + |
| 420 | (CMD_Q_STATUS_INCR * i); |
| 421 | cmd_q->int_ok = 1 << (i * 2); |
| 422 | cmd_q->int_err = 1 << ((i * 2) + 1); |
| 423 | |
Gary R Hook | bb4e89b | 2016-07-26 19:10:13 -0500 | [diff] [blame] | 424 | cmd_q->free_slots = ccp_get_free_slots(cmd_q); |
Gary R Hook | ea0375a | 2016-03-01 13:49:25 -0600 | [diff] [blame] | 425 | |
| 426 | init_waitqueue_head(&cmd_q->int_queue); |
| 427 | |
| 428 | /* Build queue interrupt mask (two interrupts per queue) */ |
Gary R Hook | a0a2324 | 2017-04-21 10:50:05 -0500 | [diff] [blame] | 429 | ccp->qim |= cmd_q->int_ok | cmd_q->int_err; |
Gary R Hook | ea0375a | 2016-03-01 13:49:25 -0600 | [diff] [blame] | 430 | |
| 431 | #ifdef CONFIG_ARM64 |
| 432 | /* For arm64 set the recommended queue cache settings */ |
| 433 | iowrite32(ccp->axcache, ccp->io_regs + CMD_Q_CACHE_BASE + |
| 434 | (CMD_Q_CACHE_INC * i)); |
| 435 | #endif |
| 436 | |
| 437 | dev_dbg(dev, "queue #%u available\n", i); |
| 438 | } |
| 439 | if (ccp->cmd_q_count == 0) { |
| 440 | dev_notice(dev, "no command queues available\n"); |
| 441 | ret = -EIO; |
| 442 | goto e_pool; |
| 443 | } |
| 444 | dev_notice(dev, "%u command queues available\n", ccp->cmd_q_count); |
| 445 | |
| 446 | /* Disable and clear interrupts until ready */ |
Gary R Hook | a0a2324 | 2017-04-21 10:50:05 -0500 | [diff] [blame] | 447 | ccp_disable_queue_interrupts(ccp); |
Gary R Hook | ea0375a | 2016-03-01 13:49:25 -0600 | [diff] [blame] | 448 | for (i = 0; i < ccp->cmd_q_count; i++) { |
| 449 | cmd_q = &ccp->cmd_q[i]; |
| 450 | |
| 451 | ioread32(cmd_q->reg_int_status); |
| 452 | ioread32(cmd_q->reg_status); |
| 453 | } |
Gary R Hook | a0a2324 | 2017-04-21 10:50:05 -0500 | [diff] [blame] | 454 | iowrite32(ccp->qim, ccp->io_regs + IRQ_STATUS_REG); |
Gary R Hook | ea0375a | 2016-03-01 13:49:25 -0600 | [diff] [blame] | 455 | |
| 456 | /* Request an irq */ |
| 457 | ret = ccp->get_irq(ccp); |
| 458 | if (ret) { |
| 459 | dev_err(dev, "unable to allocate an IRQ\n"); |
| 460 | goto e_pool; |
| 461 | } |
| 462 | |
| 463 | /* Initialize the queues used to wait for KSB space and suspend */ |
Gary R Hook | 956ee21 | 2016-07-26 19:09:40 -0500 | [diff] [blame] | 464 | init_waitqueue_head(&ccp->sb_queue); |
Gary R Hook | ea0375a | 2016-03-01 13:49:25 -0600 | [diff] [blame] | 465 | init_waitqueue_head(&ccp->suspend_queue); |
| 466 | |
Gary R Hook | a0a2324 | 2017-04-21 10:50:05 -0500 | [diff] [blame] | 467 | /* Initialize the ISR tasklet? */ |
| 468 | if (ccp->use_tasklet) |
| 469 | tasklet_init(&ccp->irq_tasklet, ccp_irq_bh, |
| 470 | (unsigned long)ccp); |
| 471 | |
Gary R Hook | 4b394a2 | 2016-07-26 19:10:21 -0500 | [diff] [blame] | 472 | dev_dbg(dev, "Starting threads...\n"); |
Gary R Hook | ea0375a | 2016-03-01 13:49:25 -0600 | [diff] [blame] | 473 | /* Create a kthread for each queue */ |
| 474 | for (i = 0; i < ccp->cmd_q_count; i++) { |
| 475 | struct task_struct *kthread; |
| 476 | |
| 477 | cmd_q = &ccp->cmd_q[i]; |
| 478 | |
| 479 | kthread = kthread_create(ccp_cmd_queue_thread, cmd_q, |
| 480 | "%s-q%u", ccp->name, cmd_q->id); |
| 481 | if (IS_ERR(kthread)) { |
| 482 | dev_err(dev, "error creating queue thread (%ld)\n", |
| 483 | PTR_ERR(kthread)); |
| 484 | ret = PTR_ERR(kthread); |
| 485 | goto e_kthread; |
| 486 | } |
| 487 | |
| 488 | cmd_q->kthread = kthread; |
| 489 | wake_up_process(kthread); |
| 490 | } |
| 491 | |
Gary R Hook | 4b394a2 | 2016-07-26 19:10:21 -0500 | [diff] [blame] | 492 | dev_dbg(dev, "Enabling interrupts...\n"); |
| 493 | /* Enable interrupts */ |
Gary R Hook | a0a2324 | 2017-04-21 10:50:05 -0500 | [diff] [blame] | 494 | ccp_enable_queue_interrupts(ccp); |
Gary R Hook | 4b394a2 | 2016-07-26 19:10:21 -0500 | [diff] [blame] | 495 | |
| 496 | dev_dbg(dev, "Registering device...\n"); |
| 497 | ccp_add_device(ccp); |
| 498 | |
Gary R Hook | 084935b | 2016-07-26 19:10:31 -0500 | [diff] [blame] | 499 | ret = ccp_register_rng(ccp); |
| 500 | if (ret) |
Gary R Hook | ea0375a | 2016-03-01 13:49:25 -0600 | [diff] [blame] | 501 | goto e_kthread; |
Gary R Hook | ea0375a | 2016-03-01 13:49:25 -0600 | [diff] [blame] | 502 | |
Gary R Hook | 58ea8ab | 2016-04-18 09:21:44 -0500 | [diff] [blame] | 503 | /* Register the DMA engine support */ |
| 504 | ret = ccp_dmaengine_register(ccp); |
| 505 | if (ret) |
| 506 | goto e_hwrng; |
| 507 | |
Gary R Hook | ea0375a | 2016-03-01 13:49:25 -0600 | [diff] [blame] | 508 | return 0; |
| 509 | |
Gary R Hook | 58ea8ab | 2016-04-18 09:21:44 -0500 | [diff] [blame] | 510 | e_hwrng: |
Gary R Hook | 084935b | 2016-07-26 19:10:31 -0500 | [diff] [blame] | 511 | ccp_unregister_rng(ccp); |
Gary R Hook | 58ea8ab | 2016-04-18 09:21:44 -0500 | [diff] [blame] | 512 | |
Gary R Hook | ea0375a | 2016-03-01 13:49:25 -0600 | [diff] [blame] | 513 | e_kthread: |
| 514 | for (i = 0; i < ccp->cmd_q_count; i++) |
| 515 | if (ccp->cmd_q[i].kthread) |
| 516 | kthread_stop(ccp->cmd_q[i].kthread); |
| 517 | |
| 518 | ccp->free_irq(ccp); |
| 519 | |
| 520 | e_pool: |
| 521 | for (i = 0; i < ccp->cmd_q_count; i++) |
| 522 | dma_pool_destroy(ccp->cmd_q[i].dma_pool); |
| 523 | |
| 524 | return ret; |
| 525 | } |
| 526 | |
| 527 | static void ccp_destroy(struct ccp_device *ccp) |
| 528 | { |
| 529 | struct ccp_cmd_queue *cmd_q; |
| 530 | struct ccp_cmd *cmd; |
Gary R Hook | a0a2324 | 2017-04-21 10:50:05 -0500 | [diff] [blame] | 531 | unsigned int i; |
Gary R Hook | ea0375a | 2016-03-01 13:49:25 -0600 | [diff] [blame] | 532 | |
Gary R Hook | 4b394a2 | 2016-07-26 19:10:21 -0500 | [diff] [blame] | 533 | /* Unregister the DMA engine */ |
| 534 | ccp_dmaengine_unregister(ccp); |
| 535 | |
| 536 | /* Unregister the RNG */ |
Gary R Hook | 084935b | 2016-07-26 19:10:31 -0500 | [diff] [blame] | 537 | ccp_unregister_rng(ccp); |
Gary R Hook | 4b394a2 | 2016-07-26 19:10:21 -0500 | [diff] [blame] | 538 | |
| 539 | /* Remove this device from the list of available units */ |
Gary R Hook | ea0375a | 2016-03-01 13:49:25 -0600 | [diff] [blame] | 540 | ccp_del_device(ccp); |
| 541 | |
Gary R Hook | ea0375a | 2016-03-01 13:49:25 -0600 | [diff] [blame] | 542 | /* Disable and clear interrupts */ |
Gary R Hook | a0a2324 | 2017-04-21 10:50:05 -0500 | [diff] [blame] | 543 | ccp_disable_queue_interrupts(ccp); |
Gary R Hook | ea0375a | 2016-03-01 13:49:25 -0600 | [diff] [blame] | 544 | for (i = 0; i < ccp->cmd_q_count; i++) { |
| 545 | cmd_q = &ccp->cmd_q[i]; |
| 546 | |
| 547 | ioread32(cmd_q->reg_int_status); |
| 548 | ioread32(cmd_q->reg_status); |
| 549 | } |
Gary R Hook | a0a2324 | 2017-04-21 10:50:05 -0500 | [diff] [blame] | 550 | iowrite32(ccp->qim, ccp->io_regs + IRQ_STATUS_REG); |
Gary R Hook | ea0375a | 2016-03-01 13:49:25 -0600 | [diff] [blame] | 551 | |
Gary R Hook | 8256e68 | 2016-07-26 19:10:02 -0500 | [diff] [blame] | 552 | /* Stop the queue kthreads */ |
| 553 | for (i = 0; i < ccp->cmd_q_count; i++) |
| 554 | if (ccp->cmd_q[i].kthread) |
| 555 | kthread_stop(ccp->cmd_q[i].kthread); |
| 556 | |
Gary R Hook | ea0375a | 2016-03-01 13:49:25 -0600 | [diff] [blame] | 557 | ccp->free_irq(ccp); |
| 558 | |
| 559 | for (i = 0; i < ccp->cmd_q_count; i++) |
| 560 | dma_pool_destroy(ccp->cmd_q[i].dma_pool); |
| 561 | |
| 562 | /* Flush the cmd and backlog queue */ |
| 563 | while (!list_empty(&ccp->cmd)) { |
| 564 | /* Invoke the callback directly with an error code */ |
| 565 | cmd = list_first_entry(&ccp->cmd, struct ccp_cmd, entry); |
| 566 | list_del(&cmd->entry); |
| 567 | cmd->callback(cmd->data, -ENODEV); |
| 568 | } |
| 569 | while (!list_empty(&ccp->backlog)) { |
| 570 | /* Invoke the callback directly with an error code */ |
| 571 | cmd = list_first_entry(&ccp->backlog, struct ccp_cmd, entry); |
| 572 | list_del(&cmd->entry); |
| 573 | cmd->callback(cmd->data, -ENODEV); |
| 574 | } |
| 575 | } |
| 576 | |
Julia Lawall | bc197b2a | 2016-05-01 13:52:55 +0200 | [diff] [blame] | 577 | static const struct ccp_actions ccp3_actions = { |
Gary R Hook | a43eb98 | 2016-07-26 19:09:31 -0500 | [diff] [blame] | 578 | .aes = ccp_perform_aes, |
| 579 | .xts_aes = ccp_perform_xts_aes, |
| 580 | .sha = ccp_perform_sha, |
| 581 | .rsa = ccp_perform_rsa, |
| 582 | .passthru = ccp_perform_passthru, |
| 583 | .ecc = ccp_perform_ecc, |
Gary R Hook | 58a690b | 2016-07-26 19:09:50 -0500 | [diff] [blame] | 584 | .sballoc = ccp_alloc_ksb, |
| 585 | .sbfree = ccp_free_ksb, |
Gary R Hook | ea0375a | 2016-03-01 13:49:25 -0600 | [diff] [blame] | 586 | .init = ccp_init, |
| 587 | .destroy = ccp_destroy, |
Gary R Hook | bb4e89b | 2016-07-26 19:10:13 -0500 | [diff] [blame] | 588 | .get_free_slots = ccp_get_free_slots, |
Gary R Hook | ea0375a | 2016-03-01 13:49:25 -0600 | [diff] [blame] | 589 | .irqhandler = ccp_irq_handler, |
| 590 | }; |
| 591 | |
Gary R Hook | 9ddb9dc | 2016-09-28 11:53:47 -0500 | [diff] [blame] | 592 | const struct ccp_vdata ccpv3 = { |
Gary R Hook | ea0375a | 2016-03-01 13:49:25 -0600 | [diff] [blame] | 593 | .version = CCP_VERSION(3, 0), |
Gary R Hook | 4b394a2 | 2016-07-26 19:10:21 -0500 | [diff] [blame] | 594 | .setup = NULL, |
Gary R Hook | ea0375a | 2016-03-01 13:49:25 -0600 | [diff] [blame] | 595 | .perform = &ccp3_actions, |
Gary R Hook | fba8855 | 2016-07-26 19:09:20 -0500 | [diff] [blame] | 596 | .bar = 2, |
| 597 | .offset = 0x20000, |
Gary R Hook | ea0375a | 2016-03-01 13:49:25 -0600 | [diff] [blame] | 598 | }; |