Gary R Hook | 4b394a2 | 2016-07-26 19:10:21 -0500 | [diff] [blame] | 1 | /* |
| 2 | * AMD Cryptographic Coprocessor (CCP) driver |
| 3 | * |
| 4 | * Copyright (C) 2016 Advanced Micro Devices, Inc. |
| 5 | * |
| 6 | * Author: Gary R Hook <gary.hook@amd.com> |
| 7 | * |
| 8 | * This program is free software; you can redistribute it and/or modify |
| 9 | * it under the terms of the GNU General Public License version 2 as |
| 10 | * published by the Free Software Foundation. |
| 11 | */ |
| 12 | |
| 13 | #include <linux/module.h> |
| 14 | #include <linux/kernel.h> |
| 15 | #include <linux/pci.h> |
| 16 | #include <linux/kthread.h> |
| 17 | #include <linux/dma-mapping.h> |
| 18 | #include <linux/interrupt.h> |
| 19 | #include <linux/compiler.h> |
| 20 | #include <linux/ccp.h> |
| 21 | |
| 22 | #include "ccp-dev.h" |
| 23 | |
| 24 | static u32 ccp_lsb_alloc(struct ccp_cmd_queue *cmd_q, unsigned int count) |
| 25 | { |
| 26 | struct ccp_device *ccp; |
| 27 | int start; |
| 28 | |
| 29 | /* First look at the map for the queue */ |
| 30 | if (cmd_q->lsb >= 0) { |
| 31 | start = (u32)bitmap_find_next_zero_area(cmd_q->lsbmap, |
| 32 | LSB_SIZE, |
| 33 | 0, count, 0); |
| 34 | if (start < LSB_SIZE) { |
| 35 | bitmap_set(cmd_q->lsbmap, start, count); |
| 36 | return start + cmd_q->lsb * LSB_SIZE; |
| 37 | } |
| 38 | } |
| 39 | |
| 40 | /* No joy; try to get an entry from the shared blocks */ |
| 41 | ccp = cmd_q->ccp; |
| 42 | for (;;) { |
| 43 | mutex_lock(&ccp->sb_mutex); |
| 44 | |
| 45 | start = (u32)bitmap_find_next_zero_area(ccp->lsbmap, |
| 46 | MAX_LSB_CNT * LSB_SIZE, |
| 47 | 0, |
| 48 | count, 0); |
| 49 | if (start <= MAX_LSB_CNT * LSB_SIZE) { |
| 50 | bitmap_set(ccp->lsbmap, start, count); |
| 51 | |
| 52 | mutex_unlock(&ccp->sb_mutex); |
| 53 | return start * LSB_ITEM_SIZE; |
| 54 | } |
| 55 | |
| 56 | ccp->sb_avail = 0; |
| 57 | |
| 58 | mutex_unlock(&ccp->sb_mutex); |
| 59 | |
| 60 | /* Wait for KSB entries to become available */ |
| 61 | if (wait_event_interruptible(ccp->sb_queue, ccp->sb_avail)) |
| 62 | return 0; |
| 63 | } |
| 64 | } |
| 65 | |
| 66 | static void ccp_lsb_free(struct ccp_cmd_queue *cmd_q, unsigned int start, |
| 67 | unsigned int count) |
| 68 | { |
| 69 | int lsbno = start / LSB_SIZE; |
| 70 | |
| 71 | if (!start) |
| 72 | return; |
| 73 | |
| 74 | if (cmd_q->lsb == lsbno) { |
| 75 | /* An entry from the private LSB */ |
| 76 | bitmap_clear(cmd_q->lsbmap, start % LSB_SIZE, count); |
| 77 | } else { |
| 78 | /* From the shared LSBs */ |
| 79 | struct ccp_device *ccp = cmd_q->ccp; |
| 80 | |
| 81 | mutex_lock(&ccp->sb_mutex); |
| 82 | bitmap_clear(ccp->lsbmap, start, count); |
| 83 | ccp->sb_avail = 1; |
| 84 | mutex_unlock(&ccp->sb_mutex); |
| 85 | wake_up_interruptible_all(&ccp->sb_queue); |
| 86 | } |
| 87 | } |
| 88 | |
| 89 | /* CCP version 5: Union to define the function field (cmd_reg1/dword0) */ |
| 90 | union ccp_function { |
| 91 | struct { |
| 92 | u16 size:7; |
| 93 | u16 encrypt:1; |
| 94 | u16 mode:5; |
| 95 | u16 type:2; |
| 96 | } aes; |
| 97 | struct { |
| 98 | u16 size:7; |
| 99 | u16 encrypt:1; |
| 100 | u16 rsvd:5; |
| 101 | u16 type:2; |
| 102 | } aes_xts; |
| 103 | struct { |
| 104 | u16 rsvd1:10; |
| 105 | u16 type:4; |
| 106 | u16 rsvd2:1; |
| 107 | } sha; |
| 108 | struct { |
| 109 | u16 mode:3; |
| 110 | u16 size:12; |
| 111 | } rsa; |
| 112 | struct { |
| 113 | u16 byteswap:2; |
| 114 | u16 bitwise:3; |
| 115 | u16 reflect:2; |
| 116 | u16 rsvd:8; |
| 117 | } pt; |
| 118 | struct { |
| 119 | u16 rsvd:13; |
| 120 | } zlib; |
| 121 | struct { |
| 122 | u16 size:10; |
| 123 | u16 type:2; |
| 124 | u16 mode:3; |
| 125 | } ecc; |
| 126 | u16 raw; |
| 127 | }; |
| 128 | |
| 129 | #define CCP_AES_SIZE(p) ((p)->aes.size) |
| 130 | #define CCP_AES_ENCRYPT(p) ((p)->aes.encrypt) |
| 131 | #define CCP_AES_MODE(p) ((p)->aes.mode) |
| 132 | #define CCP_AES_TYPE(p) ((p)->aes.type) |
| 133 | #define CCP_XTS_SIZE(p) ((p)->aes_xts.size) |
Gary R Hook | dcb3a4b | 2017-07-25 14:12:11 -0500 | [diff] [blame] | 134 | #define CCP_XTS_TYPE(p) ((p)->aes_xts.type) |
Gary R Hook | 4b394a2 | 2016-07-26 19:10:21 -0500 | [diff] [blame] | 135 | #define CCP_XTS_ENCRYPT(p) ((p)->aes_xts.encrypt) |
| 136 | #define CCP_SHA_TYPE(p) ((p)->sha.type) |
| 137 | #define CCP_RSA_SIZE(p) ((p)->rsa.size) |
| 138 | #define CCP_PT_BYTESWAP(p) ((p)->pt.byteswap) |
| 139 | #define CCP_PT_BITWISE(p) ((p)->pt.bitwise) |
| 140 | #define CCP_ECC_MODE(p) ((p)->ecc.mode) |
| 141 | #define CCP_ECC_AFFINE(p) ((p)->ecc.one) |
| 142 | |
| 143 | /* Word 0 */ |
| 144 | #define CCP5_CMD_DW0(p) ((p)->dw0) |
| 145 | #define CCP5_CMD_SOC(p) (CCP5_CMD_DW0(p).soc) |
| 146 | #define CCP5_CMD_IOC(p) (CCP5_CMD_DW0(p).ioc) |
| 147 | #define CCP5_CMD_INIT(p) (CCP5_CMD_DW0(p).init) |
| 148 | #define CCP5_CMD_EOM(p) (CCP5_CMD_DW0(p).eom) |
| 149 | #define CCP5_CMD_FUNCTION(p) (CCP5_CMD_DW0(p).function) |
| 150 | #define CCP5_CMD_ENGINE(p) (CCP5_CMD_DW0(p).engine) |
| 151 | #define CCP5_CMD_PROT(p) (CCP5_CMD_DW0(p).prot) |
| 152 | |
| 153 | /* Word 1 */ |
| 154 | #define CCP5_CMD_DW1(p) ((p)->length) |
| 155 | #define CCP5_CMD_LEN(p) (CCP5_CMD_DW1(p)) |
| 156 | |
| 157 | /* Word 2 */ |
| 158 | #define CCP5_CMD_DW2(p) ((p)->src_lo) |
| 159 | #define CCP5_CMD_SRC_LO(p) (CCP5_CMD_DW2(p)) |
| 160 | |
| 161 | /* Word 3 */ |
| 162 | #define CCP5_CMD_DW3(p) ((p)->dw3) |
| 163 | #define CCP5_CMD_SRC_MEM(p) ((p)->dw3.src_mem) |
| 164 | #define CCP5_CMD_SRC_HI(p) ((p)->dw3.src_hi) |
| 165 | #define CCP5_CMD_LSB_ID(p) ((p)->dw3.lsb_cxt_id) |
| 166 | #define CCP5_CMD_FIX_SRC(p) ((p)->dw3.fixed) |
| 167 | |
| 168 | /* Words 4/5 */ |
| 169 | #define CCP5_CMD_DW4(p) ((p)->dw4) |
| 170 | #define CCP5_CMD_DST_LO(p) (CCP5_CMD_DW4(p).dst_lo) |
| 171 | #define CCP5_CMD_DW5(p) ((p)->dw5.fields.dst_hi) |
| 172 | #define CCP5_CMD_DST_HI(p) (CCP5_CMD_DW5(p)) |
| 173 | #define CCP5_CMD_DST_MEM(p) ((p)->dw5.fields.dst_mem) |
| 174 | #define CCP5_CMD_FIX_DST(p) ((p)->dw5.fields.fixed) |
| 175 | #define CCP5_CMD_SHA_LO(p) ((p)->dw4.sha_len_lo) |
| 176 | #define CCP5_CMD_SHA_HI(p) ((p)->dw5.sha_len_hi) |
| 177 | |
| 178 | /* Word 6/7 */ |
| 179 | #define CCP5_CMD_DW6(p) ((p)->key_lo) |
| 180 | #define CCP5_CMD_KEY_LO(p) (CCP5_CMD_DW6(p)) |
| 181 | #define CCP5_CMD_DW7(p) ((p)->dw7) |
| 182 | #define CCP5_CMD_KEY_HI(p) ((p)->dw7.key_hi) |
| 183 | #define CCP5_CMD_KEY_MEM(p) ((p)->dw7.key_mem) |
| 184 | |
| 185 | static inline u32 low_address(unsigned long addr) |
| 186 | { |
| 187 | return (u64)addr & 0x0ffffffff; |
| 188 | } |
| 189 | |
| 190 | static inline u32 high_address(unsigned long addr) |
| 191 | { |
| 192 | return ((u64)addr >> 32) & 0x00000ffff; |
| 193 | } |
| 194 | |
| 195 | static unsigned int ccp5_get_free_slots(struct ccp_cmd_queue *cmd_q) |
| 196 | { |
| 197 | unsigned int head_idx, n; |
| 198 | u32 head_lo, queue_start; |
| 199 | |
| 200 | queue_start = low_address(cmd_q->qdma_tail); |
| 201 | head_lo = ioread32(cmd_q->reg_head_lo); |
| 202 | head_idx = (head_lo - queue_start) / sizeof(struct ccp5_desc); |
| 203 | |
| 204 | n = head_idx + COMMANDS_PER_QUEUE - cmd_q->qidx - 1; |
| 205 | |
| 206 | return n % COMMANDS_PER_QUEUE; /* Always one unused spot */ |
| 207 | } |
| 208 | |
| 209 | static int ccp5_do_cmd(struct ccp5_desc *desc, |
| 210 | struct ccp_cmd_queue *cmd_q) |
| 211 | { |
| 212 | u32 *mP; |
| 213 | __le32 *dP; |
| 214 | u32 tail; |
| 215 | int i; |
| 216 | int ret = 0; |
| 217 | |
| 218 | if (CCP5_CMD_SOC(desc)) { |
| 219 | CCP5_CMD_IOC(desc) = 1; |
| 220 | CCP5_CMD_SOC(desc) = 0; |
| 221 | } |
| 222 | mutex_lock(&cmd_q->q_mutex); |
| 223 | |
| 224 | mP = (u32 *) &cmd_q->qbase[cmd_q->qidx]; |
| 225 | dP = (__le32 *) desc; |
| 226 | for (i = 0; i < 8; i++) |
| 227 | mP[i] = cpu_to_le32(dP[i]); /* handle endianness */ |
| 228 | |
| 229 | cmd_q->qidx = (cmd_q->qidx + 1) % COMMANDS_PER_QUEUE; |
| 230 | |
| 231 | /* The data used by this command must be flushed to memory */ |
| 232 | wmb(); |
| 233 | |
| 234 | /* Write the new tail address back to the queue register */ |
| 235 | tail = low_address(cmd_q->qdma_tail + cmd_q->qidx * Q_DESC_SIZE); |
| 236 | iowrite32(tail, cmd_q->reg_tail_lo); |
| 237 | |
| 238 | /* Turn the queue back on using our cached control register */ |
| 239 | iowrite32(cmd_q->qcontrol | CMD5_Q_RUN, cmd_q->reg_control); |
| 240 | mutex_unlock(&cmd_q->q_mutex); |
| 241 | |
| 242 | if (CCP5_CMD_IOC(desc)) { |
| 243 | /* Wait for the job to complete */ |
| 244 | ret = wait_event_interruptible(cmd_q->int_queue, |
| 245 | cmd_q->int_rcvd); |
| 246 | if (ret || cmd_q->cmd_error) { |
Gary R Hook | 81422ba | 2016-09-28 11:53:56 -0500 | [diff] [blame] | 247 | if (cmd_q->cmd_error) |
| 248 | ccp_log_error(cmd_q->ccp, |
| 249 | cmd_q->cmd_error); |
Gary R Hook | 4b394a2 | 2016-07-26 19:10:21 -0500 | [diff] [blame] | 250 | /* A version 5 device doesn't use Job IDs... */ |
| 251 | if (!ret) |
| 252 | ret = -EIO; |
| 253 | } |
| 254 | cmd_q->int_rcvd = 0; |
| 255 | } |
| 256 | |
| 257 | return 0; |
| 258 | } |
| 259 | |
| 260 | static int ccp5_perform_aes(struct ccp_op *op) |
| 261 | { |
| 262 | struct ccp5_desc desc; |
| 263 | union ccp_function function; |
| 264 | u32 key_addr = op->sb_key * LSB_ITEM_SIZE; |
| 265 | |
| 266 | /* Zero out all the fields of the command desc */ |
| 267 | memset(&desc, 0, Q_DESC_SIZE); |
| 268 | |
| 269 | CCP5_CMD_ENGINE(&desc) = CCP_ENGINE_AES; |
| 270 | |
| 271 | CCP5_CMD_SOC(&desc) = op->soc; |
| 272 | CCP5_CMD_IOC(&desc) = 1; |
| 273 | CCP5_CMD_INIT(&desc) = op->init; |
| 274 | CCP5_CMD_EOM(&desc) = op->eom; |
| 275 | CCP5_CMD_PROT(&desc) = 0; |
| 276 | |
| 277 | function.raw = 0; |
| 278 | CCP_AES_ENCRYPT(&function) = op->u.aes.action; |
| 279 | CCP_AES_MODE(&function) = op->u.aes.mode; |
| 280 | CCP_AES_TYPE(&function) = op->u.aes.type; |
| 281 | if (op->u.aes.mode == CCP_AES_MODE_CFB) |
| 282 | CCP_AES_SIZE(&function) = 0x7f; |
| 283 | |
| 284 | CCP5_CMD_FUNCTION(&desc) = function.raw; |
| 285 | |
| 286 | CCP5_CMD_LEN(&desc) = op->src.u.dma.length; |
| 287 | |
| 288 | CCP5_CMD_SRC_LO(&desc) = ccp_addr_lo(&op->src.u.dma); |
| 289 | CCP5_CMD_SRC_HI(&desc) = ccp_addr_hi(&op->src.u.dma); |
| 290 | CCP5_CMD_SRC_MEM(&desc) = CCP_MEMTYPE_SYSTEM; |
| 291 | |
| 292 | CCP5_CMD_DST_LO(&desc) = ccp_addr_lo(&op->dst.u.dma); |
| 293 | CCP5_CMD_DST_HI(&desc) = ccp_addr_hi(&op->dst.u.dma); |
| 294 | CCP5_CMD_DST_MEM(&desc) = CCP_MEMTYPE_SYSTEM; |
| 295 | |
| 296 | CCP5_CMD_KEY_LO(&desc) = lower_32_bits(key_addr); |
| 297 | CCP5_CMD_KEY_HI(&desc) = 0; |
| 298 | CCP5_CMD_KEY_MEM(&desc) = CCP_MEMTYPE_SB; |
| 299 | CCP5_CMD_LSB_ID(&desc) = op->sb_ctx; |
| 300 | |
| 301 | return ccp5_do_cmd(&desc, op->cmd_q); |
| 302 | } |
| 303 | |
| 304 | static int ccp5_perform_xts_aes(struct ccp_op *op) |
| 305 | { |
| 306 | struct ccp5_desc desc; |
| 307 | union ccp_function function; |
| 308 | u32 key_addr = op->sb_key * LSB_ITEM_SIZE; |
| 309 | |
| 310 | /* Zero out all the fields of the command desc */ |
| 311 | memset(&desc, 0, Q_DESC_SIZE); |
| 312 | |
| 313 | CCP5_CMD_ENGINE(&desc) = CCP_ENGINE_XTS_AES_128; |
| 314 | |
| 315 | CCP5_CMD_SOC(&desc) = op->soc; |
| 316 | CCP5_CMD_IOC(&desc) = 1; |
| 317 | CCP5_CMD_INIT(&desc) = op->init; |
| 318 | CCP5_CMD_EOM(&desc) = op->eom; |
| 319 | CCP5_CMD_PROT(&desc) = 0; |
| 320 | |
| 321 | function.raw = 0; |
Gary R Hook | dcb3a4b | 2017-07-25 14:12:11 -0500 | [diff] [blame] | 322 | CCP_XTS_TYPE(&function) = op->u.xts.type; |
Gary R Hook | 4b394a2 | 2016-07-26 19:10:21 -0500 | [diff] [blame] | 323 | CCP_XTS_ENCRYPT(&function) = op->u.xts.action; |
| 324 | CCP_XTS_SIZE(&function) = op->u.xts.unit_size; |
| 325 | CCP5_CMD_FUNCTION(&desc) = function.raw; |
| 326 | |
| 327 | CCP5_CMD_LEN(&desc) = op->src.u.dma.length; |
| 328 | |
| 329 | CCP5_CMD_SRC_LO(&desc) = ccp_addr_lo(&op->src.u.dma); |
| 330 | CCP5_CMD_SRC_HI(&desc) = ccp_addr_hi(&op->src.u.dma); |
| 331 | CCP5_CMD_SRC_MEM(&desc) = CCP_MEMTYPE_SYSTEM; |
| 332 | |
| 333 | CCP5_CMD_DST_LO(&desc) = ccp_addr_lo(&op->dst.u.dma); |
| 334 | CCP5_CMD_DST_HI(&desc) = ccp_addr_hi(&op->dst.u.dma); |
| 335 | CCP5_CMD_DST_MEM(&desc) = CCP_MEMTYPE_SYSTEM; |
| 336 | |
| 337 | CCP5_CMD_KEY_LO(&desc) = lower_32_bits(key_addr); |
| 338 | CCP5_CMD_KEY_HI(&desc) = 0; |
| 339 | CCP5_CMD_KEY_MEM(&desc) = CCP_MEMTYPE_SB; |
| 340 | CCP5_CMD_LSB_ID(&desc) = op->sb_ctx; |
| 341 | |
| 342 | return ccp5_do_cmd(&desc, op->cmd_q); |
| 343 | } |
| 344 | |
| 345 | static int ccp5_perform_sha(struct ccp_op *op) |
| 346 | { |
| 347 | struct ccp5_desc desc; |
| 348 | union ccp_function function; |
| 349 | |
| 350 | /* Zero out all the fields of the command desc */ |
| 351 | memset(&desc, 0, Q_DESC_SIZE); |
| 352 | |
| 353 | CCP5_CMD_ENGINE(&desc) = CCP_ENGINE_SHA; |
| 354 | |
| 355 | CCP5_CMD_SOC(&desc) = op->soc; |
| 356 | CCP5_CMD_IOC(&desc) = 1; |
| 357 | CCP5_CMD_INIT(&desc) = 1; |
| 358 | CCP5_CMD_EOM(&desc) = op->eom; |
| 359 | CCP5_CMD_PROT(&desc) = 0; |
| 360 | |
| 361 | function.raw = 0; |
| 362 | CCP_SHA_TYPE(&function) = op->u.sha.type; |
| 363 | CCP5_CMD_FUNCTION(&desc) = function.raw; |
| 364 | |
| 365 | CCP5_CMD_LEN(&desc) = op->src.u.dma.length; |
| 366 | |
| 367 | CCP5_CMD_SRC_LO(&desc) = ccp_addr_lo(&op->src.u.dma); |
| 368 | CCP5_CMD_SRC_HI(&desc) = ccp_addr_hi(&op->src.u.dma); |
| 369 | CCP5_CMD_SRC_MEM(&desc) = CCP_MEMTYPE_SYSTEM; |
| 370 | |
| 371 | CCP5_CMD_LSB_ID(&desc) = op->sb_ctx; |
| 372 | |
| 373 | if (op->eom) { |
| 374 | CCP5_CMD_SHA_LO(&desc) = lower_32_bits(op->u.sha.msg_bits); |
| 375 | CCP5_CMD_SHA_HI(&desc) = upper_32_bits(op->u.sha.msg_bits); |
| 376 | } else { |
| 377 | CCP5_CMD_SHA_LO(&desc) = 0; |
| 378 | CCP5_CMD_SHA_HI(&desc) = 0; |
| 379 | } |
| 380 | |
| 381 | return ccp5_do_cmd(&desc, op->cmd_q); |
| 382 | } |
| 383 | |
| 384 | static int ccp5_perform_rsa(struct ccp_op *op) |
| 385 | { |
| 386 | struct ccp5_desc desc; |
| 387 | union ccp_function function; |
| 388 | |
| 389 | /* Zero out all the fields of the command desc */ |
| 390 | memset(&desc, 0, Q_DESC_SIZE); |
| 391 | |
| 392 | CCP5_CMD_ENGINE(&desc) = CCP_ENGINE_RSA; |
| 393 | |
| 394 | CCP5_CMD_SOC(&desc) = op->soc; |
| 395 | CCP5_CMD_IOC(&desc) = 1; |
| 396 | CCP5_CMD_INIT(&desc) = 0; |
| 397 | CCP5_CMD_EOM(&desc) = 1; |
| 398 | CCP5_CMD_PROT(&desc) = 0; |
| 399 | |
| 400 | function.raw = 0; |
| 401 | CCP_RSA_SIZE(&function) = op->u.rsa.mod_size; |
| 402 | CCP5_CMD_FUNCTION(&desc) = function.raw; |
| 403 | |
| 404 | CCP5_CMD_LEN(&desc) = op->u.rsa.input_len; |
| 405 | |
| 406 | /* Source is from external memory */ |
| 407 | CCP5_CMD_SRC_LO(&desc) = ccp_addr_lo(&op->src.u.dma); |
| 408 | CCP5_CMD_SRC_HI(&desc) = ccp_addr_hi(&op->src.u.dma); |
| 409 | CCP5_CMD_SRC_MEM(&desc) = CCP_MEMTYPE_SYSTEM; |
| 410 | |
| 411 | /* Destination is in external memory */ |
| 412 | CCP5_CMD_DST_LO(&desc) = ccp_addr_lo(&op->dst.u.dma); |
| 413 | CCP5_CMD_DST_HI(&desc) = ccp_addr_hi(&op->dst.u.dma); |
| 414 | CCP5_CMD_DST_MEM(&desc) = CCP_MEMTYPE_SYSTEM; |
| 415 | |
| 416 | /* Key (Exponent) is in external memory */ |
| 417 | CCP5_CMD_KEY_LO(&desc) = ccp_addr_lo(&op->exp.u.dma); |
| 418 | CCP5_CMD_KEY_HI(&desc) = ccp_addr_hi(&op->exp.u.dma); |
| 419 | CCP5_CMD_KEY_MEM(&desc) = CCP_MEMTYPE_SYSTEM; |
| 420 | |
| 421 | return ccp5_do_cmd(&desc, op->cmd_q); |
| 422 | } |
| 423 | |
| 424 | static int ccp5_perform_passthru(struct ccp_op *op) |
| 425 | { |
| 426 | struct ccp5_desc desc; |
| 427 | union ccp_function function; |
| 428 | struct ccp_dma_info *saddr = &op->src.u.dma; |
| 429 | struct ccp_dma_info *daddr = &op->dst.u.dma; |
| 430 | |
| 431 | memset(&desc, 0, Q_DESC_SIZE); |
| 432 | |
| 433 | CCP5_CMD_ENGINE(&desc) = CCP_ENGINE_PASSTHRU; |
| 434 | |
| 435 | CCP5_CMD_SOC(&desc) = 0; |
| 436 | CCP5_CMD_IOC(&desc) = 1; |
| 437 | CCP5_CMD_INIT(&desc) = 0; |
| 438 | CCP5_CMD_EOM(&desc) = op->eom; |
| 439 | CCP5_CMD_PROT(&desc) = 0; |
| 440 | |
| 441 | function.raw = 0; |
| 442 | CCP_PT_BYTESWAP(&function) = op->u.passthru.byte_swap; |
| 443 | CCP_PT_BITWISE(&function) = op->u.passthru.bit_mod; |
| 444 | CCP5_CMD_FUNCTION(&desc) = function.raw; |
| 445 | |
| 446 | /* Length of source data is always 256 bytes */ |
| 447 | if (op->src.type == CCP_MEMTYPE_SYSTEM) |
| 448 | CCP5_CMD_LEN(&desc) = saddr->length; |
| 449 | else |
| 450 | CCP5_CMD_LEN(&desc) = daddr->length; |
| 451 | |
| 452 | if (op->src.type == CCP_MEMTYPE_SYSTEM) { |
| 453 | CCP5_CMD_SRC_LO(&desc) = ccp_addr_lo(&op->src.u.dma); |
| 454 | CCP5_CMD_SRC_HI(&desc) = ccp_addr_hi(&op->src.u.dma); |
| 455 | CCP5_CMD_SRC_MEM(&desc) = CCP_MEMTYPE_SYSTEM; |
| 456 | |
| 457 | if (op->u.passthru.bit_mod != CCP_PASSTHRU_BITWISE_NOOP) |
| 458 | CCP5_CMD_LSB_ID(&desc) = op->sb_key; |
| 459 | } else { |
| 460 | u32 key_addr = op->src.u.sb * CCP_SB_BYTES; |
| 461 | |
| 462 | CCP5_CMD_SRC_LO(&desc) = lower_32_bits(key_addr); |
| 463 | CCP5_CMD_SRC_HI(&desc) = 0; |
| 464 | CCP5_CMD_SRC_MEM(&desc) = CCP_MEMTYPE_SB; |
| 465 | } |
| 466 | |
| 467 | if (op->dst.type == CCP_MEMTYPE_SYSTEM) { |
| 468 | CCP5_CMD_DST_LO(&desc) = ccp_addr_lo(&op->dst.u.dma); |
| 469 | CCP5_CMD_DST_HI(&desc) = ccp_addr_hi(&op->dst.u.dma); |
| 470 | CCP5_CMD_DST_MEM(&desc) = CCP_MEMTYPE_SYSTEM; |
| 471 | } else { |
| 472 | u32 key_addr = op->dst.u.sb * CCP_SB_BYTES; |
| 473 | |
| 474 | CCP5_CMD_DST_LO(&desc) = lower_32_bits(key_addr); |
| 475 | CCP5_CMD_DST_HI(&desc) = 0; |
| 476 | CCP5_CMD_DST_MEM(&desc) = CCP_MEMTYPE_SB; |
| 477 | } |
| 478 | |
| 479 | return ccp5_do_cmd(&desc, op->cmd_q); |
| 480 | } |
| 481 | |
| 482 | static int ccp5_perform_ecc(struct ccp_op *op) |
| 483 | { |
| 484 | struct ccp5_desc desc; |
| 485 | union ccp_function function; |
| 486 | |
| 487 | /* Zero out all the fields of the command desc */ |
| 488 | memset(&desc, 0, Q_DESC_SIZE); |
| 489 | |
| 490 | CCP5_CMD_ENGINE(&desc) = CCP_ENGINE_ECC; |
| 491 | |
| 492 | CCP5_CMD_SOC(&desc) = 0; |
| 493 | CCP5_CMD_IOC(&desc) = 1; |
| 494 | CCP5_CMD_INIT(&desc) = 0; |
| 495 | CCP5_CMD_EOM(&desc) = 1; |
| 496 | CCP5_CMD_PROT(&desc) = 0; |
| 497 | |
| 498 | function.raw = 0; |
| 499 | function.ecc.mode = op->u.ecc.function; |
| 500 | CCP5_CMD_FUNCTION(&desc) = function.raw; |
| 501 | |
| 502 | CCP5_CMD_LEN(&desc) = op->src.u.dma.length; |
| 503 | |
| 504 | CCP5_CMD_SRC_LO(&desc) = ccp_addr_lo(&op->src.u.dma); |
| 505 | CCP5_CMD_SRC_HI(&desc) = ccp_addr_hi(&op->src.u.dma); |
| 506 | CCP5_CMD_SRC_MEM(&desc) = CCP_MEMTYPE_SYSTEM; |
| 507 | |
| 508 | CCP5_CMD_DST_LO(&desc) = ccp_addr_lo(&op->dst.u.dma); |
| 509 | CCP5_CMD_DST_HI(&desc) = ccp_addr_hi(&op->dst.u.dma); |
| 510 | CCP5_CMD_DST_MEM(&desc) = CCP_MEMTYPE_SYSTEM; |
| 511 | |
| 512 | return ccp5_do_cmd(&desc, op->cmd_q); |
| 513 | } |
| 514 | |
| 515 | static int ccp_find_lsb_regions(struct ccp_cmd_queue *cmd_q, u64 status) |
| 516 | { |
| 517 | int q_mask = 1 << cmd_q->id; |
| 518 | int queues = 0; |
| 519 | int j; |
| 520 | |
| 521 | /* Build a bit mask to know which LSBs this queue has access to. |
| 522 | * Don't bother with segment 0 as it has special privileges. |
| 523 | */ |
| 524 | for (j = 1; j < MAX_LSB_CNT; j++) { |
| 525 | if (status & q_mask) |
| 526 | bitmap_set(cmd_q->lsbmask, j, 1); |
| 527 | status >>= LSB_REGION_WIDTH; |
| 528 | } |
| 529 | queues = bitmap_weight(cmd_q->lsbmask, MAX_LSB_CNT); |
| 530 | dev_info(cmd_q->ccp->dev, "Queue %d can access %d LSB regions\n", |
| 531 | cmd_q->id, queues); |
| 532 | |
| 533 | return queues ? 0 : -EINVAL; |
| 534 | } |
| 535 | |
| 536 | |
| 537 | static int ccp_find_and_assign_lsb_to_q(struct ccp_device *ccp, |
| 538 | int lsb_cnt, int n_lsbs, |
| 539 | unsigned long *lsb_pub) |
| 540 | { |
| 541 | DECLARE_BITMAP(qlsb, MAX_LSB_CNT); |
| 542 | int bitno; |
| 543 | int qlsb_wgt; |
| 544 | int i; |
| 545 | |
| 546 | /* For each queue: |
| 547 | * If the count of potential LSBs available to a queue matches the |
| 548 | * ordinal given to us in lsb_cnt: |
| 549 | * Copy the mask of possible LSBs for this queue into "qlsb"; |
| 550 | * For each bit in qlsb, see if the corresponding bit in the |
| 551 | * aggregation mask is set; if so, we have a match. |
| 552 | * If we have a match, clear the bit in the aggregation to |
| 553 | * mark it as no longer available. |
| 554 | * If there is no match, clear the bit in qlsb and keep looking. |
| 555 | */ |
| 556 | for (i = 0; i < ccp->cmd_q_count; i++) { |
| 557 | struct ccp_cmd_queue *cmd_q = &ccp->cmd_q[i]; |
| 558 | |
| 559 | qlsb_wgt = bitmap_weight(cmd_q->lsbmask, MAX_LSB_CNT); |
| 560 | |
| 561 | if (qlsb_wgt == lsb_cnt) { |
| 562 | bitmap_copy(qlsb, cmd_q->lsbmask, MAX_LSB_CNT); |
| 563 | |
| 564 | bitno = find_first_bit(qlsb, MAX_LSB_CNT); |
| 565 | while (bitno < MAX_LSB_CNT) { |
| 566 | if (test_bit(bitno, lsb_pub)) { |
| 567 | /* We found an available LSB |
| 568 | * that this queue can access |
| 569 | */ |
| 570 | cmd_q->lsb = bitno; |
| 571 | bitmap_clear(lsb_pub, bitno, 1); |
| 572 | dev_info(ccp->dev, |
| 573 | "Queue %d gets LSB %d\n", |
| 574 | i, bitno); |
| 575 | break; |
| 576 | } |
| 577 | bitmap_clear(qlsb, bitno, 1); |
| 578 | bitno = find_first_bit(qlsb, MAX_LSB_CNT); |
| 579 | } |
| 580 | if (bitno >= MAX_LSB_CNT) |
| 581 | return -EINVAL; |
| 582 | n_lsbs--; |
| 583 | } |
| 584 | } |
| 585 | return n_lsbs; |
| 586 | } |
| 587 | |
| 588 | /* For each queue, from the most- to least-constrained: |
| 589 | * find an LSB that can be assigned to the queue. If there are N queues that |
| 590 | * can only use M LSBs, where N > M, fail; otherwise, every queue will get a |
| 591 | * dedicated LSB. Remaining LSB regions become a shared resource. |
| 592 | * If we have fewer LSBs than queues, all LSB regions become shared resources. |
| 593 | */ |
| 594 | static int ccp_assign_lsbs(struct ccp_device *ccp) |
| 595 | { |
| 596 | DECLARE_BITMAP(lsb_pub, MAX_LSB_CNT); |
| 597 | DECLARE_BITMAP(qlsb, MAX_LSB_CNT); |
| 598 | int n_lsbs = 0; |
| 599 | int bitno; |
| 600 | int i, lsb_cnt; |
| 601 | int rc = 0; |
| 602 | |
| 603 | bitmap_zero(lsb_pub, MAX_LSB_CNT); |
| 604 | |
| 605 | /* Create an aggregate bitmap to get a total count of available LSBs */ |
| 606 | for (i = 0; i < ccp->cmd_q_count; i++) |
| 607 | bitmap_or(lsb_pub, |
| 608 | lsb_pub, ccp->cmd_q[i].lsbmask, |
| 609 | MAX_LSB_CNT); |
| 610 | |
| 611 | n_lsbs = bitmap_weight(lsb_pub, MAX_LSB_CNT); |
| 612 | |
| 613 | if (n_lsbs >= ccp->cmd_q_count) { |
| 614 | /* We have enough LSBS to give every queue a private LSB. |
| 615 | * Brute force search to start with the queues that are more |
| 616 | * constrained in LSB choice. When an LSB is privately |
| 617 | * assigned, it is removed from the public mask. |
| 618 | * This is an ugly N squared algorithm with some optimization. |
| 619 | */ |
| 620 | for (lsb_cnt = 1; |
| 621 | n_lsbs && (lsb_cnt <= MAX_LSB_CNT); |
| 622 | lsb_cnt++) { |
| 623 | rc = ccp_find_and_assign_lsb_to_q(ccp, lsb_cnt, n_lsbs, |
| 624 | lsb_pub); |
| 625 | if (rc < 0) |
| 626 | return -EINVAL; |
| 627 | n_lsbs = rc; |
| 628 | } |
| 629 | } |
| 630 | |
| 631 | rc = 0; |
| 632 | /* What's left of the LSBs, according to the public mask, now become |
| 633 | * shared. Any zero bits in the lsb_pub mask represent an LSB region |
| 634 | * that can't be used as a shared resource, so mark the LSB slots for |
| 635 | * them as "in use". |
| 636 | */ |
| 637 | bitmap_copy(qlsb, lsb_pub, MAX_LSB_CNT); |
| 638 | |
| 639 | bitno = find_first_zero_bit(qlsb, MAX_LSB_CNT); |
| 640 | while (bitno < MAX_LSB_CNT) { |
| 641 | bitmap_set(ccp->lsbmap, bitno * LSB_SIZE, LSB_SIZE); |
| 642 | bitmap_set(qlsb, bitno, 1); |
| 643 | bitno = find_first_zero_bit(qlsb, MAX_LSB_CNT); |
| 644 | } |
| 645 | |
| 646 | return rc; |
| 647 | } |
| 648 | |
Gary R Hook | f8d0509 | 2017-04-21 10:50:14 -0500 | [diff] [blame] | 649 | static void ccp5_disable_queue_interrupts(struct ccp_device *ccp) |
| 650 | { |
| 651 | unsigned int i; |
| 652 | |
| 653 | for (i = 0; i < ccp->cmd_q_count; i++) |
| 654 | iowrite32(0x0, ccp->cmd_q[i].reg_int_enable); |
| 655 | } |
| 656 | |
| 657 | static void ccp5_enable_queue_interrupts(struct ccp_device *ccp) |
| 658 | { |
| 659 | unsigned int i; |
| 660 | |
| 661 | for (i = 0; i < ccp->cmd_q_count; i++) |
| 662 | iowrite32(SUPPORTED_INTERRUPTS, ccp->cmd_q[i].reg_int_enable); |
| 663 | } |
| 664 | |
| 665 | static void ccp5_irq_bh(unsigned long data) |
| 666 | { |
| 667 | struct ccp_device *ccp = (struct ccp_device *)data; |
| 668 | u32 status; |
| 669 | unsigned int i; |
| 670 | |
| 671 | for (i = 0; i < ccp->cmd_q_count; i++) { |
| 672 | struct ccp_cmd_queue *cmd_q = &ccp->cmd_q[i]; |
| 673 | |
| 674 | status = ioread32(cmd_q->reg_interrupt_status); |
| 675 | |
| 676 | if (status) { |
| 677 | cmd_q->int_status = status; |
| 678 | cmd_q->q_status = ioread32(cmd_q->reg_status); |
| 679 | cmd_q->q_int_status = ioread32(cmd_q->reg_int_status); |
| 680 | |
| 681 | /* On error, only save the first error value */ |
| 682 | if ((status & INT_ERROR) && !cmd_q->cmd_error) |
| 683 | cmd_q->cmd_error = CMD_Q_ERROR(cmd_q->q_status); |
| 684 | |
| 685 | cmd_q->int_rcvd = 1; |
| 686 | |
| 687 | /* Acknowledge the interrupt and wake the kthread */ |
| 688 | iowrite32(status, cmd_q->reg_interrupt_status); |
| 689 | wake_up_interruptible(&cmd_q->int_queue); |
| 690 | } |
| 691 | } |
| 692 | ccp5_enable_queue_interrupts(ccp); |
| 693 | } |
| 694 | |
| 695 | static irqreturn_t ccp5_irq_handler(int irq, void *data) |
| 696 | { |
| 697 | struct device *dev = data; |
| 698 | struct ccp_device *ccp = dev_get_drvdata(dev); |
| 699 | |
| 700 | ccp5_disable_queue_interrupts(ccp); |
| 701 | if (ccp->use_tasklet) |
| 702 | tasklet_schedule(&ccp->irq_tasklet); |
| 703 | else |
| 704 | ccp5_irq_bh((unsigned long)ccp); |
| 705 | return IRQ_HANDLED; |
| 706 | } |
| 707 | |
Gary R Hook | 4b394a2 | 2016-07-26 19:10:21 -0500 | [diff] [blame] | 708 | static int ccp5_init(struct ccp_device *ccp) |
| 709 | { |
| 710 | struct device *dev = ccp->dev; |
| 711 | struct ccp_cmd_queue *cmd_q; |
| 712 | struct dma_pool *dma_pool; |
| 713 | char dma_pool_name[MAX_DMAPOOL_NAME_LEN]; |
| 714 | unsigned int qmr, qim, i; |
| 715 | u64 status; |
| 716 | u32 status_lo, status_hi; |
| 717 | int ret; |
| 718 | |
| 719 | /* Find available queues */ |
| 720 | qim = 0; |
| 721 | qmr = ioread32(ccp->io_regs + Q_MASK_REG); |
| 722 | for (i = 0; i < MAX_HW_QUEUES; i++) { |
| 723 | |
| 724 | if (!(qmr & (1 << i))) |
| 725 | continue; |
| 726 | |
| 727 | /* Allocate a dma pool for this queue */ |
| 728 | snprintf(dma_pool_name, sizeof(dma_pool_name), "%s_q%d", |
| 729 | ccp->name, i); |
| 730 | dma_pool = dma_pool_create(dma_pool_name, dev, |
| 731 | CCP_DMAPOOL_MAX_SIZE, |
| 732 | CCP_DMAPOOL_ALIGN, 0); |
| 733 | if (!dma_pool) { |
| 734 | dev_err(dev, "unable to allocate dma pool\n"); |
| 735 | ret = -ENOMEM; |
| 736 | } |
| 737 | |
| 738 | cmd_q = &ccp->cmd_q[ccp->cmd_q_count]; |
| 739 | ccp->cmd_q_count++; |
| 740 | |
| 741 | cmd_q->ccp = ccp; |
| 742 | cmd_q->id = i; |
| 743 | cmd_q->dma_pool = dma_pool; |
| 744 | mutex_init(&cmd_q->q_mutex); |
| 745 | |
| 746 | /* Page alignment satisfies our needs for N <= 128 */ |
| 747 | BUILD_BUG_ON(COMMANDS_PER_QUEUE > 128); |
| 748 | cmd_q->qsize = Q_SIZE(Q_DESC_SIZE); |
| 749 | cmd_q->qbase = dma_zalloc_coherent(dev, cmd_q->qsize, |
| 750 | &cmd_q->qbase_dma, |
| 751 | GFP_KERNEL); |
| 752 | if (!cmd_q->qbase) { |
| 753 | dev_err(dev, "unable to allocate command queue\n"); |
| 754 | ret = -ENOMEM; |
| 755 | goto e_pool; |
| 756 | } |
| 757 | |
| 758 | cmd_q->qidx = 0; |
| 759 | /* Preset some register values and masks that are queue |
| 760 | * number dependent |
| 761 | */ |
| 762 | cmd_q->reg_control = ccp->io_regs + |
| 763 | CMD5_Q_STATUS_INCR * (i + 1); |
| 764 | cmd_q->reg_tail_lo = cmd_q->reg_control + CMD5_Q_TAIL_LO_BASE; |
| 765 | cmd_q->reg_head_lo = cmd_q->reg_control + CMD5_Q_HEAD_LO_BASE; |
| 766 | cmd_q->reg_int_enable = cmd_q->reg_control + |
| 767 | CMD5_Q_INT_ENABLE_BASE; |
| 768 | cmd_q->reg_interrupt_status = cmd_q->reg_control + |
| 769 | CMD5_Q_INTERRUPT_STATUS_BASE; |
| 770 | cmd_q->reg_status = cmd_q->reg_control + CMD5_Q_STATUS_BASE; |
| 771 | cmd_q->reg_int_status = cmd_q->reg_control + |
| 772 | CMD5_Q_INT_STATUS_BASE; |
| 773 | cmd_q->reg_dma_status = cmd_q->reg_control + |
| 774 | CMD5_Q_DMA_STATUS_BASE; |
| 775 | cmd_q->reg_dma_read_status = cmd_q->reg_control + |
| 776 | CMD5_Q_DMA_READ_STATUS_BASE; |
| 777 | cmd_q->reg_dma_write_status = cmd_q->reg_control + |
| 778 | CMD5_Q_DMA_WRITE_STATUS_BASE; |
| 779 | |
| 780 | init_waitqueue_head(&cmd_q->int_queue); |
| 781 | |
| 782 | dev_dbg(dev, "queue #%u available\n", i); |
| 783 | } |
| 784 | if (ccp->cmd_q_count == 0) { |
| 785 | dev_notice(dev, "no command queues available\n"); |
| 786 | ret = -EIO; |
| 787 | goto e_pool; |
| 788 | } |
| 789 | dev_notice(dev, "%u command queues available\n", ccp->cmd_q_count); |
| 790 | |
| 791 | /* Turn off the queues and disable interrupts until ready */ |
Gary R Hook | f8d0509 | 2017-04-21 10:50:14 -0500 | [diff] [blame] | 792 | ccp5_disable_queue_interrupts(ccp); |
Gary R Hook | 4b394a2 | 2016-07-26 19:10:21 -0500 | [diff] [blame] | 793 | for (i = 0; i < ccp->cmd_q_count; i++) { |
| 794 | cmd_q = &ccp->cmd_q[i]; |
| 795 | |
| 796 | cmd_q->qcontrol = 0; /* Start with nothing */ |
| 797 | iowrite32(cmd_q->qcontrol, cmd_q->reg_control); |
| 798 | |
Gary R Hook | 4b394a2 | 2016-07-26 19:10:21 -0500 | [diff] [blame] | 799 | ioread32(cmd_q->reg_int_status); |
| 800 | ioread32(cmd_q->reg_status); |
| 801 | |
Gary R Hook | f8d0509 | 2017-04-21 10:50:14 -0500 | [diff] [blame] | 802 | /* Clear the interrupt status */ |
Gary R Hook | 36dffff | 2017-04-20 15:24:09 -0500 | [diff] [blame] | 803 | iowrite32(SUPPORTED_INTERRUPTS, cmd_q->reg_interrupt_status); |
Gary R Hook | 4b394a2 | 2016-07-26 19:10:21 -0500 | [diff] [blame] | 804 | } |
| 805 | |
| 806 | dev_dbg(dev, "Requesting an IRQ...\n"); |
| 807 | /* Request an irq */ |
| 808 | ret = ccp->get_irq(ccp); |
| 809 | if (ret) { |
| 810 | dev_err(dev, "unable to allocate an IRQ\n"); |
| 811 | goto e_pool; |
| 812 | } |
Gary R Hook | f8d0509 | 2017-04-21 10:50:14 -0500 | [diff] [blame] | 813 | /* Initialize the ISR tasklet */ |
| 814 | if (ccp->use_tasklet) |
| 815 | tasklet_init(&ccp->irq_tasklet, ccp5_irq_bh, |
| 816 | (unsigned long)ccp); |
Gary R Hook | 4b394a2 | 2016-07-26 19:10:21 -0500 | [diff] [blame] | 817 | |
| 818 | /* Initialize the queue used to suspend */ |
| 819 | init_waitqueue_head(&ccp->suspend_queue); |
| 820 | |
| 821 | dev_dbg(dev, "Loading LSB map...\n"); |
| 822 | /* Copy the private LSB mask to the public registers */ |
| 823 | status_lo = ioread32(ccp->io_regs + LSB_PRIVATE_MASK_LO_OFFSET); |
| 824 | status_hi = ioread32(ccp->io_regs + LSB_PRIVATE_MASK_HI_OFFSET); |
| 825 | iowrite32(status_lo, ccp->io_regs + LSB_PUBLIC_MASK_LO_OFFSET); |
| 826 | iowrite32(status_hi, ccp->io_regs + LSB_PUBLIC_MASK_HI_OFFSET); |
| 827 | status = ((u64)status_hi<<30) | (u64)status_lo; |
| 828 | |
| 829 | dev_dbg(dev, "Configuring virtual queues...\n"); |
| 830 | /* Configure size of each virtual queue accessible to host */ |
| 831 | for (i = 0; i < ccp->cmd_q_count; i++) { |
| 832 | u32 dma_addr_lo; |
| 833 | u32 dma_addr_hi; |
| 834 | |
| 835 | cmd_q = &ccp->cmd_q[i]; |
| 836 | |
| 837 | cmd_q->qcontrol &= ~(CMD5_Q_SIZE << CMD5_Q_SHIFT); |
| 838 | cmd_q->qcontrol |= QUEUE_SIZE_VAL << CMD5_Q_SHIFT; |
| 839 | |
| 840 | cmd_q->qdma_tail = cmd_q->qbase_dma; |
| 841 | dma_addr_lo = low_address(cmd_q->qdma_tail); |
| 842 | iowrite32((u32)dma_addr_lo, cmd_q->reg_tail_lo); |
| 843 | iowrite32((u32)dma_addr_lo, cmd_q->reg_head_lo); |
| 844 | |
| 845 | dma_addr_hi = high_address(cmd_q->qdma_tail); |
| 846 | cmd_q->qcontrol |= (dma_addr_hi << 16); |
| 847 | iowrite32(cmd_q->qcontrol, cmd_q->reg_control); |
| 848 | |
| 849 | /* Find the LSB regions accessible to the queue */ |
| 850 | ccp_find_lsb_regions(cmd_q, status); |
| 851 | cmd_q->lsb = -1; /* Unassigned value */ |
| 852 | } |
| 853 | |
| 854 | dev_dbg(dev, "Assigning LSBs...\n"); |
| 855 | ret = ccp_assign_lsbs(ccp); |
| 856 | if (ret) { |
| 857 | dev_err(dev, "Unable to assign LSBs (%d)\n", ret); |
| 858 | goto e_irq; |
| 859 | } |
| 860 | |
| 861 | /* Optimization: pre-allocate LSB slots for each queue */ |
| 862 | for (i = 0; i < ccp->cmd_q_count; i++) { |
| 863 | ccp->cmd_q[i].sb_key = ccp_lsb_alloc(&ccp->cmd_q[i], 2); |
| 864 | ccp->cmd_q[i].sb_ctx = ccp_lsb_alloc(&ccp->cmd_q[i], 2); |
| 865 | } |
| 866 | |
| 867 | dev_dbg(dev, "Starting threads...\n"); |
| 868 | /* Create a kthread for each queue */ |
| 869 | for (i = 0; i < ccp->cmd_q_count; i++) { |
| 870 | struct task_struct *kthread; |
| 871 | |
| 872 | cmd_q = &ccp->cmd_q[i]; |
| 873 | |
| 874 | kthread = kthread_create(ccp_cmd_queue_thread, cmd_q, |
| 875 | "%s-q%u", ccp->name, cmd_q->id); |
| 876 | if (IS_ERR(kthread)) { |
| 877 | dev_err(dev, "error creating queue thread (%ld)\n", |
| 878 | PTR_ERR(kthread)); |
| 879 | ret = PTR_ERR(kthread); |
| 880 | goto e_kthread; |
| 881 | } |
| 882 | |
| 883 | cmd_q->kthread = kthread; |
| 884 | wake_up_process(kthread); |
| 885 | } |
| 886 | |
| 887 | dev_dbg(dev, "Enabling interrupts...\n"); |
Gary R Hook | f8d0509 | 2017-04-21 10:50:14 -0500 | [diff] [blame] | 888 | ccp5_enable_queue_interrupts(ccp); |
Gary R Hook | 4b394a2 | 2016-07-26 19:10:21 -0500 | [diff] [blame] | 889 | |
| 890 | dev_dbg(dev, "Registering device...\n"); |
| 891 | /* Put this on the unit list to make it available */ |
| 892 | ccp_add_device(ccp); |
| 893 | |
Gary R Hook | 084935b | 2016-07-26 19:10:31 -0500 | [diff] [blame] | 894 | ret = ccp_register_rng(ccp); |
| 895 | if (ret) |
| 896 | goto e_kthread; |
| 897 | |
Gary R Hook | 99d90b2 | 2016-07-26 19:10:40 -0500 | [diff] [blame] | 898 | /* Register the DMA engine support */ |
| 899 | ret = ccp_dmaengine_register(ccp); |
| 900 | if (ret) |
Gary R Hook | 9ddb9dc | 2016-09-28 11:53:47 -0500 | [diff] [blame] | 901 | goto e_hwrng; |
Gary R Hook | 99d90b2 | 2016-07-26 19:10:40 -0500 | [diff] [blame] | 902 | |
Gary R Hook | 4b394a2 | 2016-07-26 19:10:21 -0500 | [diff] [blame] | 903 | return 0; |
| 904 | |
Gary R Hook | 9ddb9dc | 2016-09-28 11:53:47 -0500 | [diff] [blame] | 905 | e_hwrng: |
| 906 | ccp_unregister_rng(ccp); |
| 907 | |
Gary R Hook | 4b394a2 | 2016-07-26 19:10:21 -0500 | [diff] [blame] | 908 | e_kthread: |
| 909 | for (i = 0; i < ccp->cmd_q_count; i++) |
| 910 | if (ccp->cmd_q[i].kthread) |
| 911 | kthread_stop(ccp->cmd_q[i].kthread); |
| 912 | |
| 913 | e_irq: |
| 914 | ccp->free_irq(ccp); |
| 915 | |
| 916 | e_pool: |
| 917 | for (i = 0; i < ccp->cmd_q_count; i++) |
| 918 | dma_pool_destroy(ccp->cmd_q[i].dma_pool); |
| 919 | |
| 920 | return ret; |
| 921 | } |
| 922 | |
| 923 | static void ccp5_destroy(struct ccp_device *ccp) |
| 924 | { |
| 925 | struct device *dev = ccp->dev; |
| 926 | struct ccp_cmd_queue *cmd_q; |
| 927 | struct ccp_cmd *cmd; |
| 928 | unsigned int i; |
| 929 | |
Gary R Hook | 99d90b2 | 2016-07-26 19:10:40 -0500 | [diff] [blame] | 930 | /* Unregister the DMA engine */ |
| 931 | ccp_dmaengine_unregister(ccp); |
| 932 | |
Gary R Hook | 084935b | 2016-07-26 19:10:31 -0500 | [diff] [blame] | 933 | /* Unregister the RNG */ |
| 934 | ccp_unregister_rng(ccp); |
| 935 | |
Gary R Hook | 4b394a2 | 2016-07-26 19:10:21 -0500 | [diff] [blame] | 936 | /* Remove this device from the list of available units first */ |
| 937 | ccp_del_device(ccp); |
| 938 | |
| 939 | /* Disable and clear interrupts */ |
Gary R Hook | f8d0509 | 2017-04-21 10:50:14 -0500 | [diff] [blame] | 940 | ccp5_disable_queue_interrupts(ccp); |
Gary R Hook | 4b394a2 | 2016-07-26 19:10:21 -0500 | [diff] [blame] | 941 | for (i = 0; i < ccp->cmd_q_count; i++) { |
| 942 | cmd_q = &ccp->cmd_q[i]; |
| 943 | |
| 944 | /* Turn off the run bit */ |
| 945 | iowrite32(cmd_q->qcontrol & ~CMD5_Q_RUN, cmd_q->reg_control); |
| 946 | |
Gary R Hook | 4b394a2 | 2016-07-26 19:10:21 -0500 | [diff] [blame] | 947 | /* Clear the interrupt status */ |
Gary R Hook | 93424b2 | 2017-04-20 15:24:22 -0500 | [diff] [blame] | 948 | iowrite32(SUPPORTED_INTERRUPTS, cmd_q->reg_interrupt_status); |
Gary R Hook | 4b394a2 | 2016-07-26 19:10:21 -0500 | [diff] [blame] | 949 | ioread32(cmd_q->reg_int_status); |
| 950 | ioread32(cmd_q->reg_status); |
| 951 | } |
| 952 | |
| 953 | /* Stop the queue kthreads */ |
| 954 | for (i = 0; i < ccp->cmd_q_count; i++) |
| 955 | if (ccp->cmd_q[i].kthread) |
| 956 | kthread_stop(ccp->cmd_q[i].kthread); |
| 957 | |
| 958 | ccp->free_irq(ccp); |
| 959 | |
| 960 | for (i = 0; i < ccp->cmd_q_count; i++) { |
| 961 | cmd_q = &ccp->cmd_q[i]; |
| 962 | dma_free_coherent(dev, cmd_q->qsize, cmd_q->qbase, |
| 963 | cmd_q->qbase_dma); |
| 964 | } |
| 965 | |
| 966 | /* Flush the cmd and backlog queue */ |
| 967 | while (!list_empty(&ccp->cmd)) { |
| 968 | /* Invoke the callback directly with an error code */ |
| 969 | cmd = list_first_entry(&ccp->cmd, struct ccp_cmd, entry); |
| 970 | list_del(&cmd->entry); |
| 971 | cmd->callback(cmd->data, -ENODEV); |
| 972 | } |
| 973 | while (!list_empty(&ccp->backlog)) { |
| 974 | /* Invoke the callback directly with an error code */ |
| 975 | cmd = list_first_entry(&ccp->backlog, struct ccp_cmd, entry); |
| 976 | list_del(&cmd->entry); |
| 977 | cmd->callback(cmd->data, -ENODEV); |
| 978 | } |
| 979 | } |
| 980 | |
Gary R Hook | 4b394a2 | 2016-07-26 19:10:21 -0500 | [diff] [blame] | 981 | static void ccp5_config(struct ccp_device *ccp) |
| 982 | { |
| 983 | /* Public side */ |
Gary R Hook | 1105ccf | 2017-01-27 15:28:45 -0600 | [diff] [blame] | 984 | iowrite32(0x0, ccp->io_regs + CMD5_REQID_CONFIG_OFFSET); |
Gary R Hook | 4b394a2 | 2016-07-26 19:10:21 -0500 | [diff] [blame] | 985 | } |
| 986 | |
Gary R Hook | e14e7d1 | 2016-07-26 19:10:49 -0500 | [diff] [blame] | 987 | static void ccp5other_config(struct ccp_device *ccp) |
| 988 | { |
| 989 | int i; |
| 990 | u32 rnd; |
| 991 | |
| 992 | /* We own all of the queues on the NTB CCP */ |
| 993 | |
| 994 | iowrite32(0x00012D57, ccp->io_regs + CMD5_TRNG_CTL_OFFSET); |
| 995 | iowrite32(0x00000003, ccp->io_regs + CMD5_CONFIG_0_OFFSET); |
| 996 | for (i = 0; i < 12; i++) { |
| 997 | rnd = ioread32(ccp->io_regs + TRNG_OUT_REG); |
| 998 | iowrite32(rnd, ccp->io_regs + CMD5_AES_MASK_OFFSET); |
| 999 | } |
| 1000 | |
| 1001 | iowrite32(0x0000001F, ccp->io_regs + CMD5_QUEUE_MASK_OFFSET); |
| 1002 | iowrite32(0x00005B6D, ccp->io_regs + CMD5_QUEUE_PRIO_OFFSET); |
| 1003 | iowrite32(0x00000000, ccp->io_regs + CMD5_CMD_TIMEOUT_OFFSET); |
| 1004 | |
| 1005 | iowrite32(0x3FFFFFFF, ccp->io_regs + LSB_PRIVATE_MASK_LO_OFFSET); |
| 1006 | iowrite32(0x000003FF, ccp->io_regs + LSB_PRIVATE_MASK_HI_OFFSET); |
| 1007 | |
| 1008 | iowrite32(0x00108823, ccp->io_regs + CMD5_CLK_GATE_CTL_OFFSET); |
| 1009 | |
| 1010 | ccp5_config(ccp); |
| 1011 | } |
| 1012 | |
| 1013 | /* Version 5 adds some function, but is essentially the same as v5 */ |
Gary R Hook | 4b394a2 | 2016-07-26 19:10:21 -0500 | [diff] [blame] | 1014 | static const struct ccp_actions ccp5_actions = { |
| 1015 | .aes = ccp5_perform_aes, |
| 1016 | .xts_aes = ccp5_perform_xts_aes, |
| 1017 | .sha = ccp5_perform_sha, |
| 1018 | .rsa = ccp5_perform_rsa, |
| 1019 | .passthru = ccp5_perform_passthru, |
| 1020 | .ecc = ccp5_perform_ecc, |
| 1021 | .sballoc = ccp_lsb_alloc, |
| 1022 | .sbfree = ccp_lsb_free, |
| 1023 | .init = ccp5_init, |
| 1024 | .destroy = ccp5_destroy, |
| 1025 | .get_free_slots = ccp5_get_free_slots, |
| 1026 | .irqhandler = ccp5_irq_handler, |
| 1027 | }; |
| 1028 | |
Gary R Hook | 9ddb9dc | 2016-09-28 11:53:47 -0500 | [diff] [blame] | 1029 | const struct ccp_vdata ccpv5a = { |
Gary R Hook | 4b394a2 | 2016-07-26 19:10:21 -0500 | [diff] [blame] | 1030 | .version = CCP_VERSION(5, 0), |
| 1031 | .setup = ccp5_config, |
| 1032 | .perform = &ccp5_actions, |
| 1033 | .bar = 2, |
| 1034 | .offset = 0x0, |
| 1035 | }; |
Gary R Hook | e14e7d1 | 2016-07-26 19:10:49 -0500 | [diff] [blame] | 1036 | |
Gary R Hook | 9ddb9dc | 2016-09-28 11:53:47 -0500 | [diff] [blame] | 1037 | const struct ccp_vdata ccpv5b = { |
Gary R Hook | e14e7d1 | 2016-07-26 19:10:49 -0500 | [diff] [blame] | 1038 | .version = CCP_VERSION(5, 0), |
| 1039 | .setup = ccp5other_config, |
| 1040 | .perform = &ccp5_actions, |
| 1041 | .bar = 2, |
| 1042 | .offset = 0x0, |
| 1043 | }; |