Boris BREZILLON | db509a4 | 2015-06-18 15:46:21 +0200 | [diff] [blame] | 1 | /* |
| 2 | * Provide TDMA helper functions used by cipher and hash algorithm |
| 3 | * implementations. |
| 4 | * |
| 5 | * Author: Boris Brezillon <boris.brezillon@free-electrons.com> |
| 6 | * Author: Arnaud Ebalard <arno@natisbad.org> |
| 7 | * |
| 8 | * This work is based on an initial version written by |
| 9 | * Sebastian Andrzej Siewior < sebastian at breakpoint dot cc > |
| 10 | * |
| 11 | * This program is free software; you can redistribute it and/or modify it |
| 12 | * under the terms of the GNU General Public License version 2 as published |
| 13 | * by the Free Software Foundation. |
| 14 | */ |
| 15 | |
| 16 | #include "cesa.h" |
| 17 | |
| 18 | bool mv_cesa_req_dma_iter_next_transfer(struct mv_cesa_dma_iter *iter, |
| 19 | struct mv_cesa_sg_dma_iter *sgiter, |
| 20 | unsigned int len) |
| 21 | { |
| 22 | if (!sgiter->sg) |
| 23 | return false; |
| 24 | |
| 25 | sgiter->op_offset += len; |
| 26 | sgiter->offset += len; |
| 27 | if (sgiter->offset == sg_dma_len(sgiter->sg)) { |
| 28 | if (sg_is_last(sgiter->sg)) |
| 29 | return false; |
| 30 | sgiter->offset = 0; |
| 31 | sgiter->sg = sg_next(sgiter->sg); |
| 32 | } |
| 33 | |
| 34 | if (sgiter->op_offset == iter->op_len) |
| 35 | return false; |
| 36 | |
| 37 | return true; |
| 38 | } |
| 39 | |
Romain Perier | 53da740 | 2016-06-21 10:08:35 +0200 | [diff] [blame] | 40 | void mv_cesa_dma_step(struct mv_cesa_req *dreq) |
Boris BREZILLON | db509a4 | 2015-06-18 15:46:21 +0200 | [diff] [blame] | 41 | { |
Romain Perier | 53da740 | 2016-06-21 10:08:35 +0200 | [diff] [blame] | 42 | struct mv_cesa_engine *engine = dreq->engine; |
Boris BREZILLON | db509a4 | 2015-06-18 15:46:21 +0200 | [diff] [blame] | 43 | |
Russell King | b150856 | 2015-10-18 18:31:00 +0100 | [diff] [blame] | 44 | writel_relaxed(0, engine->regs + CESA_SA_CFG); |
Boris BREZILLON | db509a4 | 2015-06-18 15:46:21 +0200 | [diff] [blame] | 45 | |
| 46 | mv_cesa_set_int_mask(engine, CESA_SA_INT_ACC0_IDMA_DONE); |
Russell King | b150856 | 2015-10-18 18:31:00 +0100 | [diff] [blame] | 47 | writel_relaxed(CESA_TDMA_DST_BURST_128B | CESA_TDMA_SRC_BURST_128B | |
| 48 | CESA_TDMA_NO_BYTE_SWAP | CESA_TDMA_EN, |
| 49 | engine->regs + CESA_TDMA_CONTROL); |
Boris BREZILLON | db509a4 | 2015-06-18 15:46:21 +0200 | [diff] [blame] | 50 | |
Russell King | b150856 | 2015-10-18 18:31:00 +0100 | [diff] [blame] | 51 | writel_relaxed(CESA_SA_CFG_ACT_CH0_IDMA | CESA_SA_CFG_MULTI_PKT | |
| 52 | CESA_SA_CFG_CH0_W_IDMA | CESA_SA_CFG_PARA_DIS, |
| 53 | engine->regs + CESA_SA_CFG); |
| 54 | writel_relaxed(dreq->chain.first->cur_dma, |
| 55 | engine->regs + CESA_TDMA_NEXT_ADDR); |
Romain Perier | f628308 | 2016-06-21 10:08:32 +0200 | [diff] [blame] | 56 | BUG_ON(readl(engine->regs + CESA_SA_CMD) & |
| 57 | CESA_SA_CMD_EN_CESA_SA_ACCL0); |
Boris BREZILLON | db509a4 | 2015-06-18 15:46:21 +0200 | [diff] [blame] | 58 | writel(CESA_SA_CMD_EN_CESA_SA_ACCL0, engine->regs + CESA_SA_CMD); |
| 59 | } |
| 60 | |
Romain Perier | 53da740 | 2016-06-21 10:08:35 +0200 | [diff] [blame] | 61 | void mv_cesa_dma_cleanup(struct mv_cesa_req *dreq) |
Boris BREZILLON | db509a4 | 2015-06-18 15:46:21 +0200 | [diff] [blame] | 62 | { |
| 63 | struct mv_cesa_tdma_desc *tdma; |
| 64 | |
| 65 | for (tdma = dreq->chain.first; tdma;) { |
| 66 | struct mv_cesa_tdma_desc *old_tdma = tdma; |
Romain Perier | b99acf7 | 2016-06-21 10:08:33 +0200 | [diff] [blame] | 67 | u32 type = tdma->flags & CESA_TDMA_TYPE_MSK; |
Boris BREZILLON | db509a4 | 2015-06-18 15:46:21 +0200 | [diff] [blame] | 68 | |
Romain Perier | b99acf7 | 2016-06-21 10:08:33 +0200 | [diff] [blame] | 69 | if (type == CESA_TDMA_OP) |
Boris BREZILLON | db509a4 | 2015-06-18 15:46:21 +0200 | [diff] [blame] | 70 | dma_pool_free(cesa_dev->dma->op_pool, tdma->op, |
| 71 | le32_to_cpu(tdma->src)); |
Romain Perier | bac8e80 | 2016-06-21 10:08:34 +0200 | [diff] [blame] | 72 | else if (type == CESA_TDMA_IV) |
| 73 | dma_pool_free(cesa_dev->dma->iv_pool, tdma->data, |
| 74 | le32_to_cpu(tdma->dst)); |
Boris BREZILLON | db509a4 | 2015-06-18 15:46:21 +0200 | [diff] [blame] | 75 | |
| 76 | tdma = tdma->next; |
| 77 | dma_pool_free(cesa_dev->dma->tdma_desc_pool, old_tdma, |
Russell King | 5d75413 | 2015-10-18 18:31:05 +0100 | [diff] [blame] | 78 | old_tdma->cur_dma); |
Boris BREZILLON | db509a4 | 2015-06-18 15:46:21 +0200 | [diff] [blame] | 79 | } |
| 80 | |
| 81 | dreq->chain.first = NULL; |
| 82 | dreq->chain.last = NULL; |
| 83 | } |
| 84 | |
Romain Perier | 53da740 | 2016-06-21 10:08:35 +0200 | [diff] [blame] | 85 | void mv_cesa_dma_prepare(struct mv_cesa_req *dreq, |
Boris BREZILLON | db509a4 | 2015-06-18 15:46:21 +0200 | [diff] [blame] | 86 | struct mv_cesa_engine *engine) |
| 87 | { |
| 88 | struct mv_cesa_tdma_desc *tdma; |
| 89 | |
| 90 | for (tdma = dreq->chain.first; tdma; tdma = tdma->next) { |
| 91 | if (tdma->flags & CESA_TDMA_DST_IN_SRAM) |
| 92 | tdma->dst = cpu_to_le32(tdma->dst + engine->sram_dma); |
| 93 | |
| 94 | if (tdma->flags & CESA_TDMA_SRC_IN_SRAM) |
| 95 | tdma->src = cpu_to_le32(tdma->src + engine->sram_dma); |
| 96 | |
Romain Perier | b99acf7 | 2016-06-21 10:08:33 +0200 | [diff] [blame] | 97 | if ((tdma->flags & CESA_TDMA_TYPE_MSK) == CESA_TDMA_OP) |
Boris BREZILLON | db509a4 | 2015-06-18 15:46:21 +0200 | [diff] [blame] | 98 | mv_cesa_adjust_op(engine, tdma->op); |
| 99 | } |
| 100 | } |
| 101 | |
Romain Perier | 85030c5 | 2016-06-21 10:08:39 +0200 | [diff] [blame] | 102 | void mv_cesa_tdma_chain(struct mv_cesa_engine *engine, |
| 103 | struct mv_cesa_req *dreq) |
| 104 | { |
| 105 | if (engine->chain.first == NULL && engine->chain.last == NULL) { |
| 106 | engine->chain.first = dreq->chain.first; |
| 107 | engine->chain.last = dreq->chain.last; |
| 108 | } else { |
| 109 | struct mv_cesa_tdma_desc *last; |
| 110 | |
| 111 | last = engine->chain.last; |
| 112 | last->next = dreq->chain.first; |
| 113 | engine->chain.last = dreq->chain.last; |
| 114 | |
Romain Perier | bdc2571 | 2016-12-14 15:15:07 +0100 | [diff] [blame] | 115 | /* |
| 116 | * Break the DMA chain if the CESA_TDMA_BREAK_CHAIN is set on |
| 117 | * the last element of the current chain, or if the request |
| 118 | * being queued needs the IV regs to be set before lauching |
| 119 | * the request. |
| 120 | */ |
| 121 | if (!(last->flags & CESA_TDMA_BREAK_CHAIN) && |
| 122 | !(dreq->chain.first->flags & CESA_TDMA_SET_STATE)) |
Romain Perier | 85030c5 | 2016-06-21 10:08:39 +0200 | [diff] [blame] | 123 | last->next_dma = dreq->chain.first->cur_dma; |
| 124 | } |
| 125 | } |
| 126 | |
| 127 | int mv_cesa_tdma_process(struct mv_cesa_engine *engine, u32 status) |
| 128 | { |
| 129 | struct crypto_async_request *req = NULL; |
| 130 | struct mv_cesa_tdma_desc *tdma = NULL, *next = NULL; |
| 131 | dma_addr_t tdma_cur; |
| 132 | int res = 0; |
| 133 | |
| 134 | tdma_cur = readl(engine->regs + CESA_TDMA_CUR); |
| 135 | |
| 136 | for (tdma = engine->chain.first; tdma; tdma = next) { |
| 137 | spin_lock_bh(&engine->lock); |
| 138 | next = tdma->next; |
| 139 | spin_unlock_bh(&engine->lock); |
| 140 | |
| 141 | if (tdma->flags & CESA_TDMA_END_OF_REQ) { |
| 142 | struct crypto_async_request *backlog = NULL; |
| 143 | struct mv_cesa_ctx *ctx; |
| 144 | u32 current_status; |
| 145 | |
| 146 | spin_lock_bh(&engine->lock); |
| 147 | /* |
| 148 | * if req is NULL, this means we're processing the |
| 149 | * request in engine->req. |
| 150 | */ |
| 151 | if (!req) |
| 152 | req = engine->req; |
| 153 | else |
| 154 | req = mv_cesa_dequeue_req_locked(engine, |
| 155 | &backlog); |
| 156 | |
| 157 | /* Re-chaining to the next request */ |
| 158 | engine->chain.first = tdma->next; |
| 159 | tdma->next = NULL; |
| 160 | |
| 161 | /* If this is the last request, clear the chain */ |
| 162 | if (engine->chain.first == NULL) |
| 163 | engine->chain.last = NULL; |
| 164 | spin_unlock_bh(&engine->lock); |
| 165 | |
| 166 | ctx = crypto_tfm_ctx(req->tfm); |
| 167 | current_status = (tdma->cur_dma == tdma_cur) ? |
| 168 | status : CESA_SA_INT_ACC0_IDMA_DONE; |
| 169 | res = ctx->ops->process(req, current_status); |
| 170 | ctx->ops->complete(req); |
| 171 | |
| 172 | if (res == 0) |
| 173 | mv_cesa_engine_enqueue_complete_request(engine, |
| 174 | req); |
| 175 | |
| 176 | if (backlog) |
| 177 | backlog->complete(backlog, -EINPROGRESS); |
| 178 | } |
| 179 | |
| 180 | if (res || tdma->cur_dma == tdma_cur) |
| 181 | break; |
| 182 | } |
| 183 | |
| 184 | /* Save the last request in error to engine->req, so that the core |
| 185 | * knows which request was fautly */ |
| 186 | if (res) { |
| 187 | spin_lock_bh(&engine->lock); |
| 188 | engine->req = req; |
| 189 | spin_unlock_bh(&engine->lock); |
| 190 | } |
| 191 | |
| 192 | return res; |
| 193 | } |
| 194 | |
Boris BREZILLON | db509a4 | 2015-06-18 15:46:21 +0200 | [diff] [blame] | 195 | static struct mv_cesa_tdma_desc * |
| 196 | mv_cesa_dma_add_desc(struct mv_cesa_tdma_chain *chain, gfp_t flags) |
| 197 | { |
| 198 | struct mv_cesa_tdma_desc *new_tdma = NULL; |
| 199 | dma_addr_t dma_handle; |
| 200 | |
Julia Lawall | 472d640 | 2016-04-29 22:09:11 +0200 | [diff] [blame] | 201 | new_tdma = dma_pool_zalloc(cesa_dev->dma->tdma_desc_pool, flags, |
| 202 | &dma_handle); |
Boris BREZILLON | db509a4 | 2015-06-18 15:46:21 +0200 | [diff] [blame] | 203 | if (!new_tdma) |
| 204 | return ERR_PTR(-ENOMEM); |
| 205 | |
Russell King | 5d75413 | 2015-10-18 18:31:05 +0100 | [diff] [blame] | 206 | new_tdma->cur_dma = dma_handle; |
Boris BREZILLON | db509a4 | 2015-06-18 15:46:21 +0200 | [diff] [blame] | 207 | if (chain->last) { |
Russell King | 5d75413 | 2015-10-18 18:31:05 +0100 | [diff] [blame] | 208 | chain->last->next_dma = cpu_to_le32(dma_handle); |
Boris BREZILLON | db509a4 | 2015-06-18 15:46:21 +0200 | [diff] [blame] | 209 | chain->last->next = new_tdma; |
| 210 | } else { |
| 211 | chain->first = new_tdma; |
| 212 | } |
| 213 | |
| 214 | chain->last = new_tdma; |
| 215 | |
| 216 | return new_tdma; |
| 217 | } |
| 218 | |
Romain Perier | bac8e80 | 2016-06-21 10:08:34 +0200 | [diff] [blame] | 219 | int mv_cesa_dma_add_iv_op(struct mv_cesa_tdma_chain *chain, dma_addr_t src, |
| 220 | u32 size, u32 flags, gfp_t gfp_flags) |
| 221 | { |
| 222 | |
| 223 | struct mv_cesa_tdma_desc *tdma; |
| 224 | u8 *iv; |
| 225 | dma_addr_t dma_handle; |
| 226 | |
| 227 | tdma = mv_cesa_dma_add_desc(chain, gfp_flags); |
| 228 | if (IS_ERR(tdma)) |
| 229 | return PTR_ERR(tdma); |
| 230 | |
Romain Perier | aa6416e | 2016-07-18 11:32:24 +0200 | [diff] [blame] | 231 | iv = dma_pool_alloc(cesa_dev->dma->iv_pool, gfp_flags, &dma_handle); |
Romain Perier | bac8e80 | 2016-06-21 10:08:34 +0200 | [diff] [blame] | 232 | if (!iv) |
| 233 | return -ENOMEM; |
| 234 | |
| 235 | tdma->byte_cnt = cpu_to_le32(size | BIT(31)); |
| 236 | tdma->src = src; |
| 237 | tdma->dst = cpu_to_le32(dma_handle); |
| 238 | tdma->data = iv; |
| 239 | |
| 240 | flags &= (CESA_TDMA_DST_IN_SRAM | CESA_TDMA_SRC_IN_SRAM); |
| 241 | tdma->flags = flags | CESA_TDMA_IV; |
| 242 | return 0; |
| 243 | } |
| 244 | |
Boris BREZILLON | db509a4 | 2015-06-18 15:46:21 +0200 | [diff] [blame] | 245 | struct mv_cesa_op_ctx *mv_cesa_dma_add_op(struct mv_cesa_tdma_chain *chain, |
| 246 | const struct mv_cesa_op_ctx *op_templ, |
| 247 | bool skip_ctx, |
| 248 | gfp_t flags) |
| 249 | { |
| 250 | struct mv_cesa_tdma_desc *tdma; |
| 251 | struct mv_cesa_op_ctx *op; |
| 252 | dma_addr_t dma_handle; |
Russell King | 6de59d4 | 2015-10-18 18:31:26 +0100 | [diff] [blame] | 253 | unsigned int size; |
Boris BREZILLON | db509a4 | 2015-06-18 15:46:21 +0200 | [diff] [blame] | 254 | |
| 255 | tdma = mv_cesa_dma_add_desc(chain, flags); |
| 256 | if (IS_ERR(tdma)) |
| 257 | return ERR_CAST(tdma); |
| 258 | |
| 259 | op = dma_pool_alloc(cesa_dev->dma->op_pool, flags, &dma_handle); |
| 260 | if (!op) |
| 261 | return ERR_PTR(-ENOMEM); |
| 262 | |
| 263 | *op = *op_templ; |
| 264 | |
Russell King | 6de59d4 | 2015-10-18 18:31:26 +0100 | [diff] [blame] | 265 | size = skip_ctx ? sizeof(op->desc) : sizeof(*op); |
| 266 | |
Boris BREZILLON | db509a4 | 2015-06-18 15:46:21 +0200 | [diff] [blame] | 267 | tdma = chain->last; |
| 268 | tdma->op = op; |
Russell King | 6de59d4 | 2015-10-18 18:31:26 +0100 | [diff] [blame] | 269 | tdma->byte_cnt = cpu_to_le32(size | BIT(31)); |
Russell King | ea1f662 | 2015-10-18 18:31:20 +0100 | [diff] [blame] | 270 | tdma->src = cpu_to_le32(dma_handle); |
Thomas Petazzoni | 36225b9 | 2016-08-09 11:03:14 +0200 | [diff] [blame] | 271 | tdma->dst = CESA_SA_CFG_SRAM_OFFSET; |
Boris BREZILLON | db509a4 | 2015-06-18 15:46:21 +0200 | [diff] [blame] | 272 | tdma->flags = CESA_TDMA_DST_IN_SRAM | CESA_TDMA_OP; |
| 273 | |
| 274 | return op; |
| 275 | } |
| 276 | |
| 277 | int mv_cesa_dma_add_data_transfer(struct mv_cesa_tdma_chain *chain, |
| 278 | dma_addr_t dst, dma_addr_t src, u32 size, |
| 279 | u32 flags, gfp_t gfp_flags) |
| 280 | { |
| 281 | struct mv_cesa_tdma_desc *tdma; |
| 282 | |
| 283 | tdma = mv_cesa_dma_add_desc(chain, gfp_flags); |
| 284 | if (IS_ERR(tdma)) |
| 285 | return PTR_ERR(tdma); |
| 286 | |
Russell King | 6de59d4 | 2015-10-18 18:31:26 +0100 | [diff] [blame] | 287 | tdma->byte_cnt = cpu_to_le32(size | BIT(31)); |
Boris BREZILLON | db509a4 | 2015-06-18 15:46:21 +0200 | [diff] [blame] | 288 | tdma->src = src; |
| 289 | tdma->dst = dst; |
| 290 | |
| 291 | flags &= (CESA_TDMA_DST_IN_SRAM | CESA_TDMA_SRC_IN_SRAM); |
| 292 | tdma->flags = flags | CESA_TDMA_DATA; |
| 293 | |
| 294 | return 0; |
| 295 | } |
| 296 | |
Russell King | 35622ea | 2015-10-18 18:31:10 +0100 | [diff] [blame] | 297 | int mv_cesa_dma_add_dummy_launch(struct mv_cesa_tdma_chain *chain, gfp_t flags) |
Boris BREZILLON | db509a4 | 2015-06-18 15:46:21 +0200 | [diff] [blame] | 298 | { |
| 299 | struct mv_cesa_tdma_desc *tdma; |
| 300 | |
| 301 | tdma = mv_cesa_dma_add_desc(chain, flags); |
| 302 | if (IS_ERR(tdma)) |
| 303 | return PTR_ERR(tdma); |
| 304 | |
| 305 | return 0; |
| 306 | } |
| 307 | |
Russell King | 35622ea | 2015-10-18 18:31:10 +0100 | [diff] [blame] | 308 | int mv_cesa_dma_add_dummy_end(struct mv_cesa_tdma_chain *chain, gfp_t flags) |
Boris BREZILLON | db509a4 | 2015-06-18 15:46:21 +0200 | [diff] [blame] | 309 | { |
| 310 | struct mv_cesa_tdma_desc *tdma; |
| 311 | |
| 312 | tdma = mv_cesa_dma_add_desc(chain, flags); |
| 313 | if (IS_ERR(tdma)) |
| 314 | return PTR_ERR(tdma); |
| 315 | |
Russell King | 6de59d4 | 2015-10-18 18:31:26 +0100 | [diff] [blame] | 316 | tdma->byte_cnt = cpu_to_le32(BIT(31)); |
Boris BREZILLON | db509a4 | 2015-06-18 15:46:21 +0200 | [diff] [blame] | 317 | |
| 318 | return 0; |
| 319 | } |
| 320 | |
| 321 | int mv_cesa_dma_add_op_transfers(struct mv_cesa_tdma_chain *chain, |
| 322 | struct mv_cesa_dma_iter *dma_iter, |
| 323 | struct mv_cesa_sg_dma_iter *sgiter, |
| 324 | gfp_t gfp_flags) |
| 325 | { |
| 326 | u32 flags = sgiter->dir == DMA_TO_DEVICE ? |
| 327 | CESA_TDMA_DST_IN_SRAM : CESA_TDMA_SRC_IN_SRAM; |
| 328 | unsigned int len; |
| 329 | |
| 330 | do { |
| 331 | dma_addr_t dst, src; |
| 332 | int ret; |
| 333 | |
| 334 | len = mv_cesa_req_dma_iter_transfer_len(dma_iter, sgiter); |
| 335 | if (sgiter->dir == DMA_TO_DEVICE) { |
| 336 | dst = CESA_SA_DATA_SRAM_OFFSET + sgiter->op_offset; |
| 337 | src = sg_dma_address(sgiter->sg) + sgiter->offset; |
| 338 | } else { |
| 339 | dst = sg_dma_address(sgiter->sg) + sgiter->offset; |
| 340 | src = CESA_SA_DATA_SRAM_OFFSET + sgiter->op_offset; |
| 341 | } |
| 342 | |
| 343 | ret = mv_cesa_dma_add_data_transfer(chain, dst, src, len, |
| 344 | flags, gfp_flags); |
| 345 | if (ret) |
| 346 | return ret; |
| 347 | |
| 348 | } while (mv_cesa_req_dma_iter_next_transfer(dma_iter, sgiter, len)); |
| 349 | |
| 350 | return 0; |
| 351 | } |