Frank Haverkamp | eaf4722 | 2013-12-09 13:30:40 +0100 | [diff] [blame] | 1 | /** |
| 2 | * IBM Accelerator Family 'GenWQE' |
| 3 | * |
| 4 | * (C) Copyright IBM Corp. 2013 |
| 5 | * |
| 6 | * Author: Frank Haverkamp <haver@linux.vnet.ibm.com> |
| 7 | * Author: Joerg-Stephan Vogt <jsvogt@de.ibm.com> |
| 8 | * Author: Michael Jung <mijung@de.ibm.com> |
| 9 | * Author: Michael Ruettger <michael@ibmra.de> |
| 10 | * |
| 11 | * This program is free software; you can redistribute it and/or modify |
| 12 | * it under the terms of the GNU General Public License (version 2 only) |
| 13 | * as published by the Free Software Foundation. |
| 14 | * |
| 15 | * This program is distributed in the hope that it will be useful, |
| 16 | * but WITHOUT ANY WARRANTY; without even the implied warranty of |
| 17 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
| 18 | * GNU General Public License for more details. |
| 19 | */ |
| 20 | |
| 21 | /* |
| 22 | * Device Driver Control Block (DDCB) queue support. Definition of |
| 23 | * interrupt handlers for queue support as well as triggering the |
| 24 | * health monitor code in case of problems. The current hardware uses |
| 25 | * an MSI interrupt which is shared between error handling and |
| 26 | * functional code. |
| 27 | */ |
| 28 | |
| 29 | #include <linux/types.h> |
| 30 | #include <linux/module.h> |
| 31 | #include <linux/sched.h> |
| 32 | #include <linux/wait.h> |
| 33 | #include <linux/pci.h> |
| 34 | #include <linux/string.h> |
| 35 | #include <linux/dma-mapping.h> |
| 36 | #include <linux/delay.h> |
| 37 | #include <linux/module.h> |
| 38 | #include <linux/interrupt.h> |
| 39 | #include <linux/crc-itu-t.h> |
| 40 | |
Frank Haverkamp | 90b4e97 | 2014-01-07 15:41:24 +0100 | [diff] [blame] | 41 | #include "card_base.h" |
Frank Haverkamp | eaf4722 | 2013-12-09 13:30:40 +0100 | [diff] [blame] | 42 | #include "card_ddcb.h" |
| 43 | |
| 44 | /* |
| 45 | * N: next DDCB, this is where the next DDCB will be put. |
| 46 | * A: active DDCB, this is where the code will look for the next completion. |
| 47 | * x: DDCB is enqueued, we are waiting for its completion. |
| 48 | |
| 49 | * Situation (1): Empty queue |
| 50 | * +---+---+---+---+---+---+---+---+ |
| 51 | * | 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 | |
| 52 | * | | | | | | | | | |
| 53 | * +---+---+---+---+---+---+---+---+ |
| 54 | * A/N |
| 55 | * enqueued_ddcbs = A - N = 2 - 2 = 0 |
| 56 | * |
| 57 | * Situation (2): Wrapped, N > A |
| 58 | * +---+---+---+---+---+---+---+---+ |
| 59 | * | 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 | |
| 60 | * | | | x | x | | | | | |
| 61 | * +---+---+---+---+---+---+---+---+ |
| 62 | * A N |
| 63 | * enqueued_ddcbs = N - A = 4 - 2 = 2 |
| 64 | * |
| 65 | * Situation (3): Queue wrapped, A > N |
| 66 | * +---+---+---+---+---+---+---+---+ |
| 67 | * | 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 | |
| 68 | * | x | x | | | x | x | x | x | |
| 69 | * +---+---+---+---+---+---+---+---+ |
| 70 | * N A |
| 71 | * enqueued_ddcbs = queue_max - (A - N) = 8 - (4 - 2) = 6 |
| 72 | * |
| 73 | * Situation (4a): Queue full N > A |
| 74 | * +---+---+---+---+---+---+---+---+ |
| 75 | * | 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 | |
| 76 | * | x | x | x | x | x | x | x | | |
| 77 | * +---+---+---+---+---+---+---+---+ |
| 78 | * A N |
| 79 | * |
| 80 | * enqueued_ddcbs = N - A = 7 - 0 = 7 |
| 81 | * |
| 82 | * Situation (4a): Queue full A > N |
| 83 | * +---+---+---+---+---+---+---+---+ |
| 84 | * | 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 | |
| 85 | * | x | x | x | | x | x | x | x | |
| 86 | * +---+---+---+---+---+---+---+---+ |
| 87 | * N A |
| 88 | * enqueued_ddcbs = queue_max - (A - N) = 8 - (4 - 3) = 7 |
| 89 | */ |
| 90 | |
| 91 | static int queue_empty(struct ddcb_queue *queue) |
| 92 | { |
| 93 | return queue->ddcb_next == queue->ddcb_act; |
| 94 | } |
| 95 | |
| 96 | static int queue_enqueued_ddcbs(struct ddcb_queue *queue) |
| 97 | { |
| 98 | if (queue->ddcb_next >= queue->ddcb_act) |
| 99 | return queue->ddcb_next - queue->ddcb_act; |
| 100 | |
| 101 | return queue->ddcb_max - (queue->ddcb_act - queue->ddcb_next); |
| 102 | } |
| 103 | |
| 104 | static int queue_free_ddcbs(struct ddcb_queue *queue) |
| 105 | { |
| 106 | int free_ddcbs = queue->ddcb_max - queue_enqueued_ddcbs(queue) - 1; |
| 107 | |
| 108 | if (WARN_ON_ONCE(free_ddcbs < 0)) { /* must never ever happen! */ |
| 109 | return 0; |
| 110 | } |
| 111 | return free_ddcbs; |
| 112 | } |
| 113 | |
| 114 | /* |
| 115 | * Use of the PRIV field in the DDCB for queue debugging: |
| 116 | * |
| 117 | * (1) Trying to get rid of a DDCB which saw a timeout: |
| 118 | * pddcb->priv[6] = 0xcc; # cleared |
| 119 | * |
| 120 | * (2) Append a DDCB via NEXT bit: |
| 121 | * pddcb->priv[7] = 0xaa; # appended |
| 122 | * |
| 123 | * (3) DDCB needed tapping: |
| 124 | * pddcb->priv[7] = 0xbb; # tapped |
| 125 | * |
| 126 | * (4) DDCB marked as correctly finished: |
| 127 | * pddcb->priv[6] = 0xff; # finished |
| 128 | */ |
| 129 | |
| 130 | static inline void ddcb_mark_tapped(struct ddcb *pddcb) |
| 131 | { |
| 132 | pddcb->priv[7] = 0xbb; /* tapped */ |
| 133 | } |
| 134 | |
| 135 | static inline void ddcb_mark_appended(struct ddcb *pddcb) |
| 136 | { |
| 137 | pddcb->priv[7] = 0xaa; /* appended */ |
| 138 | } |
| 139 | |
| 140 | static inline void ddcb_mark_cleared(struct ddcb *pddcb) |
| 141 | { |
| 142 | pddcb->priv[6] = 0xcc; /* cleared */ |
| 143 | } |
| 144 | |
| 145 | static inline void ddcb_mark_finished(struct ddcb *pddcb) |
| 146 | { |
| 147 | pddcb->priv[6] = 0xff; /* finished */ |
| 148 | } |
| 149 | |
| 150 | static inline void ddcb_mark_unused(struct ddcb *pddcb) |
| 151 | { |
| 152 | pddcb->priv_64 = cpu_to_be64(0); /* not tapped */ |
| 153 | } |
| 154 | |
| 155 | /** |
| 156 | * genwqe_crc16() - Generate 16-bit crc as required for DDCBs |
| 157 | * @buff: pointer to data buffer |
| 158 | * @len: length of data for calculation |
| 159 | * @init: initial crc (0xffff at start) |
| 160 | * |
| 161 | * Polynomial = x^16 + x^12 + x^5 + 1 (0x1021) |
| 162 | * Example: 4 bytes 0x01 0x02 0x03 0x04 with init = 0xffff |
| 163 | * should result in a crc16 of 0x89c3 |
| 164 | * |
| 165 | * Return: crc16 checksum in big endian format ! |
| 166 | */ |
| 167 | static inline u16 genwqe_crc16(const u8 *buff, size_t len, u16 init) |
| 168 | { |
| 169 | return crc_itu_t(init, buff, len); |
| 170 | } |
| 171 | |
| 172 | static void print_ddcb_info(struct genwqe_dev *cd, struct ddcb_queue *queue) |
| 173 | { |
| 174 | int i; |
| 175 | struct ddcb *pddcb; |
| 176 | unsigned long flags; |
| 177 | struct pci_dev *pci_dev = cd->pci_dev; |
| 178 | |
| 179 | spin_lock_irqsave(&cd->print_lock, flags); |
| 180 | |
| 181 | dev_info(&pci_dev->dev, |
| 182 | "DDCB list for card #%d (ddcb_act=%d / ddcb_next=%d):\n", |
| 183 | cd->card_idx, queue->ddcb_act, queue->ddcb_next); |
| 184 | |
| 185 | pddcb = queue->ddcb_vaddr; |
| 186 | for (i = 0; i < queue->ddcb_max; i++) { |
| 187 | dev_err(&pci_dev->dev, |
| 188 | " %c %-3d: RETC=%03x SEQ=%04x " |
| 189 | "HSI=%02X SHI=%02x PRIV=%06llx CMD=%03x\n", |
| 190 | i == queue->ddcb_act ? '>' : ' ', |
| 191 | i, |
| 192 | be16_to_cpu(pddcb->retc_16), |
| 193 | be16_to_cpu(pddcb->seqnum_16), |
| 194 | pddcb->hsi, |
| 195 | pddcb->shi, |
| 196 | be64_to_cpu(pddcb->priv_64), |
| 197 | pddcb->cmd); |
| 198 | pddcb++; |
| 199 | } |
| 200 | spin_unlock_irqrestore(&cd->print_lock, flags); |
| 201 | } |
| 202 | |
| 203 | struct genwqe_ddcb_cmd *ddcb_requ_alloc(void) |
| 204 | { |
| 205 | struct ddcb_requ *req; |
| 206 | |
| 207 | req = kzalloc(sizeof(*req), GFP_ATOMIC); |
| 208 | if (!req) |
| 209 | return NULL; |
| 210 | |
| 211 | return &req->cmd; |
| 212 | } |
| 213 | |
| 214 | void ddcb_requ_free(struct genwqe_ddcb_cmd *cmd) |
| 215 | { |
| 216 | struct ddcb_requ *req = container_of(cmd, struct ddcb_requ, cmd); |
| 217 | kfree(req); |
| 218 | } |
| 219 | |
| 220 | static inline enum genwqe_requ_state ddcb_requ_get_state(struct ddcb_requ *req) |
| 221 | { |
| 222 | return req->req_state; |
| 223 | } |
| 224 | |
| 225 | static inline void ddcb_requ_set_state(struct ddcb_requ *req, |
| 226 | enum genwqe_requ_state new_state) |
| 227 | { |
| 228 | req->req_state = new_state; |
| 229 | } |
| 230 | |
| 231 | static inline int ddcb_requ_collect_debug_data(struct ddcb_requ *req) |
| 232 | { |
| 233 | return req->cmd.ddata_addr != 0x0; |
| 234 | } |
| 235 | |
| 236 | /** |
| 237 | * ddcb_requ_finished() - Returns the hardware state of the associated DDCB |
| 238 | * @cd: pointer to genwqe device descriptor |
| 239 | * @req: DDCB work request |
| 240 | * |
| 241 | * Status of ddcb_requ mirrors this hardware state, but is copied in |
| 242 | * the ddcb_requ on interrupt/polling function. The lowlevel code |
| 243 | * should check the hardware state directly, the higher level code |
| 244 | * should check the copy. |
| 245 | * |
| 246 | * This function will also return true if the state of the queue is |
| 247 | * not GENWQE_CARD_USED. This enables us to purge all DDCBs in the |
| 248 | * shutdown case. |
| 249 | */ |
| 250 | static int ddcb_requ_finished(struct genwqe_dev *cd, struct ddcb_requ *req) |
| 251 | { |
| 252 | return (ddcb_requ_get_state(req) == GENWQE_REQU_FINISHED) || |
| 253 | (cd->card_state != GENWQE_CARD_USED); |
| 254 | } |
| 255 | |
| 256 | /** |
| 257 | * enqueue_ddcb() - Enqueue a DDCB |
| 258 | * @cd: pointer to genwqe device descriptor |
| 259 | * @queue: queue this operation should be done on |
| 260 | * @ddcb_no: pointer to ddcb number being tapped |
| 261 | * |
| 262 | * Start execution of DDCB by tapping or append to queue via NEXT |
| 263 | * bit. This is done by an atomic 'compare and swap' instruction and |
| 264 | * checking SHI and HSI of the previous DDCB. |
| 265 | * |
| 266 | * This function must only be called with ddcb_lock held. |
| 267 | * |
| 268 | * Return: 1 if new DDCB is appended to previous |
| 269 | * 2 if DDCB queue is tapped via register/simulation |
| 270 | */ |
| 271 | #define RET_DDCB_APPENDED 1 |
| 272 | #define RET_DDCB_TAPPED 2 |
| 273 | |
| 274 | static int enqueue_ddcb(struct genwqe_dev *cd, struct ddcb_queue *queue, |
| 275 | struct ddcb *pddcb, int ddcb_no) |
| 276 | { |
| 277 | unsigned int try; |
| 278 | int prev_no; |
| 279 | struct ddcb *prev_ddcb; |
Frank Haverkamp | 58d66ce | 2013-12-20 16:26:10 +0100 | [diff] [blame] | 280 | __be32 old, new, icrc_hsi_shi; |
Frank Haverkamp | eaf4722 | 2013-12-09 13:30:40 +0100 | [diff] [blame] | 281 | u64 num; |
| 282 | |
| 283 | /* |
| 284 | * For performance checks a Dispatch Timestamp can be put into |
| 285 | * DDCB It is supposed to use the SLU's free running counter, |
| 286 | * but this requires PCIe cycles. |
| 287 | */ |
| 288 | ddcb_mark_unused(pddcb); |
| 289 | |
| 290 | /* check previous DDCB if already fetched */ |
| 291 | prev_no = (ddcb_no == 0) ? queue->ddcb_max - 1 : ddcb_no - 1; |
| 292 | prev_ddcb = &queue->ddcb_vaddr[prev_no]; |
| 293 | |
| 294 | /* |
| 295 | * It might have happened that the HSI.FETCHED bit is |
| 296 | * set. Retry in this case. Therefore I expect maximum 2 times |
| 297 | * trying. |
| 298 | */ |
| 299 | ddcb_mark_appended(pddcb); |
| 300 | for (try = 0; try < 2; try++) { |
| 301 | old = prev_ddcb->icrc_hsi_shi_32; /* read SHI/HSI in BE32 */ |
| 302 | |
| 303 | /* try to append via NEXT bit if prev DDCB is not completed */ |
| 304 | if ((old & DDCB_COMPLETED_BE32) != 0x00000000) |
| 305 | break; |
| 306 | |
| 307 | new = (old | DDCB_NEXT_BE32); |
Frank Haverkamp | 68fe8ac | 2014-03-20 15:11:03 +0100 | [diff] [blame] | 308 | |
| 309 | wmb(); |
Frank Haverkamp | eaf4722 | 2013-12-09 13:30:40 +0100 | [diff] [blame] | 310 | icrc_hsi_shi = cmpxchg(&prev_ddcb->icrc_hsi_shi_32, old, new); |
| 311 | |
| 312 | if (icrc_hsi_shi == old) |
| 313 | return RET_DDCB_APPENDED; /* appended to queue */ |
| 314 | } |
| 315 | |
| 316 | /* Queue must be re-started by updating QUEUE_OFFSET */ |
| 317 | ddcb_mark_tapped(pddcb); |
| 318 | num = (u64)ddcb_no << 8; |
Frank Haverkamp | 68fe8ac | 2014-03-20 15:11:03 +0100 | [diff] [blame] | 319 | |
| 320 | wmb(); |
Frank Haverkamp | eaf4722 | 2013-12-09 13:30:40 +0100 | [diff] [blame] | 321 | __genwqe_writeq(cd, queue->IO_QUEUE_OFFSET, num); /* start queue */ |
| 322 | |
| 323 | return RET_DDCB_TAPPED; |
| 324 | } |
| 325 | |
| 326 | /** |
| 327 | * copy_ddcb_results() - Copy output state from real DDCB to request |
| 328 | * |
| 329 | * Copy DDCB ASV to request struct. There is no endian |
| 330 | * conversion made, since data structure in ASV is still |
| 331 | * unknown here. |
| 332 | * |
| 333 | * This is needed by: |
| 334 | * - genwqe_purge_ddcb() |
| 335 | * - genwqe_check_ddcb_queue() |
| 336 | */ |
| 337 | static void copy_ddcb_results(struct ddcb_requ *req, int ddcb_no) |
| 338 | { |
| 339 | struct ddcb_queue *queue = req->queue; |
| 340 | struct ddcb *pddcb = &queue->ddcb_vaddr[req->num]; |
| 341 | |
| 342 | memcpy(&req->cmd.asv[0], &pddcb->asv[0], DDCB_ASV_LENGTH); |
| 343 | |
| 344 | /* copy status flags of the variant part */ |
| 345 | req->cmd.vcrc = be16_to_cpu(pddcb->vcrc_16); |
| 346 | req->cmd.deque_ts = be64_to_cpu(pddcb->deque_ts_64); |
| 347 | req->cmd.cmplt_ts = be64_to_cpu(pddcb->cmplt_ts_64); |
| 348 | |
| 349 | req->cmd.attn = be16_to_cpu(pddcb->attn_16); |
| 350 | req->cmd.progress = be32_to_cpu(pddcb->progress_32); |
| 351 | req->cmd.retc = be16_to_cpu(pddcb->retc_16); |
| 352 | |
| 353 | if (ddcb_requ_collect_debug_data(req)) { |
| 354 | int prev_no = (ddcb_no == 0) ? |
| 355 | queue->ddcb_max - 1 : ddcb_no - 1; |
| 356 | struct ddcb *prev_pddcb = &queue->ddcb_vaddr[prev_no]; |
| 357 | |
| 358 | memcpy(&req->debug_data.ddcb_finished, pddcb, |
| 359 | sizeof(req->debug_data.ddcb_finished)); |
| 360 | memcpy(&req->debug_data.ddcb_prev, prev_pddcb, |
| 361 | sizeof(req->debug_data.ddcb_prev)); |
| 362 | } |
| 363 | } |
| 364 | |
| 365 | /** |
| 366 | * genwqe_check_ddcb_queue() - Checks DDCB queue for completed work equests. |
| 367 | * @cd: pointer to genwqe device descriptor |
| 368 | * |
| 369 | * Return: Number of DDCBs which were finished |
| 370 | */ |
| 371 | static int genwqe_check_ddcb_queue(struct genwqe_dev *cd, |
| 372 | struct ddcb_queue *queue) |
| 373 | { |
| 374 | unsigned long flags; |
| 375 | int ddcbs_finished = 0; |
| 376 | struct pci_dev *pci_dev = cd->pci_dev; |
| 377 | |
| 378 | spin_lock_irqsave(&queue->ddcb_lock, flags); |
| 379 | |
| 380 | /* FIXME avoid soft locking CPU */ |
| 381 | while (!queue_empty(queue) && (ddcbs_finished < queue->ddcb_max)) { |
| 382 | |
| 383 | struct ddcb *pddcb; |
| 384 | struct ddcb_requ *req; |
| 385 | u16 vcrc, vcrc_16, retc_16; |
| 386 | |
| 387 | pddcb = &queue->ddcb_vaddr[queue->ddcb_act]; |
| 388 | |
| 389 | if ((pddcb->icrc_hsi_shi_32 & DDCB_COMPLETED_BE32) == |
| 390 | 0x00000000) |
| 391 | goto go_home; /* not completed, continue waiting */ |
| 392 | |
| 393 | /* Note: DDCB could be purged */ |
| 394 | |
| 395 | req = queue->ddcb_req[queue->ddcb_act]; |
| 396 | if (req == NULL) { |
| 397 | /* this occurs if DDCB is purged, not an error */ |
| 398 | /* Move active DDCB further; Nothing to do anymore. */ |
| 399 | goto pick_next_one; |
| 400 | } |
| 401 | |
| 402 | /* |
| 403 | * HSI=0x44 (fetched and completed), but RETC is |
| 404 | * 0x101, or even worse 0x000. |
| 405 | * |
| 406 | * In case of seeing the queue in inconsistent state |
| 407 | * we read the errcnts and the queue status to provide |
| 408 | * a trigger for our PCIe analyzer stop capturing. |
| 409 | */ |
| 410 | retc_16 = be16_to_cpu(pddcb->retc_16); |
| 411 | if ((pddcb->hsi == 0x44) && (retc_16 <= 0x101)) { |
| 412 | u64 errcnts, status; |
| 413 | u64 ddcb_offs = (u64)pddcb - (u64)queue->ddcb_vaddr; |
| 414 | |
| 415 | errcnts = __genwqe_readq(cd, queue->IO_QUEUE_ERRCNTS); |
| 416 | status = __genwqe_readq(cd, queue->IO_QUEUE_STATUS); |
| 417 | |
| 418 | dev_err(&pci_dev->dev, |
| 419 | "[%s] SEQN=%04x HSI=%02x RETC=%03x " |
| 420 | " Q_ERRCNTS=%016llx Q_STATUS=%016llx\n" |
| 421 | " DDCB_DMA_ADDR=%016llx\n", |
| 422 | __func__, be16_to_cpu(pddcb->seqnum_16), |
| 423 | pddcb->hsi, retc_16, errcnts, status, |
| 424 | queue->ddcb_daddr + ddcb_offs); |
| 425 | } |
| 426 | |
| 427 | copy_ddcb_results(req, queue->ddcb_act); |
| 428 | queue->ddcb_req[queue->ddcb_act] = NULL; /* take from queue */ |
| 429 | |
| 430 | dev_dbg(&pci_dev->dev, "FINISHED DDCB#%d\n", req->num); |
| 431 | genwqe_hexdump(pci_dev, pddcb, sizeof(*pddcb)); |
| 432 | |
| 433 | ddcb_mark_finished(pddcb); |
| 434 | |
| 435 | /* calculate CRC_16 to see if VCRC is correct */ |
| 436 | vcrc = genwqe_crc16(pddcb->asv, |
| 437 | VCRC_LENGTH(req->cmd.asv_length), |
| 438 | 0xffff); |
| 439 | vcrc_16 = be16_to_cpu(pddcb->vcrc_16); |
| 440 | if (vcrc != vcrc_16) { |
| 441 | printk_ratelimited(KERN_ERR |
| 442 | "%s %s: err: wrong VCRC pre=%02x vcrc_len=%d " |
| 443 | "bytes vcrc_data=%04x is not vcrc_card=%04x\n", |
| 444 | GENWQE_DEVNAME, dev_name(&pci_dev->dev), |
| 445 | pddcb->pre, VCRC_LENGTH(req->cmd.asv_length), |
| 446 | vcrc, vcrc_16); |
| 447 | } |
| 448 | |
| 449 | ddcb_requ_set_state(req, GENWQE_REQU_FINISHED); |
| 450 | queue->ddcbs_completed++; |
| 451 | queue->ddcbs_in_flight--; |
| 452 | |
| 453 | /* wake up process waiting for this DDCB */ |
| 454 | wake_up_interruptible(&queue->ddcb_waitqs[queue->ddcb_act]); |
| 455 | |
| 456 | pick_next_one: |
| 457 | queue->ddcb_act = (queue->ddcb_act + 1) % queue->ddcb_max; |
| 458 | ddcbs_finished++; |
| 459 | } |
| 460 | |
| 461 | go_home: |
| 462 | spin_unlock_irqrestore(&queue->ddcb_lock, flags); |
| 463 | return ddcbs_finished; |
| 464 | } |
| 465 | |
| 466 | /** |
| 467 | * __genwqe_wait_ddcb(): Waits until DDCB is completed |
| 468 | * @cd: pointer to genwqe device descriptor |
| 469 | * @req: pointer to requsted DDCB parameters |
| 470 | * |
| 471 | * The Service Layer will update the RETC in DDCB when processing is |
| 472 | * pending or done. |
| 473 | * |
| 474 | * Return: > 0 remaining jiffies, DDCB completed |
| 475 | * -ETIMEDOUT when timeout |
| 476 | * -ERESTARTSYS when ^C |
| 477 | * -EINVAL when unknown error condition |
| 478 | * |
| 479 | * When an error is returned the called needs to ensure that |
| 480 | * purge_ddcb() is being called to get the &req removed from the |
| 481 | * queue. |
| 482 | */ |
| 483 | int __genwqe_wait_ddcb(struct genwqe_dev *cd, struct ddcb_requ *req) |
| 484 | { |
| 485 | int rc; |
| 486 | unsigned int ddcb_no; |
| 487 | struct ddcb_queue *queue; |
| 488 | struct pci_dev *pci_dev = cd->pci_dev; |
| 489 | |
| 490 | if (req == NULL) |
| 491 | return -EINVAL; |
| 492 | |
| 493 | queue = req->queue; |
| 494 | if (queue == NULL) |
| 495 | return -EINVAL; |
| 496 | |
| 497 | ddcb_no = req->num; |
| 498 | if (ddcb_no >= queue->ddcb_max) |
| 499 | return -EINVAL; |
| 500 | |
| 501 | rc = wait_event_interruptible_timeout(queue->ddcb_waitqs[ddcb_no], |
| 502 | ddcb_requ_finished(cd, req), |
| 503 | genwqe_ddcb_software_timeout * HZ); |
| 504 | |
| 505 | /* |
| 506 | * We need to distinguish 3 cases here: |
| 507 | * 1. rc == 0 timeout occured |
| 508 | * 2. rc == -ERESTARTSYS signal received |
| 509 | * 3. rc > 0 remaining jiffies condition is true |
| 510 | */ |
| 511 | if (rc == 0) { |
| 512 | struct ddcb_queue *queue = req->queue; |
| 513 | struct ddcb *pddcb; |
| 514 | |
| 515 | /* |
| 516 | * Timeout may be caused by long task switching time. |
| 517 | * When timeout happens, check if the request has |
| 518 | * meanwhile completed. |
| 519 | */ |
| 520 | genwqe_check_ddcb_queue(cd, req->queue); |
| 521 | if (ddcb_requ_finished(cd, req)) |
| 522 | return rc; |
| 523 | |
| 524 | dev_err(&pci_dev->dev, |
| 525 | "[%s] err: DDCB#%d timeout rc=%d state=%d req @ %p\n", |
| 526 | __func__, req->num, rc, ddcb_requ_get_state(req), |
| 527 | req); |
| 528 | dev_err(&pci_dev->dev, |
| 529 | "[%s] IO_QUEUE_STATUS=0x%016llx\n", __func__, |
| 530 | __genwqe_readq(cd, queue->IO_QUEUE_STATUS)); |
| 531 | |
| 532 | pddcb = &queue->ddcb_vaddr[req->num]; |
| 533 | genwqe_hexdump(pci_dev, pddcb, sizeof(*pddcb)); |
| 534 | |
| 535 | print_ddcb_info(cd, req->queue); |
| 536 | return -ETIMEDOUT; |
| 537 | |
| 538 | } else if (rc == -ERESTARTSYS) { |
| 539 | return rc; |
| 540 | /* |
| 541 | * EINTR: Stops the application |
| 542 | * ERESTARTSYS: Restartable systemcall; called again |
| 543 | */ |
| 544 | |
| 545 | } else if (rc < 0) { |
| 546 | dev_err(&pci_dev->dev, |
| 547 | "[%s] err: DDCB#%d unknown result (rc=%d) %d!\n", |
| 548 | __func__, req->num, rc, ddcb_requ_get_state(req)); |
| 549 | return -EINVAL; |
| 550 | } |
| 551 | |
| 552 | /* Severe error occured. Driver is forced to stop operation */ |
| 553 | if (cd->card_state != GENWQE_CARD_USED) { |
| 554 | dev_err(&pci_dev->dev, |
| 555 | "[%s] err: DDCB#%d forced to stop (rc=%d)\n", |
| 556 | __func__, req->num, rc); |
| 557 | return -EIO; |
| 558 | } |
| 559 | return rc; |
| 560 | } |
| 561 | |
| 562 | /** |
| 563 | * get_next_ddcb() - Get next available DDCB |
| 564 | * @cd: pointer to genwqe device descriptor |
| 565 | * |
| 566 | * DDCB's content is completely cleared but presets for PRE and |
| 567 | * SEQNUM. This function must only be called when ddcb_lock is held. |
| 568 | * |
| 569 | * Return: NULL if no empty DDCB available otherwise ptr to next DDCB. |
| 570 | */ |
| 571 | static struct ddcb *get_next_ddcb(struct genwqe_dev *cd, |
| 572 | struct ddcb_queue *queue, |
| 573 | int *num) |
| 574 | { |
| 575 | u64 *pu64; |
| 576 | struct ddcb *pddcb; |
| 577 | |
| 578 | if (queue_free_ddcbs(queue) == 0) /* queue is full */ |
| 579 | return NULL; |
| 580 | |
| 581 | /* find new ddcb */ |
| 582 | pddcb = &queue->ddcb_vaddr[queue->ddcb_next]; |
| 583 | |
| 584 | /* if it is not completed, we are not allowed to use it */ |
| 585 | /* barrier(); */ |
| 586 | if ((pddcb->icrc_hsi_shi_32 & DDCB_COMPLETED_BE32) == 0x00000000) |
| 587 | return NULL; |
| 588 | |
| 589 | *num = queue->ddcb_next; /* internal DDCB number */ |
| 590 | queue->ddcb_next = (queue->ddcb_next + 1) % queue->ddcb_max; |
| 591 | |
| 592 | /* clear important DDCB fields */ |
| 593 | pu64 = (u64 *)pddcb; |
| 594 | pu64[0] = 0ULL; /* offs 0x00 (ICRC,HSI,SHI,...) */ |
| 595 | pu64[1] = 0ULL; /* offs 0x01 (ACFUNC,CMD...) */ |
| 596 | |
| 597 | /* destroy previous results in ASV */ |
| 598 | pu64[0x80/8] = 0ULL; /* offs 0x80 (ASV + 0) */ |
| 599 | pu64[0x88/8] = 0ULL; /* offs 0x88 (ASV + 0x08) */ |
| 600 | pu64[0x90/8] = 0ULL; /* offs 0x90 (ASV + 0x10) */ |
| 601 | pu64[0x98/8] = 0ULL; /* offs 0x98 (ASV + 0x18) */ |
| 602 | pu64[0xd0/8] = 0ULL; /* offs 0xd0 (RETC,ATTN...) */ |
| 603 | |
| 604 | pddcb->pre = DDCB_PRESET_PRE; /* 128 */ |
| 605 | pddcb->seqnum_16 = cpu_to_be16(queue->ddcb_seq++); |
| 606 | return pddcb; |
| 607 | } |
| 608 | |
| 609 | /** |
| 610 | * __genwqe_purge_ddcb() - Remove a DDCB from the workqueue |
| 611 | * @cd: genwqe device descriptor |
| 612 | * @req: DDCB request |
| 613 | * |
| 614 | * This will fail when the request was already FETCHED. In this case |
| 615 | * we need to wait until it is finished. Else the DDCB can be |
| 616 | * reused. This function also ensures that the request data structure |
| 617 | * is removed from ddcb_req[]. |
| 618 | * |
| 619 | * Do not forget to call this function when genwqe_wait_ddcb() fails, |
| 620 | * such that the request gets really removed from ddcb_req[]. |
| 621 | * |
| 622 | * Return: 0 success |
| 623 | */ |
| 624 | int __genwqe_purge_ddcb(struct genwqe_dev *cd, struct ddcb_requ *req) |
| 625 | { |
| 626 | struct ddcb *pddcb = NULL; |
| 627 | unsigned int t; |
| 628 | unsigned long flags; |
| 629 | struct ddcb_queue *queue = req->queue; |
| 630 | struct pci_dev *pci_dev = cd->pci_dev; |
Frank Haverkamp | eaf4722 | 2013-12-09 13:30:40 +0100 | [diff] [blame] | 631 | u64 queue_status; |
Frank Haverkamp | 58d66ce | 2013-12-20 16:26:10 +0100 | [diff] [blame] | 632 | __be32 icrc_hsi_shi = 0x0000; |
| 633 | __be32 old, new; |
Frank Haverkamp | eaf4722 | 2013-12-09 13:30:40 +0100 | [diff] [blame] | 634 | |
| 635 | /* unsigned long flags; */ |
| 636 | if (genwqe_ddcb_software_timeout <= 0) { |
| 637 | dev_err(&pci_dev->dev, |
| 638 | "[%s] err: software timeout is not set!\n", __func__); |
| 639 | return -EFAULT; |
| 640 | } |
| 641 | |
| 642 | pddcb = &queue->ddcb_vaddr[req->num]; |
| 643 | |
| 644 | for (t = 0; t < genwqe_ddcb_software_timeout * 10; t++) { |
| 645 | |
| 646 | spin_lock_irqsave(&queue->ddcb_lock, flags); |
| 647 | |
| 648 | /* Check if req was meanwhile finished */ |
| 649 | if (ddcb_requ_get_state(req) == GENWQE_REQU_FINISHED) |
| 650 | goto go_home; |
| 651 | |
| 652 | /* try to set PURGE bit if FETCHED/COMPLETED are not set */ |
| 653 | old = pddcb->icrc_hsi_shi_32; /* read SHI/HSI in BE32 */ |
| 654 | if ((old & DDCB_FETCHED_BE32) == 0x00000000) { |
| 655 | |
| 656 | new = (old | DDCB_PURGE_BE32); |
| 657 | icrc_hsi_shi = cmpxchg(&pddcb->icrc_hsi_shi_32, |
| 658 | old, new); |
| 659 | if (icrc_hsi_shi == old) |
| 660 | goto finish_ddcb; |
| 661 | } |
| 662 | |
| 663 | /* normal finish with HSI bit */ |
| 664 | barrier(); |
| 665 | icrc_hsi_shi = pddcb->icrc_hsi_shi_32; |
| 666 | if (icrc_hsi_shi & DDCB_COMPLETED_BE32) |
| 667 | goto finish_ddcb; |
| 668 | |
| 669 | spin_unlock_irqrestore(&queue->ddcb_lock, flags); |
| 670 | |
| 671 | /* |
| 672 | * Here the check_ddcb() function will most likely |
| 673 | * discover this DDCB to be finished some point in |
| 674 | * time. It will mark the req finished and free it up |
| 675 | * in the list. |
| 676 | */ |
| 677 | |
| 678 | copy_ddcb_results(req, req->num); /* for the failing case */ |
| 679 | msleep(100); /* sleep for 1/10 second and try again */ |
| 680 | continue; |
| 681 | |
| 682 | finish_ddcb: |
| 683 | copy_ddcb_results(req, req->num); |
| 684 | ddcb_requ_set_state(req, GENWQE_REQU_FINISHED); |
| 685 | queue->ddcbs_in_flight--; |
| 686 | queue->ddcb_req[req->num] = NULL; /* delete from array */ |
| 687 | ddcb_mark_cleared(pddcb); |
| 688 | |
| 689 | /* Move active DDCB further; Nothing to do here anymore. */ |
| 690 | |
| 691 | /* |
| 692 | * We need to ensure that there is at least one free |
| 693 | * DDCB in the queue. To do that, we must update |
| 694 | * ddcb_act only if the COMPLETED bit is set for the |
| 695 | * DDCB we are working on else we treat that DDCB even |
| 696 | * if we PURGED it as occupied (hardware is supposed |
| 697 | * to set the COMPLETED bit yet!). |
| 698 | */ |
| 699 | icrc_hsi_shi = pddcb->icrc_hsi_shi_32; |
| 700 | if ((icrc_hsi_shi & DDCB_COMPLETED_BE32) && |
| 701 | (queue->ddcb_act == req->num)) { |
| 702 | queue->ddcb_act = ((queue->ddcb_act + 1) % |
| 703 | queue->ddcb_max); |
| 704 | } |
| 705 | go_home: |
| 706 | spin_unlock_irqrestore(&queue->ddcb_lock, flags); |
| 707 | return 0; |
| 708 | } |
| 709 | |
| 710 | /* |
| 711 | * If the card is dead and the queue is forced to stop, we |
| 712 | * might see this in the queue status register. |
| 713 | */ |
| 714 | queue_status = __genwqe_readq(cd, queue->IO_QUEUE_STATUS); |
| 715 | |
| 716 | dev_dbg(&pci_dev->dev, "UN/FINISHED DDCB#%d\n", req->num); |
| 717 | genwqe_hexdump(pci_dev, pddcb, sizeof(*pddcb)); |
| 718 | |
| 719 | dev_err(&pci_dev->dev, |
| 720 | "[%s] err: DDCB#%d not purged and not completed " |
| 721 | "after %d seconds QSTAT=%016llx!!\n", |
| 722 | __func__, req->num, genwqe_ddcb_software_timeout, |
| 723 | queue_status); |
| 724 | |
| 725 | print_ddcb_info(cd, req->queue); |
| 726 | |
| 727 | return -EFAULT; |
| 728 | } |
| 729 | |
| 730 | int genwqe_init_debug_data(struct genwqe_dev *cd, struct genwqe_debug_data *d) |
| 731 | { |
| 732 | int len; |
| 733 | struct pci_dev *pci_dev = cd->pci_dev; |
| 734 | |
| 735 | if (d == NULL) { |
| 736 | dev_err(&pci_dev->dev, |
| 737 | "[%s] err: invalid memory for debug data!\n", |
| 738 | __func__); |
| 739 | return -EFAULT; |
| 740 | } |
| 741 | |
| 742 | len = sizeof(d->driver_version); |
| 743 | snprintf(d->driver_version, len, "%s", DRV_VERS_STRING); |
| 744 | d->slu_unitcfg = cd->slu_unitcfg; |
| 745 | d->app_unitcfg = cd->app_unitcfg; |
| 746 | return 0; |
| 747 | } |
| 748 | |
| 749 | /** |
| 750 | * __genwqe_enqueue_ddcb() - Enqueue a DDCB |
| 751 | * @cd: pointer to genwqe device descriptor |
| 752 | * @req: pointer to DDCB execution request |
| 753 | * |
| 754 | * Return: 0 if enqueuing succeeded |
| 755 | * -EIO if card is unusable/PCIe problems |
| 756 | * -EBUSY if enqueuing failed |
| 757 | */ |
| 758 | int __genwqe_enqueue_ddcb(struct genwqe_dev *cd, struct ddcb_requ *req) |
| 759 | { |
| 760 | struct ddcb *pddcb; |
| 761 | unsigned long flags; |
| 762 | struct ddcb_queue *queue; |
| 763 | struct pci_dev *pci_dev = cd->pci_dev; |
| 764 | u16 icrc; |
| 765 | |
| 766 | if (cd->card_state != GENWQE_CARD_USED) { |
| 767 | printk_ratelimited(KERN_ERR |
| 768 | "%s %s: [%s] Card is unusable/PCIe problem Req#%d\n", |
| 769 | GENWQE_DEVNAME, dev_name(&pci_dev->dev), |
| 770 | __func__, req->num); |
| 771 | return -EIO; |
| 772 | } |
| 773 | |
| 774 | queue = req->queue = &cd->queue; |
| 775 | |
| 776 | /* FIXME circumvention to improve performance when no irq is |
| 777 | * there. |
| 778 | */ |
| 779 | if (genwqe_polling_enabled) |
| 780 | genwqe_check_ddcb_queue(cd, queue); |
| 781 | |
| 782 | /* |
| 783 | * It must be ensured to process all DDCBs in successive |
| 784 | * order. Use a lock here in order to prevent nested DDCB |
| 785 | * enqueuing. |
| 786 | */ |
| 787 | spin_lock_irqsave(&queue->ddcb_lock, flags); |
| 788 | |
| 789 | pddcb = get_next_ddcb(cd, queue, &req->num); /* get ptr and num */ |
| 790 | if (pddcb == NULL) { |
| 791 | spin_unlock_irqrestore(&queue->ddcb_lock, flags); |
| 792 | queue->busy++; |
| 793 | return -EBUSY; |
| 794 | } |
| 795 | |
| 796 | if (queue->ddcb_req[req->num] != NULL) { |
| 797 | spin_unlock_irqrestore(&queue->ddcb_lock, flags); |
| 798 | |
| 799 | dev_err(&pci_dev->dev, |
| 800 | "[%s] picked DDCB %d with req=%p still in use!!\n", |
| 801 | __func__, req->num, req); |
| 802 | return -EFAULT; |
| 803 | } |
| 804 | ddcb_requ_set_state(req, GENWQE_REQU_ENQUEUED); |
| 805 | queue->ddcb_req[req->num] = req; |
| 806 | |
| 807 | pddcb->cmdopts_16 = cpu_to_be16(req->cmd.cmdopts); |
| 808 | pddcb->cmd = req->cmd.cmd; |
| 809 | pddcb->acfunc = req->cmd.acfunc; /* functional unit */ |
| 810 | |
| 811 | /* |
| 812 | * We know that we can get retc 0x104 with CRC error, do not |
| 813 | * stop the queue in those cases for this command. XDIR = 1 |
| 814 | * does not work for old SLU versions. |
| 815 | * |
| 816 | * Last bitstream with the old XDIR behavior had SLU_ID |
| 817 | * 0x34199. |
| 818 | */ |
| 819 | if ((cd->slu_unitcfg & 0xFFFF0ull) > 0x34199ull) |
| 820 | pddcb->xdir = 0x1; |
| 821 | else |
| 822 | pddcb->xdir = 0x0; |
| 823 | |
| 824 | |
| 825 | pddcb->psp = (((req->cmd.asiv_length / 8) << 4) | |
| 826 | ((req->cmd.asv_length / 8))); |
| 827 | pddcb->disp_ts_64 = cpu_to_be64(req->cmd.disp_ts); |
| 828 | |
| 829 | /* |
| 830 | * If copying the whole DDCB_ASIV_LENGTH is impacting |
| 831 | * performance we need to change it to |
| 832 | * req->cmd.asiv_length. But simulation benefits from some |
| 833 | * non-architectured bits behind the architectured content. |
| 834 | * |
| 835 | * How much data is copied depends on the availability of the |
| 836 | * ATS field, which was introduced late. If the ATS field is |
| 837 | * supported ASIV is 8 bytes shorter than it used to be. Since |
| 838 | * the ATS field is copied too, the code should do exactly |
| 839 | * what it did before, but I wanted to make copying of the ATS |
| 840 | * field very explicit. |
| 841 | */ |
| 842 | if (genwqe_get_slu_id(cd) <= 0x2) { |
| 843 | memcpy(&pddcb->__asiv[0], /* destination */ |
| 844 | &req->cmd.__asiv[0], /* source */ |
| 845 | DDCB_ASIV_LENGTH); /* req->cmd.asiv_length */ |
| 846 | } else { |
Frank Haverkamp | 58d66ce | 2013-12-20 16:26:10 +0100 | [diff] [blame] | 847 | pddcb->n.ats_64 = cpu_to_be64(req->cmd.ats); |
| 848 | memcpy(&pddcb->n.asiv[0], /* destination */ |
Frank Haverkamp | eaf4722 | 2013-12-09 13:30:40 +0100 | [diff] [blame] | 849 | &req->cmd.asiv[0], /* source */ |
| 850 | DDCB_ASIV_LENGTH_ATS); /* req->cmd.asiv_length */ |
| 851 | } |
| 852 | |
| 853 | pddcb->icrc_hsi_shi_32 = cpu_to_be32(0x00000000); /* for crc */ |
| 854 | |
| 855 | /* |
| 856 | * Calculate CRC_16 for corresponding range PSP(7:4). Include |
| 857 | * empty 4 bytes prior to the data. |
| 858 | */ |
| 859 | icrc = genwqe_crc16((const u8 *)pddcb, |
| 860 | ICRC_LENGTH(req->cmd.asiv_length), 0xffff); |
| 861 | pddcb->icrc_hsi_shi_32 = cpu_to_be32((u32)icrc << 16); |
| 862 | |
| 863 | /* enable DDCB completion irq */ |
| 864 | if (!genwqe_polling_enabled) |
| 865 | pddcb->icrc_hsi_shi_32 |= DDCB_INTR_BE32; |
| 866 | |
| 867 | dev_dbg(&pci_dev->dev, "INPUT DDCB#%d\n", req->num); |
| 868 | genwqe_hexdump(pci_dev, pddcb, sizeof(*pddcb)); |
| 869 | |
| 870 | if (ddcb_requ_collect_debug_data(req)) { |
| 871 | /* use the kernel copy of debug data. copying back to |
| 872 | user buffer happens later */ |
| 873 | |
| 874 | genwqe_init_debug_data(cd, &req->debug_data); |
| 875 | memcpy(&req->debug_data.ddcb_before, pddcb, |
| 876 | sizeof(req->debug_data.ddcb_before)); |
| 877 | } |
| 878 | |
| 879 | enqueue_ddcb(cd, queue, pddcb, req->num); |
| 880 | queue->ddcbs_in_flight++; |
| 881 | |
| 882 | if (queue->ddcbs_in_flight > queue->ddcbs_max_in_flight) |
| 883 | queue->ddcbs_max_in_flight = queue->ddcbs_in_flight; |
| 884 | |
| 885 | ddcb_requ_set_state(req, GENWQE_REQU_TAPPED); |
| 886 | spin_unlock_irqrestore(&queue->ddcb_lock, flags); |
| 887 | wake_up_interruptible(&cd->queue_waitq); |
| 888 | |
| 889 | return 0; |
| 890 | } |
| 891 | |
| 892 | /** |
| 893 | * __genwqe_execute_raw_ddcb() - Setup and execute DDCB |
| 894 | * @cd: pointer to genwqe device descriptor |
| 895 | * @req: user provided DDCB request |
| 896 | */ |
| 897 | int __genwqe_execute_raw_ddcb(struct genwqe_dev *cd, |
| 898 | struct genwqe_ddcb_cmd *cmd) |
| 899 | { |
| 900 | int rc = 0; |
| 901 | struct pci_dev *pci_dev = cd->pci_dev; |
| 902 | struct ddcb_requ *req = container_of(cmd, struct ddcb_requ, cmd); |
| 903 | |
| 904 | if (cmd->asiv_length > DDCB_ASIV_LENGTH) { |
| 905 | dev_err(&pci_dev->dev, "[%s] err: wrong asiv_length of %d\n", |
| 906 | __func__, cmd->asiv_length); |
| 907 | return -EINVAL; |
| 908 | } |
| 909 | if (cmd->asv_length > DDCB_ASV_LENGTH) { |
| 910 | dev_err(&pci_dev->dev, "[%s] err: wrong asv_length of %d\n", |
| 911 | __func__, cmd->asiv_length); |
| 912 | return -EINVAL; |
| 913 | } |
| 914 | rc = __genwqe_enqueue_ddcb(cd, req); |
| 915 | if (rc != 0) |
| 916 | return rc; |
| 917 | |
| 918 | rc = __genwqe_wait_ddcb(cd, req); |
| 919 | if (rc < 0) /* error or signal interrupt */ |
| 920 | goto err_exit; |
| 921 | |
| 922 | if (ddcb_requ_collect_debug_data(req)) { |
Frank Haverkamp | 58d66ce | 2013-12-20 16:26:10 +0100 | [diff] [blame] | 923 | if (copy_to_user((struct genwqe_debug_data __user *) |
| 924 | (unsigned long)cmd->ddata_addr, |
Frank Haverkamp | eaf4722 | 2013-12-09 13:30:40 +0100 | [diff] [blame] | 925 | &req->debug_data, |
| 926 | sizeof(struct genwqe_debug_data))) |
| 927 | return -EFAULT; |
| 928 | } |
| 929 | |
| 930 | /* |
| 931 | * Higher values than 0x102 indicate completion with faults, |
| 932 | * lower values than 0x102 indicate processing faults. Note |
| 933 | * that DDCB might have been purged. E.g. Cntl+C. |
| 934 | */ |
| 935 | if (cmd->retc != DDCB_RETC_COMPLETE) { |
| 936 | /* This might happen e.g. flash read, and needs to be |
| 937 | handled by the upper layer code. */ |
| 938 | rc = -EBADMSG; /* not processed/error retc */ |
| 939 | } |
| 940 | |
| 941 | return rc; |
| 942 | |
| 943 | err_exit: |
| 944 | __genwqe_purge_ddcb(cd, req); |
| 945 | |
| 946 | if (ddcb_requ_collect_debug_data(req)) { |
Frank Haverkamp | 58d66ce | 2013-12-20 16:26:10 +0100 | [diff] [blame] | 947 | if (copy_to_user((struct genwqe_debug_data __user *) |
| 948 | (unsigned long)cmd->ddata_addr, |
Frank Haverkamp | eaf4722 | 2013-12-09 13:30:40 +0100 | [diff] [blame] | 949 | &req->debug_data, |
| 950 | sizeof(struct genwqe_debug_data))) |
| 951 | return -EFAULT; |
| 952 | } |
| 953 | return rc; |
| 954 | } |
| 955 | |
| 956 | /** |
| 957 | * genwqe_next_ddcb_ready() - Figure out if the next DDCB is already finished |
| 958 | * |
| 959 | * We use this as condition for our wait-queue code. |
| 960 | */ |
| 961 | static int genwqe_next_ddcb_ready(struct genwqe_dev *cd) |
| 962 | { |
| 963 | unsigned long flags; |
| 964 | struct ddcb *pddcb; |
| 965 | struct ddcb_queue *queue = &cd->queue; |
| 966 | |
| 967 | spin_lock_irqsave(&queue->ddcb_lock, flags); |
| 968 | |
| 969 | if (queue_empty(queue)) { /* emtpy queue */ |
| 970 | spin_unlock_irqrestore(&queue->ddcb_lock, flags); |
| 971 | return 0; |
| 972 | } |
| 973 | |
| 974 | pddcb = &queue->ddcb_vaddr[queue->ddcb_act]; |
| 975 | if (pddcb->icrc_hsi_shi_32 & DDCB_COMPLETED_BE32) { /* ddcb ready */ |
| 976 | spin_unlock_irqrestore(&queue->ddcb_lock, flags); |
| 977 | return 1; |
| 978 | } |
| 979 | |
| 980 | spin_unlock_irqrestore(&queue->ddcb_lock, flags); |
| 981 | return 0; |
| 982 | } |
| 983 | |
| 984 | /** |
| 985 | * genwqe_ddcbs_in_flight() - Check how many DDCBs are in flight |
| 986 | * |
| 987 | * Keep track on the number of DDCBs which ware currently in the |
| 988 | * queue. This is needed for statistics as well as conditon if we want |
| 989 | * to wait or better do polling in case of no interrupts available. |
| 990 | */ |
| 991 | int genwqe_ddcbs_in_flight(struct genwqe_dev *cd) |
| 992 | { |
| 993 | unsigned long flags; |
| 994 | int ddcbs_in_flight = 0; |
| 995 | struct ddcb_queue *queue = &cd->queue; |
| 996 | |
| 997 | spin_lock_irqsave(&queue->ddcb_lock, flags); |
| 998 | ddcbs_in_flight += queue->ddcbs_in_flight; |
| 999 | spin_unlock_irqrestore(&queue->ddcb_lock, flags); |
| 1000 | |
| 1001 | return ddcbs_in_flight; |
| 1002 | } |
| 1003 | |
| 1004 | static int setup_ddcb_queue(struct genwqe_dev *cd, struct ddcb_queue *queue) |
| 1005 | { |
| 1006 | int rc, i; |
| 1007 | struct ddcb *pddcb; |
| 1008 | u64 val64; |
| 1009 | unsigned int queue_size; |
| 1010 | struct pci_dev *pci_dev = cd->pci_dev; |
| 1011 | |
| 1012 | if (genwqe_ddcb_max < 2) |
| 1013 | return -EINVAL; |
| 1014 | |
| 1015 | queue_size = roundup(genwqe_ddcb_max * sizeof(struct ddcb), PAGE_SIZE); |
| 1016 | |
| 1017 | queue->ddcbs_in_flight = 0; /* statistics */ |
| 1018 | queue->ddcbs_max_in_flight = 0; |
| 1019 | queue->ddcbs_completed = 0; |
| 1020 | queue->busy = 0; |
| 1021 | |
| 1022 | queue->ddcb_seq = 0x100; /* start sequence number */ |
| 1023 | queue->ddcb_max = genwqe_ddcb_max; /* module parameter */ |
| 1024 | queue->ddcb_vaddr = __genwqe_alloc_consistent(cd, queue_size, |
| 1025 | &queue->ddcb_daddr); |
| 1026 | if (queue->ddcb_vaddr == NULL) { |
| 1027 | dev_err(&pci_dev->dev, |
| 1028 | "[%s] **err: could not allocate DDCB **\n", __func__); |
| 1029 | return -ENOMEM; |
| 1030 | } |
| 1031 | memset(queue->ddcb_vaddr, 0, queue_size); |
| 1032 | |
| 1033 | queue->ddcb_req = kzalloc(sizeof(struct ddcb_requ *) * |
| 1034 | queue->ddcb_max, GFP_KERNEL); |
| 1035 | if (!queue->ddcb_req) { |
| 1036 | rc = -ENOMEM; |
| 1037 | goto free_ddcbs; |
| 1038 | } |
| 1039 | |
| 1040 | queue->ddcb_waitqs = kzalloc(sizeof(wait_queue_head_t) * |
| 1041 | queue->ddcb_max, GFP_KERNEL); |
| 1042 | if (!queue->ddcb_waitqs) { |
| 1043 | rc = -ENOMEM; |
| 1044 | goto free_requs; |
| 1045 | } |
| 1046 | |
| 1047 | for (i = 0; i < queue->ddcb_max; i++) { |
| 1048 | pddcb = &queue->ddcb_vaddr[i]; /* DDCBs */ |
| 1049 | pddcb->icrc_hsi_shi_32 = DDCB_COMPLETED_BE32; |
| 1050 | pddcb->retc_16 = cpu_to_be16(0xfff); |
| 1051 | |
| 1052 | queue->ddcb_req[i] = NULL; /* requests */ |
| 1053 | init_waitqueue_head(&queue->ddcb_waitqs[i]); /* waitqueues */ |
| 1054 | } |
| 1055 | |
| 1056 | queue->ddcb_act = 0; |
| 1057 | queue->ddcb_next = 0; /* queue is empty */ |
| 1058 | |
| 1059 | spin_lock_init(&queue->ddcb_lock); |
| 1060 | init_waitqueue_head(&queue->ddcb_waitq); |
| 1061 | |
| 1062 | val64 = ((u64)(queue->ddcb_max - 1) << 8); /* lastptr */ |
| 1063 | __genwqe_writeq(cd, queue->IO_QUEUE_CONFIG, 0x07); /* iCRC/vCRC */ |
| 1064 | __genwqe_writeq(cd, queue->IO_QUEUE_SEGMENT, queue->ddcb_daddr); |
| 1065 | __genwqe_writeq(cd, queue->IO_QUEUE_INITSQN, queue->ddcb_seq); |
| 1066 | __genwqe_writeq(cd, queue->IO_QUEUE_WRAP, val64); |
| 1067 | return 0; |
| 1068 | |
| 1069 | free_requs: |
| 1070 | kfree(queue->ddcb_req); |
| 1071 | queue->ddcb_req = NULL; |
| 1072 | free_ddcbs: |
| 1073 | __genwqe_free_consistent(cd, queue_size, queue->ddcb_vaddr, |
| 1074 | queue->ddcb_daddr); |
| 1075 | queue->ddcb_vaddr = NULL; |
| 1076 | queue->ddcb_daddr = 0ull; |
| 1077 | return -ENODEV; |
| 1078 | |
| 1079 | } |
| 1080 | |
| 1081 | static int ddcb_queue_initialized(struct ddcb_queue *queue) |
| 1082 | { |
| 1083 | return queue->ddcb_vaddr != NULL; |
| 1084 | } |
| 1085 | |
| 1086 | static void free_ddcb_queue(struct genwqe_dev *cd, struct ddcb_queue *queue) |
| 1087 | { |
| 1088 | unsigned int queue_size; |
| 1089 | |
| 1090 | queue_size = roundup(queue->ddcb_max * sizeof(struct ddcb), PAGE_SIZE); |
| 1091 | |
| 1092 | kfree(queue->ddcb_req); |
| 1093 | queue->ddcb_req = NULL; |
| 1094 | |
| 1095 | if (queue->ddcb_vaddr) { |
| 1096 | __genwqe_free_consistent(cd, queue_size, queue->ddcb_vaddr, |
| 1097 | queue->ddcb_daddr); |
| 1098 | queue->ddcb_vaddr = NULL; |
| 1099 | queue->ddcb_daddr = 0ull; |
| 1100 | } |
| 1101 | } |
| 1102 | |
| 1103 | static irqreturn_t genwqe_pf_isr(int irq, void *dev_id) |
| 1104 | { |
| 1105 | u64 gfir; |
| 1106 | struct genwqe_dev *cd = (struct genwqe_dev *)dev_id; |
| 1107 | struct pci_dev *pci_dev = cd->pci_dev; |
| 1108 | |
| 1109 | /* |
| 1110 | * In case of fatal FIR error the queue is stopped, such that |
| 1111 | * we can safely check it without risking anything. |
| 1112 | */ |
| 1113 | cd->irqs_processed++; |
| 1114 | wake_up_interruptible(&cd->queue_waitq); |
| 1115 | |
| 1116 | /* |
| 1117 | * Checking for errors before kicking the queue might be |
| 1118 | * safer, but slower for the good-case ... See above. |
| 1119 | */ |
| 1120 | gfir = __genwqe_readq(cd, IO_SLC_CFGREG_GFIR); |
| 1121 | if ((gfir & GFIR_ERR_TRIGGER) != 0x0) { |
| 1122 | |
| 1123 | wake_up_interruptible(&cd->health_waitq); |
| 1124 | |
| 1125 | /* |
| 1126 | * By default GFIRs causes recovery actions. This |
| 1127 | * count is just for debug when recovery is masked. |
| 1128 | */ |
| 1129 | printk_ratelimited(KERN_ERR |
| 1130 | "%s %s: [%s] GFIR=%016llx\n", |
| 1131 | GENWQE_DEVNAME, dev_name(&pci_dev->dev), |
| 1132 | __func__, gfir); |
| 1133 | } |
| 1134 | |
| 1135 | return IRQ_HANDLED; |
| 1136 | } |
| 1137 | |
| 1138 | static irqreturn_t genwqe_vf_isr(int irq, void *dev_id) |
| 1139 | { |
| 1140 | struct genwqe_dev *cd = (struct genwqe_dev *)dev_id; |
| 1141 | |
| 1142 | cd->irqs_processed++; |
| 1143 | wake_up_interruptible(&cd->queue_waitq); |
| 1144 | |
| 1145 | return IRQ_HANDLED; |
| 1146 | } |
| 1147 | |
| 1148 | /** |
| 1149 | * genwqe_card_thread() - Work thread for the DDCB queue |
| 1150 | * |
| 1151 | * The idea is to check if there are DDCBs in processing. If there are |
| 1152 | * some finished DDCBs, we process them and wakeup the |
| 1153 | * requestors. Otherwise we give other processes time using |
| 1154 | * cond_resched(). |
| 1155 | */ |
| 1156 | static int genwqe_card_thread(void *data) |
| 1157 | { |
| 1158 | int should_stop = 0, rc = 0; |
| 1159 | struct genwqe_dev *cd = (struct genwqe_dev *)data; |
| 1160 | |
| 1161 | while (!kthread_should_stop()) { |
| 1162 | |
| 1163 | genwqe_check_ddcb_queue(cd, &cd->queue); |
| 1164 | |
| 1165 | if (genwqe_polling_enabled) { |
| 1166 | rc = wait_event_interruptible_timeout( |
| 1167 | cd->queue_waitq, |
| 1168 | genwqe_ddcbs_in_flight(cd) || |
| 1169 | (should_stop = kthread_should_stop()), 1); |
| 1170 | } else { |
| 1171 | rc = wait_event_interruptible_timeout( |
| 1172 | cd->queue_waitq, |
| 1173 | genwqe_next_ddcb_ready(cd) || |
| 1174 | (should_stop = kthread_should_stop()), HZ); |
| 1175 | } |
| 1176 | if (should_stop) |
| 1177 | break; |
| 1178 | |
| 1179 | /* |
| 1180 | * Avoid soft lockups on heavy loads; we do not want |
| 1181 | * to disable our interrupts. |
| 1182 | */ |
| 1183 | cond_resched(); |
| 1184 | } |
| 1185 | return 0; |
| 1186 | } |
| 1187 | |
| 1188 | /** |
| 1189 | * genwqe_setup_service_layer() - Setup DDCB queue |
| 1190 | * @cd: pointer to genwqe device descriptor |
| 1191 | * |
| 1192 | * Allocate DDCBs. Configure Service Layer Controller (SLC). |
| 1193 | * |
| 1194 | * Return: 0 success |
| 1195 | */ |
| 1196 | int genwqe_setup_service_layer(struct genwqe_dev *cd) |
| 1197 | { |
| 1198 | int rc; |
| 1199 | struct ddcb_queue *queue; |
| 1200 | struct pci_dev *pci_dev = cd->pci_dev; |
| 1201 | |
| 1202 | if (genwqe_is_privileged(cd)) { |
| 1203 | rc = genwqe_card_reset(cd); |
| 1204 | if (rc < 0) { |
| 1205 | dev_err(&pci_dev->dev, |
| 1206 | "[%s] err: reset failed.\n", __func__); |
| 1207 | return rc; |
| 1208 | } |
| 1209 | genwqe_read_softreset(cd); |
| 1210 | } |
| 1211 | |
| 1212 | queue = &cd->queue; |
| 1213 | queue->IO_QUEUE_CONFIG = IO_SLC_QUEUE_CONFIG; |
| 1214 | queue->IO_QUEUE_STATUS = IO_SLC_QUEUE_STATUS; |
| 1215 | queue->IO_QUEUE_SEGMENT = IO_SLC_QUEUE_SEGMENT; |
| 1216 | queue->IO_QUEUE_INITSQN = IO_SLC_QUEUE_INITSQN; |
| 1217 | queue->IO_QUEUE_OFFSET = IO_SLC_QUEUE_OFFSET; |
| 1218 | queue->IO_QUEUE_WRAP = IO_SLC_QUEUE_WRAP; |
| 1219 | queue->IO_QUEUE_WTIME = IO_SLC_QUEUE_WTIME; |
| 1220 | queue->IO_QUEUE_ERRCNTS = IO_SLC_QUEUE_ERRCNTS; |
| 1221 | queue->IO_QUEUE_LRW = IO_SLC_QUEUE_LRW; |
| 1222 | |
| 1223 | rc = setup_ddcb_queue(cd, queue); |
| 1224 | if (rc != 0) { |
| 1225 | rc = -ENODEV; |
| 1226 | goto err_out; |
| 1227 | } |
| 1228 | |
| 1229 | init_waitqueue_head(&cd->queue_waitq); |
| 1230 | cd->card_thread = kthread_run(genwqe_card_thread, cd, |
| 1231 | GENWQE_DEVNAME "%d_thread", |
| 1232 | cd->card_idx); |
| 1233 | if (IS_ERR(cd->card_thread)) { |
| 1234 | rc = PTR_ERR(cd->card_thread); |
| 1235 | cd->card_thread = NULL; |
| 1236 | goto stop_free_queue; |
| 1237 | } |
| 1238 | |
| 1239 | rc = genwqe_set_interrupt_capability(cd, GENWQE_MSI_IRQS); |
| 1240 | if (rc > 0) |
| 1241 | rc = genwqe_set_interrupt_capability(cd, rc); |
| 1242 | if (rc != 0) { |
| 1243 | rc = -ENODEV; |
| 1244 | goto stop_kthread; |
| 1245 | } |
| 1246 | |
| 1247 | /* |
| 1248 | * We must have all wait-queues initialized when we enable the |
| 1249 | * interrupts. Otherwise we might crash if we get an early |
| 1250 | * irq. |
| 1251 | */ |
| 1252 | init_waitqueue_head(&cd->health_waitq); |
| 1253 | |
| 1254 | if (genwqe_is_privileged(cd)) { |
| 1255 | rc = request_irq(pci_dev->irq, genwqe_pf_isr, IRQF_SHARED, |
| 1256 | GENWQE_DEVNAME, cd); |
| 1257 | } else { |
| 1258 | rc = request_irq(pci_dev->irq, genwqe_vf_isr, IRQF_SHARED, |
| 1259 | GENWQE_DEVNAME, cd); |
| 1260 | } |
| 1261 | if (rc < 0) { |
| 1262 | dev_err(&pci_dev->dev, "irq %d not free.\n", pci_dev->irq); |
| 1263 | goto stop_irq_cap; |
| 1264 | } |
| 1265 | |
| 1266 | cd->card_state = GENWQE_CARD_USED; |
| 1267 | return 0; |
| 1268 | |
| 1269 | stop_irq_cap: |
| 1270 | genwqe_reset_interrupt_capability(cd); |
| 1271 | stop_kthread: |
| 1272 | kthread_stop(cd->card_thread); |
| 1273 | cd->card_thread = NULL; |
| 1274 | stop_free_queue: |
| 1275 | free_ddcb_queue(cd, queue); |
| 1276 | err_out: |
| 1277 | return rc; |
| 1278 | } |
| 1279 | |
| 1280 | /** |
| 1281 | * queue_wake_up_all() - Handles fatal error case |
| 1282 | * |
| 1283 | * The PCI device got unusable and we have to stop all pending |
| 1284 | * requests as fast as we can. The code after this must purge the |
| 1285 | * DDCBs in question and ensure that all mappings are freed. |
| 1286 | */ |
| 1287 | static int queue_wake_up_all(struct genwqe_dev *cd) |
| 1288 | { |
| 1289 | unsigned int i; |
| 1290 | unsigned long flags; |
| 1291 | struct ddcb_queue *queue = &cd->queue; |
| 1292 | |
| 1293 | spin_lock_irqsave(&queue->ddcb_lock, flags); |
| 1294 | |
| 1295 | for (i = 0; i < queue->ddcb_max; i++) |
| 1296 | wake_up_interruptible(&queue->ddcb_waitqs[queue->ddcb_act]); |
| 1297 | |
| 1298 | spin_unlock_irqrestore(&queue->ddcb_lock, flags); |
| 1299 | |
| 1300 | return 0; |
| 1301 | } |
| 1302 | |
| 1303 | /** |
| 1304 | * genwqe_finish_queue() - Remove any genwqe devices and user-interfaces |
| 1305 | * |
| 1306 | * Relies on the pre-condition that there are no users of the card |
| 1307 | * device anymore e.g. with open file-descriptors. |
| 1308 | * |
| 1309 | * This function must be robust enough to be called twice. |
| 1310 | */ |
| 1311 | int genwqe_finish_queue(struct genwqe_dev *cd) |
| 1312 | { |
Colin Ian King | ebb2c96 | 2014-03-20 15:11:04 +0100 | [diff] [blame] | 1313 | int i, rc = 0, in_flight; |
Frank Haverkamp | eaf4722 | 2013-12-09 13:30:40 +0100 | [diff] [blame] | 1314 | int waitmax = genwqe_ddcb_software_timeout; |
| 1315 | struct pci_dev *pci_dev = cd->pci_dev; |
| 1316 | struct ddcb_queue *queue = &cd->queue; |
| 1317 | |
| 1318 | if (!ddcb_queue_initialized(queue)) |
| 1319 | return 0; |
| 1320 | |
| 1321 | /* Do not wipe out the error state. */ |
| 1322 | if (cd->card_state == GENWQE_CARD_USED) |
| 1323 | cd->card_state = GENWQE_CARD_UNUSED; |
| 1324 | |
| 1325 | /* Wake up all requests in the DDCB queue such that they |
| 1326 | should be removed nicely. */ |
| 1327 | queue_wake_up_all(cd); |
| 1328 | |
| 1329 | /* We must wait to get rid of the DDCBs in flight */ |
| 1330 | for (i = 0; i < waitmax; i++) { |
| 1331 | in_flight = genwqe_ddcbs_in_flight(cd); |
| 1332 | |
| 1333 | if (in_flight == 0) |
| 1334 | break; |
| 1335 | |
| 1336 | dev_dbg(&pci_dev->dev, |
| 1337 | " DEBUG [%d/%d] waiting for queue to get empty: " |
| 1338 | "%d requests!\n", i, waitmax, in_flight); |
| 1339 | |
| 1340 | /* |
| 1341 | * Severe severe error situation: The card itself has |
| 1342 | * 16 DDCB queues, each queue has e.g. 32 entries, |
| 1343 | * each DDBC has a hardware timeout of currently 250 |
| 1344 | * msec but the PFs have a hardware timeout of 8 sec |
| 1345 | * ... so I take something large. |
| 1346 | */ |
| 1347 | msleep(1000); |
| 1348 | } |
| 1349 | if (i == waitmax) { |
| 1350 | dev_err(&pci_dev->dev, " [%s] err: queue is not empty!!\n", |
| 1351 | __func__); |
| 1352 | rc = -EIO; |
| 1353 | } |
| 1354 | return rc; |
| 1355 | } |
| 1356 | |
| 1357 | /** |
| 1358 | * genwqe_release_service_layer() - Shutdown DDCB queue |
| 1359 | * @cd: genwqe device descriptor |
| 1360 | * |
| 1361 | * This function must be robust enough to be called twice. |
| 1362 | */ |
| 1363 | int genwqe_release_service_layer(struct genwqe_dev *cd) |
| 1364 | { |
| 1365 | struct pci_dev *pci_dev = cd->pci_dev; |
| 1366 | |
| 1367 | if (!ddcb_queue_initialized(&cd->queue)) |
| 1368 | return 1; |
| 1369 | |
| 1370 | free_irq(pci_dev->irq, cd); |
| 1371 | genwqe_reset_interrupt_capability(cd); |
| 1372 | |
| 1373 | if (cd->card_thread != NULL) { |
| 1374 | kthread_stop(cd->card_thread); |
| 1375 | cd->card_thread = NULL; |
| 1376 | } |
| 1377 | |
| 1378 | free_ddcb_queue(cd, &cd->queue); |
| 1379 | return 0; |
| 1380 | } |