blob: 353ee0cc733d8b7773f70393b92c24f9c9b02e56 [file] [log] [blame]
Frank Haverkampeaf47222013-12-09 13:30:40 +01001/**
2 * IBM Accelerator Family 'GenWQE'
3 *
4 * (C) Copyright IBM Corp. 2013
5 *
6 * Author: Frank Haverkamp <haver@linux.vnet.ibm.com>
7 * Author: Joerg-Stephan Vogt <jsvogt@de.ibm.com>
Frank Haverkamp26d8f6f2014-09-10 16:37:48 +02008 * Author: Michael Jung <mijung@gmx.net>
Frank Haverkampeaf47222013-12-09 13:30:40 +01009 * Author: Michael Ruettger <michael@ibmra.de>
10 *
11 * This program is free software; you can redistribute it and/or modify
12 * it under the terms of the GNU General Public License (version 2 only)
13 * as published by the Free Software Foundation.
14 *
15 * This program is distributed in the hope that it will be useful,
16 * but WITHOUT ANY WARRANTY; without even the implied warranty of
17 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18 * GNU General Public License for more details.
19 */
20
21/*
22 * Device Driver Control Block (DDCB) queue support. Definition of
23 * interrupt handlers for queue support as well as triggering the
24 * health monitor code in case of problems. The current hardware uses
25 * an MSI interrupt which is shared between error handling and
26 * functional code.
27 */
28
29#include <linux/types.h>
30#include <linux/module.h>
31#include <linux/sched.h>
32#include <linux/wait.h>
33#include <linux/pci.h>
34#include <linux/string.h>
35#include <linux/dma-mapping.h>
36#include <linux/delay.h>
37#include <linux/module.h>
38#include <linux/interrupt.h>
39#include <linux/crc-itu-t.h>
40
Frank Haverkamp90b4e972014-01-07 15:41:24 +010041#include "card_base.h"
Frank Haverkampeaf47222013-12-09 13:30:40 +010042#include "card_ddcb.h"
43
44/*
45 * N: next DDCB, this is where the next DDCB will be put.
46 * A: active DDCB, this is where the code will look for the next completion.
47 * x: DDCB is enqueued, we are waiting for its completion.
48
49 * Situation (1): Empty queue
50 * +---+---+---+---+---+---+---+---+
51 * | 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 |
52 * | | | | | | | | |
53 * +---+---+---+---+---+---+---+---+
54 * A/N
55 * enqueued_ddcbs = A - N = 2 - 2 = 0
56 *
57 * Situation (2): Wrapped, N > A
58 * +---+---+---+---+---+---+---+---+
59 * | 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 |
60 * | | | x | x | | | | |
61 * +---+---+---+---+---+---+---+---+
62 * A N
63 * enqueued_ddcbs = N - A = 4 - 2 = 2
64 *
65 * Situation (3): Queue wrapped, A > N
66 * +---+---+---+---+---+---+---+---+
67 * | 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 |
68 * | x | x | | | x | x | x | x |
69 * +---+---+---+---+---+---+---+---+
70 * N A
71 * enqueued_ddcbs = queue_max - (A - N) = 8 - (4 - 2) = 6
72 *
73 * Situation (4a): Queue full N > A
74 * +---+---+---+---+---+---+---+---+
75 * | 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 |
76 * | x | x | x | x | x | x | x | |
77 * +---+---+---+---+---+---+---+---+
78 * A N
79 *
80 * enqueued_ddcbs = N - A = 7 - 0 = 7
81 *
82 * Situation (4a): Queue full A > N
83 * +---+---+---+---+---+---+---+---+
84 * | 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 |
85 * | x | x | x | | x | x | x | x |
86 * +---+---+---+---+---+---+---+---+
87 * N A
88 * enqueued_ddcbs = queue_max - (A - N) = 8 - (4 - 3) = 7
89 */
90
91static int queue_empty(struct ddcb_queue *queue)
92{
93 return queue->ddcb_next == queue->ddcb_act;
94}
95
96static int queue_enqueued_ddcbs(struct ddcb_queue *queue)
97{
98 if (queue->ddcb_next >= queue->ddcb_act)
99 return queue->ddcb_next - queue->ddcb_act;
100
101 return queue->ddcb_max - (queue->ddcb_act - queue->ddcb_next);
102}
103
104static int queue_free_ddcbs(struct ddcb_queue *queue)
105{
106 int free_ddcbs = queue->ddcb_max - queue_enqueued_ddcbs(queue) - 1;
107
108 if (WARN_ON_ONCE(free_ddcbs < 0)) { /* must never ever happen! */
109 return 0;
110 }
111 return free_ddcbs;
112}
113
114/*
115 * Use of the PRIV field in the DDCB for queue debugging:
116 *
117 * (1) Trying to get rid of a DDCB which saw a timeout:
118 * pddcb->priv[6] = 0xcc; # cleared
119 *
120 * (2) Append a DDCB via NEXT bit:
121 * pddcb->priv[7] = 0xaa; # appended
122 *
123 * (3) DDCB needed tapping:
124 * pddcb->priv[7] = 0xbb; # tapped
125 *
126 * (4) DDCB marked as correctly finished:
127 * pddcb->priv[6] = 0xff; # finished
128 */
129
130static inline void ddcb_mark_tapped(struct ddcb *pddcb)
131{
132 pddcb->priv[7] = 0xbb; /* tapped */
133}
134
135static inline void ddcb_mark_appended(struct ddcb *pddcb)
136{
137 pddcb->priv[7] = 0xaa; /* appended */
138}
139
140static inline void ddcb_mark_cleared(struct ddcb *pddcb)
141{
142 pddcb->priv[6] = 0xcc; /* cleared */
143}
144
145static inline void ddcb_mark_finished(struct ddcb *pddcb)
146{
147 pddcb->priv[6] = 0xff; /* finished */
148}
149
150static inline void ddcb_mark_unused(struct ddcb *pddcb)
151{
152 pddcb->priv_64 = cpu_to_be64(0); /* not tapped */
153}
154
155/**
156 * genwqe_crc16() - Generate 16-bit crc as required for DDCBs
157 * @buff: pointer to data buffer
158 * @len: length of data for calculation
159 * @init: initial crc (0xffff at start)
160 *
161 * Polynomial = x^16 + x^12 + x^5 + 1 (0x1021)
162 * Example: 4 bytes 0x01 0x02 0x03 0x04 with init = 0xffff
163 * should result in a crc16 of 0x89c3
164 *
165 * Return: crc16 checksum in big endian format !
166 */
167static inline u16 genwqe_crc16(const u8 *buff, size_t len, u16 init)
168{
169 return crc_itu_t(init, buff, len);
170}
171
172static void print_ddcb_info(struct genwqe_dev *cd, struct ddcb_queue *queue)
173{
174 int i;
175 struct ddcb *pddcb;
176 unsigned long flags;
177 struct pci_dev *pci_dev = cd->pci_dev;
178
179 spin_lock_irqsave(&cd->print_lock, flags);
180
181 dev_info(&pci_dev->dev,
182 "DDCB list for card #%d (ddcb_act=%d / ddcb_next=%d):\n",
183 cd->card_idx, queue->ddcb_act, queue->ddcb_next);
184
185 pddcb = queue->ddcb_vaddr;
186 for (i = 0; i < queue->ddcb_max; i++) {
187 dev_err(&pci_dev->dev,
Frank Haverkampd9c11d42014-09-10 16:37:51 +0200188 " %c %-3d: RETC=%03x SEQ=%04x HSI=%02X SHI=%02x PRIV=%06llx CMD=%03x\n",
Frank Haverkampeaf47222013-12-09 13:30:40 +0100189 i == queue->ddcb_act ? '>' : ' ',
190 i,
191 be16_to_cpu(pddcb->retc_16),
192 be16_to_cpu(pddcb->seqnum_16),
193 pddcb->hsi,
194 pddcb->shi,
195 be64_to_cpu(pddcb->priv_64),
196 pddcb->cmd);
197 pddcb++;
198 }
199 spin_unlock_irqrestore(&cd->print_lock, flags);
200}
201
202struct genwqe_ddcb_cmd *ddcb_requ_alloc(void)
203{
204 struct ddcb_requ *req;
205
Sebastian Ott19f77672015-09-29 18:23:50 +0200206 req = kzalloc(sizeof(*req), GFP_KERNEL);
Frank Haverkampeaf47222013-12-09 13:30:40 +0100207 if (!req)
208 return NULL;
209
210 return &req->cmd;
211}
212
213void ddcb_requ_free(struct genwqe_ddcb_cmd *cmd)
214{
215 struct ddcb_requ *req = container_of(cmd, struct ddcb_requ, cmd);
Frank Haverkampd9c11d42014-09-10 16:37:51 +0200216
Frank Haverkampeaf47222013-12-09 13:30:40 +0100217 kfree(req);
218}
219
220static inline enum genwqe_requ_state ddcb_requ_get_state(struct ddcb_requ *req)
221{
222 return req->req_state;
223}
224
225static inline void ddcb_requ_set_state(struct ddcb_requ *req,
226 enum genwqe_requ_state new_state)
227{
228 req->req_state = new_state;
229}
230
231static inline int ddcb_requ_collect_debug_data(struct ddcb_requ *req)
232{
233 return req->cmd.ddata_addr != 0x0;
234}
235
236/**
237 * ddcb_requ_finished() - Returns the hardware state of the associated DDCB
238 * @cd: pointer to genwqe device descriptor
239 * @req: DDCB work request
240 *
241 * Status of ddcb_requ mirrors this hardware state, but is copied in
242 * the ddcb_requ on interrupt/polling function. The lowlevel code
243 * should check the hardware state directly, the higher level code
244 * should check the copy.
245 *
246 * This function will also return true if the state of the queue is
247 * not GENWQE_CARD_USED. This enables us to purge all DDCBs in the
248 * shutdown case.
249 */
250static int ddcb_requ_finished(struct genwqe_dev *cd, struct ddcb_requ *req)
251{
252 return (ddcb_requ_get_state(req) == GENWQE_REQU_FINISHED) ||
253 (cd->card_state != GENWQE_CARD_USED);
254}
255
256/**
257 * enqueue_ddcb() - Enqueue a DDCB
258 * @cd: pointer to genwqe device descriptor
259 * @queue: queue this operation should be done on
260 * @ddcb_no: pointer to ddcb number being tapped
261 *
262 * Start execution of DDCB by tapping or append to queue via NEXT
263 * bit. This is done by an atomic 'compare and swap' instruction and
264 * checking SHI and HSI of the previous DDCB.
265 *
266 * This function must only be called with ddcb_lock held.
267 *
268 * Return: 1 if new DDCB is appended to previous
269 * 2 if DDCB queue is tapped via register/simulation
270 */
271#define RET_DDCB_APPENDED 1
272#define RET_DDCB_TAPPED 2
273
274static int enqueue_ddcb(struct genwqe_dev *cd, struct ddcb_queue *queue,
275 struct ddcb *pddcb, int ddcb_no)
276{
277 unsigned int try;
278 int prev_no;
279 struct ddcb *prev_ddcb;
Frank Haverkamp58d66ce2013-12-20 16:26:10 +0100280 __be32 old, new, icrc_hsi_shi;
Frank Haverkampeaf47222013-12-09 13:30:40 +0100281 u64 num;
282
283 /*
284 * For performance checks a Dispatch Timestamp can be put into
285 * DDCB It is supposed to use the SLU's free running counter,
286 * but this requires PCIe cycles.
287 */
288 ddcb_mark_unused(pddcb);
289
290 /* check previous DDCB if already fetched */
291 prev_no = (ddcb_no == 0) ? queue->ddcb_max - 1 : ddcb_no - 1;
292 prev_ddcb = &queue->ddcb_vaddr[prev_no];
293
294 /*
295 * It might have happened that the HSI.FETCHED bit is
296 * set. Retry in this case. Therefore I expect maximum 2 times
297 * trying.
298 */
299 ddcb_mark_appended(pddcb);
300 for (try = 0; try < 2; try++) {
301 old = prev_ddcb->icrc_hsi_shi_32; /* read SHI/HSI in BE32 */
302
303 /* try to append via NEXT bit if prev DDCB is not completed */
304 if ((old & DDCB_COMPLETED_BE32) != 0x00000000)
305 break;
306
307 new = (old | DDCB_NEXT_BE32);
Frank Haverkamp68fe8ac2014-03-20 15:11:03 +0100308
Frank Haverkampd9c11d42014-09-10 16:37:51 +0200309 wmb(); /* need to ensure write ordering */
Frank Haverkampeaf47222013-12-09 13:30:40 +0100310 icrc_hsi_shi = cmpxchg(&prev_ddcb->icrc_hsi_shi_32, old, new);
311
312 if (icrc_hsi_shi == old)
313 return RET_DDCB_APPENDED; /* appended to queue */
314 }
315
316 /* Queue must be re-started by updating QUEUE_OFFSET */
317 ddcb_mark_tapped(pddcb);
318 num = (u64)ddcb_no << 8;
Frank Haverkamp68fe8ac2014-03-20 15:11:03 +0100319
Frank Haverkampd9c11d42014-09-10 16:37:51 +0200320 wmb(); /* need to ensure write ordering */
Frank Haverkampeaf47222013-12-09 13:30:40 +0100321 __genwqe_writeq(cd, queue->IO_QUEUE_OFFSET, num); /* start queue */
322
323 return RET_DDCB_TAPPED;
324}
325
326/**
327 * copy_ddcb_results() - Copy output state from real DDCB to request
328 *
329 * Copy DDCB ASV to request struct. There is no endian
330 * conversion made, since data structure in ASV is still
331 * unknown here.
332 *
333 * This is needed by:
334 * - genwqe_purge_ddcb()
335 * - genwqe_check_ddcb_queue()
336 */
337static void copy_ddcb_results(struct ddcb_requ *req, int ddcb_no)
338{
339 struct ddcb_queue *queue = req->queue;
340 struct ddcb *pddcb = &queue->ddcb_vaddr[req->num];
341
342 memcpy(&req->cmd.asv[0], &pddcb->asv[0], DDCB_ASV_LENGTH);
343
344 /* copy status flags of the variant part */
345 req->cmd.vcrc = be16_to_cpu(pddcb->vcrc_16);
346 req->cmd.deque_ts = be64_to_cpu(pddcb->deque_ts_64);
347 req->cmd.cmplt_ts = be64_to_cpu(pddcb->cmplt_ts_64);
348
349 req->cmd.attn = be16_to_cpu(pddcb->attn_16);
350 req->cmd.progress = be32_to_cpu(pddcb->progress_32);
351 req->cmd.retc = be16_to_cpu(pddcb->retc_16);
352
353 if (ddcb_requ_collect_debug_data(req)) {
354 int prev_no = (ddcb_no == 0) ?
355 queue->ddcb_max - 1 : ddcb_no - 1;
356 struct ddcb *prev_pddcb = &queue->ddcb_vaddr[prev_no];
357
358 memcpy(&req->debug_data.ddcb_finished, pddcb,
359 sizeof(req->debug_data.ddcb_finished));
360 memcpy(&req->debug_data.ddcb_prev, prev_pddcb,
361 sizeof(req->debug_data.ddcb_prev));
362 }
363}
364
365/**
366 * genwqe_check_ddcb_queue() - Checks DDCB queue for completed work equests.
367 * @cd: pointer to genwqe device descriptor
368 *
369 * Return: Number of DDCBs which were finished
370 */
371static int genwqe_check_ddcb_queue(struct genwqe_dev *cd,
372 struct ddcb_queue *queue)
373{
374 unsigned long flags;
375 int ddcbs_finished = 0;
376 struct pci_dev *pci_dev = cd->pci_dev;
377
378 spin_lock_irqsave(&queue->ddcb_lock, flags);
379
380 /* FIXME avoid soft locking CPU */
381 while (!queue_empty(queue) && (ddcbs_finished < queue->ddcb_max)) {
382
383 struct ddcb *pddcb;
384 struct ddcb_requ *req;
385 u16 vcrc, vcrc_16, retc_16;
386
387 pddcb = &queue->ddcb_vaddr[queue->ddcb_act];
388
389 if ((pddcb->icrc_hsi_shi_32 & DDCB_COMPLETED_BE32) ==
390 0x00000000)
391 goto go_home; /* not completed, continue waiting */
392
Eberhard S. Amann08e49062014-09-10 16:37:52 +0200393 wmb(); /* Add sync to decouple prev. read operations */
Frank Haverkampeaf47222013-12-09 13:30:40 +0100394
Eberhard S. Amann08e49062014-09-10 16:37:52 +0200395 /* Note: DDCB could be purged */
Frank Haverkampeaf47222013-12-09 13:30:40 +0100396 req = queue->ddcb_req[queue->ddcb_act];
397 if (req == NULL) {
398 /* this occurs if DDCB is purged, not an error */
399 /* Move active DDCB further; Nothing to do anymore. */
400 goto pick_next_one;
401 }
402
403 /*
404 * HSI=0x44 (fetched and completed), but RETC is
405 * 0x101, or even worse 0x000.
406 *
407 * In case of seeing the queue in inconsistent state
408 * we read the errcnts and the queue status to provide
409 * a trigger for our PCIe analyzer stop capturing.
410 */
411 retc_16 = be16_to_cpu(pddcb->retc_16);
412 if ((pddcb->hsi == 0x44) && (retc_16 <= 0x101)) {
413 u64 errcnts, status;
414 u64 ddcb_offs = (u64)pddcb - (u64)queue->ddcb_vaddr;
415
416 errcnts = __genwqe_readq(cd, queue->IO_QUEUE_ERRCNTS);
417 status = __genwqe_readq(cd, queue->IO_QUEUE_STATUS);
418
419 dev_err(&pci_dev->dev,
Frank Haverkampd9c11d42014-09-10 16:37:51 +0200420 "[%s] SEQN=%04x HSI=%02x RETC=%03x Q_ERRCNTS=%016llx Q_STATUS=%016llx DDCB_DMA_ADDR=%016llx\n",
Frank Haverkampeaf47222013-12-09 13:30:40 +0100421 __func__, be16_to_cpu(pddcb->seqnum_16),
422 pddcb->hsi, retc_16, errcnts, status,
423 queue->ddcb_daddr + ddcb_offs);
424 }
425
426 copy_ddcb_results(req, queue->ddcb_act);
427 queue->ddcb_req[queue->ddcb_act] = NULL; /* take from queue */
428
429 dev_dbg(&pci_dev->dev, "FINISHED DDCB#%d\n", req->num);
430 genwqe_hexdump(pci_dev, pddcb, sizeof(*pddcb));
431
432 ddcb_mark_finished(pddcb);
433
434 /* calculate CRC_16 to see if VCRC is correct */
435 vcrc = genwqe_crc16(pddcb->asv,
436 VCRC_LENGTH(req->cmd.asv_length),
437 0xffff);
438 vcrc_16 = be16_to_cpu(pddcb->vcrc_16);
439 if (vcrc != vcrc_16) {
440 printk_ratelimited(KERN_ERR
Frank Haverkampd9c11d42014-09-10 16:37:51 +0200441 "%s %s: err: wrong VCRC pre=%02x vcrc_len=%d bytes vcrc_data=%04x is not vcrc_card=%04x\n",
Frank Haverkampeaf47222013-12-09 13:30:40 +0100442 GENWQE_DEVNAME, dev_name(&pci_dev->dev),
443 pddcb->pre, VCRC_LENGTH(req->cmd.asv_length),
444 vcrc, vcrc_16);
445 }
446
447 ddcb_requ_set_state(req, GENWQE_REQU_FINISHED);
448 queue->ddcbs_completed++;
449 queue->ddcbs_in_flight--;
450
Frank Haverkamp1451f412014-09-10 16:37:53 +0200451 /* wake up process waiting for this DDCB, and
452 processes on the busy queue */
Frank Haverkampeaf47222013-12-09 13:30:40 +0100453 wake_up_interruptible(&queue->ddcb_waitqs[queue->ddcb_act]);
Frank Haverkamp1451f412014-09-10 16:37:53 +0200454 wake_up_interruptible(&queue->busy_waitq);
Frank Haverkampeaf47222013-12-09 13:30:40 +0100455
456pick_next_one:
457 queue->ddcb_act = (queue->ddcb_act + 1) % queue->ddcb_max;
458 ddcbs_finished++;
459 }
460
461 go_home:
462 spin_unlock_irqrestore(&queue->ddcb_lock, flags);
463 return ddcbs_finished;
464}
465
466/**
467 * __genwqe_wait_ddcb(): Waits until DDCB is completed
468 * @cd: pointer to genwqe device descriptor
469 * @req: pointer to requsted DDCB parameters
470 *
471 * The Service Layer will update the RETC in DDCB when processing is
472 * pending or done.
473 *
474 * Return: > 0 remaining jiffies, DDCB completed
475 * -ETIMEDOUT when timeout
476 * -ERESTARTSYS when ^C
477 * -EINVAL when unknown error condition
478 *
479 * When an error is returned the called needs to ensure that
480 * purge_ddcb() is being called to get the &req removed from the
481 * queue.
482 */
483int __genwqe_wait_ddcb(struct genwqe_dev *cd, struct ddcb_requ *req)
484{
485 int rc;
486 unsigned int ddcb_no;
487 struct ddcb_queue *queue;
488 struct pci_dev *pci_dev = cd->pci_dev;
489
490 if (req == NULL)
491 return -EINVAL;
492
493 queue = req->queue;
494 if (queue == NULL)
495 return -EINVAL;
496
497 ddcb_no = req->num;
498 if (ddcb_no >= queue->ddcb_max)
499 return -EINVAL;
500
501 rc = wait_event_interruptible_timeout(queue->ddcb_waitqs[ddcb_no],
502 ddcb_requ_finished(cd, req),
503 genwqe_ddcb_software_timeout * HZ);
504
505 /*
506 * We need to distinguish 3 cases here:
507 * 1. rc == 0 timeout occured
508 * 2. rc == -ERESTARTSYS signal received
509 * 3. rc > 0 remaining jiffies condition is true
510 */
511 if (rc == 0) {
512 struct ddcb_queue *queue = req->queue;
513 struct ddcb *pddcb;
514
515 /*
516 * Timeout may be caused by long task switching time.
517 * When timeout happens, check if the request has
518 * meanwhile completed.
519 */
520 genwqe_check_ddcb_queue(cd, req->queue);
521 if (ddcb_requ_finished(cd, req))
522 return rc;
523
524 dev_err(&pci_dev->dev,
525 "[%s] err: DDCB#%d timeout rc=%d state=%d req @ %p\n",
526 __func__, req->num, rc, ddcb_requ_get_state(req),
527 req);
528 dev_err(&pci_dev->dev,
529 "[%s] IO_QUEUE_STATUS=0x%016llx\n", __func__,
530 __genwqe_readq(cd, queue->IO_QUEUE_STATUS));
531
532 pddcb = &queue->ddcb_vaddr[req->num];
533 genwqe_hexdump(pci_dev, pddcb, sizeof(*pddcb));
534
535 print_ddcb_info(cd, req->queue);
536 return -ETIMEDOUT;
537
538 } else if (rc == -ERESTARTSYS) {
539 return rc;
540 /*
541 * EINTR: Stops the application
542 * ERESTARTSYS: Restartable systemcall; called again
543 */
544
545 } else if (rc < 0) {
546 dev_err(&pci_dev->dev,
547 "[%s] err: DDCB#%d unknown result (rc=%d) %d!\n",
548 __func__, req->num, rc, ddcb_requ_get_state(req));
549 return -EINVAL;
550 }
551
552 /* Severe error occured. Driver is forced to stop operation */
553 if (cd->card_state != GENWQE_CARD_USED) {
554 dev_err(&pci_dev->dev,
555 "[%s] err: DDCB#%d forced to stop (rc=%d)\n",
556 __func__, req->num, rc);
557 return -EIO;
558 }
559 return rc;
560}
561
562/**
563 * get_next_ddcb() - Get next available DDCB
564 * @cd: pointer to genwqe device descriptor
565 *
566 * DDCB's content is completely cleared but presets for PRE and
567 * SEQNUM. This function must only be called when ddcb_lock is held.
568 *
569 * Return: NULL if no empty DDCB available otherwise ptr to next DDCB.
570 */
571static struct ddcb *get_next_ddcb(struct genwqe_dev *cd,
572 struct ddcb_queue *queue,
573 int *num)
574{
575 u64 *pu64;
576 struct ddcb *pddcb;
577
578 if (queue_free_ddcbs(queue) == 0) /* queue is full */
579 return NULL;
580
581 /* find new ddcb */
582 pddcb = &queue->ddcb_vaddr[queue->ddcb_next];
583
584 /* if it is not completed, we are not allowed to use it */
585 /* barrier(); */
586 if ((pddcb->icrc_hsi_shi_32 & DDCB_COMPLETED_BE32) == 0x00000000)
587 return NULL;
588
589 *num = queue->ddcb_next; /* internal DDCB number */
590 queue->ddcb_next = (queue->ddcb_next + 1) % queue->ddcb_max;
591
592 /* clear important DDCB fields */
593 pu64 = (u64 *)pddcb;
594 pu64[0] = 0ULL; /* offs 0x00 (ICRC,HSI,SHI,...) */
595 pu64[1] = 0ULL; /* offs 0x01 (ACFUNC,CMD...) */
596
597 /* destroy previous results in ASV */
598 pu64[0x80/8] = 0ULL; /* offs 0x80 (ASV + 0) */
599 pu64[0x88/8] = 0ULL; /* offs 0x88 (ASV + 0x08) */
600 pu64[0x90/8] = 0ULL; /* offs 0x90 (ASV + 0x10) */
601 pu64[0x98/8] = 0ULL; /* offs 0x98 (ASV + 0x18) */
602 pu64[0xd0/8] = 0ULL; /* offs 0xd0 (RETC,ATTN...) */
603
604 pddcb->pre = DDCB_PRESET_PRE; /* 128 */
605 pddcb->seqnum_16 = cpu_to_be16(queue->ddcb_seq++);
606 return pddcb;
607}
608
609/**
610 * __genwqe_purge_ddcb() - Remove a DDCB from the workqueue
611 * @cd: genwqe device descriptor
612 * @req: DDCB request
613 *
614 * This will fail when the request was already FETCHED. In this case
615 * we need to wait until it is finished. Else the DDCB can be
616 * reused. This function also ensures that the request data structure
617 * is removed from ddcb_req[].
618 *
619 * Do not forget to call this function when genwqe_wait_ddcb() fails,
620 * such that the request gets really removed from ddcb_req[].
621 *
622 * Return: 0 success
623 */
624int __genwqe_purge_ddcb(struct genwqe_dev *cd, struct ddcb_requ *req)
625{
626 struct ddcb *pddcb = NULL;
627 unsigned int t;
628 unsigned long flags;
629 struct ddcb_queue *queue = req->queue;
630 struct pci_dev *pci_dev = cd->pci_dev;
Frank Haverkampeaf47222013-12-09 13:30:40 +0100631 u64 queue_status;
Frank Haverkamp58d66ce2013-12-20 16:26:10 +0100632 __be32 icrc_hsi_shi = 0x0000;
633 __be32 old, new;
Frank Haverkampeaf47222013-12-09 13:30:40 +0100634
635 /* unsigned long flags; */
636 if (genwqe_ddcb_software_timeout <= 0) {
637 dev_err(&pci_dev->dev,
638 "[%s] err: software timeout is not set!\n", __func__);
639 return -EFAULT;
640 }
641
642 pddcb = &queue->ddcb_vaddr[req->num];
643
644 for (t = 0; t < genwqe_ddcb_software_timeout * 10; t++) {
645
646 spin_lock_irqsave(&queue->ddcb_lock, flags);
647
648 /* Check if req was meanwhile finished */
649 if (ddcb_requ_get_state(req) == GENWQE_REQU_FINISHED)
650 goto go_home;
651
652 /* try to set PURGE bit if FETCHED/COMPLETED are not set */
653 old = pddcb->icrc_hsi_shi_32; /* read SHI/HSI in BE32 */
654 if ((old & DDCB_FETCHED_BE32) == 0x00000000) {
655
656 new = (old | DDCB_PURGE_BE32);
657 icrc_hsi_shi = cmpxchg(&pddcb->icrc_hsi_shi_32,
658 old, new);
659 if (icrc_hsi_shi == old)
660 goto finish_ddcb;
661 }
662
663 /* normal finish with HSI bit */
664 barrier();
665 icrc_hsi_shi = pddcb->icrc_hsi_shi_32;
666 if (icrc_hsi_shi & DDCB_COMPLETED_BE32)
667 goto finish_ddcb;
668
669 spin_unlock_irqrestore(&queue->ddcb_lock, flags);
670
671 /*
672 * Here the check_ddcb() function will most likely
673 * discover this DDCB to be finished some point in
674 * time. It will mark the req finished and free it up
675 * in the list.
676 */
677
678 copy_ddcb_results(req, req->num); /* for the failing case */
679 msleep(100); /* sleep for 1/10 second and try again */
680 continue;
681
682finish_ddcb:
683 copy_ddcb_results(req, req->num);
684 ddcb_requ_set_state(req, GENWQE_REQU_FINISHED);
685 queue->ddcbs_in_flight--;
686 queue->ddcb_req[req->num] = NULL; /* delete from array */
687 ddcb_mark_cleared(pddcb);
688
689 /* Move active DDCB further; Nothing to do here anymore. */
690
691 /*
692 * We need to ensure that there is at least one free
693 * DDCB in the queue. To do that, we must update
694 * ddcb_act only if the COMPLETED bit is set for the
695 * DDCB we are working on else we treat that DDCB even
696 * if we PURGED it as occupied (hardware is supposed
697 * to set the COMPLETED bit yet!).
698 */
699 icrc_hsi_shi = pddcb->icrc_hsi_shi_32;
700 if ((icrc_hsi_shi & DDCB_COMPLETED_BE32) &&
701 (queue->ddcb_act == req->num)) {
702 queue->ddcb_act = ((queue->ddcb_act + 1) %
703 queue->ddcb_max);
704 }
705go_home:
706 spin_unlock_irqrestore(&queue->ddcb_lock, flags);
707 return 0;
708 }
709
710 /*
711 * If the card is dead and the queue is forced to stop, we
712 * might see this in the queue status register.
713 */
714 queue_status = __genwqe_readq(cd, queue->IO_QUEUE_STATUS);
715
716 dev_dbg(&pci_dev->dev, "UN/FINISHED DDCB#%d\n", req->num);
717 genwqe_hexdump(pci_dev, pddcb, sizeof(*pddcb));
718
719 dev_err(&pci_dev->dev,
Frank Haverkampd9c11d42014-09-10 16:37:51 +0200720 "[%s] err: DDCB#%d not purged and not completed after %d seconds QSTAT=%016llx!!\n",
Frank Haverkampeaf47222013-12-09 13:30:40 +0100721 __func__, req->num, genwqe_ddcb_software_timeout,
722 queue_status);
723
724 print_ddcb_info(cd, req->queue);
725
726 return -EFAULT;
727}
728
729int genwqe_init_debug_data(struct genwqe_dev *cd, struct genwqe_debug_data *d)
730{
731 int len;
732 struct pci_dev *pci_dev = cd->pci_dev;
733
734 if (d == NULL) {
735 dev_err(&pci_dev->dev,
736 "[%s] err: invalid memory for debug data!\n",
737 __func__);
738 return -EFAULT;
739 }
740
741 len = sizeof(d->driver_version);
Frank Haverkamp64df2ec2014-09-10 16:37:47 +0200742 snprintf(d->driver_version, len, "%s", DRV_VERSION);
Frank Haverkampeaf47222013-12-09 13:30:40 +0100743 d->slu_unitcfg = cd->slu_unitcfg;
744 d->app_unitcfg = cd->app_unitcfg;
745 return 0;
746}
747
748/**
749 * __genwqe_enqueue_ddcb() - Enqueue a DDCB
Frank Haverkamp1451f412014-09-10 16:37:53 +0200750 * @cd: pointer to genwqe device descriptor
751 * @req: pointer to DDCB execution request
752 * @f_flags: file mode: blocking, non-blocking
Frank Haverkampeaf47222013-12-09 13:30:40 +0100753 *
754 * Return: 0 if enqueuing succeeded
755 * -EIO if card is unusable/PCIe problems
756 * -EBUSY if enqueuing failed
757 */
Frank Haverkamp1451f412014-09-10 16:37:53 +0200758int __genwqe_enqueue_ddcb(struct genwqe_dev *cd, struct ddcb_requ *req,
759 unsigned int f_flags)
Frank Haverkampeaf47222013-12-09 13:30:40 +0100760{
761 struct ddcb *pddcb;
762 unsigned long flags;
763 struct ddcb_queue *queue;
764 struct pci_dev *pci_dev = cd->pci_dev;
765 u16 icrc;
766
Frank Haverkamp1451f412014-09-10 16:37:53 +0200767 retry:
Frank Haverkampeaf47222013-12-09 13:30:40 +0100768 if (cd->card_state != GENWQE_CARD_USED) {
769 printk_ratelimited(KERN_ERR
770 "%s %s: [%s] Card is unusable/PCIe problem Req#%d\n",
771 GENWQE_DEVNAME, dev_name(&pci_dev->dev),
772 __func__, req->num);
773 return -EIO;
774 }
775
776 queue = req->queue = &cd->queue;
777
778 /* FIXME circumvention to improve performance when no irq is
779 * there.
780 */
781 if (genwqe_polling_enabled)
782 genwqe_check_ddcb_queue(cd, queue);
783
784 /*
785 * It must be ensured to process all DDCBs in successive
786 * order. Use a lock here in order to prevent nested DDCB
787 * enqueuing.
788 */
789 spin_lock_irqsave(&queue->ddcb_lock, flags);
790
791 pddcb = get_next_ddcb(cd, queue, &req->num); /* get ptr and num */
792 if (pddcb == NULL) {
Frank Haverkamp1451f412014-09-10 16:37:53 +0200793 int rc;
794
Frank Haverkampeaf47222013-12-09 13:30:40 +0100795 spin_unlock_irqrestore(&queue->ddcb_lock, flags);
Frank Haverkamp1451f412014-09-10 16:37:53 +0200796
797 if (f_flags & O_NONBLOCK) {
798 queue->return_on_busy++;
799 return -EBUSY;
800 }
801
802 queue->wait_on_busy++;
803 rc = wait_event_interruptible(queue->busy_waitq,
804 queue_free_ddcbs(queue) != 0);
805 dev_dbg(&pci_dev->dev, "[%s] waiting for free DDCB: rc=%d\n",
806 __func__, rc);
807 if (rc == -ERESTARTSYS)
808 return rc; /* interrupted by a signal */
809
810 goto retry;
Frank Haverkampeaf47222013-12-09 13:30:40 +0100811 }
812
813 if (queue->ddcb_req[req->num] != NULL) {
814 spin_unlock_irqrestore(&queue->ddcb_lock, flags);
815
816 dev_err(&pci_dev->dev,
817 "[%s] picked DDCB %d with req=%p still in use!!\n",
818 __func__, req->num, req);
819 return -EFAULT;
820 }
821 ddcb_requ_set_state(req, GENWQE_REQU_ENQUEUED);
822 queue->ddcb_req[req->num] = req;
823
824 pddcb->cmdopts_16 = cpu_to_be16(req->cmd.cmdopts);
825 pddcb->cmd = req->cmd.cmd;
826 pddcb->acfunc = req->cmd.acfunc; /* functional unit */
827
828 /*
829 * We know that we can get retc 0x104 with CRC error, do not
830 * stop the queue in those cases for this command. XDIR = 1
831 * does not work for old SLU versions.
832 *
833 * Last bitstream with the old XDIR behavior had SLU_ID
834 * 0x34199.
835 */
836 if ((cd->slu_unitcfg & 0xFFFF0ull) > 0x34199ull)
837 pddcb->xdir = 0x1;
838 else
839 pddcb->xdir = 0x0;
840
841
842 pddcb->psp = (((req->cmd.asiv_length / 8) << 4) |
843 ((req->cmd.asv_length / 8)));
844 pddcb->disp_ts_64 = cpu_to_be64(req->cmd.disp_ts);
845
846 /*
847 * If copying the whole DDCB_ASIV_LENGTH is impacting
848 * performance we need to change it to
849 * req->cmd.asiv_length. But simulation benefits from some
850 * non-architectured bits behind the architectured content.
851 *
852 * How much data is copied depends on the availability of the
853 * ATS field, which was introduced late. If the ATS field is
854 * supported ASIV is 8 bytes shorter than it used to be. Since
855 * the ATS field is copied too, the code should do exactly
856 * what it did before, but I wanted to make copying of the ATS
857 * field very explicit.
858 */
859 if (genwqe_get_slu_id(cd) <= 0x2) {
860 memcpy(&pddcb->__asiv[0], /* destination */
861 &req->cmd.__asiv[0], /* source */
862 DDCB_ASIV_LENGTH); /* req->cmd.asiv_length */
863 } else {
Frank Haverkamp58d66ce2013-12-20 16:26:10 +0100864 pddcb->n.ats_64 = cpu_to_be64(req->cmd.ats);
865 memcpy(&pddcb->n.asiv[0], /* destination */
Frank Haverkampeaf47222013-12-09 13:30:40 +0100866 &req->cmd.asiv[0], /* source */
867 DDCB_ASIV_LENGTH_ATS); /* req->cmd.asiv_length */
868 }
869
870 pddcb->icrc_hsi_shi_32 = cpu_to_be32(0x00000000); /* for crc */
871
872 /*
873 * Calculate CRC_16 for corresponding range PSP(7:4). Include
874 * empty 4 bytes prior to the data.
875 */
876 icrc = genwqe_crc16((const u8 *)pddcb,
877 ICRC_LENGTH(req->cmd.asiv_length), 0xffff);
878 pddcb->icrc_hsi_shi_32 = cpu_to_be32((u32)icrc << 16);
879
880 /* enable DDCB completion irq */
881 if (!genwqe_polling_enabled)
882 pddcb->icrc_hsi_shi_32 |= DDCB_INTR_BE32;
883
884 dev_dbg(&pci_dev->dev, "INPUT DDCB#%d\n", req->num);
885 genwqe_hexdump(pci_dev, pddcb, sizeof(*pddcb));
886
887 if (ddcb_requ_collect_debug_data(req)) {
888 /* use the kernel copy of debug data. copying back to
889 user buffer happens later */
890
891 genwqe_init_debug_data(cd, &req->debug_data);
892 memcpy(&req->debug_data.ddcb_before, pddcb,
893 sizeof(req->debug_data.ddcb_before));
894 }
895
896 enqueue_ddcb(cd, queue, pddcb, req->num);
897 queue->ddcbs_in_flight++;
898
899 if (queue->ddcbs_in_flight > queue->ddcbs_max_in_flight)
900 queue->ddcbs_max_in_flight = queue->ddcbs_in_flight;
901
902 ddcb_requ_set_state(req, GENWQE_REQU_TAPPED);
903 spin_unlock_irqrestore(&queue->ddcb_lock, flags);
904 wake_up_interruptible(&cd->queue_waitq);
905
906 return 0;
907}
908
909/**
910 * __genwqe_execute_raw_ddcb() - Setup and execute DDCB
911 * @cd: pointer to genwqe device descriptor
912 * @req: user provided DDCB request
Frank Haverkamp1451f412014-09-10 16:37:53 +0200913 * @f_flags: file mode: blocking, non-blocking
Frank Haverkampeaf47222013-12-09 13:30:40 +0100914 */
915int __genwqe_execute_raw_ddcb(struct genwqe_dev *cd,
Frank Haverkamp1451f412014-09-10 16:37:53 +0200916 struct genwqe_ddcb_cmd *cmd,
917 unsigned int f_flags)
Frank Haverkampeaf47222013-12-09 13:30:40 +0100918{
919 int rc = 0;
920 struct pci_dev *pci_dev = cd->pci_dev;
921 struct ddcb_requ *req = container_of(cmd, struct ddcb_requ, cmd);
922
923 if (cmd->asiv_length > DDCB_ASIV_LENGTH) {
924 dev_err(&pci_dev->dev, "[%s] err: wrong asiv_length of %d\n",
925 __func__, cmd->asiv_length);
926 return -EINVAL;
927 }
928 if (cmd->asv_length > DDCB_ASV_LENGTH) {
929 dev_err(&pci_dev->dev, "[%s] err: wrong asv_length of %d\n",
930 __func__, cmd->asiv_length);
931 return -EINVAL;
932 }
Frank Haverkamp1451f412014-09-10 16:37:53 +0200933 rc = __genwqe_enqueue_ddcb(cd, req, f_flags);
Frank Haverkampeaf47222013-12-09 13:30:40 +0100934 if (rc != 0)
935 return rc;
936
937 rc = __genwqe_wait_ddcb(cd, req);
938 if (rc < 0) /* error or signal interrupt */
939 goto err_exit;
940
941 if (ddcb_requ_collect_debug_data(req)) {
Frank Haverkamp58d66ce2013-12-20 16:26:10 +0100942 if (copy_to_user((struct genwqe_debug_data __user *)
943 (unsigned long)cmd->ddata_addr,
Frank Haverkampeaf47222013-12-09 13:30:40 +0100944 &req->debug_data,
945 sizeof(struct genwqe_debug_data)))
946 return -EFAULT;
947 }
948
949 /*
950 * Higher values than 0x102 indicate completion with faults,
951 * lower values than 0x102 indicate processing faults. Note
952 * that DDCB might have been purged. E.g. Cntl+C.
953 */
954 if (cmd->retc != DDCB_RETC_COMPLETE) {
955 /* This might happen e.g. flash read, and needs to be
956 handled by the upper layer code. */
957 rc = -EBADMSG; /* not processed/error retc */
958 }
959
960 return rc;
961
962 err_exit:
963 __genwqe_purge_ddcb(cd, req);
964
965 if (ddcb_requ_collect_debug_data(req)) {
Frank Haverkamp58d66ce2013-12-20 16:26:10 +0100966 if (copy_to_user((struct genwqe_debug_data __user *)
967 (unsigned long)cmd->ddata_addr,
Frank Haverkampeaf47222013-12-09 13:30:40 +0100968 &req->debug_data,
969 sizeof(struct genwqe_debug_data)))
970 return -EFAULT;
971 }
972 return rc;
973}
974
975/**
976 * genwqe_next_ddcb_ready() - Figure out if the next DDCB is already finished
977 *
978 * We use this as condition for our wait-queue code.
979 */
980static int genwqe_next_ddcb_ready(struct genwqe_dev *cd)
981{
982 unsigned long flags;
983 struct ddcb *pddcb;
984 struct ddcb_queue *queue = &cd->queue;
985
986 spin_lock_irqsave(&queue->ddcb_lock, flags);
987
988 if (queue_empty(queue)) { /* emtpy queue */
989 spin_unlock_irqrestore(&queue->ddcb_lock, flags);
990 return 0;
991 }
992
993 pddcb = &queue->ddcb_vaddr[queue->ddcb_act];
994 if (pddcb->icrc_hsi_shi_32 & DDCB_COMPLETED_BE32) { /* ddcb ready */
995 spin_unlock_irqrestore(&queue->ddcb_lock, flags);
996 return 1;
997 }
998
999 spin_unlock_irqrestore(&queue->ddcb_lock, flags);
1000 return 0;
1001}
1002
1003/**
1004 * genwqe_ddcbs_in_flight() - Check how many DDCBs are in flight
1005 *
1006 * Keep track on the number of DDCBs which ware currently in the
1007 * queue. This is needed for statistics as well as conditon if we want
1008 * to wait or better do polling in case of no interrupts available.
1009 */
1010int genwqe_ddcbs_in_flight(struct genwqe_dev *cd)
1011{
1012 unsigned long flags;
1013 int ddcbs_in_flight = 0;
1014 struct ddcb_queue *queue = &cd->queue;
1015
1016 spin_lock_irqsave(&queue->ddcb_lock, flags);
1017 ddcbs_in_flight += queue->ddcbs_in_flight;
1018 spin_unlock_irqrestore(&queue->ddcb_lock, flags);
1019
1020 return ddcbs_in_flight;
1021}
1022
1023static int setup_ddcb_queue(struct genwqe_dev *cd, struct ddcb_queue *queue)
1024{
1025 int rc, i;
1026 struct ddcb *pddcb;
1027 u64 val64;
1028 unsigned int queue_size;
1029 struct pci_dev *pci_dev = cd->pci_dev;
1030
1031 if (genwqe_ddcb_max < 2)
1032 return -EINVAL;
1033
1034 queue_size = roundup(genwqe_ddcb_max * sizeof(struct ddcb), PAGE_SIZE);
1035
1036 queue->ddcbs_in_flight = 0; /* statistics */
1037 queue->ddcbs_max_in_flight = 0;
1038 queue->ddcbs_completed = 0;
Frank Haverkamp1451f412014-09-10 16:37:53 +02001039 queue->return_on_busy = 0;
1040 queue->wait_on_busy = 0;
Frank Haverkampeaf47222013-12-09 13:30:40 +01001041
1042 queue->ddcb_seq = 0x100; /* start sequence number */
1043 queue->ddcb_max = genwqe_ddcb_max; /* module parameter */
1044 queue->ddcb_vaddr = __genwqe_alloc_consistent(cd, queue_size,
1045 &queue->ddcb_daddr);
1046 if (queue->ddcb_vaddr == NULL) {
1047 dev_err(&pci_dev->dev,
1048 "[%s] **err: could not allocate DDCB **\n", __func__);
1049 return -ENOMEM;
1050 }
1051 memset(queue->ddcb_vaddr, 0, queue_size);
1052
1053 queue->ddcb_req = kzalloc(sizeof(struct ddcb_requ *) *
1054 queue->ddcb_max, GFP_KERNEL);
1055 if (!queue->ddcb_req) {
1056 rc = -ENOMEM;
1057 goto free_ddcbs;
1058 }
1059
1060 queue->ddcb_waitqs = kzalloc(sizeof(wait_queue_head_t) *
1061 queue->ddcb_max, GFP_KERNEL);
1062 if (!queue->ddcb_waitqs) {
1063 rc = -ENOMEM;
1064 goto free_requs;
1065 }
1066
1067 for (i = 0; i < queue->ddcb_max; i++) {
1068 pddcb = &queue->ddcb_vaddr[i]; /* DDCBs */
1069 pddcb->icrc_hsi_shi_32 = DDCB_COMPLETED_BE32;
1070 pddcb->retc_16 = cpu_to_be16(0xfff);
1071
1072 queue->ddcb_req[i] = NULL; /* requests */
1073 init_waitqueue_head(&queue->ddcb_waitqs[i]); /* waitqueues */
1074 }
1075
1076 queue->ddcb_act = 0;
1077 queue->ddcb_next = 0; /* queue is empty */
1078
1079 spin_lock_init(&queue->ddcb_lock);
Frank Haverkamp1451f412014-09-10 16:37:53 +02001080 init_waitqueue_head(&queue->busy_waitq);
Frank Haverkampeaf47222013-12-09 13:30:40 +01001081
1082 val64 = ((u64)(queue->ddcb_max - 1) << 8); /* lastptr */
1083 __genwqe_writeq(cd, queue->IO_QUEUE_CONFIG, 0x07); /* iCRC/vCRC */
1084 __genwqe_writeq(cd, queue->IO_QUEUE_SEGMENT, queue->ddcb_daddr);
1085 __genwqe_writeq(cd, queue->IO_QUEUE_INITSQN, queue->ddcb_seq);
1086 __genwqe_writeq(cd, queue->IO_QUEUE_WRAP, val64);
1087 return 0;
1088
1089 free_requs:
1090 kfree(queue->ddcb_req);
1091 queue->ddcb_req = NULL;
1092 free_ddcbs:
1093 __genwqe_free_consistent(cd, queue_size, queue->ddcb_vaddr,
1094 queue->ddcb_daddr);
1095 queue->ddcb_vaddr = NULL;
1096 queue->ddcb_daddr = 0ull;
1097 return -ENODEV;
1098
1099}
1100
1101static int ddcb_queue_initialized(struct ddcb_queue *queue)
1102{
1103 return queue->ddcb_vaddr != NULL;
1104}
1105
1106static void free_ddcb_queue(struct genwqe_dev *cd, struct ddcb_queue *queue)
1107{
1108 unsigned int queue_size;
1109
1110 queue_size = roundup(queue->ddcb_max * sizeof(struct ddcb), PAGE_SIZE);
1111
1112 kfree(queue->ddcb_req);
1113 queue->ddcb_req = NULL;
1114
1115 if (queue->ddcb_vaddr) {
1116 __genwqe_free_consistent(cd, queue_size, queue->ddcb_vaddr,
1117 queue->ddcb_daddr);
1118 queue->ddcb_vaddr = NULL;
1119 queue->ddcb_daddr = 0ull;
1120 }
1121}
1122
1123static irqreturn_t genwqe_pf_isr(int irq, void *dev_id)
1124{
1125 u64 gfir;
1126 struct genwqe_dev *cd = (struct genwqe_dev *)dev_id;
1127 struct pci_dev *pci_dev = cd->pci_dev;
1128
1129 /*
1130 * In case of fatal FIR error the queue is stopped, such that
1131 * we can safely check it without risking anything.
1132 */
1133 cd->irqs_processed++;
1134 wake_up_interruptible(&cd->queue_waitq);
1135
1136 /*
1137 * Checking for errors before kicking the queue might be
1138 * safer, but slower for the good-case ... See above.
1139 */
1140 gfir = __genwqe_readq(cd, IO_SLC_CFGREG_GFIR);
Kleber Sacilotto de Souzafb145452014-06-04 10:57:51 -03001141 if (((gfir & GFIR_ERR_TRIGGER) != 0x0) &&
1142 !pci_channel_offline(pci_dev)) {
1143
1144 if (cd->use_platform_recovery) {
1145 /*
1146 * Since we use raw accessors, EEH errors won't be
1147 * detected by the platform until we do a non-raw
1148 * MMIO or config space read
1149 */
1150 readq(cd->mmio + IO_SLC_CFGREG_GFIR);
1151
1152 /* Don't do anything if the PCI channel is frozen */
1153 if (pci_channel_offline(pci_dev))
1154 goto exit;
1155 }
Frank Haverkampeaf47222013-12-09 13:30:40 +01001156
1157 wake_up_interruptible(&cd->health_waitq);
1158
1159 /*
1160 * By default GFIRs causes recovery actions. This
1161 * count is just for debug when recovery is masked.
1162 */
Kleber Sacilotto de Souzafb145452014-06-04 10:57:51 -03001163 dev_err_ratelimited(&pci_dev->dev,
1164 "[%s] GFIR=%016llx\n",
1165 __func__, gfir);
Frank Haverkampeaf47222013-12-09 13:30:40 +01001166 }
1167
Kleber Sacilotto de Souzafb145452014-06-04 10:57:51 -03001168 exit:
Frank Haverkampeaf47222013-12-09 13:30:40 +01001169 return IRQ_HANDLED;
1170}
1171
1172static irqreturn_t genwqe_vf_isr(int irq, void *dev_id)
1173{
1174 struct genwqe_dev *cd = (struct genwqe_dev *)dev_id;
1175
1176 cd->irqs_processed++;
1177 wake_up_interruptible(&cd->queue_waitq);
1178
1179 return IRQ_HANDLED;
1180}
1181
1182/**
1183 * genwqe_card_thread() - Work thread for the DDCB queue
1184 *
1185 * The idea is to check if there are DDCBs in processing. If there are
1186 * some finished DDCBs, we process them and wakeup the
1187 * requestors. Otherwise we give other processes time using
1188 * cond_resched().
1189 */
1190static int genwqe_card_thread(void *data)
1191{
1192 int should_stop = 0, rc = 0;
1193 struct genwqe_dev *cd = (struct genwqe_dev *)data;
1194
1195 while (!kthread_should_stop()) {
1196
1197 genwqe_check_ddcb_queue(cd, &cd->queue);
1198
1199 if (genwqe_polling_enabled) {
1200 rc = wait_event_interruptible_timeout(
1201 cd->queue_waitq,
1202 genwqe_ddcbs_in_flight(cd) ||
1203 (should_stop = kthread_should_stop()), 1);
1204 } else {
1205 rc = wait_event_interruptible_timeout(
1206 cd->queue_waitq,
1207 genwqe_next_ddcb_ready(cd) ||
1208 (should_stop = kthread_should_stop()), HZ);
1209 }
1210 if (should_stop)
1211 break;
1212
1213 /*
1214 * Avoid soft lockups on heavy loads; we do not want
1215 * to disable our interrupts.
1216 */
1217 cond_resched();
1218 }
1219 return 0;
1220}
1221
1222/**
1223 * genwqe_setup_service_layer() - Setup DDCB queue
1224 * @cd: pointer to genwqe device descriptor
1225 *
1226 * Allocate DDCBs. Configure Service Layer Controller (SLC).
1227 *
1228 * Return: 0 success
1229 */
1230int genwqe_setup_service_layer(struct genwqe_dev *cd)
1231{
1232 int rc;
1233 struct ddcb_queue *queue;
1234 struct pci_dev *pci_dev = cd->pci_dev;
1235
1236 if (genwqe_is_privileged(cd)) {
1237 rc = genwqe_card_reset(cd);
1238 if (rc < 0) {
1239 dev_err(&pci_dev->dev,
1240 "[%s] err: reset failed.\n", __func__);
1241 return rc;
1242 }
1243 genwqe_read_softreset(cd);
1244 }
1245
1246 queue = &cd->queue;
1247 queue->IO_QUEUE_CONFIG = IO_SLC_QUEUE_CONFIG;
1248 queue->IO_QUEUE_STATUS = IO_SLC_QUEUE_STATUS;
1249 queue->IO_QUEUE_SEGMENT = IO_SLC_QUEUE_SEGMENT;
1250 queue->IO_QUEUE_INITSQN = IO_SLC_QUEUE_INITSQN;
1251 queue->IO_QUEUE_OFFSET = IO_SLC_QUEUE_OFFSET;
1252 queue->IO_QUEUE_WRAP = IO_SLC_QUEUE_WRAP;
1253 queue->IO_QUEUE_WTIME = IO_SLC_QUEUE_WTIME;
1254 queue->IO_QUEUE_ERRCNTS = IO_SLC_QUEUE_ERRCNTS;
1255 queue->IO_QUEUE_LRW = IO_SLC_QUEUE_LRW;
1256
1257 rc = setup_ddcb_queue(cd, queue);
1258 if (rc != 0) {
1259 rc = -ENODEV;
1260 goto err_out;
1261 }
1262
1263 init_waitqueue_head(&cd->queue_waitq);
1264 cd->card_thread = kthread_run(genwqe_card_thread, cd,
1265 GENWQE_DEVNAME "%d_thread",
1266 cd->card_idx);
1267 if (IS_ERR(cd->card_thread)) {
1268 rc = PTR_ERR(cd->card_thread);
1269 cd->card_thread = NULL;
1270 goto stop_free_queue;
1271 }
1272
1273 rc = genwqe_set_interrupt_capability(cd, GENWQE_MSI_IRQS);
Frank Haverkamp2d880cc2014-09-10 16:37:49 +02001274 if (rc)
Frank Haverkampeaf47222013-12-09 13:30:40 +01001275 goto stop_kthread;
Frank Haverkampeaf47222013-12-09 13:30:40 +01001276
1277 /*
1278 * We must have all wait-queues initialized when we enable the
1279 * interrupts. Otherwise we might crash if we get an early
1280 * irq.
1281 */
1282 init_waitqueue_head(&cd->health_waitq);
1283
1284 if (genwqe_is_privileged(cd)) {
1285 rc = request_irq(pci_dev->irq, genwqe_pf_isr, IRQF_SHARED,
1286 GENWQE_DEVNAME, cd);
1287 } else {
1288 rc = request_irq(pci_dev->irq, genwqe_vf_isr, IRQF_SHARED,
1289 GENWQE_DEVNAME, cd);
1290 }
1291 if (rc < 0) {
1292 dev_err(&pci_dev->dev, "irq %d not free.\n", pci_dev->irq);
1293 goto stop_irq_cap;
1294 }
1295
1296 cd->card_state = GENWQE_CARD_USED;
1297 return 0;
1298
1299 stop_irq_cap:
1300 genwqe_reset_interrupt_capability(cd);
1301 stop_kthread:
1302 kthread_stop(cd->card_thread);
1303 cd->card_thread = NULL;
1304 stop_free_queue:
1305 free_ddcb_queue(cd, queue);
1306 err_out:
1307 return rc;
1308}
1309
1310/**
1311 * queue_wake_up_all() - Handles fatal error case
1312 *
1313 * The PCI device got unusable and we have to stop all pending
1314 * requests as fast as we can. The code after this must purge the
1315 * DDCBs in question and ensure that all mappings are freed.
1316 */
1317static int queue_wake_up_all(struct genwqe_dev *cd)
1318{
1319 unsigned int i;
1320 unsigned long flags;
1321 struct ddcb_queue *queue = &cd->queue;
1322
1323 spin_lock_irqsave(&queue->ddcb_lock, flags);
1324
1325 for (i = 0; i < queue->ddcb_max; i++)
1326 wake_up_interruptible(&queue->ddcb_waitqs[queue->ddcb_act]);
1327
Frank Haverkamp1451f412014-09-10 16:37:53 +02001328 wake_up_interruptible(&queue->busy_waitq);
Frank Haverkampeaf47222013-12-09 13:30:40 +01001329 spin_unlock_irqrestore(&queue->ddcb_lock, flags);
1330
1331 return 0;
1332}
1333
1334/**
1335 * genwqe_finish_queue() - Remove any genwqe devices and user-interfaces
1336 *
1337 * Relies on the pre-condition that there are no users of the card
1338 * device anymore e.g. with open file-descriptors.
1339 *
1340 * This function must be robust enough to be called twice.
1341 */
1342int genwqe_finish_queue(struct genwqe_dev *cd)
1343{
Colin Ian Kingebb2c962014-03-20 15:11:04 +01001344 int i, rc = 0, in_flight;
Frank Haverkampeaf47222013-12-09 13:30:40 +01001345 int waitmax = genwqe_ddcb_software_timeout;
1346 struct pci_dev *pci_dev = cd->pci_dev;
1347 struct ddcb_queue *queue = &cd->queue;
1348
1349 if (!ddcb_queue_initialized(queue))
1350 return 0;
1351
1352 /* Do not wipe out the error state. */
1353 if (cd->card_state == GENWQE_CARD_USED)
1354 cd->card_state = GENWQE_CARD_UNUSED;
1355
1356 /* Wake up all requests in the DDCB queue such that they
1357 should be removed nicely. */
1358 queue_wake_up_all(cd);
1359
1360 /* We must wait to get rid of the DDCBs in flight */
1361 for (i = 0; i < waitmax; i++) {
1362 in_flight = genwqe_ddcbs_in_flight(cd);
1363
1364 if (in_flight == 0)
1365 break;
1366
1367 dev_dbg(&pci_dev->dev,
Frank Haverkampd9c11d42014-09-10 16:37:51 +02001368 " DEBUG [%d/%d] waiting for queue to get empty: %d requests!\n",
1369 i, waitmax, in_flight);
Frank Haverkampeaf47222013-12-09 13:30:40 +01001370
1371 /*
1372 * Severe severe error situation: The card itself has
1373 * 16 DDCB queues, each queue has e.g. 32 entries,
1374 * each DDBC has a hardware timeout of currently 250
1375 * msec but the PFs have a hardware timeout of 8 sec
1376 * ... so I take something large.
1377 */
1378 msleep(1000);
1379 }
1380 if (i == waitmax) {
1381 dev_err(&pci_dev->dev, " [%s] err: queue is not empty!!\n",
1382 __func__);
1383 rc = -EIO;
1384 }
1385 return rc;
1386}
1387
1388/**
1389 * genwqe_release_service_layer() - Shutdown DDCB queue
1390 * @cd: genwqe device descriptor
1391 *
1392 * This function must be robust enough to be called twice.
1393 */
1394int genwqe_release_service_layer(struct genwqe_dev *cd)
1395{
1396 struct pci_dev *pci_dev = cd->pci_dev;
1397
1398 if (!ddcb_queue_initialized(&cd->queue))
1399 return 1;
1400
1401 free_irq(pci_dev->irq, cd);
1402 genwqe_reset_interrupt_capability(cd);
1403
1404 if (cd->card_thread != NULL) {
1405 kthread_stop(cd->card_thread);
1406 cd->card_thread = NULL;
1407 }
1408
1409 free_ddcb_queue(cd, &cd->queue);
1410 return 0;
1411}