blob: 4a500582eef09229532ebbee90cef80efd36538a [file] [log] [blame]
Frank Haverkamp67f4add2013-12-09 13:30:41 +01001/**
2 * IBM Accelerator Family 'GenWQE'
3 *
4 * (C) Copyright IBM Corp. 2013
5 *
6 * Author: Frank Haverkamp <haver@linux.vnet.ibm.com>
7 * Author: Joerg-Stephan Vogt <jsvogt@de.ibm.com>
8 * Author: Michael Jung <mijung@de.ibm.com>
9 * Author: Michael Ruettger <michael@ibmra.de>
10 *
11 * This program is free software; you can redistribute it and/or modify
12 * it under the terms of the GNU General Public License (version 2 only)
13 * as published by the Free Software Foundation.
14 *
15 * This program is distributed in the hope that it will be useful,
16 * but WITHOUT ANY WARRANTY; without even the implied warranty of
17 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18 * GNU General Public License for more details.
19 */
20
21/*
22 * Miscelanous functionality used in the other GenWQE driver parts.
23 */
24
25#include <linux/kernel.h>
26#include <linux/dma-mapping.h>
27#include <linux/sched.h>
28#include <linux/vmalloc.h>
29#include <linux/page-flags.h>
30#include <linux/scatterlist.h>
31#include <linux/hugetlb.h>
32#include <linux/iommu.h>
33#include <linux/delay.h>
34#include <linux/pci.h>
35#include <linux/dma-mapping.h>
36#include <linux/ctype.h>
37#include <linux/module.h>
38#include <linux/platform_device.h>
39#include <linux/delay.h>
40#include <asm/pgtable.h>
41
42#include "genwqe_driver.h"
43#include "card_base.h"
44#include "card_ddcb.h"
45
46/**
47 * __genwqe_writeq() - Write 64-bit register
48 * @cd: genwqe device descriptor
49 * @byte_offs: byte offset within BAR
50 * @val: 64-bit value
51 *
52 * Return: 0 if success; < 0 if error
53 */
54int __genwqe_writeq(struct genwqe_dev *cd, u64 byte_offs, u64 val)
55{
Kleber Sacilotto de Souzafb145452014-06-04 10:57:51 -030056 struct pci_dev *pci_dev = cd->pci_dev;
57
Frank Haverkamp67f4add2013-12-09 13:30:41 +010058 if (cd->err_inject & GENWQE_INJECT_HARDWARE_FAILURE)
59 return -EIO;
60
61 if (cd->mmio == NULL)
62 return -EIO;
63
Kleber Sacilotto de Souzafb145452014-06-04 10:57:51 -030064 if (pci_channel_offline(pci_dev))
65 return -EIO;
66
Frank Haverkampa45a0252013-12-20 20:27:20 +010067 __raw_writeq((__force u64)cpu_to_be64(val), cd->mmio + byte_offs);
Frank Haverkamp67f4add2013-12-09 13:30:41 +010068 return 0;
69}
70
71/**
72 * __genwqe_readq() - Read 64-bit register
73 * @cd: genwqe device descriptor
74 * @byte_offs: offset within BAR
75 *
76 * Return: value from register
77 */
78u64 __genwqe_readq(struct genwqe_dev *cd, u64 byte_offs)
79{
Frank Haverkamp67f4add2013-12-09 13:30:41 +010080 if (cd->err_inject & GENWQE_INJECT_HARDWARE_FAILURE)
81 return 0xffffffffffffffffull;
82
83 if ((cd->err_inject & GENWQE_INJECT_GFIR_FATAL) &&
84 (byte_offs == IO_SLC_CFGREG_GFIR))
85 return 0x000000000000ffffull;
86
87 if ((cd->err_inject & GENWQE_INJECT_GFIR_INFO) &&
88 (byte_offs == IO_SLC_CFGREG_GFIR))
89 return 0x00000000ffff0000ull;
90
91 if (cd->mmio == NULL)
92 return 0xffffffffffffffffull;
93
Frank Haverkamp58d66ce2013-12-20 16:26:10 +010094 return be64_to_cpu((__force __be64)__raw_readq(cd->mmio + byte_offs));
Frank Haverkamp67f4add2013-12-09 13:30:41 +010095}
96
97/**
98 * __genwqe_writel() - Write 32-bit register
99 * @cd: genwqe device descriptor
100 * @byte_offs: byte offset within BAR
101 * @val: 32-bit value
102 *
103 * Return: 0 if success; < 0 if error
104 */
105int __genwqe_writel(struct genwqe_dev *cd, u64 byte_offs, u32 val)
106{
Kleber Sacilotto de Souzafb145452014-06-04 10:57:51 -0300107 struct pci_dev *pci_dev = cd->pci_dev;
108
Frank Haverkamp67f4add2013-12-09 13:30:41 +0100109 if (cd->err_inject & GENWQE_INJECT_HARDWARE_FAILURE)
110 return -EIO;
111
112 if (cd->mmio == NULL)
113 return -EIO;
114
Kleber Sacilotto de Souzafb145452014-06-04 10:57:51 -0300115 if (pci_channel_offline(pci_dev))
116 return -EIO;
117
Frank Haverkamp58d66ce2013-12-20 16:26:10 +0100118 __raw_writel((__force u32)cpu_to_be32(val), cd->mmio + byte_offs);
Frank Haverkamp67f4add2013-12-09 13:30:41 +0100119 return 0;
120}
121
122/**
123 * __genwqe_readl() - Read 32-bit register
124 * @cd: genwqe device descriptor
125 * @byte_offs: offset within BAR
126 *
127 * Return: Value from register
128 */
129u32 __genwqe_readl(struct genwqe_dev *cd, u64 byte_offs)
130{
131 if (cd->err_inject & GENWQE_INJECT_HARDWARE_FAILURE)
132 return 0xffffffff;
133
134 if (cd->mmio == NULL)
135 return 0xffffffff;
136
Frank Haverkamp58d66ce2013-12-20 16:26:10 +0100137 return be32_to_cpu((__force __be32)__raw_readl(cd->mmio + byte_offs));
Frank Haverkamp67f4add2013-12-09 13:30:41 +0100138}
139
140/**
141 * genwqe_read_app_id() - Extract app_id
142 *
143 * app_unitcfg need to be filled with valid data first
144 */
145int genwqe_read_app_id(struct genwqe_dev *cd, char *app_name, int len)
146{
147 int i, j;
148 u32 app_id = (u32)cd->app_unitcfg;
149
150 memset(app_name, 0, len);
151 for (i = 0, j = 0; j < min(len, 4); j++) {
152 char ch = (char)((app_id >> (24 - j*8)) & 0xff);
153 if (ch == ' ')
154 continue;
155 app_name[i++] = isprint(ch) ? ch : 'X';
156 }
157 return i;
158}
159
160/**
161 * genwqe_init_crc32() - Prepare a lookup table for fast crc32 calculations
162 *
163 * Existing kernel functions seem to use a different polynom,
164 * therefore we could not use them here.
165 *
166 * Genwqe's Polynomial = 0x20044009
167 */
168#define CRC32_POLYNOMIAL 0x20044009
169static u32 crc32_tab[256]; /* crc32 lookup table */
170
171void genwqe_init_crc32(void)
172{
173 int i, j;
174 u32 crc;
175
176 for (i = 0; i < 256; i++) {
177 crc = i << 24;
178 for (j = 0; j < 8; j++) {
179 if (crc & 0x80000000)
180 crc = (crc << 1) ^ CRC32_POLYNOMIAL;
181 else
182 crc = (crc << 1);
183 }
184 crc32_tab[i] = crc;
185 }
186}
187
188/**
189 * genwqe_crc32() - Generate 32-bit crc as required for DDCBs
190 * @buff: pointer to data buffer
191 * @len: length of data for calculation
192 * @init: initial crc (0xffffffff at start)
193 *
194 * polynomial = x^32 * + x^29 + x^18 + x^14 + x^3 + 1 (0x20044009)
195
196 * Example: 4 bytes 0x01 0x02 0x03 0x04 with init=0xffffffff should
197 * result in a crc32 of 0xf33cb7d3.
198 *
199 * The existing kernel crc functions did not cover this polynom yet.
200 *
201 * Return: crc32 checksum.
202 */
203u32 genwqe_crc32(u8 *buff, size_t len, u32 init)
204{
205 int i;
206 u32 crc;
207
208 crc = init;
209 while (len--) {
210 i = ((crc >> 24) ^ *buff++) & 0xFF;
211 crc = (crc << 8) ^ crc32_tab[i];
212 }
213 return crc;
214}
215
216void *__genwqe_alloc_consistent(struct genwqe_dev *cd, size_t size,
217 dma_addr_t *dma_handle)
218{
219 if (get_order(size) > MAX_ORDER)
220 return NULL;
221
222 return pci_alloc_consistent(cd->pci_dev, size, dma_handle);
223}
224
225void __genwqe_free_consistent(struct genwqe_dev *cd, size_t size,
226 void *vaddr, dma_addr_t dma_handle)
227{
228 if (vaddr == NULL)
229 return;
230
231 pci_free_consistent(cd->pci_dev, size, vaddr, dma_handle);
232}
233
234static void genwqe_unmap_pages(struct genwqe_dev *cd, dma_addr_t *dma_list,
235 int num_pages)
236{
237 int i;
238 struct pci_dev *pci_dev = cd->pci_dev;
239
240 for (i = 0; (i < num_pages) && (dma_list[i] != 0x0); i++) {
241 pci_unmap_page(pci_dev, dma_list[i],
242 PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
243 dma_list[i] = 0x0;
244 }
245}
246
247static int genwqe_map_pages(struct genwqe_dev *cd,
248 struct page **page_list, int num_pages,
249 dma_addr_t *dma_list)
250{
251 int i;
252 struct pci_dev *pci_dev = cd->pci_dev;
253
254 /* establish DMA mapping for requested pages */
255 for (i = 0; i < num_pages; i++) {
256 dma_addr_t daddr;
257
258 dma_list[i] = 0x0;
259 daddr = pci_map_page(pci_dev, page_list[i],
260 0, /* map_offs */
261 PAGE_SIZE,
262 PCI_DMA_BIDIRECTIONAL); /* FIXME rd/rw */
263
264 if (pci_dma_mapping_error(pci_dev, daddr)) {
265 dev_err(&pci_dev->dev,
266 "[%s] err: no dma addr daddr=%016llx!\n",
267 __func__, (long long)daddr);
268 goto err;
269 }
270
271 dma_list[i] = daddr;
272 }
273 return 0;
274
275 err:
276 genwqe_unmap_pages(cd, dma_list, num_pages);
277 return -EIO;
278}
279
280static int genwqe_sgl_size(int num_pages)
281{
282 int len, num_tlb = num_pages / 7;
283
284 len = sizeof(struct sg_entry) * (num_pages+num_tlb + 1);
285 return roundup(len, PAGE_SIZE);
286}
287
Frank Haverkamp718f7622014-03-20 15:11:05 +0100288/**
289 * genwqe_alloc_sync_sgl() - Allocate memory for sgl and overlapping pages
290 *
291 * Allocates memory for sgl and overlapping pages. Pages which might
292 * overlap other user-space memory blocks are being cached for DMAs,
293 * such that we do not run into syncronization issues. Data is copied
294 * from user-space into the cached pages.
295 */
296int genwqe_alloc_sync_sgl(struct genwqe_dev *cd, struct genwqe_sgl *sgl,
297 void __user *user_addr, size_t user_size)
Frank Haverkamp67f4add2013-12-09 13:30:41 +0100298{
Frank Haverkamp718f7622014-03-20 15:11:05 +0100299 int rc;
Frank Haverkamp67f4add2013-12-09 13:30:41 +0100300 struct pci_dev *pci_dev = cd->pci_dev;
Frank Haverkamp67f4add2013-12-09 13:30:41 +0100301
Frank Haverkamp718f7622014-03-20 15:11:05 +0100302 sgl->fpage_offs = offset_in_page((unsigned long)user_addr);
303 sgl->fpage_size = min_t(size_t, PAGE_SIZE-sgl->fpage_offs, user_size);
304 sgl->nr_pages = DIV_ROUND_UP(sgl->fpage_offs + user_size, PAGE_SIZE);
305 sgl->lpage_size = (user_size - sgl->fpage_size) % PAGE_SIZE;
306
307 dev_dbg(&pci_dev->dev, "[%s] uaddr=%p usize=%8ld nr_pages=%ld "
308 "fpage_offs=%lx fpage_size=%ld lpage_size=%ld\n",
309 __func__, user_addr, user_size, sgl->nr_pages,
310 sgl->fpage_offs, sgl->fpage_size, sgl->lpage_size);
311
312 sgl->user_addr = user_addr;
313 sgl->user_size = user_size;
314 sgl->sgl_size = genwqe_sgl_size(sgl->nr_pages);
315
316 if (get_order(sgl->sgl_size) > MAX_ORDER) {
Frank Haverkamp67f4add2013-12-09 13:30:41 +0100317 dev_err(&pci_dev->dev,
318 "[%s] err: too much memory requested!\n", __func__);
Frank Haverkamp718f7622014-03-20 15:11:05 +0100319 return -ENOMEM;
Frank Haverkamp67f4add2013-12-09 13:30:41 +0100320 }
321
Frank Haverkamp718f7622014-03-20 15:11:05 +0100322 sgl->sgl = __genwqe_alloc_consistent(cd, sgl->sgl_size,
323 &sgl->sgl_dma_addr);
324 if (sgl->sgl == NULL) {
Frank Haverkamp67f4add2013-12-09 13:30:41 +0100325 dev_err(&pci_dev->dev,
326 "[%s] err: no memory available!\n", __func__);
Frank Haverkamp718f7622014-03-20 15:11:05 +0100327 return -ENOMEM;
Frank Haverkamp67f4add2013-12-09 13:30:41 +0100328 }
329
Frank Haverkamp718f7622014-03-20 15:11:05 +0100330 /* Only use buffering on incomplete pages */
331 if ((sgl->fpage_size != 0) && (sgl->fpage_size != PAGE_SIZE)) {
332 sgl->fpage = __genwqe_alloc_consistent(cd, PAGE_SIZE,
333 &sgl->fpage_dma_addr);
334 if (sgl->fpage == NULL)
335 goto err_out;
336
337 /* Sync with user memory */
338 if (copy_from_user(sgl->fpage + sgl->fpage_offs,
339 user_addr, sgl->fpage_size)) {
340 rc = -EFAULT;
341 goto err_out;
342 }
343 }
344 if (sgl->lpage_size != 0) {
345 sgl->lpage = __genwqe_alloc_consistent(cd, PAGE_SIZE,
346 &sgl->lpage_dma_addr);
347 if (sgl->lpage == NULL)
348 goto err_out1;
349
350 /* Sync with user memory */
351 if (copy_from_user(sgl->lpage, user_addr + user_size -
352 sgl->lpage_size, sgl->lpage_size)) {
353 rc = -EFAULT;
354 goto err_out1;
355 }
356 }
357 return 0;
358
359 err_out1:
360 __genwqe_free_consistent(cd, PAGE_SIZE, sgl->fpage,
361 sgl->fpage_dma_addr);
362 err_out:
363 __genwqe_free_consistent(cd, sgl->sgl_size, sgl->sgl,
364 sgl->sgl_dma_addr);
365 return -ENOMEM;
Frank Haverkamp67f4add2013-12-09 13:30:41 +0100366}
367
Frank Haverkamp718f7622014-03-20 15:11:05 +0100368int genwqe_setup_sgl(struct genwqe_dev *cd, struct genwqe_sgl *sgl,
369 dma_addr_t *dma_list)
Frank Haverkamp67f4add2013-12-09 13:30:41 +0100370{
371 int i = 0, j = 0, p;
372 unsigned long dma_offs, map_offs;
Frank Haverkamp67f4add2013-12-09 13:30:41 +0100373 dma_addr_t prev_daddr = 0;
374 struct sg_entry *s, *last_s = NULL;
Frank Haverkamp718f7622014-03-20 15:11:05 +0100375 size_t size = sgl->user_size;
Frank Haverkamp67f4add2013-12-09 13:30:41 +0100376
377 dma_offs = 128; /* next block if needed/dma_offset */
Frank Haverkamp718f7622014-03-20 15:11:05 +0100378 map_offs = sgl->fpage_offs; /* offset in first page */
Frank Haverkamp67f4add2013-12-09 13:30:41 +0100379
Frank Haverkamp718f7622014-03-20 15:11:05 +0100380 s = &sgl->sgl[0]; /* first set of 8 entries */
Frank Haverkamp67f4add2013-12-09 13:30:41 +0100381 p = 0; /* page */
Frank Haverkamp718f7622014-03-20 15:11:05 +0100382 while (p < sgl->nr_pages) {
Frank Haverkamp67f4add2013-12-09 13:30:41 +0100383 dma_addr_t daddr;
384 unsigned int size_to_map;
385
386 /* always write the chaining entry, cleanup is done later */
387 j = 0;
Frank Haverkamp718f7622014-03-20 15:11:05 +0100388 s[j].target_addr = cpu_to_be64(sgl->sgl_dma_addr + dma_offs);
Frank Haverkamp67f4add2013-12-09 13:30:41 +0100389 s[j].len = cpu_to_be32(128);
390 s[j].flags = cpu_to_be32(SG_CHAINED);
391 j++;
392
393 while (j < 8) {
394 /* DMA mapping for requested page, offs, size */
395 size_to_map = min(size, PAGE_SIZE - map_offs);
Frank Haverkamp718f7622014-03-20 15:11:05 +0100396
397 if ((p == 0) && (sgl->fpage != NULL)) {
398 daddr = sgl->fpage_dma_addr + map_offs;
399
400 } else if ((p == sgl->nr_pages - 1) &&
401 (sgl->lpage != NULL)) {
402 daddr = sgl->lpage_dma_addr;
403 } else {
404 daddr = dma_list[p] + map_offs;
405 }
406
Frank Haverkamp67f4add2013-12-09 13:30:41 +0100407 size -= size_to_map;
408 map_offs = 0;
409
410 if (prev_daddr == daddr) {
411 u32 prev_len = be32_to_cpu(last_s->len);
412
413 /* pr_info("daddr combining: "
414 "%016llx/%08x -> %016llx\n",
415 prev_daddr, prev_len, daddr); */
416
417 last_s->len = cpu_to_be32(prev_len +
418 size_to_map);
419
420 p++; /* process next page */
Frank Haverkamp718f7622014-03-20 15:11:05 +0100421 if (p == sgl->nr_pages)
Frank Haverkamp67f4add2013-12-09 13:30:41 +0100422 goto fixup; /* nothing to do */
423
424 prev_daddr = daddr + size_to_map;
425 continue;
426 }
427
428 /* start new entry */
429 s[j].target_addr = cpu_to_be64(daddr);
430 s[j].len = cpu_to_be32(size_to_map);
431 s[j].flags = cpu_to_be32(SG_DATA);
432 prev_daddr = daddr + size_to_map;
433 last_s = &s[j];
434 j++;
435
436 p++; /* process next page */
Frank Haverkamp718f7622014-03-20 15:11:05 +0100437 if (p == sgl->nr_pages)
Frank Haverkamp67f4add2013-12-09 13:30:41 +0100438 goto fixup; /* nothing to do */
439 }
440 dma_offs += 128;
441 s += 8; /* continue 8 elements further */
442 }
443 fixup:
444 if (j == 1) { /* combining happend on last entry! */
445 s -= 8; /* full shift needed on previous sgl block */
446 j = 7; /* shift all elements */
447 }
448
449 for (i = 0; i < j; i++) /* move elements 1 up */
450 s[i] = s[i + 1];
451
452 s[i].target_addr = cpu_to_be64(0);
453 s[i].len = cpu_to_be32(0);
454 s[i].flags = cpu_to_be32(SG_END_LIST);
455 return 0;
456}
457
Frank Haverkamp718f7622014-03-20 15:11:05 +0100458/**
459 * genwqe_free_sync_sgl() - Free memory for sgl and overlapping pages
460 *
461 * After the DMA transfer has been completed we free the memory for
462 * the sgl and the cached pages. Data is being transfered from cached
463 * pages into user-space buffers.
464 */
465int genwqe_free_sync_sgl(struct genwqe_dev *cd, struct genwqe_sgl *sgl)
Frank Haverkamp67f4add2013-12-09 13:30:41 +0100466{
Christian Engelmayer63fa80c2014-05-13 22:47:03 +0200467 int rc = 0;
Frank Haverkamp718f7622014-03-20 15:11:05 +0100468 struct pci_dev *pci_dev = cd->pci_dev;
469
470 if (sgl->fpage) {
471 if (copy_to_user(sgl->user_addr, sgl->fpage + sgl->fpage_offs,
472 sgl->fpage_size)) {
473 dev_err(&pci_dev->dev, "[%s] err: copying fpage!\n",
474 __func__);
475 rc = -EFAULT;
476 }
477 __genwqe_free_consistent(cd, PAGE_SIZE, sgl->fpage,
478 sgl->fpage_dma_addr);
479 sgl->fpage = NULL;
480 sgl->fpage_dma_addr = 0;
481 }
482 if (sgl->lpage) {
483 if (copy_to_user(sgl->user_addr + sgl->user_size -
484 sgl->lpage_size, sgl->lpage,
485 sgl->lpage_size)) {
486 dev_err(&pci_dev->dev, "[%s] err: copying lpage!\n",
487 __func__);
488 rc = -EFAULT;
489 }
490 __genwqe_free_consistent(cd, PAGE_SIZE, sgl->lpage,
491 sgl->lpage_dma_addr);
492 sgl->lpage = NULL;
493 sgl->lpage_dma_addr = 0;
494 }
495 __genwqe_free_consistent(cd, sgl->sgl_size, sgl->sgl,
496 sgl->sgl_dma_addr);
497
498 sgl->sgl = NULL;
499 sgl->sgl_dma_addr = 0x0;
500 sgl->sgl_size = 0;
501 return rc;
Frank Haverkamp67f4add2013-12-09 13:30:41 +0100502}
503
504/**
505 * free_user_pages() - Give pinned pages back
506 *
507 * Documentation of get_user_pages is in mm/memory.c:
508 *
509 * If the page is written to, set_page_dirty (or set_page_dirty_lock,
510 * as appropriate) must be called after the page is finished with, and
511 * before put_page is called.
512 *
513 * FIXME Could be of use to others and might belong in the generic
514 * code, if others agree. E.g.
515 * ll_free_user_pages in drivers/staging/lustre/lustre/llite/rw26.c
516 * ceph_put_page_vector in net/ceph/pagevec.c
517 * maybe more?
518 */
519static int free_user_pages(struct page **page_list, unsigned int nr_pages,
520 int dirty)
521{
522 unsigned int i;
523
524 for (i = 0; i < nr_pages; i++) {
525 if (page_list[i] != NULL) {
526 if (dirty)
527 set_page_dirty_lock(page_list[i]);
528 put_page(page_list[i]);
529 }
530 }
531 return 0;
532}
533
534/**
535 * genwqe_user_vmap() - Map user-space memory to virtual kernel memory
536 * @cd: pointer to genwqe device
537 * @m: mapping params
538 * @uaddr: user virtual address
539 * @size: size of memory to be mapped
540 *
541 * We need to think about how we could speed this up. Of course it is
542 * not a good idea to do this over and over again, like we are
543 * currently doing it. Nevertheless, I am curious where on the path
544 * the performance is spend. Most probably within the memory
545 * allocation functions, but maybe also in the DMA mapping code.
546 *
547 * Restrictions: The maximum size of the possible mapping currently depends
548 * on the amount of memory we can get using kzalloc() for the
549 * page_list and pci_alloc_consistent for the sg_list.
550 * The sg_list is currently itself not scattered, which could
551 * be fixed with some effort. The page_list must be split into
552 * PAGE_SIZE chunks too. All that will make the complicated
553 * code more complicated.
554 *
555 * Return: 0 if success
556 */
557int genwqe_user_vmap(struct genwqe_dev *cd, struct dma_mapping *m, void *uaddr,
558 unsigned long size, struct ddcb_requ *req)
559{
560 int rc = -EINVAL;
561 unsigned long data, offs;
562 struct pci_dev *pci_dev = cd->pci_dev;
563
564 if ((uaddr == NULL) || (size == 0)) {
565 m->size = 0; /* mark unused and not added */
566 return -EINVAL;
567 }
568 m->u_vaddr = uaddr;
569 m->size = size;
570
571 /* determine space needed for page_list. */
572 data = (unsigned long)uaddr;
573 offs = offset_in_page(data);
574 m->nr_pages = DIV_ROUND_UP(offs + size, PAGE_SIZE);
575
576 m->page_list = kcalloc(m->nr_pages,
577 sizeof(struct page *) + sizeof(dma_addr_t),
578 GFP_KERNEL);
579 if (!m->page_list) {
580 dev_err(&pci_dev->dev, "err: alloc page_list failed\n");
581 m->nr_pages = 0;
582 m->u_vaddr = NULL;
583 m->size = 0; /* mark unused and not added */
584 return -ENOMEM;
585 }
586 m->dma_list = (dma_addr_t *)(m->page_list + m->nr_pages);
587
588 /* pin user pages in memory */
589 rc = get_user_pages_fast(data & PAGE_MASK, /* page aligned addr */
590 m->nr_pages,
591 1, /* write by caller */
592 m->page_list); /* ptrs to pages */
593
594 /* assumption: get_user_pages can be killed by signals. */
595 if (rc < m->nr_pages) {
596 free_user_pages(m->page_list, rc, 0);
597 rc = -EFAULT;
598 goto fail_get_user_pages;
599 }
600
601 rc = genwqe_map_pages(cd, m->page_list, m->nr_pages, m->dma_list);
602 if (rc != 0)
603 goto fail_free_user_pages;
604
605 return 0;
606
607 fail_free_user_pages:
608 free_user_pages(m->page_list, m->nr_pages, 0);
609
610 fail_get_user_pages:
611 kfree(m->page_list);
612 m->page_list = NULL;
613 m->dma_list = NULL;
614 m->nr_pages = 0;
615 m->u_vaddr = NULL;
616 m->size = 0; /* mark unused and not added */
617 return rc;
618}
619
620/**
621 * genwqe_user_vunmap() - Undo mapping of user-space mem to virtual kernel
622 * memory
623 * @cd: pointer to genwqe device
624 * @m: mapping params
625 */
626int genwqe_user_vunmap(struct genwqe_dev *cd, struct dma_mapping *m,
627 struct ddcb_requ *req)
628{
629 struct pci_dev *pci_dev = cd->pci_dev;
630
631 if (!dma_mapping_used(m)) {
632 dev_err(&pci_dev->dev, "[%s] err: mapping %p not used!\n",
633 __func__, m);
634 return -EINVAL;
635 }
636
637 if (m->dma_list)
638 genwqe_unmap_pages(cd, m->dma_list, m->nr_pages);
639
640 if (m->page_list) {
641 free_user_pages(m->page_list, m->nr_pages, 1);
642
643 kfree(m->page_list);
644 m->page_list = NULL;
645 m->dma_list = NULL;
646 m->nr_pages = 0;
647 }
648
649 m->u_vaddr = NULL;
650 m->size = 0; /* mark as unused and not added */
651 return 0;
652}
653
654/**
655 * genwqe_card_type() - Get chip type SLU Configuration Register
656 * @cd: pointer to the genwqe device descriptor
657 * Return: 0: Altera Stratix-IV 230
658 * 1: Altera Stratix-IV 530
659 * 2: Altera Stratix-V A4
660 * 3: Altera Stratix-V A7
661 */
662u8 genwqe_card_type(struct genwqe_dev *cd)
663{
664 u64 card_type = cd->slu_unitcfg;
665 return (u8)((card_type & IO_SLU_UNITCFG_TYPE_MASK) >> 20);
666}
667
668/**
669 * genwqe_card_reset() - Reset the card
670 * @cd: pointer to the genwqe device descriptor
671 */
672int genwqe_card_reset(struct genwqe_dev *cd)
673{
674 u64 softrst;
675 struct pci_dev *pci_dev = cd->pci_dev;
676
677 if (!genwqe_is_privileged(cd))
678 return -ENODEV;
679
680 /* new SL */
681 __genwqe_writeq(cd, IO_SLC_CFGREG_SOFTRESET, 0x1ull);
682 msleep(1000);
683 __genwqe_readq(cd, IO_HSU_FIR_CLR);
684 __genwqe_readq(cd, IO_APP_FIR_CLR);
685 __genwqe_readq(cd, IO_SLU_FIR_CLR);
686
687 /*
688 * Read-modify-write to preserve the stealth bits
689 *
690 * For SL >= 039, Stealth WE bit allows removing
691 * the read-modify-wrote.
692 * r-m-w may require a mask 0x3C to avoid hitting hard
693 * reset again for error reset (should be 0, chicken).
694 */
695 softrst = __genwqe_readq(cd, IO_SLC_CFGREG_SOFTRESET) & 0x3cull;
696 __genwqe_writeq(cd, IO_SLC_CFGREG_SOFTRESET, softrst | 0x2ull);
697
698 /* give ERRORRESET some time to finish */
699 msleep(50);
700
701 if (genwqe_need_err_masking(cd)) {
702 dev_info(&pci_dev->dev,
703 "[%s] masking errors for old bitstreams\n", __func__);
704 __genwqe_writeq(cd, IO_SLC_MISC_DEBUG, 0x0aull);
705 }
706 return 0;
707}
708
709int genwqe_read_softreset(struct genwqe_dev *cd)
710{
711 u64 bitstream;
712
713 if (!genwqe_is_privileged(cd))
714 return -ENODEV;
715
716 bitstream = __genwqe_readq(cd, IO_SLU_BITSTREAM) & 0x1;
717 cd->softreset = (bitstream == 0) ? 0x8ull : 0xcull;
718 return 0;
719}
720
721/**
722 * genwqe_set_interrupt_capability() - Configure MSI capability structure
723 * @cd: pointer to the device
724 * Return: 0 if no error
725 */
726int genwqe_set_interrupt_capability(struct genwqe_dev *cd, int count)
727{
728 int rc;
729 struct pci_dev *pci_dev = cd->pci_dev;
730
Alexander Gordeeva30d0102014-04-14 09:14:06 +0200731 rc = pci_enable_msi_exact(pci_dev, count);
Frank Haverkamp67f4add2013-12-09 13:30:41 +0100732 if (rc == 0)
733 cd->flags |= GENWQE_FLAG_MSI_ENABLED;
734 return rc;
735}
736
737/**
738 * genwqe_reset_interrupt_capability() - Undo genwqe_set_interrupt_capability()
739 * @cd: pointer to the device
740 */
741void genwqe_reset_interrupt_capability(struct genwqe_dev *cd)
742{
743 struct pci_dev *pci_dev = cd->pci_dev;
744
745 if (cd->flags & GENWQE_FLAG_MSI_ENABLED) {
746 pci_disable_msi(pci_dev);
747 cd->flags &= ~GENWQE_FLAG_MSI_ENABLED;
748 }
749}
750
751/**
752 * set_reg_idx() - Fill array with data. Ignore illegal offsets.
753 * @cd: card device
754 * @r: debug register array
755 * @i: index to desired entry
756 * @m: maximum possible entries
757 * @addr: addr which is read
758 * @index: index in debug array
759 * @val: read value
760 */
761static int set_reg_idx(struct genwqe_dev *cd, struct genwqe_reg *r,
762 unsigned int *i, unsigned int m, u32 addr, u32 idx,
763 u64 val)
764{
765 if (WARN_ON_ONCE(*i >= m))
766 return -EFAULT;
767
768 r[*i].addr = addr;
769 r[*i].idx = idx;
770 r[*i].val = val;
771 ++*i;
772 return 0;
773}
774
775static int set_reg(struct genwqe_dev *cd, struct genwqe_reg *r,
776 unsigned int *i, unsigned int m, u32 addr, u64 val)
777{
778 return set_reg_idx(cd, r, i, m, addr, 0, val);
779}
780
781int genwqe_read_ffdc_regs(struct genwqe_dev *cd, struct genwqe_reg *regs,
782 unsigned int max_regs, int all)
783{
784 unsigned int i, j, idx = 0;
785 u32 ufir_addr, ufec_addr, sfir_addr, sfec_addr;
786 u64 gfir, sluid, appid, ufir, ufec, sfir, sfec;
787
788 /* Global FIR */
789 gfir = __genwqe_readq(cd, IO_SLC_CFGREG_GFIR);
790 set_reg(cd, regs, &idx, max_regs, IO_SLC_CFGREG_GFIR, gfir);
791
792 /* UnitCfg for SLU */
793 sluid = __genwqe_readq(cd, IO_SLU_UNITCFG); /* 0x00000000 */
794 set_reg(cd, regs, &idx, max_regs, IO_SLU_UNITCFG, sluid);
795
796 /* UnitCfg for APP */
797 appid = __genwqe_readq(cd, IO_APP_UNITCFG); /* 0x02000000 */
798 set_reg(cd, regs, &idx, max_regs, IO_APP_UNITCFG, appid);
799
800 /* Check all chip Units */
801 for (i = 0; i < GENWQE_MAX_UNITS; i++) {
802
803 /* Unit FIR */
804 ufir_addr = (i << 24) | 0x008;
805 ufir = __genwqe_readq(cd, ufir_addr);
806 set_reg(cd, regs, &idx, max_regs, ufir_addr, ufir);
807
808 /* Unit FEC */
809 ufec_addr = (i << 24) | 0x018;
810 ufec = __genwqe_readq(cd, ufec_addr);
811 set_reg(cd, regs, &idx, max_regs, ufec_addr, ufec);
812
813 for (j = 0; j < 64; j++) {
814 /* wherever there is a primary 1, read the 2ndary */
815 if (!all && (!(ufir & (1ull << j))))
816 continue;
817
818 sfir_addr = (i << 24) | (0x100 + 8 * j);
819 sfir = __genwqe_readq(cd, sfir_addr);
820 set_reg(cd, regs, &idx, max_regs, sfir_addr, sfir);
821
822 sfec_addr = (i << 24) | (0x300 + 8 * j);
823 sfec = __genwqe_readq(cd, sfec_addr);
824 set_reg(cd, regs, &idx, max_regs, sfec_addr, sfec);
825 }
826 }
827
828 /* fill with invalid data until end */
829 for (i = idx; i < max_regs; i++) {
830 regs[i].addr = 0xffffffff;
831 regs[i].val = 0xffffffffffffffffull;
832 }
833 return idx;
834}
835
836/**
837 * genwqe_ffdc_buff_size() - Calculates the number of dump registers
838 */
839int genwqe_ffdc_buff_size(struct genwqe_dev *cd, int uid)
840{
841 int entries = 0, ring, traps, traces, trace_entries;
842 u32 eevptr_addr, l_addr, d_len, d_type;
843 u64 eevptr, val, addr;
844
845 eevptr_addr = GENWQE_UID_OFFS(uid) | IO_EXTENDED_ERROR_POINTER;
846 eevptr = __genwqe_readq(cd, eevptr_addr);
847
848 if ((eevptr != 0x0) && (eevptr != -1ull)) {
849 l_addr = GENWQE_UID_OFFS(uid) | eevptr;
850
851 while (1) {
852 val = __genwqe_readq(cd, l_addr);
853
854 if ((val == 0x0) || (val == -1ull))
855 break;
856
857 /* 38:24 */
858 d_len = (val & 0x0000007fff000000ull) >> 24;
859
860 /* 39 */
861 d_type = (val & 0x0000008000000000ull) >> 36;
862
863 if (d_type) { /* repeat */
864 entries += d_len;
865 } else { /* size in bytes! */
866 entries += d_len >> 3;
867 }
868
869 l_addr += 8;
870 }
871 }
872
873 for (ring = 0; ring < 8; ring++) {
874 addr = GENWQE_UID_OFFS(uid) | IO_EXTENDED_DIAG_MAP(ring);
875 val = __genwqe_readq(cd, addr);
876
877 if ((val == 0x0ull) || (val == -1ull))
878 continue;
879
880 traps = (val >> 24) & 0xff;
881 traces = (val >> 16) & 0xff;
882 trace_entries = val & 0xffff;
883
884 entries += traps + (traces * trace_entries);
885 }
886 return entries;
887}
888
889/**
890 * genwqe_ffdc_buff_read() - Implements LogoutExtendedErrorRegisters procedure
891 */
892int genwqe_ffdc_buff_read(struct genwqe_dev *cd, int uid,
893 struct genwqe_reg *regs, unsigned int max_regs)
894{
895 int i, traps, traces, trace, trace_entries, trace_entry, ring;
896 unsigned int idx = 0;
897 u32 eevptr_addr, l_addr, d_addr, d_len, d_type;
898 u64 eevptr, e, val, addr;
899
900 eevptr_addr = GENWQE_UID_OFFS(uid) | IO_EXTENDED_ERROR_POINTER;
901 eevptr = __genwqe_readq(cd, eevptr_addr);
902
903 if ((eevptr != 0x0) && (eevptr != 0xffffffffffffffffull)) {
904 l_addr = GENWQE_UID_OFFS(uid) | eevptr;
905 while (1) {
906 e = __genwqe_readq(cd, l_addr);
907 if ((e == 0x0) || (e == 0xffffffffffffffffull))
908 break;
909
910 d_addr = (e & 0x0000000000ffffffull); /* 23:0 */
911 d_len = (e & 0x0000007fff000000ull) >> 24; /* 38:24 */
912 d_type = (e & 0x0000008000000000ull) >> 36; /* 39 */
913 d_addr |= GENWQE_UID_OFFS(uid);
914
915 if (d_type) {
916 for (i = 0; i < (int)d_len; i++) {
917 val = __genwqe_readq(cd, d_addr);
918 set_reg_idx(cd, regs, &idx, max_regs,
919 d_addr, i, val);
920 }
921 } else {
922 d_len >>= 3; /* Size in bytes! */
923 for (i = 0; i < (int)d_len; i++, d_addr += 8) {
924 val = __genwqe_readq(cd, d_addr);
925 set_reg_idx(cd, regs, &idx, max_regs,
926 d_addr, 0, val);
927 }
928 }
929 l_addr += 8;
930 }
931 }
932
933 /*
934 * To save time, there are only 6 traces poplulated on Uid=2,
935 * Ring=1. each with iters=512.
936 */
937 for (ring = 0; ring < 8; ring++) { /* 0 is fls, 1 is fds,
938 2...7 are ASI rings */
939 addr = GENWQE_UID_OFFS(uid) | IO_EXTENDED_DIAG_MAP(ring);
940 val = __genwqe_readq(cd, addr);
941
942 if ((val == 0x0ull) || (val == -1ull))
943 continue;
944
945 traps = (val >> 24) & 0xff; /* Number of Traps */
946 traces = (val >> 16) & 0xff; /* Number of Traces */
947 trace_entries = val & 0xffff; /* Entries per trace */
948
949 /* Note: This is a combined loop that dumps both the traps */
950 /* (for the trace == 0 case) as well as the traces 1 to */
951 /* 'traces'. */
952 for (trace = 0; trace <= traces; trace++) {
953 u32 diag_sel =
954 GENWQE_EXTENDED_DIAG_SELECTOR(ring, trace);
955
956 addr = (GENWQE_UID_OFFS(uid) |
957 IO_EXTENDED_DIAG_SELECTOR);
958 __genwqe_writeq(cd, addr, diag_sel);
959
960 for (trace_entry = 0;
961 trace_entry < (trace ? trace_entries : traps);
962 trace_entry++) {
963 addr = (GENWQE_UID_OFFS(uid) |
964 IO_EXTENDED_DIAG_READ_MBX);
965 val = __genwqe_readq(cd, addr);
966 set_reg_idx(cd, regs, &idx, max_regs, addr,
967 (diag_sel<<16) | trace_entry, val);
968 }
969 }
970 }
971 return 0;
972}
973
974/**
975 * genwqe_write_vreg() - Write register in virtual window
976 *
977 * Note, these registers are only accessible to the PF through the
978 * VF-window. It is not intended for the VF to access.
979 */
980int genwqe_write_vreg(struct genwqe_dev *cd, u32 reg, u64 val, int func)
981{
982 __genwqe_writeq(cd, IO_PF_SLC_VIRTUAL_WINDOW, func & 0xf);
983 __genwqe_writeq(cd, reg, val);
984 return 0;
985}
986
987/**
988 * genwqe_read_vreg() - Read register in virtual window
989 *
990 * Note, these registers are only accessible to the PF through the
991 * VF-window. It is not intended for the VF to access.
992 */
993u64 genwqe_read_vreg(struct genwqe_dev *cd, u32 reg, int func)
994{
995 __genwqe_writeq(cd, IO_PF_SLC_VIRTUAL_WINDOW, func & 0xf);
996 return __genwqe_readq(cd, reg);
997}
998
999/**
1000 * genwqe_base_clock_frequency() - Deteremine base clock frequency of the card
1001 *
1002 * Note: From a design perspective it turned out to be a bad idea to
1003 * use codes here to specifiy the frequency/speed values. An old
1004 * driver cannot understand new codes and is therefore always a
1005 * problem. Better is to measure out the value or put the
1006 * speed/frequency directly into a register which is always a valid
1007 * value for old as well as for new software.
1008 *
1009 * Return: Card clock in MHz
1010 */
1011int genwqe_base_clock_frequency(struct genwqe_dev *cd)
1012{
1013 u16 speed; /* MHz MHz MHz MHz */
1014 static const int speed_grade[] = { 250, 200, 166, 175 };
1015
1016 speed = (u16)((cd->slu_unitcfg >> 28) & 0x0full);
1017 if (speed >= ARRAY_SIZE(speed_grade))
1018 return 0; /* illegal value */
1019
1020 return speed_grade[speed];
1021}
1022
1023/**
1024 * genwqe_stop_traps() - Stop traps
1025 *
1026 * Before reading out the analysis data, we need to stop the traps.
1027 */
1028void genwqe_stop_traps(struct genwqe_dev *cd)
1029{
1030 __genwqe_writeq(cd, IO_SLC_MISC_DEBUG_SET, 0xcull);
1031}
1032
1033/**
1034 * genwqe_start_traps() - Start traps
1035 *
1036 * After having read the data, we can/must enable the traps again.
1037 */
1038void genwqe_start_traps(struct genwqe_dev *cd)
1039{
1040 __genwqe_writeq(cd, IO_SLC_MISC_DEBUG_CLR, 0xcull);
1041
1042 if (genwqe_need_err_masking(cd))
1043 __genwqe_writeq(cd, IO_SLC_MISC_DEBUG, 0x0aull);
1044}