blob: 1cb6e51e6bda97aadb4ba88bd46123544dcc4140 [file] [log] [blame]
Jon Masonfce8a7b2012-11-16 19:27:12 -07001/*
2 * This file is provided under a dual BSD/GPLv2 license. When using or
3 * redistributing this file, you may do so under either license.
4 *
5 * GPL LICENSE SUMMARY
6 *
7 * Copyright(c) 2012 Intel Corporation. All rights reserved.
8 *
9 * This program is free software; you can redistribute it and/or modify
10 * it under the terms of version 2 of the GNU General Public License as
11 * published by the Free Software Foundation.
12 *
13 * BSD LICENSE
14 *
15 * Copyright(c) 2012 Intel Corporation. All rights reserved.
16 *
17 * Redistribution and use in source and binary forms, with or without
18 * modification, are permitted provided that the following conditions
19 * are met:
20 *
21 * * Redistributions of source code must retain the above copyright
22 * notice, this list of conditions and the following disclaimer.
23 * * Redistributions in binary form must reproduce the above copy
24 * notice, this list of conditions and the following disclaimer in
25 * the documentation and/or other materials provided with the
26 * distribution.
27 * * Neither the name of Intel Corporation nor the names of its
28 * contributors may be used to endorse or promote products derived
29 * from this software without specific prior written permission.
30 *
31 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
32 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
33 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
34 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
35 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
36 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
37 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
38 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
39 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
40 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
41 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
42 *
43 * Intel PCIe NTB Linux driver
44 *
45 * Contact Information:
46 * Jon Mason <jon.mason@intel.com>
47 */
48#include <linux/debugfs.h>
Jon Mason113bf1c2012-11-16 18:52:57 -070049#include <linux/delay.h>
Jon Masonfce8a7b2012-11-16 19:27:12 -070050#include <linux/init.h>
51#include <linux/interrupt.h>
52#include <linux/module.h>
53#include <linux/pci.h>
Jon Mason113bf1c2012-11-16 18:52:57 -070054#include <linux/random.h>
Jon Masonfce8a7b2012-11-16 19:27:12 -070055#include <linux/slab.h>
56#include "ntb_hw.h"
57#include "ntb_regs.h"
58
59#define NTB_NAME "Intel(R) PCI-E Non-Transparent Bridge Driver"
Jon Masondb3bb3f2013-07-30 15:44:05 -070060#define NTB_VER "1.0"
Jon Masonfce8a7b2012-11-16 19:27:12 -070061
62MODULE_DESCRIPTION(NTB_NAME);
63MODULE_VERSION(NTB_VER);
64MODULE_LICENSE("Dual BSD/GPL");
65MODULE_AUTHOR("Intel Corporation");
66
Jon Mason948d3a62013-04-18 17:07:36 -070067static bool xeon_errata_workaround = true;
68module_param(xeon_errata_workaround, bool, 0644);
69MODULE_PARM_DESC(xeon_errata_workaround, "Workaround for the Xeon Errata");
70
Jon Masonfce8a7b2012-11-16 19:27:12 -070071enum {
Jon Masoned6c24e2013-07-15 16:43:54 -070072 NTB_CONN_TRANSPARENT = 0,
Jon Masonfce8a7b2012-11-16 19:27:12 -070073 NTB_CONN_B2B,
74 NTB_CONN_RP,
75};
76
77enum {
78 NTB_DEV_USD = 0,
79 NTB_DEV_DSD,
80};
81
82enum {
83 SNB_HW = 0,
84 BWD_HW,
85};
86
Jon Mason1517a3f2013-07-30 15:58:49 -070087static struct dentry *debugfs_dir;
88
Jon Mason113bf1c2012-11-16 18:52:57 -070089#define BWD_LINK_RECOVERY_TIME 500
90
Jon Masonfce8a7b2012-11-16 19:27:12 -070091/* Translate memory window 0,1 to BAR 2,4 */
Jon Mason948d3a62013-04-18 17:07:36 -070092#define MW_TO_BAR(mw) (mw * NTB_MAX_NUM_MW + 2)
Jon Masonfce8a7b2012-11-16 19:27:12 -070093
94static DEFINE_PCI_DEVICE_TABLE(ntb_pci_tbl) = {
95 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_BWD)},
96 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_JSF)},
Jon Masonfce8a7b2012-11-16 19:27:12 -070097 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_SNB)},
Jon Masonbe4dac02012-09-28 11:38:48 -070098 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_IVT)},
99 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_HSX)},
100 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_PS_JSF)},
101 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_PS_SNB)},
102 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_PS_IVT)},
103 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_PS_HSX)},
104 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_SS_JSF)},
105 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_SS_SNB)},
106 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_SS_IVT)},
107 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_SS_HSX)},
Jon Masonfce8a7b2012-11-16 19:27:12 -0700108 {0}
109};
110MODULE_DEVICE_TABLE(pci, ntb_pci_tbl);
111
112/**
113 * ntb_register_event_callback() - register event callback
114 * @ndev: pointer to ntb_device instance
115 * @func: callback function to register
116 *
117 * This function registers a callback for any HW driver events such as link
118 * up/down, power management notices and etc.
119 *
120 * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
121 */
122int ntb_register_event_callback(struct ntb_device *ndev,
Jon Mason74465642013-01-21 15:28:52 -0700123 void (*func)(void *handle, enum ntb_hw_event event))
Jon Masonfce8a7b2012-11-16 19:27:12 -0700124{
125 if (ndev->event_cb)
126 return -EINVAL;
127
128 ndev->event_cb = func;
129
130 return 0;
131}
132
133/**
134 * ntb_unregister_event_callback() - unregisters the event callback
135 * @ndev: pointer to ntb_device instance
136 *
137 * This function unregisters the existing callback from transport
138 */
139void ntb_unregister_event_callback(struct ntb_device *ndev)
140{
141 ndev->event_cb = NULL;
142}
143
144/**
145 * ntb_register_db_callback() - register a callback for doorbell interrupt
146 * @ndev: pointer to ntb_device instance
147 * @idx: doorbell index to register callback, zero based
Jon Masonf9a2cf82013-07-29 16:46:43 -0700148 * @data: pointer to be returned to caller with every callback
Jon Masonfce8a7b2012-11-16 19:27:12 -0700149 * @func: callback function to register
150 *
151 * This function registers a callback function for the doorbell interrupt
152 * on the primary side. The function will unmask the doorbell as well to
153 * allow interrupt.
154 *
155 * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
156 */
157int ntb_register_db_callback(struct ntb_device *ndev, unsigned int idx,
158 void *data, void (*func)(void *data, int db_num))
159{
160 unsigned long mask;
161
162 if (idx >= ndev->max_cbs || ndev->db_cb[idx].callback) {
163 dev_warn(&ndev->pdev->dev, "Invalid Index.\n");
164 return -EINVAL;
165 }
166
167 ndev->db_cb[idx].callback = func;
168 ndev->db_cb[idx].data = data;
169
170 /* unmask interrupt */
Jon Mason49793882013-07-15 15:53:54 -0700171 mask = readw(ndev->reg_ofs.ldb_mask);
Jon Masonfce8a7b2012-11-16 19:27:12 -0700172 clear_bit(idx * ndev->bits_per_vector, &mask);
Jon Mason49793882013-07-15 15:53:54 -0700173 writew(mask, ndev->reg_ofs.ldb_mask);
Jon Masonfce8a7b2012-11-16 19:27:12 -0700174
175 return 0;
176}
177
178/**
179 * ntb_unregister_db_callback() - unregister a callback for doorbell interrupt
180 * @ndev: pointer to ntb_device instance
181 * @idx: doorbell index to register callback, zero based
182 *
183 * This function unregisters a callback function for the doorbell interrupt
184 * on the primary side. The function will also mask the said doorbell.
185 */
186void ntb_unregister_db_callback(struct ntb_device *ndev, unsigned int idx)
187{
188 unsigned long mask;
189
190 if (idx >= ndev->max_cbs || !ndev->db_cb[idx].callback)
191 return;
192
Jon Mason49793882013-07-15 15:53:54 -0700193 mask = readw(ndev->reg_ofs.ldb_mask);
Jon Masonfce8a7b2012-11-16 19:27:12 -0700194 set_bit(idx * ndev->bits_per_vector, &mask);
Jon Mason49793882013-07-15 15:53:54 -0700195 writew(mask, ndev->reg_ofs.ldb_mask);
Jon Masonfce8a7b2012-11-16 19:27:12 -0700196
197 ndev->db_cb[idx].callback = NULL;
198}
199
200/**
201 * ntb_find_transport() - find the transport pointer
202 * @transport: pointer to pci device
203 *
204 * Given the pci device pointer, return the transport pointer passed in when
205 * the transport attached when it was inited.
206 *
207 * RETURNS: pointer to transport.
208 */
209void *ntb_find_transport(struct pci_dev *pdev)
210{
211 struct ntb_device *ndev = pci_get_drvdata(pdev);
212 return ndev->ntb_transport;
213}
214
215/**
216 * ntb_register_transport() - Register NTB transport with NTB HW driver
217 * @transport: transport identifier
218 *
219 * This function allows a transport to reserve the hardware driver for
220 * NTB usage.
221 *
222 * RETURNS: pointer to ntb_device, NULL on error.
223 */
224struct ntb_device *ntb_register_transport(struct pci_dev *pdev, void *transport)
225{
226 struct ntb_device *ndev = pci_get_drvdata(pdev);
227
228 if (ndev->ntb_transport)
229 return NULL;
230
231 ndev->ntb_transport = transport;
232 return ndev;
233}
234
235/**
236 * ntb_unregister_transport() - Unregister the transport with the NTB HW driver
237 * @ndev - ntb_device of the transport to be freed
238 *
239 * This function unregisters the transport from the HW driver and performs any
240 * necessary cleanups.
241 */
242void ntb_unregister_transport(struct ntb_device *ndev)
243{
244 int i;
245
246 if (!ndev->ntb_transport)
247 return;
248
249 for (i = 0; i < ndev->max_cbs; i++)
250 ntb_unregister_db_callback(ndev, i);
251
252 ntb_unregister_event_callback(ndev);
253 ndev->ntb_transport = NULL;
254}
255
256/**
Jon Masonfce8a7b2012-11-16 19:27:12 -0700257 * ntb_write_local_spad() - write to the secondary scratchpad register
258 * @ndev: pointer to ntb_device instance
259 * @idx: index to the scratchpad register, 0 based
260 * @val: the data value to put into the register
261 *
262 * This function allows writing of a 32bit value to the indexed scratchpad
263 * register. This writes over the data mirrored to the local scratchpad register
264 * by the remote system.
265 *
266 * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
267 */
268int ntb_write_local_spad(struct ntb_device *ndev, unsigned int idx, u32 val)
269{
270 if (idx >= ndev->limits.max_spads)
271 return -EINVAL;
272
273 dev_dbg(&ndev->pdev->dev, "Writing %x to local scratch pad index %d\n",
274 val, idx);
275 writel(val, ndev->reg_ofs.spad_read + idx * 4);
276
277 return 0;
278}
279
280/**
281 * ntb_read_local_spad() - read from the primary scratchpad register
282 * @ndev: pointer to ntb_device instance
283 * @idx: index to scratchpad register, 0 based
284 * @val: pointer to 32bit integer for storing the register value
285 *
286 * This function allows reading of the 32bit scratchpad register on
287 * the primary (internal) side. This allows the local system to read data
288 * written and mirrored to the scratchpad register by the remote system.
289 *
290 * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
291 */
292int ntb_read_local_spad(struct ntb_device *ndev, unsigned int idx, u32 *val)
293{
294 if (idx >= ndev->limits.max_spads)
295 return -EINVAL;
296
297 *val = readl(ndev->reg_ofs.spad_write + idx * 4);
298 dev_dbg(&ndev->pdev->dev,
299 "Reading %x from local scratch pad index %d\n", *val, idx);
300
301 return 0;
302}
303
304/**
305 * ntb_write_remote_spad() - write to the secondary scratchpad register
306 * @ndev: pointer to ntb_device instance
307 * @idx: index to the scratchpad register, 0 based
308 * @val: the data value to put into the register
309 *
310 * This function allows writing of a 32bit value to the indexed scratchpad
311 * register. The register resides on the secondary (external) side. This allows
312 * the local system to write data to be mirrored to the remote systems
313 * scratchpad register.
314 *
315 * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
316 */
317int ntb_write_remote_spad(struct ntb_device *ndev, unsigned int idx, u32 val)
318{
319 if (idx >= ndev->limits.max_spads)
320 return -EINVAL;
321
322 dev_dbg(&ndev->pdev->dev, "Writing %x to remote scratch pad index %d\n",
323 val, idx);
324 writel(val, ndev->reg_ofs.spad_write + idx * 4);
325
326 return 0;
327}
328
329/**
330 * ntb_read_remote_spad() - read from the primary scratchpad register
331 * @ndev: pointer to ntb_device instance
332 * @idx: index to scratchpad register, 0 based
333 * @val: pointer to 32bit integer for storing the register value
334 *
335 * This function allows reading of the 32bit scratchpad register on
336 * the primary (internal) side. This alloows the local system to read the data
337 * it wrote to be mirrored on the remote system.
338 *
339 * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
340 */
341int ntb_read_remote_spad(struct ntb_device *ndev, unsigned int idx, u32 *val)
342{
343 if (idx >= ndev->limits.max_spads)
344 return -EINVAL;
345
346 *val = readl(ndev->reg_ofs.spad_read + idx * 4);
347 dev_dbg(&ndev->pdev->dev,
348 "Reading %x from remote scratch pad index %d\n", *val, idx);
349
350 return 0;
351}
352
353/**
Jon Mason282a2fe2013-02-12 09:52:50 -0700354 * ntb_get_mw_base() - get addr for the NTB memory window
355 * @ndev: pointer to ntb_device instance
356 * @mw: memory window number
357 *
358 * This function provides the base address of the memory window specified.
359 *
360 * RETURNS: address, or NULL on error.
361 */
362resource_size_t ntb_get_mw_base(struct ntb_device *ndev, unsigned int mw)
363{
364 if (mw >= ntb_max_mw(ndev))
365 return 0;
366
367 return pci_resource_start(ndev->pdev, MW_TO_BAR(mw));
368}
369
370/**
Jon Masonfce8a7b2012-11-16 19:27:12 -0700371 * ntb_get_mw_vbase() - get virtual addr for the NTB memory window
372 * @ndev: pointer to ntb_device instance
373 * @mw: memory window number
374 *
375 * This function provides the base virtual address of the memory window
376 * specified.
377 *
378 * RETURNS: pointer to virtual address, or NULL on error.
379 */
Jon Mason74465642013-01-21 15:28:52 -0700380void __iomem *ntb_get_mw_vbase(struct ntb_device *ndev, unsigned int mw)
Jon Masonfce8a7b2012-11-16 19:27:12 -0700381{
Jon Mason948d3a62013-04-18 17:07:36 -0700382 if (mw >= ntb_max_mw(ndev))
Jon Masonfce8a7b2012-11-16 19:27:12 -0700383 return NULL;
384
385 return ndev->mw[mw].vbase;
386}
387
388/**
389 * ntb_get_mw_size() - return size of NTB memory window
390 * @ndev: pointer to ntb_device instance
391 * @mw: memory window number
392 *
393 * This function provides the physical size of the memory window specified
394 *
395 * RETURNS: the size of the memory window or zero on error
396 */
Jon Masonac477af2013-01-21 16:40:39 -0700397u64 ntb_get_mw_size(struct ntb_device *ndev, unsigned int mw)
Jon Masonfce8a7b2012-11-16 19:27:12 -0700398{
Jon Mason948d3a62013-04-18 17:07:36 -0700399 if (mw >= ntb_max_mw(ndev))
Jon Masonfce8a7b2012-11-16 19:27:12 -0700400 return 0;
401
402 return ndev->mw[mw].bar_sz;
403}
404
405/**
406 * ntb_set_mw_addr - set the memory window address
407 * @ndev: pointer to ntb_device instance
408 * @mw: memory window number
409 * @addr: base address for data
410 *
411 * This function sets the base physical address of the memory window. This
412 * memory address is where data from the remote system will be transfered into
413 * or out of depending on how the transport is configured.
414 */
415void ntb_set_mw_addr(struct ntb_device *ndev, unsigned int mw, u64 addr)
416{
Jon Mason948d3a62013-04-18 17:07:36 -0700417 if (mw >= ntb_max_mw(ndev))
Jon Masonfce8a7b2012-11-16 19:27:12 -0700418 return;
419
420 dev_dbg(&ndev->pdev->dev, "Writing addr %Lx to BAR %d\n", addr,
421 MW_TO_BAR(mw));
422
423 ndev->mw[mw].phys_addr = addr;
424
425 switch (MW_TO_BAR(mw)) {
426 case NTB_BAR_23:
Jon Mason49793882013-07-15 15:53:54 -0700427 writeq(addr, ndev->reg_ofs.bar2_xlat);
Jon Masonfce8a7b2012-11-16 19:27:12 -0700428 break;
429 case NTB_BAR_45:
Jon Mason49793882013-07-15 15:53:54 -0700430 writeq(addr, ndev->reg_ofs.bar4_xlat);
Jon Masonfce8a7b2012-11-16 19:27:12 -0700431 break;
432 }
433}
434
435/**
Jon Mason49793882013-07-15 15:53:54 -0700436 * ntb_ring_doorbell() - Set the doorbell on the secondary/external side
Jon Masonfce8a7b2012-11-16 19:27:12 -0700437 * @ndev: pointer to ntb_device instance
438 * @db: doorbell to ring
439 *
440 * This function allows triggering of a doorbell on the secondary/external
441 * side that will initiate an interrupt on the remote host
442 *
443 * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
444 */
Jon Mason49793882013-07-15 15:53:54 -0700445void ntb_ring_doorbell(struct ntb_device *ndev, unsigned int db)
Jon Masonfce8a7b2012-11-16 19:27:12 -0700446{
447 dev_dbg(&ndev->pdev->dev, "%s: ringing doorbell %d\n", __func__, db);
448
449 if (ndev->hw_type == BWD_HW)
Jon Mason49793882013-07-15 15:53:54 -0700450 writeq((u64) 1 << db, ndev->reg_ofs.rdb);
Jon Masonfce8a7b2012-11-16 19:27:12 -0700451 else
452 writew(((1 << ndev->bits_per_vector) - 1) <<
Jon Mason49793882013-07-15 15:53:54 -0700453 (db * ndev->bits_per_vector), ndev->reg_ofs.rdb);
Jon Masonfce8a7b2012-11-16 19:27:12 -0700454}
455
Jon Mason113bf1c2012-11-16 18:52:57 -0700456static void bwd_recover_link(struct ntb_device *ndev)
457{
458 u32 status;
459
460 /* Driver resets the NTB ModPhy lanes - magic! */
461 writeb(0xe0, ndev->reg_base + BWD_MODPHY_PCSREG6);
462 writeb(0x40, ndev->reg_base + BWD_MODPHY_PCSREG4);
463 writeb(0x60, ndev->reg_base + BWD_MODPHY_PCSREG4);
464 writeb(0x60, ndev->reg_base + BWD_MODPHY_PCSREG6);
465
466 /* Driver waits 100ms to allow the NTB ModPhy to settle */
467 msleep(100);
468
469 /* Clear AER Errors, write to clear */
470 status = readl(ndev->reg_base + BWD_ERRCORSTS_OFFSET);
471 dev_dbg(&ndev->pdev->dev, "ERRCORSTS = %x\n", status);
472 status &= PCI_ERR_COR_REP_ROLL;
473 writel(status, ndev->reg_base + BWD_ERRCORSTS_OFFSET);
474
475 /* Clear unexpected electrical idle event in LTSSM, write to clear */
476 status = readl(ndev->reg_base + BWD_LTSSMERRSTS0_OFFSET);
477 dev_dbg(&ndev->pdev->dev, "LTSSMERRSTS0 = %x\n", status);
478 status |= BWD_LTSSMERRSTS0_UNEXPECTEDEI;
479 writel(status, ndev->reg_base + BWD_LTSSMERRSTS0_OFFSET);
480
481 /* Clear DeSkew Buffer error, write to clear */
482 status = readl(ndev->reg_base + BWD_DESKEWSTS_OFFSET);
483 dev_dbg(&ndev->pdev->dev, "DESKEWSTS = %x\n", status);
484 status |= BWD_DESKEWSTS_DBERR;
485 writel(status, ndev->reg_base + BWD_DESKEWSTS_OFFSET);
486
487 status = readl(ndev->reg_base + BWD_IBSTERRRCRVSTS0_OFFSET);
488 dev_dbg(&ndev->pdev->dev, "IBSTERRRCRVSTS0 = %x\n", status);
489 status &= BWD_IBIST_ERR_OFLOW;
490 writel(status, ndev->reg_base + BWD_IBSTERRRCRVSTS0_OFFSET);
491
492 /* Releases the NTB state machine to allow the link to retrain */
493 status = readl(ndev->reg_base + BWD_LTSSMSTATEJMP_OFFSET);
494 dev_dbg(&ndev->pdev->dev, "LTSSMSTATEJMP = %x\n", status);
495 status &= ~BWD_LTSSMSTATEJMP_FORCEDETECT;
496 writel(status, ndev->reg_base + BWD_LTSSMSTATEJMP_OFFSET);
497}
498
Jon Masonfce8a7b2012-11-16 19:27:12 -0700499static void ntb_link_event(struct ntb_device *ndev, int link_state)
500{
501 unsigned int event;
502
503 if (ndev->link_status == link_state)
504 return;
505
506 if (link_state == NTB_LINK_UP) {
507 u16 status;
508
509 dev_info(&ndev->pdev->dev, "Link Up\n");
510 ndev->link_status = NTB_LINK_UP;
511 event = NTB_EVENT_HW_LINK_UP;
512
Jon Masoned6c24e2013-07-15 16:43:54 -0700513 if (ndev->hw_type == BWD_HW ||
514 ndev->conn_type == NTB_CONN_TRANSPARENT)
Jon Masonfce8a7b2012-11-16 19:27:12 -0700515 status = readw(ndev->reg_ofs.lnk_stat);
516 else {
517 int rc = pci_read_config_word(ndev->pdev,
518 SNB_LINK_STATUS_OFFSET,
519 &status);
520 if (rc)
521 return;
522 }
Jon Mason113bf1c2012-11-16 18:52:57 -0700523
524 ndev->link_width = (status & NTB_LINK_WIDTH_MASK) >> 4;
525 ndev->link_speed = (status & NTB_LINK_SPEED_MASK);
Jon Masonfce8a7b2012-11-16 19:27:12 -0700526 dev_info(&ndev->pdev->dev, "Link Width %d, Link Speed %d\n",
Jon Mason113bf1c2012-11-16 18:52:57 -0700527 ndev->link_width, ndev->link_speed);
Jon Masonfce8a7b2012-11-16 19:27:12 -0700528 } else {
529 dev_info(&ndev->pdev->dev, "Link Down\n");
530 ndev->link_status = NTB_LINK_DOWN;
531 event = NTB_EVENT_HW_LINK_DOWN;
Jon Mason113bf1c2012-11-16 18:52:57 -0700532 /* Don't modify link width/speed, we need it in link recovery */
Jon Masonfce8a7b2012-11-16 19:27:12 -0700533 }
534
535 /* notify the upper layer if we have an event change */
536 if (ndev->event_cb)
537 ndev->event_cb(ndev->ntb_transport, event);
538}
539
540static int ntb_link_status(struct ntb_device *ndev)
541{
542 int link_state;
543
544 if (ndev->hw_type == BWD_HW) {
545 u32 ntb_cntl;
546
547 ntb_cntl = readl(ndev->reg_ofs.lnk_cntl);
548 if (ntb_cntl & BWD_CNTL_LINK_DOWN)
549 link_state = NTB_LINK_DOWN;
550 else
551 link_state = NTB_LINK_UP;
552 } else {
553 u16 status;
554 int rc;
555
556 rc = pci_read_config_word(ndev->pdev, SNB_LINK_STATUS_OFFSET,
557 &status);
558 if (rc)
559 return rc;
560
561 if (status & NTB_LINK_STATUS_ACTIVE)
562 link_state = NTB_LINK_UP;
563 else
564 link_state = NTB_LINK_DOWN;
565 }
566
567 ntb_link_event(ndev, link_state);
568
569 return 0;
570}
571
Jon Mason113bf1c2012-11-16 18:52:57 -0700572static void bwd_link_recovery(struct work_struct *work)
573{
574 struct ntb_device *ndev = container_of(work, struct ntb_device,
575 lr_timer.work);
576 u32 status32;
577
578 bwd_recover_link(ndev);
579 /* There is a potential race between the 2 NTB devices recovering at the
580 * same time. If the times are the same, the link will not recover and
581 * the driver will be stuck in this loop forever. Add a random interval
582 * to the recovery time to prevent this race.
583 */
584 msleep(BWD_LINK_RECOVERY_TIME + prandom_u32() % BWD_LINK_RECOVERY_TIME);
585
586 status32 = readl(ndev->reg_base + BWD_LTSSMSTATEJMP_OFFSET);
587 if (status32 & BWD_LTSSMSTATEJMP_FORCEDETECT)
588 goto retry;
589
590 status32 = readl(ndev->reg_base + BWD_IBSTERRRCRVSTS0_OFFSET);
591 if (status32 & BWD_IBIST_ERR_OFLOW)
592 goto retry;
593
594 status32 = readl(ndev->reg_ofs.lnk_cntl);
595 if (!(status32 & BWD_CNTL_LINK_DOWN)) {
596 unsigned char speed, width;
597 u16 status16;
598
599 status16 = readw(ndev->reg_ofs.lnk_stat);
600 width = (status16 & NTB_LINK_WIDTH_MASK) >> 4;
601 speed = (status16 & NTB_LINK_SPEED_MASK);
602 if (ndev->link_width != width || ndev->link_speed != speed)
603 goto retry;
604 }
605
606 schedule_delayed_work(&ndev->hb_timer, NTB_HB_TIMEOUT);
607 return;
608
609retry:
610 schedule_delayed_work(&ndev->lr_timer, NTB_HB_TIMEOUT);
611}
612
Jon Masonfce8a7b2012-11-16 19:27:12 -0700613/* BWD doesn't have link status interrupt, poll on that platform */
614static void bwd_link_poll(struct work_struct *work)
615{
616 struct ntb_device *ndev = container_of(work, struct ntb_device,
617 hb_timer.work);
618 unsigned long ts = jiffies;
619
620 /* If we haven't gotten an interrupt in a while, check the BWD link
621 * status bit
622 */
623 if (ts > ndev->last_ts + NTB_HB_TIMEOUT) {
624 int rc = ntb_link_status(ndev);
625 if (rc)
626 dev_err(&ndev->pdev->dev,
627 "Error determining link status\n");
Jon Mason113bf1c2012-11-16 18:52:57 -0700628
629 /* Check to see if a link error is the cause of the link down */
630 if (ndev->link_status == NTB_LINK_DOWN) {
631 u32 status32 = readl(ndev->reg_base +
632 BWD_LTSSMSTATEJMP_OFFSET);
633 if (status32 & BWD_LTSSMSTATEJMP_FORCEDETECT) {
634 schedule_delayed_work(&ndev->lr_timer, 0);
635 return;
636 }
637 }
Jon Masonfce8a7b2012-11-16 19:27:12 -0700638 }
639
640 schedule_delayed_work(&ndev->hb_timer, NTB_HB_TIMEOUT);
641}
642
643static int ntb_xeon_setup(struct ntb_device *ndev)
644{
645 int rc;
646 u8 val;
647
648 ndev->hw_type = SNB_HW;
649
650 rc = pci_read_config_byte(ndev->pdev, NTB_PPD_OFFSET, &val);
651 if (rc)
652 return rc;
653
Jon Masonfce8a7b2012-11-16 19:27:12 -0700654 if (val & SNB_PPD_DEV_TYPE)
Jon Masonfce8a7b2012-11-16 19:27:12 -0700655 ndev->dev_type = NTB_DEV_USD;
Jon Masonb6750cf2013-05-31 14:05:53 -0700656 else
657 ndev->dev_type = NTB_DEV_DSD;
Jon Masonfce8a7b2012-11-16 19:27:12 -0700658
Jon Masoned6c24e2013-07-15 16:43:54 -0700659 switch (val & SNB_PPD_CONN_TYPE) {
660 case NTB_CONN_B2B:
661 dev_info(&ndev->pdev->dev, "Conn Type = B2B\n");
662 ndev->conn_type = NTB_CONN_B2B;
663 ndev->reg_ofs.ldb = ndev->reg_base + SNB_PDOORBELL_OFFSET;
664 ndev->reg_ofs.ldb_mask = ndev->reg_base + SNB_PDBMSK_OFFSET;
665 ndev->reg_ofs.spad_read = ndev->reg_base + SNB_SPAD_OFFSET;
666 ndev->reg_ofs.bar2_xlat = ndev->reg_base + SNB_SBAR2XLAT_OFFSET;
667 ndev->reg_ofs.bar4_xlat = ndev->reg_base + SNB_SBAR4XLAT_OFFSET;
668 ndev->limits.max_spads = SNB_MAX_B2B_SPADS;
669
670 /* There is a Xeon hardware errata related to writes to
671 * SDOORBELL or B2BDOORBELL in conjunction with inbound access
672 * to NTB MMIO Space, which may hang the system. To workaround
673 * this use the second memory window to access the interrupt and
674 * scratch pad registers on the remote system.
675 */
676 if (xeon_errata_workaround) {
677 if (!ndev->mw[1].bar_sz)
678 return -EINVAL;
679
680 ndev->limits.max_mw = SNB_ERRATA_MAX_MW;
681 ndev->reg_ofs.spad_write = ndev->mw[1].vbase +
682 SNB_SPAD_OFFSET;
683 ndev->reg_ofs.rdb = ndev->mw[1].vbase +
684 SNB_PDOORBELL_OFFSET;
685
686 /* Set the Limit register to 4k, the minimum size, to
687 * prevent an illegal access
688 */
689 writeq(ndev->mw[1].bar_sz + 0x1000, ndev->reg_base +
690 SNB_PBAR4LMT_OFFSET);
691 } else {
692 ndev->limits.max_mw = SNB_MAX_MW;
693 ndev->reg_ofs.spad_write = ndev->reg_base +
694 SNB_B2B_SPAD_OFFSET;
695 ndev->reg_ofs.rdb = ndev->reg_base +
696 SNB_B2B_DOORBELL_OFFSET;
697
698 /* Disable the Limit register, just incase it is set to
699 * something silly
700 */
701 writeq(0, ndev->reg_base + SNB_PBAR4LMT_OFFSET);
702 }
703
704 /* The Xeon errata workaround requires setting SBAR Base
705 * addresses to known values, so that the PBAR XLAT can be
706 * pointed at SBAR0 of the remote system.
707 */
708 if (ndev->dev_type == NTB_DEV_USD) {
709 writeq(SNB_MBAR23_DSD_ADDR, ndev->reg_base +
710 SNB_PBAR2XLAT_OFFSET);
711 if (xeon_errata_workaround)
712 writeq(SNB_MBAR01_DSD_ADDR, ndev->reg_base +
713 SNB_PBAR4XLAT_OFFSET);
714 else {
715 writeq(SNB_MBAR45_DSD_ADDR, ndev->reg_base +
716 SNB_PBAR4XLAT_OFFSET);
717 /* B2B_XLAT_OFFSET is a 64bit register, but can
718 * only take 32bit writes
719 */
720 writel(SNB_MBAR01_DSD_ADDR & 0xffffffff,
721 ndev->reg_base + SNB_B2B_XLAT_OFFSETL);
722 writel(SNB_MBAR01_DSD_ADDR >> 32,
723 ndev->reg_base + SNB_B2B_XLAT_OFFSETU);
724 }
725
726 writeq(SNB_MBAR01_USD_ADDR, ndev->reg_base +
727 SNB_SBAR0BASE_OFFSET);
728 writeq(SNB_MBAR23_USD_ADDR, ndev->reg_base +
729 SNB_SBAR2BASE_OFFSET);
730 writeq(SNB_MBAR45_USD_ADDR, ndev->reg_base +
731 SNB_SBAR4BASE_OFFSET);
732 } else {
733 writeq(SNB_MBAR23_USD_ADDR, ndev->reg_base +
734 SNB_PBAR2XLAT_OFFSET);
735 if (xeon_errata_workaround)
736 writeq(SNB_MBAR01_USD_ADDR, ndev->reg_base +
737 SNB_PBAR4XLAT_OFFSET);
738 else {
739 writeq(SNB_MBAR45_USD_ADDR, ndev->reg_base +
740 SNB_PBAR4XLAT_OFFSET);
741 /* B2B_XLAT_OFFSET is a 64bit register, but can
742 * only take 32bit writes
743 */
744 writel(SNB_MBAR01_DSD_ADDR & 0xffffffff,
745 ndev->reg_base + SNB_B2B_XLAT_OFFSETL);
746 writel(SNB_MBAR01_USD_ADDR >> 32,
747 ndev->reg_base + SNB_B2B_XLAT_OFFSETU);
748 }
749 writeq(SNB_MBAR01_DSD_ADDR, ndev->reg_base +
750 SNB_SBAR0BASE_OFFSET);
751 writeq(SNB_MBAR23_DSD_ADDR, ndev->reg_base +
752 SNB_SBAR2BASE_OFFSET);
753 writeq(SNB_MBAR45_DSD_ADDR, ndev->reg_base +
754 SNB_SBAR4BASE_OFFSET);
755 }
756 break;
757 case NTB_CONN_RP:
758 dev_info(&ndev->pdev->dev, "Conn Type = RP\n");
759 ndev->conn_type = NTB_CONN_RP;
760
761 if (xeon_errata_workaround) {
762 dev_err(&ndev->pdev->dev,
763 "NTB-RP disabled due to hardware errata. To disregard this warning and potentially lock-up the system, add the parameter 'xeon_errata_workaround=0'.\n");
764 return -EINVAL;
765 }
766
767 /* Scratch pads need to have exclusive access from the primary
768 * or secondary side. Halve the num spads so that each side can
769 * have an equal amount.
770 */
771 ndev->limits.max_spads = SNB_MAX_COMPAT_SPADS / 2;
772 /* Note: The SDOORBELL is the cause of the errata. You REALLY
773 * don't want to touch it.
774 */
775 ndev->reg_ofs.rdb = ndev->reg_base + SNB_SDOORBELL_OFFSET;
776 ndev->reg_ofs.ldb = ndev->reg_base + SNB_PDOORBELL_OFFSET;
777 ndev->reg_ofs.ldb_mask = ndev->reg_base + SNB_PDBMSK_OFFSET;
778 /* Offset the start of the spads to correspond to whether it is
779 * primary or secondary
780 */
781 ndev->reg_ofs.spad_write = ndev->reg_base + SNB_SPAD_OFFSET +
782 ndev->limits.max_spads * 4;
783 ndev->reg_ofs.spad_read = ndev->reg_base + SNB_SPAD_OFFSET;
784 ndev->reg_ofs.bar2_xlat = ndev->reg_base + SNB_SBAR2XLAT_OFFSET;
785 ndev->reg_ofs.bar4_xlat = ndev->reg_base + SNB_SBAR4XLAT_OFFSET;
786 ndev->limits.max_mw = SNB_MAX_MW;
787 break;
788 case NTB_CONN_TRANSPARENT:
789 dev_info(&ndev->pdev->dev, "Conn Type = TRANSPARENT\n");
790 ndev->conn_type = NTB_CONN_TRANSPARENT;
791 /* Scratch pads need to have exclusive access from the primary
792 * or secondary side. Halve the num spads so that each side can
793 * have an equal amount.
794 */
795 ndev->limits.max_spads = SNB_MAX_COMPAT_SPADS / 2;
796 ndev->reg_ofs.rdb = ndev->reg_base + SNB_PDOORBELL_OFFSET;
797 ndev->reg_ofs.ldb = ndev->reg_base + SNB_SDOORBELL_OFFSET;
798 ndev->reg_ofs.ldb_mask = ndev->reg_base + SNB_SDBMSK_OFFSET;
799 ndev->reg_ofs.spad_write = ndev->reg_base + SNB_SPAD_OFFSET;
800 /* Offset the start of the spads to correspond to whether it is
801 * primary or secondary
802 */
803 ndev->reg_ofs.spad_read = ndev->reg_base + SNB_SPAD_OFFSET +
804 ndev->limits.max_spads * 4;
805 ndev->reg_ofs.bar2_xlat = ndev->reg_base + SNB_PBAR2XLAT_OFFSET;
806 ndev->reg_ofs.bar4_xlat = ndev->reg_base + SNB_PBAR4XLAT_OFFSET;
807
808 ndev->limits.max_mw = SNB_MAX_MW;
809 break;
810 default:
811 /* Most likely caused by the remote NTB-RP device not being
812 * configured
813 */
814 dev_err(&ndev->pdev->dev, "Unknown PPD %x\n", val);
815 return -EINVAL;
816 }
817
Jon Masonfce8a7b2012-11-16 19:27:12 -0700818 ndev->reg_ofs.lnk_cntl = ndev->reg_base + SNB_NTBCNTL_OFFSET;
Jon Masoned6c24e2013-07-15 16:43:54 -0700819 ndev->reg_ofs.lnk_stat = ndev->reg_base + SNB_SLINK_STATUS_OFFSET;
Jon Masonfce8a7b2012-11-16 19:27:12 -0700820 ndev->reg_ofs.spci_cmd = ndev->reg_base + SNB_PCICMD_OFFSET;
821
Jon Masonfce8a7b2012-11-16 19:27:12 -0700822 ndev->limits.max_db_bits = SNB_MAX_DB_BITS;
823 ndev->limits.msix_cnt = SNB_MSIX_CNT;
824 ndev->bits_per_vector = SNB_DB_BITS_PER_VEC;
825
826 return 0;
827}
828
829static int ntb_bwd_setup(struct ntb_device *ndev)
830{
831 int rc;
832 u32 val;
833
834 ndev->hw_type = BWD_HW;
835
836 rc = pci_read_config_dword(ndev->pdev, NTB_PPD_OFFSET, &val);
837 if (rc)
838 return rc;
839
840 switch ((val & BWD_PPD_CONN_TYPE) >> 8) {
841 case NTB_CONN_B2B:
842 ndev->conn_type = NTB_CONN_B2B;
843 break;
844 case NTB_CONN_RP:
845 default:
Jon Masonb1ef0042013-07-15 15:33:18 -0700846 dev_err(&ndev->pdev->dev, "Unsupported NTB configuration\n");
Jon Masonfce8a7b2012-11-16 19:27:12 -0700847 return -EINVAL;
848 }
849
850 if (val & BWD_PPD_DEV_TYPE)
851 ndev->dev_type = NTB_DEV_DSD;
852 else
853 ndev->dev_type = NTB_DEV_USD;
854
855 /* Initiate PCI-E link training */
856 rc = pci_write_config_dword(ndev->pdev, NTB_PPD_OFFSET,
857 val | BWD_PPD_INIT_LINK);
858 if (rc)
859 return rc;
860
Jon Mason49793882013-07-15 15:53:54 -0700861 ndev->reg_ofs.ldb = ndev->reg_base + BWD_PDOORBELL_OFFSET;
862 ndev->reg_ofs.ldb_mask = ndev->reg_base + BWD_PDBMSK_OFFSET;
Jon Masonb1ef0042013-07-15 15:33:18 -0700863 ndev->reg_ofs.rdb = ndev->reg_base + BWD_B2B_DOORBELL_OFFSET;
Jon Mason49793882013-07-15 15:53:54 -0700864 ndev->reg_ofs.bar2_xlat = ndev->reg_base + BWD_SBAR2XLAT_OFFSET;
865 ndev->reg_ofs.bar4_xlat = ndev->reg_base + BWD_SBAR4XLAT_OFFSET;
Jon Masonfce8a7b2012-11-16 19:27:12 -0700866 ndev->reg_ofs.lnk_cntl = ndev->reg_base + BWD_NTBCNTL_OFFSET;
867 ndev->reg_ofs.lnk_stat = ndev->reg_base + BWD_LINK_STATUS_OFFSET;
868 ndev->reg_ofs.spad_read = ndev->reg_base + BWD_SPAD_OFFSET;
Jon Masonb1ef0042013-07-15 15:33:18 -0700869 ndev->reg_ofs.spad_write = ndev->reg_base + BWD_B2B_SPAD_OFFSET;
Jon Masonfce8a7b2012-11-16 19:27:12 -0700870 ndev->reg_ofs.spci_cmd = ndev->reg_base + BWD_PCICMD_OFFSET;
Jon Mason948d3a62013-04-18 17:07:36 -0700871 ndev->limits.max_mw = BWD_MAX_MW;
Jon Masonb1ef0042013-07-15 15:33:18 -0700872 ndev->limits.max_spads = BWD_MAX_SPADS;
Jon Masonfce8a7b2012-11-16 19:27:12 -0700873 ndev->limits.max_db_bits = BWD_MAX_DB_BITS;
874 ndev->limits.msix_cnt = BWD_MSIX_CNT;
875 ndev->bits_per_vector = BWD_DB_BITS_PER_VEC;
876
877 /* Since bwd doesn't have a link interrupt, setup a poll timer */
878 INIT_DELAYED_WORK(&ndev->hb_timer, bwd_link_poll);
Jon Mason113bf1c2012-11-16 18:52:57 -0700879 INIT_DELAYED_WORK(&ndev->lr_timer, bwd_link_recovery);
Jon Masonfce8a7b2012-11-16 19:27:12 -0700880 schedule_delayed_work(&ndev->hb_timer, NTB_HB_TIMEOUT);
881
882 return 0;
883}
884
Greg Kroah-Hartman78a61ab2013-01-17 19:17:42 -0800885static int ntb_device_setup(struct ntb_device *ndev)
Jon Masonfce8a7b2012-11-16 19:27:12 -0700886{
887 int rc;
888
889 switch (ndev->pdev->device) {
Jon Masonbe4dac02012-09-28 11:38:48 -0700890 case PCI_DEVICE_ID_INTEL_NTB_SS_JSF:
891 case PCI_DEVICE_ID_INTEL_NTB_SS_SNB:
892 case PCI_DEVICE_ID_INTEL_NTB_SS_IVT:
893 case PCI_DEVICE_ID_INTEL_NTB_SS_HSX:
894 case PCI_DEVICE_ID_INTEL_NTB_PS_JSF:
895 case PCI_DEVICE_ID_INTEL_NTB_PS_SNB:
896 case PCI_DEVICE_ID_INTEL_NTB_PS_IVT:
897 case PCI_DEVICE_ID_INTEL_NTB_PS_HSX:
Jon Masonfce8a7b2012-11-16 19:27:12 -0700898 case PCI_DEVICE_ID_INTEL_NTB_B2B_JSF:
899 case PCI_DEVICE_ID_INTEL_NTB_B2B_SNB:
Jon Masonbe4dac02012-09-28 11:38:48 -0700900 case PCI_DEVICE_ID_INTEL_NTB_B2B_IVT:
901 case PCI_DEVICE_ID_INTEL_NTB_B2B_HSX:
Jon Masonfce8a7b2012-11-16 19:27:12 -0700902 rc = ntb_xeon_setup(ndev);
903 break;
904 case PCI_DEVICE_ID_INTEL_NTB_B2B_BWD:
905 rc = ntb_bwd_setup(ndev);
906 break;
907 default:
908 rc = -ENODEV;
909 }
910
Jon Mason3b12a0d2013-07-15 13:23:47 -0700911 if (rc)
912 return rc;
913
Jon Masonb6750cf2013-05-31 14:05:53 -0700914 dev_info(&ndev->pdev->dev, "Device Type = %s\n",
915 ndev->dev_type == NTB_DEV_USD ? "USD/DSP" : "DSD/USP");
916
Jon Masoned6c24e2013-07-15 16:43:54 -0700917 if (ndev->conn_type == NTB_CONN_B2B)
918 /* Enable Bus Master and Memory Space on the secondary side */
919 writew(PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER,
920 ndev->reg_ofs.spci_cmd);
Jon Masonfce8a7b2012-11-16 19:27:12 -0700921
Jon Mason3b12a0d2013-07-15 13:23:47 -0700922 return 0;
Jon Masonfce8a7b2012-11-16 19:27:12 -0700923}
924
925static void ntb_device_free(struct ntb_device *ndev)
926{
Jon Mason113bf1c2012-11-16 18:52:57 -0700927 if (ndev->hw_type == BWD_HW) {
Jon Masonfce8a7b2012-11-16 19:27:12 -0700928 cancel_delayed_work_sync(&ndev->hb_timer);
Jon Mason113bf1c2012-11-16 18:52:57 -0700929 cancel_delayed_work_sync(&ndev->lr_timer);
930 }
Jon Masonfce8a7b2012-11-16 19:27:12 -0700931}
932
933static irqreturn_t bwd_callback_msix_irq(int irq, void *data)
934{
935 struct ntb_db_cb *db_cb = data;
936 struct ntb_device *ndev = db_cb->ndev;
937
938 dev_dbg(&ndev->pdev->dev, "MSI-X irq %d received for DB %d\n", irq,
939 db_cb->db_num);
940
941 if (db_cb->callback)
942 db_cb->callback(db_cb->data, db_cb->db_num);
943
944 /* No need to check for the specific HB irq, any interrupt means
945 * we're connected.
946 */
947 ndev->last_ts = jiffies;
948
Jon Mason49793882013-07-15 15:53:54 -0700949 writeq((u64) 1 << db_cb->db_num, ndev->reg_ofs.ldb);
Jon Masonfce8a7b2012-11-16 19:27:12 -0700950
951 return IRQ_HANDLED;
952}
953
954static irqreturn_t xeon_callback_msix_irq(int irq, void *data)
955{
956 struct ntb_db_cb *db_cb = data;
957 struct ntb_device *ndev = db_cb->ndev;
958
959 dev_dbg(&ndev->pdev->dev, "MSI-X irq %d received for DB %d\n", irq,
960 db_cb->db_num);
961
962 if (db_cb->callback)
963 db_cb->callback(db_cb->data, db_cb->db_num);
964
965 /* On Sandybridge, there are 16 bits in the interrupt register
966 * but only 4 vectors. So, 5 bits are assigned to the first 3
967 * vectors, with the 4th having a single bit for link
968 * interrupts.
969 */
970 writew(((1 << ndev->bits_per_vector) - 1) <<
Jon Mason49793882013-07-15 15:53:54 -0700971 (db_cb->db_num * ndev->bits_per_vector), ndev->reg_ofs.ldb);
Jon Masonfce8a7b2012-11-16 19:27:12 -0700972
973 return IRQ_HANDLED;
974}
975
976/* Since we do not have a HW doorbell in BWD, this is only used in JF/JT */
977static irqreturn_t xeon_event_msix_irq(int irq, void *dev)
978{
979 struct ntb_device *ndev = dev;
980 int rc;
981
982 dev_dbg(&ndev->pdev->dev, "MSI-X irq %d received for Events\n", irq);
983
984 rc = ntb_link_status(ndev);
985 if (rc)
986 dev_err(&ndev->pdev->dev, "Error determining link status\n");
987
988 /* bit 15 is always the link bit */
Jon Mason49793882013-07-15 15:53:54 -0700989 writew(1 << ndev->limits.max_db_bits, ndev->reg_ofs.ldb);
Jon Masonfce8a7b2012-11-16 19:27:12 -0700990
991 return IRQ_HANDLED;
992}
993
994static irqreturn_t ntb_interrupt(int irq, void *dev)
995{
996 struct ntb_device *ndev = dev;
997 unsigned int i = 0;
998
999 if (ndev->hw_type == BWD_HW) {
Jon Mason49793882013-07-15 15:53:54 -07001000 u64 ldb = readq(ndev->reg_ofs.ldb);
Jon Masonfce8a7b2012-11-16 19:27:12 -07001001
Jon Mason49793882013-07-15 15:53:54 -07001002 dev_dbg(&ndev->pdev->dev, "irq %d - ldb = %Lx\n", irq, ldb);
Jon Masonfce8a7b2012-11-16 19:27:12 -07001003
Jon Mason49793882013-07-15 15:53:54 -07001004 while (ldb) {
1005 i = __ffs(ldb);
1006 ldb &= ldb - 1;
Jon Masonfce8a7b2012-11-16 19:27:12 -07001007 bwd_callback_msix_irq(irq, &ndev->db_cb[i]);
1008 }
1009 } else {
Jon Mason49793882013-07-15 15:53:54 -07001010 u16 ldb = readw(ndev->reg_ofs.ldb);
Jon Masonfce8a7b2012-11-16 19:27:12 -07001011
Jon Mason49793882013-07-15 15:53:54 -07001012 dev_dbg(&ndev->pdev->dev, "irq %d - ldb = %x\n", irq, ldb);
Jon Masonfce8a7b2012-11-16 19:27:12 -07001013
Jon Mason49793882013-07-15 15:53:54 -07001014 if (ldb & SNB_DB_HW_LINK) {
Jon Masonfce8a7b2012-11-16 19:27:12 -07001015 xeon_event_msix_irq(irq, dev);
Jon Mason49793882013-07-15 15:53:54 -07001016 ldb &= ~SNB_DB_HW_LINK;
Jon Masonfce8a7b2012-11-16 19:27:12 -07001017 }
1018
Jon Mason49793882013-07-15 15:53:54 -07001019 while (ldb) {
1020 i = __ffs(ldb);
1021 ldb &= ldb - 1;
Jon Masonfce8a7b2012-11-16 19:27:12 -07001022 xeon_callback_msix_irq(irq, &ndev->db_cb[i]);
1023 }
1024 }
1025
1026 return IRQ_HANDLED;
1027}
1028
1029static int ntb_setup_msix(struct ntb_device *ndev)
1030{
1031 struct pci_dev *pdev = ndev->pdev;
1032 struct msix_entry *msix;
1033 int msix_entries;
Yijing Wang73f47ca2013-08-08 21:09:34 +08001034 int rc, i;
Jon Masonfce8a7b2012-11-16 19:27:12 -07001035 u16 val;
1036
Yijing Wang73f47ca2013-08-08 21:09:34 +08001037 if (!pdev->msix_cap) {
Jon Masonfce8a7b2012-11-16 19:27:12 -07001038 rc = -EIO;
1039 goto err;
1040 }
1041
Yijing Wang73f47ca2013-08-08 21:09:34 +08001042 rc = pci_read_config_word(pdev, pdev->msix_cap + PCI_MSIX_FLAGS, &val);
Jon Masonfce8a7b2012-11-16 19:27:12 -07001043 if (rc)
1044 goto err;
1045
1046 msix_entries = msix_table_size(val);
1047 if (msix_entries > ndev->limits.msix_cnt) {
1048 rc = -EINVAL;
1049 goto err;
1050 }
1051
1052 ndev->msix_entries = kmalloc(sizeof(struct msix_entry) * msix_entries,
1053 GFP_KERNEL);
1054 if (!ndev->msix_entries) {
1055 rc = -ENOMEM;
1056 goto err;
1057 }
1058
1059 for (i = 0; i < msix_entries; i++)
1060 ndev->msix_entries[i].entry = i;
1061
1062 rc = pci_enable_msix(pdev, ndev->msix_entries, msix_entries);
1063 if (rc < 0)
1064 goto err1;
1065 if (rc > 0) {
1066 /* On SNB, the link interrupt is always tied to 4th vector. If
1067 * we can't get all 4, then we can't use MSI-X.
1068 */
1069 if (ndev->hw_type != BWD_HW) {
1070 rc = -EIO;
1071 goto err1;
1072 }
1073
1074 dev_warn(&pdev->dev,
1075 "Only %d MSI-X vectors. Limiting the number of queues to that number.\n",
1076 rc);
1077 msix_entries = rc;
1078 }
1079
1080 for (i = 0; i < msix_entries; i++) {
1081 msix = &ndev->msix_entries[i];
1082 WARN_ON(!msix->vector);
1083
1084 /* Use the last MSI-X vector for Link status */
1085 if (ndev->hw_type == BWD_HW) {
1086 rc = request_irq(msix->vector, bwd_callback_msix_irq, 0,
1087 "ntb-callback-msix", &ndev->db_cb[i]);
1088 if (rc)
1089 goto err2;
1090 } else {
1091 if (i == msix_entries - 1) {
1092 rc = request_irq(msix->vector,
1093 xeon_event_msix_irq, 0,
1094 "ntb-event-msix", ndev);
1095 if (rc)
1096 goto err2;
1097 } else {
1098 rc = request_irq(msix->vector,
1099 xeon_callback_msix_irq, 0,
1100 "ntb-callback-msix",
1101 &ndev->db_cb[i]);
1102 if (rc)
1103 goto err2;
1104 }
1105 }
1106 }
1107
1108 ndev->num_msix = msix_entries;
1109 if (ndev->hw_type == BWD_HW)
1110 ndev->max_cbs = msix_entries;
1111 else
1112 ndev->max_cbs = msix_entries - 1;
1113
1114 return 0;
1115
1116err2:
1117 while (--i >= 0) {
1118 msix = &ndev->msix_entries[i];
1119 if (ndev->hw_type != BWD_HW && i == ndev->num_msix - 1)
1120 free_irq(msix->vector, ndev);
1121 else
1122 free_irq(msix->vector, &ndev->db_cb[i]);
1123 }
1124 pci_disable_msix(pdev);
1125err1:
1126 kfree(ndev->msix_entries);
1127 dev_err(&pdev->dev, "Error allocating MSI-X interrupt\n");
1128err:
1129 ndev->num_msix = 0;
1130 return rc;
1131}
1132
1133static int ntb_setup_msi(struct ntb_device *ndev)
1134{
1135 struct pci_dev *pdev = ndev->pdev;
1136 int rc;
1137
1138 rc = pci_enable_msi(pdev);
1139 if (rc)
1140 return rc;
1141
1142 rc = request_irq(pdev->irq, ntb_interrupt, 0, "ntb-msi", ndev);
1143 if (rc) {
1144 pci_disable_msi(pdev);
1145 dev_err(&pdev->dev, "Error allocating MSI interrupt\n");
1146 return rc;
1147 }
1148
1149 return 0;
1150}
1151
1152static int ntb_setup_intx(struct ntb_device *ndev)
1153{
1154 struct pci_dev *pdev = ndev->pdev;
1155 int rc;
1156
1157 pci_msi_off(pdev);
1158
1159 /* Verify intx is enabled */
1160 pci_intx(pdev, 1);
1161
1162 rc = request_irq(pdev->irq, ntb_interrupt, IRQF_SHARED, "ntb-intx",
1163 ndev);
1164 if (rc)
1165 return rc;
1166
1167 return 0;
1168}
1169
Greg Kroah-Hartman78a61ab2013-01-17 19:17:42 -08001170static int ntb_setup_interrupts(struct ntb_device *ndev)
Jon Masonfce8a7b2012-11-16 19:27:12 -07001171{
1172 int rc;
1173
1174 /* On BWD, disable all interrupts. On SNB, disable all but Link
1175 * Interrupt. The rest will be unmasked as callbacks are registered.
1176 */
1177 if (ndev->hw_type == BWD_HW)
Jon Mason49793882013-07-15 15:53:54 -07001178 writeq(~0, ndev->reg_ofs.ldb_mask);
Jon Masonfce8a7b2012-11-16 19:27:12 -07001179 else
1180 writew(~(1 << ndev->limits.max_db_bits),
Jon Mason49793882013-07-15 15:53:54 -07001181 ndev->reg_ofs.ldb_mask);
Jon Masonfce8a7b2012-11-16 19:27:12 -07001182
1183 rc = ntb_setup_msix(ndev);
1184 if (!rc)
1185 goto done;
1186
1187 ndev->bits_per_vector = 1;
1188 ndev->max_cbs = ndev->limits.max_db_bits;
1189
1190 rc = ntb_setup_msi(ndev);
1191 if (!rc)
1192 goto done;
1193
1194 rc = ntb_setup_intx(ndev);
1195 if (rc) {
1196 dev_err(&ndev->pdev->dev, "no usable interrupts\n");
1197 return rc;
1198 }
1199
1200done:
1201 return 0;
1202}
1203
Greg Kroah-Hartman78a61ab2013-01-17 19:17:42 -08001204static void ntb_free_interrupts(struct ntb_device *ndev)
Jon Masonfce8a7b2012-11-16 19:27:12 -07001205{
1206 struct pci_dev *pdev = ndev->pdev;
1207
1208 /* mask interrupts */
1209 if (ndev->hw_type == BWD_HW)
Jon Mason49793882013-07-15 15:53:54 -07001210 writeq(~0, ndev->reg_ofs.ldb_mask);
Jon Masonfce8a7b2012-11-16 19:27:12 -07001211 else
Jon Mason49793882013-07-15 15:53:54 -07001212 writew(~0, ndev->reg_ofs.ldb_mask);
Jon Masonfce8a7b2012-11-16 19:27:12 -07001213
1214 if (ndev->num_msix) {
1215 struct msix_entry *msix;
1216 u32 i;
1217
1218 for (i = 0; i < ndev->num_msix; i++) {
1219 msix = &ndev->msix_entries[i];
1220 if (ndev->hw_type != BWD_HW && i == ndev->num_msix - 1)
1221 free_irq(msix->vector, ndev);
1222 else
1223 free_irq(msix->vector, &ndev->db_cb[i]);
1224 }
1225 pci_disable_msix(pdev);
1226 } else {
1227 free_irq(pdev->irq, ndev);
1228
1229 if (pci_dev_msi_enabled(pdev))
1230 pci_disable_msi(pdev);
1231 }
1232}
1233
Greg Kroah-Hartman78a61ab2013-01-17 19:17:42 -08001234static int ntb_create_callbacks(struct ntb_device *ndev)
Jon Masonfce8a7b2012-11-16 19:27:12 -07001235{
1236 int i;
1237
Jon Masonf9a2cf82013-07-29 16:46:43 -07001238 /* Chicken-egg issue. We won't know how many callbacks are necessary
Jon Masonfce8a7b2012-11-16 19:27:12 -07001239 * until we see how many MSI-X vectors we get, but these pointers need
Jon Masonf9a2cf82013-07-29 16:46:43 -07001240 * to be passed into the MSI-X register function. So, we allocate the
Jon Masonfce8a7b2012-11-16 19:27:12 -07001241 * max, knowing that they might not all be used, to work around this.
1242 */
1243 ndev->db_cb = kcalloc(ndev->limits.max_db_bits,
1244 sizeof(struct ntb_db_cb),
1245 GFP_KERNEL);
1246 if (!ndev->db_cb)
1247 return -ENOMEM;
1248
1249 for (i = 0; i < ndev->limits.max_db_bits; i++) {
1250 ndev->db_cb[i].db_num = i;
1251 ndev->db_cb[i].ndev = ndev;
1252 }
1253
1254 return 0;
1255}
1256
1257static void ntb_free_callbacks(struct ntb_device *ndev)
1258{
1259 int i;
1260
1261 for (i = 0; i < ndev->limits.max_db_bits; i++)
1262 ntb_unregister_db_callback(ndev, i);
1263
1264 kfree(ndev->db_cb);
1265}
1266
Jon Mason1517a3f2013-07-30 15:58:49 -07001267static void ntb_setup_debugfs(struct ntb_device *ndev)
1268{
1269 if (!debugfs_initialized())
1270 return;
1271
1272 if (!debugfs_dir)
1273 debugfs_dir = debugfs_create_dir(KBUILD_MODNAME, NULL);
1274
1275 ndev->debugfs_dir = debugfs_create_dir(pci_name(ndev->pdev),
1276 debugfs_dir);
1277}
1278
1279static void ntb_free_debugfs(struct ntb_device *ndev)
1280{
1281 debugfs_remove_recursive(ndev->debugfs_dir);
1282
1283 if (debugfs_dir && simple_empty(debugfs_dir)) {
1284 debugfs_remove_recursive(debugfs_dir);
1285 debugfs_dir = NULL;
1286 }
1287}
1288
Greg Kroah-Hartman78a61ab2013-01-17 19:17:42 -08001289static int ntb_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
Jon Masonfce8a7b2012-11-16 19:27:12 -07001290{
1291 struct ntb_device *ndev;
1292 int rc, i;
1293
1294 ndev = kzalloc(sizeof(struct ntb_device), GFP_KERNEL);
1295 if (!ndev)
1296 return -ENOMEM;
1297
1298 ndev->pdev = pdev;
1299 ndev->link_status = NTB_LINK_DOWN;
1300 pci_set_drvdata(pdev, ndev);
Jon Mason1517a3f2013-07-30 15:58:49 -07001301 ntb_setup_debugfs(ndev);
Jon Masonfce8a7b2012-11-16 19:27:12 -07001302
1303 rc = pci_enable_device(pdev);
1304 if (rc)
1305 goto err;
1306
1307 pci_set_master(ndev->pdev);
1308
1309 rc = pci_request_selected_regions(pdev, NTB_BAR_MASK, KBUILD_MODNAME);
1310 if (rc)
1311 goto err1;
1312
1313 ndev->reg_base = pci_ioremap_bar(pdev, NTB_BAR_MMIO);
1314 if (!ndev->reg_base) {
1315 dev_warn(&pdev->dev, "Cannot remap BAR 0\n");
1316 rc = -EIO;
1317 goto err2;
1318 }
1319
Jon Mason948d3a62013-04-18 17:07:36 -07001320 for (i = 0; i < NTB_MAX_NUM_MW; i++) {
Jon Masonfce8a7b2012-11-16 19:27:12 -07001321 ndev->mw[i].bar_sz = pci_resource_len(pdev, MW_TO_BAR(i));
1322 ndev->mw[i].vbase =
1323 ioremap_wc(pci_resource_start(pdev, MW_TO_BAR(i)),
1324 ndev->mw[i].bar_sz);
Jon Mason113fc502013-01-30 11:40:52 -07001325 dev_info(&pdev->dev, "MW %d size %llu\n", i,
Jon Masonac477af2013-01-21 16:40:39 -07001326 (unsigned long long) ndev->mw[i].bar_sz);
Jon Masonfce8a7b2012-11-16 19:27:12 -07001327 if (!ndev->mw[i].vbase) {
1328 dev_warn(&pdev->dev, "Cannot remap BAR %d\n",
1329 MW_TO_BAR(i));
1330 rc = -EIO;
1331 goto err3;
1332 }
1333 }
1334
1335 rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(64));
1336 if (rc) {
1337 rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
1338 if (rc)
1339 goto err3;
1340
1341 dev_warn(&pdev->dev, "Cannot DMA highmem\n");
1342 }
1343
1344 rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
1345 if (rc) {
1346 rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32));
1347 if (rc)
1348 goto err3;
1349
1350 dev_warn(&pdev->dev, "Cannot DMA consistent highmem\n");
1351 }
1352
1353 rc = ntb_device_setup(ndev);
1354 if (rc)
1355 goto err3;
1356
1357 rc = ntb_create_callbacks(ndev);
1358 if (rc)
1359 goto err4;
1360
1361 rc = ntb_setup_interrupts(ndev);
1362 if (rc)
1363 goto err5;
1364
1365 /* The scratchpad registers keep the values between rmmod/insmod,
1366 * blast them now
1367 */
1368 for (i = 0; i < ndev->limits.max_spads; i++) {
1369 ntb_write_local_spad(ndev, i, 0);
1370 ntb_write_remote_spad(ndev, i, 0);
1371 }
1372
1373 rc = ntb_transport_init(pdev);
1374 if (rc)
1375 goto err6;
1376
1377 /* Let's bring the NTB link up */
1378 writel(NTB_CNTL_BAR23_SNOOP | NTB_CNTL_BAR45_SNOOP,
1379 ndev->reg_ofs.lnk_cntl);
1380
1381 return 0;
1382
1383err6:
1384 ntb_free_interrupts(ndev);
1385err5:
1386 ntb_free_callbacks(ndev);
1387err4:
1388 ntb_device_free(ndev);
1389err3:
1390 for (i--; i >= 0; i--)
1391 iounmap(ndev->mw[i].vbase);
1392 iounmap(ndev->reg_base);
1393err2:
1394 pci_release_selected_regions(pdev, NTB_BAR_MASK);
1395err1:
1396 pci_disable_device(pdev);
1397err:
Jon Mason1517a3f2013-07-30 15:58:49 -07001398 ntb_free_debugfs(ndev);
Jon Masonfce8a7b2012-11-16 19:27:12 -07001399 kfree(ndev);
1400
1401 dev_err(&pdev->dev, "Error loading %s module\n", KBUILD_MODNAME);
1402 return rc;
1403}
1404
Greg Kroah-Hartman78a61ab2013-01-17 19:17:42 -08001405static void ntb_pci_remove(struct pci_dev *pdev)
Jon Masonfce8a7b2012-11-16 19:27:12 -07001406{
1407 struct ntb_device *ndev = pci_get_drvdata(pdev);
1408 int i;
1409 u32 ntb_cntl;
1410
1411 /* Bring NTB link down */
1412 ntb_cntl = readl(ndev->reg_ofs.lnk_cntl);
Jon Masoned6c24e2013-07-15 16:43:54 -07001413 ntb_cntl |= NTB_CNTL_LINK_DISABLE;
Jon Masonfce8a7b2012-11-16 19:27:12 -07001414 writel(ntb_cntl, ndev->reg_ofs.lnk_cntl);
1415
1416 ntb_transport_free(ndev->ntb_transport);
1417
1418 ntb_free_interrupts(ndev);
1419 ntb_free_callbacks(ndev);
1420 ntb_device_free(ndev);
1421
Jon Mason948d3a62013-04-18 17:07:36 -07001422 for (i = 0; i < NTB_MAX_NUM_MW; i++)
Jon Masonfce8a7b2012-11-16 19:27:12 -07001423 iounmap(ndev->mw[i].vbase);
1424
1425 iounmap(ndev->reg_base);
1426 pci_release_selected_regions(pdev, NTB_BAR_MASK);
1427 pci_disable_device(pdev);
Jon Mason1517a3f2013-07-30 15:58:49 -07001428 ntb_free_debugfs(ndev);
Jon Masonfce8a7b2012-11-16 19:27:12 -07001429 kfree(ndev);
1430}
1431
1432static struct pci_driver ntb_pci_driver = {
1433 .name = KBUILD_MODNAME,
1434 .id_table = ntb_pci_tbl,
1435 .probe = ntb_pci_probe,
Greg Kroah-Hartman78a61ab2013-01-17 19:17:42 -08001436 .remove = ntb_pci_remove,
Jon Masonfce8a7b2012-11-16 19:27:12 -07001437};
1438module_pci_driver(ntb_pci_driver);