blob: 5738d315248b202b4a26aff084b07e819a80855c [file] [log] [blame]
Benjamin Herrenschmidt184cd4a2011-11-15 17:29:08 +00001/*
2 * Support PCI/PCIe on PowerNV platforms
3 *
4 * Copyright 2011 Benjamin Herrenschmidt, IBM Corp.
5 *
6 * This program is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU General Public License
8 * as published by the Free Software Foundation; either version
9 * 2 of the License, or (at your option) any later version.
10 */
11
Benjamin Herrenschmidtcee72d52011-11-29 18:22:53 +000012#undef DEBUG
Benjamin Herrenschmidt184cd4a2011-11-15 17:29:08 +000013
14#include <linux/kernel.h>
15#include <linux/pci.h>
Gavin Shan361f2a22014-04-24 18:00:25 +100016#include <linux/crash_dump.h>
Gavin Shan37c367f2013-06-20 18:13:25 +080017#include <linux/debugfs.h>
Benjamin Herrenschmidt184cd4a2011-11-15 17:29:08 +000018#include <linux/delay.h>
19#include <linux/string.h>
20#include <linux/init.h>
21#include <linux/bootmem.h>
22#include <linux/irq.h>
23#include <linux/io.h>
24#include <linux/msi.h>
Benjamin Herrenschmidtcd15b042014-02-11 11:32:38 +110025#include <linux/memblock.h>
Alexey Kardashevskiyac9a5882015-06-05 16:34:56 +100026#include <linux/iommu.h>
Alexey Kardashevskiye57080f2015-06-05 16:35:13 +100027#include <linux/rculist.h>
Alexey Kardashevskiy4793d652015-06-05 16:35:20 +100028#include <linux/sizes.h>
Benjamin Herrenschmidt184cd4a2011-11-15 17:29:08 +000029
30#include <asm/sections.h>
31#include <asm/io.h>
32#include <asm/prom.h>
33#include <asm/pci-bridge.h>
34#include <asm/machdep.h>
Gavin Shanfb1b55d2013-03-05 21:12:37 +000035#include <asm/msi_bitmap.h>
Benjamin Herrenschmidt184cd4a2011-11-15 17:29:08 +000036#include <asm/ppc-pci.h>
37#include <asm/opal.h>
38#include <asm/iommu.h>
39#include <asm/tce.h>
Gavin Shan137436c2013-04-25 19:20:59 +000040#include <asm/xics.h>
Gavin Shan37c367f2013-06-20 18:13:25 +080041#include <asm/debug.h>
Guo Chao262af552014-07-21 14:42:30 +100042#include <asm/firmware.h>
Ian Munsie80c49c72014-10-08 19:54:57 +110043#include <asm/pnv-pci.h>
Alexey Kardashevskiyaca69132015-06-05 16:35:17 +100044#include <asm/mmzone.h>
Ian Munsie80c49c72014-10-08 19:54:57 +110045
Michael Neulingec249dd2015-05-27 16:07:16 +100046#include <misc/cxl-base.h>
Benjamin Herrenschmidt184cd4a2011-11-15 17:29:08 +000047
48#include "powernv.h"
49#include "pci.h"
50
Wei Yang781a8682015-03-25 16:23:57 +080051/* 256M DMA window, 4K TCE pages, 8 bytes TCE */
52#define TCE32_TABLE_SIZE ((0x10000000 / 0x1000) * 8)
53
Alexey Kardashevskiybbb845c2015-06-05 16:35:19 +100054#define POWERNV_IOMMU_DEFAULT_LEVELS 1
55#define POWERNV_IOMMU_MAX_LEVELS 5
56
Alexey Kardashevskiyaca69132015-06-05 16:35:17 +100057static void pnv_pci_ioda2_table_free_pages(struct iommu_table *tbl);
58
Joe Perches6d31c2f2014-09-21 10:55:06 -070059static void pe_level_printk(const struct pnv_ioda_pe *pe, const char *level,
60 const char *fmt, ...)
61{
62 struct va_format vaf;
63 va_list args;
64 char pfix[32];
Benjamin Herrenschmidt184cd4a2011-11-15 17:29:08 +000065
Joe Perches6d31c2f2014-09-21 10:55:06 -070066 va_start(args, fmt);
67
68 vaf.fmt = fmt;
69 vaf.va = &args;
70
Wei Yang781a8682015-03-25 16:23:57 +080071 if (pe->flags & PNV_IODA_PE_DEV)
Joe Perches6d31c2f2014-09-21 10:55:06 -070072 strlcpy(pfix, dev_name(&pe->pdev->dev), sizeof(pfix));
Wei Yang781a8682015-03-25 16:23:57 +080073 else if (pe->flags & (PNV_IODA_PE_BUS | PNV_IODA_PE_BUS_ALL))
Joe Perches6d31c2f2014-09-21 10:55:06 -070074 sprintf(pfix, "%04x:%02x ",
75 pci_domain_nr(pe->pbus), pe->pbus->number);
Wei Yang781a8682015-03-25 16:23:57 +080076#ifdef CONFIG_PCI_IOV
77 else if (pe->flags & PNV_IODA_PE_VF)
78 sprintf(pfix, "%04x:%02x:%2x.%d",
79 pci_domain_nr(pe->parent_dev->bus),
80 (pe->rid & 0xff00) >> 8,
81 PCI_SLOT(pe->rid), PCI_FUNC(pe->rid));
82#endif /* CONFIG_PCI_IOV*/
Joe Perches6d31c2f2014-09-21 10:55:06 -070083
84 printk("%spci %s: [PE# %.3d] %pV",
85 level, pfix, pe->pe_number, &vaf);
86
87 va_end(args);
88}
89
90#define pe_err(pe, fmt, ...) \
91 pe_level_printk(pe, KERN_ERR, fmt, ##__VA_ARGS__)
92#define pe_warn(pe, fmt, ...) \
93 pe_level_printk(pe, KERN_WARNING, fmt, ##__VA_ARGS__)
94#define pe_info(pe, fmt, ...) \
95 pe_level_printk(pe, KERN_INFO, fmt, ##__VA_ARGS__)
Benjamin Herrenschmidt184cd4a2011-11-15 17:29:08 +000096
Thadeu Lima de Souza Cascardo4e287842014-10-23 19:19:35 -020097static bool pnv_iommu_bypass_disabled __read_mostly;
98
99static int __init iommu_setup(char *str)
100{
101 if (!str)
102 return -EINVAL;
103
104 while (*str) {
105 if (!strncmp(str, "nobypass", 8)) {
106 pnv_iommu_bypass_disabled = true;
107 pr_info("PowerNV: IOMMU bypass window disabled.\n");
108 break;
109 }
110 str += strcspn(str, ",");
111 if (*str == ',')
112 str++;
113 }
114
115 return 0;
116}
117early_param("iommu", iommu_setup);
118
Alexey Kardashevskiy8e0a1612013-08-28 18:37:43 +1000119/*
120 * stdcix is only supposed to be used in hypervisor real mode as per
121 * the architecture spec
122 */
123static inline void __raw_rm_writeq(u64 val, volatile void __iomem *paddr)
124{
125 __asm__ __volatile__("stdcix %0,0,%1"
126 : : "r" (val), "r" (paddr) : "memory");
127}
128
Guo Chao262af552014-07-21 14:42:30 +1000129static inline bool pnv_pci_is_mem_pref_64(unsigned long flags)
130{
131 return ((flags & (IORESOURCE_MEM_64 | IORESOURCE_PREFETCH)) ==
132 (IORESOURCE_MEM_64 | IORESOURCE_PREFETCH));
133}
134
Gavin Shan4b82ab12014-11-12 13:36:07 +1100135static void pnv_ioda_reserve_pe(struct pnv_phb *phb, int pe_no)
136{
137 if (!(pe_no >= 0 && pe_no < phb->ioda.total_pe)) {
138 pr_warn("%s: Invalid PE %d on PHB#%x\n",
139 __func__, pe_no, phb->hose->global_number);
140 return;
141 }
142
143 if (test_and_set_bit(pe_no, phb->ioda.pe_alloc)) {
144 pr_warn("%s: PE %d was assigned on PHB#%x\n",
145 __func__, pe_no, phb->hose->global_number);
146 return;
147 }
148
149 phb->ioda.pe_array[pe_no].phb = phb;
150 phb->ioda.pe_array[pe_no].pe_number = pe_no;
151}
152
Greg Kroah-Hartmancad5cef2012-12-21 14:04:10 -0800153static int pnv_ioda_alloc_pe(struct pnv_phb *phb)
Benjamin Herrenschmidt184cd4a2011-11-15 17:29:08 +0000154{
155 unsigned long pe;
156
157 do {
158 pe = find_next_zero_bit(phb->ioda.pe_alloc,
159 phb->ioda.total_pe, 0);
160 if (pe >= phb->ioda.total_pe)
161 return IODA_INVALID_PE;
162 } while(test_and_set_bit(pe, phb->ioda.pe_alloc));
163
Gavin Shan4cce9552013-04-25 19:21:00 +0000164 phb->ioda.pe_array[pe].phb = phb;
Benjamin Herrenschmidt184cd4a2011-11-15 17:29:08 +0000165 phb->ioda.pe_array[pe].pe_number = pe;
166 return pe;
167}
168
Greg Kroah-Hartmancad5cef2012-12-21 14:04:10 -0800169static void pnv_ioda_free_pe(struct pnv_phb *phb, int pe)
Benjamin Herrenschmidt184cd4a2011-11-15 17:29:08 +0000170{
171 WARN_ON(phb->ioda.pe_array[pe].pdev);
172
173 memset(&phb->ioda.pe_array[pe], 0, sizeof(struct pnv_ioda_pe));
174 clear_bit(pe, phb->ioda.pe_alloc);
175}
176
Guo Chao262af552014-07-21 14:42:30 +1000177/* The default M64 BAR is shared by all PEs */
178static int pnv_ioda2_init_m64(struct pnv_phb *phb)
179{
180 const char *desc;
181 struct resource *r;
182 s64 rc;
183
184 /* Configure the default M64 BAR */
185 rc = opal_pci_set_phb_mem_window(phb->opal_id,
186 OPAL_M64_WINDOW_TYPE,
187 phb->ioda.m64_bar_idx,
188 phb->ioda.m64_base,
189 0, /* unused */
190 phb->ioda.m64_size);
191 if (rc != OPAL_SUCCESS) {
192 desc = "configuring";
193 goto fail;
194 }
195
196 /* Enable the default M64 BAR */
197 rc = opal_pci_phb_mmio_enable(phb->opal_id,
198 OPAL_M64_WINDOW_TYPE,
199 phb->ioda.m64_bar_idx,
200 OPAL_ENABLE_M64_SPLIT);
201 if (rc != OPAL_SUCCESS) {
202 desc = "enabling";
203 goto fail;
204 }
205
206 /* Mark the M64 BAR assigned */
207 set_bit(phb->ioda.m64_bar_idx, &phb->ioda.m64_bar_alloc);
208
209 /*
210 * Strip off the segment used by the reserved PE, which is
211 * expected to be 0 or last one of PE capabicity.
212 */
213 r = &phb->hose->mem_resources[1];
214 if (phb->ioda.reserved_pe == 0)
215 r->start += phb->ioda.m64_segsize;
216 else if (phb->ioda.reserved_pe == (phb->ioda.total_pe - 1))
217 r->end -= phb->ioda.m64_segsize;
218 else
219 pr_warn(" Cannot strip M64 segment for reserved PE#%d\n",
220 phb->ioda.reserved_pe);
221
222 return 0;
223
224fail:
225 pr_warn(" Failure %lld %s M64 BAR#%d\n",
226 rc, desc, phb->ioda.m64_bar_idx);
227 opal_pci_phb_mmio_enable(phb->opal_id,
228 OPAL_M64_WINDOW_TYPE,
229 phb->ioda.m64_bar_idx,
230 OPAL_DISABLE_M64);
231 return -EIO;
232}
233
Gavin Shan5ef73562014-11-12 13:36:06 +1100234static void pnv_ioda2_reserve_m64_pe(struct pnv_phb *phb)
Guo Chao262af552014-07-21 14:42:30 +1000235{
236 resource_size_t sgsz = phb->ioda.m64_segsize;
237 struct pci_dev *pdev;
238 struct resource *r;
239 int base, step, i;
240
241 /*
242 * Root bus always has full M64 range and root port has
243 * M64 range used in reality. So we're checking root port
244 * instead of root bus.
245 */
246 list_for_each_entry(pdev, &phb->hose->bus->devices, bus_list) {
Gavin Shan4b82ab12014-11-12 13:36:07 +1100247 for (i = 0; i < PCI_BRIDGE_RESOURCE_NUM; i++) {
248 r = &pdev->resource[PCI_BRIDGE_RESOURCES + i];
Guo Chao262af552014-07-21 14:42:30 +1000249 if (!r->parent ||
250 !pnv_pci_is_mem_pref_64(r->flags))
251 continue;
252
253 base = (r->start - phb->ioda.m64_base) / sgsz;
254 for (step = 0; step < resource_size(r) / sgsz; step++)
Gavin Shan4b82ab12014-11-12 13:36:07 +1100255 pnv_ioda_reserve_pe(phb, base + step);
Guo Chao262af552014-07-21 14:42:30 +1000256 }
257 }
258}
259
260static int pnv_ioda2_pick_m64_pe(struct pnv_phb *phb,
261 struct pci_bus *bus, int all)
262{
263 resource_size_t segsz = phb->ioda.m64_segsize;
264 struct pci_dev *pdev;
265 struct resource *r;
266 struct pnv_ioda_pe *master_pe, *pe;
267 unsigned long size, *pe_alloc;
268 bool found;
269 int start, i, j;
270
271 /* Root bus shouldn't use M64 */
272 if (pci_is_root_bus(bus))
273 return IODA_INVALID_PE;
274
275 /* We support only one M64 window on each bus */
276 found = false;
277 pci_bus_for_each_resource(bus, r, i) {
278 if (r && r->parent &&
279 pnv_pci_is_mem_pref_64(r->flags)) {
280 found = true;
281 break;
282 }
283 }
284
285 /* No M64 window found ? */
286 if (!found)
287 return IODA_INVALID_PE;
288
289 /* Allocate bitmap */
290 size = _ALIGN_UP(phb->ioda.total_pe / 8, sizeof(unsigned long));
291 pe_alloc = kzalloc(size, GFP_KERNEL);
292 if (!pe_alloc) {
293 pr_warn("%s: Out of memory !\n",
294 __func__);
295 return IODA_INVALID_PE;
296 }
297
298 /*
299 * Figure out reserved PE numbers by the PE
300 * the its child PEs.
301 */
302 start = (r->start - phb->ioda.m64_base) / segsz;
303 for (i = 0; i < resource_size(r) / segsz; i++)
304 set_bit(start + i, pe_alloc);
305
306 if (all)
307 goto done;
308
309 /*
310 * If the PE doesn't cover all subordinate buses,
311 * we need subtract from reserved PEs for children.
312 */
313 list_for_each_entry(pdev, &bus->devices, bus_list) {
314 if (!pdev->subordinate)
315 continue;
316
317 pci_bus_for_each_resource(pdev->subordinate, r, i) {
318 if (!r || !r->parent ||
319 !pnv_pci_is_mem_pref_64(r->flags))
320 continue;
321
322 start = (r->start - phb->ioda.m64_base) / segsz;
323 for (j = 0; j < resource_size(r) / segsz ; j++)
324 clear_bit(start + j, pe_alloc);
325 }
326 }
327
328 /*
329 * the current bus might not own M64 window and that's all
330 * contributed by its child buses. For the case, we needn't
331 * pick M64 dependent PE#.
332 */
333 if (bitmap_empty(pe_alloc, phb->ioda.total_pe)) {
334 kfree(pe_alloc);
335 return IODA_INVALID_PE;
336 }
337
338 /*
339 * Figure out the master PE and put all slave PEs to master
340 * PE's list to form compound PE.
341 */
342done:
343 master_pe = NULL;
344 i = -1;
345 while ((i = find_next_bit(pe_alloc, phb->ioda.total_pe, i + 1)) <
346 phb->ioda.total_pe) {
347 pe = &phb->ioda.pe_array[i];
Guo Chao262af552014-07-21 14:42:30 +1000348
349 if (!master_pe) {
350 pe->flags |= PNV_IODA_PE_MASTER;
351 INIT_LIST_HEAD(&pe->slaves);
352 master_pe = pe;
353 } else {
354 pe->flags |= PNV_IODA_PE_SLAVE;
355 pe->master = master_pe;
356 list_add_tail(&pe->list, &master_pe->slaves);
357 }
358 }
359
360 kfree(pe_alloc);
361 return master_pe->pe_number;
362}
363
364static void __init pnv_ioda_parse_m64_window(struct pnv_phb *phb)
365{
366 struct pci_controller *hose = phb->hose;
367 struct device_node *dn = hose->dn;
368 struct resource *res;
369 const u32 *r;
370 u64 pci_addr;
371
Gavin Shan1665c4a2014-11-12 13:36:04 +1100372 /* FIXME: Support M64 for P7IOC */
373 if (phb->type != PNV_PHB_IODA2) {
374 pr_info(" Not support M64 window\n");
375 return;
376 }
377
Guo Chao262af552014-07-21 14:42:30 +1000378 if (!firmware_has_feature(FW_FEATURE_OPALv3)) {
379 pr_info(" Firmware too old to support M64 window\n");
380 return;
381 }
382
383 r = of_get_property(dn, "ibm,opal-m64-window", NULL);
384 if (!r) {
385 pr_info(" No <ibm,opal-m64-window> on %s\n",
386 dn->full_name);
387 return;
388 }
389
Guo Chao262af552014-07-21 14:42:30 +1000390 res = &hose->mem_resources[1];
391 res->start = of_translate_address(dn, r + 2);
392 res->end = res->start + of_read_number(r + 4, 2) - 1;
393 res->flags = (IORESOURCE_MEM | IORESOURCE_MEM_64 | IORESOURCE_PREFETCH);
394 pci_addr = of_read_number(r, 2);
395 hose->mem_offset[1] = res->start - pci_addr;
396
397 phb->ioda.m64_size = resource_size(res);
398 phb->ioda.m64_segsize = phb->ioda.m64_size / phb->ioda.total_pe;
399 phb->ioda.m64_base = pci_addr;
400
Wei Yange9863e62014-12-12 12:39:37 +0800401 pr_info(" MEM64 0x%016llx..0x%016llx -> 0x%016llx\n",
402 res->start, res->end, pci_addr);
403
Guo Chao262af552014-07-21 14:42:30 +1000404 /* Use last M64 BAR to cover M64 window */
405 phb->ioda.m64_bar_idx = 15;
406 phb->init_m64 = pnv_ioda2_init_m64;
Gavin Shan5ef73562014-11-12 13:36:06 +1100407 phb->reserve_m64_pe = pnv_ioda2_reserve_m64_pe;
Guo Chao262af552014-07-21 14:42:30 +1000408 phb->pick_m64_pe = pnv_ioda2_pick_m64_pe;
409}
410
Gavin Shan49dec922014-07-21 14:42:33 +1000411static void pnv_ioda_freeze_pe(struct pnv_phb *phb, int pe_no)
412{
413 struct pnv_ioda_pe *pe = &phb->ioda.pe_array[pe_no];
414 struct pnv_ioda_pe *slave;
415 s64 rc;
416
417 /* Fetch master PE */
418 if (pe->flags & PNV_IODA_PE_SLAVE) {
419 pe = pe->master;
Gavin Shanec8e4e92014-11-12 13:36:10 +1100420 if (WARN_ON(!pe || !(pe->flags & PNV_IODA_PE_MASTER)))
421 return;
422
Gavin Shan49dec922014-07-21 14:42:33 +1000423 pe_no = pe->pe_number;
424 }
425
426 /* Freeze master PE */
427 rc = opal_pci_eeh_freeze_set(phb->opal_id,
428 pe_no,
429 OPAL_EEH_ACTION_SET_FREEZE_ALL);
430 if (rc != OPAL_SUCCESS) {
431 pr_warn("%s: Failure %lld freezing PHB#%x-PE#%x\n",
432 __func__, rc, phb->hose->global_number, pe_no);
433 return;
434 }
435
436 /* Freeze slave PEs */
437 if (!(pe->flags & PNV_IODA_PE_MASTER))
438 return;
439
440 list_for_each_entry(slave, &pe->slaves, list) {
441 rc = opal_pci_eeh_freeze_set(phb->opal_id,
442 slave->pe_number,
443 OPAL_EEH_ACTION_SET_FREEZE_ALL);
444 if (rc != OPAL_SUCCESS)
445 pr_warn("%s: Failure %lld freezing PHB#%x-PE#%x\n",
446 __func__, rc, phb->hose->global_number,
447 slave->pe_number);
448 }
449}
450
Anton Blancharde51df2c2014-08-20 08:55:18 +1000451static int pnv_ioda_unfreeze_pe(struct pnv_phb *phb, int pe_no, int opt)
Gavin Shan49dec922014-07-21 14:42:33 +1000452{
453 struct pnv_ioda_pe *pe, *slave;
454 s64 rc;
455
456 /* Find master PE */
457 pe = &phb->ioda.pe_array[pe_no];
458 if (pe->flags & PNV_IODA_PE_SLAVE) {
459 pe = pe->master;
460 WARN_ON(!pe || !(pe->flags & PNV_IODA_PE_MASTER));
461 pe_no = pe->pe_number;
462 }
463
464 /* Clear frozen state for master PE */
465 rc = opal_pci_eeh_freeze_clear(phb->opal_id, pe_no, opt);
466 if (rc != OPAL_SUCCESS) {
467 pr_warn("%s: Failure %lld clear %d on PHB#%x-PE#%x\n",
468 __func__, rc, opt, phb->hose->global_number, pe_no);
469 return -EIO;
470 }
471
472 if (!(pe->flags & PNV_IODA_PE_MASTER))
473 return 0;
474
475 /* Clear frozen state for slave PEs */
476 list_for_each_entry(slave, &pe->slaves, list) {
477 rc = opal_pci_eeh_freeze_clear(phb->opal_id,
478 slave->pe_number,
479 opt);
480 if (rc != OPAL_SUCCESS) {
481 pr_warn("%s: Failure %lld clear %d on PHB#%x-PE#%x\n",
482 __func__, rc, opt, phb->hose->global_number,
483 slave->pe_number);
484 return -EIO;
485 }
486 }
487
488 return 0;
489}
490
491static int pnv_ioda_get_pe_state(struct pnv_phb *phb, int pe_no)
492{
493 struct pnv_ioda_pe *slave, *pe;
494 u8 fstate, state;
495 __be16 pcierr;
496 s64 rc;
497
498 /* Sanity check on PE number */
499 if (pe_no < 0 || pe_no >= phb->ioda.total_pe)
500 return OPAL_EEH_STOPPED_PERM_UNAVAIL;
501
502 /*
503 * Fetch the master PE and the PE instance might be
504 * not initialized yet.
505 */
506 pe = &phb->ioda.pe_array[pe_no];
507 if (pe->flags & PNV_IODA_PE_SLAVE) {
508 pe = pe->master;
509 WARN_ON(!pe || !(pe->flags & PNV_IODA_PE_MASTER));
510 pe_no = pe->pe_number;
511 }
512
513 /* Check the master PE */
514 rc = opal_pci_eeh_freeze_status(phb->opal_id, pe_no,
515 &state, &pcierr, NULL);
516 if (rc != OPAL_SUCCESS) {
517 pr_warn("%s: Failure %lld getting "
518 "PHB#%x-PE#%x state\n",
519 __func__, rc,
520 phb->hose->global_number, pe_no);
521 return OPAL_EEH_STOPPED_TEMP_UNAVAIL;
522 }
523
524 /* Check the slave PE */
525 if (!(pe->flags & PNV_IODA_PE_MASTER))
526 return state;
527
528 list_for_each_entry(slave, &pe->slaves, list) {
529 rc = opal_pci_eeh_freeze_status(phb->opal_id,
530 slave->pe_number,
531 &fstate,
532 &pcierr,
533 NULL);
534 if (rc != OPAL_SUCCESS) {
535 pr_warn("%s: Failure %lld getting "
536 "PHB#%x-PE#%x state\n",
537 __func__, rc,
538 phb->hose->global_number, slave->pe_number);
539 return OPAL_EEH_STOPPED_TEMP_UNAVAIL;
540 }
541
542 /*
543 * Override the result based on the ascending
544 * priority.
545 */
546 if (fstate > state)
547 state = fstate;
548 }
549
550 return state;
551}
552
Benjamin Herrenschmidt184cd4a2011-11-15 17:29:08 +0000553/* Currently those 2 are only used when MSIs are enabled, this will change
554 * but in the meantime, we need to protect them to avoid warnings
555 */
556#ifdef CONFIG_PCI_MSI
Greg Kroah-Hartmancad5cef2012-12-21 14:04:10 -0800557static struct pnv_ioda_pe *pnv_ioda_get_pe(struct pci_dev *dev)
Benjamin Herrenschmidt184cd4a2011-11-15 17:29:08 +0000558{
559 struct pci_controller *hose = pci_bus_to_host(dev->bus);
560 struct pnv_phb *phb = hose->private_data;
Benjamin Herrenschmidtb72c1f62013-05-21 22:58:21 +0000561 struct pci_dn *pdn = pci_get_pdn(dev);
Benjamin Herrenschmidt184cd4a2011-11-15 17:29:08 +0000562
563 if (!pdn)
564 return NULL;
565 if (pdn->pe_number == IODA_INVALID_PE)
566 return NULL;
567 return &phb->ioda.pe_array[pdn->pe_number];
568}
Benjamin Herrenschmidt184cd4a2011-11-15 17:29:08 +0000569#endif /* CONFIG_PCI_MSI */
570
Gavin Shanb131a842014-11-12 13:36:08 +1100571static int pnv_ioda_set_one_peltv(struct pnv_phb *phb,
572 struct pnv_ioda_pe *parent,
573 struct pnv_ioda_pe *child,
574 bool is_add)
575{
576 const char *desc = is_add ? "adding" : "removing";
577 uint8_t op = is_add ? OPAL_ADD_PE_TO_DOMAIN :
578 OPAL_REMOVE_PE_FROM_DOMAIN;
579 struct pnv_ioda_pe *slave;
580 long rc;
581
582 /* Parent PE affects child PE */
583 rc = opal_pci_set_peltv(phb->opal_id, parent->pe_number,
584 child->pe_number, op);
585 if (rc != OPAL_SUCCESS) {
586 pe_warn(child, "OPAL error %ld %s to parent PELTV\n",
587 rc, desc);
588 return -ENXIO;
589 }
590
591 if (!(child->flags & PNV_IODA_PE_MASTER))
592 return 0;
593
594 /* Compound case: parent PE affects slave PEs */
595 list_for_each_entry(slave, &child->slaves, list) {
596 rc = opal_pci_set_peltv(phb->opal_id, parent->pe_number,
597 slave->pe_number, op);
598 if (rc != OPAL_SUCCESS) {
599 pe_warn(slave, "OPAL error %ld %s to parent PELTV\n",
600 rc, desc);
601 return -ENXIO;
602 }
603 }
604
605 return 0;
606}
607
608static int pnv_ioda_set_peltv(struct pnv_phb *phb,
609 struct pnv_ioda_pe *pe,
610 bool is_add)
611{
612 struct pnv_ioda_pe *slave;
Wei Yang781a8682015-03-25 16:23:57 +0800613 struct pci_dev *pdev = NULL;
Gavin Shanb131a842014-11-12 13:36:08 +1100614 int ret;
615
616 /*
617 * Clear PE frozen state. If it's master PE, we need
618 * clear slave PE frozen state as well.
619 */
620 if (is_add) {
621 opal_pci_eeh_freeze_clear(phb->opal_id, pe->pe_number,
622 OPAL_EEH_ACTION_CLEAR_FREEZE_ALL);
623 if (pe->flags & PNV_IODA_PE_MASTER) {
624 list_for_each_entry(slave, &pe->slaves, list)
625 opal_pci_eeh_freeze_clear(phb->opal_id,
626 slave->pe_number,
627 OPAL_EEH_ACTION_CLEAR_FREEZE_ALL);
628 }
629 }
630
631 /*
632 * Associate PE in PELT. We need add the PE into the
633 * corresponding PELT-V as well. Otherwise, the error
634 * originated from the PE might contribute to other
635 * PEs.
636 */
637 ret = pnv_ioda_set_one_peltv(phb, pe, pe, is_add);
638 if (ret)
639 return ret;
640
641 /* For compound PEs, any one affects all of them */
642 if (pe->flags & PNV_IODA_PE_MASTER) {
643 list_for_each_entry(slave, &pe->slaves, list) {
644 ret = pnv_ioda_set_one_peltv(phb, slave, pe, is_add);
645 if (ret)
646 return ret;
647 }
648 }
649
650 if (pe->flags & (PNV_IODA_PE_BUS_ALL | PNV_IODA_PE_BUS))
651 pdev = pe->pbus->self;
Wei Yang781a8682015-03-25 16:23:57 +0800652 else if (pe->flags & PNV_IODA_PE_DEV)
Gavin Shanb131a842014-11-12 13:36:08 +1100653 pdev = pe->pdev->bus->self;
Wei Yang781a8682015-03-25 16:23:57 +0800654#ifdef CONFIG_PCI_IOV
655 else if (pe->flags & PNV_IODA_PE_VF)
656 pdev = pe->parent_dev->bus->self;
657#endif /* CONFIG_PCI_IOV */
Gavin Shanb131a842014-11-12 13:36:08 +1100658 while (pdev) {
659 struct pci_dn *pdn = pci_get_pdn(pdev);
660 struct pnv_ioda_pe *parent;
661
662 if (pdn && pdn->pe_number != IODA_INVALID_PE) {
663 parent = &phb->ioda.pe_array[pdn->pe_number];
664 ret = pnv_ioda_set_one_peltv(phb, parent, pe, is_add);
665 if (ret)
666 return ret;
667 }
668
669 pdev = pdev->bus->self;
670 }
671
672 return 0;
673}
674
Wei Yang781a8682015-03-25 16:23:57 +0800675#ifdef CONFIG_PCI_IOV
676static int pnv_ioda_deconfigure_pe(struct pnv_phb *phb, struct pnv_ioda_pe *pe)
677{
678 struct pci_dev *parent;
679 uint8_t bcomp, dcomp, fcomp;
680 int64_t rc;
681 long rid_end, rid;
682
683 /* Currently, we just deconfigure VF PE. Bus PE will always there.*/
684 if (pe->pbus) {
685 int count;
686
687 dcomp = OPAL_IGNORE_RID_DEVICE_NUMBER;
688 fcomp = OPAL_IGNORE_RID_FUNCTION_NUMBER;
689 parent = pe->pbus->self;
690 if (pe->flags & PNV_IODA_PE_BUS_ALL)
691 count = pe->pbus->busn_res.end - pe->pbus->busn_res.start + 1;
692 else
693 count = 1;
694
695 switch(count) {
696 case 1: bcomp = OpalPciBusAll; break;
697 case 2: bcomp = OpalPciBus7Bits; break;
698 case 4: bcomp = OpalPciBus6Bits; break;
699 case 8: bcomp = OpalPciBus5Bits; break;
700 case 16: bcomp = OpalPciBus4Bits; break;
701 case 32: bcomp = OpalPciBus3Bits; break;
702 default:
703 dev_err(&pe->pbus->dev, "Number of subordinate buses %d unsupported\n",
704 count);
705 /* Do an exact match only */
706 bcomp = OpalPciBusAll;
707 }
708 rid_end = pe->rid + (count << 8);
709 } else {
710 if (pe->flags & PNV_IODA_PE_VF)
711 parent = pe->parent_dev;
712 else
713 parent = pe->pdev->bus->self;
714 bcomp = OpalPciBusAll;
715 dcomp = OPAL_COMPARE_RID_DEVICE_NUMBER;
716 fcomp = OPAL_COMPARE_RID_FUNCTION_NUMBER;
717 rid_end = pe->rid + 1;
718 }
719
720 /* Clear the reverse map */
721 for (rid = pe->rid; rid < rid_end; rid++)
722 phb->ioda.pe_rmap[rid] = 0;
723
724 /* Release from all parents PELT-V */
725 while (parent) {
726 struct pci_dn *pdn = pci_get_pdn(parent);
727 if (pdn && pdn->pe_number != IODA_INVALID_PE) {
728 rc = opal_pci_set_peltv(phb->opal_id, pdn->pe_number,
729 pe->pe_number, OPAL_REMOVE_PE_FROM_DOMAIN);
730 /* XXX What to do in case of error ? */
731 }
732 parent = parent->bus->self;
733 }
734
735 opal_pci_eeh_freeze_set(phb->opal_id, pe->pe_number,
736 OPAL_EEH_ACTION_CLEAR_FREEZE_ALL);
737
738 /* Disassociate PE in PELT */
739 rc = opal_pci_set_peltv(phb->opal_id, pe->pe_number,
740 pe->pe_number, OPAL_REMOVE_PE_FROM_DOMAIN);
741 if (rc)
742 pe_warn(pe, "OPAL error %ld remove self from PELTV\n", rc);
743 rc = opal_pci_set_pe(phb->opal_id, pe->pe_number, pe->rid,
744 bcomp, dcomp, fcomp, OPAL_UNMAP_PE);
745 if (rc)
746 pe_err(pe, "OPAL error %ld trying to setup PELT table\n", rc);
747
748 pe->pbus = NULL;
749 pe->pdev = NULL;
750 pe->parent_dev = NULL;
751
752 return 0;
753}
754#endif /* CONFIG_PCI_IOV */
755
Greg Kroah-Hartmancad5cef2012-12-21 14:04:10 -0800756static int pnv_ioda_configure_pe(struct pnv_phb *phb, struct pnv_ioda_pe *pe)
Benjamin Herrenschmidt184cd4a2011-11-15 17:29:08 +0000757{
758 struct pci_dev *parent;
759 uint8_t bcomp, dcomp, fcomp;
760 long rc, rid_end, rid;
761
762 /* Bus validation ? */
763 if (pe->pbus) {
764 int count;
765
766 dcomp = OPAL_IGNORE_RID_DEVICE_NUMBER;
767 fcomp = OPAL_IGNORE_RID_FUNCTION_NUMBER;
768 parent = pe->pbus->self;
Gavin Shanfb446ad2012-08-20 03:49:14 +0000769 if (pe->flags & PNV_IODA_PE_BUS_ALL)
770 count = pe->pbus->busn_res.end - pe->pbus->busn_res.start + 1;
771 else
772 count = 1;
773
Benjamin Herrenschmidt184cd4a2011-11-15 17:29:08 +0000774 switch(count) {
775 case 1: bcomp = OpalPciBusAll; break;
776 case 2: bcomp = OpalPciBus7Bits; break;
777 case 4: bcomp = OpalPciBus6Bits; break;
778 case 8: bcomp = OpalPciBus5Bits; break;
779 case 16: bcomp = OpalPciBus4Bits; break;
780 case 32: bcomp = OpalPciBus3Bits; break;
781 default:
Wei Yang781a8682015-03-25 16:23:57 +0800782 dev_err(&pe->pbus->dev, "Number of subordinate buses %d unsupported\n",
783 count);
Benjamin Herrenschmidt184cd4a2011-11-15 17:29:08 +0000784 /* Do an exact match only */
785 bcomp = OpalPciBusAll;
786 }
787 rid_end = pe->rid + (count << 8);
788 } else {
Wei Yang781a8682015-03-25 16:23:57 +0800789#ifdef CONFIG_PCI_IOV
790 if (pe->flags & PNV_IODA_PE_VF)
791 parent = pe->parent_dev;
792 else
793#endif /* CONFIG_PCI_IOV */
794 parent = pe->pdev->bus->self;
Benjamin Herrenschmidt184cd4a2011-11-15 17:29:08 +0000795 bcomp = OpalPciBusAll;
796 dcomp = OPAL_COMPARE_RID_DEVICE_NUMBER;
797 fcomp = OPAL_COMPARE_RID_FUNCTION_NUMBER;
798 rid_end = pe->rid + 1;
799 }
800
Gavin Shan631ad692013-11-04 16:32:46 +0800801 /*
802 * Associate PE in PELT. We need add the PE into the
803 * corresponding PELT-V as well. Otherwise, the error
804 * originated from the PE might contribute to other
805 * PEs.
806 */
Benjamin Herrenschmidt184cd4a2011-11-15 17:29:08 +0000807 rc = opal_pci_set_pe(phb->opal_id, pe->pe_number, pe->rid,
808 bcomp, dcomp, fcomp, OPAL_MAP_PE);
809 if (rc) {
810 pe_err(pe, "OPAL error %ld trying to setup PELT table\n", rc);
811 return -ENXIO;
812 }
Gavin Shan631ad692013-11-04 16:32:46 +0800813
Gavin Shanb131a842014-11-12 13:36:08 +1100814 /* Configure PELTV */
815 pnv_ioda_set_peltv(phb, pe, true);
Benjamin Herrenschmidt184cd4a2011-11-15 17:29:08 +0000816
Benjamin Herrenschmidt184cd4a2011-11-15 17:29:08 +0000817 /* Setup reverse map */
818 for (rid = pe->rid; rid < rid_end; rid++)
819 phb->ioda.pe_rmap[rid] = pe->pe_number;
820
821 /* Setup one MVTs on IODA1 */
Gavin Shan4773f762014-11-12 13:36:09 +1100822 if (phb->type != PNV_PHB_IODA1) {
823 pe->mve_number = 0;
824 goto out;
825 }
826
827 pe->mve_number = pe->pe_number;
828 rc = opal_pci_set_mve(phb->opal_id, pe->mve_number, pe->pe_number);
829 if (rc != OPAL_SUCCESS) {
830 pe_err(pe, "OPAL error %ld setting up MVE %d\n",
831 rc, pe->mve_number);
832 pe->mve_number = -1;
833 } else {
834 rc = opal_pci_set_mve_enable(phb->opal_id,
835 pe->mve_number, OPAL_ENABLE_MVE);
Benjamin Herrenschmidt184cd4a2011-11-15 17:29:08 +0000836 if (rc) {
Gavin Shan4773f762014-11-12 13:36:09 +1100837 pe_err(pe, "OPAL error %ld enabling MVE %d\n",
Benjamin Herrenschmidt184cd4a2011-11-15 17:29:08 +0000838 rc, pe->mve_number);
839 pe->mve_number = -1;
Benjamin Herrenschmidt184cd4a2011-11-15 17:29:08 +0000840 }
Gavin Shan4773f762014-11-12 13:36:09 +1100841 }
Benjamin Herrenschmidt184cd4a2011-11-15 17:29:08 +0000842
Gavin Shan4773f762014-11-12 13:36:09 +1100843out:
Benjamin Herrenschmidt184cd4a2011-11-15 17:29:08 +0000844 return 0;
845}
846
Greg Kroah-Hartmancad5cef2012-12-21 14:04:10 -0800847static void pnv_ioda_link_pe_by_weight(struct pnv_phb *phb,
848 struct pnv_ioda_pe *pe)
Benjamin Herrenschmidt184cd4a2011-11-15 17:29:08 +0000849{
850 struct pnv_ioda_pe *lpe;
851
Gavin Shan7ebdf952012-08-20 03:49:15 +0000852 list_for_each_entry(lpe, &phb->ioda.pe_dma_list, dma_link) {
Benjamin Herrenschmidt184cd4a2011-11-15 17:29:08 +0000853 if (lpe->dma_weight < pe->dma_weight) {
Gavin Shan7ebdf952012-08-20 03:49:15 +0000854 list_add_tail(&pe->dma_link, &lpe->dma_link);
Benjamin Herrenschmidt184cd4a2011-11-15 17:29:08 +0000855 return;
856 }
857 }
Gavin Shan7ebdf952012-08-20 03:49:15 +0000858 list_add_tail(&pe->dma_link, &phb->ioda.pe_dma_list);
Benjamin Herrenschmidt184cd4a2011-11-15 17:29:08 +0000859}
860
861static unsigned int pnv_ioda_dma_weight(struct pci_dev *dev)
862{
863 /* This is quite simplistic. The "base" weight of a device
864 * is 10. 0 means no DMA is to be accounted for it.
865 */
866
867 /* If it's a bridge, no DMA */
868 if (dev->hdr_type != PCI_HEADER_TYPE_NORMAL)
869 return 0;
870
871 /* Reduce the weight of slow USB controllers */
872 if (dev->class == PCI_CLASS_SERIAL_USB_UHCI ||
873 dev->class == PCI_CLASS_SERIAL_USB_OHCI ||
874 dev->class == PCI_CLASS_SERIAL_USB_EHCI)
875 return 3;
876
877 /* Increase the weight of RAID (includes Obsidian) */
878 if ((dev->class >> 8) == PCI_CLASS_STORAGE_RAID)
879 return 15;
880
881 /* Default */
882 return 10;
883}
884
Wei Yang781a8682015-03-25 16:23:57 +0800885#ifdef CONFIG_PCI_IOV
886static int pnv_pci_vf_resource_shift(struct pci_dev *dev, int offset)
887{
888 struct pci_dn *pdn = pci_get_pdn(dev);
889 int i;
890 struct resource *res, res2;
891 resource_size_t size;
892 u16 num_vfs;
893
894 if (!dev->is_physfn)
895 return -EINVAL;
896
897 /*
898 * "offset" is in VFs. The M64 windows are sized so that when they
899 * are segmented, each segment is the same size as the IOV BAR.
900 * Each segment is in a separate PE, and the high order bits of the
901 * address are the PE number. Therefore, each VF's BAR is in a
902 * separate PE, and changing the IOV BAR start address changes the
903 * range of PEs the VFs are in.
904 */
905 num_vfs = pdn->num_vfs;
906 for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) {
907 res = &dev->resource[i + PCI_IOV_RESOURCES];
908 if (!res->flags || !res->parent)
909 continue;
910
911 if (!pnv_pci_is_mem_pref_64(res->flags))
912 continue;
913
914 /*
915 * The actual IOV BAR range is determined by the start address
916 * and the actual size for num_vfs VFs BAR. This check is to
917 * make sure that after shifting, the range will not overlap
918 * with another device.
919 */
920 size = pci_iov_resource_size(dev, i + PCI_IOV_RESOURCES);
921 res2.flags = res->flags;
922 res2.start = res->start + (size * offset);
923 res2.end = res2.start + (size * num_vfs) - 1;
924
925 if (res2.end > res->end) {
926 dev_err(&dev->dev, "VF BAR%d: %pR would extend past %pR (trying to enable %d VFs shifted by %d)\n",
927 i, &res2, res, num_vfs, offset);
928 return -EBUSY;
929 }
930 }
931
932 /*
933 * After doing so, there would be a "hole" in the /proc/iomem when
934 * offset is a positive value. It looks like the device return some
935 * mmio back to the system, which actually no one could use it.
936 */
937 for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) {
938 res = &dev->resource[i + PCI_IOV_RESOURCES];
939 if (!res->flags || !res->parent)
940 continue;
941
942 if (!pnv_pci_is_mem_pref_64(res->flags))
943 continue;
944
945 size = pci_iov_resource_size(dev, i + PCI_IOV_RESOURCES);
946 res2 = *res;
947 res->start += size * offset;
948
949 dev_info(&dev->dev, "VF BAR%d: %pR shifted to %pR (enabling %d VFs shifted by %d)\n",
950 i, &res2, res, num_vfs, offset);
951 pci_update_resource(dev, i + PCI_IOV_RESOURCES);
952 }
953 return 0;
954}
955#endif /* CONFIG_PCI_IOV */
956
Gavin Shanfb446ad2012-08-20 03:49:14 +0000957#if 0
Greg Kroah-Hartmancad5cef2012-12-21 14:04:10 -0800958static struct pnv_ioda_pe *pnv_ioda_setup_dev_PE(struct pci_dev *dev)
Benjamin Herrenschmidt184cd4a2011-11-15 17:29:08 +0000959{
960 struct pci_controller *hose = pci_bus_to_host(dev->bus);
961 struct pnv_phb *phb = hose->private_data;
Benjamin Herrenschmidtb72c1f62013-05-21 22:58:21 +0000962 struct pci_dn *pdn = pci_get_pdn(dev);
Benjamin Herrenschmidt184cd4a2011-11-15 17:29:08 +0000963 struct pnv_ioda_pe *pe;
964 int pe_num;
965
966 if (!pdn) {
967 pr_err("%s: Device tree node not associated properly\n",
968 pci_name(dev));
969 return NULL;
970 }
971 if (pdn->pe_number != IODA_INVALID_PE)
972 return NULL;
973
974 /* PE#0 has been pre-set */
975 if (dev->bus->number == 0)
976 pe_num = 0;
977 else
978 pe_num = pnv_ioda_alloc_pe(phb);
979 if (pe_num == IODA_INVALID_PE) {
980 pr_warning("%s: Not enough PE# available, disabling device\n",
981 pci_name(dev));
982 return NULL;
983 }
984
985 /* NOTE: We get only one ref to the pci_dev for the pdn, not for the
986 * pointer in the PE data structure, both should be destroyed at the
987 * same time. However, this needs to be looked at more closely again
988 * once we actually start removing things (Hotplug, SR-IOV, ...)
989 *
990 * At some point we want to remove the PDN completely anyways
991 */
992 pe = &phb->ioda.pe_array[pe_num];
993 pci_dev_get(dev);
994 pdn->pcidev = dev;
995 pdn->pe_number = pe_num;
996 pe->pdev = dev;
997 pe->pbus = NULL;
998 pe->tce32_seg = -1;
999 pe->mve_number = -1;
1000 pe->rid = dev->bus->number << 8 | pdn->devfn;
1001
1002 pe_info(pe, "Associated device to PE\n");
1003
1004 if (pnv_ioda_configure_pe(phb, pe)) {
1005 /* XXX What do we do here ? */
1006 if (pe_num)
1007 pnv_ioda_free_pe(phb, pe_num);
1008 pdn->pe_number = IODA_INVALID_PE;
1009 pe->pdev = NULL;
1010 pci_dev_put(dev);
1011 return NULL;
1012 }
1013
1014 /* Assign a DMA weight to the device */
1015 pe->dma_weight = pnv_ioda_dma_weight(dev);
1016 if (pe->dma_weight != 0) {
1017 phb->ioda.dma_weight += pe->dma_weight;
1018 phb->ioda.dma_pe_count++;
1019 }
1020
1021 /* Link the PE */
1022 pnv_ioda_link_pe_by_weight(phb, pe);
1023
1024 return pe;
1025}
Gavin Shanfb446ad2012-08-20 03:49:14 +00001026#endif /* Useful for SRIOV case */
Benjamin Herrenschmidt184cd4a2011-11-15 17:29:08 +00001027
1028static void pnv_ioda_setup_same_PE(struct pci_bus *bus, struct pnv_ioda_pe *pe)
1029{
1030 struct pci_dev *dev;
1031
1032 list_for_each_entry(dev, &bus->devices, bus_list) {
Benjamin Herrenschmidtb72c1f62013-05-21 22:58:21 +00001033 struct pci_dn *pdn = pci_get_pdn(dev);
Benjamin Herrenschmidt184cd4a2011-11-15 17:29:08 +00001034
1035 if (pdn == NULL) {
1036 pr_warn("%s: No device node associated with device !\n",
1037 pci_name(dev));
1038 continue;
1039 }
Benjamin Herrenschmidt184cd4a2011-11-15 17:29:08 +00001040 pdn->pe_number = pe->pe_number;
1041 pe->dma_weight += pnv_ioda_dma_weight(dev);
Gavin Shanfb446ad2012-08-20 03:49:14 +00001042 if ((pe->flags & PNV_IODA_PE_BUS_ALL) && dev->subordinate)
Benjamin Herrenschmidt184cd4a2011-11-15 17:29:08 +00001043 pnv_ioda_setup_same_PE(dev->subordinate, pe);
1044 }
1045}
1046
Gavin Shanfb446ad2012-08-20 03:49:14 +00001047/*
1048 * There're 2 types of PCI bus sensitive PEs: One that is compromised of
1049 * single PCI bus. Another one that contains the primary PCI bus and its
1050 * subordinate PCI devices and buses. The second type of PE is normally
1051 * orgiriated by PCIe-to-PCI bridge or PLX switch downstream ports.
1052 */
Greg Kroah-Hartmancad5cef2012-12-21 14:04:10 -08001053static void pnv_ioda_setup_bus_PE(struct pci_bus *bus, int all)
Benjamin Herrenschmidt184cd4a2011-11-15 17:29:08 +00001054{
Gavin Shanfb446ad2012-08-20 03:49:14 +00001055 struct pci_controller *hose = pci_bus_to_host(bus);
Benjamin Herrenschmidt184cd4a2011-11-15 17:29:08 +00001056 struct pnv_phb *phb = hose->private_data;
Benjamin Herrenschmidt184cd4a2011-11-15 17:29:08 +00001057 struct pnv_ioda_pe *pe;
Guo Chao262af552014-07-21 14:42:30 +10001058 int pe_num = IODA_INVALID_PE;
Benjamin Herrenschmidt184cd4a2011-11-15 17:29:08 +00001059
Guo Chao262af552014-07-21 14:42:30 +10001060 /* Check if PE is determined by M64 */
1061 if (phb->pick_m64_pe)
1062 pe_num = phb->pick_m64_pe(phb, bus, all);
1063
1064 /* The PE number isn't pinned by M64 */
1065 if (pe_num == IODA_INVALID_PE)
1066 pe_num = pnv_ioda_alloc_pe(phb);
1067
Benjamin Herrenschmidt184cd4a2011-11-15 17:29:08 +00001068 if (pe_num == IODA_INVALID_PE) {
Gavin Shanfb446ad2012-08-20 03:49:14 +00001069 pr_warning("%s: Not enough PE# available for PCI bus %04x:%02x\n",
1070 __func__, pci_domain_nr(bus), bus->number);
Benjamin Herrenschmidt184cd4a2011-11-15 17:29:08 +00001071 return;
1072 }
1073
1074 pe = &phb->ioda.pe_array[pe_num];
Guo Chao262af552014-07-21 14:42:30 +10001075 pe->flags |= (all ? PNV_IODA_PE_BUS_ALL : PNV_IODA_PE_BUS);
Benjamin Herrenschmidt184cd4a2011-11-15 17:29:08 +00001076 pe->pbus = bus;
1077 pe->pdev = NULL;
1078 pe->tce32_seg = -1;
1079 pe->mve_number = -1;
Yinghai Lub918c622012-05-17 18:51:11 -07001080 pe->rid = bus->busn_res.start << 8;
Benjamin Herrenschmidt184cd4a2011-11-15 17:29:08 +00001081 pe->dma_weight = 0;
1082
Gavin Shanfb446ad2012-08-20 03:49:14 +00001083 if (all)
1084 pe_info(pe, "Secondary bus %d..%d associated with PE#%d\n",
1085 bus->busn_res.start, bus->busn_res.end, pe_num);
1086 else
1087 pe_info(pe, "Secondary bus %d associated with PE#%d\n",
1088 bus->busn_res.start, pe_num);
Benjamin Herrenschmidt184cd4a2011-11-15 17:29:08 +00001089
1090 if (pnv_ioda_configure_pe(phb, pe)) {
1091 /* XXX What do we do here ? */
1092 if (pe_num)
1093 pnv_ioda_free_pe(phb, pe_num);
1094 pe->pbus = NULL;
1095 return;
1096 }
1097
1098 /* Associate it with all child devices */
1099 pnv_ioda_setup_same_PE(bus, pe);
1100
Gavin Shan7ebdf952012-08-20 03:49:15 +00001101 /* Put PE to the list */
1102 list_add_tail(&pe->list, &phb->ioda.pe_list);
1103
Benjamin Herrenschmidt184cd4a2011-11-15 17:29:08 +00001104 /* Account for one DMA PE if at least one DMA capable device exist
1105 * below the bridge
1106 */
1107 if (pe->dma_weight != 0) {
1108 phb->ioda.dma_weight += pe->dma_weight;
1109 phb->ioda.dma_pe_count++;
1110 }
1111
1112 /* Link the PE */
1113 pnv_ioda_link_pe_by_weight(phb, pe);
1114}
1115
Greg Kroah-Hartmancad5cef2012-12-21 14:04:10 -08001116static void pnv_ioda_setup_PEs(struct pci_bus *bus)
Benjamin Herrenschmidt184cd4a2011-11-15 17:29:08 +00001117{
1118 struct pci_dev *dev;
Gavin Shanfb446ad2012-08-20 03:49:14 +00001119
1120 pnv_ioda_setup_bus_PE(bus, 0);
Benjamin Herrenschmidt184cd4a2011-11-15 17:29:08 +00001121
1122 list_for_each_entry(dev, &bus->devices, bus_list) {
Gavin Shanfb446ad2012-08-20 03:49:14 +00001123 if (dev->subordinate) {
1124 if (pci_pcie_type(dev) == PCI_EXP_TYPE_PCI_BRIDGE)
1125 pnv_ioda_setup_bus_PE(dev->subordinate, 1);
1126 else
1127 pnv_ioda_setup_PEs(dev->subordinate);
1128 }
1129 }
1130}
1131
1132/*
1133 * Configure PEs so that the downstream PCI buses and devices
1134 * could have their associated PE#. Unfortunately, we didn't
1135 * figure out the way to identify the PLX bridge yet. So we
1136 * simply put the PCI bus and the subordinate behind the root
1137 * port to PE# here. The game rule here is expected to be changed
1138 * as soon as we can detected PLX bridge correctly.
1139 */
Greg Kroah-Hartmancad5cef2012-12-21 14:04:10 -08001140static void pnv_pci_ioda_setup_PEs(void)
Gavin Shanfb446ad2012-08-20 03:49:14 +00001141{
1142 struct pci_controller *hose, *tmp;
Guo Chao262af552014-07-21 14:42:30 +10001143 struct pnv_phb *phb;
Gavin Shanfb446ad2012-08-20 03:49:14 +00001144
1145 list_for_each_entry_safe(hose, tmp, &hose_list, list_node) {
Guo Chao262af552014-07-21 14:42:30 +10001146 phb = hose->private_data;
1147
1148 /* M64 layout might affect PE allocation */
Gavin Shan5ef73562014-11-12 13:36:06 +11001149 if (phb->reserve_m64_pe)
1150 phb->reserve_m64_pe(phb);
Guo Chao262af552014-07-21 14:42:30 +10001151
Gavin Shanfb446ad2012-08-20 03:49:14 +00001152 pnv_ioda_setup_PEs(hose->bus);
Benjamin Herrenschmidt184cd4a2011-11-15 17:29:08 +00001153 }
1154}
1155
Gavin Shana8b2f822015-03-25 16:23:52 +08001156#ifdef CONFIG_PCI_IOV
Wei Yang781a8682015-03-25 16:23:57 +08001157static int pnv_pci_vf_release_m64(struct pci_dev *pdev)
1158{
1159 struct pci_bus *bus;
1160 struct pci_controller *hose;
1161 struct pnv_phb *phb;
1162 struct pci_dn *pdn;
Wei Yang02639b02015-03-25 16:23:59 +08001163 int i, j;
Wei Yang781a8682015-03-25 16:23:57 +08001164
1165 bus = pdev->bus;
1166 hose = pci_bus_to_host(bus);
1167 phb = hose->private_data;
1168 pdn = pci_get_pdn(pdev);
1169
Wei Yang02639b02015-03-25 16:23:59 +08001170 for (i = 0; i < PCI_SRIOV_NUM_BARS; i++)
1171 for (j = 0; j < M64_PER_IOV; j++) {
1172 if (pdn->m64_wins[i][j] == IODA_INVALID_M64)
1173 continue;
1174 opal_pci_phb_mmio_enable(phb->opal_id,
1175 OPAL_M64_WINDOW_TYPE, pdn->m64_wins[i][j], 0);
1176 clear_bit(pdn->m64_wins[i][j], &phb->ioda.m64_bar_alloc);
1177 pdn->m64_wins[i][j] = IODA_INVALID_M64;
1178 }
Wei Yang781a8682015-03-25 16:23:57 +08001179
1180 return 0;
1181}
1182
Wei Yang02639b02015-03-25 16:23:59 +08001183static int pnv_pci_vf_assign_m64(struct pci_dev *pdev, u16 num_vfs)
Wei Yang781a8682015-03-25 16:23:57 +08001184{
1185 struct pci_bus *bus;
1186 struct pci_controller *hose;
1187 struct pnv_phb *phb;
1188 struct pci_dn *pdn;
1189 unsigned int win;
1190 struct resource *res;
Wei Yang02639b02015-03-25 16:23:59 +08001191 int i, j;
Wei Yang781a8682015-03-25 16:23:57 +08001192 int64_t rc;
Wei Yang02639b02015-03-25 16:23:59 +08001193 int total_vfs;
1194 resource_size_t size, start;
1195 int pe_num;
1196 int vf_groups;
1197 int vf_per_group;
Wei Yang781a8682015-03-25 16:23:57 +08001198
1199 bus = pdev->bus;
1200 hose = pci_bus_to_host(bus);
1201 phb = hose->private_data;
1202 pdn = pci_get_pdn(pdev);
Wei Yang02639b02015-03-25 16:23:59 +08001203 total_vfs = pci_sriov_get_totalvfs(pdev);
Wei Yang781a8682015-03-25 16:23:57 +08001204
1205 /* Initialize the m64_wins to IODA_INVALID_M64 */
1206 for (i = 0; i < PCI_SRIOV_NUM_BARS; i++)
Wei Yang02639b02015-03-25 16:23:59 +08001207 for (j = 0; j < M64_PER_IOV; j++)
1208 pdn->m64_wins[i][j] = IODA_INVALID_M64;
1209
1210 if (pdn->m64_per_iov == M64_PER_IOV) {
1211 vf_groups = (num_vfs <= M64_PER_IOV) ? num_vfs: M64_PER_IOV;
1212 vf_per_group = (num_vfs <= M64_PER_IOV)? 1:
1213 roundup_pow_of_two(num_vfs) / pdn->m64_per_iov;
1214 } else {
1215 vf_groups = 1;
1216 vf_per_group = 1;
1217 }
Wei Yang781a8682015-03-25 16:23:57 +08001218
1219 for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) {
1220 res = &pdev->resource[i + PCI_IOV_RESOURCES];
1221 if (!res->flags || !res->parent)
1222 continue;
1223
1224 if (!pnv_pci_is_mem_pref_64(res->flags))
1225 continue;
1226
Wei Yang02639b02015-03-25 16:23:59 +08001227 for (j = 0; j < vf_groups; j++) {
1228 do {
1229 win = find_next_zero_bit(&phb->ioda.m64_bar_alloc,
1230 phb->ioda.m64_bar_idx + 1, 0);
Wei Yang781a8682015-03-25 16:23:57 +08001231
Wei Yang02639b02015-03-25 16:23:59 +08001232 if (win >= phb->ioda.m64_bar_idx + 1)
1233 goto m64_failed;
1234 } while (test_and_set_bit(win, &phb->ioda.m64_bar_alloc));
Wei Yang781a8682015-03-25 16:23:57 +08001235
Wei Yang02639b02015-03-25 16:23:59 +08001236 pdn->m64_wins[i][j] = win;
Wei Yang781a8682015-03-25 16:23:57 +08001237
Wei Yang02639b02015-03-25 16:23:59 +08001238 if (pdn->m64_per_iov == M64_PER_IOV) {
1239 size = pci_iov_resource_size(pdev,
1240 PCI_IOV_RESOURCES + i);
1241 size = size * vf_per_group;
1242 start = res->start + size * j;
1243 } else {
1244 size = resource_size(res);
1245 start = res->start;
1246 }
1247
1248 /* Map the M64 here */
1249 if (pdn->m64_per_iov == M64_PER_IOV) {
1250 pe_num = pdn->offset + j;
1251 rc = opal_pci_map_pe_mmio_window(phb->opal_id,
1252 pe_num, OPAL_M64_WINDOW_TYPE,
1253 pdn->m64_wins[i][j], 0);
1254 }
1255
1256 rc = opal_pci_set_phb_mem_window(phb->opal_id,
Wei Yang781a8682015-03-25 16:23:57 +08001257 OPAL_M64_WINDOW_TYPE,
Wei Yang02639b02015-03-25 16:23:59 +08001258 pdn->m64_wins[i][j],
1259 start,
Wei Yang781a8682015-03-25 16:23:57 +08001260 0, /* unused */
Wei Yang02639b02015-03-25 16:23:59 +08001261 size);
Wei Yang781a8682015-03-25 16:23:57 +08001262
Wei Yang02639b02015-03-25 16:23:59 +08001263
1264 if (rc != OPAL_SUCCESS) {
1265 dev_err(&pdev->dev, "Failed to map M64 window #%d: %lld\n",
1266 win, rc);
1267 goto m64_failed;
1268 }
1269
1270 if (pdn->m64_per_iov == M64_PER_IOV)
1271 rc = opal_pci_phb_mmio_enable(phb->opal_id,
1272 OPAL_M64_WINDOW_TYPE, pdn->m64_wins[i][j], 2);
1273 else
1274 rc = opal_pci_phb_mmio_enable(phb->opal_id,
1275 OPAL_M64_WINDOW_TYPE, pdn->m64_wins[i][j], 1);
1276
1277 if (rc != OPAL_SUCCESS) {
1278 dev_err(&pdev->dev, "Failed to enable M64 window #%d: %llx\n",
1279 win, rc);
1280 goto m64_failed;
1281 }
Wei Yang781a8682015-03-25 16:23:57 +08001282 }
1283 }
1284 return 0;
1285
1286m64_failed:
1287 pnv_pci_vf_release_m64(pdev);
1288 return -EBUSY;
1289}
1290
Alexey Kardashevskiyc035e372015-06-05 16:35:21 +10001291static long pnv_pci_ioda2_unset_window(struct iommu_table_group *table_group,
1292 int num);
1293static void pnv_pci_ioda2_set_bypass(struct pnv_ioda_pe *pe, bool enable);
1294
Wei Yang781a8682015-03-25 16:23:57 +08001295static void pnv_pci_ioda2_release_dma_pe(struct pci_dev *dev, struct pnv_ioda_pe *pe)
1296{
Wei Yang781a8682015-03-25 16:23:57 +08001297 struct iommu_table *tbl;
Wei Yang781a8682015-03-25 16:23:57 +08001298 int64_t rc;
1299
Alexey Kardashevskiyb348aa62015-06-05 16:35:08 +10001300 tbl = pe->table_group.tables[0];
Alexey Kardashevskiyc035e372015-06-05 16:35:21 +10001301 rc = pnv_pci_ioda2_unset_window(&pe->table_group, 0);
Wei Yang781a8682015-03-25 16:23:57 +08001302 if (rc)
1303 pe_warn(pe, "OPAL error %ld release DMA window\n", rc);
1304
Alexey Kardashevskiyc035e372015-06-05 16:35:21 +10001305 pnv_pci_ioda2_set_bypass(pe, false);
Alexey Kardashevskiy0eaf4de2015-06-05 16:35:09 +10001306 if (pe->table_group.group) {
1307 iommu_group_put(pe->table_group.group);
1308 BUG_ON(pe->table_group.group);
Alexey Kardashevskiyac9a5882015-06-05 16:34:56 +10001309 }
Alexey Kardashevskiyaca69132015-06-05 16:35:17 +10001310 pnv_pci_ioda2_table_free_pages(tbl);
Wei Yang781a8682015-03-25 16:23:57 +08001311 iommu_free_table(tbl, of_node_full_name(dev->dev.of_node));
Wei Yang781a8682015-03-25 16:23:57 +08001312}
1313
Wei Yang02639b02015-03-25 16:23:59 +08001314static void pnv_ioda_release_vf_PE(struct pci_dev *pdev, u16 num_vfs)
Wei Yang781a8682015-03-25 16:23:57 +08001315{
1316 struct pci_bus *bus;
1317 struct pci_controller *hose;
1318 struct pnv_phb *phb;
1319 struct pnv_ioda_pe *pe, *pe_n;
1320 struct pci_dn *pdn;
Wei Yang02639b02015-03-25 16:23:59 +08001321 u16 vf_index;
1322 int64_t rc;
Wei Yang781a8682015-03-25 16:23:57 +08001323
1324 bus = pdev->bus;
1325 hose = pci_bus_to_host(bus);
1326 phb = hose->private_data;
Wei Yang02639b02015-03-25 16:23:59 +08001327 pdn = pci_get_pdn(pdev);
Wei Yang781a8682015-03-25 16:23:57 +08001328
1329 if (!pdev->is_physfn)
1330 return;
1331
Wei Yang02639b02015-03-25 16:23:59 +08001332 if (pdn->m64_per_iov == M64_PER_IOV && num_vfs > M64_PER_IOV) {
1333 int vf_group;
1334 int vf_per_group;
1335 int vf_index1;
1336
1337 vf_per_group = roundup_pow_of_two(num_vfs) / pdn->m64_per_iov;
1338
1339 for (vf_group = 0; vf_group < M64_PER_IOV; vf_group++)
1340 for (vf_index = vf_group * vf_per_group;
1341 vf_index < (vf_group + 1) * vf_per_group &&
1342 vf_index < num_vfs;
1343 vf_index++)
1344 for (vf_index1 = vf_group * vf_per_group;
1345 vf_index1 < (vf_group + 1) * vf_per_group &&
1346 vf_index1 < num_vfs;
1347 vf_index1++){
1348
1349 rc = opal_pci_set_peltv(phb->opal_id,
1350 pdn->offset + vf_index,
1351 pdn->offset + vf_index1,
1352 OPAL_REMOVE_PE_FROM_DOMAIN);
1353
1354 if (rc)
1355 dev_warn(&pdev->dev, "%s: Failed to unlink same group PE#%d(%lld)\n",
1356 __func__,
1357 pdn->offset + vf_index1, rc);
1358 }
1359 }
1360
Wei Yang781a8682015-03-25 16:23:57 +08001361 list_for_each_entry_safe(pe, pe_n, &phb->ioda.pe_list, list) {
1362 if (pe->parent_dev != pdev)
1363 continue;
1364
1365 pnv_pci_ioda2_release_dma_pe(pdev, pe);
1366
1367 /* Remove from list */
1368 mutex_lock(&phb->ioda.pe_list_mutex);
1369 list_del(&pe->list);
1370 mutex_unlock(&phb->ioda.pe_list_mutex);
1371
1372 pnv_ioda_deconfigure_pe(phb, pe);
1373
1374 pnv_ioda_free_pe(phb, pe->pe_number);
1375 }
1376}
1377
1378void pnv_pci_sriov_disable(struct pci_dev *pdev)
1379{
1380 struct pci_bus *bus;
1381 struct pci_controller *hose;
1382 struct pnv_phb *phb;
1383 struct pci_dn *pdn;
1384 struct pci_sriov *iov;
1385 u16 num_vfs;
1386
1387 bus = pdev->bus;
1388 hose = pci_bus_to_host(bus);
1389 phb = hose->private_data;
1390 pdn = pci_get_pdn(pdev);
1391 iov = pdev->sriov;
1392 num_vfs = pdn->num_vfs;
1393
1394 /* Release VF PEs */
Wei Yang02639b02015-03-25 16:23:59 +08001395 pnv_ioda_release_vf_PE(pdev, num_vfs);
Wei Yang781a8682015-03-25 16:23:57 +08001396
1397 if (phb->type == PNV_PHB_IODA2) {
Wei Yang02639b02015-03-25 16:23:59 +08001398 if (pdn->m64_per_iov == 1)
1399 pnv_pci_vf_resource_shift(pdev, -pdn->offset);
Wei Yang781a8682015-03-25 16:23:57 +08001400
1401 /* Release M64 windows */
1402 pnv_pci_vf_release_m64(pdev);
1403
1404 /* Release PE numbers */
1405 bitmap_clear(phb->ioda.pe_alloc, pdn->offset, num_vfs);
1406 pdn->offset = 0;
1407 }
1408}
1409
1410static void pnv_pci_ioda2_setup_dma_pe(struct pnv_phb *phb,
1411 struct pnv_ioda_pe *pe);
1412static void pnv_ioda_setup_vf_PE(struct pci_dev *pdev, u16 num_vfs)
1413{
1414 struct pci_bus *bus;
1415 struct pci_controller *hose;
1416 struct pnv_phb *phb;
1417 struct pnv_ioda_pe *pe;
1418 int pe_num;
1419 u16 vf_index;
1420 struct pci_dn *pdn;
Wei Yang02639b02015-03-25 16:23:59 +08001421 int64_t rc;
Wei Yang781a8682015-03-25 16:23:57 +08001422
1423 bus = pdev->bus;
1424 hose = pci_bus_to_host(bus);
1425 phb = hose->private_data;
1426 pdn = pci_get_pdn(pdev);
1427
1428 if (!pdev->is_physfn)
1429 return;
1430
1431 /* Reserve PE for each VF */
1432 for (vf_index = 0; vf_index < num_vfs; vf_index++) {
1433 pe_num = pdn->offset + vf_index;
1434
1435 pe = &phb->ioda.pe_array[pe_num];
1436 pe->pe_number = pe_num;
1437 pe->phb = phb;
1438 pe->flags = PNV_IODA_PE_VF;
1439 pe->pbus = NULL;
1440 pe->parent_dev = pdev;
1441 pe->tce32_seg = -1;
1442 pe->mve_number = -1;
1443 pe->rid = (pci_iov_virtfn_bus(pdev, vf_index) << 8) |
1444 pci_iov_virtfn_devfn(pdev, vf_index);
1445
1446 pe_info(pe, "VF %04d:%02d:%02d.%d associated with PE#%d\n",
1447 hose->global_number, pdev->bus->number,
1448 PCI_SLOT(pci_iov_virtfn_devfn(pdev, vf_index)),
1449 PCI_FUNC(pci_iov_virtfn_devfn(pdev, vf_index)), pe_num);
1450
1451 if (pnv_ioda_configure_pe(phb, pe)) {
1452 /* XXX What do we do here ? */
1453 if (pe_num)
1454 pnv_ioda_free_pe(phb, pe_num);
1455 pe->pdev = NULL;
1456 continue;
1457 }
1458
Wei Yang781a8682015-03-25 16:23:57 +08001459 /* Put PE to the list */
1460 mutex_lock(&phb->ioda.pe_list_mutex);
1461 list_add_tail(&pe->list, &phb->ioda.pe_list);
1462 mutex_unlock(&phb->ioda.pe_list_mutex);
1463
1464 pnv_pci_ioda2_setup_dma_pe(phb, pe);
1465 }
Wei Yang02639b02015-03-25 16:23:59 +08001466
1467 if (pdn->m64_per_iov == M64_PER_IOV && num_vfs > M64_PER_IOV) {
1468 int vf_group;
1469 int vf_per_group;
1470 int vf_index1;
1471
1472 vf_per_group = roundup_pow_of_two(num_vfs) / pdn->m64_per_iov;
1473
1474 for (vf_group = 0; vf_group < M64_PER_IOV; vf_group++) {
1475 for (vf_index = vf_group * vf_per_group;
1476 vf_index < (vf_group + 1) * vf_per_group &&
1477 vf_index < num_vfs;
1478 vf_index++) {
1479 for (vf_index1 = vf_group * vf_per_group;
1480 vf_index1 < (vf_group + 1) * vf_per_group &&
1481 vf_index1 < num_vfs;
1482 vf_index1++) {
1483
1484 rc = opal_pci_set_peltv(phb->opal_id,
1485 pdn->offset + vf_index,
1486 pdn->offset + vf_index1,
1487 OPAL_ADD_PE_TO_DOMAIN);
1488
1489 if (rc)
1490 dev_warn(&pdev->dev, "%s: Failed to link same group PE#%d(%lld)\n",
1491 __func__,
1492 pdn->offset + vf_index1, rc);
1493 }
1494 }
1495 }
1496 }
Wei Yang781a8682015-03-25 16:23:57 +08001497}
1498
1499int pnv_pci_sriov_enable(struct pci_dev *pdev, u16 num_vfs)
1500{
1501 struct pci_bus *bus;
1502 struct pci_controller *hose;
1503 struct pnv_phb *phb;
1504 struct pci_dn *pdn;
1505 int ret;
1506
1507 bus = pdev->bus;
1508 hose = pci_bus_to_host(bus);
1509 phb = hose->private_data;
1510 pdn = pci_get_pdn(pdev);
1511
1512 if (phb->type == PNV_PHB_IODA2) {
1513 /* Calculate available PE for required VFs */
1514 mutex_lock(&phb->ioda.pe_alloc_mutex);
1515 pdn->offset = bitmap_find_next_zero_area(
1516 phb->ioda.pe_alloc, phb->ioda.total_pe,
1517 0, num_vfs, 0);
1518 if (pdn->offset >= phb->ioda.total_pe) {
1519 mutex_unlock(&phb->ioda.pe_alloc_mutex);
1520 dev_info(&pdev->dev, "Failed to enable VF%d\n", num_vfs);
1521 pdn->offset = 0;
1522 return -EBUSY;
1523 }
1524 bitmap_set(phb->ioda.pe_alloc, pdn->offset, num_vfs);
1525 pdn->num_vfs = num_vfs;
1526 mutex_unlock(&phb->ioda.pe_alloc_mutex);
1527
1528 /* Assign M64 window accordingly */
Wei Yang02639b02015-03-25 16:23:59 +08001529 ret = pnv_pci_vf_assign_m64(pdev, num_vfs);
Wei Yang781a8682015-03-25 16:23:57 +08001530 if (ret) {
1531 dev_info(&pdev->dev, "Not enough M64 window resources\n");
1532 goto m64_failed;
1533 }
1534
1535 /*
1536 * When using one M64 BAR to map one IOV BAR, we need to shift
1537 * the IOV BAR according to the PE# allocated to the VFs.
1538 * Otherwise, the PE# for the VF will conflict with others.
1539 */
Wei Yang02639b02015-03-25 16:23:59 +08001540 if (pdn->m64_per_iov == 1) {
1541 ret = pnv_pci_vf_resource_shift(pdev, pdn->offset);
1542 if (ret)
1543 goto m64_failed;
1544 }
Wei Yang781a8682015-03-25 16:23:57 +08001545 }
1546
1547 /* Setup VF PEs */
1548 pnv_ioda_setup_vf_PE(pdev, num_vfs);
1549
1550 return 0;
1551
1552m64_failed:
1553 bitmap_clear(phb->ioda.pe_alloc, pdn->offset, num_vfs);
1554 pdn->offset = 0;
1555
1556 return ret;
1557}
1558
Gavin Shana8b2f822015-03-25 16:23:52 +08001559int pcibios_sriov_disable(struct pci_dev *pdev)
1560{
Wei Yang781a8682015-03-25 16:23:57 +08001561 pnv_pci_sriov_disable(pdev);
1562
Gavin Shana8b2f822015-03-25 16:23:52 +08001563 /* Release PCI data */
1564 remove_dev_pci_data(pdev);
1565 return 0;
1566}
1567
1568int pcibios_sriov_enable(struct pci_dev *pdev, u16 num_vfs)
1569{
1570 /* Allocate PCI data */
1571 add_dev_pci_data(pdev);
Wei Yang781a8682015-03-25 16:23:57 +08001572
1573 pnv_pci_sriov_enable(pdev, num_vfs);
Gavin Shana8b2f822015-03-25 16:23:52 +08001574 return 0;
1575}
1576#endif /* CONFIG_PCI_IOV */
1577
Gavin Shan959c9bd2013-04-25 19:21:02 +00001578static void pnv_pci_ioda_dma_dev_setup(struct pnv_phb *phb, struct pci_dev *pdev)
Benjamin Herrenschmidt184cd4a2011-11-15 17:29:08 +00001579{
Benjamin Herrenschmidtb72c1f62013-05-21 22:58:21 +00001580 struct pci_dn *pdn = pci_get_pdn(pdev);
Gavin Shan959c9bd2013-04-25 19:21:02 +00001581 struct pnv_ioda_pe *pe;
Benjamin Herrenschmidt184cd4a2011-11-15 17:29:08 +00001582
Gavin Shan959c9bd2013-04-25 19:21:02 +00001583 /*
1584 * The function can be called while the PE#
1585 * hasn't been assigned. Do nothing for the
1586 * case.
1587 */
1588 if (!pdn || pdn->pe_number == IODA_INVALID_PE)
1589 return;
Benjamin Herrenschmidt184cd4a2011-11-15 17:29:08 +00001590
Gavin Shan959c9bd2013-04-25 19:21:02 +00001591 pe = &phb->ioda.pe_array[pdn->pe_number];
Benjamin Herrenschmidtcd15b042014-02-11 11:32:38 +11001592 WARN_ON(get_dma_ops(&pdev->dev) != &dma_iommu_ops);
Alexey Kardashevskiyb348aa62015-06-05 16:35:08 +10001593 set_iommu_table_base(&pdev->dev, pe->table_group.tables[0]);
Alexey Kardashevskiy46170822015-06-05 16:34:54 +10001594 /*
1595 * Note: iommu_add_device() will fail here as
1596 * for physical PE: the device is already added by now;
1597 * for virtual PE: sysfs entries are not ready yet and
1598 * tce_iommu_bus_notifier will add the device to a group later.
1599 */
Benjamin Herrenschmidt184cd4a2011-11-15 17:29:08 +00001600}
1601
Daniel Axtens763d2d82015-04-28 15:12:07 +10001602static int pnv_pci_ioda_dma_set_mask(struct pci_dev *pdev, u64 dma_mask)
Benjamin Herrenschmidtcd15b042014-02-11 11:32:38 +11001603{
Daniel Axtens763d2d82015-04-28 15:12:07 +10001604 struct pci_controller *hose = pci_bus_to_host(pdev->bus);
1605 struct pnv_phb *phb = hose->private_data;
Benjamin Herrenschmidtcd15b042014-02-11 11:32:38 +11001606 struct pci_dn *pdn = pci_get_pdn(pdev);
1607 struct pnv_ioda_pe *pe;
1608 uint64_t top;
1609 bool bypass = false;
1610
1611 if (WARN_ON(!pdn || pdn->pe_number == IODA_INVALID_PE))
1612 return -ENODEV;;
1613
1614 pe = &phb->ioda.pe_array[pdn->pe_number];
1615 if (pe->tce_bypass_enabled) {
1616 top = pe->tce_bypass_base + memblock_end_of_DRAM() - 1;
1617 bypass = (dma_mask >= top);
1618 }
1619
1620 if (bypass) {
1621 dev_info(&pdev->dev, "Using 64-bit DMA iommu bypass\n");
1622 set_dma_ops(&pdev->dev, &dma_direct_ops);
1623 set_dma_offset(&pdev->dev, pe->tce_bypass_base);
1624 } else {
1625 dev_info(&pdev->dev, "Using 32-bit DMA via iommu\n");
1626 set_dma_ops(&pdev->dev, &dma_iommu_ops);
Alexey Kardashevskiyb348aa62015-06-05 16:35:08 +10001627 set_iommu_table_base(&pdev->dev, pe->table_group.tables[0]);
Benjamin Herrenschmidtcd15b042014-02-11 11:32:38 +11001628 }
Brian W Harta32305b2014-07-31 14:24:37 -05001629 *pdev->dev.dma_mask = dma_mask;
Benjamin Herrenschmidtcd15b042014-02-11 11:32:38 +11001630 return 0;
1631}
1632
Gavin Shanfe7e85c2014-09-30 12:39:10 +10001633static u64 pnv_pci_ioda_dma_get_required_mask(struct pnv_phb *phb,
1634 struct pci_dev *pdev)
1635{
1636 struct pci_dn *pdn = pci_get_pdn(pdev);
1637 struct pnv_ioda_pe *pe;
1638 u64 end, mask;
1639
1640 if (WARN_ON(!pdn || pdn->pe_number == IODA_INVALID_PE))
1641 return 0;
1642
1643 pe = &phb->ioda.pe_array[pdn->pe_number];
1644 if (!pe->tce_bypass_enabled)
1645 return __dma_get_required_mask(&pdev->dev);
1646
1647
1648 end = pe->tce_bypass_base + memblock_end_of_DRAM();
1649 mask = 1ULL << (fls64(end) - 1);
1650 mask += mask - 1;
1651
1652 return mask;
1653}
1654
Gavin Shandff4a392014-07-15 17:00:55 +10001655static void pnv_ioda_setup_bus_dma(struct pnv_ioda_pe *pe,
Alexey Kardashevskiyea30e992015-06-05 16:34:53 +10001656 struct pci_bus *bus)
Benjamin Herrenschmidt74251fe2013-07-01 17:54:09 +10001657{
1658 struct pci_dev *dev;
1659
1660 list_for_each_entry(dev, &bus->devices, bus_list) {
Alexey Kardashevskiyb348aa62015-06-05 16:35:08 +10001661 set_iommu_table_base(&dev->dev, pe->table_group.tables[0]);
Alexey Kardashevskiy46170822015-06-05 16:34:54 +10001662 iommu_add_device(&dev->dev);
Gavin Shandff4a392014-07-15 17:00:55 +10001663
Alexey Kardashevskiy5c89a872015-06-18 11:41:36 +10001664 if ((pe->flags & PNV_IODA_PE_BUS_ALL) && dev->subordinate)
Alexey Kardashevskiyea30e992015-06-05 16:34:53 +10001665 pnv_ioda_setup_bus_dma(pe, dev->subordinate);
Benjamin Herrenschmidt74251fe2013-07-01 17:54:09 +10001666 }
1667}
1668
Alexey Kardashevskiydecbda22015-06-05 16:35:07 +10001669static void pnv_pci_ioda1_tce_invalidate(struct iommu_table *tbl,
1670 unsigned long index, unsigned long npages, bool rm)
Gavin Shan4cce9552013-04-25 19:21:00 +00001671{
Alexey Kardashevskiy0eaf4de2015-06-05 16:35:09 +10001672 struct iommu_table_group_link *tgl = list_first_entry_or_null(
1673 &tbl->it_group_list, struct iommu_table_group_link,
1674 next);
1675 struct pnv_ioda_pe *pe = container_of(tgl->table_group,
Alexey Kardashevskiyb348aa62015-06-05 16:35:08 +10001676 struct pnv_ioda_pe, table_group);
Benjamin Herrenschmidt3ad26e52013-10-11 18:23:53 +11001677 __be64 __iomem *invalidate = rm ?
Alexey Kardashevskiy5780fb02015-06-05 16:35:12 +10001678 (__be64 __iomem *)pe->phb->ioda.tce_inval_reg_phys :
1679 pe->phb->ioda.tce_inval_reg;
Gavin Shan4cce9552013-04-25 19:21:00 +00001680 unsigned long start, end, inc;
Alexey Kardashevskiyb0376c92014-06-06 18:44:01 +10001681 const unsigned shift = tbl->it_page_shift;
Gavin Shan4cce9552013-04-25 19:21:00 +00001682
Alexey Kardashevskiydecbda22015-06-05 16:35:07 +10001683 start = __pa(((__be64 *)tbl->it_base) + index - tbl->it_offset);
1684 end = __pa(((__be64 *)tbl->it_base) + index - tbl->it_offset +
1685 npages - 1);
Gavin Shan4cce9552013-04-25 19:21:00 +00001686
1687 /* BML uses this case for p6/p7/galaxy2: Shift addr and put in node */
1688 if (tbl->it_busno) {
Alexey Kardashevskiyb0376c92014-06-06 18:44:01 +10001689 start <<= shift;
1690 end <<= shift;
1691 inc = 128ull << shift;
Gavin Shan4cce9552013-04-25 19:21:00 +00001692 start |= tbl->it_busno;
1693 end |= tbl->it_busno;
1694 } else if (tbl->it_type & TCE_PCI_SWINV_PAIR) {
1695 /* p7ioc-style invalidation, 2 TCEs per write */
1696 start |= (1ull << 63);
1697 end |= (1ull << 63);
1698 inc = 16;
1699 } else {
1700 /* Default (older HW) */
1701 inc = 128;
1702 }
1703
1704 end |= inc - 1; /* round up end to be different than start */
1705
1706 mb(); /* Ensure above stores are visible */
1707 while (start <= end) {
Alexey Kardashevskiy8e0a1612013-08-28 18:37:43 +10001708 if (rm)
Benjamin Herrenschmidt3ad26e52013-10-11 18:23:53 +11001709 __raw_rm_writeq(cpu_to_be64(start), invalidate);
Alexey Kardashevskiy8e0a1612013-08-28 18:37:43 +10001710 else
Benjamin Herrenschmidt3ad26e52013-10-11 18:23:53 +11001711 __raw_writeq(cpu_to_be64(start), invalidate);
Gavin Shan4cce9552013-04-25 19:21:00 +00001712 start += inc;
1713 }
1714
1715 /*
1716 * The iommu layer will do another mb() for us on build()
1717 * and we don't care on free()
1718 */
1719}
1720
Alexey Kardashevskiydecbda22015-06-05 16:35:07 +10001721static int pnv_ioda1_tce_build(struct iommu_table *tbl, long index,
1722 long npages, unsigned long uaddr,
1723 enum dma_data_direction direction,
1724 struct dma_attrs *attrs)
1725{
1726 int ret = pnv_tce_build(tbl, index, npages, uaddr, direction,
1727 attrs);
1728
1729 if (!ret && (tbl->it_type & TCE_PCI_SWINV_CREATE))
1730 pnv_pci_ioda1_tce_invalidate(tbl, index, npages, false);
1731
1732 return ret;
1733}
1734
Alexey Kardashevskiy05c6cfb2015-06-05 16:35:15 +10001735#ifdef CONFIG_IOMMU_API
1736static int pnv_ioda1_tce_xchg(struct iommu_table *tbl, long index,
1737 unsigned long *hpa, enum dma_data_direction *direction)
1738{
1739 long ret = pnv_tce_xchg(tbl, index, hpa, direction);
1740
1741 if (!ret && (tbl->it_type &
1742 (TCE_PCI_SWINV_CREATE | TCE_PCI_SWINV_FREE)))
1743 pnv_pci_ioda1_tce_invalidate(tbl, index, 1, false);
1744
1745 return ret;
1746}
1747#endif
1748
Alexey Kardashevskiydecbda22015-06-05 16:35:07 +10001749static void pnv_ioda1_tce_free(struct iommu_table *tbl, long index,
1750 long npages)
1751{
1752 pnv_tce_free(tbl, index, npages);
1753
1754 if (tbl->it_type & TCE_PCI_SWINV_FREE)
1755 pnv_pci_ioda1_tce_invalidate(tbl, index, npages, false);
1756}
1757
Alexey Kardashevskiyda004c32015-06-05 16:35:06 +10001758static struct iommu_table_ops pnv_ioda1_iommu_ops = {
Alexey Kardashevskiydecbda22015-06-05 16:35:07 +10001759 .set = pnv_ioda1_tce_build,
Alexey Kardashevskiy05c6cfb2015-06-05 16:35:15 +10001760#ifdef CONFIG_IOMMU_API
1761 .exchange = pnv_ioda1_tce_xchg,
1762#endif
Alexey Kardashevskiydecbda22015-06-05 16:35:07 +10001763 .clear = pnv_ioda1_tce_free,
Alexey Kardashevskiyda004c32015-06-05 16:35:06 +10001764 .get = pnv_tce_get,
1765};
1766
Alexey Kardashevskiy5780fb02015-06-05 16:35:12 +10001767static inline void pnv_pci_ioda2_tce_invalidate_entire(struct pnv_ioda_pe *pe)
1768{
1769 /* 01xb - invalidate TCEs that match the specified PE# */
1770 unsigned long val = (0x4ull << 60) | (pe->pe_number & 0xFF);
1771 struct pnv_phb *phb = pe->phb;
1772
1773 if (!phb->ioda.tce_inval_reg)
1774 return;
1775
1776 mb(); /* Ensure above stores are visible */
1777 __raw_writeq(cpu_to_be64(val), phb->ioda.tce_inval_reg);
1778}
1779
Alexey Kardashevskiye57080f2015-06-05 16:35:13 +10001780static void pnv_pci_ioda2_do_tce_invalidate(unsigned pe_number, bool rm,
1781 __be64 __iomem *invalidate, unsigned shift,
1782 unsigned long index, unsigned long npages)
Gavin Shan4cce9552013-04-25 19:21:00 +00001783{
1784 unsigned long start, end, inc;
Gavin Shan4cce9552013-04-25 19:21:00 +00001785
1786 /* We'll invalidate DMA address in PE scope */
Alexey Kardashevskiyb0376c92014-06-06 18:44:01 +10001787 start = 0x2ull << 60;
Alexey Kardashevskiye57080f2015-06-05 16:35:13 +10001788 start |= (pe_number & 0xFF);
Gavin Shan4cce9552013-04-25 19:21:00 +00001789 end = start;
1790
1791 /* Figure out the start, end and step */
Alexey Kardashevskiydecbda22015-06-05 16:35:07 +10001792 start |= (index << shift);
1793 end |= ((index + npages - 1) << shift);
Alexey Kardashevskiyb0376c92014-06-06 18:44:01 +10001794 inc = (0x1ull << shift);
Gavin Shan4cce9552013-04-25 19:21:00 +00001795 mb();
1796
1797 while (start <= end) {
Alexey Kardashevskiy8e0a1612013-08-28 18:37:43 +10001798 if (rm)
Benjamin Herrenschmidt3ad26e52013-10-11 18:23:53 +11001799 __raw_rm_writeq(cpu_to_be64(start), invalidate);
Alexey Kardashevskiy8e0a1612013-08-28 18:37:43 +10001800 else
Benjamin Herrenschmidt3ad26e52013-10-11 18:23:53 +11001801 __raw_writeq(cpu_to_be64(start), invalidate);
Gavin Shan4cce9552013-04-25 19:21:00 +00001802 start += inc;
1803 }
1804}
1805
Alexey Kardashevskiye57080f2015-06-05 16:35:13 +10001806static void pnv_pci_ioda2_tce_invalidate(struct iommu_table *tbl,
1807 unsigned long index, unsigned long npages, bool rm)
1808{
1809 struct iommu_table_group_link *tgl;
1810
1811 list_for_each_entry_rcu(tgl, &tbl->it_group_list, next) {
1812 struct pnv_ioda_pe *pe = container_of(tgl->table_group,
1813 struct pnv_ioda_pe, table_group);
1814 __be64 __iomem *invalidate = rm ?
1815 (__be64 __iomem *)pe->phb->ioda.tce_inval_reg_phys :
1816 pe->phb->ioda.tce_inval_reg;
1817
1818 pnv_pci_ioda2_do_tce_invalidate(pe->pe_number, rm,
1819 invalidate, tbl->it_page_shift,
1820 index, npages);
1821 }
1822}
1823
Alexey Kardashevskiydecbda22015-06-05 16:35:07 +10001824static int pnv_ioda2_tce_build(struct iommu_table *tbl, long index,
1825 long npages, unsigned long uaddr,
1826 enum dma_data_direction direction,
1827 struct dma_attrs *attrs)
Gavin Shan4cce9552013-04-25 19:21:00 +00001828{
Alexey Kardashevskiydecbda22015-06-05 16:35:07 +10001829 int ret = pnv_tce_build(tbl, index, npages, uaddr, direction,
1830 attrs);
Gavin Shan4cce9552013-04-25 19:21:00 +00001831
Alexey Kardashevskiydecbda22015-06-05 16:35:07 +10001832 if (!ret && (tbl->it_type & TCE_PCI_SWINV_CREATE))
1833 pnv_pci_ioda2_tce_invalidate(tbl, index, npages, false);
1834
1835 return ret;
1836}
1837
Alexey Kardashevskiy05c6cfb2015-06-05 16:35:15 +10001838#ifdef CONFIG_IOMMU_API
1839static int pnv_ioda2_tce_xchg(struct iommu_table *tbl, long index,
1840 unsigned long *hpa, enum dma_data_direction *direction)
1841{
1842 long ret = pnv_tce_xchg(tbl, index, hpa, direction);
1843
1844 if (!ret && (tbl->it_type &
1845 (TCE_PCI_SWINV_CREATE | TCE_PCI_SWINV_FREE)))
1846 pnv_pci_ioda2_tce_invalidate(tbl, index, 1, false);
1847
1848 return ret;
1849}
1850#endif
1851
Alexey Kardashevskiydecbda22015-06-05 16:35:07 +10001852static void pnv_ioda2_tce_free(struct iommu_table *tbl, long index,
1853 long npages)
1854{
1855 pnv_tce_free(tbl, index, npages);
1856
1857 if (tbl->it_type & TCE_PCI_SWINV_FREE)
1858 pnv_pci_ioda2_tce_invalidate(tbl, index, npages, false);
Gavin Shan4cce9552013-04-25 19:21:00 +00001859}
1860
Alexey Kardashevskiy4793d652015-06-05 16:35:20 +10001861static void pnv_ioda2_table_free(struct iommu_table *tbl)
1862{
1863 pnv_pci_ioda2_table_free_pages(tbl);
1864 iommu_free_table(tbl, "pnv");
1865}
1866
Alexey Kardashevskiyda004c32015-06-05 16:35:06 +10001867static struct iommu_table_ops pnv_ioda2_iommu_ops = {
Alexey Kardashevskiydecbda22015-06-05 16:35:07 +10001868 .set = pnv_ioda2_tce_build,
Alexey Kardashevskiy05c6cfb2015-06-05 16:35:15 +10001869#ifdef CONFIG_IOMMU_API
1870 .exchange = pnv_ioda2_tce_xchg,
1871#endif
Alexey Kardashevskiydecbda22015-06-05 16:35:07 +10001872 .clear = pnv_ioda2_tce_free,
Alexey Kardashevskiyda004c32015-06-05 16:35:06 +10001873 .get = pnv_tce_get,
Alexey Kardashevskiy4793d652015-06-05 16:35:20 +10001874 .free = pnv_ioda2_table_free,
Alexey Kardashevskiyda004c32015-06-05 16:35:06 +10001875};
1876
Greg Kroah-Hartmancad5cef2012-12-21 14:04:10 -08001877static void pnv_pci_ioda_setup_dma_pe(struct pnv_phb *phb,
1878 struct pnv_ioda_pe *pe, unsigned int base,
1879 unsigned int segs)
Benjamin Herrenschmidt184cd4a2011-11-15 17:29:08 +00001880{
1881
1882 struct page *tce_mem = NULL;
Benjamin Herrenschmidt184cd4a2011-11-15 17:29:08 +00001883 struct iommu_table *tbl;
1884 unsigned int i;
1885 int64_t rc;
1886 void *addr;
1887
Benjamin Herrenschmidt184cd4a2011-11-15 17:29:08 +00001888 /* XXX FIXME: Handle 64-bit only DMA devices */
1889 /* XXX FIXME: Provide 64-bit DMA facilities & non-4K TCE tables etc.. */
1890 /* XXX FIXME: Allocate multi-level tables on PHB3 */
1891
1892 /* We shouldn't already have a 32-bit DMA associated */
1893 if (WARN_ON(pe->tce32_seg >= 0))
1894 return;
1895
Alexey Kardashevskiy0eaf4de2015-06-05 16:35:09 +10001896 tbl = pnv_pci_table_alloc(phb->hose->node);
Alexey Kardashevskiyb348aa62015-06-05 16:35:08 +10001897 iommu_register_group(&pe->table_group, phb->hose->global_number,
1898 pe->pe_number);
Alexey Kardashevskiy0eaf4de2015-06-05 16:35:09 +10001899 pnv_pci_link_table_and_group(phb->hose->node, 0, tbl, &pe->table_group);
Alexey Kardashevskiyc5773822015-06-05 16:34:55 +10001900
Benjamin Herrenschmidt184cd4a2011-11-15 17:29:08 +00001901 /* Grab a 32-bit TCE table */
1902 pe->tce32_seg = base;
1903 pe_info(pe, " Setting up 32-bit TCE table at %08x..%08x\n",
1904 (base << 28), ((base + segs) << 28) - 1);
1905
1906 /* XXX Currently, we allocate one big contiguous table for the
1907 * TCEs. We only really need one chunk per 256M of TCE space
1908 * (ie per segment) but that's an optimization for later, it
1909 * requires some added smarts with our get/put_tce implementation
1910 */
1911 tce_mem = alloc_pages_node(phb->hose->node, GFP_KERNEL,
1912 get_order(TCE32_TABLE_SIZE * segs));
1913 if (!tce_mem) {
1914 pe_err(pe, " Failed to allocate a 32-bit TCE memory\n");
1915 goto fail;
1916 }
1917 addr = page_address(tce_mem);
1918 memset(addr, 0, TCE32_TABLE_SIZE * segs);
1919
1920 /* Configure HW */
1921 for (i = 0; i < segs; i++) {
1922 rc = opal_pci_map_pe_dma_window(phb->opal_id,
1923 pe->pe_number,
1924 base + i, 1,
1925 __pa(addr) + TCE32_TABLE_SIZE * i,
1926 TCE32_TABLE_SIZE, 0x1000);
1927 if (rc) {
1928 pe_err(pe, " Failed to configure 32-bit TCE table,"
1929 " err %ld\n", rc);
1930 goto fail;
1931 }
1932 }
1933
1934 /* Setup linux iommu table */
Benjamin Herrenschmidt184cd4a2011-11-15 17:29:08 +00001935 pnv_pci_setup_iommu_table(tbl, addr, TCE32_TABLE_SIZE * segs,
Alexey Kardashevskiy8fa5d452014-06-06 18:44:03 +10001936 base << 28, IOMMU_PAGE_SHIFT_4K);
Benjamin Herrenschmidt184cd4a2011-11-15 17:29:08 +00001937
1938 /* OPAL variant of P7IOC SW invalidated TCEs */
Alexey Kardashevskiy5780fb02015-06-05 16:35:12 +10001939 if (phb->ioda.tce_inval_reg)
Gavin Shan65fd7662014-04-24 18:00:28 +10001940 tbl->it_type |= (TCE_PCI_SWINV_CREATE |
1941 TCE_PCI_SWINV_FREE |
1942 TCE_PCI_SWINV_PAIR);
Alexey Kardashevskiy5780fb02015-06-05 16:35:12 +10001943
Alexey Kardashevskiyda004c32015-06-05 16:35:06 +10001944 tbl->it_ops = &pnv_ioda1_iommu_ops;
Alexey Kardashevskiy4793d652015-06-05 16:35:20 +10001945 pe->table_group.tce32_start = tbl->it_offset << tbl->it_page_shift;
1946 pe->table_group.tce32_size = tbl->it_size << tbl->it_page_shift;
Benjamin Herrenschmidt184cd4a2011-11-15 17:29:08 +00001947 iommu_init_table(tbl, phb->hose->node);
1948
Wei Yang781a8682015-03-25 16:23:57 +08001949 if (pe->flags & PNV_IODA_PE_DEV) {
Alexey Kardashevskiy46170822015-06-05 16:34:54 +10001950 /*
1951 * Setting table base here only for carrying iommu_group
1952 * further down to let iommu_add_device() do the job.
1953 * pnv_pci_ioda_dma_dev_setup will override it later anyway.
1954 */
1955 set_iommu_table_base(&pe->pdev->dev, tbl);
1956 iommu_add_device(&pe->pdev->dev);
Alexey Kardashevskiyc5773822015-06-05 16:34:55 +10001957 } else if (pe->flags & (PNV_IODA_PE_BUS | PNV_IODA_PE_BUS_ALL))
Alexey Kardashevskiyea30e992015-06-05 16:34:53 +10001958 pnv_ioda_setup_bus_dma(pe, pe->pbus);
Benjamin Herrenschmidt74251fe2013-07-01 17:54:09 +10001959
Benjamin Herrenschmidt184cd4a2011-11-15 17:29:08 +00001960 return;
1961 fail:
1962 /* XXX Failure: Try to fallback to 64-bit only ? */
1963 if (pe->tce32_seg >= 0)
1964 pe->tce32_seg = -1;
1965 if (tce_mem)
1966 __free_pages(tce_mem, get_order(TCE32_TABLE_SIZE * segs));
Alexey Kardashevskiy0eaf4de2015-06-05 16:35:09 +10001967 if (tbl) {
1968 pnv_pci_unlink_table_and_group(tbl, &pe->table_group);
1969 iommu_free_table(tbl, "pnv");
1970 }
Benjamin Herrenschmidt184cd4a2011-11-15 17:29:08 +00001971}
1972
Alexey Kardashevskiy43cb60a2015-06-05 16:35:18 +10001973static long pnv_pci_ioda2_set_window(struct iommu_table_group *table_group,
1974 int num, struct iommu_table *tbl)
1975{
1976 struct pnv_ioda_pe *pe = container_of(table_group, struct pnv_ioda_pe,
1977 table_group);
1978 struct pnv_phb *phb = pe->phb;
1979 int64_t rc;
Alexey Kardashevskiybbb845c2015-06-05 16:35:19 +10001980 const unsigned long size = tbl->it_indirect_levels ?
1981 tbl->it_level_size : tbl->it_size;
Alexey Kardashevskiy43cb60a2015-06-05 16:35:18 +10001982 const __u64 start_addr = tbl->it_offset << tbl->it_page_shift;
1983 const __u64 win_size = tbl->it_size << tbl->it_page_shift;
1984
Alexey Kardashevskiy4793d652015-06-05 16:35:20 +10001985 pe_info(pe, "Setting up window#%d %llx..%llx pg=%x\n", num,
Alexey Kardashevskiy43cb60a2015-06-05 16:35:18 +10001986 start_addr, start_addr + win_size - 1,
1987 IOMMU_PAGE_SIZE(tbl));
1988
1989 /*
1990 * Map TCE table through TVT. The TVE index is the PE number
1991 * shifted by 1 bit for 32-bits DMA space.
1992 */
1993 rc = opal_pci_map_pe_dma_window(phb->opal_id,
1994 pe->pe_number,
Alexey Kardashevskiy4793d652015-06-05 16:35:20 +10001995 (pe->pe_number << 1) + num,
Alexey Kardashevskiybbb845c2015-06-05 16:35:19 +10001996 tbl->it_indirect_levels + 1,
Alexey Kardashevskiy43cb60a2015-06-05 16:35:18 +10001997 __pa(tbl->it_base),
Alexey Kardashevskiybbb845c2015-06-05 16:35:19 +10001998 size << 3,
Alexey Kardashevskiy43cb60a2015-06-05 16:35:18 +10001999 IOMMU_PAGE_SIZE(tbl));
2000 if (rc) {
2001 pe_err(pe, "Failed to configure TCE table, err %ld\n", rc);
2002 return rc;
2003 }
2004
2005 pnv_pci_link_table_and_group(phb->hose->node, num,
2006 tbl, &pe->table_group);
2007 pnv_pci_ioda2_tce_invalidate_entire(pe);
2008
2009 return 0;
2010}
2011
Alexey Kardashevskiyf87a8862015-06-05 16:35:10 +10002012static void pnv_pci_ioda2_set_bypass(struct pnv_ioda_pe *pe, bool enable)
Benjamin Herrenschmidtcd15b042014-02-11 11:32:38 +11002013{
Benjamin Herrenschmidtcd15b042014-02-11 11:32:38 +11002014 uint16_t window_id = (pe->pe_number << 1 ) + 1;
2015 int64_t rc;
2016
2017 pe_info(pe, "%sabling 64-bit DMA bypass\n", enable ? "En" : "Dis");
2018 if (enable) {
2019 phys_addr_t top = memblock_end_of_DRAM();
2020
2021 top = roundup_pow_of_two(top);
2022 rc = opal_pci_map_pe_dma_window_real(pe->phb->opal_id,
2023 pe->pe_number,
2024 window_id,
2025 pe->tce_bypass_base,
2026 top);
2027 } else {
2028 rc = opal_pci_map_pe_dma_window_real(pe->phb->opal_id,
2029 pe->pe_number,
2030 window_id,
2031 pe->tce_bypass_base,
2032 0);
Benjamin Herrenschmidtcd15b042014-02-11 11:32:38 +11002033 }
2034 if (rc)
2035 pe_err(pe, "OPAL error %lld configuring bypass window\n", rc);
2036 else
2037 pe->tce_bypass_enabled = enable;
2038}
2039
Alexey Kardashevskiy4793d652015-06-05 16:35:20 +10002040static long pnv_pci_ioda2_table_alloc_pages(int nid, __u64 bus_offset,
2041 __u32 page_shift, __u64 window_size, __u32 levels,
2042 struct iommu_table *tbl);
2043
2044static long pnv_pci_ioda2_create_table(struct iommu_table_group *table_group,
2045 int num, __u32 page_shift, __u64 window_size, __u32 levels,
2046 struct iommu_table **ptbl)
2047{
2048 struct pnv_ioda_pe *pe = container_of(table_group, struct pnv_ioda_pe,
2049 table_group);
2050 int nid = pe->phb->hose->node;
2051 __u64 bus_offset = num ? pe->tce_bypass_base : table_group->tce32_start;
2052 long ret;
2053 struct iommu_table *tbl;
2054
2055 tbl = pnv_pci_table_alloc(nid);
2056 if (!tbl)
2057 return -ENOMEM;
2058
2059 ret = pnv_pci_ioda2_table_alloc_pages(nid,
2060 bus_offset, page_shift, window_size,
2061 levels, tbl);
2062 if (ret) {
2063 iommu_free_table(tbl, "pnv");
2064 return ret;
2065 }
2066
2067 tbl->it_ops = &pnv_ioda2_iommu_ops;
2068 if (pe->phb->ioda.tce_inval_reg)
2069 tbl->it_type |= (TCE_PCI_SWINV_CREATE | TCE_PCI_SWINV_FREE);
2070
2071 *ptbl = tbl;
2072
2073 return 0;
2074}
2075
Alexey Kardashevskiy46d3e1e2015-06-05 16:35:23 +10002076static long pnv_pci_ioda2_setup_default_config(struct pnv_ioda_pe *pe)
2077{
2078 struct iommu_table *tbl = NULL;
2079 long rc;
2080
2081 rc = pnv_pci_ioda2_create_table(&pe->table_group, 0,
2082 IOMMU_PAGE_SHIFT_4K,
2083 pe->table_group.tce32_size,
2084 POWERNV_IOMMU_DEFAULT_LEVELS, &tbl);
2085 if (rc) {
2086 pe_err(pe, "Failed to create 32-bit TCE table, err %ld",
2087 rc);
2088 return rc;
2089 }
2090
2091 iommu_init_table(tbl, pe->phb->hose->node);
2092
2093 rc = pnv_pci_ioda2_set_window(&pe->table_group, 0, tbl);
2094 if (rc) {
2095 pe_err(pe, "Failed to configure 32-bit TCE table, err %ld\n",
2096 rc);
2097 pnv_ioda2_table_free(tbl);
2098 return rc;
2099 }
2100
2101 if (!pnv_iommu_bypass_disabled)
2102 pnv_pci_ioda2_set_bypass(pe, true);
2103
2104 /* OPAL variant of PHB3 invalidated TCEs */
2105 if (pe->phb->ioda.tce_inval_reg)
2106 tbl->it_type |= (TCE_PCI_SWINV_CREATE | TCE_PCI_SWINV_FREE);
2107
2108 /*
2109 * Setting table base here only for carrying iommu_group
2110 * further down to let iommu_add_device() do the job.
2111 * pnv_pci_ioda_dma_dev_setup will override it later anyway.
2112 */
2113 if (pe->flags & PNV_IODA_PE_DEV)
2114 set_iommu_table_base(&pe->pdev->dev, tbl);
2115
2116 return 0;
2117}
2118
Alexey Kardashevskiyb5926432015-06-15 17:49:59 +10002119#if defined(CONFIG_IOMMU_API) || defined(CONFIG_PCI_IOV)
2120static long pnv_pci_ioda2_unset_window(struct iommu_table_group *table_group,
2121 int num)
2122{
2123 struct pnv_ioda_pe *pe = container_of(table_group, struct pnv_ioda_pe,
2124 table_group);
2125 struct pnv_phb *phb = pe->phb;
2126 long ret;
2127
2128 pe_info(pe, "Removing DMA window #%d\n", num);
2129
2130 ret = opal_pci_map_pe_dma_window(phb->opal_id, pe->pe_number,
2131 (pe->pe_number << 1) + num,
2132 0/* levels */, 0/* table address */,
2133 0/* table size */, 0/* page size */);
2134 if (ret)
2135 pe_warn(pe, "Unmapping failed, ret = %ld\n", ret);
2136 else
2137 pnv_pci_ioda2_tce_invalidate_entire(pe);
2138
2139 pnv_pci_unlink_table_and_group(table_group->tables[num], table_group);
2140
2141 return ret;
2142}
2143#endif
2144
Alexey Kardashevskiyf87a8862015-06-05 16:35:10 +10002145#ifdef CONFIG_IOMMU_API
Alexey Kardashevskiy00547192015-06-05 16:35:22 +10002146static unsigned long pnv_pci_ioda2_get_table_size(__u32 page_shift,
2147 __u64 window_size, __u32 levels)
2148{
2149 unsigned long bytes = 0;
2150 const unsigned window_shift = ilog2(window_size);
2151 unsigned entries_shift = window_shift - page_shift;
2152 unsigned table_shift = entries_shift + 3;
2153 unsigned long tce_table_size = max(0x1000UL, 1UL << table_shift);
2154 unsigned long direct_table_size;
2155
2156 if (!levels || (levels > POWERNV_IOMMU_MAX_LEVELS) ||
2157 (window_size > memory_hotplug_max()) ||
2158 !is_power_of_2(window_size))
2159 return 0;
2160
2161 /* Calculate a direct table size from window_size and levels */
2162 entries_shift = (entries_shift + levels - 1) / levels;
2163 table_shift = entries_shift + 3;
2164 table_shift = max_t(unsigned, table_shift, PAGE_SHIFT);
2165 direct_table_size = 1UL << table_shift;
2166
2167 for ( ; levels; --levels) {
2168 bytes += _ALIGN_UP(tce_table_size, direct_table_size);
2169
2170 tce_table_size /= direct_table_size;
2171 tce_table_size <<= 3;
2172 tce_table_size = _ALIGN_UP(tce_table_size, direct_table_size);
2173 }
2174
2175 return bytes;
2176}
2177
Alexey Kardashevskiyf87a8862015-06-05 16:35:10 +10002178static void pnv_ioda2_take_ownership(struct iommu_table_group *table_group)
Benjamin Herrenschmidtcd15b042014-02-11 11:32:38 +11002179{
Alexey Kardashevskiyf87a8862015-06-05 16:35:10 +10002180 struct pnv_ioda_pe *pe = container_of(table_group, struct pnv_ioda_pe,
2181 table_group);
Alexey Kardashevskiy46d3e1e2015-06-05 16:35:23 +10002182 /* Store @tbl as pnv_pci_ioda2_unset_window() resets it */
2183 struct iommu_table *tbl = pe->table_group.tables[0];
Benjamin Herrenschmidtcd15b042014-02-11 11:32:38 +11002184
Alexey Kardashevskiyf87a8862015-06-05 16:35:10 +10002185 pnv_pci_ioda2_set_bypass(pe, false);
Alexey Kardashevskiy46d3e1e2015-06-05 16:35:23 +10002186 pnv_pci_ioda2_unset_window(&pe->table_group, 0);
2187 pnv_ioda2_table_free(tbl);
Benjamin Herrenschmidtcd15b042014-02-11 11:32:38 +11002188}
2189
Alexey Kardashevskiyf87a8862015-06-05 16:35:10 +10002190static void pnv_ioda2_release_ownership(struct iommu_table_group *table_group)
2191{
2192 struct pnv_ioda_pe *pe = container_of(table_group, struct pnv_ioda_pe,
2193 table_group);
2194
Alexey Kardashevskiy46d3e1e2015-06-05 16:35:23 +10002195 pnv_pci_ioda2_setup_default_config(pe);
Alexey Kardashevskiyf87a8862015-06-05 16:35:10 +10002196}
2197
2198static struct iommu_table_group_ops pnv_pci_ioda2_ops = {
Alexey Kardashevskiy00547192015-06-05 16:35:22 +10002199 .get_table_size = pnv_pci_ioda2_get_table_size,
Alexey Kardashevskiy4793d652015-06-05 16:35:20 +10002200 .create_table = pnv_pci_ioda2_create_table,
2201 .set_window = pnv_pci_ioda2_set_window,
2202 .unset_window = pnv_pci_ioda2_unset_window,
Alexey Kardashevskiyf87a8862015-06-05 16:35:10 +10002203 .take_ownership = pnv_ioda2_take_ownership,
2204 .release_ownership = pnv_ioda2_release_ownership,
2205};
2206#endif
2207
Alexey Kardashevskiy5780fb02015-06-05 16:35:12 +10002208static void pnv_pci_ioda_setup_opal_tce_kill(struct pnv_phb *phb)
2209{
2210 const __be64 *swinvp;
2211
2212 /* OPAL variant of PHB3 invalidated TCEs */
2213 swinvp = of_get_property(phb->hose->dn, "ibm,opal-tce-kill", NULL);
2214 if (!swinvp)
2215 return;
2216
2217 phb->ioda.tce_inval_reg_phys = be64_to_cpup(swinvp);
2218 phb->ioda.tce_inval_reg = ioremap(phb->ioda.tce_inval_reg_phys, 8);
2219}
2220
Alexey Kardashevskiybbb845c2015-06-05 16:35:19 +10002221static __be64 *pnv_pci_ioda2_table_do_alloc_pages(int nid, unsigned shift,
2222 unsigned levels, unsigned long limit,
2223 unsigned long *current_offset)
Alexey Kardashevskiyaca69132015-06-05 16:35:17 +10002224{
2225 struct page *tce_mem = NULL;
Alexey Kardashevskiybbb845c2015-06-05 16:35:19 +10002226 __be64 *addr, *tmp;
Alexey Kardashevskiyaca69132015-06-05 16:35:17 +10002227 unsigned order = max_t(unsigned, shift, PAGE_SHIFT) - PAGE_SHIFT;
Alexey Kardashevskiybbb845c2015-06-05 16:35:19 +10002228 unsigned long allocated = 1UL << (order + PAGE_SHIFT);
2229 unsigned entries = 1UL << (shift - 3);
2230 long i;
Alexey Kardashevskiyaca69132015-06-05 16:35:17 +10002231
2232 tce_mem = alloc_pages_node(nid, GFP_KERNEL, order);
2233 if (!tce_mem) {
2234 pr_err("Failed to allocate a TCE memory, order=%d\n", order);
2235 return NULL;
2236 }
2237 addr = page_address(tce_mem);
Alexey Kardashevskiybbb845c2015-06-05 16:35:19 +10002238 memset(addr, 0, allocated);
2239
2240 --levels;
2241 if (!levels) {
2242 *current_offset += allocated;
2243 return addr;
2244 }
2245
2246 for (i = 0; i < entries; ++i) {
2247 tmp = pnv_pci_ioda2_table_do_alloc_pages(nid, shift,
2248 levels, limit, current_offset);
2249 if (!tmp)
2250 break;
2251
2252 addr[i] = cpu_to_be64(__pa(tmp) |
2253 TCE_PCI_READ | TCE_PCI_WRITE);
2254
2255 if (*current_offset >= limit)
2256 break;
2257 }
Alexey Kardashevskiyaca69132015-06-05 16:35:17 +10002258
2259 return addr;
2260}
2261
Alexey Kardashevskiybbb845c2015-06-05 16:35:19 +10002262static void pnv_pci_ioda2_table_do_free_pages(__be64 *addr,
2263 unsigned long size, unsigned level);
2264
Alexey Kardashevskiyaca69132015-06-05 16:35:17 +10002265static long pnv_pci_ioda2_table_alloc_pages(int nid, __u64 bus_offset,
Alexey Kardashevskiybbb845c2015-06-05 16:35:19 +10002266 __u32 page_shift, __u64 window_size, __u32 levels,
2267 struct iommu_table *tbl)
Alexey Kardashevskiyaca69132015-06-05 16:35:17 +10002268{
2269 void *addr;
Alexey Kardashevskiybbb845c2015-06-05 16:35:19 +10002270 unsigned long offset = 0, level_shift;
Alexey Kardashevskiyaca69132015-06-05 16:35:17 +10002271 const unsigned window_shift = ilog2(window_size);
2272 unsigned entries_shift = window_shift - page_shift;
2273 unsigned table_shift = max_t(unsigned, entries_shift + 3, PAGE_SHIFT);
2274 const unsigned long tce_table_size = 1UL << table_shift;
2275
Alexey Kardashevskiybbb845c2015-06-05 16:35:19 +10002276 if (!levels || (levels > POWERNV_IOMMU_MAX_LEVELS))
2277 return -EINVAL;
2278
Alexey Kardashevskiyaca69132015-06-05 16:35:17 +10002279 if ((window_size > memory_hotplug_max()) || !is_power_of_2(window_size))
2280 return -EINVAL;
2281
Alexey Kardashevskiybbb845c2015-06-05 16:35:19 +10002282 /* Adjust direct table size from window_size and levels */
2283 entries_shift = (entries_shift + levels - 1) / levels;
2284 level_shift = entries_shift + 3;
2285 level_shift = max_t(unsigned, level_shift, PAGE_SHIFT);
2286
Alexey Kardashevskiyaca69132015-06-05 16:35:17 +10002287 /* Allocate TCE table */
Alexey Kardashevskiybbb845c2015-06-05 16:35:19 +10002288 addr = pnv_pci_ioda2_table_do_alloc_pages(nid, level_shift,
2289 levels, tce_table_size, &offset);
2290
2291 /* addr==NULL means that the first level allocation failed */
Alexey Kardashevskiyaca69132015-06-05 16:35:17 +10002292 if (!addr)
2293 return -ENOMEM;
2294
Alexey Kardashevskiybbb845c2015-06-05 16:35:19 +10002295 /*
2296 * First level was allocated but some lower level failed as
2297 * we did not allocate as much as we wanted,
2298 * release partially allocated table.
2299 */
2300 if (offset < tce_table_size) {
2301 pnv_pci_ioda2_table_do_free_pages(addr,
2302 1ULL << (level_shift - 3), levels - 1);
2303 return -ENOMEM;
2304 }
2305
Alexey Kardashevskiyaca69132015-06-05 16:35:17 +10002306 /* Setup linux iommu table */
2307 pnv_pci_setup_iommu_table(tbl, addr, tce_table_size, bus_offset,
2308 page_shift);
Alexey Kardashevskiybbb845c2015-06-05 16:35:19 +10002309 tbl->it_level_size = 1ULL << (level_shift - 3);
2310 tbl->it_indirect_levels = levels - 1;
Alexey Kardashevskiy00547192015-06-05 16:35:22 +10002311 tbl->it_allocated_size = offset;
Alexey Kardashevskiyaca69132015-06-05 16:35:17 +10002312
2313 pr_devel("Created TCE table: ws=%08llx ts=%lx @%08llx\n",
2314 window_size, tce_table_size, bus_offset);
2315
2316 return 0;
2317}
2318
Alexey Kardashevskiybbb845c2015-06-05 16:35:19 +10002319static void pnv_pci_ioda2_table_do_free_pages(__be64 *addr,
2320 unsigned long size, unsigned level)
2321{
2322 const unsigned long addr_ul = (unsigned long) addr &
2323 ~(TCE_PCI_READ | TCE_PCI_WRITE);
2324
2325 if (level) {
2326 long i;
2327 u64 *tmp = (u64 *) addr_ul;
2328
2329 for (i = 0; i < size; ++i) {
2330 unsigned long hpa = be64_to_cpu(tmp[i]);
2331
2332 if (!(hpa & (TCE_PCI_READ | TCE_PCI_WRITE)))
2333 continue;
2334
2335 pnv_pci_ioda2_table_do_free_pages(__va(hpa), size,
2336 level - 1);
2337 }
2338 }
2339
2340 free_pages(addr_ul, get_order(size << 3));
2341}
2342
Alexey Kardashevskiyaca69132015-06-05 16:35:17 +10002343static void pnv_pci_ioda2_table_free_pages(struct iommu_table *tbl)
2344{
Alexey Kardashevskiybbb845c2015-06-05 16:35:19 +10002345 const unsigned long size = tbl->it_indirect_levels ?
2346 tbl->it_level_size : tbl->it_size;
2347
Alexey Kardashevskiyaca69132015-06-05 16:35:17 +10002348 if (!tbl->it_size)
2349 return;
2350
Alexey Kardashevskiybbb845c2015-06-05 16:35:19 +10002351 pnv_pci_ioda2_table_do_free_pages((__be64 *)tbl->it_base, size,
2352 tbl->it_indirect_levels);
Alexey Kardashevskiyaca69132015-06-05 16:35:17 +10002353}
2354
Gavin Shan373f5652013-04-25 19:21:01 +00002355static void pnv_pci_ioda2_setup_dma_pe(struct pnv_phb *phb,
2356 struct pnv_ioda_pe *pe)
2357{
Gavin Shan373f5652013-04-25 19:21:01 +00002358 int64_t rc;
2359
2360 /* We shouldn't already have a 32-bit DMA associated */
2361 if (WARN_ON(pe->tce32_seg >= 0))
2362 return;
2363
Alexey Kardashevskiyf87a8862015-06-05 16:35:10 +10002364 /* TVE #1 is selected by PCI address bit 59 */
2365 pe->tce_bypass_base = 1ull << 59;
2366
Alexey Kardashevskiyb348aa62015-06-05 16:35:08 +10002367 iommu_register_group(&pe->table_group, phb->hose->global_number,
2368 pe->pe_number);
Alexey Kardashevskiyc5773822015-06-05 16:34:55 +10002369
Gavin Shan373f5652013-04-25 19:21:01 +00002370 /* The PE will reserve all possible 32-bits space */
2371 pe->tce32_seg = 0;
Gavin Shan373f5652013-04-25 19:21:01 +00002372 pe_info(pe, "Setting up 32-bit TCE table at 0..%08x\n",
Alexey Kardashevskiyaca69132015-06-05 16:35:17 +10002373 phb->ioda.m32_pci_base);
Gavin Shan373f5652013-04-25 19:21:01 +00002374
Alexey Kardashevskiye5aad1e2015-06-05 16:35:16 +10002375 /* Setup linux iommu table */
Alexey Kardashevskiy4793d652015-06-05 16:35:20 +10002376 pe->table_group.tce32_start = 0;
2377 pe->table_group.tce32_size = phb->ioda.m32_pci_base;
2378 pe->table_group.max_dynamic_windows_supported =
2379 IOMMU_TABLE_GROUP_MAX_TABLES;
2380 pe->table_group.max_levels = POWERNV_IOMMU_MAX_LEVELS;
2381 pe->table_group.pgsizes = SZ_4K | SZ_64K | SZ_16M;
Alexey Kardashevskiye5aad1e2015-06-05 16:35:16 +10002382#ifdef CONFIG_IOMMU_API
2383 pe->table_group.ops = &pnv_pci_ioda2_ops;
2384#endif
2385
Alexey Kardashevskiy46d3e1e2015-06-05 16:35:23 +10002386 rc = pnv_pci_ioda2_setup_default_config(pe);
Gavin Shan373f5652013-04-25 19:21:01 +00002387 if (rc) {
Alexey Kardashevskiy46d3e1e2015-06-05 16:35:23 +10002388 if (pe->tce32_seg >= 0)
2389 pe->tce32_seg = -1;
2390 return;
Gavin Shan373f5652013-04-25 19:21:01 +00002391 }
2392
Alexey Kardashevskiy46d3e1e2015-06-05 16:35:23 +10002393 if (pe->flags & PNV_IODA_PE_DEV)
Alexey Kardashevskiy46170822015-06-05 16:34:54 +10002394 iommu_add_device(&pe->pdev->dev);
Alexey Kardashevskiy46d3e1e2015-06-05 16:35:23 +10002395 else if (pe->flags & (PNV_IODA_PE_BUS | PNV_IODA_PE_BUS_ALL))
Alexey Kardashevskiyea30e992015-06-05 16:34:53 +10002396 pnv_ioda_setup_bus_dma(pe, pe->pbus);
Gavin Shan373f5652013-04-25 19:21:01 +00002397}
2398
Greg Kroah-Hartmancad5cef2012-12-21 14:04:10 -08002399static void pnv_ioda_setup_dma(struct pnv_phb *phb)
Benjamin Herrenschmidt184cd4a2011-11-15 17:29:08 +00002400{
2401 struct pci_controller *hose = phb->hose;
2402 unsigned int residual, remaining, segs, tw, base;
2403 struct pnv_ioda_pe *pe;
2404
2405 /* If we have more PE# than segments available, hand out one
2406 * per PE until we run out and let the rest fail. If not,
2407 * then we assign at least one segment per PE, plus more based
2408 * on the amount of devices under that PE
2409 */
2410 if (phb->ioda.dma_pe_count > phb->ioda.tce32_count)
2411 residual = 0;
2412 else
2413 residual = phb->ioda.tce32_count -
2414 phb->ioda.dma_pe_count;
2415
2416 pr_info("PCI: Domain %04x has %ld available 32-bit DMA segments\n",
2417 hose->global_number, phb->ioda.tce32_count);
2418 pr_info("PCI: %d PE# for a total weight of %d\n",
2419 phb->ioda.dma_pe_count, phb->ioda.dma_weight);
2420
Alexey Kardashevskiy5780fb02015-06-05 16:35:12 +10002421 pnv_pci_ioda_setup_opal_tce_kill(phb);
2422
Benjamin Herrenschmidt184cd4a2011-11-15 17:29:08 +00002423 /* Walk our PE list and configure their DMA segments, hand them
2424 * out one base segment plus any residual segments based on
2425 * weight
2426 */
2427 remaining = phb->ioda.tce32_count;
2428 tw = phb->ioda.dma_weight;
2429 base = 0;
Gavin Shan7ebdf952012-08-20 03:49:15 +00002430 list_for_each_entry(pe, &phb->ioda.pe_dma_list, dma_link) {
Benjamin Herrenschmidt184cd4a2011-11-15 17:29:08 +00002431 if (!pe->dma_weight)
2432 continue;
2433 if (!remaining) {
2434 pe_warn(pe, "No DMA32 resources available\n");
2435 continue;
2436 }
2437 segs = 1;
2438 if (residual) {
2439 segs += ((pe->dma_weight * residual) + (tw / 2)) / tw;
2440 if (segs > remaining)
2441 segs = remaining;
2442 }
Gavin Shan373f5652013-04-25 19:21:01 +00002443
2444 /*
2445 * For IODA2 compliant PHB3, we needn't care about the weight.
2446 * The all available 32-bits DMA space will be assigned to
2447 * the specific PE.
2448 */
2449 if (phb->type == PNV_PHB_IODA1) {
2450 pe_info(pe, "DMA weight %d, assigned %d DMA32 segments\n",
2451 pe->dma_weight, segs);
2452 pnv_pci_ioda_setup_dma_pe(phb, pe, base, segs);
2453 } else {
2454 pe_info(pe, "Assign DMA32 space\n");
2455 segs = 0;
2456 pnv_pci_ioda2_setup_dma_pe(phb, pe);
2457 }
2458
Benjamin Herrenschmidt184cd4a2011-11-15 17:29:08 +00002459 remaining -= segs;
2460 base += segs;
2461 }
2462}
2463
2464#ifdef CONFIG_PCI_MSI
Gavin Shan137436c2013-04-25 19:20:59 +00002465static void pnv_ioda2_msi_eoi(struct irq_data *d)
2466{
2467 unsigned int hw_irq = (unsigned int)irqd_to_hwirq(d);
2468 struct irq_chip *chip = irq_data_get_irq_chip(d);
2469 struct pnv_phb *phb = container_of(chip, struct pnv_phb,
2470 ioda.irq_chip);
2471 int64_t rc;
2472
2473 rc = opal_pci_msi_eoi(phb->opal_id, hw_irq);
2474 WARN_ON_ONCE(rc);
2475
2476 icp_native_eoi(d);
2477}
2478
Ian Munsiefd9a1c22014-10-08 19:54:55 +11002479
2480static void set_msi_irq_chip(struct pnv_phb *phb, unsigned int virq)
2481{
2482 struct irq_data *idata;
2483 struct irq_chip *ichip;
2484
2485 if (phb->type != PNV_PHB_IODA2)
2486 return;
2487
2488 if (!phb->ioda.irq_chip_init) {
2489 /*
2490 * First time we setup an MSI IRQ, we need to setup the
2491 * corresponding IRQ chip to route correctly.
2492 */
2493 idata = irq_get_irq_data(virq);
2494 ichip = irq_data_get_irq_chip(idata);
2495 phb->ioda.irq_chip_init = 1;
2496 phb->ioda.irq_chip = *ichip;
2497 phb->ioda.irq_chip.irq_eoi = pnv_ioda2_msi_eoi;
2498 }
2499 irq_set_chip(virq, &phb->ioda.irq_chip);
2500}
2501
Ian Munsie80c49c72014-10-08 19:54:57 +11002502#ifdef CONFIG_CXL_BASE
2503
Ryan Grimm6f963ec2015-01-28 20:16:04 -06002504struct device_node *pnv_pci_get_phb_node(struct pci_dev *dev)
Ian Munsie80c49c72014-10-08 19:54:57 +11002505{
2506 struct pci_controller *hose = pci_bus_to_host(dev->bus);
2507
Ryan Grimm6f963ec2015-01-28 20:16:04 -06002508 return of_node_get(hose->dn);
Ian Munsie80c49c72014-10-08 19:54:57 +11002509}
Ryan Grimm6f963ec2015-01-28 20:16:04 -06002510EXPORT_SYMBOL(pnv_pci_get_phb_node);
Ian Munsie80c49c72014-10-08 19:54:57 +11002511
Ryan Grimm1212aa12015-01-19 11:52:50 -06002512int pnv_phb_to_cxl_mode(struct pci_dev *dev, uint64_t mode)
Ian Munsie80c49c72014-10-08 19:54:57 +11002513{
2514 struct pci_controller *hose = pci_bus_to_host(dev->bus);
2515 struct pnv_phb *phb = hose->private_data;
2516 struct pnv_ioda_pe *pe;
2517 int rc;
2518
2519 pe = pnv_ioda_get_pe(dev);
2520 if (!pe)
2521 return -ENODEV;
2522
2523 pe_info(pe, "Switching PHB to CXL\n");
2524
Ryan Grimm1212aa12015-01-19 11:52:50 -06002525 rc = opal_pci_set_phb_cxl_mode(phb->opal_id, mode, pe->pe_number);
Ian Munsie80c49c72014-10-08 19:54:57 +11002526 if (rc)
2527 dev_err(&dev->dev, "opal_pci_set_phb_cxl_mode failed: %i\n", rc);
2528
2529 return rc;
2530}
Ryan Grimm1212aa12015-01-19 11:52:50 -06002531EXPORT_SYMBOL(pnv_phb_to_cxl_mode);
Ian Munsie80c49c72014-10-08 19:54:57 +11002532
2533/* Find PHB for cxl dev and allocate MSI hwirqs?
2534 * Returns the absolute hardware IRQ number
2535 */
2536int pnv_cxl_alloc_hwirqs(struct pci_dev *dev, int num)
2537{
2538 struct pci_controller *hose = pci_bus_to_host(dev->bus);
2539 struct pnv_phb *phb = hose->private_data;
2540 int hwirq = msi_bitmap_alloc_hwirqs(&phb->msi_bmp, num);
2541
2542 if (hwirq < 0) {
2543 dev_warn(&dev->dev, "Failed to find a free MSI\n");
2544 return -ENOSPC;
2545 }
2546
2547 return phb->msi_base + hwirq;
2548}
2549EXPORT_SYMBOL(pnv_cxl_alloc_hwirqs);
2550
2551void pnv_cxl_release_hwirqs(struct pci_dev *dev, int hwirq, int num)
2552{
2553 struct pci_controller *hose = pci_bus_to_host(dev->bus);
2554 struct pnv_phb *phb = hose->private_data;
2555
2556 msi_bitmap_free_hwirqs(&phb->msi_bmp, hwirq - phb->msi_base, num);
2557}
2558EXPORT_SYMBOL(pnv_cxl_release_hwirqs);
2559
2560void pnv_cxl_release_hwirq_ranges(struct cxl_irq_ranges *irqs,
2561 struct pci_dev *dev)
2562{
2563 struct pci_controller *hose = pci_bus_to_host(dev->bus);
2564 struct pnv_phb *phb = hose->private_data;
2565 int i, hwirq;
2566
2567 for (i = 1; i < CXL_IRQ_RANGES; i++) {
2568 if (!irqs->range[i])
2569 continue;
2570 pr_devel("cxl release irq range 0x%x: offset: 0x%lx limit: %ld\n",
2571 i, irqs->offset[i],
2572 irqs->range[i]);
2573 hwirq = irqs->offset[i] - phb->msi_base;
2574 msi_bitmap_free_hwirqs(&phb->msi_bmp, hwirq,
2575 irqs->range[i]);
2576 }
2577}
2578EXPORT_SYMBOL(pnv_cxl_release_hwirq_ranges);
2579
2580int pnv_cxl_alloc_hwirq_ranges(struct cxl_irq_ranges *irqs,
2581 struct pci_dev *dev, int num)
2582{
2583 struct pci_controller *hose = pci_bus_to_host(dev->bus);
2584 struct pnv_phb *phb = hose->private_data;
2585 int i, hwirq, try;
2586
2587 memset(irqs, 0, sizeof(struct cxl_irq_ranges));
2588
2589 /* 0 is reserved for the multiplexed PSL DSI interrupt */
2590 for (i = 1; i < CXL_IRQ_RANGES && num; i++) {
2591 try = num;
2592 while (try) {
2593 hwirq = msi_bitmap_alloc_hwirqs(&phb->msi_bmp, try);
2594 if (hwirq >= 0)
2595 break;
2596 try /= 2;
2597 }
2598 if (!try)
2599 goto fail;
2600
2601 irqs->offset[i] = phb->msi_base + hwirq;
2602 irqs->range[i] = try;
2603 pr_devel("cxl alloc irq range 0x%x: offset: 0x%lx limit: %li\n",
2604 i, irqs->offset[i], irqs->range[i]);
2605 num -= try;
2606 }
2607 if (num)
2608 goto fail;
2609
2610 return 0;
2611fail:
2612 pnv_cxl_release_hwirq_ranges(irqs, dev);
2613 return -ENOSPC;
2614}
2615EXPORT_SYMBOL(pnv_cxl_alloc_hwirq_ranges);
2616
2617int pnv_cxl_get_irq_count(struct pci_dev *dev)
2618{
2619 struct pci_controller *hose = pci_bus_to_host(dev->bus);
2620 struct pnv_phb *phb = hose->private_data;
2621
2622 return phb->msi_bmp.irq_count;
2623}
2624EXPORT_SYMBOL(pnv_cxl_get_irq_count);
2625
2626int pnv_cxl_ioda_msi_setup(struct pci_dev *dev, unsigned int hwirq,
2627 unsigned int virq)
2628{
2629 struct pci_controller *hose = pci_bus_to_host(dev->bus);
2630 struct pnv_phb *phb = hose->private_data;
2631 unsigned int xive_num = hwirq - phb->msi_base;
2632 struct pnv_ioda_pe *pe;
2633 int rc;
2634
2635 if (!(pe = pnv_ioda_get_pe(dev)))
2636 return -ENODEV;
2637
2638 /* Assign XIVE to PE */
2639 rc = opal_pci_set_xive_pe(phb->opal_id, pe->pe_number, xive_num);
2640 if (rc) {
2641 pe_warn(pe, "%s: OPAL error %d setting msi_base 0x%x "
2642 "hwirq 0x%x XIVE 0x%x PE\n",
2643 pci_name(dev), rc, phb->msi_base, hwirq, xive_num);
2644 return -EIO;
2645 }
2646 set_msi_irq_chip(phb, virq);
2647
2648 return 0;
2649}
2650EXPORT_SYMBOL(pnv_cxl_ioda_msi_setup);
2651#endif
2652
Benjamin Herrenschmidt184cd4a2011-11-15 17:29:08 +00002653static int pnv_pci_ioda_msi_setup(struct pnv_phb *phb, struct pci_dev *dev,
Gavin Shan137436c2013-04-25 19:20:59 +00002654 unsigned int hwirq, unsigned int virq,
2655 unsigned int is_64, struct msi_msg *msg)
Benjamin Herrenschmidt184cd4a2011-11-15 17:29:08 +00002656{
2657 struct pnv_ioda_pe *pe = pnv_ioda_get_pe(dev);
2658 unsigned int xive_num = hwirq - phb->msi_base;
Benjamin Herrenschmidt3a1a4662013-09-23 12:05:01 +10002659 __be32 data;
Benjamin Herrenschmidt184cd4a2011-11-15 17:29:08 +00002660 int rc;
2661
2662 /* No PE assigned ? bail out ... no MSI for you ! */
2663 if (pe == NULL)
2664 return -ENXIO;
2665
2666 /* Check if we have an MVE */
2667 if (pe->mve_number < 0)
2668 return -ENXIO;
2669
Benjamin Herrenschmidtb72c1f62013-05-21 22:58:21 +00002670 /* Force 32-bit MSI on some broken devices */
Benjamin Herrenschmidt36074382014-10-07 16:12:36 +11002671 if (dev->no_64bit_msi)
Benjamin Herrenschmidtb72c1f62013-05-21 22:58:21 +00002672 is_64 = 0;
2673
Benjamin Herrenschmidt184cd4a2011-11-15 17:29:08 +00002674 /* Assign XIVE to PE */
2675 rc = opal_pci_set_xive_pe(phb->opal_id, pe->pe_number, xive_num);
2676 if (rc) {
2677 pr_warn("%s: OPAL error %d setting XIVE %d PE\n",
2678 pci_name(dev), rc, xive_num);
2679 return -EIO;
2680 }
2681
2682 if (is_64) {
Benjamin Herrenschmidt3a1a4662013-09-23 12:05:01 +10002683 __be64 addr64;
2684
Benjamin Herrenschmidt184cd4a2011-11-15 17:29:08 +00002685 rc = opal_get_msi_64(phb->opal_id, pe->mve_number, xive_num, 1,
2686 &addr64, &data);
2687 if (rc) {
2688 pr_warn("%s: OPAL error %d getting 64-bit MSI data\n",
2689 pci_name(dev), rc);
2690 return -EIO;
2691 }
Benjamin Herrenschmidt3a1a4662013-09-23 12:05:01 +10002692 msg->address_hi = be64_to_cpu(addr64) >> 32;
2693 msg->address_lo = be64_to_cpu(addr64) & 0xfffffffful;
Benjamin Herrenschmidt184cd4a2011-11-15 17:29:08 +00002694 } else {
Benjamin Herrenschmidt3a1a4662013-09-23 12:05:01 +10002695 __be32 addr32;
2696
Benjamin Herrenschmidt184cd4a2011-11-15 17:29:08 +00002697 rc = opal_get_msi_32(phb->opal_id, pe->mve_number, xive_num, 1,
2698 &addr32, &data);
2699 if (rc) {
2700 pr_warn("%s: OPAL error %d getting 32-bit MSI data\n",
2701 pci_name(dev), rc);
2702 return -EIO;
2703 }
2704 msg->address_hi = 0;
Benjamin Herrenschmidt3a1a4662013-09-23 12:05:01 +10002705 msg->address_lo = be32_to_cpu(addr32);
Benjamin Herrenschmidt184cd4a2011-11-15 17:29:08 +00002706 }
Benjamin Herrenschmidt3a1a4662013-09-23 12:05:01 +10002707 msg->data = be32_to_cpu(data);
Benjamin Herrenschmidt184cd4a2011-11-15 17:29:08 +00002708
Ian Munsiefd9a1c22014-10-08 19:54:55 +11002709 set_msi_irq_chip(phb, virq);
Gavin Shan137436c2013-04-25 19:20:59 +00002710
Benjamin Herrenschmidt184cd4a2011-11-15 17:29:08 +00002711 pr_devel("%s: %s-bit MSI on hwirq %x (xive #%d),"
2712 " address=%x_%08x data=%x PE# %d\n",
2713 pci_name(dev), is_64 ? "64" : "32", hwirq, xive_num,
2714 msg->address_hi, msg->address_lo, data, pe->pe_number);
2715
2716 return 0;
2717}
2718
2719static void pnv_pci_init_ioda_msis(struct pnv_phb *phb)
2720{
Gavin Shanfb1b55d2013-03-05 21:12:37 +00002721 unsigned int count;
Benjamin Herrenschmidt184cd4a2011-11-15 17:29:08 +00002722 const __be32 *prop = of_get_property(phb->hose->dn,
2723 "ibm,opal-msi-ranges", NULL);
2724 if (!prop) {
2725 /* BML Fallback */
2726 prop = of_get_property(phb->hose->dn, "msi-ranges", NULL);
2727 }
2728 if (!prop)
2729 return;
2730
2731 phb->msi_base = be32_to_cpup(prop);
Gavin Shanfb1b55d2013-03-05 21:12:37 +00002732 count = be32_to_cpup(prop + 1);
2733 if (msi_bitmap_alloc(&phb->msi_bmp, count, phb->hose->dn)) {
Benjamin Herrenschmidt184cd4a2011-11-15 17:29:08 +00002734 pr_err("PCI %d: Failed to allocate MSI bitmap !\n",
2735 phb->hose->global_number);
2736 return;
2737 }
Gavin Shanfb1b55d2013-03-05 21:12:37 +00002738
Benjamin Herrenschmidt184cd4a2011-11-15 17:29:08 +00002739 phb->msi_setup = pnv_pci_ioda_msi_setup;
2740 phb->msi32_support = 1;
2741 pr_info(" Allocated bitmap for %d MSIs (base IRQ 0x%x)\n",
Gavin Shanfb1b55d2013-03-05 21:12:37 +00002742 count, phb->msi_base);
Benjamin Herrenschmidt184cd4a2011-11-15 17:29:08 +00002743}
2744#else
2745static void pnv_pci_init_ioda_msis(struct pnv_phb *phb) { }
2746#endif /* CONFIG_PCI_MSI */
2747
Wei Yang6e628c72015-03-25 16:23:55 +08002748#ifdef CONFIG_PCI_IOV
2749static void pnv_pci_ioda_fixup_iov_resources(struct pci_dev *pdev)
2750{
2751 struct pci_controller *hose;
2752 struct pnv_phb *phb;
2753 struct resource *res;
2754 int i;
2755 resource_size_t size;
2756 struct pci_dn *pdn;
Wei Yang5b88ec22015-03-25 16:23:58 +08002757 int mul, total_vfs;
Wei Yang6e628c72015-03-25 16:23:55 +08002758
2759 if (!pdev->is_physfn || pdev->is_added)
2760 return;
2761
2762 hose = pci_bus_to_host(pdev->bus);
2763 phb = hose->private_data;
2764
2765 pdn = pci_get_pdn(pdev);
2766 pdn->vfs_expanded = 0;
2767
Wei Yang5b88ec22015-03-25 16:23:58 +08002768 total_vfs = pci_sriov_get_totalvfs(pdev);
2769 pdn->m64_per_iov = 1;
2770 mul = phb->ioda.total_pe;
2771
2772 for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) {
2773 res = &pdev->resource[i + PCI_IOV_RESOURCES];
2774 if (!res->flags || res->parent)
2775 continue;
2776 if (!pnv_pci_is_mem_pref_64(res->flags)) {
2777 dev_warn(&pdev->dev, " non M64 VF BAR%d: %pR\n",
2778 i, res);
2779 continue;
2780 }
2781
2782 size = pci_iov_resource_size(pdev, i + PCI_IOV_RESOURCES);
2783
2784 /* bigger than 64M */
2785 if (size > (1 << 26)) {
2786 dev_info(&pdev->dev, "PowerNV: VF BAR%d: %pR IOV size is bigger than 64M, roundup power2\n",
2787 i, res);
2788 pdn->m64_per_iov = M64_PER_IOV;
2789 mul = roundup_pow_of_two(total_vfs);
2790 break;
2791 }
2792 }
2793
Wei Yang6e628c72015-03-25 16:23:55 +08002794 for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) {
2795 res = &pdev->resource[i + PCI_IOV_RESOURCES];
2796 if (!res->flags || res->parent)
2797 continue;
2798 if (!pnv_pci_is_mem_pref_64(res->flags)) {
2799 dev_warn(&pdev->dev, "Skipping expanding VF BAR%d: %pR\n",
2800 i, res);
2801 continue;
2802 }
2803
2804 dev_dbg(&pdev->dev, " Fixing VF BAR%d: %pR to\n", i, res);
2805 size = pci_iov_resource_size(pdev, i + PCI_IOV_RESOURCES);
Wei Yang5b88ec22015-03-25 16:23:58 +08002806 res->end = res->start + size * mul - 1;
Wei Yang6e628c72015-03-25 16:23:55 +08002807 dev_dbg(&pdev->dev, " %pR\n", res);
2808 dev_info(&pdev->dev, "VF BAR%d: %pR (expanded to %d VFs for PE alignment)",
Wei Yang5b88ec22015-03-25 16:23:58 +08002809 i, res, mul);
Wei Yang6e628c72015-03-25 16:23:55 +08002810 }
Wei Yang5b88ec22015-03-25 16:23:58 +08002811 pdn->vfs_expanded = mul;
Wei Yang6e628c72015-03-25 16:23:55 +08002812}
2813#endif /* CONFIG_PCI_IOV */
2814
Gavin Shan11685be2012-08-20 03:49:16 +00002815/*
2816 * This function is supposed to be called on basis of PE from top
2817 * to bottom style. So the the I/O or MMIO segment assigned to
2818 * parent PE could be overrided by its child PEs if necessary.
2819 */
Greg Kroah-Hartmancad5cef2012-12-21 14:04:10 -08002820static void pnv_ioda_setup_pe_seg(struct pci_controller *hose,
2821 struct pnv_ioda_pe *pe)
Gavin Shan11685be2012-08-20 03:49:16 +00002822{
2823 struct pnv_phb *phb = hose->private_data;
2824 struct pci_bus_region region;
2825 struct resource *res;
2826 int i, index;
2827 int rc;
2828
2829 /*
2830 * NOTE: We only care PCI bus based PE for now. For PCI
2831 * device based PE, for example SRIOV sensitive VF should
2832 * be figured out later.
2833 */
2834 BUG_ON(!(pe->flags & (PNV_IODA_PE_BUS | PNV_IODA_PE_BUS_ALL)));
2835
2836 pci_bus_for_each_resource(pe->pbus, res, i) {
2837 if (!res || !res->flags ||
2838 res->start > res->end)
2839 continue;
2840
2841 if (res->flags & IORESOURCE_IO) {
2842 region.start = res->start - phb->ioda.io_pci_base;
2843 region.end = res->end - phb->ioda.io_pci_base;
2844 index = region.start / phb->ioda.io_segsize;
2845
2846 while (index < phb->ioda.total_pe &&
2847 region.start <= region.end) {
2848 phb->ioda.io_segmap[index] = pe->pe_number;
2849 rc = opal_pci_map_pe_mmio_window(phb->opal_id,
2850 pe->pe_number, OPAL_IO_WINDOW_TYPE, 0, index);
2851 if (rc != OPAL_SUCCESS) {
2852 pr_err("%s: OPAL error %d when mapping IO "
2853 "segment #%d to PE#%d\n",
2854 __func__, rc, index, pe->pe_number);
2855 break;
2856 }
2857
2858 region.start += phb->ioda.io_segsize;
2859 index++;
2860 }
Gavin Shan027fa022015-03-27 11:29:00 +11002861 } else if ((res->flags & IORESOURCE_MEM) &&
2862 !pnv_pci_is_mem_pref_64(res->flags)) {
Gavin Shan11685be2012-08-20 03:49:16 +00002863 region.start = res->start -
Benjamin Herrenschmidt3fd47f02013-05-06 13:40:40 +10002864 hose->mem_offset[0] -
Gavin Shan11685be2012-08-20 03:49:16 +00002865 phb->ioda.m32_pci_base;
2866 region.end = res->end -
Benjamin Herrenschmidt3fd47f02013-05-06 13:40:40 +10002867 hose->mem_offset[0] -
Gavin Shan11685be2012-08-20 03:49:16 +00002868 phb->ioda.m32_pci_base;
2869 index = region.start / phb->ioda.m32_segsize;
2870
2871 while (index < phb->ioda.total_pe &&
2872 region.start <= region.end) {
2873 phb->ioda.m32_segmap[index] = pe->pe_number;
2874 rc = opal_pci_map_pe_mmio_window(phb->opal_id,
2875 pe->pe_number, OPAL_M32_WINDOW_TYPE, 0, index);
2876 if (rc != OPAL_SUCCESS) {
2877 pr_err("%s: OPAL error %d when mapping M32 "
2878 "segment#%d to PE#%d",
2879 __func__, rc, index, pe->pe_number);
2880 break;
2881 }
2882
2883 region.start += phb->ioda.m32_segsize;
2884 index++;
2885 }
2886 }
2887 }
2888}
2889
Greg Kroah-Hartmancad5cef2012-12-21 14:04:10 -08002890static void pnv_pci_ioda_setup_seg(void)
Gavin Shan11685be2012-08-20 03:49:16 +00002891{
2892 struct pci_controller *tmp, *hose;
2893 struct pnv_phb *phb;
2894 struct pnv_ioda_pe *pe;
2895
2896 list_for_each_entry_safe(hose, tmp, &hose_list, list_node) {
2897 phb = hose->private_data;
2898 list_for_each_entry(pe, &phb->ioda.pe_list, list) {
2899 pnv_ioda_setup_pe_seg(hose, pe);
2900 }
2901 }
2902}
2903
Greg Kroah-Hartmancad5cef2012-12-21 14:04:10 -08002904static void pnv_pci_ioda_setup_DMA(void)
Gavin Shan13395c42012-08-20 03:49:17 +00002905{
2906 struct pci_controller *hose, *tmp;
Gavin Shandb1266c2012-08-20 03:49:18 +00002907 struct pnv_phb *phb;
Gavin Shan13395c42012-08-20 03:49:17 +00002908
2909 list_for_each_entry_safe(hose, tmp, &hose_list, list_node) {
2910 pnv_ioda_setup_dma(hose->private_data);
Gavin Shandb1266c2012-08-20 03:49:18 +00002911
2912 /* Mark the PHB initialization done */
2913 phb = hose->private_data;
2914 phb->initialized = 1;
Gavin Shan13395c42012-08-20 03:49:17 +00002915 }
2916}
2917
Gavin Shan37c367f2013-06-20 18:13:25 +08002918static void pnv_pci_ioda_create_dbgfs(void)
2919{
2920#ifdef CONFIG_DEBUG_FS
2921 struct pci_controller *hose, *tmp;
2922 struct pnv_phb *phb;
2923 char name[16];
2924
2925 list_for_each_entry_safe(hose, tmp, &hose_list, list_node) {
2926 phb = hose->private_data;
2927
2928 sprintf(name, "PCI%04x", hose->global_number);
2929 phb->dbgfs = debugfs_create_dir(name, powerpc_debugfs_root);
2930 if (!phb->dbgfs)
2931 pr_warning("%s: Error on creating debugfs on PHB#%x\n",
2932 __func__, hose->global_number);
2933 }
2934#endif /* CONFIG_DEBUG_FS */
2935}
2936
Greg Kroah-Hartmancad5cef2012-12-21 14:04:10 -08002937static void pnv_pci_ioda_fixup(void)
Gavin Shanfb446ad2012-08-20 03:49:14 +00002938{
2939 pnv_pci_ioda_setup_PEs();
Gavin Shan11685be2012-08-20 03:49:16 +00002940 pnv_pci_ioda_setup_seg();
Gavin Shan13395c42012-08-20 03:49:17 +00002941 pnv_pci_ioda_setup_DMA();
Gavin Shane9cc17d2013-06-20 13:21:14 +08002942
Gavin Shan37c367f2013-06-20 18:13:25 +08002943 pnv_pci_ioda_create_dbgfs();
2944
Gavin Shane9cc17d2013-06-20 13:21:14 +08002945#ifdef CONFIG_EEH
Gavin Shane9cc17d2013-06-20 13:21:14 +08002946 eeh_init();
Mike Qiudadcd6d2014-06-26 02:58:47 -04002947 eeh_addr_cache_build();
Gavin Shane9cc17d2013-06-20 13:21:14 +08002948#endif
Gavin Shanfb446ad2012-08-20 03:49:14 +00002949}
2950
Gavin Shan271fd032012-09-11 16:59:47 -06002951/*
2952 * Returns the alignment for I/O or memory windows for P2P
2953 * bridges. That actually depends on how PEs are segmented.
2954 * For now, we return I/O or M32 segment size for PE sensitive
2955 * P2P bridges. Otherwise, the default values (4KiB for I/O,
2956 * 1MiB for memory) will be returned.
2957 *
2958 * The current PCI bus might be put into one PE, which was
2959 * create against the parent PCI bridge. For that case, we
2960 * needn't enlarge the alignment so that we can save some
2961 * resources.
2962 */
2963static resource_size_t pnv_pci_window_alignment(struct pci_bus *bus,
2964 unsigned long type)
2965{
2966 struct pci_dev *bridge;
2967 struct pci_controller *hose = pci_bus_to_host(bus);
2968 struct pnv_phb *phb = hose->private_data;
2969 int num_pci_bridges = 0;
2970
2971 bridge = bus->self;
2972 while (bridge) {
2973 if (pci_pcie_type(bridge) == PCI_EXP_TYPE_PCI_BRIDGE) {
2974 num_pci_bridges++;
2975 if (num_pci_bridges >= 2)
2976 return 1;
2977 }
2978
2979 bridge = bridge->bus->self;
2980 }
2981
Guo Chao262af552014-07-21 14:42:30 +10002982 /* We fail back to M32 if M64 isn't supported */
2983 if (phb->ioda.m64_segsize &&
2984 pnv_pci_is_mem_pref_64(type))
2985 return phb->ioda.m64_segsize;
Gavin Shan271fd032012-09-11 16:59:47 -06002986 if (type & IORESOURCE_MEM)
2987 return phb->ioda.m32_segsize;
2988
2989 return phb->ioda.io_segsize;
2990}
2991
Wei Yang5350ab32015-03-25 16:23:56 +08002992#ifdef CONFIG_PCI_IOV
2993static resource_size_t pnv_pci_iov_resource_alignment(struct pci_dev *pdev,
2994 int resno)
2995{
2996 struct pci_dn *pdn = pci_get_pdn(pdev);
2997 resource_size_t align, iov_align;
2998
2999 iov_align = resource_size(&pdev->resource[resno]);
3000 if (iov_align)
3001 return iov_align;
3002
3003 align = pci_iov_resource_size(pdev, resno);
3004 if (pdn->vfs_expanded)
3005 return pdn->vfs_expanded * align;
3006
3007 return align;
3008}
3009#endif /* CONFIG_PCI_IOV */
3010
Benjamin Herrenschmidt184cd4a2011-11-15 17:29:08 +00003011/* Prevent enabling devices for which we couldn't properly
3012 * assign a PE
3013 */
Daniel Axtensc88c2a12015-03-31 16:00:41 +11003014static bool pnv_pci_enable_device_hook(struct pci_dev *dev)
Benjamin Herrenschmidt184cd4a2011-11-15 17:29:08 +00003015{
Gavin Shandb1266c2012-08-20 03:49:18 +00003016 struct pci_controller *hose = pci_bus_to_host(dev->bus);
3017 struct pnv_phb *phb = hose->private_data;
3018 struct pci_dn *pdn;
Benjamin Herrenschmidt184cd4a2011-11-15 17:29:08 +00003019
Gavin Shandb1266c2012-08-20 03:49:18 +00003020 /* The function is probably called while the PEs have
3021 * not be created yet. For example, resource reassignment
3022 * during PCI probe period. We just skip the check if
3023 * PEs isn't ready.
3024 */
3025 if (!phb->initialized)
Daniel Axtensc88c2a12015-03-31 16:00:41 +11003026 return true;
Gavin Shandb1266c2012-08-20 03:49:18 +00003027
Benjamin Herrenschmidtb72c1f62013-05-21 22:58:21 +00003028 pdn = pci_get_pdn(dev);
Benjamin Herrenschmidt184cd4a2011-11-15 17:29:08 +00003029 if (!pdn || pdn->pe_number == IODA_INVALID_PE)
Daniel Axtensc88c2a12015-03-31 16:00:41 +11003030 return false;
Gavin Shandb1266c2012-08-20 03:49:18 +00003031
Daniel Axtensc88c2a12015-03-31 16:00:41 +11003032 return true;
Benjamin Herrenschmidt184cd4a2011-11-15 17:29:08 +00003033}
3034
3035static u32 pnv_ioda_bdfn_to_pe(struct pnv_phb *phb, struct pci_bus *bus,
3036 u32 devfn)
3037{
3038 return phb->ioda.pe_rmap[(bus->number << 8) | devfn];
3039}
3040
Michael Neuling7a8e6bb2015-05-27 16:06:59 +10003041static void pnv_pci_ioda_shutdown(struct pci_controller *hose)
Benjamin Herrenschmidt73ed1482013-05-10 16:59:18 +10003042{
Michael Neuling7a8e6bb2015-05-27 16:06:59 +10003043 struct pnv_phb *phb = hose->private_data;
3044
Gavin Shand1a85ee2014-09-30 12:39:05 +10003045 opal_pci_reset(phb->opal_id, OPAL_RESET_PCI_IODA_TABLE,
Benjamin Herrenschmidt73ed1482013-05-10 16:59:18 +10003046 OPAL_ASSERT_RESET);
3047}
3048
Daniel Axtens92ae0352015-04-28 15:12:05 +10003049static const struct pci_controller_ops pnv_pci_ioda_controller_ops = {
3050 .dma_dev_setup = pnv_pci_dma_dev_setup,
3051#ifdef CONFIG_PCI_MSI
3052 .setup_msi_irqs = pnv_setup_msi_irqs,
3053 .teardown_msi_irqs = pnv_teardown_msi_irqs,
3054#endif
3055 .enable_device_hook = pnv_pci_enable_device_hook,
3056 .window_alignment = pnv_pci_window_alignment,
3057 .reset_secondary_bus = pnv_pci_reset_secondary_bus,
Daniel Axtens763d2d82015-04-28 15:12:07 +10003058 .dma_set_mask = pnv_pci_ioda_dma_set_mask,
Michael Neuling7a8e6bb2015-05-27 16:06:59 +10003059 .shutdown = pnv_pci_ioda_shutdown,
Daniel Axtens92ae0352015-04-28 15:12:05 +10003060};
3061
Anton Blancharde51df2c2014-08-20 08:55:18 +10003062static void __init pnv_pci_init_ioda_phb(struct device_node *np,
3063 u64 hub_id, int ioda_type)
Benjamin Herrenschmidt184cd4a2011-11-15 17:29:08 +00003064{
3065 struct pci_controller *hose;
Benjamin Herrenschmidt184cd4a2011-11-15 17:29:08 +00003066 struct pnv_phb *phb;
Gavin Shan81846162013-12-26 09:29:40 +08003067 unsigned long size, m32map_off, pemap_off, iomap_off = 0;
Alistair Popplec681b932013-09-23 12:04:57 +10003068 const __be64 *prop64;
Benjamin Herrenschmidt3a1a4662013-09-23 12:05:01 +10003069 const __be32 *prop32;
Gavin Shanf1b7cc32013-07-31 16:47:01 +08003070 int len;
Benjamin Herrenschmidt184cd4a2011-11-15 17:29:08 +00003071 u64 phb_id;
3072 void *aux;
3073 long rc;
3074
Gavin Shan58d714e2013-07-31 16:47:00 +08003075 pr_info("Initializing IODA%d OPAL PHB %s\n", ioda_type, np->full_name);
Benjamin Herrenschmidt184cd4a2011-11-15 17:29:08 +00003076
3077 prop64 = of_get_property(np, "ibm,opal-phbid", NULL);
3078 if (!prop64) {
3079 pr_err(" Missing \"ibm,opal-phbid\" property !\n");
3080 return;
3081 }
3082 phb_id = be64_to_cpup(prop64);
3083 pr_debug(" PHB-ID : 0x%016llx\n", phb_id);
3084
Michael Ellermane39f223f2014-11-18 16:47:35 +11003085 phb = memblock_virt_alloc(sizeof(struct pnv_phb), 0);
Gavin Shan58d714e2013-07-31 16:47:00 +08003086
3087 /* Allocate PCI controller */
Gavin Shan58d714e2013-07-31 16:47:00 +08003088 phb->hose = hose = pcibios_alloc_controller(np);
3089 if (!phb->hose) {
3090 pr_err(" Can't allocate PCI controller for %s\n",
Benjamin Herrenschmidt184cd4a2011-11-15 17:29:08 +00003091 np->full_name);
Michael Ellermane39f223f2014-11-18 16:47:35 +11003092 memblock_free(__pa(phb), sizeof(struct pnv_phb));
Benjamin Herrenschmidt184cd4a2011-11-15 17:29:08 +00003093 return;
3094 }
3095
3096 spin_lock_init(&phb->lock);
Gavin Shanf1b7cc32013-07-31 16:47:01 +08003097 prop32 = of_get_property(np, "bus-range", &len);
3098 if (prop32 && len == 8) {
Benjamin Herrenschmidt3a1a4662013-09-23 12:05:01 +10003099 hose->first_busno = be32_to_cpu(prop32[0]);
3100 hose->last_busno = be32_to_cpu(prop32[1]);
Gavin Shanf1b7cc32013-07-31 16:47:01 +08003101 } else {
3102 pr_warn(" Broken <bus-range> on %s\n", np->full_name);
3103 hose->first_busno = 0;
3104 hose->last_busno = 0xff;
3105 }
Benjamin Herrenschmidt184cd4a2011-11-15 17:29:08 +00003106 hose->private_data = phb;
Gavin Shane9cc17d2013-06-20 13:21:14 +08003107 phb->hub_id = hub_id;
Benjamin Herrenschmidt184cd4a2011-11-15 17:29:08 +00003108 phb->opal_id = phb_id;
Gavin Shanaa0c0332013-04-25 19:20:57 +00003109 phb->type = ioda_type;
Wei Yang781a8682015-03-25 16:23:57 +08003110 mutex_init(&phb->ioda.pe_alloc_mutex);
Benjamin Herrenschmidt184cd4a2011-11-15 17:29:08 +00003111
Benjamin Herrenschmidtcee72d52011-11-29 18:22:53 +00003112 /* Detect specific models for error handling */
3113 if (of_device_is_compatible(np, "ibm,p7ioc-pciex"))
3114 phb->model = PNV_PHB_MODEL_P7IOC;
Benjamin Herrenschmidtf3d40c22013-05-04 14:24:32 +00003115 else if (of_device_is_compatible(np, "ibm,power8-pciex"))
Gavin Shanaa0c0332013-04-25 19:20:57 +00003116 phb->model = PNV_PHB_MODEL_PHB3;
Benjamin Herrenschmidtcee72d52011-11-29 18:22:53 +00003117 else
3118 phb->model = PNV_PHB_MODEL_UNKNOWN;
3119
Gavin Shanaa0c0332013-04-25 19:20:57 +00003120 /* Parse 32-bit and IO ranges (if any) */
Gavin Shan2f1ec022013-07-31 16:47:02 +08003121 pci_process_bridge_OF_ranges(hose, np, !hose->global_number);
Benjamin Herrenschmidt184cd4a2011-11-15 17:29:08 +00003122
Gavin Shanaa0c0332013-04-25 19:20:57 +00003123 /* Get registers */
Benjamin Herrenschmidt184cd4a2011-11-15 17:29:08 +00003124 phb->regs = of_iomap(np, 0);
3125 if (phb->regs == NULL)
3126 pr_err(" Failed to map registers !\n");
3127
Benjamin Herrenschmidt184cd4a2011-11-15 17:29:08 +00003128 /* Initialize more IODA stuff */
Gavin Shan36954dc2013-11-04 16:32:47 +08003129 phb->ioda.total_pe = 1;
Gavin Shanaa0c0332013-04-25 19:20:57 +00003130 prop32 = of_get_property(np, "ibm,opal-num-pes", NULL);
Gavin Shan36954dc2013-11-04 16:32:47 +08003131 if (prop32)
Benjamin Herrenschmidt3a1a4662013-09-23 12:05:01 +10003132 phb->ioda.total_pe = be32_to_cpup(prop32);
Gavin Shan36954dc2013-11-04 16:32:47 +08003133 prop32 = of_get_property(np, "ibm,opal-reserved-pe", NULL);
3134 if (prop32)
3135 phb->ioda.reserved_pe = be32_to_cpup(prop32);
Guo Chao262af552014-07-21 14:42:30 +10003136
3137 /* Parse 64-bit MMIO range */
3138 pnv_ioda_parse_m64_window(phb);
3139
Benjamin Herrenschmidt184cd4a2011-11-15 17:29:08 +00003140 phb->ioda.m32_size = resource_size(&hose->mem_resources[0]);
Gavin Shanaa0c0332013-04-25 19:20:57 +00003141 /* FW Has already off top 64k of M32 space (MSI space) */
Benjamin Herrenschmidt184cd4a2011-11-15 17:29:08 +00003142 phb->ioda.m32_size += 0x10000;
3143
3144 phb->ioda.m32_segsize = phb->ioda.m32_size / phb->ioda.total_pe;
Benjamin Herrenschmidt3fd47f02013-05-06 13:40:40 +10003145 phb->ioda.m32_pci_base = hose->mem_resources[0].start - hose->mem_offset[0];
Benjamin Herrenschmidt184cd4a2011-11-15 17:29:08 +00003146 phb->ioda.io_size = hose->pci_io_size;
3147 phb->ioda.io_segsize = phb->ioda.io_size / phb->ioda.total_pe;
3148 phb->ioda.io_pci_base = 0; /* XXX calculate this ? */
3149
Gavin Shanc35d2a82013-07-31 16:47:04 +08003150 /* Allocate aux data & arrays. We don't have IO ports on PHB3 */
Benjamin Herrenschmidt184cd4a2011-11-15 17:29:08 +00003151 size = _ALIGN_UP(phb->ioda.total_pe / 8, sizeof(unsigned long));
3152 m32map_off = size;
Gavin Shane47747f2012-08-20 03:49:19 +00003153 size += phb->ioda.total_pe * sizeof(phb->ioda.m32_segmap[0]);
Gavin Shanc35d2a82013-07-31 16:47:04 +08003154 if (phb->type == PNV_PHB_IODA1) {
3155 iomap_off = size;
3156 size += phb->ioda.total_pe * sizeof(phb->ioda.io_segmap[0]);
3157 }
Benjamin Herrenschmidt184cd4a2011-11-15 17:29:08 +00003158 pemap_off = size;
3159 size += phb->ioda.total_pe * sizeof(struct pnv_ioda_pe);
Michael Ellermane39f223f2014-11-18 16:47:35 +11003160 aux = memblock_virt_alloc(size, 0);
Benjamin Herrenschmidt184cd4a2011-11-15 17:29:08 +00003161 phb->ioda.pe_alloc = aux;
3162 phb->ioda.m32_segmap = aux + m32map_off;
Gavin Shanc35d2a82013-07-31 16:47:04 +08003163 if (phb->type == PNV_PHB_IODA1)
3164 phb->ioda.io_segmap = aux + iomap_off;
Benjamin Herrenschmidt184cd4a2011-11-15 17:29:08 +00003165 phb->ioda.pe_array = aux + pemap_off;
Gavin Shan36954dc2013-11-04 16:32:47 +08003166 set_bit(phb->ioda.reserved_pe, phb->ioda.pe_alloc);
Benjamin Herrenschmidt184cd4a2011-11-15 17:29:08 +00003167
Gavin Shan7ebdf952012-08-20 03:49:15 +00003168 INIT_LIST_HEAD(&phb->ioda.pe_dma_list);
Benjamin Herrenschmidt184cd4a2011-11-15 17:29:08 +00003169 INIT_LIST_HEAD(&phb->ioda.pe_list);
Wei Yang781a8682015-03-25 16:23:57 +08003170 mutex_init(&phb->ioda.pe_list_mutex);
Benjamin Herrenschmidt184cd4a2011-11-15 17:29:08 +00003171
3172 /* Calculate how many 32-bit TCE segments we have */
3173 phb->ioda.tce32_count = phb->ioda.m32_pci_base >> 28;
3174
Gavin Shanaa0c0332013-04-25 19:20:57 +00003175#if 0 /* We should really do that ... */
Benjamin Herrenschmidt184cd4a2011-11-15 17:29:08 +00003176 rc = opal_pci_set_phb_mem_window(opal->phb_id,
3177 window_type,
3178 window_num,
3179 starting_real_address,
3180 starting_pci_address,
3181 segment_size);
3182#endif
3183
Guo Chao262af552014-07-21 14:42:30 +10003184 pr_info(" %03d (%03d) PE's M32: 0x%x [segment=0x%x]\n",
3185 phb->ioda.total_pe, phb->ioda.reserved_pe,
3186 phb->ioda.m32_size, phb->ioda.m32_segsize);
3187 if (phb->ioda.m64_size)
3188 pr_info(" M64: 0x%lx [segment=0x%lx]\n",
3189 phb->ioda.m64_size, phb->ioda.m64_segsize);
3190 if (phb->ioda.io_size)
3191 pr_info(" IO: 0x%x [segment=0x%x]\n",
3192 phb->ioda.io_size, phb->ioda.io_segsize);
3193
Benjamin Herrenschmidt184cd4a2011-11-15 17:29:08 +00003194
Benjamin Herrenschmidt184cd4a2011-11-15 17:29:08 +00003195 phb->hose->ops = &pnv_pci_ops;
Gavin Shan49dec922014-07-21 14:42:33 +10003196 phb->get_pe_state = pnv_ioda_get_pe_state;
3197 phb->freeze_pe = pnv_ioda_freeze_pe;
3198 phb->unfreeze_pe = pnv_ioda_unfreeze_pe;
Benjamin Herrenschmidt184cd4a2011-11-15 17:29:08 +00003199
3200 /* Setup RID -> PE mapping function */
3201 phb->bdfn_to_pe = pnv_ioda_bdfn_to_pe;
3202
3203 /* Setup TCEs */
3204 phb->dma_dev_setup = pnv_pci_ioda_dma_dev_setup;
Gavin Shanfe7e85c2014-09-30 12:39:10 +10003205 phb->dma_get_required_mask = pnv_pci_ioda_dma_get_required_mask;
Benjamin Herrenschmidt184cd4a2011-11-15 17:29:08 +00003206
3207 /* Setup MSI support */
3208 pnv_pci_init_ioda_msis(phb);
3209
Gavin Shanc40a4212012-08-20 03:49:20 +00003210 /*
3211 * We pass the PCI probe flag PCI_REASSIGN_ALL_RSRC here
3212 * to let the PCI core do resource assignment. It's supposed
3213 * that the PCI core will do correct I/O and MMIO alignment
3214 * for the P2P bridge bars so that each PCI bus (excluding
3215 * the child P2P bridges) can form individual PE.
Benjamin Herrenschmidt184cd4a2011-11-15 17:29:08 +00003216 */
Gavin Shanfb446ad2012-08-20 03:49:14 +00003217 ppc_md.pcibios_fixup = pnv_pci_ioda_fixup;
Daniel Axtens92ae0352015-04-28 15:12:05 +10003218 hose->controller_ops = pnv_pci_ioda_controller_ops;
Michael Ellermanad30cb92015-04-14 09:29:23 +10003219
Wei Yang6e628c72015-03-25 16:23:55 +08003220#ifdef CONFIG_PCI_IOV
3221 ppc_md.pcibios_fixup_sriov = pnv_pci_ioda_fixup_iov_resources;
Wei Yang5350ab32015-03-25 16:23:56 +08003222 ppc_md.pcibios_iov_resource_alignment = pnv_pci_iov_resource_alignment;
Michael Ellermanad30cb92015-04-14 09:29:23 +10003223#endif
3224
Gavin Shanc40a4212012-08-20 03:49:20 +00003225 pci_add_flags(PCI_REASSIGN_ALL_RSRC);
Benjamin Herrenschmidt184cd4a2011-11-15 17:29:08 +00003226
3227 /* Reset IODA tables to a clean state */
Gavin Shand1a85ee2014-09-30 12:39:05 +10003228 rc = opal_pci_reset(phb_id, OPAL_RESET_PCI_IODA_TABLE, OPAL_ASSERT_RESET);
Benjamin Herrenschmidt184cd4a2011-11-15 17:29:08 +00003229 if (rc)
Benjamin Herrenschmidtf11fe552011-11-29 18:22:50 +00003230 pr_warning(" OPAL Error %ld performing IODA table reset !\n", rc);
Gavin Shan361f2a22014-04-24 18:00:25 +10003231
3232 /* If we're running in kdump kerenl, the previous kerenl never
3233 * shutdown PCI devices correctly. We already got IODA table
3234 * cleaned out. So we have to issue PHB reset to stop all PCI
3235 * transactions from previous kerenl.
3236 */
3237 if (is_kdump_kernel()) {
3238 pr_info(" Issue PHB reset ...\n");
Gavin Shancadf3642015-02-16 14:45:47 +11003239 pnv_eeh_phb_reset(hose, EEH_RESET_FUNDAMENTAL);
3240 pnv_eeh_phb_reset(hose, EEH_RESET_DEACTIVATE);
Gavin Shan361f2a22014-04-24 18:00:25 +10003241 }
Guo Chao262af552014-07-21 14:42:30 +10003242
Gavin Shan9e9e8932014-11-12 13:36:05 +11003243 /* Remove M64 resource if we can't configure it successfully */
3244 if (!phb->init_m64 || phb->init_m64(phb))
Guo Chao262af552014-07-21 14:42:30 +10003245 hose->mem_resources[1].flags = 0;
Gavin Shanaa0c0332013-04-25 19:20:57 +00003246}
3247
Bjorn Helgaas67975002013-07-02 12:20:03 -06003248void __init pnv_pci_init_ioda2_phb(struct device_node *np)
Gavin Shanaa0c0332013-04-25 19:20:57 +00003249{
Gavin Shane9cc17d2013-06-20 13:21:14 +08003250 pnv_pci_init_ioda_phb(np, 0, PNV_PHB_IODA2);
Benjamin Herrenschmidt184cd4a2011-11-15 17:29:08 +00003251}
3252
3253void __init pnv_pci_init_ioda_hub(struct device_node *np)
3254{
3255 struct device_node *phbn;
Alistair Popplec681b932013-09-23 12:04:57 +10003256 const __be64 *prop64;
Benjamin Herrenschmidt184cd4a2011-11-15 17:29:08 +00003257 u64 hub_id;
3258
3259 pr_info("Probing IODA IO-Hub %s\n", np->full_name);
3260
3261 prop64 = of_get_property(np, "ibm,opal-hubid", NULL);
3262 if (!prop64) {
3263 pr_err(" Missing \"ibm,opal-hubid\" property !\n");
3264 return;
3265 }
3266 hub_id = be64_to_cpup(prop64);
3267 pr_devel(" HUB-ID : 0x%016llx\n", hub_id);
3268
3269 /* Count child PHBs */
3270 for_each_child_of_node(np, phbn) {
3271 /* Look for IODA1 PHBs */
3272 if (of_device_is_compatible(phbn, "ibm,ioda-phb"))
Gavin Shane9cc17d2013-06-20 13:21:14 +08003273 pnv_pci_init_ioda_phb(phbn, hub_id, PNV_PHB_IODA1);
Benjamin Herrenschmidt184cd4a2011-11-15 17:29:08 +00003274 }
3275}