David S. Miller | 8f6a93a | 2006-02-09 21:32:07 -0800 | [diff] [blame] | 1 | /* pci_sun4v.c: SUN4V specific PCI controller support. |
| 2 | * |
David S. Miller | d284142 | 2008-02-08 18:05:46 -0800 | [diff] [blame] | 3 | * Copyright (C) 2006, 2007, 2008 David S. Miller (davem@davemloft.net) |
David S. Miller | 8f6a93a | 2006-02-09 21:32:07 -0800 | [diff] [blame] | 4 | */ |
| 5 | |
| 6 | #include <linux/kernel.h> |
| 7 | #include <linux/types.h> |
| 8 | #include <linux/pci.h> |
| 9 | #include <linux/init.h> |
| 10 | #include <linux/slab.h> |
| 11 | #include <linux/interrupt.h> |
David S. Miller | 1839794 | 2006-02-10 00:08:26 -0800 | [diff] [blame] | 12 | #include <linux/percpu.h> |
David S. Miller | 35a17eb | 2007-02-10 17:41:02 -0800 | [diff] [blame] | 13 | #include <linux/irq.h> |
| 14 | #include <linux/msi.h> |
Paul Gortmaker | 7b64db6 | 2011-07-18 15:57:46 -0400 | [diff] [blame] | 15 | #include <linux/export.h> |
David S. Miller | 59db810 | 2007-05-23 18:00:46 -0700 | [diff] [blame] | 16 | #include <linux/log2.h> |
David S. Miller | 3822b50 | 2008-08-30 02:50:29 -0700 | [diff] [blame] | 17 | #include <linux/of_device.h> |
Sowmini Varadhan | bb620c3 | 2015-04-09 15:33:31 -0400 | [diff] [blame] | 18 | #include <linux/iommu-common.h> |
David S. Miller | 8f6a93a | 2006-02-09 21:32:07 -0800 | [diff] [blame] | 19 | |
David S. Miller | 8f6a93a | 2006-02-09 21:32:07 -0800 | [diff] [blame] | 20 | #include <asm/iommu.h> |
| 21 | #include <asm/irq.h> |
David S. Miller | 8f6a93a | 2006-02-09 21:32:07 -0800 | [diff] [blame] | 22 | #include <asm/hypervisor.h> |
David S. Miller | e87dc35 | 2006-06-21 18:18:47 -0700 | [diff] [blame] | 23 | #include <asm/prom.h> |
David S. Miller | 8f6a93a | 2006-02-09 21:32:07 -0800 | [diff] [blame] | 24 | |
| 25 | #include "pci_impl.h" |
| 26 | #include "iommu_common.h" |
| 27 | |
David S. Miller | bade562 | 2006-02-09 22:05:54 -0800 | [diff] [blame] | 28 | #include "pci_sun4v.h" |
| 29 | |
David S. Miller | 3822b50 | 2008-08-30 02:50:29 -0700 | [diff] [blame] | 30 | #define DRIVER_NAME "pci_sun4v" |
| 31 | #define PFX DRIVER_NAME ": " |
| 32 | |
chris hyser | 8914391 | 2016-09-28 12:19:45 -0700 | [diff] [blame] | 33 | static unsigned long vpci_major; |
| 34 | static unsigned long vpci_minor; |
| 35 | |
| 36 | struct vpci_version { |
| 37 | unsigned long major; |
| 38 | unsigned long minor; |
| 39 | }; |
| 40 | |
| 41 | /* Ordered from largest major to lowest */ |
| 42 | static struct vpci_version vpci_versions[] = { |
| 43 | { .major = 2, .minor = 0 }, |
| 44 | { .major = 1, .minor = 1 }, |
| 45 | }; |
David S. Miller | e01c0d6 | 2007-05-25 01:04:15 -0700 | [diff] [blame] | 46 | |
Tushar Dave | f0248c1 | 2016-10-28 10:12:41 -0700 | [diff] [blame] | 47 | static unsigned long vatu_major = 1; |
| 48 | static unsigned long vatu_minor = 1; |
| 49 | |
David S. Miller | 7c8f486 | 2006-02-13 21:50:27 -0800 | [diff] [blame] | 50 | #define PGLIST_NENTS (PAGE_SIZE / sizeof(u64)) |
David S. Miller | 1839794 | 2006-02-10 00:08:26 -0800 | [diff] [blame] | 51 | |
David S. Miller | 16ce82d | 2007-04-26 21:08:21 -0700 | [diff] [blame] | 52 | struct iommu_batch { |
David S. Miller | ad7ad57 | 2007-07-27 22:39:14 -0700 | [diff] [blame] | 53 | struct device *dev; /* Device mapping is for. */ |
David S. Miller | 6a32fd4 | 2006-02-19 22:21:32 -0800 | [diff] [blame] | 54 | unsigned long prot; /* IOMMU page protections */ |
| 55 | unsigned long entry; /* Index into IOTSB. */ |
| 56 | u64 *pglist; /* List of physical pages */ |
| 57 | unsigned long npages; /* Number of pages in list. */ |
David S. Miller | 1839794 | 2006-02-10 00:08:26 -0800 | [diff] [blame] | 58 | }; |
| 59 | |
David S. Miller | ad7ad57 | 2007-07-27 22:39:14 -0700 | [diff] [blame] | 60 | static DEFINE_PER_CPU(struct iommu_batch, iommu_batch); |
David S. Miller | d3ae4b5 | 2008-09-09 23:54:02 -0700 | [diff] [blame] | 61 | static int iommu_batch_initialized; |
David S. Miller | 6a32fd4 | 2006-02-19 22:21:32 -0800 | [diff] [blame] | 62 | |
| 63 | /* Interrupts must be disabled. */ |
David S. Miller | ad7ad57 | 2007-07-27 22:39:14 -0700 | [diff] [blame] | 64 | static inline void iommu_batch_start(struct device *dev, unsigned long prot, unsigned long entry) |
David S. Miller | 6a32fd4 | 2006-02-19 22:21:32 -0800 | [diff] [blame] | 65 | { |
Christoph Lameter | 494fc42 | 2014-08-17 12:30:54 -0500 | [diff] [blame] | 66 | struct iommu_batch *p = this_cpu_ptr(&iommu_batch); |
David S. Miller | 6a32fd4 | 2006-02-19 22:21:32 -0800 | [diff] [blame] | 67 | |
David S. Miller | ad7ad57 | 2007-07-27 22:39:14 -0700 | [diff] [blame] | 68 | p->dev = dev; |
David S. Miller | 6a32fd4 | 2006-02-19 22:21:32 -0800 | [diff] [blame] | 69 | p->prot = prot; |
| 70 | p->entry = entry; |
| 71 | p->npages = 0; |
| 72 | } |
| 73 | |
| 74 | /* Interrupts must be disabled. */ |
Tushar Dave | f08978b | 2016-10-28 10:12:44 -0700 | [diff] [blame] | 75 | static long iommu_batch_flush(struct iommu_batch *p, u64 mask) |
David S. Miller | 6a32fd4 | 2006-02-19 22:21:32 -0800 | [diff] [blame] | 76 | { |
David S. Miller | ad7ad57 | 2007-07-27 22:39:14 -0700 | [diff] [blame] | 77 | struct pci_pbm_info *pbm = p->dev->archdata.host_controller; |
Tushar Dave | f08978b | 2016-10-28 10:12:44 -0700 | [diff] [blame] | 78 | u64 *pglist = p->pglist; |
| 79 | u64 index_count; |
David S. Miller | a2fb23a | 2007-02-28 23:35:04 -0800 | [diff] [blame] | 80 | unsigned long devhandle = pbm->devhandle; |
David S. Miller | 6a32fd4 | 2006-02-19 22:21:32 -0800 | [diff] [blame] | 81 | unsigned long prot = p->prot; |
| 82 | unsigned long entry = p->entry; |
David S. Miller | 6a32fd4 | 2006-02-19 22:21:32 -0800 | [diff] [blame] | 83 | unsigned long npages = p->npages; |
Tushar Dave | f08978b | 2016-10-28 10:12:44 -0700 | [diff] [blame] | 84 | unsigned long iotsb_num; |
| 85 | unsigned long ret; |
| 86 | long num; |
David S. Miller | 6a32fd4 | 2006-02-19 22:21:32 -0800 | [diff] [blame] | 87 | |
chris hyser | aa7bde1 | 2016-09-28 12:19:50 -0700 | [diff] [blame] | 88 | /* VPCI maj=1, min=[0,1] only supports read and write */ |
| 89 | if (vpci_major < 2) |
| 90 | prot &= (HV_PCI_MAP_ATTR_READ | HV_PCI_MAP_ATTR_WRITE); |
| 91 | |
David S. Miller | d82965c | 2006-02-20 01:42:51 -0800 | [diff] [blame] | 92 | while (npages != 0) { |
Tushar Dave | f08978b | 2016-10-28 10:12:44 -0700 | [diff] [blame] | 93 | if (mask <= DMA_BIT_MASK(32)) { |
| 94 | num = pci_sun4v_iommu_map(devhandle, |
| 95 | HV_PCI_TSBID(0, entry), |
| 96 | npages, |
| 97 | prot, |
| 98 | __pa(pglist)); |
| 99 | if (unlikely(num < 0)) { |
| 100 | pr_err_ratelimited("%s: IOMMU map of [%08lx:%08llx:%lx:%lx:%lx] failed with status %ld\n", |
| 101 | __func__, |
| 102 | devhandle, |
| 103 | HV_PCI_TSBID(0, entry), |
| 104 | npages, prot, __pa(pglist), |
| 105 | num); |
| 106 | return -1; |
| 107 | } |
| 108 | } else { |
| 109 | index_count = HV_PCI_IOTSB_INDEX_COUNT(npages, entry), |
| 110 | iotsb_num = pbm->iommu->atu->iotsb->iotsb_num; |
| 111 | ret = pci_sun4v_iotsb_map(devhandle, |
| 112 | iotsb_num, |
| 113 | index_count, |
| 114 | prot, |
| 115 | __pa(pglist), |
| 116 | &num); |
| 117 | if (unlikely(ret != HV_EOK)) { |
| 118 | pr_err_ratelimited("%s: ATU map of [%08lx:%lx:%llx:%lx:%lx] failed with status %ld\n", |
| 119 | __func__, |
| 120 | devhandle, iotsb_num, |
| 121 | index_count, prot, |
| 122 | __pa(pglist), ret); |
| 123 | return -1; |
| 124 | } |
David S. Miller | 6a32fd4 | 2006-02-19 22:21:32 -0800 | [diff] [blame] | 125 | } |
David S. Miller | 6a32fd4 | 2006-02-19 22:21:32 -0800 | [diff] [blame] | 126 | entry += num; |
| 127 | npages -= num; |
| 128 | pglist += num; |
David S. Miller | d82965c | 2006-02-20 01:42:51 -0800 | [diff] [blame] | 129 | } |
David S. Miller | 6a32fd4 | 2006-02-19 22:21:32 -0800 | [diff] [blame] | 130 | |
| 131 | p->entry = entry; |
| 132 | p->npages = 0; |
| 133 | |
| 134 | return 0; |
| 135 | } |
| 136 | |
Tushar Dave | f08978b | 2016-10-28 10:12:44 -0700 | [diff] [blame] | 137 | static inline void iommu_batch_new_entry(unsigned long entry, u64 mask) |
David S. Miller | 13fa14e | 2008-02-09 03:11:01 -0800 | [diff] [blame] | 138 | { |
Christoph Lameter | 494fc42 | 2014-08-17 12:30:54 -0500 | [diff] [blame] | 139 | struct iommu_batch *p = this_cpu_ptr(&iommu_batch); |
David S. Miller | 13fa14e | 2008-02-09 03:11:01 -0800 | [diff] [blame] | 140 | |
| 141 | if (p->entry + p->npages == entry) |
| 142 | return; |
| 143 | if (p->entry != ~0UL) |
Tushar Dave | f08978b | 2016-10-28 10:12:44 -0700 | [diff] [blame] | 144 | iommu_batch_flush(p, mask); |
David S. Miller | 13fa14e | 2008-02-09 03:11:01 -0800 | [diff] [blame] | 145 | p->entry = entry; |
| 146 | } |
| 147 | |
David S. Miller | 6a32fd4 | 2006-02-19 22:21:32 -0800 | [diff] [blame] | 148 | /* Interrupts must be disabled. */ |
Tushar Dave | f08978b | 2016-10-28 10:12:44 -0700 | [diff] [blame] | 149 | static inline long iommu_batch_add(u64 phys_page, u64 mask) |
David S. Miller | 6a32fd4 | 2006-02-19 22:21:32 -0800 | [diff] [blame] | 150 | { |
Christoph Lameter | 494fc42 | 2014-08-17 12:30:54 -0500 | [diff] [blame] | 151 | struct iommu_batch *p = this_cpu_ptr(&iommu_batch); |
David S. Miller | 6a32fd4 | 2006-02-19 22:21:32 -0800 | [diff] [blame] | 152 | |
| 153 | BUG_ON(p->npages >= PGLIST_NENTS); |
| 154 | |
| 155 | p->pglist[p->npages++] = phys_page; |
| 156 | if (p->npages == PGLIST_NENTS) |
Tushar Dave | f08978b | 2016-10-28 10:12:44 -0700 | [diff] [blame] | 157 | return iommu_batch_flush(p, mask); |
David S. Miller | 6a32fd4 | 2006-02-19 22:21:32 -0800 | [diff] [blame] | 158 | |
| 159 | return 0; |
| 160 | } |
| 161 | |
| 162 | /* Interrupts must be disabled. */ |
Tushar Dave | f08978b | 2016-10-28 10:12:44 -0700 | [diff] [blame] | 163 | static inline long iommu_batch_end(u64 mask) |
David S. Miller | 6a32fd4 | 2006-02-19 22:21:32 -0800 | [diff] [blame] | 164 | { |
Christoph Lameter | 494fc42 | 2014-08-17 12:30:54 -0500 | [diff] [blame] | 165 | struct iommu_batch *p = this_cpu_ptr(&iommu_batch); |
David S. Miller | 6a32fd4 | 2006-02-19 22:21:32 -0800 | [diff] [blame] | 166 | |
| 167 | BUG_ON(p->npages >= PGLIST_NENTS); |
| 168 | |
Tushar Dave | f08978b | 2016-10-28 10:12:44 -0700 | [diff] [blame] | 169 | return iommu_batch_flush(p, mask); |
David S. Miller | 6a32fd4 | 2006-02-19 22:21:32 -0800 | [diff] [blame] | 170 | } |
David S. Miller | 1839794 | 2006-02-10 00:08:26 -0800 | [diff] [blame] | 171 | |
David S. Miller | ad7ad57 | 2007-07-27 22:39:14 -0700 | [diff] [blame] | 172 | static void *dma_4v_alloc_coherent(struct device *dev, size_t size, |
Andrzej Pietrasiewicz | c416258 | 2012-03-27 14:56:55 +0200 | [diff] [blame] | 173 | dma_addr_t *dma_addrp, gfp_t gfp, |
Krzysztof Kozlowski | 00085f1 | 2016-08-03 13:46:00 -0700 | [diff] [blame] | 174 | unsigned long attrs) |
David S. Miller | 8f6a93a | 2006-02-09 21:32:07 -0800 | [diff] [blame] | 175 | { |
Tushar Dave | f08978b | 2016-10-28 10:12:44 -0700 | [diff] [blame] | 176 | u64 mask; |
David S. Miller | 7c8f486 | 2006-02-13 21:50:27 -0800 | [diff] [blame] | 177 | unsigned long flags, order, first_page, npages, n; |
chris hyser | aa7bde1 | 2016-09-28 12:19:50 -0700 | [diff] [blame] | 178 | unsigned long prot = 0; |
David S. Miller | c1b1a5f | 2008-03-19 04:52:48 -0700 | [diff] [blame] | 179 | struct iommu *iommu; |
Tushar Dave | f08978b | 2016-10-28 10:12:44 -0700 | [diff] [blame] | 180 | struct atu *atu; |
| 181 | struct iommu_map_table *tbl; |
David S. Miller | c1b1a5f | 2008-03-19 04:52:48 -0700 | [diff] [blame] | 182 | struct page *page; |
David S. Miller | 1839794 | 2006-02-10 00:08:26 -0800 | [diff] [blame] | 183 | void *ret; |
| 184 | long entry; |
David S. Miller | c1b1a5f | 2008-03-19 04:52:48 -0700 | [diff] [blame] | 185 | int nid; |
David S. Miller | 1839794 | 2006-02-10 00:08:26 -0800 | [diff] [blame] | 186 | |
| 187 | size = IO_PAGE_ALIGN(size); |
| 188 | order = get_order(size); |
David S. Miller | 6a32fd4 | 2006-02-19 22:21:32 -0800 | [diff] [blame] | 189 | if (unlikely(order >= MAX_ORDER)) |
David S. Miller | 1839794 | 2006-02-10 00:08:26 -0800 | [diff] [blame] | 190 | return NULL; |
| 191 | |
| 192 | npages = size >> IO_PAGE_SHIFT; |
David S. Miller | 1839794 | 2006-02-10 00:08:26 -0800 | [diff] [blame] | 193 | |
chris hyser | aa7bde1 | 2016-09-28 12:19:50 -0700 | [diff] [blame] | 194 | if (attrs & DMA_ATTR_WEAK_ORDERING) |
| 195 | prot = HV_PCI_MAP_ATTR_RELAXED_ORDER; |
| 196 | |
David S. Miller | c1b1a5f | 2008-03-19 04:52:48 -0700 | [diff] [blame] | 197 | nid = dev->archdata.numa_node; |
| 198 | page = alloc_pages_node(nid, gfp, order); |
| 199 | if (unlikely(!page)) |
David S. Miller | 1839794 | 2006-02-10 00:08:26 -0800 | [diff] [blame] | 200 | return NULL; |
David S. Miller | e7a0453 | 2006-02-15 22:25:27 -0800 | [diff] [blame] | 201 | |
David S. Miller | c1b1a5f | 2008-03-19 04:52:48 -0700 | [diff] [blame] | 202 | first_page = (unsigned long) page_address(page); |
David S. Miller | 1839794 | 2006-02-10 00:08:26 -0800 | [diff] [blame] | 203 | memset((char *)first_page, 0, PAGE_SIZE << order); |
| 204 | |
David S. Miller | ad7ad57 | 2007-07-27 22:39:14 -0700 | [diff] [blame] | 205 | iommu = dev->archdata.iommu; |
Tushar Dave | f08978b | 2016-10-28 10:12:44 -0700 | [diff] [blame] | 206 | atu = iommu->atu; |
David S. Miller | 1839794 | 2006-02-10 00:08:26 -0800 | [diff] [blame] | 207 | |
Tushar Dave | f08978b | 2016-10-28 10:12:44 -0700 | [diff] [blame] | 208 | mask = dev->coherent_dma_mask; |
| 209 | if (mask <= DMA_BIT_MASK(32)) |
| 210 | tbl = &iommu->tbl; |
| 211 | else |
| 212 | tbl = &atu->tbl; |
| 213 | |
| 214 | entry = iommu_tbl_range_alloc(dev, tbl, npages, NULL, |
Sowmini Varadhan | bb620c3 | 2015-04-09 15:33:31 -0400 | [diff] [blame] | 215 | (unsigned long)(-1), 0); |
David S. Miller | 1839794 | 2006-02-10 00:08:26 -0800 | [diff] [blame] | 216 | |
David S. Miller | d618382 | 2015-11-04 11:30:57 -0800 | [diff] [blame] | 217 | if (unlikely(entry == IOMMU_ERROR_CODE)) |
David S. Miller | d284142 | 2008-02-08 18:05:46 -0800 | [diff] [blame] | 218 | goto range_alloc_fail; |
David S. Miller | 1839794 | 2006-02-10 00:08:26 -0800 | [diff] [blame] | 219 | |
Tushar Dave | f08978b | 2016-10-28 10:12:44 -0700 | [diff] [blame] | 220 | *dma_addrp = (tbl->table_map_base + (entry << IO_PAGE_SHIFT)); |
David S. Miller | 1839794 | 2006-02-10 00:08:26 -0800 | [diff] [blame] | 221 | ret = (void *) first_page; |
| 222 | first_page = __pa(first_page); |
| 223 | |
David S. Miller | 6a32fd4 | 2006-02-19 22:21:32 -0800 | [diff] [blame] | 224 | local_irq_save(flags); |
David S. Miller | 1839794 | 2006-02-10 00:08:26 -0800 | [diff] [blame] | 225 | |
David S. Miller | ad7ad57 | 2007-07-27 22:39:14 -0700 | [diff] [blame] | 226 | iommu_batch_start(dev, |
chris hyser | aa7bde1 | 2016-09-28 12:19:50 -0700 | [diff] [blame] | 227 | (HV_PCI_MAP_ATTR_READ | prot | |
David S. Miller | ad7ad57 | 2007-07-27 22:39:14 -0700 | [diff] [blame] | 228 | HV_PCI_MAP_ATTR_WRITE), |
| 229 | entry); |
David S. Miller | 1839794 | 2006-02-10 00:08:26 -0800 | [diff] [blame] | 230 | |
David S. Miller | 6a32fd4 | 2006-02-19 22:21:32 -0800 | [diff] [blame] | 231 | for (n = 0; n < npages; n++) { |
Tushar Dave | f08978b | 2016-10-28 10:12:44 -0700 | [diff] [blame] | 232 | long err = iommu_batch_add(first_page + (n * PAGE_SIZE), mask); |
David S. Miller | 6a32fd4 | 2006-02-19 22:21:32 -0800 | [diff] [blame] | 233 | if (unlikely(err < 0L)) |
| 234 | goto iommu_map_fail; |
| 235 | } |
David S. Miller | 1839794 | 2006-02-10 00:08:26 -0800 | [diff] [blame] | 236 | |
Tushar Dave | f08978b | 2016-10-28 10:12:44 -0700 | [diff] [blame] | 237 | if (unlikely(iommu_batch_end(mask) < 0L)) |
David S. Miller | 6a32fd4 | 2006-02-19 22:21:32 -0800 | [diff] [blame] | 238 | goto iommu_map_fail; |
David S. Miller | 1839794 | 2006-02-10 00:08:26 -0800 | [diff] [blame] | 239 | |
David S. Miller | 6a32fd4 | 2006-02-19 22:21:32 -0800 | [diff] [blame] | 240 | local_irq_restore(flags); |
David S. Miller | 1839794 | 2006-02-10 00:08:26 -0800 | [diff] [blame] | 241 | |
| 242 | return ret; |
David S. Miller | 6a32fd4 | 2006-02-19 22:21:32 -0800 | [diff] [blame] | 243 | |
| 244 | iommu_map_fail: |
Tushar Dave | f08978b | 2016-10-28 10:12:44 -0700 | [diff] [blame] | 245 | iommu_tbl_range_free(tbl, *dma_addrp, npages, IOMMU_ERROR_CODE); |
David S. Miller | 6a32fd4 | 2006-02-19 22:21:32 -0800 | [diff] [blame] | 246 | |
David S. Miller | d284142 | 2008-02-08 18:05:46 -0800 | [diff] [blame] | 247 | range_alloc_fail: |
David S. Miller | 6a32fd4 | 2006-02-19 22:21:32 -0800 | [diff] [blame] | 248 | free_pages(first_page, order); |
| 249 | return NULL; |
David S. Miller | 8f6a93a | 2006-02-09 21:32:07 -0800 | [diff] [blame] | 250 | } |
| 251 | |
Tushar Dave | 5116ab4 | 2016-10-28 10:12:43 -0700 | [diff] [blame] | 252 | unsigned long dma_4v_iotsb_bind(unsigned long devhandle, |
| 253 | unsigned long iotsb_num, |
| 254 | struct pci_bus *bus_dev) |
| 255 | { |
| 256 | struct pci_dev *pdev; |
| 257 | unsigned long err; |
| 258 | unsigned int bus; |
| 259 | unsigned int device; |
| 260 | unsigned int fun; |
| 261 | |
| 262 | list_for_each_entry(pdev, &bus_dev->devices, bus_list) { |
| 263 | if (pdev->subordinate) { |
| 264 | /* No need to bind pci bridge */ |
| 265 | dma_4v_iotsb_bind(devhandle, iotsb_num, |
| 266 | pdev->subordinate); |
| 267 | } else { |
| 268 | bus = bus_dev->number; |
| 269 | device = PCI_SLOT(pdev->devfn); |
| 270 | fun = PCI_FUNC(pdev->devfn); |
| 271 | err = pci_sun4v_iotsb_bind(devhandle, iotsb_num, |
| 272 | HV_PCI_DEVICE_BUILD(bus, |
| 273 | device, |
| 274 | fun)); |
| 275 | |
| 276 | /* If bind fails for one device it is going to fail |
| 277 | * for rest of the devices because we are sharing |
| 278 | * IOTSB. So in case of failure simply return with |
| 279 | * error. |
| 280 | */ |
| 281 | if (err) |
| 282 | return err; |
| 283 | } |
| 284 | } |
| 285 | |
| 286 | return 0; |
| 287 | } |
| 288 | |
Tushar Dave | f08978b | 2016-10-28 10:12:44 -0700 | [diff] [blame] | 289 | static void dma_4v_iommu_demap(struct device *dev, unsigned long devhandle, |
| 290 | dma_addr_t dvma, unsigned long iotsb_num, |
| 291 | unsigned long entry, unsigned long npages) |
Sowmini Varadhan | bb620c3 | 2015-04-09 15:33:31 -0400 | [diff] [blame] | 292 | { |
Sowmini Varadhan | bb620c3 | 2015-04-09 15:33:31 -0400 | [diff] [blame] | 293 | unsigned long num, flags; |
Tushar Dave | f08978b | 2016-10-28 10:12:44 -0700 | [diff] [blame] | 294 | unsigned long ret; |
Sowmini Varadhan | bb620c3 | 2015-04-09 15:33:31 -0400 | [diff] [blame] | 295 | |
| 296 | local_irq_save(flags); |
| 297 | do { |
Tushar Dave | f08978b | 2016-10-28 10:12:44 -0700 | [diff] [blame] | 298 | if (dvma <= DMA_BIT_MASK(32)) { |
| 299 | num = pci_sun4v_iommu_demap(devhandle, |
| 300 | HV_PCI_TSBID(0, entry), |
| 301 | npages); |
| 302 | } else { |
| 303 | ret = pci_sun4v_iotsb_demap(devhandle, iotsb_num, |
| 304 | entry, npages, &num); |
| 305 | if (unlikely(ret != HV_EOK)) { |
| 306 | pr_err_ratelimited("pci_iotsb_demap() failed with error: %ld\n", |
| 307 | ret); |
| 308 | } |
| 309 | } |
Sowmini Varadhan | bb620c3 | 2015-04-09 15:33:31 -0400 | [diff] [blame] | 310 | entry += num; |
| 311 | npages -= num; |
| 312 | } while (npages != 0); |
| 313 | local_irq_restore(flags); |
| 314 | } |
| 315 | |
David S. Miller | ad7ad57 | 2007-07-27 22:39:14 -0700 | [diff] [blame] | 316 | static void dma_4v_free_coherent(struct device *dev, size_t size, void *cpu, |
Krzysztof Kozlowski | 00085f1 | 2016-08-03 13:46:00 -0700 | [diff] [blame] | 317 | dma_addr_t dvma, unsigned long attrs) |
David S. Miller | 8f6a93a | 2006-02-09 21:32:07 -0800 | [diff] [blame] | 318 | { |
David S. Miller | a2fb23a | 2007-02-28 23:35:04 -0800 | [diff] [blame] | 319 | struct pci_pbm_info *pbm; |
David S. Miller | 16ce82d | 2007-04-26 21:08:21 -0700 | [diff] [blame] | 320 | struct iommu *iommu; |
Tushar Dave | f08978b | 2016-10-28 10:12:44 -0700 | [diff] [blame] | 321 | struct atu *atu; |
| 322 | struct iommu_map_table *tbl; |
Sowmini Varadhan | bb620c3 | 2015-04-09 15:33:31 -0400 | [diff] [blame] | 323 | unsigned long order, npages, entry; |
Tushar Dave | f08978b | 2016-10-28 10:12:44 -0700 | [diff] [blame] | 324 | unsigned long iotsb_num; |
David S. Miller | 7c8f486 | 2006-02-13 21:50:27 -0800 | [diff] [blame] | 325 | u32 devhandle; |
David S. Miller | 1839794 | 2006-02-10 00:08:26 -0800 | [diff] [blame] | 326 | |
| 327 | npages = IO_PAGE_ALIGN(size) >> IO_PAGE_SHIFT; |
David S. Miller | ad7ad57 | 2007-07-27 22:39:14 -0700 | [diff] [blame] | 328 | iommu = dev->archdata.iommu; |
| 329 | pbm = dev->archdata.host_controller; |
Tushar Dave | f08978b | 2016-10-28 10:12:44 -0700 | [diff] [blame] | 330 | atu = iommu->atu; |
David S. Miller | a2fb23a | 2007-02-28 23:35:04 -0800 | [diff] [blame] | 331 | devhandle = pbm->devhandle; |
Tushar Dave | f08978b | 2016-10-28 10:12:44 -0700 | [diff] [blame] | 332 | |
| 333 | if (dvma <= DMA_BIT_MASK(32)) { |
| 334 | tbl = &iommu->tbl; |
| 335 | iotsb_num = 0; /* we don't care for legacy iommu */ |
| 336 | } else { |
| 337 | tbl = &atu->tbl; |
| 338 | iotsb_num = atu->iotsb->iotsb_num; |
| 339 | } |
| 340 | entry = ((dvma - tbl->table_map_base) >> IO_PAGE_SHIFT); |
| 341 | dma_4v_iommu_demap(dev, devhandle, dvma, iotsb_num, entry, npages); |
| 342 | iommu_tbl_range_free(tbl, dvma, npages, IOMMU_ERROR_CODE); |
David S. Miller | 1839794 | 2006-02-10 00:08:26 -0800 | [diff] [blame] | 343 | order = get_order(size); |
| 344 | if (order < 10) |
| 345 | free_pages((unsigned long)cpu, order); |
David S. Miller | 8f6a93a | 2006-02-09 21:32:07 -0800 | [diff] [blame] | 346 | } |
| 347 | |
FUJITA Tomonori | 797a756 | 2009-05-14 16:23:10 +0000 | [diff] [blame] | 348 | static dma_addr_t dma_4v_map_page(struct device *dev, struct page *page, |
| 349 | unsigned long offset, size_t sz, |
FUJITA Tomonori | bc0a14f | 2009-08-10 11:53:12 +0900 | [diff] [blame] | 350 | enum dma_data_direction direction, |
Krzysztof Kozlowski | 00085f1 | 2016-08-03 13:46:00 -0700 | [diff] [blame] | 351 | unsigned long attrs) |
David S. Miller | 8f6a93a | 2006-02-09 21:32:07 -0800 | [diff] [blame] | 352 | { |
David S. Miller | 16ce82d | 2007-04-26 21:08:21 -0700 | [diff] [blame] | 353 | struct iommu *iommu; |
Tushar Dave | f08978b | 2016-10-28 10:12:44 -0700 | [diff] [blame] | 354 | struct atu *atu; |
| 355 | struct iommu_map_table *tbl; |
| 356 | u64 mask; |
David S. Miller | 1839794 | 2006-02-10 00:08:26 -0800 | [diff] [blame] | 357 | unsigned long flags, npages, oaddr; |
David S. Miller | 7c8f486 | 2006-02-13 21:50:27 -0800 | [diff] [blame] | 358 | unsigned long i, base_paddr; |
David S. Miller | 1839794 | 2006-02-10 00:08:26 -0800 | [diff] [blame] | 359 | unsigned long prot; |
Tushar Dave | f08978b | 2016-10-28 10:12:44 -0700 | [diff] [blame] | 360 | dma_addr_t bus_addr, ret; |
David S. Miller | 1839794 | 2006-02-10 00:08:26 -0800 | [diff] [blame] | 361 | long entry; |
David S. Miller | 1839794 | 2006-02-10 00:08:26 -0800 | [diff] [blame] | 362 | |
David S. Miller | ad7ad57 | 2007-07-27 22:39:14 -0700 | [diff] [blame] | 363 | iommu = dev->archdata.iommu; |
Tushar Dave | f08978b | 2016-10-28 10:12:44 -0700 | [diff] [blame] | 364 | atu = iommu->atu; |
David S. Miller | 1839794 | 2006-02-10 00:08:26 -0800 | [diff] [blame] | 365 | |
David S. Miller | ad7ad57 | 2007-07-27 22:39:14 -0700 | [diff] [blame] | 366 | if (unlikely(direction == DMA_NONE)) |
David S. Miller | 1839794 | 2006-02-10 00:08:26 -0800 | [diff] [blame] | 367 | goto bad; |
| 368 | |
FUJITA Tomonori | 797a756 | 2009-05-14 16:23:10 +0000 | [diff] [blame] | 369 | oaddr = (unsigned long)(page_address(page) + offset); |
David S. Miller | 1839794 | 2006-02-10 00:08:26 -0800 | [diff] [blame] | 370 | npages = IO_PAGE_ALIGN(oaddr + sz) - (oaddr & IO_PAGE_MASK); |
| 371 | npages >>= IO_PAGE_SHIFT; |
David S. Miller | 1839794 | 2006-02-10 00:08:26 -0800 | [diff] [blame] | 372 | |
Tushar Dave | f08978b | 2016-10-28 10:12:44 -0700 | [diff] [blame] | 373 | mask = *dev->dma_mask; |
| 374 | if (mask <= DMA_BIT_MASK(32)) |
| 375 | tbl = &iommu->tbl; |
| 376 | else |
| 377 | tbl = &atu->tbl; |
| 378 | |
| 379 | entry = iommu_tbl_range_alloc(dev, tbl, npages, NULL, |
Sowmini Varadhan | bb620c3 | 2015-04-09 15:33:31 -0400 | [diff] [blame] | 380 | (unsigned long)(-1), 0); |
David S. Miller | 1839794 | 2006-02-10 00:08:26 -0800 | [diff] [blame] | 381 | |
David S. Miller | d618382 | 2015-11-04 11:30:57 -0800 | [diff] [blame] | 382 | if (unlikely(entry == IOMMU_ERROR_CODE)) |
David S. Miller | 1839794 | 2006-02-10 00:08:26 -0800 | [diff] [blame] | 383 | goto bad; |
| 384 | |
Tushar Dave | f08978b | 2016-10-28 10:12:44 -0700 | [diff] [blame] | 385 | bus_addr = (tbl->table_map_base + (entry << IO_PAGE_SHIFT)); |
David S. Miller | 1839794 | 2006-02-10 00:08:26 -0800 | [diff] [blame] | 386 | ret = bus_addr | (oaddr & ~IO_PAGE_MASK); |
| 387 | base_paddr = __pa(oaddr & IO_PAGE_MASK); |
| 388 | prot = HV_PCI_MAP_ATTR_READ; |
David S. Miller | ad7ad57 | 2007-07-27 22:39:14 -0700 | [diff] [blame] | 389 | if (direction != DMA_TO_DEVICE) |
David S. Miller | 1839794 | 2006-02-10 00:08:26 -0800 | [diff] [blame] | 390 | prot |= HV_PCI_MAP_ATTR_WRITE; |
| 391 | |
chris hyser | aa7bde1 | 2016-09-28 12:19:50 -0700 | [diff] [blame] | 392 | if (attrs & DMA_ATTR_WEAK_ORDERING) |
| 393 | prot |= HV_PCI_MAP_ATTR_RELAXED_ORDER; |
| 394 | |
David S. Miller | 6a32fd4 | 2006-02-19 22:21:32 -0800 | [diff] [blame] | 395 | local_irq_save(flags); |
David S. Miller | 1839794 | 2006-02-10 00:08:26 -0800 | [diff] [blame] | 396 | |
David S. Miller | ad7ad57 | 2007-07-27 22:39:14 -0700 | [diff] [blame] | 397 | iommu_batch_start(dev, prot, entry); |
David S. Miller | 1839794 | 2006-02-10 00:08:26 -0800 | [diff] [blame] | 398 | |
David S. Miller | 6a32fd4 | 2006-02-19 22:21:32 -0800 | [diff] [blame] | 399 | for (i = 0; i < npages; i++, base_paddr += IO_PAGE_SIZE) { |
Tushar Dave | f08978b | 2016-10-28 10:12:44 -0700 | [diff] [blame] | 400 | long err = iommu_batch_add(base_paddr, mask); |
David S. Miller | 6a32fd4 | 2006-02-19 22:21:32 -0800 | [diff] [blame] | 401 | if (unlikely(err < 0L)) |
| 402 | goto iommu_map_fail; |
| 403 | } |
Tushar Dave | f08978b | 2016-10-28 10:12:44 -0700 | [diff] [blame] | 404 | if (unlikely(iommu_batch_end(mask) < 0L)) |
David S. Miller | 6a32fd4 | 2006-02-19 22:21:32 -0800 | [diff] [blame] | 405 | goto iommu_map_fail; |
David S. Miller | 1839794 | 2006-02-10 00:08:26 -0800 | [diff] [blame] | 406 | |
David S. Miller | 6a32fd4 | 2006-02-19 22:21:32 -0800 | [diff] [blame] | 407 | local_irq_restore(flags); |
David S. Miller | 1839794 | 2006-02-10 00:08:26 -0800 | [diff] [blame] | 408 | |
| 409 | return ret; |
| 410 | |
| 411 | bad: |
| 412 | if (printk_ratelimit()) |
| 413 | WARN_ON(1); |
David S. Miller | ad7ad57 | 2007-07-27 22:39:14 -0700 | [diff] [blame] | 414 | return DMA_ERROR_CODE; |
David S. Miller | 6a32fd4 | 2006-02-19 22:21:32 -0800 | [diff] [blame] | 415 | |
| 416 | iommu_map_fail: |
Tushar Dave | f08978b | 2016-10-28 10:12:44 -0700 | [diff] [blame] | 417 | iommu_tbl_range_free(tbl, bus_addr, npages, IOMMU_ERROR_CODE); |
David S. Miller | ad7ad57 | 2007-07-27 22:39:14 -0700 | [diff] [blame] | 418 | return DMA_ERROR_CODE; |
David S. Miller | 8f6a93a | 2006-02-09 21:32:07 -0800 | [diff] [blame] | 419 | } |
| 420 | |
FUJITA Tomonori | 797a756 | 2009-05-14 16:23:10 +0000 | [diff] [blame] | 421 | static void dma_4v_unmap_page(struct device *dev, dma_addr_t bus_addr, |
FUJITA Tomonori | bc0a14f | 2009-08-10 11:53:12 +0900 | [diff] [blame] | 422 | size_t sz, enum dma_data_direction direction, |
Krzysztof Kozlowski | 00085f1 | 2016-08-03 13:46:00 -0700 | [diff] [blame] | 423 | unsigned long attrs) |
David S. Miller | 8f6a93a | 2006-02-09 21:32:07 -0800 | [diff] [blame] | 424 | { |
David S. Miller | a2fb23a | 2007-02-28 23:35:04 -0800 | [diff] [blame] | 425 | struct pci_pbm_info *pbm; |
David S. Miller | 16ce82d | 2007-04-26 21:08:21 -0700 | [diff] [blame] | 426 | struct iommu *iommu; |
Tushar Dave | f08978b | 2016-10-28 10:12:44 -0700 | [diff] [blame] | 427 | struct atu *atu; |
| 428 | struct iommu_map_table *tbl; |
Sowmini Varadhan | bb620c3 | 2015-04-09 15:33:31 -0400 | [diff] [blame] | 429 | unsigned long npages; |
Tushar Dave | f08978b | 2016-10-28 10:12:44 -0700 | [diff] [blame] | 430 | unsigned long iotsb_num; |
David S. Miller | 1839794 | 2006-02-10 00:08:26 -0800 | [diff] [blame] | 431 | long entry; |
David S. Miller | 7c8f486 | 2006-02-13 21:50:27 -0800 | [diff] [blame] | 432 | u32 devhandle; |
David S. Miller | 1839794 | 2006-02-10 00:08:26 -0800 | [diff] [blame] | 433 | |
David S. Miller | ad7ad57 | 2007-07-27 22:39:14 -0700 | [diff] [blame] | 434 | if (unlikely(direction == DMA_NONE)) { |
David S. Miller | 1839794 | 2006-02-10 00:08:26 -0800 | [diff] [blame] | 435 | if (printk_ratelimit()) |
| 436 | WARN_ON(1); |
| 437 | return; |
| 438 | } |
| 439 | |
David S. Miller | ad7ad57 | 2007-07-27 22:39:14 -0700 | [diff] [blame] | 440 | iommu = dev->archdata.iommu; |
| 441 | pbm = dev->archdata.host_controller; |
Tushar Dave | f08978b | 2016-10-28 10:12:44 -0700 | [diff] [blame] | 442 | atu = iommu->atu; |
David S. Miller | a2fb23a | 2007-02-28 23:35:04 -0800 | [diff] [blame] | 443 | devhandle = pbm->devhandle; |
David S. Miller | 1839794 | 2006-02-10 00:08:26 -0800 | [diff] [blame] | 444 | |
| 445 | npages = IO_PAGE_ALIGN(bus_addr + sz) - (bus_addr & IO_PAGE_MASK); |
| 446 | npages >>= IO_PAGE_SHIFT; |
| 447 | bus_addr &= IO_PAGE_MASK; |
Tushar Dave | f08978b | 2016-10-28 10:12:44 -0700 | [diff] [blame] | 448 | |
| 449 | if (bus_addr <= DMA_BIT_MASK(32)) { |
| 450 | iotsb_num = 0; /* we don't care for legacy iommu */ |
| 451 | tbl = &iommu->tbl; |
| 452 | } else { |
| 453 | iotsb_num = atu->iotsb->iotsb_num; |
| 454 | tbl = &atu->tbl; |
| 455 | } |
| 456 | entry = (bus_addr - tbl->table_map_base) >> IO_PAGE_SHIFT; |
| 457 | dma_4v_iommu_demap(dev, devhandle, bus_addr, iotsb_num, entry, npages); |
| 458 | iommu_tbl_range_free(tbl, bus_addr, npages, IOMMU_ERROR_CODE); |
David S. Miller | 1839794 | 2006-02-10 00:08:26 -0800 | [diff] [blame] | 459 | } |
| 460 | |
David S. Miller | ad7ad57 | 2007-07-27 22:39:14 -0700 | [diff] [blame] | 461 | static int dma_4v_map_sg(struct device *dev, struct scatterlist *sglist, |
FUJITA Tomonori | bc0a14f | 2009-08-10 11:53:12 +0900 | [diff] [blame] | 462 | int nelems, enum dma_data_direction direction, |
Krzysztof Kozlowski | 00085f1 | 2016-08-03 13:46:00 -0700 | [diff] [blame] | 463 | unsigned long attrs) |
David S. Miller | 8f6a93a | 2006-02-09 21:32:07 -0800 | [diff] [blame] | 464 | { |
David S. Miller | 13fa14e | 2008-02-09 03:11:01 -0800 | [diff] [blame] | 465 | struct scatterlist *s, *outs, *segstart; |
| 466 | unsigned long flags, handle, prot; |
| 467 | dma_addr_t dma_next = 0, dma_addr; |
| 468 | unsigned int max_seg_size; |
FUJITA Tomonori | f088025 | 2008-03-28 15:55:41 -0700 | [diff] [blame] | 469 | unsigned long seg_boundary_size; |
David S. Miller | 13fa14e | 2008-02-09 03:11:01 -0800 | [diff] [blame] | 470 | int outcount, incount, i; |
David S. Miller | 16ce82d | 2007-04-26 21:08:21 -0700 | [diff] [blame] | 471 | struct iommu *iommu; |
Tushar Dave | f08978b | 2016-10-28 10:12:44 -0700 | [diff] [blame] | 472 | struct atu *atu; |
| 473 | struct iommu_map_table *tbl; |
| 474 | u64 mask; |
FUJITA Tomonori | f088025 | 2008-03-28 15:55:41 -0700 | [diff] [blame] | 475 | unsigned long base_shift; |
David S. Miller | 13fa14e | 2008-02-09 03:11:01 -0800 | [diff] [blame] | 476 | long err; |
David S. Miller | 1839794 | 2006-02-10 00:08:26 -0800 | [diff] [blame] | 477 | |
David S. Miller | 13fa14e | 2008-02-09 03:11:01 -0800 | [diff] [blame] | 478 | BUG_ON(direction == DMA_NONE); |
David S. Miller | 1839794 | 2006-02-10 00:08:26 -0800 | [diff] [blame] | 479 | |
David S. Miller | ad7ad57 | 2007-07-27 22:39:14 -0700 | [diff] [blame] | 480 | iommu = dev->archdata.iommu; |
Tushar Dave | f08978b | 2016-10-28 10:12:44 -0700 | [diff] [blame] | 481 | atu = iommu->atu; |
| 482 | |
David S. Miller | 13fa14e | 2008-02-09 03:11:01 -0800 | [diff] [blame] | 483 | if (nelems == 0 || !iommu) |
| 484 | return 0; |
David S. Miller | 1839794 | 2006-02-10 00:08:26 -0800 | [diff] [blame] | 485 | |
David S. Miller | 1839794 | 2006-02-10 00:08:26 -0800 | [diff] [blame] | 486 | prot = HV_PCI_MAP_ATTR_READ; |
David S. Miller | ad7ad57 | 2007-07-27 22:39:14 -0700 | [diff] [blame] | 487 | if (direction != DMA_TO_DEVICE) |
David S. Miller | 1839794 | 2006-02-10 00:08:26 -0800 | [diff] [blame] | 488 | prot |= HV_PCI_MAP_ATTR_WRITE; |
| 489 | |
chris hyser | aa7bde1 | 2016-09-28 12:19:50 -0700 | [diff] [blame] | 490 | if (attrs & DMA_ATTR_WEAK_ORDERING) |
| 491 | prot |= HV_PCI_MAP_ATTR_RELAXED_ORDER; |
| 492 | |
David S. Miller | 13fa14e | 2008-02-09 03:11:01 -0800 | [diff] [blame] | 493 | outs = s = segstart = &sglist[0]; |
| 494 | outcount = 1; |
| 495 | incount = nelems; |
| 496 | handle = 0; |
David S. Miller | 38192d5 | 2008-02-06 03:50:26 -0800 | [diff] [blame] | 497 | |
David S. Miller | 13fa14e | 2008-02-09 03:11:01 -0800 | [diff] [blame] | 498 | /* Init first segment length for backout at failure */ |
| 499 | outs->dma_length = 0; |
David S. Miller | 38192d5 | 2008-02-06 03:50:26 -0800 | [diff] [blame] | 500 | |
Sowmini Varadhan | bb620c3 | 2015-04-09 15:33:31 -0400 | [diff] [blame] | 501 | local_irq_save(flags); |
David S. Miller | 38192d5 | 2008-02-06 03:50:26 -0800 | [diff] [blame] | 502 | |
David S. Miller | 13fa14e | 2008-02-09 03:11:01 -0800 | [diff] [blame] | 503 | iommu_batch_start(dev, prot, ~0UL); |
David S. Miller | 38192d5 | 2008-02-06 03:50:26 -0800 | [diff] [blame] | 504 | |
David S. Miller | 13fa14e | 2008-02-09 03:11:01 -0800 | [diff] [blame] | 505 | max_seg_size = dma_get_max_seg_size(dev); |
FUJITA Tomonori | f088025 | 2008-03-28 15:55:41 -0700 | [diff] [blame] | 506 | seg_boundary_size = ALIGN(dma_get_seg_boundary(dev) + 1, |
| 507 | IO_PAGE_SIZE) >> IO_PAGE_SHIFT; |
Tushar Dave | f08978b | 2016-10-28 10:12:44 -0700 | [diff] [blame] | 508 | |
| 509 | mask = *dev->dma_mask; |
| 510 | if (mask <= DMA_BIT_MASK(32)) |
| 511 | tbl = &iommu->tbl; |
| 512 | else |
| 513 | tbl = &atu->tbl; |
| 514 | |
| 515 | base_shift = tbl->table_map_base >> IO_PAGE_SHIFT; |
| 516 | |
David S. Miller | 13fa14e | 2008-02-09 03:11:01 -0800 | [diff] [blame] | 517 | for_each_sg(sglist, s, nelems, i) { |
FUJITA Tomonori | f088025 | 2008-03-28 15:55:41 -0700 | [diff] [blame] | 518 | unsigned long paddr, npages, entry, out_entry = 0, slen; |
David S. Miller | 38192d5 | 2008-02-06 03:50:26 -0800 | [diff] [blame] | 519 | |
David S. Miller | 13fa14e | 2008-02-09 03:11:01 -0800 | [diff] [blame] | 520 | slen = s->length; |
| 521 | /* Sanity check */ |
| 522 | if (slen == 0) { |
| 523 | dma_next = 0; |
| 524 | continue; |
David S. Miller | 38192d5 | 2008-02-06 03:50:26 -0800 | [diff] [blame] | 525 | } |
David S. Miller | 13fa14e | 2008-02-09 03:11:01 -0800 | [diff] [blame] | 526 | /* Allocate iommu entries for that segment */ |
| 527 | paddr = (unsigned long) SG_ENT_PHYS_ADDRESS(s); |
Joerg Roedel | 0fcff28 | 2008-10-15 22:02:14 -0700 | [diff] [blame] | 528 | npages = iommu_num_pages(paddr, slen, IO_PAGE_SIZE); |
Tushar Dave | f08978b | 2016-10-28 10:12:44 -0700 | [diff] [blame] | 529 | entry = iommu_tbl_range_alloc(dev, tbl, npages, |
Sowmini Varadhan | bb620c3 | 2015-04-09 15:33:31 -0400 | [diff] [blame] | 530 | &handle, (unsigned long)(-1), 0); |
David S. Miller | 13fa14e | 2008-02-09 03:11:01 -0800 | [diff] [blame] | 531 | |
| 532 | /* Handle failure */ |
David S. Miller | d618382 | 2015-11-04 11:30:57 -0800 | [diff] [blame] | 533 | if (unlikely(entry == IOMMU_ERROR_CODE)) { |
Tushar Dave | f08978b | 2016-10-28 10:12:44 -0700 | [diff] [blame] | 534 | pr_err_ratelimited("iommu_alloc failed, iommu %p paddr %lx npages %lx\n", |
| 535 | tbl, paddr, npages); |
David S. Miller | 13fa14e | 2008-02-09 03:11:01 -0800 | [diff] [blame] | 536 | goto iommu_map_failed; |
| 537 | } |
| 538 | |
Tushar Dave | f08978b | 2016-10-28 10:12:44 -0700 | [diff] [blame] | 539 | iommu_batch_new_entry(entry, mask); |
David S. Miller | 13fa14e | 2008-02-09 03:11:01 -0800 | [diff] [blame] | 540 | |
| 541 | /* Convert entry to a dma_addr_t */ |
Tushar Dave | f08978b | 2016-10-28 10:12:44 -0700 | [diff] [blame] | 542 | dma_addr = tbl->table_map_base + (entry << IO_PAGE_SHIFT); |
David S. Miller | 13fa14e | 2008-02-09 03:11:01 -0800 | [diff] [blame] | 543 | dma_addr |= (s->offset & ~IO_PAGE_MASK); |
| 544 | |
| 545 | /* Insert into HW table */ |
| 546 | paddr &= IO_PAGE_MASK; |
| 547 | while (npages--) { |
Tushar Dave | f08978b | 2016-10-28 10:12:44 -0700 | [diff] [blame] | 548 | err = iommu_batch_add(paddr, mask); |
David S. Miller | 13fa14e | 2008-02-09 03:11:01 -0800 | [diff] [blame] | 549 | if (unlikely(err < 0L)) |
| 550 | goto iommu_map_failed; |
| 551 | paddr += IO_PAGE_SIZE; |
| 552 | } |
| 553 | |
| 554 | /* If we are in an open segment, try merging */ |
| 555 | if (segstart != s) { |
| 556 | /* We cannot merge if: |
| 557 | * - allocated dma_addr isn't contiguous to previous allocation |
| 558 | */ |
| 559 | if ((dma_addr != dma_next) || |
FUJITA Tomonori | f088025 | 2008-03-28 15:55:41 -0700 | [diff] [blame] | 560 | (outs->dma_length + s->length > max_seg_size) || |
| 561 | (is_span_boundary(out_entry, base_shift, |
| 562 | seg_boundary_size, outs, s))) { |
David S. Miller | 13fa14e | 2008-02-09 03:11:01 -0800 | [diff] [blame] | 563 | /* Can't merge: create a new segment */ |
| 564 | segstart = s; |
| 565 | outcount++; |
| 566 | outs = sg_next(outs); |
| 567 | } else { |
| 568 | outs->dma_length += s->length; |
| 569 | } |
| 570 | } |
| 571 | |
| 572 | if (segstart == s) { |
| 573 | /* This is a new segment, fill entries */ |
| 574 | outs->dma_address = dma_addr; |
| 575 | outs->dma_length = slen; |
FUJITA Tomonori | f088025 | 2008-03-28 15:55:41 -0700 | [diff] [blame] | 576 | out_entry = entry; |
David S. Miller | 13fa14e | 2008-02-09 03:11:01 -0800 | [diff] [blame] | 577 | } |
| 578 | |
| 579 | /* Calculate next page pointer for contiguous check */ |
| 580 | dma_next = dma_addr + slen; |
David S. Miller | 38192d5 | 2008-02-06 03:50:26 -0800 | [diff] [blame] | 581 | } |
| 582 | |
Tushar Dave | f08978b | 2016-10-28 10:12:44 -0700 | [diff] [blame] | 583 | err = iommu_batch_end(mask); |
David S. Miller | 38192d5 | 2008-02-06 03:50:26 -0800 | [diff] [blame] | 584 | |
David S. Miller | 6a32fd4 | 2006-02-19 22:21:32 -0800 | [diff] [blame] | 585 | if (unlikely(err < 0L)) |
| 586 | goto iommu_map_failed; |
David S. Miller | 1839794 | 2006-02-10 00:08:26 -0800 | [diff] [blame] | 587 | |
Sowmini Varadhan | bb620c3 | 2015-04-09 15:33:31 -0400 | [diff] [blame] | 588 | local_irq_restore(flags); |
David S. Miller | 1839794 | 2006-02-10 00:08:26 -0800 | [diff] [blame] | 589 | |
David S. Miller | 13fa14e | 2008-02-09 03:11:01 -0800 | [diff] [blame] | 590 | if (outcount < incount) { |
| 591 | outs = sg_next(outs); |
| 592 | outs->dma_address = DMA_ERROR_CODE; |
| 593 | outs->dma_length = 0; |
| 594 | } |
| 595 | |
| 596 | return outcount; |
David S. Miller | 6a32fd4 | 2006-02-19 22:21:32 -0800 | [diff] [blame] | 597 | |
| 598 | iommu_map_failed: |
David S. Miller | 13fa14e | 2008-02-09 03:11:01 -0800 | [diff] [blame] | 599 | for_each_sg(sglist, s, nelems, i) { |
| 600 | if (s->dma_length != 0) { |
| 601 | unsigned long vaddr, npages; |
| 602 | |
| 603 | vaddr = s->dma_address & IO_PAGE_MASK; |
Joerg Roedel | 0fcff28 | 2008-10-15 22:02:14 -0700 | [diff] [blame] | 604 | npages = iommu_num_pages(s->dma_address, s->dma_length, |
| 605 | IO_PAGE_SIZE); |
Tushar Dave | f08978b | 2016-10-28 10:12:44 -0700 | [diff] [blame] | 606 | iommu_tbl_range_free(tbl, vaddr, npages, |
David S. Miller | d618382 | 2015-11-04 11:30:57 -0800 | [diff] [blame] | 607 | IOMMU_ERROR_CODE); |
David S. Miller | 13fa14e | 2008-02-09 03:11:01 -0800 | [diff] [blame] | 608 | /* XXX demap? XXX */ |
| 609 | s->dma_address = DMA_ERROR_CODE; |
| 610 | s->dma_length = 0; |
| 611 | } |
| 612 | if (s == outs) |
| 613 | break; |
| 614 | } |
Sowmini Varadhan | bb620c3 | 2015-04-09 15:33:31 -0400 | [diff] [blame] | 615 | local_irq_restore(flags); |
David S. Miller | 6a32fd4 | 2006-02-19 22:21:32 -0800 | [diff] [blame] | 616 | |
| 617 | return 0; |
David S. Miller | 8f6a93a | 2006-02-09 21:32:07 -0800 | [diff] [blame] | 618 | } |
| 619 | |
David S. Miller | ad7ad57 | 2007-07-27 22:39:14 -0700 | [diff] [blame] | 620 | static void dma_4v_unmap_sg(struct device *dev, struct scatterlist *sglist, |
FUJITA Tomonori | bc0a14f | 2009-08-10 11:53:12 +0900 | [diff] [blame] | 621 | int nelems, enum dma_data_direction direction, |
Krzysztof Kozlowski | 00085f1 | 2016-08-03 13:46:00 -0700 | [diff] [blame] | 622 | unsigned long attrs) |
David S. Miller | 8f6a93a | 2006-02-09 21:32:07 -0800 | [diff] [blame] | 623 | { |
David S. Miller | a2fb23a | 2007-02-28 23:35:04 -0800 | [diff] [blame] | 624 | struct pci_pbm_info *pbm; |
David S. Miller | 13fa14e | 2008-02-09 03:11:01 -0800 | [diff] [blame] | 625 | struct scatterlist *sg; |
David S. Miller | 38192d5 | 2008-02-06 03:50:26 -0800 | [diff] [blame] | 626 | struct iommu *iommu; |
Tushar Dave | f08978b | 2016-10-28 10:12:44 -0700 | [diff] [blame] | 627 | struct atu *atu; |
Sowmini Varadhan | bb620c3 | 2015-04-09 15:33:31 -0400 | [diff] [blame] | 628 | unsigned long flags, entry; |
Tushar Dave | f08978b | 2016-10-28 10:12:44 -0700 | [diff] [blame] | 629 | unsigned long iotsb_num; |
David S. Miller | 13fa14e | 2008-02-09 03:11:01 -0800 | [diff] [blame] | 630 | u32 devhandle; |
David S. Miller | 1839794 | 2006-02-10 00:08:26 -0800 | [diff] [blame] | 631 | |
David S. Miller | 13fa14e | 2008-02-09 03:11:01 -0800 | [diff] [blame] | 632 | BUG_ON(direction == DMA_NONE); |
David S. Miller | 1839794 | 2006-02-10 00:08:26 -0800 | [diff] [blame] | 633 | |
David S. Miller | ad7ad57 | 2007-07-27 22:39:14 -0700 | [diff] [blame] | 634 | iommu = dev->archdata.iommu; |
| 635 | pbm = dev->archdata.host_controller; |
Tushar Dave | f08978b | 2016-10-28 10:12:44 -0700 | [diff] [blame] | 636 | atu = iommu->atu; |
David S. Miller | a2fb23a | 2007-02-28 23:35:04 -0800 | [diff] [blame] | 637 | devhandle = pbm->devhandle; |
David S. Miller | 1839794 | 2006-02-10 00:08:26 -0800 | [diff] [blame] | 638 | |
Sowmini Varadhan | bb620c3 | 2015-04-09 15:33:31 -0400 | [diff] [blame] | 639 | local_irq_save(flags); |
David S. Miller | 1839794 | 2006-02-10 00:08:26 -0800 | [diff] [blame] | 640 | |
David S. Miller | 13fa14e | 2008-02-09 03:11:01 -0800 | [diff] [blame] | 641 | sg = sglist; |
| 642 | while (nelems--) { |
| 643 | dma_addr_t dma_handle = sg->dma_address; |
| 644 | unsigned int len = sg->dma_length; |
Sowmini Varadhan | bb620c3 | 2015-04-09 15:33:31 -0400 | [diff] [blame] | 645 | unsigned long npages; |
Tushar Dave | f08978b | 2016-10-28 10:12:44 -0700 | [diff] [blame] | 646 | struct iommu_map_table *tbl; |
Sowmini Varadhan | bb620c3 | 2015-04-09 15:33:31 -0400 | [diff] [blame] | 647 | unsigned long shift = IO_PAGE_SHIFT; |
David S. Miller | 1839794 | 2006-02-10 00:08:26 -0800 | [diff] [blame] | 648 | |
David S. Miller | 13fa14e | 2008-02-09 03:11:01 -0800 | [diff] [blame] | 649 | if (!len) |
| 650 | break; |
Joerg Roedel | 0fcff28 | 2008-10-15 22:02:14 -0700 | [diff] [blame] | 651 | npages = iommu_num_pages(dma_handle, len, IO_PAGE_SIZE); |
Tushar Dave | f08978b | 2016-10-28 10:12:44 -0700 | [diff] [blame] | 652 | |
| 653 | if (dma_handle <= DMA_BIT_MASK(32)) { |
| 654 | iotsb_num = 0; /* we don't care for legacy iommu */ |
| 655 | tbl = &iommu->tbl; |
| 656 | } else { |
| 657 | iotsb_num = atu->iotsb->iotsb_num; |
| 658 | tbl = &atu->tbl; |
| 659 | } |
Sowmini Varadhan | bb620c3 | 2015-04-09 15:33:31 -0400 | [diff] [blame] | 660 | entry = ((dma_handle - tbl->table_map_base) >> shift); |
Tushar Dave | f08978b | 2016-10-28 10:12:44 -0700 | [diff] [blame] | 661 | dma_4v_iommu_demap(dev, devhandle, dma_handle, iotsb_num, |
| 662 | entry, npages); |
| 663 | iommu_tbl_range_free(tbl, dma_handle, npages, |
David S. Miller | d618382 | 2015-11-04 11:30:57 -0800 | [diff] [blame] | 664 | IOMMU_ERROR_CODE); |
David S. Miller | 13fa14e | 2008-02-09 03:11:01 -0800 | [diff] [blame] | 665 | sg = sg_next(sg); |
| 666 | } |
David S. Miller | 1839794 | 2006-02-10 00:08:26 -0800 | [diff] [blame] | 667 | |
Sowmini Varadhan | bb620c3 | 2015-04-09 15:33:31 -0400 | [diff] [blame] | 668 | local_irq_restore(flags); |
David S. Miller | 8f6a93a | 2006-02-09 21:32:07 -0800 | [diff] [blame] | 669 | } |
| 670 | |
FUJITA Tomonori | 02f7a18 | 2009-08-10 11:53:13 +0900 | [diff] [blame] | 671 | static struct dma_map_ops sun4v_dma_ops = { |
Andrzej Pietrasiewicz | c416258 | 2012-03-27 14:56:55 +0200 | [diff] [blame] | 672 | .alloc = dma_4v_alloc_coherent, |
| 673 | .free = dma_4v_free_coherent, |
FUJITA Tomonori | 797a756 | 2009-05-14 16:23:10 +0000 | [diff] [blame] | 674 | .map_page = dma_4v_map_page, |
| 675 | .unmap_page = dma_4v_unmap_page, |
David S. Miller | ad7ad57 | 2007-07-27 22:39:14 -0700 | [diff] [blame] | 676 | .map_sg = dma_4v_map_sg, |
| 677 | .unmap_sg = dma_4v_unmap_sg, |
David S. Miller | 8f6a93a | 2006-02-09 21:32:07 -0800 | [diff] [blame] | 678 | }; |
| 679 | |
Greg Kroah-Hartman | 7c9503b | 2012-12-21 14:03:26 -0800 | [diff] [blame] | 680 | static void pci_sun4v_scan_bus(struct pci_pbm_info *pbm, struct device *parent) |
David S. Miller | bade562 | 2006-02-09 22:05:54 -0800 | [diff] [blame] | 681 | { |
David S. Miller | e87dc35 | 2006-06-21 18:18:47 -0700 | [diff] [blame] | 682 | struct property *prop; |
| 683 | struct device_node *dp; |
| 684 | |
Grant Likely | 61c7a08 | 2010-04-13 16:12:29 -0700 | [diff] [blame] | 685 | dp = pbm->op->dev.of_node; |
David S. Miller | 34768bc | 2007-05-07 23:06:27 -0700 | [diff] [blame] | 686 | prop = of_find_property(dp, "66mhz-capable", NULL); |
| 687 | pbm->is_66mhz_capable = (prop != NULL); |
David S. Miller | e822358a | 2008-09-01 18:32:22 -0700 | [diff] [blame] | 688 | pbm->pci_bus = pci_scan_one_pbm(pbm, parent); |
David S. Miller | c260926 | 2006-02-12 22:18:52 -0800 | [diff] [blame] | 689 | |
| 690 | /* XXX register error interrupt handlers XXX */ |
David S. Miller | bade562 | 2006-02-09 22:05:54 -0800 | [diff] [blame] | 691 | } |
| 692 | |
Greg Kroah-Hartman | 7c9503b | 2012-12-21 14:03:26 -0800 | [diff] [blame] | 693 | static unsigned long probe_existing_entries(struct pci_pbm_info *pbm, |
Sowmini Varadhan | bb620c3 | 2015-04-09 15:33:31 -0400 | [diff] [blame] | 694 | struct iommu_map_table *iommu) |
David S. Miller | 1839794 | 2006-02-10 00:08:26 -0800 | [diff] [blame] | 695 | { |
Sowmini Varadhan | bb620c3 | 2015-04-09 15:33:31 -0400 | [diff] [blame] | 696 | struct iommu_pool *pool; |
| 697 | unsigned long i, pool_nr, cnt = 0; |
David S. Miller | 7c8f486 | 2006-02-13 21:50:27 -0800 | [diff] [blame] | 698 | u32 devhandle; |
David S. Miller | 1839794 | 2006-02-10 00:08:26 -0800 | [diff] [blame] | 699 | |
| 700 | devhandle = pbm->devhandle; |
Sowmini Varadhan | bb620c3 | 2015-04-09 15:33:31 -0400 | [diff] [blame] | 701 | for (pool_nr = 0; pool_nr < iommu->nr_pools; pool_nr++) { |
| 702 | pool = &(iommu->pools[pool_nr]); |
| 703 | for (i = pool->start; i <= pool->end; i++) { |
| 704 | unsigned long ret, io_attrs, ra; |
David S. Miller | 1839794 | 2006-02-10 00:08:26 -0800 | [diff] [blame] | 705 | |
Sowmini Varadhan | bb620c3 | 2015-04-09 15:33:31 -0400 | [diff] [blame] | 706 | ret = pci_sun4v_iommu_getmap(devhandle, |
| 707 | HV_PCI_TSBID(0, i), |
| 708 | &io_attrs, &ra); |
| 709 | if (ret == HV_EOK) { |
| 710 | if (page_in_phys_avail(ra)) { |
| 711 | pci_sun4v_iommu_demap(devhandle, |
| 712 | HV_PCI_TSBID(0, |
| 713 | i), 1); |
| 714 | } else { |
| 715 | cnt++; |
| 716 | __set_bit(i, iommu->map); |
| 717 | } |
David S. Miller | c2a5a46 | 2006-06-22 00:01:56 -0700 | [diff] [blame] | 718 | } |
David S. Miller | e7a0453 | 2006-02-15 22:25:27 -0800 | [diff] [blame] | 719 | } |
David S. Miller | 1839794 | 2006-02-10 00:08:26 -0800 | [diff] [blame] | 720 | } |
David S. Miller | e7a0453 | 2006-02-15 22:25:27 -0800 | [diff] [blame] | 721 | return cnt; |
David S. Miller | 1839794 | 2006-02-10 00:08:26 -0800 | [diff] [blame] | 722 | } |
| 723 | |
Tushar Dave | f0248c1 | 2016-10-28 10:12:41 -0700 | [diff] [blame] | 724 | static int pci_sun4v_atu_alloc_iotsb(struct pci_pbm_info *pbm) |
| 725 | { |
| 726 | struct atu *atu = pbm->iommu->atu; |
| 727 | struct atu_iotsb *iotsb; |
| 728 | void *table; |
| 729 | u64 table_size; |
| 730 | u64 iotsb_num; |
| 731 | unsigned long order; |
| 732 | unsigned long err; |
| 733 | |
| 734 | iotsb = kzalloc(sizeof(*iotsb), GFP_KERNEL); |
| 735 | if (!iotsb) { |
| 736 | err = -ENOMEM; |
| 737 | goto out_err; |
| 738 | } |
| 739 | atu->iotsb = iotsb; |
| 740 | |
| 741 | /* calculate size of IOTSB */ |
| 742 | table_size = (atu->size / IO_PAGE_SIZE) * 8; |
| 743 | order = get_order(table_size); |
| 744 | table = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, order); |
| 745 | if (!table) { |
| 746 | err = -ENOMEM; |
| 747 | goto table_failed; |
| 748 | } |
| 749 | iotsb->table = table; |
| 750 | iotsb->ra = __pa(table); |
| 751 | iotsb->dvma_size = atu->size; |
| 752 | iotsb->dvma_base = atu->base; |
| 753 | iotsb->table_size = table_size; |
| 754 | iotsb->page_size = IO_PAGE_SIZE; |
| 755 | |
| 756 | /* configure and register IOTSB with HV */ |
| 757 | err = pci_sun4v_iotsb_conf(pbm->devhandle, |
| 758 | iotsb->ra, |
| 759 | iotsb->table_size, |
| 760 | iotsb->page_size, |
| 761 | iotsb->dvma_base, |
| 762 | &iotsb_num); |
| 763 | if (err) { |
| 764 | pr_err(PFX "pci_iotsb_conf failed error: %ld\n", err); |
| 765 | goto iotsb_conf_failed; |
| 766 | } |
| 767 | iotsb->iotsb_num = iotsb_num; |
| 768 | |
Tushar Dave | 5116ab4 | 2016-10-28 10:12:43 -0700 | [diff] [blame] | 769 | err = dma_4v_iotsb_bind(pbm->devhandle, iotsb_num, pbm->pci_bus); |
| 770 | if (err) { |
| 771 | pr_err(PFX "pci_iotsb_bind failed error: %ld\n", err); |
| 772 | goto iotsb_conf_failed; |
| 773 | } |
| 774 | |
Tushar Dave | f0248c1 | 2016-10-28 10:12:41 -0700 | [diff] [blame] | 775 | return 0; |
| 776 | |
| 777 | iotsb_conf_failed: |
| 778 | free_pages((unsigned long)table, order); |
| 779 | table_failed: |
| 780 | kfree(iotsb); |
| 781 | out_err: |
| 782 | return err; |
| 783 | } |
| 784 | |
| 785 | static int pci_sun4v_atu_init(struct pci_pbm_info *pbm) |
| 786 | { |
| 787 | struct atu *atu = pbm->iommu->atu; |
| 788 | unsigned long err; |
| 789 | const u64 *ranges; |
Tushar Dave | 31f077d | 2016-10-28 10:12:42 -0700 | [diff] [blame] | 790 | u64 map_size, num_iotte; |
| 791 | u64 dma_mask; |
Tushar Dave | f0248c1 | 2016-10-28 10:12:41 -0700 | [diff] [blame] | 792 | const u32 *page_size; |
| 793 | int len; |
| 794 | |
| 795 | ranges = of_get_property(pbm->op->dev.of_node, "iommu-address-ranges", |
| 796 | &len); |
| 797 | if (!ranges) { |
| 798 | pr_err(PFX "No iommu-address-ranges\n"); |
| 799 | return -EINVAL; |
| 800 | } |
| 801 | |
| 802 | page_size = of_get_property(pbm->op->dev.of_node, "iommu-pagesizes", |
| 803 | NULL); |
| 804 | if (!page_size) { |
| 805 | pr_err(PFX "No iommu-pagesizes\n"); |
| 806 | return -EINVAL; |
| 807 | } |
| 808 | |
| 809 | /* There are 4 iommu-address-ranges supported. Each range is pair of |
| 810 | * {base, size}. The ranges[0] and ranges[1] are 32bit address space |
| 811 | * while ranges[2] and ranges[3] are 64bit space. We want to use 64bit |
| 812 | * address ranges to support 64bit addressing. Because 'size' for |
| 813 | * address ranges[2] and ranges[3] are same we can select either of |
| 814 | * ranges[2] or ranges[3] for mapping. However due to 'size' is too |
| 815 | * large for OS to allocate IOTSB we are using fix size 32G |
| 816 | * (ATU_64_SPACE_SIZE) which is more than enough for all PCIe devices |
| 817 | * to share. |
| 818 | */ |
| 819 | atu->ranges = (struct atu_ranges *)ranges; |
| 820 | atu->base = atu->ranges[3].base; |
| 821 | atu->size = ATU_64_SPACE_SIZE; |
| 822 | |
| 823 | /* Create IOTSB */ |
| 824 | err = pci_sun4v_atu_alloc_iotsb(pbm); |
| 825 | if (err) { |
| 826 | pr_err(PFX "Error creating ATU IOTSB\n"); |
| 827 | return err; |
| 828 | } |
| 829 | |
Tushar Dave | 31f077d | 2016-10-28 10:12:42 -0700 | [diff] [blame] | 830 | /* Create ATU iommu map. |
| 831 | * One bit represents one iotte in IOTSB table. |
| 832 | */ |
| 833 | dma_mask = (roundup_pow_of_two(atu->size) - 1UL); |
| 834 | num_iotte = atu->size / IO_PAGE_SIZE; |
| 835 | map_size = num_iotte / 8; |
| 836 | atu->tbl.table_map_base = atu->base; |
| 837 | atu->dma_addr_mask = dma_mask; |
| 838 | atu->tbl.map = kzalloc(map_size, GFP_KERNEL); |
| 839 | if (!atu->tbl.map) |
| 840 | return -ENOMEM; |
| 841 | |
| 842 | iommu_tbl_pool_init(&atu->tbl, num_iotte, IO_PAGE_SHIFT, |
| 843 | NULL, false /* no large_pool */, |
| 844 | 0 /* default npools */, |
| 845 | false /* want span boundary checking */); |
| 846 | |
Tushar Dave | f0248c1 | 2016-10-28 10:12:41 -0700 | [diff] [blame] | 847 | return 0; |
| 848 | } |
| 849 | |
Greg Kroah-Hartman | 7c9503b | 2012-12-21 14:03:26 -0800 | [diff] [blame] | 850 | static int pci_sun4v_iommu_init(struct pci_pbm_info *pbm) |
David S. Miller | bade562 | 2006-02-09 22:05:54 -0800 | [diff] [blame] | 851 | { |
David S. Miller | 8aef727 | 2008-09-01 20:23:18 -0700 | [diff] [blame] | 852 | static const u32 vdma_default[] = { 0x80000000, 0x80000000 }; |
David S. Miller | 16ce82d | 2007-04-26 21:08:21 -0700 | [diff] [blame] | 853 | struct iommu *iommu = pbm->iommu; |
David S. Miller | c6fee08 | 2011-02-26 23:40:02 -0800 | [diff] [blame] | 854 | unsigned long num_tsb_entries, sz; |
David S. Miller | 8aef727 | 2008-09-01 20:23:18 -0700 | [diff] [blame] | 855 | u32 dma_mask, dma_offset; |
| 856 | const u32 *vdma; |
David S. Miller | 1839794 | 2006-02-10 00:08:26 -0800 | [diff] [blame] | 857 | |
Grant Likely | 61c7a08 | 2010-04-13 16:12:29 -0700 | [diff] [blame] | 858 | vdma = of_get_property(pbm->op->dev.of_node, "virtual-dma", NULL); |
David S. Miller | 8aef727 | 2008-09-01 20:23:18 -0700 | [diff] [blame] | 859 | if (!vdma) |
| 860 | vdma = vdma_default; |
David S. Miller | 1839794 | 2006-02-10 00:08:26 -0800 | [diff] [blame] | 861 | |
David S. Miller | 59db810 | 2007-05-23 18:00:46 -0700 | [diff] [blame] | 862 | if ((vdma[0] | vdma[1]) & ~IO_PAGE_MASK) { |
David S. Miller | 3822b50 | 2008-08-30 02:50:29 -0700 | [diff] [blame] | 863 | printk(KERN_ERR PFX "Strange virtual-dma[%08x:%08x].\n", |
| 864 | vdma[0], vdma[1]); |
| 865 | return -EINVAL; |
Peter Senna Tschudin | 20b739f | 2012-09-12 07:03:11 +0000 | [diff] [blame] | 866 | } |
David S. Miller | 1839794 | 2006-02-10 00:08:26 -0800 | [diff] [blame] | 867 | |
David S. Miller | 59db810 | 2007-05-23 18:00:46 -0700 | [diff] [blame] | 868 | dma_mask = (roundup_pow_of_two(vdma[1]) - 1UL); |
| 869 | num_tsb_entries = vdma[1] / IO_PAGE_SIZE; |
David S. Miller | 1839794 | 2006-02-10 00:08:26 -0800 | [diff] [blame] | 870 | |
| 871 | dma_offset = vdma[0]; |
| 872 | |
| 873 | /* Setup initial software IOMMU state. */ |
David S. Miller | c12f048 | 2015-04-18 12:31:25 -0700 | [diff] [blame] | 874 | spin_lock_init(&iommu->lock); |
David S. Miller | 1839794 | 2006-02-10 00:08:26 -0800 | [diff] [blame] | 875 | iommu->ctx_lowest_free = 1; |
Sowmini Varadhan | bb620c3 | 2015-04-09 15:33:31 -0400 | [diff] [blame] | 876 | iommu->tbl.table_map_base = dma_offset; |
David S. Miller | 1839794 | 2006-02-10 00:08:26 -0800 | [diff] [blame] | 877 | iommu->dma_addr_mask = dma_mask; |
| 878 | |
| 879 | /* Allocate and initialize the free area map. */ |
David S. Miller | 59db810 | 2007-05-23 18:00:46 -0700 | [diff] [blame] | 880 | sz = (num_tsb_entries + 7) / 8; |
David S. Miller | 1839794 | 2006-02-10 00:08:26 -0800 | [diff] [blame] | 881 | sz = (sz + 7UL) & ~7UL; |
Sowmini Varadhan | bb620c3 | 2015-04-09 15:33:31 -0400 | [diff] [blame] | 882 | iommu->tbl.map = kzalloc(sz, GFP_KERNEL); |
| 883 | if (!iommu->tbl.map) { |
David S. Miller | 3822b50 | 2008-08-30 02:50:29 -0700 | [diff] [blame] | 884 | printk(KERN_ERR PFX "Error, kmalloc(arena.map) failed.\n"); |
| 885 | return -ENOMEM; |
David S. Miller | 1839794 | 2006-02-10 00:08:26 -0800 | [diff] [blame] | 886 | } |
Sowmini Varadhan | bb620c3 | 2015-04-09 15:33:31 -0400 | [diff] [blame] | 887 | iommu_tbl_pool_init(&iommu->tbl, num_tsb_entries, IO_PAGE_SHIFT, |
| 888 | NULL, false /* no large_pool */, |
| 889 | 0 /* default npools */, |
| 890 | false /* want span boundary checking */); |
| 891 | sz = probe_existing_entries(pbm, &iommu->tbl); |
David S. Miller | c2a5a46 | 2006-06-22 00:01:56 -0700 | [diff] [blame] | 892 | if (sz) |
| 893 | printk("%s: Imported %lu TSB entries from OBP\n", |
| 894 | pbm->name, sz); |
David S. Miller | 3822b50 | 2008-08-30 02:50:29 -0700 | [diff] [blame] | 895 | |
| 896 | return 0; |
David S. Miller | bade562 | 2006-02-09 22:05:54 -0800 | [diff] [blame] | 897 | } |
| 898 | |
David S. Miller | 35a17eb | 2007-02-10 17:41:02 -0800 | [diff] [blame] | 899 | #ifdef CONFIG_PCI_MSI |
| 900 | struct pci_sun4v_msiq_entry { |
| 901 | u64 version_type; |
| 902 | #define MSIQ_VERSION_MASK 0xffffffff00000000UL |
| 903 | #define MSIQ_VERSION_SHIFT 32 |
| 904 | #define MSIQ_TYPE_MASK 0x00000000000000ffUL |
| 905 | #define MSIQ_TYPE_SHIFT 0 |
| 906 | #define MSIQ_TYPE_NONE 0x00 |
| 907 | #define MSIQ_TYPE_MSG 0x01 |
| 908 | #define MSIQ_TYPE_MSI32 0x02 |
| 909 | #define MSIQ_TYPE_MSI64 0x03 |
| 910 | #define MSIQ_TYPE_INTX 0x08 |
| 911 | #define MSIQ_TYPE_NONE2 0xff |
| 912 | |
| 913 | u64 intx_sysino; |
| 914 | u64 reserved1; |
| 915 | u64 stick; |
| 916 | u64 req_id; /* bus/device/func */ |
| 917 | #define MSIQ_REQID_BUS_MASK 0xff00UL |
| 918 | #define MSIQ_REQID_BUS_SHIFT 8 |
| 919 | #define MSIQ_REQID_DEVICE_MASK 0x00f8UL |
| 920 | #define MSIQ_REQID_DEVICE_SHIFT 3 |
| 921 | #define MSIQ_REQID_FUNC_MASK 0x0007UL |
| 922 | #define MSIQ_REQID_FUNC_SHIFT 0 |
| 923 | |
| 924 | u64 msi_address; |
| 925 | |
Simon Arlott | e5dd42e | 2007-05-11 13:52:08 -0700 | [diff] [blame] | 926 | /* The format of this value is message type dependent. |
David S. Miller | 35a17eb | 2007-02-10 17:41:02 -0800 | [diff] [blame] | 927 | * For MSI bits 15:0 are the data from the MSI packet. |
| 928 | * For MSI-X bits 31:0 are the data from the MSI packet. |
| 929 | * For MSG, the message code and message routing code where: |
| 930 | * bits 39:32 is the bus/device/fn of the msg target-id |
| 931 | * bits 18:16 is the message routing code |
| 932 | * bits 7:0 is the message code |
| 933 | * For INTx the low order 2-bits are: |
| 934 | * 00 - INTA |
| 935 | * 01 - INTB |
| 936 | * 10 - INTC |
| 937 | * 11 - INTD |
| 938 | */ |
| 939 | u64 msi_data; |
| 940 | |
| 941 | u64 reserved2; |
| 942 | }; |
| 943 | |
David S. Miller | 759f89e | 2007-10-11 03:16:13 -0700 | [diff] [blame] | 944 | static int pci_sun4v_get_head(struct pci_pbm_info *pbm, unsigned long msiqid, |
| 945 | unsigned long *head) |
David S. Miller | 35a17eb | 2007-02-10 17:41:02 -0800 | [diff] [blame] | 946 | { |
David S. Miller | 759f89e | 2007-10-11 03:16:13 -0700 | [diff] [blame] | 947 | unsigned long err, limit; |
David S. Miller | 35a17eb | 2007-02-10 17:41:02 -0800 | [diff] [blame] | 948 | |
David S. Miller | 759f89e | 2007-10-11 03:16:13 -0700 | [diff] [blame] | 949 | err = pci_sun4v_msiq_gethead(pbm->devhandle, msiqid, head); |
David S. Miller | 35a17eb | 2007-02-10 17:41:02 -0800 | [diff] [blame] | 950 | if (unlikely(err)) |
David S. Miller | 759f89e | 2007-10-11 03:16:13 -0700 | [diff] [blame] | 951 | return -ENXIO; |
David S. Miller | 35a17eb | 2007-02-10 17:41:02 -0800 | [diff] [blame] | 952 | |
David S. Miller | 759f89e | 2007-10-11 03:16:13 -0700 | [diff] [blame] | 953 | limit = pbm->msiq_ent_count * sizeof(struct pci_sun4v_msiq_entry); |
| 954 | if (unlikely(*head >= limit)) |
| 955 | return -EFBIG; |
David S. Miller | 35a17eb | 2007-02-10 17:41:02 -0800 | [diff] [blame] | 956 | |
| 957 | return 0; |
| 958 | } |
| 959 | |
David S. Miller | 759f89e | 2007-10-11 03:16:13 -0700 | [diff] [blame] | 960 | static int pci_sun4v_dequeue_msi(struct pci_pbm_info *pbm, |
| 961 | unsigned long msiqid, unsigned long *head, |
| 962 | unsigned long *msi) |
David S. Miller | 35a17eb | 2007-02-10 17:41:02 -0800 | [diff] [blame] | 963 | { |
David S. Miller | 759f89e | 2007-10-11 03:16:13 -0700 | [diff] [blame] | 964 | struct pci_sun4v_msiq_entry *ep; |
| 965 | unsigned long err, type; |
| 966 | |
| 967 | /* Note: void pointer arithmetic, 'head' is a byte offset */ |
| 968 | ep = (pbm->msi_queues + ((msiqid - pbm->msiq_first) * |
| 969 | (pbm->msiq_ent_count * |
| 970 | sizeof(struct pci_sun4v_msiq_entry))) + |
| 971 | *head); |
| 972 | |
| 973 | if ((ep->version_type & MSIQ_TYPE_MASK) == 0) |
| 974 | return 0; |
| 975 | |
| 976 | type = (ep->version_type & MSIQ_TYPE_MASK) >> MSIQ_TYPE_SHIFT; |
| 977 | if (unlikely(type != MSIQ_TYPE_MSI32 && |
| 978 | type != MSIQ_TYPE_MSI64)) |
| 979 | return -EINVAL; |
| 980 | |
| 981 | *msi = ep->msi_data; |
| 982 | |
| 983 | err = pci_sun4v_msi_setstate(pbm->devhandle, |
| 984 | ep->msi_data /* msi_num */, |
| 985 | HV_MSISTATE_IDLE); |
| 986 | if (unlikely(err)) |
| 987 | return -ENXIO; |
| 988 | |
| 989 | /* Clear the entry. */ |
| 990 | ep->version_type &= ~MSIQ_TYPE_MASK; |
| 991 | |
| 992 | (*head) += sizeof(struct pci_sun4v_msiq_entry); |
| 993 | if (*head >= |
| 994 | (pbm->msiq_ent_count * sizeof(struct pci_sun4v_msiq_entry))) |
| 995 | *head = 0; |
| 996 | |
| 997 | return 1; |
David S. Miller | 35a17eb | 2007-02-10 17:41:02 -0800 | [diff] [blame] | 998 | } |
| 999 | |
David S. Miller | 759f89e | 2007-10-11 03:16:13 -0700 | [diff] [blame] | 1000 | static int pci_sun4v_set_head(struct pci_pbm_info *pbm, unsigned long msiqid, |
| 1001 | unsigned long head) |
| 1002 | { |
| 1003 | unsigned long err; |
| 1004 | |
| 1005 | err = pci_sun4v_msiq_sethead(pbm->devhandle, msiqid, head); |
| 1006 | if (unlikely(err)) |
| 1007 | return -EINVAL; |
| 1008 | |
| 1009 | return 0; |
| 1010 | } |
| 1011 | |
| 1012 | static int pci_sun4v_msi_setup(struct pci_pbm_info *pbm, unsigned long msiqid, |
| 1013 | unsigned long msi, int is_msi64) |
| 1014 | { |
| 1015 | if (pci_sun4v_msi_setmsiq(pbm->devhandle, msi, msiqid, |
| 1016 | (is_msi64 ? |
| 1017 | HV_MSITYPE_MSI64 : HV_MSITYPE_MSI32))) |
| 1018 | return -ENXIO; |
| 1019 | if (pci_sun4v_msi_setstate(pbm->devhandle, msi, HV_MSISTATE_IDLE)) |
| 1020 | return -ENXIO; |
| 1021 | if (pci_sun4v_msi_setvalid(pbm->devhandle, msi, HV_MSIVALID_VALID)) |
| 1022 | return -ENXIO; |
| 1023 | return 0; |
| 1024 | } |
| 1025 | |
| 1026 | static int pci_sun4v_msi_teardown(struct pci_pbm_info *pbm, unsigned long msi) |
| 1027 | { |
| 1028 | unsigned long err, msiqid; |
| 1029 | |
| 1030 | err = pci_sun4v_msi_getmsiq(pbm->devhandle, msi, &msiqid); |
| 1031 | if (err) |
| 1032 | return -ENXIO; |
| 1033 | |
| 1034 | pci_sun4v_msi_setvalid(pbm->devhandle, msi, HV_MSIVALID_INVALID); |
| 1035 | |
| 1036 | return 0; |
| 1037 | } |
| 1038 | |
| 1039 | static int pci_sun4v_msiq_alloc(struct pci_pbm_info *pbm) |
David S. Miller | 35a17eb | 2007-02-10 17:41:02 -0800 | [diff] [blame] | 1040 | { |
| 1041 | unsigned long q_size, alloc_size, pages, order; |
| 1042 | int i; |
| 1043 | |
| 1044 | q_size = pbm->msiq_ent_count * sizeof(struct pci_sun4v_msiq_entry); |
| 1045 | alloc_size = (pbm->msiq_num * q_size); |
| 1046 | order = get_order(alloc_size); |
| 1047 | pages = __get_free_pages(GFP_KERNEL | __GFP_COMP, order); |
| 1048 | if (pages == 0UL) { |
| 1049 | printk(KERN_ERR "MSI: Cannot allocate MSI queues (o=%lu).\n", |
| 1050 | order); |
| 1051 | return -ENOMEM; |
| 1052 | } |
| 1053 | memset((char *)pages, 0, PAGE_SIZE << order); |
| 1054 | pbm->msi_queues = (void *) pages; |
| 1055 | |
| 1056 | for (i = 0; i < pbm->msiq_num; i++) { |
| 1057 | unsigned long err, base = __pa(pages + (i * q_size)); |
| 1058 | unsigned long ret1, ret2; |
| 1059 | |
| 1060 | err = pci_sun4v_msiq_conf(pbm->devhandle, |
| 1061 | pbm->msiq_first + i, |
| 1062 | base, pbm->msiq_ent_count); |
| 1063 | if (err) { |
| 1064 | printk(KERN_ERR "MSI: msiq register fails (err=%lu)\n", |
| 1065 | err); |
| 1066 | goto h_error; |
| 1067 | } |
| 1068 | |
| 1069 | err = pci_sun4v_msiq_info(pbm->devhandle, |
| 1070 | pbm->msiq_first + i, |
| 1071 | &ret1, &ret2); |
| 1072 | if (err) { |
| 1073 | printk(KERN_ERR "MSI: Cannot read msiq (err=%lu)\n", |
| 1074 | err); |
| 1075 | goto h_error; |
| 1076 | } |
| 1077 | if (ret1 != base || ret2 != pbm->msiq_ent_count) { |
| 1078 | printk(KERN_ERR "MSI: Bogus qconf " |
| 1079 | "expected[%lx:%x] got[%lx:%lx]\n", |
| 1080 | base, pbm->msiq_ent_count, |
| 1081 | ret1, ret2); |
| 1082 | goto h_error; |
| 1083 | } |
| 1084 | } |
| 1085 | |
| 1086 | return 0; |
| 1087 | |
| 1088 | h_error: |
| 1089 | free_pages(pages, order); |
| 1090 | return -EINVAL; |
| 1091 | } |
| 1092 | |
David S. Miller | 759f89e | 2007-10-11 03:16:13 -0700 | [diff] [blame] | 1093 | static void pci_sun4v_msiq_free(struct pci_pbm_info *pbm) |
David S. Miller | 35a17eb | 2007-02-10 17:41:02 -0800 | [diff] [blame] | 1094 | { |
David S. Miller | 759f89e | 2007-10-11 03:16:13 -0700 | [diff] [blame] | 1095 | unsigned long q_size, alloc_size, pages, order; |
David S. Miller | 35a17eb | 2007-02-10 17:41:02 -0800 | [diff] [blame] | 1096 | int i; |
| 1097 | |
David S. Miller | 759f89e | 2007-10-11 03:16:13 -0700 | [diff] [blame] | 1098 | for (i = 0; i < pbm->msiq_num; i++) { |
| 1099 | unsigned long msiqid = pbm->msiq_first + i; |
| 1100 | |
| 1101 | (void) pci_sun4v_msiq_conf(pbm->devhandle, msiqid, 0UL, 0); |
David S. Miller | 35a17eb | 2007-02-10 17:41:02 -0800 | [diff] [blame] | 1102 | } |
| 1103 | |
David S. Miller | 759f89e | 2007-10-11 03:16:13 -0700 | [diff] [blame] | 1104 | q_size = pbm->msiq_ent_count * sizeof(struct pci_sun4v_msiq_entry); |
| 1105 | alloc_size = (pbm->msiq_num * q_size); |
| 1106 | order = get_order(alloc_size); |
| 1107 | |
| 1108 | pages = (unsigned long) pbm->msi_queues; |
| 1109 | |
| 1110 | free_pages(pages, order); |
| 1111 | |
| 1112 | pbm->msi_queues = NULL; |
David S. Miller | 35a17eb | 2007-02-10 17:41:02 -0800 | [diff] [blame] | 1113 | } |
| 1114 | |
David S. Miller | 759f89e | 2007-10-11 03:16:13 -0700 | [diff] [blame] | 1115 | static int pci_sun4v_msiq_build_irq(struct pci_pbm_info *pbm, |
| 1116 | unsigned long msiqid, |
| 1117 | unsigned long devino) |
David S. Miller | 35a17eb | 2007-02-10 17:41:02 -0800 | [diff] [blame] | 1118 | { |
Sam Ravnborg | 44ed3c0 | 2011-01-22 11:32:20 +0000 | [diff] [blame] | 1119 | unsigned int irq = sun4v_build_irq(pbm->devhandle, devino); |
David S. Miller | 35a17eb | 2007-02-10 17:41:02 -0800 | [diff] [blame] | 1120 | |
Sam Ravnborg | 44ed3c0 | 2011-01-22 11:32:20 +0000 | [diff] [blame] | 1121 | if (!irq) |
David S. Miller | 759f89e | 2007-10-11 03:16:13 -0700 | [diff] [blame] | 1122 | return -ENOMEM; |
David S. Miller | 35a17eb | 2007-02-10 17:41:02 -0800 | [diff] [blame] | 1123 | |
David S. Miller | 35a17eb | 2007-02-10 17:41:02 -0800 | [diff] [blame] | 1124 | if (pci_sun4v_msiq_setvalid(pbm->devhandle, msiqid, HV_MSIQ_VALID)) |
David S. Miller | 759f89e | 2007-10-11 03:16:13 -0700 | [diff] [blame] | 1125 | return -EINVAL; |
David S. Miller | 7cc8583 | 2011-12-22 13:23:59 -0800 | [diff] [blame] | 1126 | if (pci_sun4v_msiq_setstate(pbm->devhandle, msiqid, HV_MSIQSTATE_IDLE)) |
| 1127 | return -EINVAL; |
David S. Miller | 35a17eb | 2007-02-10 17:41:02 -0800 | [diff] [blame] | 1128 | |
Sam Ravnborg | 44ed3c0 | 2011-01-22 11:32:20 +0000 | [diff] [blame] | 1129 | return irq; |
David S. Miller | 35a17eb | 2007-02-10 17:41:02 -0800 | [diff] [blame] | 1130 | } |
| 1131 | |
David S. Miller | 759f89e | 2007-10-11 03:16:13 -0700 | [diff] [blame] | 1132 | static const struct sparc64_msiq_ops pci_sun4v_msiq_ops = { |
| 1133 | .get_head = pci_sun4v_get_head, |
| 1134 | .dequeue_msi = pci_sun4v_dequeue_msi, |
| 1135 | .set_head = pci_sun4v_set_head, |
| 1136 | .msi_setup = pci_sun4v_msi_setup, |
| 1137 | .msi_teardown = pci_sun4v_msi_teardown, |
| 1138 | .msiq_alloc = pci_sun4v_msiq_alloc, |
| 1139 | .msiq_free = pci_sun4v_msiq_free, |
| 1140 | .msiq_build_irq = pci_sun4v_msiq_build_irq, |
| 1141 | }; |
David S. Miller | e9870c4 | 2007-05-07 23:28:50 -0700 | [diff] [blame] | 1142 | |
| 1143 | static void pci_sun4v_msi_init(struct pci_pbm_info *pbm) |
| 1144 | { |
David S. Miller | 759f89e | 2007-10-11 03:16:13 -0700 | [diff] [blame] | 1145 | sparc64_pbm_msi_init(pbm, &pci_sun4v_msiq_ops); |
David S. Miller | e9870c4 | 2007-05-07 23:28:50 -0700 | [diff] [blame] | 1146 | } |
David S. Miller | 35a17eb | 2007-02-10 17:41:02 -0800 | [diff] [blame] | 1147 | #else /* CONFIG_PCI_MSI */ |
| 1148 | static void pci_sun4v_msi_init(struct pci_pbm_info *pbm) |
| 1149 | { |
| 1150 | } |
| 1151 | #endif /* !(CONFIG_PCI_MSI) */ |
| 1152 | |
Greg Kroah-Hartman | 7c9503b | 2012-12-21 14:03:26 -0800 | [diff] [blame] | 1153 | static int pci_sun4v_pbm_init(struct pci_pbm_info *pbm, |
| 1154 | struct platform_device *op, u32 devhandle) |
David S. Miller | bade562 | 2006-02-09 22:05:54 -0800 | [diff] [blame] | 1155 | { |
Grant Likely | 61c7a08 | 2010-04-13 16:12:29 -0700 | [diff] [blame] | 1156 | struct device_node *dp = op->dev.of_node; |
David S. Miller | 3822b50 | 2008-08-30 02:50:29 -0700 | [diff] [blame] | 1157 | int err; |
David S. Miller | bade562 | 2006-02-09 22:05:54 -0800 | [diff] [blame] | 1158 | |
David S. Miller | c1b1a5f | 2008-03-19 04:52:48 -0700 | [diff] [blame] | 1159 | pbm->numa_node = of_node_to_nid(dp); |
| 1160 | |
David S. Miller | ca3dd88 | 2007-05-09 02:35:27 -0700 | [diff] [blame] | 1161 | pbm->pci_ops = &sun4v_pci_ops; |
| 1162 | pbm->config_space_reg_bits = 12; |
David S. Miller | 34768bc | 2007-05-07 23:06:27 -0700 | [diff] [blame] | 1163 | |
David S. Miller | 6c108f1 | 2007-05-07 23:49:01 -0700 | [diff] [blame] | 1164 | pbm->index = pci_num_pbms++; |
| 1165 | |
David S. Miller | 22fecba | 2008-09-10 00:19:28 -0700 | [diff] [blame] | 1166 | pbm->op = op; |
David S. Miller | bade562 | 2006-02-09 22:05:54 -0800 | [diff] [blame] | 1167 | |
David S. Miller | 3833789 | 2006-02-12 22:06:53 -0800 | [diff] [blame] | 1168 | pbm->devhandle = devhandle; |
David S. Miller | bade562 | 2006-02-09 22:05:54 -0800 | [diff] [blame] | 1169 | |
David S. Miller | e87dc35 | 2006-06-21 18:18:47 -0700 | [diff] [blame] | 1170 | pbm->name = dp->full_name; |
David S. Miller | bade562 | 2006-02-09 22:05:54 -0800 | [diff] [blame] | 1171 | |
David S. Miller | e87dc35 | 2006-06-21 18:18:47 -0700 | [diff] [blame] | 1172 | printk("%s: SUN4V PCI Bus Module\n", pbm->name); |
David S. Miller | c1b1a5f | 2008-03-19 04:52:48 -0700 | [diff] [blame] | 1173 | printk("%s: On NUMA node %d\n", pbm->name, pbm->numa_node); |
David S. Miller | bade562 | 2006-02-09 22:05:54 -0800 | [diff] [blame] | 1174 | |
David S. Miller | 9fd8b64 | 2007-03-08 21:55:49 -0800 | [diff] [blame] | 1175 | pci_determine_mem_io_space(pbm); |
David S. Miller | bade562 | 2006-02-09 22:05:54 -0800 | [diff] [blame] | 1176 | |
David S. Miller | cfa0652 | 2007-05-07 21:51:41 -0700 | [diff] [blame] | 1177 | pci_get_pbm_props(pbm); |
David S. Miller | 3822b50 | 2008-08-30 02:50:29 -0700 | [diff] [blame] | 1178 | |
| 1179 | err = pci_sun4v_iommu_init(pbm); |
| 1180 | if (err) |
| 1181 | return err; |
| 1182 | |
David S. Miller | 35a17eb | 2007-02-10 17:41:02 -0800 | [diff] [blame] | 1183 | pci_sun4v_msi_init(pbm); |
David S. Miller | 3822b50 | 2008-08-30 02:50:29 -0700 | [diff] [blame] | 1184 | |
David S. Miller | e822358a | 2008-09-01 18:32:22 -0700 | [diff] [blame] | 1185 | pci_sun4v_scan_bus(pbm, &op->dev); |
David S. Miller | 3822b50 | 2008-08-30 02:50:29 -0700 | [diff] [blame] | 1186 | |
Tushar Dave | f0248c1 | 2016-10-28 10:12:41 -0700 | [diff] [blame] | 1187 | /* if atu_init fails its not complete failure. |
| 1188 | * we can still continue using legacy iommu. |
| 1189 | */ |
| 1190 | if (pbm->iommu->atu) { |
| 1191 | err = pci_sun4v_atu_init(pbm); |
| 1192 | if (err) { |
| 1193 | kfree(pbm->iommu->atu); |
| 1194 | pbm->iommu->atu = NULL; |
| 1195 | pr_err(PFX "ATU init failed, err=%d\n", err); |
| 1196 | } |
| 1197 | } |
| 1198 | |
David S. Miller | d3ae4b5 | 2008-09-09 23:54:02 -0700 | [diff] [blame] | 1199 | pbm->next = pci_pbm_root; |
| 1200 | pci_pbm_root = pbm; |
| 1201 | |
David S. Miller | 3822b50 | 2008-08-30 02:50:29 -0700 | [diff] [blame] | 1202 | return 0; |
David S. Miller | bade562 | 2006-02-09 22:05:54 -0800 | [diff] [blame] | 1203 | } |
| 1204 | |
Greg Kroah-Hartman | 7c9503b | 2012-12-21 14:03:26 -0800 | [diff] [blame] | 1205 | static int pci_sun4v_probe(struct platform_device *op) |
David S. Miller | 8f6a93a | 2006-02-09 21:32:07 -0800 | [diff] [blame] | 1206 | { |
David S. Miller | 3822b50 | 2008-08-30 02:50:29 -0700 | [diff] [blame] | 1207 | const struct linux_prom64_registers *regs; |
David S. Miller | e01c0d6 | 2007-05-25 01:04:15 -0700 | [diff] [blame] | 1208 | static int hvapi_negotiated = 0; |
David S. Miller | 34768bc | 2007-05-07 23:06:27 -0700 | [diff] [blame] | 1209 | struct pci_pbm_info *pbm; |
David S. Miller | 3822b50 | 2008-08-30 02:50:29 -0700 | [diff] [blame] | 1210 | struct device_node *dp; |
David S. Miller | 16ce82d | 2007-04-26 21:08:21 -0700 | [diff] [blame] | 1211 | struct iommu *iommu; |
Tushar Dave | f0248c1 | 2016-10-28 10:12:41 -0700 | [diff] [blame] | 1212 | struct atu *atu; |
David S. Miller | 7c8f486 | 2006-02-13 21:50:27 -0800 | [diff] [blame] | 1213 | u32 devhandle; |
chris hyser | 8914391 | 2016-09-28 12:19:45 -0700 | [diff] [blame] | 1214 | int i, err = -ENODEV; |
Tushar Dave | f0248c1 | 2016-10-28 10:12:41 -0700 | [diff] [blame] | 1215 | static bool hv_atu = true; |
David S. Miller | 3833789 | 2006-02-12 22:06:53 -0800 | [diff] [blame] | 1216 | |
Grant Likely | 61c7a08 | 2010-04-13 16:12:29 -0700 | [diff] [blame] | 1217 | dp = op->dev.of_node; |
David S. Miller | 3822b50 | 2008-08-30 02:50:29 -0700 | [diff] [blame] | 1218 | |
David S. Miller | e01c0d6 | 2007-05-25 01:04:15 -0700 | [diff] [blame] | 1219 | if (!hvapi_negotiated++) { |
chris hyser | 8914391 | 2016-09-28 12:19:45 -0700 | [diff] [blame] | 1220 | for (i = 0; i < ARRAY_SIZE(vpci_versions); i++) { |
| 1221 | vpci_major = vpci_versions[i].major; |
| 1222 | vpci_minor = vpci_versions[i].minor; |
| 1223 | |
| 1224 | err = sun4v_hvapi_register(HV_GRP_PCI, vpci_major, |
| 1225 | &vpci_minor); |
| 1226 | if (!err) |
| 1227 | break; |
| 1228 | } |
David S. Miller | e01c0d6 | 2007-05-25 01:04:15 -0700 | [diff] [blame] | 1229 | |
| 1230 | if (err) { |
chris hyser | 8914391 | 2016-09-28 12:19:45 -0700 | [diff] [blame] | 1231 | pr_err(PFX "Could not register hvapi, err=%d\n", err); |
David S. Miller | 3822b50 | 2008-08-30 02:50:29 -0700 | [diff] [blame] | 1232 | return err; |
David S. Miller | e01c0d6 | 2007-05-25 01:04:15 -0700 | [diff] [blame] | 1233 | } |
chris hyser | 8914391 | 2016-09-28 12:19:45 -0700 | [diff] [blame] | 1234 | pr_info(PFX "Registered hvapi major[%lu] minor[%lu]\n", |
| 1235 | vpci_major, vpci_minor); |
David S. Miller | ad7ad57 | 2007-07-27 22:39:14 -0700 | [diff] [blame] | 1236 | |
Tushar Dave | f0248c1 | 2016-10-28 10:12:41 -0700 | [diff] [blame] | 1237 | err = sun4v_hvapi_register(HV_GRP_ATU, vatu_major, &vatu_minor); |
| 1238 | if (err) { |
| 1239 | /* don't return an error if we fail to register the |
| 1240 | * ATU group, but ATU hcalls won't be available. |
| 1241 | */ |
| 1242 | hv_atu = false; |
| 1243 | pr_err(PFX "Could not register hvapi ATU err=%d\n", |
| 1244 | err); |
| 1245 | } else { |
| 1246 | pr_info(PFX "Registered hvapi ATU major[%lu] minor[%lu]\n", |
| 1247 | vatu_major, vatu_minor); |
| 1248 | } |
| 1249 | |
David S. Miller | ad7ad57 | 2007-07-27 22:39:14 -0700 | [diff] [blame] | 1250 | dma_ops = &sun4v_dma_ops; |
David S. Miller | e01c0d6 | 2007-05-25 01:04:15 -0700 | [diff] [blame] | 1251 | } |
| 1252 | |
David S. Miller | 3822b50 | 2008-08-30 02:50:29 -0700 | [diff] [blame] | 1253 | regs = of_get_property(dp, "reg", NULL); |
David S. Miller | d7472c3 | 2008-08-31 01:33:52 -0700 | [diff] [blame] | 1254 | err = -ENODEV; |
David S. Miller | 3822b50 | 2008-08-30 02:50:29 -0700 | [diff] [blame] | 1255 | if (!regs) { |
| 1256 | printk(KERN_ERR PFX "Could not find config registers\n"); |
David S. Miller | d7472c3 | 2008-08-31 01:33:52 -0700 | [diff] [blame] | 1257 | goto out_err; |
Cyrill Gorcunov | 75c6d14 | 2007-11-20 17:32:19 -0800 | [diff] [blame] | 1258 | } |
David S. Miller | e87dc35 | 2006-06-21 18:18:47 -0700 | [diff] [blame] | 1259 | devhandle = (regs->phys_addr >> 32UL) & 0x0fffffff; |
David S. Miller | 3833789 | 2006-02-12 22:06:53 -0800 | [diff] [blame] | 1260 | |
David S. Miller | d7472c3 | 2008-08-31 01:33:52 -0700 | [diff] [blame] | 1261 | err = -ENOMEM; |
David S. Miller | d3ae4b5 | 2008-09-09 23:54:02 -0700 | [diff] [blame] | 1262 | if (!iommu_batch_initialized) { |
| 1263 | for_each_possible_cpu(i) { |
| 1264 | unsigned long page = get_zeroed_page(GFP_KERNEL); |
David S. Miller | 7c8f486 | 2006-02-13 21:50:27 -0800 | [diff] [blame] | 1265 | |
David S. Miller | d3ae4b5 | 2008-09-09 23:54:02 -0700 | [diff] [blame] | 1266 | if (!page) |
| 1267 | goto out_err; |
David S. Miller | 7c8f486 | 2006-02-13 21:50:27 -0800 | [diff] [blame] | 1268 | |
David S. Miller | d3ae4b5 | 2008-09-09 23:54:02 -0700 | [diff] [blame] | 1269 | per_cpu(iommu_batch, i).pglist = (u64 *) page; |
| 1270 | } |
| 1271 | iommu_batch_initialized = 1; |
David S. Miller | bade562 | 2006-02-09 22:05:54 -0800 | [diff] [blame] | 1272 | } |
David S. Miller | 7c8f486 | 2006-02-13 21:50:27 -0800 | [diff] [blame] | 1273 | |
David S. Miller | d3ae4b5 | 2008-09-09 23:54:02 -0700 | [diff] [blame] | 1274 | pbm = kzalloc(sizeof(*pbm), GFP_KERNEL); |
| 1275 | if (!pbm) { |
| 1276 | printk(KERN_ERR PFX "Could not allocate pci_pbm_info\n"); |
David S. Miller | d7472c3 | 2008-08-31 01:33:52 -0700 | [diff] [blame] | 1277 | goto out_err; |
David S. Miller | 3822b50 | 2008-08-30 02:50:29 -0700 | [diff] [blame] | 1278 | } |
David S. Miller | 7c8f486 | 2006-02-13 21:50:27 -0800 | [diff] [blame] | 1279 | |
David S. Miller | d3ae4b5 | 2008-09-09 23:54:02 -0700 | [diff] [blame] | 1280 | iommu = kzalloc(sizeof(struct iommu), GFP_KERNEL); |
David S. Miller | 3822b50 | 2008-08-30 02:50:29 -0700 | [diff] [blame] | 1281 | if (!iommu) { |
David S. Miller | d3ae4b5 | 2008-09-09 23:54:02 -0700 | [diff] [blame] | 1282 | printk(KERN_ERR PFX "Could not allocate pbm iommu\n"); |
David S. Miller | d7472c3 | 2008-08-31 01:33:52 -0700 | [diff] [blame] | 1283 | goto out_free_controller; |
David S. Miller | 3822b50 | 2008-08-30 02:50:29 -0700 | [diff] [blame] | 1284 | } |
David S. Miller | 7c8f486 | 2006-02-13 21:50:27 -0800 | [diff] [blame] | 1285 | |
David S. Miller | d3ae4b5 | 2008-09-09 23:54:02 -0700 | [diff] [blame] | 1286 | pbm->iommu = iommu; |
Tushar Dave | f0248c1 | 2016-10-28 10:12:41 -0700 | [diff] [blame] | 1287 | iommu->atu = NULL; |
| 1288 | if (hv_atu) { |
| 1289 | atu = kzalloc(sizeof(*atu), GFP_KERNEL); |
| 1290 | if (!atu) |
| 1291 | pr_err(PFX "Could not allocate atu\n"); |
| 1292 | else |
| 1293 | iommu->atu = atu; |
| 1294 | } |
David S. Miller | bade562 | 2006-02-09 22:05:54 -0800 | [diff] [blame] | 1295 | |
David S. Miller | d3ae4b5 | 2008-09-09 23:54:02 -0700 | [diff] [blame] | 1296 | err = pci_sun4v_pbm_init(pbm, op, devhandle); |
| 1297 | if (err) |
| 1298 | goto out_free_iommu; |
David S. Miller | 7c8f486 | 2006-02-13 21:50:27 -0800 | [diff] [blame] | 1299 | |
David S. Miller | d3ae4b5 | 2008-09-09 23:54:02 -0700 | [diff] [blame] | 1300 | dev_set_drvdata(&op->dev, pbm); |
David S. Miller | bade562 | 2006-02-09 22:05:54 -0800 | [diff] [blame] | 1301 | |
David S. Miller | d3ae4b5 | 2008-09-09 23:54:02 -0700 | [diff] [blame] | 1302 | return 0; |
David S. Miller | 7c8f486 | 2006-02-13 21:50:27 -0800 | [diff] [blame] | 1303 | |
David S. Miller | d3ae4b5 | 2008-09-09 23:54:02 -0700 | [diff] [blame] | 1304 | out_free_iommu: |
Tushar Dave | f0248c1 | 2016-10-28 10:12:41 -0700 | [diff] [blame] | 1305 | kfree(iommu->atu); |
David S. Miller | d3ae4b5 | 2008-09-09 23:54:02 -0700 | [diff] [blame] | 1306 | kfree(pbm->iommu); |
David S. Miller | d7472c3 | 2008-08-31 01:33:52 -0700 | [diff] [blame] | 1307 | |
| 1308 | out_free_controller: |
David S. Miller | d3ae4b5 | 2008-09-09 23:54:02 -0700 | [diff] [blame] | 1309 | kfree(pbm); |
David S. Miller | d7472c3 | 2008-08-31 01:33:52 -0700 | [diff] [blame] | 1310 | |
| 1311 | out_err: |
| 1312 | return err; |
David S. Miller | 8f6a93a | 2006-02-09 21:32:07 -0800 | [diff] [blame] | 1313 | } |
David S. Miller | 3822b50 | 2008-08-30 02:50:29 -0700 | [diff] [blame] | 1314 | |
David S. Miller | 3628aa0 | 2011-03-30 17:37:56 -0700 | [diff] [blame] | 1315 | static const struct of_device_id pci_sun4v_match[] = { |
David S. Miller | 3822b50 | 2008-08-30 02:50:29 -0700 | [diff] [blame] | 1316 | { |
| 1317 | .name = "pci", |
| 1318 | .compatible = "SUNW,sun4v-pci", |
| 1319 | }, |
| 1320 | {}, |
| 1321 | }; |
| 1322 | |
Grant Likely | 4ebb24f | 2011-02-22 20:01:33 -0700 | [diff] [blame] | 1323 | static struct platform_driver pci_sun4v_driver = { |
Grant Likely | 4018294 | 2010-04-13 16:13:02 -0700 | [diff] [blame] | 1324 | .driver = { |
| 1325 | .name = DRIVER_NAME, |
Grant Likely | 4018294 | 2010-04-13 16:13:02 -0700 | [diff] [blame] | 1326 | .of_match_table = pci_sun4v_match, |
| 1327 | }, |
David S. Miller | 3822b50 | 2008-08-30 02:50:29 -0700 | [diff] [blame] | 1328 | .probe = pci_sun4v_probe, |
| 1329 | }; |
| 1330 | |
| 1331 | static int __init pci_sun4v_init(void) |
| 1332 | { |
Grant Likely | 4ebb24f | 2011-02-22 20:01:33 -0700 | [diff] [blame] | 1333 | return platform_driver_register(&pci_sun4v_driver); |
David S. Miller | 3822b50 | 2008-08-30 02:50:29 -0700 | [diff] [blame] | 1334 | } |
| 1335 | |
| 1336 | subsys_initcall(pci_sun4v_init); |