blob: 61d317428610ab3d953211366d2e938284f0d72a [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
Benjamin Herrenschmidt1beb6a72005-12-14 13:10:10 +11002 * arch/powerpc/sysdev/dart_iommu.c
Linus Torvalds1da177e2005-04-16 15:20:36 -07003 *
Olof Johansson91f14482005-11-21 02:12:32 -06004 * Copyright (C) 2004 Olof Johansson <olof@lixom.net>, IBM Corporation
Benjamin Herrenschmidt1beb6a72005-12-14 13:10:10 +11005 * Copyright (C) 2005 Benjamin Herrenschmidt <benh@kernel.crashing.org>,
6 * IBM Corporation
Linus Torvalds1da177e2005-04-16 15:20:36 -07007 *
8 * Based on pSeries_iommu.c:
9 * Copyright (C) 2001 Mike Corrigan & Dave Engebretsen, IBM Corporation
Olof Johansson91f14482005-11-21 02:12:32 -060010 * Copyright (C) 2004 Olof Johansson <olof@lixom.net>, IBM Corporation
Linus Torvalds1da177e2005-04-16 15:20:36 -070011 *
Benjamin Herrenschmidt1beb6a72005-12-14 13:10:10 +110012 * Dynamic DMA mapping support, Apple U3, U4 & IBM CPC925 "DART" iommu.
Linus Torvalds1da177e2005-04-16 15:20:36 -070013 *
Benjamin Herrenschmidt1beb6a72005-12-14 13:10:10 +110014 *
Linus Torvalds1da177e2005-04-16 15:20:36 -070015 * This program is free software; you can redistribute it and/or modify
16 * it under the terms of the GNU General Public License as published by
17 * the Free Software Foundation; either version 2 of the License, or
18 * (at your option) any later version.
Benjamin Herrenschmidt1beb6a72005-12-14 13:10:10 +110019 *
Linus Torvalds1da177e2005-04-16 15:20:36 -070020 * This program is distributed in the hope that it will be useful,
21 * but WITHOUT ANY WARRANTY; without even the implied warranty of
22 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
23 * GNU General Public License for more details.
Benjamin Herrenschmidt1beb6a72005-12-14 13:10:10 +110024 *
Linus Torvalds1da177e2005-04-16 15:20:36 -070025 * You should have received a copy of the GNU General Public License
26 * along with this program; if not, write to the Free Software
27 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
28 */
29
30#include <linux/config.h>
31#include <linux/init.h>
32#include <linux/types.h>
33#include <linux/slab.h>
34#include <linux/mm.h>
35#include <linux/spinlock.h>
36#include <linux/string.h>
37#include <linux/pci.h>
38#include <linux/dma-mapping.h>
39#include <linux/vmalloc.h>
40#include <asm/io.h>
41#include <asm/prom.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070042#include <asm/iommu.h>
43#include <asm/pci-bridge.h>
44#include <asm/machdep.h>
45#include <asm/abs_addr.h>
46#include <asm/cacheflush.h>
47#include <asm/lmb.h>
Stephen Rothwelld3878992005-09-28 02:50:25 +100048#include <asm/ppc-pci.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070049
David Gibson9933f292005-11-02 15:13:20 +110050#include "dart.h"
51
Linus Torvalds1da177e2005-04-16 15:20:36 -070052extern int iommu_force_on;
53
Linus Torvalds1da177e2005-04-16 15:20:36 -070054/* Physical base address and size of the DART table */
55unsigned long dart_tablebase; /* exported to htab_initialize */
56static unsigned long dart_tablesize;
57
58/* Virtual base address of the DART table */
59static u32 *dart_vbase;
60
61/* Mapped base address for the dart */
Al Viro6fa2ffe2006-02-01 07:28:02 -050062static unsigned int __iomem *dart;
Linus Torvalds1da177e2005-04-16 15:20:36 -070063
64/* Dummy val that entries are set to when unused */
65static unsigned int dart_emptyval;
66
Benjamin Herrenschmidt1beb6a72005-12-14 13:10:10 +110067static struct iommu_table iommu_table_dart;
68static int iommu_table_dart_inited;
Linus Torvalds1da177e2005-04-16 15:20:36 -070069static int dart_dirty;
Benjamin Herrenschmidt1beb6a72005-12-14 13:10:10 +110070static int dart_is_u4;
Linus Torvalds1da177e2005-04-16 15:20:36 -070071
72#define DBG(...)
73
74static inline void dart_tlb_invalidate_all(void)
75{
76 unsigned long l = 0;
Benjamin Herrenschmidt1beb6a72005-12-14 13:10:10 +110077 unsigned int reg, inv_bit;
Linus Torvalds1da177e2005-04-16 15:20:36 -070078 unsigned long limit;
79
80 DBG("dart: flush\n");
81
82 /* To invalidate the DART, set the DARTCNTL_FLUSHTLB bit in the
83 * control register and wait for it to clear.
84 *
85 * Gotcha: Sometimes, the DART won't detect that the bit gets
86 * set. If so, clear it and set it again.
Benjamin Herrenschmidt1beb6a72005-12-14 13:10:10 +110087 */
Linus Torvalds1da177e2005-04-16 15:20:36 -070088
89 limit = 0;
90
Benjamin Herrenschmidt1beb6a72005-12-14 13:10:10 +110091 inv_bit = dart_is_u4 ? DART_CNTL_U4_FLUSHTLB : DART_CNTL_U3_FLUSHTLB;
Linus Torvalds1da177e2005-04-16 15:20:36 -070092retry:
Linus Torvalds1da177e2005-04-16 15:20:36 -070093 l = 0;
Benjamin Herrenschmidt1beb6a72005-12-14 13:10:10 +110094 reg = DART_IN(DART_CNTL);
95 reg |= inv_bit;
96 DART_OUT(DART_CNTL, reg);
97
98 while ((DART_IN(DART_CNTL) & inv_bit) && l < (1L << limit))
Linus Torvalds1da177e2005-04-16 15:20:36 -070099 l++;
Benjamin Herrenschmidt1beb6a72005-12-14 13:10:10 +1100100 if (l == (1L << limit)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700101 if (limit < 4) {
102 limit++;
Benjamin Herrenschmidt1beb6a72005-12-14 13:10:10 +1100103 reg = DART_IN(DART_CNTL);
104 reg &= ~inv_bit;
105 DART_OUT(DART_CNTL, reg);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700106 goto retry;
107 } else
Benjamin Herrenschmidt1beb6a72005-12-14 13:10:10 +1100108 panic("DART: TLB did not flush after waiting a long "
Linus Torvalds1da177e2005-04-16 15:20:36 -0700109 "time. Buggy U3 ?");
110 }
111}
112
113static void dart_flush(struct iommu_table *tbl)
114{
115 if (dart_dirty)
116 dart_tlb_invalidate_all();
117 dart_dirty = 0;
118}
119
Benjamin Herrenschmidt1beb6a72005-12-14 13:10:10 +1100120static void dart_build(struct iommu_table *tbl, long index,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700121 long npages, unsigned long uaddr,
122 enum dma_data_direction direction)
123{
124 unsigned int *dp;
125 unsigned int rpn;
126
127 DBG("dart: build at: %lx, %lx, addr: %x\n", index, npages, uaddr);
128
Olof Johanssond0035c622005-09-20 13:46:44 +1000129 index <<= DART_PAGE_FACTOR;
130 npages <<= DART_PAGE_FACTOR;
131
Linus Torvalds1da177e2005-04-16 15:20:36 -0700132 dp = ((unsigned int*)tbl->it_base) + index;
Benjamin Herrenschmidt1beb6a72005-12-14 13:10:10 +1100133
Linus Torvalds1da177e2005-04-16 15:20:36 -0700134 /* On U3, all memory is contigous, so we can move this
135 * out of the loop.
136 */
137 while (npages--) {
Olof Johanssond0035c622005-09-20 13:46:44 +1000138 rpn = virt_to_abs(uaddr) >> DART_PAGE_SHIFT;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700139
140 *(dp++) = DARTMAP_VALID | (rpn & DARTMAP_RPNMASK);
141
Olof Johanssond0035c622005-09-20 13:46:44 +1000142 uaddr += DART_PAGE_SIZE;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700143 }
144
145 dart_dirty = 1;
146}
147
148
149static void dart_free(struct iommu_table *tbl, long index, long npages)
150{
151 unsigned int *dp;
Benjamin Herrenschmidt1beb6a72005-12-14 13:10:10 +1100152
Linus Torvalds1da177e2005-04-16 15:20:36 -0700153 /* We don't worry about flushing the TLB cache. The only drawback of
154 * not doing it is that we won't catch buggy device drivers doing
155 * bad DMAs, but then no 32-bit architecture ever does either.
156 */
157
158 DBG("dart: free at: %lx, %lx\n", index, npages);
159
Olof Johanssond0035c622005-09-20 13:46:44 +1000160 index <<= DART_PAGE_FACTOR;
161 npages <<= DART_PAGE_FACTOR;
162
Linus Torvalds1da177e2005-04-16 15:20:36 -0700163 dp = ((unsigned int *)tbl->it_base) + index;
Benjamin Herrenschmidt1beb6a72005-12-14 13:10:10 +1100164
Linus Torvalds1da177e2005-04-16 15:20:36 -0700165 while (npages--)
166 *(dp++) = dart_emptyval;
167}
168
169
170static int dart_init(struct device_node *dart_node)
171{
Linus Torvalds1da177e2005-04-16 15:20:36 -0700172 unsigned int i;
Benjamin Herrenschmidt1beb6a72005-12-14 13:10:10 +1100173 unsigned long tmp, base, size;
174 struct resource r;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700175
176 if (dart_tablebase == 0 || dart_tablesize == 0) {
Benjamin Herrenschmidt1beb6a72005-12-14 13:10:10 +1100177 printk(KERN_INFO "DART: table not allocated, using "
178 "direct DMA\n");
Linus Torvalds1da177e2005-04-16 15:20:36 -0700179 return -ENODEV;
180 }
181
Benjamin Herrenschmidt1beb6a72005-12-14 13:10:10 +1100182 if (of_address_to_resource(dart_node, 0, &r))
183 panic("DART: can't get register base ! ");
184
Linus Torvalds1da177e2005-04-16 15:20:36 -0700185 /* Make sure nothing from the DART range remains in the CPU cache
186 * from a previous mapping that existed before the kernel took
187 * over
188 */
Benjamin Herrenschmidt1beb6a72005-12-14 13:10:10 +1100189 flush_dcache_phys_range(dart_tablebase,
190 dart_tablebase + dart_tablesize);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700191
192 /* Allocate a spare page to map all invalid DART pages. We need to do
193 * that to work around what looks like a problem with the HT bridge
194 * prefetching into invalid pages and corrupting data
195 */
Olof Johanssond0035c622005-09-20 13:46:44 +1000196 tmp = lmb_alloc(DART_PAGE_SIZE, DART_PAGE_SIZE);
Benjamin Herrenschmidt1beb6a72005-12-14 13:10:10 +1100197 dart_emptyval = DARTMAP_VALID | ((tmp >> DART_PAGE_SHIFT) &
198 DARTMAP_RPNMASK);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700199
Benjamin Herrenschmidt1beb6a72005-12-14 13:10:10 +1100200 /* Map in DART registers */
201 dart = ioremap(r.start, r.end - r.start + 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700202 if (dart == NULL)
Benjamin Herrenschmidt1beb6a72005-12-14 13:10:10 +1100203 panic("DART: Cannot map registers!");
Linus Torvalds1da177e2005-04-16 15:20:36 -0700204
Benjamin Herrenschmidt1beb6a72005-12-14 13:10:10 +1100205 /* Map in DART table */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700206 dart_vbase = ioremap(virt_to_abs(dart_tablebase), dart_tablesize);
207
208 /* Fill initial table */
209 for (i = 0; i < dart_tablesize/4; i++)
210 dart_vbase[i] = dart_emptyval;
211
212 /* Initialize DART with table base and enable it. */
Benjamin Herrenschmidt1beb6a72005-12-14 13:10:10 +1100213 base = dart_tablebase >> DART_PAGE_SHIFT;
214 size = dart_tablesize >> DART_PAGE_SHIFT;
215 if (dart_is_u4) {
Benjamin Herrenschmidt56c8eae2005-12-19 16:49:07 +1100216 size &= DART_SIZE_U4_SIZE_MASK;
Benjamin Herrenschmidt1beb6a72005-12-14 13:10:10 +1100217 DART_OUT(DART_BASE_U4, base);
218 DART_OUT(DART_SIZE_U4, size);
219 DART_OUT(DART_CNTL, DART_CNTL_U4_ENABLE);
220 } else {
Benjamin Herrenschmidt56c8eae2005-12-19 16:49:07 +1100221 size &= DART_CNTL_U3_SIZE_MASK;
Benjamin Herrenschmidt1beb6a72005-12-14 13:10:10 +1100222 DART_OUT(DART_CNTL,
223 DART_CNTL_U3_ENABLE |
224 (base << DART_CNTL_U3_BASE_SHIFT) |
225 (size << DART_CNTL_U3_SIZE_SHIFT));
226 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700227
228 /* Invalidate DART to get rid of possible stale TLBs */
229 dart_tlb_invalidate_all();
230
Benjamin Herrenschmidt1beb6a72005-12-14 13:10:10 +1100231 printk(KERN_INFO "DART IOMMU initialized for %s type chipset\n",
232 dart_is_u4 ? "U4" : "U3");
Linus Torvalds1da177e2005-04-16 15:20:36 -0700233
234 return 0;
235}
236
Benjamin Herrenschmidt1beb6a72005-12-14 13:10:10 +1100237static void iommu_table_dart_setup(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700238{
Benjamin Herrenschmidt1beb6a72005-12-14 13:10:10 +1100239 iommu_table_dart.it_busno = 0;
240 iommu_table_dart.it_offset = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700241 /* it_size is in number of entries */
Benjamin Herrenschmidt1beb6a72005-12-14 13:10:10 +1100242 iommu_table_dart.it_size = (dart_tablesize / sizeof(u32)) >> DART_PAGE_FACTOR;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700243
244 /* Initialize the common IOMMU code */
Benjamin Herrenschmidt1beb6a72005-12-14 13:10:10 +1100245 iommu_table_dart.it_base = (unsigned long)dart_vbase;
246 iommu_table_dart.it_index = 0;
247 iommu_table_dart.it_blocksize = 1;
248 iommu_init_table(&iommu_table_dart);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700249
250 /* Reserve the last page of the DART to avoid possible prefetch
251 * past the DART mapped area
252 */
Benjamin Herrenschmidt1beb6a72005-12-14 13:10:10 +1100253 set_bit(iommu_table_dart.it_size - 1, iommu_table_dart.it_map);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700254}
255
Benjamin Herrenschmidt1beb6a72005-12-14 13:10:10 +1100256static void iommu_dev_setup_dart(struct pci_dev *dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700257{
258 struct device_node *dn;
259
260 /* We only have one iommu table on the mac for now, which makes
261 * things simple. Setup all PCI devices to point to this table
262 *
263 * We must use pci_device_to_OF_node() to make sure that
264 * we get the real "final" pointer to the device in the
265 * pci_dev sysdata and not the temporary PHB one
266 */
267 dn = pci_device_to_OF_node(dev);
268
269 if (dn)
Benjamin Herrenschmidt1beb6a72005-12-14 13:10:10 +1100270 PCI_DN(dn)->iommu_table = &iommu_table_dart;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700271}
272
Benjamin Herrenschmidt1beb6a72005-12-14 13:10:10 +1100273static void iommu_bus_setup_dart(struct pci_bus *bus)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700274{
275 struct device_node *dn;
276
Benjamin Herrenschmidt1beb6a72005-12-14 13:10:10 +1100277 if (!iommu_table_dart_inited) {
278 iommu_table_dart_inited = 1;
279 iommu_table_dart_setup();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700280 }
281
282 dn = pci_bus_to_OF_node(bus);
283
284 if (dn)
Benjamin Herrenschmidt1beb6a72005-12-14 13:10:10 +1100285 PCI_DN(dn)->iommu_table = &iommu_table_dart;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700286}
287
288static void iommu_dev_setup_null(struct pci_dev *dev) { }
289static void iommu_bus_setup_null(struct pci_bus *bus) { }
290
Benjamin Herrenschmidt1beb6a72005-12-14 13:10:10 +1100291void iommu_init_early_dart(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700292{
293 struct device_node *dn;
294
295 /* Find the DART in the device-tree */
296 dn = of_find_compatible_node(NULL, "dart", "u3-dart");
Benjamin Herrenschmidt1beb6a72005-12-14 13:10:10 +1100297 if (dn == NULL) {
298 dn = of_find_compatible_node(NULL, "dart", "u4-dart");
299 if (dn == NULL)
300 goto bail;
301 dart_is_u4 = 1;
302 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700303
304 /* Setup low level TCE operations for the core IOMMU code */
305 ppc_md.tce_build = dart_build;
306 ppc_md.tce_free = dart_free;
307 ppc_md.tce_flush = dart_flush;
308
309 /* Initialize the DART HW */
Benjamin Herrenschmidt1beb6a72005-12-14 13:10:10 +1100310 if (dart_init(dn) == 0) {
311 ppc_md.iommu_dev_setup = iommu_dev_setup_dart;
312 ppc_md.iommu_bus_setup = iommu_bus_setup_dart;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700313
314 /* Setup pci_dma ops */
315 pci_iommu_init();
Benjamin Herrenschmidt1beb6a72005-12-14 13:10:10 +1100316
317 return;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700318 }
Benjamin Herrenschmidt1beb6a72005-12-14 13:10:10 +1100319
320 bail:
321 /* If init failed, use direct iommu and null setup functions */
322 ppc_md.iommu_dev_setup = iommu_dev_setup_null;
323 ppc_md.iommu_bus_setup = iommu_bus_setup_null;
324
325 /* Setup pci_dma ops */
326 pci_direct_iommu_init();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700327}
328
329
Benjamin Herrenschmidt1beb6a72005-12-14 13:10:10 +1100330void __init alloc_dart_table(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700331{
332 /* Only reserve DART space if machine has more than 2GB of RAM
333 * or if requested with iommu=on on cmdline.
334 */
335 if (lmb_end_of_DRAM() <= 0x80000000ull && !iommu_force_on)
336 return;
337
338 /* 512 pages (2MB) is max DART tablesize. */
339 dart_tablesize = 1UL << 21;
340 /* 16MB (1 << 24) alignment. We allocate a full 16Mb chuck since we
341 * will blow up an entire large page anyway in the kernel mapping
342 */
343 dart_tablebase = (unsigned long)
344 abs_to_virt(lmb_alloc_base(1UL<<24, 1UL<<24, 0x80000000L));
345
Benjamin Herrenschmidt1beb6a72005-12-14 13:10:10 +1100346 printk(KERN_INFO "DART table allocated at: %lx\n", dart_tablebase);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700347}