blob: 623a68af8b18dec5a6346046503ac7dbd05c4096 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * Dynamic DMA mapping support.
3 *
Jan Beulich563aaf02007-02-05 18:51:25 -08004 * This implementation is a fallback for platforms that do not support
Linus Torvalds1da177e2005-04-16 15:20:36 -07005 * I/O TLBs (aka DMA address translation hardware).
6 * Copyright (C) 2000 Asit Mallick <Asit.K.Mallick@intel.com>
7 * Copyright (C) 2000 Goutham Rao <goutham.rao@intel.com>
8 * Copyright (C) 2000, 2003 Hewlett-Packard Co
9 * David Mosberger-Tang <davidm@hpl.hp.com>
10 *
11 * 03/05/07 davidm Switch from PCI-DMA to generic device DMA API.
12 * 00/12/13 davidm Rename to swiotlb.c and add mark_clean() to avoid
13 * unnecessary i-cache flushing.
John W. Linville569c8bf2005-09-29 14:45:24 -070014 * 04/07/.. ak Better overflow handling. Assorted fixes.
15 * 05/09/10 linville Add support for syncing ranges, support syncing for
16 * DMA_BIDIRECTIONAL mappings, miscellaneous cleanup.
Linus Torvalds1da177e2005-04-16 15:20:36 -070017 */
18
19#include <linux/cache.h>
Tony Luck17e5ad62005-09-29 15:52:13 -070020#include <linux/dma-mapping.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070021#include <linux/mm.h>
22#include <linux/module.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070023#include <linux/spinlock.h>
24#include <linux/string.h>
25#include <linux/types.h>
26#include <linux/ctype.h>
27
28#include <asm/io.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070029#include <asm/dma.h>
Tony Luck17e5ad62005-09-29 15:52:13 -070030#include <asm/scatterlist.h>
Jan Beulich51099002007-02-05 18:53:04 -080031#include <asm/swiotlb.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070032
33#include <linux/init.h>
34#include <linux/bootmem.h>
35
36#define OFFSET(val,align) ((unsigned long) \
37 ( (val) & ( (align) - 1)))
38
Jan Beulich51099002007-02-05 18:53:04 -080039#ifndef SG_ENT_VIRT_ADDRESS
Linus Torvalds1da177e2005-04-16 15:20:36 -070040#define SG_ENT_VIRT_ADDRESS(sg) (page_address((sg)->page) + (sg)->offset)
Jan Beulich93fbff62007-02-05 18:49:45 -080041#define SG_ENT_PHYS_ADDRESS(sg) virt_to_bus(SG_ENT_VIRT_ADDRESS(sg))
Jan Beulich51099002007-02-05 18:53:04 -080042#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -070043
44/*
45 * Maximum allowable number of contiguous slabs to map,
46 * must be a power of 2. What is the appropriate value ?
47 * The complexity of {map,unmap}_single is linearly dependent on this value.
48 */
49#define IO_TLB_SEGSIZE 128
50
51/*
52 * log of the size of each IO TLB slab. The number of slabs is command line
53 * controllable.
54 */
55#define IO_TLB_SHIFT 11
56
Alex Williamson0b9afed2005-09-06 11:20:49 -060057#define SLABS_PER_PAGE (1 << (PAGE_SHIFT - IO_TLB_SHIFT))
58
59/*
60 * Minimum IO TLB size to bother booting with. Systems with mainly
61 * 64bit capable cards will only lightly use the swiotlb. If we can't
62 * allocate a contiguous 1MB, we're probably in trouble anyway.
63 */
64#define IO_TLB_MIN_SLABS ((1<<20) >> IO_TLB_SHIFT)
65
John W. Linvillede69e0f2005-09-29 14:44:57 -070066/*
67 * Enumeration for sync targets
68 */
69enum dma_sync_target {
70 SYNC_FOR_CPU = 0,
71 SYNC_FOR_DEVICE = 1,
72};
73
Linus Torvalds1da177e2005-04-16 15:20:36 -070074int swiotlb_force;
75
76/*
77 * Used to do a quick range check in swiotlb_unmap_single and
78 * swiotlb_sync_single_*, to see if the memory was in fact allocated by this
79 * API.
80 */
81static char *io_tlb_start, *io_tlb_end;
82
83/*
84 * The number of IO TLB blocks (in groups of 64) betweeen io_tlb_start and
85 * io_tlb_end. This is command line adjustable via setup_io_tlb_npages.
86 */
87static unsigned long io_tlb_nslabs;
88
89/*
90 * When the IOMMU overflows we return a fallback buffer. This sets the size.
91 */
92static unsigned long io_tlb_overflow = 32*1024;
93
94void *io_tlb_overflow_buffer;
95
96/*
97 * This is a free list describing the number of free entries available from
98 * each index
99 */
100static unsigned int *io_tlb_list;
101static unsigned int io_tlb_index;
102
103/*
104 * We need to save away the original address corresponding to a mapped entry
105 * for the sync operations.
106 */
Jan Beulich51099002007-02-05 18:53:04 -0800107#ifndef SWIOTLB_ARCH_HAS_IO_TLB_ADDR_T
108typedef char *io_tlb_addr_t;
109#define swiotlb_orig_addr_null(buffer) (!(buffer))
110#define ptr_to_io_tlb_addr(ptr) (ptr)
111#define page_to_io_tlb_addr(pg, off) (page_address(pg) + (off))
112#define sg_to_io_tlb_addr(sg) SG_ENT_VIRT_ADDRESS(sg)
113#endif
114static io_tlb_addr_t *io_tlb_orig_addr;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700115
116/*
117 * Protect the above data structures in the map and unmap calls
118 */
119static DEFINE_SPINLOCK(io_tlb_lock);
120
Jan Beulich51099002007-02-05 18:53:04 -0800121#ifdef SWIOTLB_EXTRA_VARIABLES
122SWIOTLB_EXTRA_VARIABLES;
123#endif
124
125#ifndef SWIOTLB_ARCH_HAS_SETUP_IO_TLB_NPAGES
Linus Torvalds1da177e2005-04-16 15:20:36 -0700126static int __init
127setup_io_tlb_npages(char *str)
128{
129 if (isdigit(*str)) {
Alex Williamsone8579e72005-08-04 13:06:00 -0700130 io_tlb_nslabs = simple_strtoul(str, &str, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700131 /* avoid tail segment of size < IO_TLB_SEGSIZE */
132 io_tlb_nslabs = ALIGN(io_tlb_nslabs, IO_TLB_SEGSIZE);
133 }
134 if (*str == ',')
135 ++str;
136 if (!strcmp(str, "force"))
137 swiotlb_force = 1;
138 return 1;
139}
Jan Beulich51099002007-02-05 18:53:04 -0800140#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -0700141__setup("swiotlb=", setup_io_tlb_npages);
142/* make io_tlb_overflow tunable too? */
143
Jan Beulich51099002007-02-05 18:53:04 -0800144#ifndef swiotlb_adjust_size
145#define swiotlb_adjust_size(size) ((void)0)
146#endif
147
148#ifndef swiotlb_adjust_seg
149#define swiotlb_adjust_seg(start, size) ((void)0)
150#endif
151
152#ifndef swiotlb_print_info
153#define swiotlb_print_info(bytes) \
154 printk(KERN_INFO "Placing %luMB software IO TLB between 0x%lx - " \
155 "0x%lx\n", bytes >> 20, \
156 virt_to_bus(io_tlb_start), virt_to_bus(io_tlb_end))
157#endif
158
Linus Torvalds1da177e2005-04-16 15:20:36 -0700159/*
160 * Statically reserve bounce buffer space and initialize bounce buffer data
Tony Luck17e5ad62005-09-29 15:52:13 -0700161 * structures for the software IO TLB used to implement the DMA API.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700162 */
Jan Beulich563aaf02007-02-05 18:51:25 -0800163void __init
164swiotlb_init_with_default_size(size_t default_size)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700165{
Jan Beulich563aaf02007-02-05 18:51:25 -0800166 unsigned long i, bytes;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700167
168 if (!io_tlb_nslabs) {
Alex Williamsone8579e72005-08-04 13:06:00 -0700169 io_tlb_nslabs = (default_size >> IO_TLB_SHIFT);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700170 io_tlb_nslabs = ALIGN(io_tlb_nslabs, IO_TLB_SEGSIZE);
171 }
Jan Beulich51099002007-02-05 18:53:04 -0800172 swiotlb_adjust_size(io_tlb_nslabs);
173 swiotlb_adjust_size(io_tlb_overflow);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700174
Jan Beulich563aaf02007-02-05 18:51:25 -0800175 bytes = io_tlb_nslabs << IO_TLB_SHIFT;
176
Linus Torvalds1da177e2005-04-16 15:20:36 -0700177 /*
178 * Get IO TLB memory from the low pages
179 */
Jan Beulich563aaf02007-02-05 18:51:25 -0800180 io_tlb_start = alloc_bootmem_low_pages(bytes);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700181 if (!io_tlb_start)
182 panic("Cannot allocate SWIOTLB buffer");
Jan Beulich563aaf02007-02-05 18:51:25 -0800183 io_tlb_end = io_tlb_start + bytes;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700184
185 /*
186 * Allocate and initialize the free list array. This array is used
187 * to find contiguous free memory regions of size up to IO_TLB_SEGSIZE
188 * between io_tlb_start and io_tlb_end.
189 */
190 io_tlb_list = alloc_bootmem(io_tlb_nslabs * sizeof(int));
Jan Beulich51099002007-02-05 18:53:04 -0800191 for (i = 0; i < io_tlb_nslabs; i++) {
192 if ( !(i % IO_TLB_SEGSIZE) )
193 swiotlb_adjust_seg(io_tlb_start + (i << IO_TLB_SHIFT),
194 IO_TLB_SEGSIZE << IO_TLB_SHIFT);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700195 io_tlb_list[i] = IO_TLB_SEGSIZE - OFFSET(i, IO_TLB_SEGSIZE);
Jan Beulich51099002007-02-05 18:53:04 -0800196 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700197 io_tlb_index = 0;
Jan Beulich51099002007-02-05 18:53:04 -0800198 io_tlb_orig_addr = alloc_bootmem(io_tlb_nslabs * sizeof(io_tlb_addr_t));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700199
200 /*
201 * Get the overflow emergency buffer
202 */
203 io_tlb_overflow_buffer = alloc_bootmem_low(io_tlb_overflow);
Jan Beulich563aaf02007-02-05 18:51:25 -0800204 if (!io_tlb_overflow_buffer)
205 panic("Cannot allocate SWIOTLB overflow buffer!\n");
Jan Beulich51099002007-02-05 18:53:04 -0800206 swiotlb_adjust_seg(io_tlb_overflow_buffer, io_tlb_overflow);
Jan Beulich563aaf02007-02-05 18:51:25 -0800207
Jan Beulich51099002007-02-05 18:53:04 -0800208 swiotlb_print_info(bytes);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700209}
Jan Beulich51099002007-02-05 18:53:04 -0800210#ifndef __swiotlb_init_with_default_size
211#define __swiotlb_init_with_default_size swiotlb_init_with_default_size
212#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -0700213
Jan Beulich563aaf02007-02-05 18:51:25 -0800214void __init
215swiotlb_init(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700216{
Jan Beulich51099002007-02-05 18:53:04 -0800217 __swiotlb_init_with_default_size(64 * (1<<20)); /* default to 64MB */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700218}
219
Jan Beulich51099002007-02-05 18:53:04 -0800220#ifdef SWIOTLB_ARCH_NEED_LATE_INIT
Alex Williamson0b9afed2005-09-06 11:20:49 -0600221/*
222 * Systems with larger DMA zones (those that don't support ISA) can
223 * initialize the swiotlb later using the slab allocator if needed.
224 * This should be just like above, but with some error catching.
225 */
226int
Jan Beulich563aaf02007-02-05 18:51:25 -0800227swiotlb_late_init_with_default_size(size_t default_size)
Alex Williamson0b9afed2005-09-06 11:20:49 -0600228{
Jan Beulich563aaf02007-02-05 18:51:25 -0800229 unsigned long i, bytes, req_nslabs = io_tlb_nslabs;
Alex Williamson0b9afed2005-09-06 11:20:49 -0600230 unsigned int order;
231
232 if (!io_tlb_nslabs) {
233 io_tlb_nslabs = (default_size >> IO_TLB_SHIFT);
234 io_tlb_nslabs = ALIGN(io_tlb_nslabs, IO_TLB_SEGSIZE);
235 }
236
237 /*
238 * Get IO TLB memory from the low pages
239 */
Jan Beulich563aaf02007-02-05 18:51:25 -0800240 order = get_order(io_tlb_nslabs << IO_TLB_SHIFT);
Alex Williamson0b9afed2005-09-06 11:20:49 -0600241 io_tlb_nslabs = SLABS_PER_PAGE << order;
Jan Beulich563aaf02007-02-05 18:51:25 -0800242 bytes = io_tlb_nslabs << IO_TLB_SHIFT;
Alex Williamson0b9afed2005-09-06 11:20:49 -0600243
244 while ((SLABS_PER_PAGE << order) > IO_TLB_MIN_SLABS) {
245 io_tlb_start = (char *)__get_free_pages(GFP_DMA | __GFP_NOWARN,
246 order);
247 if (io_tlb_start)
248 break;
249 order--;
250 }
251
252 if (!io_tlb_start)
253 goto cleanup1;
254
Jan Beulich563aaf02007-02-05 18:51:25 -0800255 if (order != get_order(bytes)) {
Alex Williamson0b9afed2005-09-06 11:20:49 -0600256 printk(KERN_WARNING "Warning: only able to allocate %ld MB "
257 "for software IO TLB\n", (PAGE_SIZE << order) >> 20);
258 io_tlb_nslabs = SLABS_PER_PAGE << order;
Jan Beulich563aaf02007-02-05 18:51:25 -0800259 bytes = io_tlb_nslabs << IO_TLB_SHIFT;
Alex Williamson0b9afed2005-09-06 11:20:49 -0600260 }
Jan Beulich563aaf02007-02-05 18:51:25 -0800261 io_tlb_end = io_tlb_start + bytes;
262 memset(io_tlb_start, 0, bytes);
Alex Williamson0b9afed2005-09-06 11:20:49 -0600263
264 /*
265 * Allocate and initialize the free list array. This array is used
266 * to find contiguous free memory regions of size up to IO_TLB_SEGSIZE
267 * between io_tlb_start and io_tlb_end.
268 */
269 io_tlb_list = (unsigned int *)__get_free_pages(GFP_KERNEL,
270 get_order(io_tlb_nslabs * sizeof(int)));
271 if (!io_tlb_list)
272 goto cleanup2;
273
274 for (i = 0; i < io_tlb_nslabs; i++)
275 io_tlb_list[i] = IO_TLB_SEGSIZE - OFFSET(i, IO_TLB_SEGSIZE);
276 io_tlb_index = 0;
277
Jan Beulich51099002007-02-05 18:53:04 -0800278 io_tlb_orig_addr = (io_tlb_addr_t *)__get_free_pages(GFP_KERNEL,
279 get_order(io_tlb_nslabs * sizeof(io_tlb_addr_t)));
Alex Williamson0b9afed2005-09-06 11:20:49 -0600280 if (!io_tlb_orig_addr)
281 goto cleanup3;
282
Jan Beulich51099002007-02-05 18:53:04 -0800283 memset(io_tlb_orig_addr, 0, io_tlb_nslabs * sizeof(io_tlb_addr_t));
Alex Williamson0b9afed2005-09-06 11:20:49 -0600284
285 /*
286 * Get the overflow emergency buffer
287 */
288 io_tlb_overflow_buffer = (void *)__get_free_pages(GFP_DMA,
289 get_order(io_tlb_overflow));
290 if (!io_tlb_overflow_buffer)
291 goto cleanup4;
292
Jan Beulich51099002007-02-05 18:53:04 -0800293 swiotlb_print_info(bytes);
Alex Williamson0b9afed2005-09-06 11:20:49 -0600294
295 return 0;
296
297cleanup4:
Jan Beulich51099002007-02-05 18:53:04 -0800298 free_pages((unsigned long)io_tlb_orig_addr,
299 get_order(io_tlb_nslabs * sizeof(io_tlb_addr_t)));
Alex Williamson0b9afed2005-09-06 11:20:49 -0600300 io_tlb_orig_addr = NULL;
301cleanup3:
Jan Beulich51099002007-02-05 18:53:04 -0800302 free_pages((unsigned long)io_tlb_list,
303 get_order(io_tlb_nslabs * sizeof(int)));
Alex Williamson0b9afed2005-09-06 11:20:49 -0600304 io_tlb_list = NULL;
Alex Williamson0b9afed2005-09-06 11:20:49 -0600305cleanup2:
Jan Beulich563aaf02007-02-05 18:51:25 -0800306 io_tlb_end = NULL;
Alex Williamson0b9afed2005-09-06 11:20:49 -0600307 free_pages((unsigned long)io_tlb_start, order);
308 io_tlb_start = NULL;
309cleanup1:
310 io_tlb_nslabs = req_nslabs;
311 return -ENOMEM;
312}
Jan Beulich51099002007-02-05 18:53:04 -0800313#endif
Alex Williamson0b9afed2005-09-06 11:20:49 -0600314
Jan Beulich51099002007-02-05 18:53:04 -0800315#ifndef SWIOTLB_ARCH_HAS_NEEDS_MAPPING
Andrew Mortonbe6b0262007-02-12 00:52:17 -0800316static int
Linus Torvalds1da177e2005-04-16 15:20:36 -0700317address_needs_mapping(struct device *hwdev, dma_addr_t addr)
318{
319 dma_addr_t mask = 0xffffffff;
320 /* If the device has a mask, use it, otherwise default to 32 bits */
321 if (hwdev && hwdev->dma_mask)
322 mask = *hwdev->dma_mask;
323 return (addr & ~mask) != 0;
324}
325
Jan Beulich51099002007-02-05 18:53:04 -0800326static inline int range_needs_mapping(const void *ptr, size_t size)
327{
328 return swiotlb_force;
329}
330
331static inline int order_needs_mapping(unsigned int order)
332{
333 return 0;
334}
335#endif
336
337static void
338__sync_single(io_tlb_addr_t buffer, char *dma_addr, size_t size, int dir)
339{
340#ifndef SWIOTLB_ARCH_HAS_SYNC_SINGLE
341 if (dir == DMA_TO_DEVICE)
342 memcpy(dma_addr, buffer, size);
343 else
344 memcpy(buffer, dma_addr, size);
345#else
346 __swiotlb_arch_sync_single(buffer, dma_addr, size, dir);
347#endif
348}
349
Linus Torvalds1da177e2005-04-16 15:20:36 -0700350/*
351 * Allocates bounce buffer and returns its kernel virtual address.
352 */
353static void *
Jan Beulich51099002007-02-05 18:53:04 -0800354map_single(struct device *hwdev, io_tlb_addr_t buffer, size_t size, int dir)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700355{
356 unsigned long flags;
357 char *dma_addr;
358 unsigned int nslots, stride, index, wrap;
359 int i;
360
361 /*
362 * For mappings greater than a page, we limit the stride (and
363 * hence alignment) to a page size.
364 */
365 nslots = ALIGN(size, 1 << IO_TLB_SHIFT) >> IO_TLB_SHIFT;
366 if (size > PAGE_SIZE)
367 stride = (1 << (PAGE_SHIFT - IO_TLB_SHIFT));
368 else
369 stride = 1;
370
Eric Sesterhenn34814542006-03-24 18:47:11 +0100371 BUG_ON(!nslots);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700372
373 /*
374 * Find suitable number of IO TLB entries size that will fit this
375 * request and allocate a buffer from that IO TLB pool.
376 */
377 spin_lock_irqsave(&io_tlb_lock, flags);
378 {
379 wrap = index = ALIGN(io_tlb_index, stride);
380
381 if (index >= io_tlb_nslabs)
382 wrap = index = 0;
383
384 do {
385 /*
386 * If we find a slot that indicates we have 'nslots'
387 * number of contiguous buffers, we allocate the
388 * buffers from that slot and mark the entries as '0'
389 * indicating unavailable.
390 */
391 if (io_tlb_list[index] >= nslots) {
392 int count = 0;
393
394 for (i = index; i < (int) (index + nslots); i++)
395 io_tlb_list[i] = 0;
396 for (i = index - 1; (OFFSET(i, IO_TLB_SEGSIZE) != IO_TLB_SEGSIZE -1) && io_tlb_list[i]; i--)
397 io_tlb_list[i] = ++count;
398 dma_addr = io_tlb_start + (index << IO_TLB_SHIFT);
399
400 /*
401 * Update the indices to avoid searching in
402 * the next round.
403 */
404 io_tlb_index = ((index + nslots) < io_tlb_nslabs
405 ? (index + nslots) : 0);
406
407 goto found;
408 }
409 index += stride;
410 if (index >= io_tlb_nslabs)
411 index = 0;
412 } while (index != wrap);
413
414 spin_unlock_irqrestore(&io_tlb_lock, flags);
415 return NULL;
416 }
417 found:
418 spin_unlock_irqrestore(&io_tlb_lock, flags);
419
420 /*
421 * Save away the mapping from the original address to the DMA address.
422 * This is needed when we sync the memory. Then we sync the buffer if
423 * needed.
424 */
425 io_tlb_orig_addr[index] = buffer;
426 if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL)
Jan Beulich51099002007-02-05 18:53:04 -0800427 __sync_single(buffer, dma_addr, size, DMA_TO_DEVICE);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700428
429 return dma_addr;
430}
431
432/*
433 * dma_addr is the kernel virtual address of the bounce buffer to unmap.
434 */
435static void
436unmap_single(struct device *hwdev, char *dma_addr, size_t size, int dir)
437{
438 unsigned long flags;
439 int i, count, nslots = ALIGN(size, 1 << IO_TLB_SHIFT) >> IO_TLB_SHIFT;
440 int index = (dma_addr - io_tlb_start) >> IO_TLB_SHIFT;
Jan Beulich51099002007-02-05 18:53:04 -0800441 io_tlb_addr_t buffer = io_tlb_orig_addr[index];
Linus Torvalds1da177e2005-04-16 15:20:36 -0700442
443 /*
444 * First, sync the memory before unmapping the entry
445 */
Jan Beulich51099002007-02-05 18:53:04 -0800446 if (!swiotlb_orig_addr_null(buffer)
447 && ((dir == DMA_FROM_DEVICE) || (dir == DMA_BIDIRECTIONAL)))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700448 /*
449 * bounce... copy the data back into the original buffer * and
450 * delete the bounce buffer.
451 */
Jan Beulich51099002007-02-05 18:53:04 -0800452 __sync_single(buffer, dma_addr, size, DMA_FROM_DEVICE);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700453
454 /*
455 * Return the buffer to the free list by setting the corresponding
456 * entries to indicate the number of contigous entries available.
457 * While returning the entries to the free list, we merge the entries
458 * with slots below and above the pool being returned.
459 */
460 spin_lock_irqsave(&io_tlb_lock, flags);
461 {
462 count = ((index + nslots) < ALIGN(index + 1, IO_TLB_SEGSIZE) ?
463 io_tlb_list[index + nslots] : 0);
464 /*
465 * Step 1: return the slots to the free list, merging the
466 * slots with superceeding slots
467 */
468 for (i = index + nslots - 1; i >= index; i--)
469 io_tlb_list[i] = ++count;
470 /*
471 * Step 2: merge the returned slots with the preceding slots,
472 * if available (non zero)
473 */
474 for (i = index - 1; (OFFSET(i, IO_TLB_SEGSIZE) != IO_TLB_SEGSIZE -1) && io_tlb_list[i]; i--)
475 io_tlb_list[i] = ++count;
476 }
477 spin_unlock_irqrestore(&io_tlb_lock, flags);
478}
479
480static void
John W. Linvillede69e0f2005-09-29 14:44:57 -0700481sync_single(struct device *hwdev, char *dma_addr, size_t size,
482 int dir, int target)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700483{
484 int index = (dma_addr - io_tlb_start) >> IO_TLB_SHIFT;
Jan Beulich51099002007-02-05 18:53:04 -0800485 io_tlb_addr_t buffer = io_tlb_orig_addr[index];
Linus Torvalds1da177e2005-04-16 15:20:36 -0700486
John W. Linvillede69e0f2005-09-29 14:44:57 -0700487 switch (target) {
488 case SYNC_FOR_CPU:
489 if (likely(dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL))
Jan Beulich51099002007-02-05 18:53:04 -0800490 __sync_single(buffer, dma_addr, size, DMA_FROM_DEVICE);
Eric Sesterhenn34814542006-03-24 18:47:11 +0100491 else
492 BUG_ON(dir != DMA_TO_DEVICE);
John W. Linvillede69e0f2005-09-29 14:44:57 -0700493 break;
494 case SYNC_FOR_DEVICE:
495 if (likely(dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL))
Jan Beulich51099002007-02-05 18:53:04 -0800496 __sync_single(buffer, dma_addr, size, DMA_TO_DEVICE);
Eric Sesterhenn34814542006-03-24 18:47:11 +0100497 else
498 BUG_ON(dir != DMA_FROM_DEVICE);
John W. Linvillede69e0f2005-09-29 14:44:57 -0700499 break;
500 default:
Linus Torvalds1da177e2005-04-16 15:20:36 -0700501 BUG();
John W. Linvillede69e0f2005-09-29 14:44:57 -0700502 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700503}
504
Jan Beulich51099002007-02-05 18:53:04 -0800505#ifdef SWIOTLB_ARCH_NEED_ALLOC
506
Linus Torvalds1da177e2005-04-16 15:20:36 -0700507void *
508swiotlb_alloc_coherent(struct device *hwdev, size_t size,
Al Viro06a54492005-10-21 03:21:03 -0400509 dma_addr_t *dma_handle, gfp_t flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700510{
Jan Beulich563aaf02007-02-05 18:51:25 -0800511 dma_addr_t dev_addr;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700512 void *ret;
513 int order = get_order(size);
514
515 /*
516 * XXX fix me: the DMA API should pass us an explicit DMA mask
517 * instead, or use ZONE_DMA32 (ia64 overloads ZONE_DMA to be a ~32
518 * bit range instead of a 16MB one).
519 */
520 flags |= GFP_DMA;
521
Jan Beulich51099002007-02-05 18:53:04 -0800522 if (!order_needs_mapping(order))
523 ret = (void *)__get_free_pages(flags, order);
524 else
525 ret = NULL;
Jan Beulich93fbff62007-02-05 18:49:45 -0800526 if (ret && address_needs_mapping(hwdev, virt_to_bus(ret))) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700527 /*
528 * The allocated memory isn't reachable by the device.
529 * Fall back on swiotlb_map_single().
530 */
531 free_pages((unsigned long) ret, order);
532 ret = NULL;
533 }
534 if (!ret) {
535 /*
536 * We are either out of memory or the device can't DMA
537 * to GFP_DMA memory; fall back on
538 * swiotlb_map_single(), which will grab memory from
539 * the lowest available address range.
540 */
541 dma_addr_t handle;
542 handle = swiotlb_map_single(NULL, NULL, size, DMA_FROM_DEVICE);
Muli Ben-Yehuda17a941d2006-01-11 22:44:42 +0100543 if (swiotlb_dma_mapping_error(handle))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700544 return NULL;
545
Jan Beulich93fbff62007-02-05 18:49:45 -0800546 ret = bus_to_virt(handle);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700547 }
548
549 memset(ret, 0, size);
Jan Beulich93fbff62007-02-05 18:49:45 -0800550 dev_addr = virt_to_bus(ret);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700551
552 /* Confirm address can be DMA'd by device */
553 if (address_needs_mapping(hwdev, dev_addr)) {
Jan Beulich563aaf02007-02-05 18:51:25 -0800554 printk("hwdev DMA mask = 0x%016Lx, dev_addr = 0x%016Lx\n",
555 (unsigned long long)*hwdev->dma_mask,
556 (unsigned long long)dev_addr);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700557 panic("swiotlb_alloc_coherent: allocated memory is out of "
558 "range for device");
559 }
560 *dma_handle = dev_addr;
561 return ret;
562}
Jan Beulich51099002007-02-05 18:53:04 -0800563EXPORT_SYMBOL(swiotlb_alloc_coherent);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700564
565void
566swiotlb_free_coherent(struct device *hwdev, size_t size, void *vaddr,
567 dma_addr_t dma_handle)
568{
569 if (!(vaddr >= (void *)io_tlb_start
570 && vaddr < (void *)io_tlb_end))
571 free_pages((unsigned long) vaddr, get_order(size));
572 else
573 /* DMA_TO_DEVICE to avoid memcpy in unmap_single */
574 swiotlb_unmap_single (hwdev, dma_handle, size, DMA_TO_DEVICE);
575}
Jan Beulich51099002007-02-05 18:53:04 -0800576EXPORT_SYMBOL(swiotlb_free_coherent);
577
578#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -0700579
580static void
581swiotlb_full(struct device *dev, size_t size, int dir, int do_panic)
582{
583 /*
584 * Ran out of IOMMU space for this operation. This is very bad.
585 * Unfortunately the drivers cannot handle this operation properly.
Tony Luck17e5ad62005-09-29 15:52:13 -0700586 * unless they check for dma_mapping_error (most don't)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700587 * When the mapping is small enough return a static buffer to limit
588 * the damage, or panic when the transfer is too big.
589 */
Jan Beulich563aaf02007-02-05 18:51:25 -0800590 printk(KERN_ERR "DMA: Out of SW-IOMMU space for %zu bytes at "
Linus Torvalds1da177e2005-04-16 15:20:36 -0700591 "device %s\n", size, dev ? dev->bus_id : "?");
592
593 if (size > io_tlb_overflow && do_panic) {
Tony Luck17e5ad62005-09-29 15:52:13 -0700594 if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
595 panic("DMA: Memory would be corrupted\n");
596 if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL)
597 panic("DMA: Random memory would be DMAed\n");
Linus Torvalds1da177e2005-04-16 15:20:36 -0700598 }
599}
600
601/*
602 * Map a single buffer of the indicated size for DMA in streaming mode. The
Tony Luck17e5ad62005-09-29 15:52:13 -0700603 * physical address to use is returned.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700604 *
605 * Once the device is given the dma address, the device owns this memory until
606 * either swiotlb_unmap_single or swiotlb_dma_sync_single is performed.
607 */
608dma_addr_t
609swiotlb_map_single(struct device *hwdev, void *ptr, size_t size, int dir)
610{
Jan Beulich563aaf02007-02-05 18:51:25 -0800611 dma_addr_t dev_addr = virt_to_bus(ptr);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700612 void *map;
613
Eric Sesterhenn34814542006-03-24 18:47:11 +0100614 BUG_ON(dir == DMA_NONE);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700615 /*
616 * If the pointer passed in happens to be in the device's DMA window,
617 * we can safely return the device addr and not worry about bounce
618 * buffering it.
619 */
Jan Beulich51099002007-02-05 18:53:04 -0800620 if (!range_needs_mapping(ptr, size)
621 && !address_needs_mapping(hwdev, dev_addr))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700622 return dev_addr;
623
624 /*
625 * Oh well, have to allocate and map a bounce buffer.
626 */
Jan Beulich51099002007-02-05 18:53:04 -0800627 map = map_single(hwdev, ptr_to_io_tlb_addr(ptr), size, dir);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700628 if (!map) {
629 swiotlb_full(hwdev, size, dir, 1);
630 map = io_tlb_overflow_buffer;
631 }
632
Jan Beulich93fbff62007-02-05 18:49:45 -0800633 dev_addr = virt_to_bus(map);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700634
635 /*
636 * Ensure that the address returned is DMA'ble
637 */
638 if (address_needs_mapping(hwdev, dev_addr))
639 panic("map_single: bounce buffer is not DMA'ble");
640
641 return dev_addr;
642}
643
644/*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700645 * Unmap a single streaming mode DMA translation. The dma_addr and size must
646 * match what was provided for in a previous swiotlb_map_single call. All
647 * other usages are undefined.
648 *
649 * After this call, reads by the cpu to the buffer are guaranteed to see
650 * whatever the device wrote there.
651 */
652void
653swiotlb_unmap_single(struct device *hwdev, dma_addr_t dev_addr, size_t size,
654 int dir)
655{
Jan Beulich93fbff62007-02-05 18:49:45 -0800656 char *dma_addr = bus_to_virt(dev_addr);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700657
Eric Sesterhenn34814542006-03-24 18:47:11 +0100658 BUG_ON(dir == DMA_NONE);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700659 if (dma_addr >= io_tlb_start && dma_addr < io_tlb_end)
660 unmap_single(hwdev, dma_addr, size, dir);
661 else if (dir == DMA_FROM_DEVICE)
Jan Beulichcde14bb2007-02-05 18:46:40 -0800662 dma_mark_clean(dma_addr, size);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700663}
664
665/*
666 * Make physical memory consistent for a single streaming mode DMA translation
667 * after a transfer.
668 *
669 * If you perform a swiotlb_map_single() but wish to interrogate the buffer
Tony Luck17e5ad62005-09-29 15:52:13 -0700670 * using the cpu, yet do not wish to teardown the dma mapping, you must
671 * call this function before doing so. At the next point you give the dma
Linus Torvalds1da177e2005-04-16 15:20:36 -0700672 * address back to the card, you must first perform a
673 * swiotlb_dma_sync_for_device, and then the device again owns the buffer
674 */
Andrew Mortonbe6b0262007-02-12 00:52:17 -0800675static void
John W. Linville8270f3f2005-09-29 14:43:32 -0700676swiotlb_sync_single(struct device *hwdev, dma_addr_t dev_addr,
John W. Linvillede69e0f2005-09-29 14:44:57 -0700677 size_t size, int dir, int target)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700678{
Jan Beulich93fbff62007-02-05 18:49:45 -0800679 char *dma_addr = bus_to_virt(dev_addr);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700680
Eric Sesterhenn34814542006-03-24 18:47:11 +0100681 BUG_ON(dir == DMA_NONE);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700682 if (dma_addr >= io_tlb_start && dma_addr < io_tlb_end)
John W. Linvillede69e0f2005-09-29 14:44:57 -0700683 sync_single(hwdev, dma_addr, size, dir, target);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700684 else if (dir == DMA_FROM_DEVICE)
Jan Beulichcde14bb2007-02-05 18:46:40 -0800685 dma_mark_clean(dma_addr, size);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700686}
687
688void
John W. Linville8270f3f2005-09-29 14:43:32 -0700689swiotlb_sync_single_for_cpu(struct device *hwdev, dma_addr_t dev_addr,
690 size_t size, int dir)
691{
John W. Linvillede69e0f2005-09-29 14:44:57 -0700692 swiotlb_sync_single(hwdev, dev_addr, size, dir, SYNC_FOR_CPU);
John W. Linville8270f3f2005-09-29 14:43:32 -0700693}
694
695void
Linus Torvalds1da177e2005-04-16 15:20:36 -0700696swiotlb_sync_single_for_device(struct device *hwdev, dma_addr_t dev_addr,
697 size_t size, int dir)
698{
John W. Linvillede69e0f2005-09-29 14:44:57 -0700699 swiotlb_sync_single(hwdev, dev_addr, size, dir, SYNC_FOR_DEVICE);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700700}
701
702/*
John W. Linville878a97c2005-09-29 14:44:23 -0700703 * Same as above, but for a sub-range of the mapping.
704 */
Andrew Mortonbe6b0262007-02-12 00:52:17 -0800705static void
John W. Linville878a97c2005-09-29 14:44:23 -0700706swiotlb_sync_single_range(struct device *hwdev, dma_addr_t dev_addr,
John W. Linvillede69e0f2005-09-29 14:44:57 -0700707 unsigned long offset, size_t size,
708 int dir, int target)
John W. Linville878a97c2005-09-29 14:44:23 -0700709{
Jan Beulich93fbff62007-02-05 18:49:45 -0800710 char *dma_addr = bus_to_virt(dev_addr) + offset;
John W. Linville878a97c2005-09-29 14:44:23 -0700711
Eric Sesterhenn34814542006-03-24 18:47:11 +0100712 BUG_ON(dir == DMA_NONE);
John W. Linville878a97c2005-09-29 14:44:23 -0700713 if (dma_addr >= io_tlb_start && dma_addr < io_tlb_end)
John W. Linvillede69e0f2005-09-29 14:44:57 -0700714 sync_single(hwdev, dma_addr, size, dir, target);
John W. Linville878a97c2005-09-29 14:44:23 -0700715 else if (dir == DMA_FROM_DEVICE)
Jan Beulichcde14bb2007-02-05 18:46:40 -0800716 dma_mark_clean(dma_addr, size);
John W. Linville878a97c2005-09-29 14:44:23 -0700717}
718
719void
720swiotlb_sync_single_range_for_cpu(struct device *hwdev, dma_addr_t dev_addr,
721 unsigned long offset, size_t size, int dir)
722{
John W. Linvillede69e0f2005-09-29 14:44:57 -0700723 swiotlb_sync_single_range(hwdev, dev_addr, offset, size, dir,
724 SYNC_FOR_CPU);
John W. Linville878a97c2005-09-29 14:44:23 -0700725}
726
727void
728swiotlb_sync_single_range_for_device(struct device *hwdev, dma_addr_t dev_addr,
729 unsigned long offset, size_t size, int dir)
730{
John W. Linvillede69e0f2005-09-29 14:44:57 -0700731 swiotlb_sync_single_range(hwdev, dev_addr, offset, size, dir,
732 SYNC_FOR_DEVICE);
John W. Linville878a97c2005-09-29 14:44:23 -0700733}
734
735/*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700736 * Map a set of buffers described by scatterlist in streaming mode for DMA.
737 * This is the scatter-gather version of the above swiotlb_map_single
738 * interface. Here the scatter gather list elements are each tagged with the
739 * appropriate dma address and length. They are obtained via
740 * sg_dma_{address,length}(SG).
741 *
742 * NOTE: An implementation may be able to use a smaller number of
743 * DMA address/length pairs than there are SG table elements.
744 * (for example via virtual mapping capabilities)
745 * The routine returns the number of addr/length pairs actually
746 * used, at most nents.
747 *
748 * Device ownership issues as mentioned above for swiotlb_map_single are the
749 * same here.
750 */
751int
752swiotlb_map_sg(struct device *hwdev, struct scatterlist *sg, int nelems,
753 int dir)
754{
Jan Beulich563aaf02007-02-05 18:51:25 -0800755 dma_addr_t dev_addr;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700756 int i;
757
Eric Sesterhenn34814542006-03-24 18:47:11 +0100758 BUG_ON(dir == DMA_NONE);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700759
760 for (i = 0; i < nelems; i++, sg++) {
Jan Beulich51099002007-02-05 18:53:04 -0800761 dev_addr = SG_ENT_PHYS_ADDRESS(sg);
762 if (range_needs_mapping(SG_ENT_VIRT_ADDRESS(sg), sg->length)
763 || address_needs_mapping(hwdev, dev_addr)) {
764 void *map = map_single(hwdev, sg_to_io_tlb_addr(sg), sg->length, dir);
Andi Kleen7e870232005-12-20 14:45:19 +0100765 if (!map) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700766 /* Don't panic here, we expect map_sg users
767 to do proper error handling. */
768 swiotlb_full(hwdev, sg->length, dir, 0);
769 swiotlb_unmap_sg(hwdev, sg - i, i, dir);
770 sg[0].dma_length = 0;
771 return 0;
772 }
Jan Beulichcde14bb2007-02-05 18:46:40 -0800773 sg->dma_address = virt_to_bus(map);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700774 } else
775 sg->dma_address = dev_addr;
776 sg->dma_length = sg->length;
777 }
778 return nelems;
779}
780
781/*
782 * Unmap a set of streaming mode DMA translations. Again, cpu read rules
783 * concerning calls here are the same as for swiotlb_unmap_single() above.
784 */
785void
786swiotlb_unmap_sg(struct device *hwdev, struct scatterlist *sg, int nelems,
787 int dir)
788{
789 int i;
790
Eric Sesterhenn34814542006-03-24 18:47:11 +0100791 BUG_ON(dir == DMA_NONE);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700792
793 for (i = 0; i < nelems; i++, sg++)
794 if (sg->dma_address != SG_ENT_PHYS_ADDRESS(sg))
Jan Beulich93fbff62007-02-05 18:49:45 -0800795 unmap_single(hwdev, bus_to_virt(sg->dma_address),
796 sg->dma_length, dir);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700797 else if (dir == DMA_FROM_DEVICE)
Jan Beulichcde14bb2007-02-05 18:46:40 -0800798 dma_mark_clean(SG_ENT_VIRT_ADDRESS(sg), sg->dma_length);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700799}
800
801/*
802 * Make physical memory consistent for a set of streaming mode DMA translations
803 * after a transfer.
804 *
805 * The same as swiotlb_sync_single_* but for a scatter-gather list, same rules
806 * and usage.
807 */
Andrew Mortonbe6b0262007-02-12 00:52:17 -0800808static void
John W. Linville8270f3f2005-09-29 14:43:32 -0700809swiotlb_sync_sg(struct device *hwdev, struct scatterlist *sg,
John W. Linvillede69e0f2005-09-29 14:44:57 -0700810 int nelems, int dir, int target)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700811{
812 int i;
813
Eric Sesterhenn34814542006-03-24 18:47:11 +0100814 BUG_ON(dir == DMA_NONE);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700815
816 for (i = 0; i < nelems; i++, sg++)
817 if (sg->dma_address != SG_ENT_PHYS_ADDRESS(sg))
Jan Beulich93fbff62007-02-05 18:49:45 -0800818 sync_single(hwdev, bus_to_virt(sg->dma_address),
John W. Linvillede69e0f2005-09-29 14:44:57 -0700819 sg->dma_length, dir, target);
Jan Beulichcde14bb2007-02-05 18:46:40 -0800820 else if (dir == DMA_FROM_DEVICE)
821 dma_mark_clean(SG_ENT_VIRT_ADDRESS(sg), sg->dma_length);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700822}
823
824void
John W. Linville8270f3f2005-09-29 14:43:32 -0700825swiotlb_sync_sg_for_cpu(struct device *hwdev, struct scatterlist *sg,
826 int nelems, int dir)
827{
John W. Linvillede69e0f2005-09-29 14:44:57 -0700828 swiotlb_sync_sg(hwdev, sg, nelems, dir, SYNC_FOR_CPU);
John W. Linville8270f3f2005-09-29 14:43:32 -0700829}
830
831void
Linus Torvalds1da177e2005-04-16 15:20:36 -0700832swiotlb_sync_sg_for_device(struct device *hwdev, struct scatterlist *sg,
833 int nelems, int dir)
834{
John W. Linvillede69e0f2005-09-29 14:44:57 -0700835 swiotlb_sync_sg(hwdev, sg, nelems, dir, SYNC_FOR_DEVICE);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700836}
837
Jan Beulich51099002007-02-05 18:53:04 -0800838#ifdef SWIOTLB_ARCH_NEED_MAP_PAGE
839
840dma_addr_t
841swiotlb_map_page(struct device *hwdev, struct page *page,
842 unsigned long offset, size_t size,
843 enum dma_data_direction direction)
844{
845 dma_addr_t dev_addr;
846 char *map;
847
848 dev_addr = page_to_bus(page) + offset;
849 if (address_needs_mapping(hwdev, dev_addr)) {
850 map = map_single(hwdev, page_to_io_tlb_addr(page, offset), size, direction);
851 if (!map) {
852 swiotlb_full(hwdev, size, direction, 1);
853 map = io_tlb_overflow_buffer;
854 }
855 dev_addr = virt_to_bus(map);
856 }
857
858 return dev_addr;
859}
860
861void
862swiotlb_unmap_page(struct device *hwdev, dma_addr_t dev_addr,
863 size_t size, enum dma_data_direction direction)
864{
865 char *dma_addr = bus_to_virt(dev_addr);
866
867 BUG_ON(direction == DMA_NONE);
868 if (dma_addr >= io_tlb_start && dma_addr < io_tlb_end)
869 unmap_single(hwdev, dma_addr, size, direction);
870 else if (direction == DMA_FROM_DEVICE)
871 dma_mark_clean(dma_addr, size);
872}
873
874#endif
875
Linus Torvalds1da177e2005-04-16 15:20:36 -0700876int
877swiotlb_dma_mapping_error(dma_addr_t dma_addr)
878{
Jan Beulich93fbff62007-02-05 18:49:45 -0800879 return (dma_addr == virt_to_bus(io_tlb_overflow_buffer));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700880}
881
882/*
Tony Luck17e5ad62005-09-29 15:52:13 -0700883 * Return whether the given device DMA address mask can be supported
Linus Torvalds1da177e2005-04-16 15:20:36 -0700884 * properly. For example, if your device can only drive the low 24-bits
Tony Luck17e5ad62005-09-29 15:52:13 -0700885 * during bus mastering, then you would pass 0x00ffffff as the mask to
Linus Torvalds1da177e2005-04-16 15:20:36 -0700886 * this function.
887 */
Jan Beulich51099002007-02-05 18:53:04 -0800888#ifndef __swiotlb_dma_supported
889#define __swiotlb_dma_supported(hwdev, mask) (virt_to_bus(io_tlb_end - 1) <= (mask))
890#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -0700891int
Jan Beulich563aaf02007-02-05 18:51:25 -0800892swiotlb_dma_supported(struct device *hwdev, u64 mask)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700893{
Jan Beulich51099002007-02-05 18:53:04 -0800894 return __swiotlb_dma_supported(hwdev, mask);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700895}
896
897EXPORT_SYMBOL(swiotlb_init);
898EXPORT_SYMBOL(swiotlb_map_single);
899EXPORT_SYMBOL(swiotlb_unmap_single);
900EXPORT_SYMBOL(swiotlb_map_sg);
901EXPORT_SYMBOL(swiotlb_unmap_sg);
902EXPORT_SYMBOL(swiotlb_sync_single_for_cpu);
903EXPORT_SYMBOL(swiotlb_sync_single_for_device);
John W. Linville878a97c2005-09-29 14:44:23 -0700904EXPORT_SYMBOL_GPL(swiotlb_sync_single_range_for_cpu);
905EXPORT_SYMBOL_GPL(swiotlb_sync_single_range_for_device);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700906EXPORT_SYMBOL(swiotlb_sync_sg_for_cpu);
907EXPORT_SYMBOL(swiotlb_sync_sg_for_device);
908EXPORT_SYMBOL(swiotlb_dma_mapping_error);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700909EXPORT_SYMBOL(swiotlb_dma_supported);