blob: fa2dc4e5f9baca6a9ae5c71534c8b4caa16557d6 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * Dynamic DMA mapping support.
3 *
Jan Beulich563aaf02007-02-05 18:51:25 -08004 * This implementation is a fallback for platforms that do not support
Linus Torvalds1da177e2005-04-16 15:20:36 -07005 * I/O TLBs (aka DMA address translation hardware).
6 * Copyright (C) 2000 Asit Mallick <Asit.K.Mallick@intel.com>
7 * Copyright (C) 2000 Goutham Rao <goutham.rao@intel.com>
8 * Copyright (C) 2000, 2003 Hewlett-Packard Co
9 * David Mosberger-Tang <davidm@hpl.hp.com>
10 *
11 * 03/05/07 davidm Switch from PCI-DMA to generic device DMA API.
12 * 00/12/13 davidm Rename to swiotlb.c and add mark_clean() to avoid
13 * unnecessary i-cache flushing.
John W. Linville569c8bf2005-09-29 14:45:24 -070014 * 04/07/.. ak Better overflow handling. Assorted fixes.
15 * 05/09/10 linville Add support for syncing ranges, support syncing for
16 * DMA_BIDIRECTIONAL mappings, miscellaneous cleanup.
Linus Torvalds1da177e2005-04-16 15:20:36 -070017 */
18
19#include <linux/cache.h>
Tony Luck17e5ad62005-09-29 15:52:13 -070020#include <linux/dma-mapping.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070021#include <linux/mm.h>
22#include <linux/module.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070023#include <linux/spinlock.h>
Jeremy Fitzhardinge8c5df162008-12-16 12:17:26 -080024#include <linux/swiotlb.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070025#include <linux/string.h>
Ian Campbell0016fde2008-12-16 12:17:27 -080026#include <linux/swiotlb.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070027#include <linux/types.h>
28#include <linux/ctype.h>
Jeremy Fitzhardingeef9b1892008-12-16 12:17:33 -080029#include <linux/highmem.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070030
31#include <asm/io.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070032#include <asm/dma.h>
Tony Luck17e5ad62005-09-29 15:52:13 -070033#include <asm/scatterlist.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070034
35#include <linux/init.h>
36#include <linux/bootmem.h>
FUJITA Tomonoria8522502008-04-29 00:59:36 -070037#include <linux/iommu-helper.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070038
39#define OFFSET(val,align) ((unsigned long) \
40 ( (val) & ( (align) - 1)))
41
Alex Williamson0b9afed2005-09-06 11:20:49 -060042#define SLABS_PER_PAGE (1 << (PAGE_SHIFT - IO_TLB_SHIFT))
43
44/*
45 * Minimum IO TLB size to bother booting with. Systems with mainly
46 * 64bit capable cards will only lightly use the swiotlb. If we can't
47 * allocate a contiguous 1MB, we're probably in trouble anyway.
48 */
49#define IO_TLB_MIN_SLABS ((1<<20) >> IO_TLB_SHIFT)
50
John W. Linvillede69e0f2005-09-29 14:44:57 -070051/*
52 * Enumeration for sync targets
53 */
54enum dma_sync_target {
55 SYNC_FOR_CPU = 0,
56 SYNC_FOR_DEVICE = 1,
57};
58
Linus Torvalds1da177e2005-04-16 15:20:36 -070059int swiotlb_force;
60
61/*
62 * Used to do a quick range check in swiotlb_unmap_single and
63 * swiotlb_sync_single_*, to see if the memory was in fact allocated by this
64 * API.
65 */
66static char *io_tlb_start, *io_tlb_end;
67
68/*
69 * The number of IO TLB blocks (in groups of 64) betweeen io_tlb_start and
70 * io_tlb_end. This is command line adjustable via setup_io_tlb_npages.
71 */
72static unsigned long io_tlb_nslabs;
73
74/*
75 * When the IOMMU overflows we return a fallback buffer. This sets the size.
76 */
77static unsigned long io_tlb_overflow = 32*1024;
78
79void *io_tlb_overflow_buffer;
80
81/*
82 * This is a free list describing the number of free entries available from
83 * each index
84 */
85static unsigned int *io_tlb_list;
86static unsigned int io_tlb_index;
87
88/*
89 * We need to save away the original address corresponding to a mapped entry
90 * for the sync operations.
91 */
Jeremy Fitzhardingeef9b1892008-12-16 12:17:33 -080092static struct swiotlb_phys_addr {
93 struct page *page;
94 unsigned int offset;
95} *io_tlb_orig_addr;
Linus Torvalds1da177e2005-04-16 15:20:36 -070096
97/*
98 * Protect the above data structures in the map and unmap calls
99 */
100static DEFINE_SPINLOCK(io_tlb_lock);
101
102static int __init
103setup_io_tlb_npages(char *str)
104{
105 if (isdigit(*str)) {
Alex Williamsone8579e72005-08-04 13:06:00 -0700106 io_tlb_nslabs = simple_strtoul(str, &str, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700107 /* avoid tail segment of size < IO_TLB_SEGSIZE */
108 io_tlb_nslabs = ALIGN(io_tlb_nslabs, IO_TLB_SEGSIZE);
109 }
110 if (*str == ',')
111 ++str;
112 if (!strcmp(str, "force"))
113 swiotlb_force = 1;
114 return 1;
115}
116__setup("swiotlb=", setup_io_tlb_npages);
117/* make io_tlb_overflow tunable too? */
118
Jeremy Fitzhardinge8c5df162008-12-16 12:17:26 -0800119void * __weak swiotlb_alloc_boot(size_t size, unsigned long nslabs)
120{
121 return alloc_bootmem_low_pages(size);
122}
123
124void * __weak swiotlb_alloc(unsigned order, unsigned long nslabs)
125{
126 return (void *)__get_free_pages(GFP_DMA | __GFP_NOWARN, order);
127}
128
Ian Campbelle08e1f72008-12-16 12:17:30 -0800129dma_addr_t __weak swiotlb_phys_to_bus(phys_addr_t paddr)
130{
131 return paddr;
132}
133
134phys_addr_t __weak swiotlb_bus_to_phys(dma_addr_t baddr)
135{
136 return baddr;
137}
138
139static dma_addr_t swiotlb_virt_to_bus(volatile void *address)
140{
141 return swiotlb_phys_to_bus(virt_to_phys(address));
142}
143
144static void *swiotlb_bus_to_virt(dma_addr_t address)
145{
146 return phys_to_virt(swiotlb_bus_to_phys(address));
147}
148
Ian Campbellb81ea272008-12-16 12:17:31 -0800149int __weak swiotlb_arch_range_needs_mapping(void *ptr, size_t size)
150{
151 return 0;
152}
153
Jeremy Fitzhardingeef9b1892008-12-16 12:17:33 -0800154static dma_addr_t swiotlb_sg_to_bus(struct scatterlist *sg)
155{
156 return swiotlb_phys_to_bus(page_to_phys(sg_page(sg)) + sg->offset);
157}
158
Ian Campbell2e5b2b82008-12-16 12:17:34 -0800159static void swiotlb_print_info(unsigned long bytes)
160{
161 phys_addr_t pstart, pend;
162 dma_addr_t bstart, bend;
163
164 pstart = virt_to_phys(io_tlb_start);
165 pend = virt_to_phys(io_tlb_end);
166
167 bstart = swiotlb_phys_to_bus(pstart);
168 bend = swiotlb_phys_to_bus(pend);
169
170 printk(KERN_INFO "Placing %luMB software IO TLB between %p - %p\n",
171 bytes >> 20, io_tlb_start, io_tlb_end);
172 if (pstart != bstart || pend != bend)
173 printk(KERN_INFO "software IO TLB at phys %#llx - %#llx"
174 " bus %#llx - %#llx\n",
175 (unsigned long long)pstart,
176 (unsigned long long)pend,
177 (unsigned long long)bstart,
178 (unsigned long long)bend);
179 else
180 printk(KERN_INFO "software IO TLB at phys %#llx - %#llx\n",
181 (unsigned long long)pstart,
182 (unsigned long long)pend);
183}
184
Linus Torvalds1da177e2005-04-16 15:20:36 -0700185/*
186 * Statically reserve bounce buffer space and initialize bounce buffer data
Tony Luck17e5ad62005-09-29 15:52:13 -0700187 * structures for the software IO TLB used to implement the DMA API.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700188 */
Jan Beulich563aaf02007-02-05 18:51:25 -0800189void __init
190swiotlb_init_with_default_size(size_t default_size)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700191{
Jan Beulich563aaf02007-02-05 18:51:25 -0800192 unsigned long i, bytes;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700193
194 if (!io_tlb_nslabs) {
Alex Williamsone8579e72005-08-04 13:06:00 -0700195 io_tlb_nslabs = (default_size >> IO_TLB_SHIFT);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700196 io_tlb_nslabs = ALIGN(io_tlb_nslabs, IO_TLB_SEGSIZE);
197 }
198
Jan Beulich563aaf02007-02-05 18:51:25 -0800199 bytes = io_tlb_nslabs << IO_TLB_SHIFT;
200
Linus Torvalds1da177e2005-04-16 15:20:36 -0700201 /*
202 * Get IO TLB memory from the low pages
203 */
Jeremy Fitzhardinge8c5df162008-12-16 12:17:26 -0800204 io_tlb_start = swiotlb_alloc_boot(bytes, io_tlb_nslabs);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700205 if (!io_tlb_start)
206 panic("Cannot allocate SWIOTLB buffer");
Jan Beulich563aaf02007-02-05 18:51:25 -0800207 io_tlb_end = io_tlb_start + bytes;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700208
209 /*
210 * Allocate and initialize the free list array. This array is used
211 * to find contiguous free memory regions of size up to IO_TLB_SEGSIZE
212 * between io_tlb_start and io_tlb_end.
213 */
214 io_tlb_list = alloc_bootmem(io_tlb_nslabs * sizeof(int));
Tony Luck25667d62007-03-06 13:31:45 -0800215 for (i = 0; i < io_tlb_nslabs; i++)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700216 io_tlb_list[i] = IO_TLB_SEGSIZE - OFFSET(i, IO_TLB_SEGSIZE);
217 io_tlb_index = 0;
Jeremy Fitzhardingeef9b1892008-12-16 12:17:33 -0800218 io_tlb_orig_addr = alloc_bootmem(io_tlb_nslabs * sizeof(struct swiotlb_phys_addr));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700219
220 /*
221 * Get the overflow emergency buffer
222 */
223 io_tlb_overflow_buffer = alloc_bootmem_low(io_tlb_overflow);
Jan Beulich563aaf02007-02-05 18:51:25 -0800224 if (!io_tlb_overflow_buffer)
225 panic("Cannot allocate SWIOTLB overflow buffer!\n");
226
Ian Campbell2e5b2b82008-12-16 12:17:34 -0800227 swiotlb_print_info(bytes);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700228}
229
Jan Beulich563aaf02007-02-05 18:51:25 -0800230void __init
231swiotlb_init(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700232{
Tony Luck25667d62007-03-06 13:31:45 -0800233 swiotlb_init_with_default_size(64 * (1<<20)); /* default to 64MB */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700234}
235
Alex Williamson0b9afed2005-09-06 11:20:49 -0600236/*
237 * Systems with larger DMA zones (those that don't support ISA) can
238 * initialize the swiotlb later using the slab allocator if needed.
239 * This should be just like above, but with some error catching.
240 */
241int
Jan Beulich563aaf02007-02-05 18:51:25 -0800242swiotlb_late_init_with_default_size(size_t default_size)
Alex Williamson0b9afed2005-09-06 11:20:49 -0600243{
Jan Beulich563aaf02007-02-05 18:51:25 -0800244 unsigned long i, bytes, req_nslabs = io_tlb_nslabs;
Alex Williamson0b9afed2005-09-06 11:20:49 -0600245 unsigned int order;
246
247 if (!io_tlb_nslabs) {
248 io_tlb_nslabs = (default_size >> IO_TLB_SHIFT);
249 io_tlb_nslabs = ALIGN(io_tlb_nslabs, IO_TLB_SEGSIZE);
250 }
251
252 /*
253 * Get IO TLB memory from the low pages
254 */
Jan Beulich563aaf02007-02-05 18:51:25 -0800255 order = get_order(io_tlb_nslabs << IO_TLB_SHIFT);
Alex Williamson0b9afed2005-09-06 11:20:49 -0600256 io_tlb_nslabs = SLABS_PER_PAGE << order;
Jan Beulich563aaf02007-02-05 18:51:25 -0800257 bytes = io_tlb_nslabs << IO_TLB_SHIFT;
Alex Williamson0b9afed2005-09-06 11:20:49 -0600258
259 while ((SLABS_PER_PAGE << order) > IO_TLB_MIN_SLABS) {
Jeremy Fitzhardinge8c5df162008-12-16 12:17:26 -0800260 io_tlb_start = swiotlb_alloc(order, io_tlb_nslabs);
Alex Williamson0b9afed2005-09-06 11:20:49 -0600261 if (io_tlb_start)
262 break;
263 order--;
264 }
265
266 if (!io_tlb_start)
267 goto cleanup1;
268
Jan Beulich563aaf02007-02-05 18:51:25 -0800269 if (order != get_order(bytes)) {
Alex Williamson0b9afed2005-09-06 11:20:49 -0600270 printk(KERN_WARNING "Warning: only able to allocate %ld MB "
271 "for software IO TLB\n", (PAGE_SIZE << order) >> 20);
272 io_tlb_nslabs = SLABS_PER_PAGE << order;
Jan Beulich563aaf02007-02-05 18:51:25 -0800273 bytes = io_tlb_nslabs << IO_TLB_SHIFT;
Alex Williamson0b9afed2005-09-06 11:20:49 -0600274 }
Jan Beulich563aaf02007-02-05 18:51:25 -0800275 io_tlb_end = io_tlb_start + bytes;
276 memset(io_tlb_start, 0, bytes);
Alex Williamson0b9afed2005-09-06 11:20:49 -0600277
278 /*
279 * Allocate and initialize the free list array. This array is used
280 * to find contiguous free memory regions of size up to IO_TLB_SEGSIZE
281 * between io_tlb_start and io_tlb_end.
282 */
283 io_tlb_list = (unsigned int *)__get_free_pages(GFP_KERNEL,
284 get_order(io_tlb_nslabs * sizeof(int)));
285 if (!io_tlb_list)
286 goto cleanup2;
287
288 for (i = 0; i < io_tlb_nslabs; i++)
289 io_tlb_list[i] = IO_TLB_SEGSIZE - OFFSET(i, IO_TLB_SEGSIZE);
290 io_tlb_index = 0;
291
Jeremy Fitzhardingeef9b1892008-12-16 12:17:33 -0800292 io_tlb_orig_addr = (struct swiotlb_phys_addr *)__get_free_pages(GFP_KERNEL,
293 get_order(io_tlb_nslabs * sizeof(struct swiotlb_phys_addr)));
Alex Williamson0b9afed2005-09-06 11:20:49 -0600294 if (!io_tlb_orig_addr)
295 goto cleanup3;
296
Jeremy Fitzhardingeef9b1892008-12-16 12:17:33 -0800297 memset(io_tlb_orig_addr, 0, io_tlb_nslabs * sizeof(struct swiotlb_phys_addr));
Alex Williamson0b9afed2005-09-06 11:20:49 -0600298
299 /*
300 * Get the overflow emergency buffer
301 */
302 io_tlb_overflow_buffer = (void *)__get_free_pages(GFP_DMA,
303 get_order(io_tlb_overflow));
304 if (!io_tlb_overflow_buffer)
305 goto cleanup4;
306
Ian Campbell2e5b2b82008-12-16 12:17:34 -0800307 swiotlb_print_info(bytes);
Alex Williamson0b9afed2005-09-06 11:20:49 -0600308
309 return 0;
310
311cleanup4:
Tony Luck25667d62007-03-06 13:31:45 -0800312 free_pages((unsigned long)io_tlb_orig_addr, get_order(io_tlb_nslabs *
313 sizeof(char *)));
Alex Williamson0b9afed2005-09-06 11:20:49 -0600314 io_tlb_orig_addr = NULL;
315cleanup3:
Tony Luck25667d62007-03-06 13:31:45 -0800316 free_pages((unsigned long)io_tlb_list, get_order(io_tlb_nslabs *
317 sizeof(int)));
Alex Williamson0b9afed2005-09-06 11:20:49 -0600318 io_tlb_list = NULL;
Alex Williamson0b9afed2005-09-06 11:20:49 -0600319cleanup2:
Jan Beulich563aaf02007-02-05 18:51:25 -0800320 io_tlb_end = NULL;
Alex Williamson0b9afed2005-09-06 11:20:49 -0600321 free_pages((unsigned long)io_tlb_start, order);
322 io_tlb_start = NULL;
323cleanup1:
324 io_tlb_nslabs = req_nslabs;
325 return -ENOMEM;
326}
327
Andrew Mortonbe6b0262007-02-12 00:52:17 -0800328static int
FUJITA Tomonori27979822008-09-10 01:06:49 +0900329address_needs_mapping(struct device *hwdev, dma_addr_t addr, size_t size)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700330{
FUJITA Tomonori07a2c012008-09-19 02:02:05 +0900331 return !is_buffer_dma_capable(dma_get_mask(hwdev), addr, size);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700332}
333
Ian Campbellb81ea272008-12-16 12:17:31 -0800334static inline int range_needs_mapping(void *ptr, size_t size)
335{
336 return swiotlb_force || swiotlb_arch_range_needs_mapping(ptr, size);
337}
338
FUJITA Tomonori640aebf2008-09-08 18:53:50 +0900339static int is_swiotlb_buffer(char *addr)
340{
341 return addr >= io_tlb_start && addr < io_tlb_end;
342}
343
Jeremy Fitzhardingeef9b1892008-12-16 12:17:33 -0800344static struct swiotlb_phys_addr swiotlb_bus_to_phys_addr(char *dma_addr)
Jeremy Fitzhardinge1b548f62008-12-16 12:17:32 -0800345{
Jeremy Fitzhardingeef9b1892008-12-16 12:17:33 -0800346 int index = (dma_addr - io_tlb_start) >> IO_TLB_SHIFT;
347 struct swiotlb_phys_addr buffer = io_tlb_orig_addr[index];
348 buffer.offset += (long)dma_addr & ((1 << IO_TLB_SHIFT) - 1);
349 buffer.page += buffer.offset >> PAGE_SHIFT;
350 buffer.offset &= PAGE_SIZE - 1;
351 return buffer;
352}
353
354static void
355__sync_single(struct swiotlb_phys_addr buffer, char *dma_addr, size_t size, int dir)
356{
357 if (PageHighMem(buffer.page)) {
358 size_t len, bytes;
359 char *dev, *host, *kmp;
360
361 len = size;
362 while (len != 0) {
363 unsigned long flags;
364
365 bytes = len;
366 if ((bytes + buffer.offset) > PAGE_SIZE)
367 bytes = PAGE_SIZE - buffer.offset;
368 local_irq_save(flags); /* protects KM_BOUNCE_READ */
369 kmp = kmap_atomic(buffer.page, KM_BOUNCE_READ);
370 dev = dma_addr + size - len;
371 host = kmp + buffer.offset;
372 if (dir == DMA_FROM_DEVICE)
373 memcpy(host, dev, bytes);
374 else
375 memcpy(dev, host, bytes);
376 kunmap_atomic(kmp, KM_BOUNCE_READ);
377 local_irq_restore(flags);
378 len -= bytes;
379 buffer.page++;
380 buffer.offset = 0;
381 }
382 } else {
383 void *v = page_address(buffer.page) + buffer.offset;
384
385 if (dir == DMA_TO_DEVICE)
386 memcpy(dma_addr, v, size);
387 else
388 memcpy(v, dma_addr, size);
389 }
Jeremy Fitzhardinge1b548f62008-12-16 12:17:32 -0800390}
391
Linus Torvalds1da177e2005-04-16 15:20:36 -0700392/*
393 * Allocates bounce buffer and returns its kernel virtual address.
394 */
395static void *
Jeremy Fitzhardingeef9b1892008-12-16 12:17:33 -0800396map_single(struct device *hwdev, struct swiotlb_phys_addr buffer, size_t size, int dir)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700397{
398 unsigned long flags;
399 char *dma_addr;
400 unsigned int nslots, stride, index, wrap;
401 int i;
FUJITA Tomonori681cc5c2008-02-04 22:28:16 -0800402 unsigned long start_dma_addr;
403 unsigned long mask;
404 unsigned long offset_slots;
405 unsigned long max_slots;
Jeremy Fitzhardingeef9b1892008-12-16 12:17:33 -0800406 struct swiotlb_phys_addr slot_buf;
FUJITA Tomonori681cc5c2008-02-04 22:28:16 -0800407
408 mask = dma_get_seg_boundary(hwdev);
Ian Campbelle08e1f72008-12-16 12:17:30 -0800409 start_dma_addr = swiotlb_virt_to_bus(io_tlb_start) & mask;
FUJITA Tomonori681cc5c2008-02-04 22:28:16 -0800410
411 offset_slots = ALIGN(start_dma_addr, 1 << IO_TLB_SHIFT) >> IO_TLB_SHIFT;
Ian Campbella5ddde42008-12-16 12:17:29 -0800412
413 /*
414 * Carefully handle integer overflow which can occur when mask == ~0UL.
415 */
Jan Beulichb15a3892008-03-13 09:13:30 +0000416 max_slots = mask + 1
417 ? ALIGN(mask + 1, 1 << IO_TLB_SHIFT) >> IO_TLB_SHIFT
418 : 1UL << (BITS_PER_LONG - IO_TLB_SHIFT);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700419
420 /*
421 * For mappings greater than a page, we limit the stride (and
422 * hence alignment) to a page size.
423 */
424 nslots = ALIGN(size, 1 << IO_TLB_SHIFT) >> IO_TLB_SHIFT;
425 if (size > PAGE_SIZE)
426 stride = (1 << (PAGE_SHIFT - IO_TLB_SHIFT));
427 else
428 stride = 1;
429
Eric Sesterhenn34814542006-03-24 18:47:11 +0100430 BUG_ON(!nslots);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700431
432 /*
433 * Find suitable number of IO TLB entries size that will fit this
434 * request and allocate a buffer from that IO TLB pool.
435 */
436 spin_lock_irqsave(&io_tlb_lock, flags);
Andrew Mortona7133a12008-04-29 00:59:36 -0700437 index = ALIGN(io_tlb_index, stride);
438 if (index >= io_tlb_nslabs)
439 index = 0;
440 wrap = index;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700441
Andrew Mortona7133a12008-04-29 00:59:36 -0700442 do {
FUJITA Tomonoria8522502008-04-29 00:59:36 -0700443 while (iommu_is_span_boundary(index, nslots, offset_slots,
444 max_slots)) {
Jan Beulichb15a3892008-03-13 09:13:30 +0000445 index += stride;
446 if (index >= io_tlb_nslabs)
447 index = 0;
Andrew Mortona7133a12008-04-29 00:59:36 -0700448 if (index == wrap)
449 goto not_found;
450 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700451
Andrew Mortona7133a12008-04-29 00:59:36 -0700452 /*
453 * If we find a slot that indicates we have 'nslots' number of
454 * contiguous buffers, we allocate the buffers from that slot
455 * and mark the entries as '0' indicating unavailable.
456 */
457 if (io_tlb_list[index] >= nslots) {
458 int count = 0;
459
460 for (i = index; i < (int) (index + nslots); i++)
461 io_tlb_list[i] = 0;
462 for (i = index - 1; (OFFSET(i, IO_TLB_SEGSIZE) != IO_TLB_SEGSIZE - 1) && io_tlb_list[i]; i--)
463 io_tlb_list[i] = ++count;
464 dma_addr = io_tlb_start + (index << IO_TLB_SHIFT);
465
466 /*
467 * Update the indices to avoid searching in the next
468 * round.
469 */
470 io_tlb_index = ((index + nslots) < io_tlb_nslabs
471 ? (index + nslots) : 0);
472
473 goto found;
474 }
475 index += stride;
476 if (index >= io_tlb_nslabs)
477 index = 0;
478 } while (index != wrap);
479
480not_found:
481 spin_unlock_irqrestore(&io_tlb_lock, flags);
482 return NULL;
483found:
Linus Torvalds1da177e2005-04-16 15:20:36 -0700484 spin_unlock_irqrestore(&io_tlb_lock, flags);
485
486 /*
487 * Save away the mapping from the original address to the DMA address.
488 * This is needed when we sync the memory. Then we sync the buffer if
489 * needed.
490 */
Jeremy Fitzhardingeef9b1892008-12-16 12:17:33 -0800491 slot_buf = buffer;
492 for (i = 0; i < nslots; i++) {
493 slot_buf.page += slot_buf.offset >> PAGE_SHIFT;
494 slot_buf.offset &= PAGE_SIZE - 1;
495 io_tlb_orig_addr[index+i] = slot_buf;
496 slot_buf.offset += 1 << IO_TLB_SHIFT;
497 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700498 if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL)
Jeremy Fitzhardinge1b548f62008-12-16 12:17:32 -0800499 __sync_single(buffer, dma_addr, size, DMA_TO_DEVICE);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700500
501 return dma_addr;
502}
503
504/*
505 * dma_addr is the kernel virtual address of the bounce buffer to unmap.
506 */
507static void
508unmap_single(struct device *hwdev, char *dma_addr, size_t size, int dir)
509{
510 unsigned long flags;
511 int i, count, nslots = ALIGN(size, 1 << IO_TLB_SHIFT) >> IO_TLB_SHIFT;
512 int index = (dma_addr - io_tlb_start) >> IO_TLB_SHIFT;
Jeremy Fitzhardingeef9b1892008-12-16 12:17:33 -0800513 struct swiotlb_phys_addr buffer = swiotlb_bus_to_phys_addr(dma_addr);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700514
515 /*
516 * First, sync the memory before unmapping the entry
517 */
Jeremy Fitzhardingeef9b1892008-12-16 12:17:33 -0800518 if ((dir == DMA_FROM_DEVICE) || (dir == DMA_BIDIRECTIONAL))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700519 /*
520 * bounce... copy the data back into the original buffer * and
521 * delete the bounce buffer.
522 */
Jeremy Fitzhardinge1b548f62008-12-16 12:17:32 -0800523 __sync_single(buffer, dma_addr, size, DMA_FROM_DEVICE);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700524
525 /*
526 * Return the buffer to the free list by setting the corresponding
527 * entries to indicate the number of contigous entries available.
528 * While returning the entries to the free list, we merge the entries
529 * with slots below and above the pool being returned.
530 */
531 spin_lock_irqsave(&io_tlb_lock, flags);
532 {
533 count = ((index + nslots) < ALIGN(index + 1, IO_TLB_SEGSIZE) ?
534 io_tlb_list[index + nslots] : 0);
535 /*
536 * Step 1: return the slots to the free list, merging the
537 * slots with superceeding slots
538 */
539 for (i = index + nslots - 1; i >= index; i--)
540 io_tlb_list[i] = ++count;
541 /*
542 * Step 2: merge the returned slots with the preceding slots,
543 * if available (non zero)
544 */
545 for (i = index - 1; (OFFSET(i, IO_TLB_SEGSIZE) != IO_TLB_SEGSIZE -1) && io_tlb_list[i]; i--)
546 io_tlb_list[i] = ++count;
547 }
548 spin_unlock_irqrestore(&io_tlb_lock, flags);
549}
550
551static void
John W. Linvillede69e0f2005-09-29 14:44:57 -0700552sync_single(struct device *hwdev, char *dma_addr, size_t size,
553 int dir, int target)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700554{
Jeremy Fitzhardingeef9b1892008-12-16 12:17:33 -0800555 struct swiotlb_phys_addr buffer = swiotlb_bus_to_phys_addr(dma_addr);
Keir Fraserdf336d12007-07-21 04:37:24 -0700556
John W. Linvillede69e0f2005-09-29 14:44:57 -0700557 switch (target) {
558 case SYNC_FOR_CPU:
559 if (likely(dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL))
Jeremy Fitzhardinge1b548f62008-12-16 12:17:32 -0800560 __sync_single(buffer, dma_addr, size, DMA_FROM_DEVICE);
Eric Sesterhenn34814542006-03-24 18:47:11 +0100561 else
562 BUG_ON(dir != DMA_TO_DEVICE);
John W. Linvillede69e0f2005-09-29 14:44:57 -0700563 break;
564 case SYNC_FOR_DEVICE:
565 if (likely(dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL))
Jeremy Fitzhardinge1b548f62008-12-16 12:17:32 -0800566 __sync_single(buffer, dma_addr, size, DMA_TO_DEVICE);
Eric Sesterhenn34814542006-03-24 18:47:11 +0100567 else
568 BUG_ON(dir != DMA_FROM_DEVICE);
John W. Linvillede69e0f2005-09-29 14:44:57 -0700569 break;
570 default:
Linus Torvalds1da177e2005-04-16 15:20:36 -0700571 BUG();
John W. Linvillede69e0f2005-09-29 14:44:57 -0700572 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700573}
574
575void *
576swiotlb_alloc_coherent(struct device *hwdev, size_t size,
Al Viro06a54492005-10-21 03:21:03 -0400577 dma_addr_t *dma_handle, gfp_t flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700578{
Jan Beulich563aaf02007-02-05 18:51:25 -0800579 dma_addr_t dev_addr;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700580 void *ret;
581 int order = get_order(size);
FUJITA Tomonori1e74f302008-11-17 16:24:34 +0900582 u64 dma_mask = DMA_32BIT_MASK;
583
584 if (hwdev && hwdev->coherent_dma_mask)
585 dma_mask = hwdev->coherent_dma_mask;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700586
Tony Luck25667d62007-03-06 13:31:45 -0800587 ret = (void *)__get_free_pages(flags, order);
Ian Campbelle08e1f72008-12-16 12:17:30 -0800588 if (ret && !is_buffer_dma_capable(dma_mask, swiotlb_virt_to_bus(ret), size)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700589 /*
590 * The allocated memory isn't reachable by the device.
591 * Fall back on swiotlb_map_single().
592 */
593 free_pages((unsigned long) ret, order);
594 ret = NULL;
595 }
596 if (!ret) {
597 /*
598 * We are either out of memory or the device can't DMA
599 * to GFP_DMA memory; fall back on
600 * swiotlb_map_single(), which will grab memory from
601 * the lowest available address range.
602 */
Jeremy Fitzhardingeef9b1892008-12-16 12:17:33 -0800603 struct swiotlb_phys_addr buffer;
604 buffer.page = virt_to_page(NULL);
605 buffer.offset = 0;
606 ret = map_single(hwdev, buffer, size, DMA_FROM_DEVICE);
FUJITA Tomonori9dfda122008-09-08 18:53:48 +0900607 if (!ret)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700608 return NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700609 }
610
611 memset(ret, 0, size);
Ian Campbelle08e1f72008-12-16 12:17:30 -0800612 dev_addr = swiotlb_virt_to_bus(ret);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700613
614 /* Confirm address can be DMA'd by device */
FUJITA Tomonori1e74f302008-11-17 16:24:34 +0900615 if (!is_buffer_dma_capable(dma_mask, dev_addr, size)) {
Jan Beulich563aaf02007-02-05 18:51:25 -0800616 printk("hwdev DMA mask = 0x%016Lx, dev_addr = 0x%016Lx\n",
FUJITA Tomonori1e74f302008-11-17 16:24:34 +0900617 (unsigned long long)dma_mask,
Jan Beulich563aaf02007-02-05 18:51:25 -0800618 (unsigned long long)dev_addr);
FUJITA Tomonoria2b89b52008-10-23 18:42:03 +0900619
620 /* DMA_TO_DEVICE to avoid memcpy in unmap_single */
621 unmap_single(hwdev, ret, size, DMA_TO_DEVICE);
622 return NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700623 }
624 *dma_handle = dev_addr;
625 return ret;
626}
627
628void
629swiotlb_free_coherent(struct device *hwdev, size_t size, void *vaddr,
630 dma_addr_t dma_handle)
631{
David Brownellaa248862007-08-10 13:10:27 -0700632 WARN_ON(irqs_disabled());
FUJITA Tomonori640aebf2008-09-08 18:53:50 +0900633 if (!is_swiotlb_buffer(vaddr))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700634 free_pages((unsigned long) vaddr, get_order(size));
635 else
636 /* DMA_TO_DEVICE to avoid memcpy in unmap_single */
FUJITA Tomonori21f6c4d2008-09-08 18:53:49 +0900637 unmap_single(hwdev, vaddr, size, DMA_TO_DEVICE);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700638}
639
640static void
641swiotlb_full(struct device *dev, size_t size, int dir, int do_panic)
642{
643 /*
644 * Ran out of IOMMU space for this operation. This is very bad.
645 * Unfortunately the drivers cannot handle this operation properly.
Tony Luck17e5ad62005-09-29 15:52:13 -0700646 * unless they check for dma_mapping_error (most don't)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700647 * When the mapping is small enough return a static buffer to limit
648 * the damage, or panic when the transfer is too big.
649 */
Jan Beulich563aaf02007-02-05 18:51:25 -0800650 printk(KERN_ERR "DMA: Out of SW-IOMMU space for %zu bytes at "
Linus Torvalds1da177e2005-04-16 15:20:36 -0700651 "device %s\n", size, dev ? dev->bus_id : "?");
652
653 if (size > io_tlb_overflow && do_panic) {
Tony Luck17e5ad62005-09-29 15:52:13 -0700654 if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
655 panic("DMA: Memory would be corrupted\n");
656 if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL)
657 panic("DMA: Random memory would be DMAed\n");
Linus Torvalds1da177e2005-04-16 15:20:36 -0700658 }
659}
660
661/*
662 * Map a single buffer of the indicated size for DMA in streaming mode. The
Tony Luck17e5ad62005-09-29 15:52:13 -0700663 * physical address to use is returned.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700664 *
665 * Once the device is given the dma address, the device owns this memory until
666 * either swiotlb_unmap_single or swiotlb_dma_sync_single is performed.
667 */
668dma_addr_t
Arthur Kepner309df0c2008-04-29 01:00:32 -0700669swiotlb_map_single_attrs(struct device *hwdev, void *ptr, size_t size,
670 int dir, struct dma_attrs *attrs)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700671{
Ian Campbelle08e1f72008-12-16 12:17:30 -0800672 dma_addr_t dev_addr = swiotlb_virt_to_bus(ptr);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700673 void *map;
Jeremy Fitzhardingeef9b1892008-12-16 12:17:33 -0800674 struct swiotlb_phys_addr buffer;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700675
Eric Sesterhenn34814542006-03-24 18:47:11 +0100676 BUG_ON(dir == DMA_NONE);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700677 /*
678 * If the pointer passed in happens to be in the device's DMA window,
679 * we can safely return the device addr and not worry about bounce
680 * buffering it.
681 */
Ian Campbellb81ea272008-12-16 12:17:31 -0800682 if (!address_needs_mapping(hwdev, dev_addr, size) &&
683 !range_needs_mapping(ptr, size))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700684 return dev_addr;
685
686 /*
687 * Oh well, have to allocate and map a bounce buffer.
688 */
Jeremy Fitzhardingeef9b1892008-12-16 12:17:33 -0800689 buffer.page = virt_to_page(ptr);
690 buffer.offset = (unsigned long)ptr & ~PAGE_MASK;
691 map = map_single(hwdev, buffer, size, dir);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700692 if (!map) {
693 swiotlb_full(hwdev, size, dir, 1);
694 map = io_tlb_overflow_buffer;
695 }
696
Ian Campbelle08e1f72008-12-16 12:17:30 -0800697 dev_addr = swiotlb_virt_to_bus(map);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700698
699 /*
700 * Ensure that the address returned is DMA'ble
701 */
FUJITA Tomonori27979822008-09-10 01:06:49 +0900702 if (address_needs_mapping(hwdev, dev_addr, size))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700703 panic("map_single: bounce buffer is not DMA'ble");
704
705 return dev_addr;
706}
Arthur Kepner309df0c2008-04-29 01:00:32 -0700707EXPORT_SYMBOL(swiotlb_map_single_attrs);
708
709dma_addr_t
710swiotlb_map_single(struct device *hwdev, void *ptr, size_t size, int dir)
711{
712 return swiotlb_map_single_attrs(hwdev, ptr, size, dir, NULL);
713}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700714
715/*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700716 * Unmap a single streaming mode DMA translation. The dma_addr and size must
717 * match what was provided for in a previous swiotlb_map_single call. All
718 * other usages are undefined.
719 *
720 * After this call, reads by the cpu to the buffer are guaranteed to see
721 * whatever the device wrote there.
722 */
723void
Arthur Kepner309df0c2008-04-29 01:00:32 -0700724swiotlb_unmap_single_attrs(struct device *hwdev, dma_addr_t dev_addr,
725 size_t size, int dir, struct dma_attrs *attrs)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700726{
Ian Campbelle08e1f72008-12-16 12:17:30 -0800727 char *dma_addr = swiotlb_bus_to_virt(dev_addr);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700728
Eric Sesterhenn34814542006-03-24 18:47:11 +0100729 BUG_ON(dir == DMA_NONE);
FUJITA Tomonori640aebf2008-09-08 18:53:50 +0900730 if (is_swiotlb_buffer(dma_addr))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700731 unmap_single(hwdev, dma_addr, size, dir);
732 else if (dir == DMA_FROM_DEVICE)
Jan Beulichcde14bb2007-02-05 18:46:40 -0800733 dma_mark_clean(dma_addr, size);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700734}
Arthur Kepner309df0c2008-04-29 01:00:32 -0700735EXPORT_SYMBOL(swiotlb_unmap_single_attrs);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700736
Arthur Kepner309df0c2008-04-29 01:00:32 -0700737void
738swiotlb_unmap_single(struct device *hwdev, dma_addr_t dev_addr, size_t size,
739 int dir)
740{
741 return swiotlb_unmap_single_attrs(hwdev, dev_addr, size, dir, NULL);
742}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700743/*
744 * Make physical memory consistent for a single streaming mode DMA translation
745 * after a transfer.
746 *
747 * If you perform a swiotlb_map_single() but wish to interrogate the buffer
Tony Luck17e5ad62005-09-29 15:52:13 -0700748 * using the cpu, yet do not wish to teardown the dma mapping, you must
749 * call this function before doing so. At the next point you give the dma
Linus Torvalds1da177e2005-04-16 15:20:36 -0700750 * address back to the card, you must first perform a
751 * swiotlb_dma_sync_for_device, and then the device again owns the buffer
752 */
Andrew Mortonbe6b0262007-02-12 00:52:17 -0800753static void
John W. Linville8270f3f2005-09-29 14:43:32 -0700754swiotlb_sync_single(struct device *hwdev, dma_addr_t dev_addr,
John W. Linvillede69e0f2005-09-29 14:44:57 -0700755 size_t size, int dir, int target)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700756{
Ian Campbelle08e1f72008-12-16 12:17:30 -0800757 char *dma_addr = swiotlb_bus_to_virt(dev_addr);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700758
Eric Sesterhenn34814542006-03-24 18:47:11 +0100759 BUG_ON(dir == DMA_NONE);
FUJITA Tomonori640aebf2008-09-08 18:53:50 +0900760 if (is_swiotlb_buffer(dma_addr))
John W. Linvillede69e0f2005-09-29 14:44:57 -0700761 sync_single(hwdev, dma_addr, size, dir, target);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700762 else if (dir == DMA_FROM_DEVICE)
Jan Beulichcde14bb2007-02-05 18:46:40 -0800763 dma_mark_clean(dma_addr, size);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700764}
765
766void
John W. Linville8270f3f2005-09-29 14:43:32 -0700767swiotlb_sync_single_for_cpu(struct device *hwdev, dma_addr_t dev_addr,
768 size_t size, int dir)
769{
John W. Linvillede69e0f2005-09-29 14:44:57 -0700770 swiotlb_sync_single(hwdev, dev_addr, size, dir, SYNC_FOR_CPU);
John W. Linville8270f3f2005-09-29 14:43:32 -0700771}
772
773void
Linus Torvalds1da177e2005-04-16 15:20:36 -0700774swiotlb_sync_single_for_device(struct device *hwdev, dma_addr_t dev_addr,
775 size_t size, int dir)
776{
John W. Linvillede69e0f2005-09-29 14:44:57 -0700777 swiotlb_sync_single(hwdev, dev_addr, size, dir, SYNC_FOR_DEVICE);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700778}
779
780/*
John W. Linville878a97c2005-09-29 14:44:23 -0700781 * Same as above, but for a sub-range of the mapping.
782 */
Andrew Mortonbe6b0262007-02-12 00:52:17 -0800783static void
John W. Linville878a97c2005-09-29 14:44:23 -0700784swiotlb_sync_single_range(struct device *hwdev, dma_addr_t dev_addr,
John W. Linvillede69e0f2005-09-29 14:44:57 -0700785 unsigned long offset, size_t size,
786 int dir, int target)
John W. Linville878a97c2005-09-29 14:44:23 -0700787{
Ian Campbelle08e1f72008-12-16 12:17:30 -0800788 char *dma_addr = swiotlb_bus_to_virt(dev_addr) + offset;
John W. Linville878a97c2005-09-29 14:44:23 -0700789
Eric Sesterhenn34814542006-03-24 18:47:11 +0100790 BUG_ON(dir == DMA_NONE);
FUJITA Tomonori640aebf2008-09-08 18:53:50 +0900791 if (is_swiotlb_buffer(dma_addr))
John W. Linvillede69e0f2005-09-29 14:44:57 -0700792 sync_single(hwdev, dma_addr, size, dir, target);
John W. Linville878a97c2005-09-29 14:44:23 -0700793 else if (dir == DMA_FROM_DEVICE)
Jan Beulichcde14bb2007-02-05 18:46:40 -0800794 dma_mark_clean(dma_addr, size);
John W. Linville878a97c2005-09-29 14:44:23 -0700795}
796
797void
798swiotlb_sync_single_range_for_cpu(struct device *hwdev, dma_addr_t dev_addr,
799 unsigned long offset, size_t size, int dir)
800{
John W. Linvillede69e0f2005-09-29 14:44:57 -0700801 swiotlb_sync_single_range(hwdev, dev_addr, offset, size, dir,
802 SYNC_FOR_CPU);
John W. Linville878a97c2005-09-29 14:44:23 -0700803}
804
805void
806swiotlb_sync_single_range_for_device(struct device *hwdev, dma_addr_t dev_addr,
807 unsigned long offset, size_t size, int dir)
808{
John W. Linvillede69e0f2005-09-29 14:44:57 -0700809 swiotlb_sync_single_range(hwdev, dev_addr, offset, size, dir,
810 SYNC_FOR_DEVICE);
John W. Linville878a97c2005-09-29 14:44:23 -0700811}
812
Arthur Kepner309df0c2008-04-29 01:00:32 -0700813void swiotlb_unmap_sg_attrs(struct device *, struct scatterlist *, int, int,
814 struct dma_attrs *);
John W. Linville878a97c2005-09-29 14:44:23 -0700815/*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700816 * Map a set of buffers described by scatterlist in streaming mode for DMA.
817 * This is the scatter-gather version of the above swiotlb_map_single
818 * interface. Here the scatter gather list elements are each tagged with the
819 * appropriate dma address and length. They are obtained via
820 * sg_dma_{address,length}(SG).
821 *
822 * NOTE: An implementation may be able to use a smaller number of
823 * DMA address/length pairs than there are SG table elements.
824 * (for example via virtual mapping capabilities)
825 * The routine returns the number of addr/length pairs actually
826 * used, at most nents.
827 *
828 * Device ownership issues as mentioned above for swiotlb_map_single are the
829 * same here.
830 */
831int
Arthur Kepner309df0c2008-04-29 01:00:32 -0700832swiotlb_map_sg_attrs(struct device *hwdev, struct scatterlist *sgl, int nelems,
833 int dir, struct dma_attrs *attrs)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700834{
Jens Axboedbfd49f2007-05-11 14:56:18 +0200835 struct scatterlist *sg;
Jeremy Fitzhardingeef9b1892008-12-16 12:17:33 -0800836 struct swiotlb_phys_addr buffer;
Jan Beulich563aaf02007-02-05 18:51:25 -0800837 dma_addr_t dev_addr;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700838 int i;
839
Eric Sesterhenn34814542006-03-24 18:47:11 +0100840 BUG_ON(dir == DMA_NONE);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700841
Jens Axboedbfd49f2007-05-11 14:56:18 +0200842 for_each_sg(sgl, sg, nelems, i) {
Jeremy Fitzhardingeef9b1892008-12-16 12:17:33 -0800843 dev_addr = swiotlb_sg_to_bus(sg);
Ian Campbellb81ea272008-12-16 12:17:31 -0800844 if (range_needs_mapping(sg_virt(sg), sg->length) ||
FUJITA Tomonori27979822008-09-10 01:06:49 +0900845 address_needs_mapping(hwdev, dev_addr, sg->length)) {
Jeremy Fitzhardingeef9b1892008-12-16 12:17:33 -0800846 void *map;
847 buffer.page = sg_page(sg);
848 buffer.offset = sg->offset;
849 map = map_single(hwdev, buffer, sg->length, dir);
Andi Kleen7e870232005-12-20 14:45:19 +0100850 if (!map) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700851 /* Don't panic here, we expect map_sg users
852 to do proper error handling. */
853 swiotlb_full(hwdev, sg->length, dir, 0);
Arthur Kepner309df0c2008-04-29 01:00:32 -0700854 swiotlb_unmap_sg_attrs(hwdev, sgl, i, dir,
855 attrs);
Jens Axboedbfd49f2007-05-11 14:56:18 +0200856 sgl[0].dma_length = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700857 return 0;
858 }
Ian Campbelle08e1f72008-12-16 12:17:30 -0800859 sg->dma_address = swiotlb_virt_to_bus(map);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700860 } else
861 sg->dma_address = dev_addr;
862 sg->dma_length = sg->length;
863 }
864 return nelems;
865}
Arthur Kepner309df0c2008-04-29 01:00:32 -0700866EXPORT_SYMBOL(swiotlb_map_sg_attrs);
867
868int
869swiotlb_map_sg(struct device *hwdev, struct scatterlist *sgl, int nelems,
870 int dir)
871{
872 return swiotlb_map_sg_attrs(hwdev, sgl, nelems, dir, NULL);
873}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700874
875/*
876 * Unmap a set of streaming mode DMA translations. Again, cpu read rules
877 * concerning calls here are the same as for swiotlb_unmap_single() above.
878 */
879void
Arthur Kepner309df0c2008-04-29 01:00:32 -0700880swiotlb_unmap_sg_attrs(struct device *hwdev, struct scatterlist *sgl,
881 int nelems, int dir, struct dma_attrs *attrs)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700882{
Jens Axboedbfd49f2007-05-11 14:56:18 +0200883 struct scatterlist *sg;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700884 int i;
885
Eric Sesterhenn34814542006-03-24 18:47:11 +0100886 BUG_ON(dir == DMA_NONE);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700887
Jens Axboedbfd49f2007-05-11 14:56:18 +0200888 for_each_sg(sgl, sg, nelems, i) {
Jeremy Fitzhardingeef9b1892008-12-16 12:17:33 -0800889 if (sg->dma_address != swiotlb_sg_to_bus(sg))
Ian Campbelle08e1f72008-12-16 12:17:30 -0800890 unmap_single(hwdev, swiotlb_bus_to_virt(sg->dma_address),
Jan Beulich93fbff62007-02-05 18:49:45 -0800891 sg->dma_length, dir);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700892 else if (dir == DMA_FROM_DEVICE)
Jeremy Fitzhardingeef9b1892008-12-16 12:17:33 -0800893 dma_mark_clean(swiotlb_bus_to_virt(sg->dma_address), sg->dma_length);
Jens Axboedbfd49f2007-05-11 14:56:18 +0200894 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700895}
Arthur Kepner309df0c2008-04-29 01:00:32 -0700896EXPORT_SYMBOL(swiotlb_unmap_sg_attrs);
897
898void
899swiotlb_unmap_sg(struct device *hwdev, struct scatterlist *sgl, int nelems,
900 int dir)
901{
902 return swiotlb_unmap_sg_attrs(hwdev, sgl, nelems, dir, NULL);
903}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700904
905/*
906 * Make physical memory consistent for a set of streaming mode DMA translations
907 * after a transfer.
908 *
909 * The same as swiotlb_sync_single_* but for a scatter-gather list, same rules
910 * and usage.
911 */
Andrew Mortonbe6b0262007-02-12 00:52:17 -0800912static void
Jens Axboedbfd49f2007-05-11 14:56:18 +0200913swiotlb_sync_sg(struct device *hwdev, struct scatterlist *sgl,
John W. Linvillede69e0f2005-09-29 14:44:57 -0700914 int nelems, int dir, int target)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700915{
Jens Axboedbfd49f2007-05-11 14:56:18 +0200916 struct scatterlist *sg;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700917 int i;
918
Eric Sesterhenn34814542006-03-24 18:47:11 +0100919 BUG_ON(dir == DMA_NONE);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700920
Jens Axboedbfd49f2007-05-11 14:56:18 +0200921 for_each_sg(sgl, sg, nelems, i) {
Jeremy Fitzhardingeef9b1892008-12-16 12:17:33 -0800922 if (sg->dma_address != swiotlb_sg_to_bus(sg))
Ian Campbelle08e1f72008-12-16 12:17:30 -0800923 sync_single(hwdev, swiotlb_bus_to_virt(sg->dma_address),
John W. Linvillede69e0f2005-09-29 14:44:57 -0700924 sg->dma_length, dir, target);
Jan Beulichcde14bb2007-02-05 18:46:40 -0800925 else if (dir == DMA_FROM_DEVICE)
Jeremy Fitzhardingeef9b1892008-12-16 12:17:33 -0800926 dma_mark_clean(swiotlb_bus_to_virt(sg->dma_address), sg->dma_length);
Jens Axboedbfd49f2007-05-11 14:56:18 +0200927 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700928}
929
930void
John W. Linville8270f3f2005-09-29 14:43:32 -0700931swiotlb_sync_sg_for_cpu(struct device *hwdev, struct scatterlist *sg,
932 int nelems, int dir)
933{
John W. Linvillede69e0f2005-09-29 14:44:57 -0700934 swiotlb_sync_sg(hwdev, sg, nelems, dir, SYNC_FOR_CPU);
John W. Linville8270f3f2005-09-29 14:43:32 -0700935}
936
937void
Linus Torvalds1da177e2005-04-16 15:20:36 -0700938swiotlb_sync_sg_for_device(struct device *hwdev, struct scatterlist *sg,
939 int nelems, int dir)
940{
John W. Linvillede69e0f2005-09-29 14:44:57 -0700941 swiotlb_sync_sg(hwdev, sg, nelems, dir, SYNC_FOR_DEVICE);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700942}
943
944int
FUJITA Tomonori8d8bb392008-07-25 19:44:49 -0700945swiotlb_dma_mapping_error(struct device *hwdev, dma_addr_t dma_addr)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700946{
Ian Campbelle08e1f72008-12-16 12:17:30 -0800947 return (dma_addr == swiotlb_virt_to_bus(io_tlb_overflow_buffer));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700948}
949
950/*
Tony Luck17e5ad62005-09-29 15:52:13 -0700951 * Return whether the given device DMA address mask can be supported
Linus Torvalds1da177e2005-04-16 15:20:36 -0700952 * properly. For example, if your device can only drive the low 24-bits
Tony Luck17e5ad62005-09-29 15:52:13 -0700953 * during bus mastering, then you would pass 0x00ffffff as the mask to
Linus Torvalds1da177e2005-04-16 15:20:36 -0700954 * this function.
955 */
956int
Jan Beulich563aaf02007-02-05 18:51:25 -0800957swiotlb_dma_supported(struct device *hwdev, u64 mask)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700958{
Ian Campbelle08e1f72008-12-16 12:17:30 -0800959 return swiotlb_virt_to_bus(io_tlb_end - 1) <= mask;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700960}
961
Linus Torvalds1da177e2005-04-16 15:20:36 -0700962EXPORT_SYMBOL(swiotlb_map_single);
963EXPORT_SYMBOL(swiotlb_unmap_single);
964EXPORT_SYMBOL(swiotlb_map_sg);
965EXPORT_SYMBOL(swiotlb_unmap_sg);
966EXPORT_SYMBOL(swiotlb_sync_single_for_cpu);
967EXPORT_SYMBOL(swiotlb_sync_single_for_device);
John W. Linville878a97c2005-09-29 14:44:23 -0700968EXPORT_SYMBOL_GPL(swiotlb_sync_single_range_for_cpu);
969EXPORT_SYMBOL_GPL(swiotlb_sync_single_range_for_device);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700970EXPORT_SYMBOL(swiotlb_sync_sg_for_cpu);
971EXPORT_SYMBOL(swiotlb_sync_sg_for_device);
972EXPORT_SYMBOL(swiotlb_dma_mapping_error);
Tony Luck25667d62007-03-06 13:31:45 -0800973EXPORT_SYMBOL(swiotlb_alloc_coherent);
974EXPORT_SYMBOL(swiotlb_free_coherent);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700975EXPORT_SYMBOL(swiotlb_dma_supported);