blob: 7f5e21b9c16b90ffaa7fffd3cee96a65a511302b [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * Dynamic DMA mapping support.
3 *
Jan Beulich563aaf02007-02-05 18:51:25 -08004 * This implementation is a fallback for platforms that do not support
Linus Torvalds1da177e2005-04-16 15:20:36 -07005 * I/O TLBs (aka DMA address translation hardware).
6 * Copyright (C) 2000 Asit Mallick <Asit.K.Mallick@intel.com>
7 * Copyright (C) 2000 Goutham Rao <goutham.rao@intel.com>
8 * Copyright (C) 2000, 2003 Hewlett-Packard Co
9 * David Mosberger-Tang <davidm@hpl.hp.com>
10 *
11 * 03/05/07 davidm Switch from PCI-DMA to generic device DMA API.
12 * 00/12/13 davidm Rename to swiotlb.c and add mark_clean() to avoid
13 * unnecessary i-cache flushing.
John W. Linville569c8bf2005-09-29 14:45:24 -070014 * 04/07/.. ak Better overflow handling. Assorted fixes.
15 * 05/09/10 linville Add support for syncing ranges, support syncing for
16 * DMA_BIDIRECTIONAL mappings, miscellaneous cleanup.
Linus Torvalds1da177e2005-04-16 15:20:36 -070017 */
18
19#include <linux/cache.h>
Tony Luck17e5ad62005-09-29 15:52:13 -070020#include <linux/dma-mapping.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070021#include <linux/mm.h>
22#include <linux/module.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070023#include <linux/spinlock.h>
Jeremy Fitzhardinge8c5df162008-12-16 12:17:26 -080024#include <linux/swiotlb.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070025#include <linux/string.h>
26#include <linux/types.h>
27#include <linux/ctype.h>
Jeremy Fitzhardingeef9b1892008-12-16 12:17:33 -080028#include <linux/highmem.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070029
30#include <asm/io.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070031#include <asm/dma.h>
Tony Luck17e5ad62005-09-29 15:52:13 -070032#include <asm/scatterlist.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070033
34#include <linux/init.h>
35#include <linux/bootmem.h>
FUJITA Tomonoria8522502008-04-29 00:59:36 -070036#include <linux/iommu-helper.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070037
38#define OFFSET(val,align) ((unsigned long) \
39 ( (val) & ( (align) - 1)))
40
Alex Williamson0b9afed2005-09-06 11:20:49 -060041#define SLABS_PER_PAGE (1 << (PAGE_SHIFT - IO_TLB_SHIFT))
42
43/*
44 * Minimum IO TLB size to bother booting with. Systems with mainly
45 * 64bit capable cards will only lightly use the swiotlb. If we can't
46 * allocate a contiguous 1MB, we're probably in trouble anyway.
47 */
48#define IO_TLB_MIN_SLABS ((1<<20) >> IO_TLB_SHIFT)
49
John W. Linvillede69e0f2005-09-29 14:44:57 -070050/*
51 * Enumeration for sync targets
52 */
53enum dma_sync_target {
54 SYNC_FOR_CPU = 0,
55 SYNC_FOR_DEVICE = 1,
56};
57
Linus Torvalds1da177e2005-04-16 15:20:36 -070058int swiotlb_force;
59
60/*
61 * Used to do a quick range check in swiotlb_unmap_single and
62 * swiotlb_sync_single_*, to see if the memory was in fact allocated by this
63 * API.
64 */
65static char *io_tlb_start, *io_tlb_end;
66
67/*
68 * The number of IO TLB blocks (in groups of 64) betweeen io_tlb_start and
69 * io_tlb_end. This is command line adjustable via setup_io_tlb_npages.
70 */
71static unsigned long io_tlb_nslabs;
72
73/*
74 * When the IOMMU overflows we return a fallback buffer. This sets the size.
75 */
76static unsigned long io_tlb_overflow = 32*1024;
77
78void *io_tlb_overflow_buffer;
79
80/*
81 * This is a free list describing the number of free entries available from
82 * each index
83 */
84static unsigned int *io_tlb_list;
85static unsigned int io_tlb_index;
86
87/*
88 * We need to save away the original address corresponding to a mapped entry
89 * for the sync operations.
90 */
Jeremy Fitzhardingeef9b1892008-12-16 12:17:33 -080091static struct swiotlb_phys_addr {
92 struct page *page;
93 unsigned int offset;
94} *io_tlb_orig_addr;
Linus Torvalds1da177e2005-04-16 15:20:36 -070095
96/*
97 * Protect the above data structures in the map and unmap calls
98 */
99static DEFINE_SPINLOCK(io_tlb_lock);
100
101static int __init
102setup_io_tlb_npages(char *str)
103{
104 if (isdigit(*str)) {
Alex Williamsone8579e72005-08-04 13:06:00 -0700105 io_tlb_nslabs = simple_strtoul(str, &str, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700106 /* avoid tail segment of size < IO_TLB_SEGSIZE */
107 io_tlb_nslabs = ALIGN(io_tlb_nslabs, IO_TLB_SEGSIZE);
108 }
109 if (*str == ',')
110 ++str;
111 if (!strcmp(str, "force"))
112 swiotlb_force = 1;
113 return 1;
114}
115__setup("swiotlb=", setup_io_tlb_npages);
116/* make io_tlb_overflow tunable too? */
117
Roland Dreier79ff56e2008-12-30 20:18:00 -0800118void * __weak __init swiotlb_alloc_boot(size_t size, unsigned long nslabs)
Jeremy Fitzhardinge8c5df162008-12-16 12:17:26 -0800119{
120 return alloc_bootmem_low_pages(size);
121}
122
123void * __weak swiotlb_alloc(unsigned order, unsigned long nslabs)
124{
125 return (void *)__get_free_pages(GFP_DMA | __GFP_NOWARN, order);
126}
127
Ian Campbelle08e1f72008-12-16 12:17:30 -0800128dma_addr_t __weak swiotlb_phys_to_bus(phys_addr_t paddr)
129{
130 return paddr;
131}
132
133phys_addr_t __weak swiotlb_bus_to_phys(dma_addr_t baddr)
134{
135 return baddr;
136}
137
138static dma_addr_t swiotlb_virt_to_bus(volatile void *address)
139{
140 return swiotlb_phys_to_bus(virt_to_phys(address));
141}
142
143static void *swiotlb_bus_to_virt(dma_addr_t address)
144{
145 return phys_to_virt(swiotlb_bus_to_phys(address));
146}
147
Ian Campbellb81ea272008-12-16 12:17:31 -0800148int __weak swiotlb_arch_range_needs_mapping(void *ptr, size_t size)
149{
150 return 0;
151}
152
Jeremy Fitzhardingeef9b1892008-12-16 12:17:33 -0800153static dma_addr_t swiotlb_sg_to_bus(struct scatterlist *sg)
154{
155 return swiotlb_phys_to_bus(page_to_phys(sg_page(sg)) + sg->offset);
156}
157
Ian Campbell2e5b2b82008-12-16 12:17:34 -0800158static void swiotlb_print_info(unsigned long bytes)
159{
160 phys_addr_t pstart, pend;
161 dma_addr_t bstart, bend;
162
163 pstart = virt_to_phys(io_tlb_start);
164 pend = virt_to_phys(io_tlb_end);
165
166 bstart = swiotlb_phys_to_bus(pstart);
167 bend = swiotlb_phys_to_bus(pend);
168
169 printk(KERN_INFO "Placing %luMB software IO TLB between %p - %p\n",
170 bytes >> 20, io_tlb_start, io_tlb_end);
171 if (pstart != bstart || pend != bend)
172 printk(KERN_INFO "software IO TLB at phys %#llx - %#llx"
173 " bus %#llx - %#llx\n",
174 (unsigned long long)pstart,
175 (unsigned long long)pend,
176 (unsigned long long)bstart,
177 (unsigned long long)bend);
178 else
179 printk(KERN_INFO "software IO TLB at phys %#llx - %#llx\n",
180 (unsigned long long)pstart,
181 (unsigned long long)pend);
182}
183
Linus Torvalds1da177e2005-04-16 15:20:36 -0700184/*
185 * Statically reserve bounce buffer space and initialize bounce buffer data
Tony Luck17e5ad62005-09-29 15:52:13 -0700186 * structures for the software IO TLB used to implement the DMA API.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700187 */
Jan Beulich563aaf02007-02-05 18:51:25 -0800188void __init
189swiotlb_init_with_default_size(size_t default_size)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700190{
Jan Beulich563aaf02007-02-05 18:51:25 -0800191 unsigned long i, bytes;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700192
193 if (!io_tlb_nslabs) {
Alex Williamsone8579e72005-08-04 13:06:00 -0700194 io_tlb_nslabs = (default_size >> IO_TLB_SHIFT);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700195 io_tlb_nslabs = ALIGN(io_tlb_nslabs, IO_TLB_SEGSIZE);
196 }
197
Jan Beulich563aaf02007-02-05 18:51:25 -0800198 bytes = io_tlb_nslabs << IO_TLB_SHIFT;
199
Linus Torvalds1da177e2005-04-16 15:20:36 -0700200 /*
201 * Get IO TLB memory from the low pages
202 */
Jeremy Fitzhardinge8c5df162008-12-16 12:17:26 -0800203 io_tlb_start = swiotlb_alloc_boot(bytes, io_tlb_nslabs);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700204 if (!io_tlb_start)
205 panic("Cannot allocate SWIOTLB buffer");
Jan Beulich563aaf02007-02-05 18:51:25 -0800206 io_tlb_end = io_tlb_start + bytes;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700207
208 /*
209 * Allocate and initialize the free list array. This array is used
210 * to find contiguous free memory regions of size up to IO_TLB_SEGSIZE
211 * between io_tlb_start and io_tlb_end.
212 */
213 io_tlb_list = alloc_bootmem(io_tlb_nslabs * sizeof(int));
Tony Luck25667d62007-03-06 13:31:45 -0800214 for (i = 0; i < io_tlb_nslabs; i++)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700215 io_tlb_list[i] = IO_TLB_SEGSIZE - OFFSET(i, IO_TLB_SEGSIZE);
216 io_tlb_index = 0;
Jeremy Fitzhardingeef9b1892008-12-16 12:17:33 -0800217 io_tlb_orig_addr = alloc_bootmem(io_tlb_nslabs * sizeof(struct swiotlb_phys_addr));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700218
219 /*
220 * Get the overflow emergency buffer
221 */
222 io_tlb_overflow_buffer = alloc_bootmem_low(io_tlb_overflow);
Jan Beulich563aaf02007-02-05 18:51:25 -0800223 if (!io_tlb_overflow_buffer)
224 panic("Cannot allocate SWIOTLB overflow buffer!\n");
225
Ian Campbell2e5b2b82008-12-16 12:17:34 -0800226 swiotlb_print_info(bytes);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700227}
228
Jan Beulich563aaf02007-02-05 18:51:25 -0800229void __init
230swiotlb_init(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700231{
Tony Luck25667d62007-03-06 13:31:45 -0800232 swiotlb_init_with_default_size(64 * (1<<20)); /* default to 64MB */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700233}
234
Alex Williamson0b9afed2005-09-06 11:20:49 -0600235/*
236 * Systems with larger DMA zones (those that don't support ISA) can
237 * initialize the swiotlb later using the slab allocator if needed.
238 * This should be just like above, but with some error catching.
239 */
240int
Jan Beulich563aaf02007-02-05 18:51:25 -0800241swiotlb_late_init_with_default_size(size_t default_size)
Alex Williamson0b9afed2005-09-06 11:20:49 -0600242{
Jan Beulich563aaf02007-02-05 18:51:25 -0800243 unsigned long i, bytes, req_nslabs = io_tlb_nslabs;
Alex Williamson0b9afed2005-09-06 11:20:49 -0600244 unsigned int order;
245
246 if (!io_tlb_nslabs) {
247 io_tlb_nslabs = (default_size >> IO_TLB_SHIFT);
248 io_tlb_nslabs = ALIGN(io_tlb_nslabs, IO_TLB_SEGSIZE);
249 }
250
251 /*
252 * Get IO TLB memory from the low pages
253 */
Jan Beulich563aaf02007-02-05 18:51:25 -0800254 order = get_order(io_tlb_nslabs << IO_TLB_SHIFT);
Alex Williamson0b9afed2005-09-06 11:20:49 -0600255 io_tlb_nslabs = SLABS_PER_PAGE << order;
Jan Beulich563aaf02007-02-05 18:51:25 -0800256 bytes = io_tlb_nslabs << IO_TLB_SHIFT;
Alex Williamson0b9afed2005-09-06 11:20:49 -0600257
258 while ((SLABS_PER_PAGE << order) > IO_TLB_MIN_SLABS) {
Jeremy Fitzhardinge8c5df162008-12-16 12:17:26 -0800259 io_tlb_start = swiotlb_alloc(order, io_tlb_nslabs);
Alex Williamson0b9afed2005-09-06 11:20:49 -0600260 if (io_tlb_start)
261 break;
262 order--;
263 }
264
265 if (!io_tlb_start)
266 goto cleanup1;
267
Jan Beulich563aaf02007-02-05 18:51:25 -0800268 if (order != get_order(bytes)) {
Alex Williamson0b9afed2005-09-06 11:20:49 -0600269 printk(KERN_WARNING "Warning: only able to allocate %ld MB "
270 "for software IO TLB\n", (PAGE_SIZE << order) >> 20);
271 io_tlb_nslabs = SLABS_PER_PAGE << order;
Jan Beulich563aaf02007-02-05 18:51:25 -0800272 bytes = io_tlb_nslabs << IO_TLB_SHIFT;
Alex Williamson0b9afed2005-09-06 11:20:49 -0600273 }
Jan Beulich563aaf02007-02-05 18:51:25 -0800274 io_tlb_end = io_tlb_start + bytes;
275 memset(io_tlb_start, 0, bytes);
Alex Williamson0b9afed2005-09-06 11:20:49 -0600276
277 /*
278 * Allocate and initialize the free list array. This array is used
279 * to find contiguous free memory regions of size up to IO_TLB_SEGSIZE
280 * between io_tlb_start and io_tlb_end.
281 */
282 io_tlb_list = (unsigned int *)__get_free_pages(GFP_KERNEL,
283 get_order(io_tlb_nslabs * sizeof(int)));
284 if (!io_tlb_list)
285 goto cleanup2;
286
287 for (i = 0; i < io_tlb_nslabs; i++)
288 io_tlb_list[i] = IO_TLB_SEGSIZE - OFFSET(i, IO_TLB_SEGSIZE);
289 io_tlb_index = 0;
290
Jeremy Fitzhardingeef9b1892008-12-16 12:17:33 -0800291 io_tlb_orig_addr = (struct swiotlb_phys_addr *)__get_free_pages(GFP_KERNEL,
292 get_order(io_tlb_nslabs * sizeof(struct swiotlb_phys_addr)));
Alex Williamson0b9afed2005-09-06 11:20:49 -0600293 if (!io_tlb_orig_addr)
294 goto cleanup3;
295
Jeremy Fitzhardingeef9b1892008-12-16 12:17:33 -0800296 memset(io_tlb_orig_addr, 0, io_tlb_nslabs * sizeof(struct swiotlb_phys_addr));
Alex Williamson0b9afed2005-09-06 11:20:49 -0600297
298 /*
299 * Get the overflow emergency buffer
300 */
301 io_tlb_overflow_buffer = (void *)__get_free_pages(GFP_DMA,
302 get_order(io_tlb_overflow));
303 if (!io_tlb_overflow_buffer)
304 goto cleanup4;
305
Ian Campbell2e5b2b82008-12-16 12:17:34 -0800306 swiotlb_print_info(bytes);
Alex Williamson0b9afed2005-09-06 11:20:49 -0600307
308 return 0;
309
310cleanup4:
Tony Luck25667d62007-03-06 13:31:45 -0800311 free_pages((unsigned long)io_tlb_orig_addr, get_order(io_tlb_nslabs *
312 sizeof(char *)));
Alex Williamson0b9afed2005-09-06 11:20:49 -0600313 io_tlb_orig_addr = NULL;
314cleanup3:
Tony Luck25667d62007-03-06 13:31:45 -0800315 free_pages((unsigned long)io_tlb_list, get_order(io_tlb_nslabs *
316 sizeof(int)));
Alex Williamson0b9afed2005-09-06 11:20:49 -0600317 io_tlb_list = NULL;
Alex Williamson0b9afed2005-09-06 11:20:49 -0600318cleanup2:
Jan Beulich563aaf02007-02-05 18:51:25 -0800319 io_tlb_end = NULL;
Alex Williamson0b9afed2005-09-06 11:20:49 -0600320 free_pages((unsigned long)io_tlb_start, order);
321 io_tlb_start = NULL;
322cleanup1:
323 io_tlb_nslabs = req_nslabs;
324 return -ENOMEM;
325}
326
Andrew Mortonbe6b0262007-02-12 00:52:17 -0800327static int
FUJITA Tomonori27979822008-09-10 01:06:49 +0900328address_needs_mapping(struct device *hwdev, dma_addr_t addr, size_t size)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700329{
FUJITA Tomonori07a2c012008-09-19 02:02:05 +0900330 return !is_buffer_dma_capable(dma_get_mask(hwdev), addr, size);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700331}
332
Ian Campbellb81ea272008-12-16 12:17:31 -0800333static inline int range_needs_mapping(void *ptr, size_t size)
334{
335 return swiotlb_force || swiotlb_arch_range_needs_mapping(ptr, size);
336}
337
FUJITA Tomonori640aebf2008-09-08 18:53:50 +0900338static int is_swiotlb_buffer(char *addr)
339{
340 return addr >= io_tlb_start && addr < io_tlb_end;
341}
342
Jeremy Fitzhardingeef9b1892008-12-16 12:17:33 -0800343static struct swiotlb_phys_addr swiotlb_bus_to_phys_addr(char *dma_addr)
Jeremy Fitzhardinge1b548f62008-12-16 12:17:32 -0800344{
Jeremy Fitzhardingeef9b1892008-12-16 12:17:33 -0800345 int index = (dma_addr - io_tlb_start) >> IO_TLB_SHIFT;
346 struct swiotlb_phys_addr buffer = io_tlb_orig_addr[index];
347 buffer.offset += (long)dma_addr & ((1 << IO_TLB_SHIFT) - 1);
348 buffer.page += buffer.offset >> PAGE_SHIFT;
349 buffer.offset &= PAGE_SIZE - 1;
350 return buffer;
351}
352
353static void
354__sync_single(struct swiotlb_phys_addr buffer, char *dma_addr, size_t size, int dir)
355{
356 if (PageHighMem(buffer.page)) {
357 size_t len, bytes;
358 char *dev, *host, *kmp;
359
360 len = size;
361 while (len != 0) {
362 unsigned long flags;
363
364 bytes = len;
365 if ((bytes + buffer.offset) > PAGE_SIZE)
366 bytes = PAGE_SIZE - buffer.offset;
367 local_irq_save(flags); /* protects KM_BOUNCE_READ */
368 kmp = kmap_atomic(buffer.page, KM_BOUNCE_READ);
369 dev = dma_addr + size - len;
370 host = kmp + buffer.offset;
371 if (dir == DMA_FROM_DEVICE)
372 memcpy(host, dev, bytes);
373 else
374 memcpy(dev, host, bytes);
375 kunmap_atomic(kmp, KM_BOUNCE_READ);
376 local_irq_restore(flags);
377 len -= bytes;
378 buffer.page++;
379 buffer.offset = 0;
380 }
381 } else {
382 void *v = page_address(buffer.page) + buffer.offset;
383
384 if (dir == DMA_TO_DEVICE)
385 memcpy(dma_addr, v, size);
386 else
387 memcpy(v, dma_addr, size);
388 }
Jeremy Fitzhardinge1b548f62008-12-16 12:17:32 -0800389}
390
Linus Torvalds1da177e2005-04-16 15:20:36 -0700391/*
392 * Allocates bounce buffer and returns its kernel virtual address.
393 */
394static void *
Jeremy Fitzhardingeef9b1892008-12-16 12:17:33 -0800395map_single(struct device *hwdev, struct swiotlb_phys_addr buffer, size_t size, int dir)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700396{
397 unsigned long flags;
398 char *dma_addr;
399 unsigned int nslots, stride, index, wrap;
400 int i;
FUJITA Tomonori681cc5c2008-02-04 22:28:16 -0800401 unsigned long start_dma_addr;
402 unsigned long mask;
403 unsigned long offset_slots;
404 unsigned long max_slots;
Jeremy Fitzhardingeef9b1892008-12-16 12:17:33 -0800405 struct swiotlb_phys_addr slot_buf;
FUJITA Tomonori681cc5c2008-02-04 22:28:16 -0800406
407 mask = dma_get_seg_boundary(hwdev);
Ian Campbelle08e1f72008-12-16 12:17:30 -0800408 start_dma_addr = swiotlb_virt_to_bus(io_tlb_start) & mask;
FUJITA Tomonori681cc5c2008-02-04 22:28:16 -0800409
410 offset_slots = ALIGN(start_dma_addr, 1 << IO_TLB_SHIFT) >> IO_TLB_SHIFT;
Ian Campbella5ddde4a2008-12-16 12:17:29 -0800411
412 /*
413 * Carefully handle integer overflow which can occur when mask == ~0UL.
414 */
Jan Beulichb15a38912008-03-13 09:13:30 +0000415 max_slots = mask + 1
416 ? ALIGN(mask + 1, 1 << IO_TLB_SHIFT) >> IO_TLB_SHIFT
417 : 1UL << (BITS_PER_LONG - IO_TLB_SHIFT);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700418
419 /*
420 * For mappings greater than a page, we limit the stride (and
421 * hence alignment) to a page size.
422 */
423 nslots = ALIGN(size, 1 << IO_TLB_SHIFT) >> IO_TLB_SHIFT;
424 if (size > PAGE_SIZE)
425 stride = (1 << (PAGE_SHIFT - IO_TLB_SHIFT));
426 else
427 stride = 1;
428
Eric Sesterhenn34814542006-03-24 18:47:11 +0100429 BUG_ON(!nslots);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700430
431 /*
432 * Find suitable number of IO TLB entries size that will fit this
433 * request and allocate a buffer from that IO TLB pool.
434 */
435 spin_lock_irqsave(&io_tlb_lock, flags);
Andrew Mortona7133a12008-04-29 00:59:36 -0700436 index = ALIGN(io_tlb_index, stride);
437 if (index >= io_tlb_nslabs)
438 index = 0;
439 wrap = index;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700440
Andrew Mortona7133a12008-04-29 00:59:36 -0700441 do {
FUJITA Tomonoria8522502008-04-29 00:59:36 -0700442 while (iommu_is_span_boundary(index, nslots, offset_slots,
443 max_slots)) {
Jan Beulichb15a38912008-03-13 09:13:30 +0000444 index += stride;
445 if (index >= io_tlb_nslabs)
446 index = 0;
Andrew Mortona7133a12008-04-29 00:59:36 -0700447 if (index == wrap)
448 goto not_found;
449 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700450
Andrew Mortona7133a12008-04-29 00:59:36 -0700451 /*
452 * If we find a slot that indicates we have 'nslots' number of
453 * contiguous buffers, we allocate the buffers from that slot
454 * and mark the entries as '0' indicating unavailable.
455 */
456 if (io_tlb_list[index] >= nslots) {
457 int count = 0;
458
459 for (i = index; i < (int) (index + nslots); i++)
460 io_tlb_list[i] = 0;
461 for (i = index - 1; (OFFSET(i, IO_TLB_SEGSIZE) != IO_TLB_SEGSIZE - 1) && io_tlb_list[i]; i--)
462 io_tlb_list[i] = ++count;
463 dma_addr = io_tlb_start + (index << IO_TLB_SHIFT);
464
465 /*
466 * Update the indices to avoid searching in the next
467 * round.
468 */
469 io_tlb_index = ((index + nslots) < io_tlb_nslabs
470 ? (index + nslots) : 0);
471
472 goto found;
473 }
474 index += stride;
475 if (index >= io_tlb_nslabs)
476 index = 0;
477 } while (index != wrap);
478
479not_found:
480 spin_unlock_irqrestore(&io_tlb_lock, flags);
481 return NULL;
482found:
Linus Torvalds1da177e2005-04-16 15:20:36 -0700483 spin_unlock_irqrestore(&io_tlb_lock, flags);
484
485 /*
486 * Save away the mapping from the original address to the DMA address.
487 * This is needed when we sync the memory. Then we sync the buffer if
488 * needed.
489 */
Jeremy Fitzhardingeef9b1892008-12-16 12:17:33 -0800490 slot_buf = buffer;
491 for (i = 0; i < nslots; i++) {
492 slot_buf.page += slot_buf.offset >> PAGE_SHIFT;
493 slot_buf.offset &= PAGE_SIZE - 1;
494 io_tlb_orig_addr[index+i] = slot_buf;
495 slot_buf.offset += 1 << IO_TLB_SHIFT;
496 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700497 if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL)
Jeremy Fitzhardinge1b548f62008-12-16 12:17:32 -0800498 __sync_single(buffer, dma_addr, size, DMA_TO_DEVICE);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700499
500 return dma_addr;
501}
502
503/*
504 * dma_addr is the kernel virtual address of the bounce buffer to unmap.
505 */
506static void
507unmap_single(struct device *hwdev, char *dma_addr, size_t size, int dir)
508{
509 unsigned long flags;
510 int i, count, nslots = ALIGN(size, 1 << IO_TLB_SHIFT) >> IO_TLB_SHIFT;
511 int index = (dma_addr - io_tlb_start) >> IO_TLB_SHIFT;
Jeremy Fitzhardingeef9b1892008-12-16 12:17:33 -0800512 struct swiotlb_phys_addr buffer = swiotlb_bus_to_phys_addr(dma_addr);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700513
514 /*
515 * First, sync the memory before unmapping the entry
516 */
Jeremy Fitzhardingeef9b1892008-12-16 12:17:33 -0800517 if ((dir == DMA_FROM_DEVICE) || (dir == DMA_BIDIRECTIONAL))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700518 /*
519 * bounce... copy the data back into the original buffer * and
520 * delete the bounce buffer.
521 */
Jeremy Fitzhardinge1b548f62008-12-16 12:17:32 -0800522 __sync_single(buffer, dma_addr, size, DMA_FROM_DEVICE);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700523
524 /*
525 * Return the buffer to the free list by setting the corresponding
526 * entries to indicate the number of contigous entries available.
527 * While returning the entries to the free list, we merge the entries
528 * with slots below and above the pool being returned.
529 */
530 spin_lock_irqsave(&io_tlb_lock, flags);
531 {
532 count = ((index + nslots) < ALIGN(index + 1, IO_TLB_SEGSIZE) ?
533 io_tlb_list[index + nslots] : 0);
534 /*
535 * Step 1: return the slots to the free list, merging the
536 * slots with superceeding slots
537 */
538 for (i = index + nslots - 1; i >= index; i--)
539 io_tlb_list[i] = ++count;
540 /*
541 * Step 2: merge the returned slots with the preceding slots,
542 * if available (non zero)
543 */
544 for (i = index - 1; (OFFSET(i, IO_TLB_SEGSIZE) != IO_TLB_SEGSIZE -1) && io_tlb_list[i]; i--)
545 io_tlb_list[i] = ++count;
546 }
547 spin_unlock_irqrestore(&io_tlb_lock, flags);
548}
549
550static void
John W. Linvillede69e0f2005-09-29 14:44:57 -0700551sync_single(struct device *hwdev, char *dma_addr, size_t size,
552 int dir, int target)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700553{
Jeremy Fitzhardingeef9b1892008-12-16 12:17:33 -0800554 struct swiotlb_phys_addr buffer = swiotlb_bus_to_phys_addr(dma_addr);
Keir Fraserdf336d12007-07-21 04:37:24 -0700555
John W. Linvillede69e0f2005-09-29 14:44:57 -0700556 switch (target) {
557 case SYNC_FOR_CPU:
558 if (likely(dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL))
Jeremy Fitzhardinge1b548f62008-12-16 12:17:32 -0800559 __sync_single(buffer, dma_addr, size, DMA_FROM_DEVICE);
Eric Sesterhenn34814542006-03-24 18:47:11 +0100560 else
561 BUG_ON(dir != DMA_TO_DEVICE);
John W. Linvillede69e0f2005-09-29 14:44:57 -0700562 break;
563 case SYNC_FOR_DEVICE:
564 if (likely(dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL))
Jeremy Fitzhardinge1b548f62008-12-16 12:17:32 -0800565 __sync_single(buffer, dma_addr, size, DMA_TO_DEVICE);
Eric Sesterhenn34814542006-03-24 18:47:11 +0100566 else
567 BUG_ON(dir != DMA_FROM_DEVICE);
John W. Linvillede69e0f2005-09-29 14:44:57 -0700568 break;
569 default:
Linus Torvalds1da177e2005-04-16 15:20:36 -0700570 BUG();
John W. Linvillede69e0f2005-09-29 14:44:57 -0700571 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700572}
573
574void *
575swiotlb_alloc_coherent(struct device *hwdev, size_t size,
Al Viro06a54492005-10-21 03:21:03 -0400576 dma_addr_t *dma_handle, gfp_t flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700577{
Jan Beulich563aaf02007-02-05 18:51:25 -0800578 dma_addr_t dev_addr;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700579 void *ret;
580 int order = get_order(size);
FUJITA Tomonori1e74f302008-11-17 16:24:34 +0900581 u64 dma_mask = DMA_32BIT_MASK;
582
583 if (hwdev && hwdev->coherent_dma_mask)
584 dma_mask = hwdev->coherent_dma_mask;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700585
Tony Luck25667d62007-03-06 13:31:45 -0800586 ret = (void *)__get_free_pages(flags, order);
Ian Campbelle08e1f72008-12-16 12:17:30 -0800587 if (ret && !is_buffer_dma_capable(dma_mask, swiotlb_virt_to_bus(ret), size)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700588 /*
589 * The allocated memory isn't reachable by the device.
590 * Fall back on swiotlb_map_single().
591 */
592 free_pages((unsigned long) ret, order);
593 ret = NULL;
594 }
595 if (!ret) {
596 /*
597 * We are either out of memory or the device can't DMA
598 * to GFP_DMA memory; fall back on
599 * swiotlb_map_single(), which will grab memory from
600 * the lowest available address range.
601 */
Jeremy Fitzhardingeef9b1892008-12-16 12:17:33 -0800602 struct swiotlb_phys_addr buffer;
603 buffer.page = virt_to_page(NULL);
604 buffer.offset = 0;
605 ret = map_single(hwdev, buffer, size, DMA_FROM_DEVICE);
FUJITA Tomonori9dfda122008-09-08 18:53:48 +0900606 if (!ret)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700607 return NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700608 }
609
610 memset(ret, 0, size);
Ian Campbelle08e1f72008-12-16 12:17:30 -0800611 dev_addr = swiotlb_virt_to_bus(ret);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700612
613 /* Confirm address can be DMA'd by device */
FUJITA Tomonori1e74f302008-11-17 16:24:34 +0900614 if (!is_buffer_dma_capable(dma_mask, dev_addr, size)) {
Jan Beulich563aaf02007-02-05 18:51:25 -0800615 printk("hwdev DMA mask = 0x%016Lx, dev_addr = 0x%016Lx\n",
FUJITA Tomonori1e74f302008-11-17 16:24:34 +0900616 (unsigned long long)dma_mask,
Jan Beulich563aaf02007-02-05 18:51:25 -0800617 (unsigned long long)dev_addr);
FUJITA Tomonoria2b89b52008-10-23 18:42:03 +0900618
619 /* DMA_TO_DEVICE to avoid memcpy in unmap_single */
620 unmap_single(hwdev, ret, size, DMA_TO_DEVICE);
621 return NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700622 }
623 *dma_handle = dev_addr;
624 return ret;
625}
626
627void
628swiotlb_free_coherent(struct device *hwdev, size_t size, void *vaddr,
629 dma_addr_t dma_handle)
630{
David Brownellaa248862007-08-10 13:10:27 -0700631 WARN_ON(irqs_disabled());
FUJITA Tomonori640aebf2008-09-08 18:53:50 +0900632 if (!is_swiotlb_buffer(vaddr))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700633 free_pages((unsigned long) vaddr, get_order(size));
634 else
635 /* DMA_TO_DEVICE to avoid memcpy in unmap_single */
FUJITA Tomonori21f6c4d2008-09-08 18:53:49 +0900636 unmap_single(hwdev, vaddr, size, DMA_TO_DEVICE);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700637}
638
639static void
640swiotlb_full(struct device *dev, size_t size, int dir, int do_panic)
641{
642 /*
643 * Ran out of IOMMU space for this operation. This is very bad.
644 * Unfortunately the drivers cannot handle this operation properly.
Tony Luck17e5ad62005-09-29 15:52:13 -0700645 * unless they check for dma_mapping_error (most don't)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700646 * When the mapping is small enough return a static buffer to limit
647 * the damage, or panic when the transfer is too big.
648 */
Jan Beulich563aaf02007-02-05 18:51:25 -0800649 printk(KERN_ERR "DMA: Out of SW-IOMMU space for %zu bytes at "
Linus Torvalds1da177e2005-04-16 15:20:36 -0700650 "device %s\n", size, dev ? dev->bus_id : "?");
651
652 if (size > io_tlb_overflow && do_panic) {
Tony Luck17e5ad62005-09-29 15:52:13 -0700653 if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
654 panic("DMA: Memory would be corrupted\n");
655 if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL)
656 panic("DMA: Random memory would be DMAed\n");
Linus Torvalds1da177e2005-04-16 15:20:36 -0700657 }
658}
659
660/*
661 * Map a single buffer of the indicated size for DMA in streaming mode. The
Tony Luck17e5ad62005-09-29 15:52:13 -0700662 * physical address to use is returned.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700663 *
664 * Once the device is given the dma address, the device owns this memory until
665 * either swiotlb_unmap_single or swiotlb_dma_sync_single is performed.
666 */
667dma_addr_t
Arthur Kepner309df0c2008-04-29 01:00:32 -0700668swiotlb_map_single_attrs(struct device *hwdev, void *ptr, size_t size,
669 int dir, struct dma_attrs *attrs)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700670{
Ian Campbelle08e1f72008-12-16 12:17:30 -0800671 dma_addr_t dev_addr = swiotlb_virt_to_bus(ptr);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700672 void *map;
Jeremy Fitzhardingeef9b1892008-12-16 12:17:33 -0800673 struct swiotlb_phys_addr buffer;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700674
Eric Sesterhenn34814542006-03-24 18:47:11 +0100675 BUG_ON(dir == DMA_NONE);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700676 /*
677 * If the pointer passed in happens to be in the device's DMA window,
678 * we can safely return the device addr and not worry about bounce
679 * buffering it.
680 */
Ian Campbellb81ea272008-12-16 12:17:31 -0800681 if (!address_needs_mapping(hwdev, dev_addr, size) &&
682 !range_needs_mapping(ptr, size))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700683 return dev_addr;
684
685 /*
686 * Oh well, have to allocate and map a bounce buffer.
687 */
Jeremy Fitzhardingeef9b1892008-12-16 12:17:33 -0800688 buffer.page = virt_to_page(ptr);
689 buffer.offset = (unsigned long)ptr & ~PAGE_MASK;
690 map = map_single(hwdev, buffer, size, dir);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700691 if (!map) {
692 swiotlb_full(hwdev, size, dir, 1);
693 map = io_tlb_overflow_buffer;
694 }
695
Ian Campbelle08e1f72008-12-16 12:17:30 -0800696 dev_addr = swiotlb_virt_to_bus(map);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700697
698 /*
699 * Ensure that the address returned is DMA'ble
700 */
FUJITA Tomonori27979822008-09-10 01:06:49 +0900701 if (address_needs_mapping(hwdev, dev_addr, size))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700702 panic("map_single: bounce buffer is not DMA'ble");
703
704 return dev_addr;
705}
Arthur Kepner309df0c2008-04-29 01:00:32 -0700706EXPORT_SYMBOL(swiotlb_map_single_attrs);
707
708dma_addr_t
709swiotlb_map_single(struct device *hwdev, void *ptr, size_t size, int dir)
710{
711 return swiotlb_map_single_attrs(hwdev, ptr, size, dir, NULL);
712}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700713
714/*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700715 * Unmap a single streaming mode DMA translation. The dma_addr and size must
716 * match what was provided for in a previous swiotlb_map_single call. All
717 * other usages are undefined.
718 *
719 * After this call, reads by the cpu to the buffer are guaranteed to see
720 * whatever the device wrote there.
721 */
722void
Arthur Kepner309df0c2008-04-29 01:00:32 -0700723swiotlb_unmap_single_attrs(struct device *hwdev, dma_addr_t dev_addr,
724 size_t size, int dir, struct dma_attrs *attrs)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700725{
Ian Campbelle08e1f72008-12-16 12:17:30 -0800726 char *dma_addr = swiotlb_bus_to_virt(dev_addr);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700727
Eric Sesterhenn34814542006-03-24 18:47:11 +0100728 BUG_ON(dir == DMA_NONE);
FUJITA Tomonori640aebf2008-09-08 18:53:50 +0900729 if (is_swiotlb_buffer(dma_addr))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700730 unmap_single(hwdev, dma_addr, size, dir);
731 else if (dir == DMA_FROM_DEVICE)
Jan Beulichcde14bb2007-02-05 18:46:40 -0800732 dma_mark_clean(dma_addr, size);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700733}
Arthur Kepner309df0c2008-04-29 01:00:32 -0700734EXPORT_SYMBOL(swiotlb_unmap_single_attrs);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700735
Arthur Kepner309df0c2008-04-29 01:00:32 -0700736void
737swiotlb_unmap_single(struct device *hwdev, dma_addr_t dev_addr, size_t size,
738 int dir)
739{
740 return swiotlb_unmap_single_attrs(hwdev, dev_addr, size, dir, NULL);
741}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700742/*
743 * Make physical memory consistent for a single streaming mode DMA translation
744 * after a transfer.
745 *
746 * If you perform a swiotlb_map_single() but wish to interrogate the buffer
Tony Luck17e5ad62005-09-29 15:52:13 -0700747 * using the cpu, yet do not wish to teardown the dma mapping, you must
748 * call this function before doing so. At the next point you give the dma
Linus Torvalds1da177e2005-04-16 15:20:36 -0700749 * address back to the card, you must first perform a
750 * swiotlb_dma_sync_for_device, and then the device again owns the buffer
751 */
Andrew Mortonbe6b0262007-02-12 00:52:17 -0800752static void
John W. Linville8270f3f2005-09-29 14:43:32 -0700753swiotlb_sync_single(struct device *hwdev, dma_addr_t dev_addr,
John W. Linvillede69e0f2005-09-29 14:44:57 -0700754 size_t size, int dir, int target)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700755{
Ian Campbelle08e1f72008-12-16 12:17:30 -0800756 char *dma_addr = swiotlb_bus_to_virt(dev_addr);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700757
Eric Sesterhenn34814542006-03-24 18:47:11 +0100758 BUG_ON(dir == DMA_NONE);
FUJITA Tomonori640aebf2008-09-08 18:53:50 +0900759 if (is_swiotlb_buffer(dma_addr))
John W. Linvillede69e0f2005-09-29 14:44:57 -0700760 sync_single(hwdev, dma_addr, size, dir, target);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700761 else if (dir == DMA_FROM_DEVICE)
Jan Beulichcde14bb2007-02-05 18:46:40 -0800762 dma_mark_clean(dma_addr, size);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700763}
764
765void
John W. Linville8270f3f2005-09-29 14:43:32 -0700766swiotlb_sync_single_for_cpu(struct device *hwdev, dma_addr_t dev_addr,
767 size_t size, int dir)
768{
John W. Linvillede69e0f2005-09-29 14:44:57 -0700769 swiotlb_sync_single(hwdev, dev_addr, size, dir, SYNC_FOR_CPU);
John W. Linville8270f3f2005-09-29 14:43:32 -0700770}
771
772void
Linus Torvalds1da177e2005-04-16 15:20:36 -0700773swiotlb_sync_single_for_device(struct device *hwdev, dma_addr_t dev_addr,
774 size_t size, int dir)
775{
John W. Linvillede69e0f2005-09-29 14:44:57 -0700776 swiotlb_sync_single(hwdev, dev_addr, size, dir, SYNC_FOR_DEVICE);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700777}
778
779/*
John W. Linville878a97c2005-09-29 14:44:23 -0700780 * Same as above, but for a sub-range of the mapping.
781 */
Andrew Mortonbe6b0262007-02-12 00:52:17 -0800782static void
John W. Linville878a97c2005-09-29 14:44:23 -0700783swiotlb_sync_single_range(struct device *hwdev, dma_addr_t dev_addr,
John W. Linvillede69e0f2005-09-29 14:44:57 -0700784 unsigned long offset, size_t size,
785 int dir, int target)
John W. Linville878a97c2005-09-29 14:44:23 -0700786{
Ian Campbelle08e1f72008-12-16 12:17:30 -0800787 char *dma_addr = swiotlb_bus_to_virt(dev_addr) + offset;
John W. Linville878a97c2005-09-29 14:44:23 -0700788
Eric Sesterhenn34814542006-03-24 18:47:11 +0100789 BUG_ON(dir == DMA_NONE);
FUJITA Tomonori640aebf2008-09-08 18:53:50 +0900790 if (is_swiotlb_buffer(dma_addr))
John W. Linvillede69e0f2005-09-29 14:44:57 -0700791 sync_single(hwdev, dma_addr, size, dir, target);
John W. Linville878a97c2005-09-29 14:44:23 -0700792 else if (dir == DMA_FROM_DEVICE)
Jan Beulichcde14bb2007-02-05 18:46:40 -0800793 dma_mark_clean(dma_addr, size);
John W. Linville878a97c2005-09-29 14:44:23 -0700794}
795
796void
797swiotlb_sync_single_range_for_cpu(struct device *hwdev, dma_addr_t dev_addr,
798 unsigned long offset, size_t size, int dir)
799{
John W. Linvillede69e0f2005-09-29 14:44:57 -0700800 swiotlb_sync_single_range(hwdev, dev_addr, offset, size, dir,
801 SYNC_FOR_CPU);
John W. Linville878a97c2005-09-29 14:44:23 -0700802}
803
804void
805swiotlb_sync_single_range_for_device(struct device *hwdev, dma_addr_t dev_addr,
806 unsigned long offset, size_t size, int dir)
807{
John W. Linvillede69e0f2005-09-29 14:44:57 -0700808 swiotlb_sync_single_range(hwdev, dev_addr, offset, size, dir,
809 SYNC_FOR_DEVICE);
John W. Linville878a97c2005-09-29 14:44:23 -0700810}
811
Arthur Kepner309df0c2008-04-29 01:00:32 -0700812void swiotlb_unmap_sg_attrs(struct device *, struct scatterlist *, int, int,
813 struct dma_attrs *);
John W. Linville878a97c2005-09-29 14:44:23 -0700814/*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700815 * Map a set of buffers described by scatterlist in streaming mode for DMA.
816 * This is the scatter-gather version of the above swiotlb_map_single
817 * interface. Here the scatter gather list elements are each tagged with the
818 * appropriate dma address and length. They are obtained via
819 * sg_dma_{address,length}(SG).
820 *
821 * NOTE: An implementation may be able to use a smaller number of
822 * DMA address/length pairs than there are SG table elements.
823 * (for example via virtual mapping capabilities)
824 * The routine returns the number of addr/length pairs actually
825 * used, at most nents.
826 *
827 * Device ownership issues as mentioned above for swiotlb_map_single are the
828 * same here.
829 */
830int
Arthur Kepner309df0c2008-04-29 01:00:32 -0700831swiotlb_map_sg_attrs(struct device *hwdev, struct scatterlist *sgl, int nelems,
832 int dir, struct dma_attrs *attrs)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700833{
Jens Axboedbfd49f2007-05-11 14:56:18 +0200834 struct scatterlist *sg;
Jeremy Fitzhardingeef9b1892008-12-16 12:17:33 -0800835 struct swiotlb_phys_addr buffer;
Jan Beulich563aaf02007-02-05 18:51:25 -0800836 dma_addr_t dev_addr;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700837 int i;
838
Eric Sesterhenn34814542006-03-24 18:47:11 +0100839 BUG_ON(dir == DMA_NONE);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700840
Jens Axboedbfd49f2007-05-11 14:56:18 +0200841 for_each_sg(sgl, sg, nelems, i) {
Jeremy Fitzhardingeef9b1892008-12-16 12:17:33 -0800842 dev_addr = swiotlb_sg_to_bus(sg);
Ian Campbellb81ea272008-12-16 12:17:31 -0800843 if (range_needs_mapping(sg_virt(sg), sg->length) ||
FUJITA Tomonori27979822008-09-10 01:06:49 +0900844 address_needs_mapping(hwdev, dev_addr, sg->length)) {
Jeremy Fitzhardingeef9b1892008-12-16 12:17:33 -0800845 void *map;
846 buffer.page = sg_page(sg);
847 buffer.offset = sg->offset;
848 map = map_single(hwdev, buffer, sg->length, dir);
Andi Kleen7e870232005-12-20 14:45:19 +0100849 if (!map) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700850 /* Don't panic here, we expect map_sg users
851 to do proper error handling. */
852 swiotlb_full(hwdev, sg->length, dir, 0);
Arthur Kepner309df0c2008-04-29 01:00:32 -0700853 swiotlb_unmap_sg_attrs(hwdev, sgl, i, dir,
854 attrs);
Jens Axboedbfd49f2007-05-11 14:56:18 +0200855 sgl[0].dma_length = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700856 return 0;
857 }
Ian Campbelle08e1f72008-12-16 12:17:30 -0800858 sg->dma_address = swiotlb_virt_to_bus(map);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700859 } else
860 sg->dma_address = dev_addr;
861 sg->dma_length = sg->length;
862 }
863 return nelems;
864}
Arthur Kepner309df0c2008-04-29 01:00:32 -0700865EXPORT_SYMBOL(swiotlb_map_sg_attrs);
866
867int
868swiotlb_map_sg(struct device *hwdev, struct scatterlist *sgl, int nelems,
869 int dir)
870{
871 return swiotlb_map_sg_attrs(hwdev, sgl, nelems, dir, NULL);
872}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700873
874/*
875 * Unmap a set of streaming mode DMA translations. Again, cpu read rules
876 * concerning calls here are the same as for swiotlb_unmap_single() above.
877 */
878void
Arthur Kepner309df0c2008-04-29 01:00:32 -0700879swiotlb_unmap_sg_attrs(struct device *hwdev, struct scatterlist *sgl,
880 int nelems, int dir, struct dma_attrs *attrs)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700881{
Jens Axboedbfd49f2007-05-11 14:56:18 +0200882 struct scatterlist *sg;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700883 int i;
884
Eric Sesterhenn34814542006-03-24 18:47:11 +0100885 BUG_ON(dir == DMA_NONE);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700886
Jens Axboedbfd49f2007-05-11 14:56:18 +0200887 for_each_sg(sgl, sg, nelems, i) {
Jeremy Fitzhardingeef9b1892008-12-16 12:17:33 -0800888 if (sg->dma_address != swiotlb_sg_to_bus(sg))
Ian Campbelle08e1f72008-12-16 12:17:30 -0800889 unmap_single(hwdev, swiotlb_bus_to_virt(sg->dma_address),
Jan Beulich93fbff62007-02-05 18:49:45 -0800890 sg->dma_length, dir);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700891 else if (dir == DMA_FROM_DEVICE)
Jeremy Fitzhardingeef9b1892008-12-16 12:17:33 -0800892 dma_mark_clean(swiotlb_bus_to_virt(sg->dma_address), sg->dma_length);
Jens Axboedbfd49f2007-05-11 14:56:18 +0200893 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700894}
Arthur Kepner309df0c2008-04-29 01:00:32 -0700895EXPORT_SYMBOL(swiotlb_unmap_sg_attrs);
896
897void
898swiotlb_unmap_sg(struct device *hwdev, struct scatterlist *sgl, int nelems,
899 int dir)
900{
901 return swiotlb_unmap_sg_attrs(hwdev, sgl, nelems, dir, NULL);
902}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700903
904/*
905 * Make physical memory consistent for a set of streaming mode DMA translations
906 * after a transfer.
907 *
908 * The same as swiotlb_sync_single_* but for a scatter-gather list, same rules
909 * and usage.
910 */
Andrew Mortonbe6b0262007-02-12 00:52:17 -0800911static void
Jens Axboedbfd49f2007-05-11 14:56:18 +0200912swiotlb_sync_sg(struct device *hwdev, struct scatterlist *sgl,
John W. Linvillede69e0f2005-09-29 14:44:57 -0700913 int nelems, int dir, int target)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700914{
Jens Axboedbfd49f2007-05-11 14:56:18 +0200915 struct scatterlist *sg;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700916 int i;
917
Eric Sesterhenn34814542006-03-24 18:47:11 +0100918 BUG_ON(dir == DMA_NONE);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700919
Jens Axboedbfd49f2007-05-11 14:56:18 +0200920 for_each_sg(sgl, sg, nelems, i) {
Jeremy Fitzhardingeef9b1892008-12-16 12:17:33 -0800921 if (sg->dma_address != swiotlb_sg_to_bus(sg))
Ian Campbelle08e1f72008-12-16 12:17:30 -0800922 sync_single(hwdev, swiotlb_bus_to_virt(sg->dma_address),
John W. Linvillede69e0f2005-09-29 14:44:57 -0700923 sg->dma_length, dir, target);
Jan Beulichcde14bb2007-02-05 18:46:40 -0800924 else if (dir == DMA_FROM_DEVICE)
Jeremy Fitzhardingeef9b1892008-12-16 12:17:33 -0800925 dma_mark_clean(swiotlb_bus_to_virt(sg->dma_address), sg->dma_length);
Jens Axboedbfd49f2007-05-11 14:56:18 +0200926 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700927}
928
929void
John W. Linville8270f3f2005-09-29 14:43:32 -0700930swiotlb_sync_sg_for_cpu(struct device *hwdev, struct scatterlist *sg,
931 int nelems, int dir)
932{
John W. Linvillede69e0f2005-09-29 14:44:57 -0700933 swiotlb_sync_sg(hwdev, sg, nelems, dir, SYNC_FOR_CPU);
John W. Linville8270f3f2005-09-29 14:43:32 -0700934}
935
936void
Linus Torvalds1da177e2005-04-16 15:20:36 -0700937swiotlb_sync_sg_for_device(struct device *hwdev, struct scatterlist *sg,
938 int nelems, int dir)
939{
John W. Linvillede69e0f2005-09-29 14:44:57 -0700940 swiotlb_sync_sg(hwdev, sg, nelems, dir, SYNC_FOR_DEVICE);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700941}
942
943int
FUJITA Tomonori8d8bb392008-07-25 19:44:49 -0700944swiotlb_dma_mapping_error(struct device *hwdev, dma_addr_t dma_addr)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700945{
Ian Campbelle08e1f72008-12-16 12:17:30 -0800946 return (dma_addr == swiotlb_virt_to_bus(io_tlb_overflow_buffer));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700947}
948
949/*
Tony Luck17e5ad62005-09-29 15:52:13 -0700950 * Return whether the given device DMA address mask can be supported
Linus Torvalds1da177e2005-04-16 15:20:36 -0700951 * properly. For example, if your device can only drive the low 24-bits
Tony Luck17e5ad62005-09-29 15:52:13 -0700952 * during bus mastering, then you would pass 0x00ffffff as the mask to
Linus Torvalds1da177e2005-04-16 15:20:36 -0700953 * this function.
954 */
955int
Jan Beulich563aaf02007-02-05 18:51:25 -0800956swiotlb_dma_supported(struct device *hwdev, u64 mask)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700957{
Ian Campbelle08e1f72008-12-16 12:17:30 -0800958 return swiotlb_virt_to_bus(io_tlb_end - 1) <= mask;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700959}
960
Linus Torvalds1da177e2005-04-16 15:20:36 -0700961EXPORT_SYMBOL(swiotlb_map_single);
962EXPORT_SYMBOL(swiotlb_unmap_single);
963EXPORT_SYMBOL(swiotlb_map_sg);
964EXPORT_SYMBOL(swiotlb_unmap_sg);
965EXPORT_SYMBOL(swiotlb_sync_single_for_cpu);
966EXPORT_SYMBOL(swiotlb_sync_single_for_device);
John W. Linville878a97c2005-09-29 14:44:23 -0700967EXPORT_SYMBOL_GPL(swiotlb_sync_single_range_for_cpu);
968EXPORT_SYMBOL_GPL(swiotlb_sync_single_range_for_device);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700969EXPORT_SYMBOL(swiotlb_sync_sg_for_cpu);
970EXPORT_SYMBOL(swiotlb_sync_sg_for_device);
971EXPORT_SYMBOL(swiotlb_dma_mapping_error);
Tony Luck25667d62007-03-06 13:31:45 -0800972EXPORT_SYMBOL(swiotlb_alloc_coherent);
973EXPORT_SYMBOL(swiotlb_free_coherent);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700974EXPORT_SYMBOL(swiotlb_dma_supported);