blob: f503d02e403bbf78bdb421402ff6a6dfddf6f742 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * This file is subject to the terms and conditions of the GNU General Public
3 * License. See the file "COPYING" in the main directory of this archive
4 * for more details.
5 *
6 * Copyright (C) 2000 Ani Joshi <ajoshi@unixbox.com>
Ralf Baechle9a88cbb2006-11-16 02:56:12 +00007 * Copyright (C) 2000, 2001, 06 Ralf Baechle <ralf@linux-mips.org>
Linus Torvalds1da177e2005-04-16 15:20:36 -07008 * swiped from i386, and cloned for MIPS by Geert, polished by Ralf.
9 */
Ralf Baechle9a88cbb2006-11-16 02:56:12 +000010
Linus Torvalds1da177e2005-04-16 15:20:36 -070011#include <linux/types.h>
Ralf Baechle9a88cbb2006-11-16 02:56:12 +000012#include <linux/dma-mapping.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070013#include <linux/mm.h>
14#include <linux/module.h>
15#include <linux/string.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070016
17#include <asm/cache.h>
18#include <asm/io.h>
19
Ralf Baechle9a88cbb2006-11-16 02:56:12 +000020#include <dma-coherence.h>
21
Linus Torvalds1da177e2005-04-16 15:20:36 -070022/*
23 * Warning on the terminology - Linux calls an uncached area coherent;
24 * MIPS terminology calls memory areas with hardware maintained coherency
25 * coherent.
26 */
27
Ralf Baechle9a88cbb2006-11-16 02:56:12 +000028static inline int cpu_is_noncoherent_r10000(struct device *dev)
29{
30 return !plat_device_is_coherent(dev) &&
31 (current_cpu_data.cputype == CPU_R10000 &&
32 current_cpu_data.cputype == CPU_R12000);
33}
34
Linus Torvalds1da177e2005-04-16 15:20:36 -070035void *dma_alloc_noncoherent(struct device *dev, size_t size,
Al Viro185a8ff2005-10-21 03:21:23 -040036 dma_addr_t * dma_handle, gfp_t gfp)
Linus Torvalds1da177e2005-04-16 15:20:36 -070037{
38 void *ret;
Ralf Baechle9a88cbb2006-11-16 02:56:12 +000039
Linus Torvalds1da177e2005-04-16 15:20:36 -070040 /* ignore region specifiers */
41 gfp &= ~(__GFP_DMA | __GFP_HIGHMEM);
42
43 if (dev == NULL || (dev->coherent_dma_mask < 0xffffffff))
44 gfp |= GFP_DMA;
45 ret = (void *) __get_free_pages(gfp, get_order(size));
46
47 if (ret != NULL) {
48 memset(ret, 0, size);
Ralf Baechle9a88cbb2006-11-16 02:56:12 +000049 *dma_handle = plat_map_dma_mem(dev, ret, size);
Linus Torvalds1da177e2005-04-16 15:20:36 -070050 }
51
52 return ret;
53}
54
55EXPORT_SYMBOL(dma_alloc_noncoherent);
56
57void *dma_alloc_coherent(struct device *dev, size_t size,
Al Viro185a8ff2005-10-21 03:21:23 -040058 dma_addr_t * dma_handle, gfp_t gfp)
Linus Torvalds1da177e2005-04-16 15:20:36 -070059{
60 void *ret;
61
Ralf Baechle9a88cbb2006-11-16 02:56:12 +000062 /* ignore region specifiers */
63 gfp &= ~(__GFP_DMA | __GFP_HIGHMEM);
64
65 if (dev == NULL || (dev->coherent_dma_mask < 0xffffffff))
66 gfp |= GFP_DMA;
67 ret = (void *) __get_free_pages(gfp, get_order(size));
68
Linus Torvalds1da177e2005-04-16 15:20:36 -070069 if (ret) {
Ralf Baechle9a88cbb2006-11-16 02:56:12 +000070 memset(ret, 0, size);
71 *dma_handle = plat_map_dma_mem(dev, ret, size);
72
73 if (!plat_device_is_coherent(dev)) {
74 dma_cache_wback_inv((unsigned long) ret, size);
75 ret = UNCAC_ADDR(ret);
76 }
Linus Torvalds1da177e2005-04-16 15:20:36 -070077 }
78
79 return ret;
80}
81
82EXPORT_SYMBOL(dma_alloc_coherent);
83
84void dma_free_noncoherent(struct device *dev, size_t size, void *vaddr,
85 dma_addr_t dma_handle)
86{
87 free_pages((unsigned long) vaddr, get_order(size));
88}
89
90EXPORT_SYMBOL(dma_free_noncoherent);
91
92void dma_free_coherent(struct device *dev, size_t size, void *vaddr,
93 dma_addr_t dma_handle)
94{
95 unsigned long addr = (unsigned long) vaddr;
96
Ralf Baechle9a88cbb2006-11-16 02:56:12 +000097 if (!plat_device_is_coherent(dev))
98 addr = CAC_ADDR(addr);
99
Linus Torvalds1da177e2005-04-16 15:20:36 -0700100 free_pages(addr, get_order(size));
101}
102
103EXPORT_SYMBOL(dma_free_coherent);
104
105static inline void __dma_sync(unsigned long addr, size_t size,
106 enum dma_data_direction direction)
107{
108 switch (direction) {
109 case DMA_TO_DEVICE:
110 dma_cache_wback(addr, size);
111 break;
112
113 case DMA_FROM_DEVICE:
114 dma_cache_inv(addr, size);
115 break;
116
117 case DMA_BIDIRECTIONAL:
118 dma_cache_wback_inv(addr, size);
119 break;
120
121 default:
122 BUG();
123 }
124}
125
126dma_addr_t dma_map_single(struct device *dev, void *ptr, size_t size,
127 enum dma_data_direction direction)
128{
129 unsigned long addr = (unsigned long) ptr;
130
Ralf Baechle9a88cbb2006-11-16 02:56:12 +0000131 if (!plat_device_is_coherent(dev))
132 __dma_sync(addr, size, direction);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700133
Ralf Baechle9a88cbb2006-11-16 02:56:12 +0000134 return plat_map_dma_mem(dev, ptr, size);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700135}
136
137EXPORT_SYMBOL(dma_map_single);
138
139void dma_unmap_single(struct device *dev, dma_addr_t dma_addr, size_t size,
140 enum dma_data_direction direction)
141{
Ralf Baechle9a88cbb2006-11-16 02:56:12 +0000142 if (cpu_is_noncoherent_r10000(dev))
143 __dma_sync(plat_dma_addr_to_phys(dma_addr) + PAGE_OFFSET, size,
144 direction);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700145
Ralf Baechle9a88cbb2006-11-16 02:56:12 +0000146 plat_unmap_dma_mem(dma_addr);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700147}
148
149EXPORT_SYMBOL(dma_unmap_single);
150
151int dma_map_sg(struct device *dev, struct scatterlist *sg, int nents,
152 enum dma_data_direction direction)
153{
154 int i;
155
156 BUG_ON(direction == DMA_NONE);
157
158 for (i = 0; i < nents; i++, sg++) {
159 unsigned long addr;
Ralf Baechle42a3b4f2005-09-03 15:56:17 -0700160
Linus Torvalds1da177e2005-04-16 15:20:36 -0700161 addr = (unsigned long) page_address(sg->page);
Ralf Baechle9a88cbb2006-11-16 02:56:12 +0000162 if (!plat_device_is_coherent(dev) && addr)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700163 __dma_sync(addr + sg->offset, sg->length, direction);
Ralf Baechle9a88cbb2006-11-16 02:56:12 +0000164 sg->dma_address = plat_map_dma_mem_page(dev, sg->page) +
165 sg->offset;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700166 }
167
168 return nents;
169}
170
171EXPORT_SYMBOL(dma_map_sg);
172
173dma_addr_t dma_map_page(struct device *dev, struct page *page,
174 unsigned long offset, size_t size, enum dma_data_direction direction)
175{
Linus Torvalds1da177e2005-04-16 15:20:36 -0700176 BUG_ON(direction == DMA_NONE);
177
Ralf Baechle9a88cbb2006-11-16 02:56:12 +0000178 if (!plat_device_is_coherent(dev)) {
179 unsigned long addr;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700180
Ralf Baechle9a88cbb2006-11-16 02:56:12 +0000181 addr = (unsigned long) page_address(page) + offset;
182 dma_cache_wback_inv(addr, size);
183 }
184
185 return plat_map_dma_mem_page(dev, page) + offset;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700186}
187
188EXPORT_SYMBOL(dma_map_page);
189
190void dma_unmap_page(struct device *dev, dma_addr_t dma_address, size_t size,
191 enum dma_data_direction direction)
192{
193 BUG_ON(direction == DMA_NONE);
194
Ralf Baechle9a88cbb2006-11-16 02:56:12 +0000195 if (!plat_device_is_coherent(dev) && direction != DMA_TO_DEVICE) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700196 unsigned long addr;
197
Ralf Baechle9a88cbb2006-11-16 02:56:12 +0000198 addr = plat_dma_addr_to_phys(dma_address);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700199 dma_cache_wback_inv(addr, size);
200 }
Ralf Baechle9a88cbb2006-11-16 02:56:12 +0000201
202 plat_unmap_dma_mem(dma_address);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700203}
204
205EXPORT_SYMBOL(dma_unmap_page);
206
207void dma_unmap_sg(struct device *dev, struct scatterlist *sg, int nhwentries,
208 enum dma_data_direction direction)
209{
210 unsigned long addr;
211 int i;
212
213 BUG_ON(direction == DMA_NONE);
214
Linus Torvalds1da177e2005-04-16 15:20:36 -0700215 for (i = 0; i < nhwentries; i++, sg++) {
Ralf Baechle9a88cbb2006-11-16 02:56:12 +0000216 if (!plat_device_is_coherent(dev) &&
217 direction != DMA_TO_DEVICE) {
218 addr = (unsigned long) page_address(sg->page);
219 if (addr)
220 __dma_sync(addr + sg->offset, sg->length,
221 direction);
222 }
223 plat_unmap_dma_mem(sg->dma_address);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700224 }
225}
226
227EXPORT_SYMBOL(dma_unmap_sg);
228
229void dma_sync_single_for_cpu(struct device *dev, dma_addr_t dma_handle,
230 size_t size, enum dma_data_direction direction)
231{
Linus Torvalds1da177e2005-04-16 15:20:36 -0700232 BUG_ON(direction == DMA_NONE);
Ralf Baechle42a3b4f2005-09-03 15:56:17 -0700233
Ralf Baechle9a88cbb2006-11-16 02:56:12 +0000234 if (cpu_is_noncoherent_r10000(dev)) {
235 unsigned long addr;
236
237 addr = PAGE_OFFSET + plat_dma_addr_to_phys(dma_handle);
238 __dma_sync(addr, size, direction);
239 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700240}
241
242EXPORT_SYMBOL(dma_sync_single_for_cpu);
243
244void dma_sync_single_for_device(struct device *dev, dma_addr_t dma_handle,
245 size_t size, enum dma_data_direction direction)
246{
Linus Torvalds1da177e2005-04-16 15:20:36 -0700247 BUG_ON(direction == DMA_NONE);
248
Thomas Bogendoerfer9b43fb62007-02-23 19:58:48 +0100249 if (!plat_device_is_coherent(dev)) {
Ralf Baechle9a88cbb2006-11-16 02:56:12 +0000250 unsigned long addr;
251
Thomas Bogendoerfer9b43fb62007-02-23 19:58:48 +0100252 addr = PAGE_OFFSET + plat_dma_addr_to_phys(dma_handle);
Ralf Baechle9a88cbb2006-11-16 02:56:12 +0000253 __dma_sync(addr, size, direction);
254 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700255}
256
257EXPORT_SYMBOL(dma_sync_single_for_device);
258
259void dma_sync_single_range_for_cpu(struct device *dev, dma_addr_t dma_handle,
260 unsigned long offset, size_t size, enum dma_data_direction direction)
261{
Linus Torvalds1da177e2005-04-16 15:20:36 -0700262 BUG_ON(direction == DMA_NONE);
263
Ralf Baechle9a88cbb2006-11-16 02:56:12 +0000264 if (cpu_is_noncoherent_r10000(dev)) {
265 unsigned long addr;
266
267 addr = PAGE_OFFSET + plat_dma_addr_to_phys(dma_handle);
268 __dma_sync(addr + offset, size, direction);
269 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700270}
271
272EXPORT_SYMBOL(dma_sync_single_range_for_cpu);
273
274void dma_sync_single_range_for_device(struct device *dev, dma_addr_t dma_handle,
275 unsigned long offset, size_t size, enum dma_data_direction direction)
276{
Linus Torvalds1da177e2005-04-16 15:20:36 -0700277 BUG_ON(direction == DMA_NONE);
278
Thomas Bogendoerfer9b43fb62007-02-23 19:58:48 +0100279 if (!plat_device_is_coherent(dev)) {
Ralf Baechle9a88cbb2006-11-16 02:56:12 +0000280 unsigned long addr;
281
282 addr = PAGE_OFFSET + plat_dma_addr_to_phys(dma_handle);
283 __dma_sync(addr + offset, size, direction);
284 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700285}
286
287EXPORT_SYMBOL(dma_sync_single_range_for_device);
288
289void dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg, int nelems,
290 enum dma_data_direction direction)
291{
292 int i;
Ralf Baechle42a3b4f2005-09-03 15:56:17 -0700293
Linus Torvalds1da177e2005-04-16 15:20:36 -0700294 BUG_ON(direction == DMA_NONE);
Ralf Baechle42a3b4f2005-09-03 15:56:17 -0700295
Linus Torvalds1da177e2005-04-16 15:20:36 -0700296 /* Make sure that gcc doesn't leave the empty loop body. */
Ralf Baechle9a88cbb2006-11-16 02:56:12 +0000297 for (i = 0; i < nelems; i++, sg++) {
Ralf Baechle5b648a92007-03-02 11:42:11 +0000298 if (cpu_is_noncoherent_r10000(dev))
Ralf Baechle9a88cbb2006-11-16 02:56:12 +0000299 __dma_sync((unsigned long)page_address(sg->page),
300 sg->length, direction);
301 plat_unmap_dma_mem(sg->dma_address);
302 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700303}
304
305EXPORT_SYMBOL(dma_sync_sg_for_cpu);
306
307void dma_sync_sg_for_device(struct device *dev, struct scatterlist *sg, int nelems,
308 enum dma_data_direction direction)
309{
310 int i;
311
312 BUG_ON(direction == DMA_NONE);
313
314 /* Make sure that gcc doesn't leave the empty loop body. */
Ralf Baechle9a88cbb2006-11-16 02:56:12 +0000315 for (i = 0; i < nelems; i++, sg++) {
316 if (!plat_device_is_coherent(dev))
317 __dma_sync((unsigned long)page_address(sg->page),
318 sg->length, direction);
319 plat_unmap_dma_mem(sg->dma_address);
320 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700321}
322
323EXPORT_SYMBOL(dma_sync_sg_for_device);
324
325int dma_mapping_error(dma_addr_t dma_addr)
326{
327 return 0;
328}
329
330EXPORT_SYMBOL(dma_mapping_error);
331
332int dma_supported(struct device *dev, u64 mask)
333{
334 /*
335 * we fall back to GFP_DMA when the mask isn't all 1s,
336 * so we can't guarantee allocations that must be
337 * within a tighter range than GFP_DMA..
338 */
339 if (mask < 0x00ffffff)
340 return 0;
341
342 return 1;
343}
344
345EXPORT_SYMBOL(dma_supported);
346
Ralf Baechlef67637e2006-12-06 20:38:54 -0800347int dma_is_consistent(struct device *dev, dma_addr_t dma_addr)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700348{
Ralf Baechle9a88cbb2006-11-16 02:56:12 +0000349 return plat_device_is_coherent(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700350}
351
352EXPORT_SYMBOL(dma_is_consistent);
353
Ralf Baechled3fa72e2006-12-06 20:38:56 -0800354void dma_cache_sync(struct device *dev, void *vaddr, size_t size,
Ralf Baechle9a88cbb2006-11-16 02:56:12 +0000355 enum dma_data_direction direction)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700356{
Ralf Baechle9a88cbb2006-11-16 02:56:12 +0000357 BUG_ON(direction == DMA_NONE);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700358
Ralf Baechle9a88cbb2006-11-16 02:56:12 +0000359 if (!plat_device_is_coherent(dev))
360 dma_cache_wback_inv((unsigned long)vaddr, size);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700361}
362
363EXPORT_SYMBOL(dma_cache_sync);