Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1 | /* |
| 2 | * This file is subject to the terms and conditions of the GNU General Public |
| 3 | * License. See the file "COPYING" in the main directory of this archive |
| 4 | * for more details. |
| 5 | * |
| 6 | * Copyright (C) 2000 Ani Joshi <ajoshi@unixbox.com> |
Ralf Baechle | 7034228 | 2013-01-22 12:59:30 +0100 | [diff] [blame] | 7 | * Copyright (C) 2000, 2001, 06 Ralf Baechle <ralf@linux-mips.org> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 8 | * swiped from i386, and cloned for MIPS by Geert, polished by Ralf. |
| 9 | */ |
Ralf Baechle | 9a88cbb | 2006-11-16 02:56:12 +0000 | [diff] [blame] | 10 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 11 | #include <linux/types.h> |
Ralf Baechle | 9a88cbb | 2006-11-16 02:56:12 +0000 | [diff] [blame] | 12 | #include <linux/dma-mapping.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 13 | #include <linux/mm.h> |
| 14 | #include <linux/module.h> |
Jens Axboe | 4fcc47a | 2007-10-23 12:32:34 +0200 | [diff] [blame] | 15 | #include <linux/scatterlist.h> |
Ralf Baechle | 6e86b0b | 2007-10-29 19:35:33 +0000 | [diff] [blame] | 16 | #include <linux/string.h> |
Tejun Heo | 5a0e3ad | 2010-03-24 17:04:11 +0900 | [diff] [blame] | 17 | #include <linux/gfp.h> |
Dezhong Diao | e36863a | 2010-10-13 16:57:35 -0700 | [diff] [blame] | 18 | #include <linux/highmem.h> |
Zubair Lutfullah Kakakhel | f464938 | 2014-07-16 16:51:32 +0100 | [diff] [blame] | 19 | #include <linux/dma-contiguous.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 20 | |
| 21 | #include <asm/cache.h> |
Ralf Baechle | 69f24d1 | 2013-09-17 10:25:47 +0200 | [diff] [blame] | 22 | #include <asm/cpu-type.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 23 | #include <asm/io.h> |
| 24 | |
Ralf Baechle | 9a88cbb | 2006-11-16 02:56:12 +0000 | [diff] [blame] | 25 | #include <dma-coherence.h> |
| 26 | |
Felix Fietkau | 885014b | 2013-09-27 14:41:44 +0200 | [diff] [blame] | 27 | #ifdef CONFIG_DMA_MAYBE_COHERENT |
Steven J. Hill | b6d92b4 | 2013-03-25 13:47:29 -0500 | [diff] [blame] | 28 | int coherentio = 0; /* User defined DMA coherency from command line. */ |
| 29 | EXPORT_SYMBOL_GPL(coherentio); |
| 30 | int hw_coherentio = 0; /* Actual hardware supported DMA coherency setting. */ |
| 31 | |
| 32 | static int __init setcoherentio(char *str) |
| 33 | { |
| 34 | coherentio = 1; |
| 35 | pr_info("Hardware DMA cache coherency (command line)\n"); |
| 36 | return 0; |
| 37 | } |
| 38 | early_param("coherentio", setcoherentio); |
| 39 | |
| 40 | static int __init setnocoherentio(char *str) |
| 41 | { |
| 42 | coherentio = 0; |
| 43 | pr_info("Software DMA cache coherency (command line)\n"); |
| 44 | return 0; |
| 45 | } |
| 46 | early_param("nocoherentio", setnocoherentio); |
Felix Fietkau | 885014b | 2013-09-27 14:41:44 +0200 | [diff] [blame] | 47 | #endif |
Steven J. Hill | b6d92b4 | 2013-03-25 13:47:29 -0500 | [diff] [blame] | 48 | |
Dezhong Diao | e36863a | 2010-10-13 16:57:35 -0700 | [diff] [blame] | 49 | static inline struct page *dma_addr_to_page(struct device *dev, |
Kevin Cernekee | 3807ef3 | 2009-04-23 17:25:12 -0700 | [diff] [blame] | 50 | dma_addr_t dma_addr) |
Franck Bui-Huu | c9d0696 | 2007-03-19 17:36:42 +0100 | [diff] [blame] | 51 | { |
Dezhong Diao | e36863a | 2010-10-13 16:57:35 -0700 | [diff] [blame] | 52 | return pfn_to_page( |
| 53 | plat_dma_addr_to_phys(dev, dma_addr) >> PAGE_SHIFT); |
Franck Bui-Huu | c9d0696 | 2007-03-19 17:36:42 +0100 | [diff] [blame] | 54 | } |
| 55 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 56 | /* |
Jim Quinlan | f86f55d | 2013-08-27 16:57:51 -0400 | [diff] [blame] | 57 | * The affected CPUs below in 'cpu_needs_post_dma_flush()' can |
| 58 | * speculatively fill random cachelines with stale data at any time, |
| 59 | * requiring an extra flush post-DMA. |
| 60 | * |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 61 | * Warning on the terminology - Linux calls an uncached area coherent; |
| 62 | * MIPS terminology calls memory areas with hardware maintained coherency |
| 63 | * coherent. |
Ralf Baechle | 0dc294c | 2014-11-11 22:22:03 +0100 | [diff] [blame] | 64 | * |
| 65 | * Note that the R14000 and R16000 should also be checked for in this |
| 66 | * condition. However this function is only called on non-I/O-coherent |
| 67 | * systems and only the R10000 and R12000 are used in such systems, the |
| 68 | * SGI IP28 Indigo² rsp. SGI IP32 aka O2. |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 69 | */ |
Jim Quinlan | f86f55d | 2013-08-27 16:57:51 -0400 | [diff] [blame] | 70 | static inline int cpu_needs_post_dma_flush(struct device *dev) |
Ralf Baechle | 9a88cbb | 2006-11-16 02:56:12 +0000 | [diff] [blame] | 71 | { |
| 72 | return !plat_device_is_coherent(dev) && |
Jerin Jacob | d451e73 | 2013-09-03 17:31:54 +0530 | [diff] [blame] | 73 | (boot_cpu_type() == CPU_R10000 || |
Ralf Baechle | eb37e6d | 2013-09-06 19:08:25 +0200 | [diff] [blame] | 74 | boot_cpu_type() == CPU_R12000 || |
| 75 | boot_cpu_type() == CPU_BMIPS5000); |
Ralf Baechle | 9a88cbb | 2006-11-16 02:56:12 +0000 | [diff] [blame] | 76 | } |
| 77 | |
Ralf Baechle | cce335ae | 2007-11-03 02:05:43 +0000 | [diff] [blame] | 78 | static gfp_t massage_gfp_flags(const struct device *dev, gfp_t gfp) |
| 79 | { |
Ralf Baechle | a2e715a | 2010-09-02 23:22:23 +0200 | [diff] [blame] | 80 | gfp_t dma_flag; |
| 81 | |
Ralf Baechle | cce335ae | 2007-11-03 02:05:43 +0000 | [diff] [blame] | 82 | /* ignore region specifiers */ |
| 83 | gfp &= ~(__GFP_DMA | __GFP_DMA32 | __GFP_HIGHMEM); |
| 84 | |
Ralf Baechle | a2e715a | 2010-09-02 23:22:23 +0200 | [diff] [blame] | 85 | #ifdef CONFIG_ISA |
Ralf Baechle | cce335ae | 2007-11-03 02:05:43 +0000 | [diff] [blame] | 86 | if (dev == NULL) |
Ralf Baechle | a2e715a | 2010-09-02 23:22:23 +0200 | [diff] [blame] | 87 | dma_flag = __GFP_DMA; |
Ralf Baechle | cce335ae | 2007-11-03 02:05:43 +0000 | [diff] [blame] | 88 | else |
| 89 | #endif |
Ralf Baechle | a2e715a | 2010-09-02 23:22:23 +0200 | [diff] [blame] | 90 | #if defined(CONFIG_ZONE_DMA32) && defined(CONFIG_ZONE_DMA) |
Ralf Baechle | cce335ae | 2007-11-03 02:05:43 +0000 | [diff] [blame] | 91 | if (dev->coherent_dma_mask < DMA_BIT_MASK(32)) |
Ralf Baechle | a2e715a | 2010-09-02 23:22:23 +0200 | [diff] [blame] | 92 | dma_flag = __GFP_DMA; |
| 93 | else if (dev->coherent_dma_mask < DMA_BIT_MASK(64)) |
| 94 | dma_flag = __GFP_DMA32; |
Ralf Baechle | cce335ae | 2007-11-03 02:05:43 +0000 | [diff] [blame] | 95 | else |
| 96 | #endif |
Ralf Baechle | a2e715a | 2010-09-02 23:22:23 +0200 | [diff] [blame] | 97 | #if defined(CONFIG_ZONE_DMA32) && !defined(CONFIG_ZONE_DMA) |
| 98 | if (dev->coherent_dma_mask < DMA_BIT_MASK(64)) |
| 99 | dma_flag = __GFP_DMA32; |
| 100 | else |
| 101 | #endif |
| 102 | #if defined(CONFIG_ZONE_DMA) && !defined(CONFIG_ZONE_DMA32) |
| 103 | if (dev->coherent_dma_mask < DMA_BIT_MASK(64)) |
| 104 | dma_flag = __GFP_DMA; |
| 105 | else |
| 106 | #endif |
| 107 | dma_flag = 0; |
Ralf Baechle | cce335ae | 2007-11-03 02:05:43 +0000 | [diff] [blame] | 108 | |
| 109 | /* Don't invoke OOM killer */ |
| 110 | gfp |= __GFP_NORETRY; |
| 111 | |
Ralf Baechle | a2e715a | 2010-09-02 23:22:23 +0200 | [diff] [blame] | 112 | return gfp | dma_flag; |
Ralf Baechle | cce335ae | 2007-11-03 02:05:43 +0000 | [diff] [blame] | 113 | } |
| 114 | |
Christoph Hellwig | 1e89375 | 2015-09-09 15:39:42 -0700 | [diff] [blame] | 115 | static void *mips_dma_alloc_noncoherent(struct device *dev, size_t size, |
Al Viro | 185a8ff | 2005-10-21 03:21:23 -0400 | [diff] [blame] | 116 | dma_addr_t * dma_handle, gfp_t gfp) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 117 | { |
| 118 | void *ret; |
Ralf Baechle | 9a88cbb | 2006-11-16 02:56:12 +0000 | [diff] [blame] | 119 | |
Ralf Baechle | cce335ae | 2007-11-03 02:05:43 +0000 | [diff] [blame] | 120 | gfp = massage_gfp_flags(dev, gfp); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 121 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 122 | ret = (void *) __get_free_pages(gfp, get_order(size)); |
| 123 | |
| 124 | if (ret != NULL) { |
| 125 | memset(ret, 0, size); |
Ralf Baechle | 9a88cbb | 2006-11-16 02:56:12 +0000 | [diff] [blame] | 126 | *dma_handle = plat_map_dma_mem(dev, ret, size); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 127 | } |
| 128 | |
| 129 | return ret; |
| 130 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 131 | |
David Daney | 48e1fd5 | 2010-10-01 13:27:32 -0700 | [diff] [blame] | 132 | static void *mips_dma_alloc_coherent(struct device *dev, size_t size, |
Andrzej Pietrasiewicz | e8d51e5 | 2012-03-27 14:32:21 +0200 | [diff] [blame] | 133 | dma_addr_t * dma_handle, gfp_t gfp, struct dma_attrs *attrs) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 134 | { |
| 135 | void *ret; |
Zubair Lutfullah Kakakhel | f464938 | 2014-07-16 16:51:32 +0100 | [diff] [blame] | 136 | struct page *page = NULL; |
| 137 | unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 138 | |
Christoph Hellwig | 1e89375 | 2015-09-09 15:39:42 -0700 | [diff] [blame] | 139 | /* |
| 140 | * XXX: seems like the coherent and non-coherent implementations could |
| 141 | * be consolidated. |
| 142 | */ |
| 143 | if (dma_get_attr(DMA_ATTR_NON_CONSISTENT, attrs)) |
| 144 | return mips_dma_alloc_noncoherent(dev, size, dma_handle, gfp); |
| 145 | |
Ralf Baechle | cce335ae | 2007-11-03 02:05:43 +0000 | [diff] [blame] | 146 | gfp = massage_gfp_flags(dev, gfp); |
Ralf Baechle | 9a88cbb | 2006-11-16 02:56:12 +0000 | [diff] [blame] | 147 | |
Zubair Lutfullah Kakakhel | f464938 | 2014-07-16 16:51:32 +0100 | [diff] [blame] | 148 | if (IS_ENABLED(CONFIG_DMA_CMA) && !(gfp & GFP_ATOMIC)) |
| 149 | page = dma_alloc_from_contiguous(dev, |
| 150 | count, get_order(size)); |
| 151 | if (!page) |
| 152 | page = alloc_pages(gfp, get_order(size)); |
Ralf Baechle | 9a88cbb | 2006-11-16 02:56:12 +0000 | [diff] [blame] | 153 | |
Zubair Lutfullah Kakakhel | f464938 | 2014-07-16 16:51:32 +0100 | [diff] [blame] | 154 | if (!page) |
| 155 | return NULL; |
Ralf Baechle | 9a88cbb | 2006-11-16 02:56:12 +0000 | [diff] [blame] | 156 | |
Zubair Lutfullah Kakakhel | f464938 | 2014-07-16 16:51:32 +0100 | [diff] [blame] | 157 | ret = page_address(page); |
| 158 | memset(ret, 0, size); |
| 159 | *dma_handle = plat_map_dma_mem(dev, ret, size); |
| 160 | if (!plat_device_is_coherent(dev)) { |
| 161 | dma_cache_wback_inv((unsigned long) ret, size); |
| 162 | if (!hw_coherentio) |
| 163 | ret = UNCAC_ADDR(ret); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 164 | } |
| 165 | |
| 166 | return ret; |
| 167 | } |
| 168 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 169 | |
Christoph Hellwig | 1e89375 | 2015-09-09 15:39:42 -0700 | [diff] [blame] | 170 | static void mips_dma_free_noncoherent(struct device *dev, size_t size, |
| 171 | void *vaddr, dma_addr_t dma_handle) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 172 | { |
Kevin Cernekee | d3f634b | 2009-04-23 17:03:43 -0700 | [diff] [blame] | 173 | plat_unmap_dma_mem(dev, dma_handle, size, DMA_BIDIRECTIONAL); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 174 | free_pages((unsigned long) vaddr, get_order(size)); |
| 175 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 176 | |
David Daney | 48e1fd5 | 2010-10-01 13:27:32 -0700 | [diff] [blame] | 177 | static void mips_dma_free_coherent(struct device *dev, size_t size, void *vaddr, |
Andrzej Pietrasiewicz | e8d51e5 | 2012-03-27 14:32:21 +0200 | [diff] [blame] | 178 | dma_addr_t dma_handle, struct dma_attrs *attrs) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 179 | { |
| 180 | unsigned long addr = (unsigned long) vaddr; |
Zubair Lutfullah Kakakhel | f464938 | 2014-07-16 16:51:32 +0100 | [diff] [blame] | 181 | unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT; |
| 182 | struct page *page = NULL; |
Yoichi Yuasa | f8ac0425 | 2009-06-04 00:16:04 +0900 | [diff] [blame] | 183 | |
Christoph Hellwig | 1e89375 | 2015-09-09 15:39:42 -0700 | [diff] [blame] | 184 | if (dma_get_attr(DMA_ATTR_NON_CONSISTENT, attrs)) { |
| 185 | mips_dma_free_noncoherent(dev, size, vaddr, dma_handle); |
| 186 | return; |
| 187 | } |
| 188 | |
Kevin Cernekee | d3f634b | 2009-04-23 17:03:43 -0700 | [diff] [blame] | 189 | plat_unmap_dma_mem(dev, dma_handle, size, DMA_BIDIRECTIONAL); |
David Daney | 11531ac | 2008-12-10 18:14:45 -0800 | [diff] [blame] | 190 | |
Steven J. Hill | b6d92b4 | 2013-03-25 13:47:29 -0500 | [diff] [blame] | 191 | if (!plat_device_is_coherent(dev) && !hw_coherentio) |
Ralf Baechle | 9a88cbb | 2006-11-16 02:56:12 +0000 | [diff] [blame] | 192 | addr = CAC_ADDR(addr); |
| 193 | |
Zubair Lutfullah Kakakhel | f464938 | 2014-07-16 16:51:32 +0100 | [diff] [blame] | 194 | page = virt_to_page((void *) addr); |
| 195 | |
| 196 | if (!dma_release_from_contiguous(dev, page, count)) |
| 197 | __free_pages(page, get_order(size)); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 198 | } |
| 199 | |
Alex Smith | 8c17246 | 2015-07-30 12:03:42 +0100 | [diff] [blame] | 200 | static int mips_dma_mmap(struct device *dev, struct vm_area_struct *vma, |
| 201 | void *cpu_addr, dma_addr_t dma_addr, size_t size, |
| 202 | struct dma_attrs *attrs) |
| 203 | { |
| 204 | unsigned long user_count = (vma->vm_end - vma->vm_start) >> PAGE_SHIFT; |
| 205 | unsigned long count = PAGE_ALIGN(size) >> PAGE_SHIFT; |
| 206 | unsigned long addr = (unsigned long)cpu_addr; |
| 207 | unsigned long off = vma->vm_pgoff; |
| 208 | unsigned long pfn; |
| 209 | int ret = -ENXIO; |
| 210 | |
| 211 | if (!plat_device_is_coherent(dev) && !hw_coherentio) |
| 212 | addr = CAC_ADDR(addr); |
| 213 | |
| 214 | pfn = page_to_pfn(virt_to_page((void *)addr)); |
| 215 | |
| 216 | if (dma_get_attr(DMA_ATTR_WRITE_COMBINE, attrs)) |
| 217 | vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot); |
| 218 | else |
| 219 | vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); |
| 220 | |
| 221 | if (dma_mmap_from_coherent(dev, vma, cpu_addr, size, &ret)) |
| 222 | return ret; |
| 223 | |
| 224 | if (off < count && user_count <= (count - off)) { |
| 225 | ret = remap_pfn_range(vma, vma->vm_start, |
| 226 | pfn + off, |
| 227 | user_count << PAGE_SHIFT, |
| 228 | vma->vm_page_prot); |
| 229 | } |
| 230 | |
| 231 | return ret; |
| 232 | } |
| 233 | |
Dezhong Diao | e36863a | 2010-10-13 16:57:35 -0700 | [diff] [blame] | 234 | static inline void __dma_sync_virtual(void *addr, size_t size, |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 235 | enum dma_data_direction direction) |
| 236 | { |
| 237 | switch (direction) { |
| 238 | case DMA_TO_DEVICE: |
Dezhong Diao | e36863a | 2010-10-13 16:57:35 -0700 | [diff] [blame] | 239 | dma_cache_wback((unsigned long)addr, size); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 240 | break; |
| 241 | |
| 242 | case DMA_FROM_DEVICE: |
Dezhong Diao | e36863a | 2010-10-13 16:57:35 -0700 | [diff] [blame] | 243 | dma_cache_inv((unsigned long)addr, size); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 244 | break; |
| 245 | |
| 246 | case DMA_BIDIRECTIONAL: |
Dezhong Diao | e36863a | 2010-10-13 16:57:35 -0700 | [diff] [blame] | 247 | dma_cache_wback_inv((unsigned long)addr, size); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 248 | break; |
| 249 | |
| 250 | default: |
| 251 | BUG(); |
| 252 | } |
| 253 | } |
| 254 | |
Dezhong Diao | e36863a | 2010-10-13 16:57:35 -0700 | [diff] [blame] | 255 | /* |
| 256 | * A single sg entry may refer to multiple physically contiguous |
| 257 | * pages. But we still need to process highmem pages individually. |
| 258 | * If highmem is not configured then the bulk of this loop gets |
| 259 | * optimized out. |
| 260 | */ |
| 261 | static inline void __dma_sync(struct page *page, |
| 262 | unsigned long offset, size_t size, enum dma_data_direction direction) |
| 263 | { |
| 264 | size_t left = size; |
| 265 | |
| 266 | do { |
| 267 | size_t len = left; |
| 268 | |
| 269 | if (PageHighMem(page)) { |
| 270 | void *addr; |
| 271 | |
| 272 | if (offset + len > PAGE_SIZE) { |
| 273 | if (offset >= PAGE_SIZE) { |
| 274 | page += offset >> PAGE_SHIFT; |
| 275 | offset &= ~PAGE_MASK; |
| 276 | } |
| 277 | len = PAGE_SIZE - offset; |
| 278 | } |
| 279 | |
| 280 | addr = kmap_atomic(page); |
| 281 | __dma_sync_virtual(addr + offset, len, direction); |
| 282 | kunmap_atomic(addr); |
| 283 | } else |
| 284 | __dma_sync_virtual(page_address(page) + offset, |
| 285 | size, direction); |
| 286 | offset = 0; |
| 287 | page++; |
| 288 | left -= len; |
| 289 | } while (left); |
| 290 | } |
| 291 | |
David Daney | 48e1fd5 | 2010-10-01 13:27:32 -0700 | [diff] [blame] | 292 | static void mips_dma_unmap_page(struct device *dev, dma_addr_t dma_addr, |
| 293 | size_t size, enum dma_data_direction direction, struct dma_attrs *attrs) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 294 | { |
Jim Quinlan | f86f55d | 2013-08-27 16:57:51 -0400 | [diff] [blame] | 295 | if (cpu_needs_post_dma_flush(dev)) |
Dezhong Diao | e36863a | 2010-10-13 16:57:35 -0700 | [diff] [blame] | 296 | __dma_sync(dma_addr_to_page(dev, dma_addr), |
| 297 | dma_addr & ~PAGE_MASK, size, direction); |
Ralf Baechle | 0acbfc6 | 2015-03-27 15:10:30 +0100 | [diff] [blame] | 298 | plat_post_dma_flush(dev); |
Kevin Cernekee | d3f634b | 2009-04-23 17:03:43 -0700 | [diff] [blame] | 299 | plat_unmap_dma_mem(dev, dma_addr, size, direction); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 300 | } |
| 301 | |
Akinobu Mita | 1e51714c | 2015-05-01 22:56:38 +0900 | [diff] [blame] | 302 | static int mips_dma_map_sg(struct device *dev, struct scatterlist *sglist, |
David Daney | 48e1fd5 | 2010-10-01 13:27:32 -0700 | [diff] [blame] | 303 | int nents, enum dma_data_direction direction, struct dma_attrs *attrs) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 304 | { |
| 305 | int i; |
Akinobu Mita | 1e51714c | 2015-05-01 22:56:38 +0900 | [diff] [blame] | 306 | struct scatterlist *sg; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 307 | |
Akinobu Mita | 1e51714c | 2015-05-01 22:56:38 +0900 | [diff] [blame] | 308 | for_each_sg(sglist, sg, nents, i) { |
Dezhong Diao | e36863a | 2010-10-13 16:57:35 -0700 | [diff] [blame] | 309 | if (!plat_device_is_coherent(dev)) |
| 310 | __dma_sync(sg_page(sg), sg->offset, sg->length, |
| 311 | direction); |
Jayachandran C | 4954a9a | 2013-06-10 06:28:08 +0000 | [diff] [blame] | 312 | #ifdef CONFIG_NEED_SG_DMA_LENGTH |
| 313 | sg->dma_length = sg->length; |
| 314 | #endif |
Dezhong Diao | e36863a | 2010-10-13 16:57:35 -0700 | [diff] [blame] | 315 | sg->dma_address = plat_map_dma_mem_page(dev, sg_page(sg)) + |
| 316 | sg->offset; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 317 | } |
| 318 | |
| 319 | return nents; |
| 320 | } |
| 321 | |
David Daney | 48e1fd5 | 2010-10-01 13:27:32 -0700 | [diff] [blame] | 322 | static dma_addr_t mips_dma_map_page(struct device *dev, struct page *page, |
| 323 | unsigned long offset, size_t size, enum dma_data_direction direction, |
| 324 | struct dma_attrs *attrs) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 325 | { |
David Daney | 48e1fd5 | 2010-10-01 13:27:32 -0700 | [diff] [blame] | 326 | if (!plat_device_is_coherent(dev)) |
Dezhong Diao | e36863a | 2010-10-13 16:57:35 -0700 | [diff] [blame] | 327 | __dma_sync(page, offset, size, direction); |
Ralf Baechle | 9a88cbb | 2006-11-16 02:56:12 +0000 | [diff] [blame] | 328 | |
Dezhong Diao | e36863a | 2010-10-13 16:57:35 -0700 | [diff] [blame] | 329 | return plat_map_dma_mem_page(dev, page) + offset; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 330 | } |
| 331 | |
Akinobu Mita | 1e51714c | 2015-05-01 22:56:38 +0900 | [diff] [blame] | 332 | static void mips_dma_unmap_sg(struct device *dev, struct scatterlist *sglist, |
David Daney | 48e1fd5 | 2010-10-01 13:27:32 -0700 | [diff] [blame] | 333 | int nhwentries, enum dma_data_direction direction, |
| 334 | struct dma_attrs *attrs) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 335 | { |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 336 | int i; |
Akinobu Mita | 1e51714c | 2015-05-01 22:56:38 +0900 | [diff] [blame] | 337 | struct scatterlist *sg; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 338 | |
Akinobu Mita | 1e51714c | 2015-05-01 22:56:38 +0900 | [diff] [blame] | 339 | for_each_sg(sglist, sg, nhwentries, i) { |
Ralf Baechle | 9a88cbb | 2006-11-16 02:56:12 +0000 | [diff] [blame] | 340 | if (!plat_device_is_coherent(dev) && |
Dezhong Diao | e36863a | 2010-10-13 16:57:35 -0700 | [diff] [blame] | 341 | direction != DMA_TO_DEVICE) |
| 342 | __dma_sync(sg_page(sg), sg->offset, sg->length, |
| 343 | direction); |
Kevin Cernekee | d3f634b | 2009-04-23 17:03:43 -0700 | [diff] [blame] | 344 | plat_unmap_dma_mem(dev, sg->dma_address, sg->length, direction); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 345 | } |
| 346 | } |
| 347 | |
David Daney | 48e1fd5 | 2010-10-01 13:27:32 -0700 | [diff] [blame] | 348 | static void mips_dma_sync_single_for_cpu(struct device *dev, |
| 349 | dma_addr_t dma_handle, size_t size, enum dma_data_direction direction) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 350 | { |
Jim Quinlan | f86f55d | 2013-08-27 16:57:51 -0400 | [diff] [blame] | 351 | if (cpu_needs_post_dma_flush(dev)) |
Dezhong Diao | e36863a | 2010-10-13 16:57:35 -0700 | [diff] [blame] | 352 | __dma_sync(dma_addr_to_page(dev, dma_handle), |
| 353 | dma_handle & ~PAGE_MASK, size, direction); |
Ralf Baechle | 0acbfc6 | 2015-03-27 15:10:30 +0100 | [diff] [blame] | 354 | plat_post_dma_flush(dev); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 355 | } |
| 356 | |
David Daney | 48e1fd5 | 2010-10-01 13:27:32 -0700 | [diff] [blame] | 357 | static void mips_dma_sync_single_for_device(struct device *dev, |
| 358 | dma_addr_t dma_handle, size_t size, enum dma_data_direction direction) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 359 | { |
Dezhong Diao | e36863a | 2010-10-13 16:57:35 -0700 | [diff] [blame] | 360 | if (!plat_device_is_coherent(dev)) |
| 361 | __dma_sync(dma_addr_to_page(dev, dma_handle), |
| 362 | dma_handle & ~PAGE_MASK, size, direction); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 363 | } |
| 364 | |
David Daney | 48e1fd5 | 2010-10-01 13:27:32 -0700 | [diff] [blame] | 365 | static void mips_dma_sync_sg_for_cpu(struct device *dev, |
Akinobu Mita | 1e51714c | 2015-05-01 22:56:38 +0900 | [diff] [blame] | 366 | struct scatterlist *sglist, int nelems, |
| 367 | enum dma_data_direction direction) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 368 | { |
| 369 | int i; |
Akinobu Mita | 1e51714c | 2015-05-01 22:56:38 +0900 | [diff] [blame] | 370 | struct scatterlist *sg; |
Ralf Baechle | 42a3b4f | 2005-09-03 15:56:17 -0700 | [diff] [blame] | 371 | |
Akinobu Mita | 1e51714c | 2015-05-01 22:56:38 +0900 | [diff] [blame] | 372 | if (cpu_needs_post_dma_flush(dev)) { |
| 373 | for_each_sg(sglist, sg, nelems, i) { |
Dezhong Diao | e36863a | 2010-10-13 16:57:35 -0700 | [diff] [blame] | 374 | __dma_sync(sg_page(sg), sg->offset, sg->length, |
| 375 | direction); |
Akinobu Mita | 1e51714c | 2015-05-01 22:56:38 +0900 | [diff] [blame] | 376 | } |
| 377 | } |
Ralf Baechle | 0acbfc6 | 2015-03-27 15:10:30 +0100 | [diff] [blame] | 378 | plat_post_dma_flush(dev); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 379 | } |
| 380 | |
David Daney | 48e1fd5 | 2010-10-01 13:27:32 -0700 | [diff] [blame] | 381 | static void mips_dma_sync_sg_for_device(struct device *dev, |
Akinobu Mita | 1e51714c | 2015-05-01 22:56:38 +0900 | [diff] [blame] | 382 | struct scatterlist *sglist, int nelems, |
| 383 | enum dma_data_direction direction) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 384 | { |
| 385 | int i; |
Akinobu Mita | 1e51714c | 2015-05-01 22:56:38 +0900 | [diff] [blame] | 386 | struct scatterlist *sg; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 387 | |
Akinobu Mita | 1e51714c | 2015-05-01 22:56:38 +0900 | [diff] [blame] | 388 | if (!plat_device_is_coherent(dev)) { |
| 389 | for_each_sg(sglist, sg, nelems, i) { |
Dezhong Diao | e36863a | 2010-10-13 16:57:35 -0700 | [diff] [blame] | 390 | __dma_sync(sg_page(sg), sg->offset, sg->length, |
| 391 | direction); |
Akinobu Mita | 1e51714c | 2015-05-01 22:56:38 +0900 | [diff] [blame] | 392 | } |
| 393 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 394 | } |
| 395 | |
David Daney | 48e1fd5 | 2010-10-01 13:27:32 -0700 | [diff] [blame] | 396 | int mips_dma_mapping_error(struct device *dev, dma_addr_t dma_addr) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 397 | { |
Felix Fietkau | 4e7f726 | 2013-08-15 11:28:30 +0200 | [diff] [blame] | 398 | return 0; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 399 | } |
| 400 | |
David Daney | 48e1fd5 | 2010-10-01 13:27:32 -0700 | [diff] [blame] | 401 | int mips_dma_supported(struct device *dev, u64 mask) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 402 | { |
David Daney | 843aef4 | 2008-12-11 15:33:36 -0800 | [diff] [blame] | 403 | return plat_dma_supported(dev, mask); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 404 | } |
| 405 | |
Ralf Baechle | a3aad4a | 2010-12-09 19:14:09 +0000 | [diff] [blame] | 406 | void dma_cache_sync(struct device *dev, void *vaddr, size_t size, |
David Daney | 48e1fd5 | 2010-10-01 13:27:32 -0700 | [diff] [blame] | 407 | enum dma_data_direction direction) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 408 | { |
Ralf Baechle | 9a88cbb | 2006-11-16 02:56:12 +0000 | [diff] [blame] | 409 | BUG_ON(direction == DMA_NONE); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 410 | |
Ralf Baechle | 9a88cbb | 2006-11-16 02:56:12 +0000 | [diff] [blame] | 411 | if (!plat_device_is_coherent(dev)) |
Dezhong Diao | e36863a | 2010-10-13 16:57:35 -0700 | [diff] [blame] | 412 | __dma_sync_virtual(vaddr, size, direction); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 413 | } |
| 414 | |
Ralf Baechle | a3aad4a | 2010-12-09 19:14:09 +0000 | [diff] [blame] | 415 | EXPORT_SYMBOL(dma_cache_sync); |
| 416 | |
David Daney | 48e1fd5 | 2010-10-01 13:27:32 -0700 | [diff] [blame] | 417 | static struct dma_map_ops mips_default_dma_map_ops = { |
Andrzej Pietrasiewicz | e8d51e5 | 2012-03-27 14:32:21 +0200 | [diff] [blame] | 418 | .alloc = mips_dma_alloc_coherent, |
| 419 | .free = mips_dma_free_coherent, |
Alex Smith | 8c17246 | 2015-07-30 12:03:42 +0100 | [diff] [blame] | 420 | .mmap = mips_dma_mmap, |
David Daney | 48e1fd5 | 2010-10-01 13:27:32 -0700 | [diff] [blame] | 421 | .map_page = mips_dma_map_page, |
| 422 | .unmap_page = mips_dma_unmap_page, |
| 423 | .map_sg = mips_dma_map_sg, |
| 424 | .unmap_sg = mips_dma_unmap_sg, |
| 425 | .sync_single_for_cpu = mips_dma_sync_single_for_cpu, |
| 426 | .sync_single_for_device = mips_dma_sync_single_for_device, |
| 427 | .sync_sg_for_cpu = mips_dma_sync_sg_for_cpu, |
| 428 | .sync_sg_for_device = mips_dma_sync_sg_for_device, |
| 429 | .mapping_error = mips_dma_mapping_error, |
| 430 | .dma_supported = mips_dma_supported |
| 431 | }; |
| 432 | |
| 433 | struct dma_map_ops *mips_dma_map_ops = &mips_default_dma_map_ops; |
| 434 | EXPORT_SYMBOL(mips_dma_map_ops); |
| 435 | |
| 436 | #define PREALLOC_DMA_DEBUG_ENTRIES (1 << 16) |
| 437 | |
| 438 | static int __init mips_dma_init(void) |
| 439 | { |
| 440 | dma_debug_init(PREALLOC_DMA_DEBUG_ENTRIES); |
| 441 | |
| 442 | return 0; |
| 443 | } |
| 444 | fs_initcall(mips_dma_init); |