Konrad Rzeszutek Wilk | b097186 | 2010-05-11 10:05:49 -0400 | [diff] [blame] | 1 | /* |
| 2 | * Copyright 2010 |
| 3 | * by Konrad Rzeszutek Wilk <konrad.wilk@oracle.com> |
| 4 | * |
| 5 | * This code provides a IOMMU for Xen PV guests with PCI passthrough. |
| 6 | * |
| 7 | * This program is free software; you can redistribute it and/or modify |
| 8 | * it under the terms of the GNU General Public License v2.0 as published by |
| 9 | * the Free Software Foundation |
| 10 | * |
| 11 | * This program is distributed in the hope that it will be useful, |
| 12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of |
| 13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
| 14 | * GNU General Public License for more details. |
| 15 | * |
| 16 | * PV guests under Xen are running in an non-contiguous memory architecture. |
| 17 | * |
| 18 | * When PCI pass-through is utilized, this necessitates an IOMMU for |
| 19 | * translating bus (DMA) to virtual and vice-versa and also providing a |
| 20 | * mechanism to have contiguous pages for device drivers operations (say DMA |
| 21 | * operations). |
| 22 | * |
| 23 | * Specifically, under Xen the Linux idea of pages is an illusion. It |
| 24 | * assumes that pages start at zero and go up to the available memory. To |
| 25 | * help with that, the Linux Xen MMU provides a lookup mechanism to |
| 26 | * translate the page frame numbers (PFN) to machine frame numbers (MFN) |
| 27 | * and vice-versa. The MFN are the "real" frame numbers. Furthermore |
| 28 | * memory is not contiguous. Xen hypervisor stitches memory for guests |
| 29 | * from different pools, which means there is no guarantee that PFN==MFN |
| 30 | * and PFN+1==MFN+1. Lastly with Xen 4.0, pages (in debug mode) are |
| 31 | * allocated in descending order (high to low), meaning the guest might |
| 32 | * never get any MFN's under the 4GB mark. |
| 33 | * |
| 34 | */ |
| 35 | |
Joe Perches | 283c097 | 2013-06-28 03:21:41 -0700 | [diff] [blame] | 36 | #define pr_fmt(fmt) "xen:" KBUILD_MODNAME ": " fmt |
| 37 | |
Konrad Rzeszutek Wilk | b097186 | 2010-05-11 10:05:49 -0400 | [diff] [blame] | 38 | #include <linux/bootmem.h> |
| 39 | #include <linux/dma-mapping.h> |
Paul Gortmaker | 63c9744 | 2011-07-10 13:22:07 -0400 | [diff] [blame] | 40 | #include <linux/export.h> |
Konrad Rzeszutek Wilk | b097186 | 2010-05-11 10:05:49 -0400 | [diff] [blame] | 41 | #include <xen/swiotlb-xen.h> |
| 42 | #include <xen/page.h> |
| 43 | #include <xen/xen-ops.h> |
Konrad Rzeszutek Wilk | f4b2f07 | 2011-07-22 12:46:43 -0400 | [diff] [blame] | 44 | #include <xen/hvc-console.h> |
Zoltan Kiss | 2b2b614 | 2013-09-04 21:11:05 +0100 | [diff] [blame] | 45 | |
Stefano Stabellini | 83862cc | 2013-10-10 13:40:44 +0000 | [diff] [blame] | 46 | #include <asm/dma-mapping.h> |
Stefano Stabellini | 1b65c4e | 2013-10-10 13:41:10 +0000 | [diff] [blame] | 47 | #include <asm/xen/page-coherent.h> |
Konrad Rzeszutek Wilk | e1d8f62 | 2013-11-08 15:36:09 -0500 | [diff] [blame] | 48 | |
Zoltan Kiss | 2b2b614 | 2013-09-04 21:11:05 +0100 | [diff] [blame] | 49 | #include <trace/events/swiotlb.h> |
Konrad Rzeszutek Wilk | b097186 | 2010-05-11 10:05:49 -0400 | [diff] [blame] | 50 | /* |
| 51 | * Used to do a quick range check in swiotlb_tbl_unmap_single and |
| 52 | * swiotlb_tbl_sync_single_*, to see if the memory was in fact allocated by this |
| 53 | * API. |
| 54 | */ |
| 55 | |
Stefano Stabellini | 83862cc | 2013-10-10 13:40:44 +0000 | [diff] [blame] | 56 | #ifndef CONFIG_X86 |
| 57 | static unsigned long dma_alloc_coherent_mask(struct device *dev, |
| 58 | gfp_t gfp) |
| 59 | { |
| 60 | unsigned long dma_mask = 0; |
| 61 | |
| 62 | dma_mask = dev->coherent_dma_mask; |
| 63 | if (!dma_mask) |
| 64 | dma_mask = (gfp & GFP_DMA) ? DMA_BIT_MASK(24) : DMA_BIT_MASK(32); |
| 65 | |
| 66 | return dma_mask; |
| 67 | } |
| 68 | #endif |
| 69 | |
Konrad Rzeszutek Wilk | b097186 | 2010-05-11 10:05:49 -0400 | [diff] [blame] | 70 | static char *xen_io_tlb_start, *xen_io_tlb_end; |
| 71 | static unsigned long xen_io_tlb_nslabs; |
| 72 | /* |
| 73 | * Quick lookup value of the bus address of the IOTLB. |
| 74 | */ |
| 75 | |
Konrad Rzeszutek Wilk | b8b0f55 | 2012-08-21 14:49:34 -0400 | [diff] [blame] | 76 | static u64 start_dma_addr; |
Konrad Rzeszutek Wilk | b097186 | 2010-05-11 10:05:49 -0400 | [diff] [blame] | 77 | |
Ian Campbell | e17b2f1 | 2014-01-20 11:30:41 +0000 | [diff] [blame] | 78 | /* |
Julien Grall | 9435cce | 2015-09-09 15:18:45 +0100 | [diff] [blame] | 79 | * Both of these functions should avoid XEN_PFN_PHYS because phys_addr_t |
Ian Campbell | e17b2f1 | 2014-01-20 11:30:41 +0000 | [diff] [blame] | 80 | * can be 32bit when dma_addr_t is 64bit leading to a loss in |
| 81 | * information if the shift is done before casting to 64bit. |
| 82 | */ |
Stefano Stabellini | 6b42a7e | 2013-10-25 10:33:27 +0000 | [diff] [blame] | 83 | static inline dma_addr_t xen_phys_to_bus(phys_addr_t paddr) |
Konrad Rzeszutek Wilk | b097186 | 2010-05-11 10:05:49 -0400 | [diff] [blame] | 84 | { |
Julien Grall | 9435cce | 2015-09-09 15:18:45 +0100 | [diff] [blame] | 85 | unsigned long bfn = pfn_to_bfn(XEN_PFN_DOWN(paddr)); |
| 86 | dma_addr_t dma = (dma_addr_t)bfn << XEN_PAGE_SHIFT; |
Ian Campbell | e17b2f1 | 2014-01-20 11:30:41 +0000 | [diff] [blame] | 87 | |
Julien Grall | 9435cce | 2015-09-09 15:18:45 +0100 | [diff] [blame] | 88 | dma |= paddr & ~XEN_PAGE_MASK; |
Ian Campbell | e17b2f1 | 2014-01-20 11:30:41 +0000 | [diff] [blame] | 89 | |
| 90 | return dma; |
Konrad Rzeszutek Wilk | b097186 | 2010-05-11 10:05:49 -0400 | [diff] [blame] | 91 | } |
| 92 | |
Stefano Stabellini | 6b42a7e | 2013-10-25 10:33:27 +0000 | [diff] [blame] | 93 | static inline phys_addr_t xen_bus_to_phys(dma_addr_t baddr) |
Konrad Rzeszutek Wilk | b097186 | 2010-05-11 10:05:49 -0400 | [diff] [blame] | 94 | { |
Julien Grall | 9435cce | 2015-09-09 15:18:45 +0100 | [diff] [blame] | 95 | unsigned long xen_pfn = bfn_to_pfn(XEN_PFN_DOWN(baddr)); |
| 96 | dma_addr_t dma = (dma_addr_t)xen_pfn << XEN_PAGE_SHIFT; |
Ian Campbell | e17b2f1 | 2014-01-20 11:30:41 +0000 | [diff] [blame] | 97 | phys_addr_t paddr = dma; |
| 98 | |
Julien Grall | 9435cce | 2015-09-09 15:18:45 +0100 | [diff] [blame] | 99 | paddr |= baddr & ~XEN_PAGE_MASK; |
Ian Campbell | e17b2f1 | 2014-01-20 11:30:41 +0000 | [diff] [blame] | 100 | |
| 101 | return paddr; |
Konrad Rzeszutek Wilk | b097186 | 2010-05-11 10:05:49 -0400 | [diff] [blame] | 102 | } |
| 103 | |
Stefano Stabellini | 6b42a7e | 2013-10-25 10:33:27 +0000 | [diff] [blame] | 104 | static inline dma_addr_t xen_virt_to_bus(void *address) |
Konrad Rzeszutek Wilk | b097186 | 2010-05-11 10:05:49 -0400 | [diff] [blame] | 105 | { |
| 106 | return xen_phys_to_bus(virt_to_phys(address)); |
| 107 | } |
| 108 | |
Julien Grall | 9435cce | 2015-09-09 15:18:45 +0100 | [diff] [blame] | 109 | static int check_pages_physically_contiguous(unsigned long xen_pfn, |
Konrad Rzeszutek Wilk | b097186 | 2010-05-11 10:05:49 -0400 | [diff] [blame] | 110 | unsigned int offset, |
| 111 | size_t length) |
| 112 | { |
Julien Grall | 32e0987 | 2015-08-07 17:34:35 +0100 | [diff] [blame] | 113 | unsigned long next_bfn; |
Konrad Rzeszutek Wilk | b097186 | 2010-05-11 10:05:49 -0400 | [diff] [blame] | 114 | int i; |
| 115 | int nr_pages; |
| 116 | |
Julien Grall | 9435cce | 2015-09-09 15:18:45 +0100 | [diff] [blame] | 117 | next_bfn = pfn_to_bfn(xen_pfn); |
| 118 | nr_pages = (offset + length + XEN_PAGE_SIZE-1) >> XEN_PAGE_SHIFT; |
Konrad Rzeszutek Wilk | b097186 | 2010-05-11 10:05:49 -0400 | [diff] [blame] | 119 | |
| 120 | for (i = 1; i < nr_pages; i++) { |
Julien Grall | 9435cce | 2015-09-09 15:18:45 +0100 | [diff] [blame] | 121 | if (pfn_to_bfn(++xen_pfn) != ++next_bfn) |
Konrad Rzeszutek Wilk | b097186 | 2010-05-11 10:05:49 -0400 | [diff] [blame] | 122 | return 0; |
| 123 | } |
| 124 | return 1; |
| 125 | } |
| 126 | |
Stefano Stabellini | 6b42a7e | 2013-10-25 10:33:27 +0000 | [diff] [blame] | 127 | static inline int range_straddles_page_boundary(phys_addr_t p, size_t size) |
Konrad Rzeszutek Wilk | b097186 | 2010-05-11 10:05:49 -0400 | [diff] [blame] | 128 | { |
Julien Grall | 9435cce | 2015-09-09 15:18:45 +0100 | [diff] [blame] | 129 | unsigned long xen_pfn = XEN_PFN_DOWN(p); |
| 130 | unsigned int offset = p & ~XEN_PAGE_MASK; |
Konrad Rzeszutek Wilk | b097186 | 2010-05-11 10:05:49 -0400 | [diff] [blame] | 131 | |
Julien Grall | 9435cce | 2015-09-09 15:18:45 +0100 | [diff] [blame] | 132 | if (offset + size <= XEN_PAGE_SIZE) |
Konrad Rzeszutek Wilk | b097186 | 2010-05-11 10:05:49 -0400 | [diff] [blame] | 133 | return 0; |
Julien Grall | 9435cce | 2015-09-09 15:18:45 +0100 | [diff] [blame] | 134 | if (check_pages_physically_contiguous(xen_pfn, offset, size)) |
Konrad Rzeszutek Wilk | b097186 | 2010-05-11 10:05:49 -0400 | [diff] [blame] | 135 | return 0; |
| 136 | return 1; |
| 137 | } |
| 138 | |
| 139 | static int is_xen_swiotlb_buffer(dma_addr_t dma_addr) |
| 140 | { |
Julien Grall | 9435cce | 2015-09-09 15:18:45 +0100 | [diff] [blame] | 141 | unsigned long bfn = XEN_PFN_DOWN(dma_addr); |
| 142 | unsigned long xen_pfn = bfn_to_local_pfn(bfn); |
| 143 | phys_addr_t paddr = XEN_PFN_PHYS(xen_pfn); |
Konrad Rzeszutek Wilk | b097186 | 2010-05-11 10:05:49 -0400 | [diff] [blame] | 144 | |
| 145 | /* If the address is outside our domain, it CAN |
| 146 | * have the same virtual address as another address |
| 147 | * in our domain. Therefore _only_ check address within our domain. |
| 148 | */ |
Julien Grall | 9435cce | 2015-09-09 15:18:45 +0100 | [diff] [blame] | 149 | if (pfn_valid(PFN_DOWN(paddr))) { |
Konrad Rzeszutek Wilk | b097186 | 2010-05-11 10:05:49 -0400 | [diff] [blame] | 150 | return paddr >= virt_to_phys(xen_io_tlb_start) && |
| 151 | paddr < virt_to_phys(xen_io_tlb_end); |
| 152 | } |
| 153 | return 0; |
| 154 | } |
| 155 | |
| 156 | static int max_dma_bits = 32; |
| 157 | |
| 158 | static int |
| 159 | xen_swiotlb_fixup(void *buf, size_t size, unsigned long nslabs) |
| 160 | { |
| 161 | int i, rc; |
| 162 | int dma_bits; |
Stefano Stabellini | 6990890 | 2013-10-09 16:56:32 +0000 | [diff] [blame] | 163 | dma_addr_t dma_handle; |
Stefano Stabellini | 1b65c4e | 2013-10-10 13:41:10 +0000 | [diff] [blame] | 164 | phys_addr_t p = virt_to_phys(buf); |
Konrad Rzeszutek Wilk | b097186 | 2010-05-11 10:05:49 -0400 | [diff] [blame] | 165 | |
| 166 | dma_bits = get_order(IO_TLB_SEGSIZE << IO_TLB_SHIFT) + PAGE_SHIFT; |
| 167 | |
| 168 | i = 0; |
| 169 | do { |
| 170 | int slabs = min(nslabs - i, (unsigned long)IO_TLB_SEGSIZE); |
| 171 | |
| 172 | do { |
| 173 | rc = xen_create_contiguous_region( |
Stefano Stabellini | 1b65c4e | 2013-10-10 13:41:10 +0000 | [diff] [blame] | 174 | p + (i << IO_TLB_SHIFT), |
Konrad Rzeszutek Wilk | b097186 | 2010-05-11 10:05:49 -0400 | [diff] [blame] | 175 | get_order(slabs << IO_TLB_SHIFT), |
Stefano Stabellini | 6990890 | 2013-10-09 16:56:32 +0000 | [diff] [blame] | 176 | dma_bits, &dma_handle); |
Konrad Rzeszutek Wilk | b097186 | 2010-05-11 10:05:49 -0400 | [diff] [blame] | 177 | } while (rc && dma_bits++ < max_dma_bits); |
| 178 | if (rc) |
| 179 | return rc; |
| 180 | |
| 181 | i += slabs; |
| 182 | } while (i < nslabs); |
| 183 | return 0; |
| 184 | } |
Konrad Rzeszutek Wilk | 1cef36a | 2012-08-23 13:55:26 -0400 | [diff] [blame] | 185 | static unsigned long xen_set_nslabs(unsigned long nr_tbl) |
Konrad Rzeszutek Wilk | b097186 | 2010-05-11 10:05:49 -0400 | [diff] [blame] | 186 | { |
Konrad Rzeszutek Wilk | 1cef36a | 2012-08-23 13:55:26 -0400 | [diff] [blame] | 187 | if (!nr_tbl) { |
FUJITA Tomonori | 5f98ecd | 2011-06-05 11:47:29 +0900 | [diff] [blame] | 188 | xen_io_tlb_nslabs = (64 * 1024 * 1024 >> IO_TLB_SHIFT); |
| 189 | xen_io_tlb_nslabs = ALIGN(xen_io_tlb_nslabs, IO_TLB_SEGSIZE); |
Konrad Rzeszutek Wilk | 1cef36a | 2012-08-23 13:55:26 -0400 | [diff] [blame] | 190 | } else |
| 191 | xen_io_tlb_nslabs = nr_tbl; |
Konrad Rzeszutek Wilk | b097186 | 2010-05-11 10:05:49 -0400 | [diff] [blame] | 192 | |
Konrad Rzeszutek Wilk | 1cef36a | 2012-08-23 13:55:26 -0400 | [diff] [blame] | 193 | return xen_io_tlb_nslabs << IO_TLB_SHIFT; |
| 194 | } |
Konrad Rzeszutek Wilk | 5bab786 | 2012-08-23 14:03:55 -0400 | [diff] [blame] | 195 | |
| 196 | enum xen_swiotlb_err { |
| 197 | XEN_SWIOTLB_UNKNOWN = 0, |
| 198 | XEN_SWIOTLB_ENOMEM, |
| 199 | XEN_SWIOTLB_EFIXUP |
| 200 | }; |
| 201 | |
| 202 | static const char *xen_swiotlb_error(enum xen_swiotlb_err err) |
| 203 | { |
| 204 | switch (err) { |
| 205 | case XEN_SWIOTLB_ENOMEM: |
| 206 | return "Cannot allocate Xen-SWIOTLB buffer\n"; |
| 207 | case XEN_SWIOTLB_EFIXUP: |
| 208 | return "Failed to get contiguous memory for DMA from Xen!\n"\ |
| 209 | "You either: don't have the permissions, do not have"\ |
| 210 | " enough free memory under 4GB, or the hypervisor memory"\ |
| 211 | " is too fragmented!"; |
| 212 | default: |
| 213 | break; |
| 214 | } |
| 215 | return ""; |
| 216 | } |
Konrad Rzeszutek Wilk | b827760 | 2012-08-23 14:36:15 -0400 | [diff] [blame] | 217 | int __ref xen_swiotlb_init(int verbose, bool early) |
Konrad Rzeszutek Wilk | b097186 | 2010-05-11 10:05:49 -0400 | [diff] [blame] | 218 | { |
Konrad Rzeszutek Wilk | b827760 | 2012-08-23 14:36:15 -0400 | [diff] [blame] | 219 | unsigned long bytes, order; |
Konrad Rzeszutek Wilk | b097186 | 2010-05-11 10:05:49 -0400 | [diff] [blame] | 220 | int rc = -ENOMEM; |
Konrad Rzeszutek Wilk | 5bab786 | 2012-08-23 14:03:55 -0400 | [diff] [blame] | 221 | enum xen_swiotlb_err m_ret = XEN_SWIOTLB_UNKNOWN; |
Konrad Rzeszutek Wilk | b097186 | 2010-05-11 10:05:49 -0400 | [diff] [blame] | 222 | unsigned int repeat = 3; |
| 223 | |
Konrad Rzeszutek Wilk | 1cef36a | 2012-08-23 13:55:26 -0400 | [diff] [blame] | 224 | xen_io_tlb_nslabs = swiotlb_nr_tbl(); |
Konrad Rzeszutek Wilk | b097186 | 2010-05-11 10:05:49 -0400 | [diff] [blame] | 225 | retry: |
Konrad Rzeszutek Wilk | 1cef36a | 2012-08-23 13:55:26 -0400 | [diff] [blame] | 226 | bytes = xen_set_nslabs(xen_io_tlb_nslabs); |
Konrad Rzeszutek Wilk | b827760 | 2012-08-23 14:36:15 -0400 | [diff] [blame] | 227 | order = get_order(xen_io_tlb_nslabs << IO_TLB_SHIFT); |
Konrad Rzeszutek Wilk | b097186 | 2010-05-11 10:05:49 -0400 | [diff] [blame] | 228 | /* |
| 229 | * Get IO TLB memory from any location. |
| 230 | */ |
Konrad Rzeszutek Wilk | b827760 | 2012-08-23 14:36:15 -0400 | [diff] [blame] | 231 | if (early) |
| 232 | xen_io_tlb_start = alloc_bootmem_pages(PAGE_ALIGN(bytes)); |
| 233 | else { |
| 234 | #define SLABS_PER_PAGE (1 << (PAGE_SHIFT - IO_TLB_SHIFT)) |
| 235 | #define IO_TLB_MIN_SLABS ((1<<20) >> IO_TLB_SHIFT) |
| 236 | while ((SLABS_PER_PAGE << order) > IO_TLB_MIN_SLABS) { |
Stefano Stabellini | 8746515 | 2015-04-24 10:16:40 +0100 | [diff] [blame] | 237 | xen_io_tlb_start = (void *)xen_get_swiotlb_free_pages(order); |
Konrad Rzeszutek Wilk | b827760 | 2012-08-23 14:36:15 -0400 | [diff] [blame] | 238 | if (xen_io_tlb_start) |
| 239 | break; |
| 240 | order--; |
| 241 | } |
| 242 | if (order != get_order(bytes)) { |
Joe Perches | 283c097 | 2013-06-28 03:21:41 -0700 | [diff] [blame] | 243 | pr_warn("Warning: only able to allocate %ld MB for software IO TLB\n", |
| 244 | (PAGE_SIZE << order) >> 20); |
Konrad Rzeszutek Wilk | b827760 | 2012-08-23 14:36:15 -0400 | [diff] [blame] | 245 | xen_io_tlb_nslabs = SLABS_PER_PAGE << order; |
| 246 | bytes = xen_io_tlb_nslabs << IO_TLB_SHIFT; |
| 247 | } |
| 248 | } |
Konrad Rzeszutek Wilk | f4b2f07 | 2011-07-22 12:46:43 -0400 | [diff] [blame] | 249 | if (!xen_io_tlb_start) { |
Konrad Rzeszutek Wilk | 5bab786 | 2012-08-23 14:03:55 -0400 | [diff] [blame] | 250 | m_ret = XEN_SWIOTLB_ENOMEM; |
Konrad Rzeszutek Wilk | f4b2f07 | 2011-07-22 12:46:43 -0400 | [diff] [blame] | 251 | goto error; |
| 252 | } |
Konrad Rzeszutek Wilk | b097186 | 2010-05-11 10:05:49 -0400 | [diff] [blame] | 253 | xen_io_tlb_end = xen_io_tlb_start + bytes; |
| 254 | /* |
| 255 | * And replace that memory with pages under 4GB. |
| 256 | */ |
| 257 | rc = xen_swiotlb_fixup(xen_io_tlb_start, |
| 258 | bytes, |
| 259 | xen_io_tlb_nslabs); |
Konrad Rzeszutek Wilk | f4b2f07 | 2011-07-22 12:46:43 -0400 | [diff] [blame] | 260 | if (rc) { |
Konrad Rzeszutek Wilk | b827760 | 2012-08-23 14:36:15 -0400 | [diff] [blame] | 261 | if (early) |
| 262 | free_bootmem(__pa(xen_io_tlb_start), PAGE_ALIGN(bytes)); |
| 263 | else { |
| 264 | free_pages((unsigned long)xen_io_tlb_start, order); |
| 265 | xen_io_tlb_start = NULL; |
| 266 | } |
Konrad Rzeszutek Wilk | 5bab786 | 2012-08-23 14:03:55 -0400 | [diff] [blame] | 267 | m_ret = XEN_SWIOTLB_EFIXUP; |
Konrad Rzeszutek Wilk | b097186 | 2010-05-11 10:05:49 -0400 | [diff] [blame] | 268 | goto error; |
Konrad Rzeszutek Wilk | f4b2f07 | 2011-07-22 12:46:43 -0400 | [diff] [blame] | 269 | } |
Konrad Rzeszutek Wilk | b097186 | 2010-05-11 10:05:49 -0400 | [diff] [blame] | 270 | start_dma_addr = xen_virt_to_bus(xen_io_tlb_start); |
Konrad Rzeszutek Wilk | c468bde | 2012-09-17 10:20:09 -0400 | [diff] [blame] | 271 | if (early) { |
Yinghai Lu | ac2cbab | 2013-01-24 12:20:16 -0800 | [diff] [blame] | 272 | if (swiotlb_init_with_tbl(xen_io_tlb_start, xen_io_tlb_nslabs, |
| 273 | verbose)) |
| 274 | panic("Cannot allocate SWIOTLB buffer"); |
Konrad Rzeszutek Wilk | c468bde | 2012-09-17 10:20:09 -0400 | [diff] [blame] | 275 | rc = 0; |
| 276 | } else |
Konrad Rzeszutek Wilk | b827760 | 2012-08-23 14:36:15 -0400 | [diff] [blame] | 277 | rc = swiotlb_late_init_with_tbl(xen_io_tlb_start, xen_io_tlb_nslabs); |
| 278 | return rc; |
Konrad Rzeszutek Wilk | b097186 | 2010-05-11 10:05:49 -0400 | [diff] [blame] | 279 | error: |
Konrad Rzeszutek Wilk | f4b2f07 | 2011-07-22 12:46:43 -0400 | [diff] [blame] | 280 | if (repeat--) { |
| 281 | xen_io_tlb_nslabs = max(1024UL, /* Min is 2MB */ |
| 282 | (xen_io_tlb_nslabs >> 1)); |
Joe Perches | 283c097 | 2013-06-28 03:21:41 -0700 | [diff] [blame] | 283 | pr_info("Lowering to %luMB\n", |
| 284 | (xen_io_tlb_nslabs << IO_TLB_SHIFT) >> 20); |
Konrad Rzeszutek Wilk | f4b2f07 | 2011-07-22 12:46:43 -0400 | [diff] [blame] | 285 | goto retry; |
| 286 | } |
Joe Perches | 283c097 | 2013-06-28 03:21:41 -0700 | [diff] [blame] | 287 | pr_err("%s (rc:%d)\n", xen_swiotlb_error(m_ret), rc); |
Konrad Rzeszutek Wilk | b827760 | 2012-08-23 14:36:15 -0400 | [diff] [blame] | 288 | if (early) |
| 289 | panic("%s (rc:%d)", xen_swiotlb_error(m_ret), rc); |
| 290 | else |
| 291 | free_pages((unsigned long)xen_io_tlb_start, order); |
| 292 | return rc; |
Konrad Rzeszutek Wilk | b097186 | 2010-05-11 10:05:49 -0400 | [diff] [blame] | 293 | } |
Konrad Rzeszutek Wilk | b097186 | 2010-05-11 10:05:49 -0400 | [diff] [blame] | 294 | void * |
| 295 | xen_swiotlb_alloc_coherent(struct device *hwdev, size_t size, |
Andrzej Pietrasiewicz | baa676f | 2012-03-27 14:28:18 +0200 | [diff] [blame] | 296 | dma_addr_t *dma_handle, gfp_t flags, |
Krzysztof Kozlowski | 00085f1 | 2016-08-03 13:46:00 -0700 | [diff] [blame] | 297 | unsigned long attrs) |
Konrad Rzeszutek Wilk | b097186 | 2010-05-11 10:05:49 -0400 | [diff] [blame] | 298 | { |
| 299 | void *ret; |
| 300 | int order = get_order(size); |
| 301 | u64 dma_mask = DMA_BIT_MASK(32); |
Konrad Rzeszutek Wilk | 6810df8 | 2011-08-25 16:13:54 -0400 | [diff] [blame] | 302 | phys_addr_t phys; |
| 303 | dma_addr_t dev_addr; |
Konrad Rzeszutek Wilk | b097186 | 2010-05-11 10:05:49 -0400 | [diff] [blame] | 304 | |
| 305 | /* |
| 306 | * Ignore region specifiers - the kernel's ideas of |
| 307 | * pseudo-phys memory layout has nothing to do with the |
| 308 | * machine physical layout. We can't allocate highmem |
| 309 | * because we can't return a pointer to it. |
| 310 | */ |
| 311 | flags &= ~(__GFP_DMA | __GFP_HIGHMEM); |
| 312 | |
Stefano Stabellini | 1b65c4e | 2013-10-10 13:41:10 +0000 | [diff] [blame] | 313 | /* On ARM this function returns an ioremap'ped virtual address for |
| 314 | * which virt_to_phys doesn't return the corresponding physical |
| 315 | * address. In fact on ARM virt_to_phys only works for kernel direct |
| 316 | * mapped RAM memory. Also see comment below. |
| 317 | */ |
| 318 | ret = xen_alloc_coherent_pages(hwdev, size, dma_handle, flags, attrs); |
Konrad Rzeszutek Wilk | b097186 | 2010-05-11 10:05:49 -0400 | [diff] [blame] | 319 | |
Konrad Rzeszutek Wilk | 6810df8 | 2011-08-25 16:13:54 -0400 | [diff] [blame] | 320 | if (!ret) |
| 321 | return ret; |
Konrad Rzeszutek Wilk | b097186 | 2010-05-11 10:05:49 -0400 | [diff] [blame] | 322 | |
Konrad Rzeszutek Wilk | 6810df8 | 2011-08-25 16:13:54 -0400 | [diff] [blame] | 323 | if (hwdev && hwdev->coherent_dma_mask) |
Ronny Hegewald | b5031ed | 2012-08-31 09:57:52 +0000 | [diff] [blame] | 324 | dma_mask = dma_alloc_coherent_mask(hwdev, flags); |
Konrad Rzeszutek Wilk | 6810df8 | 2011-08-25 16:13:54 -0400 | [diff] [blame] | 325 | |
Stefano Stabellini | 1b65c4e | 2013-10-10 13:41:10 +0000 | [diff] [blame] | 326 | /* At this point dma_handle is the physical address, next we are |
| 327 | * going to set it to the machine address. |
| 328 | * Do not use virt_to_phys(ret) because on ARM it doesn't correspond |
| 329 | * to *dma_handle. */ |
| 330 | phys = *dma_handle; |
Konrad Rzeszutek Wilk | 6810df8 | 2011-08-25 16:13:54 -0400 | [diff] [blame] | 331 | dev_addr = xen_phys_to_bus(phys); |
| 332 | if (((dev_addr + size - 1 <= dma_mask)) && |
| 333 | !range_straddles_page_boundary(phys, size)) |
| 334 | *dma_handle = dev_addr; |
| 335 | else { |
Stefano Stabellini | 1b65c4e | 2013-10-10 13:41:10 +0000 | [diff] [blame] | 336 | if (xen_create_contiguous_region(phys, order, |
Stefano Stabellini | 6990890 | 2013-10-09 16:56:32 +0000 | [diff] [blame] | 337 | fls64(dma_mask), dma_handle) != 0) { |
Stefano Stabellini | 1b65c4e | 2013-10-10 13:41:10 +0000 | [diff] [blame] | 338 | xen_free_coherent_pages(hwdev, size, ret, (dma_addr_t)phys, attrs); |
Konrad Rzeszutek Wilk | b097186 | 2010-05-11 10:05:49 -0400 | [diff] [blame] | 339 | return NULL; |
| 340 | } |
Konrad Rzeszutek Wilk | b097186 | 2010-05-11 10:05:49 -0400 | [diff] [blame] | 341 | } |
Konrad Rzeszutek Wilk | 6810df8 | 2011-08-25 16:13:54 -0400 | [diff] [blame] | 342 | memset(ret, 0, size); |
Konrad Rzeszutek Wilk | b097186 | 2010-05-11 10:05:49 -0400 | [diff] [blame] | 343 | return ret; |
| 344 | } |
| 345 | EXPORT_SYMBOL_GPL(xen_swiotlb_alloc_coherent); |
| 346 | |
| 347 | void |
| 348 | xen_swiotlb_free_coherent(struct device *hwdev, size_t size, void *vaddr, |
Krzysztof Kozlowski | 00085f1 | 2016-08-03 13:46:00 -0700 | [diff] [blame] | 349 | dma_addr_t dev_addr, unsigned long attrs) |
Konrad Rzeszutek Wilk | b097186 | 2010-05-11 10:05:49 -0400 | [diff] [blame] | 350 | { |
| 351 | int order = get_order(size); |
Konrad Rzeszutek Wilk | 6810df8 | 2011-08-25 16:13:54 -0400 | [diff] [blame] | 352 | phys_addr_t phys; |
| 353 | u64 dma_mask = DMA_BIT_MASK(32); |
Konrad Rzeszutek Wilk | b097186 | 2010-05-11 10:05:49 -0400 | [diff] [blame] | 354 | |
Konrad Rzeszutek Wilk | 6810df8 | 2011-08-25 16:13:54 -0400 | [diff] [blame] | 355 | if (hwdev && hwdev->coherent_dma_mask) |
| 356 | dma_mask = hwdev->coherent_dma_mask; |
| 357 | |
Stefano Stabellini | 1b65c4e | 2013-10-10 13:41:10 +0000 | [diff] [blame] | 358 | /* do not use virt_to_phys because on ARM it doesn't return you the |
| 359 | * physical address */ |
| 360 | phys = xen_bus_to_phys(dev_addr); |
Konrad Rzeszutek Wilk | 6810df8 | 2011-08-25 16:13:54 -0400 | [diff] [blame] | 361 | |
| 362 | if (((dev_addr + size - 1 > dma_mask)) || |
| 363 | range_straddles_page_boundary(phys, size)) |
Stefano Stabellini | 1b65c4e | 2013-10-10 13:41:10 +0000 | [diff] [blame] | 364 | xen_destroy_contiguous_region(phys, order); |
Konrad Rzeszutek Wilk | 6810df8 | 2011-08-25 16:13:54 -0400 | [diff] [blame] | 365 | |
Stefano Stabellini | 1b65c4e | 2013-10-10 13:41:10 +0000 | [diff] [blame] | 366 | xen_free_coherent_pages(hwdev, size, vaddr, (dma_addr_t)phys, attrs); |
Konrad Rzeszutek Wilk | b097186 | 2010-05-11 10:05:49 -0400 | [diff] [blame] | 367 | } |
| 368 | EXPORT_SYMBOL_GPL(xen_swiotlb_free_coherent); |
| 369 | |
| 370 | |
| 371 | /* |
| 372 | * Map a single buffer of the indicated size for DMA in streaming mode. The |
| 373 | * physical address to use is returned. |
| 374 | * |
| 375 | * Once the device is given the dma address, the device owns this memory until |
| 376 | * either xen_swiotlb_unmap_page or xen_swiotlb_dma_sync_single is performed. |
| 377 | */ |
| 378 | dma_addr_t xen_swiotlb_map_page(struct device *dev, struct page *page, |
| 379 | unsigned long offset, size_t size, |
| 380 | enum dma_data_direction dir, |
Krzysztof Kozlowski | 00085f1 | 2016-08-03 13:46:00 -0700 | [diff] [blame] | 381 | unsigned long attrs) |
Konrad Rzeszutek Wilk | b097186 | 2010-05-11 10:05:49 -0400 | [diff] [blame] | 382 | { |
Alexander Duyck | e05ed4d | 2012-10-15 10:19:39 -0700 | [diff] [blame] | 383 | phys_addr_t map, phys = page_to_phys(page) + offset; |
Konrad Rzeszutek Wilk | b097186 | 2010-05-11 10:05:49 -0400 | [diff] [blame] | 384 | dma_addr_t dev_addr = xen_phys_to_bus(phys); |
Konrad Rzeszutek Wilk | b097186 | 2010-05-11 10:05:49 -0400 | [diff] [blame] | 385 | |
| 386 | BUG_ON(dir == DMA_NONE); |
| 387 | /* |
| 388 | * If the address happens to be in the device's DMA window, |
| 389 | * we can safely return the device addr and not worry about bounce |
| 390 | * buffering it. |
| 391 | */ |
| 392 | if (dma_capable(dev, dev_addr, size) && |
Stefano Stabellini | a4dba13 | 2014-11-21 11:07:39 +0000 | [diff] [blame] | 393 | !range_straddles_page_boundary(phys, size) && |
Julien Grall | 291be10 | 2015-09-09 15:17:33 +0100 | [diff] [blame] | 394 | !xen_arch_need_swiotlb(dev, phys, dev_addr) && |
Geert Uytterhoeven | 1fd1e6c | 2016-12-16 14:28:41 +0100 | [diff] [blame] | 395 | (swiotlb_force != SWIOTLB_FORCE)) { |
Stefano Stabellini | 6cf0546 | 2013-10-25 10:33:25 +0000 | [diff] [blame] | 396 | /* we are not interested in the dma_addr returned by |
| 397 | * xen_dma_map_page, only in the potential cache flushes executed |
| 398 | * by the function. */ |
Stefano Stabellini | a0f2dee | 2014-11-21 11:04:39 +0000 | [diff] [blame] | 399 | xen_dma_map_page(dev, page, dev_addr, offset, size, dir, attrs); |
Konrad Rzeszutek Wilk | b097186 | 2010-05-11 10:05:49 -0400 | [diff] [blame] | 400 | return dev_addr; |
Stefano Stabellini | 6cf0546 | 2013-10-25 10:33:25 +0000 | [diff] [blame] | 401 | } |
Konrad Rzeszutek Wilk | b097186 | 2010-05-11 10:05:49 -0400 | [diff] [blame] | 402 | |
| 403 | /* |
| 404 | * Oh well, have to allocate and map a bounce buffer. |
| 405 | */ |
Zoltan Kiss | 2b2b614 | 2013-09-04 21:11:05 +0100 | [diff] [blame] | 406 | trace_swiotlb_bounced(dev, dev_addr, size, swiotlb_force); |
| 407 | |
Konrad Rzeszutek Wilk | b097186 | 2010-05-11 10:05:49 -0400 | [diff] [blame] | 408 | map = swiotlb_tbl_map_single(dev, start_dma_addr, phys, size, dir); |
Alexander Duyck | e05ed4d | 2012-10-15 10:19:39 -0700 | [diff] [blame] | 409 | if (map == SWIOTLB_MAP_ERROR) |
Konrad Rzeszutek Wilk | b097186 | 2010-05-11 10:05:49 -0400 | [diff] [blame] | 410 | return DMA_ERROR_CODE; |
| 411 | |
Stefano Stabellini | 6cf0546 | 2013-10-25 10:33:25 +0000 | [diff] [blame] | 412 | xen_dma_map_page(dev, pfn_to_page(map >> PAGE_SHIFT), |
Stefano Stabellini | a0f2dee | 2014-11-21 11:04:39 +0000 | [diff] [blame] | 413 | dev_addr, map & ~PAGE_MASK, size, dir, attrs); |
Alexander Duyck | e05ed4d | 2012-10-15 10:19:39 -0700 | [diff] [blame] | 414 | dev_addr = xen_phys_to_bus(map); |
Konrad Rzeszutek Wilk | b097186 | 2010-05-11 10:05:49 -0400 | [diff] [blame] | 415 | |
| 416 | /* |
| 417 | * Ensure that the address returned is DMA'ble |
| 418 | */ |
Konrad Rzeszutek Wilk | ab2a47b | 2011-07-22 12:51:48 -0400 | [diff] [blame] | 419 | if (!dma_capable(dev, dev_addr, size)) { |
Alexander Duyck | 61ca08c | 2012-10-15 10:19:44 -0700 | [diff] [blame] | 420 | swiotlb_tbl_unmap_single(dev, map, size, dir); |
Konrad Rzeszutek Wilk | ab2a47b | 2011-07-22 12:51:48 -0400 | [diff] [blame] | 421 | dev_addr = 0; |
| 422 | } |
Konrad Rzeszutek Wilk | b097186 | 2010-05-11 10:05:49 -0400 | [diff] [blame] | 423 | return dev_addr; |
| 424 | } |
| 425 | EXPORT_SYMBOL_GPL(xen_swiotlb_map_page); |
| 426 | |
| 427 | /* |
| 428 | * Unmap a single streaming mode DMA translation. The dma_addr and size must |
| 429 | * match what was provided for in a previous xen_swiotlb_map_page call. All |
| 430 | * other usages are undefined. |
| 431 | * |
| 432 | * After this call, reads by the cpu to the buffer are guaranteed to see |
| 433 | * whatever the device wrote there. |
| 434 | */ |
| 435 | static void xen_unmap_single(struct device *hwdev, dma_addr_t dev_addr, |
Stefano Stabellini | 6cf0546 | 2013-10-25 10:33:25 +0000 | [diff] [blame] | 436 | size_t size, enum dma_data_direction dir, |
Krzysztof Kozlowski | 00085f1 | 2016-08-03 13:46:00 -0700 | [diff] [blame] | 437 | unsigned long attrs) |
Konrad Rzeszutek Wilk | b097186 | 2010-05-11 10:05:49 -0400 | [diff] [blame] | 438 | { |
| 439 | phys_addr_t paddr = xen_bus_to_phys(dev_addr); |
| 440 | |
| 441 | BUG_ON(dir == DMA_NONE); |
| 442 | |
Stefano Stabellini | d6883e6 | 2014-11-21 11:09:39 +0000 | [diff] [blame] | 443 | xen_dma_unmap_page(hwdev, dev_addr, size, dir, attrs); |
Stefano Stabellini | 6cf0546 | 2013-10-25 10:33:25 +0000 | [diff] [blame] | 444 | |
Konrad Rzeszutek Wilk | b097186 | 2010-05-11 10:05:49 -0400 | [diff] [blame] | 445 | /* NOTE: We use dev_addr here, not paddr! */ |
| 446 | if (is_xen_swiotlb_buffer(dev_addr)) { |
David Vrabel | 4ef8e3f | 2014-12-10 14:48:43 +0000 | [diff] [blame] | 447 | swiotlb_tbl_unmap_single(hwdev, paddr, size, dir); |
Konrad Rzeszutek Wilk | b097186 | 2010-05-11 10:05:49 -0400 | [diff] [blame] | 448 | return; |
| 449 | } |
| 450 | |
| 451 | if (dir != DMA_FROM_DEVICE) |
| 452 | return; |
| 453 | |
| 454 | /* |
| 455 | * phys_to_virt doesn't work with hihgmem page but we could |
| 456 | * call dma_mark_clean() with hihgmem page here. However, we |
| 457 | * are fine since dma_mark_clean() is null on POWERPC. We can |
| 458 | * make dma_mark_clean() take a physical address if necessary. |
| 459 | */ |
| 460 | dma_mark_clean(phys_to_virt(paddr), size); |
| 461 | } |
| 462 | |
| 463 | void xen_swiotlb_unmap_page(struct device *hwdev, dma_addr_t dev_addr, |
| 464 | size_t size, enum dma_data_direction dir, |
Krzysztof Kozlowski | 00085f1 | 2016-08-03 13:46:00 -0700 | [diff] [blame] | 465 | unsigned long attrs) |
Konrad Rzeszutek Wilk | b097186 | 2010-05-11 10:05:49 -0400 | [diff] [blame] | 466 | { |
Stefano Stabellini | 6cf0546 | 2013-10-25 10:33:25 +0000 | [diff] [blame] | 467 | xen_unmap_single(hwdev, dev_addr, size, dir, attrs); |
Konrad Rzeszutek Wilk | b097186 | 2010-05-11 10:05:49 -0400 | [diff] [blame] | 468 | } |
| 469 | EXPORT_SYMBOL_GPL(xen_swiotlb_unmap_page); |
| 470 | |
| 471 | /* |
| 472 | * Make physical memory consistent for a single streaming mode DMA translation |
| 473 | * after a transfer. |
| 474 | * |
| 475 | * If you perform a xen_swiotlb_map_page() but wish to interrogate the buffer |
| 476 | * using the cpu, yet do not wish to teardown the dma mapping, you must |
| 477 | * call this function before doing so. At the next point you give the dma |
| 478 | * address back to the card, you must first perform a |
| 479 | * xen_swiotlb_dma_sync_for_device, and then the device again owns the buffer |
| 480 | */ |
| 481 | static void |
| 482 | xen_swiotlb_sync_single(struct device *hwdev, dma_addr_t dev_addr, |
| 483 | size_t size, enum dma_data_direction dir, |
| 484 | enum dma_sync_target target) |
| 485 | { |
| 486 | phys_addr_t paddr = xen_bus_to_phys(dev_addr); |
| 487 | |
| 488 | BUG_ON(dir == DMA_NONE); |
| 489 | |
Stefano Stabellini | 6cf0546 | 2013-10-25 10:33:25 +0000 | [diff] [blame] | 490 | if (target == SYNC_FOR_CPU) |
Stefano Stabellini | d6883e6 | 2014-11-21 11:09:39 +0000 | [diff] [blame] | 491 | xen_dma_sync_single_for_cpu(hwdev, dev_addr, size, dir); |
Stefano Stabellini | 6cf0546 | 2013-10-25 10:33:25 +0000 | [diff] [blame] | 492 | |
Konrad Rzeszutek Wilk | b097186 | 2010-05-11 10:05:49 -0400 | [diff] [blame] | 493 | /* NOTE: We use dev_addr here, not paddr! */ |
Stefano Stabellini | 6cf0546 | 2013-10-25 10:33:25 +0000 | [diff] [blame] | 494 | if (is_xen_swiotlb_buffer(dev_addr)) |
Alexander Duyck | fbfda89 | 2012-10-15 10:19:49 -0700 | [diff] [blame] | 495 | swiotlb_tbl_sync_single(hwdev, paddr, size, dir, target); |
Stefano Stabellini | 6cf0546 | 2013-10-25 10:33:25 +0000 | [diff] [blame] | 496 | |
| 497 | if (target == SYNC_FOR_DEVICE) |
Stefano Stabellini | 9490c6c | 2014-11-21 16:55:12 +0000 | [diff] [blame] | 498 | xen_dma_sync_single_for_device(hwdev, dev_addr, size, dir); |
Konrad Rzeszutek Wilk | b097186 | 2010-05-11 10:05:49 -0400 | [diff] [blame] | 499 | |
| 500 | if (dir != DMA_FROM_DEVICE) |
| 501 | return; |
| 502 | |
| 503 | dma_mark_clean(phys_to_virt(paddr), size); |
| 504 | } |
| 505 | |
| 506 | void |
| 507 | xen_swiotlb_sync_single_for_cpu(struct device *hwdev, dma_addr_t dev_addr, |
| 508 | size_t size, enum dma_data_direction dir) |
| 509 | { |
| 510 | xen_swiotlb_sync_single(hwdev, dev_addr, size, dir, SYNC_FOR_CPU); |
| 511 | } |
| 512 | EXPORT_SYMBOL_GPL(xen_swiotlb_sync_single_for_cpu); |
| 513 | |
| 514 | void |
| 515 | xen_swiotlb_sync_single_for_device(struct device *hwdev, dma_addr_t dev_addr, |
| 516 | size_t size, enum dma_data_direction dir) |
| 517 | { |
| 518 | xen_swiotlb_sync_single(hwdev, dev_addr, size, dir, SYNC_FOR_DEVICE); |
| 519 | } |
| 520 | EXPORT_SYMBOL_GPL(xen_swiotlb_sync_single_for_device); |
| 521 | |
| 522 | /* |
| 523 | * Map a set of buffers described by scatterlist in streaming mode for DMA. |
| 524 | * This is the scatter-gather version of the above xen_swiotlb_map_page |
| 525 | * interface. Here the scatter gather list elements are each tagged with the |
| 526 | * appropriate dma address and length. They are obtained via |
| 527 | * sg_dma_{address,length}(SG). |
| 528 | * |
| 529 | * NOTE: An implementation may be able to use a smaller number of |
| 530 | * DMA address/length pairs than there are SG table elements. |
| 531 | * (for example via virtual mapping capabilities) |
| 532 | * The routine returns the number of addr/length pairs actually |
| 533 | * used, at most nents. |
| 534 | * |
| 535 | * Device ownership issues as mentioned above for xen_swiotlb_map_page are the |
| 536 | * same here. |
| 537 | */ |
| 538 | int |
| 539 | xen_swiotlb_map_sg_attrs(struct device *hwdev, struct scatterlist *sgl, |
| 540 | int nelems, enum dma_data_direction dir, |
Krzysztof Kozlowski | 00085f1 | 2016-08-03 13:46:00 -0700 | [diff] [blame] | 541 | unsigned long attrs) |
Konrad Rzeszutek Wilk | b097186 | 2010-05-11 10:05:49 -0400 | [diff] [blame] | 542 | { |
| 543 | struct scatterlist *sg; |
| 544 | int i; |
| 545 | |
| 546 | BUG_ON(dir == DMA_NONE); |
| 547 | |
| 548 | for_each_sg(sgl, sg, nelems, i) { |
| 549 | phys_addr_t paddr = sg_phys(sg); |
| 550 | dma_addr_t dev_addr = xen_phys_to_bus(paddr); |
| 551 | |
Geert Uytterhoeven | 1fd1e6c | 2016-12-16 14:28:41 +0100 | [diff] [blame] | 552 | if (swiotlb_force == SWIOTLB_FORCE || |
Julien Grall | 291be10 | 2015-09-09 15:17:33 +0100 | [diff] [blame] | 553 | xen_arch_need_swiotlb(hwdev, paddr, dev_addr) || |
Konrad Rzeszutek Wilk | b097186 | 2010-05-11 10:05:49 -0400 | [diff] [blame] | 554 | !dma_capable(hwdev, dev_addr, sg->length) || |
| 555 | range_straddles_page_boundary(paddr, sg->length)) { |
Alexander Duyck | e05ed4d | 2012-10-15 10:19:39 -0700 | [diff] [blame] | 556 | phys_addr_t map = swiotlb_tbl_map_single(hwdev, |
| 557 | start_dma_addr, |
| 558 | sg_phys(sg), |
| 559 | sg->length, |
| 560 | dir); |
| 561 | if (map == SWIOTLB_MAP_ERROR) { |
Stefano Stabellini | 783d028 | 2013-10-25 10:33:26 +0000 | [diff] [blame] | 562 | dev_warn(hwdev, "swiotlb buffer is full\n"); |
Konrad Rzeszutek Wilk | b097186 | 2010-05-11 10:05:49 -0400 | [diff] [blame] | 563 | /* Don't panic here, we expect map_sg users |
| 564 | to do proper error handling. */ |
| 565 | xen_swiotlb_unmap_sg_attrs(hwdev, sgl, i, dir, |
| 566 | attrs); |
Stefano Stabellini | 781575c | 2013-08-05 17:30:48 +0100 | [diff] [blame] | 567 | sg_dma_len(sgl) = 0; |
Stefano Stabellini | 1517760 | 2013-10-29 00:37:37 +0000 | [diff] [blame] | 568 | return 0; |
Konrad Rzeszutek Wilk | b097186 | 2010-05-11 10:05:49 -0400 | [diff] [blame] | 569 | } |
Stefano Stabellini | 71bfae9 | 2013-11-15 14:22:15 +0000 | [diff] [blame] | 570 | xen_dma_map_page(hwdev, pfn_to_page(map >> PAGE_SHIFT), |
Stefano Stabellini | a0f2dee | 2014-11-21 11:04:39 +0000 | [diff] [blame] | 571 | dev_addr, |
Stefano Stabellini | 71bfae9 | 2013-11-15 14:22:15 +0000 | [diff] [blame] | 572 | map & ~PAGE_MASK, |
| 573 | sg->length, |
| 574 | dir, |
| 575 | attrs); |
Alexander Duyck | e05ed4d | 2012-10-15 10:19:39 -0700 | [diff] [blame] | 576 | sg->dma_address = xen_phys_to_bus(map); |
Stefano Stabellini | 6cf0546 | 2013-10-25 10:33:25 +0000 | [diff] [blame] | 577 | } else { |
| 578 | /* we are not interested in the dma_addr returned by |
| 579 | * xen_dma_map_page, only in the potential cache flushes executed |
| 580 | * by the function. */ |
| 581 | xen_dma_map_page(hwdev, pfn_to_page(paddr >> PAGE_SHIFT), |
Stefano Stabellini | a0f2dee | 2014-11-21 11:04:39 +0000 | [diff] [blame] | 582 | dev_addr, |
Stefano Stabellini | 6cf0546 | 2013-10-25 10:33:25 +0000 | [diff] [blame] | 583 | paddr & ~PAGE_MASK, |
| 584 | sg->length, |
| 585 | dir, |
| 586 | attrs); |
Konrad Rzeszutek Wilk | b097186 | 2010-05-11 10:05:49 -0400 | [diff] [blame] | 587 | sg->dma_address = dev_addr; |
Stefano Stabellini | 6cf0546 | 2013-10-25 10:33:25 +0000 | [diff] [blame] | 588 | } |
Stefano Stabellini | 781575c | 2013-08-05 17:30:48 +0100 | [diff] [blame] | 589 | sg_dma_len(sg) = sg->length; |
Konrad Rzeszutek Wilk | b097186 | 2010-05-11 10:05:49 -0400 | [diff] [blame] | 590 | } |
| 591 | return nelems; |
| 592 | } |
| 593 | EXPORT_SYMBOL_GPL(xen_swiotlb_map_sg_attrs); |
| 594 | |
Konrad Rzeszutek Wilk | b097186 | 2010-05-11 10:05:49 -0400 | [diff] [blame] | 595 | /* |
| 596 | * Unmap a set of streaming mode DMA translations. Again, cpu read rules |
| 597 | * concerning calls here are the same as for swiotlb_unmap_page() above. |
| 598 | */ |
| 599 | void |
| 600 | xen_swiotlb_unmap_sg_attrs(struct device *hwdev, struct scatterlist *sgl, |
| 601 | int nelems, enum dma_data_direction dir, |
Krzysztof Kozlowski | 00085f1 | 2016-08-03 13:46:00 -0700 | [diff] [blame] | 602 | unsigned long attrs) |
Konrad Rzeszutek Wilk | b097186 | 2010-05-11 10:05:49 -0400 | [diff] [blame] | 603 | { |
| 604 | struct scatterlist *sg; |
| 605 | int i; |
| 606 | |
| 607 | BUG_ON(dir == DMA_NONE); |
| 608 | |
| 609 | for_each_sg(sgl, sg, nelems, i) |
Stefano Stabellini | 6cf0546 | 2013-10-25 10:33:25 +0000 | [diff] [blame] | 610 | xen_unmap_single(hwdev, sg->dma_address, sg_dma_len(sg), dir, attrs); |
Konrad Rzeszutek Wilk | b097186 | 2010-05-11 10:05:49 -0400 | [diff] [blame] | 611 | |
| 612 | } |
| 613 | EXPORT_SYMBOL_GPL(xen_swiotlb_unmap_sg_attrs); |
| 614 | |
Konrad Rzeszutek Wilk | b097186 | 2010-05-11 10:05:49 -0400 | [diff] [blame] | 615 | /* |
| 616 | * Make physical memory consistent for a set of streaming mode DMA translations |
| 617 | * after a transfer. |
| 618 | * |
| 619 | * The same as swiotlb_sync_single_* but for a scatter-gather list, same rules |
| 620 | * and usage. |
| 621 | */ |
| 622 | static void |
| 623 | xen_swiotlb_sync_sg(struct device *hwdev, struct scatterlist *sgl, |
| 624 | int nelems, enum dma_data_direction dir, |
| 625 | enum dma_sync_target target) |
| 626 | { |
| 627 | struct scatterlist *sg; |
| 628 | int i; |
| 629 | |
| 630 | for_each_sg(sgl, sg, nelems, i) |
| 631 | xen_swiotlb_sync_single(hwdev, sg->dma_address, |
Stefano Stabellini | 781575c | 2013-08-05 17:30:48 +0100 | [diff] [blame] | 632 | sg_dma_len(sg), dir, target); |
Konrad Rzeszutek Wilk | b097186 | 2010-05-11 10:05:49 -0400 | [diff] [blame] | 633 | } |
| 634 | |
| 635 | void |
| 636 | xen_swiotlb_sync_sg_for_cpu(struct device *hwdev, struct scatterlist *sg, |
| 637 | int nelems, enum dma_data_direction dir) |
| 638 | { |
| 639 | xen_swiotlb_sync_sg(hwdev, sg, nelems, dir, SYNC_FOR_CPU); |
| 640 | } |
| 641 | EXPORT_SYMBOL_GPL(xen_swiotlb_sync_sg_for_cpu); |
| 642 | |
| 643 | void |
| 644 | xen_swiotlb_sync_sg_for_device(struct device *hwdev, struct scatterlist *sg, |
| 645 | int nelems, enum dma_data_direction dir) |
| 646 | { |
| 647 | xen_swiotlb_sync_sg(hwdev, sg, nelems, dir, SYNC_FOR_DEVICE); |
| 648 | } |
| 649 | EXPORT_SYMBOL_GPL(xen_swiotlb_sync_sg_for_device); |
| 650 | |
| 651 | int |
| 652 | xen_swiotlb_dma_mapping_error(struct device *hwdev, dma_addr_t dma_addr) |
| 653 | { |
| 654 | return !dma_addr; |
| 655 | } |
| 656 | EXPORT_SYMBOL_GPL(xen_swiotlb_dma_mapping_error); |
| 657 | |
| 658 | /* |
| 659 | * Return whether the given device DMA address mask can be supported |
| 660 | * properly. For example, if your device can only drive the low 24-bits |
| 661 | * during bus mastering, then you would pass 0x00ffffff as the mask to |
| 662 | * this function. |
| 663 | */ |
| 664 | int |
| 665 | xen_swiotlb_dma_supported(struct device *hwdev, u64 mask) |
| 666 | { |
| 667 | return xen_virt_to_bus(xen_io_tlb_end - 1) <= mask; |
| 668 | } |
| 669 | EXPORT_SYMBOL_GPL(xen_swiotlb_dma_supported); |
Stefano Stabellini | eb1ddc0 | 2013-10-09 16:56:33 +0000 | [diff] [blame] | 670 | |
| 671 | int |
| 672 | xen_swiotlb_set_dma_mask(struct device *dev, u64 dma_mask) |
| 673 | { |
| 674 | if (!dev->dma_mask || !xen_swiotlb_dma_supported(dev, dma_mask)) |
| 675 | return -EIO; |
| 676 | |
| 677 | *dev->dma_mask = dma_mask; |
| 678 | |
| 679 | return 0; |
| 680 | } |
| 681 | EXPORT_SYMBOL_GPL(xen_swiotlb_set_dma_mask); |