blob: a1e2e18e09613caa4158cc06c9a783b5c4549f70 [file] [log] [blame]
Michal Simek3a0d7a42010-02-22 12:16:08 +01001/*
2 * Microblaze support for cache consistent memory.
3 * Copyright (C) 2010 Michal Simek <monstr@monstr.eu>
4 * Copyright (C) 2010 PetaLogix
5 * Copyright (C) 2005 John Williams <jwilliams@itee.uq.edu.au>
6 *
7 * Based on PowerPC version derived from arch/arm/mm/consistent.c
8 * Copyright (C) 2001 Dan Malek (dmalek@jlc.net)
9 * Copyright (C) 2000 Russell King
10 *
11 * This program is free software; you can redistribute it and/or modify
12 * it under the terms of the GNU General Public License version 2 as
13 * published by the Free Software Foundation.
14 */
15
16#include <linux/module.h>
17#include <linux/signal.h>
18#include <linux/sched.h>
19#include <linux/kernel.h>
20#include <linux/errno.h>
21#include <linux/string.h>
22#include <linux/types.h>
23#include <linux/ptrace.h>
24#include <linux/mman.h>
25#include <linux/mm.h>
26#include <linux/swap.h>
27#include <linux/stddef.h>
28#include <linux/vmalloc.h>
29#include <linux/init.h>
30#include <linux/delay.h>
31#include <linux/bootmem.h>
32#include <linux/highmem.h>
33#include <linux/pci.h>
34#include <linux/interrupt.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090035#include <linux/gfp.h>
Michal Simek3a0d7a42010-02-22 12:16:08 +010036
37#include <asm/pgalloc.h>
38#include <linux/io.h>
39#include <linux/hardirq.h>
40#include <asm/mmu_context.h>
41#include <asm/mmu.h>
42#include <linux/uaccess.h>
43#include <asm/pgtable.h>
44#include <asm/cpuinfo.h>
Michal Simekf1525762010-04-10 17:34:06 +020045#include <asm/tlbflush.h>
Michal Simek3a0d7a42010-02-22 12:16:08 +010046
47#ifndef CONFIG_MMU
Michal Simek3a0d7a42010-02-22 12:16:08 +010048/* I have to use dcache values because I can't relate on ram size */
Michal Simekf1525762010-04-10 17:34:06 +020049# define UNCACHED_SHADOW_MASK (cpuinfo.dcache_high - cpuinfo.dcache_base + 1)
50#endif
Michal Simek3a0d7a42010-02-22 12:16:08 +010051
52/*
53 * Consistent memory allocators. Used for DMA devices that want to
54 * share uncached memory with the processor core.
55 * My crufty no-MMU approach is simple. In the HW platform we can optionally
56 * mirror the DDR up above the processor cacheable region. So, memory accessed
57 * in this mirror region will not be cached. It's alloced from the same
58 * pool as normal memory, but the handle we return is shifted up into the
59 * uncached region. This will no doubt cause big problems if memory allocated
60 * here is not also freed properly. -- JW
61 */
Michal Simekcd44da12011-02-07 11:42:37 +010062void *consistent_alloc(gfp_t gfp, size_t size, dma_addr_t *dma_handle)
Michal Simek3a0d7a42010-02-22 12:16:08 +010063{
Michal Simekf1525762010-04-10 17:34:06 +020064 unsigned long order, vaddr;
65 void *ret;
66 unsigned int i, err = 0;
67 struct page *page, *end;
Michal Simek3a0d7a42010-02-22 12:16:08 +010068
Michal Simekf1525762010-04-10 17:34:06 +020069#ifdef CONFIG_MMU
Michal Simek3a0d7a42010-02-22 12:16:08 +010070 phys_addr_t pa;
71 struct vm_struct *area;
Michal Simekf1525762010-04-10 17:34:06 +020072 unsigned long va;
73#endif
Michal Simek3a0d7a42010-02-22 12:16:08 +010074
75 if (in_interrupt())
76 BUG();
77
78 /* Only allocate page size areas. */
79 size = PAGE_ALIGN(size);
80 order = get_order(size);
81
Michal Simekf1525762010-04-10 17:34:06 +020082 vaddr = __get_free_pages(gfp, order);
83 if (!vaddr)
Michal Simek3a0d7a42010-02-22 12:16:08 +010084 return NULL;
Michal Simek3a0d7a42010-02-22 12:16:08 +010085
86 /*
87 * we need to ensure that there are no cachelines in use,
88 * or worse dirty in this area.
89 */
Michal Simekf1525762010-04-10 17:34:06 +020090 flush_dcache_range(virt_to_phys((void *)vaddr),
91 virt_to_phys((void *)vaddr) + size);
Michal Simek3a0d7a42010-02-22 12:16:08 +010092
Michal Simekf1525762010-04-10 17:34:06 +020093#ifndef CONFIG_MMU
94 ret = (void *)vaddr;
95 /*
96 * Here's the magic! Note if the uncached shadow is not implemented,
97 * it's up to the calling code to also test that condition and make
98 * other arranegments, such as manually flushing the cache and so on.
99 */
100# ifdef CONFIG_XILINX_UNCACHED_SHADOW
101 ret = (void *)((unsigned) ret | UNCACHED_SHADOW_MASK);
102# endif
103 if ((unsigned int)ret > cpuinfo.dcache_base &&
104 (unsigned int)ret < cpuinfo.dcache_high)
105 printk(KERN_WARNING
106 "ERROR: Your cache coherent area is CACHED!!!\n");
107
108 /* dma_handle is same as physical (shadowed) address */
109 *dma_handle = (dma_addr_t)ret;
110#else
Michal Simek3a0d7a42010-02-22 12:16:08 +0100111 /* Allocate some common virtual space to map the new pages. */
112 area = get_vm_area(size, VM_ALLOC);
Michal Simekf1525762010-04-10 17:34:06 +0200113 if (!area) {
114 free_pages(vaddr, order);
Michal Simek3a0d7a42010-02-22 12:16:08 +0100115 return NULL;
116 }
117 va = (unsigned long) area->addr;
118 ret = (void *)va;
119
120 /* This gives us the real physical address of the first page. */
Michal Simekf1525762010-04-10 17:34:06 +0200121 *dma_handle = pa = virt_to_bus((void *)vaddr);
122#endif
Michal Simek3a0d7a42010-02-22 12:16:08 +0100123
124 /*
Michal Simekf1525762010-04-10 17:34:06 +0200125 * free wasted pages. We skip the first page since we know
126 * that it will have count = 1 and won't require freeing.
127 * We also mark the pages in use as reserved so that
128 * remap_page_range works.
Michal Simek3a0d7a42010-02-22 12:16:08 +0100129 */
Michal Simekf1525762010-04-10 17:34:06 +0200130 page = virt_to_page(vaddr);
131 end = page + (1 << order);
132
133 split_page(page, order);
134
135 for (i = 0; i < size && err == 0; i += PAGE_SIZE) {
136#ifdef CONFIG_MMU
137 /* MS: This is the whole magic - use cache inhibit pages */
138 err = map_page(va + i, pa + i, _PAGE_KERNEL | _PAGE_NO_CACHE);
139#endif
140
141 SetPageReserved(page);
142 page++;
Michal Simek3a0d7a42010-02-22 12:16:08 +0100143 }
144
Michal Simekf1525762010-04-10 17:34:06 +0200145 /* Free the otherwise unused pages. */
146 while (page < end) {
147 __free_page(page);
148 page++;
149 }
Michal Simek3a0d7a42010-02-22 12:16:08 +0100150
151 if (err) {
Michal Simekf1525762010-04-10 17:34:06 +0200152 free_pages(vaddr, order);
Michal Simek3a0d7a42010-02-22 12:16:08 +0100153 return NULL;
154 }
155
156 return ret;
157}
Michal Simek3a0d7a42010-02-22 12:16:08 +0100158EXPORT_SYMBOL(consistent_alloc);
159
160/*
161 * free page(s) as defined by the above mapping.
162 */
Michal Simekf1525762010-04-10 17:34:06 +0200163void consistent_free(size_t size, void *vaddr)
Michal Simek3a0d7a42010-02-22 12:16:08 +0100164{
Michal Simekf1525762010-04-10 17:34:06 +0200165 struct page *page;
166
Michal Simek3a0d7a42010-02-22 12:16:08 +0100167 if (in_interrupt())
168 BUG();
169
Michal Simekf1525762010-04-10 17:34:06 +0200170 size = PAGE_ALIGN(size);
171
172#ifndef CONFIG_MMU
Michal Simek3a0d7a42010-02-22 12:16:08 +0100173 /* Clear SHADOW_MASK bit in address, and free as per usual */
Michal Simekf1525762010-04-10 17:34:06 +0200174# ifdef CONFIG_XILINX_UNCACHED_SHADOW
Michal Simek3a0d7a42010-02-22 12:16:08 +0100175 vaddr = (void *)((unsigned)vaddr & ~UNCACHED_SHADOW_MASK);
Michal Simekf1525762010-04-10 17:34:06 +0200176# endif
177 page = virt_to_page(vaddr);
178
179 do {
180 ClearPageReserved(page);
181 __free_page(page);
182 page++;
183 } while (size -= PAGE_SIZE);
184#else
185 do {
186 pte_t *ptep;
187 unsigned long pfn;
188
189 ptep = pte_offset_kernel(pmd_offset(pgd_offset_k(
190 (unsigned int)vaddr),
191 (unsigned int)vaddr),
192 (unsigned int)vaddr);
193 if (!pte_none(*ptep) && pte_present(*ptep)) {
194 pfn = pte_pfn(*ptep);
195 pte_clear(&init_mm, (unsigned int)vaddr, ptep);
196 if (pfn_valid(pfn)) {
197 page = pfn_to_page(pfn);
198
199 ClearPageReserved(page);
200 __free_page(page);
201 }
202 }
203 vaddr += PAGE_SIZE;
204 } while (size -= PAGE_SIZE);
205
206 /* flush tlb */
207 flush_tlb_all();
Michal Simek3a0d7a42010-02-22 12:16:08 +0100208#endif
Michal Simek3a0d7a42010-02-22 12:16:08 +0100209}
210EXPORT_SYMBOL(consistent_free);
211
212/*
213 * make an area consistent.
214 */
215void consistent_sync(void *vaddr, size_t size, int direction)
216{
217 unsigned long start;
218 unsigned long end;
219
220 start = (unsigned long)vaddr;
221
222 /* Convert start address back down to unshadowed memory region */
223#ifdef CONFIG_XILINX_UNCACHED_SHADOW
224 start &= ~UNCACHED_SHADOW_MASK;
225#endif
226 end = start + size;
227
228 switch (direction) {
229 case PCI_DMA_NONE:
230 BUG();
231 case PCI_DMA_FROMDEVICE: /* invalidate only */
Michal Simek385e1ef2010-04-29 13:02:17 +0200232 invalidate_dcache_range(start, end);
Michal Simek3a0d7a42010-02-22 12:16:08 +0100233 break;
234 case PCI_DMA_TODEVICE: /* writeback only */
235 flush_dcache_range(start, end);
236 break;
237 case PCI_DMA_BIDIRECTIONAL: /* writeback and invalidate */
238 flush_dcache_range(start, end);
239 break;
240 }
241}
242EXPORT_SYMBOL(consistent_sync);
243
244/*
245 * consistent_sync_page makes memory consistent. identical
246 * to consistent_sync, but takes a struct page instead of a
247 * virtual address
248 */
249void consistent_sync_page(struct page *page, unsigned long offset,
250 size_t size, int direction)
251{
252 unsigned long start = (unsigned long)page_address(page) + offset;
253 consistent_sync((void *)start, size, direction);
254}
255EXPORT_SYMBOL(consistent_sync_page);