blob: 24659952c2784de64a53dc2e889ab616bd19b12b [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * linux/arch/arm/mm/cache-v6.S
3 *
4 * Copyright (C) 2001 Deep Blue Solutions Ltd.
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 as
8 * published by the Free Software Foundation.
9 *
10 * This is the "shell" of the ARMv6 processor support.
11 */
12#include <linux/linkage.h>
13#include <linux/init.h>
14#include <asm/assembler.h>
Will Deaconc5102f52012-04-27 13:08:53 +010015#include <asm/errno.h>
Catalin Marinas32cfb1b2009-10-06 17:57:09 +010016#include <asm/unwind.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070017
18#include "proc-macros.S"
19
20#define HARVARD_CACHE
21#define CACHE_LINE_SIZE 32
22#define D_CACHE_LINE_SIZE 32
Gen FUKATSU217874f2005-09-30 16:09:17 +010023#define BTB_FLUSH_SIZE 8
Linus Torvalds1da177e2005-04-16 15:20:36 -070024
Catalin Marinas9cba3cc2009-04-30 17:06:03 +010025/*
Tony Lindgren81d11952010-09-21 17:16:40 +010026 * v6_flush_icache_all()
Catalin Marinas9cba3cc2009-04-30 17:06:03 +010027 *
Tony Lindgren81d11952010-09-21 17:16:40 +010028 * Flush the whole I-cache.
29 *
30 * ARM1136 erratum 411920 - Invalidate Instruction Cache operation can fail.
31 * This erratum is present in 1136, 1156 and 1176. It does not affect the
32 * MPCore.
33 *
34 * Registers:
35 * r0 - set to 0
36 * r1 - corrupted
Catalin Marinas9cba3cc2009-04-30 17:06:03 +010037 */
Tony Lindgren81d11952010-09-21 17:16:40 +010038ENTRY(v6_flush_icache_all)
Catalin Marinas9cba3cc2009-04-30 17:06:03 +010039 mov r0, #0
Tony Lindgren81d11952010-09-21 17:16:40 +010040#ifdef CONFIG_ARM_ERRATA_411920
Catalin Marinas9cba3cc2009-04-30 17:06:03 +010041 mrs r1, cpsr
42 cpsid ifa @ disable interrupts
43 mcr p15, 0, r0, c7, c5, 0 @ invalidate entire I-cache
44 mcr p15, 0, r0, c7, c5, 0 @ invalidate entire I-cache
45 mcr p15, 0, r0, c7, c5, 0 @ invalidate entire I-cache
46 mcr p15, 0, r0, c7, c5, 0 @ invalidate entire I-cache
47 msr cpsr_cx, r1 @ restore interrupts
48 .rept 11 @ ARM Ltd recommends at least
49 nop @ 11 NOPs
50 .endr
Tony Lindgren81d11952010-09-21 17:16:40 +010051#else
52 mcr p15, 0, r0, c7, c5, 0 @ invalidate I-cache
Catalin Marinas9cba3cc2009-04-30 17:06:03 +010053#endif
Russell King6ebbf2c2014-06-30 16:29:12 +010054 ret lr
Tony Lindgren81d11952010-09-21 17:16:40 +010055ENDPROC(v6_flush_icache_all)
Catalin Marinas9cba3cc2009-04-30 17:06:03 +010056
Linus Torvalds1da177e2005-04-16 15:20:36 -070057/*
58 * v6_flush_cache_all()
59 *
60 * Flush the entire cache.
61 *
62 * It is assumed that:
63 */
64ENTRY(v6_flush_kern_cache_all)
65 mov r0, #0
66#ifdef HARVARD_CACHE
67 mcr p15, 0, r0, c7, c14, 0 @ D cache clean+invalidate
Catalin Marinas9cba3cc2009-04-30 17:06:03 +010068#ifndef CONFIG_ARM_ERRATA_411920
Linus Torvalds1da177e2005-04-16 15:20:36 -070069 mcr p15, 0, r0, c7, c5, 0 @ I+BTB cache invalidate
70#else
Tony Lindgren81d11952010-09-21 17:16:40 +010071 b v6_flush_icache_all
Catalin Marinas9cba3cc2009-04-30 17:06:03 +010072#endif
73#else
Linus Torvalds1da177e2005-04-16 15:20:36 -070074 mcr p15, 0, r0, c7, c15, 0 @ Cache clean+invalidate
75#endif
Russell King6ebbf2c2014-06-30 16:29:12 +010076 ret lr
Linus Torvalds1da177e2005-04-16 15:20:36 -070077
78/*
79 * v6_flush_cache_all()
80 *
81 * Flush all TLB entries in a particular address space
82 *
83 * - mm - mm_struct describing address space
84 */
85ENTRY(v6_flush_user_cache_all)
86 /*FALLTHROUGH*/
87
88/*
89 * v6_flush_cache_range(start, end, flags)
90 *
91 * Flush a range of TLB entries in the specified address space.
92 *
93 * - start - start address (may not be aligned)
94 * - end - end address (exclusive, may not be aligned)
95 * - flags - vm_area_struct flags describing address space
96 *
97 * It is assumed that:
98 * - we have a VIPT cache.
99 */
100ENTRY(v6_flush_user_cache_range)
Russell King6ebbf2c2014-06-30 16:29:12 +0100101 ret lr
Linus Torvalds1da177e2005-04-16 15:20:36 -0700102
103/*
104 * v6_coherent_kern_range(start,end)
105 *
106 * Ensure that the I and D caches are coherent within specified
107 * region. This is typically used when code has been written to
108 * a memory region, and will be executed.
109 *
110 * - start - virtual start address of region
111 * - end - virtual end address of region
112 *
113 * It is assumed that:
114 * - the Icache does not read data from the write buffer
115 */
116ENTRY(v6_coherent_kern_range)
117 /* FALLTHROUGH */
118
119/*
120 * v6_coherent_user_range(start,end)
121 *
122 * Ensure that the I and D caches are coherent within specified
123 * region. This is typically used when code has been written to
124 * a memory region, and will be executed.
125 *
126 * - start - virtual start address of region
127 * - end - virtual end address of region
128 *
129 * It is assumed that:
130 * - the Icache does not read data from the write buffer
131 */
132ENTRY(v6_coherent_user_range)
Catalin Marinas32cfb1b2009-10-06 17:57:09 +0100133 UNWIND(.fnstart )
Linus Torvalds1da177e2005-04-16 15:20:36 -0700134#ifdef HARVARD_CACHE
Nicolas Pitre18afea02006-02-01 19:26:01 +0000135 bic r0, r0, #CACHE_LINE_SIZE - 1
Catalin Marinas32cfb1b2009-10-06 17:57:09 +01001361:
137 USER( mcr p15, 0, r0, c7, c10, 1 ) @ clean D line
Nicolas Pitre18afea02006-02-01 19:26:01 +0000138 add r0, r0, #CACHE_LINE_SIZE
Linus Torvalds1da177e2005-04-16 15:20:36 -0700139 cmp r0, r1
140 blo 1b
Nicolas Pitre18afea02006-02-01 19:26:01 +0000141#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -0700142 mov r0, #0
Catalin Marinas141fa402006-03-10 22:26:47 +0000143#ifdef HARVARD_CACHE
Linus Torvalds1da177e2005-04-16 15:20:36 -0700144 mcr p15, 0, r0, c7, c10, 4 @ drain write buffer
Catalin Marinas9cba3cc2009-04-30 17:06:03 +0100145#ifndef CONFIG_ARM_ERRATA_411920
Catalin Marinas141fa402006-03-10 22:26:47 +0000146 mcr p15, 0, r0, c7, c5, 0 @ I+BTB cache invalidate
147#else
Tony Lindgren81d11952010-09-21 17:16:40 +0100148 b v6_flush_icache_all
Catalin Marinas9cba3cc2009-04-30 17:06:03 +0100149#endif
150#else
Catalin Marinas141fa402006-03-10 22:26:47 +0000151 mcr p15, 0, r0, c7, c5, 6 @ invalidate BTB
Linus Torvalds1da177e2005-04-16 15:20:36 -0700152#endif
Russell King6ebbf2c2014-06-30 16:29:12 +0100153 ret lr
Linus Torvalds1da177e2005-04-16 15:20:36 -0700154
155/*
Catalin Marinas32cfb1b2009-10-06 17:57:09 +0100156 * Fault handling for the cache operation above. If the virtual address in r0
Will Deaconc5102f52012-04-27 13:08:53 +0100157 * isn't mapped, fail with -EFAULT.
Catalin Marinas32cfb1b2009-10-06 17:57:09 +0100158 */
1599001:
Will Deaconc5102f52012-04-27 13:08:53 +0100160 mov r0, #-EFAULT
Russell King6ebbf2c2014-06-30 16:29:12 +0100161 ret lr
Catalin Marinas32cfb1b2009-10-06 17:57:09 +0100162 UNWIND(.fnend )
163ENDPROC(v6_coherent_user_range)
164ENDPROC(v6_coherent_kern_range)
165
166/*
Russell King2c9b9c82009-11-26 12:56:21 +0000167 * v6_flush_kern_dcache_area(void *addr, size_t size)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700168 *
169 * Ensure that the data held in the page kaddr is written back
170 * to the page in question.
171 *
Russell King2c9b9c82009-11-26 12:56:21 +0000172 * - addr - kernel address
173 * - size - region size
Linus Torvalds1da177e2005-04-16 15:20:36 -0700174 */
Russell King2c9b9c82009-11-26 12:56:21 +0000175ENTRY(v6_flush_kern_dcache_area)
176 add r1, r0, r1
Will Deacona248b132011-05-26 11:20:19 +0100177 bic r0, r0, #D_CACHE_LINE_SIZE - 1
Linus Torvalds1da177e2005-04-16 15:20:36 -07001781:
179#ifdef HARVARD_CACHE
180 mcr p15, 0, r0, c7, c14, 1 @ clean & invalidate D line
181#else
182 mcr p15, 0, r0, c7, c15, 1 @ clean & invalidate unified line
183#endif
184 add r0, r0, #D_CACHE_LINE_SIZE
185 cmp r0, r1
186 blo 1b
187#ifdef HARVARD_CACHE
188 mov r0, #0
189 mcr p15, 0, r0, c7, c10, 4
190#endif
Russell King6ebbf2c2014-06-30 16:29:12 +0100191 ret lr
Linus Torvalds1da177e2005-04-16 15:20:36 -0700192
193
194/*
195 * v6_dma_inv_range(start,end)
196 *
197 * Invalidate the data cache within the specified region; we will
198 * be performing a DMA operation in this region and we want to
199 * purge old data in the cache.
200 *
201 * - start - virtual start address of region
202 * - end - virtual end address of region
203 */
Russell King702b94b2009-11-26 16:24:19 +0000204v6_dma_inv_range:
Valentine Barshak85b093b2010-12-14 00:03:16 +0100205#ifdef CONFIG_DMA_CACHE_RWFO
206 ldrb r2, [r0] @ read for ownership
207 strb r2, [r0] @ write for ownership
208#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -0700209 tst r0, #D_CACHE_LINE_SIZE - 1
210 bic r0, r0, #D_CACHE_LINE_SIZE - 1
211#ifdef HARVARD_CACHE
212 mcrne p15, 0, r0, c7, c10, 1 @ clean D line
213#else
214 mcrne p15, 0, r0, c7, c11, 1 @ clean unified line
215#endif
216 tst r1, #D_CACHE_LINE_SIZE - 1
Valentine Barshak85b093b2010-12-14 00:03:16 +0100217#ifdef CONFIG_DMA_CACHE_RWFO
218 ldrneb r2, [r1, #-1] @ read for ownership
219 strneb r2, [r1, #-1] @ write for ownership
220#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -0700221 bic r1, r1, #D_CACHE_LINE_SIZE - 1
222#ifdef HARVARD_CACHE
223 mcrne p15, 0, r1, c7, c14, 1 @ clean & invalidate D line
224#else
225 mcrne p15, 0, r1, c7, c15, 1 @ clean & invalidate unified line
226#endif
2271:
228#ifdef HARVARD_CACHE
229 mcr p15, 0, r0, c7, c6, 1 @ invalidate D line
230#else
231 mcr p15, 0, r0, c7, c7, 1 @ invalidate unified line
232#endif
233 add r0, r0, #D_CACHE_LINE_SIZE
234 cmp r0, r1
Valentine Barshak85b093b2010-12-14 00:03:16 +0100235#ifdef CONFIG_DMA_CACHE_RWFO
236 ldrlo r2, [r0] @ read for ownership
237 strlo r2, [r0] @ write for ownership
238#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -0700239 blo 1b
240 mov r0, #0
241 mcr p15, 0, r0, c7, c10, 4 @ drain write buffer
Russell King6ebbf2c2014-06-30 16:29:12 +0100242 ret lr
Linus Torvalds1da177e2005-04-16 15:20:36 -0700243
244/*
245 * v6_dma_clean_range(start,end)
246 * - start - virtual start address of region
247 * - end - virtual end address of region
248 */
Russell King702b94b2009-11-26 16:24:19 +0000249v6_dma_clean_range:
Linus Torvalds1da177e2005-04-16 15:20:36 -0700250 bic r0, r0, #D_CACHE_LINE_SIZE - 1
2511:
Catalin Marinasad642d92010-06-21 15:10:07 +0100252#ifdef CONFIG_DMA_CACHE_RWFO
Catalin Marinasf4d64772010-05-07 16:26:24 +0100253 ldr r2, [r0] @ read for ownership
254#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -0700255#ifdef HARVARD_CACHE
256 mcr p15, 0, r0, c7, c10, 1 @ clean D line
257#else
258 mcr p15, 0, r0, c7, c11, 1 @ clean unified line
259#endif
260 add r0, r0, #D_CACHE_LINE_SIZE
261 cmp r0, r1
262 blo 1b
263 mov r0, #0
264 mcr p15, 0, r0, c7, c10, 4 @ drain write buffer
Russell King6ebbf2c2014-06-30 16:29:12 +0100265 ret lr
Linus Torvalds1da177e2005-04-16 15:20:36 -0700266
267/*
268 * v6_dma_flush_range(start,end)
269 * - start - virtual start address of region
270 * - end - virtual end address of region
271 */
272ENTRY(v6_dma_flush_range)
Valentine Barshak85b093b2010-12-14 00:03:16 +0100273#ifdef CONFIG_DMA_CACHE_RWFO
274 ldrb r2, [r0] @ read for ownership
275 strb r2, [r0] @ write for ownership
276#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -0700277 bic r0, r0, #D_CACHE_LINE_SIZE - 1
2781:
279#ifdef HARVARD_CACHE
280 mcr p15, 0, r0, c7, c14, 1 @ clean & invalidate D line
281#else
282 mcr p15, 0, r0, c7, c15, 1 @ clean & invalidate line
283#endif
284 add r0, r0, #D_CACHE_LINE_SIZE
285 cmp r0, r1
Valentine Barshak85b093b2010-12-14 00:03:16 +0100286#ifdef CONFIG_DMA_CACHE_RWFO
287 ldrlob r2, [r0] @ read for ownership
288 strlob r2, [r0] @ write for ownership
289#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -0700290 blo 1b
291 mov r0, #0
292 mcr p15, 0, r0, c7, c10, 4 @ drain write buffer
Russell King6ebbf2c2014-06-30 16:29:12 +0100293 ret lr
Linus Torvalds1da177e2005-04-16 15:20:36 -0700294
Russell Kinga9c91472009-11-26 16:19:58 +0000295/*
296 * dma_map_area(start, size, dir)
297 * - start - kernel virtual start address
298 * - size - size of region
299 * - dir - DMA direction
300 */
301ENTRY(v6_dma_map_area)
302 add r1, r1, r0
Russell King2ffe2da2009-10-31 16:52:16 +0000303 teq r2, #DMA_FROM_DEVICE
304 beq v6_dma_inv_range
Catalin Marinasad642d92010-06-21 15:10:07 +0100305#ifndef CONFIG_DMA_CACHE_RWFO
306 b v6_dma_clean_range
307#else
Catalin Marinasf4d64772010-05-07 16:26:24 +0100308 teq r2, #DMA_TO_DEVICE
309 beq v6_dma_clean_range
310 b v6_dma_flush_range
Catalin Marinasad642d92010-06-21 15:10:07 +0100311#endif
Russell Kinga9c91472009-11-26 16:19:58 +0000312ENDPROC(v6_dma_map_area)
313
314/*
315 * dma_unmap_area(start, size, dir)
316 * - start - kernel virtual start address
317 * - size - size of region
318 * - dir - DMA direction
319 */
320ENTRY(v6_dma_unmap_area)
Catalin Marinasad642d92010-06-21 15:10:07 +0100321#ifndef CONFIG_DMA_CACHE_RWFO
322 add r1, r1, r0
323 teq r2, #DMA_TO_DEVICE
324 bne v6_dma_inv_range
325#endif
Russell King6ebbf2c2014-06-30 16:29:12 +0100326 ret lr
Russell Kinga9c91472009-11-26 16:19:58 +0000327ENDPROC(v6_dma_unmap_area)
328
Lorenzo Pieralisi031bd872012-09-06 18:35:13 +0530329 .globl v6_flush_kern_cache_louis
330 .equ v6_flush_kern_cache_louis, v6_flush_kern_cache_all
331
Linus Torvalds1da177e2005-04-16 15:20:36 -0700332 __INITDATA
333
Dave Martin641d8232011-06-23 17:16:04 +0100334 @ define struct cpu_cache_fns (see <asm/cacheflush.h> and proc-macros.S)
335 define_cache_functions v6