Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1 | /* |
| 2 | * linux/arch/arm/mm/cache-v6.S |
| 3 | * |
| 4 | * Copyright (C) 2001 Deep Blue Solutions Ltd. |
| 5 | * |
| 6 | * This program is free software; you can redistribute it and/or modify |
| 7 | * it under the terms of the GNU General Public License version 2 as |
| 8 | * published by the Free Software Foundation. |
| 9 | * |
| 10 | * This is the "shell" of the ARMv6 processor support. |
| 11 | */ |
| 12 | #include <linux/linkage.h> |
| 13 | #include <linux/init.h> |
| 14 | #include <asm/assembler.h> |
Will Deacon | c5102f5 | 2012-04-27 13:08:53 +0100 | [diff] [blame] | 15 | #include <asm/errno.h> |
Catalin Marinas | 32cfb1b | 2009-10-06 17:57:09 +0100 | [diff] [blame] | 16 | #include <asm/unwind.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 17 | |
| 18 | #include "proc-macros.S" |
| 19 | |
| 20 | #define HARVARD_CACHE |
| 21 | #define CACHE_LINE_SIZE 32 |
| 22 | #define D_CACHE_LINE_SIZE 32 |
Gen FUKATSU | 217874f | 2005-09-30 16:09:17 +0100 | [diff] [blame] | 23 | #define BTB_FLUSH_SIZE 8 |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 24 | |
Catalin Marinas | 9cba3cc | 2009-04-30 17:06:03 +0100 | [diff] [blame] | 25 | /* |
Tony Lindgren | 81d1195 | 2010-09-21 17:16:40 +0100 | [diff] [blame] | 26 | * v6_flush_icache_all() |
Catalin Marinas | 9cba3cc | 2009-04-30 17:06:03 +0100 | [diff] [blame] | 27 | * |
Tony Lindgren | 81d1195 | 2010-09-21 17:16:40 +0100 | [diff] [blame] | 28 | * Flush the whole I-cache. |
| 29 | * |
| 30 | * ARM1136 erratum 411920 - Invalidate Instruction Cache operation can fail. |
| 31 | * This erratum is present in 1136, 1156 and 1176. It does not affect the |
| 32 | * MPCore. |
| 33 | * |
| 34 | * Registers: |
| 35 | * r0 - set to 0 |
| 36 | * r1 - corrupted |
Catalin Marinas | 9cba3cc | 2009-04-30 17:06:03 +0100 | [diff] [blame] | 37 | */ |
Tony Lindgren | 81d1195 | 2010-09-21 17:16:40 +0100 | [diff] [blame] | 38 | ENTRY(v6_flush_icache_all) |
Catalin Marinas | 9cba3cc | 2009-04-30 17:06:03 +0100 | [diff] [blame] | 39 | mov r0, #0 |
Tony Lindgren | 81d1195 | 2010-09-21 17:16:40 +0100 | [diff] [blame] | 40 | #ifdef CONFIG_ARM_ERRATA_411920 |
Catalin Marinas | 9cba3cc | 2009-04-30 17:06:03 +0100 | [diff] [blame] | 41 | mrs r1, cpsr |
| 42 | cpsid ifa @ disable interrupts |
| 43 | mcr p15, 0, r0, c7, c5, 0 @ invalidate entire I-cache |
| 44 | mcr p15, 0, r0, c7, c5, 0 @ invalidate entire I-cache |
| 45 | mcr p15, 0, r0, c7, c5, 0 @ invalidate entire I-cache |
| 46 | mcr p15, 0, r0, c7, c5, 0 @ invalidate entire I-cache |
| 47 | msr cpsr_cx, r1 @ restore interrupts |
| 48 | .rept 11 @ ARM Ltd recommends at least |
| 49 | nop @ 11 NOPs |
| 50 | .endr |
Tony Lindgren | 81d1195 | 2010-09-21 17:16:40 +0100 | [diff] [blame] | 51 | #else |
| 52 | mcr p15, 0, r0, c7, c5, 0 @ invalidate I-cache |
Catalin Marinas | 9cba3cc | 2009-04-30 17:06:03 +0100 | [diff] [blame] | 53 | #endif |
Tony Lindgren | 81d1195 | 2010-09-21 17:16:40 +0100 | [diff] [blame] | 54 | mov pc, lr |
| 55 | ENDPROC(v6_flush_icache_all) |
Catalin Marinas | 9cba3cc | 2009-04-30 17:06:03 +0100 | [diff] [blame] | 56 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 57 | /* |
| 58 | * v6_flush_cache_all() |
| 59 | * |
| 60 | * Flush the entire cache. |
| 61 | * |
| 62 | * It is assumed that: |
| 63 | */ |
| 64 | ENTRY(v6_flush_kern_cache_all) |
| 65 | mov r0, #0 |
| 66 | #ifdef HARVARD_CACHE |
| 67 | mcr p15, 0, r0, c7, c14, 0 @ D cache clean+invalidate |
Catalin Marinas | 9cba3cc | 2009-04-30 17:06:03 +0100 | [diff] [blame] | 68 | #ifndef CONFIG_ARM_ERRATA_411920 |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 69 | mcr p15, 0, r0, c7, c5, 0 @ I+BTB cache invalidate |
| 70 | #else |
Tony Lindgren | 81d1195 | 2010-09-21 17:16:40 +0100 | [diff] [blame] | 71 | b v6_flush_icache_all |
Catalin Marinas | 9cba3cc | 2009-04-30 17:06:03 +0100 | [diff] [blame] | 72 | #endif |
| 73 | #else |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 74 | mcr p15, 0, r0, c7, c15, 0 @ Cache clean+invalidate |
| 75 | #endif |
| 76 | mov pc, lr |
| 77 | |
| 78 | /* |
| 79 | * v6_flush_cache_all() |
| 80 | * |
| 81 | * Flush all TLB entries in a particular address space |
| 82 | * |
| 83 | * - mm - mm_struct describing address space |
| 84 | */ |
| 85 | ENTRY(v6_flush_user_cache_all) |
| 86 | /*FALLTHROUGH*/ |
| 87 | |
| 88 | /* |
| 89 | * v6_flush_cache_range(start, end, flags) |
| 90 | * |
| 91 | * Flush a range of TLB entries in the specified address space. |
| 92 | * |
| 93 | * - start - start address (may not be aligned) |
| 94 | * - end - end address (exclusive, may not be aligned) |
| 95 | * - flags - vm_area_struct flags describing address space |
| 96 | * |
| 97 | * It is assumed that: |
| 98 | * - we have a VIPT cache. |
| 99 | */ |
| 100 | ENTRY(v6_flush_user_cache_range) |
| 101 | mov pc, lr |
| 102 | |
| 103 | /* |
| 104 | * v6_coherent_kern_range(start,end) |
| 105 | * |
| 106 | * Ensure that the I and D caches are coherent within specified |
| 107 | * region. This is typically used when code has been written to |
| 108 | * a memory region, and will be executed. |
| 109 | * |
| 110 | * - start - virtual start address of region |
| 111 | * - end - virtual end address of region |
| 112 | * |
| 113 | * It is assumed that: |
| 114 | * - the Icache does not read data from the write buffer |
| 115 | */ |
| 116 | ENTRY(v6_coherent_kern_range) |
| 117 | /* FALLTHROUGH */ |
| 118 | |
| 119 | /* |
| 120 | * v6_coherent_user_range(start,end) |
| 121 | * |
| 122 | * Ensure that the I and D caches are coherent within specified |
| 123 | * region. This is typically used when code has been written to |
| 124 | * a memory region, and will be executed. |
| 125 | * |
| 126 | * - start - virtual start address of region |
| 127 | * - end - virtual end address of region |
| 128 | * |
| 129 | * It is assumed that: |
| 130 | * - the Icache does not read data from the write buffer |
| 131 | */ |
| 132 | ENTRY(v6_coherent_user_range) |
Catalin Marinas | 32cfb1b | 2009-10-06 17:57:09 +0100 | [diff] [blame] | 133 | UNWIND(.fnstart ) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 134 | #ifdef HARVARD_CACHE |
Nicolas Pitre | 18afea0 | 2006-02-01 19:26:01 +0000 | [diff] [blame] | 135 | bic r0, r0, #CACHE_LINE_SIZE - 1 |
Catalin Marinas | 32cfb1b | 2009-10-06 17:57:09 +0100 | [diff] [blame] | 136 | 1: |
| 137 | USER( mcr p15, 0, r0, c7, c10, 1 ) @ clean D line |
Nicolas Pitre | 18afea0 | 2006-02-01 19:26:01 +0000 | [diff] [blame] | 138 | add r0, r0, #CACHE_LINE_SIZE |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 139 | cmp r0, r1 |
| 140 | blo 1b |
Nicolas Pitre | 18afea0 | 2006-02-01 19:26:01 +0000 | [diff] [blame] | 141 | #endif |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 142 | mov r0, #0 |
Catalin Marinas | 141fa40 | 2006-03-10 22:26:47 +0000 | [diff] [blame] | 143 | #ifdef HARVARD_CACHE |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 144 | mcr p15, 0, r0, c7, c10, 4 @ drain write buffer |
Catalin Marinas | 9cba3cc | 2009-04-30 17:06:03 +0100 | [diff] [blame] | 145 | #ifndef CONFIG_ARM_ERRATA_411920 |
Catalin Marinas | 141fa40 | 2006-03-10 22:26:47 +0000 | [diff] [blame] | 146 | mcr p15, 0, r0, c7, c5, 0 @ I+BTB cache invalidate |
| 147 | #else |
Tony Lindgren | 81d1195 | 2010-09-21 17:16:40 +0100 | [diff] [blame] | 148 | b v6_flush_icache_all |
Catalin Marinas | 9cba3cc | 2009-04-30 17:06:03 +0100 | [diff] [blame] | 149 | #endif |
| 150 | #else |
Catalin Marinas | 141fa40 | 2006-03-10 22:26:47 +0000 | [diff] [blame] | 151 | mcr p15, 0, r0, c7, c5, 6 @ invalidate BTB |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 152 | #endif |
| 153 | mov pc, lr |
| 154 | |
| 155 | /* |
Catalin Marinas | 32cfb1b | 2009-10-06 17:57:09 +0100 | [diff] [blame] | 156 | * Fault handling for the cache operation above. If the virtual address in r0 |
Will Deacon | c5102f5 | 2012-04-27 13:08:53 +0100 | [diff] [blame] | 157 | * isn't mapped, fail with -EFAULT. |
Catalin Marinas | 32cfb1b | 2009-10-06 17:57:09 +0100 | [diff] [blame] | 158 | */ |
| 159 | 9001: |
Will Deacon | c5102f5 | 2012-04-27 13:08:53 +0100 | [diff] [blame] | 160 | mov r0, #-EFAULT |
| 161 | mov pc, lr |
Catalin Marinas | 32cfb1b | 2009-10-06 17:57:09 +0100 | [diff] [blame] | 162 | UNWIND(.fnend ) |
| 163 | ENDPROC(v6_coherent_user_range) |
| 164 | ENDPROC(v6_coherent_kern_range) |
| 165 | |
| 166 | /* |
Russell King | 2c9b9c8 | 2009-11-26 12:56:21 +0000 | [diff] [blame] | 167 | * v6_flush_kern_dcache_area(void *addr, size_t size) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 168 | * |
| 169 | * Ensure that the data held in the page kaddr is written back |
| 170 | * to the page in question. |
| 171 | * |
Russell King | 2c9b9c8 | 2009-11-26 12:56:21 +0000 | [diff] [blame] | 172 | * - addr - kernel address |
| 173 | * - size - region size |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 174 | */ |
Russell King | 2c9b9c8 | 2009-11-26 12:56:21 +0000 | [diff] [blame] | 175 | ENTRY(v6_flush_kern_dcache_area) |
| 176 | add r1, r0, r1 |
Will Deacon | a248b13 | 2011-05-26 11:20:19 +0100 | [diff] [blame] | 177 | bic r0, r0, #D_CACHE_LINE_SIZE - 1 |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 178 | 1: |
| 179 | #ifdef HARVARD_CACHE |
| 180 | mcr p15, 0, r0, c7, c14, 1 @ clean & invalidate D line |
| 181 | #else |
| 182 | mcr p15, 0, r0, c7, c15, 1 @ clean & invalidate unified line |
| 183 | #endif |
| 184 | add r0, r0, #D_CACHE_LINE_SIZE |
| 185 | cmp r0, r1 |
| 186 | blo 1b |
| 187 | #ifdef HARVARD_CACHE |
| 188 | mov r0, #0 |
| 189 | mcr p15, 0, r0, c7, c10, 4 |
| 190 | #endif |
| 191 | mov pc, lr |
| 192 | |
| 193 | |
| 194 | /* |
| 195 | * v6_dma_inv_range(start,end) |
| 196 | * |
| 197 | * Invalidate the data cache within the specified region; we will |
| 198 | * be performing a DMA operation in this region and we want to |
| 199 | * purge old data in the cache. |
| 200 | * |
| 201 | * - start - virtual start address of region |
| 202 | * - end - virtual end address of region |
| 203 | */ |
Russell King | 702b94b | 2009-11-26 16:24:19 +0000 | [diff] [blame] | 204 | v6_dma_inv_range: |
Valentine Barshak | 85b093b | 2010-12-14 00:03:16 +0100 | [diff] [blame] | 205 | #ifdef CONFIG_DMA_CACHE_RWFO |
| 206 | ldrb r2, [r0] @ read for ownership |
| 207 | strb r2, [r0] @ write for ownership |
| 208 | #endif |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 209 | tst r0, #D_CACHE_LINE_SIZE - 1 |
| 210 | bic r0, r0, #D_CACHE_LINE_SIZE - 1 |
| 211 | #ifdef HARVARD_CACHE |
| 212 | mcrne p15, 0, r0, c7, c10, 1 @ clean D line |
| 213 | #else |
| 214 | mcrne p15, 0, r0, c7, c11, 1 @ clean unified line |
| 215 | #endif |
| 216 | tst r1, #D_CACHE_LINE_SIZE - 1 |
Valentine Barshak | 85b093b | 2010-12-14 00:03:16 +0100 | [diff] [blame] | 217 | #ifdef CONFIG_DMA_CACHE_RWFO |
| 218 | ldrneb r2, [r1, #-1] @ read for ownership |
| 219 | strneb r2, [r1, #-1] @ write for ownership |
| 220 | #endif |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 221 | bic r1, r1, #D_CACHE_LINE_SIZE - 1 |
| 222 | #ifdef HARVARD_CACHE |
| 223 | mcrne p15, 0, r1, c7, c14, 1 @ clean & invalidate D line |
| 224 | #else |
| 225 | mcrne p15, 0, r1, c7, c15, 1 @ clean & invalidate unified line |
| 226 | #endif |
| 227 | 1: |
| 228 | #ifdef HARVARD_CACHE |
| 229 | mcr p15, 0, r0, c7, c6, 1 @ invalidate D line |
| 230 | #else |
| 231 | mcr p15, 0, r0, c7, c7, 1 @ invalidate unified line |
| 232 | #endif |
| 233 | add r0, r0, #D_CACHE_LINE_SIZE |
| 234 | cmp r0, r1 |
Valentine Barshak | 85b093b | 2010-12-14 00:03:16 +0100 | [diff] [blame] | 235 | #ifdef CONFIG_DMA_CACHE_RWFO |
| 236 | ldrlo r2, [r0] @ read for ownership |
| 237 | strlo r2, [r0] @ write for ownership |
| 238 | #endif |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 239 | blo 1b |
| 240 | mov r0, #0 |
| 241 | mcr p15, 0, r0, c7, c10, 4 @ drain write buffer |
| 242 | mov pc, lr |
| 243 | |
| 244 | /* |
| 245 | * v6_dma_clean_range(start,end) |
| 246 | * - start - virtual start address of region |
| 247 | * - end - virtual end address of region |
| 248 | */ |
Russell King | 702b94b | 2009-11-26 16:24:19 +0000 | [diff] [blame] | 249 | v6_dma_clean_range: |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 250 | bic r0, r0, #D_CACHE_LINE_SIZE - 1 |
| 251 | 1: |
Catalin Marinas | ad642d9 | 2010-06-21 15:10:07 +0100 | [diff] [blame] | 252 | #ifdef CONFIG_DMA_CACHE_RWFO |
Catalin Marinas | f4d6477 | 2010-05-07 16:26:24 +0100 | [diff] [blame] | 253 | ldr r2, [r0] @ read for ownership |
| 254 | #endif |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 255 | #ifdef HARVARD_CACHE |
| 256 | mcr p15, 0, r0, c7, c10, 1 @ clean D line |
| 257 | #else |
| 258 | mcr p15, 0, r0, c7, c11, 1 @ clean unified line |
| 259 | #endif |
| 260 | add r0, r0, #D_CACHE_LINE_SIZE |
| 261 | cmp r0, r1 |
| 262 | blo 1b |
| 263 | mov r0, #0 |
| 264 | mcr p15, 0, r0, c7, c10, 4 @ drain write buffer |
| 265 | mov pc, lr |
| 266 | |
| 267 | /* |
| 268 | * v6_dma_flush_range(start,end) |
| 269 | * - start - virtual start address of region |
| 270 | * - end - virtual end address of region |
| 271 | */ |
| 272 | ENTRY(v6_dma_flush_range) |
Valentine Barshak | 85b093b | 2010-12-14 00:03:16 +0100 | [diff] [blame] | 273 | #ifdef CONFIG_DMA_CACHE_RWFO |
| 274 | ldrb r2, [r0] @ read for ownership |
| 275 | strb r2, [r0] @ write for ownership |
| 276 | #endif |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 277 | bic r0, r0, #D_CACHE_LINE_SIZE - 1 |
| 278 | 1: |
| 279 | #ifdef HARVARD_CACHE |
| 280 | mcr p15, 0, r0, c7, c14, 1 @ clean & invalidate D line |
| 281 | #else |
| 282 | mcr p15, 0, r0, c7, c15, 1 @ clean & invalidate line |
| 283 | #endif |
| 284 | add r0, r0, #D_CACHE_LINE_SIZE |
| 285 | cmp r0, r1 |
Valentine Barshak | 85b093b | 2010-12-14 00:03:16 +0100 | [diff] [blame] | 286 | #ifdef CONFIG_DMA_CACHE_RWFO |
| 287 | ldrlob r2, [r0] @ read for ownership |
| 288 | strlob r2, [r0] @ write for ownership |
| 289 | #endif |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 290 | blo 1b |
| 291 | mov r0, #0 |
| 292 | mcr p15, 0, r0, c7, c10, 4 @ drain write buffer |
| 293 | mov pc, lr |
| 294 | |
Russell King | a9c9147 | 2009-11-26 16:19:58 +0000 | [diff] [blame] | 295 | /* |
| 296 | * dma_map_area(start, size, dir) |
| 297 | * - start - kernel virtual start address |
| 298 | * - size - size of region |
| 299 | * - dir - DMA direction |
| 300 | */ |
| 301 | ENTRY(v6_dma_map_area) |
| 302 | add r1, r1, r0 |
Russell King | 2ffe2da | 2009-10-31 16:52:16 +0000 | [diff] [blame] | 303 | teq r2, #DMA_FROM_DEVICE |
| 304 | beq v6_dma_inv_range |
Catalin Marinas | ad642d9 | 2010-06-21 15:10:07 +0100 | [diff] [blame] | 305 | #ifndef CONFIG_DMA_CACHE_RWFO |
| 306 | b v6_dma_clean_range |
| 307 | #else |
Catalin Marinas | f4d6477 | 2010-05-07 16:26:24 +0100 | [diff] [blame] | 308 | teq r2, #DMA_TO_DEVICE |
| 309 | beq v6_dma_clean_range |
| 310 | b v6_dma_flush_range |
Catalin Marinas | ad642d9 | 2010-06-21 15:10:07 +0100 | [diff] [blame] | 311 | #endif |
Russell King | a9c9147 | 2009-11-26 16:19:58 +0000 | [diff] [blame] | 312 | ENDPROC(v6_dma_map_area) |
| 313 | |
| 314 | /* |
| 315 | * dma_unmap_area(start, size, dir) |
| 316 | * - start - kernel virtual start address |
| 317 | * - size - size of region |
| 318 | * - dir - DMA direction |
| 319 | */ |
| 320 | ENTRY(v6_dma_unmap_area) |
Catalin Marinas | ad642d9 | 2010-06-21 15:10:07 +0100 | [diff] [blame] | 321 | #ifndef CONFIG_DMA_CACHE_RWFO |
| 322 | add r1, r1, r0 |
| 323 | teq r2, #DMA_TO_DEVICE |
| 324 | bne v6_dma_inv_range |
| 325 | #endif |
Russell King | a9c9147 | 2009-11-26 16:19:58 +0000 | [diff] [blame] | 326 | mov pc, lr |
| 327 | ENDPROC(v6_dma_unmap_area) |
| 328 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 329 | __INITDATA |
| 330 | |
Dave Martin | 641d823 | 2011-06-23 17:16:04 +0100 | [diff] [blame] | 331 | @ define struct cpu_cache_fns (see <asm/cacheflush.h> and proc-macros.S) |
| 332 | define_cache_functions v6 |