Soby Mathew | f24307d | 2016-05-05 12:31:57 +0100 | [diff] [blame] | 1 | /* |
| 2 | * Copyright (c) 2016, ARM Limited and Contributors. All rights reserved. |
| 3 | * |
dp-arm | 82cb2c1 | 2017-05-03 09:38:09 +0100 | [diff] [blame] | 4 | * SPDX-License-Identifier: BSD-3-Clause |
Soby Mathew | f24307d | 2016-05-05 12:31:57 +0100 | [diff] [blame] | 5 | */ |
| 6 | |
| 7 | #include <arch.h> |
| 8 | #include <asm_macros.S> |
| 9 | |
| 10 | .globl flush_dcache_range |
| 11 | .globl clean_dcache_range |
| 12 | .globl inv_dcache_range |
| 13 | .globl dcsw_op_louis |
| 14 | .globl dcsw_op_all |
| 15 | .globl dcsw_op_level1 |
| 16 | .globl dcsw_op_level2 |
| 17 | .globl dcsw_op_level3 |
| 18 | |
| 19 | /* |
| 20 | * This macro can be used for implementing various data cache operations `op` |
| 21 | */ |
| 22 | .macro do_dcache_maintenance_by_mva op, coproc, opc1, CRn, CRm, opc2 |
Soby Mathew | 3ec5204 | 2017-06-15 16:18:45 +0100 | [diff] [blame] | 23 | /* Exit early if size is zero */ |
| 24 | cmp r1, #0 |
| 25 | beq exit_loop_\op |
Soby Mathew | f24307d | 2016-05-05 12:31:57 +0100 | [diff] [blame] | 26 | dcache_line_size r2, r3 |
| 27 | add r1, r0, r1 |
| 28 | sub r3, r2, #1 |
| 29 | bic r0, r0, r3 |
| 30 | loop_\op: |
| 31 | stcopr r0, \coproc, \opc1, \CRn, \CRm, \opc2 |
| 32 | add r0, r0, r2 |
| 33 | cmp r0, r1 |
| 34 | blo loop_\op |
| 35 | dsb sy |
Soby Mathew | 3ec5204 | 2017-06-15 16:18:45 +0100 | [diff] [blame] | 36 | exit_loop_\op: |
Soby Mathew | f24307d | 2016-05-05 12:31:57 +0100 | [diff] [blame] | 37 | bx lr |
| 38 | .endm |
| 39 | |
| 40 | /* ------------------------------------------ |
| 41 | * Clean+Invalidate from base address till |
| 42 | * size. 'r0' = addr, 'r1' = size |
| 43 | * ------------------------------------------ |
| 44 | */ |
| 45 | func flush_dcache_range |
| 46 | do_dcache_maintenance_by_mva cimvac, DCCIMVAC |
| 47 | endfunc flush_dcache_range |
| 48 | |
| 49 | /* ------------------------------------------ |
| 50 | * Clean from base address till size. |
| 51 | * 'r0' = addr, 'r1' = size |
| 52 | * ------------------------------------------ |
| 53 | */ |
| 54 | func clean_dcache_range |
| 55 | do_dcache_maintenance_by_mva cmvac, DCCMVAC |
| 56 | endfunc clean_dcache_range |
| 57 | |
| 58 | /* ------------------------------------------ |
| 59 | * Invalidate from base address till |
| 60 | * size. 'r0' = addr, 'r1' = size |
| 61 | * ------------------------------------------ |
| 62 | */ |
| 63 | func inv_dcache_range |
| 64 | do_dcache_maintenance_by_mva imvac, DCIMVAC |
| 65 | endfunc inv_dcache_range |
| 66 | |
| 67 | /* ---------------------------------------------------------------- |
| 68 | * Data cache operations by set/way to the level specified |
| 69 | * |
| 70 | * The main function, do_dcsw_op requires: |
| 71 | * r0: The operation type (DC_OP_ISW, DC_OP_CISW, DC_OP_CSW), |
| 72 | * as defined in arch.h |
| 73 | * r1: The cache level to begin operation from |
| 74 | * r2: clidr_el1 |
| 75 | * r3: The last cache level to operate on |
| 76 | * and will carry out the operation on each data cache from level 0 |
| 77 | * to the level in r3 in sequence |
| 78 | * |
| 79 | * The dcsw_op macro sets up the r2 and r3 parameters based on |
| 80 | * clidr_el1 cache information before invoking the main function |
| 81 | * ---------------------------------------------------------------- |
| 82 | */ |
| 83 | |
| 84 | .macro dcsw_op shift, fw, ls |
| 85 | ldcopr r2, CLIDR |
| 86 | ubfx r3, r2, \shift, \fw |
| 87 | lsl r3, r3, \ls |
| 88 | mov r1, #0 |
| 89 | b do_dcsw_op |
| 90 | .endm |
| 91 | |
| 92 | func do_dcsw_op |
| 93 | push {r4-r12,lr} |
| 94 | adr r11, dcsw_loop_table // compute cache op based on the operation type |
| 95 | add r6, r11, r0, lsl #3 // cache op is 2x32-bit instructions |
| 96 | loop1: |
| 97 | add r10, r1, r1, LSR #1 // Work out 3x current cache level |
| 98 | mov r12, r2, LSR r10 // extract cache type bits from clidr |
| 99 | and r12, r12, #7 // mask the bits for current cache only |
| 100 | cmp r12, #2 // see what cache we have at this level |
Douglas Raillard | 355a5d0 | 2017-03-07 16:36:14 +0000 | [diff] [blame] | 101 | blo level_done // no cache or only instruction cache at this level |
Soby Mathew | f24307d | 2016-05-05 12:31:57 +0100 | [diff] [blame] | 102 | |
| 103 | stcopr r1, CSSELR // select current cache level in csselr |
| 104 | isb // isb to sych the new cssr&csidr |
| 105 | ldcopr r12, CCSIDR // read the new ccsidr |
| 106 | and r10, r12, #7 // extract the length of the cache lines |
| 107 | add r10, r10, #4 // add 4 (r10 = line length offset) |
| 108 | ubfx r4, r12, #3, #10 // r4 = maximum way number (right aligned) |
| 109 | clz r5, r4 // r5 = the bit position of the way size increment |
| 110 | mov r9, r4 // r9 working copy of the aligned max way number |
| 111 | |
| 112 | loop2: |
| 113 | ubfx r7, r12, #13, #15 // r7 = max set number (right aligned) |
| 114 | |
| 115 | loop3: |
| 116 | orr r0, r1, r9, LSL r5 // factor in the way number and cache level into r0 |
| 117 | orr r0, r0, r7, LSL r10 // factor in the set number |
| 118 | |
| 119 | blx r6 |
| 120 | subs r7, r7, #1 // decrement the set number |
Douglas Raillard | 355a5d0 | 2017-03-07 16:36:14 +0000 | [diff] [blame] | 121 | bhs loop3 |
Soby Mathew | f24307d | 2016-05-05 12:31:57 +0100 | [diff] [blame] | 122 | subs r9, r9, #1 // decrement the way number |
Douglas Raillard | 355a5d0 | 2017-03-07 16:36:14 +0000 | [diff] [blame] | 123 | bhs loop2 |
Soby Mathew | f24307d | 2016-05-05 12:31:57 +0100 | [diff] [blame] | 124 | level_done: |
| 125 | add r1, r1, #2 // increment the cache number |
| 126 | cmp r3, r1 |
Joel Hutton | f999fac | 2019-04-09 14:45:34 +0100 | [diff] [blame^] | 127 | // Ensure completion of previous cache maintenance instruction. Note |
| 128 | // this also mitigates erratum 814220 on Cortex-A7 |
| 129 | dsb sy |
Douglas Raillard | 355a5d0 | 2017-03-07 16:36:14 +0000 | [diff] [blame] | 130 | bhi loop1 |
Soby Mathew | f24307d | 2016-05-05 12:31:57 +0100 | [diff] [blame] | 131 | |
| 132 | mov r6, #0 |
| 133 | stcopr r6, CSSELR //select cache level 0 in csselr |
| 134 | dsb sy |
| 135 | isb |
| 136 | pop {r4-r12,pc} |
| 137 | |
| 138 | dcsw_loop_table: |
| 139 | stcopr r0, DCISW |
| 140 | bx lr |
| 141 | stcopr r0, DCCISW |
| 142 | bx lr |
| 143 | stcopr r0, DCCSW |
| 144 | bx lr |
| 145 | |
| 146 | endfunc do_dcsw_op |
| 147 | |
| 148 | /* --------------------------------------------------------------- |
| 149 | * Data cache operations by set/way till PoU. |
| 150 | * |
| 151 | * The function requires : |
| 152 | * r0: The operation type (DC_OP_ISW, DC_OP_CISW, DC_OP_CSW), |
| 153 | * as defined in arch.h |
| 154 | * --------------------------------------------------------------- |
| 155 | */ |
| 156 | func dcsw_op_louis |
| 157 | dcsw_op #LOUIS_SHIFT, #CLIDR_FIELD_WIDTH, #LEVEL_SHIFT |
| 158 | endfunc dcsw_op_louis |
| 159 | |
| 160 | /* --------------------------------------------------------------- |
| 161 | * Data cache operations by set/way till PoC. |
| 162 | * |
| 163 | * The function requires : |
| 164 | * r0: The operation type (DC_OP_ISW, DC_OP_CISW, DC_OP_CSW), |
| 165 | * as defined in arch.h |
| 166 | * --------------------------------------------------------------- |
| 167 | */ |
| 168 | func dcsw_op_all |
| 169 | dcsw_op #LOC_SHIFT, #CLIDR_FIELD_WIDTH, #LEVEL_SHIFT |
| 170 | endfunc dcsw_op_all |
| 171 | |
| 172 | |
| 173 | /* --------------------------------------------------------------- |
| 174 | * Helper macro for data cache operations by set/way for the |
| 175 | * level specified |
| 176 | * --------------------------------------------------------------- |
| 177 | */ |
| 178 | .macro dcsw_op_level level |
| 179 | ldcopr r2, CLIDR |
| 180 | mov r3, \level |
| 181 | sub r1, r3, #2 |
| 182 | b do_dcsw_op |
| 183 | .endm |
| 184 | |
| 185 | /* --------------------------------------------------------------- |
| 186 | * Data cache operations by set/way for level 1 cache |
| 187 | * |
| 188 | * The main function, do_dcsw_op requires: |
| 189 | * r0: The operation type (DC_OP_ISW, DC_OP_CISW, DC_OP_CSW), |
| 190 | * as defined in arch.h |
| 191 | * --------------------------------------------------------------- |
| 192 | */ |
| 193 | func dcsw_op_level1 |
| 194 | dcsw_op_level #(1 << LEVEL_SHIFT) |
| 195 | endfunc dcsw_op_level1 |
| 196 | |
| 197 | /* --------------------------------------------------------------- |
| 198 | * Data cache operations by set/way for level 2 cache |
| 199 | * |
| 200 | * The main function, do_dcsw_op requires: |
| 201 | * r0: The operation type (DC_OP_ISW, DC_OP_CISW, DC_OP_CSW), |
| 202 | * as defined in arch.h |
| 203 | * --------------------------------------------------------------- |
| 204 | */ |
| 205 | func dcsw_op_level2 |
| 206 | dcsw_op_level #(2 << LEVEL_SHIFT) |
| 207 | endfunc dcsw_op_level2 |
| 208 | |
| 209 | /* --------------------------------------------------------------- |
| 210 | * Data cache operations by set/way for level 3 cache |
| 211 | * |
| 212 | * The main function, do_dcsw_op requires: |
| 213 | * r0: The operation type (DC_OP_ISW, DC_OP_CISW, DC_OP_CSW), |
| 214 | * as defined in arch.h |
| 215 | * --------------------------------------------------------------- |
| 216 | */ |
| 217 | func dcsw_op_level3 |
| 218 | dcsw_op_level #(3 << LEVEL_SHIFT) |
| 219 | endfunc dcsw_op_level3 |