4 * Copyright (C) 2001 Deep Blue Solutions Ltd.
5 * Copyright (C) 2012 ARM Ltd.
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License version 2 as
9 * published by the Free Software Foundation.
11 * This program is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
16 * You should have received a copy of the GNU General Public License
17 * along with this program. If not, see <http://www.gnu.org/licenses/>.
20 #include <linux/linkage.h>
21 #include <linux/init.h>
22 #include <asm/assembler.h>
23 #include <asm/cpufeature.h>
24 #include <asm/alternative-asm.h>
26 #include "proc-macros.S"
29 * __flush_dcache_all()
31 * Flush the whole D-cache.
33 * Corrupted registers: x0-x7, x9-x11
36 dmb sy // ensure ordering with previous memory accesses
37 mrs x0, clidr_el1 // read clidr
38 and x3, x0, #0x7000000 // extract loc from clidr
39 lsr x3, x3, #23 // left align loc bit field
40 cbz x3, finished // if loc is 0, then no need to clean
41 mov x10, #0 // start clean at cache level 0
43 add x2, x10, x10, lsr #1 // work out 3x current cache level
44 lsr x1, x0, x2 // extract cache type bits from clidr
45 and x1, x1, #7 // mask of the bits for current cache only
46 cmp x1, #2 // see what cache we have at this level
47 b.lt skip // skip if no cache, or just i-cache
48 save_and_disable_irqs x9 // make CSSELR and CCSIDR access atomic
49 msr csselr_el1, x10 // select current cache level in csselr
50 isb // isb to sych the new cssr&csidr
51 mrs x1, ccsidr_el1 // read the new ccsidr
53 and x2, x1, #7 // extract the length of the cache lines
54 add x2, x2, #4 // add 4 (line length offset)
56 and x4, x4, x1, lsr #3 // find maximum number on the way size
57 clz w5, w4 // find bit position of way size increment
59 and x7, x7, x1, lsr #13 // extract max number of the index size
61 mov x9, x4 // create working copy of max way size
64 orr x11, x10, x6 // factor way and cache number into x11
66 orr x11, x11, x6 // factor index number into x11
67 dc cisw, x11 // clean & invalidate by set/way
68 subs x9, x9, #1 // decrement the way
70 subs x7, x7, #1 // decrement the index
73 add x10, x10, #2 // increment cache number
77 mov x10, #0 // swith back to cache level 0
78 msr csselr_el1, x10 // select current cache level in csselr
82 ENDPROC(__flush_dcache_all)
87 * Flush the entire cache system. The data cache flush is now achieved
88 * using atomic clean / invalidates working outwards from L1 cache. This
89 * is done using Set/Way based cache maintainance instructions. The
90 * instruction cache can still be invalidated back to the point of
91 * unification in a single instruction.
93 ENTRY(flush_cache_all)
97 ic ialluis // I+BTB cache invalidate
99 ENDPROC(flush_cache_all)
102 * flush_icache_range(start,end)
104 * Ensure that the I and D caches are coherent within specified region.
105 * This is typically used when code has been written to a memory region,
106 * and will be executed.
108 * - start - virtual start address of region
109 * - end - virtual end address of region
111 ENTRY(flush_icache_range)
115 * __flush_cache_user_range(start,end)
117 * Ensure that the I and D caches are coherent within specified region.
118 * This is typically used when code has been written to a memory region,
119 * and will be executed.
121 * - start - virtual start address of region
122 * - end - virtual end address of region
124 ENTRY(__flush_cache_user_range)
125 dcache_line_size x2, x3
129 USER(9f, dc cvau, x4 ) // clean D line to PoU
135 icache_line_size x2, x3
139 USER(9f, ic ivau, x4 ) // invalidate I line PoU
143 9: // ignore any faulting cache operation
147 ENDPROC(flush_icache_range)
148 ENDPROC(__flush_cache_user_range)
151 * __flush_dcache_area(kaddr, size)
153 * Ensure that the data held in the page kaddr is written back to the
156 * - kaddr - kernel address
157 * - size - size in question
159 ENTRY(__flush_dcache_area)
160 dcache_line_size x2, x3
164 1: dc civac, x0 // clean & invalidate D line / unified line
170 ENDPROC(__flush_dcache_area)
173 * __inval_cache_range(start, end)
174 * - start - start address of region
175 * - end - end address of region
177 ENTRY(__inval_cache_range)
181 * __dma_inv_range(start, end)
182 * - start - virtual start address of region
183 * - end - virtual end address of region
186 dcache_line_size x2, x3
188 tst x1, x3 // end cache line aligned?
191 dc civac, x1 // clean & invalidate D / U line
192 1: tst x0, x3 // start cache line aligned?
195 dc civac, x0 // clean & invalidate D / U line
197 2: dc ivac, x0 // invalidate D / U line
203 ENDPROC(__inval_cache_range)
204 ENDPROC(__dma_inv_range)
207 * __dma_clean_range(start, end)
208 * - start - virtual start address of region
209 * - end - virtual end address of region
212 dcache_line_size x2, x3
215 1: alternative_insn "dc cvac, x0", "dc civac, x0", ARM64_WORKAROUND_CLEAN_CACHE
221 ENDPROC(__dma_clean_range)
224 * __dma_flush_range(start, end)
225 * - start - virtual start address of region
226 * - end - virtual end address of region
228 ENTRY(__dma_flush_range)
229 dcache_line_size x2, x3
232 1: dc civac, x0 // clean & invalidate D / U line
238 ENDPROC(__dma_flush_range)
241 * __dma_map_area(start, size, dir)
242 * - start - kernel virtual start address
243 * - size - size of region
244 * - dir - DMA direction
246 ENTRY(__dma_map_area)
248 cmp w2, #DMA_FROM_DEVICE
251 ENDPROC(__dma_map_area)
254 * __dma_unmap_area(start, size, dir)
255 * - start - kernel virtual start address
256 * - size - size of region
257 * - dir - DMA direction
259 ENTRY(__dma_unmap_area)
261 cmp w2, #DMA_TO_DEVICE
264 ENDPROC(__dma_unmap_area)