Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1 | /* |
| 2 | * linux/arch/arm/mm/flush.c |
| 3 | * |
| 4 | * Copyright (C) 1995-2002 Russell King |
| 5 | * |
| 6 | * This program is free software; you can redistribute it and/or modify |
| 7 | * it under the terms of the GNU General Public License version 2 as |
| 8 | * published by the Free Software Foundation. |
| 9 | */ |
| 10 | #include <linux/module.h> |
| 11 | #include <linux/mm.h> |
| 12 | #include <linux/pagemap.h> |
| 13 | |
| 14 | #include <asm/cacheflush.h> |
| 15 | #include <asm/system.h> |
Russell King | 8d802d2 | 2005-05-10 17:31:43 +0100 | [diff] [blame] | 16 | #include <asm/tlbflush.h> |
| 17 | |
| 18 | #ifdef CONFIG_CPU_CACHE_VIPT |
Russell King | d7b6b35 | 2005-09-08 15:32:23 +0100 | [diff] [blame] | 19 | |
Catalin Marinas | 481467d | 2005-09-30 16:07:04 +0100 | [diff] [blame^] | 20 | #define ALIAS_FLUSH_START 0xffff4000 |
| 21 | |
| 22 | #define TOP_PTE(x) pte_offset_kernel(top_pmd, x) |
| 23 | |
| 24 | static void flush_pfn_alias(unsigned long pfn, unsigned long vaddr) |
| 25 | { |
| 26 | unsigned long to = ALIAS_FLUSH_START + (CACHE_COLOUR(vaddr) << PAGE_SHIFT); |
| 27 | |
| 28 | set_pte(TOP_PTE(to), pfn_pte(pfn, PAGE_KERNEL)); |
| 29 | flush_tlb_kernel_page(to); |
| 30 | |
| 31 | asm( "mcrr p15, 0, %1, %0, c14\n" |
| 32 | " mcrr p15, 0, %1, %0, c5\n" |
| 33 | : |
| 34 | : "r" (to), "r" (to + PAGE_SIZE - L1_CACHE_BYTES) |
| 35 | : "cc"); |
| 36 | } |
| 37 | |
Russell King | d7b6b35 | 2005-09-08 15:32:23 +0100 | [diff] [blame] | 38 | void flush_cache_mm(struct mm_struct *mm) |
| 39 | { |
| 40 | if (cache_is_vivt()) { |
| 41 | if (cpu_isset(smp_processor_id(), mm->cpu_vm_mask)) |
| 42 | __cpuc_flush_user_all(); |
| 43 | return; |
| 44 | } |
| 45 | |
| 46 | if (cache_is_vipt_aliasing()) { |
| 47 | asm( "mcr p15, 0, %0, c7, c14, 0\n" |
| 48 | " mcr p15, 0, %0, c7, c5, 0\n" |
| 49 | " mcr p15, 0, %0, c7, c10, 4" |
| 50 | : |
| 51 | : "r" (0) |
| 52 | : "cc"); |
| 53 | } |
| 54 | } |
| 55 | |
| 56 | void flush_cache_range(struct vm_area_struct *vma, unsigned long start, unsigned long end) |
| 57 | { |
| 58 | if (cache_is_vivt()) { |
| 59 | if (cpu_isset(smp_processor_id(), vma->vm_mm->cpu_vm_mask)) |
| 60 | __cpuc_flush_user_range(start & PAGE_MASK, PAGE_ALIGN(end), |
| 61 | vma->vm_flags); |
| 62 | return; |
| 63 | } |
| 64 | |
| 65 | if (cache_is_vipt_aliasing()) { |
| 66 | asm( "mcr p15, 0, %0, c7, c14, 0\n" |
| 67 | " mcr p15, 0, %0, c7, c5, 0\n" |
| 68 | " mcr p15, 0, %0, c7, c10, 4" |
| 69 | : |
| 70 | : "r" (0) |
| 71 | : "cc"); |
| 72 | } |
| 73 | } |
| 74 | |
| 75 | void flush_cache_page(struct vm_area_struct *vma, unsigned long user_addr, unsigned long pfn) |
| 76 | { |
| 77 | if (cache_is_vivt()) { |
| 78 | if (cpu_isset(smp_processor_id(), vma->vm_mm->cpu_vm_mask)) { |
| 79 | unsigned long addr = user_addr & PAGE_MASK; |
| 80 | __cpuc_flush_user_range(addr, addr + PAGE_SIZE, vma->vm_flags); |
| 81 | } |
| 82 | return; |
| 83 | } |
| 84 | |
| 85 | if (cache_is_vipt_aliasing()) |
| 86 | flush_pfn_alias(pfn, user_addr); |
| 87 | } |
Russell King | 8d802d2 | 2005-05-10 17:31:43 +0100 | [diff] [blame] | 88 | #else |
| 89 | #define flush_pfn_alias(pfn,vaddr) do { } while (0) |
| 90 | #endif |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 91 | |
Russell King | 8830f04 | 2005-06-20 09:51:03 +0100 | [diff] [blame] | 92 | void __flush_dcache_page(struct address_space *mapping, struct page *page) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 93 | { |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 94 | /* |
| 95 | * Writeback any data associated with the kernel mapping of this |
| 96 | * page. This ensures that data in the physical page is mutually |
| 97 | * coherent with the kernels mapping. |
| 98 | */ |
| 99 | __cpuc_flush_dcache_page(page_address(page)); |
| 100 | |
| 101 | /* |
Russell King | 8830f04 | 2005-06-20 09:51:03 +0100 | [diff] [blame] | 102 | * If this is a page cache page, and we have an aliasing VIPT cache, |
| 103 | * we only need to do one flush - which would be at the relevant |
Russell King | 8d802d2 | 2005-05-10 17:31:43 +0100 | [diff] [blame] | 104 | * userspace colour, which is congruent with page->index. |
| 105 | */ |
Russell King | 8830f04 | 2005-06-20 09:51:03 +0100 | [diff] [blame] | 106 | if (mapping && cache_is_vipt_aliasing()) |
| 107 | flush_pfn_alias(page_to_pfn(page), |
| 108 | page->index << PAGE_CACHE_SHIFT); |
| 109 | } |
| 110 | |
| 111 | static void __flush_dcache_aliases(struct address_space *mapping, struct page *page) |
| 112 | { |
| 113 | struct mm_struct *mm = current->active_mm; |
| 114 | struct vm_area_struct *mpnt; |
| 115 | struct prio_tree_iter iter; |
| 116 | pgoff_t pgoff; |
Russell King | 8d802d2 | 2005-05-10 17:31:43 +0100 | [diff] [blame] | 117 | |
| 118 | /* |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 119 | * There are possible user space mappings of this page: |
| 120 | * - VIVT cache: we need to also write back and invalidate all user |
| 121 | * data in the current VM view associated with this page. |
| 122 | * - aliasing VIPT: we only need to find one mapping of this page. |
| 123 | */ |
| 124 | pgoff = page->index << (PAGE_CACHE_SHIFT - PAGE_SHIFT); |
| 125 | |
| 126 | flush_dcache_mmap_lock(mapping); |
| 127 | vma_prio_tree_foreach(mpnt, &iter, &mapping->i_mmap, pgoff, pgoff) { |
| 128 | unsigned long offset; |
| 129 | |
| 130 | /* |
| 131 | * If this VMA is not in our MM, we can ignore it. |
| 132 | */ |
| 133 | if (mpnt->vm_mm != mm) |
| 134 | continue; |
| 135 | if (!(mpnt->vm_flags & VM_MAYSHARE)) |
| 136 | continue; |
| 137 | offset = (pgoff - mpnt->vm_pgoff) << PAGE_SHIFT; |
| 138 | flush_cache_page(mpnt, mpnt->vm_start + offset, page_to_pfn(page)); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 139 | } |
| 140 | flush_dcache_mmap_unlock(mapping); |
| 141 | } |
| 142 | |
| 143 | /* |
| 144 | * Ensure cache coherency between kernel mapping and userspace mapping |
| 145 | * of this page. |
| 146 | * |
| 147 | * We have three cases to consider: |
| 148 | * - VIPT non-aliasing cache: fully coherent so nothing required. |
| 149 | * - VIVT: fully aliasing, so we need to handle every alias in our |
| 150 | * current VM view. |
| 151 | * - VIPT aliasing: need to handle one alias in our current VM view. |
| 152 | * |
| 153 | * If we need to handle aliasing: |
| 154 | * If the page only exists in the page cache and there are no user |
| 155 | * space mappings, we can be lazy and remember that we may have dirty |
| 156 | * kernel cache lines for later. Otherwise, we assume we have |
| 157 | * aliasing mappings. |
| 158 | */ |
| 159 | void flush_dcache_page(struct page *page) |
| 160 | { |
| 161 | struct address_space *mapping = page_mapping(page); |
| 162 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 163 | if (mapping && !mapping_mapped(mapping)) |
| 164 | set_bit(PG_dcache_dirty, &page->flags); |
Russell King | 8830f04 | 2005-06-20 09:51:03 +0100 | [diff] [blame] | 165 | else { |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 166 | __flush_dcache_page(mapping, page); |
Russell King | 8830f04 | 2005-06-20 09:51:03 +0100 | [diff] [blame] | 167 | if (mapping && cache_is_vivt()) |
| 168 | __flush_dcache_aliases(mapping, page); |
| 169 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 170 | } |
| 171 | EXPORT_SYMBOL(flush_dcache_page); |