Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1 | /* |
| 2 | * This file contains the routines for flushing entries from the |
| 3 | * TLB and MMU hash table. |
| 4 | * |
| 5 | * Derived from arch/ppc64/mm/init.c: |
| 6 | * Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org) |
| 7 | * |
| 8 | * Modifications by Paul Mackerras (PowerMac) (paulus@cs.anu.edu.au) |
| 9 | * and Cort Dougan (PReP) (cort@cs.nmt.edu) |
| 10 | * Copyright (C) 1996 Paul Mackerras |
| 11 | * Amiga/APUS changes by Jesper Skov (jskov@cygnus.co.uk). |
| 12 | * |
| 13 | * Derived from "arch/i386/mm/init.c" |
| 14 | * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds |
| 15 | * |
| 16 | * Dave Engebretsen <engebret@us.ibm.com> |
| 17 | * Rework for PPC64 port. |
| 18 | * |
| 19 | * This program is free software; you can redistribute it and/or |
| 20 | * modify it under the terms of the GNU General Public License |
| 21 | * as published by the Free Software Foundation; either version |
| 22 | * 2 of the License, or (at your option) any later version. |
| 23 | */ |
Benjamin Herrenschmidt | 3c726f8 | 2005-11-07 11:06:55 +1100 | [diff] [blame] | 24 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 25 | #include <linux/kernel.h> |
| 26 | #include <linux/mm.h> |
| 27 | #include <linux/init.h> |
| 28 | #include <linux/percpu.h> |
| 29 | #include <linux/hardirq.h> |
| 30 | #include <asm/pgalloc.h> |
| 31 | #include <asm/tlbflush.h> |
| 32 | #include <asm/tlb.h> |
Benjamin Herrenschmidt | 3c726f8 | 2005-11-07 11:06:55 +1100 | [diff] [blame] | 33 | #include <asm/bug.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 34 | |
| 35 | DEFINE_PER_CPU(struct ppc64_tlb_batch, ppc64_tlb_batch); |
| 36 | |
| 37 | /* This is declared as we are using the more or less generic |
Jon Mason | 2ef9481 | 2006-01-23 10:58:20 -0600 | [diff] [blame] | 38 | * include/asm-powerpc/tlb.h file -- tgall |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 39 | */ |
| 40 | DEFINE_PER_CPU(struct mmu_gather, mmu_gathers); |
| 41 | DEFINE_PER_CPU(struct pte_freelist_batch *, pte_freelist_cur); |
| 42 | unsigned long pte_freelist_forced_free; |
| 43 | |
David Gibson | e28f7fa | 2005-08-05 19:39:06 +1000 | [diff] [blame] | 44 | struct pte_freelist_batch |
| 45 | { |
| 46 | struct rcu_head rcu; |
| 47 | unsigned int index; |
| 48 | pgtable_free_t tables[0]; |
| 49 | }; |
| 50 | |
| 51 | DEFINE_PER_CPU(struct pte_freelist_batch *, pte_freelist_cur); |
| 52 | unsigned long pte_freelist_forced_free; |
| 53 | |
| 54 | #define PTE_FREELIST_SIZE \ |
| 55 | ((PAGE_SIZE - sizeof(struct pte_freelist_batch)) \ |
| 56 | / sizeof(pgtable_free_t)) |
| 57 | |
| 58 | #ifdef CONFIG_SMP |
| 59 | static void pte_free_smp_sync(void *arg) |
| 60 | { |
| 61 | /* Do nothing, just ensure we sync with all CPUs */ |
| 62 | } |
| 63 | #endif |
| 64 | |
| 65 | /* This is only called when we are critically out of memory |
| 66 | * (and fail to get a page in pte_free_tlb). |
| 67 | */ |
| 68 | static void pgtable_free_now(pgtable_free_t pgf) |
| 69 | { |
| 70 | pte_freelist_forced_free++; |
| 71 | |
| 72 | smp_call_function(pte_free_smp_sync, NULL, 0, 1); |
| 73 | |
| 74 | pgtable_free(pgf); |
| 75 | } |
| 76 | |
| 77 | static void pte_free_rcu_callback(struct rcu_head *head) |
| 78 | { |
| 79 | struct pte_freelist_batch *batch = |
| 80 | container_of(head, struct pte_freelist_batch, rcu); |
| 81 | unsigned int i; |
| 82 | |
| 83 | for (i = 0; i < batch->index; i++) |
| 84 | pgtable_free(batch->tables[i]); |
| 85 | |
| 86 | free_page((unsigned long)batch); |
| 87 | } |
| 88 | |
| 89 | static void pte_free_submit(struct pte_freelist_batch *batch) |
| 90 | { |
| 91 | INIT_RCU_HEAD(&batch->rcu); |
| 92 | call_rcu(&batch->rcu, pte_free_rcu_callback); |
| 93 | } |
| 94 | |
| 95 | void pgtable_free_tlb(struct mmu_gather *tlb, pgtable_free_t pgf) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 96 | { |
Hugh Dickins | 01edcd8 | 2005-11-23 13:37:39 -0800 | [diff] [blame] | 97 | /* This is safe since tlb_gather_mmu has disabled preemption */ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 98 | cpumask_t local_cpumask = cpumask_of_cpu(smp_processor_id()); |
| 99 | struct pte_freelist_batch **batchp = &__get_cpu_var(pte_freelist_cur); |
| 100 | |
| 101 | if (atomic_read(&tlb->mm->mm_users) < 2 || |
| 102 | cpus_equal(tlb->mm->cpu_vm_mask, local_cpumask)) { |
David Gibson | e28f7fa | 2005-08-05 19:39:06 +1000 | [diff] [blame] | 103 | pgtable_free(pgf); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 104 | return; |
| 105 | } |
| 106 | |
| 107 | if (*batchp == NULL) { |
| 108 | *batchp = (struct pte_freelist_batch *)__get_free_page(GFP_ATOMIC); |
| 109 | if (*batchp == NULL) { |
David Gibson | e28f7fa | 2005-08-05 19:39:06 +1000 | [diff] [blame] | 110 | pgtable_free_now(pgf); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 111 | return; |
| 112 | } |
| 113 | (*batchp)->index = 0; |
| 114 | } |
David Gibson | e28f7fa | 2005-08-05 19:39:06 +1000 | [diff] [blame] | 115 | (*batchp)->tables[(*batchp)->index++] = pgf; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 116 | if ((*batchp)->index == PTE_FREELIST_SIZE) { |
| 117 | pte_free_submit(*batchp); |
| 118 | *batchp = NULL; |
| 119 | } |
| 120 | } |
| 121 | |
| 122 | /* |
Benjamin Herrenschmidt | a741e67 | 2007-04-10 17:09:37 +1000 | [diff] [blame^] | 123 | * A linux PTE was changed and the corresponding hash table entry |
| 124 | * neesd to be flushed. This function will either perform the flush |
| 125 | * immediately or will batch it up if the current CPU has an active |
| 126 | * batch on it. |
| 127 | * |
| 128 | * Must be called from within some kind of spinlock/non-preempt region... |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 129 | */ |
Benjamin Herrenschmidt | a741e67 | 2007-04-10 17:09:37 +1000 | [diff] [blame^] | 130 | void hpte_need_flush(struct mm_struct *mm, unsigned long addr, |
| 131 | pte_t *ptep, unsigned long pte, int huge) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 132 | { |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 133 | struct ppc64_tlb_batch *batch = &__get_cpu_var(ppc64_tlb_batch); |
Benjamin Herrenschmidt | a741e67 | 2007-04-10 17:09:37 +1000 | [diff] [blame^] | 134 | unsigned long vsid, vaddr; |
Paul Mackerras | bf72aeb | 2006-06-15 10:45:18 +1000 | [diff] [blame] | 135 | unsigned int psize; |
Benjamin Herrenschmidt | a741e67 | 2007-04-10 17:09:37 +1000 | [diff] [blame^] | 136 | real_pte_t rpte; |
Benjamin Herrenschmidt | 61b1a94 | 2005-09-20 13:52:50 +1000 | [diff] [blame] | 137 | int i; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 138 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 139 | i = batch->index; |
| 140 | |
Benjamin Herrenschmidt | 3c726f8 | 2005-11-07 11:06:55 +1100 | [diff] [blame] | 141 | /* We mask the address for the base page size. Huge pages will |
| 142 | * have applied their own masking already |
| 143 | */ |
| 144 | addr &= PAGE_MASK; |
| 145 | |
| 146 | /* Get page size (maybe move back to caller) */ |
| 147 | if (huge) { |
| 148 | #ifdef CONFIG_HUGETLB_PAGE |
| 149 | psize = mmu_huge_psize; |
| 150 | #else |
| 151 | BUG(); |
Michael Ellerman | 2d69ff3 | 2006-07-13 17:54:44 +1000 | [diff] [blame] | 152 | psize = pte_pagesize_index(pte); /* shutup gcc */ |
Benjamin Herrenschmidt | 3c726f8 | 2005-11-07 11:06:55 +1100 | [diff] [blame] | 153 | #endif |
Paul Mackerras | bf72aeb | 2006-06-15 10:45:18 +1000 | [diff] [blame] | 154 | } else |
| 155 | psize = pte_pagesize_index(pte); |
Benjamin Herrenschmidt | 3c726f8 | 2005-11-07 11:06:55 +1100 | [diff] [blame] | 156 | |
Benjamin Herrenschmidt | a741e67 | 2007-04-10 17:09:37 +1000 | [diff] [blame^] | 157 | /* Build full vaddr */ |
| 158 | if (!is_kernel_addr(addr)) { |
| 159 | vsid = get_vsid(mm->context.id, addr); |
| 160 | WARN_ON(vsid == 0); |
| 161 | } else |
| 162 | vsid = get_kernel_vsid(addr); |
| 163 | vaddr = (vsid << 28 ) | (addr & 0x0fffffff); |
| 164 | rpte = __real_pte(__pte(pte), ptep); |
| 165 | |
| 166 | /* |
| 167 | * Check if we have an active batch on this CPU. If not, just |
| 168 | * flush now and return. For now, we don global invalidates |
| 169 | * in that case, might be worth testing the mm cpu mask though |
| 170 | * and decide to use local invalidates instead... |
| 171 | */ |
| 172 | if (!batch->active) { |
| 173 | flush_hash_page(vaddr, rpte, psize, 0); |
| 174 | return; |
| 175 | } |
| 176 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 177 | /* |
| 178 | * This can happen when we are in the middle of a TLB batch and |
| 179 | * we encounter memory pressure (eg copy_page_range when it tries |
| 180 | * to allocate a new pte). If we have to reclaim memory and end |
| 181 | * up scanning and resetting referenced bits then our batch context |
| 182 | * will change mid stream. |
Benjamin Herrenschmidt | 3c726f8 | 2005-11-07 11:06:55 +1100 | [diff] [blame] | 183 | * |
| 184 | * We also need to ensure only one page size is present in a given |
| 185 | * batch |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 186 | */ |
Benjamin Herrenschmidt | 3c726f8 | 2005-11-07 11:06:55 +1100 | [diff] [blame] | 187 | if (i != 0 && (mm != batch->mm || batch->psize != psize)) { |
Benjamin Herrenschmidt | a741e67 | 2007-04-10 17:09:37 +1000 | [diff] [blame^] | 188 | __flush_tlb_pending(batch); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 189 | i = 0; |
| 190 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 191 | if (i == 0) { |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 192 | batch->mm = mm; |
Benjamin Herrenschmidt | 3c726f8 | 2005-11-07 11:06:55 +1100 | [diff] [blame] | 193 | batch->psize = psize; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 194 | } |
Benjamin Herrenschmidt | a741e67 | 2007-04-10 17:09:37 +1000 | [diff] [blame^] | 195 | batch->pte[i] = rpte; |
| 196 | batch->vaddr[i] = vaddr; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 197 | batch->index = ++i; |
| 198 | if (i >= PPC64_TLB_BATCH_NR) |
Benjamin Herrenschmidt | a741e67 | 2007-04-10 17:09:37 +1000 | [diff] [blame^] | 199 | __flush_tlb_pending(batch); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 200 | } |
| 201 | |
Benjamin Herrenschmidt | a741e67 | 2007-04-10 17:09:37 +1000 | [diff] [blame^] | 202 | /* |
| 203 | * This function is called when terminating an mmu batch or when a batch |
| 204 | * is full. It will perform the flush of all the entries currently stored |
| 205 | * in a batch. |
| 206 | * |
| 207 | * Must be called from within some kind of spinlock/non-preempt region... |
| 208 | */ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 209 | void __flush_tlb_pending(struct ppc64_tlb_batch *batch) |
| 210 | { |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 211 | cpumask_t tmp; |
Benjamin Herrenschmidt | a741e67 | 2007-04-10 17:09:37 +1000 | [diff] [blame^] | 212 | int i, local = 0; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 213 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 214 | i = batch->index; |
Benjamin Herrenschmidt | a741e67 | 2007-04-10 17:09:37 +1000 | [diff] [blame^] | 215 | tmp = cpumask_of_cpu(smp_processor_id()); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 216 | if (cpus_equal(batch->mm->cpu_vm_mask, tmp)) |
| 217 | local = 1; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 218 | if (i == 1) |
Benjamin Herrenschmidt | 3c726f8 | 2005-11-07 11:06:55 +1100 | [diff] [blame] | 219 | flush_hash_page(batch->vaddr[0], batch->pte[0], |
| 220 | batch->psize, local); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 221 | else |
Benjamin Herrenschmidt | 61b1a94 | 2005-09-20 13:52:50 +1000 | [diff] [blame] | 222 | flush_hash_range(i, local); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 223 | batch->index = 0; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 224 | } |
| 225 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 226 | void pte_free_finish(void) |
| 227 | { |
Hugh Dickins | 01edcd8 | 2005-11-23 13:37:39 -0800 | [diff] [blame] | 228 | /* This is safe since tlb_gather_mmu has disabled preemption */ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 229 | struct pte_freelist_batch **batchp = &__get_cpu_var(pte_freelist_cur); |
| 230 | |
| 231 | if (*batchp == NULL) |
| 232 | return; |
| 233 | pte_free_submit(*batchp); |
| 234 | *batchp = NULL; |
| 235 | } |