blob: 83d89bcb44afcace24b757b02ce5305c1e762afe [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/* arch/sparc64/mm/tlb.c
2 *
3 * Copyright (C) 2004 David S. Miller <davem@redhat.com>
4 */
5
6#include <linux/kernel.h>
7#include <linux/init.h>
8#include <linux/percpu.h>
9#include <linux/mm.h>
10#include <linux/swap.h>
David S. Millerc9f29462006-04-30 22:54:27 -070011#include <linux/preempt.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070012
13#include <asm/pgtable.h>
14#include <asm/pgalloc.h>
15#include <asm/tlbflush.h>
16#include <asm/cacheflush.h>
17#include <asm/mmu_context.h>
18#include <asm/tlb.h>
19
20/* Heavily inspired by the ppc64 code. */
21
Peter Zijlstra90f08e32011-05-24 17:11:50 -070022static DEFINE_PER_CPU(struct tlb_batch, tlb_batch);
Linus Torvalds1da177e2005-04-16 15:20:36 -070023
24void flush_tlb_pending(void)
25{
Peter Zijlstra90f08e32011-05-24 17:11:50 -070026 struct tlb_batch *tb = &get_cpu_var(tlb_batch);
David S. Millerf36391d2013-04-19 17:26:26 -040027 struct mm_struct *mm = tb->mm;
Linus Torvalds1da177e2005-04-16 15:20:36 -070028
David S. Millerf36391d2013-04-19 17:26:26 -040029 if (!tb->tlb_nr)
30 goto out;
David S. Miller74bf4312006-01-31 18:29:18 -080031
David S. Millerf36391d2013-04-19 17:26:26 -040032 flush_tsb_user(tb);
33
34 if (CTX_VALID(mm->context)) {
35 if (tb->tlb_nr == 1) {
36 global_flush_tlb_page(mm, tb->vaddrs[0]);
37 } else {
Linus Torvalds1da177e2005-04-16 15:20:36 -070038#ifdef CONFIG_SMP
Peter Zijlstra90f08e32011-05-24 17:11:50 -070039 smp_flush_tlb_pending(tb->mm, tb->tlb_nr,
40 &tb->vaddrs[0]);
Linus Torvalds1da177e2005-04-16 15:20:36 -070041#else
Peter Zijlstra90f08e32011-05-24 17:11:50 -070042 __flush_tlb_pending(CTX_HWBITS(tb->mm->context),
43 tb->tlb_nr, &tb->vaddrs[0]);
Linus Torvalds1da177e2005-04-16 15:20:36 -070044#endif
45 }
Linus Torvalds1da177e2005-04-16 15:20:36 -070046 }
David S. Millerc9f29462006-04-30 22:54:27 -070047
David S. Millerf36391d2013-04-19 17:26:26 -040048 tb->tlb_nr = 0;
49
50out:
Peter Zijlstra90f08e32011-05-24 17:11:50 -070051 put_cpu_var(tlb_batch);
Linus Torvalds1da177e2005-04-16 15:20:36 -070052}
53
David S. Millerf36391d2013-04-19 17:26:26 -040054void arch_enter_lazy_mmu_mode(void)
55{
56 struct tlb_batch *tb = &__get_cpu_var(tlb_batch);
57
58 tb->active = 1;
59}
60
61void arch_leave_lazy_mmu_mode(void)
62{
63 struct tlb_batch *tb = &__get_cpu_var(tlb_batch);
64
65 if (tb->tlb_nr)
66 flush_tlb_pending();
67 tb->active = 0;
68}
69
David Miller9e695d22012-10-08 16:34:29 -070070static void tlb_batch_add_one(struct mm_struct *mm, unsigned long vaddr,
71 bool exec)
Linus Torvalds1da177e2005-04-16 15:20:36 -070072{
Peter Zijlstra90f08e32011-05-24 17:11:50 -070073 struct tlb_batch *tb = &get_cpu_var(tlb_batch);
Linus Torvalds1da177e2005-04-16 15:20:36 -070074 unsigned long nr;
75
76 vaddr &= PAGE_MASK;
David Miller9e695d22012-10-08 16:34:29 -070077 if (exec)
Linus Torvalds1da177e2005-04-16 15:20:36 -070078 vaddr |= 0x1UL;
79
David Miller9e695d22012-10-08 16:34:29 -070080 nr = tb->tlb_nr;
81
82 if (unlikely(nr != 0 && mm != tb->mm)) {
83 flush_tlb_pending();
84 nr = 0;
85 }
86
David S. Millerf36391d2013-04-19 17:26:26 -040087 if (!tb->active) {
88 global_flush_tlb_page(mm, vaddr);
89 flush_tsb_user_page(mm, vaddr);
David S. Millerf0af9702013-04-24 16:52:18 -070090 goto out;
David S. Millerf36391d2013-04-19 17:26:26 -040091 }
92
David Miller9e695d22012-10-08 16:34:29 -070093 if (nr == 0)
94 tb->mm = mm;
95
96 tb->vaddrs[nr] = vaddr;
97 tb->tlb_nr = ++nr;
98 if (nr >= TLB_BATCH_NR)
99 flush_tlb_pending();
100
David S. Millerf0af9702013-04-24 16:52:18 -0700101out:
David Miller9e695d22012-10-08 16:34:29 -0700102 put_cpu_var(tlb_batch);
103}
104
105void tlb_batch_add(struct mm_struct *mm, unsigned long vaddr,
106 pte_t *ptep, pte_t orig, int fullmm)
107{
David S. Miller7a591cf2006-02-26 19:44:50 -0800108 if (tlb_type != hypervisor &&
109 pte_dirty(orig)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700110 unsigned long paddr, pfn = pte_pfn(orig);
111 struct address_space *mapping;
112 struct page *page;
113
114 if (!pfn_valid(pfn))
115 goto no_cache_flush;
116
117 page = pfn_to_page(pfn);
118 if (PageReserved(page))
119 goto no_cache_flush;
120
121 /* A real file page? */
122 mapping = page_mapping(page);
123 if (!mapping)
124 goto no_cache_flush;
125
126 paddr = (unsigned long) page_address(page);
127 if ((paddr ^ vaddr) & (1 << 13))
128 flush_dcache_page_all(mm, page);
129 }
130
131no_cache_flush:
David Miller9e695d22012-10-08 16:34:29 -0700132 if (!fullmm)
133 tlb_batch_add_one(mm, vaddr, pte_exec(orig));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700134}
David Miller9e695d22012-10-08 16:34:29 -0700135
136#ifdef CONFIG_TRANSPARENT_HUGEPAGE
137static void tlb_batch_pmd_scan(struct mm_struct *mm, unsigned long vaddr,
138 pmd_t pmd, bool exec)
139{
140 unsigned long end;
141 pte_t *pte;
142
143 pte = pte_offset_map(&pmd, vaddr);
144 end = vaddr + HPAGE_SIZE;
145 while (vaddr < end) {
146 if (pte_val(*pte) & _PAGE_VALID)
147 tlb_batch_add_one(mm, vaddr, exec);
148 pte++;
149 vaddr += PAGE_SIZE;
150 }
151 pte_unmap(pte);
152}
153
154void set_pmd_at(struct mm_struct *mm, unsigned long addr,
155 pmd_t *pmdp, pmd_t pmd)
156{
157 pmd_t orig = *pmdp;
158
159 *pmdp = pmd;
160
161 if (mm == &init_mm)
162 return;
163
164 if ((pmd_val(pmd) ^ pmd_val(orig)) & PMD_ISHUGE) {
165 if (pmd_val(pmd) & PMD_ISHUGE)
166 mm->context.huge_pte_count++;
167 else
168 mm->context.huge_pte_count--;
David S. Miller0fbebed2013-02-19 22:34:10 -0800169
170 /* Do not try to allocate the TSB hash table if we
171 * don't have one already. We have various locks held
172 * and thus we'll end up doing a GFP_KERNEL allocation
173 * in an atomic context.
174 *
175 * Instead, we let the first TLB miss on a hugepage
176 * take care of this.
177 */
David Miller9e695d22012-10-08 16:34:29 -0700178 }
179
180 if (!pmd_none(orig)) {
181 bool exec = ((pmd_val(orig) & PMD_HUGE_EXEC) != 0);
182
183 addr &= HPAGE_MASK;
184 if (pmd_val(orig) & PMD_ISHUGE)
185 tlb_batch_add_one(mm, addr, exec);
186 else
187 tlb_batch_pmd_scan(mm, addr, orig, exec);
188 }
189}
190
191void pgtable_trans_huge_deposit(struct mm_struct *mm, pgtable_t pgtable)
192{
193 struct list_head *lh = (struct list_head *) pgtable;
194
195 assert_spin_locked(&mm->page_table_lock);
196
197 /* FIFO */
198 if (!mm->pmd_huge_pte)
199 INIT_LIST_HEAD(lh);
200 else
201 list_add(lh, (struct list_head *) mm->pmd_huge_pte);
202 mm->pmd_huge_pte = pgtable;
203}
204
205pgtable_t pgtable_trans_huge_withdraw(struct mm_struct *mm)
206{
207 struct list_head *lh;
208 pgtable_t pgtable;
209
210 assert_spin_locked(&mm->page_table_lock);
211
212 /* FIFO */
213 pgtable = mm->pmd_huge_pte;
214 lh = (struct list_head *) pgtable;
215 if (list_empty(lh))
216 mm->pmd_huge_pte = NULL;
217 else {
218 mm->pmd_huge_pte = (pgtable_t) lh->next;
219 list_del(lh);
220 }
221 pte_val(pgtable[0]) = 0;
222 pte_val(pgtable[1]) = 0;
223
224 return pgtable;
225}
226#endif /* CONFIG_TRANSPARENT_HUGEPAGE */