blob: c814e144a3f0724b1d4ce56a74ca03b70cc82f25 [file] [log] [blame]
Glauber Costac048fdf2008-03-03 14:12:54 -03001#include <linux/init.h>
2
3#include <linux/mm.h>
Glauber Costac048fdf2008-03-03 14:12:54 -03004#include <linux/spinlock.h>
5#include <linux/smp.h>
Glauber Costac048fdf2008-03-03 14:12:54 -03006#include <linux/interrupt.h>
Tejun Heo6dd01be2009-01-21 17:26:06 +09007#include <linux/module.h>
Glauber Costac048fdf2008-03-03 14:12:54 -03008
Glauber Costac048fdf2008-03-03 14:12:54 -03009#include <asm/tlbflush.h>
Glauber Costac048fdf2008-03-03 14:12:54 -030010#include <asm/mmu_context.h>
Tejun Heo6dd01be2009-01-21 17:26:06 +090011#include <asm/apic.h>
Tejun Heobdbcdd42009-01-21 17:26:06 +090012#include <asm/uv/uv.h>
Glauber Costa5af55732008-03-25 13:28:56 -030013
Brian Gerst9eb912d2009-01-19 00:38:57 +090014DEFINE_PER_CPU_SHARED_ALIGNED(struct tlb_state, cpu_tlbstate)
15 = { &init_mm, 0, };
16
Glauber Costac048fdf2008-03-03 14:12:54 -030017/*
18 * Smarter SMP flushing macros.
19 * c/o Linus Torvalds.
20 *
21 * These mean you can really definitely utterly forget about
22 * writing to user space from interrupts. (Its not allowed anyway).
23 *
24 * Optimizations Manfred Spraul <manfred@colorfullife.com>
25 *
26 * More scalable flush, from Andi Kleen
27 *
28 * To avoid global state use 8 different call vectors.
29 * Each CPU uses a specific vector to trigger flushes on other
30 * CPUs. Depending on the received vector the target CPUs look into
Frederik Deweerdt09b3ec72009-01-12 22:35:42 +010031 * the right array slot for the flush data.
Glauber Costac048fdf2008-03-03 14:12:54 -030032 *
33 * With more than 8 CPUs they are hashed to the 8 available
34 * vectors. The limited global vector space forces us to this right now.
35 * In future when interrupts are split into per CPU domains this could be
36 * fixed, at the cost of triggering multiple IPIs in some cases.
37 */
38
39union smp_flush_state {
40 struct {
Glauber Costac048fdf2008-03-03 14:12:54 -030041 struct mm_struct *flush_mm;
42 unsigned long flush_va;
43 spinlock_t tlbstate_lock;
Rusty Russell4595f962009-01-10 21:58:09 -080044 DECLARE_BITMAP(flush_cpumask, NR_CPUS);
Glauber Costac048fdf2008-03-03 14:12:54 -030045 };
Frederik Deweerdt09b3ec72009-01-12 22:35:42 +010046 char pad[CONFIG_X86_INTERNODE_CACHE_BYTES];
47} ____cacheline_internodealigned_in_smp;
Glauber Costac048fdf2008-03-03 14:12:54 -030048
49/* State is put into the per CPU data section, but padded
50 to a full cache line because other CPUs can access it and we don't
51 want false sharing in the per cpu data segment. */
Frederik Deweerdt09b3ec72009-01-12 22:35:42 +010052static union smp_flush_state flush_state[NUM_INVALIDATE_TLB_VECTORS];
Glauber Costac048fdf2008-03-03 14:12:54 -030053
54/*
55 * We cannot call mmdrop() because we are in interrupt context,
56 * instead update mm->cpu_vm_mask.
57 */
58void leave_mm(int cpu)
59{
Brian Gerst9eb912d2009-01-19 00:38:57 +090060 if (percpu_read(cpu_tlbstate.state) == TLBSTATE_OK)
Glauber Costac048fdf2008-03-03 14:12:54 -030061 BUG();
Brian Gerst9eb912d2009-01-19 00:38:57 +090062 cpu_clear(cpu, percpu_read(cpu_tlbstate.active_mm)->cpu_vm_mask);
Glauber Costac048fdf2008-03-03 14:12:54 -030063 load_cr3(swapper_pg_dir);
64}
65EXPORT_SYMBOL_GPL(leave_mm);
66
67/*
68 *
69 * The flush IPI assumes that a thread switch happens in this order:
70 * [cpu0: the cpu that switches]
71 * 1) switch_mm() either 1a) or 1b)
72 * 1a) thread switch to a different mm
73 * 1a1) cpu_clear(cpu, old_mm->cpu_vm_mask);
74 * Stop ipi delivery for the old mm. This is not synchronized with
75 * the other cpus, but smp_invalidate_interrupt ignore flush ipis
76 * for the wrong mm, and in the worst case we perform a superfluous
77 * tlb flush.
78 * 1a2) set cpu mmu_state to TLBSTATE_OK
79 * Now the smp_invalidate_interrupt won't call leave_mm if cpu0
80 * was in lazy tlb mode.
81 * 1a3) update cpu active_mm
82 * Now cpu0 accepts tlb flushes for the new mm.
83 * 1a4) cpu_set(cpu, new_mm->cpu_vm_mask);
84 * Now the other cpus will send tlb flush ipis.
85 * 1a4) change cr3.
86 * 1b) thread switch without mm change
87 * cpu active_mm is correct, cpu0 already handles
88 * flush ipis.
89 * 1b1) set cpu mmu_state to TLBSTATE_OK
90 * 1b2) test_and_set the cpu bit in cpu_vm_mask.
91 * Atomically set the bit [other cpus will start sending flush ipis],
92 * and test the bit.
93 * 1b3) if the bit was 0: leave_mm was called, flush the tlb.
94 * 2) switch %%esp, ie current
95 *
96 * The interrupt must handle 2 special cases:
97 * - cr3 is changed before %%esp, ie. it cannot use current->{active_,}mm.
98 * - the cpu performs speculative tlb reads, i.e. even if the cpu only
99 * runs in kernel space, the cpu could load tlb entries for user space
100 * pages.
101 *
102 * The good news is that cpu mmu_state is local to each cpu, no
103 * write/read ordering problems.
104 */
105
106/*
107 * TLB flush IPI:
108 *
109 * 1) Flush the tlb entries if the cpu uses the mm that's being flushed.
110 * 2) Leave the mm if we are in the lazy tlb mode.
111 *
112 * Interrupts are disabled.
113 */
114
Tejun Heo02cf94c2009-01-21 17:26:06 +0900115/*
116 * FIXME: use of asmlinkage is not consistent. On x86_64 it's noop
117 * but still used for documentation purpose but the usage is slightly
118 * inconsistent. On x86_32, asmlinkage is regparm(0) but interrupt
119 * entry calls in with the first parameter in %eax. Maybe define
120 * intrlinkage?
121 */
122#ifdef CONFIG_X86_64
123asmlinkage
124#endif
125void smp_invalidate_interrupt(struct pt_regs *regs)
Glauber Costac048fdf2008-03-03 14:12:54 -0300126{
Tejun Heo6dd01be2009-01-21 17:26:06 +0900127 unsigned int cpu;
128 unsigned int sender;
Glauber Costac048fdf2008-03-03 14:12:54 -0300129 union smp_flush_state *f;
130
131 cpu = smp_processor_id();
132 /*
133 * orig_rax contains the negated interrupt vector.
134 * Use that to determine where the sender put the data.
135 */
136 sender = ~regs->orig_ax - INVALIDATE_TLB_VECTOR_START;
Frederik Deweerdt09b3ec72009-01-12 22:35:42 +0100137 f = &flush_state[sender];
Glauber Costac048fdf2008-03-03 14:12:54 -0300138
Rusty Russell4595f962009-01-10 21:58:09 -0800139 if (!cpumask_test_cpu(cpu, to_cpumask(f->flush_cpumask)))
Glauber Costac048fdf2008-03-03 14:12:54 -0300140 goto out;
141 /*
142 * This was a BUG() but until someone can quote me the
143 * line from the intel manual that guarantees an IPI to
144 * multiple CPUs is retried _only_ on the erroring CPUs
145 * its staying as a return
146 *
147 * BUG();
148 */
149
Brian Gerst9eb912d2009-01-19 00:38:57 +0900150 if (f->flush_mm == percpu_read(cpu_tlbstate.active_mm)) {
151 if (percpu_read(cpu_tlbstate.state) == TLBSTATE_OK) {
Glauber Costac048fdf2008-03-03 14:12:54 -0300152 if (f->flush_va == TLB_FLUSH_ALL)
153 local_flush_tlb();
154 else
155 __flush_tlb_one(f->flush_va);
156 } else
157 leave_mm(cpu);
158 }
159out:
160 ack_APIC_irq();
Tejun Heo6dd01be2009-01-21 17:26:06 +0900161 smp_mb__before_clear_bit();
Rusty Russell4595f962009-01-10 21:58:09 -0800162 cpumask_clear_cpu(cpu, to_cpumask(f->flush_cpumask));
Tejun Heo6dd01be2009-01-21 17:26:06 +0900163 smp_mb__after_clear_bit();
Hiroshi Shimamoto8ae93662008-12-12 15:52:26 -0800164 inc_irq_stat(irq_tlb_count);
Glauber Costac048fdf2008-03-03 14:12:54 -0300165}
166
Rusty Russell4595f962009-01-10 21:58:09 -0800167static void flush_tlb_others_ipi(const struct cpumask *cpumask,
168 struct mm_struct *mm, unsigned long va)
Glauber Costac048fdf2008-03-03 14:12:54 -0300169{
Tejun Heo6dd01be2009-01-21 17:26:06 +0900170 unsigned int sender;
Glauber Costac048fdf2008-03-03 14:12:54 -0300171 union smp_flush_state *f;
Cliff Wickman18129242008-06-02 08:56:14 -0500172
Glauber Costac048fdf2008-03-03 14:12:54 -0300173 /* Caller has disabled preemption */
174 sender = smp_processor_id() % NUM_INVALIDATE_TLB_VECTORS;
Frederik Deweerdt09b3ec72009-01-12 22:35:42 +0100175 f = &flush_state[sender];
Glauber Costac048fdf2008-03-03 14:12:54 -0300176
177 /*
178 * Could avoid this lock when
179 * num_online_cpus() <= NUM_INVALIDATE_TLB_VECTORS, but it is
180 * probably not worth checking this for a cache-hot lock.
181 */
182 spin_lock(&f->tlbstate_lock);
183
184 f->flush_mm = mm;
185 f->flush_va = va;
Linus Torvaldsb04e6372009-08-21 09:48:10 -0700186 if (cpumask_andnot(to_cpumask(f->flush_cpumask), cpumask, cpumask_of(smp_processor_id()))) {
187 /*
188 * We have to send the IPI only to
189 * CPUs affected.
190 */
191 apic->send_IPI_mask(to_cpumask(f->flush_cpumask),
192 INVALIDATE_TLB_VECTOR_START + sender);
Glauber Costac048fdf2008-03-03 14:12:54 -0300193
Linus Torvaldsb04e6372009-08-21 09:48:10 -0700194 while (!cpumask_empty(to_cpumask(f->flush_cpumask)))
195 cpu_relax();
196 }
Glauber Costac048fdf2008-03-03 14:12:54 -0300197
198 f->flush_mm = NULL;
199 f->flush_va = 0;
200 spin_unlock(&f->tlbstate_lock);
201}
202
Rusty Russell4595f962009-01-10 21:58:09 -0800203void native_flush_tlb_others(const struct cpumask *cpumask,
204 struct mm_struct *mm, unsigned long va)
205{
206 if (is_uv_system()) {
Tejun Heobdbcdd42009-01-21 17:26:06 +0900207 unsigned int cpu;
Rusty Russell4595f962009-01-10 21:58:09 -0800208
Tejun Heobdbcdd42009-01-21 17:26:06 +0900209 cpu = get_cpu();
210 cpumask = uv_flush_tlb_others(cpumask, mm, va, cpu);
211 if (cpumask)
212 flush_tlb_others_ipi(cpumask, mm, va);
213 put_cpu();
Mike Travis0e219902009-01-10 21:58:10 -0800214 return;
Rusty Russell4595f962009-01-10 21:58:09 -0800215 }
216 flush_tlb_others_ipi(cpumask, mm, va);
217}
218
Ingo Molnara4928cf2008-04-23 13:20:56 +0200219static int __cpuinit init_smp_flush(void)
Glauber Costac048fdf2008-03-03 14:12:54 -0300220{
221 int i;
222
Frederik Deweerdt09b3ec72009-01-12 22:35:42 +0100223 for (i = 0; i < ARRAY_SIZE(flush_state); i++)
224 spin_lock_init(&flush_state[i].tlbstate_lock);
Akinobu Mita7c04e642008-04-19 23:55:17 +0900225
Glauber Costac048fdf2008-03-03 14:12:54 -0300226 return 0;
227}
228core_initcall(init_smp_flush);
229
230void flush_tlb_current_task(void)
231{
232 struct mm_struct *mm = current->mm;
Glauber Costac048fdf2008-03-03 14:12:54 -0300233
234 preempt_disable();
Glauber Costac048fdf2008-03-03 14:12:54 -0300235
236 local_flush_tlb();
Rusty Russell4595f962009-01-10 21:58:09 -0800237 if (cpumask_any_but(&mm->cpu_vm_mask, smp_processor_id()) < nr_cpu_ids)
238 flush_tlb_others(&mm->cpu_vm_mask, mm, TLB_FLUSH_ALL);
Glauber Costac048fdf2008-03-03 14:12:54 -0300239 preempt_enable();
240}
241
242void flush_tlb_mm(struct mm_struct *mm)
243{
Glauber Costac048fdf2008-03-03 14:12:54 -0300244 preempt_disable();
Glauber Costac048fdf2008-03-03 14:12:54 -0300245
246 if (current->active_mm == mm) {
247 if (current->mm)
248 local_flush_tlb();
249 else
250 leave_mm(smp_processor_id());
251 }
Rusty Russell4595f962009-01-10 21:58:09 -0800252 if (cpumask_any_but(&mm->cpu_vm_mask, smp_processor_id()) < nr_cpu_ids)
253 flush_tlb_others(&mm->cpu_vm_mask, mm, TLB_FLUSH_ALL);
Glauber Costac048fdf2008-03-03 14:12:54 -0300254
255 preempt_enable();
256}
257
258void flush_tlb_page(struct vm_area_struct *vma, unsigned long va)
259{
260 struct mm_struct *mm = vma->vm_mm;
Glauber Costac048fdf2008-03-03 14:12:54 -0300261
262 preempt_disable();
Glauber Costac048fdf2008-03-03 14:12:54 -0300263
264 if (current->active_mm == mm) {
265 if (current->mm)
266 __flush_tlb_one(va);
267 else
268 leave_mm(smp_processor_id());
269 }
270
Rusty Russell4595f962009-01-10 21:58:09 -0800271 if (cpumask_any_but(&mm->cpu_vm_mask, smp_processor_id()) < nr_cpu_ids)
272 flush_tlb_others(&mm->cpu_vm_mask, mm, va);
Glauber Costac048fdf2008-03-03 14:12:54 -0300273
274 preempt_enable();
275}
276
277static void do_flush_tlb_all(void *info)
278{
279 unsigned long cpu = smp_processor_id();
280
281 __flush_tlb_all();
Brian Gerst9eb912d2009-01-19 00:38:57 +0900282 if (percpu_read(cpu_tlbstate.state) == TLBSTATE_LAZY)
Glauber Costac048fdf2008-03-03 14:12:54 -0300283 leave_mm(cpu);
284}
285
286void flush_tlb_all(void)
287{
Jens Axboe15c8b6c2008-05-09 09:39:44 +0200288 on_each_cpu(do_flush_tlb_all, NULL, 1);
Glauber Costac048fdf2008-03-03 14:12:54 -0300289}