blob: be4f34c30a0b14c955377a5c6002fdb3e5b7a74e [file] [log] [blame]
Benjamin Herrenschmidt5e696612008-12-18 19:13:24 +00001/*
2 * This file contains the routines for handling the MMU on those
3 * PowerPC implementations where the MMU is not using the hash
4 * table, such as 8xx, 4xx, BookE's etc...
5 *
6 * Copyright 2008 Ben Herrenschmidt <benh@kernel.crashing.org>
7 * IBM Corp.
8 *
9 * Derived from previous arch/powerpc/mm/mmu_context.c
10 * and arch/powerpc/include/asm/mmu_context.h
11 *
12 * This program is free software; you can redistribute it and/or
13 * modify it under the terms of the GNU General Public License
14 * as published by the Free Software Foundation; either version
15 * 2 of the License, or (at your option) any later version.
16 *
Benjamin Herrenschmidt2ca8cf72008-12-18 19:13:29 +000017 * TODO:
18 *
19 * - The global context lock will not scale very well
20 * - The maps should be dynamically allocated to allow for processors
21 * that support more PID bits at runtime
22 * - Implement flush_tlb_mm() by making the context stale and picking
23 * a new one
24 * - More aggressively clear stale map bits and maybe find some way to
25 * also clear mm->cpu_vm_mask bits when processes are migrated
Benjamin Herrenschmidt5e696612008-12-18 19:13:24 +000026 */
27
Benjamin Herrenschmidtf1167fb2009-11-04 13:39:52 +000028//#define DEBUG_MAP_CONSISTENCY
29//#define DEBUG_CLAMP_LAST_CONTEXT 31
Benjamin Herrenschmidtfcce8102009-07-23 23:15:10 +000030//#define DEBUG_HARDER
31
32/* We don't use DEBUG because it tends to be compiled in always nowadays
33 * and this would generate way too much output
34 */
35#ifdef DEBUG_HARDER
36#define pr_hard(args...) printk(KERN_DEBUG args)
37#define pr_hardcont(args...) printk(KERN_CONT args)
38#else
39#define pr_hard(args...) do { } while(0)
40#define pr_hardcont(args...) do { } while(0)
41#endif
Benjamin Herrenschmidt2ca8cf72008-12-18 19:13:29 +000042
43#include <linux/kernel.h>
Benjamin Herrenschmidt5e696612008-12-18 19:13:24 +000044#include <linux/mm.h>
45#include <linux/init.h>
Benjamin Herrenschmidt77520352008-12-18 19:13:48 +000046#include <linux/spinlock.h>
47#include <linux/bootmem.h>
48#include <linux/notifier.h>
49#include <linux/cpu.h>
Benjamin Herrenschmidt5e696612008-12-18 19:13:24 +000050
51#include <asm/mmu_context.h>
52#include <asm/tlbflush.h>
53
Benjamin Herrenschmidt77520352008-12-18 19:13:48 +000054static unsigned int first_context, last_context;
Benjamin Herrenschmidt2ca8cf72008-12-18 19:13:29 +000055static unsigned int next_context, nr_free_contexts;
Benjamin Herrenschmidt77520352008-12-18 19:13:48 +000056static unsigned long *context_map;
57static unsigned long *stale_map[NR_CPUS];
58static struct mm_struct **context_mm;
Benjamin Herrenschmidtb46b6942009-06-02 18:53:37 +000059static DEFINE_SPINLOCK(context_lock);
Benjamin Herrenschmidt5e696612008-12-18 19:13:24 +000060
Benjamin Herrenschmidt77520352008-12-18 19:13:48 +000061#define CTX_MAP_SIZE \
62 (sizeof(unsigned long) * (last_context / BITS_PER_LONG + 1))
63
64
Benjamin Herrenschmidt5e696612008-12-18 19:13:24 +000065/* Steal a context from a task that has one at the moment.
Benjamin Herrenschmidt2ca8cf72008-12-18 19:13:29 +000066 *
67 * This is used when we are running out of available PID numbers
68 * on the processors.
69 *
Benjamin Herrenschmidt5e696612008-12-18 19:13:24 +000070 * This isn't an LRU system, it just frees up each context in
71 * turn (sort-of pseudo-random replacement :). This would be the
72 * place to implement an LRU scheme if anyone was motivated to do it.
73 * -- paulus
Benjamin Herrenschmidt2ca8cf72008-12-18 19:13:29 +000074 *
75 * For context stealing, we use a slightly different approach for
76 * SMP and UP. Basically, the UP one is simpler and doesn't use
77 * the stale map as we can just flush the local CPU
78 * -- benh
Benjamin Herrenschmidt5e696612008-12-18 19:13:24 +000079 */
Benjamin Herrenschmidt2ca8cf72008-12-18 19:13:29 +000080#ifdef CONFIG_SMP
81static unsigned int steal_context_smp(unsigned int id)
Benjamin Herrenschmidt5e696612008-12-18 19:13:24 +000082{
83 struct mm_struct *mm;
Benjamin Herrenschmidtfcce8102009-07-23 23:15:10 +000084 unsigned int cpu, max, i;
Benjamin Herrenschmidt5e696612008-12-18 19:13:24 +000085
Benjamin Herrenschmidt77520352008-12-18 19:13:48 +000086 max = last_context - first_context;
Benjamin Herrenschmidt5e696612008-12-18 19:13:24 +000087
Benjamin Herrenschmidt2ca8cf72008-12-18 19:13:29 +000088 /* Attempt to free next_context first and then loop until we manage */
89 while (max--) {
90 /* Pick up the victim mm */
91 mm = context_mm[id];
Benjamin Herrenschmidt5e696612008-12-18 19:13:24 +000092
Benjamin Herrenschmidt2ca8cf72008-12-18 19:13:29 +000093 /* We have a candidate victim, check if it's active, on SMP
94 * we cannot steal active contexts
95 */
96 if (mm->context.active) {
97 id++;
Benjamin Herrenschmidt77520352008-12-18 19:13:48 +000098 if (id > last_context)
99 id = first_context;
Benjamin Herrenschmidt2ca8cf72008-12-18 19:13:29 +0000100 continue;
101 }
Benjamin Herrenschmidtfcce8102009-07-23 23:15:10 +0000102 pr_hardcont(" | steal %d from 0x%p", id, mm);
Benjamin Herrenschmidt5e696612008-12-18 19:13:24 +0000103
Benjamin Herrenschmidt2ca8cf72008-12-18 19:13:29 +0000104 /* Mark this mm has having no context anymore */
105 mm->context.id = MMU_NO_CONTEXT;
Benjamin Herrenschmidt5e696612008-12-18 19:13:24 +0000106
Benjamin Herrenschmidtfcce8102009-07-23 23:15:10 +0000107 /* Mark it stale on all CPUs that used this mm. For threaded
108 * implementations, we set it on all threads on each core
109 * represented in the mask. A future implementation will use
110 * a core map instead but this will do for now.
111 */
112 for_each_cpu(cpu, mm_cpumask(mm)) {
113 for (i = cpu_first_thread_in_core(cpu);
114 i <= cpu_last_thread_in_core(cpu); i++)
115 __set_bit(id, stale_map[i]);
116 cpu = i - 1;
117 }
Benjamin Herrenschmidt2ca8cf72008-12-18 19:13:29 +0000118 return id;
Benjamin Herrenschmidt5e696612008-12-18 19:13:24 +0000119 }
Benjamin Herrenschmidt2ca8cf72008-12-18 19:13:29 +0000120
121 /* This will happen if you have more CPUs than available contexts,
122 * all we can do here is wait a bit and try again
123 */
124 spin_unlock(&context_lock);
125 cpu_relax();
126 spin_lock(&context_lock);
Benjamin Herrenschmidt3035c862009-05-19 16:56:42 +0000127
128 /* This will cause the caller to try again */
129 return MMU_NO_CONTEXT;
Benjamin Herrenschmidt5e696612008-12-18 19:13:24 +0000130}
Benjamin Herrenschmidt2ca8cf72008-12-18 19:13:29 +0000131#endif /* CONFIG_SMP */
132
133/* Note that this will also be called on SMP if all other CPUs are
134 * offlined, which means that it may be called for cpu != 0. For
135 * this to work, we somewhat assume that CPUs that are onlined
136 * come up with a fully clean TLB (or are cleaned when offlined)
137 */
138static unsigned int steal_context_up(unsigned int id)
139{
140 struct mm_struct *mm;
141 int cpu = smp_processor_id();
142
143 /* Pick up the victim mm */
144 mm = context_mm[id];
145
Benjamin Herrenschmidtfcce8102009-07-23 23:15:10 +0000146 pr_hardcont(" | steal %d from 0x%p", id, mm);
Benjamin Herrenschmidt2ca8cf72008-12-18 19:13:29 +0000147
Benjamin Herrenschmidt2ca8cf72008-12-18 19:13:29 +0000148 /* Flush the TLB for that context */
149 local_flush_tlb_mm(mm);
150
Hideo Saito8e359612009-05-24 15:33:34 +0000151 /* Mark this mm has having no context anymore */
152 mm->context.id = MMU_NO_CONTEXT;
153
Benjamin Herrenschmidt2ca8cf72008-12-18 19:13:29 +0000154 /* XXX This clear should ultimately be part of local_flush_tlb_mm */
155 __clear_bit(id, stale_map[cpu]);
156
157 return id;
158}
159
160#ifdef DEBUG_MAP_CONSISTENCY
161static void context_check_map(void)
162{
163 unsigned int id, nrf, nact;
164
165 nrf = nact = 0;
Benjamin Herrenschmidt77520352008-12-18 19:13:48 +0000166 for (id = first_context; id <= last_context; id++) {
Benjamin Herrenschmidt2ca8cf72008-12-18 19:13:29 +0000167 int used = test_bit(id, context_map);
168 if (!used)
169 nrf++;
170 if (used != (context_mm[id] != NULL))
171 pr_err("MMU: Context %d is %s and MM is %p !\n",
172 id, used ? "used" : "free", context_mm[id]);
173 if (context_mm[id] != NULL)
174 nact += context_mm[id]->context.active;
175 }
176 if (nrf != nr_free_contexts) {
177 pr_err("MMU: Free context count out of sync ! (%d vs %d)\n",
178 nr_free_contexts, nrf);
179 nr_free_contexts = nrf;
180 }
181 if (nact > num_online_cpus())
182 pr_err("MMU: More active contexts than CPUs ! (%d vs %d)\n",
183 nact, num_online_cpus());
Benjamin Herrenschmidt77520352008-12-18 19:13:48 +0000184 if (first_context > 0 && !test_bit(0, context_map))
185 pr_err("MMU: Context 0 has been freed !!!\n");
Benjamin Herrenschmidt2ca8cf72008-12-18 19:13:29 +0000186}
187#else
188static void context_check_map(void) { }
189#endif
Benjamin Herrenschmidt5e696612008-12-18 19:13:24 +0000190
191void switch_mmu_context(struct mm_struct *prev, struct mm_struct *next)
192{
Kumar Gala67050b52009-08-04 22:33:32 -0500193 unsigned int i, id, cpu = smp_processor_id();
Benjamin Herrenschmidt2ca8cf72008-12-18 19:13:29 +0000194 unsigned long *map;
Benjamin Herrenschmidt5e696612008-12-18 19:13:24 +0000195
Benjamin Herrenschmidt2ca8cf72008-12-18 19:13:29 +0000196 /* No lockless fast path .. yet */
197 spin_lock(&context_lock);
198
Benjamin Herrenschmidtfcce8102009-07-23 23:15:10 +0000199 pr_hard("[%d] activating context for mm @%p, active=%d, id=%d",
200 cpu, next, next->context.active, next->context.id);
Benjamin Herrenschmidt2ca8cf72008-12-18 19:13:29 +0000201
202#ifdef CONFIG_SMP
203 /* Mark us active and the previous one not anymore */
204 next->context.active++;
205 if (prev) {
Benjamin Herrenschmidtfcce8102009-07-23 23:15:10 +0000206 pr_hardcont(" (old=0x%p a=%d)", prev, prev->context.active);
Benjamin Herrenschmidt2ca8cf72008-12-18 19:13:29 +0000207 WARN_ON(prev->context.active < 1);
208 prev->context.active--;
209 }
Benjamin Herrenschmidt3035c862009-05-19 16:56:42 +0000210
211 again:
Benjamin Herrenschmidt2ca8cf72008-12-18 19:13:29 +0000212#endif /* CONFIG_SMP */
213
214 /* If we already have a valid assigned context, skip all that */
215 id = next->context.id;
Benjamin Herrenschmidtfcce8102009-07-23 23:15:10 +0000216 if (likely(id != MMU_NO_CONTEXT)) {
217#ifdef DEBUG_MAP_CONSISTENCY
218 if (context_mm[id] != next)
219 pr_err("MMU: mm 0x%p has id %d but context_mm[%d] says 0x%p\n",
220 next, id, id, context_mm[id]);
221#endif
Benjamin Herrenschmidt2ca8cf72008-12-18 19:13:29 +0000222 goto ctxt_ok;
Benjamin Herrenschmidtfcce8102009-07-23 23:15:10 +0000223 }
Benjamin Herrenschmidt2ca8cf72008-12-18 19:13:29 +0000224
225 /* We really don't have a context, let's try to acquire one */
226 id = next_context;
Benjamin Herrenschmidt77520352008-12-18 19:13:48 +0000227 if (id > last_context)
228 id = first_context;
Benjamin Herrenschmidt2ca8cf72008-12-18 19:13:29 +0000229 map = context_map;
230
231 /* No more free contexts, let's try to steal one */
232 if (nr_free_contexts == 0) {
233#ifdef CONFIG_SMP
234 if (num_online_cpus() > 1) {
235 id = steal_context_smp(id);
Benjamin Herrenschmidt3035c862009-05-19 16:56:42 +0000236 if (id == MMU_NO_CONTEXT)
237 goto again;
Kumar Gala5156ddc2009-07-29 23:04:25 -0500238 goto stolen;
Benjamin Herrenschmidt2ca8cf72008-12-18 19:13:29 +0000239 }
240#endif /* CONFIG_SMP */
241 id = steal_context_up(id);
242 goto stolen;
243 }
244 nr_free_contexts--;
245
246 /* We know there's at least one free context, try to find it */
247 while (__test_and_set_bit(id, map)) {
Benjamin Herrenschmidt77520352008-12-18 19:13:48 +0000248 id = find_next_zero_bit(map, last_context+1, id);
249 if (id > last_context)
250 id = first_context;
Benjamin Herrenschmidt2ca8cf72008-12-18 19:13:29 +0000251 }
252 stolen:
253 next_context = id + 1;
254 context_mm[id] = next;
255 next->context.id = id;
Benjamin Herrenschmidtfcce8102009-07-23 23:15:10 +0000256 pr_hardcont(" | new id=%d,nrf=%d", id, nr_free_contexts);
Benjamin Herrenschmidt2ca8cf72008-12-18 19:13:29 +0000257
258 context_check_map();
259 ctxt_ok:
260
261 /* If that context got marked stale on this CPU, then flush the
262 * local TLB for it and unmark it before we use it
263 */
264 if (test_bit(id, stale_map[cpu])) {
Benjamin Herrenschmidtfcce8102009-07-23 23:15:10 +0000265 pr_hardcont(" | stale flush %d [%d..%d]",
266 id, cpu_first_thread_in_core(cpu),
267 cpu_last_thread_in_core(cpu));
268
Benjamin Herrenschmidt2ca8cf72008-12-18 19:13:29 +0000269 local_flush_tlb_mm(next);
270
271 /* XXX This clear should ultimately be part of local_flush_tlb_mm */
Kumar Gala67050b52009-08-04 22:33:32 -0500272 for (i = cpu_first_thread_in_core(cpu);
273 i <= cpu_last_thread_in_core(cpu); i++) {
274 __clear_bit(id, stale_map[i]);
275 }
Benjamin Herrenschmidt2ca8cf72008-12-18 19:13:29 +0000276 }
277
278 /* Flick the MMU and release lock */
Benjamin Herrenschmidtfcce8102009-07-23 23:15:10 +0000279 pr_hardcont(" -> %d\n", id);
Benjamin Herrenschmidt2ca8cf72008-12-18 19:13:29 +0000280 set_context(id, next->pgd);
281 spin_unlock(&context_lock);
Benjamin Herrenschmidt5e696612008-12-18 19:13:24 +0000282}
283
284/*
285 * Set up the context for a new address space.
286 */
287int init_new_context(struct task_struct *t, struct mm_struct *mm)
288{
Benjamin Herrenschmidtfcce8102009-07-23 23:15:10 +0000289 pr_hard("initing context for mm @%p\n", mm);
290
Benjamin Herrenschmidt2ca8cf72008-12-18 19:13:29 +0000291 mm->context.id = MMU_NO_CONTEXT;
292 mm->context.active = 0;
293
Benjamin Herrenschmidt5e696612008-12-18 19:13:24 +0000294 return 0;
295}
296
297/*
298 * We're finished using the context for an address space.
299 */
300void destroy_context(struct mm_struct *mm)
301{
Benjamin Herrenschmidtb46b6942009-06-02 18:53:37 +0000302 unsigned long flags;
Benjamin Herrenschmidt2ca8cf72008-12-18 19:13:29 +0000303 unsigned int id;
304
305 if (mm->context.id == MMU_NO_CONTEXT)
306 return;
307
308 WARN_ON(mm->context.active != 0);
309
Benjamin Herrenschmidtb46b6942009-06-02 18:53:37 +0000310 spin_lock_irqsave(&context_lock, flags);
Benjamin Herrenschmidt2ca8cf72008-12-18 19:13:29 +0000311 id = mm->context.id;
312 if (id != MMU_NO_CONTEXT) {
313 __clear_bit(id, context_map);
314 mm->context.id = MMU_NO_CONTEXT;
315#ifdef DEBUG_MAP_CONSISTENCY
316 mm->context.active = 0;
Benjamin Herrenschmidt2ca8cf72008-12-18 19:13:29 +0000317#endif
Benjamin Herrenschmidt3035c862009-05-19 16:56:42 +0000318 context_mm[id] = NULL;
Benjamin Herrenschmidt2ca8cf72008-12-18 19:13:29 +0000319 nr_free_contexts++;
Benjamin Herrenschmidt5e696612008-12-18 19:13:24 +0000320 }
Benjamin Herrenschmidtb46b6942009-06-02 18:53:37 +0000321 spin_unlock_irqrestore(&context_lock, flags);
Benjamin Herrenschmidt5e696612008-12-18 19:13:24 +0000322}
323
Benjamin Herrenschmidt77520352008-12-18 19:13:48 +0000324#ifdef CONFIG_SMP
325
326static int __cpuinit mmu_context_cpu_notify(struct notifier_block *self,
327 unsigned long action, void *hcpu)
328{
329 unsigned int cpu = (unsigned int)(long)hcpu;
Benjamin Herrenschmidtfcce8102009-07-23 23:15:10 +0000330#ifdef CONFIG_HOTPLUG_CPU
331 struct task_struct *p;
332#endif
Benjamin Herrenschmidt77520352008-12-18 19:13:48 +0000333 /* We don't touch CPU 0 map, it's allocated at aboot and kept
334 * around forever
335 */
336 if (cpu == 0)
337 return NOTIFY_OK;
338
339 switch (action) {
340 case CPU_ONLINE:
341 case CPU_ONLINE_FROZEN:
Michael Ellermana1ac38a2009-06-17 18:13:54 +0000342 pr_devel("MMU: Allocating stale context map for CPU %d\n", cpu);
Benjamin Herrenschmidt77520352008-12-18 19:13:48 +0000343 stale_map[cpu] = kzalloc(CTX_MAP_SIZE, GFP_KERNEL);
344 break;
345#ifdef CONFIG_HOTPLUG_CPU
346 case CPU_DEAD:
347 case CPU_DEAD_FROZEN:
Michael Ellermana1ac38a2009-06-17 18:13:54 +0000348 pr_devel("MMU: Freeing stale context map for CPU %d\n", cpu);
Benjamin Herrenschmidt77520352008-12-18 19:13:48 +0000349 kfree(stale_map[cpu]);
350 stale_map[cpu] = NULL;
Benjamin Herrenschmidtfcce8102009-07-23 23:15:10 +0000351
352 /* We also clear the cpu_vm_mask bits of CPUs going away */
353 read_lock(&tasklist_lock);
354 for_each_process(p) {
355 if (p->mm)
356 cpu_mask_clear_cpu(cpu, mm_cpumask(p->mm));
357 }
358 read_unlock(&tasklist_lock);
359 break;
360#endif /* CONFIG_HOTPLUG_CPU */
Benjamin Herrenschmidt77520352008-12-18 19:13:48 +0000361 }
362 return NOTIFY_OK;
363}
364
365static struct notifier_block __cpuinitdata mmu_context_cpu_nb = {
366 .notifier_call = mmu_context_cpu_notify,
367};
368
369#endif /* CONFIG_SMP */
Benjamin Herrenschmidt5e696612008-12-18 19:13:24 +0000370
371/*
372 * Initialize the context management stuff.
373 */
374void __init mmu_context_init(void)
375{
Benjamin Herrenschmidt2ca8cf72008-12-18 19:13:29 +0000376 /* Mark init_mm as being active on all possible CPUs since
377 * we'll get called with prev == init_mm the first time
378 * we schedule on a given CPU
379 */
380 init_mm.context.active = NR_CPUS;
381
Benjamin Herrenschmidt5e696612008-12-18 19:13:24 +0000382 /*
Benjamin Herrenschmidt77520352008-12-18 19:13:48 +0000383 * The MPC8xx has only 16 contexts. We rotate through them on each
384 * task switch. A better way would be to keep track of tasks that
385 * own contexts, and implement an LRU usage. That way very active
386 * tasks don't always have to pay the TLB reload overhead. The
387 * kernel pages are mapped shared, so the kernel can run on behalf
388 * of any task that makes a kernel entry. Shared does not mean they
389 * are not protected, just that the ASID comparison is not performed.
390 * -- Dan
391 *
392 * The IBM4xx has 256 contexts, so we can just rotate through these
393 * as a way of "switching" contexts. If the TID of the TLB is zero,
394 * the PID/TID comparison is disabled, so we can use a TID of zero
395 * to represent all kernel pages as shared among all contexts.
396 * -- Dan
397 */
398 if (mmu_has_feature(MMU_FTR_TYPE_8xx)) {
399 first_context = 0;
400 last_context = 15;
401 } else {
402 first_context = 1;
403 last_context = 255;
404 }
405
406#ifdef DEBUG_CLAMP_LAST_CONTEXT
407 last_context = DEBUG_CLAMP_LAST_CONTEXT;
408#endif
409 /*
410 * Allocate the maps used by context management
411 */
412 context_map = alloc_bootmem(CTX_MAP_SIZE);
413 context_mm = alloc_bootmem(sizeof(void *) * (last_context + 1));
414 stale_map[0] = alloc_bootmem(CTX_MAP_SIZE);
415
416#ifdef CONFIG_SMP
417 register_cpu_notifier(&mmu_context_cpu_nb);
418#endif
419
420 printk(KERN_INFO
Benjamin Herrenschmidtff7c6602009-03-19 19:34:13 +0000421 "MMU: Allocated %zu bytes of context maps for %d contexts\n",
Benjamin Herrenschmidt77520352008-12-18 19:13:48 +0000422 2 * CTX_MAP_SIZE + (sizeof(void *) * (last_context + 1)),
423 last_context - first_context + 1);
424
425 /*
Benjamin Herrenschmidt5e696612008-12-18 19:13:24 +0000426 * Some processors have too few contexts to reserve one for
427 * init_mm, and require using context 0 for a normal task.
428 * Other processors reserve the use of context zero for the kernel.
Benjamin Herrenschmidt77520352008-12-18 19:13:48 +0000429 * This code assumes first_context < 32.
Benjamin Herrenschmidt5e696612008-12-18 19:13:24 +0000430 */
Benjamin Herrenschmidt77520352008-12-18 19:13:48 +0000431 context_map[0] = (1 << first_context) - 1;
432 next_context = first_context;
433 nr_free_contexts = last_context - first_context + 1;
Benjamin Herrenschmidt5e696612008-12-18 19:13:24 +0000434}
435