blob: c8c8b9ed02e09b57a79bee1ab5db9c337be0d338 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
Russell Kingd84b4712006-08-21 19:23:38 +01002 * linux/arch/arm/mm/context.c
Linus Torvalds1da177e2005-04-16 15:20:36 -07003 *
4 * Copyright (C) 2002-2003 Deep Blue Solutions Ltd, all rights reserved.
Will Deaconb5466f82012-06-15 14:47:31 +01005 * Copyright (C) 2012 ARM Limited
6 *
7 * Author: Will Deacon <will.deacon@arm.com>
Linus Torvalds1da177e2005-04-16 15:20:36 -07008 *
9 * This program is free software; you can redistribute it and/or modify
10 * it under the terms of the GNU General Public License version 2 as
11 * published by the Free Software Foundation.
12 */
13#include <linux/init.h>
14#include <linux/sched.h>
15#include <linux/mm.h>
Catalin Marinas11805bc2010-01-26 19:09:42 +010016#include <linux/smp.h>
17#include <linux/percpu.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070018
19#include <asm/mmu_context.h>
Will Deaconb5466f82012-06-15 14:47:31 +010020#include <asm/smp_plat.h>
Will Deacon575320d2012-07-06 15:43:03 +010021#include <asm/thread_notify.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070022#include <asm/tlbflush.h>
Cyril Chemparathy1fc84ae2012-07-16 17:20:17 -040023#include <asm/proc-fns.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070024
Will Deaconb5466f82012-06-15 14:47:31 +010025/*
26 * On ARMv6, we have the following structure in the Context ID:
27 *
28 * 31 7 0
29 * +-------------------------+-----------+
30 * | process ID | ASID |
31 * +-------------------------+-----------+
32 * | context ID |
33 * +-------------------------------------+
34 *
35 * The ASID is used to tag entries in the CPU caches and TLBs.
36 * The context ID is used by debuggers and trace logic, and
37 * should be unique within all running processes.
Ben Dooks9520a5b2013-02-11 12:25:06 +010038 *
Will Deacon5d497502013-12-17 19:17:54 +010039 * In big endian operation, the two 32 bit words are swapped if accessed
40 * by non-64-bit operations.
Will Deaconb5466f82012-06-15 14:47:31 +010041 */
42#define ASID_FIRST_VERSION (1ULL << ASID_BITS)
Marc Zyngierb8e4a472013-06-21 12:06:55 +010043#define NUM_USER_ASIDS ASID_FIRST_VERSION
Will Deaconb5466f82012-06-15 14:47:31 +010044
Thomas Gleixnerbd31b852009-07-03 08:44:46 -050045static DEFINE_RAW_SPINLOCK(cpu_asid_lock);
Will Deaconbf51bb82012-08-01 14:57:49 +010046static atomic64_t asid_generation = ATOMIC64_INIT(ASID_FIRST_VERSION);
47static DECLARE_BITMAP(asid_map, NUM_USER_ASIDS);
Will Deaconb5466f82012-06-15 14:47:31 +010048
Marc Zyngier0d0752b2013-06-21 12:07:27 +010049static DEFINE_PER_CPU(atomic64_t, active_asids);
Will Deaconb5466f82012-06-15 14:47:31 +010050static DEFINE_PER_CPU(u64, reserved_asids);
51static cpumask_t tlb_flush_pending;
Linus Torvalds1da177e2005-04-16 15:20:36 -070052
Marc Zyngier0d0752b2013-06-21 12:07:27 +010053#ifdef CONFIG_ARM_ERRATA_798181
54void a15_erratum_get_cpumask(int this_cpu, struct mm_struct *mm,
55 cpumask_t *mask)
56{
57 int cpu;
58 unsigned long flags;
59 u64 context_id, asid;
60
61 raw_spin_lock_irqsave(&cpu_asid_lock, flags);
62 context_id = mm->context.id.counter;
63 for_each_online_cpu(cpu) {
64 if (cpu == this_cpu)
65 continue;
66 /*
67 * We only need to send an IPI if the other CPUs are
68 * running the same ASID as the one being invalidated.
69 */
70 asid = per_cpu(active_asids, cpu).counter;
71 if (asid == 0)
72 asid = per_cpu(reserved_asids, cpu);
73 if (context_id == asid)
74 cpumask_set_cpu(cpu, mask);
75 }
76 raw_spin_unlock_irqrestore(&cpu_asid_lock, flags);
77}
78#endif
79
Catalin Marinas14d8c952011-11-22 17:30:31 +000080#ifdef CONFIG_ARM_LPAE
Will Deacone1a58482013-12-17 19:17:11 +010081/*
82 * With LPAE, the ASID and page tables are updated atomicly, so there is
83 * no need for a reserved set of tables (the active ASID tracking prevents
84 * any issues across a rollover).
85 */
86#define cpu_set_reserved_ttbr0()
Catalin Marinas14d8c952011-11-22 17:30:31 +000087#else
Will Deaconb5466f82012-06-15 14:47:31 +010088static void cpu_set_reserved_ttbr0(void)
Will Deacon3c5f7e72011-05-31 15:38:43 +010089{
90 u32 ttb;
Will Deacone1a58482013-12-17 19:17:11 +010091 /*
92 * Copy TTBR1 into TTBR0.
93 * This points at swapper_pg_dir, which contains only global
94 * entries so any speculative walks are perfectly safe.
95 */
Will Deacon3c5f7e72011-05-31 15:38:43 +010096 asm volatile(
97 " mrc p15, 0, %0, c2, c0, 1 @ read TTBR1\n"
98 " mcr p15, 0, %0, c2, c0, 0 @ set TTBR0\n"
99 : "=r" (ttb));
100 isb();
101}
Catalin Marinas14d8c952011-11-22 17:30:31 +0000102#endif
103
Will Deacon575320d2012-07-06 15:43:03 +0100104#ifdef CONFIG_PID_IN_CONTEXTIDR
105static int contextidr_notifier(struct notifier_block *unused, unsigned long cmd,
106 void *t)
107{
108 u32 contextidr;
109 pid_t pid;
110 struct thread_info *thread = t;
111
112 if (cmd != THREAD_NOTIFY_SWITCH)
113 return NOTIFY_DONE;
114
115 pid = task_pid_nr(thread->task) << ASID_BITS;
116 asm volatile(
117 " mrc p15, 0, %0, c13, c0, 1\n"
Will Deaconae3790b2012-08-24 15:21:52 +0100118 " and %0, %0, %2\n"
119 " orr %0, %0, %1\n"
120 " mcr p15, 0, %0, c13, c0, 1\n"
Will Deacon575320d2012-07-06 15:43:03 +0100121 : "=r" (contextidr), "+r" (pid)
Will Deaconae3790b2012-08-24 15:21:52 +0100122 : "I" (~ASID_MASK));
Will Deacon575320d2012-07-06 15:43:03 +0100123 isb();
124
125 return NOTIFY_OK;
126}
127
128static struct notifier_block contextidr_notifier_block = {
129 .notifier_call = contextidr_notifier,
130};
131
132static int __init contextidr_notifier_init(void)
133{
134 return thread_register_notifier(&contextidr_notifier_block);
135}
136arch_initcall(contextidr_notifier_init);
137#endif
138
Will Deaconb5466f82012-06-15 14:47:31 +0100139static void flush_context(unsigned int cpu)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700140{
Will Deaconb5466f82012-06-15 14:47:31 +0100141 int i;
Will Deaconbf51bb82012-08-01 14:57:49 +0100142 u64 asid;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700143
Will Deaconbf51bb82012-08-01 14:57:49 +0100144 /* Update the list of reserved ASIDs and the ASID bitmap. */
145 bitmap_clear(asid_map, 0, NUM_USER_ASIDS);
146 for_each_possible_cpu(i) {
Will Deacon8e648062015-01-29 16:41:46 +0100147 asid = atomic64_xchg(&per_cpu(active_asids, i), 0);
148 /*
149 * If this CPU has already been through a
150 * rollover, but hasn't run another task in
151 * the meantime, we must preserve its reserved
152 * ASID, as this is the only trace we have of
153 * the process it is still running.
154 */
155 if (asid == 0)
156 asid = per_cpu(reserved_asids, i);
157 __set_bit(asid & ~ASID_MASK, asid_map);
Will Deaconbf51bb82012-08-01 14:57:49 +0100158 per_cpu(reserved_asids, i) = asid;
159 }
Will Deaconb5466f82012-06-15 14:47:31 +0100160
161 /* Queue a TLB invalidate and flush the I-cache if necessary. */
Will Deaconf0915782013-02-11 13:47:48 +0000162 cpumask_setall(&tlb_flush_pending);
Will Deaconb5466f82012-06-15 14:47:31 +0100163
164 if (icache_is_vivt_asid_tagged())
Catalin Marinas11805bc2010-01-26 19:09:42 +0100165 __flush_icache_all();
Catalin Marinas11805bc2010-01-26 19:09:42 +0100166}
167
Will Deacon40ee0682015-12-02 14:31:25 +0100168static bool check_update_reserved_asid(u64 asid, u64 newasid)
Catalin Marinas11805bc2010-01-26 19:09:42 +0100169{
Will Deaconb5466f82012-06-15 14:47:31 +0100170 int cpu;
Will Deacon40ee0682015-12-02 14:31:25 +0100171 bool hit = false;
172
173 /*
174 * Iterate over the set of reserved ASIDs looking for a match.
175 * If we find one, then we can update our mm to use newasid
176 * (i.e. the same ASID in the current generation) but we can't
177 * exit the loop early, since we need to ensure that all copies
178 * of the old ASID are updated to reflect the mm. Failure to do
179 * so could result in us missing the reserved ASID in a future
180 * generation.
181 */
182 for_each_possible_cpu(cpu) {
183 if (per_cpu(reserved_asids, cpu) == asid) {
184 hit = true;
185 per_cpu(reserved_asids, cpu) = newasid;
186 }
187 }
188
189 return hit;
Will Deaconb5466f82012-06-15 14:47:31 +0100190}
Catalin Marinas11805bc2010-01-26 19:09:42 +0100191
Will Deacon8a4e3a92013-02-28 17:47:36 +0100192static u64 new_context(struct mm_struct *mm, unsigned int cpu)
Will Deaconb5466f82012-06-15 14:47:31 +0100193{
Will Deacona7a04102013-12-17 19:17:31 +0100194 static u32 cur_idx = 1;
Will Deacon8a4e3a92013-02-28 17:47:36 +0100195 u64 asid = atomic64_read(&mm->context.id);
Will Deaconbf51bb82012-08-01 14:57:49 +0100196 u64 generation = atomic64_read(&asid_generation);
Will Deaconb5466f82012-06-15 14:47:31 +0100197
Will Deacona3912632014-11-14 11:37:34 +0100198 if (asid != 0) {
Will Deacon40ee0682015-12-02 14:31:25 +0100199 u64 newasid = generation | (asid & ~ASID_MASK);
200
Catalin Marinas11805bc2010-01-26 19:09:42 +0100201 /*
Will Deacona3912632014-11-14 11:37:34 +0100202 * If our current ASID was active during a rollover, we
203 * can continue to use it and this was just a false alarm.
Catalin Marinas11805bc2010-01-26 19:09:42 +0100204 */
Will Deacon40ee0682015-12-02 14:31:25 +0100205 if (check_update_reserved_asid(asid, newasid))
206 return newasid;
Will Deacona3912632014-11-14 11:37:34 +0100207
Will Deaconb5466f82012-06-15 14:47:31 +0100208 /*
Will Deacona3912632014-11-14 11:37:34 +0100209 * We had a valid ASID in a previous life, so try to re-use
210 * it if possible.,
Will Deaconb5466f82012-06-15 14:47:31 +0100211 */
Will Deacona3912632014-11-14 11:37:34 +0100212 asid &= ~ASID_MASK;
213 if (!__test_and_set_bit(asid, asid_map))
Will Deacon40ee0682015-12-02 14:31:25 +0100214 return newasid;
Catalin Marinas11805bc2010-01-26 19:09:42 +0100215 }
Catalin Marinas11805bc2010-01-26 19:09:42 +0100216
Will Deacona3912632014-11-14 11:37:34 +0100217 /*
218 * Allocate a free ASID. If we can't find one, take a note of the
219 * currently active ASIDs and mark the TLBs as requiring flushes.
220 * We always count from ASID #1, as we reserve ASID #0 to switch
221 * via TTBR0 and to avoid speculative page table walks from hitting
222 * in any partial walk caches, which could be populated from
223 * overlapping level-1 descriptors used to map both the module
224 * area and the userspace stack.
225 */
226 asid = find_next_zero_bit(asid_map, NUM_USER_ASIDS, cur_idx);
227 if (asid == NUM_USER_ASIDS) {
228 generation = atomic64_add_return(ASID_FIRST_VERSION,
229 &asid_generation);
230 flush_context(cpu);
231 asid = find_next_zero_bit(asid_map, NUM_USER_ASIDS, 1);
232 }
233
234 __set_bit(asid, asid_map);
235 cur_idx = asid;
Will Deacona3912632014-11-14 11:37:34 +0100236 cpumask_clear(mm_cpumask(mm));
Will Deacon40ee0682015-12-02 14:31:25 +0100237 return asid | generation;
Catalin Marinas11805bc2010-01-26 19:09:42 +0100238}
239
Will Deaconb5466f82012-06-15 14:47:31 +0100240void check_and_switch_context(struct mm_struct *mm, struct task_struct *tsk)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700241{
Will Deaconb5466f82012-06-15 14:47:31 +0100242 unsigned long flags;
243 unsigned int cpu = smp_processor_id();
Will Deacon8a4e3a92013-02-28 17:47:36 +0100244 u64 asid;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700245
Nicolas Pitre3e996752012-11-25 03:24:32 +0100246 if (unlikely(mm->context.vmalloc_seq != init_mm.context.vmalloc_seq))
247 __check_vmalloc_seq(mm);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700248
249 /*
Will Deacon5d497502013-12-17 19:17:54 +0100250 * We cannot update the pgd and the ASID atomicly with classic
251 * MMU, so switch exclusively to global mappings to avoid
252 * speculative page table walking with the wrong TTBR.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700253 */
Will Deaconb5466f82012-06-15 14:47:31 +0100254 cpu_set_reserved_ttbr0();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700255
Will Deacon8a4e3a92013-02-28 17:47:36 +0100256 asid = atomic64_read(&mm->context.id);
257 if (!((asid ^ atomic64_read(&asid_generation)) >> ASID_BITS)
258 && atomic64_xchg(&per_cpu(active_asids, cpu), asid))
Will Deacon4b883162012-07-27 12:31:35 +0100259 goto switch_mm_fastpath;
260
Will Deaconb5466f82012-06-15 14:47:31 +0100261 raw_spin_lock_irqsave(&cpu_asid_lock, flags);
262 /* Check that our ASID belongs to the current generation. */
Will Deacon8a4e3a92013-02-28 17:47:36 +0100263 asid = atomic64_read(&mm->context.id);
264 if ((asid ^ atomic64_read(&asid_generation)) >> ASID_BITS) {
265 asid = new_context(mm, cpu);
266 atomic64_set(&mm->context.id, asid);
267 }
Will Deaconb5466f82012-06-15 14:47:31 +0100268
Will Deacon89c7e4b2013-02-28 17:48:40 +0100269 if (cpumask_test_and_clear_cpu(cpu, &tlb_flush_pending)) {
270 local_flush_bp_all();
Will Deaconb5466f82012-06-15 14:47:31 +0100271 local_flush_tlb_all();
Will Deacon89c7e4b2013-02-28 17:48:40 +0100272 }
Will Deacon37f47e32013-02-28 17:47:20 +0100273
Will Deacon8a4e3a92013-02-28 17:47:36 +0100274 atomic64_set(&per_cpu(active_asids, cpu), asid);
Will Deacon37f47e32013-02-28 17:47:20 +0100275 cpumask_set_cpu(cpu, mm_cpumask(mm));
Will Deaconb5466f82012-06-15 14:47:31 +0100276 raw_spin_unlock_irqrestore(&cpu_asid_lock, flags);
277
Will Deacon4b883162012-07-27 12:31:35 +0100278switch_mm_fastpath:
Will Deaconb5466f82012-06-15 14:47:31 +0100279 cpu_switch_mm(mm->pgd, mm);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700280}