blob: c0f6198565eb63592a3cbebaf7c8e538e987b157 [file] [log] [blame]
Pekka Paalanen8b7d89d2008-05-12 21:20:56 +02001/* Support for MMIO probes.
2 * Benfit many code from kprobes
3 * (C) 2002 Louis Zhuang <louis.zhuang@intel.com>.
4 * 2007 Alexander Eichner
5 * 2008 Pekka Paalanen <pq@iki.fi>
6 */
7
Joe Perches1bd591a2009-12-09 10:45:36 -08008#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
9
Pekka Paalanen0fd0e3d2008-05-12 21:20:57 +020010#include <linux/list.h>
Ingo Molnar668a6c32008-05-19 13:35:24 +020011#include <linux/rculist.h>
Pekka Paalanen8b7d89d2008-05-12 21:20:56 +020012#include <linux/spinlock.h>
13#include <linux/hash.h>
14#include <linux/init.h>
15#include <linux/module.h>
Pekka Paalanen8b7d89d2008-05-12 21:20:56 +020016#include <linux/kernel.h>
Pekka Paalanen8b7d89d2008-05-12 21:20:56 +020017#include <linux/uaccess.h>
18#include <linux/ptrace.h>
19#include <linux/preempt.h>
Pekka Paalanenf5136382008-05-12 21:20:57 +020020#include <linux/percpu.h>
Pekka Paalanen0fd0e3d2008-05-12 21:20:57 +020021#include <linux/kdebug.h>
Pekka Paalanend61fc442008-05-12 21:20:57 +020022#include <linux/mutex.h>
Pekka Paalanen970e6fa2008-05-12 21:21:03 +020023#include <linux/io.h>
Pekka Paalanen8b7d89d2008-05-12 21:20:56 +020024#include <asm/cacheflush.h>
Pekka Paalanen8b7d89d2008-05-12 21:20:56 +020025#include <asm/tlbflush.h>
Pekka Paalanen970e6fa2008-05-12 21:21:03 +020026#include <linux/errno.h>
Pekka Paalanen13829532008-05-12 21:20:58 +020027#include <asm/debugreg.h>
Pekka Paalanen0fd0e3d2008-05-12 21:20:57 +020028#include <linux/mmiotrace.h>
Pekka Paalanen8b7d89d2008-05-12 21:20:56 +020029
Pekka Paalanen8b7d89d2008-05-12 21:20:56 +020030#define KMMIO_PAGE_HASH_BITS 4
31#define KMMIO_PAGE_TABLE_SIZE (1 << KMMIO_PAGE_HASH_BITS)
32
Pekka Paalanen0fd0e3d2008-05-12 21:20:57 +020033struct kmmio_fault_page {
34 struct list_head list;
35 struct kmmio_fault_page *release_next;
36 unsigned long page; /* location of the fault page */
Stuart Bennett46e91d02009-04-28 20:17:50 +010037 pteval_t old_presence; /* page presence prior to arming */
Pekka Paalanen5359b582009-03-01 16:11:58 +020038 bool armed;
Pekka Paalanen0fd0e3d2008-05-12 21:20:57 +020039
40 /*
41 * Number of times this page has been registered as a part
42 * of a probe. If zero, page is disarmed and this may be freed.
Pekka Paalanen340430c2009-02-24 21:44:15 +020043 * Used only by writers (RCU) and post_kmmio_handler().
44 * Protected by kmmio_lock, when linked into kmmio_page_table.
Pekka Paalanen0fd0e3d2008-05-12 21:20:57 +020045 */
46 int count;
47};
48
49struct kmmio_delayed_release {
50 struct rcu_head rcu;
51 struct kmmio_fault_page *release_list;
52};
53
Pekka Paalanen8b7d89d2008-05-12 21:20:56 +020054struct kmmio_context {
55 struct kmmio_fault_page *fpage;
56 struct kmmio_probe *probe;
57 unsigned long saved_flags;
Pekka Paalanen0fd0e3d2008-05-12 21:20:57 +020058 unsigned long addr;
Pekka Paalanen8b7d89d2008-05-12 21:20:56 +020059 int active;
60};
61
Pekka Paalanen8b7d89d2008-05-12 21:20:56 +020062static DEFINE_SPINLOCK(kmmio_lock);
63
Pekka Paalanen13829532008-05-12 21:20:58 +020064/* Protected by kmmio_lock */
Pekka Paalanen8b7d89d2008-05-12 21:20:56 +020065unsigned int kmmio_count;
Pekka Paalanen0fd0e3d2008-05-12 21:20:57 +020066
67/* Read-protected by RCU, write-protected by kmmio_lock. */
Pekka Paalanen8b7d89d2008-05-12 21:20:56 +020068static struct list_head kmmio_page_table[KMMIO_PAGE_TABLE_SIZE];
69static LIST_HEAD(kmmio_probes);
70
Pekka Paalanen0fd0e3d2008-05-12 21:20:57 +020071static struct list_head *kmmio_page_list(unsigned long page)
72{
73 return &kmmio_page_table[hash_long(page, KMMIO_PAGE_HASH_BITS)];
74}
75
Pekka Paalanenf5136382008-05-12 21:20:57 +020076/* Accessed per-cpu */
77static DEFINE_PER_CPU(struct kmmio_context, kmmio_ctx);
Pekka Paalanen8b7d89d2008-05-12 21:20:56 +020078
Pekka Paalanen8b7d89d2008-05-12 21:20:56 +020079/*
80 * this is basically a dynamic stabbing problem:
81 * Could use the existing prio tree code or
82 * Possible better implementations:
83 * The Interval Skip List: A Data Structure for Finding All Intervals That
84 * Overlap a Point (might be simple)
85 * Space Efficient Dynamic Stabbing with Fast Queries - Mikkel Thorup
86 */
Pekka Paalanen0fd0e3d2008-05-12 21:20:57 +020087/* Get the kmmio at this addr (if any). You must be holding RCU read lock. */
Pekka Paalanen8b7d89d2008-05-12 21:20:56 +020088static struct kmmio_probe *get_kmmio_probe(unsigned long addr)
89{
90 struct kmmio_probe *p;
Pekka Paalanen0fd0e3d2008-05-12 21:20:57 +020091 list_for_each_entry_rcu(p, &kmmio_probes, list) {
Stuart Bennett33015c82009-04-28 20:17:48 +010092 if (addr >= p->addr && addr < (p->addr + p->len))
Pekka Paalanen8b7d89d2008-05-12 21:20:56 +020093 return p;
94 }
95 return NULL;
96}
97
Pekka Paalanen0fd0e3d2008-05-12 21:20:57 +020098/* You must be holding RCU read lock. */
Pekka Paalanen8b7d89d2008-05-12 21:20:56 +020099static struct kmmio_fault_page *get_kmmio_fault_page(unsigned long page)
100{
Pekka Paalanen0fd0e3d2008-05-12 21:20:57 +0200101 struct list_head *head;
Stuart Bennett0492e1b2009-04-28 20:17:49 +0100102 struct kmmio_fault_page *f;
Pekka Paalanen8b7d89d2008-05-12 21:20:56 +0200103
104 page &= PAGE_MASK;
Pekka Paalanen0fd0e3d2008-05-12 21:20:57 +0200105 head = kmmio_page_list(page);
Stuart Bennett0492e1b2009-04-28 20:17:49 +0100106 list_for_each_entry_rcu(f, head, list) {
107 if (f->page == page)
108 return f;
Pekka Paalanen8b7d89d2008-05-12 21:20:56 +0200109 }
Pekka Paalanen8b7d89d2008-05-12 21:20:56 +0200110 return NULL;
111}
112
Stuart Bennett46e91d02009-04-28 20:17:50 +0100113static void clear_pmd_presence(pmd_t *pmd, bool clear, pmdval_t *old)
Pekka Paalanen0b700a62009-03-01 16:12:48 +0200114{
115 pmdval_t v = pmd_val(*pmd);
Stuart Bennett46e91d02009-04-28 20:17:50 +0100116 if (clear) {
117 *old = v & _PAGE_PRESENT;
118 v &= ~_PAGE_PRESENT;
119 } else /* presume this has been called with clear==true previously */
120 v |= *old;
Pekka Paalanen0b700a62009-03-01 16:12:48 +0200121 set_pmd(pmd, __pmd(v));
122}
123
Stuart Bennett46e91d02009-04-28 20:17:50 +0100124static void clear_pte_presence(pte_t *pte, bool clear, pteval_t *old)
Pekka Paalanen0b700a62009-03-01 16:12:48 +0200125{
126 pteval_t v = pte_val(*pte);
Stuart Bennett46e91d02009-04-28 20:17:50 +0100127 if (clear) {
128 *old = v & _PAGE_PRESENT;
129 v &= ~_PAGE_PRESENT;
130 } else /* presume this has been called with clear==true previously */
131 v |= *old;
Pekka Paalanen0b700a62009-03-01 16:12:48 +0200132 set_pte_atomic(pte, __pte(v));
133}
134
Stuart Bennett46e91d02009-04-28 20:17:50 +0100135static int clear_page_presence(struct kmmio_fault_page *f, bool clear)
Pekka Paalanen8b7d89d2008-05-12 21:20:56 +0200136{
Pekka Paalanen790e2a22008-05-12 21:21:14 +0200137 unsigned int level;
Stuart Bennett46e91d02009-04-28 20:17:50 +0100138 pte_t *pte = lookup_address(f->page, &level);
Pekka Paalanen8b7d89d2008-05-12 21:20:56 +0200139
Pekka Paalanen75bb8832008-05-12 21:20:56 +0200140 if (!pte) {
Joe Perches1bd591a2009-12-09 10:45:36 -0800141 pr_err("no pte for page 0x%08lx\n", f->page);
Stuart Bennette9d54ca2009-01-30 17:38:59 +0000142 return -1;
Pekka Paalanen75bb8832008-05-12 21:20:56 +0200143 }
144
Pekka Paalanen13829532008-05-12 21:20:58 +0200145 switch (level) {
146 case PG_LEVEL_2M:
Stuart Bennett46e91d02009-04-28 20:17:50 +0100147 clear_pmd_presence((pmd_t *)pte, clear, &f->old_presence);
Pekka Paalanen13829532008-05-12 21:20:58 +0200148 break;
Pekka Paalanen13829532008-05-12 21:20:58 +0200149 case PG_LEVEL_4K:
Stuart Bennett46e91d02009-04-28 20:17:50 +0100150 clear_pte_presence(pte, clear, &f->old_presence);
Pekka Paalanen13829532008-05-12 21:20:58 +0200151 break;
Pekka Paalanen13829532008-05-12 21:20:58 +0200152 default:
Joe Perches1bd591a2009-12-09 10:45:36 -0800153 pr_err("unexpected page level 0x%x.\n", level);
Stuart Bennette9d54ca2009-01-30 17:38:59 +0000154 return -1;
Pekka Paalanen8b7d89d2008-05-12 21:20:56 +0200155 }
156
Stuart Bennett46e91d02009-04-28 20:17:50 +0100157 __flush_tlb_one(f->page);
Stuart Bennette9d54ca2009-01-30 17:38:59 +0000158 return 0;
Pekka Paalanen13829532008-05-12 21:20:58 +0200159}
Pekka Paalanen75bb8832008-05-12 21:20:56 +0200160
Pekka Paalanen5359b582009-03-01 16:11:58 +0200161/*
162 * Mark the given page as not present. Access to it will trigger a fault.
163 *
164 * Struct kmmio_fault_page is protected by RCU and kmmio_lock, but the
165 * protection is ignored here. RCU read lock is assumed held, so the struct
166 * will not disappear unexpectedly. Furthermore, the caller must guarantee,
167 * that double arming the same virtual address (page) cannot occur.
168 *
169 * Double disarming on the other hand is allowed, and may occur when a fault
170 * and mmiotrace shutdown happen simultaneously.
171 */
172static int arm_kmmio_fault_page(struct kmmio_fault_page *f)
Pekka Paalanen13829532008-05-12 21:20:58 +0200173{
Pekka Paalanen5359b582009-03-01 16:11:58 +0200174 int ret;
Joe Perches1bd591a2009-12-09 10:45:36 -0800175 WARN_ONCE(f->armed, KERN_ERR pr_fmt("kmmio page already armed.\n"));
Pekka Paalanen5359b582009-03-01 16:11:58 +0200176 if (f->armed) {
Joe Perches1bd591a2009-12-09 10:45:36 -0800177 pr_warning("double-arm: page 0x%08lx, ref %d, old %d\n",
178 f->page, f->count, !!f->old_presence);
Pekka Paalanen5359b582009-03-01 16:11:58 +0200179 }
Stuart Bennett46e91d02009-04-28 20:17:50 +0100180 ret = clear_page_presence(f, true);
Joe Perches1bd591a2009-12-09 10:45:36 -0800181 WARN_ONCE(ret < 0, KERN_ERR pr_fmt("arming 0x%08lx failed.\n"),
182 f->page);
Pekka Paalanen5359b582009-03-01 16:11:58 +0200183 f->armed = true;
Stuart Bennette9d54ca2009-01-30 17:38:59 +0000184 return ret;
Pekka Paalanen8b7d89d2008-05-12 21:20:56 +0200185}
186
Pekka Paalanen5359b582009-03-01 16:11:58 +0200187/** Restore the given page to saved presence state. */
188static void disarm_kmmio_fault_page(struct kmmio_fault_page *f)
Pekka Paalanen8b7d89d2008-05-12 21:20:56 +0200189{
Stuart Bennett46e91d02009-04-28 20:17:50 +0100190 int ret = clear_page_presence(f, false);
Pekka Paalanen5359b582009-03-01 16:11:58 +0200191 WARN_ONCE(ret < 0,
192 KERN_ERR "kmmio disarming 0x%08lx failed.\n", f->page);
193 f->armed = false;
Pekka Paalanen8b7d89d2008-05-12 21:20:56 +0200194}
195
196/*
Pekka Paalanen0fd0e3d2008-05-12 21:20:57 +0200197 * This is being called from do_page_fault().
198 *
199 * We may be in an interrupt or a critical section. Also prefecthing may
200 * trigger a page fault. We may be in the middle of process switch.
201 * We cannot take any locks, because we could be executing especially
202 * within a kmmio critical section.
203 *
204 * Local interrupts are disabled, so preemption cannot happen.
205 * Do not enable interrupts, do not sleep, and watch out for other CPUs.
206 */
207/*
Pekka Paalanen8b7d89d2008-05-12 21:20:56 +0200208 * Interrupts are disabled on entry as trap3 is an interrupt gate
André Goddard Rosaaf901ca2009-11-14 13:09:05 -0200209 * and they remain disabled throughout this function.
Pekka Paalanen8b7d89d2008-05-12 21:20:56 +0200210 */
Pekka Paalanen0fd0e3d2008-05-12 21:20:57 +0200211int kmmio_handler(struct pt_regs *regs, unsigned long addr)
Pekka Paalanen8b7d89d2008-05-12 21:20:56 +0200212{
Pekka Paalanen0fd0e3d2008-05-12 21:20:57 +0200213 struct kmmio_context *ctx;
214 struct kmmio_fault_page *faultpage;
Pekka Paalanen13829532008-05-12 21:20:58 +0200215 int ret = 0; /* default to fault not handled */
Pekka Paalanen8b7d89d2008-05-12 21:20:56 +0200216
217 /*
218 * Preemption is now disabled to prevent process switch during
219 * single stepping. We can only handle one active kmmio trace
220 * per cpu, so ensure that we finish it before something else
Pekka Paalanend61fc442008-05-12 21:20:57 +0200221 * gets to run. We also hold the RCU read lock over single
222 * stepping to avoid looking up the probe and kmmio_fault_page
223 * again.
Pekka Paalanen8b7d89d2008-05-12 21:20:56 +0200224 */
225 preempt_disable();
Pekka Paalanen0fd0e3d2008-05-12 21:20:57 +0200226 rcu_read_lock();
Pekka Paalanend61fc442008-05-12 21:20:57 +0200227
Pekka Paalanen0fd0e3d2008-05-12 21:20:57 +0200228 faultpage = get_kmmio_fault_page(addr);
229 if (!faultpage) {
230 /*
231 * Either this page fault is not caused by kmmio, or
232 * another CPU just pulled the kmmio probe from under
Pekka Paalanen13829532008-05-12 21:20:58 +0200233 * our feet. The latter case should not be possible.
Pekka Paalanen0fd0e3d2008-05-12 21:20:57 +0200234 */
235 goto no_kmmio;
236 }
237
238 ctx = &get_cpu_var(kmmio_ctx);
Pekka Paalanen8b7d89d2008-05-12 21:20:56 +0200239 if (ctx->active) {
Pekka Paalanen13829532008-05-12 21:20:58 +0200240 if (addr == ctx->addr) {
241 /*
Stuart Bennett3e39aa12009-02-05 11:02:02 +0000242 * A second fault on the same page means some other
243 * condition needs handling by do_page_fault(), the
244 * page really not being present is the most common.
Pekka Paalanen13829532008-05-12 21:20:58 +0200245 */
Joe Perches1bd591a2009-12-09 10:45:36 -0800246 pr_debug("secondary hit for 0x%08lx CPU %d.\n",
247 addr, smp_processor_id());
Stuart Bennett3e39aa12009-02-05 11:02:02 +0000248
249 if (!faultpage->old_presence)
Joe Perches1bd591a2009-12-09 10:45:36 -0800250 pr_info("unexpected secondary hit for address 0x%08lx on CPU %d.\n",
251 addr, smp_processor_id());
Stuart Bennett3e39aa12009-02-05 11:02:02 +0000252 } else {
253 /*
254 * Prevent overwriting already in-flight context.
255 * This should not happen, let's hope disarming at
256 * least prevents a panic.
257 */
Joe Perches1bd591a2009-12-09 10:45:36 -0800258 pr_emerg("recursive probe hit on CPU %d, for address 0x%08lx. Ignoring.\n",
259 smp_processor_id(), addr);
260 pr_emerg("previous hit was at 0x%08lx.\n", ctx->addr);
Stuart Bennett3e39aa12009-02-05 11:02:02 +0000261 disarm_kmmio_fault_page(faultpage);
262 }
Pekka Paalanen0fd0e3d2008-05-12 21:20:57 +0200263 goto no_kmmio_ctx;
Pekka Paalanen8b7d89d2008-05-12 21:20:56 +0200264 }
265 ctx->active++;
266
Pekka Paalanen0fd0e3d2008-05-12 21:20:57 +0200267 ctx->fpage = faultpage;
Pekka Paalanen8b7d89d2008-05-12 21:20:56 +0200268 ctx->probe = get_kmmio_probe(addr);
Ingo Molnar49023162008-05-12 21:20:58 +0200269 ctx->saved_flags = (regs->flags & (X86_EFLAGS_TF | X86_EFLAGS_IF));
Pekka Paalanen0fd0e3d2008-05-12 21:20:57 +0200270 ctx->addr = addr;
Pekka Paalanen8b7d89d2008-05-12 21:20:56 +0200271
272 if (ctx->probe && ctx->probe->pre_handler)
273 ctx->probe->pre_handler(ctx->probe, regs, addr);
274
Pekka Paalanend61fc442008-05-12 21:20:57 +0200275 /*
276 * Enable single-stepping and disable interrupts for the faulting
277 * context. Local interrupts must not get enabled during stepping.
278 */
Ingo Molnar49023162008-05-12 21:20:58 +0200279 regs->flags |= X86_EFLAGS_TF;
280 regs->flags &= ~X86_EFLAGS_IF;
Pekka Paalanen8b7d89d2008-05-12 21:20:56 +0200281
Pekka Paalanen0fd0e3d2008-05-12 21:20:57 +0200282 /* Now we set present bit in PTE and single step. */
Pekka Paalanen5359b582009-03-01 16:11:58 +0200283 disarm_kmmio_fault_page(ctx->fpage);
Pekka Paalanen8b7d89d2008-05-12 21:20:56 +0200284
Pekka Paalanend61fc442008-05-12 21:20:57 +0200285 /*
286 * If another cpu accesses the same page while we are stepping,
287 * the access will not be caught. It will simply succeed and the
288 * only downside is we lose the event. If this becomes a problem,
289 * the user should drop to single cpu before tracing.
290 */
291
Pekka Paalanenf5136382008-05-12 21:20:57 +0200292 put_cpu_var(kmmio_ctx);
Pekka Paalanen13829532008-05-12 21:20:58 +0200293 return 1; /* fault handled */
Pekka Paalanen8b7d89d2008-05-12 21:20:56 +0200294
Pekka Paalanen0fd0e3d2008-05-12 21:20:57 +0200295no_kmmio_ctx:
Pekka Paalanenf5136382008-05-12 21:20:57 +0200296 put_cpu_var(kmmio_ctx);
Pekka Paalanen0fd0e3d2008-05-12 21:20:57 +0200297no_kmmio:
298 rcu_read_unlock();
299 preempt_enable_no_resched();
Pekka Paalanen13829532008-05-12 21:20:58 +0200300 return ret;
Pekka Paalanen8b7d89d2008-05-12 21:20:56 +0200301}
302
303/*
304 * Interrupts are disabled on entry as trap1 is an interrupt gate
André Goddard Rosaaf901ca2009-11-14 13:09:05 -0200305 * and they remain disabled throughout this function.
Pekka Paalanen0fd0e3d2008-05-12 21:20:57 +0200306 * This must always get called as the pair to kmmio_handler().
Pekka Paalanen8b7d89d2008-05-12 21:20:56 +0200307 */
308static int post_kmmio_handler(unsigned long condition, struct pt_regs *regs)
309{
Pekka Paalanenf5136382008-05-12 21:20:57 +0200310 int ret = 0;
311 struct kmmio_context *ctx = &get_cpu_var(kmmio_ctx);
Pekka Paalanen8b7d89d2008-05-12 21:20:56 +0200312
Pekka Paalanen13829532008-05-12 21:20:58 +0200313 if (!ctx->active) {
Stuart Bennett0f9a6232009-04-28 20:17:51 +0100314 /*
315 * debug traps without an active context are due to either
316 * something external causing them (f.e. using a debugger while
317 * mmio tracing enabled), or erroneous behaviour
318 */
Joe Perches1bd591a2009-12-09 10:45:36 -0800319 pr_warning("unexpected debug trap on CPU %d.\n",
320 smp_processor_id());
Pekka Paalanenf5136382008-05-12 21:20:57 +0200321 goto out;
Pekka Paalanen13829532008-05-12 21:20:58 +0200322 }
Pekka Paalanen8b7d89d2008-05-12 21:20:56 +0200323
324 if (ctx->probe && ctx->probe->post_handler)
325 ctx->probe->post_handler(ctx->probe, condition, regs);
326
Pekka Paalanen340430c2009-02-24 21:44:15 +0200327 /* Prevent racing against release_kmmio_fault_page(). */
328 spin_lock(&kmmio_lock);
329 if (ctx->fpage->count)
330 arm_kmmio_fault_page(ctx->fpage);
331 spin_unlock(&kmmio_lock);
Pekka Paalanen8b7d89d2008-05-12 21:20:56 +0200332
Ingo Molnar49023162008-05-12 21:20:58 +0200333 regs->flags &= ~X86_EFLAGS_TF;
Pekka Paalanen8b7d89d2008-05-12 21:20:56 +0200334 regs->flags |= ctx->saved_flags;
335
336 /* These were acquired in kmmio_handler(). */
337 ctx->active--;
Pekka Paalanen0fd0e3d2008-05-12 21:20:57 +0200338 BUG_ON(ctx->active);
Pekka Paalanend61fc442008-05-12 21:20:57 +0200339 rcu_read_unlock();
Pekka Paalanen8b7d89d2008-05-12 21:20:56 +0200340 preempt_enable_no_resched();
341
342 /*
343 * if somebody else is singlestepping across a probe point, flags
344 * will have TF set, in which case, continue the remaining processing
345 * of do_debug, as if this is not a probe hit.
346 */
Ingo Molnar49023162008-05-12 21:20:58 +0200347 if (!(regs->flags & X86_EFLAGS_TF))
Pekka Paalanenf5136382008-05-12 21:20:57 +0200348 ret = 1;
Pekka Paalanenf5136382008-05-12 21:20:57 +0200349out:
350 put_cpu_var(kmmio_ctx);
351 return ret;
Pekka Paalanen8b7d89d2008-05-12 21:20:56 +0200352}
353
Pekka Paalanen0fd0e3d2008-05-12 21:20:57 +0200354/* You must be holding kmmio_lock. */
Pekka Paalanen8b7d89d2008-05-12 21:20:56 +0200355static int add_kmmio_fault_page(unsigned long page)
356{
357 struct kmmio_fault_page *f;
358
359 page &= PAGE_MASK;
360 f = get_kmmio_fault_page(page);
361 if (f) {
Pekka Paalanen0fd0e3d2008-05-12 21:20:57 +0200362 if (!f->count)
Pekka Paalanen5359b582009-03-01 16:11:58 +0200363 arm_kmmio_fault_page(f);
Pekka Paalanen8b7d89d2008-05-12 21:20:56 +0200364 f->count++;
365 return 0;
366 }
367
Pekka Paalanen5359b582009-03-01 16:11:58 +0200368 f = kzalloc(sizeof(*f), GFP_ATOMIC);
Pekka Paalanen8b7d89d2008-05-12 21:20:56 +0200369 if (!f)
370 return -1;
371
372 f->count = 1;
373 f->page = page;
Pekka Paalanen8b7d89d2008-05-12 21:20:56 +0200374
Pekka Paalanen5359b582009-03-01 16:11:58 +0200375 if (arm_kmmio_fault_page(f)) {
Stuart Bennette9d54ca2009-01-30 17:38:59 +0000376 kfree(f);
377 return -1;
378 }
379
380 list_add_rcu(&f->list, kmmio_page_list(f->page));
Pekka Paalanen8b7d89d2008-05-12 21:20:56 +0200381
382 return 0;
383}
384
Pekka Paalanen0fd0e3d2008-05-12 21:20:57 +0200385/* You must be holding kmmio_lock. */
386static void release_kmmio_fault_page(unsigned long page,
387 struct kmmio_fault_page **release_list)
Pekka Paalanen8b7d89d2008-05-12 21:20:56 +0200388{
389 struct kmmio_fault_page *f;
390
391 page &= PAGE_MASK;
392 f = get_kmmio_fault_page(page);
393 if (!f)
394 return;
395
396 f->count--;
Pekka Paalanen0fd0e3d2008-05-12 21:20:57 +0200397 BUG_ON(f->count < 0);
Pekka Paalanen8b7d89d2008-05-12 21:20:56 +0200398 if (!f->count) {
Pekka Paalanen5359b582009-03-01 16:11:58 +0200399 disarm_kmmio_fault_page(f);
Pekka Paalanen0fd0e3d2008-05-12 21:20:57 +0200400 f->release_next = *release_list;
401 *release_list = f;
Pekka Paalanen8b7d89d2008-05-12 21:20:56 +0200402 }
403}
404
Pekka Paalanen87e547f2008-05-12 21:21:03 +0200405/*
406 * With page-unaligned ioremaps, one or two armed pages may contain
407 * addresses from outside the intended mapping. Events for these addresses
408 * are currently silently dropped. The events may result only from programming
409 * mistakes by accessing addresses before the beginning or past the end of a
410 * mapping.
411 */
Pekka Paalanen8b7d89d2008-05-12 21:20:56 +0200412int register_kmmio_probe(struct kmmio_probe *p)
413{
Pekka Paalanend61fc442008-05-12 21:20:57 +0200414 unsigned long flags;
Pekka Paalanen8b7d89d2008-05-12 21:20:56 +0200415 int ret = 0;
416 unsigned long size = 0;
Pekka Paalanen87e547f2008-05-12 21:21:03 +0200417 const unsigned long size_lim = p->len + (p->addr & ~PAGE_MASK);
Pekka Paalanen8b7d89d2008-05-12 21:20:56 +0200418
Pekka Paalanend61fc442008-05-12 21:20:57 +0200419 spin_lock_irqsave(&kmmio_lock, flags);
Pekka Paalanen8b7d89d2008-05-12 21:20:56 +0200420 if (get_kmmio_probe(p->addr)) {
421 ret = -EEXIST;
422 goto out;
423 }
Pekka Paalanend61fc442008-05-12 21:20:57 +0200424 kmmio_count++;
Pekka Paalanen0fd0e3d2008-05-12 21:20:57 +0200425 list_add_rcu(&p->list, &kmmio_probes);
Pekka Paalanen87e547f2008-05-12 21:21:03 +0200426 while (size < size_lim) {
Pekka Paalanen8b7d89d2008-05-12 21:20:56 +0200427 if (add_kmmio_fault_page(p->addr + size))
Joe Perches1bd591a2009-12-09 10:45:36 -0800428 pr_err("Unable to set page fault.\n");
Pekka Paalanen8b7d89d2008-05-12 21:20:56 +0200429 size += PAGE_SIZE;
430 }
Pekka Paalanen8b7d89d2008-05-12 21:20:56 +0200431out:
Pekka Paalanend61fc442008-05-12 21:20:57 +0200432 spin_unlock_irqrestore(&kmmio_lock, flags);
Pekka Paalanen8b7d89d2008-05-12 21:20:56 +0200433 /*
434 * XXX: What should I do here?
435 * Here was a call to global_flush_tlb(), but it does not exist
Pekka Paalanen0fd0e3d2008-05-12 21:20:57 +0200436 * anymore. It seems it's not needed after all.
Pekka Paalanen8b7d89d2008-05-12 21:20:56 +0200437 */
438 return ret;
439}
Pekka Paalanen0fd0e3d2008-05-12 21:20:57 +0200440EXPORT_SYMBOL(register_kmmio_probe);
Pekka Paalanen8b7d89d2008-05-12 21:20:56 +0200441
Pekka Paalanen0fd0e3d2008-05-12 21:20:57 +0200442static void rcu_free_kmmio_fault_pages(struct rcu_head *head)
Pekka Paalanen8b7d89d2008-05-12 21:20:56 +0200443{
Pekka Paalanen0fd0e3d2008-05-12 21:20:57 +0200444 struct kmmio_delayed_release *dr = container_of(
445 head,
446 struct kmmio_delayed_release,
447 rcu);
Stuart Bennett0492e1b2009-04-28 20:17:49 +0100448 struct kmmio_fault_page *f = dr->release_list;
449 while (f) {
450 struct kmmio_fault_page *next = f->release_next;
451 BUG_ON(f->count);
452 kfree(f);
453 f = next;
Pekka Paalanen8b7d89d2008-05-12 21:20:56 +0200454 }
Pekka Paalanen0fd0e3d2008-05-12 21:20:57 +0200455 kfree(dr);
456}
457
458static void remove_kmmio_fault_pages(struct rcu_head *head)
459{
Stuart Bennettd0fc63f2009-03-08 20:21:35 +0200460 struct kmmio_delayed_release *dr =
461 container_of(head, struct kmmio_delayed_release, rcu);
Stuart Bennett0492e1b2009-04-28 20:17:49 +0100462 struct kmmio_fault_page *f = dr->release_list;
Pekka Paalanen0fd0e3d2008-05-12 21:20:57 +0200463 struct kmmio_fault_page **prevp = &dr->release_list;
464 unsigned long flags;
Stuart Bennettd0fc63f2009-03-08 20:21:35 +0200465
Pekka Paalanen0fd0e3d2008-05-12 21:20:57 +0200466 spin_lock_irqsave(&kmmio_lock, flags);
Stuart Bennett0492e1b2009-04-28 20:17:49 +0100467 while (f) {
468 if (!f->count) {
469 list_del_rcu(&f->list);
470 prevp = &f->release_next;
Stuart Bennettd0fc63f2009-03-08 20:21:35 +0200471 } else {
Stuart Bennett0492e1b2009-04-28 20:17:49 +0100472 *prevp = f->release_next;
Stuart Bennettd0fc63f2009-03-08 20:21:35 +0200473 }
Stuart Bennett0492e1b2009-04-28 20:17:49 +0100474 f = f->release_next;
Pekka Paalanen0fd0e3d2008-05-12 21:20:57 +0200475 }
476 spin_unlock_irqrestore(&kmmio_lock, flags);
Stuart Bennettd0fc63f2009-03-08 20:21:35 +0200477
Pekka Paalanen0fd0e3d2008-05-12 21:20:57 +0200478 /* This is the real RCU destroy call. */
479 call_rcu(&dr->rcu, rcu_free_kmmio_fault_pages);
Pekka Paalanen8b7d89d2008-05-12 21:20:56 +0200480}
481
482/*
Pekka Paalanen0fd0e3d2008-05-12 21:20:57 +0200483 * Remove a kmmio probe. You have to synchronize_rcu() before you can be
Pekka Paalanend61fc442008-05-12 21:20:57 +0200484 * sure that the callbacks will not be called anymore. Only after that
485 * you may actually release your struct kmmio_probe.
Pekka Paalanen8b7d89d2008-05-12 21:20:56 +0200486 *
Pekka Paalanen0fd0e3d2008-05-12 21:20:57 +0200487 * Unregistering a kmmio fault page has three steps:
488 * 1. release_kmmio_fault_page()
489 * Disarm the page, wait a grace period to let all faults finish.
490 * 2. remove_kmmio_fault_pages()
491 * Remove the pages from kmmio_page_table.
492 * 3. rcu_free_kmmio_fault_pages()
Shaun Patterson80550392009-12-05 10:41:34 -0500493 * Actually free the kmmio_fault_page structs as with RCU.
Pekka Paalanen8b7d89d2008-05-12 21:20:56 +0200494 */
Pekka Paalanen0fd0e3d2008-05-12 21:20:57 +0200495void unregister_kmmio_probe(struct kmmio_probe *p)
Pekka Paalanen8b7d89d2008-05-12 21:20:56 +0200496{
Pekka Paalanend61fc442008-05-12 21:20:57 +0200497 unsigned long flags;
Pekka Paalanen0fd0e3d2008-05-12 21:20:57 +0200498 unsigned long size = 0;
Pekka Paalanen87e547f2008-05-12 21:21:03 +0200499 const unsigned long size_lim = p->len + (p->addr & ~PAGE_MASK);
Pekka Paalanen0fd0e3d2008-05-12 21:20:57 +0200500 struct kmmio_fault_page *release_list = NULL;
501 struct kmmio_delayed_release *drelease;
502
Pekka Paalanend61fc442008-05-12 21:20:57 +0200503 spin_lock_irqsave(&kmmio_lock, flags);
Pekka Paalanen87e547f2008-05-12 21:21:03 +0200504 while (size < size_lim) {
Pekka Paalanen0fd0e3d2008-05-12 21:20:57 +0200505 release_kmmio_fault_page(p->addr + size, &release_list);
506 size += PAGE_SIZE;
507 }
508 list_del_rcu(&p->list);
509 kmmio_count--;
Pekka Paalanend61fc442008-05-12 21:20:57 +0200510 spin_unlock_irqrestore(&kmmio_lock, flags);
Pekka Paalanen0fd0e3d2008-05-12 21:20:57 +0200511
512 drelease = kmalloc(sizeof(*drelease), GFP_ATOMIC);
513 if (!drelease) {
Joe Perches1bd591a2009-12-09 10:45:36 -0800514 pr_crit("leaking kmmio_fault_page objects.\n");
Pekka Paalanen0fd0e3d2008-05-12 21:20:57 +0200515 return;
516 }
517 drelease->release_list = release_list;
518
519 /*
520 * This is not really RCU here. We have just disarmed a set of
521 * pages so that they cannot trigger page faults anymore. However,
522 * we cannot remove the pages from kmmio_page_table,
523 * because a probe hit might be in flight on another CPU. The
524 * pages are collected into a list, and they will be removed from
525 * kmmio_page_table when it is certain that no probe hit related to
526 * these pages can be in flight. RCU grace period sounds like a
527 * good choice.
528 *
529 * If we removed the pages too early, kmmio page fault handler might
530 * not find the respective kmmio_fault_page and determine it's not
531 * a kmmio fault, when it actually is. This would lead to madness.
532 */
533 call_rcu(&drelease->rcu, remove_kmmio_fault_pages);
Pekka Paalanen8b7d89d2008-05-12 21:20:56 +0200534}
Pekka Paalanen0fd0e3d2008-05-12 21:20:57 +0200535EXPORT_SYMBOL(unregister_kmmio_probe);
Pekka Paalanen8b7d89d2008-05-12 21:20:56 +0200536
Stuart Bennett0f9a6232009-04-28 20:17:51 +0100537static int
538kmmio_die_notifier(struct notifier_block *nb, unsigned long val, void *args)
Pekka Paalanen8b7d89d2008-05-12 21:20:56 +0200539{
540 struct die_args *arg = args;
541
Pekka Paalanen13829532008-05-12 21:20:58 +0200542 if (val == DIE_DEBUG && (arg->err & DR_STEP))
K.Prasad62edab92009-06-01 23:47:06 +0530543 if (post_kmmio_handler(arg->err, arg->regs) == 1) {
544 /*
545 * Reset the BS bit in dr6 (pointed by args->err) to
546 * denote completion of processing
547 */
548 (*(unsigned long *)ERR_PTR(arg->err)) &= ~DR_STEP;
Pekka Paalanen8b7d89d2008-05-12 21:20:56 +0200549 return NOTIFY_STOP;
K.Prasad62edab92009-06-01 23:47:06 +0530550 }
Pekka Paalanen8b7d89d2008-05-12 21:20:56 +0200551
552 return NOTIFY_DONE;
553}
Pekka Paalanen13829532008-05-12 21:20:58 +0200554
555static struct notifier_block nb_die = {
556 .notifier_call = kmmio_die_notifier
557};
558
Stuart Bennett0f9a6232009-04-28 20:17:51 +0100559int kmmio_init(void)
Pekka Paalanen13829532008-05-12 21:20:58 +0200560{
561 int i;
Stuart Bennett0f9a6232009-04-28 20:17:51 +0100562
Pekka Paalanen13829532008-05-12 21:20:58 +0200563 for (i = 0; i < KMMIO_PAGE_TABLE_SIZE; i++)
564 INIT_LIST_HEAD(&kmmio_page_table[i]);
Stuart Bennett0f9a6232009-04-28 20:17:51 +0100565
Pekka Paalanen13829532008-05-12 21:20:58 +0200566 return register_die_notifier(&nb_die);
567}
Stuart Bennett0f9a6232009-04-28 20:17:51 +0100568
569void kmmio_cleanup(void)
570{
571 int i;
572
573 unregister_die_notifier(&nb_die);
574 for (i = 0; i < KMMIO_PAGE_TABLE_SIZE; i++) {
575 WARN_ONCE(!list_empty(&kmmio_page_table[i]),
576 KERN_ERR "kmmio_page_table not empty at cleanup, any further tracing will leak memory.\n");
577 }
578}