blob: ea54f2647868726428faa630114d6fcd673c4735 [file] [log] [blame]
Mathieu Desnoyers8256e472007-10-18 23:41:06 -07001/*
2 * Copyright (C) 2007 Mathieu Desnoyers
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License as published by
6 * the Free Software Foundation; either version 2 of the License, or
7 * (at your option) any later version.
8 *
9 * This program is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
13 *
14 * You should have received a copy of the GNU General Public License
15 * along with this program; if not, write to the Free Software
16 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
17 */
18#include <linux/module.h>
19#include <linux/mutex.h>
20#include <linux/types.h>
21#include <linux/jhash.h>
22#include <linux/list.h>
23#include <linux/rcupdate.h>
24#include <linux/marker.h>
25#include <linux/err.h>
Robert P. J. Day1aeb2722008-04-29 00:59:25 -070026#include <linux/slab.h>
Mathieu Desnoyers8256e472007-10-18 23:41:06 -070027
28extern struct marker __start___markers[];
29extern struct marker __stop___markers[];
30
Mathieu Desnoyersfb40bd72008-02-13 15:03:37 -080031/* Set to 1 to enable marker debug output */
Adrian Bunkab883af2008-04-30 00:54:30 -070032static const int marker_debug;
Mathieu Desnoyers8256e472007-10-18 23:41:06 -070033
34/*
Mathieu Desnoyersfb40bd72008-02-13 15:03:37 -080035 * markers_mutex nests inside module_mutex. Markers mutex protects the builtin
36 * and module markers and the hash table.
Mathieu Desnoyers8256e472007-10-18 23:41:06 -070037 */
Mathieu Desnoyersfb40bd72008-02-13 15:03:37 -080038static DEFINE_MUTEX(markers_mutex);
Mathieu Desnoyers8256e472007-10-18 23:41:06 -070039
40/*
41 * Marker hash table, containing the active markers.
42 * Protected by module_mutex.
43 */
44#define MARKER_HASH_BITS 6
45#define MARKER_TABLE_SIZE (1 << MARKER_HASH_BITS)
Lai Jiangshan4de62742008-10-15 14:56:47 +080046static struct hlist_head marker_table[MARKER_TABLE_SIZE];
Mathieu Desnoyers8256e472007-10-18 23:41:06 -070047
Mathieu Desnoyersfb40bd72008-02-13 15:03:37 -080048/*
49 * Note about RCU :
50 * It is used to make sure every handler has finished using its private data
51 * between two consecutive operation (add or remove) on a given marker. It is
52 * also used to delay the free of multiple probes array until a quiescent state
53 * is reached.
54 * marker entries modifications are protected by the markers_mutex.
55 */
Mathieu Desnoyers8256e472007-10-18 23:41:06 -070056struct marker_entry {
57 struct hlist_node hlist;
58 char *format;
Mathieu Desnoyersdc102a82008-05-12 21:21:09 +020059 /* Probe wrapper */
60 void (*call)(const struct marker *mdata, void *call_private, ...);
Mathieu Desnoyersfb40bd72008-02-13 15:03:37 -080061 struct marker_probe_closure single;
62 struct marker_probe_closure *multi;
Mathieu Desnoyers8256e472007-10-18 23:41:06 -070063 int refcount; /* Number of times armed. 0 if disarmed. */
Mathieu Desnoyersed86a592008-10-01 12:03:25 -040064 struct rcu_head rcu;
65 void *oldptr;
Lai Jiangshan1b7ae372008-10-10 14:43:57 +080066 int rcu_pending;
Harvey Harrisonde4fc642008-02-23 15:23:33 -080067 unsigned char ptype:1;
Lai Jiangshan0eec4812008-10-15 14:56:37 +080068 unsigned char format_allocated:1;
Mathieu Desnoyers8256e472007-10-18 23:41:06 -070069 char name[0]; /* Contains name'\0'format'\0' */
70};
71
Mathieu Desnoyers8256e472007-10-18 23:41:06 -070072/**
73 * __mark_empty_function - Empty probe callback
Mathieu Desnoyersfb40bd72008-02-13 15:03:37 -080074 * @probe_private: probe private data
75 * @call_private: call site private data
Mathieu Desnoyers8256e472007-10-18 23:41:06 -070076 * @fmt: format string
77 * @...: variable argument list
78 *
79 * Empty callback provided as a probe to the markers. By providing this to a
80 * disabled marker, we make sure the execution flow is always valid even
81 * though the function pointer change and the marker enabling are two distinct
82 * operations that modifies the execution flow of preemptible code.
83 */
Mathieu Desnoyers021aeb02008-11-14 17:47:37 -050084notrace void __mark_empty_function(void *probe_private, void *call_private,
Mathieu Desnoyersfb40bd72008-02-13 15:03:37 -080085 const char *fmt, va_list *args)
Mathieu Desnoyers8256e472007-10-18 23:41:06 -070086{
87}
88EXPORT_SYMBOL_GPL(__mark_empty_function);
89
90/*
Mathieu Desnoyersfb40bd72008-02-13 15:03:37 -080091 * marker_probe_cb Callback that prepares the variable argument list for probes.
92 * @mdata: pointer of type struct marker
93 * @call_private: caller site private data
Mathieu Desnoyersfb40bd72008-02-13 15:03:37 -080094 * @...: Variable argument list.
95 *
96 * Since we do not use "typical" pointer based RCU in the 1 argument case, we
97 * need to put a full smp_rmb() in this branch. This is why we do not use
98 * rcu_dereference() for the pointer read.
99 */
Mathieu Desnoyers021aeb02008-11-14 17:47:37 -0500100notrace void marker_probe_cb(const struct marker *mdata,
101 void *call_private, ...)
Mathieu Desnoyersfb40bd72008-02-13 15:03:37 -0800102{
103 va_list args;
104 char ptype;
105
106 /*
Mathieu Desnoyerse2d3b75d2008-09-29 11:08:03 -0400107 * rcu_read_lock_sched does two things : disabling preemption to make
108 * sure the teardown of the callbacks can be done correctly when they
109 * are in modules and they insure RCU read coherency.
Mathieu Desnoyersfb40bd72008-02-13 15:03:37 -0800110 */
Mathieu Desnoyers021aeb02008-11-14 17:47:37 -0500111 rcu_read_lock_sched_notrace();
Mathieu Desnoyers58336112008-03-24 12:29:49 -0700112 ptype = mdata->ptype;
Mathieu Desnoyersfb40bd72008-02-13 15:03:37 -0800113 if (likely(!ptype)) {
114 marker_probe_func *func;
115 /* Must read the ptype before ptr. They are not data dependant,
116 * so we put an explicit smp_rmb() here. */
117 smp_rmb();
Mathieu Desnoyers58336112008-03-24 12:29:49 -0700118 func = mdata->single.func;
Mathieu Desnoyersfb40bd72008-02-13 15:03:37 -0800119 /* Must read the ptr before private data. They are not data
120 * dependant, so we put an explicit smp_rmb() here. */
121 smp_rmb();
Mathieu Desnoyersdc102a82008-05-12 21:21:09 +0200122 va_start(args, call_private);
123 func(mdata->single.probe_private, call_private, mdata->format,
124 &args);
Mathieu Desnoyersfb40bd72008-02-13 15:03:37 -0800125 va_end(args);
126 } else {
127 struct marker_probe_closure *multi;
128 int i;
129 /*
Mathieu Desnoyers5def9a32008-07-29 22:33:31 -0700130 * Read mdata->ptype before mdata->multi.
131 */
132 smp_rmb();
133 multi = mdata->multi;
134 /*
Mathieu Desnoyersfb40bd72008-02-13 15:03:37 -0800135 * multi points to an array, therefore accessing the array
136 * depends on reading multi. However, even in this case,
137 * we must insure that the pointer is read _before_ the array
138 * data. Same as rcu_dereference, but we need a full smp_rmb()
139 * in the fast path, so put the explicit barrier here.
140 */
141 smp_read_barrier_depends();
Mathieu Desnoyersfb40bd72008-02-13 15:03:37 -0800142 for (i = 0; multi[i].func; i++) {
Mathieu Desnoyersdc102a82008-05-12 21:21:09 +0200143 va_start(args, call_private);
144 multi[i].func(multi[i].probe_private, call_private,
145 mdata->format, &args);
Mathieu Desnoyersfb40bd72008-02-13 15:03:37 -0800146 va_end(args);
147 }
148 }
Mathieu Desnoyers021aeb02008-11-14 17:47:37 -0500149 rcu_read_unlock_sched_notrace();
Mathieu Desnoyersfb40bd72008-02-13 15:03:37 -0800150}
151EXPORT_SYMBOL_GPL(marker_probe_cb);
152
153/*
154 * marker_probe_cb Callback that does not prepare the variable argument list.
155 * @mdata: pointer of type struct marker
156 * @call_private: caller site private data
Mathieu Desnoyersfb40bd72008-02-13 15:03:37 -0800157 * @...: Variable argument list.
158 *
159 * Should be connected to markers "MARK_NOARGS".
160 */
Mathieu Desnoyers021aeb02008-11-14 17:47:37 -0500161static notrace void marker_probe_cb_noarg(const struct marker *mdata,
162 void *call_private, ...)
Mathieu Desnoyersfb40bd72008-02-13 15:03:37 -0800163{
164 va_list args; /* not initialized */
165 char ptype;
166
Mathieu Desnoyers021aeb02008-11-14 17:47:37 -0500167 rcu_read_lock_sched_notrace();
Mathieu Desnoyers58336112008-03-24 12:29:49 -0700168 ptype = mdata->ptype;
Mathieu Desnoyersfb40bd72008-02-13 15:03:37 -0800169 if (likely(!ptype)) {
170 marker_probe_func *func;
171 /* Must read the ptype before ptr. They are not data dependant,
172 * so we put an explicit smp_rmb() here. */
173 smp_rmb();
Mathieu Desnoyers58336112008-03-24 12:29:49 -0700174 func = mdata->single.func;
Mathieu Desnoyersfb40bd72008-02-13 15:03:37 -0800175 /* Must read the ptr before private data. They are not data
176 * dependant, so we put an explicit smp_rmb() here. */
177 smp_rmb();
Mathieu Desnoyersdc102a82008-05-12 21:21:09 +0200178 func(mdata->single.probe_private, call_private, mdata->format,
179 &args);
Mathieu Desnoyersfb40bd72008-02-13 15:03:37 -0800180 } else {
181 struct marker_probe_closure *multi;
182 int i;
183 /*
Mathieu Desnoyers5def9a32008-07-29 22:33:31 -0700184 * Read mdata->ptype before mdata->multi.
185 */
186 smp_rmb();
187 multi = mdata->multi;
188 /*
Mathieu Desnoyersfb40bd72008-02-13 15:03:37 -0800189 * multi points to an array, therefore accessing the array
190 * depends on reading multi. However, even in this case,
191 * we must insure that the pointer is read _before_ the array
192 * data. Same as rcu_dereference, but we need a full smp_rmb()
193 * in the fast path, so put the explicit barrier here.
194 */
195 smp_read_barrier_depends();
Mathieu Desnoyersfb40bd72008-02-13 15:03:37 -0800196 for (i = 0; multi[i].func; i++)
Mathieu Desnoyersdc102a82008-05-12 21:21:09 +0200197 multi[i].func(multi[i].probe_private, call_private,
198 mdata->format, &args);
Mathieu Desnoyersfb40bd72008-02-13 15:03:37 -0800199 }
Mathieu Desnoyers021aeb02008-11-14 17:47:37 -0500200 rcu_read_unlock_sched_notrace();
Mathieu Desnoyersfb40bd72008-02-13 15:03:37 -0800201}
Mathieu Desnoyersfb40bd72008-02-13 15:03:37 -0800202
Mathieu Desnoyersed86a592008-10-01 12:03:25 -0400203static void free_old_closure(struct rcu_head *head)
204{
205 struct marker_entry *entry = container_of(head,
206 struct marker_entry, rcu);
207 kfree(entry->oldptr);
208 /* Make sure we free the data before setting the pending flag to 0 */
209 smp_wmb();
210 entry->rcu_pending = 0;
211}
212
Mathieu Desnoyersfb40bd72008-02-13 15:03:37 -0800213static void debug_print_probes(struct marker_entry *entry)
214{
215 int i;
216
217 if (!marker_debug)
218 return;
219
220 if (!entry->ptype) {
221 printk(KERN_DEBUG "Single probe : %p %p\n",
222 entry->single.func,
223 entry->single.probe_private);
224 } else {
225 for (i = 0; entry->multi[i].func; i++)
226 printk(KERN_DEBUG "Multi probe %d : %p %p\n", i,
227 entry->multi[i].func,
228 entry->multi[i].probe_private);
229 }
230}
231
232static struct marker_probe_closure *
233marker_entry_add_probe(struct marker_entry *entry,
234 marker_probe_func *probe, void *probe_private)
235{
236 int nr_probes = 0;
237 struct marker_probe_closure *old, *new;
238
239 WARN_ON(!probe);
240
241 debug_print_probes(entry);
242 old = entry->multi;
243 if (!entry->ptype) {
244 if (entry->single.func == probe &&
245 entry->single.probe_private == probe_private)
246 return ERR_PTR(-EBUSY);
247 if (entry->single.func == __mark_empty_function) {
248 /* 0 -> 1 probes */
249 entry->single.func = probe;
250 entry->single.probe_private = probe_private;
251 entry->refcount = 1;
252 entry->ptype = 0;
253 debug_print_probes(entry);
254 return NULL;
255 } else {
256 /* 1 -> 2 probes */
257 nr_probes = 1;
258 old = NULL;
259 }
260 } else {
261 /* (N -> N+1), (N != 0, 1) probes */
262 for (nr_probes = 0; old[nr_probes].func; nr_probes++)
263 if (old[nr_probes].func == probe
264 && old[nr_probes].probe_private
265 == probe_private)
266 return ERR_PTR(-EBUSY);
267 }
268 /* + 2 : one for new probe, one for NULL func */
269 new = kzalloc((nr_probes + 2) * sizeof(struct marker_probe_closure),
270 GFP_KERNEL);
271 if (new == NULL)
272 return ERR_PTR(-ENOMEM);
273 if (!old)
274 new[0] = entry->single;
275 else
276 memcpy(new, old,
277 nr_probes * sizeof(struct marker_probe_closure));
278 new[nr_probes].func = probe;
279 new[nr_probes].probe_private = probe_private;
280 entry->refcount = nr_probes + 1;
281 entry->multi = new;
282 entry->ptype = 1;
283 debug_print_probes(entry);
284 return old;
285}
286
287static struct marker_probe_closure *
288marker_entry_remove_probe(struct marker_entry *entry,
289 marker_probe_func *probe, void *probe_private)
290{
291 int nr_probes = 0, nr_del = 0, i;
292 struct marker_probe_closure *old, *new;
293
294 old = entry->multi;
295
296 debug_print_probes(entry);
297 if (!entry->ptype) {
298 /* 0 -> N is an error */
299 WARN_ON(entry->single.func == __mark_empty_function);
300 /* 1 -> 0 probes */
301 WARN_ON(probe && entry->single.func != probe);
302 WARN_ON(entry->single.probe_private != probe_private);
303 entry->single.func = __mark_empty_function;
304 entry->refcount = 0;
305 entry->ptype = 0;
306 debug_print_probes(entry);
307 return NULL;
308 } else {
309 /* (N -> M), (N > 1, M >= 0) probes */
310 for (nr_probes = 0; old[nr_probes].func; nr_probes++) {
311 if ((!probe || old[nr_probes].func == probe)
312 && old[nr_probes].probe_private
313 == probe_private)
314 nr_del++;
315 }
316 }
317
318 if (nr_probes - nr_del == 0) {
319 /* N -> 0, (N > 1) */
320 entry->single.func = __mark_empty_function;
321 entry->refcount = 0;
322 entry->ptype = 0;
323 } else if (nr_probes - nr_del == 1) {
324 /* N -> 1, (N > 1) */
325 for (i = 0; old[i].func; i++)
326 if ((probe && old[i].func != probe) ||
327 old[i].probe_private != probe_private)
328 entry->single = old[i];
329 entry->refcount = 1;
330 entry->ptype = 0;
331 } else {
332 int j = 0;
333 /* N -> M, (N > 1, M > 1) */
334 /* + 1 for NULL */
335 new = kzalloc((nr_probes - nr_del + 1)
336 * sizeof(struct marker_probe_closure), GFP_KERNEL);
337 if (new == NULL)
338 return ERR_PTR(-ENOMEM);
339 for (i = 0; old[i].func; i++)
340 if ((probe && old[i].func != probe) ||
341 old[i].probe_private != probe_private)
342 new[j++] = old[i];
343 entry->refcount = nr_probes - nr_del;
344 entry->ptype = 1;
345 entry->multi = new;
346 }
347 debug_print_probes(entry);
348 return old;
349}
350
351/*
Mathieu Desnoyers8256e472007-10-18 23:41:06 -0700352 * Get marker if the marker is present in the marker hash table.
353 * Must be called with markers_mutex held.
354 * Returns NULL if not present.
355 */
356static struct marker_entry *get_marker(const char *name)
357{
358 struct hlist_head *head;
359 struct hlist_node *node;
360 struct marker_entry *e;
361 u32 hash = jhash(name, strlen(name), 0);
362
363 head = &marker_table[hash & ((1 << MARKER_HASH_BITS)-1)];
364 hlist_for_each_entry(e, node, head, hlist) {
365 if (!strcmp(name, e->name))
366 return e;
367 }
368 return NULL;
369}
370
371/*
372 * Add the marker to the marker hash table. Must be called with markers_mutex
373 * held.
374 */
Mathieu Desnoyersfb40bd72008-02-13 15:03:37 -0800375static struct marker_entry *add_marker(const char *name, const char *format)
Mathieu Desnoyers8256e472007-10-18 23:41:06 -0700376{
377 struct hlist_head *head;
378 struct hlist_node *node;
379 struct marker_entry *e;
380 size_t name_len = strlen(name) + 1;
381 size_t format_len = 0;
382 u32 hash = jhash(name, name_len-1, 0);
383
384 if (format)
385 format_len = strlen(format) + 1;
386 head = &marker_table[hash & ((1 << MARKER_HASH_BITS)-1)];
387 hlist_for_each_entry(e, node, head, hlist) {
388 if (!strcmp(name, e->name)) {
389 printk(KERN_NOTICE
Mathieu Desnoyersfb40bd72008-02-13 15:03:37 -0800390 "Marker %s busy\n", name);
391 return ERR_PTR(-EBUSY); /* Already there */
Mathieu Desnoyers8256e472007-10-18 23:41:06 -0700392 }
393 }
394 /*
395 * Using kmalloc here to allocate a variable length element. Could
396 * cause some memory fragmentation if overused.
397 */
398 e = kmalloc(sizeof(struct marker_entry) + name_len + format_len,
399 GFP_KERNEL);
400 if (!e)
Mathieu Desnoyersfb40bd72008-02-13 15:03:37 -0800401 return ERR_PTR(-ENOMEM);
Mathieu Desnoyers8256e472007-10-18 23:41:06 -0700402 memcpy(&e->name[0], name, name_len);
403 if (format) {
404 e->format = &e->name[name_len];
405 memcpy(e->format, format, format_len);
Mathieu Desnoyersfb40bd72008-02-13 15:03:37 -0800406 if (strcmp(e->format, MARK_NOARGS) == 0)
407 e->call = marker_probe_cb_noarg;
408 else
409 e->call = marker_probe_cb;
Mathieu Desnoyers8256e472007-10-18 23:41:06 -0700410 trace_mark(core_marker_format, "name %s format %s",
411 e->name, e->format);
Mathieu Desnoyersfb40bd72008-02-13 15:03:37 -0800412 } else {
Mathieu Desnoyers8256e472007-10-18 23:41:06 -0700413 e->format = NULL;
Mathieu Desnoyersfb40bd72008-02-13 15:03:37 -0800414 e->call = marker_probe_cb;
415 }
416 e->single.func = __mark_empty_function;
417 e->single.probe_private = NULL;
418 e->multi = NULL;
419 e->ptype = 0;
Lai Jiangshan0eec4812008-10-15 14:56:37 +0800420 e->format_allocated = 0;
Mathieu Desnoyers8256e472007-10-18 23:41:06 -0700421 e->refcount = 0;
Mathieu Desnoyersed86a592008-10-01 12:03:25 -0400422 e->rcu_pending = 0;
Mathieu Desnoyers8256e472007-10-18 23:41:06 -0700423 hlist_add_head(&e->hlist, head);
Mathieu Desnoyersfb40bd72008-02-13 15:03:37 -0800424 return e;
Mathieu Desnoyers8256e472007-10-18 23:41:06 -0700425}
426
427/*
428 * Remove the marker from the marker hash table. Must be called with mutex_lock
429 * held.
430 */
Mathieu Desnoyersfb40bd72008-02-13 15:03:37 -0800431static int remove_marker(const char *name)
Mathieu Desnoyers8256e472007-10-18 23:41:06 -0700432{
433 struct hlist_head *head;
434 struct hlist_node *node;
435 struct marker_entry *e;
436 int found = 0;
437 size_t len = strlen(name) + 1;
Mathieu Desnoyers8256e472007-10-18 23:41:06 -0700438 u32 hash = jhash(name, len-1, 0);
439
440 head = &marker_table[hash & ((1 << MARKER_HASH_BITS)-1)];
441 hlist_for_each_entry(e, node, head, hlist) {
442 if (!strcmp(name, e->name)) {
443 found = 1;
444 break;
445 }
446 }
Mathieu Desnoyersfb40bd72008-02-13 15:03:37 -0800447 if (!found)
448 return -ENOENT;
449 if (e->single.func != __mark_empty_function)
450 return -EBUSY;
451 hlist_del(&e->hlist);
Lai Jiangshan0eec4812008-10-15 14:56:37 +0800452 if (e->format_allocated)
453 kfree(e->format);
Mathieu Desnoyersed86a592008-10-01 12:03:25 -0400454 /* Make sure the call_rcu has been executed */
455 if (e->rcu_pending)
456 rcu_barrier_sched();
Mathieu Desnoyersfb40bd72008-02-13 15:03:37 -0800457 kfree(e);
458 return 0;
Mathieu Desnoyers8256e472007-10-18 23:41:06 -0700459}
460
461/*
462 * Set the mark_entry format to the format found in the element.
463 */
Lai Jiangshan0eec4812008-10-15 14:56:37 +0800464static int marker_set_format(struct marker_entry *entry, const char *format)
Mathieu Desnoyers8256e472007-10-18 23:41:06 -0700465{
Lai Jiangshan0eec4812008-10-15 14:56:37 +0800466 entry->format = kstrdup(format, GFP_KERNEL);
467 if (!entry->format)
Mathieu Desnoyers8256e472007-10-18 23:41:06 -0700468 return -ENOMEM;
Lai Jiangshan0eec4812008-10-15 14:56:37 +0800469 entry->format_allocated = 1;
470
Mathieu Desnoyers8256e472007-10-18 23:41:06 -0700471 trace_mark(core_marker_format, "name %s format %s",
Lai Jiangshan0eec4812008-10-15 14:56:37 +0800472 entry->name, entry->format);
Mathieu Desnoyers8256e472007-10-18 23:41:06 -0700473 return 0;
474}
475
476/*
477 * Sets the probe callback corresponding to one marker.
478 */
Lai Jiangshan0eec4812008-10-15 14:56:37 +0800479static int set_marker(struct marker_entry *entry, struct marker *elem,
Mathieu Desnoyersfb40bd72008-02-13 15:03:37 -0800480 int active)
Mathieu Desnoyers8256e472007-10-18 23:41:06 -0700481{
Mathieu Desnoyersc1df1bd2008-11-14 17:47:39 -0500482 int ret = 0;
Lai Jiangshan0eec4812008-10-15 14:56:37 +0800483 WARN_ON(strcmp(entry->name, elem->name) != 0);
Mathieu Desnoyers8256e472007-10-18 23:41:06 -0700484
Lai Jiangshan0eec4812008-10-15 14:56:37 +0800485 if (entry->format) {
486 if (strcmp(entry->format, elem->format) != 0) {
Mathieu Desnoyers8256e472007-10-18 23:41:06 -0700487 printk(KERN_NOTICE
488 "Format mismatch for probe %s "
489 "(%s), marker (%s)\n",
Lai Jiangshan0eec4812008-10-15 14:56:37 +0800490 entry->name,
491 entry->format,
Mathieu Desnoyers8256e472007-10-18 23:41:06 -0700492 elem->format);
493 return -EPERM;
494 }
495 } else {
496 ret = marker_set_format(entry, elem->format);
497 if (ret)
498 return ret;
499 }
Mathieu Desnoyersfb40bd72008-02-13 15:03:37 -0800500
501 /*
502 * probe_cb setup (statically known) is done here. It is
503 * asynchronous with the rest of execution, therefore we only
504 * pass from a "safe" callback (with argument) to an "unsafe"
505 * callback (does not set arguments).
506 */
Lai Jiangshan0eec4812008-10-15 14:56:37 +0800507 elem->call = entry->call;
Mathieu Desnoyersfb40bd72008-02-13 15:03:37 -0800508 /*
509 * Sanity check :
510 * We only update the single probe private data when the ptr is
511 * set to a _non_ single probe! (0 -> 1 and N -> 1, N != 1)
512 */
513 WARN_ON(elem->single.func != __mark_empty_function
Lai Jiangshan0eec4812008-10-15 14:56:37 +0800514 && elem->single.probe_private != entry->single.probe_private
515 && !elem->ptype);
516 elem->single.probe_private = entry->single.probe_private;
Mathieu Desnoyersfb40bd72008-02-13 15:03:37 -0800517 /*
518 * Make sure the private data is valid when we update the
519 * single probe ptr.
520 */
521 smp_wmb();
Lai Jiangshan0eec4812008-10-15 14:56:37 +0800522 elem->single.func = entry->single.func;
Mathieu Desnoyersfb40bd72008-02-13 15:03:37 -0800523 /*
524 * We also make sure that the new probe callbacks array is consistent
525 * before setting a pointer to it.
526 */
Lai Jiangshan0eec4812008-10-15 14:56:37 +0800527 rcu_assign_pointer(elem->multi, entry->multi);
Mathieu Desnoyersfb40bd72008-02-13 15:03:37 -0800528 /*
529 * Update the function or multi probe array pointer before setting the
530 * ptype.
531 */
532 smp_wmb();
Lai Jiangshan0eec4812008-10-15 14:56:37 +0800533 elem->ptype = entry->ptype;
Mathieu Desnoyersc1df1bd2008-11-14 17:47:39 -0500534
535 if (elem->tp_name && (active ^ elem->state)) {
536 WARN_ON(!elem->tp_cb);
537 /*
538 * It is ok to directly call the probe registration because type
539 * checking has been done in the __trace_mark_tp() macro.
540 */
541
542 if (active) {
543 /*
544 * try_module_get should always succeed because we hold
545 * lock_module() to get the tp_cb address.
546 */
547 ret = try_module_get(__module_text_address(
548 (unsigned long)elem->tp_cb));
549 BUG_ON(!ret);
550 ret = tracepoint_probe_register_noupdate(
551 elem->tp_name,
552 elem->tp_cb);
553 } else {
554 ret = tracepoint_probe_unregister_noupdate(
555 elem->tp_name,
556 elem->tp_cb);
557 /*
558 * tracepoint_probe_update_all() must be called
559 * before the module containing tp_cb is unloaded.
560 */
561 module_put(__module_text_address(
562 (unsigned long)elem->tp_cb));
563 }
564 }
Mathieu Desnoyersfb40bd72008-02-13 15:03:37 -0800565 elem->state = active;
566
Mathieu Desnoyersc1df1bd2008-11-14 17:47:39 -0500567 return ret;
Mathieu Desnoyers8256e472007-10-18 23:41:06 -0700568}
569
570/*
571 * Disable a marker and its probe callback.
Mathieu Desnoyersfd3c36f2008-03-24 12:29:47 -0700572 * Note: only waiting an RCU period after setting elem->call to the empty
573 * function insures that the original callback is not used anymore. This insured
Mathieu Desnoyerse2d3b75d2008-09-29 11:08:03 -0400574 * by rcu_read_lock_sched around the call site.
Mathieu Desnoyers8256e472007-10-18 23:41:06 -0700575 */
576static void disable_marker(struct marker *elem)
577{
Mathieu Desnoyersc1df1bd2008-11-14 17:47:39 -0500578 int ret;
579
Mathieu Desnoyersfb40bd72008-02-13 15:03:37 -0800580 /* leave "call" as is. It is known statically. */
Mathieu Desnoyersc1df1bd2008-11-14 17:47:39 -0500581 if (elem->tp_name && elem->state) {
582 WARN_ON(!elem->tp_cb);
583 /*
584 * It is ok to directly call the probe registration because type
585 * checking has been done in the __trace_mark_tp() macro.
586 */
587 ret = tracepoint_probe_unregister_noupdate(elem->tp_name,
588 elem->tp_cb);
589 WARN_ON(ret);
590 /*
591 * tracepoint_probe_update_all() must be called
592 * before the module containing tp_cb is unloaded.
593 */
594 module_put(__module_text_address((unsigned long)elem->tp_cb));
595 }
Mathieu Desnoyers8256e472007-10-18 23:41:06 -0700596 elem->state = 0;
Mathieu Desnoyersfb40bd72008-02-13 15:03:37 -0800597 elem->single.func = __mark_empty_function;
598 /* Update the function before setting the ptype */
599 smp_wmb();
600 elem->ptype = 0; /* single probe */
Mathieu Desnoyers8256e472007-10-18 23:41:06 -0700601 /*
602 * Leave the private data and id there, because removal is racy and
Mathieu Desnoyersfd3c36f2008-03-24 12:29:47 -0700603 * should be done only after an RCU period. These are never used until
604 * the next initialization anyway.
Mathieu Desnoyers8256e472007-10-18 23:41:06 -0700605 */
606}
607
608/**
609 * marker_update_probe_range - Update a probe range
610 * @begin: beginning of the range
611 * @end: end of the range
Mathieu Desnoyers8256e472007-10-18 23:41:06 -0700612 *
613 * Updates the probe callback corresponding to a range of markers.
Mathieu Desnoyers8256e472007-10-18 23:41:06 -0700614 */
615void marker_update_probe_range(struct marker *begin,
Mathieu Desnoyersfb40bd72008-02-13 15:03:37 -0800616 struct marker *end)
Mathieu Desnoyers8256e472007-10-18 23:41:06 -0700617{
618 struct marker *iter;
619 struct marker_entry *mark_entry;
620
Mathieu Desnoyers314de8a2007-11-14 16:59:48 -0800621 mutex_lock(&markers_mutex);
Mathieu Desnoyers8256e472007-10-18 23:41:06 -0700622 for (iter = begin; iter < end; iter++) {
623 mark_entry = get_marker(iter->name);
Mathieu Desnoyersfb40bd72008-02-13 15:03:37 -0800624 if (mark_entry) {
Lai Jiangshan0eec4812008-10-15 14:56:37 +0800625 set_marker(mark_entry, iter, !!mark_entry->refcount);
Mathieu Desnoyers8256e472007-10-18 23:41:06 -0700626 /*
627 * ignore error, continue
628 */
Mathieu Desnoyers8256e472007-10-18 23:41:06 -0700629 } else {
630 disable_marker(iter);
631 }
632 }
Mathieu Desnoyers314de8a2007-11-14 16:59:48 -0800633 mutex_unlock(&markers_mutex);
Mathieu Desnoyers8256e472007-10-18 23:41:06 -0700634}
635
636/*
637 * Update probes, removing the faulty probes.
Mathieu Desnoyersfb40bd72008-02-13 15:03:37 -0800638 *
639 * Internal callback only changed before the first probe is connected to it.
640 * Single probe private data can only be changed on 0 -> 1 and 2 -> 1
641 * transitions. All other transitions will leave the old private data valid.
642 * This makes the non-atomicity of the callback/private data updates valid.
643 *
644 * "special case" updates :
645 * 0 -> 1 callback
646 * 1 -> 0 callback
647 * 1 -> 2 callbacks
648 * 2 -> 1 callbacks
649 * Other updates all behave the same, just like the 2 -> 3 or 3 -> 2 updates.
650 * Site effect : marker_set_format may delete the marker entry (creating a
651 * replacement).
Mathieu Desnoyers8256e472007-10-18 23:41:06 -0700652 */
Mathieu Desnoyersfb40bd72008-02-13 15:03:37 -0800653static void marker_update_probes(void)
Mathieu Desnoyers8256e472007-10-18 23:41:06 -0700654{
Mathieu Desnoyers8256e472007-10-18 23:41:06 -0700655 /* Core kernel markers */
Mathieu Desnoyersfb40bd72008-02-13 15:03:37 -0800656 marker_update_probe_range(__start___markers, __stop___markers);
Mathieu Desnoyers8256e472007-10-18 23:41:06 -0700657 /* Markers in modules. */
Mathieu Desnoyersfb40bd72008-02-13 15:03:37 -0800658 module_update_markers();
Mathieu Desnoyersc1df1bd2008-11-14 17:47:39 -0500659 tracepoint_probe_update_all();
Mathieu Desnoyers8256e472007-10-18 23:41:06 -0700660}
661
662/**
663 * marker_probe_register - Connect a probe to a marker
664 * @name: marker name
665 * @format: format string
666 * @probe: probe handler
Mathieu Desnoyersfb40bd72008-02-13 15:03:37 -0800667 * @probe_private: probe private data
Mathieu Desnoyers8256e472007-10-18 23:41:06 -0700668 *
669 * private data must be a valid allocated memory address, or NULL.
670 * Returns 0 if ok, error value on error.
Mathieu Desnoyersfb40bd72008-02-13 15:03:37 -0800671 * The probe address must at least be aligned on the architecture pointer size.
Mathieu Desnoyers8256e472007-10-18 23:41:06 -0700672 */
673int marker_probe_register(const char *name, const char *format,
Mathieu Desnoyersfb40bd72008-02-13 15:03:37 -0800674 marker_probe_func *probe, void *probe_private)
Mathieu Desnoyers8256e472007-10-18 23:41:06 -0700675{
676 struct marker_entry *entry;
Mathieu Desnoyers314de8a2007-11-14 16:59:48 -0800677 int ret = 0;
Mathieu Desnoyersfb40bd72008-02-13 15:03:37 -0800678 struct marker_probe_closure *old;
Mathieu Desnoyers8256e472007-10-18 23:41:06 -0700679
680 mutex_lock(&markers_mutex);
681 entry = get_marker(name);
Mathieu Desnoyersfb40bd72008-02-13 15:03:37 -0800682 if (!entry) {
683 entry = add_marker(name, format);
Lai Jiangshan48043bc2008-10-08 10:23:36 +0800684 if (IS_ERR(entry))
Mathieu Desnoyersfb40bd72008-02-13 15:03:37 -0800685 ret = PTR_ERR(entry);
Lai Jiangshan48043bc2008-10-08 10:23:36 +0800686 } else if (format) {
687 if (!entry->format)
Lai Jiangshan0eec4812008-10-15 14:56:37 +0800688 ret = marker_set_format(entry, format);
Lai Jiangshan48043bc2008-10-08 10:23:36 +0800689 else if (strcmp(entry->format, format))
690 ret = -EPERM;
Mathieu Desnoyersfb40bd72008-02-13 15:03:37 -0800691 }
Lai Jiangshan48043bc2008-10-08 10:23:36 +0800692 if (ret)
693 goto end;
694
Mathieu Desnoyersed86a592008-10-01 12:03:25 -0400695 /*
696 * If we detect that a call_rcu is pending for this marker,
697 * make sure it's executed now.
698 */
699 if (entry->rcu_pending)
700 rcu_barrier_sched();
Mathieu Desnoyersfb40bd72008-02-13 15:03:37 -0800701 old = marker_entry_add_probe(entry, probe, probe_private);
702 if (IS_ERR(old)) {
703 ret = PTR_ERR(old);
Mathieu Desnoyers8256e472007-10-18 23:41:06 -0700704 goto end;
705 }
Mathieu Desnoyers314de8a2007-11-14 16:59:48 -0800706 mutex_unlock(&markers_mutex);
Mathieu Desnoyers2bdba312008-11-14 17:47:35 -0500707 marker_update_probes();
Mathieu Desnoyersfb40bd72008-02-13 15:03:37 -0800708 mutex_lock(&markers_mutex);
709 entry = get_marker(name);
Mathieu Desnoyers2bdba312008-11-14 17:47:35 -0500710 if (!entry)
711 goto end;
Mathieu Desnoyersed86a592008-10-01 12:03:25 -0400712 if (entry->rcu_pending)
713 rcu_barrier_sched();
714 entry->oldptr = old;
715 entry->rcu_pending = 1;
716 /* write rcu_pending before calling the RCU callback */
717 smp_wmb();
718 call_rcu_sched(&entry->rcu, free_old_closure);
Mathieu Desnoyers8256e472007-10-18 23:41:06 -0700719end:
720 mutex_unlock(&markers_mutex);
Mathieu Desnoyers8256e472007-10-18 23:41:06 -0700721 return ret;
722}
723EXPORT_SYMBOL_GPL(marker_probe_register);
724
725/**
726 * marker_probe_unregister - Disconnect a probe from a marker
727 * @name: marker name
Mathieu Desnoyersfb40bd72008-02-13 15:03:37 -0800728 * @probe: probe function pointer
729 * @probe_private: probe private data
Mathieu Desnoyers8256e472007-10-18 23:41:06 -0700730 *
731 * Returns the private data given to marker_probe_register, or an ERR_PTR().
Mathieu Desnoyersfb40bd72008-02-13 15:03:37 -0800732 * We do not need to call a synchronize_sched to make sure the probes have
733 * finished running before doing a module unload, because the module unload
734 * itself uses stop_machine(), which insures that every preempt disabled section
735 * have finished.
Mathieu Desnoyers8256e472007-10-18 23:41:06 -0700736 */
Mathieu Desnoyersfb40bd72008-02-13 15:03:37 -0800737int marker_probe_unregister(const char *name,
738 marker_probe_func *probe, void *probe_private)
Mathieu Desnoyers8256e472007-10-18 23:41:06 -0700739{
Mathieu Desnoyers8256e472007-10-18 23:41:06 -0700740 struct marker_entry *entry;
Mathieu Desnoyersfb40bd72008-02-13 15:03:37 -0800741 struct marker_probe_closure *old;
Jesper Juhl544adb42008-03-04 14:29:00 -0800742 int ret = -ENOENT;
Mathieu Desnoyers8256e472007-10-18 23:41:06 -0700743
744 mutex_lock(&markers_mutex);
745 entry = get_marker(name);
Jesper Juhl544adb42008-03-04 14:29:00 -0800746 if (!entry)
Mathieu Desnoyers8256e472007-10-18 23:41:06 -0700747 goto end;
Mathieu Desnoyersed86a592008-10-01 12:03:25 -0400748 if (entry->rcu_pending)
749 rcu_barrier_sched();
Mathieu Desnoyersfb40bd72008-02-13 15:03:37 -0800750 old = marker_entry_remove_probe(entry, probe, probe_private);
Mathieu Desnoyers314de8a2007-11-14 16:59:48 -0800751 mutex_unlock(&markers_mutex);
Mathieu Desnoyers2bdba312008-11-14 17:47:35 -0500752 marker_update_probes();
Mathieu Desnoyersfb40bd72008-02-13 15:03:37 -0800753 mutex_lock(&markers_mutex);
754 entry = get_marker(name);
Jesper Juhl544adb42008-03-04 14:29:00 -0800755 if (!entry)
756 goto end;
Mathieu Desnoyersed86a592008-10-01 12:03:25 -0400757 if (entry->rcu_pending)
758 rcu_barrier_sched();
759 entry->oldptr = old;
760 entry->rcu_pending = 1;
761 /* write rcu_pending before calling the RCU callback */
762 smp_wmb();
763 call_rcu_sched(&entry->rcu, free_old_closure);
Mathieu Desnoyersfb40bd72008-02-13 15:03:37 -0800764 remove_marker(name); /* Ignore busy error message */
Jesper Juhl544adb42008-03-04 14:29:00 -0800765 ret = 0;
Mathieu Desnoyers8256e472007-10-18 23:41:06 -0700766end:
767 mutex_unlock(&markers_mutex);
Mathieu Desnoyersfb40bd72008-02-13 15:03:37 -0800768 return ret;
Mathieu Desnoyers8256e472007-10-18 23:41:06 -0700769}
770EXPORT_SYMBOL_GPL(marker_probe_unregister);
771
Mathieu Desnoyersfb40bd72008-02-13 15:03:37 -0800772static struct marker_entry *
773get_marker_from_private_data(marker_probe_func *probe, void *probe_private)
Mathieu Desnoyers8256e472007-10-18 23:41:06 -0700774{
Mathieu Desnoyersfb40bd72008-02-13 15:03:37 -0800775 struct marker_entry *entry;
776 unsigned int i;
Mathieu Desnoyers8256e472007-10-18 23:41:06 -0700777 struct hlist_head *head;
778 struct hlist_node *node;
Mathieu Desnoyers8256e472007-10-18 23:41:06 -0700779
Mathieu Desnoyers8256e472007-10-18 23:41:06 -0700780 for (i = 0; i < MARKER_TABLE_SIZE; i++) {
781 head = &marker_table[i];
782 hlist_for_each_entry(entry, node, head, hlist) {
Mathieu Desnoyersfb40bd72008-02-13 15:03:37 -0800783 if (!entry->ptype) {
784 if (entry->single.func == probe
785 && entry->single.probe_private
786 == probe_private)
787 return entry;
788 } else {
789 struct marker_probe_closure *closure;
790 closure = entry->multi;
791 for (i = 0; closure[i].func; i++) {
792 if (closure[i].func == probe &&
793 closure[i].probe_private
794 == probe_private)
795 return entry;
796 }
Mathieu Desnoyers8256e472007-10-18 23:41:06 -0700797 }
798 }
799 }
Mathieu Desnoyersfb40bd72008-02-13 15:03:37 -0800800 return NULL;
801}
802
803/**
804 * marker_probe_unregister_private_data - Disconnect a probe from a marker
805 * @probe: probe function
806 * @probe_private: probe private data
807 *
808 * Unregister a probe by providing the registered private data.
809 * Only removes the first marker found in hash table.
810 * Return 0 on success or error value.
811 * We do not need to call a synchronize_sched to make sure the probes have
812 * finished running before doing a module unload, because the module unload
813 * itself uses stop_machine(), which insures that every preempt disabled section
814 * have finished.
815 */
816int marker_probe_unregister_private_data(marker_probe_func *probe,
817 void *probe_private)
818{
819 struct marker_entry *entry;
820 int ret = 0;
821 struct marker_probe_closure *old;
822
823 mutex_lock(&markers_mutex);
824 entry = get_marker_from_private_data(probe, probe_private);
825 if (!entry) {
826 ret = -ENOENT;
Mathieu Desnoyers8256e472007-10-18 23:41:06 -0700827 goto end;
828 }
Mathieu Desnoyersed86a592008-10-01 12:03:25 -0400829 if (entry->rcu_pending)
830 rcu_barrier_sched();
Mathieu Desnoyersfb40bd72008-02-13 15:03:37 -0800831 old = marker_entry_remove_probe(entry, NULL, probe_private);
Mathieu Desnoyers314de8a2007-11-14 16:59:48 -0800832 mutex_unlock(&markers_mutex);
Mathieu Desnoyers2bdba312008-11-14 17:47:35 -0500833 marker_update_probes();
Mathieu Desnoyersfb40bd72008-02-13 15:03:37 -0800834 mutex_lock(&markers_mutex);
835 entry = get_marker_from_private_data(probe, probe_private);
Mathieu Desnoyers2bdba312008-11-14 17:47:35 -0500836 if (!entry)
837 goto end;
Mathieu Desnoyersed86a592008-10-01 12:03:25 -0400838 if (entry->rcu_pending)
839 rcu_barrier_sched();
840 entry->oldptr = old;
841 entry->rcu_pending = 1;
842 /* write rcu_pending before calling the RCU callback */
843 smp_wmb();
844 call_rcu_sched(&entry->rcu, free_old_closure);
Mathieu Desnoyersfb40bd72008-02-13 15:03:37 -0800845 remove_marker(entry->name); /* Ignore busy error message */
Mathieu Desnoyers8256e472007-10-18 23:41:06 -0700846end:
847 mutex_unlock(&markers_mutex);
Mathieu Desnoyersfb40bd72008-02-13 15:03:37 -0800848 return ret;
Mathieu Desnoyers8256e472007-10-18 23:41:06 -0700849}
850EXPORT_SYMBOL_GPL(marker_probe_unregister_private_data);
851
852/**
Mathieu Desnoyers8256e472007-10-18 23:41:06 -0700853 * marker_get_private_data - Get a marker's probe private data
854 * @name: marker name
Mathieu Desnoyersfb40bd72008-02-13 15:03:37 -0800855 * @probe: probe to match
856 * @num: get the nth matching probe's private data
Mathieu Desnoyers8256e472007-10-18 23:41:06 -0700857 *
Mathieu Desnoyersfb40bd72008-02-13 15:03:37 -0800858 * Returns the nth private data pointer (starting from 0) matching, or an
859 * ERR_PTR.
Mathieu Desnoyers8256e472007-10-18 23:41:06 -0700860 * Returns the private data pointer, or an ERR_PTR.
861 * The private data pointer should _only_ be dereferenced if the caller is the
862 * owner of the data, or its content could vanish. This is mostly used to
863 * confirm that a caller is the owner of a registered probe.
864 */
Mathieu Desnoyersfb40bd72008-02-13 15:03:37 -0800865void *marker_get_private_data(const char *name, marker_probe_func *probe,
866 int num)
Mathieu Desnoyers8256e472007-10-18 23:41:06 -0700867{
868 struct hlist_head *head;
869 struct hlist_node *node;
870 struct marker_entry *e;
871 size_t name_len = strlen(name) + 1;
872 u32 hash = jhash(name, name_len-1, 0);
Mathieu Desnoyersfb40bd72008-02-13 15:03:37 -0800873 int i;
Mathieu Desnoyers8256e472007-10-18 23:41:06 -0700874
875 head = &marker_table[hash & ((1 << MARKER_HASH_BITS)-1)];
876 hlist_for_each_entry(e, node, head, hlist) {
877 if (!strcmp(name, e->name)) {
Mathieu Desnoyersfb40bd72008-02-13 15:03:37 -0800878 if (!e->ptype) {
879 if (num == 0 && e->single.func == probe)
880 return e->single.probe_private;
Mathieu Desnoyersfb40bd72008-02-13 15:03:37 -0800881 } else {
882 struct marker_probe_closure *closure;
883 int match = 0;
884 closure = e->multi;
885 for (i = 0; closure[i].func; i++) {
886 if (closure[i].func != probe)
887 continue;
888 if (match++ == num)
889 return closure[i].probe_private;
890 }
891 }
Zhaolei5d9881e2008-10-22 11:38:01 +0800892 break;
Mathieu Desnoyers8256e472007-10-18 23:41:06 -0700893 }
894 }
895 return ERR_PTR(-ENOENT);
896}
897EXPORT_SYMBOL_GPL(marker_get_private_data);
Mathieu Desnoyersa4192462008-11-14 17:47:38 -0500898
Ingo Molnar227a8372008-11-16 09:50:34 +0100899#ifdef CONFIG_MODULES
900
Mathieu Desnoyersa4192462008-11-14 17:47:38 -0500901int marker_module_notify(struct notifier_block *self,
902 unsigned long val, void *data)
903{
904 struct module *mod = data;
905
906 switch (val) {
907 case MODULE_STATE_COMING:
908 marker_update_probe_range(mod->markers,
909 mod->markers + mod->num_markers);
910 break;
911 case MODULE_STATE_GOING:
912 marker_update_probe_range(mod->markers,
913 mod->markers + mod->num_markers);
914 break;
915 }
916 return 0;
917}
918
919struct notifier_block marker_module_nb = {
920 .notifier_call = marker_module_notify,
921 .priority = 0,
922};
923
924static int init_markers(void)
925{
926 return register_module_notifier(&marker_module_nb);
927}
928__initcall(init_markers);
Ingo Molnar227a8372008-11-16 09:50:34 +0100929
930#endif /* CONFIG_MODULES */