blob: 59bb2b9ec0e2e05494ffa2b814eb30c9a7a721d8 [file] [log] [blame]
Catalin Marinas3c7b4e62009-06-11 13:22:39 +01001/*
2 * mm/kmemleak.c
3 *
4 * Copyright (C) 2008 ARM Limited
5 * Written by Catalin Marinas <catalin.marinas@arm.com>
6 *
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License version 2 as
9 * published by the Free Software Foundation.
10 *
11 * This program is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
15 *
16 * You should have received a copy of the GNU General Public License
17 * along with this program; if not, write to the Free Software
18 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
19 *
20 *
21 * For more information on the algorithm and kmemleak usage, please see
22 * Documentation/kmemleak.txt.
23 *
24 * Notes on locking
25 * ----------------
26 *
27 * The following locks and mutexes are used by kmemleak:
28 *
29 * - kmemleak_lock (rwlock): protects the object_list modifications and
30 * accesses to the object_tree_root. The object_list is the main list
31 * holding the metadata (struct kmemleak_object) for the allocated memory
Michel Lespinasse85d3a312012-10-08 16:31:27 -070032 * blocks. The object_tree_root is a red black tree used to look-up
Catalin Marinas3c7b4e62009-06-11 13:22:39 +010033 * metadata based on a pointer to the corresponding memory block. The
34 * kmemleak_object structures are added to the object_list and
35 * object_tree_root in the create_object() function called from the
36 * kmemleak_alloc() callback and removed in delete_object() called from the
37 * kmemleak_free() callback
38 * - kmemleak_object.lock (spinlock): protects a kmemleak_object. Accesses to
39 * the metadata (e.g. count) are protected by this lock. Note that some
40 * members of this structure may be protected by other means (atomic or
41 * kmemleak_lock). This lock is also held when scanning the corresponding
42 * memory block to avoid the kernel freeing it via the kmemleak_free()
43 * callback. This is less heavyweight than holding a global lock like
44 * kmemleak_lock during scanning
45 * - scan_mutex (mutex): ensures that only one thread may scan the memory for
46 * unreferenced objects at a time. The gray_list contains the objects which
47 * are already referenced or marked as false positives and need to be
48 * scanned. This list is only modified during a scanning episode when the
49 * scan_mutex is held. At the end of a scan, the gray_list is always empty.
50 * Note that the kmemleak_object.use_count is incremented when an object is
Catalin Marinas4698c1f2009-06-26 17:38:27 +010051 * added to the gray_list and therefore cannot be freed. This mutex also
52 * prevents multiple users of the "kmemleak" debugfs file together with
53 * modifications to the memory scanning parameters including the scan_thread
54 * pointer
Catalin Marinas3c7b4e62009-06-11 13:22:39 +010055 *
Catalin Marinas93ada572015-06-24 16:58:37 -070056 * Locks and mutexes are acquired/nested in the following order:
Catalin Marinas9d5a4c72015-06-24 16:58:34 -070057 *
Catalin Marinas93ada572015-06-24 16:58:37 -070058 * scan_mutex [-> object->lock] -> kmemleak_lock -> other_object->lock (SINGLE_DEPTH_NESTING)
59 *
60 * No kmemleak_lock and object->lock nesting is allowed outside scan_mutex
61 * regions.
Catalin Marinas9d5a4c72015-06-24 16:58:34 -070062 *
Catalin Marinas3c7b4e62009-06-11 13:22:39 +010063 * The kmemleak_object structures have a use_count incremented or decremented
64 * using the get_object()/put_object() functions. When the use_count becomes
65 * 0, this count can no longer be incremented and put_object() schedules the
66 * kmemleak_object freeing via an RCU callback. All calls to the get_object()
67 * function must be protected by rcu_read_lock() to avoid accessing a freed
68 * structure.
69 */
70
Joe Perchesae281062009-06-23 14:40:26 +010071#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
72
Catalin Marinas3c7b4e62009-06-11 13:22:39 +010073#include <linux/init.h>
74#include <linux/kernel.h>
75#include <linux/list.h>
76#include <linux/sched.h>
77#include <linux/jiffies.h>
78#include <linux/delay.h>
Paul Gortmakerb95f1b312011-10-16 02:01:52 -040079#include <linux/export.h>
Catalin Marinas3c7b4e62009-06-11 13:22:39 +010080#include <linux/kthread.h>
Michel Lespinasse85d3a312012-10-08 16:31:27 -070081#include <linux/rbtree.h>
Catalin Marinas3c7b4e62009-06-11 13:22:39 +010082#include <linux/fs.h>
83#include <linux/debugfs.h>
84#include <linux/seq_file.h>
85#include <linux/cpumask.h>
86#include <linux/spinlock.h>
87#include <linux/mutex.h>
88#include <linux/rcupdate.h>
89#include <linux/stacktrace.h>
90#include <linux/cache.h>
91#include <linux/percpu.h>
92#include <linux/hardirq.h>
Catalin Marinas9099dae2016-10-11 13:55:11 -070093#include <linux/bootmem.h>
94#include <linux/pfn.h>
Catalin Marinas3c7b4e62009-06-11 13:22:39 +010095#include <linux/mmzone.h>
96#include <linux/slab.h>
97#include <linux/thread_info.h>
98#include <linux/err.h>
99#include <linux/uaccess.h>
100#include <linux/string.h>
101#include <linux/nodemask.h>
102#include <linux/mm.h>
Catalin Marinas179a8102009-09-07 10:14:42 +0100103#include <linux/workqueue.h>
Catalin Marinas04609ccc2009-10-28 13:33:12 +0000104#include <linux/crc32.h>
Catalin Marinas3c7b4e62009-06-11 13:22:39 +0100105
106#include <asm/sections.h>
107#include <asm/processor.h>
Arun Sharma600634972011-07-26 16:09:06 -0700108#include <linux/atomic.h>
Catalin Marinas3c7b4e62009-06-11 13:22:39 +0100109
Andrey Ryabinine79ed2f2015-02-13 14:39:49 -0800110#include <linux/kasan.h>
Pekka Enberg8e019362009-08-27 14:50:00 +0100111#include <linux/kmemcheck.h>
Catalin Marinas3c7b4e62009-06-11 13:22:39 +0100112#include <linux/kmemleak.h>
Laura Abbott029aeff2011-11-15 23:49:09 +0000113#include <linux/memory_hotplug.h>
Catalin Marinas3c7b4e62009-06-11 13:22:39 +0100114
115/*
116 * Kmemleak configuration and common defines.
117 */
118#define MAX_TRACE 16 /* stack trace length */
Catalin Marinas3c7b4e62009-06-11 13:22:39 +0100119#define MSECS_MIN_AGE 5000 /* minimum object age for reporting */
Catalin Marinas3c7b4e62009-06-11 13:22:39 +0100120#define SECS_FIRST_SCAN 60 /* delay before the first scan */
121#define SECS_SCAN_WAIT 600 /* subsequent auto scanning delay */
Catalin Marinasaf986032009-08-27 14:29:12 +0100122#define MAX_SCAN_SIZE 4096 /* maximum size of a scanned block */
Catalin Marinas3c7b4e62009-06-11 13:22:39 +0100123
124#define BYTES_PER_POINTER sizeof(void *)
125
Catalin Marinas216c04b2009-06-17 18:29:02 +0100126/* GFP bitmask for kmemleak internal allocations */
Vladimir Davydov20b5c302016-01-14 15:18:08 -0800127#define gfp_kmemleak_mask(gfp) (((gfp) & (GFP_KERNEL | GFP_ATOMIC)) | \
Catalin Marinas6ae4bd12011-01-27 10:30:26 +0000128 __GFP_NORETRY | __GFP_NOMEMALLOC | \
129 __GFP_NOWARN)
Catalin Marinas216c04b2009-06-17 18:29:02 +0100130
Catalin Marinas3c7b4e62009-06-11 13:22:39 +0100131/* scanning area inside a memory block */
132struct kmemleak_scan_area {
133 struct hlist_node node;
Catalin Marinasc017b4b2009-10-28 13:33:09 +0000134 unsigned long start;
135 size_t size;
Catalin Marinas3c7b4e62009-06-11 13:22:39 +0100136};
137
Luis R. Rodrigueza1084c82009-09-04 17:44:52 -0700138#define KMEMLEAK_GREY 0
139#define KMEMLEAK_BLACK -1
140
Catalin Marinas3c7b4e62009-06-11 13:22:39 +0100141/*
142 * Structure holding the metadata for each allocated memory block.
143 * Modifications to such objects should be made while holding the
144 * object->lock. Insertions or deletions from object_list, gray_list or
Michel Lespinasse85d3a312012-10-08 16:31:27 -0700145 * rb_node are already protected by the corresponding locks or mutex (see
Catalin Marinas3c7b4e62009-06-11 13:22:39 +0100146 * the notes on locking above). These objects are reference-counted
147 * (use_count) and freed using the RCU mechanism.
148 */
149struct kmemleak_object {
150 spinlock_t lock;
151 unsigned long flags; /* object status flags */
152 struct list_head object_list;
153 struct list_head gray_list;
Michel Lespinasse85d3a312012-10-08 16:31:27 -0700154 struct rb_node rb_node;
Catalin Marinas3c7b4e62009-06-11 13:22:39 +0100155 struct rcu_head rcu; /* object_list lockless traversal */
156 /* object usage count; object freed when use_count == 0 */
157 atomic_t use_count;
158 unsigned long pointer;
159 size_t size;
160 /* minimum number of a pointers found before it is considered leak */
161 int min_count;
162 /* the total number of pointers found pointing to this object */
163 int count;
Catalin Marinas04609ccc2009-10-28 13:33:12 +0000164 /* checksum for detecting modified objects */
165 u32 checksum;
Catalin Marinas3c7b4e62009-06-11 13:22:39 +0100166 /* memory ranges to be scanned inside an object (empty for all) */
167 struct hlist_head area_list;
168 unsigned long trace[MAX_TRACE];
169 unsigned int trace_len;
170 unsigned long jiffies; /* creation timestamp */
171 pid_t pid; /* pid of the current task */
172 char comm[TASK_COMM_LEN]; /* executable name */
173};
174
175/* flag representing the memory block allocation status */
176#define OBJECT_ALLOCATED (1 << 0)
177/* flag set after the first reporting of an unreference object */
178#define OBJECT_REPORTED (1 << 1)
179/* flag set to not scan the object */
180#define OBJECT_NO_SCAN (1 << 2)
181
Sergey Senozhatsky0494e082009-08-27 14:29:18 +0100182/* number of bytes to print per line; must be 16 or 32 */
183#define HEX_ROW_SIZE 16
184/* number of bytes to print at a time (1, 2, 4, 8) */
185#define HEX_GROUP_SIZE 1
186/* include ASCII after the hex output */
187#define HEX_ASCII 1
188/* max number of lines to be printed */
189#define HEX_MAX_LINES 2
190
Catalin Marinas3c7b4e62009-06-11 13:22:39 +0100191/* the list of all allocated objects */
192static LIST_HEAD(object_list);
193/* the list of gray-colored objects (see color_gray comment below) */
194static LIST_HEAD(gray_list);
Michel Lespinasse85d3a312012-10-08 16:31:27 -0700195/* search tree for object boundaries */
196static struct rb_root object_tree_root = RB_ROOT;
197/* rw_lock protecting the access to object_list and object_tree_root */
Catalin Marinas3c7b4e62009-06-11 13:22:39 +0100198static DEFINE_RWLOCK(kmemleak_lock);
199
200/* allocation caches for kmemleak internal data */
201static struct kmem_cache *object_cache;
202static struct kmem_cache *scan_area_cache;
203
204/* set if tracing memory operations is enabled */
Li Zefan8910ae82014-04-03 14:46:29 -0700205static int kmemleak_enabled;
Catalin Marinasc5f3b1a2015-06-24 16:58:26 -0700206/* same as above but only for the kmemleak_free() callback */
207static int kmemleak_free_enabled;
Catalin Marinas3c7b4e62009-06-11 13:22:39 +0100208/* set in the late_initcall if there were no errors */
Li Zefan8910ae82014-04-03 14:46:29 -0700209static int kmemleak_initialized;
Catalin Marinas3c7b4e62009-06-11 13:22:39 +0100210/* enables or disables early logging of the memory operations */
Li Zefan8910ae82014-04-03 14:46:29 -0700211static int kmemleak_early_log = 1;
Catalin Marinas5f790202011-09-28 12:17:03 +0100212/* set if a kmemleak warning was issued */
Li Zefan8910ae82014-04-03 14:46:29 -0700213static int kmemleak_warning;
Catalin Marinas5f790202011-09-28 12:17:03 +0100214/* set if a fatal kmemleak error has occurred */
Li Zefan8910ae82014-04-03 14:46:29 -0700215static int kmemleak_error;
Catalin Marinas3c7b4e62009-06-11 13:22:39 +0100216
217/* minimum and maximum address that may be valid pointers */
218static unsigned long min_addr = ULONG_MAX;
219static unsigned long max_addr;
220
Catalin Marinas3c7b4e62009-06-11 13:22:39 +0100221static struct task_struct *scan_thread;
Catalin Marinasacf49682009-06-26 17:38:29 +0100222/* used to avoid reporting of recently allocated objects */
Catalin Marinas3c7b4e62009-06-11 13:22:39 +0100223static unsigned long jiffies_min_age;
Catalin Marinasacf49682009-06-26 17:38:29 +0100224static unsigned long jiffies_last_scan;
Catalin Marinas3c7b4e62009-06-11 13:22:39 +0100225/* delay between automatic memory scannings */
226static signed long jiffies_scan_wait;
227/* enables or disables the task stacks scanning */
Catalin Marinase0a2a162009-06-26 17:38:25 +0100228static int kmemleak_stack_scan = 1;
Catalin Marinas4698c1f2009-06-26 17:38:27 +0100229/* protects the memory scanning, parameters and debug/kmemleak file access */
Catalin Marinas3c7b4e62009-06-11 13:22:39 +0100230static DEFINE_MUTEX(scan_mutex);
Jason Baronab0155a2010-07-19 11:54:17 +0100231/* setting kmemleak=on, will set this var, skipping the disable */
232static int kmemleak_skip_disable;
Li Zefandc9b3f42014-04-03 14:46:26 -0700233/* If there are leaks that can be reported */
234static bool kmemleak_found_leaks;
Catalin Marinas3c7b4e62009-06-11 13:22:39 +0100235
Catalin Marinas3c7b4e62009-06-11 13:22:39 +0100236/*
Catalin Marinas20301172009-06-17 18:29:04 +0100237 * Early object allocation/freeing logging. Kmemleak is initialized after the
Catalin Marinas3c7b4e62009-06-11 13:22:39 +0100238 * kernel allocator. However, both the kernel allocator and kmemleak may
Catalin Marinas20301172009-06-17 18:29:04 +0100239 * allocate memory blocks which need to be tracked. Kmemleak defines an
Catalin Marinas3c7b4e62009-06-11 13:22:39 +0100240 * arbitrary buffer to hold the allocation/freeing information before it is
241 * fully initialized.
242 */
243
244/* kmemleak operation type for early logging */
245enum {
246 KMEMLEAK_ALLOC,
Catalin Marinasf528f0b2011-09-26 17:12:53 +0100247 KMEMLEAK_ALLOC_PERCPU,
Catalin Marinas3c7b4e62009-06-11 13:22:39 +0100248 KMEMLEAK_FREE,
Catalin Marinas53238a62009-07-07 10:33:00 +0100249 KMEMLEAK_FREE_PART,
Catalin Marinasf528f0b2011-09-26 17:12:53 +0100250 KMEMLEAK_FREE_PERCPU,
Catalin Marinas3c7b4e62009-06-11 13:22:39 +0100251 KMEMLEAK_NOT_LEAK,
252 KMEMLEAK_IGNORE,
253 KMEMLEAK_SCAN_AREA,
254 KMEMLEAK_NO_SCAN
255};
256
257/*
258 * Structure holding the information passed to kmemleak callbacks during the
259 * early logging.
260 */
261struct early_log {
262 int op_type; /* kmemleak operation type */
263 const void *ptr; /* allocated/freed memory block */
264 size_t size; /* memory block size */
265 int min_count; /* minimum reference count */
Catalin Marinasfd678962009-08-27 14:29:17 +0100266 unsigned long trace[MAX_TRACE]; /* stack trace */
267 unsigned int trace_len; /* stack trace length */
Catalin Marinas3c7b4e62009-06-11 13:22:39 +0100268};
269
270/* early logging buffer and current position */
Catalin Marinasa6186d82009-08-27 14:29:16 +0100271static struct early_log
272 early_log[CONFIG_DEBUG_KMEMLEAK_EARLY_LOG_SIZE] __initdata;
273static int crt_early_log __initdata;
Catalin Marinas3c7b4e62009-06-11 13:22:39 +0100274
275static void kmemleak_disable(void);
276
277/*
278 * Print a warning and dump the stack trace.
279 */
Catalin Marinas5f790202011-09-28 12:17:03 +0100280#define kmemleak_warn(x...) do { \
Joe Perches598d8092016-03-17 14:19:44 -0700281 pr_warn(x); \
Catalin Marinas5f790202011-09-28 12:17:03 +0100282 dump_stack(); \
Li Zefan8910ae82014-04-03 14:46:29 -0700283 kmemleak_warning = 1; \
Catalin Marinas3c7b4e62009-06-11 13:22:39 +0100284} while (0)
285
286/*
Lucas De Marchi25985ed2011-03-30 22:57:33 -0300287 * Macro invoked when a serious kmemleak condition occurred and cannot be
Catalin Marinas20301172009-06-17 18:29:04 +0100288 * recovered from. Kmemleak will be disabled and further allocation/freeing
Catalin Marinas3c7b4e62009-06-11 13:22:39 +0100289 * tracing no longer available.
290 */
Catalin Marinas000814f2009-06-17 18:29:03 +0100291#define kmemleak_stop(x...) do { \
Catalin Marinas3c7b4e62009-06-11 13:22:39 +0100292 kmemleak_warn(x); \
293 kmemleak_disable(); \
294} while (0)
295
296/*
Sergey Senozhatsky0494e082009-08-27 14:29:18 +0100297 * Printing of the objects hex dump to the seq file. The number of lines to be
298 * printed is limited to HEX_MAX_LINES to prevent seq file spamming. The
299 * actual number of printed bytes depends on HEX_ROW_SIZE. It must be called
300 * with the object->lock held.
301 */
302static void hex_dump_object(struct seq_file *seq,
303 struct kmemleak_object *object)
304{
305 const u8 *ptr = (const u8 *)object->pointer;
Andy Shevchenko6fc37c42015-09-09 15:38:45 -0700306 size_t len;
Sergey Senozhatsky0494e082009-08-27 14:29:18 +0100307
308 /* limit the number of lines to HEX_MAX_LINES */
Andy Shevchenko6fc37c42015-09-09 15:38:45 -0700309 len = min_t(size_t, object->size, HEX_MAX_LINES * HEX_ROW_SIZE);
Sergey Senozhatsky0494e082009-08-27 14:29:18 +0100310
Andy Shevchenko6fc37c42015-09-09 15:38:45 -0700311 seq_printf(seq, " hex dump (first %zu bytes):\n", len);
Dmitry Vyukov5c335fe2016-06-24 14:50:07 -0700312 kasan_disable_current();
Andy Shevchenko6fc37c42015-09-09 15:38:45 -0700313 seq_hex_dump(seq, " ", DUMP_PREFIX_NONE, HEX_ROW_SIZE,
314 HEX_GROUP_SIZE, ptr, len, HEX_ASCII);
Dmitry Vyukov5c335fe2016-06-24 14:50:07 -0700315 kasan_enable_current();
Sergey Senozhatsky0494e082009-08-27 14:29:18 +0100316}
317
318/*
Catalin Marinas3c7b4e62009-06-11 13:22:39 +0100319 * Object colors, encoded with count and min_count:
320 * - white - orphan object, not enough references to it (count < min_count)
321 * - gray - not orphan, not marked as false positive (min_count == 0) or
322 * sufficient references to it (count >= min_count)
323 * - black - ignore, it doesn't contain references (e.g. text section)
324 * (min_count == -1). No function defined for this color.
325 * Newly created objects don't have any color assigned (object->count == -1)
326 * before the next memory scan when they become white.
327 */
Luis R. Rodriguez4a558dd2009-09-08 16:34:50 +0100328static bool color_white(const struct kmemleak_object *object)
Catalin Marinas3c7b4e62009-06-11 13:22:39 +0100329{
Luis R. Rodrigueza1084c82009-09-04 17:44:52 -0700330 return object->count != KMEMLEAK_BLACK &&
331 object->count < object->min_count;
Catalin Marinas3c7b4e62009-06-11 13:22:39 +0100332}
333
Luis R. Rodriguez4a558dd2009-09-08 16:34:50 +0100334static bool color_gray(const struct kmemleak_object *object)
Catalin Marinas3c7b4e62009-06-11 13:22:39 +0100335{
Luis R. Rodrigueza1084c82009-09-04 17:44:52 -0700336 return object->min_count != KMEMLEAK_BLACK &&
337 object->count >= object->min_count;
Catalin Marinas3c7b4e62009-06-11 13:22:39 +0100338}
339
340/*
Catalin Marinas3c7b4e62009-06-11 13:22:39 +0100341 * Objects are considered unreferenced only if their color is white, they have
342 * not be deleted and have a minimum age to avoid false positives caused by
343 * pointers temporarily stored in CPU registers.
344 */
Luis R. Rodriguez4a558dd2009-09-08 16:34:50 +0100345static bool unreferenced_object(struct kmemleak_object *object)
Catalin Marinas3c7b4e62009-06-11 13:22:39 +0100346{
Catalin Marinas04609ccc2009-10-28 13:33:12 +0000347 return (color_white(object) && object->flags & OBJECT_ALLOCATED) &&
Catalin Marinasacf49682009-06-26 17:38:29 +0100348 time_before_eq(object->jiffies + jiffies_min_age,
349 jiffies_last_scan);
Catalin Marinas3c7b4e62009-06-11 13:22:39 +0100350}
351
352/*
Catalin Marinasbab4a342009-06-26 17:38:26 +0100353 * Printing of the unreferenced objects information to the seq file. The
354 * print_unreferenced function must be called with the object->lock held.
Catalin Marinas3c7b4e62009-06-11 13:22:39 +0100355 */
Catalin Marinas3c7b4e62009-06-11 13:22:39 +0100356static void print_unreferenced(struct seq_file *seq,
357 struct kmemleak_object *object)
358{
359 int i;
Catalin Marinasfefdd332009-10-28 13:33:12 +0000360 unsigned int msecs_age = jiffies_to_msecs(jiffies - object->jiffies);
Catalin Marinas3c7b4e62009-06-11 13:22:39 +0100361
Catalin Marinasbab4a342009-06-26 17:38:26 +0100362 seq_printf(seq, "unreferenced object 0x%08lx (size %zu):\n",
363 object->pointer, object->size);
Catalin Marinasfefdd332009-10-28 13:33:12 +0000364 seq_printf(seq, " comm \"%s\", pid %d, jiffies %lu (age %d.%03ds)\n",
365 object->comm, object->pid, object->jiffies,
366 msecs_age / 1000, msecs_age % 1000);
Sergey Senozhatsky0494e082009-08-27 14:29:18 +0100367 hex_dump_object(seq, object);
Catalin Marinasbab4a342009-06-26 17:38:26 +0100368 seq_printf(seq, " backtrace:\n");
Catalin Marinas3c7b4e62009-06-11 13:22:39 +0100369
370 for (i = 0; i < object->trace_len; i++) {
371 void *ptr = (void *)object->trace[i];
Catalin Marinasbab4a342009-06-26 17:38:26 +0100372 seq_printf(seq, " [<%p>] %pS\n", ptr, ptr);
Catalin Marinas3c7b4e62009-06-11 13:22:39 +0100373 }
374}
375
376/*
377 * Print the kmemleak_object information. This function is used mainly for
378 * debugging special cases when kmemleak operations. It must be called with
379 * the object->lock held.
380 */
381static void dump_object_info(struct kmemleak_object *object)
382{
383 struct stack_trace trace;
384
385 trace.nr_entries = object->trace_len;
386 trace.entries = object->trace;
387
Joe Perchesae281062009-06-23 14:40:26 +0100388 pr_notice("Object 0x%08lx (size %zu):\n",
Michel Lespinasse85d3a312012-10-08 16:31:27 -0700389 object->pointer, object->size);
Catalin Marinas3c7b4e62009-06-11 13:22:39 +0100390 pr_notice(" comm \"%s\", pid %d, jiffies %lu\n",
391 object->comm, object->pid, object->jiffies);
392 pr_notice(" min_count = %d\n", object->min_count);
393 pr_notice(" count = %d\n", object->count);
Catalin Marinas189d84e2009-08-27 14:29:15 +0100394 pr_notice(" flags = 0x%lx\n", object->flags);
Jianpeng Maaae0ad72014-06-06 14:38:16 -0700395 pr_notice(" checksum = %u\n", object->checksum);
Catalin Marinas3c7b4e62009-06-11 13:22:39 +0100396 pr_notice(" backtrace:\n");
397 print_stack_trace(&trace, 4);
398}
399
400/*
Michel Lespinasse85d3a312012-10-08 16:31:27 -0700401 * Look-up a memory block metadata (kmemleak_object) in the object search
Catalin Marinas3c7b4e62009-06-11 13:22:39 +0100402 * tree based on a pointer value. If alias is 0, only values pointing to the
403 * beginning of the memory block are allowed. The kmemleak_lock must be held
404 * when calling this function.
405 */
406static struct kmemleak_object *lookup_object(unsigned long ptr, int alias)
407{
Michel Lespinasse85d3a312012-10-08 16:31:27 -0700408 struct rb_node *rb = object_tree_root.rb_node;
Catalin Marinas3c7b4e62009-06-11 13:22:39 +0100409
Michel Lespinasse85d3a312012-10-08 16:31:27 -0700410 while (rb) {
411 struct kmemleak_object *object =
412 rb_entry(rb, struct kmemleak_object, rb_node);
413 if (ptr < object->pointer)
414 rb = object->rb_node.rb_left;
415 else if (object->pointer + object->size <= ptr)
416 rb = object->rb_node.rb_right;
417 else if (object->pointer == ptr || alias)
418 return object;
419 else {
Catalin Marinas5f790202011-09-28 12:17:03 +0100420 kmemleak_warn("Found object by alias at 0x%08lx\n",
421 ptr);
Catalin Marinasa7686a42010-07-19 11:54:16 +0100422 dump_object_info(object);
Michel Lespinasse85d3a312012-10-08 16:31:27 -0700423 break;
Catalin Marinas3c7b4e62009-06-11 13:22:39 +0100424 }
Michel Lespinasse85d3a312012-10-08 16:31:27 -0700425 }
426 return NULL;
Catalin Marinas3c7b4e62009-06-11 13:22:39 +0100427}
428
429/*
430 * Increment the object use_count. Return 1 if successful or 0 otherwise. Note
431 * that once an object's use_count reached 0, the RCU freeing was already
432 * registered and the object should no longer be used. This function must be
433 * called under the protection of rcu_read_lock().
434 */
435static int get_object(struct kmemleak_object *object)
436{
437 return atomic_inc_not_zero(&object->use_count);
438}
439
440/*
441 * RCU callback to free a kmemleak_object.
442 */
443static void free_object_rcu(struct rcu_head *rcu)
444{
Sasha Levinb67bfe02013-02-27 17:06:00 -0800445 struct hlist_node *tmp;
Catalin Marinas3c7b4e62009-06-11 13:22:39 +0100446 struct kmemleak_scan_area *area;
447 struct kmemleak_object *object =
448 container_of(rcu, struct kmemleak_object, rcu);
449
450 /*
451 * Once use_count is 0 (guaranteed by put_object), there is no other
452 * code accessing this object, hence no need for locking.
453 */
Sasha Levinb67bfe02013-02-27 17:06:00 -0800454 hlist_for_each_entry_safe(area, tmp, &object->area_list, node) {
455 hlist_del(&area->node);
Catalin Marinas3c7b4e62009-06-11 13:22:39 +0100456 kmem_cache_free(scan_area_cache, area);
457 }
458 kmem_cache_free(object_cache, object);
459}
460
461/*
462 * Decrement the object use_count. Once the count is 0, free the object using
463 * an RCU callback. Since put_object() may be called via the kmemleak_free() ->
464 * delete_object() path, the delayed RCU freeing ensures that there is no
465 * recursive call to the kernel allocator. Lock-less RCU object_list traversal
466 * is also possible.
467 */
468static void put_object(struct kmemleak_object *object)
469{
470 if (!atomic_dec_and_test(&object->use_count))
471 return;
472
473 /* should only get here after delete_object was called */
474 WARN_ON(object->flags & OBJECT_ALLOCATED);
475
476 call_rcu(&object->rcu, free_object_rcu);
477}
478
479/*
Michel Lespinasse85d3a312012-10-08 16:31:27 -0700480 * Look up an object in the object search tree and increase its use_count.
Catalin Marinas3c7b4e62009-06-11 13:22:39 +0100481 */
482static struct kmemleak_object *find_and_get_object(unsigned long ptr, int alias)
483{
484 unsigned long flags;
Alexey Klimov9fbed252015-11-05 18:45:57 -0800485 struct kmemleak_object *object;
Catalin Marinas3c7b4e62009-06-11 13:22:39 +0100486
487 rcu_read_lock();
488 read_lock_irqsave(&kmemleak_lock, flags);
Catalin Marinas93ada572015-06-24 16:58:37 -0700489 object = lookup_object(ptr, alias);
Catalin Marinas3c7b4e62009-06-11 13:22:39 +0100490 read_unlock_irqrestore(&kmemleak_lock, flags);
491
492 /* check whether the object is still available */
493 if (object && !get_object(object))
494 object = NULL;
495 rcu_read_unlock();
496
497 return object;
498}
499
500/*
Catalin Marinase781a9a2015-06-24 16:58:29 -0700501 * Look up an object in the object search tree and remove it from both
502 * object_tree_root and object_list. The returned object's use_count should be
503 * at least 1, as initially set by create_object().
504 */
505static struct kmemleak_object *find_and_remove_object(unsigned long ptr, int alias)
506{
507 unsigned long flags;
508 struct kmemleak_object *object;
509
510 write_lock_irqsave(&kmemleak_lock, flags);
511 object = lookup_object(ptr, alias);
512 if (object) {
513 rb_erase(&object->rb_node, &object_tree_root);
514 list_del_rcu(&object->object_list);
515 }
516 write_unlock_irqrestore(&kmemleak_lock, flags);
517
518 return object;
519}
520
521/*
Catalin Marinasfd678962009-08-27 14:29:17 +0100522 * Save stack trace to the given array of MAX_TRACE size.
523 */
524static int __save_stack_trace(unsigned long *trace)
525{
526 struct stack_trace stack_trace;
527
528 stack_trace.max_entries = MAX_TRACE;
529 stack_trace.nr_entries = 0;
530 stack_trace.entries = trace;
531 stack_trace.skip = 2;
532 save_stack_trace(&stack_trace);
533
534 return stack_trace.nr_entries;
535}
536
537/*
Catalin Marinas3c7b4e62009-06-11 13:22:39 +0100538 * Create the metadata (struct kmemleak_object) corresponding to an allocated
539 * memory block and add it to the object_list and object_tree_root.
540 */
Catalin Marinasfd678962009-08-27 14:29:17 +0100541static struct kmemleak_object *create_object(unsigned long ptr, size_t size,
542 int min_count, gfp_t gfp)
Catalin Marinas3c7b4e62009-06-11 13:22:39 +0100543{
544 unsigned long flags;
Michel Lespinasse85d3a312012-10-08 16:31:27 -0700545 struct kmemleak_object *object, *parent;
546 struct rb_node **link, *rb_parent;
Catalin Marinas3c7b4e62009-06-11 13:22:39 +0100547
Catalin Marinas6ae4bd12011-01-27 10:30:26 +0000548 object = kmem_cache_alloc(object_cache, gfp_kmemleak_mask(gfp));
Catalin Marinas3c7b4e62009-06-11 13:22:39 +0100549 if (!object) {
Joe Perches598d8092016-03-17 14:19:44 -0700550 pr_warn("Cannot allocate a kmemleak_object structure\n");
Catalin Marinas6ae4bd12011-01-27 10:30:26 +0000551 kmemleak_disable();
Catalin Marinasfd678962009-08-27 14:29:17 +0100552 return NULL;
Catalin Marinas3c7b4e62009-06-11 13:22:39 +0100553 }
554
555 INIT_LIST_HEAD(&object->object_list);
556 INIT_LIST_HEAD(&object->gray_list);
557 INIT_HLIST_HEAD(&object->area_list);
558 spin_lock_init(&object->lock);
559 atomic_set(&object->use_count, 1);
Catalin Marinas04609ccc2009-10-28 13:33:12 +0000560 object->flags = OBJECT_ALLOCATED;
Catalin Marinas3c7b4e62009-06-11 13:22:39 +0100561 object->pointer = ptr;
562 object->size = size;
563 object->min_count = min_count;
Catalin Marinas04609ccc2009-10-28 13:33:12 +0000564 object->count = 0; /* white color initially */
Catalin Marinas3c7b4e62009-06-11 13:22:39 +0100565 object->jiffies = jiffies;
Catalin Marinas04609ccc2009-10-28 13:33:12 +0000566 object->checksum = 0;
Catalin Marinas3c7b4e62009-06-11 13:22:39 +0100567
568 /* task information */
569 if (in_irq()) {
570 object->pid = 0;
571 strncpy(object->comm, "hardirq", sizeof(object->comm));
Dmitry Vyukov478cf2d2019-07-11 20:53:39 -0700572 } else if (in_serving_softirq()) {
Catalin Marinas3c7b4e62009-06-11 13:22:39 +0100573 object->pid = 0;
574 strncpy(object->comm, "softirq", sizeof(object->comm));
575 } else {
576 object->pid = current->pid;
577 /*
578 * There is a small chance of a race with set_task_comm(),
579 * however using get_task_comm() here may cause locking
580 * dependency issues with current->alloc_lock. In the worst
581 * case, the command line is not correct.
582 */
583 strncpy(object->comm, current->comm, sizeof(object->comm));
584 }
585
586 /* kernel backtrace */
Catalin Marinasfd678962009-08-27 14:29:17 +0100587 object->trace_len = __save_stack_trace(object->trace);
Catalin Marinas3c7b4e62009-06-11 13:22:39 +0100588
Catalin Marinas3c7b4e62009-06-11 13:22:39 +0100589 write_lock_irqsave(&kmemleak_lock, flags);
Luis R. Rodriguez0580a182009-09-08 17:32:34 +0100590
Catalin Marinas3c7b4e62009-06-11 13:22:39 +0100591 min_addr = min(min_addr, ptr);
592 max_addr = max(max_addr, ptr + size);
Michel Lespinasse85d3a312012-10-08 16:31:27 -0700593 link = &object_tree_root.rb_node;
594 rb_parent = NULL;
595 while (*link) {
596 rb_parent = *link;
597 parent = rb_entry(rb_parent, struct kmemleak_object, rb_node);
598 if (ptr + size <= parent->pointer)
599 link = &parent->rb_node.rb_left;
600 else if (parent->pointer + parent->size <= ptr)
601 link = &parent->rb_node.rb_right;
602 else {
Joe Perches756a025f02016-03-17 14:19:47 -0700603 kmemleak_stop("Cannot insert 0x%lx into the object search tree (overlaps existing)\n",
Michel Lespinasse85d3a312012-10-08 16:31:27 -0700604 ptr);
Catalin Marinas9d5a4c72015-06-24 16:58:34 -0700605 /*
606 * No need for parent->lock here since "parent" cannot
607 * be freed while the kmemleak_lock is held.
608 */
609 dump_object_info(parent);
Michel Lespinasse85d3a312012-10-08 16:31:27 -0700610 kmem_cache_free(object_cache, object);
Catalin Marinas9d5a4c72015-06-24 16:58:34 -0700611 object = NULL;
Michel Lespinasse85d3a312012-10-08 16:31:27 -0700612 goto out;
613 }
Catalin Marinas3c7b4e62009-06-11 13:22:39 +0100614 }
Michel Lespinasse85d3a312012-10-08 16:31:27 -0700615 rb_link_node(&object->rb_node, rb_parent, link);
616 rb_insert_color(&object->rb_node, &object_tree_root);
617
Catalin Marinas3c7b4e62009-06-11 13:22:39 +0100618 list_add_tail_rcu(&object->object_list, &object_list);
619out:
620 write_unlock_irqrestore(&kmemleak_lock, flags);
Catalin Marinasfd678962009-08-27 14:29:17 +0100621 return object;
Catalin Marinas3c7b4e62009-06-11 13:22:39 +0100622}
623
624/*
Catalin Marinase781a9a2015-06-24 16:58:29 -0700625 * Mark the object as not allocated and schedule RCU freeing via put_object().
Catalin Marinas3c7b4e62009-06-11 13:22:39 +0100626 */
Catalin Marinas53238a62009-07-07 10:33:00 +0100627static void __delete_object(struct kmemleak_object *object)
Catalin Marinas3c7b4e62009-06-11 13:22:39 +0100628{
629 unsigned long flags;
Catalin Marinas3c7b4e62009-06-11 13:22:39 +0100630
Catalin Marinas3c7b4e62009-06-11 13:22:39 +0100631 WARN_ON(!(object->flags & OBJECT_ALLOCATED));
Catalin Marinase781a9a2015-06-24 16:58:29 -0700632 WARN_ON(atomic_read(&object->use_count) < 1);
Catalin Marinas3c7b4e62009-06-11 13:22:39 +0100633
634 /*
635 * Locking here also ensures that the corresponding memory block
636 * cannot be freed when it is being scanned.
637 */
638 spin_lock_irqsave(&object->lock, flags);
Catalin Marinas3c7b4e62009-06-11 13:22:39 +0100639 object->flags &= ~OBJECT_ALLOCATED;
640 spin_unlock_irqrestore(&object->lock, flags);
641 put_object(object);
642}
643
644/*
Catalin Marinas53238a62009-07-07 10:33:00 +0100645 * Look up the metadata (struct kmemleak_object) corresponding to ptr and
646 * delete it.
647 */
648static void delete_object_full(unsigned long ptr)
649{
650 struct kmemleak_object *object;
651
Catalin Marinase781a9a2015-06-24 16:58:29 -0700652 object = find_and_remove_object(ptr, 0);
Catalin Marinas53238a62009-07-07 10:33:00 +0100653 if (!object) {
654#ifdef DEBUG
655 kmemleak_warn("Freeing unknown object at 0x%08lx\n",
656 ptr);
657#endif
658 return;
659 }
660 __delete_object(object);
Catalin Marinas53238a62009-07-07 10:33:00 +0100661}
662
663/*
664 * Look up the metadata (struct kmemleak_object) corresponding to ptr and
665 * delete it. If the memory block is partially freed, the function may create
666 * additional metadata for the remaining parts of the block.
667 */
668static void delete_object_part(unsigned long ptr, size_t size)
669{
670 struct kmemleak_object *object;
671 unsigned long start, end;
672
Catalin Marinase781a9a2015-06-24 16:58:29 -0700673 object = find_and_remove_object(ptr, 1);
Catalin Marinas53238a62009-07-07 10:33:00 +0100674 if (!object) {
675#ifdef DEBUG
Joe Perches756a025f02016-03-17 14:19:47 -0700676 kmemleak_warn("Partially freeing unknown object at 0x%08lx (size %zu)\n",
677 ptr, size);
Catalin Marinas53238a62009-07-07 10:33:00 +0100678#endif
679 return;
680 }
Catalin Marinas53238a62009-07-07 10:33:00 +0100681
682 /*
683 * Create one or two objects that may result from the memory block
684 * split. Note that partial freeing is only done by free_bootmem() and
685 * this happens before kmemleak_init() is called. The path below is
686 * only executed during early log recording in kmemleak_init(), so
687 * GFP_KERNEL is enough.
688 */
689 start = object->pointer;
690 end = object->pointer + object->size;
691 if (ptr > start)
692 create_object(start, ptr - start, object->min_count,
693 GFP_KERNEL);
694 if (ptr + size < end)
695 create_object(ptr + size, end - ptr - size, object->min_count,
696 GFP_KERNEL);
697
Catalin Marinase781a9a2015-06-24 16:58:29 -0700698 __delete_object(object);
Catalin Marinas53238a62009-07-07 10:33:00 +0100699}
Luis R. Rodrigueza1084c82009-09-04 17:44:52 -0700700
701static void __paint_it(struct kmemleak_object *object, int color)
702{
703 object->min_count = color;
704 if (color == KMEMLEAK_BLACK)
705 object->flags |= OBJECT_NO_SCAN;
706}
707
708static void paint_it(struct kmemleak_object *object, int color)
709{
710 unsigned long flags;
711
712 spin_lock_irqsave(&object->lock, flags);
713 __paint_it(object, color);
714 spin_unlock_irqrestore(&object->lock, flags);
715}
716
717static void paint_ptr(unsigned long ptr, int color)
718{
719 struct kmemleak_object *object;
720
721 object = find_and_get_object(ptr, 0);
722 if (!object) {
Joe Perches756a025f02016-03-17 14:19:47 -0700723 kmemleak_warn("Trying to color unknown object at 0x%08lx as %s\n",
724 ptr,
Luis R. Rodrigueza1084c82009-09-04 17:44:52 -0700725 (color == KMEMLEAK_GREY) ? "Grey" :
726 (color == KMEMLEAK_BLACK) ? "Black" : "Unknown");
727 return;
728 }
729 paint_it(object, color);
730 put_object(object);
731}
732
Catalin Marinas53238a62009-07-07 10:33:00 +0100733/*
Holger Hans Peter Freyther145b64b2010-07-22 19:54:13 +0800734 * Mark an object permanently as gray-colored so that it can no longer be
Catalin Marinas3c7b4e62009-06-11 13:22:39 +0100735 * reported as a leak. This is used in general to mark a false positive.
736 */
737static void make_gray_object(unsigned long ptr)
738{
Luis R. Rodrigueza1084c82009-09-04 17:44:52 -0700739 paint_ptr(ptr, KMEMLEAK_GREY);
Catalin Marinas3c7b4e62009-06-11 13:22:39 +0100740}
741
742/*
743 * Mark the object as black-colored so that it is ignored from scans and
744 * reporting.
745 */
746static void make_black_object(unsigned long ptr)
747{
Luis R. Rodrigueza1084c82009-09-04 17:44:52 -0700748 paint_ptr(ptr, KMEMLEAK_BLACK);
Catalin Marinas3c7b4e62009-06-11 13:22:39 +0100749}
750
751/*
752 * Add a scanning area to the object. If at least one such area is added,
753 * kmemleak will only scan these ranges rather than the whole memory block.
754 */
Catalin Marinasc017b4b2009-10-28 13:33:09 +0000755static void add_scan_area(unsigned long ptr, size_t size, gfp_t gfp)
Catalin Marinas3c7b4e62009-06-11 13:22:39 +0100756{
757 unsigned long flags;
758 struct kmemleak_object *object;
759 struct kmemleak_scan_area *area;
760
Catalin Marinasc017b4b2009-10-28 13:33:09 +0000761 object = find_and_get_object(ptr, 1);
Catalin Marinas3c7b4e62009-06-11 13:22:39 +0100762 if (!object) {
Joe Perchesae281062009-06-23 14:40:26 +0100763 kmemleak_warn("Adding scan area to unknown object at 0x%08lx\n",
764 ptr);
Catalin Marinas3c7b4e62009-06-11 13:22:39 +0100765 return;
766 }
767
Catalin Marinas6ae4bd12011-01-27 10:30:26 +0000768 area = kmem_cache_alloc(scan_area_cache, gfp_kmemleak_mask(gfp));
Catalin Marinas3c7b4e62009-06-11 13:22:39 +0100769 if (!area) {
Joe Perches598d8092016-03-17 14:19:44 -0700770 pr_warn("Cannot allocate a scan area\n");
Catalin Marinas3c7b4e62009-06-11 13:22:39 +0100771 goto out;
772 }
773
774 spin_lock_irqsave(&object->lock, flags);
Catalin Marinas7f88f882013-11-12 15:07:45 -0800775 if (size == SIZE_MAX) {
776 size = object->pointer + object->size - ptr;
777 } else if (ptr + size > object->pointer + object->size) {
Joe Perchesae281062009-06-23 14:40:26 +0100778 kmemleak_warn("Scan area larger than object 0x%08lx\n", ptr);
Catalin Marinas3c7b4e62009-06-11 13:22:39 +0100779 dump_object_info(object);
780 kmem_cache_free(scan_area_cache, area);
781 goto out_unlock;
782 }
783
784 INIT_HLIST_NODE(&area->node);
Catalin Marinasc017b4b2009-10-28 13:33:09 +0000785 area->start = ptr;
786 area->size = size;
Catalin Marinas3c7b4e62009-06-11 13:22:39 +0100787
788 hlist_add_head(&area->node, &object->area_list);
789out_unlock:
790 spin_unlock_irqrestore(&object->lock, flags);
791out:
792 put_object(object);
793}
794
795/*
796 * Set the OBJECT_NO_SCAN flag for the object corresponding to the give
797 * pointer. Such object will not be scanned by kmemleak but references to it
798 * are searched.
799 */
800static void object_no_scan(unsigned long ptr)
801{
802 unsigned long flags;
803 struct kmemleak_object *object;
804
805 object = find_and_get_object(ptr, 0);
806 if (!object) {
Joe Perchesae281062009-06-23 14:40:26 +0100807 kmemleak_warn("Not scanning unknown object at 0x%08lx\n", ptr);
Catalin Marinas3c7b4e62009-06-11 13:22:39 +0100808 return;
809 }
810
811 spin_lock_irqsave(&object->lock, flags);
812 object->flags |= OBJECT_NO_SCAN;
813 spin_unlock_irqrestore(&object->lock, flags);
814 put_object(object);
815}
816
817/*
818 * Log an early kmemleak_* call to the early_log buffer. These calls will be
819 * processed later once kmemleak is fully initialized.
820 */
Catalin Marinasa6186d82009-08-27 14:29:16 +0100821static void __init log_early(int op_type, const void *ptr, size_t size,
Catalin Marinasc017b4b2009-10-28 13:33:09 +0000822 int min_count)
Catalin Marinas3c7b4e62009-06-11 13:22:39 +0100823{
824 unsigned long flags;
825 struct early_log *log;
826
Li Zefan8910ae82014-04-03 14:46:29 -0700827 if (kmemleak_error) {
Catalin Marinasb6693002011-09-28 17:22:56 +0100828 /* kmemleak stopped recording, just count the requests */
829 crt_early_log++;
830 return;
831 }
832
Catalin Marinas3c7b4e62009-06-11 13:22:39 +0100833 if (crt_early_log >= ARRAY_SIZE(early_log)) {
Wang Kai21cd3a62015-09-08 15:03:41 -0700834 crt_early_log++;
Catalin Marinasa9d90582009-06-25 10:16:11 +0100835 kmemleak_disable();
Catalin Marinas3c7b4e62009-06-11 13:22:39 +0100836 return;
837 }
838
839 /*
840 * There is no need for locking since the kernel is still in UP mode
841 * at this stage. Disabling the IRQs is enough.
842 */
843 local_irq_save(flags);
844 log = &early_log[crt_early_log];
845 log->op_type = op_type;
846 log->ptr = ptr;
847 log->size = size;
848 log->min_count = min_count;
Catalin Marinas5f790202011-09-28 12:17:03 +0100849 log->trace_len = __save_stack_trace(log->trace);
Catalin Marinas3c7b4e62009-06-11 13:22:39 +0100850 crt_early_log++;
851 local_irq_restore(flags);
852}
853
854/*
Catalin Marinasfd678962009-08-27 14:29:17 +0100855 * Log an early allocated block and populate the stack trace.
856 */
857static void early_alloc(struct early_log *log)
858{
859 struct kmemleak_object *object;
860 unsigned long flags;
861 int i;
862
Li Zefan8910ae82014-04-03 14:46:29 -0700863 if (!kmemleak_enabled || !log->ptr || IS_ERR(log->ptr))
Catalin Marinasfd678962009-08-27 14:29:17 +0100864 return;
865
866 /*
867 * RCU locking needed to ensure object is not freed via put_object().
868 */
869 rcu_read_lock();
870 object = create_object((unsigned long)log->ptr, log->size,
Tetsuo Handac1bcd6b2009-10-09 10:39:24 +0100871 log->min_count, GFP_ATOMIC);
Catalin Marinas0d5d1aa2009-10-09 10:30:34 +0100872 if (!object)
873 goto out;
Catalin Marinasfd678962009-08-27 14:29:17 +0100874 spin_lock_irqsave(&object->lock, flags);
875 for (i = 0; i < log->trace_len; i++)
876 object->trace[i] = log->trace[i];
877 object->trace_len = log->trace_len;
878 spin_unlock_irqrestore(&object->lock, flags);
Catalin Marinas0d5d1aa2009-10-09 10:30:34 +0100879out:
Catalin Marinasfd678962009-08-27 14:29:17 +0100880 rcu_read_unlock();
881}
882
Catalin Marinasf528f0b2011-09-26 17:12:53 +0100883/*
884 * Log an early allocated block and populate the stack trace.
885 */
886static void early_alloc_percpu(struct early_log *log)
887{
888 unsigned int cpu;
889 const void __percpu *ptr = log->ptr;
890
891 for_each_possible_cpu(cpu) {
892 log->ptr = per_cpu_ptr(ptr, cpu);
893 early_alloc(log);
894 }
895}
896
Catalin Marinasa2b6bf62010-07-19 11:54:17 +0100897/**
898 * kmemleak_alloc - register a newly allocated object
899 * @ptr: pointer to beginning of the object
900 * @size: size of the object
901 * @min_count: minimum number of references to this object. If during memory
902 * scanning a number of references less than @min_count is found,
903 * the object is reported as a memory leak. If @min_count is 0,
904 * the object is never reported as a leak. If @min_count is -1,
905 * the object is ignored (not scanned and not reported as a leak)
906 * @gfp: kmalloc() flags used for kmemleak internal memory allocations
907 *
908 * This function is called from the kernel allocators when a new object
909 * (memory block) is allocated (kmem_cache_alloc, kmalloc, vmalloc etc.).
Catalin Marinas3c7b4e62009-06-11 13:22:39 +0100910 */
Catalin Marinasa6186d82009-08-27 14:29:16 +0100911void __ref kmemleak_alloc(const void *ptr, size_t size, int min_count,
912 gfp_t gfp)
Catalin Marinas3c7b4e62009-06-11 13:22:39 +0100913{
914 pr_debug("%s(0x%p, %zu, %d)\n", __func__, ptr, size, min_count);
915
Li Zefan8910ae82014-04-03 14:46:29 -0700916 if (kmemleak_enabled && ptr && !IS_ERR(ptr))
Catalin Marinas3c7b4e62009-06-11 13:22:39 +0100917 create_object((unsigned long)ptr, size, min_count, gfp);
Li Zefan8910ae82014-04-03 14:46:29 -0700918 else if (kmemleak_early_log)
Catalin Marinasc017b4b2009-10-28 13:33:09 +0000919 log_early(KMEMLEAK_ALLOC, ptr, size, min_count);
Catalin Marinas3c7b4e62009-06-11 13:22:39 +0100920}
921EXPORT_SYMBOL_GPL(kmemleak_alloc);
922
Catalin Marinasa2b6bf62010-07-19 11:54:17 +0100923/**
Catalin Marinasf528f0b2011-09-26 17:12:53 +0100924 * kmemleak_alloc_percpu - register a newly allocated __percpu object
925 * @ptr: __percpu pointer to beginning of the object
926 * @size: size of the object
Larry Finger8a8c35f2015-06-24 16:58:51 -0700927 * @gfp: flags used for kmemleak internal memory allocations
Catalin Marinasf528f0b2011-09-26 17:12:53 +0100928 *
929 * This function is called from the kernel percpu allocator when a new object
Larry Finger8a8c35f2015-06-24 16:58:51 -0700930 * (memory block) is allocated (alloc_percpu).
Catalin Marinasf528f0b2011-09-26 17:12:53 +0100931 */
Larry Finger8a8c35f2015-06-24 16:58:51 -0700932void __ref kmemleak_alloc_percpu(const void __percpu *ptr, size_t size,
933 gfp_t gfp)
Catalin Marinasf528f0b2011-09-26 17:12:53 +0100934{
935 unsigned int cpu;
936
937 pr_debug("%s(0x%p, %zu)\n", __func__, ptr, size);
938
939 /*
940 * Percpu allocations are only scanned and not reported as leaks
941 * (min_count is set to 0).
942 */
Li Zefan8910ae82014-04-03 14:46:29 -0700943 if (kmemleak_enabled && ptr && !IS_ERR(ptr))
Catalin Marinasf528f0b2011-09-26 17:12:53 +0100944 for_each_possible_cpu(cpu)
945 create_object((unsigned long)per_cpu_ptr(ptr, cpu),
Larry Finger8a8c35f2015-06-24 16:58:51 -0700946 size, 0, gfp);
Li Zefan8910ae82014-04-03 14:46:29 -0700947 else if (kmemleak_early_log)
Catalin Marinasf528f0b2011-09-26 17:12:53 +0100948 log_early(KMEMLEAK_ALLOC_PERCPU, ptr, size, 0);
949}
950EXPORT_SYMBOL_GPL(kmemleak_alloc_percpu);
951
952/**
Catalin Marinasa2b6bf62010-07-19 11:54:17 +0100953 * kmemleak_free - unregister a previously registered object
954 * @ptr: pointer to beginning of the object
955 *
956 * This function is called from the kernel allocators when an object (memory
957 * block) is freed (kmem_cache_free, kfree, vfree etc.).
Catalin Marinas3c7b4e62009-06-11 13:22:39 +0100958 */
Catalin Marinasa6186d82009-08-27 14:29:16 +0100959void __ref kmemleak_free(const void *ptr)
Catalin Marinas3c7b4e62009-06-11 13:22:39 +0100960{
961 pr_debug("%s(0x%p)\n", __func__, ptr);
962
Catalin Marinasc5f3b1a2015-06-24 16:58:26 -0700963 if (kmemleak_free_enabled && ptr && !IS_ERR(ptr))
Catalin Marinas53238a62009-07-07 10:33:00 +0100964 delete_object_full((unsigned long)ptr);
Li Zefan8910ae82014-04-03 14:46:29 -0700965 else if (kmemleak_early_log)
Catalin Marinasc017b4b2009-10-28 13:33:09 +0000966 log_early(KMEMLEAK_FREE, ptr, 0, 0);
Catalin Marinas3c7b4e62009-06-11 13:22:39 +0100967}
968EXPORT_SYMBOL_GPL(kmemleak_free);
969
Catalin Marinasa2b6bf62010-07-19 11:54:17 +0100970/**
971 * kmemleak_free_part - partially unregister a previously registered object
972 * @ptr: pointer to the beginning or inside the object. This also
973 * represents the start of the range to be freed
974 * @size: size to be unregistered
975 *
976 * This function is called when only a part of a memory block is freed
977 * (usually from the bootmem allocator).
Catalin Marinas53238a62009-07-07 10:33:00 +0100978 */
Catalin Marinasa6186d82009-08-27 14:29:16 +0100979void __ref kmemleak_free_part(const void *ptr, size_t size)
Catalin Marinas53238a62009-07-07 10:33:00 +0100980{
981 pr_debug("%s(0x%p)\n", __func__, ptr);
982
Li Zefan8910ae82014-04-03 14:46:29 -0700983 if (kmemleak_enabled && ptr && !IS_ERR(ptr))
Catalin Marinas53238a62009-07-07 10:33:00 +0100984 delete_object_part((unsigned long)ptr, size);
Li Zefan8910ae82014-04-03 14:46:29 -0700985 else if (kmemleak_early_log)
Catalin Marinasc017b4b2009-10-28 13:33:09 +0000986 log_early(KMEMLEAK_FREE_PART, ptr, size, 0);
Catalin Marinas53238a62009-07-07 10:33:00 +0100987}
988EXPORT_SYMBOL_GPL(kmemleak_free_part);
989
Catalin Marinasa2b6bf62010-07-19 11:54:17 +0100990/**
Catalin Marinasf528f0b2011-09-26 17:12:53 +0100991 * kmemleak_free_percpu - unregister a previously registered __percpu object
992 * @ptr: __percpu pointer to beginning of the object
993 *
994 * This function is called from the kernel percpu allocator when an object
995 * (memory block) is freed (free_percpu).
996 */
997void __ref kmemleak_free_percpu(const void __percpu *ptr)
998{
999 unsigned int cpu;
1000
1001 pr_debug("%s(0x%p)\n", __func__, ptr);
1002
Catalin Marinasc5f3b1a2015-06-24 16:58:26 -07001003 if (kmemleak_free_enabled && ptr && !IS_ERR(ptr))
Catalin Marinasf528f0b2011-09-26 17:12:53 +01001004 for_each_possible_cpu(cpu)
1005 delete_object_full((unsigned long)per_cpu_ptr(ptr,
1006 cpu));
Li Zefan8910ae82014-04-03 14:46:29 -07001007 else if (kmemleak_early_log)
Catalin Marinasf528f0b2011-09-26 17:12:53 +01001008 log_early(KMEMLEAK_FREE_PERCPU, ptr, 0, 0);
1009}
1010EXPORT_SYMBOL_GPL(kmemleak_free_percpu);
1011
1012/**
Catalin Marinasffe2c742014-06-06 14:38:17 -07001013 * kmemleak_update_trace - update object allocation stack trace
1014 * @ptr: pointer to beginning of the object
1015 *
1016 * Override the object allocation stack trace for cases where the actual
1017 * allocation place is not always useful.
1018 */
1019void __ref kmemleak_update_trace(const void *ptr)
1020{
1021 struct kmemleak_object *object;
1022 unsigned long flags;
1023
1024 pr_debug("%s(0x%p)\n", __func__, ptr);
1025
1026 if (!kmemleak_enabled || IS_ERR_OR_NULL(ptr))
1027 return;
1028
1029 object = find_and_get_object((unsigned long)ptr, 1);
1030 if (!object) {
1031#ifdef DEBUG
1032 kmemleak_warn("Updating stack trace for unknown object at %p\n",
1033 ptr);
1034#endif
1035 return;
1036 }
1037
1038 spin_lock_irqsave(&object->lock, flags);
1039 object->trace_len = __save_stack_trace(object->trace);
1040 spin_unlock_irqrestore(&object->lock, flags);
1041
1042 put_object(object);
1043}
1044EXPORT_SYMBOL(kmemleak_update_trace);
1045
1046/**
Catalin Marinasa2b6bf62010-07-19 11:54:17 +01001047 * kmemleak_not_leak - mark an allocated object as false positive
1048 * @ptr: pointer to beginning of the object
1049 *
1050 * Calling this function on an object will cause the memory block to no longer
1051 * be reported as leak and always be scanned.
Catalin Marinas3c7b4e62009-06-11 13:22:39 +01001052 */
Catalin Marinasa6186d82009-08-27 14:29:16 +01001053void __ref kmemleak_not_leak(const void *ptr)
Catalin Marinas3c7b4e62009-06-11 13:22:39 +01001054{
1055 pr_debug("%s(0x%p)\n", __func__, ptr);
1056
Li Zefan8910ae82014-04-03 14:46:29 -07001057 if (kmemleak_enabled && ptr && !IS_ERR(ptr))
Catalin Marinas3c7b4e62009-06-11 13:22:39 +01001058 make_gray_object((unsigned long)ptr);
Li Zefan8910ae82014-04-03 14:46:29 -07001059 else if (kmemleak_early_log)
Catalin Marinasc017b4b2009-10-28 13:33:09 +00001060 log_early(KMEMLEAK_NOT_LEAK, ptr, 0, 0);
Catalin Marinas3c7b4e62009-06-11 13:22:39 +01001061}
1062EXPORT_SYMBOL(kmemleak_not_leak);
1063
Catalin Marinasa2b6bf62010-07-19 11:54:17 +01001064/**
1065 * kmemleak_ignore - ignore an allocated object
1066 * @ptr: pointer to beginning of the object
1067 *
1068 * Calling this function on an object will cause the memory block to be
1069 * ignored (not scanned and not reported as a leak). This is usually done when
1070 * it is known that the corresponding block is not a leak and does not contain
1071 * any references to other allocated memory blocks.
Catalin Marinas3c7b4e62009-06-11 13:22:39 +01001072 */
Catalin Marinasa6186d82009-08-27 14:29:16 +01001073void __ref kmemleak_ignore(const void *ptr)
Catalin Marinas3c7b4e62009-06-11 13:22:39 +01001074{
1075 pr_debug("%s(0x%p)\n", __func__, ptr);
1076
Li Zefan8910ae82014-04-03 14:46:29 -07001077 if (kmemleak_enabled && ptr && !IS_ERR(ptr))
Catalin Marinas3c7b4e62009-06-11 13:22:39 +01001078 make_black_object((unsigned long)ptr);
Li Zefan8910ae82014-04-03 14:46:29 -07001079 else if (kmemleak_early_log)
Catalin Marinasc017b4b2009-10-28 13:33:09 +00001080 log_early(KMEMLEAK_IGNORE, ptr, 0, 0);
Catalin Marinas3c7b4e62009-06-11 13:22:39 +01001081}
1082EXPORT_SYMBOL(kmemleak_ignore);
1083
Catalin Marinasa2b6bf62010-07-19 11:54:17 +01001084/**
1085 * kmemleak_scan_area - limit the range to be scanned in an allocated object
1086 * @ptr: pointer to beginning or inside the object. This also
1087 * represents the start of the scan area
1088 * @size: size of the scan area
1089 * @gfp: kmalloc() flags used for kmemleak internal memory allocations
1090 *
1091 * This function is used when it is known that only certain parts of an object
1092 * contain references to other objects. Kmemleak will only scan these areas
1093 * reducing the number false negatives.
Catalin Marinas3c7b4e62009-06-11 13:22:39 +01001094 */
Catalin Marinasc017b4b2009-10-28 13:33:09 +00001095void __ref kmemleak_scan_area(const void *ptr, size_t size, gfp_t gfp)
Catalin Marinas3c7b4e62009-06-11 13:22:39 +01001096{
1097 pr_debug("%s(0x%p)\n", __func__, ptr);
1098
Li Zefan8910ae82014-04-03 14:46:29 -07001099 if (kmemleak_enabled && ptr && size && !IS_ERR(ptr))
Catalin Marinasc017b4b2009-10-28 13:33:09 +00001100 add_scan_area((unsigned long)ptr, size, gfp);
Li Zefan8910ae82014-04-03 14:46:29 -07001101 else if (kmemleak_early_log)
Catalin Marinasc017b4b2009-10-28 13:33:09 +00001102 log_early(KMEMLEAK_SCAN_AREA, ptr, size, 0);
Catalin Marinas3c7b4e62009-06-11 13:22:39 +01001103}
1104EXPORT_SYMBOL(kmemleak_scan_area);
1105
Catalin Marinasa2b6bf62010-07-19 11:54:17 +01001106/**
1107 * kmemleak_no_scan - do not scan an allocated object
1108 * @ptr: pointer to beginning of the object
1109 *
1110 * This function notifies kmemleak not to scan the given memory block. Useful
1111 * in situations where it is known that the given object does not contain any
1112 * references to other objects. Kmemleak will not scan such objects reducing
1113 * the number of false negatives.
Catalin Marinas3c7b4e62009-06-11 13:22:39 +01001114 */
Catalin Marinasa6186d82009-08-27 14:29:16 +01001115void __ref kmemleak_no_scan(const void *ptr)
Catalin Marinas3c7b4e62009-06-11 13:22:39 +01001116{
1117 pr_debug("%s(0x%p)\n", __func__, ptr);
1118
Li Zefan8910ae82014-04-03 14:46:29 -07001119 if (kmemleak_enabled && ptr && !IS_ERR(ptr))
Catalin Marinas3c7b4e62009-06-11 13:22:39 +01001120 object_no_scan((unsigned long)ptr);
Li Zefan8910ae82014-04-03 14:46:29 -07001121 else if (kmemleak_early_log)
Catalin Marinasc017b4b2009-10-28 13:33:09 +00001122 log_early(KMEMLEAK_NO_SCAN, ptr, 0, 0);
Catalin Marinas3c7b4e62009-06-11 13:22:39 +01001123}
1124EXPORT_SYMBOL(kmemleak_no_scan);
1125
Catalin Marinas9099dae2016-10-11 13:55:11 -07001126/**
1127 * kmemleak_alloc_phys - similar to kmemleak_alloc but taking a physical
1128 * address argument
1129 */
1130void __ref kmemleak_alloc_phys(phys_addr_t phys, size_t size, int min_count,
1131 gfp_t gfp)
1132{
1133 if (!IS_ENABLED(CONFIG_HIGHMEM) || PHYS_PFN(phys) < max_low_pfn)
1134 kmemleak_alloc(__va(phys), size, min_count, gfp);
1135}
1136EXPORT_SYMBOL(kmemleak_alloc_phys);
1137
1138/**
1139 * kmemleak_free_part_phys - similar to kmemleak_free_part but taking a
1140 * physical address argument
1141 */
1142void __ref kmemleak_free_part_phys(phys_addr_t phys, size_t size)
1143{
1144 if (!IS_ENABLED(CONFIG_HIGHMEM) || PHYS_PFN(phys) < max_low_pfn)
1145 kmemleak_free_part(__va(phys), size);
1146}
1147EXPORT_SYMBOL(kmemleak_free_part_phys);
1148
1149/**
1150 * kmemleak_not_leak_phys - similar to kmemleak_not_leak but taking a physical
1151 * address argument
1152 */
1153void __ref kmemleak_not_leak_phys(phys_addr_t phys)
1154{
1155 if (!IS_ENABLED(CONFIG_HIGHMEM) || PHYS_PFN(phys) < max_low_pfn)
1156 kmemleak_not_leak(__va(phys));
1157}
1158EXPORT_SYMBOL(kmemleak_not_leak_phys);
1159
1160/**
1161 * kmemleak_ignore_phys - similar to kmemleak_ignore but taking a physical
1162 * address argument
1163 */
1164void __ref kmemleak_ignore_phys(phys_addr_t phys)
1165{
1166 if (!IS_ENABLED(CONFIG_HIGHMEM) || PHYS_PFN(phys) < max_low_pfn)
1167 kmemleak_ignore(__va(phys));
1168}
1169EXPORT_SYMBOL(kmemleak_ignore_phys);
1170
Catalin Marinas3c7b4e62009-06-11 13:22:39 +01001171/*
Catalin Marinas04609ccc2009-10-28 13:33:12 +00001172 * Update an object's checksum and return true if it was modified.
1173 */
1174static bool update_checksum(struct kmemleak_object *object)
1175{
1176 u32 old_csum = object->checksum;
1177
1178 if (!kmemcheck_is_obj_initialized(object->pointer, object->size))
1179 return false;
1180
Andrey Ryabinine79ed2f2015-02-13 14:39:49 -08001181 kasan_disable_current();
Catalin Marinas04609ccc2009-10-28 13:33:12 +00001182 object->checksum = crc32(0, (void *)object->pointer, object->size);
Andrey Ryabinine79ed2f2015-02-13 14:39:49 -08001183 kasan_enable_current();
1184
Catalin Marinas04609ccc2009-10-28 13:33:12 +00001185 return object->checksum != old_csum;
1186}
1187
1188/*
Catalin Marinas3c7b4e62009-06-11 13:22:39 +01001189 * Memory scanning is a long process and it needs to be interruptable. This
Lucas De Marchi25985ed2011-03-30 22:57:33 -03001190 * function checks whether such interrupt condition occurred.
Catalin Marinas3c7b4e62009-06-11 13:22:39 +01001191 */
1192static int scan_should_stop(void)
1193{
Li Zefan8910ae82014-04-03 14:46:29 -07001194 if (!kmemleak_enabled)
Catalin Marinas3c7b4e62009-06-11 13:22:39 +01001195 return 1;
1196
1197 /*
1198 * This function may be called from either process or kthread context,
1199 * hence the need to check for both stop conditions.
1200 */
1201 if (current->mm)
1202 return signal_pending(current);
1203 else
1204 return kthread_should_stop();
1205
1206 return 0;
1207}
1208
1209/*
1210 * Scan a memory block (exclusive range) for valid pointers and add those
1211 * found to the gray list.
1212 */
1213static void scan_block(void *_start, void *_end,
Catalin Marinas93ada572015-06-24 16:58:37 -07001214 struct kmemleak_object *scanned)
Catalin Marinas3c7b4e62009-06-11 13:22:39 +01001215{
1216 unsigned long *ptr;
1217 unsigned long *start = PTR_ALIGN(_start, BYTES_PER_POINTER);
1218 unsigned long *end = _end - (BYTES_PER_POINTER - 1);
Catalin Marinas93ada572015-06-24 16:58:37 -07001219 unsigned long flags;
Catalin Marinas3c7b4e62009-06-11 13:22:39 +01001220
Catalin Marinas93ada572015-06-24 16:58:37 -07001221 read_lock_irqsave(&kmemleak_lock, flags);
Catalin Marinas3c7b4e62009-06-11 13:22:39 +01001222 for (ptr = start; ptr < end; ptr++) {
Catalin Marinas3c7b4e62009-06-11 13:22:39 +01001223 struct kmemleak_object *object;
Pekka Enberg8e019362009-08-27 14:50:00 +01001224 unsigned long pointer;
Catalin Marinas3c7b4e62009-06-11 13:22:39 +01001225
1226 if (scan_should_stop())
1227 break;
1228
Pekka Enberg8e019362009-08-27 14:50:00 +01001229 /* don't scan uninitialized memory */
1230 if (!kmemcheck_is_obj_initialized((unsigned long)ptr,
1231 BYTES_PER_POINTER))
1232 continue;
1233
Andrey Ryabinine79ed2f2015-02-13 14:39:49 -08001234 kasan_disable_current();
Pekka Enberg8e019362009-08-27 14:50:00 +01001235 pointer = *ptr;
Andrey Ryabinine79ed2f2015-02-13 14:39:49 -08001236 kasan_enable_current();
Pekka Enberg8e019362009-08-27 14:50:00 +01001237
Catalin Marinas93ada572015-06-24 16:58:37 -07001238 if (pointer < min_addr || pointer >= max_addr)
1239 continue;
1240
1241 /*
1242 * No need for get_object() here since we hold kmemleak_lock.
1243 * object->use_count cannot be dropped to 0 while the object
1244 * is still present in object_tree_root and object_list
1245 * (with updates protected by kmemleak_lock).
1246 */
1247 object = lookup_object(pointer, 1);
Catalin Marinas3c7b4e62009-06-11 13:22:39 +01001248 if (!object)
1249 continue;
Catalin Marinas93ada572015-06-24 16:58:37 -07001250 if (object == scanned)
Catalin Marinas3c7b4e62009-06-11 13:22:39 +01001251 /* self referenced, ignore */
Catalin Marinas3c7b4e62009-06-11 13:22:39 +01001252 continue;
Catalin Marinas3c7b4e62009-06-11 13:22:39 +01001253
1254 /*
1255 * Avoid the lockdep recursive warning on object->lock being
1256 * previously acquired in scan_object(). These locks are
1257 * enclosed by scan_mutex.
1258 */
Catalin Marinas93ada572015-06-24 16:58:37 -07001259 spin_lock_nested(&object->lock, SINGLE_DEPTH_NESTING);
Catalin Marinas3c7b4e62009-06-11 13:22:39 +01001260 if (!color_white(object)) {
1261 /* non-orphan, ignored or new */
Catalin Marinas93ada572015-06-24 16:58:37 -07001262 spin_unlock(&object->lock);
Catalin Marinas3c7b4e62009-06-11 13:22:39 +01001263 continue;
1264 }
1265
1266 /*
1267 * Increase the object's reference count (number of pointers
1268 * to the memory block). If this count reaches the required
1269 * minimum, the object's color will become gray and it will be
1270 * added to the gray_list.
1271 */
1272 object->count++;
Catalin Marinas0587da42009-10-28 13:33:11 +00001273 if (color_gray(object)) {
Catalin Marinas93ada572015-06-24 16:58:37 -07001274 /* put_object() called when removing from gray_list */
1275 WARN_ON(!get_object(object));
Catalin Marinas3c7b4e62009-06-11 13:22:39 +01001276 list_add_tail(&object->gray_list, &gray_list);
Catalin Marinas0587da42009-10-28 13:33:11 +00001277 }
Catalin Marinas93ada572015-06-24 16:58:37 -07001278 spin_unlock(&object->lock);
1279 }
1280 read_unlock_irqrestore(&kmemleak_lock, flags);
1281}
Catalin Marinas0587da42009-10-28 13:33:11 +00001282
Catalin Marinas93ada572015-06-24 16:58:37 -07001283/*
1284 * Scan a large memory block in MAX_SCAN_SIZE chunks to reduce the latency.
1285 */
1286static void scan_large_block(void *start, void *end)
1287{
1288 void *next;
1289
1290 while (start < end) {
1291 next = min(start + MAX_SCAN_SIZE, end);
1292 scan_block(start, next, NULL);
1293 start = next;
1294 cond_resched();
Catalin Marinas3c7b4e62009-06-11 13:22:39 +01001295 }
1296}
1297
1298/*
1299 * Scan a memory block corresponding to a kmemleak_object. A condition is
1300 * that object->use_count >= 1.
1301 */
1302static void scan_object(struct kmemleak_object *object)
1303{
1304 struct kmemleak_scan_area *area;
Catalin Marinas3c7b4e62009-06-11 13:22:39 +01001305 unsigned long flags;
1306
1307 /*
Uwe Kleine-König21ae2952009-10-07 15:21:09 +02001308 * Once the object->lock is acquired, the corresponding memory block
1309 * cannot be freed (the same lock is acquired in delete_object).
Catalin Marinas3c7b4e62009-06-11 13:22:39 +01001310 */
1311 spin_lock_irqsave(&object->lock, flags);
1312 if (object->flags & OBJECT_NO_SCAN)
1313 goto out;
1314 if (!(object->flags & OBJECT_ALLOCATED))
1315 /* already freed object */
1316 goto out;
Catalin Marinasaf986032009-08-27 14:29:12 +01001317 if (hlist_empty(&object->area_list)) {
1318 void *start = (void *)object->pointer;
1319 void *end = (void *)(object->pointer + object->size);
Catalin Marinas93ada572015-06-24 16:58:37 -07001320 void *next;
Catalin Marinasaf986032009-08-27 14:29:12 +01001321
Catalin Marinas93ada572015-06-24 16:58:37 -07001322 do {
1323 next = min(start + MAX_SCAN_SIZE, end);
1324 scan_block(start, next, object);
1325
1326 start = next;
1327 if (start >= end)
1328 break;
Catalin Marinasaf986032009-08-27 14:29:12 +01001329
1330 spin_unlock_irqrestore(&object->lock, flags);
1331 cond_resched();
1332 spin_lock_irqsave(&object->lock, flags);
Catalin Marinas93ada572015-06-24 16:58:37 -07001333 } while (object->flags & OBJECT_ALLOCATED);
Catalin Marinasaf986032009-08-27 14:29:12 +01001334 } else
Sasha Levinb67bfe02013-02-27 17:06:00 -08001335 hlist_for_each_entry(area, &object->area_list, node)
Catalin Marinasc017b4b2009-10-28 13:33:09 +00001336 scan_block((void *)area->start,
1337 (void *)(area->start + area->size),
Catalin Marinas93ada572015-06-24 16:58:37 -07001338 object);
Catalin Marinas3c7b4e62009-06-11 13:22:39 +01001339out:
1340 spin_unlock_irqrestore(&object->lock, flags);
1341}
1342
1343/*
Catalin Marinas04609ccc2009-10-28 13:33:12 +00001344 * Scan the objects already referenced (gray objects). More objects will be
1345 * referenced and, if there are no memory leaks, all the objects are scanned.
1346 */
1347static void scan_gray_list(void)
1348{
1349 struct kmemleak_object *object, *tmp;
1350
1351 /*
1352 * The list traversal is safe for both tail additions and removals
1353 * from inside the loop. The kmemleak objects cannot be freed from
1354 * outside the loop because their use_count was incremented.
1355 */
1356 object = list_entry(gray_list.next, typeof(*object), gray_list);
1357 while (&object->gray_list != &gray_list) {
1358 cond_resched();
1359
1360 /* may add new objects to the list */
1361 if (!scan_should_stop())
1362 scan_object(object);
1363
1364 tmp = list_entry(object->gray_list.next, typeof(*object),
1365 gray_list);
1366
1367 /* remove the object from the list and release it */
1368 list_del(&object->gray_list);
1369 put_object(object);
1370
1371 object = tmp;
1372 }
1373 WARN_ON(!list_empty(&gray_list));
1374}
1375
1376/*
Catalin Marinas3c7b4e62009-06-11 13:22:39 +01001377 * Scan data sections and all the referenced memory blocks allocated via the
1378 * kernel's standard allocators. This function must be called with the
1379 * scan_mutex held.
1380 */
1381static void kmemleak_scan(void)
1382{
1383 unsigned long flags;
Catalin Marinas04609ccc2009-10-28 13:33:12 +00001384 struct kmemleak_object *object;
Catalin Marinas3c7b4e62009-06-11 13:22:39 +01001385 int i;
Catalin Marinas4698c1f2009-06-26 17:38:27 +01001386 int new_leaks = 0;
Catalin Marinas3c7b4e62009-06-11 13:22:39 +01001387
Catalin Marinasacf49682009-06-26 17:38:29 +01001388 jiffies_last_scan = jiffies;
1389
Catalin Marinas3c7b4e62009-06-11 13:22:39 +01001390 /* prepare the kmemleak_object's */
1391 rcu_read_lock();
1392 list_for_each_entry_rcu(object, &object_list, object_list) {
1393 spin_lock_irqsave(&object->lock, flags);
1394#ifdef DEBUG
1395 /*
1396 * With a few exceptions there should be a maximum of
1397 * 1 reference to any object at this point.
1398 */
1399 if (atomic_read(&object->use_count) > 1) {
Joe Perchesae281062009-06-23 14:40:26 +01001400 pr_debug("object->use_count = %d\n",
Catalin Marinas3c7b4e62009-06-11 13:22:39 +01001401 atomic_read(&object->use_count));
1402 dump_object_info(object);
1403 }
1404#endif
1405 /* reset the reference count (whiten the object) */
1406 object->count = 0;
1407 if (color_gray(object) && get_object(object))
1408 list_add_tail(&object->gray_list, &gray_list);
1409
1410 spin_unlock_irqrestore(&object->lock, flags);
1411 }
1412 rcu_read_unlock();
1413
1414 /* data/bss scanning */
Catalin Marinas93ada572015-06-24 16:58:37 -07001415 scan_large_block(_sdata, _edata);
1416 scan_large_block(__bss_start, __bss_stop);
Jakub Kicinskid7c19b02016-11-10 10:46:44 -08001417 scan_large_block(__start_data_ro_after_init, __end_data_ro_after_init);
Catalin Marinas3c7b4e62009-06-11 13:22:39 +01001418
1419#ifdef CONFIG_SMP
1420 /* per-cpu sections scanning */
1421 for_each_possible_cpu(i)
Catalin Marinas93ada572015-06-24 16:58:37 -07001422 scan_large_block(__per_cpu_start + per_cpu_offset(i),
1423 __per_cpu_end + per_cpu_offset(i));
Catalin Marinas3c7b4e62009-06-11 13:22:39 +01001424#endif
1425
1426 /*
Laura Abbott029aeff2011-11-15 23:49:09 +00001427 * Struct page scanning for each node.
Catalin Marinas3c7b4e62009-06-11 13:22:39 +01001428 */
Vladimir Davydovbfc8c902014-06-04 16:07:18 -07001429 get_online_mems();
Catalin Marinas3c7b4e62009-06-11 13:22:39 +01001430 for_each_online_node(i) {
Cody P Schafer108bcc92013-02-22 16:35:23 -08001431 unsigned long start_pfn = node_start_pfn(i);
1432 unsigned long end_pfn = node_end_pfn(i);
Catalin Marinas3c7b4e62009-06-11 13:22:39 +01001433 unsigned long pfn;
1434
1435 for (pfn = start_pfn; pfn < end_pfn; pfn++) {
1436 struct page *page;
1437
1438 if (!pfn_valid(pfn))
1439 continue;
1440 page = pfn_to_page(pfn);
1441 /* only scan if page is in use */
1442 if (page_count(page) == 0)
1443 continue;
Catalin Marinas93ada572015-06-24 16:58:37 -07001444 scan_block(page, page + 1, NULL);
Andrew Morton3cb026d2017-12-14 15:32:31 -08001445 if (!(pfn & 63))
Yisheng Xie7b862382017-11-29 16:11:08 -08001446 cond_resched();
Catalin Marinas3c7b4e62009-06-11 13:22:39 +01001447 }
1448 }
Vladimir Davydovbfc8c902014-06-04 16:07:18 -07001449 put_online_mems();
Catalin Marinas3c7b4e62009-06-11 13:22:39 +01001450
1451 /*
Catalin Marinas43ed5d62009-09-01 11:12:44 +01001452 * Scanning the task stacks (may introduce false negatives).
Catalin Marinas3c7b4e62009-06-11 13:22:39 +01001453 */
1454 if (kmemleak_stack_scan) {
Catalin Marinas43ed5d62009-09-01 11:12:44 +01001455 struct task_struct *p, *g;
1456
Catalin Marinas3c7b4e62009-06-11 13:22:39 +01001457 read_lock(&tasklist_lock);
Catalin Marinas43ed5d62009-09-01 11:12:44 +01001458 do_each_thread(g, p) {
Catalin Marinas37df49f2016-10-27 17:46:47 -07001459 void *stack = try_get_task_stack(p);
1460 if (stack) {
1461 scan_block(stack, stack + THREAD_SIZE, NULL);
1462 put_task_stack(p);
1463 }
Catalin Marinas43ed5d62009-09-01 11:12:44 +01001464 } while_each_thread(g, p);
Catalin Marinas3c7b4e62009-06-11 13:22:39 +01001465 read_unlock(&tasklist_lock);
1466 }
1467
1468 /*
1469 * Scan the objects already referenced from the sections scanned
Catalin Marinas04609ccc2009-10-28 13:33:12 +00001470 * above.
Catalin Marinas3c7b4e62009-06-11 13:22:39 +01001471 */
Catalin Marinas04609ccc2009-10-28 13:33:12 +00001472 scan_gray_list();
Catalin Marinas25873622009-07-07 10:32:58 +01001473
1474 /*
Catalin Marinas04609ccc2009-10-28 13:33:12 +00001475 * Check for new or unreferenced objects modified since the previous
1476 * scan and color them gray until the next scan.
Catalin Marinas25873622009-07-07 10:32:58 +01001477 */
1478 rcu_read_lock();
1479 list_for_each_entry_rcu(object, &object_list, object_list) {
1480 spin_lock_irqsave(&object->lock, flags);
Catalin Marinas04609ccc2009-10-28 13:33:12 +00001481 if (color_white(object) && (object->flags & OBJECT_ALLOCATED)
1482 && update_checksum(object) && get_object(object)) {
1483 /* color it gray temporarily */
1484 object->count = object->min_count;
Catalin Marinas25873622009-07-07 10:32:58 +01001485 list_add_tail(&object->gray_list, &gray_list);
1486 }
1487 spin_unlock_irqrestore(&object->lock, flags);
1488 }
1489 rcu_read_unlock();
1490
Catalin Marinas04609ccc2009-10-28 13:33:12 +00001491 /*
1492 * Re-scan the gray list for modified unreferenced objects.
1493 */
1494 scan_gray_list();
Catalin Marinas4698c1f2009-06-26 17:38:27 +01001495
1496 /*
Catalin Marinas04609ccc2009-10-28 13:33:12 +00001497 * If scanning was stopped do not report any new unreferenced objects.
Catalin Marinas17bb9e02009-06-29 17:13:56 +01001498 */
Catalin Marinas04609ccc2009-10-28 13:33:12 +00001499 if (scan_should_stop())
Catalin Marinas17bb9e02009-06-29 17:13:56 +01001500 return;
1501
1502 /*
Catalin Marinas4698c1f2009-06-26 17:38:27 +01001503 * Scanning result reporting.
1504 */
1505 rcu_read_lock();
1506 list_for_each_entry_rcu(object, &object_list, object_list) {
1507 spin_lock_irqsave(&object->lock, flags);
1508 if (unreferenced_object(object) &&
1509 !(object->flags & OBJECT_REPORTED)) {
1510 object->flags |= OBJECT_REPORTED;
1511 new_leaks++;
1512 }
1513 spin_unlock_irqrestore(&object->lock, flags);
1514 }
1515 rcu_read_unlock();
1516
Li Zefandc9b3f42014-04-03 14:46:26 -07001517 if (new_leaks) {
1518 kmemleak_found_leaks = true;
1519
Joe Perches756a025f02016-03-17 14:19:47 -07001520 pr_info("%d new suspected memory leaks (see /sys/kernel/debug/kmemleak)\n",
1521 new_leaks);
Li Zefandc9b3f42014-04-03 14:46:26 -07001522 }
Catalin Marinas4698c1f2009-06-26 17:38:27 +01001523
Catalin Marinas3c7b4e62009-06-11 13:22:39 +01001524}
1525
1526/*
1527 * Thread function performing automatic memory scanning. Unreferenced objects
1528 * at the end of a memory scan are reported but only the first time.
1529 */
1530static int kmemleak_scan_thread(void *arg)
1531{
1532 static int first_run = 1;
1533
Joe Perchesae281062009-06-23 14:40:26 +01001534 pr_info("Automatic memory scanning thread started\n");
Catalin Marinasbf2a76b2009-07-07 10:32:55 +01001535 set_user_nice(current, 10);
Catalin Marinas3c7b4e62009-06-11 13:22:39 +01001536
1537 /*
1538 * Wait before the first scan to allow the system to fully initialize.
1539 */
1540 if (first_run) {
Vegard Nossum98c42d92016-07-28 15:48:32 -07001541 signed long timeout = msecs_to_jiffies(SECS_FIRST_SCAN * 1000);
Catalin Marinas3c7b4e62009-06-11 13:22:39 +01001542 first_run = 0;
Vegard Nossum98c42d92016-07-28 15:48:32 -07001543 while (timeout && !kthread_should_stop())
1544 timeout = schedule_timeout_interruptible(timeout);
Catalin Marinas3c7b4e62009-06-11 13:22:39 +01001545 }
1546
1547 while (!kthread_should_stop()) {
Catalin Marinas3c7b4e62009-06-11 13:22:39 +01001548 signed long timeout = jiffies_scan_wait;
1549
1550 mutex_lock(&scan_mutex);
Catalin Marinas3c7b4e62009-06-11 13:22:39 +01001551 kmemleak_scan();
Catalin Marinas3c7b4e62009-06-11 13:22:39 +01001552 mutex_unlock(&scan_mutex);
Catalin Marinas4698c1f2009-06-26 17:38:27 +01001553
Catalin Marinas3c7b4e62009-06-11 13:22:39 +01001554 /* wait before the next scan */
1555 while (timeout && !kthread_should_stop())
1556 timeout = schedule_timeout_interruptible(timeout);
1557 }
1558
Joe Perchesae281062009-06-23 14:40:26 +01001559 pr_info("Automatic memory scanning thread ended\n");
Catalin Marinas3c7b4e62009-06-11 13:22:39 +01001560
1561 return 0;
1562}
1563
1564/*
1565 * Start the automatic memory scanning thread. This function must be called
Catalin Marinas4698c1f2009-06-26 17:38:27 +01001566 * with the scan_mutex held.
Catalin Marinas3c7b4e62009-06-11 13:22:39 +01001567 */
Luis R. Rodriguez7eb0d5e2009-09-08 17:31:45 +01001568static void start_scan_thread(void)
Catalin Marinas3c7b4e62009-06-11 13:22:39 +01001569{
1570 if (scan_thread)
1571 return;
1572 scan_thread = kthread_run(kmemleak_scan_thread, NULL, "kmemleak");
1573 if (IS_ERR(scan_thread)) {
Joe Perches598d8092016-03-17 14:19:44 -07001574 pr_warn("Failed to create the scan thread\n");
Catalin Marinas3c7b4e62009-06-11 13:22:39 +01001575 scan_thread = NULL;
1576 }
1577}
1578
1579/*
Vinayak Menon51db6742018-03-28 16:01:16 -07001580 * Stop the automatic memory scanning thread.
Catalin Marinas3c7b4e62009-06-11 13:22:39 +01001581 */
Luis R. Rodriguez7eb0d5e2009-09-08 17:31:45 +01001582static void stop_scan_thread(void)
Catalin Marinas3c7b4e62009-06-11 13:22:39 +01001583{
1584 if (scan_thread) {
1585 kthread_stop(scan_thread);
1586 scan_thread = NULL;
1587 }
1588}
1589
1590/*
1591 * Iterate over the object_list and return the first valid object at or after
1592 * the required position with its use_count incremented. The function triggers
1593 * a memory scanning when the pos argument points to the first position.
1594 */
1595static void *kmemleak_seq_start(struct seq_file *seq, loff_t *pos)
1596{
1597 struct kmemleak_object *object;
1598 loff_t n = *pos;
Catalin Marinasb87324d2009-07-07 10:32:58 +01001599 int err;
1600
1601 err = mutex_lock_interruptible(&scan_mutex);
1602 if (err < 0)
1603 return ERR_PTR(err);
Catalin Marinas3c7b4e62009-06-11 13:22:39 +01001604
Catalin Marinas3c7b4e62009-06-11 13:22:39 +01001605 rcu_read_lock();
1606 list_for_each_entry_rcu(object, &object_list, object_list) {
1607 if (n-- > 0)
1608 continue;
1609 if (get_object(object))
1610 goto out;
1611 }
1612 object = NULL;
1613out:
Catalin Marinas3c7b4e62009-06-11 13:22:39 +01001614 return object;
1615}
1616
1617/*
1618 * Return the next object in the object_list. The function decrements the
1619 * use_count of the previous object and increases that of the next one.
1620 */
1621static void *kmemleak_seq_next(struct seq_file *seq, void *v, loff_t *pos)
1622{
1623 struct kmemleak_object *prev_obj = v;
1624 struct kmemleak_object *next_obj = NULL;
Michael Wang58fac092012-08-17 12:33:34 +08001625 struct kmemleak_object *obj = prev_obj;
Catalin Marinas3c7b4e62009-06-11 13:22:39 +01001626
1627 ++(*pos);
Catalin Marinas3c7b4e62009-06-11 13:22:39 +01001628
Michael Wang58fac092012-08-17 12:33:34 +08001629 list_for_each_entry_continue_rcu(obj, &object_list, object_list) {
Catalin Marinas52c3ce42011-04-27 16:44:26 +01001630 if (get_object(obj)) {
1631 next_obj = obj;
Catalin Marinas3c7b4e62009-06-11 13:22:39 +01001632 break;
Catalin Marinas52c3ce42011-04-27 16:44:26 +01001633 }
Catalin Marinas3c7b4e62009-06-11 13:22:39 +01001634 }
Catalin Marinas288c8572009-07-07 10:32:57 +01001635
Catalin Marinas3c7b4e62009-06-11 13:22:39 +01001636 put_object(prev_obj);
1637 return next_obj;
1638}
1639
1640/*
1641 * Decrement the use_count of the last object required, if any.
1642 */
1643static void kmemleak_seq_stop(struct seq_file *seq, void *v)
1644{
Catalin Marinasb87324d2009-07-07 10:32:58 +01001645 if (!IS_ERR(v)) {
1646 /*
1647 * kmemleak_seq_start may return ERR_PTR if the scan_mutex
1648 * waiting was interrupted, so only release it if !IS_ERR.
1649 */
Catalin Marinasf5886c72009-07-29 16:26:57 +01001650 rcu_read_unlock();
Catalin Marinasb87324d2009-07-07 10:32:58 +01001651 mutex_unlock(&scan_mutex);
1652 if (v)
1653 put_object(v);
1654 }
Catalin Marinas3c7b4e62009-06-11 13:22:39 +01001655}
1656
1657/*
1658 * Print the information for an unreferenced object to the seq file.
1659 */
1660static int kmemleak_seq_show(struct seq_file *seq, void *v)
1661{
1662 struct kmemleak_object *object = v;
1663 unsigned long flags;
1664
1665 spin_lock_irqsave(&object->lock, flags);
Catalin Marinas288c8572009-07-07 10:32:57 +01001666 if ((object->flags & OBJECT_REPORTED) && unreferenced_object(object))
Catalin Marinas17bb9e02009-06-29 17:13:56 +01001667 print_unreferenced(seq, object);
Catalin Marinas3c7b4e62009-06-11 13:22:39 +01001668 spin_unlock_irqrestore(&object->lock, flags);
1669 return 0;
1670}
1671
1672static const struct seq_operations kmemleak_seq_ops = {
1673 .start = kmemleak_seq_start,
1674 .next = kmemleak_seq_next,
1675 .stop = kmemleak_seq_stop,
1676 .show = kmemleak_seq_show,
1677};
1678
1679static int kmemleak_open(struct inode *inode, struct file *file)
1680{
Catalin Marinasb87324d2009-07-07 10:32:58 +01001681 return seq_open(file, &kmemleak_seq_ops);
Catalin Marinas3c7b4e62009-06-11 13:22:39 +01001682}
1683
Catalin Marinas189d84e2009-08-27 14:29:15 +01001684static int dump_str_object_info(const char *str)
1685{
1686 unsigned long flags;
1687 struct kmemleak_object *object;
1688 unsigned long addr;
1689
Abhijit Pawardc053732012-12-18 14:23:27 -08001690 if (kstrtoul(str, 0, &addr))
1691 return -EINVAL;
Catalin Marinas189d84e2009-08-27 14:29:15 +01001692 object = find_and_get_object(addr, 0);
1693 if (!object) {
1694 pr_info("Unknown object at 0x%08lx\n", addr);
1695 return -EINVAL;
1696 }
1697
1698 spin_lock_irqsave(&object->lock, flags);
1699 dump_object_info(object);
1700 spin_unlock_irqrestore(&object->lock, flags);
1701
1702 put_object(object);
1703 return 0;
1704}
1705
Catalin Marinas3c7b4e62009-06-11 13:22:39 +01001706/*
Luis R. Rodriguez30b37102009-09-04 17:44:51 -07001707 * We use grey instead of black to ensure we can do future scans on the same
1708 * objects. If we did not do future scans these black objects could
1709 * potentially contain references to newly allocated objects in the future and
1710 * we'd end up with false positives.
1711 */
1712static void kmemleak_clear(void)
1713{
1714 struct kmemleak_object *object;
1715 unsigned long flags;
1716
1717 rcu_read_lock();
1718 list_for_each_entry_rcu(object, &object_list, object_list) {
1719 spin_lock_irqsave(&object->lock, flags);
1720 if ((object->flags & OBJECT_REPORTED) &&
1721 unreferenced_object(object))
Luis R. Rodrigueza1084c82009-09-04 17:44:52 -07001722 __paint_it(object, KMEMLEAK_GREY);
Luis R. Rodriguez30b37102009-09-04 17:44:51 -07001723 spin_unlock_irqrestore(&object->lock, flags);
1724 }
1725 rcu_read_unlock();
Li Zefandc9b3f42014-04-03 14:46:26 -07001726
1727 kmemleak_found_leaks = false;
Luis R. Rodriguez30b37102009-09-04 17:44:51 -07001728}
1729
Li Zefanc89da702014-04-03 14:46:27 -07001730static void __kmemleak_do_cleanup(void);
1731
Luis R. Rodriguez30b37102009-09-04 17:44:51 -07001732/*
Catalin Marinas3c7b4e62009-06-11 13:22:39 +01001733 * File write operation to configure kmemleak at run-time. The following
1734 * commands can be written to the /sys/kernel/debug/kmemleak file:
1735 * off - disable kmemleak (irreversible)
1736 * stack=on - enable the task stacks scanning
1737 * stack=off - disable the tasks stacks scanning
1738 * scan=on - start the automatic memory scanning thread
1739 * scan=off - stop the automatic memory scanning thread
1740 * scan=... - set the automatic memory scanning period in seconds (0 to
1741 * disable it)
Catalin Marinas4698c1f2009-06-26 17:38:27 +01001742 * scan - trigger a memory scan
Luis R. Rodriguez30b37102009-09-04 17:44:51 -07001743 * clear - mark all current reported unreferenced kmemleak objects as
Li Zefanc89da702014-04-03 14:46:27 -07001744 * grey to ignore printing them, or free all kmemleak objects
1745 * if kmemleak has been disabled.
Catalin Marinas189d84e2009-08-27 14:29:15 +01001746 * dump=... - dump information about the object found at the given address
Catalin Marinas3c7b4e62009-06-11 13:22:39 +01001747 */
1748static ssize_t kmemleak_write(struct file *file, const char __user *user_buf,
1749 size_t size, loff_t *ppos)
1750{
1751 char buf[64];
1752 int buf_size;
Catalin Marinasb87324d2009-07-07 10:32:58 +01001753 int ret;
Catalin Marinas3c7b4e62009-06-11 13:22:39 +01001754
1755 buf_size = min(size, (sizeof(buf) - 1));
1756 if (strncpy_from_user(buf, user_buf, buf_size) < 0)
1757 return -EFAULT;
1758 buf[buf_size] = 0;
1759
Catalin Marinasb87324d2009-07-07 10:32:58 +01001760 ret = mutex_lock_interruptible(&scan_mutex);
1761 if (ret < 0)
1762 return ret;
1763
Li Zefanc89da702014-04-03 14:46:27 -07001764 if (strncmp(buf, "clear", 5) == 0) {
Li Zefan8910ae82014-04-03 14:46:29 -07001765 if (kmemleak_enabled)
Li Zefanc89da702014-04-03 14:46:27 -07001766 kmemleak_clear();
1767 else
1768 __kmemleak_do_cleanup();
1769 goto out;
1770 }
1771
Li Zefan8910ae82014-04-03 14:46:29 -07001772 if (!kmemleak_enabled) {
Li Zefanc89da702014-04-03 14:46:27 -07001773 ret = -EBUSY;
1774 goto out;
1775 }
1776
Catalin Marinas3c7b4e62009-06-11 13:22:39 +01001777 if (strncmp(buf, "off", 3) == 0)
1778 kmemleak_disable();
1779 else if (strncmp(buf, "stack=on", 8) == 0)
1780 kmemleak_stack_scan = 1;
1781 else if (strncmp(buf, "stack=off", 9) == 0)
1782 kmemleak_stack_scan = 0;
1783 else if (strncmp(buf, "scan=on", 7) == 0)
1784 start_scan_thread();
1785 else if (strncmp(buf, "scan=off", 8) == 0)
1786 stop_scan_thread();
1787 else if (strncmp(buf, "scan=", 5) == 0) {
1788 unsigned long secs;
Catalin Marinas3c7b4e62009-06-11 13:22:39 +01001789
Jingoo Han3dbb95f2013-09-11 14:20:25 -07001790 ret = kstrtoul(buf + 5, 0, &secs);
Catalin Marinasb87324d2009-07-07 10:32:58 +01001791 if (ret < 0)
1792 goto out;
Catalin Marinas3c7b4e62009-06-11 13:22:39 +01001793 stop_scan_thread();
1794 if (secs) {
1795 jiffies_scan_wait = msecs_to_jiffies(secs * 1000);
1796 start_scan_thread();
1797 }
Catalin Marinas4698c1f2009-06-26 17:38:27 +01001798 } else if (strncmp(buf, "scan", 4) == 0)
1799 kmemleak_scan();
Catalin Marinas189d84e2009-08-27 14:29:15 +01001800 else if (strncmp(buf, "dump=", 5) == 0)
1801 ret = dump_str_object_info(buf + 5);
Catalin Marinas4698c1f2009-06-26 17:38:27 +01001802 else
Catalin Marinasb87324d2009-07-07 10:32:58 +01001803 ret = -EINVAL;
1804
1805out:
1806 mutex_unlock(&scan_mutex);
1807 if (ret < 0)
1808 return ret;
Catalin Marinas3c7b4e62009-06-11 13:22:39 +01001809
1810 /* ignore the rest of the buffer, only one command at a time */
1811 *ppos += size;
1812 return size;
1813}
1814
1815static const struct file_operations kmemleak_fops = {
1816 .owner = THIS_MODULE,
1817 .open = kmemleak_open,
1818 .read = seq_read,
1819 .write = kmemleak_write,
1820 .llseek = seq_lseek,
Li Zefan5f3bf192014-04-03 14:46:28 -07001821 .release = seq_release,
Catalin Marinas3c7b4e62009-06-11 13:22:39 +01001822};
1823
Li Zefanc89da702014-04-03 14:46:27 -07001824static void __kmemleak_do_cleanup(void)
1825{
1826 struct kmemleak_object *object;
1827
1828 rcu_read_lock();
1829 list_for_each_entry_rcu(object, &object_list, object_list)
1830 delete_object_full(object->pointer);
1831 rcu_read_unlock();
1832}
1833
Catalin Marinas3c7b4e62009-06-11 13:22:39 +01001834/*
Catalin Marinas74341702011-09-29 11:50:07 +01001835 * Stop the memory scanning thread and free the kmemleak internal objects if
1836 * no previous scan thread (otherwise, kmemleak may still have some useful
1837 * information on memory leaks).
Catalin Marinas3c7b4e62009-06-11 13:22:39 +01001838 */
Catalin Marinas179a8102009-09-07 10:14:42 +01001839static void kmemleak_do_cleanup(struct work_struct *work)
Catalin Marinas3c7b4e62009-06-11 13:22:39 +01001840{
Catalin Marinas4698c1f2009-06-26 17:38:27 +01001841 stop_scan_thread();
1842
Vinayak Menon51db6742018-03-28 16:01:16 -07001843 mutex_lock(&scan_mutex);
Catalin Marinasc5f3b1a2015-06-24 16:58:26 -07001844 /*
Vinayak Menon51db6742018-03-28 16:01:16 -07001845 * Once it is made sure that kmemleak_scan has stopped, it is safe to no
1846 * longer track object freeing. Ordering of the scan thread stopping and
1847 * the memory accesses below is guaranteed by the kthread_stop()
1848 * function.
Catalin Marinasc5f3b1a2015-06-24 16:58:26 -07001849 */
1850 kmemleak_free_enabled = 0;
Vinayak Menon51db6742018-03-28 16:01:16 -07001851 mutex_unlock(&scan_mutex);
Catalin Marinasc5f3b1a2015-06-24 16:58:26 -07001852
Li Zefanc89da702014-04-03 14:46:27 -07001853 if (!kmemleak_found_leaks)
1854 __kmemleak_do_cleanup();
1855 else
Joe Perches756a025f02016-03-17 14:19:47 -07001856 pr_info("Kmemleak disabled without freeing internal data. Reclaim the memory with \"echo clear > /sys/kernel/debug/kmemleak\".\n");
Catalin Marinas3c7b4e62009-06-11 13:22:39 +01001857}
1858
Catalin Marinas179a8102009-09-07 10:14:42 +01001859static DECLARE_WORK(cleanup_work, kmemleak_do_cleanup);
Catalin Marinas3c7b4e62009-06-11 13:22:39 +01001860
1861/*
1862 * Disable kmemleak. No memory allocation/freeing will be traced once this
1863 * function is called. Disabling kmemleak is an irreversible operation.
1864 */
1865static void kmemleak_disable(void)
1866{
1867 /* atomically check whether it was already invoked */
Li Zefan8910ae82014-04-03 14:46:29 -07001868 if (cmpxchg(&kmemleak_error, 0, 1))
Catalin Marinas3c7b4e62009-06-11 13:22:39 +01001869 return;
1870
1871 /* stop any memory operation tracing */
Li Zefan8910ae82014-04-03 14:46:29 -07001872 kmemleak_enabled = 0;
Catalin Marinas3c7b4e62009-06-11 13:22:39 +01001873
1874 /* check whether it is too early for a kernel thread */
Li Zefan8910ae82014-04-03 14:46:29 -07001875 if (kmemleak_initialized)
Catalin Marinas179a8102009-09-07 10:14:42 +01001876 schedule_work(&cleanup_work);
Catalin Marinasc5f3b1a2015-06-24 16:58:26 -07001877 else
1878 kmemleak_free_enabled = 0;
Catalin Marinas3c7b4e62009-06-11 13:22:39 +01001879
1880 pr_info("Kernel memory leak detector disabled\n");
1881}
1882
1883/*
1884 * Allow boot-time kmemleak disabling (enabled by default).
1885 */
1886static int kmemleak_boot_config(char *str)
1887{
1888 if (!str)
1889 return -EINVAL;
1890 if (strcmp(str, "off") == 0)
1891 kmemleak_disable();
Jason Baronab0155a2010-07-19 11:54:17 +01001892 else if (strcmp(str, "on") == 0)
1893 kmemleak_skip_disable = 1;
1894 else
Catalin Marinas3c7b4e62009-06-11 13:22:39 +01001895 return -EINVAL;
1896 return 0;
1897}
1898early_param("kmemleak", kmemleak_boot_config);
1899
Catalin Marinas5f790202011-09-28 12:17:03 +01001900static void __init print_log_trace(struct early_log *log)
1901{
1902 struct stack_trace trace;
1903
1904 trace.nr_entries = log->trace_len;
1905 trace.entries = log->trace;
1906
1907 pr_notice("Early log backtrace:\n");
1908 print_stack_trace(&trace, 2);
1909}
1910
Catalin Marinas3c7b4e62009-06-11 13:22:39 +01001911/*
Catalin Marinas20301172009-06-17 18:29:04 +01001912 * Kmemleak initialization.
Catalin Marinas3c7b4e62009-06-11 13:22:39 +01001913 */
1914void __init kmemleak_init(void)
1915{
1916 int i;
1917 unsigned long flags;
1918
Jason Baronab0155a2010-07-19 11:54:17 +01001919#ifdef CONFIG_DEBUG_KMEMLEAK_DEFAULT_OFF
1920 if (!kmemleak_skip_disable) {
Catalin Marinas3551a922014-05-09 15:36:59 -07001921 kmemleak_early_log = 0;
Jason Baronab0155a2010-07-19 11:54:17 +01001922 kmemleak_disable();
1923 return;
1924 }
1925#endif
1926
Catalin Marinas3c7b4e62009-06-11 13:22:39 +01001927 jiffies_min_age = msecs_to_jiffies(MSECS_MIN_AGE);
1928 jiffies_scan_wait = msecs_to_jiffies(SECS_SCAN_WAIT * 1000);
1929
1930 object_cache = KMEM_CACHE(kmemleak_object, SLAB_NOLEAKTRACE);
1931 scan_area_cache = KMEM_CACHE(kmemleak_scan_area, SLAB_NOLEAKTRACE);
Catalin Marinas3c7b4e62009-06-11 13:22:39 +01001932
Wang Kai21cd3a62015-09-08 15:03:41 -07001933 if (crt_early_log > ARRAY_SIZE(early_log))
Joe Perches598d8092016-03-17 14:19:44 -07001934 pr_warn("Early log buffer exceeded (%d), please increase DEBUG_KMEMLEAK_EARLY_LOG_SIZE\n",
1935 crt_early_log);
Catalin Marinasb6693002011-09-28 17:22:56 +01001936
Catalin Marinas3c7b4e62009-06-11 13:22:39 +01001937 /* the kernel is still in UP mode, so disabling the IRQs is enough */
1938 local_irq_save(flags);
Catalin Marinas3551a922014-05-09 15:36:59 -07001939 kmemleak_early_log = 0;
Li Zefan8910ae82014-04-03 14:46:29 -07001940 if (kmemleak_error) {
Catalin Marinasb6693002011-09-28 17:22:56 +01001941 local_irq_restore(flags);
1942 return;
Catalin Marinasc5f3b1a2015-06-24 16:58:26 -07001943 } else {
Li Zefan8910ae82014-04-03 14:46:29 -07001944 kmemleak_enabled = 1;
Catalin Marinasc5f3b1a2015-06-24 16:58:26 -07001945 kmemleak_free_enabled = 1;
1946 }
Catalin Marinas3c7b4e62009-06-11 13:22:39 +01001947 local_irq_restore(flags);
1948
1949 /*
1950 * This is the point where tracking allocations is safe. Automatic
1951 * scanning is started during the late initcall. Add the early logged
1952 * callbacks to the kmemleak infrastructure.
1953 */
1954 for (i = 0; i < crt_early_log; i++) {
1955 struct early_log *log = &early_log[i];
1956
1957 switch (log->op_type) {
1958 case KMEMLEAK_ALLOC:
Catalin Marinasfd678962009-08-27 14:29:17 +01001959 early_alloc(log);
Catalin Marinas3c7b4e62009-06-11 13:22:39 +01001960 break;
Catalin Marinasf528f0b2011-09-26 17:12:53 +01001961 case KMEMLEAK_ALLOC_PERCPU:
1962 early_alloc_percpu(log);
1963 break;
Catalin Marinas3c7b4e62009-06-11 13:22:39 +01001964 case KMEMLEAK_FREE:
1965 kmemleak_free(log->ptr);
1966 break;
Catalin Marinas53238a62009-07-07 10:33:00 +01001967 case KMEMLEAK_FREE_PART:
1968 kmemleak_free_part(log->ptr, log->size);
1969 break;
Catalin Marinasf528f0b2011-09-26 17:12:53 +01001970 case KMEMLEAK_FREE_PERCPU:
1971 kmemleak_free_percpu(log->ptr);
1972 break;
Catalin Marinas3c7b4e62009-06-11 13:22:39 +01001973 case KMEMLEAK_NOT_LEAK:
1974 kmemleak_not_leak(log->ptr);
1975 break;
1976 case KMEMLEAK_IGNORE:
1977 kmemleak_ignore(log->ptr);
1978 break;
1979 case KMEMLEAK_SCAN_AREA:
Catalin Marinasc017b4b2009-10-28 13:33:09 +00001980 kmemleak_scan_area(log->ptr, log->size, GFP_KERNEL);
Catalin Marinas3c7b4e62009-06-11 13:22:39 +01001981 break;
1982 case KMEMLEAK_NO_SCAN:
1983 kmemleak_no_scan(log->ptr);
1984 break;
1985 default:
Catalin Marinas5f790202011-09-28 12:17:03 +01001986 kmemleak_warn("Unknown early log operation: %d\n",
1987 log->op_type);
1988 }
1989
Li Zefan8910ae82014-04-03 14:46:29 -07001990 if (kmemleak_warning) {
Catalin Marinas5f790202011-09-28 12:17:03 +01001991 print_log_trace(log);
Li Zefan8910ae82014-04-03 14:46:29 -07001992 kmemleak_warning = 0;
Catalin Marinas3c7b4e62009-06-11 13:22:39 +01001993 }
1994 }
1995}
1996
1997/*
1998 * Late initialization function.
1999 */
2000static int __init kmemleak_late_init(void)
2001{
2002 struct dentry *dentry;
2003
Li Zefan8910ae82014-04-03 14:46:29 -07002004 kmemleak_initialized = 1;
Catalin Marinas3c7b4e62009-06-11 13:22:39 +01002005
Li Zefan8910ae82014-04-03 14:46:29 -07002006 if (kmemleak_error) {
Catalin Marinas3c7b4e62009-06-11 13:22:39 +01002007 /*
Lucas De Marchi25985ed2011-03-30 22:57:33 -03002008 * Some error occurred and kmemleak was disabled. There is a
Catalin Marinas3c7b4e62009-06-11 13:22:39 +01002009 * small chance that kmemleak_disable() was called immediately
2010 * after setting kmemleak_initialized and we may end up with
2011 * two clean-up threads but serialized by scan_mutex.
2012 */
Catalin Marinas179a8102009-09-07 10:14:42 +01002013 schedule_work(&cleanup_work);
Catalin Marinas3c7b4e62009-06-11 13:22:39 +01002014 return -ENOMEM;
2015 }
2016
2017 dentry = debugfs_create_file("kmemleak", S_IRUGO, NULL, NULL,
2018 &kmemleak_fops);
2019 if (!dentry)
Joe Perches598d8092016-03-17 14:19:44 -07002020 pr_warn("Failed to create the debugfs kmemleak file\n");
Catalin Marinas4698c1f2009-06-26 17:38:27 +01002021 mutex_lock(&scan_mutex);
Catalin Marinas3c7b4e62009-06-11 13:22:39 +01002022 start_scan_thread();
Catalin Marinas4698c1f2009-06-26 17:38:27 +01002023 mutex_unlock(&scan_mutex);
Catalin Marinas3c7b4e62009-06-11 13:22:39 +01002024
2025 pr_info("Kernel memory leak detector initialized\n");
2026
2027 return 0;
2028}
2029late_initcall(kmemleak_late_init);