blob: 98ac20bc0de9a3ad443ef9f8bd54cb1c58ed5de4 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * linux/mm/slab.c
3 * Written by Mark Hemment, 1996/97.
4 * (markhe@nextd.demon.co.uk)
5 *
6 * kmem_cache_destroy() + some cleanup - 1999 Andrea Arcangeli
7 *
8 * Major cleanup, different bufctl logic, per-cpu arrays
9 * (c) 2000 Manfred Spraul
10 *
11 * Cleanup, make the head arrays unconditional, preparation for NUMA
12 * (c) 2002 Manfred Spraul
13 *
14 * An implementation of the Slab Allocator as described in outline in;
15 * UNIX Internals: The New Frontiers by Uresh Vahalia
16 * Pub: Prentice Hall ISBN 0-13-101908-2
17 * or with a little more detail in;
18 * The Slab Allocator: An Object-Caching Kernel Memory Allocator
19 * Jeff Bonwick (Sun Microsystems).
20 * Presented at: USENIX Summer 1994 Technical Conference
21 *
22 * The memory is organized in caches, one cache for each object type.
23 * (e.g. inode_cache, dentry_cache, buffer_head, vm_area_struct)
24 * Each cache consists out of many slabs (they are small (usually one
25 * page long) and always contiguous), and each slab contains multiple
26 * initialized objects.
27 *
28 * This means, that your constructor is used only for newly allocated
29 * slabs and you must pass objects with the same intializations to
30 * kmem_cache_free.
31 *
32 * Each cache can only support one memory type (GFP_DMA, GFP_HIGHMEM,
33 * normal). If you need a special memory type, then must create a new
34 * cache for that memory type.
35 *
36 * In order to reduce fragmentation, the slabs are sorted in 3 groups:
37 * full slabs with 0 free objects
38 * partial slabs
39 * empty slabs with no allocated objects
40 *
41 * If partial slabs exist, then new allocations come from these slabs,
42 * otherwise from empty slabs or new slabs are allocated.
43 *
44 * kmem_cache_destroy() CAN CRASH if you try to allocate from the cache
45 * during kmem_cache_destroy(). The caller must prevent concurrent allocs.
46 *
47 * Each cache has a short per-cpu head array, most allocs
48 * and frees go into that array, and if that array overflows, then 1/2
49 * of the entries in the array are given back into the global cache.
50 * The head array is strictly LIFO and should improve the cache hit rates.
51 * On SMP, it additionally reduces the spinlock operations.
52 *
Andrew Mortona737b3e2006-03-22 00:08:11 -080053 * The c_cpuarray may not be read with enabled local interrupts -
Linus Torvalds1da177e2005-04-16 15:20:36 -070054 * it's changed with a smp_call_function().
55 *
56 * SMP synchronization:
57 * constructors and destructors are called without any locking.
Pekka Enberg343e0d72006-02-01 03:05:50 -080058 * Several members in struct kmem_cache and struct slab never change, they
Linus Torvalds1da177e2005-04-16 15:20:36 -070059 * are accessed without any locking.
60 * The per-cpu arrays are never accessed from the wrong cpu, no locking,
61 * and local interrupts are disabled so slab code is preempt-safe.
62 * The non-constant members are protected with a per-cache irq spinlock.
63 *
64 * Many thanks to Mark Hemment, who wrote another per-cpu slab patch
65 * in 2000 - many ideas in the current implementation are derived from
66 * his patch.
67 *
68 * Further notes from the original documentation:
69 *
70 * 11 April '97. Started multi-threading - markhe
Ingo Molnarfc0abb12006-01-18 17:42:33 -080071 * The global cache-chain is protected by the mutex 'cache_chain_mutex'.
Linus Torvalds1da177e2005-04-16 15:20:36 -070072 * The sem is only needed when accessing/extending the cache-chain, which
73 * can never happen inside an interrupt (kmem_cache_create(),
74 * kmem_cache_shrink() and kmem_cache_reap()).
75 *
76 * At present, each engine can be growing a cache. This should be blocked.
77 *
Christoph Lametere498be72005-09-09 13:03:32 -070078 * 15 March 2005. NUMA slab allocator.
79 * Shai Fultheim <shai@scalex86.org>.
80 * Shobhit Dayal <shobhit@calsoftinc.com>
81 * Alok N Kataria <alokk@calsoftinc.com>
82 * Christoph Lameter <christoph@lameter.com>
83 *
84 * Modified the slab allocator to be node aware on NUMA systems.
85 * Each node has its own list of partial, free and full slabs.
86 * All object allocations for a node occur from node specific slab lists.
Linus Torvalds1da177e2005-04-16 15:20:36 -070087 */
88
89#include <linux/config.h>
90#include <linux/slab.h>
91#include <linux/mm.h>
92#include <linux/swap.h>
93#include <linux/cache.h>
94#include <linux/interrupt.h>
95#include <linux/init.h>
96#include <linux/compiler.h>
Paul Jackson101a5002006-03-24 03:16:07 -080097#include <linux/cpuset.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070098#include <linux/seq_file.h>
99#include <linux/notifier.h>
100#include <linux/kallsyms.h>
101#include <linux/cpu.h>
102#include <linux/sysctl.h>
103#include <linux/module.h>
104#include <linux/rcupdate.h>
Paulo Marques543537b2005-06-23 00:09:02 -0700105#include <linux/string.h>
Christoph Lametere498be72005-09-09 13:03:32 -0700106#include <linux/nodemask.h>
Christoph Lameterdc85da12006-01-18 17:42:36 -0800107#include <linux/mempolicy.h>
Ingo Molnarfc0abb12006-01-18 17:42:33 -0800108#include <linux/mutex.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700109
110#include <asm/uaccess.h>
111#include <asm/cacheflush.h>
112#include <asm/tlbflush.h>
113#include <asm/page.h>
114
115/*
116 * DEBUG - 1 for kmem_cache_create() to honour; SLAB_DEBUG_INITIAL,
117 * SLAB_RED_ZONE & SLAB_POISON.
118 * 0 for faster, smaller code (especially in the critical paths).
119 *
120 * STATS - 1 to collect stats for /proc/slabinfo.
121 * 0 for faster, smaller code (especially in the critical paths).
122 *
123 * FORCED_DEBUG - 1 enables SLAB_RED_ZONE and SLAB_POISON (if possible)
124 */
125
126#ifdef CONFIG_DEBUG_SLAB
127#define DEBUG 1
128#define STATS 1
129#define FORCED_DEBUG 1
130#else
131#define DEBUG 0
132#define STATS 0
133#define FORCED_DEBUG 0
134#endif
135
Linus Torvalds1da177e2005-04-16 15:20:36 -0700136/* Shouldn't this be in a header file somewhere? */
137#define BYTES_PER_WORD sizeof(void *)
138
139#ifndef cache_line_size
140#define cache_line_size() L1_CACHE_BYTES
141#endif
142
143#ifndef ARCH_KMALLOC_MINALIGN
144/*
145 * Enforce a minimum alignment for the kmalloc caches.
146 * Usually, the kmalloc caches are cache_line_size() aligned, except when
147 * DEBUG and FORCED_DEBUG are enabled, then they are BYTES_PER_WORD aligned.
148 * Some archs want to perform DMA into kmalloc caches and need a guaranteed
149 * alignment larger than BYTES_PER_WORD. ARCH_KMALLOC_MINALIGN allows that.
150 * Note that this flag disables some debug features.
151 */
152#define ARCH_KMALLOC_MINALIGN 0
153#endif
154
155#ifndef ARCH_SLAB_MINALIGN
156/*
157 * Enforce a minimum alignment for all caches.
158 * Intended for archs that get misalignment faults even for BYTES_PER_WORD
159 * aligned buffers. Includes ARCH_KMALLOC_MINALIGN.
160 * If possible: Do not enable this flag for CONFIG_DEBUG_SLAB, it disables
161 * some debug features.
162 */
163#define ARCH_SLAB_MINALIGN 0
164#endif
165
166#ifndef ARCH_KMALLOC_FLAGS
167#define ARCH_KMALLOC_FLAGS SLAB_HWCACHE_ALIGN
168#endif
169
170/* Legal flag mask for kmem_cache_create(). */
171#if DEBUG
172# define CREATE_MASK (SLAB_DEBUG_INITIAL | SLAB_RED_ZONE | \
173 SLAB_POISON | SLAB_HWCACHE_ALIGN | \
Christoph Lameterac2b8982006-03-22 00:08:15 -0800174 SLAB_CACHE_DMA | \
Linus Torvalds1da177e2005-04-16 15:20:36 -0700175 SLAB_MUST_HWCACHE_ALIGN | SLAB_STORE_USER | \
176 SLAB_RECLAIM_ACCOUNT | SLAB_PANIC | \
Paul Jackson101a5002006-03-24 03:16:07 -0800177 SLAB_DESTROY_BY_RCU | SLAB_MEM_SPREAD)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700178#else
Christoph Lameterac2b8982006-03-22 00:08:15 -0800179# define CREATE_MASK (SLAB_HWCACHE_ALIGN | \
Linus Torvalds1da177e2005-04-16 15:20:36 -0700180 SLAB_CACHE_DMA | SLAB_MUST_HWCACHE_ALIGN | \
181 SLAB_RECLAIM_ACCOUNT | SLAB_PANIC | \
Paul Jackson101a5002006-03-24 03:16:07 -0800182 SLAB_DESTROY_BY_RCU | SLAB_MEM_SPREAD)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700183#endif
184
185/*
186 * kmem_bufctl_t:
187 *
188 * Bufctl's are used for linking objs within a slab
189 * linked offsets.
190 *
191 * This implementation relies on "struct page" for locating the cache &
192 * slab an object belongs to.
193 * This allows the bufctl structure to be small (one int), but limits
194 * the number of objects a slab (not a cache) can contain when off-slab
195 * bufctls are used. The limit is the size of the largest general cache
196 * that does not use off-slab slabs.
197 * For 32bit archs with 4 kB pages, is this 56.
198 * This is not serious, as it is only for large objects, when it is unwise
199 * to have too many per slab.
200 * Note: This limit can be raised by introducing a general cache whose size
201 * is less than 512 (PAGE_SIZE<<3), but greater than 256.
202 */
203
Kyle Moffettfa5b08d2005-09-03 15:55:03 -0700204typedef unsigned int kmem_bufctl_t;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700205#define BUFCTL_END (((kmem_bufctl_t)(~0U))-0)
206#define BUFCTL_FREE (((kmem_bufctl_t)(~0U))-1)
Al Viro871751e2006-03-25 03:06:39 -0800207#define BUFCTL_ACTIVE (((kmem_bufctl_t)(~0U))-2)
208#define SLAB_LIMIT (((kmem_bufctl_t)(~0U))-3)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700209
Linus Torvalds1da177e2005-04-16 15:20:36 -0700210/*
211 * struct slab
212 *
213 * Manages the objs in a slab. Placed either at the beginning of mem allocated
214 * for a slab, or allocated from an general cache.
215 * Slabs are chained into three list: fully used, partial, fully free slabs.
216 */
217struct slab {
Pekka Enbergb28a02d2006-01-08 01:00:37 -0800218 struct list_head list;
219 unsigned long colouroff;
220 void *s_mem; /* including colour offset */
221 unsigned int inuse; /* num of objs active in slab */
222 kmem_bufctl_t free;
223 unsigned short nodeid;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700224};
225
226/*
227 * struct slab_rcu
228 *
229 * slab_destroy on a SLAB_DESTROY_BY_RCU cache uses this structure to
230 * arrange for kmem_freepages to be called via RCU. This is useful if
231 * we need to approach a kernel structure obliquely, from its address
232 * obtained without the usual locking. We can lock the structure to
233 * stabilize it and check it's still at the given address, only if we
234 * can be sure that the memory has not been meanwhile reused for some
235 * other kind of object (which our subsystem's lock might corrupt).
236 *
237 * rcu_read_lock before reading the address, then rcu_read_unlock after
238 * taking the spinlock within the structure expected at that address.
239 *
240 * We assume struct slab_rcu can overlay struct slab when destroying.
241 */
242struct slab_rcu {
Pekka Enbergb28a02d2006-01-08 01:00:37 -0800243 struct rcu_head head;
Pekka Enberg343e0d72006-02-01 03:05:50 -0800244 struct kmem_cache *cachep;
Pekka Enbergb28a02d2006-01-08 01:00:37 -0800245 void *addr;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700246};
247
248/*
249 * struct array_cache
250 *
Linus Torvalds1da177e2005-04-16 15:20:36 -0700251 * Purpose:
252 * - LIFO ordering, to hand out cache-warm objects from _alloc
253 * - reduce the number of linked list operations
254 * - reduce spinlock operations
255 *
256 * The limit is stored in the per-cpu structure to reduce the data cache
257 * footprint.
258 *
259 */
260struct array_cache {
261 unsigned int avail;
262 unsigned int limit;
263 unsigned int batchcount;
264 unsigned int touched;
Christoph Lametere498be72005-09-09 13:03:32 -0700265 spinlock_t lock;
Andrew Mortona737b3e2006-03-22 00:08:11 -0800266 void *entry[0]; /*
267 * Must have this definition in here for the proper
268 * alignment of array_cache. Also simplifies accessing
269 * the entries.
270 * [0] is for gcc 2.95. It should really be [].
271 */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700272};
273
Andrew Mortona737b3e2006-03-22 00:08:11 -0800274/*
275 * bootstrap: The caches do not work without cpuarrays anymore, but the
276 * cpuarrays are allocated from the generic caches...
Linus Torvalds1da177e2005-04-16 15:20:36 -0700277 */
278#define BOOT_CPUCACHE_ENTRIES 1
279struct arraycache_init {
280 struct array_cache cache;
Pekka Enbergb28a02d2006-01-08 01:00:37 -0800281 void *entries[BOOT_CPUCACHE_ENTRIES];
Linus Torvalds1da177e2005-04-16 15:20:36 -0700282};
283
284/*
Christoph Lametere498be72005-09-09 13:03:32 -0700285 * The slab lists for all objects.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700286 */
287struct kmem_list3 {
Pekka Enbergb28a02d2006-01-08 01:00:37 -0800288 struct list_head slabs_partial; /* partial list first, better asm code */
289 struct list_head slabs_full;
290 struct list_head slabs_free;
291 unsigned long free_objects;
Pekka Enbergb28a02d2006-01-08 01:00:37 -0800292 unsigned int free_limit;
Ravikiran G Thirumalai2e1217c2006-02-04 23:27:56 -0800293 unsigned int colour_next; /* Per-node cache coloring */
Pekka Enbergb28a02d2006-01-08 01:00:37 -0800294 spinlock_t list_lock;
295 struct array_cache *shared; /* shared per node */
296 struct array_cache **alien; /* on other nodes */
Christoph Lameter35386e32006-03-22 00:09:05 -0800297 unsigned long next_reap; /* updated without locking */
298 int free_touched; /* updated without locking */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700299};
300
Christoph Lametere498be72005-09-09 13:03:32 -0700301/*
302 * Need this for bootstrapping a per node allocator.
303 */
304#define NUM_INIT_LISTS (2 * MAX_NUMNODES + 1)
305struct kmem_list3 __initdata initkmem_list3[NUM_INIT_LISTS];
306#define CACHE_CACHE 0
307#define SIZE_AC 1
308#define SIZE_L3 (1 + MAX_NUMNODES)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700309
Christoph Lametere498be72005-09-09 13:03:32 -0700310/*
Andrew Mortona737b3e2006-03-22 00:08:11 -0800311 * This function must be completely optimized away if a constant is passed to
312 * it. Mostly the same as what is in linux/slab.h except it returns an index.
Christoph Lametere498be72005-09-09 13:03:32 -0700313 */
Ivan Kokshaysky7243cc02005-09-22 21:43:58 -0700314static __always_inline int index_of(const size_t size)
Christoph Lametere498be72005-09-09 13:03:32 -0700315{
Steven Rostedt5ec8a842006-02-01 03:05:44 -0800316 extern void __bad_size(void);
317
Christoph Lametere498be72005-09-09 13:03:32 -0700318 if (__builtin_constant_p(size)) {
319 int i = 0;
320
321#define CACHE(x) \
322 if (size <=x) \
323 return i; \
324 else \
325 i++;
326#include "linux/kmalloc_sizes.h"
327#undef CACHE
Steven Rostedt5ec8a842006-02-01 03:05:44 -0800328 __bad_size();
Ivan Kokshaysky7243cc02005-09-22 21:43:58 -0700329 } else
Steven Rostedt5ec8a842006-02-01 03:05:44 -0800330 __bad_size();
Christoph Lametere498be72005-09-09 13:03:32 -0700331 return 0;
332}
333
Ingo Molnare0a42722006-06-23 02:03:46 -0700334static int slab_early_init = 1;
335
Christoph Lametere498be72005-09-09 13:03:32 -0700336#define INDEX_AC index_of(sizeof(struct arraycache_init))
337#define INDEX_L3 index_of(sizeof(struct kmem_list3))
338
Pekka Enberg5295a742006-02-01 03:05:48 -0800339static void kmem_list3_init(struct kmem_list3 *parent)
Christoph Lametere498be72005-09-09 13:03:32 -0700340{
341 INIT_LIST_HEAD(&parent->slabs_full);
342 INIT_LIST_HEAD(&parent->slabs_partial);
343 INIT_LIST_HEAD(&parent->slabs_free);
344 parent->shared = NULL;
345 parent->alien = NULL;
Ravikiran G Thirumalai2e1217c2006-02-04 23:27:56 -0800346 parent->colour_next = 0;
Christoph Lametere498be72005-09-09 13:03:32 -0700347 spin_lock_init(&parent->list_lock);
348 parent->free_objects = 0;
349 parent->free_touched = 0;
350}
351
Andrew Mortona737b3e2006-03-22 00:08:11 -0800352#define MAKE_LIST(cachep, listp, slab, nodeid) \
353 do { \
354 INIT_LIST_HEAD(listp); \
355 list_splice(&(cachep->nodelists[nodeid]->slab), listp); \
Christoph Lametere498be72005-09-09 13:03:32 -0700356 } while (0)
357
Andrew Mortona737b3e2006-03-22 00:08:11 -0800358#define MAKE_ALL_LISTS(cachep, ptr, nodeid) \
359 do { \
Christoph Lametere498be72005-09-09 13:03:32 -0700360 MAKE_LIST((cachep), (&(ptr)->slabs_full), slabs_full, nodeid); \
361 MAKE_LIST((cachep), (&(ptr)->slabs_partial), slabs_partial, nodeid); \
362 MAKE_LIST((cachep), (&(ptr)->slabs_free), slabs_free, nodeid); \
363 } while (0)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700364
365/*
Pekka Enberg343e0d72006-02-01 03:05:50 -0800366 * struct kmem_cache
Linus Torvalds1da177e2005-04-16 15:20:36 -0700367 *
368 * manages a cache.
369 */
Pekka Enbergb28a02d2006-01-08 01:00:37 -0800370
Pekka J Enberg2109a2d2005-11-07 00:58:01 -0800371struct kmem_cache {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700372/* 1) per-cpu data, touched during every alloc/free */
Pekka Enbergb28a02d2006-01-08 01:00:37 -0800373 struct array_cache *array[NR_CPUS];
Ravikiran G Thirumalaib5d8ca72006-03-22 00:08:12 -0800374/* 2) Cache tunables. Protected by cache_chain_mutex */
Pekka Enbergb28a02d2006-01-08 01:00:37 -0800375 unsigned int batchcount;
376 unsigned int limit;
377 unsigned int shared;
Ravikiran G Thirumalaib5d8ca72006-03-22 00:08:12 -0800378
Manfred Spraul3dafccf2006-02-01 03:05:42 -0800379 unsigned int buffer_size;
Ravikiran G Thirumalaib5d8ca72006-03-22 00:08:12 -0800380/* 3) touched by every alloc & free from the backend */
Pekka Enbergb28a02d2006-01-08 01:00:37 -0800381 struct kmem_list3 *nodelists[MAX_NUMNODES];
Ravikiran G Thirumalaib5d8ca72006-03-22 00:08:12 -0800382
Andrew Mortona737b3e2006-03-22 00:08:11 -0800383 unsigned int flags; /* constant flags */
384 unsigned int num; /* # of objs per slab */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700385
Ravikiran G Thirumalaib5d8ca72006-03-22 00:08:12 -0800386/* 4) cache_grow/shrink */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700387 /* order of pgs per slab (2^n) */
Pekka Enbergb28a02d2006-01-08 01:00:37 -0800388 unsigned int gfporder;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700389
390 /* force GFP flags, e.g. GFP_DMA */
Pekka Enbergb28a02d2006-01-08 01:00:37 -0800391 gfp_t gfpflags;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700392
Andrew Mortona737b3e2006-03-22 00:08:11 -0800393 size_t colour; /* cache colouring range */
Pekka Enbergb28a02d2006-01-08 01:00:37 -0800394 unsigned int colour_off; /* colour offset */
Pekka Enberg343e0d72006-02-01 03:05:50 -0800395 struct kmem_cache *slabp_cache;
Pekka Enbergb28a02d2006-01-08 01:00:37 -0800396 unsigned int slab_size;
Andrew Mortona737b3e2006-03-22 00:08:11 -0800397 unsigned int dflags; /* dynamic flags */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700398
399 /* constructor func */
Pekka Enberg343e0d72006-02-01 03:05:50 -0800400 void (*ctor) (void *, struct kmem_cache *, unsigned long);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700401
402 /* de-constructor func */
Pekka Enberg343e0d72006-02-01 03:05:50 -0800403 void (*dtor) (void *, struct kmem_cache *, unsigned long);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700404
Ravikiran G Thirumalaib5d8ca72006-03-22 00:08:12 -0800405/* 5) cache creation/removal */
Pekka Enbergb28a02d2006-01-08 01:00:37 -0800406 const char *name;
407 struct list_head next;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700408
Ravikiran G Thirumalaib5d8ca72006-03-22 00:08:12 -0800409/* 6) statistics */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700410#if STATS
Pekka Enbergb28a02d2006-01-08 01:00:37 -0800411 unsigned long num_active;
412 unsigned long num_allocations;
413 unsigned long high_mark;
414 unsigned long grown;
415 unsigned long reaped;
416 unsigned long errors;
417 unsigned long max_freeable;
418 unsigned long node_allocs;
419 unsigned long node_frees;
Ravikiran G Thirumalaifb7faf32006-04-10 22:52:54 -0700420 unsigned long node_overflow;
Pekka Enbergb28a02d2006-01-08 01:00:37 -0800421 atomic_t allochit;
422 atomic_t allocmiss;
423 atomic_t freehit;
424 atomic_t freemiss;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700425#endif
426#if DEBUG
Manfred Spraul3dafccf2006-02-01 03:05:42 -0800427 /*
428 * If debugging is enabled, then the allocator can add additional
429 * fields and/or padding to every object. buffer_size contains the total
430 * object size including these internal fields, the following two
431 * variables contain the offset to the user object and its size.
432 */
433 int obj_offset;
434 int obj_size;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700435#endif
436};
437
438#define CFLGS_OFF_SLAB (0x80000000UL)
439#define OFF_SLAB(x) ((x)->flags & CFLGS_OFF_SLAB)
440
441#define BATCHREFILL_LIMIT 16
Andrew Mortona737b3e2006-03-22 00:08:11 -0800442/*
443 * Optimization question: fewer reaps means less probability for unnessary
444 * cpucache drain/refill cycles.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700445 *
Adrian Bunkdc6f3f22005-11-08 16:44:08 +0100446 * OTOH the cpuarrays can contain lots of objects,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700447 * which could lock up otherwise freeable slabs.
448 */
449#define REAPTIMEOUT_CPUC (2*HZ)
450#define REAPTIMEOUT_LIST3 (4*HZ)
451
452#if STATS
453#define STATS_INC_ACTIVE(x) ((x)->num_active++)
454#define STATS_DEC_ACTIVE(x) ((x)->num_active--)
455#define STATS_INC_ALLOCED(x) ((x)->num_allocations++)
456#define STATS_INC_GROWN(x) ((x)->grown++)
457#define STATS_INC_REAPED(x) ((x)->reaped++)
Andrew Mortona737b3e2006-03-22 00:08:11 -0800458#define STATS_SET_HIGH(x) \
459 do { \
460 if ((x)->num_active > (x)->high_mark) \
461 (x)->high_mark = (x)->num_active; \
462 } while (0)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700463#define STATS_INC_ERR(x) ((x)->errors++)
464#define STATS_INC_NODEALLOCS(x) ((x)->node_allocs++)
Christoph Lametere498be72005-09-09 13:03:32 -0700465#define STATS_INC_NODEFREES(x) ((x)->node_frees++)
Ravikiran G Thirumalaifb7faf32006-04-10 22:52:54 -0700466#define STATS_INC_ACOVERFLOW(x) ((x)->node_overflow++)
Andrew Mortona737b3e2006-03-22 00:08:11 -0800467#define STATS_SET_FREEABLE(x, i) \
468 do { \
469 if ((x)->max_freeable < i) \
470 (x)->max_freeable = i; \
471 } while (0)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700472#define STATS_INC_ALLOCHIT(x) atomic_inc(&(x)->allochit)
473#define STATS_INC_ALLOCMISS(x) atomic_inc(&(x)->allocmiss)
474#define STATS_INC_FREEHIT(x) atomic_inc(&(x)->freehit)
475#define STATS_INC_FREEMISS(x) atomic_inc(&(x)->freemiss)
476#else
477#define STATS_INC_ACTIVE(x) do { } while (0)
478#define STATS_DEC_ACTIVE(x) do { } while (0)
479#define STATS_INC_ALLOCED(x) do { } while (0)
480#define STATS_INC_GROWN(x) do { } while (0)
481#define STATS_INC_REAPED(x) do { } while (0)
482#define STATS_SET_HIGH(x) do { } while (0)
483#define STATS_INC_ERR(x) do { } while (0)
484#define STATS_INC_NODEALLOCS(x) do { } while (0)
Christoph Lametere498be72005-09-09 13:03:32 -0700485#define STATS_INC_NODEFREES(x) do { } while (0)
Ravikiran G Thirumalaifb7faf32006-04-10 22:52:54 -0700486#define STATS_INC_ACOVERFLOW(x) do { } while (0)
Andrew Mortona737b3e2006-03-22 00:08:11 -0800487#define STATS_SET_FREEABLE(x, i) do { } while (0)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700488#define STATS_INC_ALLOCHIT(x) do { } while (0)
489#define STATS_INC_ALLOCMISS(x) do { } while (0)
490#define STATS_INC_FREEHIT(x) do { } while (0)
491#define STATS_INC_FREEMISS(x) do { } while (0)
492#endif
493
494#if DEBUG
Andrew Mortona737b3e2006-03-22 00:08:11 -0800495/*
496 * Magic nums for obj red zoning.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700497 * Placed in the first word before and the first word after an obj.
498 */
499#define RED_INACTIVE 0x5A2CF071UL /* when obj is inactive */
500#define RED_ACTIVE 0x170FC2A5UL /* when obj is active */
501
502/* ...and for poisoning */
503#define POISON_INUSE 0x5a /* for use-uninitialised poisoning */
504#define POISON_FREE 0x6b /* for use-after-free poisoning */
505#define POISON_END 0xa5 /* end-byte of poisoning */
506
Andrew Mortona737b3e2006-03-22 00:08:11 -0800507/*
508 * memory layout of objects:
Linus Torvalds1da177e2005-04-16 15:20:36 -0700509 * 0 : objp
Manfred Spraul3dafccf2006-02-01 03:05:42 -0800510 * 0 .. cachep->obj_offset - BYTES_PER_WORD - 1: padding. This ensures that
Linus Torvalds1da177e2005-04-16 15:20:36 -0700511 * the end of an object is aligned with the end of the real
512 * allocation. Catches writes behind the end of the allocation.
Manfred Spraul3dafccf2006-02-01 03:05:42 -0800513 * cachep->obj_offset - BYTES_PER_WORD .. cachep->obj_offset - 1:
Linus Torvalds1da177e2005-04-16 15:20:36 -0700514 * redzone word.
Manfred Spraul3dafccf2006-02-01 03:05:42 -0800515 * cachep->obj_offset: The real object.
516 * cachep->buffer_size - 2* BYTES_PER_WORD: redzone word [BYTES_PER_WORD long]
Andrew Mortona737b3e2006-03-22 00:08:11 -0800517 * cachep->buffer_size - 1* BYTES_PER_WORD: last caller address
518 * [BYTES_PER_WORD long]
Linus Torvalds1da177e2005-04-16 15:20:36 -0700519 */
Pekka Enberg343e0d72006-02-01 03:05:50 -0800520static int obj_offset(struct kmem_cache *cachep)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700521{
Manfred Spraul3dafccf2006-02-01 03:05:42 -0800522 return cachep->obj_offset;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700523}
524
Pekka Enberg343e0d72006-02-01 03:05:50 -0800525static int obj_size(struct kmem_cache *cachep)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700526{
Manfred Spraul3dafccf2006-02-01 03:05:42 -0800527 return cachep->obj_size;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700528}
529
Pekka Enberg343e0d72006-02-01 03:05:50 -0800530static unsigned long *dbg_redzone1(struct kmem_cache *cachep, void *objp)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700531{
532 BUG_ON(!(cachep->flags & SLAB_RED_ZONE));
Manfred Spraul3dafccf2006-02-01 03:05:42 -0800533 return (unsigned long*) (objp+obj_offset(cachep)-BYTES_PER_WORD);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700534}
535
Pekka Enberg343e0d72006-02-01 03:05:50 -0800536static unsigned long *dbg_redzone2(struct kmem_cache *cachep, void *objp)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700537{
538 BUG_ON(!(cachep->flags & SLAB_RED_ZONE));
539 if (cachep->flags & SLAB_STORE_USER)
Manfred Spraul3dafccf2006-02-01 03:05:42 -0800540 return (unsigned long *)(objp + cachep->buffer_size -
Pekka Enbergb28a02d2006-01-08 01:00:37 -0800541 2 * BYTES_PER_WORD);
Manfred Spraul3dafccf2006-02-01 03:05:42 -0800542 return (unsigned long *)(objp + cachep->buffer_size - BYTES_PER_WORD);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700543}
544
Pekka Enberg343e0d72006-02-01 03:05:50 -0800545static void **dbg_userword(struct kmem_cache *cachep, void *objp)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700546{
547 BUG_ON(!(cachep->flags & SLAB_STORE_USER));
Manfred Spraul3dafccf2006-02-01 03:05:42 -0800548 return (void **)(objp + cachep->buffer_size - BYTES_PER_WORD);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700549}
550
551#else
552
Manfred Spraul3dafccf2006-02-01 03:05:42 -0800553#define obj_offset(x) 0
554#define obj_size(cachep) (cachep->buffer_size)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700555#define dbg_redzone1(cachep, objp) ({BUG(); (unsigned long *)NULL;})
556#define dbg_redzone2(cachep, objp) ({BUG(); (unsigned long *)NULL;})
557#define dbg_userword(cachep, objp) ({BUG(); (void **)NULL;})
558
559#endif
560
561/*
Andrew Mortona737b3e2006-03-22 00:08:11 -0800562 * Maximum size of an obj (in 2^order pages) and absolute limit for the gfp
563 * order.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700564 */
565#if defined(CONFIG_LARGE_ALLOCS)
566#define MAX_OBJ_ORDER 13 /* up to 32Mb */
567#define MAX_GFP_ORDER 13 /* up to 32Mb */
568#elif defined(CONFIG_MMU)
569#define MAX_OBJ_ORDER 5 /* 32 pages */
570#define MAX_GFP_ORDER 5 /* 32 pages */
571#else
572#define MAX_OBJ_ORDER 8 /* up to 1Mb */
573#define MAX_GFP_ORDER 8 /* up to 1Mb */
574#endif
575
576/*
577 * Do not go above this order unless 0 objects fit into the slab.
578 */
579#define BREAK_GFP_ORDER_HI 1
580#define BREAK_GFP_ORDER_LO 0
581static int slab_break_gfp_order = BREAK_GFP_ORDER_LO;
582
Andrew Mortona737b3e2006-03-22 00:08:11 -0800583/*
584 * Functions for storing/retrieving the cachep and or slab from the page
585 * allocator. These are used to find the slab an obj belongs to. With kfree(),
586 * these are used to find the cache which an obj belongs to.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700587 */
Pekka Enberg065d41c2005-11-13 16:06:46 -0800588static inline void page_set_cache(struct page *page, struct kmem_cache *cache)
589{
590 page->lru.next = (struct list_head *)cache;
591}
592
593static inline struct kmem_cache *page_get_cache(struct page *page)
594{
Nick Piggin84097512006-03-22 00:08:34 -0800595 if (unlikely(PageCompound(page)))
596 page = (struct page *)page_private(page);
Pekka Enbergddc2e812006-06-23 02:03:40 -0700597 BUG_ON(!PageSlab(page));
Pekka Enberg065d41c2005-11-13 16:06:46 -0800598 return (struct kmem_cache *)page->lru.next;
599}
600
601static inline void page_set_slab(struct page *page, struct slab *slab)
602{
603 page->lru.prev = (struct list_head *)slab;
604}
605
606static inline struct slab *page_get_slab(struct page *page)
607{
Nick Piggin84097512006-03-22 00:08:34 -0800608 if (unlikely(PageCompound(page)))
609 page = (struct page *)page_private(page);
Pekka Enbergddc2e812006-06-23 02:03:40 -0700610 BUG_ON(!PageSlab(page));
Pekka Enberg065d41c2005-11-13 16:06:46 -0800611 return (struct slab *)page->lru.prev;
612}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700613
Pekka Enberg6ed5eb22006-02-01 03:05:49 -0800614static inline struct kmem_cache *virt_to_cache(const void *obj)
615{
616 struct page *page = virt_to_page(obj);
617 return page_get_cache(page);
618}
619
620static inline struct slab *virt_to_slab(const void *obj)
621{
622 struct page *page = virt_to_page(obj);
623 return page_get_slab(page);
624}
625
Pekka Enberg8fea4e92006-03-22 00:08:10 -0800626static inline void *index_to_obj(struct kmem_cache *cache, struct slab *slab,
627 unsigned int idx)
628{
629 return slab->s_mem + cache->buffer_size * idx;
630}
631
632static inline unsigned int obj_to_index(struct kmem_cache *cache,
633 struct slab *slab, void *obj)
634{
635 return (unsigned)(obj - slab->s_mem) / cache->buffer_size;
636}
637
Andrew Mortona737b3e2006-03-22 00:08:11 -0800638/*
639 * These are the default caches for kmalloc. Custom caches can have other sizes.
640 */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700641struct cache_sizes malloc_sizes[] = {
642#define CACHE(x) { .cs_size = (x) },
643#include <linux/kmalloc_sizes.h>
644 CACHE(ULONG_MAX)
645#undef CACHE
646};
647EXPORT_SYMBOL(malloc_sizes);
648
649/* Must match cache_sizes above. Out of line to keep cache footprint low. */
650struct cache_names {
651 char *name;
652 char *name_dma;
653};
654
655static struct cache_names __initdata cache_names[] = {
656#define CACHE(x) { .name = "size-" #x, .name_dma = "size-" #x "(DMA)" },
657#include <linux/kmalloc_sizes.h>
Pekka Enbergb28a02d2006-01-08 01:00:37 -0800658 {NULL,}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700659#undef CACHE
660};
661
662static struct arraycache_init initarray_cache __initdata =
Pekka Enbergb28a02d2006-01-08 01:00:37 -0800663 { {0, BOOT_CPUCACHE_ENTRIES, 1, 0} };
Linus Torvalds1da177e2005-04-16 15:20:36 -0700664static struct arraycache_init initarray_generic =
Pekka Enbergb28a02d2006-01-08 01:00:37 -0800665 { {0, BOOT_CPUCACHE_ENTRIES, 1, 0} };
Linus Torvalds1da177e2005-04-16 15:20:36 -0700666
667/* internal cache of cache description objs */
Pekka Enberg343e0d72006-02-01 03:05:50 -0800668static struct kmem_cache cache_cache = {
Pekka Enbergb28a02d2006-01-08 01:00:37 -0800669 .batchcount = 1,
670 .limit = BOOT_CPUCACHE_ENTRIES,
671 .shared = 1,
Pekka Enberg343e0d72006-02-01 03:05:50 -0800672 .buffer_size = sizeof(struct kmem_cache),
Pekka Enbergb28a02d2006-01-08 01:00:37 -0800673 .name = "kmem_cache",
Linus Torvalds1da177e2005-04-16 15:20:36 -0700674#if DEBUG
Pekka Enberg343e0d72006-02-01 03:05:50 -0800675 .obj_size = sizeof(struct kmem_cache),
Linus Torvalds1da177e2005-04-16 15:20:36 -0700676#endif
677};
678
679/* Guard access to the cache-chain. */
Ingo Molnarfc0abb12006-01-18 17:42:33 -0800680static DEFINE_MUTEX(cache_chain_mutex);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700681static struct list_head cache_chain;
682
683/*
Andrew Mortona737b3e2006-03-22 00:08:11 -0800684 * vm_enough_memory() looks at this to determine how many slab-allocated pages
685 * are possibly freeable under pressure
Linus Torvalds1da177e2005-04-16 15:20:36 -0700686 *
687 * SLAB_RECLAIM_ACCOUNT turns this on per-slab
688 */
689atomic_t slab_reclaim_pages;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700690
691/*
692 * chicken and egg problem: delay the per-cpu array allocation
693 * until the general caches are up.
694 */
695static enum {
696 NONE,
Christoph Lametere498be72005-09-09 13:03:32 -0700697 PARTIAL_AC,
698 PARTIAL_L3,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700699 FULL
700} g_cpucache_up;
701
Mike Kravetz39d24e62006-05-15 09:44:13 -0700702/*
703 * used by boot code to determine if it can use slab based allocator
704 */
705int slab_is_available(void)
706{
707 return g_cpucache_up == FULL;
708}
709
Linus Torvalds1da177e2005-04-16 15:20:36 -0700710static DEFINE_PER_CPU(struct work_struct, reap_work);
711
Andrew Mortona737b3e2006-03-22 00:08:11 -0800712static void free_block(struct kmem_cache *cachep, void **objpp, int len,
713 int node);
Pekka Enberg343e0d72006-02-01 03:05:50 -0800714static void enable_cpucache(struct kmem_cache *cachep);
Pekka Enbergb28a02d2006-01-08 01:00:37 -0800715static void cache_reap(void *unused);
Pekka Enberg343e0d72006-02-01 03:05:50 -0800716static int __node_shrink(struct kmem_cache *cachep, int node);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700717
Pekka Enberg343e0d72006-02-01 03:05:50 -0800718static inline struct array_cache *cpu_cache_get(struct kmem_cache *cachep)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700719{
720 return cachep->array[smp_processor_id()];
721}
722
Andrew Mortona737b3e2006-03-22 00:08:11 -0800723static inline struct kmem_cache *__find_general_cachep(size_t size,
724 gfp_t gfpflags)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700725{
726 struct cache_sizes *csizep = malloc_sizes;
727
728#if DEBUG
729 /* This happens if someone tries to call
Pekka Enbergb28a02d2006-01-08 01:00:37 -0800730 * kmem_cache_create(), or __kmalloc(), before
731 * the generic caches are initialized.
732 */
Alok Katariac7e43c72005-09-14 12:17:53 -0700733 BUG_ON(malloc_sizes[INDEX_AC].cs_cachep == NULL);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700734#endif
735 while (size > csizep->cs_size)
736 csizep++;
737
738 /*
Martin Hicks0abf40c2005-09-03 15:54:54 -0700739 * Really subtle: The last entry with cs->cs_size==ULONG_MAX
Linus Torvalds1da177e2005-04-16 15:20:36 -0700740 * has cs_{dma,}cachep==NULL. Thus no special case
741 * for large kmalloc calls required.
742 */
743 if (unlikely(gfpflags & GFP_DMA))
744 return csizep->cs_dmacachep;
745 return csizep->cs_cachep;
746}
747
Pekka Enberg343e0d72006-02-01 03:05:50 -0800748struct kmem_cache *kmem_find_general_cachep(size_t size, gfp_t gfpflags)
Manfred Spraul97e2bde2005-05-01 08:58:38 -0700749{
750 return __find_general_cachep(size, gfpflags);
751}
752EXPORT_SYMBOL(kmem_find_general_cachep);
753
Steven Rostedtfbaccac2006-02-01 03:05:45 -0800754static size_t slab_mgmt_size(size_t nr_objs, size_t align)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700755{
Steven Rostedtfbaccac2006-02-01 03:05:45 -0800756 return ALIGN(sizeof(struct slab)+nr_objs*sizeof(kmem_bufctl_t), align);
757}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700758
Andrew Mortona737b3e2006-03-22 00:08:11 -0800759/*
760 * Calculate the number of objects and left-over bytes for a given buffer size.
761 */
Steven Rostedtfbaccac2006-02-01 03:05:45 -0800762static void cache_estimate(unsigned long gfporder, size_t buffer_size,
763 size_t align, int flags, size_t *left_over,
764 unsigned int *num)
765{
766 int nr_objs;
767 size_t mgmt_size;
768 size_t slab_size = PAGE_SIZE << gfporder;
769
770 /*
771 * The slab management structure can be either off the slab or
772 * on it. For the latter case, the memory allocated for a
773 * slab is used for:
774 *
775 * - The struct slab
776 * - One kmem_bufctl_t for each object
777 * - Padding to respect alignment of @align
778 * - @buffer_size bytes for each object
779 *
780 * If the slab management structure is off the slab, then the
781 * alignment will already be calculated into the size. Because
782 * the slabs are all pages aligned, the objects will be at the
783 * correct alignment when allocated.
784 */
785 if (flags & CFLGS_OFF_SLAB) {
786 mgmt_size = 0;
787 nr_objs = slab_size / buffer_size;
788
789 if (nr_objs > SLAB_LIMIT)
790 nr_objs = SLAB_LIMIT;
791 } else {
792 /*
793 * Ignore padding for the initial guess. The padding
794 * is at most @align-1 bytes, and @buffer_size is at
795 * least @align. In the worst case, this result will
796 * be one greater than the number of objects that fit
797 * into the memory allocation when taking the padding
798 * into account.
799 */
800 nr_objs = (slab_size - sizeof(struct slab)) /
801 (buffer_size + sizeof(kmem_bufctl_t));
802
803 /*
804 * This calculated number will be either the right
805 * amount, or one greater than what we want.
806 */
807 if (slab_mgmt_size(nr_objs, align) + nr_objs*buffer_size
808 > slab_size)
809 nr_objs--;
810
811 if (nr_objs > SLAB_LIMIT)
812 nr_objs = SLAB_LIMIT;
813
814 mgmt_size = slab_mgmt_size(nr_objs, align);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700815 }
Steven Rostedtfbaccac2006-02-01 03:05:45 -0800816 *num = nr_objs;
817 *left_over = slab_size - nr_objs*buffer_size - mgmt_size;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700818}
819
820#define slab_error(cachep, msg) __slab_error(__FUNCTION__, cachep, msg)
821
Andrew Mortona737b3e2006-03-22 00:08:11 -0800822static void __slab_error(const char *function, struct kmem_cache *cachep,
823 char *msg)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700824{
825 printk(KERN_ERR "slab error in %s(): cache `%s': %s\n",
Pekka Enbergb28a02d2006-01-08 01:00:37 -0800826 function, cachep->name, msg);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700827 dump_stack();
828}
829
Christoph Lameter8fce4d82006-03-09 17:33:54 -0800830#ifdef CONFIG_NUMA
831/*
832 * Special reaping functions for NUMA systems called from cache_reap().
833 * These take care of doing round robin flushing of alien caches (containing
834 * objects freed on different nodes from which they were allocated) and the
835 * flushing of remote pcps by calling drain_node_pages.
836 */
837static DEFINE_PER_CPU(unsigned long, reap_node);
838
839static void init_reap_node(int cpu)
840{
841 int node;
842
843 node = next_node(cpu_to_node(cpu), node_online_map);
844 if (node == MAX_NUMNODES)
Paul Jackson442295c2006-03-22 00:09:11 -0800845 node = first_node(node_online_map);
Christoph Lameter8fce4d82006-03-09 17:33:54 -0800846
847 __get_cpu_var(reap_node) = node;
848}
849
850static void next_reap_node(void)
851{
852 int node = __get_cpu_var(reap_node);
853
854 /*
855 * Also drain per cpu pages on remote zones
856 */
857 if (node != numa_node_id())
858 drain_node_pages(node);
859
860 node = next_node(node, node_online_map);
861 if (unlikely(node >= MAX_NUMNODES))
862 node = first_node(node_online_map);
863 __get_cpu_var(reap_node) = node;
864}
865
866#else
867#define init_reap_node(cpu) do { } while (0)
868#define next_reap_node(void) do { } while (0)
869#endif
870
Linus Torvalds1da177e2005-04-16 15:20:36 -0700871/*
872 * Initiate the reap timer running on the target CPU. We run at around 1 to 2Hz
873 * via the workqueue/eventd.
874 * Add the CPU number into the expiration time to minimize the possibility of
875 * the CPUs getting into lockstep and contending for the global cache chain
876 * lock.
877 */
878static void __devinit start_cpu_timer(int cpu)
879{
880 struct work_struct *reap_work = &per_cpu(reap_work, cpu);
881
882 /*
883 * When this gets called from do_initcalls via cpucache_init(),
884 * init_workqueues() has already run, so keventd will be setup
885 * at that time.
886 */
887 if (keventd_up() && reap_work->func == NULL) {
Christoph Lameter8fce4d82006-03-09 17:33:54 -0800888 init_reap_node(cpu);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700889 INIT_WORK(reap_work, cache_reap, NULL);
890 schedule_delayed_work_on(cpu, reap_work, HZ + 3 * cpu);
891 }
892}
893
Christoph Lametere498be72005-09-09 13:03:32 -0700894static struct array_cache *alloc_arraycache(int node, int entries,
Pekka Enbergb28a02d2006-01-08 01:00:37 -0800895 int batchcount)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700896{
Pekka Enbergb28a02d2006-01-08 01:00:37 -0800897 int memsize = sizeof(void *) * entries + sizeof(struct array_cache);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700898 struct array_cache *nc = NULL;
899
Christoph Lametere498be72005-09-09 13:03:32 -0700900 nc = kmalloc_node(memsize, GFP_KERNEL, node);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700901 if (nc) {
902 nc->avail = 0;
903 nc->limit = entries;
904 nc->batchcount = batchcount;
905 nc->touched = 0;
Christoph Lametere498be72005-09-09 13:03:32 -0700906 spin_lock_init(&nc->lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700907 }
908 return nc;
909}
910
Christoph Lameter3ded1752006-03-25 03:06:44 -0800911/*
912 * Transfer objects in one arraycache to another.
913 * Locking must be handled by the caller.
914 *
915 * Return the number of entries transferred.
916 */
917static int transfer_objects(struct array_cache *to,
918 struct array_cache *from, unsigned int max)
919{
920 /* Figure out how many entries to transfer */
921 int nr = min(min(from->avail, max), to->limit - to->avail);
922
923 if (!nr)
924 return 0;
925
926 memcpy(to->entry + to->avail, from->entry + from->avail -nr,
927 sizeof(void *) *nr);
928
929 from->avail -= nr;
930 to->avail += nr;
931 to->touched = 1;
932 return nr;
933}
934
Christoph Lametere498be72005-09-09 13:03:32 -0700935#ifdef CONFIG_NUMA
Pekka Enberg343e0d72006-02-01 03:05:50 -0800936static void *__cache_alloc_node(struct kmem_cache *, gfp_t, int);
Paul Jacksonc61afb12006-03-24 03:16:08 -0800937static void *alternate_node_alloc(struct kmem_cache *, gfp_t);
Christoph Lameterdc85da12006-01-18 17:42:36 -0800938
Pekka Enberg5295a742006-02-01 03:05:48 -0800939static struct array_cache **alloc_alien_cache(int node, int limit)
Christoph Lametere498be72005-09-09 13:03:32 -0700940{
941 struct array_cache **ac_ptr;
Pekka Enbergb28a02d2006-01-08 01:00:37 -0800942 int memsize = sizeof(void *) * MAX_NUMNODES;
Christoph Lametere498be72005-09-09 13:03:32 -0700943 int i;
944
945 if (limit > 1)
946 limit = 12;
947 ac_ptr = kmalloc_node(memsize, GFP_KERNEL, node);
948 if (ac_ptr) {
949 for_each_node(i) {
950 if (i == node || !node_online(i)) {
951 ac_ptr[i] = NULL;
952 continue;
953 }
954 ac_ptr[i] = alloc_arraycache(node, limit, 0xbaadf00d);
955 if (!ac_ptr[i]) {
Pekka Enbergb28a02d2006-01-08 01:00:37 -0800956 for (i--; i <= 0; i--)
Christoph Lametere498be72005-09-09 13:03:32 -0700957 kfree(ac_ptr[i]);
958 kfree(ac_ptr);
959 return NULL;
960 }
961 }
962 }
963 return ac_ptr;
964}
965
Pekka Enberg5295a742006-02-01 03:05:48 -0800966static void free_alien_cache(struct array_cache **ac_ptr)
Christoph Lametere498be72005-09-09 13:03:32 -0700967{
968 int i;
969
970 if (!ac_ptr)
971 return;
Christoph Lametere498be72005-09-09 13:03:32 -0700972 for_each_node(i)
Pekka Enbergb28a02d2006-01-08 01:00:37 -0800973 kfree(ac_ptr[i]);
Christoph Lametere498be72005-09-09 13:03:32 -0700974 kfree(ac_ptr);
975}
976
Pekka Enberg343e0d72006-02-01 03:05:50 -0800977static void __drain_alien_cache(struct kmem_cache *cachep,
Pekka Enberg5295a742006-02-01 03:05:48 -0800978 struct array_cache *ac, int node)
Christoph Lametere498be72005-09-09 13:03:32 -0700979{
980 struct kmem_list3 *rl3 = cachep->nodelists[node];
981
982 if (ac->avail) {
983 spin_lock(&rl3->list_lock);
Christoph Lametere00946f2006-03-25 03:06:45 -0800984 /*
985 * Stuff objects into the remote nodes shared array first.
986 * That way we could avoid the overhead of putting the objects
987 * into the free lists and getting them back later.
988 */
shin, jacob693f7d32006-04-28 10:54:37 -0500989 if (rl3->shared)
990 transfer_objects(rl3->shared, ac, ac->limit);
Christoph Lametere00946f2006-03-25 03:06:45 -0800991
Christoph Lameterff694162005-09-22 21:44:02 -0700992 free_block(cachep, ac->entry, ac->avail, node);
Christoph Lametere498be72005-09-09 13:03:32 -0700993 ac->avail = 0;
994 spin_unlock(&rl3->list_lock);
995 }
996}
997
Christoph Lameter8fce4d82006-03-09 17:33:54 -0800998/*
999 * Called from cache_reap() to regularly drain alien caches round robin.
1000 */
1001static void reap_alien(struct kmem_cache *cachep, struct kmem_list3 *l3)
1002{
1003 int node = __get_cpu_var(reap_node);
1004
1005 if (l3->alien) {
1006 struct array_cache *ac = l3->alien[node];
Christoph Lametere00946f2006-03-25 03:06:45 -08001007
1008 if (ac && ac->avail && spin_trylock_irq(&ac->lock)) {
Christoph Lameter8fce4d82006-03-09 17:33:54 -08001009 __drain_alien_cache(cachep, ac, node);
1010 spin_unlock_irq(&ac->lock);
1011 }
1012 }
1013}
1014
Andrew Mortona737b3e2006-03-22 00:08:11 -08001015static void drain_alien_cache(struct kmem_cache *cachep,
1016 struct array_cache **alien)
Christoph Lametere498be72005-09-09 13:03:32 -07001017{
Pekka Enbergb28a02d2006-01-08 01:00:37 -08001018 int i = 0;
Christoph Lametere498be72005-09-09 13:03:32 -07001019 struct array_cache *ac;
1020 unsigned long flags;
1021
1022 for_each_online_node(i) {
Ravikiran G Thirumalai4484ebf2006-02-04 23:27:59 -08001023 ac = alien[i];
Christoph Lametere498be72005-09-09 13:03:32 -07001024 if (ac) {
1025 spin_lock_irqsave(&ac->lock, flags);
1026 __drain_alien_cache(cachep, ac, i);
1027 spin_unlock_irqrestore(&ac->lock, flags);
1028 }
1029 }
1030}
Pekka Enberg729bd0b2006-06-23 02:03:05 -07001031
1032static inline int cache_free_alien(struct kmem_cache *cachep, void *objp)
1033{
1034 struct slab *slabp = virt_to_slab(objp);
1035 int nodeid = slabp->nodeid;
1036 struct kmem_list3 *l3;
1037 struct array_cache *alien = NULL;
1038
1039 /*
1040 * Make sure we are not freeing a object from another node to the array
1041 * cache on this cpu.
1042 */
1043 if (likely(slabp->nodeid == numa_node_id()))
1044 return 0;
1045
1046 l3 = cachep->nodelists[numa_node_id()];
1047 STATS_INC_NODEFREES(cachep);
1048 if (l3->alien && l3->alien[nodeid]) {
1049 alien = l3->alien[nodeid];
1050 spin_lock(&alien->lock);
1051 if (unlikely(alien->avail == alien->limit)) {
1052 STATS_INC_ACOVERFLOW(cachep);
1053 __drain_alien_cache(cachep, alien, nodeid);
1054 }
1055 alien->entry[alien->avail++] = objp;
1056 spin_unlock(&alien->lock);
1057 } else {
1058 spin_lock(&(cachep->nodelists[nodeid])->list_lock);
1059 free_block(cachep, &objp, 1, nodeid);
1060 spin_unlock(&(cachep->nodelists[nodeid])->list_lock);
1061 }
1062 return 1;
1063}
1064
Christoph Lametere498be72005-09-09 13:03:32 -07001065#else
Linus Torvalds7a21ef62006-02-05 11:26:38 -08001066
Ravikiran G Thirumalai4484ebf2006-02-04 23:27:59 -08001067#define drain_alien_cache(cachep, alien) do { } while (0)
Christoph Lameter8fce4d82006-03-09 17:33:54 -08001068#define reap_alien(cachep, l3) do { } while (0)
Ravikiran G Thirumalai4484ebf2006-02-04 23:27:59 -08001069
Linus Torvalds7a21ef62006-02-05 11:26:38 -08001070static inline struct array_cache **alloc_alien_cache(int node, int limit)
1071{
1072 return (struct array_cache **) 0x01020304ul;
1073}
1074
Ravikiran G Thirumalai4484ebf2006-02-04 23:27:59 -08001075static inline void free_alien_cache(struct array_cache **ac_ptr)
1076{
1077}
Linus Torvalds7a21ef62006-02-05 11:26:38 -08001078
Pekka Enberg729bd0b2006-06-23 02:03:05 -07001079static inline int cache_free_alien(struct kmem_cache *cachep, void *objp)
1080{
1081 return 0;
1082}
1083
Christoph Lametere498be72005-09-09 13:03:32 -07001084#endif
1085
Chandra Seetharaman83d722f2006-04-24 19:35:21 -07001086static int cpuup_callback(struct notifier_block *nfb,
Pekka Enbergb28a02d2006-01-08 01:00:37 -08001087 unsigned long action, void *hcpu)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001088{
1089 long cpu = (long)hcpu;
Pekka Enberg343e0d72006-02-01 03:05:50 -08001090 struct kmem_cache *cachep;
Christoph Lametere498be72005-09-09 13:03:32 -07001091 struct kmem_list3 *l3 = NULL;
1092 int node = cpu_to_node(cpu);
1093 int memsize = sizeof(struct kmem_list3);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001094
1095 switch (action) {
1096 case CPU_UP_PREPARE:
Ingo Molnarfc0abb12006-01-18 17:42:33 -08001097 mutex_lock(&cache_chain_mutex);
Andrew Mortona737b3e2006-03-22 00:08:11 -08001098 /*
1099 * We need to do this right in the beginning since
Christoph Lametere498be72005-09-09 13:03:32 -07001100 * alloc_arraycache's are going to use this list.
1101 * kmalloc_node allows us to add the slab to the right
1102 * kmem_list3 and not this cpu's kmem_list3
1103 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001104
Christoph Lametere498be72005-09-09 13:03:32 -07001105 list_for_each_entry(cachep, &cache_chain, next) {
Andrew Mortona737b3e2006-03-22 00:08:11 -08001106 /*
1107 * Set up the size64 kmemlist for cpu before we can
Christoph Lametere498be72005-09-09 13:03:32 -07001108 * begin anything. Make sure some other cpu on this
1109 * node has not already allocated this
1110 */
1111 if (!cachep->nodelists[node]) {
Andrew Mortona737b3e2006-03-22 00:08:11 -08001112 l3 = kmalloc_node(memsize, GFP_KERNEL, node);
1113 if (!l3)
Christoph Lametere498be72005-09-09 13:03:32 -07001114 goto bad;
1115 kmem_list3_init(l3);
1116 l3->next_reap = jiffies + REAPTIMEOUT_LIST3 +
Pekka Enbergb28a02d2006-01-08 01:00:37 -08001117 ((unsigned long)cachep) % REAPTIMEOUT_LIST3;
Christoph Lametere498be72005-09-09 13:03:32 -07001118
Ravikiran G Thirumalai4484ebf2006-02-04 23:27:59 -08001119 /*
1120 * The l3s don't come and go as CPUs come and
1121 * go. cache_chain_mutex is sufficient
1122 * protection here.
1123 */
Christoph Lametere498be72005-09-09 13:03:32 -07001124 cachep->nodelists[node] = l3;
1125 }
1126
1127 spin_lock_irq(&cachep->nodelists[node]->list_lock);
1128 cachep->nodelists[node]->free_limit =
Andrew Mortona737b3e2006-03-22 00:08:11 -08001129 (1 + nr_cpus_node(node)) *
1130 cachep->batchcount + cachep->num;
Christoph Lametere498be72005-09-09 13:03:32 -07001131 spin_unlock_irq(&cachep->nodelists[node]->list_lock);
1132 }
1133
Andrew Mortona737b3e2006-03-22 00:08:11 -08001134 /*
1135 * Now we can go ahead with allocating the shared arrays and
1136 * array caches
1137 */
Christoph Lametere498be72005-09-09 13:03:32 -07001138 list_for_each_entry(cachep, &cache_chain, next) {
Tobias Klausercd105df2006-01-08 01:00:59 -08001139 struct array_cache *nc;
Ravikiran G Thirumalai4484ebf2006-02-04 23:27:59 -08001140 struct array_cache *shared;
1141 struct array_cache **alien;
Tobias Klausercd105df2006-01-08 01:00:59 -08001142
Christoph Lametere498be72005-09-09 13:03:32 -07001143 nc = alloc_arraycache(node, cachep->limit,
Ravikiran G Thirumalai4484ebf2006-02-04 23:27:59 -08001144 cachep->batchcount);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001145 if (!nc)
1146 goto bad;
Ravikiran G Thirumalai4484ebf2006-02-04 23:27:59 -08001147 shared = alloc_arraycache(node,
1148 cachep->shared * cachep->batchcount,
1149 0xbaadf00d);
1150 if (!shared)
1151 goto bad;
Linus Torvalds7a21ef62006-02-05 11:26:38 -08001152
Ravikiran G Thirumalai4484ebf2006-02-04 23:27:59 -08001153 alien = alloc_alien_cache(node, cachep->limit);
1154 if (!alien)
1155 goto bad;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001156 cachep->array[cpu] = nc;
Christoph Lametere498be72005-09-09 13:03:32 -07001157 l3 = cachep->nodelists[node];
1158 BUG_ON(!l3);
Christoph Lametere498be72005-09-09 13:03:32 -07001159
Ravikiran G Thirumalai4484ebf2006-02-04 23:27:59 -08001160 spin_lock_irq(&l3->list_lock);
1161 if (!l3->shared) {
1162 /*
1163 * We are serialised from CPU_DEAD or
1164 * CPU_UP_CANCELLED by the cpucontrol lock
1165 */
1166 l3->shared = shared;
1167 shared = NULL;
Christoph Lametere498be72005-09-09 13:03:32 -07001168 }
Ravikiran G Thirumalai4484ebf2006-02-04 23:27:59 -08001169#ifdef CONFIG_NUMA
1170 if (!l3->alien) {
1171 l3->alien = alien;
1172 alien = NULL;
1173 }
1174#endif
1175 spin_unlock_irq(&l3->list_lock);
Ravikiran G Thirumalai4484ebf2006-02-04 23:27:59 -08001176 kfree(shared);
1177 free_alien_cache(alien);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001178 }
Ingo Molnarfc0abb12006-01-18 17:42:33 -08001179 mutex_unlock(&cache_chain_mutex);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001180 break;
1181 case CPU_ONLINE:
1182 start_cpu_timer(cpu);
1183 break;
1184#ifdef CONFIG_HOTPLUG_CPU
1185 case CPU_DEAD:
Ravikiran G Thirumalai4484ebf2006-02-04 23:27:59 -08001186 /*
1187 * Even if all the cpus of a node are down, we don't free the
1188 * kmem_list3 of any cache. This to avoid a race between
1189 * cpu_down, and a kmalloc allocation from another cpu for
1190 * memory from the node of the cpu going down. The list3
1191 * structure is usually allocated from kmem_cache_create() and
1192 * gets destroyed at kmem_cache_destroy().
1193 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001194 /* fall thru */
1195 case CPU_UP_CANCELED:
Ingo Molnarfc0abb12006-01-18 17:42:33 -08001196 mutex_lock(&cache_chain_mutex);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001197 list_for_each_entry(cachep, &cache_chain, next) {
1198 struct array_cache *nc;
Ravikiran G Thirumalai4484ebf2006-02-04 23:27:59 -08001199 struct array_cache *shared;
1200 struct array_cache **alien;
Christoph Lametere498be72005-09-09 13:03:32 -07001201 cpumask_t mask;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001202
Christoph Lametere498be72005-09-09 13:03:32 -07001203 mask = node_to_cpumask(node);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001204 /* cpu is dead; no one can alloc from it. */
1205 nc = cachep->array[cpu];
1206 cachep->array[cpu] = NULL;
Christoph Lametere498be72005-09-09 13:03:32 -07001207 l3 = cachep->nodelists[node];
1208
1209 if (!l3)
Ravikiran G Thirumalai4484ebf2006-02-04 23:27:59 -08001210 goto free_array_cache;
Christoph Lametere498be72005-09-09 13:03:32 -07001211
Ravikiran G Thirumalaica3b9b92006-02-04 23:27:58 -08001212 spin_lock_irq(&l3->list_lock);
Christoph Lametere498be72005-09-09 13:03:32 -07001213
1214 /* Free limit for this kmem_list3 */
1215 l3->free_limit -= cachep->batchcount;
1216 if (nc)
Christoph Lameterff694162005-09-22 21:44:02 -07001217 free_block(cachep, nc->entry, nc->avail, node);
Christoph Lametere498be72005-09-09 13:03:32 -07001218
1219 if (!cpus_empty(mask)) {
Ravikiran G Thirumalaica3b9b92006-02-04 23:27:58 -08001220 spin_unlock_irq(&l3->list_lock);
Ravikiran G Thirumalai4484ebf2006-02-04 23:27:59 -08001221 goto free_array_cache;
Pekka Enbergb28a02d2006-01-08 01:00:37 -08001222 }
Christoph Lametere498be72005-09-09 13:03:32 -07001223
Ravikiran G Thirumalai4484ebf2006-02-04 23:27:59 -08001224 shared = l3->shared;
1225 if (shared) {
Christoph Lametere498be72005-09-09 13:03:32 -07001226 free_block(cachep, l3->shared->entry,
Pekka Enbergb28a02d2006-01-08 01:00:37 -08001227 l3->shared->avail, node);
Christoph Lametere498be72005-09-09 13:03:32 -07001228 l3->shared = NULL;
1229 }
Christoph Lametere498be72005-09-09 13:03:32 -07001230
Ravikiran G Thirumalai4484ebf2006-02-04 23:27:59 -08001231 alien = l3->alien;
1232 l3->alien = NULL;
1233
1234 spin_unlock_irq(&l3->list_lock);
1235
1236 kfree(shared);
1237 if (alien) {
1238 drain_alien_cache(cachep, alien);
1239 free_alien_cache(alien);
Christoph Lametere498be72005-09-09 13:03:32 -07001240 }
Ravikiran G Thirumalai4484ebf2006-02-04 23:27:59 -08001241free_array_cache:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001242 kfree(nc);
1243 }
Ravikiran G Thirumalai4484ebf2006-02-04 23:27:59 -08001244 /*
1245 * In the previous loop, all the objects were freed to
1246 * the respective cache's slabs, now we can go ahead and
1247 * shrink each nodelist to its limit.
1248 */
1249 list_for_each_entry(cachep, &cache_chain, next) {
1250 l3 = cachep->nodelists[node];
1251 if (!l3)
1252 continue;
1253 spin_lock_irq(&l3->list_lock);
1254 /* free slabs belonging to this node */
1255 __node_shrink(cachep, node);
1256 spin_unlock_irq(&l3->list_lock);
1257 }
Ingo Molnarfc0abb12006-01-18 17:42:33 -08001258 mutex_unlock(&cache_chain_mutex);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001259 break;
1260#endif
1261 }
1262 return NOTIFY_OK;
Andrew Mortona737b3e2006-03-22 00:08:11 -08001263bad:
Ingo Molnarfc0abb12006-01-18 17:42:33 -08001264 mutex_unlock(&cache_chain_mutex);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001265 return NOTIFY_BAD;
1266}
1267
1268static struct notifier_block cpucache_notifier = { &cpuup_callback, NULL, 0 };
1269
Christoph Lametere498be72005-09-09 13:03:32 -07001270/*
1271 * swap the static kmem_list3 with kmalloced memory
1272 */
Andrew Mortona737b3e2006-03-22 00:08:11 -08001273static void init_list(struct kmem_cache *cachep, struct kmem_list3 *list,
1274 int nodeid)
Christoph Lametere498be72005-09-09 13:03:32 -07001275{
1276 struct kmem_list3 *ptr;
1277
1278 BUG_ON(cachep->nodelists[nodeid] != list);
1279 ptr = kmalloc_node(sizeof(struct kmem_list3), GFP_KERNEL, nodeid);
1280 BUG_ON(!ptr);
1281
1282 local_irq_disable();
1283 memcpy(ptr, list, sizeof(struct kmem_list3));
1284 MAKE_ALL_LISTS(cachep, ptr, nodeid);
1285 cachep->nodelists[nodeid] = ptr;
1286 local_irq_enable();
1287}
1288
Andrew Mortona737b3e2006-03-22 00:08:11 -08001289/*
1290 * Initialisation. Called after the page allocator have been initialised and
1291 * before smp_init().
Linus Torvalds1da177e2005-04-16 15:20:36 -07001292 */
1293void __init kmem_cache_init(void)
1294{
1295 size_t left_over;
1296 struct cache_sizes *sizes;
1297 struct cache_names *names;
Christoph Lametere498be72005-09-09 13:03:32 -07001298 int i;
Jack Steiner07ed76b2006-03-07 21:55:46 -08001299 int order;
Christoph Lametere498be72005-09-09 13:03:32 -07001300
1301 for (i = 0; i < NUM_INIT_LISTS; i++) {
1302 kmem_list3_init(&initkmem_list3[i]);
1303 if (i < MAX_NUMNODES)
1304 cache_cache.nodelists[i] = NULL;
1305 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001306
1307 /*
1308 * Fragmentation resistance on low memory - only use bigger
1309 * page orders on machines with more than 32MB of memory.
1310 */
1311 if (num_physpages > (32 << 20) >> PAGE_SHIFT)
1312 slab_break_gfp_order = BREAK_GFP_ORDER_HI;
1313
Linus Torvalds1da177e2005-04-16 15:20:36 -07001314 /* Bootstrap is tricky, because several objects are allocated
1315 * from caches that do not exist yet:
Andrew Mortona737b3e2006-03-22 00:08:11 -08001316 * 1) initialize the cache_cache cache: it contains the struct
1317 * kmem_cache structures of all caches, except cache_cache itself:
1318 * cache_cache is statically allocated.
Christoph Lametere498be72005-09-09 13:03:32 -07001319 * Initially an __init data area is used for the head array and the
1320 * kmem_list3 structures, it's replaced with a kmalloc allocated
1321 * array at the end of the bootstrap.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001322 * 2) Create the first kmalloc cache.
Pekka Enberg343e0d72006-02-01 03:05:50 -08001323 * The struct kmem_cache for the new cache is allocated normally.
Christoph Lametere498be72005-09-09 13:03:32 -07001324 * An __init data area is used for the head array.
1325 * 3) Create the remaining kmalloc caches, with minimally sized
1326 * head arrays.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001327 * 4) Replace the __init data head arrays for cache_cache and the first
1328 * kmalloc cache with kmalloc allocated arrays.
Christoph Lametere498be72005-09-09 13:03:32 -07001329 * 5) Replace the __init data for kmem_list3 for cache_cache and
1330 * the other cache's with kmalloc allocated memory.
1331 * 6) Resize the head arrays of the kmalloc caches to their final sizes.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001332 */
1333
1334 /* 1) create the cache_cache */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001335 INIT_LIST_HEAD(&cache_chain);
1336 list_add(&cache_cache.next, &cache_chain);
1337 cache_cache.colour_off = cache_line_size();
1338 cache_cache.array[smp_processor_id()] = &initarray_cache.cache;
Christoph Lametere498be72005-09-09 13:03:32 -07001339 cache_cache.nodelists[numa_node_id()] = &initkmem_list3[CACHE_CACHE];
Linus Torvalds1da177e2005-04-16 15:20:36 -07001340
Andrew Mortona737b3e2006-03-22 00:08:11 -08001341 cache_cache.buffer_size = ALIGN(cache_cache.buffer_size,
1342 cache_line_size());
Linus Torvalds1da177e2005-04-16 15:20:36 -07001343
Jack Steiner07ed76b2006-03-07 21:55:46 -08001344 for (order = 0; order < MAX_ORDER; order++) {
1345 cache_estimate(order, cache_cache.buffer_size,
1346 cache_line_size(), 0, &left_over, &cache_cache.num);
1347 if (cache_cache.num)
1348 break;
1349 }
Eric Sesterhenn40094fa2006-04-02 13:49:25 +02001350 BUG_ON(!cache_cache.num);
Jack Steiner07ed76b2006-03-07 21:55:46 -08001351 cache_cache.gfporder = order;
Pekka Enbergb28a02d2006-01-08 01:00:37 -08001352 cache_cache.colour = left_over / cache_cache.colour_off;
Pekka Enbergb28a02d2006-01-08 01:00:37 -08001353 cache_cache.slab_size = ALIGN(cache_cache.num * sizeof(kmem_bufctl_t) +
1354 sizeof(struct slab), cache_line_size());
Linus Torvalds1da177e2005-04-16 15:20:36 -07001355
1356 /* 2+3) create the kmalloc caches */
1357 sizes = malloc_sizes;
1358 names = cache_names;
1359
Andrew Mortona737b3e2006-03-22 00:08:11 -08001360 /*
1361 * Initialize the caches that provide memory for the array cache and the
1362 * kmem_list3 structures first. Without this, further allocations will
1363 * bug.
Christoph Lametere498be72005-09-09 13:03:32 -07001364 */
1365
1366 sizes[INDEX_AC].cs_cachep = kmem_cache_create(names[INDEX_AC].name,
Andrew Mortona737b3e2006-03-22 00:08:11 -08001367 sizes[INDEX_AC].cs_size,
1368 ARCH_KMALLOC_MINALIGN,
1369 ARCH_KMALLOC_FLAGS|SLAB_PANIC,
1370 NULL, NULL);
Christoph Lametere498be72005-09-09 13:03:32 -07001371
Andrew Mortona737b3e2006-03-22 00:08:11 -08001372 if (INDEX_AC != INDEX_L3) {
Christoph Lametere498be72005-09-09 13:03:32 -07001373 sizes[INDEX_L3].cs_cachep =
Andrew Mortona737b3e2006-03-22 00:08:11 -08001374 kmem_cache_create(names[INDEX_L3].name,
1375 sizes[INDEX_L3].cs_size,
1376 ARCH_KMALLOC_MINALIGN,
1377 ARCH_KMALLOC_FLAGS|SLAB_PANIC,
1378 NULL, NULL);
1379 }
Christoph Lametere498be72005-09-09 13:03:32 -07001380
Ingo Molnare0a42722006-06-23 02:03:46 -07001381 slab_early_init = 0;
1382
Linus Torvalds1da177e2005-04-16 15:20:36 -07001383 while (sizes->cs_size != ULONG_MAX) {
Christoph Lametere498be72005-09-09 13:03:32 -07001384 /*
1385 * For performance, all the general caches are L1 aligned.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001386 * This should be particularly beneficial on SMP boxes, as it
1387 * eliminates "false sharing".
1388 * Note for systems short on memory removing the alignment will
Christoph Lametere498be72005-09-09 13:03:32 -07001389 * allow tighter packing of the smaller caches.
1390 */
Andrew Mortona737b3e2006-03-22 00:08:11 -08001391 if (!sizes->cs_cachep) {
Christoph Lametere498be72005-09-09 13:03:32 -07001392 sizes->cs_cachep = kmem_cache_create(names->name,
Andrew Mortona737b3e2006-03-22 00:08:11 -08001393 sizes->cs_size,
1394 ARCH_KMALLOC_MINALIGN,
1395 ARCH_KMALLOC_FLAGS|SLAB_PANIC,
1396 NULL, NULL);
1397 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001398
Linus Torvalds1da177e2005-04-16 15:20:36 -07001399 sizes->cs_dmacachep = kmem_cache_create(names->name_dma,
Andrew Mortona737b3e2006-03-22 00:08:11 -08001400 sizes->cs_size,
1401 ARCH_KMALLOC_MINALIGN,
1402 ARCH_KMALLOC_FLAGS|SLAB_CACHE_DMA|
1403 SLAB_PANIC,
1404 NULL, NULL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001405 sizes++;
1406 names++;
1407 }
1408 /* 4) Replace the bootstrap head arrays */
1409 {
Pekka Enbergb28a02d2006-01-08 01:00:37 -08001410 void *ptr;
Christoph Lametere498be72005-09-09 13:03:32 -07001411
Linus Torvalds1da177e2005-04-16 15:20:36 -07001412 ptr = kmalloc(sizeof(struct arraycache_init), GFP_KERNEL);
Christoph Lametere498be72005-09-09 13:03:32 -07001413
Linus Torvalds1da177e2005-04-16 15:20:36 -07001414 local_irq_disable();
Pekka Enberg9a2dba42006-02-01 03:05:49 -08001415 BUG_ON(cpu_cache_get(&cache_cache) != &initarray_cache.cache);
1416 memcpy(ptr, cpu_cache_get(&cache_cache),
Pekka Enbergb28a02d2006-01-08 01:00:37 -08001417 sizeof(struct arraycache_init));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001418 cache_cache.array[smp_processor_id()] = ptr;
1419 local_irq_enable();
Christoph Lametere498be72005-09-09 13:03:32 -07001420
Linus Torvalds1da177e2005-04-16 15:20:36 -07001421 ptr = kmalloc(sizeof(struct arraycache_init), GFP_KERNEL);
Christoph Lametere498be72005-09-09 13:03:32 -07001422
Linus Torvalds1da177e2005-04-16 15:20:36 -07001423 local_irq_disable();
Pekka Enberg9a2dba42006-02-01 03:05:49 -08001424 BUG_ON(cpu_cache_get(malloc_sizes[INDEX_AC].cs_cachep)
Pekka Enbergb28a02d2006-01-08 01:00:37 -08001425 != &initarray_generic.cache);
Pekka Enberg9a2dba42006-02-01 03:05:49 -08001426 memcpy(ptr, cpu_cache_get(malloc_sizes[INDEX_AC].cs_cachep),
Pekka Enbergb28a02d2006-01-08 01:00:37 -08001427 sizeof(struct arraycache_init));
Christoph Lametere498be72005-09-09 13:03:32 -07001428 malloc_sizes[INDEX_AC].cs_cachep->array[smp_processor_id()] =
Pekka Enbergb28a02d2006-01-08 01:00:37 -08001429 ptr;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001430 local_irq_enable();
1431 }
Christoph Lametere498be72005-09-09 13:03:32 -07001432 /* 5) Replace the bootstrap kmem_list3's */
1433 {
1434 int node;
1435 /* Replace the static kmem_list3 structures for the boot cpu */
1436 init_list(&cache_cache, &initkmem_list3[CACHE_CACHE],
Pekka Enbergb28a02d2006-01-08 01:00:37 -08001437 numa_node_id());
Linus Torvalds1da177e2005-04-16 15:20:36 -07001438
Christoph Lametere498be72005-09-09 13:03:32 -07001439 for_each_online_node(node) {
1440 init_list(malloc_sizes[INDEX_AC].cs_cachep,
Pekka Enbergb28a02d2006-01-08 01:00:37 -08001441 &initkmem_list3[SIZE_AC + node], node);
Christoph Lametere498be72005-09-09 13:03:32 -07001442
1443 if (INDEX_AC != INDEX_L3) {
1444 init_list(malloc_sizes[INDEX_L3].cs_cachep,
Pekka Enbergb28a02d2006-01-08 01:00:37 -08001445 &initkmem_list3[SIZE_L3 + node],
1446 node);
Christoph Lametere498be72005-09-09 13:03:32 -07001447 }
1448 }
1449 }
1450
1451 /* 6) resize the head arrays to their final sizes */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001452 {
Pekka Enberg343e0d72006-02-01 03:05:50 -08001453 struct kmem_cache *cachep;
Ingo Molnarfc0abb12006-01-18 17:42:33 -08001454 mutex_lock(&cache_chain_mutex);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001455 list_for_each_entry(cachep, &cache_chain, next)
Andrew Mortona737b3e2006-03-22 00:08:11 -08001456 enable_cpucache(cachep);
Ingo Molnarfc0abb12006-01-18 17:42:33 -08001457 mutex_unlock(&cache_chain_mutex);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001458 }
1459
1460 /* Done! */
1461 g_cpucache_up = FULL;
1462
Andrew Mortona737b3e2006-03-22 00:08:11 -08001463 /*
1464 * Register a cpu startup notifier callback that initializes
1465 * cpu_cache_get for all new cpus
Linus Torvalds1da177e2005-04-16 15:20:36 -07001466 */
1467 register_cpu_notifier(&cpucache_notifier);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001468
Andrew Mortona737b3e2006-03-22 00:08:11 -08001469 /*
1470 * The reap timers are started later, with a module init call: That part
1471 * of the kernel is not yet operational.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001472 */
1473}
1474
1475static int __init cpucache_init(void)
1476{
1477 int cpu;
1478
Andrew Mortona737b3e2006-03-22 00:08:11 -08001479 /*
1480 * Register the timers that return unneeded pages to the page allocator
Linus Torvalds1da177e2005-04-16 15:20:36 -07001481 */
Christoph Lametere498be72005-09-09 13:03:32 -07001482 for_each_online_cpu(cpu)
Andrew Mortona737b3e2006-03-22 00:08:11 -08001483 start_cpu_timer(cpu);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001484 return 0;
1485}
Linus Torvalds1da177e2005-04-16 15:20:36 -07001486__initcall(cpucache_init);
1487
1488/*
1489 * Interface to system's page allocator. No need to hold the cache-lock.
1490 *
1491 * If we requested dmaable memory, we will get it. Even if we
1492 * did not request dmaable memory, we might get it, but that
1493 * would be relatively rare and ignorable.
1494 */
Pekka Enberg343e0d72006-02-01 03:05:50 -08001495static void *kmem_getpages(struct kmem_cache *cachep, gfp_t flags, int nodeid)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001496{
1497 struct page *page;
Christoph Hellwige1b6aa62006-06-23 02:03:17 -07001498 int nr_pages;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001499 int i;
1500
Luke Yangd6fef9d2006-04-10 22:52:56 -07001501#ifndef CONFIG_MMU
Christoph Hellwige1b6aa62006-06-23 02:03:17 -07001502 /*
1503 * Nommu uses slab's for process anonymous memory allocations, and thus
1504 * requires __GFP_COMP to properly refcount higher order allocations
Luke Yangd6fef9d2006-04-10 22:52:56 -07001505 */
Christoph Hellwige1b6aa62006-06-23 02:03:17 -07001506 flags |= __GFP_COMP;
Luke Yangd6fef9d2006-04-10 22:52:56 -07001507#endif
Christoph Hellwige1b6aa62006-06-23 02:03:17 -07001508 flags |= cachep->gfpflags;
1509
1510 page = alloc_pages_node(nodeid, flags, cachep->gfporder);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001511 if (!page)
1512 return NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001513
Christoph Hellwige1b6aa62006-06-23 02:03:17 -07001514 nr_pages = (1 << cachep->gfporder);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001515 if (cachep->flags & SLAB_RECLAIM_ACCOUNT)
Christoph Hellwige1b6aa62006-06-23 02:03:17 -07001516 atomic_add(nr_pages, &slab_reclaim_pages);
1517 add_page_state(nr_slab, nr_pages);
1518 for (i = 0; i < nr_pages; i++)
1519 __SetPageSlab(page + i);
1520 return page_address(page);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001521}
1522
1523/*
1524 * Interface to system's page release.
1525 */
Pekka Enberg343e0d72006-02-01 03:05:50 -08001526static void kmem_freepages(struct kmem_cache *cachep, void *addr)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001527{
Pekka Enbergb28a02d2006-01-08 01:00:37 -08001528 unsigned long i = (1 << cachep->gfporder);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001529 struct page *page = virt_to_page(addr);
1530 const unsigned long nr_freed = i;
1531
1532 while (i--) {
Nick Pigginf205b2f2006-03-22 00:08:02 -08001533 BUG_ON(!PageSlab(page));
1534 __ClearPageSlab(page);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001535 page++;
1536 }
1537 sub_page_state(nr_slab, nr_freed);
1538 if (current->reclaim_state)
1539 current->reclaim_state->reclaimed_slab += nr_freed;
1540 free_pages((unsigned long)addr, cachep->gfporder);
Pekka Enbergb28a02d2006-01-08 01:00:37 -08001541 if (cachep->flags & SLAB_RECLAIM_ACCOUNT)
1542 atomic_sub(1 << cachep->gfporder, &slab_reclaim_pages);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001543}
1544
1545static void kmem_rcu_free(struct rcu_head *head)
1546{
Pekka Enbergb28a02d2006-01-08 01:00:37 -08001547 struct slab_rcu *slab_rcu = (struct slab_rcu *)head;
Pekka Enberg343e0d72006-02-01 03:05:50 -08001548 struct kmem_cache *cachep = slab_rcu->cachep;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001549
1550 kmem_freepages(cachep, slab_rcu->addr);
1551 if (OFF_SLAB(cachep))
1552 kmem_cache_free(cachep->slabp_cache, slab_rcu);
1553}
1554
1555#if DEBUG
1556
1557#ifdef CONFIG_DEBUG_PAGEALLOC
Pekka Enberg343e0d72006-02-01 03:05:50 -08001558static void store_stackinfo(struct kmem_cache *cachep, unsigned long *addr,
Pekka Enbergb28a02d2006-01-08 01:00:37 -08001559 unsigned long caller)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001560{
Manfred Spraul3dafccf2006-02-01 03:05:42 -08001561 int size = obj_size(cachep);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001562
Manfred Spraul3dafccf2006-02-01 03:05:42 -08001563 addr = (unsigned long *)&((char *)addr)[obj_offset(cachep)];
Linus Torvalds1da177e2005-04-16 15:20:36 -07001564
Pekka Enbergb28a02d2006-01-08 01:00:37 -08001565 if (size < 5 * sizeof(unsigned long))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001566 return;
1567
Pekka Enbergb28a02d2006-01-08 01:00:37 -08001568 *addr++ = 0x12345678;
1569 *addr++ = caller;
1570 *addr++ = smp_processor_id();
1571 size -= 3 * sizeof(unsigned long);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001572 {
1573 unsigned long *sptr = &caller;
1574 unsigned long svalue;
1575
1576 while (!kstack_end(sptr)) {
1577 svalue = *sptr++;
1578 if (kernel_text_address(svalue)) {
Pekka Enbergb28a02d2006-01-08 01:00:37 -08001579 *addr++ = svalue;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001580 size -= sizeof(unsigned long);
1581 if (size <= sizeof(unsigned long))
1582 break;
1583 }
1584 }
1585
1586 }
Pekka Enbergb28a02d2006-01-08 01:00:37 -08001587 *addr++ = 0x87654321;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001588}
1589#endif
1590
Pekka Enberg343e0d72006-02-01 03:05:50 -08001591static void poison_obj(struct kmem_cache *cachep, void *addr, unsigned char val)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001592{
Manfred Spraul3dafccf2006-02-01 03:05:42 -08001593 int size = obj_size(cachep);
1594 addr = &((char *)addr)[obj_offset(cachep)];
Linus Torvalds1da177e2005-04-16 15:20:36 -07001595
1596 memset(addr, val, size);
Pekka Enbergb28a02d2006-01-08 01:00:37 -08001597 *(unsigned char *)(addr + size - 1) = POISON_END;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001598}
1599
1600static void dump_line(char *data, int offset, int limit)
1601{
1602 int i;
1603 printk(KERN_ERR "%03x:", offset);
Andrew Mortona737b3e2006-03-22 00:08:11 -08001604 for (i = 0; i < limit; i++)
Pekka Enbergb28a02d2006-01-08 01:00:37 -08001605 printk(" %02x", (unsigned char)data[offset + i]);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001606 printk("\n");
1607}
1608#endif
1609
1610#if DEBUG
1611
Pekka Enberg343e0d72006-02-01 03:05:50 -08001612static void print_objinfo(struct kmem_cache *cachep, void *objp, int lines)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001613{
1614 int i, size;
1615 char *realobj;
1616
1617 if (cachep->flags & SLAB_RED_ZONE) {
1618 printk(KERN_ERR "Redzone: 0x%lx/0x%lx.\n",
Andrew Mortona737b3e2006-03-22 00:08:11 -08001619 *dbg_redzone1(cachep, objp),
1620 *dbg_redzone2(cachep, objp));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001621 }
1622
1623 if (cachep->flags & SLAB_STORE_USER) {
1624 printk(KERN_ERR "Last user: [<%p>]",
Andrew Mortona737b3e2006-03-22 00:08:11 -08001625 *dbg_userword(cachep, objp));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001626 print_symbol("(%s)",
Andrew Mortona737b3e2006-03-22 00:08:11 -08001627 (unsigned long)*dbg_userword(cachep, objp));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001628 printk("\n");
1629 }
Manfred Spraul3dafccf2006-02-01 03:05:42 -08001630 realobj = (char *)objp + obj_offset(cachep);
1631 size = obj_size(cachep);
Pekka Enbergb28a02d2006-01-08 01:00:37 -08001632 for (i = 0; i < size && lines; i += 16, lines--) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001633 int limit;
1634 limit = 16;
Pekka Enbergb28a02d2006-01-08 01:00:37 -08001635 if (i + limit > size)
1636 limit = size - i;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001637 dump_line(realobj, i, limit);
1638 }
1639}
1640
Pekka Enberg343e0d72006-02-01 03:05:50 -08001641static void check_poison_obj(struct kmem_cache *cachep, void *objp)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001642{
1643 char *realobj;
1644 int size, i;
1645 int lines = 0;
1646
Manfred Spraul3dafccf2006-02-01 03:05:42 -08001647 realobj = (char *)objp + obj_offset(cachep);
1648 size = obj_size(cachep);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001649
Pekka Enbergb28a02d2006-01-08 01:00:37 -08001650 for (i = 0; i < size; i++) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001651 char exp = POISON_FREE;
Pekka Enbergb28a02d2006-01-08 01:00:37 -08001652 if (i == size - 1)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001653 exp = POISON_END;
1654 if (realobj[i] != exp) {
1655 int limit;
1656 /* Mismatch ! */
1657 /* Print header */
1658 if (lines == 0) {
Pekka Enbergb28a02d2006-01-08 01:00:37 -08001659 printk(KERN_ERR
Andrew Mortona737b3e2006-03-22 00:08:11 -08001660 "Slab corruption: start=%p, len=%d\n",
1661 realobj, size);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001662 print_objinfo(cachep, objp, 0);
1663 }
1664 /* Hexdump the affected line */
Pekka Enbergb28a02d2006-01-08 01:00:37 -08001665 i = (i / 16) * 16;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001666 limit = 16;
Pekka Enbergb28a02d2006-01-08 01:00:37 -08001667 if (i + limit > size)
1668 limit = size - i;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001669 dump_line(realobj, i, limit);
1670 i += 16;
1671 lines++;
1672 /* Limit to 5 lines */
1673 if (lines > 5)
1674 break;
1675 }
1676 }
1677 if (lines != 0) {
1678 /* Print some data about the neighboring objects, if they
1679 * exist:
1680 */
Pekka Enberg6ed5eb22006-02-01 03:05:49 -08001681 struct slab *slabp = virt_to_slab(objp);
Pekka Enberg8fea4e92006-03-22 00:08:10 -08001682 unsigned int objnr;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001683
Pekka Enberg8fea4e92006-03-22 00:08:10 -08001684 objnr = obj_to_index(cachep, slabp, objp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001685 if (objnr) {
Pekka Enberg8fea4e92006-03-22 00:08:10 -08001686 objp = index_to_obj(cachep, slabp, objnr - 1);
Manfred Spraul3dafccf2006-02-01 03:05:42 -08001687 realobj = (char *)objp + obj_offset(cachep);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001688 printk(KERN_ERR "Prev obj: start=%p, len=%d\n",
Pekka Enbergb28a02d2006-01-08 01:00:37 -08001689 realobj, size);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001690 print_objinfo(cachep, objp, 2);
1691 }
Pekka Enbergb28a02d2006-01-08 01:00:37 -08001692 if (objnr + 1 < cachep->num) {
Pekka Enberg8fea4e92006-03-22 00:08:10 -08001693 objp = index_to_obj(cachep, slabp, objnr + 1);
Manfred Spraul3dafccf2006-02-01 03:05:42 -08001694 realobj = (char *)objp + obj_offset(cachep);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001695 printk(KERN_ERR "Next obj: start=%p, len=%d\n",
Pekka Enbergb28a02d2006-01-08 01:00:37 -08001696 realobj, size);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001697 print_objinfo(cachep, objp, 2);
1698 }
1699 }
1700}
1701#endif
1702
Linus Torvalds1da177e2005-04-16 15:20:36 -07001703#if DEBUG
Matthew Dobson12dd36f2006-02-01 03:05:46 -08001704/**
Randy Dunlap911851e2006-03-22 00:08:14 -08001705 * slab_destroy_objs - destroy a slab and its objects
1706 * @cachep: cache pointer being destroyed
1707 * @slabp: slab pointer being destroyed
1708 *
1709 * Call the registered destructor for each object in a slab that is being
1710 * destroyed.
Matthew Dobson12dd36f2006-02-01 03:05:46 -08001711 */
Pekka Enberg343e0d72006-02-01 03:05:50 -08001712static void slab_destroy_objs(struct kmem_cache *cachep, struct slab *slabp)
Matthew Dobson12dd36f2006-02-01 03:05:46 -08001713{
Linus Torvalds1da177e2005-04-16 15:20:36 -07001714 int i;
1715 for (i = 0; i < cachep->num; i++) {
Pekka Enberg8fea4e92006-03-22 00:08:10 -08001716 void *objp = index_to_obj(cachep, slabp, i);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001717
1718 if (cachep->flags & SLAB_POISON) {
1719#ifdef CONFIG_DEBUG_PAGEALLOC
Andrew Mortona737b3e2006-03-22 00:08:11 -08001720 if (cachep->buffer_size % PAGE_SIZE == 0 &&
1721 OFF_SLAB(cachep))
Pekka Enbergb28a02d2006-01-08 01:00:37 -08001722 kernel_map_pages(virt_to_page(objp),
Andrew Mortona737b3e2006-03-22 00:08:11 -08001723 cachep->buffer_size / PAGE_SIZE, 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001724 else
1725 check_poison_obj(cachep, objp);
1726#else
1727 check_poison_obj(cachep, objp);
1728#endif
1729 }
1730 if (cachep->flags & SLAB_RED_ZONE) {
1731 if (*dbg_redzone1(cachep, objp) != RED_INACTIVE)
1732 slab_error(cachep, "start of a freed object "
Pekka Enbergb28a02d2006-01-08 01:00:37 -08001733 "was overwritten");
Linus Torvalds1da177e2005-04-16 15:20:36 -07001734 if (*dbg_redzone2(cachep, objp) != RED_INACTIVE)
1735 slab_error(cachep, "end of a freed object "
Pekka Enbergb28a02d2006-01-08 01:00:37 -08001736 "was overwritten");
Linus Torvalds1da177e2005-04-16 15:20:36 -07001737 }
1738 if (cachep->dtor && !(cachep->flags & SLAB_POISON))
Manfred Spraul3dafccf2006-02-01 03:05:42 -08001739 (cachep->dtor) (objp + obj_offset(cachep), cachep, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001740 }
Matthew Dobson12dd36f2006-02-01 03:05:46 -08001741}
Linus Torvalds1da177e2005-04-16 15:20:36 -07001742#else
Pekka Enberg343e0d72006-02-01 03:05:50 -08001743static void slab_destroy_objs(struct kmem_cache *cachep, struct slab *slabp)
Matthew Dobson12dd36f2006-02-01 03:05:46 -08001744{
Linus Torvalds1da177e2005-04-16 15:20:36 -07001745 if (cachep->dtor) {
1746 int i;
1747 for (i = 0; i < cachep->num; i++) {
Pekka Enberg8fea4e92006-03-22 00:08:10 -08001748 void *objp = index_to_obj(cachep, slabp, i);
Pekka Enbergb28a02d2006-01-08 01:00:37 -08001749 (cachep->dtor) (objp, cachep, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001750 }
1751 }
Matthew Dobson12dd36f2006-02-01 03:05:46 -08001752}
Linus Torvalds1da177e2005-04-16 15:20:36 -07001753#endif
1754
Randy Dunlap911851e2006-03-22 00:08:14 -08001755/**
1756 * slab_destroy - destroy and release all objects in a slab
1757 * @cachep: cache pointer being destroyed
1758 * @slabp: slab pointer being destroyed
1759 *
Matthew Dobson12dd36f2006-02-01 03:05:46 -08001760 * Destroy all the objs in a slab, and release the mem back to the system.
Andrew Mortona737b3e2006-03-22 00:08:11 -08001761 * Before calling the slab must have been unlinked from the cache. The
1762 * cache-lock is not held/needed.
Matthew Dobson12dd36f2006-02-01 03:05:46 -08001763 */
Pekka Enberg343e0d72006-02-01 03:05:50 -08001764static void slab_destroy(struct kmem_cache *cachep, struct slab *slabp)
Matthew Dobson12dd36f2006-02-01 03:05:46 -08001765{
1766 void *addr = slabp->s_mem - slabp->colouroff;
1767
1768 slab_destroy_objs(cachep, slabp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001769 if (unlikely(cachep->flags & SLAB_DESTROY_BY_RCU)) {
1770 struct slab_rcu *slab_rcu;
1771
Pekka Enbergb28a02d2006-01-08 01:00:37 -08001772 slab_rcu = (struct slab_rcu *)slabp;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001773 slab_rcu->cachep = cachep;
1774 slab_rcu->addr = addr;
1775 call_rcu(&slab_rcu->head, kmem_rcu_free);
1776 } else {
1777 kmem_freepages(cachep, addr);
1778 if (OFF_SLAB(cachep))
1779 kmem_cache_free(cachep->slabp_cache, slabp);
1780 }
1781}
1782
Andrew Mortona737b3e2006-03-22 00:08:11 -08001783/*
1784 * For setting up all the kmem_list3s for cache whose buffer_size is same as
1785 * size of kmem_list3.
1786 */
Pekka Enberg343e0d72006-02-01 03:05:50 -08001787static void set_up_list3s(struct kmem_cache *cachep, int index)
Christoph Lametere498be72005-09-09 13:03:32 -07001788{
1789 int node;
1790
1791 for_each_online_node(node) {
Pekka Enbergb28a02d2006-01-08 01:00:37 -08001792 cachep->nodelists[node] = &initkmem_list3[index + node];
Christoph Lametere498be72005-09-09 13:03:32 -07001793 cachep->nodelists[node]->next_reap = jiffies +
Pekka Enbergb28a02d2006-01-08 01:00:37 -08001794 REAPTIMEOUT_LIST3 +
1795 ((unsigned long)cachep) % REAPTIMEOUT_LIST3;
Christoph Lametere498be72005-09-09 13:03:32 -07001796 }
1797}
1798
Linus Torvalds1da177e2005-04-16 15:20:36 -07001799/**
Randy.Dunlapa70773d2006-02-01 03:05:52 -08001800 * calculate_slab_order - calculate size (page order) of slabs
1801 * @cachep: pointer to the cache that is being created
1802 * @size: size of objects to be created in this cache.
1803 * @align: required alignment for the objects.
1804 * @flags: slab allocation flags
1805 *
1806 * Also calculates the number of objects per slab.
Pekka Enberg4d268eb2006-01-08 01:00:36 -08001807 *
1808 * This could be made much more intelligent. For now, try to avoid using
1809 * high order pages for slabs. When the gfp() functions are more friendly
1810 * towards high-order requests, this should be changed.
1811 */
Andrew Mortona737b3e2006-03-22 00:08:11 -08001812static size_t calculate_slab_order(struct kmem_cache *cachep,
Randy Dunlapee13d782006-02-01 03:05:53 -08001813 size_t size, size_t align, unsigned long flags)
Pekka Enberg4d268eb2006-01-08 01:00:36 -08001814{
Ingo Molnarb1ab41c2006-06-02 15:44:58 +02001815 unsigned long offslab_limit;
Pekka Enberg4d268eb2006-01-08 01:00:36 -08001816 size_t left_over = 0;
Linus Torvalds9888e6f2006-03-06 17:44:43 -08001817 int gfporder;
Pekka Enberg4d268eb2006-01-08 01:00:36 -08001818
Andrew Mortona737b3e2006-03-22 00:08:11 -08001819 for (gfporder = 0; gfporder <= MAX_GFP_ORDER; gfporder++) {
Pekka Enberg4d268eb2006-01-08 01:00:36 -08001820 unsigned int num;
1821 size_t remainder;
1822
Linus Torvalds9888e6f2006-03-06 17:44:43 -08001823 cache_estimate(gfporder, size, align, flags, &remainder, &num);
Pekka Enberg4d268eb2006-01-08 01:00:36 -08001824 if (!num)
1825 continue;
Linus Torvalds9888e6f2006-03-06 17:44:43 -08001826
Ingo Molnarb1ab41c2006-06-02 15:44:58 +02001827 if (flags & CFLGS_OFF_SLAB) {
1828 /*
1829 * Max number of objs-per-slab for caches which
1830 * use off-slab slabs. Needed to avoid a possible
1831 * looping condition in cache_grow().
1832 */
1833 offslab_limit = size - sizeof(struct slab);
1834 offslab_limit /= sizeof(kmem_bufctl_t);
1835
1836 if (num > offslab_limit)
1837 break;
1838 }
Pekka Enberg4d268eb2006-01-08 01:00:36 -08001839
Linus Torvalds9888e6f2006-03-06 17:44:43 -08001840 /* Found something acceptable - save it away */
Pekka Enberg4d268eb2006-01-08 01:00:36 -08001841 cachep->num = num;
Linus Torvalds9888e6f2006-03-06 17:44:43 -08001842 cachep->gfporder = gfporder;
Pekka Enberg4d268eb2006-01-08 01:00:36 -08001843 left_over = remainder;
1844
1845 /*
Linus Torvaldsf78bb8a2006-03-08 10:33:05 -08001846 * A VFS-reclaimable slab tends to have most allocations
1847 * as GFP_NOFS and we really don't want to have to be allocating
1848 * higher-order pages when we are unable to shrink dcache.
1849 */
1850 if (flags & SLAB_RECLAIM_ACCOUNT)
1851 break;
1852
1853 /*
Pekka Enberg4d268eb2006-01-08 01:00:36 -08001854 * Large number of objects is good, but very large slabs are
1855 * currently bad for the gfp()s.
1856 */
Linus Torvalds9888e6f2006-03-06 17:44:43 -08001857 if (gfporder >= slab_break_gfp_order)
Pekka Enberg4d268eb2006-01-08 01:00:36 -08001858 break;
1859
Linus Torvalds9888e6f2006-03-06 17:44:43 -08001860 /*
1861 * Acceptable internal fragmentation?
1862 */
Andrew Mortona737b3e2006-03-22 00:08:11 -08001863 if (left_over * 8 <= (PAGE_SIZE << gfporder))
Pekka Enberg4d268eb2006-01-08 01:00:36 -08001864 break;
1865 }
1866 return left_over;
1867}
1868
Pekka Enbergf30cf7d2006-03-22 00:08:11 -08001869static void setup_cpu_cache(struct kmem_cache *cachep)
1870{
1871 if (g_cpucache_up == FULL) {
1872 enable_cpucache(cachep);
1873 return;
1874 }
1875 if (g_cpucache_up == NONE) {
1876 /*
1877 * Note: the first kmem_cache_create must create the cache
1878 * that's used by kmalloc(24), otherwise the creation of
1879 * further caches will BUG().
1880 */
1881 cachep->array[smp_processor_id()] = &initarray_generic.cache;
1882
1883 /*
1884 * If the cache that's used by kmalloc(sizeof(kmem_list3)) is
1885 * the first cache, then we need to set up all its list3s,
1886 * otherwise the creation of further caches will BUG().
1887 */
1888 set_up_list3s(cachep, SIZE_AC);
1889 if (INDEX_AC == INDEX_L3)
1890 g_cpucache_up = PARTIAL_L3;
1891 else
1892 g_cpucache_up = PARTIAL_AC;
1893 } else {
1894 cachep->array[smp_processor_id()] =
1895 kmalloc(sizeof(struct arraycache_init), GFP_KERNEL);
1896
1897 if (g_cpucache_up == PARTIAL_AC) {
1898 set_up_list3s(cachep, SIZE_L3);
1899 g_cpucache_up = PARTIAL_L3;
1900 } else {
1901 int node;
1902 for_each_online_node(node) {
1903 cachep->nodelists[node] =
1904 kmalloc_node(sizeof(struct kmem_list3),
1905 GFP_KERNEL, node);
1906 BUG_ON(!cachep->nodelists[node]);
1907 kmem_list3_init(cachep->nodelists[node]);
1908 }
1909 }
1910 }
1911 cachep->nodelists[numa_node_id()]->next_reap =
1912 jiffies + REAPTIMEOUT_LIST3 +
1913 ((unsigned long)cachep) % REAPTIMEOUT_LIST3;
1914
1915 cpu_cache_get(cachep)->avail = 0;
1916 cpu_cache_get(cachep)->limit = BOOT_CPUCACHE_ENTRIES;
1917 cpu_cache_get(cachep)->batchcount = 1;
1918 cpu_cache_get(cachep)->touched = 0;
1919 cachep->batchcount = 1;
1920 cachep->limit = BOOT_CPUCACHE_ENTRIES;
1921}
1922
Pekka Enberg4d268eb2006-01-08 01:00:36 -08001923/**
Linus Torvalds1da177e2005-04-16 15:20:36 -07001924 * kmem_cache_create - Create a cache.
1925 * @name: A string which is used in /proc/slabinfo to identify this cache.
1926 * @size: The size of objects to be created in this cache.
1927 * @align: The required alignment for the objects.
1928 * @flags: SLAB flags
1929 * @ctor: A constructor for the objects.
1930 * @dtor: A destructor for the objects.
1931 *
1932 * Returns a ptr to the cache on success, NULL on failure.
1933 * Cannot be called within a int, but can be interrupted.
1934 * The @ctor is run when new pages are allocated by the cache
1935 * and the @dtor is run before the pages are handed back.
1936 *
1937 * @name must be valid until the cache is destroyed. This implies that
Andrew Mortona737b3e2006-03-22 00:08:11 -08001938 * the module calling this has to destroy the cache before getting unloaded.
1939 *
Linus Torvalds1da177e2005-04-16 15:20:36 -07001940 * The flags are
1941 *
1942 * %SLAB_POISON - Poison the slab with a known test pattern (a5a5a5a5)
1943 * to catch references to uninitialised memory.
1944 *
1945 * %SLAB_RED_ZONE - Insert `Red' zones around the allocated memory to check
1946 * for buffer overruns.
1947 *
Linus Torvalds1da177e2005-04-16 15:20:36 -07001948 * %SLAB_HWCACHE_ALIGN - Align the objects in this cache to a hardware
1949 * cacheline. This can be beneficial if you're counting cycles as closely
1950 * as davem.
1951 */
Pekka Enberg343e0d72006-02-01 03:05:50 -08001952struct kmem_cache *
Linus Torvalds1da177e2005-04-16 15:20:36 -07001953kmem_cache_create (const char *name, size_t size, size_t align,
Andrew Mortona737b3e2006-03-22 00:08:11 -08001954 unsigned long flags,
1955 void (*ctor)(void*, struct kmem_cache *, unsigned long),
Pekka Enberg343e0d72006-02-01 03:05:50 -08001956 void (*dtor)(void*, struct kmem_cache *, unsigned long))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001957{
1958 size_t left_over, slab_size, ralign;
Christoph Hellwig7a7c3812006-06-23 02:03:17 -07001959 struct kmem_cache *cachep = NULL, *pc;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001960
1961 /*
1962 * Sanity checks... these are all serious usage bugs.
1963 */
Andrew Mortona737b3e2006-03-22 00:08:11 -08001964 if (!name || in_interrupt() || (size < BYTES_PER_WORD) ||
Pekka Enbergb28a02d2006-01-08 01:00:37 -08001965 (size > (1 << MAX_OBJ_ORDER) * PAGE_SIZE) || (dtor && !ctor)) {
Andrew Mortona737b3e2006-03-22 00:08:11 -08001966 printk(KERN_ERR "%s: Early error in slab %s\n", __FUNCTION__,
1967 name);
Pekka Enbergb28a02d2006-01-08 01:00:37 -08001968 BUG();
1969 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001970
Ravikiran G Thirumalaif0188f42006-02-10 01:51:13 -08001971 /*
1972 * Prevent CPUs from coming and going.
1973 * lock_cpu_hotplug() nests outside cache_chain_mutex
1974 */
1975 lock_cpu_hotplug();
1976
Ingo Molnarfc0abb12006-01-18 17:42:33 -08001977 mutex_lock(&cache_chain_mutex);
Andrew Morton4f12bb42005-11-07 00:58:00 -08001978
Christoph Hellwig7a7c3812006-06-23 02:03:17 -07001979 list_for_each_entry(pc, &cache_chain, next) {
Andrew Morton4f12bb42005-11-07 00:58:00 -08001980 mm_segment_t old_fs = get_fs();
1981 char tmp;
1982 int res;
1983
1984 /*
1985 * This happens when the module gets unloaded and doesn't
1986 * destroy its slab cache and no-one else reuses the vmalloc
1987 * area of the module. Print a warning.
1988 */
1989 set_fs(KERNEL_DS);
1990 res = __get_user(tmp, pc->name);
1991 set_fs(old_fs);
1992 if (res) {
1993 printk("SLAB: cache with size %d has lost its name\n",
Manfred Spraul3dafccf2006-02-01 03:05:42 -08001994 pc->buffer_size);
Andrew Morton4f12bb42005-11-07 00:58:00 -08001995 continue;
1996 }
1997
Pekka Enbergb28a02d2006-01-08 01:00:37 -08001998 if (!strcmp(pc->name, name)) {
Andrew Morton4f12bb42005-11-07 00:58:00 -08001999 printk("kmem_cache_create: duplicate cache %s\n", name);
2000 dump_stack();
2001 goto oops;
2002 }
2003 }
2004
Linus Torvalds1da177e2005-04-16 15:20:36 -07002005#if DEBUG
2006 WARN_ON(strchr(name, ' ')); /* It confuses parsers */
2007 if ((flags & SLAB_DEBUG_INITIAL) && !ctor) {
2008 /* No constructor, but inital state check requested */
2009 printk(KERN_ERR "%s: No con, but init state check "
Pekka Enbergb28a02d2006-01-08 01:00:37 -08002010 "requested - %s\n", __FUNCTION__, name);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002011 flags &= ~SLAB_DEBUG_INITIAL;
2012 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002013#if FORCED_DEBUG
2014 /*
2015 * Enable redzoning and last user accounting, except for caches with
2016 * large objects, if the increased size would increase the object size
2017 * above the next power of two: caches with object sizes just above a
2018 * power of two have a significant amount of internal fragmentation.
2019 */
Andrew Mortona737b3e2006-03-22 00:08:11 -08002020 if (size < 4096 || fls(size - 1) == fls(size-1 + 3 * BYTES_PER_WORD))
Pekka Enbergb28a02d2006-01-08 01:00:37 -08002021 flags |= SLAB_RED_ZONE | SLAB_STORE_USER;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002022 if (!(flags & SLAB_DESTROY_BY_RCU))
2023 flags |= SLAB_POISON;
2024#endif
2025 if (flags & SLAB_DESTROY_BY_RCU)
2026 BUG_ON(flags & SLAB_POISON);
2027#endif
2028 if (flags & SLAB_DESTROY_BY_RCU)
2029 BUG_ON(dtor);
2030
2031 /*
Andrew Mortona737b3e2006-03-22 00:08:11 -08002032 * Always checks flags, a caller might be expecting debug support which
2033 * isn't available.
Linus Torvalds1da177e2005-04-16 15:20:36 -07002034 */
Eric Sesterhenn40094fa2006-04-02 13:49:25 +02002035 BUG_ON(flags & ~CREATE_MASK);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002036
Andrew Mortona737b3e2006-03-22 00:08:11 -08002037 /*
2038 * Check that size is in terms of words. This is needed to avoid
Linus Torvalds1da177e2005-04-16 15:20:36 -07002039 * unaligned accesses for some archs when redzoning is used, and makes
2040 * sure any on-slab bufctl's are also correctly aligned.
2041 */
Pekka Enbergb28a02d2006-01-08 01:00:37 -08002042 if (size & (BYTES_PER_WORD - 1)) {
2043 size += (BYTES_PER_WORD - 1);
2044 size &= ~(BYTES_PER_WORD - 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002045 }
2046
Andrew Mortona737b3e2006-03-22 00:08:11 -08002047 /* calculate the final buffer alignment: */
2048
Linus Torvalds1da177e2005-04-16 15:20:36 -07002049 /* 1) arch recommendation: can be overridden for debug */
2050 if (flags & SLAB_HWCACHE_ALIGN) {
Andrew Mortona737b3e2006-03-22 00:08:11 -08002051 /*
2052 * Default alignment: as specified by the arch code. Except if
2053 * an object is really small, then squeeze multiple objects into
2054 * one cacheline.
Linus Torvalds1da177e2005-04-16 15:20:36 -07002055 */
2056 ralign = cache_line_size();
Pekka Enbergb28a02d2006-01-08 01:00:37 -08002057 while (size <= ralign / 2)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002058 ralign /= 2;
2059 } else {
2060 ralign = BYTES_PER_WORD;
2061 }
2062 /* 2) arch mandated alignment: disables debug if necessary */
2063 if (ralign < ARCH_SLAB_MINALIGN) {
2064 ralign = ARCH_SLAB_MINALIGN;
2065 if (ralign > BYTES_PER_WORD)
Pekka Enbergb28a02d2006-01-08 01:00:37 -08002066 flags &= ~(SLAB_RED_ZONE | SLAB_STORE_USER);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002067 }
2068 /* 3) caller mandated alignment: disables debug if necessary */
2069 if (ralign < align) {
2070 ralign = align;
2071 if (ralign > BYTES_PER_WORD)
Pekka Enbergb28a02d2006-01-08 01:00:37 -08002072 flags &= ~(SLAB_RED_ZONE | SLAB_STORE_USER);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002073 }
Andrew Mortona737b3e2006-03-22 00:08:11 -08002074 /*
2075 * 4) Store it. Note that the debug code below can reduce
Linus Torvalds1da177e2005-04-16 15:20:36 -07002076 * the alignment to BYTES_PER_WORD.
2077 */
2078 align = ralign;
2079
2080 /* Get cache's description obj. */
Pekka Enbergc5e3b832006-03-25 03:06:43 -08002081 cachep = kmem_cache_zalloc(&cache_cache, SLAB_KERNEL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002082 if (!cachep)
Andrew Morton4f12bb42005-11-07 00:58:00 -08002083 goto oops;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002084
2085#if DEBUG
Manfred Spraul3dafccf2006-02-01 03:05:42 -08002086 cachep->obj_size = size;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002087
2088 if (flags & SLAB_RED_ZONE) {
2089 /* redzoning only works with word aligned caches */
2090 align = BYTES_PER_WORD;
2091
2092 /* add space for red zone words */
Manfred Spraul3dafccf2006-02-01 03:05:42 -08002093 cachep->obj_offset += BYTES_PER_WORD;
Pekka Enbergb28a02d2006-01-08 01:00:37 -08002094 size += 2 * BYTES_PER_WORD;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002095 }
2096 if (flags & SLAB_STORE_USER) {
2097 /* user store requires word alignment and
2098 * one word storage behind the end of the real
2099 * object.
2100 */
2101 align = BYTES_PER_WORD;
2102 size += BYTES_PER_WORD;
2103 }
2104#if FORCED_DEBUG && defined(CONFIG_DEBUG_PAGEALLOC)
Pekka Enbergb28a02d2006-01-08 01:00:37 -08002105 if (size >= malloc_sizes[INDEX_L3 + 1].cs_size
Manfred Spraul3dafccf2006-02-01 03:05:42 -08002106 && cachep->obj_size > cache_line_size() && size < PAGE_SIZE) {
2107 cachep->obj_offset += PAGE_SIZE - size;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002108 size = PAGE_SIZE;
2109 }
2110#endif
2111#endif
2112
Ingo Molnare0a42722006-06-23 02:03:46 -07002113 /*
2114 * Determine if the slab management is 'on' or 'off' slab.
2115 * (bootstrapping cannot cope with offslab caches so don't do
2116 * it too early on.)
2117 */
2118 if ((size >= (PAGE_SIZE >> 3)) && !slab_early_init)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002119 /*
2120 * Size is large, assume best to place the slab management obj
2121 * off-slab (should allow better packing of objs).
2122 */
2123 flags |= CFLGS_OFF_SLAB;
2124
2125 size = ALIGN(size, align);
2126
Linus Torvaldsf78bb8a2006-03-08 10:33:05 -08002127 left_over = calculate_slab_order(cachep, size, align, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002128
2129 if (!cachep->num) {
2130 printk("kmem_cache_create: couldn't create cache %s.\n", name);
2131 kmem_cache_free(&cache_cache, cachep);
2132 cachep = NULL;
Andrew Morton4f12bb42005-11-07 00:58:00 -08002133 goto oops;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002134 }
Pekka Enbergb28a02d2006-01-08 01:00:37 -08002135 slab_size = ALIGN(cachep->num * sizeof(kmem_bufctl_t)
2136 + sizeof(struct slab), align);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002137
2138 /*
2139 * If the slab has been placed off-slab, and we have enough space then
2140 * move it on-slab. This is at the expense of any extra colouring.
2141 */
2142 if (flags & CFLGS_OFF_SLAB && left_over >= slab_size) {
2143 flags &= ~CFLGS_OFF_SLAB;
2144 left_over -= slab_size;
2145 }
2146
2147 if (flags & CFLGS_OFF_SLAB) {
2148 /* really off slab. No need for manual alignment */
Pekka Enbergb28a02d2006-01-08 01:00:37 -08002149 slab_size =
2150 cachep->num * sizeof(kmem_bufctl_t) + sizeof(struct slab);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002151 }
2152
2153 cachep->colour_off = cache_line_size();
2154 /* Offset must be a multiple of the alignment. */
2155 if (cachep->colour_off < align)
2156 cachep->colour_off = align;
Pekka Enbergb28a02d2006-01-08 01:00:37 -08002157 cachep->colour = left_over / cachep->colour_off;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002158 cachep->slab_size = slab_size;
2159 cachep->flags = flags;
2160 cachep->gfpflags = 0;
2161 if (flags & SLAB_CACHE_DMA)
2162 cachep->gfpflags |= GFP_DMA;
Manfred Spraul3dafccf2006-02-01 03:05:42 -08002163 cachep->buffer_size = size;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002164
2165 if (flags & CFLGS_OFF_SLAB)
Victor Fuscob2d55072005-09-10 00:26:36 -07002166 cachep->slabp_cache = kmem_find_general_cachep(slab_size, 0u);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002167 cachep->ctor = ctor;
2168 cachep->dtor = dtor;
2169 cachep->name = name;
2170
Linus Torvalds1da177e2005-04-16 15:20:36 -07002171
Pekka Enbergf30cf7d2006-03-22 00:08:11 -08002172 setup_cpu_cache(cachep);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002173
Linus Torvalds1da177e2005-04-16 15:20:36 -07002174 /* cache setup completed, link it into the list */
2175 list_add(&cachep->next, &cache_chain);
Andrew Mortona737b3e2006-03-22 00:08:11 -08002176oops:
Linus Torvalds1da177e2005-04-16 15:20:36 -07002177 if (!cachep && (flags & SLAB_PANIC))
2178 panic("kmem_cache_create(): failed to create slab `%s'\n",
Pekka Enbergb28a02d2006-01-08 01:00:37 -08002179 name);
Ingo Molnarfc0abb12006-01-18 17:42:33 -08002180 mutex_unlock(&cache_chain_mutex);
Ravikiran G Thirumalaif0188f42006-02-10 01:51:13 -08002181 unlock_cpu_hotplug();
Linus Torvalds1da177e2005-04-16 15:20:36 -07002182 return cachep;
2183}
2184EXPORT_SYMBOL(kmem_cache_create);
2185
2186#if DEBUG
2187static void check_irq_off(void)
2188{
2189 BUG_ON(!irqs_disabled());
2190}
2191
2192static void check_irq_on(void)
2193{
2194 BUG_ON(irqs_disabled());
2195}
2196
Pekka Enberg343e0d72006-02-01 03:05:50 -08002197static void check_spinlock_acquired(struct kmem_cache *cachep)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002198{
2199#ifdef CONFIG_SMP
2200 check_irq_off();
Christoph Lametere498be72005-09-09 13:03:32 -07002201 assert_spin_locked(&cachep->nodelists[numa_node_id()]->list_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002202#endif
2203}
Christoph Lametere498be72005-09-09 13:03:32 -07002204
Pekka Enberg343e0d72006-02-01 03:05:50 -08002205static void check_spinlock_acquired_node(struct kmem_cache *cachep, int node)
Christoph Lametere498be72005-09-09 13:03:32 -07002206{
2207#ifdef CONFIG_SMP
2208 check_irq_off();
2209 assert_spin_locked(&cachep->nodelists[node]->list_lock);
2210#endif
2211}
2212
Linus Torvalds1da177e2005-04-16 15:20:36 -07002213#else
2214#define check_irq_off() do { } while(0)
2215#define check_irq_on() do { } while(0)
2216#define check_spinlock_acquired(x) do { } while(0)
Christoph Lametere498be72005-09-09 13:03:32 -07002217#define check_spinlock_acquired_node(x, y) do { } while(0)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002218#endif
2219
Christoph Lameteraab22072006-03-22 00:09:06 -08002220static void drain_array(struct kmem_cache *cachep, struct kmem_list3 *l3,
2221 struct array_cache *ac,
2222 int force, int node);
2223
Linus Torvalds1da177e2005-04-16 15:20:36 -07002224static void do_drain(void *arg)
2225{
Andrew Mortona737b3e2006-03-22 00:08:11 -08002226 struct kmem_cache *cachep = arg;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002227 struct array_cache *ac;
Christoph Lameterff694162005-09-22 21:44:02 -07002228 int node = numa_node_id();
Linus Torvalds1da177e2005-04-16 15:20:36 -07002229
2230 check_irq_off();
Pekka Enberg9a2dba42006-02-01 03:05:49 -08002231 ac = cpu_cache_get(cachep);
Christoph Lameterff694162005-09-22 21:44:02 -07002232 spin_lock(&cachep->nodelists[node]->list_lock);
2233 free_block(cachep, ac->entry, ac->avail, node);
2234 spin_unlock(&cachep->nodelists[node]->list_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002235 ac->avail = 0;
2236}
2237
Pekka Enberg343e0d72006-02-01 03:05:50 -08002238static void drain_cpu_caches(struct kmem_cache *cachep)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002239{
Christoph Lametere498be72005-09-09 13:03:32 -07002240 struct kmem_list3 *l3;
2241 int node;
2242
Andrew Mortona07fa392006-03-22 00:08:17 -08002243 on_each_cpu(do_drain, cachep, 1, 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002244 check_irq_on();
Pekka Enbergb28a02d2006-01-08 01:00:37 -08002245 for_each_online_node(node) {
Christoph Lametere498be72005-09-09 13:03:32 -07002246 l3 = cachep->nodelists[node];
Roland Dreiera4523a82006-05-15 11:41:00 -07002247 if (l3 && l3->alien)
2248 drain_alien_cache(cachep, l3->alien);
2249 }
2250
2251 for_each_online_node(node) {
2252 l3 = cachep->nodelists[node];
2253 if (l3)
Christoph Lameteraab22072006-03-22 00:09:06 -08002254 drain_array(cachep, l3, l3->shared, 1, node);
Christoph Lametere498be72005-09-09 13:03:32 -07002255 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002256}
2257
Pekka Enberg343e0d72006-02-01 03:05:50 -08002258static int __node_shrink(struct kmem_cache *cachep, int node)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002259{
2260 struct slab *slabp;
Christoph Lametere498be72005-09-09 13:03:32 -07002261 struct kmem_list3 *l3 = cachep->nodelists[node];
Linus Torvalds1da177e2005-04-16 15:20:36 -07002262 int ret;
2263
Christoph Lametere498be72005-09-09 13:03:32 -07002264 for (;;) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07002265 struct list_head *p;
2266
Christoph Lametere498be72005-09-09 13:03:32 -07002267 p = l3->slabs_free.prev;
2268 if (p == &l3->slabs_free)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002269 break;
2270
Christoph Lametere498be72005-09-09 13:03:32 -07002271 slabp = list_entry(l3->slabs_free.prev, struct slab, list);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002272#if DEBUG
Eric Sesterhenn40094fa2006-04-02 13:49:25 +02002273 BUG_ON(slabp->inuse);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002274#endif
2275 list_del(&slabp->list);
2276
Christoph Lametere498be72005-09-09 13:03:32 -07002277 l3->free_objects -= cachep->num;
2278 spin_unlock_irq(&l3->list_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002279 slab_destroy(cachep, slabp);
Christoph Lametere498be72005-09-09 13:03:32 -07002280 spin_lock_irq(&l3->list_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002281 }
Pekka Enbergb28a02d2006-01-08 01:00:37 -08002282 ret = !list_empty(&l3->slabs_full) || !list_empty(&l3->slabs_partial);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002283 return ret;
2284}
2285
Pekka Enberg343e0d72006-02-01 03:05:50 -08002286static int __cache_shrink(struct kmem_cache *cachep)
Christoph Lametere498be72005-09-09 13:03:32 -07002287{
2288 int ret = 0, i = 0;
2289 struct kmem_list3 *l3;
2290
2291 drain_cpu_caches(cachep);
2292
2293 check_irq_on();
2294 for_each_online_node(i) {
2295 l3 = cachep->nodelists[i];
2296 if (l3) {
2297 spin_lock_irq(&l3->list_lock);
2298 ret += __node_shrink(cachep, i);
2299 spin_unlock_irq(&l3->list_lock);
2300 }
2301 }
2302 return (ret ? 1 : 0);
2303}
2304
Linus Torvalds1da177e2005-04-16 15:20:36 -07002305/**
2306 * kmem_cache_shrink - Shrink a cache.
2307 * @cachep: The cache to shrink.
2308 *
2309 * Releases as many slabs as possible for a cache.
2310 * To help debugging, a zero exit status indicates all slabs were released.
2311 */
Pekka Enberg343e0d72006-02-01 03:05:50 -08002312int kmem_cache_shrink(struct kmem_cache *cachep)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002313{
Eric Sesterhenn40094fa2006-04-02 13:49:25 +02002314 BUG_ON(!cachep || in_interrupt());
Linus Torvalds1da177e2005-04-16 15:20:36 -07002315
2316 return __cache_shrink(cachep);
2317}
2318EXPORT_SYMBOL(kmem_cache_shrink);
2319
2320/**
2321 * kmem_cache_destroy - delete a cache
2322 * @cachep: the cache to destroy
2323 *
Pekka Enberg343e0d72006-02-01 03:05:50 -08002324 * Remove a struct kmem_cache object from the slab cache.
Linus Torvalds1da177e2005-04-16 15:20:36 -07002325 * Returns 0 on success.
2326 *
2327 * It is expected this function will be called by a module when it is
2328 * unloaded. This will remove the cache completely, and avoid a duplicate
2329 * cache being allocated each time a module is loaded and unloaded, if the
2330 * module doesn't have persistent in-kernel storage across loads and unloads.
2331 *
2332 * The cache must be empty before calling this function.
2333 *
2334 * The caller must guarantee that noone will allocate memory from the cache
2335 * during the kmem_cache_destroy().
2336 */
Pekka Enberg343e0d72006-02-01 03:05:50 -08002337int kmem_cache_destroy(struct kmem_cache *cachep)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002338{
2339 int i;
Christoph Lametere498be72005-09-09 13:03:32 -07002340 struct kmem_list3 *l3;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002341
Eric Sesterhenn40094fa2006-04-02 13:49:25 +02002342 BUG_ON(!cachep || in_interrupt());
Linus Torvalds1da177e2005-04-16 15:20:36 -07002343
2344 /* Don't let CPUs to come and go */
2345 lock_cpu_hotplug();
2346
2347 /* Find the cache in the chain of caches. */
Ingo Molnarfc0abb12006-01-18 17:42:33 -08002348 mutex_lock(&cache_chain_mutex);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002349 /*
2350 * the chain is never empty, cache_cache is never destroyed
2351 */
2352 list_del(&cachep->next);
Ingo Molnarfc0abb12006-01-18 17:42:33 -08002353 mutex_unlock(&cache_chain_mutex);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002354
2355 if (__cache_shrink(cachep)) {
2356 slab_error(cachep, "Can't free all objects");
Ingo Molnarfc0abb12006-01-18 17:42:33 -08002357 mutex_lock(&cache_chain_mutex);
Pekka Enbergb28a02d2006-01-08 01:00:37 -08002358 list_add(&cachep->next, &cache_chain);
Ingo Molnarfc0abb12006-01-18 17:42:33 -08002359 mutex_unlock(&cache_chain_mutex);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002360 unlock_cpu_hotplug();
2361 return 1;
2362 }
2363
2364 if (unlikely(cachep->flags & SLAB_DESTROY_BY_RCU))
Paul E. McKenneyfbd568a3e2005-05-01 08:59:04 -07002365 synchronize_rcu();
Linus Torvalds1da177e2005-04-16 15:20:36 -07002366
Christoph Lametere498be72005-09-09 13:03:32 -07002367 for_each_online_cpu(i)
Pekka Enbergb28a02d2006-01-08 01:00:37 -08002368 kfree(cachep->array[i]);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002369
2370 /* NUMA: free the list3 structures */
Christoph Lametere498be72005-09-09 13:03:32 -07002371 for_each_online_node(i) {
Andrew Mortona737b3e2006-03-22 00:08:11 -08002372 l3 = cachep->nodelists[i];
2373 if (l3) {
Christoph Lametere498be72005-09-09 13:03:32 -07002374 kfree(l3->shared);
2375 free_alien_cache(l3->alien);
2376 kfree(l3);
2377 }
2378 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002379 kmem_cache_free(&cache_cache, cachep);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002380 unlock_cpu_hotplug();
Linus Torvalds1da177e2005-04-16 15:20:36 -07002381 return 0;
2382}
2383EXPORT_SYMBOL(kmem_cache_destroy);
2384
2385/* Get the memory for a slab management obj. */
Pekka Enberg343e0d72006-02-01 03:05:50 -08002386static struct slab *alloc_slabmgmt(struct kmem_cache *cachep, void *objp,
Ravikiran G Thirumalai5b74ada2006-04-10 22:52:53 -07002387 int colour_off, gfp_t local_flags,
2388 int nodeid)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002389{
2390 struct slab *slabp;
Pekka Enbergb28a02d2006-01-08 01:00:37 -08002391
Linus Torvalds1da177e2005-04-16 15:20:36 -07002392 if (OFF_SLAB(cachep)) {
2393 /* Slab management obj is off-slab. */
Ravikiran G Thirumalai5b74ada2006-04-10 22:52:53 -07002394 slabp = kmem_cache_alloc_node(cachep->slabp_cache,
2395 local_flags, nodeid);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002396 if (!slabp)
2397 return NULL;
2398 } else {
Pekka Enbergb28a02d2006-01-08 01:00:37 -08002399 slabp = objp + colour_off;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002400 colour_off += cachep->slab_size;
2401 }
2402 slabp->inuse = 0;
2403 slabp->colouroff = colour_off;
Pekka Enbergb28a02d2006-01-08 01:00:37 -08002404 slabp->s_mem = objp + colour_off;
Ravikiran G Thirumalai5b74ada2006-04-10 22:52:53 -07002405 slabp->nodeid = nodeid;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002406 return slabp;
2407}
2408
2409static inline kmem_bufctl_t *slab_bufctl(struct slab *slabp)
2410{
Pekka Enbergb28a02d2006-01-08 01:00:37 -08002411 return (kmem_bufctl_t *) (slabp + 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002412}
2413
Pekka Enberg343e0d72006-02-01 03:05:50 -08002414static void cache_init_objs(struct kmem_cache *cachep,
Pekka Enbergb28a02d2006-01-08 01:00:37 -08002415 struct slab *slabp, unsigned long ctor_flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002416{
2417 int i;
2418
2419 for (i = 0; i < cachep->num; i++) {
Pekka Enberg8fea4e92006-03-22 00:08:10 -08002420 void *objp = index_to_obj(cachep, slabp, i);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002421#if DEBUG
2422 /* need to poison the objs? */
2423 if (cachep->flags & SLAB_POISON)
2424 poison_obj(cachep, objp, POISON_FREE);
2425 if (cachep->flags & SLAB_STORE_USER)
2426 *dbg_userword(cachep, objp) = NULL;
2427
2428 if (cachep->flags & SLAB_RED_ZONE) {
2429 *dbg_redzone1(cachep, objp) = RED_INACTIVE;
2430 *dbg_redzone2(cachep, objp) = RED_INACTIVE;
2431 }
2432 /*
Andrew Mortona737b3e2006-03-22 00:08:11 -08002433 * Constructors are not allowed to allocate memory from the same
2434 * cache which they are a constructor for. Otherwise, deadlock.
2435 * They must also be threaded.
Linus Torvalds1da177e2005-04-16 15:20:36 -07002436 */
2437 if (cachep->ctor && !(cachep->flags & SLAB_POISON))
Manfred Spraul3dafccf2006-02-01 03:05:42 -08002438 cachep->ctor(objp + obj_offset(cachep), cachep,
Pekka Enbergb28a02d2006-01-08 01:00:37 -08002439 ctor_flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002440
2441 if (cachep->flags & SLAB_RED_ZONE) {
2442 if (*dbg_redzone2(cachep, objp) != RED_INACTIVE)
2443 slab_error(cachep, "constructor overwrote the"
Pekka Enbergb28a02d2006-01-08 01:00:37 -08002444 " end of an object");
Linus Torvalds1da177e2005-04-16 15:20:36 -07002445 if (*dbg_redzone1(cachep, objp) != RED_INACTIVE)
2446 slab_error(cachep, "constructor overwrote the"
Pekka Enbergb28a02d2006-01-08 01:00:37 -08002447 " start of an object");
Linus Torvalds1da177e2005-04-16 15:20:36 -07002448 }
Andrew Mortona737b3e2006-03-22 00:08:11 -08002449 if ((cachep->buffer_size % PAGE_SIZE) == 0 &&
2450 OFF_SLAB(cachep) && cachep->flags & SLAB_POISON)
Pekka Enbergb28a02d2006-01-08 01:00:37 -08002451 kernel_map_pages(virt_to_page(objp),
Manfred Spraul3dafccf2006-02-01 03:05:42 -08002452 cachep->buffer_size / PAGE_SIZE, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002453#else
2454 if (cachep->ctor)
2455 cachep->ctor(objp, cachep, ctor_flags);
2456#endif
Pekka Enbergb28a02d2006-01-08 01:00:37 -08002457 slab_bufctl(slabp)[i] = i + 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002458 }
Pekka Enbergb28a02d2006-01-08 01:00:37 -08002459 slab_bufctl(slabp)[i - 1] = BUFCTL_END;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002460 slabp->free = 0;
2461}
2462
Pekka Enberg343e0d72006-02-01 03:05:50 -08002463static void kmem_flagcheck(struct kmem_cache *cachep, gfp_t flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002464{
Andrew Mortona737b3e2006-03-22 00:08:11 -08002465 if (flags & SLAB_DMA)
2466 BUG_ON(!(cachep->gfpflags & GFP_DMA));
2467 else
2468 BUG_ON(cachep->gfpflags & GFP_DMA);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002469}
2470
Andrew Mortona737b3e2006-03-22 00:08:11 -08002471static void *slab_get_obj(struct kmem_cache *cachep, struct slab *slabp,
2472 int nodeid)
Matthew Dobson78d382d2006-02-01 03:05:47 -08002473{
Pekka Enberg8fea4e92006-03-22 00:08:10 -08002474 void *objp = index_to_obj(cachep, slabp, slabp->free);
Matthew Dobson78d382d2006-02-01 03:05:47 -08002475 kmem_bufctl_t next;
2476
2477 slabp->inuse++;
2478 next = slab_bufctl(slabp)[slabp->free];
2479#if DEBUG
2480 slab_bufctl(slabp)[slabp->free] = BUFCTL_FREE;
2481 WARN_ON(slabp->nodeid != nodeid);
2482#endif
2483 slabp->free = next;
2484
2485 return objp;
2486}
2487
Andrew Mortona737b3e2006-03-22 00:08:11 -08002488static void slab_put_obj(struct kmem_cache *cachep, struct slab *slabp,
2489 void *objp, int nodeid)
Matthew Dobson78d382d2006-02-01 03:05:47 -08002490{
Pekka Enberg8fea4e92006-03-22 00:08:10 -08002491 unsigned int objnr = obj_to_index(cachep, slabp, objp);
Matthew Dobson78d382d2006-02-01 03:05:47 -08002492
2493#if DEBUG
2494 /* Verify that the slab belongs to the intended node */
2495 WARN_ON(slabp->nodeid != nodeid);
2496
Al Viro871751e2006-03-25 03:06:39 -08002497 if (slab_bufctl(slabp)[objnr] + 1 <= SLAB_LIMIT + 1) {
Matthew Dobson78d382d2006-02-01 03:05:47 -08002498 printk(KERN_ERR "slab: double free detected in cache "
Andrew Mortona737b3e2006-03-22 00:08:11 -08002499 "'%s', objp %p\n", cachep->name, objp);
Matthew Dobson78d382d2006-02-01 03:05:47 -08002500 BUG();
2501 }
2502#endif
2503 slab_bufctl(slabp)[objnr] = slabp->free;
2504 slabp->free = objnr;
2505 slabp->inuse--;
2506}
2507
Pekka Enberg47768742006-06-23 02:03:07 -07002508/*
2509 * Map pages beginning at addr to the given cache and slab. This is required
2510 * for the slab allocator to be able to lookup the cache and slab of a
2511 * virtual address for kfree, ksize, kmem_ptr_validate, and slab debugging.
2512 */
2513static void slab_map_pages(struct kmem_cache *cache, struct slab *slab,
2514 void *addr)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002515{
Pekka Enberg47768742006-06-23 02:03:07 -07002516 int nr_pages;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002517 struct page *page;
2518
Pekka Enberg47768742006-06-23 02:03:07 -07002519 page = virt_to_page(addr);
Nick Piggin84097512006-03-22 00:08:34 -08002520
Pekka Enberg47768742006-06-23 02:03:07 -07002521 nr_pages = 1;
Nick Piggin84097512006-03-22 00:08:34 -08002522 if (likely(!PageCompound(page)))
Pekka Enberg47768742006-06-23 02:03:07 -07002523 nr_pages <<= cache->gfporder;
2524
Linus Torvalds1da177e2005-04-16 15:20:36 -07002525 do {
Pekka Enberg47768742006-06-23 02:03:07 -07002526 page_set_cache(page, cache);
2527 page_set_slab(page, slab);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002528 page++;
Pekka Enberg47768742006-06-23 02:03:07 -07002529 } while (--nr_pages);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002530}
2531
2532/*
2533 * Grow (by 1) the number of slabs within a cache. This is called by
2534 * kmem_cache_alloc() when there are no active objs left in a cache.
2535 */
Pekka Enberg343e0d72006-02-01 03:05:50 -08002536static int cache_grow(struct kmem_cache *cachep, gfp_t flags, int nodeid)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002537{
Pekka Enbergb28a02d2006-01-08 01:00:37 -08002538 struct slab *slabp;
2539 void *objp;
2540 size_t offset;
2541 gfp_t local_flags;
2542 unsigned long ctor_flags;
Christoph Lametere498be72005-09-09 13:03:32 -07002543 struct kmem_list3 *l3;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002544
Andrew Mortona737b3e2006-03-22 00:08:11 -08002545 /*
2546 * Be lazy and only check for valid flags here, keeping it out of the
2547 * critical path in kmem_cache_alloc().
Linus Torvalds1da177e2005-04-16 15:20:36 -07002548 */
Eric Sesterhenn40094fa2006-04-02 13:49:25 +02002549 BUG_ON(flags & ~(SLAB_DMA | SLAB_LEVEL_MASK | SLAB_NO_GROW));
Linus Torvalds1da177e2005-04-16 15:20:36 -07002550 if (flags & SLAB_NO_GROW)
2551 return 0;
2552
2553 ctor_flags = SLAB_CTOR_CONSTRUCTOR;
2554 local_flags = (flags & SLAB_LEVEL_MASK);
2555 if (!(local_flags & __GFP_WAIT))
2556 /*
2557 * Not allowed to sleep. Need to tell a constructor about
2558 * this - it might need to know...
2559 */
2560 ctor_flags |= SLAB_CTOR_ATOMIC;
2561
Ravikiran G Thirumalai2e1217c2006-02-04 23:27:56 -08002562 /* Take the l3 list lock to change the colour_next on this node */
Linus Torvalds1da177e2005-04-16 15:20:36 -07002563 check_irq_off();
Ravikiran G Thirumalai2e1217c2006-02-04 23:27:56 -08002564 l3 = cachep->nodelists[nodeid];
2565 spin_lock(&l3->list_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002566
2567 /* Get colour for the slab, and cal the next value. */
Ravikiran G Thirumalai2e1217c2006-02-04 23:27:56 -08002568 offset = l3->colour_next;
2569 l3->colour_next++;
2570 if (l3->colour_next >= cachep->colour)
2571 l3->colour_next = 0;
2572 spin_unlock(&l3->list_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002573
Ravikiran G Thirumalai2e1217c2006-02-04 23:27:56 -08002574 offset *= cachep->colour_off;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002575
2576 if (local_flags & __GFP_WAIT)
2577 local_irq_enable();
2578
2579 /*
2580 * The test for missing atomic flag is performed here, rather than
2581 * the more obvious place, simply to reduce the critical path length
2582 * in kmem_cache_alloc(). If a caller is seriously mis-behaving they
2583 * will eventually be caught here (where it matters).
2584 */
2585 kmem_flagcheck(cachep, flags);
2586
Andrew Mortona737b3e2006-03-22 00:08:11 -08002587 /*
2588 * Get mem for the objs. Attempt to allocate a physical page from
2589 * 'nodeid'.
Christoph Lametere498be72005-09-09 13:03:32 -07002590 */
Andrew Mortona737b3e2006-03-22 00:08:11 -08002591 objp = kmem_getpages(cachep, flags, nodeid);
2592 if (!objp)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002593 goto failed;
2594
2595 /* Get slab management. */
Ravikiran G Thirumalai5b74ada2006-04-10 22:52:53 -07002596 slabp = alloc_slabmgmt(cachep, objp, offset, local_flags, nodeid);
Andrew Mortona737b3e2006-03-22 00:08:11 -08002597 if (!slabp)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002598 goto opps1;
2599
Christoph Lametere498be72005-09-09 13:03:32 -07002600 slabp->nodeid = nodeid;
Pekka Enberg47768742006-06-23 02:03:07 -07002601 slab_map_pages(cachep, slabp, objp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002602
2603 cache_init_objs(cachep, slabp, ctor_flags);
2604
2605 if (local_flags & __GFP_WAIT)
2606 local_irq_disable();
2607 check_irq_off();
Christoph Lametere498be72005-09-09 13:03:32 -07002608 spin_lock(&l3->list_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002609
2610 /* Make slab active. */
Christoph Lametere498be72005-09-09 13:03:32 -07002611 list_add_tail(&slabp->list, &(l3->slabs_free));
Linus Torvalds1da177e2005-04-16 15:20:36 -07002612 STATS_INC_GROWN(cachep);
Christoph Lametere498be72005-09-09 13:03:32 -07002613 l3->free_objects += cachep->num;
2614 spin_unlock(&l3->list_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002615 return 1;
Andrew Mortona737b3e2006-03-22 00:08:11 -08002616opps1:
Linus Torvalds1da177e2005-04-16 15:20:36 -07002617 kmem_freepages(cachep, objp);
Andrew Mortona737b3e2006-03-22 00:08:11 -08002618failed:
Linus Torvalds1da177e2005-04-16 15:20:36 -07002619 if (local_flags & __GFP_WAIT)
2620 local_irq_disable();
2621 return 0;
2622}
2623
2624#if DEBUG
2625
2626/*
2627 * Perform extra freeing checks:
2628 * - detect bad pointers.
2629 * - POISON/RED_ZONE checking
2630 * - destructor calls, for caches with POISON+dtor
2631 */
2632static void kfree_debugcheck(const void *objp)
2633{
2634 struct page *page;
2635
2636 if (!virt_addr_valid(objp)) {
2637 printk(KERN_ERR "kfree_debugcheck: out of range ptr %lxh.\n",
Pekka Enbergb28a02d2006-01-08 01:00:37 -08002638 (unsigned long)objp);
2639 BUG();
Linus Torvalds1da177e2005-04-16 15:20:36 -07002640 }
2641 page = virt_to_page(objp);
2642 if (!PageSlab(page)) {
Pekka Enbergb28a02d2006-01-08 01:00:37 -08002643 printk(KERN_ERR "kfree_debugcheck: bad ptr %lxh.\n",
2644 (unsigned long)objp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002645 BUG();
2646 }
2647}
2648
Pekka Enberg58ce1fd2006-06-23 02:03:24 -07002649static inline void verify_redzone_free(struct kmem_cache *cache, void *obj)
2650{
2651 unsigned long redzone1, redzone2;
2652
2653 redzone1 = *dbg_redzone1(cache, obj);
2654 redzone2 = *dbg_redzone2(cache, obj);
2655
2656 /*
2657 * Redzone is ok.
2658 */
2659 if (redzone1 == RED_ACTIVE && redzone2 == RED_ACTIVE)
2660 return;
2661
2662 if (redzone1 == RED_INACTIVE && redzone2 == RED_INACTIVE)
2663 slab_error(cache, "double free detected");
2664 else
2665 slab_error(cache, "memory outside object was overwritten");
2666
2667 printk(KERN_ERR "%p: redzone 1:0x%lx, redzone 2:0x%lx.\n",
2668 obj, redzone1, redzone2);
2669}
2670
Pekka Enberg343e0d72006-02-01 03:05:50 -08002671static void *cache_free_debugcheck(struct kmem_cache *cachep, void *objp,
Pekka Enbergb28a02d2006-01-08 01:00:37 -08002672 void *caller)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002673{
2674 struct page *page;
2675 unsigned int objnr;
2676 struct slab *slabp;
2677
Manfred Spraul3dafccf2006-02-01 03:05:42 -08002678 objp -= obj_offset(cachep);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002679 kfree_debugcheck(objp);
2680 page = virt_to_page(objp);
2681
Pekka Enberg065d41c2005-11-13 16:06:46 -08002682 slabp = page_get_slab(page);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002683
2684 if (cachep->flags & SLAB_RED_ZONE) {
Pekka Enberg58ce1fd2006-06-23 02:03:24 -07002685 verify_redzone_free(cachep, objp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002686 *dbg_redzone1(cachep, objp) = RED_INACTIVE;
2687 *dbg_redzone2(cachep, objp) = RED_INACTIVE;
2688 }
2689 if (cachep->flags & SLAB_STORE_USER)
2690 *dbg_userword(cachep, objp) = caller;
2691
Pekka Enberg8fea4e92006-03-22 00:08:10 -08002692 objnr = obj_to_index(cachep, slabp, objp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002693
2694 BUG_ON(objnr >= cachep->num);
Pekka Enberg8fea4e92006-03-22 00:08:10 -08002695 BUG_ON(objp != index_to_obj(cachep, slabp, objnr));
Linus Torvalds1da177e2005-04-16 15:20:36 -07002696
2697 if (cachep->flags & SLAB_DEBUG_INITIAL) {
Andrew Mortona737b3e2006-03-22 00:08:11 -08002698 /*
2699 * Need to call the slab's constructor so the caller can
2700 * perform a verify of its state (debugging). Called without
2701 * the cache-lock held.
Linus Torvalds1da177e2005-04-16 15:20:36 -07002702 */
Manfred Spraul3dafccf2006-02-01 03:05:42 -08002703 cachep->ctor(objp + obj_offset(cachep),
Pekka Enbergb28a02d2006-01-08 01:00:37 -08002704 cachep, SLAB_CTOR_CONSTRUCTOR | SLAB_CTOR_VERIFY);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002705 }
2706 if (cachep->flags & SLAB_POISON && cachep->dtor) {
2707 /* we want to cache poison the object,
2708 * call the destruction callback
2709 */
Manfred Spraul3dafccf2006-02-01 03:05:42 -08002710 cachep->dtor(objp + obj_offset(cachep), cachep, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002711 }
Al Viro871751e2006-03-25 03:06:39 -08002712#ifdef CONFIG_DEBUG_SLAB_LEAK
2713 slab_bufctl(slabp)[objnr] = BUFCTL_FREE;
2714#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07002715 if (cachep->flags & SLAB_POISON) {
2716#ifdef CONFIG_DEBUG_PAGEALLOC
Andrew Mortona737b3e2006-03-22 00:08:11 -08002717 if ((cachep->buffer_size % PAGE_SIZE)==0 && OFF_SLAB(cachep)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07002718 store_stackinfo(cachep, objp, (unsigned long)caller);
Pekka Enbergb28a02d2006-01-08 01:00:37 -08002719 kernel_map_pages(virt_to_page(objp),
Manfred Spraul3dafccf2006-02-01 03:05:42 -08002720 cachep->buffer_size / PAGE_SIZE, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002721 } else {
2722 poison_obj(cachep, objp, POISON_FREE);
2723 }
2724#else
2725 poison_obj(cachep, objp, POISON_FREE);
2726#endif
2727 }
2728 return objp;
2729}
2730
Pekka Enberg343e0d72006-02-01 03:05:50 -08002731static void check_slabp(struct kmem_cache *cachep, struct slab *slabp)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002732{
2733 kmem_bufctl_t i;
2734 int entries = 0;
Pekka Enbergb28a02d2006-01-08 01:00:37 -08002735
Linus Torvalds1da177e2005-04-16 15:20:36 -07002736 /* Check slab's freelist to see if this obj is there. */
2737 for (i = slabp->free; i != BUFCTL_END; i = slab_bufctl(slabp)[i]) {
2738 entries++;
2739 if (entries > cachep->num || i >= cachep->num)
2740 goto bad;
2741 }
2742 if (entries != cachep->num - slabp->inuse) {
Andrew Mortona737b3e2006-03-22 00:08:11 -08002743bad:
2744 printk(KERN_ERR "slab: Internal list corruption detected in "
2745 "cache '%s'(%d), slabp %p(%d). Hexdump:\n",
2746 cachep->name, cachep->num, slabp, slabp->inuse);
Pekka Enbergb28a02d2006-01-08 01:00:37 -08002747 for (i = 0;
Linus Torvalds264132b2006-03-06 12:10:07 -08002748 i < sizeof(*slabp) + cachep->num * sizeof(kmem_bufctl_t);
Pekka Enbergb28a02d2006-01-08 01:00:37 -08002749 i++) {
Andrew Mortona737b3e2006-03-22 00:08:11 -08002750 if (i % 16 == 0)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002751 printk("\n%03x:", i);
Pekka Enbergb28a02d2006-01-08 01:00:37 -08002752 printk(" %02x", ((unsigned char *)slabp)[i]);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002753 }
2754 printk("\n");
2755 BUG();
2756 }
2757}
2758#else
2759#define kfree_debugcheck(x) do { } while(0)
2760#define cache_free_debugcheck(x,objp,z) (objp)
2761#define check_slabp(x,y) do { } while(0)
2762#endif
2763
Pekka Enberg343e0d72006-02-01 03:05:50 -08002764static void *cache_alloc_refill(struct kmem_cache *cachep, gfp_t flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002765{
2766 int batchcount;
2767 struct kmem_list3 *l3;
2768 struct array_cache *ac;
2769
2770 check_irq_off();
Pekka Enberg9a2dba42006-02-01 03:05:49 -08002771 ac = cpu_cache_get(cachep);
Andrew Mortona737b3e2006-03-22 00:08:11 -08002772retry:
Linus Torvalds1da177e2005-04-16 15:20:36 -07002773 batchcount = ac->batchcount;
2774 if (!ac->touched && batchcount > BATCHREFILL_LIMIT) {
Andrew Mortona737b3e2006-03-22 00:08:11 -08002775 /*
2776 * If there was little recent activity on this cache, then
2777 * perform only a partial refill. Otherwise we could generate
2778 * refill bouncing.
Linus Torvalds1da177e2005-04-16 15:20:36 -07002779 */
2780 batchcount = BATCHREFILL_LIMIT;
2781 }
Christoph Lametere498be72005-09-09 13:03:32 -07002782 l3 = cachep->nodelists[numa_node_id()];
Linus Torvalds1da177e2005-04-16 15:20:36 -07002783
Christoph Lametere498be72005-09-09 13:03:32 -07002784 BUG_ON(ac->avail > 0 || !l3);
2785 spin_lock(&l3->list_lock);
2786
Christoph Lameter3ded1752006-03-25 03:06:44 -08002787 /* See if we can refill from the shared array */
2788 if (l3->shared && transfer_objects(ac, l3->shared, batchcount))
2789 goto alloc_done;
2790
Linus Torvalds1da177e2005-04-16 15:20:36 -07002791 while (batchcount > 0) {
2792 struct list_head *entry;
2793 struct slab *slabp;
2794 /* Get slab alloc is to come from. */
2795 entry = l3->slabs_partial.next;
2796 if (entry == &l3->slabs_partial) {
2797 l3->free_touched = 1;
2798 entry = l3->slabs_free.next;
2799 if (entry == &l3->slabs_free)
2800 goto must_grow;
2801 }
2802
2803 slabp = list_entry(entry, struct slab, list);
2804 check_slabp(cachep, slabp);
2805 check_spinlock_acquired(cachep);
2806 while (slabp->inuse < cachep->num && batchcount--) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07002807 STATS_INC_ALLOCED(cachep);
2808 STATS_INC_ACTIVE(cachep);
2809 STATS_SET_HIGH(cachep);
2810
Matthew Dobson78d382d2006-02-01 03:05:47 -08002811 ac->entry[ac->avail++] = slab_get_obj(cachep, slabp,
2812 numa_node_id());
Linus Torvalds1da177e2005-04-16 15:20:36 -07002813 }
2814 check_slabp(cachep, slabp);
2815
2816 /* move slabp to correct slabp list: */
2817 list_del(&slabp->list);
2818 if (slabp->free == BUFCTL_END)
2819 list_add(&slabp->list, &l3->slabs_full);
2820 else
2821 list_add(&slabp->list, &l3->slabs_partial);
2822 }
2823
Andrew Mortona737b3e2006-03-22 00:08:11 -08002824must_grow:
Linus Torvalds1da177e2005-04-16 15:20:36 -07002825 l3->free_objects -= ac->avail;
Andrew Mortona737b3e2006-03-22 00:08:11 -08002826alloc_done:
Christoph Lametere498be72005-09-09 13:03:32 -07002827 spin_unlock(&l3->list_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002828
2829 if (unlikely(!ac->avail)) {
2830 int x;
Christoph Lametere498be72005-09-09 13:03:32 -07002831 x = cache_grow(cachep, flags, numa_node_id());
2832
Andrew Mortona737b3e2006-03-22 00:08:11 -08002833 /* cache_grow can reenable interrupts, then ac could change. */
Pekka Enberg9a2dba42006-02-01 03:05:49 -08002834 ac = cpu_cache_get(cachep);
Andrew Mortona737b3e2006-03-22 00:08:11 -08002835 if (!x && ac->avail == 0) /* no objects in sight? abort */
Linus Torvalds1da177e2005-04-16 15:20:36 -07002836 return NULL;
2837
Andrew Mortona737b3e2006-03-22 00:08:11 -08002838 if (!ac->avail) /* objects refilled by interrupt? */
Linus Torvalds1da177e2005-04-16 15:20:36 -07002839 goto retry;
2840 }
2841 ac->touched = 1;
Christoph Lametere498be72005-09-09 13:03:32 -07002842 return ac->entry[--ac->avail];
Linus Torvalds1da177e2005-04-16 15:20:36 -07002843}
2844
Andrew Mortona737b3e2006-03-22 00:08:11 -08002845static inline void cache_alloc_debugcheck_before(struct kmem_cache *cachep,
2846 gfp_t flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002847{
2848 might_sleep_if(flags & __GFP_WAIT);
2849#if DEBUG
2850 kmem_flagcheck(cachep, flags);
2851#endif
2852}
2853
2854#if DEBUG
Andrew Mortona737b3e2006-03-22 00:08:11 -08002855static void *cache_alloc_debugcheck_after(struct kmem_cache *cachep,
2856 gfp_t flags, void *objp, void *caller)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002857{
Pekka Enbergb28a02d2006-01-08 01:00:37 -08002858 if (!objp)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002859 return objp;
Pekka Enbergb28a02d2006-01-08 01:00:37 -08002860 if (cachep->flags & SLAB_POISON) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07002861#ifdef CONFIG_DEBUG_PAGEALLOC
Manfred Spraul3dafccf2006-02-01 03:05:42 -08002862 if ((cachep->buffer_size % PAGE_SIZE) == 0 && OFF_SLAB(cachep))
Pekka Enbergb28a02d2006-01-08 01:00:37 -08002863 kernel_map_pages(virt_to_page(objp),
Manfred Spraul3dafccf2006-02-01 03:05:42 -08002864 cachep->buffer_size / PAGE_SIZE, 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002865 else
2866 check_poison_obj(cachep, objp);
2867#else
2868 check_poison_obj(cachep, objp);
2869#endif
2870 poison_obj(cachep, objp, POISON_INUSE);
2871 }
2872 if (cachep->flags & SLAB_STORE_USER)
2873 *dbg_userword(cachep, objp) = caller;
2874
2875 if (cachep->flags & SLAB_RED_ZONE) {
Andrew Mortona737b3e2006-03-22 00:08:11 -08002876 if (*dbg_redzone1(cachep, objp) != RED_INACTIVE ||
2877 *dbg_redzone2(cachep, objp) != RED_INACTIVE) {
2878 slab_error(cachep, "double free, or memory outside"
2879 " object was overwritten");
Pekka Enbergb28a02d2006-01-08 01:00:37 -08002880 printk(KERN_ERR
Andrew Mortona737b3e2006-03-22 00:08:11 -08002881 "%p: redzone 1:0x%lx, redzone 2:0x%lx\n",
2882 objp, *dbg_redzone1(cachep, objp),
2883 *dbg_redzone2(cachep, objp));
Linus Torvalds1da177e2005-04-16 15:20:36 -07002884 }
2885 *dbg_redzone1(cachep, objp) = RED_ACTIVE;
2886 *dbg_redzone2(cachep, objp) = RED_ACTIVE;
2887 }
Al Viro871751e2006-03-25 03:06:39 -08002888#ifdef CONFIG_DEBUG_SLAB_LEAK
2889 {
2890 struct slab *slabp;
2891 unsigned objnr;
2892
2893 slabp = page_get_slab(virt_to_page(objp));
2894 objnr = (unsigned)(objp - slabp->s_mem) / cachep->buffer_size;
2895 slab_bufctl(slabp)[objnr] = BUFCTL_ACTIVE;
2896 }
2897#endif
Manfred Spraul3dafccf2006-02-01 03:05:42 -08002898 objp += obj_offset(cachep);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002899 if (cachep->ctor && cachep->flags & SLAB_POISON) {
Pekka Enbergb28a02d2006-01-08 01:00:37 -08002900 unsigned long ctor_flags = SLAB_CTOR_CONSTRUCTOR;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002901
2902 if (!(flags & __GFP_WAIT))
2903 ctor_flags |= SLAB_CTOR_ATOMIC;
2904
2905 cachep->ctor(objp, cachep, ctor_flags);
Pekka Enbergb28a02d2006-01-08 01:00:37 -08002906 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002907 return objp;
2908}
2909#else
2910#define cache_alloc_debugcheck_after(a,b,objp,d) (objp)
2911#endif
2912
Pekka Enberg343e0d72006-02-01 03:05:50 -08002913static inline void *____cache_alloc(struct kmem_cache *cachep, gfp_t flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002914{
Pekka Enbergb28a02d2006-01-08 01:00:37 -08002915 void *objp;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002916 struct array_cache *ac;
2917
Christoph Lameterdc85da12006-01-18 17:42:36 -08002918#ifdef CONFIG_NUMA
Paul Jacksonb2455392006-03-24 03:16:12 -08002919 if (unlikely(current->flags & (PF_SPREAD_SLAB | PF_MEMPOLICY))) {
Paul Jacksonc61afb12006-03-24 03:16:08 -08002920 objp = alternate_node_alloc(cachep, flags);
2921 if (objp != NULL)
2922 return objp;
Paul Jackson101a5002006-03-24 03:16:07 -08002923 }
Christoph Lameterdc85da12006-01-18 17:42:36 -08002924#endif
2925
Alok N Kataria5c382302005-09-27 21:45:46 -07002926 check_irq_off();
Pekka Enberg9a2dba42006-02-01 03:05:49 -08002927 ac = cpu_cache_get(cachep);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002928 if (likely(ac->avail)) {
2929 STATS_INC_ALLOCHIT(cachep);
2930 ac->touched = 1;
Christoph Lametere498be72005-09-09 13:03:32 -07002931 objp = ac->entry[--ac->avail];
Linus Torvalds1da177e2005-04-16 15:20:36 -07002932 } else {
2933 STATS_INC_ALLOCMISS(cachep);
2934 objp = cache_alloc_refill(cachep, flags);
2935 }
Alok N Kataria5c382302005-09-27 21:45:46 -07002936 return objp;
2937}
2938
Andrew Mortona737b3e2006-03-22 00:08:11 -08002939static __always_inline void *__cache_alloc(struct kmem_cache *cachep,
2940 gfp_t flags, void *caller)
Alok N Kataria5c382302005-09-27 21:45:46 -07002941{
2942 unsigned long save_flags;
Pekka Enbergb28a02d2006-01-08 01:00:37 -08002943 void *objp;
Alok N Kataria5c382302005-09-27 21:45:46 -07002944
2945 cache_alloc_debugcheck_before(cachep, flags);
2946
2947 local_irq_save(save_flags);
2948 objp = ____cache_alloc(cachep, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002949 local_irq_restore(save_flags);
Eric Dumazet34342e82005-09-03 15:55:06 -07002950 objp = cache_alloc_debugcheck_after(cachep, flags, objp,
Pekka Enberg7fd6b142006-02-01 03:05:52 -08002951 caller);
Eric Dumazet34342e82005-09-03 15:55:06 -07002952 prefetchw(objp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002953 return objp;
2954}
2955
Christoph Lametere498be72005-09-09 13:03:32 -07002956#ifdef CONFIG_NUMA
2957/*
Paul Jacksonb2455392006-03-24 03:16:12 -08002958 * Try allocating on another node if PF_SPREAD_SLAB|PF_MEMPOLICY.
Paul Jacksonc61afb12006-03-24 03:16:08 -08002959 *
2960 * If we are in_interrupt, then process context, including cpusets and
2961 * mempolicy, may not apply and should not be used for allocation policy.
2962 */
2963static void *alternate_node_alloc(struct kmem_cache *cachep, gfp_t flags)
2964{
2965 int nid_alloc, nid_here;
2966
2967 if (in_interrupt())
2968 return NULL;
2969 nid_alloc = nid_here = numa_node_id();
2970 if (cpuset_do_slab_mem_spread() && (cachep->flags & SLAB_MEM_SPREAD))
2971 nid_alloc = cpuset_mem_spread_node();
2972 else if (current->mempolicy)
2973 nid_alloc = slab_node(current->mempolicy);
2974 if (nid_alloc != nid_here)
2975 return __cache_alloc_node(cachep, flags, nid_alloc);
2976 return NULL;
2977}
2978
2979/*
Christoph Lametere498be72005-09-09 13:03:32 -07002980 * A interface to enable slab creation on nodeid
Linus Torvalds1da177e2005-04-16 15:20:36 -07002981 */
Andrew Mortona737b3e2006-03-22 00:08:11 -08002982static void *__cache_alloc_node(struct kmem_cache *cachep, gfp_t flags,
2983 int nodeid)
Christoph Lametere498be72005-09-09 13:03:32 -07002984{
2985 struct list_head *entry;
Pekka Enbergb28a02d2006-01-08 01:00:37 -08002986 struct slab *slabp;
2987 struct kmem_list3 *l3;
2988 void *obj;
Pekka Enbergb28a02d2006-01-08 01:00:37 -08002989 int x;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002990
Pekka Enbergb28a02d2006-01-08 01:00:37 -08002991 l3 = cachep->nodelists[nodeid];
2992 BUG_ON(!l3);
Christoph Lametere498be72005-09-09 13:03:32 -07002993
Andrew Mortona737b3e2006-03-22 00:08:11 -08002994retry:
Ravikiran G Thirumalaica3b9b92006-02-04 23:27:58 -08002995 check_irq_off();
Pekka Enbergb28a02d2006-01-08 01:00:37 -08002996 spin_lock(&l3->list_lock);
2997 entry = l3->slabs_partial.next;
2998 if (entry == &l3->slabs_partial) {
2999 l3->free_touched = 1;
3000 entry = l3->slabs_free.next;
3001 if (entry == &l3->slabs_free)
3002 goto must_grow;
3003 }
Christoph Lametere498be72005-09-09 13:03:32 -07003004
Pekka Enbergb28a02d2006-01-08 01:00:37 -08003005 slabp = list_entry(entry, struct slab, list);
3006 check_spinlock_acquired_node(cachep, nodeid);
3007 check_slabp(cachep, slabp);
Christoph Lametere498be72005-09-09 13:03:32 -07003008
Pekka Enbergb28a02d2006-01-08 01:00:37 -08003009 STATS_INC_NODEALLOCS(cachep);
3010 STATS_INC_ACTIVE(cachep);
3011 STATS_SET_HIGH(cachep);
Christoph Lametere498be72005-09-09 13:03:32 -07003012
Pekka Enbergb28a02d2006-01-08 01:00:37 -08003013 BUG_ON(slabp->inuse == cachep->num);
Christoph Lametere498be72005-09-09 13:03:32 -07003014
Matthew Dobson78d382d2006-02-01 03:05:47 -08003015 obj = slab_get_obj(cachep, slabp, nodeid);
Pekka Enbergb28a02d2006-01-08 01:00:37 -08003016 check_slabp(cachep, slabp);
3017 l3->free_objects--;
3018 /* move slabp to correct slabp list: */
3019 list_del(&slabp->list);
Christoph Lametere498be72005-09-09 13:03:32 -07003020
Andrew Mortona737b3e2006-03-22 00:08:11 -08003021 if (slabp->free == BUFCTL_END)
Pekka Enbergb28a02d2006-01-08 01:00:37 -08003022 list_add(&slabp->list, &l3->slabs_full);
Andrew Mortona737b3e2006-03-22 00:08:11 -08003023 else
Pekka Enbergb28a02d2006-01-08 01:00:37 -08003024 list_add(&slabp->list, &l3->slabs_partial);
Christoph Lametere498be72005-09-09 13:03:32 -07003025
Pekka Enbergb28a02d2006-01-08 01:00:37 -08003026 spin_unlock(&l3->list_lock);
3027 goto done;
Christoph Lametere498be72005-09-09 13:03:32 -07003028
Andrew Mortona737b3e2006-03-22 00:08:11 -08003029must_grow:
Pekka Enbergb28a02d2006-01-08 01:00:37 -08003030 spin_unlock(&l3->list_lock);
3031 x = cache_grow(cachep, flags, nodeid);
Christoph Lametere498be72005-09-09 13:03:32 -07003032
Pekka Enbergb28a02d2006-01-08 01:00:37 -08003033 if (!x)
3034 return NULL;
Christoph Lametere498be72005-09-09 13:03:32 -07003035
Pekka Enbergb28a02d2006-01-08 01:00:37 -08003036 goto retry;
Andrew Mortona737b3e2006-03-22 00:08:11 -08003037done:
Pekka Enbergb28a02d2006-01-08 01:00:37 -08003038 return obj;
Christoph Lametere498be72005-09-09 13:03:32 -07003039}
3040#endif
3041
3042/*
3043 * Caller needs to acquire correct kmem_list's list_lock
3044 */
Pekka Enberg343e0d72006-02-01 03:05:50 -08003045static void free_block(struct kmem_cache *cachep, void **objpp, int nr_objects,
Pekka Enbergb28a02d2006-01-08 01:00:37 -08003046 int node)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003047{
3048 int i;
Christoph Lametere498be72005-09-09 13:03:32 -07003049 struct kmem_list3 *l3;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003050
3051 for (i = 0; i < nr_objects; i++) {
3052 void *objp = objpp[i];
3053 struct slab *slabp;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003054
Pekka Enberg6ed5eb22006-02-01 03:05:49 -08003055 slabp = virt_to_slab(objp);
Christoph Lameterff694162005-09-22 21:44:02 -07003056 l3 = cachep->nodelists[node];
Linus Torvalds1da177e2005-04-16 15:20:36 -07003057 list_del(&slabp->list);
Christoph Lameterff694162005-09-22 21:44:02 -07003058 check_spinlock_acquired_node(cachep, node);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003059 check_slabp(cachep, slabp);
Matthew Dobson78d382d2006-02-01 03:05:47 -08003060 slab_put_obj(cachep, slabp, objp, node);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003061 STATS_DEC_ACTIVE(cachep);
Christoph Lametere498be72005-09-09 13:03:32 -07003062 l3->free_objects++;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003063 check_slabp(cachep, slabp);
3064
3065 /* fixup slab chains */
3066 if (slabp->inuse == 0) {
Christoph Lametere498be72005-09-09 13:03:32 -07003067 if (l3->free_objects > l3->free_limit) {
3068 l3->free_objects -= cachep->num;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003069 slab_destroy(cachep, slabp);
3070 } else {
Christoph Lametere498be72005-09-09 13:03:32 -07003071 list_add(&slabp->list, &l3->slabs_free);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003072 }
3073 } else {
3074 /* Unconditionally move a slab to the end of the
3075 * partial list on free - maximum time for the
3076 * other objects to be freed, too.
3077 */
Christoph Lametere498be72005-09-09 13:03:32 -07003078 list_add_tail(&slabp->list, &l3->slabs_partial);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003079 }
3080 }
3081}
3082
Pekka Enberg343e0d72006-02-01 03:05:50 -08003083static void cache_flusharray(struct kmem_cache *cachep, struct array_cache *ac)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003084{
3085 int batchcount;
Christoph Lametere498be72005-09-09 13:03:32 -07003086 struct kmem_list3 *l3;
Christoph Lameterff694162005-09-22 21:44:02 -07003087 int node = numa_node_id();
Linus Torvalds1da177e2005-04-16 15:20:36 -07003088
3089 batchcount = ac->batchcount;
3090#if DEBUG
3091 BUG_ON(!batchcount || batchcount > ac->avail);
3092#endif
3093 check_irq_off();
Christoph Lameterff694162005-09-22 21:44:02 -07003094 l3 = cachep->nodelists[node];
Christoph Lametere498be72005-09-09 13:03:32 -07003095 spin_lock(&l3->list_lock);
3096 if (l3->shared) {
3097 struct array_cache *shared_array = l3->shared;
Pekka Enbergb28a02d2006-01-08 01:00:37 -08003098 int max = shared_array->limit - shared_array->avail;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003099 if (max) {
3100 if (batchcount > max)
3101 batchcount = max;
Christoph Lametere498be72005-09-09 13:03:32 -07003102 memcpy(&(shared_array->entry[shared_array->avail]),
Pekka Enbergb28a02d2006-01-08 01:00:37 -08003103 ac->entry, sizeof(void *) * batchcount);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003104 shared_array->avail += batchcount;
3105 goto free_done;
3106 }
3107 }
3108
Christoph Lameterff694162005-09-22 21:44:02 -07003109 free_block(cachep, ac->entry, batchcount, node);
Andrew Mortona737b3e2006-03-22 00:08:11 -08003110free_done:
Linus Torvalds1da177e2005-04-16 15:20:36 -07003111#if STATS
3112 {
3113 int i = 0;
3114 struct list_head *p;
3115
Christoph Lametere498be72005-09-09 13:03:32 -07003116 p = l3->slabs_free.next;
3117 while (p != &(l3->slabs_free)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07003118 struct slab *slabp;
3119
3120 slabp = list_entry(p, struct slab, list);
3121 BUG_ON(slabp->inuse);
3122
3123 i++;
3124 p = p->next;
3125 }
3126 STATS_SET_FREEABLE(cachep, i);
3127 }
3128#endif
Christoph Lametere498be72005-09-09 13:03:32 -07003129 spin_unlock(&l3->list_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003130 ac->avail -= batchcount;
Andrew Mortona737b3e2006-03-22 00:08:11 -08003131 memmove(ac->entry, &(ac->entry[batchcount]), sizeof(void *)*ac->avail);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003132}
3133
3134/*
Andrew Mortona737b3e2006-03-22 00:08:11 -08003135 * Release an obj back to its cache. If the obj has a constructed state, it must
3136 * be in this state _before_ it is released. Called with disabled ints.
Linus Torvalds1da177e2005-04-16 15:20:36 -07003137 */
Pekka Enberg343e0d72006-02-01 03:05:50 -08003138static inline void __cache_free(struct kmem_cache *cachep, void *objp)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003139{
Pekka Enberg9a2dba42006-02-01 03:05:49 -08003140 struct array_cache *ac = cpu_cache_get(cachep);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003141
3142 check_irq_off();
3143 objp = cache_free_debugcheck(cachep, objp, __builtin_return_address(0));
3144
Pekka Enberg729bd0b2006-06-23 02:03:05 -07003145 if (cache_free_alien(cachep, objp))
3146 return;
Christoph Lametere498be72005-09-09 13:03:32 -07003147
Linus Torvalds1da177e2005-04-16 15:20:36 -07003148 if (likely(ac->avail < ac->limit)) {
3149 STATS_INC_FREEHIT(cachep);
Christoph Lametere498be72005-09-09 13:03:32 -07003150 ac->entry[ac->avail++] = objp;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003151 return;
3152 } else {
3153 STATS_INC_FREEMISS(cachep);
3154 cache_flusharray(cachep, ac);
Christoph Lametere498be72005-09-09 13:03:32 -07003155 ac->entry[ac->avail++] = objp;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003156 }
3157}
3158
3159/**
3160 * kmem_cache_alloc - Allocate an object
3161 * @cachep: The cache to allocate from.
3162 * @flags: See kmalloc().
3163 *
3164 * Allocate an object from this cache. The flags are only relevant
3165 * if the cache has no available objects.
3166 */
Pekka Enberg343e0d72006-02-01 03:05:50 -08003167void *kmem_cache_alloc(struct kmem_cache *cachep, gfp_t flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003168{
Pekka Enberg7fd6b142006-02-01 03:05:52 -08003169 return __cache_alloc(cachep, flags, __builtin_return_address(0));
Linus Torvalds1da177e2005-04-16 15:20:36 -07003170}
3171EXPORT_SYMBOL(kmem_cache_alloc);
3172
3173/**
Pekka Enberga8c0f9a2006-03-25 03:06:42 -08003174 * kmem_cache_alloc - Allocate an object. The memory is set to zero.
3175 * @cache: The cache to allocate from.
3176 * @flags: See kmalloc().
3177 *
3178 * Allocate an object from this cache and set the allocated memory to zero.
3179 * The flags are only relevant if the cache has no available objects.
3180 */
3181void *kmem_cache_zalloc(struct kmem_cache *cache, gfp_t flags)
3182{
3183 void *ret = __cache_alloc(cache, flags, __builtin_return_address(0));
3184 if (ret)
3185 memset(ret, 0, obj_size(cache));
3186 return ret;
3187}
3188EXPORT_SYMBOL(kmem_cache_zalloc);
3189
3190/**
Linus Torvalds1da177e2005-04-16 15:20:36 -07003191 * kmem_ptr_validate - check if an untrusted pointer might
3192 * be a slab entry.
3193 * @cachep: the cache we're checking against
3194 * @ptr: pointer to validate
3195 *
3196 * This verifies that the untrusted pointer looks sane:
3197 * it is _not_ a guarantee that the pointer is actually
3198 * part of the slab cache in question, but it at least
3199 * validates that the pointer can be dereferenced and
3200 * looks half-way sane.
3201 *
3202 * Currently only used for dentry validation.
3203 */
Pekka Enberg343e0d72006-02-01 03:05:50 -08003204int fastcall kmem_ptr_validate(struct kmem_cache *cachep, void *ptr)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003205{
Pekka Enbergb28a02d2006-01-08 01:00:37 -08003206 unsigned long addr = (unsigned long)ptr;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003207 unsigned long min_addr = PAGE_OFFSET;
Pekka Enbergb28a02d2006-01-08 01:00:37 -08003208 unsigned long align_mask = BYTES_PER_WORD - 1;
Manfred Spraul3dafccf2006-02-01 03:05:42 -08003209 unsigned long size = cachep->buffer_size;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003210 struct page *page;
3211
3212 if (unlikely(addr < min_addr))
3213 goto out;
3214 if (unlikely(addr > (unsigned long)high_memory - size))
3215 goto out;
3216 if (unlikely(addr & align_mask))
3217 goto out;
3218 if (unlikely(!kern_addr_valid(addr)))
3219 goto out;
3220 if (unlikely(!kern_addr_valid(addr + size - 1)))
3221 goto out;
3222 page = virt_to_page(ptr);
3223 if (unlikely(!PageSlab(page)))
3224 goto out;
Pekka Enberg065d41c2005-11-13 16:06:46 -08003225 if (unlikely(page_get_cache(page) != cachep))
Linus Torvalds1da177e2005-04-16 15:20:36 -07003226 goto out;
3227 return 1;
Andrew Mortona737b3e2006-03-22 00:08:11 -08003228out:
Linus Torvalds1da177e2005-04-16 15:20:36 -07003229 return 0;
3230}
3231
3232#ifdef CONFIG_NUMA
3233/**
3234 * kmem_cache_alloc_node - Allocate an object on the specified node
3235 * @cachep: The cache to allocate from.
3236 * @flags: See kmalloc().
3237 * @nodeid: node number of the target node.
3238 *
3239 * Identical to kmem_cache_alloc, except that this function is slow
3240 * and can sleep. And it will allocate memory on the given node, which
3241 * can improve the performance for cpu bound structures.
Christoph Lametere498be72005-09-09 13:03:32 -07003242 * New and improved: it will now make sure that the object gets
3243 * put on the correct node list so that there is no false sharing.
Linus Torvalds1da177e2005-04-16 15:20:36 -07003244 */
Pekka Enberg343e0d72006-02-01 03:05:50 -08003245void *kmem_cache_alloc_node(struct kmem_cache *cachep, gfp_t flags, int nodeid)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003246{
Christoph Lametere498be72005-09-09 13:03:32 -07003247 unsigned long save_flags;
3248 void *ptr;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003249
Christoph Lametere498be72005-09-09 13:03:32 -07003250 cache_alloc_debugcheck_before(cachep, flags);
3251 local_irq_save(save_flags);
Christoph Lameter18f820f2006-02-01 03:05:43 -08003252
3253 if (nodeid == -1 || nodeid == numa_node_id() ||
Andrew Mortona737b3e2006-03-22 00:08:11 -08003254 !cachep->nodelists[nodeid])
Alok N Kataria5c382302005-09-27 21:45:46 -07003255 ptr = ____cache_alloc(cachep, flags);
3256 else
3257 ptr = __cache_alloc_node(cachep, flags, nodeid);
Christoph Lametere498be72005-09-09 13:03:32 -07003258 local_irq_restore(save_flags);
Christoph Lameter18f820f2006-02-01 03:05:43 -08003259
3260 ptr = cache_alloc_debugcheck_after(cachep, flags, ptr,
3261 __builtin_return_address(0));
Linus Torvalds1da177e2005-04-16 15:20:36 -07003262
Christoph Lametere498be72005-09-09 13:03:32 -07003263 return ptr;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003264}
3265EXPORT_SYMBOL(kmem_cache_alloc_node);
3266
Al Virodd0fc662005-10-07 07:46:04 +01003267void *kmalloc_node(size_t size, gfp_t flags, int node)
Manfred Spraul97e2bde2005-05-01 08:58:38 -07003268{
Pekka Enberg343e0d72006-02-01 03:05:50 -08003269 struct kmem_cache *cachep;
Manfred Spraul97e2bde2005-05-01 08:58:38 -07003270
3271 cachep = kmem_find_general_cachep(size, flags);
3272 if (unlikely(cachep == NULL))
3273 return NULL;
3274 return kmem_cache_alloc_node(cachep, flags, node);
3275}
3276EXPORT_SYMBOL(kmalloc_node);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003277#endif
3278
3279/**
Paul Drynoff800590f2006-06-23 02:03:48 -07003280 * __do_kmalloc - allocate memory
Linus Torvalds1da177e2005-04-16 15:20:36 -07003281 * @size: how many bytes of memory are required.
Paul Drynoff800590f2006-06-23 02:03:48 -07003282 * @flags: the type of memory to allocate (see kmalloc).
Randy Dunlap911851e2006-03-22 00:08:14 -08003283 * @caller: function caller for debug tracking of the caller
Linus Torvalds1da177e2005-04-16 15:20:36 -07003284 */
Pekka Enberg7fd6b142006-02-01 03:05:52 -08003285static __always_inline void *__do_kmalloc(size_t size, gfp_t flags,
3286 void *caller)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003287{
Pekka Enberg343e0d72006-02-01 03:05:50 -08003288 struct kmem_cache *cachep;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003289
Manfred Spraul97e2bde2005-05-01 08:58:38 -07003290 /* If you want to save a few bytes .text space: replace
3291 * __ with kmem_.
3292 * Then kmalloc uses the uninlined functions instead of the inline
3293 * functions.
3294 */
3295 cachep = __find_general_cachep(size, flags);
Andrew Mortondbdb9042005-09-23 13:24:10 -07003296 if (unlikely(cachep == NULL))
3297 return NULL;
Pekka Enberg7fd6b142006-02-01 03:05:52 -08003298 return __cache_alloc(cachep, flags, caller);
3299}
3300
Pekka Enberg7fd6b142006-02-01 03:05:52 -08003301
3302void *__kmalloc(size_t size, gfp_t flags)
3303{
Al Viro871751e2006-03-25 03:06:39 -08003304#ifndef CONFIG_DEBUG_SLAB
Pekka Enberg7fd6b142006-02-01 03:05:52 -08003305 return __do_kmalloc(size, flags, NULL);
Al Viro871751e2006-03-25 03:06:39 -08003306#else
3307 return __do_kmalloc(size, flags, __builtin_return_address(0));
3308#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07003309}
3310EXPORT_SYMBOL(__kmalloc);
3311
Al Viro871751e2006-03-25 03:06:39 -08003312#ifdef CONFIG_DEBUG_SLAB
Pekka Enberg7fd6b142006-02-01 03:05:52 -08003313void *__kmalloc_track_caller(size_t size, gfp_t flags, void *caller)
3314{
3315 return __do_kmalloc(size, flags, caller);
3316}
3317EXPORT_SYMBOL(__kmalloc_track_caller);
Pekka Enberg7fd6b142006-02-01 03:05:52 -08003318#endif
3319
Linus Torvalds1da177e2005-04-16 15:20:36 -07003320#ifdef CONFIG_SMP
3321/**
3322 * __alloc_percpu - allocate one copy of the object for every present
3323 * cpu in the system, zeroing them.
3324 * Objects should be dereferenced using the per_cpu_ptr macro only.
3325 *
3326 * @size: how many bytes of memory are required.
Linus Torvalds1da177e2005-04-16 15:20:36 -07003327 */
Pekka Enbergf9f75002006-01-08 01:00:33 -08003328void *__alloc_percpu(size_t size)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003329{
3330 int i;
Pekka Enbergb28a02d2006-01-08 01:00:37 -08003331 struct percpu_data *pdata = kmalloc(sizeof(*pdata), GFP_KERNEL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003332
3333 if (!pdata)
3334 return NULL;
3335
Christoph Lametere498be72005-09-09 13:03:32 -07003336 /*
3337 * Cannot use for_each_online_cpu since a cpu may come online
3338 * and we have no way of figuring out how to fix the array
3339 * that we have allocated then....
3340 */
KAMEZAWA Hiroyuki0a945022006-03-28 01:56:37 -08003341 for_each_possible_cpu(i) {
Christoph Lametere498be72005-09-09 13:03:32 -07003342 int node = cpu_to_node(i);
3343
3344 if (node_online(node))
3345 pdata->ptrs[i] = kmalloc_node(size, GFP_KERNEL, node);
3346 else
3347 pdata->ptrs[i] = kmalloc(size, GFP_KERNEL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003348
3349 if (!pdata->ptrs[i])
3350 goto unwind_oom;
3351 memset(pdata->ptrs[i], 0, size);
3352 }
3353
3354 /* Catch derefs w/o wrappers */
Pekka Enbergb28a02d2006-01-08 01:00:37 -08003355 return (void *)(~(unsigned long)pdata);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003356
Andrew Mortona737b3e2006-03-22 00:08:11 -08003357unwind_oom:
Linus Torvalds1da177e2005-04-16 15:20:36 -07003358 while (--i >= 0) {
3359 if (!cpu_possible(i))
3360 continue;
3361 kfree(pdata->ptrs[i]);
3362 }
3363 kfree(pdata);
3364 return NULL;
3365}
3366EXPORT_SYMBOL(__alloc_percpu);
3367#endif
3368
3369/**
3370 * kmem_cache_free - Deallocate an object
3371 * @cachep: The cache the allocation was from.
3372 * @objp: The previously allocated object.
3373 *
3374 * Free an object which was previously allocated from this
3375 * cache.
3376 */
Pekka Enberg343e0d72006-02-01 03:05:50 -08003377void kmem_cache_free(struct kmem_cache *cachep, void *objp)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003378{
3379 unsigned long flags;
3380
Pekka Enbergddc2e812006-06-23 02:03:40 -07003381 BUG_ON(virt_to_cache(objp) != cachep);
3382
Linus Torvalds1da177e2005-04-16 15:20:36 -07003383 local_irq_save(flags);
3384 __cache_free(cachep, objp);
3385 local_irq_restore(flags);
3386}
3387EXPORT_SYMBOL(kmem_cache_free);
3388
3389/**
Linus Torvalds1da177e2005-04-16 15:20:36 -07003390 * kfree - free previously allocated memory
3391 * @objp: pointer returned by kmalloc.
3392 *
Pekka Enberg80e93ef2005-09-09 13:10:16 -07003393 * If @objp is NULL, no operation is performed.
3394 *
Linus Torvalds1da177e2005-04-16 15:20:36 -07003395 * Don't free memory not originally allocated by kmalloc()
3396 * or you will run into trouble.
3397 */
3398void kfree(const void *objp)
3399{
Pekka Enberg343e0d72006-02-01 03:05:50 -08003400 struct kmem_cache *c;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003401 unsigned long flags;
3402
3403 if (unlikely(!objp))
3404 return;
3405 local_irq_save(flags);
3406 kfree_debugcheck(objp);
Pekka Enberg6ed5eb22006-02-01 03:05:49 -08003407 c = virt_to_cache(objp);
Manfred Spraul3dafccf2006-02-01 03:05:42 -08003408 mutex_debug_check_no_locks_freed(objp, obj_size(c));
Pekka Enbergb28a02d2006-01-08 01:00:37 -08003409 __cache_free(c, (void *)objp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003410 local_irq_restore(flags);
3411}
3412EXPORT_SYMBOL(kfree);
3413
3414#ifdef CONFIG_SMP
3415/**
3416 * free_percpu - free previously allocated percpu memory
3417 * @objp: pointer returned by alloc_percpu.
3418 *
3419 * Don't free memory not originally allocated by alloc_percpu()
3420 * The complemented objp is to check for that.
3421 */
Pekka Enbergb28a02d2006-01-08 01:00:37 -08003422void free_percpu(const void *objp)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003423{
3424 int i;
Pekka Enbergb28a02d2006-01-08 01:00:37 -08003425 struct percpu_data *p = (struct percpu_data *)(~(unsigned long)objp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003426
Christoph Lametere498be72005-09-09 13:03:32 -07003427 /*
3428 * We allocate for all cpus so we cannot use for online cpu here.
3429 */
KAMEZAWA Hiroyuki0a945022006-03-28 01:56:37 -08003430 for_each_possible_cpu(i)
Pekka Enbergb28a02d2006-01-08 01:00:37 -08003431 kfree(p->ptrs[i]);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003432 kfree(p);
3433}
3434EXPORT_SYMBOL(free_percpu);
3435#endif
3436
Pekka Enberg343e0d72006-02-01 03:05:50 -08003437unsigned int kmem_cache_size(struct kmem_cache *cachep)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003438{
Manfred Spraul3dafccf2006-02-01 03:05:42 -08003439 return obj_size(cachep);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003440}
3441EXPORT_SYMBOL(kmem_cache_size);
3442
Pekka Enberg343e0d72006-02-01 03:05:50 -08003443const char *kmem_cache_name(struct kmem_cache *cachep)
Arnaldo Carvalho de Melo19449722005-06-18 22:46:19 -07003444{
3445 return cachep->name;
3446}
3447EXPORT_SYMBOL_GPL(kmem_cache_name);
3448
Christoph Lametere498be72005-09-09 13:03:32 -07003449/*
Christoph Lameter0718dc22006-03-25 03:06:47 -08003450 * This initializes kmem_list3 or resizes varioius caches for all nodes.
Christoph Lametere498be72005-09-09 13:03:32 -07003451 */
Pekka Enberg343e0d72006-02-01 03:05:50 -08003452static int alloc_kmemlist(struct kmem_cache *cachep)
Christoph Lametere498be72005-09-09 13:03:32 -07003453{
3454 int node;
3455 struct kmem_list3 *l3;
Christoph Lametercafeb022006-03-25 03:06:46 -08003456 struct array_cache *new_shared;
3457 struct array_cache **new_alien;
Christoph Lametere498be72005-09-09 13:03:32 -07003458
3459 for_each_online_node(node) {
Christoph Lametercafeb022006-03-25 03:06:46 -08003460
Andrew Mortona737b3e2006-03-22 00:08:11 -08003461 new_alien = alloc_alien_cache(node, cachep->limit);
3462 if (!new_alien)
Christoph Lametere498be72005-09-09 13:03:32 -07003463 goto fail;
Christoph Lametercafeb022006-03-25 03:06:46 -08003464
Christoph Lameter0718dc22006-03-25 03:06:47 -08003465 new_shared = alloc_arraycache(node,
3466 cachep->shared*cachep->batchcount,
Andrew Mortona737b3e2006-03-22 00:08:11 -08003467 0xbaadf00d);
Christoph Lameter0718dc22006-03-25 03:06:47 -08003468 if (!new_shared) {
3469 free_alien_cache(new_alien);
Christoph Lametere498be72005-09-09 13:03:32 -07003470 goto fail;
Christoph Lameter0718dc22006-03-25 03:06:47 -08003471 }
Christoph Lametercafeb022006-03-25 03:06:46 -08003472
Andrew Mortona737b3e2006-03-22 00:08:11 -08003473 l3 = cachep->nodelists[node];
3474 if (l3) {
Christoph Lametercafeb022006-03-25 03:06:46 -08003475 struct array_cache *shared = l3->shared;
3476
Christoph Lametere498be72005-09-09 13:03:32 -07003477 spin_lock_irq(&l3->list_lock);
3478
Christoph Lametercafeb022006-03-25 03:06:46 -08003479 if (shared)
Christoph Lameter0718dc22006-03-25 03:06:47 -08003480 free_block(cachep, shared->entry,
3481 shared->avail, node);
Christoph Lametere498be72005-09-09 13:03:32 -07003482
Christoph Lametercafeb022006-03-25 03:06:46 -08003483 l3->shared = new_shared;
3484 if (!l3->alien) {
Christoph Lametere498be72005-09-09 13:03:32 -07003485 l3->alien = new_alien;
3486 new_alien = NULL;
3487 }
Pekka Enbergb28a02d2006-01-08 01:00:37 -08003488 l3->free_limit = (1 + nr_cpus_node(node)) *
Andrew Mortona737b3e2006-03-22 00:08:11 -08003489 cachep->batchcount + cachep->num;
Christoph Lametere498be72005-09-09 13:03:32 -07003490 spin_unlock_irq(&l3->list_lock);
Christoph Lametercafeb022006-03-25 03:06:46 -08003491 kfree(shared);
Christoph Lametere498be72005-09-09 13:03:32 -07003492 free_alien_cache(new_alien);
3493 continue;
3494 }
Andrew Mortona737b3e2006-03-22 00:08:11 -08003495 l3 = kmalloc_node(sizeof(struct kmem_list3), GFP_KERNEL, node);
Christoph Lameter0718dc22006-03-25 03:06:47 -08003496 if (!l3) {
3497 free_alien_cache(new_alien);
3498 kfree(new_shared);
Christoph Lametere498be72005-09-09 13:03:32 -07003499 goto fail;
Christoph Lameter0718dc22006-03-25 03:06:47 -08003500 }
Christoph Lametere498be72005-09-09 13:03:32 -07003501
3502 kmem_list3_init(l3);
3503 l3->next_reap = jiffies + REAPTIMEOUT_LIST3 +
Andrew Mortona737b3e2006-03-22 00:08:11 -08003504 ((unsigned long)cachep) % REAPTIMEOUT_LIST3;
Christoph Lametercafeb022006-03-25 03:06:46 -08003505 l3->shared = new_shared;
Christoph Lametere498be72005-09-09 13:03:32 -07003506 l3->alien = new_alien;
Pekka Enbergb28a02d2006-01-08 01:00:37 -08003507 l3->free_limit = (1 + nr_cpus_node(node)) *
Andrew Mortona737b3e2006-03-22 00:08:11 -08003508 cachep->batchcount + cachep->num;
Christoph Lametere498be72005-09-09 13:03:32 -07003509 cachep->nodelists[node] = l3;
3510 }
Christoph Lametercafeb022006-03-25 03:06:46 -08003511 return 0;
Christoph Lameter0718dc22006-03-25 03:06:47 -08003512
Andrew Mortona737b3e2006-03-22 00:08:11 -08003513fail:
Christoph Lameter0718dc22006-03-25 03:06:47 -08003514 if (!cachep->next.next) {
3515 /* Cache is not active yet. Roll back what we did */
3516 node--;
3517 while (node >= 0) {
3518 if (cachep->nodelists[node]) {
3519 l3 = cachep->nodelists[node];
3520
3521 kfree(l3->shared);
3522 free_alien_cache(l3->alien);
3523 kfree(l3);
3524 cachep->nodelists[node] = NULL;
3525 }
3526 node--;
3527 }
3528 }
Christoph Lametercafeb022006-03-25 03:06:46 -08003529 return -ENOMEM;
Christoph Lametere498be72005-09-09 13:03:32 -07003530}
3531
Linus Torvalds1da177e2005-04-16 15:20:36 -07003532struct ccupdate_struct {
Pekka Enberg343e0d72006-02-01 03:05:50 -08003533 struct kmem_cache *cachep;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003534 struct array_cache *new[NR_CPUS];
3535};
3536
3537static void do_ccupdate_local(void *info)
3538{
Andrew Mortona737b3e2006-03-22 00:08:11 -08003539 struct ccupdate_struct *new = info;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003540 struct array_cache *old;
3541
3542 check_irq_off();
Pekka Enberg9a2dba42006-02-01 03:05:49 -08003543 old = cpu_cache_get(new->cachep);
Christoph Lametere498be72005-09-09 13:03:32 -07003544
Linus Torvalds1da177e2005-04-16 15:20:36 -07003545 new->cachep->array[smp_processor_id()] = new->new[smp_processor_id()];
3546 new->new[smp_processor_id()] = old;
3547}
3548
Ravikiran G Thirumalaib5d8ca72006-03-22 00:08:12 -08003549/* Always called with the cache_chain_mutex held */
Andrew Mortona737b3e2006-03-22 00:08:11 -08003550static int do_tune_cpucache(struct kmem_cache *cachep, int limit,
3551 int batchcount, int shared)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003552{
3553 struct ccupdate_struct new;
Christoph Lametere498be72005-09-09 13:03:32 -07003554 int i, err;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003555
Pekka Enbergb28a02d2006-01-08 01:00:37 -08003556 memset(&new.new, 0, sizeof(new.new));
Christoph Lametere498be72005-09-09 13:03:32 -07003557 for_each_online_cpu(i) {
Andrew Mortona737b3e2006-03-22 00:08:11 -08003558 new.new[i] = alloc_arraycache(cpu_to_node(i), limit,
3559 batchcount);
Christoph Lametere498be72005-09-09 13:03:32 -07003560 if (!new.new[i]) {
Pekka Enbergb28a02d2006-01-08 01:00:37 -08003561 for (i--; i >= 0; i--)
3562 kfree(new.new[i]);
Christoph Lametere498be72005-09-09 13:03:32 -07003563 return -ENOMEM;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003564 }
3565 }
3566 new.cachep = cachep;
3567
Andrew Mortona07fa392006-03-22 00:08:17 -08003568 on_each_cpu(do_ccupdate_local, (void *)&new, 1, 1);
Christoph Lametere498be72005-09-09 13:03:32 -07003569
Linus Torvalds1da177e2005-04-16 15:20:36 -07003570 check_irq_on();
Linus Torvalds1da177e2005-04-16 15:20:36 -07003571 cachep->batchcount = batchcount;
3572 cachep->limit = limit;
Christoph Lametere498be72005-09-09 13:03:32 -07003573 cachep->shared = shared;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003574
Christoph Lametere498be72005-09-09 13:03:32 -07003575 for_each_online_cpu(i) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07003576 struct array_cache *ccold = new.new[i];
3577 if (!ccold)
3578 continue;
Christoph Lametere498be72005-09-09 13:03:32 -07003579 spin_lock_irq(&cachep->nodelists[cpu_to_node(i)]->list_lock);
Christoph Lameterff694162005-09-22 21:44:02 -07003580 free_block(cachep, ccold->entry, ccold->avail, cpu_to_node(i));
Christoph Lametere498be72005-09-09 13:03:32 -07003581 spin_unlock_irq(&cachep->nodelists[cpu_to_node(i)]->list_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003582 kfree(ccold);
3583 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07003584
Christoph Lametere498be72005-09-09 13:03:32 -07003585 err = alloc_kmemlist(cachep);
3586 if (err) {
3587 printk(KERN_ERR "alloc_kmemlist failed for %s, error %d.\n",
Pekka Enbergb28a02d2006-01-08 01:00:37 -08003588 cachep->name, -err);
Christoph Lametere498be72005-09-09 13:03:32 -07003589 BUG();
Linus Torvalds1da177e2005-04-16 15:20:36 -07003590 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07003591 return 0;
3592}
3593
Ravikiran G Thirumalaib5d8ca72006-03-22 00:08:12 -08003594/* Called with cache_chain_mutex held always */
Pekka Enberg343e0d72006-02-01 03:05:50 -08003595static void enable_cpucache(struct kmem_cache *cachep)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003596{
3597 int err;
3598 int limit, shared;
3599
Andrew Mortona737b3e2006-03-22 00:08:11 -08003600 /*
3601 * The head array serves three purposes:
Linus Torvalds1da177e2005-04-16 15:20:36 -07003602 * - create a LIFO ordering, i.e. return objects that are cache-warm
3603 * - reduce the number of spinlock operations.
Andrew Mortona737b3e2006-03-22 00:08:11 -08003604 * - reduce the number of linked list operations on the slab and
Linus Torvalds1da177e2005-04-16 15:20:36 -07003605 * bufctl chains: array operations are cheaper.
3606 * The numbers are guessed, we should auto-tune as described by
3607 * Bonwick.
3608 */
Manfred Spraul3dafccf2006-02-01 03:05:42 -08003609 if (cachep->buffer_size > 131072)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003610 limit = 1;
Manfred Spraul3dafccf2006-02-01 03:05:42 -08003611 else if (cachep->buffer_size > PAGE_SIZE)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003612 limit = 8;
Manfred Spraul3dafccf2006-02-01 03:05:42 -08003613 else if (cachep->buffer_size > 1024)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003614 limit = 24;
Manfred Spraul3dafccf2006-02-01 03:05:42 -08003615 else if (cachep->buffer_size > 256)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003616 limit = 54;
3617 else
3618 limit = 120;
3619
Andrew Mortona737b3e2006-03-22 00:08:11 -08003620 /*
3621 * CPU bound tasks (e.g. network routing) can exhibit cpu bound
Linus Torvalds1da177e2005-04-16 15:20:36 -07003622 * allocation behaviour: Most allocs on one cpu, most free operations
3623 * on another cpu. For these cases, an efficient object passing between
3624 * cpus is necessary. This is provided by a shared array. The array
3625 * replaces Bonwick's magazine layer.
3626 * On uniprocessor, it's functionally equivalent (but less efficient)
3627 * to a larger limit. Thus disabled by default.
3628 */
3629 shared = 0;
3630#ifdef CONFIG_SMP
Manfred Spraul3dafccf2006-02-01 03:05:42 -08003631 if (cachep->buffer_size <= PAGE_SIZE)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003632 shared = 8;
3633#endif
3634
3635#if DEBUG
Andrew Mortona737b3e2006-03-22 00:08:11 -08003636 /*
3637 * With debugging enabled, large batchcount lead to excessively long
3638 * periods with disabled local interrupts. Limit the batchcount
Linus Torvalds1da177e2005-04-16 15:20:36 -07003639 */
3640 if (limit > 32)
3641 limit = 32;
3642#endif
Pekka Enbergb28a02d2006-01-08 01:00:37 -08003643 err = do_tune_cpucache(cachep, limit, (limit + 1) / 2, shared);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003644 if (err)
3645 printk(KERN_ERR "enable_cpucache failed for %s, error %d.\n",
Pekka Enbergb28a02d2006-01-08 01:00:37 -08003646 cachep->name, -err);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003647}
3648
Christoph Lameter1b552532006-03-22 00:09:07 -08003649/*
3650 * Drain an array if it contains any elements taking the l3 lock only if
Christoph Lameterb18e7e62006-03-22 00:09:07 -08003651 * necessary. Note that the l3 listlock also protects the array_cache
3652 * if drain_array() is used on the shared array.
Christoph Lameter1b552532006-03-22 00:09:07 -08003653 */
3654void drain_array(struct kmem_cache *cachep, struct kmem_list3 *l3,
3655 struct array_cache *ac, int force, int node)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003656{
3657 int tofree;
3658
Christoph Lameter1b552532006-03-22 00:09:07 -08003659 if (!ac || !ac->avail)
3660 return;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003661 if (ac->touched && !force) {
3662 ac->touched = 0;
Christoph Lameterb18e7e62006-03-22 00:09:07 -08003663 } else {
Christoph Lameter1b552532006-03-22 00:09:07 -08003664 spin_lock_irq(&l3->list_lock);
Christoph Lameterb18e7e62006-03-22 00:09:07 -08003665 if (ac->avail) {
3666 tofree = force ? ac->avail : (ac->limit + 4) / 5;
3667 if (tofree > ac->avail)
3668 tofree = (ac->avail + 1) / 2;
3669 free_block(cachep, ac->entry, tofree, node);
3670 ac->avail -= tofree;
3671 memmove(ac->entry, &(ac->entry[tofree]),
3672 sizeof(void *) * ac->avail);
3673 }
Christoph Lameter1b552532006-03-22 00:09:07 -08003674 spin_unlock_irq(&l3->list_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003675 }
3676}
3677
3678/**
3679 * cache_reap - Reclaim memory from caches.
Randy Dunlap1e5d5332005-11-07 01:01:06 -08003680 * @unused: unused parameter
Linus Torvalds1da177e2005-04-16 15:20:36 -07003681 *
3682 * Called from workqueue/eventd every few seconds.
3683 * Purpose:
3684 * - clear the per-cpu caches for this CPU.
3685 * - return freeable pages to the main free memory pool.
3686 *
Andrew Mortona737b3e2006-03-22 00:08:11 -08003687 * If we cannot acquire the cache chain mutex then just give up - we'll try
3688 * again on the next iteration.
Linus Torvalds1da177e2005-04-16 15:20:36 -07003689 */
3690static void cache_reap(void *unused)
3691{
Christoph Hellwig7a7c3812006-06-23 02:03:17 -07003692 struct kmem_cache *searchp;
Christoph Lametere498be72005-09-09 13:03:32 -07003693 struct kmem_list3 *l3;
Christoph Lameteraab22072006-03-22 00:09:06 -08003694 int node = numa_node_id();
Linus Torvalds1da177e2005-04-16 15:20:36 -07003695
Ingo Molnarfc0abb12006-01-18 17:42:33 -08003696 if (!mutex_trylock(&cache_chain_mutex)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07003697 /* Give up. Setup the next iteration. */
Pekka Enbergb28a02d2006-01-08 01:00:37 -08003698 schedule_delayed_work(&__get_cpu_var(reap_work),
3699 REAPTIMEOUT_CPUC);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003700 return;
3701 }
3702
Christoph Hellwig7a7c3812006-06-23 02:03:17 -07003703 list_for_each_entry(searchp, &cache_chain, next) {
Pekka Enbergb28a02d2006-01-08 01:00:37 -08003704 struct list_head *p;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003705 int tofree;
3706 struct slab *slabp;
3707
Linus Torvalds1da177e2005-04-16 15:20:36 -07003708 check_irq_on();
3709
Christoph Lameter35386e32006-03-22 00:09:05 -08003710 /*
3711 * We only take the l3 lock if absolutely necessary and we
3712 * have established with reasonable certainty that
3713 * we can do some work if the lock was obtained.
3714 */
Christoph Lameteraab22072006-03-22 00:09:06 -08003715 l3 = searchp->nodelists[node];
Christoph Lameter35386e32006-03-22 00:09:05 -08003716
Christoph Lameter8fce4d82006-03-09 17:33:54 -08003717 reap_alien(searchp, l3);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003718
Christoph Lameteraab22072006-03-22 00:09:06 -08003719 drain_array(searchp, l3, cpu_cache_get(searchp), 0, node);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003720
Christoph Lameter35386e32006-03-22 00:09:05 -08003721 /*
3722 * These are racy checks but it does not matter
3723 * if we skip one check or scan twice.
3724 */
Christoph Lametere498be72005-09-09 13:03:32 -07003725 if (time_after(l3->next_reap, jiffies))
Christoph Lameter35386e32006-03-22 00:09:05 -08003726 goto next;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003727
Christoph Lametere498be72005-09-09 13:03:32 -07003728 l3->next_reap = jiffies + REAPTIMEOUT_LIST3;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003729
Christoph Lameteraab22072006-03-22 00:09:06 -08003730 drain_array(searchp, l3, l3->shared, 0, node);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003731
Christoph Lametere498be72005-09-09 13:03:32 -07003732 if (l3->free_touched) {
3733 l3->free_touched = 0;
Christoph Lameter35386e32006-03-22 00:09:05 -08003734 goto next;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003735 }
3736
Andrew Mortona737b3e2006-03-22 00:08:11 -08003737 tofree = (l3->free_limit + 5 * searchp->num - 1) /
3738 (5 * searchp->num);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003739 do {
Christoph Lameter35386e32006-03-22 00:09:05 -08003740 /*
3741 * Do not lock if there are no free blocks.
3742 */
3743 if (list_empty(&l3->slabs_free))
Linus Torvalds1da177e2005-04-16 15:20:36 -07003744 break;
3745
Christoph Lameter35386e32006-03-22 00:09:05 -08003746 spin_lock_irq(&l3->list_lock);
3747 p = l3->slabs_free.next;
3748 if (p == &(l3->slabs_free)) {
3749 spin_unlock_irq(&l3->list_lock);
3750 break;
3751 }
3752
Linus Torvalds1da177e2005-04-16 15:20:36 -07003753 slabp = list_entry(p, struct slab, list);
3754 BUG_ON(slabp->inuse);
3755 list_del(&slabp->list);
3756 STATS_INC_REAPED(searchp);
3757
Andrew Mortona737b3e2006-03-22 00:08:11 -08003758 /*
3759 * Safe to drop the lock. The slab is no longer linked
3760 * to the cache. searchp cannot disappear, we hold
Linus Torvalds1da177e2005-04-16 15:20:36 -07003761 * cache_chain_lock
3762 */
Christoph Lametere498be72005-09-09 13:03:32 -07003763 l3->free_objects -= searchp->num;
3764 spin_unlock_irq(&l3->list_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003765 slab_destroy(searchp, slabp);
Pekka Enbergb28a02d2006-01-08 01:00:37 -08003766 } while (--tofree > 0);
Christoph Lameter35386e32006-03-22 00:09:05 -08003767next:
Linus Torvalds1da177e2005-04-16 15:20:36 -07003768 cond_resched();
3769 }
3770 check_irq_on();
Ingo Molnarfc0abb12006-01-18 17:42:33 -08003771 mutex_unlock(&cache_chain_mutex);
Christoph Lameter8fce4d82006-03-09 17:33:54 -08003772 next_reap_node();
Andrew Mortona737b3e2006-03-22 00:08:11 -08003773 /* Set up the next iteration */
Manfred Spraulcd61ef62005-11-07 00:58:02 -08003774 schedule_delayed_work(&__get_cpu_var(reap_work), REAPTIMEOUT_CPUC);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003775}
3776
3777#ifdef CONFIG_PROC_FS
3778
Pekka Enberg85289f92006-01-08 01:00:36 -08003779static void print_slabinfo_header(struct seq_file *m)
3780{
3781 /*
3782 * Output format version, so at least we can change it
3783 * without _too_ many complaints.
3784 */
3785#if STATS
3786 seq_puts(m, "slabinfo - version: 2.1 (statistics)\n");
3787#else
3788 seq_puts(m, "slabinfo - version: 2.1\n");
3789#endif
3790 seq_puts(m, "# name <active_objs> <num_objs> <objsize> "
3791 "<objperslab> <pagesperslab>");
3792 seq_puts(m, " : tunables <limit> <batchcount> <sharedfactor>");
3793 seq_puts(m, " : slabdata <active_slabs> <num_slabs> <sharedavail>");
3794#if STATS
3795 seq_puts(m, " : globalstat <listallocs> <maxobjs> <grown> <reaped> "
Ravikiran G Thirumalaifb7faf32006-04-10 22:52:54 -07003796 "<error> <maxfreeable> <nodeallocs> <remotefrees> <alienoverflow>");
Pekka Enberg85289f92006-01-08 01:00:36 -08003797 seq_puts(m, " : cpustat <allochit> <allocmiss> <freehit> <freemiss>");
3798#endif
3799 seq_putc(m, '\n');
3800}
3801
Linus Torvalds1da177e2005-04-16 15:20:36 -07003802static void *s_start(struct seq_file *m, loff_t *pos)
3803{
3804 loff_t n = *pos;
3805 struct list_head *p;
3806
Ingo Molnarfc0abb12006-01-18 17:42:33 -08003807 mutex_lock(&cache_chain_mutex);
Pekka Enberg85289f92006-01-08 01:00:36 -08003808 if (!n)
3809 print_slabinfo_header(m);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003810 p = cache_chain.next;
3811 while (n--) {
3812 p = p->next;
3813 if (p == &cache_chain)
3814 return NULL;
3815 }
Pekka Enberg343e0d72006-02-01 03:05:50 -08003816 return list_entry(p, struct kmem_cache, next);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003817}
3818
3819static void *s_next(struct seq_file *m, void *p, loff_t *pos)
3820{
Pekka Enberg343e0d72006-02-01 03:05:50 -08003821 struct kmem_cache *cachep = p;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003822 ++*pos;
Andrew Mortona737b3e2006-03-22 00:08:11 -08003823 return cachep->next.next == &cache_chain ?
3824 NULL : list_entry(cachep->next.next, struct kmem_cache, next);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003825}
3826
3827static void s_stop(struct seq_file *m, void *p)
3828{
Ingo Molnarfc0abb12006-01-18 17:42:33 -08003829 mutex_unlock(&cache_chain_mutex);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003830}
3831
3832static int s_show(struct seq_file *m, void *p)
3833{
Pekka Enberg343e0d72006-02-01 03:05:50 -08003834 struct kmem_cache *cachep = p;
Pekka Enbergb28a02d2006-01-08 01:00:37 -08003835 struct slab *slabp;
3836 unsigned long active_objs;
3837 unsigned long num_objs;
3838 unsigned long active_slabs = 0;
3839 unsigned long num_slabs, free_objects = 0, shared_avail = 0;
Christoph Lametere498be72005-09-09 13:03:32 -07003840 const char *name;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003841 char *error = NULL;
Christoph Lametere498be72005-09-09 13:03:32 -07003842 int node;
3843 struct kmem_list3 *l3;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003844
Linus Torvalds1da177e2005-04-16 15:20:36 -07003845 active_objs = 0;
3846 num_slabs = 0;
Christoph Lametere498be72005-09-09 13:03:32 -07003847 for_each_online_node(node) {
3848 l3 = cachep->nodelists[node];
3849 if (!l3)
3850 continue;
3851
Ravikiran G Thirumalaica3b9b92006-02-04 23:27:58 -08003852 check_irq_on();
3853 spin_lock_irq(&l3->list_lock);
Christoph Lametere498be72005-09-09 13:03:32 -07003854
Christoph Hellwig7a7c3812006-06-23 02:03:17 -07003855 list_for_each_entry(slabp, &l3->slabs_full, list) {
Christoph Lametere498be72005-09-09 13:03:32 -07003856 if (slabp->inuse != cachep->num && !error)
3857 error = "slabs_full accounting error";
3858 active_objs += cachep->num;
3859 active_slabs++;
3860 }
Christoph Hellwig7a7c3812006-06-23 02:03:17 -07003861 list_for_each_entry(slabp, &l3->slabs_partial, list) {
Christoph Lametere498be72005-09-09 13:03:32 -07003862 if (slabp->inuse == cachep->num && !error)
3863 error = "slabs_partial inuse accounting error";
3864 if (!slabp->inuse && !error)
3865 error = "slabs_partial/inuse accounting error";
3866 active_objs += slabp->inuse;
3867 active_slabs++;
3868 }
Christoph Hellwig7a7c3812006-06-23 02:03:17 -07003869 list_for_each_entry(slabp, &l3->slabs_free, list) {
Christoph Lametere498be72005-09-09 13:03:32 -07003870 if (slabp->inuse && !error)
3871 error = "slabs_free/inuse accounting error";
3872 num_slabs++;
3873 }
3874 free_objects += l3->free_objects;
Ravikiran G Thirumalai4484ebf2006-02-04 23:27:59 -08003875 if (l3->shared)
3876 shared_avail += l3->shared->avail;
Christoph Lametere498be72005-09-09 13:03:32 -07003877
Ravikiran G Thirumalaica3b9b92006-02-04 23:27:58 -08003878 spin_unlock_irq(&l3->list_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003879 }
Pekka Enbergb28a02d2006-01-08 01:00:37 -08003880 num_slabs += active_slabs;
3881 num_objs = num_slabs * cachep->num;
Christoph Lametere498be72005-09-09 13:03:32 -07003882 if (num_objs - active_objs != free_objects && !error)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003883 error = "free_objects accounting error";
3884
Pekka Enbergb28a02d2006-01-08 01:00:37 -08003885 name = cachep->name;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003886 if (error)
3887 printk(KERN_ERR "slab: cache %s error: %s\n", name, error);
3888
3889 seq_printf(m, "%-17s %6lu %6lu %6u %4u %4d",
Manfred Spraul3dafccf2006-02-01 03:05:42 -08003890 name, active_objs, num_objs, cachep->buffer_size,
Pekka Enbergb28a02d2006-01-08 01:00:37 -08003891 cachep->num, (1 << cachep->gfporder));
Linus Torvalds1da177e2005-04-16 15:20:36 -07003892 seq_printf(m, " : tunables %4u %4u %4u",
Pekka Enbergb28a02d2006-01-08 01:00:37 -08003893 cachep->limit, cachep->batchcount, cachep->shared);
Christoph Lametere498be72005-09-09 13:03:32 -07003894 seq_printf(m, " : slabdata %6lu %6lu %6lu",
Pekka Enbergb28a02d2006-01-08 01:00:37 -08003895 active_slabs, num_slabs, shared_avail);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003896#if STATS
Pekka Enbergb28a02d2006-01-08 01:00:37 -08003897 { /* list3 stats */
Linus Torvalds1da177e2005-04-16 15:20:36 -07003898 unsigned long high = cachep->high_mark;
3899 unsigned long allocs = cachep->num_allocations;
3900 unsigned long grown = cachep->grown;
3901 unsigned long reaped = cachep->reaped;
3902 unsigned long errors = cachep->errors;
3903 unsigned long max_freeable = cachep->max_freeable;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003904 unsigned long node_allocs = cachep->node_allocs;
Christoph Lametere498be72005-09-09 13:03:32 -07003905 unsigned long node_frees = cachep->node_frees;
Ravikiran G Thirumalaifb7faf32006-04-10 22:52:54 -07003906 unsigned long overflows = cachep->node_overflow;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003907
Christoph Lametere498be72005-09-09 13:03:32 -07003908 seq_printf(m, " : globalstat %7lu %6lu %5lu %4lu \
Ravikiran G Thirumalaifb7faf32006-04-10 22:52:54 -07003909 %4lu %4lu %4lu %4lu %4lu", allocs, high, grown,
Andrew Mortona737b3e2006-03-22 00:08:11 -08003910 reaped, errors, max_freeable, node_allocs,
Ravikiran G Thirumalaifb7faf32006-04-10 22:52:54 -07003911 node_frees, overflows);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003912 }
3913 /* cpu stats */
3914 {
3915 unsigned long allochit = atomic_read(&cachep->allochit);
3916 unsigned long allocmiss = atomic_read(&cachep->allocmiss);
3917 unsigned long freehit = atomic_read(&cachep->freehit);
3918 unsigned long freemiss = atomic_read(&cachep->freemiss);
3919
3920 seq_printf(m, " : cpustat %6lu %6lu %6lu %6lu",
Pekka Enbergb28a02d2006-01-08 01:00:37 -08003921 allochit, allocmiss, freehit, freemiss);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003922 }
3923#endif
3924 seq_putc(m, '\n');
Linus Torvalds1da177e2005-04-16 15:20:36 -07003925 return 0;
3926}
3927
3928/*
3929 * slabinfo_op - iterator that generates /proc/slabinfo
3930 *
3931 * Output layout:
3932 * cache-name
3933 * num-active-objs
3934 * total-objs
3935 * object size
3936 * num-active-slabs
3937 * total-slabs
3938 * num-pages-per-slab
3939 * + further values on SMP and with statistics enabled
3940 */
3941
3942struct seq_operations slabinfo_op = {
Pekka Enbergb28a02d2006-01-08 01:00:37 -08003943 .start = s_start,
3944 .next = s_next,
3945 .stop = s_stop,
3946 .show = s_show,
Linus Torvalds1da177e2005-04-16 15:20:36 -07003947};
3948
3949#define MAX_SLABINFO_WRITE 128
3950/**
3951 * slabinfo_write - Tuning for the slab allocator
3952 * @file: unused
3953 * @buffer: user buffer
3954 * @count: data length
3955 * @ppos: unused
3956 */
Pekka Enbergb28a02d2006-01-08 01:00:37 -08003957ssize_t slabinfo_write(struct file *file, const char __user * buffer,
3958 size_t count, loff_t *ppos)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003959{
Pekka Enbergb28a02d2006-01-08 01:00:37 -08003960 char kbuf[MAX_SLABINFO_WRITE + 1], *tmp;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003961 int limit, batchcount, shared, res;
Christoph Hellwig7a7c3812006-06-23 02:03:17 -07003962 struct kmem_cache *cachep;
Pekka Enbergb28a02d2006-01-08 01:00:37 -08003963
Linus Torvalds1da177e2005-04-16 15:20:36 -07003964 if (count > MAX_SLABINFO_WRITE)
3965 return -EINVAL;
3966 if (copy_from_user(&kbuf, buffer, count))
3967 return -EFAULT;
Pekka Enbergb28a02d2006-01-08 01:00:37 -08003968 kbuf[MAX_SLABINFO_WRITE] = '\0';
Linus Torvalds1da177e2005-04-16 15:20:36 -07003969
3970 tmp = strchr(kbuf, ' ');
3971 if (!tmp)
3972 return -EINVAL;
3973 *tmp = '\0';
3974 tmp++;
3975 if (sscanf(tmp, " %d %d %d", &limit, &batchcount, &shared) != 3)
3976 return -EINVAL;
3977
3978 /* Find the cache in the chain of caches. */
Ingo Molnarfc0abb12006-01-18 17:42:33 -08003979 mutex_lock(&cache_chain_mutex);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003980 res = -EINVAL;
Christoph Hellwig7a7c3812006-06-23 02:03:17 -07003981 list_for_each_entry(cachep, &cache_chain, next) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07003982 if (!strcmp(cachep->name, kbuf)) {
Andrew Mortona737b3e2006-03-22 00:08:11 -08003983 if (limit < 1 || batchcount < 1 ||
3984 batchcount > limit || shared < 0) {
Christoph Lametere498be72005-09-09 13:03:32 -07003985 res = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003986 } else {
Christoph Lametere498be72005-09-09 13:03:32 -07003987 res = do_tune_cpucache(cachep, limit,
Pekka Enbergb28a02d2006-01-08 01:00:37 -08003988 batchcount, shared);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003989 }
3990 break;
3991 }
3992 }
Ingo Molnarfc0abb12006-01-18 17:42:33 -08003993 mutex_unlock(&cache_chain_mutex);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003994 if (res >= 0)
3995 res = count;
3996 return res;
3997}
Al Viro871751e2006-03-25 03:06:39 -08003998
3999#ifdef CONFIG_DEBUG_SLAB_LEAK
4000
4001static void *leaks_start(struct seq_file *m, loff_t *pos)
4002{
4003 loff_t n = *pos;
4004 struct list_head *p;
4005
4006 mutex_lock(&cache_chain_mutex);
4007 p = cache_chain.next;
4008 while (n--) {
4009 p = p->next;
4010 if (p == &cache_chain)
4011 return NULL;
4012 }
4013 return list_entry(p, struct kmem_cache, next);
4014}
4015
4016static inline int add_caller(unsigned long *n, unsigned long v)
4017{
4018 unsigned long *p;
4019 int l;
4020 if (!v)
4021 return 1;
4022 l = n[1];
4023 p = n + 2;
4024 while (l) {
4025 int i = l/2;
4026 unsigned long *q = p + 2 * i;
4027 if (*q == v) {
4028 q[1]++;
4029 return 1;
4030 }
4031 if (*q > v) {
4032 l = i;
4033 } else {
4034 p = q + 2;
4035 l -= i + 1;
4036 }
4037 }
4038 if (++n[1] == n[0])
4039 return 0;
4040 memmove(p + 2, p, n[1] * 2 * sizeof(unsigned long) - ((void *)p - (void *)n));
4041 p[0] = v;
4042 p[1] = 1;
4043 return 1;
4044}
4045
4046static void handle_slab(unsigned long *n, struct kmem_cache *c, struct slab *s)
4047{
4048 void *p;
4049 int i;
4050 if (n[0] == n[1])
4051 return;
4052 for (i = 0, p = s->s_mem; i < c->num; i++, p += c->buffer_size) {
4053 if (slab_bufctl(s)[i] != BUFCTL_ACTIVE)
4054 continue;
4055 if (!add_caller(n, (unsigned long)*dbg_userword(c, p)))
4056 return;
4057 }
4058}
4059
4060static void show_symbol(struct seq_file *m, unsigned long address)
4061{
4062#ifdef CONFIG_KALLSYMS
4063 char *modname;
4064 const char *name;
4065 unsigned long offset, size;
4066 char namebuf[KSYM_NAME_LEN+1];
4067
4068 name = kallsyms_lookup(address, &size, &offset, &modname, namebuf);
4069
4070 if (name) {
4071 seq_printf(m, "%s+%#lx/%#lx", name, offset, size);
4072 if (modname)
4073 seq_printf(m, " [%s]", modname);
4074 return;
4075 }
4076#endif
4077 seq_printf(m, "%p", (void *)address);
4078}
4079
4080static int leaks_show(struct seq_file *m, void *p)
4081{
4082 struct kmem_cache *cachep = p;
Al Viro871751e2006-03-25 03:06:39 -08004083 struct slab *slabp;
4084 struct kmem_list3 *l3;
4085 const char *name;
4086 unsigned long *n = m->private;
4087 int node;
4088 int i;
4089
4090 if (!(cachep->flags & SLAB_STORE_USER))
4091 return 0;
4092 if (!(cachep->flags & SLAB_RED_ZONE))
4093 return 0;
4094
4095 /* OK, we can do it */
4096
4097 n[1] = 0;
4098
4099 for_each_online_node(node) {
4100 l3 = cachep->nodelists[node];
4101 if (!l3)
4102 continue;
4103
4104 check_irq_on();
4105 spin_lock_irq(&l3->list_lock);
4106
Christoph Hellwig7a7c3812006-06-23 02:03:17 -07004107 list_for_each_entry(slabp, &l3->slabs_full, list)
Al Viro871751e2006-03-25 03:06:39 -08004108 handle_slab(n, cachep, slabp);
Christoph Hellwig7a7c3812006-06-23 02:03:17 -07004109 list_for_each_entry(slabp, &l3->slabs_partial, list)
Al Viro871751e2006-03-25 03:06:39 -08004110 handle_slab(n, cachep, slabp);
Al Viro871751e2006-03-25 03:06:39 -08004111 spin_unlock_irq(&l3->list_lock);
4112 }
4113 name = cachep->name;
4114 if (n[0] == n[1]) {
4115 /* Increase the buffer size */
4116 mutex_unlock(&cache_chain_mutex);
4117 m->private = kzalloc(n[0] * 4 * sizeof(unsigned long), GFP_KERNEL);
4118 if (!m->private) {
4119 /* Too bad, we are really out */
4120 m->private = n;
4121 mutex_lock(&cache_chain_mutex);
4122 return -ENOMEM;
4123 }
4124 *(unsigned long *)m->private = n[0] * 2;
4125 kfree(n);
4126 mutex_lock(&cache_chain_mutex);
4127 /* Now make sure this entry will be retried */
4128 m->count = m->size;
4129 return 0;
4130 }
4131 for (i = 0; i < n[1]; i++) {
4132 seq_printf(m, "%s: %lu ", name, n[2*i+3]);
4133 show_symbol(m, n[2*i+2]);
4134 seq_putc(m, '\n');
4135 }
4136 return 0;
4137}
4138
4139struct seq_operations slabstats_op = {
4140 .start = leaks_start,
4141 .next = s_next,
4142 .stop = s_stop,
4143 .show = leaks_show,
4144};
4145#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07004146#endif
4147
Manfred Spraul00e145b2005-09-03 15:55:07 -07004148/**
4149 * ksize - get the actual amount of memory allocated for a given object
4150 * @objp: Pointer to the object
4151 *
4152 * kmalloc may internally round up allocations and return more memory
4153 * than requested. ksize() can be used to determine the actual amount of
4154 * memory allocated. The caller may use this additional memory, even though
4155 * a smaller amount of memory was initially specified with the kmalloc call.
4156 * The caller must guarantee that objp points to a valid object previously
4157 * allocated with either kmalloc() or kmem_cache_alloc(). The object
4158 * must not be freed during the duration of the call.
4159 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07004160unsigned int ksize(const void *objp)
4161{
Manfred Spraul00e145b2005-09-03 15:55:07 -07004162 if (unlikely(objp == NULL))
4163 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004164
Pekka Enberg6ed5eb22006-02-01 03:05:49 -08004165 return obj_size(virt_to_cache(objp));
Linus Torvalds1da177e2005-04-16 15:20:36 -07004166}