blob: f1b644eb39d816b3865400caf8d589d458756a17 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * linux/mm/slab.c
3 * Written by Mark Hemment, 1996/97.
4 * (markhe@nextd.demon.co.uk)
5 *
6 * kmem_cache_destroy() + some cleanup - 1999 Andrea Arcangeli
7 *
8 * Major cleanup, different bufctl logic, per-cpu arrays
9 * (c) 2000 Manfred Spraul
10 *
11 * Cleanup, make the head arrays unconditional, preparation for NUMA
12 * (c) 2002 Manfred Spraul
13 *
14 * An implementation of the Slab Allocator as described in outline in;
15 * UNIX Internals: The New Frontiers by Uresh Vahalia
16 * Pub: Prentice Hall ISBN 0-13-101908-2
17 * or with a little more detail in;
18 * The Slab Allocator: An Object-Caching Kernel Memory Allocator
19 * Jeff Bonwick (Sun Microsystems).
20 * Presented at: USENIX Summer 1994 Technical Conference
21 *
22 * The memory is organized in caches, one cache for each object type.
23 * (e.g. inode_cache, dentry_cache, buffer_head, vm_area_struct)
24 * Each cache consists out of many slabs (they are small (usually one
25 * page long) and always contiguous), and each slab contains multiple
26 * initialized objects.
27 *
28 * This means, that your constructor is used only for newly allocated
29 * slabs and you must pass objects with the same intializations to
30 * kmem_cache_free.
31 *
32 * Each cache can only support one memory type (GFP_DMA, GFP_HIGHMEM,
33 * normal). If you need a special memory type, then must create a new
34 * cache for that memory type.
35 *
36 * In order to reduce fragmentation, the slabs are sorted in 3 groups:
37 * full slabs with 0 free objects
38 * partial slabs
39 * empty slabs with no allocated objects
40 *
41 * If partial slabs exist, then new allocations come from these slabs,
42 * otherwise from empty slabs or new slabs are allocated.
43 *
44 * kmem_cache_destroy() CAN CRASH if you try to allocate from the cache
45 * during kmem_cache_destroy(). The caller must prevent concurrent allocs.
46 *
47 * Each cache has a short per-cpu head array, most allocs
48 * and frees go into that array, and if that array overflows, then 1/2
49 * of the entries in the array are given back into the global cache.
50 * The head array is strictly LIFO and should improve the cache hit rates.
51 * On SMP, it additionally reduces the spinlock operations.
52 *
Andrew Mortona737b3e2006-03-22 00:08:11 -080053 * The c_cpuarray may not be read with enabled local interrupts -
Linus Torvalds1da177e2005-04-16 15:20:36 -070054 * it's changed with a smp_call_function().
55 *
56 * SMP synchronization:
57 * constructors and destructors are called without any locking.
Pekka Enberg343e0d72006-02-01 03:05:50 -080058 * Several members in struct kmem_cache and struct slab never change, they
Linus Torvalds1da177e2005-04-16 15:20:36 -070059 * are accessed without any locking.
60 * The per-cpu arrays are never accessed from the wrong cpu, no locking,
61 * and local interrupts are disabled so slab code is preempt-safe.
62 * The non-constant members are protected with a per-cache irq spinlock.
63 *
64 * Many thanks to Mark Hemment, who wrote another per-cpu slab patch
65 * in 2000 - many ideas in the current implementation are derived from
66 * his patch.
67 *
68 * Further notes from the original documentation:
69 *
70 * 11 April '97. Started multi-threading - markhe
Ingo Molnarfc0abb12006-01-18 17:42:33 -080071 * The global cache-chain is protected by the mutex 'cache_chain_mutex'.
Linus Torvalds1da177e2005-04-16 15:20:36 -070072 * The sem is only needed when accessing/extending the cache-chain, which
73 * can never happen inside an interrupt (kmem_cache_create(),
74 * kmem_cache_shrink() and kmem_cache_reap()).
75 *
76 * At present, each engine can be growing a cache. This should be blocked.
77 *
Christoph Lametere498be72005-09-09 13:03:32 -070078 * 15 March 2005. NUMA slab allocator.
79 * Shai Fultheim <shai@scalex86.org>.
80 * Shobhit Dayal <shobhit@calsoftinc.com>
81 * Alok N Kataria <alokk@calsoftinc.com>
82 * Christoph Lameter <christoph@lameter.com>
83 *
84 * Modified the slab allocator to be node aware on NUMA systems.
85 * Each node has its own list of partial, free and full slabs.
86 * All object allocations for a node occur from node specific slab lists.
Linus Torvalds1da177e2005-04-16 15:20:36 -070087 */
88
89#include <linux/config.h>
90#include <linux/slab.h>
91#include <linux/mm.h>
92#include <linux/swap.h>
93#include <linux/cache.h>
94#include <linux/interrupt.h>
95#include <linux/init.h>
96#include <linux/compiler.h>
Paul Jackson101a5002006-03-24 03:16:07 -080097#include <linux/cpuset.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070098#include <linux/seq_file.h>
99#include <linux/notifier.h>
100#include <linux/kallsyms.h>
101#include <linux/cpu.h>
102#include <linux/sysctl.h>
103#include <linux/module.h>
104#include <linux/rcupdate.h>
Paulo Marques543537b2005-06-23 00:09:02 -0700105#include <linux/string.h>
Christoph Lametere498be72005-09-09 13:03:32 -0700106#include <linux/nodemask.h>
Christoph Lameterdc85da12006-01-18 17:42:36 -0800107#include <linux/mempolicy.h>
Ingo Molnarfc0abb12006-01-18 17:42:33 -0800108#include <linux/mutex.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700109
110#include <asm/uaccess.h>
111#include <asm/cacheflush.h>
112#include <asm/tlbflush.h>
113#include <asm/page.h>
114
115/*
116 * DEBUG - 1 for kmem_cache_create() to honour; SLAB_DEBUG_INITIAL,
117 * SLAB_RED_ZONE & SLAB_POISON.
118 * 0 for faster, smaller code (especially in the critical paths).
119 *
120 * STATS - 1 to collect stats for /proc/slabinfo.
121 * 0 for faster, smaller code (especially in the critical paths).
122 *
123 * FORCED_DEBUG - 1 enables SLAB_RED_ZONE and SLAB_POISON (if possible)
124 */
125
126#ifdef CONFIG_DEBUG_SLAB
127#define DEBUG 1
128#define STATS 1
129#define FORCED_DEBUG 1
130#else
131#define DEBUG 0
132#define STATS 0
133#define FORCED_DEBUG 0
134#endif
135
Linus Torvalds1da177e2005-04-16 15:20:36 -0700136/* Shouldn't this be in a header file somewhere? */
137#define BYTES_PER_WORD sizeof(void *)
138
139#ifndef cache_line_size
140#define cache_line_size() L1_CACHE_BYTES
141#endif
142
143#ifndef ARCH_KMALLOC_MINALIGN
144/*
145 * Enforce a minimum alignment for the kmalloc caches.
146 * Usually, the kmalloc caches are cache_line_size() aligned, except when
147 * DEBUG and FORCED_DEBUG are enabled, then they are BYTES_PER_WORD aligned.
148 * Some archs want to perform DMA into kmalloc caches and need a guaranteed
149 * alignment larger than BYTES_PER_WORD. ARCH_KMALLOC_MINALIGN allows that.
150 * Note that this flag disables some debug features.
151 */
152#define ARCH_KMALLOC_MINALIGN 0
153#endif
154
155#ifndef ARCH_SLAB_MINALIGN
156/*
157 * Enforce a minimum alignment for all caches.
158 * Intended for archs that get misalignment faults even for BYTES_PER_WORD
159 * aligned buffers. Includes ARCH_KMALLOC_MINALIGN.
160 * If possible: Do not enable this flag for CONFIG_DEBUG_SLAB, it disables
161 * some debug features.
162 */
163#define ARCH_SLAB_MINALIGN 0
164#endif
165
166#ifndef ARCH_KMALLOC_FLAGS
167#define ARCH_KMALLOC_FLAGS SLAB_HWCACHE_ALIGN
168#endif
169
170/* Legal flag mask for kmem_cache_create(). */
171#if DEBUG
172# define CREATE_MASK (SLAB_DEBUG_INITIAL | SLAB_RED_ZONE | \
173 SLAB_POISON | SLAB_HWCACHE_ALIGN | \
Christoph Lameterac2b8982006-03-22 00:08:15 -0800174 SLAB_CACHE_DMA | \
Linus Torvalds1da177e2005-04-16 15:20:36 -0700175 SLAB_MUST_HWCACHE_ALIGN | SLAB_STORE_USER | \
176 SLAB_RECLAIM_ACCOUNT | SLAB_PANIC | \
Paul Jackson101a5002006-03-24 03:16:07 -0800177 SLAB_DESTROY_BY_RCU | SLAB_MEM_SPREAD)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700178#else
Christoph Lameterac2b8982006-03-22 00:08:15 -0800179# define CREATE_MASK (SLAB_HWCACHE_ALIGN | \
Linus Torvalds1da177e2005-04-16 15:20:36 -0700180 SLAB_CACHE_DMA | SLAB_MUST_HWCACHE_ALIGN | \
181 SLAB_RECLAIM_ACCOUNT | SLAB_PANIC | \
Paul Jackson101a5002006-03-24 03:16:07 -0800182 SLAB_DESTROY_BY_RCU | SLAB_MEM_SPREAD)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700183#endif
184
185/*
186 * kmem_bufctl_t:
187 *
188 * Bufctl's are used for linking objs within a slab
189 * linked offsets.
190 *
191 * This implementation relies on "struct page" for locating the cache &
192 * slab an object belongs to.
193 * This allows the bufctl structure to be small (one int), but limits
194 * the number of objects a slab (not a cache) can contain when off-slab
195 * bufctls are used. The limit is the size of the largest general cache
196 * that does not use off-slab slabs.
197 * For 32bit archs with 4 kB pages, is this 56.
198 * This is not serious, as it is only for large objects, when it is unwise
199 * to have too many per slab.
200 * Note: This limit can be raised by introducing a general cache whose size
201 * is less than 512 (PAGE_SIZE<<3), but greater than 256.
202 */
203
Kyle Moffettfa5b08d2005-09-03 15:55:03 -0700204typedef unsigned int kmem_bufctl_t;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700205#define BUFCTL_END (((kmem_bufctl_t)(~0U))-0)
206#define BUFCTL_FREE (((kmem_bufctl_t)(~0U))-1)
Al Viro871751e2006-03-25 03:06:39 -0800207#define BUFCTL_ACTIVE (((kmem_bufctl_t)(~0U))-2)
208#define SLAB_LIMIT (((kmem_bufctl_t)(~0U))-3)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700209
Linus Torvalds1da177e2005-04-16 15:20:36 -0700210/*
211 * struct slab
212 *
213 * Manages the objs in a slab. Placed either at the beginning of mem allocated
214 * for a slab, or allocated from an general cache.
215 * Slabs are chained into three list: fully used, partial, fully free slabs.
216 */
217struct slab {
Pekka Enbergb28a02d2006-01-08 01:00:37 -0800218 struct list_head list;
219 unsigned long colouroff;
220 void *s_mem; /* including colour offset */
221 unsigned int inuse; /* num of objs active in slab */
222 kmem_bufctl_t free;
223 unsigned short nodeid;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700224};
225
226/*
227 * struct slab_rcu
228 *
229 * slab_destroy on a SLAB_DESTROY_BY_RCU cache uses this structure to
230 * arrange for kmem_freepages to be called via RCU. This is useful if
231 * we need to approach a kernel structure obliquely, from its address
232 * obtained without the usual locking. We can lock the structure to
233 * stabilize it and check it's still at the given address, only if we
234 * can be sure that the memory has not been meanwhile reused for some
235 * other kind of object (which our subsystem's lock might corrupt).
236 *
237 * rcu_read_lock before reading the address, then rcu_read_unlock after
238 * taking the spinlock within the structure expected at that address.
239 *
240 * We assume struct slab_rcu can overlay struct slab when destroying.
241 */
242struct slab_rcu {
Pekka Enbergb28a02d2006-01-08 01:00:37 -0800243 struct rcu_head head;
Pekka Enberg343e0d72006-02-01 03:05:50 -0800244 struct kmem_cache *cachep;
Pekka Enbergb28a02d2006-01-08 01:00:37 -0800245 void *addr;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700246};
247
248/*
249 * struct array_cache
250 *
Linus Torvalds1da177e2005-04-16 15:20:36 -0700251 * Purpose:
252 * - LIFO ordering, to hand out cache-warm objects from _alloc
253 * - reduce the number of linked list operations
254 * - reduce spinlock operations
255 *
256 * The limit is stored in the per-cpu structure to reduce the data cache
257 * footprint.
258 *
259 */
260struct array_cache {
261 unsigned int avail;
262 unsigned int limit;
263 unsigned int batchcount;
264 unsigned int touched;
Christoph Lametere498be72005-09-09 13:03:32 -0700265 spinlock_t lock;
Andrew Mortona737b3e2006-03-22 00:08:11 -0800266 void *entry[0]; /*
267 * Must have this definition in here for the proper
268 * alignment of array_cache. Also simplifies accessing
269 * the entries.
270 * [0] is for gcc 2.95. It should really be [].
271 */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700272};
273
Andrew Mortona737b3e2006-03-22 00:08:11 -0800274/*
275 * bootstrap: The caches do not work without cpuarrays anymore, but the
276 * cpuarrays are allocated from the generic caches...
Linus Torvalds1da177e2005-04-16 15:20:36 -0700277 */
278#define BOOT_CPUCACHE_ENTRIES 1
279struct arraycache_init {
280 struct array_cache cache;
Pekka Enbergb28a02d2006-01-08 01:00:37 -0800281 void *entries[BOOT_CPUCACHE_ENTRIES];
Linus Torvalds1da177e2005-04-16 15:20:36 -0700282};
283
284/*
Christoph Lametere498be72005-09-09 13:03:32 -0700285 * The slab lists for all objects.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700286 */
287struct kmem_list3 {
Pekka Enbergb28a02d2006-01-08 01:00:37 -0800288 struct list_head slabs_partial; /* partial list first, better asm code */
289 struct list_head slabs_full;
290 struct list_head slabs_free;
291 unsigned long free_objects;
Pekka Enbergb28a02d2006-01-08 01:00:37 -0800292 unsigned int free_limit;
Ravikiran G Thirumalai2e1217c2006-02-04 23:27:56 -0800293 unsigned int colour_next; /* Per-node cache coloring */
Pekka Enbergb28a02d2006-01-08 01:00:37 -0800294 spinlock_t list_lock;
295 struct array_cache *shared; /* shared per node */
296 struct array_cache **alien; /* on other nodes */
Christoph Lameter35386e32006-03-22 00:09:05 -0800297 unsigned long next_reap; /* updated without locking */
298 int free_touched; /* updated without locking */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700299};
300
Christoph Lametere498be72005-09-09 13:03:32 -0700301/*
302 * Need this for bootstrapping a per node allocator.
303 */
304#define NUM_INIT_LISTS (2 * MAX_NUMNODES + 1)
305struct kmem_list3 __initdata initkmem_list3[NUM_INIT_LISTS];
306#define CACHE_CACHE 0
307#define SIZE_AC 1
308#define SIZE_L3 (1 + MAX_NUMNODES)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700309
Christoph Lametere498be72005-09-09 13:03:32 -0700310/*
Andrew Mortona737b3e2006-03-22 00:08:11 -0800311 * This function must be completely optimized away if a constant is passed to
312 * it. Mostly the same as what is in linux/slab.h except it returns an index.
Christoph Lametere498be72005-09-09 13:03:32 -0700313 */
Ivan Kokshaysky7243cc02005-09-22 21:43:58 -0700314static __always_inline int index_of(const size_t size)
Christoph Lametere498be72005-09-09 13:03:32 -0700315{
Steven Rostedt5ec8a842006-02-01 03:05:44 -0800316 extern void __bad_size(void);
317
Christoph Lametere498be72005-09-09 13:03:32 -0700318 if (__builtin_constant_p(size)) {
319 int i = 0;
320
321#define CACHE(x) \
322 if (size <=x) \
323 return i; \
324 else \
325 i++;
326#include "linux/kmalloc_sizes.h"
327#undef CACHE
Steven Rostedt5ec8a842006-02-01 03:05:44 -0800328 __bad_size();
Ivan Kokshaysky7243cc02005-09-22 21:43:58 -0700329 } else
Steven Rostedt5ec8a842006-02-01 03:05:44 -0800330 __bad_size();
Christoph Lametere498be72005-09-09 13:03:32 -0700331 return 0;
332}
333
334#define INDEX_AC index_of(sizeof(struct arraycache_init))
335#define INDEX_L3 index_of(sizeof(struct kmem_list3))
336
Pekka Enberg5295a742006-02-01 03:05:48 -0800337static void kmem_list3_init(struct kmem_list3 *parent)
Christoph Lametere498be72005-09-09 13:03:32 -0700338{
339 INIT_LIST_HEAD(&parent->slabs_full);
340 INIT_LIST_HEAD(&parent->slabs_partial);
341 INIT_LIST_HEAD(&parent->slabs_free);
342 parent->shared = NULL;
343 parent->alien = NULL;
Ravikiran G Thirumalai2e1217c2006-02-04 23:27:56 -0800344 parent->colour_next = 0;
Christoph Lametere498be72005-09-09 13:03:32 -0700345 spin_lock_init(&parent->list_lock);
346 parent->free_objects = 0;
347 parent->free_touched = 0;
348}
349
Andrew Mortona737b3e2006-03-22 00:08:11 -0800350#define MAKE_LIST(cachep, listp, slab, nodeid) \
351 do { \
352 INIT_LIST_HEAD(listp); \
353 list_splice(&(cachep->nodelists[nodeid]->slab), listp); \
Christoph Lametere498be72005-09-09 13:03:32 -0700354 } while (0)
355
Andrew Mortona737b3e2006-03-22 00:08:11 -0800356#define MAKE_ALL_LISTS(cachep, ptr, nodeid) \
357 do { \
Christoph Lametere498be72005-09-09 13:03:32 -0700358 MAKE_LIST((cachep), (&(ptr)->slabs_full), slabs_full, nodeid); \
359 MAKE_LIST((cachep), (&(ptr)->slabs_partial), slabs_partial, nodeid); \
360 MAKE_LIST((cachep), (&(ptr)->slabs_free), slabs_free, nodeid); \
361 } while (0)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700362
363/*
Pekka Enberg343e0d72006-02-01 03:05:50 -0800364 * struct kmem_cache
Linus Torvalds1da177e2005-04-16 15:20:36 -0700365 *
366 * manages a cache.
367 */
Pekka Enbergb28a02d2006-01-08 01:00:37 -0800368
Pekka J Enberg2109a2d2005-11-07 00:58:01 -0800369struct kmem_cache {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700370/* 1) per-cpu data, touched during every alloc/free */
Pekka Enbergb28a02d2006-01-08 01:00:37 -0800371 struct array_cache *array[NR_CPUS];
Ravikiran G Thirumalaib5d8ca72006-03-22 00:08:12 -0800372/* 2) Cache tunables. Protected by cache_chain_mutex */
Pekka Enbergb28a02d2006-01-08 01:00:37 -0800373 unsigned int batchcount;
374 unsigned int limit;
375 unsigned int shared;
Ravikiran G Thirumalaib5d8ca72006-03-22 00:08:12 -0800376
Manfred Spraul3dafccf2006-02-01 03:05:42 -0800377 unsigned int buffer_size;
Ravikiran G Thirumalaib5d8ca72006-03-22 00:08:12 -0800378/* 3) touched by every alloc & free from the backend */
Pekka Enbergb28a02d2006-01-08 01:00:37 -0800379 struct kmem_list3 *nodelists[MAX_NUMNODES];
Ravikiran G Thirumalaib5d8ca72006-03-22 00:08:12 -0800380
Andrew Mortona737b3e2006-03-22 00:08:11 -0800381 unsigned int flags; /* constant flags */
382 unsigned int num; /* # of objs per slab */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700383
Ravikiran G Thirumalaib5d8ca72006-03-22 00:08:12 -0800384/* 4) cache_grow/shrink */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700385 /* order of pgs per slab (2^n) */
Pekka Enbergb28a02d2006-01-08 01:00:37 -0800386 unsigned int gfporder;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700387
388 /* force GFP flags, e.g. GFP_DMA */
Pekka Enbergb28a02d2006-01-08 01:00:37 -0800389 gfp_t gfpflags;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700390
Andrew Mortona737b3e2006-03-22 00:08:11 -0800391 size_t colour; /* cache colouring range */
Pekka Enbergb28a02d2006-01-08 01:00:37 -0800392 unsigned int colour_off; /* colour offset */
Pekka Enberg343e0d72006-02-01 03:05:50 -0800393 struct kmem_cache *slabp_cache;
Pekka Enbergb28a02d2006-01-08 01:00:37 -0800394 unsigned int slab_size;
Andrew Mortona737b3e2006-03-22 00:08:11 -0800395 unsigned int dflags; /* dynamic flags */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700396
397 /* constructor func */
Pekka Enberg343e0d72006-02-01 03:05:50 -0800398 void (*ctor) (void *, struct kmem_cache *, unsigned long);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700399
400 /* de-constructor func */
Pekka Enberg343e0d72006-02-01 03:05:50 -0800401 void (*dtor) (void *, struct kmem_cache *, unsigned long);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700402
Ravikiran G Thirumalaib5d8ca72006-03-22 00:08:12 -0800403/* 5) cache creation/removal */
Pekka Enbergb28a02d2006-01-08 01:00:37 -0800404 const char *name;
405 struct list_head next;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700406
Ravikiran G Thirumalaib5d8ca72006-03-22 00:08:12 -0800407/* 6) statistics */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700408#if STATS
Pekka Enbergb28a02d2006-01-08 01:00:37 -0800409 unsigned long num_active;
410 unsigned long num_allocations;
411 unsigned long high_mark;
412 unsigned long grown;
413 unsigned long reaped;
414 unsigned long errors;
415 unsigned long max_freeable;
416 unsigned long node_allocs;
417 unsigned long node_frees;
Ravikiran G Thirumalaifb7faf32006-04-10 22:52:54 -0700418 unsigned long node_overflow;
Pekka Enbergb28a02d2006-01-08 01:00:37 -0800419 atomic_t allochit;
420 atomic_t allocmiss;
421 atomic_t freehit;
422 atomic_t freemiss;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700423#endif
424#if DEBUG
Manfred Spraul3dafccf2006-02-01 03:05:42 -0800425 /*
426 * If debugging is enabled, then the allocator can add additional
427 * fields and/or padding to every object. buffer_size contains the total
428 * object size including these internal fields, the following two
429 * variables contain the offset to the user object and its size.
430 */
431 int obj_offset;
432 int obj_size;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700433#endif
434};
435
436#define CFLGS_OFF_SLAB (0x80000000UL)
437#define OFF_SLAB(x) ((x)->flags & CFLGS_OFF_SLAB)
438
439#define BATCHREFILL_LIMIT 16
Andrew Mortona737b3e2006-03-22 00:08:11 -0800440/*
441 * Optimization question: fewer reaps means less probability for unnessary
442 * cpucache drain/refill cycles.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700443 *
Adrian Bunkdc6f3f22005-11-08 16:44:08 +0100444 * OTOH the cpuarrays can contain lots of objects,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700445 * which could lock up otherwise freeable slabs.
446 */
447#define REAPTIMEOUT_CPUC (2*HZ)
448#define REAPTIMEOUT_LIST3 (4*HZ)
449
450#if STATS
451#define STATS_INC_ACTIVE(x) ((x)->num_active++)
452#define STATS_DEC_ACTIVE(x) ((x)->num_active--)
453#define STATS_INC_ALLOCED(x) ((x)->num_allocations++)
454#define STATS_INC_GROWN(x) ((x)->grown++)
455#define STATS_INC_REAPED(x) ((x)->reaped++)
Andrew Mortona737b3e2006-03-22 00:08:11 -0800456#define STATS_SET_HIGH(x) \
457 do { \
458 if ((x)->num_active > (x)->high_mark) \
459 (x)->high_mark = (x)->num_active; \
460 } while (0)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700461#define STATS_INC_ERR(x) ((x)->errors++)
462#define STATS_INC_NODEALLOCS(x) ((x)->node_allocs++)
Christoph Lametere498be72005-09-09 13:03:32 -0700463#define STATS_INC_NODEFREES(x) ((x)->node_frees++)
Ravikiran G Thirumalaifb7faf32006-04-10 22:52:54 -0700464#define STATS_INC_ACOVERFLOW(x) ((x)->node_overflow++)
Andrew Mortona737b3e2006-03-22 00:08:11 -0800465#define STATS_SET_FREEABLE(x, i) \
466 do { \
467 if ((x)->max_freeable < i) \
468 (x)->max_freeable = i; \
469 } while (0)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700470#define STATS_INC_ALLOCHIT(x) atomic_inc(&(x)->allochit)
471#define STATS_INC_ALLOCMISS(x) atomic_inc(&(x)->allocmiss)
472#define STATS_INC_FREEHIT(x) atomic_inc(&(x)->freehit)
473#define STATS_INC_FREEMISS(x) atomic_inc(&(x)->freemiss)
474#else
475#define STATS_INC_ACTIVE(x) do { } while (0)
476#define STATS_DEC_ACTIVE(x) do { } while (0)
477#define STATS_INC_ALLOCED(x) do { } while (0)
478#define STATS_INC_GROWN(x) do { } while (0)
479#define STATS_INC_REAPED(x) do { } while (0)
480#define STATS_SET_HIGH(x) do { } while (0)
481#define STATS_INC_ERR(x) do { } while (0)
482#define STATS_INC_NODEALLOCS(x) do { } while (0)
Christoph Lametere498be72005-09-09 13:03:32 -0700483#define STATS_INC_NODEFREES(x) do { } while (0)
Ravikiran G Thirumalaifb7faf32006-04-10 22:52:54 -0700484#define STATS_INC_ACOVERFLOW(x) do { } while (0)
Andrew Mortona737b3e2006-03-22 00:08:11 -0800485#define STATS_SET_FREEABLE(x, i) do { } while (0)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700486#define STATS_INC_ALLOCHIT(x) do { } while (0)
487#define STATS_INC_ALLOCMISS(x) do { } while (0)
488#define STATS_INC_FREEHIT(x) do { } while (0)
489#define STATS_INC_FREEMISS(x) do { } while (0)
490#endif
491
492#if DEBUG
Andrew Mortona737b3e2006-03-22 00:08:11 -0800493/*
494 * Magic nums for obj red zoning.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700495 * Placed in the first word before and the first word after an obj.
496 */
497#define RED_INACTIVE 0x5A2CF071UL /* when obj is inactive */
498#define RED_ACTIVE 0x170FC2A5UL /* when obj is active */
499
500/* ...and for poisoning */
501#define POISON_INUSE 0x5a /* for use-uninitialised poisoning */
502#define POISON_FREE 0x6b /* for use-after-free poisoning */
503#define POISON_END 0xa5 /* end-byte of poisoning */
504
Andrew Mortona737b3e2006-03-22 00:08:11 -0800505/*
506 * memory layout of objects:
Linus Torvalds1da177e2005-04-16 15:20:36 -0700507 * 0 : objp
Manfred Spraul3dafccf2006-02-01 03:05:42 -0800508 * 0 .. cachep->obj_offset - BYTES_PER_WORD - 1: padding. This ensures that
Linus Torvalds1da177e2005-04-16 15:20:36 -0700509 * the end of an object is aligned with the end of the real
510 * allocation. Catches writes behind the end of the allocation.
Manfred Spraul3dafccf2006-02-01 03:05:42 -0800511 * cachep->obj_offset - BYTES_PER_WORD .. cachep->obj_offset - 1:
Linus Torvalds1da177e2005-04-16 15:20:36 -0700512 * redzone word.
Manfred Spraul3dafccf2006-02-01 03:05:42 -0800513 * cachep->obj_offset: The real object.
514 * cachep->buffer_size - 2* BYTES_PER_WORD: redzone word [BYTES_PER_WORD long]
Andrew Mortona737b3e2006-03-22 00:08:11 -0800515 * cachep->buffer_size - 1* BYTES_PER_WORD: last caller address
516 * [BYTES_PER_WORD long]
Linus Torvalds1da177e2005-04-16 15:20:36 -0700517 */
Pekka Enberg343e0d72006-02-01 03:05:50 -0800518static int obj_offset(struct kmem_cache *cachep)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700519{
Manfred Spraul3dafccf2006-02-01 03:05:42 -0800520 return cachep->obj_offset;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700521}
522
Pekka Enberg343e0d72006-02-01 03:05:50 -0800523static int obj_size(struct kmem_cache *cachep)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700524{
Manfred Spraul3dafccf2006-02-01 03:05:42 -0800525 return cachep->obj_size;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700526}
527
Pekka Enberg343e0d72006-02-01 03:05:50 -0800528static unsigned long *dbg_redzone1(struct kmem_cache *cachep, void *objp)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700529{
530 BUG_ON(!(cachep->flags & SLAB_RED_ZONE));
Manfred Spraul3dafccf2006-02-01 03:05:42 -0800531 return (unsigned long*) (objp+obj_offset(cachep)-BYTES_PER_WORD);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700532}
533
Pekka Enberg343e0d72006-02-01 03:05:50 -0800534static unsigned long *dbg_redzone2(struct kmem_cache *cachep, void *objp)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700535{
536 BUG_ON(!(cachep->flags & SLAB_RED_ZONE));
537 if (cachep->flags & SLAB_STORE_USER)
Manfred Spraul3dafccf2006-02-01 03:05:42 -0800538 return (unsigned long *)(objp + cachep->buffer_size -
Pekka Enbergb28a02d2006-01-08 01:00:37 -0800539 2 * BYTES_PER_WORD);
Manfred Spraul3dafccf2006-02-01 03:05:42 -0800540 return (unsigned long *)(objp + cachep->buffer_size - BYTES_PER_WORD);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700541}
542
Pekka Enberg343e0d72006-02-01 03:05:50 -0800543static void **dbg_userword(struct kmem_cache *cachep, void *objp)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700544{
545 BUG_ON(!(cachep->flags & SLAB_STORE_USER));
Manfred Spraul3dafccf2006-02-01 03:05:42 -0800546 return (void **)(objp + cachep->buffer_size - BYTES_PER_WORD);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700547}
548
549#else
550
Manfred Spraul3dafccf2006-02-01 03:05:42 -0800551#define obj_offset(x) 0
552#define obj_size(cachep) (cachep->buffer_size)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700553#define dbg_redzone1(cachep, objp) ({BUG(); (unsigned long *)NULL;})
554#define dbg_redzone2(cachep, objp) ({BUG(); (unsigned long *)NULL;})
555#define dbg_userword(cachep, objp) ({BUG(); (void **)NULL;})
556
557#endif
558
559/*
Andrew Mortona737b3e2006-03-22 00:08:11 -0800560 * Maximum size of an obj (in 2^order pages) and absolute limit for the gfp
561 * order.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700562 */
563#if defined(CONFIG_LARGE_ALLOCS)
564#define MAX_OBJ_ORDER 13 /* up to 32Mb */
565#define MAX_GFP_ORDER 13 /* up to 32Mb */
566#elif defined(CONFIG_MMU)
567#define MAX_OBJ_ORDER 5 /* 32 pages */
568#define MAX_GFP_ORDER 5 /* 32 pages */
569#else
570#define MAX_OBJ_ORDER 8 /* up to 1Mb */
571#define MAX_GFP_ORDER 8 /* up to 1Mb */
572#endif
573
574/*
575 * Do not go above this order unless 0 objects fit into the slab.
576 */
577#define BREAK_GFP_ORDER_HI 1
578#define BREAK_GFP_ORDER_LO 0
579static int slab_break_gfp_order = BREAK_GFP_ORDER_LO;
580
Andrew Mortona737b3e2006-03-22 00:08:11 -0800581/*
582 * Functions for storing/retrieving the cachep and or slab from the page
583 * allocator. These are used to find the slab an obj belongs to. With kfree(),
584 * these are used to find the cache which an obj belongs to.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700585 */
Pekka Enberg065d41c2005-11-13 16:06:46 -0800586static inline void page_set_cache(struct page *page, struct kmem_cache *cache)
587{
588 page->lru.next = (struct list_head *)cache;
589}
590
591static inline struct kmem_cache *page_get_cache(struct page *page)
592{
Nick Piggin84097512006-03-22 00:08:34 -0800593 if (unlikely(PageCompound(page)))
594 page = (struct page *)page_private(page);
Pekka Enberg065d41c2005-11-13 16:06:46 -0800595 return (struct kmem_cache *)page->lru.next;
596}
597
598static inline void page_set_slab(struct page *page, struct slab *slab)
599{
600 page->lru.prev = (struct list_head *)slab;
601}
602
603static inline struct slab *page_get_slab(struct page *page)
604{
Nick Piggin84097512006-03-22 00:08:34 -0800605 if (unlikely(PageCompound(page)))
606 page = (struct page *)page_private(page);
Pekka Enberg065d41c2005-11-13 16:06:46 -0800607 return (struct slab *)page->lru.prev;
608}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700609
Pekka Enberg6ed5eb2212006-02-01 03:05:49 -0800610static inline struct kmem_cache *virt_to_cache(const void *obj)
611{
612 struct page *page = virt_to_page(obj);
613 return page_get_cache(page);
614}
615
616static inline struct slab *virt_to_slab(const void *obj)
617{
618 struct page *page = virt_to_page(obj);
619 return page_get_slab(page);
620}
621
Pekka Enberg8fea4e92006-03-22 00:08:10 -0800622static inline void *index_to_obj(struct kmem_cache *cache, struct slab *slab,
623 unsigned int idx)
624{
625 return slab->s_mem + cache->buffer_size * idx;
626}
627
628static inline unsigned int obj_to_index(struct kmem_cache *cache,
629 struct slab *slab, void *obj)
630{
631 return (unsigned)(obj - slab->s_mem) / cache->buffer_size;
632}
633
Andrew Mortona737b3e2006-03-22 00:08:11 -0800634/*
635 * These are the default caches for kmalloc. Custom caches can have other sizes.
636 */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700637struct cache_sizes malloc_sizes[] = {
638#define CACHE(x) { .cs_size = (x) },
639#include <linux/kmalloc_sizes.h>
640 CACHE(ULONG_MAX)
641#undef CACHE
642};
643EXPORT_SYMBOL(malloc_sizes);
644
645/* Must match cache_sizes above. Out of line to keep cache footprint low. */
646struct cache_names {
647 char *name;
648 char *name_dma;
649};
650
651static struct cache_names __initdata cache_names[] = {
652#define CACHE(x) { .name = "size-" #x, .name_dma = "size-" #x "(DMA)" },
653#include <linux/kmalloc_sizes.h>
Pekka Enbergb28a02d2006-01-08 01:00:37 -0800654 {NULL,}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700655#undef CACHE
656};
657
658static struct arraycache_init initarray_cache __initdata =
Pekka Enbergb28a02d2006-01-08 01:00:37 -0800659 { {0, BOOT_CPUCACHE_ENTRIES, 1, 0} };
Linus Torvalds1da177e2005-04-16 15:20:36 -0700660static struct arraycache_init initarray_generic =
Pekka Enbergb28a02d2006-01-08 01:00:37 -0800661 { {0, BOOT_CPUCACHE_ENTRIES, 1, 0} };
Linus Torvalds1da177e2005-04-16 15:20:36 -0700662
663/* internal cache of cache description objs */
Pekka Enberg343e0d72006-02-01 03:05:50 -0800664static struct kmem_cache cache_cache = {
Pekka Enbergb28a02d2006-01-08 01:00:37 -0800665 .batchcount = 1,
666 .limit = BOOT_CPUCACHE_ENTRIES,
667 .shared = 1,
Pekka Enberg343e0d72006-02-01 03:05:50 -0800668 .buffer_size = sizeof(struct kmem_cache),
Pekka Enbergb28a02d2006-01-08 01:00:37 -0800669 .name = "kmem_cache",
Linus Torvalds1da177e2005-04-16 15:20:36 -0700670#if DEBUG
Pekka Enberg343e0d72006-02-01 03:05:50 -0800671 .obj_size = sizeof(struct kmem_cache),
Linus Torvalds1da177e2005-04-16 15:20:36 -0700672#endif
673};
674
675/* Guard access to the cache-chain. */
Ingo Molnarfc0abb12006-01-18 17:42:33 -0800676static DEFINE_MUTEX(cache_chain_mutex);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700677static struct list_head cache_chain;
678
679/*
Andrew Mortona737b3e2006-03-22 00:08:11 -0800680 * vm_enough_memory() looks at this to determine how many slab-allocated pages
681 * are possibly freeable under pressure
Linus Torvalds1da177e2005-04-16 15:20:36 -0700682 *
683 * SLAB_RECLAIM_ACCOUNT turns this on per-slab
684 */
685atomic_t slab_reclaim_pages;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700686
687/*
688 * chicken and egg problem: delay the per-cpu array allocation
689 * until the general caches are up.
690 */
691static enum {
692 NONE,
Christoph Lametere498be72005-09-09 13:03:32 -0700693 PARTIAL_AC,
694 PARTIAL_L3,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700695 FULL
696} g_cpucache_up;
697
Mike Kravetz39d24e62006-05-15 09:44:13 -0700698/*
699 * used by boot code to determine if it can use slab based allocator
700 */
701int slab_is_available(void)
702{
703 return g_cpucache_up == FULL;
704}
705
Linus Torvalds1da177e2005-04-16 15:20:36 -0700706static DEFINE_PER_CPU(struct work_struct, reap_work);
707
Andrew Mortona737b3e2006-03-22 00:08:11 -0800708static void free_block(struct kmem_cache *cachep, void **objpp, int len,
709 int node);
Pekka Enberg343e0d72006-02-01 03:05:50 -0800710static void enable_cpucache(struct kmem_cache *cachep);
Pekka Enbergb28a02d2006-01-08 01:00:37 -0800711static void cache_reap(void *unused);
Pekka Enberg343e0d72006-02-01 03:05:50 -0800712static int __node_shrink(struct kmem_cache *cachep, int node);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700713
Pekka Enberg343e0d72006-02-01 03:05:50 -0800714static inline struct array_cache *cpu_cache_get(struct kmem_cache *cachep)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700715{
716 return cachep->array[smp_processor_id()];
717}
718
Andrew Mortona737b3e2006-03-22 00:08:11 -0800719static inline struct kmem_cache *__find_general_cachep(size_t size,
720 gfp_t gfpflags)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700721{
722 struct cache_sizes *csizep = malloc_sizes;
723
724#if DEBUG
725 /* This happens if someone tries to call
Pekka Enbergb28a02d2006-01-08 01:00:37 -0800726 * kmem_cache_create(), or __kmalloc(), before
727 * the generic caches are initialized.
728 */
Alok Katariac7e43c72005-09-14 12:17:53 -0700729 BUG_ON(malloc_sizes[INDEX_AC].cs_cachep == NULL);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700730#endif
731 while (size > csizep->cs_size)
732 csizep++;
733
734 /*
Martin Hicks0abf40c2005-09-03 15:54:54 -0700735 * Really subtle: The last entry with cs->cs_size==ULONG_MAX
Linus Torvalds1da177e2005-04-16 15:20:36 -0700736 * has cs_{dma,}cachep==NULL. Thus no special case
737 * for large kmalloc calls required.
738 */
739 if (unlikely(gfpflags & GFP_DMA))
740 return csizep->cs_dmacachep;
741 return csizep->cs_cachep;
742}
743
Pekka Enberg343e0d72006-02-01 03:05:50 -0800744struct kmem_cache *kmem_find_general_cachep(size_t size, gfp_t gfpflags)
Manfred Spraul97e2bde2005-05-01 08:58:38 -0700745{
746 return __find_general_cachep(size, gfpflags);
747}
748EXPORT_SYMBOL(kmem_find_general_cachep);
749
Steven Rostedtfbaccac2006-02-01 03:05:45 -0800750static size_t slab_mgmt_size(size_t nr_objs, size_t align)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700751{
Steven Rostedtfbaccac2006-02-01 03:05:45 -0800752 return ALIGN(sizeof(struct slab)+nr_objs*sizeof(kmem_bufctl_t), align);
753}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700754
Andrew Mortona737b3e2006-03-22 00:08:11 -0800755/*
756 * Calculate the number of objects and left-over bytes for a given buffer size.
757 */
Steven Rostedtfbaccac2006-02-01 03:05:45 -0800758static void cache_estimate(unsigned long gfporder, size_t buffer_size,
759 size_t align, int flags, size_t *left_over,
760 unsigned int *num)
761{
762 int nr_objs;
763 size_t mgmt_size;
764 size_t slab_size = PAGE_SIZE << gfporder;
765
766 /*
767 * The slab management structure can be either off the slab or
768 * on it. For the latter case, the memory allocated for a
769 * slab is used for:
770 *
771 * - The struct slab
772 * - One kmem_bufctl_t for each object
773 * - Padding to respect alignment of @align
774 * - @buffer_size bytes for each object
775 *
776 * If the slab management structure is off the slab, then the
777 * alignment will already be calculated into the size. Because
778 * the slabs are all pages aligned, the objects will be at the
779 * correct alignment when allocated.
780 */
781 if (flags & CFLGS_OFF_SLAB) {
782 mgmt_size = 0;
783 nr_objs = slab_size / buffer_size;
784
785 if (nr_objs > SLAB_LIMIT)
786 nr_objs = SLAB_LIMIT;
787 } else {
788 /*
789 * Ignore padding for the initial guess. The padding
790 * is at most @align-1 bytes, and @buffer_size is at
791 * least @align. In the worst case, this result will
792 * be one greater than the number of objects that fit
793 * into the memory allocation when taking the padding
794 * into account.
795 */
796 nr_objs = (slab_size - sizeof(struct slab)) /
797 (buffer_size + sizeof(kmem_bufctl_t));
798
799 /*
800 * This calculated number will be either the right
801 * amount, or one greater than what we want.
802 */
803 if (slab_mgmt_size(nr_objs, align) + nr_objs*buffer_size
804 > slab_size)
805 nr_objs--;
806
807 if (nr_objs > SLAB_LIMIT)
808 nr_objs = SLAB_LIMIT;
809
810 mgmt_size = slab_mgmt_size(nr_objs, align);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700811 }
Steven Rostedtfbaccac2006-02-01 03:05:45 -0800812 *num = nr_objs;
813 *left_over = slab_size - nr_objs*buffer_size - mgmt_size;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700814}
815
816#define slab_error(cachep, msg) __slab_error(__FUNCTION__, cachep, msg)
817
Andrew Mortona737b3e2006-03-22 00:08:11 -0800818static void __slab_error(const char *function, struct kmem_cache *cachep,
819 char *msg)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700820{
821 printk(KERN_ERR "slab error in %s(): cache `%s': %s\n",
Pekka Enbergb28a02d2006-01-08 01:00:37 -0800822 function, cachep->name, msg);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700823 dump_stack();
824}
825
Christoph Lameter8fce4d82006-03-09 17:33:54 -0800826#ifdef CONFIG_NUMA
827/*
828 * Special reaping functions for NUMA systems called from cache_reap().
829 * These take care of doing round robin flushing of alien caches (containing
830 * objects freed on different nodes from which they were allocated) and the
831 * flushing of remote pcps by calling drain_node_pages.
832 */
833static DEFINE_PER_CPU(unsigned long, reap_node);
834
835static void init_reap_node(int cpu)
836{
837 int node;
838
839 node = next_node(cpu_to_node(cpu), node_online_map);
840 if (node == MAX_NUMNODES)
Paul Jackson442295c2006-03-22 00:09:11 -0800841 node = first_node(node_online_map);
Christoph Lameter8fce4d82006-03-09 17:33:54 -0800842
843 __get_cpu_var(reap_node) = node;
844}
845
846static void next_reap_node(void)
847{
848 int node = __get_cpu_var(reap_node);
849
850 /*
851 * Also drain per cpu pages on remote zones
852 */
853 if (node != numa_node_id())
854 drain_node_pages(node);
855
856 node = next_node(node, node_online_map);
857 if (unlikely(node >= MAX_NUMNODES))
858 node = first_node(node_online_map);
859 __get_cpu_var(reap_node) = node;
860}
861
862#else
863#define init_reap_node(cpu) do { } while (0)
864#define next_reap_node(void) do { } while (0)
865#endif
866
Linus Torvalds1da177e2005-04-16 15:20:36 -0700867/*
868 * Initiate the reap timer running on the target CPU. We run at around 1 to 2Hz
869 * via the workqueue/eventd.
870 * Add the CPU number into the expiration time to minimize the possibility of
871 * the CPUs getting into lockstep and contending for the global cache chain
872 * lock.
873 */
874static void __devinit start_cpu_timer(int cpu)
875{
876 struct work_struct *reap_work = &per_cpu(reap_work, cpu);
877
878 /*
879 * When this gets called from do_initcalls via cpucache_init(),
880 * init_workqueues() has already run, so keventd will be setup
881 * at that time.
882 */
883 if (keventd_up() && reap_work->func == NULL) {
Christoph Lameter8fce4d82006-03-09 17:33:54 -0800884 init_reap_node(cpu);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700885 INIT_WORK(reap_work, cache_reap, NULL);
886 schedule_delayed_work_on(cpu, reap_work, HZ + 3 * cpu);
887 }
888}
889
Christoph Lametere498be72005-09-09 13:03:32 -0700890static struct array_cache *alloc_arraycache(int node, int entries,
Pekka Enbergb28a02d2006-01-08 01:00:37 -0800891 int batchcount)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700892{
Pekka Enbergb28a02d2006-01-08 01:00:37 -0800893 int memsize = sizeof(void *) * entries + sizeof(struct array_cache);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700894 struct array_cache *nc = NULL;
895
Christoph Lametere498be72005-09-09 13:03:32 -0700896 nc = kmalloc_node(memsize, GFP_KERNEL, node);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700897 if (nc) {
898 nc->avail = 0;
899 nc->limit = entries;
900 nc->batchcount = batchcount;
901 nc->touched = 0;
Christoph Lametere498be72005-09-09 13:03:32 -0700902 spin_lock_init(&nc->lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700903 }
904 return nc;
905}
906
Christoph Lameter3ded1752006-03-25 03:06:44 -0800907/*
908 * Transfer objects in one arraycache to another.
909 * Locking must be handled by the caller.
910 *
911 * Return the number of entries transferred.
912 */
913static int transfer_objects(struct array_cache *to,
914 struct array_cache *from, unsigned int max)
915{
916 /* Figure out how many entries to transfer */
917 int nr = min(min(from->avail, max), to->limit - to->avail);
918
919 if (!nr)
920 return 0;
921
922 memcpy(to->entry + to->avail, from->entry + from->avail -nr,
923 sizeof(void *) *nr);
924
925 from->avail -= nr;
926 to->avail += nr;
927 to->touched = 1;
928 return nr;
929}
930
Christoph Lametere498be72005-09-09 13:03:32 -0700931#ifdef CONFIG_NUMA
Pekka Enberg343e0d72006-02-01 03:05:50 -0800932static void *__cache_alloc_node(struct kmem_cache *, gfp_t, int);
Paul Jacksonc61afb12006-03-24 03:16:08 -0800933static void *alternate_node_alloc(struct kmem_cache *, gfp_t);
Christoph Lameterdc85da12006-01-18 17:42:36 -0800934
Pekka Enberg5295a742006-02-01 03:05:48 -0800935static struct array_cache **alloc_alien_cache(int node, int limit)
Christoph Lametere498be72005-09-09 13:03:32 -0700936{
937 struct array_cache **ac_ptr;
Pekka Enbergb28a02d2006-01-08 01:00:37 -0800938 int memsize = sizeof(void *) * MAX_NUMNODES;
Christoph Lametere498be72005-09-09 13:03:32 -0700939 int i;
940
941 if (limit > 1)
942 limit = 12;
943 ac_ptr = kmalloc_node(memsize, GFP_KERNEL, node);
944 if (ac_ptr) {
945 for_each_node(i) {
946 if (i == node || !node_online(i)) {
947 ac_ptr[i] = NULL;
948 continue;
949 }
950 ac_ptr[i] = alloc_arraycache(node, limit, 0xbaadf00d);
951 if (!ac_ptr[i]) {
Pekka Enbergb28a02d2006-01-08 01:00:37 -0800952 for (i--; i <= 0; i--)
Christoph Lametere498be72005-09-09 13:03:32 -0700953 kfree(ac_ptr[i]);
954 kfree(ac_ptr);
955 return NULL;
956 }
957 }
958 }
959 return ac_ptr;
960}
961
Pekka Enberg5295a742006-02-01 03:05:48 -0800962static void free_alien_cache(struct array_cache **ac_ptr)
Christoph Lametere498be72005-09-09 13:03:32 -0700963{
964 int i;
965
966 if (!ac_ptr)
967 return;
Christoph Lametere498be72005-09-09 13:03:32 -0700968 for_each_node(i)
Pekka Enbergb28a02d2006-01-08 01:00:37 -0800969 kfree(ac_ptr[i]);
Christoph Lametere498be72005-09-09 13:03:32 -0700970 kfree(ac_ptr);
971}
972
Pekka Enberg343e0d72006-02-01 03:05:50 -0800973static void __drain_alien_cache(struct kmem_cache *cachep,
Pekka Enberg5295a742006-02-01 03:05:48 -0800974 struct array_cache *ac, int node)
Christoph Lametere498be72005-09-09 13:03:32 -0700975{
976 struct kmem_list3 *rl3 = cachep->nodelists[node];
977
978 if (ac->avail) {
979 spin_lock(&rl3->list_lock);
Christoph Lametere00946f2006-03-25 03:06:45 -0800980 /*
981 * Stuff objects into the remote nodes shared array first.
982 * That way we could avoid the overhead of putting the objects
983 * into the free lists and getting them back later.
984 */
shin, jacob693f7d32006-04-28 10:54:37 -0500985 if (rl3->shared)
986 transfer_objects(rl3->shared, ac, ac->limit);
Christoph Lametere00946f2006-03-25 03:06:45 -0800987
Christoph Lameterff694162005-09-22 21:44:02 -0700988 free_block(cachep, ac->entry, ac->avail, node);
Christoph Lametere498be72005-09-09 13:03:32 -0700989 ac->avail = 0;
990 spin_unlock(&rl3->list_lock);
991 }
992}
993
Christoph Lameter8fce4d82006-03-09 17:33:54 -0800994/*
995 * Called from cache_reap() to regularly drain alien caches round robin.
996 */
997static void reap_alien(struct kmem_cache *cachep, struct kmem_list3 *l3)
998{
999 int node = __get_cpu_var(reap_node);
1000
1001 if (l3->alien) {
1002 struct array_cache *ac = l3->alien[node];
Christoph Lametere00946f2006-03-25 03:06:45 -08001003
1004 if (ac && ac->avail && spin_trylock_irq(&ac->lock)) {
Christoph Lameter8fce4d82006-03-09 17:33:54 -08001005 __drain_alien_cache(cachep, ac, node);
1006 spin_unlock_irq(&ac->lock);
1007 }
1008 }
1009}
1010
Andrew Mortona737b3e2006-03-22 00:08:11 -08001011static void drain_alien_cache(struct kmem_cache *cachep,
1012 struct array_cache **alien)
Christoph Lametere498be72005-09-09 13:03:32 -07001013{
Pekka Enbergb28a02d2006-01-08 01:00:37 -08001014 int i = 0;
Christoph Lametere498be72005-09-09 13:03:32 -07001015 struct array_cache *ac;
1016 unsigned long flags;
1017
1018 for_each_online_node(i) {
Ravikiran G Thirumalai4484ebf2006-02-04 23:27:59 -08001019 ac = alien[i];
Christoph Lametere498be72005-09-09 13:03:32 -07001020 if (ac) {
1021 spin_lock_irqsave(&ac->lock, flags);
1022 __drain_alien_cache(cachep, ac, i);
1023 spin_unlock_irqrestore(&ac->lock, flags);
1024 }
1025 }
1026}
1027#else
Linus Torvalds7a21ef62006-02-05 11:26:38 -08001028
Ravikiran G Thirumalai4484ebf2006-02-04 23:27:59 -08001029#define drain_alien_cache(cachep, alien) do { } while (0)
Christoph Lameter8fce4d82006-03-09 17:33:54 -08001030#define reap_alien(cachep, l3) do { } while (0)
Ravikiran G Thirumalai4484ebf2006-02-04 23:27:59 -08001031
Linus Torvalds7a21ef62006-02-05 11:26:38 -08001032static inline struct array_cache **alloc_alien_cache(int node, int limit)
1033{
1034 return (struct array_cache **) 0x01020304ul;
1035}
1036
Ravikiran G Thirumalai4484ebf2006-02-04 23:27:59 -08001037static inline void free_alien_cache(struct array_cache **ac_ptr)
1038{
1039}
Linus Torvalds7a21ef62006-02-05 11:26:38 -08001040
Christoph Lametere498be72005-09-09 13:03:32 -07001041#endif
1042
Chandra Seetharaman83d722f2006-04-24 19:35:21 -07001043static int cpuup_callback(struct notifier_block *nfb,
Pekka Enbergb28a02d2006-01-08 01:00:37 -08001044 unsigned long action, void *hcpu)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001045{
1046 long cpu = (long)hcpu;
Pekka Enberg343e0d72006-02-01 03:05:50 -08001047 struct kmem_cache *cachep;
Christoph Lametere498be72005-09-09 13:03:32 -07001048 struct kmem_list3 *l3 = NULL;
1049 int node = cpu_to_node(cpu);
1050 int memsize = sizeof(struct kmem_list3);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001051
1052 switch (action) {
1053 case CPU_UP_PREPARE:
Ingo Molnarfc0abb12006-01-18 17:42:33 -08001054 mutex_lock(&cache_chain_mutex);
Andrew Mortona737b3e2006-03-22 00:08:11 -08001055 /*
1056 * We need to do this right in the beginning since
Christoph Lametere498be72005-09-09 13:03:32 -07001057 * alloc_arraycache's are going to use this list.
1058 * kmalloc_node allows us to add the slab to the right
1059 * kmem_list3 and not this cpu's kmem_list3
1060 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001061
Christoph Lametere498be72005-09-09 13:03:32 -07001062 list_for_each_entry(cachep, &cache_chain, next) {
Andrew Mortona737b3e2006-03-22 00:08:11 -08001063 /*
1064 * Set up the size64 kmemlist for cpu before we can
Christoph Lametere498be72005-09-09 13:03:32 -07001065 * begin anything. Make sure some other cpu on this
1066 * node has not already allocated this
1067 */
1068 if (!cachep->nodelists[node]) {
Andrew Mortona737b3e2006-03-22 00:08:11 -08001069 l3 = kmalloc_node(memsize, GFP_KERNEL, node);
1070 if (!l3)
Christoph Lametere498be72005-09-09 13:03:32 -07001071 goto bad;
1072 kmem_list3_init(l3);
1073 l3->next_reap = jiffies + REAPTIMEOUT_LIST3 +
Pekka Enbergb28a02d2006-01-08 01:00:37 -08001074 ((unsigned long)cachep) % REAPTIMEOUT_LIST3;
Christoph Lametere498be72005-09-09 13:03:32 -07001075
Ravikiran G Thirumalai4484ebf2006-02-04 23:27:59 -08001076 /*
1077 * The l3s don't come and go as CPUs come and
1078 * go. cache_chain_mutex is sufficient
1079 * protection here.
1080 */
Christoph Lametere498be72005-09-09 13:03:32 -07001081 cachep->nodelists[node] = l3;
1082 }
1083
1084 spin_lock_irq(&cachep->nodelists[node]->list_lock);
1085 cachep->nodelists[node]->free_limit =
Andrew Mortona737b3e2006-03-22 00:08:11 -08001086 (1 + nr_cpus_node(node)) *
1087 cachep->batchcount + cachep->num;
Christoph Lametere498be72005-09-09 13:03:32 -07001088 spin_unlock_irq(&cachep->nodelists[node]->list_lock);
1089 }
1090
Andrew Mortona737b3e2006-03-22 00:08:11 -08001091 /*
1092 * Now we can go ahead with allocating the shared arrays and
1093 * array caches
1094 */
Christoph Lametere498be72005-09-09 13:03:32 -07001095 list_for_each_entry(cachep, &cache_chain, next) {
Tobias Klausercd105df2006-01-08 01:00:59 -08001096 struct array_cache *nc;
Ravikiran G Thirumalai4484ebf2006-02-04 23:27:59 -08001097 struct array_cache *shared;
1098 struct array_cache **alien;
Tobias Klausercd105df2006-01-08 01:00:59 -08001099
Christoph Lametere498be72005-09-09 13:03:32 -07001100 nc = alloc_arraycache(node, cachep->limit,
Ravikiran G Thirumalai4484ebf2006-02-04 23:27:59 -08001101 cachep->batchcount);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001102 if (!nc)
1103 goto bad;
Ravikiran G Thirumalai4484ebf2006-02-04 23:27:59 -08001104 shared = alloc_arraycache(node,
1105 cachep->shared * cachep->batchcount,
1106 0xbaadf00d);
1107 if (!shared)
1108 goto bad;
Linus Torvalds7a21ef62006-02-05 11:26:38 -08001109
Ravikiran G Thirumalai4484ebf2006-02-04 23:27:59 -08001110 alien = alloc_alien_cache(node, cachep->limit);
1111 if (!alien)
1112 goto bad;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001113 cachep->array[cpu] = nc;
Christoph Lametere498be72005-09-09 13:03:32 -07001114 l3 = cachep->nodelists[node];
1115 BUG_ON(!l3);
Christoph Lametere498be72005-09-09 13:03:32 -07001116
Ravikiran G Thirumalai4484ebf2006-02-04 23:27:59 -08001117 spin_lock_irq(&l3->list_lock);
1118 if (!l3->shared) {
1119 /*
1120 * We are serialised from CPU_DEAD or
1121 * CPU_UP_CANCELLED by the cpucontrol lock
1122 */
1123 l3->shared = shared;
1124 shared = NULL;
Christoph Lametere498be72005-09-09 13:03:32 -07001125 }
Ravikiran G Thirumalai4484ebf2006-02-04 23:27:59 -08001126#ifdef CONFIG_NUMA
1127 if (!l3->alien) {
1128 l3->alien = alien;
1129 alien = NULL;
1130 }
1131#endif
1132 spin_unlock_irq(&l3->list_lock);
Ravikiran G Thirumalai4484ebf2006-02-04 23:27:59 -08001133 kfree(shared);
1134 free_alien_cache(alien);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001135 }
Ingo Molnarfc0abb12006-01-18 17:42:33 -08001136 mutex_unlock(&cache_chain_mutex);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001137 break;
1138 case CPU_ONLINE:
1139 start_cpu_timer(cpu);
1140 break;
1141#ifdef CONFIG_HOTPLUG_CPU
1142 case CPU_DEAD:
Ravikiran G Thirumalai4484ebf2006-02-04 23:27:59 -08001143 /*
1144 * Even if all the cpus of a node are down, we don't free the
1145 * kmem_list3 of any cache. This to avoid a race between
1146 * cpu_down, and a kmalloc allocation from another cpu for
1147 * memory from the node of the cpu going down. The list3
1148 * structure is usually allocated from kmem_cache_create() and
1149 * gets destroyed at kmem_cache_destroy().
1150 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001151 /* fall thru */
1152 case CPU_UP_CANCELED:
Ingo Molnarfc0abb12006-01-18 17:42:33 -08001153 mutex_lock(&cache_chain_mutex);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001154 list_for_each_entry(cachep, &cache_chain, next) {
1155 struct array_cache *nc;
Ravikiran G Thirumalai4484ebf2006-02-04 23:27:59 -08001156 struct array_cache *shared;
1157 struct array_cache **alien;
Christoph Lametere498be72005-09-09 13:03:32 -07001158 cpumask_t mask;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001159
Christoph Lametere498be72005-09-09 13:03:32 -07001160 mask = node_to_cpumask(node);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001161 /* cpu is dead; no one can alloc from it. */
1162 nc = cachep->array[cpu];
1163 cachep->array[cpu] = NULL;
Christoph Lametere498be72005-09-09 13:03:32 -07001164 l3 = cachep->nodelists[node];
1165
1166 if (!l3)
Ravikiran G Thirumalai4484ebf2006-02-04 23:27:59 -08001167 goto free_array_cache;
Christoph Lametere498be72005-09-09 13:03:32 -07001168
Ravikiran G Thirumalaica3b9b92006-02-04 23:27:58 -08001169 spin_lock_irq(&l3->list_lock);
Christoph Lametere498be72005-09-09 13:03:32 -07001170
1171 /* Free limit for this kmem_list3 */
1172 l3->free_limit -= cachep->batchcount;
1173 if (nc)
Christoph Lameterff694162005-09-22 21:44:02 -07001174 free_block(cachep, nc->entry, nc->avail, node);
Christoph Lametere498be72005-09-09 13:03:32 -07001175
1176 if (!cpus_empty(mask)) {
Ravikiran G Thirumalaica3b9b92006-02-04 23:27:58 -08001177 spin_unlock_irq(&l3->list_lock);
Ravikiran G Thirumalai4484ebf2006-02-04 23:27:59 -08001178 goto free_array_cache;
Pekka Enbergb28a02d2006-01-08 01:00:37 -08001179 }
Christoph Lametere498be72005-09-09 13:03:32 -07001180
Ravikiran G Thirumalai4484ebf2006-02-04 23:27:59 -08001181 shared = l3->shared;
1182 if (shared) {
Christoph Lametere498be72005-09-09 13:03:32 -07001183 free_block(cachep, l3->shared->entry,
Pekka Enbergb28a02d2006-01-08 01:00:37 -08001184 l3->shared->avail, node);
Christoph Lametere498be72005-09-09 13:03:32 -07001185 l3->shared = NULL;
1186 }
Christoph Lametere498be72005-09-09 13:03:32 -07001187
Ravikiran G Thirumalai4484ebf2006-02-04 23:27:59 -08001188 alien = l3->alien;
1189 l3->alien = NULL;
1190
1191 spin_unlock_irq(&l3->list_lock);
1192
1193 kfree(shared);
1194 if (alien) {
1195 drain_alien_cache(cachep, alien);
1196 free_alien_cache(alien);
Christoph Lametere498be72005-09-09 13:03:32 -07001197 }
Ravikiran G Thirumalai4484ebf2006-02-04 23:27:59 -08001198free_array_cache:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001199 kfree(nc);
1200 }
Ravikiran G Thirumalai4484ebf2006-02-04 23:27:59 -08001201 /*
1202 * In the previous loop, all the objects were freed to
1203 * the respective cache's slabs, now we can go ahead and
1204 * shrink each nodelist to its limit.
1205 */
1206 list_for_each_entry(cachep, &cache_chain, next) {
1207 l3 = cachep->nodelists[node];
1208 if (!l3)
1209 continue;
1210 spin_lock_irq(&l3->list_lock);
1211 /* free slabs belonging to this node */
1212 __node_shrink(cachep, node);
1213 spin_unlock_irq(&l3->list_lock);
1214 }
Ingo Molnarfc0abb12006-01-18 17:42:33 -08001215 mutex_unlock(&cache_chain_mutex);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001216 break;
1217#endif
1218 }
1219 return NOTIFY_OK;
Andrew Mortona737b3e2006-03-22 00:08:11 -08001220bad:
Ingo Molnarfc0abb12006-01-18 17:42:33 -08001221 mutex_unlock(&cache_chain_mutex);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001222 return NOTIFY_BAD;
1223}
1224
1225static struct notifier_block cpucache_notifier = { &cpuup_callback, NULL, 0 };
1226
Christoph Lametere498be72005-09-09 13:03:32 -07001227/*
1228 * swap the static kmem_list3 with kmalloced memory
1229 */
Andrew Mortona737b3e2006-03-22 00:08:11 -08001230static void init_list(struct kmem_cache *cachep, struct kmem_list3 *list,
1231 int nodeid)
Christoph Lametere498be72005-09-09 13:03:32 -07001232{
1233 struct kmem_list3 *ptr;
1234
1235 BUG_ON(cachep->nodelists[nodeid] != list);
1236 ptr = kmalloc_node(sizeof(struct kmem_list3), GFP_KERNEL, nodeid);
1237 BUG_ON(!ptr);
1238
1239 local_irq_disable();
1240 memcpy(ptr, list, sizeof(struct kmem_list3));
1241 MAKE_ALL_LISTS(cachep, ptr, nodeid);
1242 cachep->nodelists[nodeid] = ptr;
1243 local_irq_enable();
1244}
1245
Andrew Mortona737b3e2006-03-22 00:08:11 -08001246/*
1247 * Initialisation. Called after the page allocator have been initialised and
1248 * before smp_init().
Linus Torvalds1da177e2005-04-16 15:20:36 -07001249 */
1250void __init kmem_cache_init(void)
1251{
1252 size_t left_over;
1253 struct cache_sizes *sizes;
1254 struct cache_names *names;
Christoph Lametere498be72005-09-09 13:03:32 -07001255 int i;
Jack Steiner07ed76b2006-03-07 21:55:46 -08001256 int order;
Christoph Lametere498be72005-09-09 13:03:32 -07001257
1258 for (i = 0; i < NUM_INIT_LISTS; i++) {
1259 kmem_list3_init(&initkmem_list3[i]);
1260 if (i < MAX_NUMNODES)
1261 cache_cache.nodelists[i] = NULL;
1262 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001263
1264 /*
1265 * Fragmentation resistance on low memory - only use bigger
1266 * page orders on machines with more than 32MB of memory.
1267 */
1268 if (num_physpages > (32 << 20) >> PAGE_SHIFT)
1269 slab_break_gfp_order = BREAK_GFP_ORDER_HI;
1270
Linus Torvalds1da177e2005-04-16 15:20:36 -07001271 /* Bootstrap is tricky, because several objects are allocated
1272 * from caches that do not exist yet:
Andrew Mortona737b3e2006-03-22 00:08:11 -08001273 * 1) initialize the cache_cache cache: it contains the struct
1274 * kmem_cache structures of all caches, except cache_cache itself:
1275 * cache_cache is statically allocated.
Christoph Lametere498be72005-09-09 13:03:32 -07001276 * Initially an __init data area is used for the head array and the
1277 * kmem_list3 structures, it's replaced with a kmalloc allocated
1278 * array at the end of the bootstrap.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001279 * 2) Create the first kmalloc cache.
Pekka Enberg343e0d72006-02-01 03:05:50 -08001280 * The struct kmem_cache for the new cache is allocated normally.
Christoph Lametere498be72005-09-09 13:03:32 -07001281 * An __init data area is used for the head array.
1282 * 3) Create the remaining kmalloc caches, with minimally sized
1283 * head arrays.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001284 * 4) Replace the __init data head arrays for cache_cache and the first
1285 * kmalloc cache with kmalloc allocated arrays.
Christoph Lametere498be72005-09-09 13:03:32 -07001286 * 5) Replace the __init data for kmem_list3 for cache_cache and
1287 * the other cache's with kmalloc allocated memory.
1288 * 6) Resize the head arrays of the kmalloc caches to their final sizes.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001289 */
1290
1291 /* 1) create the cache_cache */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001292 INIT_LIST_HEAD(&cache_chain);
1293 list_add(&cache_cache.next, &cache_chain);
1294 cache_cache.colour_off = cache_line_size();
1295 cache_cache.array[smp_processor_id()] = &initarray_cache.cache;
Christoph Lametere498be72005-09-09 13:03:32 -07001296 cache_cache.nodelists[numa_node_id()] = &initkmem_list3[CACHE_CACHE];
Linus Torvalds1da177e2005-04-16 15:20:36 -07001297
Andrew Mortona737b3e2006-03-22 00:08:11 -08001298 cache_cache.buffer_size = ALIGN(cache_cache.buffer_size,
1299 cache_line_size());
Linus Torvalds1da177e2005-04-16 15:20:36 -07001300
Jack Steiner07ed76b2006-03-07 21:55:46 -08001301 for (order = 0; order < MAX_ORDER; order++) {
1302 cache_estimate(order, cache_cache.buffer_size,
1303 cache_line_size(), 0, &left_over, &cache_cache.num);
1304 if (cache_cache.num)
1305 break;
1306 }
Eric Sesterhenn40094fa2006-04-02 13:49:25 +02001307 BUG_ON(!cache_cache.num);
Jack Steiner07ed76b2006-03-07 21:55:46 -08001308 cache_cache.gfporder = order;
Pekka Enbergb28a02d2006-01-08 01:00:37 -08001309 cache_cache.colour = left_over / cache_cache.colour_off;
Pekka Enbergb28a02d2006-01-08 01:00:37 -08001310 cache_cache.slab_size = ALIGN(cache_cache.num * sizeof(kmem_bufctl_t) +
1311 sizeof(struct slab), cache_line_size());
Linus Torvalds1da177e2005-04-16 15:20:36 -07001312
1313 /* 2+3) create the kmalloc caches */
1314 sizes = malloc_sizes;
1315 names = cache_names;
1316
Andrew Mortona737b3e2006-03-22 00:08:11 -08001317 /*
1318 * Initialize the caches that provide memory for the array cache and the
1319 * kmem_list3 structures first. Without this, further allocations will
1320 * bug.
Christoph Lametere498be72005-09-09 13:03:32 -07001321 */
1322
1323 sizes[INDEX_AC].cs_cachep = kmem_cache_create(names[INDEX_AC].name,
Andrew Mortona737b3e2006-03-22 00:08:11 -08001324 sizes[INDEX_AC].cs_size,
1325 ARCH_KMALLOC_MINALIGN,
1326 ARCH_KMALLOC_FLAGS|SLAB_PANIC,
1327 NULL, NULL);
Christoph Lametere498be72005-09-09 13:03:32 -07001328
Andrew Mortona737b3e2006-03-22 00:08:11 -08001329 if (INDEX_AC != INDEX_L3) {
Christoph Lametere498be72005-09-09 13:03:32 -07001330 sizes[INDEX_L3].cs_cachep =
Andrew Mortona737b3e2006-03-22 00:08:11 -08001331 kmem_cache_create(names[INDEX_L3].name,
1332 sizes[INDEX_L3].cs_size,
1333 ARCH_KMALLOC_MINALIGN,
1334 ARCH_KMALLOC_FLAGS|SLAB_PANIC,
1335 NULL, NULL);
1336 }
Christoph Lametere498be72005-09-09 13:03:32 -07001337
Linus Torvalds1da177e2005-04-16 15:20:36 -07001338 while (sizes->cs_size != ULONG_MAX) {
Christoph Lametere498be72005-09-09 13:03:32 -07001339 /*
1340 * For performance, all the general caches are L1 aligned.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001341 * This should be particularly beneficial on SMP boxes, as it
1342 * eliminates "false sharing".
1343 * Note for systems short on memory removing the alignment will
Christoph Lametere498be72005-09-09 13:03:32 -07001344 * allow tighter packing of the smaller caches.
1345 */
Andrew Mortona737b3e2006-03-22 00:08:11 -08001346 if (!sizes->cs_cachep) {
Christoph Lametere498be72005-09-09 13:03:32 -07001347 sizes->cs_cachep = kmem_cache_create(names->name,
Andrew Mortona737b3e2006-03-22 00:08:11 -08001348 sizes->cs_size,
1349 ARCH_KMALLOC_MINALIGN,
1350 ARCH_KMALLOC_FLAGS|SLAB_PANIC,
1351 NULL, NULL);
1352 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001353
Linus Torvalds1da177e2005-04-16 15:20:36 -07001354 sizes->cs_dmacachep = kmem_cache_create(names->name_dma,
Andrew Mortona737b3e2006-03-22 00:08:11 -08001355 sizes->cs_size,
1356 ARCH_KMALLOC_MINALIGN,
1357 ARCH_KMALLOC_FLAGS|SLAB_CACHE_DMA|
1358 SLAB_PANIC,
1359 NULL, NULL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001360 sizes++;
1361 names++;
1362 }
1363 /* 4) Replace the bootstrap head arrays */
1364 {
Pekka Enbergb28a02d2006-01-08 01:00:37 -08001365 void *ptr;
Christoph Lametere498be72005-09-09 13:03:32 -07001366
Linus Torvalds1da177e2005-04-16 15:20:36 -07001367 ptr = kmalloc(sizeof(struct arraycache_init), GFP_KERNEL);
Christoph Lametere498be72005-09-09 13:03:32 -07001368
Linus Torvalds1da177e2005-04-16 15:20:36 -07001369 local_irq_disable();
Pekka Enberg9a2dba42006-02-01 03:05:49 -08001370 BUG_ON(cpu_cache_get(&cache_cache) != &initarray_cache.cache);
1371 memcpy(ptr, cpu_cache_get(&cache_cache),
Pekka Enbergb28a02d2006-01-08 01:00:37 -08001372 sizeof(struct arraycache_init));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001373 cache_cache.array[smp_processor_id()] = ptr;
1374 local_irq_enable();
Christoph Lametere498be72005-09-09 13:03:32 -07001375
Linus Torvalds1da177e2005-04-16 15:20:36 -07001376 ptr = kmalloc(sizeof(struct arraycache_init), GFP_KERNEL);
Christoph Lametere498be72005-09-09 13:03:32 -07001377
Linus Torvalds1da177e2005-04-16 15:20:36 -07001378 local_irq_disable();
Pekka Enberg9a2dba42006-02-01 03:05:49 -08001379 BUG_ON(cpu_cache_get(malloc_sizes[INDEX_AC].cs_cachep)
Pekka Enbergb28a02d2006-01-08 01:00:37 -08001380 != &initarray_generic.cache);
Pekka Enberg9a2dba42006-02-01 03:05:49 -08001381 memcpy(ptr, cpu_cache_get(malloc_sizes[INDEX_AC].cs_cachep),
Pekka Enbergb28a02d2006-01-08 01:00:37 -08001382 sizeof(struct arraycache_init));
Christoph Lametere498be72005-09-09 13:03:32 -07001383 malloc_sizes[INDEX_AC].cs_cachep->array[smp_processor_id()] =
Pekka Enbergb28a02d2006-01-08 01:00:37 -08001384 ptr;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001385 local_irq_enable();
1386 }
Christoph Lametere498be72005-09-09 13:03:32 -07001387 /* 5) Replace the bootstrap kmem_list3's */
1388 {
1389 int node;
1390 /* Replace the static kmem_list3 structures for the boot cpu */
1391 init_list(&cache_cache, &initkmem_list3[CACHE_CACHE],
Pekka Enbergb28a02d2006-01-08 01:00:37 -08001392 numa_node_id());
Linus Torvalds1da177e2005-04-16 15:20:36 -07001393
Christoph Lametere498be72005-09-09 13:03:32 -07001394 for_each_online_node(node) {
1395 init_list(malloc_sizes[INDEX_AC].cs_cachep,
Pekka Enbergb28a02d2006-01-08 01:00:37 -08001396 &initkmem_list3[SIZE_AC + node], node);
Christoph Lametere498be72005-09-09 13:03:32 -07001397
1398 if (INDEX_AC != INDEX_L3) {
1399 init_list(malloc_sizes[INDEX_L3].cs_cachep,
Pekka Enbergb28a02d2006-01-08 01:00:37 -08001400 &initkmem_list3[SIZE_L3 + node],
1401 node);
Christoph Lametere498be72005-09-09 13:03:32 -07001402 }
1403 }
1404 }
1405
1406 /* 6) resize the head arrays to their final sizes */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001407 {
Pekka Enberg343e0d72006-02-01 03:05:50 -08001408 struct kmem_cache *cachep;
Ingo Molnarfc0abb12006-01-18 17:42:33 -08001409 mutex_lock(&cache_chain_mutex);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001410 list_for_each_entry(cachep, &cache_chain, next)
Andrew Mortona737b3e2006-03-22 00:08:11 -08001411 enable_cpucache(cachep);
Ingo Molnarfc0abb12006-01-18 17:42:33 -08001412 mutex_unlock(&cache_chain_mutex);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001413 }
1414
1415 /* Done! */
1416 g_cpucache_up = FULL;
1417
Andrew Mortona737b3e2006-03-22 00:08:11 -08001418 /*
1419 * Register a cpu startup notifier callback that initializes
1420 * cpu_cache_get for all new cpus
Linus Torvalds1da177e2005-04-16 15:20:36 -07001421 */
1422 register_cpu_notifier(&cpucache_notifier);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001423
Andrew Mortona737b3e2006-03-22 00:08:11 -08001424 /*
1425 * The reap timers are started later, with a module init call: That part
1426 * of the kernel is not yet operational.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001427 */
1428}
1429
1430static int __init cpucache_init(void)
1431{
1432 int cpu;
1433
Andrew Mortona737b3e2006-03-22 00:08:11 -08001434 /*
1435 * Register the timers that return unneeded pages to the page allocator
Linus Torvalds1da177e2005-04-16 15:20:36 -07001436 */
Christoph Lametere498be72005-09-09 13:03:32 -07001437 for_each_online_cpu(cpu)
Andrew Mortona737b3e2006-03-22 00:08:11 -08001438 start_cpu_timer(cpu);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001439 return 0;
1440}
Linus Torvalds1da177e2005-04-16 15:20:36 -07001441__initcall(cpucache_init);
1442
1443/*
1444 * Interface to system's page allocator. No need to hold the cache-lock.
1445 *
1446 * If we requested dmaable memory, we will get it. Even if we
1447 * did not request dmaable memory, we might get it, but that
1448 * would be relatively rare and ignorable.
1449 */
Pekka Enberg343e0d72006-02-01 03:05:50 -08001450static void *kmem_getpages(struct kmem_cache *cachep, gfp_t flags, int nodeid)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001451{
1452 struct page *page;
1453 void *addr;
1454 int i;
1455
1456 flags |= cachep->gfpflags;
Luke Yangd6fef9d2006-04-10 22:52:56 -07001457#ifndef CONFIG_MMU
1458 /* nommu uses slab's for process anonymous memory allocations, so
1459 * requires __GFP_COMP to properly refcount higher order allocations"
1460 */
1461 page = alloc_pages_node(nodeid, (flags | __GFP_COMP), cachep->gfporder);
1462#else
Christoph Lameter50c85a12005-11-13 16:06:47 -08001463 page = alloc_pages_node(nodeid, flags, cachep->gfporder);
Luke Yangd6fef9d2006-04-10 22:52:56 -07001464#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07001465 if (!page)
1466 return NULL;
1467 addr = page_address(page);
1468
1469 i = (1 << cachep->gfporder);
1470 if (cachep->flags & SLAB_RECLAIM_ACCOUNT)
1471 atomic_add(i, &slab_reclaim_pages);
1472 add_page_state(nr_slab, i);
1473 while (i--) {
Nick Pigginf205b2f2006-03-22 00:08:02 -08001474 __SetPageSlab(page);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001475 page++;
1476 }
1477 return addr;
1478}
1479
1480/*
1481 * Interface to system's page release.
1482 */
Pekka Enberg343e0d72006-02-01 03:05:50 -08001483static void kmem_freepages(struct kmem_cache *cachep, void *addr)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001484{
Pekka Enbergb28a02d2006-01-08 01:00:37 -08001485 unsigned long i = (1 << cachep->gfporder);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001486 struct page *page = virt_to_page(addr);
1487 const unsigned long nr_freed = i;
1488
1489 while (i--) {
Nick Pigginf205b2f2006-03-22 00:08:02 -08001490 BUG_ON(!PageSlab(page));
1491 __ClearPageSlab(page);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001492 page++;
1493 }
1494 sub_page_state(nr_slab, nr_freed);
1495 if (current->reclaim_state)
1496 current->reclaim_state->reclaimed_slab += nr_freed;
1497 free_pages((unsigned long)addr, cachep->gfporder);
Pekka Enbergb28a02d2006-01-08 01:00:37 -08001498 if (cachep->flags & SLAB_RECLAIM_ACCOUNT)
1499 atomic_sub(1 << cachep->gfporder, &slab_reclaim_pages);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001500}
1501
1502static void kmem_rcu_free(struct rcu_head *head)
1503{
Pekka Enbergb28a02d2006-01-08 01:00:37 -08001504 struct slab_rcu *slab_rcu = (struct slab_rcu *)head;
Pekka Enberg343e0d72006-02-01 03:05:50 -08001505 struct kmem_cache *cachep = slab_rcu->cachep;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001506
1507 kmem_freepages(cachep, slab_rcu->addr);
1508 if (OFF_SLAB(cachep))
1509 kmem_cache_free(cachep->slabp_cache, slab_rcu);
1510}
1511
1512#if DEBUG
1513
1514#ifdef CONFIG_DEBUG_PAGEALLOC
Pekka Enberg343e0d72006-02-01 03:05:50 -08001515static void store_stackinfo(struct kmem_cache *cachep, unsigned long *addr,
Pekka Enbergb28a02d2006-01-08 01:00:37 -08001516 unsigned long caller)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001517{
Manfred Spraul3dafccf2006-02-01 03:05:42 -08001518 int size = obj_size(cachep);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001519
Manfred Spraul3dafccf2006-02-01 03:05:42 -08001520 addr = (unsigned long *)&((char *)addr)[obj_offset(cachep)];
Linus Torvalds1da177e2005-04-16 15:20:36 -07001521
Pekka Enbergb28a02d2006-01-08 01:00:37 -08001522 if (size < 5 * sizeof(unsigned long))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001523 return;
1524
Pekka Enbergb28a02d2006-01-08 01:00:37 -08001525 *addr++ = 0x12345678;
1526 *addr++ = caller;
1527 *addr++ = smp_processor_id();
1528 size -= 3 * sizeof(unsigned long);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001529 {
1530 unsigned long *sptr = &caller;
1531 unsigned long svalue;
1532
1533 while (!kstack_end(sptr)) {
1534 svalue = *sptr++;
1535 if (kernel_text_address(svalue)) {
Pekka Enbergb28a02d2006-01-08 01:00:37 -08001536 *addr++ = svalue;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001537 size -= sizeof(unsigned long);
1538 if (size <= sizeof(unsigned long))
1539 break;
1540 }
1541 }
1542
1543 }
Pekka Enbergb28a02d2006-01-08 01:00:37 -08001544 *addr++ = 0x87654321;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001545}
1546#endif
1547
Pekka Enberg343e0d72006-02-01 03:05:50 -08001548static void poison_obj(struct kmem_cache *cachep, void *addr, unsigned char val)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001549{
Manfred Spraul3dafccf2006-02-01 03:05:42 -08001550 int size = obj_size(cachep);
1551 addr = &((char *)addr)[obj_offset(cachep)];
Linus Torvalds1da177e2005-04-16 15:20:36 -07001552
1553 memset(addr, val, size);
Pekka Enbergb28a02d2006-01-08 01:00:37 -08001554 *(unsigned char *)(addr + size - 1) = POISON_END;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001555}
1556
1557static void dump_line(char *data, int offset, int limit)
1558{
1559 int i;
1560 printk(KERN_ERR "%03x:", offset);
Andrew Mortona737b3e2006-03-22 00:08:11 -08001561 for (i = 0; i < limit; i++)
Pekka Enbergb28a02d2006-01-08 01:00:37 -08001562 printk(" %02x", (unsigned char)data[offset + i]);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001563 printk("\n");
1564}
1565#endif
1566
1567#if DEBUG
1568
Pekka Enberg343e0d72006-02-01 03:05:50 -08001569static void print_objinfo(struct kmem_cache *cachep, void *objp, int lines)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001570{
1571 int i, size;
1572 char *realobj;
1573
1574 if (cachep->flags & SLAB_RED_ZONE) {
1575 printk(KERN_ERR "Redzone: 0x%lx/0x%lx.\n",
Andrew Mortona737b3e2006-03-22 00:08:11 -08001576 *dbg_redzone1(cachep, objp),
1577 *dbg_redzone2(cachep, objp));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001578 }
1579
1580 if (cachep->flags & SLAB_STORE_USER) {
1581 printk(KERN_ERR "Last user: [<%p>]",
Andrew Mortona737b3e2006-03-22 00:08:11 -08001582 *dbg_userword(cachep, objp));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001583 print_symbol("(%s)",
Andrew Mortona737b3e2006-03-22 00:08:11 -08001584 (unsigned long)*dbg_userword(cachep, objp));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001585 printk("\n");
1586 }
Manfred Spraul3dafccf2006-02-01 03:05:42 -08001587 realobj = (char *)objp + obj_offset(cachep);
1588 size = obj_size(cachep);
Pekka Enbergb28a02d2006-01-08 01:00:37 -08001589 for (i = 0; i < size && lines; i += 16, lines--) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001590 int limit;
1591 limit = 16;
Pekka Enbergb28a02d2006-01-08 01:00:37 -08001592 if (i + limit > size)
1593 limit = size - i;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001594 dump_line(realobj, i, limit);
1595 }
1596}
1597
Pekka Enberg343e0d72006-02-01 03:05:50 -08001598static void check_poison_obj(struct kmem_cache *cachep, void *objp)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001599{
1600 char *realobj;
1601 int size, i;
1602 int lines = 0;
1603
Manfred Spraul3dafccf2006-02-01 03:05:42 -08001604 realobj = (char *)objp + obj_offset(cachep);
1605 size = obj_size(cachep);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001606
Pekka Enbergb28a02d2006-01-08 01:00:37 -08001607 for (i = 0; i < size; i++) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001608 char exp = POISON_FREE;
Pekka Enbergb28a02d2006-01-08 01:00:37 -08001609 if (i == size - 1)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001610 exp = POISON_END;
1611 if (realobj[i] != exp) {
1612 int limit;
1613 /* Mismatch ! */
1614 /* Print header */
1615 if (lines == 0) {
Pekka Enbergb28a02d2006-01-08 01:00:37 -08001616 printk(KERN_ERR
Andrew Mortona737b3e2006-03-22 00:08:11 -08001617 "Slab corruption: start=%p, len=%d\n",
1618 realobj, size);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001619 print_objinfo(cachep, objp, 0);
1620 }
1621 /* Hexdump the affected line */
Pekka Enbergb28a02d2006-01-08 01:00:37 -08001622 i = (i / 16) * 16;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001623 limit = 16;
Pekka Enbergb28a02d2006-01-08 01:00:37 -08001624 if (i + limit > size)
1625 limit = size - i;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001626 dump_line(realobj, i, limit);
1627 i += 16;
1628 lines++;
1629 /* Limit to 5 lines */
1630 if (lines > 5)
1631 break;
1632 }
1633 }
1634 if (lines != 0) {
1635 /* Print some data about the neighboring objects, if they
1636 * exist:
1637 */
Pekka Enberg6ed5eb2212006-02-01 03:05:49 -08001638 struct slab *slabp = virt_to_slab(objp);
Pekka Enberg8fea4e92006-03-22 00:08:10 -08001639 unsigned int objnr;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001640
Pekka Enberg8fea4e92006-03-22 00:08:10 -08001641 objnr = obj_to_index(cachep, slabp, objp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001642 if (objnr) {
Pekka Enberg8fea4e92006-03-22 00:08:10 -08001643 objp = index_to_obj(cachep, slabp, objnr - 1);
Manfred Spraul3dafccf2006-02-01 03:05:42 -08001644 realobj = (char *)objp + obj_offset(cachep);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001645 printk(KERN_ERR "Prev obj: start=%p, len=%d\n",
Pekka Enbergb28a02d2006-01-08 01:00:37 -08001646 realobj, size);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001647 print_objinfo(cachep, objp, 2);
1648 }
Pekka Enbergb28a02d2006-01-08 01:00:37 -08001649 if (objnr + 1 < cachep->num) {
Pekka Enberg8fea4e92006-03-22 00:08:10 -08001650 objp = index_to_obj(cachep, slabp, objnr + 1);
Manfred Spraul3dafccf2006-02-01 03:05:42 -08001651 realobj = (char *)objp + obj_offset(cachep);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001652 printk(KERN_ERR "Next obj: start=%p, len=%d\n",
Pekka Enbergb28a02d2006-01-08 01:00:37 -08001653 realobj, size);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001654 print_objinfo(cachep, objp, 2);
1655 }
1656 }
1657}
1658#endif
1659
Linus Torvalds1da177e2005-04-16 15:20:36 -07001660#if DEBUG
Matthew Dobson12dd36f2006-02-01 03:05:46 -08001661/**
Randy Dunlap911851e2006-03-22 00:08:14 -08001662 * slab_destroy_objs - destroy a slab and its objects
1663 * @cachep: cache pointer being destroyed
1664 * @slabp: slab pointer being destroyed
1665 *
1666 * Call the registered destructor for each object in a slab that is being
1667 * destroyed.
Matthew Dobson12dd36f2006-02-01 03:05:46 -08001668 */
Pekka Enberg343e0d72006-02-01 03:05:50 -08001669static void slab_destroy_objs(struct kmem_cache *cachep, struct slab *slabp)
Matthew Dobson12dd36f2006-02-01 03:05:46 -08001670{
Linus Torvalds1da177e2005-04-16 15:20:36 -07001671 int i;
1672 for (i = 0; i < cachep->num; i++) {
Pekka Enberg8fea4e92006-03-22 00:08:10 -08001673 void *objp = index_to_obj(cachep, slabp, i);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001674
1675 if (cachep->flags & SLAB_POISON) {
1676#ifdef CONFIG_DEBUG_PAGEALLOC
Andrew Mortona737b3e2006-03-22 00:08:11 -08001677 if (cachep->buffer_size % PAGE_SIZE == 0 &&
1678 OFF_SLAB(cachep))
Pekka Enbergb28a02d2006-01-08 01:00:37 -08001679 kernel_map_pages(virt_to_page(objp),
Andrew Mortona737b3e2006-03-22 00:08:11 -08001680 cachep->buffer_size / PAGE_SIZE, 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001681 else
1682 check_poison_obj(cachep, objp);
1683#else
1684 check_poison_obj(cachep, objp);
1685#endif
1686 }
1687 if (cachep->flags & SLAB_RED_ZONE) {
1688 if (*dbg_redzone1(cachep, objp) != RED_INACTIVE)
1689 slab_error(cachep, "start of a freed object "
Pekka Enbergb28a02d2006-01-08 01:00:37 -08001690 "was overwritten");
Linus Torvalds1da177e2005-04-16 15:20:36 -07001691 if (*dbg_redzone2(cachep, objp) != RED_INACTIVE)
1692 slab_error(cachep, "end of a freed object "
Pekka Enbergb28a02d2006-01-08 01:00:37 -08001693 "was overwritten");
Linus Torvalds1da177e2005-04-16 15:20:36 -07001694 }
1695 if (cachep->dtor && !(cachep->flags & SLAB_POISON))
Manfred Spraul3dafccf2006-02-01 03:05:42 -08001696 (cachep->dtor) (objp + obj_offset(cachep), cachep, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001697 }
Matthew Dobson12dd36f2006-02-01 03:05:46 -08001698}
Linus Torvalds1da177e2005-04-16 15:20:36 -07001699#else
Pekka Enberg343e0d72006-02-01 03:05:50 -08001700static void slab_destroy_objs(struct kmem_cache *cachep, struct slab *slabp)
Matthew Dobson12dd36f2006-02-01 03:05:46 -08001701{
Linus Torvalds1da177e2005-04-16 15:20:36 -07001702 if (cachep->dtor) {
1703 int i;
1704 for (i = 0; i < cachep->num; i++) {
Pekka Enberg8fea4e92006-03-22 00:08:10 -08001705 void *objp = index_to_obj(cachep, slabp, i);
Pekka Enbergb28a02d2006-01-08 01:00:37 -08001706 (cachep->dtor) (objp, cachep, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001707 }
1708 }
Matthew Dobson12dd36f2006-02-01 03:05:46 -08001709}
Linus Torvalds1da177e2005-04-16 15:20:36 -07001710#endif
1711
Randy Dunlap911851e2006-03-22 00:08:14 -08001712/**
1713 * slab_destroy - destroy and release all objects in a slab
1714 * @cachep: cache pointer being destroyed
1715 * @slabp: slab pointer being destroyed
1716 *
Matthew Dobson12dd36f2006-02-01 03:05:46 -08001717 * Destroy all the objs in a slab, and release the mem back to the system.
Andrew Mortona737b3e2006-03-22 00:08:11 -08001718 * Before calling the slab must have been unlinked from the cache. The
1719 * cache-lock is not held/needed.
Matthew Dobson12dd36f2006-02-01 03:05:46 -08001720 */
Pekka Enberg343e0d72006-02-01 03:05:50 -08001721static void slab_destroy(struct kmem_cache *cachep, struct slab *slabp)
Matthew Dobson12dd36f2006-02-01 03:05:46 -08001722{
1723 void *addr = slabp->s_mem - slabp->colouroff;
1724
1725 slab_destroy_objs(cachep, slabp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001726 if (unlikely(cachep->flags & SLAB_DESTROY_BY_RCU)) {
1727 struct slab_rcu *slab_rcu;
1728
Pekka Enbergb28a02d2006-01-08 01:00:37 -08001729 slab_rcu = (struct slab_rcu *)slabp;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001730 slab_rcu->cachep = cachep;
1731 slab_rcu->addr = addr;
1732 call_rcu(&slab_rcu->head, kmem_rcu_free);
1733 } else {
1734 kmem_freepages(cachep, addr);
1735 if (OFF_SLAB(cachep))
1736 kmem_cache_free(cachep->slabp_cache, slabp);
1737 }
1738}
1739
Andrew Mortona737b3e2006-03-22 00:08:11 -08001740/*
1741 * For setting up all the kmem_list3s for cache whose buffer_size is same as
1742 * size of kmem_list3.
1743 */
Pekka Enberg343e0d72006-02-01 03:05:50 -08001744static void set_up_list3s(struct kmem_cache *cachep, int index)
Christoph Lametere498be72005-09-09 13:03:32 -07001745{
1746 int node;
1747
1748 for_each_online_node(node) {
Pekka Enbergb28a02d2006-01-08 01:00:37 -08001749 cachep->nodelists[node] = &initkmem_list3[index + node];
Christoph Lametere498be72005-09-09 13:03:32 -07001750 cachep->nodelists[node]->next_reap = jiffies +
Pekka Enbergb28a02d2006-01-08 01:00:37 -08001751 REAPTIMEOUT_LIST3 +
1752 ((unsigned long)cachep) % REAPTIMEOUT_LIST3;
Christoph Lametere498be72005-09-09 13:03:32 -07001753 }
1754}
1755
Linus Torvalds1da177e2005-04-16 15:20:36 -07001756/**
Randy.Dunlapa70773d2006-02-01 03:05:52 -08001757 * calculate_slab_order - calculate size (page order) of slabs
1758 * @cachep: pointer to the cache that is being created
1759 * @size: size of objects to be created in this cache.
1760 * @align: required alignment for the objects.
1761 * @flags: slab allocation flags
1762 *
1763 * Also calculates the number of objects per slab.
Pekka Enberg4d268eb2006-01-08 01:00:36 -08001764 *
1765 * This could be made much more intelligent. For now, try to avoid using
1766 * high order pages for slabs. When the gfp() functions are more friendly
1767 * towards high-order requests, this should be changed.
1768 */
Andrew Mortona737b3e2006-03-22 00:08:11 -08001769static size_t calculate_slab_order(struct kmem_cache *cachep,
Randy Dunlapee13d782006-02-01 03:05:53 -08001770 size_t size, size_t align, unsigned long flags)
Pekka Enberg4d268eb2006-01-08 01:00:36 -08001771{
Ingo Molnarb1ab41c2006-06-02 15:44:58 +02001772 unsigned long offslab_limit;
Pekka Enberg4d268eb2006-01-08 01:00:36 -08001773 size_t left_over = 0;
Linus Torvalds9888e6f2006-03-06 17:44:43 -08001774 int gfporder;
Pekka Enberg4d268eb2006-01-08 01:00:36 -08001775
Andrew Mortona737b3e2006-03-22 00:08:11 -08001776 for (gfporder = 0; gfporder <= MAX_GFP_ORDER; gfporder++) {
Pekka Enberg4d268eb2006-01-08 01:00:36 -08001777 unsigned int num;
1778 size_t remainder;
1779
Linus Torvalds9888e6f2006-03-06 17:44:43 -08001780 cache_estimate(gfporder, size, align, flags, &remainder, &num);
Pekka Enberg4d268eb2006-01-08 01:00:36 -08001781 if (!num)
1782 continue;
Linus Torvalds9888e6f2006-03-06 17:44:43 -08001783
Ingo Molnarb1ab41c2006-06-02 15:44:58 +02001784 if (flags & CFLGS_OFF_SLAB) {
1785 /*
1786 * Max number of objs-per-slab for caches which
1787 * use off-slab slabs. Needed to avoid a possible
1788 * looping condition in cache_grow().
1789 */
1790 offslab_limit = size - sizeof(struct slab);
1791 offslab_limit /= sizeof(kmem_bufctl_t);
1792
1793 if (num > offslab_limit)
1794 break;
1795 }
Pekka Enberg4d268eb2006-01-08 01:00:36 -08001796
Linus Torvalds9888e6f2006-03-06 17:44:43 -08001797 /* Found something acceptable - save it away */
Pekka Enberg4d268eb2006-01-08 01:00:36 -08001798 cachep->num = num;
Linus Torvalds9888e6f2006-03-06 17:44:43 -08001799 cachep->gfporder = gfporder;
Pekka Enberg4d268eb2006-01-08 01:00:36 -08001800 left_over = remainder;
1801
1802 /*
Linus Torvaldsf78bb8a2006-03-08 10:33:05 -08001803 * A VFS-reclaimable slab tends to have most allocations
1804 * as GFP_NOFS and we really don't want to have to be allocating
1805 * higher-order pages when we are unable to shrink dcache.
1806 */
1807 if (flags & SLAB_RECLAIM_ACCOUNT)
1808 break;
1809
1810 /*
Pekka Enberg4d268eb2006-01-08 01:00:36 -08001811 * Large number of objects is good, but very large slabs are
1812 * currently bad for the gfp()s.
1813 */
Linus Torvalds9888e6f2006-03-06 17:44:43 -08001814 if (gfporder >= slab_break_gfp_order)
Pekka Enberg4d268eb2006-01-08 01:00:36 -08001815 break;
1816
Linus Torvalds9888e6f2006-03-06 17:44:43 -08001817 /*
1818 * Acceptable internal fragmentation?
1819 */
Andrew Mortona737b3e2006-03-22 00:08:11 -08001820 if (left_over * 8 <= (PAGE_SIZE << gfporder))
Pekka Enberg4d268eb2006-01-08 01:00:36 -08001821 break;
1822 }
1823 return left_over;
1824}
1825
Pekka Enbergf30cf7d2006-03-22 00:08:11 -08001826static void setup_cpu_cache(struct kmem_cache *cachep)
1827{
1828 if (g_cpucache_up == FULL) {
1829 enable_cpucache(cachep);
1830 return;
1831 }
1832 if (g_cpucache_up == NONE) {
1833 /*
1834 * Note: the first kmem_cache_create must create the cache
1835 * that's used by kmalloc(24), otherwise the creation of
1836 * further caches will BUG().
1837 */
1838 cachep->array[smp_processor_id()] = &initarray_generic.cache;
1839
1840 /*
1841 * If the cache that's used by kmalloc(sizeof(kmem_list3)) is
1842 * the first cache, then we need to set up all its list3s,
1843 * otherwise the creation of further caches will BUG().
1844 */
1845 set_up_list3s(cachep, SIZE_AC);
1846 if (INDEX_AC == INDEX_L3)
1847 g_cpucache_up = PARTIAL_L3;
1848 else
1849 g_cpucache_up = PARTIAL_AC;
1850 } else {
1851 cachep->array[smp_processor_id()] =
1852 kmalloc(sizeof(struct arraycache_init), GFP_KERNEL);
1853
1854 if (g_cpucache_up == PARTIAL_AC) {
1855 set_up_list3s(cachep, SIZE_L3);
1856 g_cpucache_up = PARTIAL_L3;
1857 } else {
1858 int node;
1859 for_each_online_node(node) {
1860 cachep->nodelists[node] =
1861 kmalloc_node(sizeof(struct kmem_list3),
1862 GFP_KERNEL, node);
1863 BUG_ON(!cachep->nodelists[node]);
1864 kmem_list3_init(cachep->nodelists[node]);
1865 }
1866 }
1867 }
1868 cachep->nodelists[numa_node_id()]->next_reap =
1869 jiffies + REAPTIMEOUT_LIST3 +
1870 ((unsigned long)cachep) % REAPTIMEOUT_LIST3;
1871
1872 cpu_cache_get(cachep)->avail = 0;
1873 cpu_cache_get(cachep)->limit = BOOT_CPUCACHE_ENTRIES;
1874 cpu_cache_get(cachep)->batchcount = 1;
1875 cpu_cache_get(cachep)->touched = 0;
1876 cachep->batchcount = 1;
1877 cachep->limit = BOOT_CPUCACHE_ENTRIES;
1878}
1879
Pekka Enberg4d268eb2006-01-08 01:00:36 -08001880/**
Linus Torvalds1da177e2005-04-16 15:20:36 -07001881 * kmem_cache_create - Create a cache.
1882 * @name: A string which is used in /proc/slabinfo to identify this cache.
1883 * @size: The size of objects to be created in this cache.
1884 * @align: The required alignment for the objects.
1885 * @flags: SLAB flags
1886 * @ctor: A constructor for the objects.
1887 * @dtor: A destructor for the objects.
1888 *
1889 * Returns a ptr to the cache on success, NULL on failure.
1890 * Cannot be called within a int, but can be interrupted.
1891 * The @ctor is run when new pages are allocated by the cache
1892 * and the @dtor is run before the pages are handed back.
1893 *
1894 * @name must be valid until the cache is destroyed. This implies that
Andrew Mortona737b3e2006-03-22 00:08:11 -08001895 * the module calling this has to destroy the cache before getting unloaded.
1896 *
Linus Torvalds1da177e2005-04-16 15:20:36 -07001897 * The flags are
1898 *
1899 * %SLAB_POISON - Poison the slab with a known test pattern (a5a5a5a5)
1900 * to catch references to uninitialised memory.
1901 *
1902 * %SLAB_RED_ZONE - Insert `Red' zones around the allocated memory to check
1903 * for buffer overruns.
1904 *
Linus Torvalds1da177e2005-04-16 15:20:36 -07001905 * %SLAB_HWCACHE_ALIGN - Align the objects in this cache to a hardware
1906 * cacheline. This can be beneficial if you're counting cycles as closely
1907 * as davem.
1908 */
Pekka Enberg343e0d72006-02-01 03:05:50 -08001909struct kmem_cache *
Linus Torvalds1da177e2005-04-16 15:20:36 -07001910kmem_cache_create (const char *name, size_t size, size_t align,
Andrew Mortona737b3e2006-03-22 00:08:11 -08001911 unsigned long flags,
1912 void (*ctor)(void*, struct kmem_cache *, unsigned long),
Pekka Enberg343e0d72006-02-01 03:05:50 -08001913 void (*dtor)(void*, struct kmem_cache *, unsigned long))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001914{
1915 size_t left_over, slab_size, ralign;
Pekka Enberg343e0d72006-02-01 03:05:50 -08001916 struct kmem_cache *cachep = NULL;
Andrew Morton4f12bb42005-11-07 00:58:00 -08001917 struct list_head *p;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001918
1919 /*
1920 * Sanity checks... these are all serious usage bugs.
1921 */
Andrew Mortona737b3e2006-03-22 00:08:11 -08001922 if (!name || in_interrupt() || (size < BYTES_PER_WORD) ||
Pekka Enbergb28a02d2006-01-08 01:00:37 -08001923 (size > (1 << MAX_OBJ_ORDER) * PAGE_SIZE) || (dtor && !ctor)) {
Andrew Mortona737b3e2006-03-22 00:08:11 -08001924 printk(KERN_ERR "%s: Early error in slab %s\n", __FUNCTION__,
1925 name);
Pekka Enbergb28a02d2006-01-08 01:00:37 -08001926 BUG();
1927 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001928
Ravikiran G Thirumalaif0188f42006-02-10 01:51:13 -08001929 /*
1930 * Prevent CPUs from coming and going.
1931 * lock_cpu_hotplug() nests outside cache_chain_mutex
1932 */
1933 lock_cpu_hotplug();
1934
Ingo Molnarfc0abb12006-01-18 17:42:33 -08001935 mutex_lock(&cache_chain_mutex);
Andrew Morton4f12bb42005-11-07 00:58:00 -08001936
1937 list_for_each(p, &cache_chain) {
Pekka Enberg343e0d72006-02-01 03:05:50 -08001938 struct kmem_cache *pc = list_entry(p, struct kmem_cache, next);
Andrew Morton4f12bb42005-11-07 00:58:00 -08001939 mm_segment_t old_fs = get_fs();
1940 char tmp;
1941 int res;
1942
1943 /*
1944 * This happens when the module gets unloaded and doesn't
1945 * destroy its slab cache and no-one else reuses the vmalloc
1946 * area of the module. Print a warning.
1947 */
1948 set_fs(KERNEL_DS);
1949 res = __get_user(tmp, pc->name);
1950 set_fs(old_fs);
1951 if (res) {
1952 printk("SLAB: cache with size %d has lost its name\n",
Manfred Spraul3dafccf2006-02-01 03:05:42 -08001953 pc->buffer_size);
Andrew Morton4f12bb42005-11-07 00:58:00 -08001954 continue;
1955 }
1956
Pekka Enbergb28a02d2006-01-08 01:00:37 -08001957 if (!strcmp(pc->name, name)) {
Andrew Morton4f12bb42005-11-07 00:58:00 -08001958 printk("kmem_cache_create: duplicate cache %s\n", name);
1959 dump_stack();
1960 goto oops;
1961 }
1962 }
1963
Linus Torvalds1da177e2005-04-16 15:20:36 -07001964#if DEBUG
1965 WARN_ON(strchr(name, ' ')); /* It confuses parsers */
1966 if ((flags & SLAB_DEBUG_INITIAL) && !ctor) {
1967 /* No constructor, but inital state check requested */
1968 printk(KERN_ERR "%s: No con, but init state check "
Pekka Enbergb28a02d2006-01-08 01:00:37 -08001969 "requested - %s\n", __FUNCTION__, name);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001970 flags &= ~SLAB_DEBUG_INITIAL;
1971 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001972#if FORCED_DEBUG
1973 /*
1974 * Enable redzoning and last user accounting, except for caches with
1975 * large objects, if the increased size would increase the object size
1976 * above the next power of two: caches with object sizes just above a
1977 * power of two have a significant amount of internal fragmentation.
1978 */
Andrew Mortona737b3e2006-03-22 00:08:11 -08001979 if (size < 4096 || fls(size - 1) == fls(size-1 + 3 * BYTES_PER_WORD))
Pekka Enbergb28a02d2006-01-08 01:00:37 -08001980 flags |= SLAB_RED_ZONE | SLAB_STORE_USER;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001981 if (!(flags & SLAB_DESTROY_BY_RCU))
1982 flags |= SLAB_POISON;
1983#endif
1984 if (flags & SLAB_DESTROY_BY_RCU)
1985 BUG_ON(flags & SLAB_POISON);
1986#endif
1987 if (flags & SLAB_DESTROY_BY_RCU)
1988 BUG_ON(dtor);
1989
1990 /*
Andrew Mortona737b3e2006-03-22 00:08:11 -08001991 * Always checks flags, a caller might be expecting debug support which
1992 * isn't available.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001993 */
Eric Sesterhenn40094fa2006-04-02 13:49:25 +02001994 BUG_ON(flags & ~CREATE_MASK);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001995
Andrew Mortona737b3e2006-03-22 00:08:11 -08001996 /*
1997 * Check that size is in terms of words. This is needed to avoid
Linus Torvalds1da177e2005-04-16 15:20:36 -07001998 * unaligned accesses for some archs when redzoning is used, and makes
1999 * sure any on-slab bufctl's are also correctly aligned.
2000 */
Pekka Enbergb28a02d2006-01-08 01:00:37 -08002001 if (size & (BYTES_PER_WORD - 1)) {
2002 size += (BYTES_PER_WORD - 1);
2003 size &= ~(BYTES_PER_WORD - 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002004 }
2005
Andrew Mortona737b3e2006-03-22 00:08:11 -08002006 /* calculate the final buffer alignment: */
2007
Linus Torvalds1da177e2005-04-16 15:20:36 -07002008 /* 1) arch recommendation: can be overridden for debug */
2009 if (flags & SLAB_HWCACHE_ALIGN) {
Andrew Mortona737b3e2006-03-22 00:08:11 -08002010 /*
2011 * Default alignment: as specified by the arch code. Except if
2012 * an object is really small, then squeeze multiple objects into
2013 * one cacheline.
Linus Torvalds1da177e2005-04-16 15:20:36 -07002014 */
2015 ralign = cache_line_size();
Pekka Enbergb28a02d2006-01-08 01:00:37 -08002016 while (size <= ralign / 2)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002017 ralign /= 2;
2018 } else {
2019 ralign = BYTES_PER_WORD;
2020 }
2021 /* 2) arch mandated alignment: disables debug if necessary */
2022 if (ralign < ARCH_SLAB_MINALIGN) {
2023 ralign = ARCH_SLAB_MINALIGN;
2024 if (ralign > BYTES_PER_WORD)
Pekka Enbergb28a02d2006-01-08 01:00:37 -08002025 flags &= ~(SLAB_RED_ZONE | SLAB_STORE_USER);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002026 }
2027 /* 3) caller mandated alignment: disables debug if necessary */
2028 if (ralign < align) {
2029 ralign = align;
2030 if (ralign > BYTES_PER_WORD)
Pekka Enbergb28a02d2006-01-08 01:00:37 -08002031 flags &= ~(SLAB_RED_ZONE | SLAB_STORE_USER);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002032 }
Andrew Mortona737b3e2006-03-22 00:08:11 -08002033 /*
2034 * 4) Store it. Note that the debug code below can reduce
Linus Torvalds1da177e2005-04-16 15:20:36 -07002035 * the alignment to BYTES_PER_WORD.
2036 */
2037 align = ralign;
2038
2039 /* Get cache's description obj. */
Pekka Enbergc5e3b832006-03-25 03:06:43 -08002040 cachep = kmem_cache_zalloc(&cache_cache, SLAB_KERNEL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002041 if (!cachep)
Andrew Morton4f12bb42005-11-07 00:58:00 -08002042 goto oops;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002043
2044#if DEBUG
Manfred Spraul3dafccf2006-02-01 03:05:42 -08002045 cachep->obj_size = size;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002046
2047 if (flags & SLAB_RED_ZONE) {
2048 /* redzoning only works with word aligned caches */
2049 align = BYTES_PER_WORD;
2050
2051 /* add space for red zone words */
Manfred Spraul3dafccf2006-02-01 03:05:42 -08002052 cachep->obj_offset += BYTES_PER_WORD;
Pekka Enbergb28a02d2006-01-08 01:00:37 -08002053 size += 2 * BYTES_PER_WORD;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002054 }
2055 if (flags & SLAB_STORE_USER) {
2056 /* user store requires word alignment and
2057 * one word storage behind the end of the real
2058 * object.
2059 */
2060 align = BYTES_PER_WORD;
2061 size += BYTES_PER_WORD;
2062 }
2063#if FORCED_DEBUG && defined(CONFIG_DEBUG_PAGEALLOC)
Pekka Enbergb28a02d2006-01-08 01:00:37 -08002064 if (size >= malloc_sizes[INDEX_L3 + 1].cs_size
Manfred Spraul3dafccf2006-02-01 03:05:42 -08002065 && cachep->obj_size > cache_line_size() && size < PAGE_SIZE) {
2066 cachep->obj_offset += PAGE_SIZE - size;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002067 size = PAGE_SIZE;
2068 }
2069#endif
2070#endif
2071
2072 /* Determine if the slab management is 'on' or 'off' slab. */
Pekka Enbergb28a02d2006-01-08 01:00:37 -08002073 if (size >= (PAGE_SIZE >> 3))
Linus Torvalds1da177e2005-04-16 15:20:36 -07002074 /*
2075 * Size is large, assume best to place the slab management obj
2076 * off-slab (should allow better packing of objs).
2077 */
2078 flags |= CFLGS_OFF_SLAB;
2079
2080 size = ALIGN(size, align);
2081
Linus Torvaldsf78bb8a2006-03-08 10:33:05 -08002082 left_over = calculate_slab_order(cachep, size, align, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002083
2084 if (!cachep->num) {
2085 printk("kmem_cache_create: couldn't create cache %s.\n", name);
2086 kmem_cache_free(&cache_cache, cachep);
2087 cachep = NULL;
Andrew Morton4f12bb42005-11-07 00:58:00 -08002088 goto oops;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002089 }
Pekka Enbergb28a02d2006-01-08 01:00:37 -08002090 slab_size = ALIGN(cachep->num * sizeof(kmem_bufctl_t)
2091 + sizeof(struct slab), align);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002092
2093 /*
2094 * If the slab has been placed off-slab, and we have enough space then
2095 * move it on-slab. This is at the expense of any extra colouring.
2096 */
2097 if (flags & CFLGS_OFF_SLAB && left_over >= slab_size) {
2098 flags &= ~CFLGS_OFF_SLAB;
2099 left_over -= slab_size;
2100 }
2101
2102 if (flags & CFLGS_OFF_SLAB) {
2103 /* really off slab. No need for manual alignment */
Pekka Enbergb28a02d2006-01-08 01:00:37 -08002104 slab_size =
2105 cachep->num * sizeof(kmem_bufctl_t) + sizeof(struct slab);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002106 }
2107
2108 cachep->colour_off = cache_line_size();
2109 /* Offset must be a multiple of the alignment. */
2110 if (cachep->colour_off < align)
2111 cachep->colour_off = align;
Pekka Enbergb28a02d2006-01-08 01:00:37 -08002112 cachep->colour = left_over / cachep->colour_off;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002113 cachep->slab_size = slab_size;
2114 cachep->flags = flags;
2115 cachep->gfpflags = 0;
2116 if (flags & SLAB_CACHE_DMA)
2117 cachep->gfpflags |= GFP_DMA;
Manfred Spraul3dafccf2006-02-01 03:05:42 -08002118 cachep->buffer_size = size;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002119
2120 if (flags & CFLGS_OFF_SLAB)
Victor Fuscob2d55072005-09-10 00:26:36 -07002121 cachep->slabp_cache = kmem_find_general_cachep(slab_size, 0u);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002122 cachep->ctor = ctor;
2123 cachep->dtor = dtor;
2124 cachep->name = name;
2125
Linus Torvalds1da177e2005-04-16 15:20:36 -07002126
Pekka Enbergf30cf7d2006-03-22 00:08:11 -08002127 setup_cpu_cache(cachep);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002128
Linus Torvalds1da177e2005-04-16 15:20:36 -07002129 /* cache setup completed, link it into the list */
2130 list_add(&cachep->next, &cache_chain);
Andrew Mortona737b3e2006-03-22 00:08:11 -08002131oops:
Linus Torvalds1da177e2005-04-16 15:20:36 -07002132 if (!cachep && (flags & SLAB_PANIC))
2133 panic("kmem_cache_create(): failed to create slab `%s'\n",
Pekka Enbergb28a02d2006-01-08 01:00:37 -08002134 name);
Ingo Molnarfc0abb12006-01-18 17:42:33 -08002135 mutex_unlock(&cache_chain_mutex);
Ravikiran G Thirumalaif0188f42006-02-10 01:51:13 -08002136 unlock_cpu_hotplug();
Linus Torvalds1da177e2005-04-16 15:20:36 -07002137 return cachep;
2138}
2139EXPORT_SYMBOL(kmem_cache_create);
2140
2141#if DEBUG
2142static void check_irq_off(void)
2143{
2144 BUG_ON(!irqs_disabled());
2145}
2146
2147static void check_irq_on(void)
2148{
2149 BUG_ON(irqs_disabled());
2150}
2151
Pekka Enberg343e0d72006-02-01 03:05:50 -08002152static void check_spinlock_acquired(struct kmem_cache *cachep)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002153{
2154#ifdef CONFIG_SMP
2155 check_irq_off();
Christoph Lametere498be72005-09-09 13:03:32 -07002156 assert_spin_locked(&cachep->nodelists[numa_node_id()]->list_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002157#endif
2158}
Christoph Lametere498be72005-09-09 13:03:32 -07002159
Pekka Enberg343e0d72006-02-01 03:05:50 -08002160static void check_spinlock_acquired_node(struct kmem_cache *cachep, int node)
Christoph Lametere498be72005-09-09 13:03:32 -07002161{
2162#ifdef CONFIG_SMP
2163 check_irq_off();
2164 assert_spin_locked(&cachep->nodelists[node]->list_lock);
2165#endif
2166}
2167
Linus Torvalds1da177e2005-04-16 15:20:36 -07002168#else
2169#define check_irq_off() do { } while(0)
2170#define check_irq_on() do { } while(0)
2171#define check_spinlock_acquired(x) do { } while(0)
Christoph Lametere498be72005-09-09 13:03:32 -07002172#define check_spinlock_acquired_node(x, y) do { } while(0)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002173#endif
2174
Christoph Lameteraab22072006-03-22 00:09:06 -08002175static void drain_array(struct kmem_cache *cachep, struct kmem_list3 *l3,
2176 struct array_cache *ac,
2177 int force, int node);
2178
Linus Torvalds1da177e2005-04-16 15:20:36 -07002179static void do_drain(void *arg)
2180{
Andrew Mortona737b3e2006-03-22 00:08:11 -08002181 struct kmem_cache *cachep = arg;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002182 struct array_cache *ac;
Christoph Lameterff694162005-09-22 21:44:02 -07002183 int node = numa_node_id();
Linus Torvalds1da177e2005-04-16 15:20:36 -07002184
2185 check_irq_off();
Pekka Enberg9a2dba42006-02-01 03:05:49 -08002186 ac = cpu_cache_get(cachep);
Christoph Lameterff694162005-09-22 21:44:02 -07002187 spin_lock(&cachep->nodelists[node]->list_lock);
2188 free_block(cachep, ac->entry, ac->avail, node);
2189 spin_unlock(&cachep->nodelists[node]->list_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002190 ac->avail = 0;
2191}
2192
Pekka Enberg343e0d72006-02-01 03:05:50 -08002193static void drain_cpu_caches(struct kmem_cache *cachep)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002194{
Christoph Lametere498be72005-09-09 13:03:32 -07002195 struct kmem_list3 *l3;
2196 int node;
2197
Andrew Mortona07fa392006-03-22 00:08:17 -08002198 on_each_cpu(do_drain, cachep, 1, 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002199 check_irq_on();
Pekka Enbergb28a02d2006-01-08 01:00:37 -08002200 for_each_online_node(node) {
Christoph Lametere498be72005-09-09 13:03:32 -07002201 l3 = cachep->nodelists[node];
Roland Dreiera4523a82006-05-15 11:41:00 -07002202 if (l3 && l3->alien)
2203 drain_alien_cache(cachep, l3->alien);
2204 }
2205
2206 for_each_online_node(node) {
2207 l3 = cachep->nodelists[node];
2208 if (l3)
Christoph Lameteraab22072006-03-22 00:09:06 -08002209 drain_array(cachep, l3, l3->shared, 1, node);
Christoph Lametere498be72005-09-09 13:03:32 -07002210 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002211}
2212
Pekka Enberg343e0d72006-02-01 03:05:50 -08002213static int __node_shrink(struct kmem_cache *cachep, int node)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002214{
2215 struct slab *slabp;
Christoph Lametere498be72005-09-09 13:03:32 -07002216 struct kmem_list3 *l3 = cachep->nodelists[node];
Linus Torvalds1da177e2005-04-16 15:20:36 -07002217 int ret;
2218
Christoph Lametere498be72005-09-09 13:03:32 -07002219 for (;;) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07002220 struct list_head *p;
2221
Christoph Lametere498be72005-09-09 13:03:32 -07002222 p = l3->slabs_free.prev;
2223 if (p == &l3->slabs_free)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002224 break;
2225
Christoph Lametere498be72005-09-09 13:03:32 -07002226 slabp = list_entry(l3->slabs_free.prev, struct slab, list);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002227#if DEBUG
Eric Sesterhenn40094fa2006-04-02 13:49:25 +02002228 BUG_ON(slabp->inuse);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002229#endif
2230 list_del(&slabp->list);
2231
Christoph Lametere498be72005-09-09 13:03:32 -07002232 l3->free_objects -= cachep->num;
2233 spin_unlock_irq(&l3->list_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002234 slab_destroy(cachep, slabp);
Christoph Lametere498be72005-09-09 13:03:32 -07002235 spin_lock_irq(&l3->list_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002236 }
Pekka Enbergb28a02d2006-01-08 01:00:37 -08002237 ret = !list_empty(&l3->slabs_full) || !list_empty(&l3->slabs_partial);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002238 return ret;
2239}
2240
Pekka Enberg343e0d72006-02-01 03:05:50 -08002241static int __cache_shrink(struct kmem_cache *cachep)
Christoph Lametere498be72005-09-09 13:03:32 -07002242{
2243 int ret = 0, i = 0;
2244 struct kmem_list3 *l3;
2245
2246 drain_cpu_caches(cachep);
2247
2248 check_irq_on();
2249 for_each_online_node(i) {
2250 l3 = cachep->nodelists[i];
2251 if (l3) {
2252 spin_lock_irq(&l3->list_lock);
2253 ret += __node_shrink(cachep, i);
2254 spin_unlock_irq(&l3->list_lock);
2255 }
2256 }
2257 return (ret ? 1 : 0);
2258}
2259
Linus Torvalds1da177e2005-04-16 15:20:36 -07002260/**
2261 * kmem_cache_shrink - Shrink a cache.
2262 * @cachep: The cache to shrink.
2263 *
2264 * Releases as many slabs as possible for a cache.
2265 * To help debugging, a zero exit status indicates all slabs were released.
2266 */
Pekka Enberg343e0d72006-02-01 03:05:50 -08002267int kmem_cache_shrink(struct kmem_cache *cachep)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002268{
Eric Sesterhenn40094fa2006-04-02 13:49:25 +02002269 BUG_ON(!cachep || in_interrupt());
Linus Torvalds1da177e2005-04-16 15:20:36 -07002270
2271 return __cache_shrink(cachep);
2272}
2273EXPORT_SYMBOL(kmem_cache_shrink);
2274
2275/**
2276 * kmem_cache_destroy - delete a cache
2277 * @cachep: the cache to destroy
2278 *
Pekka Enberg343e0d72006-02-01 03:05:50 -08002279 * Remove a struct kmem_cache object from the slab cache.
Linus Torvalds1da177e2005-04-16 15:20:36 -07002280 * Returns 0 on success.
2281 *
2282 * It is expected this function will be called by a module when it is
2283 * unloaded. This will remove the cache completely, and avoid a duplicate
2284 * cache being allocated each time a module is loaded and unloaded, if the
2285 * module doesn't have persistent in-kernel storage across loads and unloads.
2286 *
2287 * The cache must be empty before calling this function.
2288 *
2289 * The caller must guarantee that noone will allocate memory from the cache
2290 * during the kmem_cache_destroy().
2291 */
Pekka Enberg343e0d72006-02-01 03:05:50 -08002292int kmem_cache_destroy(struct kmem_cache *cachep)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002293{
2294 int i;
Christoph Lametere498be72005-09-09 13:03:32 -07002295 struct kmem_list3 *l3;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002296
Eric Sesterhenn40094fa2006-04-02 13:49:25 +02002297 BUG_ON(!cachep || in_interrupt());
Linus Torvalds1da177e2005-04-16 15:20:36 -07002298
2299 /* Don't let CPUs to come and go */
2300 lock_cpu_hotplug();
2301
2302 /* Find the cache in the chain of caches. */
Ingo Molnarfc0abb12006-01-18 17:42:33 -08002303 mutex_lock(&cache_chain_mutex);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002304 /*
2305 * the chain is never empty, cache_cache is never destroyed
2306 */
2307 list_del(&cachep->next);
Ingo Molnarfc0abb12006-01-18 17:42:33 -08002308 mutex_unlock(&cache_chain_mutex);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002309
2310 if (__cache_shrink(cachep)) {
2311 slab_error(cachep, "Can't free all objects");
Ingo Molnarfc0abb12006-01-18 17:42:33 -08002312 mutex_lock(&cache_chain_mutex);
Pekka Enbergb28a02d2006-01-08 01:00:37 -08002313 list_add(&cachep->next, &cache_chain);
Ingo Molnarfc0abb12006-01-18 17:42:33 -08002314 mutex_unlock(&cache_chain_mutex);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002315 unlock_cpu_hotplug();
2316 return 1;
2317 }
2318
2319 if (unlikely(cachep->flags & SLAB_DESTROY_BY_RCU))
Paul E. McKenneyfbd568a3e2005-05-01 08:59:04 -07002320 synchronize_rcu();
Linus Torvalds1da177e2005-04-16 15:20:36 -07002321
Christoph Lametere498be72005-09-09 13:03:32 -07002322 for_each_online_cpu(i)
Pekka Enbergb28a02d2006-01-08 01:00:37 -08002323 kfree(cachep->array[i]);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002324
2325 /* NUMA: free the list3 structures */
Christoph Lametere498be72005-09-09 13:03:32 -07002326 for_each_online_node(i) {
Andrew Mortona737b3e2006-03-22 00:08:11 -08002327 l3 = cachep->nodelists[i];
2328 if (l3) {
Christoph Lametere498be72005-09-09 13:03:32 -07002329 kfree(l3->shared);
2330 free_alien_cache(l3->alien);
2331 kfree(l3);
2332 }
2333 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002334 kmem_cache_free(&cache_cache, cachep);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002335 unlock_cpu_hotplug();
Linus Torvalds1da177e2005-04-16 15:20:36 -07002336 return 0;
2337}
2338EXPORT_SYMBOL(kmem_cache_destroy);
2339
2340/* Get the memory for a slab management obj. */
Pekka Enberg343e0d72006-02-01 03:05:50 -08002341static struct slab *alloc_slabmgmt(struct kmem_cache *cachep, void *objp,
Ravikiran G Thirumalai5b74ada2006-04-10 22:52:53 -07002342 int colour_off, gfp_t local_flags,
2343 int nodeid)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002344{
2345 struct slab *slabp;
Pekka Enbergb28a02d2006-01-08 01:00:37 -08002346
Linus Torvalds1da177e2005-04-16 15:20:36 -07002347 if (OFF_SLAB(cachep)) {
2348 /* Slab management obj is off-slab. */
Ravikiran G Thirumalai5b74ada2006-04-10 22:52:53 -07002349 slabp = kmem_cache_alloc_node(cachep->slabp_cache,
2350 local_flags, nodeid);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002351 if (!slabp)
2352 return NULL;
2353 } else {
Pekka Enbergb28a02d2006-01-08 01:00:37 -08002354 slabp = objp + colour_off;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002355 colour_off += cachep->slab_size;
2356 }
2357 slabp->inuse = 0;
2358 slabp->colouroff = colour_off;
Pekka Enbergb28a02d2006-01-08 01:00:37 -08002359 slabp->s_mem = objp + colour_off;
Ravikiran G Thirumalai5b74ada2006-04-10 22:52:53 -07002360 slabp->nodeid = nodeid;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002361 return slabp;
2362}
2363
2364static inline kmem_bufctl_t *slab_bufctl(struct slab *slabp)
2365{
Pekka Enbergb28a02d2006-01-08 01:00:37 -08002366 return (kmem_bufctl_t *) (slabp + 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002367}
2368
Pekka Enberg343e0d72006-02-01 03:05:50 -08002369static void cache_init_objs(struct kmem_cache *cachep,
Pekka Enbergb28a02d2006-01-08 01:00:37 -08002370 struct slab *slabp, unsigned long ctor_flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002371{
2372 int i;
2373
2374 for (i = 0; i < cachep->num; i++) {
Pekka Enberg8fea4e92006-03-22 00:08:10 -08002375 void *objp = index_to_obj(cachep, slabp, i);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002376#if DEBUG
2377 /* need to poison the objs? */
2378 if (cachep->flags & SLAB_POISON)
2379 poison_obj(cachep, objp, POISON_FREE);
2380 if (cachep->flags & SLAB_STORE_USER)
2381 *dbg_userword(cachep, objp) = NULL;
2382
2383 if (cachep->flags & SLAB_RED_ZONE) {
2384 *dbg_redzone1(cachep, objp) = RED_INACTIVE;
2385 *dbg_redzone2(cachep, objp) = RED_INACTIVE;
2386 }
2387 /*
Andrew Mortona737b3e2006-03-22 00:08:11 -08002388 * Constructors are not allowed to allocate memory from the same
2389 * cache which they are a constructor for. Otherwise, deadlock.
2390 * They must also be threaded.
Linus Torvalds1da177e2005-04-16 15:20:36 -07002391 */
2392 if (cachep->ctor && !(cachep->flags & SLAB_POISON))
Manfred Spraul3dafccf2006-02-01 03:05:42 -08002393 cachep->ctor(objp + obj_offset(cachep), cachep,
Pekka Enbergb28a02d2006-01-08 01:00:37 -08002394 ctor_flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002395
2396 if (cachep->flags & SLAB_RED_ZONE) {
2397 if (*dbg_redzone2(cachep, objp) != RED_INACTIVE)
2398 slab_error(cachep, "constructor overwrote the"
Pekka Enbergb28a02d2006-01-08 01:00:37 -08002399 " end of an object");
Linus Torvalds1da177e2005-04-16 15:20:36 -07002400 if (*dbg_redzone1(cachep, objp) != RED_INACTIVE)
2401 slab_error(cachep, "constructor overwrote the"
Pekka Enbergb28a02d2006-01-08 01:00:37 -08002402 " start of an object");
Linus Torvalds1da177e2005-04-16 15:20:36 -07002403 }
Andrew Mortona737b3e2006-03-22 00:08:11 -08002404 if ((cachep->buffer_size % PAGE_SIZE) == 0 &&
2405 OFF_SLAB(cachep) && cachep->flags & SLAB_POISON)
Pekka Enbergb28a02d2006-01-08 01:00:37 -08002406 kernel_map_pages(virt_to_page(objp),
Manfred Spraul3dafccf2006-02-01 03:05:42 -08002407 cachep->buffer_size / PAGE_SIZE, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002408#else
2409 if (cachep->ctor)
2410 cachep->ctor(objp, cachep, ctor_flags);
2411#endif
Pekka Enbergb28a02d2006-01-08 01:00:37 -08002412 slab_bufctl(slabp)[i] = i + 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002413 }
Pekka Enbergb28a02d2006-01-08 01:00:37 -08002414 slab_bufctl(slabp)[i - 1] = BUFCTL_END;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002415 slabp->free = 0;
2416}
2417
Pekka Enberg343e0d72006-02-01 03:05:50 -08002418static void kmem_flagcheck(struct kmem_cache *cachep, gfp_t flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002419{
Andrew Mortona737b3e2006-03-22 00:08:11 -08002420 if (flags & SLAB_DMA)
2421 BUG_ON(!(cachep->gfpflags & GFP_DMA));
2422 else
2423 BUG_ON(cachep->gfpflags & GFP_DMA);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002424}
2425
Andrew Mortona737b3e2006-03-22 00:08:11 -08002426static void *slab_get_obj(struct kmem_cache *cachep, struct slab *slabp,
2427 int nodeid)
Matthew Dobson78d382d2006-02-01 03:05:47 -08002428{
Pekka Enberg8fea4e92006-03-22 00:08:10 -08002429 void *objp = index_to_obj(cachep, slabp, slabp->free);
Matthew Dobson78d382d2006-02-01 03:05:47 -08002430 kmem_bufctl_t next;
2431
2432 slabp->inuse++;
2433 next = slab_bufctl(slabp)[slabp->free];
2434#if DEBUG
2435 slab_bufctl(slabp)[slabp->free] = BUFCTL_FREE;
2436 WARN_ON(slabp->nodeid != nodeid);
2437#endif
2438 slabp->free = next;
2439
2440 return objp;
2441}
2442
Andrew Mortona737b3e2006-03-22 00:08:11 -08002443static void slab_put_obj(struct kmem_cache *cachep, struct slab *slabp,
2444 void *objp, int nodeid)
Matthew Dobson78d382d2006-02-01 03:05:47 -08002445{
Pekka Enberg8fea4e92006-03-22 00:08:10 -08002446 unsigned int objnr = obj_to_index(cachep, slabp, objp);
Matthew Dobson78d382d2006-02-01 03:05:47 -08002447
2448#if DEBUG
2449 /* Verify that the slab belongs to the intended node */
2450 WARN_ON(slabp->nodeid != nodeid);
2451
Al Viro871751e2006-03-25 03:06:39 -08002452 if (slab_bufctl(slabp)[objnr] + 1 <= SLAB_LIMIT + 1) {
Matthew Dobson78d382d2006-02-01 03:05:47 -08002453 printk(KERN_ERR "slab: double free detected in cache "
Andrew Mortona737b3e2006-03-22 00:08:11 -08002454 "'%s', objp %p\n", cachep->name, objp);
Matthew Dobson78d382d2006-02-01 03:05:47 -08002455 BUG();
2456 }
2457#endif
2458 slab_bufctl(slabp)[objnr] = slabp->free;
2459 slabp->free = objnr;
2460 slabp->inuse--;
2461}
2462
Andrew Mortona737b3e2006-03-22 00:08:11 -08002463static void set_slab_attr(struct kmem_cache *cachep, struct slab *slabp,
2464 void *objp)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002465{
2466 int i;
2467 struct page *page;
2468
2469 /* Nasty!!!!!! I hope this is OK. */
Linus Torvalds1da177e2005-04-16 15:20:36 -07002470 page = virt_to_page(objp);
Nick Piggin84097512006-03-22 00:08:34 -08002471
2472 i = 1;
2473 if (likely(!PageCompound(page)))
2474 i <<= cachep->gfporder;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002475 do {
Pekka Enberg065d41c2005-11-13 16:06:46 -08002476 page_set_cache(page, cachep);
2477 page_set_slab(page, slabp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002478 page++;
2479 } while (--i);
2480}
2481
2482/*
2483 * Grow (by 1) the number of slabs within a cache. This is called by
2484 * kmem_cache_alloc() when there are no active objs left in a cache.
2485 */
Pekka Enberg343e0d72006-02-01 03:05:50 -08002486static int cache_grow(struct kmem_cache *cachep, gfp_t flags, int nodeid)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002487{
Pekka Enbergb28a02d2006-01-08 01:00:37 -08002488 struct slab *slabp;
2489 void *objp;
2490 size_t offset;
2491 gfp_t local_flags;
2492 unsigned long ctor_flags;
Christoph Lametere498be72005-09-09 13:03:32 -07002493 struct kmem_list3 *l3;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002494
Andrew Mortona737b3e2006-03-22 00:08:11 -08002495 /*
2496 * Be lazy and only check for valid flags here, keeping it out of the
2497 * critical path in kmem_cache_alloc().
Linus Torvalds1da177e2005-04-16 15:20:36 -07002498 */
Eric Sesterhenn40094fa2006-04-02 13:49:25 +02002499 BUG_ON(flags & ~(SLAB_DMA | SLAB_LEVEL_MASK | SLAB_NO_GROW));
Linus Torvalds1da177e2005-04-16 15:20:36 -07002500 if (flags & SLAB_NO_GROW)
2501 return 0;
2502
2503 ctor_flags = SLAB_CTOR_CONSTRUCTOR;
2504 local_flags = (flags & SLAB_LEVEL_MASK);
2505 if (!(local_flags & __GFP_WAIT))
2506 /*
2507 * Not allowed to sleep. Need to tell a constructor about
2508 * this - it might need to know...
2509 */
2510 ctor_flags |= SLAB_CTOR_ATOMIC;
2511
Ravikiran G Thirumalai2e1217c2006-02-04 23:27:56 -08002512 /* Take the l3 list lock to change the colour_next on this node */
Linus Torvalds1da177e2005-04-16 15:20:36 -07002513 check_irq_off();
Ravikiran G Thirumalai2e1217c2006-02-04 23:27:56 -08002514 l3 = cachep->nodelists[nodeid];
2515 spin_lock(&l3->list_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002516
2517 /* Get colour for the slab, and cal the next value. */
Ravikiran G Thirumalai2e1217c2006-02-04 23:27:56 -08002518 offset = l3->colour_next;
2519 l3->colour_next++;
2520 if (l3->colour_next >= cachep->colour)
2521 l3->colour_next = 0;
2522 spin_unlock(&l3->list_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002523
Ravikiran G Thirumalai2e1217c2006-02-04 23:27:56 -08002524 offset *= cachep->colour_off;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002525
2526 if (local_flags & __GFP_WAIT)
2527 local_irq_enable();
2528
2529 /*
2530 * The test for missing atomic flag is performed here, rather than
2531 * the more obvious place, simply to reduce the critical path length
2532 * in kmem_cache_alloc(). If a caller is seriously mis-behaving they
2533 * will eventually be caught here (where it matters).
2534 */
2535 kmem_flagcheck(cachep, flags);
2536
Andrew Mortona737b3e2006-03-22 00:08:11 -08002537 /*
2538 * Get mem for the objs. Attempt to allocate a physical page from
2539 * 'nodeid'.
Christoph Lametere498be72005-09-09 13:03:32 -07002540 */
Andrew Mortona737b3e2006-03-22 00:08:11 -08002541 objp = kmem_getpages(cachep, flags, nodeid);
2542 if (!objp)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002543 goto failed;
2544
2545 /* Get slab management. */
Ravikiran G Thirumalai5b74ada2006-04-10 22:52:53 -07002546 slabp = alloc_slabmgmt(cachep, objp, offset, local_flags, nodeid);
Andrew Mortona737b3e2006-03-22 00:08:11 -08002547 if (!slabp)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002548 goto opps1;
2549
Christoph Lametere498be72005-09-09 13:03:32 -07002550 slabp->nodeid = nodeid;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002551 set_slab_attr(cachep, slabp, objp);
2552
2553 cache_init_objs(cachep, slabp, ctor_flags);
2554
2555 if (local_flags & __GFP_WAIT)
2556 local_irq_disable();
2557 check_irq_off();
Christoph Lametere498be72005-09-09 13:03:32 -07002558 spin_lock(&l3->list_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002559
2560 /* Make slab active. */
Christoph Lametere498be72005-09-09 13:03:32 -07002561 list_add_tail(&slabp->list, &(l3->slabs_free));
Linus Torvalds1da177e2005-04-16 15:20:36 -07002562 STATS_INC_GROWN(cachep);
Christoph Lametere498be72005-09-09 13:03:32 -07002563 l3->free_objects += cachep->num;
2564 spin_unlock(&l3->list_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002565 return 1;
Andrew Mortona737b3e2006-03-22 00:08:11 -08002566opps1:
Linus Torvalds1da177e2005-04-16 15:20:36 -07002567 kmem_freepages(cachep, objp);
Andrew Mortona737b3e2006-03-22 00:08:11 -08002568failed:
Linus Torvalds1da177e2005-04-16 15:20:36 -07002569 if (local_flags & __GFP_WAIT)
2570 local_irq_disable();
2571 return 0;
2572}
2573
2574#if DEBUG
2575
2576/*
2577 * Perform extra freeing checks:
2578 * - detect bad pointers.
2579 * - POISON/RED_ZONE checking
2580 * - destructor calls, for caches with POISON+dtor
2581 */
2582static void kfree_debugcheck(const void *objp)
2583{
2584 struct page *page;
2585
2586 if (!virt_addr_valid(objp)) {
2587 printk(KERN_ERR "kfree_debugcheck: out of range ptr %lxh.\n",
Pekka Enbergb28a02d2006-01-08 01:00:37 -08002588 (unsigned long)objp);
2589 BUG();
Linus Torvalds1da177e2005-04-16 15:20:36 -07002590 }
2591 page = virt_to_page(objp);
2592 if (!PageSlab(page)) {
Pekka Enbergb28a02d2006-01-08 01:00:37 -08002593 printk(KERN_ERR "kfree_debugcheck: bad ptr %lxh.\n",
2594 (unsigned long)objp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002595 BUG();
2596 }
2597}
2598
Pekka Enberg343e0d72006-02-01 03:05:50 -08002599static void *cache_free_debugcheck(struct kmem_cache *cachep, void *objp,
Pekka Enbergb28a02d2006-01-08 01:00:37 -08002600 void *caller)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002601{
2602 struct page *page;
2603 unsigned int objnr;
2604 struct slab *slabp;
2605
Manfred Spraul3dafccf2006-02-01 03:05:42 -08002606 objp -= obj_offset(cachep);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002607 kfree_debugcheck(objp);
2608 page = virt_to_page(objp);
2609
Pekka Enberg065d41c2005-11-13 16:06:46 -08002610 if (page_get_cache(page) != cachep) {
Andrew Mortona737b3e2006-03-22 00:08:11 -08002611 printk(KERN_ERR "mismatch in kmem_cache_free: expected "
2612 "cache %p, got %p\n",
Pekka Enbergb28a02d2006-01-08 01:00:37 -08002613 page_get_cache(page), cachep);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002614 printk(KERN_ERR "%p is %s.\n", cachep, cachep->name);
Pekka Enbergb28a02d2006-01-08 01:00:37 -08002615 printk(KERN_ERR "%p is %s.\n", page_get_cache(page),
2616 page_get_cache(page)->name);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002617 WARN_ON(1);
2618 }
Pekka Enberg065d41c2005-11-13 16:06:46 -08002619 slabp = page_get_slab(page);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002620
2621 if (cachep->flags & SLAB_RED_ZONE) {
Andrew Mortona737b3e2006-03-22 00:08:11 -08002622 if (*dbg_redzone1(cachep, objp) != RED_ACTIVE ||
2623 *dbg_redzone2(cachep, objp) != RED_ACTIVE) {
2624 slab_error(cachep, "double free, or memory outside"
2625 " object was overwritten");
2626 printk(KERN_ERR "%p: redzone 1:0x%lx, "
2627 "redzone 2:0x%lx.\n",
Pekka Enbergb28a02d2006-01-08 01:00:37 -08002628 objp, *dbg_redzone1(cachep, objp),
2629 *dbg_redzone2(cachep, objp));
Linus Torvalds1da177e2005-04-16 15:20:36 -07002630 }
2631 *dbg_redzone1(cachep, objp) = RED_INACTIVE;
2632 *dbg_redzone2(cachep, objp) = RED_INACTIVE;
2633 }
2634 if (cachep->flags & SLAB_STORE_USER)
2635 *dbg_userword(cachep, objp) = caller;
2636
Pekka Enberg8fea4e92006-03-22 00:08:10 -08002637 objnr = obj_to_index(cachep, slabp, objp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002638
2639 BUG_ON(objnr >= cachep->num);
Pekka Enberg8fea4e92006-03-22 00:08:10 -08002640 BUG_ON(objp != index_to_obj(cachep, slabp, objnr));
Linus Torvalds1da177e2005-04-16 15:20:36 -07002641
2642 if (cachep->flags & SLAB_DEBUG_INITIAL) {
Andrew Mortona737b3e2006-03-22 00:08:11 -08002643 /*
2644 * Need to call the slab's constructor so the caller can
2645 * perform a verify of its state (debugging). Called without
2646 * the cache-lock held.
Linus Torvalds1da177e2005-04-16 15:20:36 -07002647 */
Manfred Spraul3dafccf2006-02-01 03:05:42 -08002648 cachep->ctor(objp + obj_offset(cachep),
Pekka Enbergb28a02d2006-01-08 01:00:37 -08002649 cachep, SLAB_CTOR_CONSTRUCTOR | SLAB_CTOR_VERIFY);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002650 }
2651 if (cachep->flags & SLAB_POISON && cachep->dtor) {
2652 /* we want to cache poison the object,
2653 * call the destruction callback
2654 */
Manfred Spraul3dafccf2006-02-01 03:05:42 -08002655 cachep->dtor(objp + obj_offset(cachep), cachep, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002656 }
Al Viro871751e2006-03-25 03:06:39 -08002657#ifdef CONFIG_DEBUG_SLAB_LEAK
2658 slab_bufctl(slabp)[objnr] = BUFCTL_FREE;
2659#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07002660 if (cachep->flags & SLAB_POISON) {
2661#ifdef CONFIG_DEBUG_PAGEALLOC
Andrew Mortona737b3e2006-03-22 00:08:11 -08002662 if ((cachep->buffer_size % PAGE_SIZE)==0 && OFF_SLAB(cachep)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07002663 store_stackinfo(cachep, objp, (unsigned long)caller);
Pekka Enbergb28a02d2006-01-08 01:00:37 -08002664 kernel_map_pages(virt_to_page(objp),
Manfred Spraul3dafccf2006-02-01 03:05:42 -08002665 cachep->buffer_size / PAGE_SIZE, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002666 } else {
2667 poison_obj(cachep, objp, POISON_FREE);
2668 }
2669#else
2670 poison_obj(cachep, objp, POISON_FREE);
2671#endif
2672 }
2673 return objp;
2674}
2675
Pekka Enberg343e0d72006-02-01 03:05:50 -08002676static void check_slabp(struct kmem_cache *cachep, struct slab *slabp)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002677{
2678 kmem_bufctl_t i;
2679 int entries = 0;
Pekka Enbergb28a02d2006-01-08 01:00:37 -08002680
Linus Torvalds1da177e2005-04-16 15:20:36 -07002681 /* Check slab's freelist to see if this obj is there. */
2682 for (i = slabp->free; i != BUFCTL_END; i = slab_bufctl(slabp)[i]) {
2683 entries++;
2684 if (entries > cachep->num || i >= cachep->num)
2685 goto bad;
2686 }
2687 if (entries != cachep->num - slabp->inuse) {
Andrew Mortona737b3e2006-03-22 00:08:11 -08002688bad:
2689 printk(KERN_ERR "slab: Internal list corruption detected in "
2690 "cache '%s'(%d), slabp %p(%d). Hexdump:\n",
2691 cachep->name, cachep->num, slabp, slabp->inuse);
Pekka Enbergb28a02d2006-01-08 01:00:37 -08002692 for (i = 0;
Linus Torvalds264132b2006-03-06 12:10:07 -08002693 i < sizeof(*slabp) + cachep->num * sizeof(kmem_bufctl_t);
Pekka Enbergb28a02d2006-01-08 01:00:37 -08002694 i++) {
Andrew Mortona737b3e2006-03-22 00:08:11 -08002695 if (i % 16 == 0)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002696 printk("\n%03x:", i);
Pekka Enbergb28a02d2006-01-08 01:00:37 -08002697 printk(" %02x", ((unsigned char *)slabp)[i]);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002698 }
2699 printk("\n");
2700 BUG();
2701 }
2702}
2703#else
2704#define kfree_debugcheck(x) do { } while(0)
2705#define cache_free_debugcheck(x,objp,z) (objp)
2706#define check_slabp(x,y) do { } while(0)
2707#endif
2708
Pekka Enberg343e0d72006-02-01 03:05:50 -08002709static void *cache_alloc_refill(struct kmem_cache *cachep, gfp_t flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002710{
2711 int batchcount;
2712 struct kmem_list3 *l3;
2713 struct array_cache *ac;
2714
2715 check_irq_off();
Pekka Enberg9a2dba42006-02-01 03:05:49 -08002716 ac = cpu_cache_get(cachep);
Andrew Mortona737b3e2006-03-22 00:08:11 -08002717retry:
Linus Torvalds1da177e2005-04-16 15:20:36 -07002718 batchcount = ac->batchcount;
2719 if (!ac->touched && batchcount > BATCHREFILL_LIMIT) {
Andrew Mortona737b3e2006-03-22 00:08:11 -08002720 /*
2721 * If there was little recent activity on this cache, then
2722 * perform only a partial refill. Otherwise we could generate
2723 * refill bouncing.
Linus Torvalds1da177e2005-04-16 15:20:36 -07002724 */
2725 batchcount = BATCHREFILL_LIMIT;
2726 }
Christoph Lametere498be72005-09-09 13:03:32 -07002727 l3 = cachep->nodelists[numa_node_id()];
Linus Torvalds1da177e2005-04-16 15:20:36 -07002728
Christoph Lametere498be72005-09-09 13:03:32 -07002729 BUG_ON(ac->avail > 0 || !l3);
2730 spin_lock(&l3->list_lock);
2731
Christoph Lameter3ded1752006-03-25 03:06:44 -08002732 /* See if we can refill from the shared array */
2733 if (l3->shared && transfer_objects(ac, l3->shared, batchcount))
2734 goto alloc_done;
2735
Linus Torvalds1da177e2005-04-16 15:20:36 -07002736 while (batchcount > 0) {
2737 struct list_head *entry;
2738 struct slab *slabp;
2739 /* Get slab alloc is to come from. */
2740 entry = l3->slabs_partial.next;
2741 if (entry == &l3->slabs_partial) {
2742 l3->free_touched = 1;
2743 entry = l3->slabs_free.next;
2744 if (entry == &l3->slabs_free)
2745 goto must_grow;
2746 }
2747
2748 slabp = list_entry(entry, struct slab, list);
2749 check_slabp(cachep, slabp);
2750 check_spinlock_acquired(cachep);
2751 while (slabp->inuse < cachep->num && batchcount--) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07002752 STATS_INC_ALLOCED(cachep);
2753 STATS_INC_ACTIVE(cachep);
2754 STATS_SET_HIGH(cachep);
2755
Matthew Dobson78d382d2006-02-01 03:05:47 -08002756 ac->entry[ac->avail++] = slab_get_obj(cachep, slabp,
2757 numa_node_id());
Linus Torvalds1da177e2005-04-16 15:20:36 -07002758 }
2759 check_slabp(cachep, slabp);
2760
2761 /* move slabp to correct slabp list: */
2762 list_del(&slabp->list);
2763 if (slabp->free == BUFCTL_END)
2764 list_add(&slabp->list, &l3->slabs_full);
2765 else
2766 list_add(&slabp->list, &l3->slabs_partial);
2767 }
2768
Andrew Mortona737b3e2006-03-22 00:08:11 -08002769must_grow:
Linus Torvalds1da177e2005-04-16 15:20:36 -07002770 l3->free_objects -= ac->avail;
Andrew Mortona737b3e2006-03-22 00:08:11 -08002771alloc_done:
Christoph Lametere498be72005-09-09 13:03:32 -07002772 spin_unlock(&l3->list_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002773
2774 if (unlikely(!ac->avail)) {
2775 int x;
Christoph Lametere498be72005-09-09 13:03:32 -07002776 x = cache_grow(cachep, flags, numa_node_id());
2777
Andrew Mortona737b3e2006-03-22 00:08:11 -08002778 /* cache_grow can reenable interrupts, then ac could change. */
Pekka Enberg9a2dba42006-02-01 03:05:49 -08002779 ac = cpu_cache_get(cachep);
Andrew Mortona737b3e2006-03-22 00:08:11 -08002780 if (!x && ac->avail == 0) /* no objects in sight? abort */
Linus Torvalds1da177e2005-04-16 15:20:36 -07002781 return NULL;
2782
Andrew Mortona737b3e2006-03-22 00:08:11 -08002783 if (!ac->avail) /* objects refilled by interrupt? */
Linus Torvalds1da177e2005-04-16 15:20:36 -07002784 goto retry;
2785 }
2786 ac->touched = 1;
Christoph Lametere498be72005-09-09 13:03:32 -07002787 return ac->entry[--ac->avail];
Linus Torvalds1da177e2005-04-16 15:20:36 -07002788}
2789
Andrew Mortona737b3e2006-03-22 00:08:11 -08002790static inline void cache_alloc_debugcheck_before(struct kmem_cache *cachep,
2791 gfp_t flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002792{
2793 might_sleep_if(flags & __GFP_WAIT);
2794#if DEBUG
2795 kmem_flagcheck(cachep, flags);
2796#endif
2797}
2798
2799#if DEBUG
Andrew Mortona737b3e2006-03-22 00:08:11 -08002800static void *cache_alloc_debugcheck_after(struct kmem_cache *cachep,
2801 gfp_t flags, void *objp, void *caller)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002802{
Pekka Enbergb28a02d2006-01-08 01:00:37 -08002803 if (!objp)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002804 return objp;
Pekka Enbergb28a02d2006-01-08 01:00:37 -08002805 if (cachep->flags & SLAB_POISON) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07002806#ifdef CONFIG_DEBUG_PAGEALLOC
Manfred Spraul3dafccf2006-02-01 03:05:42 -08002807 if ((cachep->buffer_size % PAGE_SIZE) == 0 && OFF_SLAB(cachep))
Pekka Enbergb28a02d2006-01-08 01:00:37 -08002808 kernel_map_pages(virt_to_page(objp),
Manfred Spraul3dafccf2006-02-01 03:05:42 -08002809 cachep->buffer_size / PAGE_SIZE, 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002810 else
2811 check_poison_obj(cachep, objp);
2812#else
2813 check_poison_obj(cachep, objp);
2814#endif
2815 poison_obj(cachep, objp, POISON_INUSE);
2816 }
2817 if (cachep->flags & SLAB_STORE_USER)
2818 *dbg_userword(cachep, objp) = caller;
2819
2820 if (cachep->flags & SLAB_RED_ZONE) {
Andrew Mortona737b3e2006-03-22 00:08:11 -08002821 if (*dbg_redzone1(cachep, objp) != RED_INACTIVE ||
2822 *dbg_redzone2(cachep, objp) != RED_INACTIVE) {
2823 slab_error(cachep, "double free, or memory outside"
2824 " object was overwritten");
Pekka Enbergb28a02d2006-01-08 01:00:37 -08002825 printk(KERN_ERR
Andrew Mortona737b3e2006-03-22 00:08:11 -08002826 "%p: redzone 1:0x%lx, redzone 2:0x%lx\n",
2827 objp, *dbg_redzone1(cachep, objp),
2828 *dbg_redzone2(cachep, objp));
Linus Torvalds1da177e2005-04-16 15:20:36 -07002829 }
2830 *dbg_redzone1(cachep, objp) = RED_ACTIVE;
2831 *dbg_redzone2(cachep, objp) = RED_ACTIVE;
2832 }
Al Viro871751e2006-03-25 03:06:39 -08002833#ifdef CONFIG_DEBUG_SLAB_LEAK
2834 {
2835 struct slab *slabp;
2836 unsigned objnr;
2837
2838 slabp = page_get_slab(virt_to_page(objp));
2839 objnr = (unsigned)(objp - slabp->s_mem) / cachep->buffer_size;
2840 slab_bufctl(slabp)[objnr] = BUFCTL_ACTIVE;
2841 }
2842#endif
Manfred Spraul3dafccf2006-02-01 03:05:42 -08002843 objp += obj_offset(cachep);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002844 if (cachep->ctor && cachep->flags & SLAB_POISON) {
Pekka Enbergb28a02d2006-01-08 01:00:37 -08002845 unsigned long ctor_flags = SLAB_CTOR_CONSTRUCTOR;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002846
2847 if (!(flags & __GFP_WAIT))
2848 ctor_flags |= SLAB_CTOR_ATOMIC;
2849
2850 cachep->ctor(objp, cachep, ctor_flags);
Pekka Enbergb28a02d2006-01-08 01:00:37 -08002851 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002852 return objp;
2853}
2854#else
2855#define cache_alloc_debugcheck_after(a,b,objp,d) (objp)
2856#endif
2857
Pekka Enberg343e0d72006-02-01 03:05:50 -08002858static inline void *____cache_alloc(struct kmem_cache *cachep, gfp_t flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002859{
Pekka Enbergb28a02d2006-01-08 01:00:37 -08002860 void *objp;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002861 struct array_cache *ac;
2862
Christoph Lameterdc85da12006-01-18 17:42:36 -08002863#ifdef CONFIG_NUMA
Paul Jacksonb2455392006-03-24 03:16:12 -08002864 if (unlikely(current->flags & (PF_SPREAD_SLAB | PF_MEMPOLICY))) {
Paul Jacksonc61afb12006-03-24 03:16:08 -08002865 objp = alternate_node_alloc(cachep, flags);
2866 if (objp != NULL)
2867 return objp;
Paul Jackson101a5002006-03-24 03:16:07 -08002868 }
Christoph Lameterdc85da12006-01-18 17:42:36 -08002869#endif
2870
Alok N Kataria5c382302005-09-27 21:45:46 -07002871 check_irq_off();
Pekka Enberg9a2dba42006-02-01 03:05:49 -08002872 ac = cpu_cache_get(cachep);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002873 if (likely(ac->avail)) {
2874 STATS_INC_ALLOCHIT(cachep);
2875 ac->touched = 1;
Christoph Lametere498be72005-09-09 13:03:32 -07002876 objp = ac->entry[--ac->avail];
Linus Torvalds1da177e2005-04-16 15:20:36 -07002877 } else {
2878 STATS_INC_ALLOCMISS(cachep);
2879 objp = cache_alloc_refill(cachep, flags);
2880 }
Alok N Kataria5c382302005-09-27 21:45:46 -07002881 return objp;
2882}
2883
Andrew Mortona737b3e2006-03-22 00:08:11 -08002884static __always_inline void *__cache_alloc(struct kmem_cache *cachep,
2885 gfp_t flags, void *caller)
Alok N Kataria5c382302005-09-27 21:45:46 -07002886{
2887 unsigned long save_flags;
Pekka Enbergb28a02d2006-01-08 01:00:37 -08002888 void *objp;
Alok N Kataria5c382302005-09-27 21:45:46 -07002889
2890 cache_alloc_debugcheck_before(cachep, flags);
2891
2892 local_irq_save(save_flags);
2893 objp = ____cache_alloc(cachep, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002894 local_irq_restore(save_flags);
Eric Dumazet34342e82005-09-03 15:55:06 -07002895 objp = cache_alloc_debugcheck_after(cachep, flags, objp,
Pekka Enberg7fd6b142006-02-01 03:05:52 -08002896 caller);
Eric Dumazet34342e82005-09-03 15:55:06 -07002897 prefetchw(objp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002898 return objp;
2899}
2900
Christoph Lametere498be72005-09-09 13:03:32 -07002901#ifdef CONFIG_NUMA
2902/*
Paul Jacksonb2455392006-03-24 03:16:12 -08002903 * Try allocating on another node if PF_SPREAD_SLAB|PF_MEMPOLICY.
Paul Jacksonc61afb12006-03-24 03:16:08 -08002904 *
2905 * If we are in_interrupt, then process context, including cpusets and
2906 * mempolicy, may not apply and should not be used for allocation policy.
2907 */
2908static void *alternate_node_alloc(struct kmem_cache *cachep, gfp_t flags)
2909{
2910 int nid_alloc, nid_here;
2911
2912 if (in_interrupt())
2913 return NULL;
2914 nid_alloc = nid_here = numa_node_id();
2915 if (cpuset_do_slab_mem_spread() && (cachep->flags & SLAB_MEM_SPREAD))
2916 nid_alloc = cpuset_mem_spread_node();
2917 else if (current->mempolicy)
2918 nid_alloc = slab_node(current->mempolicy);
2919 if (nid_alloc != nid_here)
2920 return __cache_alloc_node(cachep, flags, nid_alloc);
2921 return NULL;
2922}
2923
2924/*
Christoph Lametere498be72005-09-09 13:03:32 -07002925 * A interface to enable slab creation on nodeid
Linus Torvalds1da177e2005-04-16 15:20:36 -07002926 */
Andrew Mortona737b3e2006-03-22 00:08:11 -08002927static void *__cache_alloc_node(struct kmem_cache *cachep, gfp_t flags,
2928 int nodeid)
Christoph Lametere498be72005-09-09 13:03:32 -07002929{
2930 struct list_head *entry;
Pekka Enbergb28a02d2006-01-08 01:00:37 -08002931 struct slab *slabp;
2932 struct kmem_list3 *l3;
2933 void *obj;
Pekka Enbergb28a02d2006-01-08 01:00:37 -08002934 int x;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002935
Pekka Enbergb28a02d2006-01-08 01:00:37 -08002936 l3 = cachep->nodelists[nodeid];
2937 BUG_ON(!l3);
Christoph Lametere498be72005-09-09 13:03:32 -07002938
Andrew Mortona737b3e2006-03-22 00:08:11 -08002939retry:
Ravikiran G Thirumalaica3b9b92006-02-04 23:27:58 -08002940 check_irq_off();
Pekka Enbergb28a02d2006-01-08 01:00:37 -08002941 spin_lock(&l3->list_lock);
2942 entry = l3->slabs_partial.next;
2943 if (entry == &l3->slabs_partial) {
2944 l3->free_touched = 1;
2945 entry = l3->slabs_free.next;
2946 if (entry == &l3->slabs_free)
2947 goto must_grow;
2948 }
Christoph Lametere498be72005-09-09 13:03:32 -07002949
Pekka Enbergb28a02d2006-01-08 01:00:37 -08002950 slabp = list_entry(entry, struct slab, list);
2951 check_spinlock_acquired_node(cachep, nodeid);
2952 check_slabp(cachep, slabp);
Christoph Lametere498be72005-09-09 13:03:32 -07002953
Pekka Enbergb28a02d2006-01-08 01:00:37 -08002954 STATS_INC_NODEALLOCS(cachep);
2955 STATS_INC_ACTIVE(cachep);
2956 STATS_SET_HIGH(cachep);
Christoph Lametere498be72005-09-09 13:03:32 -07002957
Pekka Enbergb28a02d2006-01-08 01:00:37 -08002958 BUG_ON(slabp->inuse == cachep->num);
Christoph Lametere498be72005-09-09 13:03:32 -07002959
Matthew Dobson78d382d2006-02-01 03:05:47 -08002960 obj = slab_get_obj(cachep, slabp, nodeid);
Pekka Enbergb28a02d2006-01-08 01:00:37 -08002961 check_slabp(cachep, slabp);
2962 l3->free_objects--;
2963 /* move slabp to correct slabp list: */
2964 list_del(&slabp->list);
Christoph Lametere498be72005-09-09 13:03:32 -07002965
Andrew Mortona737b3e2006-03-22 00:08:11 -08002966 if (slabp->free == BUFCTL_END)
Pekka Enbergb28a02d2006-01-08 01:00:37 -08002967 list_add(&slabp->list, &l3->slabs_full);
Andrew Mortona737b3e2006-03-22 00:08:11 -08002968 else
Pekka Enbergb28a02d2006-01-08 01:00:37 -08002969 list_add(&slabp->list, &l3->slabs_partial);
Christoph Lametere498be72005-09-09 13:03:32 -07002970
Pekka Enbergb28a02d2006-01-08 01:00:37 -08002971 spin_unlock(&l3->list_lock);
2972 goto done;
Christoph Lametere498be72005-09-09 13:03:32 -07002973
Andrew Mortona737b3e2006-03-22 00:08:11 -08002974must_grow:
Pekka Enbergb28a02d2006-01-08 01:00:37 -08002975 spin_unlock(&l3->list_lock);
2976 x = cache_grow(cachep, flags, nodeid);
Christoph Lametere498be72005-09-09 13:03:32 -07002977
Pekka Enbergb28a02d2006-01-08 01:00:37 -08002978 if (!x)
2979 return NULL;
Christoph Lametere498be72005-09-09 13:03:32 -07002980
Pekka Enbergb28a02d2006-01-08 01:00:37 -08002981 goto retry;
Andrew Mortona737b3e2006-03-22 00:08:11 -08002982done:
Pekka Enbergb28a02d2006-01-08 01:00:37 -08002983 return obj;
Christoph Lametere498be72005-09-09 13:03:32 -07002984}
2985#endif
2986
2987/*
2988 * Caller needs to acquire correct kmem_list's list_lock
2989 */
Pekka Enberg343e0d72006-02-01 03:05:50 -08002990static void free_block(struct kmem_cache *cachep, void **objpp, int nr_objects,
Pekka Enbergb28a02d2006-01-08 01:00:37 -08002991 int node)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002992{
2993 int i;
Christoph Lametere498be72005-09-09 13:03:32 -07002994 struct kmem_list3 *l3;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002995
2996 for (i = 0; i < nr_objects; i++) {
2997 void *objp = objpp[i];
2998 struct slab *slabp;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002999
Pekka Enberg6ed5eb2212006-02-01 03:05:49 -08003000 slabp = virt_to_slab(objp);
Christoph Lameterff694162005-09-22 21:44:02 -07003001 l3 = cachep->nodelists[node];
Linus Torvalds1da177e2005-04-16 15:20:36 -07003002 list_del(&slabp->list);
Christoph Lameterff694162005-09-22 21:44:02 -07003003 check_spinlock_acquired_node(cachep, node);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003004 check_slabp(cachep, slabp);
Matthew Dobson78d382d2006-02-01 03:05:47 -08003005 slab_put_obj(cachep, slabp, objp, node);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003006 STATS_DEC_ACTIVE(cachep);
Christoph Lametere498be72005-09-09 13:03:32 -07003007 l3->free_objects++;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003008 check_slabp(cachep, slabp);
3009
3010 /* fixup slab chains */
3011 if (slabp->inuse == 0) {
Christoph Lametere498be72005-09-09 13:03:32 -07003012 if (l3->free_objects > l3->free_limit) {
3013 l3->free_objects -= cachep->num;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003014 slab_destroy(cachep, slabp);
3015 } else {
Christoph Lametere498be72005-09-09 13:03:32 -07003016 list_add(&slabp->list, &l3->slabs_free);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003017 }
3018 } else {
3019 /* Unconditionally move a slab to the end of the
3020 * partial list on free - maximum time for the
3021 * other objects to be freed, too.
3022 */
Christoph Lametere498be72005-09-09 13:03:32 -07003023 list_add_tail(&slabp->list, &l3->slabs_partial);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003024 }
3025 }
3026}
3027
Pekka Enberg343e0d72006-02-01 03:05:50 -08003028static void cache_flusharray(struct kmem_cache *cachep, struct array_cache *ac)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003029{
3030 int batchcount;
Christoph Lametere498be72005-09-09 13:03:32 -07003031 struct kmem_list3 *l3;
Christoph Lameterff694162005-09-22 21:44:02 -07003032 int node = numa_node_id();
Linus Torvalds1da177e2005-04-16 15:20:36 -07003033
3034 batchcount = ac->batchcount;
3035#if DEBUG
3036 BUG_ON(!batchcount || batchcount > ac->avail);
3037#endif
3038 check_irq_off();
Christoph Lameterff694162005-09-22 21:44:02 -07003039 l3 = cachep->nodelists[node];
Christoph Lametere498be72005-09-09 13:03:32 -07003040 spin_lock(&l3->list_lock);
3041 if (l3->shared) {
3042 struct array_cache *shared_array = l3->shared;
Pekka Enbergb28a02d2006-01-08 01:00:37 -08003043 int max = shared_array->limit - shared_array->avail;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003044 if (max) {
3045 if (batchcount > max)
3046 batchcount = max;
Christoph Lametere498be72005-09-09 13:03:32 -07003047 memcpy(&(shared_array->entry[shared_array->avail]),
Pekka Enbergb28a02d2006-01-08 01:00:37 -08003048 ac->entry, sizeof(void *) * batchcount);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003049 shared_array->avail += batchcount;
3050 goto free_done;
3051 }
3052 }
3053
Christoph Lameterff694162005-09-22 21:44:02 -07003054 free_block(cachep, ac->entry, batchcount, node);
Andrew Mortona737b3e2006-03-22 00:08:11 -08003055free_done:
Linus Torvalds1da177e2005-04-16 15:20:36 -07003056#if STATS
3057 {
3058 int i = 0;
3059 struct list_head *p;
3060
Christoph Lametere498be72005-09-09 13:03:32 -07003061 p = l3->slabs_free.next;
3062 while (p != &(l3->slabs_free)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07003063 struct slab *slabp;
3064
3065 slabp = list_entry(p, struct slab, list);
3066 BUG_ON(slabp->inuse);
3067
3068 i++;
3069 p = p->next;
3070 }
3071 STATS_SET_FREEABLE(cachep, i);
3072 }
3073#endif
Christoph Lametere498be72005-09-09 13:03:32 -07003074 spin_unlock(&l3->list_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003075 ac->avail -= batchcount;
Andrew Mortona737b3e2006-03-22 00:08:11 -08003076 memmove(ac->entry, &(ac->entry[batchcount]), sizeof(void *)*ac->avail);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003077}
3078
3079/*
Andrew Mortona737b3e2006-03-22 00:08:11 -08003080 * Release an obj back to its cache. If the obj has a constructed state, it must
3081 * be in this state _before_ it is released. Called with disabled ints.
Linus Torvalds1da177e2005-04-16 15:20:36 -07003082 */
Pekka Enberg343e0d72006-02-01 03:05:50 -08003083static inline void __cache_free(struct kmem_cache *cachep, void *objp)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003084{
Pekka Enberg9a2dba42006-02-01 03:05:49 -08003085 struct array_cache *ac = cpu_cache_get(cachep);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003086
3087 check_irq_off();
3088 objp = cache_free_debugcheck(cachep, objp, __builtin_return_address(0));
3089
Christoph Lametere498be72005-09-09 13:03:32 -07003090 /* Make sure we are not freeing a object from another
3091 * node to the array cache on this cpu.
3092 */
3093#ifdef CONFIG_NUMA
3094 {
3095 struct slab *slabp;
Pekka Enberg6ed5eb2212006-02-01 03:05:49 -08003096 slabp = virt_to_slab(objp);
Christoph Lametere498be72005-09-09 13:03:32 -07003097 if (unlikely(slabp->nodeid != numa_node_id())) {
3098 struct array_cache *alien = NULL;
3099 int nodeid = slabp->nodeid;
Andrew Mortona737b3e2006-03-22 00:08:11 -08003100 struct kmem_list3 *l3;
Christoph Lametere498be72005-09-09 13:03:32 -07003101
Andrew Mortona737b3e2006-03-22 00:08:11 -08003102 l3 = cachep->nodelists[numa_node_id()];
Christoph Lametere498be72005-09-09 13:03:32 -07003103 STATS_INC_NODEFREES(cachep);
3104 if (l3->alien && l3->alien[nodeid]) {
3105 alien = l3->alien[nodeid];
3106 spin_lock(&alien->lock);
Ravikiran G Thirumalaifb7faf32006-04-10 22:52:54 -07003107 if (unlikely(alien->avail == alien->limit)) {
3108 STATS_INC_ACOVERFLOW(cachep);
Christoph Lametere498be72005-09-09 13:03:32 -07003109 __drain_alien_cache(cachep,
Pekka Enbergb28a02d2006-01-08 01:00:37 -08003110 alien, nodeid);
Ravikiran G Thirumalaifb7faf32006-04-10 22:52:54 -07003111 }
Christoph Lametere498be72005-09-09 13:03:32 -07003112 alien->entry[alien->avail++] = objp;
3113 spin_unlock(&alien->lock);
3114 } else {
3115 spin_lock(&(cachep->nodelists[nodeid])->
Pekka Enbergb28a02d2006-01-08 01:00:37 -08003116 list_lock);
Christoph Lameterff694162005-09-22 21:44:02 -07003117 free_block(cachep, &objp, 1, nodeid);
Christoph Lametere498be72005-09-09 13:03:32 -07003118 spin_unlock(&(cachep->nodelists[nodeid])->
Pekka Enbergb28a02d2006-01-08 01:00:37 -08003119 list_lock);
Christoph Lametere498be72005-09-09 13:03:32 -07003120 }
3121 return;
3122 }
3123 }
3124#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07003125 if (likely(ac->avail < ac->limit)) {
3126 STATS_INC_FREEHIT(cachep);
Christoph Lametere498be72005-09-09 13:03:32 -07003127 ac->entry[ac->avail++] = objp;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003128 return;
3129 } else {
3130 STATS_INC_FREEMISS(cachep);
3131 cache_flusharray(cachep, ac);
Christoph Lametere498be72005-09-09 13:03:32 -07003132 ac->entry[ac->avail++] = objp;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003133 }
3134}
3135
3136/**
3137 * kmem_cache_alloc - Allocate an object
3138 * @cachep: The cache to allocate from.
3139 * @flags: See kmalloc().
3140 *
3141 * Allocate an object from this cache. The flags are only relevant
3142 * if the cache has no available objects.
3143 */
Pekka Enberg343e0d72006-02-01 03:05:50 -08003144void *kmem_cache_alloc(struct kmem_cache *cachep, gfp_t flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003145{
Pekka Enberg7fd6b142006-02-01 03:05:52 -08003146 return __cache_alloc(cachep, flags, __builtin_return_address(0));
Linus Torvalds1da177e2005-04-16 15:20:36 -07003147}
3148EXPORT_SYMBOL(kmem_cache_alloc);
3149
3150/**
Pekka Enberga8c0f9a2006-03-25 03:06:42 -08003151 * kmem_cache_alloc - Allocate an object. The memory is set to zero.
3152 * @cache: The cache to allocate from.
3153 * @flags: See kmalloc().
3154 *
3155 * Allocate an object from this cache and set the allocated memory to zero.
3156 * The flags are only relevant if the cache has no available objects.
3157 */
3158void *kmem_cache_zalloc(struct kmem_cache *cache, gfp_t flags)
3159{
3160 void *ret = __cache_alloc(cache, flags, __builtin_return_address(0));
3161 if (ret)
3162 memset(ret, 0, obj_size(cache));
3163 return ret;
3164}
3165EXPORT_SYMBOL(kmem_cache_zalloc);
3166
3167/**
Linus Torvalds1da177e2005-04-16 15:20:36 -07003168 * kmem_ptr_validate - check if an untrusted pointer might
3169 * be a slab entry.
3170 * @cachep: the cache we're checking against
3171 * @ptr: pointer to validate
3172 *
3173 * This verifies that the untrusted pointer looks sane:
3174 * it is _not_ a guarantee that the pointer is actually
3175 * part of the slab cache in question, but it at least
3176 * validates that the pointer can be dereferenced and
3177 * looks half-way sane.
3178 *
3179 * Currently only used for dentry validation.
3180 */
Pekka Enberg343e0d72006-02-01 03:05:50 -08003181int fastcall kmem_ptr_validate(struct kmem_cache *cachep, void *ptr)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003182{
Pekka Enbergb28a02d2006-01-08 01:00:37 -08003183 unsigned long addr = (unsigned long)ptr;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003184 unsigned long min_addr = PAGE_OFFSET;
Pekka Enbergb28a02d2006-01-08 01:00:37 -08003185 unsigned long align_mask = BYTES_PER_WORD - 1;
Manfred Spraul3dafccf2006-02-01 03:05:42 -08003186 unsigned long size = cachep->buffer_size;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003187 struct page *page;
3188
3189 if (unlikely(addr < min_addr))
3190 goto out;
3191 if (unlikely(addr > (unsigned long)high_memory - size))
3192 goto out;
3193 if (unlikely(addr & align_mask))
3194 goto out;
3195 if (unlikely(!kern_addr_valid(addr)))
3196 goto out;
3197 if (unlikely(!kern_addr_valid(addr + size - 1)))
3198 goto out;
3199 page = virt_to_page(ptr);
3200 if (unlikely(!PageSlab(page)))
3201 goto out;
Pekka Enberg065d41c2005-11-13 16:06:46 -08003202 if (unlikely(page_get_cache(page) != cachep))
Linus Torvalds1da177e2005-04-16 15:20:36 -07003203 goto out;
3204 return 1;
Andrew Mortona737b3e2006-03-22 00:08:11 -08003205out:
Linus Torvalds1da177e2005-04-16 15:20:36 -07003206 return 0;
3207}
3208
3209#ifdef CONFIG_NUMA
3210/**
3211 * kmem_cache_alloc_node - Allocate an object on the specified node
3212 * @cachep: The cache to allocate from.
3213 * @flags: See kmalloc().
3214 * @nodeid: node number of the target node.
3215 *
3216 * Identical to kmem_cache_alloc, except that this function is slow
3217 * and can sleep. And it will allocate memory on the given node, which
3218 * can improve the performance for cpu bound structures.
Christoph Lametere498be72005-09-09 13:03:32 -07003219 * New and improved: it will now make sure that the object gets
3220 * put on the correct node list so that there is no false sharing.
Linus Torvalds1da177e2005-04-16 15:20:36 -07003221 */
Pekka Enberg343e0d72006-02-01 03:05:50 -08003222void *kmem_cache_alloc_node(struct kmem_cache *cachep, gfp_t flags, int nodeid)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003223{
Christoph Lametere498be72005-09-09 13:03:32 -07003224 unsigned long save_flags;
3225 void *ptr;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003226
Christoph Lametere498be72005-09-09 13:03:32 -07003227 cache_alloc_debugcheck_before(cachep, flags);
3228 local_irq_save(save_flags);
Christoph Lameter18f820f2006-02-01 03:05:43 -08003229
3230 if (nodeid == -1 || nodeid == numa_node_id() ||
Andrew Mortona737b3e2006-03-22 00:08:11 -08003231 !cachep->nodelists[nodeid])
Alok N Kataria5c382302005-09-27 21:45:46 -07003232 ptr = ____cache_alloc(cachep, flags);
3233 else
3234 ptr = __cache_alloc_node(cachep, flags, nodeid);
Christoph Lametere498be72005-09-09 13:03:32 -07003235 local_irq_restore(save_flags);
Christoph Lameter18f820f2006-02-01 03:05:43 -08003236
3237 ptr = cache_alloc_debugcheck_after(cachep, flags, ptr,
3238 __builtin_return_address(0));
Linus Torvalds1da177e2005-04-16 15:20:36 -07003239
Christoph Lametere498be72005-09-09 13:03:32 -07003240 return ptr;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003241}
3242EXPORT_SYMBOL(kmem_cache_alloc_node);
3243
Al Virodd0fc662005-10-07 07:46:04 +01003244void *kmalloc_node(size_t size, gfp_t flags, int node)
Manfred Spraul97e2bde2005-05-01 08:58:38 -07003245{
Pekka Enberg343e0d72006-02-01 03:05:50 -08003246 struct kmem_cache *cachep;
Manfred Spraul97e2bde2005-05-01 08:58:38 -07003247
3248 cachep = kmem_find_general_cachep(size, flags);
3249 if (unlikely(cachep == NULL))
3250 return NULL;
3251 return kmem_cache_alloc_node(cachep, flags, node);
3252}
3253EXPORT_SYMBOL(kmalloc_node);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003254#endif
3255
3256/**
3257 * kmalloc - allocate memory
3258 * @size: how many bytes of memory are required.
3259 * @flags: the type of memory to allocate.
Randy Dunlap911851e2006-03-22 00:08:14 -08003260 * @caller: function caller for debug tracking of the caller
Linus Torvalds1da177e2005-04-16 15:20:36 -07003261 *
3262 * kmalloc is the normal method of allocating memory
3263 * in the kernel.
3264 *
3265 * The @flags argument may be one of:
3266 *
3267 * %GFP_USER - Allocate memory on behalf of user. May sleep.
3268 *
3269 * %GFP_KERNEL - Allocate normal kernel ram. May sleep.
3270 *
3271 * %GFP_ATOMIC - Allocation will not sleep. Use inside interrupt handlers.
3272 *
3273 * Additionally, the %GFP_DMA flag may be set to indicate the memory
3274 * must be suitable for DMA. This can mean different things on different
3275 * platforms. For example, on i386, it means that the memory must come
3276 * from the first 16MB.
3277 */
Pekka Enberg7fd6b142006-02-01 03:05:52 -08003278static __always_inline void *__do_kmalloc(size_t size, gfp_t flags,
3279 void *caller)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003280{
Pekka Enberg343e0d72006-02-01 03:05:50 -08003281 struct kmem_cache *cachep;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003282
Manfred Spraul97e2bde2005-05-01 08:58:38 -07003283 /* If you want to save a few bytes .text space: replace
3284 * __ with kmem_.
3285 * Then kmalloc uses the uninlined functions instead of the inline
3286 * functions.
3287 */
3288 cachep = __find_general_cachep(size, flags);
Andrew Mortondbdb9042005-09-23 13:24:10 -07003289 if (unlikely(cachep == NULL))
3290 return NULL;
Pekka Enberg7fd6b142006-02-01 03:05:52 -08003291 return __cache_alloc(cachep, flags, caller);
3292}
3293
Pekka Enberg7fd6b142006-02-01 03:05:52 -08003294
3295void *__kmalloc(size_t size, gfp_t flags)
3296{
Al Viro871751e2006-03-25 03:06:39 -08003297#ifndef CONFIG_DEBUG_SLAB
Pekka Enberg7fd6b142006-02-01 03:05:52 -08003298 return __do_kmalloc(size, flags, NULL);
Al Viro871751e2006-03-25 03:06:39 -08003299#else
3300 return __do_kmalloc(size, flags, __builtin_return_address(0));
3301#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07003302}
3303EXPORT_SYMBOL(__kmalloc);
3304
Al Viro871751e2006-03-25 03:06:39 -08003305#ifdef CONFIG_DEBUG_SLAB
Pekka Enberg7fd6b142006-02-01 03:05:52 -08003306void *__kmalloc_track_caller(size_t size, gfp_t flags, void *caller)
3307{
3308 return __do_kmalloc(size, flags, caller);
3309}
3310EXPORT_SYMBOL(__kmalloc_track_caller);
Pekka Enberg7fd6b142006-02-01 03:05:52 -08003311#endif
3312
Linus Torvalds1da177e2005-04-16 15:20:36 -07003313#ifdef CONFIG_SMP
3314/**
3315 * __alloc_percpu - allocate one copy of the object for every present
3316 * cpu in the system, zeroing them.
3317 * Objects should be dereferenced using the per_cpu_ptr macro only.
3318 *
3319 * @size: how many bytes of memory are required.
Linus Torvalds1da177e2005-04-16 15:20:36 -07003320 */
Pekka Enbergf9f75002006-01-08 01:00:33 -08003321void *__alloc_percpu(size_t size)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003322{
3323 int i;
Pekka Enbergb28a02d2006-01-08 01:00:37 -08003324 struct percpu_data *pdata = kmalloc(sizeof(*pdata), GFP_KERNEL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003325
3326 if (!pdata)
3327 return NULL;
3328
Christoph Lametere498be72005-09-09 13:03:32 -07003329 /*
3330 * Cannot use for_each_online_cpu since a cpu may come online
3331 * and we have no way of figuring out how to fix the array
3332 * that we have allocated then....
3333 */
KAMEZAWA Hiroyuki0a945022006-03-28 01:56:37 -08003334 for_each_possible_cpu(i) {
Christoph Lametere498be72005-09-09 13:03:32 -07003335 int node = cpu_to_node(i);
3336
3337 if (node_online(node))
3338 pdata->ptrs[i] = kmalloc_node(size, GFP_KERNEL, node);
3339 else
3340 pdata->ptrs[i] = kmalloc(size, GFP_KERNEL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003341
3342 if (!pdata->ptrs[i])
3343 goto unwind_oom;
3344 memset(pdata->ptrs[i], 0, size);
3345 }
3346
3347 /* Catch derefs w/o wrappers */
Pekka Enbergb28a02d2006-01-08 01:00:37 -08003348 return (void *)(~(unsigned long)pdata);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003349
Andrew Mortona737b3e2006-03-22 00:08:11 -08003350unwind_oom:
Linus Torvalds1da177e2005-04-16 15:20:36 -07003351 while (--i >= 0) {
3352 if (!cpu_possible(i))
3353 continue;
3354 kfree(pdata->ptrs[i]);
3355 }
3356 kfree(pdata);
3357 return NULL;
3358}
3359EXPORT_SYMBOL(__alloc_percpu);
3360#endif
3361
3362/**
3363 * kmem_cache_free - Deallocate an object
3364 * @cachep: The cache the allocation was from.
3365 * @objp: The previously allocated object.
3366 *
3367 * Free an object which was previously allocated from this
3368 * cache.
3369 */
Pekka Enberg343e0d72006-02-01 03:05:50 -08003370void kmem_cache_free(struct kmem_cache *cachep, void *objp)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003371{
3372 unsigned long flags;
3373
3374 local_irq_save(flags);
3375 __cache_free(cachep, objp);
3376 local_irq_restore(flags);
3377}
3378EXPORT_SYMBOL(kmem_cache_free);
3379
3380/**
Linus Torvalds1da177e2005-04-16 15:20:36 -07003381 * kfree - free previously allocated memory
3382 * @objp: pointer returned by kmalloc.
3383 *
Pekka Enberg80e93ef2005-09-09 13:10:16 -07003384 * If @objp is NULL, no operation is performed.
3385 *
Linus Torvalds1da177e2005-04-16 15:20:36 -07003386 * Don't free memory not originally allocated by kmalloc()
3387 * or you will run into trouble.
3388 */
3389void kfree(const void *objp)
3390{
Pekka Enberg343e0d72006-02-01 03:05:50 -08003391 struct kmem_cache *c;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003392 unsigned long flags;
3393
3394 if (unlikely(!objp))
3395 return;
3396 local_irq_save(flags);
3397 kfree_debugcheck(objp);
Pekka Enberg6ed5eb2212006-02-01 03:05:49 -08003398 c = virt_to_cache(objp);
Manfred Spraul3dafccf2006-02-01 03:05:42 -08003399 mutex_debug_check_no_locks_freed(objp, obj_size(c));
Pekka Enbergb28a02d2006-01-08 01:00:37 -08003400 __cache_free(c, (void *)objp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003401 local_irq_restore(flags);
3402}
3403EXPORT_SYMBOL(kfree);
3404
3405#ifdef CONFIG_SMP
3406/**
3407 * free_percpu - free previously allocated percpu memory
3408 * @objp: pointer returned by alloc_percpu.
3409 *
3410 * Don't free memory not originally allocated by alloc_percpu()
3411 * The complemented objp is to check for that.
3412 */
Pekka Enbergb28a02d2006-01-08 01:00:37 -08003413void free_percpu(const void *objp)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003414{
3415 int i;
Pekka Enbergb28a02d2006-01-08 01:00:37 -08003416 struct percpu_data *p = (struct percpu_data *)(~(unsigned long)objp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003417
Christoph Lametere498be72005-09-09 13:03:32 -07003418 /*
3419 * We allocate for all cpus so we cannot use for online cpu here.
3420 */
KAMEZAWA Hiroyuki0a945022006-03-28 01:56:37 -08003421 for_each_possible_cpu(i)
Pekka Enbergb28a02d2006-01-08 01:00:37 -08003422 kfree(p->ptrs[i]);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003423 kfree(p);
3424}
3425EXPORT_SYMBOL(free_percpu);
3426#endif
3427
Pekka Enberg343e0d72006-02-01 03:05:50 -08003428unsigned int kmem_cache_size(struct kmem_cache *cachep)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003429{
Manfred Spraul3dafccf2006-02-01 03:05:42 -08003430 return obj_size(cachep);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003431}
3432EXPORT_SYMBOL(kmem_cache_size);
3433
Pekka Enberg343e0d72006-02-01 03:05:50 -08003434const char *kmem_cache_name(struct kmem_cache *cachep)
Arnaldo Carvalho de Melo19449722005-06-18 22:46:19 -07003435{
3436 return cachep->name;
3437}
3438EXPORT_SYMBOL_GPL(kmem_cache_name);
3439
Christoph Lametere498be72005-09-09 13:03:32 -07003440/*
Christoph Lameter0718dc22006-03-25 03:06:47 -08003441 * This initializes kmem_list3 or resizes varioius caches for all nodes.
Christoph Lametere498be72005-09-09 13:03:32 -07003442 */
Pekka Enberg343e0d72006-02-01 03:05:50 -08003443static int alloc_kmemlist(struct kmem_cache *cachep)
Christoph Lametere498be72005-09-09 13:03:32 -07003444{
3445 int node;
3446 struct kmem_list3 *l3;
Christoph Lametercafeb022006-03-25 03:06:46 -08003447 struct array_cache *new_shared;
3448 struct array_cache **new_alien;
Christoph Lametere498be72005-09-09 13:03:32 -07003449
3450 for_each_online_node(node) {
Christoph Lametercafeb022006-03-25 03:06:46 -08003451
Andrew Mortona737b3e2006-03-22 00:08:11 -08003452 new_alien = alloc_alien_cache(node, cachep->limit);
3453 if (!new_alien)
Christoph Lametere498be72005-09-09 13:03:32 -07003454 goto fail;
Christoph Lametercafeb022006-03-25 03:06:46 -08003455
Christoph Lameter0718dc22006-03-25 03:06:47 -08003456 new_shared = alloc_arraycache(node,
3457 cachep->shared*cachep->batchcount,
Andrew Mortona737b3e2006-03-22 00:08:11 -08003458 0xbaadf00d);
Christoph Lameter0718dc22006-03-25 03:06:47 -08003459 if (!new_shared) {
3460 free_alien_cache(new_alien);
Christoph Lametere498be72005-09-09 13:03:32 -07003461 goto fail;
Christoph Lameter0718dc22006-03-25 03:06:47 -08003462 }
Christoph Lametercafeb022006-03-25 03:06:46 -08003463
Andrew Mortona737b3e2006-03-22 00:08:11 -08003464 l3 = cachep->nodelists[node];
3465 if (l3) {
Christoph Lametercafeb022006-03-25 03:06:46 -08003466 struct array_cache *shared = l3->shared;
3467
Christoph Lametere498be72005-09-09 13:03:32 -07003468 spin_lock_irq(&l3->list_lock);
3469
Christoph Lametercafeb022006-03-25 03:06:46 -08003470 if (shared)
Christoph Lameter0718dc22006-03-25 03:06:47 -08003471 free_block(cachep, shared->entry,
3472 shared->avail, node);
Christoph Lametere498be72005-09-09 13:03:32 -07003473
Christoph Lametercafeb022006-03-25 03:06:46 -08003474 l3->shared = new_shared;
3475 if (!l3->alien) {
Christoph Lametere498be72005-09-09 13:03:32 -07003476 l3->alien = new_alien;
3477 new_alien = NULL;
3478 }
Pekka Enbergb28a02d2006-01-08 01:00:37 -08003479 l3->free_limit = (1 + nr_cpus_node(node)) *
Andrew Mortona737b3e2006-03-22 00:08:11 -08003480 cachep->batchcount + cachep->num;
Christoph Lametere498be72005-09-09 13:03:32 -07003481 spin_unlock_irq(&l3->list_lock);
Christoph Lametercafeb022006-03-25 03:06:46 -08003482 kfree(shared);
Christoph Lametere498be72005-09-09 13:03:32 -07003483 free_alien_cache(new_alien);
3484 continue;
3485 }
Andrew Mortona737b3e2006-03-22 00:08:11 -08003486 l3 = kmalloc_node(sizeof(struct kmem_list3), GFP_KERNEL, node);
Christoph Lameter0718dc22006-03-25 03:06:47 -08003487 if (!l3) {
3488 free_alien_cache(new_alien);
3489 kfree(new_shared);
Christoph Lametere498be72005-09-09 13:03:32 -07003490 goto fail;
Christoph Lameter0718dc22006-03-25 03:06:47 -08003491 }
Christoph Lametere498be72005-09-09 13:03:32 -07003492
3493 kmem_list3_init(l3);
3494 l3->next_reap = jiffies + REAPTIMEOUT_LIST3 +
Andrew Mortona737b3e2006-03-22 00:08:11 -08003495 ((unsigned long)cachep) % REAPTIMEOUT_LIST3;
Christoph Lametercafeb022006-03-25 03:06:46 -08003496 l3->shared = new_shared;
Christoph Lametere498be72005-09-09 13:03:32 -07003497 l3->alien = new_alien;
Pekka Enbergb28a02d2006-01-08 01:00:37 -08003498 l3->free_limit = (1 + nr_cpus_node(node)) *
Andrew Mortona737b3e2006-03-22 00:08:11 -08003499 cachep->batchcount + cachep->num;
Christoph Lametere498be72005-09-09 13:03:32 -07003500 cachep->nodelists[node] = l3;
3501 }
Christoph Lametercafeb022006-03-25 03:06:46 -08003502 return 0;
Christoph Lameter0718dc22006-03-25 03:06:47 -08003503
Andrew Mortona737b3e2006-03-22 00:08:11 -08003504fail:
Christoph Lameter0718dc22006-03-25 03:06:47 -08003505 if (!cachep->next.next) {
3506 /* Cache is not active yet. Roll back what we did */
3507 node--;
3508 while (node >= 0) {
3509 if (cachep->nodelists[node]) {
3510 l3 = cachep->nodelists[node];
3511
3512 kfree(l3->shared);
3513 free_alien_cache(l3->alien);
3514 kfree(l3);
3515 cachep->nodelists[node] = NULL;
3516 }
3517 node--;
3518 }
3519 }
Christoph Lametercafeb022006-03-25 03:06:46 -08003520 return -ENOMEM;
Christoph Lametere498be72005-09-09 13:03:32 -07003521}
3522
Linus Torvalds1da177e2005-04-16 15:20:36 -07003523struct ccupdate_struct {
Pekka Enberg343e0d72006-02-01 03:05:50 -08003524 struct kmem_cache *cachep;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003525 struct array_cache *new[NR_CPUS];
3526};
3527
3528static void do_ccupdate_local(void *info)
3529{
Andrew Mortona737b3e2006-03-22 00:08:11 -08003530 struct ccupdate_struct *new = info;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003531 struct array_cache *old;
3532
3533 check_irq_off();
Pekka Enberg9a2dba42006-02-01 03:05:49 -08003534 old = cpu_cache_get(new->cachep);
Christoph Lametere498be72005-09-09 13:03:32 -07003535
Linus Torvalds1da177e2005-04-16 15:20:36 -07003536 new->cachep->array[smp_processor_id()] = new->new[smp_processor_id()];
3537 new->new[smp_processor_id()] = old;
3538}
3539
Ravikiran G Thirumalaib5d8ca72006-03-22 00:08:12 -08003540/* Always called with the cache_chain_mutex held */
Andrew Mortona737b3e2006-03-22 00:08:11 -08003541static int do_tune_cpucache(struct kmem_cache *cachep, int limit,
3542 int batchcount, int shared)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003543{
3544 struct ccupdate_struct new;
Christoph Lametere498be72005-09-09 13:03:32 -07003545 int i, err;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003546
Pekka Enbergb28a02d2006-01-08 01:00:37 -08003547 memset(&new.new, 0, sizeof(new.new));
Christoph Lametere498be72005-09-09 13:03:32 -07003548 for_each_online_cpu(i) {
Andrew Mortona737b3e2006-03-22 00:08:11 -08003549 new.new[i] = alloc_arraycache(cpu_to_node(i), limit,
3550 batchcount);
Christoph Lametere498be72005-09-09 13:03:32 -07003551 if (!new.new[i]) {
Pekka Enbergb28a02d2006-01-08 01:00:37 -08003552 for (i--; i >= 0; i--)
3553 kfree(new.new[i]);
Christoph Lametere498be72005-09-09 13:03:32 -07003554 return -ENOMEM;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003555 }
3556 }
3557 new.cachep = cachep;
3558
Andrew Mortona07fa392006-03-22 00:08:17 -08003559 on_each_cpu(do_ccupdate_local, (void *)&new, 1, 1);
Christoph Lametere498be72005-09-09 13:03:32 -07003560
Linus Torvalds1da177e2005-04-16 15:20:36 -07003561 check_irq_on();
Linus Torvalds1da177e2005-04-16 15:20:36 -07003562 cachep->batchcount = batchcount;
3563 cachep->limit = limit;
Christoph Lametere498be72005-09-09 13:03:32 -07003564 cachep->shared = shared;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003565
Christoph Lametere498be72005-09-09 13:03:32 -07003566 for_each_online_cpu(i) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07003567 struct array_cache *ccold = new.new[i];
3568 if (!ccold)
3569 continue;
Christoph Lametere498be72005-09-09 13:03:32 -07003570 spin_lock_irq(&cachep->nodelists[cpu_to_node(i)]->list_lock);
Christoph Lameterff694162005-09-22 21:44:02 -07003571 free_block(cachep, ccold->entry, ccold->avail, cpu_to_node(i));
Christoph Lametere498be72005-09-09 13:03:32 -07003572 spin_unlock_irq(&cachep->nodelists[cpu_to_node(i)]->list_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003573 kfree(ccold);
3574 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07003575
Christoph Lametere498be72005-09-09 13:03:32 -07003576 err = alloc_kmemlist(cachep);
3577 if (err) {
3578 printk(KERN_ERR "alloc_kmemlist failed for %s, error %d.\n",
Pekka Enbergb28a02d2006-01-08 01:00:37 -08003579 cachep->name, -err);
Christoph Lametere498be72005-09-09 13:03:32 -07003580 BUG();
Linus Torvalds1da177e2005-04-16 15:20:36 -07003581 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07003582 return 0;
3583}
3584
Ravikiran G Thirumalaib5d8ca72006-03-22 00:08:12 -08003585/* Called with cache_chain_mutex held always */
Pekka Enberg343e0d72006-02-01 03:05:50 -08003586static void enable_cpucache(struct kmem_cache *cachep)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003587{
3588 int err;
3589 int limit, shared;
3590
Andrew Mortona737b3e2006-03-22 00:08:11 -08003591 /*
3592 * The head array serves three purposes:
Linus Torvalds1da177e2005-04-16 15:20:36 -07003593 * - create a LIFO ordering, i.e. return objects that are cache-warm
3594 * - reduce the number of spinlock operations.
Andrew Mortona737b3e2006-03-22 00:08:11 -08003595 * - reduce the number of linked list operations on the slab and
Linus Torvalds1da177e2005-04-16 15:20:36 -07003596 * bufctl chains: array operations are cheaper.
3597 * The numbers are guessed, we should auto-tune as described by
3598 * Bonwick.
3599 */
Manfred Spraul3dafccf2006-02-01 03:05:42 -08003600 if (cachep->buffer_size > 131072)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003601 limit = 1;
Manfred Spraul3dafccf2006-02-01 03:05:42 -08003602 else if (cachep->buffer_size > PAGE_SIZE)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003603 limit = 8;
Manfred Spraul3dafccf2006-02-01 03:05:42 -08003604 else if (cachep->buffer_size > 1024)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003605 limit = 24;
Manfred Spraul3dafccf2006-02-01 03:05:42 -08003606 else if (cachep->buffer_size > 256)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003607 limit = 54;
3608 else
3609 limit = 120;
3610
Andrew Mortona737b3e2006-03-22 00:08:11 -08003611 /*
3612 * CPU bound tasks (e.g. network routing) can exhibit cpu bound
Linus Torvalds1da177e2005-04-16 15:20:36 -07003613 * allocation behaviour: Most allocs on one cpu, most free operations
3614 * on another cpu. For these cases, an efficient object passing between
3615 * cpus is necessary. This is provided by a shared array. The array
3616 * replaces Bonwick's magazine layer.
3617 * On uniprocessor, it's functionally equivalent (but less efficient)
3618 * to a larger limit. Thus disabled by default.
3619 */
3620 shared = 0;
3621#ifdef CONFIG_SMP
Manfred Spraul3dafccf2006-02-01 03:05:42 -08003622 if (cachep->buffer_size <= PAGE_SIZE)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003623 shared = 8;
3624#endif
3625
3626#if DEBUG
Andrew Mortona737b3e2006-03-22 00:08:11 -08003627 /*
3628 * With debugging enabled, large batchcount lead to excessively long
3629 * periods with disabled local interrupts. Limit the batchcount
Linus Torvalds1da177e2005-04-16 15:20:36 -07003630 */
3631 if (limit > 32)
3632 limit = 32;
3633#endif
Pekka Enbergb28a02d2006-01-08 01:00:37 -08003634 err = do_tune_cpucache(cachep, limit, (limit + 1) / 2, shared);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003635 if (err)
3636 printk(KERN_ERR "enable_cpucache failed for %s, error %d.\n",
Pekka Enbergb28a02d2006-01-08 01:00:37 -08003637 cachep->name, -err);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003638}
3639
Christoph Lameter1b552532006-03-22 00:09:07 -08003640/*
3641 * Drain an array if it contains any elements taking the l3 lock only if
Christoph Lameterb18e7e62006-03-22 00:09:07 -08003642 * necessary. Note that the l3 listlock also protects the array_cache
3643 * if drain_array() is used on the shared array.
Christoph Lameter1b552532006-03-22 00:09:07 -08003644 */
3645void drain_array(struct kmem_cache *cachep, struct kmem_list3 *l3,
3646 struct array_cache *ac, int force, int node)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003647{
3648 int tofree;
3649
Christoph Lameter1b552532006-03-22 00:09:07 -08003650 if (!ac || !ac->avail)
3651 return;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003652 if (ac->touched && !force) {
3653 ac->touched = 0;
Christoph Lameterb18e7e62006-03-22 00:09:07 -08003654 } else {
Christoph Lameter1b552532006-03-22 00:09:07 -08003655 spin_lock_irq(&l3->list_lock);
Christoph Lameterb18e7e62006-03-22 00:09:07 -08003656 if (ac->avail) {
3657 tofree = force ? ac->avail : (ac->limit + 4) / 5;
3658 if (tofree > ac->avail)
3659 tofree = (ac->avail + 1) / 2;
3660 free_block(cachep, ac->entry, tofree, node);
3661 ac->avail -= tofree;
3662 memmove(ac->entry, &(ac->entry[tofree]),
3663 sizeof(void *) * ac->avail);
3664 }
Christoph Lameter1b552532006-03-22 00:09:07 -08003665 spin_unlock_irq(&l3->list_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003666 }
3667}
3668
3669/**
3670 * cache_reap - Reclaim memory from caches.
Randy Dunlap1e5d5332005-11-07 01:01:06 -08003671 * @unused: unused parameter
Linus Torvalds1da177e2005-04-16 15:20:36 -07003672 *
3673 * Called from workqueue/eventd every few seconds.
3674 * Purpose:
3675 * - clear the per-cpu caches for this CPU.
3676 * - return freeable pages to the main free memory pool.
3677 *
Andrew Mortona737b3e2006-03-22 00:08:11 -08003678 * If we cannot acquire the cache chain mutex then just give up - we'll try
3679 * again on the next iteration.
Linus Torvalds1da177e2005-04-16 15:20:36 -07003680 */
3681static void cache_reap(void *unused)
3682{
3683 struct list_head *walk;
Christoph Lametere498be72005-09-09 13:03:32 -07003684 struct kmem_list3 *l3;
Christoph Lameteraab22072006-03-22 00:09:06 -08003685 int node = numa_node_id();
Linus Torvalds1da177e2005-04-16 15:20:36 -07003686
Ingo Molnarfc0abb12006-01-18 17:42:33 -08003687 if (!mutex_trylock(&cache_chain_mutex)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07003688 /* Give up. Setup the next iteration. */
Pekka Enbergb28a02d2006-01-08 01:00:37 -08003689 schedule_delayed_work(&__get_cpu_var(reap_work),
3690 REAPTIMEOUT_CPUC);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003691 return;
3692 }
3693
3694 list_for_each(walk, &cache_chain) {
Pekka Enberg343e0d72006-02-01 03:05:50 -08003695 struct kmem_cache *searchp;
Pekka Enbergb28a02d2006-01-08 01:00:37 -08003696 struct list_head *p;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003697 int tofree;
3698 struct slab *slabp;
3699
Pekka Enberg343e0d72006-02-01 03:05:50 -08003700 searchp = list_entry(walk, struct kmem_cache, next);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003701 check_irq_on();
3702
Christoph Lameter35386e32006-03-22 00:09:05 -08003703 /*
3704 * We only take the l3 lock if absolutely necessary and we
3705 * have established with reasonable certainty that
3706 * we can do some work if the lock was obtained.
3707 */
Christoph Lameteraab22072006-03-22 00:09:06 -08003708 l3 = searchp->nodelists[node];
Christoph Lameter35386e32006-03-22 00:09:05 -08003709
Christoph Lameter8fce4d82006-03-09 17:33:54 -08003710 reap_alien(searchp, l3);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003711
Christoph Lameteraab22072006-03-22 00:09:06 -08003712 drain_array(searchp, l3, cpu_cache_get(searchp), 0, node);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003713
Christoph Lameter35386e32006-03-22 00:09:05 -08003714 /*
3715 * These are racy checks but it does not matter
3716 * if we skip one check or scan twice.
3717 */
Christoph Lametere498be72005-09-09 13:03:32 -07003718 if (time_after(l3->next_reap, jiffies))
Christoph Lameter35386e32006-03-22 00:09:05 -08003719 goto next;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003720
Christoph Lametere498be72005-09-09 13:03:32 -07003721 l3->next_reap = jiffies + REAPTIMEOUT_LIST3;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003722
Christoph Lameteraab22072006-03-22 00:09:06 -08003723 drain_array(searchp, l3, l3->shared, 0, node);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003724
Christoph Lametere498be72005-09-09 13:03:32 -07003725 if (l3->free_touched) {
3726 l3->free_touched = 0;
Christoph Lameter35386e32006-03-22 00:09:05 -08003727 goto next;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003728 }
3729
Andrew Mortona737b3e2006-03-22 00:08:11 -08003730 tofree = (l3->free_limit + 5 * searchp->num - 1) /
3731 (5 * searchp->num);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003732 do {
Christoph Lameter35386e32006-03-22 00:09:05 -08003733 /*
3734 * Do not lock if there are no free blocks.
3735 */
3736 if (list_empty(&l3->slabs_free))
Linus Torvalds1da177e2005-04-16 15:20:36 -07003737 break;
3738
Christoph Lameter35386e32006-03-22 00:09:05 -08003739 spin_lock_irq(&l3->list_lock);
3740 p = l3->slabs_free.next;
3741 if (p == &(l3->slabs_free)) {
3742 spin_unlock_irq(&l3->list_lock);
3743 break;
3744 }
3745
Linus Torvalds1da177e2005-04-16 15:20:36 -07003746 slabp = list_entry(p, struct slab, list);
3747 BUG_ON(slabp->inuse);
3748 list_del(&slabp->list);
3749 STATS_INC_REAPED(searchp);
3750
Andrew Mortona737b3e2006-03-22 00:08:11 -08003751 /*
3752 * Safe to drop the lock. The slab is no longer linked
3753 * to the cache. searchp cannot disappear, we hold
Linus Torvalds1da177e2005-04-16 15:20:36 -07003754 * cache_chain_lock
3755 */
Christoph Lametere498be72005-09-09 13:03:32 -07003756 l3->free_objects -= searchp->num;
3757 spin_unlock_irq(&l3->list_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003758 slab_destroy(searchp, slabp);
Pekka Enbergb28a02d2006-01-08 01:00:37 -08003759 } while (--tofree > 0);
Christoph Lameter35386e32006-03-22 00:09:05 -08003760next:
Linus Torvalds1da177e2005-04-16 15:20:36 -07003761 cond_resched();
3762 }
3763 check_irq_on();
Ingo Molnarfc0abb12006-01-18 17:42:33 -08003764 mutex_unlock(&cache_chain_mutex);
Christoph Lameter8fce4d82006-03-09 17:33:54 -08003765 next_reap_node();
Andrew Mortona737b3e2006-03-22 00:08:11 -08003766 /* Set up the next iteration */
Manfred Spraulcd61ef62005-11-07 00:58:02 -08003767 schedule_delayed_work(&__get_cpu_var(reap_work), REAPTIMEOUT_CPUC);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003768}
3769
3770#ifdef CONFIG_PROC_FS
3771
Pekka Enberg85289f92006-01-08 01:00:36 -08003772static void print_slabinfo_header(struct seq_file *m)
3773{
3774 /*
3775 * Output format version, so at least we can change it
3776 * without _too_ many complaints.
3777 */
3778#if STATS
3779 seq_puts(m, "slabinfo - version: 2.1 (statistics)\n");
3780#else
3781 seq_puts(m, "slabinfo - version: 2.1\n");
3782#endif
3783 seq_puts(m, "# name <active_objs> <num_objs> <objsize> "
3784 "<objperslab> <pagesperslab>");
3785 seq_puts(m, " : tunables <limit> <batchcount> <sharedfactor>");
3786 seq_puts(m, " : slabdata <active_slabs> <num_slabs> <sharedavail>");
3787#if STATS
3788 seq_puts(m, " : globalstat <listallocs> <maxobjs> <grown> <reaped> "
Ravikiran G Thirumalaifb7faf32006-04-10 22:52:54 -07003789 "<error> <maxfreeable> <nodeallocs> <remotefrees> <alienoverflow>");
Pekka Enberg85289f92006-01-08 01:00:36 -08003790 seq_puts(m, " : cpustat <allochit> <allocmiss> <freehit> <freemiss>");
3791#endif
3792 seq_putc(m, '\n');
3793}
3794
Linus Torvalds1da177e2005-04-16 15:20:36 -07003795static void *s_start(struct seq_file *m, loff_t *pos)
3796{
3797 loff_t n = *pos;
3798 struct list_head *p;
3799
Ingo Molnarfc0abb12006-01-18 17:42:33 -08003800 mutex_lock(&cache_chain_mutex);
Pekka Enberg85289f92006-01-08 01:00:36 -08003801 if (!n)
3802 print_slabinfo_header(m);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003803 p = cache_chain.next;
3804 while (n--) {
3805 p = p->next;
3806 if (p == &cache_chain)
3807 return NULL;
3808 }
Pekka Enberg343e0d72006-02-01 03:05:50 -08003809 return list_entry(p, struct kmem_cache, next);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003810}
3811
3812static void *s_next(struct seq_file *m, void *p, loff_t *pos)
3813{
Pekka Enberg343e0d72006-02-01 03:05:50 -08003814 struct kmem_cache *cachep = p;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003815 ++*pos;
Andrew Mortona737b3e2006-03-22 00:08:11 -08003816 return cachep->next.next == &cache_chain ?
3817 NULL : list_entry(cachep->next.next, struct kmem_cache, next);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003818}
3819
3820static void s_stop(struct seq_file *m, void *p)
3821{
Ingo Molnarfc0abb12006-01-18 17:42:33 -08003822 mutex_unlock(&cache_chain_mutex);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003823}
3824
3825static int s_show(struct seq_file *m, void *p)
3826{
Pekka Enberg343e0d72006-02-01 03:05:50 -08003827 struct kmem_cache *cachep = p;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003828 struct list_head *q;
Pekka Enbergb28a02d2006-01-08 01:00:37 -08003829 struct slab *slabp;
3830 unsigned long active_objs;
3831 unsigned long num_objs;
3832 unsigned long active_slabs = 0;
3833 unsigned long num_slabs, free_objects = 0, shared_avail = 0;
Christoph Lametere498be72005-09-09 13:03:32 -07003834 const char *name;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003835 char *error = NULL;
Christoph Lametere498be72005-09-09 13:03:32 -07003836 int node;
3837 struct kmem_list3 *l3;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003838
Linus Torvalds1da177e2005-04-16 15:20:36 -07003839 active_objs = 0;
3840 num_slabs = 0;
Christoph Lametere498be72005-09-09 13:03:32 -07003841 for_each_online_node(node) {
3842 l3 = cachep->nodelists[node];
3843 if (!l3)
3844 continue;
3845
Ravikiran G Thirumalaica3b9b92006-02-04 23:27:58 -08003846 check_irq_on();
3847 spin_lock_irq(&l3->list_lock);
Christoph Lametere498be72005-09-09 13:03:32 -07003848
Pekka Enbergb28a02d2006-01-08 01:00:37 -08003849 list_for_each(q, &l3->slabs_full) {
Christoph Lametere498be72005-09-09 13:03:32 -07003850 slabp = list_entry(q, struct slab, list);
3851 if (slabp->inuse != cachep->num && !error)
3852 error = "slabs_full accounting error";
3853 active_objs += cachep->num;
3854 active_slabs++;
3855 }
Pekka Enbergb28a02d2006-01-08 01:00:37 -08003856 list_for_each(q, &l3->slabs_partial) {
Christoph Lametere498be72005-09-09 13:03:32 -07003857 slabp = list_entry(q, struct slab, list);
3858 if (slabp->inuse == cachep->num && !error)
3859 error = "slabs_partial inuse accounting error";
3860 if (!slabp->inuse && !error)
3861 error = "slabs_partial/inuse accounting error";
3862 active_objs += slabp->inuse;
3863 active_slabs++;
3864 }
Pekka Enbergb28a02d2006-01-08 01:00:37 -08003865 list_for_each(q, &l3->slabs_free) {
Christoph Lametere498be72005-09-09 13:03:32 -07003866 slabp = list_entry(q, struct slab, list);
3867 if (slabp->inuse && !error)
3868 error = "slabs_free/inuse accounting error";
3869 num_slabs++;
3870 }
3871 free_objects += l3->free_objects;
Ravikiran G Thirumalai4484ebf2006-02-04 23:27:59 -08003872 if (l3->shared)
3873 shared_avail += l3->shared->avail;
Christoph Lametere498be72005-09-09 13:03:32 -07003874
Ravikiran G Thirumalaica3b9b92006-02-04 23:27:58 -08003875 spin_unlock_irq(&l3->list_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003876 }
Pekka Enbergb28a02d2006-01-08 01:00:37 -08003877 num_slabs += active_slabs;
3878 num_objs = num_slabs * cachep->num;
Christoph Lametere498be72005-09-09 13:03:32 -07003879 if (num_objs - active_objs != free_objects && !error)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003880 error = "free_objects accounting error";
3881
Pekka Enbergb28a02d2006-01-08 01:00:37 -08003882 name = cachep->name;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003883 if (error)
3884 printk(KERN_ERR "slab: cache %s error: %s\n", name, error);
3885
3886 seq_printf(m, "%-17s %6lu %6lu %6u %4u %4d",
Manfred Spraul3dafccf2006-02-01 03:05:42 -08003887 name, active_objs, num_objs, cachep->buffer_size,
Pekka Enbergb28a02d2006-01-08 01:00:37 -08003888 cachep->num, (1 << cachep->gfporder));
Linus Torvalds1da177e2005-04-16 15:20:36 -07003889 seq_printf(m, " : tunables %4u %4u %4u",
Pekka Enbergb28a02d2006-01-08 01:00:37 -08003890 cachep->limit, cachep->batchcount, cachep->shared);
Christoph Lametere498be72005-09-09 13:03:32 -07003891 seq_printf(m, " : slabdata %6lu %6lu %6lu",
Pekka Enbergb28a02d2006-01-08 01:00:37 -08003892 active_slabs, num_slabs, shared_avail);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003893#if STATS
Pekka Enbergb28a02d2006-01-08 01:00:37 -08003894 { /* list3 stats */
Linus Torvalds1da177e2005-04-16 15:20:36 -07003895 unsigned long high = cachep->high_mark;
3896 unsigned long allocs = cachep->num_allocations;
3897 unsigned long grown = cachep->grown;
3898 unsigned long reaped = cachep->reaped;
3899 unsigned long errors = cachep->errors;
3900 unsigned long max_freeable = cachep->max_freeable;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003901 unsigned long node_allocs = cachep->node_allocs;
Christoph Lametere498be72005-09-09 13:03:32 -07003902 unsigned long node_frees = cachep->node_frees;
Ravikiran G Thirumalaifb7faf32006-04-10 22:52:54 -07003903 unsigned long overflows = cachep->node_overflow;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003904
Christoph Lametere498be72005-09-09 13:03:32 -07003905 seq_printf(m, " : globalstat %7lu %6lu %5lu %4lu \
Ravikiran G Thirumalaifb7faf32006-04-10 22:52:54 -07003906 %4lu %4lu %4lu %4lu %4lu", allocs, high, grown,
Andrew Mortona737b3e2006-03-22 00:08:11 -08003907 reaped, errors, max_freeable, node_allocs,
Ravikiran G Thirumalaifb7faf32006-04-10 22:52:54 -07003908 node_frees, overflows);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003909 }
3910 /* cpu stats */
3911 {
3912 unsigned long allochit = atomic_read(&cachep->allochit);
3913 unsigned long allocmiss = atomic_read(&cachep->allocmiss);
3914 unsigned long freehit = atomic_read(&cachep->freehit);
3915 unsigned long freemiss = atomic_read(&cachep->freemiss);
3916
3917 seq_printf(m, " : cpustat %6lu %6lu %6lu %6lu",
Pekka Enbergb28a02d2006-01-08 01:00:37 -08003918 allochit, allocmiss, freehit, freemiss);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003919 }
3920#endif
3921 seq_putc(m, '\n');
Linus Torvalds1da177e2005-04-16 15:20:36 -07003922 return 0;
3923}
3924
3925/*
3926 * slabinfo_op - iterator that generates /proc/slabinfo
3927 *
3928 * Output layout:
3929 * cache-name
3930 * num-active-objs
3931 * total-objs
3932 * object size
3933 * num-active-slabs
3934 * total-slabs
3935 * num-pages-per-slab
3936 * + further values on SMP and with statistics enabled
3937 */
3938
3939struct seq_operations slabinfo_op = {
Pekka Enbergb28a02d2006-01-08 01:00:37 -08003940 .start = s_start,
3941 .next = s_next,
3942 .stop = s_stop,
3943 .show = s_show,
Linus Torvalds1da177e2005-04-16 15:20:36 -07003944};
3945
3946#define MAX_SLABINFO_WRITE 128
3947/**
3948 * slabinfo_write - Tuning for the slab allocator
3949 * @file: unused
3950 * @buffer: user buffer
3951 * @count: data length
3952 * @ppos: unused
3953 */
Pekka Enbergb28a02d2006-01-08 01:00:37 -08003954ssize_t slabinfo_write(struct file *file, const char __user * buffer,
3955 size_t count, loff_t *ppos)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003956{
Pekka Enbergb28a02d2006-01-08 01:00:37 -08003957 char kbuf[MAX_SLABINFO_WRITE + 1], *tmp;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003958 int limit, batchcount, shared, res;
3959 struct list_head *p;
Pekka Enbergb28a02d2006-01-08 01:00:37 -08003960
Linus Torvalds1da177e2005-04-16 15:20:36 -07003961 if (count > MAX_SLABINFO_WRITE)
3962 return -EINVAL;
3963 if (copy_from_user(&kbuf, buffer, count))
3964 return -EFAULT;
Pekka Enbergb28a02d2006-01-08 01:00:37 -08003965 kbuf[MAX_SLABINFO_WRITE] = '\0';
Linus Torvalds1da177e2005-04-16 15:20:36 -07003966
3967 tmp = strchr(kbuf, ' ');
3968 if (!tmp)
3969 return -EINVAL;
3970 *tmp = '\0';
3971 tmp++;
3972 if (sscanf(tmp, " %d %d %d", &limit, &batchcount, &shared) != 3)
3973 return -EINVAL;
3974
3975 /* Find the cache in the chain of caches. */
Ingo Molnarfc0abb12006-01-18 17:42:33 -08003976 mutex_lock(&cache_chain_mutex);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003977 res = -EINVAL;
Pekka Enbergb28a02d2006-01-08 01:00:37 -08003978 list_for_each(p, &cache_chain) {
Andrew Mortona737b3e2006-03-22 00:08:11 -08003979 struct kmem_cache *cachep;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003980
Andrew Mortona737b3e2006-03-22 00:08:11 -08003981 cachep = list_entry(p, struct kmem_cache, next);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003982 if (!strcmp(cachep->name, kbuf)) {
Andrew Mortona737b3e2006-03-22 00:08:11 -08003983 if (limit < 1 || batchcount < 1 ||
3984 batchcount > limit || shared < 0) {
Christoph Lametere498be72005-09-09 13:03:32 -07003985 res = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003986 } else {
Christoph Lametere498be72005-09-09 13:03:32 -07003987 res = do_tune_cpucache(cachep, limit,
Pekka Enbergb28a02d2006-01-08 01:00:37 -08003988 batchcount, shared);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003989 }
3990 break;
3991 }
3992 }
Ingo Molnarfc0abb12006-01-18 17:42:33 -08003993 mutex_unlock(&cache_chain_mutex);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003994 if (res >= 0)
3995 res = count;
3996 return res;
3997}
Al Viro871751e2006-03-25 03:06:39 -08003998
3999#ifdef CONFIG_DEBUG_SLAB_LEAK
4000
4001static void *leaks_start(struct seq_file *m, loff_t *pos)
4002{
4003 loff_t n = *pos;
4004 struct list_head *p;
4005
4006 mutex_lock(&cache_chain_mutex);
4007 p = cache_chain.next;
4008 while (n--) {
4009 p = p->next;
4010 if (p == &cache_chain)
4011 return NULL;
4012 }
4013 return list_entry(p, struct kmem_cache, next);
4014}
4015
4016static inline int add_caller(unsigned long *n, unsigned long v)
4017{
4018 unsigned long *p;
4019 int l;
4020 if (!v)
4021 return 1;
4022 l = n[1];
4023 p = n + 2;
4024 while (l) {
4025 int i = l/2;
4026 unsigned long *q = p + 2 * i;
4027 if (*q == v) {
4028 q[1]++;
4029 return 1;
4030 }
4031 if (*q > v) {
4032 l = i;
4033 } else {
4034 p = q + 2;
4035 l -= i + 1;
4036 }
4037 }
4038 if (++n[1] == n[0])
4039 return 0;
4040 memmove(p + 2, p, n[1] * 2 * sizeof(unsigned long) - ((void *)p - (void *)n));
4041 p[0] = v;
4042 p[1] = 1;
4043 return 1;
4044}
4045
4046static void handle_slab(unsigned long *n, struct kmem_cache *c, struct slab *s)
4047{
4048 void *p;
4049 int i;
4050 if (n[0] == n[1])
4051 return;
4052 for (i = 0, p = s->s_mem; i < c->num; i++, p += c->buffer_size) {
4053 if (slab_bufctl(s)[i] != BUFCTL_ACTIVE)
4054 continue;
4055 if (!add_caller(n, (unsigned long)*dbg_userword(c, p)))
4056 return;
4057 }
4058}
4059
4060static void show_symbol(struct seq_file *m, unsigned long address)
4061{
4062#ifdef CONFIG_KALLSYMS
4063 char *modname;
4064 const char *name;
4065 unsigned long offset, size;
4066 char namebuf[KSYM_NAME_LEN+1];
4067
4068 name = kallsyms_lookup(address, &size, &offset, &modname, namebuf);
4069
4070 if (name) {
4071 seq_printf(m, "%s+%#lx/%#lx", name, offset, size);
4072 if (modname)
4073 seq_printf(m, " [%s]", modname);
4074 return;
4075 }
4076#endif
4077 seq_printf(m, "%p", (void *)address);
4078}
4079
4080static int leaks_show(struct seq_file *m, void *p)
4081{
4082 struct kmem_cache *cachep = p;
4083 struct list_head *q;
4084 struct slab *slabp;
4085 struct kmem_list3 *l3;
4086 const char *name;
4087 unsigned long *n = m->private;
4088 int node;
4089 int i;
4090
4091 if (!(cachep->flags & SLAB_STORE_USER))
4092 return 0;
4093 if (!(cachep->flags & SLAB_RED_ZONE))
4094 return 0;
4095
4096 /* OK, we can do it */
4097
4098 n[1] = 0;
4099
4100 for_each_online_node(node) {
4101 l3 = cachep->nodelists[node];
4102 if (!l3)
4103 continue;
4104
4105 check_irq_on();
4106 spin_lock_irq(&l3->list_lock);
4107
4108 list_for_each(q, &l3->slabs_full) {
4109 slabp = list_entry(q, struct slab, list);
4110 handle_slab(n, cachep, slabp);
4111 }
4112 list_for_each(q, &l3->slabs_partial) {
4113 slabp = list_entry(q, struct slab, list);
4114 handle_slab(n, cachep, slabp);
4115 }
4116 spin_unlock_irq(&l3->list_lock);
4117 }
4118 name = cachep->name;
4119 if (n[0] == n[1]) {
4120 /* Increase the buffer size */
4121 mutex_unlock(&cache_chain_mutex);
4122 m->private = kzalloc(n[0] * 4 * sizeof(unsigned long), GFP_KERNEL);
4123 if (!m->private) {
4124 /* Too bad, we are really out */
4125 m->private = n;
4126 mutex_lock(&cache_chain_mutex);
4127 return -ENOMEM;
4128 }
4129 *(unsigned long *)m->private = n[0] * 2;
4130 kfree(n);
4131 mutex_lock(&cache_chain_mutex);
4132 /* Now make sure this entry will be retried */
4133 m->count = m->size;
4134 return 0;
4135 }
4136 for (i = 0; i < n[1]; i++) {
4137 seq_printf(m, "%s: %lu ", name, n[2*i+3]);
4138 show_symbol(m, n[2*i+2]);
4139 seq_putc(m, '\n');
4140 }
4141 return 0;
4142}
4143
4144struct seq_operations slabstats_op = {
4145 .start = leaks_start,
4146 .next = s_next,
4147 .stop = s_stop,
4148 .show = leaks_show,
4149};
4150#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07004151#endif
4152
Manfred Spraul00e145b2005-09-03 15:55:07 -07004153/**
4154 * ksize - get the actual amount of memory allocated for a given object
4155 * @objp: Pointer to the object
4156 *
4157 * kmalloc may internally round up allocations and return more memory
4158 * than requested. ksize() can be used to determine the actual amount of
4159 * memory allocated. The caller may use this additional memory, even though
4160 * a smaller amount of memory was initially specified with the kmalloc call.
4161 * The caller must guarantee that objp points to a valid object previously
4162 * allocated with either kmalloc() or kmem_cache_alloc(). The object
4163 * must not be freed during the duration of the call.
4164 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07004165unsigned int ksize(const void *objp)
4166{
Manfred Spraul00e145b2005-09-03 15:55:07 -07004167 if (unlikely(objp == NULL))
4168 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004169
Pekka Enberg6ed5eb2212006-02-01 03:05:49 -08004170 return obj_size(virt_to_cache(objp));
Linus Torvalds1da177e2005-04-16 15:20:36 -07004171}