blob: f0bd7857ab3bed2adf6649e60dda6ad712ef0b92 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * linux/mm/slab.c
3 * Written by Mark Hemment, 1996/97.
4 * (markhe@nextd.demon.co.uk)
5 *
6 * kmem_cache_destroy() + some cleanup - 1999 Andrea Arcangeli
7 *
8 * Major cleanup, different bufctl logic, per-cpu arrays
9 * (c) 2000 Manfred Spraul
10 *
11 * Cleanup, make the head arrays unconditional, preparation for NUMA
12 * (c) 2002 Manfred Spraul
13 *
14 * An implementation of the Slab Allocator as described in outline in;
15 * UNIX Internals: The New Frontiers by Uresh Vahalia
16 * Pub: Prentice Hall ISBN 0-13-101908-2
17 * or with a little more detail in;
18 * The Slab Allocator: An Object-Caching Kernel Memory Allocator
19 * Jeff Bonwick (Sun Microsystems).
20 * Presented at: USENIX Summer 1994 Technical Conference
21 *
22 * The memory is organized in caches, one cache for each object type.
23 * (e.g. inode_cache, dentry_cache, buffer_head, vm_area_struct)
24 * Each cache consists out of many slabs (they are small (usually one
25 * page long) and always contiguous), and each slab contains multiple
26 * initialized objects.
27 *
28 * This means, that your constructor is used only for newly allocated
Simon Arlott183ff222007-10-20 01:27:18 +020029 * slabs and you must pass objects with the same initializations to
Linus Torvalds1da177e2005-04-16 15:20:36 -070030 * kmem_cache_free.
31 *
32 * Each cache can only support one memory type (GFP_DMA, GFP_HIGHMEM,
33 * normal). If you need a special memory type, then must create a new
34 * cache for that memory type.
35 *
36 * In order to reduce fragmentation, the slabs are sorted in 3 groups:
37 * full slabs with 0 free objects
38 * partial slabs
39 * empty slabs with no allocated objects
40 *
41 * If partial slabs exist, then new allocations come from these slabs,
42 * otherwise from empty slabs or new slabs are allocated.
43 *
44 * kmem_cache_destroy() CAN CRASH if you try to allocate from the cache
45 * during kmem_cache_destroy(). The caller must prevent concurrent allocs.
46 *
47 * Each cache has a short per-cpu head array, most allocs
48 * and frees go into that array, and if that array overflows, then 1/2
49 * of the entries in the array are given back into the global cache.
50 * The head array is strictly LIFO and should improve the cache hit rates.
51 * On SMP, it additionally reduces the spinlock operations.
52 *
Andrew Mortona737b3e2006-03-22 00:08:11 -080053 * The c_cpuarray may not be read with enabled local interrupts -
Linus Torvalds1da177e2005-04-16 15:20:36 -070054 * it's changed with a smp_call_function().
55 *
56 * SMP synchronization:
57 * constructors and destructors are called without any locking.
Pekka Enberg343e0d72006-02-01 03:05:50 -080058 * Several members in struct kmem_cache and struct slab never change, they
Linus Torvalds1da177e2005-04-16 15:20:36 -070059 * are accessed without any locking.
60 * The per-cpu arrays are never accessed from the wrong cpu, no locking,
61 * and local interrupts are disabled so slab code is preempt-safe.
62 * The non-constant members are protected with a per-cache irq spinlock.
63 *
64 * Many thanks to Mark Hemment, who wrote another per-cpu slab patch
65 * in 2000 - many ideas in the current implementation are derived from
66 * his patch.
67 *
68 * Further notes from the original documentation:
69 *
70 * 11 April '97. Started multi-threading - markhe
Ingo Molnarfc0abb12006-01-18 17:42:33 -080071 * The global cache-chain is protected by the mutex 'cache_chain_mutex'.
Linus Torvalds1da177e2005-04-16 15:20:36 -070072 * The sem is only needed when accessing/extending the cache-chain, which
73 * can never happen inside an interrupt (kmem_cache_create(),
74 * kmem_cache_shrink() and kmem_cache_reap()).
75 *
76 * At present, each engine can be growing a cache. This should be blocked.
77 *
Christoph Lametere498be72005-09-09 13:03:32 -070078 * 15 March 2005. NUMA slab allocator.
79 * Shai Fultheim <shai@scalex86.org>.
80 * Shobhit Dayal <shobhit@calsoftinc.com>
81 * Alok N Kataria <alokk@calsoftinc.com>
82 * Christoph Lameter <christoph@lameter.com>
83 *
84 * Modified the slab allocator to be node aware on NUMA systems.
85 * Each node has its own list of partial, free and full slabs.
86 * All object allocations for a node occur from node specific slab lists.
Linus Torvalds1da177e2005-04-16 15:20:36 -070087 */
88
Linus Torvalds1da177e2005-04-16 15:20:36 -070089#include <linux/slab.h>
90#include <linux/mm.h>
Randy Dunlapc9cf5522006-06-27 02:53:52 -070091#include <linux/poison.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070092#include <linux/swap.h>
93#include <linux/cache.h>
94#include <linux/interrupt.h>
95#include <linux/init.h>
96#include <linux/compiler.h>
Paul Jackson101a5002006-03-24 03:16:07 -080097#include <linux/cpuset.h>
Alexey Dobriyana0ec95a2008-10-06 00:59:10 +040098#include <linux/proc_fs.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070099#include <linux/seq_file.h>
100#include <linux/notifier.h>
101#include <linux/kallsyms.h>
102#include <linux/cpu.h>
103#include <linux/sysctl.h>
104#include <linux/module.h>
105#include <linux/rcupdate.h>
Paulo Marques543537b2005-06-23 00:09:02 -0700106#include <linux/string.h>
Andrew Morton138ae662006-12-06 20:36:41 -0800107#include <linux/uaccess.h>
Christoph Lametere498be72005-09-09 13:03:32 -0700108#include <linux/nodemask.h>
Catalin Marinasd5cff632009-06-11 13:22:40 +0100109#include <linux/kmemleak.h>
Christoph Lameterdc85da12006-01-18 17:42:36 -0800110#include <linux/mempolicy.h>
Ingo Molnarfc0abb12006-01-18 17:42:33 -0800111#include <linux/mutex.h>
Akinobu Mita8a8b6502006-12-08 02:39:44 -0800112#include <linux/fault-inject.h>
Ingo Molnare7eebaf2006-06-27 02:54:55 -0700113#include <linux/rtmutex.h>
Eric Dumazet6a2d7a92006-12-13 00:34:27 -0800114#include <linux/reciprocal_div.h>
Thomas Gleixner3ac7fe52008-04-30 00:55:01 -0700115#include <linux/debugobjects.h>
Pekka Enbergc175eea2008-05-09 20:35:53 +0200116#include <linux/kmemcheck.h>
David Rientjes8f9f8d92010-03-27 19:40:47 -0700117#include <linux/memory.h>
Linus Torvalds268bb0c2011-05-20 12:50:29 -0700118#include <linux/prefetch.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700119
Linus Torvalds1da177e2005-04-16 15:20:36 -0700120#include <asm/cacheflush.h>
121#include <asm/tlbflush.h>
122#include <asm/page.h>
123
Steven Rostedt4dee6b62012-01-09 17:15:42 -0500124#include <trace/events/kmem.h>
125
Linus Torvalds1da177e2005-04-16 15:20:36 -0700126/*
Christoph Lameter50953fe2007-05-06 14:50:16 -0700127 * DEBUG - 1 for kmem_cache_create() to honour; SLAB_RED_ZONE & SLAB_POISON.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700128 * 0 for faster, smaller code (especially in the critical paths).
129 *
130 * STATS - 1 to collect stats for /proc/slabinfo.
131 * 0 for faster, smaller code (especially in the critical paths).
132 *
133 * FORCED_DEBUG - 1 enables SLAB_RED_ZONE and SLAB_POISON (if possible)
134 */
135
136#ifdef CONFIG_DEBUG_SLAB
137#define DEBUG 1
138#define STATS 1
139#define FORCED_DEBUG 1
140#else
141#define DEBUG 0
142#define STATS 0
143#define FORCED_DEBUG 0
144#endif
145
Linus Torvalds1da177e2005-04-16 15:20:36 -0700146/* Shouldn't this be in a header file somewhere? */
147#define BYTES_PER_WORD sizeof(void *)
David Woodhouse87a927c2007-07-04 21:26:44 -0400148#define REDZONE_ALIGN max(BYTES_PER_WORD, __alignof__(unsigned long long))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700149
Linus Torvalds1da177e2005-04-16 15:20:36 -0700150#ifndef ARCH_KMALLOC_FLAGS
151#define ARCH_KMALLOC_FLAGS SLAB_HWCACHE_ALIGN
152#endif
153
154/* Legal flag mask for kmem_cache_create(). */
155#if DEBUG
Christoph Lameter50953fe2007-05-06 14:50:16 -0700156# define CREATE_MASK (SLAB_RED_ZONE | \
Linus Torvalds1da177e2005-04-16 15:20:36 -0700157 SLAB_POISON | SLAB_HWCACHE_ALIGN | \
Christoph Lameterac2b8982006-03-22 00:08:15 -0800158 SLAB_CACHE_DMA | \
Christoph Lameter5af60832007-05-06 14:49:56 -0700159 SLAB_STORE_USER | \
Linus Torvalds1da177e2005-04-16 15:20:36 -0700160 SLAB_RECLAIM_ACCOUNT | SLAB_PANIC | \
Thomas Gleixner3ac7fe52008-04-30 00:55:01 -0700161 SLAB_DESTROY_BY_RCU | SLAB_MEM_SPREAD | \
Pekka Enbergc175eea2008-05-09 20:35:53 +0200162 SLAB_DEBUG_OBJECTS | SLAB_NOLEAKTRACE | SLAB_NOTRACK)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700163#else
Christoph Lameterac2b8982006-03-22 00:08:15 -0800164# define CREATE_MASK (SLAB_HWCACHE_ALIGN | \
Christoph Lameter5af60832007-05-06 14:49:56 -0700165 SLAB_CACHE_DMA | \
Linus Torvalds1da177e2005-04-16 15:20:36 -0700166 SLAB_RECLAIM_ACCOUNT | SLAB_PANIC | \
Thomas Gleixner3ac7fe52008-04-30 00:55:01 -0700167 SLAB_DESTROY_BY_RCU | SLAB_MEM_SPREAD | \
Pekka Enbergc175eea2008-05-09 20:35:53 +0200168 SLAB_DEBUG_OBJECTS | SLAB_NOLEAKTRACE | SLAB_NOTRACK)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700169#endif
170
171/*
172 * kmem_bufctl_t:
173 *
174 * Bufctl's are used for linking objs within a slab
175 * linked offsets.
176 *
177 * This implementation relies on "struct page" for locating the cache &
178 * slab an object belongs to.
179 * This allows the bufctl structure to be small (one int), but limits
180 * the number of objects a slab (not a cache) can contain when off-slab
181 * bufctls are used. The limit is the size of the largest general cache
182 * that does not use off-slab slabs.
183 * For 32bit archs with 4 kB pages, is this 56.
184 * This is not serious, as it is only for large objects, when it is unwise
185 * to have too many per slab.
186 * Note: This limit can be raised by introducing a general cache whose size
187 * is less than 512 (PAGE_SIZE<<3), but greater than 256.
188 */
189
Kyle Moffettfa5b08d2005-09-03 15:55:03 -0700190typedef unsigned int kmem_bufctl_t;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700191#define BUFCTL_END (((kmem_bufctl_t)(~0U))-0)
192#define BUFCTL_FREE (((kmem_bufctl_t)(~0U))-1)
Al Viro871751e2006-03-25 03:06:39 -0800193#define BUFCTL_ACTIVE (((kmem_bufctl_t)(~0U))-2)
194#define SLAB_LIMIT (((kmem_bufctl_t)(~0U))-3)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700195
Linus Torvalds1da177e2005-04-16 15:20:36 -0700196/*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700197 * struct slab_rcu
198 *
199 * slab_destroy on a SLAB_DESTROY_BY_RCU cache uses this structure to
200 * arrange for kmem_freepages to be called via RCU. This is useful if
201 * we need to approach a kernel structure obliquely, from its address
202 * obtained without the usual locking. We can lock the structure to
203 * stabilize it and check it's still at the given address, only if we
204 * can be sure that the memory has not been meanwhile reused for some
205 * other kind of object (which our subsystem's lock might corrupt).
206 *
207 * rcu_read_lock before reading the address, then rcu_read_unlock after
208 * taking the spinlock within the structure expected at that address.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700209 */
210struct slab_rcu {
Pekka Enbergb28a02d2006-01-08 01:00:37 -0800211 struct rcu_head head;
Pekka Enberg343e0d72006-02-01 03:05:50 -0800212 struct kmem_cache *cachep;
Pekka Enbergb28a02d2006-01-08 01:00:37 -0800213 void *addr;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700214};
215
216/*
Lai Jiangshan5bfe53a2011-03-10 15:22:24 +0800217 * struct slab
218 *
219 * Manages the objs in a slab. Placed either at the beginning of mem allocated
220 * for a slab, or allocated from an general cache.
221 * Slabs are chained into three list: fully used, partial, fully free slabs.
222 */
223struct slab {
224 union {
225 struct {
226 struct list_head list;
227 unsigned long colouroff;
228 void *s_mem; /* including colour offset */
229 unsigned int inuse; /* num of objs active in slab */
230 kmem_bufctl_t free;
231 unsigned short nodeid;
232 };
233 struct slab_rcu __slab_cover_slab_rcu;
234 };
235};
236
237/*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700238 * struct array_cache
239 *
Linus Torvalds1da177e2005-04-16 15:20:36 -0700240 * Purpose:
241 * - LIFO ordering, to hand out cache-warm objects from _alloc
242 * - reduce the number of linked list operations
243 * - reduce spinlock operations
244 *
245 * The limit is stored in the per-cpu structure to reduce the data cache
246 * footprint.
247 *
248 */
249struct array_cache {
250 unsigned int avail;
251 unsigned int limit;
252 unsigned int batchcount;
253 unsigned int touched;
Christoph Lametere498be72005-09-09 13:03:32 -0700254 spinlock_t lock;
Robert P. J. Daybda5b652007-10-16 23:30:05 -0700255 void *entry[]; /*
Andrew Mortona737b3e2006-03-22 00:08:11 -0800256 * Must have this definition in here for the proper
257 * alignment of array_cache. Also simplifies accessing
258 * the entries.
Andrew Mortona737b3e2006-03-22 00:08:11 -0800259 */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700260};
261
Andrew Mortona737b3e2006-03-22 00:08:11 -0800262/*
263 * bootstrap: The caches do not work without cpuarrays anymore, but the
264 * cpuarrays are allocated from the generic caches...
Linus Torvalds1da177e2005-04-16 15:20:36 -0700265 */
266#define BOOT_CPUCACHE_ENTRIES 1
267struct arraycache_init {
268 struct array_cache cache;
Pekka Enbergb28a02d2006-01-08 01:00:37 -0800269 void *entries[BOOT_CPUCACHE_ENTRIES];
Linus Torvalds1da177e2005-04-16 15:20:36 -0700270};
271
272/*
Christoph Lametere498be72005-09-09 13:03:32 -0700273 * The slab lists for all objects.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700274 */
275struct kmem_list3 {
Pekka Enbergb28a02d2006-01-08 01:00:37 -0800276 struct list_head slabs_partial; /* partial list first, better asm code */
277 struct list_head slabs_full;
278 struct list_head slabs_free;
279 unsigned long free_objects;
Pekka Enbergb28a02d2006-01-08 01:00:37 -0800280 unsigned int free_limit;
Ravikiran G Thirumalai2e1217c2006-02-04 23:27:56 -0800281 unsigned int colour_next; /* Per-node cache coloring */
Pekka Enbergb28a02d2006-01-08 01:00:37 -0800282 spinlock_t list_lock;
283 struct array_cache *shared; /* shared per node */
284 struct array_cache **alien; /* on other nodes */
Christoph Lameter35386e32006-03-22 00:09:05 -0800285 unsigned long next_reap; /* updated without locking */
286 int free_touched; /* updated without locking */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700287};
288
Christoph Lametere498be72005-09-09 13:03:32 -0700289/*
290 * Need this for bootstrapping a per node allocator.
291 */
Pekka Enberg556a1692008-01-25 08:20:51 +0200292#define NUM_INIT_LISTS (3 * MAX_NUMNODES)
H Hartley Sweeten68a1b192011-01-11 17:49:32 -0600293static struct kmem_list3 __initdata initkmem_list3[NUM_INIT_LISTS];
Christoph Lametere498be72005-09-09 13:03:32 -0700294#define CACHE_CACHE 0
Pekka Enberg556a1692008-01-25 08:20:51 +0200295#define SIZE_AC MAX_NUMNODES
296#define SIZE_L3 (2 * MAX_NUMNODES)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700297
Christoph Lametered11d9e2006-06-30 01:55:45 -0700298static int drain_freelist(struct kmem_cache *cache,
299 struct kmem_list3 *l3, int tofree);
300static void free_block(struct kmem_cache *cachep, void **objpp, int len,
301 int node);
Pekka Enberg83b519e2009-06-10 19:40:04 +0300302static int enable_cpucache(struct kmem_cache *cachep, gfp_t gfp);
David Howells65f27f32006-11-22 14:55:48 +0000303static void cache_reap(struct work_struct *unused);
Christoph Lametered11d9e2006-06-30 01:55:45 -0700304
Christoph Lametere498be72005-09-09 13:03:32 -0700305/*
Andrew Mortona737b3e2006-03-22 00:08:11 -0800306 * This function must be completely optimized away if a constant is passed to
307 * it. Mostly the same as what is in linux/slab.h except it returns an index.
Christoph Lametere498be72005-09-09 13:03:32 -0700308 */
Ivan Kokshaysky7243cc02005-09-22 21:43:58 -0700309static __always_inline int index_of(const size_t size)
Christoph Lametere498be72005-09-09 13:03:32 -0700310{
Steven Rostedt5ec8a842006-02-01 03:05:44 -0800311 extern void __bad_size(void);
312
Christoph Lametere498be72005-09-09 13:03:32 -0700313 if (__builtin_constant_p(size)) {
314 int i = 0;
315
316#define CACHE(x) \
317 if (size <=x) \
318 return i; \
319 else \
320 i++;
Joe Perches1c61fc42008-03-05 13:58:17 -0800321#include <linux/kmalloc_sizes.h>
Christoph Lametere498be72005-09-09 13:03:32 -0700322#undef CACHE
Steven Rostedt5ec8a842006-02-01 03:05:44 -0800323 __bad_size();
Ivan Kokshaysky7243cc02005-09-22 21:43:58 -0700324 } else
Steven Rostedt5ec8a842006-02-01 03:05:44 -0800325 __bad_size();
Christoph Lametere498be72005-09-09 13:03:32 -0700326 return 0;
327}
328
Ingo Molnare0a42722006-06-23 02:03:46 -0700329static int slab_early_init = 1;
330
Christoph Lametere498be72005-09-09 13:03:32 -0700331#define INDEX_AC index_of(sizeof(struct arraycache_init))
332#define INDEX_L3 index_of(sizeof(struct kmem_list3))
333
Pekka Enberg5295a742006-02-01 03:05:48 -0800334static void kmem_list3_init(struct kmem_list3 *parent)
Christoph Lametere498be72005-09-09 13:03:32 -0700335{
336 INIT_LIST_HEAD(&parent->slabs_full);
337 INIT_LIST_HEAD(&parent->slabs_partial);
338 INIT_LIST_HEAD(&parent->slabs_free);
339 parent->shared = NULL;
340 parent->alien = NULL;
Ravikiran G Thirumalai2e1217c2006-02-04 23:27:56 -0800341 parent->colour_next = 0;
Christoph Lametere498be72005-09-09 13:03:32 -0700342 spin_lock_init(&parent->list_lock);
343 parent->free_objects = 0;
344 parent->free_touched = 0;
345}
346
Andrew Mortona737b3e2006-03-22 00:08:11 -0800347#define MAKE_LIST(cachep, listp, slab, nodeid) \
348 do { \
349 INIT_LIST_HEAD(listp); \
350 list_splice(&(cachep->nodelists[nodeid]->slab), listp); \
Christoph Lametere498be72005-09-09 13:03:32 -0700351 } while (0)
352
Andrew Mortona737b3e2006-03-22 00:08:11 -0800353#define MAKE_ALL_LISTS(cachep, ptr, nodeid) \
354 do { \
Christoph Lametere498be72005-09-09 13:03:32 -0700355 MAKE_LIST((cachep), (&(ptr)->slabs_full), slabs_full, nodeid); \
356 MAKE_LIST((cachep), (&(ptr)->slabs_partial), slabs_partial, nodeid); \
357 MAKE_LIST((cachep), (&(ptr)->slabs_free), slabs_free, nodeid); \
358 } while (0)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700359
Linus Torvalds1da177e2005-04-16 15:20:36 -0700360#define CFLGS_OFF_SLAB (0x80000000UL)
361#define OFF_SLAB(x) ((x)->flags & CFLGS_OFF_SLAB)
362
363#define BATCHREFILL_LIMIT 16
Andrew Mortona737b3e2006-03-22 00:08:11 -0800364/*
365 * Optimization question: fewer reaps means less probability for unnessary
366 * cpucache drain/refill cycles.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700367 *
Adrian Bunkdc6f3f22005-11-08 16:44:08 +0100368 * OTOH the cpuarrays can contain lots of objects,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700369 * which could lock up otherwise freeable slabs.
370 */
371#define REAPTIMEOUT_CPUC (2*HZ)
372#define REAPTIMEOUT_LIST3 (4*HZ)
373
374#if STATS
375#define STATS_INC_ACTIVE(x) ((x)->num_active++)
376#define STATS_DEC_ACTIVE(x) ((x)->num_active--)
377#define STATS_INC_ALLOCED(x) ((x)->num_allocations++)
378#define STATS_INC_GROWN(x) ((x)->grown++)
Christoph Lametered11d9e2006-06-30 01:55:45 -0700379#define STATS_ADD_REAPED(x,y) ((x)->reaped += (y))
Andrew Mortona737b3e2006-03-22 00:08:11 -0800380#define STATS_SET_HIGH(x) \
381 do { \
382 if ((x)->num_active > (x)->high_mark) \
383 (x)->high_mark = (x)->num_active; \
384 } while (0)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700385#define STATS_INC_ERR(x) ((x)->errors++)
386#define STATS_INC_NODEALLOCS(x) ((x)->node_allocs++)
Christoph Lametere498be72005-09-09 13:03:32 -0700387#define STATS_INC_NODEFREES(x) ((x)->node_frees++)
Ravikiran G Thirumalaifb7faf32006-04-10 22:52:54 -0700388#define STATS_INC_ACOVERFLOW(x) ((x)->node_overflow++)
Andrew Mortona737b3e2006-03-22 00:08:11 -0800389#define STATS_SET_FREEABLE(x, i) \
390 do { \
391 if ((x)->max_freeable < i) \
392 (x)->max_freeable = i; \
393 } while (0)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700394#define STATS_INC_ALLOCHIT(x) atomic_inc(&(x)->allochit)
395#define STATS_INC_ALLOCMISS(x) atomic_inc(&(x)->allocmiss)
396#define STATS_INC_FREEHIT(x) atomic_inc(&(x)->freehit)
397#define STATS_INC_FREEMISS(x) atomic_inc(&(x)->freemiss)
398#else
399#define STATS_INC_ACTIVE(x) do { } while (0)
400#define STATS_DEC_ACTIVE(x) do { } while (0)
401#define STATS_INC_ALLOCED(x) do { } while (0)
402#define STATS_INC_GROWN(x) do { } while (0)
Andi Kleen4e60c862010-08-09 17:19:03 -0700403#define STATS_ADD_REAPED(x,y) do { (void)(y); } while (0)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700404#define STATS_SET_HIGH(x) do { } while (0)
405#define STATS_INC_ERR(x) do { } while (0)
406#define STATS_INC_NODEALLOCS(x) do { } while (0)
Christoph Lametere498be72005-09-09 13:03:32 -0700407#define STATS_INC_NODEFREES(x) do { } while (0)
Ravikiran G Thirumalaifb7faf32006-04-10 22:52:54 -0700408#define STATS_INC_ACOVERFLOW(x) do { } while (0)
Andrew Mortona737b3e2006-03-22 00:08:11 -0800409#define STATS_SET_FREEABLE(x, i) do { } while (0)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700410#define STATS_INC_ALLOCHIT(x) do { } while (0)
411#define STATS_INC_ALLOCMISS(x) do { } while (0)
412#define STATS_INC_FREEHIT(x) do { } while (0)
413#define STATS_INC_FREEMISS(x) do { } while (0)
414#endif
415
416#if DEBUG
Linus Torvalds1da177e2005-04-16 15:20:36 -0700417
Andrew Mortona737b3e2006-03-22 00:08:11 -0800418/*
419 * memory layout of objects:
Linus Torvalds1da177e2005-04-16 15:20:36 -0700420 * 0 : objp
Manfred Spraul3dafccf2006-02-01 03:05:42 -0800421 * 0 .. cachep->obj_offset - BYTES_PER_WORD - 1: padding. This ensures that
Linus Torvalds1da177e2005-04-16 15:20:36 -0700422 * the end of an object is aligned with the end of the real
423 * allocation. Catches writes behind the end of the allocation.
Manfred Spraul3dafccf2006-02-01 03:05:42 -0800424 * cachep->obj_offset - BYTES_PER_WORD .. cachep->obj_offset - 1:
Linus Torvalds1da177e2005-04-16 15:20:36 -0700425 * redzone word.
Manfred Spraul3dafccf2006-02-01 03:05:42 -0800426 * cachep->obj_offset: The real object.
427 * cachep->buffer_size - 2* BYTES_PER_WORD: redzone word [BYTES_PER_WORD long]
Andrew Mortona737b3e2006-03-22 00:08:11 -0800428 * cachep->buffer_size - 1* BYTES_PER_WORD: last caller address
429 * [BYTES_PER_WORD long]
Linus Torvalds1da177e2005-04-16 15:20:36 -0700430 */
Pekka Enberg343e0d72006-02-01 03:05:50 -0800431static int obj_offset(struct kmem_cache *cachep)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700432{
Manfred Spraul3dafccf2006-02-01 03:05:42 -0800433 return cachep->obj_offset;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700434}
435
Pekka Enberg343e0d72006-02-01 03:05:50 -0800436static int obj_size(struct kmem_cache *cachep)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700437{
Manfred Spraul3dafccf2006-02-01 03:05:42 -0800438 return cachep->obj_size;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700439}
440
David Woodhouseb46b8f12007-05-08 00:22:59 -0700441static unsigned long long *dbg_redzone1(struct kmem_cache *cachep, void *objp)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700442{
443 BUG_ON(!(cachep->flags & SLAB_RED_ZONE));
David Woodhouseb46b8f12007-05-08 00:22:59 -0700444 return (unsigned long long*) (objp + obj_offset(cachep) -
445 sizeof(unsigned long long));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700446}
447
David Woodhouseb46b8f12007-05-08 00:22:59 -0700448static unsigned long long *dbg_redzone2(struct kmem_cache *cachep, void *objp)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700449{
450 BUG_ON(!(cachep->flags & SLAB_RED_ZONE));
451 if (cachep->flags & SLAB_STORE_USER)
David Woodhouseb46b8f12007-05-08 00:22:59 -0700452 return (unsigned long long *)(objp + cachep->buffer_size -
453 sizeof(unsigned long long) -
David Woodhouse87a927c2007-07-04 21:26:44 -0400454 REDZONE_ALIGN);
David Woodhouseb46b8f12007-05-08 00:22:59 -0700455 return (unsigned long long *) (objp + cachep->buffer_size -
456 sizeof(unsigned long long));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700457}
458
Pekka Enberg343e0d72006-02-01 03:05:50 -0800459static void **dbg_userword(struct kmem_cache *cachep, void *objp)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700460{
461 BUG_ON(!(cachep->flags & SLAB_STORE_USER));
Manfred Spraul3dafccf2006-02-01 03:05:42 -0800462 return (void **)(objp + cachep->buffer_size - BYTES_PER_WORD);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700463}
464
465#else
466
Manfred Spraul3dafccf2006-02-01 03:05:42 -0800467#define obj_offset(x) 0
468#define obj_size(cachep) (cachep->buffer_size)
David Woodhouseb46b8f12007-05-08 00:22:59 -0700469#define dbg_redzone1(cachep, objp) ({BUG(); (unsigned long long *)NULL;})
470#define dbg_redzone2(cachep, objp) ({BUG(); (unsigned long long *)NULL;})
Linus Torvalds1da177e2005-04-16 15:20:36 -0700471#define dbg_userword(cachep, objp) ({BUG(); (void **)NULL;})
472
473#endif
474
Li Zefan0f24f122009-12-11 15:45:30 +0800475#ifdef CONFIG_TRACING
Eduard - Gabriel Munteanu36555752008-08-10 20:14:05 +0300476size_t slab_buffer_size(struct kmem_cache *cachep)
477{
478 return cachep->buffer_size;
479}
480EXPORT_SYMBOL(slab_buffer_size);
481#endif
482
Linus Torvalds1da177e2005-04-16 15:20:36 -0700483/*
David Rientjes3df1ccc2011-10-18 22:09:28 -0700484 * Do not go above this order unless 0 objects fit into the slab or
485 * overridden on the command line.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700486 */
David Rientjes543585c2011-10-18 22:09:24 -0700487#define SLAB_MAX_ORDER_HI 1
488#define SLAB_MAX_ORDER_LO 0
489static int slab_max_order = SLAB_MAX_ORDER_LO;
David Rientjes3df1ccc2011-10-18 22:09:28 -0700490static bool slab_max_order_set __initdata;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700491
Andrew Mortona737b3e2006-03-22 00:08:11 -0800492/*
493 * Functions for storing/retrieving the cachep and or slab from the page
494 * allocator. These are used to find the slab an obj belongs to. With kfree(),
495 * these are used to find the cache which an obj belongs to.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700496 */
Pekka Enberg065d41c2005-11-13 16:06:46 -0800497static inline void page_set_cache(struct page *page, struct kmem_cache *cache)
498{
499 page->lru.next = (struct list_head *)cache;
500}
501
502static inline struct kmem_cache *page_get_cache(struct page *page)
503{
Christoph Lameterd85f3382007-05-06 14:49:39 -0700504 page = compound_head(page);
Pekka Enbergddc2e812006-06-23 02:03:40 -0700505 BUG_ON(!PageSlab(page));
Pekka Enberg065d41c2005-11-13 16:06:46 -0800506 return (struct kmem_cache *)page->lru.next;
507}
508
509static inline void page_set_slab(struct page *page, struct slab *slab)
510{
511 page->lru.prev = (struct list_head *)slab;
512}
513
514static inline struct slab *page_get_slab(struct page *page)
515{
Pekka Enbergddc2e812006-06-23 02:03:40 -0700516 BUG_ON(!PageSlab(page));
Pekka Enberg065d41c2005-11-13 16:06:46 -0800517 return (struct slab *)page->lru.prev;
518}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700519
Pekka Enberg6ed5eb2212006-02-01 03:05:49 -0800520static inline struct kmem_cache *virt_to_cache(const void *obj)
521{
Christoph Lameterb49af682007-05-06 14:49:41 -0700522 struct page *page = virt_to_head_page(obj);
Pekka Enberg6ed5eb2212006-02-01 03:05:49 -0800523 return page_get_cache(page);
524}
525
526static inline struct slab *virt_to_slab(const void *obj)
527{
Christoph Lameterb49af682007-05-06 14:49:41 -0700528 struct page *page = virt_to_head_page(obj);
Pekka Enberg6ed5eb2212006-02-01 03:05:49 -0800529 return page_get_slab(page);
530}
531
Pekka Enberg8fea4e92006-03-22 00:08:10 -0800532static inline void *index_to_obj(struct kmem_cache *cache, struct slab *slab,
533 unsigned int idx)
534{
535 return slab->s_mem + cache->buffer_size * idx;
536}
537
Eric Dumazet6a2d7a92006-12-13 00:34:27 -0800538/*
539 * We want to avoid an expensive divide : (offset / cache->buffer_size)
540 * Using the fact that buffer_size is a constant for a particular cache,
541 * we can replace (offset / cache->buffer_size) by
542 * reciprocal_divide(offset, cache->reciprocal_buffer_size)
543 */
544static inline unsigned int obj_to_index(const struct kmem_cache *cache,
545 const struct slab *slab, void *obj)
Pekka Enberg8fea4e92006-03-22 00:08:10 -0800546{
Eric Dumazet6a2d7a92006-12-13 00:34:27 -0800547 u32 offset = (obj - slab->s_mem);
548 return reciprocal_divide(offset, cache->reciprocal_buffer_size);
Pekka Enberg8fea4e92006-03-22 00:08:10 -0800549}
550
Andrew Mortona737b3e2006-03-22 00:08:11 -0800551/*
552 * These are the default caches for kmalloc. Custom caches can have other sizes.
553 */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700554struct cache_sizes malloc_sizes[] = {
555#define CACHE(x) { .cs_size = (x) },
556#include <linux/kmalloc_sizes.h>
557 CACHE(ULONG_MAX)
558#undef CACHE
559};
560EXPORT_SYMBOL(malloc_sizes);
561
562/* Must match cache_sizes above. Out of line to keep cache footprint low. */
563struct cache_names {
564 char *name;
565 char *name_dma;
566};
567
568static struct cache_names __initdata cache_names[] = {
569#define CACHE(x) { .name = "size-" #x, .name_dma = "size-" #x "(DMA)" },
570#include <linux/kmalloc_sizes.h>
Pekka Enbergb28a02d2006-01-08 01:00:37 -0800571 {NULL,}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700572#undef CACHE
573};
574
575static struct arraycache_init initarray_cache __initdata =
Pekka Enbergb28a02d2006-01-08 01:00:37 -0800576 { {0, BOOT_CPUCACHE_ENTRIES, 1, 0} };
Linus Torvalds1da177e2005-04-16 15:20:36 -0700577static struct arraycache_init initarray_generic =
Pekka Enbergb28a02d2006-01-08 01:00:37 -0800578 { {0, BOOT_CPUCACHE_ENTRIES, 1, 0} };
Linus Torvalds1da177e2005-04-16 15:20:36 -0700579
580/* internal cache of cache description objs */
Eric Dumazetb56efcf2011-07-20 19:04:23 +0200581static struct kmem_list3 *cache_cache_nodelists[MAX_NUMNODES];
Pekka Enberg343e0d72006-02-01 03:05:50 -0800582static struct kmem_cache cache_cache = {
Eric Dumazetb56efcf2011-07-20 19:04:23 +0200583 .nodelists = cache_cache_nodelists,
Pekka Enbergb28a02d2006-01-08 01:00:37 -0800584 .batchcount = 1,
585 .limit = BOOT_CPUCACHE_ENTRIES,
586 .shared = 1,
Pekka Enberg343e0d72006-02-01 03:05:50 -0800587 .buffer_size = sizeof(struct kmem_cache),
Pekka Enbergb28a02d2006-01-08 01:00:37 -0800588 .name = "kmem_cache",
Linus Torvalds1da177e2005-04-16 15:20:36 -0700589};
590
Ravikiran G Thirumalai056c6242006-09-25 23:31:38 -0700591#define BAD_ALIEN_MAGIC 0x01020304ul
592
Linus Torvalds1da177e2005-04-16 15:20:36 -0700593/*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700594 * chicken and egg problem: delay the per-cpu array allocation
595 * until the general caches are up.
596 */
597static enum {
598 NONE,
Christoph Lametere498be72005-09-09 13:03:32 -0700599 PARTIAL_AC,
600 PARTIAL_L3,
Pekka Enberg8429db52009-06-12 15:58:59 +0300601 EARLY,
Peter Zijlstra52cef182011-11-28 21:12:40 +0100602 LATE,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700603 FULL
604} g_cpucache_up;
605
Mike Kravetz39d24e62006-05-15 09:44:13 -0700606/*
607 * used by boot code to determine if it can use slab based allocator
608 */
609int slab_is_available(void)
610{
Pekka Enberg8429db52009-06-12 15:58:59 +0300611 return g_cpucache_up >= EARLY;
Mike Kravetz39d24e62006-05-15 09:44:13 -0700612}
613
Pekka Enbergce79ddc2009-11-23 22:01:15 +0200614#ifdef CONFIG_LOCKDEP
615
616/*
617 * Slab sometimes uses the kmalloc slabs to store the slab headers
618 * for other slabs "off slab".
619 * The locking for this is tricky in that it nests within the locks
620 * of all other slabs in a few places; to deal with this special
621 * locking we put on-slab caches into a separate lock-class.
622 *
623 * We set lock class for alien array caches which are up during init.
624 * The lock annotation will be lost if all cpus of a node goes down and
625 * then comes back up during hotplug
626 */
627static struct lock_class_key on_slab_l3_key;
628static struct lock_class_key on_slab_alc_key;
629
Peter Zijlstra83835b32011-07-22 15:26:05 +0200630static struct lock_class_key debugobj_l3_key;
631static struct lock_class_key debugobj_alc_key;
632
633static void slab_set_lock_classes(struct kmem_cache *cachep,
634 struct lock_class_key *l3_key, struct lock_class_key *alc_key,
635 int q)
636{
637 struct array_cache **alc;
638 struct kmem_list3 *l3;
639 int r;
640
641 l3 = cachep->nodelists[q];
642 if (!l3)
643 return;
644
645 lockdep_set_class(&l3->list_lock, l3_key);
646 alc = l3->alien;
647 /*
648 * FIXME: This check for BAD_ALIEN_MAGIC
649 * should go away when common slab code is taught to
650 * work even without alien caches.
651 * Currently, non NUMA code returns BAD_ALIEN_MAGIC
652 * for alloc_alien_cache,
653 */
654 if (!alc || (unsigned long)alc == BAD_ALIEN_MAGIC)
655 return;
656 for_each_node(r) {
657 if (alc[r])
658 lockdep_set_class(&alc[r]->lock, alc_key);
659 }
660}
661
662static void slab_set_debugobj_lock_classes_node(struct kmem_cache *cachep, int node)
663{
664 slab_set_lock_classes(cachep, &debugobj_l3_key, &debugobj_alc_key, node);
665}
666
667static void slab_set_debugobj_lock_classes(struct kmem_cache *cachep)
668{
669 int node;
670
671 for_each_online_node(node)
672 slab_set_debugobj_lock_classes_node(cachep, node);
673}
674
Pekka Enbergce79ddc2009-11-23 22:01:15 +0200675static void init_node_lock_keys(int q)
676{
677 struct cache_sizes *s = malloc_sizes;
678
Peter Zijlstra52cef182011-11-28 21:12:40 +0100679 if (g_cpucache_up < LATE)
Pekka Enbergce79ddc2009-11-23 22:01:15 +0200680 return;
681
682 for (s = malloc_sizes; s->cs_size != ULONG_MAX; s++) {
Pekka Enbergce79ddc2009-11-23 22:01:15 +0200683 struct kmem_list3 *l3;
Pekka Enbergce79ddc2009-11-23 22:01:15 +0200684
685 l3 = s->cs_cachep->nodelists[q];
686 if (!l3 || OFF_SLAB(s->cs_cachep))
Pekka Enberg00afa752009-12-27 14:33:14 +0200687 continue;
Peter Zijlstra83835b32011-07-22 15:26:05 +0200688
689 slab_set_lock_classes(s->cs_cachep, &on_slab_l3_key,
690 &on_slab_alc_key, q);
Pekka Enbergce79ddc2009-11-23 22:01:15 +0200691 }
692}
693
694static inline void init_lock_keys(void)
695{
696 int node;
697
698 for_each_node(node)
699 init_node_lock_keys(node);
700}
701#else
702static void init_node_lock_keys(int q)
703{
704}
705
706static inline void init_lock_keys(void)
707{
708}
Peter Zijlstra83835b32011-07-22 15:26:05 +0200709
710static void slab_set_debugobj_lock_classes_node(struct kmem_cache *cachep, int node)
711{
712}
713
714static void slab_set_debugobj_lock_classes(struct kmem_cache *cachep)
715{
716}
Pekka Enbergce79ddc2009-11-23 22:01:15 +0200717#endif
718
719/*
720 * Guard access to the cache-chain.
721 */
722static DEFINE_MUTEX(cache_chain_mutex);
723static struct list_head cache_chain;
724
Tejun Heo1871e522009-10-29 22:34:13 +0900725static DEFINE_PER_CPU(struct delayed_work, slab_reap_work);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700726
Pekka Enberg343e0d72006-02-01 03:05:50 -0800727static inline struct array_cache *cpu_cache_get(struct kmem_cache *cachep)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700728{
729 return cachep->array[smp_processor_id()];
730}
731
Andrew Mortona737b3e2006-03-22 00:08:11 -0800732static inline struct kmem_cache *__find_general_cachep(size_t size,
733 gfp_t gfpflags)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700734{
735 struct cache_sizes *csizep = malloc_sizes;
736
737#if DEBUG
738 /* This happens if someone tries to call
Pekka Enbergb28a02d2006-01-08 01:00:37 -0800739 * kmem_cache_create(), or __kmalloc(), before
740 * the generic caches are initialized.
741 */
Alok Katariac7e43c72005-09-14 12:17:53 -0700742 BUG_ON(malloc_sizes[INDEX_AC].cs_cachep == NULL);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700743#endif
Christoph Lameter6cb8f912007-07-17 04:03:22 -0700744 if (!size)
745 return ZERO_SIZE_PTR;
746
Linus Torvalds1da177e2005-04-16 15:20:36 -0700747 while (size > csizep->cs_size)
748 csizep++;
749
750 /*
Martin Hicks0abf40c2005-09-03 15:54:54 -0700751 * Really subtle: The last entry with cs->cs_size==ULONG_MAX
Linus Torvalds1da177e2005-04-16 15:20:36 -0700752 * has cs_{dma,}cachep==NULL. Thus no special case
753 * for large kmalloc calls required.
754 */
Christoph Lameter4b51d662007-02-10 01:43:10 -0800755#ifdef CONFIG_ZONE_DMA
Linus Torvalds1da177e2005-04-16 15:20:36 -0700756 if (unlikely(gfpflags & GFP_DMA))
757 return csizep->cs_dmacachep;
Christoph Lameter4b51d662007-02-10 01:43:10 -0800758#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -0700759 return csizep->cs_cachep;
760}
761
Adrian Bunkb2213852006-09-25 23:31:02 -0700762static struct kmem_cache *kmem_find_general_cachep(size_t size, gfp_t gfpflags)
Manfred Spraul97e2bde2005-05-01 08:58:38 -0700763{
764 return __find_general_cachep(size, gfpflags);
765}
Manfred Spraul97e2bde2005-05-01 08:58:38 -0700766
Steven Rostedtfbaccac2006-02-01 03:05:45 -0800767static size_t slab_mgmt_size(size_t nr_objs, size_t align)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700768{
Steven Rostedtfbaccac2006-02-01 03:05:45 -0800769 return ALIGN(sizeof(struct slab)+nr_objs*sizeof(kmem_bufctl_t), align);
770}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700771
Andrew Mortona737b3e2006-03-22 00:08:11 -0800772/*
773 * Calculate the number of objects and left-over bytes for a given buffer size.
774 */
Steven Rostedtfbaccac2006-02-01 03:05:45 -0800775static void cache_estimate(unsigned long gfporder, size_t buffer_size,
776 size_t align, int flags, size_t *left_over,
777 unsigned int *num)
778{
779 int nr_objs;
780 size_t mgmt_size;
781 size_t slab_size = PAGE_SIZE << gfporder;
782
783 /*
784 * The slab management structure can be either off the slab or
785 * on it. For the latter case, the memory allocated for a
786 * slab is used for:
787 *
788 * - The struct slab
789 * - One kmem_bufctl_t for each object
790 * - Padding to respect alignment of @align
791 * - @buffer_size bytes for each object
792 *
793 * If the slab management structure is off the slab, then the
794 * alignment will already be calculated into the size. Because
795 * the slabs are all pages aligned, the objects will be at the
796 * correct alignment when allocated.
797 */
798 if (flags & CFLGS_OFF_SLAB) {
799 mgmt_size = 0;
800 nr_objs = slab_size / buffer_size;
801
802 if (nr_objs > SLAB_LIMIT)
803 nr_objs = SLAB_LIMIT;
804 } else {
805 /*
806 * Ignore padding for the initial guess. The padding
807 * is at most @align-1 bytes, and @buffer_size is at
808 * least @align. In the worst case, this result will
809 * be one greater than the number of objects that fit
810 * into the memory allocation when taking the padding
811 * into account.
812 */
813 nr_objs = (slab_size - sizeof(struct slab)) /
814 (buffer_size + sizeof(kmem_bufctl_t));
815
816 /*
817 * This calculated number will be either the right
818 * amount, or one greater than what we want.
819 */
820 if (slab_mgmt_size(nr_objs, align) + nr_objs*buffer_size
821 > slab_size)
822 nr_objs--;
823
824 if (nr_objs > SLAB_LIMIT)
825 nr_objs = SLAB_LIMIT;
826
827 mgmt_size = slab_mgmt_size(nr_objs, align);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700828 }
Steven Rostedtfbaccac2006-02-01 03:05:45 -0800829 *num = nr_objs;
830 *left_over = slab_size - nr_objs*buffer_size - mgmt_size;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700831}
832
Harvey Harrisond40cee22008-04-30 00:55:07 -0700833#define slab_error(cachep, msg) __slab_error(__func__, cachep, msg)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700834
Andrew Mortona737b3e2006-03-22 00:08:11 -0800835static void __slab_error(const char *function, struct kmem_cache *cachep,
836 char *msg)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700837{
838 printk(KERN_ERR "slab error in %s(): cache `%s': %s\n",
Pekka Enbergb28a02d2006-01-08 01:00:37 -0800839 function, cachep->name, msg);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700840 dump_stack();
841}
842
Paul Menage3395ee02006-12-06 20:32:16 -0800843/*
844 * By default on NUMA we use alien caches to stage the freeing of
845 * objects allocated from other nodes. This causes massive memory
846 * inefficiencies when using fake NUMA setup to split memory into a
847 * large number of small nodes, so it can be disabled on the command
848 * line
849 */
850
851static int use_alien_caches __read_mostly = 1;
852static int __init noaliencache_setup(char *s)
853{
854 use_alien_caches = 0;
855 return 1;
856}
857__setup("noaliencache", noaliencache_setup);
858
David Rientjes3df1ccc2011-10-18 22:09:28 -0700859static int __init slab_max_order_setup(char *str)
860{
861 get_option(&str, &slab_max_order);
862 slab_max_order = slab_max_order < 0 ? 0 :
863 min(slab_max_order, MAX_ORDER - 1);
864 slab_max_order_set = true;
865
866 return 1;
867}
868__setup("slab_max_order=", slab_max_order_setup);
869
Christoph Lameter8fce4d82006-03-09 17:33:54 -0800870#ifdef CONFIG_NUMA
871/*
872 * Special reaping functions for NUMA systems called from cache_reap().
873 * These take care of doing round robin flushing of alien caches (containing
874 * objects freed on different nodes from which they were allocated) and the
875 * flushing of remote pcps by calling drain_node_pages.
876 */
Tejun Heo1871e522009-10-29 22:34:13 +0900877static DEFINE_PER_CPU(unsigned long, slab_reap_node);
Christoph Lameter8fce4d82006-03-09 17:33:54 -0800878
879static void init_reap_node(int cpu)
880{
881 int node;
882
Lee Schermerhorn7d6e6d02010-05-26 14:45:03 -0700883 node = next_node(cpu_to_mem(cpu), node_online_map);
Christoph Lameter8fce4d82006-03-09 17:33:54 -0800884 if (node == MAX_NUMNODES)
Paul Jackson442295c2006-03-22 00:09:11 -0800885 node = first_node(node_online_map);
Christoph Lameter8fce4d82006-03-09 17:33:54 -0800886
Tejun Heo1871e522009-10-29 22:34:13 +0900887 per_cpu(slab_reap_node, cpu) = node;
Christoph Lameter8fce4d82006-03-09 17:33:54 -0800888}
889
890static void next_reap_node(void)
891{
Christoph Lameter909ea962010-12-08 16:22:55 +0100892 int node = __this_cpu_read(slab_reap_node);
Christoph Lameter8fce4d82006-03-09 17:33:54 -0800893
Christoph Lameter8fce4d82006-03-09 17:33:54 -0800894 node = next_node(node, node_online_map);
895 if (unlikely(node >= MAX_NUMNODES))
896 node = first_node(node_online_map);
Christoph Lameter909ea962010-12-08 16:22:55 +0100897 __this_cpu_write(slab_reap_node, node);
Christoph Lameter8fce4d82006-03-09 17:33:54 -0800898}
899
900#else
901#define init_reap_node(cpu) do { } while (0)
902#define next_reap_node(void) do { } while (0)
903#endif
904
Linus Torvalds1da177e2005-04-16 15:20:36 -0700905/*
906 * Initiate the reap timer running on the target CPU. We run at around 1 to 2Hz
907 * via the workqueue/eventd.
908 * Add the CPU number into the expiration time to minimize the possibility of
909 * the CPUs getting into lockstep and contending for the global cache chain
910 * lock.
911 */
Adrian Bunk897e6792007-07-15 23:38:20 -0700912static void __cpuinit start_cpu_timer(int cpu)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700913{
Tejun Heo1871e522009-10-29 22:34:13 +0900914 struct delayed_work *reap_work = &per_cpu(slab_reap_work, cpu);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700915
916 /*
917 * When this gets called from do_initcalls via cpucache_init(),
918 * init_workqueues() has already run, so keventd will be setup
919 * at that time.
920 */
David Howells52bad642006-11-22 14:54:01 +0000921 if (keventd_up() && reap_work->work.func == NULL) {
Christoph Lameter8fce4d82006-03-09 17:33:54 -0800922 init_reap_node(cpu);
Arjan van de Ven78b43532010-07-19 10:59:42 -0700923 INIT_DELAYED_WORK_DEFERRABLE(reap_work, cache_reap);
Arjan van de Ven2b284212006-12-10 02:21:28 -0800924 schedule_delayed_work_on(cpu, reap_work,
925 __round_jiffies_relative(HZ, cpu));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700926 }
927}
928
Christoph Lametere498be72005-09-09 13:03:32 -0700929static struct array_cache *alloc_arraycache(int node, int entries,
Pekka Enberg83b519e2009-06-10 19:40:04 +0300930 int batchcount, gfp_t gfp)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700931{
Pekka Enbergb28a02d2006-01-08 01:00:37 -0800932 int memsize = sizeof(void *) * entries + sizeof(struct array_cache);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700933 struct array_cache *nc = NULL;
934
Pekka Enberg83b519e2009-06-10 19:40:04 +0300935 nc = kmalloc_node(memsize, gfp, node);
Catalin Marinasd5cff632009-06-11 13:22:40 +0100936 /*
937 * The array_cache structures contain pointers to free object.
Lucas De Marchi25985ed2011-03-30 22:57:33 -0300938 * However, when such objects are allocated or transferred to another
Catalin Marinasd5cff632009-06-11 13:22:40 +0100939 * cache the pointers are not cleared and they could be counted as
940 * valid references during a kmemleak scan. Therefore, kmemleak must
941 * not scan such objects.
942 */
943 kmemleak_no_scan(nc);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700944 if (nc) {
945 nc->avail = 0;
946 nc->limit = entries;
947 nc->batchcount = batchcount;
948 nc->touched = 0;
Christoph Lametere498be72005-09-09 13:03:32 -0700949 spin_lock_init(&nc->lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700950 }
951 return nc;
952}
953
Christoph Lameter3ded1752006-03-25 03:06:44 -0800954/*
955 * Transfer objects in one arraycache to another.
956 * Locking must be handled by the caller.
957 *
958 * Return the number of entries transferred.
959 */
960static int transfer_objects(struct array_cache *to,
961 struct array_cache *from, unsigned int max)
962{
963 /* Figure out how many entries to transfer */
Hagen Paul Pfeifer732eacc2010-10-26 14:22:23 -0700964 int nr = min3(from->avail, max, to->limit - to->avail);
Christoph Lameter3ded1752006-03-25 03:06:44 -0800965
966 if (!nr)
967 return 0;
968
969 memcpy(to->entry + to->avail, from->entry + from->avail -nr,
970 sizeof(void *) *nr);
971
972 from->avail -= nr;
973 to->avail += nr;
Christoph Lameter3ded1752006-03-25 03:06:44 -0800974 return nr;
975}
976
Christoph Lameter765c4502006-09-27 01:50:08 -0700977#ifndef CONFIG_NUMA
978
979#define drain_alien_cache(cachep, alien) do { } while (0)
980#define reap_alien(cachep, l3) do { } while (0)
981
Pekka Enberg83b519e2009-06-10 19:40:04 +0300982static inline struct array_cache **alloc_alien_cache(int node, int limit, gfp_t gfp)
Christoph Lameter765c4502006-09-27 01:50:08 -0700983{
984 return (struct array_cache **)BAD_ALIEN_MAGIC;
985}
986
987static inline void free_alien_cache(struct array_cache **ac_ptr)
988{
989}
990
991static inline int cache_free_alien(struct kmem_cache *cachep, void *objp)
992{
993 return 0;
994}
995
996static inline void *alternate_node_alloc(struct kmem_cache *cachep,
997 gfp_t flags)
998{
999 return NULL;
1000}
1001
Christoph Hellwig8b98c162006-12-06 20:32:30 -08001002static inline void *____cache_alloc_node(struct kmem_cache *cachep,
Christoph Lameter765c4502006-09-27 01:50:08 -07001003 gfp_t flags, int nodeid)
1004{
1005 return NULL;
1006}
1007
1008#else /* CONFIG_NUMA */
1009
Christoph Hellwig8b98c162006-12-06 20:32:30 -08001010static void *____cache_alloc_node(struct kmem_cache *, gfp_t, int);
Paul Jacksonc61afb12006-03-24 03:16:08 -08001011static void *alternate_node_alloc(struct kmem_cache *, gfp_t);
Christoph Lameterdc85da12006-01-18 17:42:36 -08001012
Pekka Enberg83b519e2009-06-10 19:40:04 +03001013static struct array_cache **alloc_alien_cache(int node, int limit, gfp_t gfp)
Christoph Lametere498be72005-09-09 13:03:32 -07001014{
1015 struct array_cache **ac_ptr;
Christoph Lameter8ef82862007-02-20 13:57:52 -08001016 int memsize = sizeof(void *) * nr_node_ids;
Christoph Lametere498be72005-09-09 13:03:32 -07001017 int i;
1018
1019 if (limit > 1)
1020 limit = 12;
Haicheng Lif3186a92010-01-06 15:25:23 +08001021 ac_ptr = kzalloc_node(memsize, gfp, node);
Christoph Lametere498be72005-09-09 13:03:32 -07001022 if (ac_ptr) {
1023 for_each_node(i) {
Haicheng Lif3186a92010-01-06 15:25:23 +08001024 if (i == node || !node_online(i))
Christoph Lametere498be72005-09-09 13:03:32 -07001025 continue;
Pekka Enberg83b519e2009-06-10 19:40:04 +03001026 ac_ptr[i] = alloc_arraycache(node, limit, 0xbaadf00d, gfp);
Christoph Lametere498be72005-09-09 13:03:32 -07001027 if (!ac_ptr[i]) {
Akinobu Mitacc550de2007-11-14 16:58:35 -08001028 for (i--; i >= 0; i--)
Christoph Lametere498be72005-09-09 13:03:32 -07001029 kfree(ac_ptr[i]);
1030 kfree(ac_ptr);
1031 return NULL;
1032 }
1033 }
1034 }
1035 return ac_ptr;
1036}
1037
Pekka Enberg5295a742006-02-01 03:05:48 -08001038static void free_alien_cache(struct array_cache **ac_ptr)
Christoph Lametere498be72005-09-09 13:03:32 -07001039{
1040 int i;
1041
1042 if (!ac_ptr)
1043 return;
Christoph Lametere498be72005-09-09 13:03:32 -07001044 for_each_node(i)
Pekka Enbergb28a02d2006-01-08 01:00:37 -08001045 kfree(ac_ptr[i]);
Christoph Lametere498be72005-09-09 13:03:32 -07001046 kfree(ac_ptr);
1047}
1048
Pekka Enberg343e0d72006-02-01 03:05:50 -08001049static void __drain_alien_cache(struct kmem_cache *cachep,
Pekka Enberg5295a742006-02-01 03:05:48 -08001050 struct array_cache *ac, int node)
Christoph Lametere498be72005-09-09 13:03:32 -07001051{
1052 struct kmem_list3 *rl3 = cachep->nodelists[node];
1053
1054 if (ac->avail) {
1055 spin_lock(&rl3->list_lock);
Christoph Lametere00946f2006-03-25 03:06:45 -08001056 /*
1057 * Stuff objects into the remote nodes shared array first.
1058 * That way we could avoid the overhead of putting the objects
1059 * into the free lists and getting them back later.
1060 */
shin, jacob693f7d32006-04-28 10:54:37 -05001061 if (rl3->shared)
1062 transfer_objects(rl3->shared, ac, ac->limit);
Christoph Lametere00946f2006-03-25 03:06:45 -08001063
Christoph Lameterff694162005-09-22 21:44:02 -07001064 free_block(cachep, ac->entry, ac->avail, node);
Christoph Lametere498be72005-09-09 13:03:32 -07001065 ac->avail = 0;
1066 spin_unlock(&rl3->list_lock);
1067 }
1068}
1069
Christoph Lameter8fce4d82006-03-09 17:33:54 -08001070/*
1071 * Called from cache_reap() to regularly drain alien caches round robin.
1072 */
1073static void reap_alien(struct kmem_cache *cachep, struct kmem_list3 *l3)
1074{
Christoph Lameter909ea962010-12-08 16:22:55 +01001075 int node = __this_cpu_read(slab_reap_node);
Christoph Lameter8fce4d82006-03-09 17:33:54 -08001076
1077 if (l3->alien) {
1078 struct array_cache *ac = l3->alien[node];
Christoph Lametere00946f2006-03-25 03:06:45 -08001079
1080 if (ac && ac->avail && spin_trylock_irq(&ac->lock)) {
Christoph Lameter8fce4d82006-03-09 17:33:54 -08001081 __drain_alien_cache(cachep, ac, node);
1082 spin_unlock_irq(&ac->lock);
1083 }
1084 }
1085}
1086
Andrew Mortona737b3e2006-03-22 00:08:11 -08001087static void drain_alien_cache(struct kmem_cache *cachep,
1088 struct array_cache **alien)
Christoph Lametere498be72005-09-09 13:03:32 -07001089{
Pekka Enbergb28a02d2006-01-08 01:00:37 -08001090 int i = 0;
Christoph Lametere498be72005-09-09 13:03:32 -07001091 struct array_cache *ac;
1092 unsigned long flags;
1093
1094 for_each_online_node(i) {
Ravikiran G Thirumalai4484ebf2006-02-04 23:27:59 -08001095 ac = alien[i];
Christoph Lametere498be72005-09-09 13:03:32 -07001096 if (ac) {
1097 spin_lock_irqsave(&ac->lock, flags);
1098 __drain_alien_cache(cachep, ac, i);
1099 spin_unlock_irqrestore(&ac->lock, flags);
1100 }
1101 }
1102}
Pekka Enberg729bd0b2006-06-23 02:03:05 -07001103
Ingo Molnar873623d2006-07-13 14:44:38 +02001104static inline int cache_free_alien(struct kmem_cache *cachep, void *objp)
Pekka Enberg729bd0b2006-06-23 02:03:05 -07001105{
1106 struct slab *slabp = virt_to_slab(objp);
1107 int nodeid = slabp->nodeid;
1108 struct kmem_list3 *l3;
1109 struct array_cache *alien = NULL;
Pekka Enberg1ca4cb22006-10-06 00:43:52 -07001110 int node;
1111
Lee Schermerhorn7d6e6d02010-05-26 14:45:03 -07001112 node = numa_mem_id();
Pekka Enberg729bd0b2006-06-23 02:03:05 -07001113
1114 /*
1115 * Make sure we are not freeing a object from another node to the array
1116 * cache on this cpu.
1117 */
Siddha, Suresh B62918a02007-05-02 19:27:18 +02001118 if (likely(slabp->nodeid == node))
Pekka Enberg729bd0b2006-06-23 02:03:05 -07001119 return 0;
1120
Pekka Enberg1ca4cb22006-10-06 00:43:52 -07001121 l3 = cachep->nodelists[node];
Pekka Enberg729bd0b2006-06-23 02:03:05 -07001122 STATS_INC_NODEFREES(cachep);
1123 if (l3->alien && l3->alien[nodeid]) {
1124 alien = l3->alien[nodeid];
Ingo Molnar873623d2006-07-13 14:44:38 +02001125 spin_lock(&alien->lock);
Pekka Enberg729bd0b2006-06-23 02:03:05 -07001126 if (unlikely(alien->avail == alien->limit)) {
1127 STATS_INC_ACOVERFLOW(cachep);
1128 __drain_alien_cache(cachep, alien, nodeid);
1129 }
1130 alien->entry[alien->avail++] = objp;
1131 spin_unlock(&alien->lock);
1132 } else {
1133 spin_lock(&(cachep->nodelists[nodeid])->list_lock);
1134 free_block(cachep, &objp, 1, nodeid);
1135 spin_unlock(&(cachep->nodelists[nodeid])->list_lock);
1136 }
1137 return 1;
1138}
Christoph Lametere498be72005-09-09 13:03:32 -07001139#endif
1140
David Rientjes8f9f8d92010-03-27 19:40:47 -07001141/*
1142 * Allocates and initializes nodelists for a node on each slab cache, used for
1143 * either memory or cpu hotplug. If memory is being hot-added, the kmem_list3
1144 * will be allocated off-node since memory is not yet online for the new node.
1145 * When hotplugging memory or a cpu, existing nodelists are not replaced if
1146 * already in use.
1147 *
1148 * Must hold cache_chain_mutex.
1149 */
1150static int init_cache_nodelists_node(int node)
1151{
1152 struct kmem_cache *cachep;
1153 struct kmem_list3 *l3;
1154 const int memsize = sizeof(struct kmem_list3);
1155
1156 list_for_each_entry(cachep, &cache_chain, next) {
1157 /*
1158 * Set up the size64 kmemlist for cpu before we can
1159 * begin anything. Make sure some other cpu on this
1160 * node has not already allocated this
1161 */
1162 if (!cachep->nodelists[node]) {
1163 l3 = kmalloc_node(memsize, GFP_KERNEL, node);
1164 if (!l3)
1165 return -ENOMEM;
1166 kmem_list3_init(l3);
1167 l3->next_reap = jiffies + REAPTIMEOUT_LIST3 +
1168 ((unsigned long)cachep) % REAPTIMEOUT_LIST3;
1169
1170 /*
1171 * The l3s don't come and go as CPUs come and
1172 * go. cache_chain_mutex is sufficient
1173 * protection here.
1174 */
1175 cachep->nodelists[node] = l3;
1176 }
1177
1178 spin_lock_irq(&cachep->nodelists[node]->list_lock);
1179 cachep->nodelists[node]->free_limit =
1180 (1 + nr_cpus_node(node)) *
1181 cachep->batchcount + cachep->num;
1182 spin_unlock_irq(&cachep->nodelists[node]->list_lock);
1183 }
1184 return 0;
1185}
1186
Akinobu Mitafbf1e472007-10-18 03:05:09 -07001187static void __cpuinit cpuup_canceled(long cpu)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001188{
Akinobu Mitafbf1e472007-10-18 03:05:09 -07001189 struct kmem_cache *cachep;
1190 struct kmem_list3 *l3 = NULL;
Lee Schermerhorn7d6e6d02010-05-26 14:45:03 -07001191 int node = cpu_to_mem(cpu);
Rusty Russella70f7302009-03-13 14:49:46 +10301192 const struct cpumask *mask = cpumask_of_node(node);
Akinobu Mitafbf1e472007-10-18 03:05:09 -07001193
1194 list_for_each_entry(cachep, &cache_chain, next) {
1195 struct array_cache *nc;
1196 struct array_cache *shared;
1197 struct array_cache **alien;
Akinobu Mitafbf1e472007-10-18 03:05:09 -07001198
Akinobu Mitafbf1e472007-10-18 03:05:09 -07001199 /* cpu is dead; no one can alloc from it. */
1200 nc = cachep->array[cpu];
1201 cachep->array[cpu] = NULL;
1202 l3 = cachep->nodelists[node];
1203
1204 if (!l3)
1205 goto free_array_cache;
1206
1207 spin_lock_irq(&l3->list_lock);
1208
1209 /* Free limit for this kmem_list3 */
1210 l3->free_limit -= cachep->batchcount;
1211 if (nc)
1212 free_block(cachep, nc->entry, nc->avail, node);
1213
Rusty Russell58463c12009-12-17 11:43:12 -06001214 if (!cpumask_empty(mask)) {
Akinobu Mitafbf1e472007-10-18 03:05:09 -07001215 spin_unlock_irq(&l3->list_lock);
1216 goto free_array_cache;
1217 }
1218
1219 shared = l3->shared;
1220 if (shared) {
1221 free_block(cachep, shared->entry,
1222 shared->avail, node);
1223 l3->shared = NULL;
1224 }
1225
1226 alien = l3->alien;
1227 l3->alien = NULL;
1228
1229 spin_unlock_irq(&l3->list_lock);
1230
1231 kfree(shared);
1232 if (alien) {
1233 drain_alien_cache(cachep, alien);
1234 free_alien_cache(alien);
1235 }
1236free_array_cache:
1237 kfree(nc);
1238 }
1239 /*
1240 * In the previous loop, all the objects were freed to
1241 * the respective cache's slabs, now we can go ahead and
1242 * shrink each nodelist to its limit.
1243 */
1244 list_for_each_entry(cachep, &cache_chain, next) {
1245 l3 = cachep->nodelists[node];
1246 if (!l3)
1247 continue;
1248 drain_freelist(cachep, l3, l3->free_objects);
1249 }
1250}
1251
1252static int __cpuinit cpuup_prepare(long cpu)
1253{
Pekka Enberg343e0d72006-02-01 03:05:50 -08001254 struct kmem_cache *cachep;
Christoph Lametere498be72005-09-09 13:03:32 -07001255 struct kmem_list3 *l3 = NULL;
Lee Schermerhorn7d6e6d02010-05-26 14:45:03 -07001256 int node = cpu_to_mem(cpu);
David Rientjes8f9f8d92010-03-27 19:40:47 -07001257 int err;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001258
Akinobu Mitafbf1e472007-10-18 03:05:09 -07001259 /*
1260 * We need to do this right in the beginning since
1261 * alloc_arraycache's are going to use this list.
1262 * kmalloc_node allows us to add the slab to the right
1263 * kmem_list3 and not this cpu's kmem_list3
1264 */
David Rientjes8f9f8d92010-03-27 19:40:47 -07001265 err = init_cache_nodelists_node(node);
1266 if (err < 0)
1267 goto bad;
Akinobu Mitafbf1e472007-10-18 03:05:09 -07001268
1269 /*
1270 * Now we can go ahead with allocating the shared arrays and
1271 * array caches
1272 */
1273 list_for_each_entry(cachep, &cache_chain, next) {
1274 struct array_cache *nc;
1275 struct array_cache *shared = NULL;
1276 struct array_cache **alien = NULL;
1277
1278 nc = alloc_arraycache(node, cachep->limit,
Pekka Enberg83b519e2009-06-10 19:40:04 +03001279 cachep->batchcount, GFP_KERNEL);
Akinobu Mitafbf1e472007-10-18 03:05:09 -07001280 if (!nc)
1281 goto bad;
1282 if (cachep->shared) {
1283 shared = alloc_arraycache(node,
1284 cachep->shared * cachep->batchcount,
Pekka Enberg83b519e2009-06-10 19:40:04 +03001285 0xbaadf00d, GFP_KERNEL);
Akinobu Mita12d00f62007-10-18 03:05:11 -07001286 if (!shared) {
1287 kfree(nc);
Akinobu Mitafbf1e472007-10-18 03:05:09 -07001288 goto bad;
Akinobu Mita12d00f62007-10-18 03:05:11 -07001289 }
Akinobu Mitafbf1e472007-10-18 03:05:09 -07001290 }
1291 if (use_alien_caches) {
Pekka Enberg83b519e2009-06-10 19:40:04 +03001292 alien = alloc_alien_cache(node, cachep->limit, GFP_KERNEL);
Akinobu Mita12d00f62007-10-18 03:05:11 -07001293 if (!alien) {
1294 kfree(shared);
1295 kfree(nc);
Akinobu Mitafbf1e472007-10-18 03:05:09 -07001296 goto bad;
Akinobu Mita12d00f62007-10-18 03:05:11 -07001297 }
Akinobu Mitafbf1e472007-10-18 03:05:09 -07001298 }
1299 cachep->array[cpu] = nc;
1300 l3 = cachep->nodelists[node];
1301 BUG_ON(!l3);
1302
1303 spin_lock_irq(&l3->list_lock);
1304 if (!l3->shared) {
1305 /*
1306 * We are serialised from CPU_DEAD or
1307 * CPU_UP_CANCELLED by the cpucontrol lock
1308 */
1309 l3->shared = shared;
1310 shared = NULL;
1311 }
1312#ifdef CONFIG_NUMA
1313 if (!l3->alien) {
1314 l3->alien = alien;
1315 alien = NULL;
1316 }
1317#endif
1318 spin_unlock_irq(&l3->list_lock);
1319 kfree(shared);
1320 free_alien_cache(alien);
Peter Zijlstra83835b32011-07-22 15:26:05 +02001321 if (cachep->flags & SLAB_DEBUG_OBJECTS)
1322 slab_set_debugobj_lock_classes_node(cachep, node);
Akinobu Mitafbf1e472007-10-18 03:05:09 -07001323 }
Pekka Enbergce79ddc2009-11-23 22:01:15 +02001324 init_node_lock_keys(node);
1325
Akinobu Mitafbf1e472007-10-18 03:05:09 -07001326 return 0;
1327bad:
Akinobu Mita12d00f62007-10-18 03:05:11 -07001328 cpuup_canceled(cpu);
Akinobu Mitafbf1e472007-10-18 03:05:09 -07001329 return -ENOMEM;
1330}
1331
1332static int __cpuinit cpuup_callback(struct notifier_block *nfb,
1333 unsigned long action, void *hcpu)
1334{
1335 long cpu = (long)hcpu;
1336 int err = 0;
1337
Linus Torvalds1da177e2005-04-16 15:20:36 -07001338 switch (action) {
Heiko Carstens38c3bd92007-05-09 02:34:05 -07001339 case CPU_UP_PREPARE:
Rafael J. Wysocki8bb78442007-05-09 02:35:10 -07001340 case CPU_UP_PREPARE_FROZEN:
Gautham R Shenoy95402b32008-01-25 21:08:02 +01001341 mutex_lock(&cache_chain_mutex);
Akinobu Mitafbf1e472007-10-18 03:05:09 -07001342 err = cpuup_prepare(cpu);
Gautham R Shenoy95402b32008-01-25 21:08:02 +01001343 mutex_unlock(&cache_chain_mutex);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001344 break;
1345 case CPU_ONLINE:
Rafael J. Wysocki8bb78442007-05-09 02:35:10 -07001346 case CPU_ONLINE_FROZEN:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001347 start_cpu_timer(cpu);
1348 break;
1349#ifdef CONFIG_HOTPLUG_CPU
Christoph Lameter5830c592007-05-09 02:34:22 -07001350 case CPU_DOWN_PREPARE:
Rafael J. Wysocki8bb78442007-05-09 02:35:10 -07001351 case CPU_DOWN_PREPARE_FROZEN:
Christoph Lameter5830c592007-05-09 02:34:22 -07001352 /*
1353 * Shutdown cache reaper. Note that the cache_chain_mutex is
1354 * held so that if cache_reap() is invoked it cannot do
1355 * anything expensive but will only modify reap_work
1356 * and reschedule the timer.
1357 */
Tejun Heoafe2c512010-12-14 16:21:17 +01001358 cancel_delayed_work_sync(&per_cpu(slab_reap_work, cpu));
Christoph Lameter5830c592007-05-09 02:34:22 -07001359 /* Now the cache_reaper is guaranteed to be not running. */
Tejun Heo1871e522009-10-29 22:34:13 +09001360 per_cpu(slab_reap_work, cpu).work.func = NULL;
Christoph Lameter5830c592007-05-09 02:34:22 -07001361 break;
1362 case CPU_DOWN_FAILED:
Rafael J. Wysocki8bb78442007-05-09 02:35:10 -07001363 case CPU_DOWN_FAILED_FROZEN:
Christoph Lameter5830c592007-05-09 02:34:22 -07001364 start_cpu_timer(cpu);
1365 break;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001366 case CPU_DEAD:
Rafael J. Wysocki8bb78442007-05-09 02:35:10 -07001367 case CPU_DEAD_FROZEN:
Ravikiran G Thirumalai4484ebf2006-02-04 23:27:59 -08001368 /*
1369 * Even if all the cpus of a node are down, we don't free the
1370 * kmem_list3 of any cache. This to avoid a race between
1371 * cpu_down, and a kmalloc allocation from another cpu for
1372 * memory from the node of the cpu going down. The list3
1373 * structure is usually allocated from kmem_cache_create() and
1374 * gets destroyed at kmem_cache_destroy().
1375 */
Simon Arlott183ff222007-10-20 01:27:18 +02001376 /* fall through */
Ravikiran G Thirumalai8f5be202006-12-06 20:32:14 -08001377#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07001378 case CPU_UP_CANCELED:
Rafael J. Wysocki8bb78442007-05-09 02:35:10 -07001379 case CPU_UP_CANCELED_FROZEN:
Gautham R Shenoy95402b32008-01-25 21:08:02 +01001380 mutex_lock(&cache_chain_mutex);
Akinobu Mitafbf1e472007-10-18 03:05:09 -07001381 cpuup_canceled(cpu);
Ingo Molnarfc0abb12006-01-18 17:42:33 -08001382 mutex_unlock(&cache_chain_mutex);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001383 break;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001384 }
Akinobu Mitaeac40682010-05-26 14:43:32 -07001385 return notifier_from_errno(err);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001386}
1387
Chandra Seetharaman74b85f32006-06-27 02:54:09 -07001388static struct notifier_block __cpuinitdata cpucache_notifier = {
1389 &cpuup_callback, NULL, 0
1390};
Linus Torvalds1da177e2005-04-16 15:20:36 -07001391
David Rientjes8f9f8d92010-03-27 19:40:47 -07001392#if defined(CONFIG_NUMA) && defined(CONFIG_MEMORY_HOTPLUG)
1393/*
1394 * Drains freelist for a node on each slab cache, used for memory hot-remove.
1395 * Returns -EBUSY if all objects cannot be drained so that the node is not
1396 * removed.
1397 *
1398 * Must hold cache_chain_mutex.
1399 */
1400static int __meminit drain_cache_nodelists_node(int node)
1401{
1402 struct kmem_cache *cachep;
1403 int ret = 0;
1404
1405 list_for_each_entry(cachep, &cache_chain, next) {
1406 struct kmem_list3 *l3;
1407
1408 l3 = cachep->nodelists[node];
1409 if (!l3)
1410 continue;
1411
1412 drain_freelist(cachep, l3, l3->free_objects);
1413
1414 if (!list_empty(&l3->slabs_full) ||
1415 !list_empty(&l3->slabs_partial)) {
1416 ret = -EBUSY;
1417 break;
1418 }
1419 }
1420 return ret;
1421}
1422
1423static int __meminit slab_memory_callback(struct notifier_block *self,
1424 unsigned long action, void *arg)
1425{
1426 struct memory_notify *mnb = arg;
1427 int ret = 0;
1428 int nid;
1429
1430 nid = mnb->status_change_nid;
1431 if (nid < 0)
1432 goto out;
1433
1434 switch (action) {
1435 case MEM_GOING_ONLINE:
1436 mutex_lock(&cache_chain_mutex);
1437 ret = init_cache_nodelists_node(nid);
1438 mutex_unlock(&cache_chain_mutex);
1439 break;
1440 case MEM_GOING_OFFLINE:
1441 mutex_lock(&cache_chain_mutex);
1442 ret = drain_cache_nodelists_node(nid);
1443 mutex_unlock(&cache_chain_mutex);
1444 break;
1445 case MEM_ONLINE:
1446 case MEM_OFFLINE:
1447 case MEM_CANCEL_ONLINE:
1448 case MEM_CANCEL_OFFLINE:
1449 break;
1450 }
1451out:
Prarit Bhargava5fda1bd2011-03-22 16:30:49 -07001452 return notifier_from_errno(ret);
David Rientjes8f9f8d92010-03-27 19:40:47 -07001453}
1454#endif /* CONFIG_NUMA && CONFIG_MEMORY_HOTPLUG */
1455
Christoph Lametere498be72005-09-09 13:03:32 -07001456/*
1457 * swap the static kmem_list3 with kmalloced memory
1458 */
David Rientjes8f9f8d92010-03-27 19:40:47 -07001459static void __init init_list(struct kmem_cache *cachep, struct kmem_list3 *list,
1460 int nodeid)
Christoph Lametere498be72005-09-09 13:03:32 -07001461{
1462 struct kmem_list3 *ptr;
1463
Pekka Enberg83b519e2009-06-10 19:40:04 +03001464 ptr = kmalloc_node(sizeof(struct kmem_list3), GFP_NOWAIT, nodeid);
Christoph Lametere498be72005-09-09 13:03:32 -07001465 BUG_ON(!ptr);
1466
Christoph Lametere498be72005-09-09 13:03:32 -07001467 memcpy(ptr, list, sizeof(struct kmem_list3));
Ingo Molnar2b2d5492006-07-03 00:25:28 -07001468 /*
1469 * Do not assume that spinlocks can be initialized via memcpy:
1470 */
1471 spin_lock_init(&ptr->list_lock);
1472
Christoph Lametere498be72005-09-09 13:03:32 -07001473 MAKE_ALL_LISTS(cachep, ptr, nodeid);
1474 cachep->nodelists[nodeid] = ptr;
Christoph Lametere498be72005-09-09 13:03:32 -07001475}
1476
Andrew Mortona737b3e2006-03-22 00:08:11 -08001477/*
Pekka Enberg556a1692008-01-25 08:20:51 +02001478 * For setting up all the kmem_list3s for cache whose buffer_size is same as
1479 * size of kmem_list3.
1480 */
1481static void __init set_up_list3s(struct kmem_cache *cachep, int index)
1482{
1483 int node;
1484
1485 for_each_online_node(node) {
1486 cachep->nodelists[node] = &initkmem_list3[index + node];
1487 cachep->nodelists[node]->next_reap = jiffies +
1488 REAPTIMEOUT_LIST3 +
1489 ((unsigned long)cachep) % REAPTIMEOUT_LIST3;
1490 }
1491}
1492
1493/*
Andrew Mortona737b3e2006-03-22 00:08:11 -08001494 * Initialisation. Called after the page allocator have been initialised and
1495 * before smp_init().
Linus Torvalds1da177e2005-04-16 15:20:36 -07001496 */
1497void __init kmem_cache_init(void)
1498{
1499 size_t left_over;
1500 struct cache_sizes *sizes;
1501 struct cache_names *names;
Christoph Lametere498be72005-09-09 13:03:32 -07001502 int i;
Jack Steiner07ed76b2006-03-07 21:55:46 -08001503 int order;
Pekka Enberg1ca4cb22006-10-06 00:43:52 -07001504 int node;
Christoph Lametere498be72005-09-09 13:03:32 -07001505
Mel Gormanb6e68bc2009-06-16 15:32:16 -07001506 if (num_possible_nodes() == 1)
Siddha, Suresh B62918a02007-05-02 19:27:18 +02001507 use_alien_caches = 0;
1508
Christoph Lametere498be72005-09-09 13:03:32 -07001509 for (i = 0; i < NUM_INIT_LISTS; i++) {
1510 kmem_list3_init(&initkmem_list3[i]);
1511 if (i < MAX_NUMNODES)
1512 cache_cache.nodelists[i] = NULL;
1513 }
Pekka Enberg556a1692008-01-25 08:20:51 +02001514 set_up_list3s(&cache_cache, CACHE_CACHE);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001515
1516 /*
1517 * Fragmentation resistance on low memory - only use bigger
David Rientjes3df1ccc2011-10-18 22:09:28 -07001518 * page orders on machines with more than 32MB of memory if
1519 * not overridden on the command line.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001520 */
David Rientjes3df1ccc2011-10-18 22:09:28 -07001521 if (!slab_max_order_set && totalram_pages > (32 << 20) >> PAGE_SHIFT)
David Rientjes543585c2011-10-18 22:09:24 -07001522 slab_max_order = SLAB_MAX_ORDER_HI;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001523
Linus Torvalds1da177e2005-04-16 15:20:36 -07001524 /* Bootstrap is tricky, because several objects are allocated
1525 * from caches that do not exist yet:
Andrew Mortona737b3e2006-03-22 00:08:11 -08001526 * 1) initialize the cache_cache cache: it contains the struct
1527 * kmem_cache structures of all caches, except cache_cache itself:
1528 * cache_cache is statically allocated.
Christoph Lametere498be72005-09-09 13:03:32 -07001529 * Initially an __init data area is used for the head array and the
1530 * kmem_list3 structures, it's replaced with a kmalloc allocated
1531 * array at the end of the bootstrap.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001532 * 2) Create the first kmalloc cache.
Pekka Enberg343e0d72006-02-01 03:05:50 -08001533 * The struct kmem_cache for the new cache is allocated normally.
Christoph Lametere498be72005-09-09 13:03:32 -07001534 * An __init data area is used for the head array.
1535 * 3) Create the remaining kmalloc caches, with minimally sized
1536 * head arrays.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001537 * 4) Replace the __init data head arrays for cache_cache and the first
1538 * kmalloc cache with kmalloc allocated arrays.
Christoph Lametere498be72005-09-09 13:03:32 -07001539 * 5) Replace the __init data for kmem_list3 for cache_cache and
1540 * the other cache's with kmalloc allocated memory.
1541 * 6) Resize the head arrays of the kmalloc caches to their final sizes.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001542 */
1543
Lee Schermerhorn7d6e6d02010-05-26 14:45:03 -07001544 node = numa_mem_id();
Pekka Enberg1ca4cb22006-10-06 00:43:52 -07001545
Linus Torvalds1da177e2005-04-16 15:20:36 -07001546 /* 1) create the cache_cache */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001547 INIT_LIST_HEAD(&cache_chain);
1548 list_add(&cache_cache.next, &cache_chain);
1549 cache_cache.colour_off = cache_line_size();
1550 cache_cache.array[smp_processor_id()] = &initarray_cache.cache;
Daniel Yeisleyec1f5ee2008-03-25 23:59:08 +02001551 cache_cache.nodelists[node] = &initkmem_list3[CACHE_CACHE + node];
Linus Torvalds1da177e2005-04-16 15:20:36 -07001552
Eric Dumazet8da34302007-05-06 14:49:29 -07001553 /*
Eric Dumazetb56efcf2011-07-20 19:04:23 +02001554 * struct kmem_cache size depends on nr_node_ids & nr_cpu_ids
Eric Dumazet8da34302007-05-06 14:49:29 -07001555 */
Eric Dumazetb56efcf2011-07-20 19:04:23 +02001556 cache_cache.buffer_size = offsetof(struct kmem_cache, array[nr_cpu_ids]) +
1557 nr_node_ids * sizeof(struct kmem_list3 *);
Eric Dumazet8da34302007-05-06 14:49:29 -07001558#if DEBUG
1559 cache_cache.obj_size = cache_cache.buffer_size;
1560#endif
Andrew Mortona737b3e2006-03-22 00:08:11 -08001561 cache_cache.buffer_size = ALIGN(cache_cache.buffer_size,
1562 cache_line_size());
Eric Dumazet6a2d7a92006-12-13 00:34:27 -08001563 cache_cache.reciprocal_buffer_size =
1564 reciprocal_value(cache_cache.buffer_size);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001565
Jack Steiner07ed76b2006-03-07 21:55:46 -08001566 for (order = 0; order < MAX_ORDER; order++) {
1567 cache_estimate(order, cache_cache.buffer_size,
1568 cache_line_size(), 0, &left_over, &cache_cache.num);
1569 if (cache_cache.num)
1570 break;
1571 }
Eric Sesterhenn40094fa2006-04-02 13:49:25 +02001572 BUG_ON(!cache_cache.num);
Jack Steiner07ed76b2006-03-07 21:55:46 -08001573 cache_cache.gfporder = order;
Pekka Enbergb28a02d2006-01-08 01:00:37 -08001574 cache_cache.colour = left_over / cache_cache.colour_off;
Pekka Enbergb28a02d2006-01-08 01:00:37 -08001575 cache_cache.slab_size = ALIGN(cache_cache.num * sizeof(kmem_bufctl_t) +
1576 sizeof(struct slab), cache_line_size());
Linus Torvalds1da177e2005-04-16 15:20:36 -07001577
1578 /* 2+3) create the kmalloc caches */
1579 sizes = malloc_sizes;
1580 names = cache_names;
1581
Andrew Mortona737b3e2006-03-22 00:08:11 -08001582 /*
1583 * Initialize the caches that provide memory for the array cache and the
1584 * kmem_list3 structures first. Without this, further allocations will
1585 * bug.
Christoph Lametere498be72005-09-09 13:03:32 -07001586 */
1587
1588 sizes[INDEX_AC].cs_cachep = kmem_cache_create(names[INDEX_AC].name,
Andrew Mortona737b3e2006-03-22 00:08:11 -08001589 sizes[INDEX_AC].cs_size,
1590 ARCH_KMALLOC_MINALIGN,
1591 ARCH_KMALLOC_FLAGS|SLAB_PANIC,
Paul Mundt20c2df82007-07-20 10:11:58 +09001592 NULL);
Christoph Lametere498be72005-09-09 13:03:32 -07001593
Andrew Mortona737b3e2006-03-22 00:08:11 -08001594 if (INDEX_AC != INDEX_L3) {
Christoph Lametere498be72005-09-09 13:03:32 -07001595 sizes[INDEX_L3].cs_cachep =
Andrew Mortona737b3e2006-03-22 00:08:11 -08001596 kmem_cache_create(names[INDEX_L3].name,
1597 sizes[INDEX_L3].cs_size,
1598 ARCH_KMALLOC_MINALIGN,
1599 ARCH_KMALLOC_FLAGS|SLAB_PANIC,
Paul Mundt20c2df82007-07-20 10:11:58 +09001600 NULL);
Andrew Mortona737b3e2006-03-22 00:08:11 -08001601 }
Christoph Lametere498be72005-09-09 13:03:32 -07001602
Ingo Molnare0a42722006-06-23 02:03:46 -07001603 slab_early_init = 0;
1604
Linus Torvalds1da177e2005-04-16 15:20:36 -07001605 while (sizes->cs_size != ULONG_MAX) {
Christoph Lametere498be72005-09-09 13:03:32 -07001606 /*
1607 * For performance, all the general caches are L1 aligned.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001608 * This should be particularly beneficial on SMP boxes, as it
1609 * eliminates "false sharing".
1610 * Note for systems short on memory removing the alignment will
Christoph Lametere498be72005-09-09 13:03:32 -07001611 * allow tighter packing of the smaller caches.
1612 */
Andrew Mortona737b3e2006-03-22 00:08:11 -08001613 if (!sizes->cs_cachep) {
Christoph Lametere498be72005-09-09 13:03:32 -07001614 sizes->cs_cachep = kmem_cache_create(names->name,
Andrew Mortona737b3e2006-03-22 00:08:11 -08001615 sizes->cs_size,
1616 ARCH_KMALLOC_MINALIGN,
1617 ARCH_KMALLOC_FLAGS|SLAB_PANIC,
Paul Mundt20c2df82007-07-20 10:11:58 +09001618 NULL);
Andrew Mortona737b3e2006-03-22 00:08:11 -08001619 }
Christoph Lameter4b51d662007-02-10 01:43:10 -08001620#ifdef CONFIG_ZONE_DMA
1621 sizes->cs_dmacachep = kmem_cache_create(
1622 names->name_dma,
Andrew Mortona737b3e2006-03-22 00:08:11 -08001623 sizes->cs_size,
1624 ARCH_KMALLOC_MINALIGN,
1625 ARCH_KMALLOC_FLAGS|SLAB_CACHE_DMA|
1626 SLAB_PANIC,
Paul Mundt20c2df82007-07-20 10:11:58 +09001627 NULL);
Christoph Lameter4b51d662007-02-10 01:43:10 -08001628#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07001629 sizes++;
1630 names++;
1631 }
1632 /* 4) Replace the bootstrap head arrays */
1633 {
Ingo Molnar2b2d5492006-07-03 00:25:28 -07001634 struct array_cache *ptr;
Christoph Lametere498be72005-09-09 13:03:32 -07001635
Pekka Enberg83b519e2009-06-10 19:40:04 +03001636 ptr = kmalloc(sizeof(struct arraycache_init), GFP_NOWAIT);
Christoph Lametere498be72005-09-09 13:03:32 -07001637
Pekka Enberg9a2dba4b2006-02-01 03:05:49 -08001638 BUG_ON(cpu_cache_get(&cache_cache) != &initarray_cache.cache);
1639 memcpy(ptr, cpu_cache_get(&cache_cache),
Pekka Enbergb28a02d2006-01-08 01:00:37 -08001640 sizeof(struct arraycache_init));
Ingo Molnar2b2d5492006-07-03 00:25:28 -07001641 /*
1642 * Do not assume that spinlocks can be initialized via memcpy:
1643 */
1644 spin_lock_init(&ptr->lock);
1645
Linus Torvalds1da177e2005-04-16 15:20:36 -07001646 cache_cache.array[smp_processor_id()] = ptr;
Christoph Lametere498be72005-09-09 13:03:32 -07001647
Pekka Enberg83b519e2009-06-10 19:40:04 +03001648 ptr = kmalloc(sizeof(struct arraycache_init), GFP_NOWAIT);
Christoph Lametere498be72005-09-09 13:03:32 -07001649
Pekka Enberg9a2dba4b2006-02-01 03:05:49 -08001650 BUG_ON(cpu_cache_get(malloc_sizes[INDEX_AC].cs_cachep)
Pekka Enbergb28a02d2006-01-08 01:00:37 -08001651 != &initarray_generic.cache);
Pekka Enberg9a2dba4b2006-02-01 03:05:49 -08001652 memcpy(ptr, cpu_cache_get(malloc_sizes[INDEX_AC].cs_cachep),
Pekka Enbergb28a02d2006-01-08 01:00:37 -08001653 sizeof(struct arraycache_init));
Ingo Molnar2b2d5492006-07-03 00:25:28 -07001654 /*
1655 * Do not assume that spinlocks can be initialized via memcpy:
1656 */
1657 spin_lock_init(&ptr->lock);
1658
Christoph Lametere498be72005-09-09 13:03:32 -07001659 malloc_sizes[INDEX_AC].cs_cachep->array[smp_processor_id()] =
Pekka Enbergb28a02d2006-01-08 01:00:37 -08001660 ptr;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001661 }
Christoph Lametere498be72005-09-09 13:03:32 -07001662 /* 5) Replace the bootstrap kmem_list3's */
1663 {
Pekka Enberg1ca4cb22006-10-06 00:43:52 -07001664 int nid;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001665
Mel Gorman9c09a952008-01-24 05:49:54 -08001666 for_each_online_node(nid) {
Daniel Yeisleyec1f5ee2008-03-25 23:59:08 +02001667 init_list(&cache_cache, &initkmem_list3[CACHE_CACHE + nid], nid);
Pekka Enberg556a1692008-01-25 08:20:51 +02001668
Christoph Lametere498be72005-09-09 13:03:32 -07001669 init_list(malloc_sizes[INDEX_AC].cs_cachep,
Pekka Enberg1ca4cb22006-10-06 00:43:52 -07001670 &initkmem_list3[SIZE_AC + nid], nid);
Christoph Lametere498be72005-09-09 13:03:32 -07001671
1672 if (INDEX_AC != INDEX_L3) {
1673 init_list(malloc_sizes[INDEX_L3].cs_cachep,
Pekka Enberg1ca4cb22006-10-06 00:43:52 -07001674 &initkmem_list3[SIZE_L3 + nid], nid);
Christoph Lametere498be72005-09-09 13:03:32 -07001675 }
1676 }
1677 }
1678
Pekka Enberg8429db52009-06-12 15:58:59 +03001679 g_cpucache_up = EARLY;
Pekka Enberg8429db52009-06-12 15:58:59 +03001680}
Ravikiran G Thirumalai056c6242006-09-25 23:31:38 -07001681
Pekka Enberg8429db52009-06-12 15:58:59 +03001682void __init kmem_cache_init_late(void)
1683{
1684 struct kmem_cache *cachep;
1685
Peter Zijlstra52cef182011-11-28 21:12:40 +01001686 g_cpucache_up = LATE;
1687
Peter Zijlstra30765b92011-07-28 23:22:56 +02001688 /* Annotate slab for lockdep -- annotate the malloc caches */
1689 init_lock_keys();
1690
Pekka Enberg8429db52009-06-12 15:58:59 +03001691 /* 6) resize the head arrays to their final sizes */
1692 mutex_lock(&cache_chain_mutex);
1693 list_for_each_entry(cachep, &cache_chain, next)
1694 if (enable_cpucache(cachep, GFP_NOWAIT))
1695 BUG();
1696 mutex_unlock(&cache_chain_mutex);
Ravikiran G Thirumalai056c6242006-09-25 23:31:38 -07001697
Linus Torvalds1da177e2005-04-16 15:20:36 -07001698 /* Done! */
1699 g_cpucache_up = FULL;
1700
Andrew Mortona737b3e2006-03-22 00:08:11 -08001701 /*
1702 * Register a cpu startup notifier callback that initializes
1703 * cpu_cache_get for all new cpus
Linus Torvalds1da177e2005-04-16 15:20:36 -07001704 */
1705 register_cpu_notifier(&cpucache_notifier);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001706
David Rientjes8f9f8d92010-03-27 19:40:47 -07001707#ifdef CONFIG_NUMA
1708 /*
1709 * Register a memory hotplug callback that initializes and frees
1710 * nodelists.
1711 */
1712 hotplug_memory_notifier(slab_memory_callback, SLAB_CALLBACK_PRI);
1713#endif
1714
Andrew Mortona737b3e2006-03-22 00:08:11 -08001715 /*
1716 * The reap timers are started later, with a module init call: That part
1717 * of the kernel is not yet operational.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001718 */
1719}
1720
1721static int __init cpucache_init(void)
1722{
1723 int cpu;
1724
Andrew Mortona737b3e2006-03-22 00:08:11 -08001725 /*
1726 * Register the timers that return unneeded pages to the page allocator
Linus Torvalds1da177e2005-04-16 15:20:36 -07001727 */
Christoph Lametere498be72005-09-09 13:03:32 -07001728 for_each_online_cpu(cpu)
Andrew Mortona737b3e2006-03-22 00:08:11 -08001729 start_cpu_timer(cpu);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001730 return 0;
1731}
Linus Torvalds1da177e2005-04-16 15:20:36 -07001732__initcall(cpucache_init);
1733
1734/*
1735 * Interface to system's page allocator. No need to hold the cache-lock.
1736 *
1737 * If we requested dmaable memory, we will get it. Even if we
1738 * did not request dmaable memory, we might get it, but that
1739 * would be relatively rare and ignorable.
1740 */
Pekka Enberg343e0d72006-02-01 03:05:50 -08001741static void *kmem_getpages(struct kmem_cache *cachep, gfp_t flags, int nodeid)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001742{
1743 struct page *page;
Christoph Hellwige1b6aa62006-06-23 02:03:17 -07001744 int nr_pages;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001745 int i;
1746
Luke Yangd6fef9d2006-04-10 22:52:56 -07001747#ifndef CONFIG_MMU
Christoph Hellwige1b6aa62006-06-23 02:03:17 -07001748 /*
1749 * Nommu uses slab's for process anonymous memory allocations, and thus
1750 * requires __GFP_COMP to properly refcount higher order allocations
Luke Yangd6fef9d2006-04-10 22:52:56 -07001751 */
Christoph Hellwige1b6aa62006-06-23 02:03:17 -07001752 flags |= __GFP_COMP;
Luke Yangd6fef9d2006-04-10 22:52:56 -07001753#endif
Christoph Lameter765c4502006-09-27 01:50:08 -07001754
Christoph Lameter3c517a62006-12-06 20:33:29 -08001755 flags |= cachep->gfpflags;
Mel Gormane12ba742007-10-16 01:25:52 -07001756 if (cachep->flags & SLAB_RECLAIM_ACCOUNT)
1757 flags |= __GFP_RECLAIMABLE;
Christoph Hellwige1b6aa62006-06-23 02:03:17 -07001758
Linus Torvalds517d0862009-06-16 19:50:13 -07001759 page = alloc_pages_exact_node(nodeid, flags | __GFP_NOTRACK, cachep->gfporder);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001760 if (!page)
1761 return NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001762
Christoph Hellwige1b6aa62006-06-23 02:03:17 -07001763 nr_pages = (1 << cachep->gfporder);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001764 if (cachep->flags & SLAB_RECLAIM_ACCOUNT)
Christoph Lameter972d1a72006-09-25 23:31:51 -07001765 add_zone_page_state(page_zone(page),
1766 NR_SLAB_RECLAIMABLE, nr_pages);
1767 else
1768 add_zone_page_state(page_zone(page),
1769 NR_SLAB_UNRECLAIMABLE, nr_pages);
Christoph Hellwige1b6aa62006-06-23 02:03:17 -07001770 for (i = 0; i < nr_pages; i++)
1771 __SetPageSlab(page + i);
Pekka Enbergc175eea2008-05-09 20:35:53 +02001772
Vegard Nossumb1eeab62008-11-25 16:55:53 +01001773 if (kmemcheck_enabled && !(cachep->flags & SLAB_NOTRACK)) {
1774 kmemcheck_alloc_shadow(page, cachep->gfporder, flags, nodeid);
1775
1776 if (cachep->ctor)
1777 kmemcheck_mark_uninitialized_pages(page, nr_pages);
1778 else
1779 kmemcheck_mark_unallocated_pages(page, nr_pages);
1780 }
Pekka Enbergc175eea2008-05-09 20:35:53 +02001781
Christoph Hellwige1b6aa62006-06-23 02:03:17 -07001782 return page_address(page);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001783}
1784
1785/*
1786 * Interface to system's page release.
1787 */
Pekka Enberg343e0d72006-02-01 03:05:50 -08001788static void kmem_freepages(struct kmem_cache *cachep, void *addr)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001789{
Pekka Enbergb28a02d2006-01-08 01:00:37 -08001790 unsigned long i = (1 << cachep->gfporder);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001791 struct page *page = virt_to_page(addr);
1792 const unsigned long nr_freed = i;
1793
Vegard Nossumb1eeab62008-11-25 16:55:53 +01001794 kmemcheck_free_shadow(page, cachep->gfporder);
Pekka Enbergc175eea2008-05-09 20:35:53 +02001795
Christoph Lameter972d1a72006-09-25 23:31:51 -07001796 if (cachep->flags & SLAB_RECLAIM_ACCOUNT)
1797 sub_zone_page_state(page_zone(page),
1798 NR_SLAB_RECLAIMABLE, nr_freed);
1799 else
1800 sub_zone_page_state(page_zone(page),
1801 NR_SLAB_UNRECLAIMABLE, nr_freed);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001802 while (i--) {
Nick Pigginf205b2f2006-03-22 00:08:02 -08001803 BUG_ON(!PageSlab(page));
1804 __ClearPageSlab(page);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001805 page++;
1806 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001807 if (current->reclaim_state)
1808 current->reclaim_state->reclaimed_slab += nr_freed;
1809 free_pages((unsigned long)addr, cachep->gfporder);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001810}
1811
1812static void kmem_rcu_free(struct rcu_head *head)
1813{
Pekka Enbergb28a02d2006-01-08 01:00:37 -08001814 struct slab_rcu *slab_rcu = (struct slab_rcu *)head;
Pekka Enberg343e0d72006-02-01 03:05:50 -08001815 struct kmem_cache *cachep = slab_rcu->cachep;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001816
1817 kmem_freepages(cachep, slab_rcu->addr);
1818 if (OFF_SLAB(cachep))
1819 kmem_cache_free(cachep->slabp_cache, slab_rcu);
1820}
1821
1822#if DEBUG
1823
1824#ifdef CONFIG_DEBUG_PAGEALLOC
Pekka Enberg343e0d72006-02-01 03:05:50 -08001825static void store_stackinfo(struct kmem_cache *cachep, unsigned long *addr,
Pekka Enbergb28a02d2006-01-08 01:00:37 -08001826 unsigned long caller)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001827{
Manfred Spraul3dafccf2006-02-01 03:05:42 -08001828 int size = obj_size(cachep);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001829
Manfred Spraul3dafccf2006-02-01 03:05:42 -08001830 addr = (unsigned long *)&((char *)addr)[obj_offset(cachep)];
Linus Torvalds1da177e2005-04-16 15:20:36 -07001831
Pekka Enbergb28a02d2006-01-08 01:00:37 -08001832 if (size < 5 * sizeof(unsigned long))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001833 return;
1834
Pekka Enbergb28a02d2006-01-08 01:00:37 -08001835 *addr++ = 0x12345678;
1836 *addr++ = caller;
1837 *addr++ = smp_processor_id();
1838 size -= 3 * sizeof(unsigned long);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001839 {
1840 unsigned long *sptr = &caller;
1841 unsigned long svalue;
1842
1843 while (!kstack_end(sptr)) {
1844 svalue = *sptr++;
1845 if (kernel_text_address(svalue)) {
Pekka Enbergb28a02d2006-01-08 01:00:37 -08001846 *addr++ = svalue;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001847 size -= sizeof(unsigned long);
1848 if (size <= sizeof(unsigned long))
1849 break;
1850 }
1851 }
1852
1853 }
Pekka Enbergb28a02d2006-01-08 01:00:37 -08001854 *addr++ = 0x87654321;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001855}
1856#endif
1857
Pekka Enberg343e0d72006-02-01 03:05:50 -08001858static void poison_obj(struct kmem_cache *cachep, void *addr, unsigned char val)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001859{
Manfred Spraul3dafccf2006-02-01 03:05:42 -08001860 int size = obj_size(cachep);
1861 addr = &((char *)addr)[obj_offset(cachep)];
Linus Torvalds1da177e2005-04-16 15:20:36 -07001862
1863 memset(addr, val, size);
Pekka Enbergb28a02d2006-01-08 01:00:37 -08001864 *(unsigned char *)(addr + size - 1) = POISON_END;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001865}
1866
1867static void dump_line(char *data, int offset, int limit)
1868{
1869 int i;
Dave Jonesaa83aa42006-09-29 01:59:51 -07001870 unsigned char error = 0;
1871 int bad_count = 0;
1872
Sebastian Andrzej Siewiorfdde6ab2011-07-29 18:22:13 +02001873 printk(KERN_ERR "%03x: ", offset);
Dave Jonesaa83aa42006-09-29 01:59:51 -07001874 for (i = 0; i < limit; i++) {
1875 if (data[offset + i] != POISON_FREE) {
1876 error = data[offset + i];
1877 bad_count++;
1878 }
Dave Jonesaa83aa42006-09-29 01:59:51 -07001879 }
Sebastian Andrzej Siewiorfdde6ab2011-07-29 18:22:13 +02001880 print_hex_dump(KERN_CONT, "", 0, 16, 1,
1881 &data[offset], limit, 1);
Dave Jonesaa83aa42006-09-29 01:59:51 -07001882
1883 if (bad_count == 1) {
1884 error ^= POISON_FREE;
1885 if (!(error & (error - 1))) {
1886 printk(KERN_ERR "Single bit error detected. Probably "
1887 "bad RAM.\n");
1888#ifdef CONFIG_X86
1889 printk(KERN_ERR "Run memtest86+ or a similar memory "
1890 "test tool.\n");
1891#else
1892 printk(KERN_ERR "Run a memory test tool.\n");
1893#endif
1894 }
1895 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001896}
1897#endif
1898
1899#if DEBUG
1900
Pekka Enberg343e0d72006-02-01 03:05:50 -08001901static void print_objinfo(struct kmem_cache *cachep, void *objp, int lines)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001902{
1903 int i, size;
1904 char *realobj;
1905
1906 if (cachep->flags & SLAB_RED_ZONE) {
David Woodhouseb46b8f12007-05-08 00:22:59 -07001907 printk(KERN_ERR "Redzone: 0x%llx/0x%llx.\n",
Andrew Mortona737b3e2006-03-22 00:08:11 -08001908 *dbg_redzone1(cachep, objp),
1909 *dbg_redzone2(cachep, objp));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001910 }
1911
1912 if (cachep->flags & SLAB_STORE_USER) {
1913 printk(KERN_ERR "Last user: [<%p>]",
Andrew Mortona737b3e2006-03-22 00:08:11 -08001914 *dbg_userword(cachep, objp));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001915 print_symbol("(%s)",
Andrew Mortona737b3e2006-03-22 00:08:11 -08001916 (unsigned long)*dbg_userword(cachep, objp));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001917 printk("\n");
1918 }
Manfred Spraul3dafccf2006-02-01 03:05:42 -08001919 realobj = (char *)objp + obj_offset(cachep);
1920 size = obj_size(cachep);
Pekka Enbergb28a02d2006-01-08 01:00:37 -08001921 for (i = 0; i < size && lines; i += 16, lines--) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001922 int limit;
1923 limit = 16;
Pekka Enbergb28a02d2006-01-08 01:00:37 -08001924 if (i + limit > size)
1925 limit = size - i;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001926 dump_line(realobj, i, limit);
1927 }
1928}
1929
Pekka Enberg343e0d72006-02-01 03:05:50 -08001930static void check_poison_obj(struct kmem_cache *cachep, void *objp)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001931{
1932 char *realobj;
1933 int size, i;
1934 int lines = 0;
1935
Manfred Spraul3dafccf2006-02-01 03:05:42 -08001936 realobj = (char *)objp + obj_offset(cachep);
1937 size = obj_size(cachep);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001938
Pekka Enbergb28a02d2006-01-08 01:00:37 -08001939 for (i = 0; i < size; i++) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001940 char exp = POISON_FREE;
Pekka Enbergb28a02d2006-01-08 01:00:37 -08001941 if (i == size - 1)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001942 exp = POISON_END;
1943 if (realobj[i] != exp) {
1944 int limit;
1945 /* Mismatch ! */
1946 /* Print header */
1947 if (lines == 0) {
Pekka Enbergb28a02d2006-01-08 01:00:37 -08001948 printk(KERN_ERR
Dave Jonesface37f2011-11-15 15:03:52 -08001949 "Slab corruption (%s): %s start=%p, len=%d\n",
1950 print_tainted(), cachep->name, realobj, size);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001951 print_objinfo(cachep, objp, 0);
1952 }
1953 /* Hexdump the affected line */
Pekka Enbergb28a02d2006-01-08 01:00:37 -08001954 i = (i / 16) * 16;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001955 limit = 16;
Pekka Enbergb28a02d2006-01-08 01:00:37 -08001956 if (i + limit > size)
1957 limit = size - i;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001958 dump_line(realobj, i, limit);
1959 i += 16;
1960 lines++;
1961 /* Limit to 5 lines */
1962 if (lines > 5)
1963 break;
1964 }
1965 }
1966 if (lines != 0) {
1967 /* Print some data about the neighboring objects, if they
1968 * exist:
1969 */
Pekka Enberg6ed5eb2212006-02-01 03:05:49 -08001970 struct slab *slabp = virt_to_slab(objp);
Pekka Enberg8fea4e92006-03-22 00:08:10 -08001971 unsigned int objnr;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001972
Pekka Enberg8fea4e92006-03-22 00:08:10 -08001973 objnr = obj_to_index(cachep, slabp, objp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001974 if (objnr) {
Pekka Enberg8fea4e92006-03-22 00:08:10 -08001975 objp = index_to_obj(cachep, slabp, objnr - 1);
Manfred Spraul3dafccf2006-02-01 03:05:42 -08001976 realobj = (char *)objp + obj_offset(cachep);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001977 printk(KERN_ERR "Prev obj: start=%p, len=%d\n",
Pekka Enbergb28a02d2006-01-08 01:00:37 -08001978 realobj, size);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001979 print_objinfo(cachep, objp, 2);
1980 }
Pekka Enbergb28a02d2006-01-08 01:00:37 -08001981 if (objnr + 1 < cachep->num) {
Pekka Enberg8fea4e92006-03-22 00:08:10 -08001982 objp = index_to_obj(cachep, slabp, objnr + 1);
Manfred Spraul3dafccf2006-02-01 03:05:42 -08001983 realobj = (char *)objp + obj_offset(cachep);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001984 printk(KERN_ERR "Next obj: start=%p, len=%d\n",
Pekka Enbergb28a02d2006-01-08 01:00:37 -08001985 realobj, size);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001986 print_objinfo(cachep, objp, 2);
1987 }
1988 }
1989}
1990#endif
1991
Linus Torvalds1da177e2005-04-16 15:20:36 -07001992#if DEBUG
Rabin Vincente79aec22008-07-04 00:40:32 +05301993static void slab_destroy_debugcheck(struct kmem_cache *cachep, struct slab *slabp)
Matthew Dobson12dd36f2006-02-01 03:05:46 -08001994{
Linus Torvalds1da177e2005-04-16 15:20:36 -07001995 int i;
1996 for (i = 0; i < cachep->num; i++) {
Pekka Enberg8fea4e92006-03-22 00:08:10 -08001997 void *objp = index_to_obj(cachep, slabp, i);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001998
1999 if (cachep->flags & SLAB_POISON) {
2000#ifdef CONFIG_DEBUG_PAGEALLOC
Andrew Mortona737b3e2006-03-22 00:08:11 -08002001 if (cachep->buffer_size % PAGE_SIZE == 0 &&
2002 OFF_SLAB(cachep))
Pekka Enbergb28a02d2006-01-08 01:00:37 -08002003 kernel_map_pages(virt_to_page(objp),
Andrew Mortona737b3e2006-03-22 00:08:11 -08002004 cachep->buffer_size / PAGE_SIZE, 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002005 else
2006 check_poison_obj(cachep, objp);
2007#else
2008 check_poison_obj(cachep, objp);
2009#endif
2010 }
2011 if (cachep->flags & SLAB_RED_ZONE) {
2012 if (*dbg_redzone1(cachep, objp) != RED_INACTIVE)
2013 slab_error(cachep, "start of a freed object "
Pekka Enbergb28a02d2006-01-08 01:00:37 -08002014 "was overwritten");
Linus Torvalds1da177e2005-04-16 15:20:36 -07002015 if (*dbg_redzone2(cachep, objp) != RED_INACTIVE)
2016 slab_error(cachep, "end of a freed object "
Pekka Enbergb28a02d2006-01-08 01:00:37 -08002017 "was overwritten");
Linus Torvalds1da177e2005-04-16 15:20:36 -07002018 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002019 }
Matthew Dobson12dd36f2006-02-01 03:05:46 -08002020}
Linus Torvalds1da177e2005-04-16 15:20:36 -07002021#else
Rabin Vincente79aec22008-07-04 00:40:32 +05302022static void slab_destroy_debugcheck(struct kmem_cache *cachep, struct slab *slabp)
Matthew Dobson12dd36f2006-02-01 03:05:46 -08002023{
Matthew Dobson12dd36f2006-02-01 03:05:46 -08002024}
Linus Torvalds1da177e2005-04-16 15:20:36 -07002025#endif
2026
Randy Dunlap911851e2006-03-22 00:08:14 -08002027/**
2028 * slab_destroy - destroy and release all objects in a slab
2029 * @cachep: cache pointer being destroyed
2030 * @slabp: slab pointer being destroyed
2031 *
Matthew Dobson12dd36f2006-02-01 03:05:46 -08002032 * Destroy all the objs in a slab, and release the mem back to the system.
Andrew Mortona737b3e2006-03-22 00:08:11 -08002033 * Before calling the slab must have been unlinked from the cache. The
2034 * cache-lock is not held/needed.
Matthew Dobson12dd36f2006-02-01 03:05:46 -08002035 */
Pekka Enberg343e0d72006-02-01 03:05:50 -08002036static void slab_destroy(struct kmem_cache *cachep, struct slab *slabp)
Matthew Dobson12dd36f2006-02-01 03:05:46 -08002037{
2038 void *addr = slabp->s_mem - slabp->colouroff;
2039
Rabin Vincente79aec22008-07-04 00:40:32 +05302040 slab_destroy_debugcheck(cachep, slabp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002041 if (unlikely(cachep->flags & SLAB_DESTROY_BY_RCU)) {
2042 struct slab_rcu *slab_rcu;
2043
Pekka Enbergb28a02d2006-01-08 01:00:37 -08002044 slab_rcu = (struct slab_rcu *)slabp;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002045 slab_rcu->cachep = cachep;
2046 slab_rcu->addr = addr;
2047 call_rcu(&slab_rcu->head, kmem_rcu_free);
2048 } else {
2049 kmem_freepages(cachep, addr);
Ingo Molnar873623d2006-07-13 14:44:38 +02002050 if (OFF_SLAB(cachep))
2051 kmem_cache_free(cachep->slabp_cache, slabp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002052 }
2053}
2054
Christoph Lameter117f6eb2006-09-25 23:31:37 -07002055static void __kmem_cache_destroy(struct kmem_cache *cachep)
2056{
2057 int i;
2058 struct kmem_list3 *l3;
2059
2060 for_each_online_cpu(i)
2061 kfree(cachep->array[i]);
2062
2063 /* NUMA: free the list3 structures */
2064 for_each_online_node(i) {
2065 l3 = cachep->nodelists[i];
2066 if (l3) {
2067 kfree(l3->shared);
2068 free_alien_cache(l3->alien);
2069 kfree(l3);
2070 }
2071 }
2072 kmem_cache_free(&cache_cache, cachep);
2073}
2074
2075
Linus Torvalds1da177e2005-04-16 15:20:36 -07002076/**
Randy.Dunlapa70773d2006-02-01 03:05:52 -08002077 * calculate_slab_order - calculate size (page order) of slabs
2078 * @cachep: pointer to the cache that is being created
2079 * @size: size of objects to be created in this cache.
2080 * @align: required alignment for the objects.
2081 * @flags: slab allocation flags
2082 *
2083 * Also calculates the number of objects per slab.
Pekka Enberg4d268eb2006-01-08 01:00:36 -08002084 *
2085 * This could be made much more intelligent. For now, try to avoid using
2086 * high order pages for slabs. When the gfp() functions are more friendly
2087 * towards high-order requests, this should be changed.
2088 */
Andrew Mortona737b3e2006-03-22 00:08:11 -08002089static size_t calculate_slab_order(struct kmem_cache *cachep,
Randy Dunlapee13d782006-02-01 03:05:53 -08002090 size_t size, size_t align, unsigned long flags)
Pekka Enberg4d268eb2006-01-08 01:00:36 -08002091{
Ingo Molnarb1ab41c2006-06-02 15:44:58 +02002092 unsigned long offslab_limit;
Pekka Enberg4d268eb2006-01-08 01:00:36 -08002093 size_t left_over = 0;
Linus Torvalds9888e6f2006-03-06 17:44:43 -08002094 int gfporder;
Pekka Enberg4d268eb2006-01-08 01:00:36 -08002095
Christoph Lameter0aa817f2007-05-16 22:11:01 -07002096 for (gfporder = 0; gfporder <= KMALLOC_MAX_ORDER; gfporder++) {
Pekka Enberg4d268eb2006-01-08 01:00:36 -08002097 unsigned int num;
2098 size_t remainder;
2099
Linus Torvalds9888e6f2006-03-06 17:44:43 -08002100 cache_estimate(gfporder, size, align, flags, &remainder, &num);
Pekka Enberg4d268eb2006-01-08 01:00:36 -08002101 if (!num)
2102 continue;
Linus Torvalds9888e6f2006-03-06 17:44:43 -08002103
Ingo Molnarb1ab41c2006-06-02 15:44:58 +02002104 if (flags & CFLGS_OFF_SLAB) {
2105 /*
2106 * Max number of objs-per-slab for caches which
2107 * use off-slab slabs. Needed to avoid a possible
2108 * looping condition in cache_grow().
2109 */
2110 offslab_limit = size - sizeof(struct slab);
2111 offslab_limit /= sizeof(kmem_bufctl_t);
2112
2113 if (num > offslab_limit)
2114 break;
2115 }
Pekka Enberg4d268eb2006-01-08 01:00:36 -08002116
Linus Torvalds9888e6f2006-03-06 17:44:43 -08002117 /* Found something acceptable - save it away */
Pekka Enberg4d268eb2006-01-08 01:00:36 -08002118 cachep->num = num;
Linus Torvalds9888e6f2006-03-06 17:44:43 -08002119 cachep->gfporder = gfporder;
Pekka Enberg4d268eb2006-01-08 01:00:36 -08002120 left_over = remainder;
2121
2122 /*
Linus Torvaldsf78bb8a2006-03-08 10:33:05 -08002123 * A VFS-reclaimable slab tends to have most allocations
2124 * as GFP_NOFS and we really don't want to have to be allocating
2125 * higher-order pages when we are unable to shrink dcache.
2126 */
2127 if (flags & SLAB_RECLAIM_ACCOUNT)
2128 break;
2129
2130 /*
Pekka Enberg4d268eb2006-01-08 01:00:36 -08002131 * Large number of objects is good, but very large slabs are
2132 * currently bad for the gfp()s.
2133 */
David Rientjes543585c2011-10-18 22:09:24 -07002134 if (gfporder >= slab_max_order)
Pekka Enberg4d268eb2006-01-08 01:00:36 -08002135 break;
2136
Linus Torvalds9888e6f2006-03-06 17:44:43 -08002137 /*
2138 * Acceptable internal fragmentation?
2139 */
Andrew Mortona737b3e2006-03-22 00:08:11 -08002140 if (left_over * 8 <= (PAGE_SIZE << gfporder))
Pekka Enberg4d268eb2006-01-08 01:00:36 -08002141 break;
2142 }
2143 return left_over;
2144}
2145
Pekka Enberg83b519e2009-06-10 19:40:04 +03002146static int __init_refok setup_cpu_cache(struct kmem_cache *cachep, gfp_t gfp)
Pekka Enbergf30cf7d2006-03-22 00:08:11 -08002147{
Christoph Lameter2ed3a4e2006-09-25 23:31:38 -07002148 if (g_cpucache_up == FULL)
Pekka Enberg83b519e2009-06-10 19:40:04 +03002149 return enable_cpucache(cachep, gfp);
Christoph Lameter2ed3a4e2006-09-25 23:31:38 -07002150
Pekka Enbergf30cf7d2006-03-22 00:08:11 -08002151 if (g_cpucache_up == NONE) {
2152 /*
2153 * Note: the first kmem_cache_create must create the cache
2154 * that's used by kmalloc(24), otherwise the creation of
2155 * further caches will BUG().
2156 */
2157 cachep->array[smp_processor_id()] = &initarray_generic.cache;
2158
2159 /*
2160 * If the cache that's used by kmalloc(sizeof(kmem_list3)) is
2161 * the first cache, then we need to set up all its list3s,
2162 * otherwise the creation of further caches will BUG().
2163 */
2164 set_up_list3s(cachep, SIZE_AC);
2165 if (INDEX_AC == INDEX_L3)
2166 g_cpucache_up = PARTIAL_L3;
2167 else
2168 g_cpucache_up = PARTIAL_AC;
2169 } else {
2170 cachep->array[smp_processor_id()] =
Pekka Enberg83b519e2009-06-10 19:40:04 +03002171 kmalloc(sizeof(struct arraycache_init), gfp);
Pekka Enbergf30cf7d2006-03-22 00:08:11 -08002172
2173 if (g_cpucache_up == PARTIAL_AC) {
2174 set_up_list3s(cachep, SIZE_L3);
2175 g_cpucache_up = PARTIAL_L3;
2176 } else {
2177 int node;
Pekka Enberg556a1692008-01-25 08:20:51 +02002178 for_each_online_node(node) {
Pekka Enbergf30cf7d2006-03-22 00:08:11 -08002179 cachep->nodelists[node] =
2180 kmalloc_node(sizeof(struct kmem_list3),
Pekka Enbergeb91f1d2009-06-12 14:56:09 +03002181 gfp, node);
Pekka Enbergf30cf7d2006-03-22 00:08:11 -08002182 BUG_ON(!cachep->nodelists[node]);
2183 kmem_list3_init(cachep->nodelists[node]);
2184 }
2185 }
2186 }
Lee Schermerhorn7d6e6d02010-05-26 14:45:03 -07002187 cachep->nodelists[numa_mem_id()]->next_reap =
Pekka Enbergf30cf7d2006-03-22 00:08:11 -08002188 jiffies + REAPTIMEOUT_LIST3 +
2189 ((unsigned long)cachep) % REAPTIMEOUT_LIST3;
2190
2191 cpu_cache_get(cachep)->avail = 0;
2192 cpu_cache_get(cachep)->limit = BOOT_CPUCACHE_ENTRIES;
2193 cpu_cache_get(cachep)->batchcount = 1;
2194 cpu_cache_get(cachep)->touched = 0;
2195 cachep->batchcount = 1;
2196 cachep->limit = BOOT_CPUCACHE_ENTRIES;
Christoph Lameter2ed3a4e2006-09-25 23:31:38 -07002197 return 0;
Pekka Enbergf30cf7d2006-03-22 00:08:11 -08002198}
2199
Pekka Enberg4d268eb2006-01-08 01:00:36 -08002200/**
Linus Torvalds1da177e2005-04-16 15:20:36 -07002201 * kmem_cache_create - Create a cache.
2202 * @name: A string which is used in /proc/slabinfo to identify this cache.
2203 * @size: The size of objects to be created in this cache.
2204 * @align: The required alignment for the objects.
2205 * @flags: SLAB flags
2206 * @ctor: A constructor for the objects.
Linus Torvalds1da177e2005-04-16 15:20:36 -07002207 *
2208 * Returns a ptr to the cache on success, NULL on failure.
2209 * Cannot be called within a int, but can be interrupted.
Paul Mundt20c2df82007-07-20 10:11:58 +09002210 * The @ctor is run when new pages are allocated by the cache.
Linus Torvalds1da177e2005-04-16 15:20:36 -07002211 *
2212 * @name must be valid until the cache is destroyed. This implies that
Andrew Mortona737b3e2006-03-22 00:08:11 -08002213 * the module calling this has to destroy the cache before getting unloaded.
2214 *
Linus Torvalds1da177e2005-04-16 15:20:36 -07002215 * The flags are
2216 *
2217 * %SLAB_POISON - Poison the slab with a known test pattern (a5a5a5a5)
2218 * to catch references to uninitialised memory.
2219 *
2220 * %SLAB_RED_ZONE - Insert `Red' zones around the allocated memory to check
2221 * for buffer overruns.
2222 *
Linus Torvalds1da177e2005-04-16 15:20:36 -07002223 * %SLAB_HWCACHE_ALIGN - Align the objects in this cache to a hardware
2224 * cacheline. This can be beneficial if you're counting cycles as closely
2225 * as davem.
2226 */
Pekka Enberg343e0d72006-02-01 03:05:50 -08002227struct kmem_cache *
Linus Torvalds1da177e2005-04-16 15:20:36 -07002228kmem_cache_create (const char *name, size_t size, size_t align,
Alexey Dobriyan51cc5062008-07-25 19:45:34 -07002229 unsigned long flags, void (*ctor)(void *))
Linus Torvalds1da177e2005-04-16 15:20:36 -07002230{
2231 size_t left_over, slab_size, ralign;
Christoph Hellwig7a7c3812006-06-23 02:03:17 -07002232 struct kmem_cache *cachep = NULL, *pc;
Pekka Enberg83b519e2009-06-10 19:40:04 +03002233 gfp_t gfp;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002234
2235 /*
2236 * Sanity checks... these are all serious usage bugs.
2237 */
Andrew Mortona737b3e2006-03-22 00:08:11 -08002238 if (!name || in_interrupt() || (size < BYTES_PER_WORD) ||
Paul Mundt20c2df82007-07-20 10:11:58 +09002239 size > KMALLOC_MAX_SIZE) {
Harvey Harrisond40cee22008-04-30 00:55:07 -07002240 printk(KERN_ERR "%s: Early error in slab %s\n", __func__,
Andrew Mortona737b3e2006-03-22 00:08:11 -08002241 name);
Pekka Enbergb28a02d2006-01-08 01:00:37 -08002242 BUG();
2243 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002244
Ravikiran G Thirumalaif0188f42006-02-10 01:51:13 -08002245 /*
Ravikiran G Thirumalai8f5be202006-12-06 20:32:14 -08002246 * We use cache_chain_mutex to ensure a consistent view of
Rusty Russell174596a2009-01-01 10:12:29 +10302247 * cpu_online_mask as well. Please see cpuup_callback
Ravikiran G Thirumalaif0188f42006-02-10 01:51:13 -08002248 */
Pekka Enberg83b519e2009-06-10 19:40:04 +03002249 if (slab_is_available()) {
2250 get_online_cpus();
2251 mutex_lock(&cache_chain_mutex);
2252 }
Andrew Morton4f12bb42005-11-07 00:58:00 -08002253
Christoph Hellwig7a7c3812006-06-23 02:03:17 -07002254 list_for_each_entry(pc, &cache_chain, next) {
Andrew Morton4f12bb42005-11-07 00:58:00 -08002255 char tmp;
2256 int res;
2257
2258 /*
2259 * This happens when the module gets unloaded and doesn't
2260 * destroy its slab cache and no-one else reuses the vmalloc
2261 * area of the module. Print a warning.
2262 */
Andrew Morton138ae662006-12-06 20:36:41 -08002263 res = probe_kernel_address(pc->name, tmp);
Andrew Morton4f12bb42005-11-07 00:58:00 -08002264 if (res) {
matzeb4169522007-05-06 14:49:52 -07002265 printk(KERN_ERR
2266 "SLAB: cache with size %d has lost its name\n",
Manfred Spraul3dafccf2006-02-01 03:05:42 -08002267 pc->buffer_size);
Andrew Morton4f12bb42005-11-07 00:58:00 -08002268 continue;
2269 }
2270
Pekka Enbergb28a02d2006-01-08 01:00:37 -08002271 if (!strcmp(pc->name, name)) {
matzeb4169522007-05-06 14:49:52 -07002272 printk(KERN_ERR
2273 "kmem_cache_create: duplicate cache %s\n", name);
Andrew Morton4f12bb42005-11-07 00:58:00 -08002274 dump_stack();
2275 goto oops;
2276 }
2277 }
2278
Linus Torvalds1da177e2005-04-16 15:20:36 -07002279#if DEBUG
2280 WARN_ON(strchr(name, ' ')); /* It confuses parsers */
Linus Torvalds1da177e2005-04-16 15:20:36 -07002281#if FORCED_DEBUG
2282 /*
2283 * Enable redzoning and last user accounting, except for caches with
2284 * large objects, if the increased size would increase the object size
2285 * above the next power of two: caches with object sizes just above a
2286 * power of two have a significant amount of internal fragmentation.
2287 */
David Woodhouse87a927c2007-07-04 21:26:44 -04002288 if (size < 4096 || fls(size - 1) == fls(size-1 + REDZONE_ALIGN +
2289 2 * sizeof(unsigned long long)))
Pekka Enbergb28a02d2006-01-08 01:00:37 -08002290 flags |= SLAB_RED_ZONE | SLAB_STORE_USER;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002291 if (!(flags & SLAB_DESTROY_BY_RCU))
2292 flags |= SLAB_POISON;
2293#endif
2294 if (flags & SLAB_DESTROY_BY_RCU)
2295 BUG_ON(flags & SLAB_POISON);
2296#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07002297 /*
Andrew Mortona737b3e2006-03-22 00:08:11 -08002298 * Always checks flags, a caller might be expecting debug support which
2299 * isn't available.
Linus Torvalds1da177e2005-04-16 15:20:36 -07002300 */
Eric Sesterhenn40094fa2006-04-02 13:49:25 +02002301 BUG_ON(flags & ~CREATE_MASK);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002302
Andrew Mortona737b3e2006-03-22 00:08:11 -08002303 /*
2304 * Check that size is in terms of words. This is needed to avoid
Linus Torvalds1da177e2005-04-16 15:20:36 -07002305 * unaligned accesses for some archs when redzoning is used, and makes
2306 * sure any on-slab bufctl's are also correctly aligned.
2307 */
Pekka Enbergb28a02d2006-01-08 01:00:37 -08002308 if (size & (BYTES_PER_WORD - 1)) {
2309 size += (BYTES_PER_WORD - 1);
2310 size &= ~(BYTES_PER_WORD - 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002311 }
2312
Andrew Mortona737b3e2006-03-22 00:08:11 -08002313 /* calculate the final buffer alignment: */
2314
Linus Torvalds1da177e2005-04-16 15:20:36 -07002315 /* 1) arch recommendation: can be overridden for debug */
2316 if (flags & SLAB_HWCACHE_ALIGN) {
Andrew Mortona737b3e2006-03-22 00:08:11 -08002317 /*
2318 * Default alignment: as specified by the arch code. Except if
2319 * an object is really small, then squeeze multiple objects into
2320 * one cacheline.
Linus Torvalds1da177e2005-04-16 15:20:36 -07002321 */
2322 ralign = cache_line_size();
Pekka Enbergb28a02d2006-01-08 01:00:37 -08002323 while (size <= ralign / 2)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002324 ralign /= 2;
2325 } else {
2326 ralign = BYTES_PER_WORD;
2327 }
Pekka Enbergca5f9702006-09-25 23:31:25 -07002328
2329 /*
David Woodhouse87a927c2007-07-04 21:26:44 -04002330 * Redzoning and user store require word alignment or possibly larger.
2331 * Note this will be overridden by architecture or caller mandated
2332 * alignment if either is greater than BYTES_PER_WORD.
Pekka Enbergca5f9702006-09-25 23:31:25 -07002333 */
David Woodhouse87a927c2007-07-04 21:26:44 -04002334 if (flags & SLAB_STORE_USER)
2335 ralign = BYTES_PER_WORD;
2336
2337 if (flags & SLAB_RED_ZONE) {
2338 ralign = REDZONE_ALIGN;
2339 /* If redzoning, ensure that the second redzone is suitably
2340 * aligned, by adjusting the object size accordingly. */
2341 size += REDZONE_ALIGN - 1;
2342 size &= ~(REDZONE_ALIGN - 1);
2343 }
Pekka Enbergca5f9702006-09-25 23:31:25 -07002344
Kevin Hilmana44b56d2006-12-06 20:32:11 -08002345 /* 2) arch mandated alignment */
Linus Torvalds1da177e2005-04-16 15:20:36 -07002346 if (ralign < ARCH_SLAB_MINALIGN) {
2347 ralign = ARCH_SLAB_MINALIGN;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002348 }
Kevin Hilmana44b56d2006-12-06 20:32:11 -08002349 /* 3) caller mandated alignment */
Linus Torvalds1da177e2005-04-16 15:20:36 -07002350 if (ralign < align) {
2351 ralign = align;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002352 }
Pekka Enberg3ff84a72011-02-14 17:46:21 +02002353 /* disable debug if necessary */
2354 if (ralign > __alignof__(unsigned long long))
Kevin Hilmana44b56d2006-12-06 20:32:11 -08002355 flags &= ~(SLAB_RED_ZONE | SLAB_STORE_USER);
Andrew Mortona737b3e2006-03-22 00:08:11 -08002356 /*
Pekka Enbergca5f9702006-09-25 23:31:25 -07002357 * 4) Store it.
Linus Torvalds1da177e2005-04-16 15:20:36 -07002358 */
2359 align = ralign;
2360
Pekka Enberg83b519e2009-06-10 19:40:04 +03002361 if (slab_is_available())
2362 gfp = GFP_KERNEL;
2363 else
2364 gfp = GFP_NOWAIT;
2365
Linus Torvalds1da177e2005-04-16 15:20:36 -07002366 /* Get cache's description obj. */
Pekka Enberg83b519e2009-06-10 19:40:04 +03002367 cachep = kmem_cache_zalloc(&cache_cache, gfp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002368 if (!cachep)
Andrew Morton4f12bb42005-11-07 00:58:00 -08002369 goto oops;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002370
Eric Dumazetb56efcf2011-07-20 19:04:23 +02002371 cachep->nodelists = (struct kmem_list3 **)&cachep->array[nr_cpu_ids];
Linus Torvalds1da177e2005-04-16 15:20:36 -07002372#if DEBUG
Manfred Spraul3dafccf2006-02-01 03:05:42 -08002373 cachep->obj_size = size;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002374
Pekka Enbergca5f9702006-09-25 23:31:25 -07002375 /*
2376 * Both debugging options require word-alignment which is calculated
2377 * into align above.
2378 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07002379 if (flags & SLAB_RED_ZONE) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07002380 /* add space for red zone words */
Pekka Enberg3ff84a72011-02-14 17:46:21 +02002381 cachep->obj_offset += sizeof(unsigned long long);
2382 size += 2 * sizeof(unsigned long long);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002383 }
2384 if (flags & SLAB_STORE_USER) {
Pekka Enbergca5f9702006-09-25 23:31:25 -07002385 /* user store requires one word storage behind the end of
David Woodhouse87a927c2007-07-04 21:26:44 -04002386 * the real object. But if the second red zone needs to be
2387 * aligned to 64 bits, we must allow that much space.
Linus Torvalds1da177e2005-04-16 15:20:36 -07002388 */
David Woodhouse87a927c2007-07-04 21:26:44 -04002389 if (flags & SLAB_RED_ZONE)
2390 size += REDZONE_ALIGN;
2391 else
2392 size += BYTES_PER_WORD;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002393 }
2394#if FORCED_DEBUG && defined(CONFIG_DEBUG_PAGEALLOC)
Pekka Enbergb28a02d2006-01-08 01:00:37 -08002395 if (size >= malloc_sizes[INDEX_L3 + 1].cs_size
Carsten Otte1ab335d2010-08-06 18:19:22 +02002396 && cachep->obj_size > cache_line_size() && ALIGN(size, align) < PAGE_SIZE) {
2397 cachep->obj_offset += PAGE_SIZE - ALIGN(size, align);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002398 size = PAGE_SIZE;
2399 }
2400#endif
2401#endif
2402
Ingo Molnare0a42722006-06-23 02:03:46 -07002403 /*
2404 * Determine if the slab management is 'on' or 'off' slab.
2405 * (bootstrapping cannot cope with offslab caches so don't do
Catalin Marinase7cb55b2009-10-28 13:33:08 +00002406 * it too early on. Always use on-slab management when
2407 * SLAB_NOLEAKTRACE to avoid recursive calls into kmemleak)
Ingo Molnare0a42722006-06-23 02:03:46 -07002408 */
Catalin Marinase7cb55b2009-10-28 13:33:08 +00002409 if ((size >= (PAGE_SIZE >> 3)) && !slab_early_init &&
2410 !(flags & SLAB_NOLEAKTRACE))
Linus Torvalds1da177e2005-04-16 15:20:36 -07002411 /*
2412 * Size is large, assume best to place the slab management obj
2413 * off-slab (should allow better packing of objs).
2414 */
2415 flags |= CFLGS_OFF_SLAB;
2416
2417 size = ALIGN(size, align);
2418
Linus Torvaldsf78bb8a2006-03-08 10:33:05 -08002419 left_over = calculate_slab_order(cachep, size, align, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002420
2421 if (!cachep->num) {
matzeb4169522007-05-06 14:49:52 -07002422 printk(KERN_ERR
2423 "kmem_cache_create: couldn't create cache %s.\n", name);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002424 kmem_cache_free(&cache_cache, cachep);
2425 cachep = NULL;
Andrew Morton4f12bb42005-11-07 00:58:00 -08002426 goto oops;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002427 }
Pekka Enbergb28a02d2006-01-08 01:00:37 -08002428 slab_size = ALIGN(cachep->num * sizeof(kmem_bufctl_t)
2429 + sizeof(struct slab), align);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002430
2431 /*
2432 * If the slab has been placed off-slab, and we have enough space then
2433 * move it on-slab. This is at the expense of any extra colouring.
2434 */
2435 if (flags & CFLGS_OFF_SLAB && left_over >= slab_size) {
2436 flags &= ~CFLGS_OFF_SLAB;
2437 left_over -= slab_size;
2438 }
2439
2440 if (flags & CFLGS_OFF_SLAB) {
2441 /* really off slab. No need for manual alignment */
Pekka Enbergb28a02d2006-01-08 01:00:37 -08002442 slab_size =
2443 cachep->num * sizeof(kmem_bufctl_t) + sizeof(struct slab);
Ron Lee67461362009-05-22 04:58:22 +09302444
2445#ifdef CONFIG_PAGE_POISONING
2446 /* If we're going to use the generic kernel_map_pages()
2447 * poisoning, then it's going to smash the contents of
2448 * the redzone and userword anyhow, so switch them off.
2449 */
2450 if (size % PAGE_SIZE == 0 && flags & SLAB_POISON)
2451 flags &= ~(SLAB_RED_ZONE | SLAB_STORE_USER);
2452#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07002453 }
2454
2455 cachep->colour_off = cache_line_size();
2456 /* Offset must be a multiple of the alignment. */
2457 if (cachep->colour_off < align)
2458 cachep->colour_off = align;
Pekka Enbergb28a02d2006-01-08 01:00:37 -08002459 cachep->colour = left_over / cachep->colour_off;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002460 cachep->slab_size = slab_size;
2461 cachep->flags = flags;
2462 cachep->gfpflags = 0;
Christoph Lameter4b51d662007-02-10 01:43:10 -08002463 if (CONFIG_ZONE_DMA_FLAG && (flags & SLAB_CACHE_DMA))
Linus Torvalds1da177e2005-04-16 15:20:36 -07002464 cachep->gfpflags |= GFP_DMA;
Manfred Spraul3dafccf2006-02-01 03:05:42 -08002465 cachep->buffer_size = size;
Eric Dumazet6a2d7a92006-12-13 00:34:27 -08002466 cachep->reciprocal_buffer_size = reciprocal_value(size);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002467
Ravikiran G Thirumalaie5ac9c52006-09-25 23:31:34 -07002468 if (flags & CFLGS_OFF_SLAB) {
Victor Fuscob2d55072005-09-10 00:26:36 -07002469 cachep->slabp_cache = kmem_find_general_cachep(slab_size, 0u);
Ravikiran G Thirumalaie5ac9c52006-09-25 23:31:34 -07002470 /*
2471 * This is a possibility for one of the malloc_sizes caches.
2472 * But since we go off slab only for object size greater than
2473 * PAGE_SIZE/8, and malloc_sizes gets created in ascending order,
2474 * this should not happen at all.
2475 * But leave a BUG_ON for some lucky dude.
2476 */
Christoph Lameter6cb8f912007-07-17 04:03:22 -07002477 BUG_ON(ZERO_OR_NULL_PTR(cachep->slabp_cache));
Ravikiran G Thirumalaie5ac9c52006-09-25 23:31:34 -07002478 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002479 cachep->ctor = ctor;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002480 cachep->name = name;
2481
Pekka Enberg83b519e2009-06-10 19:40:04 +03002482 if (setup_cpu_cache(cachep, gfp)) {
Christoph Lameter2ed3a4e2006-09-25 23:31:38 -07002483 __kmem_cache_destroy(cachep);
2484 cachep = NULL;
2485 goto oops;
2486 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002487
Peter Zijlstra83835b32011-07-22 15:26:05 +02002488 if (flags & SLAB_DEBUG_OBJECTS) {
2489 /*
2490 * Would deadlock through slab_destroy()->call_rcu()->
2491 * debug_object_activate()->kmem_cache_alloc().
2492 */
2493 WARN_ON_ONCE(flags & SLAB_DESTROY_BY_RCU);
2494
2495 slab_set_debugobj_lock_classes(cachep);
2496 }
2497
Linus Torvalds1da177e2005-04-16 15:20:36 -07002498 /* cache setup completed, link it into the list */
2499 list_add(&cachep->next, &cache_chain);
Andrew Mortona737b3e2006-03-22 00:08:11 -08002500oops:
Linus Torvalds1da177e2005-04-16 15:20:36 -07002501 if (!cachep && (flags & SLAB_PANIC))
2502 panic("kmem_cache_create(): failed to create slab `%s'\n",
Pekka Enbergb28a02d2006-01-08 01:00:37 -08002503 name);
Pekka Enberg83b519e2009-06-10 19:40:04 +03002504 if (slab_is_available()) {
2505 mutex_unlock(&cache_chain_mutex);
2506 put_online_cpus();
2507 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002508 return cachep;
2509}
2510EXPORT_SYMBOL(kmem_cache_create);
2511
2512#if DEBUG
2513static void check_irq_off(void)
2514{
2515 BUG_ON(!irqs_disabled());
2516}
2517
2518static void check_irq_on(void)
2519{
2520 BUG_ON(irqs_disabled());
2521}
2522
Pekka Enberg343e0d72006-02-01 03:05:50 -08002523static void check_spinlock_acquired(struct kmem_cache *cachep)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002524{
2525#ifdef CONFIG_SMP
2526 check_irq_off();
Lee Schermerhorn7d6e6d02010-05-26 14:45:03 -07002527 assert_spin_locked(&cachep->nodelists[numa_mem_id()]->list_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002528#endif
2529}
Christoph Lametere498be72005-09-09 13:03:32 -07002530
Pekka Enberg343e0d72006-02-01 03:05:50 -08002531static void check_spinlock_acquired_node(struct kmem_cache *cachep, int node)
Christoph Lametere498be72005-09-09 13:03:32 -07002532{
2533#ifdef CONFIG_SMP
2534 check_irq_off();
2535 assert_spin_locked(&cachep->nodelists[node]->list_lock);
2536#endif
2537}
2538
Linus Torvalds1da177e2005-04-16 15:20:36 -07002539#else
2540#define check_irq_off() do { } while(0)
2541#define check_irq_on() do { } while(0)
2542#define check_spinlock_acquired(x) do { } while(0)
Christoph Lametere498be72005-09-09 13:03:32 -07002543#define check_spinlock_acquired_node(x, y) do { } while(0)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002544#endif
2545
Christoph Lameteraab22072006-03-22 00:09:06 -08002546static void drain_array(struct kmem_cache *cachep, struct kmem_list3 *l3,
2547 struct array_cache *ac,
2548 int force, int node);
2549
Linus Torvalds1da177e2005-04-16 15:20:36 -07002550static void do_drain(void *arg)
2551{
Andrew Mortona737b3e2006-03-22 00:08:11 -08002552 struct kmem_cache *cachep = arg;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002553 struct array_cache *ac;
Lee Schermerhorn7d6e6d02010-05-26 14:45:03 -07002554 int node = numa_mem_id();
Linus Torvalds1da177e2005-04-16 15:20:36 -07002555
2556 check_irq_off();
Pekka Enberg9a2dba4b2006-02-01 03:05:49 -08002557 ac = cpu_cache_get(cachep);
Christoph Lameterff694162005-09-22 21:44:02 -07002558 spin_lock(&cachep->nodelists[node]->list_lock);
2559 free_block(cachep, ac->entry, ac->avail, node);
2560 spin_unlock(&cachep->nodelists[node]->list_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002561 ac->avail = 0;
2562}
2563
Pekka Enberg343e0d72006-02-01 03:05:50 -08002564static void drain_cpu_caches(struct kmem_cache *cachep)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002565{
Christoph Lametere498be72005-09-09 13:03:32 -07002566 struct kmem_list3 *l3;
2567 int node;
2568
Jens Axboe15c8b6c2008-05-09 09:39:44 +02002569 on_each_cpu(do_drain, cachep, 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002570 check_irq_on();
Pekka Enbergb28a02d2006-01-08 01:00:37 -08002571 for_each_online_node(node) {
Christoph Lametere498be72005-09-09 13:03:32 -07002572 l3 = cachep->nodelists[node];
Roland Dreiera4523a82006-05-15 11:41:00 -07002573 if (l3 && l3->alien)
2574 drain_alien_cache(cachep, l3->alien);
2575 }
2576
2577 for_each_online_node(node) {
2578 l3 = cachep->nodelists[node];
2579 if (l3)
Christoph Lameteraab22072006-03-22 00:09:06 -08002580 drain_array(cachep, l3, l3->shared, 1, node);
Christoph Lametere498be72005-09-09 13:03:32 -07002581 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002582}
2583
Christoph Lametered11d9e2006-06-30 01:55:45 -07002584/*
2585 * Remove slabs from the list of free slabs.
2586 * Specify the number of slabs to drain in tofree.
2587 *
2588 * Returns the actual number of slabs released.
2589 */
2590static int drain_freelist(struct kmem_cache *cache,
2591 struct kmem_list3 *l3, int tofree)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002592{
Christoph Lametered11d9e2006-06-30 01:55:45 -07002593 struct list_head *p;
2594 int nr_freed;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002595 struct slab *slabp;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002596
Christoph Lametered11d9e2006-06-30 01:55:45 -07002597 nr_freed = 0;
2598 while (nr_freed < tofree && !list_empty(&l3->slabs_free)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07002599
Christoph Lametered11d9e2006-06-30 01:55:45 -07002600 spin_lock_irq(&l3->list_lock);
Christoph Lametere498be72005-09-09 13:03:32 -07002601 p = l3->slabs_free.prev;
Christoph Lametered11d9e2006-06-30 01:55:45 -07002602 if (p == &l3->slabs_free) {
2603 spin_unlock_irq(&l3->list_lock);
2604 goto out;
2605 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002606
Christoph Lametered11d9e2006-06-30 01:55:45 -07002607 slabp = list_entry(p, struct slab, list);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002608#if DEBUG
Eric Sesterhenn40094fa2006-04-02 13:49:25 +02002609 BUG_ON(slabp->inuse);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002610#endif
2611 list_del(&slabp->list);
Christoph Lametered11d9e2006-06-30 01:55:45 -07002612 /*
2613 * Safe to drop the lock. The slab is no longer linked
2614 * to the cache.
2615 */
2616 l3->free_objects -= cache->num;
Christoph Lametere498be72005-09-09 13:03:32 -07002617 spin_unlock_irq(&l3->list_lock);
Christoph Lametered11d9e2006-06-30 01:55:45 -07002618 slab_destroy(cache, slabp);
2619 nr_freed++;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002620 }
Christoph Lametered11d9e2006-06-30 01:55:45 -07002621out:
2622 return nr_freed;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002623}
2624
Ravikiran G Thirumalai8f5be202006-12-06 20:32:14 -08002625/* Called with cache_chain_mutex held to protect against cpu hotplug */
Pekka Enberg343e0d72006-02-01 03:05:50 -08002626static int __cache_shrink(struct kmem_cache *cachep)
Christoph Lametere498be72005-09-09 13:03:32 -07002627{
2628 int ret = 0, i = 0;
2629 struct kmem_list3 *l3;
2630
2631 drain_cpu_caches(cachep);
2632
2633 check_irq_on();
2634 for_each_online_node(i) {
2635 l3 = cachep->nodelists[i];
Christoph Lametered11d9e2006-06-30 01:55:45 -07002636 if (!l3)
2637 continue;
2638
2639 drain_freelist(cachep, l3, l3->free_objects);
2640
2641 ret += !list_empty(&l3->slabs_full) ||
2642 !list_empty(&l3->slabs_partial);
Christoph Lametere498be72005-09-09 13:03:32 -07002643 }
2644 return (ret ? 1 : 0);
2645}
2646
Linus Torvalds1da177e2005-04-16 15:20:36 -07002647/**
2648 * kmem_cache_shrink - Shrink a cache.
2649 * @cachep: The cache to shrink.
2650 *
2651 * Releases as many slabs as possible for a cache.
2652 * To help debugging, a zero exit status indicates all slabs were released.
2653 */
Pekka Enberg343e0d72006-02-01 03:05:50 -08002654int kmem_cache_shrink(struct kmem_cache *cachep)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002655{
Ravikiran G Thirumalai8f5be202006-12-06 20:32:14 -08002656 int ret;
Eric Sesterhenn40094fa2006-04-02 13:49:25 +02002657 BUG_ON(!cachep || in_interrupt());
Linus Torvalds1da177e2005-04-16 15:20:36 -07002658
Gautham R Shenoy95402b32008-01-25 21:08:02 +01002659 get_online_cpus();
Ravikiran G Thirumalai8f5be202006-12-06 20:32:14 -08002660 mutex_lock(&cache_chain_mutex);
2661 ret = __cache_shrink(cachep);
2662 mutex_unlock(&cache_chain_mutex);
Gautham R Shenoy95402b32008-01-25 21:08:02 +01002663 put_online_cpus();
Ravikiran G Thirumalai8f5be202006-12-06 20:32:14 -08002664 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002665}
2666EXPORT_SYMBOL(kmem_cache_shrink);
2667
2668/**
2669 * kmem_cache_destroy - delete a cache
2670 * @cachep: the cache to destroy
2671 *
Robert P. J. Day72fd4a32007-02-10 01:45:59 -08002672 * Remove a &struct kmem_cache object from the slab cache.
Linus Torvalds1da177e2005-04-16 15:20:36 -07002673 *
2674 * It is expected this function will be called by a module when it is
2675 * unloaded. This will remove the cache completely, and avoid a duplicate
2676 * cache being allocated each time a module is loaded and unloaded, if the
2677 * module doesn't have persistent in-kernel storage across loads and unloads.
2678 *
2679 * The cache must be empty before calling this function.
2680 *
Lucas De Marchi25985ed2011-03-30 22:57:33 -03002681 * The caller must guarantee that no one will allocate memory from the cache
Linus Torvalds1da177e2005-04-16 15:20:36 -07002682 * during the kmem_cache_destroy().
2683 */
Alexey Dobriyan133d2052006-09-27 01:49:41 -07002684void kmem_cache_destroy(struct kmem_cache *cachep)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002685{
Eric Sesterhenn40094fa2006-04-02 13:49:25 +02002686 BUG_ON(!cachep || in_interrupt());
Linus Torvalds1da177e2005-04-16 15:20:36 -07002687
Linus Torvalds1da177e2005-04-16 15:20:36 -07002688 /* Find the cache in the chain of caches. */
Gautham R Shenoy95402b32008-01-25 21:08:02 +01002689 get_online_cpus();
Ingo Molnarfc0abb12006-01-18 17:42:33 -08002690 mutex_lock(&cache_chain_mutex);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002691 /*
2692 * the chain is never empty, cache_cache is never destroyed
2693 */
2694 list_del(&cachep->next);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002695 if (__cache_shrink(cachep)) {
2696 slab_error(cachep, "Can't free all objects");
Pekka Enbergb28a02d2006-01-08 01:00:37 -08002697 list_add(&cachep->next, &cache_chain);
Ingo Molnarfc0abb12006-01-18 17:42:33 -08002698 mutex_unlock(&cache_chain_mutex);
Gautham R Shenoy95402b32008-01-25 21:08:02 +01002699 put_online_cpus();
Alexey Dobriyan133d2052006-09-27 01:49:41 -07002700 return;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002701 }
2702
2703 if (unlikely(cachep->flags & SLAB_DESTROY_BY_RCU))
Paul E. McKenney7ed9f7e2009-06-25 12:31:37 -07002704 rcu_barrier();
Linus Torvalds1da177e2005-04-16 15:20:36 -07002705
Christoph Lameter117f6eb2006-09-25 23:31:37 -07002706 __kmem_cache_destroy(cachep);
Ravikiran G Thirumalai8f5be202006-12-06 20:32:14 -08002707 mutex_unlock(&cache_chain_mutex);
Gautham R Shenoy95402b32008-01-25 21:08:02 +01002708 put_online_cpus();
Linus Torvalds1da177e2005-04-16 15:20:36 -07002709}
2710EXPORT_SYMBOL(kmem_cache_destroy);
2711
Ravikiran G Thirumalaie5ac9c52006-09-25 23:31:34 -07002712/*
2713 * Get the memory for a slab management obj.
2714 * For a slab cache when the slab descriptor is off-slab, slab descriptors
2715 * always come from malloc_sizes caches. The slab descriptor cannot
2716 * come from the same cache which is getting created because,
2717 * when we are searching for an appropriate cache for these
2718 * descriptors in kmem_cache_create, we search through the malloc_sizes array.
2719 * If we are creating a malloc_sizes cache here it would not be visible to
2720 * kmem_find_general_cachep till the initialization is complete.
2721 * Hence we cannot have slabp_cache same as the original cache.
2722 */
Pekka Enberg343e0d72006-02-01 03:05:50 -08002723static struct slab *alloc_slabmgmt(struct kmem_cache *cachep, void *objp,
Ravikiran G Thirumalai5b74ada2006-04-10 22:52:53 -07002724 int colour_off, gfp_t local_flags,
2725 int nodeid)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002726{
2727 struct slab *slabp;
Pekka Enbergb28a02d2006-01-08 01:00:37 -08002728
Linus Torvalds1da177e2005-04-16 15:20:36 -07002729 if (OFF_SLAB(cachep)) {
2730 /* Slab management obj is off-slab. */
Ravikiran G Thirumalai5b74ada2006-04-10 22:52:53 -07002731 slabp = kmem_cache_alloc_node(cachep->slabp_cache,
Pekka Enberg8759ec52008-11-26 10:01:31 +02002732 local_flags, nodeid);
Catalin Marinasd5cff632009-06-11 13:22:40 +01002733 /*
2734 * If the first object in the slab is leaked (it's allocated
2735 * but no one has a reference to it), we want to make sure
2736 * kmemleak does not treat the ->s_mem pointer as a reference
2737 * to the object. Otherwise we will not report the leak.
2738 */
Catalin Marinasc017b4b2009-10-28 13:33:09 +00002739 kmemleak_scan_area(&slabp->list, sizeof(struct list_head),
2740 local_flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002741 if (!slabp)
2742 return NULL;
2743 } else {
Pekka Enbergb28a02d2006-01-08 01:00:37 -08002744 slabp = objp + colour_off;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002745 colour_off += cachep->slab_size;
2746 }
2747 slabp->inuse = 0;
2748 slabp->colouroff = colour_off;
Pekka Enbergb28a02d2006-01-08 01:00:37 -08002749 slabp->s_mem = objp + colour_off;
Ravikiran G Thirumalai5b74ada2006-04-10 22:52:53 -07002750 slabp->nodeid = nodeid;
Marcin Slusarze51bfd02008-02-10 11:21:54 +01002751 slabp->free = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002752 return slabp;
2753}
2754
2755static inline kmem_bufctl_t *slab_bufctl(struct slab *slabp)
2756{
Pekka Enbergb28a02d2006-01-08 01:00:37 -08002757 return (kmem_bufctl_t *) (slabp + 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002758}
2759
Pekka Enberg343e0d72006-02-01 03:05:50 -08002760static void cache_init_objs(struct kmem_cache *cachep,
Christoph Lametera35afb82007-05-16 22:10:57 -07002761 struct slab *slabp)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002762{
2763 int i;
2764
2765 for (i = 0; i < cachep->num; i++) {
Pekka Enberg8fea4e92006-03-22 00:08:10 -08002766 void *objp = index_to_obj(cachep, slabp, i);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002767#if DEBUG
2768 /* need to poison the objs? */
2769 if (cachep->flags & SLAB_POISON)
2770 poison_obj(cachep, objp, POISON_FREE);
2771 if (cachep->flags & SLAB_STORE_USER)
2772 *dbg_userword(cachep, objp) = NULL;
2773
2774 if (cachep->flags & SLAB_RED_ZONE) {
2775 *dbg_redzone1(cachep, objp) = RED_INACTIVE;
2776 *dbg_redzone2(cachep, objp) = RED_INACTIVE;
2777 }
2778 /*
Andrew Mortona737b3e2006-03-22 00:08:11 -08002779 * Constructors are not allowed to allocate memory from the same
2780 * cache which they are a constructor for. Otherwise, deadlock.
2781 * They must also be threaded.
Linus Torvalds1da177e2005-04-16 15:20:36 -07002782 */
2783 if (cachep->ctor && !(cachep->flags & SLAB_POISON))
Alexey Dobriyan51cc5062008-07-25 19:45:34 -07002784 cachep->ctor(objp + obj_offset(cachep));
Linus Torvalds1da177e2005-04-16 15:20:36 -07002785
2786 if (cachep->flags & SLAB_RED_ZONE) {
2787 if (*dbg_redzone2(cachep, objp) != RED_INACTIVE)
2788 slab_error(cachep, "constructor overwrote the"
Pekka Enbergb28a02d2006-01-08 01:00:37 -08002789 " end of an object");
Linus Torvalds1da177e2005-04-16 15:20:36 -07002790 if (*dbg_redzone1(cachep, objp) != RED_INACTIVE)
2791 slab_error(cachep, "constructor overwrote the"
Pekka Enbergb28a02d2006-01-08 01:00:37 -08002792 " start of an object");
Linus Torvalds1da177e2005-04-16 15:20:36 -07002793 }
Andrew Mortona737b3e2006-03-22 00:08:11 -08002794 if ((cachep->buffer_size % PAGE_SIZE) == 0 &&
2795 OFF_SLAB(cachep) && cachep->flags & SLAB_POISON)
Pekka Enbergb28a02d2006-01-08 01:00:37 -08002796 kernel_map_pages(virt_to_page(objp),
Manfred Spraul3dafccf2006-02-01 03:05:42 -08002797 cachep->buffer_size / PAGE_SIZE, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002798#else
2799 if (cachep->ctor)
Alexey Dobriyan51cc5062008-07-25 19:45:34 -07002800 cachep->ctor(objp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002801#endif
Pekka Enbergb28a02d2006-01-08 01:00:37 -08002802 slab_bufctl(slabp)[i] = i + 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002803 }
Pekka Enbergb28a02d2006-01-08 01:00:37 -08002804 slab_bufctl(slabp)[i - 1] = BUFCTL_END;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002805}
2806
Pekka Enberg343e0d72006-02-01 03:05:50 -08002807static void kmem_flagcheck(struct kmem_cache *cachep, gfp_t flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002808{
Christoph Lameter4b51d662007-02-10 01:43:10 -08002809 if (CONFIG_ZONE_DMA_FLAG) {
2810 if (flags & GFP_DMA)
2811 BUG_ON(!(cachep->gfpflags & GFP_DMA));
2812 else
2813 BUG_ON(cachep->gfpflags & GFP_DMA);
2814 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002815}
2816
Andrew Mortona737b3e2006-03-22 00:08:11 -08002817static void *slab_get_obj(struct kmem_cache *cachep, struct slab *slabp,
2818 int nodeid)
Matthew Dobson78d382d2006-02-01 03:05:47 -08002819{
Pekka Enberg8fea4e92006-03-22 00:08:10 -08002820 void *objp = index_to_obj(cachep, slabp, slabp->free);
Matthew Dobson78d382d2006-02-01 03:05:47 -08002821 kmem_bufctl_t next;
2822
2823 slabp->inuse++;
2824 next = slab_bufctl(slabp)[slabp->free];
2825#if DEBUG
2826 slab_bufctl(slabp)[slabp->free] = BUFCTL_FREE;
2827 WARN_ON(slabp->nodeid != nodeid);
2828#endif
2829 slabp->free = next;
2830
2831 return objp;
2832}
2833
Andrew Mortona737b3e2006-03-22 00:08:11 -08002834static void slab_put_obj(struct kmem_cache *cachep, struct slab *slabp,
2835 void *objp, int nodeid)
Matthew Dobson78d382d2006-02-01 03:05:47 -08002836{
Pekka Enberg8fea4e92006-03-22 00:08:10 -08002837 unsigned int objnr = obj_to_index(cachep, slabp, objp);
Matthew Dobson78d382d2006-02-01 03:05:47 -08002838
2839#if DEBUG
2840 /* Verify that the slab belongs to the intended node */
2841 WARN_ON(slabp->nodeid != nodeid);
2842
Al Viro871751e2006-03-25 03:06:39 -08002843 if (slab_bufctl(slabp)[objnr] + 1 <= SLAB_LIMIT + 1) {
Matthew Dobson78d382d2006-02-01 03:05:47 -08002844 printk(KERN_ERR "slab: double free detected in cache "
Andrew Mortona737b3e2006-03-22 00:08:11 -08002845 "'%s', objp %p\n", cachep->name, objp);
Matthew Dobson78d382d2006-02-01 03:05:47 -08002846 BUG();
2847 }
2848#endif
2849 slab_bufctl(slabp)[objnr] = slabp->free;
2850 slabp->free = objnr;
2851 slabp->inuse--;
2852}
2853
Pekka Enberg47768742006-06-23 02:03:07 -07002854/*
2855 * Map pages beginning at addr to the given cache and slab. This is required
2856 * for the slab allocator to be able to lookup the cache and slab of a
Nick Pigginccd35fb2011-01-07 17:49:17 +11002857 * virtual address for kfree, ksize, and slab debugging.
Pekka Enberg47768742006-06-23 02:03:07 -07002858 */
2859static void slab_map_pages(struct kmem_cache *cache, struct slab *slab,
2860 void *addr)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002861{
Pekka Enberg47768742006-06-23 02:03:07 -07002862 int nr_pages;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002863 struct page *page;
2864
Pekka Enberg47768742006-06-23 02:03:07 -07002865 page = virt_to_page(addr);
Nick Piggin84097512006-03-22 00:08:34 -08002866
Pekka Enberg47768742006-06-23 02:03:07 -07002867 nr_pages = 1;
Nick Piggin84097512006-03-22 00:08:34 -08002868 if (likely(!PageCompound(page)))
Pekka Enberg47768742006-06-23 02:03:07 -07002869 nr_pages <<= cache->gfporder;
2870
Linus Torvalds1da177e2005-04-16 15:20:36 -07002871 do {
Pekka Enberg47768742006-06-23 02:03:07 -07002872 page_set_cache(page, cache);
2873 page_set_slab(page, slab);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002874 page++;
Pekka Enberg47768742006-06-23 02:03:07 -07002875 } while (--nr_pages);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002876}
2877
2878/*
2879 * Grow (by 1) the number of slabs within a cache. This is called by
2880 * kmem_cache_alloc() when there are no active objs left in a cache.
2881 */
Christoph Lameter3c517a62006-12-06 20:33:29 -08002882static int cache_grow(struct kmem_cache *cachep,
2883 gfp_t flags, int nodeid, void *objp)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002884{
Pekka Enbergb28a02d2006-01-08 01:00:37 -08002885 struct slab *slabp;
Pekka Enbergb28a02d2006-01-08 01:00:37 -08002886 size_t offset;
2887 gfp_t local_flags;
Christoph Lametere498be72005-09-09 13:03:32 -07002888 struct kmem_list3 *l3;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002889
Andrew Mortona737b3e2006-03-22 00:08:11 -08002890 /*
2891 * Be lazy and only check for valid flags here, keeping it out of the
2892 * critical path in kmem_cache_alloc().
Linus Torvalds1da177e2005-04-16 15:20:36 -07002893 */
Christoph Lameter6cb06222007-10-16 01:25:41 -07002894 BUG_ON(flags & GFP_SLAB_BUG_MASK);
2895 local_flags = flags & (GFP_CONSTRAINT_MASK|GFP_RECLAIM_MASK);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002896
Ravikiran G Thirumalai2e1217c2006-02-04 23:27:56 -08002897 /* Take the l3 list lock to change the colour_next on this node */
Linus Torvalds1da177e2005-04-16 15:20:36 -07002898 check_irq_off();
Ravikiran G Thirumalai2e1217c2006-02-04 23:27:56 -08002899 l3 = cachep->nodelists[nodeid];
2900 spin_lock(&l3->list_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002901
2902 /* Get colour for the slab, and cal the next value. */
Ravikiran G Thirumalai2e1217c2006-02-04 23:27:56 -08002903 offset = l3->colour_next;
2904 l3->colour_next++;
2905 if (l3->colour_next >= cachep->colour)
2906 l3->colour_next = 0;
2907 spin_unlock(&l3->list_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002908
Ravikiran G Thirumalai2e1217c2006-02-04 23:27:56 -08002909 offset *= cachep->colour_off;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002910
2911 if (local_flags & __GFP_WAIT)
2912 local_irq_enable();
2913
2914 /*
2915 * The test for missing atomic flag is performed here, rather than
2916 * the more obvious place, simply to reduce the critical path length
2917 * in kmem_cache_alloc(). If a caller is seriously mis-behaving they
2918 * will eventually be caught here (where it matters).
2919 */
2920 kmem_flagcheck(cachep, flags);
2921
Andrew Mortona737b3e2006-03-22 00:08:11 -08002922 /*
2923 * Get mem for the objs. Attempt to allocate a physical page from
2924 * 'nodeid'.
Christoph Lametere498be72005-09-09 13:03:32 -07002925 */
Christoph Lameter3c517a62006-12-06 20:33:29 -08002926 if (!objp)
Andrew Mortonb8c1c5d2007-07-24 12:02:40 -07002927 objp = kmem_getpages(cachep, local_flags, nodeid);
Andrew Mortona737b3e2006-03-22 00:08:11 -08002928 if (!objp)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002929 goto failed;
2930
2931 /* Get slab management. */
Christoph Lameter3c517a62006-12-06 20:33:29 -08002932 slabp = alloc_slabmgmt(cachep, objp, offset,
Christoph Lameter6cb06222007-10-16 01:25:41 -07002933 local_flags & ~GFP_CONSTRAINT_MASK, nodeid);
Andrew Mortona737b3e2006-03-22 00:08:11 -08002934 if (!slabp)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002935 goto opps1;
2936
Pekka Enberg47768742006-06-23 02:03:07 -07002937 slab_map_pages(cachep, slabp, objp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002938
Christoph Lametera35afb82007-05-16 22:10:57 -07002939 cache_init_objs(cachep, slabp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002940
2941 if (local_flags & __GFP_WAIT)
2942 local_irq_disable();
2943 check_irq_off();
Christoph Lametere498be72005-09-09 13:03:32 -07002944 spin_lock(&l3->list_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002945
2946 /* Make slab active. */
Christoph Lametere498be72005-09-09 13:03:32 -07002947 list_add_tail(&slabp->list, &(l3->slabs_free));
Linus Torvalds1da177e2005-04-16 15:20:36 -07002948 STATS_INC_GROWN(cachep);
Christoph Lametere498be72005-09-09 13:03:32 -07002949 l3->free_objects += cachep->num;
2950 spin_unlock(&l3->list_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002951 return 1;
Andrew Mortona737b3e2006-03-22 00:08:11 -08002952opps1:
Linus Torvalds1da177e2005-04-16 15:20:36 -07002953 kmem_freepages(cachep, objp);
Andrew Mortona737b3e2006-03-22 00:08:11 -08002954failed:
Linus Torvalds1da177e2005-04-16 15:20:36 -07002955 if (local_flags & __GFP_WAIT)
2956 local_irq_disable();
2957 return 0;
2958}
2959
2960#if DEBUG
2961
2962/*
2963 * Perform extra freeing checks:
2964 * - detect bad pointers.
2965 * - POISON/RED_ZONE checking
Linus Torvalds1da177e2005-04-16 15:20:36 -07002966 */
2967static void kfree_debugcheck(const void *objp)
2968{
Linus Torvalds1da177e2005-04-16 15:20:36 -07002969 if (!virt_addr_valid(objp)) {
2970 printk(KERN_ERR "kfree_debugcheck: out of range ptr %lxh.\n",
Pekka Enbergb28a02d2006-01-08 01:00:37 -08002971 (unsigned long)objp);
2972 BUG();
Linus Torvalds1da177e2005-04-16 15:20:36 -07002973 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002974}
2975
Pekka Enberg58ce1fd2006-06-23 02:03:24 -07002976static inline void verify_redzone_free(struct kmem_cache *cache, void *obj)
2977{
David Woodhouseb46b8f12007-05-08 00:22:59 -07002978 unsigned long long redzone1, redzone2;
Pekka Enberg58ce1fd2006-06-23 02:03:24 -07002979
2980 redzone1 = *dbg_redzone1(cache, obj);
2981 redzone2 = *dbg_redzone2(cache, obj);
2982
2983 /*
2984 * Redzone is ok.
2985 */
2986 if (redzone1 == RED_ACTIVE && redzone2 == RED_ACTIVE)
2987 return;
2988
2989 if (redzone1 == RED_INACTIVE && redzone2 == RED_INACTIVE)
2990 slab_error(cache, "double free detected");
2991 else
2992 slab_error(cache, "memory outside object was overwritten");
2993
David Woodhouseb46b8f12007-05-08 00:22:59 -07002994 printk(KERN_ERR "%p: redzone 1:0x%llx, redzone 2:0x%llx.\n",
Pekka Enberg58ce1fd2006-06-23 02:03:24 -07002995 obj, redzone1, redzone2);
2996}
2997
Pekka Enberg343e0d72006-02-01 03:05:50 -08002998static void *cache_free_debugcheck(struct kmem_cache *cachep, void *objp,
Pekka Enbergb28a02d2006-01-08 01:00:37 -08002999 void *caller)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003000{
3001 struct page *page;
3002 unsigned int objnr;
3003 struct slab *slabp;
3004
Matthew Wilcox80cbd912007-11-29 12:05:13 -07003005 BUG_ON(virt_to_cache(objp) != cachep);
3006
Manfred Spraul3dafccf2006-02-01 03:05:42 -08003007 objp -= obj_offset(cachep);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003008 kfree_debugcheck(objp);
Christoph Lameterb49af682007-05-06 14:49:41 -07003009 page = virt_to_head_page(objp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003010
Pekka Enberg065d41c2005-11-13 16:06:46 -08003011 slabp = page_get_slab(page);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003012
3013 if (cachep->flags & SLAB_RED_ZONE) {
Pekka Enberg58ce1fd2006-06-23 02:03:24 -07003014 verify_redzone_free(cachep, objp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003015 *dbg_redzone1(cachep, objp) = RED_INACTIVE;
3016 *dbg_redzone2(cachep, objp) = RED_INACTIVE;
3017 }
3018 if (cachep->flags & SLAB_STORE_USER)
3019 *dbg_userword(cachep, objp) = caller;
3020
Pekka Enberg8fea4e92006-03-22 00:08:10 -08003021 objnr = obj_to_index(cachep, slabp, objp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003022
3023 BUG_ON(objnr >= cachep->num);
Pekka Enberg8fea4e92006-03-22 00:08:10 -08003024 BUG_ON(objp != index_to_obj(cachep, slabp, objnr));
Linus Torvalds1da177e2005-04-16 15:20:36 -07003025
Al Viro871751e2006-03-25 03:06:39 -08003026#ifdef CONFIG_DEBUG_SLAB_LEAK
3027 slab_bufctl(slabp)[objnr] = BUFCTL_FREE;
3028#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07003029 if (cachep->flags & SLAB_POISON) {
3030#ifdef CONFIG_DEBUG_PAGEALLOC
Andrew Mortona737b3e2006-03-22 00:08:11 -08003031 if ((cachep->buffer_size % PAGE_SIZE)==0 && OFF_SLAB(cachep)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07003032 store_stackinfo(cachep, objp, (unsigned long)caller);
Pekka Enbergb28a02d2006-01-08 01:00:37 -08003033 kernel_map_pages(virt_to_page(objp),
Manfred Spraul3dafccf2006-02-01 03:05:42 -08003034 cachep->buffer_size / PAGE_SIZE, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003035 } else {
3036 poison_obj(cachep, objp, POISON_FREE);
3037 }
3038#else
3039 poison_obj(cachep, objp, POISON_FREE);
3040#endif
3041 }
3042 return objp;
3043}
3044
Pekka Enberg343e0d72006-02-01 03:05:50 -08003045static void check_slabp(struct kmem_cache *cachep, struct slab *slabp)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003046{
3047 kmem_bufctl_t i;
3048 int entries = 0;
Pekka Enbergb28a02d2006-01-08 01:00:37 -08003049
Linus Torvalds1da177e2005-04-16 15:20:36 -07003050 /* Check slab's freelist to see if this obj is there. */
3051 for (i = slabp->free; i != BUFCTL_END; i = slab_bufctl(slabp)[i]) {
3052 entries++;
3053 if (entries > cachep->num || i >= cachep->num)
3054 goto bad;
3055 }
3056 if (entries != cachep->num - slabp->inuse) {
Andrew Mortona737b3e2006-03-22 00:08:11 -08003057bad:
3058 printk(KERN_ERR "slab: Internal list corruption detected in "
Dave Jonesface37f2011-11-15 15:03:52 -08003059 "cache '%s'(%d), slabp %p(%d). Tainted(%s). Hexdump:\n",
3060 cachep->name, cachep->num, slabp, slabp->inuse,
3061 print_tainted());
Sebastian Andrzej Siewiorfdde6ab2011-07-29 18:22:13 +02003062 print_hex_dump(KERN_ERR, "", DUMP_PREFIX_OFFSET, 16, 1, slabp,
3063 sizeof(*slabp) + cachep->num * sizeof(kmem_bufctl_t),
3064 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003065 BUG();
3066 }
3067}
3068#else
3069#define kfree_debugcheck(x) do { } while(0)
3070#define cache_free_debugcheck(x,objp,z) (objp)
3071#define check_slabp(x,y) do { } while(0)
3072#endif
3073
Pekka Enberg343e0d72006-02-01 03:05:50 -08003074static void *cache_alloc_refill(struct kmem_cache *cachep, gfp_t flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003075{
3076 int batchcount;
3077 struct kmem_list3 *l3;
3078 struct array_cache *ac;
Pekka Enberg1ca4cb22006-10-06 00:43:52 -07003079 int node;
3080
Andrew Mortona737b3e2006-03-22 00:08:11 -08003081retry:
Joe Korty6d2144d2008-03-05 15:04:59 -08003082 check_irq_off();
Lee Schermerhorn7d6e6d02010-05-26 14:45:03 -07003083 node = numa_mem_id();
Joe Korty6d2144d2008-03-05 15:04:59 -08003084 ac = cpu_cache_get(cachep);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003085 batchcount = ac->batchcount;
3086 if (!ac->touched && batchcount > BATCHREFILL_LIMIT) {
Andrew Mortona737b3e2006-03-22 00:08:11 -08003087 /*
3088 * If there was little recent activity on this cache, then
3089 * perform only a partial refill. Otherwise we could generate
3090 * refill bouncing.
Linus Torvalds1da177e2005-04-16 15:20:36 -07003091 */
3092 batchcount = BATCHREFILL_LIMIT;
3093 }
Pekka Enberg1ca4cb22006-10-06 00:43:52 -07003094 l3 = cachep->nodelists[node];
Linus Torvalds1da177e2005-04-16 15:20:36 -07003095
Christoph Lametere498be72005-09-09 13:03:32 -07003096 BUG_ON(ac->avail > 0 || !l3);
3097 spin_lock(&l3->list_lock);
3098
Christoph Lameter3ded1752006-03-25 03:06:44 -08003099 /* See if we can refill from the shared array */
Nick Piggin44b57f12010-01-27 22:27:40 +11003100 if (l3->shared && transfer_objects(ac, l3->shared, batchcount)) {
3101 l3->shared->touched = 1;
Christoph Lameter3ded1752006-03-25 03:06:44 -08003102 goto alloc_done;
Nick Piggin44b57f12010-01-27 22:27:40 +11003103 }
Christoph Lameter3ded1752006-03-25 03:06:44 -08003104
Linus Torvalds1da177e2005-04-16 15:20:36 -07003105 while (batchcount > 0) {
3106 struct list_head *entry;
3107 struct slab *slabp;
3108 /* Get slab alloc is to come from. */
3109 entry = l3->slabs_partial.next;
3110 if (entry == &l3->slabs_partial) {
3111 l3->free_touched = 1;
3112 entry = l3->slabs_free.next;
3113 if (entry == &l3->slabs_free)
3114 goto must_grow;
3115 }
3116
3117 slabp = list_entry(entry, struct slab, list);
3118 check_slabp(cachep, slabp);
3119 check_spinlock_acquired(cachep);
Pekka Enberg714b81712007-05-06 14:49:03 -07003120
3121 /*
3122 * The slab was either on partial or free list so
3123 * there must be at least one object available for
3124 * allocation.
3125 */
roel kluin249b9f32008-10-29 17:18:07 -04003126 BUG_ON(slabp->inuse >= cachep->num);
Pekka Enberg714b81712007-05-06 14:49:03 -07003127
Linus Torvalds1da177e2005-04-16 15:20:36 -07003128 while (slabp->inuse < cachep->num && batchcount--) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07003129 STATS_INC_ALLOCED(cachep);
3130 STATS_INC_ACTIVE(cachep);
3131 STATS_SET_HIGH(cachep);
3132
Matthew Dobson78d382d2006-02-01 03:05:47 -08003133 ac->entry[ac->avail++] = slab_get_obj(cachep, slabp,
Pekka Enberg1ca4cb22006-10-06 00:43:52 -07003134 node);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003135 }
3136 check_slabp(cachep, slabp);
3137
3138 /* move slabp to correct slabp list: */
3139 list_del(&slabp->list);
3140 if (slabp->free == BUFCTL_END)
3141 list_add(&slabp->list, &l3->slabs_full);
3142 else
3143 list_add(&slabp->list, &l3->slabs_partial);
3144 }
3145
Andrew Mortona737b3e2006-03-22 00:08:11 -08003146must_grow:
Linus Torvalds1da177e2005-04-16 15:20:36 -07003147 l3->free_objects -= ac->avail;
Andrew Mortona737b3e2006-03-22 00:08:11 -08003148alloc_done:
Christoph Lametere498be72005-09-09 13:03:32 -07003149 spin_unlock(&l3->list_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003150
3151 if (unlikely(!ac->avail)) {
3152 int x;
Christoph Lameter3c517a62006-12-06 20:33:29 -08003153 x = cache_grow(cachep, flags | GFP_THISNODE, node, NULL);
Christoph Lametere498be72005-09-09 13:03:32 -07003154
Andrew Mortona737b3e2006-03-22 00:08:11 -08003155 /* cache_grow can reenable interrupts, then ac could change. */
Pekka Enberg9a2dba4b2006-02-01 03:05:49 -08003156 ac = cpu_cache_get(cachep);
Andrew Mortona737b3e2006-03-22 00:08:11 -08003157 if (!x && ac->avail == 0) /* no objects in sight? abort */
Linus Torvalds1da177e2005-04-16 15:20:36 -07003158 return NULL;
3159
Andrew Mortona737b3e2006-03-22 00:08:11 -08003160 if (!ac->avail) /* objects refilled by interrupt? */
Linus Torvalds1da177e2005-04-16 15:20:36 -07003161 goto retry;
3162 }
3163 ac->touched = 1;
Christoph Lametere498be72005-09-09 13:03:32 -07003164 return ac->entry[--ac->avail];
Linus Torvalds1da177e2005-04-16 15:20:36 -07003165}
3166
Andrew Mortona737b3e2006-03-22 00:08:11 -08003167static inline void cache_alloc_debugcheck_before(struct kmem_cache *cachep,
3168 gfp_t flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003169{
3170 might_sleep_if(flags & __GFP_WAIT);
3171#if DEBUG
3172 kmem_flagcheck(cachep, flags);
3173#endif
3174}
3175
3176#if DEBUG
Andrew Mortona737b3e2006-03-22 00:08:11 -08003177static void *cache_alloc_debugcheck_after(struct kmem_cache *cachep,
3178 gfp_t flags, void *objp, void *caller)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003179{
Pekka Enbergb28a02d2006-01-08 01:00:37 -08003180 if (!objp)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003181 return objp;
Pekka Enbergb28a02d2006-01-08 01:00:37 -08003182 if (cachep->flags & SLAB_POISON) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07003183#ifdef CONFIG_DEBUG_PAGEALLOC
Manfred Spraul3dafccf2006-02-01 03:05:42 -08003184 if ((cachep->buffer_size % PAGE_SIZE) == 0 && OFF_SLAB(cachep))
Pekka Enbergb28a02d2006-01-08 01:00:37 -08003185 kernel_map_pages(virt_to_page(objp),
Manfred Spraul3dafccf2006-02-01 03:05:42 -08003186 cachep->buffer_size / PAGE_SIZE, 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003187 else
3188 check_poison_obj(cachep, objp);
3189#else
3190 check_poison_obj(cachep, objp);
3191#endif
3192 poison_obj(cachep, objp, POISON_INUSE);
3193 }
3194 if (cachep->flags & SLAB_STORE_USER)
3195 *dbg_userword(cachep, objp) = caller;
3196
3197 if (cachep->flags & SLAB_RED_ZONE) {
Andrew Mortona737b3e2006-03-22 00:08:11 -08003198 if (*dbg_redzone1(cachep, objp) != RED_INACTIVE ||
3199 *dbg_redzone2(cachep, objp) != RED_INACTIVE) {
3200 slab_error(cachep, "double free, or memory outside"
3201 " object was overwritten");
Pekka Enbergb28a02d2006-01-08 01:00:37 -08003202 printk(KERN_ERR
David Woodhouseb46b8f12007-05-08 00:22:59 -07003203 "%p: redzone 1:0x%llx, redzone 2:0x%llx\n",
Andrew Mortona737b3e2006-03-22 00:08:11 -08003204 objp, *dbg_redzone1(cachep, objp),
3205 *dbg_redzone2(cachep, objp));
Linus Torvalds1da177e2005-04-16 15:20:36 -07003206 }
3207 *dbg_redzone1(cachep, objp) = RED_ACTIVE;
3208 *dbg_redzone2(cachep, objp) = RED_ACTIVE;
3209 }
Al Viro871751e2006-03-25 03:06:39 -08003210#ifdef CONFIG_DEBUG_SLAB_LEAK
3211 {
3212 struct slab *slabp;
3213 unsigned objnr;
3214
Christoph Lameterb49af682007-05-06 14:49:41 -07003215 slabp = page_get_slab(virt_to_head_page(objp));
Al Viro871751e2006-03-25 03:06:39 -08003216 objnr = (unsigned)(objp - slabp->s_mem) / cachep->buffer_size;
3217 slab_bufctl(slabp)[objnr] = BUFCTL_ACTIVE;
3218 }
3219#endif
Manfred Spraul3dafccf2006-02-01 03:05:42 -08003220 objp += obj_offset(cachep);
Christoph Lameter4f104932007-05-06 14:50:17 -07003221 if (cachep->ctor && cachep->flags & SLAB_POISON)
Alexey Dobriyan51cc5062008-07-25 19:45:34 -07003222 cachep->ctor(objp);
Tetsuo Handa7ea466f2011-07-21 09:42:45 +09003223 if (ARCH_SLAB_MINALIGN &&
3224 ((unsigned long)objp & (ARCH_SLAB_MINALIGN-1))) {
Kevin Hilmana44b56d2006-12-06 20:32:11 -08003225 printk(KERN_ERR "0x%p: not aligned to ARCH_SLAB_MINALIGN=%d\n",
Hugh Dickinsc2251502011-07-11 13:35:08 -07003226 objp, (int)ARCH_SLAB_MINALIGN);
Kevin Hilmana44b56d2006-12-06 20:32:11 -08003227 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07003228 return objp;
3229}
3230#else
3231#define cache_alloc_debugcheck_after(a,b,objp,d) (objp)
3232#endif
3233
Akinobu Mita773ff602008-12-23 19:37:01 +09003234static bool slab_should_failslab(struct kmem_cache *cachep, gfp_t flags)
Akinobu Mita8a8b6502006-12-08 02:39:44 -08003235{
3236 if (cachep == &cache_cache)
Akinobu Mita773ff602008-12-23 19:37:01 +09003237 return false;
Akinobu Mita8a8b6502006-12-08 02:39:44 -08003238
Dmitry Monakhov4c13dd32010-02-26 09:36:12 +03003239 return should_failslab(obj_size(cachep), flags, cachep->flags);
Akinobu Mita8a8b6502006-12-08 02:39:44 -08003240}
3241
Pekka Enberg343e0d72006-02-01 03:05:50 -08003242static inline void *____cache_alloc(struct kmem_cache *cachep, gfp_t flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003243{
Pekka Enbergb28a02d2006-01-08 01:00:37 -08003244 void *objp;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003245 struct array_cache *ac;
3246
Alok N Kataria5c382302005-09-27 21:45:46 -07003247 check_irq_off();
Akinobu Mita8a8b6502006-12-08 02:39:44 -08003248
Pekka Enberg9a2dba4b2006-02-01 03:05:49 -08003249 ac = cpu_cache_get(cachep);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003250 if (likely(ac->avail)) {
3251 STATS_INC_ALLOCHIT(cachep);
3252 ac->touched = 1;
Christoph Lametere498be72005-09-09 13:03:32 -07003253 objp = ac->entry[--ac->avail];
Linus Torvalds1da177e2005-04-16 15:20:36 -07003254 } else {
3255 STATS_INC_ALLOCMISS(cachep);
3256 objp = cache_alloc_refill(cachep, flags);
J. R. Okajimaddbf2e82009-12-02 16:55:50 +09003257 /*
3258 * the 'ac' may be updated by cache_alloc_refill(),
3259 * and kmemleak_erase() requires its correct value.
3260 */
3261 ac = cpu_cache_get(cachep);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003262 }
Catalin Marinasd5cff632009-06-11 13:22:40 +01003263 /*
3264 * To avoid a false negative, if an object that is in one of the
3265 * per-CPU caches is leaked, we need to make sure kmemleak doesn't
3266 * treat the array pointers as a reference to the object.
3267 */
J. R. Okajimaf3d8b532009-12-02 16:55:49 +09003268 if (objp)
3269 kmemleak_erase(&ac->entry[ac->avail]);
Alok N Kataria5c382302005-09-27 21:45:46 -07003270 return objp;
3271}
3272
Christoph Lametere498be72005-09-09 13:03:32 -07003273#ifdef CONFIG_NUMA
3274/*
Paul Jacksonb2455392006-03-24 03:16:12 -08003275 * Try allocating on another node if PF_SPREAD_SLAB|PF_MEMPOLICY.
Paul Jacksonc61afb12006-03-24 03:16:08 -08003276 *
3277 * If we are in_interrupt, then process context, including cpusets and
3278 * mempolicy, may not apply and should not be used for allocation policy.
3279 */
3280static void *alternate_node_alloc(struct kmem_cache *cachep, gfp_t flags)
3281{
3282 int nid_alloc, nid_here;
3283
Christoph Lameter765c4502006-09-27 01:50:08 -07003284 if (in_interrupt() || (flags & __GFP_THISNODE))
Paul Jacksonc61afb12006-03-24 03:16:08 -08003285 return NULL;
Lee Schermerhorn7d6e6d02010-05-26 14:45:03 -07003286 nid_alloc = nid_here = numa_mem_id();
Miao Xiec0ff7452010-05-24 14:32:08 -07003287 get_mems_allowed();
Paul Jacksonc61afb12006-03-24 03:16:08 -08003288 if (cpuset_do_slab_mem_spread() && (cachep->flags & SLAB_MEM_SPREAD))
Jack Steiner6adef3e2010-05-26 14:42:49 -07003289 nid_alloc = cpuset_slab_spread_node();
Paul Jacksonc61afb12006-03-24 03:16:08 -08003290 else if (current->mempolicy)
3291 nid_alloc = slab_node(current->mempolicy);
Miao Xiec0ff7452010-05-24 14:32:08 -07003292 put_mems_allowed();
Paul Jacksonc61afb12006-03-24 03:16:08 -08003293 if (nid_alloc != nid_here)
Christoph Hellwig8b98c162006-12-06 20:32:30 -08003294 return ____cache_alloc_node(cachep, flags, nid_alloc);
Paul Jacksonc61afb12006-03-24 03:16:08 -08003295 return NULL;
3296}
3297
3298/*
Christoph Lameter765c4502006-09-27 01:50:08 -07003299 * Fallback function if there was no memory available and no objects on a
Christoph Lameter3c517a62006-12-06 20:33:29 -08003300 * certain node and fall back is permitted. First we scan all the
3301 * available nodelists for available objects. If that fails then we
3302 * perform an allocation without specifying a node. This allows the page
3303 * allocator to do its reclaim / fallback magic. We then insert the
3304 * slab into the proper nodelist and then allocate from it.
Christoph Lameter765c4502006-09-27 01:50:08 -07003305 */
Pekka Enberg8c8cc2c2007-02-10 01:42:53 -08003306static void *fallback_alloc(struct kmem_cache *cache, gfp_t flags)
Christoph Lameter765c4502006-09-27 01:50:08 -07003307{
Pekka Enberg8c8cc2c2007-02-10 01:42:53 -08003308 struct zonelist *zonelist;
3309 gfp_t local_flags;
Mel Gormandd1a2392008-04-28 02:12:17 -07003310 struct zoneref *z;
Mel Gorman54a6eb52008-04-28 02:12:16 -07003311 struct zone *zone;
3312 enum zone_type high_zoneidx = gfp_zone(flags);
Christoph Lameter765c4502006-09-27 01:50:08 -07003313 void *obj = NULL;
Christoph Lameter3c517a62006-12-06 20:33:29 -08003314 int nid;
Pekka Enberg8c8cc2c2007-02-10 01:42:53 -08003315
3316 if (flags & __GFP_THISNODE)
3317 return NULL;
3318
Miao Xiec0ff7452010-05-24 14:32:08 -07003319 get_mems_allowed();
Mel Gorman0e884602008-04-28 02:12:14 -07003320 zonelist = node_zonelist(slab_node(current->mempolicy), flags);
Christoph Lameter6cb06222007-10-16 01:25:41 -07003321 local_flags = flags & (GFP_CONSTRAINT_MASK|GFP_RECLAIM_MASK);
Christoph Lameter765c4502006-09-27 01:50:08 -07003322
Christoph Lameter3c517a62006-12-06 20:33:29 -08003323retry:
3324 /*
3325 * Look through allowed nodes for objects available
3326 * from existing per node queues.
3327 */
Mel Gorman54a6eb52008-04-28 02:12:16 -07003328 for_each_zone_zonelist(zone, z, zonelist, high_zoneidx) {
3329 nid = zone_to_nid(zone);
Christoph Lameteraedb0eb2006-10-21 10:24:16 -07003330
Mel Gorman54a6eb52008-04-28 02:12:16 -07003331 if (cpuset_zone_allowed_hardwall(zone, flags) &&
Christoph Lameter3c517a62006-12-06 20:33:29 -08003332 cache->nodelists[nid] &&
Christoph Lameter481c5342008-06-21 16:46:35 -07003333 cache->nodelists[nid]->free_objects) {
Christoph Lameter3c517a62006-12-06 20:33:29 -08003334 obj = ____cache_alloc_node(cache,
3335 flags | GFP_THISNODE, nid);
Christoph Lameter481c5342008-06-21 16:46:35 -07003336 if (obj)
3337 break;
3338 }
Christoph Lameter3c517a62006-12-06 20:33:29 -08003339 }
3340
Christoph Lametercfce6602007-05-06 14:50:17 -07003341 if (!obj) {
Christoph Lameter3c517a62006-12-06 20:33:29 -08003342 /*
3343 * This allocation will be performed within the constraints
3344 * of the current cpuset / memory policy requirements.
3345 * We may trigger various forms of reclaim on the allowed
3346 * set and go into memory reserves if necessary.
3347 */
Christoph Lameterdd47ea72006-12-13 00:34:11 -08003348 if (local_flags & __GFP_WAIT)
3349 local_irq_enable();
3350 kmem_flagcheck(cache, flags);
Lee Schermerhorn7d6e6d02010-05-26 14:45:03 -07003351 obj = kmem_getpages(cache, local_flags, numa_mem_id());
Christoph Lameterdd47ea72006-12-13 00:34:11 -08003352 if (local_flags & __GFP_WAIT)
3353 local_irq_disable();
Christoph Lameter3c517a62006-12-06 20:33:29 -08003354 if (obj) {
3355 /*
3356 * Insert into the appropriate per node queues
3357 */
3358 nid = page_to_nid(virt_to_page(obj));
3359 if (cache_grow(cache, flags, nid, obj)) {
3360 obj = ____cache_alloc_node(cache,
3361 flags | GFP_THISNODE, nid);
3362 if (!obj)
3363 /*
3364 * Another processor may allocate the
3365 * objects in the slab since we are
3366 * not holding any locks.
3367 */
3368 goto retry;
3369 } else {
Hugh Dickinsb6a60452007-01-05 16:36:36 -08003370 /* cache_grow already freed obj */
Christoph Lameter3c517a62006-12-06 20:33:29 -08003371 obj = NULL;
3372 }
3373 }
Christoph Lameteraedb0eb2006-10-21 10:24:16 -07003374 }
Miao Xiec0ff7452010-05-24 14:32:08 -07003375 put_mems_allowed();
Christoph Lameter765c4502006-09-27 01:50:08 -07003376 return obj;
3377}
3378
3379/*
Christoph Lametere498be72005-09-09 13:03:32 -07003380 * A interface to enable slab creation on nodeid
Linus Torvalds1da177e2005-04-16 15:20:36 -07003381 */
Christoph Hellwig8b98c162006-12-06 20:32:30 -08003382static void *____cache_alloc_node(struct kmem_cache *cachep, gfp_t flags,
Andrew Mortona737b3e2006-03-22 00:08:11 -08003383 int nodeid)
Christoph Lametere498be72005-09-09 13:03:32 -07003384{
3385 struct list_head *entry;
Pekka Enbergb28a02d2006-01-08 01:00:37 -08003386 struct slab *slabp;
3387 struct kmem_list3 *l3;
3388 void *obj;
Pekka Enbergb28a02d2006-01-08 01:00:37 -08003389 int x;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003390
Pekka Enbergb28a02d2006-01-08 01:00:37 -08003391 l3 = cachep->nodelists[nodeid];
3392 BUG_ON(!l3);
Christoph Lametere498be72005-09-09 13:03:32 -07003393
Andrew Mortona737b3e2006-03-22 00:08:11 -08003394retry:
Ravikiran G Thirumalaica3b9b92006-02-04 23:27:58 -08003395 check_irq_off();
Pekka Enbergb28a02d2006-01-08 01:00:37 -08003396 spin_lock(&l3->list_lock);
3397 entry = l3->slabs_partial.next;
3398 if (entry == &l3->slabs_partial) {
3399 l3->free_touched = 1;
3400 entry = l3->slabs_free.next;
3401 if (entry == &l3->slabs_free)
3402 goto must_grow;
3403 }
Christoph Lametere498be72005-09-09 13:03:32 -07003404
Pekka Enbergb28a02d2006-01-08 01:00:37 -08003405 slabp = list_entry(entry, struct slab, list);
3406 check_spinlock_acquired_node(cachep, nodeid);
3407 check_slabp(cachep, slabp);
Christoph Lametere498be72005-09-09 13:03:32 -07003408
Pekka Enbergb28a02d2006-01-08 01:00:37 -08003409 STATS_INC_NODEALLOCS(cachep);
3410 STATS_INC_ACTIVE(cachep);
3411 STATS_SET_HIGH(cachep);
Christoph Lametere498be72005-09-09 13:03:32 -07003412
Pekka Enbergb28a02d2006-01-08 01:00:37 -08003413 BUG_ON(slabp->inuse == cachep->num);
Christoph Lametere498be72005-09-09 13:03:32 -07003414
Matthew Dobson78d382d2006-02-01 03:05:47 -08003415 obj = slab_get_obj(cachep, slabp, nodeid);
Pekka Enbergb28a02d2006-01-08 01:00:37 -08003416 check_slabp(cachep, slabp);
3417 l3->free_objects--;
3418 /* move slabp to correct slabp list: */
3419 list_del(&slabp->list);
Christoph Lametere498be72005-09-09 13:03:32 -07003420
Andrew Mortona737b3e2006-03-22 00:08:11 -08003421 if (slabp->free == BUFCTL_END)
Pekka Enbergb28a02d2006-01-08 01:00:37 -08003422 list_add(&slabp->list, &l3->slabs_full);
Andrew Mortona737b3e2006-03-22 00:08:11 -08003423 else
Pekka Enbergb28a02d2006-01-08 01:00:37 -08003424 list_add(&slabp->list, &l3->slabs_partial);
Christoph Lametere498be72005-09-09 13:03:32 -07003425
Pekka Enbergb28a02d2006-01-08 01:00:37 -08003426 spin_unlock(&l3->list_lock);
3427 goto done;
Christoph Lametere498be72005-09-09 13:03:32 -07003428
Andrew Mortona737b3e2006-03-22 00:08:11 -08003429must_grow:
Pekka Enbergb28a02d2006-01-08 01:00:37 -08003430 spin_unlock(&l3->list_lock);
Christoph Lameter3c517a62006-12-06 20:33:29 -08003431 x = cache_grow(cachep, flags | GFP_THISNODE, nodeid, NULL);
Christoph Lameter765c4502006-09-27 01:50:08 -07003432 if (x)
3433 goto retry;
Christoph Lametere498be72005-09-09 13:03:32 -07003434
Pekka Enberg8c8cc2c2007-02-10 01:42:53 -08003435 return fallback_alloc(cachep, flags);
Christoph Lameter765c4502006-09-27 01:50:08 -07003436
Andrew Mortona737b3e2006-03-22 00:08:11 -08003437done:
Pekka Enbergb28a02d2006-01-08 01:00:37 -08003438 return obj;
Christoph Lametere498be72005-09-09 13:03:32 -07003439}
Pekka Enberg8c8cc2c2007-02-10 01:42:53 -08003440
3441/**
3442 * kmem_cache_alloc_node - Allocate an object on the specified node
3443 * @cachep: The cache to allocate from.
3444 * @flags: See kmalloc().
3445 * @nodeid: node number of the target node.
3446 * @caller: return address of caller, used for debug information
3447 *
3448 * Identical to kmem_cache_alloc but it will allocate memory on the given
3449 * node, which can improve the performance for cpu bound structures.
3450 *
3451 * Fallback to other node is possible if __GFP_THISNODE is not set.
3452 */
3453static __always_inline void *
3454__cache_alloc_node(struct kmem_cache *cachep, gfp_t flags, int nodeid,
3455 void *caller)
3456{
3457 unsigned long save_flags;
3458 void *ptr;
Lee Schermerhorn7d6e6d02010-05-26 14:45:03 -07003459 int slab_node = numa_mem_id();
Pekka Enberg8c8cc2c2007-02-10 01:42:53 -08003460
Benjamin Herrenschmidtdcce2842009-06-18 13:24:12 +10003461 flags &= gfp_allowed_mask;
Pekka Enberg7e85ee02009-06-12 14:03:06 +03003462
Nick Piggincf40bd12009-01-21 08:12:39 +01003463 lockdep_trace_alloc(flags);
3464
Akinobu Mita773ff602008-12-23 19:37:01 +09003465 if (slab_should_failslab(cachep, flags))
Akinobu Mita824ebef2007-05-06 14:49:58 -07003466 return NULL;
3467
Pekka Enberg8c8cc2c2007-02-10 01:42:53 -08003468 cache_alloc_debugcheck_before(cachep, flags);
3469 local_irq_save(save_flags);
3470
Andrew Mortoneacbbae2011-07-28 13:59:49 -07003471 if (nodeid == NUMA_NO_NODE)
Lee Schermerhorn7d6e6d02010-05-26 14:45:03 -07003472 nodeid = slab_node;
Pekka Enberg8c8cc2c2007-02-10 01:42:53 -08003473
3474 if (unlikely(!cachep->nodelists[nodeid])) {
3475 /* Node not bootstrapped yet */
3476 ptr = fallback_alloc(cachep, flags);
3477 goto out;
3478 }
3479
Lee Schermerhorn7d6e6d02010-05-26 14:45:03 -07003480 if (nodeid == slab_node) {
Pekka Enberg8c8cc2c2007-02-10 01:42:53 -08003481 /*
3482 * Use the locally cached objects if possible.
3483 * However ____cache_alloc does not allow fallback
3484 * to other nodes. It may fail while we still have
3485 * objects on other nodes available.
3486 */
3487 ptr = ____cache_alloc(cachep, flags);
3488 if (ptr)
3489 goto out;
3490 }
3491 /* ___cache_alloc_node can fall back to other nodes */
3492 ptr = ____cache_alloc_node(cachep, flags, nodeid);
3493 out:
3494 local_irq_restore(save_flags);
3495 ptr = cache_alloc_debugcheck_after(cachep, flags, ptr, caller);
Catalin Marinasd5cff632009-06-11 13:22:40 +01003496 kmemleak_alloc_recursive(ptr, obj_size(cachep), 1, cachep->flags,
3497 flags);
Pekka Enberg8c8cc2c2007-02-10 01:42:53 -08003498
Pekka Enbergc175eea2008-05-09 20:35:53 +02003499 if (likely(ptr))
3500 kmemcheck_slab_alloc(cachep, flags, ptr, obj_size(cachep));
3501
Christoph Lameterd07dbea2007-07-17 04:03:23 -07003502 if (unlikely((flags & __GFP_ZERO) && ptr))
3503 memset(ptr, 0, obj_size(cachep));
3504
Pekka Enberg8c8cc2c2007-02-10 01:42:53 -08003505 return ptr;
3506}
3507
3508static __always_inline void *
3509__do_cache_alloc(struct kmem_cache *cache, gfp_t flags)
3510{
3511 void *objp;
3512
3513 if (unlikely(current->flags & (PF_SPREAD_SLAB | PF_MEMPOLICY))) {
3514 objp = alternate_node_alloc(cache, flags);
3515 if (objp)
3516 goto out;
3517 }
3518 objp = ____cache_alloc(cache, flags);
3519
3520 /*
3521 * We may just have run out of memory on the local node.
3522 * ____cache_alloc_node() knows how to locate memory on other nodes
3523 */
Lee Schermerhorn7d6e6d02010-05-26 14:45:03 -07003524 if (!objp)
3525 objp = ____cache_alloc_node(cache, flags, numa_mem_id());
Pekka Enberg8c8cc2c2007-02-10 01:42:53 -08003526
3527 out:
3528 return objp;
3529}
3530#else
3531
3532static __always_inline void *
3533__do_cache_alloc(struct kmem_cache *cachep, gfp_t flags)
3534{
3535 return ____cache_alloc(cachep, flags);
3536}
3537
3538#endif /* CONFIG_NUMA */
3539
3540static __always_inline void *
3541__cache_alloc(struct kmem_cache *cachep, gfp_t flags, void *caller)
3542{
3543 unsigned long save_flags;
3544 void *objp;
3545
Benjamin Herrenschmidtdcce2842009-06-18 13:24:12 +10003546 flags &= gfp_allowed_mask;
Pekka Enberg7e85ee02009-06-12 14:03:06 +03003547
Nick Piggincf40bd12009-01-21 08:12:39 +01003548 lockdep_trace_alloc(flags);
3549
Akinobu Mita773ff602008-12-23 19:37:01 +09003550 if (slab_should_failslab(cachep, flags))
Akinobu Mita824ebef2007-05-06 14:49:58 -07003551 return NULL;
3552
Pekka Enberg8c8cc2c2007-02-10 01:42:53 -08003553 cache_alloc_debugcheck_before(cachep, flags);
3554 local_irq_save(save_flags);
3555 objp = __do_cache_alloc(cachep, flags);
3556 local_irq_restore(save_flags);
3557 objp = cache_alloc_debugcheck_after(cachep, flags, objp, caller);
Catalin Marinasd5cff632009-06-11 13:22:40 +01003558 kmemleak_alloc_recursive(objp, obj_size(cachep), 1, cachep->flags,
3559 flags);
Pekka Enberg8c8cc2c2007-02-10 01:42:53 -08003560 prefetchw(objp);
3561
Pekka Enbergc175eea2008-05-09 20:35:53 +02003562 if (likely(objp))
3563 kmemcheck_slab_alloc(cachep, flags, objp, obj_size(cachep));
3564
Christoph Lameterd07dbea2007-07-17 04:03:23 -07003565 if (unlikely((flags & __GFP_ZERO) && objp))
3566 memset(objp, 0, obj_size(cachep));
3567
Pekka Enberg8c8cc2c2007-02-10 01:42:53 -08003568 return objp;
3569}
Christoph Lametere498be72005-09-09 13:03:32 -07003570
3571/*
3572 * Caller needs to acquire correct kmem_list's list_lock
3573 */
Pekka Enberg343e0d72006-02-01 03:05:50 -08003574static void free_block(struct kmem_cache *cachep, void **objpp, int nr_objects,
Pekka Enbergb28a02d2006-01-08 01:00:37 -08003575 int node)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003576{
3577 int i;
Christoph Lametere498be72005-09-09 13:03:32 -07003578 struct kmem_list3 *l3;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003579
3580 for (i = 0; i < nr_objects; i++) {
3581 void *objp = objpp[i];
3582 struct slab *slabp;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003583
Pekka Enberg6ed5eb2212006-02-01 03:05:49 -08003584 slabp = virt_to_slab(objp);
Christoph Lameterff694162005-09-22 21:44:02 -07003585 l3 = cachep->nodelists[node];
Linus Torvalds1da177e2005-04-16 15:20:36 -07003586 list_del(&slabp->list);
Christoph Lameterff694162005-09-22 21:44:02 -07003587 check_spinlock_acquired_node(cachep, node);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003588 check_slabp(cachep, slabp);
Matthew Dobson78d382d2006-02-01 03:05:47 -08003589 slab_put_obj(cachep, slabp, objp, node);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003590 STATS_DEC_ACTIVE(cachep);
Christoph Lametere498be72005-09-09 13:03:32 -07003591 l3->free_objects++;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003592 check_slabp(cachep, slabp);
3593
3594 /* fixup slab chains */
3595 if (slabp->inuse == 0) {
Christoph Lametere498be72005-09-09 13:03:32 -07003596 if (l3->free_objects > l3->free_limit) {
3597 l3->free_objects -= cachep->num;
Ravikiran G Thirumalaie5ac9c52006-09-25 23:31:34 -07003598 /* No need to drop any previously held
3599 * lock here, even if we have a off-slab slab
3600 * descriptor it is guaranteed to come from
3601 * a different cache, refer to comments before
3602 * alloc_slabmgmt.
3603 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07003604 slab_destroy(cachep, slabp);
3605 } else {
Christoph Lametere498be72005-09-09 13:03:32 -07003606 list_add(&slabp->list, &l3->slabs_free);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003607 }
3608 } else {
3609 /* Unconditionally move a slab to the end of the
3610 * partial list on free - maximum time for the
3611 * other objects to be freed, too.
3612 */
Christoph Lametere498be72005-09-09 13:03:32 -07003613 list_add_tail(&slabp->list, &l3->slabs_partial);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003614 }
3615 }
3616}
3617
Pekka Enberg343e0d72006-02-01 03:05:50 -08003618static void cache_flusharray(struct kmem_cache *cachep, struct array_cache *ac)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003619{
3620 int batchcount;
Christoph Lametere498be72005-09-09 13:03:32 -07003621 struct kmem_list3 *l3;
Lee Schermerhorn7d6e6d02010-05-26 14:45:03 -07003622 int node = numa_mem_id();
Linus Torvalds1da177e2005-04-16 15:20:36 -07003623
3624 batchcount = ac->batchcount;
3625#if DEBUG
3626 BUG_ON(!batchcount || batchcount > ac->avail);
3627#endif
3628 check_irq_off();
Christoph Lameterff694162005-09-22 21:44:02 -07003629 l3 = cachep->nodelists[node];
Ingo Molnar873623d2006-07-13 14:44:38 +02003630 spin_lock(&l3->list_lock);
Christoph Lametere498be72005-09-09 13:03:32 -07003631 if (l3->shared) {
3632 struct array_cache *shared_array = l3->shared;
Pekka Enbergb28a02d2006-01-08 01:00:37 -08003633 int max = shared_array->limit - shared_array->avail;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003634 if (max) {
3635 if (batchcount > max)
3636 batchcount = max;
Christoph Lametere498be72005-09-09 13:03:32 -07003637 memcpy(&(shared_array->entry[shared_array->avail]),
Pekka Enbergb28a02d2006-01-08 01:00:37 -08003638 ac->entry, sizeof(void *) * batchcount);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003639 shared_array->avail += batchcount;
3640 goto free_done;
3641 }
3642 }
3643
Christoph Lameterff694162005-09-22 21:44:02 -07003644 free_block(cachep, ac->entry, batchcount, node);
Andrew Mortona737b3e2006-03-22 00:08:11 -08003645free_done:
Linus Torvalds1da177e2005-04-16 15:20:36 -07003646#if STATS
3647 {
3648 int i = 0;
3649 struct list_head *p;
3650
Christoph Lametere498be72005-09-09 13:03:32 -07003651 p = l3->slabs_free.next;
3652 while (p != &(l3->slabs_free)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07003653 struct slab *slabp;
3654
3655 slabp = list_entry(p, struct slab, list);
3656 BUG_ON(slabp->inuse);
3657
3658 i++;
3659 p = p->next;
3660 }
3661 STATS_SET_FREEABLE(cachep, i);
3662 }
3663#endif
Christoph Lametere498be72005-09-09 13:03:32 -07003664 spin_unlock(&l3->list_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003665 ac->avail -= batchcount;
Andrew Mortona737b3e2006-03-22 00:08:11 -08003666 memmove(ac->entry, &(ac->entry[batchcount]), sizeof(void *)*ac->avail);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003667}
3668
3669/*
Andrew Mortona737b3e2006-03-22 00:08:11 -08003670 * Release an obj back to its cache. If the obj has a constructed state, it must
3671 * be in this state _before_ it is released. Called with disabled ints.
Linus Torvalds1da177e2005-04-16 15:20:36 -07003672 */
Suleiman Souhlala947eb92011-06-02 00:16:42 -07003673static inline void __cache_free(struct kmem_cache *cachep, void *objp,
3674 void *caller)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003675{
Pekka Enberg9a2dba4b2006-02-01 03:05:49 -08003676 struct array_cache *ac = cpu_cache_get(cachep);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003677
3678 check_irq_off();
Catalin Marinasd5cff632009-06-11 13:22:40 +01003679 kmemleak_free_recursive(objp, cachep->flags);
Suleiman Souhlala947eb92011-06-02 00:16:42 -07003680 objp = cache_free_debugcheck(cachep, objp, caller);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003681
Pekka Enbergc175eea2008-05-09 20:35:53 +02003682 kmemcheck_slab_free(cachep, objp, obj_size(cachep));
3683
Siddha, Suresh B1807a1a2007-08-22 14:01:49 -07003684 /*
3685 * Skip calling cache_free_alien() when the platform is not numa.
3686 * This will avoid cache misses that happen while accessing slabp (which
3687 * is per page memory reference) to get nodeid. Instead use a global
3688 * variable to skip the call, which is mostly likely to be present in
3689 * the cache.
3690 */
Mel Gormanb6e68bc2009-06-16 15:32:16 -07003691 if (nr_online_nodes > 1 && cache_free_alien(cachep, objp))
Pekka Enberg729bd0b2006-06-23 02:03:05 -07003692 return;
Christoph Lametere498be72005-09-09 13:03:32 -07003693
Linus Torvalds1da177e2005-04-16 15:20:36 -07003694 if (likely(ac->avail < ac->limit)) {
3695 STATS_INC_FREEHIT(cachep);
Christoph Lametere498be72005-09-09 13:03:32 -07003696 ac->entry[ac->avail++] = objp;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003697 return;
3698 } else {
3699 STATS_INC_FREEMISS(cachep);
3700 cache_flusharray(cachep, ac);
Christoph Lametere498be72005-09-09 13:03:32 -07003701 ac->entry[ac->avail++] = objp;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003702 }
3703}
3704
3705/**
3706 * kmem_cache_alloc - Allocate an object
3707 * @cachep: The cache to allocate from.
3708 * @flags: See kmalloc().
3709 *
3710 * Allocate an object from this cache. The flags are only relevant
3711 * if the cache has no available objects.
3712 */
Pekka Enberg343e0d72006-02-01 03:05:50 -08003713void *kmem_cache_alloc(struct kmem_cache *cachep, gfp_t flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003714{
Eduard - Gabriel Munteanu36555752008-08-10 20:14:05 +03003715 void *ret = __cache_alloc(cachep, flags, __builtin_return_address(0));
3716
Eduard - Gabriel Munteanuca2b84cb2009-03-23 15:12:24 +02003717 trace_kmem_cache_alloc(_RET_IP_, ret,
3718 obj_size(cachep), cachep->buffer_size, flags);
Eduard - Gabriel Munteanu36555752008-08-10 20:14:05 +03003719
3720 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003721}
3722EXPORT_SYMBOL(kmem_cache_alloc);
3723
Li Zefan0f24f122009-12-11 15:45:30 +08003724#ifdef CONFIG_TRACING
Steven Rostedt85beb582010-11-24 16:23:34 -05003725void *
3726kmem_cache_alloc_trace(size_t size, struct kmem_cache *cachep, gfp_t flags)
Eduard - Gabriel Munteanu36555752008-08-10 20:14:05 +03003727{
Steven Rostedt85beb582010-11-24 16:23:34 -05003728 void *ret;
3729
3730 ret = __cache_alloc(cachep, flags, __builtin_return_address(0));
3731
3732 trace_kmalloc(_RET_IP_, ret,
3733 size, slab_buffer_size(cachep), flags);
3734 return ret;
Eduard - Gabriel Munteanu36555752008-08-10 20:14:05 +03003735}
Steven Rostedt85beb582010-11-24 16:23:34 -05003736EXPORT_SYMBOL(kmem_cache_alloc_trace);
Eduard - Gabriel Munteanu36555752008-08-10 20:14:05 +03003737#endif
3738
Linus Torvalds1da177e2005-04-16 15:20:36 -07003739#ifdef CONFIG_NUMA
Christoph Hellwig8b98c162006-12-06 20:32:30 -08003740void *kmem_cache_alloc_node(struct kmem_cache *cachep, gfp_t flags, int nodeid)
3741{
Eduard - Gabriel Munteanu36555752008-08-10 20:14:05 +03003742 void *ret = __cache_alloc_node(cachep, flags, nodeid,
3743 __builtin_return_address(0));
3744
Eduard - Gabriel Munteanuca2b84cb2009-03-23 15:12:24 +02003745 trace_kmem_cache_alloc_node(_RET_IP_, ret,
3746 obj_size(cachep), cachep->buffer_size,
3747 flags, nodeid);
Eduard - Gabriel Munteanu36555752008-08-10 20:14:05 +03003748
3749 return ret;
Christoph Hellwig8b98c162006-12-06 20:32:30 -08003750}
Linus Torvalds1da177e2005-04-16 15:20:36 -07003751EXPORT_SYMBOL(kmem_cache_alloc_node);
3752
Li Zefan0f24f122009-12-11 15:45:30 +08003753#ifdef CONFIG_TRACING
Steven Rostedt85beb582010-11-24 16:23:34 -05003754void *kmem_cache_alloc_node_trace(size_t size,
3755 struct kmem_cache *cachep,
3756 gfp_t flags,
3757 int nodeid)
Eduard - Gabriel Munteanu36555752008-08-10 20:14:05 +03003758{
Steven Rostedt85beb582010-11-24 16:23:34 -05003759 void *ret;
3760
3761 ret = __cache_alloc_node(cachep, flags, nodeid,
Eduard - Gabriel Munteanu36555752008-08-10 20:14:05 +03003762 __builtin_return_address(0));
Steven Rostedt85beb582010-11-24 16:23:34 -05003763 trace_kmalloc_node(_RET_IP_, ret,
3764 size, slab_buffer_size(cachep),
3765 flags, nodeid);
3766 return ret;
Eduard - Gabriel Munteanu36555752008-08-10 20:14:05 +03003767}
Steven Rostedt85beb582010-11-24 16:23:34 -05003768EXPORT_SYMBOL(kmem_cache_alloc_node_trace);
Eduard - Gabriel Munteanu36555752008-08-10 20:14:05 +03003769#endif
3770
Christoph Hellwig8b98c162006-12-06 20:32:30 -08003771static __always_inline void *
3772__do_kmalloc_node(size_t size, gfp_t flags, int node, void *caller)
Manfred Spraul97e2bde2005-05-01 08:58:38 -07003773{
Pekka Enberg343e0d72006-02-01 03:05:50 -08003774 struct kmem_cache *cachep;
Manfred Spraul97e2bde2005-05-01 08:58:38 -07003775
3776 cachep = kmem_find_general_cachep(size, flags);
Christoph Lameter6cb8f912007-07-17 04:03:22 -07003777 if (unlikely(ZERO_OR_NULL_PTR(cachep)))
3778 return cachep;
Steven Rostedt85beb582010-11-24 16:23:34 -05003779 return kmem_cache_alloc_node_trace(size, cachep, flags, node);
Manfred Spraul97e2bde2005-05-01 08:58:38 -07003780}
Christoph Hellwig8b98c162006-12-06 20:32:30 -08003781
Li Zefan0bb38a52009-12-11 15:45:50 +08003782#if defined(CONFIG_DEBUG_SLAB) || defined(CONFIG_TRACING)
Christoph Hellwig8b98c162006-12-06 20:32:30 -08003783void *__kmalloc_node(size_t size, gfp_t flags, int node)
3784{
3785 return __do_kmalloc_node(size, flags, node,
3786 __builtin_return_address(0));
3787}
Christoph Hellwigdbe5e692006-09-25 23:31:36 -07003788EXPORT_SYMBOL(__kmalloc_node);
Christoph Hellwig8b98c162006-12-06 20:32:30 -08003789
3790void *__kmalloc_node_track_caller(size_t size, gfp_t flags,
Eduard - Gabriel Munteanuce71e272008-08-19 20:43:25 +03003791 int node, unsigned long caller)
Christoph Hellwig8b98c162006-12-06 20:32:30 -08003792{
Eduard - Gabriel Munteanuce71e272008-08-19 20:43:25 +03003793 return __do_kmalloc_node(size, flags, node, (void *)caller);
Christoph Hellwig8b98c162006-12-06 20:32:30 -08003794}
3795EXPORT_SYMBOL(__kmalloc_node_track_caller);
3796#else
3797void *__kmalloc_node(size_t size, gfp_t flags, int node)
3798{
3799 return __do_kmalloc_node(size, flags, node, NULL);
3800}
3801EXPORT_SYMBOL(__kmalloc_node);
Li Zefan0bb38a52009-12-11 15:45:50 +08003802#endif /* CONFIG_DEBUG_SLAB || CONFIG_TRACING */
Christoph Hellwig8b98c162006-12-06 20:32:30 -08003803#endif /* CONFIG_NUMA */
Linus Torvalds1da177e2005-04-16 15:20:36 -07003804
3805/**
Paul Drynoff800590f2006-06-23 02:03:48 -07003806 * __do_kmalloc - allocate memory
Linus Torvalds1da177e2005-04-16 15:20:36 -07003807 * @size: how many bytes of memory are required.
Paul Drynoff800590f2006-06-23 02:03:48 -07003808 * @flags: the type of memory to allocate (see kmalloc).
Randy Dunlap911851e2006-03-22 00:08:14 -08003809 * @caller: function caller for debug tracking of the caller
Linus Torvalds1da177e2005-04-16 15:20:36 -07003810 */
Pekka Enberg7fd6b142006-02-01 03:05:52 -08003811static __always_inline void *__do_kmalloc(size_t size, gfp_t flags,
3812 void *caller)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003813{
Pekka Enberg343e0d72006-02-01 03:05:50 -08003814 struct kmem_cache *cachep;
Eduard - Gabriel Munteanu36555752008-08-10 20:14:05 +03003815 void *ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003816
Manfred Spraul97e2bde2005-05-01 08:58:38 -07003817 /* If you want to save a few bytes .text space: replace
3818 * __ with kmem_.
3819 * Then kmalloc uses the uninlined functions instead of the inline
3820 * functions.
3821 */
3822 cachep = __find_general_cachep(size, flags);
Linus Torvaldsa5c96d82007-07-19 13:17:15 -07003823 if (unlikely(ZERO_OR_NULL_PTR(cachep)))
3824 return cachep;
Eduard - Gabriel Munteanu36555752008-08-10 20:14:05 +03003825 ret = __cache_alloc(cachep, flags, caller);
3826
Eduard - Gabriel Munteanuca2b84cb2009-03-23 15:12:24 +02003827 trace_kmalloc((unsigned long) caller, ret,
3828 size, cachep->buffer_size, flags);
Eduard - Gabriel Munteanu36555752008-08-10 20:14:05 +03003829
3830 return ret;
Pekka Enberg7fd6b142006-02-01 03:05:52 -08003831}
3832
Pekka Enberg7fd6b142006-02-01 03:05:52 -08003833
Li Zefan0bb38a52009-12-11 15:45:50 +08003834#if defined(CONFIG_DEBUG_SLAB) || defined(CONFIG_TRACING)
Pekka Enberg7fd6b142006-02-01 03:05:52 -08003835void *__kmalloc(size_t size, gfp_t flags)
3836{
Al Viro871751e2006-03-25 03:06:39 -08003837 return __do_kmalloc(size, flags, __builtin_return_address(0));
Linus Torvalds1da177e2005-04-16 15:20:36 -07003838}
3839EXPORT_SYMBOL(__kmalloc);
3840
Eduard - Gabriel Munteanuce71e272008-08-19 20:43:25 +03003841void *__kmalloc_track_caller(size_t size, gfp_t flags, unsigned long caller)
Pekka Enberg7fd6b142006-02-01 03:05:52 -08003842{
Eduard - Gabriel Munteanuce71e272008-08-19 20:43:25 +03003843 return __do_kmalloc(size, flags, (void *)caller);
Pekka Enberg7fd6b142006-02-01 03:05:52 -08003844}
3845EXPORT_SYMBOL(__kmalloc_track_caller);
Christoph Hellwig1d2c8ee2006-10-04 02:15:25 -07003846
3847#else
3848void *__kmalloc(size_t size, gfp_t flags)
3849{
3850 return __do_kmalloc(size, flags, NULL);
3851}
3852EXPORT_SYMBOL(__kmalloc);
Pekka Enberg7fd6b142006-02-01 03:05:52 -08003853#endif
3854
Linus Torvalds1da177e2005-04-16 15:20:36 -07003855/**
3856 * kmem_cache_free - Deallocate an object
3857 * @cachep: The cache the allocation was from.
3858 * @objp: The previously allocated object.
3859 *
3860 * Free an object which was previously allocated from this
3861 * cache.
3862 */
Pekka Enberg343e0d72006-02-01 03:05:50 -08003863void kmem_cache_free(struct kmem_cache *cachep, void *objp)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003864{
3865 unsigned long flags;
3866
3867 local_irq_save(flags);
Ingo Molnar898552c2007-02-10 01:44:57 -08003868 debug_check_no_locks_freed(objp, obj_size(cachep));
Thomas Gleixner3ac7fe52008-04-30 00:55:01 -07003869 if (!(cachep->flags & SLAB_DEBUG_OBJECTS))
3870 debug_check_no_obj_freed(objp, obj_size(cachep));
Suleiman Souhlala947eb92011-06-02 00:16:42 -07003871 __cache_free(cachep, objp, __builtin_return_address(0));
Linus Torvalds1da177e2005-04-16 15:20:36 -07003872 local_irq_restore(flags);
Eduard - Gabriel Munteanu36555752008-08-10 20:14:05 +03003873
Eduard - Gabriel Munteanuca2b84cb2009-03-23 15:12:24 +02003874 trace_kmem_cache_free(_RET_IP_, objp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003875}
3876EXPORT_SYMBOL(kmem_cache_free);
3877
3878/**
Linus Torvalds1da177e2005-04-16 15:20:36 -07003879 * kfree - free previously allocated memory
3880 * @objp: pointer returned by kmalloc.
3881 *
Pekka Enberg80e93ef2005-09-09 13:10:16 -07003882 * If @objp is NULL, no operation is performed.
3883 *
Linus Torvalds1da177e2005-04-16 15:20:36 -07003884 * Don't free memory not originally allocated by kmalloc()
3885 * or you will run into trouble.
3886 */
3887void kfree(const void *objp)
3888{
Pekka Enberg343e0d72006-02-01 03:05:50 -08003889 struct kmem_cache *c;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003890 unsigned long flags;
3891
Pekka Enberg2121db72009-03-25 11:05:57 +02003892 trace_kfree(_RET_IP_, objp);
3893
Christoph Lameter6cb8f912007-07-17 04:03:22 -07003894 if (unlikely(ZERO_OR_NULL_PTR(objp)))
Linus Torvalds1da177e2005-04-16 15:20:36 -07003895 return;
3896 local_irq_save(flags);
3897 kfree_debugcheck(objp);
Pekka Enberg6ed5eb2212006-02-01 03:05:49 -08003898 c = virt_to_cache(objp);
Ingo Molnarf9b84042006-06-27 02:54:49 -07003899 debug_check_no_locks_freed(objp, obj_size(c));
Thomas Gleixner3ac7fe52008-04-30 00:55:01 -07003900 debug_check_no_obj_freed(objp, obj_size(c));
Suleiman Souhlala947eb92011-06-02 00:16:42 -07003901 __cache_free(c, (void *)objp, __builtin_return_address(0));
Linus Torvalds1da177e2005-04-16 15:20:36 -07003902 local_irq_restore(flags);
3903}
3904EXPORT_SYMBOL(kfree);
3905
Pekka Enberg343e0d72006-02-01 03:05:50 -08003906unsigned int kmem_cache_size(struct kmem_cache *cachep)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003907{
Manfred Spraul3dafccf2006-02-01 03:05:42 -08003908 return obj_size(cachep);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003909}
3910EXPORT_SYMBOL(kmem_cache_size);
3911
Christoph Lametere498be72005-09-09 13:03:32 -07003912/*
Simon Arlott183ff222007-10-20 01:27:18 +02003913 * This initializes kmem_list3 or resizes various caches for all nodes.
Christoph Lametere498be72005-09-09 13:03:32 -07003914 */
Pekka Enberg83b519e2009-06-10 19:40:04 +03003915static int alloc_kmemlist(struct kmem_cache *cachep, gfp_t gfp)
Christoph Lametere498be72005-09-09 13:03:32 -07003916{
3917 int node;
3918 struct kmem_list3 *l3;
Christoph Lametercafeb022006-03-25 03:06:46 -08003919 struct array_cache *new_shared;
Paul Menage3395ee02006-12-06 20:32:16 -08003920 struct array_cache **new_alien = NULL;
Christoph Lametere498be72005-09-09 13:03:32 -07003921
Mel Gorman9c09a952008-01-24 05:49:54 -08003922 for_each_online_node(node) {
Christoph Lametercafeb022006-03-25 03:06:46 -08003923
Paul Menage3395ee02006-12-06 20:32:16 -08003924 if (use_alien_caches) {
Pekka Enberg83b519e2009-06-10 19:40:04 +03003925 new_alien = alloc_alien_cache(node, cachep->limit, gfp);
Paul Menage3395ee02006-12-06 20:32:16 -08003926 if (!new_alien)
3927 goto fail;
3928 }
Christoph Lametercafeb022006-03-25 03:06:46 -08003929
Eric Dumazet63109842007-05-06 14:49:28 -07003930 new_shared = NULL;
3931 if (cachep->shared) {
3932 new_shared = alloc_arraycache(node,
Christoph Lameter0718dc22006-03-25 03:06:47 -08003933 cachep->shared*cachep->batchcount,
Pekka Enberg83b519e2009-06-10 19:40:04 +03003934 0xbaadf00d, gfp);
Eric Dumazet63109842007-05-06 14:49:28 -07003935 if (!new_shared) {
3936 free_alien_cache(new_alien);
3937 goto fail;
3938 }
Christoph Lameter0718dc22006-03-25 03:06:47 -08003939 }
Christoph Lametercafeb022006-03-25 03:06:46 -08003940
Andrew Mortona737b3e2006-03-22 00:08:11 -08003941 l3 = cachep->nodelists[node];
3942 if (l3) {
Christoph Lametercafeb022006-03-25 03:06:46 -08003943 struct array_cache *shared = l3->shared;
3944
Christoph Lametere498be72005-09-09 13:03:32 -07003945 spin_lock_irq(&l3->list_lock);
3946
Christoph Lametercafeb022006-03-25 03:06:46 -08003947 if (shared)
Christoph Lameter0718dc22006-03-25 03:06:47 -08003948 free_block(cachep, shared->entry,
3949 shared->avail, node);
Christoph Lametere498be72005-09-09 13:03:32 -07003950
Christoph Lametercafeb022006-03-25 03:06:46 -08003951 l3->shared = new_shared;
3952 if (!l3->alien) {
Christoph Lametere498be72005-09-09 13:03:32 -07003953 l3->alien = new_alien;
3954 new_alien = NULL;
3955 }
Pekka Enbergb28a02d2006-01-08 01:00:37 -08003956 l3->free_limit = (1 + nr_cpus_node(node)) *
Andrew Mortona737b3e2006-03-22 00:08:11 -08003957 cachep->batchcount + cachep->num;
Christoph Lametere498be72005-09-09 13:03:32 -07003958 spin_unlock_irq(&l3->list_lock);
Christoph Lametercafeb022006-03-25 03:06:46 -08003959 kfree(shared);
Christoph Lametere498be72005-09-09 13:03:32 -07003960 free_alien_cache(new_alien);
3961 continue;
3962 }
Pekka Enberg83b519e2009-06-10 19:40:04 +03003963 l3 = kmalloc_node(sizeof(struct kmem_list3), gfp, node);
Christoph Lameter0718dc22006-03-25 03:06:47 -08003964 if (!l3) {
3965 free_alien_cache(new_alien);
3966 kfree(new_shared);
Christoph Lametere498be72005-09-09 13:03:32 -07003967 goto fail;
Christoph Lameter0718dc22006-03-25 03:06:47 -08003968 }
Christoph Lametere498be72005-09-09 13:03:32 -07003969
3970 kmem_list3_init(l3);
3971 l3->next_reap = jiffies + REAPTIMEOUT_LIST3 +
Andrew Mortona737b3e2006-03-22 00:08:11 -08003972 ((unsigned long)cachep) % REAPTIMEOUT_LIST3;
Christoph Lametercafeb022006-03-25 03:06:46 -08003973 l3->shared = new_shared;
Christoph Lametere498be72005-09-09 13:03:32 -07003974 l3->alien = new_alien;
Pekka Enbergb28a02d2006-01-08 01:00:37 -08003975 l3->free_limit = (1 + nr_cpus_node(node)) *
Andrew Mortona737b3e2006-03-22 00:08:11 -08003976 cachep->batchcount + cachep->num;
Christoph Lametere498be72005-09-09 13:03:32 -07003977 cachep->nodelists[node] = l3;
3978 }
Christoph Lametercafeb022006-03-25 03:06:46 -08003979 return 0;
Christoph Lameter0718dc22006-03-25 03:06:47 -08003980
Andrew Mortona737b3e2006-03-22 00:08:11 -08003981fail:
Christoph Lameter0718dc22006-03-25 03:06:47 -08003982 if (!cachep->next.next) {
3983 /* Cache is not active yet. Roll back what we did */
3984 node--;
3985 while (node >= 0) {
3986 if (cachep->nodelists[node]) {
3987 l3 = cachep->nodelists[node];
3988
3989 kfree(l3->shared);
3990 free_alien_cache(l3->alien);
3991 kfree(l3);
3992 cachep->nodelists[node] = NULL;
3993 }
3994 node--;
3995 }
3996 }
Christoph Lametercafeb022006-03-25 03:06:46 -08003997 return -ENOMEM;
Christoph Lametere498be72005-09-09 13:03:32 -07003998}
3999
Linus Torvalds1da177e2005-04-16 15:20:36 -07004000struct ccupdate_struct {
Pekka Enberg343e0d72006-02-01 03:05:50 -08004001 struct kmem_cache *cachep;
Eric Dumazetacfe7d72011-07-25 08:55:42 +02004002 struct array_cache *new[0];
Linus Torvalds1da177e2005-04-16 15:20:36 -07004003};
4004
4005static void do_ccupdate_local(void *info)
4006{
Andrew Mortona737b3e2006-03-22 00:08:11 -08004007 struct ccupdate_struct *new = info;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004008 struct array_cache *old;
4009
4010 check_irq_off();
Pekka Enberg9a2dba4b2006-02-01 03:05:49 -08004011 old = cpu_cache_get(new->cachep);
Christoph Lametere498be72005-09-09 13:03:32 -07004012
Linus Torvalds1da177e2005-04-16 15:20:36 -07004013 new->cachep->array[smp_processor_id()] = new->new[smp_processor_id()];
4014 new->new[smp_processor_id()] = old;
4015}
4016
Ravikiran G Thirumalaib5d8ca72006-03-22 00:08:12 -08004017/* Always called with the cache_chain_mutex held */
Andrew Mortona737b3e2006-03-22 00:08:11 -08004018static int do_tune_cpucache(struct kmem_cache *cachep, int limit,
Pekka Enberg83b519e2009-06-10 19:40:04 +03004019 int batchcount, int shared, gfp_t gfp)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004020{
Siddha, Suresh Bd2e7b7d2006-09-25 23:31:47 -07004021 struct ccupdate_struct *new;
Christoph Lameter2ed3a4e2006-09-25 23:31:38 -07004022 int i;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004023
Eric Dumazetacfe7d72011-07-25 08:55:42 +02004024 new = kzalloc(sizeof(*new) + nr_cpu_ids * sizeof(struct array_cache *),
4025 gfp);
Siddha, Suresh Bd2e7b7d2006-09-25 23:31:47 -07004026 if (!new)
4027 return -ENOMEM;
4028
Christoph Lametere498be72005-09-09 13:03:32 -07004029 for_each_online_cpu(i) {
Lee Schermerhorn7d6e6d02010-05-26 14:45:03 -07004030 new->new[i] = alloc_arraycache(cpu_to_mem(i), limit,
Pekka Enberg83b519e2009-06-10 19:40:04 +03004031 batchcount, gfp);
Siddha, Suresh Bd2e7b7d2006-09-25 23:31:47 -07004032 if (!new->new[i]) {
Pekka Enbergb28a02d2006-01-08 01:00:37 -08004033 for (i--; i >= 0; i--)
Siddha, Suresh Bd2e7b7d2006-09-25 23:31:47 -07004034 kfree(new->new[i]);
4035 kfree(new);
Christoph Lametere498be72005-09-09 13:03:32 -07004036 return -ENOMEM;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004037 }
4038 }
Siddha, Suresh Bd2e7b7d2006-09-25 23:31:47 -07004039 new->cachep = cachep;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004040
Jens Axboe15c8b6c2008-05-09 09:39:44 +02004041 on_each_cpu(do_ccupdate_local, (void *)new, 1);
Christoph Lametere498be72005-09-09 13:03:32 -07004042
Linus Torvalds1da177e2005-04-16 15:20:36 -07004043 check_irq_on();
Linus Torvalds1da177e2005-04-16 15:20:36 -07004044 cachep->batchcount = batchcount;
4045 cachep->limit = limit;
Christoph Lametere498be72005-09-09 13:03:32 -07004046 cachep->shared = shared;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004047
Christoph Lametere498be72005-09-09 13:03:32 -07004048 for_each_online_cpu(i) {
Siddha, Suresh Bd2e7b7d2006-09-25 23:31:47 -07004049 struct array_cache *ccold = new->new[i];
Linus Torvalds1da177e2005-04-16 15:20:36 -07004050 if (!ccold)
4051 continue;
Lee Schermerhorn7d6e6d02010-05-26 14:45:03 -07004052 spin_lock_irq(&cachep->nodelists[cpu_to_mem(i)]->list_lock);
4053 free_block(cachep, ccold->entry, ccold->avail, cpu_to_mem(i));
4054 spin_unlock_irq(&cachep->nodelists[cpu_to_mem(i)]->list_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004055 kfree(ccold);
4056 }
Siddha, Suresh Bd2e7b7d2006-09-25 23:31:47 -07004057 kfree(new);
Pekka Enberg83b519e2009-06-10 19:40:04 +03004058 return alloc_kmemlist(cachep, gfp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004059}
4060
Ravikiran G Thirumalaib5d8ca72006-03-22 00:08:12 -08004061/* Called with cache_chain_mutex held always */
Pekka Enberg83b519e2009-06-10 19:40:04 +03004062static int enable_cpucache(struct kmem_cache *cachep, gfp_t gfp)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004063{
4064 int err;
4065 int limit, shared;
4066
Andrew Mortona737b3e2006-03-22 00:08:11 -08004067 /*
4068 * The head array serves three purposes:
Linus Torvalds1da177e2005-04-16 15:20:36 -07004069 * - create a LIFO ordering, i.e. return objects that are cache-warm
4070 * - reduce the number of spinlock operations.
Andrew Mortona737b3e2006-03-22 00:08:11 -08004071 * - reduce the number of linked list operations on the slab and
Linus Torvalds1da177e2005-04-16 15:20:36 -07004072 * bufctl chains: array operations are cheaper.
4073 * The numbers are guessed, we should auto-tune as described by
4074 * Bonwick.
4075 */
Manfred Spraul3dafccf2006-02-01 03:05:42 -08004076 if (cachep->buffer_size > 131072)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004077 limit = 1;
Manfred Spraul3dafccf2006-02-01 03:05:42 -08004078 else if (cachep->buffer_size > PAGE_SIZE)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004079 limit = 8;
Manfred Spraul3dafccf2006-02-01 03:05:42 -08004080 else if (cachep->buffer_size > 1024)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004081 limit = 24;
Manfred Spraul3dafccf2006-02-01 03:05:42 -08004082 else if (cachep->buffer_size > 256)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004083 limit = 54;
4084 else
4085 limit = 120;
4086
Andrew Mortona737b3e2006-03-22 00:08:11 -08004087 /*
4088 * CPU bound tasks (e.g. network routing) can exhibit cpu bound
Linus Torvalds1da177e2005-04-16 15:20:36 -07004089 * allocation behaviour: Most allocs on one cpu, most free operations
4090 * on another cpu. For these cases, an efficient object passing between
4091 * cpus is necessary. This is provided by a shared array. The array
4092 * replaces Bonwick's magazine layer.
4093 * On uniprocessor, it's functionally equivalent (but less efficient)
4094 * to a larger limit. Thus disabled by default.
4095 */
4096 shared = 0;
Eric Dumazet364fbb22007-05-06 14:49:27 -07004097 if (cachep->buffer_size <= PAGE_SIZE && num_possible_cpus() > 1)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004098 shared = 8;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004099
4100#if DEBUG
Andrew Mortona737b3e2006-03-22 00:08:11 -08004101 /*
4102 * With debugging enabled, large batchcount lead to excessively long
4103 * periods with disabled local interrupts. Limit the batchcount
Linus Torvalds1da177e2005-04-16 15:20:36 -07004104 */
4105 if (limit > 32)
4106 limit = 32;
4107#endif
Pekka Enberg83b519e2009-06-10 19:40:04 +03004108 err = do_tune_cpucache(cachep, limit, (limit + 1) / 2, shared, gfp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004109 if (err)
4110 printk(KERN_ERR "enable_cpucache failed for %s, error %d.\n",
Pekka Enbergb28a02d2006-01-08 01:00:37 -08004111 cachep->name, -err);
Christoph Lameter2ed3a4e2006-09-25 23:31:38 -07004112 return err;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004113}
4114
Christoph Lameter1b552532006-03-22 00:09:07 -08004115/*
4116 * Drain an array if it contains any elements taking the l3 lock only if
Christoph Lameterb18e7e62006-03-22 00:09:07 -08004117 * necessary. Note that the l3 listlock also protects the array_cache
4118 * if drain_array() is used on the shared array.
Christoph Lameter1b552532006-03-22 00:09:07 -08004119 */
H Hartley Sweeten68a1b192011-01-11 17:49:32 -06004120static void drain_array(struct kmem_cache *cachep, struct kmem_list3 *l3,
Christoph Lameter1b552532006-03-22 00:09:07 -08004121 struct array_cache *ac, int force, int node)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004122{
4123 int tofree;
4124
Christoph Lameter1b552532006-03-22 00:09:07 -08004125 if (!ac || !ac->avail)
4126 return;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004127 if (ac->touched && !force) {
4128 ac->touched = 0;
Christoph Lameterb18e7e62006-03-22 00:09:07 -08004129 } else {
Christoph Lameter1b552532006-03-22 00:09:07 -08004130 spin_lock_irq(&l3->list_lock);
Christoph Lameterb18e7e62006-03-22 00:09:07 -08004131 if (ac->avail) {
4132 tofree = force ? ac->avail : (ac->limit + 4) / 5;
4133 if (tofree > ac->avail)
4134 tofree = (ac->avail + 1) / 2;
4135 free_block(cachep, ac->entry, tofree, node);
4136 ac->avail -= tofree;
4137 memmove(ac->entry, &(ac->entry[tofree]),
4138 sizeof(void *) * ac->avail);
4139 }
Christoph Lameter1b552532006-03-22 00:09:07 -08004140 spin_unlock_irq(&l3->list_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004141 }
4142}
4143
4144/**
4145 * cache_reap - Reclaim memory from caches.
Randy Dunlap05fb6bf2007-02-28 20:12:13 -08004146 * @w: work descriptor
Linus Torvalds1da177e2005-04-16 15:20:36 -07004147 *
4148 * Called from workqueue/eventd every few seconds.
4149 * Purpose:
4150 * - clear the per-cpu caches for this CPU.
4151 * - return freeable pages to the main free memory pool.
4152 *
Andrew Mortona737b3e2006-03-22 00:08:11 -08004153 * If we cannot acquire the cache chain mutex then just give up - we'll try
4154 * again on the next iteration.
Linus Torvalds1da177e2005-04-16 15:20:36 -07004155 */
Christoph Lameter7c5cae32007-02-10 01:42:55 -08004156static void cache_reap(struct work_struct *w)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004157{
Christoph Hellwig7a7c3812006-06-23 02:03:17 -07004158 struct kmem_cache *searchp;
Christoph Lametere498be72005-09-09 13:03:32 -07004159 struct kmem_list3 *l3;
Lee Schermerhorn7d6e6d02010-05-26 14:45:03 -07004160 int node = numa_mem_id();
Jean Delvarebf6aede2009-04-02 16:56:54 -07004161 struct delayed_work *work = to_delayed_work(w);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004162
Christoph Lameter7c5cae32007-02-10 01:42:55 -08004163 if (!mutex_trylock(&cache_chain_mutex))
Linus Torvalds1da177e2005-04-16 15:20:36 -07004164 /* Give up. Setup the next iteration. */
Christoph Lameter7c5cae32007-02-10 01:42:55 -08004165 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004166
Christoph Hellwig7a7c3812006-06-23 02:03:17 -07004167 list_for_each_entry(searchp, &cache_chain, next) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07004168 check_irq_on();
4169
Christoph Lameter35386e32006-03-22 00:09:05 -08004170 /*
4171 * We only take the l3 lock if absolutely necessary and we
4172 * have established with reasonable certainty that
4173 * we can do some work if the lock was obtained.
4174 */
Christoph Lameteraab22072006-03-22 00:09:06 -08004175 l3 = searchp->nodelists[node];
Christoph Lameter35386e32006-03-22 00:09:05 -08004176
Christoph Lameter8fce4d82006-03-09 17:33:54 -08004177 reap_alien(searchp, l3);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004178
Christoph Lameteraab22072006-03-22 00:09:06 -08004179 drain_array(searchp, l3, cpu_cache_get(searchp), 0, node);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004180
Christoph Lameter35386e32006-03-22 00:09:05 -08004181 /*
4182 * These are racy checks but it does not matter
4183 * if we skip one check or scan twice.
4184 */
Christoph Lametere498be72005-09-09 13:03:32 -07004185 if (time_after(l3->next_reap, jiffies))
Christoph Lameter35386e32006-03-22 00:09:05 -08004186 goto next;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004187
Christoph Lametere498be72005-09-09 13:03:32 -07004188 l3->next_reap = jiffies + REAPTIMEOUT_LIST3;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004189
Christoph Lameteraab22072006-03-22 00:09:06 -08004190 drain_array(searchp, l3, l3->shared, 0, node);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004191
Christoph Lametered11d9e2006-06-30 01:55:45 -07004192 if (l3->free_touched)
Christoph Lametere498be72005-09-09 13:03:32 -07004193 l3->free_touched = 0;
Christoph Lametered11d9e2006-06-30 01:55:45 -07004194 else {
4195 int freed;
4196
4197 freed = drain_freelist(searchp, l3, (l3->free_limit +
4198 5 * searchp->num - 1) / (5 * searchp->num));
4199 STATS_ADD_REAPED(searchp, freed);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004200 }
Christoph Lameter35386e32006-03-22 00:09:05 -08004201next:
Linus Torvalds1da177e2005-04-16 15:20:36 -07004202 cond_resched();
4203 }
4204 check_irq_on();
Ingo Molnarfc0abb12006-01-18 17:42:33 -08004205 mutex_unlock(&cache_chain_mutex);
Christoph Lameter8fce4d82006-03-09 17:33:54 -08004206 next_reap_node();
Christoph Lameter7c5cae32007-02-10 01:42:55 -08004207out:
Andrew Mortona737b3e2006-03-22 00:08:11 -08004208 /* Set up the next iteration */
Christoph Lameter7c5cae32007-02-10 01:42:55 -08004209 schedule_delayed_work(work, round_jiffies_relative(REAPTIMEOUT_CPUC));
Linus Torvalds1da177e2005-04-16 15:20:36 -07004210}
4211
Linus Torvalds158a9622008-01-02 13:04:48 -08004212#ifdef CONFIG_SLABINFO
Linus Torvalds1da177e2005-04-16 15:20:36 -07004213
Pekka Enberg85289f92006-01-08 01:00:36 -08004214static void print_slabinfo_header(struct seq_file *m)
4215{
4216 /*
4217 * Output format version, so at least we can change it
4218 * without _too_ many complaints.
4219 */
4220#if STATS
4221 seq_puts(m, "slabinfo - version: 2.1 (statistics)\n");
4222#else
4223 seq_puts(m, "slabinfo - version: 2.1\n");
4224#endif
4225 seq_puts(m, "# name <active_objs> <num_objs> <objsize> "
4226 "<objperslab> <pagesperslab>");
4227 seq_puts(m, " : tunables <limit> <batchcount> <sharedfactor>");
4228 seq_puts(m, " : slabdata <active_slabs> <num_slabs> <sharedavail>");
4229#if STATS
4230 seq_puts(m, " : globalstat <listallocs> <maxobjs> <grown> <reaped> "
Ravikiran G Thirumalaifb7faf32006-04-10 22:52:54 -07004231 "<error> <maxfreeable> <nodeallocs> <remotefrees> <alienoverflow>");
Pekka Enberg85289f92006-01-08 01:00:36 -08004232 seq_puts(m, " : cpustat <allochit> <allocmiss> <freehit> <freemiss>");
4233#endif
4234 seq_putc(m, '\n');
4235}
4236
Linus Torvalds1da177e2005-04-16 15:20:36 -07004237static void *s_start(struct seq_file *m, loff_t *pos)
4238{
4239 loff_t n = *pos;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004240
Ingo Molnarfc0abb12006-01-18 17:42:33 -08004241 mutex_lock(&cache_chain_mutex);
Pekka Enberg85289f92006-01-08 01:00:36 -08004242 if (!n)
4243 print_slabinfo_header(m);
Pavel Emelianovb92151b2007-07-15 23:38:04 -07004244
4245 return seq_list_start(&cache_chain, *pos);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004246}
4247
4248static void *s_next(struct seq_file *m, void *p, loff_t *pos)
4249{
Pavel Emelianovb92151b2007-07-15 23:38:04 -07004250 return seq_list_next(p, &cache_chain, pos);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004251}
4252
4253static void s_stop(struct seq_file *m, void *p)
4254{
Ingo Molnarfc0abb12006-01-18 17:42:33 -08004255 mutex_unlock(&cache_chain_mutex);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004256}
4257
4258static int s_show(struct seq_file *m, void *p)
4259{
Pavel Emelianovb92151b2007-07-15 23:38:04 -07004260 struct kmem_cache *cachep = list_entry(p, struct kmem_cache, next);
Pekka Enbergb28a02d2006-01-08 01:00:37 -08004261 struct slab *slabp;
4262 unsigned long active_objs;
4263 unsigned long num_objs;
4264 unsigned long active_slabs = 0;
4265 unsigned long num_slabs, free_objects = 0, shared_avail = 0;
Christoph Lametere498be72005-09-09 13:03:32 -07004266 const char *name;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004267 char *error = NULL;
Christoph Lametere498be72005-09-09 13:03:32 -07004268 int node;
4269 struct kmem_list3 *l3;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004270
Linus Torvalds1da177e2005-04-16 15:20:36 -07004271 active_objs = 0;
4272 num_slabs = 0;
Christoph Lametere498be72005-09-09 13:03:32 -07004273 for_each_online_node(node) {
4274 l3 = cachep->nodelists[node];
4275 if (!l3)
4276 continue;
4277
Ravikiran G Thirumalaica3b9b92006-02-04 23:27:58 -08004278 check_irq_on();
4279 spin_lock_irq(&l3->list_lock);
Christoph Lametere498be72005-09-09 13:03:32 -07004280
Christoph Hellwig7a7c3812006-06-23 02:03:17 -07004281 list_for_each_entry(slabp, &l3->slabs_full, list) {
Christoph Lametere498be72005-09-09 13:03:32 -07004282 if (slabp->inuse != cachep->num && !error)
4283 error = "slabs_full accounting error";
4284 active_objs += cachep->num;
4285 active_slabs++;
4286 }
Christoph Hellwig7a7c3812006-06-23 02:03:17 -07004287 list_for_each_entry(slabp, &l3->slabs_partial, list) {
Christoph Lametere498be72005-09-09 13:03:32 -07004288 if (slabp->inuse == cachep->num && !error)
4289 error = "slabs_partial inuse accounting error";
4290 if (!slabp->inuse && !error)
4291 error = "slabs_partial/inuse accounting error";
4292 active_objs += slabp->inuse;
4293 active_slabs++;
4294 }
Christoph Hellwig7a7c3812006-06-23 02:03:17 -07004295 list_for_each_entry(slabp, &l3->slabs_free, list) {
Christoph Lametere498be72005-09-09 13:03:32 -07004296 if (slabp->inuse && !error)
4297 error = "slabs_free/inuse accounting error";
4298 num_slabs++;
4299 }
4300 free_objects += l3->free_objects;
Ravikiran G Thirumalai4484ebf2006-02-04 23:27:59 -08004301 if (l3->shared)
4302 shared_avail += l3->shared->avail;
Christoph Lametere498be72005-09-09 13:03:32 -07004303
Ravikiran G Thirumalaica3b9b92006-02-04 23:27:58 -08004304 spin_unlock_irq(&l3->list_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004305 }
Pekka Enbergb28a02d2006-01-08 01:00:37 -08004306 num_slabs += active_slabs;
4307 num_objs = num_slabs * cachep->num;
Christoph Lametere498be72005-09-09 13:03:32 -07004308 if (num_objs - active_objs != free_objects && !error)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004309 error = "free_objects accounting error";
4310
Pekka Enbergb28a02d2006-01-08 01:00:37 -08004311 name = cachep->name;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004312 if (error)
4313 printk(KERN_ERR "slab: cache %s error: %s\n", name, error);
4314
4315 seq_printf(m, "%-17s %6lu %6lu %6u %4u %4d",
Manfred Spraul3dafccf2006-02-01 03:05:42 -08004316 name, active_objs, num_objs, cachep->buffer_size,
Pekka Enbergb28a02d2006-01-08 01:00:37 -08004317 cachep->num, (1 << cachep->gfporder));
Linus Torvalds1da177e2005-04-16 15:20:36 -07004318 seq_printf(m, " : tunables %4u %4u %4u",
Pekka Enbergb28a02d2006-01-08 01:00:37 -08004319 cachep->limit, cachep->batchcount, cachep->shared);
Christoph Lametere498be72005-09-09 13:03:32 -07004320 seq_printf(m, " : slabdata %6lu %6lu %6lu",
Pekka Enbergb28a02d2006-01-08 01:00:37 -08004321 active_slabs, num_slabs, shared_avail);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004322#if STATS
Pekka Enbergb28a02d2006-01-08 01:00:37 -08004323 { /* list3 stats */
Linus Torvalds1da177e2005-04-16 15:20:36 -07004324 unsigned long high = cachep->high_mark;
4325 unsigned long allocs = cachep->num_allocations;
4326 unsigned long grown = cachep->grown;
4327 unsigned long reaped = cachep->reaped;
4328 unsigned long errors = cachep->errors;
4329 unsigned long max_freeable = cachep->max_freeable;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004330 unsigned long node_allocs = cachep->node_allocs;
Christoph Lametere498be72005-09-09 13:03:32 -07004331 unsigned long node_frees = cachep->node_frees;
Ravikiran G Thirumalaifb7faf32006-04-10 22:52:54 -07004332 unsigned long overflows = cachep->node_overflow;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004333
Joe Perchese92dd4f2010-03-26 19:27:58 -07004334 seq_printf(m, " : globalstat %7lu %6lu %5lu %4lu "
4335 "%4lu %4lu %4lu %4lu %4lu",
4336 allocs, high, grown,
4337 reaped, errors, max_freeable, node_allocs,
4338 node_frees, overflows);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004339 }
4340 /* cpu stats */
4341 {
4342 unsigned long allochit = atomic_read(&cachep->allochit);
4343 unsigned long allocmiss = atomic_read(&cachep->allocmiss);
4344 unsigned long freehit = atomic_read(&cachep->freehit);
4345 unsigned long freemiss = atomic_read(&cachep->freemiss);
4346
4347 seq_printf(m, " : cpustat %6lu %6lu %6lu %6lu",
Pekka Enbergb28a02d2006-01-08 01:00:37 -08004348 allochit, allocmiss, freehit, freemiss);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004349 }
4350#endif
4351 seq_putc(m, '\n');
Linus Torvalds1da177e2005-04-16 15:20:36 -07004352 return 0;
4353}
4354
4355/*
4356 * slabinfo_op - iterator that generates /proc/slabinfo
4357 *
4358 * Output layout:
4359 * cache-name
4360 * num-active-objs
4361 * total-objs
4362 * object size
4363 * num-active-slabs
4364 * total-slabs
4365 * num-pages-per-slab
4366 * + further values on SMP and with statistics enabled
4367 */
4368
Alexey Dobriyan7b3c3a52008-10-06 02:42:17 +04004369static const struct seq_operations slabinfo_op = {
Pekka Enbergb28a02d2006-01-08 01:00:37 -08004370 .start = s_start,
4371 .next = s_next,
4372 .stop = s_stop,
4373 .show = s_show,
Linus Torvalds1da177e2005-04-16 15:20:36 -07004374};
4375
4376#define MAX_SLABINFO_WRITE 128
4377/**
4378 * slabinfo_write - Tuning for the slab allocator
4379 * @file: unused
4380 * @buffer: user buffer
4381 * @count: data length
4382 * @ppos: unused
4383 */
H Hartley Sweeten68a1b192011-01-11 17:49:32 -06004384static ssize_t slabinfo_write(struct file *file, const char __user *buffer,
Pekka Enbergb28a02d2006-01-08 01:00:37 -08004385 size_t count, loff_t *ppos)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004386{
Pekka Enbergb28a02d2006-01-08 01:00:37 -08004387 char kbuf[MAX_SLABINFO_WRITE + 1], *tmp;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004388 int limit, batchcount, shared, res;
Christoph Hellwig7a7c3812006-06-23 02:03:17 -07004389 struct kmem_cache *cachep;
Pekka Enbergb28a02d2006-01-08 01:00:37 -08004390
Linus Torvalds1da177e2005-04-16 15:20:36 -07004391 if (count > MAX_SLABINFO_WRITE)
4392 return -EINVAL;
4393 if (copy_from_user(&kbuf, buffer, count))
4394 return -EFAULT;
Pekka Enbergb28a02d2006-01-08 01:00:37 -08004395 kbuf[MAX_SLABINFO_WRITE] = '\0';
Linus Torvalds1da177e2005-04-16 15:20:36 -07004396
4397 tmp = strchr(kbuf, ' ');
4398 if (!tmp)
4399 return -EINVAL;
4400 *tmp = '\0';
4401 tmp++;
4402 if (sscanf(tmp, " %d %d %d", &limit, &batchcount, &shared) != 3)
4403 return -EINVAL;
4404
4405 /* Find the cache in the chain of caches. */
Ingo Molnarfc0abb12006-01-18 17:42:33 -08004406 mutex_lock(&cache_chain_mutex);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004407 res = -EINVAL;
Christoph Hellwig7a7c3812006-06-23 02:03:17 -07004408 list_for_each_entry(cachep, &cache_chain, next) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07004409 if (!strcmp(cachep->name, kbuf)) {
Andrew Mortona737b3e2006-03-22 00:08:11 -08004410 if (limit < 1 || batchcount < 1 ||
4411 batchcount > limit || shared < 0) {
Christoph Lametere498be72005-09-09 13:03:32 -07004412 res = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004413 } else {
Christoph Lametere498be72005-09-09 13:03:32 -07004414 res = do_tune_cpucache(cachep, limit,
Pekka Enberg83b519e2009-06-10 19:40:04 +03004415 batchcount, shared,
4416 GFP_KERNEL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004417 }
4418 break;
4419 }
4420 }
Ingo Molnarfc0abb12006-01-18 17:42:33 -08004421 mutex_unlock(&cache_chain_mutex);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004422 if (res >= 0)
4423 res = count;
4424 return res;
4425}
Al Viro871751e2006-03-25 03:06:39 -08004426
Alexey Dobriyan7b3c3a52008-10-06 02:42:17 +04004427static int slabinfo_open(struct inode *inode, struct file *file)
4428{
4429 return seq_open(file, &slabinfo_op);
4430}
4431
4432static const struct file_operations proc_slabinfo_operations = {
4433 .open = slabinfo_open,
4434 .read = seq_read,
4435 .write = slabinfo_write,
4436 .llseek = seq_lseek,
4437 .release = seq_release,
4438};
4439
Al Viro871751e2006-03-25 03:06:39 -08004440#ifdef CONFIG_DEBUG_SLAB_LEAK
4441
4442static void *leaks_start(struct seq_file *m, loff_t *pos)
4443{
Al Viro871751e2006-03-25 03:06:39 -08004444 mutex_lock(&cache_chain_mutex);
Pavel Emelianovb92151b2007-07-15 23:38:04 -07004445 return seq_list_start(&cache_chain, *pos);
Al Viro871751e2006-03-25 03:06:39 -08004446}
4447
4448static inline int add_caller(unsigned long *n, unsigned long v)
4449{
4450 unsigned long *p;
4451 int l;
4452 if (!v)
4453 return 1;
4454 l = n[1];
4455 p = n + 2;
4456 while (l) {
4457 int i = l/2;
4458 unsigned long *q = p + 2 * i;
4459 if (*q == v) {
4460 q[1]++;
4461 return 1;
4462 }
4463 if (*q > v) {
4464 l = i;
4465 } else {
4466 p = q + 2;
4467 l -= i + 1;
4468 }
4469 }
4470 if (++n[1] == n[0])
4471 return 0;
4472 memmove(p + 2, p, n[1] * 2 * sizeof(unsigned long) - ((void *)p - (void *)n));
4473 p[0] = v;
4474 p[1] = 1;
4475 return 1;
4476}
4477
4478static void handle_slab(unsigned long *n, struct kmem_cache *c, struct slab *s)
4479{
4480 void *p;
4481 int i;
4482 if (n[0] == n[1])
4483 return;
4484 for (i = 0, p = s->s_mem; i < c->num; i++, p += c->buffer_size) {
4485 if (slab_bufctl(s)[i] != BUFCTL_ACTIVE)
4486 continue;
4487 if (!add_caller(n, (unsigned long)*dbg_userword(c, p)))
4488 return;
4489 }
4490}
4491
4492static void show_symbol(struct seq_file *m, unsigned long address)
4493{
4494#ifdef CONFIG_KALLSYMS
Al Viro871751e2006-03-25 03:06:39 -08004495 unsigned long offset, size;
Tejun Heo9281ace2007-07-17 04:03:51 -07004496 char modname[MODULE_NAME_LEN], name[KSYM_NAME_LEN];
Al Viro871751e2006-03-25 03:06:39 -08004497
Alexey Dobriyana5c43da2007-05-08 00:28:47 -07004498 if (lookup_symbol_attrs(address, &size, &offset, modname, name) == 0) {
Al Viro871751e2006-03-25 03:06:39 -08004499 seq_printf(m, "%s+%#lx/%#lx", name, offset, size);
Alexey Dobriyana5c43da2007-05-08 00:28:47 -07004500 if (modname[0])
Al Viro871751e2006-03-25 03:06:39 -08004501 seq_printf(m, " [%s]", modname);
4502 return;
4503 }
4504#endif
4505 seq_printf(m, "%p", (void *)address);
4506}
4507
4508static int leaks_show(struct seq_file *m, void *p)
4509{
Pavel Emelianovb92151b2007-07-15 23:38:04 -07004510 struct kmem_cache *cachep = list_entry(p, struct kmem_cache, next);
Al Viro871751e2006-03-25 03:06:39 -08004511 struct slab *slabp;
4512 struct kmem_list3 *l3;
4513 const char *name;
4514 unsigned long *n = m->private;
4515 int node;
4516 int i;
4517
4518 if (!(cachep->flags & SLAB_STORE_USER))
4519 return 0;
4520 if (!(cachep->flags & SLAB_RED_ZONE))
4521 return 0;
4522
4523 /* OK, we can do it */
4524
4525 n[1] = 0;
4526
4527 for_each_online_node(node) {
4528 l3 = cachep->nodelists[node];
4529 if (!l3)
4530 continue;
4531
4532 check_irq_on();
4533 spin_lock_irq(&l3->list_lock);
4534
Christoph Hellwig7a7c3812006-06-23 02:03:17 -07004535 list_for_each_entry(slabp, &l3->slabs_full, list)
Al Viro871751e2006-03-25 03:06:39 -08004536 handle_slab(n, cachep, slabp);
Christoph Hellwig7a7c3812006-06-23 02:03:17 -07004537 list_for_each_entry(slabp, &l3->slabs_partial, list)
Al Viro871751e2006-03-25 03:06:39 -08004538 handle_slab(n, cachep, slabp);
Al Viro871751e2006-03-25 03:06:39 -08004539 spin_unlock_irq(&l3->list_lock);
4540 }
4541 name = cachep->name;
4542 if (n[0] == n[1]) {
4543 /* Increase the buffer size */
4544 mutex_unlock(&cache_chain_mutex);
4545 m->private = kzalloc(n[0] * 4 * sizeof(unsigned long), GFP_KERNEL);
4546 if (!m->private) {
4547 /* Too bad, we are really out */
4548 m->private = n;
4549 mutex_lock(&cache_chain_mutex);
4550 return -ENOMEM;
4551 }
4552 *(unsigned long *)m->private = n[0] * 2;
4553 kfree(n);
4554 mutex_lock(&cache_chain_mutex);
4555 /* Now make sure this entry will be retried */
4556 m->count = m->size;
4557 return 0;
4558 }
4559 for (i = 0; i < n[1]; i++) {
4560 seq_printf(m, "%s: %lu ", name, n[2*i+3]);
4561 show_symbol(m, n[2*i+2]);
4562 seq_putc(m, '\n');
4563 }
Siddha, Suresh Bd2e7b7d2006-09-25 23:31:47 -07004564
Al Viro871751e2006-03-25 03:06:39 -08004565 return 0;
4566}
4567
Alexey Dobriyana0ec95a2008-10-06 00:59:10 +04004568static const struct seq_operations slabstats_op = {
Al Viro871751e2006-03-25 03:06:39 -08004569 .start = leaks_start,
4570 .next = s_next,
4571 .stop = s_stop,
4572 .show = leaks_show,
4573};
Alexey Dobriyana0ec95a2008-10-06 00:59:10 +04004574
4575static int slabstats_open(struct inode *inode, struct file *file)
4576{
4577 unsigned long *n = kzalloc(PAGE_SIZE, GFP_KERNEL);
4578 int ret = -ENOMEM;
4579 if (n) {
4580 ret = seq_open(file, &slabstats_op);
4581 if (!ret) {
4582 struct seq_file *m = file->private_data;
4583 *n = PAGE_SIZE / (2 * sizeof(unsigned long));
4584 m->private = n;
4585 n = NULL;
4586 }
4587 kfree(n);
4588 }
4589 return ret;
4590}
4591
4592static const struct file_operations proc_slabstats_operations = {
4593 .open = slabstats_open,
4594 .read = seq_read,
4595 .llseek = seq_lseek,
4596 .release = seq_release_private,
4597};
Al Viro871751e2006-03-25 03:06:39 -08004598#endif
Alexey Dobriyana0ec95a2008-10-06 00:59:10 +04004599
4600static int __init slab_proc_init(void)
4601{
Vasiliy Kulikovab067e92011-09-27 21:54:53 +04004602 proc_create("slabinfo",S_IWUSR|S_IRUSR,NULL,&proc_slabinfo_operations);
Alexey Dobriyana0ec95a2008-10-06 00:59:10 +04004603#ifdef CONFIG_DEBUG_SLAB_LEAK
4604 proc_create("slab_allocators", 0, NULL, &proc_slabstats_operations);
4605#endif
4606 return 0;
4607}
4608module_init(slab_proc_init);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004609#endif
4610
Manfred Spraul00e145b2005-09-03 15:55:07 -07004611/**
4612 * ksize - get the actual amount of memory allocated for a given object
4613 * @objp: Pointer to the object
4614 *
4615 * kmalloc may internally round up allocations and return more memory
4616 * than requested. ksize() can be used to determine the actual amount of
4617 * memory allocated. The caller may use this additional memory, even though
4618 * a smaller amount of memory was initially specified with the kmalloc call.
4619 * The caller must guarantee that objp points to a valid object previously
4620 * allocated with either kmalloc() or kmem_cache_alloc(). The object
4621 * must not be freed during the duration of the call.
4622 */
Pekka Enbergfd76bab2007-05-06 14:48:40 -07004623size_t ksize(const void *objp)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004624{
Christoph Lameteref8b4522007-10-16 01:24:46 -07004625 BUG_ON(!objp);
4626 if (unlikely(objp == ZERO_SIZE_PTR))
Manfred Spraul00e145b2005-09-03 15:55:07 -07004627 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004628
Pekka Enberg6ed5eb2212006-02-01 03:05:49 -08004629 return obj_size(virt_to_cache(objp));
Linus Torvalds1da177e2005-04-16 15:20:36 -07004630}
Kirill A. Shutemovb1aabec2009-02-10 15:21:44 +02004631EXPORT_SYMBOL(ksize);