blob: addb20a6d67d53e73e2dba2da6492ac0ee93aa38 [file] [log] [blame]
Christoph Lameter81819f02007-05-06 14:49:36 -07001/*
2 * SLUB: A slab allocator that limits cache line use instead of queuing
3 * objects in per cpu and per node lists.
4 *
5 * The allocator synchronizes using per slab locks and only
6 * uses a centralized lock to manage a pool of partial slabs.
7 *
8 * (C) 2007 SGI, Christoph Lameter <clameter@sgi.com>
9 */
10
11#include <linux/mm.h>
12#include <linux/module.h>
13#include <linux/bit_spinlock.h>
14#include <linux/interrupt.h>
15#include <linux/bitops.h>
16#include <linux/slab.h>
17#include <linux/seq_file.h>
18#include <linux/cpu.h>
19#include <linux/cpuset.h>
20#include <linux/mempolicy.h>
21#include <linux/ctype.h>
22#include <linux/kallsyms.h>
23
24/*
25 * Lock order:
26 * 1. slab_lock(page)
27 * 2. slab->list_lock
28 *
29 * The slab_lock protects operations on the object of a particular
30 * slab and its metadata in the page struct. If the slab lock
31 * has been taken then no allocations nor frees can be performed
32 * on the objects in the slab nor can the slab be added or removed
33 * from the partial or full lists since this would mean modifying
34 * the page_struct of the slab.
35 *
36 * The list_lock protects the partial and full list on each node and
37 * the partial slab counter. If taken then no new slabs may be added or
38 * removed from the lists nor make the number of partial slabs be modified.
39 * (Note that the total number of slabs is an atomic value that may be
40 * modified without taking the list lock).
41 *
42 * The list_lock is a centralized lock and thus we avoid taking it as
43 * much as possible. As long as SLUB does not have to handle partial
44 * slabs, operations can continue without any centralized lock. F.e.
45 * allocating a long series of objects that fill up slabs does not require
46 * the list lock.
47 *
48 * The lock order is sometimes inverted when we are trying to get a slab
49 * off a list. We take the list_lock and then look for a page on the list
50 * to use. While we do that objects in the slabs may be freed. We can
51 * only operate on the slab if we have also taken the slab_lock. So we use
52 * a slab_trylock() on the slab. If trylock was successful then no frees
53 * can occur anymore and we can use the slab for allocations etc. If the
54 * slab_trylock() does not succeed then frees are in progress in the slab and
55 * we must stay away from it for a while since we may cause a bouncing
56 * cacheline if we try to acquire the lock. So go onto the next slab.
57 * If all pages are busy then we may allocate a new slab instead of reusing
58 * a partial slab. A new slab has noone operating on it and thus there is
59 * no danger of cacheline contention.
60 *
61 * Interrupts are disabled during allocation and deallocation in order to
62 * make the slab allocator safe to use in the context of an irq. In addition
63 * interrupts are disabled to ensure that the processor does not change
64 * while handling per_cpu slabs, due to kernel preemption.
65 *
66 * SLUB assigns one slab for allocation to each processor.
67 * Allocations only occur from these slabs called cpu slabs.
68 *
Christoph Lameter672bba32007-05-09 02:32:39 -070069 * Slabs with free elements are kept on a partial list and during regular
70 * operations no list for full slabs is used. If an object in a full slab is
Christoph Lameter81819f02007-05-06 14:49:36 -070071 * freed then the slab will show up again on the partial lists.
Christoph Lameter672bba32007-05-09 02:32:39 -070072 * We track full slabs for debugging purposes though because otherwise we
73 * cannot scan all objects.
Christoph Lameter81819f02007-05-06 14:49:36 -070074 *
75 * Slabs are freed when they become empty. Teardown and setup is
76 * minimal so we rely on the page allocators per cpu caches for
77 * fast frees and allocs.
78 *
79 * Overloading of page flags that are otherwise used for LRU management.
80 *
Christoph Lameter4b6f0752007-05-16 22:10:53 -070081 * PageActive The slab is frozen and exempt from list processing.
82 * This means that the slab is dedicated to a purpose
83 * such as satisfying allocations for a specific
84 * processor. Objects may be freed in the slab while
85 * it is frozen but slab_free will then skip the usual
86 * list operations. It is up to the processor holding
87 * the slab to integrate the slab into the slab lists
88 * when the slab is no longer needed.
89 *
90 * One use of this flag is to mark slabs that are
91 * used for allocations. Then such a slab becomes a cpu
92 * slab. The cpu slab may be equipped with an additional
Christoph Lameter894b8782007-05-10 03:15:16 -070093 * lockless_freelist that allows lockless access to
94 * free objects in addition to the regular freelist
95 * that requires the slab lock.
Christoph Lameter81819f02007-05-06 14:49:36 -070096 *
97 * PageError Slab requires special handling due to debug
98 * options set. This moves slab handling out of
Christoph Lameter894b8782007-05-10 03:15:16 -070099 * the fast path and disables lockless freelists.
Christoph Lameter81819f02007-05-06 14:49:36 -0700100 */
101
Christoph Lameter5577bd82007-05-16 22:10:56 -0700102#define FROZEN (1 << PG_active)
103
104#ifdef CONFIG_SLUB_DEBUG
105#define SLABDEBUG (1 << PG_error)
106#else
107#define SLABDEBUG 0
108#endif
109
Christoph Lameter4b6f0752007-05-16 22:10:53 -0700110static inline int SlabFrozen(struct page *page)
111{
Christoph Lameter5577bd82007-05-16 22:10:56 -0700112 return page->flags & FROZEN;
Christoph Lameter4b6f0752007-05-16 22:10:53 -0700113}
114
115static inline void SetSlabFrozen(struct page *page)
116{
Christoph Lameter5577bd82007-05-16 22:10:56 -0700117 page->flags |= FROZEN;
Christoph Lameter4b6f0752007-05-16 22:10:53 -0700118}
119
120static inline void ClearSlabFrozen(struct page *page)
121{
Christoph Lameter5577bd82007-05-16 22:10:56 -0700122 page->flags &= ~FROZEN;
Christoph Lameter4b6f0752007-05-16 22:10:53 -0700123}
124
Christoph Lameter35e5d7e2007-05-09 02:32:42 -0700125static inline int SlabDebug(struct page *page)
126{
Christoph Lameter5577bd82007-05-16 22:10:56 -0700127 return page->flags & SLABDEBUG;
Christoph Lameter35e5d7e2007-05-09 02:32:42 -0700128}
129
130static inline void SetSlabDebug(struct page *page)
131{
Christoph Lameter5577bd82007-05-16 22:10:56 -0700132 page->flags |= SLABDEBUG;
Christoph Lameter35e5d7e2007-05-09 02:32:42 -0700133}
134
135static inline void ClearSlabDebug(struct page *page)
136{
Christoph Lameter5577bd82007-05-16 22:10:56 -0700137 page->flags &= ~SLABDEBUG;
Christoph Lameter35e5d7e2007-05-09 02:32:42 -0700138}
139
Christoph Lameter81819f02007-05-06 14:49:36 -0700140/*
141 * Issues still to be resolved:
142 *
143 * - The per cpu array is updated for each new slab and and is a remote
144 * cacheline for most nodes. This could become a bouncing cacheline given
Christoph Lameter672bba32007-05-09 02:32:39 -0700145 * enough frequent updates. There are 16 pointers in a cacheline, so at
146 * max 16 cpus could compete for the cacheline which may be okay.
Christoph Lameter81819f02007-05-06 14:49:36 -0700147 *
148 * - Support PAGE_ALLOC_DEBUG. Should be easy to do.
149 *
Christoph Lameter81819f02007-05-06 14:49:36 -0700150 * - Variable sizing of the per node arrays
151 */
152
153/* Enable to test recovery from slab corruption on boot */
154#undef SLUB_RESILIENCY_TEST
155
156#if PAGE_SHIFT <= 12
157
158/*
159 * Small page size. Make sure that we do not fragment memory
160 */
161#define DEFAULT_MAX_ORDER 1
162#define DEFAULT_MIN_OBJECTS 4
163
164#else
165
166/*
167 * Large page machines are customarily able to handle larger
168 * page orders.
169 */
170#define DEFAULT_MAX_ORDER 2
171#define DEFAULT_MIN_OBJECTS 8
172
173#endif
174
175/*
Christoph Lameter2086d262007-05-06 14:49:46 -0700176 * Mininum number of partial slabs. These will be left on the partial
177 * lists even if they are empty. kmem_cache_shrink may reclaim them.
178 */
Christoph Lametere95eed52007-05-06 14:49:44 -0700179#define MIN_PARTIAL 2
180
Christoph Lameter2086d262007-05-06 14:49:46 -0700181/*
182 * Maximum number of desirable partial slabs.
183 * The existence of more partial slabs makes kmem_cache_shrink
184 * sort the partial list by the number of objects in the.
185 */
186#define MAX_PARTIAL 10
187
Christoph Lameter81819f02007-05-06 14:49:36 -0700188#define DEBUG_DEFAULT_FLAGS (SLAB_DEBUG_FREE | SLAB_RED_ZONE | \
189 SLAB_POISON | SLAB_STORE_USER)
Christoph Lameter672bba32007-05-09 02:32:39 -0700190
Christoph Lameter81819f02007-05-06 14:49:36 -0700191/*
192 * Set of flags that will prevent slab merging
193 */
194#define SLUB_NEVER_MERGE (SLAB_RED_ZONE | SLAB_POISON | SLAB_STORE_USER | \
195 SLAB_TRACE | SLAB_DESTROY_BY_RCU)
196
197#define SLUB_MERGE_SAME (SLAB_DEBUG_FREE | SLAB_RECLAIM_ACCOUNT | \
198 SLAB_CACHE_DMA)
199
200#ifndef ARCH_KMALLOC_MINALIGN
Christoph Lameter47bfdc02007-05-06 14:49:37 -0700201#define ARCH_KMALLOC_MINALIGN __alignof__(unsigned long long)
Christoph Lameter81819f02007-05-06 14:49:36 -0700202#endif
203
204#ifndef ARCH_SLAB_MINALIGN
Christoph Lameter47bfdc02007-05-06 14:49:37 -0700205#define ARCH_SLAB_MINALIGN __alignof__(unsigned long long)
Christoph Lameter81819f02007-05-06 14:49:36 -0700206#endif
207
Christoph Lameter6300ea72007-07-17 04:03:20 -0700208/*
209 * The page->inuse field is 16 bit thus we have this limitation
210 */
211#define MAX_OBJECTS_PER_SLAB 65535
212
Christoph Lameter81819f02007-05-06 14:49:36 -0700213/* Internal SLUB flags */
Christoph Lameter1ceef402007-08-07 15:11:48 -0700214#define __OBJECT_POISON 0x80000000 /* Poison object */
215#define __SYSFS_ADD_DEFERRED 0x40000000 /* Not yet visible via sysfs */
Christoph Lameter81819f02007-05-06 14:49:36 -0700216
Christoph Lameter65c02d42007-05-09 02:32:35 -0700217/* Not all arches define cache_line_size */
218#ifndef cache_line_size
219#define cache_line_size() L1_CACHE_BYTES
220#endif
221
Christoph Lameter81819f02007-05-06 14:49:36 -0700222static int kmem_size = sizeof(struct kmem_cache);
223
224#ifdef CONFIG_SMP
225static struct notifier_block slab_notifier;
226#endif
227
228static enum {
229 DOWN, /* No slab functionality available */
230 PARTIAL, /* kmem_cache_open() works but kmalloc does not */
Christoph Lameter672bba32007-05-09 02:32:39 -0700231 UP, /* Everything works but does not show up in sysfs */
Christoph Lameter81819f02007-05-06 14:49:36 -0700232 SYSFS /* Sysfs up */
233} slab_state = DOWN;
234
235/* A list of all slab caches on the system */
236static DECLARE_RWSEM(slub_lock);
Adrian Bunk5af328a2007-07-17 04:03:27 -0700237static LIST_HEAD(slab_caches);
Christoph Lameter81819f02007-05-06 14:49:36 -0700238
Christoph Lameter02cbc872007-05-09 02:32:43 -0700239/*
240 * Tracking user of a slab.
241 */
242struct track {
243 void *addr; /* Called from address */
244 int cpu; /* Was running on cpu */
245 int pid; /* Pid context */
246 unsigned long when; /* When did the operation occur */
247};
248
249enum track_item { TRACK_ALLOC, TRACK_FREE };
250
Christoph Lameter41ecc552007-05-09 02:32:44 -0700251#if defined(CONFIG_SYSFS) && defined(CONFIG_SLUB_DEBUG)
Christoph Lameter81819f02007-05-06 14:49:36 -0700252static int sysfs_slab_add(struct kmem_cache *);
253static int sysfs_slab_alias(struct kmem_cache *, const char *);
254static void sysfs_slab_remove(struct kmem_cache *);
255#else
Christoph Lameter0c710012007-07-17 04:03:24 -0700256static inline int sysfs_slab_add(struct kmem_cache *s) { return 0; }
257static inline int sysfs_slab_alias(struct kmem_cache *s, const char *p)
258 { return 0; }
259static inline void sysfs_slab_remove(struct kmem_cache *s) {}
Christoph Lameter81819f02007-05-06 14:49:36 -0700260#endif
261
262/********************************************************************
263 * Core slab cache functions
264 *******************************************************************/
265
266int slab_is_available(void)
267{
268 return slab_state >= UP;
269}
270
271static inline struct kmem_cache_node *get_node(struct kmem_cache *s, int node)
272{
273#ifdef CONFIG_NUMA
274 return s->node[node];
275#else
276 return &s->local_node;
277#endif
278}
279
Christoph Lameter02cbc872007-05-09 02:32:43 -0700280static inline int check_valid_pointer(struct kmem_cache *s,
281 struct page *page, const void *object)
282{
283 void *base;
284
285 if (!object)
286 return 1;
287
288 base = page_address(page);
289 if (object < base || object >= base + s->objects * s->size ||
290 (object - base) % s->size) {
291 return 0;
292 }
293
294 return 1;
295}
296
Christoph Lameter81819f02007-05-06 14:49:36 -0700297/*
Christoph Lameter7656c722007-05-09 02:32:40 -0700298 * Slow version of get and set free pointer.
299 *
300 * This version requires touching the cache lines of kmem_cache which
301 * we avoid to do in the fast alloc free paths. There we obtain the offset
302 * from the page struct.
303 */
304static inline void *get_freepointer(struct kmem_cache *s, void *object)
305{
306 return *(void **)(object + s->offset);
307}
308
309static inline void set_freepointer(struct kmem_cache *s, void *object, void *fp)
310{
311 *(void **)(object + s->offset) = fp;
312}
313
314/* Loop over all objects in a slab */
315#define for_each_object(__p, __s, __addr) \
316 for (__p = (__addr); __p < (__addr) + (__s)->objects * (__s)->size;\
317 __p += (__s)->size)
318
319/* Scan freelist */
320#define for_each_free_object(__p, __s, __free) \
321 for (__p = (__free); __p; __p = get_freepointer((__s), __p))
322
323/* Determine object index from a given position */
324static inline int slab_index(void *p, struct kmem_cache *s, void *addr)
325{
326 return (p - addr) / s->size;
327}
328
Christoph Lameter41ecc552007-05-09 02:32:44 -0700329#ifdef CONFIG_SLUB_DEBUG
330/*
331 * Debug settings:
332 */
Christoph Lameterf0630ff2007-07-15 23:38:14 -0700333#ifdef CONFIG_SLUB_DEBUG_ON
334static int slub_debug = DEBUG_DEFAULT_FLAGS;
335#else
Christoph Lameter41ecc552007-05-09 02:32:44 -0700336static int slub_debug;
Christoph Lameterf0630ff2007-07-15 23:38:14 -0700337#endif
Christoph Lameter41ecc552007-05-09 02:32:44 -0700338
339static char *slub_debug_slabs;
340
Christoph Lameter7656c722007-05-09 02:32:40 -0700341/*
Christoph Lameter81819f02007-05-06 14:49:36 -0700342 * Object debugging
343 */
344static void print_section(char *text, u8 *addr, unsigned int length)
345{
346 int i, offset;
347 int newline = 1;
348 char ascii[17];
349
350 ascii[16] = 0;
351
352 for (i = 0; i < length; i++) {
353 if (newline) {
Christoph Lameter24922682007-07-17 04:03:18 -0700354 printk(KERN_ERR "%8s 0x%p: ", text, addr + i);
Christoph Lameter81819f02007-05-06 14:49:36 -0700355 newline = 0;
356 }
357 printk(" %02x", addr[i]);
358 offset = i % 16;
359 ascii[offset] = isgraph(addr[i]) ? addr[i] : '.';
360 if (offset == 15) {
361 printk(" %s\n",ascii);
362 newline = 1;
363 }
364 }
365 if (!newline) {
366 i %= 16;
367 while (i < 16) {
368 printk(" ");
369 ascii[i] = ' ';
370 i++;
371 }
372 printk(" %s\n", ascii);
373 }
374}
375
Christoph Lameter81819f02007-05-06 14:49:36 -0700376static struct track *get_track(struct kmem_cache *s, void *object,
377 enum track_item alloc)
378{
379 struct track *p;
380
381 if (s->offset)
382 p = object + s->offset + sizeof(void *);
383 else
384 p = object + s->inuse;
385
386 return p + alloc;
387}
388
389static void set_track(struct kmem_cache *s, void *object,
390 enum track_item alloc, void *addr)
391{
392 struct track *p;
393
394 if (s->offset)
395 p = object + s->offset + sizeof(void *);
396 else
397 p = object + s->inuse;
398
399 p += alloc;
400 if (addr) {
401 p->addr = addr;
402 p->cpu = smp_processor_id();
403 p->pid = current ? current->pid : -1;
404 p->when = jiffies;
405 } else
406 memset(p, 0, sizeof(struct track));
407}
408
Christoph Lameter81819f02007-05-06 14:49:36 -0700409static void init_tracking(struct kmem_cache *s, void *object)
410{
Christoph Lameter24922682007-07-17 04:03:18 -0700411 if (!(s->flags & SLAB_STORE_USER))
412 return;
413
414 set_track(s, object, TRACK_FREE, NULL);
415 set_track(s, object, TRACK_ALLOC, NULL);
Christoph Lameter81819f02007-05-06 14:49:36 -0700416}
417
418static void print_track(const char *s, struct track *t)
419{
420 if (!t->addr)
421 return;
422
Christoph Lameter24922682007-07-17 04:03:18 -0700423 printk(KERN_ERR "INFO: %s in ", s);
Christoph Lameter81819f02007-05-06 14:49:36 -0700424 __print_symbol("%s", (unsigned long)t->addr);
Christoph Lameter24922682007-07-17 04:03:18 -0700425 printk(" age=%lu cpu=%u pid=%d\n", jiffies - t->when, t->cpu, t->pid);
Christoph Lameter81819f02007-05-06 14:49:36 -0700426}
427
Christoph Lameter24922682007-07-17 04:03:18 -0700428static void print_tracking(struct kmem_cache *s, void *object)
429{
430 if (!(s->flags & SLAB_STORE_USER))
431 return;
432
433 print_track("Allocated", get_track(s, object, TRACK_ALLOC));
434 print_track("Freed", get_track(s, object, TRACK_FREE));
435}
436
437static void print_page_info(struct page *page)
438{
439 printk(KERN_ERR "INFO: Slab 0x%p used=%u fp=0x%p flags=0x%04lx\n",
440 page, page->inuse, page->freelist, page->flags);
441
442}
443
444static void slab_bug(struct kmem_cache *s, char *fmt, ...)
445{
446 va_list args;
447 char buf[100];
448
449 va_start(args, fmt);
450 vsnprintf(buf, sizeof(buf), fmt, args);
451 va_end(args);
452 printk(KERN_ERR "========================================"
453 "=====================================\n");
454 printk(KERN_ERR "BUG %s: %s\n", s->name, buf);
455 printk(KERN_ERR "----------------------------------------"
456 "-------------------------------------\n\n");
457}
458
459static void slab_fix(struct kmem_cache *s, char *fmt, ...)
460{
461 va_list args;
462 char buf[100];
463
464 va_start(args, fmt);
465 vsnprintf(buf, sizeof(buf), fmt, args);
466 va_end(args);
467 printk(KERN_ERR "FIX %s: %s\n", s->name, buf);
468}
469
470static void print_trailer(struct kmem_cache *s, struct page *page, u8 *p)
Christoph Lameter81819f02007-05-06 14:49:36 -0700471{
472 unsigned int off; /* Offset of last byte */
Christoph Lameter24922682007-07-17 04:03:18 -0700473 u8 *addr = page_address(page);
474
475 print_tracking(s, p);
476
477 print_page_info(page);
478
479 printk(KERN_ERR "INFO: Object 0x%p @offset=%tu fp=0x%p\n\n",
480 p, p - addr, get_freepointer(s, p));
481
482 if (p > addr + 16)
483 print_section("Bytes b4", p - 16, 16);
484
485 print_section("Object", p, min(s->objsize, 128));
Christoph Lameter81819f02007-05-06 14:49:36 -0700486
487 if (s->flags & SLAB_RED_ZONE)
488 print_section("Redzone", p + s->objsize,
489 s->inuse - s->objsize);
490
Christoph Lameter81819f02007-05-06 14:49:36 -0700491 if (s->offset)
492 off = s->offset + sizeof(void *);
493 else
494 off = s->inuse;
495
Christoph Lameter24922682007-07-17 04:03:18 -0700496 if (s->flags & SLAB_STORE_USER)
Christoph Lameter81819f02007-05-06 14:49:36 -0700497 off += 2 * sizeof(struct track);
Christoph Lameter81819f02007-05-06 14:49:36 -0700498
499 if (off != s->size)
500 /* Beginning of the filler is the free pointer */
Christoph Lameter24922682007-07-17 04:03:18 -0700501 print_section("Padding", p + off, s->size - off);
502
503 dump_stack();
Christoph Lameter81819f02007-05-06 14:49:36 -0700504}
505
506static void object_err(struct kmem_cache *s, struct page *page,
507 u8 *object, char *reason)
508{
Christoph Lameter24922682007-07-17 04:03:18 -0700509 slab_bug(s, reason);
510 print_trailer(s, page, object);
Christoph Lameter81819f02007-05-06 14:49:36 -0700511}
512
Christoph Lameter24922682007-07-17 04:03:18 -0700513static void slab_err(struct kmem_cache *s, struct page *page, char *fmt, ...)
Christoph Lameter81819f02007-05-06 14:49:36 -0700514{
515 va_list args;
516 char buf[100];
517
Christoph Lameter24922682007-07-17 04:03:18 -0700518 va_start(args, fmt);
519 vsnprintf(buf, sizeof(buf), fmt, args);
Christoph Lameter81819f02007-05-06 14:49:36 -0700520 va_end(args);
Christoph Lameter24922682007-07-17 04:03:18 -0700521 slab_bug(s, fmt);
522 print_page_info(page);
Christoph Lameter81819f02007-05-06 14:49:36 -0700523 dump_stack();
524}
525
526static void init_object(struct kmem_cache *s, void *object, int active)
527{
528 u8 *p = object;
529
530 if (s->flags & __OBJECT_POISON) {
531 memset(p, POISON_FREE, s->objsize - 1);
532 p[s->objsize -1] = POISON_END;
533 }
534
535 if (s->flags & SLAB_RED_ZONE)
536 memset(p + s->objsize,
537 active ? SLUB_RED_ACTIVE : SLUB_RED_INACTIVE,
538 s->inuse - s->objsize);
539}
540
Christoph Lameter24922682007-07-17 04:03:18 -0700541static u8 *check_bytes(u8 *start, unsigned int value, unsigned int bytes)
Christoph Lameter81819f02007-05-06 14:49:36 -0700542{
543 while (bytes) {
544 if (*start != (u8)value)
Christoph Lameter24922682007-07-17 04:03:18 -0700545 return start;
Christoph Lameter81819f02007-05-06 14:49:36 -0700546 start++;
547 bytes--;
548 }
Christoph Lameter24922682007-07-17 04:03:18 -0700549 return NULL;
550}
551
552static void restore_bytes(struct kmem_cache *s, char *message, u8 data,
553 void *from, void *to)
554{
555 slab_fix(s, "Restoring 0x%p-0x%p=0x%x\n", from, to - 1, data);
556 memset(from, data, to - from);
557}
558
559static int check_bytes_and_report(struct kmem_cache *s, struct page *page,
560 u8 *object, char *what,
561 u8* start, unsigned int value, unsigned int bytes)
562{
563 u8 *fault;
564 u8 *end;
565
566 fault = check_bytes(start, value, bytes);
567 if (!fault)
568 return 1;
569
570 end = start + bytes;
571 while (end > fault && end[-1] == value)
572 end--;
573
574 slab_bug(s, "%s overwritten", what);
575 printk(KERN_ERR "INFO: 0x%p-0x%p. First byte 0x%x instead of 0x%x\n",
576 fault, end - 1, fault[0], value);
577 print_trailer(s, page, object);
578
579 restore_bytes(s, what, value, fault, end);
580 return 0;
Christoph Lameter81819f02007-05-06 14:49:36 -0700581}
582
Christoph Lameter81819f02007-05-06 14:49:36 -0700583/*
584 * Object layout:
585 *
586 * object address
587 * Bytes of the object to be managed.
588 * If the freepointer may overlay the object then the free
589 * pointer is the first word of the object.
Christoph Lameter672bba32007-05-09 02:32:39 -0700590 *
Christoph Lameter81819f02007-05-06 14:49:36 -0700591 * Poisoning uses 0x6b (POISON_FREE) and the last byte is
592 * 0xa5 (POISON_END)
593 *
594 * object + s->objsize
595 * Padding to reach word boundary. This is also used for Redzoning.
Christoph Lameter672bba32007-05-09 02:32:39 -0700596 * Padding is extended by another word if Redzoning is enabled and
597 * objsize == inuse.
598 *
Christoph Lameter81819f02007-05-06 14:49:36 -0700599 * We fill with 0xbb (RED_INACTIVE) for inactive objects and with
600 * 0xcc (RED_ACTIVE) for objects in use.
601 *
602 * object + s->inuse
Christoph Lameter672bba32007-05-09 02:32:39 -0700603 * Meta data starts here.
604 *
Christoph Lameter81819f02007-05-06 14:49:36 -0700605 * A. Free pointer (if we cannot overwrite object on free)
606 * B. Tracking data for SLAB_STORE_USER
Christoph Lameter672bba32007-05-09 02:32:39 -0700607 * C. Padding to reach required alignment boundary or at mininum
608 * one word if debuggin is on to be able to detect writes
609 * before the word boundary.
610 *
611 * Padding is done using 0x5a (POISON_INUSE)
Christoph Lameter81819f02007-05-06 14:49:36 -0700612 *
613 * object + s->size
Christoph Lameter672bba32007-05-09 02:32:39 -0700614 * Nothing is used beyond s->size.
Christoph Lameter81819f02007-05-06 14:49:36 -0700615 *
Christoph Lameter672bba32007-05-09 02:32:39 -0700616 * If slabcaches are merged then the objsize and inuse boundaries are mostly
617 * ignored. And therefore no slab options that rely on these boundaries
Christoph Lameter81819f02007-05-06 14:49:36 -0700618 * may be used with merged slabcaches.
619 */
620
Christoph Lameter81819f02007-05-06 14:49:36 -0700621static int check_pad_bytes(struct kmem_cache *s, struct page *page, u8 *p)
622{
623 unsigned long off = s->inuse; /* The end of info */
624
625 if (s->offset)
626 /* Freepointer is placed after the object. */
627 off += sizeof(void *);
628
629 if (s->flags & SLAB_STORE_USER)
630 /* We also have user information there */
631 off += 2 * sizeof(struct track);
632
633 if (s->size == off)
634 return 1;
635
Christoph Lameter24922682007-07-17 04:03:18 -0700636 return check_bytes_and_report(s, page, p, "Object padding",
637 p + off, POISON_INUSE, s->size - off);
Christoph Lameter81819f02007-05-06 14:49:36 -0700638}
639
640static int slab_pad_check(struct kmem_cache *s, struct page *page)
641{
Christoph Lameter24922682007-07-17 04:03:18 -0700642 u8 *start;
643 u8 *fault;
644 u8 *end;
645 int length;
646 int remainder;
Christoph Lameter81819f02007-05-06 14:49:36 -0700647
648 if (!(s->flags & SLAB_POISON))
649 return 1;
650
Christoph Lameter24922682007-07-17 04:03:18 -0700651 start = page_address(page);
652 end = start + (PAGE_SIZE << s->order);
Christoph Lameter81819f02007-05-06 14:49:36 -0700653 length = s->objects * s->size;
Christoph Lameter24922682007-07-17 04:03:18 -0700654 remainder = end - (start + length);
Christoph Lameter81819f02007-05-06 14:49:36 -0700655 if (!remainder)
656 return 1;
657
Christoph Lameter24922682007-07-17 04:03:18 -0700658 fault = check_bytes(start + length, POISON_INUSE, remainder);
659 if (!fault)
660 return 1;
661 while (end > fault && end[-1] == POISON_INUSE)
662 end--;
663
664 slab_err(s, page, "Padding overwritten. 0x%p-0x%p", fault, end - 1);
665 print_section("Padding", start, length);
666
667 restore_bytes(s, "slab padding", POISON_INUSE, start, end);
668 return 0;
Christoph Lameter81819f02007-05-06 14:49:36 -0700669}
670
671static int check_object(struct kmem_cache *s, struct page *page,
672 void *object, int active)
673{
674 u8 *p = object;
675 u8 *endobject = object + s->objsize;
676
677 if (s->flags & SLAB_RED_ZONE) {
678 unsigned int red =
679 active ? SLUB_RED_ACTIVE : SLUB_RED_INACTIVE;
680
Christoph Lameter24922682007-07-17 04:03:18 -0700681 if (!check_bytes_and_report(s, page, object, "Redzone",
682 endobject, red, s->inuse - s->objsize))
Christoph Lameter81819f02007-05-06 14:49:36 -0700683 return 0;
Christoph Lameter81819f02007-05-06 14:49:36 -0700684 } else {
Christoph Lameter24922682007-07-17 04:03:18 -0700685 if ((s->flags & SLAB_POISON) && s->objsize < s->inuse)
686 check_bytes_and_report(s, page, p, "Alignment padding", endobject,
687 POISON_INUSE, s->inuse - s->objsize);
Christoph Lameter81819f02007-05-06 14:49:36 -0700688 }
689
690 if (s->flags & SLAB_POISON) {
691 if (!active && (s->flags & __OBJECT_POISON) &&
Christoph Lameter24922682007-07-17 04:03:18 -0700692 (!check_bytes_and_report(s, page, p, "Poison", p,
693 POISON_FREE, s->objsize - 1) ||
694 !check_bytes_and_report(s, page, p, "Poison",
695 p + s->objsize -1, POISON_END, 1)))
Christoph Lameter81819f02007-05-06 14:49:36 -0700696 return 0;
Christoph Lameter81819f02007-05-06 14:49:36 -0700697 /*
698 * check_pad_bytes cleans up on its own.
699 */
700 check_pad_bytes(s, page, p);
701 }
702
703 if (!s->offset && active)
704 /*
705 * Object and freepointer overlap. Cannot check
706 * freepointer while object is allocated.
707 */
708 return 1;
709
710 /* Check free pointer validity */
711 if (!check_valid_pointer(s, page, get_freepointer(s, p))) {
712 object_err(s, page, p, "Freepointer corrupt");
713 /*
714 * No choice but to zap it and thus loose the remainder
715 * of the free objects in this slab. May cause
Christoph Lameter672bba32007-05-09 02:32:39 -0700716 * another error because the object count is now wrong.
Christoph Lameter81819f02007-05-06 14:49:36 -0700717 */
718 set_freepointer(s, p, NULL);
719 return 0;
720 }
721 return 1;
722}
723
724static int check_slab(struct kmem_cache *s, struct page *page)
725{
726 VM_BUG_ON(!irqs_disabled());
727
728 if (!PageSlab(page)) {
Christoph Lameter24922682007-07-17 04:03:18 -0700729 slab_err(s, page, "Not a valid slab page");
Christoph Lameter81819f02007-05-06 14:49:36 -0700730 return 0;
731 }
732 if (page->offset * sizeof(void *) != s->offset) {
Christoph Lameter24922682007-07-17 04:03:18 -0700733 slab_err(s, page, "Corrupted offset %lu",
734 (unsigned long)(page->offset * sizeof(void *)));
Christoph Lameter81819f02007-05-06 14:49:36 -0700735 return 0;
736 }
737 if (page->inuse > s->objects) {
Christoph Lameter24922682007-07-17 04:03:18 -0700738 slab_err(s, page, "inuse %u > max %u",
739 s->name, page->inuse, s->objects);
Christoph Lameter81819f02007-05-06 14:49:36 -0700740 return 0;
741 }
742 /* Slab_pad_check fixes things up after itself */
743 slab_pad_check(s, page);
744 return 1;
745}
746
747/*
Christoph Lameter672bba32007-05-09 02:32:39 -0700748 * Determine if a certain object on a page is on the freelist. Must hold the
749 * slab lock to guarantee that the chains are in a consistent state.
Christoph Lameter81819f02007-05-06 14:49:36 -0700750 */
751static int on_freelist(struct kmem_cache *s, struct page *page, void *search)
752{
753 int nr = 0;
754 void *fp = page->freelist;
755 void *object = NULL;
756
757 while (fp && nr <= s->objects) {
758 if (fp == search)
759 return 1;
760 if (!check_valid_pointer(s, page, fp)) {
761 if (object) {
762 object_err(s, page, object,
763 "Freechain corrupt");
764 set_freepointer(s, object, NULL);
765 break;
766 } else {
Christoph Lameter24922682007-07-17 04:03:18 -0700767 slab_err(s, page, "Freepointer corrupt");
Christoph Lameter81819f02007-05-06 14:49:36 -0700768 page->freelist = NULL;
769 page->inuse = s->objects;
Christoph Lameter24922682007-07-17 04:03:18 -0700770 slab_fix(s, "Freelist cleared");
Christoph Lameter81819f02007-05-06 14:49:36 -0700771 return 0;
772 }
773 break;
774 }
775 object = fp;
776 fp = get_freepointer(s, object);
777 nr++;
778 }
779
780 if (page->inuse != s->objects - nr) {
Christoph Lameter70d71222007-05-06 14:49:47 -0700781 slab_err(s, page, "Wrong object count. Counter is %d but "
Christoph Lameter24922682007-07-17 04:03:18 -0700782 "counted were %d", page->inuse, s->objects - nr);
Christoph Lameter81819f02007-05-06 14:49:36 -0700783 page->inuse = s->objects - nr;
Christoph Lameter24922682007-07-17 04:03:18 -0700784 slab_fix(s, "Object count adjusted.");
Christoph Lameter81819f02007-05-06 14:49:36 -0700785 }
786 return search == NULL;
787}
788
Christoph Lameter3ec09742007-05-16 22:11:00 -0700789static void trace(struct kmem_cache *s, struct page *page, void *object, int alloc)
790{
791 if (s->flags & SLAB_TRACE) {
792 printk(KERN_INFO "TRACE %s %s 0x%p inuse=%d fp=0x%p\n",
793 s->name,
794 alloc ? "alloc" : "free",
795 object, page->inuse,
796 page->freelist);
797
798 if (!alloc)
799 print_section("Object", (void *)object, s->objsize);
800
801 dump_stack();
802 }
803}
804
Christoph Lameter643b1132007-05-06 14:49:42 -0700805/*
Christoph Lameter672bba32007-05-09 02:32:39 -0700806 * Tracking of fully allocated slabs for debugging purposes.
Christoph Lameter643b1132007-05-06 14:49:42 -0700807 */
Christoph Lametere95eed52007-05-06 14:49:44 -0700808static void add_full(struct kmem_cache_node *n, struct page *page)
Christoph Lameter643b1132007-05-06 14:49:42 -0700809{
Christoph Lameter643b1132007-05-06 14:49:42 -0700810 spin_lock(&n->list_lock);
811 list_add(&page->lru, &n->full);
812 spin_unlock(&n->list_lock);
813}
814
815static void remove_full(struct kmem_cache *s, struct page *page)
816{
817 struct kmem_cache_node *n;
818
819 if (!(s->flags & SLAB_STORE_USER))
820 return;
821
822 n = get_node(s, page_to_nid(page));
823
824 spin_lock(&n->list_lock);
825 list_del(&page->lru);
826 spin_unlock(&n->list_lock);
827}
828
Christoph Lameter3ec09742007-05-16 22:11:00 -0700829static void setup_object_debug(struct kmem_cache *s, struct page *page,
830 void *object)
831{
832 if (!(s->flags & (SLAB_STORE_USER|SLAB_RED_ZONE|__OBJECT_POISON)))
833 return;
834
835 init_object(s, object, 0);
836 init_tracking(s, object);
837}
838
839static int alloc_debug_processing(struct kmem_cache *s, struct page *page,
840 void *object, void *addr)
Christoph Lameter81819f02007-05-06 14:49:36 -0700841{
842 if (!check_slab(s, page))
843 goto bad;
844
845 if (object && !on_freelist(s, page, object)) {
Christoph Lameter24922682007-07-17 04:03:18 -0700846 object_err(s, page, object, "Object already allocated");
Christoph Lameter70d71222007-05-06 14:49:47 -0700847 goto bad;
Christoph Lameter81819f02007-05-06 14:49:36 -0700848 }
849
850 if (!check_valid_pointer(s, page, object)) {
851 object_err(s, page, object, "Freelist Pointer check fails");
Christoph Lameter70d71222007-05-06 14:49:47 -0700852 goto bad;
Christoph Lameter81819f02007-05-06 14:49:36 -0700853 }
854
Christoph Lameter3ec09742007-05-16 22:11:00 -0700855 if (object && !check_object(s, page, object, 0))
Christoph Lameter81819f02007-05-06 14:49:36 -0700856 goto bad;
Christoph Lameter81819f02007-05-06 14:49:36 -0700857
Christoph Lameter3ec09742007-05-16 22:11:00 -0700858 /* Success perform special debug activities for allocs */
859 if (s->flags & SLAB_STORE_USER)
860 set_track(s, object, TRACK_ALLOC, addr);
861 trace(s, page, object, 1);
862 init_object(s, object, 1);
Christoph Lameter81819f02007-05-06 14:49:36 -0700863 return 1;
Christoph Lameter3ec09742007-05-16 22:11:00 -0700864
Christoph Lameter81819f02007-05-06 14:49:36 -0700865bad:
866 if (PageSlab(page)) {
867 /*
868 * If this is a slab page then lets do the best we can
869 * to avoid issues in the future. Marking all objects
Christoph Lameter672bba32007-05-09 02:32:39 -0700870 * as used avoids touching the remaining objects.
Christoph Lameter81819f02007-05-06 14:49:36 -0700871 */
Christoph Lameter24922682007-07-17 04:03:18 -0700872 slab_fix(s, "Marking all objects used");
Christoph Lameter81819f02007-05-06 14:49:36 -0700873 page->inuse = s->objects;
874 page->freelist = NULL;
875 /* Fix up fields that may be corrupted */
876 page->offset = s->offset / sizeof(void *);
877 }
878 return 0;
879}
880
Christoph Lameter3ec09742007-05-16 22:11:00 -0700881static int free_debug_processing(struct kmem_cache *s, struct page *page,
882 void *object, void *addr)
Christoph Lameter81819f02007-05-06 14:49:36 -0700883{
884 if (!check_slab(s, page))
885 goto fail;
886
887 if (!check_valid_pointer(s, page, object)) {
Christoph Lameter70d71222007-05-06 14:49:47 -0700888 slab_err(s, page, "Invalid object pointer 0x%p", object);
Christoph Lameter81819f02007-05-06 14:49:36 -0700889 goto fail;
890 }
891
892 if (on_freelist(s, page, object)) {
Christoph Lameter24922682007-07-17 04:03:18 -0700893 object_err(s, page, object, "Object already free");
Christoph Lameter81819f02007-05-06 14:49:36 -0700894 goto fail;
895 }
896
897 if (!check_object(s, page, object, 1))
898 return 0;
899
900 if (unlikely(s != page->slab)) {
901 if (!PageSlab(page))
Christoph Lameter70d71222007-05-06 14:49:47 -0700902 slab_err(s, page, "Attempt to free object(0x%p) "
903 "outside of slab", object);
Christoph Lameter81819f02007-05-06 14:49:36 -0700904 else
Christoph Lameter70d71222007-05-06 14:49:47 -0700905 if (!page->slab) {
Christoph Lameter81819f02007-05-06 14:49:36 -0700906 printk(KERN_ERR
Christoph Lameter70d71222007-05-06 14:49:47 -0700907 "SLUB <none>: no slab for object 0x%p.\n",
Christoph Lameter81819f02007-05-06 14:49:36 -0700908 object);
Christoph Lameter70d71222007-05-06 14:49:47 -0700909 dump_stack();
910 }
Christoph Lameter81819f02007-05-06 14:49:36 -0700911 else
Christoph Lameter24922682007-07-17 04:03:18 -0700912 object_err(s, page, object,
913 "page slab pointer corrupt.");
Christoph Lameter81819f02007-05-06 14:49:36 -0700914 goto fail;
915 }
Christoph Lameter3ec09742007-05-16 22:11:00 -0700916
917 /* Special debug activities for freeing objects */
918 if (!SlabFrozen(page) && !page->freelist)
919 remove_full(s, page);
920 if (s->flags & SLAB_STORE_USER)
921 set_track(s, object, TRACK_FREE, addr);
922 trace(s, page, object, 0);
923 init_object(s, object, 0);
Christoph Lameter81819f02007-05-06 14:49:36 -0700924 return 1;
Christoph Lameter3ec09742007-05-16 22:11:00 -0700925
Christoph Lameter81819f02007-05-06 14:49:36 -0700926fail:
Christoph Lameter24922682007-07-17 04:03:18 -0700927 slab_fix(s, "Object at 0x%p not freed", object);
Christoph Lameter81819f02007-05-06 14:49:36 -0700928 return 0;
929}
930
Christoph Lameter41ecc552007-05-09 02:32:44 -0700931static int __init setup_slub_debug(char *str)
932{
Christoph Lameterf0630ff2007-07-15 23:38:14 -0700933 slub_debug = DEBUG_DEFAULT_FLAGS;
934 if (*str++ != '=' || !*str)
935 /*
936 * No options specified. Switch on full debugging.
937 */
938 goto out;
Christoph Lameter41ecc552007-05-09 02:32:44 -0700939
940 if (*str == ',')
Christoph Lameterf0630ff2007-07-15 23:38:14 -0700941 /*
942 * No options but restriction on slabs. This means full
943 * debugging for slabs matching a pattern.
944 */
945 goto check_slabs;
946
947 slub_debug = 0;
948 if (*str == '-')
949 /*
950 * Switch off all debugging measures.
951 */
952 goto out;
953
954 /*
955 * Determine which debug features should be switched on
956 */
957 for ( ;*str && *str != ','; str++) {
958 switch (tolower(*str)) {
959 case 'f':
960 slub_debug |= SLAB_DEBUG_FREE;
961 break;
962 case 'z':
963 slub_debug |= SLAB_RED_ZONE;
964 break;
965 case 'p':
966 slub_debug |= SLAB_POISON;
967 break;
968 case 'u':
969 slub_debug |= SLAB_STORE_USER;
970 break;
971 case 't':
972 slub_debug |= SLAB_TRACE;
973 break;
974 default:
975 printk(KERN_ERR "slub_debug option '%c' "
976 "unknown. skipped\n",*str);
977 }
978 }
979
980check_slabs:
981 if (*str == ',')
Christoph Lameter41ecc552007-05-09 02:32:44 -0700982 slub_debug_slabs = str + 1;
Christoph Lameterf0630ff2007-07-15 23:38:14 -0700983out:
Christoph Lameter41ecc552007-05-09 02:32:44 -0700984 return 1;
985}
986
987__setup("slub_debug", setup_slub_debug);
988
Christoph Lameterba0268a2007-09-11 15:24:11 -0700989static unsigned long kmem_cache_flags(unsigned long objsize,
990 unsigned long flags, const char *name,
991 void (*ctor)(void *, struct kmem_cache *, unsigned long))
Christoph Lameter41ecc552007-05-09 02:32:44 -0700992{
993 /*
994 * The page->offset field is only 16 bit wide. This is an offset
995 * in units of words from the beginning of an object. If the slab
996 * size is bigger then we cannot move the free pointer behind the
997 * object anymore.
998 *
999 * On 32 bit platforms the limit is 256k. On 64bit platforms
1000 * the limit is 512k.
1001 *
Christoph Lameterc59def9f2007-05-16 22:10:50 -07001002 * Debugging or ctor may create a need to move the free
Christoph Lameter41ecc552007-05-09 02:32:44 -07001003 * pointer. Fail if this happens.
1004 */
Christoph Lameterba0268a2007-09-11 15:24:11 -07001005 if (objsize >= 65535 * sizeof(void *)) {
1006 BUG_ON(flags & (SLAB_RED_ZONE | SLAB_POISON |
Christoph Lameter41ecc552007-05-09 02:32:44 -07001007 SLAB_STORE_USER | SLAB_DESTROY_BY_RCU));
Christoph Lameterba0268a2007-09-11 15:24:11 -07001008 BUG_ON(ctor);
1009 } else {
Christoph Lameter41ecc552007-05-09 02:32:44 -07001010 /*
1011 * Enable debugging if selected on the kernel commandline.
1012 */
1013 if (slub_debug && (!slub_debug_slabs ||
Christoph Lameterba0268a2007-09-11 15:24:11 -07001014 strncmp(slub_debug_slabs, name,
Christoph Lameter41ecc552007-05-09 02:32:44 -07001015 strlen(slub_debug_slabs)) == 0))
Christoph Lameterba0268a2007-09-11 15:24:11 -07001016 flags |= slub_debug;
1017 }
1018
1019 return flags;
Christoph Lameter41ecc552007-05-09 02:32:44 -07001020}
1021#else
Christoph Lameter3ec09742007-05-16 22:11:00 -07001022static inline void setup_object_debug(struct kmem_cache *s,
1023 struct page *page, void *object) {}
Christoph Lameter41ecc552007-05-09 02:32:44 -07001024
Christoph Lameter3ec09742007-05-16 22:11:00 -07001025static inline int alloc_debug_processing(struct kmem_cache *s,
1026 struct page *page, void *object, void *addr) { return 0; }
Christoph Lameter41ecc552007-05-09 02:32:44 -07001027
Christoph Lameter3ec09742007-05-16 22:11:00 -07001028static inline int free_debug_processing(struct kmem_cache *s,
1029 struct page *page, void *object, void *addr) { return 0; }
Christoph Lameter41ecc552007-05-09 02:32:44 -07001030
Christoph Lameter41ecc552007-05-09 02:32:44 -07001031static inline int slab_pad_check(struct kmem_cache *s, struct page *page)
1032 { return 1; }
1033static inline int check_object(struct kmem_cache *s, struct page *page,
1034 void *object, int active) { return 1; }
Christoph Lameter3ec09742007-05-16 22:11:00 -07001035static inline void add_full(struct kmem_cache_node *n, struct page *page) {}
Christoph Lameterba0268a2007-09-11 15:24:11 -07001036static inline unsigned long kmem_cache_flags(unsigned long objsize,
1037 unsigned long flags, const char *name,
1038 void (*ctor)(void *, struct kmem_cache *, unsigned long))
1039{
1040 return flags;
1041}
Christoph Lameter41ecc552007-05-09 02:32:44 -07001042#define slub_debug 0
1043#endif
Christoph Lameter81819f02007-05-06 14:49:36 -07001044/*
1045 * Slab allocation and freeing
1046 */
1047static struct page *allocate_slab(struct kmem_cache *s, gfp_t flags, int node)
1048{
1049 struct page * page;
1050 int pages = 1 << s->order;
1051
1052 if (s->order)
1053 flags |= __GFP_COMP;
1054
1055 if (s->flags & SLAB_CACHE_DMA)
1056 flags |= SLUB_DMA;
1057
1058 if (node == -1)
1059 page = alloc_pages(flags, s->order);
1060 else
1061 page = alloc_pages_node(node, flags, s->order);
1062
1063 if (!page)
1064 return NULL;
1065
1066 mod_zone_page_state(page_zone(page),
1067 (s->flags & SLAB_RECLAIM_ACCOUNT) ?
1068 NR_SLAB_RECLAIMABLE : NR_SLAB_UNRECLAIMABLE,
1069 pages);
1070
1071 return page;
1072}
1073
1074static void setup_object(struct kmem_cache *s, struct page *page,
1075 void *object)
1076{
Christoph Lameter3ec09742007-05-16 22:11:00 -07001077 setup_object_debug(s, page, object);
Christoph Lameter4f104932007-05-06 14:50:17 -07001078 if (unlikely(s->ctor))
Christoph Lametera35afb82007-05-16 22:10:57 -07001079 s->ctor(object, s, 0);
Christoph Lameter81819f02007-05-06 14:49:36 -07001080}
1081
1082static struct page *new_slab(struct kmem_cache *s, gfp_t flags, int node)
1083{
1084 struct page *page;
1085 struct kmem_cache_node *n;
1086 void *start;
1087 void *end;
1088 void *last;
1089 void *p;
1090
Christoph Lameterd07dbea2007-07-17 04:03:23 -07001091 BUG_ON(flags & ~(GFP_DMA | __GFP_ZERO | GFP_LEVEL_MASK));
Christoph Lameter81819f02007-05-06 14:49:36 -07001092
1093 if (flags & __GFP_WAIT)
1094 local_irq_enable();
1095
1096 page = allocate_slab(s, flags & GFP_LEVEL_MASK, node);
1097 if (!page)
1098 goto out;
1099
1100 n = get_node(s, page_to_nid(page));
1101 if (n)
1102 atomic_long_inc(&n->nr_slabs);
1103 page->offset = s->offset / sizeof(void *);
1104 page->slab = s;
1105 page->flags |= 1 << PG_slab;
1106 if (s->flags & (SLAB_DEBUG_FREE | SLAB_RED_ZONE | SLAB_POISON |
1107 SLAB_STORE_USER | SLAB_TRACE))
Christoph Lameter35e5d7e2007-05-09 02:32:42 -07001108 SetSlabDebug(page);
Christoph Lameter81819f02007-05-06 14:49:36 -07001109
1110 start = page_address(page);
1111 end = start + s->objects * s->size;
1112
1113 if (unlikely(s->flags & SLAB_POISON))
1114 memset(start, POISON_INUSE, PAGE_SIZE << s->order);
1115
1116 last = start;
Christoph Lameter7656c722007-05-09 02:32:40 -07001117 for_each_object(p, s, start) {
Christoph Lameter81819f02007-05-06 14:49:36 -07001118 setup_object(s, page, last);
1119 set_freepointer(s, last, p);
1120 last = p;
1121 }
1122 setup_object(s, page, last);
1123 set_freepointer(s, last, NULL);
1124
1125 page->freelist = start;
Christoph Lameter894b8782007-05-10 03:15:16 -07001126 page->lockless_freelist = NULL;
Christoph Lameter81819f02007-05-06 14:49:36 -07001127 page->inuse = 0;
1128out:
1129 if (flags & __GFP_WAIT)
1130 local_irq_disable();
1131 return page;
1132}
1133
1134static void __free_slab(struct kmem_cache *s, struct page *page)
1135{
1136 int pages = 1 << s->order;
1137
Christoph Lameterc59def9f2007-05-16 22:10:50 -07001138 if (unlikely(SlabDebug(page))) {
Christoph Lameter81819f02007-05-06 14:49:36 -07001139 void *p;
1140
1141 slab_pad_check(s, page);
Christoph Lameterc59def9f2007-05-16 22:10:50 -07001142 for_each_object(p, s, page_address(page))
Christoph Lameter81819f02007-05-06 14:49:36 -07001143 check_object(s, page, p, 0);
Peter Zijlstra2208b762007-07-26 20:54:34 +02001144 ClearSlabDebug(page);
Christoph Lameter81819f02007-05-06 14:49:36 -07001145 }
1146
1147 mod_zone_page_state(page_zone(page),
1148 (s->flags & SLAB_RECLAIM_ACCOUNT) ?
1149 NR_SLAB_RECLAIMABLE : NR_SLAB_UNRECLAIMABLE,
1150 - pages);
1151
1152 page->mapping = NULL;
1153 __free_pages(page, s->order);
1154}
1155
1156static void rcu_free_slab(struct rcu_head *h)
1157{
1158 struct page *page;
1159
1160 page = container_of((struct list_head *)h, struct page, lru);
1161 __free_slab(page->slab, page);
1162}
1163
1164static void free_slab(struct kmem_cache *s, struct page *page)
1165{
1166 if (unlikely(s->flags & SLAB_DESTROY_BY_RCU)) {
1167 /*
1168 * RCU free overloads the RCU head over the LRU
1169 */
1170 struct rcu_head *head = (void *)&page->lru;
1171
1172 call_rcu(head, rcu_free_slab);
1173 } else
1174 __free_slab(s, page);
1175}
1176
1177static void discard_slab(struct kmem_cache *s, struct page *page)
1178{
1179 struct kmem_cache_node *n = get_node(s, page_to_nid(page));
1180
1181 atomic_long_dec(&n->nr_slabs);
1182 reset_page_mapcount(page);
Christoph Lameter35e5d7e2007-05-09 02:32:42 -07001183 __ClearPageSlab(page);
Christoph Lameter81819f02007-05-06 14:49:36 -07001184 free_slab(s, page);
1185}
1186
1187/*
1188 * Per slab locking using the pagelock
1189 */
1190static __always_inline void slab_lock(struct page *page)
1191{
1192 bit_spin_lock(PG_locked, &page->flags);
1193}
1194
1195static __always_inline void slab_unlock(struct page *page)
1196{
1197 bit_spin_unlock(PG_locked, &page->flags);
1198}
1199
1200static __always_inline int slab_trylock(struct page *page)
1201{
1202 int rc = 1;
1203
1204 rc = bit_spin_trylock(PG_locked, &page->flags);
1205 return rc;
1206}
1207
1208/*
1209 * Management of partially allocated slabs
1210 */
Christoph Lametere95eed52007-05-06 14:49:44 -07001211static void add_partial_tail(struct kmem_cache_node *n, struct page *page)
Christoph Lameter81819f02007-05-06 14:49:36 -07001212{
Christoph Lametere95eed52007-05-06 14:49:44 -07001213 spin_lock(&n->list_lock);
1214 n->nr_partial++;
1215 list_add_tail(&page->lru, &n->partial);
1216 spin_unlock(&n->list_lock);
1217}
Christoph Lameter81819f02007-05-06 14:49:36 -07001218
Christoph Lametere95eed52007-05-06 14:49:44 -07001219static void add_partial(struct kmem_cache_node *n, struct page *page)
1220{
Christoph Lameter81819f02007-05-06 14:49:36 -07001221 spin_lock(&n->list_lock);
1222 n->nr_partial++;
1223 list_add(&page->lru, &n->partial);
1224 spin_unlock(&n->list_lock);
1225}
1226
1227static void remove_partial(struct kmem_cache *s,
1228 struct page *page)
1229{
1230 struct kmem_cache_node *n = get_node(s, page_to_nid(page));
1231
1232 spin_lock(&n->list_lock);
1233 list_del(&page->lru);
1234 n->nr_partial--;
1235 spin_unlock(&n->list_lock);
1236}
1237
1238/*
Christoph Lameter672bba32007-05-09 02:32:39 -07001239 * Lock slab and remove from the partial list.
Christoph Lameter81819f02007-05-06 14:49:36 -07001240 *
Christoph Lameter672bba32007-05-09 02:32:39 -07001241 * Must hold list_lock.
Christoph Lameter81819f02007-05-06 14:49:36 -07001242 */
Christoph Lameter4b6f0752007-05-16 22:10:53 -07001243static inline int lock_and_freeze_slab(struct kmem_cache_node *n, struct page *page)
Christoph Lameter81819f02007-05-06 14:49:36 -07001244{
1245 if (slab_trylock(page)) {
1246 list_del(&page->lru);
1247 n->nr_partial--;
Christoph Lameter4b6f0752007-05-16 22:10:53 -07001248 SetSlabFrozen(page);
Christoph Lameter81819f02007-05-06 14:49:36 -07001249 return 1;
1250 }
1251 return 0;
1252}
1253
1254/*
Christoph Lameter672bba32007-05-09 02:32:39 -07001255 * Try to allocate a partial slab from a specific node.
Christoph Lameter81819f02007-05-06 14:49:36 -07001256 */
1257static struct page *get_partial_node(struct kmem_cache_node *n)
1258{
1259 struct page *page;
1260
1261 /*
1262 * Racy check. If we mistakenly see no partial slabs then we
1263 * just allocate an empty slab. If we mistakenly try to get a
Christoph Lameter672bba32007-05-09 02:32:39 -07001264 * partial slab and there is none available then get_partials()
1265 * will return NULL.
Christoph Lameter81819f02007-05-06 14:49:36 -07001266 */
1267 if (!n || !n->nr_partial)
1268 return NULL;
1269
1270 spin_lock(&n->list_lock);
1271 list_for_each_entry(page, &n->partial, lru)
Christoph Lameter4b6f0752007-05-16 22:10:53 -07001272 if (lock_and_freeze_slab(n, page))
Christoph Lameter81819f02007-05-06 14:49:36 -07001273 goto out;
1274 page = NULL;
1275out:
1276 spin_unlock(&n->list_lock);
1277 return page;
1278}
1279
1280/*
Christoph Lameter672bba32007-05-09 02:32:39 -07001281 * Get a page from somewhere. Search in increasing NUMA distances.
Christoph Lameter81819f02007-05-06 14:49:36 -07001282 */
1283static struct page *get_any_partial(struct kmem_cache *s, gfp_t flags)
1284{
1285#ifdef CONFIG_NUMA
1286 struct zonelist *zonelist;
1287 struct zone **z;
1288 struct page *page;
1289
1290 /*
Christoph Lameter672bba32007-05-09 02:32:39 -07001291 * The defrag ratio allows a configuration of the tradeoffs between
1292 * inter node defragmentation and node local allocations. A lower
1293 * defrag_ratio increases the tendency to do local allocations
1294 * instead of attempting to obtain partial slabs from other nodes.
Christoph Lameter81819f02007-05-06 14:49:36 -07001295 *
Christoph Lameter672bba32007-05-09 02:32:39 -07001296 * If the defrag_ratio is set to 0 then kmalloc() always
1297 * returns node local objects. If the ratio is higher then kmalloc()
1298 * may return off node objects because partial slabs are obtained
1299 * from other nodes and filled up.
Christoph Lameter81819f02007-05-06 14:49:36 -07001300 *
1301 * If /sys/slab/xx/defrag_ratio is set to 100 (which makes
Christoph Lameter672bba32007-05-09 02:32:39 -07001302 * defrag_ratio = 1000) then every (well almost) allocation will
1303 * first attempt to defrag slab caches on other nodes. This means
1304 * scanning over all nodes to look for partial slabs which may be
1305 * expensive if we do it every time we are trying to find a slab
1306 * with available objects.
Christoph Lameter81819f02007-05-06 14:49:36 -07001307 */
1308 if (!s->defrag_ratio || get_cycles() % 1024 > s->defrag_ratio)
1309 return NULL;
1310
1311 zonelist = &NODE_DATA(slab_node(current->mempolicy))
1312 ->node_zonelists[gfp_zone(flags)];
1313 for (z = zonelist->zones; *z; z++) {
1314 struct kmem_cache_node *n;
1315
1316 n = get_node(s, zone_to_nid(*z));
1317
1318 if (n && cpuset_zone_allowed_hardwall(*z, flags) &&
Christoph Lametere95eed52007-05-06 14:49:44 -07001319 n->nr_partial > MIN_PARTIAL) {
Christoph Lameter81819f02007-05-06 14:49:36 -07001320 page = get_partial_node(n);
1321 if (page)
1322 return page;
1323 }
1324 }
1325#endif
1326 return NULL;
1327}
1328
1329/*
1330 * Get a partial page, lock it and return it.
1331 */
1332static struct page *get_partial(struct kmem_cache *s, gfp_t flags, int node)
1333{
1334 struct page *page;
1335 int searchnode = (node == -1) ? numa_node_id() : node;
1336
1337 page = get_partial_node(get_node(s, searchnode));
1338 if (page || (flags & __GFP_THISNODE))
1339 return page;
1340
1341 return get_any_partial(s, flags);
1342}
1343
1344/*
1345 * Move a page back to the lists.
1346 *
1347 * Must be called with the slab lock held.
1348 *
1349 * On exit the slab lock will have been dropped.
1350 */
Christoph Lameter4b6f0752007-05-16 22:10:53 -07001351static void unfreeze_slab(struct kmem_cache *s, struct page *page)
Christoph Lameter81819f02007-05-06 14:49:36 -07001352{
Christoph Lametere95eed52007-05-06 14:49:44 -07001353 struct kmem_cache_node *n = get_node(s, page_to_nid(page));
1354
Christoph Lameter4b6f0752007-05-16 22:10:53 -07001355 ClearSlabFrozen(page);
Christoph Lameter81819f02007-05-06 14:49:36 -07001356 if (page->inuse) {
Christoph Lametere95eed52007-05-06 14:49:44 -07001357
Christoph Lameter81819f02007-05-06 14:49:36 -07001358 if (page->freelist)
Christoph Lametere95eed52007-05-06 14:49:44 -07001359 add_partial(n, page);
Christoph Lameter35e5d7e2007-05-09 02:32:42 -07001360 else if (SlabDebug(page) && (s->flags & SLAB_STORE_USER))
Christoph Lametere95eed52007-05-06 14:49:44 -07001361 add_full(n, page);
Christoph Lameter81819f02007-05-06 14:49:36 -07001362 slab_unlock(page);
Christoph Lametere95eed52007-05-06 14:49:44 -07001363
Christoph Lameter81819f02007-05-06 14:49:36 -07001364 } else {
Christoph Lametere95eed52007-05-06 14:49:44 -07001365 if (n->nr_partial < MIN_PARTIAL) {
1366 /*
Christoph Lameter672bba32007-05-09 02:32:39 -07001367 * Adding an empty slab to the partial slabs in order
1368 * to avoid page allocator overhead. This slab needs
1369 * to come after the other slabs with objects in
1370 * order to fill them up. That way the size of the
1371 * partial list stays small. kmem_cache_shrink can
1372 * reclaim empty slabs from the partial list.
Christoph Lametere95eed52007-05-06 14:49:44 -07001373 */
1374 add_partial_tail(n, page);
1375 slab_unlock(page);
1376 } else {
1377 slab_unlock(page);
1378 discard_slab(s, page);
1379 }
Christoph Lameter81819f02007-05-06 14:49:36 -07001380 }
1381}
1382
1383/*
1384 * Remove the cpu slab
1385 */
1386static void deactivate_slab(struct kmem_cache *s, struct page *page, int cpu)
1387{
Christoph Lameter894b8782007-05-10 03:15:16 -07001388 /*
1389 * Merge cpu freelist into freelist. Typically we get here
1390 * because both freelists are empty. So this is unlikely
1391 * to occur.
1392 */
1393 while (unlikely(page->lockless_freelist)) {
1394 void **object;
1395
1396 /* Retrieve object from cpu_freelist */
1397 object = page->lockless_freelist;
1398 page->lockless_freelist = page->lockless_freelist[page->offset];
1399
1400 /* And put onto the regular freelist */
1401 object[page->offset] = page->freelist;
1402 page->freelist = object;
1403 page->inuse--;
1404 }
Christoph Lameter81819f02007-05-06 14:49:36 -07001405 s->cpu_slab[cpu] = NULL;
Christoph Lameter4b6f0752007-05-16 22:10:53 -07001406 unfreeze_slab(s, page);
Christoph Lameter81819f02007-05-06 14:49:36 -07001407}
1408
Christoph Lameter0c710012007-07-17 04:03:24 -07001409static inline void flush_slab(struct kmem_cache *s, struct page *page, int cpu)
Christoph Lameter81819f02007-05-06 14:49:36 -07001410{
1411 slab_lock(page);
1412 deactivate_slab(s, page, cpu);
1413}
1414
1415/*
1416 * Flush cpu slab.
1417 * Called from IPI handler with interrupts disabled.
1418 */
Christoph Lameter0c710012007-07-17 04:03:24 -07001419static inline void __flush_cpu_slab(struct kmem_cache *s, int cpu)
Christoph Lameter81819f02007-05-06 14:49:36 -07001420{
1421 struct page *page = s->cpu_slab[cpu];
1422
1423 if (likely(page))
1424 flush_slab(s, page, cpu);
1425}
1426
1427static void flush_cpu_slab(void *d)
1428{
1429 struct kmem_cache *s = d;
1430 int cpu = smp_processor_id();
1431
1432 __flush_cpu_slab(s, cpu);
1433}
1434
1435static void flush_all(struct kmem_cache *s)
1436{
1437#ifdef CONFIG_SMP
1438 on_each_cpu(flush_cpu_slab, s, 1, 1);
1439#else
1440 unsigned long flags;
1441
1442 local_irq_save(flags);
1443 flush_cpu_slab(s);
1444 local_irq_restore(flags);
1445#endif
1446}
1447
1448/*
Christoph Lameter894b8782007-05-10 03:15:16 -07001449 * Slow path. The lockless freelist is empty or we need to perform
1450 * debugging duties.
Christoph Lameter81819f02007-05-06 14:49:36 -07001451 *
Christoph Lameter894b8782007-05-10 03:15:16 -07001452 * Interrupts are disabled.
Christoph Lameter81819f02007-05-06 14:49:36 -07001453 *
Christoph Lameter894b8782007-05-10 03:15:16 -07001454 * Processing is still very fast if new objects have been freed to the
1455 * regular freelist. In that case we simply take over the regular freelist
1456 * as the lockless freelist and zap the regular freelist.
Christoph Lameter81819f02007-05-06 14:49:36 -07001457 *
Christoph Lameter894b8782007-05-10 03:15:16 -07001458 * If that is not working then we fall back to the partial lists. We take the
1459 * first element of the freelist as the object to allocate now and move the
1460 * rest of the freelist to the lockless freelist.
1461 *
1462 * And if we were unable to get a new slab from the partial slab lists then
1463 * we need to allocate a new slab. This is slowest path since we may sleep.
Christoph Lameter81819f02007-05-06 14:49:36 -07001464 */
Christoph Lameter894b8782007-05-10 03:15:16 -07001465static void *__slab_alloc(struct kmem_cache *s,
1466 gfp_t gfpflags, int node, void *addr, struct page *page)
Christoph Lameter81819f02007-05-06 14:49:36 -07001467{
Christoph Lameter81819f02007-05-06 14:49:36 -07001468 void **object;
Christoph Lameter894b8782007-05-10 03:15:16 -07001469 int cpu = smp_processor_id();
Christoph Lameter81819f02007-05-06 14:49:36 -07001470
Christoph Lameter81819f02007-05-06 14:49:36 -07001471 if (!page)
1472 goto new_slab;
1473
1474 slab_lock(page);
1475 if (unlikely(node != -1 && page_to_nid(page) != node))
1476 goto another_slab;
Christoph Lameter894b8782007-05-10 03:15:16 -07001477load_freelist:
Christoph Lameter81819f02007-05-06 14:49:36 -07001478 object = page->freelist;
1479 if (unlikely(!object))
1480 goto another_slab;
Christoph Lameter35e5d7e2007-05-09 02:32:42 -07001481 if (unlikely(SlabDebug(page)))
Christoph Lameter81819f02007-05-06 14:49:36 -07001482 goto debug;
1483
Christoph Lameter894b8782007-05-10 03:15:16 -07001484 object = page->freelist;
1485 page->lockless_freelist = object[page->offset];
1486 page->inuse = s->objects;
1487 page->freelist = NULL;
Christoph Lameter81819f02007-05-06 14:49:36 -07001488 slab_unlock(page);
Christoph Lameter81819f02007-05-06 14:49:36 -07001489 return object;
1490
1491another_slab:
1492 deactivate_slab(s, page, cpu);
1493
1494new_slab:
1495 page = get_partial(s, gfpflags, node);
Christoph Lameter894b8782007-05-10 03:15:16 -07001496 if (page) {
Christoph Lameter81819f02007-05-06 14:49:36 -07001497 s->cpu_slab[cpu] = page;
Christoph Lameter894b8782007-05-10 03:15:16 -07001498 goto load_freelist;
Christoph Lameter81819f02007-05-06 14:49:36 -07001499 }
1500
1501 page = new_slab(s, gfpflags, node);
1502 if (page) {
1503 cpu = smp_processor_id();
1504 if (s->cpu_slab[cpu]) {
1505 /*
Christoph Lameter672bba32007-05-09 02:32:39 -07001506 * Someone else populated the cpu_slab while we
1507 * enabled interrupts, or we have gotten scheduled
1508 * on another cpu. The page may not be on the
1509 * requested node even if __GFP_THISNODE was
1510 * specified. So we need to recheck.
Christoph Lameter81819f02007-05-06 14:49:36 -07001511 */
1512 if (node == -1 ||
1513 page_to_nid(s->cpu_slab[cpu]) == node) {
1514 /*
1515 * Current cpuslab is acceptable and we
1516 * want the current one since its cache hot
1517 */
1518 discard_slab(s, page);
1519 page = s->cpu_slab[cpu];
1520 slab_lock(page);
Christoph Lameter894b8782007-05-10 03:15:16 -07001521 goto load_freelist;
Christoph Lameter81819f02007-05-06 14:49:36 -07001522 }
Christoph Lameter672bba32007-05-09 02:32:39 -07001523 /* New slab does not fit our expectations */
Christoph Lameter81819f02007-05-06 14:49:36 -07001524 flush_slab(s, s->cpu_slab[cpu], cpu);
1525 }
1526 slab_lock(page);
Christoph Lameter4b6f0752007-05-16 22:10:53 -07001527 SetSlabFrozen(page);
1528 s->cpu_slab[cpu] = page;
1529 goto load_freelist;
Christoph Lameter81819f02007-05-06 14:49:36 -07001530 }
Christoph Lameter81819f02007-05-06 14:49:36 -07001531 return NULL;
1532debug:
Christoph Lameter894b8782007-05-10 03:15:16 -07001533 object = page->freelist;
Christoph Lameter3ec09742007-05-16 22:11:00 -07001534 if (!alloc_debug_processing(s, page, object, addr))
Christoph Lameter81819f02007-05-06 14:49:36 -07001535 goto another_slab;
Christoph Lameter894b8782007-05-10 03:15:16 -07001536
1537 page->inuse++;
1538 page->freelist = object[page->offset];
1539 slab_unlock(page);
1540 return object;
1541}
1542
1543/*
1544 * Inlined fastpath so that allocation functions (kmalloc, kmem_cache_alloc)
1545 * have the fastpath folded into their functions. So no function call
1546 * overhead for requests that can be satisfied on the fastpath.
1547 *
1548 * The fastpath works by first checking if the lockless freelist can be used.
1549 * If not then __slab_alloc is called for slow processing.
1550 *
1551 * Otherwise we can simply pick the next object from the lockless free list.
1552 */
1553static void __always_inline *slab_alloc(struct kmem_cache *s,
Christoph Lameterce15fea2007-07-17 04:03:28 -07001554 gfp_t gfpflags, int node, void *addr)
Christoph Lameter894b8782007-05-10 03:15:16 -07001555{
1556 struct page *page;
1557 void **object;
1558 unsigned long flags;
1559
1560 local_irq_save(flags);
1561 page = s->cpu_slab[smp_processor_id()];
1562 if (unlikely(!page || !page->lockless_freelist ||
1563 (node != -1 && page_to_nid(page) != node)))
1564
1565 object = __slab_alloc(s, gfpflags, node, addr, page);
1566
1567 else {
1568 object = page->lockless_freelist;
1569 page->lockless_freelist = object[page->offset];
1570 }
1571 local_irq_restore(flags);
Christoph Lameterd07dbea2007-07-17 04:03:23 -07001572
1573 if (unlikely((gfpflags & __GFP_ZERO) && object))
Christoph Lameterce15fea2007-07-17 04:03:28 -07001574 memset(object, 0, s->objsize);
Christoph Lameterd07dbea2007-07-17 04:03:23 -07001575
Christoph Lameter894b8782007-05-10 03:15:16 -07001576 return object;
Christoph Lameter81819f02007-05-06 14:49:36 -07001577}
1578
1579void *kmem_cache_alloc(struct kmem_cache *s, gfp_t gfpflags)
1580{
Christoph Lameterce15fea2007-07-17 04:03:28 -07001581 return slab_alloc(s, gfpflags, -1, __builtin_return_address(0));
Christoph Lameter81819f02007-05-06 14:49:36 -07001582}
1583EXPORT_SYMBOL(kmem_cache_alloc);
1584
1585#ifdef CONFIG_NUMA
1586void *kmem_cache_alloc_node(struct kmem_cache *s, gfp_t gfpflags, int node)
1587{
Christoph Lameterce15fea2007-07-17 04:03:28 -07001588 return slab_alloc(s, gfpflags, node, __builtin_return_address(0));
Christoph Lameter81819f02007-05-06 14:49:36 -07001589}
1590EXPORT_SYMBOL(kmem_cache_alloc_node);
1591#endif
1592
1593/*
Christoph Lameter894b8782007-05-10 03:15:16 -07001594 * Slow patch handling. This may still be called frequently since objects
1595 * have a longer lifetime than the cpu slabs in most processing loads.
Christoph Lameter81819f02007-05-06 14:49:36 -07001596 *
Christoph Lameter894b8782007-05-10 03:15:16 -07001597 * So we still attempt to reduce cache line usage. Just take the slab
1598 * lock and free the item. If there is no additional partial page
1599 * handling required then we can return immediately.
Christoph Lameter81819f02007-05-06 14:49:36 -07001600 */
Christoph Lameter894b8782007-05-10 03:15:16 -07001601static void __slab_free(struct kmem_cache *s, struct page *page,
Christoph Lameter77c5e2d2007-05-06 14:49:42 -07001602 void *x, void *addr)
Christoph Lameter81819f02007-05-06 14:49:36 -07001603{
1604 void *prior;
1605 void **object = (void *)x;
Christoph Lameter81819f02007-05-06 14:49:36 -07001606
Christoph Lameter81819f02007-05-06 14:49:36 -07001607 slab_lock(page);
1608
Christoph Lameter35e5d7e2007-05-09 02:32:42 -07001609 if (unlikely(SlabDebug(page)))
Christoph Lameter81819f02007-05-06 14:49:36 -07001610 goto debug;
1611checks_ok:
1612 prior = object[page->offset] = page->freelist;
1613 page->freelist = object;
1614 page->inuse--;
1615
Christoph Lameter4b6f0752007-05-16 22:10:53 -07001616 if (unlikely(SlabFrozen(page)))
Christoph Lameter81819f02007-05-06 14:49:36 -07001617 goto out_unlock;
1618
1619 if (unlikely(!page->inuse))
1620 goto slab_empty;
1621
1622 /*
1623 * Objects left in the slab. If it
1624 * was not on the partial list before
1625 * then add it.
1626 */
1627 if (unlikely(!prior))
Christoph Lametere95eed52007-05-06 14:49:44 -07001628 add_partial(get_node(s, page_to_nid(page)), page);
Christoph Lameter81819f02007-05-06 14:49:36 -07001629
1630out_unlock:
1631 slab_unlock(page);
Christoph Lameter81819f02007-05-06 14:49:36 -07001632 return;
1633
1634slab_empty:
1635 if (prior)
1636 /*
Christoph Lameter672bba32007-05-09 02:32:39 -07001637 * Slab still on the partial list.
Christoph Lameter81819f02007-05-06 14:49:36 -07001638 */
1639 remove_partial(s, page);
1640
1641 slab_unlock(page);
1642 discard_slab(s, page);
Christoph Lameter81819f02007-05-06 14:49:36 -07001643 return;
1644
1645debug:
Christoph Lameter3ec09742007-05-16 22:11:00 -07001646 if (!free_debug_processing(s, page, x, addr))
Christoph Lameter77c5e2d2007-05-06 14:49:42 -07001647 goto out_unlock;
Christoph Lameter77c5e2d2007-05-06 14:49:42 -07001648 goto checks_ok;
Christoph Lameter81819f02007-05-06 14:49:36 -07001649}
1650
Christoph Lameter894b8782007-05-10 03:15:16 -07001651/*
1652 * Fastpath with forced inlining to produce a kfree and kmem_cache_free that
1653 * can perform fastpath freeing without additional function calls.
1654 *
1655 * The fastpath is only possible if we are freeing to the current cpu slab
1656 * of this processor. This typically the case if we have just allocated
1657 * the item before.
1658 *
1659 * If fastpath is not possible then fall back to __slab_free where we deal
1660 * with all sorts of special processing.
1661 */
1662static void __always_inline slab_free(struct kmem_cache *s,
1663 struct page *page, void *x, void *addr)
1664{
1665 void **object = (void *)x;
1666 unsigned long flags;
1667
1668 local_irq_save(flags);
Peter Zijlstra02febdf2007-07-26 20:01:38 +02001669 debug_check_no_locks_freed(object, s->objsize);
Christoph Lameter894b8782007-05-10 03:15:16 -07001670 if (likely(page == s->cpu_slab[smp_processor_id()] &&
1671 !SlabDebug(page))) {
1672 object[page->offset] = page->lockless_freelist;
1673 page->lockless_freelist = object;
1674 } else
1675 __slab_free(s, page, x, addr);
1676
1677 local_irq_restore(flags);
1678}
1679
Christoph Lameter81819f02007-05-06 14:49:36 -07001680void kmem_cache_free(struct kmem_cache *s, void *x)
1681{
Christoph Lameter77c5e2d2007-05-06 14:49:42 -07001682 struct page *page;
Christoph Lameter81819f02007-05-06 14:49:36 -07001683
Christoph Lameterb49af682007-05-06 14:49:41 -07001684 page = virt_to_head_page(x);
Christoph Lameter81819f02007-05-06 14:49:36 -07001685
Christoph Lameter77c5e2d2007-05-06 14:49:42 -07001686 slab_free(s, page, x, __builtin_return_address(0));
Christoph Lameter81819f02007-05-06 14:49:36 -07001687}
1688EXPORT_SYMBOL(kmem_cache_free);
1689
1690/* Figure out on which slab object the object resides */
1691static struct page *get_object_page(const void *x)
1692{
Christoph Lameterb49af682007-05-06 14:49:41 -07001693 struct page *page = virt_to_head_page(x);
Christoph Lameter81819f02007-05-06 14:49:36 -07001694
1695 if (!PageSlab(page))
1696 return NULL;
1697
1698 return page;
1699}
1700
1701/*
Christoph Lameter672bba32007-05-09 02:32:39 -07001702 * Object placement in a slab is made very easy because we always start at
1703 * offset 0. If we tune the size of the object to the alignment then we can
1704 * get the required alignment by putting one properly sized object after
1705 * another.
Christoph Lameter81819f02007-05-06 14:49:36 -07001706 *
1707 * Notice that the allocation order determines the sizes of the per cpu
1708 * caches. Each processor has always one slab available for allocations.
1709 * Increasing the allocation order reduces the number of times that slabs
Christoph Lameter672bba32007-05-09 02:32:39 -07001710 * must be moved on and off the partial lists and is therefore a factor in
Christoph Lameter81819f02007-05-06 14:49:36 -07001711 * locking overhead.
Christoph Lameter81819f02007-05-06 14:49:36 -07001712 */
1713
1714/*
1715 * Mininum / Maximum order of slab pages. This influences locking overhead
1716 * and slab fragmentation. A higher order reduces the number of partial slabs
1717 * and increases the number of allocations possible without having to
1718 * take the list_lock.
1719 */
1720static int slub_min_order;
1721static int slub_max_order = DEFAULT_MAX_ORDER;
Christoph Lameter81819f02007-05-06 14:49:36 -07001722static int slub_min_objects = DEFAULT_MIN_OBJECTS;
1723
1724/*
1725 * Merge control. If this is set then no merging of slab caches will occur.
Christoph Lameter672bba32007-05-09 02:32:39 -07001726 * (Could be removed. This was introduced to pacify the merge skeptics.)
Christoph Lameter81819f02007-05-06 14:49:36 -07001727 */
1728static int slub_nomerge;
1729
1730/*
Christoph Lameter81819f02007-05-06 14:49:36 -07001731 * Calculate the order of allocation given an slab object size.
1732 *
Christoph Lameter672bba32007-05-09 02:32:39 -07001733 * The order of allocation has significant impact on performance and other
1734 * system components. Generally order 0 allocations should be preferred since
1735 * order 0 does not cause fragmentation in the page allocator. Larger objects
1736 * be problematic to put into order 0 slabs because there may be too much
1737 * unused space left. We go to a higher order if more than 1/8th of the slab
1738 * would be wasted.
Christoph Lameter81819f02007-05-06 14:49:36 -07001739 *
Christoph Lameter672bba32007-05-09 02:32:39 -07001740 * In order to reach satisfactory performance we must ensure that a minimum
1741 * number of objects is in one slab. Otherwise we may generate too much
1742 * activity on the partial lists which requires taking the list_lock. This is
1743 * less a concern for large slabs though which are rarely used.
Christoph Lameter81819f02007-05-06 14:49:36 -07001744 *
Christoph Lameter672bba32007-05-09 02:32:39 -07001745 * slub_max_order specifies the order where we begin to stop considering the
1746 * number of objects in a slab as critical. If we reach slub_max_order then
1747 * we try to keep the page order as low as possible. So we accept more waste
1748 * of space in favor of a small page order.
1749 *
1750 * Higher order allocations also allow the placement of more objects in a
1751 * slab and thereby reduce object handling overhead. If the user has
1752 * requested a higher mininum order then we start with that one instead of
1753 * the smallest order which will fit the object.
Christoph Lameter81819f02007-05-06 14:49:36 -07001754 */
Christoph Lameter5e6d4442007-05-09 02:32:46 -07001755static inline int slab_order(int size, int min_objects,
1756 int max_order, int fract_leftover)
Christoph Lameter81819f02007-05-06 14:49:36 -07001757{
1758 int order;
1759 int rem;
Christoph Lameter6300ea72007-07-17 04:03:20 -07001760 int min_order = slub_min_order;
Christoph Lameter81819f02007-05-06 14:49:36 -07001761
Christoph Lameter6300ea72007-07-17 04:03:20 -07001762 /*
1763 * If we would create too many object per slab then reduce
1764 * the slab order even if it goes below slub_min_order.
1765 */
1766 while (min_order > 0 &&
1767 (PAGE_SIZE << min_order) >= MAX_OBJECTS_PER_SLAB * size)
1768 min_order--;
1769
1770 for (order = max(min_order,
Christoph Lameter5e6d4442007-05-09 02:32:46 -07001771 fls(min_objects * size - 1) - PAGE_SHIFT);
1772 order <= max_order; order++) {
1773
Christoph Lameter81819f02007-05-06 14:49:36 -07001774 unsigned long slab_size = PAGE_SIZE << order;
1775
Christoph Lameter5e6d4442007-05-09 02:32:46 -07001776 if (slab_size < min_objects * size)
Christoph Lameter81819f02007-05-06 14:49:36 -07001777 continue;
1778
Christoph Lameter81819f02007-05-06 14:49:36 -07001779 rem = slab_size % size;
1780
Christoph Lameter5e6d4442007-05-09 02:32:46 -07001781 if (rem <= slab_size / fract_leftover)
Christoph Lameter81819f02007-05-06 14:49:36 -07001782 break;
1783
Christoph Lameter6300ea72007-07-17 04:03:20 -07001784 /* If the next size is too high then exit now */
1785 if (slab_size * 2 >= MAX_OBJECTS_PER_SLAB * size)
1786 break;
Christoph Lameter81819f02007-05-06 14:49:36 -07001787 }
Christoph Lameter672bba32007-05-09 02:32:39 -07001788
Christoph Lameter81819f02007-05-06 14:49:36 -07001789 return order;
1790}
1791
Christoph Lameter5e6d4442007-05-09 02:32:46 -07001792static inline int calculate_order(int size)
1793{
1794 int order;
1795 int min_objects;
1796 int fraction;
1797
1798 /*
1799 * Attempt to find best configuration for a slab. This
1800 * works by first attempting to generate a layout with
1801 * the best configuration and backing off gradually.
1802 *
1803 * First we reduce the acceptable waste in a slab. Then
1804 * we reduce the minimum objects required in a slab.
1805 */
1806 min_objects = slub_min_objects;
1807 while (min_objects > 1) {
1808 fraction = 8;
1809 while (fraction >= 4) {
1810 order = slab_order(size, min_objects,
1811 slub_max_order, fraction);
1812 if (order <= slub_max_order)
1813 return order;
1814 fraction /= 2;
1815 }
1816 min_objects /= 2;
1817 }
1818
1819 /*
1820 * We were unable to place multiple objects in a slab. Now
1821 * lets see if we can place a single object there.
1822 */
1823 order = slab_order(size, 1, slub_max_order, 1);
1824 if (order <= slub_max_order)
1825 return order;
1826
1827 /*
1828 * Doh this slab cannot be placed using slub_max_order.
1829 */
1830 order = slab_order(size, 1, MAX_ORDER, 1);
1831 if (order <= MAX_ORDER)
1832 return order;
1833 return -ENOSYS;
1834}
1835
Christoph Lameter81819f02007-05-06 14:49:36 -07001836/*
Christoph Lameter672bba32007-05-09 02:32:39 -07001837 * Figure out what the alignment of the objects will be.
Christoph Lameter81819f02007-05-06 14:49:36 -07001838 */
1839static unsigned long calculate_alignment(unsigned long flags,
1840 unsigned long align, unsigned long size)
1841{
1842 /*
1843 * If the user wants hardware cache aligned objects then
1844 * follow that suggestion if the object is sufficiently
1845 * large.
1846 *
1847 * The hardware cache alignment cannot override the
1848 * specified alignment though. If that is greater
1849 * then use it.
1850 */
Christoph Lameter5af60832007-05-06 14:49:56 -07001851 if ((flags & SLAB_HWCACHE_ALIGN) &&
Christoph Lameter65c02d42007-05-09 02:32:35 -07001852 size > cache_line_size() / 2)
1853 return max_t(unsigned long, align, cache_line_size());
Christoph Lameter81819f02007-05-06 14:49:36 -07001854
1855 if (align < ARCH_SLAB_MINALIGN)
1856 return ARCH_SLAB_MINALIGN;
1857
1858 return ALIGN(align, sizeof(void *));
1859}
1860
1861static void init_kmem_cache_node(struct kmem_cache_node *n)
1862{
1863 n->nr_partial = 0;
1864 atomic_long_set(&n->nr_slabs, 0);
1865 spin_lock_init(&n->list_lock);
1866 INIT_LIST_HEAD(&n->partial);
Christoph Lameter8ab13722007-07-17 04:03:32 -07001867#ifdef CONFIG_SLUB_DEBUG
Christoph Lameter643b1132007-05-06 14:49:42 -07001868 INIT_LIST_HEAD(&n->full);
Christoph Lameter8ab13722007-07-17 04:03:32 -07001869#endif
Christoph Lameter81819f02007-05-06 14:49:36 -07001870}
1871
1872#ifdef CONFIG_NUMA
1873/*
1874 * No kmalloc_node yet so do it by hand. We know that this is the first
1875 * slab on the node for this slabcache. There are no concurrent accesses
1876 * possible.
1877 *
1878 * Note that this function only works on the kmalloc_node_cache
1879 * when allocating for the kmalloc_node_cache.
1880 */
1881static struct kmem_cache_node * __init early_kmem_cache_node_alloc(gfp_t gfpflags,
1882 int node)
1883{
1884 struct page *page;
1885 struct kmem_cache_node *n;
1886
1887 BUG_ON(kmalloc_caches->size < sizeof(struct kmem_cache_node));
1888
Christoph Lametera2f92ee2007-08-22 14:01:57 -07001889 page = new_slab(kmalloc_caches, gfpflags, node);
Christoph Lameter81819f02007-05-06 14:49:36 -07001890
1891 BUG_ON(!page);
Christoph Lametera2f92ee2007-08-22 14:01:57 -07001892 if (page_to_nid(page) != node) {
1893 printk(KERN_ERR "SLUB: Unable to allocate memory from "
1894 "node %d\n", node);
1895 printk(KERN_ERR "SLUB: Allocating a useless per node structure "
1896 "in order to be able to continue\n");
1897 }
1898
Christoph Lameter81819f02007-05-06 14:49:36 -07001899 n = page->freelist;
1900 BUG_ON(!n);
1901 page->freelist = get_freepointer(kmalloc_caches, n);
1902 page->inuse++;
1903 kmalloc_caches->node[node] = n;
Christoph Lameter8ab13722007-07-17 04:03:32 -07001904#ifdef CONFIG_SLUB_DEBUG
Christoph Lameterd45f39c2007-07-17 04:03:21 -07001905 init_object(kmalloc_caches, n, 1);
1906 init_tracking(kmalloc_caches, n);
Christoph Lameter8ab13722007-07-17 04:03:32 -07001907#endif
Christoph Lameter81819f02007-05-06 14:49:36 -07001908 init_kmem_cache_node(n);
1909 atomic_long_inc(&n->nr_slabs);
Christoph Lametere95eed52007-05-06 14:49:44 -07001910 add_partial(n, page);
Christoph Lameterdbc55fa2007-07-03 09:31:04 -07001911
1912 /*
1913 * new_slab() disables interupts. If we do not reenable interrupts here
1914 * then bootup would continue with interrupts disabled.
1915 */
1916 local_irq_enable();
Christoph Lameter81819f02007-05-06 14:49:36 -07001917 return n;
1918}
1919
1920static void free_kmem_cache_nodes(struct kmem_cache *s)
1921{
1922 int node;
1923
1924 for_each_online_node(node) {
1925 struct kmem_cache_node *n = s->node[node];
1926 if (n && n != &s->local_node)
1927 kmem_cache_free(kmalloc_caches, n);
1928 s->node[node] = NULL;
1929 }
1930}
1931
1932static int init_kmem_cache_nodes(struct kmem_cache *s, gfp_t gfpflags)
1933{
1934 int node;
1935 int local_node;
1936
1937 if (slab_state >= UP)
1938 local_node = page_to_nid(virt_to_page(s));
1939 else
1940 local_node = 0;
1941
1942 for_each_online_node(node) {
1943 struct kmem_cache_node *n;
1944
1945 if (local_node == node)
1946 n = &s->local_node;
1947 else {
1948 if (slab_state == DOWN) {
1949 n = early_kmem_cache_node_alloc(gfpflags,
1950 node);
1951 continue;
1952 }
1953 n = kmem_cache_alloc_node(kmalloc_caches,
1954 gfpflags, node);
1955
1956 if (!n) {
1957 free_kmem_cache_nodes(s);
1958 return 0;
1959 }
1960
1961 }
1962 s->node[node] = n;
1963 init_kmem_cache_node(n);
1964 }
1965 return 1;
1966}
1967#else
1968static void free_kmem_cache_nodes(struct kmem_cache *s)
1969{
1970}
1971
1972static int init_kmem_cache_nodes(struct kmem_cache *s, gfp_t gfpflags)
1973{
1974 init_kmem_cache_node(&s->local_node);
1975 return 1;
1976}
1977#endif
1978
1979/*
1980 * calculate_sizes() determines the order and the distribution of data within
1981 * a slab object.
1982 */
1983static int calculate_sizes(struct kmem_cache *s)
1984{
1985 unsigned long flags = s->flags;
1986 unsigned long size = s->objsize;
1987 unsigned long align = s->align;
1988
1989 /*
1990 * Determine if we can poison the object itself. If the user of
1991 * the slab may touch the object after free or before allocation
1992 * then we should never poison the object itself.
1993 */
1994 if ((flags & SLAB_POISON) && !(flags & SLAB_DESTROY_BY_RCU) &&
Christoph Lameterc59def9f2007-05-16 22:10:50 -07001995 !s->ctor)
Christoph Lameter81819f02007-05-06 14:49:36 -07001996 s->flags |= __OBJECT_POISON;
1997 else
1998 s->flags &= ~__OBJECT_POISON;
1999
2000 /*
2001 * Round up object size to the next word boundary. We can only
2002 * place the free pointer at word boundaries and this determines
2003 * the possible location of the free pointer.
2004 */
2005 size = ALIGN(size, sizeof(void *));
2006
Christoph Lameter41ecc552007-05-09 02:32:44 -07002007#ifdef CONFIG_SLUB_DEBUG
Christoph Lameter81819f02007-05-06 14:49:36 -07002008 /*
Christoph Lameter672bba32007-05-09 02:32:39 -07002009 * If we are Redzoning then check if there is some space between the
Christoph Lameter81819f02007-05-06 14:49:36 -07002010 * end of the object and the free pointer. If not then add an
Christoph Lameter672bba32007-05-09 02:32:39 -07002011 * additional word to have some bytes to store Redzone information.
Christoph Lameter81819f02007-05-06 14:49:36 -07002012 */
2013 if ((flags & SLAB_RED_ZONE) && size == s->objsize)
2014 size += sizeof(void *);
Christoph Lameter41ecc552007-05-09 02:32:44 -07002015#endif
Christoph Lameter81819f02007-05-06 14:49:36 -07002016
2017 /*
Christoph Lameter672bba32007-05-09 02:32:39 -07002018 * With that we have determined the number of bytes in actual use
2019 * by the object. This is the potential offset to the free pointer.
Christoph Lameter81819f02007-05-06 14:49:36 -07002020 */
2021 s->inuse = size;
2022
2023 if (((flags & (SLAB_DESTROY_BY_RCU | SLAB_POISON)) ||
Christoph Lameterc59def9f2007-05-16 22:10:50 -07002024 s->ctor)) {
Christoph Lameter81819f02007-05-06 14:49:36 -07002025 /*
2026 * Relocate free pointer after the object if it is not
2027 * permitted to overwrite the first word of the object on
2028 * kmem_cache_free.
2029 *
2030 * This is the case if we do RCU, have a constructor or
2031 * destructor or are poisoning the objects.
2032 */
2033 s->offset = size;
2034 size += sizeof(void *);
2035 }
2036
Christoph Lameterc12b3c62007-05-23 13:57:31 -07002037#ifdef CONFIG_SLUB_DEBUG
Christoph Lameter81819f02007-05-06 14:49:36 -07002038 if (flags & SLAB_STORE_USER)
2039 /*
2040 * Need to store information about allocs and frees after
2041 * the object.
2042 */
2043 size += 2 * sizeof(struct track);
2044
Christoph Lameterbe7b3fb2007-05-09 02:32:36 -07002045 if (flags & SLAB_RED_ZONE)
Christoph Lameter81819f02007-05-06 14:49:36 -07002046 /*
2047 * Add some empty padding so that we can catch
2048 * overwrites from earlier objects rather than let
2049 * tracking information or the free pointer be
2050 * corrupted if an user writes before the start
2051 * of the object.
2052 */
2053 size += sizeof(void *);
Christoph Lameter41ecc552007-05-09 02:32:44 -07002054#endif
Christoph Lameter672bba32007-05-09 02:32:39 -07002055
Christoph Lameter81819f02007-05-06 14:49:36 -07002056 /*
2057 * Determine the alignment based on various parameters that the
Christoph Lameter65c02d42007-05-09 02:32:35 -07002058 * user specified and the dynamic determination of cache line size
2059 * on bootup.
Christoph Lameter81819f02007-05-06 14:49:36 -07002060 */
2061 align = calculate_alignment(flags, align, s->objsize);
2062
2063 /*
2064 * SLUB stores one object immediately after another beginning from
2065 * offset 0. In order to align the objects we have to simply size
2066 * each object to conform to the alignment.
2067 */
2068 size = ALIGN(size, align);
2069 s->size = size;
2070
2071 s->order = calculate_order(size);
2072 if (s->order < 0)
2073 return 0;
2074
2075 /*
2076 * Determine the number of objects per slab
2077 */
2078 s->objects = (PAGE_SIZE << s->order) / size;
2079
2080 /*
2081 * Verify that the number of objects is within permitted limits.
2082 * The page->inuse field is only 16 bit wide! So we cannot have
2083 * more than 64k objects per slab.
2084 */
Christoph Lameter6300ea72007-07-17 04:03:20 -07002085 if (!s->objects || s->objects > MAX_OBJECTS_PER_SLAB)
Christoph Lameter81819f02007-05-06 14:49:36 -07002086 return 0;
2087 return 1;
2088
2089}
2090
Christoph Lameter81819f02007-05-06 14:49:36 -07002091static int kmem_cache_open(struct kmem_cache *s, gfp_t gfpflags,
2092 const char *name, size_t size,
2093 size_t align, unsigned long flags,
Christoph Lameterc59def9f2007-05-16 22:10:50 -07002094 void (*ctor)(void *, struct kmem_cache *, unsigned long))
Christoph Lameter81819f02007-05-06 14:49:36 -07002095{
2096 memset(s, 0, kmem_size);
2097 s->name = name;
2098 s->ctor = ctor;
Christoph Lameter81819f02007-05-06 14:49:36 -07002099 s->objsize = size;
Christoph Lameter81819f02007-05-06 14:49:36 -07002100 s->align = align;
Christoph Lameterba0268a2007-09-11 15:24:11 -07002101 s->flags = kmem_cache_flags(size, flags, name, ctor);
Christoph Lameter81819f02007-05-06 14:49:36 -07002102
2103 if (!calculate_sizes(s))
2104 goto error;
2105
2106 s->refcount = 1;
2107#ifdef CONFIG_NUMA
2108 s->defrag_ratio = 100;
2109#endif
2110
2111 if (init_kmem_cache_nodes(s, gfpflags & ~SLUB_DMA))
2112 return 1;
2113error:
2114 if (flags & SLAB_PANIC)
2115 panic("Cannot create slab %s size=%lu realsize=%u "
2116 "order=%u offset=%u flags=%lx\n",
2117 s->name, (unsigned long)size, s->size, s->order,
2118 s->offset, flags);
2119 return 0;
2120}
Christoph Lameter81819f02007-05-06 14:49:36 -07002121
2122/*
2123 * Check if a given pointer is valid
2124 */
2125int kmem_ptr_validate(struct kmem_cache *s, const void *object)
2126{
2127 struct page * page;
Christoph Lameter81819f02007-05-06 14:49:36 -07002128
2129 page = get_object_page(object);
2130
2131 if (!page || s != page->slab)
2132 /* No slab or wrong slab */
2133 return 0;
2134
Christoph Lameterabcd08a2007-05-09 02:32:37 -07002135 if (!check_valid_pointer(s, page, object))
Christoph Lameter81819f02007-05-06 14:49:36 -07002136 return 0;
2137
2138 /*
2139 * We could also check if the object is on the slabs freelist.
2140 * But this would be too expensive and it seems that the main
2141 * purpose of kmem_ptr_valid is to check if the object belongs
2142 * to a certain slab.
2143 */
2144 return 1;
2145}
2146EXPORT_SYMBOL(kmem_ptr_validate);
2147
2148/*
2149 * Determine the size of a slab object
2150 */
2151unsigned int kmem_cache_size(struct kmem_cache *s)
2152{
2153 return s->objsize;
2154}
2155EXPORT_SYMBOL(kmem_cache_size);
2156
2157const char *kmem_cache_name(struct kmem_cache *s)
2158{
2159 return s->name;
2160}
2161EXPORT_SYMBOL(kmem_cache_name);
2162
2163/*
Christoph Lameter672bba32007-05-09 02:32:39 -07002164 * Attempt to free all slabs on a node. Return the number of slabs we
2165 * were unable to free.
Christoph Lameter81819f02007-05-06 14:49:36 -07002166 */
2167static int free_list(struct kmem_cache *s, struct kmem_cache_node *n,
2168 struct list_head *list)
2169{
2170 int slabs_inuse = 0;
2171 unsigned long flags;
2172 struct page *page, *h;
2173
2174 spin_lock_irqsave(&n->list_lock, flags);
2175 list_for_each_entry_safe(page, h, list, lru)
2176 if (!page->inuse) {
2177 list_del(&page->lru);
2178 discard_slab(s, page);
2179 } else
2180 slabs_inuse++;
2181 spin_unlock_irqrestore(&n->list_lock, flags);
2182 return slabs_inuse;
2183}
2184
2185/*
Christoph Lameter672bba32007-05-09 02:32:39 -07002186 * Release all resources used by a slab cache.
Christoph Lameter81819f02007-05-06 14:49:36 -07002187 */
Christoph Lameter0c710012007-07-17 04:03:24 -07002188static inline int kmem_cache_close(struct kmem_cache *s)
Christoph Lameter81819f02007-05-06 14:49:36 -07002189{
2190 int node;
2191
2192 flush_all(s);
2193
2194 /* Attempt to free all objects */
2195 for_each_online_node(node) {
2196 struct kmem_cache_node *n = get_node(s, node);
2197
Christoph Lameter2086d262007-05-06 14:49:46 -07002198 n->nr_partial -= free_list(s, n, &n->partial);
Christoph Lameter81819f02007-05-06 14:49:36 -07002199 if (atomic_long_read(&n->nr_slabs))
2200 return 1;
2201 }
2202 free_kmem_cache_nodes(s);
2203 return 0;
2204}
2205
2206/*
2207 * Close a cache and release the kmem_cache structure
2208 * (must be used for caches created using kmem_cache_create)
2209 */
2210void kmem_cache_destroy(struct kmem_cache *s)
2211{
2212 down_write(&slub_lock);
2213 s->refcount--;
2214 if (!s->refcount) {
2215 list_del(&s->list);
Christoph Lametera0e1d1b2007-07-17 04:03:31 -07002216 up_write(&slub_lock);
Christoph Lameter81819f02007-05-06 14:49:36 -07002217 if (kmem_cache_close(s))
2218 WARN_ON(1);
2219 sysfs_slab_remove(s);
2220 kfree(s);
Christoph Lametera0e1d1b2007-07-17 04:03:31 -07002221 } else
2222 up_write(&slub_lock);
Christoph Lameter81819f02007-05-06 14:49:36 -07002223}
2224EXPORT_SYMBOL(kmem_cache_destroy);
2225
2226/********************************************************************
2227 * Kmalloc subsystem
2228 *******************************************************************/
2229
2230struct kmem_cache kmalloc_caches[KMALLOC_SHIFT_HIGH + 1] __cacheline_aligned;
2231EXPORT_SYMBOL(kmalloc_caches);
2232
2233#ifdef CONFIG_ZONE_DMA
2234static struct kmem_cache *kmalloc_caches_dma[KMALLOC_SHIFT_HIGH + 1];
2235#endif
2236
2237static int __init setup_slub_min_order(char *str)
2238{
2239 get_option (&str, &slub_min_order);
2240
2241 return 1;
2242}
2243
2244__setup("slub_min_order=", setup_slub_min_order);
2245
2246static int __init setup_slub_max_order(char *str)
2247{
2248 get_option (&str, &slub_max_order);
2249
2250 return 1;
2251}
2252
2253__setup("slub_max_order=", setup_slub_max_order);
2254
2255static int __init setup_slub_min_objects(char *str)
2256{
2257 get_option (&str, &slub_min_objects);
2258
2259 return 1;
2260}
2261
2262__setup("slub_min_objects=", setup_slub_min_objects);
2263
2264static int __init setup_slub_nomerge(char *str)
2265{
2266 slub_nomerge = 1;
2267 return 1;
2268}
2269
2270__setup("slub_nomerge", setup_slub_nomerge);
2271
Christoph Lameter81819f02007-05-06 14:49:36 -07002272static struct kmem_cache *create_kmalloc_cache(struct kmem_cache *s,
2273 const char *name, int size, gfp_t gfp_flags)
2274{
2275 unsigned int flags = 0;
2276
2277 if (gfp_flags & SLUB_DMA)
2278 flags = SLAB_CACHE_DMA;
2279
2280 down_write(&slub_lock);
2281 if (!kmem_cache_open(s, gfp_flags, name, size, ARCH_KMALLOC_MINALIGN,
Christoph Lameterc59def9f2007-05-16 22:10:50 -07002282 flags, NULL))
Christoph Lameter81819f02007-05-06 14:49:36 -07002283 goto panic;
2284
2285 list_add(&s->list, &slab_caches);
2286 up_write(&slub_lock);
2287 if (sysfs_slab_add(s))
2288 goto panic;
2289 return s;
2290
2291panic:
2292 panic("Creation of kmalloc slab %s size=%d failed.\n", name, size);
2293}
2294
Christoph Lameter2e443fd2007-07-17 04:03:24 -07002295#ifdef CONFIG_ZONE_DMA
Christoph Lameter1ceef402007-08-07 15:11:48 -07002296
2297static void sysfs_add_func(struct work_struct *w)
2298{
2299 struct kmem_cache *s;
2300
2301 down_write(&slub_lock);
2302 list_for_each_entry(s, &slab_caches, list) {
2303 if (s->flags & __SYSFS_ADD_DEFERRED) {
2304 s->flags &= ~__SYSFS_ADD_DEFERRED;
2305 sysfs_slab_add(s);
2306 }
2307 }
2308 up_write(&slub_lock);
2309}
2310
2311static DECLARE_WORK(sysfs_add_work, sysfs_add_func);
2312
Christoph Lameter2e443fd2007-07-17 04:03:24 -07002313static noinline struct kmem_cache *dma_kmalloc_cache(int index, gfp_t flags)
2314{
2315 struct kmem_cache *s;
Christoph Lameter2e443fd2007-07-17 04:03:24 -07002316 char *text;
2317 size_t realsize;
2318
2319 s = kmalloc_caches_dma[index];
2320 if (s)
2321 return s;
2322
2323 /* Dynamically create dma cache */
Christoph Lameter1ceef402007-08-07 15:11:48 -07002324 if (flags & __GFP_WAIT)
2325 down_write(&slub_lock);
2326 else {
2327 if (!down_write_trylock(&slub_lock))
2328 goto out;
2329 }
2330
2331 if (kmalloc_caches_dma[index])
2332 goto unlock_out;
Christoph Lameter2e443fd2007-07-17 04:03:24 -07002333
Christoph Lameter7b55f622007-07-17 04:03:27 -07002334 realsize = kmalloc_caches[index].objsize;
Christoph Lameter1ceef402007-08-07 15:11:48 -07002335 text = kasprintf(flags & ~SLUB_DMA, "kmalloc_dma-%d", (unsigned int)realsize),
2336 s = kmalloc(kmem_size, flags & ~SLUB_DMA);
2337
2338 if (!s || !text || !kmem_cache_open(s, flags, text,
2339 realsize, ARCH_KMALLOC_MINALIGN,
2340 SLAB_CACHE_DMA|__SYSFS_ADD_DEFERRED, NULL)) {
2341 kfree(s);
2342 kfree(text);
2343 goto unlock_out;
Christoph Lameterdfce8642007-07-17 04:03:25 -07002344 }
Christoph Lameter1ceef402007-08-07 15:11:48 -07002345
2346 list_add(&s->list, &slab_caches);
2347 kmalloc_caches_dma[index] = s;
2348
2349 schedule_work(&sysfs_add_work);
2350
2351unlock_out:
Christoph Lameterdfce8642007-07-17 04:03:25 -07002352 up_write(&slub_lock);
Christoph Lameter1ceef402007-08-07 15:11:48 -07002353out:
Christoph Lameterdfce8642007-07-17 04:03:25 -07002354 return kmalloc_caches_dma[index];
Christoph Lameter2e443fd2007-07-17 04:03:24 -07002355}
2356#endif
2357
Christoph Lameterf1b26332007-07-17 04:03:26 -07002358/*
2359 * Conversion table for small slabs sizes / 8 to the index in the
2360 * kmalloc array. This is necessary for slabs < 192 since we have non power
2361 * of two cache sizes there. The size of larger slabs can be determined using
2362 * fls.
2363 */
2364static s8 size_index[24] = {
2365 3, /* 8 */
2366 4, /* 16 */
2367 5, /* 24 */
2368 5, /* 32 */
2369 6, /* 40 */
2370 6, /* 48 */
2371 6, /* 56 */
2372 6, /* 64 */
2373 1, /* 72 */
2374 1, /* 80 */
2375 1, /* 88 */
2376 1, /* 96 */
2377 7, /* 104 */
2378 7, /* 112 */
2379 7, /* 120 */
2380 7, /* 128 */
2381 2, /* 136 */
2382 2, /* 144 */
2383 2, /* 152 */
2384 2, /* 160 */
2385 2, /* 168 */
2386 2, /* 176 */
2387 2, /* 184 */
2388 2 /* 192 */
2389};
2390
Christoph Lameter81819f02007-05-06 14:49:36 -07002391static struct kmem_cache *get_slab(size_t size, gfp_t flags)
2392{
Christoph Lameterf1b26332007-07-17 04:03:26 -07002393 int index;
Christoph Lameter81819f02007-05-06 14:49:36 -07002394
Christoph Lameterf1b26332007-07-17 04:03:26 -07002395 if (size <= 192) {
2396 if (!size)
2397 return ZERO_SIZE_PTR;
Christoph Lameter81819f02007-05-06 14:49:36 -07002398
Christoph Lameterf1b26332007-07-17 04:03:26 -07002399 index = size_index[(size - 1) / 8];
2400 } else {
2401 if (size > KMALLOC_MAX_SIZE)
2402 return NULL;
2403
2404 index = fls(size - 1);
2405 }
Christoph Lameter81819f02007-05-06 14:49:36 -07002406
2407#ifdef CONFIG_ZONE_DMA
Christoph Lameterf1b26332007-07-17 04:03:26 -07002408 if (unlikely((flags & SLUB_DMA)))
Christoph Lameter2e443fd2007-07-17 04:03:24 -07002409 return dma_kmalloc_cache(index, flags);
Christoph Lameterf1b26332007-07-17 04:03:26 -07002410
Christoph Lameter81819f02007-05-06 14:49:36 -07002411#endif
2412 return &kmalloc_caches[index];
2413}
2414
2415void *__kmalloc(size_t size, gfp_t flags)
2416{
2417 struct kmem_cache *s = get_slab(size, flags);
2418
Christoph Lameter6cb8f912007-07-17 04:03:22 -07002419 if (ZERO_OR_NULL_PTR(s))
2420 return s;
2421
Christoph Lameterce15fea2007-07-17 04:03:28 -07002422 return slab_alloc(s, flags, -1, __builtin_return_address(0));
Christoph Lameter81819f02007-05-06 14:49:36 -07002423}
2424EXPORT_SYMBOL(__kmalloc);
2425
2426#ifdef CONFIG_NUMA
2427void *__kmalloc_node(size_t size, gfp_t flags, int node)
2428{
2429 struct kmem_cache *s = get_slab(size, flags);
2430
Christoph Lameter6cb8f912007-07-17 04:03:22 -07002431 if (ZERO_OR_NULL_PTR(s))
2432 return s;
2433
Christoph Lameterce15fea2007-07-17 04:03:28 -07002434 return slab_alloc(s, flags, node, __builtin_return_address(0));
Christoph Lameter81819f02007-05-06 14:49:36 -07002435}
2436EXPORT_SYMBOL(__kmalloc_node);
2437#endif
2438
2439size_t ksize(const void *object)
2440{
Christoph Lameter272c1d22007-06-08 13:46:49 -07002441 struct page *page;
Christoph Lameter81819f02007-05-06 14:49:36 -07002442 struct kmem_cache *s;
2443
Linus Torvalds9550b102007-07-19 13:21:34 -07002444 if (ZERO_OR_NULL_PTR(object))
Christoph Lameter272c1d22007-06-08 13:46:49 -07002445 return 0;
2446
2447 page = get_object_page(object);
Christoph Lameter81819f02007-05-06 14:49:36 -07002448 BUG_ON(!page);
2449 s = page->slab;
2450 BUG_ON(!s);
2451
2452 /*
2453 * Debugging requires use of the padding between object
2454 * and whatever may come after it.
2455 */
2456 if (s->flags & (SLAB_RED_ZONE | SLAB_POISON))
2457 return s->objsize;
2458
2459 /*
2460 * If we have the need to store the freelist pointer
2461 * back there or track user information then we can
2462 * only use the space before that information.
2463 */
2464 if (s->flags & (SLAB_DESTROY_BY_RCU | SLAB_STORE_USER))
2465 return s->inuse;
2466
2467 /*
2468 * Else we can use all the padding etc for the allocation
2469 */
2470 return s->size;
2471}
2472EXPORT_SYMBOL(ksize);
2473
2474void kfree(const void *x)
2475{
2476 struct kmem_cache *s;
2477 struct page *page;
2478
Christoph Lameter272c1d22007-06-08 13:46:49 -07002479 /*
2480 * This has to be an unsigned comparison. According to Linus
2481 * some gcc version treat a pointer as a signed entity. Then
2482 * this comparison would be true for all "negative" pointers
2483 * (which would cover the whole upper half of the address space).
2484 */
Christoph Lameter6cb8f912007-07-17 04:03:22 -07002485 if (ZERO_OR_NULL_PTR(x))
Christoph Lameter81819f02007-05-06 14:49:36 -07002486 return;
2487
Christoph Lameterb49af682007-05-06 14:49:41 -07002488 page = virt_to_head_page(x);
Christoph Lameter81819f02007-05-06 14:49:36 -07002489 s = page->slab;
2490
Christoph Lameter77c5e2d2007-05-06 14:49:42 -07002491 slab_free(s, page, (void *)x, __builtin_return_address(0));
Christoph Lameter81819f02007-05-06 14:49:36 -07002492}
2493EXPORT_SYMBOL(kfree);
2494
Christoph Lameter2086d262007-05-06 14:49:46 -07002495/*
Christoph Lameter672bba32007-05-09 02:32:39 -07002496 * kmem_cache_shrink removes empty slabs from the partial lists and sorts
2497 * the remaining slabs by the number of items in use. The slabs with the
2498 * most items in use come first. New allocations will then fill those up
2499 * and thus they can be removed from the partial lists.
2500 *
2501 * The slabs with the least items are placed last. This results in them
2502 * being allocated from last increasing the chance that the last objects
2503 * are freed in them.
Christoph Lameter2086d262007-05-06 14:49:46 -07002504 */
2505int kmem_cache_shrink(struct kmem_cache *s)
2506{
2507 int node;
2508 int i;
2509 struct kmem_cache_node *n;
2510 struct page *page;
2511 struct page *t;
2512 struct list_head *slabs_by_inuse =
2513 kmalloc(sizeof(struct list_head) * s->objects, GFP_KERNEL);
2514 unsigned long flags;
2515
2516 if (!slabs_by_inuse)
2517 return -ENOMEM;
2518
2519 flush_all(s);
2520 for_each_online_node(node) {
2521 n = get_node(s, node);
2522
2523 if (!n->nr_partial)
2524 continue;
2525
2526 for (i = 0; i < s->objects; i++)
2527 INIT_LIST_HEAD(slabs_by_inuse + i);
2528
2529 spin_lock_irqsave(&n->list_lock, flags);
2530
2531 /*
Christoph Lameter672bba32007-05-09 02:32:39 -07002532 * Build lists indexed by the items in use in each slab.
Christoph Lameter2086d262007-05-06 14:49:46 -07002533 *
Christoph Lameter672bba32007-05-09 02:32:39 -07002534 * Note that concurrent frees may occur while we hold the
2535 * list_lock. page->inuse here is the upper limit.
Christoph Lameter2086d262007-05-06 14:49:46 -07002536 */
2537 list_for_each_entry_safe(page, t, &n->partial, lru) {
2538 if (!page->inuse && slab_trylock(page)) {
2539 /*
2540 * Must hold slab lock here because slab_free
2541 * may have freed the last object and be
2542 * waiting to release the slab.
2543 */
2544 list_del(&page->lru);
2545 n->nr_partial--;
2546 slab_unlock(page);
2547 discard_slab(s, page);
2548 } else {
Christoph Lameterfcda3d82007-07-30 13:06:46 -07002549 list_move(&page->lru,
2550 slabs_by_inuse + page->inuse);
Christoph Lameter2086d262007-05-06 14:49:46 -07002551 }
2552 }
2553
Christoph Lameter2086d262007-05-06 14:49:46 -07002554 /*
Christoph Lameter672bba32007-05-09 02:32:39 -07002555 * Rebuild the partial list with the slabs filled up most
2556 * first and the least used slabs at the end.
Christoph Lameter2086d262007-05-06 14:49:46 -07002557 */
2558 for (i = s->objects - 1; i >= 0; i--)
2559 list_splice(slabs_by_inuse + i, n->partial.prev);
2560
Christoph Lameter2086d262007-05-06 14:49:46 -07002561 spin_unlock_irqrestore(&n->list_lock, flags);
2562 }
2563
2564 kfree(slabs_by_inuse);
2565 return 0;
2566}
2567EXPORT_SYMBOL(kmem_cache_shrink);
2568
Christoph Lameter81819f02007-05-06 14:49:36 -07002569/********************************************************************
2570 * Basic setup of slabs
2571 *******************************************************************/
2572
2573void __init kmem_cache_init(void)
2574{
2575 int i;
Christoph Lameter4b356be2007-06-16 10:16:13 -07002576 int caches = 0;
Christoph Lameter81819f02007-05-06 14:49:36 -07002577
2578#ifdef CONFIG_NUMA
2579 /*
2580 * Must first have the slab cache available for the allocations of the
Christoph Lameter672bba32007-05-09 02:32:39 -07002581 * struct kmem_cache_node's. There is special bootstrap code in
Christoph Lameter81819f02007-05-06 14:49:36 -07002582 * kmem_cache_open for slab_state == DOWN.
2583 */
2584 create_kmalloc_cache(&kmalloc_caches[0], "kmem_cache_node",
2585 sizeof(struct kmem_cache_node), GFP_KERNEL);
Christoph Lameter8ffa6872007-05-31 00:40:51 -07002586 kmalloc_caches[0].refcount = -1;
Christoph Lameter4b356be2007-06-16 10:16:13 -07002587 caches++;
Christoph Lameter81819f02007-05-06 14:49:36 -07002588#endif
2589
2590 /* Able to allocate the per node structures */
2591 slab_state = PARTIAL;
2592
2593 /* Caches that are not of the two-to-the-power-of size */
Christoph Lameter4b356be2007-06-16 10:16:13 -07002594 if (KMALLOC_MIN_SIZE <= 64) {
2595 create_kmalloc_cache(&kmalloc_caches[1],
Christoph Lameter81819f02007-05-06 14:49:36 -07002596 "kmalloc-96", 96, GFP_KERNEL);
Christoph Lameter4b356be2007-06-16 10:16:13 -07002597 caches++;
2598 }
2599 if (KMALLOC_MIN_SIZE <= 128) {
2600 create_kmalloc_cache(&kmalloc_caches[2],
Christoph Lameter81819f02007-05-06 14:49:36 -07002601 "kmalloc-192", 192, GFP_KERNEL);
Christoph Lameter4b356be2007-06-16 10:16:13 -07002602 caches++;
2603 }
Christoph Lameter81819f02007-05-06 14:49:36 -07002604
Christoph Lameter4b356be2007-06-16 10:16:13 -07002605 for (i = KMALLOC_SHIFT_LOW; i <= KMALLOC_SHIFT_HIGH; i++) {
Christoph Lameter81819f02007-05-06 14:49:36 -07002606 create_kmalloc_cache(&kmalloc_caches[i],
2607 "kmalloc", 1 << i, GFP_KERNEL);
Christoph Lameter4b356be2007-06-16 10:16:13 -07002608 caches++;
2609 }
Christoph Lameter81819f02007-05-06 14:49:36 -07002610
Christoph Lameterf1b26332007-07-17 04:03:26 -07002611
2612 /*
2613 * Patch up the size_index table if we have strange large alignment
2614 * requirements for the kmalloc array. This is only the case for
2615 * mips it seems. The standard arches will not generate any code here.
2616 *
2617 * Largest permitted alignment is 256 bytes due to the way we
2618 * handle the index determination for the smaller caches.
2619 *
2620 * Make sure that nothing crazy happens if someone starts tinkering
2621 * around with ARCH_KMALLOC_MINALIGN
2622 */
2623 BUILD_BUG_ON(KMALLOC_MIN_SIZE > 256 ||
2624 (KMALLOC_MIN_SIZE & (KMALLOC_MIN_SIZE - 1)));
2625
Christoph Lameter12ad6842007-07-17 04:03:28 -07002626 for (i = 8; i < KMALLOC_MIN_SIZE; i += 8)
Christoph Lameterf1b26332007-07-17 04:03:26 -07002627 size_index[(i - 1) / 8] = KMALLOC_SHIFT_LOW;
2628
Christoph Lameter81819f02007-05-06 14:49:36 -07002629 slab_state = UP;
2630
2631 /* Provide the correct kmalloc names now that the caches are up */
2632 for (i = KMALLOC_SHIFT_LOW; i <= KMALLOC_SHIFT_HIGH; i++)
2633 kmalloc_caches[i]. name =
2634 kasprintf(GFP_KERNEL, "kmalloc-%d", 1 << i);
2635
2636#ifdef CONFIG_SMP
2637 register_cpu_notifier(&slab_notifier);
2638#endif
2639
Christoph Lameterbcf889f2007-05-10 03:15:44 -07002640 kmem_size = offsetof(struct kmem_cache, cpu_slab) +
2641 nr_cpu_ids * sizeof(struct page *);
Christoph Lameter81819f02007-05-06 14:49:36 -07002642
2643 printk(KERN_INFO "SLUB: Genslabs=%d, HWalign=%d, Order=%d-%d, MinObjects=%d,"
Christoph Lameter4b356be2007-06-16 10:16:13 -07002644 " CPUs=%d, Nodes=%d\n",
2645 caches, cache_line_size(),
Christoph Lameter81819f02007-05-06 14:49:36 -07002646 slub_min_order, slub_max_order, slub_min_objects,
2647 nr_cpu_ids, nr_node_ids);
2648}
2649
2650/*
2651 * Find a mergeable slab cache
2652 */
2653static int slab_unmergeable(struct kmem_cache *s)
2654{
2655 if (slub_nomerge || (s->flags & SLUB_NEVER_MERGE))
2656 return 1;
2657
Christoph Lameterc59def9f2007-05-16 22:10:50 -07002658 if (s->ctor)
Christoph Lameter81819f02007-05-06 14:49:36 -07002659 return 1;
2660
Christoph Lameter8ffa6872007-05-31 00:40:51 -07002661 /*
2662 * We may have set a slab to be unmergeable during bootstrap.
2663 */
2664 if (s->refcount < 0)
2665 return 1;
2666
Christoph Lameter81819f02007-05-06 14:49:36 -07002667 return 0;
2668}
2669
2670static struct kmem_cache *find_mergeable(size_t size,
Christoph Lameterba0268a2007-09-11 15:24:11 -07002671 size_t align, unsigned long flags, const char *name,
Christoph Lameterc59def9f2007-05-16 22:10:50 -07002672 void (*ctor)(void *, struct kmem_cache *, unsigned long))
Christoph Lameter81819f02007-05-06 14:49:36 -07002673{
Christoph Lameter5b95a4a2007-07-17 04:03:19 -07002674 struct kmem_cache *s;
Christoph Lameter81819f02007-05-06 14:49:36 -07002675
2676 if (slub_nomerge || (flags & SLUB_NEVER_MERGE))
2677 return NULL;
2678
Christoph Lameterc59def9f2007-05-16 22:10:50 -07002679 if (ctor)
Christoph Lameter81819f02007-05-06 14:49:36 -07002680 return NULL;
2681
2682 size = ALIGN(size, sizeof(void *));
2683 align = calculate_alignment(flags, align, size);
2684 size = ALIGN(size, align);
Christoph Lameterba0268a2007-09-11 15:24:11 -07002685 flags = kmem_cache_flags(size, flags, name, NULL);
Christoph Lameter81819f02007-05-06 14:49:36 -07002686
Christoph Lameter5b95a4a2007-07-17 04:03:19 -07002687 list_for_each_entry(s, &slab_caches, list) {
Christoph Lameter81819f02007-05-06 14:49:36 -07002688 if (slab_unmergeable(s))
2689 continue;
2690
2691 if (size > s->size)
2692 continue;
2693
Christoph Lameterba0268a2007-09-11 15:24:11 -07002694 if ((flags & SLUB_MERGE_SAME) != (s->flags & SLUB_MERGE_SAME))
Christoph Lameter81819f02007-05-06 14:49:36 -07002695 continue;
2696 /*
2697 * Check if alignment is compatible.
2698 * Courtesy of Adrian Drzewiecki
2699 */
2700 if ((s->size & ~(align -1)) != s->size)
2701 continue;
2702
2703 if (s->size - size >= sizeof(void *))
2704 continue;
2705
2706 return s;
2707 }
2708 return NULL;
2709}
2710
2711struct kmem_cache *kmem_cache_create(const char *name, size_t size,
2712 size_t align, unsigned long flags,
Paul Mundt20c2df82007-07-20 10:11:58 +09002713 void (*ctor)(void *, struct kmem_cache *, unsigned long))
Christoph Lameter81819f02007-05-06 14:49:36 -07002714{
2715 struct kmem_cache *s;
2716
2717 down_write(&slub_lock);
Christoph Lameterba0268a2007-09-11 15:24:11 -07002718 s = find_mergeable(size, align, flags, name, ctor);
Christoph Lameter81819f02007-05-06 14:49:36 -07002719 if (s) {
2720 s->refcount++;
2721 /*
2722 * Adjust the object sizes so that we clear
2723 * the complete object on kzalloc.
2724 */
2725 s->objsize = max(s->objsize, (int)size);
2726 s->inuse = max_t(int, s->inuse, ALIGN(size, sizeof(void *)));
Christoph Lametera0e1d1b2007-07-17 04:03:31 -07002727 up_write(&slub_lock);
Christoph Lameter81819f02007-05-06 14:49:36 -07002728 if (sysfs_slab_alias(s, name))
2729 goto err;
Christoph Lametera0e1d1b2007-07-17 04:03:31 -07002730 return s;
2731 }
2732 s = kmalloc(kmem_size, GFP_KERNEL);
2733 if (s) {
2734 if (kmem_cache_open(s, GFP_KERNEL, name,
Christoph Lameterc59def9f2007-05-16 22:10:50 -07002735 size, align, flags, ctor)) {
Christoph Lameter81819f02007-05-06 14:49:36 -07002736 list_add(&s->list, &slab_caches);
Christoph Lametera0e1d1b2007-07-17 04:03:31 -07002737 up_write(&slub_lock);
2738 if (sysfs_slab_add(s))
2739 goto err;
2740 return s;
2741 }
2742 kfree(s);
Christoph Lameter81819f02007-05-06 14:49:36 -07002743 }
2744 up_write(&slub_lock);
Christoph Lameter81819f02007-05-06 14:49:36 -07002745
2746err:
Christoph Lameter81819f02007-05-06 14:49:36 -07002747 if (flags & SLAB_PANIC)
2748 panic("Cannot create slabcache %s\n", name);
2749 else
2750 s = NULL;
2751 return s;
2752}
2753EXPORT_SYMBOL(kmem_cache_create);
2754
Christoph Lameter81819f02007-05-06 14:49:36 -07002755#ifdef CONFIG_SMP
Christoph Lameter27390bc2007-06-01 00:47:09 -07002756/*
Christoph Lameter672bba32007-05-09 02:32:39 -07002757 * Use the cpu notifier to insure that the cpu slabs are flushed when
2758 * necessary.
Christoph Lameter81819f02007-05-06 14:49:36 -07002759 */
2760static int __cpuinit slab_cpuup_callback(struct notifier_block *nfb,
2761 unsigned long action, void *hcpu)
2762{
2763 long cpu = (long)hcpu;
Christoph Lameter5b95a4a2007-07-17 04:03:19 -07002764 struct kmem_cache *s;
2765 unsigned long flags;
Christoph Lameter81819f02007-05-06 14:49:36 -07002766
2767 switch (action) {
2768 case CPU_UP_CANCELED:
Rafael J. Wysocki8bb78442007-05-09 02:35:10 -07002769 case CPU_UP_CANCELED_FROZEN:
Christoph Lameter81819f02007-05-06 14:49:36 -07002770 case CPU_DEAD:
Rafael J. Wysocki8bb78442007-05-09 02:35:10 -07002771 case CPU_DEAD_FROZEN:
Christoph Lameter5b95a4a2007-07-17 04:03:19 -07002772 down_read(&slub_lock);
2773 list_for_each_entry(s, &slab_caches, list) {
2774 local_irq_save(flags);
2775 __flush_cpu_slab(s, cpu);
2776 local_irq_restore(flags);
2777 }
2778 up_read(&slub_lock);
Christoph Lameter81819f02007-05-06 14:49:36 -07002779 break;
2780 default:
2781 break;
2782 }
2783 return NOTIFY_OK;
2784}
2785
2786static struct notifier_block __cpuinitdata slab_notifier =
2787 { &slab_cpuup_callback, NULL, 0 };
2788
2789#endif
2790
Christoph Lameter81819f02007-05-06 14:49:36 -07002791void *__kmalloc_track_caller(size_t size, gfp_t gfpflags, void *caller)
2792{
2793 struct kmem_cache *s = get_slab(size, gfpflags);
Christoph Lameter81819f02007-05-06 14:49:36 -07002794
Christoph Lameter6cb8f912007-07-17 04:03:22 -07002795 if (ZERO_OR_NULL_PTR(s))
2796 return s;
Christoph Lameter81819f02007-05-06 14:49:36 -07002797
Christoph Lameterce15fea2007-07-17 04:03:28 -07002798 return slab_alloc(s, gfpflags, -1, caller);
Christoph Lameter81819f02007-05-06 14:49:36 -07002799}
2800
2801void *__kmalloc_node_track_caller(size_t size, gfp_t gfpflags,
2802 int node, void *caller)
2803{
2804 struct kmem_cache *s = get_slab(size, gfpflags);
Christoph Lameter81819f02007-05-06 14:49:36 -07002805
Christoph Lameter6cb8f912007-07-17 04:03:22 -07002806 if (ZERO_OR_NULL_PTR(s))
2807 return s;
Christoph Lameter81819f02007-05-06 14:49:36 -07002808
Christoph Lameterce15fea2007-07-17 04:03:28 -07002809 return slab_alloc(s, gfpflags, node, caller);
Christoph Lameter81819f02007-05-06 14:49:36 -07002810}
2811
Christoph Lameter41ecc552007-05-09 02:32:44 -07002812#if defined(CONFIG_SYSFS) && defined(CONFIG_SLUB_DEBUG)
Christoph Lameter434e2452007-07-17 04:03:30 -07002813static int validate_slab(struct kmem_cache *s, struct page *page,
2814 unsigned long *map)
Christoph Lameter53e15af2007-05-06 14:49:43 -07002815{
2816 void *p;
2817 void *addr = page_address(page);
Christoph Lameter53e15af2007-05-06 14:49:43 -07002818
2819 if (!check_slab(s, page) ||
2820 !on_freelist(s, page, NULL))
2821 return 0;
2822
2823 /* Now we know that a valid freelist exists */
2824 bitmap_zero(map, s->objects);
2825
Christoph Lameter7656c722007-05-09 02:32:40 -07002826 for_each_free_object(p, s, page->freelist) {
2827 set_bit(slab_index(p, s, addr), map);
Christoph Lameter53e15af2007-05-06 14:49:43 -07002828 if (!check_object(s, page, p, 0))
2829 return 0;
2830 }
2831
Christoph Lameter7656c722007-05-09 02:32:40 -07002832 for_each_object(p, s, addr)
2833 if (!test_bit(slab_index(p, s, addr), map))
Christoph Lameter53e15af2007-05-06 14:49:43 -07002834 if (!check_object(s, page, p, 1))
2835 return 0;
2836 return 1;
2837}
2838
Christoph Lameter434e2452007-07-17 04:03:30 -07002839static void validate_slab_slab(struct kmem_cache *s, struct page *page,
2840 unsigned long *map)
Christoph Lameter53e15af2007-05-06 14:49:43 -07002841{
2842 if (slab_trylock(page)) {
Christoph Lameter434e2452007-07-17 04:03:30 -07002843 validate_slab(s, page, map);
Christoph Lameter53e15af2007-05-06 14:49:43 -07002844 slab_unlock(page);
2845 } else
2846 printk(KERN_INFO "SLUB %s: Skipped busy slab 0x%p\n",
2847 s->name, page);
2848
2849 if (s->flags & DEBUG_DEFAULT_FLAGS) {
Christoph Lameter35e5d7e2007-05-09 02:32:42 -07002850 if (!SlabDebug(page))
2851 printk(KERN_ERR "SLUB %s: SlabDebug not set "
Christoph Lameter53e15af2007-05-06 14:49:43 -07002852 "on slab 0x%p\n", s->name, page);
2853 } else {
Christoph Lameter35e5d7e2007-05-09 02:32:42 -07002854 if (SlabDebug(page))
2855 printk(KERN_ERR "SLUB %s: SlabDebug set on "
Christoph Lameter53e15af2007-05-06 14:49:43 -07002856 "slab 0x%p\n", s->name, page);
2857 }
2858}
2859
Christoph Lameter434e2452007-07-17 04:03:30 -07002860static int validate_slab_node(struct kmem_cache *s,
2861 struct kmem_cache_node *n, unsigned long *map)
Christoph Lameter53e15af2007-05-06 14:49:43 -07002862{
2863 unsigned long count = 0;
2864 struct page *page;
2865 unsigned long flags;
2866
2867 spin_lock_irqsave(&n->list_lock, flags);
2868
2869 list_for_each_entry(page, &n->partial, lru) {
Christoph Lameter434e2452007-07-17 04:03:30 -07002870 validate_slab_slab(s, page, map);
Christoph Lameter53e15af2007-05-06 14:49:43 -07002871 count++;
2872 }
2873 if (count != n->nr_partial)
2874 printk(KERN_ERR "SLUB %s: %ld partial slabs counted but "
2875 "counter=%ld\n", s->name, count, n->nr_partial);
2876
2877 if (!(s->flags & SLAB_STORE_USER))
2878 goto out;
2879
2880 list_for_each_entry(page, &n->full, lru) {
Christoph Lameter434e2452007-07-17 04:03:30 -07002881 validate_slab_slab(s, page, map);
Christoph Lameter53e15af2007-05-06 14:49:43 -07002882 count++;
2883 }
2884 if (count != atomic_long_read(&n->nr_slabs))
2885 printk(KERN_ERR "SLUB: %s %ld slabs counted but "
2886 "counter=%ld\n", s->name, count,
2887 atomic_long_read(&n->nr_slabs));
2888
2889out:
2890 spin_unlock_irqrestore(&n->list_lock, flags);
2891 return count;
2892}
2893
Christoph Lameter434e2452007-07-17 04:03:30 -07002894static long validate_slab_cache(struct kmem_cache *s)
Christoph Lameter53e15af2007-05-06 14:49:43 -07002895{
2896 int node;
2897 unsigned long count = 0;
Christoph Lameter434e2452007-07-17 04:03:30 -07002898 unsigned long *map = kmalloc(BITS_TO_LONGS(s->objects) *
2899 sizeof(unsigned long), GFP_KERNEL);
2900
2901 if (!map)
2902 return -ENOMEM;
Christoph Lameter53e15af2007-05-06 14:49:43 -07002903
2904 flush_all(s);
2905 for_each_online_node(node) {
2906 struct kmem_cache_node *n = get_node(s, node);
2907
Christoph Lameter434e2452007-07-17 04:03:30 -07002908 count += validate_slab_node(s, n, map);
Christoph Lameter53e15af2007-05-06 14:49:43 -07002909 }
Christoph Lameter434e2452007-07-17 04:03:30 -07002910 kfree(map);
Christoph Lameter53e15af2007-05-06 14:49:43 -07002911 return count;
2912}
2913
Christoph Lameterb3459702007-05-09 02:32:41 -07002914#ifdef SLUB_RESILIENCY_TEST
2915static void resiliency_test(void)
2916{
2917 u8 *p;
2918
2919 printk(KERN_ERR "SLUB resiliency testing\n");
2920 printk(KERN_ERR "-----------------------\n");
2921 printk(KERN_ERR "A. Corruption after allocation\n");
2922
2923 p = kzalloc(16, GFP_KERNEL);
2924 p[16] = 0x12;
2925 printk(KERN_ERR "\n1. kmalloc-16: Clobber Redzone/next pointer"
2926 " 0x12->0x%p\n\n", p + 16);
2927
2928 validate_slab_cache(kmalloc_caches + 4);
2929
2930 /* Hmmm... The next two are dangerous */
2931 p = kzalloc(32, GFP_KERNEL);
2932 p[32 + sizeof(void *)] = 0x34;
2933 printk(KERN_ERR "\n2. kmalloc-32: Clobber next pointer/next slab"
2934 " 0x34 -> -0x%p\n", p);
2935 printk(KERN_ERR "If allocated object is overwritten then not detectable\n\n");
2936
2937 validate_slab_cache(kmalloc_caches + 5);
2938 p = kzalloc(64, GFP_KERNEL);
2939 p += 64 + (get_cycles() & 0xff) * sizeof(void *);
2940 *p = 0x56;
2941 printk(KERN_ERR "\n3. kmalloc-64: corrupting random byte 0x56->0x%p\n",
2942 p);
2943 printk(KERN_ERR "If allocated object is overwritten then not detectable\n\n");
2944 validate_slab_cache(kmalloc_caches + 6);
2945
2946 printk(KERN_ERR "\nB. Corruption after free\n");
2947 p = kzalloc(128, GFP_KERNEL);
2948 kfree(p);
2949 *p = 0x78;
2950 printk(KERN_ERR "1. kmalloc-128: Clobber first word 0x78->0x%p\n\n", p);
2951 validate_slab_cache(kmalloc_caches + 7);
2952
2953 p = kzalloc(256, GFP_KERNEL);
2954 kfree(p);
2955 p[50] = 0x9a;
2956 printk(KERN_ERR "\n2. kmalloc-256: Clobber 50th byte 0x9a->0x%p\n\n", p);
2957 validate_slab_cache(kmalloc_caches + 8);
2958
2959 p = kzalloc(512, GFP_KERNEL);
2960 kfree(p);
2961 p[512] = 0xab;
2962 printk(KERN_ERR "\n3. kmalloc-512: Clobber redzone 0xab->0x%p\n\n", p);
2963 validate_slab_cache(kmalloc_caches + 9);
2964}
2965#else
2966static void resiliency_test(void) {};
2967#endif
2968
Christoph Lameter88a420e2007-05-06 14:49:45 -07002969/*
Christoph Lameter672bba32007-05-09 02:32:39 -07002970 * Generate lists of code addresses where slabcache objects are allocated
Christoph Lameter88a420e2007-05-06 14:49:45 -07002971 * and freed.
2972 */
2973
2974struct location {
2975 unsigned long count;
2976 void *addr;
Christoph Lameter45edfa52007-05-09 02:32:45 -07002977 long long sum_time;
2978 long min_time;
2979 long max_time;
2980 long min_pid;
2981 long max_pid;
2982 cpumask_t cpus;
2983 nodemask_t nodes;
Christoph Lameter88a420e2007-05-06 14:49:45 -07002984};
2985
2986struct loc_track {
2987 unsigned long max;
2988 unsigned long count;
2989 struct location *loc;
2990};
2991
2992static void free_loc_track(struct loc_track *t)
2993{
2994 if (t->max)
2995 free_pages((unsigned long)t->loc,
2996 get_order(sizeof(struct location) * t->max));
2997}
2998
Christoph Lameter68dff6a2007-07-17 04:03:20 -07002999static int alloc_loc_track(struct loc_track *t, unsigned long max, gfp_t flags)
Christoph Lameter88a420e2007-05-06 14:49:45 -07003000{
3001 struct location *l;
3002 int order;
3003
Christoph Lameter88a420e2007-05-06 14:49:45 -07003004 order = get_order(sizeof(struct location) * max);
3005
Christoph Lameter68dff6a2007-07-17 04:03:20 -07003006 l = (void *)__get_free_pages(flags, order);
Christoph Lameter88a420e2007-05-06 14:49:45 -07003007 if (!l)
3008 return 0;
3009
3010 if (t->count) {
3011 memcpy(l, t->loc, sizeof(struct location) * t->count);
3012 free_loc_track(t);
3013 }
3014 t->max = max;
3015 t->loc = l;
3016 return 1;
3017}
3018
3019static int add_location(struct loc_track *t, struct kmem_cache *s,
Christoph Lameter45edfa52007-05-09 02:32:45 -07003020 const struct track *track)
Christoph Lameter88a420e2007-05-06 14:49:45 -07003021{
3022 long start, end, pos;
3023 struct location *l;
3024 void *caddr;
Christoph Lameter45edfa52007-05-09 02:32:45 -07003025 unsigned long age = jiffies - track->when;
Christoph Lameter88a420e2007-05-06 14:49:45 -07003026
3027 start = -1;
3028 end = t->count;
3029
3030 for ( ; ; ) {
3031 pos = start + (end - start + 1) / 2;
3032
3033 /*
3034 * There is nothing at "end". If we end up there
3035 * we need to add something to before end.
3036 */
3037 if (pos == end)
3038 break;
3039
3040 caddr = t->loc[pos].addr;
Christoph Lameter45edfa52007-05-09 02:32:45 -07003041 if (track->addr == caddr) {
3042
3043 l = &t->loc[pos];
3044 l->count++;
3045 if (track->when) {
3046 l->sum_time += age;
3047 if (age < l->min_time)
3048 l->min_time = age;
3049 if (age > l->max_time)
3050 l->max_time = age;
3051
3052 if (track->pid < l->min_pid)
3053 l->min_pid = track->pid;
3054 if (track->pid > l->max_pid)
3055 l->max_pid = track->pid;
3056
3057 cpu_set(track->cpu, l->cpus);
3058 }
3059 node_set(page_to_nid(virt_to_page(track)), l->nodes);
Christoph Lameter88a420e2007-05-06 14:49:45 -07003060 return 1;
3061 }
3062
Christoph Lameter45edfa52007-05-09 02:32:45 -07003063 if (track->addr < caddr)
Christoph Lameter88a420e2007-05-06 14:49:45 -07003064 end = pos;
3065 else
3066 start = pos;
3067 }
3068
3069 /*
Christoph Lameter672bba32007-05-09 02:32:39 -07003070 * Not found. Insert new tracking element.
Christoph Lameter88a420e2007-05-06 14:49:45 -07003071 */
Christoph Lameter68dff6a2007-07-17 04:03:20 -07003072 if (t->count >= t->max && !alloc_loc_track(t, 2 * t->max, GFP_ATOMIC))
Christoph Lameter88a420e2007-05-06 14:49:45 -07003073 return 0;
3074
3075 l = t->loc + pos;
3076 if (pos < t->count)
3077 memmove(l + 1, l,
3078 (t->count - pos) * sizeof(struct location));
3079 t->count++;
3080 l->count = 1;
Christoph Lameter45edfa52007-05-09 02:32:45 -07003081 l->addr = track->addr;
3082 l->sum_time = age;
3083 l->min_time = age;
3084 l->max_time = age;
3085 l->min_pid = track->pid;
3086 l->max_pid = track->pid;
3087 cpus_clear(l->cpus);
3088 cpu_set(track->cpu, l->cpus);
3089 nodes_clear(l->nodes);
3090 node_set(page_to_nid(virt_to_page(track)), l->nodes);
Christoph Lameter88a420e2007-05-06 14:49:45 -07003091 return 1;
3092}
3093
3094static void process_slab(struct loc_track *t, struct kmem_cache *s,
3095 struct page *page, enum track_item alloc)
3096{
3097 void *addr = page_address(page);
Christoph Lameter7656c722007-05-09 02:32:40 -07003098 DECLARE_BITMAP(map, s->objects);
Christoph Lameter88a420e2007-05-06 14:49:45 -07003099 void *p;
3100
3101 bitmap_zero(map, s->objects);
Christoph Lameter7656c722007-05-09 02:32:40 -07003102 for_each_free_object(p, s, page->freelist)
3103 set_bit(slab_index(p, s, addr), map);
Christoph Lameter88a420e2007-05-06 14:49:45 -07003104
Christoph Lameter7656c722007-05-09 02:32:40 -07003105 for_each_object(p, s, addr)
Christoph Lameter45edfa52007-05-09 02:32:45 -07003106 if (!test_bit(slab_index(p, s, addr), map))
3107 add_location(t, s, get_track(s, p, alloc));
Christoph Lameter88a420e2007-05-06 14:49:45 -07003108}
3109
3110static int list_locations(struct kmem_cache *s, char *buf,
3111 enum track_item alloc)
3112{
3113 int n = 0;
3114 unsigned long i;
Christoph Lameter68dff6a2007-07-17 04:03:20 -07003115 struct loc_track t = { 0, 0, NULL };
Christoph Lameter88a420e2007-05-06 14:49:45 -07003116 int node;
3117
Christoph Lameter68dff6a2007-07-17 04:03:20 -07003118 if (!alloc_loc_track(&t, PAGE_SIZE / sizeof(struct location),
3119 GFP_KERNEL))
3120 return sprintf(buf, "Out of memory\n");
Christoph Lameter88a420e2007-05-06 14:49:45 -07003121
3122 /* Push back cpu slabs */
3123 flush_all(s);
3124
3125 for_each_online_node(node) {
3126 struct kmem_cache_node *n = get_node(s, node);
3127 unsigned long flags;
3128 struct page *page;
3129
Christoph Lameter9e869432007-08-22 14:01:56 -07003130 if (!atomic_long_read(&n->nr_slabs))
Christoph Lameter88a420e2007-05-06 14:49:45 -07003131 continue;
3132
3133 spin_lock_irqsave(&n->list_lock, flags);
3134 list_for_each_entry(page, &n->partial, lru)
3135 process_slab(&t, s, page, alloc);
3136 list_for_each_entry(page, &n->full, lru)
3137 process_slab(&t, s, page, alloc);
3138 spin_unlock_irqrestore(&n->list_lock, flags);
3139 }
3140
3141 for (i = 0; i < t.count; i++) {
Christoph Lameter45edfa52007-05-09 02:32:45 -07003142 struct location *l = &t.loc[i];
Christoph Lameter88a420e2007-05-06 14:49:45 -07003143
3144 if (n > PAGE_SIZE - 100)
3145 break;
Christoph Lameter45edfa52007-05-09 02:32:45 -07003146 n += sprintf(buf + n, "%7ld ", l->count);
3147
3148 if (l->addr)
3149 n += sprint_symbol(buf + n, (unsigned long)l->addr);
Christoph Lameter88a420e2007-05-06 14:49:45 -07003150 else
3151 n += sprintf(buf + n, "<not-available>");
Christoph Lameter45edfa52007-05-09 02:32:45 -07003152
3153 if (l->sum_time != l->min_time) {
3154 unsigned long remainder;
3155
3156 n += sprintf(buf + n, " age=%ld/%ld/%ld",
3157 l->min_time,
3158 div_long_long_rem(l->sum_time, l->count, &remainder),
3159 l->max_time);
3160 } else
3161 n += sprintf(buf + n, " age=%ld",
3162 l->min_time);
3163
3164 if (l->min_pid != l->max_pid)
3165 n += sprintf(buf + n, " pid=%ld-%ld",
3166 l->min_pid, l->max_pid);
3167 else
3168 n += sprintf(buf + n, " pid=%ld",
3169 l->min_pid);
3170
Christoph Lameter84966342007-06-23 17:16:32 -07003171 if (num_online_cpus() > 1 && !cpus_empty(l->cpus) &&
3172 n < PAGE_SIZE - 60) {
Christoph Lameter45edfa52007-05-09 02:32:45 -07003173 n += sprintf(buf + n, " cpus=");
3174 n += cpulist_scnprintf(buf + n, PAGE_SIZE - n - 50,
3175 l->cpus);
3176 }
3177
Christoph Lameter84966342007-06-23 17:16:32 -07003178 if (num_online_nodes() > 1 && !nodes_empty(l->nodes) &&
3179 n < PAGE_SIZE - 60) {
Christoph Lameter45edfa52007-05-09 02:32:45 -07003180 n += sprintf(buf + n, " nodes=");
3181 n += nodelist_scnprintf(buf + n, PAGE_SIZE - n - 50,
3182 l->nodes);
3183 }
3184
Christoph Lameter88a420e2007-05-06 14:49:45 -07003185 n += sprintf(buf + n, "\n");
3186 }
3187
3188 free_loc_track(&t);
3189 if (!t.count)
3190 n += sprintf(buf, "No data\n");
3191 return n;
3192}
3193
Christoph Lameter81819f02007-05-06 14:49:36 -07003194static unsigned long count_partial(struct kmem_cache_node *n)
3195{
3196 unsigned long flags;
3197 unsigned long x = 0;
3198 struct page *page;
3199
3200 spin_lock_irqsave(&n->list_lock, flags);
3201 list_for_each_entry(page, &n->partial, lru)
3202 x += page->inuse;
3203 spin_unlock_irqrestore(&n->list_lock, flags);
3204 return x;
3205}
3206
3207enum slab_stat_type {
3208 SL_FULL,
3209 SL_PARTIAL,
3210 SL_CPU,
3211 SL_OBJECTS
3212};
3213
3214#define SO_FULL (1 << SL_FULL)
3215#define SO_PARTIAL (1 << SL_PARTIAL)
3216#define SO_CPU (1 << SL_CPU)
3217#define SO_OBJECTS (1 << SL_OBJECTS)
3218
3219static unsigned long slab_objects(struct kmem_cache *s,
3220 char *buf, unsigned long flags)
3221{
3222 unsigned long total = 0;
3223 int cpu;
3224 int node;
3225 int x;
3226 unsigned long *nodes;
3227 unsigned long *per_cpu;
3228
3229 nodes = kzalloc(2 * sizeof(unsigned long) * nr_node_ids, GFP_KERNEL);
3230 per_cpu = nodes + nr_node_ids;
3231
3232 for_each_possible_cpu(cpu) {
3233 struct page *page = s->cpu_slab[cpu];
3234 int node;
3235
3236 if (page) {
3237 node = page_to_nid(page);
3238 if (flags & SO_CPU) {
3239 int x = 0;
3240
3241 if (flags & SO_OBJECTS)
3242 x = page->inuse;
3243 else
3244 x = 1;
3245 total += x;
3246 nodes[node] += x;
3247 }
3248 per_cpu[node]++;
3249 }
3250 }
3251
3252 for_each_online_node(node) {
3253 struct kmem_cache_node *n = get_node(s, node);
3254
3255 if (flags & SO_PARTIAL) {
3256 if (flags & SO_OBJECTS)
3257 x = count_partial(n);
3258 else
3259 x = n->nr_partial;
3260 total += x;
3261 nodes[node] += x;
3262 }
3263
3264 if (flags & SO_FULL) {
Christoph Lameter9e869432007-08-22 14:01:56 -07003265 int full_slabs = atomic_long_read(&n->nr_slabs)
Christoph Lameter81819f02007-05-06 14:49:36 -07003266 - per_cpu[node]
3267 - n->nr_partial;
3268
3269 if (flags & SO_OBJECTS)
3270 x = full_slabs * s->objects;
3271 else
3272 x = full_slabs;
3273 total += x;
3274 nodes[node] += x;
3275 }
3276 }
3277
3278 x = sprintf(buf, "%lu", total);
3279#ifdef CONFIG_NUMA
3280 for_each_online_node(node)
3281 if (nodes[node])
3282 x += sprintf(buf + x, " N%d=%lu",
3283 node, nodes[node]);
3284#endif
3285 kfree(nodes);
3286 return x + sprintf(buf + x, "\n");
3287}
3288
3289static int any_slab_objects(struct kmem_cache *s)
3290{
3291 int node;
3292 int cpu;
3293
3294 for_each_possible_cpu(cpu)
3295 if (s->cpu_slab[cpu])
3296 return 1;
3297
3298 for_each_node(node) {
3299 struct kmem_cache_node *n = get_node(s, node);
3300
Christoph Lameter9e869432007-08-22 14:01:56 -07003301 if (n->nr_partial || atomic_long_read(&n->nr_slabs))
Christoph Lameter81819f02007-05-06 14:49:36 -07003302 return 1;
3303 }
3304 return 0;
3305}
3306
3307#define to_slab_attr(n) container_of(n, struct slab_attribute, attr)
3308#define to_slab(n) container_of(n, struct kmem_cache, kobj);
3309
3310struct slab_attribute {
3311 struct attribute attr;
3312 ssize_t (*show)(struct kmem_cache *s, char *buf);
3313 ssize_t (*store)(struct kmem_cache *s, const char *x, size_t count);
3314};
3315
3316#define SLAB_ATTR_RO(_name) \
3317 static struct slab_attribute _name##_attr = __ATTR_RO(_name)
3318
3319#define SLAB_ATTR(_name) \
3320 static struct slab_attribute _name##_attr = \
3321 __ATTR(_name, 0644, _name##_show, _name##_store)
3322
Christoph Lameter81819f02007-05-06 14:49:36 -07003323static ssize_t slab_size_show(struct kmem_cache *s, char *buf)
3324{
3325 return sprintf(buf, "%d\n", s->size);
3326}
3327SLAB_ATTR_RO(slab_size);
3328
3329static ssize_t align_show(struct kmem_cache *s, char *buf)
3330{
3331 return sprintf(buf, "%d\n", s->align);
3332}
3333SLAB_ATTR_RO(align);
3334
3335static ssize_t object_size_show(struct kmem_cache *s, char *buf)
3336{
3337 return sprintf(buf, "%d\n", s->objsize);
3338}
3339SLAB_ATTR_RO(object_size);
3340
3341static ssize_t objs_per_slab_show(struct kmem_cache *s, char *buf)
3342{
3343 return sprintf(buf, "%d\n", s->objects);
3344}
3345SLAB_ATTR_RO(objs_per_slab);
3346
3347static ssize_t order_show(struct kmem_cache *s, char *buf)
3348{
3349 return sprintf(buf, "%d\n", s->order);
3350}
3351SLAB_ATTR_RO(order);
3352
3353static ssize_t ctor_show(struct kmem_cache *s, char *buf)
3354{
3355 if (s->ctor) {
3356 int n = sprint_symbol(buf, (unsigned long)s->ctor);
3357
3358 return n + sprintf(buf + n, "\n");
3359 }
3360 return 0;
3361}
3362SLAB_ATTR_RO(ctor);
3363
Christoph Lameter81819f02007-05-06 14:49:36 -07003364static ssize_t aliases_show(struct kmem_cache *s, char *buf)
3365{
3366 return sprintf(buf, "%d\n", s->refcount - 1);
3367}
3368SLAB_ATTR_RO(aliases);
3369
3370static ssize_t slabs_show(struct kmem_cache *s, char *buf)
3371{
3372 return slab_objects(s, buf, SO_FULL|SO_PARTIAL|SO_CPU);
3373}
3374SLAB_ATTR_RO(slabs);
3375
3376static ssize_t partial_show(struct kmem_cache *s, char *buf)
3377{
3378 return slab_objects(s, buf, SO_PARTIAL);
3379}
3380SLAB_ATTR_RO(partial);
3381
3382static ssize_t cpu_slabs_show(struct kmem_cache *s, char *buf)
3383{
3384 return slab_objects(s, buf, SO_CPU);
3385}
3386SLAB_ATTR_RO(cpu_slabs);
3387
3388static ssize_t objects_show(struct kmem_cache *s, char *buf)
3389{
3390 return slab_objects(s, buf, SO_FULL|SO_PARTIAL|SO_CPU|SO_OBJECTS);
3391}
3392SLAB_ATTR_RO(objects);
3393
3394static ssize_t sanity_checks_show(struct kmem_cache *s, char *buf)
3395{
3396 return sprintf(buf, "%d\n", !!(s->flags & SLAB_DEBUG_FREE));
3397}
3398
3399static ssize_t sanity_checks_store(struct kmem_cache *s,
3400 const char *buf, size_t length)
3401{
3402 s->flags &= ~SLAB_DEBUG_FREE;
3403 if (buf[0] == '1')
3404 s->flags |= SLAB_DEBUG_FREE;
3405 return length;
3406}
3407SLAB_ATTR(sanity_checks);
3408
3409static ssize_t trace_show(struct kmem_cache *s, char *buf)
3410{
3411 return sprintf(buf, "%d\n", !!(s->flags & SLAB_TRACE));
3412}
3413
3414static ssize_t trace_store(struct kmem_cache *s, const char *buf,
3415 size_t length)
3416{
3417 s->flags &= ~SLAB_TRACE;
3418 if (buf[0] == '1')
3419 s->flags |= SLAB_TRACE;
3420 return length;
3421}
3422SLAB_ATTR(trace);
3423
3424static ssize_t reclaim_account_show(struct kmem_cache *s, char *buf)
3425{
3426 return sprintf(buf, "%d\n", !!(s->flags & SLAB_RECLAIM_ACCOUNT));
3427}
3428
3429static ssize_t reclaim_account_store(struct kmem_cache *s,
3430 const char *buf, size_t length)
3431{
3432 s->flags &= ~SLAB_RECLAIM_ACCOUNT;
3433 if (buf[0] == '1')
3434 s->flags |= SLAB_RECLAIM_ACCOUNT;
3435 return length;
3436}
3437SLAB_ATTR(reclaim_account);
3438
3439static ssize_t hwcache_align_show(struct kmem_cache *s, char *buf)
3440{
Christoph Lameter5af60832007-05-06 14:49:56 -07003441 return sprintf(buf, "%d\n", !!(s->flags & SLAB_HWCACHE_ALIGN));
Christoph Lameter81819f02007-05-06 14:49:36 -07003442}
3443SLAB_ATTR_RO(hwcache_align);
3444
3445#ifdef CONFIG_ZONE_DMA
3446static ssize_t cache_dma_show(struct kmem_cache *s, char *buf)
3447{
3448 return sprintf(buf, "%d\n", !!(s->flags & SLAB_CACHE_DMA));
3449}
3450SLAB_ATTR_RO(cache_dma);
3451#endif
3452
3453static ssize_t destroy_by_rcu_show(struct kmem_cache *s, char *buf)
3454{
3455 return sprintf(buf, "%d\n", !!(s->flags & SLAB_DESTROY_BY_RCU));
3456}
3457SLAB_ATTR_RO(destroy_by_rcu);
3458
3459static ssize_t red_zone_show(struct kmem_cache *s, char *buf)
3460{
3461 return sprintf(buf, "%d\n", !!(s->flags & SLAB_RED_ZONE));
3462}
3463
3464static ssize_t red_zone_store(struct kmem_cache *s,
3465 const char *buf, size_t length)
3466{
3467 if (any_slab_objects(s))
3468 return -EBUSY;
3469
3470 s->flags &= ~SLAB_RED_ZONE;
3471 if (buf[0] == '1')
3472 s->flags |= SLAB_RED_ZONE;
3473 calculate_sizes(s);
3474 return length;
3475}
3476SLAB_ATTR(red_zone);
3477
3478static ssize_t poison_show(struct kmem_cache *s, char *buf)
3479{
3480 return sprintf(buf, "%d\n", !!(s->flags & SLAB_POISON));
3481}
3482
3483static ssize_t poison_store(struct kmem_cache *s,
3484 const char *buf, size_t length)
3485{
3486 if (any_slab_objects(s))
3487 return -EBUSY;
3488
3489 s->flags &= ~SLAB_POISON;
3490 if (buf[0] == '1')
3491 s->flags |= SLAB_POISON;
3492 calculate_sizes(s);
3493 return length;
3494}
3495SLAB_ATTR(poison);
3496
3497static ssize_t store_user_show(struct kmem_cache *s, char *buf)
3498{
3499 return sprintf(buf, "%d\n", !!(s->flags & SLAB_STORE_USER));
3500}
3501
3502static ssize_t store_user_store(struct kmem_cache *s,
3503 const char *buf, size_t length)
3504{
3505 if (any_slab_objects(s))
3506 return -EBUSY;
3507
3508 s->flags &= ~SLAB_STORE_USER;
3509 if (buf[0] == '1')
3510 s->flags |= SLAB_STORE_USER;
3511 calculate_sizes(s);
3512 return length;
3513}
3514SLAB_ATTR(store_user);
3515
Christoph Lameter53e15af2007-05-06 14:49:43 -07003516static ssize_t validate_show(struct kmem_cache *s, char *buf)
3517{
3518 return 0;
3519}
3520
3521static ssize_t validate_store(struct kmem_cache *s,
3522 const char *buf, size_t length)
3523{
Christoph Lameter434e2452007-07-17 04:03:30 -07003524 int ret = -EINVAL;
3525
3526 if (buf[0] == '1') {
3527 ret = validate_slab_cache(s);
3528 if (ret >= 0)
3529 ret = length;
3530 }
3531 return ret;
Christoph Lameter53e15af2007-05-06 14:49:43 -07003532}
3533SLAB_ATTR(validate);
3534
Christoph Lameter2086d262007-05-06 14:49:46 -07003535static ssize_t shrink_show(struct kmem_cache *s, char *buf)
3536{
3537 return 0;
3538}
3539
3540static ssize_t shrink_store(struct kmem_cache *s,
3541 const char *buf, size_t length)
3542{
3543 if (buf[0] == '1') {
3544 int rc = kmem_cache_shrink(s);
3545
3546 if (rc)
3547 return rc;
3548 } else
3549 return -EINVAL;
3550 return length;
3551}
3552SLAB_ATTR(shrink);
3553
Christoph Lameter88a420e2007-05-06 14:49:45 -07003554static ssize_t alloc_calls_show(struct kmem_cache *s, char *buf)
3555{
3556 if (!(s->flags & SLAB_STORE_USER))
3557 return -ENOSYS;
3558 return list_locations(s, buf, TRACK_ALLOC);
3559}
3560SLAB_ATTR_RO(alloc_calls);
3561
3562static ssize_t free_calls_show(struct kmem_cache *s, char *buf)
3563{
3564 if (!(s->flags & SLAB_STORE_USER))
3565 return -ENOSYS;
3566 return list_locations(s, buf, TRACK_FREE);
3567}
3568SLAB_ATTR_RO(free_calls);
3569
Christoph Lameter81819f02007-05-06 14:49:36 -07003570#ifdef CONFIG_NUMA
3571static ssize_t defrag_ratio_show(struct kmem_cache *s, char *buf)
3572{
3573 return sprintf(buf, "%d\n", s->defrag_ratio / 10);
3574}
3575
3576static ssize_t defrag_ratio_store(struct kmem_cache *s,
3577 const char *buf, size_t length)
3578{
3579 int n = simple_strtoul(buf, NULL, 10);
3580
3581 if (n < 100)
3582 s->defrag_ratio = n * 10;
3583 return length;
3584}
3585SLAB_ATTR(defrag_ratio);
3586#endif
3587
3588static struct attribute * slab_attrs[] = {
3589 &slab_size_attr.attr,
3590 &object_size_attr.attr,
3591 &objs_per_slab_attr.attr,
3592 &order_attr.attr,
3593 &objects_attr.attr,
3594 &slabs_attr.attr,
3595 &partial_attr.attr,
3596 &cpu_slabs_attr.attr,
3597 &ctor_attr.attr,
Christoph Lameter81819f02007-05-06 14:49:36 -07003598 &aliases_attr.attr,
3599 &align_attr.attr,
3600 &sanity_checks_attr.attr,
3601 &trace_attr.attr,
3602 &hwcache_align_attr.attr,
3603 &reclaim_account_attr.attr,
3604 &destroy_by_rcu_attr.attr,
3605 &red_zone_attr.attr,
3606 &poison_attr.attr,
3607 &store_user_attr.attr,
Christoph Lameter53e15af2007-05-06 14:49:43 -07003608 &validate_attr.attr,
Christoph Lameter2086d262007-05-06 14:49:46 -07003609 &shrink_attr.attr,
Christoph Lameter88a420e2007-05-06 14:49:45 -07003610 &alloc_calls_attr.attr,
3611 &free_calls_attr.attr,
Christoph Lameter81819f02007-05-06 14:49:36 -07003612#ifdef CONFIG_ZONE_DMA
3613 &cache_dma_attr.attr,
3614#endif
3615#ifdef CONFIG_NUMA
3616 &defrag_ratio_attr.attr,
3617#endif
3618 NULL
3619};
3620
3621static struct attribute_group slab_attr_group = {
3622 .attrs = slab_attrs,
3623};
3624
3625static ssize_t slab_attr_show(struct kobject *kobj,
3626 struct attribute *attr,
3627 char *buf)
3628{
3629 struct slab_attribute *attribute;
3630 struct kmem_cache *s;
3631 int err;
3632
3633 attribute = to_slab_attr(attr);
3634 s = to_slab(kobj);
3635
3636 if (!attribute->show)
3637 return -EIO;
3638
3639 err = attribute->show(s, buf);
3640
3641 return err;
3642}
3643
3644static ssize_t slab_attr_store(struct kobject *kobj,
3645 struct attribute *attr,
3646 const char *buf, size_t len)
3647{
3648 struct slab_attribute *attribute;
3649 struct kmem_cache *s;
3650 int err;
3651
3652 attribute = to_slab_attr(attr);
3653 s = to_slab(kobj);
3654
3655 if (!attribute->store)
3656 return -EIO;
3657
3658 err = attribute->store(s, buf, len);
3659
3660 return err;
3661}
3662
3663static struct sysfs_ops slab_sysfs_ops = {
3664 .show = slab_attr_show,
3665 .store = slab_attr_store,
3666};
3667
3668static struct kobj_type slab_ktype = {
3669 .sysfs_ops = &slab_sysfs_ops,
3670};
3671
3672static int uevent_filter(struct kset *kset, struct kobject *kobj)
3673{
3674 struct kobj_type *ktype = get_ktype(kobj);
3675
3676 if (ktype == &slab_ktype)
3677 return 1;
3678 return 0;
3679}
3680
3681static struct kset_uevent_ops slab_uevent_ops = {
3682 .filter = uevent_filter,
3683};
3684
Adrian Bunk5af328a2007-07-17 04:03:27 -07003685static decl_subsys(slab, &slab_ktype, &slab_uevent_ops);
Christoph Lameter81819f02007-05-06 14:49:36 -07003686
3687#define ID_STR_LENGTH 64
3688
3689/* Create a unique string id for a slab cache:
3690 * format
3691 * :[flags-]size:[memory address of kmemcache]
3692 */
3693static char *create_unique_id(struct kmem_cache *s)
3694{
3695 char *name = kmalloc(ID_STR_LENGTH, GFP_KERNEL);
3696 char *p = name;
3697
3698 BUG_ON(!name);
3699
3700 *p++ = ':';
3701 /*
3702 * First flags affecting slabcache operations. We will only
3703 * get here for aliasable slabs so we do not need to support
3704 * too many flags. The flags here must cover all flags that
3705 * are matched during merging to guarantee that the id is
3706 * unique.
3707 */
3708 if (s->flags & SLAB_CACHE_DMA)
3709 *p++ = 'd';
3710 if (s->flags & SLAB_RECLAIM_ACCOUNT)
3711 *p++ = 'a';
3712 if (s->flags & SLAB_DEBUG_FREE)
3713 *p++ = 'F';
3714 if (p != name + 1)
3715 *p++ = '-';
3716 p += sprintf(p, "%07d", s->size);
3717 BUG_ON(p > name + ID_STR_LENGTH - 1);
3718 return name;
3719}
3720
3721static int sysfs_slab_add(struct kmem_cache *s)
3722{
3723 int err;
3724 const char *name;
3725 int unmergeable;
3726
3727 if (slab_state < SYSFS)
3728 /* Defer until later */
3729 return 0;
3730
3731 unmergeable = slab_unmergeable(s);
3732 if (unmergeable) {
3733 /*
3734 * Slabcache can never be merged so we can use the name proper.
3735 * This is typically the case for debug situations. In that
3736 * case we can catch duplicate names easily.
3737 */
Linus Torvalds0f9008e2007-05-07 12:31:58 -07003738 sysfs_remove_link(&slab_subsys.kobj, s->name);
Christoph Lameter81819f02007-05-06 14:49:36 -07003739 name = s->name;
3740 } else {
3741 /*
3742 * Create a unique name for the slab as a target
3743 * for the symlinks.
3744 */
3745 name = create_unique_id(s);
3746 }
3747
3748 kobj_set_kset_s(s, slab_subsys);
3749 kobject_set_name(&s->kobj, name);
3750 kobject_init(&s->kobj);
3751 err = kobject_add(&s->kobj);
3752 if (err)
3753 return err;
3754
3755 err = sysfs_create_group(&s->kobj, &slab_attr_group);
3756 if (err)
3757 return err;
3758 kobject_uevent(&s->kobj, KOBJ_ADD);
3759 if (!unmergeable) {
3760 /* Setup first alias */
3761 sysfs_slab_alias(s, s->name);
3762 kfree(name);
3763 }
3764 return 0;
3765}
3766
3767static void sysfs_slab_remove(struct kmem_cache *s)
3768{
3769 kobject_uevent(&s->kobj, KOBJ_REMOVE);
3770 kobject_del(&s->kobj);
3771}
3772
3773/*
3774 * Need to buffer aliases during bootup until sysfs becomes
3775 * available lest we loose that information.
3776 */
3777struct saved_alias {
3778 struct kmem_cache *s;
3779 const char *name;
3780 struct saved_alias *next;
3781};
3782
Adrian Bunk5af328a2007-07-17 04:03:27 -07003783static struct saved_alias *alias_list;
Christoph Lameter81819f02007-05-06 14:49:36 -07003784
3785static int sysfs_slab_alias(struct kmem_cache *s, const char *name)
3786{
3787 struct saved_alias *al;
3788
3789 if (slab_state == SYSFS) {
3790 /*
3791 * If we have a leftover link then remove it.
3792 */
Linus Torvalds0f9008e2007-05-07 12:31:58 -07003793 sysfs_remove_link(&slab_subsys.kobj, name);
3794 return sysfs_create_link(&slab_subsys.kobj,
Christoph Lameter81819f02007-05-06 14:49:36 -07003795 &s->kobj, name);
3796 }
3797
3798 al = kmalloc(sizeof(struct saved_alias), GFP_KERNEL);
3799 if (!al)
3800 return -ENOMEM;
3801
3802 al->s = s;
3803 al->name = name;
3804 al->next = alias_list;
3805 alias_list = al;
3806 return 0;
3807}
3808
3809static int __init slab_sysfs_init(void)
3810{
Christoph Lameter5b95a4a2007-07-17 04:03:19 -07003811 struct kmem_cache *s;
Christoph Lameter81819f02007-05-06 14:49:36 -07003812 int err;
3813
3814 err = subsystem_register(&slab_subsys);
3815 if (err) {
3816 printk(KERN_ERR "Cannot register slab subsystem.\n");
3817 return -ENOSYS;
3818 }
3819
Christoph Lameter26a7bd02007-05-09 02:32:39 -07003820 slab_state = SYSFS;
3821
Christoph Lameter5b95a4a2007-07-17 04:03:19 -07003822 list_for_each_entry(s, &slab_caches, list) {
Christoph Lameter26a7bd02007-05-09 02:32:39 -07003823 err = sysfs_slab_add(s);
Christoph Lameter5d540fb2007-08-30 23:56:26 -07003824 if (err)
3825 printk(KERN_ERR "SLUB: Unable to add boot slab %s"
3826 " to sysfs\n", s->name);
Christoph Lameter26a7bd02007-05-09 02:32:39 -07003827 }
Christoph Lameter81819f02007-05-06 14:49:36 -07003828
3829 while (alias_list) {
3830 struct saved_alias *al = alias_list;
3831
3832 alias_list = alias_list->next;
3833 err = sysfs_slab_alias(al->s, al->name);
Christoph Lameter5d540fb2007-08-30 23:56:26 -07003834 if (err)
3835 printk(KERN_ERR "SLUB: Unable to add boot slab alias"
3836 " %s to sysfs\n", s->name);
Christoph Lameter81819f02007-05-06 14:49:36 -07003837 kfree(al);
3838 }
3839
3840 resiliency_test();
3841 return 0;
3842}
3843
3844__initcall(slab_sysfs_init);
Christoph Lameter81819f02007-05-06 14:49:36 -07003845#endif