blob: 30354bfeb43d5b093669fb0ef62b213e0c3991f2 [file] [log] [blame]
Christoph Lameter81819f02007-05-06 14:49:36 -07001/*
2 * SLUB: A slab allocator that limits cache line use instead of queuing
3 * objects in per cpu and per node lists.
4 *
5 * The allocator synchronizes using per slab locks and only
6 * uses a centralized lock to manage a pool of partial slabs.
7 *
Christoph Lametercde53532008-07-04 09:59:22 -07008 * (C) 2007 SGI, Christoph Lameter
Christoph Lameter81819f02007-05-06 14:49:36 -07009 */
10
11#include <linux/mm.h>
Nick Piggin1eb5ac62009-05-05 19:13:44 +100012#include <linux/swap.h> /* struct reclaim_state */
Christoph Lameter81819f02007-05-06 14:49:36 -070013#include <linux/module.h>
14#include <linux/bit_spinlock.h>
15#include <linux/interrupt.h>
16#include <linux/bitops.h>
17#include <linux/slab.h>
Alexey Dobriyan7b3c3a52008-10-06 02:42:17 +040018#include <linux/proc_fs.h>
Christoph Lameter81819f02007-05-06 14:49:36 -070019#include <linux/seq_file.h>
Zhaolei02af61b2009-04-10 14:26:18 +080020#include <linux/kmemtrace.h>
Christoph Lameter81819f02007-05-06 14:49:36 -070021#include <linux/cpu.h>
22#include <linux/cpuset.h>
Catalin Marinas06f22f12009-06-11 13:23:18 +010023#include <linux/kmemleak.h>
Christoph Lameter81819f02007-05-06 14:49:36 -070024#include <linux/mempolicy.h>
25#include <linux/ctype.h>
Thomas Gleixner3ac7fe52008-04-30 00:55:01 -070026#include <linux/debugobjects.h>
Christoph Lameter81819f02007-05-06 14:49:36 -070027#include <linux/kallsyms.h>
Yasunori Gotob9049e22007-10-21 16:41:37 -070028#include <linux/memory.h>
Roman Zippelf8bd2252008-05-01 04:34:31 -070029#include <linux/math64.h>
Akinobu Mita773ff602008-12-23 19:37:01 +090030#include <linux/fault-inject.h>
Christoph Lameter81819f02007-05-06 14:49:36 -070031
32/*
33 * Lock order:
34 * 1. slab_lock(page)
35 * 2. slab->list_lock
36 *
37 * The slab_lock protects operations on the object of a particular
38 * slab and its metadata in the page struct. If the slab lock
39 * has been taken then no allocations nor frees can be performed
40 * on the objects in the slab nor can the slab be added or removed
41 * from the partial or full lists since this would mean modifying
42 * the page_struct of the slab.
43 *
44 * The list_lock protects the partial and full list on each node and
45 * the partial slab counter. If taken then no new slabs may be added or
46 * removed from the lists nor make the number of partial slabs be modified.
47 * (Note that the total number of slabs is an atomic value that may be
48 * modified without taking the list lock).
49 *
50 * The list_lock is a centralized lock and thus we avoid taking it as
51 * much as possible. As long as SLUB does not have to handle partial
52 * slabs, operations can continue without any centralized lock. F.e.
53 * allocating a long series of objects that fill up slabs does not require
54 * the list lock.
55 *
56 * The lock order is sometimes inverted when we are trying to get a slab
57 * off a list. We take the list_lock and then look for a page on the list
58 * to use. While we do that objects in the slabs may be freed. We can
59 * only operate on the slab if we have also taken the slab_lock. So we use
60 * a slab_trylock() on the slab. If trylock was successful then no frees
61 * can occur anymore and we can use the slab for allocations etc. If the
62 * slab_trylock() does not succeed then frees are in progress in the slab and
63 * we must stay away from it for a while since we may cause a bouncing
64 * cacheline if we try to acquire the lock. So go onto the next slab.
65 * If all pages are busy then we may allocate a new slab instead of reusing
66 * a partial slab. A new slab has noone operating on it and thus there is
67 * no danger of cacheline contention.
68 *
69 * Interrupts are disabled during allocation and deallocation in order to
70 * make the slab allocator safe to use in the context of an irq. In addition
71 * interrupts are disabled to ensure that the processor does not change
72 * while handling per_cpu slabs, due to kernel preemption.
73 *
74 * SLUB assigns one slab for allocation to each processor.
75 * Allocations only occur from these slabs called cpu slabs.
76 *
Christoph Lameter672bba32007-05-09 02:32:39 -070077 * Slabs with free elements are kept on a partial list and during regular
78 * operations no list for full slabs is used. If an object in a full slab is
Christoph Lameter81819f02007-05-06 14:49:36 -070079 * freed then the slab will show up again on the partial lists.
Christoph Lameter672bba32007-05-09 02:32:39 -070080 * We track full slabs for debugging purposes though because otherwise we
81 * cannot scan all objects.
Christoph Lameter81819f02007-05-06 14:49:36 -070082 *
83 * Slabs are freed when they become empty. Teardown and setup is
84 * minimal so we rely on the page allocators per cpu caches for
85 * fast frees and allocs.
86 *
87 * Overloading of page flags that are otherwise used for LRU management.
88 *
Christoph Lameter4b6f0752007-05-16 22:10:53 -070089 * PageActive The slab is frozen and exempt from list processing.
90 * This means that the slab is dedicated to a purpose
91 * such as satisfying allocations for a specific
92 * processor. Objects may be freed in the slab while
93 * it is frozen but slab_free will then skip the usual
94 * list operations. It is up to the processor holding
95 * the slab to integrate the slab into the slab lists
96 * when the slab is no longer needed.
97 *
98 * One use of this flag is to mark slabs that are
99 * used for allocations. Then such a slab becomes a cpu
100 * slab. The cpu slab may be equipped with an additional
Christoph Lameterdfb4f092007-10-16 01:26:05 -0700101 * freelist that allows lockless access to
Christoph Lameter894b8782007-05-10 03:15:16 -0700102 * free objects in addition to the regular freelist
103 * that requires the slab lock.
Christoph Lameter81819f02007-05-06 14:49:36 -0700104 *
105 * PageError Slab requires special handling due to debug
106 * options set. This moves slab handling out of
Christoph Lameter894b8782007-05-10 03:15:16 -0700107 * the fast path and disables lockless freelists.
Christoph Lameter81819f02007-05-06 14:49:36 -0700108 */
109
Christoph Lameter5577bd82007-05-16 22:10:56 -0700110#ifdef CONFIG_SLUB_DEBUG
Andy Whitcroft8a380822008-07-23 21:27:18 -0700111#define SLABDEBUG 1
Christoph Lameter5577bd82007-05-16 22:10:56 -0700112#else
113#define SLABDEBUG 0
114#endif
115
Christoph Lameter81819f02007-05-06 14:49:36 -0700116/*
117 * Issues still to be resolved:
118 *
Christoph Lameter81819f02007-05-06 14:49:36 -0700119 * - Support PAGE_ALLOC_DEBUG. Should be easy to do.
120 *
Christoph Lameter81819f02007-05-06 14:49:36 -0700121 * - Variable sizing of the per node arrays
122 */
123
124/* Enable to test recovery from slab corruption on boot */
125#undef SLUB_RESILIENCY_TEST
126
Christoph Lameter81819f02007-05-06 14:49:36 -0700127/*
Christoph Lameter2086d262007-05-06 14:49:46 -0700128 * Mininum number of partial slabs. These will be left on the partial
129 * lists even if they are empty. kmem_cache_shrink may reclaim them.
130 */
Christoph Lameter76be8952007-12-21 14:37:37 -0800131#define MIN_PARTIAL 5
Christoph Lametere95eed52007-05-06 14:49:44 -0700132
Christoph Lameter2086d262007-05-06 14:49:46 -0700133/*
134 * Maximum number of desirable partial slabs.
135 * The existence of more partial slabs makes kmem_cache_shrink
136 * sort the partial list by the number of objects in the.
137 */
138#define MAX_PARTIAL 10
139
Christoph Lameter81819f02007-05-06 14:49:36 -0700140#define DEBUG_DEFAULT_FLAGS (SLAB_DEBUG_FREE | SLAB_RED_ZONE | \
141 SLAB_POISON | SLAB_STORE_USER)
Christoph Lameter672bba32007-05-09 02:32:39 -0700142
Christoph Lameter81819f02007-05-06 14:49:36 -0700143/*
144 * Set of flags that will prevent slab merging
145 */
146#define SLUB_NEVER_MERGE (SLAB_RED_ZONE | SLAB_POISON | SLAB_STORE_USER | \
Catalin Marinas06f22f12009-06-11 13:23:18 +0100147 SLAB_TRACE | SLAB_DESTROY_BY_RCU | SLAB_NOLEAKTRACE)
Christoph Lameter81819f02007-05-06 14:49:36 -0700148
149#define SLUB_MERGE_SAME (SLAB_DEBUG_FREE | SLAB_RECLAIM_ACCOUNT | \
150 SLAB_CACHE_DMA)
151
152#ifndef ARCH_KMALLOC_MINALIGN
Christoph Lameter47bfdc02007-05-06 14:49:37 -0700153#define ARCH_KMALLOC_MINALIGN __alignof__(unsigned long long)
Christoph Lameter81819f02007-05-06 14:49:36 -0700154#endif
155
156#ifndef ARCH_SLAB_MINALIGN
Christoph Lameter47bfdc02007-05-06 14:49:37 -0700157#define ARCH_SLAB_MINALIGN __alignof__(unsigned long long)
Christoph Lameter81819f02007-05-06 14:49:36 -0700158#endif
159
Cyrill Gorcunov210b5c02008-10-22 23:00:38 +0400160#define OO_SHIFT 16
161#define OO_MASK ((1 << OO_SHIFT) - 1)
162#define MAX_OBJS_PER_PAGE 65535 /* since page.objects is u16 */
163
Christoph Lameter81819f02007-05-06 14:49:36 -0700164/* Internal SLUB flags */
Christoph Lameter1ceef402007-08-07 15:11:48 -0700165#define __OBJECT_POISON 0x80000000 /* Poison object */
166#define __SYSFS_ADD_DEFERRED 0x40000000 /* Not yet visible via sysfs */
Christoph Lameter81819f02007-05-06 14:49:36 -0700167
168static int kmem_size = sizeof(struct kmem_cache);
169
170#ifdef CONFIG_SMP
171static struct notifier_block slab_notifier;
172#endif
173
174static enum {
175 DOWN, /* No slab functionality available */
176 PARTIAL, /* kmem_cache_open() works but kmalloc does not */
Christoph Lameter672bba32007-05-09 02:32:39 -0700177 UP, /* Everything works but does not show up in sysfs */
Christoph Lameter81819f02007-05-06 14:49:36 -0700178 SYSFS /* Sysfs up */
179} slab_state = DOWN;
180
Pekka Enberg7e85ee02009-06-12 14:03:06 +0300181/*
182 * The slab allocator is initialized with interrupts disabled. Therefore, make
183 * sure early boot allocations don't accidentally enable interrupts.
184 */
185static gfp_t slab_gfp_mask __read_mostly = SLAB_GFP_BOOT_MASK;
186
Christoph Lameter81819f02007-05-06 14:49:36 -0700187/* A list of all slab caches on the system */
188static DECLARE_RWSEM(slub_lock);
Adrian Bunk5af328a2007-07-17 04:03:27 -0700189static LIST_HEAD(slab_caches);
Christoph Lameter81819f02007-05-06 14:49:36 -0700190
Christoph Lameter02cbc872007-05-09 02:32:43 -0700191/*
192 * Tracking user of a slab.
193 */
194struct track {
Eduard - Gabriel Munteanuce71e272008-08-19 20:43:25 +0300195 unsigned long addr; /* Called from address */
Christoph Lameter02cbc872007-05-09 02:32:43 -0700196 int cpu; /* Was running on cpu */
197 int pid; /* Pid context */
198 unsigned long when; /* When did the operation occur */
199};
200
201enum track_item { TRACK_ALLOC, TRACK_FREE };
202
Christoph Lameterf6acb632008-04-29 16:16:06 -0700203#ifdef CONFIG_SLUB_DEBUG
Christoph Lameter81819f02007-05-06 14:49:36 -0700204static int sysfs_slab_add(struct kmem_cache *);
205static int sysfs_slab_alias(struct kmem_cache *, const char *);
206static void sysfs_slab_remove(struct kmem_cache *);
Christoph Lameter8ff12cf2008-02-07 17:47:41 -0800207
Christoph Lameter81819f02007-05-06 14:49:36 -0700208#else
Christoph Lameter0c710012007-07-17 04:03:24 -0700209static inline int sysfs_slab_add(struct kmem_cache *s) { return 0; }
210static inline int sysfs_slab_alias(struct kmem_cache *s, const char *p)
211 { return 0; }
Christoph Lameter151c6022008-01-07 22:29:05 -0800212static inline void sysfs_slab_remove(struct kmem_cache *s)
213{
214 kfree(s);
215}
Christoph Lameter8ff12cf2008-02-07 17:47:41 -0800216
Christoph Lameter81819f02007-05-06 14:49:36 -0700217#endif
218
Christoph Lameter8ff12cf2008-02-07 17:47:41 -0800219static inline void stat(struct kmem_cache_cpu *c, enum stat_item si)
220{
221#ifdef CONFIG_SLUB_STATS
222 c->stat[si]++;
223#endif
224}
225
Christoph Lameter81819f02007-05-06 14:49:36 -0700226/********************************************************************
227 * Core slab cache functions
228 *******************************************************************/
229
230int slab_is_available(void)
231{
232 return slab_state >= UP;
233}
234
235static inline struct kmem_cache_node *get_node(struct kmem_cache *s, int node)
236{
237#ifdef CONFIG_NUMA
238 return s->node[node];
239#else
240 return &s->local_node;
241#endif
242}
243
Christoph Lameterdfb4f092007-10-16 01:26:05 -0700244static inline struct kmem_cache_cpu *get_cpu_slab(struct kmem_cache *s, int cpu)
245{
Christoph Lameter4c93c3552007-10-16 01:26:08 -0700246#ifdef CONFIG_SMP
247 return s->cpu_slab[cpu];
248#else
249 return &s->cpu_slab;
250#endif
Christoph Lameterdfb4f092007-10-16 01:26:05 -0700251}
252
Christoph Lameter6446faa2008-02-15 23:45:26 -0800253/* Verify that a pointer has an address that is valid within a slab page */
Christoph Lameter02cbc872007-05-09 02:32:43 -0700254static inline int check_valid_pointer(struct kmem_cache *s,
255 struct page *page, const void *object)
256{
257 void *base;
258
Christoph Lametera973e9d2008-03-01 13:40:44 -0800259 if (!object)
Christoph Lameter02cbc872007-05-09 02:32:43 -0700260 return 1;
261
Christoph Lametera973e9d2008-03-01 13:40:44 -0800262 base = page_address(page);
Christoph Lameter39b26462008-04-14 19:11:30 +0300263 if (object < base || object >= base + page->objects * s->size ||
Christoph Lameter02cbc872007-05-09 02:32:43 -0700264 (object - base) % s->size) {
265 return 0;
266 }
267
268 return 1;
269}
270
Christoph Lameter81819f02007-05-06 14:49:36 -0700271/*
Christoph Lameter7656c722007-05-09 02:32:40 -0700272 * Slow version of get and set free pointer.
273 *
274 * This version requires touching the cache lines of kmem_cache which
275 * we avoid to do in the fast alloc free paths. There we obtain the offset
276 * from the page struct.
277 */
278static inline void *get_freepointer(struct kmem_cache *s, void *object)
279{
280 return *(void **)(object + s->offset);
281}
282
283static inline void set_freepointer(struct kmem_cache *s, void *object, void *fp)
284{
285 *(void **)(object + s->offset) = fp;
286}
287
288/* Loop over all objects in a slab */
Christoph Lameter224a88b2008-04-14 19:11:31 +0300289#define for_each_object(__p, __s, __addr, __objects) \
290 for (__p = (__addr); __p < (__addr) + (__objects) * (__s)->size;\
Christoph Lameter7656c722007-05-09 02:32:40 -0700291 __p += (__s)->size)
292
293/* Scan freelist */
294#define for_each_free_object(__p, __s, __free) \
Christoph Lametera973e9d2008-03-01 13:40:44 -0800295 for (__p = (__free); __p; __p = get_freepointer((__s), __p))
Christoph Lameter7656c722007-05-09 02:32:40 -0700296
297/* Determine object index from a given position */
298static inline int slab_index(void *p, struct kmem_cache *s, void *addr)
299{
300 return (p - addr) / s->size;
301}
302
Christoph Lameter834f3d12008-04-14 19:11:31 +0300303static inline struct kmem_cache_order_objects oo_make(int order,
304 unsigned long size)
305{
306 struct kmem_cache_order_objects x = {
Cyrill Gorcunov210b5c02008-10-22 23:00:38 +0400307 (order << OO_SHIFT) + (PAGE_SIZE << order) / size
Christoph Lameter834f3d12008-04-14 19:11:31 +0300308 };
309
310 return x;
311}
312
313static inline int oo_order(struct kmem_cache_order_objects x)
314{
Cyrill Gorcunov210b5c02008-10-22 23:00:38 +0400315 return x.x >> OO_SHIFT;
Christoph Lameter834f3d12008-04-14 19:11:31 +0300316}
317
318static inline int oo_objects(struct kmem_cache_order_objects x)
319{
Cyrill Gorcunov210b5c02008-10-22 23:00:38 +0400320 return x.x & OO_MASK;
Christoph Lameter834f3d12008-04-14 19:11:31 +0300321}
322
Christoph Lameter41ecc552007-05-09 02:32:44 -0700323#ifdef CONFIG_SLUB_DEBUG
324/*
325 * Debug settings:
326 */
Christoph Lameterf0630ff2007-07-15 23:38:14 -0700327#ifdef CONFIG_SLUB_DEBUG_ON
328static int slub_debug = DEBUG_DEFAULT_FLAGS;
329#else
Christoph Lameter41ecc552007-05-09 02:32:44 -0700330static int slub_debug;
Christoph Lameterf0630ff2007-07-15 23:38:14 -0700331#endif
Christoph Lameter41ecc552007-05-09 02:32:44 -0700332
333static char *slub_debug_slabs;
334
Christoph Lameter7656c722007-05-09 02:32:40 -0700335/*
Christoph Lameter81819f02007-05-06 14:49:36 -0700336 * Object debugging
337 */
338static void print_section(char *text, u8 *addr, unsigned int length)
339{
340 int i, offset;
341 int newline = 1;
342 char ascii[17];
343
344 ascii[16] = 0;
345
346 for (i = 0; i < length; i++) {
347 if (newline) {
Christoph Lameter24922682007-07-17 04:03:18 -0700348 printk(KERN_ERR "%8s 0x%p: ", text, addr + i);
Christoph Lameter81819f02007-05-06 14:49:36 -0700349 newline = 0;
350 }
Pekka Enberg06428782008-01-07 23:20:27 -0800351 printk(KERN_CONT " %02x", addr[i]);
Christoph Lameter81819f02007-05-06 14:49:36 -0700352 offset = i % 16;
353 ascii[offset] = isgraph(addr[i]) ? addr[i] : '.';
354 if (offset == 15) {
Pekka Enberg06428782008-01-07 23:20:27 -0800355 printk(KERN_CONT " %s\n", ascii);
Christoph Lameter81819f02007-05-06 14:49:36 -0700356 newline = 1;
357 }
358 }
359 if (!newline) {
360 i %= 16;
361 while (i < 16) {
Pekka Enberg06428782008-01-07 23:20:27 -0800362 printk(KERN_CONT " ");
Christoph Lameter81819f02007-05-06 14:49:36 -0700363 ascii[i] = ' ';
364 i++;
365 }
Pekka Enberg06428782008-01-07 23:20:27 -0800366 printk(KERN_CONT " %s\n", ascii);
Christoph Lameter81819f02007-05-06 14:49:36 -0700367 }
368}
369
Christoph Lameter81819f02007-05-06 14:49:36 -0700370static struct track *get_track(struct kmem_cache *s, void *object,
371 enum track_item alloc)
372{
373 struct track *p;
374
375 if (s->offset)
376 p = object + s->offset + sizeof(void *);
377 else
378 p = object + s->inuse;
379
380 return p + alloc;
381}
382
383static void set_track(struct kmem_cache *s, void *object,
Eduard - Gabriel Munteanuce71e272008-08-19 20:43:25 +0300384 enum track_item alloc, unsigned long addr)
Christoph Lameter81819f02007-05-06 14:49:36 -0700385{
Akinobu Mita1a00df42009-03-07 00:36:21 +0900386 struct track *p = get_track(s, object, alloc);
Christoph Lameter81819f02007-05-06 14:49:36 -0700387
Christoph Lameter81819f02007-05-06 14:49:36 -0700388 if (addr) {
389 p->addr = addr;
390 p->cpu = smp_processor_id();
Alexey Dobriyan88e4ccf2008-06-23 02:58:37 +0400391 p->pid = current->pid;
Christoph Lameter81819f02007-05-06 14:49:36 -0700392 p->when = jiffies;
393 } else
394 memset(p, 0, sizeof(struct track));
395}
396
Christoph Lameter81819f02007-05-06 14:49:36 -0700397static void init_tracking(struct kmem_cache *s, void *object)
398{
Christoph Lameter24922682007-07-17 04:03:18 -0700399 if (!(s->flags & SLAB_STORE_USER))
400 return;
401
Eduard - Gabriel Munteanuce71e272008-08-19 20:43:25 +0300402 set_track(s, object, TRACK_FREE, 0UL);
403 set_track(s, object, TRACK_ALLOC, 0UL);
Christoph Lameter81819f02007-05-06 14:49:36 -0700404}
405
406static void print_track(const char *s, struct track *t)
407{
408 if (!t->addr)
409 return;
410
Linus Torvalds7daf7052008-07-14 12:12:53 -0700411 printk(KERN_ERR "INFO: %s in %pS age=%lu cpu=%u pid=%d\n",
Eduard - Gabriel Munteanuce71e272008-08-19 20:43:25 +0300412 s, (void *)t->addr, jiffies - t->when, t->cpu, t->pid);
Christoph Lameter81819f02007-05-06 14:49:36 -0700413}
414
Christoph Lameter24922682007-07-17 04:03:18 -0700415static void print_tracking(struct kmem_cache *s, void *object)
416{
417 if (!(s->flags & SLAB_STORE_USER))
418 return;
419
420 print_track("Allocated", get_track(s, object, TRACK_ALLOC));
421 print_track("Freed", get_track(s, object, TRACK_FREE));
422}
423
424static void print_page_info(struct page *page)
425{
Christoph Lameter39b26462008-04-14 19:11:30 +0300426 printk(KERN_ERR "INFO: Slab 0x%p objects=%u used=%u fp=0x%p flags=0x%04lx\n",
427 page, page->objects, page->inuse, page->freelist, page->flags);
Christoph Lameter24922682007-07-17 04:03:18 -0700428
429}
430
431static void slab_bug(struct kmem_cache *s, char *fmt, ...)
432{
433 va_list args;
434 char buf[100];
435
436 va_start(args, fmt);
437 vsnprintf(buf, sizeof(buf), fmt, args);
438 va_end(args);
439 printk(KERN_ERR "========================================"
440 "=====================================\n");
441 printk(KERN_ERR "BUG %s: %s\n", s->name, buf);
442 printk(KERN_ERR "----------------------------------------"
443 "-------------------------------------\n\n");
444}
445
446static void slab_fix(struct kmem_cache *s, char *fmt, ...)
447{
448 va_list args;
449 char buf[100];
450
451 va_start(args, fmt);
452 vsnprintf(buf, sizeof(buf), fmt, args);
453 va_end(args);
454 printk(KERN_ERR "FIX %s: %s\n", s->name, buf);
455}
456
457static void print_trailer(struct kmem_cache *s, struct page *page, u8 *p)
Christoph Lameter81819f02007-05-06 14:49:36 -0700458{
459 unsigned int off; /* Offset of last byte */
Christoph Lametera973e9d2008-03-01 13:40:44 -0800460 u8 *addr = page_address(page);
Christoph Lameter24922682007-07-17 04:03:18 -0700461
462 print_tracking(s, p);
463
464 print_page_info(page);
465
466 printk(KERN_ERR "INFO: Object 0x%p @offset=%tu fp=0x%p\n\n",
467 p, p - addr, get_freepointer(s, p));
468
469 if (p > addr + 16)
470 print_section("Bytes b4", p - 16, 16);
471
Pekka Enberg0ebd6522008-07-19 14:17:22 +0300472 print_section("Object", p, min_t(unsigned long, s->objsize, PAGE_SIZE));
Christoph Lameter81819f02007-05-06 14:49:36 -0700473
474 if (s->flags & SLAB_RED_ZONE)
475 print_section("Redzone", p + s->objsize,
476 s->inuse - s->objsize);
477
Christoph Lameter81819f02007-05-06 14:49:36 -0700478 if (s->offset)
479 off = s->offset + sizeof(void *);
480 else
481 off = s->inuse;
482
Christoph Lameter24922682007-07-17 04:03:18 -0700483 if (s->flags & SLAB_STORE_USER)
Christoph Lameter81819f02007-05-06 14:49:36 -0700484 off += 2 * sizeof(struct track);
Christoph Lameter81819f02007-05-06 14:49:36 -0700485
486 if (off != s->size)
487 /* Beginning of the filler is the free pointer */
Christoph Lameter24922682007-07-17 04:03:18 -0700488 print_section("Padding", p + off, s->size - off);
489
490 dump_stack();
Christoph Lameter81819f02007-05-06 14:49:36 -0700491}
492
493static void object_err(struct kmem_cache *s, struct page *page,
494 u8 *object, char *reason)
495{
Christoph Lameter3dc50632008-04-23 12:28:01 -0700496 slab_bug(s, "%s", reason);
Christoph Lameter24922682007-07-17 04:03:18 -0700497 print_trailer(s, page, object);
Christoph Lameter81819f02007-05-06 14:49:36 -0700498}
499
Christoph Lameter24922682007-07-17 04:03:18 -0700500static void slab_err(struct kmem_cache *s, struct page *page, char *fmt, ...)
Christoph Lameter81819f02007-05-06 14:49:36 -0700501{
502 va_list args;
503 char buf[100];
504
Christoph Lameter24922682007-07-17 04:03:18 -0700505 va_start(args, fmt);
506 vsnprintf(buf, sizeof(buf), fmt, args);
Christoph Lameter81819f02007-05-06 14:49:36 -0700507 va_end(args);
Christoph Lameter3dc50632008-04-23 12:28:01 -0700508 slab_bug(s, "%s", buf);
Christoph Lameter24922682007-07-17 04:03:18 -0700509 print_page_info(page);
Christoph Lameter81819f02007-05-06 14:49:36 -0700510 dump_stack();
511}
512
513static void init_object(struct kmem_cache *s, void *object, int active)
514{
515 u8 *p = object;
516
517 if (s->flags & __OBJECT_POISON) {
518 memset(p, POISON_FREE, s->objsize - 1);
Pekka Enberg06428782008-01-07 23:20:27 -0800519 p[s->objsize - 1] = POISON_END;
Christoph Lameter81819f02007-05-06 14:49:36 -0700520 }
521
522 if (s->flags & SLAB_RED_ZONE)
523 memset(p + s->objsize,
524 active ? SLUB_RED_ACTIVE : SLUB_RED_INACTIVE,
525 s->inuse - s->objsize);
526}
527
Christoph Lameter24922682007-07-17 04:03:18 -0700528static u8 *check_bytes(u8 *start, unsigned int value, unsigned int bytes)
Christoph Lameter81819f02007-05-06 14:49:36 -0700529{
530 while (bytes) {
531 if (*start != (u8)value)
Christoph Lameter24922682007-07-17 04:03:18 -0700532 return start;
Christoph Lameter81819f02007-05-06 14:49:36 -0700533 start++;
534 bytes--;
535 }
Christoph Lameter24922682007-07-17 04:03:18 -0700536 return NULL;
537}
538
539static void restore_bytes(struct kmem_cache *s, char *message, u8 data,
540 void *from, void *to)
541{
542 slab_fix(s, "Restoring 0x%p-0x%p=0x%x\n", from, to - 1, data);
543 memset(from, data, to - from);
544}
545
546static int check_bytes_and_report(struct kmem_cache *s, struct page *page,
547 u8 *object, char *what,
Pekka Enberg06428782008-01-07 23:20:27 -0800548 u8 *start, unsigned int value, unsigned int bytes)
Christoph Lameter24922682007-07-17 04:03:18 -0700549{
550 u8 *fault;
551 u8 *end;
552
553 fault = check_bytes(start, value, bytes);
554 if (!fault)
555 return 1;
556
557 end = start + bytes;
558 while (end > fault && end[-1] == value)
559 end--;
560
561 slab_bug(s, "%s overwritten", what);
562 printk(KERN_ERR "INFO: 0x%p-0x%p. First byte 0x%x instead of 0x%x\n",
563 fault, end - 1, fault[0], value);
564 print_trailer(s, page, object);
565
566 restore_bytes(s, what, value, fault, end);
567 return 0;
Christoph Lameter81819f02007-05-06 14:49:36 -0700568}
569
Christoph Lameter81819f02007-05-06 14:49:36 -0700570/*
571 * Object layout:
572 *
573 * object address
574 * Bytes of the object to be managed.
575 * If the freepointer may overlay the object then the free
576 * pointer is the first word of the object.
Christoph Lameter672bba32007-05-09 02:32:39 -0700577 *
Christoph Lameter81819f02007-05-06 14:49:36 -0700578 * Poisoning uses 0x6b (POISON_FREE) and the last byte is
579 * 0xa5 (POISON_END)
580 *
581 * object + s->objsize
582 * Padding to reach word boundary. This is also used for Redzoning.
Christoph Lameter672bba32007-05-09 02:32:39 -0700583 * Padding is extended by another word if Redzoning is enabled and
584 * objsize == inuse.
585 *
Christoph Lameter81819f02007-05-06 14:49:36 -0700586 * We fill with 0xbb (RED_INACTIVE) for inactive objects and with
587 * 0xcc (RED_ACTIVE) for objects in use.
588 *
589 * object + s->inuse
Christoph Lameter672bba32007-05-09 02:32:39 -0700590 * Meta data starts here.
591 *
Christoph Lameter81819f02007-05-06 14:49:36 -0700592 * A. Free pointer (if we cannot overwrite object on free)
593 * B. Tracking data for SLAB_STORE_USER
Christoph Lameter672bba32007-05-09 02:32:39 -0700594 * C. Padding to reach required alignment boundary or at mininum
Christoph Lameter6446faa2008-02-15 23:45:26 -0800595 * one word if debugging is on to be able to detect writes
Christoph Lameter672bba32007-05-09 02:32:39 -0700596 * before the word boundary.
597 *
598 * Padding is done using 0x5a (POISON_INUSE)
Christoph Lameter81819f02007-05-06 14:49:36 -0700599 *
600 * object + s->size
Christoph Lameter672bba32007-05-09 02:32:39 -0700601 * Nothing is used beyond s->size.
Christoph Lameter81819f02007-05-06 14:49:36 -0700602 *
Christoph Lameter672bba32007-05-09 02:32:39 -0700603 * If slabcaches are merged then the objsize and inuse boundaries are mostly
604 * ignored. And therefore no slab options that rely on these boundaries
Christoph Lameter81819f02007-05-06 14:49:36 -0700605 * may be used with merged slabcaches.
606 */
607
Christoph Lameter81819f02007-05-06 14:49:36 -0700608static int check_pad_bytes(struct kmem_cache *s, struct page *page, u8 *p)
609{
610 unsigned long off = s->inuse; /* The end of info */
611
612 if (s->offset)
613 /* Freepointer is placed after the object. */
614 off += sizeof(void *);
615
616 if (s->flags & SLAB_STORE_USER)
617 /* We also have user information there */
618 off += 2 * sizeof(struct track);
619
620 if (s->size == off)
621 return 1;
622
Christoph Lameter24922682007-07-17 04:03:18 -0700623 return check_bytes_and_report(s, page, p, "Object padding",
624 p + off, POISON_INUSE, s->size - off);
Christoph Lameter81819f02007-05-06 14:49:36 -0700625}
626
Christoph Lameter39b26462008-04-14 19:11:30 +0300627/* Check the pad bytes at the end of a slab page */
Christoph Lameter81819f02007-05-06 14:49:36 -0700628static int slab_pad_check(struct kmem_cache *s, struct page *page)
629{
Christoph Lameter24922682007-07-17 04:03:18 -0700630 u8 *start;
631 u8 *fault;
632 u8 *end;
633 int length;
634 int remainder;
Christoph Lameter81819f02007-05-06 14:49:36 -0700635
636 if (!(s->flags & SLAB_POISON))
637 return 1;
638
Christoph Lametera973e9d2008-03-01 13:40:44 -0800639 start = page_address(page);
Christoph Lameter834f3d12008-04-14 19:11:31 +0300640 length = (PAGE_SIZE << compound_order(page));
Christoph Lameter39b26462008-04-14 19:11:30 +0300641 end = start + length;
642 remainder = length % s->size;
Christoph Lameter81819f02007-05-06 14:49:36 -0700643 if (!remainder)
644 return 1;
645
Christoph Lameter39b26462008-04-14 19:11:30 +0300646 fault = check_bytes(end - remainder, POISON_INUSE, remainder);
Christoph Lameter24922682007-07-17 04:03:18 -0700647 if (!fault)
648 return 1;
649 while (end > fault && end[-1] == POISON_INUSE)
650 end--;
651
652 slab_err(s, page, "Padding overwritten. 0x%p-0x%p", fault, end - 1);
Christoph Lameter39b26462008-04-14 19:11:30 +0300653 print_section("Padding", end - remainder, remainder);
Christoph Lameter24922682007-07-17 04:03:18 -0700654
655 restore_bytes(s, "slab padding", POISON_INUSE, start, end);
656 return 0;
Christoph Lameter81819f02007-05-06 14:49:36 -0700657}
658
659static int check_object(struct kmem_cache *s, struct page *page,
660 void *object, int active)
661{
662 u8 *p = object;
663 u8 *endobject = object + s->objsize;
664
665 if (s->flags & SLAB_RED_ZONE) {
666 unsigned int red =
667 active ? SLUB_RED_ACTIVE : SLUB_RED_INACTIVE;
668
Christoph Lameter24922682007-07-17 04:03:18 -0700669 if (!check_bytes_and_report(s, page, object, "Redzone",
670 endobject, red, s->inuse - s->objsize))
Christoph Lameter81819f02007-05-06 14:49:36 -0700671 return 0;
Christoph Lameter81819f02007-05-06 14:49:36 -0700672 } else {
Ingo Molnar3adbefe2008-02-05 17:57:39 -0800673 if ((s->flags & SLAB_POISON) && s->objsize < s->inuse) {
674 check_bytes_and_report(s, page, p, "Alignment padding",
675 endobject, POISON_INUSE, s->inuse - s->objsize);
676 }
Christoph Lameter81819f02007-05-06 14:49:36 -0700677 }
678
679 if (s->flags & SLAB_POISON) {
680 if (!active && (s->flags & __OBJECT_POISON) &&
Christoph Lameter24922682007-07-17 04:03:18 -0700681 (!check_bytes_and_report(s, page, p, "Poison", p,
682 POISON_FREE, s->objsize - 1) ||
683 !check_bytes_and_report(s, page, p, "Poison",
Pekka Enberg06428782008-01-07 23:20:27 -0800684 p + s->objsize - 1, POISON_END, 1)))
Christoph Lameter81819f02007-05-06 14:49:36 -0700685 return 0;
Christoph Lameter81819f02007-05-06 14:49:36 -0700686 /*
687 * check_pad_bytes cleans up on its own.
688 */
689 check_pad_bytes(s, page, p);
690 }
691
692 if (!s->offset && active)
693 /*
694 * Object and freepointer overlap. Cannot check
695 * freepointer while object is allocated.
696 */
697 return 1;
698
699 /* Check free pointer validity */
700 if (!check_valid_pointer(s, page, get_freepointer(s, p))) {
701 object_err(s, page, p, "Freepointer corrupt");
702 /*
Nick Andrew9f6c708e2008-12-05 14:08:08 +1100703 * No choice but to zap it and thus lose the remainder
Christoph Lameter81819f02007-05-06 14:49:36 -0700704 * of the free objects in this slab. May cause
Christoph Lameter672bba32007-05-09 02:32:39 -0700705 * another error because the object count is now wrong.
Christoph Lameter81819f02007-05-06 14:49:36 -0700706 */
Christoph Lametera973e9d2008-03-01 13:40:44 -0800707 set_freepointer(s, p, NULL);
Christoph Lameter81819f02007-05-06 14:49:36 -0700708 return 0;
709 }
710 return 1;
711}
712
713static int check_slab(struct kmem_cache *s, struct page *page)
714{
Christoph Lameter39b26462008-04-14 19:11:30 +0300715 int maxobj;
716
Christoph Lameter81819f02007-05-06 14:49:36 -0700717 VM_BUG_ON(!irqs_disabled());
718
719 if (!PageSlab(page)) {
Christoph Lameter24922682007-07-17 04:03:18 -0700720 slab_err(s, page, "Not a valid slab page");
Christoph Lameter81819f02007-05-06 14:49:36 -0700721 return 0;
722 }
Christoph Lameter39b26462008-04-14 19:11:30 +0300723
724 maxobj = (PAGE_SIZE << compound_order(page)) / s->size;
725 if (page->objects > maxobj) {
726 slab_err(s, page, "objects %u > max %u",
727 s->name, page->objects, maxobj);
728 return 0;
729 }
730 if (page->inuse > page->objects) {
Christoph Lameter24922682007-07-17 04:03:18 -0700731 slab_err(s, page, "inuse %u > max %u",
Christoph Lameter39b26462008-04-14 19:11:30 +0300732 s->name, page->inuse, page->objects);
Christoph Lameter81819f02007-05-06 14:49:36 -0700733 return 0;
734 }
735 /* Slab_pad_check fixes things up after itself */
736 slab_pad_check(s, page);
737 return 1;
738}
739
740/*
Christoph Lameter672bba32007-05-09 02:32:39 -0700741 * Determine if a certain object on a page is on the freelist. Must hold the
742 * slab lock to guarantee that the chains are in a consistent state.
Christoph Lameter81819f02007-05-06 14:49:36 -0700743 */
744static int on_freelist(struct kmem_cache *s, struct page *page, void *search)
745{
746 int nr = 0;
747 void *fp = page->freelist;
748 void *object = NULL;
Christoph Lameter224a88b2008-04-14 19:11:31 +0300749 unsigned long max_objects;
Christoph Lameter81819f02007-05-06 14:49:36 -0700750
Christoph Lameter39b26462008-04-14 19:11:30 +0300751 while (fp && nr <= page->objects) {
Christoph Lameter81819f02007-05-06 14:49:36 -0700752 if (fp == search)
753 return 1;
754 if (!check_valid_pointer(s, page, fp)) {
755 if (object) {
756 object_err(s, page, object,
757 "Freechain corrupt");
Christoph Lametera973e9d2008-03-01 13:40:44 -0800758 set_freepointer(s, object, NULL);
Christoph Lameter81819f02007-05-06 14:49:36 -0700759 break;
760 } else {
Christoph Lameter24922682007-07-17 04:03:18 -0700761 slab_err(s, page, "Freepointer corrupt");
Christoph Lametera973e9d2008-03-01 13:40:44 -0800762 page->freelist = NULL;
Christoph Lameter39b26462008-04-14 19:11:30 +0300763 page->inuse = page->objects;
Christoph Lameter24922682007-07-17 04:03:18 -0700764 slab_fix(s, "Freelist cleared");
Christoph Lameter81819f02007-05-06 14:49:36 -0700765 return 0;
766 }
767 break;
768 }
769 object = fp;
770 fp = get_freepointer(s, object);
771 nr++;
772 }
773
Christoph Lameter224a88b2008-04-14 19:11:31 +0300774 max_objects = (PAGE_SIZE << compound_order(page)) / s->size;
Cyrill Gorcunov210b5c02008-10-22 23:00:38 +0400775 if (max_objects > MAX_OBJS_PER_PAGE)
776 max_objects = MAX_OBJS_PER_PAGE;
Christoph Lameter224a88b2008-04-14 19:11:31 +0300777
778 if (page->objects != max_objects) {
779 slab_err(s, page, "Wrong number of objects. Found %d but "
780 "should be %d", page->objects, max_objects);
781 page->objects = max_objects;
782 slab_fix(s, "Number of objects adjusted.");
783 }
Christoph Lameter39b26462008-04-14 19:11:30 +0300784 if (page->inuse != page->objects - nr) {
Christoph Lameter70d71222007-05-06 14:49:47 -0700785 slab_err(s, page, "Wrong object count. Counter is %d but "
Christoph Lameter39b26462008-04-14 19:11:30 +0300786 "counted were %d", page->inuse, page->objects - nr);
787 page->inuse = page->objects - nr;
Christoph Lameter24922682007-07-17 04:03:18 -0700788 slab_fix(s, "Object count adjusted.");
Christoph Lameter81819f02007-05-06 14:49:36 -0700789 }
790 return search == NULL;
791}
792
Christoph Lameter0121c6192008-04-29 16:11:12 -0700793static void trace(struct kmem_cache *s, struct page *page, void *object,
794 int alloc)
Christoph Lameter3ec09742007-05-16 22:11:00 -0700795{
796 if (s->flags & SLAB_TRACE) {
797 printk(KERN_INFO "TRACE %s %s 0x%p inuse=%d fp=0x%p\n",
798 s->name,
799 alloc ? "alloc" : "free",
800 object, page->inuse,
801 page->freelist);
802
803 if (!alloc)
804 print_section("Object", (void *)object, s->objsize);
805
806 dump_stack();
807 }
808}
809
Christoph Lameter643b1132007-05-06 14:49:42 -0700810/*
Christoph Lameter672bba32007-05-09 02:32:39 -0700811 * Tracking of fully allocated slabs for debugging purposes.
Christoph Lameter643b1132007-05-06 14:49:42 -0700812 */
Christoph Lametere95eed52007-05-06 14:49:44 -0700813static void add_full(struct kmem_cache_node *n, struct page *page)
Christoph Lameter643b1132007-05-06 14:49:42 -0700814{
Christoph Lameter643b1132007-05-06 14:49:42 -0700815 spin_lock(&n->list_lock);
816 list_add(&page->lru, &n->full);
817 spin_unlock(&n->list_lock);
818}
819
820static void remove_full(struct kmem_cache *s, struct page *page)
821{
822 struct kmem_cache_node *n;
823
824 if (!(s->flags & SLAB_STORE_USER))
825 return;
826
827 n = get_node(s, page_to_nid(page));
828
829 spin_lock(&n->list_lock);
830 list_del(&page->lru);
831 spin_unlock(&n->list_lock);
832}
833
Christoph Lameter0f389ec2008-04-14 18:53:02 +0300834/* Tracking of the number of slabs for debugging purposes */
835static inline unsigned long slabs_node(struct kmem_cache *s, int node)
836{
837 struct kmem_cache_node *n = get_node(s, node);
838
839 return atomic_long_read(&n->nr_slabs);
840}
841
Christoph Lameter205ab992008-04-14 19:11:40 +0300842static inline void inc_slabs_node(struct kmem_cache *s, int node, int objects)
Christoph Lameter0f389ec2008-04-14 18:53:02 +0300843{
844 struct kmem_cache_node *n = get_node(s, node);
845
846 /*
847 * May be called early in order to allocate a slab for the
848 * kmem_cache_node structure. Solve the chicken-egg
849 * dilemma by deferring the increment of the count during
850 * bootstrap (see early_kmem_cache_node_alloc).
851 */
Christoph Lameter205ab992008-04-14 19:11:40 +0300852 if (!NUMA_BUILD || n) {
Christoph Lameter0f389ec2008-04-14 18:53:02 +0300853 atomic_long_inc(&n->nr_slabs);
Christoph Lameter205ab992008-04-14 19:11:40 +0300854 atomic_long_add(objects, &n->total_objects);
855 }
Christoph Lameter0f389ec2008-04-14 18:53:02 +0300856}
Christoph Lameter205ab992008-04-14 19:11:40 +0300857static inline void dec_slabs_node(struct kmem_cache *s, int node, int objects)
Christoph Lameter0f389ec2008-04-14 18:53:02 +0300858{
859 struct kmem_cache_node *n = get_node(s, node);
860
861 atomic_long_dec(&n->nr_slabs);
Christoph Lameter205ab992008-04-14 19:11:40 +0300862 atomic_long_sub(objects, &n->total_objects);
Christoph Lameter0f389ec2008-04-14 18:53:02 +0300863}
864
865/* Object debug checks for alloc/free paths */
Christoph Lameter3ec09742007-05-16 22:11:00 -0700866static void setup_object_debug(struct kmem_cache *s, struct page *page,
867 void *object)
868{
869 if (!(s->flags & (SLAB_STORE_USER|SLAB_RED_ZONE|__OBJECT_POISON)))
870 return;
871
872 init_object(s, object, 0);
873 init_tracking(s, object);
874}
875
876static int alloc_debug_processing(struct kmem_cache *s, struct page *page,
Eduard - Gabriel Munteanuce71e272008-08-19 20:43:25 +0300877 void *object, unsigned long addr)
Christoph Lameter81819f02007-05-06 14:49:36 -0700878{
879 if (!check_slab(s, page))
880 goto bad;
881
Christoph Lameterd692ef62008-02-15 23:45:24 -0800882 if (!on_freelist(s, page, object)) {
Christoph Lameter24922682007-07-17 04:03:18 -0700883 object_err(s, page, object, "Object already allocated");
Christoph Lameter70d71222007-05-06 14:49:47 -0700884 goto bad;
Christoph Lameter81819f02007-05-06 14:49:36 -0700885 }
886
887 if (!check_valid_pointer(s, page, object)) {
888 object_err(s, page, object, "Freelist Pointer check fails");
Christoph Lameter70d71222007-05-06 14:49:47 -0700889 goto bad;
Christoph Lameter81819f02007-05-06 14:49:36 -0700890 }
891
Christoph Lameterd692ef62008-02-15 23:45:24 -0800892 if (!check_object(s, page, object, 0))
Christoph Lameter81819f02007-05-06 14:49:36 -0700893 goto bad;
Christoph Lameter81819f02007-05-06 14:49:36 -0700894
Christoph Lameter3ec09742007-05-16 22:11:00 -0700895 /* Success perform special debug activities for allocs */
896 if (s->flags & SLAB_STORE_USER)
897 set_track(s, object, TRACK_ALLOC, addr);
898 trace(s, page, object, 1);
899 init_object(s, object, 1);
Christoph Lameter81819f02007-05-06 14:49:36 -0700900 return 1;
Christoph Lameter3ec09742007-05-16 22:11:00 -0700901
Christoph Lameter81819f02007-05-06 14:49:36 -0700902bad:
903 if (PageSlab(page)) {
904 /*
905 * If this is a slab page then lets do the best we can
906 * to avoid issues in the future. Marking all objects
Christoph Lameter672bba32007-05-09 02:32:39 -0700907 * as used avoids touching the remaining objects.
Christoph Lameter81819f02007-05-06 14:49:36 -0700908 */
Christoph Lameter24922682007-07-17 04:03:18 -0700909 slab_fix(s, "Marking all objects used");
Christoph Lameter39b26462008-04-14 19:11:30 +0300910 page->inuse = page->objects;
Christoph Lametera973e9d2008-03-01 13:40:44 -0800911 page->freelist = NULL;
Christoph Lameter81819f02007-05-06 14:49:36 -0700912 }
913 return 0;
914}
915
Christoph Lameter3ec09742007-05-16 22:11:00 -0700916static int free_debug_processing(struct kmem_cache *s, struct page *page,
Eduard - Gabriel Munteanuce71e272008-08-19 20:43:25 +0300917 void *object, unsigned long addr)
Christoph Lameter81819f02007-05-06 14:49:36 -0700918{
919 if (!check_slab(s, page))
920 goto fail;
921
922 if (!check_valid_pointer(s, page, object)) {
Christoph Lameter70d71222007-05-06 14:49:47 -0700923 slab_err(s, page, "Invalid object pointer 0x%p", object);
Christoph Lameter81819f02007-05-06 14:49:36 -0700924 goto fail;
925 }
926
927 if (on_freelist(s, page, object)) {
Christoph Lameter24922682007-07-17 04:03:18 -0700928 object_err(s, page, object, "Object already free");
Christoph Lameter81819f02007-05-06 14:49:36 -0700929 goto fail;
930 }
931
932 if (!check_object(s, page, object, 1))
933 return 0;
934
935 if (unlikely(s != page->slab)) {
Ingo Molnar3adbefe2008-02-05 17:57:39 -0800936 if (!PageSlab(page)) {
Christoph Lameter70d71222007-05-06 14:49:47 -0700937 slab_err(s, page, "Attempt to free object(0x%p) "
938 "outside of slab", object);
Ingo Molnar3adbefe2008-02-05 17:57:39 -0800939 } else if (!page->slab) {
Christoph Lameter81819f02007-05-06 14:49:36 -0700940 printk(KERN_ERR
Christoph Lameter70d71222007-05-06 14:49:47 -0700941 "SLUB <none>: no slab for object 0x%p.\n",
Christoph Lameter81819f02007-05-06 14:49:36 -0700942 object);
Christoph Lameter70d71222007-05-06 14:49:47 -0700943 dump_stack();
Pekka Enberg06428782008-01-07 23:20:27 -0800944 } else
Christoph Lameter24922682007-07-17 04:03:18 -0700945 object_err(s, page, object,
946 "page slab pointer corrupt.");
Christoph Lameter81819f02007-05-06 14:49:36 -0700947 goto fail;
948 }
Christoph Lameter3ec09742007-05-16 22:11:00 -0700949
950 /* Special debug activities for freeing objects */
Andy Whitcroft8a380822008-07-23 21:27:18 -0700951 if (!PageSlubFrozen(page) && !page->freelist)
Christoph Lameter3ec09742007-05-16 22:11:00 -0700952 remove_full(s, page);
953 if (s->flags & SLAB_STORE_USER)
954 set_track(s, object, TRACK_FREE, addr);
955 trace(s, page, object, 0);
956 init_object(s, object, 0);
Christoph Lameter81819f02007-05-06 14:49:36 -0700957 return 1;
Christoph Lameter3ec09742007-05-16 22:11:00 -0700958
Christoph Lameter81819f02007-05-06 14:49:36 -0700959fail:
Christoph Lameter24922682007-07-17 04:03:18 -0700960 slab_fix(s, "Object at 0x%p not freed", object);
Christoph Lameter81819f02007-05-06 14:49:36 -0700961 return 0;
962}
963
Christoph Lameter41ecc552007-05-09 02:32:44 -0700964static int __init setup_slub_debug(char *str)
965{
Christoph Lameterf0630ff2007-07-15 23:38:14 -0700966 slub_debug = DEBUG_DEFAULT_FLAGS;
967 if (*str++ != '=' || !*str)
968 /*
969 * No options specified. Switch on full debugging.
970 */
971 goto out;
Christoph Lameter41ecc552007-05-09 02:32:44 -0700972
973 if (*str == ',')
Christoph Lameterf0630ff2007-07-15 23:38:14 -0700974 /*
975 * No options but restriction on slabs. This means full
976 * debugging for slabs matching a pattern.
977 */
978 goto check_slabs;
979
980 slub_debug = 0;
981 if (*str == '-')
982 /*
983 * Switch off all debugging measures.
984 */
985 goto out;
986
987 /*
988 * Determine which debug features should be switched on
989 */
Pekka Enberg06428782008-01-07 23:20:27 -0800990 for (; *str && *str != ','; str++) {
Christoph Lameterf0630ff2007-07-15 23:38:14 -0700991 switch (tolower(*str)) {
992 case 'f':
993 slub_debug |= SLAB_DEBUG_FREE;
994 break;
995 case 'z':
996 slub_debug |= SLAB_RED_ZONE;
997 break;
998 case 'p':
999 slub_debug |= SLAB_POISON;
1000 break;
1001 case 'u':
1002 slub_debug |= SLAB_STORE_USER;
1003 break;
1004 case 't':
1005 slub_debug |= SLAB_TRACE;
1006 break;
1007 default:
1008 printk(KERN_ERR "slub_debug option '%c' "
Pekka Enberg06428782008-01-07 23:20:27 -08001009 "unknown. skipped\n", *str);
Christoph Lameterf0630ff2007-07-15 23:38:14 -07001010 }
1011 }
1012
1013check_slabs:
1014 if (*str == ',')
Christoph Lameter41ecc552007-05-09 02:32:44 -07001015 slub_debug_slabs = str + 1;
Christoph Lameterf0630ff2007-07-15 23:38:14 -07001016out:
Christoph Lameter41ecc552007-05-09 02:32:44 -07001017 return 1;
1018}
1019
1020__setup("slub_debug", setup_slub_debug);
1021
Christoph Lameterba0268a2007-09-11 15:24:11 -07001022static unsigned long kmem_cache_flags(unsigned long objsize,
1023 unsigned long flags, const char *name,
Alexey Dobriyan51cc5062008-07-25 19:45:34 -07001024 void (*ctor)(void *))
Christoph Lameter41ecc552007-05-09 02:32:44 -07001025{
1026 /*
Christoph Lametere1533622008-02-15 23:45:24 -08001027 * Enable debugging if selected on the kernel commandline.
Christoph Lameter41ecc552007-05-09 02:32:44 -07001028 */
Christoph Lametere1533622008-02-15 23:45:24 -08001029 if (slub_debug && (!slub_debug_slabs ||
1030 strncmp(slub_debug_slabs, name, strlen(slub_debug_slabs)) == 0))
1031 flags |= slub_debug;
Christoph Lameterba0268a2007-09-11 15:24:11 -07001032
1033 return flags;
Christoph Lameter41ecc552007-05-09 02:32:44 -07001034}
1035#else
Christoph Lameter3ec09742007-05-16 22:11:00 -07001036static inline void setup_object_debug(struct kmem_cache *s,
1037 struct page *page, void *object) {}
Christoph Lameter41ecc552007-05-09 02:32:44 -07001038
Christoph Lameter3ec09742007-05-16 22:11:00 -07001039static inline int alloc_debug_processing(struct kmem_cache *s,
Eduard - Gabriel Munteanuce71e272008-08-19 20:43:25 +03001040 struct page *page, void *object, unsigned long addr) { return 0; }
Christoph Lameter41ecc552007-05-09 02:32:44 -07001041
Christoph Lameter3ec09742007-05-16 22:11:00 -07001042static inline int free_debug_processing(struct kmem_cache *s,
Eduard - Gabriel Munteanuce71e272008-08-19 20:43:25 +03001043 struct page *page, void *object, unsigned long addr) { return 0; }
Christoph Lameter41ecc552007-05-09 02:32:44 -07001044
Christoph Lameter41ecc552007-05-09 02:32:44 -07001045static inline int slab_pad_check(struct kmem_cache *s, struct page *page)
1046 { return 1; }
1047static inline int check_object(struct kmem_cache *s, struct page *page,
1048 void *object, int active) { return 1; }
Christoph Lameter3ec09742007-05-16 22:11:00 -07001049static inline void add_full(struct kmem_cache_node *n, struct page *page) {}
Christoph Lameterba0268a2007-09-11 15:24:11 -07001050static inline unsigned long kmem_cache_flags(unsigned long objsize,
1051 unsigned long flags, const char *name,
Alexey Dobriyan51cc5062008-07-25 19:45:34 -07001052 void (*ctor)(void *))
Christoph Lameterba0268a2007-09-11 15:24:11 -07001053{
1054 return flags;
1055}
Christoph Lameter41ecc552007-05-09 02:32:44 -07001056#define slub_debug 0
Christoph Lameter0f389ec2008-04-14 18:53:02 +03001057
1058static inline unsigned long slabs_node(struct kmem_cache *s, int node)
1059 { return 0; }
Christoph Lameter205ab992008-04-14 19:11:40 +03001060static inline void inc_slabs_node(struct kmem_cache *s, int node,
1061 int objects) {}
1062static inline void dec_slabs_node(struct kmem_cache *s, int node,
1063 int objects) {}
Christoph Lameter41ecc552007-05-09 02:32:44 -07001064#endif
Christoph Lameter205ab992008-04-14 19:11:40 +03001065
Christoph Lameter81819f02007-05-06 14:49:36 -07001066/*
1067 * Slab allocation and freeing
1068 */
Christoph Lameter65c33762008-04-14 19:11:40 +03001069static inline struct page *alloc_slab_page(gfp_t flags, int node,
1070 struct kmem_cache_order_objects oo)
1071{
1072 int order = oo_order(oo);
1073
1074 if (node == -1)
1075 return alloc_pages(flags, order);
1076 else
1077 return alloc_pages_node(node, flags, order);
1078}
1079
Christoph Lameter81819f02007-05-06 14:49:36 -07001080static struct page *allocate_slab(struct kmem_cache *s, gfp_t flags, int node)
1081{
Pekka Enberg06428782008-01-07 23:20:27 -08001082 struct page *page;
Christoph Lameter834f3d12008-04-14 19:11:31 +03001083 struct kmem_cache_order_objects oo = s->oo;
Christoph Lameter81819f02007-05-06 14:49:36 -07001084
Christoph Lameterb7a49f02008-02-14 14:21:32 -08001085 flags |= s->allocflags;
Mel Gormane12ba742007-10-16 01:25:52 -07001086
Christoph Lameter65c33762008-04-14 19:11:40 +03001087 page = alloc_slab_page(flags | __GFP_NOWARN | __GFP_NORETRY, node,
1088 oo);
1089 if (unlikely(!page)) {
1090 oo = s->min;
1091 /*
1092 * Allocation may have failed due to fragmentation.
1093 * Try a lower order alloc if possible
1094 */
1095 page = alloc_slab_page(flags, node, oo);
1096 if (!page)
1097 return NULL;
Christoph Lameter81819f02007-05-06 14:49:36 -07001098
Christoph Lameter65c33762008-04-14 19:11:40 +03001099 stat(get_cpu_slab(s, raw_smp_processor_id()), ORDER_FALLBACK);
1100 }
Christoph Lameter834f3d12008-04-14 19:11:31 +03001101 page->objects = oo_objects(oo);
Christoph Lameter81819f02007-05-06 14:49:36 -07001102 mod_zone_page_state(page_zone(page),
1103 (s->flags & SLAB_RECLAIM_ACCOUNT) ?
1104 NR_SLAB_RECLAIMABLE : NR_SLAB_UNRECLAIMABLE,
Christoph Lameter65c33762008-04-14 19:11:40 +03001105 1 << oo_order(oo));
Christoph Lameter81819f02007-05-06 14:49:36 -07001106
1107 return page;
1108}
1109
1110static void setup_object(struct kmem_cache *s, struct page *page,
1111 void *object)
1112{
Christoph Lameter3ec09742007-05-16 22:11:00 -07001113 setup_object_debug(s, page, object);
Christoph Lameter4f104932007-05-06 14:50:17 -07001114 if (unlikely(s->ctor))
Alexey Dobriyan51cc5062008-07-25 19:45:34 -07001115 s->ctor(object);
Christoph Lameter81819f02007-05-06 14:49:36 -07001116}
1117
1118static struct page *new_slab(struct kmem_cache *s, gfp_t flags, int node)
1119{
1120 struct page *page;
Christoph Lameter81819f02007-05-06 14:49:36 -07001121 void *start;
Christoph Lameter81819f02007-05-06 14:49:36 -07001122 void *last;
1123 void *p;
1124
Christoph Lameter6cb06222007-10-16 01:25:41 -07001125 BUG_ON(flags & GFP_SLAB_BUG_MASK);
Christoph Lameter81819f02007-05-06 14:49:36 -07001126
Christoph Lameter6cb06222007-10-16 01:25:41 -07001127 page = allocate_slab(s,
1128 flags & (GFP_RECLAIM_MASK | GFP_CONSTRAINT_MASK), node);
Christoph Lameter81819f02007-05-06 14:49:36 -07001129 if (!page)
1130 goto out;
1131
Christoph Lameter205ab992008-04-14 19:11:40 +03001132 inc_slabs_node(s, page_to_nid(page), page->objects);
Christoph Lameter81819f02007-05-06 14:49:36 -07001133 page->slab = s;
1134 page->flags |= 1 << PG_slab;
1135 if (s->flags & (SLAB_DEBUG_FREE | SLAB_RED_ZONE | SLAB_POISON |
1136 SLAB_STORE_USER | SLAB_TRACE))
Andy Whitcroft8a380822008-07-23 21:27:18 -07001137 __SetPageSlubDebug(page);
Christoph Lameter81819f02007-05-06 14:49:36 -07001138
1139 start = page_address(page);
Christoph Lameter81819f02007-05-06 14:49:36 -07001140
1141 if (unlikely(s->flags & SLAB_POISON))
Christoph Lameter834f3d12008-04-14 19:11:31 +03001142 memset(start, POISON_INUSE, PAGE_SIZE << compound_order(page));
Christoph Lameter81819f02007-05-06 14:49:36 -07001143
1144 last = start;
Christoph Lameter224a88b2008-04-14 19:11:31 +03001145 for_each_object(p, s, start, page->objects) {
Christoph Lameter81819f02007-05-06 14:49:36 -07001146 setup_object(s, page, last);
1147 set_freepointer(s, last, p);
1148 last = p;
1149 }
1150 setup_object(s, page, last);
Christoph Lametera973e9d2008-03-01 13:40:44 -08001151 set_freepointer(s, last, NULL);
Christoph Lameter81819f02007-05-06 14:49:36 -07001152
1153 page->freelist = start;
1154 page->inuse = 0;
1155out:
Christoph Lameter81819f02007-05-06 14:49:36 -07001156 return page;
1157}
1158
1159static void __free_slab(struct kmem_cache *s, struct page *page)
1160{
Christoph Lameter834f3d12008-04-14 19:11:31 +03001161 int order = compound_order(page);
1162 int pages = 1 << order;
Christoph Lameter81819f02007-05-06 14:49:36 -07001163
Andy Whitcroft8a380822008-07-23 21:27:18 -07001164 if (unlikely(SLABDEBUG && PageSlubDebug(page))) {
Christoph Lameter81819f02007-05-06 14:49:36 -07001165 void *p;
1166
1167 slab_pad_check(s, page);
Christoph Lameter224a88b2008-04-14 19:11:31 +03001168 for_each_object(p, s, page_address(page),
1169 page->objects)
Christoph Lameter81819f02007-05-06 14:49:36 -07001170 check_object(s, page, p, 0);
Andy Whitcroft8a380822008-07-23 21:27:18 -07001171 __ClearPageSlubDebug(page);
Christoph Lameter81819f02007-05-06 14:49:36 -07001172 }
1173
1174 mod_zone_page_state(page_zone(page),
1175 (s->flags & SLAB_RECLAIM_ACCOUNT) ?
1176 NR_SLAB_RECLAIMABLE : NR_SLAB_UNRECLAIMABLE,
Pekka Enberg06428782008-01-07 23:20:27 -08001177 -pages);
Christoph Lameter81819f02007-05-06 14:49:36 -07001178
Christoph Lameter49bd5222008-04-14 18:52:18 +03001179 __ClearPageSlab(page);
1180 reset_page_mapcount(page);
Nick Piggin1eb5ac62009-05-05 19:13:44 +10001181 if (current->reclaim_state)
1182 current->reclaim_state->reclaimed_slab += pages;
Christoph Lameter834f3d12008-04-14 19:11:31 +03001183 __free_pages(page, order);
Christoph Lameter81819f02007-05-06 14:49:36 -07001184}
1185
1186static void rcu_free_slab(struct rcu_head *h)
1187{
1188 struct page *page;
1189
1190 page = container_of((struct list_head *)h, struct page, lru);
1191 __free_slab(page->slab, page);
1192}
1193
1194static void free_slab(struct kmem_cache *s, struct page *page)
1195{
1196 if (unlikely(s->flags & SLAB_DESTROY_BY_RCU)) {
1197 /*
1198 * RCU free overloads the RCU head over the LRU
1199 */
1200 struct rcu_head *head = (void *)&page->lru;
1201
1202 call_rcu(head, rcu_free_slab);
1203 } else
1204 __free_slab(s, page);
1205}
1206
1207static void discard_slab(struct kmem_cache *s, struct page *page)
1208{
Christoph Lameter205ab992008-04-14 19:11:40 +03001209 dec_slabs_node(s, page_to_nid(page), page->objects);
Christoph Lameter81819f02007-05-06 14:49:36 -07001210 free_slab(s, page);
1211}
1212
1213/*
1214 * Per slab locking using the pagelock
1215 */
1216static __always_inline void slab_lock(struct page *page)
1217{
1218 bit_spin_lock(PG_locked, &page->flags);
1219}
1220
1221static __always_inline void slab_unlock(struct page *page)
1222{
Nick Piggina76d3542008-01-07 23:20:27 -08001223 __bit_spin_unlock(PG_locked, &page->flags);
Christoph Lameter81819f02007-05-06 14:49:36 -07001224}
1225
1226static __always_inline int slab_trylock(struct page *page)
1227{
1228 int rc = 1;
1229
1230 rc = bit_spin_trylock(PG_locked, &page->flags);
1231 return rc;
1232}
1233
1234/*
1235 * Management of partially allocated slabs
1236 */
Christoph Lameter7c2e1322008-01-07 23:20:27 -08001237static void add_partial(struct kmem_cache_node *n,
1238 struct page *page, int tail)
Christoph Lameter81819f02007-05-06 14:49:36 -07001239{
Christoph Lametere95eed52007-05-06 14:49:44 -07001240 spin_lock(&n->list_lock);
1241 n->nr_partial++;
Christoph Lameter7c2e1322008-01-07 23:20:27 -08001242 if (tail)
1243 list_add_tail(&page->lru, &n->partial);
1244 else
1245 list_add(&page->lru, &n->partial);
Christoph Lameter81819f02007-05-06 14:49:36 -07001246 spin_unlock(&n->list_lock);
1247}
1248
Christoph Lameter0121c6192008-04-29 16:11:12 -07001249static void remove_partial(struct kmem_cache *s, struct page *page)
Christoph Lameter81819f02007-05-06 14:49:36 -07001250{
1251 struct kmem_cache_node *n = get_node(s, page_to_nid(page));
1252
1253 spin_lock(&n->list_lock);
1254 list_del(&page->lru);
1255 n->nr_partial--;
1256 spin_unlock(&n->list_lock);
1257}
1258
1259/*
Christoph Lameter672bba32007-05-09 02:32:39 -07001260 * Lock slab and remove from the partial list.
Christoph Lameter81819f02007-05-06 14:49:36 -07001261 *
Christoph Lameter672bba32007-05-09 02:32:39 -07001262 * Must hold list_lock.
Christoph Lameter81819f02007-05-06 14:49:36 -07001263 */
Christoph Lameter0121c6192008-04-29 16:11:12 -07001264static inline int lock_and_freeze_slab(struct kmem_cache_node *n,
1265 struct page *page)
Christoph Lameter81819f02007-05-06 14:49:36 -07001266{
1267 if (slab_trylock(page)) {
1268 list_del(&page->lru);
1269 n->nr_partial--;
Andy Whitcroft8a380822008-07-23 21:27:18 -07001270 __SetPageSlubFrozen(page);
Christoph Lameter81819f02007-05-06 14:49:36 -07001271 return 1;
1272 }
1273 return 0;
1274}
1275
1276/*
Christoph Lameter672bba32007-05-09 02:32:39 -07001277 * Try to allocate a partial slab from a specific node.
Christoph Lameter81819f02007-05-06 14:49:36 -07001278 */
1279static struct page *get_partial_node(struct kmem_cache_node *n)
1280{
1281 struct page *page;
1282
1283 /*
1284 * Racy check. If we mistakenly see no partial slabs then we
1285 * just allocate an empty slab. If we mistakenly try to get a
Christoph Lameter672bba32007-05-09 02:32:39 -07001286 * partial slab and there is none available then get_partials()
1287 * will return NULL.
Christoph Lameter81819f02007-05-06 14:49:36 -07001288 */
1289 if (!n || !n->nr_partial)
1290 return NULL;
1291
1292 spin_lock(&n->list_lock);
1293 list_for_each_entry(page, &n->partial, lru)
Christoph Lameter4b6f0752007-05-16 22:10:53 -07001294 if (lock_and_freeze_slab(n, page))
Christoph Lameter81819f02007-05-06 14:49:36 -07001295 goto out;
1296 page = NULL;
1297out:
1298 spin_unlock(&n->list_lock);
1299 return page;
1300}
1301
1302/*
Christoph Lameter672bba32007-05-09 02:32:39 -07001303 * Get a page from somewhere. Search in increasing NUMA distances.
Christoph Lameter81819f02007-05-06 14:49:36 -07001304 */
1305static struct page *get_any_partial(struct kmem_cache *s, gfp_t flags)
1306{
1307#ifdef CONFIG_NUMA
1308 struct zonelist *zonelist;
Mel Gormandd1a2392008-04-28 02:12:17 -07001309 struct zoneref *z;
Mel Gorman54a6eb52008-04-28 02:12:16 -07001310 struct zone *zone;
1311 enum zone_type high_zoneidx = gfp_zone(flags);
Christoph Lameter81819f02007-05-06 14:49:36 -07001312 struct page *page;
1313
1314 /*
Christoph Lameter672bba32007-05-09 02:32:39 -07001315 * The defrag ratio allows a configuration of the tradeoffs between
1316 * inter node defragmentation and node local allocations. A lower
1317 * defrag_ratio increases the tendency to do local allocations
1318 * instead of attempting to obtain partial slabs from other nodes.
Christoph Lameter81819f02007-05-06 14:49:36 -07001319 *
Christoph Lameter672bba32007-05-09 02:32:39 -07001320 * If the defrag_ratio is set to 0 then kmalloc() always
1321 * returns node local objects. If the ratio is higher then kmalloc()
1322 * may return off node objects because partial slabs are obtained
1323 * from other nodes and filled up.
Christoph Lameter81819f02007-05-06 14:49:36 -07001324 *
Christoph Lameter6446faa2008-02-15 23:45:26 -08001325 * If /sys/kernel/slab/xx/defrag_ratio is set to 100 (which makes
Christoph Lameter672bba32007-05-09 02:32:39 -07001326 * defrag_ratio = 1000) then every (well almost) allocation will
1327 * first attempt to defrag slab caches on other nodes. This means
1328 * scanning over all nodes to look for partial slabs which may be
1329 * expensive if we do it every time we are trying to find a slab
1330 * with available objects.
Christoph Lameter81819f02007-05-06 14:49:36 -07001331 */
Christoph Lameter98246012008-01-07 23:20:26 -08001332 if (!s->remote_node_defrag_ratio ||
1333 get_cycles() % 1024 > s->remote_node_defrag_ratio)
Christoph Lameter81819f02007-05-06 14:49:36 -07001334 return NULL;
1335
Mel Gorman0e884602008-04-28 02:12:14 -07001336 zonelist = node_zonelist(slab_node(current->mempolicy), flags);
Mel Gorman54a6eb52008-04-28 02:12:16 -07001337 for_each_zone_zonelist(zone, z, zonelist, high_zoneidx) {
Christoph Lameter81819f02007-05-06 14:49:36 -07001338 struct kmem_cache_node *n;
1339
Mel Gorman54a6eb52008-04-28 02:12:16 -07001340 n = get_node(s, zone_to_nid(zone));
Christoph Lameter81819f02007-05-06 14:49:36 -07001341
Mel Gorman54a6eb52008-04-28 02:12:16 -07001342 if (n && cpuset_zone_allowed_hardwall(zone, flags) &&
David Rientjes3b89d7d2009-02-22 17:40:07 -08001343 n->nr_partial > s->min_partial) {
Christoph Lameter81819f02007-05-06 14:49:36 -07001344 page = get_partial_node(n);
1345 if (page)
1346 return page;
1347 }
1348 }
1349#endif
1350 return NULL;
1351}
1352
1353/*
1354 * Get a partial page, lock it and return it.
1355 */
1356static struct page *get_partial(struct kmem_cache *s, gfp_t flags, int node)
1357{
1358 struct page *page;
1359 int searchnode = (node == -1) ? numa_node_id() : node;
1360
1361 page = get_partial_node(get_node(s, searchnode));
1362 if (page || (flags & __GFP_THISNODE))
1363 return page;
1364
1365 return get_any_partial(s, flags);
1366}
1367
1368/*
1369 * Move a page back to the lists.
1370 *
1371 * Must be called with the slab lock held.
1372 *
1373 * On exit the slab lock will have been dropped.
1374 */
Christoph Lameter7c2e1322008-01-07 23:20:27 -08001375static void unfreeze_slab(struct kmem_cache *s, struct page *page, int tail)
Christoph Lameter81819f02007-05-06 14:49:36 -07001376{
Christoph Lametere95eed52007-05-06 14:49:44 -07001377 struct kmem_cache_node *n = get_node(s, page_to_nid(page));
Christoph Lameter8ff12cf2008-02-07 17:47:41 -08001378 struct kmem_cache_cpu *c = get_cpu_slab(s, smp_processor_id());
Christoph Lametere95eed52007-05-06 14:49:44 -07001379
Andy Whitcroft8a380822008-07-23 21:27:18 -07001380 __ClearPageSlubFrozen(page);
Christoph Lameter81819f02007-05-06 14:49:36 -07001381 if (page->inuse) {
Christoph Lametere95eed52007-05-06 14:49:44 -07001382
Christoph Lametera973e9d2008-03-01 13:40:44 -08001383 if (page->freelist) {
Christoph Lameter7c2e1322008-01-07 23:20:27 -08001384 add_partial(n, page, tail);
Christoph Lameter8ff12cf2008-02-07 17:47:41 -08001385 stat(c, tail ? DEACTIVATE_TO_TAIL : DEACTIVATE_TO_HEAD);
1386 } else {
1387 stat(c, DEACTIVATE_FULL);
Andy Whitcroft8a380822008-07-23 21:27:18 -07001388 if (SLABDEBUG && PageSlubDebug(page) &&
1389 (s->flags & SLAB_STORE_USER))
Christoph Lameter8ff12cf2008-02-07 17:47:41 -08001390 add_full(n, page);
1391 }
Christoph Lameter81819f02007-05-06 14:49:36 -07001392 slab_unlock(page);
1393 } else {
Christoph Lameter8ff12cf2008-02-07 17:47:41 -08001394 stat(c, DEACTIVATE_EMPTY);
David Rientjes3b89d7d2009-02-22 17:40:07 -08001395 if (n->nr_partial < s->min_partial) {
Christoph Lametere95eed52007-05-06 14:49:44 -07001396 /*
Christoph Lameter672bba32007-05-09 02:32:39 -07001397 * Adding an empty slab to the partial slabs in order
1398 * to avoid page allocator overhead. This slab needs
1399 * to come after the other slabs with objects in
Christoph Lameter6446faa2008-02-15 23:45:26 -08001400 * so that the others get filled first. That way the
1401 * size of the partial list stays small.
1402 *
Christoph Lameter0121c6192008-04-29 16:11:12 -07001403 * kmem_cache_shrink can reclaim any empty slabs from
1404 * the partial list.
Christoph Lametere95eed52007-05-06 14:49:44 -07001405 */
Christoph Lameter7c2e1322008-01-07 23:20:27 -08001406 add_partial(n, page, 1);
Christoph Lametere95eed52007-05-06 14:49:44 -07001407 slab_unlock(page);
1408 } else {
1409 slab_unlock(page);
Christoph Lameter8ff12cf2008-02-07 17:47:41 -08001410 stat(get_cpu_slab(s, raw_smp_processor_id()), FREE_SLAB);
Christoph Lametere95eed52007-05-06 14:49:44 -07001411 discard_slab(s, page);
1412 }
Christoph Lameter81819f02007-05-06 14:49:36 -07001413 }
1414}
1415
1416/*
1417 * Remove the cpu slab
1418 */
Christoph Lameterdfb4f092007-10-16 01:26:05 -07001419static void deactivate_slab(struct kmem_cache *s, struct kmem_cache_cpu *c)
Christoph Lameter81819f02007-05-06 14:49:36 -07001420{
Christoph Lameterdfb4f092007-10-16 01:26:05 -07001421 struct page *page = c->page;
Christoph Lameter7c2e1322008-01-07 23:20:27 -08001422 int tail = 1;
Christoph Lameter8ff12cf2008-02-07 17:47:41 -08001423
Christoph Lameterb773ad72008-03-04 11:10:17 -08001424 if (page->freelist)
Christoph Lameter8ff12cf2008-02-07 17:47:41 -08001425 stat(c, DEACTIVATE_REMOTE_FREES);
Christoph Lameter894b8782007-05-10 03:15:16 -07001426 /*
Christoph Lameter6446faa2008-02-15 23:45:26 -08001427 * Merge cpu freelist into slab freelist. Typically we get here
Christoph Lameter894b8782007-05-10 03:15:16 -07001428 * because both freelists are empty. So this is unlikely
1429 * to occur.
1430 */
Christoph Lametera973e9d2008-03-01 13:40:44 -08001431 while (unlikely(c->freelist)) {
Christoph Lameter894b8782007-05-10 03:15:16 -07001432 void **object;
1433
Christoph Lameter7c2e1322008-01-07 23:20:27 -08001434 tail = 0; /* Hot objects. Put the slab first */
1435
Christoph Lameter894b8782007-05-10 03:15:16 -07001436 /* Retrieve object from cpu_freelist */
Christoph Lameterdfb4f092007-10-16 01:26:05 -07001437 object = c->freelist;
Christoph Lameterb3fba8d2007-10-16 01:26:06 -07001438 c->freelist = c->freelist[c->offset];
Christoph Lameter894b8782007-05-10 03:15:16 -07001439
1440 /* And put onto the regular freelist */
Christoph Lameterb3fba8d2007-10-16 01:26:06 -07001441 object[c->offset] = page->freelist;
Christoph Lameter894b8782007-05-10 03:15:16 -07001442 page->freelist = object;
1443 page->inuse--;
1444 }
Christoph Lameterdfb4f092007-10-16 01:26:05 -07001445 c->page = NULL;
Christoph Lameter7c2e1322008-01-07 23:20:27 -08001446 unfreeze_slab(s, page, tail);
Christoph Lameter81819f02007-05-06 14:49:36 -07001447}
1448
Christoph Lameterdfb4f092007-10-16 01:26:05 -07001449static inline void flush_slab(struct kmem_cache *s, struct kmem_cache_cpu *c)
Christoph Lameter81819f02007-05-06 14:49:36 -07001450{
Christoph Lameter8ff12cf2008-02-07 17:47:41 -08001451 stat(c, CPUSLAB_FLUSH);
Christoph Lameterdfb4f092007-10-16 01:26:05 -07001452 slab_lock(c->page);
1453 deactivate_slab(s, c);
Christoph Lameter81819f02007-05-06 14:49:36 -07001454}
1455
1456/*
1457 * Flush cpu slab.
Christoph Lameter6446faa2008-02-15 23:45:26 -08001458 *
Christoph Lameter81819f02007-05-06 14:49:36 -07001459 * Called from IPI handler with interrupts disabled.
1460 */
Christoph Lameter0c710012007-07-17 04:03:24 -07001461static inline void __flush_cpu_slab(struct kmem_cache *s, int cpu)
Christoph Lameter81819f02007-05-06 14:49:36 -07001462{
Christoph Lameterdfb4f092007-10-16 01:26:05 -07001463 struct kmem_cache_cpu *c = get_cpu_slab(s, cpu);
Christoph Lameter81819f02007-05-06 14:49:36 -07001464
Christoph Lameterdfb4f092007-10-16 01:26:05 -07001465 if (likely(c && c->page))
1466 flush_slab(s, c);
Christoph Lameter81819f02007-05-06 14:49:36 -07001467}
1468
1469static void flush_cpu_slab(void *d)
1470{
1471 struct kmem_cache *s = d;
Christoph Lameter81819f02007-05-06 14:49:36 -07001472
Christoph Lameterdfb4f092007-10-16 01:26:05 -07001473 __flush_cpu_slab(s, smp_processor_id());
Christoph Lameter81819f02007-05-06 14:49:36 -07001474}
1475
1476static void flush_all(struct kmem_cache *s)
1477{
Jens Axboe15c8b6c2008-05-09 09:39:44 +02001478 on_each_cpu(flush_cpu_slab, s, 1);
Christoph Lameter81819f02007-05-06 14:49:36 -07001479}
1480
1481/*
Christoph Lameterdfb4f092007-10-16 01:26:05 -07001482 * Check if the objects in a per cpu structure fit numa
1483 * locality expectations.
1484 */
1485static inline int node_match(struct kmem_cache_cpu *c, int node)
1486{
1487#ifdef CONFIG_NUMA
1488 if (node != -1 && c->node != node)
1489 return 0;
1490#endif
1491 return 1;
1492}
1493
1494/*
Christoph Lameter894b8782007-05-10 03:15:16 -07001495 * Slow path. The lockless freelist is empty or we need to perform
1496 * debugging duties.
Christoph Lameter81819f02007-05-06 14:49:36 -07001497 *
Christoph Lameter894b8782007-05-10 03:15:16 -07001498 * Interrupts are disabled.
Christoph Lameter81819f02007-05-06 14:49:36 -07001499 *
Christoph Lameter894b8782007-05-10 03:15:16 -07001500 * Processing is still very fast if new objects have been freed to the
1501 * regular freelist. In that case we simply take over the regular freelist
1502 * as the lockless freelist and zap the regular freelist.
Christoph Lameter81819f02007-05-06 14:49:36 -07001503 *
Christoph Lameter894b8782007-05-10 03:15:16 -07001504 * If that is not working then we fall back to the partial lists. We take the
1505 * first element of the freelist as the object to allocate now and move the
1506 * rest of the freelist to the lockless freelist.
1507 *
1508 * And if we were unable to get a new slab from the partial slab lists then
Christoph Lameter6446faa2008-02-15 23:45:26 -08001509 * we need to allocate a new slab. This is the slowest path since it involves
1510 * a call to the page allocator and the setup of a new slab.
Christoph Lameter81819f02007-05-06 14:49:36 -07001511 */
Eduard - Gabriel Munteanuce71e272008-08-19 20:43:25 +03001512static void *__slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node,
1513 unsigned long addr, struct kmem_cache_cpu *c)
Christoph Lameter81819f02007-05-06 14:49:36 -07001514{
Christoph Lameter81819f02007-05-06 14:49:36 -07001515 void **object;
Christoph Lameterdfb4f092007-10-16 01:26:05 -07001516 struct page *new;
Christoph Lameter81819f02007-05-06 14:49:36 -07001517
Linus Torvaldse72e9c22008-03-27 20:56:33 -07001518 /* We handle __GFP_ZERO in the caller */
1519 gfpflags &= ~__GFP_ZERO;
1520
Christoph Lameterdfb4f092007-10-16 01:26:05 -07001521 if (!c->page)
Christoph Lameter81819f02007-05-06 14:49:36 -07001522 goto new_slab;
1523
Christoph Lameterdfb4f092007-10-16 01:26:05 -07001524 slab_lock(c->page);
1525 if (unlikely(!node_match(c, node)))
Christoph Lameter81819f02007-05-06 14:49:36 -07001526 goto another_slab;
Christoph Lameter6446faa2008-02-15 23:45:26 -08001527
Christoph Lameter8ff12cf2008-02-07 17:47:41 -08001528 stat(c, ALLOC_REFILL);
Christoph Lameter6446faa2008-02-15 23:45:26 -08001529
Christoph Lameter894b8782007-05-10 03:15:16 -07001530load_freelist:
Christoph Lameterdfb4f092007-10-16 01:26:05 -07001531 object = c->page->freelist;
Christoph Lametera973e9d2008-03-01 13:40:44 -08001532 if (unlikely(!object))
Christoph Lameter81819f02007-05-06 14:49:36 -07001533 goto another_slab;
Andy Whitcroft8a380822008-07-23 21:27:18 -07001534 if (unlikely(SLABDEBUG && PageSlubDebug(c->page)))
Christoph Lameter81819f02007-05-06 14:49:36 -07001535 goto debug;
1536
Christoph Lameterb3fba8d2007-10-16 01:26:06 -07001537 c->freelist = object[c->offset];
Christoph Lameter39b26462008-04-14 19:11:30 +03001538 c->page->inuse = c->page->objects;
Christoph Lametera973e9d2008-03-01 13:40:44 -08001539 c->page->freelist = NULL;
Christoph Lameterdfb4f092007-10-16 01:26:05 -07001540 c->node = page_to_nid(c->page);
Christoph Lameter1f842602008-01-07 23:20:30 -08001541unlock_out:
Christoph Lameterdfb4f092007-10-16 01:26:05 -07001542 slab_unlock(c->page);
Christoph Lameter8ff12cf2008-02-07 17:47:41 -08001543 stat(c, ALLOC_SLOWPATH);
Christoph Lameter81819f02007-05-06 14:49:36 -07001544 return object;
1545
1546another_slab:
Christoph Lameterdfb4f092007-10-16 01:26:05 -07001547 deactivate_slab(s, c);
Christoph Lameter81819f02007-05-06 14:49:36 -07001548
1549new_slab:
Christoph Lameterdfb4f092007-10-16 01:26:05 -07001550 new = get_partial(s, gfpflags, node);
1551 if (new) {
1552 c->page = new;
Christoph Lameter8ff12cf2008-02-07 17:47:41 -08001553 stat(c, ALLOC_FROM_PARTIAL);
Christoph Lameter894b8782007-05-10 03:15:16 -07001554 goto load_freelist;
Christoph Lameter81819f02007-05-06 14:49:36 -07001555 }
1556
Christoph Lameterb811c202007-10-16 23:25:51 -07001557 if (gfpflags & __GFP_WAIT)
1558 local_irq_enable();
1559
Christoph Lameterdfb4f092007-10-16 01:26:05 -07001560 new = new_slab(s, gfpflags, node);
Christoph Lameterb811c202007-10-16 23:25:51 -07001561
1562 if (gfpflags & __GFP_WAIT)
1563 local_irq_disable();
1564
Christoph Lameterdfb4f092007-10-16 01:26:05 -07001565 if (new) {
1566 c = get_cpu_slab(s, smp_processor_id());
Christoph Lameter8ff12cf2008-02-07 17:47:41 -08001567 stat(c, ALLOC_SLAB);
Christoph Lameter05aa3452007-11-05 11:31:58 -08001568 if (c->page)
Christoph Lameterdfb4f092007-10-16 01:26:05 -07001569 flush_slab(s, c);
Christoph Lameterdfb4f092007-10-16 01:26:05 -07001570 slab_lock(new);
Andy Whitcroft8a380822008-07-23 21:27:18 -07001571 __SetPageSlubFrozen(new);
Christoph Lameterdfb4f092007-10-16 01:26:05 -07001572 c->page = new;
Christoph Lameter4b6f0752007-05-16 22:10:53 -07001573 goto load_freelist;
Christoph Lameter81819f02007-05-06 14:49:36 -07001574 }
Christoph Lameter71c7a062008-02-14 14:28:01 -08001575 return NULL;
Christoph Lameter81819f02007-05-06 14:49:36 -07001576debug:
Christoph Lameterdfb4f092007-10-16 01:26:05 -07001577 if (!alloc_debug_processing(s, c->page, object, addr))
Christoph Lameter81819f02007-05-06 14:49:36 -07001578 goto another_slab;
Christoph Lameter894b8782007-05-10 03:15:16 -07001579
Christoph Lameterdfb4f092007-10-16 01:26:05 -07001580 c->page->inuse++;
Christoph Lameterb3fba8d2007-10-16 01:26:06 -07001581 c->page->freelist = object[c->offset];
Christoph Lameteree3c72a2007-10-16 01:26:07 -07001582 c->node = -1;
Christoph Lameter1f842602008-01-07 23:20:30 -08001583 goto unlock_out;
Christoph Lameter894b8782007-05-10 03:15:16 -07001584}
1585
1586/*
1587 * Inlined fastpath so that allocation functions (kmalloc, kmem_cache_alloc)
1588 * have the fastpath folded into their functions. So no function call
1589 * overhead for requests that can be satisfied on the fastpath.
1590 *
1591 * The fastpath works by first checking if the lockless freelist can be used.
1592 * If not then __slab_alloc is called for slow processing.
1593 *
1594 * Otherwise we can simply pick the next object from the lockless free list.
1595 */
Pekka Enberg06428782008-01-07 23:20:27 -08001596static __always_inline void *slab_alloc(struct kmem_cache *s,
Eduard - Gabriel Munteanuce71e272008-08-19 20:43:25 +03001597 gfp_t gfpflags, int node, unsigned long addr)
Christoph Lameter894b8782007-05-10 03:15:16 -07001598{
Christoph Lameter894b8782007-05-10 03:15:16 -07001599 void **object;
Christoph Lameterdfb4f092007-10-16 01:26:05 -07001600 struct kmem_cache_cpu *c;
Christoph Lameter1f842602008-01-07 23:20:30 -08001601 unsigned long flags;
Dmitry Adamushkobdb21922008-07-10 22:21:58 +02001602 unsigned int objsize;
Christoph Lameter1f842602008-01-07 23:20:30 -08001603
Pekka Enberg7e85ee02009-06-12 14:03:06 +03001604 gfpflags &= slab_gfp_mask;
1605
Nick Piggincf40bd12009-01-21 08:12:39 +01001606 lockdep_trace_alloc(gfpflags);
OGAWA Hirofumi89124d72008-11-19 21:23:59 +09001607 might_sleep_if(gfpflags & __GFP_WAIT);
Pekka Enberg3c506ef2008-12-29 11:47:05 +02001608
Akinobu Mita773ff602008-12-23 19:37:01 +09001609 if (should_failslab(s->objsize, gfpflags))
1610 return NULL;
1611
Christoph Lameter894b8782007-05-10 03:15:16 -07001612 local_irq_save(flags);
Christoph Lameterdfb4f092007-10-16 01:26:05 -07001613 c = get_cpu_slab(s, smp_processor_id());
Dmitry Adamushkobdb21922008-07-10 22:21:58 +02001614 objsize = c->objsize;
Christoph Lametera973e9d2008-03-01 13:40:44 -08001615 if (unlikely(!c->freelist || !node_match(c, node)))
Christoph Lameter894b8782007-05-10 03:15:16 -07001616
Christoph Lameterdfb4f092007-10-16 01:26:05 -07001617 object = __slab_alloc(s, gfpflags, node, addr, c);
Christoph Lameter894b8782007-05-10 03:15:16 -07001618
1619 else {
Christoph Lameterdfb4f092007-10-16 01:26:05 -07001620 object = c->freelist;
Christoph Lameterb3fba8d2007-10-16 01:26:06 -07001621 c->freelist = object[c->offset];
Christoph Lameter8ff12cf2008-02-07 17:47:41 -08001622 stat(c, ALLOC_FASTPATH);
Christoph Lameter894b8782007-05-10 03:15:16 -07001623 }
1624 local_irq_restore(flags);
Christoph Lameterd07dbea2007-07-17 04:03:23 -07001625
1626 if (unlikely((gfpflags & __GFP_ZERO) && object))
Dmitry Adamushkobdb21922008-07-10 22:21:58 +02001627 memset(object, 0, objsize);
Christoph Lameterd07dbea2007-07-17 04:03:23 -07001628
Catalin Marinas06f22f12009-06-11 13:23:18 +01001629 kmemleak_alloc_recursive(object, objsize, 1, s->flags, gfpflags);
Christoph Lameter894b8782007-05-10 03:15:16 -07001630 return object;
Christoph Lameter81819f02007-05-06 14:49:36 -07001631}
1632
1633void *kmem_cache_alloc(struct kmem_cache *s, gfp_t gfpflags)
1634{
Eduard - Gabriel Munteanu5b882be2008-08-19 20:43:26 +03001635 void *ret = slab_alloc(s, gfpflags, -1, _RET_IP_);
1636
Eduard - Gabriel Munteanuca2b84c2009-03-23 15:12:24 +02001637 trace_kmem_cache_alloc(_RET_IP_, ret, s->objsize, s->size, gfpflags);
Eduard - Gabriel Munteanu5b882be2008-08-19 20:43:26 +03001638
1639 return ret;
Christoph Lameter81819f02007-05-06 14:49:36 -07001640}
1641EXPORT_SYMBOL(kmem_cache_alloc);
1642
Eduard - Gabriel Munteanu5b882be2008-08-19 20:43:26 +03001643#ifdef CONFIG_KMEMTRACE
1644void *kmem_cache_alloc_notrace(struct kmem_cache *s, gfp_t gfpflags)
1645{
1646 return slab_alloc(s, gfpflags, -1, _RET_IP_);
1647}
1648EXPORT_SYMBOL(kmem_cache_alloc_notrace);
1649#endif
1650
Christoph Lameter81819f02007-05-06 14:49:36 -07001651#ifdef CONFIG_NUMA
1652void *kmem_cache_alloc_node(struct kmem_cache *s, gfp_t gfpflags, int node)
1653{
Eduard - Gabriel Munteanu5b882be2008-08-19 20:43:26 +03001654 void *ret = slab_alloc(s, gfpflags, node, _RET_IP_);
1655
Eduard - Gabriel Munteanuca2b84c2009-03-23 15:12:24 +02001656 trace_kmem_cache_alloc_node(_RET_IP_, ret,
1657 s->objsize, s->size, gfpflags, node);
Eduard - Gabriel Munteanu5b882be2008-08-19 20:43:26 +03001658
1659 return ret;
Christoph Lameter81819f02007-05-06 14:49:36 -07001660}
1661EXPORT_SYMBOL(kmem_cache_alloc_node);
1662#endif
1663
Eduard - Gabriel Munteanu5b882be2008-08-19 20:43:26 +03001664#ifdef CONFIG_KMEMTRACE
1665void *kmem_cache_alloc_node_notrace(struct kmem_cache *s,
1666 gfp_t gfpflags,
1667 int node)
1668{
1669 return slab_alloc(s, gfpflags, node, _RET_IP_);
1670}
1671EXPORT_SYMBOL(kmem_cache_alloc_node_notrace);
1672#endif
1673
Christoph Lameter81819f02007-05-06 14:49:36 -07001674/*
Christoph Lameter894b8782007-05-10 03:15:16 -07001675 * Slow patch handling. This may still be called frequently since objects
1676 * have a longer lifetime than the cpu slabs in most processing loads.
Christoph Lameter81819f02007-05-06 14:49:36 -07001677 *
Christoph Lameter894b8782007-05-10 03:15:16 -07001678 * So we still attempt to reduce cache line usage. Just take the slab
1679 * lock and free the item. If there is no additional partial page
1680 * handling required then we can return immediately.
Christoph Lameter81819f02007-05-06 14:49:36 -07001681 */
Christoph Lameter894b8782007-05-10 03:15:16 -07001682static void __slab_free(struct kmem_cache *s, struct page *page,
Eduard - Gabriel Munteanuce71e272008-08-19 20:43:25 +03001683 void *x, unsigned long addr, unsigned int offset)
Christoph Lameter81819f02007-05-06 14:49:36 -07001684{
1685 void *prior;
1686 void **object = (void *)x;
Christoph Lameter8ff12cf2008-02-07 17:47:41 -08001687 struct kmem_cache_cpu *c;
Christoph Lameter81819f02007-05-06 14:49:36 -07001688
Christoph Lameter8ff12cf2008-02-07 17:47:41 -08001689 c = get_cpu_slab(s, raw_smp_processor_id());
1690 stat(c, FREE_SLOWPATH);
Christoph Lameter81819f02007-05-06 14:49:36 -07001691 slab_lock(page);
1692
Andy Whitcroft8a380822008-07-23 21:27:18 -07001693 if (unlikely(SLABDEBUG && PageSlubDebug(page)))
Christoph Lameter81819f02007-05-06 14:49:36 -07001694 goto debug;
Christoph Lameter6446faa2008-02-15 23:45:26 -08001695
Christoph Lameter81819f02007-05-06 14:49:36 -07001696checks_ok:
Christoph Lameterb3fba8d2007-10-16 01:26:06 -07001697 prior = object[offset] = page->freelist;
Christoph Lameter81819f02007-05-06 14:49:36 -07001698 page->freelist = object;
1699 page->inuse--;
1700
Andy Whitcroft8a380822008-07-23 21:27:18 -07001701 if (unlikely(PageSlubFrozen(page))) {
Christoph Lameter8ff12cf2008-02-07 17:47:41 -08001702 stat(c, FREE_FROZEN);
Christoph Lameter81819f02007-05-06 14:49:36 -07001703 goto out_unlock;
Christoph Lameter8ff12cf2008-02-07 17:47:41 -08001704 }
Christoph Lameter81819f02007-05-06 14:49:36 -07001705
1706 if (unlikely(!page->inuse))
1707 goto slab_empty;
1708
1709 /*
Christoph Lameter6446faa2008-02-15 23:45:26 -08001710 * Objects left in the slab. If it was not on the partial list before
Christoph Lameter81819f02007-05-06 14:49:36 -07001711 * then add it.
1712 */
Christoph Lametera973e9d2008-03-01 13:40:44 -08001713 if (unlikely(!prior)) {
Christoph Lameter7c2e1322008-01-07 23:20:27 -08001714 add_partial(get_node(s, page_to_nid(page)), page, 1);
Christoph Lameter8ff12cf2008-02-07 17:47:41 -08001715 stat(c, FREE_ADD_PARTIAL);
1716 }
Christoph Lameter81819f02007-05-06 14:49:36 -07001717
1718out_unlock:
1719 slab_unlock(page);
Christoph Lameter81819f02007-05-06 14:49:36 -07001720 return;
1721
1722slab_empty:
Christoph Lametera973e9d2008-03-01 13:40:44 -08001723 if (prior) {
Christoph Lameter81819f02007-05-06 14:49:36 -07001724 /*
Christoph Lameter672bba32007-05-09 02:32:39 -07001725 * Slab still on the partial list.
Christoph Lameter81819f02007-05-06 14:49:36 -07001726 */
1727 remove_partial(s, page);
Christoph Lameter8ff12cf2008-02-07 17:47:41 -08001728 stat(c, FREE_REMOVE_PARTIAL);
1729 }
Christoph Lameter81819f02007-05-06 14:49:36 -07001730 slab_unlock(page);
Christoph Lameter8ff12cf2008-02-07 17:47:41 -08001731 stat(c, FREE_SLAB);
Christoph Lameter81819f02007-05-06 14:49:36 -07001732 discard_slab(s, page);
Christoph Lameter81819f02007-05-06 14:49:36 -07001733 return;
1734
1735debug:
Christoph Lameter3ec09742007-05-16 22:11:00 -07001736 if (!free_debug_processing(s, page, x, addr))
Christoph Lameter77c5e2d2007-05-06 14:49:42 -07001737 goto out_unlock;
Christoph Lameter77c5e2d2007-05-06 14:49:42 -07001738 goto checks_ok;
Christoph Lameter81819f02007-05-06 14:49:36 -07001739}
1740
Christoph Lameter894b8782007-05-10 03:15:16 -07001741/*
1742 * Fastpath with forced inlining to produce a kfree and kmem_cache_free that
1743 * can perform fastpath freeing without additional function calls.
1744 *
1745 * The fastpath is only possible if we are freeing to the current cpu slab
1746 * of this processor. This typically the case if we have just allocated
1747 * the item before.
1748 *
1749 * If fastpath is not possible then fall back to __slab_free where we deal
1750 * with all sorts of special processing.
1751 */
Pekka Enberg06428782008-01-07 23:20:27 -08001752static __always_inline void slab_free(struct kmem_cache *s,
Eduard - Gabriel Munteanuce71e272008-08-19 20:43:25 +03001753 struct page *page, void *x, unsigned long addr)
Christoph Lameter894b8782007-05-10 03:15:16 -07001754{
1755 void **object = (void *)x;
Christoph Lameterdfb4f092007-10-16 01:26:05 -07001756 struct kmem_cache_cpu *c;
Christoph Lameter1f842602008-01-07 23:20:30 -08001757 unsigned long flags;
1758
Catalin Marinas06f22f12009-06-11 13:23:18 +01001759 kmemleak_free_recursive(x, s->flags);
Christoph Lameter894b8782007-05-10 03:15:16 -07001760 local_irq_save(flags);
Christoph Lameterdfb4f092007-10-16 01:26:05 -07001761 c = get_cpu_slab(s, smp_processor_id());
Christoph Lameter27d9e4e2008-02-15 23:45:25 -08001762 debug_check_no_locks_freed(object, c->objsize);
Thomas Gleixner3ac7fe52008-04-30 00:55:01 -07001763 if (!(s->flags & SLAB_DEBUG_OBJECTS))
Pekka Enberg6047a002009-01-14 12:22:25 +02001764 debug_check_no_obj_freed(object, c->objsize);
Christoph Lameteree3c72a2007-10-16 01:26:07 -07001765 if (likely(page == c->page && c->node >= 0)) {
Christoph Lameterb3fba8d2007-10-16 01:26:06 -07001766 object[c->offset] = c->freelist;
Christoph Lameterdfb4f092007-10-16 01:26:05 -07001767 c->freelist = object;
Christoph Lameter8ff12cf2008-02-07 17:47:41 -08001768 stat(c, FREE_FASTPATH);
Christoph Lameter894b8782007-05-10 03:15:16 -07001769 } else
Christoph Lameterb3fba8d2007-10-16 01:26:06 -07001770 __slab_free(s, page, x, addr, c->offset);
Christoph Lameter894b8782007-05-10 03:15:16 -07001771
1772 local_irq_restore(flags);
1773}
1774
Christoph Lameter81819f02007-05-06 14:49:36 -07001775void kmem_cache_free(struct kmem_cache *s, void *x)
1776{
Christoph Lameter77c5e2d2007-05-06 14:49:42 -07001777 struct page *page;
Christoph Lameter81819f02007-05-06 14:49:36 -07001778
Christoph Lameterb49af682007-05-06 14:49:41 -07001779 page = virt_to_head_page(x);
Christoph Lameter81819f02007-05-06 14:49:36 -07001780
Eduard - Gabriel Munteanuce71e272008-08-19 20:43:25 +03001781 slab_free(s, page, x, _RET_IP_);
Eduard - Gabriel Munteanu5b882be2008-08-19 20:43:26 +03001782
Eduard - Gabriel Munteanuca2b84c2009-03-23 15:12:24 +02001783 trace_kmem_cache_free(_RET_IP_, x);
Christoph Lameter81819f02007-05-06 14:49:36 -07001784}
1785EXPORT_SYMBOL(kmem_cache_free);
1786
Cyrill Gorcunove9beef12008-10-28 22:02:26 +03001787/* Figure out on which slab page the object resides */
Christoph Lameter81819f02007-05-06 14:49:36 -07001788static struct page *get_object_page(const void *x)
1789{
Christoph Lameterb49af682007-05-06 14:49:41 -07001790 struct page *page = virt_to_head_page(x);
Christoph Lameter81819f02007-05-06 14:49:36 -07001791
1792 if (!PageSlab(page))
1793 return NULL;
1794
1795 return page;
1796}
1797
1798/*
Christoph Lameter672bba32007-05-09 02:32:39 -07001799 * Object placement in a slab is made very easy because we always start at
1800 * offset 0. If we tune the size of the object to the alignment then we can
1801 * get the required alignment by putting one properly sized object after
1802 * another.
Christoph Lameter81819f02007-05-06 14:49:36 -07001803 *
1804 * Notice that the allocation order determines the sizes of the per cpu
1805 * caches. Each processor has always one slab available for allocations.
1806 * Increasing the allocation order reduces the number of times that slabs
Christoph Lameter672bba32007-05-09 02:32:39 -07001807 * must be moved on and off the partial lists and is therefore a factor in
Christoph Lameter81819f02007-05-06 14:49:36 -07001808 * locking overhead.
Christoph Lameter81819f02007-05-06 14:49:36 -07001809 */
1810
1811/*
1812 * Mininum / Maximum order of slab pages. This influences locking overhead
1813 * and slab fragmentation. A higher order reduces the number of partial slabs
1814 * and increases the number of allocations possible without having to
1815 * take the list_lock.
1816 */
1817static int slub_min_order;
Christoph Lameter114e9e82008-04-14 19:11:41 +03001818static int slub_max_order = PAGE_ALLOC_COSTLY_ORDER;
Christoph Lameter9b2cd502008-04-14 19:11:41 +03001819static int slub_min_objects;
Christoph Lameter81819f02007-05-06 14:49:36 -07001820
1821/*
1822 * Merge control. If this is set then no merging of slab caches will occur.
Christoph Lameter672bba32007-05-09 02:32:39 -07001823 * (Could be removed. This was introduced to pacify the merge skeptics.)
Christoph Lameter81819f02007-05-06 14:49:36 -07001824 */
1825static int slub_nomerge;
1826
1827/*
Christoph Lameter81819f02007-05-06 14:49:36 -07001828 * Calculate the order of allocation given an slab object size.
1829 *
Christoph Lameter672bba32007-05-09 02:32:39 -07001830 * The order of allocation has significant impact on performance and other
1831 * system components. Generally order 0 allocations should be preferred since
1832 * order 0 does not cause fragmentation in the page allocator. Larger objects
1833 * be problematic to put into order 0 slabs because there may be too much
Christoph Lameterc124f5b2008-04-14 19:13:29 +03001834 * unused space left. We go to a higher order if more than 1/16th of the slab
Christoph Lameter672bba32007-05-09 02:32:39 -07001835 * would be wasted.
Christoph Lameter81819f02007-05-06 14:49:36 -07001836 *
Christoph Lameter672bba32007-05-09 02:32:39 -07001837 * In order to reach satisfactory performance we must ensure that a minimum
1838 * number of objects is in one slab. Otherwise we may generate too much
1839 * activity on the partial lists which requires taking the list_lock. This is
1840 * less a concern for large slabs though which are rarely used.
Christoph Lameter81819f02007-05-06 14:49:36 -07001841 *
Christoph Lameter672bba32007-05-09 02:32:39 -07001842 * slub_max_order specifies the order where we begin to stop considering the
1843 * number of objects in a slab as critical. If we reach slub_max_order then
1844 * we try to keep the page order as low as possible. So we accept more waste
1845 * of space in favor of a small page order.
1846 *
1847 * Higher order allocations also allow the placement of more objects in a
1848 * slab and thereby reduce object handling overhead. If the user has
1849 * requested a higher mininum order then we start with that one instead of
1850 * the smallest order which will fit the object.
Christoph Lameter81819f02007-05-06 14:49:36 -07001851 */
Christoph Lameter5e6d4442007-05-09 02:32:46 -07001852static inline int slab_order(int size, int min_objects,
1853 int max_order, int fract_leftover)
Christoph Lameter81819f02007-05-06 14:49:36 -07001854{
1855 int order;
1856 int rem;
Christoph Lameter6300ea72007-07-17 04:03:20 -07001857 int min_order = slub_min_order;
Christoph Lameter81819f02007-05-06 14:49:36 -07001858
Cyrill Gorcunov210b5c02008-10-22 23:00:38 +04001859 if ((PAGE_SIZE << min_order) / size > MAX_OBJS_PER_PAGE)
1860 return get_order(size * MAX_OBJS_PER_PAGE) - 1;
Christoph Lameter39b26462008-04-14 19:11:30 +03001861
Christoph Lameter6300ea72007-07-17 04:03:20 -07001862 for (order = max(min_order,
Christoph Lameter5e6d4442007-05-09 02:32:46 -07001863 fls(min_objects * size - 1) - PAGE_SHIFT);
1864 order <= max_order; order++) {
1865
Christoph Lameter81819f02007-05-06 14:49:36 -07001866 unsigned long slab_size = PAGE_SIZE << order;
1867
Christoph Lameter5e6d4442007-05-09 02:32:46 -07001868 if (slab_size < min_objects * size)
Christoph Lameter81819f02007-05-06 14:49:36 -07001869 continue;
1870
Christoph Lameter81819f02007-05-06 14:49:36 -07001871 rem = slab_size % size;
1872
Christoph Lameter5e6d4442007-05-09 02:32:46 -07001873 if (rem <= slab_size / fract_leftover)
Christoph Lameter81819f02007-05-06 14:49:36 -07001874 break;
1875
1876 }
Christoph Lameter672bba32007-05-09 02:32:39 -07001877
Christoph Lameter81819f02007-05-06 14:49:36 -07001878 return order;
1879}
1880
Christoph Lameter5e6d4442007-05-09 02:32:46 -07001881static inline int calculate_order(int size)
1882{
1883 int order;
1884 int min_objects;
1885 int fraction;
Zhang Yanmine8120ff2009-02-12 18:00:17 +02001886 int max_objects;
Christoph Lameter5e6d4442007-05-09 02:32:46 -07001887
1888 /*
1889 * Attempt to find best configuration for a slab. This
1890 * works by first attempting to generate a layout with
1891 * the best configuration and backing off gradually.
1892 *
1893 * First we reduce the acceptable waste in a slab. Then
1894 * we reduce the minimum objects required in a slab.
1895 */
1896 min_objects = slub_min_objects;
Christoph Lameter9b2cd502008-04-14 19:11:41 +03001897 if (!min_objects)
1898 min_objects = 4 * (fls(nr_cpu_ids) + 1);
Zhang Yanmine8120ff2009-02-12 18:00:17 +02001899 max_objects = (PAGE_SIZE << slub_max_order)/size;
1900 min_objects = min(min_objects, max_objects);
1901
Christoph Lameter5e6d4442007-05-09 02:32:46 -07001902 while (min_objects > 1) {
Christoph Lameterc124f5b2008-04-14 19:13:29 +03001903 fraction = 16;
Christoph Lameter5e6d4442007-05-09 02:32:46 -07001904 while (fraction >= 4) {
1905 order = slab_order(size, min_objects,
1906 slub_max_order, fraction);
1907 if (order <= slub_max_order)
1908 return order;
1909 fraction /= 2;
1910 }
Zhang Yanmine8120ff2009-02-12 18:00:17 +02001911 min_objects --;
Christoph Lameter5e6d4442007-05-09 02:32:46 -07001912 }
1913
1914 /*
1915 * We were unable to place multiple objects in a slab. Now
1916 * lets see if we can place a single object there.
1917 */
1918 order = slab_order(size, 1, slub_max_order, 1);
1919 if (order <= slub_max_order)
1920 return order;
1921
1922 /*
1923 * Doh this slab cannot be placed using slub_max_order.
1924 */
1925 order = slab_order(size, 1, MAX_ORDER, 1);
David Rientjes818cf592009-04-23 09:58:22 +03001926 if (order < MAX_ORDER)
Christoph Lameter5e6d4442007-05-09 02:32:46 -07001927 return order;
1928 return -ENOSYS;
1929}
1930
Christoph Lameter81819f02007-05-06 14:49:36 -07001931/*
Christoph Lameter672bba32007-05-09 02:32:39 -07001932 * Figure out what the alignment of the objects will be.
Christoph Lameter81819f02007-05-06 14:49:36 -07001933 */
1934static unsigned long calculate_alignment(unsigned long flags,
1935 unsigned long align, unsigned long size)
1936{
1937 /*
Christoph Lameter6446faa2008-02-15 23:45:26 -08001938 * If the user wants hardware cache aligned objects then follow that
1939 * suggestion if the object is sufficiently large.
Christoph Lameter81819f02007-05-06 14:49:36 -07001940 *
Christoph Lameter6446faa2008-02-15 23:45:26 -08001941 * The hardware cache alignment cannot override the specified
1942 * alignment though. If that is greater then use it.
Christoph Lameter81819f02007-05-06 14:49:36 -07001943 */
Nick Pigginb6210382008-03-05 14:05:56 -08001944 if (flags & SLAB_HWCACHE_ALIGN) {
1945 unsigned long ralign = cache_line_size();
1946 while (size <= ralign / 2)
1947 ralign /= 2;
1948 align = max(align, ralign);
1949 }
Christoph Lameter81819f02007-05-06 14:49:36 -07001950
1951 if (align < ARCH_SLAB_MINALIGN)
Nick Pigginb6210382008-03-05 14:05:56 -08001952 align = ARCH_SLAB_MINALIGN;
Christoph Lameter81819f02007-05-06 14:49:36 -07001953
1954 return ALIGN(align, sizeof(void *));
1955}
1956
Christoph Lameterdfb4f092007-10-16 01:26:05 -07001957static void init_kmem_cache_cpu(struct kmem_cache *s,
1958 struct kmem_cache_cpu *c)
1959{
1960 c->page = NULL;
Christoph Lametera973e9d2008-03-01 13:40:44 -08001961 c->freelist = NULL;
Christoph Lameterdfb4f092007-10-16 01:26:05 -07001962 c->node = 0;
Christoph Lameter42a9fdb2007-10-16 01:26:09 -07001963 c->offset = s->offset / sizeof(void *);
1964 c->objsize = s->objsize;
Pekka Enberg62f75532008-04-14 18:50:44 +03001965#ifdef CONFIG_SLUB_STATS
1966 memset(c->stat, 0, NR_SLUB_STAT_ITEMS * sizeof(unsigned));
1967#endif
Christoph Lameterdfb4f092007-10-16 01:26:05 -07001968}
1969
Pekka Enberg5595cff2008-08-05 09:28:47 +03001970static void
1971init_kmem_cache_node(struct kmem_cache_node *n, struct kmem_cache *s)
Christoph Lameter81819f02007-05-06 14:49:36 -07001972{
1973 n->nr_partial = 0;
Christoph Lameter81819f02007-05-06 14:49:36 -07001974 spin_lock_init(&n->list_lock);
1975 INIT_LIST_HEAD(&n->partial);
Christoph Lameter8ab13722007-07-17 04:03:32 -07001976#ifdef CONFIG_SLUB_DEBUG
Christoph Lameter0f389ec2008-04-14 18:53:02 +03001977 atomic_long_set(&n->nr_slabs, 0);
Salman Qazi02b71b72008-09-11 12:25:41 -07001978 atomic_long_set(&n->total_objects, 0);
Christoph Lameter643b1132007-05-06 14:49:42 -07001979 INIT_LIST_HEAD(&n->full);
Christoph Lameter8ab13722007-07-17 04:03:32 -07001980#endif
Christoph Lameter81819f02007-05-06 14:49:36 -07001981}
1982
Christoph Lameter4c93c3552007-10-16 01:26:08 -07001983#ifdef CONFIG_SMP
1984/*
1985 * Per cpu array for per cpu structures.
1986 *
1987 * The per cpu array places all kmem_cache_cpu structures from one processor
1988 * close together meaning that it becomes possible that multiple per cpu
1989 * structures are contained in one cacheline. This may be particularly
1990 * beneficial for the kmalloc caches.
1991 *
1992 * A desktop system typically has around 60-80 slabs. With 100 here we are
1993 * likely able to get per cpu structures for all caches from the array defined
1994 * here. We must be able to cover all kmalloc caches during bootstrap.
1995 *
1996 * If the per cpu array is exhausted then fall back to kmalloc
1997 * of individual cachelines. No sharing is possible then.
1998 */
1999#define NR_KMEM_CACHE_CPU 100
2000
2001static DEFINE_PER_CPU(struct kmem_cache_cpu,
2002 kmem_cache_cpu)[NR_KMEM_CACHE_CPU];
2003
2004static DEFINE_PER_CPU(struct kmem_cache_cpu *, kmem_cache_cpu_free);
Rusty Russell174596a2009-01-01 10:12:29 +10302005static DECLARE_BITMAP(kmem_cach_cpu_free_init_once, CONFIG_NR_CPUS);
Christoph Lameter4c93c3552007-10-16 01:26:08 -07002006
2007static struct kmem_cache_cpu *alloc_kmem_cache_cpu(struct kmem_cache *s,
2008 int cpu, gfp_t flags)
2009{
2010 struct kmem_cache_cpu *c = per_cpu(kmem_cache_cpu_free, cpu);
2011
2012 if (c)
2013 per_cpu(kmem_cache_cpu_free, cpu) =
2014 (void *)c->freelist;
2015 else {
2016 /* Table overflow: So allocate ourselves */
2017 c = kmalloc_node(
2018 ALIGN(sizeof(struct kmem_cache_cpu), cache_line_size()),
2019 flags, cpu_to_node(cpu));
2020 if (!c)
2021 return NULL;
2022 }
2023
2024 init_kmem_cache_cpu(s, c);
2025 return c;
2026}
2027
2028static void free_kmem_cache_cpu(struct kmem_cache_cpu *c, int cpu)
2029{
2030 if (c < per_cpu(kmem_cache_cpu, cpu) ||
David Rientjes37189092009-01-27 18:59:46 -08002031 c >= per_cpu(kmem_cache_cpu, cpu) + NR_KMEM_CACHE_CPU) {
Christoph Lameter4c93c3552007-10-16 01:26:08 -07002032 kfree(c);
2033 return;
2034 }
2035 c->freelist = (void *)per_cpu(kmem_cache_cpu_free, cpu);
2036 per_cpu(kmem_cache_cpu_free, cpu) = c;
2037}
2038
2039static void free_kmem_cache_cpus(struct kmem_cache *s)
2040{
2041 int cpu;
2042
2043 for_each_online_cpu(cpu) {
2044 struct kmem_cache_cpu *c = get_cpu_slab(s, cpu);
2045
2046 if (c) {
2047 s->cpu_slab[cpu] = NULL;
2048 free_kmem_cache_cpu(c, cpu);
2049 }
2050 }
2051}
2052
2053static int alloc_kmem_cache_cpus(struct kmem_cache *s, gfp_t flags)
2054{
2055 int cpu;
2056
2057 for_each_online_cpu(cpu) {
2058 struct kmem_cache_cpu *c = get_cpu_slab(s, cpu);
2059
2060 if (c)
2061 continue;
2062
2063 c = alloc_kmem_cache_cpu(s, cpu, flags);
2064 if (!c) {
2065 free_kmem_cache_cpus(s);
2066 return 0;
2067 }
2068 s->cpu_slab[cpu] = c;
2069 }
2070 return 1;
2071}
2072
2073/*
2074 * Initialize the per cpu array.
2075 */
2076static void init_alloc_cpu_cpu(int cpu)
2077{
2078 int i;
2079
Rusty Russell174596a2009-01-01 10:12:29 +10302080 if (cpumask_test_cpu(cpu, to_cpumask(kmem_cach_cpu_free_init_once)))
Christoph Lameter4c93c3552007-10-16 01:26:08 -07002081 return;
2082
2083 for (i = NR_KMEM_CACHE_CPU - 1; i >= 0; i--)
2084 free_kmem_cache_cpu(&per_cpu(kmem_cache_cpu, cpu)[i], cpu);
2085
Rusty Russell174596a2009-01-01 10:12:29 +10302086 cpumask_set_cpu(cpu, to_cpumask(kmem_cach_cpu_free_init_once));
Christoph Lameter4c93c3552007-10-16 01:26:08 -07002087}
2088
2089static void __init init_alloc_cpu(void)
2090{
2091 int cpu;
2092
2093 for_each_online_cpu(cpu)
2094 init_alloc_cpu_cpu(cpu);
2095 }
2096
2097#else
2098static inline void free_kmem_cache_cpus(struct kmem_cache *s) {}
2099static inline void init_alloc_cpu(void) {}
2100
2101static inline int alloc_kmem_cache_cpus(struct kmem_cache *s, gfp_t flags)
2102{
2103 init_kmem_cache_cpu(s, &s->cpu_slab);
2104 return 1;
2105}
2106#endif
2107
Christoph Lameter81819f02007-05-06 14:49:36 -07002108#ifdef CONFIG_NUMA
2109/*
2110 * No kmalloc_node yet so do it by hand. We know that this is the first
2111 * slab on the node for this slabcache. There are no concurrent accesses
2112 * possible.
2113 *
2114 * Note that this function only works on the kmalloc_node_cache
Christoph Lameter4c93c3552007-10-16 01:26:08 -07002115 * when allocating for the kmalloc_node_cache. This is used for bootstrapping
2116 * memory on a fresh node that has no slab structures yet.
Christoph Lameter81819f02007-05-06 14:49:36 -07002117 */
David Rientjes0094de92008-11-25 19:14:19 -08002118static void early_kmem_cache_node_alloc(gfp_t gfpflags, int node)
Christoph Lameter81819f02007-05-06 14:49:36 -07002119{
2120 struct page *page;
2121 struct kmem_cache_node *n;
rootba84c732008-01-07 23:20:28 -08002122 unsigned long flags;
Christoph Lameter81819f02007-05-06 14:49:36 -07002123
2124 BUG_ON(kmalloc_caches->size < sizeof(struct kmem_cache_node));
2125
Christoph Lametera2f92ee2007-08-22 14:01:57 -07002126 page = new_slab(kmalloc_caches, gfpflags, node);
Christoph Lameter81819f02007-05-06 14:49:36 -07002127
2128 BUG_ON(!page);
Christoph Lametera2f92ee2007-08-22 14:01:57 -07002129 if (page_to_nid(page) != node) {
2130 printk(KERN_ERR "SLUB: Unable to allocate memory from "
2131 "node %d\n", node);
2132 printk(KERN_ERR "SLUB: Allocating a useless per node structure "
2133 "in order to be able to continue\n");
2134 }
2135
Christoph Lameter81819f02007-05-06 14:49:36 -07002136 n = page->freelist;
2137 BUG_ON(!n);
2138 page->freelist = get_freepointer(kmalloc_caches, n);
2139 page->inuse++;
2140 kmalloc_caches->node[node] = n;
Christoph Lameter8ab13722007-07-17 04:03:32 -07002141#ifdef CONFIG_SLUB_DEBUG
Christoph Lameterd45f39c2007-07-17 04:03:21 -07002142 init_object(kmalloc_caches, n, 1);
2143 init_tracking(kmalloc_caches, n);
Christoph Lameter8ab13722007-07-17 04:03:32 -07002144#endif
Pekka Enberg5595cff2008-08-05 09:28:47 +03002145 init_kmem_cache_node(n, kmalloc_caches);
Christoph Lameter205ab992008-04-14 19:11:40 +03002146 inc_slabs_node(kmalloc_caches, node, page->objects);
Christoph Lameter6446faa2008-02-15 23:45:26 -08002147
rootba84c732008-01-07 23:20:28 -08002148 /*
2149 * lockdep requires consistent irq usage for each lock
2150 * so even though there cannot be a race this early in
2151 * the boot sequence, we still disable irqs.
2152 */
2153 local_irq_save(flags);
Christoph Lameter7c2e1322008-01-07 23:20:27 -08002154 add_partial(n, page, 0);
rootba84c732008-01-07 23:20:28 -08002155 local_irq_restore(flags);
Christoph Lameter81819f02007-05-06 14:49:36 -07002156}
2157
2158static void free_kmem_cache_nodes(struct kmem_cache *s)
2159{
2160 int node;
2161
Christoph Lameterf64dc582007-10-16 01:25:33 -07002162 for_each_node_state(node, N_NORMAL_MEMORY) {
Christoph Lameter81819f02007-05-06 14:49:36 -07002163 struct kmem_cache_node *n = s->node[node];
2164 if (n && n != &s->local_node)
2165 kmem_cache_free(kmalloc_caches, n);
2166 s->node[node] = NULL;
2167 }
2168}
2169
2170static int init_kmem_cache_nodes(struct kmem_cache *s, gfp_t gfpflags)
2171{
2172 int node;
2173 int local_node;
2174
2175 if (slab_state >= UP)
2176 local_node = page_to_nid(virt_to_page(s));
2177 else
2178 local_node = 0;
2179
Christoph Lameterf64dc582007-10-16 01:25:33 -07002180 for_each_node_state(node, N_NORMAL_MEMORY) {
Christoph Lameter81819f02007-05-06 14:49:36 -07002181 struct kmem_cache_node *n;
2182
2183 if (local_node == node)
2184 n = &s->local_node;
2185 else {
2186 if (slab_state == DOWN) {
David Rientjes0094de92008-11-25 19:14:19 -08002187 early_kmem_cache_node_alloc(gfpflags, node);
Christoph Lameter81819f02007-05-06 14:49:36 -07002188 continue;
2189 }
2190 n = kmem_cache_alloc_node(kmalloc_caches,
2191 gfpflags, node);
2192
2193 if (!n) {
2194 free_kmem_cache_nodes(s);
2195 return 0;
2196 }
2197
2198 }
2199 s->node[node] = n;
Pekka Enberg5595cff2008-08-05 09:28:47 +03002200 init_kmem_cache_node(n, s);
Christoph Lameter81819f02007-05-06 14:49:36 -07002201 }
2202 return 1;
2203}
2204#else
2205static void free_kmem_cache_nodes(struct kmem_cache *s)
2206{
2207}
2208
2209static int init_kmem_cache_nodes(struct kmem_cache *s, gfp_t gfpflags)
2210{
Pekka Enberg5595cff2008-08-05 09:28:47 +03002211 init_kmem_cache_node(&s->local_node, s);
Christoph Lameter81819f02007-05-06 14:49:36 -07002212 return 1;
2213}
2214#endif
2215
David Rientjesc0bdb232009-02-25 09:16:35 +02002216static void set_min_partial(struct kmem_cache *s, unsigned long min)
David Rientjes3b89d7d2009-02-22 17:40:07 -08002217{
2218 if (min < MIN_PARTIAL)
2219 min = MIN_PARTIAL;
2220 else if (min > MAX_PARTIAL)
2221 min = MAX_PARTIAL;
2222 s->min_partial = min;
2223}
2224
Christoph Lameter81819f02007-05-06 14:49:36 -07002225/*
2226 * calculate_sizes() determines the order and the distribution of data within
2227 * a slab object.
2228 */
Christoph Lameter06b285d2008-04-14 19:11:41 +03002229static int calculate_sizes(struct kmem_cache *s, int forced_order)
Christoph Lameter81819f02007-05-06 14:49:36 -07002230{
2231 unsigned long flags = s->flags;
2232 unsigned long size = s->objsize;
2233 unsigned long align = s->align;
Christoph Lameter834f3d12008-04-14 19:11:31 +03002234 int order;
Christoph Lameter81819f02007-05-06 14:49:36 -07002235
2236 /*
Christoph Lameterd8b42bf2008-02-15 23:45:25 -08002237 * Round up object size to the next word boundary. We can only
2238 * place the free pointer at word boundaries and this determines
2239 * the possible location of the free pointer.
2240 */
2241 size = ALIGN(size, sizeof(void *));
2242
2243#ifdef CONFIG_SLUB_DEBUG
2244 /*
Christoph Lameter81819f02007-05-06 14:49:36 -07002245 * Determine if we can poison the object itself. If the user of
2246 * the slab may touch the object after free or before allocation
2247 * then we should never poison the object itself.
2248 */
2249 if ((flags & SLAB_POISON) && !(flags & SLAB_DESTROY_BY_RCU) &&
Christoph Lameterc59def9f2007-05-16 22:10:50 -07002250 !s->ctor)
Christoph Lameter81819f02007-05-06 14:49:36 -07002251 s->flags |= __OBJECT_POISON;
2252 else
2253 s->flags &= ~__OBJECT_POISON;
2254
Christoph Lameter81819f02007-05-06 14:49:36 -07002255
2256 /*
Christoph Lameter672bba32007-05-09 02:32:39 -07002257 * If we are Redzoning then check if there is some space between the
Christoph Lameter81819f02007-05-06 14:49:36 -07002258 * end of the object and the free pointer. If not then add an
Christoph Lameter672bba32007-05-09 02:32:39 -07002259 * additional word to have some bytes to store Redzone information.
Christoph Lameter81819f02007-05-06 14:49:36 -07002260 */
2261 if ((flags & SLAB_RED_ZONE) && size == s->objsize)
2262 size += sizeof(void *);
Christoph Lameter41ecc552007-05-09 02:32:44 -07002263#endif
Christoph Lameter81819f02007-05-06 14:49:36 -07002264
2265 /*
Christoph Lameter672bba32007-05-09 02:32:39 -07002266 * With that we have determined the number of bytes in actual use
2267 * by the object. This is the potential offset to the free pointer.
Christoph Lameter81819f02007-05-06 14:49:36 -07002268 */
2269 s->inuse = size;
2270
2271 if (((flags & (SLAB_DESTROY_BY_RCU | SLAB_POISON)) ||
Christoph Lameterc59def9f2007-05-16 22:10:50 -07002272 s->ctor)) {
Christoph Lameter81819f02007-05-06 14:49:36 -07002273 /*
2274 * Relocate free pointer after the object if it is not
2275 * permitted to overwrite the first word of the object on
2276 * kmem_cache_free.
2277 *
2278 * This is the case if we do RCU, have a constructor or
2279 * destructor or are poisoning the objects.
2280 */
2281 s->offset = size;
2282 size += sizeof(void *);
2283 }
2284
Christoph Lameterc12b3c62007-05-23 13:57:31 -07002285#ifdef CONFIG_SLUB_DEBUG
Christoph Lameter81819f02007-05-06 14:49:36 -07002286 if (flags & SLAB_STORE_USER)
2287 /*
2288 * Need to store information about allocs and frees after
2289 * the object.
2290 */
2291 size += 2 * sizeof(struct track);
2292
Christoph Lameterbe7b3fb2007-05-09 02:32:36 -07002293 if (flags & SLAB_RED_ZONE)
Christoph Lameter81819f02007-05-06 14:49:36 -07002294 /*
2295 * Add some empty padding so that we can catch
2296 * overwrites from earlier objects rather than let
2297 * tracking information or the free pointer be
Frederik Schwarzer0211a9c2008-12-29 22:14:56 +01002298 * corrupted if a user writes before the start
Christoph Lameter81819f02007-05-06 14:49:36 -07002299 * of the object.
2300 */
2301 size += sizeof(void *);
Christoph Lameter41ecc552007-05-09 02:32:44 -07002302#endif
Christoph Lameter672bba32007-05-09 02:32:39 -07002303
Christoph Lameter81819f02007-05-06 14:49:36 -07002304 /*
2305 * Determine the alignment based on various parameters that the
Christoph Lameter65c02d42007-05-09 02:32:35 -07002306 * user specified and the dynamic determination of cache line size
2307 * on bootup.
Christoph Lameter81819f02007-05-06 14:49:36 -07002308 */
2309 align = calculate_alignment(flags, align, s->objsize);
2310
2311 /*
2312 * SLUB stores one object immediately after another beginning from
2313 * offset 0. In order to align the objects we have to simply size
2314 * each object to conform to the alignment.
2315 */
2316 size = ALIGN(size, align);
2317 s->size = size;
Christoph Lameter06b285d2008-04-14 19:11:41 +03002318 if (forced_order >= 0)
2319 order = forced_order;
2320 else
2321 order = calculate_order(size);
Christoph Lameter81819f02007-05-06 14:49:36 -07002322
Christoph Lameter834f3d12008-04-14 19:11:31 +03002323 if (order < 0)
Christoph Lameter81819f02007-05-06 14:49:36 -07002324 return 0;
2325
Christoph Lameterb7a49f02008-02-14 14:21:32 -08002326 s->allocflags = 0;
Christoph Lameter834f3d12008-04-14 19:11:31 +03002327 if (order)
Christoph Lameterb7a49f02008-02-14 14:21:32 -08002328 s->allocflags |= __GFP_COMP;
2329
2330 if (s->flags & SLAB_CACHE_DMA)
2331 s->allocflags |= SLUB_DMA;
2332
2333 if (s->flags & SLAB_RECLAIM_ACCOUNT)
2334 s->allocflags |= __GFP_RECLAIMABLE;
2335
Christoph Lameter81819f02007-05-06 14:49:36 -07002336 /*
2337 * Determine the number of objects per slab
2338 */
Christoph Lameter834f3d12008-04-14 19:11:31 +03002339 s->oo = oo_make(order, size);
Christoph Lameter65c33762008-04-14 19:11:40 +03002340 s->min = oo_make(get_order(size), size);
Christoph Lameter205ab992008-04-14 19:11:40 +03002341 if (oo_objects(s->oo) > oo_objects(s->max))
2342 s->max = s->oo;
Christoph Lameter81819f02007-05-06 14:49:36 -07002343
Christoph Lameter834f3d12008-04-14 19:11:31 +03002344 return !!oo_objects(s->oo);
Christoph Lameter81819f02007-05-06 14:49:36 -07002345
2346}
2347
Christoph Lameter81819f02007-05-06 14:49:36 -07002348static int kmem_cache_open(struct kmem_cache *s, gfp_t gfpflags,
2349 const char *name, size_t size,
2350 size_t align, unsigned long flags,
Alexey Dobriyan51cc5062008-07-25 19:45:34 -07002351 void (*ctor)(void *))
Christoph Lameter81819f02007-05-06 14:49:36 -07002352{
2353 memset(s, 0, kmem_size);
2354 s->name = name;
2355 s->ctor = ctor;
Christoph Lameter81819f02007-05-06 14:49:36 -07002356 s->objsize = size;
Christoph Lameter81819f02007-05-06 14:49:36 -07002357 s->align = align;
Christoph Lameterba0268a2007-09-11 15:24:11 -07002358 s->flags = kmem_cache_flags(size, flags, name, ctor);
Christoph Lameter81819f02007-05-06 14:49:36 -07002359
Christoph Lameter06b285d2008-04-14 19:11:41 +03002360 if (!calculate_sizes(s, -1))
Christoph Lameter81819f02007-05-06 14:49:36 -07002361 goto error;
2362
David Rientjes3b89d7d2009-02-22 17:40:07 -08002363 /*
2364 * The larger the object size is, the more pages we want on the partial
2365 * list to avoid pounding the page allocator excessively.
2366 */
David Rientjesc0bdb232009-02-25 09:16:35 +02002367 set_min_partial(s, ilog2(s->size));
Christoph Lameter81819f02007-05-06 14:49:36 -07002368 s->refcount = 1;
2369#ifdef CONFIG_NUMA
Christoph Lametere2cb96b2008-08-19 08:51:22 -05002370 s->remote_node_defrag_ratio = 1000;
Christoph Lameter81819f02007-05-06 14:49:36 -07002371#endif
Christoph Lameterdfb4f092007-10-16 01:26:05 -07002372 if (!init_kmem_cache_nodes(s, gfpflags & ~SLUB_DMA))
2373 goto error;
Christoph Lameter81819f02007-05-06 14:49:36 -07002374
Christoph Lameterdfb4f092007-10-16 01:26:05 -07002375 if (alloc_kmem_cache_cpus(s, gfpflags & ~SLUB_DMA))
Christoph Lameter81819f02007-05-06 14:49:36 -07002376 return 1;
Christoph Lameter4c93c3552007-10-16 01:26:08 -07002377 free_kmem_cache_nodes(s);
Christoph Lameter81819f02007-05-06 14:49:36 -07002378error:
2379 if (flags & SLAB_PANIC)
2380 panic("Cannot create slab %s size=%lu realsize=%u "
2381 "order=%u offset=%u flags=%lx\n",
Christoph Lameter834f3d12008-04-14 19:11:31 +03002382 s->name, (unsigned long)size, s->size, oo_order(s->oo),
Christoph Lameter81819f02007-05-06 14:49:36 -07002383 s->offset, flags);
2384 return 0;
2385}
Christoph Lameter81819f02007-05-06 14:49:36 -07002386
2387/*
2388 * Check if a given pointer is valid
2389 */
2390int kmem_ptr_validate(struct kmem_cache *s, const void *object)
2391{
Pekka Enberg06428782008-01-07 23:20:27 -08002392 struct page *page;
Christoph Lameter81819f02007-05-06 14:49:36 -07002393
2394 page = get_object_page(object);
2395
2396 if (!page || s != page->slab)
2397 /* No slab or wrong slab */
2398 return 0;
2399
Christoph Lameterabcd08a2007-05-09 02:32:37 -07002400 if (!check_valid_pointer(s, page, object))
Christoph Lameter81819f02007-05-06 14:49:36 -07002401 return 0;
2402
2403 /*
2404 * We could also check if the object is on the slabs freelist.
2405 * But this would be too expensive and it seems that the main
Christoph Lameter6446faa2008-02-15 23:45:26 -08002406 * purpose of kmem_ptr_valid() is to check if the object belongs
Christoph Lameter81819f02007-05-06 14:49:36 -07002407 * to a certain slab.
2408 */
2409 return 1;
2410}
2411EXPORT_SYMBOL(kmem_ptr_validate);
2412
2413/*
2414 * Determine the size of a slab object
2415 */
2416unsigned int kmem_cache_size(struct kmem_cache *s)
2417{
2418 return s->objsize;
2419}
2420EXPORT_SYMBOL(kmem_cache_size);
2421
2422const char *kmem_cache_name(struct kmem_cache *s)
2423{
2424 return s->name;
2425}
2426EXPORT_SYMBOL(kmem_cache_name);
2427
Christoph Lameter33b12c32008-04-25 12:22:43 -07002428static void list_slab_objects(struct kmem_cache *s, struct page *page,
2429 const char *text)
Christoph Lameter81819f02007-05-06 14:49:36 -07002430{
Christoph Lameter33b12c32008-04-25 12:22:43 -07002431#ifdef CONFIG_SLUB_DEBUG
2432 void *addr = page_address(page);
2433 void *p;
2434 DECLARE_BITMAP(map, page->objects);
2435
2436 bitmap_zero(map, page->objects);
2437 slab_err(s, page, "%s", text);
2438 slab_lock(page);
2439 for_each_free_object(p, s, page->freelist)
2440 set_bit(slab_index(p, s, addr), map);
2441
2442 for_each_object(p, s, addr, page->objects) {
2443
2444 if (!test_bit(slab_index(p, s, addr), map)) {
2445 printk(KERN_ERR "INFO: Object 0x%p @offset=%tu\n",
2446 p, p - addr);
2447 print_tracking(s, p);
2448 }
2449 }
2450 slab_unlock(page);
2451#endif
2452}
2453
Christoph Lameter81819f02007-05-06 14:49:36 -07002454/*
Christoph Lameter599870b2008-04-23 12:36:52 -07002455 * Attempt to free all partial slabs on a node.
Christoph Lameter81819f02007-05-06 14:49:36 -07002456 */
Christoph Lameter599870b2008-04-23 12:36:52 -07002457static void free_partial(struct kmem_cache *s, struct kmem_cache_node *n)
Christoph Lameter81819f02007-05-06 14:49:36 -07002458{
Christoph Lameter81819f02007-05-06 14:49:36 -07002459 unsigned long flags;
2460 struct page *page, *h;
2461
2462 spin_lock_irqsave(&n->list_lock, flags);
Christoph Lameter33b12c32008-04-25 12:22:43 -07002463 list_for_each_entry_safe(page, h, &n->partial, lru) {
Christoph Lameter81819f02007-05-06 14:49:36 -07002464 if (!page->inuse) {
2465 list_del(&page->lru);
2466 discard_slab(s, page);
Christoph Lameter599870b2008-04-23 12:36:52 -07002467 n->nr_partial--;
Christoph Lameter33b12c32008-04-25 12:22:43 -07002468 } else {
2469 list_slab_objects(s, page,
2470 "Objects remaining on kmem_cache_close()");
Christoph Lameter599870b2008-04-23 12:36:52 -07002471 }
Christoph Lameter33b12c32008-04-25 12:22:43 -07002472 }
Christoph Lameter81819f02007-05-06 14:49:36 -07002473 spin_unlock_irqrestore(&n->list_lock, flags);
Christoph Lameter81819f02007-05-06 14:49:36 -07002474}
2475
2476/*
Christoph Lameter672bba32007-05-09 02:32:39 -07002477 * Release all resources used by a slab cache.
Christoph Lameter81819f02007-05-06 14:49:36 -07002478 */
Christoph Lameter0c710012007-07-17 04:03:24 -07002479static inline int kmem_cache_close(struct kmem_cache *s)
Christoph Lameter81819f02007-05-06 14:49:36 -07002480{
2481 int node;
2482
2483 flush_all(s);
2484
2485 /* Attempt to free all objects */
Christoph Lameter4c93c3552007-10-16 01:26:08 -07002486 free_kmem_cache_cpus(s);
Christoph Lameterf64dc582007-10-16 01:25:33 -07002487 for_each_node_state(node, N_NORMAL_MEMORY) {
Christoph Lameter81819f02007-05-06 14:49:36 -07002488 struct kmem_cache_node *n = get_node(s, node);
2489
Christoph Lameter599870b2008-04-23 12:36:52 -07002490 free_partial(s, n);
2491 if (n->nr_partial || slabs_node(s, node))
Christoph Lameter81819f02007-05-06 14:49:36 -07002492 return 1;
2493 }
2494 free_kmem_cache_nodes(s);
2495 return 0;
2496}
2497
2498/*
2499 * Close a cache and release the kmem_cache structure
2500 * (must be used for caches created using kmem_cache_create)
2501 */
2502void kmem_cache_destroy(struct kmem_cache *s)
2503{
2504 down_write(&slub_lock);
2505 s->refcount--;
2506 if (!s->refcount) {
2507 list_del(&s->list);
Christoph Lametera0e1d1b2007-07-17 04:03:31 -07002508 up_write(&slub_lock);
Pekka Enbergd629d812008-04-23 22:31:08 +03002509 if (kmem_cache_close(s)) {
2510 printk(KERN_ERR "SLUB %s: %s called for cache that "
2511 "still has objects.\n", s->name, __func__);
2512 dump_stack();
2513 }
Christoph Lameter81819f02007-05-06 14:49:36 -07002514 sysfs_slab_remove(s);
Christoph Lametera0e1d1b2007-07-17 04:03:31 -07002515 } else
2516 up_write(&slub_lock);
Christoph Lameter81819f02007-05-06 14:49:36 -07002517}
2518EXPORT_SYMBOL(kmem_cache_destroy);
2519
2520/********************************************************************
2521 * Kmalloc subsystem
2522 *******************************************************************/
2523
Christoph Lameterffadd4d2009-02-17 12:05:07 -05002524struct kmem_cache kmalloc_caches[SLUB_PAGE_SHIFT] __cacheline_aligned;
Christoph Lameter81819f02007-05-06 14:49:36 -07002525EXPORT_SYMBOL(kmalloc_caches);
2526
Christoph Lameter81819f02007-05-06 14:49:36 -07002527static int __init setup_slub_min_order(char *str)
2528{
Pekka Enberg06428782008-01-07 23:20:27 -08002529 get_option(&str, &slub_min_order);
Christoph Lameter81819f02007-05-06 14:49:36 -07002530
2531 return 1;
2532}
2533
2534__setup("slub_min_order=", setup_slub_min_order);
2535
2536static int __init setup_slub_max_order(char *str)
2537{
Pekka Enberg06428782008-01-07 23:20:27 -08002538 get_option(&str, &slub_max_order);
David Rientjes818cf592009-04-23 09:58:22 +03002539 slub_max_order = min(slub_max_order, MAX_ORDER - 1);
Christoph Lameter81819f02007-05-06 14:49:36 -07002540
2541 return 1;
2542}
2543
2544__setup("slub_max_order=", setup_slub_max_order);
2545
2546static int __init setup_slub_min_objects(char *str)
2547{
Pekka Enberg06428782008-01-07 23:20:27 -08002548 get_option(&str, &slub_min_objects);
Christoph Lameter81819f02007-05-06 14:49:36 -07002549
2550 return 1;
2551}
2552
2553__setup("slub_min_objects=", setup_slub_min_objects);
2554
2555static int __init setup_slub_nomerge(char *str)
2556{
2557 slub_nomerge = 1;
2558 return 1;
2559}
2560
2561__setup("slub_nomerge", setup_slub_nomerge);
2562
Christoph Lameter81819f02007-05-06 14:49:36 -07002563static struct kmem_cache *create_kmalloc_cache(struct kmem_cache *s,
2564 const char *name, int size, gfp_t gfp_flags)
2565{
2566 unsigned int flags = 0;
2567
2568 if (gfp_flags & SLUB_DMA)
2569 flags = SLAB_CACHE_DMA;
2570
Pekka Enberg83b519e2009-06-10 19:40:04 +03002571 /*
2572 * This function is called with IRQs disabled during early-boot on
2573 * single CPU so there's no need to take slub_lock here.
2574 */
Christoph Lameter81819f02007-05-06 14:49:36 -07002575 if (!kmem_cache_open(s, gfp_flags, name, size, ARCH_KMALLOC_MINALIGN,
Christoph Lameter319d1e22008-04-14 19:11:41 +03002576 flags, NULL))
Christoph Lameter81819f02007-05-06 14:49:36 -07002577 goto panic;
2578
2579 list_add(&s->list, &slab_caches);
Pekka Enberg83b519e2009-06-10 19:40:04 +03002580
Christoph Lameter81819f02007-05-06 14:49:36 -07002581 if (sysfs_slab_add(s))
2582 goto panic;
2583 return s;
2584
2585panic:
2586 panic("Creation of kmalloc slab %s size=%d failed.\n", name, size);
2587}
2588
Christoph Lameter2e443fd2007-07-17 04:03:24 -07002589#ifdef CONFIG_ZONE_DMA
Christoph Lameterffadd4d2009-02-17 12:05:07 -05002590static struct kmem_cache *kmalloc_caches_dma[SLUB_PAGE_SHIFT];
Christoph Lameter1ceef402007-08-07 15:11:48 -07002591
2592static void sysfs_add_func(struct work_struct *w)
2593{
2594 struct kmem_cache *s;
2595
2596 down_write(&slub_lock);
2597 list_for_each_entry(s, &slab_caches, list) {
2598 if (s->flags & __SYSFS_ADD_DEFERRED) {
2599 s->flags &= ~__SYSFS_ADD_DEFERRED;
2600 sysfs_slab_add(s);
2601 }
2602 }
2603 up_write(&slub_lock);
2604}
2605
2606static DECLARE_WORK(sysfs_add_work, sysfs_add_func);
2607
Christoph Lameter2e443fd2007-07-17 04:03:24 -07002608static noinline struct kmem_cache *dma_kmalloc_cache(int index, gfp_t flags)
2609{
2610 struct kmem_cache *s;
Christoph Lameter2e443fd2007-07-17 04:03:24 -07002611 char *text;
2612 size_t realsize;
2613
2614 s = kmalloc_caches_dma[index];
2615 if (s)
2616 return s;
2617
2618 /* Dynamically create dma cache */
Christoph Lameter1ceef402007-08-07 15:11:48 -07002619 if (flags & __GFP_WAIT)
2620 down_write(&slub_lock);
2621 else {
2622 if (!down_write_trylock(&slub_lock))
2623 goto out;
2624 }
2625
2626 if (kmalloc_caches_dma[index])
2627 goto unlock_out;
Christoph Lameter2e443fd2007-07-17 04:03:24 -07002628
Christoph Lameter7b55f622007-07-17 04:03:27 -07002629 realsize = kmalloc_caches[index].objsize;
Ingo Molnar3adbefe2008-02-05 17:57:39 -08002630 text = kasprintf(flags & ~SLUB_DMA, "kmalloc_dma-%d",
2631 (unsigned int)realsize);
Christoph Lameter1ceef402007-08-07 15:11:48 -07002632 s = kmalloc(kmem_size, flags & ~SLUB_DMA);
2633
2634 if (!s || !text || !kmem_cache_open(s, flags, text,
2635 realsize, ARCH_KMALLOC_MINALIGN,
2636 SLAB_CACHE_DMA|__SYSFS_ADD_DEFERRED, NULL)) {
2637 kfree(s);
2638 kfree(text);
2639 goto unlock_out;
Christoph Lameterdfce8642007-07-17 04:03:25 -07002640 }
Christoph Lameter1ceef402007-08-07 15:11:48 -07002641
2642 list_add(&s->list, &slab_caches);
2643 kmalloc_caches_dma[index] = s;
2644
2645 schedule_work(&sysfs_add_work);
2646
2647unlock_out:
Christoph Lameterdfce8642007-07-17 04:03:25 -07002648 up_write(&slub_lock);
Christoph Lameter1ceef402007-08-07 15:11:48 -07002649out:
Christoph Lameterdfce8642007-07-17 04:03:25 -07002650 return kmalloc_caches_dma[index];
Christoph Lameter2e443fd2007-07-17 04:03:24 -07002651}
2652#endif
2653
Christoph Lameterf1b26332007-07-17 04:03:26 -07002654/*
2655 * Conversion table for small slabs sizes / 8 to the index in the
2656 * kmalloc array. This is necessary for slabs < 192 since we have non power
2657 * of two cache sizes there. The size of larger slabs can be determined using
2658 * fls.
2659 */
2660static s8 size_index[24] = {
2661 3, /* 8 */
2662 4, /* 16 */
2663 5, /* 24 */
2664 5, /* 32 */
2665 6, /* 40 */
2666 6, /* 48 */
2667 6, /* 56 */
2668 6, /* 64 */
2669 1, /* 72 */
2670 1, /* 80 */
2671 1, /* 88 */
2672 1, /* 96 */
2673 7, /* 104 */
2674 7, /* 112 */
2675 7, /* 120 */
2676 7, /* 128 */
2677 2, /* 136 */
2678 2, /* 144 */
2679 2, /* 152 */
2680 2, /* 160 */
2681 2, /* 168 */
2682 2, /* 176 */
2683 2, /* 184 */
2684 2 /* 192 */
2685};
2686
Christoph Lameter81819f02007-05-06 14:49:36 -07002687static struct kmem_cache *get_slab(size_t size, gfp_t flags)
2688{
Christoph Lameterf1b26332007-07-17 04:03:26 -07002689 int index;
Christoph Lameter81819f02007-05-06 14:49:36 -07002690
Christoph Lameterf1b26332007-07-17 04:03:26 -07002691 if (size <= 192) {
2692 if (!size)
2693 return ZERO_SIZE_PTR;
Christoph Lameter81819f02007-05-06 14:49:36 -07002694
Christoph Lameterf1b26332007-07-17 04:03:26 -07002695 index = size_index[(size - 1) / 8];
Christoph Lameteraadb4bc2007-10-16 01:24:38 -07002696 } else
Christoph Lameterf1b26332007-07-17 04:03:26 -07002697 index = fls(size - 1);
Christoph Lameter81819f02007-05-06 14:49:36 -07002698
2699#ifdef CONFIG_ZONE_DMA
Christoph Lameterf1b26332007-07-17 04:03:26 -07002700 if (unlikely((flags & SLUB_DMA)))
Christoph Lameter2e443fd2007-07-17 04:03:24 -07002701 return dma_kmalloc_cache(index, flags);
Christoph Lameterf1b26332007-07-17 04:03:26 -07002702
Christoph Lameter81819f02007-05-06 14:49:36 -07002703#endif
2704 return &kmalloc_caches[index];
2705}
2706
2707void *__kmalloc(size_t size, gfp_t flags)
2708{
Christoph Lameteraadb4bc2007-10-16 01:24:38 -07002709 struct kmem_cache *s;
Eduard - Gabriel Munteanu5b882be2008-08-19 20:43:26 +03002710 void *ret;
Christoph Lameter81819f02007-05-06 14:49:36 -07002711
Christoph Lameterffadd4d2009-02-17 12:05:07 -05002712 if (unlikely(size > SLUB_MAX_SIZE))
Pekka Enbergeada35e2008-02-11 22:47:46 +02002713 return kmalloc_large(size, flags);
Christoph Lameteraadb4bc2007-10-16 01:24:38 -07002714
2715 s = get_slab(size, flags);
2716
2717 if (unlikely(ZERO_OR_NULL_PTR(s)))
Christoph Lameter6cb8f912007-07-17 04:03:22 -07002718 return s;
2719
Eduard - Gabriel Munteanu5b882be2008-08-19 20:43:26 +03002720 ret = slab_alloc(s, flags, -1, _RET_IP_);
2721
Eduard - Gabriel Munteanuca2b84c2009-03-23 15:12:24 +02002722 trace_kmalloc(_RET_IP_, ret, size, s->size, flags);
Eduard - Gabriel Munteanu5b882be2008-08-19 20:43:26 +03002723
2724 return ret;
Christoph Lameter81819f02007-05-06 14:49:36 -07002725}
2726EXPORT_SYMBOL(__kmalloc);
2727
Christoph Lameterf619cfe2008-03-01 13:56:40 -08002728static void *kmalloc_large_node(size_t size, gfp_t flags, int node)
2729{
2730 struct page *page = alloc_pages_node(node, flags | __GFP_COMP,
2731 get_order(size));
2732
2733 if (page)
2734 return page_address(page);
2735 else
2736 return NULL;
2737}
2738
Christoph Lameter81819f02007-05-06 14:49:36 -07002739#ifdef CONFIG_NUMA
2740void *__kmalloc_node(size_t size, gfp_t flags, int node)
2741{
Christoph Lameteraadb4bc2007-10-16 01:24:38 -07002742 struct kmem_cache *s;
Eduard - Gabriel Munteanu5b882be2008-08-19 20:43:26 +03002743 void *ret;
Christoph Lameter81819f02007-05-06 14:49:36 -07002744
Ingo Molnar057685c2009-02-20 12:15:30 +01002745 if (unlikely(size > SLUB_MAX_SIZE)) {
Eduard - Gabriel Munteanu5b882be2008-08-19 20:43:26 +03002746 ret = kmalloc_large_node(size, flags, node);
2747
Eduard - Gabriel Munteanuca2b84c2009-03-23 15:12:24 +02002748 trace_kmalloc_node(_RET_IP_, ret,
2749 size, PAGE_SIZE << get_order(size),
2750 flags, node);
Eduard - Gabriel Munteanu5b882be2008-08-19 20:43:26 +03002751
2752 return ret;
2753 }
Christoph Lameteraadb4bc2007-10-16 01:24:38 -07002754
2755 s = get_slab(size, flags);
2756
2757 if (unlikely(ZERO_OR_NULL_PTR(s)))
Christoph Lameter6cb8f912007-07-17 04:03:22 -07002758 return s;
2759
Eduard - Gabriel Munteanu5b882be2008-08-19 20:43:26 +03002760 ret = slab_alloc(s, flags, node, _RET_IP_);
2761
Eduard - Gabriel Munteanuca2b84c2009-03-23 15:12:24 +02002762 trace_kmalloc_node(_RET_IP_, ret, size, s->size, flags, node);
Eduard - Gabriel Munteanu5b882be2008-08-19 20:43:26 +03002763
2764 return ret;
Christoph Lameter81819f02007-05-06 14:49:36 -07002765}
2766EXPORT_SYMBOL(__kmalloc_node);
2767#endif
2768
2769size_t ksize(const void *object)
2770{
Christoph Lameter272c1d22007-06-08 13:46:49 -07002771 struct page *page;
Christoph Lameter81819f02007-05-06 14:49:36 -07002772 struct kmem_cache *s;
2773
Christoph Lameteref8b4522007-10-16 01:24:46 -07002774 if (unlikely(object == ZERO_SIZE_PTR))
Christoph Lameter272c1d22007-06-08 13:46:49 -07002775 return 0;
2776
Vegard Nossum294a80a2007-12-04 23:45:30 -08002777 page = virt_to_head_page(object);
Vegard Nossum294a80a2007-12-04 23:45:30 -08002778
Pekka Enberg76994412008-05-22 19:22:25 +03002779 if (unlikely(!PageSlab(page))) {
2780 WARN_ON(!PageCompound(page));
Vegard Nossum294a80a2007-12-04 23:45:30 -08002781 return PAGE_SIZE << compound_order(page);
Pekka Enberg76994412008-05-22 19:22:25 +03002782 }
Christoph Lameter81819f02007-05-06 14:49:36 -07002783 s = page->slab;
Christoph Lameter81819f02007-05-06 14:49:36 -07002784
Christoph Lameterae20bfd2008-02-15 23:45:25 -08002785#ifdef CONFIG_SLUB_DEBUG
Christoph Lameter81819f02007-05-06 14:49:36 -07002786 /*
2787 * Debugging requires use of the padding between object
2788 * and whatever may come after it.
2789 */
2790 if (s->flags & (SLAB_RED_ZONE | SLAB_POISON))
2791 return s->objsize;
2792
Christoph Lameterae20bfd2008-02-15 23:45:25 -08002793#endif
Christoph Lameter81819f02007-05-06 14:49:36 -07002794 /*
2795 * If we have the need to store the freelist pointer
2796 * back there or track user information then we can
2797 * only use the space before that information.
2798 */
2799 if (s->flags & (SLAB_DESTROY_BY_RCU | SLAB_STORE_USER))
2800 return s->inuse;
Christoph Lameter81819f02007-05-06 14:49:36 -07002801 /*
2802 * Else we can use all the padding etc for the allocation
2803 */
2804 return s->size;
2805}
Kirill A. Shutemovb1aabec2009-02-10 15:21:44 +02002806EXPORT_SYMBOL(ksize);
Christoph Lameter81819f02007-05-06 14:49:36 -07002807
2808void kfree(const void *x)
2809{
Christoph Lameter81819f02007-05-06 14:49:36 -07002810 struct page *page;
Christoph Lameter5bb983b2008-02-07 17:47:41 -08002811 void *object = (void *)x;
Christoph Lameter81819f02007-05-06 14:49:36 -07002812
Pekka Enberg2121db72009-03-25 11:05:57 +02002813 trace_kfree(_RET_IP_, x);
2814
Satyam Sharma2408c552007-10-16 01:24:44 -07002815 if (unlikely(ZERO_OR_NULL_PTR(x)))
Christoph Lameter81819f02007-05-06 14:49:36 -07002816 return;
2817
Christoph Lameterb49af682007-05-06 14:49:41 -07002818 page = virt_to_head_page(x);
Christoph Lameteraadb4bc2007-10-16 01:24:38 -07002819 if (unlikely(!PageSlab(page))) {
Christoph Lameter09375022008-05-28 10:32:22 -07002820 BUG_ON(!PageCompound(page));
Christoph Lameteraadb4bc2007-10-16 01:24:38 -07002821 put_page(page);
2822 return;
2823 }
Eduard - Gabriel Munteanuce71e272008-08-19 20:43:25 +03002824 slab_free(page->slab, page, object, _RET_IP_);
Christoph Lameter81819f02007-05-06 14:49:36 -07002825}
2826EXPORT_SYMBOL(kfree);
2827
Christoph Lameter2086d262007-05-06 14:49:46 -07002828/*
Christoph Lameter672bba32007-05-09 02:32:39 -07002829 * kmem_cache_shrink removes empty slabs from the partial lists and sorts
2830 * the remaining slabs by the number of items in use. The slabs with the
2831 * most items in use come first. New allocations will then fill those up
2832 * and thus they can be removed from the partial lists.
2833 *
2834 * The slabs with the least items are placed last. This results in them
2835 * being allocated from last increasing the chance that the last objects
2836 * are freed in them.
Christoph Lameter2086d262007-05-06 14:49:46 -07002837 */
2838int kmem_cache_shrink(struct kmem_cache *s)
2839{
2840 int node;
2841 int i;
2842 struct kmem_cache_node *n;
2843 struct page *page;
2844 struct page *t;
Christoph Lameter205ab992008-04-14 19:11:40 +03002845 int objects = oo_objects(s->max);
Christoph Lameter2086d262007-05-06 14:49:46 -07002846 struct list_head *slabs_by_inuse =
Christoph Lameter834f3d12008-04-14 19:11:31 +03002847 kmalloc(sizeof(struct list_head) * objects, GFP_KERNEL);
Christoph Lameter2086d262007-05-06 14:49:46 -07002848 unsigned long flags;
2849
2850 if (!slabs_by_inuse)
2851 return -ENOMEM;
2852
2853 flush_all(s);
Christoph Lameterf64dc582007-10-16 01:25:33 -07002854 for_each_node_state(node, N_NORMAL_MEMORY) {
Christoph Lameter2086d262007-05-06 14:49:46 -07002855 n = get_node(s, node);
2856
2857 if (!n->nr_partial)
2858 continue;
2859
Christoph Lameter834f3d12008-04-14 19:11:31 +03002860 for (i = 0; i < objects; i++)
Christoph Lameter2086d262007-05-06 14:49:46 -07002861 INIT_LIST_HEAD(slabs_by_inuse + i);
2862
2863 spin_lock_irqsave(&n->list_lock, flags);
2864
2865 /*
Christoph Lameter672bba32007-05-09 02:32:39 -07002866 * Build lists indexed by the items in use in each slab.
Christoph Lameter2086d262007-05-06 14:49:46 -07002867 *
Christoph Lameter672bba32007-05-09 02:32:39 -07002868 * Note that concurrent frees may occur while we hold the
2869 * list_lock. page->inuse here is the upper limit.
Christoph Lameter2086d262007-05-06 14:49:46 -07002870 */
2871 list_for_each_entry_safe(page, t, &n->partial, lru) {
2872 if (!page->inuse && slab_trylock(page)) {
2873 /*
2874 * Must hold slab lock here because slab_free
2875 * may have freed the last object and be
2876 * waiting to release the slab.
2877 */
2878 list_del(&page->lru);
2879 n->nr_partial--;
2880 slab_unlock(page);
2881 discard_slab(s, page);
2882 } else {
Christoph Lameterfcda3d82007-07-30 13:06:46 -07002883 list_move(&page->lru,
2884 slabs_by_inuse + page->inuse);
Christoph Lameter2086d262007-05-06 14:49:46 -07002885 }
2886 }
2887
Christoph Lameter2086d262007-05-06 14:49:46 -07002888 /*
Christoph Lameter672bba32007-05-09 02:32:39 -07002889 * Rebuild the partial list with the slabs filled up most
2890 * first and the least used slabs at the end.
Christoph Lameter2086d262007-05-06 14:49:46 -07002891 */
Christoph Lameter834f3d12008-04-14 19:11:31 +03002892 for (i = objects - 1; i >= 0; i--)
Christoph Lameter2086d262007-05-06 14:49:46 -07002893 list_splice(slabs_by_inuse + i, n->partial.prev);
2894
Christoph Lameter2086d262007-05-06 14:49:46 -07002895 spin_unlock_irqrestore(&n->list_lock, flags);
2896 }
2897
2898 kfree(slabs_by_inuse);
2899 return 0;
2900}
2901EXPORT_SYMBOL(kmem_cache_shrink);
2902
Yasunori Gotob9049e22007-10-21 16:41:37 -07002903#if defined(CONFIG_NUMA) && defined(CONFIG_MEMORY_HOTPLUG)
2904static int slab_mem_going_offline_callback(void *arg)
2905{
2906 struct kmem_cache *s;
2907
2908 down_read(&slub_lock);
2909 list_for_each_entry(s, &slab_caches, list)
2910 kmem_cache_shrink(s);
2911 up_read(&slub_lock);
2912
2913 return 0;
2914}
2915
2916static void slab_mem_offline_callback(void *arg)
2917{
2918 struct kmem_cache_node *n;
2919 struct kmem_cache *s;
2920 struct memory_notify *marg = arg;
2921 int offline_node;
2922
2923 offline_node = marg->status_change_nid;
2924
2925 /*
2926 * If the node still has available memory. we need kmem_cache_node
2927 * for it yet.
2928 */
2929 if (offline_node < 0)
2930 return;
2931
2932 down_read(&slub_lock);
2933 list_for_each_entry(s, &slab_caches, list) {
2934 n = get_node(s, offline_node);
2935 if (n) {
2936 /*
2937 * if n->nr_slabs > 0, slabs still exist on the node
2938 * that is going down. We were unable to free them,
2939 * and offline_pages() function shoudn't call this
2940 * callback. So, we must fail.
2941 */
Christoph Lameter0f389ec2008-04-14 18:53:02 +03002942 BUG_ON(slabs_node(s, offline_node));
Yasunori Gotob9049e22007-10-21 16:41:37 -07002943
2944 s->node[offline_node] = NULL;
2945 kmem_cache_free(kmalloc_caches, n);
2946 }
2947 }
2948 up_read(&slub_lock);
2949}
2950
2951static int slab_mem_going_online_callback(void *arg)
2952{
2953 struct kmem_cache_node *n;
2954 struct kmem_cache *s;
2955 struct memory_notify *marg = arg;
2956 int nid = marg->status_change_nid;
2957 int ret = 0;
2958
2959 /*
2960 * If the node's memory is already available, then kmem_cache_node is
2961 * already created. Nothing to do.
2962 */
2963 if (nid < 0)
2964 return 0;
2965
2966 /*
Christoph Lameter0121c6192008-04-29 16:11:12 -07002967 * We are bringing a node online. No memory is available yet. We must
Yasunori Gotob9049e22007-10-21 16:41:37 -07002968 * allocate a kmem_cache_node structure in order to bring the node
2969 * online.
2970 */
2971 down_read(&slub_lock);
2972 list_for_each_entry(s, &slab_caches, list) {
2973 /*
2974 * XXX: kmem_cache_alloc_node will fallback to other nodes
2975 * since memory is not yet available from the node that
2976 * is brought up.
2977 */
2978 n = kmem_cache_alloc(kmalloc_caches, GFP_KERNEL);
2979 if (!n) {
2980 ret = -ENOMEM;
2981 goto out;
2982 }
Pekka Enberg5595cff2008-08-05 09:28:47 +03002983 init_kmem_cache_node(n, s);
Yasunori Gotob9049e22007-10-21 16:41:37 -07002984 s->node[nid] = n;
2985 }
2986out:
2987 up_read(&slub_lock);
2988 return ret;
2989}
2990
2991static int slab_memory_callback(struct notifier_block *self,
2992 unsigned long action, void *arg)
2993{
2994 int ret = 0;
2995
2996 switch (action) {
2997 case MEM_GOING_ONLINE:
2998 ret = slab_mem_going_online_callback(arg);
2999 break;
3000 case MEM_GOING_OFFLINE:
3001 ret = slab_mem_going_offline_callback(arg);
3002 break;
3003 case MEM_OFFLINE:
3004 case MEM_CANCEL_ONLINE:
3005 slab_mem_offline_callback(arg);
3006 break;
3007 case MEM_ONLINE:
3008 case MEM_CANCEL_OFFLINE:
3009 break;
3010 }
KAMEZAWA Hiroyukidc19f9d2008-12-01 13:13:48 -08003011 if (ret)
3012 ret = notifier_from_errno(ret);
3013 else
3014 ret = NOTIFY_OK;
Yasunori Gotob9049e22007-10-21 16:41:37 -07003015 return ret;
3016}
3017
3018#endif /* CONFIG_MEMORY_HOTPLUG */
3019
Christoph Lameter81819f02007-05-06 14:49:36 -07003020/********************************************************************
3021 * Basic setup of slabs
3022 *******************************************************************/
3023
3024void __init kmem_cache_init(void)
3025{
3026 int i;
Christoph Lameter4b356be2007-06-16 10:16:13 -07003027 int caches = 0;
Christoph Lameter81819f02007-05-06 14:49:36 -07003028
Christoph Lameter4c93c3552007-10-16 01:26:08 -07003029 init_alloc_cpu();
3030
Christoph Lameter81819f02007-05-06 14:49:36 -07003031#ifdef CONFIG_NUMA
3032 /*
3033 * Must first have the slab cache available for the allocations of the
Christoph Lameter672bba32007-05-09 02:32:39 -07003034 * struct kmem_cache_node's. There is special bootstrap code in
Christoph Lameter81819f02007-05-06 14:49:36 -07003035 * kmem_cache_open for slab_state == DOWN.
3036 */
3037 create_kmalloc_cache(&kmalloc_caches[0], "kmem_cache_node",
Pekka Enberg83b519e2009-06-10 19:40:04 +03003038 sizeof(struct kmem_cache_node), GFP_NOWAIT);
Christoph Lameter8ffa6872007-05-31 00:40:51 -07003039 kmalloc_caches[0].refcount = -1;
Christoph Lameter4b356be2007-06-16 10:16:13 -07003040 caches++;
Yasunori Gotob9049e22007-10-21 16:41:37 -07003041
Nadia Derbey0c40ba42008-04-29 01:00:41 -07003042 hotplug_memory_notifier(slab_memory_callback, SLAB_CALLBACK_PRI);
Christoph Lameter81819f02007-05-06 14:49:36 -07003043#endif
3044
3045 /* Able to allocate the per node structures */
3046 slab_state = PARTIAL;
3047
3048 /* Caches that are not of the two-to-the-power-of size */
Christoph Lameter4b356be2007-06-16 10:16:13 -07003049 if (KMALLOC_MIN_SIZE <= 64) {
3050 create_kmalloc_cache(&kmalloc_caches[1],
Pekka Enberg83b519e2009-06-10 19:40:04 +03003051 "kmalloc-96", 96, GFP_NOWAIT);
Christoph Lameter4b356be2007-06-16 10:16:13 -07003052 caches++;
Christoph Lameter4b356be2007-06-16 10:16:13 -07003053 create_kmalloc_cache(&kmalloc_caches[2],
Pekka Enberg83b519e2009-06-10 19:40:04 +03003054 "kmalloc-192", 192, GFP_NOWAIT);
Christoph Lameter4b356be2007-06-16 10:16:13 -07003055 caches++;
3056 }
Christoph Lameter81819f02007-05-06 14:49:36 -07003057
Christoph Lameterffadd4d2009-02-17 12:05:07 -05003058 for (i = KMALLOC_SHIFT_LOW; i < SLUB_PAGE_SHIFT; i++) {
Christoph Lameter81819f02007-05-06 14:49:36 -07003059 create_kmalloc_cache(&kmalloc_caches[i],
Pekka Enberg83b519e2009-06-10 19:40:04 +03003060 "kmalloc", 1 << i, GFP_NOWAIT);
Christoph Lameter4b356be2007-06-16 10:16:13 -07003061 caches++;
3062 }
Christoph Lameter81819f02007-05-06 14:49:36 -07003063
Christoph Lameterf1b26332007-07-17 04:03:26 -07003064
3065 /*
3066 * Patch up the size_index table if we have strange large alignment
3067 * requirements for the kmalloc array. This is only the case for
Christoph Lameter6446faa2008-02-15 23:45:26 -08003068 * MIPS it seems. The standard arches will not generate any code here.
Christoph Lameterf1b26332007-07-17 04:03:26 -07003069 *
3070 * Largest permitted alignment is 256 bytes due to the way we
3071 * handle the index determination for the smaller caches.
3072 *
3073 * Make sure that nothing crazy happens if someone starts tinkering
3074 * around with ARCH_KMALLOC_MINALIGN
3075 */
3076 BUILD_BUG_ON(KMALLOC_MIN_SIZE > 256 ||
3077 (KMALLOC_MIN_SIZE & (KMALLOC_MIN_SIZE - 1)));
3078
Christoph Lameter12ad6842007-07-17 04:03:28 -07003079 for (i = 8; i < KMALLOC_MIN_SIZE; i += 8)
Christoph Lameterf1b26332007-07-17 04:03:26 -07003080 size_index[(i - 1) / 8] = KMALLOC_SHIFT_LOW;
3081
Christoph Lameter41d54d32008-07-03 09:14:26 -05003082 if (KMALLOC_MIN_SIZE == 128) {
3083 /*
3084 * The 192 byte sized cache is not used if the alignment
3085 * is 128 byte. Redirect kmalloc to use the 256 byte cache
3086 * instead.
3087 */
3088 for (i = 128 + 8; i <= 192; i += 8)
3089 size_index[(i - 1) / 8] = 8;
3090 }
3091
Christoph Lameter81819f02007-05-06 14:49:36 -07003092 slab_state = UP;
3093
3094 /* Provide the correct kmalloc names now that the caches are up */
Christoph Lameterffadd4d2009-02-17 12:05:07 -05003095 for (i = KMALLOC_SHIFT_LOW; i < SLUB_PAGE_SHIFT; i++)
Christoph Lameter81819f02007-05-06 14:49:36 -07003096 kmalloc_caches[i]. name =
Pekka Enberg83b519e2009-06-10 19:40:04 +03003097 kasprintf(GFP_NOWAIT, "kmalloc-%d", 1 << i);
Christoph Lameter81819f02007-05-06 14:49:36 -07003098
3099#ifdef CONFIG_SMP
3100 register_cpu_notifier(&slab_notifier);
Christoph Lameter4c93c3552007-10-16 01:26:08 -07003101 kmem_size = offsetof(struct kmem_cache, cpu_slab) +
3102 nr_cpu_ids * sizeof(struct kmem_cache_cpu *);
3103#else
3104 kmem_size = sizeof(struct kmem_cache);
Christoph Lameter81819f02007-05-06 14:49:36 -07003105#endif
3106
Ingo Molnar3adbefe2008-02-05 17:57:39 -08003107 printk(KERN_INFO
3108 "SLUB: Genslabs=%d, HWalign=%d, Order=%d-%d, MinObjects=%d,"
Christoph Lameter4b356be2007-06-16 10:16:13 -07003109 " CPUs=%d, Nodes=%d\n",
3110 caches, cache_line_size(),
Christoph Lameter81819f02007-05-06 14:49:36 -07003111 slub_min_order, slub_max_order, slub_min_objects,
3112 nr_cpu_ids, nr_node_ids);
3113}
3114
Pekka Enberg7e85ee02009-06-12 14:03:06 +03003115void __init kmem_cache_init_late(void)
3116{
3117 /*
3118 * Interrupts are enabled now so all GFP allocations are safe.
3119 */
3120 slab_gfp_mask = __GFP_BITS_MASK;
3121}
3122
Christoph Lameter81819f02007-05-06 14:49:36 -07003123/*
3124 * Find a mergeable slab cache
3125 */
3126static int slab_unmergeable(struct kmem_cache *s)
3127{
3128 if (slub_nomerge || (s->flags & SLUB_NEVER_MERGE))
3129 return 1;
3130
Christoph Lameterc59def9f2007-05-16 22:10:50 -07003131 if (s->ctor)
Christoph Lameter81819f02007-05-06 14:49:36 -07003132 return 1;
3133
Christoph Lameter8ffa6872007-05-31 00:40:51 -07003134 /*
3135 * We may have set a slab to be unmergeable during bootstrap.
3136 */
3137 if (s->refcount < 0)
3138 return 1;
3139
Christoph Lameter81819f02007-05-06 14:49:36 -07003140 return 0;
3141}
3142
3143static struct kmem_cache *find_mergeable(size_t size,
Christoph Lameterba0268a2007-09-11 15:24:11 -07003144 size_t align, unsigned long flags, const char *name,
Alexey Dobriyan51cc5062008-07-25 19:45:34 -07003145 void (*ctor)(void *))
Christoph Lameter81819f02007-05-06 14:49:36 -07003146{
Christoph Lameter5b95a4a2007-07-17 04:03:19 -07003147 struct kmem_cache *s;
Christoph Lameter81819f02007-05-06 14:49:36 -07003148
3149 if (slub_nomerge || (flags & SLUB_NEVER_MERGE))
3150 return NULL;
3151
Christoph Lameterc59def9f2007-05-16 22:10:50 -07003152 if (ctor)
Christoph Lameter81819f02007-05-06 14:49:36 -07003153 return NULL;
3154
3155 size = ALIGN(size, sizeof(void *));
3156 align = calculate_alignment(flags, align, size);
3157 size = ALIGN(size, align);
Christoph Lameterba0268a2007-09-11 15:24:11 -07003158 flags = kmem_cache_flags(size, flags, name, NULL);
Christoph Lameter81819f02007-05-06 14:49:36 -07003159
Christoph Lameter5b95a4a2007-07-17 04:03:19 -07003160 list_for_each_entry(s, &slab_caches, list) {
Christoph Lameter81819f02007-05-06 14:49:36 -07003161 if (slab_unmergeable(s))
3162 continue;
3163
3164 if (size > s->size)
3165 continue;
3166
Christoph Lameterba0268a2007-09-11 15:24:11 -07003167 if ((flags & SLUB_MERGE_SAME) != (s->flags & SLUB_MERGE_SAME))
Christoph Lameter81819f02007-05-06 14:49:36 -07003168 continue;
3169 /*
3170 * Check if alignment is compatible.
3171 * Courtesy of Adrian Drzewiecki
3172 */
Pekka Enberg06428782008-01-07 23:20:27 -08003173 if ((s->size & ~(align - 1)) != s->size)
Christoph Lameter81819f02007-05-06 14:49:36 -07003174 continue;
3175
3176 if (s->size - size >= sizeof(void *))
3177 continue;
3178
3179 return s;
3180 }
3181 return NULL;
3182}
3183
3184struct kmem_cache *kmem_cache_create(const char *name, size_t size,
Alexey Dobriyan51cc5062008-07-25 19:45:34 -07003185 size_t align, unsigned long flags, void (*ctor)(void *))
Christoph Lameter81819f02007-05-06 14:49:36 -07003186{
3187 struct kmem_cache *s;
3188
3189 down_write(&slub_lock);
Christoph Lameterba0268a2007-09-11 15:24:11 -07003190 s = find_mergeable(size, align, flags, name, ctor);
Christoph Lameter81819f02007-05-06 14:49:36 -07003191 if (s) {
Christoph Lameter42a9fdb2007-10-16 01:26:09 -07003192 int cpu;
3193
Christoph Lameter81819f02007-05-06 14:49:36 -07003194 s->refcount++;
3195 /*
3196 * Adjust the object sizes so that we clear
3197 * the complete object on kzalloc.
3198 */
3199 s->objsize = max(s->objsize, (int)size);
Christoph Lameter42a9fdb2007-10-16 01:26:09 -07003200
3201 /*
3202 * And then we need to update the object size in the
3203 * per cpu structures
3204 */
3205 for_each_online_cpu(cpu)
3206 get_cpu_slab(s, cpu)->objsize = s->objsize;
Christoph Lameter6446faa2008-02-15 23:45:26 -08003207
Christoph Lameter81819f02007-05-06 14:49:36 -07003208 s->inuse = max_t(int, s->inuse, ALIGN(size, sizeof(void *)));
Christoph Lametera0e1d1b2007-07-17 04:03:31 -07003209 up_write(&slub_lock);
Christoph Lameter6446faa2008-02-15 23:45:26 -08003210
David Rientjes7b8f3b62008-12-17 22:09:46 -08003211 if (sysfs_slab_alias(s, name)) {
3212 down_write(&slub_lock);
3213 s->refcount--;
3214 up_write(&slub_lock);
Christoph Lameter81819f02007-05-06 14:49:36 -07003215 goto err;
David Rientjes7b8f3b62008-12-17 22:09:46 -08003216 }
Christoph Lametera0e1d1b2007-07-17 04:03:31 -07003217 return s;
3218 }
Christoph Lameter6446faa2008-02-15 23:45:26 -08003219
Christoph Lametera0e1d1b2007-07-17 04:03:31 -07003220 s = kmalloc(kmem_size, GFP_KERNEL);
3221 if (s) {
3222 if (kmem_cache_open(s, GFP_KERNEL, name,
Christoph Lameterc59def9f2007-05-16 22:10:50 -07003223 size, align, flags, ctor)) {
Christoph Lameter81819f02007-05-06 14:49:36 -07003224 list_add(&s->list, &slab_caches);
Christoph Lametera0e1d1b2007-07-17 04:03:31 -07003225 up_write(&slub_lock);
David Rientjes7b8f3b62008-12-17 22:09:46 -08003226 if (sysfs_slab_add(s)) {
3227 down_write(&slub_lock);
3228 list_del(&s->list);
3229 up_write(&slub_lock);
3230 kfree(s);
Christoph Lametera0e1d1b2007-07-17 04:03:31 -07003231 goto err;
David Rientjes7b8f3b62008-12-17 22:09:46 -08003232 }
Christoph Lametera0e1d1b2007-07-17 04:03:31 -07003233 return s;
3234 }
3235 kfree(s);
Christoph Lameter81819f02007-05-06 14:49:36 -07003236 }
3237 up_write(&slub_lock);
Christoph Lameter81819f02007-05-06 14:49:36 -07003238
3239err:
Christoph Lameter81819f02007-05-06 14:49:36 -07003240 if (flags & SLAB_PANIC)
3241 panic("Cannot create slabcache %s\n", name);
3242 else
3243 s = NULL;
3244 return s;
3245}
3246EXPORT_SYMBOL(kmem_cache_create);
3247
Christoph Lameter81819f02007-05-06 14:49:36 -07003248#ifdef CONFIG_SMP
Christoph Lameter27390bc2007-06-01 00:47:09 -07003249/*
Christoph Lameter672bba32007-05-09 02:32:39 -07003250 * Use the cpu notifier to insure that the cpu slabs are flushed when
3251 * necessary.
Christoph Lameter81819f02007-05-06 14:49:36 -07003252 */
3253static int __cpuinit slab_cpuup_callback(struct notifier_block *nfb,
3254 unsigned long action, void *hcpu)
3255{
3256 long cpu = (long)hcpu;
Christoph Lameter5b95a4a2007-07-17 04:03:19 -07003257 struct kmem_cache *s;
3258 unsigned long flags;
Christoph Lameter81819f02007-05-06 14:49:36 -07003259
3260 switch (action) {
Christoph Lameter4c93c3552007-10-16 01:26:08 -07003261 case CPU_UP_PREPARE:
3262 case CPU_UP_PREPARE_FROZEN:
3263 init_alloc_cpu_cpu(cpu);
3264 down_read(&slub_lock);
3265 list_for_each_entry(s, &slab_caches, list)
3266 s->cpu_slab[cpu] = alloc_kmem_cache_cpu(s, cpu,
3267 GFP_KERNEL);
3268 up_read(&slub_lock);
3269 break;
3270
Christoph Lameter81819f02007-05-06 14:49:36 -07003271 case CPU_UP_CANCELED:
Rafael J. Wysocki8bb78442007-05-09 02:35:10 -07003272 case CPU_UP_CANCELED_FROZEN:
Christoph Lameter81819f02007-05-06 14:49:36 -07003273 case CPU_DEAD:
Rafael J. Wysocki8bb78442007-05-09 02:35:10 -07003274 case CPU_DEAD_FROZEN:
Christoph Lameter5b95a4a2007-07-17 04:03:19 -07003275 down_read(&slub_lock);
3276 list_for_each_entry(s, &slab_caches, list) {
Christoph Lameter4c93c3552007-10-16 01:26:08 -07003277 struct kmem_cache_cpu *c = get_cpu_slab(s, cpu);
3278
Christoph Lameter5b95a4a2007-07-17 04:03:19 -07003279 local_irq_save(flags);
3280 __flush_cpu_slab(s, cpu);
3281 local_irq_restore(flags);
Christoph Lameter4c93c3552007-10-16 01:26:08 -07003282 free_kmem_cache_cpu(c, cpu);
3283 s->cpu_slab[cpu] = NULL;
Christoph Lameter5b95a4a2007-07-17 04:03:19 -07003284 }
3285 up_read(&slub_lock);
Christoph Lameter81819f02007-05-06 14:49:36 -07003286 break;
3287 default:
3288 break;
3289 }
3290 return NOTIFY_OK;
3291}
3292
Pekka Enberg06428782008-01-07 23:20:27 -08003293static struct notifier_block __cpuinitdata slab_notifier = {
Ingo Molnar3adbefe2008-02-05 17:57:39 -08003294 .notifier_call = slab_cpuup_callback
Pekka Enberg06428782008-01-07 23:20:27 -08003295};
Christoph Lameter81819f02007-05-06 14:49:36 -07003296
3297#endif
3298
Eduard - Gabriel Munteanuce71e272008-08-19 20:43:25 +03003299void *__kmalloc_track_caller(size_t size, gfp_t gfpflags, unsigned long caller)
Christoph Lameter81819f02007-05-06 14:49:36 -07003300{
Christoph Lameteraadb4bc2007-10-16 01:24:38 -07003301 struct kmem_cache *s;
Eduard - Gabriel Munteanu94b528d2008-08-24 20:49:35 +03003302 void *ret;
Christoph Lameteraadb4bc2007-10-16 01:24:38 -07003303
Christoph Lameterffadd4d2009-02-17 12:05:07 -05003304 if (unlikely(size > SLUB_MAX_SIZE))
Pekka Enbergeada35e2008-02-11 22:47:46 +02003305 return kmalloc_large(size, gfpflags);
3306
Christoph Lameteraadb4bc2007-10-16 01:24:38 -07003307 s = get_slab(size, gfpflags);
Christoph Lameter81819f02007-05-06 14:49:36 -07003308
Satyam Sharma2408c552007-10-16 01:24:44 -07003309 if (unlikely(ZERO_OR_NULL_PTR(s)))
Christoph Lameter6cb8f912007-07-17 04:03:22 -07003310 return s;
Christoph Lameter81819f02007-05-06 14:49:36 -07003311
Eduard - Gabriel Munteanu94b528d2008-08-24 20:49:35 +03003312 ret = slab_alloc(s, gfpflags, -1, caller);
3313
3314 /* Honor the call site pointer we recieved. */
Eduard - Gabriel Munteanuca2b84c2009-03-23 15:12:24 +02003315 trace_kmalloc(caller, ret, size, s->size, gfpflags);
Eduard - Gabriel Munteanu94b528d2008-08-24 20:49:35 +03003316
3317 return ret;
Christoph Lameter81819f02007-05-06 14:49:36 -07003318}
3319
3320void *__kmalloc_node_track_caller(size_t size, gfp_t gfpflags,
Eduard - Gabriel Munteanuce71e272008-08-19 20:43:25 +03003321 int node, unsigned long caller)
Christoph Lameter81819f02007-05-06 14:49:36 -07003322{
Christoph Lameteraadb4bc2007-10-16 01:24:38 -07003323 struct kmem_cache *s;
Eduard - Gabriel Munteanu94b528d2008-08-24 20:49:35 +03003324 void *ret;
Christoph Lameteraadb4bc2007-10-16 01:24:38 -07003325
Christoph Lameterffadd4d2009-02-17 12:05:07 -05003326 if (unlikely(size > SLUB_MAX_SIZE))
Christoph Lameterf619cfe2008-03-01 13:56:40 -08003327 return kmalloc_large_node(size, gfpflags, node);
Pekka Enbergeada35e2008-02-11 22:47:46 +02003328
Christoph Lameteraadb4bc2007-10-16 01:24:38 -07003329 s = get_slab(size, gfpflags);
Christoph Lameter81819f02007-05-06 14:49:36 -07003330
Satyam Sharma2408c552007-10-16 01:24:44 -07003331 if (unlikely(ZERO_OR_NULL_PTR(s)))
Christoph Lameter6cb8f912007-07-17 04:03:22 -07003332 return s;
Christoph Lameter81819f02007-05-06 14:49:36 -07003333
Eduard - Gabriel Munteanu94b528d2008-08-24 20:49:35 +03003334 ret = slab_alloc(s, gfpflags, node, caller);
3335
3336 /* Honor the call site pointer we recieved. */
Eduard - Gabriel Munteanuca2b84c2009-03-23 15:12:24 +02003337 trace_kmalloc_node(caller, ret, size, s->size, gfpflags, node);
Eduard - Gabriel Munteanu94b528d2008-08-24 20:49:35 +03003338
3339 return ret;
Christoph Lameter81819f02007-05-06 14:49:36 -07003340}
3341
Christoph Lameterf6acb632008-04-29 16:16:06 -07003342#ifdef CONFIG_SLUB_DEBUG
Christoph Lameter205ab992008-04-14 19:11:40 +03003343static unsigned long count_partial(struct kmem_cache_node *n,
3344 int (*get_count)(struct page *))
Christoph Lameter5b06c8532008-04-14 18:51:34 +03003345{
3346 unsigned long flags;
3347 unsigned long x = 0;
3348 struct page *page;
3349
3350 spin_lock_irqsave(&n->list_lock, flags);
3351 list_for_each_entry(page, &n->partial, lru)
Christoph Lameter205ab992008-04-14 19:11:40 +03003352 x += get_count(page);
Christoph Lameter5b06c8532008-04-14 18:51:34 +03003353 spin_unlock_irqrestore(&n->list_lock, flags);
3354 return x;
3355}
Christoph Lameter205ab992008-04-14 19:11:40 +03003356
3357static int count_inuse(struct page *page)
3358{
3359 return page->inuse;
3360}
3361
3362static int count_total(struct page *page)
3363{
3364 return page->objects;
3365}
3366
3367static int count_free(struct page *page)
3368{
3369 return page->objects - page->inuse;
3370}
Christoph Lameter5b06c8532008-04-14 18:51:34 +03003371
Christoph Lameter434e2452007-07-17 04:03:30 -07003372static int validate_slab(struct kmem_cache *s, struct page *page,
3373 unsigned long *map)
Christoph Lameter53e15af2007-05-06 14:49:43 -07003374{
3375 void *p;
Christoph Lametera973e9d2008-03-01 13:40:44 -08003376 void *addr = page_address(page);
Christoph Lameter53e15af2007-05-06 14:49:43 -07003377
3378 if (!check_slab(s, page) ||
3379 !on_freelist(s, page, NULL))
3380 return 0;
3381
3382 /* Now we know that a valid freelist exists */
Christoph Lameter39b26462008-04-14 19:11:30 +03003383 bitmap_zero(map, page->objects);
Christoph Lameter53e15af2007-05-06 14:49:43 -07003384
Christoph Lameter7656c722007-05-09 02:32:40 -07003385 for_each_free_object(p, s, page->freelist) {
3386 set_bit(slab_index(p, s, addr), map);
Christoph Lameter53e15af2007-05-06 14:49:43 -07003387 if (!check_object(s, page, p, 0))
3388 return 0;
3389 }
3390
Christoph Lameter224a88b2008-04-14 19:11:31 +03003391 for_each_object(p, s, addr, page->objects)
Christoph Lameter7656c722007-05-09 02:32:40 -07003392 if (!test_bit(slab_index(p, s, addr), map))
Christoph Lameter53e15af2007-05-06 14:49:43 -07003393 if (!check_object(s, page, p, 1))
3394 return 0;
3395 return 1;
3396}
3397
Christoph Lameter434e2452007-07-17 04:03:30 -07003398static void validate_slab_slab(struct kmem_cache *s, struct page *page,
3399 unsigned long *map)
Christoph Lameter53e15af2007-05-06 14:49:43 -07003400{
3401 if (slab_trylock(page)) {
Christoph Lameter434e2452007-07-17 04:03:30 -07003402 validate_slab(s, page, map);
Christoph Lameter53e15af2007-05-06 14:49:43 -07003403 slab_unlock(page);
3404 } else
3405 printk(KERN_INFO "SLUB %s: Skipped busy slab 0x%p\n",
3406 s->name, page);
3407
3408 if (s->flags & DEBUG_DEFAULT_FLAGS) {
Andy Whitcroft8a380822008-07-23 21:27:18 -07003409 if (!PageSlubDebug(page))
3410 printk(KERN_ERR "SLUB %s: SlubDebug not set "
Christoph Lameter53e15af2007-05-06 14:49:43 -07003411 "on slab 0x%p\n", s->name, page);
3412 } else {
Andy Whitcroft8a380822008-07-23 21:27:18 -07003413 if (PageSlubDebug(page))
3414 printk(KERN_ERR "SLUB %s: SlubDebug set on "
Christoph Lameter53e15af2007-05-06 14:49:43 -07003415 "slab 0x%p\n", s->name, page);
3416 }
3417}
3418
Christoph Lameter434e2452007-07-17 04:03:30 -07003419static int validate_slab_node(struct kmem_cache *s,
3420 struct kmem_cache_node *n, unsigned long *map)
Christoph Lameter53e15af2007-05-06 14:49:43 -07003421{
3422 unsigned long count = 0;
3423 struct page *page;
3424 unsigned long flags;
3425
3426 spin_lock_irqsave(&n->list_lock, flags);
3427
3428 list_for_each_entry(page, &n->partial, lru) {
Christoph Lameter434e2452007-07-17 04:03:30 -07003429 validate_slab_slab(s, page, map);
Christoph Lameter53e15af2007-05-06 14:49:43 -07003430 count++;
3431 }
3432 if (count != n->nr_partial)
3433 printk(KERN_ERR "SLUB %s: %ld partial slabs counted but "
3434 "counter=%ld\n", s->name, count, n->nr_partial);
3435
3436 if (!(s->flags & SLAB_STORE_USER))
3437 goto out;
3438
3439 list_for_each_entry(page, &n->full, lru) {
Christoph Lameter434e2452007-07-17 04:03:30 -07003440 validate_slab_slab(s, page, map);
Christoph Lameter53e15af2007-05-06 14:49:43 -07003441 count++;
3442 }
3443 if (count != atomic_long_read(&n->nr_slabs))
3444 printk(KERN_ERR "SLUB: %s %ld slabs counted but "
3445 "counter=%ld\n", s->name, count,
3446 atomic_long_read(&n->nr_slabs));
3447
3448out:
3449 spin_unlock_irqrestore(&n->list_lock, flags);
3450 return count;
3451}
3452
Christoph Lameter434e2452007-07-17 04:03:30 -07003453static long validate_slab_cache(struct kmem_cache *s)
Christoph Lameter53e15af2007-05-06 14:49:43 -07003454{
3455 int node;
3456 unsigned long count = 0;
Christoph Lameter205ab992008-04-14 19:11:40 +03003457 unsigned long *map = kmalloc(BITS_TO_LONGS(oo_objects(s->max)) *
Christoph Lameter434e2452007-07-17 04:03:30 -07003458 sizeof(unsigned long), GFP_KERNEL);
3459
3460 if (!map)
3461 return -ENOMEM;
Christoph Lameter53e15af2007-05-06 14:49:43 -07003462
3463 flush_all(s);
Christoph Lameterf64dc582007-10-16 01:25:33 -07003464 for_each_node_state(node, N_NORMAL_MEMORY) {
Christoph Lameter53e15af2007-05-06 14:49:43 -07003465 struct kmem_cache_node *n = get_node(s, node);
3466
Christoph Lameter434e2452007-07-17 04:03:30 -07003467 count += validate_slab_node(s, n, map);
Christoph Lameter53e15af2007-05-06 14:49:43 -07003468 }
Christoph Lameter434e2452007-07-17 04:03:30 -07003469 kfree(map);
Christoph Lameter53e15af2007-05-06 14:49:43 -07003470 return count;
3471}
3472
Christoph Lameterb3459702007-05-09 02:32:41 -07003473#ifdef SLUB_RESILIENCY_TEST
3474static void resiliency_test(void)
3475{
3476 u8 *p;
3477
3478 printk(KERN_ERR "SLUB resiliency testing\n");
3479 printk(KERN_ERR "-----------------------\n");
3480 printk(KERN_ERR "A. Corruption after allocation\n");
3481
3482 p = kzalloc(16, GFP_KERNEL);
3483 p[16] = 0x12;
3484 printk(KERN_ERR "\n1. kmalloc-16: Clobber Redzone/next pointer"
3485 " 0x12->0x%p\n\n", p + 16);
3486
3487 validate_slab_cache(kmalloc_caches + 4);
3488
3489 /* Hmmm... The next two are dangerous */
3490 p = kzalloc(32, GFP_KERNEL);
3491 p[32 + sizeof(void *)] = 0x34;
3492 printk(KERN_ERR "\n2. kmalloc-32: Clobber next pointer/next slab"
Ingo Molnar3adbefe2008-02-05 17:57:39 -08003493 " 0x34 -> -0x%p\n", p);
3494 printk(KERN_ERR
3495 "If allocated object is overwritten then not detectable\n\n");
Christoph Lameterb3459702007-05-09 02:32:41 -07003496
3497 validate_slab_cache(kmalloc_caches + 5);
3498 p = kzalloc(64, GFP_KERNEL);
3499 p += 64 + (get_cycles() & 0xff) * sizeof(void *);
3500 *p = 0x56;
3501 printk(KERN_ERR "\n3. kmalloc-64: corrupting random byte 0x56->0x%p\n",
3502 p);
Ingo Molnar3adbefe2008-02-05 17:57:39 -08003503 printk(KERN_ERR
3504 "If allocated object is overwritten then not detectable\n\n");
Christoph Lameterb3459702007-05-09 02:32:41 -07003505 validate_slab_cache(kmalloc_caches + 6);
3506
3507 printk(KERN_ERR "\nB. Corruption after free\n");
3508 p = kzalloc(128, GFP_KERNEL);
3509 kfree(p);
3510 *p = 0x78;
3511 printk(KERN_ERR "1. kmalloc-128: Clobber first word 0x78->0x%p\n\n", p);
3512 validate_slab_cache(kmalloc_caches + 7);
3513
3514 p = kzalloc(256, GFP_KERNEL);
3515 kfree(p);
3516 p[50] = 0x9a;
Ingo Molnar3adbefe2008-02-05 17:57:39 -08003517 printk(KERN_ERR "\n2. kmalloc-256: Clobber 50th byte 0x9a->0x%p\n\n",
3518 p);
Christoph Lameterb3459702007-05-09 02:32:41 -07003519 validate_slab_cache(kmalloc_caches + 8);
3520
3521 p = kzalloc(512, GFP_KERNEL);
3522 kfree(p);
3523 p[512] = 0xab;
3524 printk(KERN_ERR "\n3. kmalloc-512: Clobber redzone 0xab->0x%p\n\n", p);
3525 validate_slab_cache(kmalloc_caches + 9);
3526}
3527#else
3528static void resiliency_test(void) {};
3529#endif
3530
Christoph Lameter88a420e2007-05-06 14:49:45 -07003531/*
Christoph Lameter672bba32007-05-09 02:32:39 -07003532 * Generate lists of code addresses where slabcache objects are allocated
Christoph Lameter88a420e2007-05-06 14:49:45 -07003533 * and freed.
3534 */
3535
3536struct location {
3537 unsigned long count;
Eduard - Gabriel Munteanuce71e272008-08-19 20:43:25 +03003538 unsigned long addr;
Christoph Lameter45edfa52007-05-09 02:32:45 -07003539 long long sum_time;
3540 long min_time;
3541 long max_time;
3542 long min_pid;
3543 long max_pid;
Rusty Russell174596a2009-01-01 10:12:29 +10303544 DECLARE_BITMAP(cpus, NR_CPUS);
Christoph Lameter45edfa52007-05-09 02:32:45 -07003545 nodemask_t nodes;
Christoph Lameter88a420e2007-05-06 14:49:45 -07003546};
3547
3548struct loc_track {
3549 unsigned long max;
3550 unsigned long count;
3551 struct location *loc;
3552};
3553
3554static void free_loc_track(struct loc_track *t)
3555{
3556 if (t->max)
3557 free_pages((unsigned long)t->loc,
3558 get_order(sizeof(struct location) * t->max));
3559}
3560
Christoph Lameter68dff6a2007-07-17 04:03:20 -07003561static int alloc_loc_track(struct loc_track *t, unsigned long max, gfp_t flags)
Christoph Lameter88a420e2007-05-06 14:49:45 -07003562{
3563 struct location *l;
3564 int order;
3565
Christoph Lameter88a420e2007-05-06 14:49:45 -07003566 order = get_order(sizeof(struct location) * max);
3567
Christoph Lameter68dff6a2007-07-17 04:03:20 -07003568 l = (void *)__get_free_pages(flags, order);
Christoph Lameter88a420e2007-05-06 14:49:45 -07003569 if (!l)
3570 return 0;
3571
3572 if (t->count) {
3573 memcpy(l, t->loc, sizeof(struct location) * t->count);
3574 free_loc_track(t);
3575 }
3576 t->max = max;
3577 t->loc = l;
3578 return 1;
3579}
3580
3581static int add_location(struct loc_track *t, struct kmem_cache *s,
Christoph Lameter45edfa52007-05-09 02:32:45 -07003582 const struct track *track)
Christoph Lameter88a420e2007-05-06 14:49:45 -07003583{
3584 long start, end, pos;
3585 struct location *l;
Eduard - Gabriel Munteanuce71e272008-08-19 20:43:25 +03003586 unsigned long caddr;
Christoph Lameter45edfa52007-05-09 02:32:45 -07003587 unsigned long age = jiffies - track->when;
Christoph Lameter88a420e2007-05-06 14:49:45 -07003588
3589 start = -1;
3590 end = t->count;
3591
3592 for ( ; ; ) {
3593 pos = start + (end - start + 1) / 2;
3594
3595 /*
3596 * There is nothing at "end". If we end up there
3597 * we need to add something to before end.
3598 */
3599 if (pos == end)
3600 break;
3601
3602 caddr = t->loc[pos].addr;
Christoph Lameter45edfa52007-05-09 02:32:45 -07003603 if (track->addr == caddr) {
3604
3605 l = &t->loc[pos];
3606 l->count++;
3607 if (track->when) {
3608 l->sum_time += age;
3609 if (age < l->min_time)
3610 l->min_time = age;
3611 if (age > l->max_time)
3612 l->max_time = age;
3613
3614 if (track->pid < l->min_pid)
3615 l->min_pid = track->pid;
3616 if (track->pid > l->max_pid)
3617 l->max_pid = track->pid;
3618
Rusty Russell174596a2009-01-01 10:12:29 +10303619 cpumask_set_cpu(track->cpu,
3620 to_cpumask(l->cpus));
Christoph Lameter45edfa52007-05-09 02:32:45 -07003621 }
3622 node_set(page_to_nid(virt_to_page(track)), l->nodes);
Christoph Lameter88a420e2007-05-06 14:49:45 -07003623 return 1;
3624 }
3625
Christoph Lameter45edfa52007-05-09 02:32:45 -07003626 if (track->addr < caddr)
Christoph Lameter88a420e2007-05-06 14:49:45 -07003627 end = pos;
3628 else
3629 start = pos;
3630 }
3631
3632 /*
Christoph Lameter672bba32007-05-09 02:32:39 -07003633 * Not found. Insert new tracking element.
Christoph Lameter88a420e2007-05-06 14:49:45 -07003634 */
Christoph Lameter68dff6a2007-07-17 04:03:20 -07003635 if (t->count >= t->max && !alloc_loc_track(t, 2 * t->max, GFP_ATOMIC))
Christoph Lameter88a420e2007-05-06 14:49:45 -07003636 return 0;
3637
3638 l = t->loc + pos;
3639 if (pos < t->count)
3640 memmove(l + 1, l,
3641 (t->count - pos) * sizeof(struct location));
3642 t->count++;
3643 l->count = 1;
Christoph Lameter45edfa52007-05-09 02:32:45 -07003644 l->addr = track->addr;
3645 l->sum_time = age;
3646 l->min_time = age;
3647 l->max_time = age;
3648 l->min_pid = track->pid;
3649 l->max_pid = track->pid;
Rusty Russell174596a2009-01-01 10:12:29 +10303650 cpumask_clear(to_cpumask(l->cpus));
3651 cpumask_set_cpu(track->cpu, to_cpumask(l->cpus));
Christoph Lameter45edfa52007-05-09 02:32:45 -07003652 nodes_clear(l->nodes);
3653 node_set(page_to_nid(virt_to_page(track)), l->nodes);
Christoph Lameter88a420e2007-05-06 14:49:45 -07003654 return 1;
3655}
3656
3657static void process_slab(struct loc_track *t, struct kmem_cache *s,
3658 struct page *page, enum track_item alloc)
3659{
Christoph Lametera973e9d2008-03-01 13:40:44 -08003660 void *addr = page_address(page);
Christoph Lameter39b26462008-04-14 19:11:30 +03003661 DECLARE_BITMAP(map, page->objects);
Christoph Lameter88a420e2007-05-06 14:49:45 -07003662 void *p;
3663
Christoph Lameter39b26462008-04-14 19:11:30 +03003664 bitmap_zero(map, page->objects);
Christoph Lameter7656c722007-05-09 02:32:40 -07003665 for_each_free_object(p, s, page->freelist)
3666 set_bit(slab_index(p, s, addr), map);
Christoph Lameter88a420e2007-05-06 14:49:45 -07003667
Christoph Lameter224a88b2008-04-14 19:11:31 +03003668 for_each_object(p, s, addr, page->objects)
Christoph Lameter45edfa52007-05-09 02:32:45 -07003669 if (!test_bit(slab_index(p, s, addr), map))
3670 add_location(t, s, get_track(s, p, alloc));
Christoph Lameter88a420e2007-05-06 14:49:45 -07003671}
3672
3673static int list_locations(struct kmem_cache *s, char *buf,
3674 enum track_item alloc)
3675{
Harvey Harrisone374d482008-01-31 15:20:50 -08003676 int len = 0;
Christoph Lameter88a420e2007-05-06 14:49:45 -07003677 unsigned long i;
Christoph Lameter68dff6a2007-07-17 04:03:20 -07003678 struct loc_track t = { 0, 0, NULL };
Christoph Lameter88a420e2007-05-06 14:49:45 -07003679 int node;
3680
Christoph Lameter68dff6a2007-07-17 04:03:20 -07003681 if (!alloc_loc_track(&t, PAGE_SIZE / sizeof(struct location),
Andrew Mortonea3061d2007-10-16 01:26:09 -07003682 GFP_TEMPORARY))
Christoph Lameter68dff6a2007-07-17 04:03:20 -07003683 return sprintf(buf, "Out of memory\n");
Christoph Lameter88a420e2007-05-06 14:49:45 -07003684
3685 /* Push back cpu slabs */
3686 flush_all(s);
3687
Christoph Lameterf64dc582007-10-16 01:25:33 -07003688 for_each_node_state(node, N_NORMAL_MEMORY) {
Christoph Lameter88a420e2007-05-06 14:49:45 -07003689 struct kmem_cache_node *n = get_node(s, node);
3690 unsigned long flags;
3691 struct page *page;
3692
Christoph Lameter9e869432007-08-22 14:01:56 -07003693 if (!atomic_long_read(&n->nr_slabs))
Christoph Lameter88a420e2007-05-06 14:49:45 -07003694 continue;
3695
3696 spin_lock_irqsave(&n->list_lock, flags);
3697 list_for_each_entry(page, &n->partial, lru)
3698 process_slab(&t, s, page, alloc);
3699 list_for_each_entry(page, &n->full, lru)
3700 process_slab(&t, s, page, alloc);
3701 spin_unlock_irqrestore(&n->list_lock, flags);
3702 }
3703
3704 for (i = 0; i < t.count; i++) {
Christoph Lameter45edfa52007-05-09 02:32:45 -07003705 struct location *l = &t.loc[i];
Christoph Lameter88a420e2007-05-06 14:49:45 -07003706
Hugh Dickins9c246242008-12-09 13:14:27 -08003707 if (len > PAGE_SIZE - KSYM_SYMBOL_LEN - 100)
Christoph Lameter88a420e2007-05-06 14:49:45 -07003708 break;
Harvey Harrisone374d482008-01-31 15:20:50 -08003709 len += sprintf(buf + len, "%7ld ", l->count);
Christoph Lameter45edfa52007-05-09 02:32:45 -07003710
3711 if (l->addr)
Harvey Harrisone374d482008-01-31 15:20:50 -08003712 len += sprint_symbol(buf + len, (unsigned long)l->addr);
Christoph Lameter88a420e2007-05-06 14:49:45 -07003713 else
Harvey Harrisone374d482008-01-31 15:20:50 -08003714 len += sprintf(buf + len, "<not-available>");
Christoph Lameter45edfa52007-05-09 02:32:45 -07003715
3716 if (l->sum_time != l->min_time) {
Harvey Harrisone374d482008-01-31 15:20:50 -08003717 len += sprintf(buf + len, " age=%ld/%ld/%ld",
Roman Zippelf8bd2252008-05-01 04:34:31 -07003718 l->min_time,
3719 (long)div_u64(l->sum_time, l->count),
3720 l->max_time);
Christoph Lameter45edfa52007-05-09 02:32:45 -07003721 } else
Harvey Harrisone374d482008-01-31 15:20:50 -08003722 len += sprintf(buf + len, " age=%ld",
Christoph Lameter45edfa52007-05-09 02:32:45 -07003723 l->min_time);
3724
3725 if (l->min_pid != l->max_pid)
Harvey Harrisone374d482008-01-31 15:20:50 -08003726 len += sprintf(buf + len, " pid=%ld-%ld",
Christoph Lameter45edfa52007-05-09 02:32:45 -07003727 l->min_pid, l->max_pid);
3728 else
Harvey Harrisone374d482008-01-31 15:20:50 -08003729 len += sprintf(buf + len, " pid=%ld",
Christoph Lameter45edfa52007-05-09 02:32:45 -07003730 l->min_pid);
3731
Rusty Russell174596a2009-01-01 10:12:29 +10303732 if (num_online_cpus() > 1 &&
3733 !cpumask_empty(to_cpumask(l->cpus)) &&
Harvey Harrisone374d482008-01-31 15:20:50 -08003734 len < PAGE_SIZE - 60) {
3735 len += sprintf(buf + len, " cpus=");
3736 len += cpulist_scnprintf(buf + len, PAGE_SIZE - len - 50,
Rusty Russell174596a2009-01-01 10:12:29 +10303737 to_cpumask(l->cpus));
Christoph Lameter45edfa52007-05-09 02:32:45 -07003738 }
3739
Christoph Lameter84966342007-06-23 17:16:32 -07003740 if (num_online_nodes() > 1 && !nodes_empty(l->nodes) &&
Harvey Harrisone374d482008-01-31 15:20:50 -08003741 len < PAGE_SIZE - 60) {
3742 len += sprintf(buf + len, " nodes=");
3743 len += nodelist_scnprintf(buf + len, PAGE_SIZE - len - 50,
Christoph Lameter45edfa52007-05-09 02:32:45 -07003744 l->nodes);
3745 }
3746
Harvey Harrisone374d482008-01-31 15:20:50 -08003747 len += sprintf(buf + len, "\n");
Christoph Lameter88a420e2007-05-06 14:49:45 -07003748 }
3749
3750 free_loc_track(&t);
3751 if (!t.count)
Harvey Harrisone374d482008-01-31 15:20:50 -08003752 len += sprintf(buf, "No data\n");
3753 return len;
Christoph Lameter88a420e2007-05-06 14:49:45 -07003754}
3755
Christoph Lameter81819f02007-05-06 14:49:36 -07003756enum slab_stat_type {
Christoph Lameter205ab992008-04-14 19:11:40 +03003757 SL_ALL, /* All slabs */
3758 SL_PARTIAL, /* Only partially allocated slabs */
3759 SL_CPU, /* Only slabs used for cpu caches */
3760 SL_OBJECTS, /* Determine allocated objects not slabs */
3761 SL_TOTAL /* Determine object capacity not slabs */
Christoph Lameter81819f02007-05-06 14:49:36 -07003762};
3763
Christoph Lameter205ab992008-04-14 19:11:40 +03003764#define SO_ALL (1 << SL_ALL)
Christoph Lameter81819f02007-05-06 14:49:36 -07003765#define SO_PARTIAL (1 << SL_PARTIAL)
3766#define SO_CPU (1 << SL_CPU)
3767#define SO_OBJECTS (1 << SL_OBJECTS)
Christoph Lameter205ab992008-04-14 19:11:40 +03003768#define SO_TOTAL (1 << SL_TOTAL)
Christoph Lameter81819f02007-05-06 14:49:36 -07003769
Cyrill Gorcunov62e5c4b2008-03-02 23:28:24 +03003770static ssize_t show_slab_objects(struct kmem_cache *s,
3771 char *buf, unsigned long flags)
Christoph Lameter81819f02007-05-06 14:49:36 -07003772{
3773 unsigned long total = 0;
Christoph Lameter81819f02007-05-06 14:49:36 -07003774 int node;
3775 int x;
3776 unsigned long *nodes;
3777 unsigned long *per_cpu;
3778
3779 nodes = kzalloc(2 * sizeof(unsigned long) * nr_node_ids, GFP_KERNEL);
Cyrill Gorcunov62e5c4b2008-03-02 23:28:24 +03003780 if (!nodes)
3781 return -ENOMEM;
Christoph Lameter81819f02007-05-06 14:49:36 -07003782 per_cpu = nodes + nr_node_ids;
3783
Christoph Lameter205ab992008-04-14 19:11:40 +03003784 if (flags & SO_CPU) {
3785 int cpu;
Christoph Lameter81819f02007-05-06 14:49:36 -07003786
Christoph Lameter205ab992008-04-14 19:11:40 +03003787 for_each_possible_cpu(cpu) {
3788 struct kmem_cache_cpu *c = get_cpu_slab(s, cpu);
Christoph Lameterdfb4f092007-10-16 01:26:05 -07003789
Christoph Lameter205ab992008-04-14 19:11:40 +03003790 if (!c || c->node < 0)
3791 continue;
3792
3793 if (c->page) {
3794 if (flags & SO_TOTAL)
3795 x = c->page->objects;
3796 else if (flags & SO_OBJECTS)
3797 x = c->page->inuse;
Christoph Lameter81819f02007-05-06 14:49:36 -07003798 else
3799 x = 1;
Christoph Lameter205ab992008-04-14 19:11:40 +03003800
Christoph Lameter81819f02007-05-06 14:49:36 -07003801 total += x;
Christoph Lameter205ab992008-04-14 19:11:40 +03003802 nodes[c->node] += x;
Christoph Lameter81819f02007-05-06 14:49:36 -07003803 }
Christoph Lameter205ab992008-04-14 19:11:40 +03003804 per_cpu[c->node]++;
Christoph Lameter81819f02007-05-06 14:49:36 -07003805 }
3806 }
3807
Christoph Lameter205ab992008-04-14 19:11:40 +03003808 if (flags & SO_ALL) {
3809 for_each_node_state(node, N_NORMAL_MEMORY) {
3810 struct kmem_cache_node *n = get_node(s, node);
Christoph Lameter81819f02007-05-06 14:49:36 -07003811
Christoph Lameter205ab992008-04-14 19:11:40 +03003812 if (flags & SO_TOTAL)
3813 x = atomic_long_read(&n->total_objects);
3814 else if (flags & SO_OBJECTS)
3815 x = atomic_long_read(&n->total_objects) -
3816 count_partial(n, count_free);
3817
3818 else
3819 x = atomic_long_read(&n->nr_slabs);
3820 total += x;
3821 nodes[node] += x;
3822 }
3823
3824 } else if (flags & SO_PARTIAL) {
3825 for_each_node_state(node, N_NORMAL_MEMORY) {
3826 struct kmem_cache_node *n = get_node(s, node);
3827
3828 if (flags & SO_TOTAL)
3829 x = count_partial(n, count_total);
3830 else if (flags & SO_OBJECTS)
3831 x = count_partial(n, count_inuse);
Christoph Lameter81819f02007-05-06 14:49:36 -07003832 else
3833 x = n->nr_partial;
3834 total += x;
3835 nodes[node] += x;
3836 }
Christoph Lameter81819f02007-05-06 14:49:36 -07003837 }
Christoph Lameter81819f02007-05-06 14:49:36 -07003838 x = sprintf(buf, "%lu", total);
3839#ifdef CONFIG_NUMA
Christoph Lameterf64dc582007-10-16 01:25:33 -07003840 for_each_node_state(node, N_NORMAL_MEMORY)
Christoph Lameter81819f02007-05-06 14:49:36 -07003841 if (nodes[node])
3842 x += sprintf(buf + x, " N%d=%lu",
3843 node, nodes[node]);
3844#endif
3845 kfree(nodes);
3846 return x + sprintf(buf + x, "\n");
3847}
3848
3849static int any_slab_objects(struct kmem_cache *s)
3850{
3851 int node;
Christoph Lameterdfb4f092007-10-16 01:26:05 -07003852
3853 for_each_online_node(node) {
Christoph Lameter81819f02007-05-06 14:49:36 -07003854 struct kmem_cache_node *n = get_node(s, node);
3855
Christoph Lameterdfb4f092007-10-16 01:26:05 -07003856 if (!n)
3857 continue;
3858
Benjamin Herrenschmidt4ea33e22008-05-06 20:42:39 -07003859 if (atomic_long_read(&n->total_objects))
Christoph Lameter81819f02007-05-06 14:49:36 -07003860 return 1;
3861 }
3862 return 0;
3863}
3864
3865#define to_slab_attr(n) container_of(n, struct slab_attribute, attr)
3866#define to_slab(n) container_of(n, struct kmem_cache, kobj);
3867
3868struct slab_attribute {
3869 struct attribute attr;
3870 ssize_t (*show)(struct kmem_cache *s, char *buf);
3871 ssize_t (*store)(struct kmem_cache *s, const char *x, size_t count);
3872};
3873
3874#define SLAB_ATTR_RO(_name) \
3875 static struct slab_attribute _name##_attr = __ATTR_RO(_name)
3876
3877#define SLAB_ATTR(_name) \
3878 static struct slab_attribute _name##_attr = \
3879 __ATTR(_name, 0644, _name##_show, _name##_store)
3880
Christoph Lameter81819f02007-05-06 14:49:36 -07003881static ssize_t slab_size_show(struct kmem_cache *s, char *buf)
3882{
3883 return sprintf(buf, "%d\n", s->size);
3884}
3885SLAB_ATTR_RO(slab_size);
3886
3887static ssize_t align_show(struct kmem_cache *s, char *buf)
3888{
3889 return sprintf(buf, "%d\n", s->align);
3890}
3891SLAB_ATTR_RO(align);
3892
3893static ssize_t object_size_show(struct kmem_cache *s, char *buf)
3894{
3895 return sprintf(buf, "%d\n", s->objsize);
3896}
3897SLAB_ATTR_RO(object_size);
3898
3899static ssize_t objs_per_slab_show(struct kmem_cache *s, char *buf)
3900{
Christoph Lameter834f3d12008-04-14 19:11:31 +03003901 return sprintf(buf, "%d\n", oo_objects(s->oo));
Christoph Lameter81819f02007-05-06 14:49:36 -07003902}
3903SLAB_ATTR_RO(objs_per_slab);
3904
Christoph Lameter06b285d2008-04-14 19:11:41 +03003905static ssize_t order_store(struct kmem_cache *s,
3906 const char *buf, size_t length)
3907{
Christoph Lameter0121c6192008-04-29 16:11:12 -07003908 unsigned long order;
3909 int err;
3910
3911 err = strict_strtoul(buf, 10, &order);
3912 if (err)
3913 return err;
Christoph Lameter06b285d2008-04-14 19:11:41 +03003914
3915 if (order > slub_max_order || order < slub_min_order)
3916 return -EINVAL;
3917
3918 calculate_sizes(s, order);
3919 return length;
3920}
3921
Christoph Lameter81819f02007-05-06 14:49:36 -07003922static ssize_t order_show(struct kmem_cache *s, char *buf)
3923{
Christoph Lameter834f3d12008-04-14 19:11:31 +03003924 return sprintf(buf, "%d\n", oo_order(s->oo));
Christoph Lameter81819f02007-05-06 14:49:36 -07003925}
Christoph Lameter06b285d2008-04-14 19:11:41 +03003926SLAB_ATTR(order);
Christoph Lameter81819f02007-05-06 14:49:36 -07003927
David Rientjes73d342b2009-02-22 17:40:09 -08003928static ssize_t min_partial_show(struct kmem_cache *s, char *buf)
3929{
3930 return sprintf(buf, "%lu\n", s->min_partial);
3931}
3932
3933static ssize_t min_partial_store(struct kmem_cache *s, const char *buf,
3934 size_t length)
3935{
3936 unsigned long min;
3937 int err;
3938
3939 err = strict_strtoul(buf, 10, &min);
3940 if (err)
3941 return err;
3942
David Rientjesc0bdb232009-02-25 09:16:35 +02003943 set_min_partial(s, min);
David Rientjes73d342b2009-02-22 17:40:09 -08003944 return length;
3945}
3946SLAB_ATTR(min_partial);
3947
Christoph Lameter81819f02007-05-06 14:49:36 -07003948static ssize_t ctor_show(struct kmem_cache *s, char *buf)
3949{
3950 if (s->ctor) {
3951 int n = sprint_symbol(buf, (unsigned long)s->ctor);
3952
3953 return n + sprintf(buf + n, "\n");
3954 }
3955 return 0;
3956}
3957SLAB_ATTR_RO(ctor);
3958
Christoph Lameter81819f02007-05-06 14:49:36 -07003959static ssize_t aliases_show(struct kmem_cache *s, char *buf)
3960{
3961 return sprintf(buf, "%d\n", s->refcount - 1);
3962}
3963SLAB_ATTR_RO(aliases);
3964
3965static ssize_t slabs_show(struct kmem_cache *s, char *buf)
3966{
Christoph Lameter205ab992008-04-14 19:11:40 +03003967 return show_slab_objects(s, buf, SO_ALL);
Christoph Lameter81819f02007-05-06 14:49:36 -07003968}
3969SLAB_ATTR_RO(slabs);
3970
3971static ssize_t partial_show(struct kmem_cache *s, char *buf)
3972{
Christoph Lameterd9acf4b2008-02-15 15:22:21 -08003973 return show_slab_objects(s, buf, SO_PARTIAL);
Christoph Lameter81819f02007-05-06 14:49:36 -07003974}
3975SLAB_ATTR_RO(partial);
3976
3977static ssize_t cpu_slabs_show(struct kmem_cache *s, char *buf)
3978{
Christoph Lameterd9acf4b2008-02-15 15:22:21 -08003979 return show_slab_objects(s, buf, SO_CPU);
Christoph Lameter81819f02007-05-06 14:49:36 -07003980}
3981SLAB_ATTR_RO(cpu_slabs);
3982
3983static ssize_t objects_show(struct kmem_cache *s, char *buf)
3984{
Christoph Lameter205ab992008-04-14 19:11:40 +03003985 return show_slab_objects(s, buf, SO_ALL|SO_OBJECTS);
Christoph Lameter81819f02007-05-06 14:49:36 -07003986}
3987SLAB_ATTR_RO(objects);
3988
Christoph Lameter205ab992008-04-14 19:11:40 +03003989static ssize_t objects_partial_show(struct kmem_cache *s, char *buf)
3990{
3991 return show_slab_objects(s, buf, SO_PARTIAL|SO_OBJECTS);
3992}
3993SLAB_ATTR_RO(objects_partial);
3994
3995static ssize_t total_objects_show(struct kmem_cache *s, char *buf)
3996{
3997 return show_slab_objects(s, buf, SO_ALL|SO_TOTAL);
3998}
3999SLAB_ATTR_RO(total_objects);
4000
Christoph Lameter81819f02007-05-06 14:49:36 -07004001static ssize_t sanity_checks_show(struct kmem_cache *s, char *buf)
4002{
4003 return sprintf(buf, "%d\n", !!(s->flags & SLAB_DEBUG_FREE));
4004}
4005
4006static ssize_t sanity_checks_store(struct kmem_cache *s,
4007 const char *buf, size_t length)
4008{
4009 s->flags &= ~SLAB_DEBUG_FREE;
4010 if (buf[0] == '1')
4011 s->flags |= SLAB_DEBUG_FREE;
4012 return length;
4013}
4014SLAB_ATTR(sanity_checks);
4015
4016static ssize_t trace_show(struct kmem_cache *s, char *buf)
4017{
4018 return sprintf(buf, "%d\n", !!(s->flags & SLAB_TRACE));
4019}
4020
4021static ssize_t trace_store(struct kmem_cache *s, const char *buf,
4022 size_t length)
4023{
4024 s->flags &= ~SLAB_TRACE;
4025 if (buf[0] == '1')
4026 s->flags |= SLAB_TRACE;
4027 return length;
4028}
4029SLAB_ATTR(trace);
4030
4031static ssize_t reclaim_account_show(struct kmem_cache *s, char *buf)
4032{
4033 return sprintf(buf, "%d\n", !!(s->flags & SLAB_RECLAIM_ACCOUNT));
4034}
4035
4036static ssize_t reclaim_account_store(struct kmem_cache *s,
4037 const char *buf, size_t length)
4038{
4039 s->flags &= ~SLAB_RECLAIM_ACCOUNT;
4040 if (buf[0] == '1')
4041 s->flags |= SLAB_RECLAIM_ACCOUNT;
4042 return length;
4043}
4044SLAB_ATTR(reclaim_account);
4045
4046static ssize_t hwcache_align_show(struct kmem_cache *s, char *buf)
4047{
Christoph Lameter5af60832007-05-06 14:49:56 -07004048 return sprintf(buf, "%d\n", !!(s->flags & SLAB_HWCACHE_ALIGN));
Christoph Lameter81819f02007-05-06 14:49:36 -07004049}
4050SLAB_ATTR_RO(hwcache_align);
4051
4052#ifdef CONFIG_ZONE_DMA
4053static ssize_t cache_dma_show(struct kmem_cache *s, char *buf)
4054{
4055 return sprintf(buf, "%d\n", !!(s->flags & SLAB_CACHE_DMA));
4056}
4057SLAB_ATTR_RO(cache_dma);
4058#endif
4059
4060static ssize_t destroy_by_rcu_show(struct kmem_cache *s, char *buf)
4061{
4062 return sprintf(buf, "%d\n", !!(s->flags & SLAB_DESTROY_BY_RCU));
4063}
4064SLAB_ATTR_RO(destroy_by_rcu);
4065
4066static ssize_t red_zone_show(struct kmem_cache *s, char *buf)
4067{
4068 return sprintf(buf, "%d\n", !!(s->flags & SLAB_RED_ZONE));
4069}
4070
4071static ssize_t red_zone_store(struct kmem_cache *s,
4072 const char *buf, size_t length)
4073{
4074 if (any_slab_objects(s))
4075 return -EBUSY;
4076
4077 s->flags &= ~SLAB_RED_ZONE;
4078 if (buf[0] == '1')
4079 s->flags |= SLAB_RED_ZONE;
Christoph Lameter06b285d2008-04-14 19:11:41 +03004080 calculate_sizes(s, -1);
Christoph Lameter81819f02007-05-06 14:49:36 -07004081 return length;
4082}
4083SLAB_ATTR(red_zone);
4084
4085static ssize_t poison_show(struct kmem_cache *s, char *buf)
4086{
4087 return sprintf(buf, "%d\n", !!(s->flags & SLAB_POISON));
4088}
4089
4090static ssize_t poison_store(struct kmem_cache *s,
4091 const char *buf, size_t length)
4092{
4093 if (any_slab_objects(s))
4094 return -EBUSY;
4095
4096 s->flags &= ~SLAB_POISON;
4097 if (buf[0] == '1')
4098 s->flags |= SLAB_POISON;
Christoph Lameter06b285d2008-04-14 19:11:41 +03004099 calculate_sizes(s, -1);
Christoph Lameter81819f02007-05-06 14:49:36 -07004100 return length;
4101}
4102SLAB_ATTR(poison);
4103
4104static ssize_t store_user_show(struct kmem_cache *s, char *buf)
4105{
4106 return sprintf(buf, "%d\n", !!(s->flags & SLAB_STORE_USER));
4107}
4108
4109static ssize_t store_user_store(struct kmem_cache *s,
4110 const char *buf, size_t length)
4111{
4112 if (any_slab_objects(s))
4113 return -EBUSY;
4114
4115 s->flags &= ~SLAB_STORE_USER;
4116 if (buf[0] == '1')
4117 s->flags |= SLAB_STORE_USER;
Christoph Lameter06b285d2008-04-14 19:11:41 +03004118 calculate_sizes(s, -1);
Christoph Lameter81819f02007-05-06 14:49:36 -07004119 return length;
4120}
4121SLAB_ATTR(store_user);
4122
Christoph Lameter53e15af2007-05-06 14:49:43 -07004123static ssize_t validate_show(struct kmem_cache *s, char *buf)
4124{
4125 return 0;
4126}
4127
4128static ssize_t validate_store(struct kmem_cache *s,
4129 const char *buf, size_t length)
4130{
Christoph Lameter434e2452007-07-17 04:03:30 -07004131 int ret = -EINVAL;
4132
4133 if (buf[0] == '1') {
4134 ret = validate_slab_cache(s);
4135 if (ret >= 0)
4136 ret = length;
4137 }
4138 return ret;
Christoph Lameter53e15af2007-05-06 14:49:43 -07004139}
4140SLAB_ATTR(validate);
4141
Christoph Lameter2086d262007-05-06 14:49:46 -07004142static ssize_t shrink_show(struct kmem_cache *s, char *buf)
4143{
4144 return 0;
4145}
4146
4147static ssize_t shrink_store(struct kmem_cache *s,
4148 const char *buf, size_t length)
4149{
4150 if (buf[0] == '1') {
4151 int rc = kmem_cache_shrink(s);
4152
4153 if (rc)
4154 return rc;
4155 } else
4156 return -EINVAL;
4157 return length;
4158}
4159SLAB_ATTR(shrink);
4160
Christoph Lameter88a420e2007-05-06 14:49:45 -07004161static ssize_t alloc_calls_show(struct kmem_cache *s, char *buf)
4162{
4163 if (!(s->flags & SLAB_STORE_USER))
4164 return -ENOSYS;
4165 return list_locations(s, buf, TRACK_ALLOC);
4166}
4167SLAB_ATTR_RO(alloc_calls);
4168
4169static ssize_t free_calls_show(struct kmem_cache *s, char *buf)
4170{
4171 if (!(s->flags & SLAB_STORE_USER))
4172 return -ENOSYS;
4173 return list_locations(s, buf, TRACK_FREE);
4174}
4175SLAB_ATTR_RO(free_calls);
4176
Christoph Lameter81819f02007-05-06 14:49:36 -07004177#ifdef CONFIG_NUMA
Christoph Lameter98246012008-01-07 23:20:26 -08004178static ssize_t remote_node_defrag_ratio_show(struct kmem_cache *s, char *buf)
Christoph Lameter81819f02007-05-06 14:49:36 -07004179{
Christoph Lameter98246012008-01-07 23:20:26 -08004180 return sprintf(buf, "%d\n", s->remote_node_defrag_ratio / 10);
Christoph Lameter81819f02007-05-06 14:49:36 -07004181}
4182
Christoph Lameter98246012008-01-07 23:20:26 -08004183static ssize_t remote_node_defrag_ratio_store(struct kmem_cache *s,
Christoph Lameter81819f02007-05-06 14:49:36 -07004184 const char *buf, size_t length)
4185{
Christoph Lameter0121c6192008-04-29 16:11:12 -07004186 unsigned long ratio;
4187 int err;
Christoph Lameter81819f02007-05-06 14:49:36 -07004188
Christoph Lameter0121c6192008-04-29 16:11:12 -07004189 err = strict_strtoul(buf, 10, &ratio);
4190 if (err)
4191 return err;
4192
Christoph Lametere2cb96b2008-08-19 08:51:22 -05004193 if (ratio <= 100)
Christoph Lameter0121c6192008-04-29 16:11:12 -07004194 s->remote_node_defrag_ratio = ratio * 10;
4195
Christoph Lameter81819f02007-05-06 14:49:36 -07004196 return length;
4197}
Christoph Lameter98246012008-01-07 23:20:26 -08004198SLAB_ATTR(remote_node_defrag_ratio);
Christoph Lameter81819f02007-05-06 14:49:36 -07004199#endif
4200
Christoph Lameter8ff12cf2008-02-07 17:47:41 -08004201#ifdef CONFIG_SLUB_STATS
Christoph Lameter8ff12cf2008-02-07 17:47:41 -08004202static int show_stat(struct kmem_cache *s, char *buf, enum stat_item si)
4203{
4204 unsigned long sum = 0;
4205 int cpu;
4206 int len;
4207 int *data = kmalloc(nr_cpu_ids * sizeof(int), GFP_KERNEL);
4208
4209 if (!data)
4210 return -ENOMEM;
4211
4212 for_each_online_cpu(cpu) {
4213 unsigned x = get_cpu_slab(s, cpu)->stat[si];
4214
4215 data[cpu] = x;
4216 sum += x;
4217 }
4218
4219 len = sprintf(buf, "%lu", sum);
4220
Christoph Lameter50ef37b2008-04-14 18:52:05 +03004221#ifdef CONFIG_SMP
Christoph Lameter8ff12cf2008-02-07 17:47:41 -08004222 for_each_online_cpu(cpu) {
4223 if (data[cpu] && len < PAGE_SIZE - 20)
Christoph Lameter50ef37b2008-04-14 18:52:05 +03004224 len += sprintf(buf + len, " C%d=%u", cpu, data[cpu]);
Christoph Lameter8ff12cf2008-02-07 17:47:41 -08004225 }
Christoph Lameter50ef37b2008-04-14 18:52:05 +03004226#endif
Christoph Lameter8ff12cf2008-02-07 17:47:41 -08004227 kfree(data);
4228 return len + sprintf(buf + len, "\n");
4229}
4230
4231#define STAT_ATTR(si, text) \
4232static ssize_t text##_show(struct kmem_cache *s, char *buf) \
4233{ \
4234 return show_stat(s, buf, si); \
4235} \
4236SLAB_ATTR_RO(text); \
4237
4238STAT_ATTR(ALLOC_FASTPATH, alloc_fastpath);
4239STAT_ATTR(ALLOC_SLOWPATH, alloc_slowpath);
4240STAT_ATTR(FREE_FASTPATH, free_fastpath);
4241STAT_ATTR(FREE_SLOWPATH, free_slowpath);
4242STAT_ATTR(FREE_FROZEN, free_frozen);
4243STAT_ATTR(FREE_ADD_PARTIAL, free_add_partial);
4244STAT_ATTR(FREE_REMOVE_PARTIAL, free_remove_partial);
4245STAT_ATTR(ALLOC_FROM_PARTIAL, alloc_from_partial);
4246STAT_ATTR(ALLOC_SLAB, alloc_slab);
4247STAT_ATTR(ALLOC_REFILL, alloc_refill);
4248STAT_ATTR(FREE_SLAB, free_slab);
4249STAT_ATTR(CPUSLAB_FLUSH, cpuslab_flush);
4250STAT_ATTR(DEACTIVATE_FULL, deactivate_full);
4251STAT_ATTR(DEACTIVATE_EMPTY, deactivate_empty);
4252STAT_ATTR(DEACTIVATE_TO_HEAD, deactivate_to_head);
4253STAT_ATTR(DEACTIVATE_TO_TAIL, deactivate_to_tail);
4254STAT_ATTR(DEACTIVATE_REMOTE_FREES, deactivate_remote_frees);
Christoph Lameter65c33762008-04-14 19:11:40 +03004255STAT_ATTR(ORDER_FALLBACK, order_fallback);
Christoph Lameter8ff12cf2008-02-07 17:47:41 -08004256#endif
4257
Pekka Enberg06428782008-01-07 23:20:27 -08004258static struct attribute *slab_attrs[] = {
Christoph Lameter81819f02007-05-06 14:49:36 -07004259 &slab_size_attr.attr,
4260 &object_size_attr.attr,
4261 &objs_per_slab_attr.attr,
4262 &order_attr.attr,
David Rientjes73d342b2009-02-22 17:40:09 -08004263 &min_partial_attr.attr,
Christoph Lameter81819f02007-05-06 14:49:36 -07004264 &objects_attr.attr,
Christoph Lameter205ab992008-04-14 19:11:40 +03004265 &objects_partial_attr.attr,
4266 &total_objects_attr.attr,
Christoph Lameter81819f02007-05-06 14:49:36 -07004267 &slabs_attr.attr,
4268 &partial_attr.attr,
4269 &cpu_slabs_attr.attr,
4270 &ctor_attr.attr,
Christoph Lameter81819f02007-05-06 14:49:36 -07004271 &aliases_attr.attr,
4272 &align_attr.attr,
4273 &sanity_checks_attr.attr,
4274 &trace_attr.attr,
4275 &hwcache_align_attr.attr,
4276 &reclaim_account_attr.attr,
4277 &destroy_by_rcu_attr.attr,
4278 &red_zone_attr.attr,
4279 &poison_attr.attr,
4280 &store_user_attr.attr,
Christoph Lameter53e15af2007-05-06 14:49:43 -07004281 &validate_attr.attr,
Christoph Lameter2086d262007-05-06 14:49:46 -07004282 &shrink_attr.attr,
Christoph Lameter88a420e2007-05-06 14:49:45 -07004283 &alloc_calls_attr.attr,
4284 &free_calls_attr.attr,
Christoph Lameter81819f02007-05-06 14:49:36 -07004285#ifdef CONFIG_ZONE_DMA
4286 &cache_dma_attr.attr,
4287#endif
4288#ifdef CONFIG_NUMA
Christoph Lameter98246012008-01-07 23:20:26 -08004289 &remote_node_defrag_ratio_attr.attr,
Christoph Lameter81819f02007-05-06 14:49:36 -07004290#endif
Christoph Lameter8ff12cf2008-02-07 17:47:41 -08004291#ifdef CONFIG_SLUB_STATS
4292 &alloc_fastpath_attr.attr,
4293 &alloc_slowpath_attr.attr,
4294 &free_fastpath_attr.attr,
4295 &free_slowpath_attr.attr,
4296 &free_frozen_attr.attr,
4297 &free_add_partial_attr.attr,
4298 &free_remove_partial_attr.attr,
4299 &alloc_from_partial_attr.attr,
4300 &alloc_slab_attr.attr,
4301 &alloc_refill_attr.attr,
4302 &free_slab_attr.attr,
4303 &cpuslab_flush_attr.attr,
4304 &deactivate_full_attr.attr,
4305 &deactivate_empty_attr.attr,
4306 &deactivate_to_head_attr.attr,
4307 &deactivate_to_tail_attr.attr,
4308 &deactivate_remote_frees_attr.attr,
Christoph Lameter65c33762008-04-14 19:11:40 +03004309 &order_fallback_attr.attr,
Christoph Lameter8ff12cf2008-02-07 17:47:41 -08004310#endif
Christoph Lameter81819f02007-05-06 14:49:36 -07004311 NULL
4312};
4313
4314static struct attribute_group slab_attr_group = {
4315 .attrs = slab_attrs,
4316};
4317
4318static ssize_t slab_attr_show(struct kobject *kobj,
4319 struct attribute *attr,
4320 char *buf)
4321{
4322 struct slab_attribute *attribute;
4323 struct kmem_cache *s;
4324 int err;
4325
4326 attribute = to_slab_attr(attr);
4327 s = to_slab(kobj);
4328
4329 if (!attribute->show)
4330 return -EIO;
4331
4332 err = attribute->show(s, buf);
4333
4334 return err;
4335}
4336
4337static ssize_t slab_attr_store(struct kobject *kobj,
4338 struct attribute *attr,
4339 const char *buf, size_t len)
4340{
4341 struct slab_attribute *attribute;
4342 struct kmem_cache *s;
4343 int err;
4344
4345 attribute = to_slab_attr(attr);
4346 s = to_slab(kobj);
4347
4348 if (!attribute->store)
4349 return -EIO;
4350
4351 err = attribute->store(s, buf, len);
4352
4353 return err;
4354}
4355
Christoph Lameter151c6022008-01-07 22:29:05 -08004356static void kmem_cache_release(struct kobject *kobj)
4357{
4358 struct kmem_cache *s = to_slab(kobj);
4359
4360 kfree(s);
4361}
4362
Christoph Lameter81819f02007-05-06 14:49:36 -07004363static struct sysfs_ops slab_sysfs_ops = {
4364 .show = slab_attr_show,
4365 .store = slab_attr_store,
4366};
4367
4368static struct kobj_type slab_ktype = {
4369 .sysfs_ops = &slab_sysfs_ops,
Christoph Lameter151c6022008-01-07 22:29:05 -08004370 .release = kmem_cache_release
Christoph Lameter81819f02007-05-06 14:49:36 -07004371};
4372
4373static int uevent_filter(struct kset *kset, struct kobject *kobj)
4374{
4375 struct kobj_type *ktype = get_ktype(kobj);
4376
4377 if (ktype == &slab_ktype)
4378 return 1;
4379 return 0;
4380}
4381
4382static struct kset_uevent_ops slab_uevent_ops = {
4383 .filter = uevent_filter,
4384};
4385
Greg Kroah-Hartman27c3a312007-11-01 09:29:06 -06004386static struct kset *slab_kset;
Christoph Lameter81819f02007-05-06 14:49:36 -07004387
4388#define ID_STR_LENGTH 64
4389
4390/* Create a unique string id for a slab cache:
Christoph Lameter6446faa2008-02-15 23:45:26 -08004391 *
4392 * Format :[flags-]size
Christoph Lameter81819f02007-05-06 14:49:36 -07004393 */
4394static char *create_unique_id(struct kmem_cache *s)
4395{
4396 char *name = kmalloc(ID_STR_LENGTH, GFP_KERNEL);
4397 char *p = name;
4398
4399 BUG_ON(!name);
4400
4401 *p++ = ':';
4402 /*
4403 * First flags affecting slabcache operations. We will only
4404 * get here for aliasable slabs so we do not need to support
4405 * too many flags. The flags here must cover all flags that
4406 * are matched during merging to guarantee that the id is
4407 * unique.
4408 */
4409 if (s->flags & SLAB_CACHE_DMA)
4410 *p++ = 'd';
4411 if (s->flags & SLAB_RECLAIM_ACCOUNT)
4412 *p++ = 'a';
4413 if (s->flags & SLAB_DEBUG_FREE)
4414 *p++ = 'F';
4415 if (p != name + 1)
4416 *p++ = '-';
4417 p += sprintf(p, "%07d", s->size);
4418 BUG_ON(p > name + ID_STR_LENGTH - 1);
4419 return name;
4420}
4421
4422static int sysfs_slab_add(struct kmem_cache *s)
4423{
4424 int err;
4425 const char *name;
4426 int unmergeable;
4427
4428 if (slab_state < SYSFS)
4429 /* Defer until later */
4430 return 0;
4431
4432 unmergeable = slab_unmergeable(s);
4433 if (unmergeable) {
4434 /*
4435 * Slabcache can never be merged so we can use the name proper.
4436 * This is typically the case for debug situations. In that
4437 * case we can catch duplicate names easily.
4438 */
Greg Kroah-Hartman27c3a312007-11-01 09:29:06 -06004439 sysfs_remove_link(&slab_kset->kobj, s->name);
Christoph Lameter81819f02007-05-06 14:49:36 -07004440 name = s->name;
4441 } else {
4442 /*
4443 * Create a unique name for the slab as a target
4444 * for the symlinks.
4445 */
4446 name = create_unique_id(s);
4447 }
4448
Greg Kroah-Hartman27c3a312007-11-01 09:29:06 -06004449 s->kobj.kset = slab_kset;
Greg Kroah-Hartman1eada112007-12-17 23:05:35 -07004450 err = kobject_init_and_add(&s->kobj, &slab_ktype, NULL, name);
4451 if (err) {
4452 kobject_put(&s->kobj);
Christoph Lameter81819f02007-05-06 14:49:36 -07004453 return err;
Greg Kroah-Hartman1eada112007-12-17 23:05:35 -07004454 }
Christoph Lameter81819f02007-05-06 14:49:36 -07004455
4456 err = sysfs_create_group(&s->kobj, &slab_attr_group);
4457 if (err)
4458 return err;
4459 kobject_uevent(&s->kobj, KOBJ_ADD);
4460 if (!unmergeable) {
4461 /* Setup first alias */
4462 sysfs_slab_alias(s, s->name);
4463 kfree(name);
4464 }
4465 return 0;
4466}
4467
4468static void sysfs_slab_remove(struct kmem_cache *s)
4469{
4470 kobject_uevent(&s->kobj, KOBJ_REMOVE);
4471 kobject_del(&s->kobj);
Christoph Lameter151c6022008-01-07 22:29:05 -08004472 kobject_put(&s->kobj);
Christoph Lameter81819f02007-05-06 14:49:36 -07004473}
4474
4475/*
4476 * Need to buffer aliases during bootup until sysfs becomes
Nick Andrew9f6c708e2008-12-05 14:08:08 +11004477 * available lest we lose that information.
Christoph Lameter81819f02007-05-06 14:49:36 -07004478 */
4479struct saved_alias {
4480 struct kmem_cache *s;
4481 const char *name;
4482 struct saved_alias *next;
4483};
4484
Adrian Bunk5af328a2007-07-17 04:03:27 -07004485static struct saved_alias *alias_list;
Christoph Lameter81819f02007-05-06 14:49:36 -07004486
4487static int sysfs_slab_alias(struct kmem_cache *s, const char *name)
4488{
4489 struct saved_alias *al;
4490
4491 if (slab_state == SYSFS) {
4492 /*
4493 * If we have a leftover link then remove it.
4494 */
Greg Kroah-Hartman27c3a312007-11-01 09:29:06 -06004495 sysfs_remove_link(&slab_kset->kobj, name);
4496 return sysfs_create_link(&slab_kset->kobj, &s->kobj, name);
Christoph Lameter81819f02007-05-06 14:49:36 -07004497 }
4498
4499 al = kmalloc(sizeof(struct saved_alias), GFP_KERNEL);
4500 if (!al)
4501 return -ENOMEM;
4502
4503 al->s = s;
4504 al->name = name;
4505 al->next = alias_list;
4506 alias_list = al;
4507 return 0;
4508}
4509
4510static int __init slab_sysfs_init(void)
4511{
Christoph Lameter5b95a4a2007-07-17 04:03:19 -07004512 struct kmem_cache *s;
Christoph Lameter81819f02007-05-06 14:49:36 -07004513 int err;
4514
Greg Kroah-Hartman0ff21e42007-11-06 10:36:58 -08004515 slab_kset = kset_create_and_add("slab", &slab_uevent_ops, kernel_kobj);
Greg Kroah-Hartman27c3a312007-11-01 09:29:06 -06004516 if (!slab_kset) {
Christoph Lameter81819f02007-05-06 14:49:36 -07004517 printk(KERN_ERR "Cannot register slab subsystem.\n");
4518 return -ENOSYS;
4519 }
4520
Christoph Lameter26a7bd02007-05-09 02:32:39 -07004521 slab_state = SYSFS;
4522
Christoph Lameter5b95a4a2007-07-17 04:03:19 -07004523 list_for_each_entry(s, &slab_caches, list) {
Christoph Lameter26a7bd02007-05-09 02:32:39 -07004524 err = sysfs_slab_add(s);
Christoph Lameter5d540fb2007-08-30 23:56:26 -07004525 if (err)
4526 printk(KERN_ERR "SLUB: Unable to add boot slab %s"
4527 " to sysfs\n", s->name);
Christoph Lameter26a7bd02007-05-09 02:32:39 -07004528 }
Christoph Lameter81819f02007-05-06 14:49:36 -07004529
4530 while (alias_list) {
4531 struct saved_alias *al = alias_list;
4532
4533 alias_list = alias_list->next;
4534 err = sysfs_slab_alias(al->s, al->name);
Christoph Lameter5d540fb2007-08-30 23:56:26 -07004535 if (err)
4536 printk(KERN_ERR "SLUB: Unable to add boot slab alias"
4537 " %s to sysfs\n", s->name);
Christoph Lameter81819f02007-05-06 14:49:36 -07004538 kfree(al);
4539 }
4540
4541 resiliency_test();
4542 return 0;
4543}
4544
4545__initcall(slab_sysfs_init);
Christoph Lameter81819f02007-05-06 14:49:36 -07004546#endif
Pekka J Enberg57ed3ed2008-01-01 17:23:28 +01004547
4548/*
4549 * The /proc/slabinfo ABI
4550 */
Linus Torvalds158a9622008-01-02 13:04:48 -08004551#ifdef CONFIG_SLABINFO
Pekka J Enberg57ed3ed2008-01-01 17:23:28 +01004552static void print_slabinfo_header(struct seq_file *m)
4553{
4554 seq_puts(m, "slabinfo - version: 2.1\n");
4555 seq_puts(m, "# name <active_objs> <num_objs> <objsize> "
4556 "<objperslab> <pagesperslab>");
4557 seq_puts(m, " : tunables <limit> <batchcount> <sharedfactor>");
4558 seq_puts(m, " : slabdata <active_slabs> <num_slabs> <sharedavail>");
4559 seq_putc(m, '\n');
4560}
4561
4562static void *s_start(struct seq_file *m, loff_t *pos)
4563{
4564 loff_t n = *pos;
4565
4566 down_read(&slub_lock);
4567 if (!n)
4568 print_slabinfo_header(m);
4569
4570 return seq_list_start(&slab_caches, *pos);
4571}
4572
4573static void *s_next(struct seq_file *m, void *p, loff_t *pos)
4574{
4575 return seq_list_next(p, &slab_caches, pos);
4576}
4577
4578static void s_stop(struct seq_file *m, void *p)
4579{
4580 up_read(&slub_lock);
4581}
4582
4583static int s_show(struct seq_file *m, void *p)
4584{
4585 unsigned long nr_partials = 0;
4586 unsigned long nr_slabs = 0;
4587 unsigned long nr_inuse = 0;
Christoph Lameter205ab992008-04-14 19:11:40 +03004588 unsigned long nr_objs = 0;
4589 unsigned long nr_free = 0;
Pekka J Enberg57ed3ed2008-01-01 17:23:28 +01004590 struct kmem_cache *s;
4591 int node;
4592
4593 s = list_entry(p, struct kmem_cache, list);
4594
4595 for_each_online_node(node) {
4596 struct kmem_cache_node *n = get_node(s, node);
4597
4598 if (!n)
4599 continue;
4600
4601 nr_partials += n->nr_partial;
4602 nr_slabs += atomic_long_read(&n->nr_slabs);
Christoph Lameter205ab992008-04-14 19:11:40 +03004603 nr_objs += atomic_long_read(&n->total_objects);
4604 nr_free += count_partial(n, count_free);
Pekka J Enberg57ed3ed2008-01-01 17:23:28 +01004605 }
4606
Christoph Lameter205ab992008-04-14 19:11:40 +03004607 nr_inuse = nr_objs - nr_free;
Pekka J Enberg57ed3ed2008-01-01 17:23:28 +01004608
4609 seq_printf(m, "%-17s %6lu %6lu %6u %4u %4d", s->name, nr_inuse,
Christoph Lameter834f3d12008-04-14 19:11:31 +03004610 nr_objs, s->size, oo_objects(s->oo),
4611 (1 << oo_order(s->oo)));
Pekka J Enberg57ed3ed2008-01-01 17:23:28 +01004612 seq_printf(m, " : tunables %4u %4u %4u", 0, 0, 0);
4613 seq_printf(m, " : slabdata %6lu %6lu %6lu", nr_slabs, nr_slabs,
4614 0UL);
4615 seq_putc(m, '\n');
4616 return 0;
4617}
4618
Alexey Dobriyan7b3c3a52008-10-06 02:42:17 +04004619static const struct seq_operations slabinfo_op = {
Pekka J Enberg57ed3ed2008-01-01 17:23:28 +01004620 .start = s_start,
4621 .next = s_next,
4622 .stop = s_stop,
4623 .show = s_show,
4624};
4625
Alexey Dobriyan7b3c3a52008-10-06 02:42:17 +04004626static int slabinfo_open(struct inode *inode, struct file *file)
4627{
4628 return seq_open(file, &slabinfo_op);
4629}
4630
4631static const struct file_operations proc_slabinfo_operations = {
4632 .open = slabinfo_open,
4633 .read = seq_read,
4634 .llseek = seq_lseek,
4635 .release = seq_release,
4636};
4637
4638static int __init slab_proc_init(void)
4639{
4640 proc_create("slabinfo",S_IWUSR|S_IRUGO,NULL,&proc_slabinfo_operations);
4641 return 0;
4642}
4643module_init(slab_proc_init);
Linus Torvalds158a9622008-01-02 13:04:48 -08004644#endif /* CONFIG_SLABINFO */