blob: 46997517406ede0e987388763c9fa1d07875c1db [file] [log] [blame]
Christoph Lameter81819f02007-05-06 14:49:36 -07001/*
2 * SLUB: A slab allocator that limits cache line use instead of queuing
3 * objects in per cpu and per node lists.
4 *
Christoph Lameter881db7f2011-06-01 12:25:53 -05005 * The allocator synchronizes using per slab locks or atomic operatios
6 * and only uses a centralized lock to manage a pool of partial slabs.
Christoph Lameter81819f02007-05-06 14:49:36 -07007 *
Christoph Lametercde53532008-07-04 09:59:22 -07008 * (C) 2007 SGI, Christoph Lameter
Christoph Lameter881db7f2011-06-01 12:25:53 -05009 * (C) 2011 Linux Foundation, Christoph Lameter
Christoph Lameter81819f02007-05-06 14:49:36 -070010 */
11
12#include <linux/mm.h>
Nick Piggin1eb5ac62009-05-05 19:13:44 +100013#include <linux/swap.h> /* struct reclaim_state */
Christoph Lameter81819f02007-05-06 14:49:36 -070014#include <linux/module.h>
15#include <linux/bit_spinlock.h>
16#include <linux/interrupt.h>
17#include <linux/bitops.h>
18#include <linux/slab.h>
Christoph Lameter97d06602012-07-06 15:25:11 -050019#include "slab.h"
Alexey Dobriyan7b3c3a52008-10-06 02:42:17 +040020#include <linux/proc_fs.h>
Andrew Morton3ac38fa2013-04-29 15:08:06 -070021#include <linux/notifier.h>
Christoph Lameter81819f02007-05-06 14:49:36 -070022#include <linux/seq_file.h>
Andrey Ryabinina79316c2015-02-13 14:39:38 -080023#include <linux/kasan.h>
Vegard Nossum5a896d92008-04-04 00:54:48 +020024#include <linux/kmemcheck.h>
Christoph Lameter81819f02007-05-06 14:49:36 -070025#include <linux/cpu.h>
26#include <linux/cpuset.h>
27#include <linux/mempolicy.h>
28#include <linux/ctype.h>
Thomas Gleixner3ac7fe52008-04-30 00:55:01 -070029#include <linux/debugobjects.h>
Christoph Lameter81819f02007-05-06 14:49:36 -070030#include <linux/kallsyms.h>
Yasunori Gotob9049e22007-10-21 16:41:37 -070031#include <linux/memory.h>
Roman Zippelf8bd2252008-05-01 04:34:31 -070032#include <linux/math64.h>
Akinobu Mita773ff602008-12-23 19:37:01 +090033#include <linux/fault-inject.h>
Pekka Enbergbfa71452011-07-07 22:47:01 +030034#include <linux/stacktrace.h>
Christoph Lameter4de900b2012-01-30 15:53:51 -060035#include <linux/prefetch.h>
Glauber Costa2633d7a2012-12-18 14:22:34 -080036#include <linux/memcontrol.h>
Christoph Lameter81819f02007-05-06 14:49:36 -070037
Richard Kennedy4a923792010-10-21 10:29:19 +010038#include <trace/events/kmem.h>
39
Mel Gorman072bb0a2012-07-31 16:43:58 -070040#include "internal.h"
41
Christoph Lameter81819f02007-05-06 14:49:36 -070042/*
43 * Lock order:
Christoph Lameter18004c52012-07-06 15:25:12 -050044 * 1. slab_mutex (Global Mutex)
Christoph Lameter881db7f2011-06-01 12:25:53 -050045 * 2. node->list_lock
46 * 3. slab_lock(page) (Only on some arches and for debugging)
Christoph Lameter81819f02007-05-06 14:49:36 -070047 *
Christoph Lameter18004c52012-07-06 15:25:12 -050048 * slab_mutex
Christoph Lameter881db7f2011-06-01 12:25:53 -050049 *
Christoph Lameter18004c52012-07-06 15:25:12 -050050 * The role of the slab_mutex is to protect the list of all the slabs
Christoph Lameter881db7f2011-06-01 12:25:53 -050051 * and to synchronize major metadata changes to slab cache structures.
52 *
53 * The slab_lock is only used for debugging and on arches that do not
54 * have the ability to do a cmpxchg_double. It only protects the second
55 * double word in the page struct. Meaning
56 * A. page->freelist -> List of object free in a page
57 * B. page->counters -> Counters of objects
58 * C. page->frozen -> frozen state
59 *
60 * If a slab is frozen then it is exempt from list management. It is not
61 * on any list. The processor that froze the slab is the one who can
62 * perform list operations on the page. Other processors may put objects
63 * onto the freelist but the processor that froze the slab is the only
64 * one that can retrieve the objects from the page's freelist.
Christoph Lameter81819f02007-05-06 14:49:36 -070065 *
66 * The list_lock protects the partial and full list on each node and
67 * the partial slab counter. If taken then no new slabs may be added or
68 * removed from the lists nor make the number of partial slabs be modified.
69 * (Note that the total number of slabs is an atomic value that may be
70 * modified without taking the list lock).
71 *
72 * The list_lock is a centralized lock and thus we avoid taking it as
73 * much as possible. As long as SLUB does not have to handle partial
74 * slabs, operations can continue without any centralized lock. F.e.
75 * allocating a long series of objects that fill up slabs does not require
76 * the list lock.
Christoph Lameter81819f02007-05-06 14:49:36 -070077 * Interrupts are disabled during allocation and deallocation in order to
78 * make the slab allocator safe to use in the context of an irq. In addition
79 * interrupts are disabled to ensure that the processor does not change
80 * while handling per_cpu slabs, due to kernel preemption.
81 *
82 * SLUB assigns one slab for allocation to each processor.
83 * Allocations only occur from these slabs called cpu slabs.
84 *
Christoph Lameter672bba32007-05-09 02:32:39 -070085 * Slabs with free elements are kept on a partial list and during regular
86 * operations no list for full slabs is used. If an object in a full slab is
Christoph Lameter81819f02007-05-06 14:49:36 -070087 * freed then the slab will show up again on the partial lists.
Christoph Lameter672bba32007-05-09 02:32:39 -070088 * We track full slabs for debugging purposes though because otherwise we
89 * cannot scan all objects.
Christoph Lameter81819f02007-05-06 14:49:36 -070090 *
91 * Slabs are freed when they become empty. Teardown and setup is
92 * minimal so we rely on the page allocators per cpu caches for
93 * fast frees and allocs.
94 *
95 * Overloading of page flags that are otherwise used for LRU management.
96 *
Christoph Lameter4b6f0752007-05-16 22:10:53 -070097 * PageActive The slab is frozen and exempt from list processing.
98 * This means that the slab is dedicated to a purpose
99 * such as satisfying allocations for a specific
100 * processor. Objects may be freed in the slab while
101 * it is frozen but slab_free will then skip the usual
102 * list operations. It is up to the processor holding
103 * the slab to integrate the slab into the slab lists
104 * when the slab is no longer needed.
105 *
106 * One use of this flag is to mark slabs that are
107 * used for allocations. Then such a slab becomes a cpu
108 * slab. The cpu slab may be equipped with an additional
Christoph Lameterdfb4f092007-10-16 01:26:05 -0700109 * freelist that allows lockless access to
Christoph Lameter894b8782007-05-10 03:15:16 -0700110 * free objects in addition to the regular freelist
111 * that requires the slab lock.
Christoph Lameter81819f02007-05-06 14:49:36 -0700112 *
113 * PageError Slab requires special handling due to debug
114 * options set. This moves slab handling out of
Christoph Lameter894b8782007-05-10 03:15:16 -0700115 * the fast path and disables lockless freelists.
Christoph Lameter81819f02007-05-06 14:49:36 -0700116 */
117
Christoph Lameteraf537b02010-07-09 14:07:14 -0500118static inline int kmem_cache_debug(struct kmem_cache *s)
119{
Christoph Lameter5577bd82007-05-16 22:10:56 -0700120#ifdef CONFIG_SLUB_DEBUG
Christoph Lameteraf537b02010-07-09 14:07:14 -0500121 return unlikely(s->flags & SLAB_DEBUG_FLAGS);
Christoph Lameter5577bd82007-05-16 22:10:56 -0700122#else
Christoph Lameteraf537b02010-07-09 14:07:14 -0500123 return 0;
Christoph Lameter5577bd82007-05-16 22:10:56 -0700124#endif
Christoph Lameteraf537b02010-07-09 14:07:14 -0500125}
Christoph Lameter5577bd82007-05-16 22:10:56 -0700126
Joonsoo Kim345c9052013-06-19 14:05:52 +0900127static inline bool kmem_cache_has_cpu_partial(struct kmem_cache *s)
128{
129#ifdef CONFIG_SLUB_CPU_PARTIAL
130 return !kmem_cache_debug(s);
131#else
132 return false;
133#endif
134}
135
Christoph Lameter81819f02007-05-06 14:49:36 -0700136/*
137 * Issues still to be resolved:
138 *
Christoph Lameter81819f02007-05-06 14:49:36 -0700139 * - Support PAGE_ALLOC_DEBUG. Should be easy to do.
140 *
Christoph Lameter81819f02007-05-06 14:49:36 -0700141 * - Variable sizing of the per node arrays
142 */
143
144/* Enable to test recovery from slab corruption on boot */
145#undef SLUB_RESILIENCY_TEST
146
Christoph Lameterb789ef52011-06-01 12:25:49 -0500147/* Enable to log cmpxchg failures */
148#undef SLUB_DEBUG_CMPXCHG
149
Christoph Lameter81819f02007-05-06 14:49:36 -0700150/*
Christoph Lameter2086d262007-05-06 14:49:46 -0700151 * Mininum number of partial slabs. These will be left on the partial
152 * lists even if they are empty. kmem_cache_shrink may reclaim them.
153 */
Christoph Lameter76be8952007-12-21 14:37:37 -0800154#define MIN_PARTIAL 5
Christoph Lametere95eed52007-05-06 14:49:44 -0700155
Christoph Lameter2086d262007-05-06 14:49:46 -0700156/*
157 * Maximum number of desirable partial slabs.
158 * The existence of more partial slabs makes kmem_cache_shrink
Zhi Yong Wu721ae222013-11-08 20:47:37 +0800159 * sort the partial list by the number of objects in use.
Christoph Lameter2086d262007-05-06 14:49:46 -0700160 */
161#define MAX_PARTIAL 10
162
Christoph Lameter81819f02007-05-06 14:49:36 -0700163#define DEBUG_DEFAULT_FLAGS (SLAB_DEBUG_FREE | SLAB_RED_ZONE | \
164 SLAB_POISON | SLAB_STORE_USER)
Christoph Lameter672bba32007-05-09 02:32:39 -0700165
Christoph Lameter81819f02007-05-06 14:49:36 -0700166/*
David Rientjes3de47212009-07-27 18:30:35 -0700167 * Debugging flags that require metadata to be stored in the slab. These get
168 * disabled when slub_debug=O is used and a cache's min order increases with
169 * metadata.
David Rientjesfa5ec8a2009-07-07 00:14:14 -0700170 */
David Rientjes3de47212009-07-27 18:30:35 -0700171#define DEBUG_METADATA_FLAGS (SLAB_RED_ZONE | SLAB_POISON | SLAB_STORE_USER)
David Rientjesfa5ec8a2009-07-07 00:14:14 -0700172
Cyrill Gorcunov210b5c02008-10-22 23:00:38 +0400173#define OO_SHIFT 16
174#define OO_MASK ((1 << OO_SHIFT) - 1)
Christoph Lameter50d5c412011-06-01 12:25:45 -0500175#define MAX_OBJS_PER_PAGE 32767 /* since page.objects is u15 */
Cyrill Gorcunov210b5c02008-10-22 23:00:38 +0400176
Christoph Lameter81819f02007-05-06 14:49:36 -0700177/* Internal SLUB flags */
Christoph Lameterf90ec392010-07-09 14:07:11 -0500178#define __OBJECT_POISON 0x80000000UL /* Poison object */
Christoph Lameterb789ef52011-06-01 12:25:49 -0500179#define __CMPXCHG_DOUBLE 0x40000000UL /* Use cmpxchg_double */
Christoph Lameter81819f02007-05-06 14:49:36 -0700180
Christoph Lameter81819f02007-05-06 14:49:36 -0700181#ifdef CONFIG_SMP
182static struct notifier_block slab_notifier;
183#endif
184
Christoph Lameter02cbc872007-05-09 02:32:43 -0700185/*
186 * Tracking user of a slab.
187 */
Ben Greeard6543e32011-07-07 11:36:36 -0700188#define TRACK_ADDRS_COUNT 16
Christoph Lameter02cbc872007-05-09 02:32:43 -0700189struct track {
Eduard - Gabriel Munteanuce71e272008-08-19 20:43:25 +0300190 unsigned long addr; /* Called from address */
Ben Greeard6543e32011-07-07 11:36:36 -0700191#ifdef CONFIG_STACKTRACE
192 unsigned long addrs[TRACK_ADDRS_COUNT]; /* Called from address */
193#endif
Christoph Lameter02cbc872007-05-09 02:32:43 -0700194 int cpu; /* Was running on cpu */
195 int pid; /* Pid context */
196 unsigned long when; /* When did the operation occur */
197};
198
199enum track_item { TRACK_ALLOC, TRACK_FREE };
200
Christoph Lameterab4d5ed2010-10-05 13:57:26 -0500201#ifdef CONFIG_SYSFS
Christoph Lameter81819f02007-05-06 14:49:36 -0700202static int sysfs_slab_add(struct kmem_cache *);
203static int sysfs_slab_alias(struct kmem_cache *, const char *);
Glauber Costa107dab52012-12-18 14:23:05 -0800204static void memcg_propagate_slab_attrs(struct kmem_cache *s);
Christoph Lameter81819f02007-05-06 14:49:36 -0700205#else
Christoph Lameter0c710012007-07-17 04:03:24 -0700206static inline int sysfs_slab_add(struct kmem_cache *s) { return 0; }
207static inline int sysfs_slab_alias(struct kmem_cache *s, const char *p)
208 { return 0; }
Glauber Costa107dab52012-12-18 14:23:05 -0800209static inline void memcg_propagate_slab_attrs(struct kmem_cache *s) { }
Christoph Lameter81819f02007-05-06 14:49:36 -0700210#endif
211
Christoph Lameter4fdccdf2011-03-22 13:35:00 -0500212static inline void stat(const struct kmem_cache *s, enum stat_item si)
Christoph Lameter8ff12cf2008-02-07 17:47:41 -0800213{
214#ifdef CONFIG_SLUB_STATS
Christoph Lameter88da03a2014-04-07 15:39:42 -0700215 /*
216 * The rmw is racy on a preemptible kernel but this is acceptable, so
217 * avoid this_cpu_add()'s irq-disable overhead.
218 */
219 raw_cpu_inc(s->cpu_slab->stat[si]);
Christoph Lameter8ff12cf2008-02-07 17:47:41 -0800220#endif
221}
222
Christoph Lameter81819f02007-05-06 14:49:36 -0700223/********************************************************************
224 * Core slab cache functions
225 *******************************************************************/
226
Christoph Lameter6446faa2008-02-15 23:45:26 -0800227/* Verify that a pointer has an address that is valid within a slab page */
Christoph Lameter02cbc872007-05-09 02:32:43 -0700228static inline int check_valid_pointer(struct kmem_cache *s,
229 struct page *page, const void *object)
230{
231 void *base;
232
Christoph Lametera973e9d2008-03-01 13:40:44 -0800233 if (!object)
Christoph Lameter02cbc872007-05-09 02:32:43 -0700234 return 1;
235
Christoph Lametera973e9d2008-03-01 13:40:44 -0800236 base = page_address(page);
Christoph Lameter39b26462008-04-14 19:11:30 +0300237 if (object < base || object >= base + page->objects * s->size ||
Christoph Lameter02cbc872007-05-09 02:32:43 -0700238 (object - base) % s->size) {
239 return 0;
240 }
241
242 return 1;
243}
244
Christoph Lameter7656c722007-05-09 02:32:40 -0700245static inline void *get_freepointer(struct kmem_cache *s, void *object)
246{
247 return *(void **)(object + s->offset);
248}
249
Eric Dumazet0ad95002011-12-16 16:25:34 +0100250static void prefetch_freepointer(const struct kmem_cache *s, void *object)
251{
252 prefetch(object + s->offset);
253}
254
Christoph Lameter1393d9a2011-05-16 15:26:08 -0500255static inline void *get_freepointer_safe(struct kmem_cache *s, void *object)
256{
257 void *p;
258
259#ifdef CONFIG_DEBUG_PAGEALLOC
260 probe_kernel_read(&p, (void **)(object + s->offset), sizeof(p));
261#else
262 p = get_freepointer(s, object);
263#endif
264 return p;
265}
266
Christoph Lameter7656c722007-05-09 02:32:40 -0700267static inline void set_freepointer(struct kmem_cache *s, void *object, void *fp)
268{
269 *(void **)(object + s->offset) = fp;
270}
271
272/* Loop over all objects in a slab */
Christoph Lameter224a88b2008-04-14 19:11:31 +0300273#define for_each_object(__p, __s, __addr, __objects) \
274 for (__p = (__addr); __p < (__addr) + (__objects) * (__s)->size;\
Christoph Lameter7656c722007-05-09 02:32:40 -0700275 __p += (__s)->size)
276
Wei Yang54266642014-08-06 16:04:42 -0700277#define for_each_object_idx(__p, __idx, __s, __addr, __objects) \
278 for (__p = (__addr), __idx = 1; __idx <= __objects;\
279 __p += (__s)->size, __idx++)
280
Christoph Lameter7656c722007-05-09 02:32:40 -0700281/* Determine object index from a given position */
282static inline int slab_index(void *p, struct kmem_cache *s, void *addr)
283{
284 return (p - addr) / s->size;
285}
286
Mariusz Kozlowskid71f6062011-02-26 20:10:26 +0100287static inline size_t slab_ksize(const struct kmem_cache *s)
288{
289#ifdef CONFIG_SLUB_DEBUG
290 /*
291 * Debugging requires use of the padding between object
292 * and whatever may come after it.
293 */
294 if (s->flags & (SLAB_RED_ZONE | SLAB_POISON))
Christoph Lameter3b0efdf2012-06-13 10:24:57 -0500295 return s->object_size;
Mariusz Kozlowskid71f6062011-02-26 20:10:26 +0100296
297#endif
298 /*
299 * If we have the need to store the freelist pointer
300 * back there or track user information then we can
301 * only use the space before that information.
302 */
303 if (s->flags & (SLAB_DESTROY_BY_RCU | SLAB_STORE_USER))
304 return s->inuse;
305 /*
306 * Else we can use all the padding etc for the allocation
307 */
308 return s->size;
309}
310
Lai Jiangshanab9a0f12011-03-10 15:21:48 +0800311static inline int order_objects(int order, unsigned long size, int reserved)
312{
313 return ((PAGE_SIZE << order) - reserved) / size;
314}
315
Christoph Lameter834f3d12008-04-14 19:11:31 +0300316static inline struct kmem_cache_order_objects oo_make(int order,
Lai Jiangshanab9a0f12011-03-10 15:21:48 +0800317 unsigned long size, int reserved)
Christoph Lameter834f3d12008-04-14 19:11:31 +0300318{
319 struct kmem_cache_order_objects x = {
Lai Jiangshanab9a0f12011-03-10 15:21:48 +0800320 (order << OO_SHIFT) + order_objects(order, size, reserved)
Christoph Lameter834f3d12008-04-14 19:11:31 +0300321 };
322
323 return x;
324}
325
326static inline int oo_order(struct kmem_cache_order_objects x)
327{
Cyrill Gorcunov210b5c02008-10-22 23:00:38 +0400328 return x.x >> OO_SHIFT;
Christoph Lameter834f3d12008-04-14 19:11:31 +0300329}
330
331static inline int oo_objects(struct kmem_cache_order_objects x)
332{
Cyrill Gorcunov210b5c02008-10-22 23:00:38 +0400333 return x.x & OO_MASK;
Christoph Lameter834f3d12008-04-14 19:11:31 +0300334}
335
Christoph Lameter881db7f2011-06-01 12:25:53 -0500336/*
337 * Per slab locking using the pagelock
338 */
339static __always_inline void slab_lock(struct page *page)
340{
341 bit_spin_lock(PG_locked, &page->flags);
342}
343
344static __always_inline void slab_unlock(struct page *page)
345{
346 __bit_spin_unlock(PG_locked, &page->flags);
347}
348
Dave Hansena0320862014-01-30 15:46:09 -0800349static inline void set_page_slub_counters(struct page *page, unsigned long counters_new)
350{
351 struct page tmp;
352 tmp.counters = counters_new;
353 /*
354 * page->counters can cover frozen/inuse/objects as well
355 * as page->_count. If we assign to ->counters directly
356 * we run the risk of losing updates to page->_count, so
357 * be careful and only assign to the fields we need.
358 */
359 page->frozen = tmp.frozen;
360 page->inuse = tmp.inuse;
361 page->objects = tmp.objects;
362}
363
Christoph Lameter1d071712011-07-14 12:49:12 -0500364/* Interrupts must be disabled (for the fallback code to work right) */
365static inline bool __cmpxchg_double_slab(struct kmem_cache *s, struct page *page,
Christoph Lameterb789ef52011-06-01 12:25:49 -0500366 void *freelist_old, unsigned long counters_old,
367 void *freelist_new, unsigned long counters_new,
368 const char *n)
369{
Christoph Lameter1d071712011-07-14 12:49:12 -0500370 VM_BUG_ON(!irqs_disabled());
Heiko Carstens25654092012-01-12 17:17:33 -0800371#if defined(CONFIG_HAVE_CMPXCHG_DOUBLE) && \
372 defined(CONFIG_HAVE_ALIGNED_STRUCT_PAGE)
Christoph Lameterb789ef52011-06-01 12:25:49 -0500373 if (s->flags & __CMPXCHG_DOUBLE) {
Jan Beulichcdcd6292012-01-02 17:02:18 +0000374 if (cmpxchg_double(&page->freelist, &page->counters,
Dan Carpenter0aa9a132014-08-06 16:04:48 -0700375 freelist_old, counters_old,
376 freelist_new, counters_new))
Joe Perches6f6528a2015-04-14 15:44:31 -0700377 return true;
Christoph Lameterb789ef52011-06-01 12:25:49 -0500378 } else
379#endif
380 {
Christoph Lameter881db7f2011-06-01 12:25:53 -0500381 slab_lock(page);
Chen Gangd0e0ac92013-07-15 09:05:29 +0800382 if (page->freelist == freelist_old &&
383 page->counters == counters_old) {
Christoph Lameterb789ef52011-06-01 12:25:49 -0500384 page->freelist = freelist_new;
Dave Hansena0320862014-01-30 15:46:09 -0800385 set_page_slub_counters(page, counters_new);
Christoph Lameter881db7f2011-06-01 12:25:53 -0500386 slab_unlock(page);
Joe Perches6f6528a2015-04-14 15:44:31 -0700387 return true;
Christoph Lameterb789ef52011-06-01 12:25:49 -0500388 }
Christoph Lameter881db7f2011-06-01 12:25:53 -0500389 slab_unlock(page);
Christoph Lameterb789ef52011-06-01 12:25:49 -0500390 }
391
392 cpu_relax();
393 stat(s, CMPXCHG_DOUBLE_FAIL);
394
395#ifdef SLUB_DEBUG_CMPXCHG
Fabian Frederickf9f58282014-06-04 16:06:34 -0700396 pr_info("%s %s: cmpxchg double redo ", n, s->name);
Christoph Lameterb789ef52011-06-01 12:25:49 -0500397#endif
398
Joe Perches6f6528a2015-04-14 15:44:31 -0700399 return false;
Christoph Lameterb789ef52011-06-01 12:25:49 -0500400}
401
Christoph Lameter1d071712011-07-14 12:49:12 -0500402static inline bool cmpxchg_double_slab(struct kmem_cache *s, struct page *page,
403 void *freelist_old, unsigned long counters_old,
404 void *freelist_new, unsigned long counters_new,
405 const char *n)
406{
Heiko Carstens25654092012-01-12 17:17:33 -0800407#if defined(CONFIG_HAVE_CMPXCHG_DOUBLE) && \
408 defined(CONFIG_HAVE_ALIGNED_STRUCT_PAGE)
Christoph Lameter1d071712011-07-14 12:49:12 -0500409 if (s->flags & __CMPXCHG_DOUBLE) {
Jan Beulichcdcd6292012-01-02 17:02:18 +0000410 if (cmpxchg_double(&page->freelist, &page->counters,
Dan Carpenter0aa9a132014-08-06 16:04:48 -0700411 freelist_old, counters_old,
412 freelist_new, counters_new))
Joe Perches6f6528a2015-04-14 15:44:31 -0700413 return true;
Christoph Lameter1d071712011-07-14 12:49:12 -0500414 } else
415#endif
416 {
417 unsigned long flags;
418
419 local_irq_save(flags);
420 slab_lock(page);
Chen Gangd0e0ac92013-07-15 09:05:29 +0800421 if (page->freelist == freelist_old &&
422 page->counters == counters_old) {
Christoph Lameter1d071712011-07-14 12:49:12 -0500423 page->freelist = freelist_new;
Dave Hansena0320862014-01-30 15:46:09 -0800424 set_page_slub_counters(page, counters_new);
Christoph Lameter1d071712011-07-14 12:49:12 -0500425 slab_unlock(page);
426 local_irq_restore(flags);
Joe Perches6f6528a2015-04-14 15:44:31 -0700427 return true;
Christoph Lameter1d071712011-07-14 12:49:12 -0500428 }
429 slab_unlock(page);
430 local_irq_restore(flags);
431 }
432
433 cpu_relax();
434 stat(s, CMPXCHG_DOUBLE_FAIL);
435
436#ifdef SLUB_DEBUG_CMPXCHG
Fabian Frederickf9f58282014-06-04 16:06:34 -0700437 pr_info("%s %s: cmpxchg double redo ", n, s->name);
Christoph Lameter1d071712011-07-14 12:49:12 -0500438#endif
439
Joe Perches6f6528a2015-04-14 15:44:31 -0700440 return false;
Christoph Lameter1d071712011-07-14 12:49:12 -0500441}
442
Christoph Lameter41ecc552007-05-09 02:32:44 -0700443#ifdef CONFIG_SLUB_DEBUG
444/*
Christoph Lameter5f80b132011-04-15 14:48:13 -0500445 * Determine a map of object in use on a page.
446 *
Christoph Lameter881db7f2011-06-01 12:25:53 -0500447 * Node listlock must be held to guarantee that the page does
Christoph Lameter5f80b132011-04-15 14:48:13 -0500448 * not vanish from under us.
449 */
450static void get_map(struct kmem_cache *s, struct page *page, unsigned long *map)
451{
452 void *p;
453 void *addr = page_address(page);
454
455 for (p = page->freelist; p; p = get_freepointer(s, p))
456 set_bit(slab_index(p, s, addr), map);
457}
458
Christoph Lameter41ecc552007-05-09 02:32:44 -0700459/*
460 * Debug settings:
461 */
Andrey Ryabinin89d3c872015-11-05 18:51:23 -0800462#if defined(CONFIG_SLUB_DEBUG_ON)
Christoph Lameterf0630ff2007-07-15 23:38:14 -0700463static int slub_debug = DEBUG_DEFAULT_FLAGS;
Andrey Ryabinin89d3c872015-11-05 18:51:23 -0800464#elif defined(CONFIG_KASAN)
465static int slub_debug = SLAB_STORE_USER;
Christoph Lameterf0630ff2007-07-15 23:38:14 -0700466#else
Christoph Lameter41ecc552007-05-09 02:32:44 -0700467static int slub_debug;
Christoph Lameterf0630ff2007-07-15 23:38:14 -0700468#endif
Christoph Lameter41ecc552007-05-09 02:32:44 -0700469
470static char *slub_debug_slabs;
David Rientjesfa5ec8a2009-07-07 00:14:14 -0700471static int disable_higher_order_debug;
Christoph Lameter41ecc552007-05-09 02:32:44 -0700472
Christoph Lameter7656c722007-05-09 02:32:40 -0700473/*
Andrey Ryabinina79316c2015-02-13 14:39:38 -0800474 * slub is about to manipulate internal object metadata. This memory lies
475 * outside the range of the allocated object, so accessing it would normally
476 * be reported by kasan as a bounds error. metadata_access_enable() is used
477 * to tell kasan that these accesses are OK.
478 */
479static inline void metadata_access_enable(void)
480{
481 kasan_disable_current();
482}
483
484static inline void metadata_access_disable(void)
485{
486 kasan_enable_current();
487}
488
489/*
Christoph Lameter81819f02007-05-06 14:49:36 -0700490 * Object debugging
491 */
492static void print_section(char *text, u8 *addr, unsigned int length)
493{
Andrey Ryabinina79316c2015-02-13 14:39:38 -0800494 metadata_access_enable();
Sebastian Andrzej Siewiorffc79d22011-07-29 14:10:20 +0200495 print_hex_dump(KERN_ERR, text, DUMP_PREFIX_ADDRESS, 16, 1, addr,
496 length, 1);
Andrey Ryabinina79316c2015-02-13 14:39:38 -0800497 metadata_access_disable();
Christoph Lameter81819f02007-05-06 14:49:36 -0700498}
499
Christoph Lameter81819f02007-05-06 14:49:36 -0700500static struct track *get_track(struct kmem_cache *s, void *object,
501 enum track_item alloc)
502{
503 struct track *p;
504
505 if (s->offset)
506 p = object + s->offset + sizeof(void *);
507 else
508 p = object + s->inuse;
509
510 return p + alloc;
511}
512
513static void set_track(struct kmem_cache *s, void *object,
Eduard - Gabriel Munteanuce71e272008-08-19 20:43:25 +0300514 enum track_item alloc, unsigned long addr)
Christoph Lameter81819f02007-05-06 14:49:36 -0700515{
Akinobu Mita1a00df42009-03-07 00:36:21 +0900516 struct track *p = get_track(s, object, alloc);
Christoph Lameter81819f02007-05-06 14:49:36 -0700517
Christoph Lameter81819f02007-05-06 14:49:36 -0700518 if (addr) {
Ben Greeard6543e32011-07-07 11:36:36 -0700519#ifdef CONFIG_STACKTRACE
520 struct stack_trace trace;
521 int i;
522
523 trace.nr_entries = 0;
524 trace.max_entries = TRACK_ADDRS_COUNT;
525 trace.entries = p->addrs;
526 trace.skip = 3;
Andrey Ryabinina79316c2015-02-13 14:39:38 -0800527 metadata_access_enable();
Ben Greeard6543e32011-07-07 11:36:36 -0700528 save_stack_trace(&trace);
Andrey Ryabinina79316c2015-02-13 14:39:38 -0800529 metadata_access_disable();
Ben Greeard6543e32011-07-07 11:36:36 -0700530
531 /* See rant in lockdep.c */
532 if (trace.nr_entries != 0 &&
533 trace.entries[trace.nr_entries - 1] == ULONG_MAX)
534 trace.nr_entries--;
535
536 for (i = trace.nr_entries; i < TRACK_ADDRS_COUNT; i++)
537 p->addrs[i] = 0;
538#endif
Christoph Lameter81819f02007-05-06 14:49:36 -0700539 p->addr = addr;
540 p->cpu = smp_processor_id();
Alexey Dobriyan88e4ccf2008-06-23 02:58:37 +0400541 p->pid = current->pid;
Christoph Lameter81819f02007-05-06 14:49:36 -0700542 p->when = jiffies;
543 } else
544 memset(p, 0, sizeof(struct track));
545}
546
Christoph Lameter81819f02007-05-06 14:49:36 -0700547static void init_tracking(struct kmem_cache *s, void *object)
548{
Christoph Lameter24922682007-07-17 04:03:18 -0700549 if (!(s->flags & SLAB_STORE_USER))
550 return;
551
Eduard - Gabriel Munteanuce71e272008-08-19 20:43:25 +0300552 set_track(s, object, TRACK_FREE, 0UL);
553 set_track(s, object, TRACK_ALLOC, 0UL);
Christoph Lameter81819f02007-05-06 14:49:36 -0700554}
555
556static void print_track(const char *s, struct track *t)
557{
558 if (!t->addr)
559 return;
560
Fabian Frederickf9f58282014-06-04 16:06:34 -0700561 pr_err("INFO: %s in %pS age=%lu cpu=%u pid=%d\n",
562 s, (void *)t->addr, jiffies - t->when, t->cpu, t->pid);
Ben Greeard6543e32011-07-07 11:36:36 -0700563#ifdef CONFIG_STACKTRACE
564 {
565 int i;
566 for (i = 0; i < TRACK_ADDRS_COUNT; i++)
567 if (t->addrs[i])
Fabian Frederickf9f58282014-06-04 16:06:34 -0700568 pr_err("\t%pS\n", (void *)t->addrs[i]);
Ben Greeard6543e32011-07-07 11:36:36 -0700569 else
570 break;
571 }
572#endif
Christoph Lameter81819f02007-05-06 14:49:36 -0700573}
574
Christoph Lameter24922682007-07-17 04:03:18 -0700575static void print_tracking(struct kmem_cache *s, void *object)
576{
577 if (!(s->flags & SLAB_STORE_USER))
578 return;
579
580 print_track("Allocated", get_track(s, object, TRACK_ALLOC));
581 print_track("Freed", get_track(s, object, TRACK_FREE));
582}
583
584static void print_page_info(struct page *page)
585{
Fabian Frederickf9f58282014-06-04 16:06:34 -0700586 pr_err("INFO: Slab 0x%p objects=%u used=%u fp=0x%p flags=0x%04lx\n",
Chen Gangd0e0ac92013-07-15 09:05:29 +0800587 page, page->objects, page->inuse, page->freelist, page->flags);
Christoph Lameter24922682007-07-17 04:03:18 -0700588
589}
590
591static void slab_bug(struct kmem_cache *s, char *fmt, ...)
592{
Fabian Frederickecc42fb2014-06-04 16:06:35 -0700593 struct va_format vaf;
Christoph Lameter24922682007-07-17 04:03:18 -0700594 va_list args;
Christoph Lameter24922682007-07-17 04:03:18 -0700595
596 va_start(args, fmt);
Fabian Frederickecc42fb2014-06-04 16:06:35 -0700597 vaf.fmt = fmt;
598 vaf.va = &args;
Fabian Frederickf9f58282014-06-04 16:06:34 -0700599 pr_err("=============================================================================\n");
Fabian Frederickecc42fb2014-06-04 16:06:35 -0700600 pr_err("BUG %s (%s): %pV\n", s->name, print_tainted(), &vaf);
Fabian Frederickf9f58282014-06-04 16:06:34 -0700601 pr_err("-----------------------------------------------------------------------------\n\n");
Dave Jones645df232012-09-18 15:54:12 -0400602
Rusty Russell373d4d02013-01-21 17:17:39 +1030603 add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE);
Fabian Frederickecc42fb2014-06-04 16:06:35 -0700604 va_end(args);
Christoph Lameter24922682007-07-17 04:03:18 -0700605}
606
607static void slab_fix(struct kmem_cache *s, char *fmt, ...)
608{
Fabian Frederickecc42fb2014-06-04 16:06:35 -0700609 struct va_format vaf;
Christoph Lameter24922682007-07-17 04:03:18 -0700610 va_list args;
Christoph Lameter24922682007-07-17 04:03:18 -0700611
612 va_start(args, fmt);
Fabian Frederickecc42fb2014-06-04 16:06:35 -0700613 vaf.fmt = fmt;
614 vaf.va = &args;
615 pr_err("FIX %s: %pV\n", s->name, &vaf);
Christoph Lameter24922682007-07-17 04:03:18 -0700616 va_end(args);
Christoph Lameter24922682007-07-17 04:03:18 -0700617}
618
619static void print_trailer(struct kmem_cache *s, struct page *page, u8 *p)
Christoph Lameter81819f02007-05-06 14:49:36 -0700620{
621 unsigned int off; /* Offset of last byte */
Christoph Lametera973e9d2008-03-01 13:40:44 -0800622 u8 *addr = page_address(page);
Christoph Lameter24922682007-07-17 04:03:18 -0700623
624 print_tracking(s, p);
625
626 print_page_info(page);
627
Fabian Frederickf9f58282014-06-04 16:06:34 -0700628 pr_err("INFO: Object 0x%p @offset=%tu fp=0x%p\n\n",
629 p, p - addr, get_freepointer(s, p));
Christoph Lameter24922682007-07-17 04:03:18 -0700630
631 if (p > addr + 16)
Sebastian Andrzej Siewiorffc79d22011-07-29 14:10:20 +0200632 print_section("Bytes b4 ", p - 16, 16);
Christoph Lameter24922682007-07-17 04:03:18 -0700633
Christoph Lameter3b0efdf2012-06-13 10:24:57 -0500634 print_section("Object ", p, min_t(unsigned long, s->object_size,
Sebastian Andrzej Siewiorffc79d22011-07-29 14:10:20 +0200635 PAGE_SIZE));
Christoph Lameter81819f02007-05-06 14:49:36 -0700636 if (s->flags & SLAB_RED_ZONE)
Christoph Lameter3b0efdf2012-06-13 10:24:57 -0500637 print_section("Redzone ", p + s->object_size,
638 s->inuse - s->object_size);
Christoph Lameter81819f02007-05-06 14:49:36 -0700639
Christoph Lameter81819f02007-05-06 14:49:36 -0700640 if (s->offset)
641 off = s->offset + sizeof(void *);
642 else
643 off = s->inuse;
644
Christoph Lameter24922682007-07-17 04:03:18 -0700645 if (s->flags & SLAB_STORE_USER)
Christoph Lameter81819f02007-05-06 14:49:36 -0700646 off += 2 * sizeof(struct track);
Christoph Lameter81819f02007-05-06 14:49:36 -0700647
648 if (off != s->size)
649 /* Beginning of the filler is the free pointer */
Sebastian Andrzej Siewiorffc79d22011-07-29 14:10:20 +0200650 print_section("Padding ", p + off, s->size - off);
Christoph Lameter24922682007-07-17 04:03:18 -0700651
652 dump_stack();
Christoph Lameter81819f02007-05-06 14:49:36 -0700653}
654
Andrey Ryabinin75c66de2015-02-13 14:39:35 -0800655void object_err(struct kmem_cache *s, struct page *page,
Christoph Lameter81819f02007-05-06 14:49:36 -0700656 u8 *object, char *reason)
657{
Christoph Lameter3dc50632008-04-23 12:28:01 -0700658 slab_bug(s, "%s", reason);
Christoph Lameter24922682007-07-17 04:03:18 -0700659 print_trailer(s, page, object);
Christoph Lameter81819f02007-05-06 14:49:36 -0700660}
661
Chen Gangd0e0ac92013-07-15 09:05:29 +0800662static void slab_err(struct kmem_cache *s, struct page *page,
663 const char *fmt, ...)
Christoph Lameter81819f02007-05-06 14:49:36 -0700664{
665 va_list args;
666 char buf[100];
667
Christoph Lameter24922682007-07-17 04:03:18 -0700668 va_start(args, fmt);
669 vsnprintf(buf, sizeof(buf), fmt, args);
Christoph Lameter81819f02007-05-06 14:49:36 -0700670 va_end(args);
Christoph Lameter3dc50632008-04-23 12:28:01 -0700671 slab_bug(s, "%s", buf);
Christoph Lameter24922682007-07-17 04:03:18 -0700672 print_page_info(page);
Christoph Lameter81819f02007-05-06 14:49:36 -0700673 dump_stack();
674}
675
Christoph Lameterf7cb1932010-09-29 07:15:01 -0500676static void init_object(struct kmem_cache *s, void *object, u8 val)
Christoph Lameter81819f02007-05-06 14:49:36 -0700677{
678 u8 *p = object;
679
680 if (s->flags & __OBJECT_POISON) {
Christoph Lameter3b0efdf2012-06-13 10:24:57 -0500681 memset(p, POISON_FREE, s->object_size - 1);
682 p[s->object_size - 1] = POISON_END;
Christoph Lameter81819f02007-05-06 14:49:36 -0700683 }
684
685 if (s->flags & SLAB_RED_ZONE)
Christoph Lameter3b0efdf2012-06-13 10:24:57 -0500686 memset(p + s->object_size, val, s->inuse - s->object_size);
Christoph Lameter81819f02007-05-06 14:49:36 -0700687}
688
Christoph Lameter24922682007-07-17 04:03:18 -0700689static void restore_bytes(struct kmem_cache *s, char *message, u8 data,
690 void *from, void *to)
691{
692 slab_fix(s, "Restoring 0x%p-0x%p=0x%x\n", from, to - 1, data);
693 memset(from, data, to - from);
694}
695
696static int check_bytes_and_report(struct kmem_cache *s, struct page *page,
697 u8 *object, char *what,
Pekka Enberg06428782008-01-07 23:20:27 -0800698 u8 *start, unsigned int value, unsigned int bytes)
Christoph Lameter24922682007-07-17 04:03:18 -0700699{
700 u8 *fault;
701 u8 *end;
702
Andrey Ryabinina79316c2015-02-13 14:39:38 -0800703 metadata_access_enable();
Akinobu Mita798248202011-10-31 17:08:07 -0700704 fault = memchr_inv(start, value, bytes);
Andrey Ryabinina79316c2015-02-13 14:39:38 -0800705 metadata_access_disable();
Christoph Lameter24922682007-07-17 04:03:18 -0700706 if (!fault)
707 return 1;
708
709 end = start + bytes;
710 while (end > fault && end[-1] == value)
711 end--;
712
713 slab_bug(s, "%s overwritten", what);
Fabian Frederickf9f58282014-06-04 16:06:34 -0700714 pr_err("INFO: 0x%p-0x%p. First byte 0x%x instead of 0x%x\n",
Christoph Lameter24922682007-07-17 04:03:18 -0700715 fault, end - 1, fault[0], value);
716 print_trailer(s, page, object);
717
718 restore_bytes(s, what, value, fault, end);
719 return 0;
Christoph Lameter81819f02007-05-06 14:49:36 -0700720}
721
Christoph Lameter81819f02007-05-06 14:49:36 -0700722/*
723 * Object layout:
724 *
725 * object address
726 * Bytes of the object to be managed.
727 * If the freepointer may overlay the object then the free
728 * pointer is the first word of the object.
Christoph Lameter672bba32007-05-09 02:32:39 -0700729 *
Christoph Lameter81819f02007-05-06 14:49:36 -0700730 * Poisoning uses 0x6b (POISON_FREE) and the last byte is
731 * 0xa5 (POISON_END)
732 *
Christoph Lameter3b0efdf2012-06-13 10:24:57 -0500733 * object + s->object_size
Christoph Lameter81819f02007-05-06 14:49:36 -0700734 * Padding to reach word boundary. This is also used for Redzoning.
Christoph Lameter672bba32007-05-09 02:32:39 -0700735 * Padding is extended by another word if Redzoning is enabled and
Christoph Lameter3b0efdf2012-06-13 10:24:57 -0500736 * object_size == inuse.
Christoph Lameter672bba32007-05-09 02:32:39 -0700737 *
Christoph Lameter81819f02007-05-06 14:49:36 -0700738 * We fill with 0xbb (RED_INACTIVE) for inactive objects and with
739 * 0xcc (RED_ACTIVE) for objects in use.
740 *
741 * object + s->inuse
Christoph Lameter672bba32007-05-09 02:32:39 -0700742 * Meta data starts here.
743 *
Christoph Lameter81819f02007-05-06 14:49:36 -0700744 * A. Free pointer (if we cannot overwrite object on free)
745 * B. Tracking data for SLAB_STORE_USER
Christoph Lameter672bba32007-05-09 02:32:39 -0700746 * C. Padding to reach required alignment boundary or at mininum
Christoph Lameter6446faa2008-02-15 23:45:26 -0800747 * one word if debugging is on to be able to detect writes
Christoph Lameter672bba32007-05-09 02:32:39 -0700748 * before the word boundary.
749 *
750 * Padding is done using 0x5a (POISON_INUSE)
Christoph Lameter81819f02007-05-06 14:49:36 -0700751 *
752 * object + s->size
Christoph Lameter672bba32007-05-09 02:32:39 -0700753 * Nothing is used beyond s->size.
Christoph Lameter81819f02007-05-06 14:49:36 -0700754 *
Christoph Lameter3b0efdf2012-06-13 10:24:57 -0500755 * If slabcaches are merged then the object_size and inuse boundaries are mostly
Christoph Lameter672bba32007-05-09 02:32:39 -0700756 * ignored. And therefore no slab options that rely on these boundaries
Christoph Lameter81819f02007-05-06 14:49:36 -0700757 * may be used with merged slabcaches.
758 */
759
Christoph Lameter81819f02007-05-06 14:49:36 -0700760static int check_pad_bytes(struct kmem_cache *s, struct page *page, u8 *p)
761{
762 unsigned long off = s->inuse; /* The end of info */
763
764 if (s->offset)
765 /* Freepointer is placed after the object. */
766 off += sizeof(void *);
767
768 if (s->flags & SLAB_STORE_USER)
769 /* We also have user information there */
770 off += 2 * sizeof(struct track);
771
772 if (s->size == off)
773 return 1;
774
Christoph Lameter24922682007-07-17 04:03:18 -0700775 return check_bytes_and_report(s, page, p, "Object padding",
776 p + off, POISON_INUSE, s->size - off);
Christoph Lameter81819f02007-05-06 14:49:36 -0700777}
778
Christoph Lameter39b26462008-04-14 19:11:30 +0300779/* Check the pad bytes at the end of a slab page */
Christoph Lameter81819f02007-05-06 14:49:36 -0700780static int slab_pad_check(struct kmem_cache *s, struct page *page)
781{
Christoph Lameter24922682007-07-17 04:03:18 -0700782 u8 *start;
783 u8 *fault;
784 u8 *end;
785 int length;
786 int remainder;
Christoph Lameter81819f02007-05-06 14:49:36 -0700787
788 if (!(s->flags & SLAB_POISON))
789 return 1;
790
Christoph Lametera973e9d2008-03-01 13:40:44 -0800791 start = page_address(page);
Lai Jiangshanab9a0f12011-03-10 15:21:48 +0800792 length = (PAGE_SIZE << compound_order(page)) - s->reserved;
Christoph Lameter39b26462008-04-14 19:11:30 +0300793 end = start + length;
794 remainder = length % s->size;
Christoph Lameter81819f02007-05-06 14:49:36 -0700795 if (!remainder)
796 return 1;
797
Andrey Ryabinina79316c2015-02-13 14:39:38 -0800798 metadata_access_enable();
Akinobu Mita798248202011-10-31 17:08:07 -0700799 fault = memchr_inv(end - remainder, POISON_INUSE, remainder);
Andrey Ryabinina79316c2015-02-13 14:39:38 -0800800 metadata_access_disable();
Christoph Lameter24922682007-07-17 04:03:18 -0700801 if (!fault)
802 return 1;
803 while (end > fault && end[-1] == POISON_INUSE)
804 end--;
805
806 slab_err(s, page, "Padding overwritten. 0x%p-0x%p", fault, end - 1);
Sebastian Andrzej Siewiorffc79d22011-07-29 14:10:20 +0200807 print_section("Padding ", end - remainder, remainder);
Christoph Lameter24922682007-07-17 04:03:18 -0700808
Eric Dumazet8a3d2712009-09-03 16:08:06 +0200809 restore_bytes(s, "slab padding", POISON_INUSE, end - remainder, end);
Christoph Lameter24922682007-07-17 04:03:18 -0700810 return 0;
Christoph Lameter81819f02007-05-06 14:49:36 -0700811}
812
813static int check_object(struct kmem_cache *s, struct page *page,
Christoph Lameterf7cb1932010-09-29 07:15:01 -0500814 void *object, u8 val)
Christoph Lameter81819f02007-05-06 14:49:36 -0700815{
816 u8 *p = object;
Christoph Lameter3b0efdf2012-06-13 10:24:57 -0500817 u8 *endobject = object + s->object_size;
Christoph Lameter81819f02007-05-06 14:49:36 -0700818
819 if (s->flags & SLAB_RED_ZONE) {
Christoph Lameter24922682007-07-17 04:03:18 -0700820 if (!check_bytes_and_report(s, page, object, "Redzone",
Christoph Lameter3b0efdf2012-06-13 10:24:57 -0500821 endobject, val, s->inuse - s->object_size))
Christoph Lameter81819f02007-05-06 14:49:36 -0700822 return 0;
Christoph Lameter81819f02007-05-06 14:49:36 -0700823 } else {
Christoph Lameter3b0efdf2012-06-13 10:24:57 -0500824 if ((s->flags & SLAB_POISON) && s->object_size < s->inuse) {
Ingo Molnar3adbefe2008-02-05 17:57:39 -0800825 check_bytes_and_report(s, page, p, "Alignment padding",
Chen Gangd0e0ac92013-07-15 09:05:29 +0800826 endobject, POISON_INUSE,
827 s->inuse - s->object_size);
Ingo Molnar3adbefe2008-02-05 17:57:39 -0800828 }
Christoph Lameter81819f02007-05-06 14:49:36 -0700829 }
830
831 if (s->flags & SLAB_POISON) {
Christoph Lameterf7cb1932010-09-29 07:15:01 -0500832 if (val != SLUB_RED_ACTIVE && (s->flags & __OBJECT_POISON) &&
Christoph Lameter24922682007-07-17 04:03:18 -0700833 (!check_bytes_and_report(s, page, p, "Poison", p,
Christoph Lameter3b0efdf2012-06-13 10:24:57 -0500834 POISON_FREE, s->object_size - 1) ||
Christoph Lameter24922682007-07-17 04:03:18 -0700835 !check_bytes_and_report(s, page, p, "Poison",
Christoph Lameter3b0efdf2012-06-13 10:24:57 -0500836 p + s->object_size - 1, POISON_END, 1)))
Christoph Lameter81819f02007-05-06 14:49:36 -0700837 return 0;
Christoph Lameter81819f02007-05-06 14:49:36 -0700838 /*
839 * check_pad_bytes cleans up on its own.
840 */
841 check_pad_bytes(s, page, p);
842 }
843
Christoph Lameterf7cb1932010-09-29 07:15:01 -0500844 if (!s->offset && val == SLUB_RED_ACTIVE)
Christoph Lameter81819f02007-05-06 14:49:36 -0700845 /*
846 * Object and freepointer overlap. Cannot check
847 * freepointer while object is allocated.
848 */
849 return 1;
850
851 /* Check free pointer validity */
852 if (!check_valid_pointer(s, page, get_freepointer(s, p))) {
853 object_err(s, page, p, "Freepointer corrupt");
854 /*
Nick Andrew9f6c708e2008-12-05 14:08:08 +1100855 * No choice but to zap it and thus lose the remainder
Christoph Lameter81819f02007-05-06 14:49:36 -0700856 * of the free objects in this slab. May cause
Christoph Lameter672bba32007-05-09 02:32:39 -0700857 * another error because the object count is now wrong.
Christoph Lameter81819f02007-05-06 14:49:36 -0700858 */
Christoph Lametera973e9d2008-03-01 13:40:44 -0800859 set_freepointer(s, p, NULL);
Christoph Lameter81819f02007-05-06 14:49:36 -0700860 return 0;
861 }
862 return 1;
863}
864
865static int check_slab(struct kmem_cache *s, struct page *page)
866{
Christoph Lameter39b26462008-04-14 19:11:30 +0300867 int maxobj;
868
Christoph Lameter81819f02007-05-06 14:49:36 -0700869 VM_BUG_ON(!irqs_disabled());
870
871 if (!PageSlab(page)) {
Christoph Lameter24922682007-07-17 04:03:18 -0700872 slab_err(s, page, "Not a valid slab page");
Christoph Lameter81819f02007-05-06 14:49:36 -0700873 return 0;
874 }
Christoph Lameter39b26462008-04-14 19:11:30 +0300875
Lai Jiangshanab9a0f12011-03-10 15:21:48 +0800876 maxobj = order_objects(compound_order(page), s->size, s->reserved);
Christoph Lameter39b26462008-04-14 19:11:30 +0300877 if (page->objects > maxobj) {
878 slab_err(s, page, "objects %u > max %u",
Andrey Ryabininf6edde92014-12-10 15:42:22 -0800879 page->objects, maxobj);
Christoph Lameter39b26462008-04-14 19:11:30 +0300880 return 0;
881 }
882 if (page->inuse > page->objects) {
Christoph Lameter24922682007-07-17 04:03:18 -0700883 slab_err(s, page, "inuse %u > max %u",
Andrey Ryabininf6edde92014-12-10 15:42:22 -0800884 page->inuse, page->objects);
Christoph Lameter81819f02007-05-06 14:49:36 -0700885 return 0;
886 }
887 /* Slab_pad_check fixes things up after itself */
888 slab_pad_check(s, page);
889 return 1;
890}
891
892/*
Christoph Lameter672bba32007-05-09 02:32:39 -0700893 * Determine if a certain object on a page is on the freelist. Must hold the
894 * slab lock to guarantee that the chains are in a consistent state.
Christoph Lameter81819f02007-05-06 14:49:36 -0700895 */
896static int on_freelist(struct kmem_cache *s, struct page *page, void *search)
897{
898 int nr = 0;
Christoph Lameter881db7f2011-06-01 12:25:53 -0500899 void *fp;
Christoph Lameter81819f02007-05-06 14:49:36 -0700900 void *object = NULL;
Andrey Ryabininf6edde92014-12-10 15:42:22 -0800901 int max_objects;
Christoph Lameter81819f02007-05-06 14:49:36 -0700902
Christoph Lameter881db7f2011-06-01 12:25:53 -0500903 fp = page->freelist;
Christoph Lameter39b26462008-04-14 19:11:30 +0300904 while (fp && nr <= page->objects) {
Christoph Lameter81819f02007-05-06 14:49:36 -0700905 if (fp == search)
906 return 1;
907 if (!check_valid_pointer(s, page, fp)) {
908 if (object) {
909 object_err(s, page, object,
910 "Freechain corrupt");
Christoph Lametera973e9d2008-03-01 13:40:44 -0800911 set_freepointer(s, object, NULL);
Christoph Lameter81819f02007-05-06 14:49:36 -0700912 } else {
Christoph Lameter24922682007-07-17 04:03:18 -0700913 slab_err(s, page, "Freepointer corrupt");
Christoph Lametera973e9d2008-03-01 13:40:44 -0800914 page->freelist = NULL;
Christoph Lameter39b26462008-04-14 19:11:30 +0300915 page->inuse = page->objects;
Christoph Lameter24922682007-07-17 04:03:18 -0700916 slab_fix(s, "Freelist cleared");
Christoph Lameter81819f02007-05-06 14:49:36 -0700917 return 0;
918 }
919 break;
920 }
921 object = fp;
922 fp = get_freepointer(s, object);
923 nr++;
924 }
925
Lai Jiangshanab9a0f12011-03-10 15:21:48 +0800926 max_objects = order_objects(compound_order(page), s->size, s->reserved);
Cyrill Gorcunov210b5c02008-10-22 23:00:38 +0400927 if (max_objects > MAX_OBJS_PER_PAGE)
928 max_objects = MAX_OBJS_PER_PAGE;
Christoph Lameter224a88b2008-04-14 19:11:31 +0300929
930 if (page->objects != max_objects) {
931 slab_err(s, page, "Wrong number of objects. Found %d but "
932 "should be %d", page->objects, max_objects);
933 page->objects = max_objects;
934 slab_fix(s, "Number of objects adjusted.");
935 }
Christoph Lameter39b26462008-04-14 19:11:30 +0300936 if (page->inuse != page->objects - nr) {
Christoph Lameter70d71222007-05-06 14:49:47 -0700937 slab_err(s, page, "Wrong object count. Counter is %d but "
Christoph Lameter39b26462008-04-14 19:11:30 +0300938 "counted were %d", page->inuse, page->objects - nr);
939 page->inuse = page->objects - nr;
Christoph Lameter24922682007-07-17 04:03:18 -0700940 slab_fix(s, "Object count adjusted.");
Christoph Lameter81819f02007-05-06 14:49:36 -0700941 }
942 return search == NULL;
943}
944
Christoph Lameter0121c6192008-04-29 16:11:12 -0700945static void trace(struct kmem_cache *s, struct page *page, void *object,
946 int alloc)
Christoph Lameter3ec09742007-05-16 22:11:00 -0700947{
948 if (s->flags & SLAB_TRACE) {
Fabian Frederickf9f58282014-06-04 16:06:34 -0700949 pr_info("TRACE %s %s 0x%p inuse=%d fp=0x%p\n",
Christoph Lameter3ec09742007-05-16 22:11:00 -0700950 s->name,
951 alloc ? "alloc" : "free",
952 object, page->inuse,
953 page->freelist);
954
955 if (!alloc)
Chen Gangd0e0ac92013-07-15 09:05:29 +0800956 print_section("Object ", (void *)object,
957 s->object_size);
Christoph Lameter3ec09742007-05-16 22:11:00 -0700958
959 dump_stack();
960 }
961}
962
Christoph Lameter643b1132007-05-06 14:49:42 -0700963/*
Christoph Lameter672bba32007-05-09 02:32:39 -0700964 * Tracking of fully allocated slabs for debugging purposes.
Christoph Lameter643b1132007-05-06 14:49:42 -0700965 */
Christoph Lameter5cc6eee2011-06-01 12:25:50 -0500966static void add_full(struct kmem_cache *s,
967 struct kmem_cache_node *n, struct page *page)
Christoph Lameter643b1132007-05-06 14:49:42 -0700968{
Christoph Lameter643b1132007-05-06 14:49:42 -0700969 if (!(s->flags & SLAB_STORE_USER))
970 return;
971
David Rientjes255d0882014-02-10 14:25:39 -0800972 lockdep_assert_held(&n->list_lock);
Christoph Lameter5cc6eee2011-06-01 12:25:50 -0500973 list_add(&page->lru, &n->full);
974}
Christoph Lameter643b1132007-05-06 14:49:42 -0700975
Peter Zijlstrac65c1872014-01-10 13:23:49 +0100976static void remove_full(struct kmem_cache *s, struct kmem_cache_node *n, struct page *page)
Christoph Lameter5cc6eee2011-06-01 12:25:50 -0500977{
978 if (!(s->flags & SLAB_STORE_USER))
979 return;
980
David Rientjes255d0882014-02-10 14:25:39 -0800981 lockdep_assert_held(&n->list_lock);
Christoph Lameter643b1132007-05-06 14:49:42 -0700982 list_del(&page->lru);
Christoph Lameter643b1132007-05-06 14:49:42 -0700983}
984
Christoph Lameter0f389ec2008-04-14 18:53:02 +0300985/* Tracking of the number of slabs for debugging purposes */
986static inline unsigned long slabs_node(struct kmem_cache *s, int node)
987{
988 struct kmem_cache_node *n = get_node(s, node);
989
990 return atomic_long_read(&n->nr_slabs);
991}
992
Alexander Beregalov26c02cf2009-06-11 14:08:48 +0400993static inline unsigned long node_nr_slabs(struct kmem_cache_node *n)
994{
995 return atomic_long_read(&n->nr_slabs);
996}
997
Christoph Lameter205ab992008-04-14 19:11:40 +0300998static inline void inc_slabs_node(struct kmem_cache *s, int node, int objects)
Christoph Lameter0f389ec2008-04-14 18:53:02 +0300999{
1000 struct kmem_cache_node *n = get_node(s, node);
1001
1002 /*
1003 * May be called early in order to allocate a slab for the
1004 * kmem_cache_node structure. Solve the chicken-egg
1005 * dilemma by deferring the increment of the count during
1006 * bootstrap (see early_kmem_cache_node_alloc).
1007 */
Joonsoo Kim338b2642013-01-21 17:01:27 +09001008 if (likely(n)) {
Christoph Lameter0f389ec2008-04-14 18:53:02 +03001009 atomic_long_inc(&n->nr_slabs);
Christoph Lameter205ab992008-04-14 19:11:40 +03001010 atomic_long_add(objects, &n->total_objects);
1011 }
Christoph Lameter0f389ec2008-04-14 18:53:02 +03001012}
Christoph Lameter205ab992008-04-14 19:11:40 +03001013static inline void dec_slabs_node(struct kmem_cache *s, int node, int objects)
Christoph Lameter0f389ec2008-04-14 18:53:02 +03001014{
1015 struct kmem_cache_node *n = get_node(s, node);
1016
1017 atomic_long_dec(&n->nr_slabs);
Christoph Lameter205ab992008-04-14 19:11:40 +03001018 atomic_long_sub(objects, &n->total_objects);
Christoph Lameter0f389ec2008-04-14 18:53:02 +03001019}
1020
1021/* Object debug checks for alloc/free paths */
Christoph Lameter3ec09742007-05-16 22:11:00 -07001022static void setup_object_debug(struct kmem_cache *s, struct page *page,
1023 void *object)
1024{
1025 if (!(s->flags & (SLAB_STORE_USER|SLAB_RED_ZONE|__OBJECT_POISON)))
1026 return;
1027
Christoph Lameterf7cb1932010-09-29 07:15:01 -05001028 init_object(s, object, SLUB_RED_INACTIVE);
Christoph Lameter3ec09742007-05-16 22:11:00 -07001029 init_tracking(s, object);
1030}
1031
Chen Gangd0e0ac92013-07-15 09:05:29 +08001032static noinline int alloc_debug_processing(struct kmem_cache *s,
1033 struct page *page,
Eduard - Gabriel Munteanuce71e272008-08-19 20:43:25 +03001034 void *object, unsigned long addr)
Christoph Lameter81819f02007-05-06 14:49:36 -07001035{
1036 if (!check_slab(s, page))
1037 goto bad;
1038
Christoph Lameter81819f02007-05-06 14:49:36 -07001039 if (!check_valid_pointer(s, page, object)) {
1040 object_err(s, page, object, "Freelist Pointer check fails");
Christoph Lameter70d71222007-05-06 14:49:47 -07001041 goto bad;
Christoph Lameter81819f02007-05-06 14:49:36 -07001042 }
1043
Christoph Lameterf7cb1932010-09-29 07:15:01 -05001044 if (!check_object(s, page, object, SLUB_RED_INACTIVE))
Christoph Lameter81819f02007-05-06 14:49:36 -07001045 goto bad;
Christoph Lameter81819f02007-05-06 14:49:36 -07001046
Christoph Lameter3ec09742007-05-16 22:11:00 -07001047 /* Success perform special debug activities for allocs */
1048 if (s->flags & SLAB_STORE_USER)
1049 set_track(s, object, TRACK_ALLOC, addr);
1050 trace(s, page, object, 1);
Christoph Lameterf7cb1932010-09-29 07:15:01 -05001051 init_object(s, object, SLUB_RED_ACTIVE);
Christoph Lameter81819f02007-05-06 14:49:36 -07001052 return 1;
Christoph Lameter3ec09742007-05-16 22:11:00 -07001053
Christoph Lameter81819f02007-05-06 14:49:36 -07001054bad:
1055 if (PageSlab(page)) {
1056 /*
1057 * If this is a slab page then lets do the best we can
1058 * to avoid issues in the future. Marking all objects
Christoph Lameter672bba32007-05-09 02:32:39 -07001059 * as used avoids touching the remaining objects.
Christoph Lameter81819f02007-05-06 14:49:36 -07001060 */
Christoph Lameter24922682007-07-17 04:03:18 -07001061 slab_fix(s, "Marking all objects used");
Christoph Lameter39b26462008-04-14 19:11:30 +03001062 page->inuse = page->objects;
Christoph Lametera973e9d2008-03-01 13:40:44 -08001063 page->freelist = NULL;
Christoph Lameter81819f02007-05-06 14:49:36 -07001064 }
1065 return 0;
1066}
1067
Jesper Dangaard Brouer81084652015-11-20 15:57:46 -08001068/* Supports checking bulk free of a constructed freelist */
Christoph Lameter19c7ff92012-05-30 12:54:46 -05001069static noinline struct kmem_cache_node *free_debug_processing(
Jesper Dangaard Brouer81084652015-11-20 15:57:46 -08001070 struct kmem_cache *s, struct page *page,
1071 void *head, void *tail, int bulk_cnt,
Christoph Lameter19c7ff92012-05-30 12:54:46 -05001072 unsigned long addr, unsigned long *flags)
Christoph Lameter81819f02007-05-06 14:49:36 -07001073{
Christoph Lameter19c7ff92012-05-30 12:54:46 -05001074 struct kmem_cache_node *n = get_node(s, page_to_nid(page));
Jesper Dangaard Brouer81084652015-11-20 15:57:46 -08001075 void *object = head;
1076 int cnt = 0;
Christoph Lameter5c2e4bb2011-06-01 12:25:54 -05001077
Christoph Lameter19c7ff92012-05-30 12:54:46 -05001078 spin_lock_irqsave(&n->list_lock, *flags);
Christoph Lameter881db7f2011-06-01 12:25:53 -05001079 slab_lock(page);
1080
Christoph Lameter81819f02007-05-06 14:49:36 -07001081 if (!check_slab(s, page))
1082 goto fail;
1083
Jesper Dangaard Brouer81084652015-11-20 15:57:46 -08001084next_object:
1085 cnt++;
1086
Christoph Lameter81819f02007-05-06 14:49:36 -07001087 if (!check_valid_pointer(s, page, object)) {
Christoph Lameter70d71222007-05-06 14:49:47 -07001088 slab_err(s, page, "Invalid object pointer 0x%p", object);
Christoph Lameter81819f02007-05-06 14:49:36 -07001089 goto fail;
1090 }
1091
1092 if (on_freelist(s, page, object)) {
Christoph Lameter24922682007-07-17 04:03:18 -07001093 object_err(s, page, object, "Object already free");
Christoph Lameter81819f02007-05-06 14:49:36 -07001094 goto fail;
1095 }
1096
Christoph Lameterf7cb1932010-09-29 07:15:01 -05001097 if (!check_object(s, page, object, SLUB_RED_ACTIVE))
Christoph Lameter5c2e4bb2011-06-01 12:25:54 -05001098 goto out;
Christoph Lameter81819f02007-05-06 14:49:36 -07001099
Glauber Costa1b4f59e32012-10-22 18:05:36 +04001100 if (unlikely(s != page->slab_cache)) {
Ingo Molnar3adbefe2008-02-05 17:57:39 -08001101 if (!PageSlab(page)) {
Christoph Lameter70d71222007-05-06 14:49:47 -07001102 slab_err(s, page, "Attempt to free object(0x%p) "
1103 "outside of slab", object);
Glauber Costa1b4f59e32012-10-22 18:05:36 +04001104 } else if (!page->slab_cache) {
Fabian Frederickf9f58282014-06-04 16:06:34 -07001105 pr_err("SLUB <none>: no slab for object 0x%p.\n",
1106 object);
Christoph Lameter70d71222007-05-06 14:49:47 -07001107 dump_stack();
Pekka Enberg06428782008-01-07 23:20:27 -08001108 } else
Christoph Lameter24922682007-07-17 04:03:18 -07001109 object_err(s, page, object,
1110 "page slab pointer corrupt.");
Christoph Lameter81819f02007-05-06 14:49:36 -07001111 goto fail;
1112 }
Christoph Lameter3ec09742007-05-16 22:11:00 -07001113
Christoph Lameter3ec09742007-05-16 22:11:00 -07001114 if (s->flags & SLAB_STORE_USER)
1115 set_track(s, object, TRACK_FREE, addr);
1116 trace(s, page, object, 0);
Jesper Dangaard Brouer81084652015-11-20 15:57:46 -08001117 /* Freepointer not overwritten by init_object(), SLAB_POISON moved it */
Christoph Lameterf7cb1932010-09-29 07:15:01 -05001118 init_object(s, object, SLUB_RED_INACTIVE);
Jesper Dangaard Brouer81084652015-11-20 15:57:46 -08001119
1120 /* Reached end of constructed freelist yet? */
1121 if (object != tail) {
1122 object = get_freepointer(s, object);
1123 goto next_object;
1124 }
Christoph Lameter5c2e4bb2011-06-01 12:25:54 -05001125out:
Jesper Dangaard Brouer81084652015-11-20 15:57:46 -08001126 if (cnt != bulk_cnt)
1127 slab_err(s, page, "Bulk freelist count(%d) invalid(%d)\n",
1128 bulk_cnt, cnt);
1129
Christoph Lameter881db7f2011-06-01 12:25:53 -05001130 slab_unlock(page);
Christoph Lameter19c7ff92012-05-30 12:54:46 -05001131 /*
1132 * Keep node_lock to preserve integrity
1133 * until the object is actually freed
1134 */
1135 return n;
Christoph Lameter3ec09742007-05-16 22:11:00 -07001136
Christoph Lameter81819f02007-05-06 14:49:36 -07001137fail:
Christoph Lameter19c7ff92012-05-30 12:54:46 -05001138 slab_unlock(page);
1139 spin_unlock_irqrestore(&n->list_lock, *flags);
Christoph Lameter24922682007-07-17 04:03:18 -07001140 slab_fix(s, "Object at 0x%p not freed", object);
Christoph Lameter19c7ff92012-05-30 12:54:46 -05001141 return NULL;
Christoph Lameter81819f02007-05-06 14:49:36 -07001142}
1143
Christoph Lameter41ecc552007-05-09 02:32:44 -07001144static int __init setup_slub_debug(char *str)
1145{
Christoph Lameterf0630ff2007-07-15 23:38:14 -07001146 slub_debug = DEBUG_DEFAULT_FLAGS;
1147 if (*str++ != '=' || !*str)
1148 /*
1149 * No options specified. Switch on full debugging.
1150 */
1151 goto out;
Christoph Lameter41ecc552007-05-09 02:32:44 -07001152
1153 if (*str == ',')
Christoph Lameterf0630ff2007-07-15 23:38:14 -07001154 /*
1155 * No options but restriction on slabs. This means full
1156 * debugging for slabs matching a pattern.
1157 */
1158 goto check_slabs;
1159
1160 slub_debug = 0;
1161 if (*str == '-')
1162 /*
1163 * Switch off all debugging measures.
1164 */
1165 goto out;
1166
1167 /*
1168 * Determine which debug features should be switched on
1169 */
Pekka Enberg06428782008-01-07 23:20:27 -08001170 for (; *str && *str != ','; str++) {
Christoph Lameterf0630ff2007-07-15 23:38:14 -07001171 switch (tolower(*str)) {
1172 case 'f':
1173 slub_debug |= SLAB_DEBUG_FREE;
1174 break;
1175 case 'z':
1176 slub_debug |= SLAB_RED_ZONE;
1177 break;
1178 case 'p':
1179 slub_debug |= SLAB_POISON;
1180 break;
1181 case 'u':
1182 slub_debug |= SLAB_STORE_USER;
1183 break;
1184 case 't':
1185 slub_debug |= SLAB_TRACE;
1186 break;
Dmitry Monakhov4c13dd32010-02-26 09:36:12 +03001187 case 'a':
1188 slub_debug |= SLAB_FAILSLAB;
1189 break;
Chris J Arges08303a72015-04-14 15:44:25 -07001190 case 'o':
1191 /*
1192 * Avoid enabling debugging on caches if its minimum
1193 * order would increase as a result.
1194 */
1195 disable_higher_order_debug = 1;
1196 break;
Christoph Lameterf0630ff2007-07-15 23:38:14 -07001197 default:
Fabian Frederickf9f58282014-06-04 16:06:34 -07001198 pr_err("slub_debug option '%c' unknown. skipped\n",
1199 *str);
Christoph Lameterf0630ff2007-07-15 23:38:14 -07001200 }
1201 }
1202
1203check_slabs:
1204 if (*str == ',')
Christoph Lameter41ecc552007-05-09 02:32:44 -07001205 slub_debug_slabs = str + 1;
Christoph Lameterf0630ff2007-07-15 23:38:14 -07001206out:
Christoph Lameter41ecc552007-05-09 02:32:44 -07001207 return 1;
1208}
1209
1210__setup("slub_debug", setup_slub_debug);
1211
Joonsoo Kim423c9292014-10-09 15:26:22 -07001212unsigned long kmem_cache_flags(unsigned long object_size,
Christoph Lameterba0268a2007-09-11 15:24:11 -07001213 unsigned long flags, const char *name,
Alexey Dobriyan51cc5062008-07-25 19:45:34 -07001214 void (*ctor)(void *))
Christoph Lameter41ecc552007-05-09 02:32:44 -07001215{
1216 /*
Christoph Lametere1533622008-02-15 23:45:24 -08001217 * Enable debugging if selected on the kernel commandline.
Christoph Lameter41ecc552007-05-09 02:32:44 -07001218 */
Christoph Lameterc6f58d92013-11-07 16:29:15 +00001219 if (slub_debug && (!slub_debug_slabs || (name &&
1220 !strncmp(slub_debug_slabs, name, strlen(slub_debug_slabs)))))
David Rientjes3de47212009-07-27 18:30:35 -07001221 flags |= slub_debug;
Christoph Lameterba0268a2007-09-11 15:24:11 -07001222
1223 return flags;
Christoph Lameter41ecc552007-05-09 02:32:44 -07001224}
Jesper Dangaard Brouerb4a64712015-11-20 15:57:41 -08001225#else /* !CONFIG_SLUB_DEBUG */
Christoph Lameter3ec09742007-05-16 22:11:00 -07001226static inline void setup_object_debug(struct kmem_cache *s,
1227 struct page *page, void *object) {}
Christoph Lameter41ecc552007-05-09 02:32:44 -07001228
Christoph Lameter3ec09742007-05-16 22:11:00 -07001229static inline int alloc_debug_processing(struct kmem_cache *s,
Eduard - Gabriel Munteanuce71e272008-08-19 20:43:25 +03001230 struct page *page, void *object, unsigned long addr) { return 0; }
Christoph Lameter41ecc552007-05-09 02:32:44 -07001231
Christoph Lameter19c7ff92012-05-30 12:54:46 -05001232static inline struct kmem_cache_node *free_debug_processing(
Jesper Dangaard Brouer81084652015-11-20 15:57:46 -08001233 struct kmem_cache *s, struct page *page,
1234 void *head, void *tail, int bulk_cnt,
Christoph Lameter19c7ff92012-05-30 12:54:46 -05001235 unsigned long addr, unsigned long *flags) { return NULL; }
Christoph Lameter41ecc552007-05-09 02:32:44 -07001236
Christoph Lameter41ecc552007-05-09 02:32:44 -07001237static inline int slab_pad_check(struct kmem_cache *s, struct page *page)
1238 { return 1; }
1239static inline int check_object(struct kmem_cache *s, struct page *page,
Christoph Lameterf7cb1932010-09-29 07:15:01 -05001240 void *object, u8 val) { return 1; }
Christoph Lameter5cc6eee2011-06-01 12:25:50 -05001241static inline void add_full(struct kmem_cache *s, struct kmem_cache_node *n,
1242 struct page *page) {}
Peter Zijlstrac65c1872014-01-10 13:23:49 +01001243static inline void remove_full(struct kmem_cache *s, struct kmem_cache_node *n,
1244 struct page *page) {}
Joonsoo Kim423c9292014-10-09 15:26:22 -07001245unsigned long kmem_cache_flags(unsigned long object_size,
Christoph Lameterba0268a2007-09-11 15:24:11 -07001246 unsigned long flags, const char *name,
Alexey Dobriyan51cc5062008-07-25 19:45:34 -07001247 void (*ctor)(void *))
Christoph Lameterba0268a2007-09-11 15:24:11 -07001248{
1249 return flags;
1250}
Christoph Lameter41ecc552007-05-09 02:32:44 -07001251#define slub_debug 0
Christoph Lameter0f389ec2008-04-14 18:53:02 +03001252
Ingo Molnarfdaa45e2009-09-15 11:00:26 +02001253#define disable_higher_order_debug 0
1254
Christoph Lameter0f389ec2008-04-14 18:53:02 +03001255static inline unsigned long slabs_node(struct kmem_cache *s, int node)
1256 { return 0; }
Alexander Beregalov26c02cf2009-06-11 14:08:48 +04001257static inline unsigned long node_nr_slabs(struct kmem_cache_node *n)
1258 { return 0; }
Christoph Lameter205ab992008-04-14 19:11:40 +03001259static inline void inc_slabs_node(struct kmem_cache *s, int node,
1260 int objects) {}
1261static inline void dec_slabs_node(struct kmem_cache *s, int node,
1262 int objects) {}
Christoph Lameter7d550c52010-08-25 14:07:16 -05001263
Andrey Ryabinin02e72cc2014-08-06 16:04:18 -07001264#endif /* CONFIG_SLUB_DEBUG */
1265
1266/*
1267 * Hooks for other subsystems that check memory allocations. In a typical
1268 * production configuration these hooks all should produce no code at all.
1269 */
Roman Bobnievd56791b2013-10-08 15:58:57 -07001270static inline void kmalloc_large_node_hook(void *ptr, size_t size, gfp_t flags)
1271{
1272 kmemleak_alloc(ptr, size, 1, flags);
Andrey Ryabinin0316bec2015-02-13 14:39:42 -08001273 kasan_kmalloc_large(ptr, size);
Roman Bobnievd56791b2013-10-08 15:58:57 -07001274}
1275
1276static inline void kfree_hook(const void *x)
1277{
1278 kmemleak_free(x);
Andrey Ryabinin0316bec2015-02-13 14:39:42 -08001279 kasan_kfree_large(x);
Roman Bobnievd56791b2013-10-08 15:58:57 -07001280}
1281
Vladimir Davydov8135be52014-12-12 16:56:38 -08001282static inline struct kmem_cache *slab_pre_alloc_hook(struct kmem_cache *s,
1283 gfp_t flags)
Roman Bobnievd56791b2013-10-08 15:58:57 -07001284{
Andrey Ryabinin02e72cc2014-08-06 16:04:18 -07001285 flags &= gfp_allowed_mask;
1286 lockdep_trace_alloc(flags);
Mel Gormand0164ad2015-11-06 16:28:21 -08001287 might_sleep_if(gfpflags_allow_blocking(flags));
Andrey Ryabinin02e72cc2014-08-06 16:04:18 -07001288
Vladimir Davydov8135be52014-12-12 16:56:38 -08001289 if (should_failslab(s->object_size, flags, s->flags))
1290 return NULL;
1291
1292 return memcg_kmem_get_cache(s, flags);
Andrey Ryabinin02e72cc2014-08-06 16:04:18 -07001293}
1294
Jesper Dangaard Brouer03ec0ed2015-11-20 15:57:52 -08001295static inline void slab_post_alloc_hook(struct kmem_cache *s, gfp_t flags,
1296 size_t size, void **p)
Andrey Ryabinin02e72cc2014-08-06 16:04:18 -07001297{
Jesper Dangaard Brouer03ec0ed2015-11-20 15:57:52 -08001298 size_t i;
1299
Andrey Ryabinin02e72cc2014-08-06 16:04:18 -07001300 flags &= gfp_allowed_mask;
Jesper Dangaard Brouer03ec0ed2015-11-20 15:57:52 -08001301 for (i = 0; i < size; i++) {
1302 void *object = p[i];
1303
1304 kmemcheck_slab_alloc(s, flags, object, slab_ksize(s));
1305 kmemleak_alloc_recursive(object, s->object_size, 1,
1306 s->flags, flags);
1307 kasan_slab_alloc(s, object);
1308 }
Vladimir Davydov8135be52014-12-12 16:56:38 -08001309 memcg_kmem_put_cache(s);
Roman Bobnievd56791b2013-10-08 15:58:57 -07001310}
Christoph Lameter7d550c52010-08-25 14:07:16 -05001311
Roman Bobnievd56791b2013-10-08 15:58:57 -07001312static inline void slab_free_hook(struct kmem_cache *s, void *x)
1313{
1314 kmemleak_free_recursive(x, s->flags);
Christoph Lameter7d550c52010-08-25 14:07:16 -05001315
Andrey Ryabinin02e72cc2014-08-06 16:04:18 -07001316 /*
1317 * Trouble is that we may no longer disable interrupts in the fast path
1318 * So in order to make the debug calls that expect irqs to be
1319 * disabled we need to disable interrupts temporarily.
1320 */
1321#if defined(CONFIG_KMEMCHECK) || defined(CONFIG_LOCKDEP)
1322 {
1323 unsigned long flags;
1324
1325 local_irq_save(flags);
1326 kmemcheck_slab_free(s, x, s->object_size);
1327 debug_check_no_locks_freed(x, s->object_size);
1328 local_irq_restore(flags);
1329 }
1330#endif
1331 if (!(s->flags & SLAB_DEBUG_OBJECTS))
1332 debug_check_no_obj_freed(x, s->object_size);
Andrey Ryabinin0316bec2015-02-13 14:39:42 -08001333
1334 kasan_slab_free(s, x);
Andrey Ryabinin02e72cc2014-08-06 16:04:18 -07001335}
Christoph Lameter205ab992008-04-14 19:11:40 +03001336
Jesper Dangaard Brouer81084652015-11-20 15:57:46 -08001337static inline void slab_free_freelist_hook(struct kmem_cache *s,
1338 void *head, void *tail)
1339{
1340/*
1341 * Compiler cannot detect this function can be removed if slab_free_hook()
1342 * evaluates to nothing. Thus, catch all relevant config debug options here.
1343 */
1344#if defined(CONFIG_KMEMCHECK) || \
1345 defined(CONFIG_LOCKDEP) || \
1346 defined(CONFIG_DEBUG_KMEMLEAK) || \
1347 defined(CONFIG_DEBUG_OBJECTS_FREE) || \
1348 defined(CONFIG_KASAN)
1349
1350 void *object = head;
1351 void *tail_obj = tail ? : head;
1352
1353 do {
1354 slab_free_hook(s, object);
1355 } while ((object != tail_obj) &&
1356 (object = get_freepointer(s, object)));
1357#endif
1358}
1359
Thomas Gleixner588f8ba2015-09-04 15:45:48 -07001360static void setup_object(struct kmem_cache *s, struct page *page,
1361 void *object)
1362{
1363 setup_object_debug(s, page, object);
1364 if (unlikely(s->ctor)) {
1365 kasan_unpoison_object_data(s, object);
1366 s->ctor(object);
1367 kasan_poison_object_data(s, object);
1368 }
1369}
1370
Christoph Lameter81819f02007-05-06 14:49:36 -07001371/*
1372 * Slab allocation and freeing
1373 */
Vladimir Davydov5dfb4172014-06-04 16:06:38 -07001374static inline struct page *alloc_slab_page(struct kmem_cache *s,
1375 gfp_t flags, int node, struct kmem_cache_order_objects oo)
Christoph Lameter65c33762008-04-14 19:11:40 +03001376{
Vladimir Davydov5dfb4172014-06-04 16:06:38 -07001377 struct page *page;
Christoph Lameter65c33762008-04-14 19:11:40 +03001378 int order = oo_order(oo);
1379
Vegard Nossumb1eeab62008-11-25 16:55:53 +01001380 flags |= __GFP_NOTRACK;
1381
Christoph Lameter2154a332010-07-09 14:07:10 -05001382 if (node == NUMA_NO_NODE)
Vladimir Davydov5dfb4172014-06-04 16:06:38 -07001383 page = alloc_pages(flags, order);
Christoph Lameter65c33762008-04-14 19:11:40 +03001384 else
Vlastimil Babka96db8002015-09-08 15:03:50 -07001385 page = __alloc_pages_node(node, flags, order);
Vladimir Davydov5dfb4172014-06-04 16:06:38 -07001386
Vladimir Davydovf3ccb2c42015-11-05 18:49:01 -08001387 if (page && memcg_charge_slab(page, flags, order, s)) {
1388 __free_pages(page, order);
1389 page = NULL;
1390 }
Vladimir Davydov5dfb4172014-06-04 16:06:38 -07001391
1392 return page;
Christoph Lameter65c33762008-04-14 19:11:40 +03001393}
1394
Christoph Lameter81819f02007-05-06 14:49:36 -07001395static struct page *allocate_slab(struct kmem_cache *s, gfp_t flags, int node)
1396{
Pekka Enberg06428782008-01-07 23:20:27 -08001397 struct page *page;
Christoph Lameter834f3d12008-04-14 19:11:31 +03001398 struct kmem_cache_order_objects oo = s->oo;
Pekka Enbergba522702009-06-24 21:59:51 +03001399 gfp_t alloc_gfp;
Thomas Gleixner588f8ba2015-09-04 15:45:48 -07001400 void *start, *p;
1401 int idx, order;
Christoph Lameter81819f02007-05-06 14:49:36 -07001402
Christoph Lameter7e0528d2011-06-01 12:25:44 -05001403 flags &= gfp_allowed_mask;
1404
Mel Gormand0164ad2015-11-06 16:28:21 -08001405 if (gfpflags_allow_blocking(flags))
Christoph Lameter7e0528d2011-06-01 12:25:44 -05001406 local_irq_enable();
1407
Christoph Lameterb7a49f02008-02-14 14:21:32 -08001408 flags |= s->allocflags;
Mel Gormane12ba742007-10-16 01:25:52 -07001409
Pekka Enbergba522702009-06-24 21:59:51 +03001410 /*
1411 * Let the initial higher-order allocation fail under memory pressure
1412 * so we fall-back to the minimum order allocation.
1413 */
1414 alloc_gfp = (flags | __GFP_NOWARN | __GFP_NORETRY) & ~__GFP_NOFAIL;
Mel Gormand0164ad2015-11-06 16:28:21 -08001415 if ((alloc_gfp & __GFP_DIRECT_RECLAIM) && oo_order(oo) > oo_order(s->min))
1416 alloc_gfp = (alloc_gfp | __GFP_NOMEMALLOC) & ~__GFP_DIRECT_RECLAIM;
Pekka Enbergba522702009-06-24 21:59:51 +03001417
Vladimir Davydov5dfb4172014-06-04 16:06:38 -07001418 page = alloc_slab_page(s, alloc_gfp, node, oo);
Christoph Lameter65c33762008-04-14 19:11:40 +03001419 if (unlikely(!page)) {
1420 oo = s->min;
Joonsoo Kim80c3a992014-03-12 17:26:20 +09001421 alloc_gfp = flags;
Christoph Lameter65c33762008-04-14 19:11:40 +03001422 /*
1423 * Allocation may have failed due to fragmentation.
1424 * Try a lower order alloc if possible
1425 */
Vladimir Davydov5dfb4172014-06-04 16:06:38 -07001426 page = alloc_slab_page(s, alloc_gfp, node, oo);
Thomas Gleixner588f8ba2015-09-04 15:45:48 -07001427 if (unlikely(!page))
1428 goto out;
1429 stat(s, ORDER_FALLBACK);
Christoph Lameter65c33762008-04-14 19:11:40 +03001430 }
Vegard Nossum5a896d92008-04-04 00:54:48 +02001431
Thomas Gleixner588f8ba2015-09-04 15:45:48 -07001432 if (kmemcheck_enabled &&
1433 !(s->flags & (SLAB_NOTRACK | DEBUG_DEFAULT_FLAGS))) {
Vegard Nossumb1eeab62008-11-25 16:55:53 +01001434 int pages = 1 << oo_order(oo);
1435
Joonsoo Kim80c3a992014-03-12 17:26:20 +09001436 kmemcheck_alloc_shadow(page, oo_order(oo), alloc_gfp, node);
Vegard Nossumb1eeab62008-11-25 16:55:53 +01001437
1438 /*
1439 * Objects from caches that have a constructor don't get
1440 * cleared when they're allocated, so we need to do it here.
1441 */
1442 if (s->ctor)
1443 kmemcheck_mark_uninitialized_pages(page, pages);
1444 else
1445 kmemcheck_mark_unallocated_pages(page, pages);
Vegard Nossum5a896d92008-04-04 00:54:48 +02001446 }
1447
Christoph Lameter834f3d12008-04-14 19:11:31 +03001448 page->objects = oo_objects(oo);
Christoph Lameter81819f02007-05-06 14:49:36 -07001449
Glauber Costa1f458cb2012-12-18 14:22:50 -08001450 order = compound_order(page);
Glauber Costa1b4f59e32012-10-22 18:05:36 +04001451 page->slab_cache = s;
Joonsoo Kimc03f94c2012-05-18 00:47:47 +09001452 __SetPageSlab(page);
Michal Hocko2f064f32015-08-21 14:11:51 -07001453 if (page_is_pfmemalloc(page))
Mel Gorman072bb0a2012-07-31 16:43:58 -07001454 SetPageSlabPfmemalloc(page);
Christoph Lameter81819f02007-05-06 14:49:36 -07001455
1456 start = page_address(page);
Christoph Lameter81819f02007-05-06 14:49:36 -07001457
1458 if (unlikely(s->flags & SLAB_POISON))
Glauber Costa1f458cb2012-12-18 14:22:50 -08001459 memset(start, POISON_INUSE, PAGE_SIZE << order);
Christoph Lameter81819f02007-05-06 14:49:36 -07001460
Andrey Ryabinin0316bec2015-02-13 14:39:42 -08001461 kasan_poison_slab(page);
1462
Wei Yang54266642014-08-06 16:04:42 -07001463 for_each_object_idx(p, idx, s, start, page->objects) {
1464 setup_object(s, page, p);
1465 if (likely(idx < page->objects))
1466 set_freepointer(s, p, p + s->size);
1467 else
1468 set_freepointer(s, p, NULL);
Christoph Lameter81819f02007-05-06 14:49:36 -07001469 }
Christoph Lameter81819f02007-05-06 14:49:36 -07001470
1471 page->freelist = start;
Christoph Lametere6e82ea2011-08-09 16:12:24 -05001472 page->inuse = page->objects;
Christoph Lameter8cb0a502011-06-01 12:25:46 -05001473 page->frozen = 1;
Thomas Gleixner588f8ba2015-09-04 15:45:48 -07001474
Christoph Lameter81819f02007-05-06 14:49:36 -07001475out:
Mel Gormand0164ad2015-11-06 16:28:21 -08001476 if (gfpflags_allow_blocking(flags))
Thomas Gleixner588f8ba2015-09-04 15:45:48 -07001477 local_irq_disable();
1478 if (!page)
1479 return NULL;
1480
1481 mod_zone_page_state(page_zone(page),
1482 (s->flags & SLAB_RECLAIM_ACCOUNT) ?
1483 NR_SLAB_RECLAIMABLE : NR_SLAB_UNRECLAIMABLE,
1484 1 << oo_order(oo));
1485
1486 inc_slabs_node(s, page_to_nid(page), page->objects);
1487
Christoph Lameter81819f02007-05-06 14:49:36 -07001488 return page;
1489}
1490
Thomas Gleixner588f8ba2015-09-04 15:45:48 -07001491static struct page *new_slab(struct kmem_cache *s, gfp_t flags, int node)
1492{
1493 if (unlikely(flags & GFP_SLAB_BUG_MASK)) {
1494 pr_emerg("gfp: %u\n", flags & GFP_SLAB_BUG_MASK);
1495 BUG();
1496 }
1497
1498 return allocate_slab(s,
1499 flags & (GFP_RECLAIM_MASK | GFP_CONSTRAINT_MASK), node);
1500}
1501
Christoph Lameter81819f02007-05-06 14:49:36 -07001502static void __free_slab(struct kmem_cache *s, struct page *page)
1503{
Christoph Lameter834f3d12008-04-14 19:11:31 +03001504 int order = compound_order(page);
1505 int pages = 1 << order;
Christoph Lameter81819f02007-05-06 14:49:36 -07001506
Christoph Lameteraf537b02010-07-09 14:07:14 -05001507 if (kmem_cache_debug(s)) {
Christoph Lameter81819f02007-05-06 14:49:36 -07001508 void *p;
1509
1510 slab_pad_check(s, page);
Christoph Lameter224a88b2008-04-14 19:11:31 +03001511 for_each_object(p, s, page_address(page),
1512 page->objects)
Christoph Lameterf7cb1932010-09-29 07:15:01 -05001513 check_object(s, page, p, SLUB_RED_INACTIVE);
Christoph Lameter81819f02007-05-06 14:49:36 -07001514 }
1515
Vegard Nossumb1eeab62008-11-25 16:55:53 +01001516 kmemcheck_free_shadow(page, compound_order(page));
Vegard Nossum5a896d92008-04-04 00:54:48 +02001517
Christoph Lameter81819f02007-05-06 14:49:36 -07001518 mod_zone_page_state(page_zone(page),
1519 (s->flags & SLAB_RECLAIM_ACCOUNT) ?
1520 NR_SLAB_RECLAIMABLE : NR_SLAB_UNRECLAIMABLE,
Pekka Enberg06428782008-01-07 23:20:27 -08001521 -pages);
Christoph Lameter81819f02007-05-06 14:49:36 -07001522
Mel Gorman072bb0a2012-07-31 16:43:58 -07001523 __ClearPageSlabPfmemalloc(page);
Christoph Lameter49bd5222008-04-14 18:52:18 +03001524 __ClearPageSlab(page);
Glauber Costa1f458cb2012-12-18 14:22:50 -08001525
Mel Gorman22b751c2013-02-22 16:34:59 -08001526 page_mapcount_reset(page);
Nick Piggin1eb5ac62009-05-05 19:13:44 +10001527 if (current->reclaim_state)
1528 current->reclaim_state->reclaimed_slab += pages;
Vladimir Davydovf3ccb2c42015-11-05 18:49:01 -08001529 __free_kmem_pages(page, order);
Christoph Lameter81819f02007-05-06 14:49:36 -07001530}
1531
Lai Jiangshanda9a6382011-03-10 15:22:00 +08001532#define need_reserve_slab_rcu \
1533 (sizeof(((struct page *)NULL)->lru) < sizeof(struct rcu_head))
1534
Christoph Lameter81819f02007-05-06 14:49:36 -07001535static void rcu_free_slab(struct rcu_head *h)
1536{
1537 struct page *page;
1538
Lai Jiangshanda9a6382011-03-10 15:22:00 +08001539 if (need_reserve_slab_rcu)
1540 page = virt_to_head_page(h);
1541 else
1542 page = container_of((struct list_head *)h, struct page, lru);
1543
Glauber Costa1b4f59e32012-10-22 18:05:36 +04001544 __free_slab(page->slab_cache, page);
Christoph Lameter81819f02007-05-06 14:49:36 -07001545}
1546
1547static void free_slab(struct kmem_cache *s, struct page *page)
1548{
1549 if (unlikely(s->flags & SLAB_DESTROY_BY_RCU)) {
Lai Jiangshanda9a6382011-03-10 15:22:00 +08001550 struct rcu_head *head;
1551
1552 if (need_reserve_slab_rcu) {
1553 int order = compound_order(page);
1554 int offset = (PAGE_SIZE << order) - s->reserved;
1555
1556 VM_BUG_ON(s->reserved != sizeof(*head));
1557 head = page_address(page) + offset;
1558 } else {
Kirill A. Shutemovbc4f6102015-11-06 16:29:44 -08001559 head = &page->rcu_head;
Lai Jiangshanda9a6382011-03-10 15:22:00 +08001560 }
Christoph Lameter81819f02007-05-06 14:49:36 -07001561
1562 call_rcu(head, rcu_free_slab);
1563 } else
1564 __free_slab(s, page);
1565}
1566
1567static void discard_slab(struct kmem_cache *s, struct page *page)
1568{
Christoph Lameter205ab992008-04-14 19:11:40 +03001569 dec_slabs_node(s, page_to_nid(page), page->objects);
Christoph Lameter81819f02007-05-06 14:49:36 -07001570 free_slab(s, page);
1571}
1572
1573/*
Christoph Lameter5cc6eee2011-06-01 12:25:50 -05001574 * Management of partially allocated slabs.
Christoph Lameter81819f02007-05-06 14:49:36 -07001575 */
Steven Rostedt1e4dd942014-02-10 14:25:46 -08001576static inline void
1577__add_partial(struct kmem_cache_node *n, struct page *page, int tail)
Christoph Lameter81819f02007-05-06 14:49:36 -07001578{
Christoph Lametere95eed52007-05-06 14:49:44 -07001579 n->nr_partial++;
Shaohua Li136333d2011-08-24 08:57:52 +08001580 if (tail == DEACTIVATE_TO_TAIL)
Christoph Lameter7c2e1322008-01-07 23:20:27 -08001581 list_add_tail(&page->lru, &n->partial);
1582 else
1583 list_add(&page->lru, &n->partial);
Christoph Lameter81819f02007-05-06 14:49:36 -07001584}
1585
Steven Rostedt1e4dd942014-02-10 14:25:46 -08001586static inline void add_partial(struct kmem_cache_node *n,
1587 struct page *page, int tail)
1588{
1589 lockdep_assert_held(&n->list_lock);
1590 __add_partial(n, page, tail);
1591}
1592
1593static inline void
1594__remove_partial(struct kmem_cache_node *n, struct page *page)
1595{
1596 list_del(&page->lru);
1597 n->nr_partial--;
1598}
1599
Christoph Lameter5cc6eee2011-06-01 12:25:50 -05001600static inline void remove_partial(struct kmem_cache_node *n,
Christoph Lameter62e346a2010-09-28 08:10:28 -05001601 struct page *page)
1602{
Peter Zijlstrac65c1872014-01-10 13:23:49 +01001603 lockdep_assert_held(&n->list_lock);
Steven Rostedt1e4dd942014-02-10 14:25:46 -08001604 __remove_partial(n, page);
Christoph Lameter62e346a2010-09-28 08:10:28 -05001605}
1606
Christoph Lameter81819f02007-05-06 14:49:36 -07001607/*
Christoph Lameter7ced3712012-05-09 10:09:53 -05001608 * Remove slab from the partial list, freeze it and
1609 * return the pointer to the freelist.
Christoph Lameter81819f02007-05-06 14:49:36 -07001610 *
Christoph Lameter497b66f2011-08-09 16:12:26 -05001611 * Returns a list of objects or NULL if it fails.
Christoph Lameter81819f02007-05-06 14:49:36 -07001612 */
Christoph Lameter497b66f2011-08-09 16:12:26 -05001613static inline void *acquire_slab(struct kmem_cache *s,
Christoph Lameteracd19fd2011-08-09 16:12:25 -05001614 struct kmem_cache_node *n, struct page *page,
Joonsoo Kim633b0762013-01-21 17:01:25 +09001615 int mode, int *objects)
Christoph Lameter81819f02007-05-06 14:49:36 -07001616{
Christoph Lameter2cfb7452011-06-01 12:25:52 -05001617 void *freelist;
1618 unsigned long counters;
1619 struct page new;
1620
Peter Zijlstrac65c1872014-01-10 13:23:49 +01001621 lockdep_assert_held(&n->list_lock);
1622
Christoph Lameter2cfb7452011-06-01 12:25:52 -05001623 /*
1624 * Zap the freelist and set the frozen bit.
1625 * The old freelist is the list of objects for the
1626 * per cpu allocation list.
1627 */
Christoph Lameter7ced3712012-05-09 10:09:53 -05001628 freelist = page->freelist;
1629 counters = page->counters;
1630 new.counters = counters;
Joonsoo Kim633b0762013-01-21 17:01:25 +09001631 *objects = new.objects - new.inuse;
Pekka Enberg23910c52012-06-04 10:14:58 +03001632 if (mode) {
Christoph Lameter7ced3712012-05-09 10:09:53 -05001633 new.inuse = page->objects;
Pekka Enberg23910c52012-06-04 10:14:58 +03001634 new.freelist = NULL;
1635 } else {
1636 new.freelist = freelist;
1637 }
Christoph Lameter2cfb7452011-06-01 12:25:52 -05001638
Dave Hansena0132ac2014-01-29 14:05:50 -08001639 VM_BUG_ON(new.frozen);
Christoph Lameter7ced3712012-05-09 10:09:53 -05001640 new.frozen = 1;
Christoph Lameter2cfb7452011-06-01 12:25:52 -05001641
Christoph Lameter7ced3712012-05-09 10:09:53 -05001642 if (!__cmpxchg_double_slab(s, page,
Christoph Lameter2cfb7452011-06-01 12:25:52 -05001643 freelist, counters,
Joonsoo Kim02d76332012-05-17 00:13:02 +09001644 new.freelist, new.counters,
Christoph Lameter7ced3712012-05-09 10:09:53 -05001645 "acquire_slab"))
Christoph Lameter7ced3712012-05-09 10:09:53 -05001646 return NULL;
Christoph Lameter2cfb7452011-06-01 12:25:52 -05001647
1648 remove_partial(n, page);
Christoph Lameter7ced3712012-05-09 10:09:53 -05001649 WARN_ON(!freelist);
Christoph Lameter49e22582011-08-09 16:12:27 -05001650 return freelist;
Christoph Lameter81819f02007-05-06 14:49:36 -07001651}
1652
Joonsoo Kim633b0762013-01-21 17:01:25 +09001653static void put_cpu_partial(struct kmem_cache *s, struct page *page, int drain);
Joonsoo Kim8ba00bb2012-09-17 14:09:09 -07001654static inline bool pfmemalloc_match(struct page *page, gfp_t gfpflags);
Christoph Lameter49e22582011-08-09 16:12:27 -05001655
Christoph Lameter81819f02007-05-06 14:49:36 -07001656/*
Christoph Lameter672bba32007-05-09 02:32:39 -07001657 * Try to allocate a partial slab from a specific node.
Christoph Lameter81819f02007-05-06 14:49:36 -07001658 */
Joonsoo Kim8ba00bb2012-09-17 14:09:09 -07001659static void *get_partial_node(struct kmem_cache *s, struct kmem_cache_node *n,
1660 struct kmem_cache_cpu *c, gfp_t flags)
Christoph Lameter81819f02007-05-06 14:49:36 -07001661{
Christoph Lameter49e22582011-08-09 16:12:27 -05001662 struct page *page, *page2;
1663 void *object = NULL;
Joonsoo Kim633b0762013-01-21 17:01:25 +09001664 int available = 0;
1665 int objects;
Christoph Lameter81819f02007-05-06 14:49:36 -07001666
1667 /*
1668 * Racy check. If we mistakenly see no partial slabs then we
1669 * just allocate an empty slab. If we mistakenly try to get a
Christoph Lameter672bba32007-05-09 02:32:39 -07001670 * partial slab and there is none available then get_partials()
1671 * will return NULL.
Christoph Lameter81819f02007-05-06 14:49:36 -07001672 */
1673 if (!n || !n->nr_partial)
1674 return NULL;
1675
1676 spin_lock(&n->list_lock);
Christoph Lameter49e22582011-08-09 16:12:27 -05001677 list_for_each_entry_safe(page, page2, &n->partial, lru) {
Joonsoo Kim8ba00bb2012-09-17 14:09:09 -07001678 void *t;
Christoph Lameter49e22582011-08-09 16:12:27 -05001679
Joonsoo Kim8ba00bb2012-09-17 14:09:09 -07001680 if (!pfmemalloc_match(page, flags))
1681 continue;
1682
Joonsoo Kim633b0762013-01-21 17:01:25 +09001683 t = acquire_slab(s, n, page, object == NULL, &objects);
Christoph Lameter49e22582011-08-09 16:12:27 -05001684 if (!t)
1685 break;
1686
Joonsoo Kim633b0762013-01-21 17:01:25 +09001687 available += objects;
Alex,Shi12d79632011-09-07 10:26:36 +08001688 if (!object) {
Christoph Lameter49e22582011-08-09 16:12:27 -05001689 c->page = page;
Christoph Lameter49e22582011-08-09 16:12:27 -05001690 stat(s, ALLOC_FROM_PARTIAL);
Christoph Lameter49e22582011-08-09 16:12:27 -05001691 object = t;
Christoph Lameter49e22582011-08-09 16:12:27 -05001692 } else {
Joonsoo Kim633b0762013-01-21 17:01:25 +09001693 put_cpu_partial(s, page, 0);
Alex Shi8028dce2012-02-03 23:34:56 +08001694 stat(s, CPU_PARTIAL_NODE);
Christoph Lameter49e22582011-08-09 16:12:27 -05001695 }
Joonsoo Kim345c9052013-06-19 14:05:52 +09001696 if (!kmem_cache_has_cpu_partial(s)
1697 || available > s->cpu_partial / 2)
Christoph Lameter49e22582011-08-09 16:12:27 -05001698 break;
1699
Christoph Lameter497b66f2011-08-09 16:12:26 -05001700 }
Christoph Lameter81819f02007-05-06 14:49:36 -07001701 spin_unlock(&n->list_lock);
Christoph Lameter497b66f2011-08-09 16:12:26 -05001702 return object;
Christoph Lameter81819f02007-05-06 14:49:36 -07001703}
1704
1705/*
Christoph Lameter672bba32007-05-09 02:32:39 -07001706 * Get a page from somewhere. Search in increasing NUMA distances.
Christoph Lameter81819f02007-05-06 14:49:36 -07001707 */
Joonsoo Kimde3ec032012-01-27 00:12:23 -08001708static void *get_any_partial(struct kmem_cache *s, gfp_t flags,
Christoph Lameteracd19fd2011-08-09 16:12:25 -05001709 struct kmem_cache_cpu *c)
Christoph Lameter81819f02007-05-06 14:49:36 -07001710{
1711#ifdef CONFIG_NUMA
1712 struct zonelist *zonelist;
Mel Gormandd1a2392008-04-28 02:12:17 -07001713 struct zoneref *z;
Mel Gorman54a6eb52008-04-28 02:12:16 -07001714 struct zone *zone;
1715 enum zone_type high_zoneidx = gfp_zone(flags);
Christoph Lameter497b66f2011-08-09 16:12:26 -05001716 void *object;
Mel Gormancc9a6c82012-03-21 16:34:11 -07001717 unsigned int cpuset_mems_cookie;
Christoph Lameter81819f02007-05-06 14:49:36 -07001718
1719 /*
Christoph Lameter672bba32007-05-09 02:32:39 -07001720 * The defrag ratio allows a configuration of the tradeoffs between
1721 * inter node defragmentation and node local allocations. A lower
1722 * defrag_ratio increases the tendency to do local allocations
1723 * instead of attempting to obtain partial slabs from other nodes.
Christoph Lameter81819f02007-05-06 14:49:36 -07001724 *
Christoph Lameter672bba32007-05-09 02:32:39 -07001725 * If the defrag_ratio is set to 0 then kmalloc() always
1726 * returns node local objects. If the ratio is higher then kmalloc()
1727 * may return off node objects because partial slabs are obtained
1728 * from other nodes and filled up.
Christoph Lameter81819f02007-05-06 14:49:36 -07001729 *
Christoph Lameter6446faa2008-02-15 23:45:26 -08001730 * If /sys/kernel/slab/xx/defrag_ratio is set to 100 (which makes
Christoph Lameter672bba32007-05-09 02:32:39 -07001731 * defrag_ratio = 1000) then every (well almost) allocation will
1732 * first attempt to defrag slab caches on other nodes. This means
1733 * scanning over all nodes to look for partial slabs which may be
1734 * expensive if we do it every time we are trying to find a slab
1735 * with available objects.
Christoph Lameter81819f02007-05-06 14:49:36 -07001736 */
Christoph Lameter98246012008-01-07 23:20:26 -08001737 if (!s->remote_node_defrag_ratio ||
1738 get_cycles() % 1024 > s->remote_node_defrag_ratio)
Christoph Lameter81819f02007-05-06 14:49:36 -07001739 return NULL;
1740
Mel Gormancc9a6c82012-03-21 16:34:11 -07001741 do {
Mel Gormand26914d2014-04-03 14:47:24 -07001742 cpuset_mems_cookie = read_mems_allowed_begin();
David Rientjes2a389612014-04-07 15:37:29 -07001743 zonelist = node_zonelist(mempolicy_slab_node(), flags);
Mel Gormancc9a6c82012-03-21 16:34:11 -07001744 for_each_zone_zonelist(zone, z, zonelist, high_zoneidx) {
1745 struct kmem_cache_node *n;
Christoph Lameter81819f02007-05-06 14:49:36 -07001746
Mel Gormancc9a6c82012-03-21 16:34:11 -07001747 n = get_node(s, zone_to_nid(zone));
Christoph Lameter81819f02007-05-06 14:49:36 -07001748
Vladimir Davydovdee2f8a2014-12-12 16:58:28 -08001749 if (n && cpuset_zone_allowed(zone, flags) &&
Mel Gormancc9a6c82012-03-21 16:34:11 -07001750 n->nr_partial > s->min_partial) {
Joonsoo Kim8ba00bb2012-09-17 14:09:09 -07001751 object = get_partial_node(s, n, c, flags);
Mel Gormancc9a6c82012-03-21 16:34:11 -07001752 if (object) {
1753 /*
Mel Gormand26914d2014-04-03 14:47:24 -07001754 * Don't check read_mems_allowed_retry()
1755 * here - if mems_allowed was updated in
1756 * parallel, that was a harmless race
1757 * between allocation and the cpuset
1758 * update
Mel Gormancc9a6c82012-03-21 16:34:11 -07001759 */
Mel Gormancc9a6c82012-03-21 16:34:11 -07001760 return object;
1761 }
Miao Xiec0ff7452010-05-24 14:32:08 -07001762 }
Christoph Lameter81819f02007-05-06 14:49:36 -07001763 }
Mel Gormand26914d2014-04-03 14:47:24 -07001764 } while (read_mems_allowed_retry(cpuset_mems_cookie));
Christoph Lameter81819f02007-05-06 14:49:36 -07001765#endif
1766 return NULL;
1767}
1768
1769/*
1770 * Get a partial page, lock it and return it.
1771 */
Christoph Lameter497b66f2011-08-09 16:12:26 -05001772static void *get_partial(struct kmem_cache *s, gfp_t flags, int node,
Christoph Lameteracd19fd2011-08-09 16:12:25 -05001773 struct kmem_cache_cpu *c)
Christoph Lameter81819f02007-05-06 14:49:36 -07001774{
Christoph Lameter497b66f2011-08-09 16:12:26 -05001775 void *object;
Joonsoo Kima561ce02014-10-09 15:26:15 -07001776 int searchnode = node;
1777
1778 if (node == NUMA_NO_NODE)
1779 searchnode = numa_mem_id();
1780 else if (!node_present_pages(node))
1781 searchnode = node_to_mem_node(node);
Christoph Lameter81819f02007-05-06 14:49:36 -07001782
Joonsoo Kim8ba00bb2012-09-17 14:09:09 -07001783 object = get_partial_node(s, get_node(s, searchnode), c, flags);
Christoph Lameter497b66f2011-08-09 16:12:26 -05001784 if (object || node != NUMA_NO_NODE)
1785 return object;
Christoph Lameter81819f02007-05-06 14:49:36 -07001786
Christoph Lameteracd19fd2011-08-09 16:12:25 -05001787 return get_any_partial(s, flags, c);
Christoph Lameter81819f02007-05-06 14:49:36 -07001788}
1789
Christoph Lameter8a5ec0b2011-02-25 11:38:54 -06001790#ifdef CONFIG_PREEMPT
1791/*
1792 * Calculate the next globally unique transaction for disambiguiation
1793 * during cmpxchg. The transactions start with the cpu number and are then
1794 * incremented by CONFIG_NR_CPUS.
1795 */
1796#define TID_STEP roundup_pow_of_two(CONFIG_NR_CPUS)
1797#else
1798/*
1799 * No preemption supported therefore also no need to check for
1800 * different cpus.
1801 */
1802#define TID_STEP 1
1803#endif
1804
1805static inline unsigned long next_tid(unsigned long tid)
1806{
1807 return tid + TID_STEP;
1808}
1809
1810static inline unsigned int tid_to_cpu(unsigned long tid)
1811{
1812 return tid % TID_STEP;
1813}
1814
1815static inline unsigned long tid_to_event(unsigned long tid)
1816{
1817 return tid / TID_STEP;
1818}
1819
1820static inline unsigned int init_tid(int cpu)
1821{
1822 return cpu;
1823}
1824
1825static inline void note_cmpxchg_failure(const char *n,
1826 const struct kmem_cache *s, unsigned long tid)
1827{
1828#ifdef SLUB_DEBUG_CMPXCHG
1829 unsigned long actual_tid = __this_cpu_read(s->cpu_slab->tid);
1830
Fabian Frederickf9f58282014-06-04 16:06:34 -07001831 pr_info("%s %s: cmpxchg redo ", n, s->name);
Christoph Lameter8a5ec0b2011-02-25 11:38:54 -06001832
1833#ifdef CONFIG_PREEMPT
1834 if (tid_to_cpu(tid) != tid_to_cpu(actual_tid))
Fabian Frederickf9f58282014-06-04 16:06:34 -07001835 pr_warn("due to cpu change %d -> %d\n",
Christoph Lameter8a5ec0b2011-02-25 11:38:54 -06001836 tid_to_cpu(tid), tid_to_cpu(actual_tid));
1837 else
1838#endif
1839 if (tid_to_event(tid) != tid_to_event(actual_tid))
Fabian Frederickf9f58282014-06-04 16:06:34 -07001840 pr_warn("due to cpu running other code. Event %ld->%ld\n",
Christoph Lameter8a5ec0b2011-02-25 11:38:54 -06001841 tid_to_event(tid), tid_to_event(actual_tid));
1842 else
Fabian Frederickf9f58282014-06-04 16:06:34 -07001843 pr_warn("for unknown reason: actual=%lx was=%lx target=%lx\n",
Christoph Lameter8a5ec0b2011-02-25 11:38:54 -06001844 actual_tid, tid, next_tid(tid));
1845#endif
Christoph Lameter4fdccdf2011-03-22 13:35:00 -05001846 stat(s, CMPXCHG_DOUBLE_CPU_FAIL);
Christoph Lameter8a5ec0b2011-02-25 11:38:54 -06001847}
1848
Fengguang Wu788e1aa2012-09-28 16:34:05 +08001849static void init_kmem_cache_cpus(struct kmem_cache *s)
Christoph Lameter8a5ec0b2011-02-25 11:38:54 -06001850{
Christoph Lameter8a5ec0b2011-02-25 11:38:54 -06001851 int cpu;
1852
1853 for_each_possible_cpu(cpu)
1854 per_cpu_ptr(s->cpu_slab, cpu)->tid = init_tid(cpu);
Christoph Lameter8a5ec0b2011-02-25 11:38:54 -06001855}
Christoph Lameter2cfb7452011-06-01 12:25:52 -05001856
1857/*
1858 * Remove the cpu slab
1859 */
Chen Gangd0e0ac92013-07-15 09:05:29 +08001860static void deactivate_slab(struct kmem_cache *s, struct page *page,
1861 void *freelist)
Christoph Lameter81819f02007-05-06 14:49:36 -07001862{
Christoph Lameter2cfb7452011-06-01 12:25:52 -05001863 enum slab_modes { M_NONE, M_PARTIAL, M_FULL, M_FREE };
Christoph Lameter2cfb7452011-06-01 12:25:52 -05001864 struct kmem_cache_node *n = get_node(s, page_to_nid(page));
1865 int lock = 0;
1866 enum slab_modes l = M_NONE, m = M_NONE;
Christoph Lameter2cfb7452011-06-01 12:25:52 -05001867 void *nextfree;
Shaohua Li136333d2011-08-24 08:57:52 +08001868 int tail = DEACTIVATE_TO_HEAD;
Christoph Lameter2cfb7452011-06-01 12:25:52 -05001869 struct page new;
1870 struct page old;
Christoph Lameter8ff12cf2008-02-07 17:47:41 -08001871
Christoph Lameter2cfb7452011-06-01 12:25:52 -05001872 if (page->freelist) {
Christoph Lameter84e554e62009-12-18 16:26:23 -06001873 stat(s, DEACTIVATE_REMOTE_FREES);
Shaohua Li136333d2011-08-24 08:57:52 +08001874 tail = DEACTIVATE_TO_TAIL;
Christoph Lameter894b8782007-05-10 03:15:16 -07001875 }
Christoph Lameter2cfb7452011-06-01 12:25:52 -05001876
Christoph Lameter2cfb7452011-06-01 12:25:52 -05001877 /*
1878 * Stage one: Free all available per cpu objects back
1879 * to the page freelist while it is still frozen. Leave the
1880 * last one.
1881 *
1882 * There is no need to take the list->lock because the page
1883 * is still frozen.
1884 */
1885 while (freelist && (nextfree = get_freepointer(s, freelist))) {
1886 void *prior;
1887 unsigned long counters;
1888
1889 do {
1890 prior = page->freelist;
1891 counters = page->counters;
1892 set_freepointer(s, freelist, prior);
1893 new.counters = counters;
1894 new.inuse--;
Dave Hansena0132ac2014-01-29 14:05:50 -08001895 VM_BUG_ON(!new.frozen);
Christoph Lameter2cfb7452011-06-01 12:25:52 -05001896
Christoph Lameter1d071712011-07-14 12:49:12 -05001897 } while (!__cmpxchg_double_slab(s, page,
Christoph Lameter2cfb7452011-06-01 12:25:52 -05001898 prior, counters,
1899 freelist, new.counters,
1900 "drain percpu freelist"));
1901
1902 freelist = nextfree;
1903 }
1904
1905 /*
1906 * Stage two: Ensure that the page is unfrozen while the
1907 * list presence reflects the actual number of objects
1908 * during unfreeze.
1909 *
1910 * We setup the list membership and then perform a cmpxchg
1911 * with the count. If there is a mismatch then the page
1912 * is not unfrozen but the page is on the wrong list.
1913 *
1914 * Then we restart the process which may have to remove
1915 * the page from the list that we just put it on again
1916 * because the number of objects in the slab may have
1917 * changed.
1918 */
1919redo:
1920
1921 old.freelist = page->freelist;
1922 old.counters = page->counters;
Dave Hansena0132ac2014-01-29 14:05:50 -08001923 VM_BUG_ON(!old.frozen);
Christoph Lameter2cfb7452011-06-01 12:25:52 -05001924
1925 /* Determine target state of the slab */
1926 new.counters = old.counters;
1927 if (freelist) {
1928 new.inuse--;
1929 set_freepointer(s, freelist, old.freelist);
1930 new.freelist = freelist;
1931 } else
1932 new.freelist = old.freelist;
1933
1934 new.frozen = 0;
1935
Joonsoo Kim8a5b20a2014-07-02 15:22:35 -07001936 if (!new.inuse && n->nr_partial >= s->min_partial)
Christoph Lameter2cfb7452011-06-01 12:25:52 -05001937 m = M_FREE;
1938 else if (new.freelist) {
1939 m = M_PARTIAL;
1940 if (!lock) {
1941 lock = 1;
1942 /*
1943 * Taking the spinlock removes the possiblity
1944 * that acquire_slab() will see a slab page that
1945 * is frozen
1946 */
1947 spin_lock(&n->list_lock);
1948 }
1949 } else {
1950 m = M_FULL;
1951 if (kmem_cache_debug(s) && !lock) {
1952 lock = 1;
1953 /*
1954 * This also ensures that the scanning of full
1955 * slabs from diagnostic functions will not see
1956 * any frozen slabs.
1957 */
1958 spin_lock(&n->list_lock);
1959 }
1960 }
1961
1962 if (l != m) {
1963
1964 if (l == M_PARTIAL)
1965
1966 remove_partial(n, page);
1967
1968 else if (l == M_FULL)
1969
Peter Zijlstrac65c1872014-01-10 13:23:49 +01001970 remove_full(s, n, page);
Christoph Lameter2cfb7452011-06-01 12:25:52 -05001971
1972 if (m == M_PARTIAL) {
1973
1974 add_partial(n, page, tail);
Shaohua Li136333d2011-08-24 08:57:52 +08001975 stat(s, tail);
Christoph Lameter2cfb7452011-06-01 12:25:52 -05001976
1977 } else if (m == M_FULL) {
1978
1979 stat(s, DEACTIVATE_FULL);
1980 add_full(s, n, page);
1981
1982 }
1983 }
1984
1985 l = m;
Christoph Lameter1d071712011-07-14 12:49:12 -05001986 if (!__cmpxchg_double_slab(s, page,
Christoph Lameter2cfb7452011-06-01 12:25:52 -05001987 old.freelist, old.counters,
1988 new.freelist, new.counters,
1989 "unfreezing slab"))
1990 goto redo;
1991
Christoph Lameter2cfb7452011-06-01 12:25:52 -05001992 if (lock)
1993 spin_unlock(&n->list_lock);
1994
1995 if (m == M_FREE) {
1996 stat(s, DEACTIVATE_EMPTY);
1997 discard_slab(s, page);
1998 stat(s, FREE_SLAB);
1999 }
Christoph Lameter81819f02007-05-06 14:49:36 -07002000}
2001
Joonsoo Kimd24ac772012-05-18 22:01:17 +09002002/*
2003 * Unfreeze all the cpu partial slabs.
2004 *
Christoph Lameter59a09912012-11-28 16:23:00 +00002005 * This function must be called with interrupts disabled
2006 * for the cpu using c (or some other guarantee must be there
2007 * to guarantee no concurrent accesses).
Joonsoo Kimd24ac772012-05-18 22:01:17 +09002008 */
Christoph Lameter59a09912012-11-28 16:23:00 +00002009static void unfreeze_partials(struct kmem_cache *s,
2010 struct kmem_cache_cpu *c)
Christoph Lameter49e22582011-08-09 16:12:27 -05002011{
Joonsoo Kim345c9052013-06-19 14:05:52 +09002012#ifdef CONFIG_SLUB_CPU_PARTIAL
Joonsoo Kim43d77862012-06-09 02:23:16 +09002013 struct kmem_cache_node *n = NULL, *n2 = NULL;
Shaohua Li9ada1932011-11-14 13:34:13 +08002014 struct page *page, *discard_page = NULL;
Christoph Lameter49e22582011-08-09 16:12:27 -05002015
2016 while ((page = c->partial)) {
Christoph Lameter49e22582011-08-09 16:12:27 -05002017 struct page new;
2018 struct page old;
2019
2020 c->partial = page->next;
Joonsoo Kim43d77862012-06-09 02:23:16 +09002021
2022 n2 = get_node(s, page_to_nid(page));
2023 if (n != n2) {
2024 if (n)
2025 spin_unlock(&n->list_lock);
2026
2027 n = n2;
2028 spin_lock(&n->list_lock);
2029 }
Christoph Lameter49e22582011-08-09 16:12:27 -05002030
2031 do {
2032
2033 old.freelist = page->freelist;
2034 old.counters = page->counters;
Dave Hansena0132ac2014-01-29 14:05:50 -08002035 VM_BUG_ON(!old.frozen);
Christoph Lameter49e22582011-08-09 16:12:27 -05002036
2037 new.counters = old.counters;
2038 new.freelist = old.freelist;
2039
2040 new.frozen = 0;
2041
Joonsoo Kimd24ac772012-05-18 22:01:17 +09002042 } while (!__cmpxchg_double_slab(s, page,
Christoph Lameter49e22582011-08-09 16:12:27 -05002043 old.freelist, old.counters,
2044 new.freelist, new.counters,
2045 "unfreezing slab"));
2046
Joonsoo Kim8a5b20a2014-07-02 15:22:35 -07002047 if (unlikely(!new.inuse && n->nr_partial >= s->min_partial)) {
Shaohua Li9ada1932011-11-14 13:34:13 +08002048 page->next = discard_page;
2049 discard_page = page;
Joonsoo Kim43d77862012-06-09 02:23:16 +09002050 } else {
2051 add_partial(n, page, DEACTIVATE_TO_TAIL);
2052 stat(s, FREE_ADD_PARTIAL);
Christoph Lameter49e22582011-08-09 16:12:27 -05002053 }
2054 }
2055
2056 if (n)
2057 spin_unlock(&n->list_lock);
Shaohua Li9ada1932011-11-14 13:34:13 +08002058
2059 while (discard_page) {
2060 page = discard_page;
2061 discard_page = discard_page->next;
2062
2063 stat(s, DEACTIVATE_EMPTY);
2064 discard_slab(s, page);
2065 stat(s, FREE_SLAB);
2066 }
Joonsoo Kim345c9052013-06-19 14:05:52 +09002067#endif
Christoph Lameter49e22582011-08-09 16:12:27 -05002068}
2069
2070/*
2071 * Put a page that was just frozen (in __slab_free) into a partial page
2072 * slot if available. This is done without interrupts disabled and without
2073 * preemption disabled. The cmpxchg is racy and may put the partial page
2074 * onto a random cpus partial slot.
2075 *
2076 * If we did not find a slot then simply move all the partials to the
2077 * per node partial list.
2078 */
Joonsoo Kim633b0762013-01-21 17:01:25 +09002079static void put_cpu_partial(struct kmem_cache *s, struct page *page, int drain)
Christoph Lameter49e22582011-08-09 16:12:27 -05002080{
Joonsoo Kim345c9052013-06-19 14:05:52 +09002081#ifdef CONFIG_SLUB_CPU_PARTIAL
Christoph Lameter49e22582011-08-09 16:12:27 -05002082 struct page *oldpage;
2083 int pages;
2084 int pobjects;
2085
Vladimir Davydovd6e0b7f2015-02-12 14:59:47 -08002086 preempt_disable();
Christoph Lameter49e22582011-08-09 16:12:27 -05002087 do {
2088 pages = 0;
2089 pobjects = 0;
2090 oldpage = this_cpu_read(s->cpu_slab->partial);
2091
2092 if (oldpage) {
2093 pobjects = oldpage->pobjects;
2094 pages = oldpage->pages;
2095 if (drain && pobjects > s->cpu_partial) {
2096 unsigned long flags;
2097 /*
2098 * partial array is full. Move the existing
2099 * set to the per node partial list.
2100 */
2101 local_irq_save(flags);
Christoph Lameter59a09912012-11-28 16:23:00 +00002102 unfreeze_partials(s, this_cpu_ptr(s->cpu_slab));
Christoph Lameter49e22582011-08-09 16:12:27 -05002103 local_irq_restore(flags);
Joonsoo Kime24fc412012-06-23 03:22:38 +09002104 oldpage = NULL;
Christoph Lameter49e22582011-08-09 16:12:27 -05002105 pobjects = 0;
2106 pages = 0;
Alex Shi8028dce2012-02-03 23:34:56 +08002107 stat(s, CPU_PARTIAL_DRAIN);
Christoph Lameter49e22582011-08-09 16:12:27 -05002108 }
2109 }
2110
2111 pages++;
2112 pobjects += page->objects - page->inuse;
2113
2114 page->pages = pages;
2115 page->pobjects = pobjects;
2116 page->next = oldpage;
2117
Chen Gangd0e0ac92013-07-15 09:05:29 +08002118 } while (this_cpu_cmpxchg(s->cpu_slab->partial, oldpage, page)
2119 != oldpage);
Vladimir Davydovd6e0b7f2015-02-12 14:59:47 -08002120 if (unlikely(!s->cpu_partial)) {
2121 unsigned long flags;
2122
2123 local_irq_save(flags);
2124 unfreeze_partials(s, this_cpu_ptr(s->cpu_slab));
2125 local_irq_restore(flags);
2126 }
2127 preempt_enable();
Joonsoo Kim345c9052013-06-19 14:05:52 +09002128#endif
Christoph Lameter49e22582011-08-09 16:12:27 -05002129}
2130
Christoph Lameterdfb4f092007-10-16 01:26:05 -07002131static inline void flush_slab(struct kmem_cache *s, struct kmem_cache_cpu *c)
Christoph Lameter81819f02007-05-06 14:49:36 -07002132{
Christoph Lameter84e554e62009-12-18 16:26:23 -06002133 stat(s, CPUSLAB_FLUSH);
Christoph Lameterc17dda42012-05-09 10:09:57 -05002134 deactivate_slab(s, c->page, c->freelist);
2135
2136 c->tid = next_tid(c->tid);
2137 c->page = NULL;
2138 c->freelist = NULL;
Christoph Lameter81819f02007-05-06 14:49:36 -07002139}
2140
2141/*
2142 * Flush cpu slab.
Christoph Lameter6446faa2008-02-15 23:45:26 -08002143 *
Christoph Lameter81819f02007-05-06 14:49:36 -07002144 * Called from IPI handler with interrupts disabled.
2145 */
Christoph Lameter0c710012007-07-17 04:03:24 -07002146static inline void __flush_cpu_slab(struct kmem_cache *s, int cpu)
Christoph Lameter81819f02007-05-06 14:49:36 -07002147{
Christoph Lameter9dfc6e62009-12-18 16:26:20 -06002148 struct kmem_cache_cpu *c = per_cpu_ptr(s->cpu_slab, cpu);
Christoph Lameter81819f02007-05-06 14:49:36 -07002149
Christoph Lameter49e22582011-08-09 16:12:27 -05002150 if (likely(c)) {
2151 if (c->page)
2152 flush_slab(s, c);
2153
Christoph Lameter59a09912012-11-28 16:23:00 +00002154 unfreeze_partials(s, c);
Christoph Lameter49e22582011-08-09 16:12:27 -05002155 }
Christoph Lameter81819f02007-05-06 14:49:36 -07002156}
2157
2158static void flush_cpu_slab(void *d)
2159{
2160 struct kmem_cache *s = d;
Christoph Lameter81819f02007-05-06 14:49:36 -07002161
Christoph Lameterdfb4f092007-10-16 01:26:05 -07002162 __flush_cpu_slab(s, smp_processor_id());
Christoph Lameter81819f02007-05-06 14:49:36 -07002163}
2164
Gilad Ben-Yossefa8364d52012-03-28 14:42:44 -07002165static bool has_cpu_slab(int cpu, void *info)
2166{
2167 struct kmem_cache *s = info;
2168 struct kmem_cache_cpu *c = per_cpu_ptr(s->cpu_slab, cpu);
2169
majianpeng02e1a9c2012-05-17 17:03:26 -07002170 return c->page || c->partial;
Gilad Ben-Yossefa8364d52012-03-28 14:42:44 -07002171}
2172
Christoph Lameter81819f02007-05-06 14:49:36 -07002173static void flush_all(struct kmem_cache *s)
2174{
Gilad Ben-Yossefa8364d52012-03-28 14:42:44 -07002175 on_each_cpu_cond(has_cpu_slab, flush_cpu_slab, s, 1, GFP_ATOMIC);
Christoph Lameter81819f02007-05-06 14:49:36 -07002176}
2177
2178/*
Christoph Lameterdfb4f092007-10-16 01:26:05 -07002179 * Check if the objects in a per cpu structure fit numa
2180 * locality expectations.
2181 */
Christoph Lameter57d437d2012-05-09 10:09:59 -05002182static inline int node_match(struct page *page, int node)
Christoph Lameterdfb4f092007-10-16 01:26:05 -07002183{
2184#ifdef CONFIG_NUMA
Christoph Lameter4d7868e2013-01-23 21:45:47 +00002185 if (!page || (node != NUMA_NO_NODE && page_to_nid(page) != node))
Christoph Lameterdfb4f092007-10-16 01:26:05 -07002186 return 0;
2187#endif
2188 return 1;
2189}
2190
David Rientjes9a02d692014-06-04 16:06:36 -07002191#ifdef CONFIG_SLUB_DEBUG
Pekka Enberg781b2ba2009-06-10 18:50:32 +03002192static int count_free(struct page *page)
2193{
2194 return page->objects - page->inuse;
2195}
2196
David Rientjes9a02d692014-06-04 16:06:36 -07002197static inline unsigned long node_nr_objs(struct kmem_cache_node *n)
2198{
2199 return atomic_long_read(&n->total_objects);
2200}
2201#endif /* CONFIG_SLUB_DEBUG */
2202
2203#if defined(CONFIG_SLUB_DEBUG) || defined(CONFIG_SYSFS)
Pekka Enberg781b2ba2009-06-10 18:50:32 +03002204static unsigned long count_partial(struct kmem_cache_node *n,
2205 int (*get_count)(struct page *))
2206{
2207 unsigned long flags;
2208 unsigned long x = 0;
2209 struct page *page;
2210
2211 spin_lock_irqsave(&n->list_lock, flags);
2212 list_for_each_entry(page, &n->partial, lru)
2213 x += get_count(page);
2214 spin_unlock_irqrestore(&n->list_lock, flags);
2215 return x;
2216}
David Rientjes9a02d692014-06-04 16:06:36 -07002217#endif /* CONFIG_SLUB_DEBUG || CONFIG_SYSFS */
Alexander Beregalov26c02cf2009-06-11 14:08:48 +04002218
Pekka Enberg781b2ba2009-06-10 18:50:32 +03002219static noinline void
2220slab_out_of_memory(struct kmem_cache *s, gfp_t gfpflags, int nid)
2221{
David Rientjes9a02d692014-06-04 16:06:36 -07002222#ifdef CONFIG_SLUB_DEBUG
2223 static DEFINE_RATELIMIT_STATE(slub_oom_rs, DEFAULT_RATELIMIT_INTERVAL,
2224 DEFAULT_RATELIMIT_BURST);
Pekka Enberg781b2ba2009-06-10 18:50:32 +03002225 int node;
Christoph Lameterfa45dc22014-08-06 16:04:09 -07002226 struct kmem_cache_node *n;
Pekka Enberg781b2ba2009-06-10 18:50:32 +03002227
David Rientjes9a02d692014-06-04 16:06:36 -07002228 if ((gfpflags & __GFP_NOWARN) || !__ratelimit(&slub_oom_rs))
2229 return;
2230
Fabian Frederickf9f58282014-06-04 16:06:34 -07002231 pr_warn("SLUB: Unable to allocate memory on node %d (gfp=0x%x)\n",
Pekka Enberg781b2ba2009-06-10 18:50:32 +03002232 nid, gfpflags);
Fabian Frederickf9f58282014-06-04 16:06:34 -07002233 pr_warn(" cache: %s, object size: %d, buffer size: %d, default order: %d, min order: %d\n",
2234 s->name, s->object_size, s->size, oo_order(s->oo),
2235 oo_order(s->min));
Pekka Enberg781b2ba2009-06-10 18:50:32 +03002236
Christoph Lameter3b0efdf2012-06-13 10:24:57 -05002237 if (oo_order(s->min) > get_order(s->object_size))
Fabian Frederickf9f58282014-06-04 16:06:34 -07002238 pr_warn(" %s debugging increased min order, use slub_debug=O to disable.\n",
2239 s->name);
David Rientjesfa5ec8a2009-07-07 00:14:14 -07002240
Christoph Lameterfa45dc22014-08-06 16:04:09 -07002241 for_each_kmem_cache_node(s, node, n) {
Pekka Enberg781b2ba2009-06-10 18:50:32 +03002242 unsigned long nr_slabs;
2243 unsigned long nr_objs;
2244 unsigned long nr_free;
2245
Alexander Beregalov26c02cf2009-06-11 14:08:48 +04002246 nr_free = count_partial(n, count_free);
2247 nr_slabs = node_nr_slabs(n);
2248 nr_objs = node_nr_objs(n);
Pekka Enberg781b2ba2009-06-10 18:50:32 +03002249
Fabian Frederickf9f58282014-06-04 16:06:34 -07002250 pr_warn(" node %d: slabs: %ld, objs: %ld, free: %ld\n",
Pekka Enberg781b2ba2009-06-10 18:50:32 +03002251 node, nr_slabs, nr_objs, nr_free);
2252 }
David Rientjes9a02d692014-06-04 16:06:36 -07002253#endif
Pekka Enberg781b2ba2009-06-10 18:50:32 +03002254}
2255
Christoph Lameter497b66f2011-08-09 16:12:26 -05002256static inline void *new_slab_objects(struct kmem_cache *s, gfp_t flags,
2257 int node, struct kmem_cache_cpu **pc)
2258{
Christoph Lameter6faa6832012-05-09 10:09:51 -05002259 void *freelist;
Christoph Lameter188fd062012-05-09 10:09:55 -05002260 struct kmem_cache_cpu *c = *pc;
2261 struct page *page;
Christoph Lameter497b66f2011-08-09 16:12:26 -05002262
Christoph Lameter188fd062012-05-09 10:09:55 -05002263 freelist = get_partial(s, flags, node, c);
2264
2265 if (freelist)
2266 return freelist;
2267
2268 page = new_slab(s, flags, node);
Christoph Lameter497b66f2011-08-09 16:12:26 -05002269 if (page) {
Christoph Lameter7c8e0182014-06-04 16:07:56 -07002270 c = raw_cpu_ptr(s->cpu_slab);
Christoph Lameter497b66f2011-08-09 16:12:26 -05002271 if (c->page)
2272 flush_slab(s, c);
2273
2274 /*
2275 * No other reference to the page yet so we can
2276 * muck around with it freely without cmpxchg
2277 */
Christoph Lameter6faa6832012-05-09 10:09:51 -05002278 freelist = page->freelist;
Christoph Lameter497b66f2011-08-09 16:12:26 -05002279 page->freelist = NULL;
2280
2281 stat(s, ALLOC_SLAB);
Christoph Lameter497b66f2011-08-09 16:12:26 -05002282 c->page = page;
2283 *pc = c;
2284 } else
Christoph Lameter6faa6832012-05-09 10:09:51 -05002285 freelist = NULL;
Christoph Lameter497b66f2011-08-09 16:12:26 -05002286
Christoph Lameter6faa6832012-05-09 10:09:51 -05002287 return freelist;
Christoph Lameter497b66f2011-08-09 16:12:26 -05002288}
2289
Mel Gorman072bb0a2012-07-31 16:43:58 -07002290static inline bool pfmemalloc_match(struct page *page, gfp_t gfpflags)
2291{
2292 if (unlikely(PageSlabPfmemalloc(page)))
2293 return gfp_pfmemalloc_allowed(gfpflags);
2294
2295 return true;
2296}
2297
Christoph Lameterdfb4f092007-10-16 01:26:05 -07002298/*
Chen Gangd0e0ac92013-07-15 09:05:29 +08002299 * Check the page->freelist of a page and either transfer the freelist to the
2300 * per cpu freelist or deactivate the page.
Christoph Lameter213eeb92011-11-11 14:07:14 -06002301 *
2302 * The page is still frozen if the return value is not NULL.
2303 *
2304 * If this function returns NULL then the page has been unfrozen.
Joonsoo Kimd24ac772012-05-18 22:01:17 +09002305 *
2306 * This function must be called with interrupt disabled.
Christoph Lameter213eeb92011-11-11 14:07:14 -06002307 */
2308static inline void *get_freelist(struct kmem_cache *s, struct page *page)
2309{
2310 struct page new;
2311 unsigned long counters;
2312 void *freelist;
2313
2314 do {
2315 freelist = page->freelist;
2316 counters = page->counters;
Christoph Lameter6faa6832012-05-09 10:09:51 -05002317
Christoph Lameter213eeb92011-11-11 14:07:14 -06002318 new.counters = counters;
Dave Hansena0132ac2014-01-29 14:05:50 -08002319 VM_BUG_ON(!new.frozen);
Christoph Lameter213eeb92011-11-11 14:07:14 -06002320
2321 new.inuse = page->objects;
2322 new.frozen = freelist != NULL;
2323
Joonsoo Kimd24ac772012-05-18 22:01:17 +09002324 } while (!__cmpxchg_double_slab(s, page,
Christoph Lameter213eeb92011-11-11 14:07:14 -06002325 freelist, counters,
2326 NULL, new.counters,
2327 "get_freelist"));
2328
2329 return freelist;
2330}
2331
2332/*
Christoph Lameter894b8782007-05-10 03:15:16 -07002333 * Slow path. The lockless freelist is empty or we need to perform
2334 * debugging duties.
Christoph Lameter81819f02007-05-06 14:49:36 -07002335 *
Christoph Lameter894b8782007-05-10 03:15:16 -07002336 * Processing is still very fast if new objects have been freed to the
2337 * regular freelist. In that case we simply take over the regular freelist
2338 * as the lockless freelist and zap the regular freelist.
Christoph Lameter81819f02007-05-06 14:49:36 -07002339 *
Christoph Lameter894b8782007-05-10 03:15:16 -07002340 * If that is not working then we fall back to the partial lists. We take the
2341 * first element of the freelist as the object to allocate now and move the
2342 * rest of the freelist to the lockless freelist.
2343 *
2344 * And if we were unable to get a new slab from the partial slab lists then
Christoph Lameter6446faa2008-02-15 23:45:26 -08002345 * we need to allocate a new slab. This is the slowest path since it involves
2346 * a call to the page allocator and the setup of a new slab.
Christoph Lametera380a3c2015-11-20 15:57:35 -08002347 *
2348 * Version of __slab_alloc to use when we know that interrupts are
2349 * already disabled (which is the case for bulk allocation).
Christoph Lameter81819f02007-05-06 14:49:36 -07002350 */
Christoph Lametera380a3c2015-11-20 15:57:35 -08002351static void *___slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node,
Eduard - Gabriel Munteanuce71e272008-08-19 20:43:25 +03002352 unsigned long addr, struct kmem_cache_cpu *c)
Christoph Lameter81819f02007-05-06 14:49:36 -07002353{
Christoph Lameter6faa6832012-05-09 10:09:51 -05002354 void *freelist;
Christoph Lameterf6e7def2012-05-09 10:09:58 -05002355 struct page *page;
Christoph Lameter81819f02007-05-06 14:49:36 -07002356
Christoph Lameterf6e7def2012-05-09 10:09:58 -05002357 page = c->page;
2358 if (!page)
Christoph Lameter81819f02007-05-06 14:49:36 -07002359 goto new_slab;
Christoph Lameter49e22582011-08-09 16:12:27 -05002360redo:
Christoph Lameter6faa6832012-05-09 10:09:51 -05002361
Christoph Lameter57d437d2012-05-09 10:09:59 -05002362 if (unlikely(!node_match(page, node))) {
Joonsoo Kima561ce02014-10-09 15:26:15 -07002363 int searchnode = node;
2364
2365 if (node != NUMA_NO_NODE && !node_present_pages(node))
2366 searchnode = node_to_mem_node(node);
2367
2368 if (unlikely(!node_match(page, searchnode))) {
2369 stat(s, ALLOC_NODE_MISMATCH);
2370 deactivate_slab(s, page, c->freelist);
2371 c->page = NULL;
2372 c->freelist = NULL;
2373 goto new_slab;
2374 }
Christoph Lameterfc59c052011-06-01 12:25:56 -05002375 }
Christoph Lameter6446faa2008-02-15 23:45:26 -08002376
Mel Gorman072bb0a2012-07-31 16:43:58 -07002377 /*
2378 * By rights, we should be searching for a slab page that was
2379 * PFMEMALLOC but right now, we are losing the pfmemalloc
2380 * information when the page leaves the per-cpu allocator
2381 */
2382 if (unlikely(!pfmemalloc_match(page, gfpflags))) {
2383 deactivate_slab(s, page, c->freelist);
2384 c->page = NULL;
2385 c->freelist = NULL;
2386 goto new_slab;
2387 }
2388
Eric Dumazet73736e02011-12-13 04:57:06 +01002389 /* must check again c->freelist in case of cpu migration or IRQ */
Christoph Lameter6faa6832012-05-09 10:09:51 -05002390 freelist = c->freelist;
2391 if (freelist)
Eric Dumazet73736e02011-12-13 04:57:06 +01002392 goto load_freelist;
2393
Christoph Lameterf6e7def2012-05-09 10:09:58 -05002394 freelist = get_freelist(s, page);
Christoph Lameter6446faa2008-02-15 23:45:26 -08002395
Christoph Lameter6faa6832012-05-09 10:09:51 -05002396 if (!freelist) {
Christoph Lameter03e404a2011-06-01 12:25:58 -05002397 c->page = NULL;
2398 stat(s, DEACTIVATE_BYPASS);
Christoph Lameterfc59c052011-06-01 12:25:56 -05002399 goto new_slab;
Christoph Lameter03e404a2011-06-01 12:25:58 -05002400 }
Christoph Lameterdfb4f092007-10-16 01:26:05 -07002401
Christoph Lameter81819f02007-05-06 14:49:36 -07002402 stat(s, ALLOC_REFILL);
Christoph Lameter8ff12cf2008-02-07 17:47:41 -08002403
Christoph Lameter894b8782007-05-10 03:15:16 -07002404load_freelist:
Christoph Lameter507effe2012-05-09 10:09:52 -05002405 /*
2406 * freelist is pointing to the list of objects to be used.
2407 * page is pointing to the page from which the objects are obtained.
2408 * That page must be frozen for per cpu allocations to work.
2409 */
Dave Hansena0132ac2014-01-29 14:05:50 -08002410 VM_BUG_ON(!c->page->frozen);
Christoph Lameter6faa6832012-05-09 10:09:51 -05002411 c->freelist = get_freepointer(s, freelist);
Christoph Lameter8a5ec0b2011-02-25 11:38:54 -06002412 c->tid = next_tid(c->tid);
Christoph Lameter6faa6832012-05-09 10:09:51 -05002413 return freelist;
Christoph Lameter81819f02007-05-06 14:49:36 -07002414
Christoph Lameter81819f02007-05-06 14:49:36 -07002415new_slab:
Christoph Lameter2cfb7452011-06-01 12:25:52 -05002416
Christoph Lameter49e22582011-08-09 16:12:27 -05002417 if (c->partial) {
Christoph Lameterf6e7def2012-05-09 10:09:58 -05002418 page = c->page = c->partial;
2419 c->partial = page->next;
Christoph Lameter49e22582011-08-09 16:12:27 -05002420 stat(s, CPU_PARTIAL_ALLOC);
2421 c->freelist = NULL;
2422 goto redo;
Christoph Lameter81819f02007-05-06 14:49:36 -07002423 }
2424
Christoph Lameter188fd062012-05-09 10:09:55 -05002425 freelist = new_slab_objects(s, gfpflags, node, &c);
Christoph Lameterb811c202007-10-16 23:25:51 -07002426
Christoph Lameterf46974362012-05-09 10:09:54 -05002427 if (unlikely(!freelist)) {
David Rientjes9a02d692014-06-04 16:06:36 -07002428 slab_out_of_memory(s, gfpflags, node);
Christoph Lameterf46974362012-05-09 10:09:54 -05002429 return NULL;
Christoph Lameter81819f02007-05-06 14:49:36 -07002430 }
Christoph Lameter894b8782007-05-10 03:15:16 -07002431
Christoph Lameterf6e7def2012-05-09 10:09:58 -05002432 page = c->page;
Christoph Lameter5091b742012-07-31 16:44:00 -07002433 if (likely(!kmem_cache_debug(s) && pfmemalloc_match(page, gfpflags)))
Christoph Lameter81819f02007-05-06 14:49:36 -07002434 goto load_freelist;
Christoph Lameter894b8782007-05-10 03:15:16 -07002435
Christoph Lameter497b66f2011-08-09 16:12:26 -05002436 /* Only entered in the debug case */
Chen Gangd0e0ac92013-07-15 09:05:29 +08002437 if (kmem_cache_debug(s) &&
2438 !alloc_debug_processing(s, page, freelist, addr))
Christoph Lameter497b66f2011-08-09 16:12:26 -05002439 goto new_slab; /* Slab failed checks. Next slab needed */
Christoph Lameter2cfb7452011-06-01 12:25:52 -05002440
Christoph Lameterf6e7def2012-05-09 10:09:58 -05002441 deactivate_slab(s, page, get_freepointer(s, freelist));
Christoph Lameterc17dda42012-05-09 10:09:57 -05002442 c->page = NULL;
2443 c->freelist = NULL;
Christoph Lameter6faa6832012-05-09 10:09:51 -05002444 return freelist;
Christoph Lameter894b8782007-05-10 03:15:16 -07002445}
2446
2447/*
Christoph Lametera380a3c2015-11-20 15:57:35 -08002448 * Another one that disabled interrupt and compensates for possible
2449 * cpu changes by refetching the per cpu area pointer.
2450 */
2451static void *__slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node,
2452 unsigned long addr, struct kmem_cache_cpu *c)
2453{
2454 void *p;
2455 unsigned long flags;
2456
2457 local_irq_save(flags);
2458#ifdef CONFIG_PREEMPT
2459 /*
2460 * We may have been preempted and rescheduled on a different
2461 * cpu before disabling interrupts. Need to reload cpu area
2462 * pointer.
2463 */
2464 c = this_cpu_ptr(s->cpu_slab);
2465#endif
2466
2467 p = ___slab_alloc(s, gfpflags, node, addr, c);
2468 local_irq_restore(flags);
2469 return p;
2470}
2471
2472/*
Christoph Lameter894b8782007-05-10 03:15:16 -07002473 * Inlined fastpath so that allocation functions (kmalloc, kmem_cache_alloc)
2474 * have the fastpath folded into their functions. So no function call
2475 * overhead for requests that can be satisfied on the fastpath.
2476 *
2477 * The fastpath works by first checking if the lockless freelist can be used.
2478 * If not then __slab_alloc is called for slow processing.
2479 *
2480 * Otherwise we can simply pick the next object from the lockless free list.
2481 */
Ezequiel Garcia2b847c32012-09-08 17:47:58 -03002482static __always_inline void *slab_alloc_node(struct kmem_cache *s,
Eduard - Gabriel Munteanuce71e272008-08-19 20:43:25 +03002483 gfp_t gfpflags, int node, unsigned long addr)
Christoph Lameter894b8782007-05-10 03:15:16 -07002484{
Jesper Dangaard Brouer03ec0ed2015-11-20 15:57:52 -08002485 void *object;
Christoph Lameterdfb4f092007-10-16 01:26:05 -07002486 struct kmem_cache_cpu *c;
Christoph Lameter57d437d2012-05-09 10:09:59 -05002487 struct page *page;
Christoph Lameter8a5ec0b2011-02-25 11:38:54 -06002488 unsigned long tid;
Christoph Lameter1f842602008-01-07 23:20:30 -08002489
Vladimir Davydov8135be52014-12-12 16:56:38 -08002490 s = slab_pre_alloc_hook(s, gfpflags);
2491 if (!s)
Akinobu Mita773ff602008-12-23 19:37:01 +09002492 return NULL;
Christoph Lameter8a5ec0b2011-02-25 11:38:54 -06002493redo:
Christoph Lameter8a5ec0b2011-02-25 11:38:54 -06002494 /*
2495 * Must read kmem_cache cpu data via this cpu ptr. Preemption is
2496 * enabled. We may switch back and forth between cpus while
2497 * reading from one cpu area. That does not matter as long
2498 * as we end up on the original cpu again when doing the cmpxchg.
Christoph Lameter7cccd802013-01-23 21:45:48 +00002499 *
Joonsoo Kim9aabf812015-02-10 14:09:32 -08002500 * We should guarantee that tid and kmem_cache are retrieved on
2501 * the same cpu. It could be different if CONFIG_PREEMPT so we need
2502 * to check if it is matched or not.
Christoph Lameter8a5ec0b2011-02-25 11:38:54 -06002503 */
Joonsoo Kim9aabf812015-02-10 14:09:32 -08002504 do {
2505 tid = this_cpu_read(s->cpu_slab->tid);
2506 c = raw_cpu_ptr(s->cpu_slab);
Mark Rutland859b7a02015-03-25 15:55:23 -07002507 } while (IS_ENABLED(CONFIG_PREEMPT) &&
2508 unlikely(tid != READ_ONCE(c->tid)));
Joonsoo Kim9aabf812015-02-10 14:09:32 -08002509
2510 /*
2511 * Irqless object alloc/free algorithm used here depends on sequence
2512 * of fetching cpu_slab's data. tid should be fetched before anything
2513 * on c to guarantee that object and page associated with previous tid
2514 * won't be used with current tid. If we fetch tid first, object and
2515 * page could be one associated with next tid and our alloc/free
2516 * request will be failed. In this case, we will retry. So, no problem.
2517 */
2518 barrier();
Christoph Lameter8a5ec0b2011-02-25 11:38:54 -06002519
Christoph Lameter8a5ec0b2011-02-25 11:38:54 -06002520 /*
2521 * The transaction ids are globally unique per cpu and per operation on
2522 * a per cpu queue. Thus they can be guarantee that the cmpxchg_double
2523 * occurs on the right processor and that there was no operation on the
2524 * linked list in between.
2525 */
Christoph Lameter8a5ec0b2011-02-25 11:38:54 -06002526
Christoph Lameter9dfc6e62009-12-18 16:26:20 -06002527 object = c->freelist;
Christoph Lameter57d437d2012-05-09 10:09:59 -05002528 page = c->page;
Dave Hansen8eae1492014-06-04 16:06:37 -07002529 if (unlikely(!object || !node_match(page, node))) {
Christoph Lameterdfb4f092007-10-16 01:26:05 -07002530 object = __slab_alloc(s, gfpflags, node, addr, c);
Dave Hansen8eae1492014-06-04 16:06:37 -07002531 stat(s, ALLOC_SLOWPATH);
2532 } else {
Eric Dumazet0ad95002011-12-16 16:25:34 +01002533 void *next_object = get_freepointer_safe(s, object);
2534
Christoph Lameter8a5ec0b2011-02-25 11:38:54 -06002535 /*
Lucas De Marchi25985ed2011-03-30 22:57:33 -03002536 * The cmpxchg will only match if there was no additional
Christoph Lameter8a5ec0b2011-02-25 11:38:54 -06002537 * operation and if we are on the right processor.
2538 *
Chen Gangd0e0ac92013-07-15 09:05:29 +08002539 * The cmpxchg does the following atomically (without lock
2540 * semantics!)
Christoph Lameter8a5ec0b2011-02-25 11:38:54 -06002541 * 1. Relocate first pointer to the current per cpu area.
2542 * 2. Verify that tid and freelist have not been changed
2543 * 3. If they were not changed replace tid and freelist
2544 *
Chen Gangd0e0ac92013-07-15 09:05:29 +08002545 * Since this is without lock semantics the protection is only
2546 * against code executing on this cpu *not* from access by
2547 * other cpus.
Christoph Lameter8a5ec0b2011-02-25 11:38:54 -06002548 */
Christoph Lameter933393f2011-12-22 11:58:51 -06002549 if (unlikely(!this_cpu_cmpxchg_double(
Christoph Lameter8a5ec0b2011-02-25 11:38:54 -06002550 s->cpu_slab->freelist, s->cpu_slab->tid,
2551 object, tid,
Eric Dumazet0ad95002011-12-16 16:25:34 +01002552 next_object, next_tid(tid)))) {
Christoph Lameter8a5ec0b2011-02-25 11:38:54 -06002553
2554 note_cmpxchg_failure("slab_alloc", s, tid);
2555 goto redo;
2556 }
Eric Dumazet0ad95002011-12-16 16:25:34 +01002557 prefetch_freepointer(s, next_object);
Christoph Lameter84e554e62009-12-18 16:26:23 -06002558 stat(s, ALLOC_FASTPATH);
Christoph Lameter894b8782007-05-10 03:15:16 -07002559 }
Christoph Lameter8a5ec0b2011-02-25 11:38:54 -06002560
Pekka Enberg74e21342009-11-25 20:14:48 +02002561 if (unlikely(gfpflags & __GFP_ZERO) && object)
Christoph Lameter3b0efdf2012-06-13 10:24:57 -05002562 memset(object, 0, s->object_size);
Christoph Lameterd07dbea2007-07-17 04:03:23 -07002563
Jesper Dangaard Brouer03ec0ed2015-11-20 15:57:52 -08002564 slab_post_alloc_hook(s, gfpflags, 1, &object);
Vegard Nossum5a896d92008-04-04 00:54:48 +02002565
Christoph Lameter894b8782007-05-10 03:15:16 -07002566 return object;
Christoph Lameter81819f02007-05-06 14:49:36 -07002567}
2568
Ezequiel Garcia2b847c32012-09-08 17:47:58 -03002569static __always_inline void *slab_alloc(struct kmem_cache *s,
2570 gfp_t gfpflags, unsigned long addr)
2571{
2572 return slab_alloc_node(s, gfpflags, NUMA_NO_NODE, addr);
2573}
2574
Christoph Lameter81819f02007-05-06 14:49:36 -07002575void *kmem_cache_alloc(struct kmem_cache *s, gfp_t gfpflags)
2576{
Ezequiel Garcia2b847c32012-09-08 17:47:58 -03002577 void *ret = slab_alloc(s, gfpflags, _RET_IP_);
Eduard - Gabriel Munteanu5b882be2008-08-19 20:43:26 +03002578
Chen Gangd0e0ac92013-07-15 09:05:29 +08002579 trace_kmem_cache_alloc(_RET_IP_, ret, s->object_size,
2580 s->size, gfpflags);
Eduard - Gabriel Munteanu5b882be2008-08-19 20:43:26 +03002581
2582 return ret;
Christoph Lameter81819f02007-05-06 14:49:36 -07002583}
2584EXPORT_SYMBOL(kmem_cache_alloc);
2585
Li Zefan0f24f122009-12-11 15:45:30 +08002586#ifdef CONFIG_TRACING
Richard Kennedy4a923792010-10-21 10:29:19 +01002587void *kmem_cache_alloc_trace(struct kmem_cache *s, gfp_t gfpflags, size_t size)
Eduard - Gabriel Munteanu5b882be2008-08-19 20:43:26 +03002588{
Ezequiel Garcia2b847c32012-09-08 17:47:58 -03002589 void *ret = slab_alloc(s, gfpflags, _RET_IP_);
Richard Kennedy4a923792010-10-21 10:29:19 +01002590 trace_kmalloc(_RET_IP_, ret, size, s->size, gfpflags);
Andrey Ryabinin0316bec2015-02-13 14:39:42 -08002591 kasan_kmalloc(s, ret, size);
Richard Kennedy4a923792010-10-21 10:29:19 +01002592 return ret;
Eduard - Gabriel Munteanu5b882be2008-08-19 20:43:26 +03002593}
Richard Kennedy4a923792010-10-21 10:29:19 +01002594EXPORT_SYMBOL(kmem_cache_alloc_trace);
Eduard - Gabriel Munteanu5b882be2008-08-19 20:43:26 +03002595#endif
2596
Christoph Lameter81819f02007-05-06 14:49:36 -07002597#ifdef CONFIG_NUMA
2598void *kmem_cache_alloc_node(struct kmem_cache *s, gfp_t gfpflags, int node)
2599{
Ezequiel Garcia2b847c32012-09-08 17:47:58 -03002600 void *ret = slab_alloc_node(s, gfpflags, node, _RET_IP_);
Eduard - Gabriel Munteanu5b882be2008-08-19 20:43:26 +03002601
Eduard - Gabriel Munteanuca2b84cb2009-03-23 15:12:24 +02002602 trace_kmem_cache_alloc_node(_RET_IP_, ret,
Christoph Lameter3b0efdf2012-06-13 10:24:57 -05002603 s->object_size, s->size, gfpflags, node);
Eduard - Gabriel Munteanu5b882be2008-08-19 20:43:26 +03002604
2605 return ret;
Christoph Lameter81819f02007-05-06 14:49:36 -07002606}
2607EXPORT_SYMBOL(kmem_cache_alloc_node);
Christoph Lameter81819f02007-05-06 14:49:36 -07002608
Li Zefan0f24f122009-12-11 15:45:30 +08002609#ifdef CONFIG_TRACING
Richard Kennedy4a923792010-10-21 10:29:19 +01002610void *kmem_cache_alloc_node_trace(struct kmem_cache *s,
Eduard - Gabriel Munteanu5b882be2008-08-19 20:43:26 +03002611 gfp_t gfpflags,
Richard Kennedy4a923792010-10-21 10:29:19 +01002612 int node, size_t size)
Eduard - Gabriel Munteanu5b882be2008-08-19 20:43:26 +03002613{
Ezequiel Garcia2b847c32012-09-08 17:47:58 -03002614 void *ret = slab_alloc_node(s, gfpflags, node, _RET_IP_);
Richard Kennedy4a923792010-10-21 10:29:19 +01002615
2616 trace_kmalloc_node(_RET_IP_, ret,
2617 size, s->size, gfpflags, node);
Andrey Ryabinin0316bec2015-02-13 14:39:42 -08002618
2619 kasan_kmalloc(s, ret, size);
Richard Kennedy4a923792010-10-21 10:29:19 +01002620 return ret;
Eduard - Gabriel Munteanu5b882be2008-08-19 20:43:26 +03002621}
Richard Kennedy4a923792010-10-21 10:29:19 +01002622EXPORT_SYMBOL(kmem_cache_alloc_node_trace);
Eduard - Gabriel Munteanu5b882be2008-08-19 20:43:26 +03002623#endif
Namhyung Kim5d1f57e2010-09-29 21:02:15 +09002624#endif
Eduard - Gabriel Munteanu5b882be2008-08-19 20:43:26 +03002625
Christoph Lameter81819f02007-05-06 14:49:36 -07002626/*
Kim Phillips94e4d712015-02-10 14:09:37 -08002627 * Slow path handling. This may still be called frequently since objects
Christoph Lameter894b8782007-05-10 03:15:16 -07002628 * have a longer lifetime than the cpu slabs in most processing loads.
Christoph Lameter81819f02007-05-06 14:49:36 -07002629 *
Christoph Lameter894b8782007-05-10 03:15:16 -07002630 * So we still attempt to reduce cache line usage. Just take the slab
2631 * lock and free the item. If there is no additional partial page
2632 * handling required then we can return immediately.
Christoph Lameter81819f02007-05-06 14:49:36 -07002633 */
Christoph Lameter894b8782007-05-10 03:15:16 -07002634static void __slab_free(struct kmem_cache *s, struct page *page,
Jesper Dangaard Brouer81084652015-11-20 15:57:46 -08002635 void *head, void *tail, int cnt,
2636 unsigned long addr)
2637
Christoph Lameter81819f02007-05-06 14:49:36 -07002638{
2639 void *prior;
Christoph Lameter2cfb7452011-06-01 12:25:52 -05002640 int was_frozen;
Christoph Lameter2cfb7452011-06-01 12:25:52 -05002641 struct page new;
2642 unsigned long counters;
2643 struct kmem_cache_node *n = NULL;
Christoph Lameter61728d12011-06-01 12:25:51 -05002644 unsigned long uninitialized_var(flags);
Christoph Lameter81819f02007-05-06 14:49:36 -07002645
Christoph Lameter8a5ec0b2011-02-25 11:38:54 -06002646 stat(s, FREE_SLOWPATH);
Christoph Lameter81819f02007-05-06 14:49:36 -07002647
Christoph Lameter19c7ff92012-05-30 12:54:46 -05002648 if (kmem_cache_debug(s) &&
Jesper Dangaard Brouer81084652015-11-20 15:57:46 -08002649 !(n = free_debug_processing(s, page, head, tail, cnt,
2650 addr, &flags)))
Christoph Lameter80f08c12011-06-01 12:25:55 -05002651 return;
Christoph Lameter6446faa2008-02-15 23:45:26 -08002652
Christoph Lameter2cfb7452011-06-01 12:25:52 -05002653 do {
Joonsoo Kim837d6782012-08-16 00:02:40 +09002654 if (unlikely(n)) {
2655 spin_unlock_irqrestore(&n->list_lock, flags);
2656 n = NULL;
2657 }
Christoph Lameter2cfb7452011-06-01 12:25:52 -05002658 prior = page->freelist;
2659 counters = page->counters;
Jesper Dangaard Brouer81084652015-11-20 15:57:46 -08002660 set_freepointer(s, tail, prior);
Christoph Lameter2cfb7452011-06-01 12:25:52 -05002661 new.counters = counters;
2662 was_frozen = new.frozen;
Jesper Dangaard Brouer81084652015-11-20 15:57:46 -08002663 new.inuse -= cnt;
Joonsoo Kim837d6782012-08-16 00:02:40 +09002664 if ((!new.inuse || !prior) && !was_frozen) {
Christoph Lameter49e22582011-08-09 16:12:27 -05002665
Peter Zijlstrac65c1872014-01-10 13:23:49 +01002666 if (kmem_cache_has_cpu_partial(s) && !prior) {
Christoph Lameter49e22582011-08-09 16:12:27 -05002667
2668 /*
Chen Gangd0e0ac92013-07-15 09:05:29 +08002669 * Slab was on no list before and will be
2670 * partially empty
2671 * We can defer the list move and instead
2672 * freeze it.
Christoph Lameter49e22582011-08-09 16:12:27 -05002673 */
2674 new.frozen = 1;
2675
Peter Zijlstrac65c1872014-01-10 13:23:49 +01002676 } else { /* Needs to be taken off a list */
Christoph Lameter49e22582011-08-09 16:12:27 -05002677
LQYMGTb455def2014-12-10 15:42:13 -08002678 n = get_node(s, page_to_nid(page));
Christoph Lameter49e22582011-08-09 16:12:27 -05002679 /*
2680 * Speculatively acquire the list_lock.
2681 * If the cmpxchg does not succeed then we may
2682 * drop the list_lock without any processing.
2683 *
2684 * Otherwise the list_lock will synchronize with
2685 * other processors updating the list of slabs.
2686 */
2687 spin_lock_irqsave(&n->list_lock, flags);
2688
2689 }
Christoph Lameter2cfb7452011-06-01 12:25:52 -05002690 }
Christoph Lameter81819f02007-05-06 14:49:36 -07002691
Christoph Lameter2cfb7452011-06-01 12:25:52 -05002692 } while (!cmpxchg_double_slab(s, page,
2693 prior, counters,
Jesper Dangaard Brouer81084652015-11-20 15:57:46 -08002694 head, new.counters,
Christoph Lameter2cfb7452011-06-01 12:25:52 -05002695 "__slab_free"));
Christoph Lameter81819f02007-05-06 14:49:36 -07002696
Christoph Lameter2cfb7452011-06-01 12:25:52 -05002697 if (likely(!n)) {
Christoph Lameter49e22582011-08-09 16:12:27 -05002698
2699 /*
2700 * If we just froze the page then put it onto the
2701 * per cpu partial list.
2702 */
Alex Shi8028dce2012-02-03 23:34:56 +08002703 if (new.frozen && !was_frozen) {
Christoph Lameter49e22582011-08-09 16:12:27 -05002704 put_cpu_partial(s, page, 1);
Alex Shi8028dce2012-02-03 23:34:56 +08002705 stat(s, CPU_PARTIAL_FREE);
2706 }
Christoph Lameter49e22582011-08-09 16:12:27 -05002707 /*
Christoph Lameter2cfb7452011-06-01 12:25:52 -05002708 * The list lock was not taken therefore no list
2709 * activity can be necessary.
2710 */
LQYMGTb455def2014-12-10 15:42:13 -08002711 if (was_frozen)
2712 stat(s, FREE_FROZEN);
2713 return;
2714 }
Christoph Lameter81819f02007-05-06 14:49:36 -07002715
Joonsoo Kim8a5b20a2014-07-02 15:22:35 -07002716 if (unlikely(!new.inuse && n->nr_partial >= s->min_partial))
Joonsoo Kim837d6782012-08-16 00:02:40 +09002717 goto slab_empty;
Christoph Lameter81819f02007-05-06 14:49:36 -07002718
Joonsoo Kim837d6782012-08-16 00:02:40 +09002719 /*
2720 * Objects left in the slab. If it was not on the partial list before
2721 * then add it.
2722 */
Joonsoo Kim345c9052013-06-19 14:05:52 +09002723 if (!kmem_cache_has_cpu_partial(s) && unlikely(!prior)) {
2724 if (kmem_cache_debug(s))
Peter Zijlstrac65c1872014-01-10 13:23:49 +01002725 remove_full(s, n, page);
Joonsoo Kim837d6782012-08-16 00:02:40 +09002726 add_partial(n, page, DEACTIVATE_TO_TAIL);
2727 stat(s, FREE_ADD_PARTIAL);
Christoph Lameter81819f02007-05-06 14:49:36 -07002728 }
Christoph Lameter80f08c12011-06-01 12:25:55 -05002729 spin_unlock_irqrestore(&n->list_lock, flags);
Christoph Lameter81819f02007-05-06 14:49:36 -07002730 return;
2731
2732slab_empty:
Christoph Lametera973e9d2008-03-01 13:40:44 -08002733 if (prior) {
Christoph Lameter81819f02007-05-06 14:49:36 -07002734 /*
Christoph Lameter6fbabb22011-08-08 11:16:56 -05002735 * Slab on the partial list.
Christoph Lameter81819f02007-05-06 14:49:36 -07002736 */
Christoph Lameter5cc6eee2011-06-01 12:25:50 -05002737 remove_partial(n, page);
Christoph Lameter84e554e62009-12-18 16:26:23 -06002738 stat(s, FREE_REMOVE_PARTIAL);
Peter Zijlstrac65c1872014-01-10 13:23:49 +01002739 } else {
Christoph Lameter6fbabb22011-08-08 11:16:56 -05002740 /* Slab must be on the full list */
Peter Zijlstrac65c1872014-01-10 13:23:49 +01002741 remove_full(s, n, page);
2742 }
Christoph Lameter2cfb7452011-06-01 12:25:52 -05002743
Christoph Lameter80f08c12011-06-01 12:25:55 -05002744 spin_unlock_irqrestore(&n->list_lock, flags);
Christoph Lameter84e554e62009-12-18 16:26:23 -06002745 stat(s, FREE_SLAB);
Christoph Lameter81819f02007-05-06 14:49:36 -07002746 discard_slab(s, page);
Christoph Lameter81819f02007-05-06 14:49:36 -07002747}
2748
Christoph Lameter894b8782007-05-10 03:15:16 -07002749/*
2750 * Fastpath with forced inlining to produce a kfree and kmem_cache_free that
2751 * can perform fastpath freeing without additional function calls.
2752 *
2753 * The fastpath is only possible if we are freeing to the current cpu slab
2754 * of this processor. This typically the case if we have just allocated
2755 * the item before.
2756 *
2757 * If fastpath is not possible then fall back to __slab_free where we deal
2758 * with all sorts of special processing.
Jesper Dangaard Brouer81084652015-11-20 15:57:46 -08002759 *
2760 * Bulk free of a freelist with several objects (all pointing to the
2761 * same page) possible by specifying head and tail ptr, plus objects
2762 * count (cnt). Bulk free indicated by tail pointer being set.
Christoph Lameter894b8782007-05-10 03:15:16 -07002763 */
Jesper Dangaard Brouer81084652015-11-20 15:57:46 -08002764static __always_inline void slab_free(struct kmem_cache *s, struct page *page,
2765 void *head, void *tail, int cnt,
2766 unsigned long addr)
Christoph Lameter894b8782007-05-10 03:15:16 -07002767{
Jesper Dangaard Brouer81084652015-11-20 15:57:46 -08002768 void *tail_obj = tail ? : head;
Christoph Lameterdfb4f092007-10-16 01:26:05 -07002769 struct kmem_cache_cpu *c;
Christoph Lameter8a5ec0b2011-02-25 11:38:54 -06002770 unsigned long tid;
Christoph Lameter1f842602008-01-07 23:20:30 -08002771
Jesper Dangaard Brouer81084652015-11-20 15:57:46 -08002772 slab_free_freelist_hook(s, head, tail);
Christoph Lameterc016b0b2010-08-20 12:37:16 -05002773
Christoph Lametera24c5a02011-03-15 12:45:21 -05002774redo:
Christoph Lameter8a5ec0b2011-02-25 11:38:54 -06002775 /*
2776 * Determine the currently cpus per cpu slab.
2777 * The cpu may change afterward. However that does not matter since
2778 * data is retrieved via this pointer. If we are on the same cpu
Jesper Dangaard Brouer2ae44002015-09-04 15:45:31 -07002779 * during the cmpxchg then the free will succeed.
Christoph Lameter8a5ec0b2011-02-25 11:38:54 -06002780 */
Joonsoo Kim9aabf812015-02-10 14:09:32 -08002781 do {
2782 tid = this_cpu_read(s->cpu_slab->tid);
2783 c = raw_cpu_ptr(s->cpu_slab);
Mark Rutland859b7a02015-03-25 15:55:23 -07002784 } while (IS_ENABLED(CONFIG_PREEMPT) &&
2785 unlikely(tid != READ_ONCE(c->tid)));
Christoph Lameterc016b0b2010-08-20 12:37:16 -05002786
Joonsoo Kim9aabf812015-02-10 14:09:32 -08002787 /* Same with comment on barrier() in slab_alloc_node() */
2788 barrier();
Christoph Lameterc016b0b2010-08-20 12:37:16 -05002789
Christoph Lameter442b06b2011-05-17 16:29:31 -05002790 if (likely(page == c->page)) {
Jesper Dangaard Brouer81084652015-11-20 15:57:46 -08002791 set_freepointer(s, tail_obj, c->freelist);
Christoph Lameter8a5ec0b2011-02-25 11:38:54 -06002792
Christoph Lameter933393f2011-12-22 11:58:51 -06002793 if (unlikely(!this_cpu_cmpxchg_double(
Christoph Lameter8a5ec0b2011-02-25 11:38:54 -06002794 s->cpu_slab->freelist, s->cpu_slab->tid,
2795 c->freelist, tid,
Jesper Dangaard Brouer81084652015-11-20 15:57:46 -08002796 head, next_tid(tid)))) {
Christoph Lameter8a5ec0b2011-02-25 11:38:54 -06002797
2798 note_cmpxchg_failure("slab_free", s, tid);
2799 goto redo;
2800 }
Christoph Lameter84e554e62009-12-18 16:26:23 -06002801 stat(s, FREE_FASTPATH);
Christoph Lameter894b8782007-05-10 03:15:16 -07002802 } else
Jesper Dangaard Brouer81084652015-11-20 15:57:46 -08002803 __slab_free(s, page, head, tail_obj, cnt, addr);
Christoph Lameter894b8782007-05-10 03:15:16 -07002804
Christoph Lameter894b8782007-05-10 03:15:16 -07002805}
2806
Christoph Lameter81819f02007-05-06 14:49:36 -07002807void kmem_cache_free(struct kmem_cache *s, void *x)
2808{
Glauber Costab9ce5ef2012-12-18 14:22:46 -08002809 s = cache_from_obj(s, x);
2810 if (!s)
Christoph Lameter79576102012-09-04 23:06:14 +00002811 return;
Jesper Dangaard Brouer81084652015-11-20 15:57:46 -08002812 slab_free(s, virt_to_head_page(x), x, NULL, 1, _RET_IP_);
Eduard - Gabriel Munteanuca2b84cb2009-03-23 15:12:24 +02002813 trace_kmem_cache_free(_RET_IP_, x);
Christoph Lameter81819f02007-05-06 14:49:36 -07002814}
2815EXPORT_SYMBOL(kmem_cache_free);
2816
Jesper Dangaard Brouerd0ecd892015-11-20 15:57:49 -08002817struct detached_freelist {
2818 struct page *page;
2819 void *tail;
2820 void *freelist;
2821 int cnt;
2822};
2823
2824/*
2825 * This function progressively scans the array with free objects (with
2826 * a limited look ahead) and extract objects belonging to the same
2827 * page. It builds a detached freelist directly within the given
2828 * page/objects. This can happen without any need for
2829 * synchronization, because the objects are owned by running process.
2830 * The freelist is build up as a single linked list in the objects.
2831 * The idea is, that this detached freelist can then be bulk
2832 * transferred to the real freelist(s), but only requiring a single
2833 * synchronization primitive. Look ahead in the array is limited due
2834 * to performance reasons.
2835 */
2836static int build_detached_freelist(struct kmem_cache *s, size_t size,
2837 void **p, struct detached_freelist *df)
2838{
2839 size_t first_skipped_index = 0;
2840 int lookahead = 3;
2841 void *object;
2842
2843 /* Always re-init detached_freelist */
2844 df->page = NULL;
2845
2846 do {
2847 object = p[--size];
2848 } while (!object && size);
2849
2850 if (!object)
2851 return 0;
2852
2853 /* Start new detached freelist */
2854 set_freepointer(s, object, NULL);
2855 df->page = virt_to_head_page(object);
2856 df->tail = object;
2857 df->freelist = object;
2858 p[size] = NULL; /* mark object processed */
2859 df->cnt = 1;
2860
2861 while (size) {
2862 object = p[--size];
2863 if (!object)
2864 continue; /* Skip processed objects */
2865
2866 /* df->page is always set at this point */
2867 if (df->page == virt_to_head_page(object)) {
2868 /* Opportunity build freelist */
2869 set_freepointer(s, object, df->freelist);
2870 df->freelist = object;
2871 df->cnt++;
2872 p[size] = NULL; /* mark object processed */
2873
2874 continue;
2875 }
2876
2877 /* Limit look ahead search */
2878 if (!--lookahead)
2879 break;
2880
2881 if (!first_skipped_index)
2882 first_skipped_index = size + 1;
2883 }
2884
2885 return first_skipped_index;
2886}
2887
2888
Jesper Dangaard Brouer994eb762015-09-04 15:45:37 -07002889/* Note that interrupts must be enabled when calling this function. */
Jesper Dangaard Brouer03374512015-11-20 15:57:55 -08002890void kmem_cache_free_bulk(struct kmem_cache *orig_s, size_t size, void **p)
Christoph Lameter484748f2015-09-04 15:45:34 -07002891{
Jesper Dangaard Brouerd0ecd892015-11-20 15:57:49 -08002892 if (WARN_ON(!size))
2893 return;
Jesper Dangaard Brouerfbd02632015-09-04 15:45:43 -07002894
Jesper Dangaard Brouerd0ecd892015-11-20 15:57:49 -08002895 do {
2896 struct detached_freelist df;
Jesper Dangaard Brouer03374512015-11-20 15:57:55 -08002897 struct kmem_cache *s;
2898
2899 /* Support for memcg */
2900 s = cache_from_obj(orig_s, p[size - 1]);
Jesper Dangaard Brouerfbd02632015-09-04 15:45:43 -07002901
Jesper Dangaard Brouerd0ecd892015-11-20 15:57:49 -08002902 size = build_detached_freelist(s, size, p, &df);
2903 if (unlikely(!df.page))
2904 continue;
Jesper Dangaard Brouerfbd02632015-09-04 15:45:43 -07002905
Jesper Dangaard Brouerd0ecd892015-11-20 15:57:49 -08002906 slab_free(s, df.page, df.freelist, df.tail, df.cnt, _RET_IP_);
2907 } while (likely(size));
Christoph Lameter484748f2015-09-04 15:45:34 -07002908}
2909EXPORT_SYMBOL(kmem_cache_free_bulk);
2910
Jesper Dangaard Brouer994eb762015-09-04 15:45:37 -07002911/* Note that interrupts must be enabled when calling this function. */
Jesper Dangaard Brouer865762a2015-11-20 15:57:58 -08002912int kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags, size_t size,
2913 void **p)
Christoph Lameter484748f2015-09-04 15:45:34 -07002914{
Jesper Dangaard Brouer994eb762015-09-04 15:45:37 -07002915 struct kmem_cache_cpu *c;
2916 int i;
2917
Jesper Dangaard Brouer03ec0ed2015-11-20 15:57:52 -08002918 /* memcg and kmem_cache debug support */
2919 s = slab_pre_alloc_hook(s, flags);
2920 if (unlikely(!s))
2921 return false;
Jesper Dangaard Brouer994eb762015-09-04 15:45:37 -07002922 /*
2923 * Drain objects in the per cpu slab, while disabling local
2924 * IRQs, which protects against PREEMPT and interrupts
2925 * handlers invoking normal fastpath.
2926 */
2927 local_irq_disable();
2928 c = this_cpu_ptr(s->cpu_slab);
2929
2930 for (i = 0; i < size; i++) {
2931 void *object = c->freelist;
2932
Jesper Dangaard Brouerebe909e2015-09-04 15:45:40 -07002933 if (unlikely(!object)) {
Jesper Dangaard Brouerebe909e2015-09-04 15:45:40 -07002934 /*
2935 * Invoking slow path likely have side-effect
2936 * of re-populating per CPU c->freelist
2937 */
Christoph Lameter87098372015-11-20 15:57:38 -08002938 p[i] = ___slab_alloc(s, flags, NUMA_NO_NODE,
Jesper Dangaard Brouerebe909e2015-09-04 15:45:40 -07002939 _RET_IP_, c);
Christoph Lameter87098372015-11-20 15:57:38 -08002940 if (unlikely(!p[i]))
2941 goto error;
2942
Jesper Dangaard Brouerebe909e2015-09-04 15:45:40 -07002943 c = this_cpu_ptr(s->cpu_slab);
2944 continue; /* goto for-loop */
2945 }
Jesper Dangaard Brouer994eb762015-09-04 15:45:37 -07002946 c->freelist = get_freepointer(s, object);
2947 p[i] = object;
2948 }
2949 c->tid = next_tid(c->tid);
2950 local_irq_enable();
2951
2952 /* Clear memory outside IRQ disabled fastpath loop */
2953 if (unlikely(flags & __GFP_ZERO)) {
2954 int j;
2955
2956 for (j = 0; j < i; j++)
2957 memset(p[j], 0, s->object_size);
2958 }
2959
Jesper Dangaard Brouer03ec0ed2015-11-20 15:57:52 -08002960 /* memcg and kmem_cache debug support */
2961 slab_post_alloc_hook(s, flags, size, p);
Jesper Dangaard Brouer865762a2015-11-20 15:57:58 -08002962 return i;
Christoph Lameter87098372015-11-20 15:57:38 -08002963error:
Christoph Lameter87098372015-11-20 15:57:38 -08002964 local_irq_enable();
Jesper Dangaard Brouer03ec0ed2015-11-20 15:57:52 -08002965 slab_post_alloc_hook(s, flags, i, p);
2966 __kmem_cache_free_bulk(s, i, p);
Jesper Dangaard Brouer865762a2015-11-20 15:57:58 -08002967 return 0;
Christoph Lameter484748f2015-09-04 15:45:34 -07002968}
2969EXPORT_SYMBOL(kmem_cache_alloc_bulk);
2970
2971
Christoph Lameter81819f02007-05-06 14:49:36 -07002972/*
Christoph Lameter672bba32007-05-09 02:32:39 -07002973 * Object placement in a slab is made very easy because we always start at
2974 * offset 0. If we tune the size of the object to the alignment then we can
2975 * get the required alignment by putting one properly sized object after
2976 * another.
Christoph Lameter81819f02007-05-06 14:49:36 -07002977 *
2978 * Notice that the allocation order determines the sizes of the per cpu
2979 * caches. Each processor has always one slab available for allocations.
2980 * Increasing the allocation order reduces the number of times that slabs
Christoph Lameter672bba32007-05-09 02:32:39 -07002981 * must be moved on and off the partial lists and is therefore a factor in
Christoph Lameter81819f02007-05-06 14:49:36 -07002982 * locking overhead.
Christoph Lameter81819f02007-05-06 14:49:36 -07002983 */
2984
2985/*
2986 * Mininum / Maximum order of slab pages. This influences locking overhead
2987 * and slab fragmentation. A higher order reduces the number of partial slabs
2988 * and increases the number of allocations possible without having to
2989 * take the list_lock.
2990 */
2991static int slub_min_order;
Christoph Lameter114e9e82008-04-14 19:11:41 +03002992static int slub_max_order = PAGE_ALLOC_COSTLY_ORDER;
Christoph Lameter9b2cd502008-04-14 19:11:41 +03002993static int slub_min_objects;
Christoph Lameter81819f02007-05-06 14:49:36 -07002994
2995/*
Christoph Lameter81819f02007-05-06 14:49:36 -07002996 * Calculate the order of allocation given an slab object size.
2997 *
Christoph Lameter672bba32007-05-09 02:32:39 -07002998 * The order of allocation has significant impact on performance and other
2999 * system components. Generally order 0 allocations should be preferred since
3000 * order 0 does not cause fragmentation in the page allocator. Larger objects
3001 * be problematic to put into order 0 slabs because there may be too much
Christoph Lameterc124f5b2008-04-14 19:13:29 +03003002 * unused space left. We go to a higher order if more than 1/16th of the slab
Christoph Lameter672bba32007-05-09 02:32:39 -07003003 * would be wasted.
Christoph Lameter81819f02007-05-06 14:49:36 -07003004 *
Christoph Lameter672bba32007-05-09 02:32:39 -07003005 * In order to reach satisfactory performance we must ensure that a minimum
3006 * number of objects is in one slab. Otherwise we may generate too much
3007 * activity on the partial lists which requires taking the list_lock. This is
3008 * less a concern for large slabs though which are rarely used.
Christoph Lameter81819f02007-05-06 14:49:36 -07003009 *
Christoph Lameter672bba32007-05-09 02:32:39 -07003010 * slub_max_order specifies the order where we begin to stop considering the
3011 * number of objects in a slab as critical. If we reach slub_max_order then
3012 * we try to keep the page order as low as possible. So we accept more waste
3013 * of space in favor of a small page order.
3014 *
3015 * Higher order allocations also allow the placement of more objects in a
3016 * slab and thereby reduce object handling overhead. If the user has
3017 * requested a higher mininum order then we start with that one instead of
3018 * the smallest order which will fit the object.
Christoph Lameter81819f02007-05-06 14:49:36 -07003019 */
Christoph Lameter5e6d4442007-05-09 02:32:46 -07003020static inline int slab_order(int size, int min_objects,
Lai Jiangshanab9a0f12011-03-10 15:21:48 +08003021 int max_order, int fract_leftover, int reserved)
Christoph Lameter81819f02007-05-06 14:49:36 -07003022{
3023 int order;
3024 int rem;
Christoph Lameter6300ea72007-07-17 04:03:20 -07003025 int min_order = slub_min_order;
Christoph Lameter81819f02007-05-06 14:49:36 -07003026
Lai Jiangshanab9a0f12011-03-10 15:21:48 +08003027 if (order_objects(min_order, size, reserved) > MAX_OBJS_PER_PAGE)
Cyrill Gorcunov210b5c02008-10-22 23:00:38 +04003028 return get_order(size * MAX_OBJS_PER_PAGE) - 1;
Christoph Lameter39b26462008-04-14 19:11:30 +03003029
Wei Yang9f835702015-11-05 18:45:51 -08003030 for (order = max(min_order, get_order(min_objects * size + reserved));
Christoph Lameter5e6d4442007-05-09 02:32:46 -07003031 order <= max_order; order++) {
3032
Christoph Lameter81819f02007-05-06 14:49:36 -07003033 unsigned long slab_size = PAGE_SIZE << order;
3034
Lai Jiangshanab9a0f12011-03-10 15:21:48 +08003035 rem = (slab_size - reserved) % size;
Christoph Lameter81819f02007-05-06 14:49:36 -07003036
Christoph Lameter5e6d4442007-05-09 02:32:46 -07003037 if (rem <= slab_size / fract_leftover)
Christoph Lameter81819f02007-05-06 14:49:36 -07003038 break;
Christoph Lameter81819f02007-05-06 14:49:36 -07003039 }
Christoph Lameter672bba32007-05-09 02:32:39 -07003040
Christoph Lameter81819f02007-05-06 14:49:36 -07003041 return order;
3042}
3043
Lai Jiangshanab9a0f12011-03-10 15:21:48 +08003044static inline int calculate_order(int size, int reserved)
Christoph Lameter5e6d4442007-05-09 02:32:46 -07003045{
3046 int order;
3047 int min_objects;
3048 int fraction;
Zhang Yanmine8120ff2009-02-12 18:00:17 +02003049 int max_objects;
Christoph Lameter5e6d4442007-05-09 02:32:46 -07003050
3051 /*
3052 * Attempt to find best configuration for a slab. This
3053 * works by first attempting to generate a layout with
3054 * the best configuration and backing off gradually.
3055 *
Wei Yang422ff4d2015-11-05 18:45:46 -08003056 * First we increase the acceptable waste in a slab. Then
Christoph Lameter5e6d4442007-05-09 02:32:46 -07003057 * we reduce the minimum objects required in a slab.
3058 */
3059 min_objects = slub_min_objects;
Christoph Lameter9b2cd502008-04-14 19:11:41 +03003060 if (!min_objects)
3061 min_objects = 4 * (fls(nr_cpu_ids) + 1);
Lai Jiangshanab9a0f12011-03-10 15:21:48 +08003062 max_objects = order_objects(slub_max_order, size, reserved);
Zhang Yanmine8120ff2009-02-12 18:00:17 +02003063 min_objects = min(min_objects, max_objects);
3064
Christoph Lameter5e6d4442007-05-09 02:32:46 -07003065 while (min_objects > 1) {
Christoph Lameterc124f5b2008-04-14 19:13:29 +03003066 fraction = 16;
Christoph Lameter5e6d4442007-05-09 02:32:46 -07003067 while (fraction >= 4) {
3068 order = slab_order(size, min_objects,
Lai Jiangshanab9a0f12011-03-10 15:21:48 +08003069 slub_max_order, fraction, reserved);
Christoph Lameter5e6d4442007-05-09 02:32:46 -07003070 if (order <= slub_max_order)
3071 return order;
3072 fraction /= 2;
3073 }
Amerigo Wang5086c389c2009-08-19 21:44:13 +03003074 min_objects--;
Christoph Lameter5e6d4442007-05-09 02:32:46 -07003075 }
3076
3077 /*
3078 * We were unable to place multiple objects in a slab. Now
3079 * lets see if we can place a single object there.
3080 */
Lai Jiangshanab9a0f12011-03-10 15:21:48 +08003081 order = slab_order(size, 1, slub_max_order, 1, reserved);
Christoph Lameter5e6d4442007-05-09 02:32:46 -07003082 if (order <= slub_max_order)
3083 return order;
3084
3085 /*
3086 * Doh this slab cannot be placed using slub_max_order.
3087 */
Lai Jiangshanab9a0f12011-03-10 15:21:48 +08003088 order = slab_order(size, 1, MAX_ORDER, 1, reserved);
David Rientjes818cf592009-04-23 09:58:22 +03003089 if (order < MAX_ORDER)
Christoph Lameter5e6d4442007-05-09 02:32:46 -07003090 return order;
3091 return -ENOSYS;
3092}
3093
Pekka Enberg5595cff2008-08-05 09:28:47 +03003094static void
Joonsoo Kim40534972012-05-11 00:50:47 +09003095init_kmem_cache_node(struct kmem_cache_node *n)
Christoph Lameter81819f02007-05-06 14:49:36 -07003096{
3097 n->nr_partial = 0;
Christoph Lameter81819f02007-05-06 14:49:36 -07003098 spin_lock_init(&n->list_lock);
3099 INIT_LIST_HEAD(&n->partial);
Christoph Lameter8ab13722007-07-17 04:03:32 -07003100#ifdef CONFIG_SLUB_DEBUG
Christoph Lameter0f389ec2008-04-14 18:53:02 +03003101 atomic_long_set(&n->nr_slabs, 0);
Salman Qazi02b71b72008-09-11 12:25:41 -07003102 atomic_long_set(&n->total_objects, 0);
Christoph Lameter643b1132007-05-06 14:49:42 -07003103 INIT_LIST_HEAD(&n->full);
Christoph Lameter8ab13722007-07-17 04:03:32 -07003104#endif
Christoph Lameter81819f02007-05-06 14:49:36 -07003105}
3106
Christoph Lameter55136592010-08-20 12:37:13 -05003107static inline int alloc_kmem_cache_cpus(struct kmem_cache *s)
Christoph Lameter4c93c3552007-10-16 01:26:08 -07003108{
Christoph Lameter6c182dc2010-08-20 12:37:14 -05003109 BUILD_BUG_ON(PERCPU_DYNAMIC_EARLY_SIZE <
Christoph Lameter95a05b42013-01-10 19:14:19 +00003110 KMALLOC_SHIFT_HIGH * sizeof(struct kmem_cache_cpu));
Christoph Lameter9dfc6e62009-12-18 16:26:20 -06003111
Christoph Lameter8a5ec0b2011-02-25 11:38:54 -06003112 /*
Chris Metcalfd4d84fe2011-06-02 10:19:41 -04003113 * Must align to double word boundary for the double cmpxchg
3114 * instructions to work; see __pcpu_double_call_return_bool().
Christoph Lameter8a5ec0b2011-02-25 11:38:54 -06003115 */
Chris Metcalfd4d84fe2011-06-02 10:19:41 -04003116 s->cpu_slab = __alloc_percpu(sizeof(struct kmem_cache_cpu),
3117 2 * sizeof(void *));
Christoph Lameter9dfc6e62009-12-18 16:26:20 -06003118
Christoph Lameter8a5ec0b2011-02-25 11:38:54 -06003119 if (!s->cpu_slab)
3120 return 0;
3121
3122 init_kmem_cache_cpus(s);
3123
3124 return 1;
Christoph Lameter4c93c3552007-10-16 01:26:08 -07003125}
Christoph Lameter4c93c3552007-10-16 01:26:08 -07003126
Christoph Lameter51df1142010-08-20 12:37:15 -05003127static struct kmem_cache *kmem_cache_node;
3128
Christoph Lameter81819f02007-05-06 14:49:36 -07003129/*
3130 * No kmalloc_node yet so do it by hand. We know that this is the first
3131 * slab on the node for this slabcache. There are no concurrent accesses
3132 * possible.
3133 *
Zhi Yong Wu721ae222013-11-08 20:47:37 +08003134 * Note that this function only works on the kmem_cache_node
3135 * when allocating for the kmem_cache_node. This is used for bootstrapping
Christoph Lameter4c93c3552007-10-16 01:26:08 -07003136 * memory on a fresh node that has no slab structures yet.
Christoph Lameter81819f02007-05-06 14:49:36 -07003137 */
Christoph Lameter55136592010-08-20 12:37:13 -05003138static void early_kmem_cache_node_alloc(int node)
Christoph Lameter81819f02007-05-06 14:49:36 -07003139{
3140 struct page *page;
3141 struct kmem_cache_node *n;
3142
Christoph Lameter51df1142010-08-20 12:37:15 -05003143 BUG_ON(kmem_cache_node->size < sizeof(struct kmem_cache_node));
Christoph Lameter81819f02007-05-06 14:49:36 -07003144
Christoph Lameter51df1142010-08-20 12:37:15 -05003145 page = new_slab(kmem_cache_node, GFP_NOWAIT, node);
Christoph Lameter81819f02007-05-06 14:49:36 -07003146
3147 BUG_ON(!page);
Christoph Lametera2f92ee2007-08-22 14:01:57 -07003148 if (page_to_nid(page) != node) {
Fabian Frederickf9f58282014-06-04 16:06:34 -07003149 pr_err("SLUB: Unable to allocate memory from node %d\n", node);
3150 pr_err("SLUB: Allocating a useless per node structure in order to be able to continue\n");
Christoph Lametera2f92ee2007-08-22 14:01:57 -07003151 }
3152
Christoph Lameter81819f02007-05-06 14:49:36 -07003153 n = page->freelist;
3154 BUG_ON(!n);
Christoph Lameter51df1142010-08-20 12:37:15 -05003155 page->freelist = get_freepointer(kmem_cache_node, n);
Christoph Lametere6e82ea2011-08-09 16:12:24 -05003156 page->inuse = 1;
Christoph Lameter8cb0a502011-06-01 12:25:46 -05003157 page->frozen = 0;
Christoph Lameter51df1142010-08-20 12:37:15 -05003158 kmem_cache_node->node[node] = n;
Christoph Lameter8ab13722007-07-17 04:03:32 -07003159#ifdef CONFIG_SLUB_DEBUG
Christoph Lameterf7cb1932010-09-29 07:15:01 -05003160 init_object(kmem_cache_node, n, SLUB_RED_ACTIVE);
Christoph Lameter51df1142010-08-20 12:37:15 -05003161 init_tracking(kmem_cache_node, n);
Christoph Lameter8ab13722007-07-17 04:03:32 -07003162#endif
Andrey Ryabinin0316bec2015-02-13 14:39:42 -08003163 kasan_kmalloc(kmem_cache_node, n, sizeof(struct kmem_cache_node));
Joonsoo Kim40534972012-05-11 00:50:47 +09003164 init_kmem_cache_node(n);
Christoph Lameter51df1142010-08-20 12:37:15 -05003165 inc_slabs_node(kmem_cache_node, node, page->objects);
Christoph Lameter6446faa2008-02-15 23:45:26 -08003166
Dave Hansen67b6c902014-01-24 07:20:23 -08003167 /*
Steven Rostedt1e4dd942014-02-10 14:25:46 -08003168 * No locks need to be taken here as it has just been
3169 * initialized and there is no concurrent access.
Dave Hansen67b6c902014-01-24 07:20:23 -08003170 */
Steven Rostedt1e4dd942014-02-10 14:25:46 -08003171 __add_partial(n, page, DEACTIVATE_TO_HEAD);
Christoph Lameter81819f02007-05-06 14:49:36 -07003172}
3173
3174static void free_kmem_cache_nodes(struct kmem_cache *s)
3175{
3176 int node;
Christoph Lameterfa45dc22014-08-06 16:04:09 -07003177 struct kmem_cache_node *n;
Christoph Lameter81819f02007-05-06 14:49:36 -07003178
Christoph Lameterfa45dc22014-08-06 16:04:09 -07003179 for_each_kmem_cache_node(s, node, n) {
3180 kmem_cache_free(kmem_cache_node, n);
Christoph Lameter81819f02007-05-06 14:49:36 -07003181 s->node[node] = NULL;
3182 }
3183}
3184
Christoph Lameter55136592010-08-20 12:37:13 -05003185static int init_kmem_cache_nodes(struct kmem_cache *s)
Christoph Lameter81819f02007-05-06 14:49:36 -07003186{
3187 int node;
Christoph Lameter81819f02007-05-06 14:49:36 -07003188
Christoph Lameterf64dc582007-10-16 01:25:33 -07003189 for_each_node_state(node, N_NORMAL_MEMORY) {
Christoph Lameter81819f02007-05-06 14:49:36 -07003190 struct kmem_cache_node *n;
3191
Alexander Duyck73367bd2010-05-21 14:41:35 -07003192 if (slab_state == DOWN) {
Christoph Lameter55136592010-08-20 12:37:13 -05003193 early_kmem_cache_node_alloc(node);
Alexander Duyck73367bd2010-05-21 14:41:35 -07003194 continue;
Christoph Lameter81819f02007-05-06 14:49:36 -07003195 }
Christoph Lameter51df1142010-08-20 12:37:15 -05003196 n = kmem_cache_alloc_node(kmem_cache_node,
Christoph Lameter55136592010-08-20 12:37:13 -05003197 GFP_KERNEL, node);
Alexander Duyck73367bd2010-05-21 14:41:35 -07003198
3199 if (!n) {
3200 free_kmem_cache_nodes(s);
3201 return 0;
3202 }
3203
Christoph Lameter81819f02007-05-06 14:49:36 -07003204 s->node[node] = n;
Joonsoo Kim40534972012-05-11 00:50:47 +09003205 init_kmem_cache_node(n);
Christoph Lameter81819f02007-05-06 14:49:36 -07003206 }
3207 return 1;
3208}
Christoph Lameter81819f02007-05-06 14:49:36 -07003209
David Rientjesc0bdb232009-02-25 09:16:35 +02003210static void set_min_partial(struct kmem_cache *s, unsigned long min)
David Rientjes3b89d7d2009-02-22 17:40:07 -08003211{
3212 if (min < MIN_PARTIAL)
3213 min = MIN_PARTIAL;
3214 else if (min > MAX_PARTIAL)
3215 min = MAX_PARTIAL;
3216 s->min_partial = min;
3217}
3218
Christoph Lameter81819f02007-05-06 14:49:36 -07003219/*
3220 * calculate_sizes() determines the order and the distribution of data within
3221 * a slab object.
3222 */
Christoph Lameter06b285d2008-04-14 19:11:41 +03003223static int calculate_sizes(struct kmem_cache *s, int forced_order)
Christoph Lameter81819f02007-05-06 14:49:36 -07003224{
3225 unsigned long flags = s->flags;
Christoph Lameter3b0efdf2012-06-13 10:24:57 -05003226 unsigned long size = s->object_size;
Christoph Lameter834f3d12008-04-14 19:11:31 +03003227 int order;
Christoph Lameter81819f02007-05-06 14:49:36 -07003228
3229 /*
Christoph Lameterd8b42bf2008-02-15 23:45:25 -08003230 * Round up object size to the next word boundary. We can only
3231 * place the free pointer at word boundaries and this determines
3232 * the possible location of the free pointer.
3233 */
3234 size = ALIGN(size, sizeof(void *));
3235
3236#ifdef CONFIG_SLUB_DEBUG
3237 /*
Christoph Lameter81819f02007-05-06 14:49:36 -07003238 * Determine if we can poison the object itself. If the user of
3239 * the slab may touch the object after free or before allocation
3240 * then we should never poison the object itself.
3241 */
3242 if ((flags & SLAB_POISON) && !(flags & SLAB_DESTROY_BY_RCU) &&
Christoph Lameterc59def92007-05-16 22:10:50 -07003243 !s->ctor)
Christoph Lameter81819f02007-05-06 14:49:36 -07003244 s->flags |= __OBJECT_POISON;
3245 else
3246 s->flags &= ~__OBJECT_POISON;
3247
Christoph Lameter81819f02007-05-06 14:49:36 -07003248
3249 /*
Christoph Lameter672bba32007-05-09 02:32:39 -07003250 * If we are Redzoning then check if there is some space between the
Christoph Lameter81819f02007-05-06 14:49:36 -07003251 * end of the object and the free pointer. If not then add an
Christoph Lameter672bba32007-05-09 02:32:39 -07003252 * additional word to have some bytes to store Redzone information.
Christoph Lameter81819f02007-05-06 14:49:36 -07003253 */
Christoph Lameter3b0efdf2012-06-13 10:24:57 -05003254 if ((flags & SLAB_RED_ZONE) && size == s->object_size)
Christoph Lameter81819f02007-05-06 14:49:36 -07003255 size += sizeof(void *);
Christoph Lameter41ecc552007-05-09 02:32:44 -07003256#endif
Christoph Lameter81819f02007-05-06 14:49:36 -07003257
3258 /*
Christoph Lameter672bba32007-05-09 02:32:39 -07003259 * With that we have determined the number of bytes in actual use
3260 * by the object. This is the potential offset to the free pointer.
Christoph Lameter81819f02007-05-06 14:49:36 -07003261 */
3262 s->inuse = size;
3263
3264 if (((flags & (SLAB_DESTROY_BY_RCU | SLAB_POISON)) ||
Christoph Lameterc59def92007-05-16 22:10:50 -07003265 s->ctor)) {
Christoph Lameter81819f02007-05-06 14:49:36 -07003266 /*
3267 * Relocate free pointer after the object if it is not
3268 * permitted to overwrite the first word of the object on
3269 * kmem_cache_free.
3270 *
3271 * This is the case if we do RCU, have a constructor or
3272 * destructor or are poisoning the objects.
3273 */
3274 s->offset = size;
3275 size += sizeof(void *);
3276 }
3277
Christoph Lameterc12b3c62007-05-23 13:57:31 -07003278#ifdef CONFIG_SLUB_DEBUG
Christoph Lameter81819f02007-05-06 14:49:36 -07003279 if (flags & SLAB_STORE_USER)
3280 /*
3281 * Need to store information about allocs and frees after
3282 * the object.
3283 */
3284 size += 2 * sizeof(struct track);
3285
Christoph Lameterbe7b3fb2007-05-09 02:32:36 -07003286 if (flags & SLAB_RED_ZONE)
Christoph Lameter81819f02007-05-06 14:49:36 -07003287 /*
3288 * Add some empty padding so that we can catch
3289 * overwrites from earlier objects rather than let
3290 * tracking information or the free pointer be
Frederik Schwarzer0211a9c2008-12-29 22:14:56 +01003291 * corrupted if a user writes before the start
Christoph Lameter81819f02007-05-06 14:49:36 -07003292 * of the object.
3293 */
3294 size += sizeof(void *);
Christoph Lameter41ecc552007-05-09 02:32:44 -07003295#endif
Christoph Lameter672bba32007-05-09 02:32:39 -07003296
Christoph Lameter81819f02007-05-06 14:49:36 -07003297 /*
Christoph Lameter81819f02007-05-06 14:49:36 -07003298 * SLUB stores one object immediately after another beginning from
3299 * offset 0. In order to align the objects we have to simply size
3300 * each object to conform to the alignment.
3301 */
Christoph Lameter45906852012-11-28 16:23:16 +00003302 size = ALIGN(size, s->align);
Christoph Lameter81819f02007-05-06 14:49:36 -07003303 s->size = size;
Christoph Lameter06b285d2008-04-14 19:11:41 +03003304 if (forced_order >= 0)
3305 order = forced_order;
3306 else
Lai Jiangshanab9a0f12011-03-10 15:21:48 +08003307 order = calculate_order(size, s->reserved);
Christoph Lameter81819f02007-05-06 14:49:36 -07003308
Christoph Lameter834f3d12008-04-14 19:11:31 +03003309 if (order < 0)
Christoph Lameter81819f02007-05-06 14:49:36 -07003310 return 0;
3311
Christoph Lameterb7a49f02008-02-14 14:21:32 -08003312 s->allocflags = 0;
Christoph Lameter834f3d12008-04-14 19:11:31 +03003313 if (order)
Christoph Lameterb7a49f02008-02-14 14:21:32 -08003314 s->allocflags |= __GFP_COMP;
3315
3316 if (s->flags & SLAB_CACHE_DMA)
Christoph Lameter2c59dd62013-01-10 19:14:19 +00003317 s->allocflags |= GFP_DMA;
Christoph Lameterb7a49f02008-02-14 14:21:32 -08003318
3319 if (s->flags & SLAB_RECLAIM_ACCOUNT)
3320 s->allocflags |= __GFP_RECLAIMABLE;
3321
Christoph Lameter81819f02007-05-06 14:49:36 -07003322 /*
3323 * Determine the number of objects per slab
3324 */
Lai Jiangshanab9a0f12011-03-10 15:21:48 +08003325 s->oo = oo_make(order, size, s->reserved);
3326 s->min = oo_make(get_order(size), size, s->reserved);
Christoph Lameter205ab992008-04-14 19:11:40 +03003327 if (oo_objects(s->oo) > oo_objects(s->max))
3328 s->max = s->oo;
Christoph Lameter81819f02007-05-06 14:49:36 -07003329
Christoph Lameter834f3d12008-04-14 19:11:31 +03003330 return !!oo_objects(s->oo);
Christoph Lameter81819f02007-05-06 14:49:36 -07003331}
3332
Christoph Lameter8a13a4c2012-09-04 23:18:33 +00003333static int kmem_cache_open(struct kmem_cache *s, unsigned long flags)
Christoph Lameter81819f02007-05-06 14:49:36 -07003334{
Christoph Lameter8a13a4c2012-09-04 23:18:33 +00003335 s->flags = kmem_cache_flags(s->size, flags, s->name, s->ctor);
Lai Jiangshanab9a0f12011-03-10 15:21:48 +08003336 s->reserved = 0;
Christoph Lameter81819f02007-05-06 14:49:36 -07003337
Lai Jiangshanda9a6382011-03-10 15:22:00 +08003338 if (need_reserve_slab_rcu && (s->flags & SLAB_DESTROY_BY_RCU))
3339 s->reserved = sizeof(struct rcu_head);
Christoph Lameter81819f02007-05-06 14:49:36 -07003340
Christoph Lameter06b285d2008-04-14 19:11:41 +03003341 if (!calculate_sizes(s, -1))
Christoph Lameter81819f02007-05-06 14:49:36 -07003342 goto error;
David Rientjes3de47212009-07-27 18:30:35 -07003343 if (disable_higher_order_debug) {
3344 /*
3345 * Disable debugging flags that store metadata if the min slab
3346 * order increased.
3347 */
Christoph Lameter3b0efdf2012-06-13 10:24:57 -05003348 if (get_order(s->size) > get_order(s->object_size)) {
David Rientjes3de47212009-07-27 18:30:35 -07003349 s->flags &= ~DEBUG_METADATA_FLAGS;
3350 s->offset = 0;
3351 if (!calculate_sizes(s, -1))
3352 goto error;
3353 }
3354 }
Christoph Lameter81819f02007-05-06 14:49:36 -07003355
Heiko Carstens25654092012-01-12 17:17:33 -08003356#if defined(CONFIG_HAVE_CMPXCHG_DOUBLE) && \
3357 defined(CONFIG_HAVE_ALIGNED_STRUCT_PAGE)
Christoph Lameterb789ef52011-06-01 12:25:49 -05003358 if (system_has_cmpxchg_double() && (s->flags & SLAB_DEBUG_FLAGS) == 0)
3359 /* Enable fast mode */
3360 s->flags |= __CMPXCHG_DOUBLE;
3361#endif
3362
David Rientjes3b89d7d2009-02-22 17:40:07 -08003363 /*
3364 * The larger the object size is, the more pages we want on the partial
3365 * list to avoid pounding the page allocator excessively.
3366 */
Christoph Lameter49e22582011-08-09 16:12:27 -05003367 set_min_partial(s, ilog2(s->size) / 2);
3368
3369 /*
3370 * cpu_partial determined the maximum number of objects kept in the
3371 * per cpu partial lists of a processor.
3372 *
3373 * Per cpu partial lists mainly contain slabs that just have one
3374 * object freed. If they are used for allocation then they can be
3375 * filled up again with minimal effort. The slab will never hit the
3376 * per node partial lists and therefore no locking will be required.
3377 *
3378 * This setting also determines
3379 *
3380 * A) The number of objects from per cpu partial slabs dumped to the
3381 * per node list when we reach the limit.
Alex Shi9f264902011-09-01 11:32:18 +08003382 * B) The number of objects in cpu partial slabs to extract from the
Chen Gangd0e0ac92013-07-15 09:05:29 +08003383 * per node list when we run out of per cpu objects. We only fetch
3384 * 50% to keep some capacity around for frees.
Christoph Lameter49e22582011-08-09 16:12:27 -05003385 */
Joonsoo Kim345c9052013-06-19 14:05:52 +09003386 if (!kmem_cache_has_cpu_partial(s))
Christoph Lameter8f1e33d2011-11-23 09:24:27 -06003387 s->cpu_partial = 0;
3388 else if (s->size >= PAGE_SIZE)
Christoph Lameter49e22582011-08-09 16:12:27 -05003389 s->cpu_partial = 2;
3390 else if (s->size >= 1024)
3391 s->cpu_partial = 6;
3392 else if (s->size >= 256)
3393 s->cpu_partial = 13;
3394 else
3395 s->cpu_partial = 30;
3396
Christoph Lameter81819f02007-05-06 14:49:36 -07003397#ifdef CONFIG_NUMA
Christoph Lametere2cb96b2008-08-19 08:51:22 -05003398 s->remote_node_defrag_ratio = 1000;
Christoph Lameter81819f02007-05-06 14:49:36 -07003399#endif
Christoph Lameter55136592010-08-20 12:37:13 -05003400 if (!init_kmem_cache_nodes(s))
Christoph Lameterdfb4f092007-10-16 01:26:05 -07003401 goto error;
Christoph Lameter81819f02007-05-06 14:49:36 -07003402
Christoph Lameter55136592010-08-20 12:37:13 -05003403 if (alloc_kmem_cache_cpus(s))
Christoph Lameter278b1bb2012-09-05 00:20:34 +00003404 return 0;
Christoph Lameterff120592009-12-18 16:26:22 -06003405
Christoph Lameter4c93c3552007-10-16 01:26:08 -07003406 free_kmem_cache_nodes(s);
Christoph Lameter81819f02007-05-06 14:49:36 -07003407error:
3408 if (flags & SLAB_PANIC)
3409 panic("Cannot create slab %s size=%lu realsize=%u "
3410 "order=%u offset=%u flags=%lx\n",
Chen Gangd0e0ac92013-07-15 09:05:29 +08003411 s->name, (unsigned long)s->size, s->size,
3412 oo_order(s->oo), s->offset, flags);
Christoph Lameter278b1bb2012-09-05 00:20:34 +00003413 return -EINVAL;
Christoph Lameter81819f02007-05-06 14:49:36 -07003414}
Christoph Lameter81819f02007-05-06 14:49:36 -07003415
Christoph Lameter33b12c32008-04-25 12:22:43 -07003416static void list_slab_objects(struct kmem_cache *s, struct page *page,
3417 const char *text)
Christoph Lameter81819f02007-05-06 14:49:36 -07003418{
Christoph Lameter33b12c32008-04-25 12:22:43 -07003419#ifdef CONFIG_SLUB_DEBUG
3420 void *addr = page_address(page);
3421 void *p;
Namhyung Kima5dd5c12010-09-29 21:02:13 +09003422 unsigned long *map = kzalloc(BITS_TO_LONGS(page->objects) *
3423 sizeof(long), GFP_ATOMIC);
Eric Dumazetbbd7d572010-03-24 22:25:47 +01003424 if (!map)
3425 return;
Christoph Lameter945cf2b2012-09-04 23:18:33 +00003426 slab_err(s, page, text, s->name);
Christoph Lameter33b12c32008-04-25 12:22:43 -07003427 slab_lock(page);
Christoph Lameter33b12c32008-04-25 12:22:43 -07003428
Christoph Lameter5f80b132011-04-15 14:48:13 -05003429 get_map(s, page, map);
Christoph Lameter33b12c32008-04-25 12:22:43 -07003430 for_each_object(p, s, addr, page->objects) {
3431
3432 if (!test_bit(slab_index(p, s, addr), map)) {
Fabian Frederickf9f58282014-06-04 16:06:34 -07003433 pr_err("INFO: Object 0x%p @offset=%tu\n", p, p - addr);
Christoph Lameter33b12c32008-04-25 12:22:43 -07003434 print_tracking(s, p);
3435 }
3436 }
3437 slab_unlock(page);
Eric Dumazetbbd7d572010-03-24 22:25:47 +01003438 kfree(map);
Christoph Lameter33b12c32008-04-25 12:22:43 -07003439#endif
3440}
3441
Christoph Lameter81819f02007-05-06 14:49:36 -07003442/*
Christoph Lameter599870b2008-04-23 12:36:52 -07003443 * Attempt to free all partial slabs on a node.
Christoph Lameter69cb8e62011-08-09 16:12:22 -05003444 * This is called from kmem_cache_close(). We must be the last thread
3445 * using the cache and therefore we do not need to lock anymore.
Christoph Lameter81819f02007-05-06 14:49:36 -07003446 */
Christoph Lameter599870b2008-04-23 12:36:52 -07003447static void free_partial(struct kmem_cache *s, struct kmem_cache_node *n)
Christoph Lameter81819f02007-05-06 14:49:36 -07003448{
Christoph Lameter81819f02007-05-06 14:49:36 -07003449 struct page *page, *h;
3450
Christoph Lameter33b12c32008-04-25 12:22:43 -07003451 list_for_each_entry_safe(page, h, &n->partial, lru) {
Christoph Lameter81819f02007-05-06 14:49:36 -07003452 if (!page->inuse) {
Steven Rostedt1e4dd942014-02-10 14:25:46 -08003453 __remove_partial(n, page);
Christoph Lameter81819f02007-05-06 14:49:36 -07003454 discard_slab(s, page);
Christoph Lameter33b12c32008-04-25 12:22:43 -07003455 } else {
3456 list_slab_objects(s, page,
Christoph Lameter945cf2b2012-09-04 23:18:33 +00003457 "Objects remaining in %s on kmem_cache_close()");
Christoph Lameter599870b2008-04-23 12:36:52 -07003458 }
Christoph Lameter33b12c32008-04-25 12:22:43 -07003459 }
Christoph Lameter81819f02007-05-06 14:49:36 -07003460}
3461
3462/*
Christoph Lameter672bba32007-05-09 02:32:39 -07003463 * Release all resources used by a slab cache.
Christoph Lameter81819f02007-05-06 14:49:36 -07003464 */
Christoph Lameter0c710012007-07-17 04:03:24 -07003465static inline int kmem_cache_close(struct kmem_cache *s)
Christoph Lameter81819f02007-05-06 14:49:36 -07003466{
3467 int node;
Christoph Lameterfa45dc22014-08-06 16:04:09 -07003468 struct kmem_cache_node *n;
Christoph Lameter81819f02007-05-06 14:49:36 -07003469
3470 flush_all(s);
Christoph Lameter81819f02007-05-06 14:49:36 -07003471 /* Attempt to free all objects */
Christoph Lameterfa45dc22014-08-06 16:04:09 -07003472 for_each_kmem_cache_node(s, node, n) {
Christoph Lameter599870b2008-04-23 12:36:52 -07003473 free_partial(s, n);
3474 if (n->nr_partial || slabs_node(s, node))
Christoph Lameter81819f02007-05-06 14:49:36 -07003475 return 1;
3476 }
Christoph Lameter945cf2b2012-09-04 23:18:33 +00003477 free_percpu(s->cpu_slab);
Christoph Lameter81819f02007-05-06 14:49:36 -07003478 free_kmem_cache_nodes(s);
3479 return 0;
3480}
3481
Christoph Lameter945cf2b2012-09-04 23:18:33 +00003482int __kmem_cache_shutdown(struct kmem_cache *s)
Christoph Lameter81819f02007-05-06 14:49:36 -07003483{
Christoph Lameter41a21282014-05-06 12:50:08 -07003484 return kmem_cache_close(s);
Christoph Lameter81819f02007-05-06 14:49:36 -07003485}
Christoph Lameter81819f02007-05-06 14:49:36 -07003486
3487/********************************************************************
3488 * Kmalloc subsystem
3489 *******************************************************************/
3490
Christoph Lameter81819f02007-05-06 14:49:36 -07003491static int __init setup_slub_min_order(char *str)
3492{
Pekka Enberg06428782008-01-07 23:20:27 -08003493 get_option(&str, &slub_min_order);
Christoph Lameter81819f02007-05-06 14:49:36 -07003494
3495 return 1;
3496}
3497
3498__setup("slub_min_order=", setup_slub_min_order);
3499
3500static int __init setup_slub_max_order(char *str)
3501{
Pekka Enberg06428782008-01-07 23:20:27 -08003502 get_option(&str, &slub_max_order);
David Rientjes818cf592009-04-23 09:58:22 +03003503 slub_max_order = min(slub_max_order, MAX_ORDER - 1);
Christoph Lameter81819f02007-05-06 14:49:36 -07003504
3505 return 1;
3506}
3507
3508__setup("slub_max_order=", setup_slub_max_order);
3509
3510static int __init setup_slub_min_objects(char *str)
3511{
Pekka Enberg06428782008-01-07 23:20:27 -08003512 get_option(&str, &slub_min_objects);
Christoph Lameter81819f02007-05-06 14:49:36 -07003513
3514 return 1;
3515}
3516
3517__setup("slub_min_objects=", setup_slub_min_objects);
3518
Christoph Lameter81819f02007-05-06 14:49:36 -07003519void *__kmalloc(size_t size, gfp_t flags)
3520{
Christoph Lameteraadb4bc2007-10-16 01:24:38 -07003521 struct kmem_cache *s;
Eduard - Gabriel Munteanu5b882be2008-08-19 20:43:26 +03003522 void *ret;
Christoph Lameter81819f02007-05-06 14:49:36 -07003523
Christoph Lameter95a05b42013-01-10 19:14:19 +00003524 if (unlikely(size > KMALLOC_MAX_CACHE_SIZE))
Pekka Enbergeada35e2008-02-11 22:47:46 +02003525 return kmalloc_large(size, flags);
Christoph Lameteraadb4bc2007-10-16 01:24:38 -07003526
Christoph Lameter2c59dd62013-01-10 19:14:19 +00003527 s = kmalloc_slab(size, flags);
Christoph Lameteraadb4bc2007-10-16 01:24:38 -07003528
3529 if (unlikely(ZERO_OR_NULL_PTR(s)))
Christoph Lameter6cb8f912007-07-17 04:03:22 -07003530 return s;
3531
Ezequiel Garcia2b847c32012-09-08 17:47:58 -03003532 ret = slab_alloc(s, flags, _RET_IP_);
Eduard - Gabriel Munteanu5b882be2008-08-19 20:43:26 +03003533
Eduard - Gabriel Munteanuca2b84cb2009-03-23 15:12:24 +02003534 trace_kmalloc(_RET_IP_, ret, size, s->size, flags);
Eduard - Gabriel Munteanu5b882be2008-08-19 20:43:26 +03003535
Andrey Ryabinin0316bec2015-02-13 14:39:42 -08003536 kasan_kmalloc(s, ret, size);
3537
Eduard - Gabriel Munteanu5b882be2008-08-19 20:43:26 +03003538 return ret;
Christoph Lameter81819f02007-05-06 14:49:36 -07003539}
3540EXPORT_SYMBOL(__kmalloc);
3541
Namhyung Kim5d1f57e2010-09-29 21:02:15 +09003542#ifdef CONFIG_NUMA
Christoph Lameterf619cfe2008-03-01 13:56:40 -08003543static void *kmalloc_large_node(size_t size, gfp_t flags, int node)
3544{
Vegard Nossumb1eeab62008-11-25 16:55:53 +01003545 struct page *page;
Catalin Marinase4f7c0b42009-07-07 10:32:59 +01003546 void *ptr = NULL;
Christoph Lameterf619cfe2008-03-01 13:56:40 -08003547
Vladimir Davydov52383432014-06-04 16:06:39 -07003548 flags |= __GFP_COMP | __GFP_NOTRACK;
3549 page = alloc_kmem_pages_node(node, flags, get_order(size));
Christoph Lameterf619cfe2008-03-01 13:56:40 -08003550 if (page)
Catalin Marinase4f7c0b42009-07-07 10:32:59 +01003551 ptr = page_address(page);
3552
Roman Bobnievd56791b2013-10-08 15:58:57 -07003553 kmalloc_large_node_hook(ptr, size, flags);
Catalin Marinase4f7c0b42009-07-07 10:32:59 +01003554 return ptr;
Christoph Lameterf619cfe2008-03-01 13:56:40 -08003555}
3556
Christoph Lameter81819f02007-05-06 14:49:36 -07003557void *__kmalloc_node(size_t size, gfp_t flags, int node)
3558{
Christoph Lameteraadb4bc2007-10-16 01:24:38 -07003559 struct kmem_cache *s;
Eduard - Gabriel Munteanu5b882be2008-08-19 20:43:26 +03003560 void *ret;
Christoph Lameter81819f02007-05-06 14:49:36 -07003561
Christoph Lameter95a05b42013-01-10 19:14:19 +00003562 if (unlikely(size > KMALLOC_MAX_CACHE_SIZE)) {
Eduard - Gabriel Munteanu5b882be2008-08-19 20:43:26 +03003563 ret = kmalloc_large_node(size, flags, node);
3564
Eduard - Gabriel Munteanuca2b84cb2009-03-23 15:12:24 +02003565 trace_kmalloc_node(_RET_IP_, ret,
3566 size, PAGE_SIZE << get_order(size),
3567 flags, node);
Eduard - Gabriel Munteanu5b882be2008-08-19 20:43:26 +03003568
3569 return ret;
3570 }
Christoph Lameteraadb4bc2007-10-16 01:24:38 -07003571
Christoph Lameter2c59dd62013-01-10 19:14:19 +00003572 s = kmalloc_slab(size, flags);
Christoph Lameteraadb4bc2007-10-16 01:24:38 -07003573
3574 if (unlikely(ZERO_OR_NULL_PTR(s)))
Christoph Lameter6cb8f912007-07-17 04:03:22 -07003575 return s;
3576
Ezequiel Garcia2b847c32012-09-08 17:47:58 -03003577 ret = slab_alloc_node(s, flags, node, _RET_IP_);
Eduard - Gabriel Munteanu5b882be2008-08-19 20:43:26 +03003578
Eduard - Gabriel Munteanuca2b84cb2009-03-23 15:12:24 +02003579 trace_kmalloc_node(_RET_IP_, ret, size, s->size, flags, node);
Eduard - Gabriel Munteanu5b882be2008-08-19 20:43:26 +03003580
Andrey Ryabinin0316bec2015-02-13 14:39:42 -08003581 kasan_kmalloc(s, ret, size);
3582
Eduard - Gabriel Munteanu5b882be2008-08-19 20:43:26 +03003583 return ret;
Christoph Lameter81819f02007-05-06 14:49:36 -07003584}
3585EXPORT_SYMBOL(__kmalloc_node);
3586#endif
3587
Andrey Ryabinin0316bec2015-02-13 14:39:42 -08003588static size_t __ksize(const void *object)
Christoph Lameter81819f02007-05-06 14:49:36 -07003589{
Christoph Lameter272c1d22007-06-08 13:46:49 -07003590 struct page *page;
Christoph Lameter81819f02007-05-06 14:49:36 -07003591
Christoph Lameteref8b4522007-10-16 01:24:46 -07003592 if (unlikely(object == ZERO_SIZE_PTR))
Christoph Lameter272c1d22007-06-08 13:46:49 -07003593 return 0;
3594
Vegard Nossum294a80a2007-12-04 23:45:30 -08003595 page = virt_to_head_page(object);
Vegard Nossum294a80a2007-12-04 23:45:30 -08003596
Pekka Enberg76994412008-05-22 19:22:25 +03003597 if (unlikely(!PageSlab(page))) {
3598 WARN_ON(!PageCompound(page));
Vegard Nossum294a80a2007-12-04 23:45:30 -08003599 return PAGE_SIZE << compound_order(page);
Pekka Enberg76994412008-05-22 19:22:25 +03003600 }
Christoph Lameter81819f02007-05-06 14:49:36 -07003601
Glauber Costa1b4f59e32012-10-22 18:05:36 +04003602 return slab_ksize(page->slab_cache);
Christoph Lameter81819f02007-05-06 14:49:36 -07003603}
Andrey Ryabinin0316bec2015-02-13 14:39:42 -08003604
3605size_t ksize(const void *object)
3606{
3607 size_t size = __ksize(object);
3608 /* We assume that ksize callers could use whole allocated area,
3609 so we need unpoison this area. */
3610 kasan_krealloc(object, size);
3611 return size;
3612}
Kirill A. Shutemovb1aabec2009-02-10 15:21:44 +02003613EXPORT_SYMBOL(ksize);
Christoph Lameter81819f02007-05-06 14:49:36 -07003614
3615void kfree(const void *x)
3616{
Christoph Lameter81819f02007-05-06 14:49:36 -07003617 struct page *page;
Christoph Lameter5bb983b2008-02-07 17:47:41 -08003618 void *object = (void *)x;
Christoph Lameter81819f02007-05-06 14:49:36 -07003619
Pekka Enberg2121db72009-03-25 11:05:57 +02003620 trace_kfree(_RET_IP_, x);
3621
Satyam Sharma2408c552007-10-16 01:24:44 -07003622 if (unlikely(ZERO_OR_NULL_PTR(x)))
Christoph Lameter81819f02007-05-06 14:49:36 -07003623 return;
3624
Christoph Lameterb49af682007-05-06 14:49:41 -07003625 page = virt_to_head_page(x);
Christoph Lameteraadb4bc2007-10-16 01:24:38 -07003626 if (unlikely(!PageSlab(page))) {
Christoph Lameter09375022008-05-28 10:32:22 -07003627 BUG_ON(!PageCompound(page));
Roman Bobnievd56791b2013-10-08 15:58:57 -07003628 kfree_hook(x);
Vladimir Davydov52383432014-06-04 16:06:39 -07003629 __free_kmem_pages(page, compound_order(page));
Christoph Lameteraadb4bc2007-10-16 01:24:38 -07003630 return;
3631 }
Jesper Dangaard Brouer81084652015-11-20 15:57:46 -08003632 slab_free(page->slab_cache, page, object, NULL, 1, _RET_IP_);
Christoph Lameter81819f02007-05-06 14:49:36 -07003633}
3634EXPORT_SYMBOL(kfree);
3635
Vladimir Davydov832f37f2015-02-12 14:59:41 -08003636#define SHRINK_PROMOTE_MAX 32
3637
Christoph Lameter2086d262007-05-06 14:49:46 -07003638/*
Vladimir Davydov832f37f2015-02-12 14:59:41 -08003639 * kmem_cache_shrink discards empty slabs and promotes the slabs filled
3640 * up most to the head of the partial lists. New allocations will then
3641 * fill those up and thus they can be removed from the partial lists.
Christoph Lameter672bba32007-05-09 02:32:39 -07003642 *
3643 * The slabs with the least items are placed last. This results in them
3644 * being allocated from last increasing the chance that the last objects
3645 * are freed in them.
Christoph Lameter2086d262007-05-06 14:49:46 -07003646 */
Vladimir Davydovd6e0b7f2015-02-12 14:59:47 -08003647int __kmem_cache_shrink(struct kmem_cache *s, bool deactivate)
Christoph Lameter2086d262007-05-06 14:49:46 -07003648{
3649 int node;
3650 int i;
3651 struct kmem_cache_node *n;
3652 struct page *page;
3653 struct page *t;
Vladimir Davydov832f37f2015-02-12 14:59:41 -08003654 struct list_head discard;
3655 struct list_head promote[SHRINK_PROMOTE_MAX];
Christoph Lameter2086d262007-05-06 14:49:46 -07003656 unsigned long flags;
Vladimir Davydovce3712d2015-02-12 14:59:44 -08003657 int ret = 0;
Christoph Lameter2086d262007-05-06 14:49:46 -07003658
Vladimir Davydovd6e0b7f2015-02-12 14:59:47 -08003659 if (deactivate) {
3660 /*
3661 * Disable empty slabs caching. Used to avoid pinning offline
3662 * memory cgroups by kmem pages that can be freed.
3663 */
3664 s->cpu_partial = 0;
3665 s->min_partial = 0;
3666
3667 /*
3668 * s->cpu_partial is checked locklessly (see put_cpu_partial),
3669 * so we have to make sure the change is visible.
3670 */
3671 kick_all_cpus_sync();
3672 }
3673
Christoph Lameter2086d262007-05-06 14:49:46 -07003674 flush_all(s);
Christoph Lameterfa45dc22014-08-06 16:04:09 -07003675 for_each_kmem_cache_node(s, node, n) {
Vladimir Davydov832f37f2015-02-12 14:59:41 -08003676 INIT_LIST_HEAD(&discard);
3677 for (i = 0; i < SHRINK_PROMOTE_MAX; i++)
3678 INIT_LIST_HEAD(promote + i);
Christoph Lameter2086d262007-05-06 14:49:46 -07003679
3680 spin_lock_irqsave(&n->list_lock, flags);
3681
3682 /*
Vladimir Davydov832f37f2015-02-12 14:59:41 -08003683 * Build lists of slabs to discard or promote.
Christoph Lameter2086d262007-05-06 14:49:46 -07003684 *
Christoph Lameter672bba32007-05-09 02:32:39 -07003685 * Note that concurrent frees may occur while we hold the
3686 * list_lock. page->inuse here is the upper limit.
Christoph Lameter2086d262007-05-06 14:49:46 -07003687 */
3688 list_for_each_entry_safe(page, t, &n->partial, lru) {
Vladimir Davydov832f37f2015-02-12 14:59:41 -08003689 int free = page->objects - page->inuse;
3690
3691 /* Do not reread page->inuse */
3692 barrier();
3693
3694 /* We do not keep full slabs on the list */
3695 BUG_ON(free <= 0);
3696
3697 if (free == page->objects) {
3698 list_move(&page->lru, &discard);
Christoph Lameter69cb8e62011-08-09 16:12:22 -05003699 n->nr_partial--;
Vladimir Davydov832f37f2015-02-12 14:59:41 -08003700 } else if (free <= SHRINK_PROMOTE_MAX)
3701 list_move(&page->lru, promote + free - 1);
Christoph Lameter2086d262007-05-06 14:49:46 -07003702 }
3703
Christoph Lameter2086d262007-05-06 14:49:46 -07003704 /*
Vladimir Davydov832f37f2015-02-12 14:59:41 -08003705 * Promote the slabs filled up most to the head of the
3706 * partial list.
Christoph Lameter2086d262007-05-06 14:49:46 -07003707 */
Vladimir Davydov832f37f2015-02-12 14:59:41 -08003708 for (i = SHRINK_PROMOTE_MAX - 1; i >= 0; i--)
3709 list_splice(promote + i, &n->partial);
Christoph Lameter2086d262007-05-06 14:49:46 -07003710
Christoph Lameter2086d262007-05-06 14:49:46 -07003711 spin_unlock_irqrestore(&n->list_lock, flags);
Christoph Lameter69cb8e62011-08-09 16:12:22 -05003712
3713 /* Release empty slabs */
Vladimir Davydov832f37f2015-02-12 14:59:41 -08003714 list_for_each_entry_safe(page, t, &discard, lru)
Christoph Lameter69cb8e62011-08-09 16:12:22 -05003715 discard_slab(s, page);
Vladimir Davydovce3712d2015-02-12 14:59:44 -08003716
3717 if (slabs_node(s, node))
3718 ret = 1;
Christoph Lameter2086d262007-05-06 14:49:46 -07003719 }
3720
Vladimir Davydovce3712d2015-02-12 14:59:44 -08003721 return ret;
Christoph Lameter2086d262007-05-06 14:49:46 -07003722}
Christoph Lameter2086d262007-05-06 14:49:46 -07003723
Yasunori Gotob9049e22007-10-21 16:41:37 -07003724static int slab_mem_going_offline_callback(void *arg)
3725{
3726 struct kmem_cache *s;
3727
Christoph Lameter18004c52012-07-06 15:25:12 -05003728 mutex_lock(&slab_mutex);
Yasunori Gotob9049e22007-10-21 16:41:37 -07003729 list_for_each_entry(s, &slab_caches, list)
Vladimir Davydovd6e0b7f2015-02-12 14:59:47 -08003730 __kmem_cache_shrink(s, false);
Christoph Lameter18004c52012-07-06 15:25:12 -05003731 mutex_unlock(&slab_mutex);
Yasunori Gotob9049e22007-10-21 16:41:37 -07003732
3733 return 0;
3734}
3735
3736static void slab_mem_offline_callback(void *arg)
3737{
3738 struct kmem_cache_node *n;
3739 struct kmem_cache *s;
3740 struct memory_notify *marg = arg;
3741 int offline_node;
3742
Lai Jiangshanb9d5ab22012-12-11 16:01:05 -08003743 offline_node = marg->status_change_nid_normal;
Yasunori Gotob9049e22007-10-21 16:41:37 -07003744
3745 /*
3746 * If the node still has available memory. we need kmem_cache_node
3747 * for it yet.
3748 */
3749 if (offline_node < 0)
3750 return;
3751
Christoph Lameter18004c52012-07-06 15:25:12 -05003752 mutex_lock(&slab_mutex);
Yasunori Gotob9049e22007-10-21 16:41:37 -07003753 list_for_each_entry(s, &slab_caches, list) {
3754 n = get_node(s, offline_node);
3755 if (n) {
3756 /*
3757 * if n->nr_slabs > 0, slabs still exist on the node
3758 * that is going down. We were unable to free them,
Adam Buchbinderc9404c92009-12-18 15:40:42 -05003759 * and offline_pages() function shouldn't call this
Yasunori Gotob9049e22007-10-21 16:41:37 -07003760 * callback. So, we must fail.
3761 */
Christoph Lameter0f389ec2008-04-14 18:53:02 +03003762 BUG_ON(slabs_node(s, offline_node));
Yasunori Gotob9049e22007-10-21 16:41:37 -07003763
3764 s->node[offline_node] = NULL;
Christoph Lameter8de66a02010-08-25 14:51:14 -05003765 kmem_cache_free(kmem_cache_node, n);
Yasunori Gotob9049e22007-10-21 16:41:37 -07003766 }
3767 }
Christoph Lameter18004c52012-07-06 15:25:12 -05003768 mutex_unlock(&slab_mutex);
Yasunori Gotob9049e22007-10-21 16:41:37 -07003769}
3770
3771static int slab_mem_going_online_callback(void *arg)
3772{
3773 struct kmem_cache_node *n;
3774 struct kmem_cache *s;
3775 struct memory_notify *marg = arg;
Lai Jiangshanb9d5ab22012-12-11 16:01:05 -08003776 int nid = marg->status_change_nid_normal;
Yasunori Gotob9049e22007-10-21 16:41:37 -07003777 int ret = 0;
3778
3779 /*
3780 * If the node's memory is already available, then kmem_cache_node is
3781 * already created. Nothing to do.
3782 */
3783 if (nid < 0)
3784 return 0;
3785
3786 /*
Christoph Lameter0121c6192008-04-29 16:11:12 -07003787 * We are bringing a node online. No memory is available yet. We must
Yasunori Gotob9049e22007-10-21 16:41:37 -07003788 * allocate a kmem_cache_node structure in order to bring the node
3789 * online.
3790 */
Christoph Lameter18004c52012-07-06 15:25:12 -05003791 mutex_lock(&slab_mutex);
Yasunori Gotob9049e22007-10-21 16:41:37 -07003792 list_for_each_entry(s, &slab_caches, list) {
3793 /*
3794 * XXX: kmem_cache_alloc_node will fallback to other nodes
3795 * since memory is not yet available from the node that
3796 * is brought up.
3797 */
Christoph Lameter8de66a02010-08-25 14:51:14 -05003798 n = kmem_cache_alloc(kmem_cache_node, GFP_KERNEL);
Yasunori Gotob9049e22007-10-21 16:41:37 -07003799 if (!n) {
3800 ret = -ENOMEM;
3801 goto out;
3802 }
Joonsoo Kim40534972012-05-11 00:50:47 +09003803 init_kmem_cache_node(n);
Yasunori Gotob9049e22007-10-21 16:41:37 -07003804 s->node[nid] = n;
3805 }
3806out:
Christoph Lameter18004c52012-07-06 15:25:12 -05003807 mutex_unlock(&slab_mutex);
Yasunori Gotob9049e22007-10-21 16:41:37 -07003808 return ret;
3809}
3810
3811static int slab_memory_callback(struct notifier_block *self,
3812 unsigned long action, void *arg)
3813{
3814 int ret = 0;
3815
3816 switch (action) {
3817 case MEM_GOING_ONLINE:
3818 ret = slab_mem_going_online_callback(arg);
3819 break;
3820 case MEM_GOING_OFFLINE:
3821 ret = slab_mem_going_offline_callback(arg);
3822 break;
3823 case MEM_OFFLINE:
3824 case MEM_CANCEL_ONLINE:
3825 slab_mem_offline_callback(arg);
3826 break;
3827 case MEM_ONLINE:
3828 case MEM_CANCEL_OFFLINE:
3829 break;
3830 }
KAMEZAWA Hiroyukidc19f9d2008-12-01 13:13:48 -08003831 if (ret)
3832 ret = notifier_from_errno(ret);
3833 else
3834 ret = NOTIFY_OK;
Yasunori Gotob9049e22007-10-21 16:41:37 -07003835 return ret;
3836}
3837
Andrew Morton3ac38fa2013-04-29 15:08:06 -07003838static struct notifier_block slab_memory_callback_nb = {
3839 .notifier_call = slab_memory_callback,
3840 .priority = SLAB_CALLBACK_PRI,
3841};
Yasunori Gotob9049e22007-10-21 16:41:37 -07003842
Christoph Lameter81819f02007-05-06 14:49:36 -07003843/********************************************************************
3844 * Basic setup of slabs
3845 *******************************************************************/
3846
Christoph Lameter51df1142010-08-20 12:37:15 -05003847/*
3848 * Used for early kmem_cache structures that were allocated using
Christoph Lameterdffb4d62012-11-28 16:23:07 +00003849 * the page allocator. Allocate them properly then fix up the pointers
3850 * that may be pointing to the wrong kmem_cache structure.
Christoph Lameter51df1142010-08-20 12:37:15 -05003851 */
3852
Christoph Lameterdffb4d62012-11-28 16:23:07 +00003853static struct kmem_cache * __init bootstrap(struct kmem_cache *static_cache)
Christoph Lameter51df1142010-08-20 12:37:15 -05003854{
3855 int node;
Christoph Lameterdffb4d62012-11-28 16:23:07 +00003856 struct kmem_cache *s = kmem_cache_zalloc(kmem_cache, GFP_NOWAIT);
Christoph Lameterfa45dc22014-08-06 16:04:09 -07003857 struct kmem_cache_node *n;
Christoph Lameter51df1142010-08-20 12:37:15 -05003858
Christoph Lameterdffb4d62012-11-28 16:23:07 +00003859 memcpy(s, static_cache, kmem_cache->object_size);
Christoph Lameter51df1142010-08-20 12:37:15 -05003860
Glauber Costa7d557b32013-02-22 20:20:00 +04003861 /*
3862 * This runs very early, and only the boot processor is supposed to be
3863 * up. Even if it weren't true, IRQs are not up so we couldn't fire
3864 * IPIs around.
3865 */
3866 __flush_cpu_slab(s, smp_processor_id());
Christoph Lameterfa45dc22014-08-06 16:04:09 -07003867 for_each_kmem_cache_node(s, node, n) {
Christoph Lameter51df1142010-08-20 12:37:15 -05003868 struct page *p;
3869
Christoph Lameterfa45dc22014-08-06 16:04:09 -07003870 list_for_each_entry(p, &n->partial, lru)
3871 p->slab_cache = s;
Christoph Lameter51df1142010-08-20 12:37:15 -05003872
Li Zefan607bf322011-04-12 15:22:26 +08003873#ifdef CONFIG_SLUB_DEBUG
Christoph Lameterfa45dc22014-08-06 16:04:09 -07003874 list_for_each_entry(p, &n->full, lru)
3875 p->slab_cache = s;
Christoph Lameter51df1142010-08-20 12:37:15 -05003876#endif
Christoph Lameter51df1142010-08-20 12:37:15 -05003877 }
Vladimir Davydovf7ce3192015-02-12 14:59:20 -08003878 slab_init_memcg_params(s);
Christoph Lameterdffb4d62012-11-28 16:23:07 +00003879 list_add(&s->list, &slab_caches);
3880 return s;
Christoph Lameter51df1142010-08-20 12:37:15 -05003881}
3882
Christoph Lameter81819f02007-05-06 14:49:36 -07003883void __init kmem_cache_init(void)
3884{
Christoph Lameterdffb4d62012-11-28 16:23:07 +00003885 static __initdata struct kmem_cache boot_kmem_cache,
3886 boot_kmem_cache_node;
Christoph Lameter51df1142010-08-20 12:37:15 -05003887
Stanislaw Gruszkafc8d8622012-01-10 15:07:32 -08003888 if (debug_guardpage_minorder())
3889 slub_max_order = 0;
3890
Christoph Lameterdffb4d62012-11-28 16:23:07 +00003891 kmem_cache_node = &boot_kmem_cache_node;
3892 kmem_cache = &boot_kmem_cache;
Christoph Lameter51df1142010-08-20 12:37:15 -05003893
Christoph Lameterdffb4d62012-11-28 16:23:07 +00003894 create_boot_cache(kmem_cache_node, "kmem_cache_node",
3895 sizeof(struct kmem_cache_node), SLAB_HWCACHE_ALIGN);
Yasunori Gotob9049e22007-10-21 16:41:37 -07003896
Andrew Morton3ac38fa2013-04-29 15:08:06 -07003897 register_hotmemory_notifier(&slab_memory_callback_nb);
Christoph Lameter81819f02007-05-06 14:49:36 -07003898
3899 /* Able to allocate the per node structures */
3900 slab_state = PARTIAL;
3901
Christoph Lameterdffb4d62012-11-28 16:23:07 +00003902 create_boot_cache(kmem_cache, "kmem_cache",
3903 offsetof(struct kmem_cache, node) +
3904 nr_node_ids * sizeof(struct kmem_cache_node *),
3905 SLAB_HWCACHE_ALIGN);
Christoph Lameter8a13a4c2012-09-04 23:18:33 +00003906
Christoph Lameterdffb4d62012-11-28 16:23:07 +00003907 kmem_cache = bootstrap(&boot_kmem_cache);
Christoph Lameter81819f02007-05-06 14:49:36 -07003908
Christoph Lameter51df1142010-08-20 12:37:15 -05003909 /*
3910 * Allocate kmem_cache_node properly from the kmem_cache slab.
3911 * kmem_cache_node is separately allocated so no need to
3912 * update any list pointers.
3913 */
Christoph Lameterdffb4d62012-11-28 16:23:07 +00003914 kmem_cache_node = bootstrap(&boot_kmem_cache_node);
Christoph Lameter51df1142010-08-20 12:37:15 -05003915
3916 /* Now we can use the kmem_cache to allocate kmalloc slabs */
Daniel Sanders34cc6992015-06-24 16:55:57 -07003917 setup_kmalloc_cache_index_table();
Christoph Lameterf97d5f62013-01-10 19:12:17 +00003918 create_kmalloc_caches(0);
Christoph Lameter81819f02007-05-06 14:49:36 -07003919
3920#ifdef CONFIG_SMP
3921 register_cpu_notifier(&slab_notifier);
Christoph Lameter9dfc6e62009-12-18 16:26:20 -06003922#endif
Christoph Lameter81819f02007-05-06 14:49:36 -07003923
Fabian Frederickf9f58282014-06-04 16:06:34 -07003924 pr_info("SLUB: HWalign=%d, Order=%d-%d, MinObjects=%d, CPUs=%d, Nodes=%d\n",
Christoph Lameterf97d5f62013-01-10 19:12:17 +00003925 cache_line_size(),
Christoph Lameter81819f02007-05-06 14:49:36 -07003926 slub_min_order, slub_max_order, slub_min_objects,
3927 nr_cpu_ids, nr_node_ids);
3928}
3929
Pekka Enberg7e85ee02009-06-12 14:03:06 +03003930void __init kmem_cache_init_late(void)
3931{
Pekka Enberg7e85ee02009-06-12 14:03:06 +03003932}
3933
Glauber Costa2633d7a2012-12-18 14:22:34 -08003934struct kmem_cache *
Vladimir Davydova44cb942014-04-07 15:39:23 -07003935__kmem_cache_alias(const char *name, size_t size, size_t align,
3936 unsigned long flags, void (*ctor)(void *))
Christoph Lameter81819f02007-05-06 14:49:36 -07003937{
Vladimir Davydov426589f2015-02-12 14:59:23 -08003938 struct kmem_cache *s, *c;
Christoph Lameter81819f02007-05-06 14:49:36 -07003939
Vladimir Davydova44cb942014-04-07 15:39:23 -07003940 s = find_mergeable(size, align, flags, name, ctor);
Christoph Lameter81819f02007-05-06 14:49:36 -07003941 if (s) {
3942 s->refcount++;
Vladimir Davydov84d0ddd2014-04-07 15:39:29 -07003943
Christoph Lameter81819f02007-05-06 14:49:36 -07003944 /*
3945 * Adjust the object sizes so that we clear
3946 * the complete object on kzalloc.
3947 */
Christoph Lameter3b0efdf2012-06-13 10:24:57 -05003948 s->object_size = max(s->object_size, (int)size);
Christoph Lameter81819f02007-05-06 14:49:36 -07003949 s->inuse = max_t(int, s->inuse, ALIGN(size, sizeof(void *)));
Christoph Lameter6446faa2008-02-15 23:45:26 -08003950
Vladimir Davydov426589f2015-02-12 14:59:23 -08003951 for_each_memcg_cache(c, s) {
Vladimir Davydov84d0ddd2014-04-07 15:39:29 -07003952 c->object_size = s->object_size;
3953 c->inuse = max_t(int, c->inuse,
3954 ALIGN(size, sizeof(void *)));
3955 }
3956
David Rientjes7b8f3b62008-12-17 22:09:46 -08003957 if (sysfs_slab_alias(s, name)) {
David Rientjes7b8f3b62008-12-17 22:09:46 -08003958 s->refcount--;
Christoph Lametercbb79692012-09-05 00:18:32 +00003959 s = NULL;
David Rientjes7b8f3b62008-12-17 22:09:46 -08003960 }
Christoph Lametera0e1d1b2007-07-17 04:03:31 -07003961 }
Christoph Lameter6446faa2008-02-15 23:45:26 -08003962
Christoph Lametercbb79692012-09-05 00:18:32 +00003963 return s;
3964}
Pekka Enberg84c1cf62010-09-14 23:21:12 +03003965
Christoph Lameter8a13a4c2012-09-04 23:18:33 +00003966int __kmem_cache_create(struct kmem_cache *s, unsigned long flags)
Christoph Lametercbb79692012-09-05 00:18:32 +00003967{
Pekka Enbergaac3a162012-09-05 12:07:44 +03003968 int err;
Christoph Lameter20cea962012-07-06 15:25:13 -05003969
Pekka Enbergaac3a162012-09-05 12:07:44 +03003970 err = kmem_cache_open(s, flags);
3971 if (err)
3972 return err;
Christoph Lameter20cea962012-07-06 15:25:13 -05003973
Christoph Lameter45530c42012-11-28 16:23:07 +00003974 /* Mutex is not taken during early boot */
3975 if (slab_state <= UP)
3976 return 0;
3977
Glauber Costa107dab52012-12-18 14:23:05 -08003978 memcg_propagate_slab_attrs(s);
Pekka Enbergaac3a162012-09-05 12:07:44 +03003979 err = sysfs_slab_add(s);
Pekka Enbergaac3a162012-09-05 12:07:44 +03003980 if (err)
3981 kmem_cache_close(s);
3982
3983 return err;
Christoph Lameter81819f02007-05-06 14:49:36 -07003984}
Christoph Lameter81819f02007-05-06 14:49:36 -07003985
Christoph Lameter81819f02007-05-06 14:49:36 -07003986#ifdef CONFIG_SMP
Christoph Lameter27390bc2007-06-01 00:47:09 -07003987/*
Christoph Lameter672bba32007-05-09 02:32:39 -07003988 * Use the cpu notifier to insure that the cpu slabs are flushed when
3989 * necessary.
Christoph Lameter81819f02007-05-06 14:49:36 -07003990 */
Paul Gortmaker0db06282013-06-19 14:53:51 -04003991static int slab_cpuup_callback(struct notifier_block *nfb,
Christoph Lameter81819f02007-05-06 14:49:36 -07003992 unsigned long action, void *hcpu)
3993{
3994 long cpu = (long)hcpu;
Christoph Lameter5b95a4ac2007-07-17 04:03:19 -07003995 struct kmem_cache *s;
3996 unsigned long flags;
Christoph Lameter81819f02007-05-06 14:49:36 -07003997
3998 switch (action) {
3999 case CPU_UP_CANCELED:
Rafael J. Wysocki8bb78442007-05-09 02:35:10 -07004000 case CPU_UP_CANCELED_FROZEN:
Christoph Lameter81819f02007-05-06 14:49:36 -07004001 case CPU_DEAD:
Rafael J. Wysocki8bb78442007-05-09 02:35:10 -07004002 case CPU_DEAD_FROZEN:
Christoph Lameter18004c52012-07-06 15:25:12 -05004003 mutex_lock(&slab_mutex);
Christoph Lameter5b95a4ac2007-07-17 04:03:19 -07004004 list_for_each_entry(s, &slab_caches, list) {
4005 local_irq_save(flags);
4006 __flush_cpu_slab(s, cpu);
4007 local_irq_restore(flags);
4008 }
Christoph Lameter18004c52012-07-06 15:25:12 -05004009 mutex_unlock(&slab_mutex);
Christoph Lameter81819f02007-05-06 14:49:36 -07004010 break;
4011 default:
4012 break;
4013 }
4014 return NOTIFY_OK;
4015}
4016
Paul Gortmaker0db06282013-06-19 14:53:51 -04004017static struct notifier_block slab_notifier = {
Ingo Molnar3adbefe2008-02-05 17:57:39 -08004018 .notifier_call = slab_cpuup_callback
Pekka Enberg06428782008-01-07 23:20:27 -08004019};
Christoph Lameter81819f02007-05-06 14:49:36 -07004020
4021#endif
4022
Eduard - Gabriel Munteanuce71e272008-08-19 20:43:25 +03004023void *__kmalloc_track_caller(size_t size, gfp_t gfpflags, unsigned long caller)
Christoph Lameter81819f02007-05-06 14:49:36 -07004024{
Christoph Lameteraadb4bc2007-10-16 01:24:38 -07004025 struct kmem_cache *s;
Eduard - Gabriel Munteanu94b528d2008-08-24 20:49:35 +03004026 void *ret;
Christoph Lameteraadb4bc2007-10-16 01:24:38 -07004027
Christoph Lameter95a05b42013-01-10 19:14:19 +00004028 if (unlikely(size > KMALLOC_MAX_CACHE_SIZE))
Pekka Enbergeada35e2008-02-11 22:47:46 +02004029 return kmalloc_large(size, gfpflags);
4030
Christoph Lameter2c59dd62013-01-10 19:14:19 +00004031 s = kmalloc_slab(size, gfpflags);
Christoph Lameter81819f02007-05-06 14:49:36 -07004032
Satyam Sharma2408c552007-10-16 01:24:44 -07004033 if (unlikely(ZERO_OR_NULL_PTR(s)))
Christoph Lameter6cb8f912007-07-17 04:03:22 -07004034 return s;
Christoph Lameter81819f02007-05-06 14:49:36 -07004035
Ezequiel Garcia2b847c32012-09-08 17:47:58 -03004036 ret = slab_alloc(s, gfpflags, caller);
Eduard - Gabriel Munteanu94b528d2008-08-24 20:49:35 +03004037
Lucas De Marchi25985ed2011-03-30 22:57:33 -03004038 /* Honor the call site pointer we received. */
Eduard - Gabriel Munteanuca2b84cb2009-03-23 15:12:24 +02004039 trace_kmalloc(caller, ret, size, s->size, gfpflags);
Eduard - Gabriel Munteanu94b528d2008-08-24 20:49:35 +03004040
4041 return ret;
Christoph Lameter81819f02007-05-06 14:49:36 -07004042}
4043
Namhyung Kim5d1f57e2010-09-29 21:02:15 +09004044#ifdef CONFIG_NUMA
Christoph Lameter81819f02007-05-06 14:49:36 -07004045void *__kmalloc_node_track_caller(size_t size, gfp_t gfpflags,
Eduard - Gabriel Munteanuce71e272008-08-19 20:43:25 +03004046 int node, unsigned long caller)
Christoph Lameter81819f02007-05-06 14:49:36 -07004047{
Christoph Lameteraadb4bc2007-10-16 01:24:38 -07004048 struct kmem_cache *s;
Eduard - Gabriel Munteanu94b528d2008-08-24 20:49:35 +03004049 void *ret;
Christoph Lameteraadb4bc2007-10-16 01:24:38 -07004050
Christoph Lameter95a05b42013-01-10 19:14:19 +00004051 if (unlikely(size > KMALLOC_MAX_CACHE_SIZE)) {
Xiaotian Fengd3e14aa2010-04-08 17:26:44 +08004052 ret = kmalloc_large_node(size, gfpflags, node);
4053
4054 trace_kmalloc_node(caller, ret,
4055 size, PAGE_SIZE << get_order(size),
4056 gfpflags, node);
4057
4058 return ret;
4059 }
Pekka Enbergeada35e2008-02-11 22:47:46 +02004060
Christoph Lameter2c59dd62013-01-10 19:14:19 +00004061 s = kmalloc_slab(size, gfpflags);
Christoph Lameter81819f02007-05-06 14:49:36 -07004062
Satyam Sharma2408c552007-10-16 01:24:44 -07004063 if (unlikely(ZERO_OR_NULL_PTR(s)))
Christoph Lameter6cb8f912007-07-17 04:03:22 -07004064 return s;
Christoph Lameter81819f02007-05-06 14:49:36 -07004065
Ezequiel Garcia2b847c32012-09-08 17:47:58 -03004066 ret = slab_alloc_node(s, gfpflags, node, caller);
Eduard - Gabriel Munteanu94b528d2008-08-24 20:49:35 +03004067
Lucas De Marchi25985ed2011-03-30 22:57:33 -03004068 /* Honor the call site pointer we received. */
Eduard - Gabriel Munteanuca2b84cb2009-03-23 15:12:24 +02004069 trace_kmalloc_node(caller, ret, size, s->size, gfpflags, node);
Eduard - Gabriel Munteanu94b528d2008-08-24 20:49:35 +03004070
4071 return ret;
Christoph Lameter81819f02007-05-06 14:49:36 -07004072}
Namhyung Kim5d1f57e2010-09-29 21:02:15 +09004073#endif
Christoph Lameter81819f02007-05-06 14:49:36 -07004074
Christoph Lameterab4d5ed2010-10-05 13:57:26 -05004075#ifdef CONFIG_SYSFS
Christoph Lameter205ab992008-04-14 19:11:40 +03004076static int count_inuse(struct page *page)
4077{
4078 return page->inuse;
4079}
4080
4081static int count_total(struct page *page)
4082{
4083 return page->objects;
4084}
Christoph Lameterab4d5ed2010-10-05 13:57:26 -05004085#endif
Christoph Lameter205ab992008-04-14 19:11:40 +03004086
Christoph Lameterab4d5ed2010-10-05 13:57:26 -05004087#ifdef CONFIG_SLUB_DEBUG
Christoph Lameter434e2452007-07-17 04:03:30 -07004088static int validate_slab(struct kmem_cache *s, struct page *page,
4089 unsigned long *map)
Christoph Lameter53e15af2007-05-06 14:49:43 -07004090{
4091 void *p;
Christoph Lametera973e9d2008-03-01 13:40:44 -08004092 void *addr = page_address(page);
Christoph Lameter53e15af2007-05-06 14:49:43 -07004093
4094 if (!check_slab(s, page) ||
4095 !on_freelist(s, page, NULL))
4096 return 0;
4097
4098 /* Now we know that a valid freelist exists */
Christoph Lameter39b26462008-04-14 19:11:30 +03004099 bitmap_zero(map, page->objects);
Christoph Lameter53e15af2007-05-06 14:49:43 -07004100
Christoph Lameter5f80b132011-04-15 14:48:13 -05004101 get_map(s, page, map);
4102 for_each_object(p, s, addr, page->objects) {
4103 if (test_bit(slab_index(p, s, addr), map))
4104 if (!check_object(s, page, p, SLUB_RED_INACTIVE))
4105 return 0;
Christoph Lameter53e15af2007-05-06 14:49:43 -07004106 }
4107
Christoph Lameter224a88b2008-04-14 19:11:31 +03004108 for_each_object(p, s, addr, page->objects)
Christoph Lameter7656c722007-05-09 02:32:40 -07004109 if (!test_bit(slab_index(p, s, addr), map))
Tero Roponen37d57442010-12-01 20:04:20 +02004110 if (!check_object(s, page, p, SLUB_RED_ACTIVE))
Christoph Lameter53e15af2007-05-06 14:49:43 -07004111 return 0;
4112 return 1;
4113}
4114
Christoph Lameter434e2452007-07-17 04:03:30 -07004115static void validate_slab_slab(struct kmem_cache *s, struct page *page,
4116 unsigned long *map)
Christoph Lameter53e15af2007-05-06 14:49:43 -07004117{
Christoph Lameter881db7f2011-06-01 12:25:53 -05004118 slab_lock(page);
4119 validate_slab(s, page, map);
4120 slab_unlock(page);
Christoph Lameter53e15af2007-05-06 14:49:43 -07004121}
4122
Christoph Lameter434e2452007-07-17 04:03:30 -07004123static int validate_slab_node(struct kmem_cache *s,
4124 struct kmem_cache_node *n, unsigned long *map)
Christoph Lameter53e15af2007-05-06 14:49:43 -07004125{
4126 unsigned long count = 0;
4127 struct page *page;
4128 unsigned long flags;
4129
4130 spin_lock_irqsave(&n->list_lock, flags);
4131
4132 list_for_each_entry(page, &n->partial, lru) {
Christoph Lameter434e2452007-07-17 04:03:30 -07004133 validate_slab_slab(s, page, map);
Christoph Lameter53e15af2007-05-06 14:49:43 -07004134 count++;
4135 }
4136 if (count != n->nr_partial)
Fabian Frederickf9f58282014-06-04 16:06:34 -07004137 pr_err("SLUB %s: %ld partial slabs counted but counter=%ld\n",
4138 s->name, count, n->nr_partial);
Christoph Lameter53e15af2007-05-06 14:49:43 -07004139
4140 if (!(s->flags & SLAB_STORE_USER))
4141 goto out;
4142
4143 list_for_each_entry(page, &n->full, lru) {
Christoph Lameter434e2452007-07-17 04:03:30 -07004144 validate_slab_slab(s, page, map);
Christoph Lameter53e15af2007-05-06 14:49:43 -07004145 count++;
4146 }
4147 if (count != atomic_long_read(&n->nr_slabs))
Fabian Frederickf9f58282014-06-04 16:06:34 -07004148 pr_err("SLUB: %s %ld slabs counted but counter=%ld\n",
4149 s->name, count, atomic_long_read(&n->nr_slabs));
Christoph Lameter53e15af2007-05-06 14:49:43 -07004150
4151out:
4152 spin_unlock_irqrestore(&n->list_lock, flags);
4153 return count;
4154}
4155
Christoph Lameter434e2452007-07-17 04:03:30 -07004156static long validate_slab_cache(struct kmem_cache *s)
Christoph Lameter53e15af2007-05-06 14:49:43 -07004157{
4158 int node;
4159 unsigned long count = 0;
Christoph Lameter205ab992008-04-14 19:11:40 +03004160 unsigned long *map = kmalloc(BITS_TO_LONGS(oo_objects(s->max)) *
Christoph Lameter434e2452007-07-17 04:03:30 -07004161 sizeof(unsigned long), GFP_KERNEL);
Christoph Lameterfa45dc22014-08-06 16:04:09 -07004162 struct kmem_cache_node *n;
Christoph Lameter434e2452007-07-17 04:03:30 -07004163
4164 if (!map)
4165 return -ENOMEM;
Christoph Lameter53e15af2007-05-06 14:49:43 -07004166
4167 flush_all(s);
Christoph Lameterfa45dc22014-08-06 16:04:09 -07004168 for_each_kmem_cache_node(s, node, n)
Christoph Lameter434e2452007-07-17 04:03:30 -07004169 count += validate_slab_node(s, n, map);
Christoph Lameter434e2452007-07-17 04:03:30 -07004170 kfree(map);
Christoph Lameter53e15af2007-05-06 14:49:43 -07004171 return count;
4172}
Christoph Lameter88a420e2007-05-06 14:49:45 -07004173/*
Christoph Lameter672bba32007-05-09 02:32:39 -07004174 * Generate lists of code addresses where slabcache objects are allocated
Christoph Lameter88a420e2007-05-06 14:49:45 -07004175 * and freed.
4176 */
4177
4178struct location {
4179 unsigned long count;
Eduard - Gabriel Munteanuce71e272008-08-19 20:43:25 +03004180 unsigned long addr;
Christoph Lameter45edfa52007-05-09 02:32:45 -07004181 long long sum_time;
4182 long min_time;
4183 long max_time;
4184 long min_pid;
4185 long max_pid;
Rusty Russell174596a2009-01-01 10:12:29 +10304186 DECLARE_BITMAP(cpus, NR_CPUS);
Christoph Lameter45edfa52007-05-09 02:32:45 -07004187 nodemask_t nodes;
Christoph Lameter88a420e2007-05-06 14:49:45 -07004188};
4189
4190struct loc_track {
4191 unsigned long max;
4192 unsigned long count;
4193 struct location *loc;
4194};
4195
4196static void free_loc_track(struct loc_track *t)
4197{
4198 if (t->max)
4199 free_pages((unsigned long)t->loc,
4200 get_order(sizeof(struct location) * t->max));
4201}
4202
Christoph Lameter68dff6a2007-07-17 04:03:20 -07004203static int alloc_loc_track(struct loc_track *t, unsigned long max, gfp_t flags)
Christoph Lameter88a420e2007-05-06 14:49:45 -07004204{
4205 struct location *l;
4206 int order;
4207
Christoph Lameter88a420e2007-05-06 14:49:45 -07004208 order = get_order(sizeof(struct location) * max);
4209
Christoph Lameter68dff6a2007-07-17 04:03:20 -07004210 l = (void *)__get_free_pages(flags, order);
Christoph Lameter88a420e2007-05-06 14:49:45 -07004211 if (!l)
4212 return 0;
4213
4214 if (t->count) {
4215 memcpy(l, t->loc, sizeof(struct location) * t->count);
4216 free_loc_track(t);
4217 }
4218 t->max = max;
4219 t->loc = l;
4220 return 1;
4221}
4222
4223static int add_location(struct loc_track *t, struct kmem_cache *s,
Christoph Lameter45edfa52007-05-09 02:32:45 -07004224 const struct track *track)
Christoph Lameter88a420e2007-05-06 14:49:45 -07004225{
4226 long start, end, pos;
4227 struct location *l;
Eduard - Gabriel Munteanuce71e272008-08-19 20:43:25 +03004228 unsigned long caddr;
Christoph Lameter45edfa52007-05-09 02:32:45 -07004229 unsigned long age = jiffies - track->when;
Christoph Lameter88a420e2007-05-06 14:49:45 -07004230
4231 start = -1;
4232 end = t->count;
4233
4234 for ( ; ; ) {
4235 pos = start + (end - start + 1) / 2;
4236
4237 /*
4238 * There is nothing at "end". If we end up there
4239 * we need to add something to before end.
4240 */
4241 if (pos == end)
4242 break;
4243
4244 caddr = t->loc[pos].addr;
Christoph Lameter45edfa52007-05-09 02:32:45 -07004245 if (track->addr == caddr) {
4246
4247 l = &t->loc[pos];
4248 l->count++;
4249 if (track->when) {
4250 l->sum_time += age;
4251 if (age < l->min_time)
4252 l->min_time = age;
4253 if (age > l->max_time)
4254 l->max_time = age;
4255
4256 if (track->pid < l->min_pid)
4257 l->min_pid = track->pid;
4258 if (track->pid > l->max_pid)
4259 l->max_pid = track->pid;
4260
Rusty Russell174596a2009-01-01 10:12:29 +10304261 cpumask_set_cpu(track->cpu,
4262 to_cpumask(l->cpus));
Christoph Lameter45edfa52007-05-09 02:32:45 -07004263 }
4264 node_set(page_to_nid(virt_to_page(track)), l->nodes);
Christoph Lameter88a420e2007-05-06 14:49:45 -07004265 return 1;
4266 }
4267
Christoph Lameter45edfa52007-05-09 02:32:45 -07004268 if (track->addr < caddr)
Christoph Lameter88a420e2007-05-06 14:49:45 -07004269 end = pos;
4270 else
4271 start = pos;
4272 }
4273
4274 /*
Christoph Lameter672bba32007-05-09 02:32:39 -07004275 * Not found. Insert new tracking element.
Christoph Lameter88a420e2007-05-06 14:49:45 -07004276 */
Christoph Lameter68dff6a2007-07-17 04:03:20 -07004277 if (t->count >= t->max && !alloc_loc_track(t, 2 * t->max, GFP_ATOMIC))
Christoph Lameter88a420e2007-05-06 14:49:45 -07004278 return 0;
4279
4280 l = t->loc + pos;
4281 if (pos < t->count)
4282 memmove(l + 1, l,
4283 (t->count - pos) * sizeof(struct location));
4284 t->count++;
4285 l->count = 1;
Christoph Lameter45edfa52007-05-09 02:32:45 -07004286 l->addr = track->addr;
4287 l->sum_time = age;
4288 l->min_time = age;
4289 l->max_time = age;
4290 l->min_pid = track->pid;
4291 l->max_pid = track->pid;
Rusty Russell174596a2009-01-01 10:12:29 +10304292 cpumask_clear(to_cpumask(l->cpus));
4293 cpumask_set_cpu(track->cpu, to_cpumask(l->cpus));
Christoph Lameter45edfa52007-05-09 02:32:45 -07004294 nodes_clear(l->nodes);
4295 node_set(page_to_nid(virt_to_page(track)), l->nodes);
Christoph Lameter88a420e2007-05-06 14:49:45 -07004296 return 1;
4297}
4298
4299static void process_slab(struct loc_track *t, struct kmem_cache *s,
Eric Dumazetbbd7d572010-03-24 22:25:47 +01004300 struct page *page, enum track_item alloc,
Namhyung Kima5dd5c12010-09-29 21:02:13 +09004301 unsigned long *map)
Christoph Lameter88a420e2007-05-06 14:49:45 -07004302{
Christoph Lametera973e9d2008-03-01 13:40:44 -08004303 void *addr = page_address(page);
Christoph Lameter88a420e2007-05-06 14:49:45 -07004304 void *p;
4305
Christoph Lameter39b26462008-04-14 19:11:30 +03004306 bitmap_zero(map, page->objects);
Christoph Lameter5f80b132011-04-15 14:48:13 -05004307 get_map(s, page, map);
Christoph Lameter88a420e2007-05-06 14:49:45 -07004308
Christoph Lameter224a88b2008-04-14 19:11:31 +03004309 for_each_object(p, s, addr, page->objects)
Christoph Lameter45edfa52007-05-09 02:32:45 -07004310 if (!test_bit(slab_index(p, s, addr), map))
4311 add_location(t, s, get_track(s, p, alloc));
Christoph Lameter88a420e2007-05-06 14:49:45 -07004312}
4313
4314static int list_locations(struct kmem_cache *s, char *buf,
4315 enum track_item alloc)
4316{
Harvey Harrisone374d482008-01-31 15:20:50 -08004317 int len = 0;
Christoph Lameter88a420e2007-05-06 14:49:45 -07004318 unsigned long i;
Christoph Lameter68dff6a2007-07-17 04:03:20 -07004319 struct loc_track t = { 0, 0, NULL };
Christoph Lameter88a420e2007-05-06 14:49:45 -07004320 int node;
Eric Dumazetbbd7d572010-03-24 22:25:47 +01004321 unsigned long *map = kmalloc(BITS_TO_LONGS(oo_objects(s->max)) *
4322 sizeof(unsigned long), GFP_KERNEL);
Christoph Lameterfa45dc22014-08-06 16:04:09 -07004323 struct kmem_cache_node *n;
Christoph Lameter88a420e2007-05-06 14:49:45 -07004324
Eric Dumazetbbd7d572010-03-24 22:25:47 +01004325 if (!map || !alloc_loc_track(&t, PAGE_SIZE / sizeof(struct location),
4326 GFP_TEMPORARY)) {
4327 kfree(map);
Christoph Lameter68dff6a2007-07-17 04:03:20 -07004328 return sprintf(buf, "Out of memory\n");
Eric Dumazetbbd7d572010-03-24 22:25:47 +01004329 }
Christoph Lameter88a420e2007-05-06 14:49:45 -07004330 /* Push back cpu slabs */
4331 flush_all(s);
4332
Christoph Lameterfa45dc22014-08-06 16:04:09 -07004333 for_each_kmem_cache_node(s, node, n) {
Christoph Lameter88a420e2007-05-06 14:49:45 -07004334 unsigned long flags;
4335 struct page *page;
4336
Christoph Lameter9e869432007-08-22 14:01:56 -07004337 if (!atomic_long_read(&n->nr_slabs))
Christoph Lameter88a420e2007-05-06 14:49:45 -07004338 continue;
4339
4340 spin_lock_irqsave(&n->list_lock, flags);
4341 list_for_each_entry(page, &n->partial, lru)
Eric Dumazetbbd7d572010-03-24 22:25:47 +01004342 process_slab(&t, s, page, alloc, map);
Christoph Lameter88a420e2007-05-06 14:49:45 -07004343 list_for_each_entry(page, &n->full, lru)
Eric Dumazetbbd7d572010-03-24 22:25:47 +01004344 process_slab(&t, s, page, alloc, map);
Christoph Lameter88a420e2007-05-06 14:49:45 -07004345 spin_unlock_irqrestore(&n->list_lock, flags);
4346 }
4347
4348 for (i = 0; i < t.count; i++) {
Christoph Lameter45edfa52007-05-09 02:32:45 -07004349 struct location *l = &t.loc[i];
Christoph Lameter88a420e2007-05-06 14:49:45 -07004350
Hugh Dickins9c246242008-12-09 13:14:27 -08004351 if (len > PAGE_SIZE - KSYM_SYMBOL_LEN - 100)
Christoph Lameter88a420e2007-05-06 14:49:45 -07004352 break;
Harvey Harrisone374d482008-01-31 15:20:50 -08004353 len += sprintf(buf + len, "%7ld ", l->count);
Christoph Lameter45edfa52007-05-09 02:32:45 -07004354
4355 if (l->addr)
Joe Perches62c70bc2011-01-13 15:45:52 -08004356 len += sprintf(buf + len, "%pS", (void *)l->addr);
Christoph Lameter88a420e2007-05-06 14:49:45 -07004357 else
Harvey Harrisone374d482008-01-31 15:20:50 -08004358 len += sprintf(buf + len, "<not-available>");
Christoph Lameter45edfa52007-05-09 02:32:45 -07004359
4360 if (l->sum_time != l->min_time) {
Harvey Harrisone374d482008-01-31 15:20:50 -08004361 len += sprintf(buf + len, " age=%ld/%ld/%ld",
Roman Zippelf8bd2252008-05-01 04:34:31 -07004362 l->min_time,
4363 (long)div_u64(l->sum_time, l->count),
4364 l->max_time);
Christoph Lameter45edfa52007-05-09 02:32:45 -07004365 } else
Harvey Harrisone374d482008-01-31 15:20:50 -08004366 len += sprintf(buf + len, " age=%ld",
Christoph Lameter45edfa52007-05-09 02:32:45 -07004367 l->min_time);
4368
4369 if (l->min_pid != l->max_pid)
Harvey Harrisone374d482008-01-31 15:20:50 -08004370 len += sprintf(buf + len, " pid=%ld-%ld",
Christoph Lameter45edfa52007-05-09 02:32:45 -07004371 l->min_pid, l->max_pid);
4372 else
Harvey Harrisone374d482008-01-31 15:20:50 -08004373 len += sprintf(buf + len, " pid=%ld",
Christoph Lameter45edfa52007-05-09 02:32:45 -07004374 l->min_pid);
4375
Rusty Russell174596a2009-01-01 10:12:29 +10304376 if (num_online_cpus() > 1 &&
4377 !cpumask_empty(to_cpumask(l->cpus)) &&
Tejun Heo5024c1d2015-02-13 14:37:59 -08004378 len < PAGE_SIZE - 60)
4379 len += scnprintf(buf + len, PAGE_SIZE - len - 50,
4380 " cpus=%*pbl",
4381 cpumask_pr_args(to_cpumask(l->cpus)));
Christoph Lameter45edfa52007-05-09 02:32:45 -07004382
Christoph Lameter62bc62a2009-06-16 15:32:15 -07004383 if (nr_online_nodes > 1 && !nodes_empty(l->nodes) &&
Tejun Heo5024c1d2015-02-13 14:37:59 -08004384 len < PAGE_SIZE - 60)
4385 len += scnprintf(buf + len, PAGE_SIZE - len - 50,
4386 " nodes=%*pbl",
4387 nodemask_pr_args(&l->nodes));
Christoph Lameter45edfa52007-05-09 02:32:45 -07004388
Harvey Harrisone374d482008-01-31 15:20:50 -08004389 len += sprintf(buf + len, "\n");
Christoph Lameter88a420e2007-05-06 14:49:45 -07004390 }
4391
4392 free_loc_track(&t);
Eric Dumazetbbd7d572010-03-24 22:25:47 +01004393 kfree(map);
Christoph Lameter88a420e2007-05-06 14:49:45 -07004394 if (!t.count)
Harvey Harrisone374d482008-01-31 15:20:50 -08004395 len += sprintf(buf, "No data\n");
4396 return len;
Christoph Lameter88a420e2007-05-06 14:49:45 -07004397}
Christoph Lameterab4d5ed2010-10-05 13:57:26 -05004398#endif
Christoph Lameter88a420e2007-05-06 14:49:45 -07004399
Christoph Lametera5a84752010-10-05 13:57:27 -05004400#ifdef SLUB_RESILIENCY_TEST
David Rientjesc07b8182014-08-06 16:04:16 -07004401static void __init resiliency_test(void)
Christoph Lametera5a84752010-10-05 13:57:27 -05004402{
4403 u8 *p;
4404
Christoph Lameter95a05b42013-01-10 19:14:19 +00004405 BUILD_BUG_ON(KMALLOC_MIN_SIZE > 16 || KMALLOC_SHIFT_HIGH < 10);
Christoph Lametera5a84752010-10-05 13:57:27 -05004406
Fabian Frederickf9f58282014-06-04 16:06:34 -07004407 pr_err("SLUB resiliency testing\n");
4408 pr_err("-----------------------\n");
4409 pr_err("A. Corruption after allocation\n");
Christoph Lametera5a84752010-10-05 13:57:27 -05004410
4411 p = kzalloc(16, GFP_KERNEL);
4412 p[16] = 0x12;
Fabian Frederickf9f58282014-06-04 16:06:34 -07004413 pr_err("\n1. kmalloc-16: Clobber Redzone/next pointer 0x12->0x%p\n\n",
4414 p + 16);
Christoph Lametera5a84752010-10-05 13:57:27 -05004415
4416 validate_slab_cache(kmalloc_caches[4]);
4417
4418 /* Hmmm... The next two are dangerous */
4419 p = kzalloc(32, GFP_KERNEL);
4420 p[32 + sizeof(void *)] = 0x34;
Fabian Frederickf9f58282014-06-04 16:06:34 -07004421 pr_err("\n2. kmalloc-32: Clobber next pointer/next slab 0x34 -> -0x%p\n",
4422 p);
4423 pr_err("If allocated object is overwritten then not detectable\n\n");
Christoph Lametera5a84752010-10-05 13:57:27 -05004424
4425 validate_slab_cache(kmalloc_caches[5]);
4426 p = kzalloc(64, GFP_KERNEL);
4427 p += 64 + (get_cycles() & 0xff) * sizeof(void *);
4428 *p = 0x56;
Fabian Frederickf9f58282014-06-04 16:06:34 -07004429 pr_err("\n3. kmalloc-64: corrupting random byte 0x56->0x%p\n",
4430 p);
4431 pr_err("If allocated object is overwritten then not detectable\n\n");
Christoph Lametera5a84752010-10-05 13:57:27 -05004432 validate_slab_cache(kmalloc_caches[6]);
4433
Fabian Frederickf9f58282014-06-04 16:06:34 -07004434 pr_err("\nB. Corruption after free\n");
Christoph Lametera5a84752010-10-05 13:57:27 -05004435 p = kzalloc(128, GFP_KERNEL);
4436 kfree(p);
4437 *p = 0x78;
Fabian Frederickf9f58282014-06-04 16:06:34 -07004438 pr_err("1. kmalloc-128: Clobber first word 0x78->0x%p\n\n", p);
Christoph Lametera5a84752010-10-05 13:57:27 -05004439 validate_slab_cache(kmalloc_caches[7]);
4440
4441 p = kzalloc(256, GFP_KERNEL);
4442 kfree(p);
4443 p[50] = 0x9a;
Fabian Frederickf9f58282014-06-04 16:06:34 -07004444 pr_err("\n2. kmalloc-256: Clobber 50th byte 0x9a->0x%p\n\n", p);
Christoph Lametera5a84752010-10-05 13:57:27 -05004445 validate_slab_cache(kmalloc_caches[8]);
4446
4447 p = kzalloc(512, GFP_KERNEL);
4448 kfree(p);
4449 p[512] = 0xab;
Fabian Frederickf9f58282014-06-04 16:06:34 -07004450 pr_err("\n3. kmalloc-512: Clobber redzone 0xab->0x%p\n\n", p);
Christoph Lametera5a84752010-10-05 13:57:27 -05004451 validate_slab_cache(kmalloc_caches[9]);
4452}
4453#else
4454#ifdef CONFIG_SYSFS
4455static void resiliency_test(void) {};
4456#endif
4457#endif
4458
Christoph Lameterab4d5ed2010-10-05 13:57:26 -05004459#ifdef CONFIG_SYSFS
Christoph Lameter81819f02007-05-06 14:49:36 -07004460enum slab_stat_type {
Christoph Lameter205ab992008-04-14 19:11:40 +03004461 SL_ALL, /* All slabs */
4462 SL_PARTIAL, /* Only partially allocated slabs */
4463 SL_CPU, /* Only slabs used for cpu caches */
4464 SL_OBJECTS, /* Determine allocated objects not slabs */
4465 SL_TOTAL /* Determine object capacity not slabs */
Christoph Lameter81819f02007-05-06 14:49:36 -07004466};
4467
Christoph Lameter205ab992008-04-14 19:11:40 +03004468#define SO_ALL (1 << SL_ALL)
Christoph Lameter81819f02007-05-06 14:49:36 -07004469#define SO_PARTIAL (1 << SL_PARTIAL)
4470#define SO_CPU (1 << SL_CPU)
4471#define SO_OBJECTS (1 << SL_OBJECTS)
Christoph Lameter205ab992008-04-14 19:11:40 +03004472#define SO_TOTAL (1 << SL_TOTAL)
Christoph Lameter81819f02007-05-06 14:49:36 -07004473
Cyrill Gorcunov62e5c4b2008-03-02 23:28:24 +03004474static ssize_t show_slab_objects(struct kmem_cache *s,
4475 char *buf, unsigned long flags)
Christoph Lameter81819f02007-05-06 14:49:36 -07004476{
4477 unsigned long total = 0;
Christoph Lameter81819f02007-05-06 14:49:36 -07004478 int node;
4479 int x;
4480 unsigned long *nodes;
Christoph Lameter81819f02007-05-06 14:49:36 -07004481
Chen Gange35e1a92013-07-12 08:23:48 +08004482 nodes = kzalloc(sizeof(unsigned long) * nr_node_ids, GFP_KERNEL);
Cyrill Gorcunov62e5c4b2008-03-02 23:28:24 +03004483 if (!nodes)
4484 return -ENOMEM;
Christoph Lameter81819f02007-05-06 14:49:36 -07004485
Christoph Lameter205ab992008-04-14 19:11:40 +03004486 if (flags & SO_CPU) {
4487 int cpu;
Christoph Lameter81819f02007-05-06 14:49:36 -07004488
Christoph Lameter205ab992008-04-14 19:11:40 +03004489 for_each_possible_cpu(cpu) {
Chen Gangd0e0ac92013-07-15 09:05:29 +08004490 struct kmem_cache_cpu *c = per_cpu_ptr(s->cpu_slab,
4491 cpu);
Christoph Lameterec3ab082012-05-09 10:09:56 -05004492 int node;
Christoph Lameter49e22582011-08-09 16:12:27 -05004493 struct page *page;
Christoph Lameterdfb4f092007-10-16 01:26:05 -07004494
Jason Low4db0c3c2015-04-15 16:14:08 -07004495 page = READ_ONCE(c->page);
Christoph Lameterec3ab082012-05-09 10:09:56 -05004496 if (!page)
4497 continue;
Christoph Lameter205ab992008-04-14 19:11:40 +03004498
Christoph Lameterec3ab082012-05-09 10:09:56 -05004499 node = page_to_nid(page);
4500 if (flags & SO_TOTAL)
4501 x = page->objects;
4502 else if (flags & SO_OBJECTS)
4503 x = page->inuse;
4504 else
4505 x = 1;
Christoph Lameter49e22582011-08-09 16:12:27 -05004506
Christoph Lameterec3ab082012-05-09 10:09:56 -05004507 total += x;
4508 nodes[node] += x;
4509
Jason Low4db0c3c2015-04-15 16:14:08 -07004510 page = READ_ONCE(c->partial);
Christoph Lameter49e22582011-08-09 16:12:27 -05004511 if (page) {
Li Zefan8afb1472013-09-10 11:43:37 +08004512 node = page_to_nid(page);
4513 if (flags & SO_TOTAL)
4514 WARN_ON_ONCE(1);
4515 else if (flags & SO_OBJECTS)
4516 WARN_ON_ONCE(1);
4517 else
4518 x = page->pages;
Eric Dumazetbc6697d2011-11-22 16:02:02 +01004519 total += x;
4520 nodes[node] += x;
Christoph Lameter49e22582011-08-09 16:12:27 -05004521 }
Christoph Lameter81819f02007-05-06 14:49:36 -07004522 }
4523 }
4524
Vladimir Davydovbfc8c902014-06-04 16:07:18 -07004525 get_online_mems();
Christoph Lameterab4d5ed2010-10-05 13:57:26 -05004526#ifdef CONFIG_SLUB_DEBUG
Christoph Lameter205ab992008-04-14 19:11:40 +03004527 if (flags & SO_ALL) {
Christoph Lameterfa45dc22014-08-06 16:04:09 -07004528 struct kmem_cache_node *n;
4529
4530 for_each_kmem_cache_node(s, node, n) {
Christoph Lameter81819f02007-05-06 14:49:36 -07004531
Chen Gangd0e0ac92013-07-15 09:05:29 +08004532 if (flags & SO_TOTAL)
4533 x = atomic_long_read(&n->total_objects);
4534 else if (flags & SO_OBJECTS)
4535 x = atomic_long_read(&n->total_objects) -
4536 count_partial(n, count_free);
Christoph Lameter205ab992008-04-14 19:11:40 +03004537 else
4538 x = atomic_long_read(&n->nr_slabs);
4539 total += x;
4540 nodes[node] += x;
4541 }
4542
Christoph Lameterab4d5ed2010-10-05 13:57:26 -05004543 } else
4544#endif
4545 if (flags & SO_PARTIAL) {
Christoph Lameterfa45dc22014-08-06 16:04:09 -07004546 struct kmem_cache_node *n;
Christoph Lameter205ab992008-04-14 19:11:40 +03004547
Christoph Lameterfa45dc22014-08-06 16:04:09 -07004548 for_each_kmem_cache_node(s, node, n) {
Christoph Lameter205ab992008-04-14 19:11:40 +03004549 if (flags & SO_TOTAL)
4550 x = count_partial(n, count_total);
4551 else if (flags & SO_OBJECTS)
4552 x = count_partial(n, count_inuse);
Christoph Lameter81819f02007-05-06 14:49:36 -07004553 else
4554 x = n->nr_partial;
4555 total += x;
4556 nodes[node] += x;
4557 }
Christoph Lameter81819f02007-05-06 14:49:36 -07004558 }
Christoph Lameter81819f02007-05-06 14:49:36 -07004559 x = sprintf(buf, "%lu", total);
4560#ifdef CONFIG_NUMA
Christoph Lameterfa45dc22014-08-06 16:04:09 -07004561 for (node = 0; node < nr_node_ids; node++)
Christoph Lameter81819f02007-05-06 14:49:36 -07004562 if (nodes[node])
4563 x += sprintf(buf + x, " N%d=%lu",
4564 node, nodes[node]);
4565#endif
Vladimir Davydovbfc8c902014-06-04 16:07:18 -07004566 put_online_mems();
Christoph Lameter81819f02007-05-06 14:49:36 -07004567 kfree(nodes);
4568 return x + sprintf(buf + x, "\n");
4569}
4570
Christoph Lameterab4d5ed2010-10-05 13:57:26 -05004571#ifdef CONFIG_SLUB_DEBUG
Christoph Lameter81819f02007-05-06 14:49:36 -07004572static int any_slab_objects(struct kmem_cache *s)
4573{
4574 int node;
Christoph Lameterfa45dc22014-08-06 16:04:09 -07004575 struct kmem_cache_node *n;
Christoph Lameterdfb4f092007-10-16 01:26:05 -07004576
Christoph Lameterfa45dc22014-08-06 16:04:09 -07004577 for_each_kmem_cache_node(s, node, n)
Benjamin Herrenschmidt4ea33e22008-05-06 20:42:39 -07004578 if (atomic_long_read(&n->total_objects))
Christoph Lameter81819f02007-05-06 14:49:36 -07004579 return 1;
Christoph Lameterfa45dc22014-08-06 16:04:09 -07004580
Christoph Lameter81819f02007-05-06 14:49:36 -07004581 return 0;
4582}
Christoph Lameterab4d5ed2010-10-05 13:57:26 -05004583#endif
Christoph Lameter81819f02007-05-06 14:49:36 -07004584
4585#define to_slab_attr(n) container_of(n, struct slab_attribute, attr)
Phil Carmody497888c2011-07-14 15:07:13 +03004586#define to_slab(n) container_of(n, struct kmem_cache, kobj)
Christoph Lameter81819f02007-05-06 14:49:36 -07004587
4588struct slab_attribute {
4589 struct attribute attr;
4590 ssize_t (*show)(struct kmem_cache *s, char *buf);
4591 ssize_t (*store)(struct kmem_cache *s, const char *x, size_t count);
4592};
4593
4594#define SLAB_ATTR_RO(_name) \
Vasiliy Kulikovab067e92011-09-27 21:54:53 +04004595 static struct slab_attribute _name##_attr = \
4596 __ATTR(_name, 0400, _name##_show, NULL)
Christoph Lameter81819f02007-05-06 14:49:36 -07004597
4598#define SLAB_ATTR(_name) \
4599 static struct slab_attribute _name##_attr = \
Vasiliy Kulikovab067e92011-09-27 21:54:53 +04004600 __ATTR(_name, 0600, _name##_show, _name##_store)
Christoph Lameter81819f02007-05-06 14:49:36 -07004601
Christoph Lameter81819f02007-05-06 14:49:36 -07004602static ssize_t slab_size_show(struct kmem_cache *s, char *buf)
4603{
4604 return sprintf(buf, "%d\n", s->size);
4605}
4606SLAB_ATTR_RO(slab_size);
4607
4608static ssize_t align_show(struct kmem_cache *s, char *buf)
4609{
4610 return sprintf(buf, "%d\n", s->align);
4611}
4612SLAB_ATTR_RO(align);
4613
4614static ssize_t object_size_show(struct kmem_cache *s, char *buf)
4615{
Christoph Lameter3b0efdf2012-06-13 10:24:57 -05004616 return sprintf(buf, "%d\n", s->object_size);
Christoph Lameter81819f02007-05-06 14:49:36 -07004617}
4618SLAB_ATTR_RO(object_size);
4619
4620static ssize_t objs_per_slab_show(struct kmem_cache *s, char *buf)
4621{
Christoph Lameter834f3d12008-04-14 19:11:31 +03004622 return sprintf(buf, "%d\n", oo_objects(s->oo));
Christoph Lameter81819f02007-05-06 14:49:36 -07004623}
4624SLAB_ATTR_RO(objs_per_slab);
4625
Christoph Lameter06b285d2008-04-14 19:11:41 +03004626static ssize_t order_store(struct kmem_cache *s,
4627 const char *buf, size_t length)
4628{
Christoph Lameter0121c6192008-04-29 16:11:12 -07004629 unsigned long order;
4630 int err;
4631
Jingoo Han3dbb95f2013-09-11 14:20:25 -07004632 err = kstrtoul(buf, 10, &order);
Christoph Lameter0121c6192008-04-29 16:11:12 -07004633 if (err)
4634 return err;
Christoph Lameter06b285d2008-04-14 19:11:41 +03004635
4636 if (order > slub_max_order || order < slub_min_order)
4637 return -EINVAL;
4638
4639 calculate_sizes(s, order);
4640 return length;
4641}
4642
Christoph Lameter81819f02007-05-06 14:49:36 -07004643static ssize_t order_show(struct kmem_cache *s, char *buf)
4644{
Christoph Lameter834f3d12008-04-14 19:11:31 +03004645 return sprintf(buf, "%d\n", oo_order(s->oo));
Christoph Lameter81819f02007-05-06 14:49:36 -07004646}
Christoph Lameter06b285d2008-04-14 19:11:41 +03004647SLAB_ATTR(order);
Christoph Lameter81819f02007-05-06 14:49:36 -07004648
David Rientjes73d342b2009-02-22 17:40:09 -08004649static ssize_t min_partial_show(struct kmem_cache *s, char *buf)
4650{
4651 return sprintf(buf, "%lu\n", s->min_partial);
4652}
4653
4654static ssize_t min_partial_store(struct kmem_cache *s, const char *buf,
4655 size_t length)
4656{
4657 unsigned long min;
4658 int err;
4659
Jingoo Han3dbb95f2013-09-11 14:20:25 -07004660 err = kstrtoul(buf, 10, &min);
David Rientjes73d342b2009-02-22 17:40:09 -08004661 if (err)
4662 return err;
4663
David Rientjesc0bdb232009-02-25 09:16:35 +02004664 set_min_partial(s, min);
David Rientjes73d342b2009-02-22 17:40:09 -08004665 return length;
4666}
4667SLAB_ATTR(min_partial);
4668
Christoph Lameter49e22582011-08-09 16:12:27 -05004669static ssize_t cpu_partial_show(struct kmem_cache *s, char *buf)
4670{
4671 return sprintf(buf, "%u\n", s->cpu_partial);
4672}
4673
4674static ssize_t cpu_partial_store(struct kmem_cache *s, const char *buf,
4675 size_t length)
4676{
4677 unsigned long objects;
4678 int err;
4679
Jingoo Han3dbb95f2013-09-11 14:20:25 -07004680 err = kstrtoul(buf, 10, &objects);
Christoph Lameter49e22582011-08-09 16:12:27 -05004681 if (err)
4682 return err;
Joonsoo Kim345c9052013-06-19 14:05:52 +09004683 if (objects && !kmem_cache_has_cpu_partial(s))
David Rientjes74ee4ef2012-01-09 13:19:45 -08004684 return -EINVAL;
Christoph Lameter49e22582011-08-09 16:12:27 -05004685
4686 s->cpu_partial = objects;
4687 flush_all(s);
4688 return length;
4689}
4690SLAB_ATTR(cpu_partial);
4691
Christoph Lameter81819f02007-05-06 14:49:36 -07004692static ssize_t ctor_show(struct kmem_cache *s, char *buf)
4693{
Joe Perches62c70bc2011-01-13 15:45:52 -08004694 if (!s->ctor)
4695 return 0;
4696 return sprintf(buf, "%pS\n", s->ctor);
Christoph Lameter81819f02007-05-06 14:49:36 -07004697}
4698SLAB_ATTR_RO(ctor);
4699
Christoph Lameter81819f02007-05-06 14:49:36 -07004700static ssize_t aliases_show(struct kmem_cache *s, char *buf)
4701{
Gu Zheng4307c142014-08-06 16:04:51 -07004702 return sprintf(buf, "%d\n", s->refcount < 0 ? 0 : s->refcount - 1);
Christoph Lameter81819f02007-05-06 14:49:36 -07004703}
4704SLAB_ATTR_RO(aliases);
4705
Christoph Lameter81819f02007-05-06 14:49:36 -07004706static ssize_t partial_show(struct kmem_cache *s, char *buf)
4707{
Christoph Lameterd9acf4b2008-02-15 15:22:21 -08004708 return show_slab_objects(s, buf, SO_PARTIAL);
Christoph Lameter81819f02007-05-06 14:49:36 -07004709}
4710SLAB_ATTR_RO(partial);
4711
4712static ssize_t cpu_slabs_show(struct kmem_cache *s, char *buf)
4713{
Christoph Lameterd9acf4b2008-02-15 15:22:21 -08004714 return show_slab_objects(s, buf, SO_CPU);
Christoph Lameter81819f02007-05-06 14:49:36 -07004715}
4716SLAB_ATTR_RO(cpu_slabs);
4717
4718static ssize_t objects_show(struct kmem_cache *s, char *buf)
4719{
Christoph Lameter205ab992008-04-14 19:11:40 +03004720 return show_slab_objects(s, buf, SO_ALL|SO_OBJECTS);
Christoph Lameter81819f02007-05-06 14:49:36 -07004721}
4722SLAB_ATTR_RO(objects);
4723
Christoph Lameter205ab992008-04-14 19:11:40 +03004724static ssize_t objects_partial_show(struct kmem_cache *s, char *buf)
4725{
4726 return show_slab_objects(s, buf, SO_PARTIAL|SO_OBJECTS);
4727}
4728SLAB_ATTR_RO(objects_partial);
4729
Christoph Lameter49e22582011-08-09 16:12:27 -05004730static ssize_t slabs_cpu_partial_show(struct kmem_cache *s, char *buf)
4731{
4732 int objects = 0;
4733 int pages = 0;
4734 int cpu;
4735 int len;
4736
4737 for_each_online_cpu(cpu) {
4738 struct page *page = per_cpu_ptr(s->cpu_slab, cpu)->partial;
4739
4740 if (page) {
4741 pages += page->pages;
4742 objects += page->pobjects;
4743 }
4744 }
4745
4746 len = sprintf(buf, "%d(%d)", objects, pages);
4747
4748#ifdef CONFIG_SMP
4749 for_each_online_cpu(cpu) {
4750 struct page *page = per_cpu_ptr(s->cpu_slab, cpu) ->partial;
4751
4752 if (page && len < PAGE_SIZE - 20)
4753 len += sprintf(buf + len, " C%d=%d(%d)", cpu,
4754 page->pobjects, page->pages);
4755 }
4756#endif
4757 return len + sprintf(buf + len, "\n");
4758}
4759SLAB_ATTR_RO(slabs_cpu_partial);
4760
Christoph Lameter81819f02007-05-06 14:49:36 -07004761static ssize_t reclaim_account_show(struct kmem_cache *s, char *buf)
4762{
4763 return sprintf(buf, "%d\n", !!(s->flags & SLAB_RECLAIM_ACCOUNT));
4764}
4765
4766static ssize_t reclaim_account_store(struct kmem_cache *s,
4767 const char *buf, size_t length)
4768{
4769 s->flags &= ~SLAB_RECLAIM_ACCOUNT;
4770 if (buf[0] == '1')
4771 s->flags |= SLAB_RECLAIM_ACCOUNT;
4772 return length;
4773}
4774SLAB_ATTR(reclaim_account);
4775
4776static ssize_t hwcache_align_show(struct kmem_cache *s, char *buf)
4777{
Christoph Lameter5af60832007-05-06 14:49:56 -07004778 return sprintf(buf, "%d\n", !!(s->flags & SLAB_HWCACHE_ALIGN));
Christoph Lameter81819f02007-05-06 14:49:36 -07004779}
4780SLAB_ATTR_RO(hwcache_align);
4781
4782#ifdef CONFIG_ZONE_DMA
4783static ssize_t cache_dma_show(struct kmem_cache *s, char *buf)
4784{
4785 return sprintf(buf, "%d\n", !!(s->flags & SLAB_CACHE_DMA));
4786}
4787SLAB_ATTR_RO(cache_dma);
4788#endif
4789
4790static ssize_t destroy_by_rcu_show(struct kmem_cache *s, char *buf)
4791{
4792 return sprintf(buf, "%d\n", !!(s->flags & SLAB_DESTROY_BY_RCU));
4793}
4794SLAB_ATTR_RO(destroy_by_rcu);
4795
Lai Jiangshanab9a0f12011-03-10 15:21:48 +08004796static ssize_t reserved_show(struct kmem_cache *s, char *buf)
4797{
4798 return sprintf(buf, "%d\n", s->reserved);
4799}
4800SLAB_ATTR_RO(reserved);
4801
Christoph Lameterab4d5ed2010-10-05 13:57:26 -05004802#ifdef CONFIG_SLUB_DEBUG
Christoph Lametera5a84752010-10-05 13:57:27 -05004803static ssize_t slabs_show(struct kmem_cache *s, char *buf)
4804{
4805 return show_slab_objects(s, buf, SO_ALL);
4806}
4807SLAB_ATTR_RO(slabs);
4808
4809static ssize_t total_objects_show(struct kmem_cache *s, char *buf)
4810{
4811 return show_slab_objects(s, buf, SO_ALL|SO_TOTAL);
4812}
4813SLAB_ATTR_RO(total_objects);
4814
4815static ssize_t sanity_checks_show(struct kmem_cache *s, char *buf)
4816{
4817 return sprintf(buf, "%d\n", !!(s->flags & SLAB_DEBUG_FREE));
4818}
4819
4820static ssize_t sanity_checks_store(struct kmem_cache *s,
4821 const char *buf, size_t length)
4822{
4823 s->flags &= ~SLAB_DEBUG_FREE;
Christoph Lameterb789ef52011-06-01 12:25:49 -05004824 if (buf[0] == '1') {
4825 s->flags &= ~__CMPXCHG_DOUBLE;
Christoph Lametera5a84752010-10-05 13:57:27 -05004826 s->flags |= SLAB_DEBUG_FREE;
Christoph Lameterb789ef52011-06-01 12:25:49 -05004827 }
Christoph Lametera5a84752010-10-05 13:57:27 -05004828 return length;
4829}
4830SLAB_ATTR(sanity_checks);
4831
4832static ssize_t trace_show(struct kmem_cache *s, char *buf)
4833{
4834 return sprintf(buf, "%d\n", !!(s->flags & SLAB_TRACE));
4835}
4836
4837static ssize_t trace_store(struct kmem_cache *s, const char *buf,
4838 size_t length)
4839{
Christoph Lameterc9e16132014-10-09 15:26:11 -07004840 /*
4841 * Tracing a merged cache is going to give confusing results
4842 * as well as cause other issues like converting a mergeable
4843 * cache into an umergeable one.
4844 */
4845 if (s->refcount > 1)
4846 return -EINVAL;
4847
Christoph Lametera5a84752010-10-05 13:57:27 -05004848 s->flags &= ~SLAB_TRACE;
Christoph Lameterb789ef52011-06-01 12:25:49 -05004849 if (buf[0] == '1') {
4850 s->flags &= ~__CMPXCHG_DOUBLE;
Christoph Lametera5a84752010-10-05 13:57:27 -05004851 s->flags |= SLAB_TRACE;
Christoph Lameterb789ef52011-06-01 12:25:49 -05004852 }
Christoph Lametera5a84752010-10-05 13:57:27 -05004853 return length;
4854}
4855SLAB_ATTR(trace);
4856
Christoph Lameter81819f02007-05-06 14:49:36 -07004857static ssize_t red_zone_show(struct kmem_cache *s, char *buf)
4858{
4859 return sprintf(buf, "%d\n", !!(s->flags & SLAB_RED_ZONE));
4860}
4861
4862static ssize_t red_zone_store(struct kmem_cache *s,
4863 const char *buf, size_t length)
4864{
4865 if (any_slab_objects(s))
4866 return -EBUSY;
4867
4868 s->flags &= ~SLAB_RED_ZONE;
Christoph Lameterb789ef52011-06-01 12:25:49 -05004869 if (buf[0] == '1') {
4870 s->flags &= ~__CMPXCHG_DOUBLE;
Christoph Lameter81819f02007-05-06 14:49:36 -07004871 s->flags |= SLAB_RED_ZONE;
Christoph Lameterb789ef52011-06-01 12:25:49 -05004872 }
Christoph Lameter06b285d2008-04-14 19:11:41 +03004873 calculate_sizes(s, -1);
Christoph Lameter81819f02007-05-06 14:49:36 -07004874 return length;
4875}
4876SLAB_ATTR(red_zone);
4877
4878static ssize_t poison_show(struct kmem_cache *s, char *buf)
4879{
4880 return sprintf(buf, "%d\n", !!(s->flags & SLAB_POISON));
4881}
4882
4883static ssize_t poison_store(struct kmem_cache *s,
4884 const char *buf, size_t length)
4885{
4886 if (any_slab_objects(s))
4887 return -EBUSY;
4888
4889 s->flags &= ~SLAB_POISON;
Christoph Lameterb789ef52011-06-01 12:25:49 -05004890 if (buf[0] == '1') {
4891 s->flags &= ~__CMPXCHG_DOUBLE;
Christoph Lameter81819f02007-05-06 14:49:36 -07004892 s->flags |= SLAB_POISON;
Christoph Lameterb789ef52011-06-01 12:25:49 -05004893 }
Christoph Lameter06b285d2008-04-14 19:11:41 +03004894 calculate_sizes(s, -1);
Christoph Lameter81819f02007-05-06 14:49:36 -07004895 return length;
4896}
4897SLAB_ATTR(poison);
4898
4899static ssize_t store_user_show(struct kmem_cache *s, char *buf)
4900{
4901 return sprintf(buf, "%d\n", !!(s->flags & SLAB_STORE_USER));
4902}
4903
4904static ssize_t store_user_store(struct kmem_cache *s,
4905 const char *buf, size_t length)
4906{
4907 if (any_slab_objects(s))
4908 return -EBUSY;
4909
4910 s->flags &= ~SLAB_STORE_USER;
Christoph Lameterb789ef52011-06-01 12:25:49 -05004911 if (buf[0] == '1') {
4912 s->flags &= ~__CMPXCHG_DOUBLE;
Christoph Lameter81819f02007-05-06 14:49:36 -07004913 s->flags |= SLAB_STORE_USER;
Christoph Lameterb789ef52011-06-01 12:25:49 -05004914 }
Christoph Lameter06b285d2008-04-14 19:11:41 +03004915 calculate_sizes(s, -1);
Christoph Lameter81819f02007-05-06 14:49:36 -07004916 return length;
4917}
4918SLAB_ATTR(store_user);
4919
Christoph Lameter53e15af2007-05-06 14:49:43 -07004920static ssize_t validate_show(struct kmem_cache *s, char *buf)
4921{
4922 return 0;
4923}
4924
4925static ssize_t validate_store(struct kmem_cache *s,
4926 const char *buf, size_t length)
4927{
Christoph Lameter434e2452007-07-17 04:03:30 -07004928 int ret = -EINVAL;
4929
4930 if (buf[0] == '1') {
4931 ret = validate_slab_cache(s);
4932 if (ret >= 0)
4933 ret = length;
4934 }
4935 return ret;
Christoph Lameter53e15af2007-05-06 14:49:43 -07004936}
4937SLAB_ATTR(validate);
Christoph Lametera5a84752010-10-05 13:57:27 -05004938
4939static ssize_t alloc_calls_show(struct kmem_cache *s, char *buf)
4940{
4941 if (!(s->flags & SLAB_STORE_USER))
4942 return -ENOSYS;
4943 return list_locations(s, buf, TRACK_ALLOC);
4944}
4945SLAB_ATTR_RO(alloc_calls);
4946
4947static ssize_t free_calls_show(struct kmem_cache *s, char *buf)
4948{
4949 if (!(s->flags & SLAB_STORE_USER))
4950 return -ENOSYS;
4951 return list_locations(s, buf, TRACK_FREE);
4952}
4953SLAB_ATTR_RO(free_calls);
4954#endif /* CONFIG_SLUB_DEBUG */
4955
4956#ifdef CONFIG_FAILSLAB
4957static ssize_t failslab_show(struct kmem_cache *s, char *buf)
4958{
4959 return sprintf(buf, "%d\n", !!(s->flags & SLAB_FAILSLAB));
4960}
4961
4962static ssize_t failslab_store(struct kmem_cache *s, const char *buf,
4963 size_t length)
4964{
Christoph Lameterc9e16132014-10-09 15:26:11 -07004965 if (s->refcount > 1)
4966 return -EINVAL;
4967
Christoph Lametera5a84752010-10-05 13:57:27 -05004968 s->flags &= ~SLAB_FAILSLAB;
4969 if (buf[0] == '1')
4970 s->flags |= SLAB_FAILSLAB;
4971 return length;
4972}
4973SLAB_ATTR(failslab);
Christoph Lameterab4d5ed2010-10-05 13:57:26 -05004974#endif
Christoph Lameter53e15af2007-05-06 14:49:43 -07004975
Christoph Lameter2086d262007-05-06 14:49:46 -07004976static ssize_t shrink_show(struct kmem_cache *s, char *buf)
4977{
4978 return 0;
4979}
4980
4981static ssize_t shrink_store(struct kmem_cache *s,
4982 const char *buf, size_t length)
4983{
Vladimir Davydov832f37f2015-02-12 14:59:41 -08004984 if (buf[0] == '1')
4985 kmem_cache_shrink(s);
4986 else
Christoph Lameter2086d262007-05-06 14:49:46 -07004987 return -EINVAL;
4988 return length;
4989}
4990SLAB_ATTR(shrink);
4991
Christoph Lameter81819f02007-05-06 14:49:36 -07004992#ifdef CONFIG_NUMA
Christoph Lameter98246012008-01-07 23:20:26 -08004993static ssize_t remote_node_defrag_ratio_show(struct kmem_cache *s, char *buf)
Christoph Lameter81819f02007-05-06 14:49:36 -07004994{
Christoph Lameter98246012008-01-07 23:20:26 -08004995 return sprintf(buf, "%d\n", s->remote_node_defrag_ratio / 10);
Christoph Lameter81819f02007-05-06 14:49:36 -07004996}
4997
Christoph Lameter98246012008-01-07 23:20:26 -08004998static ssize_t remote_node_defrag_ratio_store(struct kmem_cache *s,
Christoph Lameter81819f02007-05-06 14:49:36 -07004999 const char *buf, size_t length)
5000{
Christoph Lameter0121c6192008-04-29 16:11:12 -07005001 unsigned long ratio;
5002 int err;
Christoph Lameter81819f02007-05-06 14:49:36 -07005003
Jingoo Han3dbb95f2013-09-11 14:20:25 -07005004 err = kstrtoul(buf, 10, &ratio);
Christoph Lameter0121c6192008-04-29 16:11:12 -07005005 if (err)
5006 return err;
5007
Christoph Lametere2cb96b2008-08-19 08:51:22 -05005008 if (ratio <= 100)
Christoph Lameter0121c6192008-04-29 16:11:12 -07005009 s->remote_node_defrag_ratio = ratio * 10;
5010
Christoph Lameter81819f02007-05-06 14:49:36 -07005011 return length;
5012}
Christoph Lameter98246012008-01-07 23:20:26 -08005013SLAB_ATTR(remote_node_defrag_ratio);
Christoph Lameter81819f02007-05-06 14:49:36 -07005014#endif
5015
Christoph Lameter8ff12cf2008-02-07 17:47:41 -08005016#ifdef CONFIG_SLUB_STATS
Christoph Lameter8ff12cf2008-02-07 17:47:41 -08005017static int show_stat(struct kmem_cache *s, char *buf, enum stat_item si)
5018{
5019 unsigned long sum = 0;
5020 int cpu;
5021 int len;
5022 int *data = kmalloc(nr_cpu_ids * sizeof(int), GFP_KERNEL);
5023
5024 if (!data)
5025 return -ENOMEM;
5026
5027 for_each_online_cpu(cpu) {
Christoph Lameter9dfc6e62009-12-18 16:26:20 -06005028 unsigned x = per_cpu_ptr(s->cpu_slab, cpu)->stat[si];
Christoph Lameter8ff12cf2008-02-07 17:47:41 -08005029
5030 data[cpu] = x;
5031 sum += x;
5032 }
5033
5034 len = sprintf(buf, "%lu", sum);
5035
Christoph Lameter50ef37b2008-04-14 18:52:05 +03005036#ifdef CONFIG_SMP
Christoph Lameter8ff12cf2008-02-07 17:47:41 -08005037 for_each_online_cpu(cpu) {
5038 if (data[cpu] && len < PAGE_SIZE - 20)
Christoph Lameter50ef37b2008-04-14 18:52:05 +03005039 len += sprintf(buf + len, " C%d=%u", cpu, data[cpu]);
Christoph Lameter8ff12cf2008-02-07 17:47:41 -08005040 }
Christoph Lameter50ef37b2008-04-14 18:52:05 +03005041#endif
Christoph Lameter8ff12cf2008-02-07 17:47:41 -08005042 kfree(data);
5043 return len + sprintf(buf + len, "\n");
5044}
5045
David Rientjes78eb00c2009-10-15 02:20:22 -07005046static void clear_stat(struct kmem_cache *s, enum stat_item si)
5047{
5048 int cpu;
5049
5050 for_each_online_cpu(cpu)
Christoph Lameter9dfc6e62009-12-18 16:26:20 -06005051 per_cpu_ptr(s->cpu_slab, cpu)->stat[si] = 0;
David Rientjes78eb00c2009-10-15 02:20:22 -07005052}
5053
Christoph Lameter8ff12cf2008-02-07 17:47:41 -08005054#define STAT_ATTR(si, text) \
5055static ssize_t text##_show(struct kmem_cache *s, char *buf) \
5056{ \
5057 return show_stat(s, buf, si); \
5058} \
David Rientjes78eb00c2009-10-15 02:20:22 -07005059static ssize_t text##_store(struct kmem_cache *s, \
5060 const char *buf, size_t length) \
5061{ \
5062 if (buf[0] != '0') \
5063 return -EINVAL; \
5064 clear_stat(s, si); \
5065 return length; \
5066} \
5067SLAB_ATTR(text); \
Christoph Lameter8ff12cf2008-02-07 17:47:41 -08005068
5069STAT_ATTR(ALLOC_FASTPATH, alloc_fastpath);
5070STAT_ATTR(ALLOC_SLOWPATH, alloc_slowpath);
5071STAT_ATTR(FREE_FASTPATH, free_fastpath);
5072STAT_ATTR(FREE_SLOWPATH, free_slowpath);
5073STAT_ATTR(FREE_FROZEN, free_frozen);
5074STAT_ATTR(FREE_ADD_PARTIAL, free_add_partial);
5075STAT_ATTR(FREE_REMOVE_PARTIAL, free_remove_partial);
5076STAT_ATTR(ALLOC_FROM_PARTIAL, alloc_from_partial);
5077STAT_ATTR(ALLOC_SLAB, alloc_slab);
5078STAT_ATTR(ALLOC_REFILL, alloc_refill);
Christoph Lametere36a2652011-06-01 12:25:57 -05005079STAT_ATTR(ALLOC_NODE_MISMATCH, alloc_node_mismatch);
Christoph Lameter8ff12cf2008-02-07 17:47:41 -08005080STAT_ATTR(FREE_SLAB, free_slab);
5081STAT_ATTR(CPUSLAB_FLUSH, cpuslab_flush);
5082STAT_ATTR(DEACTIVATE_FULL, deactivate_full);
5083STAT_ATTR(DEACTIVATE_EMPTY, deactivate_empty);
5084STAT_ATTR(DEACTIVATE_TO_HEAD, deactivate_to_head);
5085STAT_ATTR(DEACTIVATE_TO_TAIL, deactivate_to_tail);
5086STAT_ATTR(DEACTIVATE_REMOTE_FREES, deactivate_remote_frees);
Christoph Lameter03e404a2011-06-01 12:25:58 -05005087STAT_ATTR(DEACTIVATE_BYPASS, deactivate_bypass);
Christoph Lameter65c33762008-04-14 19:11:40 +03005088STAT_ATTR(ORDER_FALLBACK, order_fallback);
Christoph Lameterb789ef52011-06-01 12:25:49 -05005089STAT_ATTR(CMPXCHG_DOUBLE_CPU_FAIL, cmpxchg_double_cpu_fail);
5090STAT_ATTR(CMPXCHG_DOUBLE_FAIL, cmpxchg_double_fail);
Christoph Lameter49e22582011-08-09 16:12:27 -05005091STAT_ATTR(CPU_PARTIAL_ALLOC, cpu_partial_alloc);
5092STAT_ATTR(CPU_PARTIAL_FREE, cpu_partial_free);
Alex Shi8028dce2012-02-03 23:34:56 +08005093STAT_ATTR(CPU_PARTIAL_NODE, cpu_partial_node);
5094STAT_ATTR(CPU_PARTIAL_DRAIN, cpu_partial_drain);
Christoph Lameter8ff12cf2008-02-07 17:47:41 -08005095#endif
5096
Pekka Enberg06428782008-01-07 23:20:27 -08005097static struct attribute *slab_attrs[] = {
Christoph Lameter81819f02007-05-06 14:49:36 -07005098 &slab_size_attr.attr,
5099 &object_size_attr.attr,
5100 &objs_per_slab_attr.attr,
5101 &order_attr.attr,
David Rientjes73d342b2009-02-22 17:40:09 -08005102 &min_partial_attr.attr,
Christoph Lameter49e22582011-08-09 16:12:27 -05005103 &cpu_partial_attr.attr,
Christoph Lameter81819f02007-05-06 14:49:36 -07005104 &objects_attr.attr,
Christoph Lameter205ab992008-04-14 19:11:40 +03005105 &objects_partial_attr.attr,
Christoph Lameter81819f02007-05-06 14:49:36 -07005106 &partial_attr.attr,
5107 &cpu_slabs_attr.attr,
5108 &ctor_attr.attr,
Christoph Lameter81819f02007-05-06 14:49:36 -07005109 &aliases_attr.attr,
5110 &align_attr.attr,
Christoph Lameter81819f02007-05-06 14:49:36 -07005111 &hwcache_align_attr.attr,
5112 &reclaim_account_attr.attr,
5113 &destroy_by_rcu_attr.attr,
Christoph Lametera5a84752010-10-05 13:57:27 -05005114 &shrink_attr.attr,
Lai Jiangshanab9a0f12011-03-10 15:21:48 +08005115 &reserved_attr.attr,
Christoph Lameter49e22582011-08-09 16:12:27 -05005116 &slabs_cpu_partial_attr.attr,
Christoph Lameterab4d5ed2010-10-05 13:57:26 -05005117#ifdef CONFIG_SLUB_DEBUG
Christoph Lametera5a84752010-10-05 13:57:27 -05005118 &total_objects_attr.attr,
5119 &slabs_attr.attr,
5120 &sanity_checks_attr.attr,
5121 &trace_attr.attr,
Christoph Lameter81819f02007-05-06 14:49:36 -07005122 &red_zone_attr.attr,
5123 &poison_attr.attr,
5124 &store_user_attr.attr,
Christoph Lameter53e15af2007-05-06 14:49:43 -07005125 &validate_attr.attr,
Christoph Lameter88a420e2007-05-06 14:49:45 -07005126 &alloc_calls_attr.attr,
5127 &free_calls_attr.attr,
Christoph Lameterab4d5ed2010-10-05 13:57:26 -05005128#endif
Christoph Lameter81819f02007-05-06 14:49:36 -07005129#ifdef CONFIG_ZONE_DMA
5130 &cache_dma_attr.attr,
5131#endif
5132#ifdef CONFIG_NUMA
Christoph Lameter98246012008-01-07 23:20:26 -08005133 &remote_node_defrag_ratio_attr.attr,
Christoph Lameter81819f02007-05-06 14:49:36 -07005134#endif
Christoph Lameter8ff12cf2008-02-07 17:47:41 -08005135#ifdef CONFIG_SLUB_STATS
5136 &alloc_fastpath_attr.attr,
5137 &alloc_slowpath_attr.attr,
5138 &free_fastpath_attr.attr,
5139 &free_slowpath_attr.attr,
5140 &free_frozen_attr.attr,
5141 &free_add_partial_attr.attr,
5142 &free_remove_partial_attr.attr,
5143 &alloc_from_partial_attr.attr,
5144 &alloc_slab_attr.attr,
5145 &alloc_refill_attr.attr,
Christoph Lametere36a2652011-06-01 12:25:57 -05005146 &alloc_node_mismatch_attr.attr,
Christoph Lameter8ff12cf2008-02-07 17:47:41 -08005147 &free_slab_attr.attr,
5148 &cpuslab_flush_attr.attr,
5149 &deactivate_full_attr.attr,
5150 &deactivate_empty_attr.attr,
5151 &deactivate_to_head_attr.attr,
5152 &deactivate_to_tail_attr.attr,
5153 &deactivate_remote_frees_attr.attr,
Christoph Lameter03e404a2011-06-01 12:25:58 -05005154 &deactivate_bypass_attr.attr,
Christoph Lameter65c33762008-04-14 19:11:40 +03005155 &order_fallback_attr.attr,
Christoph Lameterb789ef52011-06-01 12:25:49 -05005156 &cmpxchg_double_fail_attr.attr,
5157 &cmpxchg_double_cpu_fail_attr.attr,
Christoph Lameter49e22582011-08-09 16:12:27 -05005158 &cpu_partial_alloc_attr.attr,
5159 &cpu_partial_free_attr.attr,
Alex Shi8028dce2012-02-03 23:34:56 +08005160 &cpu_partial_node_attr.attr,
5161 &cpu_partial_drain_attr.attr,
Christoph Lameter8ff12cf2008-02-07 17:47:41 -08005162#endif
Dmitry Monakhov4c13dd32010-02-26 09:36:12 +03005163#ifdef CONFIG_FAILSLAB
5164 &failslab_attr.attr,
5165#endif
5166
Christoph Lameter81819f02007-05-06 14:49:36 -07005167 NULL
5168};
5169
5170static struct attribute_group slab_attr_group = {
5171 .attrs = slab_attrs,
5172};
5173
5174static ssize_t slab_attr_show(struct kobject *kobj,
5175 struct attribute *attr,
5176 char *buf)
5177{
5178 struct slab_attribute *attribute;
5179 struct kmem_cache *s;
5180 int err;
5181
5182 attribute = to_slab_attr(attr);
5183 s = to_slab(kobj);
5184
5185 if (!attribute->show)
5186 return -EIO;
5187
5188 err = attribute->show(s, buf);
5189
5190 return err;
5191}
5192
5193static ssize_t slab_attr_store(struct kobject *kobj,
5194 struct attribute *attr,
5195 const char *buf, size_t len)
5196{
5197 struct slab_attribute *attribute;
5198 struct kmem_cache *s;
5199 int err;
5200
5201 attribute = to_slab_attr(attr);
5202 s = to_slab(kobj);
5203
5204 if (!attribute->store)
5205 return -EIO;
5206
5207 err = attribute->store(s, buf, len);
Glauber Costa107dab52012-12-18 14:23:05 -08005208#ifdef CONFIG_MEMCG_KMEM
5209 if (slab_state >= FULL && err >= 0 && is_root_cache(s)) {
Vladimir Davydov426589f2015-02-12 14:59:23 -08005210 struct kmem_cache *c;
Christoph Lameter81819f02007-05-06 14:49:36 -07005211
Glauber Costa107dab52012-12-18 14:23:05 -08005212 mutex_lock(&slab_mutex);
5213 if (s->max_attr_size < len)
5214 s->max_attr_size = len;
5215
Glauber Costaebe945c2012-12-18 14:23:10 -08005216 /*
5217 * This is a best effort propagation, so this function's return
5218 * value will be determined by the parent cache only. This is
5219 * basically because not all attributes will have a well
5220 * defined semantics for rollbacks - most of the actions will
5221 * have permanent effects.
5222 *
5223 * Returning the error value of any of the children that fail
5224 * is not 100 % defined, in the sense that users seeing the
5225 * error code won't be able to know anything about the state of
5226 * the cache.
5227 *
5228 * Only returning the error code for the parent cache at least
5229 * has well defined semantics. The cache being written to
5230 * directly either failed or succeeded, in which case we loop
5231 * through the descendants with best-effort propagation.
5232 */
Vladimir Davydov426589f2015-02-12 14:59:23 -08005233 for_each_memcg_cache(c, s)
5234 attribute->store(c, buf, len);
Glauber Costa107dab52012-12-18 14:23:05 -08005235 mutex_unlock(&slab_mutex);
5236 }
5237#endif
Christoph Lameter81819f02007-05-06 14:49:36 -07005238 return err;
5239}
5240
Glauber Costa107dab52012-12-18 14:23:05 -08005241static void memcg_propagate_slab_attrs(struct kmem_cache *s)
5242{
5243#ifdef CONFIG_MEMCG_KMEM
5244 int i;
5245 char *buffer = NULL;
Vladimir Davydov93030d82014-05-06 12:49:59 -07005246 struct kmem_cache *root_cache;
Glauber Costa107dab52012-12-18 14:23:05 -08005247
Vladimir Davydov93030d82014-05-06 12:49:59 -07005248 if (is_root_cache(s))
Glauber Costa107dab52012-12-18 14:23:05 -08005249 return;
5250
Vladimir Davydovf7ce3192015-02-12 14:59:20 -08005251 root_cache = s->memcg_params.root_cache;
Vladimir Davydov93030d82014-05-06 12:49:59 -07005252
Glauber Costa107dab52012-12-18 14:23:05 -08005253 /*
5254 * This mean this cache had no attribute written. Therefore, no point
5255 * in copying default values around
5256 */
Vladimir Davydov93030d82014-05-06 12:49:59 -07005257 if (!root_cache->max_attr_size)
Glauber Costa107dab52012-12-18 14:23:05 -08005258 return;
5259
5260 for (i = 0; i < ARRAY_SIZE(slab_attrs); i++) {
5261 char mbuf[64];
5262 char *buf;
5263 struct slab_attribute *attr = to_slab_attr(slab_attrs[i]);
5264
5265 if (!attr || !attr->store || !attr->show)
5266 continue;
5267
5268 /*
5269 * It is really bad that we have to allocate here, so we will
5270 * do it only as a fallback. If we actually allocate, though,
5271 * we can just use the allocated buffer until the end.
5272 *
5273 * Most of the slub attributes will tend to be very small in
5274 * size, but sysfs allows buffers up to a page, so they can
5275 * theoretically happen.
5276 */
5277 if (buffer)
5278 buf = buffer;
Vladimir Davydov93030d82014-05-06 12:49:59 -07005279 else if (root_cache->max_attr_size < ARRAY_SIZE(mbuf))
Glauber Costa107dab52012-12-18 14:23:05 -08005280 buf = mbuf;
5281 else {
5282 buffer = (char *) get_zeroed_page(GFP_KERNEL);
5283 if (WARN_ON(!buffer))
5284 continue;
5285 buf = buffer;
5286 }
5287
Vladimir Davydov93030d82014-05-06 12:49:59 -07005288 attr->show(root_cache, buf);
Glauber Costa107dab52012-12-18 14:23:05 -08005289 attr->store(s, buf, strlen(buf));
5290 }
5291
5292 if (buffer)
5293 free_page((unsigned long)buffer);
5294#endif
5295}
5296
Christoph Lameter41a21282014-05-06 12:50:08 -07005297static void kmem_cache_release(struct kobject *k)
5298{
5299 slab_kmem_cache_release(to_slab(k));
5300}
5301
Emese Revfy52cf25d2010-01-19 02:58:23 +01005302static const struct sysfs_ops slab_sysfs_ops = {
Christoph Lameter81819f02007-05-06 14:49:36 -07005303 .show = slab_attr_show,
5304 .store = slab_attr_store,
5305};
5306
5307static struct kobj_type slab_ktype = {
5308 .sysfs_ops = &slab_sysfs_ops,
Christoph Lameter41a21282014-05-06 12:50:08 -07005309 .release = kmem_cache_release,
Christoph Lameter81819f02007-05-06 14:49:36 -07005310};
5311
5312static int uevent_filter(struct kset *kset, struct kobject *kobj)
5313{
5314 struct kobj_type *ktype = get_ktype(kobj);
5315
5316 if (ktype == &slab_ktype)
5317 return 1;
5318 return 0;
5319}
5320
Emese Revfy9cd43612009-12-31 14:52:51 +01005321static const struct kset_uevent_ops slab_uevent_ops = {
Christoph Lameter81819f02007-05-06 14:49:36 -07005322 .filter = uevent_filter,
5323};
5324
Greg Kroah-Hartman27c3a312007-11-01 09:29:06 -06005325static struct kset *slab_kset;
Christoph Lameter81819f02007-05-06 14:49:36 -07005326
Vladimir Davydov9a417072014-04-07 15:39:31 -07005327static inline struct kset *cache_kset(struct kmem_cache *s)
5328{
5329#ifdef CONFIG_MEMCG_KMEM
5330 if (!is_root_cache(s))
Vladimir Davydovf7ce3192015-02-12 14:59:20 -08005331 return s->memcg_params.root_cache->memcg_kset;
Vladimir Davydov9a417072014-04-07 15:39:31 -07005332#endif
5333 return slab_kset;
5334}
5335
Christoph Lameter81819f02007-05-06 14:49:36 -07005336#define ID_STR_LENGTH 64
5337
5338/* Create a unique string id for a slab cache:
Christoph Lameter6446faa2008-02-15 23:45:26 -08005339 *
5340 * Format :[flags-]size
Christoph Lameter81819f02007-05-06 14:49:36 -07005341 */
5342static char *create_unique_id(struct kmem_cache *s)
5343{
5344 char *name = kmalloc(ID_STR_LENGTH, GFP_KERNEL);
5345 char *p = name;
5346
5347 BUG_ON(!name);
5348
5349 *p++ = ':';
5350 /*
5351 * First flags affecting slabcache operations. We will only
5352 * get here for aliasable slabs so we do not need to support
5353 * too many flags. The flags here must cover all flags that
5354 * are matched during merging to guarantee that the id is
5355 * unique.
5356 */
5357 if (s->flags & SLAB_CACHE_DMA)
5358 *p++ = 'd';
5359 if (s->flags & SLAB_RECLAIM_ACCOUNT)
5360 *p++ = 'a';
5361 if (s->flags & SLAB_DEBUG_FREE)
5362 *p++ = 'F';
Vegard Nossum5a896d92008-04-04 00:54:48 +02005363 if (!(s->flags & SLAB_NOTRACK))
5364 *p++ = 't';
Christoph Lameter81819f02007-05-06 14:49:36 -07005365 if (p != name + 1)
5366 *p++ = '-';
5367 p += sprintf(p, "%07d", s->size);
Glauber Costa2633d7a2012-12-18 14:22:34 -08005368
Christoph Lameter81819f02007-05-06 14:49:36 -07005369 BUG_ON(p > name + ID_STR_LENGTH - 1);
5370 return name;
5371}
5372
5373static int sysfs_slab_add(struct kmem_cache *s)
5374{
5375 int err;
5376 const char *name;
Christoph Lameter45530c42012-11-28 16:23:07 +00005377 int unmergeable = slab_unmergeable(s);
Christoph Lameter81819f02007-05-06 14:49:36 -07005378
Christoph Lameter81819f02007-05-06 14:49:36 -07005379 if (unmergeable) {
5380 /*
5381 * Slabcache can never be merged so we can use the name proper.
5382 * This is typically the case for debug situations. In that
5383 * case we can catch duplicate names easily.
5384 */
Greg Kroah-Hartman27c3a312007-11-01 09:29:06 -06005385 sysfs_remove_link(&slab_kset->kobj, s->name);
Christoph Lameter81819f02007-05-06 14:49:36 -07005386 name = s->name;
5387 } else {
5388 /*
5389 * Create a unique name for the slab as a target
5390 * for the symlinks.
5391 */
5392 name = create_unique_id(s);
5393 }
5394
Vladimir Davydov9a417072014-04-07 15:39:31 -07005395 s->kobj.kset = cache_kset(s);
Tetsuo Handa26e4f202014-01-04 16:32:31 +09005396 err = kobject_init_and_add(&s->kobj, &slab_ktype, NULL, "%s", name);
Dave Jones54b6a732014-04-07 15:39:32 -07005397 if (err)
Konstantin Khlebnikov80da0262015-09-04 15:45:51 -07005398 goto out;
Christoph Lameter81819f02007-05-06 14:49:36 -07005399
5400 err = sysfs_create_group(&s->kobj, &slab_attr_group);
Dave Jones54b6a732014-04-07 15:39:32 -07005401 if (err)
5402 goto out_del_kobj;
Vladimir Davydov9a417072014-04-07 15:39:31 -07005403
5404#ifdef CONFIG_MEMCG_KMEM
5405 if (is_root_cache(s)) {
5406 s->memcg_kset = kset_create_and_add("cgroup", NULL, &s->kobj);
5407 if (!s->memcg_kset) {
Dave Jones54b6a732014-04-07 15:39:32 -07005408 err = -ENOMEM;
5409 goto out_del_kobj;
Vladimir Davydov9a417072014-04-07 15:39:31 -07005410 }
5411 }
5412#endif
5413
Christoph Lameter81819f02007-05-06 14:49:36 -07005414 kobject_uevent(&s->kobj, KOBJ_ADD);
5415 if (!unmergeable) {
5416 /* Setup first alias */
5417 sysfs_slab_alias(s, s->name);
Christoph Lameter81819f02007-05-06 14:49:36 -07005418 }
Dave Jones54b6a732014-04-07 15:39:32 -07005419out:
5420 if (!unmergeable)
5421 kfree(name);
5422 return err;
5423out_del_kobj:
5424 kobject_del(&s->kobj);
Dave Jones54b6a732014-04-07 15:39:32 -07005425 goto out;
Christoph Lameter81819f02007-05-06 14:49:36 -07005426}
5427
Christoph Lameter41a21282014-05-06 12:50:08 -07005428void sysfs_slab_remove(struct kmem_cache *s)
Christoph Lameter81819f02007-05-06 14:49:36 -07005429{
Christoph Lameter97d06602012-07-06 15:25:11 -05005430 if (slab_state < FULL)
Christoph Lameter2bce6482010-07-19 11:39:11 -05005431 /*
5432 * Sysfs has not been setup yet so no need to remove the
5433 * cache from sysfs.
5434 */
5435 return;
5436
Vladimir Davydov9a417072014-04-07 15:39:31 -07005437#ifdef CONFIG_MEMCG_KMEM
5438 kset_unregister(s->memcg_kset);
5439#endif
Christoph Lameter81819f02007-05-06 14:49:36 -07005440 kobject_uevent(&s->kobj, KOBJ_REMOVE);
5441 kobject_del(&s->kobj);
Christoph Lameter151c6022008-01-07 22:29:05 -08005442 kobject_put(&s->kobj);
Christoph Lameter81819f02007-05-06 14:49:36 -07005443}
5444
5445/*
5446 * Need to buffer aliases during bootup until sysfs becomes
Nick Andrew9f6c708e2008-12-05 14:08:08 +11005447 * available lest we lose that information.
Christoph Lameter81819f02007-05-06 14:49:36 -07005448 */
5449struct saved_alias {
5450 struct kmem_cache *s;
5451 const char *name;
5452 struct saved_alias *next;
5453};
5454
Adrian Bunk5af328a2007-07-17 04:03:27 -07005455static struct saved_alias *alias_list;
Christoph Lameter81819f02007-05-06 14:49:36 -07005456
5457static int sysfs_slab_alias(struct kmem_cache *s, const char *name)
5458{
5459 struct saved_alias *al;
5460
Christoph Lameter97d06602012-07-06 15:25:11 -05005461 if (slab_state == FULL) {
Christoph Lameter81819f02007-05-06 14:49:36 -07005462 /*
5463 * If we have a leftover link then remove it.
5464 */
Greg Kroah-Hartman27c3a312007-11-01 09:29:06 -06005465 sysfs_remove_link(&slab_kset->kobj, name);
5466 return sysfs_create_link(&slab_kset->kobj, &s->kobj, name);
Christoph Lameter81819f02007-05-06 14:49:36 -07005467 }
5468
5469 al = kmalloc(sizeof(struct saved_alias), GFP_KERNEL);
5470 if (!al)
5471 return -ENOMEM;
5472
5473 al->s = s;
5474 al->name = name;
5475 al->next = alias_list;
5476 alias_list = al;
5477 return 0;
5478}
5479
5480static int __init slab_sysfs_init(void)
5481{
Christoph Lameter5b95a4ac2007-07-17 04:03:19 -07005482 struct kmem_cache *s;
Christoph Lameter81819f02007-05-06 14:49:36 -07005483 int err;
5484
Christoph Lameter18004c52012-07-06 15:25:12 -05005485 mutex_lock(&slab_mutex);
Christoph Lameter2bce6482010-07-19 11:39:11 -05005486
Greg Kroah-Hartman0ff21e42007-11-06 10:36:58 -08005487 slab_kset = kset_create_and_add("slab", &slab_uevent_ops, kernel_kobj);
Greg Kroah-Hartman27c3a312007-11-01 09:29:06 -06005488 if (!slab_kset) {
Christoph Lameter18004c52012-07-06 15:25:12 -05005489 mutex_unlock(&slab_mutex);
Fabian Frederickf9f58282014-06-04 16:06:34 -07005490 pr_err("Cannot register slab subsystem.\n");
Christoph Lameter81819f02007-05-06 14:49:36 -07005491 return -ENOSYS;
5492 }
5493
Christoph Lameter97d06602012-07-06 15:25:11 -05005494 slab_state = FULL;
Christoph Lameter26a7bd02007-05-09 02:32:39 -07005495
Christoph Lameter5b95a4ac2007-07-17 04:03:19 -07005496 list_for_each_entry(s, &slab_caches, list) {
Christoph Lameter26a7bd02007-05-09 02:32:39 -07005497 err = sysfs_slab_add(s);
Christoph Lameter5d540fb2007-08-30 23:56:26 -07005498 if (err)
Fabian Frederickf9f58282014-06-04 16:06:34 -07005499 pr_err("SLUB: Unable to add boot slab %s to sysfs\n",
5500 s->name);
Christoph Lameter26a7bd02007-05-09 02:32:39 -07005501 }
Christoph Lameter81819f02007-05-06 14:49:36 -07005502
5503 while (alias_list) {
5504 struct saved_alias *al = alias_list;
5505
5506 alias_list = alias_list->next;
5507 err = sysfs_slab_alias(al->s, al->name);
Christoph Lameter5d540fb2007-08-30 23:56:26 -07005508 if (err)
Fabian Frederickf9f58282014-06-04 16:06:34 -07005509 pr_err("SLUB: Unable to add boot slab alias %s to sysfs\n",
5510 al->name);
Christoph Lameter81819f02007-05-06 14:49:36 -07005511 kfree(al);
5512 }
5513
Christoph Lameter18004c52012-07-06 15:25:12 -05005514 mutex_unlock(&slab_mutex);
Christoph Lameter81819f02007-05-06 14:49:36 -07005515 resiliency_test();
5516 return 0;
5517}
5518
5519__initcall(slab_sysfs_init);
Christoph Lameterab4d5ed2010-10-05 13:57:26 -05005520#endif /* CONFIG_SYSFS */
Pekka J Enberg57ed3ed2008-01-01 17:23:28 +01005521
5522/*
5523 * The /proc/slabinfo ABI
5524 */
Linus Torvalds158a9622008-01-02 13:04:48 -08005525#ifdef CONFIG_SLABINFO
Glauber Costa0d7561c2012-10-19 18:20:27 +04005526void get_slabinfo(struct kmem_cache *s, struct slabinfo *sinfo)
Pekka J Enberg57ed3ed2008-01-01 17:23:28 +01005527{
Pekka J Enberg57ed3ed2008-01-01 17:23:28 +01005528 unsigned long nr_slabs = 0;
Christoph Lameter205ab992008-04-14 19:11:40 +03005529 unsigned long nr_objs = 0;
5530 unsigned long nr_free = 0;
Pekka J Enberg57ed3ed2008-01-01 17:23:28 +01005531 int node;
Christoph Lameterfa45dc22014-08-06 16:04:09 -07005532 struct kmem_cache_node *n;
Pekka J Enberg57ed3ed2008-01-01 17:23:28 +01005533
Christoph Lameterfa45dc22014-08-06 16:04:09 -07005534 for_each_kmem_cache_node(s, node, n) {
Wanpeng Lic17fd132013-07-04 08:33:26 +08005535 nr_slabs += node_nr_slabs(n);
5536 nr_objs += node_nr_objs(n);
Christoph Lameter205ab992008-04-14 19:11:40 +03005537 nr_free += count_partial(n, count_free);
Pekka J Enberg57ed3ed2008-01-01 17:23:28 +01005538 }
5539
Glauber Costa0d7561c2012-10-19 18:20:27 +04005540 sinfo->active_objs = nr_objs - nr_free;
5541 sinfo->num_objs = nr_objs;
5542 sinfo->active_slabs = nr_slabs;
5543 sinfo->num_slabs = nr_slabs;
5544 sinfo->objects_per_slab = oo_objects(s->oo);
5545 sinfo->cache_order = oo_order(s->oo);
Pekka J Enberg57ed3ed2008-01-01 17:23:28 +01005546}
5547
Glauber Costa0d7561c2012-10-19 18:20:27 +04005548void slabinfo_show_stats(struct seq_file *m, struct kmem_cache *s)
Alexey Dobriyan7b3c3a52008-10-06 02:42:17 +04005549{
Alexey Dobriyan7b3c3a52008-10-06 02:42:17 +04005550}
5551
Glauber Costab7454ad2012-10-19 18:20:25 +04005552ssize_t slabinfo_write(struct file *file, const char __user *buffer,
5553 size_t count, loff_t *ppos)
Alexey Dobriyan7b3c3a52008-10-06 02:42:17 +04005554{
Glauber Costab7454ad2012-10-19 18:20:25 +04005555 return -EIO;
Alexey Dobriyan7b3c3a52008-10-06 02:42:17 +04005556}
Linus Torvalds158a9622008-01-02 13:04:48 -08005557#endif /* CONFIG_SLABINFO */