blob: 33885118523c7ce695e2c901e42d76250a84a8da [file] [log] [blame]
Christoph Lameter81819f02007-05-06 14:49:36 -07001#ifndef _LINUX_SLUB_DEF_H
2#define _LINUX_SLUB_DEF_H
3
4/*
5 * SLUB : A Slab allocator without object queues.
6 *
Christoph Lametercde53532008-07-04 09:59:22 -07007 * (C) 2007 SGI, Christoph Lameter
Christoph Lameter81819f02007-05-06 14:49:36 -07008 */
Christoph Lameter81819f02007-05-06 14:49:36 -07009#include <linux/kobject.h>
10
Christoph Lameter8ff12cf2008-02-07 17:47:41 -080011enum stat_item {
12 ALLOC_FASTPATH, /* Allocation from cpu slab */
13 ALLOC_SLOWPATH, /* Allocation by getting a new cpu slab */
Zhi Yong Wua941f832013-11-08 20:47:36 +080014 FREE_FASTPATH, /* Free to cpu slab */
Christoph Lameter8ff12cf2008-02-07 17:47:41 -080015 FREE_SLOWPATH, /* Freeing not to cpu slab */
16 FREE_FROZEN, /* Freeing to frozen slab */
17 FREE_ADD_PARTIAL, /* Freeing moves slab to partial list */
18 FREE_REMOVE_PARTIAL, /* Freeing removes last object */
Alex Shi8028dce2012-02-03 23:34:56 +080019 ALLOC_FROM_PARTIAL, /* Cpu slab acquired from node partial list */
Christoph Lameter8ff12cf2008-02-07 17:47:41 -080020 ALLOC_SLAB, /* Cpu slab acquired from page allocator */
21 ALLOC_REFILL, /* Refill cpu slab from slab freelist */
Christoph Lametere36a2652011-06-01 12:25:57 -050022 ALLOC_NODE_MISMATCH, /* Switching cpu slab */
Christoph Lameter8ff12cf2008-02-07 17:47:41 -080023 FREE_SLAB, /* Slab freed to the page allocator */
24 CPUSLAB_FLUSH, /* Abandoning of the cpu slab */
25 DEACTIVATE_FULL, /* Cpu slab was full when deactivated */
26 DEACTIVATE_EMPTY, /* Cpu slab was empty when deactivated */
27 DEACTIVATE_TO_HEAD, /* Cpu slab was moved to the head of partials */
28 DEACTIVATE_TO_TAIL, /* Cpu slab was moved to the tail of partials */
29 DEACTIVATE_REMOTE_FREES,/* Slab contained remotely freed objects */
Christoph Lameter03e404a2011-06-01 12:25:58 -050030 DEACTIVATE_BYPASS, /* Implicit deactivation */
Christoph Lameter65c33762008-04-14 19:11:40 +030031 ORDER_FALLBACK, /* Number of times fallback was necessary */
Christoph Lameter4fdccdf2011-03-22 13:35:00 -050032 CMPXCHG_DOUBLE_CPU_FAIL,/* Failure of this_cpu_cmpxchg_double */
Christoph Lameterb789ef52011-06-01 12:25:49 -050033 CMPXCHG_DOUBLE_FAIL, /* Number of times that cmpxchg double did not match */
Christoph Lameter49e22582011-08-09 16:12:27 -050034 CPU_PARTIAL_ALLOC, /* Used cpu partial on alloc */
Alex Shi8028dce2012-02-03 23:34:56 +080035 CPU_PARTIAL_FREE, /* Refill cpu partial on free */
36 CPU_PARTIAL_NODE, /* Refill cpu partial from node partial */
37 CPU_PARTIAL_DRAIN, /* Drain cpu partial to node partial */
Christoph Lameter8ff12cf2008-02-07 17:47:41 -080038 NR_SLUB_STAT_ITEMS };
39
Christoph Lameterdfb4f092007-10-16 01:26:05 -070040struct kmem_cache_cpu {
Christoph Lameter8a5ec0b2011-02-25 11:38:54 -060041 void **freelist; /* Pointer to next available object */
Christoph Lameter8a5ec0b2011-02-25 11:38:54 -060042 unsigned long tid; /* Globally unique transaction id */
Christoph Lameterda89b792008-01-07 23:20:31 -080043 struct page *page; /* The slab from which we are allocating */
Christoph Lameter49e22582011-08-09 16:12:27 -050044 struct page *partial; /* Partially allocated frozen slabs */
Christoph Lameter8ff12cf2008-02-07 17:47:41 -080045#ifdef CONFIG_SLUB_STATS
46 unsigned stat[NR_SLUB_STAT_ITEMS];
47#endif
Christoph Lameter4c93c3552007-10-16 01:26:08 -070048};
Christoph Lameterdfb4f092007-10-16 01:26:05 -070049
Christoph Lameter81819f02007-05-06 14:49:36 -070050/*
Christoph Lameter834f3d12008-04-14 19:11:31 +030051 * Word size structure that can be atomically updated or read and that
52 * contains both the order and the number of objects that a slab of the
53 * given order would contain.
54 */
55struct kmem_cache_order_objects {
56 unsigned long x;
57};
58
59/*
Christoph Lameter81819f02007-05-06 14:49:36 -070060 * Slab cache management.
61 */
62struct kmem_cache {
Namhyung Kim1b5ad242010-08-07 14:29:22 +020063 struct kmem_cache_cpu __percpu *cpu_slab;
Christoph Lameter81819f02007-05-06 14:49:36 -070064 /* Used for retriving partial slabs etc */
65 unsigned long flags;
Christoph Lameter1a757fe2011-02-25 11:38:51 -060066 unsigned long min_partial;
Christoph Lameter81819f02007-05-06 14:49:36 -070067 int size; /* The size of an object including meta data */
Christoph Lameter3b0efdf2012-06-13 10:24:57 -050068 int object_size; /* The size of an object without meta data */
Christoph Lameter81819f02007-05-06 14:49:36 -070069 int offset; /* Free pointer offset. */
Alex Shi9f264902011-09-01 11:32:18 +080070 int cpu_partial; /* Number of per cpu partial objects to keep around */
Christoph Lameter834f3d12008-04-14 19:11:31 +030071 struct kmem_cache_order_objects oo;
Christoph Lameter81819f02007-05-06 14:49:36 -070072
Christoph Lameter81819f02007-05-06 14:49:36 -070073 /* Allocation and freeing of slabs */
Christoph Lameter205ab992008-04-14 19:11:40 +030074 struct kmem_cache_order_objects max;
Christoph Lameter65c33762008-04-14 19:11:40 +030075 struct kmem_cache_order_objects min;
Christoph Lameterb7a49f02008-02-14 14:21:32 -080076 gfp_t allocflags; /* gfp flags to use on each alloc */
Christoph Lameter81819f02007-05-06 14:49:36 -070077 int refcount; /* Refcount for slab cache destroy */
Alexey Dobriyan51cc5062008-07-25 19:45:34 -070078 void (*ctor)(void *);
Christoph Lameter81819f02007-05-06 14:49:36 -070079 int inuse; /* Offset to metadata */
80 int align; /* Alignment */
Lai Jiangshanab9a0f12011-03-10 15:21:48 +080081 int reserved; /* Reserved bytes at the end of slabs */
Christoph Lameter81819f02007-05-06 14:49:36 -070082 const char *name; /* Name (only for display!) */
83 struct list_head list; /* List of slab caches */
Christoph Lameterab4d5ed2010-10-05 13:57:26 -050084#ifdef CONFIG_SYSFS
Christoph Lameter81819f02007-05-06 14:49:36 -070085 struct kobject kobj; /* For sysfs */
Christoph Lameter0c710012007-07-17 04:03:24 -070086#endif
Glauber Costaba6c4962012-12-18 14:22:27 -080087#ifdef CONFIG_MEMCG_KMEM
Vladimir Davydovf7ce3192015-02-12 14:59:20 -080088 struct memcg_cache_params memcg_params;
Glauber Costa107dab52012-12-18 14:23:05 -080089 int max_attr_size; /* for propagation, maximum size of a stored attr */
Vladimir Davydov9a417072014-04-07 15:39:31 -070090#ifdef CONFIG_SYSFS
91 struct kset *memcg_kset;
92#endif
Glauber Costaba6c4962012-12-18 14:22:27 -080093#endif
Christoph Lameter81819f02007-05-06 14:49:36 -070094
95#ifdef CONFIG_NUMA
Christoph Lameter98246012008-01-07 23:20:26 -080096 /*
97 * Defragmentation by allocating from a remote node.
98 */
99 int remote_node_defrag_ratio;
Christoph Lameter81819f02007-05-06 14:49:36 -0700100#endif
Christoph Lameter7340cc82010-09-28 08:10:26 -0500101 struct kmem_cache_node *node[MAX_NUMNODES];
Christoph Lameter81819f02007-05-06 14:49:36 -0700102};
103
Christoph Lameter41a21282014-05-06 12:50:08 -0700104#ifdef CONFIG_SYSFS
105#define SLAB_SUPPORTS_SYSFS
106void sysfs_slab_remove(struct kmem_cache *);
107#else
108static inline void sysfs_slab_remove(struct kmem_cache *s)
109{
110}
111#endif
112
Andrey Ryabinin912f5fb2015-02-13 14:39:31 -0800113
114/**
115 * virt_to_obj - returns address of the beginning of object.
116 * @s: object's kmem_cache
117 * @slab_page: address of slab page
118 * @x: address within object memory range
119 *
120 * Returns address of the beginning of object
121 */
122static inline void *virt_to_obj(struct kmem_cache *s,
123 const void *slab_page,
124 const void *x)
125{
126 return (void *)x - ((x - slab_page) % s->size);
127}
128
Andrey Ryabinin75c66de2015-02-13 14:39:35 -0800129void object_err(struct kmem_cache *s, struct page *page,
130 u8 *object, char *reason);
131
Christoph Lameter81819f02007-05-06 14:49:36 -0700132#endif /* _LINUX_SLUB_DEF_H */