blob: 850d057500dece3f52497b2bed3aab51dd677852 [file] [log] [blame]
Christoph Lameter2e892f42006-12-13 00:34:23 -08001#ifndef _LINUX_SLAB_DEF_H
2#define _LINUX_SLAB_DEF_H
3
4/*
5 * Definitions unique to the original Linux SLAB allocator.
6 *
7 * What we provide here is a way to optimize the frequent kmalloc
8 * calls in the kernel by selecting the appropriate general cache
9 * if kmalloc was called with a size that can be established at
10 * compile time.
11 */
12
13#include <linux/init.h>
14#include <asm/page.h> /* kmalloc_sizes.h needs PAGE_SIZE */
15#include <asm/cache.h> /* kmalloc_sizes.h needs L1_CACHE_BYTES */
16#include <linux/compiler.h>
Zhaolei02af61b2009-04-10 14:26:18 +080017#include <linux/kmemtrace.h>
Christoph Lameter2e892f42006-12-13 00:34:23 -080018
Pekka Enberg8eae9852008-05-09 20:32:44 +020019/*
20 * struct kmem_cache
21 *
22 * manages a cache.
23 */
24
25struct kmem_cache {
26/* 1) per-cpu data, touched during every alloc/free */
27 struct array_cache *array[NR_CPUS];
28/* 2) Cache tunables. Protected by cache_chain_mutex */
29 unsigned int batchcount;
30 unsigned int limit;
31 unsigned int shared;
32
33 unsigned int buffer_size;
34 u32 reciprocal_buffer_size;
35/* 3) touched by every alloc & free from the backend */
36
37 unsigned int flags; /* constant flags */
38 unsigned int num; /* # of objs per slab */
39
40/* 4) cache_grow/shrink */
41 /* order of pgs per slab (2^n) */
42 unsigned int gfporder;
43
44 /* force GFP flags, e.g. GFP_DMA */
45 gfp_t gfpflags;
46
47 size_t colour; /* cache colouring range */
48 unsigned int colour_off; /* colour offset */
49 struct kmem_cache *slabp_cache;
50 unsigned int slab_size;
51 unsigned int dflags; /* dynamic flags */
52
53 /* constructor func */
54 void (*ctor)(void *obj);
55
56/* 5) cache creation/removal */
57 const char *name;
58 struct list_head next;
59
60/* 6) statistics */
61#ifdef CONFIG_DEBUG_SLAB
62 unsigned long num_active;
63 unsigned long num_allocations;
64 unsigned long high_mark;
65 unsigned long grown;
66 unsigned long reaped;
67 unsigned long errors;
68 unsigned long max_freeable;
69 unsigned long node_allocs;
70 unsigned long node_frees;
71 unsigned long node_overflow;
72 atomic_t allochit;
73 atomic_t allocmiss;
74 atomic_t freehit;
75 atomic_t freemiss;
76
77 /*
78 * If debugging is enabled, then the allocator can add additional
79 * fields and/or padding to every object. buffer_size contains the total
80 * object size including these internal fields, the following two
81 * variables contain the offset to the user object and its size.
82 */
83 int obj_offset;
84 int obj_size;
85#endif /* CONFIG_DEBUG_SLAB */
86
87 /*
88 * We put nodelists[] at the end of kmem_cache, because we want to size
89 * this array to nr_node_ids slots instead of MAX_NUMNODES
90 * (see kmem_cache_init())
91 * We still use [MAX_NUMNODES] and not [1] or [0] because cache_cache
92 * is statically defined, so we reserve the max number of nodes.
93 */
94 struct kmem_list3 *nodelists[MAX_NUMNODES];
95 /*
96 * Do not add fields after nodelists[]
97 */
98};
99
Christoph Lameter2e892f42006-12-13 00:34:23 -0800100/* Size description struct for general caches. */
101struct cache_sizes {
102 size_t cs_size;
103 struct kmem_cache *cs_cachep;
Christoph Lameter4b51d662007-02-10 01:43:10 -0800104#ifdef CONFIG_ZONE_DMA
Christoph Lameter2e892f42006-12-13 00:34:23 -0800105 struct kmem_cache *cs_dmacachep;
Christoph Lameter4b51d662007-02-10 01:43:10 -0800106#endif
Christoph Lameter2e892f42006-12-13 00:34:23 -0800107};
108extern struct cache_sizes malloc_sizes[];
109
Paul Mundt6193a2f2007-07-15 23:38:22 -0700110void *kmem_cache_alloc(struct kmem_cache *, gfp_t);
111void *__kmalloc(size_t size, gfp_t flags);
112
Eduard - Gabriel Munteanu36555752008-08-10 20:14:05 +0300113#ifdef CONFIG_KMEMTRACE
114extern void *kmem_cache_alloc_notrace(struct kmem_cache *cachep, gfp_t flags);
115extern size_t slab_buffer_size(struct kmem_cache *cachep);
116#else
117static __always_inline void *
118kmem_cache_alloc_notrace(struct kmem_cache *cachep, gfp_t flags)
Christoph Lameter2e892f42006-12-13 00:34:23 -0800119{
Eduard - Gabriel Munteanu36555752008-08-10 20:14:05 +0300120 return kmem_cache_alloc(cachep, flags);
121}
122static inline size_t slab_buffer_size(struct kmem_cache *cachep)
123{
124 return 0;
125}
126#endif
127
128static __always_inline void *kmalloc(size_t size, gfp_t flags)
129{
130 struct kmem_cache *cachep;
131 void *ret;
132
Christoph Lameter2e892f42006-12-13 00:34:23 -0800133 if (__builtin_constant_p(size)) {
134 int i = 0;
Christoph Lameter6cb8f912007-07-17 04:03:22 -0700135
136 if (!size)
137 return ZERO_SIZE_PTR;
138
Christoph Lameter2e892f42006-12-13 00:34:23 -0800139#define CACHE(x) \
140 if (size <= x) \
141 goto found; \
142 else \
143 i++;
Joe Perches1c61fc42008-03-05 13:58:17 -0800144#include <linux/kmalloc_sizes.h>
Christoph Lameter2e892f42006-12-13 00:34:23 -0800145#undef CACHE
Jeff Mahoney1cf3eb22009-01-27 23:48:59 +0200146 return NULL;
Christoph Lameter2e892f42006-12-13 00:34:23 -0800147found:
Christoph Lameter4b51d662007-02-10 01:43:10 -0800148#ifdef CONFIG_ZONE_DMA
149 if (flags & GFP_DMA)
Eduard - Gabriel Munteanu36555752008-08-10 20:14:05 +0300150 cachep = malloc_sizes[i].cs_dmacachep;
151 else
Christoph Lameter4b51d662007-02-10 01:43:10 -0800152#endif
Eduard - Gabriel Munteanu36555752008-08-10 20:14:05 +0300153 cachep = malloc_sizes[i].cs_cachep;
154
155 ret = kmem_cache_alloc_notrace(cachep, flags);
156
Eduard - Gabriel Munteanuca2b84cb2009-03-23 15:12:24 +0200157 trace_kmalloc(_THIS_IP_, ret,
158 size, slab_buffer_size(cachep), flags);
Eduard - Gabriel Munteanu36555752008-08-10 20:14:05 +0300159
160 return ret;
Christoph Lameter2e892f42006-12-13 00:34:23 -0800161 }
162 return __kmalloc(size, flags);
163}
164
Christoph Lameter2e892f42006-12-13 00:34:23 -0800165#ifdef CONFIG_NUMA
166extern void *__kmalloc_node(size_t size, gfp_t flags, int node);
Paul Mundt6193a2f2007-07-15 23:38:22 -0700167extern void *kmem_cache_alloc_node(struct kmem_cache *, gfp_t flags, int node);
Christoph Lameter2e892f42006-12-13 00:34:23 -0800168
Eduard - Gabriel Munteanu36555752008-08-10 20:14:05 +0300169#ifdef CONFIG_KMEMTRACE
170extern void *kmem_cache_alloc_node_notrace(struct kmem_cache *cachep,
171 gfp_t flags,
172 int nodeid);
173#else
174static __always_inline void *
175kmem_cache_alloc_node_notrace(struct kmem_cache *cachep,
176 gfp_t flags,
177 int nodeid)
Christoph Lameter2e892f42006-12-13 00:34:23 -0800178{
Eduard - Gabriel Munteanu36555752008-08-10 20:14:05 +0300179 return kmem_cache_alloc_node(cachep, flags, nodeid);
180}
181#endif
182
183static __always_inline void *kmalloc_node(size_t size, gfp_t flags, int node)
184{
185 struct kmem_cache *cachep;
186 void *ret;
187
Christoph Lameter2e892f42006-12-13 00:34:23 -0800188 if (__builtin_constant_p(size)) {
189 int i = 0;
Christoph Lameter6cb8f912007-07-17 04:03:22 -0700190
191 if (!size)
192 return ZERO_SIZE_PTR;
193
Christoph Lameter2e892f42006-12-13 00:34:23 -0800194#define CACHE(x) \
195 if (size <= x) \
196 goto found; \
197 else \
198 i++;
Joe Perches1c61fc42008-03-05 13:58:17 -0800199#include <linux/kmalloc_sizes.h>
Christoph Lameter2e892f42006-12-13 00:34:23 -0800200#undef CACHE
Jeff Mahoney1cf3eb22009-01-27 23:48:59 +0200201 return NULL;
Christoph Lameter2e892f42006-12-13 00:34:23 -0800202found:
Christoph Lameter4b51d662007-02-10 01:43:10 -0800203#ifdef CONFIG_ZONE_DMA
204 if (flags & GFP_DMA)
Eduard - Gabriel Munteanu36555752008-08-10 20:14:05 +0300205 cachep = malloc_sizes[i].cs_dmacachep;
206 else
Christoph Lameter4b51d662007-02-10 01:43:10 -0800207#endif
Eduard - Gabriel Munteanu36555752008-08-10 20:14:05 +0300208 cachep = malloc_sizes[i].cs_cachep;
209
210 ret = kmem_cache_alloc_node_notrace(cachep, flags, node);
211
Eduard - Gabriel Munteanuca2b84cb2009-03-23 15:12:24 +0200212 trace_kmalloc_node(_THIS_IP_, ret,
213 size, slab_buffer_size(cachep),
214 flags, node);
Eduard - Gabriel Munteanu36555752008-08-10 20:14:05 +0300215
216 return ret;
Christoph Lameter2e892f42006-12-13 00:34:23 -0800217 }
218 return __kmalloc_node(size, flags, node);
219}
220
221#endif /* CONFIG_NUMA */
222
Christoph Lameter2e892f42006-12-13 00:34:23 -0800223#endif /* _LINUX_SLAB_DEF_H */