blob: d00e0bacda936ecc8e5ebaac68e5959b9acca306 [file] [log] [blame]
Christoph Lameter2e892f42006-12-13 00:34:23 -08001#ifndef _LINUX_SLAB_DEF_H
2#define _LINUX_SLAB_DEF_H
3
4/*
5 * Definitions unique to the original Linux SLAB allocator.
6 *
7 * What we provide here is a way to optimize the frequent kmalloc
8 * calls in the kernel by selecting the appropriate general cache
9 * if kmalloc was called with a size that can be established at
10 * compile time.
11 */
12
13#include <linux/init.h>
14#include <asm/page.h> /* kmalloc_sizes.h needs PAGE_SIZE */
15#include <asm/cache.h> /* kmalloc_sizes.h needs L1_CACHE_BYTES */
16#include <linux/compiler.h>
Li Zefan039ca4e2010-05-26 17:22:17 +080017
18#include <trace/events/kmem.h>
Christoph Lameter2e892f42006-12-13 00:34:23 -080019
David Woodhouse1f0ce8b32010-05-19 12:01:42 +010020/*
Pekka Enberg8eae9852008-05-09 20:32:44 +020021 * struct kmem_cache
22 *
23 * manages a cache.
24 */
25
26struct kmem_cache {
Eric Dumazetb56efcf2011-07-20 19:04:23 +020027/* 1) Cache tunables. Protected by cache_chain_mutex */
Pekka Enberg8eae9852008-05-09 20:32:44 +020028 unsigned int batchcount;
29 unsigned int limit;
30 unsigned int shared;
31
32 unsigned int buffer_size;
33 u32 reciprocal_buffer_size;
Eric Dumazetb56efcf2011-07-20 19:04:23 +020034/* 2) touched by every alloc & free from the backend */
Pekka Enberg8eae9852008-05-09 20:32:44 +020035
36 unsigned int flags; /* constant flags */
37 unsigned int num; /* # of objs per slab */
38
Eric Dumazetb56efcf2011-07-20 19:04:23 +020039/* 3) cache_grow/shrink */
Pekka Enberg8eae9852008-05-09 20:32:44 +020040 /* order of pgs per slab (2^n) */
41 unsigned int gfporder;
42
43 /* force GFP flags, e.g. GFP_DMA */
44 gfp_t gfpflags;
45
46 size_t colour; /* cache colouring range */
47 unsigned int colour_off; /* colour offset */
48 struct kmem_cache *slabp_cache;
49 unsigned int slab_size;
50 unsigned int dflags; /* dynamic flags */
51
52 /* constructor func */
53 void (*ctor)(void *obj);
54
Eric Dumazetb56efcf2011-07-20 19:04:23 +020055/* 4) cache creation/removal */
Pekka Enberg8eae9852008-05-09 20:32:44 +020056 const char *name;
57 struct list_head next;
58
Eric Dumazetb56efcf2011-07-20 19:04:23 +020059/* 5) statistics */
Pekka Enberg8eae9852008-05-09 20:32:44 +020060#ifdef CONFIG_DEBUG_SLAB
61 unsigned long num_active;
62 unsigned long num_allocations;
63 unsigned long high_mark;
64 unsigned long grown;
65 unsigned long reaped;
66 unsigned long errors;
67 unsigned long max_freeable;
68 unsigned long node_allocs;
69 unsigned long node_frees;
70 unsigned long node_overflow;
71 atomic_t allochit;
72 atomic_t allocmiss;
73 atomic_t freehit;
74 atomic_t freemiss;
75
76 /*
77 * If debugging is enabled, then the allocator can add additional
78 * fields and/or padding to every object. buffer_size contains the total
79 * object size including these internal fields, the following two
80 * variables contain the offset to the user object and its size.
81 */
82 int obj_offset;
83 int obj_size;
84#endif /* CONFIG_DEBUG_SLAB */
85
Eric Dumazetb56efcf2011-07-20 19:04:23 +020086/* 6) per-cpu/per-node data, touched during every alloc/free */
Pekka Enberg8eae9852008-05-09 20:32:44 +020087 /*
Eric Dumazetb56efcf2011-07-20 19:04:23 +020088 * We put array[] at the end of kmem_cache, because we want to size
89 * this array to nr_cpu_ids slots instead of NR_CPUS
Pekka Enberg8eae9852008-05-09 20:32:44 +020090 * (see kmem_cache_init())
Eric Dumazetb56efcf2011-07-20 19:04:23 +020091 * We still use [NR_CPUS] and not [1] or [0] because cache_cache
92 * is statically defined, so we reserve the max number of cpus.
Pekka Enberg8eae9852008-05-09 20:32:44 +020093 */
Eric Dumazetb56efcf2011-07-20 19:04:23 +020094 struct kmem_list3 **nodelists;
95 struct array_cache *array[NR_CPUS];
Pekka Enberg8eae9852008-05-09 20:32:44 +020096 /*
Eric Dumazetb56efcf2011-07-20 19:04:23 +020097 * Do not add fields after array[]
Pekka Enberg8eae9852008-05-09 20:32:44 +020098 */
99};
100
Christoph Lameter2e892f42006-12-13 00:34:23 -0800101/* Size description struct for general caches. */
102struct cache_sizes {
103 size_t cs_size;
104 struct kmem_cache *cs_cachep;
Christoph Lameter4b51d662007-02-10 01:43:10 -0800105#ifdef CONFIG_ZONE_DMA
Christoph Lameter2e892f42006-12-13 00:34:23 -0800106 struct kmem_cache *cs_dmacachep;
Christoph Lameter4b51d662007-02-10 01:43:10 -0800107#endif
Christoph Lameter2e892f42006-12-13 00:34:23 -0800108};
109extern struct cache_sizes malloc_sizes[];
110
Paul Mundt6193a2f2007-07-15 23:38:22 -0700111void *kmem_cache_alloc(struct kmem_cache *, gfp_t);
112void *__kmalloc(size_t size, gfp_t flags);
113
Li Zefan0f24f122009-12-11 15:45:30 +0800114#ifdef CONFIG_TRACING
Steven Rostedt85beb582010-11-24 16:23:34 -0500115extern void *kmem_cache_alloc_trace(size_t size,
116 struct kmem_cache *cachep, gfp_t flags);
Eduard - Gabriel Munteanu36555752008-08-10 20:14:05 +0300117extern size_t slab_buffer_size(struct kmem_cache *cachep);
118#else
119static __always_inline void *
Steven Rostedt85beb582010-11-24 16:23:34 -0500120kmem_cache_alloc_trace(size_t size, struct kmem_cache *cachep, gfp_t flags)
Christoph Lameter2e892f42006-12-13 00:34:23 -0800121{
Eduard - Gabriel Munteanu36555752008-08-10 20:14:05 +0300122 return kmem_cache_alloc(cachep, flags);
123}
124static inline size_t slab_buffer_size(struct kmem_cache *cachep)
125{
126 return 0;
127}
128#endif
129
130static __always_inline void *kmalloc(size_t size, gfp_t flags)
131{
132 struct kmem_cache *cachep;
133 void *ret;
134
Christoph Lameter2e892f42006-12-13 00:34:23 -0800135 if (__builtin_constant_p(size)) {
136 int i = 0;
Christoph Lameter6cb8f912007-07-17 04:03:22 -0700137
138 if (!size)
139 return ZERO_SIZE_PTR;
140
Christoph Lameter2e892f42006-12-13 00:34:23 -0800141#define CACHE(x) \
142 if (size <= x) \
143 goto found; \
144 else \
145 i++;
Joe Perches1c61fc42008-03-05 13:58:17 -0800146#include <linux/kmalloc_sizes.h>
Christoph Lameter2e892f42006-12-13 00:34:23 -0800147#undef CACHE
Jeff Mahoney1cf3eb22009-01-27 23:48:59 +0200148 return NULL;
Christoph Lameter2e892f42006-12-13 00:34:23 -0800149found:
Christoph Lameter4b51d662007-02-10 01:43:10 -0800150#ifdef CONFIG_ZONE_DMA
151 if (flags & GFP_DMA)
Eduard - Gabriel Munteanu36555752008-08-10 20:14:05 +0300152 cachep = malloc_sizes[i].cs_dmacachep;
153 else
Christoph Lameter4b51d662007-02-10 01:43:10 -0800154#endif
Eduard - Gabriel Munteanu36555752008-08-10 20:14:05 +0300155 cachep = malloc_sizes[i].cs_cachep;
156
Steven Rostedt85beb582010-11-24 16:23:34 -0500157 ret = kmem_cache_alloc_trace(size, cachep, flags);
Eduard - Gabriel Munteanu36555752008-08-10 20:14:05 +0300158
159 return ret;
Christoph Lameter2e892f42006-12-13 00:34:23 -0800160 }
161 return __kmalloc(size, flags);
162}
163
Christoph Lameter2e892f42006-12-13 00:34:23 -0800164#ifdef CONFIG_NUMA
165extern void *__kmalloc_node(size_t size, gfp_t flags, int node);
Paul Mundt6193a2f2007-07-15 23:38:22 -0700166extern void *kmem_cache_alloc_node(struct kmem_cache *, gfp_t flags, int node);
Christoph Lameter2e892f42006-12-13 00:34:23 -0800167
Li Zefan0f24f122009-12-11 15:45:30 +0800168#ifdef CONFIG_TRACING
Steven Rostedt85beb582010-11-24 16:23:34 -0500169extern void *kmem_cache_alloc_node_trace(size_t size,
170 struct kmem_cache *cachep,
171 gfp_t flags,
172 int nodeid);
Eduard - Gabriel Munteanu36555752008-08-10 20:14:05 +0300173#else
174static __always_inline void *
Steven Rostedt85beb582010-11-24 16:23:34 -0500175kmem_cache_alloc_node_trace(size_t size,
176 struct kmem_cache *cachep,
177 gfp_t flags,
178 int nodeid)
Christoph Lameter2e892f42006-12-13 00:34:23 -0800179{
Eduard - Gabriel Munteanu36555752008-08-10 20:14:05 +0300180 return kmem_cache_alloc_node(cachep, flags, nodeid);
181}
182#endif
183
184static __always_inline void *kmalloc_node(size_t size, gfp_t flags, int node)
185{
186 struct kmem_cache *cachep;
Eduard - Gabriel Munteanu36555752008-08-10 20:14:05 +0300187
Christoph Lameter2e892f42006-12-13 00:34:23 -0800188 if (__builtin_constant_p(size)) {
189 int i = 0;
Christoph Lameter6cb8f912007-07-17 04:03:22 -0700190
191 if (!size)
192 return ZERO_SIZE_PTR;
193
Christoph Lameter2e892f42006-12-13 00:34:23 -0800194#define CACHE(x) \
195 if (size <= x) \
196 goto found; \
197 else \
198 i++;
Joe Perches1c61fc42008-03-05 13:58:17 -0800199#include <linux/kmalloc_sizes.h>
Christoph Lameter2e892f42006-12-13 00:34:23 -0800200#undef CACHE
Jeff Mahoney1cf3eb22009-01-27 23:48:59 +0200201 return NULL;
Christoph Lameter2e892f42006-12-13 00:34:23 -0800202found:
Christoph Lameter4b51d662007-02-10 01:43:10 -0800203#ifdef CONFIG_ZONE_DMA
204 if (flags & GFP_DMA)
Eduard - Gabriel Munteanu36555752008-08-10 20:14:05 +0300205 cachep = malloc_sizes[i].cs_dmacachep;
206 else
Christoph Lameter4b51d662007-02-10 01:43:10 -0800207#endif
Eduard - Gabriel Munteanu36555752008-08-10 20:14:05 +0300208 cachep = malloc_sizes[i].cs_cachep;
209
Steven Rostedt85beb582010-11-24 16:23:34 -0500210 return kmem_cache_alloc_node_trace(size, cachep, flags, node);
Christoph Lameter2e892f42006-12-13 00:34:23 -0800211 }
212 return __kmalloc_node(size, flags, node);
213}
214
215#endif /* CONFIG_NUMA */
216
Christoph Lameter2e892f42006-12-13 00:34:23 -0800217#endif /* _LINUX_SLAB_DEF_H */