Christoph Lameter | 039363f | 2012-07-06 15:25:10 -0500 | [diff] [blame] | 1 | /* |
| 2 | * Slab allocator functions that are independent of the allocator strategy |
| 3 | * |
| 4 | * (C) 2012 Christoph Lameter <cl@linux.com> |
| 5 | */ |
| 6 | #include <linux/slab.h> |
| 7 | |
| 8 | #include <linux/mm.h> |
| 9 | #include <linux/poison.h> |
| 10 | #include <linux/interrupt.h> |
| 11 | #include <linux/memory.h> |
| 12 | #include <linux/compiler.h> |
| 13 | #include <linux/module.h> |
Christoph Lameter | 20cea96 | 2012-07-06 15:25:13 -0500 | [diff] [blame] | 14 | #include <linux/cpu.h> |
| 15 | #include <linux/uaccess.h> |
Glauber Costa | b7454ad | 2012-10-19 18:20:25 +0400 | [diff] [blame] | 16 | #include <linux/seq_file.h> |
| 17 | #include <linux/proc_fs.h> |
Christoph Lameter | 039363f | 2012-07-06 15:25:10 -0500 | [diff] [blame] | 18 | #include <asm/cacheflush.h> |
| 19 | #include <asm/tlbflush.h> |
| 20 | #include <asm/page.h> |
| 21 | |
Christoph Lameter | 97d0660 | 2012-07-06 15:25:11 -0500 | [diff] [blame] | 22 | #include "slab.h" |
| 23 | |
| 24 | enum slab_state slab_state; |
Christoph Lameter | 18004c5 | 2012-07-06 15:25:12 -0500 | [diff] [blame] | 25 | LIST_HEAD(slab_caches); |
| 26 | DEFINE_MUTEX(slab_mutex); |
Christoph Lameter | 9b030cb | 2012-09-05 00:20:33 +0000 | [diff] [blame] | 27 | struct kmem_cache *kmem_cache; |
Christoph Lameter | 97d0660 | 2012-07-06 15:25:11 -0500 | [diff] [blame] | 28 | |
Shuah Khan | 77be4b1 | 2012-08-16 00:09:46 -0700 | [diff] [blame] | 29 | #ifdef CONFIG_DEBUG_VM |
| 30 | static int kmem_cache_sanity_check(const char *name, size_t size) |
| 31 | { |
| 32 | struct kmem_cache *s = NULL; |
| 33 | |
| 34 | if (!name || in_interrupt() || size < sizeof(void *) || |
| 35 | size > KMALLOC_MAX_SIZE) { |
| 36 | pr_err("kmem_cache_create(%s) integrity check failed\n", name); |
| 37 | return -EINVAL; |
| 38 | } |
| 39 | |
| 40 | list_for_each_entry(s, &slab_caches, list) { |
| 41 | char tmp; |
| 42 | int res; |
| 43 | |
| 44 | /* |
| 45 | * This happens when the module gets unloaded and doesn't |
| 46 | * destroy its slab cache and no-one else reuses the vmalloc |
| 47 | * area of the module. Print a warning. |
| 48 | */ |
| 49 | res = probe_kernel_address(s->name, tmp); |
| 50 | if (res) { |
| 51 | pr_err("Slab cache with size %d has lost its name\n", |
| 52 | s->object_size); |
| 53 | continue; |
| 54 | } |
| 55 | |
| 56 | if (!strcmp(s->name, name)) { |
| 57 | pr_err("%s (%s): Cache name already exists.\n", |
| 58 | __func__, name); |
| 59 | dump_stack(); |
| 60 | s = NULL; |
| 61 | return -EINVAL; |
| 62 | } |
| 63 | } |
| 64 | |
| 65 | WARN_ON(strchr(name, ' ')); /* It confuses parsers */ |
| 66 | return 0; |
| 67 | } |
| 68 | #else |
| 69 | static inline int kmem_cache_sanity_check(const char *name, size_t size) |
| 70 | { |
| 71 | return 0; |
| 72 | } |
| 73 | #endif |
| 74 | |
Christoph Lameter | 039363f | 2012-07-06 15:25:10 -0500 | [diff] [blame] | 75 | /* |
| 76 | * kmem_cache_create - Create a cache. |
| 77 | * @name: A string which is used in /proc/slabinfo to identify this cache. |
| 78 | * @size: The size of objects to be created in this cache. |
| 79 | * @align: The required alignment for the objects. |
| 80 | * @flags: SLAB flags |
| 81 | * @ctor: A constructor for the objects. |
| 82 | * |
| 83 | * Returns a ptr to the cache on success, NULL on failure. |
| 84 | * Cannot be called within a interrupt, but can be interrupted. |
| 85 | * The @ctor is run when new pages are allocated by the cache. |
| 86 | * |
| 87 | * The flags are |
| 88 | * |
| 89 | * %SLAB_POISON - Poison the slab with a known test pattern (a5a5a5a5) |
| 90 | * to catch references to uninitialised memory. |
| 91 | * |
| 92 | * %SLAB_RED_ZONE - Insert `Red' zones around the allocated memory to check |
| 93 | * for buffer overruns. |
| 94 | * |
| 95 | * %SLAB_HWCACHE_ALIGN - Align the objects in this cache to a hardware |
| 96 | * cacheline. This can be beneficial if you're counting cycles as closely |
| 97 | * as davem. |
| 98 | */ |
| 99 | |
| 100 | struct kmem_cache *kmem_cache_create(const char *name, size_t size, size_t align, |
| 101 | unsigned long flags, void (*ctor)(void *)) |
| 102 | { |
| 103 | struct kmem_cache *s = NULL; |
Christoph Lameter | 686d550 | 2012-09-05 00:20:33 +0000 | [diff] [blame] | 104 | int err = 0; |
Christoph Lameter | 039363f | 2012-07-06 15:25:10 -0500 | [diff] [blame] | 105 | |
Pekka Enberg | b920536 | 2012-08-16 10:12:18 +0300 | [diff] [blame] | 106 | get_online_cpus(); |
| 107 | mutex_lock(&slab_mutex); |
Christoph Lameter | 686d550 | 2012-09-05 00:20:33 +0000 | [diff] [blame] | 108 | |
| 109 | if (!kmem_cache_sanity_check(name, size) == 0) |
| 110 | goto out_locked; |
| 111 | |
| 112 | |
Christoph Lameter | cbb7969 | 2012-09-05 00:18:32 +0000 | [diff] [blame] | 113 | s = __kmem_cache_alias(name, size, align, flags, ctor); |
| 114 | if (s) |
| 115 | goto out_locked; |
| 116 | |
Christoph Lameter | 278b1bb | 2012-09-05 00:20:34 +0000 | [diff] [blame] | 117 | s = kmem_cache_zalloc(kmem_cache, GFP_KERNEL); |
Christoph Lameter | db265ec | 2012-09-04 23:18:33 +0000 | [diff] [blame] | 118 | if (s) { |
Christoph Lameter | 8a13a4c | 2012-09-04 23:18:33 +0000 | [diff] [blame] | 119 | s->object_size = s->size = size; |
| 120 | s->align = align; |
| 121 | s->ctor = ctor; |
| 122 | s->name = kstrdup(name, GFP_KERNEL); |
| 123 | if (!s->name) { |
| 124 | kmem_cache_free(kmem_cache, s); |
| 125 | err = -ENOMEM; |
| 126 | goto out_locked; |
| 127 | } |
| 128 | |
| 129 | err = __kmem_cache_create(s, flags); |
Christoph Lameter | cce89f4 | 2012-09-04 23:38:33 +0000 | [diff] [blame] | 130 | if (!err) { |
Christoph Lameter | 278b1bb | 2012-09-05 00:20:34 +0000 | [diff] [blame] | 131 | |
Christoph Lameter | cce89f4 | 2012-09-04 23:38:33 +0000 | [diff] [blame] | 132 | s->refcount = 1; |
Christoph Lameter | db265ec | 2012-09-04 23:18:33 +0000 | [diff] [blame] | 133 | list_add(&s->list, &slab_caches); |
| 134 | |
Christoph Lameter | cce89f4 | 2012-09-04 23:38:33 +0000 | [diff] [blame] | 135 | } else { |
Christoph Lameter | 8a13a4c | 2012-09-04 23:18:33 +0000 | [diff] [blame] | 136 | kfree(s->name); |
Christoph Lameter | 278b1bb | 2012-09-05 00:20:34 +0000 | [diff] [blame] | 137 | kmem_cache_free(kmem_cache, s); |
| 138 | } |
Christoph Lameter | 8a13a4c | 2012-09-04 23:18:33 +0000 | [diff] [blame] | 139 | } else |
Christoph Lameter | 278b1bb | 2012-09-05 00:20:34 +0000 | [diff] [blame] | 140 | err = -ENOMEM; |
Christoph Lameter | 7c9adf5 | 2012-09-04 23:38:33 +0000 | [diff] [blame] | 141 | |
Christoph Lameter | 686d550 | 2012-09-05 00:20:33 +0000 | [diff] [blame] | 142 | out_locked: |
Christoph Lameter | 20cea96 | 2012-07-06 15:25:13 -0500 | [diff] [blame] | 143 | mutex_unlock(&slab_mutex); |
| 144 | put_online_cpus(); |
| 145 | |
Christoph Lameter | 686d550 | 2012-09-05 00:20:33 +0000 | [diff] [blame] | 146 | if (err) { |
| 147 | |
| 148 | if (flags & SLAB_PANIC) |
| 149 | panic("kmem_cache_create: Failed to create slab '%s'. Error %d\n", |
| 150 | name, err); |
| 151 | else { |
| 152 | printk(KERN_WARNING "kmem_cache_create(%s) failed with error %d", |
| 153 | name, err); |
| 154 | dump_stack(); |
| 155 | } |
| 156 | |
| 157 | return NULL; |
| 158 | } |
Christoph Lameter | 039363f | 2012-07-06 15:25:10 -0500 | [diff] [blame] | 159 | |
| 160 | return s; |
| 161 | } |
| 162 | EXPORT_SYMBOL(kmem_cache_create); |
Christoph Lameter | 97d0660 | 2012-07-06 15:25:11 -0500 | [diff] [blame] | 163 | |
Christoph Lameter | 945cf2b | 2012-09-04 23:18:33 +0000 | [diff] [blame] | 164 | void kmem_cache_destroy(struct kmem_cache *s) |
| 165 | { |
| 166 | get_online_cpus(); |
| 167 | mutex_lock(&slab_mutex); |
| 168 | s->refcount--; |
| 169 | if (!s->refcount) { |
| 170 | list_del(&s->list); |
| 171 | |
| 172 | if (!__kmem_cache_shutdown(s)) { |
Jiri Kosina | 210ed9d | 2012-10-08 09:26:01 +0200 | [diff] [blame] | 173 | mutex_unlock(&slab_mutex); |
Christoph Lameter | 945cf2b | 2012-09-04 23:18:33 +0000 | [diff] [blame] | 174 | if (s->flags & SLAB_DESTROY_BY_RCU) |
| 175 | rcu_barrier(); |
| 176 | |
Christoph Lameter | db265ec | 2012-09-04 23:18:33 +0000 | [diff] [blame] | 177 | kfree(s->name); |
Christoph Lameter | 8f4c765 | 2012-09-05 00:18:32 +0000 | [diff] [blame] | 178 | kmem_cache_free(kmem_cache, s); |
Christoph Lameter | 945cf2b | 2012-09-04 23:18:33 +0000 | [diff] [blame] | 179 | } else { |
| 180 | list_add(&s->list, &slab_caches); |
Jiri Kosina | 210ed9d | 2012-10-08 09:26:01 +0200 | [diff] [blame] | 181 | mutex_unlock(&slab_mutex); |
Christoph Lameter | 945cf2b | 2012-09-04 23:18:33 +0000 | [diff] [blame] | 182 | printk(KERN_ERR "kmem_cache_destroy %s: Slab cache still has objects\n", |
| 183 | s->name); |
| 184 | dump_stack(); |
| 185 | } |
Jiri Kosina | 210ed9d | 2012-10-08 09:26:01 +0200 | [diff] [blame] | 186 | } else { |
| 187 | mutex_unlock(&slab_mutex); |
Christoph Lameter | 945cf2b | 2012-09-04 23:18:33 +0000 | [diff] [blame] | 188 | } |
Christoph Lameter | 945cf2b | 2012-09-04 23:18:33 +0000 | [diff] [blame] | 189 | put_online_cpus(); |
| 190 | } |
| 191 | EXPORT_SYMBOL(kmem_cache_destroy); |
| 192 | |
Christoph Lameter | 97d0660 | 2012-07-06 15:25:11 -0500 | [diff] [blame] | 193 | int slab_is_available(void) |
| 194 | { |
| 195 | return slab_state >= UP; |
| 196 | } |
Glauber Costa | b7454ad | 2012-10-19 18:20:25 +0400 | [diff] [blame] | 197 | |
| 198 | #ifdef CONFIG_SLABINFO |
Glauber Costa | bcee6e2 | 2012-10-19 18:20:26 +0400 | [diff] [blame^] | 199 | static void print_slabinfo_header(struct seq_file *m) |
| 200 | { |
| 201 | /* |
| 202 | * Output format version, so at least we can change it |
| 203 | * without _too_ many complaints. |
| 204 | */ |
| 205 | #ifdef CONFIG_DEBUG_SLAB |
| 206 | seq_puts(m, "slabinfo - version: 2.1 (statistics)\n"); |
| 207 | #else |
| 208 | seq_puts(m, "slabinfo - version: 2.1\n"); |
| 209 | #endif |
| 210 | seq_puts(m, "# name <active_objs> <num_objs> <objsize> " |
| 211 | "<objperslab> <pagesperslab>"); |
| 212 | seq_puts(m, " : tunables <limit> <batchcount> <sharedfactor>"); |
| 213 | seq_puts(m, " : slabdata <active_slabs> <num_slabs> <sharedavail>"); |
| 214 | #ifdef CONFIG_DEBUG_SLAB |
| 215 | seq_puts(m, " : globalstat <listallocs> <maxobjs> <grown> <reaped> " |
| 216 | "<error> <maxfreeable> <nodeallocs> <remotefrees> <alienoverflow>"); |
| 217 | seq_puts(m, " : cpustat <allochit> <allocmiss> <freehit> <freemiss>"); |
| 218 | #endif |
| 219 | seq_putc(m, '\n'); |
| 220 | } |
| 221 | |
Glauber Costa | b7454ad | 2012-10-19 18:20:25 +0400 | [diff] [blame] | 222 | static void *s_start(struct seq_file *m, loff_t *pos) |
| 223 | { |
| 224 | loff_t n = *pos; |
| 225 | |
| 226 | mutex_lock(&slab_mutex); |
| 227 | if (!n) |
| 228 | print_slabinfo_header(m); |
| 229 | |
| 230 | return seq_list_start(&slab_caches, *pos); |
| 231 | } |
| 232 | |
| 233 | static void *s_next(struct seq_file *m, void *p, loff_t *pos) |
| 234 | { |
| 235 | return seq_list_next(p, &slab_caches, pos); |
| 236 | } |
| 237 | |
| 238 | static void s_stop(struct seq_file *m, void *p) |
| 239 | { |
| 240 | mutex_unlock(&slab_mutex); |
| 241 | } |
| 242 | |
| 243 | static int s_show(struct seq_file *m, void *p) |
| 244 | { |
| 245 | return slabinfo_show(m, p); |
| 246 | } |
| 247 | |
| 248 | /* |
| 249 | * slabinfo_op - iterator that generates /proc/slabinfo |
| 250 | * |
| 251 | * Output layout: |
| 252 | * cache-name |
| 253 | * num-active-objs |
| 254 | * total-objs |
| 255 | * object size |
| 256 | * num-active-slabs |
| 257 | * total-slabs |
| 258 | * num-pages-per-slab |
| 259 | * + further values on SMP and with statistics enabled |
| 260 | */ |
| 261 | static const struct seq_operations slabinfo_op = { |
| 262 | .start = s_start, |
| 263 | .next = s_next, |
| 264 | .stop = s_stop, |
| 265 | .show = s_show, |
| 266 | }; |
| 267 | |
| 268 | static int slabinfo_open(struct inode *inode, struct file *file) |
| 269 | { |
| 270 | return seq_open(file, &slabinfo_op); |
| 271 | } |
| 272 | |
| 273 | static const struct file_operations proc_slabinfo_operations = { |
| 274 | .open = slabinfo_open, |
| 275 | .read = seq_read, |
| 276 | .write = slabinfo_write, |
| 277 | .llseek = seq_lseek, |
| 278 | .release = seq_release, |
| 279 | }; |
| 280 | |
| 281 | static int __init slab_proc_init(void) |
| 282 | { |
| 283 | proc_create("slabinfo", S_IRUSR, NULL, &proc_slabinfo_operations); |
| 284 | return 0; |
| 285 | } |
| 286 | module_init(slab_proc_init); |
| 287 | #endif /* CONFIG_SLABINFO */ |