Alexander Potapenko | 55834c5 | 2016-05-20 16:59:11 -0700 | [diff] [blame] | 1 | /* |
| 2 | * KASAN quarantine. |
| 3 | * |
| 4 | * Author: Alexander Potapenko <glider@google.com> |
| 5 | * Copyright (C) 2016 Google, Inc. |
| 6 | * |
| 7 | * Based on code by Dmitry Chernenkov. |
| 8 | * |
| 9 | * This program is free software; you can redistribute it and/or |
| 10 | * modify it under the terms of the GNU General Public License |
| 11 | * version 2 as published by the Free Software Foundation. |
| 12 | * |
| 13 | * This program is distributed in the hope that it will be useful, but |
| 14 | * WITHOUT ANY WARRANTY; without even the implied warranty of |
| 15 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU |
| 16 | * General Public License for more details. |
| 17 | * |
| 18 | */ |
| 19 | |
| 20 | #include <linux/gfp.h> |
| 21 | #include <linux/hash.h> |
| 22 | #include <linux/kernel.h> |
| 23 | #include <linux/mm.h> |
| 24 | #include <linux/percpu.h> |
| 25 | #include <linux/printk.h> |
| 26 | #include <linux/shrinker.h> |
| 27 | #include <linux/slab.h> |
| 28 | #include <linux/string.h> |
| 29 | #include <linux/types.h> |
| 30 | |
| 31 | #include "../slab.h" |
| 32 | #include "kasan.h" |
| 33 | |
| 34 | /* Data structure and operations for quarantine queues. */ |
| 35 | |
| 36 | /* |
| 37 | * Each queue is a signle-linked list, which also stores the total size of |
| 38 | * objects inside of it. |
| 39 | */ |
| 40 | struct qlist_head { |
| 41 | struct qlist_node *head; |
| 42 | struct qlist_node *tail; |
| 43 | size_t bytes; |
| 44 | }; |
| 45 | |
| 46 | #define QLIST_INIT { NULL, NULL, 0 } |
| 47 | |
| 48 | static bool qlist_empty(struct qlist_head *q) |
| 49 | { |
| 50 | return !q->head; |
| 51 | } |
| 52 | |
| 53 | static void qlist_init(struct qlist_head *q) |
| 54 | { |
| 55 | q->head = q->tail = NULL; |
| 56 | q->bytes = 0; |
| 57 | } |
| 58 | |
| 59 | static void qlist_put(struct qlist_head *q, struct qlist_node *qlink, |
| 60 | size_t size) |
| 61 | { |
| 62 | if (unlikely(qlist_empty(q))) |
| 63 | q->head = qlink; |
| 64 | else |
| 65 | q->tail->next = qlink; |
| 66 | q->tail = qlink; |
| 67 | qlink->next = NULL; |
| 68 | q->bytes += size; |
| 69 | } |
| 70 | |
| 71 | static void qlist_move_all(struct qlist_head *from, struct qlist_head *to) |
| 72 | { |
| 73 | if (unlikely(qlist_empty(from))) |
| 74 | return; |
| 75 | |
| 76 | if (qlist_empty(to)) { |
| 77 | *to = *from; |
| 78 | qlist_init(from); |
| 79 | return; |
| 80 | } |
| 81 | |
| 82 | to->tail->next = from->head; |
| 83 | to->tail = from->tail; |
| 84 | to->bytes += from->bytes; |
| 85 | |
| 86 | qlist_init(from); |
| 87 | } |
| 88 | |
| 89 | static void qlist_move(struct qlist_head *from, struct qlist_node *last, |
| 90 | struct qlist_head *to, size_t size) |
| 91 | { |
| 92 | if (unlikely(last == from->tail)) { |
| 93 | qlist_move_all(from, to); |
| 94 | return; |
| 95 | } |
| 96 | if (qlist_empty(to)) |
| 97 | to->head = from->head; |
| 98 | else |
| 99 | to->tail->next = from->head; |
| 100 | to->tail = last; |
| 101 | from->head = last->next; |
| 102 | last->next = NULL; |
| 103 | from->bytes -= size; |
| 104 | to->bytes += size; |
| 105 | } |
| 106 | |
| 107 | |
| 108 | /* |
| 109 | * The object quarantine consists of per-cpu queues and a global queue, |
| 110 | * guarded by quarantine_lock. |
| 111 | */ |
| 112 | static DEFINE_PER_CPU(struct qlist_head, cpu_quarantine); |
| 113 | |
| 114 | static struct qlist_head global_quarantine; |
| 115 | static DEFINE_SPINLOCK(quarantine_lock); |
| 116 | |
| 117 | /* Maximum size of the global queue. */ |
| 118 | static unsigned long quarantine_size; |
| 119 | |
| 120 | /* |
| 121 | * The fraction of physical memory the quarantine is allowed to occupy. |
| 122 | * Quarantine doesn't support memory shrinker with SLAB allocator, so we keep |
| 123 | * the ratio low to avoid OOM. |
| 124 | */ |
| 125 | #define QUARANTINE_FRACTION 32 |
| 126 | |
| 127 | #define QUARANTINE_LOW_SIZE (READ_ONCE(quarantine_size) * 3 / 4) |
| 128 | #define QUARANTINE_PERCPU_SIZE (1 << 20) |
| 129 | |
| 130 | static struct kmem_cache *qlink_to_cache(struct qlist_node *qlink) |
| 131 | { |
| 132 | return virt_to_head_page(qlink)->slab_cache; |
| 133 | } |
| 134 | |
| 135 | static void *qlink_to_object(struct qlist_node *qlink, struct kmem_cache *cache) |
| 136 | { |
| 137 | struct kasan_free_meta *free_info = |
| 138 | container_of(qlink, struct kasan_free_meta, |
| 139 | quarantine_link); |
| 140 | |
| 141 | return ((void *)free_info) - cache->kasan_info.free_meta_offset; |
| 142 | } |
| 143 | |
| 144 | static void qlink_free(struct qlist_node *qlink, struct kmem_cache *cache) |
| 145 | { |
| 146 | void *object = qlink_to_object(qlink, cache); |
Alexander Potapenko | 55834c5 | 2016-05-20 16:59:11 -0700 | [diff] [blame] | 147 | unsigned long flags; |
| 148 | |
Andrey Ryabinin | f7376ae | 2016-08-02 14:02:46 -0700 | [diff] [blame] | 149 | if (IS_ENABLED(CONFIG_SLAB)) |
| 150 | local_irq_save(flags); |
| 151 | |
Alexander Potapenko | 55834c5 | 2016-05-20 16:59:11 -0700 | [diff] [blame] | 152 | ___cache_free(cache, object, _THIS_IP_); |
Andrey Ryabinin | f7376ae | 2016-08-02 14:02:46 -0700 | [diff] [blame] | 153 | |
| 154 | if (IS_ENABLED(CONFIG_SLAB)) |
| 155 | local_irq_restore(flags); |
Alexander Potapenko | 55834c5 | 2016-05-20 16:59:11 -0700 | [diff] [blame] | 156 | } |
| 157 | |
| 158 | static void qlist_free_all(struct qlist_head *q, struct kmem_cache *cache) |
| 159 | { |
| 160 | struct qlist_node *qlink; |
| 161 | |
| 162 | if (unlikely(qlist_empty(q))) |
| 163 | return; |
| 164 | |
| 165 | qlink = q->head; |
| 166 | while (qlink) { |
| 167 | struct kmem_cache *obj_cache = |
| 168 | cache ? cache : qlink_to_cache(qlink); |
| 169 | struct qlist_node *next = qlink->next; |
| 170 | |
| 171 | qlink_free(qlink, obj_cache); |
| 172 | qlink = next; |
| 173 | } |
| 174 | qlist_init(q); |
| 175 | } |
| 176 | |
| 177 | void quarantine_put(struct kasan_free_meta *info, struct kmem_cache *cache) |
| 178 | { |
| 179 | unsigned long flags; |
| 180 | struct qlist_head *q; |
| 181 | struct qlist_head temp = QLIST_INIT; |
| 182 | |
| 183 | local_irq_save(flags); |
| 184 | |
| 185 | q = this_cpu_ptr(&cpu_quarantine); |
| 186 | qlist_put(q, &info->quarantine_link, cache->size); |
| 187 | if (unlikely(q->bytes > QUARANTINE_PERCPU_SIZE)) |
| 188 | qlist_move_all(q, &temp); |
| 189 | |
| 190 | local_irq_restore(flags); |
| 191 | |
| 192 | if (unlikely(!qlist_empty(&temp))) { |
| 193 | spin_lock_irqsave(&quarantine_lock, flags); |
| 194 | qlist_move_all(&temp, &global_quarantine); |
| 195 | spin_unlock_irqrestore(&quarantine_lock, flags); |
| 196 | } |
| 197 | } |
| 198 | |
| 199 | void quarantine_reduce(void) |
| 200 | { |
Alexander Potapenko | c3cee37 | 2016-08-02 14:02:58 -0700 | [diff] [blame] | 201 | size_t new_quarantine_size, percpu_quarantines; |
Alexander Potapenko | 55834c5 | 2016-05-20 16:59:11 -0700 | [diff] [blame] | 202 | unsigned long flags; |
| 203 | struct qlist_head to_free = QLIST_INIT; |
| 204 | size_t size_to_free = 0; |
| 205 | struct qlist_node *last; |
| 206 | |
| 207 | if (likely(READ_ONCE(global_quarantine.bytes) <= |
| 208 | READ_ONCE(quarantine_size))) |
| 209 | return; |
| 210 | |
| 211 | spin_lock_irqsave(&quarantine_lock, flags); |
| 212 | |
| 213 | /* |
| 214 | * Update quarantine size in case of hotplug. Allocate a fraction of |
| 215 | * the installed memory to quarantine minus per-cpu queue limits. |
| 216 | */ |
| 217 | new_quarantine_size = (READ_ONCE(totalram_pages) << PAGE_SHIFT) / |
| 218 | QUARANTINE_FRACTION; |
Alexander Potapenko | c3cee37 | 2016-08-02 14:02:58 -0700 | [diff] [blame] | 219 | percpu_quarantines = QUARANTINE_PERCPU_SIZE * num_online_cpus(); |
Alexander Potapenko | bcbf0d5 | 2016-08-11 15:33:06 -0700 | [diff] [blame] | 220 | new_quarantine_size = (new_quarantine_size < percpu_quarantines) ? |
| 221 | 0 : new_quarantine_size - percpu_quarantines; |
Alexander Potapenko | 55834c5 | 2016-05-20 16:59:11 -0700 | [diff] [blame] | 222 | WRITE_ONCE(quarantine_size, new_quarantine_size); |
| 223 | |
| 224 | last = global_quarantine.head; |
| 225 | while (last) { |
| 226 | struct kmem_cache *cache = qlink_to_cache(last); |
| 227 | |
| 228 | size_to_free += cache->size; |
| 229 | if (!last->next || size_to_free > |
| 230 | global_quarantine.bytes - QUARANTINE_LOW_SIZE) |
| 231 | break; |
| 232 | last = last->next; |
| 233 | } |
| 234 | qlist_move(&global_quarantine, last, &to_free, size_to_free); |
| 235 | |
| 236 | spin_unlock_irqrestore(&quarantine_lock, flags); |
| 237 | |
| 238 | qlist_free_all(&to_free, NULL); |
| 239 | } |
| 240 | |
| 241 | static void qlist_move_cache(struct qlist_head *from, |
| 242 | struct qlist_head *to, |
| 243 | struct kmem_cache *cache) |
| 244 | { |
Joonsoo Kim | 0ab686d | 2016-07-14 12:07:17 -0700 | [diff] [blame] | 245 | struct qlist_node *curr; |
Alexander Potapenko | 55834c5 | 2016-05-20 16:59:11 -0700 | [diff] [blame] | 246 | |
| 247 | if (unlikely(qlist_empty(from))) |
| 248 | return; |
| 249 | |
| 250 | curr = from->head; |
Joonsoo Kim | 0ab686d | 2016-07-14 12:07:17 -0700 | [diff] [blame] | 251 | qlist_init(from); |
Alexander Potapenko | 55834c5 | 2016-05-20 16:59:11 -0700 | [diff] [blame] | 252 | while (curr) { |
Joonsoo Kim | 0ab686d | 2016-07-14 12:07:17 -0700 | [diff] [blame] | 253 | struct qlist_node *next = curr->next; |
| 254 | struct kmem_cache *obj_cache = qlink_to_cache(curr); |
Alexander Potapenko | 55834c5 | 2016-05-20 16:59:11 -0700 | [diff] [blame] | 255 | |
Joonsoo Kim | 0ab686d | 2016-07-14 12:07:17 -0700 | [diff] [blame] | 256 | if (obj_cache == cache) |
| 257 | qlist_put(to, curr, obj_cache->size); |
| 258 | else |
| 259 | qlist_put(from, curr, obj_cache->size); |
| 260 | |
| 261 | curr = next; |
Alexander Potapenko | 55834c5 | 2016-05-20 16:59:11 -0700 | [diff] [blame] | 262 | } |
| 263 | } |
| 264 | |
| 265 | static void per_cpu_remove_cache(void *arg) |
| 266 | { |
| 267 | struct kmem_cache *cache = arg; |
| 268 | struct qlist_head to_free = QLIST_INIT; |
| 269 | struct qlist_head *q; |
| 270 | |
| 271 | q = this_cpu_ptr(&cpu_quarantine); |
| 272 | qlist_move_cache(q, &to_free, cache); |
| 273 | qlist_free_all(&to_free, cache); |
| 274 | } |
| 275 | |
| 276 | void quarantine_remove_cache(struct kmem_cache *cache) |
| 277 | { |
| 278 | unsigned long flags; |
| 279 | struct qlist_head to_free = QLIST_INIT; |
| 280 | |
| 281 | on_each_cpu(per_cpu_remove_cache, cache, 1); |
| 282 | |
| 283 | spin_lock_irqsave(&quarantine_lock, flags); |
| 284 | qlist_move_cache(&global_quarantine, &to_free, cache); |
| 285 | spin_unlock_irqrestore(&quarantine_lock, flags); |
| 286 | |
| 287 | qlist_free_all(&to_free, cache); |
| 288 | } |