Dan Magenheimer | 29f233c | 2012-04-09 17:09:27 -0600 | [diff] [blame] | 1 | /* |
| 2 | * Frontswap frontend |
| 3 | * |
| 4 | * This code provides the generic "frontend" layer to call a matching |
| 5 | * "backend" driver implementation of frontswap. See |
| 6 | * Documentation/vm/frontswap.txt for more information. |
| 7 | * |
| 8 | * Copyright (C) 2009-2012 Oracle Corp. All rights reserved. |
| 9 | * Author: Dan Magenheimer |
| 10 | * |
| 11 | * This work is licensed under the terms of the GNU GPL, version 2. |
| 12 | */ |
| 13 | |
| 14 | #include <linux/mm.h> |
| 15 | #include <linux/mman.h> |
| 16 | #include <linux/swap.h> |
| 17 | #include <linux/swapops.h> |
| 18 | #include <linux/proc_fs.h> |
| 19 | #include <linux/security.h> |
| 20 | #include <linux/capability.h> |
| 21 | #include <linux/module.h> |
| 22 | #include <linux/uaccess.h> |
| 23 | #include <linux/debugfs.h> |
| 24 | #include <linux/frontswap.h> |
| 25 | #include <linux/swapfile.h> |
| 26 | |
| 27 | /* |
| 28 | * frontswap_ops is set by frontswap_register_ops to contain the pointers |
| 29 | * to the frontswap "backend" implementation functions. |
| 30 | */ |
| 31 | static struct frontswap_ops frontswap_ops __read_mostly; |
| 32 | |
| 33 | /* |
| 34 | * This global enablement flag reduces overhead on systems where frontswap_ops |
| 35 | * has not been registered, so is preferred to the slower alternative: a |
| 36 | * function call that checks a non-global. |
| 37 | */ |
| 38 | bool frontswap_enabled __read_mostly; |
| 39 | EXPORT_SYMBOL(frontswap_enabled); |
| 40 | |
| 41 | /* |
Konrad Rzeszutek Wilk | 165c8ae | 2012-05-15 11:32:15 -0400 | [diff] [blame] | 42 | * If enabled, frontswap_store will return failure even on success. As |
Dan Magenheimer | 29f233c | 2012-04-09 17:09:27 -0600 | [diff] [blame] | 43 | * a result, the swap subsystem will always write the page to swap, in |
| 44 | * effect converting frontswap into a writethrough cache. In this mode, |
| 45 | * there is no direct reduction in swap writes, but a frontswap backend |
| 46 | * can unilaterally "reclaim" any pages in use with no data loss, thus |
| 47 | * providing increases control over maximum memory usage due to frontswap. |
| 48 | */ |
| 49 | static bool frontswap_writethrough_enabled __read_mostly; |
| 50 | |
| 51 | #ifdef CONFIG_DEBUG_FS |
| 52 | /* |
| 53 | * Counters available via /sys/kernel/debug/frontswap (if debugfs is |
| 54 | * properly configured). These are for information only so are not protected |
| 55 | * against increment races. |
| 56 | */ |
Konrad Rzeszutek Wilk | 165c8ae | 2012-05-15 11:32:15 -0400 | [diff] [blame] | 57 | static u64 frontswap_loads; |
| 58 | static u64 frontswap_succ_stores; |
| 59 | static u64 frontswap_failed_stores; |
Dan Magenheimer | 29f233c | 2012-04-09 17:09:27 -0600 | [diff] [blame] | 60 | static u64 frontswap_invalidates; |
| 61 | |
Konrad Rzeszutek Wilk | 165c8ae | 2012-05-15 11:32:15 -0400 | [diff] [blame] | 62 | static inline void inc_frontswap_loads(void) { |
| 63 | frontswap_loads++; |
Dan Magenheimer | 29f233c | 2012-04-09 17:09:27 -0600 | [diff] [blame] | 64 | } |
Konrad Rzeszutek Wilk | 165c8ae | 2012-05-15 11:32:15 -0400 | [diff] [blame] | 65 | static inline void inc_frontswap_succ_stores(void) { |
| 66 | frontswap_succ_stores++; |
Dan Magenheimer | 29f233c | 2012-04-09 17:09:27 -0600 | [diff] [blame] | 67 | } |
Konrad Rzeszutek Wilk | 165c8ae | 2012-05-15 11:32:15 -0400 | [diff] [blame] | 68 | static inline void inc_frontswap_failed_stores(void) { |
| 69 | frontswap_failed_stores++; |
Dan Magenheimer | 29f233c | 2012-04-09 17:09:27 -0600 | [diff] [blame] | 70 | } |
| 71 | static inline void inc_frontswap_invalidates(void) { |
| 72 | frontswap_invalidates++; |
| 73 | } |
| 74 | #else |
Konrad Rzeszutek Wilk | 165c8ae | 2012-05-15 11:32:15 -0400 | [diff] [blame] | 75 | static inline void inc_frontswap_loads(void) { } |
| 76 | static inline void inc_frontswap_succ_stores(void) { } |
| 77 | static inline void inc_frontswap_failed_stores(void) { } |
Dan Magenheimer | 29f233c | 2012-04-09 17:09:27 -0600 | [diff] [blame] | 78 | static inline void inc_frontswap_invalidates(void) { } |
| 79 | #endif |
| 80 | /* |
| 81 | * Register operations for frontswap, returning previous thus allowing |
| 82 | * detection of multiple backends and possible nesting. |
| 83 | */ |
| 84 | struct frontswap_ops frontswap_register_ops(struct frontswap_ops *ops) |
| 85 | { |
| 86 | struct frontswap_ops old = frontswap_ops; |
| 87 | |
| 88 | frontswap_ops = *ops; |
| 89 | frontswap_enabled = true; |
| 90 | return old; |
| 91 | } |
| 92 | EXPORT_SYMBOL(frontswap_register_ops); |
| 93 | |
| 94 | /* |
| 95 | * Enable/disable frontswap writethrough (see above). |
| 96 | */ |
| 97 | void frontswap_writethrough(bool enable) |
| 98 | { |
| 99 | frontswap_writethrough_enabled = enable; |
| 100 | } |
| 101 | EXPORT_SYMBOL(frontswap_writethrough); |
| 102 | |
| 103 | /* |
| 104 | * Called when a swap device is swapon'd. |
| 105 | */ |
| 106 | void __frontswap_init(unsigned type) |
| 107 | { |
| 108 | struct swap_info_struct *sis = swap_info[type]; |
| 109 | |
| 110 | BUG_ON(sis == NULL); |
| 111 | if (sis->frontswap_map == NULL) |
| 112 | return; |
Sasha Levin | f9f0810 | 2012-06-10 12:51:05 +0200 | [diff] [blame] | 113 | frontswap_ops.init(type); |
Dan Magenheimer | 29f233c | 2012-04-09 17:09:27 -0600 | [diff] [blame] | 114 | } |
| 115 | EXPORT_SYMBOL(__frontswap_init); |
| 116 | |
Sasha Levin | 611edfe | 2012-06-10 12:51:07 +0200 | [diff] [blame^] | 117 | static inline void __frontswap_clear(struct swap_info_struct *sis, pgoff_t offset) |
| 118 | { |
| 119 | frontswap_clear(sis, offset); |
| 120 | atomic_dec(&sis->frontswap_pages); |
| 121 | } |
| 122 | |
Dan Magenheimer | 29f233c | 2012-04-09 17:09:27 -0600 | [diff] [blame] | 123 | /* |
Konrad Rzeszutek Wilk | 165c8ae | 2012-05-15 11:32:15 -0400 | [diff] [blame] | 124 | * "Store" data from a page to frontswap and associate it with the page's |
Dan Magenheimer | 29f233c | 2012-04-09 17:09:27 -0600 | [diff] [blame] | 125 | * swaptype and offset. Page must be locked and in the swap cache. |
| 126 | * If frontswap already contains a page with matching swaptype and |
| 127 | * offset, the frontswap implmentation may either overwrite the data and |
| 128 | * return success or invalidate the page from frontswap and return failure. |
| 129 | */ |
Konrad Rzeszutek Wilk | 165c8ae | 2012-05-15 11:32:15 -0400 | [diff] [blame] | 130 | int __frontswap_store(struct page *page) |
Dan Magenheimer | 29f233c | 2012-04-09 17:09:27 -0600 | [diff] [blame] | 131 | { |
| 132 | int ret = -1, dup = 0; |
| 133 | swp_entry_t entry = { .val = page_private(page), }; |
| 134 | int type = swp_type(entry); |
| 135 | struct swap_info_struct *sis = swap_info[type]; |
| 136 | pgoff_t offset = swp_offset(entry); |
| 137 | |
| 138 | BUG_ON(!PageLocked(page)); |
| 139 | BUG_ON(sis == NULL); |
| 140 | if (frontswap_test(sis, offset)) |
| 141 | dup = 1; |
Sasha Levin | ef38359 | 2012-06-10 12:50:59 +0200 | [diff] [blame] | 142 | ret = frontswap_ops.store(type, offset, page); |
Dan Magenheimer | 29f233c | 2012-04-09 17:09:27 -0600 | [diff] [blame] | 143 | if (ret == 0) { |
| 144 | frontswap_set(sis, offset); |
Konrad Rzeszutek Wilk | 165c8ae | 2012-05-15 11:32:15 -0400 | [diff] [blame] | 145 | inc_frontswap_succ_stores(); |
Dan Magenheimer | 29f233c | 2012-04-09 17:09:27 -0600 | [diff] [blame] | 146 | if (!dup) |
| 147 | atomic_inc(&sis->frontswap_pages); |
Sasha Levin | d9674dd | 2012-06-10 12:51:04 +0200 | [diff] [blame] | 148 | } else { |
Dan Magenheimer | 29f233c | 2012-04-09 17:09:27 -0600 | [diff] [blame] | 149 | /* |
| 150 | failed dup always results in automatic invalidate of |
| 151 | the (older) page from frontswap |
| 152 | */ |
Konrad Rzeszutek Wilk | 165c8ae | 2012-05-15 11:32:15 -0400 | [diff] [blame] | 153 | inc_frontswap_failed_stores(); |
Sasha Levin | 611edfe | 2012-06-10 12:51:07 +0200 | [diff] [blame^] | 154 | if (dup) |
| 155 | __frontswap_clear(sis, offset); |
Sasha Levin | 4bb3e31 | 2012-06-10 12:51:00 +0200 | [diff] [blame] | 156 | } |
Dan Magenheimer | 29f233c | 2012-04-09 17:09:27 -0600 | [diff] [blame] | 157 | if (frontswap_writethrough_enabled) |
| 158 | /* report failure so swap also writes to swap device */ |
| 159 | ret = -1; |
| 160 | return ret; |
| 161 | } |
Konrad Rzeszutek Wilk | 165c8ae | 2012-05-15 11:32:15 -0400 | [diff] [blame] | 162 | EXPORT_SYMBOL(__frontswap_store); |
Dan Magenheimer | 29f233c | 2012-04-09 17:09:27 -0600 | [diff] [blame] | 163 | |
| 164 | /* |
| 165 | * "Get" data from frontswap associated with swaptype and offset that were |
| 166 | * specified when the data was put to frontswap and use it to fill the |
| 167 | * specified page with data. Page must be locked and in the swap cache. |
| 168 | */ |
Konrad Rzeszutek Wilk | 165c8ae | 2012-05-15 11:32:15 -0400 | [diff] [blame] | 169 | int __frontswap_load(struct page *page) |
Dan Magenheimer | 29f233c | 2012-04-09 17:09:27 -0600 | [diff] [blame] | 170 | { |
| 171 | int ret = -1; |
| 172 | swp_entry_t entry = { .val = page_private(page), }; |
| 173 | int type = swp_type(entry); |
| 174 | struct swap_info_struct *sis = swap_info[type]; |
| 175 | pgoff_t offset = swp_offset(entry); |
| 176 | |
| 177 | BUG_ON(!PageLocked(page)); |
| 178 | BUG_ON(sis == NULL); |
| 179 | if (frontswap_test(sis, offset)) |
Sasha Levin | ef38359 | 2012-06-10 12:50:59 +0200 | [diff] [blame] | 180 | ret = frontswap_ops.load(type, offset, page); |
Dan Magenheimer | 29f233c | 2012-04-09 17:09:27 -0600 | [diff] [blame] | 181 | if (ret == 0) |
Konrad Rzeszutek Wilk | 165c8ae | 2012-05-15 11:32:15 -0400 | [diff] [blame] | 182 | inc_frontswap_loads(); |
Dan Magenheimer | 29f233c | 2012-04-09 17:09:27 -0600 | [diff] [blame] | 183 | return ret; |
| 184 | } |
Konrad Rzeszutek Wilk | 165c8ae | 2012-05-15 11:32:15 -0400 | [diff] [blame] | 185 | EXPORT_SYMBOL(__frontswap_load); |
Dan Magenheimer | 29f233c | 2012-04-09 17:09:27 -0600 | [diff] [blame] | 186 | |
| 187 | /* |
| 188 | * Invalidate any data from frontswap associated with the specified swaptype |
| 189 | * and offset so that a subsequent "get" will fail. |
| 190 | */ |
| 191 | void __frontswap_invalidate_page(unsigned type, pgoff_t offset) |
| 192 | { |
| 193 | struct swap_info_struct *sis = swap_info[type]; |
| 194 | |
| 195 | BUG_ON(sis == NULL); |
| 196 | if (frontswap_test(sis, offset)) { |
Sasha Levin | ef38359 | 2012-06-10 12:50:59 +0200 | [diff] [blame] | 197 | frontswap_ops.invalidate_page(type, offset); |
Sasha Levin | 611edfe | 2012-06-10 12:51:07 +0200 | [diff] [blame^] | 198 | __frontswap_clear(sis, offset); |
Dan Magenheimer | 29f233c | 2012-04-09 17:09:27 -0600 | [diff] [blame] | 199 | inc_frontswap_invalidates(); |
| 200 | } |
| 201 | } |
| 202 | EXPORT_SYMBOL(__frontswap_invalidate_page); |
| 203 | |
| 204 | /* |
| 205 | * Invalidate all data from frontswap associated with all offsets for the |
| 206 | * specified swaptype. |
| 207 | */ |
| 208 | void __frontswap_invalidate_area(unsigned type) |
| 209 | { |
| 210 | struct swap_info_struct *sis = swap_info[type]; |
| 211 | |
| 212 | BUG_ON(sis == NULL); |
| 213 | if (sis->frontswap_map == NULL) |
| 214 | return; |
Sasha Levin | ef38359 | 2012-06-10 12:50:59 +0200 | [diff] [blame] | 215 | frontswap_ops.invalidate_area(type); |
Dan Magenheimer | 29f233c | 2012-04-09 17:09:27 -0600 | [diff] [blame] | 216 | atomic_set(&sis->frontswap_pages, 0); |
| 217 | memset(sis->frontswap_map, 0, sis->max / sizeof(long)); |
| 218 | } |
| 219 | EXPORT_SYMBOL(__frontswap_invalidate_area); |
| 220 | |
Sasha Levin | 9625344 | 2012-06-10 12:51:01 +0200 | [diff] [blame] | 221 | static unsigned long __frontswap_curr_pages(void) |
| 222 | { |
| 223 | int type; |
| 224 | unsigned long totalpages = 0; |
| 225 | struct swap_info_struct *si = NULL; |
| 226 | |
| 227 | assert_spin_locked(&swap_lock); |
| 228 | for (type = swap_list.head; type >= 0; type = si->next) { |
| 229 | si = swap_info[type]; |
| 230 | totalpages += atomic_read(&si->frontswap_pages); |
| 231 | } |
| 232 | return totalpages; |
| 233 | } |
| 234 | |
Sasha Levin | f116695 | 2012-06-10 12:51:02 +0200 | [diff] [blame] | 235 | static int __frontswap_unuse_pages(unsigned long total, unsigned long *unused, |
| 236 | int *swapid) |
| 237 | { |
| 238 | int ret = -EINVAL; |
| 239 | struct swap_info_struct *si = NULL; |
| 240 | int si_frontswap_pages; |
| 241 | unsigned long total_pages_to_unuse = total; |
| 242 | unsigned long pages = 0, pages_to_unuse = 0; |
| 243 | int type; |
| 244 | |
| 245 | assert_spin_locked(&swap_lock); |
| 246 | for (type = swap_list.head; type >= 0; type = si->next) { |
| 247 | si = swap_info[type]; |
| 248 | si_frontswap_pages = atomic_read(&si->frontswap_pages); |
| 249 | if (total_pages_to_unuse < si_frontswap_pages) { |
| 250 | pages = pages_to_unuse = total_pages_to_unuse; |
| 251 | } else { |
| 252 | pages = si_frontswap_pages; |
| 253 | pages_to_unuse = 0; /* unuse all */ |
| 254 | } |
| 255 | /* ensure there is enough RAM to fetch pages from frontswap */ |
| 256 | if (security_vm_enough_memory_mm(current->mm, pages)) { |
| 257 | ret = -ENOMEM; |
| 258 | continue; |
| 259 | } |
| 260 | vm_unacct_memory(pages); |
| 261 | *unused = pages_to_unuse; |
| 262 | *swapid = type; |
| 263 | ret = 0; |
| 264 | break; |
| 265 | } |
| 266 | |
| 267 | return ret; |
| 268 | } |
| 269 | |
Sasha Levin | 69217b4 | 2012-06-10 12:51:03 +0200 | [diff] [blame] | 270 | static int __frontswap_shrink(unsigned long target_pages, |
| 271 | unsigned long *pages_to_unuse, |
| 272 | int *type) |
| 273 | { |
| 274 | unsigned long total_pages = 0, total_pages_to_unuse; |
| 275 | |
| 276 | assert_spin_locked(&swap_lock); |
| 277 | |
| 278 | total_pages = __frontswap_curr_pages(); |
| 279 | if (total_pages <= target_pages) { |
| 280 | /* Nothing to do */ |
| 281 | *pages_to_unuse = 0; |
| 282 | return 0; |
| 283 | } |
| 284 | total_pages_to_unuse = total_pages - target_pages; |
| 285 | return __frontswap_unuse_pages(total_pages_to_unuse, pages_to_unuse, type); |
| 286 | } |
| 287 | |
Dan Magenheimer | 29f233c | 2012-04-09 17:09:27 -0600 | [diff] [blame] | 288 | /* |
| 289 | * Frontswap, like a true swap device, may unnecessarily retain pages |
| 290 | * under certain circumstances; "shrink" frontswap is essentially a |
| 291 | * "partial swapoff" and works by calling try_to_unuse to attempt to |
| 292 | * unuse enough frontswap pages to attempt to -- subject to memory |
| 293 | * constraints -- reduce the number of pages in frontswap to the |
| 294 | * number given in the parameter target_pages. |
| 295 | */ |
| 296 | void frontswap_shrink(unsigned long target_pages) |
| 297 | { |
Sasha Levin | f116695 | 2012-06-10 12:51:02 +0200 | [diff] [blame] | 298 | unsigned long pages_to_unuse = 0; |
| 299 | int type, ret; |
Dan Magenheimer | 29f233c | 2012-04-09 17:09:27 -0600 | [diff] [blame] | 300 | |
| 301 | /* |
| 302 | * we don't want to hold swap_lock while doing a very |
| 303 | * lengthy try_to_unuse, but swap_list may change |
| 304 | * so restart scan from swap_list.head each time |
| 305 | */ |
| 306 | spin_lock(&swap_lock); |
Sasha Levin | 69217b4 | 2012-06-10 12:51:03 +0200 | [diff] [blame] | 307 | ret = __frontswap_shrink(target_pages, &pages_to_unuse, &type); |
Dan Magenheimer | 29f233c | 2012-04-09 17:09:27 -0600 | [diff] [blame] | 308 | spin_unlock(&swap_lock); |
Sasha Levin | 69217b4 | 2012-06-10 12:51:03 +0200 | [diff] [blame] | 309 | if (ret == 0 && pages_to_unuse) |
| 310 | try_to_unuse(type, true, pages_to_unuse); |
Dan Magenheimer | 29f233c | 2012-04-09 17:09:27 -0600 | [diff] [blame] | 311 | return; |
| 312 | } |
| 313 | EXPORT_SYMBOL(frontswap_shrink); |
| 314 | |
| 315 | /* |
| 316 | * Count and return the number of frontswap pages across all |
| 317 | * swap devices. This is exported so that backend drivers can |
| 318 | * determine current usage without reading debugfs. |
| 319 | */ |
| 320 | unsigned long frontswap_curr_pages(void) |
| 321 | { |
Dan Magenheimer | 29f233c | 2012-04-09 17:09:27 -0600 | [diff] [blame] | 322 | unsigned long totalpages = 0; |
Dan Magenheimer | 29f233c | 2012-04-09 17:09:27 -0600 | [diff] [blame] | 323 | |
| 324 | spin_lock(&swap_lock); |
Sasha Levin | 9625344 | 2012-06-10 12:51:01 +0200 | [diff] [blame] | 325 | totalpages = __frontswap_curr_pages(); |
Dan Magenheimer | 29f233c | 2012-04-09 17:09:27 -0600 | [diff] [blame] | 326 | spin_unlock(&swap_lock); |
Sasha Levin | 9625344 | 2012-06-10 12:51:01 +0200 | [diff] [blame] | 327 | |
Dan Magenheimer | 29f233c | 2012-04-09 17:09:27 -0600 | [diff] [blame] | 328 | return totalpages; |
| 329 | } |
| 330 | EXPORT_SYMBOL(frontswap_curr_pages); |
| 331 | |
| 332 | static int __init init_frontswap(void) |
| 333 | { |
| 334 | #ifdef CONFIG_DEBUG_FS |
| 335 | struct dentry *root = debugfs_create_dir("frontswap", NULL); |
| 336 | if (root == NULL) |
| 337 | return -ENXIO; |
Konrad Rzeszutek Wilk | 165c8ae | 2012-05-15 11:32:15 -0400 | [diff] [blame] | 338 | debugfs_create_u64("loads", S_IRUGO, root, &frontswap_loads); |
| 339 | debugfs_create_u64("succ_stores", S_IRUGO, root, &frontswap_succ_stores); |
| 340 | debugfs_create_u64("failed_stores", S_IRUGO, root, |
| 341 | &frontswap_failed_stores); |
Dan Magenheimer | 29f233c | 2012-04-09 17:09:27 -0600 | [diff] [blame] | 342 | debugfs_create_u64("invalidates", S_IRUGO, |
| 343 | root, &frontswap_invalidates); |
| 344 | #endif |
| 345 | return 0; |
| 346 | } |
| 347 | |
| 348 | module_init(init_frontswap); |