blob: edf7e18a1c058294f8b2acec0f03b677179cb36a [file] [log] [blame]
Dan Magenheimer5bc20fc2011-05-26 10:02:21 -06001/*
2 * Xen implementation for transcendent memory (tmem)
3 *
Dan Magenheimerafec6e02011-06-17 15:06:20 -06004 * Copyright (C) 2009-2011 Oracle Corp. All rights reserved.
Dan Magenheimer5bc20fc2011-05-26 10:02:21 -06005 * Author: Dan Magenheimer
6 */
7
Dan Magenheimer10a7a0772013-04-30 15:27:00 -07008#include <linux/module.h>
Dan Magenheimer5bc20fc2011-05-26 10:02:21 -06009#include <linux/kernel.h>
10#include <linux/types.h>
11#include <linux/init.h>
12#include <linux/pagemap.h>
13#include <linux/cleancache.h>
Dan Magenheimerafec6e02011-06-17 15:06:20 -060014#include <linux/frontswap.h>
Dan Magenheimerafec6e02011-06-17 15:06:20 -060015
Dan Magenheimer5bc20fc2011-05-26 10:02:21 -060016#include <xen/xen.h>
17#include <xen/interface/xen.h>
18#include <asm/xen/hypercall.h>
19#include <asm/xen/page.h>
20#include <asm/xen/hypervisor.h>
Konrad Rzeszutek Wilkb8b0f552012-08-21 14:49:34 -040021#include <xen/tmem.h>
Dan Magenheimer5bc20fc2011-05-26 10:02:21 -060022
Konrad Rzeszutek Wilk0cb401d2013-05-08 15:50:59 -040023#ifndef CONFIG_XEN_TMEM_MODULE
24bool __read_mostly tmem_enabled = false;
25
26static int __init enable_tmem(char *s)
27{
28 tmem_enabled = true;
29 return 1;
30}
31__setup("tmem", enable_tmem);
32#endif
33
34#ifdef CONFIG_CLEANCACHE
35static bool disable_cleancache __read_mostly;
36static bool disable_selfballooning __read_mostly;
37#ifdef CONFIG_XEN_TMEM_MODULE
38module_param(disable_cleancache, bool, S_IRUGO);
39module_param(disable_selfballooning, bool, S_IRUGO);
40#else
41static int __init no_cleancache(char *s)
42{
43 disable_cleancache = true;
44 return 1;
45}
46__setup("nocleancache", no_cleancache);
47#endif
48#endif /* CONFIG_CLEANCACHE */
49
50#ifdef CONFIG_FRONTSWAP
51static bool disable_frontswap __read_mostly;
52static bool disable_frontswap_selfshrinking __read_mostly;
53#ifdef CONFIG_XEN_TMEM_MODULE
54module_param(disable_frontswap, bool, S_IRUGO);
55module_param(disable_frontswap_selfshrinking, bool, S_IRUGO);
56#else
57static int __init no_frontswap(char *s)
58{
59 disable_frontswap = true;
60 return 1;
61}
62__setup("nofrontswap", no_frontswap);
63#endif
64#else /* CONFIG_FRONTSWAP */
65#define disable_frontswap_selfshrinking 1
66#endif /* CONFIG_FRONTSWAP */
67
Dan Magenheimer5bc20fc2011-05-26 10:02:21 -060068#define TMEM_CONTROL 0
69#define TMEM_NEW_POOL 1
70#define TMEM_DESTROY_POOL 2
71#define TMEM_NEW_PAGE 3
72#define TMEM_PUT_PAGE 4
73#define TMEM_GET_PAGE 5
74#define TMEM_FLUSH_PAGE 6
75#define TMEM_FLUSH_OBJECT 7
76#define TMEM_READ 8
77#define TMEM_WRITE 9
78#define TMEM_XCHG 10
79
80/* Bits for HYPERVISOR_tmem_op(TMEM_NEW_POOL) */
81#define TMEM_POOL_PERSIST 1
82#define TMEM_POOL_SHARED 2
83#define TMEM_POOL_PAGESIZE_SHIFT 4
84#define TMEM_VERSION_SHIFT 24
85
86
87struct tmem_pool_uuid {
88 u64 uuid_lo;
89 u64 uuid_hi;
90};
91
92struct tmem_oid {
93 u64 oid[3];
94};
95
96#define TMEM_POOL_PRIVATE_UUID { 0, 0 }
97
98/* flags for tmem_ops.new_pool */
99#define TMEM_POOL_PERSIST 1
100#define TMEM_POOL_SHARED 2
101
102/* xen tmem foundation ops/hypercalls */
103
104static inline int xen_tmem_op(u32 tmem_cmd, u32 tmem_pool, struct tmem_oid oid,
105 u32 index, unsigned long gmfn, u32 tmem_offset, u32 pfn_offset, u32 len)
106{
107 struct tmem_op op;
108 int rc = 0;
109
110 op.cmd = tmem_cmd;
111 op.pool_id = tmem_pool;
112 op.u.gen.oid[0] = oid.oid[0];
113 op.u.gen.oid[1] = oid.oid[1];
114 op.u.gen.oid[2] = oid.oid[2];
115 op.u.gen.index = index;
116 op.u.gen.tmem_offset = tmem_offset;
117 op.u.gen.pfn_offset = pfn_offset;
118 op.u.gen.len = len;
119 set_xen_guest_handle(op.u.gen.gmfn, (void *)gmfn);
120 rc = HYPERVISOR_tmem_op(&op);
121 return rc;
122}
123
124static int xen_tmem_new_pool(struct tmem_pool_uuid uuid,
125 u32 flags, unsigned long pagesize)
126{
127 struct tmem_op op;
128 int rc = 0, pageshift;
129
130 for (pageshift = 0; pagesize != 1; pageshift++)
131 pagesize >>= 1;
132 flags |= (pageshift - 12) << TMEM_POOL_PAGESIZE_SHIFT;
133 flags |= TMEM_SPEC_VERSION << TMEM_VERSION_SHIFT;
134 op.cmd = TMEM_NEW_POOL;
135 op.u.new.uuid[0] = uuid.uuid_lo;
136 op.u.new.uuid[1] = uuid.uuid_hi;
137 op.u.new.flags = flags;
138 rc = HYPERVISOR_tmem_op(&op);
139 return rc;
140}
141
142/* xen generic tmem ops */
143
144static int xen_tmem_put_page(u32 pool_id, struct tmem_oid oid,
145 u32 index, unsigned long pfn)
146{
147 unsigned long gmfn = xen_pv_domain() ? pfn_to_mfn(pfn) : pfn;
148
149 return xen_tmem_op(TMEM_PUT_PAGE, pool_id, oid, index,
150 gmfn, 0, 0, 0);
151}
152
153static int xen_tmem_get_page(u32 pool_id, struct tmem_oid oid,
154 u32 index, unsigned long pfn)
155{
156 unsigned long gmfn = xen_pv_domain() ? pfn_to_mfn(pfn) : pfn;
157
158 return xen_tmem_op(TMEM_GET_PAGE, pool_id, oid, index,
159 gmfn, 0, 0, 0);
160}
161
162static int xen_tmem_flush_page(u32 pool_id, struct tmem_oid oid, u32 index)
163{
164 return xen_tmem_op(TMEM_FLUSH_PAGE, pool_id, oid, index,
165 0, 0, 0, 0);
166}
167
168static int xen_tmem_flush_object(u32 pool_id, struct tmem_oid oid)
169{
170 return xen_tmem_op(TMEM_FLUSH_OBJECT, pool_id, oid, 0, 0, 0, 0, 0);
171}
172
Dan Magenheimer5bc20fc2011-05-26 10:02:21 -0600173
Dan Magenheimerafec6e02011-06-17 15:06:20 -0600174#ifdef CONFIG_CLEANCACHE
175static int xen_tmem_destroy_pool(u32 pool_id)
176{
177 struct tmem_oid oid = { { 0 } };
178
179 return xen_tmem_op(TMEM_DESTROY_POOL, pool_id, oid, 0, 0, 0, 0, 0);
180}
181
Dan Magenheimer5bc20fc2011-05-26 10:02:21 -0600182/* cleancache ops */
183
184static void tmem_cleancache_put_page(int pool, struct cleancache_filekey key,
185 pgoff_t index, struct page *page)
186{
187 u32 ind = (u32) index;
188 struct tmem_oid oid = *(struct tmem_oid *)&key;
189 unsigned long pfn = page_to_pfn(page);
190
191 if (pool < 0)
192 return;
193 if (ind != index)
194 return;
195 mb(); /* ensure page is quiescent; tmem may address it with an alias */
196 (void)xen_tmem_put_page((u32)pool, oid, ind, pfn);
197}
198
199static int tmem_cleancache_get_page(int pool, struct cleancache_filekey key,
200 pgoff_t index, struct page *page)
201{
202 u32 ind = (u32) index;
203 struct tmem_oid oid = *(struct tmem_oid *)&key;
204 unsigned long pfn = page_to_pfn(page);
205 int ret;
206
207 /* translate return values to linux semantics */
208 if (pool < 0)
209 return -1;
210 if (ind != index)
211 return -1;
212 ret = xen_tmem_get_page((u32)pool, oid, ind, pfn);
213 if (ret == 1)
214 return 0;
215 else
216 return -1;
217}
218
219static void tmem_cleancache_flush_page(int pool, struct cleancache_filekey key,
220 pgoff_t index)
221{
222 u32 ind = (u32) index;
223 struct tmem_oid oid = *(struct tmem_oid *)&key;
224
225 if (pool < 0)
226 return;
227 if (ind != index)
228 return;
229 (void)xen_tmem_flush_page((u32)pool, oid, ind);
230}
231
232static void tmem_cleancache_flush_inode(int pool, struct cleancache_filekey key)
233{
234 struct tmem_oid oid = *(struct tmem_oid *)&key;
235
236 if (pool < 0)
237 return;
238 (void)xen_tmem_flush_object((u32)pool, oid);
239}
240
241static void tmem_cleancache_flush_fs(int pool)
242{
243 if (pool < 0)
244 return;
245 (void)xen_tmem_destroy_pool((u32)pool);
246}
247
248static int tmem_cleancache_init_fs(size_t pagesize)
249{
250 struct tmem_pool_uuid uuid_private = TMEM_POOL_PRIVATE_UUID;
251
252 return xen_tmem_new_pool(uuid_private, 0, pagesize);
253}
254
255static int tmem_cleancache_init_shared_fs(char *uuid, size_t pagesize)
256{
257 struct tmem_pool_uuid shared_uuid;
258
259 shared_uuid.uuid_lo = *(u64 *)uuid;
260 shared_uuid.uuid_hi = *(u64 *)(&uuid[8]);
261 return xen_tmem_new_pool(shared_uuid, TMEM_POOL_SHARED, pagesize);
262}
263
Konrad Rzeszutek Wilk833f8662013-04-30 15:26:57 -0700264static struct cleancache_ops tmem_cleancache_ops = {
Dan Magenheimer5bc20fc2011-05-26 10:02:21 -0600265 .put_page = tmem_cleancache_put_page,
266 .get_page = tmem_cleancache_get_page,
Dan Magenheimer91c6cc92012-01-12 14:03:25 -0500267 .invalidate_page = tmem_cleancache_flush_page,
268 .invalidate_inode = tmem_cleancache_flush_inode,
269 .invalidate_fs = tmem_cleancache_flush_fs,
Dan Magenheimer5bc20fc2011-05-26 10:02:21 -0600270 .init_shared_fs = tmem_cleancache_init_shared_fs,
271 .init_fs = tmem_cleancache_init_fs
272};
Dan Magenheimerafec6e02011-06-17 15:06:20 -0600273#endif
274
275#ifdef CONFIG_FRONTSWAP
276/* frontswap tmem operations */
277
278/* a single tmem poolid is used for all frontswap "types" (swapfiles) */
279static int tmem_frontswap_poolid;
280
281/*
282 * Swizzling increases objects per swaptype, increasing tmem concurrency
283 * for heavy swaploads. Later, larger nr_cpus -> larger SWIZ_BITS
284 */
285#define SWIZ_BITS 4
286#define SWIZ_MASK ((1 << SWIZ_BITS) - 1)
287#define _oswiz(_type, _ind) ((_type << SWIZ_BITS) | (_ind & SWIZ_MASK))
288#define iswiz(_ind) (_ind >> SWIZ_BITS)
289
290static inline struct tmem_oid oswiz(unsigned type, u32 ind)
291{
292 struct tmem_oid oid = { .oid = { 0 } };
293 oid.oid[0] = _oswiz(type, ind);
294 return oid;
295}
296
297/* returns 0 if the page was successfully put into frontswap, -1 if not */
Konrad Rzeszutek Wilk165c8ae2012-05-15 11:32:15 -0400298static int tmem_frontswap_store(unsigned type, pgoff_t offset,
Dan Magenheimerafec6e02011-06-17 15:06:20 -0600299 struct page *page)
300{
301 u64 ind64 = (u64)offset;
302 u32 ind = (u32)offset;
303 unsigned long pfn = page_to_pfn(page);
304 int pool = tmem_frontswap_poolid;
305 int ret;
306
307 if (pool < 0)
308 return -1;
309 if (ind64 != ind)
310 return -1;
311 mb(); /* ensure page is quiescent; tmem may address it with an alias */
312 ret = xen_tmem_put_page(pool, oswiz(type, ind), iswiz(ind), pfn);
313 /* translate Xen tmem return values to linux semantics */
314 if (ret == 1)
315 return 0;
316 else
317 return -1;
318}
319
320/*
321 * returns 0 if the page was successfully gotten from frontswap, -1 if
322 * was not present (should never happen!)
323 */
Konrad Rzeszutek Wilk165c8ae2012-05-15 11:32:15 -0400324static int tmem_frontswap_load(unsigned type, pgoff_t offset,
Dan Magenheimerafec6e02011-06-17 15:06:20 -0600325 struct page *page)
326{
327 u64 ind64 = (u64)offset;
328 u32 ind = (u32)offset;
329 unsigned long pfn = page_to_pfn(page);
330 int pool = tmem_frontswap_poolid;
331 int ret;
332
333 if (pool < 0)
334 return -1;
335 if (ind64 != ind)
336 return -1;
337 ret = xen_tmem_get_page(pool, oswiz(type, ind), iswiz(ind), pfn);
338 /* translate Xen tmem return values to linux semantics */
339 if (ret == 1)
340 return 0;
341 else
342 return -1;
343}
344
345/* flush a single page from frontswap */
346static void tmem_frontswap_flush_page(unsigned type, pgoff_t offset)
347{
348 u64 ind64 = (u64)offset;
349 u32 ind = (u32)offset;
350 int pool = tmem_frontswap_poolid;
351
352 if (pool < 0)
353 return;
354 if (ind64 != ind)
355 return;
356 (void) xen_tmem_flush_page(pool, oswiz(type, ind), iswiz(ind));
357}
358
359/* flush all pages from the passed swaptype */
360static void tmem_frontswap_flush_area(unsigned type)
361{
362 int pool = tmem_frontswap_poolid;
363 int ind;
364
365 if (pool < 0)
366 return;
367 for (ind = SWIZ_MASK; ind >= 0; ind--)
368 (void)xen_tmem_flush_object(pool, oswiz(type, ind));
369}
370
371static void tmem_frontswap_init(unsigned ignored)
372{
373 struct tmem_pool_uuid private = TMEM_POOL_PRIVATE_UUID;
374
375 /* a single tmem poolid is used for all frontswap "types" (swapfiles) */
376 if (tmem_frontswap_poolid < 0)
377 tmem_frontswap_poolid =
378 xen_tmem_new_pool(private, TMEM_POOL_PERSIST, PAGE_SIZE);
379}
380
Konrad Rzeszutek Wilk1e01c962013-04-30 15:26:51 -0700381static struct frontswap_ops tmem_frontswap_ops = {
Konrad Rzeszutek Wilk165c8ae2012-05-15 11:32:15 -0400382 .store = tmem_frontswap_store,
383 .load = tmem_frontswap_load,
Dan Magenheimer91c6cc92012-01-12 14:03:25 -0500384 .invalidate_page = tmem_frontswap_flush_page,
385 .invalidate_area = tmem_frontswap_flush_area,
Dan Magenheimerafec6e02011-06-17 15:06:20 -0600386 .init = tmem_frontswap_init
387};
388#endif
Dan Magenheimer5bc20fc2011-05-26 10:02:21 -0600389
Dan Magenheimer10a7a0772013-04-30 15:27:00 -0700390static int xen_tmem_init(void)
Dan Magenheimer5bc20fc2011-05-26 10:02:21 -0600391{
Dan Magenheimer5bc20fc2011-05-26 10:02:21 -0600392 if (!xen_domain())
393 return 0;
Dan Magenheimerafec6e02011-06-17 15:06:20 -0600394#ifdef CONFIG_FRONTSWAP
Dan Magenheimer10a7a0772013-04-30 15:27:00 -0700395 if (tmem_enabled && !disable_frontswap) {
Dan Magenheimerafec6e02011-06-17 15:06:20 -0600396 char *s = "";
Konrad Rzeszutek Wilk1e01c962013-04-30 15:26:51 -0700397 struct frontswap_ops *old_ops =
Dan Magenheimerafec6e02011-06-17 15:06:20 -0600398 frontswap_register_ops(&tmem_frontswap_ops);
399
400 tmem_frontswap_poolid = -1;
Konrad Rzeszutek Wilkf42158f2013-04-30 15:27:01 -0700401 if (IS_ERR(old_ops) || old_ops) {
402 if (IS_ERR(old_ops))
403 return PTR_ERR(old_ops);
Dan Magenheimerafec6e02011-06-17 15:06:20 -0600404 s = " (WARNING: frontswap_ops overridden)";
Konrad Rzeszutek Wilkf42158f2013-04-30 15:27:01 -0700405 }
Dan Magenheimerafec6e02011-06-17 15:06:20 -0600406 printk(KERN_INFO "frontswap enabled, RAM provided by "
Konrad Rzeszutek Wilk22230c12013-02-01 14:10:44 -0500407 "Xen Transcendent Memory%s\n", s);
Dan Magenheimerafec6e02011-06-17 15:06:20 -0600408 }
409#endif
Dan Magenheimer5bc20fc2011-05-26 10:02:21 -0600410#ifdef CONFIG_CLEANCACHE
411 BUG_ON(sizeof(struct cleancache_filekey) != sizeof(struct tmem_oid));
Dan Magenheimer10a7a0772013-04-30 15:27:00 -0700412 if (tmem_enabled && !disable_cleancache) {
Dan Magenheimer5bc20fc2011-05-26 10:02:21 -0600413 char *s = "";
Konrad Rzeszutek Wilk833f8662013-04-30 15:26:57 -0700414 struct cleancache_ops *old_ops =
Dan Magenheimerafec6e02011-06-17 15:06:20 -0600415 cleancache_register_ops(&tmem_cleancache_ops);
Konrad Rzeszutek Wilk833f8662013-04-30 15:26:57 -0700416 if (old_ops)
Dan Magenheimer5bc20fc2011-05-26 10:02:21 -0600417 s = " (WARNING: cleancache_ops overridden)";
418 printk(KERN_INFO "cleancache enabled, RAM provided by "
419 "Xen Transcendent Memory%s\n", s);
420 }
421#endif
Dan Magenheimer10a7a0772013-04-30 15:27:00 -0700422#ifdef CONFIG_XEN_SELFBALLOONING
423 xen_selfballoon_init(!disable_selfballooning,
424 !disable_frontswap_selfshrinking);
425#endif
Dan Magenheimer5bc20fc2011-05-26 10:02:21 -0600426 return 0;
427}
428
429module_init(xen_tmem_init)
Dan Magenheimer10a7a0772013-04-30 15:27:00 -0700430MODULE_LICENSE("GPL");
431MODULE_AUTHOR("Dan Magenheimer <dan.magenheimer@oracle.com>");
432MODULE_DESCRIPTION("Shim to Xen transcendent memory");