blob: 526c191e84ea9cfd32d0a6252e27046a90cfc6f4 [file] [log] [blame]
Jeremy Fitzhardinge17758262008-04-02 10:54:13 -07001/******************************************************************************
2 * balloon.c
3 *
4 * Xen balloon driver - enables returning/claiming memory to/from Xen.
5 *
6 * Copyright (c) 2003, B Dragovic
7 * Copyright (c) 2003-2004, M Williamson, K Fraser
8 * Copyright (c) 2005 Dan M. Smith, IBM Corporation
9 *
10 * This program is free software; you can redistribute it and/or
11 * modify it under the terms of the GNU General Public License version 2
12 * as published by the Free Software Foundation; or, when distributed
13 * separately from the Linux kernel or incorporated into other
14 * software packages, subject to the following license:
15 *
16 * Permission is hereby granted, free of charge, to any person obtaining a copy
17 * of this source file (the "Software"), to deal in the Software without
18 * restriction, including without limitation the rights to use, copy, modify,
19 * merge, publish, distribute, sublicense, and/or sell copies of the Software,
20 * and to permit persons to whom the Software is furnished to do so, subject to
21 * the following conditions:
22 *
23 * The above copyright notice and this permission notice shall be included in
24 * all copies or substantial portions of the Software.
25 *
26 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
27 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
28 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
29 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
30 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
31 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
32 * IN THE SOFTWARE.
33 */
34
35#include <linux/kernel.h>
36#include <linux/module.h>
37#include <linux/sched.h>
38#include <linux/errno.h>
39#include <linux/mm.h>
40#include <linux/bootmem.h>
41#include <linux/pagemap.h>
42#include <linux/highmem.h>
43#include <linux/mutex.h>
Jeremy Fitzhardinge17758262008-04-02 10:54:13 -070044#include <linux/list.h>
45#include <linux/sysdev.h>
46
47#include <asm/xen/hypervisor.h>
48#include <asm/page.h>
49#include <asm/pgalloc.h>
50#include <asm/pgtable.h>
51#include <asm/uaccess.h>
52#include <asm/tlb.h>
53
54#include <xen/interface/memory.h>
Jeremy Fitzhardinge17758262008-04-02 10:54:13 -070055#include <xen/xenbus.h>
56#include <xen/features.h>
57#include <xen/page.h>
58
59#define PAGES2KB(_p) ((_p)<<(PAGE_SHIFT-10))
60
Jeremy Fitzhardinge167e6cf2008-07-24 16:27:52 -070061#define BALLOON_CLASS_NAME "xen_memory"
Jeremy Fitzhardinge17758262008-04-02 10:54:13 -070062
63struct balloon_stats {
64 /* We aim for 'current allocation' == 'target allocation'. */
65 unsigned long current_pages;
66 unsigned long target_pages;
67 /* We may hit the hard limit in Xen. If we do then we remember it. */
68 unsigned long hard_limit;
69 /*
70 * Drivers may alter the memory reservation independently, but they
71 * must inform the balloon driver so we avoid hitting the hard limit.
72 */
73 unsigned long driver_pages;
74 /* Number of pages in high- and low-memory balloons. */
75 unsigned long balloon_low;
76 unsigned long balloon_high;
77};
78
79static DEFINE_MUTEX(balloon_mutex);
80
81static struct sys_device balloon_sysdev;
82
83static int register_balloon(struct sys_device *sysdev);
84
85/*
86 * Protects atomic reservation decrease/increase against concurrent increases.
87 * Also protects non-atomic updates of current_pages and driver_pages, and
88 * balloon lists.
89 */
90static DEFINE_SPINLOCK(balloon_lock);
91
92static struct balloon_stats balloon_stats;
93
94/* We increase/decrease in batches which fit in a page */
95static unsigned long frame_list[PAGE_SIZE / sizeof(unsigned long)];
96
97/* VM /proc information for memory */
98extern unsigned long totalram_pages;
99
100#ifdef CONFIG_HIGHMEM
101extern unsigned long totalhigh_pages;
102#define inc_totalhigh_pages() (totalhigh_pages++)
103#define dec_totalhigh_pages() (totalhigh_pages--)
104#else
105#define inc_totalhigh_pages() do {} while(0)
106#define dec_totalhigh_pages() do {} while(0)
107#endif
108
109/* List of ballooned pages, threaded through the mem_map array. */
110static LIST_HEAD(ballooned_pages);
111
112/* Main work function, always executed in process context. */
113static void balloon_process(struct work_struct *work);
114static DECLARE_WORK(balloon_worker, balloon_process);
115static struct timer_list balloon_timer;
116
117/* When ballooning out (allocating memory to return to Xen) we don't really
118 want the kernel to try too hard since that can trigger the oom killer. */
119#define GFP_BALLOON \
120 (GFP_HIGHUSER | __GFP_NOWARN | __GFP_NORETRY | __GFP_NOMEMALLOC)
121
122static void scrub_page(struct page *page)
123{
124#ifdef CONFIG_XEN_SCRUB_PAGES
Jeremy Fitzhardinge26a3e992008-11-17 09:35:00 -0800125 clear_highpage(page);
Jeremy Fitzhardinge17758262008-04-02 10:54:13 -0700126#endif
127}
128
129/* balloon_append: add the given page to the balloon. */
130static void balloon_append(struct page *page)
131{
132 /* Lowmem is re-populated first, so highmem pages go at list tail. */
133 if (PageHighMem(page)) {
134 list_add_tail(&page->lru, &ballooned_pages);
135 balloon_stats.balloon_high++;
136 dec_totalhigh_pages();
137 } else {
138 list_add(&page->lru, &ballooned_pages);
139 balloon_stats.balloon_low++;
140 }
141}
142
143/* balloon_retrieve: rescue a page from the balloon, if it is not empty. */
144static struct page *balloon_retrieve(void)
145{
146 struct page *page;
147
148 if (list_empty(&ballooned_pages))
149 return NULL;
150
151 page = list_entry(ballooned_pages.next, struct page, lru);
152 list_del(&page->lru);
153
154 if (PageHighMem(page)) {
155 balloon_stats.balloon_high--;
156 inc_totalhigh_pages();
157 }
158 else
159 balloon_stats.balloon_low--;
160
161 return page;
162}
163
164static struct page *balloon_first_page(void)
165{
166 if (list_empty(&ballooned_pages))
167 return NULL;
168 return list_entry(ballooned_pages.next, struct page, lru);
169}
170
171static struct page *balloon_next_page(struct page *page)
172{
173 struct list_head *next = page->lru.next;
174 if (next == &ballooned_pages)
175 return NULL;
176 return list_entry(next, struct page, lru);
177}
178
179static void balloon_alarm(unsigned long unused)
180{
181 schedule_work(&balloon_worker);
182}
183
184static unsigned long current_target(void)
185{
186 unsigned long target = min(balloon_stats.target_pages, balloon_stats.hard_limit);
187
188 target = min(target,
189 balloon_stats.current_pages +
190 balloon_stats.balloon_low +
191 balloon_stats.balloon_high);
192
193 return target;
194}
195
196static int increase_reservation(unsigned long nr_pages)
197{
198 unsigned long pfn, i, flags;
199 struct page *page;
200 long rc;
201 struct xen_memory_reservation reservation = {
202 .address_bits = 0,
203 .extent_order = 0,
204 .domid = DOMID_SELF
205 };
206
207 if (nr_pages > ARRAY_SIZE(frame_list))
208 nr_pages = ARRAY_SIZE(frame_list);
209
210 spin_lock_irqsave(&balloon_lock, flags);
211
212 page = balloon_first_page();
213 for (i = 0; i < nr_pages; i++) {
214 BUG_ON(page == NULL);
215 frame_list[i] = page_to_pfn(page);;
216 page = balloon_next_page(page);
217 }
218
Isaku Yamahataa90971e2008-05-26 23:31:14 +0100219 set_xen_guest_handle(reservation.extent_start, frame_list);
Jeremy Fitzhardingefde28e82008-07-24 16:28:00 -0700220 reservation.nr_extents = nr_pages;
221 rc = HYPERVISOR_memory_op(XENMEM_populate_physmap, &reservation);
Jeremy Fitzhardinge17758262008-04-02 10:54:13 -0700222 if (rc < nr_pages) {
223 if (rc > 0) {
224 int ret;
225
226 /* We hit the Xen hard limit: reprobe. */
227 reservation.nr_extents = rc;
228 ret = HYPERVISOR_memory_op(XENMEM_decrease_reservation,
Jeremy Fitzhardingefde28e82008-07-24 16:28:00 -0700229 &reservation);
Jeremy Fitzhardinge17758262008-04-02 10:54:13 -0700230 BUG_ON(ret != rc);
231 }
232 if (rc >= 0)
233 balloon_stats.hard_limit = (balloon_stats.current_pages + rc -
234 balloon_stats.driver_pages);
235 goto out;
236 }
237
238 for (i = 0; i < nr_pages; i++) {
239 page = balloon_retrieve();
240 BUG_ON(page == NULL);
241
242 pfn = page_to_pfn(page);
243 BUG_ON(!xen_feature(XENFEAT_auto_translated_physmap) &&
244 phys_to_machine_mapping_valid(pfn));
245
246 set_phys_to_machine(pfn, frame_list[i]);
247
248 /* Link back into the page tables if not highmem. */
249 if (pfn < max_low_pfn) {
250 int ret;
251 ret = HYPERVISOR_update_va_mapping(
252 (unsigned long)__va(pfn << PAGE_SHIFT),
253 mfn_pte(frame_list[i], PAGE_KERNEL),
254 0);
255 BUG_ON(ret);
256 }
257
258 /* Relinquish the page back to the allocator. */
259 ClearPageReserved(page);
260 init_page_count(page);
261 __free_page(page);
262 }
263
264 balloon_stats.current_pages += nr_pages;
265 totalram_pages = balloon_stats.current_pages;
266
267 out:
268 spin_unlock_irqrestore(&balloon_lock, flags);
269
270 return 0;
271}
272
273static int decrease_reservation(unsigned long nr_pages)
274{
275 unsigned long pfn, i, flags;
276 struct page *page;
277 int need_sleep = 0;
278 int ret;
279 struct xen_memory_reservation reservation = {
280 .address_bits = 0,
281 .extent_order = 0,
282 .domid = DOMID_SELF
283 };
284
285 if (nr_pages > ARRAY_SIZE(frame_list))
286 nr_pages = ARRAY_SIZE(frame_list);
287
288 for (i = 0; i < nr_pages; i++) {
289 if ((page = alloc_page(GFP_BALLOON)) == NULL) {
290 nr_pages = i;
291 need_sleep = 1;
292 break;
293 }
294
295 pfn = page_to_pfn(page);
296 frame_list[i] = pfn_to_mfn(pfn);
297
298 scrub_page(page);
299 }
300
301 /* Ensure that ballooned highmem pages don't have kmaps. */
302 kmap_flush_unused();
303 flush_tlb_all();
304
305 spin_lock_irqsave(&balloon_lock, flags);
306
307 /* No more mappings: invalidate P2M and add to balloon. */
308 for (i = 0; i < nr_pages; i++) {
309 pfn = mfn_to_pfn(frame_list[i]);
310 set_phys_to_machine(pfn, INVALID_P2M_ENTRY);
311 balloon_append(pfn_to_page(pfn));
312 }
313
Isaku Yamahataa90971e2008-05-26 23:31:14 +0100314 set_xen_guest_handle(reservation.extent_start, frame_list);
Jeremy Fitzhardinge17758262008-04-02 10:54:13 -0700315 reservation.nr_extents = nr_pages;
316 ret = HYPERVISOR_memory_op(XENMEM_decrease_reservation, &reservation);
317 BUG_ON(ret != nr_pages);
318
319 balloon_stats.current_pages -= nr_pages;
320 totalram_pages = balloon_stats.current_pages;
321
322 spin_unlock_irqrestore(&balloon_lock, flags);
323
324 return need_sleep;
325}
326
327/*
328 * We avoid multiple worker processes conflicting via the balloon mutex.
329 * We may of course race updates of the target counts (which are protected
330 * by the balloon lock), or with changes to the Xen hard limit, but we will
331 * recover from these in time.
332 */
333static void balloon_process(struct work_struct *work)
334{
335 int need_sleep = 0;
336 long credit;
337
338 mutex_lock(&balloon_mutex);
339
340 do {
341 credit = current_target() - balloon_stats.current_pages;
342 if (credit > 0)
343 need_sleep = (increase_reservation(credit) != 0);
344 if (credit < 0)
345 need_sleep = (decrease_reservation(-credit) != 0);
346
347#ifndef CONFIG_PREEMPT
348 if (need_resched())
349 schedule();
350#endif
351 } while ((credit != 0) && !need_sleep);
352
353 /* Schedule more work if there is some still to be done. */
354 if (current_target() != balloon_stats.current_pages)
355 mod_timer(&balloon_timer, jiffies + HZ);
356
357 mutex_unlock(&balloon_mutex);
358}
359
360/* Resets the Xen limit, sets new target, and kicks off processing. */
Adrian Bunk955d6f12008-05-26 23:31:17 +0100361static void balloon_set_new_target(unsigned long target)
Jeremy Fitzhardinge17758262008-04-02 10:54:13 -0700362{
363 /* No need for lock. Not read-modify-write updates. */
364 balloon_stats.hard_limit = ~0UL;
365 balloon_stats.target_pages = target;
366 schedule_work(&balloon_worker);
367}
368
369static struct xenbus_watch target_watch =
370{
371 .node = "memory/target"
372};
373
374/* React to a change in the target key */
375static void watch_target(struct xenbus_watch *watch,
376 const char **vec, unsigned int len)
377{
378 unsigned long long new_target;
379 int err;
380
381 err = xenbus_scanf(XBT_NIL, "memory", "target", "%llu", &new_target);
382 if (err != 1) {
383 /* This is ok (for domain0 at least) - so just return */
384 return;
385 }
386
387 /* The given memory/target value is in KiB, so it needs converting to
388 * pages. PAGE_SHIFT converts bytes to pages, hence PAGE_SHIFT - 10.
389 */
390 balloon_set_new_target(new_target >> (PAGE_SHIFT - 10));
391}
392
393static int balloon_init_watcher(struct notifier_block *notifier,
394 unsigned long event,
395 void *data)
396{
397 int err;
398
399 err = register_xenbus_watch(&target_watch);
400 if (err)
401 printk(KERN_ERR "Failed to set balloon watcher\n");
402
403 return NOTIFY_DONE;
404}
405
406static struct notifier_block xenstore_notifier;
407
408static int __init balloon_init(void)
409{
410 unsigned long pfn;
411 struct page *page;
412
Jeremy Fitzhardinge6e833582008-08-19 13:16:17 -0700413 if (!xen_pv_domain())
Jeremy Fitzhardinge17758262008-04-02 10:54:13 -0700414 return -ENODEV;
415
416 pr_info("xen_balloon: Initialising balloon driver.\n");
417
418 balloon_stats.current_pages = min(xen_start_info->nr_pages, max_pfn);
419 totalram_pages = balloon_stats.current_pages;
420 balloon_stats.target_pages = balloon_stats.current_pages;
421 balloon_stats.balloon_low = 0;
422 balloon_stats.balloon_high = 0;
423 balloon_stats.driver_pages = 0UL;
424 balloon_stats.hard_limit = ~0UL;
425
426 init_timer(&balloon_timer);
427 balloon_timer.data = 0;
428 balloon_timer.function = balloon_alarm;
429
430 register_balloon(&balloon_sysdev);
431
432 /* Initialise the balloon with excess memory space. */
433 for (pfn = xen_start_info->nr_pages; pfn < max_pfn; pfn++) {
434 page = pfn_to_page(pfn);
435 if (!PageReserved(page))
436 balloon_append(page);
437 }
438
439 target_watch.callback = watch_target;
440 xenstore_notifier.notifier_call = balloon_init_watcher;
441
442 register_xenstore_notifier(&xenstore_notifier);
443
444 return 0;
445}
446
447subsys_initcall(balloon_init);
448
449static void balloon_exit(void)
450{
451 /* XXX - release balloon here */
452 return;
453}
454
455module_exit(balloon_exit);
456
Jeremy Fitzhardinge167e6cf2008-07-24 16:27:52 -0700457#define BALLOON_SHOW(name, format, args...) \
458 static ssize_t show_##name(struct sys_device *dev, \
459 struct sysdev_attribute *attr, \
460 char *buf) \
461 { \
462 return sprintf(buf, format, ##args); \
463 } \
Jeremy Fitzhardinge17758262008-04-02 10:54:13 -0700464 static SYSDEV_ATTR(name, S_IRUGO, show_##name, NULL)
465
466BALLOON_SHOW(current_kb, "%lu\n", PAGES2KB(balloon_stats.current_pages));
467BALLOON_SHOW(low_kb, "%lu\n", PAGES2KB(balloon_stats.balloon_low));
468BALLOON_SHOW(high_kb, "%lu\n", PAGES2KB(balloon_stats.balloon_high));
469BALLOON_SHOW(hard_limit_kb,
470 (balloon_stats.hard_limit!=~0UL) ? "%lu\n" : "???\n",
471 (balloon_stats.hard_limit!=~0UL) ? PAGES2KB(balloon_stats.hard_limit) : 0);
472BALLOON_SHOW(driver_kb, "%lu\n", PAGES2KB(balloon_stats.driver_pages));
473
Jeremy Fitzhardinge167e6cf2008-07-24 16:27:52 -0700474static ssize_t show_target_kb(struct sys_device *dev, struct sysdev_attribute *attr,
475 char *buf)
Jeremy Fitzhardinge17758262008-04-02 10:54:13 -0700476{
477 return sprintf(buf, "%lu\n", PAGES2KB(balloon_stats.target_pages));
478}
479
480static ssize_t store_target_kb(struct sys_device *dev,
Andi Kleen4a0b2b42008-07-01 18:48:41 +0200481 struct sysdev_attribute *attr,
Jeremy Fitzhardinge17758262008-04-02 10:54:13 -0700482 const char *buf,
483 size_t count)
484{
Jeremy Fitzhardinge167e6cf2008-07-24 16:27:52 -0700485 char *endchar;
Jeremy Fitzhardinge17758262008-04-02 10:54:13 -0700486 unsigned long long target_bytes;
487
488 if (!capable(CAP_SYS_ADMIN))
489 return -EPERM;
490
Jeremy Fitzhardinge167e6cf2008-07-24 16:27:52 -0700491 target_bytes = memparse(buf, &endchar);
Jeremy Fitzhardinge17758262008-04-02 10:54:13 -0700492
Jeremy Fitzhardinge17758262008-04-02 10:54:13 -0700493 balloon_set_new_target(target_bytes >> PAGE_SHIFT);
494
495 return count;
496}
497
498static SYSDEV_ATTR(target_kb, S_IRUGO | S_IWUSR,
499 show_target_kb, store_target_kb);
500
501static struct sysdev_attribute *balloon_attrs[] = {
502 &attr_target_kb,
503};
504
505static struct attribute *balloon_info_attrs[] = {
506 &attr_current_kb.attr,
507 &attr_low_kb.attr,
508 &attr_high_kb.attr,
509 &attr_hard_limit_kb.attr,
510 &attr_driver_kb.attr,
511 NULL
512};
513
514static struct attribute_group balloon_info_group = {
515 .name = "info",
516 .attrs = balloon_info_attrs,
517};
518
519static struct sysdev_class balloon_sysdev_class = {
520 .name = BALLOON_CLASS_NAME,
521};
522
523static int register_balloon(struct sys_device *sysdev)
524{
525 int i, error;
526
527 error = sysdev_class_register(&balloon_sysdev_class);
528 if (error)
529 return error;
530
531 sysdev->id = 0;
532 sysdev->cls = &balloon_sysdev_class;
533
534 error = sysdev_register(sysdev);
535 if (error) {
536 sysdev_class_unregister(&balloon_sysdev_class);
537 return error;
538 }
539
540 for (i = 0; i < ARRAY_SIZE(balloon_attrs); i++) {
541 error = sysdev_create_file(sysdev, balloon_attrs[i]);
542 if (error)
543 goto fail;
544 }
545
546 error = sysfs_create_group(&sysdev->kobj, &balloon_info_group);
547 if (error)
548 goto fail;
549
550 return 0;
551
552 fail:
553 while (--i >= 0)
554 sysdev_remove_file(sysdev, balloon_attrs[i]);
555 sysdev_unregister(sysdev);
556 sysdev_class_unregister(&balloon_sysdev_class);
557 return error;
558}
559
Jeremy Fitzhardinge17758262008-04-02 10:54:13 -0700560MODULE_LICENSE("GPL");