blob: f6738d8b02bcdd55d7b3ebdde6782f3394726831 [file] [log] [blame]
Jeremy Fitzhardinge17758262008-04-02 10:54:13 -07001/******************************************************************************
2 * balloon.c
3 *
4 * Xen balloon driver - enables returning/claiming memory to/from Xen.
5 *
6 * Copyright (c) 2003, B Dragovic
7 * Copyright (c) 2003-2004, M Williamson, K Fraser
8 * Copyright (c) 2005 Dan M. Smith, IBM Corporation
9 *
10 * This program is free software; you can redistribute it and/or
11 * modify it under the terms of the GNU General Public License version 2
12 * as published by the Free Software Foundation; or, when distributed
13 * separately from the Linux kernel or incorporated into other
14 * software packages, subject to the following license:
15 *
16 * Permission is hereby granted, free of charge, to any person obtaining a copy
17 * of this source file (the "Software"), to deal in the Software without
18 * restriction, including without limitation the rights to use, copy, modify,
19 * merge, publish, distribute, sublicense, and/or sell copies of the Software,
20 * and to permit persons to whom the Software is furnished to do so, subject to
21 * the following conditions:
22 *
23 * The above copyright notice and this permission notice shall be included in
24 * all copies or substantial portions of the Software.
25 *
26 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
27 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
28 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
29 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
30 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
31 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
32 * IN THE SOFTWARE.
33 */
34
35#include <linux/kernel.h>
36#include <linux/module.h>
37#include <linux/sched.h>
38#include <linux/errno.h>
39#include <linux/mm.h>
40#include <linux/bootmem.h>
41#include <linux/pagemap.h>
42#include <linux/highmem.h>
43#include <linux/mutex.h>
Jeremy Fitzhardinge17758262008-04-02 10:54:13 -070044#include <linux/list.h>
45#include <linux/sysdev.h>
46
Jeremy Fitzhardinge17758262008-04-02 10:54:13 -070047#include <asm/page.h>
48#include <asm/pgalloc.h>
49#include <asm/pgtable.h>
50#include <asm/uaccess.h>
51#include <asm/tlb.h>
52
Jeremy Fitzhardingeecbf29c2008-12-16 12:37:07 -080053#include <asm/xen/hypervisor.h>
54#include <asm/xen/hypercall.h>
Jeremy Fitzhardinge1ccbf532009-10-06 15:11:14 -070055
56#include <xen/xen.h>
Jeremy Fitzhardingeecbf29c2008-12-16 12:37:07 -080057#include <xen/interface/xen.h>
Jeremy Fitzhardinge17758262008-04-02 10:54:13 -070058#include <xen/interface/memory.h>
Jeremy Fitzhardinge17758262008-04-02 10:54:13 -070059#include <xen/xenbus.h>
60#include <xen/features.h>
61#include <xen/page.h>
62
63#define PAGES2KB(_p) ((_p)<<(PAGE_SHIFT-10))
64
Jeremy Fitzhardinge167e6cf2008-07-24 16:27:52 -070065#define BALLOON_CLASS_NAME "xen_memory"
Jeremy Fitzhardinge17758262008-04-02 10:54:13 -070066
67struct balloon_stats {
68 /* We aim for 'current allocation' == 'target allocation'. */
69 unsigned long current_pages;
70 unsigned long target_pages;
Jeremy Fitzhardinge17758262008-04-02 10:54:13 -070071 /*
72 * Drivers may alter the memory reservation independently, but they
73 * must inform the balloon driver so we avoid hitting the hard limit.
74 */
75 unsigned long driver_pages;
76 /* Number of pages in high- and low-memory balloons. */
77 unsigned long balloon_low;
78 unsigned long balloon_high;
79};
80
81static DEFINE_MUTEX(balloon_mutex);
82
83static struct sys_device balloon_sysdev;
84
85static int register_balloon(struct sys_device *sysdev);
86
87/*
88 * Protects atomic reservation decrease/increase against concurrent increases.
89 * Also protects non-atomic updates of current_pages and driver_pages, and
90 * balloon lists.
91 */
92static DEFINE_SPINLOCK(balloon_lock);
93
94static struct balloon_stats balloon_stats;
95
96/* We increase/decrease in batches which fit in a page */
97static unsigned long frame_list[PAGE_SIZE / sizeof(unsigned long)];
98
Jeremy Fitzhardinge17758262008-04-02 10:54:13 -070099#ifdef CONFIG_HIGHMEM
Jeremy Fitzhardinge17758262008-04-02 10:54:13 -0700100#define inc_totalhigh_pages() (totalhigh_pages++)
101#define dec_totalhigh_pages() (totalhigh_pages--)
102#else
103#define inc_totalhigh_pages() do {} while(0)
104#define dec_totalhigh_pages() do {} while(0)
105#endif
106
107/* List of ballooned pages, threaded through the mem_map array. */
108static LIST_HEAD(ballooned_pages);
109
110/* Main work function, always executed in process context. */
111static void balloon_process(struct work_struct *work);
112static DECLARE_WORK(balloon_worker, balloon_process);
113static struct timer_list balloon_timer;
114
115/* When ballooning out (allocating memory to return to Xen) we don't really
116 want the kernel to try too hard since that can trigger the oom killer. */
117#define GFP_BALLOON \
118 (GFP_HIGHUSER | __GFP_NOWARN | __GFP_NORETRY | __GFP_NOMEMALLOC)
119
120static void scrub_page(struct page *page)
121{
122#ifdef CONFIG_XEN_SCRUB_PAGES
Jeremy Fitzhardinge26a3e992008-11-17 09:35:00 -0800123 clear_highpage(page);
Jeremy Fitzhardinge17758262008-04-02 10:54:13 -0700124#endif
125}
126
127/* balloon_append: add the given page to the balloon. */
128static void balloon_append(struct page *page)
129{
130 /* Lowmem is re-populated first, so highmem pages go at list tail. */
131 if (PageHighMem(page)) {
132 list_add_tail(&page->lru, &ballooned_pages);
133 balloon_stats.balloon_high++;
134 dec_totalhigh_pages();
135 } else {
136 list_add(&page->lru, &ballooned_pages);
137 balloon_stats.balloon_low++;
138 }
Gianluca Guida3d65c942009-07-30 22:54:36 +0100139
140 totalram_pages--;
Jeremy Fitzhardinge17758262008-04-02 10:54:13 -0700141}
142
143/* balloon_retrieve: rescue a page from the balloon, if it is not empty. */
144static struct page *balloon_retrieve(void)
145{
146 struct page *page;
147
148 if (list_empty(&ballooned_pages))
149 return NULL;
150
151 page = list_entry(ballooned_pages.next, struct page, lru);
152 list_del(&page->lru);
153
154 if (PageHighMem(page)) {
155 balloon_stats.balloon_high--;
156 inc_totalhigh_pages();
157 }
158 else
159 balloon_stats.balloon_low--;
160
Gianluca Guida3d65c942009-07-30 22:54:36 +0100161 totalram_pages++;
162
Jeremy Fitzhardinge17758262008-04-02 10:54:13 -0700163 return page;
164}
165
166static struct page *balloon_first_page(void)
167{
168 if (list_empty(&ballooned_pages))
169 return NULL;
170 return list_entry(ballooned_pages.next, struct page, lru);
171}
172
173static struct page *balloon_next_page(struct page *page)
174{
175 struct list_head *next = page->lru.next;
176 if (next == &ballooned_pages)
177 return NULL;
178 return list_entry(next, struct page, lru);
179}
180
181static void balloon_alarm(unsigned long unused)
182{
183 schedule_work(&balloon_worker);
184}
185
186static unsigned long current_target(void)
187{
Ian Campbellbc2c0302009-06-05 11:58:37 +0100188 unsigned long target = balloon_stats.target_pages;
Jeremy Fitzhardinge17758262008-04-02 10:54:13 -0700189
190 target = min(target,
191 balloon_stats.current_pages +
192 balloon_stats.balloon_low +
193 balloon_stats.balloon_high);
194
195 return target;
196}
197
198static int increase_reservation(unsigned long nr_pages)
199{
200 unsigned long pfn, i, flags;
201 struct page *page;
202 long rc;
203 struct xen_memory_reservation reservation = {
204 .address_bits = 0,
205 .extent_order = 0,
206 .domid = DOMID_SELF
207 };
208
209 if (nr_pages > ARRAY_SIZE(frame_list))
210 nr_pages = ARRAY_SIZE(frame_list);
211
212 spin_lock_irqsave(&balloon_lock, flags);
213
214 page = balloon_first_page();
215 for (i = 0; i < nr_pages; i++) {
216 BUG_ON(page == NULL);
Joe Perchesa419aef2009-08-18 11:18:35 -0700217 frame_list[i] = page_to_pfn(page);
Jeremy Fitzhardinge17758262008-04-02 10:54:13 -0700218 page = balloon_next_page(page);
219 }
220
Isaku Yamahataa90971e2008-05-26 23:31:14 +0100221 set_xen_guest_handle(reservation.extent_start, frame_list);
Jeremy Fitzhardingefde28e82008-07-24 16:28:00 -0700222 reservation.nr_extents = nr_pages;
223 rc = HYPERVISOR_memory_op(XENMEM_populate_physmap, &reservation);
Ian Campbellbc2c0302009-06-05 11:58:37 +0100224 if (rc < 0)
Jeremy Fitzhardinge17758262008-04-02 10:54:13 -0700225 goto out;
Jeremy Fitzhardinge17758262008-04-02 10:54:13 -0700226
Ian Campbellbc2c0302009-06-05 11:58:37 +0100227 for (i = 0; i < rc; i++) {
Jeremy Fitzhardinge17758262008-04-02 10:54:13 -0700228 page = balloon_retrieve();
229 BUG_ON(page == NULL);
230
231 pfn = page_to_pfn(page);
232 BUG_ON(!xen_feature(XENFEAT_auto_translated_physmap) &&
233 phys_to_machine_mapping_valid(pfn));
234
235 set_phys_to_machine(pfn, frame_list[i]);
236
237 /* Link back into the page tables if not highmem. */
238 if (pfn < max_low_pfn) {
239 int ret;
240 ret = HYPERVISOR_update_va_mapping(
241 (unsigned long)__va(pfn << PAGE_SHIFT),
242 mfn_pte(frame_list[i], PAGE_KERNEL),
243 0);
244 BUG_ON(ret);
245 }
246
247 /* Relinquish the page back to the allocator. */
248 ClearPageReserved(page);
249 init_page_count(page);
250 __free_page(page);
251 }
252
Ian Campbellbc2c0302009-06-05 11:58:37 +0100253 balloon_stats.current_pages += rc;
Jeremy Fitzhardinge17758262008-04-02 10:54:13 -0700254
255 out:
256 spin_unlock_irqrestore(&balloon_lock, flags);
257
Ian Campbellbc2c0302009-06-05 11:58:37 +0100258 return rc < 0 ? rc : rc != nr_pages;
Jeremy Fitzhardinge17758262008-04-02 10:54:13 -0700259}
260
261static int decrease_reservation(unsigned long nr_pages)
262{
263 unsigned long pfn, i, flags;
264 struct page *page;
265 int need_sleep = 0;
266 int ret;
267 struct xen_memory_reservation reservation = {
268 .address_bits = 0,
269 .extent_order = 0,
270 .domid = DOMID_SELF
271 };
272
273 if (nr_pages > ARRAY_SIZE(frame_list))
274 nr_pages = ARRAY_SIZE(frame_list);
275
276 for (i = 0; i < nr_pages; i++) {
277 if ((page = alloc_page(GFP_BALLOON)) == NULL) {
278 nr_pages = i;
279 need_sleep = 1;
280 break;
281 }
282
283 pfn = page_to_pfn(page);
284 frame_list[i] = pfn_to_mfn(pfn);
285
286 scrub_page(page);
Dan Magenheimer1058a752009-01-22 14:36:08 -0800287
Ian Campbellff4ce8c2009-01-23 16:26:21 +0000288 if (!PageHighMem(page)) {
289 ret = HYPERVISOR_update_va_mapping(
290 (unsigned long)__va(pfn << PAGE_SHIFT),
291 __pte_ma(0), 0);
292 BUG_ON(ret);
293 }
294
Jeremy Fitzhardinge17758262008-04-02 10:54:13 -0700295 }
296
297 /* Ensure that ballooned highmem pages don't have kmaps. */
298 kmap_flush_unused();
299 flush_tlb_all();
300
301 spin_lock_irqsave(&balloon_lock, flags);
302
303 /* No more mappings: invalidate P2M and add to balloon. */
304 for (i = 0; i < nr_pages; i++) {
305 pfn = mfn_to_pfn(frame_list[i]);
306 set_phys_to_machine(pfn, INVALID_P2M_ENTRY);
307 balloon_append(pfn_to_page(pfn));
308 }
309
Isaku Yamahataa90971e2008-05-26 23:31:14 +0100310 set_xen_guest_handle(reservation.extent_start, frame_list);
Jeremy Fitzhardinge17758262008-04-02 10:54:13 -0700311 reservation.nr_extents = nr_pages;
312 ret = HYPERVISOR_memory_op(XENMEM_decrease_reservation, &reservation);
313 BUG_ON(ret != nr_pages);
314
315 balloon_stats.current_pages -= nr_pages;
Jeremy Fitzhardinge17758262008-04-02 10:54:13 -0700316
317 spin_unlock_irqrestore(&balloon_lock, flags);
318
319 return need_sleep;
320}
321
322/*
323 * We avoid multiple worker processes conflicting via the balloon mutex.
324 * We may of course race updates of the target counts (which are protected
325 * by the balloon lock), or with changes to the Xen hard limit, but we will
326 * recover from these in time.
327 */
328static void balloon_process(struct work_struct *work)
329{
330 int need_sleep = 0;
331 long credit;
332
333 mutex_lock(&balloon_mutex);
334
335 do {
336 credit = current_target() - balloon_stats.current_pages;
337 if (credit > 0)
338 need_sleep = (increase_reservation(credit) != 0);
339 if (credit < 0)
340 need_sleep = (decrease_reservation(-credit) != 0);
341
342#ifndef CONFIG_PREEMPT
343 if (need_resched())
344 schedule();
345#endif
346 } while ((credit != 0) && !need_sleep);
347
348 /* Schedule more work if there is some still to be done. */
349 if (current_target() != balloon_stats.current_pages)
350 mod_timer(&balloon_timer, jiffies + HZ);
351
352 mutex_unlock(&balloon_mutex);
353}
354
355/* Resets the Xen limit, sets new target, and kicks off processing. */
Adrian Bunk955d6f12008-05-26 23:31:17 +0100356static void balloon_set_new_target(unsigned long target)
Jeremy Fitzhardinge17758262008-04-02 10:54:13 -0700357{
358 /* No need for lock. Not read-modify-write updates. */
Jeremy Fitzhardinge17758262008-04-02 10:54:13 -0700359 balloon_stats.target_pages = target;
360 schedule_work(&balloon_worker);
361}
362
363static struct xenbus_watch target_watch =
364{
365 .node = "memory/target"
366};
367
368/* React to a change in the target key */
369static void watch_target(struct xenbus_watch *watch,
370 const char **vec, unsigned int len)
371{
372 unsigned long long new_target;
373 int err;
374
375 err = xenbus_scanf(XBT_NIL, "memory", "target", "%llu", &new_target);
376 if (err != 1) {
377 /* This is ok (for domain0 at least) - so just return */
378 return;
379 }
380
381 /* The given memory/target value is in KiB, so it needs converting to
382 * pages. PAGE_SHIFT converts bytes to pages, hence PAGE_SHIFT - 10.
383 */
384 balloon_set_new_target(new_target >> (PAGE_SHIFT - 10));
385}
386
387static int balloon_init_watcher(struct notifier_block *notifier,
388 unsigned long event,
389 void *data)
390{
391 int err;
392
393 err = register_xenbus_watch(&target_watch);
394 if (err)
395 printk(KERN_ERR "Failed to set balloon watcher\n");
396
397 return NOTIFY_DONE;
398}
399
400static struct notifier_block xenstore_notifier;
401
402static int __init balloon_init(void)
403{
404 unsigned long pfn;
405 struct page *page;
406
Jeremy Fitzhardinge6e833582008-08-19 13:16:17 -0700407 if (!xen_pv_domain())
Jeremy Fitzhardinge17758262008-04-02 10:54:13 -0700408 return -ENODEV;
409
410 pr_info("xen_balloon: Initialising balloon driver.\n");
411
412 balloon_stats.current_pages = min(xen_start_info->nr_pages, max_pfn);
Jeremy Fitzhardinge17758262008-04-02 10:54:13 -0700413 balloon_stats.target_pages = balloon_stats.current_pages;
414 balloon_stats.balloon_low = 0;
415 balloon_stats.balloon_high = 0;
416 balloon_stats.driver_pages = 0UL;
Jeremy Fitzhardinge17758262008-04-02 10:54:13 -0700417
418 init_timer(&balloon_timer);
419 balloon_timer.data = 0;
420 balloon_timer.function = balloon_alarm;
421
422 register_balloon(&balloon_sysdev);
423
424 /* Initialise the balloon with excess memory space. */
425 for (pfn = xen_start_info->nr_pages; pfn < max_pfn; pfn++) {
426 page = pfn_to_page(pfn);
427 if (!PageReserved(page))
428 balloon_append(page);
429 }
430
431 target_watch.callback = watch_target;
432 xenstore_notifier.notifier_call = balloon_init_watcher;
433
434 register_xenstore_notifier(&xenstore_notifier);
435
436 return 0;
437}
438
439subsys_initcall(balloon_init);
440
441static void balloon_exit(void)
442{
443 /* XXX - release balloon here */
444 return;
445}
446
447module_exit(balloon_exit);
448
Jeremy Fitzhardinge167e6cf2008-07-24 16:27:52 -0700449#define BALLOON_SHOW(name, format, args...) \
450 static ssize_t show_##name(struct sys_device *dev, \
451 struct sysdev_attribute *attr, \
452 char *buf) \
453 { \
454 return sprintf(buf, format, ##args); \
455 } \
Jeremy Fitzhardinge17758262008-04-02 10:54:13 -0700456 static SYSDEV_ATTR(name, S_IRUGO, show_##name, NULL)
457
458BALLOON_SHOW(current_kb, "%lu\n", PAGES2KB(balloon_stats.current_pages));
459BALLOON_SHOW(low_kb, "%lu\n", PAGES2KB(balloon_stats.balloon_low));
460BALLOON_SHOW(high_kb, "%lu\n", PAGES2KB(balloon_stats.balloon_high));
Jeremy Fitzhardinge17758262008-04-02 10:54:13 -0700461BALLOON_SHOW(driver_kb, "%lu\n", PAGES2KB(balloon_stats.driver_pages));
462
Jeremy Fitzhardinge167e6cf2008-07-24 16:27:52 -0700463static ssize_t show_target_kb(struct sys_device *dev, struct sysdev_attribute *attr,
464 char *buf)
Jeremy Fitzhardinge17758262008-04-02 10:54:13 -0700465{
466 return sprintf(buf, "%lu\n", PAGES2KB(balloon_stats.target_pages));
467}
468
469static ssize_t store_target_kb(struct sys_device *dev,
Andi Kleen4a0b2b42008-07-01 18:48:41 +0200470 struct sysdev_attribute *attr,
Jeremy Fitzhardinge17758262008-04-02 10:54:13 -0700471 const char *buf,
472 size_t count)
473{
Jeremy Fitzhardinge167e6cf2008-07-24 16:27:52 -0700474 char *endchar;
Jeremy Fitzhardinge17758262008-04-02 10:54:13 -0700475 unsigned long long target_bytes;
476
477 if (!capable(CAP_SYS_ADMIN))
478 return -EPERM;
479
Jeremy Fitzhardinge618b2c82009-01-28 16:50:20 -0800480 target_bytes = simple_strtoull(buf, &endchar, 0) * 1024;
Jeremy Fitzhardinge17758262008-04-02 10:54:13 -0700481
Jeremy Fitzhardinge17758262008-04-02 10:54:13 -0700482 balloon_set_new_target(target_bytes >> PAGE_SHIFT);
483
484 return count;
485}
486
487static SYSDEV_ATTR(target_kb, S_IRUGO | S_IWUSR,
488 show_target_kb, store_target_kb);
489
Jeremy Fitzhardinge618b2c82009-01-28 16:50:20 -0800490
491static ssize_t show_target(struct sys_device *dev, struct sysdev_attribute *attr,
492 char *buf)
493{
494 return sprintf(buf, "%llu\n",
Jan Beulich06926982009-05-05 13:57:52 +0100495 (unsigned long long)balloon_stats.target_pages
496 << PAGE_SHIFT);
Jeremy Fitzhardinge618b2c82009-01-28 16:50:20 -0800497}
498
499static ssize_t store_target(struct sys_device *dev,
500 struct sysdev_attribute *attr,
501 const char *buf,
502 size_t count)
503{
504 char *endchar;
505 unsigned long long target_bytes;
506
507 if (!capable(CAP_SYS_ADMIN))
508 return -EPERM;
509
510 target_bytes = memparse(buf, &endchar);
511
512 balloon_set_new_target(target_bytes >> PAGE_SHIFT);
513
514 return count;
515}
516
517static SYSDEV_ATTR(target, S_IRUGO | S_IWUSR,
518 show_target, store_target);
519
520
Jeremy Fitzhardinge17758262008-04-02 10:54:13 -0700521static struct sysdev_attribute *balloon_attrs[] = {
522 &attr_target_kb,
Jeremy Fitzhardinge618b2c82009-01-28 16:50:20 -0800523 &attr_target,
Jeremy Fitzhardinge17758262008-04-02 10:54:13 -0700524};
525
526static struct attribute *balloon_info_attrs[] = {
527 &attr_current_kb.attr,
528 &attr_low_kb.attr,
529 &attr_high_kb.attr,
Jeremy Fitzhardinge17758262008-04-02 10:54:13 -0700530 &attr_driver_kb.attr,
531 NULL
532};
533
534static struct attribute_group balloon_info_group = {
535 .name = "info",
536 .attrs = balloon_info_attrs,
537};
538
539static struct sysdev_class balloon_sysdev_class = {
540 .name = BALLOON_CLASS_NAME,
541};
542
543static int register_balloon(struct sys_device *sysdev)
544{
545 int i, error;
546
547 error = sysdev_class_register(&balloon_sysdev_class);
548 if (error)
549 return error;
550
551 sysdev->id = 0;
552 sysdev->cls = &balloon_sysdev_class;
553
554 error = sysdev_register(sysdev);
555 if (error) {
556 sysdev_class_unregister(&balloon_sysdev_class);
557 return error;
558 }
559
560 for (i = 0; i < ARRAY_SIZE(balloon_attrs); i++) {
561 error = sysdev_create_file(sysdev, balloon_attrs[i]);
562 if (error)
563 goto fail;
564 }
565
566 error = sysfs_create_group(&sysdev->kobj, &balloon_info_group);
567 if (error)
568 goto fail;
569
570 return 0;
571
572 fail:
573 while (--i >= 0)
574 sysdev_remove_file(sysdev, balloon_attrs[i]);
575 sysdev_unregister(sysdev);
576 sysdev_class_unregister(&balloon_sysdev_class);
577 return error;
578}
579
Jeremy Fitzhardinge17758262008-04-02 10:54:13 -0700580MODULE_LICENSE("GPL");