blob: c444654fc33fb6f7e858824eb8ed816186bf3bda [file] [log] [blame]
Sasha Levin1e214a52011-11-03 10:20:04 +02001/*
2 * Virtio balloon implementation, inspired by Dor Laor and Marcelo
Rusty Russell6b35e402008-02-04 23:50:12 -05003 * Tosatti's implementations.
4 *
5 * Copyright 2008 Rusty Russell IBM Corporation
6 *
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License as published by
9 * the Free Software Foundation; either version 2 of the License, or
10 * (at your option) any later version.
11 *
12 * This program is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 * GNU General Public License for more details.
16 *
17 * You should have received a copy of the GNU General Public License
18 * along with this program; if not, write to the Free Software
19 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
20 */
Sasha Levin1e214a52011-11-03 10:20:04 +020021
Rusty Russell6b35e402008-02-04 23:50:12 -050022#include <linux/virtio.h>
23#include <linux/virtio_balloon.h>
24#include <linux/swap.h>
25#include <linux/kthread.h>
26#include <linux/freezer.h>
Johann Felix Soden6659a0f2008-02-06 01:40:22 -080027#include <linux/delay.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090028#include <linux/slab.h>
Paul Gortmakerb5a2c4f2011-07-03 16:20:30 -040029#include <linux/module.h>
Rafael Aquinie2250422012-12-11 16:02:45 -080030#include <linux/balloon_compaction.h>
Rusty Russell6b35e402008-02-04 23:50:12 -050031
Michael S. Tsirkin3ccc9372012-04-12 16:38:00 +030032/*
33 * Balloon device works in 4K page units. So each page is pointed to by
34 * multiple balloon pages. All memory counters in this driver are in balloon
35 * page units.
36 */
Rafael Aquinie2250422012-12-11 16:02:45 -080037#define VIRTIO_BALLOON_PAGES_PER_PAGE (unsigned)(PAGE_SIZE >> VIRTIO_BALLOON_PFN_SHIFT)
38#define VIRTIO_BALLOON_ARRAY_PFNS_MAX 256
Michael S. Tsirkin3ccc9372012-04-12 16:38:00 +030039
Rusty Russell6b35e402008-02-04 23:50:12 -050040struct virtio_balloon
41{
42 struct virtio_device *vdev;
Adam Litke9564e132009-11-30 10:14:15 -060043 struct virtqueue *inflate_vq, *deflate_vq, *stats_vq;
Rusty Russell6b35e402008-02-04 23:50:12 -050044
45 /* Where the ballooning thread waits for config to change. */
46 wait_queue_head_t config_change;
47
48 /* The thread servicing the balloon. */
49 struct task_struct *thread;
50
51 /* Waiting for host to ack the pages we released. */
Michael S. Tsirkin9c378ab2012-07-02 10:33:08 +030052 wait_queue_head_t acked;
Rusty Russell6b35e402008-02-04 23:50:12 -050053
Michael S. Tsirkin3ccc9372012-04-12 16:38:00 +030054 /* Number of balloon pages we've told the Host we're not using. */
Rusty Russell6b35e402008-02-04 23:50:12 -050055 unsigned int num_pages;
Michael S. Tsirkin3ccc9372012-04-12 16:38:00 +030056 /*
Rafael Aquinie2250422012-12-11 16:02:45 -080057 * The pages we've told the Host we're not using are enqueued
58 * at vb_dev_info->pages list.
Michael S. Tsirkin3ccc9372012-04-12 16:38:00 +030059 * Each page on this list adds VIRTIO_BALLOON_PAGES_PER_PAGE
60 * to num_pages above.
61 */
Rafael Aquinie2250422012-12-11 16:02:45 -080062 struct balloon_dev_info *vb_dev_info;
63
64 /* Synchronize access/update to this struct virtio_balloon elements */
65 struct mutex balloon_lock;
Rusty Russell6b35e402008-02-04 23:50:12 -050066
67 /* The array of pfns we tell the Host about. */
68 unsigned int num_pfns;
Rafael Aquinie2250422012-12-11 16:02:45 -080069 u32 pfns[VIRTIO_BALLOON_ARRAY_PFNS_MAX];
Adam Litke9564e132009-11-30 10:14:15 -060070
71 /* Memory statistics */
Adam Litke1f34c712009-12-10 16:35:15 -060072 int need_stats_update;
Adam Litke9564e132009-11-30 10:14:15 -060073 struct virtio_balloon_stat stats[VIRTIO_BALLOON_S_NR];
Rusty Russell6b35e402008-02-04 23:50:12 -050074};
75
76static struct virtio_device_id id_table[] = {
77 { VIRTIO_ID_BALLOON, VIRTIO_DEV_ANY_ID },
78 { 0 },
79};
80
Hollis Blanchard1b4aa2f2008-11-13 15:48:33 -060081static u32 page_to_balloon_pfn(struct page *page)
82{
83 unsigned long pfn = page_to_pfn(page);
84
85 BUILD_BUG_ON(PAGE_SHIFT < VIRTIO_BALLOON_PFN_SHIFT);
86 /* Convert pfn from Linux page size to balloon page size. */
Michael S. Tsirkin3ccc9372012-04-12 16:38:00 +030087 return pfn * VIRTIO_BALLOON_PAGES_PER_PAGE;
88}
89
90static struct page *balloon_pfn_to_page(u32 pfn)
91{
92 BUG_ON(pfn % VIRTIO_BALLOON_PAGES_PER_PAGE);
93 return pfn_to_page(pfn / VIRTIO_BALLOON_PAGES_PER_PAGE);
Hollis Blanchard1b4aa2f2008-11-13 15:48:33 -060094}
95
Rusty Russell6b35e402008-02-04 23:50:12 -050096static void balloon_ack(struct virtqueue *vq)
97{
Michael S. Tsirkin9c378ab2012-07-02 10:33:08 +030098 struct virtio_balloon *vb = vq->vdev->priv;
Rusty Russell6b35e402008-02-04 23:50:12 -050099
Michael S. Tsirkin9c378ab2012-07-02 10:33:08 +0300100 wake_up(&vb->acked);
Rusty Russell6b35e402008-02-04 23:50:12 -0500101}
102
103static void tell_host(struct virtio_balloon *vb, struct virtqueue *vq)
104{
105 struct scatterlist sg;
Michael S. Tsirkin9c378ab2012-07-02 10:33:08 +0300106 unsigned int len;
Rusty Russell6b35e402008-02-04 23:50:12 -0500107
108 sg_init_one(&sg, vb->pfns, sizeof(vb->pfns[0]) * vb->num_pfns);
109
Rusty Russell6b35e402008-02-04 23:50:12 -0500110 /* We should always be able to add one buffer to an empty queue. */
Rusty Russell92549ab2013-03-20 15:44:30 +1030111 if (virtqueue_add_outbuf(vq, &sg, 1, vb, GFP_KERNEL) < 0)
Rusty Russell6b35e402008-02-04 23:50:12 -0500112 BUG();
Michael S. Tsirkin946cfe02010-04-12 16:18:28 +0300113 virtqueue_kick(vq);
Rusty Russell6b35e402008-02-04 23:50:12 -0500114
115 /* When host has read buffer, this completes via balloon_ack */
Michael S. Tsirkin9c378ab2012-07-02 10:33:08 +0300116 wait_event(vb->acked, virtqueue_get_buf(vq, &len));
Rusty Russell6b35e402008-02-04 23:50:12 -0500117}
118
Michael S. Tsirkin3ccc9372012-04-12 16:38:00 +0300119static void set_page_pfns(u32 pfns[], struct page *page)
120{
121 unsigned int i;
122
123 /* Set balloon pfns pointing at this page.
124 * Note that the first pfn points at start of the page. */
125 for (i = 0; i < VIRTIO_BALLOON_PAGES_PER_PAGE; i++)
126 pfns[i] = page_to_balloon_pfn(page) + i;
127}
128
Rusty Russell6b35e402008-02-04 23:50:12 -0500129static void fill_balloon(struct virtio_balloon *vb, size_t num)
130{
Rafael Aquinie2250422012-12-11 16:02:45 -0800131 struct balloon_dev_info *vb_dev_info = vb->vb_dev_info;
132
Rusty Russell6b35e402008-02-04 23:50:12 -0500133 /* We can only do one array worth at a time. */
134 num = min(num, ARRAY_SIZE(vb->pfns));
135
Rafael Aquinie2250422012-12-11 16:02:45 -0800136 mutex_lock(&vb->balloon_lock);
Michael S. Tsirkin3ccc9372012-04-12 16:38:00 +0300137 for (vb->num_pfns = 0; vb->num_pfns < num;
138 vb->num_pfns += VIRTIO_BALLOON_PAGES_PER_PAGE) {
Rafael Aquinie2250422012-12-11 16:02:45 -0800139 struct page *page = balloon_page_enqueue(vb_dev_info);
140
Rusty Russell6b35e402008-02-04 23:50:12 -0500141 if (!page) {
Joe Perches800ba5e2012-10-31 09:27:22 +1030142 dev_info_ratelimited(&vb->vdev->dev,
Linus Torvaldsb7dfde92012-12-20 08:37:04 -0800143 "Out of puff! Can't get %u pages\n",
144 VIRTIO_BALLOON_PAGES_PER_PAGE);
Rusty Russell6b35e402008-02-04 23:50:12 -0500145 /* Sleep for at least 1/5 of a second before retry. */
146 msleep(200);
147 break;
148 }
Michael S. Tsirkin3ccc9372012-04-12 16:38:00 +0300149 set_page_pfns(vb->pfns + vb->num_pfns, page);
150 vb->num_pages += VIRTIO_BALLOON_PAGES_PER_PAGE;
Jiang Liu3dcc0572013-07-03 15:03:21 -0700151 adjust_managed_page_count(page, -1);
Rusty Russell6b35e402008-02-04 23:50:12 -0500152 }
153
Rafael Aquinie2250422012-12-11 16:02:45 -0800154 /* Did we get any? */
155 if (vb->num_pfns != 0)
156 tell_host(vb, vb->inflate_vq);
157 mutex_unlock(&vb->balloon_lock);
Rusty Russell6b35e402008-02-04 23:50:12 -0500158}
159
160static void release_pages_by_pfn(const u32 pfns[], unsigned int num)
161{
162 unsigned int i;
163
Michael S. Tsirkin3ccc9372012-04-12 16:38:00 +0300164 /* Find pfns pointing at start of each page, get pages and free them. */
165 for (i = 0; i < num; i += VIRTIO_BALLOON_PAGES_PER_PAGE) {
Jiang Liu3dcc0572013-07-03 15:03:21 -0700166 struct page *page = balloon_pfn_to_page(pfns[i]);
167 balloon_page_free(page);
168 adjust_managed_page_count(page, 1);
Rusty Russell6b35e402008-02-04 23:50:12 -0500169 }
170}
171
172static void leak_balloon(struct virtio_balloon *vb, size_t num)
173{
174 struct page *page;
Rafael Aquinie2250422012-12-11 16:02:45 -0800175 struct balloon_dev_info *vb_dev_info = vb->vb_dev_info;
Rusty Russell6b35e402008-02-04 23:50:12 -0500176
177 /* We can only do one array worth at a time. */
178 num = min(num, ARRAY_SIZE(vb->pfns));
179
Rafael Aquinie2250422012-12-11 16:02:45 -0800180 mutex_lock(&vb->balloon_lock);
Michael S. Tsirkin3ccc9372012-04-12 16:38:00 +0300181 for (vb->num_pfns = 0; vb->num_pfns < num;
182 vb->num_pfns += VIRTIO_BALLOON_PAGES_PER_PAGE) {
Rafael Aquinie2250422012-12-11 16:02:45 -0800183 page = balloon_page_dequeue(vb_dev_info);
184 if (!page)
185 break;
Michael S. Tsirkin3ccc9372012-04-12 16:38:00 +0300186 set_page_pfns(vb->pfns + vb->num_pfns, page);
187 vb->num_pages -= VIRTIO_BALLOON_PAGES_PER_PAGE;
Rusty Russell6b35e402008-02-04 23:50:12 -0500188 }
189
Dave Hansenbf50e692011-04-07 10:43:25 -0700190 /*
191 * Note that if
192 * virtio_has_feature(vdev, VIRTIO_BALLOON_F_MUST_TELL_HOST);
193 * is true, we *have* to do it in this order
194 */
Luiz Capitulino8c6bab42013-07-02 15:35:13 +0930195 if (vb->num_pfns != 0)
196 tell_host(vb, vb->deflate_vq);
Rafael Aquinie2250422012-12-11 16:02:45 -0800197 mutex_unlock(&vb->balloon_lock);
Dave Hansenbf50e692011-04-07 10:43:25 -0700198 release_pages_by_pfn(vb->pfns, vb->num_pfns);
Rusty Russell6b35e402008-02-04 23:50:12 -0500199}
200
Adam Litke9564e132009-11-30 10:14:15 -0600201static inline void update_stat(struct virtio_balloon *vb, int idx,
202 u16 tag, u64 val)
203{
204 BUG_ON(idx >= VIRTIO_BALLOON_S_NR);
205 vb->stats[idx].tag = tag;
206 vb->stats[idx].val = val;
207}
208
209#define pages_to_bytes(x) ((u64)(x) << PAGE_SHIFT)
210
211static void update_balloon_stats(struct virtio_balloon *vb)
212{
213 unsigned long events[NR_VM_EVENT_ITEMS];
214 struct sysinfo i;
215 int idx = 0;
216
217 all_vm_events(events);
218 si_meminfo(&i);
219
220 update_stat(vb, idx++, VIRTIO_BALLOON_S_SWAP_IN,
221 pages_to_bytes(events[PSWPIN]));
222 update_stat(vb, idx++, VIRTIO_BALLOON_S_SWAP_OUT,
223 pages_to_bytes(events[PSWPOUT]));
224 update_stat(vb, idx++, VIRTIO_BALLOON_S_MAJFLT, events[PGMAJFAULT]);
225 update_stat(vb, idx++, VIRTIO_BALLOON_S_MINFLT, events[PGFAULT]);
226 update_stat(vb, idx++, VIRTIO_BALLOON_S_MEMFREE,
227 pages_to_bytes(i.freeram));
228 update_stat(vb, idx++, VIRTIO_BALLOON_S_MEMTOT,
229 pages_to_bytes(i.totalram));
230}
231
232/*
233 * While most virtqueues communicate guest-initiated requests to the hypervisor,
234 * the stats queue operates in reverse. The driver initializes the virtqueue
235 * with a single buffer. From that point forward, all conversations consist of
236 * a hypervisor request (a call to this function) which directs us to refill
Adam Litke1f34c712009-12-10 16:35:15 -0600237 * the virtqueue with a fresh stats buffer. Since stats collection can sleep,
238 * we notify our kthread which does the actual work via stats_handle_request().
Adam Litke9564e132009-11-30 10:14:15 -0600239 */
Adam Litke1f34c712009-12-10 16:35:15 -0600240static void stats_request(struct virtqueue *vq)
Adam Litke9564e132009-11-30 10:14:15 -0600241{
Michael S. Tsirkin9c378ab2012-07-02 10:33:08 +0300242 struct virtio_balloon *vb = vq->vdev->priv;
Adam Litke9564e132009-11-30 10:14:15 -0600243
Adam Litke1f34c712009-12-10 16:35:15 -0600244 vb->need_stats_update = 1;
245 wake_up(&vb->config_change);
246}
Adam Litke9564e132009-11-30 10:14:15 -0600247
Adam Litke1f34c712009-12-10 16:35:15 -0600248static void stats_handle_request(struct virtio_balloon *vb)
249{
250 struct virtqueue *vq;
251 struct scatterlist sg;
Michael S. Tsirkin9c378ab2012-07-02 10:33:08 +0300252 unsigned int len;
Adam Litke1f34c712009-12-10 16:35:15 -0600253
254 vb->need_stats_update = 0;
Adam Litke9564e132009-11-30 10:14:15 -0600255 update_balloon_stats(vb);
256
Adam Litke1f34c712009-12-10 16:35:15 -0600257 vq = vb->stats_vq;
Michael S. Tsirkin9c378ab2012-07-02 10:33:08 +0300258 if (!virtqueue_get_buf(vq, &len))
259 return;
Adam Litke9564e132009-11-30 10:14:15 -0600260 sg_init_one(&sg, vb->stats, sizeof(vb->stats));
Rusty Russell92549ab2013-03-20 15:44:30 +1030261 if (virtqueue_add_outbuf(vq, &sg, 1, vb, GFP_KERNEL) < 0)
Adam Litke9564e132009-11-30 10:14:15 -0600262 BUG();
Michael S. Tsirkin946cfe02010-04-12 16:18:28 +0300263 virtqueue_kick(vq);
Adam Litke9564e132009-11-30 10:14:15 -0600264}
265
Rusty Russell6b35e402008-02-04 23:50:12 -0500266static void virtballoon_changed(struct virtio_device *vdev)
267{
268 struct virtio_balloon *vb = vdev->priv;
269
270 wake_up(&vb->config_change);
271}
272
Rusty Russellbdc16812008-03-17 22:58:15 -0500273static inline s64 towards_target(struct virtio_balloon *vb)
Rusty Russell6b35e402008-02-04 23:50:12 -0500274{
David Gibson1a872282012-04-12 15:36:34 +1000275 __le32 v;
276 s64 target;
277
Rusty Russell855e0c52013-10-14 18:11:51 +1030278 virtio_cread(vb->vdev, struct virtio_balloon_config, num_pages, &v);
279
David Gibson1a872282012-04-12 15:36:34 +1000280 target = le32_to_cpu(v);
281 return target - vb->num_pages;
Rusty Russell6b35e402008-02-04 23:50:12 -0500282}
283
284static void update_balloon_size(struct virtio_balloon *vb)
285{
286 __le32 actual = cpu_to_le32(vb->num_pages);
287
Rusty Russell855e0c52013-10-14 18:11:51 +1030288 virtio_cwrite(vb->vdev, struct virtio_balloon_config, num_pages,
289 &actual);
Rusty Russell6b35e402008-02-04 23:50:12 -0500290}
291
292static int balloon(void *_vballoon)
293{
294 struct virtio_balloon *vb = _vballoon;
295
296 set_freezable();
297 while (!kthread_should_stop()) {
Rusty Russellbdc16812008-03-17 22:58:15 -0500298 s64 diff;
Rusty Russell6b35e402008-02-04 23:50:12 -0500299
300 try_to_freeze();
301 wait_event_interruptible(vb->config_change,
302 (diff = towards_target(vb)) != 0
Adam Litke1f34c712009-12-10 16:35:15 -0600303 || vb->need_stats_update
Marcelo Tosatti84a139a2009-04-16 21:14:04 -0300304 || kthread_should_stop()
305 || freezing(current));
Adam Litke1f34c712009-12-10 16:35:15 -0600306 if (vb->need_stats_update)
307 stats_handle_request(vb);
Rusty Russell6b35e402008-02-04 23:50:12 -0500308 if (diff > 0)
309 fill_balloon(vb, diff);
310 else if (diff < 0)
311 leak_balloon(vb, -diff);
312 update_balloon_size(vb);
313 }
314 return 0;
315}
316
Amit Shahbe91c332011-12-22 16:58:34 +0530317static int init_vqs(struct virtio_balloon *vb)
Rusty Russell6b35e402008-02-04 23:50:12 -0500318{
Adam Litke9564e132009-11-30 10:14:15 -0600319 struct virtqueue *vqs[3];
Adam Litke1f34c712009-12-10 16:35:15 -0600320 vq_callback_t *callbacks[] = { balloon_ack, balloon_ack, stats_request };
Adam Litke9564e132009-11-30 10:14:15 -0600321 const char *names[] = { "inflate", "deflate", "stats" };
322 int err, nvqs;
Rusty Russell6b35e402008-02-04 23:50:12 -0500323
Amit Shahbe91c332011-12-22 16:58:34 +0530324 /*
325 * We expect two virtqueues: inflate and deflate, and
326 * optionally stat.
327 */
Adam Litke9564e132009-11-30 10:14:15 -0600328 nvqs = virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_STATS_VQ) ? 3 : 2;
Amit Shahbe91c332011-12-22 16:58:34 +0530329 err = vb->vdev->config->find_vqs(vb->vdev, nvqs, vqs, callbacks, names);
Michael S. Tsirkind2a7ddd2009-06-12 22:16:36 -0600330 if (err)
Amit Shahbe91c332011-12-22 16:58:34 +0530331 return err;
Rusty Russell6b35e402008-02-04 23:50:12 -0500332
Michael S. Tsirkind2a7ddd2009-06-12 22:16:36 -0600333 vb->inflate_vq = vqs[0];
334 vb->deflate_vq = vqs[1];
Adam Litke9564e132009-11-30 10:14:15 -0600335 if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_STATS_VQ)) {
336 struct scatterlist sg;
337 vb->stats_vq = vqs[2];
338
339 /*
340 * Prime this virtqueue with one buffer so the hypervisor can
341 * use it to signal us later.
342 */
343 sg_init_one(&sg, vb->stats, sizeof vb->stats);
Rusty Russell92549ab2013-03-20 15:44:30 +1030344 if (virtqueue_add_outbuf(vb->stats_vq, &sg, 1, vb, GFP_KERNEL)
Rusty Russellf96fde42012-01-12 15:44:42 +1030345 < 0)
Adam Litke9564e132009-11-30 10:14:15 -0600346 BUG();
Michael S. Tsirkin946cfe02010-04-12 16:18:28 +0300347 virtqueue_kick(vb->stats_vq);
Adam Litke9564e132009-11-30 10:14:15 -0600348 }
Amit Shahbe91c332011-12-22 16:58:34 +0530349 return 0;
350}
351
Rafael Aquinie2250422012-12-11 16:02:45 -0800352static const struct address_space_operations virtio_balloon_aops;
353#ifdef CONFIG_BALLOON_COMPACTION
354/*
355 * virtballoon_migratepage - perform the balloon page migration on behalf of
356 * a compation thread. (called under page lock)
357 * @mapping: the page->mapping which will be assigned to the new migrated page.
358 * @newpage: page that will replace the isolated page after migration finishes.
359 * @page : the isolated (old) page that is about to be migrated to newpage.
360 * @mode : compaction mode -- not used for balloon page migration.
361 *
362 * After a ballooned page gets isolated by compaction procedures, this is the
363 * function that performs the page migration on behalf of a compaction thread
364 * The page migration for virtio balloon is done in a simple swap fashion which
365 * follows these two macro steps:
366 * 1) insert newpage into vb->pages list and update the host about it;
367 * 2) update the host about the old page removed from vb->pages list;
368 *
369 * This function preforms the balloon page migration task.
370 * Called through balloon_mapping->a_ops->migratepage
371 */
372int virtballoon_migratepage(struct address_space *mapping,
373 struct page *newpage, struct page *page, enum migrate_mode mode)
374{
375 struct balloon_dev_info *vb_dev_info = balloon_page_device(page);
376 struct virtio_balloon *vb;
377 unsigned long flags;
378
379 BUG_ON(!vb_dev_info);
380
381 vb = vb_dev_info->balloon_device;
382
383 /*
384 * In order to avoid lock contention while migrating pages concurrently
385 * to leak_balloon() or fill_balloon() we just give up the balloon_lock
386 * this turn, as it is easier to retry the page migration later.
387 * This also prevents fill_balloon() getting stuck into a mutex
388 * recursion in the case it ends up triggering memory compaction
389 * while it is attempting to inflate the ballon.
390 */
391 if (!mutex_trylock(&vb->balloon_lock))
392 return -EAGAIN;
393
394 /* balloon's page migration 1st step -- inflate "newpage" */
395 spin_lock_irqsave(&vb_dev_info->pages_lock, flags);
396 balloon_page_insert(newpage, mapping, &vb_dev_info->pages);
397 vb_dev_info->isolated_pages--;
398 spin_unlock_irqrestore(&vb_dev_info->pages_lock, flags);
399 vb->num_pfns = VIRTIO_BALLOON_PAGES_PER_PAGE;
400 set_page_pfns(vb->pfns, newpage);
401 tell_host(vb, vb->inflate_vq);
402
403 /*
404 * balloon's page migration 2nd step -- deflate "page"
405 *
406 * It's safe to delete page->lru here because this page is at
407 * an isolated migration list, and this step is expected to happen here
408 */
409 balloon_page_delete(page);
410 vb->num_pfns = VIRTIO_BALLOON_PAGES_PER_PAGE;
411 set_page_pfns(vb->pfns, page);
412 tell_host(vb, vb->deflate_vq);
413
414 mutex_unlock(&vb->balloon_lock);
415
416 return MIGRATEPAGE_BALLOON_SUCCESS;
417}
418
419/* define the balloon_mapping->a_ops callback to allow balloon page migration */
420static const struct address_space_operations virtio_balloon_aops = {
421 .migratepage = virtballoon_migratepage,
422};
423#endif /* CONFIG_BALLOON_COMPACTION */
424
Amit Shahbe91c332011-12-22 16:58:34 +0530425static int virtballoon_probe(struct virtio_device *vdev)
426{
427 struct virtio_balloon *vb;
Rafael Aquinie2250422012-12-11 16:02:45 -0800428 struct address_space *vb_mapping;
429 struct balloon_dev_info *vb_devinfo;
Amit Shahbe91c332011-12-22 16:58:34 +0530430 int err;
431
432 vdev->priv = vb = kmalloc(sizeof(*vb), GFP_KERNEL);
433 if (!vb) {
434 err = -ENOMEM;
435 goto out;
436 }
437
Amit Shahbe91c332011-12-22 16:58:34 +0530438 vb->num_pages = 0;
Rafael Aquinie2250422012-12-11 16:02:45 -0800439 mutex_init(&vb->balloon_lock);
Amit Shahbe91c332011-12-22 16:58:34 +0530440 init_waitqueue_head(&vb->config_change);
Michael S. Tsirkin9c378ab2012-07-02 10:33:08 +0300441 init_waitqueue_head(&vb->acked);
Amit Shahbe91c332011-12-22 16:58:34 +0530442 vb->vdev = vdev;
443 vb->need_stats_update = 0;
444
Rafael Aquinie2250422012-12-11 16:02:45 -0800445 vb_devinfo = balloon_devinfo_alloc(vb);
446 if (IS_ERR(vb_devinfo)) {
447 err = PTR_ERR(vb_devinfo);
448 goto out_free_vb;
449 }
450
451 vb_mapping = balloon_mapping_alloc(vb_devinfo,
452 (balloon_compaction_check()) ?
453 &virtio_balloon_aops : NULL);
454 if (IS_ERR(vb_mapping)) {
455 /*
456 * IS_ERR(vb_mapping) && PTR_ERR(vb_mapping) == -EOPNOTSUPP
457 * This means !CONFIG_BALLOON_COMPACTION, otherwise we get off.
458 */
459 err = PTR_ERR(vb_mapping);
460 if (err != -EOPNOTSUPP)
461 goto out_free_vb_devinfo;
462 }
463
464 vb->vb_dev_info = vb_devinfo;
465
Amit Shahbe91c332011-12-22 16:58:34 +0530466 err = init_vqs(vb);
467 if (err)
Rafael Aquinie2250422012-12-11 16:02:45 -0800468 goto out_free_vb_mapping;
Rusty Russell6b35e402008-02-04 23:50:12 -0500469
470 vb->thread = kthread_run(balloon, vb, "vballoon");
471 if (IS_ERR(vb->thread)) {
472 err = PTR_ERR(vb->thread);
Michael S. Tsirkind2a7ddd2009-06-12 22:16:36 -0600473 goto out_del_vqs;
Rusty Russell6b35e402008-02-04 23:50:12 -0500474 }
475
Rusty Russell6b35e402008-02-04 23:50:12 -0500476 return 0;
477
Michael S. Tsirkind2a7ddd2009-06-12 22:16:36 -0600478out_del_vqs:
479 vdev->config->del_vqs(vdev);
Rafael Aquinie2250422012-12-11 16:02:45 -0800480out_free_vb_mapping:
481 balloon_mapping_free(vb_mapping);
482out_free_vb_devinfo:
483 balloon_devinfo_free(vb_devinfo);
Rusty Russell6b35e402008-02-04 23:50:12 -0500484out_free_vb:
485 kfree(vb);
486out:
487 return err;
488}
489
Amit Shahc877bab2012-04-27 00:45:57 +0530490static void remove_common(struct virtio_balloon *vb)
Rusty Russell6b35e402008-02-04 23:50:12 -0500491{
Rusty Russell6b35e402008-02-04 23:50:12 -0500492 /* There might be pages left in the balloon: free them. */
493 while (vb->num_pages)
494 leak_balloon(vb, vb->num_pages);
Amit Shahb8ae0eb2012-04-27 00:45:56 +0530495 update_balloon_size(vb);
Rusty Russell6b35e402008-02-04 23:50:12 -0500496
497 /* Now we reset the device so we can clean up the queues. */
Amit Shahc877bab2012-04-27 00:45:57 +0530498 vb->vdev->config->reset(vb->vdev);
Rusty Russell6b35e402008-02-04 23:50:12 -0500499
Amit Shahc877bab2012-04-27 00:45:57 +0530500 vb->vdev->config->del_vqs(vb->vdev);
501}
502
Greg Kroah-Hartman8590dbc2012-12-21 13:05:30 -0800503static void virtballoon_remove(struct virtio_device *vdev)
Amit Shahc877bab2012-04-27 00:45:57 +0530504{
505 struct virtio_balloon *vb = vdev->priv;
506
507 kthread_stop(vb->thread);
508 remove_common(vb);
Rafael Aquinie2250422012-12-11 16:02:45 -0800509 balloon_mapping_free(vb->vb_dev_info->mapping);
510 balloon_devinfo_free(vb->vb_dev_info);
Rusty Russell6b35e402008-02-04 23:50:12 -0500511 kfree(vb);
512}
513
Aaron Lu89107002013-09-17 09:25:23 +0930514#ifdef CONFIG_PM_SLEEP
Amit Shahe5629662011-12-22 16:58:35 +0530515static int virtballoon_freeze(struct virtio_device *vdev)
516{
Amit Shah4eb05d52012-02-29 17:42:51 +0530517 struct virtio_balloon *vb = vdev->priv;
518
Amit Shahe5629662011-12-22 16:58:35 +0530519 /*
520 * The kthread is already frozen by the PM core before this
521 * function is called.
522 */
523
Amit Shahc877bab2012-04-27 00:45:57 +0530524 remove_common(vb);
Amit Shahe5629662011-12-22 16:58:35 +0530525 return 0;
526}
527
Amit Shahc45b4162012-04-27 00:45:55 +0530528static int virtballoon_restore(struct virtio_device *vdev)
Amit Shah4eb05d52012-02-29 17:42:51 +0530529{
530 struct virtio_balloon *vb = vdev->priv;
531 int ret;
532
533 ret = init_vqs(vdev->priv);
534 if (ret)
535 return ret;
536
537 fill_balloon(vb, towards_target(vb));
538 update_balloon_size(vb);
539 return 0;
540}
Amit Shahe5629662011-12-22 16:58:35 +0530541#endif
542
Adam Litke9564e132009-11-30 10:14:15 -0600543static unsigned int features[] = {
544 VIRTIO_BALLOON_F_MUST_TELL_HOST,
545 VIRTIO_BALLOON_F_STATS_VQ,
546};
Rusty Russellc45a6812008-05-02 21:50:50 -0500547
Jeff Mahoneyd817cd52010-01-15 17:01:26 -0800548static struct virtio_driver virtio_balloon_driver = {
Rusty Russellc45a6812008-05-02 21:50:50 -0500549 .feature_table = features,
550 .feature_table_size = ARRAY_SIZE(features),
Rusty Russell6b35e402008-02-04 23:50:12 -0500551 .driver.name = KBUILD_MODNAME,
552 .driver.owner = THIS_MODULE,
553 .id_table = id_table,
554 .probe = virtballoon_probe,
Greg Kroah-Hartman8590dbc2012-12-21 13:05:30 -0800555 .remove = virtballoon_remove,
Rusty Russell6b35e402008-02-04 23:50:12 -0500556 .config_changed = virtballoon_changed,
Aaron Lu89107002013-09-17 09:25:23 +0930557#ifdef CONFIG_PM_SLEEP
Amit Shahe5629662011-12-22 16:58:35 +0530558 .freeze = virtballoon_freeze,
559 .restore = virtballoon_restore,
Amit Shahe5629662011-12-22 16:58:35 +0530560#endif
Rusty Russell6b35e402008-02-04 23:50:12 -0500561};
562
Rusty Russellb2a17022013-02-13 16:59:28 +1030563module_virtio_driver(virtio_balloon_driver);
Rusty Russell6b35e402008-02-04 23:50:12 -0500564MODULE_DEVICE_TABLE(virtio, id_table);
565MODULE_DESCRIPTION("Virtio balloon driver");
566MODULE_LICENSE("GPL");