blob: 7d8f531fb8e8a60a342f2fce3f2168fe5127db39 [file] [log] [blame]
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -07001/******************************************************************************
2 * grant_table.c
3 *
4 * Granting foreign access to our memory reservation.
5 *
6 * Copyright (c) 2005-2006, Christopher Clark
7 * Copyright (c) 2004-2005, K A Fraser
8 *
9 * This program is free software; you can redistribute it and/or
10 * modify it under the terms of the GNU General Public License version 2
11 * as published by the Free Software Foundation; or, when distributed
12 * separately from the Linux kernel or incorporated into other
13 * software packages, subject to the following license:
14 *
15 * Permission is hereby granted, free of charge, to any person obtaining a copy
16 * of this source file (the "Software"), to deal in the Software without
17 * restriction, including without limitation the rights to use, copy, modify,
18 * merge, publish, distribute, sublicense, and/or sell copies of the Software,
19 * and to permit persons to whom the Software is furnished to do so, subject to
20 * the following conditions:
21 *
22 * The above copyright notice and this permission notice shall be included in
23 * all copies or substantial portions of the Software.
24 *
25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
26 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
27 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
28 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
29 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
30 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
31 * IN THE SOFTWARE.
32 */
33
34#include <linux/module.h>
35#include <linux/sched.h>
36#include <linux/mm.h>
37#include <linux/vmalloc.h>
38#include <linux/uaccess.h>
39
40#include <xen/interface/xen.h>
41#include <xen/page.h>
42#include <xen/grant_table.h>
Jeremy Fitzhardingeecbf29c2008-12-16 12:37:07 -080043#include <asm/xen/hypercall.h>
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -070044
45#include <asm/pgtable.h>
46#include <asm/sync_bitops.h>
47
48
49/* External tools reserve first few grant table entries. */
50#define NR_RESERVED_ENTRIES 8
51#define GNTTAB_LIST_END 0xffffffff
52#define GREFS_PER_GRANT_FRAME (PAGE_SIZE / sizeof(struct grant_entry))
53
54static grant_ref_t **gnttab_list;
55static unsigned int nr_grant_frames;
56static unsigned int boot_max_nr_grant_frames;
57static int gnttab_free_count;
58static grant_ref_t gnttab_free_head;
59static DEFINE_SPINLOCK(gnttab_list_lock);
60
61static struct grant_entry *shared;
62
63static struct gnttab_free_callback *gnttab_free_callback_list;
64
65static int gnttab_expand(unsigned int req_entries);
66
67#define RPP (PAGE_SIZE / sizeof(grant_ref_t))
68
69static inline grant_ref_t *__gnttab_entry(grant_ref_t entry)
70{
71 return &gnttab_list[(entry) / RPP][(entry) % RPP];
72}
73/* This can be used as an l-value */
74#define gnttab_entry(entry) (*__gnttab_entry(entry))
75
76static int get_free_entries(unsigned count)
77{
78 unsigned long flags;
79 int ref, rc;
80 grant_ref_t head;
81
82 spin_lock_irqsave(&gnttab_list_lock, flags);
83
84 if ((gnttab_free_count < count) &&
85 ((rc = gnttab_expand(count - gnttab_free_count)) < 0)) {
86 spin_unlock_irqrestore(&gnttab_list_lock, flags);
87 return rc;
88 }
89
90 ref = head = gnttab_free_head;
91 gnttab_free_count -= count;
92 while (count-- > 1)
93 head = gnttab_entry(head);
94 gnttab_free_head = gnttab_entry(head);
95 gnttab_entry(head) = GNTTAB_LIST_END;
96
97 spin_unlock_irqrestore(&gnttab_list_lock, flags);
98
99 return ref;
100}
101
102static void do_free_callbacks(void)
103{
104 struct gnttab_free_callback *callback, *next;
105
106 callback = gnttab_free_callback_list;
107 gnttab_free_callback_list = NULL;
108
109 while (callback != NULL) {
110 next = callback->next;
111 if (gnttab_free_count >= callback->count) {
112 callback->next = NULL;
113 callback->fn(callback->arg);
114 } else {
115 callback->next = gnttab_free_callback_list;
116 gnttab_free_callback_list = callback;
117 }
118 callback = next;
119 }
120}
121
122static inline void check_free_callbacks(void)
123{
124 if (unlikely(gnttab_free_callback_list))
125 do_free_callbacks();
126}
127
128static void put_free_entry(grant_ref_t ref)
129{
130 unsigned long flags;
131 spin_lock_irqsave(&gnttab_list_lock, flags);
132 gnttab_entry(ref) = gnttab_free_head;
133 gnttab_free_head = ref;
134 gnttab_free_count++;
135 check_free_callbacks();
136 spin_unlock_irqrestore(&gnttab_list_lock, flags);
137}
138
139static void update_grant_entry(grant_ref_t ref, domid_t domid,
140 unsigned long frame, unsigned flags)
141{
142 /*
143 * Introducing a valid entry into the grant table:
144 * 1. Write ent->domid.
145 * 2. Write ent->frame:
146 * GTF_permit_access: Frame to which access is permitted.
147 * GTF_accept_transfer: Pseudo-phys frame slot being filled by new
148 * frame, or zero if none.
149 * 3. Write memory barrier (WMB).
150 * 4. Write ent->flags, inc. valid type.
151 */
152 shared[ref].frame = frame;
153 shared[ref].domid = domid;
154 wmb();
155 shared[ref].flags = flags;
156}
157
158/*
159 * Public grant-issuing interface functions
160 */
161void gnttab_grant_foreign_access_ref(grant_ref_t ref, domid_t domid,
162 unsigned long frame, int readonly)
163{
164 update_grant_entry(ref, domid, frame,
165 GTF_permit_access | (readonly ? GTF_readonly : 0));
166}
167EXPORT_SYMBOL_GPL(gnttab_grant_foreign_access_ref);
168
169int gnttab_grant_foreign_access(domid_t domid, unsigned long frame,
170 int readonly)
171{
172 int ref;
173
174 ref = get_free_entries(1);
175 if (unlikely(ref < 0))
176 return -ENOSPC;
177
178 gnttab_grant_foreign_access_ref(ref, domid, frame, readonly);
179
180 return ref;
181}
182EXPORT_SYMBOL_GPL(gnttab_grant_foreign_access);
183
184int gnttab_query_foreign_access(grant_ref_t ref)
185{
186 u16 nflags;
187
188 nflags = shared[ref].flags;
189
190 return (nflags & (GTF_reading|GTF_writing));
191}
192EXPORT_SYMBOL_GPL(gnttab_query_foreign_access);
193
194int gnttab_end_foreign_access_ref(grant_ref_t ref, int readonly)
195{
196 u16 flags, nflags;
197
198 nflags = shared[ref].flags;
199 do {
200 flags = nflags;
201 if (flags & (GTF_reading|GTF_writing)) {
202 printk(KERN_ALERT "WARNING: g.e. still in use!\n");
203 return 0;
204 }
205 } while ((nflags = sync_cmpxchg(&shared[ref].flags, flags, 0)) != flags);
206
207 return 1;
208}
209EXPORT_SYMBOL_GPL(gnttab_end_foreign_access_ref);
210
211void gnttab_end_foreign_access(grant_ref_t ref, int readonly,
212 unsigned long page)
213{
214 if (gnttab_end_foreign_access_ref(ref, readonly)) {
215 put_free_entry(ref);
216 if (page != 0)
217 free_page(page);
218 } else {
219 /* XXX This needs to be fixed so that the ref and page are
220 placed on a list to be freed up later. */
221 printk(KERN_WARNING
222 "WARNING: leaking g.e. and page still in use!\n");
223 }
224}
225EXPORT_SYMBOL_GPL(gnttab_end_foreign_access);
226
227int gnttab_grant_foreign_transfer(domid_t domid, unsigned long pfn)
228{
229 int ref;
230
231 ref = get_free_entries(1);
232 if (unlikely(ref < 0))
233 return -ENOSPC;
234 gnttab_grant_foreign_transfer_ref(ref, domid, pfn);
235
236 return ref;
237}
238EXPORT_SYMBOL_GPL(gnttab_grant_foreign_transfer);
239
240void gnttab_grant_foreign_transfer_ref(grant_ref_t ref, domid_t domid,
241 unsigned long pfn)
242{
243 update_grant_entry(ref, domid, pfn, GTF_accept_transfer);
244}
245EXPORT_SYMBOL_GPL(gnttab_grant_foreign_transfer_ref);
246
247unsigned long gnttab_end_foreign_transfer_ref(grant_ref_t ref)
248{
249 unsigned long frame;
250 u16 flags;
251
252 /*
253 * If a transfer is not even yet started, try to reclaim the grant
254 * reference and return failure (== 0).
255 */
256 while (!((flags = shared[ref].flags) & GTF_transfer_committed)) {
257 if (sync_cmpxchg(&shared[ref].flags, flags, 0) == flags)
258 return 0;
259 cpu_relax();
260 }
261
262 /* If a transfer is in progress then wait until it is completed. */
263 while (!(flags & GTF_transfer_completed)) {
264 flags = shared[ref].flags;
265 cpu_relax();
266 }
267
268 rmb(); /* Read the frame number /after/ reading completion status. */
269 frame = shared[ref].frame;
270 BUG_ON(frame == 0);
271
272 return frame;
273}
274EXPORT_SYMBOL_GPL(gnttab_end_foreign_transfer_ref);
275
276unsigned long gnttab_end_foreign_transfer(grant_ref_t ref)
277{
278 unsigned long frame = gnttab_end_foreign_transfer_ref(ref);
279 put_free_entry(ref);
280 return frame;
281}
282EXPORT_SYMBOL_GPL(gnttab_end_foreign_transfer);
283
284void gnttab_free_grant_reference(grant_ref_t ref)
285{
286 put_free_entry(ref);
287}
288EXPORT_SYMBOL_GPL(gnttab_free_grant_reference);
289
290void gnttab_free_grant_references(grant_ref_t head)
291{
292 grant_ref_t ref;
293 unsigned long flags;
294 int count = 1;
295 if (head == GNTTAB_LIST_END)
296 return;
297 spin_lock_irqsave(&gnttab_list_lock, flags);
298 ref = head;
299 while (gnttab_entry(ref) != GNTTAB_LIST_END) {
300 ref = gnttab_entry(ref);
301 count++;
302 }
303 gnttab_entry(ref) = gnttab_free_head;
304 gnttab_free_head = head;
305 gnttab_free_count += count;
306 check_free_callbacks();
307 spin_unlock_irqrestore(&gnttab_list_lock, flags);
308}
309EXPORT_SYMBOL_GPL(gnttab_free_grant_references);
310
311int gnttab_alloc_grant_references(u16 count, grant_ref_t *head)
312{
313 int h = get_free_entries(count);
314
315 if (h < 0)
316 return -ENOSPC;
317
318 *head = h;
319
320 return 0;
321}
322EXPORT_SYMBOL_GPL(gnttab_alloc_grant_references);
323
324int gnttab_empty_grant_references(const grant_ref_t *private_head)
325{
326 return (*private_head == GNTTAB_LIST_END);
327}
328EXPORT_SYMBOL_GPL(gnttab_empty_grant_references);
329
330int gnttab_claim_grant_reference(grant_ref_t *private_head)
331{
332 grant_ref_t g = *private_head;
333 if (unlikely(g == GNTTAB_LIST_END))
334 return -ENOSPC;
335 *private_head = gnttab_entry(g);
336 return g;
337}
338EXPORT_SYMBOL_GPL(gnttab_claim_grant_reference);
339
340void gnttab_release_grant_reference(grant_ref_t *private_head,
341 grant_ref_t release)
342{
343 gnttab_entry(release) = *private_head;
344 *private_head = release;
345}
346EXPORT_SYMBOL_GPL(gnttab_release_grant_reference);
347
348void gnttab_request_free_callback(struct gnttab_free_callback *callback,
349 void (*fn)(void *), void *arg, u16 count)
350{
351 unsigned long flags;
352 spin_lock_irqsave(&gnttab_list_lock, flags);
353 if (callback->next)
354 goto out;
355 callback->fn = fn;
356 callback->arg = arg;
357 callback->count = count;
358 callback->next = gnttab_free_callback_list;
359 gnttab_free_callback_list = callback;
360 check_free_callbacks();
361out:
362 spin_unlock_irqrestore(&gnttab_list_lock, flags);
363}
364EXPORT_SYMBOL_GPL(gnttab_request_free_callback);
365
366void gnttab_cancel_free_callback(struct gnttab_free_callback *callback)
367{
368 struct gnttab_free_callback **pcb;
369 unsigned long flags;
370
371 spin_lock_irqsave(&gnttab_list_lock, flags);
372 for (pcb = &gnttab_free_callback_list; *pcb; pcb = &(*pcb)->next) {
373 if (*pcb == callback) {
374 *pcb = callback->next;
375 break;
376 }
377 }
378 spin_unlock_irqrestore(&gnttab_list_lock, flags);
379}
380EXPORT_SYMBOL_GPL(gnttab_cancel_free_callback);
381
382static int grow_gnttab_list(unsigned int more_frames)
383{
384 unsigned int new_nr_grant_frames, extra_entries, i;
Michael Abd-El-Malekbbc60c12008-04-04 02:33:48 -0700385 unsigned int nr_glist_frames, new_nr_glist_frames;
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700386
387 new_nr_grant_frames = nr_grant_frames + more_frames;
388 extra_entries = more_frames * GREFS_PER_GRANT_FRAME;
389
Michael Abd-El-Malekbbc60c12008-04-04 02:33:48 -0700390 nr_glist_frames = (nr_grant_frames * GREFS_PER_GRANT_FRAME + RPP - 1) / RPP;
391 new_nr_glist_frames =
392 (new_nr_grant_frames * GREFS_PER_GRANT_FRAME + RPP - 1) / RPP;
393 for (i = nr_glist_frames; i < new_nr_glist_frames; i++) {
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700394 gnttab_list[i] = (grant_ref_t *)__get_free_page(GFP_ATOMIC);
395 if (!gnttab_list[i])
396 goto grow_nomem;
397 }
398
399
400 for (i = GREFS_PER_GRANT_FRAME * nr_grant_frames;
401 i < GREFS_PER_GRANT_FRAME * new_nr_grant_frames - 1; i++)
402 gnttab_entry(i) = i + 1;
403
404 gnttab_entry(i) = gnttab_free_head;
405 gnttab_free_head = GREFS_PER_GRANT_FRAME * nr_grant_frames;
406 gnttab_free_count += extra_entries;
407
408 nr_grant_frames = new_nr_grant_frames;
409
410 check_free_callbacks();
411
412 return 0;
413
414grow_nomem:
Michael Abd-El-Malekbbc60c12008-04-04 02:33:48 -0700415 for ( ; i >= nr_glist_frames; i--)
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700416 free_page((unsigned long) gnttab_list[i]);
417 return -ENOMEM;
418}
419
420static unsigned int __max_nr_grant_frames(void)
421{
422 struct gnttab_query_size query;
423 int rc;
424
425 query.dom = DOMID_SELF;
426
427 rc = HYPERVISOR_grant_table_op(GNTTABOP_query_size, &query, 1);
428 if ((rc < 0) || (query.status != GNTST_okay))
429 return 4; /* Legacy max supported number of frames */
430
431 return query.max_nr_frames;
432}
433
434static inline unsigned int max_nr_grant_frames(void)
435{
436 unsigned int xen_max = __max_nr_grant_frames();
437
438 if (xen_max > boot_max_nr_grant_frames)
439 return boot_max_nr_grant_frames;
440 return xen_max;
441}
442
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700443static int gnttab_map(unsigned int start_idx, unsigned int end_idx)
444{
445 struct gnttab_setup_table setup;
446 unsigned long *frames;
447 unsigned int nr_gframes = end_idx + 1;
448 int rc;
449
450 frames = kmalloc(nr_gframes * sizeof(unsigned long), GFP_ATOMIC);
451 if (!frames)
452 return -ENOMEM;
453
454 setup.dom = DOMID_SELF;
455 setup.nr_frames = nr_gframes;
Isaku Yamahata87e27cf2008-04-02 10:53:52 -0700456 set_xen_guest_handle(setup.frame_list, frames);
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700457
458 rc = HYPERVISOR_grant_table_op(GNTTABOP_setup_table, &setup, 1);
459 if (rc == -ENOSYS) {
460 kfree(frames);
461 return -ENOSYS;
462 }
463
464 BUG_ON(rc || setup.status);
465
Isaku Yamahata8d3d2102008-04-02 10:54:00 -0700466 rc = arch_gnttab_map_shared(frames, nr_gframes, max_nr_grant_frames(),
467 &shared);
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700468 BUG_ON(rc);
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700469
470 kfree(frames);
471
472 return 0;
473}
474
Jeremy Fitzhardinge0e913982008-05-26 23:31:27 +0100475int gnttab_resume(void)
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700476{
477 if (max_nr_grant_frames() < nr_grant_frames)
478 return -ENOSYS;
479 return gnttab_map(0, nr_grant_frames - 1);
480}
481
Jeremy Fitzhardinge0e913982008-05-26 23:31:27 +0100482int gnttab_suspend(void)
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700483{
Isaku Yamahata8d3d2102008-04-02 10:54:00 -0700484 arch_gnttab_unmap_shared(shared, nr_grant_frames);
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700485 return 0;
486}
487
488static int gnttab_expand(unsigned int req_entries)
489{
490 int rc;
491 unsigned int cur, extra;
492
493 cur = nr_grant_frames;
494 extra = ((req_entries + (GREFS_PER_GRANT_FRAME-1)) /
495 GREFS_PER_GRANT_FRAME);
496 if (cur + extra > max_nr_grant_frames())
497 return -ENOSPC;
498
499 rc = gnttab_map(cur, cur + extra - 1);
500 if (rc == 0)
501 rc = grow_gnttab_list(extra);
502
503 return rc;
504}
505
506static int __devinit gnttab_init(void)
507{
508 int i;
Michael Abd-El-Malekbbc60c12008-04-04 02:33:48 -0700509 unsigned int max_nr_glist_frames, nr_glist_frames;
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700510 unsigned int nr_init_grefs;
511
Jeremy Fitzhardinge6e833582008-08-19 13:16:17 -0700512 if (!xen_domain())
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700513 return -ENODEV;
514
515 nr_grant_frames = 1;
516 boot_max_nr_grant_frames = __max_nr_grant_frames();
517
518 /* Determine the maximum number of frames required for the
519 * grant reference free list on the current hypervisor.
520 */
521 max_nr_glist_frames = (boot_max_nr_grant_frames *
Michael Abd-El-Malekbbc60c12008-04-04 02:33:48 -0700522 GREFS_PER_GRANT_FRAME / RPP);
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700523
524 gnttab_list = kmalloc(max_nr_glist_frames * sizeof(grant_ref_t *),
525 GFP_KERNEL);
526 if (gnttab_list == NULL)
527 return -ENOMEM;
528
Michael Abd-El-Malekbbc60c12008-04-04 02:33:48 -0700529 nr_glist_frames = (nr_grant_frames * GREFS_PER_GRANT_FRAME + RPP - 1) / RPP;
530 for (i = 0; i < nr_glist_frames; i++) {
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700531 gnttab_list[i] = (grant_ref_t *)__get_free_page(GFP_KERNEL);
532 if (gnttab_list[i] == NULL)
533 goto ini_nomem;
534 }
535
536 if (gnttab_resume() < 0)
537 return -ENODEV;
538
539 nr_init_grefs = nr_grant_frames * GREFS_PER_GRANT_FRAME;
540
541 for (i = NR_RESERVED_ENTRIES; i < nr_init_grefs - 1; i++)
542 gnttab_entry(i) = i + 1;
543
544 gnttab_entry(nr_init_grefs - 1) = GNTTAB_LIST_END;
545 gnttab_free_count = nr_init_grefs - NR_RESERVED_ENTRIES;
546 gnttab_free_head = NR_RESERVED_ENTRIES;
547
548 printk("Grant table initialized\n");
549 return 0;
550
551 ini_nomem:
552 for (i--; i >= 0; i--)
553 free_page((unsigned long)gnttab_list[i]);
554 kfree(gnttab_list);
555 return -ENOMEM;
556}
557
558core_initcall(gnttab_init);