blob: b4d4eac761db6241042e60db3b604caa9150e91e [file] [log] [blame]
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -07001/******************************************************************************
2 * grant_table.c
3 *
4 * Granting foreign access to our memory reservation.
5 *
6 * Copyright (c) 2005-2006, Christopher Clark
7 * Copyright (c) 2004-2005, K A Fraser
8 *
9 * This program is free software; you can redistribute it and/or
10 * modify it under the terms of the GNU General Public License version 2
11 * as published by the Free Software Foundation; or, when distributed
12 * separately from the Linux kernel or incorporated into other
13 * software packages, subject to the following license:
14 *
15 * Permission is hereby granted, free of charge, to any person obtaining a copy
16 * of this source file (the "Software"), to deal in the Software without
17 * restriction, including without limitation the rights to use, copy, modify,
18 * merge, publish, distribute, sublicense, and/or sell copies of the Software,
19 * and to permit persons to whom the Software is furnished to do so, subject to
20 * the following conditions:
21 *
22 * The above copyright notice and this permission notice shall be included in
23 * all copies or substantial portions of the Software.
24 *
25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
26 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
27 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
28 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
29 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
30 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
31 * IN THE SOFTWARE.
32 */
33
34#include <linux/module.h>
35#include <linux/sched.h>
36#include <linux/mm.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090037#include <linux/slab.h>
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -070038#include <linux/vmalloc.h>
39#include <linux/uaccess.h>
Stefano Stabellini183d03c2010-05-17 17:08:21 +010040#include <linux/io.h>
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -070041
Jeremy Fitzhardinge1ccbf532009-10-06 15:11:14 -070042#include <xen/xen.h>
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -070043#include <xen/interface/xen.h>
44#include <xen/page.h>
45#include <xen/grant_table.h>
Stefano Stabellini183d03c2010-05-17 17:08:21 +010046#include <xen/interface/memory.h>
Annie Li85ff6ac2011-11-22 09:59:21 +080047#include <xen/hvc-console.h>
Jeremy Fitzhardingeecbf29c2008-12-16 12:37:07 -080048#include <asm/xen/hypercall.h>
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -070049
50#include <asm/pgtable.h>
51#include <asm/sync_bitops.h>
52
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -070053/* External tools reserve first few grant table entries. */
54#define NR_RESERVED_ENTRIES 8
55#define GNTTAB_LIST_END 0xffffffff
Annie Li85ff6ac2011-11-22 09:59:21 +080056#define GREFS_PER_GRANT_FRAME \
57(grant_table_version == 1 ? \
58(PAGE_SIZE / sizeof(struct grant_entry_v1)) : \
59(PAGE_SIZE / sizeof(union grant_entry_v2)))
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -070060
61static grant_ref_t **gnttab_list;
62static unsigned int nr_grant_frames;
63static unsigned int boot_max_nr_grant_frames;
64static int gnttab_free_count;
65static grant_ref_t gnttab_free_head;
66static DEFINE_SPINLOCK(gnttab_list_lock);
Stefano Stabellini183d03c2010-05-17 17:08:21 +010067unsigned long xen_hvm_resume_frames;
68EXPORT_SYMBOL_GPL(xen_hvm_resume_frames);
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -070069
Annie Li0f9f5a92011-11-22 09:58:06 +080070static union {
71 struct grant_entry_v1 *v1;
Annie Li85ff6ac2011-11-22 09:59:21 +080072 union grant_entry_v2 *v2;
Annie Li0f9f5a92011-11-22 09:58:06 +080073 void *addr;
74} gnttab_shared;
75
76/*This is a structure of function pointers for grant table*/
77struct gnttab_ops {
78 /*
Annie Li9dbc71d2011-12-12 18:13:57 +080079 * Mapping a list of frames for storing grant entries. Frames parameter
80 * is used to store grant table address when grant table being setup,
81 * nr_gframes is the number of frames to map grant table. Returning
82 * GNTST_okay means success and negative value means failure.
Annie Li0f9f5a92011-11-22 09:58:06 +080083 */
Annie Li9dbc71d2011-12-12 18:13:57 +080084 int (*map_frames)(unsigned long *frames, unsigned int nr_gframes);
Annie Li0f9f5a92011-11-22 09:58:06 +080085 /*
86 * Release a list of frames which are mapped in map_frames for grant
87 * entry status.
88 */
89 void (*unmap_frames)(void);
90 /*
Annie Li9dbc71d2011-12-12 18:13:57 +080091 * Introducing a valid entry into the grant table, granting the frame of
92 * this grant entry to domain for accessing or transfering. Ref
93 * parameter is reference of this introduced grant entry, domid is id of
94 * granted domain, frame is the page frame to be granted, and flags is
95 * status of the grant entry to be updated.
Annie Li0f9f5a92011-11-22 09:58:06 +080096 */
Annie Li9dbc71d2011-12-12 18:13:57 +080097 void (*update_entry)(grant_ref_t ref, domid_t domid,
98 unsigned long frame, unsigned flags);
Annie Li0f9f5a92011-11-22 09:58:06 +080099 /*
Annie Li9dbc71d2011-12-12 18:13:57 +0800100 * Stop granting a grant entry to domain for accessing. Ref parameter is
101 * reference of a grant entry whose grant access will be stopped,
102 * readonly is not in use in this function. If the grant entry is
Annie Li0f9f5a92011-11-22 09:58:06 +0800103 * currently mapped for reading or writing, just return failure(==0)
104 * directly and don't tear down the grant access. Otherwise, stop grant
105 * access for this entry and return success(==1).
106 */
Annie Li9dbc71d2011-12-12 18:13:57 +0800107 int (*end_foreign_access_ref)(grant_ref_t ref, int readonly);
Annie Li0f9f5a92011-11-22 09:58:06 +0800108 /*
Annie Li9dbc71d2011-12-12 18:13:57 +0800109 * Stop granting a grant entry to domain for transfer. Ref parameter is
110 * reference of a grant entry whose grant transfer will be stopped. If
111 * tranfer has not started, just reclaim the grant entry and return
112 * failure(==0). Otherwise, wait for the transfer to complete and then
113 * return the frame.
Annie Li0f9f5a92011-11-22 09:58:06 +0800114 */
Annie Li9dbc71d2011-12-12 18:13:57 +0800115 unsigned long (*end_foreign_transfer_ref)(grant_ref_t ref);
Annie Li0f9f5a92011-11-22 09:58:06 +0800116 /*
Annie Li9dbc71d2011-12-12 18:13:57 +0800117 * Query the status of a grant entry. Ref parameter is reference of
Annie Li0f9f5a92011-11-22 09:58:06 +0800118 * queried grant entry, return value is the status of queried entry.
119 * Detailed status(writing/reading) can be gotten from the return value
120 * by bit operations.
121 */
Annie Li9dbc71d2011-12-12 18:13:57 +0800122 int (*query_foreign_access)(grant_ref_t ref);
Annie Li66667542011-12-12 18:14:42 +0800123 /*
124 * Grant a domain to access a range of bytes within the page referred by
125 * an available grant entry. Ref parameter is reference of a grant entry
126 * which will be sub-page accessed, domid is id of grantee domain, frame
127 * is frame address of subpage grant, flags is grant type and flag
128 * information, page_off is offset of the range of bytes, and length is
129 * length of bytes to be accessed.
130 */
131 void (*update_subpage_entry)(grant_ref_t ref, domid_t domid,
132 unsigned long frame, int flags,
133 unsigned page_off, unsigned length);
Annie Li9438ce92011-12-12 18:15:07 +0800134 /*
135 * Redirect an available grant entry on domain A to another grant
136 * reference of domain B, then allow domain C to use grant reference
137 * of domain B transitively. Ref parameter is an available grant entry
138 * reference on domain A, domid is id of domain C which accesses grant
139 * entry transitively, flags is grant type and flag information,
140 * trans_domid is id of domain B whose grant entry is finally accessed
141 * transitively, trans_gref is grant entry transitive reference of
142 * domain B.
143 */
144 void (*update_trans_entry)(grant_ref_t ref, domid_t domid, int flags,
145 domid_t trans_domid, grant_ref_t trans_gref);
Annie Li0f9f5a92011-11-22 09:58:06 +0800146};
147
148static struct gnttab_ops *gnttab_interface;
149
Annie Li85ff6ac2011-11-22 09:59:21 +0800150/*This reflects status of grant entries, so act as a global value*/
151static grant_status_t *grstatus;
152
Annie Li0f9f5a92011-11-22 09:58:06 +0800153static int grant_table_version;
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700154
155static struct gnttab_free_callback *gnttab_free_callback_list;
156
157static int gnttab_expand(unsigned int req_entries);
158
159#define RPP (PAGE_SIZE / sizeof(grant_ref_t))
Annie Li85ff6ac2011-11-22 09:59:21 +0800160#define SPP (PAGE_SIZE / sizeof(grant_status_t))
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700161
162static inline grant_ref_t *__gnttab_entry(grant_ref_t entry)
163{
164 return &gnttab_list[(entry) / RPP][(entry) % RPP];
165}
166/* This can be used as an l-value */
167#define gnttab_entry(entry) (*__gnttab_entry(entry))
168
169static int get_free_entries(unsigned count)
170{
171 unsigned long flags;
Konrad Rzeszutek Wilk272800d2011-07-22 14:00:06 -0400172 int ref, rc = 0;
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700173 grant_ref_t head;
174
175 spin_lock_irqsave(&gnttab_list_lock, flags);
176
177 if ((gnttab_free_count < count) &&
178 ((rc = gnttab_expand(count - gnttab_free_count)) < 0)) {
179 spin_unlock_irqrestore(&gnttab_list_lock, flags);
180 return rc;
181 }
182
183 ref = head = gnttab_free_head;
184 gnttab_free_count -= count;
185 while (count-- > 1)
186 head = gnttab_entry(head);
187 gnttab_free_head = gnttab_entry(head);
188 gnttab_entry(head) = GNTTAB_LIST_END;
189
190 spin_unlock_irqrestore(&gnttab_list_lock, flags);
191
192 return ref;
193}
194
195static void do_free_callbacks(void)
196{
197 struct gnttab_free_callback *callback, *next;
198
199 callback = gnttab_free_callback_list;
200 gnttab_free_callback_list = NULL;
201
202 while (callback != NULL) {
203 next = callback->next;
204 if (gnttab_free_count >= callback->count) {
205 callback->next = NULL;
206 callback->fn(callback->arg);
207 } else {
208 callback->next = gnttab_free_callback_list;
209 gnttab_free_callback_list = callback;
210 }
211 callback = next;
212 }
213}
214
215static inline void check_free_callbacks(void)
216{
217 if (unlikely(gnttab_free_callback_list))
218 do_free_callbacks();
219}
220
221static void put_free_entry(grant_ref_t ref)
222{
223 unsigned long flags;
224 spin_lock_irqsave(&gnttab_list_lock, flags);
225 gnttab_entry(ref) = gnttab_free_head;
226 gnttab_free_head = ref;
227 gnttab_free_count++;
228 check_free_callbacks();
229 spin_unlock_irqrestore(&gnttab_list_lock, flags);
230}
231
Annie Li0f9f5a92011-11-22 09:58:06 +0800232/*
Annie Li85ff6ac2011-11-22 09:59:21 +0800233 * Following applies to gnttab_update_entry_v1 and gnttab_update_entry_v2.
Annie Li0f9f5a92011-11-22 09:58:06 +0800234 * Introducing a valid entry into the grant table:
235 * 1. Write ent->domid.
236 * 2. Write ent->frame:
237 * GTF_permit_access: Frame to which access is permitted.
238 * GTF_accept_transfer: Pseudo-phys frame slot being filled by new
239 * frame, or zero if none.
240 * 3. Write memory barrier (WMB).
241 * 4. Write ent->flags, inc. valid type.
242 */
243static void gnttab_update_entry_v1(grant_ref_t ref, domid_t domid,
244 unsigned long frame, unsigned flags)
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700245{
Annie Li0f9f5a92011-11-22 09:58:06 +0800246 gnttab_shared.v1[ref].domid = domid;
247 gnttab_shared.v1[ref].frame = frame;
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700248 wmb();
Annie Li0f9f5a92011-11-22 09:58:06 +0800249 gnttab_shared.v1[ref].flags = flags;
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700250}
251
Annie Li85ff6ac2011-11-22 09:59:21 +0800252static void gnttab_update_entry_v2(grant_ref_t ref, domid_t domid,
253 unsigned long frame, unsigned flags)
254{
255 gnttab_shared.v2[ref].hdr.domid = domid;
256 gnttab_shared.v2[ref].full_page.frame = frame;
257 wmb();
258 gnttab_shared.v2[ref].hdr.flags = GTF_permit_access | flags;
259}
260
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700261/*
262 * Public grant-issuing interface functions
263 */
264void gnttab_grant_foreign_access_ref(grant_ref_t ref, domid_t domid,
265 unsigned long frame, int readonly)
266{
Annie Li0f9f5a92011-11-22 09:58:06 +0800267 gnttab_interface->update_entry(ref, domid, frame,
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700268 GTF_permit_access | (readonly ? GTF_readonly : 0));
269}
270EXPORT_SYMBOL_GPL(gnttab_grant_foreign_access_ref);
271
272int gnttab_grant_foreign_access(domid_t domid, unsigned long frame,
273 int readonly)
274{
275 int ref;
276
277 ref = get_free_entries(1);
278 if (unlikely(ref < 0))
279 return -ENOSPC;
280
281 gnttab_grant_foreign_access_ref(ref, domid, frame, readonly);
282
283 return ref;
284}
285EXPORT_SYMBOL_GPL(gnttab_grant_foreign_access);
286
Annie Li66667542011-12-12 18:14:42 +0800287void gnttab_update_subpage_entry_v2(grant_ref_t ref, domid_t domid,
288 unsigned long frame, int flags,
289 unsigned page_off,
290 unsigned length)
291{
292 gnttab_shared.v2[ref].sub_page.frame = frame;
293 gnttab_shared.v2[ref].sub_page.page_off = page_off;
294 gnttab_shared.v2[ref].sub_page.length = length;
295 gnttab_shared.v2[ref].hdr.domid = domid;
296 wmb();
297 gnttab_shared.v2[ref].hdr.flags =
298 GTF_permit_access | GTF_sub_page | flags;
299}
300
301int gnttab_grant_foreign_access_subpage_ref(grant_ref_t ref, domid_t domid,
302 unsigned long frame, int flags,
303 unsigned page_off,
304 unsigned length)
305{
306 if (flags & (GTF_accept_transfer | GTF_reading |
307 GTF_writing | GTF_transitive))
308 return -EPERM;
309
310 if (gnttab_interface->update_subpage_entry == NULL)
311 return -ENOSYS;
312
313 gnttab_interface->update_subpage_entry(ref, domid, frame, flags,
314 page_off, length);
315
316 return 0;
317}
318EXPORT_SYMBOL_GPL(gnttab_grant_foreign_access_subpage_ref);
319
320int gnttab_grant_foreign_access_subpage(domid_t domid, unsigned long frame,
321 int flags, unsigned page_off,
322 unsigned length)
323{
324 int ref, rc;
325
326 ref = get_free_entries(1);
327 if (unlikely(ref < 0))
328 return -ENOSPC;
329
330 rc = gnttab_grant_foreign_access_subpage_ref(ref, domid, frame, flags,
331 page_off, length);
332 if (rc < 0) {
333 put_free_entry(ref);
334 return rc;
335 }
336
337 return ref;
338}
339EXPORT_SYMBOL_GPL(gnttab_grant_foreign_access_subpage);
340
341bool gnttab_subpage_grants_available(void)
342{
343 return gnttab_interface->update_subpage_entry != NULL;
344}
345EXPORT_SYMBOL_GPL(gnttab_subpage_grants_available);
346
Annie Li9438ce92011-12-12 18:15:07 +0800347void gnttab_update_trans_entry_v2(grant_ref_t ref, domid_t domid,
348 int flags, domid_t trans_domid,
349 grant_ref_t trans_gref)
350{
351 gnttab_shared.v2[ref].transitive.trans_domid = trans_domid;
352 gnttab_shared.v2[ref].transitive.gref = trans_gref;
353 gnttab_shared.v2[ref].hdr.domid = domid;
354 wmb();
355 gnttab_shared.v2[ref].hdr.flags =
356 GTF_permit_access | GTF_transitive | flags;
357}
358
359int gnttab_grant_foreign_access_trans_ref(grant_ref_t ref, domid_t domid,
360 int flags, domid_t trans_domid,
361 grant_ref_t trans_gref)
362{
363 if (flags & (GTF_accept_transfer | GTF_reading |
364 GTF_writing | GTF_sub_page))
365 return -EPERM;
366
367 if (gnttab_interface->update_trans_entry == NULL)
368 return -ENOSYS;
369
370 gnttab_interface->update_trans_entry(ref, domid, flags, trans_domid,
371 trans_gref);
372
373 return 0;
374}
375EXPORT_SYMBOL_GPL(gnttab_grant_foreign_access_trans_ref);
376
377int gnttab_grant_foreign_access_trans(domid_t domid, int flags,
378 domid_t trans_domid,
379 grant_ref_t trans_gref)
380{
381 int ref, rc;
382
383 ref = get_free_entries(1);
384 if (unlikely(ref < 0))
385 return -ENOSPC;
386
387 rc = gnttab_grant_foreign_access_trans_ref(ref, domid, flags,
388 trans_domid, trans_gref);
389 if (rc < 0) {
390 put_free_entry(ref);
391 return rc;
392 }
393
394 return ref;
395}
396EXPORT_SYMBOL_GPL(gnttab_grant_foreign_access_trans);
397
398bool gnttab_trans_grants_available(void)
399{
400 return gnttab_interface->update_trans_entry != NULL;
401}
402EXPORT_SYMBOL_GPL(gnttab_trans_grants_available);
403
Annie Li0f9f5a92011-11-22 09:58:06 +0800404static int gnttab_query_foreign_access_v1(grant_ref_t ref)
405{
406 return gnttab_shared.v1[ref].flags & (GTF_reading|GTF_writing);
407}
408
Annie Li85ff6ac2011-11-22 09:59:21 +0800409static int gnttab_query_foreign_access_v2(grant_ref_t ref)
410{
411 return grstatus[ref] & (GTF_reading|GTF_writing);
412}
413
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700414int gnttab_query_foreign_access(grant_ref_t ref)
415{
Annie Li0f9f5a92011-11-22 09:58:06 +0800416 return gnttab_interface->query_foreign_access(ref);
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700417}
418EXPORT_SYMBOL_GPL(gnttab_query_foreign_access);
419
Annie Li0f9f5a92011-11-22 09:58:06 +0800420static int gnttab_end_foreign_access_ref_v1(grant_ref_t ref, int readonly)
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700421{
422 u16 flags, nflags;
Annie Lib1e495b2011-11-22 09:58:47 +0800423 u16 *pflags;
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700424
Annie Lib1e495b2011-11-22 09:58:47 +0800425 pflags = &gnttab_shared.v1[ref].flags;
426 nflags = *pflags;
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700427 do {
428 flags = nflags;
429 if (flags & (GTF_reading|GTF_writing)) {
430 printk(KERN_ALERT "WARNING: g.e. still in use!\n");
431 return 0;
432 }
Annie Lib1e495b2011-11-22 09:58:47 +0800433 } while ((nflags = sync_cmpxchg(pflags, flags, 0)) != flags);
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700434
435 return 1;
436}
Annie Li0f9f5a92011-11-22 09:58:06 +0800437
Annie Li85ff6ac2011-11-22 09:59:21 +0800438static int gnttab_end_foreign_access_ref_v2(grant_ref_t ref, int readonly)
439{
440 gnttab_shared.v2[ref].hdr.flags = 0;
441 mb();
442 if (grstatus[ref] & (GTF_reading|GTF_writing)) {
443 return 0;
444 } else {
445 /* The read of grstatus needs to have acquire
446 semantics. On x86, reads already have
447 that, and we just need to protect against
448 compiler reorderings. On other
449 architectures we may need a full
450 barrier. */
451#ifdef CONFIG_X86
452 barrier();
453#else
454 mb();
455#endif
456 }
457
458 return 1;
459}
460
Annie Li0f9f5a92011-11-22 09:58:06 +0800461int gnttab_end_foreign_access_ref(grant_ref_t ref, int readonly)
462{
463 return gnttab_interface->end_foreign_access_ref(ref, readonly);
464}
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700465EXPORT_SYMBOL_GPL(gnttab_end_foreign_access_ref);
466
467void gnttab_end_foreign_access(grant_ref_t ref, int readonly,
468 unsigned long page)
469{
470 if (gnttab_end_foreign_access_ref(ref, readonly)) {
471 put_free_entry(ref);
472 if (page != 0)
473 free_page(page);
474 } else {
475 /* XXX This needs to be fixed so that the ref and page are
476 placed on a list to be freed up later. */
477 printk(KERN_WARNING
478 "WARNING: leaking g.e. and page still in use!\n");
479 }
480}
481EXPORT_SYMBOL_GPL(gnttab_end_foreign_access);
482
483int gnttab_grant_foreign_transfer(domid_t domid, unsigned long pfn)
484{
485 int ref;
486
487 ref = get_free_entries(1);
488 if (unlikely(ref < 0))
489 return -ENOSPC;
490 gnttab_grant_foreign_transfer_ref(ref, domid, pfn);
491
492 return ref;
493}
494EXPORT_SYMBOL_GPL(gnttab_grant_foreign_transfer);
495
496void gnttab_grant_foreign_transfer_ref(grant_ref_t ref, domid_t domid,
497 unsigned long pfn)
498{
Annie Li0f9f5a92011-11-22 09:58:06 +0800499 gnttab_interface->update_entry(ref, domid, pfn, GTF_accept_transfer);
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700500}
501EXPORT_SYMBOL_GPL(gnttab_grant_foreign_transfer_ref);
502
Annie Li0f9f5a92011-11-22 09:58:06 +0800503static unsigned long gnttab_end_foreign_transfer_ref_v1(grant_ref_t ref)
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700504{
505 unsigned long frame;
506 u16 flags;
Annie Lib1e495b2011-11-22 09:58:47 +0800507 u16 *pflags;
508
509 pflags = &gnttab_shared.v1[ref].flags;
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700510
511 /*
512 * If a transfer is not even yet started, try to reclaim the grant
513 * reference and return failure (== 0).
514 */
Annie Lib1e495b2011-11-22 09:58:47 +0800515 while (!((flags = *pflags) & GTF_transfer_committed)) {
516 if (sync_cmpxchg(pflags, flags, 0) == flags)
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700517 return 0;
518 cpu_relax();
519 }
520
521 /* If a transfer is in progress then wait until it is completed. */
522 while (!(flags & GTF_transfer_completed)) {
Annie Lib1e495b2011-11-22 09:58:47 +0800523 flags = *pflags;
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700524 cpu_relax();
525 }
526
527 rmb(); /* Read the frame number /after/ reading completion status. */
Annie Li0f9f5a92011-11-22 09:58:06 +0800528 frame = gnttab_shared.v1[ref].frame;
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700529 BUG_ON(frame == 0);
530
531 return frame;
532}
Annie Li0f9f5a92011-11-22 09:58:06 +0800533
Annie Li85ff6ac2011-11-22 09:59:21 +0800534static unsigned long gnttab_end_foreign_transfer_ref_v2(grant_ref_t ref)
535{
536 unsigned long frame;
537 u16 flags;
538 u16 *pflags;
539
540 pflags = &gnttab_shared.v2[ref].hdr.flags;
541
542 /*
543 * If a transfer is not even yet started, try to reclaim the grant
544 * reference and return failure (== 0).
545 */
546 while (!((flags = *pflags) & GTF_transfer_committed)) {
547 if (sync_cmpxchg(pflags, flags, 0) == flags)
548 return 0;
549 cpu_relax();
550 }
551
552 /* If a transfer is in progress then wait until it is completed. */
553 while (!(flags & GTF_transfer_completed)) {
554 flags = *pflags;
555 cpu_relax();
556 }
557
558 rmb(); /* Read the frame number /after/ reading completion status. */
559 frame = gnttab_shared.v2[ref].full_page.frame;
560 BUG_ON(frame == 0);
561
562 return frame;
563}
564
Annie Li0f9f5a92011-11-22 09:58:06 +0800565unsigned long gnttab_end_foreign_transfer_ref(grant_ref_t ref)
566{
567 return gnttab_interface->end_foreign_transfer_ref(ref);
568}
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700569EXPORT_SYMBOL_GPL(gnttab_end_foreign_transfer_ref);
570
571unsigned long gnttab_end_foreign_transfer(grant_ref_t ref)
572{
573 unsigned long frame = gnttab_end_foreign_transfer_ref(ref);
574 put_free_entry(ref);
575 return frame;
576}
577EXPORT_SYMBOL_GPL(gnttab_end_foreign_transfer);
578
579void gnttab_free_grant_reference(grant_ref_t ref)
580{
581 put_free_entry(ref);
582}
583EXPORT_SYMBOL_GPL(gnttab_free_grant_reference);
584
585void gnttab_free_grant_references(grant_ref_t head)
586{
587 grant_ref_t ref;
588 unsigned long flags;
589 int count = 1;
590 if (head == GNTTAB_LIST_END)
591 return;
592 spin_lock_irqsave(&gnttab_list_lock, flags);
593 ref = head;
594 while (gnttab_entry(ref) != GNTTAB_LIST_END) {
595 ref = gnttab_entry(ref);
596 count++;
597 }
598 gnttab_entry(ref) = gnttab_free_head;
599 gnttab_free_head = head;
600 gnttab_free_count += count;
601 check_free_callbacks();
602 spin_unlock_irqrestore(&gnttab_list_lock, flags);
603}
604EXPORT_SYMBOL_GPL(gnttab_free_grant_references);
605
606int gnttab_alloc_grant_references(u16 count, grant_ref_t *head)
607{
608 int h = get_free_entries(count);
609
610 if (h < 0)
611 return -ENOSPC;
612
613 *head = h;
614
615 return 0;
616}
617EXPORT_SYMBOL_GPL(gnttab_alloc_grant_references);
618
619int gnttab_empty_grant_references(const grant_ref_t *private_head)
620{
621 return (*private_head == GNTTAB_LIST_END);
622}
623EXPORT_SYMBOL_GPL(gnttab_empty_grant_references);
624
625int gnttab_claim_grant_reference(grant_ref_t *private_head)
626{
627 grant_ref_t g = *private_head;
628 if (unlikely(g == GNTTAB_LIST_END))
629 return -ENOSPC;
630 *private_head = gnttab_entry(g);
631 return g;
632}
633EXPORT_SYMBOL_GPL(gnttab_claim_grant_reference);
634
635void gnttab_release_grant_reference(grant_ref_t *private_head,
636 grant_ref_t release)
637{
638 gnttab_entry(release) = *private_head;
639 *private_head = release;
640}
641EXPORT_SYMBOL_GPL(gnttab_release_grant_reference);
642
643void gnttab_request_free_callback(struct gnttab_free_callback *callback,
644 void (*fn)(void *), void *arg, u16 count)
645{
646 unsigned long flags;
647 spin_lock_irqsave(&gnttab_list_lock, flags);
648 if (callback->next)
649 goto out;
650 callback->fn = fn;
651 callback->arg = arg;
652 callback->count = count;
653 callback->next = gnttab_free_callback_list;
654 gnttab_free_callback_list = callback;
655 check_free_callbacks();
656out:
657 spin_unlock_irqrestore(&gnttab_list_lock, flags);
658}
659EXPORT_SYMBOL_GPL(gnttab_request_free_callback);
660
661void gnttab_cancel_free_callback(struct gnttab_free_callback *callback)
662{
663 struct gnttab_free_callback **pcb;
664 unsigned long flags;
665
666 spin_lock_irqsave(&gnttab_list_lock, flags);
667 for (pcb = &gnttab_free_callback_list; *pcb; pcb = &(*pcb)->next) {
668 if (*pcb == callback) {
669 *pcb = callback->next;
670 break;
671 }
672 }
673 spin_unlock_irqrestore(&gnttab_list_lock, flags);
674}
675EXPORT_SYMBOL_GPL(gnttab_cancel_free_callback);
676
677static int grow_gnttab_list(unsigned int more_frames)
678{
679 unsigned int new_nr_grant_frames, extra_entries, i;
Michael Abd-El-Malekbbc60c12008-04-04 02:33:48 -0700680 unsigned int nr_glist_frames, new_nr_glist_frames;
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700681
682 new_nr_grant_frames = nr_grant_frames + more_frames;
683 extra_entries = more_frames * GREFS_PER_GRANT_FRAME;
684
Michael Abd-El-Malekbbc60c12008-04-04 02:33:48 -0700685 nr_glist_frames = (nr_grant_frames * GREFS_PER_GRANT_FRAME + RPP - 1) / RPP;
686 new_nr_glist_frames =
687 (new_nr_grant_frames * GREFS_PER_GRANT_FRAME + RPP - 1) / RPP;
688 for (i = nr_glist_frames; i < new_nr_glist_frames; i++) {
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700689 gnttab_list[i] = (grant_ref_t *)__get_free_page(GFP_ATOMIC);
690 if (!gnttab_list[i])
691 goto grow_nomem;
692 }
693
694
695 for (i = GREFS_PER_GRANT_FRAME * nr_grant_frames;
696 i < GREFS_PER_GRANT_FRAME * new_nr_grant_frames - 1; i++)
697 gnttab_entry(i) = i + 1;
698
699 gnttab_entry(i) = gnttab_free_head;
700 gnttab_free_head = GREFS_PER_GRANT_FRAME * nr_grant_frames;
701 gnttab_free_count += extra_entries;
702
703 nr_grant_frames = new_nr_grant_frames;
704
705 check_free_callbacks();
706
707 return 0;
708
709grow_nomem:
Michael Abd-El-Malekbbc60c12008-04-04 02:33:48 -0700710 for ( ; i >= nr_glist_frames; i--)
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700711 free_page((unsigned long) gnttab_list[i]);
712 return -ENOMEM;
713}
714
715static unsigned int __max_nr_grant_frames(void)
716{
717 struct gnttab_query_size query;
718 int rc;
719
720 query.dom = DOMID_SELF;
721
722 rc = HYPERVISOR_grant_table_op(GNTTABOP_query_size, &query, 1);
723 if ((rc < 0) || (query.status != GNTST_okay))
724 return 4; /* Legacy max supported number of frames */
725
726 return query.max_nr_frames;
727}
728
Stefano Stabellini183d03c2010-05-17 17:08:21 +0100729unsigned int gnttab_max_grant_frames(void)
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700730{
731 unsigned int xen_max = __max_nr_grant_frames();
732
733 if (xen_max > boot_max_nr_grant_frames)
734 return boot_max_nr_grant_frames;
735 return xen_max;
736}
Stefano Stabellini183d03c2010-05-17 17:08:21 +0100737EXPORT_SYMBOL_GPL(gnttab_max_grant_frames);
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700738
Stefano Stabellini289b7772010-12-10 14:54:44 +0000739int gnttab_map_refs(struct gnttab_map_grant_ref *map_ops,
Annie Lic1237992011-11-22 09:59:56 +0800740 struct gnttab_map_grant_ref *kmap_ops,
741 struct page **pages, unsigned int count)
Stefano Stabellini289b7772010-12-10 14:54:44 +0000742{
743 int i, ret;
744 pte_t *pte;
745 unsigned long mfn;
746
747 ret = HYPERVISOR_grant_table_op(GNTTABOP_map_grant_ref, map_ops, count);
Jeremy Fitzhardinge87f1d402010-12-13 14:42:30 +0000748 if (ret)
749 return ret;
Stefano Stabellini289b7772010-12-10 14:54:44 +0000750
Daniel De Graafaab8f112011-02-03 12:19:02 -0500751 if (xen_feature(XENFEAT_auto_translated_physmap))
752 return ret;
753
Stefano Stabellini289b7772010-12-10 14:54:44 +0000754 for (i = 0; i < count; i++) {
Ian Campbelldc4972a2011-03-04 17:38:21 +0000755 /* Do not add to override if the map failed. */
756 if (map_ops[i].status)
757 continue;
758
Konrad Rzeszutek Wilkcf8d9162011-02-28 17:58:48 -0500759 if (map_ops[i].flags & GNTMAP_contains_pte) {
760 pte = (pte_t *) (mfn_to_virt(PFN_DOWN(map_ops[i].host_addr)) +
Stefano Stabellini289b7772010-12-10 14:54:44 +0000761 (map_ops[i].host_addr & ~PAGE_MASK));
Konrad Rzeszutek Wilkcf8d9162011-02-28 17:58:48 -0500762 mfn = pte_mfn(*pte);
763 } else {
Daniel De Graaf7d17e842011-12-14 15:12:11 -0500764 mfn = PFN_DOWN(map_ops[i].dev_bus_addr);
Konrad Rzeszutek Wilkcf8d9162011-02-28 17:58:48 -0500765 }
Daniel De Graaf7d17e842011-12-14 15:12:11 -0500766 ret = m2p_add_override(mfn, pages[i], kmap_ops ?
767 &kmap_ops[i] : NULL);
Jeremy Fitzhardinge87f1d402010-12-13 14:42:30 +0000768 if (ret)
769 return ret;
Stefano Stabellini289b7772010-12-10 14:54:44 +0000770 }
771
772 return ret;
773}
774EXPORT_SYMBOL_GPL(gnttab_map_refs);
775
776int gnttab_unmap_refs(struct gnttab_unmap_grant_ref *unmap_ops,
Daniel De Graaf7d17e842011-12-14 15:12:11 -0500777 struct page **pages, unsigned int count, bool clear_pte)
Stefano Stabellini289b7772010-12-10 14:54:44 +0000778{
779 int i, ret;
780
781 ret = HYPERVISOR_grant_table_op(GNTTABOP_unmap_grant_ref, unmap_ops, count);
Jeremy Fitzhardinge87f1d402010-12-13 14:42:30 +0000782 if (ret)
783 return ret;
784
Daniel De Graafaab8f112011-02-03 12:19:02 -0500785 if (xen_feature(XENFEAT_auto_translated_physmap))
786 return ret;
787
Jeremy Fitzhardinge87f1d402010-12-13 14:42:30 +0000788 for (i = 0; i < count; i++) {
Daniel De Graaf7d17e842011-12-14 15:12:11 -0500789 ret = m2p_remove_override(pages[i], clear_pte);
Jeremy Fitzhardinge87f1d402010-12-13 14:42:30 +0000790 if (ret)
791 return ret;
792 }
Stefano Stabellini289b7772010-12-10 14:54:44 +0000793
794 return ret;
795}
796EXPORT_SYMBOL_GPL(gnttab_unmap_refs);
797
Annie Li85ff6ac2011-11-22 09:59:21 +0800798static unsigned nr_status_frames(unsigned nr_grant_frames)
799{
800 return (nr_grant_frames * GREFS_PER_GRANT_FRAME + SPP - 1) / SPP;
801}
802
Annie Li0f9f5a92011-11-22 09:58:06 +0800803static int gnttab_map_frames_v1(unsigned long *frames, unsigned int nr_gframes)
804{
805 int rc;
806
807 rc = arch_gnttab_map_shared(frames, nr_gframes,
808 gnttab_max_grant_frames(),
809 &gnttab_shared.addr);
810 BUG_ON(rc);
811
812 return 0;
813}
814
815static void gnttab_unmap_frames_v1(void)
816{
Annie Li85ff6ac2011-11-22 09:59:21 +0800817 arch_gnttab_unmap(gnttab_shared.addr, nr_grant_frames);
818}
819
820static int gnttab_map_frames_v2(unsigned long *frames, unsigned int nr_gframes)
821{
822 uint64_t *sframes;
823 unsigned int nr_sframes;
824 struct gnttab_get_status_frames getframes;
825 int rc;
826
827 nr_sframes = nr_status_frames(nr_gframes);
828
829 /* No need for kzalloc as it is initialized in following hypercall
830 * GNTTABOP_get_status_frames.
831 */
832 sframes = kmalloc(nr_sframes * sizeof(uint64_t), GFP_ATOMIC);
833 if (!sframes)
834 return -ENOMEM;
835
836 getframes.dom = DOMID_SELF;
837 getframes.nr_frames = nr_sframes;
838 set_xen_guest_handle(getframes.frame_list, sframes);
839
840 rc = HYPERVISOR_grant_table_op(GNTTABOP_get_status_frames,
841 &getframes, 1);
842 if (rc == -ENOSYS) {
843 kfree(sframes);
844 return -ENOSYS;
845 }
846
847 BUG_ON(rc || getframes.status);
848
849 rc = arch_gnttab_map_status(sframes, nr_sframes,
850 nr_status_frames(gnttab_max_grant_frames()),
851 &grstatus);
852 BUG_ON(rc);
853 kfree(sframes);
854
855 rc = arch_gnttab_map_shared(frames, nr_gframes,
856 gnttab_max_grant_frames(),
857 &gnttab_shared.addr);
858 BUG_ON(rc);
859
860 return 0;
861}
862
863static void gnttab_unmap_frames_v2(void)
864{
865 arch_gnttab_unmap(gnttab_shared.addr, nr_grant_frames);
866 arch_gnttab_unmap(grstatus, nr_status_frames(nr_grant_frames));
Annie Li0f9f5a92011-11-22 09:58:06 +0800867}
868
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700869static int gnttab_map(unsigned int start_idx, unsigned int end_idx)
870{
871 struct gnttab_setup_table setup;
872 unsigned long *frames;
873 unsigned int nr_gframes = end_idx + 1;
874 int rc;
875
Stefano Stabellini183d03c2010-05-17 17:08:21 +0100876 if (xen_hvm_domain()) {
877 struct xen_add_to_physmap xatp;
878 unsigned int i = end_idx;
879 rc = 0;
880 /*
881 * Loop backwards, so that the first hypercall has the largest
882 * index, ensuring that the table will grow only once.
883 */
884 do {
885 xatp.domid = DOMID_SELF;
886 xatp.idx = i;
887 xatp.space = XENMAPSPACE_grant_table;
888 xatp.gpfn = (xen_hvm_resume_frames >> PAGE_SHIFT) + i;
889 rc = HYPERVISOR_memory_op(XENMEM_add_to_physmap, &xatp);
890 if (rc != 0) {
891 printk(KERN_WARNING
892 "grant table add_to_physmap failed, err=%d\n", rc);
893 break;
894 }
895 } while (i-- > start_idx);
896
897 return rc;
898 }
899
Annie Li85ff6ac2011-11-22 09:59:21 +0800900 /* No need for kzalloc as it is initialized in following hypercall
901 * GNTTABOP_setup_table.
902 */
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700903 frames = kmalloc(nr_gframes * sizeof(unsigned long), GFP_ATOMIC);
904 if (!frames)
905 return -ENOMEM;
906
907 setup.dom = DOMID_SELF;
908 setup.nr_frames = nr_gframes;
Isaku Yamahata87e27cf2008-04-02 10:53:52 -0700909 set_xen_guest_handle(setup.frame_list, frames);
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700910
911 rc = HYPERVISOR_grant_table_op(GNTTABOP_setup_table, &setup, 1);
912 if (rc == -ENOSYS) {
913 kfree(frames);
914 return -ENOSYS;
915 }
916
917 BUG_ON(rc || setup.status);
918
Annie Li0f9f5a92011-11-22 09:58:06 +0800919 rc = gnttab_interface->map_frames(frames, nr_gframes);
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700920
921 kfree(frames);
922
Annie Li0f9f5a92011-11-22 09:58:06 +0800923 return rc;
924}
925
926static struct gnttab_ops gnttab_v1_ops = {
927 .map_frames = gnttab_map_frames_v1,
928 .unmap_frames = gnttab_unmap_frames_v1,
929 .update_entry = gnttab_update_entry_v1,
930 .end_foreign_access_ref = gnttab_end_foreign_access_ref_v1,
931 .end_foreign_transfer_ref = gnttab_end_foreign_transfer_ref_v1,
932 .query_foreign_access = gnttab_query_foreign_access_v1,
933};
934
Annie Li85ff6ac2011-11-22 09:59:21 +0800935static struct gnttab_ops gnttab_v2_ops = {
936 .map_frames = gnttab_map_frames_v2,
937 .unmap_frames = gnttab_unmap_frames_v2,
938 .update_entry = gnttab_update_entry_v2,
939 .end_foreign_access_ref = gnttab_end_foreign_access_ref_v2,
940 .end_foreign_transfer_ref = gnttab_end_foreign_transfer_ref_v2,
941 .query_foreign_access = gnttab_query_foreign_access_v2,
Annie Li66667542011-12-12 18:14:42 +0800942 .update_subpage_entry = gnttab_update_subpage_entry_v2,
Annie Li9438ce92011-12-12 18:15:07 +0800943 .update_trans_entry = gnttab_update_trans_entry_v2,
Annie Li85ff6ac2011-11-22 09:59:21 +0800944};
945
Annie Li0f9f5a92011-11-22 09:58:06 +0800946static void gnttab_request_version(void)
947{
Annie Li85ff6ac2011-11-22 09:59:21 +0800948 int rc;
949 struct gnttab_set_version gsv;
950
Konrad Rzeszutek Wilk69e8f432012-01-25 00:13:20 -0500951 if (xen_hvm_domain())
952 gsv.version = 1;
953 else
954 gsv.version = 2;
Annie Li85ff6ac2011-11-22 09:59:21 +0800955 rc = HYPERVISOR_grant_table_op(GNTTABOP_set_version, &gsv, 1);
Konrad Rzeszutek Wilk69e8f432012-01-25 00:13:20 -0500956 if (rc == 0 && gsv.version == 2) {
Annie Li85ff6ac2011-11-22 09:59:21 +0800957 grant_table_version = 2;
958 gnttab_interface = &gnttab_v2_ops;
959 } else if (grant_table_version == 2) {
960 /*
961 * If we've already used version 2 features,
962 * but then suddenly discover that they're not
963 * available (e.g. migrating to an older
964 * version of Xen), almost unbounded badness
965 * can happen.
966 */
967 panic("we need grant tables version 2, but only version 1 is available");
968 } else {
969 grant_table_version = 1;
970 gnttab_interface = &gnttab_v1_ops;
971 }
Annie Li0f9f5a92011-11-22 09:58:06 +0800972 printk(KERN_INFO "Grant tables using version %d layout.\n",
973 grant_table_version);
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700974}
975
Jeremy Fitzhardinge0e913982008-05-26 23:31:27 +0100976int gnttab_resume(void)
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700977{
Stefano Stabellini183d03c2010-05-17 17:08:21 +0100978 unsigned int max_nr_gframes;
979
Annie Li0f9f5a92011-11-22 09:58:06 +0800980 gnttab_request_version();
Stefano Stabellini183d03c2010-05-17 17:08:21 +0100981 max_nr_gframes = gnttab_max_grant_frames();
982 if (max_nr_gframes < nr_grant_frames)
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700983 return -ENOSYS;
Stefano Stabellini183d03c2010-05-17 17:08:21 +0100984
985 if (xen_pv_domain())
986 return gnttab_map(0, nr_grant_frames - 1);
987
Annie Li0f9f5a92011-11-22 09:58:06 +0800988 if (gnttab_shared.addr == NULL) {
989 gnttab_shared.addr = ioremap(xen_hvm_resume_frames,
990 PAGE_SIZE * max_nr_gframes);
991 if (gnttab_shared.addr == NULL) {
Stefano Stabellini183d03c2010-05-17 17:08:21 +0100992 printk(KERN_WARNING
993 "Failed to ioremap gnttab share frames!");
994 return -ENOMEM;
995 }
996 }
997
998 gnttab_map(0, nr_grant_frames - 1);
999
1000 return 0;
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -07001001}
1002
Jeremy Fitzhardinge0e913982008-05-26 23:31:27 +01001003int gnttab_suspend(void)
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -07001004{
Annie Li0f9f5a92011-11-22 09:58:06 +08001005 gnttab_interface->unmap_frames();
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -07001006 return 0;
1007}
1008
1009static int gnttab_expand(unsigned int req_entries)
1010{
1011 int rc;
1012 unsigned int cur, extra;
1013
1014 cur = nr_grant_frames;
1015 extra = ((req_entries + (GREFS_PER_GRANT_FRAME-1)) /
1016 GREFS_PER_GRANT_FRAME);
Stefano Stabellini183d03c2010-05-17 17:08:21 +01001017 if (cur + extra > gnttab_max_grant_frames())
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -07001018 return -ENOSPC;
1019
1020 rc = gnttab_map(cur, cur + extra - 1);
1021 if (rc == 0)
1022 rc = grow_gnttab_list(extra);
1023
1024 return rc;
1025}
1026
Stefano Stabellini183d03c2010-05-17 17:08:21 +01001027int gnttab_init(void)
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -07001028{
1029 int i;
Michael Abd-El-Malekbbc60c12008-04-04 02:33:48 -07001030 unsigned int max_nr_glist_frames, nr_glist_frames;
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -07001031 unsigned int nr_init_grefs;
1032
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -07001033 nr_grant_frames = 1;
1034 boot_max_nr_grant_frames = __max_nr_grant_frames();
1035
1036 /* Determine the maximum number of frames required for the
1037 * grant reference free list on the current hypervisor.
1038 */
1039 max_nr_glist_frames = (boot_max_nr_grant_frames *
Michael Abd-El-Malekbbc60c12008-04-04 02:33:48 -07001040 GREFS_PER_GRANT_FRAME / RPP);
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -07001041
1042 gnttab_list = kmalloc(max_nr_glist_frames * sizeof(grant_ref_t *),
1043 GFP_KERNEL);
1044 if (gnttab_list == NULL)
1045 return -ENOMEM;
1046
Michael Abd-El-Malekbbc60c12008-04-04 02:33:48 -07001047 nr_glist_frames = (nr_grant_frames * GREFS_PER_GRANT_FRAME + RPP - 1) / RPP;
1048 for (i = 0; i < nr_glist_frames; i++) {
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -07001049 gnttab_list[i] = (grant_ref_t *)__get_free_page(GFP_KERNEL);
1050 if (gnttab_list[i] == NULL)
1051 goto ini_nomem;
1052 }
1053
1054 if (gnttab_resume() < 0)
1055 return -ENODEV;
1056
1057 nr_init_grefs = nr_grant_frames * GREFS_PER_GRANT_FRAME;
1058
1059 for (i = NR_RESERVED_ENTRIES; i < nr_init_grefs - 1; i++)
1060 gnttab_entry(i) = i + 1;
1061
1062 gnttab_entry(nr_init_grefs - 1) = GNTTAB_LIST_END;
1063 gnttab_free_count = nr_init_grefs - NR_RESERVED_ENTRIES;
1064 gnttab_free_head = NR_RESERVED_ENTRIES;
1065
1066 printk("Grant table initialized\n");
1067 return 0;
1068
1069 ini_nomem:
1070 for (i--; i >= 0; i--)
1071 free_page((unsigned long)gnttab_list[i]);
1072 kfree(gnttab_list);
1073 return -ENOMEM;
1074}
Stefano Stabellini183d03c2010-05-17 17:08:21 +01001075EXPORT_SYMBOL_GPL(gnttab_init);
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -07001076
Stefano Stabellini183d03c2010-05-17 17:08:21 +01001077static int __devinit __gnttab_init(void)
1078{
1079 /* Delay grant-table initialization in the PV on HVM case */
1080 if (xen_hvm_domain())
1081 return 0;
1082
1083 if (!xen_pv_domain())
1084 return -ENODEV;
1085
1086 return gnttab_init();
1087}
1088
1089core_initcall(__gnttab_init);