blob: 0bfc1ef11259eccaa937eb8d0913e4e14e04832e [file] [log] [blame]
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -07001/******************************************************************************
2 * grant_table.c
3 *
4 * Granting foreign access to our memory reservation.
5 *
6 * Copyright (c) 2005-2006, Christopher Clark
7 * Copyright (c) 2004-2005, K A Fraser
8 *
9 * This program is free software; you can redistribute it and/or
10 * modify it under the terms of the GNU General Public License version 2
11 * as published by the Free Software Foundation; or, when distributed
12 * separately from the Linux kernel or incorporated into other
13 * software packages, subject to the following license:
14 *
15 * Permission is hereby granted, free of charge, to any person obtaining a copy
16 * of this source file (the "Software"), to deal in the Software without
17 * restriction, including without limitation the rights to use, copy, modify,
18 * merge, publish, distribute, sublicense, and/or sell copies of the Software,
19 * and to permit persons to whom the Software is furnished to do so, subject to
20 * the following conditions:
21 *
22 * The above copyright notice and this permission notice shall be included in
23 * all copies or substantial portions of the Software.
24 *
25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
26 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
27 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
28 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
29 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
30 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
31 * IN THE SOFTWARE.
32 */
33
34#include <linux/module.h>
35#include <linux/sched.h>
36#include <linux/mm.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090037#include <linux/slab.h>
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -070038#include <linux/vmalloc.h>
39#include <linux/uaccess.h>
Stefano Stabellini183d03c2010-05-17 17:08:21 +010040#include <linux/io.h>
Stefano Stabellinif62805f2012-04-24 11:55:43 +010041#include <linux/hardirq.h>
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -070042
Jeremy Fitzhardinge1ccbf532009-10-06 15:11:14 -070043#include <xen/xen.h>
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -070044#include <xen/interface/xen.h>
45#include <xen/page.h>
46#include <xen/grant_table.h>
Stefano Stabellini183d03c2010-05-17 17:08:21 +010047#include <xen/interface/memory.h>
Annie Li85ff6ac2011-11-22 09:59:21 +080048#include <xen/hvc-console.h>
Jeremy Fitzhardingeecbf29c2008-12-16 12:37:07 -080049#include <asm/xen/hypercall.h>
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -070050
51#include <asm/pgtable.h>
52#include <asm/sync_bitops.h>
53
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -070054/* External tools reserve first few grant table entries. */
55#define NR_RESERVED_ENTRIES 8
56#define GNTTAB_LIST_END 0xffffffff
Annie Li85ff6ac2011-11-22 09:59:21 +080057#define GREFS_PER_GRANT_FRAME \
58(grant_table_version == 1 ? \
59(PAGE_SIZE / sizeof(struct grant_entry_v1)) : \
60(PAGE_SIZE / sizeof(union grant_entry_v2)))
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -070061
62static grant_ref_t **gnttab_list;
63static unsigned int nr_grant_frames;
64static unsigned int boot_max_nr_grant_frames;
65static int gnttab_free_count;
66static grant_ref_t gnttab_free_head;
67static DEFINE_SPINLOCK(gnttab_list_lock);
Stefano Stabellini183d03c2010-05-17 17:08:21 +010068unsigned long xen_hvm_resume_frames;
69EXPORT_SYMBOL_GPL(xen_hvm_resume_frames);
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -070070
Annie Li0f9f5a92011-11-22 09:58:06 +080071static union {
72 struct grant_entry_v1 *v1;
Annie Li85ff6ac2011-11-22 09:59:21 +080073 union grant_entry_v2 *v2;
Annie Li0f9f5a92011-11-22 09:58:06 +080074 void *addr;
75} gnttab_shared;
76
77/*This is a structure of function pointers for grant table*/
78struct gnttab_ops {
79 /*
Annie Li9dbc71d2011-12-12 18:13:57 +080080 * Mapping a list of frames for storing grant entries. Frames parameter
81 * is used to store grant table address when grant table being setup,
82 * nr_gframes is the number of frames to map grant table. Returning
83 * GNTST_okay means success and negative value means failure.
Annie Li0f9f5a92011-11-22 09:58:06 +080084 */
Annie Li9dbc71d2011-12-12 18:13:57 +080085 int (*map_frames)(unsigned long *frames, unsigned int nr_gframes);
Annie Li0f9f5a92011-11-22 09:58:06 +080086 /*
87 * Release a list of frames which are mapped in map_frames for grant
88 * entry status.
89 */
90 void (*unmap_frames)(void);
91 /*
Annie Li9dbc71d2011-12-12 18:13:57 +080092 * Introducing a valid entry into the grant table, granting the frame of
93 * this grant entry to domain for accessing or transfering. Ref
94 * parameter is reference of this introduced grant entry, domid is id of
95 * granted domain, frame is the page frame to be granted, and flags is
96 * status of the grant entry to be updated.
Annie Li0f9f5a92011-11-22 09:58:06 +080097 */
Annie Li9dbc71d2011-12-12 18:13:57 +080098 void (*update_entry)(grant_ref_t ref, domid_t domid,
99 unsigned long frame, unsigned flags);
Annie Li0f9f5a92011-11-22 09:58:06 +0800100 /*
Annie Li9dbc71d2011-12-12 18:13:57 +0800101 * Stop granting a grant entry to domain for accessing. Ref parameter is
102 * reference of a grant entry whose grant access will be stopped,
103 * readonly is not in use in this function. If the grant entry is
Annie Li0f9f5a92011-11-22 09:58:06 +0800104 * currently mapped for reading or writing, just return failure(==0)
105 * directly and don't tear down the grant access. Otherwise, stop grant
106 * access for this entry and return success(==1).
107 */
Annie Li9dbc71d2011-12-12 18:13:57 +0800108 int (*end_foreign_access_ref)(grant_ref_t ref, int readonly);
Annie Li0f9f5a92011-11-22 09:58:06 +0800109 /*
Annie Li9dbc71d2011-12-12 18:13:57 +0800110 * Stop granting a grant entry to domain for transfer. Ref parameter is
111 * reference of a grant entry whose grant transfer will be stopped. If
112 * tranfer has not started, just reclaim the grant entry and return
113 * failure(==0). Otherwise, wait for the transfer to complete and then
114 * return the frame.
Annie Li0f9f5a92011-11-22 09:58:06 +0800115 */
Annie Li9dbc71d2011-12-12 18:13:57 +0800116 unsigned long (*end_foreign_transfer_ref)(grant_ref_t ref);
Annie Li0f9f5a92011-11-22 09:58:06 +0800117 /*
Annie Li9dbc71d2011-12-12 18:13:57 +0800118 * Query the status of a grant entry. Ref parameter is reference of
Annie Li0f9f5a92011-11-22 09:58:06 +0800119 * queried grant entry, return value is the status of queried entry.
120 * Detailed status(writing/reading) can be gotten from the return value
121 * by bit operations.
122 */
Annie Li9dbc71d2011-12-12 18:13:57 +0800123 int (*query_foreign_access)(grant_ref_t ref);
Annie Li66667542011-12-12 18:14:42 +0800124 /*
125 * Grant a domain to access a range of bytes within the page referred by
126 * an available grant entry. Ref parameter is reference of a grant entry
127 * which will be sub-page accessed, domid is id of grantee domain, frame
128 * is frame address of subpage grant, flags is grant type and flag
129 * information, page_off is offset of the range of bytes, and length is
130 * length of bytes to be accessed.
131 */
132 void (*update_subpage_entry)(grant_ref_t ref, domid_t domid,
133 unsigned long frame, int flags,
134 unsigned page_off, unsigned length);
Annie Li9438ce92011-12-12 18:15:07 +0800135 /*
136 * Redirect an available grant entry on domain A to another grant
137 * reference of domain B, then allow domain C to use grant reference
138 * of domain B transitively. Ref parameter is an available grant entry
139 * reference on domain A, domid is id of domain C which accesses grant
140 * entry transitively, flags is grant type and flag information,
141 * trans_domid is id of domain B whose grant entry is finally accessed
142 * transitively, trans_gref is grant entry transitive reference of
143 * domain B.
144 */
145 void (*update_trans_entry)(grant_ref_t ref, domid_t domid, int flags,
146 domid_t trans_domid, grant_ref_t trans_gref);
Annie Li0f9f5a92011-11-22 09:58:06 +0800147};
148
149static struct gnttab_ops *gnttab_interface;
150
Annie Li85ff6ac2011-11-22 09:59:21 +0800151/*This reflects status of grant entries, so act as a global value*/
152static grant_status_t *grstatus;
153
Annie Li0f9f5a92011-11-22 09:58:06 +0800154static int grant_table_version;
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700155
156static struct gnttab_free_callback *gnttab_free_callback_list;
157
158static int gnttab_expand(unsigned int req_entries);
159
160#define RPP (PAGE_SIZE / sizeof(grant_ref_t))
Annie Li85ff6ac2011-11-22 09:59:21 +0800161#define SPP (PAGE_SIZE / sizeof(grant_status_t))
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700162
163static inline grant_ref_t *__gnttab_entry(grant_ref_t entry)
164{
165 return &gnttab_list[(entry) / RPP][(entry) % RPP];
166}
167/* This can be used as an l-value */
168#define gnttab_entry(entry) (*__gnttab_entry(entry))
169
170static int get_free_entries(unsigned count)
171{
172 unsigned long flags;
Konrad Rzeszutek Wilk272800d2011-07-22 14:00:06 -0400173 int ref, rc = 0;
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700174 grant_ref_t head;
175
176 spin_lock_irqsave(&gnttab_list_lock, flags);
177
178 if ((gnttab_free_count < count) &&
179 ((rc = gnttab_expand(count - gnttab_free_count)) < 0)) {
180 spin_unlock_irqrestore(&gnttab_list_lock, flags);
181 return rc;
182 }
183
184 ref = head = gnttab_free_head;
185 gnttab_free_count -= count;
186 while (count-- > 1)
187 head = gnttab_entry(head);
188 gnttab_free_head = gnttab_entry(head);
189 gnttab_entry(head) = GNTTAB_LIST_END;
190
191 spin_unlock_irqrestore(&gnttab_list_lock, flags);
192
193 return ref;
194}
195
196static void do_free_callbacks(void)
197{
198 struct gnttab_free_callback *callback, *next;
199
200 callback = gnttab_free_callback_list;
201 gnttab_free_callback_list = NULL;
202
203 while (callback != NULL) {
204 next = callback->next;
205 if (gnttab_free_count >= callback->count) {
206 callback->next = NULL;
207 callback->fn(callback->arg);
208 } else {
209 callback->next = gnttab_free_callback_list;
210 gnttab_free_callback_list = callback;
211 }
212 callback = next;
213 }
214}
215
216static inline void check_free_callbacks(void)
217{
218 if (unlikely(gnttab_free_callback_list))
219 do_free_callbacks();
220}
221
222static void put_free_entry(grant_ref_t ref)
223{
224 unsigned long flags;
225 spin_lock_irqsave(&gnttab_list_lock, flags);
226 gnttab_entry(ref) = gnttab_free_head;
227 gnttab_free_head = ref;
228 gnttab_free_count++;
229 check_free_callbacks();
230 spin_unlock_irqrestore(&gnttab_list_lock, flags);
231}
232
Annie Li0f9f5a92011-11-22 09:58:06 +0800233/*
Annie Li85ff6ac2011-11-22 09:59:21 +0800234 * Following applies to gnttab_update_entry_v1 and gnttab_update_entry_v2.
Annie Li0f9f5a92011-11-22 09:58:06 +0800235 * Introducing a valid entry into the grant table:
236 * 1. Write ent->domid.
237 * 2. Write ent->frame:
238 * GTF_permit_access: Frame to which access is permitted.
239 * GTF_accept_transfer: Pseudo-phys frame slot being filled by new
240 * frame, or zero if none.
241 * 3. Write memory barrier (WMB).
242 * 4. Write ent->flags, inc. valid type.
243 */
244static void gnttab_update_entry_v1(grant_ref_t ref, domid_t domid,
245 unsigned long frame, unsigned flags)
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700246{
Annie Li0f9f5a92011-11-22 09:58:06 +0800247 gnttab_shared.v1[ref].domid = domid;
248 gnttab_shared.v1[ref].frame = frame;
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700249 wmb();
Annie Li0f9f5a92011-11-22 09:58:06 +0800250 gnttab_shared.v1[ref].flags = flags;
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700251}
252
Annie Li85ff6ac2011-11-22 09:59:21 +0800253static void gnttab_update_entry_v2(grant_ref_t ref, domid_t domid,
254 unsigned long frame, unsigned flags)
255{
256 gnttab_shared.v2[ref].hdr.domid = domid;
257 gnttab_shared.v2[ref].full_page.frame = frame;
258 wmb();
259 gnttab_shared.v2[ref].hdr.flags = GTF_permit_access | flags;
260}
261
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700262/*
263 * Public grant-issuing interface functions
264 */
265void gnttab_grant_foreign_access_ref(grant_ref_t ref, domid_t domid,
266 unsigned long frame, int readonly)
267{
Annie Li0f9f5a92011-11-22 09:58:06 +0800268 gnttab_interface->update_entry(ref, domid, frame,
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700269 GTF_permit_access | (readonly ? GTF_readonly : 0));
270}
271EXPORT_SYMBOL_GPL(gnttab_grant_foreign_access_ref);
272
273int gnttab_grant_foreign_access(domid_t domid, unsigned long frame,
274 int readonly)
275{
276 int ref;
277
278 ref = get_free_entries(1);
279 if (unlikely(ref < 0))
280 return -ENOSPC;
281
282 gnttab_grant_foreign_access_ref(ref, domid, frame, readonly);
283
284 return ref;
285}
286EXPORT_SYMBOL_GPL(gnttab_grant_foreign_access);
287
Annie Li66667542011-12-12 18:14:42 +0800288void gnttab_update_subpage_entry_v2(grant_ref_t ref, domid_t domid,
289 unsigned long frame, int flags,
290 unsigned page_off,
291 unsigned length)
292{
293 gnttab_shared.v2[ref].sub_page.frame = frame;
294 gnttab_shared.v2[ref].sub_page.page_off = page_off;
295 gnttab_shared.v2[ref].sub_page.length = length;
296 gnttab_shared.v2[ref].hdr.domid = domid;
297 wmb();
298 gnttab_shared.v2[ref].hdr.flags =
299 GTF_permit_access | GTF_sub_page | flags;
300}
301
302int gnttab_grant_foreign_access_subpage_ref(grant_ref_t ref, domid_t domid,
303 unsigned long frame, int flags,
304 unsigned page_off,
305 unsigned length)
306{
307 if (flags & (GTF_accept_transfer | GTF_reading |
308 GTF_writing | GTF_transitive))
309 return -EPERM;
310
311 if (gnttab_interface->update_subpage_entry == NULL)
312 return -ENOSYS;
313
314 gnttab_interface->update_subpage_entry(ref, domid, frame, flags,
315 page_off, length);
316
317 return 0;
318}
319EXPORT_SYMBOL_GPL(gnttab_grant_foreign_access_subpage_ref);
320
321int gnttab_grant_foreign_access_subpage(domid_t domid, unsigned long frame,
322 int flags, unsigned page_off,
323 unsigned length)
324{
325 int ref, rc;
326
327 ref = get_free_entries(1);
328 if (unlikely(ref < 0))
329 return -ENOSPC;
330
331 rc = gnttab_grant_foreign_access_subpage_ref(ref, domid, frame, flags,
332 page_off, length);
333 if (rc < 0) {
334 put_free_entry(ref);
335 return rc;
336 }
337
338 return ref;
339}
340EXPORT_SYMBOL_GPL(gnttab_grant_foreign_access_subpage);
341
342bool gnttab_subpage_grants_available(void)
343{
344 return gnttab_interface->update_subpage_entry != NULL;
345}
346EXPORT_SYMBOL_GPL(gnttab_subpage_grants_available);
347
Annie Li9438ce92011-12-12 18:15:07 +0800348void gnttab_update_trans_entry_v2(grant_ref_t ref, domid_t domid,
349 int flags, domid_t trans_domid,
350 grant_ref_t trans_gref)
351{
352 gnttab_shared.v2[ref].transitive.trans_domid = trans_domid;
353 gnttab_shared.v2[ref].transitive.gref = trans_gref;
354 gnttab_shared.v2[ref].hdr.domid = domid;
355 wmb();
356 gnttab_shared.v2[ref].hdr.flags =
357 GTF_permit_access | GTF_transitive | flags;
358}
359
360int gnttab_grant_foreign_access_trans_ref(grant_ref_t ref, domid_t domid,
361 int flags, domid_t trans_domid,
362 grant_ref_t trans_gref)
363{
364 if (flags & (GTF_accept_transfer | GTF_reading |
365 GTF_writing | GTF_sub_page))
366 return -EPERM;
367
368 if (gnttab_interface->update_trans_entry == NULL)
369 return -ENOSYS;
370
371 gnttab_interface->update_trans_entry(ref, domid, flags, trans_domid,
372 trans_gref);
373
374 return 0;
375}
376EXPORT_SYMBOL_GPL(gnttab_grant_foreign_access_trans_ref);
377
378int gnttab_grant_foreign_access_trans(domid_t domid, int flags,
379 domid_t trans_domid,
380 grant_ref_t trans_gref)
381{
382 int ref, rc;
383
384 ref = get_free_entries(1);
385 if (unlikely(ref < 0))
386 return -ENOSPC;
387
388 rc = gnttab_grant_foreign_access_trans_ref(ref, domid, flags,
389 trans_domid, trans_gref);
390 if (rc < 0) {
391 put_free_entry(ref);
392 return rc;
393 }
394
395 return ref;
396}
397EXPORT_SYMBOL_GPL(gnttab_grant_foreign_access_trans);
398
399bool gnttab_trans_grants_available(void)
400{
401 return gnttab_interface->update_trans_entry != NULL;
402}
403EXPORT_SYMBOL_GPL(gnttab_trans_grants_available);
404
Annie Li0f9f5a92011-11-22 09:58:06 +0800405static int gnttab_query_foreign_access_v1(grant_ref_t ref)
406{
407 return gnttab_shared.v1[ref].flags & (GTF_reading|GTF_writing);
408}
409
Annie Li85ff6ac2011-11-22 09:59:21 +0800410static int gnttab_query_foreign_access_v2(grant_ref_t ref)
411{
412 return grstatus[ref] & (GTF_reading|GTF_writing);
413}
414
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700415int gnttab_query_foreign_access(grant_ref_t ref)
416{
Annie Li0f9f5a92011-11-22 09:58:06 +0800417 return gnttab_interface->query_foreign_access(ref);
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700418}
419EXPORT_SYMBOL_GPL(gnttab_query_foreign_access);
420
Annie Li0f9f5a92011-11-22 09:58:06 +0800421static int gnttab_end_foreign_access_ref_v1(grant_ref_t ref, int readonly)
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700422{
423 u16 flags, nflags;
Annie Lib1e495b2011-11-22 09:58:47 +0800424 u16 *pflags;
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700425
Annie Lib1e495b2011-11-22 09:58:47 +0800426 pflags = &gnttab_shared.v1[ref].flags;
427 nflags = *pflags;
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700428 do {
429 flags = nflags;
Jan Beulich569ca5b2012-04-05 16:10:07 +0100430 if (flags & (GTF_reading|GTF_writing))
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700431 return 0;
Annie Lib1e495b2011-11-22 09:58:47 +0800432 } while ((nflags = sync_cmpxchg(pflags, flags, 0)) != flags);
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700433
434 return 1;
435}
Annie Li0f9f5a92011-11-22 09:58:06 +0800436
Annie Li85ff6ac2011-11-22 09:59:21 +0800437static int gnttab_end_foreign_access_ref_v2(grant_ref_t ref, int readonly)
438{
439 gnttab_shared.v2[ref].hdr.flags = 0;
440 mb();
441 if (grstatus[ref] & (GTF_reading|GTF_writing)) {
442 return 0;
443 } else {
444 /* The read of grstatus needs to have acquire
445 semantics. On x86, reads already have
446 that, and we just need to protect against
447 compiler reorderings. On other
448 architectures we may need a full
449 barrier. */
450#ifdef CONFIG_X86
451 barrier();
452#else
453 mb();
454#endif
455 }
456
457 return 1;
458}
459
Jan Beulich569ca5b2012-04-05 16:10:07 +0100460static inline int _gnttab_end_foreign_access_ref(grant_ref_t ref, int readonly)
Annie Li0f9f5a92011-11-22 09:58:06 +0800461{
462 return gnttab_interface->end_foreign_access_ref(ref, readonly);
463}
Jan Beulich569ca5b2012-04-05 16:10:07 +0100464
465int gnttab_end_foreign_access_ref(grant_ref_t ref, int readonly)
466{
467 if (_gnttab_end_foreign_access_ref(ref, readonly))
468 return 1;
469 pr_warn("WARNING: g.e. %#x still in use!\n", ref);
470 return 0;
471}
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700472EXPORT_SYMBOL_GPL(gnttab_end_foreign_access_ref);
473
Jan Beulich569ca5b2012-04-05 16:10:07 +0100474struct deferred_entry {
475 struct list_head list;
476 grant_ref_t ref;
477 bool ro;
478 uint16_t warn_delay;
479 struct page *page;
480};
481static LIST_HEAD(deferred_list);
482static void gnttab_handle_deferred(unsigned long);
483static DEFINE_TIMER(deferred_timer, gnttab_handle_deferred, 0, 0);
484
485static void gnttab_handle_deferred(unsigned long unused)
486{
487 unsigned int nr = 10;
488 struct deferred_entry *first = NULL;
489 unsigned long flags;
490
491 spin_lock_irqsave(&gnttab_list_lock, flags);
492 while (nr--) {
493 struct deferred_entry *entry
494 = list_first_entry(&deferred_list,
495 struct deferred_entry, list);
496
497 if (entry == first)
498 break;
499 list_del(&entry->list);
500 spin_unlock_irqrestore(&gnttab_list_lock, flags);
501 if (_gnttab_end_foreign_access_ref(entry->ref, entry->ro)) {
502 put_free_entry(entry->ref);
503 if (entry->page) {
504 pr_debug("freeing g.e. %#x (pfn %#lx)\n",
505 entry->ref, page_to_pfn(entry->page));
506 __free_page(entry->page);
507 } else
508 pr_info("freeing g.e. %#x\n", entry->ref);
509 kfree(entry);
510 entry = NULL;
511 } else {
512 if (!--entry->warn_delay)
513 pr_info("g.e. %#x still pending\n",
514 entry->ref);
515 if (!first)
516 first = entry;
517 }
518 spin_lock_irqsave(&gnttab_list_lock, flags);
519 if (entry)
520 list_add_tail(&entry->list, &deferred_list);
521 else if (list_empty(&deferred_list))
522 break;
523 }
524 if (!list_empty(&deferred_list) && !timer_pending(&deferred_timer)) {
525 deferred_timer.expires = jiffies + HZ;
526 add_timer(&deferred_timer);
527 }
528 spin_unlock_irqrestore(&gnttab_list_lock, flags);
529}
530
531static void gnttab_add_deferred(grant_ref_t ref, bool readonly,
532 struct page *page)
533{
534 struct deferred_entry *entry = kmalloc(sizeof(*entry), GFP_ATOMIC);
535 const char *what = KERN_WARNING "leaking";
536
537 if (entry) {
538 unsigned long flags;
539
540 entry->ref = ref;
541 entry->ro = readonly;
542 entry->page = page;
543 entry->warn_delay = 60;
544 spin_lock_irqsave(&gnttab_list_lock, flags);
545 list_add_tail(&entry->list, &deferred_list);
546 if (!timer_pending(&deferred_timer)) {
547 deferred_timer.expires = jiffies + HZ;
548 add_timer(&deferred_timer);
549 }
550 spin_unlock_irqrestore(&gnttab_list_lock, flags);
551 what = KERN_DEBUG "deferring";
552 }
553 printk("%s g.e. %#x (pfn %#lx)\n",
554 what, ref, page ? page_to_pfn(page) : -1);
555}
556
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700557void gnttab_end_foreign_access(grant_ref_t ref, int readonly,
558 unsigned long page)
559{
560 if (gnttab_end_foreign_access_ref(ref, readonly)) {
561 put_free_entry(ref);
562 if (page != 0)
563 free_page(page);
Jan Beulich569ca5b2012-04-05 16:10:07 +0100564 } else
565 gnttab_add_deferred(ref, readonly,
566 page ? virt_to_page(page) : NULL);
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700567}
568EXPORT_SYMBOL_GPL(gnttab_end_foreign_access);
569
570int gnttab_grant_foreign_transfer(domid_t domid, unsigned long pfn)
571{
572 int ref;
573
574 ref = get_free_entries(1);
575 if (unlikely(ref < 0))
576 return -ENOSPC;
577 gnttab_grant_foreign_transfer_ref(ref, domid, pfn);
578
579 return ref;
580}
581EXPORT_SYMBOL_GPL(gnttab_grant_foreign_transfer);
582
583void gnttab_grant_foreign_transfer_ref(grant_ref_t ref, domid_t domid,
584 unsigned long pfn)
585{
Annie Li0f9f5a92011-11-22 09:58:06 +0800586 gnttab_interface->update_entry(ref, domid, pfn, GTF_accept_transfer);
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700587}
588EXPORT_SYMBOL_GPL(gnttab_grant_foreign_transfer_ref);
589
Annie Li0f9f5a92011-11-22 09:58:06 +0800590static unsigned long gnttab_end_foreign_transfer_ref_v1(grant_ref_t ref)
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700591{
592 unsigned long frame;
593 u16 flags;
Annie Lib1e495b2011-11-22 09:58:47 +0800594 u16 *pflags;
595
596 pflags = &gnttab_shared.v1[ref].flags;
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700597
598 /*
599 * If a transfer is not even yet started, try to reclaim the grant
600 * reference and return failure (== 0).
601 */
Annie Lib1e495b2011-11-22 09:58:47 +0800602 while (!((flags = *pflags) & GTF_transfer_committed)) {
603 if (sync_cmpxchg(pflags, flags, 0) == flags)
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700604 return 0;
605 cpu_relax();
606 }
607
608 /* If a transfer is in progress then wait until it is completed. */
609 while (!(flags & GTF_transfer_completed)) {
Annie Lib1e495b2011-11-22 09:58:47 +0800610 flags = *pflags;
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700611 cpu_relax();
612 }
613
614 rmb(); /* Read the frame number /after/ reading completion status. */
Annie Li0f9f5a92011-11-22 09:58:06 +0800615 frame = gnttab_shared.v1[ref].frame;
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700616 BUG_ON(frame == 0);
617
618 return frame;
619}
Annie Li0f9f5a92011-11-22 09:58:06 +0800620
Annie Li85ff6ac2011-11-22 09:59:21 +0800621static unsigned long gnttab_end_foreign_transfer_ref_v2(grant_ref_t ref)
622{
623 unsigned long frame;
624 u16 flags;
625 u16 *pflags;
626
627 pflags = &gnttab_shared.v2[ref].hdr.flags;
628
629 /*
630 * If a transfer is not even yet started, try to reclaim the grant
631 * reference and return failure (== 0).
632 */
633 while (!((flags = *pflags) & GTF_transfer_committed)) {
634 if (sync_cmpxchg(pflags, flags, 0) == flags)
635 return 0;
636 cpu_relax();
637 }
638
639 /* If a transfer is in progress then wait until it is completed. */
640 while (!(flags & GTF_transfer_completed)) {
641 flags = *pflags;
642 cpu_relax();
643 }
644
645 rmb(); /* Read the frame number /after/ reading completion status. */
646 frame = gnttab_shared.v2[ref].full_page.frame;
647 BUG_ON(frame == 0);
648
649 return frame;
650}
651
Annie Li0f9f5a92011-11-22 09:58:06 +0800652unsigned long gnttab_end_foreign_transfer_ref(grant_ref_t ref)
653{
654 return gnttab_interface->end_foreign_transfer_ref(ref);
655}
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700656EXPORT_SYMBOL_GPL(gnttab_end_foreign_transfer_ref);
657
658unsigned long gnttab_end_foreign_transfer(grant_ref_t ref)
659{
660 unsigned long frame = gnttab_end_foreign_transfer_ref(ref);
661 put_free_entry(ref);
662 return frame;
663}
664EXPORT_SYMBOL_GPL(gnttab_end_foreign_transfer);
665
666void gnttab_free_grant_reference(grant_ref_t ref)
667{
668 put_free_entry(ref);
669}
670EXPORT_SYMBOL_GPL(gnttab_free_grant_reference);
671
672void gnttab_free_grant_references(grant_ref_t head)
673{
674 grant_ref_t ref;
675 unsigned long flags;
676 int count = 1;
677 if (head == GNTTAB_LIST_END)
678 return;
679 spin_lock_irqsave(&gnttab_list_lock, flags);
680 ref = head;
681 while (gnttab_entry(ref) != GNTTAB_LIST_END) {
682 ref = gnttab_entry(ref);
683 count++;
684 }
685 gnttab_entry(ref) = gnttab_free_head;
686 gnttab_free_head = head;
687 gnttab_free_count += count;
688 check_free_callbacks();
689 spin_unlock_irqrestore(&gnttab_list_lock, flags);
690}
691EXPORT_SYMBOL_GPL(gnttab_free_grant_references);
692
693int gnttab_alloc_grant_references(u16 count, grant_ref_t *head)
694{
695 int h = get_free_entries(count);
696
697 if (h < 0)
698 return -ENOSPC;
699
700 *head = h;
701
702 return 0;
703}
704EXPORT_SYMBOL_GPL(gnttab_alloc_grant_references);
705
706int gnttab_empty_grant_references(const grant_ref_t *private_head)
707{
708 return (*private_head == GNTTAB_LIST_END);
709}
710EXPORT_SYMBOL_GPL(gnttab_empty_grant_references);
711
712int gnttab_claim_grant_reference(grant_ref_t *private_head)
713{
714 grant_ref_t g = *private_head;
715 if (unlikely(g == GNTTAB_LIST_END))
716 return -ENOSPC;
717 *private_head = gnttab_entry(g);
718 return g;
719}
720EXPORT_SYMBOL_GPL(gnttab_claim_grant_reference);
721
722void gnttab_release_grant_reference(grant_ref_t *private_head,
723 grant_ref_t release)
724{
725 gnttab_entry(release) = *private_head;
726 *private_head = release;
727}
728EXPORT_SYMBOL_GPL(gnttab_release_grant_reference);
729
730void gnttab_request_free_callback(struct gnttab_free_callback *callback,
731 void (*fn)(void *), void *arg, u16 count)
732{
733 unsigned long flags;
734 spin_lock_irqsave(&gnttab_list_lock, flags);
735 if (callback->next)
736 goto out;
737 callback->fn = fn;
738 callback->arg = arg;
739 callback->count = count;
740 callback->next = gnttab_free_callback_list;
741 gnttab_free_callback_list = callback;
742 check_free_callbacks();
743out:
744 spin_unlock_irqrestore(&gnttab_list_lock, flags);
745}
746EXPORT_SYMBOL_GPL(gnttab_request_free_callback);
747
748void gnttab_cancel_free_callback(struct gnttab_free_callback *callback)
749{
750 struct gnttab_free_callback **pcb;
751 unsigned long flags;
752
753 spin_lock_irqsave(&gnttab_list_lock, flags);
754 for (pcb = &gnttab_free_callback_list; *pcb; pcb = &(*pcb)->next) {
755 if (*pcb == callback) {
756 *pcb = callback->next;
757 break;
758 }
759 }
760 spin_unlock_irqrestore(&gnttab_list_lock, flags);
761}
762EXPORT_SYMBOL_GPL(gnttab_cancel_free_callback);
763
764static int grow_gnttab_list(unsigned int more_frames)
765{
766 unsigned int new_nr_grant_frames, extra_entries, i;
Michael Abd-El-Malekbbc60c12008-04-04 02:33:48 -0700767 unsigned int nr_glist_frames, new_nr_glist_frames;
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700768
769 new_nr_grant_frames = nr_grant_frames + more_frames;
770 extra_entries = more_frames * GREFS_PER_GRANT_FRAME;
771
Michael Abd-El-Malekbbc60c12008-04-04 02:33:48 -0700772 nr_glist_frames = (nr_grant_frames * GREFS_PER_GRANT_FRAME + RPP - 1) / RPP;
773 new_nr_glist_frames =
774 (new_nr_grant_frames * GREFS_PER_GRANT_FRAME + RPP - 1) / RPP;
775 for (i = nr_glist_frames; i < new_nr_glist_frames; i++) {
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700776 gnttab_list[i] = (grant_ref_t *)__get_free_page(GFP_ATOMIC);
777 if (!gnttab_list[i])
778 goto grow_nomem;
779 }
780
781
782 for (i = GREFS_PER_GRANT_FRAME * nr_grant_frames;
783 i < GREFS_PER_GRANT_FRAME * new_nr_grant_frames - 1; i++)
784 gnttab_entry(i) = i + 1;
785
786 gnttab_entry(i) = gnttab_free_head;
787 gnttab_free_head = GREFS_PER_GRANT_FRAME * nr_grant_frames;
788 gnttab_free_count += extra_entries;
789
790 nr_grant_frames = new_nr_grant_frames;
791
792 check_free_callbacks();
793
794 return 0;
795
796grow_nomem:
Michael Abd-El-Malekbbc60c12008-04-04 02:33:48 -0700797 for ( ; i >= nr_glist_frames; i--)
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700798 free_page((unsigned long) gnttab_list[i]);
799 return -ENOMEM;
800}
801
802static unsigned int __max_nr_grant_frames(void)
803{
804 struct gnttab_query_size query;
805 int rc;
806
807 query.dom = DOMID_SELF;
808
809 rc = HYPERVISOR_grant_table_op(GNTTABOP_query_size, &query, 1);
810 if ((rc < 0) || (query.status != GNTST_okay))
811 return 4; /* Legacy max supported number of frames */
812
813 return query.max_nr_frames;
814}
815
Stefano Stabellini183d03c2010-05-17 17:08:21 +0100816unsigned int gnttab_max_grant_frames(void)
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700817{
818 unsigned int xen_max = __max_nr_grant_frames();
819
820 if (xen_max > boot_max_nr_grant_frames)
821 return boot_max_nr_grant_frames;
822 return xen_max;
823}
Stefano Stabellini183d03c2010-05-17 17:08:21 +0100824EXPORT_SYMBOL_GPL(gnttab_max_grant_frames);
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700825
Stefano Stabellini289b7772010-12-10 14:54:44 +0000826int gnttab_map_refs(struct gnttab_map_grant_ref *map_ops,
Annie Lic1237992011-11-22 09:59:56 +0800827 struct gnttab_map_grant_ref *kmap_ops,
828 struct page **pages, unsigned int count)
Stefano Stabellini289b7772010-12-10 14:54:44 +0000829{
830 int i, ret;
Stefano Stabellinif62805f2012-04-24 11:55:43 +0100831 bool lazy = false;
Stefano Stabellini289b7772010-12-10 14:54:44 +0000832 pte_t *pte;
833 unsigned long mfn;
834
835 ret = HYPERVISOR_grant_table_op(GNTTABOP_map_grant_ref, map_ops, count);
Jeremy Fitzhardinge87f1d402010-12-13 14:42:30 +0000836 if (ret)
837 return ret;
Stefano Stabellini289b7772010-12-10 14:54:44 +0000838
Daniel De Graafaab8f112011-02-03 12:19:02 -0500839 if (xen_feature(XENFEAT_auto_translated_physmap))
840 return ret;
841
Stefano Stabellinif62805f2012-04-24 11:55:43 +0100842 if (!in_interrupt() && paravirt_get_lazy_mode() == PARAVIRT_LAZY_NONE) {
843 arch_enter_lazy_mmu_mode();
844 lazy = true;
845 }
846
Stefano Stabellini289b7772010-12-10 14:54:44 +0000847 for (i = 0; i < count; i++) {
Ian Campbelldc4972a2011-03-04 17:38:21 +0000848 /* Do not add to override if the map failed. */
849 if (map_ops[i].status)
850 continue;
851
Konrad Rzeszutek Wilkcf8d9162011-02-28 17:58:48 -0500852 if (map_ops[i].flags & GNTMAP_contains_pte) {
853 pte = (pte_t *) (mfn_to_virt(PFN_DOWN(map_ops[i].host_addr)) +
Stefano Stabellini289b7772010-12-10 14:54:44 +0000854 (map_ops[i].host_addr & ~PAGE_MASK));
Konrad Rzeszutek Wilkcf8d9162011-02-28 17:58:48 -0500855 mfn = pte_mfn(*pte);
856 } else {
Daniel De Graaf7d17e842011-12-14 15:12:11 -0500857 mfn = PFN_DOWN(map_ops[i].dev_bus_addr);
Konrad Rzeszutek Wilkcf8d9162011-02-28 17:58:48 -0500858 }
Daniel De Graaf7d17e842011-12-14 15:12:11 -0500859 ret = m2p_add_override(mfn, pages[i], kmap_ops ?
860 &kmap_ops[i] : NULL);
Jeremy Fitzhardinge87f1d402010-12-13 14:42:30 +0000861 if (ret)
862 return ret;
Stefano Stabellini289b7772010-12-10 14:54:44 +0000863 }
864
Stefano Stabellinif62805f2012-04-24 11:55:43 +0100865 if (lazy)
866 arch_leave_lazy_mmu_mode();
867
Stefano Stabellini289b7772010-12-10 14:54:44 +0000868 return ret;
869}
870EXPORT_SYMBOL_GPL(gnttab_map_refs);
871
872int gnttab_unmap_refs(struct gnttab_unmap_grant_ref *unmap_ops,
Daniel De Graaf7d17e842011-12-14 15:12:11 -0500873 struct page **pages, unsigned int count, bool clear_pte)
Stefano Stabellini289b7772010-12-10 14:54:44 +0000874{
875 int i, ret;
Stefano Stabellinif62805f2012-04-24 11:55:43 +0100876 bool lazy = false;
Stefano Stabellini289b7772010-12-10 14:54:44 +0000877
878 ret = HYPERVISOR_grant_table_op(GNTTABOP_unmap_grant_ref, unmap_ops, count);
Jeremy Fitzhardinge87f1d402010-12-13 14:42:30 +0000879 if (ret)
880 return ret;
881
Daniel De Graafaab8f112011-02-03 12:19:02 -0500882 if (xen_feature(XENFEAT_auto_translated_physmap))
883 return ret;
884
Stefano Stabellinif62805f2012-04-24 11:55:43 +0100885 if (!in_interrupt() && paravirt_get_lazy_mode() == PARAVIRT_LAZY_NONE) {
886 arch_enter_lazy_mmu_mode();
887 lazy = true;
888 }
889
Jeremy Fitzhardinge87f1d402010-12-13 14:42:30 +0000890 for (i = 0; i < count; i++) {
Daniel De Graaf7d17e842011-12-14 15:12:11 -0500891 ret = m2p_remove_override(pages[i], clear_pte);
Jeremy Fitzhardinge87f1d402010-12-13 14:42:30 +0000892 if (ret)
893 return ret;
894 }
Stefano Stabellini289b7772010-12-10 14:54:44 +0000895
Stefano Stabellinif62805f2012-04-24 11:55:43 +0100896 if (lazy)
897 arch_leave_lazy_mmu_mode();
898
Stefano Stabellini289b7772010-12-10 14:54:44 +0000899 return ret;
900}
901EXPORT_SYMBOL_GPL(gnttab_unmap_refs);
902
Annie Li85ff6ac2011-11-22 09:59:21 +0800903static unsigned nr_status_frames(unsigned nr_grant_frames)
904{
905 return (nr_grant_frames * GREFS_PER_GRANT_FRAME + SPP - 1) / SPP;
906}
907
Annie Li0f9f5a92011-11-22 09:58:06 +0800908static int gnttab_map_frames_v1(unsigned long *frames, unsigned int nr_gframes)
909{
910 int rc;
911
912 rc = arch_gnttab_map_shared(frames, nr_gframes,
913 gnttab_max_grant_frames(),
914 &gnttab_shared.addr);
915 BUG_ON(rc);
916
917 return 0;
918}
919
920static void gnttab_unmap_frames_v1(void)
921{
Annie Li85ff6ac2011-11-22 09:59:21 +0800922 arch_gnttab_unmap(gnttab_shared.addr, nr_grant_frames);
923}
924
925static int gnttab_map_frames_v2(unsigned long *frames, unsigned int nr_gframes)
926{
927 uint64_t *sframes;
928 unsigned int nr_sframes;
929 struct gnttab_get_status_frames getframes;
930 int rc;
931
932 nr_sframes = nr_status_frames(nr_gframes);
933
934 /* No need for kzalloc as it is initialized in following hypercall
935 * GNTTABOP_get_status_frames.
936 */
937 sframes = kmalloc(nr_sframes * sizeof(uint64_t), GFP_ATOMIC);
938 if (!sframes)
939 return -ENOMEM;
940
941 getframes.dom = DOMID_SELF;
942 getframes.nr_frames = nr_sframes;
943 set_xen_guest_handle(getframes.frame_list, sframes);
944
945 rc = HYPERVISOR_grant_table_op(GNTTABOP_get_status_frames,
946 &getframes, 1);
947 if (rc == -ENOSYS) {
948 kfree(sframes);
949 return -ENOSYS;
950 }
951
952 BUG_ON(rc || getframes.status);
953
954 rc = arch_gnttab_map_status(sframes, nr_sframes,
955 nr_status_frames(gnttab_max_grant_frames()),
956 &grstatus);
957 BUG_ON(rc);
958 kfree(sframes);
959
960 rc = arch_gnttab_map_shared(frames, nr_gframes,
961 gnttab_max_grant_frames(),
962 &gnttab_shared.addr);
963 BUG_ON(rc);
964
965 return 0;
966}
967
968static void gnttab_unmap_frames_v2(void)
969{
970 arch_gnttab_unmap(gnttab_shared.addr, nr_grant_frames);
971 arch_gnttab_unmap(grstatus, nr_status_frames(nr_grant_frames));
Annie Li0f9f5a92011-11-22 09:58:06 +0800972}
973
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700974static int gnttab_map(unsigned int start_idx, unsigned int end_idx)
975{
976 struct gnttab_setup_table setup;
977 unsigned long *frames;
978 unsigned int nr_gframes = end_idx + 1;
979 int rc;
980
Stefano Stabellini183d03c2010-05-17 17:08:21 +0100981 if (xen_hvm_domain()) {
982 struct xen_add_to_physmap xatp;
983 unsigned int i = end_idx;
984 rc = 0;
985 /*
986 * Loop backwards, so that the first hypercall has the largest
987 * index, ensuring that the table will grow only once.
988 */
989 do {
990 xatp.domid = DOMID_SELF;
991 xatp.idx = i;
992 xatp.space = XENMAPSPACE_grant_table;
993 xatp.gpfn = (xen_hvm_resume_frames >> PAGE_SHIFT) + i;
994 rc = HYPERVISOR_memory_op(XENMEM_add_to_physmap, &xatp);
995 if (rc != 0) {
996 printk(KERN_WARNING
997 "grant table add_to_physmap failed, err=%d\n", rc);
998 break;
999 }
1000 } while (i-- > start_idx);
1001
1002 return rc;
1003 }
1004
Annie Li85ff6ac2011-11-22 09:59:21 +08001005 /* No need for kzalloc as it is initialized in following hypercall
1006 * GNTTABOP_setup_table.
1007 */
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -07001008 frames = kmalloc(nr_gframes * sizeof(unsigned long), GFP_ATOMIC);
1009 if (!frames)
1010 return -ENOMEM;
1011
1012 setup.dom = DOMID_SELF;
1013 setup.nr_frames = nr_gframes;
Isaku Yamahata87e27cf2008-04-02 10:53:52 -07001014 set_xen_guest_handle(setup.frame_list, frames);
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -07001015
1016 rc = HYPERVISOR_grant_table_op(GNTTABOP_setup_table, &setup, 1);
1017 if (rc == -ENOSYS) {
1018 kfree(frames);
1019 return -ENOSYS;
1020 }
1021
1022 BUG_ON(rc || setup.status);
1023
Annie Li0f9f5a92011-11-22 09:58:06 +08001024 rc = gnttab_interface->map_frames(frames, nr_gframes);
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -07001025
1026 kfree(frames);
1027
Annie Li0f9f5a92011-11-22 09:58:06 +08001028 return rc;
1029}
1030
1031static struct gnttab_ops gnttab_v1_ops = {
1032 .map_frames = gnttab_map_frames_v1,
1033 .unmap_frames = gnttab_unmap_frames_v1,
1034 .update_entry = gnttab_update_entry_v1,
1035 .end_foreign_access_ref = gnttab_end_foreign_access_ref_v1,
1036 .end_foreign_transfer_ref = gnttab_end_foreign_transfer_ref_v1,
1037 .query_foreign_access = gnttab_query_foreign_access_v1,
1038};
1039
Annie Li85ff6ac2011-11-22 09:59:21 +08001040static struct gnttab_ops gnttab_v2_ops = {
1041 .map_frames = gnttab_map_frames_v2,
1042 .unmap_frames = gnttab_unmap_frames_v2,
1043 .update_entry = gnttab_update_entry_v2,
1044 .end_foreign_access_ref = gnttab_end_foreign_access_ref_v2,
1045 .end_foreign_transfer_ref = gnttab_end_foreign_transfer_ref_v2,
1046 .query_foreign_access = gnttab_query_foreign_access_v2,
Annie Li66667542011-12-12 18:14:42 +08001047 .update_subpage_entry = gnttab_update_subpage_entry_v2,
Annie Li9438ce92011-12-12 18:15:07 +08001048 .update_trans_entry = gnttab_update_trans_entry_v2,
Annie Li85ff6ac2011-11-22 09:59:21 +08001049};
1050
Annie Li0f9f5a92011-11-22 09:58:06 +08001051static void gnttab_request_version(void)
1052{
Annie Li85ff6ac2011-11-22 09:59:21 +08001053 int rc;
1054 struct gnttab_set_version gsv;
1055
Konrad Rzeszutek Wilk69e8f432012-01-25 00:13:20 -05001056 if (xen_hvm_domain())
1057 gsv.version = 1;
1058 else
1059 gsv.version = 2;
Annie Li85ff6ac2011-11-22 09:59:21 +08001060 rc = HYPERVISOR_grant_table_op(GNTTABOP_set_version, &gsv, 1);
Konrad Rzeszutek Wilk69e8f432012-01-25 00:13:20 -05001061 if (rc == 0 && gsv.version == 2) {
Annie Li85ff6ac2011-11-22 09:59:21 +08001062 grant_table_version = 2;
1063 gnttab_interface = &gnttab_v2_ops;
1064 } else if (grant_table_version == 2) {
1065 /*
1066 * If we've already used version 2 features,
1067 * but then suddenly discover that they're not
1068 * available (e.g. migrating to an older
1069 * version of Xen), almost unbounded badness
1070 * can happen.
1071 */
1072 panic("we need grant tables version 2, but only version 1 is available");
1073 } else {
1074 grant_table_version = 1;
1075 gnttab_interface = &gnttab_v1_ops;
1076 }
Annie Li0f9f5a92011-11-22 09:58:06 +08001077 printk(KERN_INFO "Grant tables using version %d layout.\n",
1078 grant_table_version);
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -07001079}
1080
Jeremy Fitzhardinge0e913982008-05-26 23:31:27 +01001081int gnttab_resume(void)
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -07001082{
Stefano Stabellini183d03c2010-05-17 17:08:21 +01001083 unsigned int max_nr_gframes;
1084
Annie Li0f9f5a92011-11-22 09:58:06 +08001085 gnttab_request_version();
Stefano Stabellini183d03c2010-05-17 17:08:21 +01001086 max_nr_gframes = gnttab_max_grant_frames();
1087 if (max_nr_gframes < nr_grant_frames)
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -07001088 return -ENOSYS;
Stefano Stabellini183d03c2010-05-17 17:08:21 +01001089
1090 if (xen_pv_domain())
1091 return gnttab_map(0, nr_grant_frames - 1);
1092
Annie Li0f9f5a92011-11-22 09:58:06 +08001093 if (gnttab_shared.addr == NULL) {
1094 gnttab_shared.addr = ioremap(xen_hvm_resume_frames,
1095 PAGE_SIZE * max_nr_gframes);
1096 if (gnttab_shared.addr == NULL) {
Stefano Stabellini183d03c2010-05-17 17:08:21 +01001097 printk(KERN_WARNING
1098 "Failed to ioremap gnttab share frames!");
1099 return -ENOMEM;
1100 }
1101 }
1102
1103 gnttab_map(0, nr_grant_frames - 1);
1104
1105 return 0;
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -07001106}
1107
Jeremy Fitzhardinge0e913982008-05-26 23:31:27 +01001108int gnttab_suspend(void)
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -07001109{
Annie Li0f9f5a92011-11-22 09:58:06 +08001110 gnttab_interface->unmap_frames();
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -07001111 return 0;
1112}
1113
1114static int gnttab_expand(unsigned int req_entries)
1115{
1116 int rc;
1117 unsigned int cur, extra;
1118
1119 cur = nr_grant_frames;
1120 extra = ((req_entries + (GREFS_PER_GRANT_FRAME-1)) /
1121 GREFS_PER_GRANT_FRAME);
Stefano Stabellini183d03c2010-05-17 17:08:21 +01001122 if (cur + extra > gnttab_max_grant_frames())
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -07001123 return -ENOSPC;
1124
1125 rc = gnttab_map(cur, cur + extra - 1);
1126 if (rc == 0)
1127 rc = grow_gnttab_list(extra);
1128
1129 return rc;
1130}
1131
Stefano Stabellini183d03c2010-05-17 17:08:21 +01001132int gnttab_init(void)
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -07001133{
1134 int i;
Michael Abd-El-Malekbbc60c12008-04-04 02:33:48 -07001135 unsigned int max_nr_glist_frames, nr_glist_frames;
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -07001136 unsigned int nr_init_grefs;
Julia Lawall6b5e7d92012-04-15 11:27:12 +02001137 int ret;
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -07001138
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -07001139 nr_grant_frames = 1;
1140 boot_max_nr_grant_frames = __max_nr_grant_frames();
1141
1142 /* Determine the maximum number of frames required for the
1143 * grant reference free list on the current hypervisor.
1144 */
1145 max_nr_glist_frames = (boot_max_nr_grant_frames *
Michael Abd-El-Malekbbc60c12008-04-04 02:33:48 -07001146 GREFS_PER_GRANT_FRAME / RPP);
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -07001147
1148 gnttab_list = kmalloc(max_nr_glist_frames * sizeof(grant_ref_t *),
1149 GFP_KERNEL);
1150 if (gnttab_list == NULL)
1151 return -ENOMEM;
1152
Michael Abd-El-Malekbbc60c12008-04-04 02:33:48 -07001153 nr_glist_frames = (nr_grant_frames * GREFS_PER_GRANT_FRAME + RPP - 1) / RPP;
1154 for (i = 0; i < nr_glist_frames; i++) {
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -07001155 gnttab_list[i] = (grant_ref_t *)__get_free_page(GFP_KERNEL);
Julia Lawall6b5e7d92012-04-15 11:27:12 +02001156 if (gnttab_list[i] == NULL) {
1157 ret = -ENOMEM;
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -07001158 goto ini_nomem;
Julia Lawall6b5e7d92012-04-15 11:27:12 +02001159 }
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -07001160 }
1161
Julia Lawall6b5e7d92012-04-15 11:27:12 +02001162 if (gnttab_resume() < 0) {
1163 ret = -ENODEV;
1164 goto ini_nomem;
1165 }
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -07001166
1167 nr_init_grefs = nr_grant_frames * GREFS_PER_GRANT_FRAME;
1168
1169 for (i = NR_RESERVED_ENTRIES; i < nr_init_grefs - 1; i++)
1170 gnttab_entry(i) = i + 1;
1171
1172 gnttab_entry(nr_init_grefs - 1) = GNTTAB_LIST_END;
1173 gnttab_free_count = nr_init_grefs - NR_RESERVED_ENTRIES;
1174 gnttab_free_head = NR_RESERVED_ENTRIES;
1175
1176 printk("Grant table initialized\n");
1177 return 0;
1178
1179 ini_nomem:
1180 for (i--; i >= 0; i--)
1181 free_page((unsigned long)gnttab_list[i]);
1182 kfree(gnttab_list);
Julia Lawall6b5e7d92012-04-15 11:27:12 +02001183 return ret;
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -07001184}
Stefano Stabellini183d03c2010-05-17 17:08:21 +01001185EXPORT_SYMBOL_GPL(gnttab_init);
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -07001186
Stefano Stabellini183d03c2010-05-17 17:08:21 +01001187static int __devinit __gnttab_init(void)
1188{
1189 /* Delay grant-table initialization in the PV on HVM case */
1190 if (xen_hvm_domain())
1191 return 0;
1192
1193 if (!xen_pv_domain())
1194 return -ENODEV;
1195
1196 return gnttab_init();
1197}
1198
1199core_initcall(__gnttab_init);