blob: b91f14e831648e68093a02f804f82e4113325360 [file] [log] [blame]
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -07001/******************************************************************************
2 * grant_table.c
3 *
4 * Granting foreign access to our memory reservation.
5 *
6 * Copyright (c) 2005-2006, Christopher Clark
7 * Copyright (c) 2004-2005, K A Fraser
8 *
9 * This program is free software; you can redistribute it and/or
10 * modify it under the terms of the GNU General Public License version 2
11 * as published by the Free Software Foundation; or, when distributed
12 * separately from the Linux kernel or incorporated into other
13 * software packages, subject to the following license:
14 *
15 * Permission is hereby granted, free of charge, to any person obtaining a copy
16 * of this source file (the "Software"), to deal in the Software without
17 * restriction, including without limitation the rights to use, copy, modify,
18 * merge, publish, distribute, sublicense, and/or sell copies of the Software,
19 * and to permit persons to whom the Software is furnished to do so, subject to
20 * the following conditions:
21 *
22 * The above copyright notice and this permission notice shall be included in
23 * all copies or substantial portions of the Software.
24 *
25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
26 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
27 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
28 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
29 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
30 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
31 * IN THE SOFTWARE.
32 */
33
34#include <linux/module.h>
35#include <linux/sched.h>
36#include <linux/mm.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090037#include <linux/slab.h>
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -070038#include <linux/vmalloc.h>
39#include <linux/uaccess.h>
Stefano Stabellini183d03c2010-05-17 17:08:21 +010040#include <linux/io.h>
Andres Lagar-Cavillac5718982012-09-14 14:26:59 +000041#include <linux/delay.h>
Stefano Stabellinif62805f2012-04-24 11:55:43 +010042#include <linux/hardirq.h>
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -070043
Jeremy Fitzhardinge1ccbf532009-10-06 15:11:14 -070044#include <xen/xen.h>
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -070045#include <xen/interface/xen.h>
46#include <xen/page.h>
47#include <xen/grant_table.h>
Stefano Stabellini183d03c2010-05-17 17:08:21 +010048#include <xen/interface/memory.h>
Annie Li85ff6ac2011-11-22 09:59:21 +080049#include <xen/hvc-console.h>
Jeremy Fitzhardingeecbf29c2008-12-16 12:37:07 -080050#include <asm/xen/hypercall.h>
Stefano Stabellini4d9310e2012-08-06 15:27:09 +010051#include <asm/xen/interface.h>
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -070052
53#include <asm/pgtable.h>
54#include <asm/sync_bitops.h>
55
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -070056/* External tools reserve first few grant table entries. */
57#define NR_RESERVED_ENTRIES 8
58#define GNTTAB_LIST_END 0xffffffff
Annie Li85ff6ac2011-11-22 09:59:21 +080059#define GREFS_PER_GRANT_FRAME \
60(grant_table_version == 1 ? \
61(PAGE_SIZE / sizeof(struct grant_entry_v1)) : \
62(PAGE_SIZE / sizeof(union grant_entry_v2)))
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -070063
64static grant_ref_t **gnttab_list;
65static unsigned int nr_grant_frames;
66static unsigned int boot_max_nr_grant_frames;
67static int gnttab_free_count;
68static grant_ref_t gnttab_free_head;
69static DEFINE_SPINLOCK(gnttab_list_lock);
Stefano Stabellini183d03c2010-05-17 17:08:21 +010070unsigned long xen_hvm_resume_frames;
71EXPORT_SYMBOL_GPL(xen_hvm_resume_frames);
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -070072
Annie Li0f9f5a92011-11-22 09:58:06 +080073static union {
74 struct grant_entry_v1 *v1;
Annie Li85ff6ac2011-11-22 09:59:21 +080075 union grant_entry_v2 *v2;
Annie Li0f9f5a92011-11-22 09:58:06 +080076 void *addr;
77} gnttab_shared;
78
79/*This is a structure of function pointers for grant table*/
80struct gnttab_ops {
81 /*
Annie Li9dbc71d2011-12-12 18:13:57 +080082 * Mapping a list of frames for storing grant entries. Frames parameter
83 * is used to store grant table address when grant table being setup,
84 * nr_gframes is the number of frames to map grant table. Returning
85 * GNTST_okay means success and negative value means failure.
Annie Li0f9f5a92011-11-22 09:58:06 +080086 */
Ian Campbellef32f892012-10-17 09:39:14 +010087 int (*map_frames)(xen_pfn_t *frames, unsigned int nr_gframes);
Annie Li0f9f5a92011-11-22 09:58:06 +080088 /*
89 * Release a list of frames which are mapped in map_frames for grant
90 * entry status.
91 */
92 void (*unmap_frames)(void);
93 /*
Annie Li9dbc71d2011-12-12 18:13:57 +080094 * Introducing a valid entry into the grant table, granting the frame of
95 * this grant entry to domain for accessing or transfering. Ref
96 * parameter is reference of this introduced grant entry, domid is id of
97 * granted domain, frame is the page frame to be granted, and flags is
98 * status of the grant entry to be updated.
Annie Li0f9f5a92011-11-22 09:58:06 +080099 */
Annie Li9dbc71d2011-12-12 18:13:57 +0800100 void (*update_entry)(grant_ref_t ref, domid_t domid,
101 unsigned long frame, unsigned flags);
Annie Li0f9f5a92011-11-22 09:58:06 +0800102 /*
Annie Li9dbc71d2011-12-12 18:13:57 +0800103 * Stop granting a grant entry to domain for accessing. Ref parameter is
104 * reference of a grant entry whose grant access will be stopped,
105 * readonly is not in use in this function. If the grant entry is
Annie Li0f9f5a92011-11-22 09:58:06 +0800106 * currently mapped for reading or writing, just return failure(==0)
107 * directly and don't tear down the grant access. Otherwise, stop grant
108 * access for this entry and return success(==1).
109 */
Annie Li9dbc71d2011-12-12 18:13:57 +0800110 int (*end_foreign_access_ref)(grant_ref_t ref, int readonly);
Annie Li0f9f5a92011-11-22 09:58:06 +0800111 /*
Annie Li9dbc71d2011-12-12 18:13:57 +0800112 * Stop granting a grant entry to domain for transfer. Ref parameter is
113 * reference of a grant entry whose grant transfer will be stopped. If
114 * tranfer has not started, just reclaim the grant entry and return
115 * failure(==0). Otherwise, wait for the transfer to complete and then
116 * return the frame.
Annie Li0f9f5a92011-11-22 09:58:06 +0800117 */
Annie Li9dbc71d2011-12-12 18:13:57 +0800118 unsigned long (*end_foreign_transfer_ref)(grant_ref_t ref);
Annie Li0f9f5a92011-11-22 09:58:06 +0800119 /*
Annie Li9dbc71d2011-12-12 18:13:57 +0800120 * Query the status of a grant entry. Ref parameter is reference of
Annie Li0f9f5a92011-11-22 09:58:06 +0800121 * queried grant entry, return value is the status of queried entry.
122 * Detailed status(writing/reading) can be gotten from the return value
123 * by bit operations.
124 */
Annie Li9dbc71d2011-12-12 18:13:57 +0800125 int (*query_foreign_access)(grant_ref_t ref);
Annie Li66667542011-12-12 18:14:42 +0800126 /*
127 * Grant a domain to access a range of bytes within the page referred by
128 * an available grant entry. Ref parameter is reference of a grant entry
129 * which will be sub-page accessed, domid is id of grantee domain, frame
130 * is frame address of subpage grant, flags is grant type and flag
131 * information, page_off is offset of the range of bytes, and length is
132 * length of bytes to be accessed.
133 */
134 void (*update_subpage_entry)(grant_ref_t ref, domid_t domid,
135 unsigned long frame, int flags,
136 unsigned page_off, unsigned length);
Annie Li9438ce92011-12-12 18:15:07 +0800137 /*
138 * Redirect an available grant entry on domain A to another grant
139 * reference of domain B, then allow domain C to use grant reference
140 * of domain B transitively. Ref parameter is an available grant entry
141 * reference on domain A, domid is id of domain C which accesses grant
142 * entry transitively, flags is grant type and flag information,
143 * trans_domid is id of domain B whose grant entry is finally accessed
144 * transitively, trans_gref is grant entry transitive reference of
145 * domain B.
146 */
147 void (*update_trans_entry)(grant_ref_t ref, domid_t domid, int flags,
148 domid_t trans_domid, grant_ref_t trans_gref);
Annie Li0f9f5a92011-11-22 09:58:06 +0800149};
150
151static struct gnttab_ops *gnttab_interface;
152
Annie Li85ff6ac2011-11-22 09:59:21 +0800153/*This reflects status of grant entries, so act as a global value*/
154static grant_status_t *grstatus;
155
Annie Li0f9f5a92011-11-22 09:58:06 +0800156static int grant_table_version;
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700157
158static struct gnttab_free_callback *gnttab_free_callback_list;
159
160static int gnttab_expand(unsigned int req_entries);
161
162#define RPP (PAGE_SIZE / sizeof(grant_ref_t))
Annie Li85ff6ac2011-11-22 09:59:21 +0800163#define SPP (PAGE_SIZE / sizeof(grant_status_t))
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700164
165static inline grant_ref_t *__gnttab_entry(grant_ref_t entry)
166{
167 return &gnttab_list[(entry) / RPP][(entry) % RPP];
168}
169/* This can be used as an l-value */
170#define gnttab_entry(entry) (*__gnttab_entry(entry))
171
172static int get_free_entries(unsigned count)
173{
174 unsigned long flags;
Konrad Rzeszutek Wilk272800d2011-07-22 14:00:06 -0400175 int ref, rc = 0;
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700176 grant_ref_t head;
177
178 spin_lock_irqsave(&gnttab_list_lock, flags);
179
180 if ((gnttab_free_count < count) &&
181 ((rc = gnttab_expand(count - gnttab_free_count)) < 0)) {
182 spin_unlock_irqrestore(&gnttab_list_lock, flags);
183 return rc;
184 }
185
186 ref = head = gnttab_free_head;
187 gnttab_free_count -= count;
188 while (count-- > 1)
189 head = gnttab_entry(head);
190 gnttab_free_head = gnttab_entry(head);
191 gnttab_entry(head) = GNTTAB_LIST_END;
192
193 spin_unlock_irqrestore(&gnttab_list_lock, flags);
194
195 return ref;
196}
197
198static void do_free_callbacks(void)
199{
200 struct gnttab_free_callback *callback, *next;
201
202 callback = gnttab_free_callback_list;
203 gnttab_free_callback_list = NULL;
204
205 while (callback != NULL) {
206 next = callback->next;
207 if (gnttab_free_count >= callback->count) {
208 callback->next = NULL;
209 callback->fn(callback->arg);
210 } else {
211 callback->next = gnttab_free_callback_list;
212 gnttab_free_callback_list = callback;
213 }
214 callback = next;
215 }
216}
217
218static inline void check_free_callbacks(void)
219{
220 if (unlikely(gnttab_free_callback_list))
221 do_free_callbacks();
222}
223
224static void put_free_entry(grant_ref_t ref)
225{
226 unsigned long flags;
227 spin_lock_irqsave(&gnttab_list_lock, flags);
228 gnttab_entry(ref) = gnttab_free_head;
229 gnttab_free_head = ref;
230 gnttab_free_count++;
231 check_free_callbacks();
232 spin_unlock_irqrestore(&gnttab_list_lock, flags);
233}
234
Annie Li0f9f5a92011-11-22 09:58:06 +0800235/*
Annie Li85ff6ac2011-11-22 09:59:21 +0800236 * Following applies to gnttab_update_entry_v1 and gnttab_update_entry_v2.
Annie Li0f9f5a92011-11-22 09:58:06 +0800237 * Introducing a valid entry into the grant table:
238 * 1. Write ent->domid.
239 * 2. Write ent->frame:
240 * GTF_permit_access: Frame to which access is permitted.
241 * GTF_accept_transfer: Pseudo-phys frame slot being filled by new
242 * frame, or zero if none.
243 * 3. Write memory barrier (WMB).
244 * 4. Write ent->flags, inc. valid type.
245 */
246static void gnttab_update_entry_v1(grant_ref_t ref, domid_t domid,
247 unsigned long frame, unsigned flags)
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700248{
Annie Li0f9f5a92011-11-22 09:58:06 +0800249 gnttab_shared.v1[ref].domid = domid;
250 gnttab_shared.v1[ref].frame = frame;
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700251 wmb();
Annie Li0f9f5a92011-11-22 09:58:06 +0800252 gnttab_shared.v1[ref].flags = flags;
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700253}
254
Annie Li85ff6ac2011-11-22 09:59:21 +0800255static void gnttab_update_entry_v2(grant_ref_t ref, domid_t domid,
256 unsigned long frame, unsigned flags)
257{
258 gnttab_shared.v2[ref].hdr.domid = domid;
259 gnttab_shared.v2[ref].full_page.frame = frame;
260 wmb();
261 gnttab_shared.v2[ref].hdr.flags = GTF_permit_access | flags;
262}
263
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700264/*
265 * Public grant-issuing interface functions
266 */
267void gnttab_grant_foreign_access_ref(grant_ref_t ref, domid_t domid,
268 unsigned long frame, int readonly)
269{
Annie Li0f9f5a92011-11-22 09:58:06 +0800270 gnttab_interface->update_entry(ref, domid, frame,
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700271 GTF_permit_access | (readonly ? GTF_readonly : 0));
272}
273EXPORT_SYMBOL_GPL(gnttab_grant_foreign_access_ref);
274
275int gnttab_grant_foreign_access(domid_t domid, unsigned long frame,
276 int readonly)
277{
278 int ref;
279
280 ref = get_free_entries(1);
281 if (unlikely(ref < 0))
282 return -ENOSPC;
283
284 gnttab_grant_foreign_access_ref(ref, domid, frame, readonly);
285
286 return ref;
287}
288EXPORT_SYMBOL_GPL(gnttab_grant_foreign_access);
289
Konrad Rzeszutek Wilkb8b0f552012-08-21 14:49:34 -0400290static void gnttab_update_subpage_entry_v2(grant_ref_t ref, domid_t domid,
291 unsigned long frame, int flags,
292 unsigned page_off, unsigned length)
Annie Li66667542011-12-12 18:14:42 +0800293{
294 gnttab_shared.v2[ref].sub_page.frame = frame;
295 gnttab_shared.v2[ref].sub_page.page_off = page_off;
296 gnttab_shared.v2[ref].sub_page.length = length;
297 gnttab_shared.v2[ref].hdr.domid = domid;
298 wmb();
299 gnttab_shared.v2[ref].hdr.flags =
300 GTF_permit_access | GTF_sub_page | flags;
301}
302
303int gnttab_grant_foreign_access_subpage_ref(grant_ref_t ref, domid_t domid,
304 unsigned long frame, int flags,
305 unsigned page_off,
306 unsigned length)
307{
308 if (flags & (GTF_accept_transfer | GTF_reading |
309 GTF_writing | GTF_transitive))
310 return -EPERM;
311
312 if (gnttab_interface->update_subpage_entry == NULL)
313 return -ENOSYS;
314
315 gnttab_interface->update_subpage_entry(ref, domid, frame, flags,
316 page_off, length);
317
318 return 0;
319}
320EXPORT_SYMBOL_GPL(gnttab_grant_foreign_access_subpage_ref);
321
322int gnttab_grant_foreign_access_subpage(domid_t domid, unsigned long frame,
323 int flags, unsigned page_off,
324 unsigned length)
325{
326 int ref, rc;
327
328 ref = get_free_entries(1);
329 if (unlikely(ref < 0))
330 return -ENOSPC;
331
332 rc = gnttab_grant_foreign_access_subpage_ref(ref, domid, frame, flags,
333 page_off, length);
334 if (rc < 0) {
335 put_free_entry(ref);
336 return rc;
337 }
338
339 return ref;
340}
341EXPORT_SYMBOL_GPL(gnttab_grant_foreign_access_subpage);
342
343bool gnttab_subpage_grants_available(void)
344{
345 return gnttab_interface->update_subpage_entry != NULL;
346}
347EXPORT_SYMBOL_GPL(gnttab_subpage_grants_available);
348
Konrad Rzeszutek Wilkb8b0f552012-08-21 14:49:34 -0400349static void gnttab_update_trans_entry_v2(grant_ref_t ref, domid_t domid,
350 int flags, domid_t trans_domid,
351 grant_ref_t trans_gref)
Annie Li9438ce92011-12-12 18:15:07 +0800352{
353 gnttab_shared.v2[ref].transitive.trans_domid = trans_domid;
354 gnttab_shared.v2[ref].transitive.gref = trans_gref;
355 gnttab_shared.v2[ref].hdr.domid = domid;
356 wmb();
357 gnttab_shared.v2[ref].hdr.flags =
358 GTF_permit_access | GTF_transitive | flags;
359}
360
361int gnttab_grant_foreign_access_trans_ref(grant_ref_t ref, domid_t domid,
362 int flags, domid_t trans_domid,
363 grant_ref_t trans_gref)
364{
365 if (flags & (GTF_accept_transfer | GTF_reading |
366 GTF_writing | GTF_sub_page))
367 return -EPERM;
368
369 if (gnttab_interface->update_trans_entry == NULL)
370 return -ENOSYS;
371
372 gnttab_interface->update_trans_entry(ref, domid, flags, trans_domid,
373 trans_gref);
374
375 return 0;
376}
377EXPORT_SYMBOL_GPL(gnttab_grant_foreign_access_trans_ref);
378
379int gnttab_grant_foreign_access_trans(domid_t domid, int flags,
380 domid_t trans_domid,
381 grant_ref_t trans_gref)
382{
383 int ref, rc;
384
385 ref = get_free_entries(1);
386 if (unlikely(ref < 0))
387 return -ENOSPC;
388
389 rc = gnttab_grant_foreign_access_trans_ref(ref, domid, flags,
390 trans_domid, trans_gref);
391 if (rc < 0) {
392 put_free_entry(ref);
393 return rc;
394 }
395
396 return ref;
397}
398EXPORT_SYMBOL_GPL(gnttab_grant_foreign_access_trans);
399
400bool gnttab_trans_grants_available(void)
401{
402 return gnttab_interface->update_trans_entry != NULL;
403}
404EXPORT_SYMBOL_GPL(gnttab_trans_grants_available);
405
Annie Li0f9f5a92011-11-22 09:58:06 +0800406static int gnttab_query_foreign_access_v1(grant_ref_t ref)
407{
408 return gnttab_shared.v1[ref].flags & (GTF_reading|GTF_writing);
409}
410
Annie Li85ff6ac2011-11-22 09:59:21 +0800411static int gnttab_query_foreign_access_v2(grant_ref_t ref)
412{
413 return grstatus[ref] & (GTF_reading|GTF_writing);
414}
415
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700416int gnttab_query_foreign_access(grant_ref_t ref)
417{
Annie Li0f9f5a92011-11-22 09:58:06 +0800418 return gnttab_interface->query_foreign_access(ref);
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700419}
420EXPORT_SYMBOL_GPL(gnttab_query_foreign_access);
421
Annie Li0f9f5a92011-11-22 09:58:06 +0800422static int gnttab_end_foreign_access_ref_v1(grant_ref_t ref, int readonly)
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700423{
424 u16 flags, nflags;
Annie Lib1e495b2011-11-22 09:58:47 +0800425 u16 *pflags;
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700426
Annie Lib1e495b2011-11-22 09:58:47 +0800427 pflags = &gnttab_shared.v1[ref].flags;
428 nflags = *pflags;
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700429 do {
430 flags = nflags;
Jan Beulich569ca5b2012-04-05 16:10:07 +0100431 if (flags & (GTF_reading|GTF_writing))
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700432 return 0;
Annie Lib1e495b2011-11-22 09:58:47 +0800433 } while ((nflags = sync_cmpxchg(pflags, flags, 0)) != flags);
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700434
435 return 1;
436}
Annie Li0f9f5a92011-11-22 09:58:06 +0800437
Annie Li85ff6ac2011-11-22 09:59:21 +0800438static int gnttab_end_foreign_access_ref_v2(grant_ref_t ref, int readonly)
439{
440 gnttab_shared.v2[ref].hdr.flags = 0;
441 mb();
442 if (grstatus[ref] & (GTF_reading|GTF_writing)) {
443 return 0;
444 } else {
445 /* The read of grstatus needs to have acquire
446 semantics. On x86, reads already have
447 that, and we just need to protect against
448 compiler reorderings. On other
449 architectures we may need a full
450 barrier. */
451#ifdef CONFIG_X86
452 barrier();
453#else
454 mb();
455#endif
456 }
457
458 return 1;
459}
460
Jan Beulich569ca5b2012-04-05 16:10:07 +0100461static inline int _gnttab_end_foreign_access_ref(grant_ref_t ref, int readonly)
Annie Li0f9f5a92011-11-22 09:58:06 +0800462{
463 return gnttab_interface->end_foreign_access_ref(ref, readonly);
464}
Jan Beulich569ca5b2012-04-05 16:10:07 +0100465
466int gnttab_end_foreign_access_ref(grant_ref_t ref, int readonly)
467{
468 if (_gnttab_end_foreign_access_ref(ref, readonly))
469 return 1;
470 pr_warn("WARNING: g.e. %#x still in use!\n", ref);
471 return 0;
472}
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700473EXPORT_SYMBOL_GPL(gnttab_end_foreign_access_ref);
474
Jan Beulich569ca5b2012-04-05 16:10:07 +0100475struct deferred_entry {
476 struct list_head list;
477 grant_ref_t ref;
478 bool ro;
479 uint16_t warn_delay;
480 struct page *page;
481};
482static LIST_HEAD(deferred_list);
483static void gnttab_handle_deferred(unsigned long);
484static DEFINE_TIMER(deferred_timer, gnttab_handle_deferred, 0, 0);
485
486static void gnttab_handle_deferred(unsigned long unused)
487{
488 unsigned int nr = 10;
489 struct deferred_entry *first = NULL;
490 unsigned long flags;
491
492 spin_lock_irqsave(&gnttab_list_lock, flags);
493 while (nr--) {
494 struct deferred_entry *entry
495 = list_first_entry(&deferred_list,
496 struct deferred_entry, list);
497
498 if (entry == first)
499 break;
500 list_del(&entry->list);
501 spin_unlock_irqrestore(&gnttab_list_lock, flags);
502 if (_gnttab_end_foreign_access_ref(entry->ref, entry->ro)) {
503 put_free_entry(entry->ref);
504 if (entry->page) {
505 pr_debug("freeing g.e. %#x (pfn %#lx)\n",
506 entry->ref, page_to_pfn(entry->page));
507 __free_page(entry->page);
508 } else
509 pr_info("freeing g.e. %#x\n", entry->ref);
510 kfree(entry);
511 entry = NULL;
512 } else {
513 if (!--entry->warn_delay)
514 pr_info("g.e. %#x still pending\n",
515 entry->ref);
516 if (!first)
517 first = entry;
518 }
519 spin_lock_irqsave(&gnttab_list_lock, flags);
520 if (entry)
521 list_add_tail(&entry->list, &deferred_list);
522 else if (list_empty(&deferred_list))
523 break;
524 }
525 if (!list_empty(&deferred_list) && !timer_pending(&deferred_timer)) {
526 deferred_timer.expires = jiffies + HZ;
527 add_timer(&deferred_timer);
528 }
529 spin_unlock_irqrestore(&gnttab_list_lock, flags);
530}
531
532static void gnttab_add_deferred(grant_ref_t ref, bool readonly,
533 struct page *page)
534{
535 struct deferred_entry *entry = kmalloc(sizeof(*entry), GFP_ATOMIC);
536 const char *what = KERN_WARNING "leaking";
537
538 if (entry) {
539 unsigned long flags;
540
541 entry->ref = ref;
542 entry->ro = readonly;
543 entry->page = page;
544 entry->warn_delay = 60;
545 spin_lock_irqsave(&gnttab_list_lock, flags);
546 list_add_tail(&entry->list, &deferred_list);
547 if (!timer_pending(&deferred_timer)) {
548 deferred_timer.expires = jiffies + HZ;
549 add_timer(&deferred_timer);
550 }
551 spin_unlock_irqrestore(&gnttab_list_lock, flags);
552 what = KERN_DEBUG "deferring";
553 }
554 printk("%s g.e. %#x (pfn %#lx)\n",
555 what, ref, page ? page_to_pfn(page) : -1);
556}
557
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700558void gnttab_end_foreign_access(grant_ref_t ref, int readonly,
559 unsigned long page)
560{
561 if (gnttab_end_foreign_access_ref(ref, readonly)) {
562 put_free_entry(ref);
563 if (page != 0)
564 free_page(page);
Jan Beulich569ca5b2012-04-05 16:10:07 +0100565 } else
566 gnttab_add_deferred(ref, readonly,
567 page ? virt_to_page(page) : NULL);
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700568}
569EXPORT_SYMBOL_GPL(gnttab_end_foreign_access);
570
571int gnttab_grant_foreign_transfer(domid_t domid, unsigned long pfn)
572{
573 int ref;
574
575 ref = get_free_entries(1);
576 if (unlikely(ref < 0))
577 return -ENOSPC;
578 gnttab_grant_foreign_transfer_ref(ref, domid, pfn);
579
580 return ref;
581}
582EXPORT_SYMBOL_GPL(gnttab_grant_foreign_transfer);
583
584void gnttab_grant_foreign_transfer_ref(grant_ref_t ref, domid_t domid,
585 unsigned long pfn)
586{
Annie Li0f9f5a92011-11-22 09:58:06 +0800587 gnttab_interface->update_entry(ref, domid, pfn, GTF_accept_transfer);
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700588}
589EXPORT_SYMBOL_GPL(gnttab_grant_foreign_transfer_ref);
590
Annie Li0f9f5a92011-11-22 09:58:06 +0800591static unsigned long gnttab_end_foreign_transfer_ref_v1(grant_ref_t ref)
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700592{
593 unsigned long frame;
594 u16 flags;
Annie Lib1e495b2011-11-22 09:58:47 +0800595 u16 *pflags;
596
597 pflags = &gnttab_shared.v1[ref].flags;
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700598
599 /*
600 * If a transfer is not even yet started, try to reclaim the grant
601 * reference and return failure (== 0).
602 */
Annie Lib1e495b2011-11-22 09:58:47 +0800603 while (!((flags = *pflags) & GTF_transfer_committed)) {
604 if (sync_cmpxchg(pflags, flags, 0) == flags)
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700605 return 0;
606 cpu_relax();
607 }
608
609 /* If a transfer is in progress then wait until it is completed. */
610 while (!(flags & GTF_transfer_completed)) {
Annie Lib1e495b2011-11-22 09:58:47 +0800611 flags = *pflags;
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700612 cpu_relax();
613 }
614
615 rmb(); /* Read the frame number /after/ reading completion status. */
Annie Li0f9f5a92011-11-22 09:58:06 +0800616 frame = gnttab_shared.v1[ref].frame;
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700617 BUG_ON(frame == 0);
618
619 return frame;
620}
Annie Li0f9f5a92011-11-22 09:58:06 +0800621
Annie Li85ff6ac2011-11-22 09:59:21 +0800622static unsigned long gnttab_end_foreign_transfer_ref_v2(grant_ref_t ref)
623{
624 unsigned long frame;
625 u16 flags;
626 u16 *pflags;
627
628 pflags = &gnttab_shared.v2[ref].hdr.flags;
629
630 /*
631 * If a transfer is not even yet started, try to reclaim the grant
632 * reference and return failure (== 0).
633 */
634 while (!((flags = *pflags) & GTF_transfer_committed)) {
635 if (sync_cmpxchg(pflags, flags, 0) == flags)
636 return 0;
637 cpu_relax();
638 }
639
640 /* If a transfer is in progress then wait until it is completed. */
641 while (!(flags & GTF_transfer_completed)) {
642 flags = *pflags;
643 cpu_relax();
644 }
645
646 rmb(); /* Read the frame number /after/ reading completion status. */
647 frame = gnttab_shared.v2[ref].full_page.frame;
648 BUG_ON(frame == 0);
649
650 return frame;
651}
652
Annie Li0f9f5a92011-11-22 09:58:06 +0800653unsigned long gnttab_end_foreign_transfer_ref(grant_ref_t ref)
654{
655 return gnttab_interface->end_foreign_transfer_ref(ref);
656}
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700657EXPORT_SYMBOL_GPL(gnttab_end_foreign_transfer_ref);
658
659unsigned long gnttab_end_foreign_transfer(grant_ref_t ref)
660{
661 unsigned long frame = gnttab_end_foreign_transfer_ref(ref);
662 put_free_entry(ref);
663 return frame;
664}
665EXPORT_SYMBOL_GPL(gnttab_end_foreign_transfer);
666
667void gnttab_free_grant_reference(grant_ref_t ref)
668{
669 put_free_entry(ref);
670}
671EXPORT_SYMBOL_GPL(gnttab_free_grant_reference);
672
673void gnttab_free_grant_references(grant_ref_t head)
674{
675 grant_ref_t ref;
676 unsigned long flags;
677 int count = 1;
678 if (head == GNTTAB_LIST_END)
679 return;
680 spin_lock_irqsave(&gnttab_list_lock, flags);
681 ref = head;
682 while (gnttab_entry(ref) != GNTTAB_LIST_END) {
683 ref = gnttab_entry(ref);
684 count++;
685 }
686 gnttab_entry(ref) = gnttab_free_head;
687 gnttab_free_head = head;
688 gnttab_free_count += count;
689 check_free_callbacks();
690 spin_unlock_irqrestore(&gnttab_list_lock, flags);
691}
692EXPORT_SYMBOL_GPL(gnttab_free_grant_references);
693
694int gnttab_alloc_grant_references(u16 count, grant_ref_t *head)
695{
696 int h = get_free_entries(count);
697
698 if (h < 0)
699 return -ENOSPC;
700
701 *head = h;
702
703 return 0;
704}
705EXPORT_SYMBOL_GPL(gnttab_alloc_grant_references);
706
707int gnttab_empty_grant_references(const grant_ref_t *private_head)
708{
709 return (*private_head == GNTTAB_LIST_END);
710}
711EXPORT_SYMBOL_GPL(gnttab_empty_grant_references);
712
713int gnttab_claim_grant_reference(grant_ref_t *private_head)
714{
715 grant_ref_t g = *private_head;
716 if (unlikely(g == GNTTAB_LIST_END))
717 return -ENOSPC;
718 *private_head = gnttab_entry(g);
719 return g;
720}
721EXPORT_SYMBOL_GPL(gnttab_claim_grant_reference);
722
723void gnttab_release_grant_reference(grant_ref_t *private_head,
724 grant_ref_t release)
725{
726 gnttab_entry(release) = *private_head;
727 *private_head = release;
728}
729EXPORT_SYMBOL_GPL(gnttab_release_grant_reference);
730
731void gnttab_request_free_callback(struct gnttab_free_callback *callback,
732 void (*fn)(void *), void *arg, u16 count)
733{
734 unsigned long flags;
735 spin_lock_irqsave(&gnttab_list_lock, flags);
736 if (callback->next)
737 goto out;
738 callback->fn = fn;
739 callback->arg = arg;
740 callback->count = count;
741 callback->next = gnttab_free_callback_list;
742 gnttab_free_callback_list = callback;
743 check_free_callbacks();
744out:
745 spin_unlock_irqrestore(&gnttab_list_lock, flags);
746}
747EXPORT_SYMBOL_GPL(gnttab_request_free_callback);
748
749void gnttab_cancel_free_callback(struct gnttab_free_callback *callback)
750{
751 struct gnttab_free_callback **pcb;
752 unsigned long flags;
753
754 spin_lock_irqsave(&gnttab_list_lock, flags);
755 for (pcb = &gnttab_free_callback_list; *pcb; pcb = &(*pcb)->next) {
756 if (*pcb == callback) {
757 *pcb = callback->next;
758 break;
759 }
760 }
761 spin_unlock_irqrestore(&gnttab_list_lock, flags);
762}
763EXPORT_SYMBOL_GPL(gnttab_cancel_free_callback);
764
765static int grow_gnttab_list(unsigned int more_frames)
766{
767 unsigned int new_nr_grant_frames, extra_entries, i;
Michael Abd-El-Malekbbc60c12008-04-04 02:33:48 -0700768 unsigned int nr_glist_frames, new_nr_glist_frames;
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700769
770 new_nr_grant_frames = nr_grant_frames + more_frames;
771 extra_entries = more_frames * GREFS_PER_GRANT_FRAME;
772
Michael Abd-El-Malekbbc60c12008-04-04 02:33:48 -0700773 nr_glist_frames = (nr_grant_frames * GREFS_PER_GRANT_FRAME + RPP - 1) / RPP;
774 new_nr_glist_frames =
775 (new_nr_grant_frames * GREFS_PER_GRANT_FRAME + RPP - 1) / RPP;
776 for (i = nr_glist_frames; i < new_nr_glist_frames; i++) {
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700777 gnttab_list[i] = (grant_ref_t *)__get_free_page(GFP_ATOMIC);
778 if (!gnttab_list[i])
779 goto grow_nomem;
780 }
781
782
783 for (i = GREFS_PER_GRANT_FRAME * nr_grant_frames;
784 i < GREFS_PER_GRANT_FRAME * new_nr_grant_frames - 1; i++)
785 gnttab_entry(i) = i + 1;
786
787 gnttab_entry(i) = gnttab_free_head;
788 gnttab_free_head = GREFS_PER_GRANT_FRAME * nr_grant_frames;
789 gnttab_free_count += extra_entries;
790
791 nr_grant_frames = new_nr_grant_frames;
792
793 check_free_callbacks();
794
795 return 0;
796
797grow_nomem:
Michael Abd-El-Malekbbc60c12008-04-04 02:33:48 -0700798 for ( ; i >= nr_glist_frames; i--)
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700799 free_page((unsigned long) gnttab_list[i]);
800 return -ENOMEM;
801}
802
803static unsigned int __max_nr_grant_frames(void)
804{
805 struct gnttab_query_size query;
806 int rc;
807
808 query.dom = DOMID_SELF;
809
810 rc = HYPERVISOR_grant_table_op(GNTTABOP_query_size, &query, 1);
811 if ((rc < 0) || (query.status != GNTST_okay))
812 return 4; /* Legacy max supported number of frames */
813
814 return query.max_nr_frames;
815}
816
Stefano Stabellini183d03c2010-05-17 17:08:21 +0100817unsigned int gnttab_max_grant_frames(void)
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700818{
819 unsigned int xen_max = __max_nr_grant_frames();
820
821 if (xen_max > boot_max_nr_grant_frames)
822 return boot_max_nr_grant_frames;
823 return xen_max;
824}
Stefano Stabellini183d03c2010-05-17 17:08:21 +0100825EXPORT_SYMBOL_GPL(gnttab_max_grant_frames);
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700826
Andres Lagar-Cavillac5718982012-09-14 14:26:59 +0000827/* Handling of paged out grant targets (GNTST_eagain) */
828#define MAX_DELAY 256
829static inline void
830gnttab_retry_eagain_gop(unsigned int cmd, void *gop, int16_t *status,
831 const char *func)
832{
833 unsigned delay = 1;
834
835 do {
836 BUG_ON(HYPERVISOR_grant_table_op(cmd, gop, 1));
837 if (*status == GNTST_eagain)
838 msleep(delay++);
839 } while ((*status == GNTST_eagain) && (delay < MAX_DELAY));
840
841 if (delay >= MAX_DELAY) {
842 printk(KERN_ERR "%s: %s eagain grant\n", func, current->comm);
843 *status = GNTST_bad_page;
844 }
845}
846
847void gnttab_batch_map(struct gnttab_map_grant_ref *batch, unsigned count)
848{
849 struct gnttab_map_grant_ref *op;
850
851 if (HYPERVISOR_grant_table_op(GNTTABOP_map_grant_ref, batch, count))
852 BUG();
853 for (op = batch; op < batch + count; op++)
854 if (op->status == GNTST_eagain)
855 gnttab_retry_eagain_gop(GNTTABOP_map_grant_ref, op,
856 &op->status, __func__);
857}
858EXPORT_SYMBOL_GPL(gnttab_batch_map);
859
860void gnttab_batch_copy(struct gnttab_copy *batch, unsigned count)
861{
862 struct gnttab_copy *op;
863
864 if (HYPERVISOR_grant_table_op(GNTTABOP_copy, batch, count))
865 BUG();
866 for (op = batch; op < batch + count; op++)
867 if (op->status == GNTST_eagain)
868 gnttab_retry_eagain_gop(GNTTABOP_copy, op,
869 &op->status, __func__);
870}
871EXPORT_SYMBOL_GPL(gnttab_batch_copy);
872
Stefano Stabellini289b7772010-12-10 14:54:44 +0000873int gnttab_map_refs(struct gnttab_map_grant_ref *map_ops,
Annie Lic1237992011-11-22 09:59:56 +0800874 struct gnttab_map_grant_ref *kmap_ops,
875 struct page **pages, unsigned int count)
Stefano Stabellini289b7772010-12-10 14:54:44 +0000876{
877 int i, ret;
Stefano Stabellinif62805f2012-04-24 11:55:43 +0100878 bool lazy = false;
Stefano Stabellini289b7772010-12-10 14:54:44 +0000879 pte_t *pte;
880 unsigned long mfn;
881
882 ret = HYPERVISOR_grant_table_op(GNTTABOP_map_grant_ref, map_ops, count);
Jeremy Fitzhardinge87f1d402010-12-13 14:42:30 +0000883 if (ret)
884 return ret;
Stefano Stabellini289b7772010-12-10 14:54:44 +0000885
Andres Lagar-Cavillac5718982012-09-14 14:26:59 +0000886 /* Retry eagain maps */
887 for (i = 0; i < count; i++)
888 if (map_ops[i].status == GNTST_eagain)
889 gnttab_retry_eagain_gop(GNTTABOP_map_grant_ref, map_ops + i,
890 &map_ops[i].status, __func__);
891
Daniel De Graafaab8f112011-02-03 12:19:02 -0500892 if (xen_feature(XENFEAT_auto_translated_physmap))
893 return ret;
894
Stefano Stabellinif62805f2012-04-24 11:55:43 +0100895 if (!in_interrupt() && paravirt_get_lazy_mode() == PARAVIRT_LAZY_NONE) {
896 arch_enter_lazy_mmu_mode();
897 lazy = true;
898 }
899
Stefano Stabellini289b7772010-12-10 14:54:44 +0000900 for (i = 0; i < count; i++) {
Ian Campbelldc4972a2011-03-04 17:38:21 +0000901 /* Do not add to override if the map failed. */
902 if (map_ops[i].status)
903 continue;
904
Konrad Rzeszutek Wilkcf8d9162011-02-28 17:58:48 -0500905 if (map_ops[i].flags & GNTMAP_contains_pte) {
906 pte = (pte_t *) (mfn_to_virt(PFN_DOWN(map_ops[i].host_addr)) +
Stefano Stabellini289b7772010-12-10 14:54:44 +0000907 (map_ops[i].host_addr & ~PAGE_MASK));
Konrad Rzeszutek Wilkcf8d9162011-02-28 17:58:48 -0500908 mfn = pte_mfn(*pte);
909 } else {
Daniel De Graaf7d17e842011-12-14 15:12:11 -0500910 mfn = PFN_DOWN(map_ops[i].dev_bus_addr);
Konrad Rzeszutek Wilkcf8d9162011-02-28 17:58:48 -0500911 }
Daniel De Graaf7d17e842011-12-14 15:12:11 -0500912 ret = m2p_add_override(mfn, pages[i], kmap_ops ?
913 &kmap_ops[i] : NULL);
Jeremy Fitzhardinge87f1d402010-12-13 14:42:30 +0000914 if (ret)
915 return ret;
Stefano Stabellini289b7772010-12-10 14:54:44 +0000916 }
917
Stefano Stabellinif62805f2012-04-24 11:55:43 +0100918 if (lazy)
919 arch_leave_lazy_mmu_mode();
920
Stefano Stabellini289b7772010-12-10 14:54:44 +0000921 return ret;
922}
923EXPORT_SYMBOL_GPL(gnttab_map_refs);
924
925int gnttab_unmap_refs(struct gnttab_unmap_grant_ref *unmap_ops,
Stefano Stabellini2fc136e2012-09-12 12:44:30 +0100926 struct gnttab_map_grant_ref *kmap_ops,
927 struct page **pages, unsigned int count)
Stefano Stabellini289b7772010-12-10 14:54:44 +0000928{
929 int i, ret;
Stefano Stabellinif62805f2012-04-24 11:55:43 +0100930 bool lazy = false;
Stefano Stabellini289b7772010-12-10 14:54:44 +0000931
932 ret = HYPERVISOR_grant_table_op(GNTTABOP_unmap_grant_ref, unmap_ops, count);
Jeremy Fitzhardinge87f1d402010-12-13 14:42:30 +0000933 if (ret)
934 return ret;
935
Daniel De Graafaab8f112011-02-03 12:19:02 -0500936 if (xen_feature(XENFEAT_auto_translated_physmap))
937 return ret;
938
Stefano Stabellinif62805f2012-04-24 11:55:43 +0100939 if (!in_interrupt() && paravirt_get_lazy_mode() == PARAVIRT_LAZY_NONE) {
940 arch_enter_lazy_mmu_mode();
941 lazy = true;
942 }
943
Jeremy Fitzhardinge87f1d402010-12-13 14:42:30 +0000944 for (i = 0; i < count; i++) {
Stefano Stabellini2fc136e2012-09-12 12:44:30 +0100945 ret = m2p_remove_override(pages[i], kmap_ops ?
946 &kmap_ops[i] : NULL);
Jeremy Fitzhardinge87f1d402010-12-13 14:42:30 +0000947 if (ret)
948 return ret;
949 }
Stefano Stabellini289b7772010-12-10 14:54:44 +0000950
Stefano Stabellinif62805f2012-04-24 11:55:43 +0100951 if (lazy)
952 arch_leave_lazy_mmu_mode();
953
Stefano Stabellini289b7772010-12-10 14:54:44 +0000954 return ret;
955}
956EXPORT_SYMBOL_GPL(gnttab_unmap_refs);
957
Annie Li85ff6ac2011-11-22 09:59:21 +0800958static unsigned nr_status_frames(unsigned nr_grant_frames)
959{
960 return (nr_grant_frames * GREFS_PER_GRANT_FRAME + SPP - 1) / SPP;
961}
962
Ian Campbellef32f892012-10-17 09:39:14 +0100963static int gnttab_map_frames_v1(xen_pfn_t *frames, unsigned int nr_gframes)
Annie Li0f9f5a92011-11-22 09:58:06 +0800964{
965 int rc;
966
967 rc = arch_gnttab_map_shared(frames, nr_gframes,
968 gnttab_max_grant_frames(),
969 &gnttab_shared.addr);
970 BUG_ON(rc);
971
972 return 0;
973}
974
975static void gnttab_unmap_frames_v1(void)
976{
Annie Li85ff6ac2011-11-22 09:59:21 +0800977 arch_gnttab_unmap(gnttab_shared.addr, nr_grant_frames);
978}
979
Ian Campbellef32f892012-10-17 09:39:14 +0100980static int gnttab_map_frames_v2(xen_pfn_t *frames, unsigned int nr_gframes)
Annie Li85ff6ac2011-11-22 09:59:21 +0800981{
982 uint64_t *sframes;
983 unsigned int nr_sframes;
984 struct gnttab_get_status_frames getframes;
985 int rc;
986
987 nr_sframes = nr_status_frames(nr_gframes);
988
989 /* No need for kzalloc as it is initialized in following hypercall
990 * GNTTABOP_get_status_frames.
991 */
992 sframes = kmalloc(nr_sframes * sizeof(uint64_t), GFP_ATOMIC);
993 if (!sframes)
994 return -ENOMEM;
995
996 getframes.dom = DOMID_SELF;
997 getframes.nr_frames = nr_sframes;
998 set_xen_guest_handle(getframes.frame_list, sframes);
999
1000 rc = HYPERVISOR_grant_table_op(GNTTABOP_get_status_frames,
1001 &getframes, 1);
1002 if (rc == -ENOSYS) {
1003 kfree(sframes);
1004 return -ENOSYS;
1005 }
1006
1007 BUG_ON(rc || getframes.status);
1008
1009 rc = arch_gnttab_map_status(sframes, nr_sframes,
1010 nr_status_frames(gnttab_max_grant_frames()),
1011 &grstatus);
1012 BUG_ON(rc);
1013 kfree(sframes);
1014
1015 rc = arch_gnttab_map_shared(frames, nr_gframes,
1016 gnttab_max_grant_frames(),
1017 &gnttab_shared.addr);
1018 BUG_ON(rc);
1019
1020 return 0;
1021}
1022
1023static void gnttab_unmap_frames_v2(void)
1024{
1025 arch_gnttab_unmap(gnttab_shared.addr, nr_grant_frames);
1026 arch_gnttab_unmap(grstatus, nr_status_frames(nr_grant_frames));
Annie Li0f9f5a92011-11-22 09:58:06 +08001027}
1028
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -07001029static int gnttab_map(unsigned int start_idx, unsigned int end_idx)
1030{
1031 struct gnttab_setup_table setup;
Ian Campbellef32f892012-10-17 09:39:14 +01001032 xen_pfn_t *frames;
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -07001033 unsigned int nr_gframes = end_idx + 1;
1034 int rc;
1035
Stefano Stabellini183d03c2010-05-17 17:08:21 +01001036 if (xen_hvm_domain()) {
1037 struct xen_add_to_physmap xatp;
1038 unsigned int i = end_idx;
1039 rc = 0;
1040 /*
1041 * Loop backwards, so that the first hypercall has the largest
1042 * index, ensuring that the table will grow only once.
1043 */
1044 do {
1045 xatp.domid = DOMID_SELF;
1046 xatp.idx = i;
1047 xatp.space = XENMAPSPACE_grant_table;
1048 xatp.gpfn = (xen_hvm_resume_frames >> PAGE_SHIFT) + i;
1049 rc = HYPERVISOR_memory_op(XENMEM_add_to_physmap, &xatp);
1050 if (rc != 0) {
1051 printk(KERN_WARNING
1052 "grant table add_to_physmap failed, err=%d\n", rc);
1053 break;
1054 }
1055 } while (i-- > start_idx);
1056
1057 return rc;
1058 }
1059
Annie Li85ff6ac2011-11-22 09:59:21 +08001060 /* No need for kzalloc as it is initialized in following hypercall
1061 * GNTTABOP_setup_table.
1062 */
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -07001063 frames = kmalloc(nr_gframes * sizeof(unsigned long), GFP_ATOMIC);
1064 if (!frames)
1065 return -ENOMEM;
1066
1067 setup.dom = DOMID_SELF;
1068 setup.nr_frames = nr_gframes;
Isaku Yamahata87e27cf2008-04-02 10:53:52 -07001069 set_xen_guest_handle(setup.frame_list, frames);
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -07001070
1071 rc = HYPERVISOR_grant_table_op(GNTTABOP_setup_table, &setup, 1);
1072 if (rc == -ENOSYS) {
1073 kfree(frames);
1074 return -ENOSYS;
1075 }
1076
1077 BUG_ON(rc || setup.status);
1078
Annie Li0f9f5a92011-11-22 09:58:06 +08001079 rc = gnttab_interface->map_frames(frames, nr_gframes);
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -07001080
1081 kfree(frames);
1082
Annie Li0f9f5a92011-11-22 09:58:06 +08001083 return rc;
1084}
1085
1086static struct gnttab_ops gnttab_v1_ops = {
1087 .map_frames = gnttab_map_frames_v1,
1088 .unmap_frames = gnttab_unmap_frames_v1,
1089 .update_entry = gnttab_update_entry_v1,
1090 .end_foreign_access_ref = gnttab_end_foreign_access_ref_v1,
1091 .end_foreign_transfer_ref = gnttab_end_foreign_transfer_ref_v1,
1092 .query_foreign_access = gnttab_query_foreign_access_v1,
1093};
1094
Annie Li85ff6ac2011-11-22 09:59:21 +08001095static struct gnttab_ops gnttab_v2_ops = {
1096 .map_frames = gnttab_map_frames_v2,
1097 .unmap_frames = gnttab_unmap_frames_v2,
1098 .update_entry = gnttab_update_entry_v2,
1099 .end_foreign_access_ref = gnttab_end_foreign_access_ref_v2,
1100 .end_foreign_transfer_ref = gnttab_end_foreign_transfer_ref_v2,
1101 .query_foreign_access = gnttab_query_foreign_access_v2,
Annie Li66667542011-12-12 18:14:42 +08001102 .update_subpage_entry = gnttab_update_subpage_entry_v2,
Annie Li9438ce92011-12-12 18:15:07 +08001103 .update_trans_entry = gnttab_update_trans_entry_v2,
Annie Li85ff6ac2011-11-22 09:59:21 +08001104};
1105
Annie Li0f9f5a92011-11-22 09:58:06 +08001106static void gnttab_request_version(void)
1107{
Annie Li85ff6ac2011-11-22 09:59:21 +08001108 int rc;
1109 struct gnttab_set_version gsv;
1110
Konrad Rzeszutek Wilk69e8f432012-01-25 00:13:20 -05001111 if (xen_hvm_domain())
1112 gsv.version = 1;
1113 else
1114 gsv.version = 2;
Annie Li85ff6ac2011-11-22 09:59:21 +08001115 rc = HYPERVISOR_grant_table_op(GNTTABOP_set_version, &gsv, 1);
Konrad Rzeszutek Wilk69e8f432012-01-25 00:13:20 -05001116 if (rc == 0 && gsv.version == 2) {
Annie Li85ff6ac2011-11-22 09:59:21 +08001117 grant_table_version = 2;
1118 gnttab_interface = &gnttab_v2_ops;
1119 } else if (grant_table_version == 2) {
1120 /*
1121 * If we've already used version 2 features,
1122 * but then suddenly discover that they're not
1123 * available (e.g. migrating to an older
1124 * version of Xen), almost unbounded badness
1125 * can happen.
1126 */
1127 panic("we need grant tables version 2, but only version 1 is available");
1128 } else {
1129 grant_table_version = 1;
1130 gnttab_interface = &gnttab_v1_ops;
1131 }
Annie Li0f9f5a92011-11-22 09:58:06 +08001132 printk(KERN_INFO "Grant tables using version %d layout.\n",
1133 grant_table_version);
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -07001134}
1135
Jeremy Fitzhardinge0e913982008-05-26 23:31:27 +01001136int gnttab_resume(void)
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -07001137{
Stefano Stabellini183d03c2010-05-17 17:08:21 +01001138 unsigned int max_nr_gframes;
1139
Annie Li0f9f5a92011-11-22 09:58:06 +08001140 gnttab_request_version();
Stefano Stabellini183d03c2010-05-17 17:08:21 +01001141 max_nr_gframes = gnttab_max_grant_frames();
1142 if (max_nr_gframes < nr_grant_frames)
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -07001143 return -ENOSYS;
Stefano Stabellini183d03c2010-05-17 17:08:21 +01001144
1145 if (xen_pv_domain())
1146 return gnttab_map(0, nr_grant_frames - 1);
1147
Annie Li0f9f5a92011-11-22 09:58:06 +08001148 if (gnttab_shared.addr == NULL) {
1149 gnttab_shared.addr = ioremap(xen_hvm_resume_frames,
1150 PAGE_SIZE * max_nr_gframes);
1151 if (gnttab_shared.addr == NULL) {
Stefano Stabellini183d03c2010-05-17 17:08:21 +01001152 printk(KERN_WARNING
1153 "Failed to ioremap gnttab share frames!");
1154 return -ENOMEM;
1155 }
1156 }
1157
1158 gnttab_map(0, nr_grant_frames - 1);
1159
1160 return 0;
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -07001161}
1162
Jeremy Fitzhardinge0e913982008-05-26 23:31:27 +01001163int gnttab_suspend(void)
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -07001164{
Annie Li0f9f5a92011-11-22 09:58:06 +08001165 gnttab_interface->unmap_frames();
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -07001166 return 0;
1167}
1168
1169static int gnttab_expand(unsigned int req_entries)
1170{
1171 int rc;
1172 unsigned int cur, extra;
1173
1174 cur = nr_grant_frames;
1175 extra = ((req_entries + (GREFS_PER_GRANT_FRAME-1)) /
1176 GREFS_PER_GRANT_FRAME);
Stefano Stabellini183d03c2010-05-17 17:08:21 +01001177 if (cur + extra > gnttab_max_grant_frames())
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -07001178 return -ENOSPC;
1179
1180 rc = gnttab_map(cur, cur + extra - 1);
1181 if (rc == 0)
1182 rc = grow_gnttab_list(extra);
1183
1184 return rc;
1185}
1186
Stefano Stabellini183d03c2010-05-17 17:08:21 +01001187int gnttab_init(void)
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -07001188{
1189 int i;
Michael Abd-El-Malekbbc60c12008-04-04 02:33:48 -07001190 unsigned int max_nr_glist_frames, nr_glist_frames;
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -07001191 unsigned int nr_init_grefs;
Julia Lawall6b5e7d92012-04-15 11:27:12 +02001192 int ret;
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -07001193
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -07001194 nr_grant_frames = 1;
1195 boot_max_nr_grant_frames = __max_nr_grant_frames();
1196
1197 /* Determine the maximum number of frames required for the
1198 * grant reference free list on the current hypervisor.
1199 */
1200 max_nr_glist_frames = (boot_max_nr_grant_frames *
Michael Abd-El-Malekbbc60c12008-04-04 02:33:48 -07001201 GREFS_PER_GRANT_FRAME / RPP);
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -07001202
1203 gnttab_list = kmalloc(max_nr_glist_frames * sizeof(grant_ref_t *),
1204 GFP_KERNEL);
1205 if (gnttab_list == NULL)
1206 return -ENOMEM;
1207
Michael Abd-El-Malekbbc60c12008-04-04 02:33:48 -07001208 nr_glist_frames = (nr_grant_frames * GREFS_PER_GRANT_FRAME + RPP - 1) / RPP;
1209 for (i = 0; i < nr_glist_frames; i++) {
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -07001210 gnttab_list[i] = (grant_ref_t *)__get_free_page(GFP_KERNEL);
Julia Lawall6b5e7d92012-04-15 11:27:12 +02001211 if (gnttab_list[i] == NULL) {
1212 ret = -ENOMEM;
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -07001213 goto ini_nomem;
Julia Lawall6b5e7d92012-04-15 11:27:12 +02001214 }
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -07001215 }
1216
Julia Lawall6b5e7d92012-04-15 11:27:12 +02001217 if (gnttab_resume() < 0) {
1218 ret = -ENODEV;
1219 goto ini_nomem;
1220 }
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -07001221
1222 nr_init_grefs = nr_grant_frames * GREFS_PER_GRANT_FRAME;
1223
1224 for (i = NR_RESERVED_ENTRIES; i < nr_init_grefs - 1; i++)
1225 gnttab_entry(i) = i + 1;
1226
1227 gnttab_entry(nr_init_grefs - 1) = GNTTAB_LIST_END;
1228 gnttab_free_count = nr_init_grefs - NR_RESERVED_ENTRIES;
1229 gnttab_free_head = NR_RESERVED_ENTRIES;
1230
1231 printk("Grant table initialized\n");
1232 return 0;
1233
1234 ini_nomem:
1235 for (i--; i >= 0; i--)
1236 free_page((unsigned long)gnttab_list[i]);
1237 kfree(gnttab_list);
Julia Lawall6b5e7d92012-04-15 11:27:12 +02001238 return ret;
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -07001239}
Stefano Stabellini183d03c2010-05-17 17:08:21 +01001240EXPORT_SYMBOL_GPL(gnttab_init);
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -07001241
Stefano Stabellini183d03c2010-05-17 17:08:21 +01001242static int __devinit __gnttab_init(void)
1243{
1244 /* Delay grant-table initialization in the PV on HVM case */
1245 if (xen_hvm_domain())
1246 return 0;
1247
1248 if (!xen_pv_domain())
1249 return -ENODEV;
1250
1251 return gnttab_init();
1252}
1253
1254core_initcall(__gnttab_init);