blob: 157c0ccda3efdce0e8a3c4b9ab0937ade1d932ec [file] [log] [blame]
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -07001/******************************************************************************
2 * grant_table.c
3 *
4 * Granting foreign access to our memory reservation.
5 *
6 * Copyright (c) 2005-2006, Christopher Clark
7 * Copyright (c) 2004-2005, K A Fraser
8 *
9 * This program is free software; you can redistribute it and/or
10 * modify it under the terms of the GNU General Public License version 2
11 * as published by the Free Software Foundation; or, when distributed
12 * separately from the Linux kernel or incorporated into other
13 * software packages, subject to the following license:
14 *
15 * Permission is hereby granted, free of charge, to any person obtaining a copy
16 * of this source file (the "Software"), to deal in the Software without
17 * restriction, including without limitation the rights to use, copy, modify,
18 * merge, publish, distribute, sublicense, and/or sell copies of the Software,
19 * and to permit persons to whom the Software is furnished to do so, subject to
20 * the following conditions:
21 *
22 * The above copyright notice and this permission notice shall be included in
23 * all copies or substantial portions of the Software.
24 *
25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
26 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
27 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
28 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
29 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
30 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
31 * IN THE SOFTWARE.
32 */
33
34#include <linux/module.h>
35#include <linux/sched.h>
36#include <linux/mm.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090037#include <linux/slab.h>
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -070038#include <linux/vmalloc.h>
39#include <linux/uaccess.h>
Stefano Stabellini183d03c2010-05-17 17:08:21 +010040#include <linux/io.h>
Andres Lagar-Cavillac5718982012-09-14 14:26:59 +000041#include <linux/delay.h>
Stefano Stabellinif62805f2012-04-24 11:55:43 +010042#include <linux/hardirq.h>
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -070043
Jeremy Fitzhardinge1ccbf532009-10-06 15:11:14 -070044#include <xen/xen.h>
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -070045#include <xen/interface/xen.h>
46#include <xen/page.h>
47#include <xen/grant_table.h>
Stefano Stabellini183d03c2010-05-17 17:08:21 +010048#include <xen/interface/memory.h>
Annie Li85ff6ac2011-11-22 09:59:21 +080049#include <xen/hvc-console.h>
Jeremy Fitzhardingeecbf29c2008-12-16 12:37:07 -080050#include <asm/xen/hypercall.h>
Stefano Stabellini4d9310e2012-08-06 15:27:09 +010051#include <asm/xen/interface.h>
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -070052
53#include <asm/pgtable.h>
54#include <asm/sync_bitops.h>
55
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -070056/* External tools reserve first few grant table entries. */
57#define NR_RESERVED_ENTRIES 8
58#define GNTTAB_LIST_END 0xffffffff
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -070059
60static grant_ref_t **gnttab_list;
61static unsigned int nr_grant_frames;
62static unsigned int boot_max_nr_grant_frames;
63static int gnttab_free_count;
64static grant_ref_t gnttab_free_head;
65static DEFINE_SPINLOCK(gnttab_list_lock);
Stefano Stabellini183d03c2010-05-17 17:08:21 +010066unsigned long xen_hvm_resume_frames;
67EXPORT_SYMBOL_GPL(xen_hvm_resume_frames);
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -070068
Annie Li0f9f5a92011-11-22 09:58:06 +080069static union {
70 struct grant_entry_v1 *v1;
Annie Li85ff6ac2011-11-22 09:59:21 +080071 union grant_entry_v2 *v2;
Annie Li0f9f5a92011-11-22 09:58:06 +080072 void *addr;
73} gnttab_shared;
74
75/*This is a structure of function pointers for grant table*/
76struct gnttab_ops {
77 /*
Annie Li9dbc71d2011-12-12 18:13:57 +080078 * Mapping a list of frames for storing grant entries. Frames parameter
79 * is used to store grant table address when grant table being setup,
80 * nr_gframes is the number of frames to map grant table. Returning
81 * GNTST_okay means success and negative value means failure.
Annie Li0f9f5a92011-11-22 09:58:06 +080082 */
Ian Campbellef32f892012-10-17 09:39:14 +010083 int (*map_frames)(xen_pfn_t *frames, unsigned int nr_gframes);
Annie Li0f9f5a92011-11-22 09:58:06 +080084 /*
85 * Release a list of frames which are mapped in map_frames for grant
86 * entry status.
87 */
88 void (*unmap_frames)(void);
89 /*
Annie Li9dbc71d2011-12-12 18:13:57 +080090 * Introducing a valid entry into the grant table, granting the frame of
91 * this grant entry to domain for accessing or transfering. Ref
92 * parameter is reference of this introduced grant entry, domid is id of
93 * granted domain, frame is the page frame to be granted, and flags is
94 * status of the grant entry to be updated.
Annie Li0f9f5a92011-11-22 09:58:06 +080095 */
Annie Li9dbc71d2011-12-12 18:13:57 +080096 void (*update_entry)(grant_ref_t ref, domid_t domid,
97 unsigned long frame, unsigned flags);
Annie Li0f9f5a92011-11-22 09:58:06 +080098 /*
Annie Li9dbc71d2011-12-12 18:13:57 +080099 * Stop granting a grant entry to domain for accessing. Ref parameter is
100 * reference of a grant entry whose grant access will be stopped,
101 * readonly is not in use in this function. If the grant entry is
Annie Li0f9f5a92011-11-22 09:58:06 +0800102 * currently mapped for reading or writing, just return failure(==0)
103 * directly and don't tear down the grant access. Otherwise, stop grant
104 * access for this entry and return success(==1).
105 */
Annie Li9dbc71d2011-12-12 18:13:57 +0800106 int (*end_foreign_access_ref)(grant_ref_t ref, int readonly);
Annie Li0f9f5a92011-11-22 09:58:06 +0800107 /*
Annie Li9dbc71d2011-12-12 18:13:57 +0800108 * Stop granting a grant entry to domain for transfer. Ref parameter is
109 * reference of a grant entry whose grant transfer will be stopped. If
110 * tranfer has not started, just reclaim the grant entry and return
111 * failure(==0). Otherwise, wait for the transfer to complete and then
112 * return the frame.
Annie Li0f9f5a92011-11-22 09:58:06 +0800113 */
Annie Li9dbc71d2011-12-12 18:13:57 +0800114 unsigned long (*end_foreign_transfer_ref)(grant_ref_t ref);
Annie Li0f9f5a92011-11-22 09:58:06 +0800115 /*
Annie Li9dbc71d2011-12-12 18:13:57 +0800116 * Query the status of a grant entry. Ref parameter is reference of
Annie Li0f9f5a92011-11-22 09:58:06 +0800117 * queried grant entry, return value is the status of queried entry.
118 * Detailed status(writing/reading) can be gotten from the return value
119 * by bit operations.
120 */
Annie Li9dbc71d2011-12-12 18:13:57 +0800121 int (*query_foreign_access)(grant_ref_t ref);
Annie Li66667542011-12-12 18:14:42 +0800122 /*
123 * Grant a domain to access a range of bytes within the page referred by
124 * an available grant entry. Ref parameter is reference of a grant entry
125 * which will be sub-page accessed, domid is id of grantee domain, frame
126 * is frame address of subpage grant, flags is grant type and flag
127 * information, page_off is offset of the range of bytes, and length is
128 * length of bytes to be accessed.
129 */
130 void (*update_subpage_entry)(grant_ref_t ref, domid_t domid,
131 unsigned long frame, int flags,
132 unsigned page_off, unsigned length);
Annie Li9438ce92011-12-12 18:15:07 +0800133 /*
134 * Redirect an available grant entry on domain A to another grant
135 * reference of domain B, then allow domain C to use grant reference
136 * of domain B transitively. Ref parameter is an available grant entry
137 * reference on domain A, domid is id of domain C which accesses grant
138 * entry transitively, flags is grant type and flag information,
139 * trans_domid is id of domain B whose grant entry is finally accessed
140 * transitively, trans_gref is grant entry transitive reference of
141 * domain B.
142 */
143 void (*update_trans_entry)(grant_ref_t ref, domid_t domid, int flags,
144 domid_t trans_domid, grant_ref_t trans_gref);
Annie Li0f9f5a92011-11-22 09:58:06 +0800145};
146
147static struct gnttab_ops *gnttab_interface;
148
Annie Li85ff6ac2011-11-22 09:59:21 +0800149/*This reflects status of grant entries, so act as a global value*/
150static grant_status_t *grstatus;
151
Annie Li0f9f5a92011-11-22 09:58:06 +0800152static int grant_table_version;
Matt Wilsond0b4d642013-01-15 13:21:27 +0000153static int grefs_per_grant_frame;
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700154
155static struct gnttab_free_callback *gnttab_free_callback_list;
156
157static int gnttab_expand(unsigned int req_entries);
158
159#define RPP (PAGE_SIZE / sizeof(grant_ref_t))
Annie Li85ff6ac2011-11-22 09:59:21 +0800160#define SPP (PAGE_SIZE / sizeof(grant_status_t))
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700161
162static inline grant_ref_t *__gnttab_entry(grant_ref_t entry)
163{
164 return &gnttab_list[(entry) / RPP][(entry) % RPP];
165}
166/* This can be used as an l-value */
167#define gnttab_entry(entry) (*__gnttab_entry(entry))
168
169static int get_free_entries(unsigned count)
170{
171 unsigned long flags;
Konrad Rzeszutek Wilk272800d2011-07-22 14:00:06 -0400172 int ref, rc = 0;
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700173 grant_ref_t head;
174
175 spin_lock_irqsave(&gnttab_list_lock, flags);
176
177 if ((gnttab_free_count < count) &&
178 ((rc = gnttab_expand(count - gnttab_free_count)) < 0)) {
179 spin_unlock_irqrestore(&gnttab_list_lock, flags);
180 return rc;
181 }
182
183 ref = head = gnttab_free_head;
184 gnttab_free_count -= count;
185 while (count-- > 1)
186 head = gnttab_entry(head);
187 gnttab_free_head = gnttab_entry(head);
188 gnttab_entry(head) = GNTTAB_LIST_END;
189
190 spin_unlock_irqrestore(&gnttab_list_lock, flags);
191
192 return ref;
193}
194
195static void do_free_callbacks(void)
196{
197 struct gnttab_free_callback *callback, *next;
198
199 callback = gnttab_free_callback_list;
200 gnttab_free_callback_list = NULL;
201
202 while (callback != NULL) {
203 next = callback->next;
204 if (gnttab_free_count >= callback->count) {
205 callback->next = NULL;
206 callback->fn(callback->arg);
207 } else {
208 callback->next = gnttab_free_callback_list;
209 gnttab_free_callback_list = callback;
210 }
211 callback = next;
212 }
213}
214
215static inline void check_free_callbacks(void)
216{
217 if (unlikely(gnttab_free_callback_list))
218 do_free_callbacks();
219}
220
221static void put_free_entry(grant_ref_t ref)
222{
223 unsigned long flags;
224 spin_lock_irqsave(&gnttab_list_lock, flags);
225 gnttab_entry(ref) = gnttab_free_head;
226 gnttab_free_head = ref;
227 gnttab_free_count++;
228 check_free_callbacks();
229 spin_unlock_irqrestore(&gnttab_list_lock, flags);
230}
231
Annie Li0f9f5a92011-11-22 09:58:06 +0800232/*
Annie Li85ff6ac2011-11-22 09:59:21 +0800233 * Following applies to gnttab_update_entry_v1 and gnttab_update_entry_v2.
Annie Li0f9f5a92011-11-22 09:58:06 +0800234 * Introducing a valid entry into the grant table:
235 * 1. Write ent->domid.
236 * 2. Write ent->frame:
237 * GTF_permit_access: Frame to which access is permitted.
238 * GTF_accept_transfer: Pseudo-phys frame slot being filled by new
239 * frame, or zero if none.
240 * 3. Write memory barrier (WMB).
241 * 4. Write ent->flags, inc. valid type.
242 */
243static void gnttab_update_entry_v1(grant_ref_t ref, domid_t domid,
244 unsigned long frame, unsigned flags)
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700245{
Annie Li0f9f5a92011-11-22 09:58:06 +0800246 gnttab_shared.v1[ref].domid = domid;
247 gnttab_shared.v1[ref].frame = frame;
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700248 wmb();
Annie Li0f9f5a92011-11-22 09:58:06 +0800249 gnttab_shared.v1[ref].flags = flags;
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700250}
251
Annie Li85ff6ac2011-11-22 09:59:21 +0800252static void gnttab_update_entry_v2(grant_ref_t ref, domid_t domid,
253 unsigned long frame, unsigned flags)
254{
255 gnttab_shared.v2[ref].hdr.domid = domid;
256 gnttab_shared.v2[ref].full_page.frame = frame;
257 wmb();
258 gnttab_shared.v2[ref].hdr.flags = GTF_permit_access | flags;
259}
260
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700261/*
262 * Public grant-issuing interface functions
263 */
264void gnttab_grant_foreign_access_ref(grant_ref_t ref, domid_t domid,
265 unsigned long frame, int readonly)
266{
Annie Li0f9f5a92011-11-22 09:58:06 +0800267 gnttab_interface->update_entry(ref, domid, frame,
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700268 GTF_permit_access | (readonly ? GTF_readonly : 0));
269}
270EXPORT_SYMBOL_GPL(gnttab_grant_foreign_access_ref);
271
272int gnttab_grant_foreign_access(domid_t domid, unsigned long frame,
273 int readonly)
274{
275 int ref;
276
277 ref = get_free_entries(1);
278 if (unlikely(ref < 0))
279 return -ENOSPC;
280
281 gnttab_grant_foreign_access_ref(ref, domid, frame, readonly);
282
283 return ref;
284}
285EXPORT_SYMBOL_GPL(gnttab_grant_foreign_access);
286
Konrad Rzeszutek Wilkb8b0f552012-08-21 14:49:34 -0400287static void gnttab_update_subpage_entry_v2(grant_ref_t ref, domid_t domid,
288 unsigned long frame, int flags,
289 unsigned page_off, unsigned length)
Annie Li66667542011-12-12 18:14:42 +0800290{
291 gnttab_shared.v2[ref].sub_page.frame = frame;
292 gnttab_shared.v2[ref].sub_page.page_off = page_off;
293 gnttab_shared.v2[ref].sub_page.length = length;
294 gnttab_shared.v2[ref].hdr.domid = domid;
295 wmb();
296 gnttab_shared.v2[ref].hdr.flags =
297 GTF_permit_access | GTF_sub_page | flags;
298}
299
300int gnttab_grant_foreign_access_subpage_ref(grant_ref_t ref, domid_t domid,
301 unsigned long frame, int flags,
302 unsigned page_off,
303 unsigned length)
304{
305 if (flags & (GTF_accept_transfer | GTF_reading |
306 GTF_writing | GTF_transitive))
307 return -EPERM;
308
309 if (gnttab_interface->update_subpage_entry == NULL)
310 return -ENOSYS;
311
312 gnttab_interface->update_subpage_entry(ref, domid, frame, flags,
313 page_off, length);
314
315 return 0;
316}
317EXPORT_SYMBOL_GPL(gnttab_grant_foreign_access_subpage_ref);
318
319int gnttab_grant_foreign_access_subpage(domid_t domid, unsigned long frame,
320 int flags, unsigned page_off,
321 unsigned length)
322{
323 int ref, rc;
324
325 ref = get_free_entries(1);
326 if (unlikely(ref < 0))
327 return -ENOSPC;
328
329 rc = gnttab_grant_foreign_access_subpage_ref(ref, domid, frame, flags,
330 page_off, length);
331 if (rc < 0) {
332 put_free_entry(ref);
333 return rc;
334 }
335
336 return ref;
337}
338EXPORT_SYMBOL_GPL(gnttab_grant_foreign_access_subpage);
339
340bool gnttab_subpage_grants_available(void)
341{
342 return gnttab_interface->update_subpage_entry != NULL;
343}
344EXPORT_SYMBOL_GPL(gnttab_subpage_grants_available);
345
Konrad Rzeszutek Wilkb8b0f552012-08-21 14:49:34 -0400346static void gnttab_update_trans_entry_v2(grant_ref_t ref, domid_t domid,
347 int flags, domid_t trans_domid,
348 grant_ref_t trans_gref)
Annie Li9438ce92011-12-12 18:15:07 +0800349{
350 gnttab_shared.v2[ref].transitive.trans_domid = trans_domid;
351 gnttab_shared.v2[ref].transitive.gref = trans_gref;
352 gnttab_shared.v2[ref].hdr.domid = domid;
353 wmb();
354 gnttab_shared.v2[ref].hdr.flags =
355 GTF_permit_access | GTF_transitive | flags;
356}
357
358int gnttab_grant_foreign_access_trans_ref(grant_ref_t ref, domid_t domid,
359 int flags, domid_t trans_domid,
360 grant_ref_t trans_gref)
361{
362 if (flags & (GTF_accept_transfer | GTF_reading |
363 GTF_writing | GTF_sub_page))
364 return -EPERM;
365
366 if (gnttab_interface->update_trans_entry == NULL)
367 return -ENOSYS;
368
369 gnttab_interface->update_trans_entry(ref, domid, flags, trans_domid,
370 trans_gref);
371
372 return 0;
373}
374EXPORT_SYMBOL_GPL(gnttab_grant_foreign_access_trans_ref);
375
376int gnttab_grant_foreign_access_trans(domid_t domid, int flags,
377 domid_t trans_domid,
378 grant_ref_t trans_gref)
379{
380 int ref, rc;
381
382 ref = get_free_entries(1);
383 if (unlikely(ref < 0))
384 return -ENOSPC;
385
386 rc = gnttab_grant_foreign_access_trans_ref(ref, domid, flags,
387 trans_domid, trans_gref);
388 if (rc < 0) {
389 put_free_entry(ref);
390 return rc;
391 }
392
393 return ref;
394}
395EXPORT_SYMBOL_GPL(gnttab_grant_foreign_access_trans);
396
397bool gnttab_trans_grants_available(void)
398{
399 return gnttab_interface->update_trans_entry != NULL;
400}
401EXPORT_SYMBOL_GPL(gnttab_trans_grants_available);
402
Annie Li0f9f5a92011-11-22 09:58:06 +0800403static int gnttab_query_foreign_access_v1(grant_ref_t ref)
404{
405 return gnttab_shared.v1[ref].flags & (GTF_reading|GTF_writing);
406}
407
Annie Li85ff6ac2011-11-22 09:59:21 +0800408static int gnttab_query_foreign_access_v2(grant_ref_t ref)
409{
410 return grstatus[ref] & (GTF_reading|GTF_writing);
411}
412
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700413int gnttab_query_foreign_access(grant_ref_t ref)
414{
Annie Li0f9f5a92011-11-22 09:58:06 +0800415 return gnttab_interface->query_foreign_access(ref);
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700416}
417EXPORT_SYMBOL_GPL(gnttab_query_foreign_access);
418
Annie Li0f9f5a92011-11-22 09:58:06 +0800419static int gnttab_end_foreign_access_ref_v1(grant_ref_t ref, int readonly)
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700420{
421 u16 flags, nflags;
Annie Lib1e495b2011-11-22 09:58:47 +0800422 u16 *pflags;
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700423
Annie Lib1e495b2011-11-22 09:58:47 +0800424 pflags = &gnttab_shared.v1[ref].flags;
425 nflags = *pflags;
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700426 do {
427 flags = nflags;
Jan Beulich569ca5b2012-04-05 16:10:07 +0100428 if (flags & (GTF_reading|GTF_writing))
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700429 return 0;
Annie Lib1e495b2011-11-22 09:58:47 +0800430 } while ((nflags = sync_cmpxchg(pflags, flags, 0)) != flags);
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700431
432 return 1;
433}
Annie Li0f9f5a92011-11-22 09:58:06 +0800434
Annie Li85ff6ac2011-11-22 09:59:21 +0800435static int gnttab_end_foreign_access_ref_v2(grant_ref_t ref, int readonly)
436{
437 gnttab_shared.v2[ref].hdr.flags = 0;
438 mb();
439 if (grstatus[ref] & (GTF_reading|GTF_writing)) {
440 return 0;
441 } else {
442 /* The read of grstatus needs to have acquire
443 semantics. On x86, reads already have
444 that, and we just need to protect against
445 compiler reorderings. On other
446 architectures we may need a full
447 barrier. */
448#ifdef CONFIG_X86
449 barrier();
450#else
451 mb();
452#endif
453 }
454
455 return 1;
456}
457
Jan Beulich569ca5b2012-04-05 16:10:07 +0100458static inline int _gnttab_end_foreign_access_ref(grant_ref_t ref, int readonly)
Annie Li0f9f5a92011-11-22 09:58:06 +0800459{
460 return gnttab_interface->end_foreign_access_ref(ref, readonly);
461}
Jan Beulich569ca5b2012-04-05 16:10:07 +0100462
463int gnttab_end_foreign_access_ref(grant_ref_t ref, int readonly)
464{
465 if (_gnttab_end_foreign_access_ref(ref, readonly))
466 return 1;
467 pr_warn("WARNING: g.e. %#x still in use!\n", ref);
468 return 0;
469}
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700470EXPORT_SYMBOL_GPL(gnttab_end_foreign_access_ref);
471
Jan Beulich569ca5b2012-04-05 16:10:07 +0100472struct deferred_entry {
473 struct list_head list;
474 grant_ref_t ref;
475 bool ro;
476 uint16_t warn_delay;
477 struct page *page;
478};
479static LIST_HEAD(deferred_list);
480static void gnttab_handle_deferred(unsigned long);
481static DEFINE_TIMER(deferred_timer, gnttab_handle_deferred, 0, 0);
482
483static void gnttab_handle_deferred(unsigned long unused)
484{
485 unsigned int nr = 10;
486 struct deferred_entry *first = NULL;
487 unsigned long flags;
488
489 spin_lock_irqsave(&gnttab_list_lock, flags);
490 while (nr--) {
491 struct deferred_entry *entry
492 = list_first_entry(&deferred_list,
493 struct deferred_entry, list);
494
495 if (entry == first)
496 break;
497 list_del(&entry->list);
498 spin_unlock_irqrestore(&gnttab_list_lock, flags);
499 if (_gnttab_end_foreign_access_ref(entry->ref, entry->ro)) {
500 put_free_entry(entry->ref);
501 if (entry->page) {
502 pr_debug("freeing g.e. %#x (pfn %#lx)\n",
503 entry->ref, page_to_pfn(entry->page));
504 __free_page(entry->page);
505 } else
506 pr_info("freeing g.e. %#x\n", entry->ref);
507 kfree(entry);
508 entry = NULL;
509 } else {
510 if (!--entry->warn_delay)
511 pr_info("g.e. %#x still pending\n",
512 entry->ref);
513 if (!first)
514 first = entry;
515 }
516 spin_lock_irqsave(&gnttab_list_lock, flags);
517 if (entry)
518 list_add_tail(&entry->list, &deferred_list);
519 else if (list_empty(&deferred_list))
520 break;
521 }
522 if (!list_empty(&deferred_list) && !timer_pending(&deferred_timer)) {
523 deferred_timer.expires = jiffies + HZ;
524 add_timer(&deferred_timer);
525 }
526 spin_unlock_irqrestore(&gnttab_list_lock, flags);
527}
528
529static void gnttab_add_deferred(grant_ref_t ref, bool readonly,
530 struct page *page)
531{
532 struct deferred_entry *entry = kmalloc(sizeof(*entry), GFP_ATOMIC);
533 const char *what = KERN_WARNING "leaking";
534
535 if (entry) {
536 unsigned long flags;
537
538 entry->ref = ref;
539 entry->ro = readonly;
540 entry->page = page;
541 entry->warn_delay = 60;
542 spin_lock_irqsave(&gnttab_list_lock, flags);
543 list_add_tail(&entry->list, &deferred_list);
544 if (!timer_pending(&deferred_timer)) {
545 deferred_timer.expires = jiffies + HZ;
546 add_timer(&deferred_timer);
547 }
548 spin_unlock_irqrestore(&gnttab_list_lock, flags);
549 what = KERN_DEBUG "deferring";
550 }
551 printk("%s g.e. %#x (pfn %#lx)\n",
552 what, ref, page ? page_to_pfn(page) : -1);
553}
554
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700555void gnttab_end_foreign_access(grant_ref_t ref, int readonly,
556 unsigned long page)
557{
558 if (gnttab_end_foreign_access_ref(ref, readonly)) {
559 put_free_entry(ref);
560 if (page != 0)
561 free_page(page);
Jan Beulich569ca5b2012-04-05 16:10:07 +0100562 } else
563 gnttab_add_deferred(ref, readonly,
564 page ? virt_to_page(page) : NULL);
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700565}
566EXPORT_SYMBOL_GPL(gnttab_end_foreign_access);
567
568int gnttab_grant_foreign_transfer(domid_t domid, unsigned long pfn)
569{
570 int ref;
571
572 ref = get_free_entries(1);
573 if (unlikely(ref < 0))
574 return -ENOSPC;
575 gnttab_grant_foreign_transfer_ref(ref, domid, pfn);
576
577 return ref;
578}
579EXPORT_SYMBOL_GPL(gnttab_grant_foreign_transfer);
580
581void gnttab_grant_foreign_transfer_ref(grant_ref_t ref, domid_t domid,
582 unsigned long pfn)
583{
Annie Li0f9f5a92011-11-22 09:58:06 +0800584 gnttab_interface->update_entry(ref, domid, pfn, GTF_accept_transfer);
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700585}
586EXPORT_SYMBOL_GPL(gnttab_grant_foreign_transfer_ref);
587
Annie Li0f9f5a92011-11-22 09:58:06 +0800588static unsigned long gnttab_end_foreign_transfer_ref_v1(grant_ref_t ref)
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700589{
590 unsigned long frame;
591 u16 flags;
Annie Lib1e495b2011-11-22 09:58:47 +0800592 u16 *pflags;
593
594 pflags = &gnttab_shared.v1[ref].flags;
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700595
596 /*
597 * If a transfer is not even yet started, try to reclaim the grant
598 * reference and return failure (== 0).
599 */
Annie Lib1e495b2011-11-22 09:58:47 +0800600 while (!((flags = *pflags) & GTF_transfer_committed)) {
601 if (sync_cmpxchg(pflags, flags, 0) == flags)
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700602 return 0;
603 cpu_relax();
604 }
605
606 /* If a transfer is in progress then wait until it is completed. */
607 while (!(flags & GTF_transfer_completed)) {
Annie Lib1e495b2011-11-22 09:58:47 +0800608 flags = *pflags;
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700609 cpu_relax();
610 }
611
612 rmb(); /* Read the frame number /after/ reading completion status. */
Annie Li0f9f5a92011-11-22 09:58:06 +0800613 frame = gnttab_shared.v1[ref].frame;
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700614 BUG_ON(frame == 0);
615
616 return frame;
617}
Annie Li0f9f5a92011-11-22 09:58:06 +0800618
Annie Li85ff6ac2011-11-22 09:59:21 +0800619static unsigned long gnttab_end_foreign_transfer_ref_v2(grant_ref_t ref)
620{
621 unsigned long frame;
622 u16 flags;
623 u16 *pflags;
624
625 pflags = &gnttab_shared.v2[ref].hdr.flags;
626
627 /*
628 * If a transfer is not even yet started, try to reclaim the grant
629 * reference and return failure (== 0).
630 */
631 while (!((flags = *pflags) & GTF_transfer_committed)) {
632 if (sync_cmpxchg(pflags, flags, 0) == flags)
633 return 0;
634 cpu_relax();
635 }
636
637 /* If a transfer is in progress then wait until it is completed. */
638 while (!(flags & GTF_transfer_completed)) {
639 flags = *pflags;
640 cpu_relax();
641 }
642
643 rmb(); /* Read the frame number /after/ reading completion status. */
644 frame = gnttab_shared.v2[ref].full_page.frame;
645 BUG_ON(frame == 0);
646
647 return frame;
648}
649
Annie Li0f9f5a92011-11-22 09:58:06 +0800650unsigned long gnttab_end_foreign_transfer_ref(grant_ref_t ref)
651{
652 return gnttab_interface->end_foreign_transfer_ref(ref);
653}
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700654EXPORT_SYMBOL_GPL(gnttab_end_foreign_transfer_ref);
655
656unsigned long gnttab_end_foreign_transfer(grant_ref_t ref)
657{
658 unsigned long frame = gnttab_end_foreign_transfer_ref(ref);
659 put_free_entry(ref);
660 return frame;
661}
662EXPORT_SYMBOL_GPL(gnttab_end_foreign_transfer);
663
664void gnttab_free_grant_reference(grant_ref_t ref)
665{
666 put_free_entry(ref);
667}
668EXPORT_SYMBOL_GPL(gnttab_free_grant_reference);
669
670void gnttab_free_grant_references(grant_ref_t head)
671{
672 grant_ref_t ref;
673 unsigned long flags;
674 int count = 1;
675 if (head == GNTTAB_LIST_END)
676 return;
677 spin_lock_irqsave(&gnttab_list_lock, flags);
678 ref = head;
679 while (gnttab_entry(ref) != GNTTAB_LIST_END) {
680 ref = gnttab_entry(ref);
681 count++;
682 }
683 gnttab_entry(ref) = gnttab_free_head;
684 gnttab_free_head = head;
685 gnttab_free_count += count;
686 check_free_callbacks();
687 spin_unlock_irqrestore(&gnttab_list_lock, flags);
688}
689EXPORT_SYMBOL_GPL(gnttab_free_grant_references);
690
691int gnttab_alloc_grant_references(u16 count, grant_ref_t *head)
692{
693 int h = get_free_entries(count);
694
695 if (h < 0)
696 return -ENOSPC;
697
698 *head = h;
699
700 return 0;
701}
702EXPORT_SYMBOL_GPL(gnttab_alloc_grant_references);
703
704int gnttab_empty_grant_references(const grant_ref_t *private_head)
705{
706 return (*private_head == GNTTAB_LIST_END);
707}
708EXPORT_SYMBOL_GPL(gnttab_empty_grant_references);
709
710int gnttab_claim_grant_reference(grant_ref_t *private_head)
711{
712 grant_ref_t g = *private_head;
713 if (unlikely(g == GNTTAB_LIST_END))
714 return -ENOSPC;
715 *private_head = gnttab_entry(g);
716 return g;
717}
718EXPORT_SYMBOL_GPL(gnttab_claim_grant_reference);
719
720void gnttab_release_grant_reference(grant_ref_t *private_head,
721 grant_ref_t release)
722{
723 gnttab_entry(release) = *private_head;
724 *private_head = release;
725}
726EXPORT_SYMBOL_GPL(gnttab_release_grant_reference);
727
728void gnttab_request_free_callback(struct gnttab_free_callback *callback,
729 void (*fn)(void *), void *arg, u16 count)
730{
731 unsigned long flags;
732 spin_lock_irqsave(&gnttab_list_lock, flags);
733 if (callback->next)
734 goto out;
735 callback->fn = fn;
736 callback->arg = arg;
737 callback->count = count;
738 callback->next = gnttab_free_callback_list;
739 gnttab_free_callback_list = callback;
740 check_free_callbacks();
741out:
742 spin_unlock_irqrestore(&gnttab_list_lock, flags);
743}
744EXPORT_SYMBOL_GPL(gnttab_request_free_callback);
745
746void gnttab_cancel_free_callback(struct gnttab_free_callback *callback)
747{
748 struct gnttab_free_callback **pcb;
749 unsigned long flags;
750
751 spin_lock_irqsave(&gnttab_list_lock, flags);
752 for (pcb = &gnttab_free_callback_list; *pcb; pcb = &(*pcb)->next) {
753 if (*pcb == callback) {
754 *pcb = callback->next;
755 break;
756 }
757 }
758 spin_unlock_irqrestore(&gnttab_list_lock, flags);
759}
760EXPORT_SYMBOL_GPL(gnttab_cancel_free_callback);
761
762static int grow_gnttab_list(unsigned int more_frames)
763{
764 unsigned int new_nr_grant_frames, extra_entries, i;
Michael Abd-El-Malekbbc60c12008-04-04 02:33:48 -0700765 unsigned int nr_glist_frames, new_nr_glist_frames;
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700766
Matt Wilsond0b4d642013-01-15 13:21:27 +0000767 BUG_ON(grefs_per_grant_frame == 0);
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700768
Matt Wilsond0b4d642013-01-15 13:21:27 +0000769 new_nr_grant_frames = nr_grant_frames + more_frames;
770 extra_entries = more_frames * grefs_per_grant_frame;
771
772 nr_glist_frames = (nr_grant_frames * grefs_per_grant_frame + RPP - 1) / RPP;
Michael Abd-El-Malekbbc60c12008-04-04 02:33:48 -0700773 new_nr_glist_frames =
Matt Wilsond0b4d642013-01-15 13:21:27 +0000774 (new_nr_grant_frames * grefs_per_grant_frame + RPP - 1) / RPP;
Michael Abd-El-Malekbbc60c12008-04-04 02:33:48 -0700775 for (i = nr_glist_frames; i < new_nr_glist_frames; i++) {
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700776 gnttab_list[i] = (grant_ref_t *)__get_free_page(GFP_ATOMIC);
777 if (!gnttab_list[i])
778 goto grow_nomem;
779 }
780
781
Matt Wilsond0b4d642013-01-15 13:21:27 +0000782 for (i = grefs_per_grant_frame * nr_grant_frames;
783 i < grefs_per_grant_frame * new_nr_grant_frames - 1; i++)
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700784 gnttab_entry(i) = i + 1;
785
786 gnttab_entry(i) = gnttab_free_head;
Matt Wilsond0b4d642013-01-15 13:21:27 +0000787 gnttab_free_head = grefs_per_grant_frame * nr_grant_frames;
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700788 gnttab_free_count += extra_entries;
789
790 nr_grant_frames = new_nr_grant_frames;
791
792 check_free_callbacks();
793
794 return 0;
795
796grow_nomem:
Michael Abd-El-Malekbbc60c12008-04-04 02:33:48 -0700797 for ( ; i >= nr_glist_frames; i--)
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700798 free_page((unsigned long) gnttab_list[i]);
799 return -ENOMEM;
800}
801
802static unsigned int __max_nr_grant_frames(void)
803{
804 struct gnttab_query_size query;
805 int rc;
806
807 query.dom = DOMID_SELF;
808
809 rc = HYPERVISOR_grant_table_op(GNTTABOP_query_size, &query, 1);
810 if ((rc < 0) || (query.status != GNTST_okay))
811 return 4; /* Legacy max supported number of frames */
812
813 return query.max_nr_frames;
814}
815
Stefano Stabellini183d03c2010-05-17 17:08:21 +0100816unsigned int gnttab_max_grant_frames(void)
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700817{
818 unsigned int xen_max = __max_nr_grant_frames();
819
820 if (xen_max > boot_max_nr_grant_frames)
821 return boot_max_nr_grant_frames;
822 return xen_max;
823}
Stefano Stabellini183d03c2010-05-17 17:08:21 +0100824EXPORT_SYMBOL_GPL(gnttab_max_grant_frames);
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700825
Andres Lagar-Cavillac5718982012-09-14 14:26:59 +0000826/* Handling of paged out grant targets (GNTST_eagain) */
827#define MAX_DELAY 256
828static inline void
829gnttab_retry_eagain_gop(unsigned int cmd, void *gop, int16_t *status,
830 const char *func)
831{
832 unsigned delay = 1;
833
834 do {
835 BUG_ON(HYPERVISOR_grant_table_op(cmd, gop, 1));
836 if (*status == GNTST_eagain)
837 msleep(delay++);
838 } while ((*status == GNTST_eagain) && (delay < MAX_DELAY));
839
840 if (delay >= MAX_DELAY) {
841 printk(KERN_ERR "%s: %s eagain grant\n", func, current->comm);
842 *status = GNTST_bad_page;
843 }
844}
845
846void gnttab_batch_map(struct gnttab_map_grant_ref *batch, unsigned count)
847{
848 struct gnttab_map_grant_ref *op;
849
850 if (HYPERVISOR_grant_table_op(GNTTABOP_map_grant_ref, batch, count))
851 BUG();
852 for (op = batch; op < batch + count; op++)
853 if (op->status == GNTST_eagain)
854 gnttab_retry_eagain_gop(GNTTABOP_map_grant_ref, op,
855 &op->status, __func__);
856}
857EXPORT_SYMBOL_GPL(gnttab_batch_map);
858
859void gnttab_batch_copy(struct gnttab_copy *batch, unsigned count)
860{
861 struct gnttab_copy *op;
862
863 if (HYPERVISOR_grant_table_op(GNTTABOP_copy, batch, count))
864 BUG();
865 for (op = batch; op < batch + count; op++)
866 if (op->status == GNTST_eagain)
867 gnttab_retry_eagain_gop(GNTTABOP_copy, op,
868 &op->status, __func__);
869}
870EXPORT_SYMBOL_GPL(gnttab_batch_copy);
871
Stefano Stabellini289b7772010-12-10 14:54:44 +0000872int gnttab_map_refs(struct gnttab_map_grant_ref *map_ops,
Annie Lic1237992011-11-22 09:59:56 +0800873 struct gnttab_map_grant_ref *kmap_ops,
874 struct page **pages, unsigned int count)
Stefano Stabellini289b7772010-12-10 14:54:44 +0000875{
876 int i, ret;
Stefano Stabellinif62805f2012-04-24 11:55:43 +0100877 bool lazy = false;
Stefano Stabellini289b7772010-12-10 14:54:44 +0000878 pte_t *pte;
879 unsigned long mfn;
880
881 ret = HYPERVISOR_grant_table_op(GNTTABOP_map_grant_ref, map_ops, count);
Jeremy Fitzhardinge87f1d402010-12-13 14:42:30 +0000882 if (ret)
883 return ret;
Stefano Stabellini289b7772010-12-10 14:54:44 +0000884
Andres Lagar-Cavillac5718982012-09-14 14:26:59 +0000885 /* Retry eagain maps */
886 for (i = 0; i < count; i++)
887 if (map_ops[i].status == GNTST_eagain)
888 gnttab_retry_eagain_gop(GNTTABOP_map_grant_ref, map_ops + i,
889 &map_ops[i].status, __func__);
890
Daniel De Graafaab8f112011-02-03 12:19:02 -0500891 if (xen_feature(XENFEAT_auto_translated_physmap))
892 return ret;
893
Stefano Stabellinif62805f2012-04-24 11:55:43 +0100894 if (!in_interrupt() && paravirt_get_lazy_mode() == PARAVIRT_LAZY_NONE) {
895 arch_enter_lazy_mmu_mode();
896 lazy = true;
897 }
898
Stefano Stabellini289b7772010-12-10 14:54:44 +0000899 for (i = 0; i < count; i++) {
Ian Campbelldc4972a2011-03-04 17:38:21 +0000900 /* Do not add to override if the map failed. */
901 if (map_ops[i].status)
902 continue;
903
Konrad Rzeszutek Wilkcf8d9162011-02-28 17:58:48 -0500904 if (map_ops[i].flags & GNTMAP_contains_pte) {
905 pte = (pte_t *) (mfn_to_virt(PFN_DOWN(map_ops[i].host_addr)) +
Stefano Stabellini289b7772010-12-10 14:54:44 +0000906 (map_ops[i].host_addr & ~PAGE_MASK));
Konrad Rzeszutek Wilkcf8d9162011-02-28 17:58:48 -0500907 mfn = pte_mfn(*pte);
908 } else {
Daniel De Graaf7d17e842011-12-14 15:12:11 -0500909 mfn = PFN_DOWN(map_ops[i].dev_bus_addr);
Konrad Rzeszutek Wilkcf8d9162011-02-28 17:58:48 -0500910 }
Daniel De Graaf7d17e842011-12-14 15:12:11 -0500911 ret = m2p_add_override(mfn, pages[i], kmap_ops ?
912 &kmap_ops[i] : NULL);
Jeremy Fitzhardinge87f1d402010-12-13 14:42:30 +0000913 if (ret)
914 return ret;
Stefano Stabellini289b7772010-12-10 14:54:44 +0000915 }
916
Stefano Stabellinif62805f2012-04-24 11:55:43 +0100917 if (lazy)
918 arch_leave_lazy_mmu_mode();
919
Stefano Stabellini289b7772010-12-10 14:54:44 +0000920 return ret;
921}
922EXPORT_SYMBOL_GPL(gnttab_map_refs);
923
924int gnttab_unmap_refs(struct gnttab_unmap_grant_ref *unmap_ops,
Stefano Stabellini2fc136e2012-09-12 12:44:30 +0100925 struct gnttab_map_grant_ref *kmap_ops,
926 struct page **pages, unsigned int count)
Stefano Stabellini289b7772010-12-10 14:54:44 +0000927{
928 int i, ret;
Stefano Stabellinif62805f2012-04-24 11:55:43 +0100929 bool lazy = false;
Stefano Stabellini289b7772010-12-10 14:54:44 +0000930
931 ret = HYPERVISOR_grant_table_op(GNTTABOP_unmap_grant_ref, unmap_ops, count);
Jeremy Fitzhardinge87f1d402010-12-13 14:42:30 +0000932 if (ret)
933 return ret;
934
Daniel De Graafaab8f112011-02-03 12:19:02 -0500935 if (xen_feature(XENFEAT_auto_translated_physmap))
936 return ret;
937
Stefano Stabellinif62805f2012-04-24 11:55:43 +0100938 if (!in_interrupt() && paravirt_get_lazy_mode() == PARAVIRT_LAZY_NONE) {
939 arch_enter_lazy_mmu_mode();
940 lazy = true;
941 }
942
Jeremy Fitzhardinge87f1d402010-12-13 14:42:30 +0000943 for (i = 0; i < count; i++) {
Stefano Stabellini2fc136e2012-09-12 12:44:30 +0100944 ret = m2p_remove_override(pages[i], kmap_ops ?
945 &kmap_ops[i] : NULL);
Jeremy Fitzhardinge87f1d402010-12-13 14:42:30 +0000946 if (ret)
947 return ret;
948 }
Stefano Stabellini289b7772010-12-10 14:54:44 +0000949
Stefano Stabellinif62805f2012-04-24 11:55:43 +0100950 if (lazy)
951 arch_leave_lazy_mmu_mode();
952
Stefano Stabellini289b7772010-12-10 14:54:44 +0000953 return ret;
954}
955EXPORT_SYMBOL_GPL(gnttab_unmap_refs);
956
Annie Li85ff6ac2011-11-22 09:59:21 +0800957static unsigned nr_status_frames(unsigned nr_grant_frames)
958{
Matt Wilsond0b4d642013-01-15 13:21:27 +0000959 BUG_ON(grefs_per_grant_frame == 0);
960 return (nr_grant_frames * grefs_per_grant_frame + SPP - 1) / SPP;
Annie Li85ff6ac2011-11-22 09:59:21 +0800961}
962
Ian Campbellef32f892012-10-17 09:39:14 +0100963static int gnttab_map_frames_v1(xen_pfn_t *frames, unsigned int nr_gframes)
Annie Li0f9f5a92011-11-22 09:58:06 +0800964{
965 int rc;
966
967 rc = arch_gnttab_map_shared(frames, nr_gframes,
968 gnttab_max_grant_frames(),
969 &gnttab_shared.addr);
970 BUG_ON(rc);
971
972 return 0;
973}
974
975static void gnttab_unmap_frames_v1(void)
976{
Annie Li85ff6ac2011-11-22 09:59:21 +0800977 arch_gnttab_unmap(gnttab_shared.addr, nr_grant_frames);
978}
979
Ian Campbellef32f892012-10-17 09:39:14 +0100980static int gnttab_map_frames_v2(xen_pfn_t *frames, unsigned int nr_gframes)
Annie Li85ff6ac2011-11-22 09:59:21 +0800981{
982 uint64_t *sframes;
983 unsigned int nr_sframes;
984 struct gnttab_get_status_frames getframes;
985 int rc;
986
987 nr_sframes = nr_status_frames(nr_gframes);
988
989 /* No need for kzalloc as it is initialized in following hypercall
990 * GNTTABOP_get_status_frames.
991 */
992 sframes = kmalloc(nr_sframes * sizeof(uint64_t), GFP_ATOMIC);
993 if (!sframes)
994 return -ENOMEM;
995
996 getframes.dom = DOMID_SELF;
997 getframes.nr_frames = nr_sframes;
998 set_xen_guest_handle(getframes.frame_list, sframes);
999
1000 rc = HYPERVISOR_grant_table_op(GNTTABOP_get_status_frames,
1001 &getframes, 1);
1002 if (rc == -ENOSYS) {
1003 kfree(sframes);
1004 return -ENOSYS;
1005 }
1006
1007 BUG_ON(rc || getframes.status);
1008
1009 rc = arch_gnttab_map_status(sframes, nr_sframes,
1010 nr_status_frames(gnttab_max_grant_frames()),
1011 &grstatus);
1012 BUG_ON(rc);
1013 kfree(sframes);
1014
1015 rc = arch_gnttab_map_shared(frames, nr_gframes,
1016 gnttab_max_grant_frames(),
1017 &gnttab_shared.addr);
1018 BUG_ON(rc);
1019
1020 return 0;
1021}
1022
1023static void gnttab_unmap_frames_v2(void)
1024{
1025 arch_gnttab_unmap(gnttab_shared.addr, nr_grant_frames);
1026 arch_gnttab_unmap(grstatus, nr_status_frames(nr_grant_frames));
Annie Li0f9f5a92011-11-22 09:58:06 +08001027}
1028
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -07001029static int gnttab_map(unsigned int start_idx, unsigned int end_idx)
1030{
1031 struct gnttab_setup_table setup;
Ian Campbellef32f892012-10-17 09:39:14 +01001032 xen_pfn_t *frames;
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -07001033 unsigned int nr_gframes = end_idx + 1;
1034 int rc;
1035
Stefano Stabellini183d03c2010-05-17 17:08:21 +01001036 if (xen_hvm_domain()) {
1037 struct xen_add_to_physmap xatp;
1038 unsigned int i = end_idx;
1039 rc = 0;
1040 /*
1041 * Loop backwards, so that the first hypercall has the largest
1042 * index, ensuring that the table will grow only once.
1043 */
1044 do {
1045 xatp.domid = DOMID_SELF;
1046 xatp.idx = i;
1047 xatp.space = XENMAPSPACE_grant_table;
1048 xatp.gpfn = (xen_hvm_resume_frames >> PAGE_SHIFT) + i;
1049 rc = HYPERVISOR_memory_op(XENMEM_add_to_physmap, &xatp);
1050 if (rc != 0) {
1051 printk(KERN_WARNING
1052 "grant table add_to_physmap failed, err=%d\n", rc);
1053 break;
1054 }
1055 } while (i-- > start_idx);
1056
1057 return rc;
1058 }
1059
Annie Li85ff6ac2011-11-22 09:59:21 +08001060 /* No need for kzalloc as it is initialized in following hypercall
1061 * GNTTABOP_setup_table.
1062 */
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -07001063 frames = kmalloc(nr_gframes * sizeof(unsigned long), GFP_ATOMIC);
1064 if (!frames)
1065 return -ENOMEM;
1066
1067 setup.dom = DOMID_SELF;
1068 setup.nr_frames = nr_gframes;
Isaku Yamahata87e27cf2008-04-02 10:53:52 -07001069 set_xen_guest_handle(setup.frame_list, frames);
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -07001070
1071 rc = HYPERVISOR_grant_table_op(GNTTABOP_setup_table, &setup, 1);
1072 if (rc == -ENOSYS) {
1073 kfree(frames);
1074 return -ENOSYS;
1075 }
1076
1077 BUG_ON(rc || setup.status);
1078
Annie Li0f9f5a92011-11-22 09:58:06 +08001079 rc = gnttab_interface->map_frames(frames, nr_gframes);
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -07001080
1081 kfree(frames);
1082
Annie Li0f9f5a92011-11-22 09:58:06 +08001083 return rc;
1084}
1085
1086static struct gnttab_ops gnttab_v1_ops = {
1087 .map_frames = gnttab_map_frames_v1,
1088 .unmap_frames = gnttab_unmap_frames_v1,
1089 .update_entry = gnttab_update_entry_v1,
1090 .end_foreign_access_ref = gnttab_end_foreign_access_ref_v1,
1091 .end_foreign_transfer_ref = gnttab_end_foreign_transfer_ref_v1,
1092 .query_foreign_access = gnttab_query_foreign_access_v1,
1093};
1094
Annie Li85ff6ac2011-11-22 09:59:21 +08001095static struct gnttab_ops gnttab_v2_ops = {
1096 .map_frames = gnttab_map_frames_v2,
1097 .unmap_frames = gnttab_unmap_frames_v2,
1098 .update_entry = gnttab_update_entry_v2,
1099 .end_foreign_access_ref = gnttab_end_foreign_access_ref_v2,
1100 .end_foreign_transfer_ref = gnttab_end_foreign_transfer_ref_v2,
1101 .query_foreign_access = gnttab_query_foreign_access_v2,
Annie Li66667542011-12-12 18:14:42 +08001102 .update_subpage_entry = gnttab_update_subpage_entry_v2,
Annie Li9438ce92011-12-12 18:15:07 +08001103 .update_trans_entry = gnttab_update_trans_entry_v2,
Annie Li85ff6ac2011-11-22 09:59:21 +08001104};
1105
Annie Li0f9f5a92011-11-22 09:58:06 +08001106static void gnttab_request_version(void)
1107{
Annie Li85ff6ac2011-11-22 09:59:21 +08001108 int rc;
1109 struct gnttab_set_version gsv;
1110
Konrad Rzeszutek Wilk69e8f432012-01-25 00:13:20 -05001111 if (xen_hvm_domain())
1112 gsv.version = 1;
1113 else
1114 gsv.version = 2;
Annie Li85ff6ac2011-11-22 09:59:21 +08001115 rc = HYPERVISOR_grant_table_op(GNTTABOP_set_version, &gsv, 1);
Konrad Rzeszutek Wilk69e8f432012-01-25 00:13:20 -05001116 if (rc == 0 && gsv.version == 2) {
Annie Li85ff6ac2011-11-22 09:59:21 +08001117 grant_table_version = 2;
Matt Wilsond0b4d642013-01-15 13:21:27 +00001118 grefs_per_grant_frame = PAGE_SIZE / sizeof(union grant_entry_v2);
Annie Li85ff6ac2011-11-22 09:59:21 +08001119 gnttab_interface = &gnttab_v2_ops;
1120 } else if (grant_table_version == 2) {
1121 /*
1122 * If we've already used version 2 features,
1123 * but then suddenly discover that they're not
1124 * available (e.g. migrating to an older
1125 * version of Xen), almost unbounded badness
1126 * can happen.
1127 */
1128 panic("we need grant tables version 2, but only version 1 is available");
1129 } else {
1130 grant_table_version = 1;
Matt Wilsond0b4d642013-01-15 13:21:27 +00001131 grefs_per_grant_frame = PAGE_SIZE / sizeof(struct grant_entry_v1);
Annie Li85ff6ac2011-11-22 09:59:21 +08001132 gnttab_interface = &gnttab_v1_ops;
1133 }
Annie Li0f9f5a92011-11-22 09:58:06 +08001134 printk(KERN_INFO "Grant tables using version %d layout.\n",
1135 grant_table_version);
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -07001136}
1137
Matt Wilsond0b4d642013-01-15 13:21:27 +00001138static int gnttab_setup(void)
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -07001139{
Stefano Stabellini183d03c2010-05-17 17:08:21 +01001140 unsigned int max_nr_gframes;
1141
1142 max_nr_gframes = gnttab_max_grant_frames();
1143 if (max_nr_gframes < nr_grant_frames)
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -07001144 return -ENOSYS;
Stefano Stabellini183d03c2010-05-17 17:08:21 +01001145
1146 if (xen_pv_domain())
1147 return gnttab_map(0, nr_grant_frames - 1);
1148
Annie Li0f9f5a92011-11-22 09:58:06 +08001149 if (gnttab_shared.addr == NULL) {
1150 gnttab_shared.addr = ioremap(xen_hvm_resume_frames,
1151 PAGE_SIZE * max_nr_gframes);
1152 if (gnttab_shared.addr == NULL) {
Stefano Stabellini183d03c2010-05-17 17:08:21 +01001153 printk(KERN_WARNING
1154 "Failed to ioremap gnttab share frames!");
1155 return -ENOMEM;
1156 }
1157 }
1158
1159 gnttab_map(0, nr_grant_frames - 1);
1160
1161 return 0;
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -07001162}
1163
Matt Wilsond0b4d642013-01-15 13:21:27 +00001164int gnttab_resume(void)
1165{
1166 gnttab_request_version();
1167 return gnttab_setup();
1168}
1169
Jeremy Fitzhardinge0e913982008-05-26 23:31:27 +01001170int gnttab_suspend(void)
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -07001171{
Annie Li0f9f5a92011-11-22 09:58:06 +08001172 gnttab_interface->unmap_frames();
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -07001173 return 0;
1174}
1175
1176static int gnttab_expand(unsigned int req_entries)
1177{
1178 int rc;
1179 unsigned int cur, extra;
1180
Matt Wilsond0b4d642013-01-15 13:21:27 +00001181 BUG_ON(grefs_per_grant_frame == 0);
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -07001182 cur = nr_grant_frames;
Matt Wilsond0b4d642013-01-15 13:21:27 +00001183 extra = ((req_entries + (grefs_per_grant_frame-1)) /
1184 grefs_per_grant_frame);
Stefano Stabellini183d03c2010-05-17 17:08:21 +01001185 if (cur + extra > gnttab_max_grant_frames())
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -07001186 return -ENOSPC;
1187
1188 rc = gnttab_map(cur, cur + extra - 1);
1189 if (rc == 0)
1190 rc = grow_gnttab_list(extra);
1191
1192 return rc;
1193}
1194
Stefano Stabellini183d03c2010-05-17 17:08:21 +01001195int gnttab_init(void)
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -07001196{
1197 int i;
Michael Abd-El-Malekbbc60c12008-04-04 02:33:48 -07001198 unsigned int max_nr_glist_frames, nr_glist_frames;
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -07001199 unsigned int nr_init_grefs;
Julia Lawall6b5e7d92012-04-15 11:27:12 +02001200 int ret;
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -07001201
Matt Wilsond0b4d642013-01-15 13:21:27 +00001202 gnttab_request_version();
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -07001203 nr_grant_frames = 1;
1204 boot_max_nr_grant_frames = __max_nr_grant_frames();
1205
1206 /* Determine the maximum number of frames required for the
1207 * grant reference free list on the current hypervisor.
1208 */
Matt Wilsond0b4d642013-01-15 13:21:27 +00001209 BUG_ON(grefs_per_grant_frame == 0);
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -07001210 max_nr_glist_frames = (boot_max_nr_grant_frames *
Matt Wilsond0b4d642013-01-15 13:21:27 +00001211 grefs_per_grant_frame / RPP);
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -07001212
1213 gnttab_list = kmalloc(max_nr_glist_frames * sizeof(grant_ref_t *),
1214 GFP_KERNEL);
1215 if (gnttab_list == NULL)
1216 return -ENOMEM;
1217
Matt Wilsond0b4d642013-01-15 13:21:27 +00001218 nr_glist_frames = (nr_grant_frames * grefs_per_grant_frame + RPP - 1) / RPP;
Michael Abd-El-Malekbbc60c12008-04-04 02:33:48 -07001219 for (i = 0; i < nr_glist_frames; i++) {
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -07001220 gnttab_list[i] = (grant_ref_t *)__get_free_page(GFP_KERNEL);
Julia Lawall6b5e7d92012-04-15 11:27:12 +02001221 if (gnttab_list[i] == NULL) {
1222 ret = -ENOMEM;
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -07001223 goto ini_nomem;
Julia Lawall6b5e7d92012-04-15 11:27:12 +02001224 }
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -07001225 }
1226
Matt Wilsond0b4d642013-01-15 13:21:27 +00001227 if (gnttab_setup() < 0) {
Julia Lawall6b5e7d92012-04-15 11:27:12 +02001228 ret = -ENODEV;
1229 goto ini_nomem;
1230 }
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -07001231
Matt Wilsond0b4d642013-01-15 13:21:27 +00001232 nr_init_grefs = nr_grant_frames * grefs_per_grant_frame;
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -07001233
1234 for (i = NR_RESERVED_ENTRIES; i < nr_init_grefs - 1; i++)
1235 gnttab_entry(i) = i + 1;
1236
1237 gnttab_entry(nr_init_grefs - 1) = GNTTAB_LIST_END;
1238 gnttab_free_count = nr_init_grefs - NR_RESERVED_ENTRIES;
1239 gnttab_free_head = NR_RESERVED_ENTRIES;
1240
1241 printk("Grant table initialized\n");
1242 return 0;
1243
1244 ini_nomem:
1245 for (i--; i >= 0; i--)
1246 free_page((unsigned long)gnttab_list[i]);
1247 kfree(gnttab_list);
Julia Lawall6b5e7d92012-04-15 11:27:12 +02001248 return ret;
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -07001249}
Stefano Stabellini183d03c2010-05-17 17:08:21 +01001250EXPORT_SYMBOL_GPL(gnttab_init);
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -07001251
Greg Kroah-Hartman345a5252012-12-21 13:00:00 -08001252static int __gnttab_init(void)
Stefano Stabellini183d03c2010-05-17 17:08:21 +01001253{
1254 /* Delay grant-table initialization in the PV on HVM case */
1255 if (xen_hvm_domain())
1256 return 0;
1257
1258 if (!xen_pv_domain())
1259 return -ENODEV;
1260
1261 return gnttab_init();
1262}
1263
1264core_initcall(__gnttab_init);