blob: b84e3ab839aa06650f98e39da40343f26644532d [file] [log] [blame]
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -07001/******************************************************************************
2 * grant_table.c
3 *
4 * Granting foreign access to our memory reservation.
5 *
6 * Copyright (c) 2005-2006, Christopher Clark
7 * Copyright (c) 2004-2005, K A Fraser
8 *
9 * This program is free software; you can redistribute it and/or
10 * modify it under the terms of the GNU General Public License version 2
11 * as published by the Free Software Foundation; or, when distributed
12 * separately from the Linux kernel or incorporated into other
13 * software packages, subject to the following license:
14 *
15 * Permission is hereby granted, free of charge, to any person obtaining a copy
16 * of this source file (the "Software"), to deal in the Software without
17 * restriction, including without limitation the rights to use, copy, modify,
18 * merge, publish, distribute, sublicense, and/or sell copies of the Software,
19 * and to permit persons to whom the Software is furnished to do so, subject to
20 * the following conditions:
21 *
22 * The above copyright notice and this permission notice shall be included in
23 * all copies or substantial portions of the Software.
24 *
25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
26 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
27 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
28 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
29 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
30 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
31 * IN THE SOFTWARE.
32 */
33
Joe Perches283c0972013-06-28 03:21:41 -070034#define pr_fmt(fmt) "xen:" KBUILD_MODNAME ": " fmt
35
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -070036#include <linux/module.h>
37#include <linux/sched.h>
38#include <linux/mm.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090039#include <linux/slab.h>
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -070040#include <linux/vmalloc.h>
41#include <linux/uaccess.h>
Stefano Stabellini183d03c2010-05-17 17:08:21 +010042#include <linux/io.h>
Andres Lagar-Cavillac5718982012-09-14 14:26:59 +000043#include <linux/delay.h>
Stefano Stabellinif62805f2012-04-24 11:55:43 +010044#include <linux/hardirq.h>
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -070045
Jeremy Fitzhardinge1ccbf532009-10-06 15:11:14 -070046#include <xen/xen.h>
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -070047#include <xen/interface/xen.h>
48#include <xen/page.h>
49#include <xen/grant_table.h>
Stefano Stabellini183d03c2010-05-17 17:08:21 +010050#include <xen/interface/memory.h>
Annie Li85ff6ac2011-11-22 09:59:21 +080051#include <xen/hvc-console.h>
Stefano Stabellini3d24bbd2013-10-25 10:41:44 +000052#include <xen/swiotlb-xen.h>
Jeremy Fitzhardingeecbf29c2008-12-16 12:37:07 -080053#include <asm/xen/hypercall.h>
Stefano Stabellini4d9310e2012-08-06 15:27:09 +010054#include <asm/xen/interface.h>
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -070055
56#include <asm/pgtable.h>
57#include <asm/sync_bitops.h>
58
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -070059/* External tools reserve first few grant table entries. */
60#define NR_RESERVED_ENTRIES 8
61#define GNTTAB_LIST_END 0xffffffff
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -070062
63static grant_ref_t **gnttab_list;
64static unsigned int nr_grant_frames;
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -070065static int gnttab_free_count;
66static grant_ref_t gnttab_free_head;
67static DEFINE_SPINLOCK(gnttab_list_lock);
Konrad Rzeszutek Wilkefaf30a2014-01-06 10:40:36 -050068struct grant_frames xen_auto_xlat_grant_frames;
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -070069
Annie Li0f9f5a92011-11-22 09:58:06 +080070static union {
71 struct grant_entry_v1 *v1;
Annie Li85ff6ac2011-11-22 09:59:21 +080072 union grant_entry_v2 *v2;
Annie Li0f9f5a92011-11-22 09:58:06 +080073 void *addr;
74} gnttab_shared;
75
76/*This is a structure of function pointers for grant table*/
77struct gnttab_ops {
78 /*
Annie Li9dbc71d2011-12-12 18:13:57 +080079 * Mapping a list of frames for storing grant entries. Frames parameter
80 * is used to store grant table address when grant table being setup,
81 * nr_gframes is the number of frames to map grant table. Returning
82 * GNTST_okay means success and negative value means failure.
Annie Li0f9f5a92011-11-22 09:58:06 +080083 */
Ian Campbellef32f892012-10-17 09:39:14 +010084 int (*map_frames)(xen_pfn_t *frames, unsigned int nr_gframes);
Annie Li0f9f5a92011-11-22 09:58:06 +080085 /*
86 * Release a list of frames which are mapped in map_frames for grant
87 * entry status.
88 */
89 void (*unmap_frames)(void);
90 /*
Annie Li9dbc71d2011-12-12 18:13:57 +080091 * Introducing a valid entry into the grant table, granting the frame of
92 * this grant entry to domain for accessing or transfering. Ref
93 * parameter is reference of this introduced grant entry, domid is id of
94 * granted domain, frame is the page frame to be granted, and flags is
95 * status of the grant entry to be updated.
Annie Li0f9f5a92011-11-22 09:58:06 +080096 */
Annie Li9dbc71d2011-12-12 18:13:57 +080097 void (*update_entry)(grant_ref_t ref, domid_t domid,
98 unsigned long frame, unsigned flags);
Annie Li0f9f5a92011-11-22 09:58:06 +080099 /*
Annie Li9dbc71d2011-12-12 18:13:57 +0800100 * Stop granting a grant entry to domain for accessing. Ref parameter is
101 * reference of a grant entry whose grant access will be stopped,
102 * readonly is not in use in this function. If the grant entry is
Annie Li0f9f5a92011-11-22 09:58:06 +0800103 * currently mapped for reading or writing, just return failure(==0)
104 * directly and don't tear down the grant access. Otherwise, stop grant
105 * access for this entry and return success(==1).
106 */
Annie Li9dbc71d2011-12-12 18:13:57 +0800107 int (*end_foreign_access_ref)(grant_ref_t ref, int readonly);
Annie Li0f9f5a92011-11-22 09:58:06 +0800108 /*
Annie Li9dbc71d2011-12-12 18:13:57 +0800109 * Stop granting a grant entry to domain for transfer. Ref parameter is
110 * reference of a grant entry whose grant transfer will be stopped. If
111 * tranfer has not started, just reclaim the grant entry and return
112 * failure(==0). Otherwise, wait for the transfer to complete and then
113 * return the frame.
Annie Li0f9f5a92011-11-22 09:58:06 +0800114 */
Annie Li9dbc71d2011-12-12 18:13:57 +0800115 unsigned long (*end_foreign_transfer_ref)(grant_ref_t ref);
Annie Li0f9f5a92011-11-22 09:58:06 +0800116 /*
Annie Li9dbc71d2011-12-12 18:13:57 +0800117 * Query the status of a grant entry. Ref parameter is reference of
Annie Li0f9f5a92011-11-22 09:58:06 +0800118 * queried grant entry, return value is the status of queried entry.
119 * Detailed status(writing/reading) can be gotten from the return value
120 * by bit operations.
121 */
Annie Li9dbc71d2011-12-12 18:13:57 +0800122 int (*query_foreign_access)(grant_ref_t ref);
Annie Li66667542011-12-12 18:14:42 +0800123 /*
124 * Grant a domain to access a range of bytes within the page referred by
125 * an available grant entry. Ref parameter is reference of a grant entry
126 * which will be sub-page accessed, domid is id of grantee domain, frame
127 * is frame address of subpage grant, flags is grant type and flag
128 * information, page_off is offset of the range of bytes, and length is
129 * length of bytes to be accessed.
130 */
131 void (*update_subpage_entry)(grant_ref_t ref, domid_t domid,
132 unsigned long frame, int flags,
133 unsigned page_off, unsigned length);
Annie Li9438ce92011-12-12 18:15:07 +0800134 /*
135 * Redirect an available grant entry on domain A to another grant
136 * reference of domain B, then allow domain C to use grant reference
137 * of domain B transitively. Ref parameter is an available grant entry
138 * reference on domain A, domid is id of domain C which accesses grant
139 * entry transitively, flags is grant type and flag information,
140 * trans_domid is id of domain B whose grant entry is finally accessed
141 * transitively, trans_gref is grant entry transitive reference of
142 * domain B.
143 */
144 void (*update_trans_entry)(grant_ref_t ref, domid_t domid, int flags,
145 domid_t trans_domid, grant_ref_t trans_gref);
Annie Li0f9f5a92011-11-22 09:58:06 +0800146};
147
148static struct gnttab_ops *gnttab_interface;
149
Annie Li85ff6ac2011-11-22 09:59:21 +0800150/*This reflects status of grant entries, so act as a global value*/
151static grant_status_t *grstatus;
152
Annie Li0f9f5a92011-11-22 09:58:06 +0800153static int grant_table_version;
Matt Wilsond0b4d642013-01-15 13:21:27 +0000154static int grefs_per_grant_frame;
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700155
156static struct gnttab_free_callback *gnttab_free_callback_list;
157
158static int gnttab_expand(unsigned int req_entries);
159
160#define RPP (PAGE_SIZE / sizeof(grant_ref_t))
Annie Li85ff6ac2011-11-22 09:59:21 +0800161#define SPP (PAGE_SIZE / sizeof(grant_status_t))
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700162
163static inline grant_ref_t *__gnttab_entry(grant_ref_t entry)
164{
165 return &gnttab_list[(entry) / RPP][(entry) % RPP];
166}
167/* This can be used as an l-value */
168#define gnttab_entry(entry) (*__gnttab_entry(entry))
169
170static int get_free_entries(unsigned count)
171{
172 unsigned long flags;
Konrad Rzeszutek Wilk272800d2011-07-22 14:00:06 -0400173 int ref, rc = 0;
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700174 grant_ref_t head;
175
176 spin_lock_irqsave(&gnttab_list_lock, flags);
177
178 if ((gnttab_free_count < count) &&
179 ((rc = gnttab_expand(count - gnttab_free_count)) < 0)) {
180 spin_unlock_irqrestore(&gnttab_list_lock, flags);
181 return rc;
182 }
183
184 ref = head = gnttab_free_head;
185 gnttab_free_count -= count;
186 while (count-- > 1)
187 head = gnttab_entry(head);
188 gnttab_free_head = gnttab_entry(head);
189 gnttab_entry(head) = GNTTAB_LIST_END;
190
191 spin_unlock_irqrestore(&gnttab_list_lock, flags);
192
193 return ref;
194}
195
196static void do_free_callbacks(void)
197{
198 struct gnttab_free_callback *callback, *next;
199
200 callback = gnttab_free_callback_list;
201 gnttab_free_callback_list = NULL;
202
203 while (callback != NULL) {
204 next = callback->next;
205 if (gnttab_free_count >= callback->count) {
206 callback->next = NULL;
207 callback->fn(callback->arg);
208 } else {
209 callback->next = gnttab_free_callback_list;
210 gnttab_free_callback_list = callback;
211 }
212 callback = next;
213 }
214}
215
216static inline void check_free_callbacks(void)
217{
218 if (unlikely(gnttab_free_callback_list))
219 do_free_callbacks();
220}
221
222static void put_free_entry(grant_ref_t ref)
223{
224 unsigned long flags;
225 spin_lock_irqsave(&gnttab_list_lock, flags);
226 gnttab_entry(ref) = gnttab_free_head;
227 gnttab_free_head = ref;
228 gnttab_free_count++;
229 check_free_callbacks();
230 spin_unlock_irqrestore(&gnttab_list_lock, flags);
231}
232
Annie Li0f9f5a92011-11-22 09:58:06 +0800233/*
Annie Li85ff6ac2011-11-22 09:59:21 +0800234 * Following applies to gnttab_update_entry_v1 and gnttab_update_entry_v2.
Annie Li0f9f5a92011-11-22 09:58:06 +0800235 * Introducing a valid entry into the grant table:
236 * 1. Write ent->domid.
237 * 2. Write ent->frame:
238 * GTF_permit_access: Frame to which access is permitted.
239 * GTF_accept_transfer: Pseudo-phys frame slot being filled by new
240 * frame, or zero if none.
241 * 3. Write memory barrier (WMB).
242 * 4. Write ent->flags, inc. valid type.
243 */
244static void gnttab_update_entry_v1(grant_ref_t ref, domid_t domid,
245 unsigned long frame, unsigned flags)
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700246{
Annie Li0f9f5a92011-11-22 09:58:06 +0800247 gnttab_shared.v1[ref].domid = domid;
248 gnttab_shared.v1[ref].frame = frame;
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700249 wmb();
Annie Li0f9f5a92011-11-22 09:58:06 +0800250 gnttab_shared.v1[ref].flags = flags;
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700251}
252
Annie Li85ff6ac2011-11-22 09:59:21 +0800253static void gnttab_update_entry_v2(grant_ref_t ref, domid_t domid,
254 unsigned long frame, unsigned flags)
255{
256 gnttab_shared.v2[ref].hdr.domid = domid;
257 gnttab_shared.v2[ref].full_page.frame = frame;
258 wmb();
259 gnttab_shared.v2[ref].hdr.flags = GTF_permit_access | flags;
260}
261
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700262/*
263 * Public grant-issuing interface functions
264 */
265void gnttab_grant_foreign_access_ref(grant_ref_t ref, domid_t domid,
266 unsigned long frame, int readonly)
267{
Annie Li0f9f5a92011-11-22 09:58:06 +0800268 gnttab_interface->update_entry(ref, domid, frame,
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700269 GTF_permit_access | (readonly ? GTF_readonly : 0));
270}
271EXPORT_SYMBOL_GPL(gnttab_grant_foreign_access_ref);
272
273int gnttab_grant_foreign_access(domid_t domid, unsigned long frame,
274 int readonly)
275{
276 int ref;
277
278 ref = get_free_entries(1);
279 if (unlikely(ref < 0))
280 return -ENOSPC;
281
282 gnttab_grant_foreign_access_ref(ref, domid, frame, readonly);
283
284 return ref;
285}
286EXPORT_SYMBOL_GPL(gnttab_grant_foreign_access);
287
Konrad Rzeszutek Wilkb8b0f552012-08-21 14:49:34 -0400288static void gnttab_update_subpage_entry_v2(grant_ref_t ref, domid_t domid,
289 unsigned long frame, int flags,
290 unsigned page_off, unsigned length)
Annie Li66667542011-12-12 18:14:42 +0800291{
292 gnttab_shared.v2[ref].sub_page.frame = frame;
293 gnttab_shared.v2[ref].sub_page.page_off = page_off;
294 gnttab_shared.v2[ref].sub_page.length = length;
295 gnttab_shared.v2[ref].hdr.domid = domid;
296 wmb();
297 gnttab_shared.v2[ref].hdr.flags =
298 GTF_permit_access | GTF_sub_page | flags;
299}
300
301int gnttab_grant_foreign_access_subpage_ref(grant_ref_t ref, domid_t domid,
302 unsigned long frame, int flags,
303 unsigned page_off,
304 unsigned length)
305{
306 if (flags & (GTF_accept_transfer | GTF_reading |
307 GTF_writing | GTF_transitive))
308 return -EPERM;
309
310 if (gnttab_interface->update_subpage_entry == NULL)
311 return -ENOSYS;
312
313 gnttab_interface->update_subpage_entry(ref, domid, frame, flags,
314 page_off, length);
315
316 return 0;
317}
318EXPORT_SYMBOL_GPL(gnttab_grant_foreign_access_subpage_ref);
319
320int gnttab_grant_foreign_access_subpage(domid_t domid, unsigned long frame,
321 int flags, unsigned page_off,
322 unsigned length)
323{
324 int ref, rc;
325
326 ref = get_free_entries(1);
327 if (unlikely(ref < 0))
328 return -ENOSPC;
329
330 rc = gnttab_grant_foreign_access_subpage_ref(ref, domid, frame, flags,
331 page_off, length);
332 if (rc < 0) {
333 put_free_entry(ref);
334 return rc;
335 }
336
337 return ref;
338}
339EXPORT_SYMBOL_GPL(gnttab_grant_foreign_access_subpage);
340
341bool gnttab_subpage_grants_available(void)
342{
343 return gnttab_interface->update_subpage_entry != NULL;
344}
345EXPORT_SYMBOL_GPL(gnttab_subpage_grants_available);
346
Konrad Rzeszutek Wilkb8b0f552012-08-21 14:49:34 -0400347static void gnttab_update_trans_entry_v2(grant_ref_t ref, domid_t domid,
348 int flags, domid_t trans_domid,
349 grant_ref_t trans_gref)
Annie Li9438ce92011-12-12 18:15:07 +0800350{
351 gnttab_shared.v2[ref].transitive.trans_domid = trans_domid;
352 gnttab_shared.v2[ref].transitive.gref = trans_gref;
353 gnttab_shared.v2[ref].hdr.domid = domid;
354 wmb();
355 gnttab_shared.v2[ref].hdr.flags =
356 GTF_permit_access | GTF_transitive | flags;
357}
358
359int gnttab_grant_foreign_access_trans_ref(grant_ref_t ref, domid_t domid,
360 int flags, domid_t trans_domid,
361 grant_ref_t trans_gref)
362{
363 if (flags & (GTF_accept_transfer | GTF_reading |
364 GTF_writing | GTF_sub_page))
365 return -EPERM;
366
367 if (gnttab_interface->update_trans_entry == NULL)
368 return -ENOSYS;
369
370 gnttab_interface->update_trans_entry(ref, domid, flags, trans_domid,
371 trans_gref);
372
373 return 0;
374}
375EXPORT_SYMBOL_GPL(gnttab_grant_foreign_access_trans_ref);
376
377int gnttab_grant_foreign_access_trans(domid_t domid, int flags,
378 domid_t trans_domid,
379 grant_ref_t trans_gref)
380{
381 int ref, rc;
382
383 ref = get_free_entries(1);
384 if (unlikely(ref < 0))
385 return -ENOSPC;
386
387 rc = gnttab_grant_foreign_access_trans_ref(ref, domid, flags,
388 trans_domid, trans_gref);
389 if (rc < 0) {
390 put_free_entry(ref);
391 return rc;
392 }
393
394 return ref;
395}
396EXPORT_SYMBOL_GPL(gnttab_grant_foreign_access_trans);
397
398bool gnttab_trans_grants_available(void)
399{
400 return gnttab_interface->update_trans_entry != NULL;
401}
402EXPORT_SYMBOL_GPL(gnttab_trans_grants_available);
403
Annie Li0f9f5a92011-11-22 09:58:06 +0800404static int gnttab_query_foreign_access_v1(grant_ref_t ref)
405{
406 return gnttab_shared.v1[ref].flags & (GTF_reading|GTF_writing);
407}
408
Annie Li85ff6ac2011-11-22 09:59:21 +0800409static int gnttab_query_foreign_access_v2(grant_ref_t ref)
410{
411 return grstatus[ref] & (GTF_reading|GTF_writing);
412}
413
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700414int gnttab_query_foreign_access(grant_ref_t ref)
415{
Annie Li0f9f5a92011-11-22 09:58:06 +0800416 return gnttab_interface->query_foreign_access(ref);
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700417}
418EXPORT_SYMBOL_GPL(gnttab_query_foreign_access);
419
Annie Li0f9f5a92011-11-22 09:58:06 +0800420static int gnttab_end_foreign_access_ref_v1(grant_ref_t ref, int readonly)
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700421{
422 u16 flags, nflags;
Annie Lib1e495b2011-11-22 09:58:47 +0800423 u16 *pflags;
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700424
Annie Lib1e495b2011-11-22 09:58:47 +0800425 pflags = &gnttab_shared.v1[ref].flags;
426 nflags = *pflags;
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700427 do {
428 flags = nflags;
Jan Beulich569ca5b2012-04-05 16:10:07 +0100429 if (flags & (GTF_reading|GTF_writing))
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700430 return 0;
Annie Lib1e495b2011-11-22 09:58:47 +0800431 } while ((nflags = sync_cmpxchg(pflags, flags, 0)) != flags);
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700432
433 return 1;
434}
Annie Li0f9f5a92011-11-22 09:58:06 +0800435
Annie Li85ff6ac2011-11-22 09:59:21 +0800436static int gnttab_end_foreign_access_ref_v2(grant_ref_t ref, int readonly)
437{
438 gnttab_shared.v2[ref].hdr.flags = 0;
439 mb();
440 if (grstatus[ref] & (GTF_reading|GTF_writing)) {
441 return 0;
442 } else {
443 /* The read of grstatus needs to have acquire
444 semantics. On x86, reads already have
445 that, and we just need to protect against
446 compiler reorderings. On other
447 architectures we may need a full
448 barrier. */
449#ifdef CONFIG_X86
450 barrier();
451#else
452 mb();
453#endif
454 }
455
456 return 1;
457}
458
Jan Beulich569ca5b2012-04-05 16:10:07 +0100459static inline int _gnttab_end_foreign_access_ref(grant_ref_t ref, int readonly)
Annie Li0f9f5a92011-11-22 09:58:06 +0800460{
461 return gnttab_interface->end_foreign_access_ref(ref, readonly);
462}
Jan Beulich569ca5b2012-04-05 16:10:07 +0100463
464int gnttab_end_foreign_access_ref(grant_ref_t ref, int readonly)
465{
466 if (_gnttab_end_foreign_access_ref(ref, readonly))
467 return 1;
468 pr_warn("WARNING: g.e. %#x still in use!\n", ref);
469 return 0;
470}
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700471EXPORT_SYMBOL_GPL(gnttab_end_foreign_access_ref);
472
Jan Beulich569ca5b2012-04-05 16:10:07 +0100473struct deferred_entry {
474 struct list_head list;
475 grant_ref_t ref;
476 bool ro;
477 uint16_t warn_delay;
478 struct page *page;
479};
480static LIST_HEAD(deferred_list);
481static void gnttab_handle_deferred(unsigned long);
482static DEFINE_TIMER(deferred_timer, gnttab_handle_deferred, 0, 0);
483
484static void gnttab_handle_deferred(unsigned long unused)
485{
486 unsigned int nr = 10;
487 struct deferred_entry *first = NULL;
488 unsigned long flags;
489
490 spin_lock_irqsave(&gnttab_list_lock, flags);
491 while (nr--) {
492 struct deferred_entry *entry
493 = list_first_entry(&deferred_list,
494 struct deferred_entry, list);
495
496 if (entry == first)
497 break;
498 list_del(&entry->list);
499 spin_unlock_irqrestore(&gnttab_list_lock, flags);
500 if (_gnttab_end_foreign_access_ref(entry->ref, entry->ro)) {
501 put_free_entry(entry->ref);
502 if (entry->page) {
503 pr_debug("freeing g.e. %#x (pfn %#lx)\n",
504 entry->ref, page_to_pfn(entry->page));
505 __free_page(entry->page);
506 } else
507 pr_info("freeing g.e. %#x\n", entry->ref);
508 kfree(entry);
509 entry = NULL;
510 } else {
511 if (!--entry->warn_delay)
Joe Perches283c0972013-06-28 03:21:41 -0700512 pr_info("g.e. %#x still pending\n", entry->ref);
Jan Beulich569ca5b2012-04-05 16:10:07 +0100513 if (!first)
514 first = entry;
515 }
516 spin_lock_irqsave(&gnttab_list_lock, flags);
517 if (entry)
518 list_add_tail(&entry->list, &deferred_list);
519 else if (list_empty(&deferred_list))
520 break;
521 }
522 if (!list_empty(&deferred_list) && !timer_pending(&deferred_timer)) {
523 deferred_timer.expires = jiffies + HZ;
524 add_timer(&deferred_timer);
525 }
526 spin_unlock_irqrestore(&gnttab_list_lock, flags);
527}
528
529static void gnttab_add_deferred(grant_ref_t ref, bool readonly,
530 struct page *page)
531{
532 struct deferred_entry *entry = kmalloc(sizeof(*entry), GFP_ATOMIC);
533 const char *what = KERN_WARNING "leaking";
534
535 if (entry) {
536 unsigned long flags;
537
538 entry->ref = ref;
539 entry->ro = readonly;
540 entry->page = page;
541 entry->warn_delay = 60;
542 spin_lock_irqsave(&gnttab_list_lock, flags);
543 list_add_tail(&entry->list, &deferred_list);
544 if (!timer_pending(&deferred_timer)) {
545 deferred_timer.expires = jiffies + HZ;
546 add_timer(&deferred_timer);
547 }
548 spin_unlock_irqrestore(&gnttab_list_lock, flags);
549 what = KERN_DEBUG "deferring";
550 }
551 printk("%s g.e. %#x (pfn %#lx)\n",
552 what, ref, page ? page_to_pfn(page) : -1);
553}
554
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700555void gnttab_end_foreign_access(grant_ref_t ref, int readonly,
556 unsigned long page)
557{
558 if (gnttab_end_foreign_access_ref(ref, readonly)) {
559 put_free_entry(ref);
560 if (page != 0)
561 free_page(page);
Jan Beulich569ca5b2012-04-05 16:10:07 +0100562 } else
563 gnttab_add_deferred(ref, readonly,
564 page ? virt_to_page(page) : NULL);
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700565}
566EXPORT_SYMBOL_GPL(gnttab_end_foreign_access);
567
568int gnttab_grant_foreign_transfer(domid_t domid, unsigned long pfn)
569{
570 int ref;
571
572 ref = get_free_entries(1);
573 if (unlikely(ref < 0))
574 return -ENOSPC;
575 gnttab_grant_foreign_transfer_ref(ref, domid, pfn);
576
577 return ref;
578}
579EXPORT_SYMBOL_GPL(gnttab_grant_foreign_transfer);
580
581void gnttab_grant_foreign_transfer_ref(grant_ref_t ref, domid_t domid,
582 unsigned long pfn)
583{
Annie Li0f9f5a92011-11-22 09:58:06 +0800584 gnttab_interface->update_entry(ref, domid, pfn, GTF_accept_transfer);
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700585}
586EXPORT_SYMBOL_GPL(gnttab_grant_foreign_transfer_ref);
587
Annie Li0f9f5a92011-11-22 09:58:06 +0800588static unsigned long gnttab_end_foreign_transfer_ref_v1(grant_ref_t ref)
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700589{
590 unsigned long frame;
591 u16 flags;
Annie Lib1e495b2011-11-22 09:58:47 +0800592 u16 *pflags;
593
594 pflags = &gnttab_shared.v1[ref].flags;
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700595
596 /*
597 * If a transfer is not even yet started, try to reclaim the grant
598 * reference and return failure (== 0).
599 */
Annie Lib1e495b2011-11-22 09:58:47 +0800600 while (!((flags = *pflags) & GTF_transfer_committed)) {
601 if (sync_cmpxchg(pflags, flags, 0) == flags)
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700602 return 0;
603 cpu_relax();
604 }
605
606 /* If a transfer is in progress then wait until it is completed. */
607 while (!(flags & GTF_transfer_completed)) {
Annie Lib1e495b2011-11-22 09:58:47 +0800608 flags = *pflags;
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700609 cpu_relax();
610 }
611
612 rmb(); /* Read the frame number /after/ reading completion status. */
Annie Li0f9f5a92011-11-22 09:58:06 +0800613 frame = gnttab_shared.v1[ref].frame;
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700614 BUG_ON(frame == 0);
615
616 return frame;
617}
Annie Li0f9f5a92011-11-22 09:58:06 +0800618
Annie Li85ff6ac2011-11-22 09:59:21 +0800619static unsigned long gnttab_end_foreign_transfer_ref_v2(grant_ref_t ref)
620{
621 unsigned long frame;
622 u16 flags;
623 u16 *pflags;
624
625 pflags = &gnttab_shared.v2[ref].hdr.flags;
626
627 /*
628 * If a transfer is not even yet started, try to reclaim the grant
629 * reference and return failure (== 0).
630 */
631 while (!((flags = *pflags) & GTF_transfer_committed)) {
632 if (sync_cmpxchg(pflags, flags, 0) == flags)
633 return 0;
634 cpu_relax();
635 }
636
637 /* If a transfer is in progress then wait until it is completed. */
638 while (!(flags & GTF_transfer_completed)) {
639 flags = *pflags;
640 cpu_relax();
641 }
642
643 rmb(); /* Read the frame number /after/ reading completion status. */
644 frame = gnttab_shared.v2[ref].full_page.frame;
645 BUG_ON(frame == 0);
646
647 return frame;
648}
649
Annie Li0f9f5a92011-11-22 09:58:06 +0800650unsigned long gnttab_end_foreign_transfer_ref(grant_ref_t ref)
651{
652 return gnttab_interface->end_foreign_transfer_ref(ref);
653}
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700654EXPORT_SYMBOL_GPL(gnttab_end_foreign_transfer_ref);
655
656unsigned long gnttab_end_foreign_transfer(grant_ref_t ref)
657{
658 unsigned long frame = gnttab_end_foreign_transfer_ref(ref);
659 put_free_entry(ref);
660 return frame;
661}
662EXPORT_SYMBOL_GPL(gnttab_end_foreign_transfer);
663
664void gnttab_free_grant_reference(grant_ref_t ref)
665{
666 put_free_entry(ref);
667}
668EXPORT_SYMBOL_GPL(gnttab_free_grant_reference);
669
670void gnttab_free_grant_references(grant_ref_t head)
671{
672 grant_ref_t ref;
673 unsigned long flags;
674 int count = 1;
675 if (head == GNTTAB_LIST_END)
676 return;
677 spin_lock_irqsave(&gnttab_list_lock, flags);
678 ref = head;
679 while (gnttab_entry(ref) != GNTTAB_LIST_END) {
680 ref = gnttab_entry(ref);
681 count++;
682 }
683 gnttab_entry(ref) = gnttab_free_head;
684 gnttab_free_head = head;
685 gnttab_free_count += count;
686 check_free_callbacks();
687 spin_unlock_irqrestore(&gnttab_list_lock, flags);
688}
689EXPORT_SYMBOL_GPL(gnttab_free_grant_references);
690
691int gnttab_alloc_grant_references(u16 count, grant_ref_t *head)
692{
693 int h = get_free_entries(count);
694
695 if (h < 0)
696 return -ENOSPC;
697
698 *head = h;
699
700 return 0;
701}
702EXPORT_SYMBOL_GPL(gnttab_alloc_grant_references);
703
704int gnttab_empty_grant_references(const grant_ref_t *private_head)
705{
706 return (*private_head == GNTTAB_LIST_END);
707}
708EXPORT_SYMBOL_GPL(gnttab_empty_grant_references);
709
710int gnttab_claim_grant_reference(grant_ref_t *private_head)
711{
712 grant_ref_t g = *private_head;
713 if (unlikely(g == GNTTAB_LIST_END))
714 return -ENOSPC;
715 *private_head = gnttab_entry(g);
716 return g;
717}
718EXPORT_SYMBOL_GPL(gnttab_claim_grant_reference);
719
720void gnttab_release_grant_reference(grant_ref_t *private_head,
721 grant_ref_t release)
722{
723 gnttab_entry(release) = *private_head;
724 *private_head = release;
725}
726EXPORT_SYMBOL_GPL(gnttab_release_grant_reference);
727
728void gnttab_request_free_callback(struct gnttab_free_callback *callback,
729 void (*fn)(void *), void *arg, u16 count)
730{
731 unsigned long flags;
Roger Pau Monne5f338d92013-07-31 17:00:42 +0200732 struct gnttab_free_callback *cb;
733
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700734 spin_lock_irqsave(&gnttab_list_lock, flags);
Roger Pau Monne5f338d92013-07-31 17:00:42 +0200735
736 /* Check if the callback is already on the list */
737 cb = gnttab_free_callback_list;
738 while (cb) {
739 if (cb == callback)
740 goto out;
741 cb = cb->next;
742 }
743
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700744 callback->fn = fn;
745 callback->arg = arg;
746 callback->count = count;
747 callback->next = gnttab_free_callback_list;
748 gnttab_free_callback_list = callback;
749 check_free_callbacks();
750out:
751 spin_unlock_irqrestore(&gnttab_list_lock, flags);
752}
753EXPORT_SYMBOL_GPL(gnttab_request_free_callback);
754
755void gnttab_cancel_free_callback(struct gnttab_free_callback *callback)
756{
757 struct gnttab_free_callback **pcb;
758 unsigned long flags;
759
760 spin_lock_irqsave(&gnttab_list_lock, flags);
761 for (pcb = &gnttab_free_callback_list; *pcb; pcb = &(*pcb)->next) {
762 if (*pcb == callback) {
763 *pcb = callback->next;
764 break;
765 }
766 }
767 spin_unlock_irqrestore(&gnttab_list_lock, flags);
768}
769EXPORT_SYMBOL_GPL(gnttab_cancel_free_callback);
770
771static int grow_gnttab_list(unsigned int more_frames)
772{
773 unsigned int new_nr_grant_frames, extra_entries, i;
Michael Abd-El-Malekbbc60c12008-04-04 02:33:48 -0700774 unsigned int nr_glist_frames, new_nr_glist_frames;
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700775
Matt Wilsond0b4d642013-01-15 13:21:27 +0000776 BUG_ON(grefs_per_grant_frame == 0);
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700777
Matt Wilsond0b4d642013-01-15 13:21:27 +0000778 new_nr_grant_frames = nr_grant_frames + more_frames;
779 extra_entries = more_frames * grefs_per_grant_frame;
780
781 nr_glist_frames = (nr_grant_frames * grefs_per_grant_frame + RPP - 1) / RPP;
Michael Abd-El-Malekbbc60c12008-04-04 02:33:48 -0700782 new_nr_glist_frames =
Matt Wilsond0b4d642013-01-15 13:21:27 +0000783 (new_nr_grant_frames * grefs_per_grant_frame + RPP - 1) / RPP;
Michael Abd-El-Malekbbc60c12008-04-04 02:33:48 -0700784 for (i = nr_glist_frames; i < new_nr_glist_frames; i++) {
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700785 gnttab_list[i] = (grant_ref_t *)__get_free_page(GFP_ATOMIC);
786 if (!gnttab_list[i])
787 goto grow_nomem;
788 }
789
790
Matt Wilsond0b4d642013-01-15 13:21:27 +0000791 for (i = grefs_per_grant_frame * nr_grant_frames;
792 i < grefs_per_grant_frame * new_nr_grant_frames - 1; i++)
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700793 gnttab_entry(i) = i + 1;
794
795 gnttab_entry(i) = gnttab_free_head;
Matt Wilsond0b4d642013-01-15 13:21:27 +0000796 gnttab_free_head = grefs_per_grant_frame * nr_grant_frames;
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700797 gnttab_free_count += extra_entries;
798
799 nr_grant_frames = new_nr_grant_frames;
800
801 check_free_callbacks();
802
803 return 0;
804
805grow_nomem:
Michael Abd-El-Malekbbc60c12008-04-04 02:33:48 -0700806 for ( ; i >= nr_glist_frames; i--)
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700807 free_page((unsigned long) gnttab_list[i]);
808 return -ENOMEM;
809}
810
811static unsigned int __max_nr_grant_frames(void)
812{
813 struct gnttab_query_size query;
814 int rc;
815
816 query.dom = DOMID_SELF;
817
818 rc = HYPERVISOR_grant_table_op(GNTTABOP_query_size, &query, 1);
819 if ((rc < 0) || (query.status != GNTST_okay))
820 return 4; /* Legacy max supported number of frames */
821
822 return query.max_nr_frames;
823}
824
Stefano Stabellini183d03c2010-05-17 17:08:21 +0100825unsigned int gnttab_max_grant_frames(void)
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700826{
827 unsigned int xen_max = __max_nr_grant_frames();
Konrad Rzeszutek Wilk7f256022013-12-31 15:55:39 -0500828 static unsigned int boot_max_nr_grant_frames;
829
830 /* First time, initialize it properly. */
831 if (!boot_max_nr_grant_frames)
832 boot_max_nr_grant_frames = __max_nr_grant_frames();
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700833
834 if (xen_max > boot_max_nr_grant_frames)
835 return boot_max_nr_grant_frames;
836 return xen_max;
837}
Stefano Stabellini183d03c2010-05-17 17:08:21 +0100838EXPORT_SYMBOL_GPL(gnttab_max_grant_frames);
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -0700839
Julien Grall47c54202014-01-30 12:56:34 +0000840int gnttab_setup_auto_xlat_frames(phys_addr_t addr)
Konrad Rzeszutek Wilkefaf30a2014-01-06 10:40:36 -0500841{
842 xen_pfn_t *pfn;
843 unsigned int max_nr_gframes = __max_nr_grant_frames();
844 unsigned int i;
845 void *vaddr;
846
847 if (xen_auto_xlat_grant_frames.count)
848 return -EINVAL;
849
850 vaddr = xen_remap(addr, PAGE_SIZE * max_nr_gframes);
851 if (vaddr == NULL) {
Julien Grall47c54202014-01-30 12:56:34 +0000852 pr_warn("Failed to ioremap gnttab share frames (addr=%pa)!\n",
853 &addr);
Konrad Rzeszutek Wilkefaf30a2014-01-06 10:40:36 -0500854 return -ENOMEM;
855 }
856 pfn = kcalloc(max_nr_gframes, sizeof(pfn[0]), GFP_KERNEL);
857 if (!pfn) {
858 xen_unmap(vaddr);
859 return -ENOMEM;
860 }
861 for (i = 0; i < max_nr_gframes; i++)
862 pfn[i] = PFN_DOWN(addr) + i;
863
864 xen_auto_xlat_grant_frames.vaddr = vaddr;
865 xen_auto_xlat_grant_frames.pfn = pfn;
866 xen_auto_xlat_grant_frames.count = max_nr_gframes;
867
868 return 0;
869}
870EXPORT_SYMBOL_GPL(gnttab_setup_auto_xlat_frames);
871
872void gnttab_free_auto_xlat_frames(void)
873{
874 if (!xen_auto_xlat_grant_frames.count)
875 return;
876 kfree(xen_auto_xlat_grant_frames.pfn);
877 xen_unmap(xen_auto_xlat_grant_frames.vaddr);
878
879 xen_auto_xlat_grant_frames.pfn = NULL;
880 xen_auto_xlat_grant_frames.count = 0;
881 xen_auto_xlat_grant_frames.vaddr = NULL;
882}
883EXPORT_SYMBOL_GPL(gnttab_free_auto_xlat_frames);
884
Andres Lagar-Cavillac5718982012-09-14 14:26:59 +0000885/* Handling of paged out grant targets (GNTST_eagain) */
886#define MAX_DELAY 256
887static inline void
888gnttab_retry_eagain_gop(unsigned int cmd, void *gop, int16_t *status,
889 const char *func)
890{
891 unsigned delay = 1;
892
893 do {
894 BUG_ON(HYPERVISOR_grant_table_op(cmd, gop, 1));
895 if (*status == GNTST_eagain)
896 msleep(delay++);
897 } while ((*status == GNTST_eagain) && (delay < MAX_DELAY));
898
899 if (delay >= MAX_DELAY) {
Joe Perches283c0972013-06-28 03:21:41 -0700900 pr_err("%s: %s eagain grant\n", func, current->comm);
Andres Lagar-Cavillac5718982012-09-14 14:26:59 +0000901 *status = GNTST_bad_page;
902 }
903}
904
905void gnttab_batch_map(struct gnttab_map_grant_ref *batch, unsigned count)
906{
907 struct gnttab_map_grant_ref *op;
908
909 if (HYPERVISOR_grant_table_op(GNTTABOP_map_grant_ref, batch, count))
910 BUG();
911 for (op = batch; op < batch + count; op++)
912 if (op->status == GNTST_eagain)
913 gnttab_retry_eagain_gop(GNTTABOP_map_grant_ref, op,
914 &op->status, __func__);
915}
916EXPORT_SYMBOL_GPL(gnttab_batch_map);
917
918void gnttab_batch_copy(struct gnttab_copy *batch, unsigned count)
919{
920 struct gnttab_copy *op;
921
922 if (HYPERVISOR_grant_table_op(GNTTABOP_copy, batch, count))
923 BUG();
924 for (op = batch; op < batch + count; op++)
925 if (op->status == GNTST_eagain)
926 gnttab_retry_eagain_gop(GNTTABOP_copy, op,
927 &op->status, __func__);
928}
929EXPORT_SYMBOL_GPL(gnttab_batch_copy);
930
Konrad Rzeszutek Wilke85fc982014-02-03 06:43:59 -0500931int gnttab_map_refs(struct gnttab_map_grant_ref *map_ops,
Annie Lic1237992011-11-22 09:59:56 +0800932 struct gnttab_map_grant_ref *kmap_ops,
Konrad Rzeszutek Wilke85fc982014-02-03 06:43:59 -0500933 struct page **pages, unsigned int count)
Stefano Stabellini289b7772010-12-10 14:54:44 +0000934{
935 int i, ret;
Stefano Stabellinif62805f2012-04-24 11:55:43 +0100936 bool lazy = false;
Stefano Stabellini289b7772010-12-10 14:54:44 +0000937 pte_t *pte;
Konrad Rzeszutek Wilke85fc982014-02-03 06:43:59 -0500938 unsigned long mfn;
Stefano Stabellini289b7772010-12-10 14:54:44 +0000939
940 ret = HYPERVISOR_grant_table_op(GNTTABOP_map_grant_ref, map_ops, count);
Jeremy Fitzhardinge87f1d402010-12-13 14:42:30 +0000941 if (ret)
942 return ret;
Stefano Stabellini289b7772010-12-10 14:54:44 +0000943
Andres Lagar-Cavillac5718982012-09-14 14:26:59 +0000944 /* Retry eagain maps */
945 for (i = 0; i < count; i++)
946 if (map_ops[i].status == GNTST_eagain)
947 gnttab_retry_eagain_gop(GNTTABOP_map_grant_ref, map_ops + i,
948 &map_ops[i].status, __func__);
949
Stefano Stabellini3d24bbd2013-10-25 10:41:44 +0000950 /* this is basically a nop on x86 */
951 if (xen_feature(XENFEAT_auto_translated_physmap)) {
952 for (i = 0; i < count; i++) {
953 if (map_ops[i].status)
954 continue;
955 set_phys_to_machine(map_ops[i].host_addr >> PAGE_SHIFT,
956 map_ops[i].dev_bus_addr >> PAGE_SHIFT);
957 }
Konrad Rzeszutek Wilke85fc982014-02-03 06:43:59 -0500958 return ret;
Stefano Stabellini3d24bbd2013-10-25 10:41:44 +0000959 }
Daniel De Graafaab8f112011-02-03 12:19:02 -0500960
Konrad Rzeszutek Wilke85fc982014-02-03 06:43:59 -0500961 if (!in_interrupt() && paravirt_get_lazy_mode() == PARAVIRT_LAZY_NONE) {
Stefano Stabellinif62805f2012-04-24 11:55:43 +0100962 arch_enter_lazy_mmu_mode();
963 lazy = true;
964 }
965
Stefano Stabellini289b7772010-12-10 14:54:44 +0000966 for (i = 0; i < count; i++) {
Ian Campbelldc4972a2011-03-04 17:38:21 +0000967 /* Do not add to override if the map failed. */
968 if (map_ops[i].status)
969 continue;
970
Konrad Rzeszutek Wilkcf8d9162011-02-28 17:58:48 -0500971 if (map_ops[i].flags & GNTMAP_contains_pte) {
972 pte = (pte_t *) (mfn_to_virt(PFN_DOWN(map_ops[i].host_addr)) +
Stefano Stabellini289b7772010-12-10 14:54:44 +0000973 (map_ops[i].host_addr & ~PAGE_MASK));
Konrad Rzeszutek Wilkcf8d9162011-02-28 17:58:48 -0500974 mfn = pte_mfn(*pte);
975 } else {
Daniel De Graaf7d17e842011-12-14 15:12:11 -0500976 mfn = PFN_DOWN(map_ops[i].dev_bus_addr);
Konrad Rzeszutek Wilkcf8d9162011-02-28 17:58:48 -0500977 }
Konrad Rzeszutek Wilke85fc982014-02-03 06:43:59 -0500978 ret = m2p_add_override(mfn, pages[i], kmap_ops ?
979 &kmap_ops[i] : NULL);
Jeremy Fitzhardinge87f1d402010-12-13 14:42:30 +0000980 if (ret)
Matt Wilson14883a72013-11-20 12:11:35 -0800981 goto out;
Stefano Stabellini289b7772010-12-10 14:54:44 +0000982 }
983
Matt Wilson14883a72013-11-20 12:11:35 -0800984 out:
Stefano Stabellinif62805f2012-04-24 11:55:43 +0100985 if (lazy)
986 arch_leave_lazy_mmu_mode();
987
Stefano Stabellini289b7772010-12-10 14:54:44 +0000988 return ret;
989}
990EXPORT_SYMBOL_GPL(gnttab_map_refs);
991
Konrad Rzeszutek Wilke85fc982014-02-03 06:43:59 -0500992int gnttab_unmap_refs(struct gnttab_unmap_grant_ref *unmap_ops,
Stefano Stabellini2fc136e2012-09-12 12:44:30 +0100993 struct gnttab_map_grant_ref *kmap_ops,
Konrad Rzeszutek Wilke85fc982014-02-03 06:43:59 -0500994 struct page **pages, unsigned int count)
Stefano Stabellini289b7772010-12-10 14:54:44 +0000995{
996 int i, ret;
Stefano Stabellinif62805f2012-04-24 11:55:43 +0100997 bool lazy = false;
Stefano Stabellini289b7772010-12-10 14:54:44 +0000998
999 ret = HYPERVISOR_grant_table_op(GNTTABOP_unmap_grant_ref, unmap_ops, count);
Jeremy Fitzhardinge87f1d402010-12-13 14:42:30 +00001000 if (ret)
1001 return ret;
1002
Stefano Stabellini3d24bbd2013-10-25 10:41:44 +00001003 /* this is basically a nop on x86 */
1004 if (xen_feature(XENFEAT_auto_translated_physmap)) {
1005 for (i = 0; i < count; i++) {
1006 set_phys_to_machine(unmap_ops[i].host_addr >> PAGE_SHIFT,
1007 INVALID_P2M_ENTRY);
1008 }
Konrad Rzeszutek Wilke85fc982014-02-03 06:43:59 -05001009 return ret;
Stefano Stabellini3d24bbd2013-10-25 10:41:44 +00001010 }
Daniel De Graafaab8f112011-02-03 12:19:02 -05001011
Konrad Rzeszutek Wilke85fc982014-02-03 06:43:59 -05001012 if (!in_interrupt() && paravirt_get_lazy_mode() == PARAVIRT_LAZY_NONE) {
Stefano Stabellinif62805f2012-04-24 11:55:43 +01001013 arch_enter_lazy_mmu_mode();
1014 lazy = true;
1015 }
1016
Jeremy Fitzhardinge87f1d402010-12-13 14:42:30 +00001017 for (i = 0; i < count; i++) {
Konrad Rzeszutek Wilke85fc982014-02-03 06:43:59 -05001018 ret = m2p_remove_override(pages[i], kmap_ops ?
1019 &kmap_ops[i] : NULL);
Jeremy Fitzhardinge87f1d402010-12-13 14:42:30 +00001020 if (ret)
Matt Wilson14883a72013-11-20 12:11:35 -08001021 goto out;
Jeremy Fitzhardinge87f1d402010-12-13 14:42:30 +00001022 }
Stefano Stabellini289b7772010-12-10 14:54:44 +00001023
Matt Wilson14883a72013-11-20 12:11:35 -08001024 out:
Stefano Stabellinif62805f2012-04-24 11:55:43 +01001025 if (lazy)
1026 arch_leave_lazy_mmu_mode();
1027
Stefano Stabellini289b7772010-12-10 14:54:44 +00001028 return ret;
1029}
1030EXPORT_SYMBOL_GPL(gnttab_unmap_refs);
1031
Annie Li85ff6ac2011-11-22 09:59:21 +08001032static unsigned nr_status_frames(unsigned nr_grant_frames)
1033{
Matt Wilsond0b4d642013-01-15 13:21:27 +00001034 BUG_ON(grefs_per_grant_frame == 0);
1035 return (nr_grant_frames * grefs_per_grant_frame + SPP - 1) / SPP;
Annie Li85ff6ac2011-11-22 09:59:21 +08001036}
1037
Ian Campbellef32f892012-10-17 09:39:14 +01001038static int gnttab_map_frames_v1(xen_pfn_t *frames, unsigned int nr_gframes)
Annie Li0f9f5a92011-11-22 09:58:06 +08001039{
1040 int rc;
1041
1042 rc = arch_gnttab_map_shared(frames, nr_gframes,
1043 gnttab_max_grant_frames(),
1044 &gnttab_shared.addr);
1045 BUG_ON(rc);
1046
1047 return 0;
1048}
1049
1050static void gnttab_unmap_frames_v1(void)
1051{
Annie Li85ff6ac2011-11-22 09:59:21 +08001052 arch_gnttab_unmap(gnttab_shared.addr, nr_grant_frames);
1053}
1054
Ian Campbellef32f892012-10-17 09:39:14 +01001055static int gnttab_map_frames_v2(xen_pfn_t *frames, unsigned int nr_gframes)
Annie Li85ff6ac2011-11-22 09:59:21 +08001056{
1057 uint64_t *sframes;
1058 unsigned int nr_sframes;
1059 struct gnttab_get_status_frames getframes;
1060 int rc;
1061
1062 nr_sframes = nr_status_frames(nr_gframes);
1063
1064 /* No need for kzalloc as it is initialized in following hypercall
1065 * GNTTABOP_get_status_frames.
1066 */
1067 sframes = kmalloc(nr_sframes * sizeof(uint64_t), GFP_ATOMIC);
1068 if (!sframes)
1069 return -ENOMEM;
1070
1071 getframes.dom = DOMID_SELF;
1072 getframes.nr_frames = nr_sframes;
1073 set_xen_guest_handle(getframes.frame_list, sframes);
1074
1075 rc = HYPERVISOR_grant_table_op(GNTTABOP_get_status_frames,
1076 &getframes, 1);
1077 if (rc == -ENOSYS) {
1078 kfree(sframes);
1079 return -ENOSYS;
1080 }
1081
1082 BUG_ON(rc || getframes.status);
1083
1084 rc = arch_gnttab_map_status(sframes, nr_sframes,
1085 nr_status_frames(gnttab_max_grant_frames()),
1086 &grstatus);
1087 BUG_ON(rc);
1088 kfree(sframes);
1089
1090 rc = arch_gnttab_map_shared(frames, nr_gframes,
1091 gnttab_max_grant_frames(),
1092 &gnttab_shared.addr);
1093 BUG_ON(rc);
1094
1095 return 0;
1096}
1097
1098static void gnttab_unmap_frames_v2(void)
1099{
1100 arch_gnttab_unmap(gnttab_shared.addr, nr_grant_frames);
1101 arch_gnttab_unmap(grstatus, nr_status_frames(nr_grant_frames));
Annie Li0f9f5a92011-11-22 09:58:06 +08001102}
1103
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -07001104static int gnttab_map(unsigned int start_idx, unsigned int end_idx)
1105{
1106 struct gnttab_setup_table setup;
Ian Campbellef32f892012-10-17 09:39:14 +01001107 xen_pfn_t *frames;
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -07001108 unsigned int nr_gframes = end_idx + 1;
1109 int rc;
1110
Konrad Rzeszutek Wilk6926f6d2014-01-03 10:20:18 -05001111 if (xen_feature(XENFEAT_auto_translated_physmap)) {
Stefano Stabellini183d03c2010-05-17 17:08:21 +01001112 struct xen_add_to_physmap xatp;
1113 unsigned int i = end_idx;
1114 rc = 0;
Konrad Rzeszutek Wilkefaf30a2014-01-06 10:40:36 -05001115 BUG_ON(xen_auto_xlat_grant_frames.count < nr_gframes);
Stefano Stabellini183d03c2010-05-17 17:08:21 +01001116 /*
1117 * Loop backwards, so that the first hypercall has the largest
1118 * index, ensuring that the table will grow only once.
1119 */
1120 do {
1121 xatp.domid = DOMID_SELF;
1122 xatp.idx = i;
1123 xatp.space = XENMAPSPACE_grant_table;
Konrad Rzeszutek Wilkefaf30a2014-01-06 10:40:36 -05001124 xatp.gpfn = xen_auto_xlat_grant_frames.pfn[i];
Stefano Stabellini183d03c2010-05-17 17:08:21 +01001125 rc = HYPERVISOR_memory_op(XENMEM_add_to_physmap, &xatp);
1126 if (rc != 0) {
Joe Perches283c0972013-06-28 03:21:41 -07001127 pr_warn("grant table add_to_physmap failed, err=%d\n",
1128 rc);
Stefano Stabellini183d03c2010-05-17 17:08:21 +01001129 break;
1130 }
1131 } while (i-- > start_idx);
1132
1133 return rc;
1134 }
1135
Annie Li85ff6ac2011-11-22 09:59:21 +08001136 /* No need for kzalloc as it is initialized in following hypercall
1137 * GNTTABOP_setup_table.
1138 */
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -07001139 frames = kmalloc(nr_gframes * sizeof(unsigned long), GFP_ATOMIC);
1140 if (!frames)
1141 return -ENOMEM;
1142
1143 setup.dom = DOMID_SELF;
1144 setup.nr_frames = nr_gframes;
Isaku Yamahata87e27cf2008-04-02 10:53:52 -07001145 set_xen_guest_handle(setup.frame_list, frames);
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -07001146
1147 rc = HYPERVISOR_grant_table_op(GNTTABOP_setup_table, &setup, 1);
1148 if (rc == -ENOSYS) {
1149 kfree(frames);
1150 return -ENOSYS;
1151 }
1152
1153 BUG_ON(rc || setup.status);
1154
Annie Li0f9f5a92011-11-22 09:58:06 +08001155 rc = gnttab_interface->map_frames(frames, nr_gframes);
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -07001156
1157 kfree(frames);
1158
Annie Li0f9f5a92011-11-22 09:58:06 +08001159 return rc;
1160}
1161
1162static struct gnttab_ops gnttab_v1_ops = {
1163 .map_frames = gnttab_map_frames_v1,
1164 .unmap_frames = gnttab_unmap_frames_v1,
1165 .update_entry = gnttab_update_entry_v1,
1166 .end_foreign_access_ref = gnttab_end_foreign_access_ref_v1,
1167 .end_foreign_transfer_ref = gnttab_end_foreign_transfer_ref_v1,
1168 .query_foreign_access = gnttab_query_foreign_access_v1,
1169};
1170
Annie Li85ff6ac2011-11-22 09:59:21 +08001171static struct gnttab_ops gnttab_v2_ops = {
1172 .map_frames = gnttab_map_frames_v2,
1173 .unmap_frames = gnttab_unmap_frames_v2,
1174 .update_entry = gnttab_update_entry_v2,
1175 .end_foreign_access_ref = gnttab_end_foreign_access_ref_v2,
1176 .end_foreign_transfer_ref = gnttab_end_foreign_transfer_ref_v2,
1177 .query_foreign_access = gnttab_query_foreign_access_v2,
Annie Li66667542011-12-12 18:14:42 +08001178 .update_subpage_entry = gnttab_update_subpage_entry_v2,
Annie Li9438ce92011-12-12 18:15:07 +08001179 .update_trans_entry = gnttab_update_trans_entry_v2,
Annie Li85ff6ac2011-11-22 09:59:21 +08001180};
1181
Annie Li0f9f5a92011-11-22 09:58:06 +08001182static void gnttab_request_version(void)
1183{
Annie Li85ff6ac2011-11-22 09:59:21 +08001184 int rc;
1185 struct gnttab_set_version gsv;
1186
Konrad Rzeszutek Wilk11c7ff12014-01-06 10:44:39 -05001187 gsv.version = 1;
1188
Annie Li85ff6ac2011-11-22 09:59:21 +08001189 rc = HYPERVISOR_grant_table_op(GNTTABOP_set_version, &gsv, 1);
Konrad Rzeszutek Wilk69e8f432012-01-25 00:13:20 -05001190 if (rc == 0 && gsv.version == 2) {
Annie Li85ff6ac2011-11-22 09:59:21 +08001191 grant_table_version = 2;
Matt Wilsond0b4d642013-01-15 13:21:27 +00001192 grefs_per_grant_frame = PAGE_SIZE / sizeof(union grant_entry_v2);
Annie Li85ff6ac2011-11-22 09:59:21 +08001193 gnttab_interface = &gnttab_v2_ops;
1194 } else if (grant_table_version == 2) {
1195 /*
1196 * If we've already used version 2 features,
1197 * but then suddenly discover that they're not
1198 * available (e.g. migrating to an older
1199 * version of Xen), almost unbounded badness
1200 * can happen.
1201 */
1202 panic("we need grant tables version 2, but only version 1 is available");
1203 } else {
1204 grant_table_version = 1;
Matt Wilsond0b4d642013-01-15 13:21:27 +00001205 grefs_per_grant_frame = PAGE_SIZE / sizeof(struct grant_entry_v1);
Annie Li85ff6ac2011-11-22 09:59:21 +08001206 gnttab_interface = &gnttab_v1_ops;
1207 }
Joe Perches283c0972013-06-28 03:21:41 -07001208 pr_info("Grant tables using version %d layout\n", grant_table_version);
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -07001209}
1210
Matt Wilsond0b4d642013-01-15 13:21:27 +00001211static int gnttab_setup(void)
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -07001212{
Stefano Stabellini183d03c2010-05-17 17:08:21 +01001213 unsigned int max_nr_gframes;
1214
1215 max_nr_gframes = gnttab_max_grant_frames();
1216 if (max_nr_gframes < nr_grant_frames)
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -07001217 return -ENOSYS;
Stefano Stabellini183d03c2010-05-17 17:08:21 +01001218
Konrad Rzeszutek Wilk45684752013-12-31 16:33:31 -05001219 if (xen_feature(XENFEAT_auto_translated_physmap) && gnttab_shared.addr == NULL) {
Konrad Rzeszutek Wilkefaf30a2014-01-06 10:40:36 -05001220 gnttab_shared.addr = xen_auto_xlat_grant_frames.vaddr;
Annie Li0f9f5a92011-11-22 09:58:06 +08001221 if (gnttab_shared.addr == NULL) {
Konrad Rzeszutek Wilkefaf30a2014-01-06 10:40:36 -05001222 pr_warn("gnttab share frames (addr=0x%08lx) is not mapped!\n",
1223 (unsigned long)xen_auto_xlat_grant_frames.vaddr);
Stefano Stabellini183d03c2010-05-17 17:08:21 +01001224 return -ENOMEM;
1225 }
1226 }
Konrad Rzeszutek Wilk45684752013-12-31 16:33:31 -05001227 return gnttab_map(0, nr_grant_frames - 1);
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -07001228}
1229
Matt Wilsond0b4d642013-01-15 13:21:27 +00001230int gnttab_resume(void)
1231{
1232 gnttab_request_version();
1233 return gnttab_setup();
1234}
1235
Jeremy Fitzhardinge0e913982008-05-26 23:31:27 +01001236int gnttab_suspend(void)
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -07001237{
Annie Li0f9f5a92011-11-22 09:58:06 +08001238 gnttab_interface->unmap_frames();
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -07001239 return 0;
1240}
1241
1242static int gnttab_expand(unsigned int req_entries)
1243{
1244 int rc;
1245 unsigned int cur, extra;
1246
Matt Wilsond0b4d642013-01-15 13:21:27 +00001247 BUG_ON(grefs_per_grant_frame == 0);
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -07001248 cur = nr_grant_frames;
Matt Wilsond0b4d642013-01-15 13:21:27 +00001249 extra = ((req_entries + (grefs_per_grant_frame-1)) /
1250 grefs_per_grant_frame);
Stefano Stabellini183d03c2010-05-17 17:08:21 +01001251 if (cur + extra > gnttab_max_grant_frames())
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -07001252 return -ENOSPC;
1253
1254 rc = gnttab_map(cur, cur + extra - 1);
1255 if (rc == 0)
1256 rc = grow_gnttab_list(extra);
1257
1258 return rc;
1259}
1260
Stefano Stabellini183d03c2010-05-17 17:08:21 +01001261int gnttab_init(void)
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -07001262{
1263 int i;
Michael Abd-El-Malekbbc60c12008-04-04 02:33:48 -07001264 unsigned int max_nr_glist_frames, nr_glist_frames;
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -07001265 unsigned int nr_init_grefs;
Julia Lawall6b5e7d92012-04-15 11:27:12 +02001266 int ret;
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -07001267
Matt Wilsond0b4d642013-01-15 13:21:27 +00001268 gnttab_request_version();
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -07001269 nr_grant_frames = 1;
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -07001270
1271 /* Determine the maximum number of frames required for the
1272 * grant reference free list on the current hypervisor.
1273 */
Matt Wilsond0b4d642013-01-15 13:21:27 +00001274 BUG_ON(grefs_per_grant_frame == 0);
Konrad Rzeszutek Wilk7f256022013-12-31 15:55:39 -05001275 max_nr_glist_frames = (gnttab_max_grant_frames() *
Matt Wilsond0b4d642013-01-15 13:21:27 +00001276 grefs_per_grant_frame / RPP);
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -07001277
1278 gnttab_list = kmalloc(max_nr_glist_frames * sizeof(grant_ref_t *),
1279 GFP_KERNEL);
1280 if (gnttab_list == NULL)
1281 return -ENOMEM;
1282
Matt Wilsond0b4d642013-01-15 13:21:27 +00001283 nr_glist_frames = (nr_grant_frames * grefs_per_grant_frame + RPP - 1) / RPP;
Michael Abd-El-Malekbbc60c12008-04-04 02:33:48 -07001284 for (i = 0; i < nr_glist_frames; i++) {
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -07001285 gnttab_list[i] = (grant_ref_t *)__get_free_page(GFP_KERNEL);
Julia Lawall6b5e7d92012-04-15 11:27:12 +02001286 if (gnttab_list[i] == NULL) {
1287 ret = -ENOMEM;
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -07001288 goto ini_nomem;
Julia Lawall6b5e7d92012-04-15 11:27:12 +02001289 }
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -07001290 }
1291
Matt Wilsond0b4d642013-01-15 13:21:27 +00001292 if (gnttab_setup() < 0) {
Julia Lawall6b5e7d92012-04-15 11:27:12 +02001293 ret = -ENODEV;
1294 goto ini_nomem;
1295 }
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -07001296
Matt Wilsond0b4d642013-01-15 13:21:27 +00001297 nr_init_grefs = nr_grant_frames * grefs_per_grant_frame;
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -07001298
1299 for (i = NR_RESERVED_ENTRIES; i < nr_init_grefs - 1; i++)
1300 gnttab_entry(i) = i + 1;
1301
1302 gnttab_entry(nr_init_grefs - 1) = GNTTAB_LIST_END;
1303 gnttab_free_count = nr_init_grefs - NR_RESERVED_ENTRIES;
1304 gnttab_free_head = NR_RESERVED_ENTRIES;
1305
1306 printk("Grant table initialized\n");
1307 return 0;
1308
1309 ini_nomem:
1310 for (i--; i >= 0; i--)
1311 free_page((unsigned long)gnttab_list[i]);
1312 kfree(gnttab_list);
Julia Lawall6b5e7d92012-04-15 11:27:12 +02001313 return ret;
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -07001314}
Stefano Stabellini183d03c2010-05-17 17:08:21 +01001315EXPORT_SYMBOL_GPL(gnttab_init);
Jeremy Fitzhardingead9a8612007-07-17 18:37:06 -07001316
Greg Kroah-Hartman345a5252012-12-21 13:00:00 -08001317static int __gnttab_init(void)
Stefano Stabellini183d03c2010-05-17 17:08:21 +01001318{
1319 /* Delay grant-table initialization in the PV on HVM case */
1320 if (xen_hvm_domain())
1321 return 0;
1322
1323 if (!xen_pv_domain())
1324 return -ENODEV;
1325
1326 return gnttab_init();
1327}
Konrad Rzeszutek Wilk6926f6d2014-01-03 10:20:18 -05001328/* Starts after core_initcall so that xen_pvh_gnttab_setup can be called
1329 * beforehand to initialize xen_auto_xlat_grant_frames. */
1330core_initcall_sync(__gnttab_init);