Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 1 | /****************************************************************************** |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 2 | * |
| 3 | * Back-end of the driver for virtual block devices. This portion of the |
| 4 | * driver exports a 'unified' block-device interface that can be accessed |
| 5 | * by any operating system that implements a compatible front end. A |
| 6 | * reference front-end implementation can be found in: |
Konrad Rzeszutek Wilk | a1397fa | 2011-04-14 17:05:23 -0400 | [diff] [blame] | 7 | * drivers/block/xen-blkfront.c |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 8 | * |
| 9 | * Copyright (c) 2003-2004, Keir Fraser & Steve Hand |
| 10 | * Copyright (c) 2005, Christopher Clark |
| 11 | * |
| 12 | * This program is free software; you can redistribute it and/or |
| 13 | * modify it under the terms of the GNU General Public License version 2 |
| 14 | * as published by the Free Software Foundation; or, when distributed |
| 15 | * separately from the Linux kernel or incorporated into other |
| 16 | * software packages, subject to the following license: |
| 17 | * |
| 18 | * Permission is hereby granted, free of charge, to any person obtaining a copy |
| 19 | * of this source file (the "Software"), to deal in the Software without |
| 20 | * restriction, including without limitation the rights to use, copy, modify, |
| 21 | * merge, publish, distribute, sublicense, and/or sell copies of the Software, |
| 22 | * and to permit persons to whom the Software is furnished to do so, subject to |
| 23 | * the following conditions: |
| 24 | * |
| 25 | * The above copyright notice and this permission notice shall be included in |
| 26 | * all copies or substantial portions of the Software. |
| 27 | * |
| 28 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
| 29 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
| 30 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE |
| 31 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER |
| 32 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING |
| 33 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS |
| 34 | * IN THE SOFTWARE. |
| 35 | */ |
| 36 | |
| 37 | #include <linux/spinlock.h> |
| 38 | #include <linux/kthread.h> |
| 39 | #include <linux/list.h> |
| 40 | #include <linux/delay.h> |
Jeremy Fitzhardinge | 8812293 | 2009-02-09 12:05:51 -0800 | [diff] [blame] | 41 | #include <linux/freezer.h> |
Roger Pau Monne | 0a8704a | 2012-10-24 18:58:45 +0200 | [diff] [blame] | 42 | #include <linux/bitmap.h> |
Jeremy Fitzhardinge | afd91d0 | 2009-09-15 14:12:37 -0700 | [diff] [blame] | 43 | |
Jeremy Fitzhardinge | 8812293 | 2009-02-09 12:05:51 -0800 | [diff] [blame] | 44 | #include <xen/events.h> |
| 45 | #include <xen/page.h> |
Stefano Stabellini | e79affc | 2012-08-08 17:21:14 +0000 | [diff] [blame] | 46 | #include <xen/xen.h> |
Jeremy Fitzhardinge | 8812293 | 2009-02-09 12:05:51 -0800 | [diff] [blame] | 47 | #include <asm/xen/hypervisor.h> |
| 48 | #include <asm/xen/hypercall.h> |
Roger Pau Monne | 087ffec | 2013-02-14 11:12:09 +0100 | [diff] [blame] | 49 | #include <xen/balloon.h> |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 50 | #include "common.h" |
| 51 | |
| 52 | /* |
| 53 | * These are rather arbitrary. They are fairly large because adjacent requests |
| 54 | * pulled from a communication ring are quite likely to end up being part of |
| 55 | * the same scatter/gather request at the disc. |
| 56 | * |
Konrad Rzeszutek Wilk | 8b6bf74 | 2011-04-20 11:50:43 -0400 | [diff] [blame] | 57 | * ** TRY INCREASING 'xen_blkif_reqs' IF WRITE SPEEDS SEEM TOO LOW ** |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 58 | * |
| 59 | * This will increase the chances of being able to write whole tracks. |
| 60 | * 64 should be enough to keep us competitive with Linux. |
| 61 | */ |
Konrad Rzeszutek Wilk | 8b6bf74 | 2011-04-20 11:50:43 -0400 | [diff] [blame] | 62 | static int xen_blkif_reqs = 64; |
| 63 | module_param_named(reqs, xen_blkif_reqs, int, 0); |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 64 | MODULE_PARM_DESC(reqs, "Number of blkback requests to allocate"); |
| 65 | |
| 66 | /* Run-time switchable: /sys/module/blkback/parameters/ */ |
Konrad Rzeszutek Wilk | 2e9977c | 2011-04-14 17:42:07 -0400 | [diff] [blame] | 67 | static unsigned int log_stats; |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 68 | module_param(log_stats, int, 0644); |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 69 | |
| 70 | /* |
| 71 | * Each outstanding request that we've passed to the lower device layers has a |
| 72 | * 'pending_req' allocated to it. Each buffer_head that completes decrements |
| 73 | * the pendcnt towards zero. When it hits zero, the specified domain has a |
| 74 | * response queued for it, with the saved 'id' passed back. |
| 75 | */ |
Konrad Rzeszutek Wilk | 2e9977c | 2011-04-14 17:42:07 -0400 | [diff] [blame] | 76 | struct pending_req { |
Konrad Rzeszutek Wilk | 30fd150 | 2011-05-12 16:47:48 -0400 | [diff] [blame] | 77 | struct xen_blkif *blkif; |
Konrad Rzeszutek Wilk | 01f37f2 | 2011-05-11 15:57:09 -0400 | [diff] [blame] | 78 | u64 id; |
| 79 | int nr_pages; |
| 80 | atomic_t pendcnt; |
| 81 | unsigned short operation; |
| 82 | int status; |
| 83 | struct list_head free_list; |
Roger Pau Monne | 0a8704a | 2012-10-24 18:58:45 +0200 | [diff] [blame] | 84 | DECLARE_BITMAP(unmap_seg, BLKIF_MAX_SEGMENTS_PER_REQUEST); |
Konrad Rzeszutek Wilk | 2e9977c | 2011-04-14 17:42:07 -0400 | [diff] [blame] | 85 | }; |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 86 | |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 87 | #define BLKBACK_INVALID_HANDLE (~0) |
| 88 | |
Konrad Rzeszutek Wilk | e8e2887 | 2011-02-25 10:51:29 -0500 | [diff] [blame] | 89 | struct xen_blkbk { |
Konrad Rzeszutek Wilk | 2e9977c | 2011-04-14 17:42:07 -0400 | [diff] [blame] | 90 | struct pending_req *pending_reqs; |
Konrad Rzeszutek Wilk | a1397fa | 2011-04-14 17:05:23 -0400 | [diff] [blame] | 91 | /* List of all 'pending_req' available */ |
Konrad Rzeszutek Wilk | e8e2887 | 2011-02-25 10:51:29 -0500 | [diff] [blame] | 92 | struct list_head pending_free; |
Konrad Rzeszutek Wilk | a1397fa | 2011-04-14 17:05:23 -0400 | [diff] [blame] | 93 | /* And its spinlock. */ |
Konrad Rzeszutek Wilk | e8e2887 | 2011-02-25 10:51:29 -0500 | [diff] [blame] | 94 | spinlock_t pending_free_lock; |
| 95 | wait_queue_head_t pending_free_wq; |
Konrad Rzeszutek Wilk | a1397fa | 2011-04-14 17:05:23 -0400 | [diff] [blame] | 96 | /* The list of all pages that are available. */ |
Konrad Rzeszutek Wilk | e8e2887 | 2011-02-25 10:51:29 -0500 | [diff] [blame] | 97 | struct page **pending_pages; |
Konrad Rzeszutek Wilk | a1397fa | 2011-04-14 17:05:23 -0400 | [diff] [blame] | 98 | /* And the grant handles that are available. */ |
Konrad Rzeszutek Wilk | e8e2887 | 2011-02-25 10:51:29 -0500 | [diff] [blame] | 99 | grant_handle_t *pending_grant_handles; |
| 100 | }; |
| 101 | |
| 102 | static struct xen_blkbk *blkbk; |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 103 | |
Konrad Rzeszutek Wilk | a1397fa | 2011-04-14 17:05:23 -0400 | [diff] [blame] | 104 | /* |
Roger Pau Monne | 0a8704a | 2012-10-24 18:58:45 +0200 | [diff] [blame] | 105 | * Maximum number of grant pages that can be mapped in blkback. |
| 106 | * BLKIF_MAX_SEGMENTS_PER_REQUEST * RING_SIZE is the maximum number of |
| 107 | * pages that blkback will persistently map. |
| 108 | * Currently, this is: |
| 109 | * RING_SIZE = 32 (for all known ring types) |
| 110 | * BLKIF_MAX_SEGMENTS_PER_REQUEST = 11 |
| 111 | * sizeof(struct persistent_gnt) = 48 |
| 112 | * So the maximum memory used to store the grants is: |
| 113 | * 32 * 11 * 48 = 16896 bytes |
| 114 | */ |
| 115 | static inline unsigned int max_mapped_grant_pages(enum blkif_protocol protocol) |
| 116 | { |
| 117 | switch (protocol) { |
| 118 | case BLKIF_PROTOCOL_NATIVE: |
| 119 | return __CONST_RING_SIZE(blkif, PAGE_SIZE) * |
| 120 | BLKIF_MAX_SEGMENTS_PER_REQUEST; |
| 121 | case BLKIF_PROTOCOL_X86_32: |
| 122 | return __CONST_RING_SIZE(blkif_x86_32, PAGE_SIZE) * |
| 123 | BLKIF_MAX_SEGMENTS_PER_REQUEST; |
| 124 | case BLKIF_PROTOCOL_X86_64: |
| 125 | return __CONST_RING_SIZE(blkif_x86_64, PAGE_SIZE) * |
| 126 | BLKIF_MAX_SEGMENTS_PER_REQUEST; |
| 127 | default: |
| 128 | BUG(); |
| 129 | } |
| 130 | return 0; |
| 131 | } |
| 132 | |
| 133 | |
| 134 | /* |
Konrad Rzeszutek Wilk | a1397fa | 2011-04-14 17:05:23 -0400 | [diff] [blame] | 135 | * Little helpful macro to figure out the index and virtual address of the |
| 136 | * pending_pages[..]. For each 'pending_req' we have have up to |
| 137 | * BLKIF_MAX_SEGMENTS_PER_REQUEST (11) pages. The seg would be from 0 through |
Konrad Rzeszutek Wilk | 01f37f2 | 2011-05-11 15:57:09 -0400 | [diff] [blame] | 138 | * 10 and would index in the pending_pages[..]. |
| 139 | */ |
Konrad Rzeszutek Wilk | 2e9977c | 2011-04-14 17:42:07 -0400 | [diff] [blame] | 140 | static inline int vaddr_pagenr(struct pending_req *req, int seg) |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 141 | { |
Konrad Rzeszutek Wilk | 2e9977c | 2011-04-14 17:42:07 -0400 | [diff] [blame] | 142 | return (req - blkbk->pending_reqs) * |
| 143 | BLKIF_MAX_SEGMENTS_PER_REQUEST + seg; |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 144 | } |
| 145 | |
Jan Beulich | efe08a3 | 2010-02-05 14:19:33 -0500 | [diff] [blame] | 146 | #define pending_page(req, seg) pending_pages[vaddr_pagenr(req, seg)] |
| 147 | |
Konrad Rzeszutek Wilk | 2e9977c | 2011-04-14 17:42:07 -0400 | [diff] [blame] | 148 | static inline unsigned long vaddr(struct pending_req *req, int seg) |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 149 | { |
Konrad Rzeszutek Wilk | e8e2887 | 2011-02-25 10:51:29 -0500 | [diff] [blame] | 150 | unsigned long pfn = page_to_pfn(blkbk->pending_page(req, seg)); |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 151 | return (unsigned long)pfn_to_kaddr(pfn); |
| 152 | } |
| 153 | |
| 154 | #define pending_handle(_req, _seg) \ |
Konrad Rzeszutek Wilk | e8e2887 | 2011-02-25 10:51:29 -0500 | [diff] [blame] | 155 | (blkbk->pending_grant_handles[vaddr_pagenr(_req, _seg)]) |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 156 | |
| 157 | |
Konrad Rzeszutek Wilk | 30fd150 | 2011-05-12 16:47:48 -0400 | [diff] [blame] | 158 | static int do_block_io_op(struct xen_blkif *blkif); |
| 159 | static int dispatch_rw_block_io(struct xen_blkif *blkif, |
Konrad Rzeszutek Wilk | fc53bf7 | 2011-05-05 13:37:23 -0400 | [diff] [blame] | 160 | struct blkif_request *req, |
| 161 | struct pending_req *pending_req); |
Konrad Rzeszutek Wilk | 30fd150 | 2011-05-12 16:47:48 -0400 | [diff] [blame] | 162 | static void make_response(struct xen_blkif *blkif, u64 id, |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 163 | unsigned short op, int st); |
| 164 | |
Roger Pau Monne | 7dc3411 | 2012-12-04 15:21:52 +0100 | [diff] [blame] | 165 | #define foreach_grant_safe(pos, n, rbtree, node) \ |
| 166 | for ((pos) = container_of(rb_first((rbtree)), typeof(*(pos)), node), \ |
| 167 | (n) = rb_next(&(pos)->node); \ |
Roger Pau Monne | 0a8704a | 2012-10-24 18:58:45 +0200 | [diff] [blame] | 168 | &(pos)->node != NULL; \ |
Roger Pau Monne | 7dc3411 | 2012-12-04 15:21:52 +0100 | [diff] [blame] | 169 | (pos) = container_of(n, typeof(*(pos)), node), \ |
| 170 | (n) = (&(pos)->node != NULL) ? rb_next(&(pos)->node) : NULL) |
Roger Pau Monne | 0a8704a | 2012-10-24 18:58:45 +0200 | [diff] [blame] | 171 | |
| 172 | |
| 173 | static void add_persistent_gnt(struct rb_root *root, |
| 174 | struct persistent_gnt *persistent_gnt) |
| 175 | { |
| 176 | struct rb_node **new = &(root->rb_node), *parent = NULL; |
| 177 | struct persistent_gnt *this; |
| 178 | |
| 179 | /* Figure out where to put new node */ |
| 180 | while (*new) { |
| 181 | this = container_of(*new, struct persistent_gnt, node); |
| 182 | |
| 183 | parent = *new; |
| 184 | if (persistent_gnt->gnt < this->gnt) |
| 185 | new = &((*new)->rb_left); |
| 186 | else if (persistent_gnt->gnt > this->gnt) |
| 187 | new = &((*new)->rb_right); |
| 188 | else { |
| 189 | pr_alert(DRV_PFX " trying to add a gref that's already in the tree\n"); |
| 190 | BUG(); |
| 191 | } |
| 192 | } |
| 193 | |
| 194 | /* Add new node and rebalance tree. */ |
| 195 | rb_link_node(&(persistent_gnt->node), parent, new); |
| 196 | rb_insert_color(&(persistent_gnt->node), root); |
| 197 | } |
| 198 | |
| 199 | static struct persistent_gnt *get_persistent_gnt(struct rb_root *root, |
| 200 | grant_ref_t gref) |
| 201 | { |
| 202 | struct persistent_gnt *data; |
| 203 | struct rb_node *node = root->rb_node; |
| 204 | |
| 205 | while (node) { |
| 206 | data = container_of(node, struct persistent_gnt, node); |
| 207 | |
| 208 | if (gref < data->gnt) |
| 209 | node = node->rb_left; |
| 210 | else if (gref > data->gnt) |
| 211 | node = node->rb_right; |
| 212 | else |
| 213 | return data; |
| 214 | } |
| 215 | return NULL; |
| 216 | } |
| 217 | |
Roger Pau Monne | 4d4f270 | 2012-11-16 19:26:48 +0100 | [diff] [blame] | 218 | static void free_persistent_gnts(struct rb_root *root, unsigned int num) |
| 219 | { |
| 220 | struct gnttab_unmap_grant_ref unmap[BLKIF_MAX_SEGMENTS_PER_REQUEST]; |
| 221 | struct page *pages[BLKIF_MAX_SEGMENTS_PER_REQUEST]; |
| 222 | struct persistent_gnt *persistent_gnt; |
Roger Pau Monne | 7dc3411 | 2012-12-04 15:21:52 +0100 | [diff] [blame] | 223 | struct rb_node *n; |
Roger Pau Monne | 4d4f270 | 2012-11-16 19:26:48 +0100 | [diff] [blame] | 224 | int ret = 0; |
| 225 | int segs_to_unmap = 0; |
| 226 | |
Roger Pau Monne | 7dc3411 | 2012-12-04 15:21:52 +0100 | [diff] [blame] | 227 | foreach_grant_safe(persistent_gnt, n, root, node) { |
Roger Pau Monne | 4d4f270 | 2012-11-16 19:26:48 +0100 | [diff] [blame] | 228 | BUG_ON(persistent_gnt->handle == |
| 229 | BLKBACK_INVALID_HANDLE); |
| 230 | gnttab_set_unmap_op(&unmap[segs_to_unmap], |
| 231 | (unsigned long) pfn_to_kaddr(page_to_pfn( |
| 232 | persistent_gnt->page)), |
| 233 | GNTMAP_host_map, |
| 234 | persistent_gnt->handle); |
| 235 | |
| 236 | pages[segs_to_unmap] = persistent_gnt->page; |
Roger Pau Monne | 4d4f270 | 2012-11-16 19:26:48 +0100 | [diff] [blame] | 237 | |
| 238 | if (++segs_to_unmap == BLKIF_MAX_SEGMENTS_PER_REQUEST || |
| 239 | !rb_next(&persistent_gnt->node)) { |
| 240 | ret = gnttab_unmap_refs(unmap, NULL, pages, |
| 241 | segs_to_unmap); |
| 242 | BUG_ON(ret); |
Roger Pau Monne | 087ffec | 2013-02-14 11:12:09 +0100 | [diff] [blame] | 243 | free_xenballooned_pages(segs_to_unmap, pages); |
Roger Pau Monne | 4d4f270 | 2012-11-16 19:26:48 +0100 | [diff] [blame] | 244 | segs_to_unmap = 0; |
| 245 | } |
Roger Pau Monne | 7dc3411 | 2012-12-04 15:21:52 +0100 | [diff] [blame] | 246 | |
| 247 | rb_erase(&persistent_gnt->node, root); |
| 248 | kfree(persistent_gnt); |
| 249 | num--; |
Roger Pau Monne | 4d4f270 | 2012-11-16 19:26:48 +0100 | [diff] [blame] | 250 | } |
| 251 | BUG_ON(num != 0); |
| 252 | } |
| 253 | |
Konrad Rzeszutek Wilk | a1397fa | 2011-04-14 17:05:23 -0400 | [diff] [blame] | 254 | /* |
| 255 | * Retrieve from the 'pending_reqs' a free pending_req structure to be used. |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 256 | */ |
Konrad Rzeszutek Wilk | 2e9977c | 2011-04-14 17:42:07 -0400 | [diff] [blame] | 257 | static struct pending_req *alloc_req(void) |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 258 | { |
Konrad Rzeszutek Wilk | 2e9977c | 2011-04-14 17:42:07 -0400 | [diff] [blame] | 259 | struct pending_req *req = NULL; |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 260 | unsigned long flags; |
| 261 | |
Konrad Rzeszutek Wilk | e8e2887 | 2011-02-25 10:51:29 -0500 | [diff] [blame] | 262 | spin_lock_irqsave(&blkbk->pending_free_lock, flags); |
| 263 | if (!list_empty(&blkbk->pending_free)) { |
Konrad Rzeszutek Wilk | 2e9977c | 2011-04-14 17:42:07 -0400 | [diff] [blame] | 264 | req = list_entry(blkbk->pending_free.next, struct pending_req, |
| 265 | free_list); |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 266 | list_del(&req->free_list); |
| 267 | } |
Konrad Rzeszutek Wilk | e8e2887 | 2011-02-25 10:51:29 -0500 | [diff] [blame] | 268 | spin_unlock_irqrestore(&blkbk->pending_free_lock, flags); |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 269 | return req; |
| 270 | } |
| 271 | |
Konrad Rzeszutek Wilk | a1397fa | 2011-04-14 17:05:23 -0400 | [diff] [blame] | 272 | /* |
| 273 | * Return the 'pending_req' structure back to the freepool. We also |
| 274 | * wake up the thread if it was waiting for a free page. |
| 275 | */ |
Konrad Rzeszutek Wilk | 2e9977c | 2011-04-14 17:42:07 -0400 | [diff] [blame] | 276 | static void free_req(struct pending_req *req) |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 277 | { |
| 278 | unsigned long flags; |
| 279 | int was_empty; |
| 280 | |
Konrad Rzeszutek Wilk | e8e2887 | 2011-02-25 10:51:29 -0500 | [diff] [blame] | 281 | spin_lock_irqsave(&blkbk->pending_free_lock, flags); |
| 282 | was_empty = list_empty(&blkbk->pending_free); |
| 283 | list_add(&req->free_list, &blkbk->pending_free); |
| 284 | spin_unlock_irqrestore(&blkbk->pending_free_lock, flags); |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 285 | if (was_empty) |
Konrad Rzeszutek Wilk | e8e2887 | 2011-02-25 10:51:29 -0500 | [diff] [blame] | 286 | wake_up(&blkbk->pending_free_wq); |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 287 | } |
| 288 | |
Konrad Rzeszutek Wilk | a1397fa | 2011-04-14 17:05:23 -0400 | [diff] [blame] | 289 | /* |
Konrad Rzeszutek Wilk | ee9ff85 | 2011-04-20 10:57:29 -0400 | [diff] [blame] | 290 | * Routines for managing virtual block devices (vbds). |
| 291 | */ |
Konrad Rzeszutek Wilk | 3d81473 | 2011-05-12 16:53:56 -0400 | [diff] [blame] | 292 | static int xen_vbd_translate(struct phys_req *req, struct xen_blkif *blkif, |
| 293 | int operation) |
Konrad Rzeszutek Wilk | ee9ff85 | 2011-04-20 10:57:29 -0400 | [diff] [blame] | 294 | { |
Konrad Rzeszutek Wilk | 3d81473 | 2011-05-12 16:53:56 -0400 | [diff] [blame] | 295 | struct xen_vbd *vbd = &blkif->vbd; |
Konrad Rzeszutek Wilk | ee9ff85 | 2011-04-20 10:57:29 -0400 | [diff] [blame] | 296 | int rc = -EACCES; |
| 297 | |
| 298 | if ((operation != READ) && vbd->readonly) |
| 299 | goto out; |
| 300 | |
Jan Beulich | 8ab5215 | 2011-05-17 11:07:05 +0100 | [diff] [blame] | 301 | if (likely(req->nr_sects)) { |
| 302 | blkif_sector_t end = req->sector_number + req->nr_sects; |
| 303 | |
| 304 | if (unlikely(end < req->sector_number)) |
| 305 | goto out; |
| 306 | if (unlikely(end > vbd_sz(vbd))) |
| 307 | goto out; |
| 308 | } |
Konrad Rzeszutek Wilk | ee9ff85 | 2011-04-20 10:57:29 -0400 | [diff] [blame] | 309 | |
| 310 | req->dev = vbd->pdevice; |
| 311 | req->bdev = vbd->bdev; |
| 312 | rc = 0; |
| 313 | |
| 314 | out: |
| 315 | return rc; |
| 316 | } |
| 317 | |
Konrad Rzeszutek Wilk | 3d81473 | 2011-05-12 16:53:56 -0400 | [diff] [blame] | 318 | static void xen_vbd_resize(struct xen_blkif *blkif) |
Konrad Rzeszutek Wilk | ee9ff85 | 2011-04-20 10:57:29 -0400 | [diff] [blame] | 319 | { |
Konrad Rzeszutek Wilk | 3d81473 | 2011-05-12 16:53:56 -0400 | [diff] [blame] | 320 | struct xen_vbd *vbd = &blkif->vbd; |
Konrad Rzeszutek Wilk | ee9ff85 | 2011-04-20 10:57:29 -0400 | [diff] [blame] | 321 | struct xenbus_transaction xbt; |
| 322 | int err; |
Konrad Rzeszutek Wilk | 8b6bf74 | 2011-04-20 11:50:43 -0400 | [diff] [blame] | 323 | struct xenbus_device *dev = xen_blkbk_xenbus(blkif->be); |
Konrad Rzeszutek Wilk | 42c7841 | 2011-04-20 11:21:43 -0400 | [diff] [blame] | 324 | unsigned long long new_size = vbd_sz(vbd); |
Konrad Rzeszutek Wilk | ee9ff85 | 2011-04-20 10:57:29 -0400 | [diff] [blame] | 325 | |
Konrad Rzeszutek Wilk | 22b20f2 | 2011-05-12 16:43:12 -0400 | [diff] [blame] | 326 | pr_info(DRV_PFX "VBD Resize: Domid: %d, Device: (%d, %d)\n", |
Konrad Rzeszutek Wilk | ee9ff85 | 2011-04-20 10:57:29 -0400 | [diff] [blame] | 327 | blkif->domid, MAJOR(vbd->pdevice), MINOR(vbd->pdevice)); |
Konrad Rzeszutek Wilk | 22b20f2 | 2011-05-12 16:43:12 -0400 | [diff] [blame] | 328 | pr_info(DRV_PFX "VBD Resize: new size %llu\n", new_size); |
Konrad Rzeszutek Wilk | ee9ff85 | 2011-04-20 10:57:29 -0400 | [diff] [blame] | 329 | vbd->size = new_size; |
| 330 | again: |
| 331 | err = xenbus_transaction_start(&xbt); |
| 332 | if (err) { |
Konrad Rzeszutek Wilk | 22b20f2 | 2011-05-12 16:43:12 -0400 | [diff] [blame] | 333 | pr_warn(DRV_PFX "Error starting transaction"); |
Konrad Rzeszutek Wilk | ee9ff85 | 2011-04-20 10:57:29 -0400 | [diff] [blame] | 334 | return; |
| 335 | } |
| 336 | err = xenbus_printf(xbt, dev->nodename, "sectors", "%llu", |
Konrad Rzeszutek Wilk | 42c7841 | 2011-04-20 11:21:43 -0400 | [diff] [blame] | 337 | (unsigned long long)vbd_sz(vbd)); |
Konrad Rzeszutek Wilk | ee9ff85 | 2011-04-20 10:57:29 -0400 | [diff] [blame] | 338 | if (err) { |
Konrad Rzeszutek Wilk | 22b20f2 | 2011-05-12 16:43:12 -0400 | [diff] [blame] | 339 | pr_warn(DRV_PFX "Error writing new size"); |
Konrad Rzeszutek Wilk | ee9ff85 | 2011-04-20 10:57:29 -0400 | [diff] [blame] | 340 | goto abort; |
| 341 | } |
| 342 | /* |
| 343 | * Write the current state; we will use this to synchronize |
| 344 | * the front-end. If the current state is "connected" the |
| 345 | * front-end will get the new size information online. |
| 346 | */ |
| 347 | err = xenbus_printf(xbt, dev->nodename, "state", "%d", dev->state); |
| 348 | if (err) { |
Konrad Rzeszutek Wilk | 22b20f2 | 2011-05-12 16:43:12 -0400 | [diff] [blame] | 349 | pr_warn(DRV_PFX "Error writing the state"); |
Konrad Rzeszutek Wilk | ee9ff85 | 2011-04-20 10:57:29 -0400 | [diff] [blame] | 350 | goto abort; |
| 351 | } |
| 352 | |
| 353 | err = xenbus_transaction_end(xbt, 0); |
| 354 | if (err == -EAGAIN) |
| 355 | goto again; |
| 356 | if (err) |
Konrad Rzeszutek Wilk | 22b20f2 | 2011-05-12 16:43:12 -0400 | [diff] [blame] | 357 | pr_warn(DRV_PFX "Error ending transaction"); |
Laszlo Ersek | 496b318 | 2011-05-13 09:45:40 -0400 | [diff] [blame] | 358 | return; |
Konrad Rzeszutek Wilk | ee9ff85 | 2011-04-20 10:57:29 -0400 | [diff] [blame] | 359 | abort: |
| 360 | xenbus_transaction_end(xbt, 1); |
| 361 | } |
| 362 | |
| 363 | /* |
Konrad Rzeszutek Wilk | b0aef17 | 2011-04-15 10:58:05 -0400 | [diff] [blame] | 364 | * Notification from the guest OS. |
| 365 | */ |
Konrad Rzeszutek Wilk | 30fd150 | 2011-05-12 16:47:48 -0400 | [diff] [blame] | 366 | static void blkif_notify_work(struct xen_blkif *blkif) |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 367 | { |
Konrad Rzeszutek Wilk | b0aef17 | 2011-04-15 10:58:05 -0400 | [diff] [blame] | 368 | blkif->waiting_reqs = 1; |
| 369 | wake_up(&blkif->wq); |
| 370 | } |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 371 | |
Konrad Rzeszutek Wilk | 8b6bf74 | 2011-04-20 11:50:43 -0400 | [diff] [blame] | 372 | irqreturn_t xen_blkif_be_int(int irq, void *dev_id) |
Konrad Rzeszutek Wilk | b0aef17 | 2011-04-15 10:58:05 -0400 | [diff] [blame] | 373 | { |
| 374 | blkif_notify_work(dev_id); |
| 375 | return IRQ_HANDLED; |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 376 | } |
| 377 | |
Konrad Rzeszutek Wilk | 2e9977c | 2011-04-14 17:42:07 -0400 | [diff] [blame] | 378 | /* |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 379 | * SCHEDULER FUNCTIONS |
| 380 | */ |
| 381 | |
Konrad Rzeszutek Wilk | 30fd150 | 2011-05-12 16:47:48 -0400 | [diff] [blame] | 382 | static void print_stats(struct xen_blkif *blkif) |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 383 | { |
Li Dongyang | b3cb0d6 | 2011-09-01 18:39:10 +0800 | [diff] [blame] | 384 | pr_info("xen-blkback (%s): oo %3d | rd %4d | wr %4d | f %4d" |
| 385 | " | ds %4d\n", |
Konrad Rzeszutek Wilk | ebe8190 | 2011-05-12 16:42:31 -0400 | [diff] [blame] | 386 | current->comm, blkif->st_oo_req, |
Li Dongyang | b3cb0d6 | 2011-09-01 18:39:10 +0800 | [diff] [blame] | 387 | blkif->st_rd_req, blkif->st_wr_req, |
| 388 | blkif->st_f_req, blkif->st_ds_req); |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 389 | blkif->st_print = jiffies + msecs_to_jiffies(10 * 1000); |
| 390 | blkif->st_rd_req = 0; |
| 391 | blkif->st_wr_req = 0; |
| 392 | blkif->st_oo_req = 0; |
Li Dongyang | b3cb0d6 | 2011-09-01 18:39:10 +0800 | [diff] [blame] | 393 | blkif->st_ds_req = 0; |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 394 | } |
| 395 | |
Konrad Rzeszutek Wilk | 8b6bf74 | 2011-04-20 11:50:43 -0400 | [diff] [blame] | 396 | int xen_blkif_schedule(void *arg) |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 397 | { |
Konrad Rzeszutek Wilk | 30fd150 | 2011-05-12 16:47:48 -0400 | [diff] [blame] | 398 | struct xen_blkif *blkif = arg; |
Konrad Rzeszutek Wilk | 3d81473 | 2011-05-12 16:53:56 -0400 | [diff] [blame] | 399 | struct xen_vbd *vbd = &blkif->vbd; |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 400 | |
Konrad Rzeszutek Wilk | 8b6bf74 | 2011-04-20 11:50:43 -0400 | [diff] [blame] | 401 | xen_blkif_get(blkif); |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 402 | |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 403 | while (!kthread_should_stop()) { |
| 404 | if (try_to_freeze()) |
| 405 | continue; |
Konrad Rzeszutek Wilk | 42c7841 | 2011-04-20 11:21:43 -0400 | [diff] [blame] | 406 | if (unlikely(vbd->size != vbd_sz(vbd))) |
Konrad Rzeszutek Wilk | 3d81473 | 2011-05-12 16:53:56 -0400 | [diff] [blame] | 407 | xen_vbd_resize(blkif); |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 408 | |
| 409 | wait_event_interruptible( |
| 410 | blkif->wq, |
| 411 | blkif->waiting_reqs || kthread_should_stop()); |
| 412 | wait_event_interruptible( |
Konrad Rzeszutek Wilk | e8e2887 | 2011-02-25 10:51:29 -0500 | [diff] [blame] | 413 | blkbk->pending_free_wq, |
Konrad Rzeszutek Wilk | 2e9977c | 2011-04-14 17:42:07 -0400 | [diff] [blame] | 414 | !list_empty(&blkbk->pending_free) || |
| 415 | kthread_should_stop()); |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 416 | |
| 417 | blkif->waiting_reqs = 0; |
| 418 | smp_mb(); /* clear flag *before* checking for work */ |
| 419 | |
| 420 | if (do_block_io_op(blkif)) |
| 421 | blkif->waiting_reqs = 1; |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 422 | |
| 423 | if (log_stats && time_after(jiffies, blkif->st_print)) |
| 424 | print_stats(blkif); |
| 425 | } |
| 426 | |
Roger Pau Monne | 0a8704a | 2012-10-24 18:58:45 +0200 | [diff] [blame] | 427 | /* Free all persistent grant pages */ |
Roger Pau Monne | 4d4f270 | 2012-11-16 19:26:48 +0100 | [diff] [blame] | 428 | if (!RB_EMPTY_ROOT(&blkif->persistent_gnts)) |
| 429 | free_persistent_gnts(&blkif->persistent_gnts, |
| 430 | blkif->persistent_gnt_c); |
Roger Pau Monne | 0a8704a | 2012-10-24 18:58:45 +0200 | [diff] [blame] | 431 | |
Roger Pau Monne | 0a8704a | 2012-10-24 18:58:45 +0200 | [diff] [blame] | 432 | BUG_ON(!RB_EMPTY_ROOT(&blkif->persistent_gnts)); |
Roger Pau Monne | 4d4f270 | 2012-11-16 19:26:48 +0100 | [diff] [blame] | 433 | blkif->persistent_gnt_c = 0; |
Roger Pau Monne | 0a8704a | 2012-10-24 18:58:45 +0200 | [diff] [blame] | 434 | |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 435 | if (log_stats) |
| 436 | print_stats(blkif); |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 437 | |
| 438 | blkif->xenblkd = NULL; |
Konrad Rzeszutek Wilk | 8b6bf74 | 2011-04-20 11:50:43 -0400 | [diff] [blame] | 439 | xen_blkif_put(blkif); |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 440 | |
| 441 | return 0; |
| 442 | } |
| 443 | |
Konrad Rzeszutek Wilk | 1a95fe6 | 2011-04-15 11:35:13 -0400 | [diff] [blame] | 444 | struct seg_buf { |
| 445 | unsigned long buf; |
| 446 | unsigned int nsec; |
| 447 | }; |
Konrad Rzeszutek Wilk | a1397fa | 2011-04-14 17:05:23 -0400 | [diff] [blame] | 448 | /* |
Konrad Rzeszutek Wilk | b0aef17 | 2011-04-15 10:58:05 -0400 | [diff] [blame] | 449 | * Unmap the grant references, and also remove the M2P over-rides |
| 450 | * used in the 'pending_req'. |
Konrad Rzeszutek Wilk | 01f37f2 | 2011-05-11 15:57:09 -0400 | [diff] [blame] | 451 | */ |
Konrad Rzeszutek Wilk | 9f3aedf | 2011-04-15 11:50:34 -0400 | [diff] [blame] | 452 | static void xen_blkbk_unmap(struct pending_req *req) |
Konrad Rzeszutek Wilk | b0aef17 | 2011-04-15 10:58:05 -0400 | [diff] [blame] | 453 | { |
| 454 | struct gnttab_unmap_grant_ref unmap[BLKIF_MAX_SEGMENTS_PER_REQUEST]; |
Daniel De Graaf | 4f14faa | 2011-11-28 11:49:03 -0500 | [diff] [blame] | 455 | struct page *pages[BLKIF_MAX_SEGMENTS_PER_REQUEST]; |
Konrad Rzeszutek Wilk | b0aef17 | 2011-04-15 10:58:05 -0400 | [diff] [blame] | 456 | unsigned int i, invcount = 0; |
| 457 | grant_handle_t handle; |
| 458 | int ret; |
| 459 | |
| 460 | for (i = 0; i < req->nr_pages; i++) { |
Roger Pau Monne | 0a8704a | 2012-10-24 18:58:45 +0200 | [diff] [blame] | 461 | if (!test_bit(i, req->unmap_seg)) |
| 462 | continue; |
Konrad Rzeszutek Wilk | b0aef17 | 2011-04-15 10:58:05 -0400 | [diff] [blame] | 463 | handle = pending_handle(req, i); |
| 464 | if (handle == BLKBACK_INVALID_HANDLE) |
| 465 | continue; |
| 466 | gnttab_set_unmap_op(&unmap[invcount], vaddr(req, i), |
| 467 | GNTMAP_host_map, handle); |
| 468 | pending_handle(req, i) = BLKBACK_INVALID_HANDLE; |
Daniel De Graaf | 4f14faa | 2011-11-28 11:49:03 -0500 | [diff] [blame] | 469 | pages[invcount] = virt_to_page(vaddr(req, i)); |
Konrad Rzeszutek Wilk | b0aef17 | 2011-04-15 10:58:05 -0400 | [diff] [blame] | 470 | invcount++; |
| 471 | } |
| 472 | |
Stefano Stabellini | 2fc136e | 2012-09-12 12:44:30 +0100 | [diff] [blame] | 473 | ret = gnttab_unmap_refs(unmap, NULL, pages, invcount); |
Konrad Rzeszutek Wilk | b0aef17 | 2011-04-15 10:58:05 -0400 | [diff] [blame] | 474 | BUG_ON(ret); |
Konrad Rzeszutek Wilk | b0aef17 | 2011-04-15 10:58:05 -0400 | [diff] [blame] | 475 | } |
Konrad Rzeszutek Wilk | 01f37f2 | 2011-05-11 15:57:09 -0400 | [diff] [blame] | 476 | |
| 477 | static int xen_blkbk_map(struct blkif_request *req, |
| 478 | struct pending_req *pending_req, |
Roger Pau Monne | 0a8704a | 2012-10-24 18:58:45 +0200 | [diff] [blame] | 479 | struct seg_buf seg[], |
| 480 | struct page *pages[]) |
Konrad Rzeszutek Wilk | 1a95fe6 | 2011-04-15 11:35:13 -0400 | [diff] [blame] | 481 | { |
| 482 | struct gnttab_map_grant_ref map[BLKIF_MAX_SEGMENTS_PER_REQUEST]; |
Roger Pau Monne | 0a8704a | 2012-10-24 18:58:45 +0200 | [diff] [blame] | 483 | struct persistent_gnt *persistent_gnts[BLKIF_MAX_SEGMENTS_PER_REQUEST]; |
| 484 | struct page *pages_to_gnt[BLKIF_MAX_SEGMENTS_PER_REQUEST]; |
| 485 | struct persistent_gnt *persistent_gnt = NULL; |
| 486 | struct xen_blkif *blkif = pending_req->blkif; |
| 487 | phys_addr_t addr = 0; |
| 488 | int i, j; |
| 489 | bool new_map; |
Konrad Rzeszutek Wilk | 97e3683 | 2011-10-12 12:12:36 -0400 | [diff] [blame] | 490 | int nseg = req->u.rw.nr_segments; |
Roger Pau Monne | 0a8704a | 2012-10-24 18:58:45 +0200 | [diff] [blame] | 491 | int segs_to_map = 0; |
Konrad Rzeszutek Wilk | 1a95fe6 | 2011-04-15 11:35:13 -0400 | [diff] [blame] | 492 | int ret = 0; |
Roger Pau Monne | 0a8704a | 2012-10-24 18:58:45 +0200 | [diff] [blame] | 493 | int use_persistent_gnts; |
| 494 | |
| 495 | use_persistent_gnts = (blkif->vbd.feature_gnt_persistent); |
| 496 | |
| 497 | BUG_ON(blkif->persistent_gnt_c > |
| 498 | max_mapped_grant_pages(pending_req->blkif->blk_protocol)); |
Konrad Rzeszutek Wilk | 01f37f2 | 2011-05-11 15:57:09 -0400 | [diff] [blame] | 499 | |
| 500 | /* |
| 501 | * Fill out preq.nr_sects with proper amount of sectors, and setup |
Konrad Rzeszutek Wilk | 1a95fe6 | 2011-04-15 11:35:13 -0400 | [diff] [blame] | 502 | * assign map[..] with the PFN of the page in our domain with the |
| 503 | * corresponding grant reference for each page. |
| 504 | */ |
| 505 | for (i = 0; i < nseg; i++) { |
| 506 | uint32_t flags; |
| 507 | |
Roger Pau Monne | 0a8704a | 2012-10-24 18:58:45 +0200 | [diff] [blame] | 508 | if (use_persistent_gnts) |
| 509 | persistent_gnt = get_persistent_gnt( |
| 510 | &blkif->persistent_gnts, |
| 511 | req->u.rw.seg[i].gref); |
| 512 | |
| 513 | if (persistent_gnt) { |
| 514 | /* |
| 515 | * We are using persistent grants and |
| 516 | * the grant is already mapped |
| 517 | */ |
| 518 | new_map = false; |
| 519 | } else if (use_persistent_gnts && |
| 520 | blkif->persistent_gnt_c < |
| 521 | max_mapped_grant_pages(blkif->blk_protocol)) { |
| 522 | /* |
| 523 | * We are using persistent grants, the grant is |
| 524 | * not mapped but we have room for it |
| 525 | */ |
| 526 | new_map = true; |
Roger Pau Monne | cb5bd4d | 2012-11-02 16:43:04 +0100 | [diff] [blame] | 527 | persistent_gnt = kmalloc( |
Roger Pau Monne | 0a8704a | 2012-10-24 18:58:45 +0200 | [diff] [blame] | 528 | sizeof(struct persistent_gnt), |
| 529 | GFP_KERNEL); |
| 530 | if (!persistent_gnt) |
| 531 | return -ENOMEM; |
Roger Pau Monne | 087ffec | 2013-02-14 11:12:09 +0100 | [diff] [blame] | 532 | if (alloc_xenballooned_pages(1, &persistent_gnt->page, |
| 533 | false)) { |
Roger Pau Monne | 0a8704a | 2012-10-24 18:58:45 +0200 | [diff] [blame] | 534 | kfree(persistent_gnt); |
| 535 | return -ENOMEM; |
| 536 | } |
| 537 | persistent_gnt->gnt = req->u.rw.seg[i].gref; |
Roger Pau Monne | cb5bd4d | 2012-11-02 16:43:04 +0100 | [diff] [blame] | 538 | persistent_gnt->handle = BLKBACK_INVALID_HANDLE; |
Roger Pau Monne | 0a8704a | 2012-10-24 18:58:45 +0200 | [diff] [blame] | 539 | |
| 540 | pages_to_gnt[segs_to_map] = |
| 541 | persistent_gnt->page; |
| 542 | addr = (unsigned long) pfn_to_kaddr( |
| 543 | page_to_pfn(persistent_gnt->page)); |
| 544 | |
| 545 | add_persistent_gnt(&blkif->persistent_gnts, |
| 546 | persistent_gnt); |
| 547 | blkif->persistent_gnt_c++; |
| 548 | pr_debug(DRV_PFX " grant %u added to the tree of persistent grants, using %u/%u\n", |
| 549 | persistent_gnt->gnt, blkif->persistent_gnt_c, |
| 550 | max_mapped_grant_pages(blkif->blk_protocol)); |
| 551 | } else { |
| 552 | /* |
| 553 | * We are either using persistent grants and |
| 554 | * hit the maximum limit of grants mapped, |
| 555 | * or we are not using persistent grants. |
| 556 | */ |
| 557 | if (use_persistent_gnts && |
| 558 | !blkif->vbd.overflow_max_grants) { |
| 559 | blkif->vbd.overflow_max_grants = 1; |
| 560 | pr_alert(DRV_PFX " domain %u, device %#x is using maximum number of persistent grants\n", |
| 561 | blkif->domid, blkif->vbd.handle); |
| 562 | } |
| 563 | new_map = true; |
| 564 | pages[i] = blkbk->pending_page(pending_req, i); |
| 565 | addr = vaddr(pending_req, i); |
| 566 | pages_to_gnt[segs_to_map] = |
| 567 | blkbk->pending_page(pending_req, i); |
| 568 | } |
| 569 | |
| 570 | if (persistent_gnt) { |
| 571 | pages[i] = persistent_gnt->page; |
| 572 | persistent_gnts[i] = persistent_gnt; |
| 573 | } else { |
| 574 | persistent_gnts[i] = NULL; |
| 575 | } |
| 576 | |
| 577 | if (new_map) { |
| 578 | flags = GNTMAP_host_map; |
| 579 | if (!persistent_gnt && |
| 580 | (pending_req->operation != BLKIF_OP_READ)) |
| 581 | flags |= GNTMAP_readonly; |
| 582 | gnttab_set_map_op(&map[segs_to_map++], addr, |
| 583 | flags, req->u.rw.seg[i].gref, |
| 584 | blkif->domid); |
| 585 | } |
Konrad Rzeszutek Wilk | 1a95fe6 | 2011-04-15 11:35:13 -0400 | [diff] [blame] | 586 | } |
| 587 | |
Roger Pau Monne | 0a8704a | 2012-10-24 18:58:45 +0200 | [diff] [blame] | 588 | if (segs_to_map) { |
| 589 | ret = gnttab_map_refs(map, NULL, pages_to_gnt, segs_to_map); |
| 590 | BUG_ON(ret); |
| 591 | } |
Konrad Rzeszutek Wilk | 1a95fe6 | 2011-04-15 11:35:13 -0400 | [diff] [blame] | 592 | |
Konrad Rzeszutek Wilk | 01f37f2 | 2011-05-11 15:57:09 -0400 | [diff] [blame] | 593 | /* |
| 594 | * Now swizzle the MFN in our domain with the MFN from the other domain |
Konrad Rzeszutek Wilk | 1a95fe6 | 2011-04-15 11:35:13 -0400 | [diff] [blame] | 595 | * so that when we access vaddr(pending_req,i) it has the contents of |
| 596 | * the page from the other domain. |
| 597 | */ |
Roger Pau Monne | 0a8704a | 2012-10-24 18:58:45 +0200 | [diff] [blame] | 598 | bitmap_zero(pending_req->unmap_seg, BLKIF_MAX_SEGMENTS_PER_REQUEST); |
| 599 | for (i = 0, j = 0; i < nseg; i++) { |
Roger Pau Monne | cb5bd4d | 2012-11-02 16:43:04 +0100 | [diff] [blame] | 600 | if (!persistent_gnts[i] || |
| 601 | persistent_gnts[i]->handle == BLKBACK_INVALID_HANDLE) { |
Roger Pau Monne | 0a8704a | 2012-10-24 18:58:45 +0200 | [diff] [blame] | 602 | /* This is a newly mapped grant */ |
| 603 | BUG_ON(j >= segs_to_map); |
| 604 | if (unlikely(map[j].status != 0)) { |
| 605 | pr_debug(DRV_PFX "invalid buffer -- could not remap it\n"); |
| 606 | map[j].handle = BLKBACK_INVALID_HANDLE; |
| 607 | ret |= 1; |
| 608 | if (persistent_gnts[i]) { |
| 609 | rb_erase(&persistent_gnts[i]->node, |
| 610 | &blkif->persistent_gnts); |
| 611 | blkif->persistent_gnt_c--; |
| 612 | kfree(persistent_gnts[i]); |
| 613 | persistent_gnts[i] = NULL; |
| 614 | } |
| 615 | } |
Konrad Rzeszutek Wilk | 1a95fe6 | 2011-04-15 11:35:13 -0400 | [diff] [blame] | 616 | } |
Roger Pau Monne | 0a8704a | 2012-10-24 18:58:45 +0200 | [diff] [blame] | 617 | if (persistent_gnts[i]) { |
Roger Pau Monne | cb5bd4d | 2012-11-02 16:43:04 +0100 | [diff] [blame] | 618 | if (persistent_gnts[i]->handle == |
| 619 | BLKBACK_INVALID_HANDLE) { |
Roger Pau Monne | 0a8704a | 2012-10-24 18:58:45 +0200 | [diff] [blame] | 620 | /* |
| 621 | * If this is a new persistent grant |
| 622 | * save the handler |
| 623 | */ |
| 624 | persistent_gnts[i]->handle = map[j].handle; |
| 625 | persistent_gnts[i]->dev_bus_addr = |
| 626 | map[j++].dev_bus_addr; |
| 627 | } |
| 628 | pending_handle(pending_req, i) = |
| 629 | persistent_gnts[i]->handle; |
Konrad Rzeszutek Wilk | 1a95fe6 | 2011-04-15 11:35:13 -0400 | [diff] [blame] | 630 | |
Roger Pau Monne | 0a8704a | 2012-10-24 18:58:45 +0200 | [diff] [blame] | 631 | if (ret) |
| 632 | continue; |
Konrad Rzeszutek Wilk | 1a95fe6 | 2011-04-15 11:35:13 -0400 | [diff] [blame] | 633 | |
Roger Pau Monne | 0a8704a | 2012-10-24 18:58:45 +0200 | [diff] [blame] | 634 | seg[i].buf = persistent_gnts[i]->dev_bus_addr | |
| 635 | (req->u.rw.seg[i].first_sect << 9); |
| 636 | } else { |
| 637 | pending_handle(pending_req, i) = map[j].handle; |
| 638 | bitmap_set(pending_req->unmap_seg, i, 1); |
Konrad Rzeszutek Wilk | 1a95fe6 | 2011-04-15 11:35:13 -0400 | [diff] [blame] | 639 | |
Roger Pau Monne | 0a8704a | 2012-10-24 18:58:45 +0200 | [diff] [blame] | 640 | if (ret) { |
| 641 | j++; |
| 642 | continue; |
| 643 | } |
| 644 | |
| 645 | seg[i].buf = map[j++].dev_bus_addr | |
| 646 | (req->u.rw.seg[i].first_sect << 9); |
| 647 | } |
Konrad Rzeszutek Wilk | 1a95fe6 | 2011-04-15 11:35:13 -0400 | [diff] [blame] | 648 | } |
| 649 | return ret; |
| 650 | } |
| 651 | |
Konrad Rzeszutek Wilk | 4214635 | 2011-10-12 17:26:47 -0400 | [diff] [blame] | 652 | static int dispatch_discard_io(struct xen_blkif *blkif, |
| 653 | struct blkif_request *req) |
Li Dongyang | b3cb0d6 | 2011-09-01 18:39:10 +0800 | [diff] [blame] | 654 | { |
| 655 | int err = 0; |
| 656 | int status = BLKIF_RSP_OKAY; |
| 657 | struct block_device *bdev = blkif->vbd.bdev; |
Konrad Rzeszutek Wilk | 4dae767 | 2012-03-13 18:43:23 -0400 | [diff] [blame] | 658 | unsigned long secure; |
Li Dongyang | b3cb0d6 | 2011-09-01 18:39:10 +0800 | [diff] [blame] | 659 | |
Konrad Rzeszutek Wilk | 4214635 | 2011-10-12 17:26:47 -0400 | [diff] [blame] | 660 | blkif->st_ds_req++; |
| 661 | |
| 662 | xen_blkif_get(blkif); |
Konrad Rzeszutek Wilk | 4dae767 | 2012-03-13 18:43:23 -0400 | [diff] [blame] | 663 | secure = (blkif->vbd.discard_secure && |
| 664 | (req->u.discard.flag & BLKIF_DISCARD_SECURE)) ? |
| 665 | BLKDEV_DISCARD_SECURE : 0; |
| 666 | |
| 667 | err = blkdev_issue_discard(bdev, req->u.discard.sector_number, |
| 668 | req->u.discard.nr_sectors, |
| 669 | GFP_KERNEL, secure); |
Li Dongyang | b3cb0d6 | 2011-09-01 18:39:10 +0800 | [diff] [blame] | 670 | |
| 671 | if (err == -EOPNOTSUPP) { |
| 672 | pr_debug(DRV_PFX "discard op failed, not supported\n"); |
| 673 | status = BLKIF_RSP_EOPNOTSUPP; |
| 674 | } else if (err) |
| 675 | status = BLKIF_RSP_ERROR; |
| 676 | |
Konrad Rzeszutek Wilk | 97e3683 | 2011-10-12 12:12:36 -0400 | [diff] [blame] | 677 | make_response(blkif, req->u.discard.id, req->operation, status); |
Konrad Rzeszutek Wilk | 4214635 | 2011-10-12 17:26:47 -0400 | [diff] [blame] | 678 | xen_blkif_put(blkif); |
| 679 | return err; |
Li Dongyang | b3cb0d6 | 2011-09-01 18:39:10 +0800 | [diff] [blame] | 680 | } |
| 681 | |
Konrad Rzeszutek Wilk | 29bde09 | 2011-10-10 00:42:22 -0400 | [diff] [blame] | 682 | static void xen_blk_drain_io(struct xen_blkif *blkif) |
| 683 | { |
| 684 | atomic_set(&blkif->drain, 1); |
| 685 | do { |
Konrad Rzeszutek Wilk | 6927d92 | 2011-10-17 14:27:48 -0400 | [diff] [blame] | 686 | /* The initial value is one, and one refcnt taken at the |
| 687 | * start of the xen_blkif_schedule thread. */ |
| 688 | if (atomic_read(&blkif->refcnt) <= 2) |
| 689 | break; |
Konrad Rzeszutek Wilk | 29bde09 | 2011-10-10 00:42:22 -0400 | [diff] [blame] | 690 | wait_for_completion_interruptible_timeout( |
| 691 | &blkif->drain_complete, HZ); |
| 692 | |
| 693 | if (!atomic_read(&blkif->drain)) |
| 694 | break; |
Konrad Rzeszutek Wilk | 29bde09 | 2011-10-10 00:42:22 -0400 | [diff] [blame] | 695 | } while (!kthread_should_stop()); |
| 696 | atomic_set(&blkif->drain, 0); |
| 697 | } |
| 698 | |
Konrad Rzeszutek Wilk | b0aef17 | 2011-04-15 10:58:05 -0400 | [diff] [blame] | 699 | /* |
Konrad Rzeszutek Wilk | a1397fa | 2011-04-14 17:05:23 -0400 | [diff] [blame] | 700 | * Completion callback on the bio's. Called as bh->b_end_io() |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 701 | */ |
| 702 | |
Konrad Rzeszutek Wilk | 2e9977c | 2011-04-14 17:42:07 -0400 | [diff] [blame] | 703 | static void __end_block_io_op(struct pending_req *pending_req, int error) |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 704 | { |
| 705 | /* An error fails the entire request. */ |
Konrad Rzeszutek Wilk | 24f567f | 2011-05-04 17:07:27 -0400 | [diff] [blame] | 706 | if ((pending_req->operation == BLKIF_OP_FLUSH_DISKCACHE) && |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 707 | (error == -EOPNOTSUPP)) { |
Konrad Rzeszutek Wilk | 22b20f2 | 2011-05-12 16:43:12 -0400 | [diff] [blame] | 708 | pr_debug(DRV_PFX "flush diskcache op failed, not supported\n"); |
Konrad Rzeszutek Wilk | 24f567f | 2011-05-04 17:07:27 -0400 | [diff] [blame] | 709 | xen_blkbk_flush_diskcache(XBT_NIL, pending_req->blkif->be, 0); |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 710 | pending_req->status = BLKIF_RSP_EOPNOTSUPP; |
Konrad Rzeszutek Wilk | 29bde09 | 2011-10-10 00:42:22 -0400 | [diff] [blame] | 711 | } else if ((pending_req->operation == BLKIF_OP_WRITE_BARRIER) && |
| 712 | (error == -EOPNOTSUPP)) { |
| 713 | pr_debug(DRV_PFX "write barrier op failed, not supported\n"); |
| 714 | xen_blkbk_barrier(XBT_NIL, pending_req->blkif->be, 0); |
| 715 | pending_req->status = BLKIF_RSP_EOPNOTSUPP; |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 716 | } else if (error) { |
Konrad Rzeszutek Wilk | 22b20f2 | 2011-05-12 16:43:12 -0400 | [diff] [blame] | 717 | pr_debug(DRV_PFX "Buffer not up-to-date at end of operation," |
Konrad Rzeszutek Wilk | ebe8190 | 2011-05-12 16:42:31 -0400 | [diff] [blame] | 718 | " error=%d\n", error); |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 719 | pending_req->status = BLKIF_RSP_ERROR; |
| 720 | } |
| 721 | |
Konrad Rzeszutek Wilk | 01f37f2 | 2011-05-11 15:57:09 -0400 | [diff] [blame] | 722 | /* |
| 723 | * If all of the bio's have completed it is time to unmap |
Konrad Rzeszutek Wilk | a1397fa | 2011-04-14 17:05:23 -0400 | [diff] [blame] | 724 | * the grant references associated with 'request' and provide |
Konrad Rzeszutek Wilk | 2e9977c | 2011-04-14 17:42:07 -0400 | [diff] [blame] | 725 | * the proper response on the ring. |
| 726 | */ |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 727 | if (atomic_dec_and_test(&pending_req->pendcnt)) { |
Konrad Rzeszutek Wilk | 9f3aedf | 2011-04-15 11:50:34 -0400 | [diff] [blame] | 728 | xen_blkbk_unmap(pending_req); |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 729 | make_response(pending_req->blkif, pending_req->id, |
| 730 | pending_req->operation, pending_req->status); |
Konrad Rzeszutek Wilk | 8b6bf74 | 2011-04-20 11:50:43 -0400 | [diff] [blame] | 731 | xen_blkif_put(pending_req->blkif); |
Konrad Rzeszutek Wilk | 29bde09 | 2011-10-10 00:42:22 -0400 | [diff] [blame] | 732 | if (atomic_read(&pending_req->blkif->refcnt) <= 2) { |
| 733 | if (atomic_read(&pending_req->blkif->drain)) |
| 734 | complete(&pending_req->blkif->drain_complete); |
| 735 | } |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 736 | free_req(pending_req); |
| 737 | } |
| 738 | } |
| 739 | |
Konrad Rzeszutek Wilk | a1397fa | 2011-04-14 17:05:23 -0400 | [diff] [blame] | 740 | /* |
| 741 | * bio callback. |
| 742 | */ |
Jeremy Fitzhardinge | 8812293 | 2009-02-09 12:05:51 -0800 | [diff] [blame] | 743 | static void end_block_io_op(struct bio *bio, int error) |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 744 | { |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 745 | __end_block_io_op(bio->bi_private, error); |
| 746 | bio_put(bio); |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 747 | } |
| 748 | |
| 749 | |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 750 | |
Konrad Rzeszutek Wilk | a1397fa | 2011-04-14 17:05:23 -0400 | [diff] [blame] | 751 | /* |
| 752 | * Function to copy the from the ring buffer the 'struct blkif_request' |
| 753 | * (which has the sectors we want, number of them, grant references, etc), |
| 754 | * and transmute it to the block API to hand it over to the proper block disk. |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 755 | */ |
Daniel Stodden | b4726a9 | 2011-05-28 13:21:10 -0700 | [diff] [blame] | 756 | static int |
| 757 | __do_block_io_op(struct xen_blkif *blkif) |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 758 | { |
Jeremy Fitzhardinge | 8812293 | 2009-02-09 12:05:51 -0800 | [diff] [blame] | 759 | union blkif_back_rings *blk_rings = &blkif->blk_rings; |
| 760 | struct blkif_request req; |
Konrad Rzeszutek Wilk | 2e9977c | 2011-04-14 17:42:07 -0400 | [diff] [blame] | 761 | struct pending_req *pending_req; |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 762 | RING_IDX rc, rp; |
| 763 | int more_to_do = 0; |
| 764 | |
| 765 | rc = blk_rings->common.req_cons; |
| 766 | rp = blk_rings->common.sring->req_prod; |
| 767 | rmb(); /* Ensure we see queued requests up to 'rp'. */ |
| 768 | |
| 769 | while (rc != rp) { |
| 770 | |
| 771 | if (RING_REQUEST_CONS_OVERFLOW(&blk_rings->common, rc)) |
| 772 | break; |
| 773 | |
Keir Fraser | 8270b45 | 2009-03-06 08:29:15 +0000 | [diff] [blame] | 774 | if (kthread_should_stop()) { |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 775 | more_to_do = 1; |
| 776 | break; |
| 777 | } |
| 778 | |
Keir Fraser | 8270b45 | 2009-03-06 08:29:15 +0000 | [diff] [blame] | 779 | pending_req = alloc_req(); |
| 780 | if (NULL == pending_req) { |
| 781 | blkif->st_oo_req++; |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 782 | more_to_do = 1; |
| 783 | break; |
| 784 | } |
| 785 | |
| 786 | switch (blkif->blk_protocol) { |
| 787 | case BLKIF_PROTOCOL_NATIVE: |
| 788 | memcpy(&req, RING_GET_REQUEST(&blk_rings->native, rc), sizeof(req)); |
| 789 | break; |
| 790 | case BLKIF_PROTOCOL_X86_32: |
| 791 | blkif_get_x86_32_req(&req, RING_GET_REQUEST(&blk_rings->x86_32, rc)); |
| 792 | break; |
| 793 | case BLKIF_PROTOCOL_X86_64: |
| 794 | blkif_get_x86_64_req(&req, RING_GET_REQUEST(&blk_rings->x86_64, rc)); |
| 795 | break; |
| 796 | default: |
| 797 | BUG(); |
| 798 | } |
| 799 | blk_rings->common.req_cons = ++rc; /* before make_response() */ |
| 800 | |
| 801 | /* Apply all sanity checks to /private copy/ of request. */ |
| 802 | barrier(); |
Konrad Rzeszutek Wilk | 4214635 | 2011-10-12 17:26:47 -0400 | [diff] [blame] | 803 | if (unlikely(req.operation == BLKIF_OP_DISCARD)) { |
| 804 | free_req(pending_req); |
| 805 | if (dispatch_discard_io(blkif, &req)) |
| 806 | break; |
| 807 | } else if (dispatch_rw_block_io(blkif, &req, pending_req)) |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 808 | break; |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 809 | |
| 810 | /* Yield point for this unbounded loop. */ |
| 811 | cond_resched(); |
| 812 | } |
| 813 | |
| 814 | return more_to_do; |
| 815 | } |
| 816 | |
Daniel Stodden | b4726a9 | 2011-05-28 13:21:10 -0700 | [diff] [blame] | 817 | static int |
| 818 | do_block_io_op(struct xen_blkif *blkif) |
| 819 | { |
| 820 | union blkif_back_rings *blk_rings = &blkif->blk_rings; |
| 821 | int more_to_do; |
| 822 | |
| 823 | do { |
| 824 | more_to_do = __do_block_io_op(blkif); |
| 825 | if (more_to_do) |
| 826 | break; |
| 827 | |
| 828 | RING_FINAL_CHECK_FOR_REQUESTS(&blk_rings->common, more_to_do); |
| 829 | } while (more_to_do); |
| 830 | |
| 831 | return more_to_do; |
| 832 | } |
Konrad Rzeszutek Wilk | a1397fa | 2011-04-14 17:05:23 -0400 | [diff] [blame] | 833 | /* |
Konrad Rzeszutek Wilk | 01f37f2 | 2011-05-11 15:57:09 -0400 | [diff] [blame] | 834 | * Transmutation of the 'struct blkif_request' to a proper 'struct bio' |
| 835 | * and call the 'submit_bio' to pass it to the underlying storage. |
Konrad Rzeszutek Wilk | a1397fa | 2011-04-14 17:05:23 -0400 | [diff] [blame] | 836 | */ |
Konrad Rzeszutek Wilk | 30fd150 | 2011-05-12 16:47:48 -0400 | [diff] [blame] | 837 | static int dispatch_rw_block_io(struct xen_blkif *blkif, |
| 838 | struct blkif_request *req, |
| 839 | struct pending_req *pending_req) |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 840 | { |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 841 | struct phys_req preq; |
Konrad Rzeszutek Wilk | 1a95fe6 | 2011-04-15 11:35:13 -0400 | [diff] [blame] | 842 | struct seg_buf seg[BLKIF_MAX_SEGMENTS_PER_REQUEST]; |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 843 | unsigned int nseg; |
| 844 | struct bio *bio = NULL; |
Konrad Rzeszutek Wilk | 7708992 | 2011-04-15 10:51:27 -0400 | [diff] [blame] | 845 | struct bio *biolist[BLKIF_MAX_SEGMENTS_PER_REQUEST]; |
Konrad Rzeszutek Wilk | 1a95fe6 | 2011-04-15 11:35:13 -0400 | [diff] [blame] | 846 | int i, nbio = 0; |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 847 | int operation; |
Konrad Rzeszutek Wilk | a19be5f | 2011-04-27 12:40:11 -0400 | [diff] [blame] | 848 | struct blk_plug plug; |
Konrad Rzeszutek Wilk | 29bde09 | 2011-10-10 00:42:22 -0400 | [diff] [blame] | 849 | bool drain = false; |
Roger Pau Monne | 0a8704a | 2012-10-24 18:58:45 +0200 | [diff] [blame] | 850 | struct page *pages[BLKIF_MAX_SEGMENTS_PER_REQUEST]; |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 851 | |
| 852 | switch (req->operation) { |
| 853 | case BLKIF_OP_READ: |
Konrad Rzeszutek Wilk | fc53bf7 | 2011-05-05 13:37:23 -0400 | [diff] [blame] | 854 | blkif->st_rd_req++; |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 855 | operation = READ; |
| 856 | break; |
| 857 | case BLKIF_OP_WRITE: |
Konrad Rzeszutek Wilk | fc53bf7 | 2011-05-05 13:37:23 -0400 | [diff] [blame] | 858 | blkif->st_wr_req++; |
Konrad Rzeszutek Wilk | 013c3ca | 2011-04-26 16:24:18 -0400 | [diff] [blame] | 859 | operation = WRITE_ODIRECT; |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 860 | break; |
Konrad Rzeszutek Wilk | 29bde09 | 2011-10-10 00:42:22 -0400 | [diff] [blame] | 861 | case BLKIF_OP_WRITE_BARRIER: |
| 862 | drain = true; |
Konrad Rzeszutek Wilk | 24f567f | 2011-05-04 17:07:27 -0400 | [diff] [blame] | 863 | case BLKIF_OP_FLUSH_DISKCACHE: |
Konrad Rzeszutek Wilk | fc53bf7 | 2011-05-05 13:37:23 -0400 | [diff] [blame] | 864 | blkif->st_f_req++; |
Konrad Rzeszutek Wilk | 24f567f | 2011-05-04 17:07:27 -0400 | [diff] [blame] | 865 | operation = WRITE_FLUSH; |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 866 | break; |
| 867 | default: |
| 868 | operation = 0; /* make gcc happy */ |
Konrad Rzeszutek Wilk | fc53bf7 | 2011-05-05 13:37:23 -0400 | [diff] [blame] | 869 | goto fail_response; |
| 870 | break; |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 871 | } |
| 872 | |
Konrad Rzeszutek Wilk | 4214635 | 2011-10-12 17:26:47 -0400 | [diff] [blame] | 873 | /* Check that the number of segments is sane. */ |
| 874 | nseg = req->u.rw.nr_segments; |
Konrad Rzeszutek Wilk | 97e3683 | 2011-10-12 12:12:36 -0400 | [diff] [blame] | 875 | |
Konrad Rzeszutek Wilk | 4214635 | 2011-10-12 17:26:47 -0400 | [diff] [blame] | 876 | if (unlikely(nseg == 0 && operation != WRITE_FLUSH) || |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 877 | unlikely(nseg > BLKIF_MAX_SEGMENTS_PER_REQUEST)) { |
Konrad Rzeszutek Wilk | 22b20f2 | 2011-05-12 16:43:12 -0400 | [diff] [blame] | 878 | pr_debug(DRV_PFX "Bad number of segments in request (%d)\n", |
Konrad Rzeszutek Wilk | ebe8190 | 2011-05-12 16:42:31 -0400 | [diff] [blame] | 879 | nseg); |
Konrad Rzeszutek Wilk | 1a95fe6 | 2011-04-15 11:35:13 -0400 | [diff] [blame] | 880 | /* Haven't submitted any bio's yet. */ |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 881 | goto fail_response; |
| 882 | } |
| 883 | |
Konrad Rzeszutek Wilk | c35950b | 2011-03-01 16:22:28 -0500 | [diff] [blame] | 884 | preq.sector_number = req->u.rw.sector_number; |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 885 | preq.nr_sects = 0; |
| 886 | |
| 887 | pending_req->blkif = blkif; |
Konrad Rzeszutek Wilk | 97e3683 | 2011-10-12 12:12:36 -0400 | [diff] [blame] | 888 | pending_req->id = req->u.rw.id; |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 889 | pending_req->operation = req->operation; |
| 890 | pending_req->status = BLKIF_RSP_OKAY; |
| 891 | pending_req->nr_pages = nseg; |
Konrad Rzeszutek Wilk | e935049 | 2011-04-18 11:34:55 -0400 | [diff] [blame] | 892 | |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 893 | for (i = 0; i < nseg; i++) { |
Konrad Rzeszutek Wilk | c35950b | 2011-03-01 16:22:28 -0500 | [diff] [blame] | 894 | seg[i].nsec = req->u.rw.seg[i].last_sect - |
| 895 | req->u.rw.seg[i].first_sect + 1; |
Konrad Rzeszutek Wilk | c35950b | 2011-03-01 16:22:28 -0500 | [diff] [blame] | 896 | if ((req->u.rw.seg[i].last_sect >= (PAGE_SIZE >> 9)) || |
| 897 | (req->u.rw.seg[i].last_sect < req->u.rw.seg[i].first_sect)) |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 898 | goto fail_response; |
| 899 | preq.nr_sects += seg[i].nsec; |
Konrad Rzeszutek Wilk | 976222e | 2011-04-15 11:38:29 -0400 | [diff] [blame] | 900 | |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 901 | } |
| 902 | |
Konrad Rzeszutek Wilk | 3d81473 | 2011-05-12 16:53:56 -0400 | [diff] [blame] | 903 | if (xen_vbd_translate(&preq, blkif, operation) != 0) { |
Konrad Rzeszutek Wilk | 22b20f2 | 2011-05-12 16:43:12 -0400 | [diff] [blame] | 904 | pr_debug(DRV_PFX "access denied: %s of [%llu,%llu] on dev=%04x\n", |
Konrad Rzeszutek Wilk | ebe8190 | 2011-05-12 16:42:31 -0400 | [diff] [blame] | 905 | operation == READ ? "read" : "write", |
| 906 | preq.sector_number, |
Chen Gang | a72d900 | 2013-02-28 10:34:23 +0800 | [diff] [blame^] | 907 | preq.sector_number + preq.nr_sects, |
| 908 | blkif->vbd.pdevice); |
Konrad Rzeszutek Wilk | 1a95fe6 | 2011-04-15 11:35:13 -0400 | [diff] [blame] | 909 | goto fail_response; |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 910 | } |
Konrad Rzeszutek Wilk | 01f37f2 | 2011-05-11 15:57:09 -0400 | [diff] [blame] | 911 | |
| 912 | /* |
Konrad Rzeszutek Wilk | 3d81473 | 2011-05-12 16:53:56 -0400 | [diff] [blame] | 913 | * This check _MUST_ be done after xen_vbd_translate as the preq.bdev |
Konrad Rzeszutek Wilk | 01f37f2 | 2011-05-11 15:57:09 -0400 | [diff] [blame] | 914 | * is set there. |
| 915 | */ |
Konrad Rzeszutek Wilk | e935049 | 2011-04-18 11:34:55 -0400 | [diff] [blame] | 916 | for (i = 0; i < nseg; i++) { |
| 917 | if (((int)preq.sector_number|(int)seg[i].nsec) & |
| 918 | ((bdev_logical_block_size(preq.bdev) >> 9) - 1)) { |
Konrad Rzeszutek Wilk | 22b20f2 | 2011-05-12 16:43:12 -0400 | [diff] [blame] | 919 | pr_debug(DRV_PFX "Misaligned I/O request from domain %d", |
Konrad Rzeszutek Wilk | ebe8190 | 2011-05-12 16:42:31 -0400 | [diff] [blame] | 920 | blkif->domid); |
Konrad Rzeszutek Wilk | e935049 | 2011-04-18 11:34:55 -0400 | [diff] [blame] | 921 | goto fail_response; |
| 922 | } |
| 923 | } |
Konrad Rzeszutek Wilk | 01f37f2 | 2011-05-11 15:57:09 -0400 | [diff] [blame] | 924 | |
Konrad Rzeszutek Wilk | 29bde09 | 2011-10-10 00:42:22 -0400 | [diff] [blame] | 925 | /* Wait on all outstanding I/O's and once that has been completed |
| 926 | * issue the WRITE_FLUSH. |
| 927 | */ |
| 928 | if (drain) |
| 929 | xen_blk_drain_io(pending_req->blkif); |
| 930 | |
Konrad Rzeszutek Wilk | 01f37f2 | 2011-05-11 15:57:09 -0400 | [diff] [blame] | 931 | /* |
| 932 | * If we have failed at this point, we need to undo the M2P override, |
Konrad Rzeszutek Wilk | 1a95fe6 | 2011-04-15 11:35:13 -0400 | [diff] [blame] | 933 | * set gnttab_set_unmap_op on all of the grant references and perform |
| 934 | * the hypercall to unmap the grants - that is all done in |
Konrad Rzeszutek Wilk | 9f3aedf | 2011-04-15 11:50:34 -0400 | [diff] [blame] | 935 | * xen_blkbk_unmap. |
Konrad Rzeszutek Wilk | 1a95fe6 | 2011-04-15 11:35:13 -0400 | [diff] [blame] | 936 | */ |
Roger Pau Monne | 0a8704a | 2012-10-24 18:58:45 +0200 | [diff] [blame] | 937 | if (xen_blkbk_map(req, pending_req, seg, pages)) |
Konrad Rzeszutek Wilk | 1a95fe6 | 2011-04-15 11:35:13 -0400 | [diff] [blame] | 938 | goto fail_flush; |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 939 | |
Li Dongyang | b3cb0d6 | 2011-09-01 18:39:10 +0800 | [diff] [blame] | 940 | /* |
| 941 | * This corresponding xen_blkif_put is done in __end_block_io_op, or |
| 942 | * below (in "!bio") if we are handling a BLKIF_OP_DISCARD. |
| 943 | */ |
Konrad Rzeszutek Wilk | 8b6bf74 | 2011-04-20 11:50:43 -0400 | [diff] [blame] | 944 | xen_blkif_get(blkif); |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 945 | |
| 946 | for (i = 0; i < nseg; i++) { |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 947 | while ((bio == NULL) || |
| 948 | (bio_add_page(bio, |
Roger Pau Monne | 0a8704a | 2012-10-24 18:58:45 +0200 | [diff] [blame] | 949 | pages[i], |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 950 | seg[i].nsec << 9, |
| 951 | seg[i].buf & ~PAGE_MASK) == 0)) { |
Konrad Rzeszutek Wilk | 2e9977c | 2011-04-14 17:42:07 -0400 | [diff] [blame] | 952 | |
Konrad Rzeszutek Wilk | 03e0edf | 2011-05-12 16:19:23 -0400 | [diff] [blame] | 953 | bio = bio_alloc(GFP_KERNEL, nseg-i); |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 954 | if (unlikely(bio == NULL)) |
| 955 | goto fail_put_bio; |
| 956 | |
Konrad Rzeszutek Wilk | 03e0edf | 2011-05-12 16:19:23 -0400 | [diff] [blame] | 957 | biolist[nbio++] = bio; |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 958 | bio->bi_bdev = preq.bdev; |
| 959 | bio->bi_private = pending_req; |
| 960 | bio->bi_end_io = end_block_io_op; |
| 961 | bio->bi_sector = preq.sector_number; |
| 962 | } |
| 963 | |
| 964 | preq.sector_number += seg[i].nsec; |
| 965 | } |
| 966 | |
Li Dongyang | b3cb0d6 | 2011-09-01 18:39:10 +0800 | [diff] [blame] | 967 | /* This will be hit if the operation was a flush or discard. */ |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 968 | if (!bio) { |
Konrad Rzeszutek Wilk | 4214635 | 2011-10-12 17:26:47 -0400 | [diff] [blame] | 969 | BUG_ON(operation != WRITE_FLUSH); |
Konrad Rzeszutek Wilk | b0f8012 | 2011-05-12 16:23:06 -0400 | [diff] [blame] | 970 | |
Konrad Rzeszutek Wilk | 4214635 | 2011-10-12 17:26:47 -0400 | [diff] [blame] | 971 | bio = bio_alloc(GFP_KERNEL, 0); |
| 972 | if (unlikely(bio == NULL)) |
| 973 | goto fail_put_bio; |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 974 | |
Konrad Rzeszutek Wilk | 4214635 | 2011-10-12 17:26:47 -0400 | [diff] [blame] | 975 | biolist[nbio++] = bio; |
| 976 | bio->bi_bdev = preq.bdev; |
| 977 | bio->bi_private = pending_req; |
| 978 | bio->bi_end_io = end_block_io_op; |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 979 | } |
| 980 | |
Konrad Rzeszutek Wilk | 01f37f2 | 2011-05-11 15:57:09 -0400 | [diff] [blame] | 981 | /* |
| 982 | * We set it one so that the last submit_bio does not have to call |
Konrad Rzeszutek Wilk | 7708992 | 2011-04-15 10:51:27 -0400 | [diff] [blame] | 983 | * atomic_inc. |
| 984 | */ |
| 985 | atomic_set(&pending_req->pendcnt, nbio); |
| 986 | |
Konrad Rzeszutek Wilk | a19be5f | 2011-04-27 12:40:11 -0400 | [diff] [blame] | 987 | /* Get a reference count for the disk queue and start sending I/O */ |
| 988 | blk_start_plug(&plug); |
| 989 | |
Konrad Rzeszutek Wilk | 7708992 | 2011-04-15 10:51:27 -0400 | [diff] [blame] | 990 | for (i = 0; i < nbio; i++) |
| 991 | submit_bio(operation, biolist[i]); |
| 992 | |
Konrad Rzeszutek Wilk | a19be5f | 2011-04-27 12:40:11 -0400 | [diff] [blame] | 993 | /* Let the I/Os go.. */ |
Konrad Rzeszutek Wilk | 3d68b39 | 2011-05-05 13:42:10 -0400 | [diff] [blame] | 994 | blk_finish_plug(&plug); |
Konrad Rzeszutek Wilk | a19be5f | 2011-04-27 12:40:11 -0400 | [diff] [blame] | 995 | |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 996 | if (operation == READ) |
| 997 | blkif->st_rd_sect += preq.nr_sects; |
Konrad Rzeszutek Wilk | 5c62cb4 | 2011-10-10 12:33:21 -0400 | [diff] [blame] | 998 | else if (operation & WRITE) |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 999 | blkif->st_wr_sect += preq.nr_sects; |
| 1000 | |
Konrad Rzeszutek Wilk | fc53bf7 | 2011-05-05 13:37:23 -0400 | [diff] [blame] | 1001 | return 0; |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 1002 | |
| 1003 | fail_flush: |
Konrad Rzeszutek Wilk | 9f3aedf | 2011-04-15 11:50:34 -0400 | [diff] [blame] | 1004 | xen_blkbk_unmap(pending_req); |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 1005 | fail_response: |
Konrad Rzeszutek Wilk | 0faa8cc | 2011-04-14 17:58:19 -0400 | [diff] [blame] | 1006 | /* Haven't submitted any bio's yet. */ |
Konrad Rzeszutek Wilk | 97e3683 | 2011-10-12 12:12:36 -0400 | [diff] [blame] | 1007 | make_response(blkif, req->u.rw.id, req->operation, BLKIF_RSP_ERROR); |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 1008 | free_req(pending_req); |
| 1009 | msleep(1); /* back off a bit */ |
Konrad Rzeszutek Wilk | fc53bf7 | 2011-05-05 13:37:23 -0400 | [diff] [blame] | 1010 | return -EIO; |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 1011 | |
| 1012 | fail_put_bio: |
Konrad Rzeszutek Wilk | 03e0edf | 2011-05-12 16:19:23 -0400 | [diff] [blame] | 1013 | for (i = 0; i < nbio; i++) |
Konrad Rzeszutek Wilk | 7708992 | 2011-04-15 10:51:27 -0400 | [diff] [blame] | 1014 | bio_put(biolist[i]); |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 1015 | __end_block_io_op(pending_req, -EINVAL); |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 1016 | msleep(1); /* back off a bit */ |
Konrad Rzeszutek Wilk | fc53bf7 | 2011-05-05 13:37:23 -0400 | [diff] [blame] | 1017 | return -EIO; |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 1018 | } |
| 1019 | |
| 1020 | |
| 1021 | |
Konrad Rzeszutek Wilk | a1397fa | 2011-04-14 17:05:23 -0400 | [diff] [blame] | 1022 | /* |
| 1023 | * Put a response on the ring on how the operation fared. |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 1024 | */ |
Konrad Rzeszutek Wilk | 30fd150 | 2011-05-12 16:47:48 -0400 | [diff] [blame] | 1025 | static void make_response(struct xen_blkif *blkif, u64 id, |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 1026 | unsigned short op, int st) |
| 1027 | { |
Jeremy Fitzhardinge | 8812293 | 2009-02-09 12:05:51 -0800 | [diff] [blame] | 1028 | struct blkif_response resp; |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 1029 | unsigned long flags; |
Jeremy Fitzhardinge | 8812293 | 2009-02-09 12:05:51 -0800 | [diff] [blame] | 1030 | union blkif_back_rings *blk_rings = &blkif->blk_rings; |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 1031 | int notify; |
| 1032 | |
| 1033 | resp.id = id; |
| 1034 | resp.operation = op; |
| 1035 | resp.status = st; |
| 1036 | |
| 1037 | spin_lock_irqsave(&blkif->blk_ring_lock, flags); |
| 1038 | /* Place on the response ring for the relevant domain. */ |
| 1039 | switch (blkif->blk_protocol) { |
| 1040 | case BLKIF_PROTOCOL_NATIVE: |
| 1041 | memcpy(RING_GET_RESPONSE(&blk_rings->native, blk_rings->native.rsp_prod_pvt), |
| 1042 | &resp, sizeof(resp)); |
| 1043 | break; |
| 1044 | case BLKIF_PROTOCOL_X86_32: |
| 1045 | memcpy(RING_GET_RESPONSE(&blk_rings->x86_32, blk_rings->x86_32.rsp_prod_pvt), |
| 1046 | &resp, sizeof(resp)); |
| 1047 | break; |
| 1048 | case BLKIF_PROTOCOL_X86_64: |
| 1049 | memcpy(RING_GET_RESPONSE(&blk_rings->x86_64, blk_rings->x86_64.rsp_prod_pvt), |
| 1050 | &resp, sizeof(resp)); |
| 1051 | break; |
| 1052 | default: |
| 1053 | BUG(); |
| 1054 | } |
| 1055 | blk_rings->common.rsp_prod_pvt++; |
| 1056 | RING_PUSH_RESPONSES_AND_CHECK_NOTIFY(&blk_rings->common, notify); |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 1057 | spin_unlock_irqrestore(&blkif->blk_ring_lock, flags); |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 1058 | if (notify) |
| 1059 | notify_remote_via_irq(blkif->irq); |
| 1060 | } |
| 1061 | |
Konrad Rzeszutek Wilk | 8b6bf74 | 2011-04-20 11:50:43 -0400 | [diff] [blame] | 1062 | static int __init xen_blkif_init(void) |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 1063 | { |
| 1064 | int i, mmap_pages; |
Konrad Rzeszutek Wilk | 8770b26 | 2009-10-08 13:23:09 -0400 | [diff] [blame] | 1065 | int rc = 0; |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 1066 | |
Daniel De Graaf | b2167ba | 2011-11-28 11:49:05 -0500 | [diff] [blame] | 1067 | if (!xen_domain()) |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 1068 | return -ENODEV; |
| 1069 | |
Konrad Rzeszutek Wilk | 2e9977c | 2011-04-14 17:42:07 -0400 | [diff] [blame] | 1070 | blkbk = kzalloc(sizeof(struct xen_blkbk), GFP_KERNEL); |
Konrad Rzeszutek Wilk | e8e2887 | 2011-02-25 10:51:29 -0500 | [diff] [blame] | 1071 | if (!blkbk) { |
Konrad Rzeszutek Wilk | 22b20f2 | 2011-05-12 16:43:12 -0400 | [diff] [blame] | 1072 | pr_alert(DRV_PFX "%s: out of memory!\n", __func__); |
Konrad Rzeszutek Wilk | e8e2887 | 2011-02-25 10:51:29 -0500 | [diff] [blame] | 1073 | return -ENOMEM; |
| 1074 | } |
| 1075 | |
Konrad Rzeszutek Wilk | 8b6bf74 | 2011-04-20 11:50:43 -0400 | [diff] [blame] | 1076 | mmap_pages = xen_blkif_reqs * BLKIF_MAX_SEGMENTS_PER_REQUEST; |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 1077 | |
Jan Beulich | 8e6dc6f | 2011-09-16 08:38:09 +0100 | [diff] [blame] | 1078 | blkbk->pending_reqs = kzalloc(sizeof(blkbk->pending_reqs[0]) * |
Konrad Rzeszutek Wilk | 8b6bf74 | 2011-04-20 11:50:43 -0400 | [diff] [blame] | 1079 | xen_blkif_reqs, GFP_KERNEL); |
Jan Beulich | 8e6dc6f | 2011-09-16 08:38:09 +0100 | [diff] [blame] | 1080 | blkbk->pending_grant_handles = kmalloc(sizeof(blkbk->pending_grant_handles[0]) * |
Konrad Rzeszutek Wilk | a742b02 | 2011-03-14 12:41:26 -0400 | [diff] [blame] | 1081 | mmap_pages, GFP_KERNEL); |
| 1082 | blkbk->pending_pages = kzalloc(sizeof(blkbk->pending_pages[0]) * |
| 1083 | mmap_pages, GFP_KERNEL); |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 1084 | |
Konrad Rzeszutek Wilk | 2e9977c | 2011-04-14 17:42:07 -0400 | [diff] [blame] | 1085 | if (!blkbk->pending_reqs || !blkbk->pending_grant_handles || |
| 1086 | !blkbk->pending_pages) { |
Konrad Rzeszutek Wilk | 8770b26 | 2009-10-08 13:23:09 -0400 | [diff] [blame] | 1087 | rc = -ENOMEM; |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 1088 | goto out_of_memory; |
Konrad Rzeszutek Wilk | 8770b26 | 2009-10-08 13:23:09 -0400 | [diff] [blame] | 1089 | } |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 1090 | |
Konrad Rzeszutek Wilk | 464fb41 | 2011-03-01 16:26:10 -0500 | [diff] [blame] | 1091 | for (i = 0; i < mmap_pages; i++) { |
Konrad Rzeszutek Wilk | e8e2887 | 2011-02-25 10:51:29 -0500 | [diff] [blame] | 1092 | blkbk->pending_grant_handles[i] = BLKBACK_INVALID_HANDLE; |
Konrad Rzeszutek Wilk | a742b02 | 2011-03-14 12:41:26 -0400 | [diff] [blame] | 1093 | blkbk->pending_pages[i] = alloc_page(GFP_KERNEL); |
Konrad Rzeszutek Wilk | 464fb41 | 2011-03-01 16:26:10 -0500 | [diff] [blame] | 1094 | if (blkbk->pending_pages[i] == NULL) { |
| 1095 | rc = -ENOMEM; |
| 1096 | goto out_of_memory; |
| 1097 | } |
| 1098 | } |
Konrad Rzeszutek Wilk | 8b6bf74 | 2011-04-20 11:50:43 -0400 | [diff] [blame] | 1099 | rc = xen_blkif_interface_init(); |
Konrad Rzeszutek Wilk | 8770b26 | 2009-10-08 13:23:09 -0400 | [diff] [blame] | 1100 | if (rc) |
| 1101 | goto failed_init; |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 1102 | |
Konrad Rzeszutek Wilk | e8e2887 | 2011-02-25 10:51:29 -0500 | [diff] [blame] | 1103 | INIT_LIST_HEAD(&blkbk->pending_free); |
| 1104 | spin_lock_init(&blkbk->pending_free_lock); |
| 1105 | init_waitqueue_head(&blkbk->pending_free_wq); |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 1106 | |
Konrad Rzeszutek Wilk | 8b6bf74 | 2011-04-20 11:50:43 -0400 | [diff] [blame] | 1107 | for (i = 0; i < xen_blkif_reqs; i++) |
Konrad Rzeszutek Wilk | 2e9977c | 2011-04-14 17:42:07 -0400 | [diff] [blame] | 1108 | list_add_tail(&blkbk->pending_reqs[i].free_list, |
| 1109 | &blkbk->pending_free); |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 1110 | |
Konrad Rzeszutek Wilk | 8b6bf74 | 2011-04-20 11:50:43 -0400 | [diff] [blame] | 1111 | rc = xen_blkif_xenbus_init(); |
Konrad Rzeszutek Wilk | 8770b26 | 2009-10-08 13:23:09 -0400 | [diff] [blame] | 1112 | if (rc) |
| 1113 | goto failed_init; |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 1114 | |
| 1115 | return 0; |
| 1116 | |
| 1117 | out_of_memory: |
Konrad Rzeszutek Wilk | 22b20f2 | 2011-05-12 16:43:12 -0400 | [diff] [blame] | 1118 | pr_alert(DRV_PFX "%s: out of memory\n", __func__); |
Konrad Rzeszutek Wilk | 8770b26 | 2009-10-08 13:23:09 -0400 | [diff] [blame] | 1119 | failed_init: |
Konrad Rzeszutek Wilk | e8e2887 | 2011-02-25 10:51:29 -0500 | [diff] [blame] | 1120 | kfree(blkbk->pending_reqs); |
Konrad Rzeszutek Wilk | a742b02 | 2011-03-14 12:41:26 -0400 | [diff] [blame] | 1121 | kfree(blkbk->pending_grant_handles); |
Dan Carpenter | 9b83c77 | 2011-05-27 09:27:16 +0300 | [diff] [blame] | 1122 | if (blkbk->pending_pages) { |
| 1123 | for (i = 0; i < mmap_pages; i++) { |
| 1124 | if (blkbk->pending_pages[i]) |
| 1125 | __free_page(blkbk->pending_pages[i]); |
| 1126 | } |
| 1127 | kfree(blkbk->pending_pages); |
Konrad Rzeszutek Wilk | 464fb41 | 2011-03-01 16:26:10 -0500 | [diff] [blame] | 1128 | } |
Konrad Rzeszutek Wilk | a742b02 | 2011-03-14 12:41:26 -0400 | [diff] [blame] | 1129 | kfree(blkbk); |
Konrad Rzeszutek Wilk | e8e2887 | 2011-02-25 10:51:29 -0500 | [diff] [blame] | 1130 | blkbk = NULL; |
Konrad Rzeszutek Wilk | 8770b26 | 2009-10-08 13:23:09 -0400 | [diff] [blame] | 1131 | return rc; |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 1132 | } |
| 1133 | |
Konrad Rzeszutek Wilk | 8b6bf74 | 2011-04-20 11:50:43 -0400 | [diff] [blame] | 1134 | module_init(xen_blkif_init); |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 1135 | |
| 1136 | MODULE_LICENSE("Dual BSD/GPL"); |
Bastian Blank | a7e9357 | 2011-06-29 14:40:50 +0200 | [diff] [blame] | 1137 | MODULE_ALIAS("xen-backend:vbd"); |