Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 1 | /****************************************************************************** |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 2 | * |
| 3 | * Back-end of the driver for virtual block devices. This portion of the |
| 4 | * driver exports a 'unified' block-device interface that can be accessed |
| 5 | * by any operating system that implements a compatible front end. A |
| 6 | * reference front-end implementation can be found in: |
Konrad Rzeszutek Wilk | a1397fa | 2011-04-14 17:05:23 -0400 | [diff] [blame] | 7 | * drivers/block/xen-blkfront.c |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 8 | * |
| 9 | * Copyright (c) 2003-2004, Keir Fraser & Steve Hand |
| 10 | * Copyright (c) 2005, Christopher Clark |
| 11 | * |
| 12 | * This program is free software; you can redistribute it and/or |
| 13 | * modify it under the terms of the GNU General Public License version 2 |
| 14 | * as published by the Free Software Foundation; or, when distributed |
| 15 | * separately from the Linux kernel or incorporated into other |
| 16 | * software packages, subject to the following license: |
| 17 | * |
| 18 | * Permission is hereby granted, free of charge, to any person obtaining a copy |
| 19 | * of this source file (the "Software"), to deal in the Software without |
| 20 | * restriction, including without limitation the rights to use, copy, modify, |
| 21 | * merge, publish, distribute, sublicense, and/or sell copies of the Software, |
| 22 | * and to permit persons to whom the Software is furnished to do so, subject to |
| 23 | * the following conditions: |
| 24 | * |
| 25 | * The above copyright notice and this permission notice shall be included in |
| 26 | * all copies or substantial portions of the Software. |
| 27 | * |
| 28 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
| 29 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
| 30 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE |
| 31 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER |
| 32 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING |
| 33 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS |
| 34 | * IN THE SOFTWARE. |
| 35 | */ |
| 36 | |
| 37 | #include <linux/spinlock.h> |
| 38 | #include <linux/kthread.h> |
| 39 | #include <linux/list.h> |
| 40 | #include <linux/delay.h> |
Jeremy Fitzhardinge | 8812293 | 2009-02-09 12:05:51 -0800 | [diff] [blame] | 41 | #include <linux/freezer.h> |
Roger Pau Monne | 0a8704a | 2012-10-24 18:58:45 +0200 | [diff] [blame^] | 42 | #include <linux/bitmap.h> |
Jeremy Fitzhardinge | afd91d0 | 2009-09-15 14:12:37 -0700 | [diff] [blame] | 43 | |
Jeremy Fitzhardinge | 8812293 | 2009-02-09 12:05:51 -0800 | [diff] [blame] | 44 | #include <xen/events.h> |
| 45 | #include <xen/page.h> |
Stefano Stabellini | e79affc | 2012-08-08 17:21:14 +0000 | [diff] [blame] | 46 | #include <xen/xen.h> |
Jeremy Fitzhardinge | 8812293 | 2009-02-09 12:05:51 -0800 | [diff] [blame] | 47 | #include <asm/xen/hypervisor.h> |
| 48 | #include <asm/xen/hypercall.h> |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 49 | #include "common.h" |
| 50 | |
| 51 | /* |
| 52 | * These are rather arbitrary. They are fairly large because adjacent requests |
| 53 | * pulled from a communication ring are quite likely to end up being part of |
| 54 | * the same scatter/gather request at the disc. |
| 55 | * |
Konrad Rzeszutek Wilk | 8b6bf74 | 2011-04-20 11:50:43 -0400 | [diff] [blame] | 56 | * ** TRY INCREASING 'xen_blkif_reqs' IF WRITE SPEEDS SEEM TOO LOW ** |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 57 | * |
| 58 | * This will increase the chances of being able to write whole tracks. |
| 59 | * 64 should be enough to keep us competitive with Linux. |
| 60 | */ |
Konrad Rzeszutek Wilk | 8b6bf74 | 2011-04-20 11:50:43 -0400 | [diff] [blame] | 61 | static int xen_blkif_reqs = 64; |
| 62 | module_param_named(reqs, xen_blkif_reqs, int, 0); |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 63 | MODULE_PARM_DESC(reqs, "Number of blkback requests to allocate"); |
| 64 | |
| 65 | /* Run-time switchable: /sys/module/blkback/parameters/ */ |
Konrad Rzeszutek Wilk | 2e9977c | 2011-04-14 17:42:07 -0400 | [diff] [blame] | 66 | static unsigned int log_stats; |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 67 | module_param(log_stats, int, 0644); |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 68 | |
| 69 | /* |
| 70 | * Each outstanding request that we've passed to the lower device layers has a |
| 71 | * 'pending_req' allocated to it. Each buffer_head that completes decrements |
| 72 | * the pendcnt towards zero. When it hits zero, the specified domain has a |
| 73 | * response queued for it, with the saved 'id' passed back. |
| 74 | */ |
Konrad Rzeszutek Wilk | 2e9977c | 2011-04-14 17:42:07 -0400 | [diff] [blame] | 75 | struct pending_req { |
Konrad Rzeszutek Wilk | 30fd150 | 2011-05-12 16:47:48 -0400 | [diff] [blame] | 76 | struct xen_blkif *blkif; |
Konrad Rzeszutek Wilk | 01f37f2 | 2011-05-11 15:57:09 -0400 | [diff] [blame] | 77 | u64 id; |
| 78 | int nr_pages; |
| 79 | atomic_t pendcnt; |
| 80 | unsigned short operation; |
| 81 | int status; |
| 82 | struct list_head free_list; |
Roger Pau Monne | 0a8704a | 2012-10-24 18:58:45 +0200 | [diff] [blame^] | 83 | DECLARE_BITMAP(unmap_seg, BLKIF_MAX_SEGMENTS_PER_REQUEST); |
Konrad Rzeszutek Wilk | 2e9977c | 2011-04-14 17:42:07 -0400 | [diff] [blame] | 84 | }; |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 85 | |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 86 | #define BLKBACK_INVALID_HANDLE (~0) |
| 87 | |
Konrad Rzeszutek Wilk | e8e2887 | 2011-02-25 10:51:29 -0500 | [diff] [blame] | 88 | struct xen_blkbk { |
Konrad Rzeszutek Wilk | 2e9977c | 2011-04-14 17:42:07 -0400 | [diff] [blame] | 89 | struct pending_req *pending_reqs; |
Konrad Rzeszutek Wilk | a1397fa | 2011-04-14 17:05:23 -0400 | [diff] [blame] | 90 | /* List of all 'pending_req' available */ |
Konrad Rzeszutek Wilk | e8e2887 | 2011-02-25 10:51:29 -0500 | [diff] [blame] | 91 | struct list_head pending_free; |
Konrad Rzeszutek Wilk | a1397fa | 2011-04-14 17:05:23 -0400 | [diff] [blame] | 92 | /* And its spinlock. */ |
Konrad Rzeszutek Wilk | e8e2887 | 2011-02-25 10:51:29 -0500 | [diff] [blame] | 93 | spinlock_t pending_free_lock; |
| 94 | wait_queue_head_t pending_free_wq; |
Konrad Rzeszutek Wilk | a1397fa | 2011-04-14 17:05:23 -0400 | [diff] [blame] | 95 | /* The list of all pages that are available. */ |
Konrad Rzeszutek Wilk | e8e2887 | 2011-02-25 10:51:29 -0500 | [diff] [blame] | 96 | struct page **pending_pages; |
Konrad Rzeszutek Wilk | a1397fa | 2011-04-14 17:05:23 -0400 | [diff] [blame] | 97 | /* And the grant handles that are available. */ |
Konrad Rzeszutek Wilk | e8e2887 | 2011-02-25 10:51:29 -0500 | [diff] [blame] | 98 | grant_handle_t *pending_grant_handles; |
| 99 | }; |
| 100 | |
| 101 | static struct xen_blkbk *blkbk; |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 102 | |
Konrad Rzeszutek Wilk | a1397fa | 2011-04-14 17:05:23 -0400 | [diff] [blame] | 103 | /* |
Roger Pau Monne | 0a8704a | 2012-10-24 18:58:45 +0200 | [diff] [blame^] | 104 | * Maximum number of grant pages that can be mapped in blkback. |
| 105 | * BLKIF_MAX_SEGMENTS_PER_REQUEST * RING_SIZE is the maximum number of |
| 106 | * pages that blkback will persistently map. |
| 107 | * Currently, this is: |
| 108 | * RING_SIZE = 32 (for all known ring types) |
| 109 | * BLKIF_MAX_SEGMENTS_PER_REQUEST = 11 |
| 110 | * sizeof(struct persistent_gnt) = 48 |
| 111 | * So the maximum memory used to store the grants is: |
| 112 | * 32 * 11 * 48 = 16896 bytes |
| 113 | */ |
| 114 | static inline unsigned int max_mapped_grant_pages(enum blkif_protocol protocol) |
| 115 | { |
| 116 | switch (protocol) { |
| 117 | case BLKIF_PROTOCOL_NATIVE: |
| 118 | return __CONST_RING_SIZE(blkif, PAGE_SIZE) * |
| 119 | BLKIF_MAX_SEGMENTS_PER_REQUEST; |
| 120 | case BLKIF_PROTOCOL_X86_32: |
| 121 | return __CONST_RING_SIZE(blkif_x86_32, PAGE_SIZE) * |
| 122 | BLKIF_MAX_SEGMENTS_PER_REQUEST; |
| 123 | case BLKIF_PROTOCOL_X86_64: |
| 124 | return __CONST_RING_SIZE(blkif_x86_64, PAGE_SIZE) * |
| 125 | BLKIF_MAX_SEGMENTS_PER_REQUEST; |
| 126 | default: |
| 127 | BUG(); |
| 128 | } |
| 129 | return 0; |
| 130 | } |
| 131 | |
| 132 | |
| 133 | /* |
Konrad Rzeszutek Wilk | a1397fa | 2011-04-14 17:05:23 -0400 | [diff] [blame] | 134 | * Little helpful macro to figure out the index and virtual address of the |
| 135 | * pending_pages[..]. For each 'pending_req' we have have up to |
| 136 | * BLKIF_MAX_SEGMENTS_PER_REQUEST (11) pages. The seg would be from 0 through |
Konrad Rzeszutek Wilk | 01f37f2 | 2011-05-11 15:57:09 -0400 | [diff] [blame] | 137 | * 10 and would index in the pending_pages[..]. |
| 138 | */ |
Konrad Rzeszutek Wilk | 2e9977c | 2011-04-14 17:42:07 -0400 | [diff] [blame] | 139 | static inline int vaddr_pagenr(struct pending_req *req, int seg) |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 140 | { |
Konrad Rzeszutek Wilk | 2e9977c | 2011-04-14 17:42:07 -0400 | [diff] [blame] | 141 | return (req - blkbk->pending_reqs) * |
| 142 | BLKIF_MAX_SEGMENTS_PER_REQUEST + seg; |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 143 | } |
| 144 | |
Jan Beulich | efe08a3 | 2010-02-05 14:19:33 -0500 | [diff] [blame] | 145 | #define pending_page(req, seg) pending_pages[vaddr_pagenr(req, seg)] |
| 146 | |
Konrad Rzeszutek Wilk | 2e9977c | 2011-04-14 17:42:07 -0400 | [diff] [blame] | 147 | static inline unsigned long vaddr(struct pending_req *req, int seg) |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 148 | { |
Konrad Rzeszutek Wilk | e8e2887 | 2011-02-25 10:51:29 -0500 | [diff] [blame] | 149 | unsigned long pfn = page_to_pfn(blkbk->pending_page(req, seg)); |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 150 | return (unsigned long)pfn_to_kaddr(pfn); |
| 151 | } |
| 152 | |
| 153 | #define pending_handle(_req, _seg) \ |
Konrad Rzeszutek Wilk | e8e2887 | 2011-02-25 10:51:29 -0500 | [diff] [blame] | 154 | (blkbk->pending_grant_handles[vaddr_pagenr(_req, _seg)]) |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 155 | |
| 156 | |
Konrad Rzeszutek Wilk | 30fd150 | 2011-05-12 16:47:48 -0400 | [diff] [blame] | 157 | static int do_block_io_op(struct xen_blkif *blkif); |
| 158 | static int dispatch_rw_block_io(struct xen_blkif *blkif, |
Konrad Rzeszutek Wilk | fc53bf7 | 2011-05-05 13:37:23 -0400 | [diff] [blame] | 159 | struct blkif_request *req, |
| 160 | struct pending_req *pending_req); |
Konrad Rzeszutek Wilk | 30fd150 | 2011-05-12 16:47:48 -0400 | [diff] [blame] | 161 | static void make_response(struct xen_blkif *blkif, u64 id, |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 162 | unsigned short op, int st); |
| 163 | |
Roger Pau Monne | 0a8704a | 2012-10-24 18:58:45 +0200 | [diff] [blame^] | 164 | #define foreach_grant(pos, rbtree, node) \ |
| 165 | for ((pos) = container_of(rb_first((rbtree)), typeof(*(pos)), node); \ |
| 166 | &(pos)->node != NULL; \ |
| 167 | (pos) = container_of(rb_next(&(pos)->node), typeof(*(pos)), node)) |
| 168 | |
| 169 | |
| 170 | static void add_persistent_gnt(struct rb_root *root, |
| 171 | struct persistent_gnt *persistent_gnt) |
| 172 | { |
| 173 | struct rb_node **new = &(root->rb_node), *parent = NULL; |
| 174 | struct persistent_gnt *this; |
| 175 | |
| 176 | /* Figure out where to put new node */ |
| 177 | while (*new) { |
| 178 | this = container_of(*new, struct persistent_gnt, node); |
| 179 | |
| 180 | parent = *new; |
| 181 | if (persistent_gnt->gnt < this->gnt) |
| 182 | new = &((*new)->rb_left); |
| 183 | else if (persistent_gnt->gnt > this->gnt) |
| 184 | new = &((*new)->rb_right); |
| 185 | else { |
| 186 | pr_alert(DRV_PFX " trying to add a gref that's already in the tree\n"); |
| 187 | BUG(); |
| 188 | } |
| 189 | } |
| 190 | |
| 191 | /* Add new node and rebalance tree. */ |
| 192 | rb_link_node(&(persistent_gnt->node), parent, new); |
| 193 | rb_insert_color(&(persistent_gnt->node), root); |
| 194 | } |
| 195 | |
| 196 | static struct persistent_gnt *get_persistent_gnt(struct rb_root *root, |
| 197 | grant_ref_t gref) |
| 198 | { |
| 199 | struct persistent_gnt *data; |
| 200 | struct rb_node *node = root->rb_node; |
| 201 | |
| 202 | while (node) { |
| 203 | data = container_of(node, struct persistent_gnt, node); |
| 204 | |
| 205 | if (gref < data->gnt) |
| 206 | node = node->rb_left; |
| 207 | else if (gref > data->gnt) |
| 208 | node = node->rb_right; |
| 209 | else |
| 210 | return data; |
| 211 | } |
| 212 | return NULL; |
| 213 | } |
| 214 | |
Konrad Rzeszutek Wilk | a1397fa | 2011-04-14 17:05:23 -0400 | [diff] [blame] | 215 | /* |
| 216 | * Retrieve from the 'pending_reqs' a free pending_req structure to be used. |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 217 | */ |
Konrad Rzeszutek Wilk | 2e9977c | 2011-04-14 17:42:07 -0400 | [diff] [blame] | 218 | static struct pending_req *alloc_req(void) |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 219 | { |
Konrad Rzeszutek Wilk | 2e9977c | 2011-04-14 17:42:07 -0400 | [diff] [blame] | 220 | struct pending_req *req = NULL; |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 221 | unsigned long flags; |
| 222 | |
Konrad Rzeszutek Wilk | e8e2887 | 2011-02-25 10:51:29 -0500 | [diff] [blame] | 223 | spin_lock_irqsave(&blkbk->pending_free_lock, flags); |
| 224 | if (!list_empty(&blkbk->pending_free)) { |
Konrad Rzeszutek Wilk | 2e9977c | 2011-04-14 17:42:07 -0400 | [diff] [blame] | 225 | req = list_entry(blkbk->pending_free.next, struct pending_req, |
| 226 | free_list); |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 227 | list_del(&req->free_list); |
| 228 | } |
Konrad Rzeszutek Wilk | e8e2887 | 2011-02-25 10:51:29 -0500 | [diff] [blame] | 229 | spin_unlock_irqrestore(&blkbk->pending_free_lock, flags); |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 230 | return req; |
| 231 | } |
| 232 | |
Konrad Rzeszutek Wilk | a1397fa | 2011-04-14 17:05:23 -0400 | [diff] [blame] | 233 | /* |
| 234 | * Return the 'pending_req' structure back to the freepool. We also |
| 235 | * wake up the thread if it was waiting for a free page. |
| 236 | */ |
Konrad Rzeszutek Wilk | 2e9977c | 2011-04-14 17:42:07 -0400 | [diff] [blame] | 237 | static void free_req(struct pending_req *req) |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 238 | { |
| 239 | unsigned long flags; |
| 240 | int was_empty; |
| 241 | |
Konrad Rzeszutek Wilk | e8e2887 | 2011-02-25 10:51:29 -0500 | [diff] [blame] | 242 | spin_lock_irqsave(&blkbk->pending_free_lock, flags); |
| 243 | was_empty = list_empty(&blkbk->pending_free); |
| 244 | list_add(&req->free_list, &blkbk->pending_free); |
| 245 | spin_unlock_irqrestore(&blkbk->pending_free_lock, flags); |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 246 | if (was_empty) |
Konrad Rzeszutek Wilk | e8e2887 | 2011-02-25 10:51:29 -0500 | [diff] [blame] | 247 | wake_up(&blkbk->pending_free_wq); |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 248 | } |
| 249 | |
Konrad Rzeszutek Wilk | a1397fa | 2011-04-14 17:05:23 -0400 | [diff] [blame] | 250 | /* |
Konrad Rzeszutek Wilk | ee9ff85 | 2011-04-20 10:57:29 -0400 | [diff] [blame] | 251 | * Routines for managing virtual block devices (vbds). |
| 252 | */ |
Konrad Rzeszutek Wilk | 3d81473 | 2011-05-12 16:53:56 -0400 | [diff] [blame] | 253 | static int xen_vbd_translate(struct phys_req *req, struct xen_blkif *blkif, |
| 254 | int operation) |
Konrad Rzeszutek Wilk | ee9ff85 | 2011-04-20 10:57:29 -0400 | [diff] [blame] | 255 | { |
Konrad Rzeszutek Wilk | 3d81473 | 2011-05-12 16:53:56 -0400 | [diff] [blame] | 256 | struct xen_vbd *vbd = &blkif->vbd; |
Konrad Rzeszutek Wilk | ee9ff85 | 2011-04-20 10:57:29 -0400 | [diff] [blame] | 257 | int rc = -EACCES; |
| 258 | |
| 259 | if ((operation != READ) && vbd->readonly) |
| 260 | goto out; |
| 261 | |
Jan Beulich | 8ab5215 | 2011-05-17 11:07:05 +0100 | [diff] [blame] | 262 | if (likely(req->nr_sects)) { |
| 263 | blkif_sector_t end = req->sector_number + req->nr_sects; |
| 264 | |
| 265 | if (unlikely(end < req->sector_number)) |
| 266 | goto out; |
| 267 | if (unlikely(end > vbd_sz(vbd))) |
| 268 | goto out; |
| 269 | } |
Konrad Rzeszutek Wilk | ee9ff85 | 2011-04-20 10:57:29 -0400 | [diff] [blame] | 270 | |
| 271 | req->dev = vbd->pdevice; |
| 272 | req->bdev = vbd->bdev; |
| 273 | rc = 0; |
| 274 | |
| 275 | out: |
| 276 | return rc; |
| 277 | } |
| 278 | |
Konrad Rzeszutek Wilk | 3d81473 | 2011-05-12 16:53:56 -0400 | [diff] [blame] | 279 | static void xen_vbd_resize(struct xen_blkif *blkif) |
Konrad Rzeszutek Wilk | ee9ff85 | 2011-04-20 10:57:29 -0400 | [diff] [blame] | 280 | { |
Konrad Rzeszutek Wilk | 3d81473 | 2011-05-12 16:53:56 -0400 | [diff] [blame] | 281 | struct xen_vbd *vbd = &blkif->vbd; |
Konrad Rzeszutek Wilk | ee9ff85 | 2011-04-20 10:57:29 -0400 | [diff] [blame] | 282 | struct xenbus_transaction xbt; |
| 283 | int err; |
Konrad Rzeszutek Wilk | 8b6bf74 | 2011-04-20 11:50:43 -0400 | [diff] [blame] | 284 | struct xenbus_device *dev = xen_blkbk_xenbus(blkif->be); |
Konrad Rzeszutek Wilk | 42c7841 | 2011-04-20 11:21:43 -0400 | [diff] [blame] | 285 | unsigned long long new_size = vbd_sz(vbd); |
Konrad Rzeszutek Wilk | ee9ff85 | 2011-04-20 10:57:29 -0400 | [diff] [blame] | 286 | |
Konrad Rzeszutek Wilk | 22b20f2 | 2011-05-12 16:43:12 -0400 | [diff] [blame] | 287 | pr_info(DRV_PFX "VBD Resize: Domid: %d, Device: (%d, %d)\n", |
Konrad Rzeszutek Wilk | ee9ff85 | 2011-04-20 10:57:29 -0400 | [diff] [blame] | 288 | blkif->domid, MAJOR(vbd->pdevice), MINOR(vbd->pdevice)); |
Konrad Rzeszutek Wilk | 22b20f2 | 2011-05-12 16:43:12 -0400 | [diff] [blame] | 289 | pr_info(DRV_PFX "VBD Resize: new size %llu\n", new_size); |
Konrad Rzeszutek Wilk | ee9ff85 | 2011-04-20 10:57:29 -0400 | [diff] [blame] | 290 | vbd->size = new_size; |
| 291 | again: |
| 292 | err = xenbus_transaction_start(&xbt); |
| 293 | if (err) { |
Konrad Rzeszutek Wilk | 22b20f2 | 2011-05-12 16:43:12 -0400 | [diff] [blame] | 294 | pr_warn(DRV_PFX "Error starting transaction"); |
Konrad Rzeszutek Wilk | ee9ff85 | 2011-04-20 10:57:29 -0400 | [diff] [blame] | 295 | return; |
| 296 | } |
| 297 | err = xenbus_printf(xbt, dev->nodename, "sectors", "%llu", |
Konrad Rzeszutek Wilk | 42c7841 | 2011-04-20 11:21:43 -0400 | [diff] [blame] | 298 | (unsigned long long)vbd_sz(vbd)); |
Konrad Rzeszutek Wilk | ee9ff85 | 2011-04-20 10:57:29 -0400 | [diff] [blame] | 299 | if (err) { |
Konrad Rzeszutek Wilk | 22b20f2 | 2011-05-12 16:43:12 -0400 | [diff] [blame] | 300 | pr_warn(DRV_PFX "Error writing new size"); |
Konrad Rzeszutek Wilk | ee9ff85 | 2011-04-20 10:57:29 -0400 | [diff] [blame] | 301 | goto abort; |
| 302 | } |
| 303 | /* |
| 304 | * Write the current state; we will use this to synchronize |
| 305 | * the front-end. If the current state is "connected" the |
| 306 | * front-end will get the new size information online. |
| 307 | */ |
| 308 | err = xenbus_printf(xbt, dev->nodename, "state", "%d", dev->state); |
| 309 | if (err) { |
Konrad Rzeszutek Wilk | 22b20f2 | 2011-05-12 16:43:12 -0400 | [diff] [blame] | 310 | pr_warn(DRV_PFX "Error writing the state"); |
Konrad Rzeszutek Wilk | ee9ff85 | 2011-04-20 10:57:29 -0400 | [diff] [blame] | 311 | goto abort; |
| 312 | } |
| 313 | |
| 314 | err = xenbus_transaction_end(xbt, 0); |
| 315 | if (err == -EAGAIN) |
| 316 | goto again; |
| 317 | if (err) |
Konrad Rzeszutek Wilk | 22b20f2 | 2011-05-12 16:43:12 -0400 | [diff] [blame] | 318 | pr_warn(DRV_PFX "Error ending transaction"); |
Laszlo Ersek | 496b318 | 2011-05-13 09:45:40 -0400 | [diff] [blame] | 319 | return; |
Konrad Rzeszutek Wilk | ee9ff85 | 2011-04-20 10:57:29 -0400 | [diff] [blame] | 320 | abort: |
| 321 | xenbus_transaction_end(xbt, 1); |
| 322 | } |
| 323 | |
| 324 | /* |
Konrad Rzeszutek Wilk | b0aef17 | 2011-04-15 10:58:05 -0400 | [diff] [blame] | 325 | * Notification from the guest OS. |
| 326 | */ |
Konrad Rzeszutek Wilk | 30fd150 | 2011-05-12 16:47:48 -0400 | [diff] [blame] | 327 | static void blkif_notify_work(struct xen_blkif *blkif) |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 328 | { |
Konrad Rzeszutek Wilk | b0aef17 | 2011-04-15 10:58:05 -0400 | [diff] [blame] | 329 | blkif->waiting_reqs = 1; |
| 330 | wake_up(&blkif->wq); |
| 331 | } |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 332 | |
Konrad Rzeszutek Wilk | 8b6bf74 | 2011-04-20 11:50:43 -0400 | [diff] [blame] | 333 | irqreturn_t xen_blkif_be_int(int irq, void *dev_id) |
Konrad Rzeszutek Wilk | b0aef17 | 2011-04-15 10:58:05 -0400 | [diff] [blame] | 334 | { |
| 335 | blkif_notify_work(dev_id); |
| 336 | return IRQ_HANDLED; |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 337 | } |
| 338 | |
Konrad Rzeszutek Wilk | 2e9977c | 2011-04-14 17:42:07 -0400 | [diff] [blame] | 339 | /* |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 340 | * SCHEDULER FUNCTIONS |
| 341 | */ |
| 342 | |
Konrad Rzeszutek Wilk | 30fd150 | 2011-05-12 16:47:48 -0400 | [diff] [blame] | 343 | static void print_stats(struct xen_blkif *blkif) |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 344 | { |
Li Dongyang | b3cb0d6 | 2011-09-01 18:39:10 +0800 | [diff] [blame] | 345 | pr_info("xen-blkback (%s): oo %3d | rd %4d | wr %4d | f %4d" |
| 346 | " | ds %4d\n", |
Konrad Rzeszutek Wilk | ebe8190 | 2011-05-12 16:42:31 -0400 | [diff] [blame] | 347 | current->comm, blkif->st_oo_req, |
Li Dongyang | b3cb0d6 | 2011-09-01 18:39:10 +0800 | [diff] [blame] | 348 | blkif->st_rd_req, blkif->st_wr_req, |
| 349 | blkif->st_f_req, blkif->st_ds_req); |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 350 | blkif->st_print = jiffies + msecs_to_jiffies(10 * 1000); |
| 351 | blkif->st_rd_req = 0; |
| 352 | blkif->st_wr_req = 0; |
| 353 | blkif->st_oo_req = 0; |
Li Dongyang | b3cb0d6 | 2011-09-01 18:39:10 +0800 | [diff] [blame] | 354 | blkif->st_ds_req = 0; |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 355 | } |
| 356 | |
Konrad Rzeszutek Wilk | 8b6bf74 | 2011-04-20 11:50:43 -0400 | [diff] [blame] | 357 | int xen_blkif_schedule(void *arg) |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 358 | { |
Konrad Rzeszutek Wilk | 30fd150 | 2011-05-12 16:47:48 -0400 | [diff] [blame] | 359 | struct xen_blkif *blkif = arg; |
Konrad Rzeszutek Wilk | 3d81473 | 2011-05-12 16:53:56 -0400 | [diff] [blame] | 360 | struct xen_vbd *vbd = &blkif->vbd; |
Roger Pau Monne | 0a8704a | 2012-10-24 18:58:45 +0200 | [diff] [blame^] | 361 | struct gnttab_unmap_grant_ref unmap[BLKIF_MAX_SEGMENTS_PER_REQUEST]; |
| 362 | struct page *pages[BLKIF_MAX_SEGMENTS_PER_REQUEST]; |
| 363 | struct persistent_gnt *persistent_gnt; |
| 364 | int ret = 0; |
| 365 | int segs_to_unmap = 0; |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 366 | |
Konrad Rzeszutek Wilk | 8b6bf74 | 2011-04-20 11:50:43 -0400 | [diff] [blame] | 367 | xen_blkif_get(blkif); |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 368 | |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 369 | while (!kthread_should_stop()) { |
| 370 | if (try_to_freeze()) |
| 371 | continue; |
Konrad Rzeszutek Wilk | 42c7841 | 2011-04-20 11:21:43 -0400 | [diff] [blame] | 372 | if (unlikely(vbd->size != vbd_sz(vbd))) |
Konrad Rzeszutek Wilk | 3d81473 | 2011-05-12 16:53:56 -0400 | [diff] [blame] | 373 | xen_vbd_resize(blkif); |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 374 | |
| 375 | wait_event_interruptible( |
| 376 | blkif->wq, |
| 377 | blkif->waiting_reqs || kthread_should_stop()); |
| 378 | wait_event_interruptible( |
Konrad Rzeszutek Wilk | e8e2887 | 2011-02-25 10:51:29 -0500 | [diff] [blame] | 379 | blkbk->pending_free_wq, |
Konrad Rzeszutek Wilk | 2e9977c | 2011-04-14 17:42:07 -0400 | [diff] [blame] | 380 | !list_empty(&blkbk->pending_free) || |
| 381 | kthread_should_stop()); |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 382 | |
| 383 | blkif->waiting_reqs = 0; |
| 384 | smp_mb(); /* clear flag *before* checking for work */ |
| 385 | |
| 386 | if (do_block_io_op(blkif)) |
| 387 | blkif->waiting_reqs = 1; |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 388 | |
| 389 | if (log_stats && time_after(jiffies, blkif->st_print)) |
| 390 | print_stats(blkif); |
| 391 | } |
| 392 | |
Roger Pau Monne | 0a8704a | 2012-10-24 18:58:45 +0200 | [diff] [blame^] | 393 | /* Free all persistent grant pages */ |
| 394 | if (!RB_EMPTY_ROOT(&blkif->persistent_gnts)) { |
| 395 | foreach_grant(persistent_gnt, &blkif->persistent_gnts, node) { |
| 396 | BUG_ON(persistent_gnt->handle == |
| 397 | BLKBACK_INVALID_HANDLE); |
| 398 | gnttab_set_unmap_op(&unmap[segs_to_unmap], |
| 399 | (unsigned long) pfn_to_kaddr(page_to_pfn( |
| 400 | persistent_gnt->page)), |
| 401 | GNTMAP_host_map, |
| 402 | persistent_gnt->handle); |
| 403 | |
| 404 | pages[segs_to_unmap] = persistent_gnt->page; |
| 405 | rb_erase(&persistent_gnt->node, |
| 406 | &blkif->persistent_gnts); |
| 407 | kfree(persistent_gnt); |
| 408 | blkif->persistent_gnt_c--; |
| 409 | |
| 410 | if (++segs_to_unmap == BLKIF_MAX_SEGMENTS_PER_REQUEST || |
| 411 | !rb_next(&persistent_gnt->node)) { |
| 412 | ret = gnttab_unmap_refs(unmap, NULL, pages, |
| 413 | segs_to_unmap); |
| 414 | BUG_ON(ret); |
| 415 | segs_to_unmap = 0; |
| 416 | } |
| 417 | } |
| 418 | } |
| 419 | |
| 420 | BUG_ON(blkif->persistent_gnt_c != 0); |
| 421 | BUG_ON(!RB_EMPTY_ROOT(&blkif->persistent_gnts)); |
| 422 | |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 423 | if (log_stats) |
| 424 | print_stats(blkif); |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 425 | |
| 426 | blkif->xenblkd = NULL; |
Konrad Rzeszutek Wilk | 8b6bf74 | 2011-04-20 11:50:43 -0400 | [diff] [blame] | 427 | xen_blkif_put(blkif); |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 428 | |
| 429 | return 0; |
| 430 | } |
| 431 | |
Konrad Rzeszutek Wilk | 1a95fe6 | 2011-04-15 11:35:13 -0400 | [diff] [blame] | 432 | struct seg_buf { |
| 433 | unsigned long buf; |
| 434 | unsigned int nsec; |
| 435 | }; |
Konrad Rzeszutek Wilk | a1397fa | 2011-04-14 17:05:23 -0400 | [diff] [blame] | 436 | /* |
Konrad Rzeszutek Wilk | b0aef17 | 2011-04-15 10:58:05 -0400 | [diff] [blame] | 437 | * Unmap the grant references, and also remove the M2P over-rides |
| 438 | * used in the 'pending_req'. |
Konrad Rzeszutek Wilk | 01f37f2 | 2011-05-11 15:57:09 -0400 | [diff] [blame] | 439 | */ |
Konrad Rzeszutek Wilk | 9f3aedf | 2011-04-15 11:50:34 -0400 | [diff] [blame] | 440 | static void xen_blkbk_unmap(struct pending_req *req) |
Konrad Rzeszutek Wilk | b0aef17 | 2011-04-15 10:58:05 -0400 | [diff] [blame] | 441 | { |
| 442 | struct gnttab_unmap_grant_ref unmap[BLKIF_MAX_SEGMENTS_PER_REQUEST]; |
Daniel De Graaf | 4f14faa | 2011-11-28 11:49:03 -0500 | [diff] [blame] | 443 | struct page *pages[BLKIF_MAX_SEGMENTS_PER_REQUEST]; |
Konrad Rzeszutek Wilk | b0aef17 | 2011-04-15 10:58:05 -0400 | [diff] [blame] | 444 | unsigned int i, invcount = 0; |
| 445 | grant_handle_t handle; |
| 446 | int ret; |
| 447 | |
| 448 | for (i = 0; i < req->nr_pages; i++) { |
Roger Pau Monne | 0a8704a | 2012-10-24 18:58:45 +0200 | [diff] [blame^] | 449 | if (!test_bit(i, req->unmap_seg)) |
| 450 | continue; |
Konrad Rzeszutek Wilk | b0aef17 | 2011-04-15 10:58:05 -0400 | [diff] [blame] | 451 | handle = pending_handle(req, i); |
| 452 | if (handle == BLKBACK_INVALID_HANDLE) |
| 453 | continue; |
| 454 | gnttab_set_unmap_op(&unmap[invcount], vaddr(req, i), |
| 455 | GNTMAP_host_map, handle); |
| 456 | pending_handle(req, i) = BLKBACK_INVALID_HANDLE; |
Daniel De Graaf | 4f14faa | 2011-11-28 11:49:03 -0500 | [diff] [blame] | 457 | pages[invcount] = virt_to_page(vaddr(req, i)); |
Konrad Rzeszutek Wilk | b0aef17 | 2011-04-15 10:58:05 -0400 | [diff] [blame] | 458 | invcount++; |
| 459 | } |
| 460 | |
Stefano Stabellini | 2fc136e | 2012-09-12 12:44:30 +0100 | [diff] [blame] | 461 | ret = gnttab_unmap_refs(unmap, NULL, pages, invcount); |
Konrad Rzeszutek Wilk | b0aef17 | 2011-04-15 10:58:05 -0400 | [diff] [blame] | 462 | BUG_ON(ret); |
Konrad Rzeszutek Wilk | b0aef17 | 2011-04-15 10:58:05 -0400 | [diff] [blame] | 463 | } |
Konrad Rzeszutek Wilk | 01f37f2 | 2011-05-11 15:57:09 -0400 | [diff] [blame] | 464 | |
| 465 | static int xen_blkbk_map(struct blkif_request *req, |
| 466 | struct pending_req *pending_req, |
Roger Pau Monne | 0a8704a | 2012-10-24 18:58:45 +0200 | [diff] [blame^] | 467 | struct seg_buf seg[], |
| 468 | struct page *pages[]) |
Konrad Rzeszutek Wilk | 1a95fe6 | 2011-04-15 11:35:13 -0400 | [diff] [blame] | 469 | { |
| 470 | struct gnttab_map_grant_ref map[BLKIF_MAX_SEGMENTS_PER_REQUEST]; |
Roger Pau Monne | 0a8704a | 2012-10-24 18:58:45 +0200 | [diff] [blame^] | 471 | struct persistent_gnt *persistent_gnts[BLKIF_MAX_SEGMENTS_PER_REQUEST]; |
| 472 | struct page *pages_to_gnt[BLKIF_MAX_SEGMENTS_PER_REQUEST]; |
| 473 | struct persistent_gnt *persistent_gnt = NULL; |
| 474 | struct xen_blkif *blkif = pending_req->blkif; |
| 475 | phys_addr_t addr = 0; |
| 476 | int i, j; |
| 477 | bool new_map; |
Konrad Rzeszutek Wilk | 97e3683 | 2011-10-12 12:12:36 -0400 | [diff] [blame] | 478 | int nseg = req->u.rw.nr_segments; |
Roger Pau Monne | 0a8704a | 2012-10-24 18:58:45 +0200 | [diff] [blame^] | 479 | int segs_to_map = 0; |
Konrad Rzeszutek Wilk | 1a95fe6 | 2011-04-15 11:35:13 -0400 | [diff] [blame] | 480 | int ret = 0; |
Roger Pau Monne | 0a8704a | 2012-10-24 18:58:45 +0200 | [diff] [blame^] | 481 | int use_persistent_gnts; |
| 482 | |
| 483 | use_persistent_gnts = (blkif->vbd.feature_gnt_persistent); |
| 484 | |
| 485 | BUG_ON(blkif->persistent_gnt_c > |
| 486 | max_mapped_grant_pages(pending_req->blkif->blk_protocol)); |
Konrad Rzeszutek Wilk | 01f37f2 | 2011-05-11 15:57:09 -0400 | [diff] [blame] | 487 | |
| 488 | /* |
| 489 | * Fill out preq.nr_sects with proper amount of sectors, and setup |
Konrad Rzeszutek Wilk | 1a95fe6 | 2011-04-15 11:35:13 -0400 | [diff] [blame] | 490 | * assign map[..] with the PFN of the page in our domain with the |
| 491 | * corresponding grant reference for each page. |
| 492 | */ |
| 493 | for (i = 0; i < nseg; i++) { |
| 494 | uint32_t flags; |
| 495 | |
Roger Pau Monne | 0a8704a | 2012-10-24 18:58:45 +0200 | [diff] [blame^] | 496 | if (use_persistent_gnts) |
| 497 | persistent_gnt = get_persistent_gnt( |
| 498 | &blkif->persistent_gnts, |
| 499 | req->u.rw.seg[i].gref); |
| 500 | |
| 501 | if (persistent_gnt) { |
| 502 | /* |
| 503 | * We are using persistent grants and |
| 504 | * the grant is already mapped |
| 505 | */ |
| 506 | new_map = false; |
| 507 | } else if (use_persistent_gnts && |
| 508 | blkif->persistent_gnt_c < |
| 509 | max_mapped_grant_pages(blkif->blk_protocol)) { |
| 510 | /* |
| 511 | * We are using persistent grants, the grant is |
| 512 | * not mapped but we have room for it |
| 513 | */ |
| 514 | new_map = true; |
| 515 | persistent_gnt = kzalloc( |
| 516 | sizeof(struct persistent_gnt), |
| 517 | GFP_KERNEL); |
| 518 | if (!persistent_gnt) |
| 519 | return -ENOMEM; |
| 520 | persistent_gnt->page = alloc_page(GFP_KERNEL); |
| 521 | if (!persistent_gnt->page) { |
| 522 | kfree(persistent_gnt); |
| 523 | return -ENOMEM; |
| 524 | } |
| 525 | persistent_gnt->gnt = req->u.rw.seg[i].gref; |
| 526 | |
| 527 | pages_to_gnt[segs_to_map] = |
| 528 | persistent_gnt->page; |
| 529 | addr = (unsigned long) pfn_to_kaddr( |
| 530 | page_to_pfn(persistent_gnt->page)); |
| 531 | |
| 532 | add_persistent_gnt(&blkif->persistent_gnts, |
| 533 | persistent_gnt); |
| 534 | blkif->persistent_gnt_c++; |
| 535 | pr_debug(DRV_PFX " grant %u added to the tree of persistent grants, using %u/%u\n", |
| 536 | persistent_gnt->gnt, blkif->persistent_gnt_c, |
| 537 | max_mapped_grant_pages(blkif->blk_protocol)); |
| 538 | } else { |
| 539 | /* |
| 540 | * We are either using persistent grants and |
| 541 | * hit the maximum limit of grants mapped, |
| 542 | * or we are not using persistent grants. |
| 543 | */ |
| 544 | if (use_persistent_gnts && |
| 545 | !blkif->vbd.overflow_max_grants) { |
| 546 | blkif->vbd.overflow_max_grants = 1; |
| 547 | pr_alert(DRV_PFX " domain %u, device %#x is using maximum number of persistent grants\n", |
| 548 | blkif->domid, blkif->vbd.handle); |
| 549 | } |
| 550 | new_map = true; |
| 551 | pages[i] = blkbk->pending_page(pending_req, i); |
| 552 | addr = vaddr(pending_req, i); |
| 553 | pages_to_gnt[segs_to_map] = |
| 554 | blkbk->pending_page(pending_req, i); |
| 555 | } |
| 556 | |
| 557 | if (persistent_gnt) { |
| 558 | pages[i] = persistent_gnt->page; |
| 559 | persistent_gnts[i] = persistent_gnt; |
| 560 | } else { |
| 561 | persistent_gnts[i] = NULL; |
| 562 | } |
| 563 | |
| 564 | if (new_map) { |
| 565 | flags = GNTMAP_host_map; |
| 566 | if (!persistent_gnt && |
| 567 | (pending_req->operation != BLKIF_OP_READ)) |
| 568 | flags |= GNTMAP_readonly; |
| 569 | gnttab_set_map_op(&map[segs_to_map++], addr, |
| 570 | flags, req->u.rw.seg[i].gref, |
| 571 | blkif->domid); |
| 572 | } |
Konrad Rzeszutek Wilk | 1a95fe6 | 2011-04-15 11:35:13 -0400 | [diff] [blame] | 573 | } |
| 574 | |
Roger Pau Monne | 0a8704a | 2012-10-24 18:58:45 +0200 | [diff] [blame^] | 575 | if (segs_to_map) { |
| 576 | ret = gnttab_map_refs(map, NULL, pages_to_gnt, segs_to_map); |
| 577 | BUG_ON(ret); |
| 578 | } |
Konrad Rzeszutek Wilk | 1a95fe6 | 2011-04-15 11:35:13 -0400 | [diff] [blame] | 579 | |
Konrad Rzeszutek Wilk | 01f37f2 | 2011-05-11 15:57:09 -0400 | [diff] [blame] | 580 | /* |
| 581 | * Now swizzle the MFN in our domain with the MFN from the other domain |
Konrad Rzeszutek Wilk | 1a95fe6 | 2011-04-15 11:35:13 -0400 | [diff] [blame] | 582 | * so that when we access vaddr(pending_req,i) it has the contents of |
| 583 | * the page from the other domain. |
| 584 | */ |
Roger Pau Monne | 0a8704a | 2012-10-24 18:58:45 +0200 | [diff] [blame^] | 585 | bitmap_zero(pending_req->unmap_seg, BLKIF_MAX_SEGMENTS_PER_REQUEST); |
| 586 | for (i = 0, j = 0; i < nseg; i++) { |
| 587 | if (!persistent_gnts[i] || !persistent_gnts[i]->handle) { |
| 588 | /* This is a newly mapped grant */ |
| 589 | BUG_ON(j >= segs_to_map); |
| 590 | if (unlikely(map[j].status != 0)) { |
| 591 | pr_debug(DRV_PFX "invalid buffer -- could not remap it\n"); |
| 592 | map[j].handle = BLKBACK_INVALID_HANDLE; |
| 593 | ret |= 1; |
| 594 | if (persistent_gnts[i]) { |
| 595 | rb_erase(&persistent_gnts[i]->node, |
| 596 | &blkif->persistent_gnts); |
| 597 | blkif->persistent_gnt_c--; |
| 598 | kfree(persistent_gnts[i]); |
| 599 | persistent_gnts[i] = NULL; |
| 600 | } |
| 601 | } |
Konrad Rzeszutek Wilk | 1a95fe6 | 2011-04-15 11:35:13 -0400 | [diff] [blame] | 602 | } |
Roger Pau Monne | 0a8704a | 2012-10-24 18:58:45 +0200 | [diff] [blame^] | 603 | if (persistent_gnts[i]) { |
| 604 | if (!persistent_gnts[i]->handle) { |
| 605 | /* |
| 606 | * If this is a new persistent grant |
| 607 | * save the handler |
| 608 | */ |
| 609 | persistent_gnts[i]->handle = map[j].handle; |
| 610 | persistent_gnts[i]->dev_bus_addr = |
| 611 | map[j++].dev_bus_addr; |
| 612 | } |
| 613 | pending_handle(pending_req, i) = |
| 614 | persistent_gnts[i]->handle; |
Konrad Rzeszutek Wilk | 1a95fe6 | 2011-04-15 11:35:13 -0400 | [diff] [blame] | 615 | |
Roger Pau Monne | 0a8704a | 2012-10-24 18:58:45 +0200 | [diff] [blame^] | 616 | if (ret) |
| 617 | continue; |
Konrad Rzeszutek Wilk | 1a95fe6 | 2011-04-15 11:35:13 -0400 | [diff] [blame] | 618 | |
Roger Pau Monne | 0a8704a | 2012-10-24 18:58:45 +0200 | [diff] [blame^] | 619 | seg[i].buf = persistent_gnts[i]->dev_bus_addr | |
| 620 | (req->u.rw.seg[i].first_sect << 9); |
| 621 | } else { |
| 622 | pending_handle(pending_req, i) = map[j].handle; |
| 623 | bitmap_set(pending_req->unmap_seg, i, 1); |
Konrad Rzeszutek Wilk | 1a95fe6 | 2011-04-15 11:35:13 -0400 | [diff] [blame] | 624 | |
Roger Pau Monne | 0a8704a | 2012-10-24 18:58:45 +0200 | [diff] [blame^] | 625 | if (ret) { |
| 626 | j++; |
| 627 | continue; |
| 628 | } |
| 629 | |
| 630 | seg[i].buf = map[j++].dev_bus_addr | |
| 631 | (req->u.rw.seg[i].first_sect << 9); |
| 632 | } |
Konrad Rzeszutek Wilk | 1a95fe6 | 2011-04-15 11:35:13 -0400 | [diff] [blame] | 633 | } |
| 634 | return ret; |
| 635 | } |
| 636 | |
Konrad Rzeszutek Wilk | 4214635 | 2011-10-12 17:26:47 -0400 | [diff] [blame] | 637 | static int dispatch_discard_io(struct xen_blkif *blkif, |
| 638 | struct blkif_request *req) |
Li Dongyang | b3cb0d6 | 2011-09-01 18:39:10 +0800 | [diff] [blame] | 639 | { |
| 640 | int err = 0; |
| 641 | int status = BLKIF_RSP_OKAY; |
| 642 | struct block_device *bdev = blkif->vbd.bdev; |
Konrad Rzeszutek Wilk | 4dae767 | 2012-03-13 18:43:23 -0400 | [diff] [blame] | 643 | unsigned long secure; |
Li Dongyang | b3cb0d6 | 2011-09-01 18:39:10 +0800 | [diff] [blame] | 644 | |
Konrad Rzeszutek Wilk | 4214635 | 2011-10-12 17:26:47 -0400 | [diff] [blame] | 645 | blkif->st_ds_req++; |
| 646 | |
| 647 | xen_blkif_get(blkif); |
Konrad Rzeszutek Wilk | 4dae767 | 2012-03-13 18:43:23 -0400 | [diff] [blame] | 648 | secure = (blkif->vbd.discard_secure && |
| 649 | (req->u.discard.flag & BLKIF_DISCARD_SECURE)) ? |
| 650 | BLKDEV_DISCARD_SECURE : 0; |
| 651 | |
| 652 | err = blkdev_issue_discard(bdev, req->u.discard.sector_number, |
| 653 | req->u.discard.nr_sectors, |
| 654 | GFP_KERNEL, secure); |
Li Dongyang | b3cb0d6 | 2011-09-01 18:39:10 +0800 | [diff] [blame] | 655 | |
| 656 | if (err == -EOPNOTSUPP) { |
| 657 | pr_debug(DRV_PFX "discard op failed, not supported\n"); |
| 658 | status = BLKIF_RSP_EOPNOTSUPP; |
| 659 | } else if (err) |
| 660 | status = BLKIF_RSP_ERROR; |
| 661 | |
Konrad Rzeszutek Wilk | 97e3683 | 2011-10-12 12:12:36 -0400 | [diff] [blame] | 662 | make_response(blkif, req->u.discard.id, req->operation, status); |
Konrad Rzeszutek Wilk | 4214635 | 2011-10-12 17:26:47 -0400 | [diff] [blame] | 663 | xen_blkif_put(blkif); |
| 664 | return err; |
Li Dongyang | b3cb0d6 | 2011-09-01 18:39:10 +0800 | [diff] [blame] | 665 | } |
| 666 | |
Konrad Rzeszutek Wilk | 29bde09 | 2011-10-10 00:42:22 -0400 | [diff] [blame] | 667 | static void xen_blk_drain_io(struct xen_blkif *blkif) |
| 668 | { |
| 669 | atomic_set(&blkif->drain, 1); |
| 670 | do { |
Konrad Rzeszutek Wilk | 6927d92 | 2011-10-17 14:27:48 -0400 | [diff] [blame] | 671 | /* The initial value is one, and one refcnt taken at the |
| 672 | * start of the xen_blkif_schedule thread. */ |
| 673 | if (atomic_read(&blkif->refcnt) <= 2) |
| 674 | break; |
Konrad Rzeszutek Wilk | 29bde09 | 2011-10-10 00:42:22 -0400 | [diff] [blame] | 675 | wait_for_completion_interruptible_timeout( |
| 676 | &blkif->drain_complete, HZ); |
| 677 | |
| 678 | if (!atomic_read(&blkif->drain)) |
| 679 | break; |
Konrad Rzeszutek Wilk | 29bde09 | 2011-10-10 00:42:22 -0400 | [diff] [blame] | 680 | } while (!kthread_should_stop()); |
| 681 | atomic_set(&blkif->drain, 0); |
| 682 | } |
| 683 | |
Konrad Rzeszutek Wilk | b0aef17 | 2011-04-15 10:58:05 -0400 | [diff] [blame] | 684 | /* |
Konrad Rzeszutek Wilk | a1397fa | 2011-04-14 17:05:23 -0400 | [diff] [blame] | 685 | * Completion callback on the bio's. Called as bh->b_end_io() |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 686 | */ |
| 687 | |
Konrad Rzeszutek Wilk | 2e9977c | 2011-04-14 17:42:07 -0400 | [diff] [blame] | 688 | static void __end_block_io_op(struct pending_req *pending_req, int error) |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 689 | { |
| 690 | /* An error fails the entire request. */ |
Konrad Rzeszutek Wilk | 24f567f | 2011-05-04 17:07:27 -0400 | [diff] [blame] | 691 | if ((pending_req->operation == BLKIF_OP_FLUSH_DISKCACHE) && |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 692 | (error == -EOPNOTSUPP)) { |
Konrad Rzeszutek Wilk | 22b20f2 | 2011-05-12 16:43:12 -0400 | [diff] [blame] | 693 | pr_debug(DRV_PFX "flush diskcache op failed, not supported\n"); |
Konrad Rzeszutek Wilk | 24f567f | 2011-05-04 17:07:27 -0400 | [diff] [blame] | 694 | xen_blkbk_flush_diskcache(XBT_NIL, pending_req->blkif->be, 0); |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 695 | pending_req->status = BLKIF_RSP_EOPNOTSUPP; |
Konrad Rzeszutek Wilk | 29bde09 | 2011-10-10 00:42:22 -0400 | [diff] [blame] | 696 | } else if ((pending_req->operation == BLKIF_OP_WRITE_BARRIER) && |
| 697 | (error == -EOPNOTSUPP)) { |
| 698 | pr_debug(DRV_PFX "write barrier op failed, not supported\n"); |
| 699 | xen_blkbk_barrier(XBT_NIL, pending_req->blkif->be, 0); |
| 700 | pending_req->status = BLKIF_RSP_EOPNOTSUPP; |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 701 | } else if (error) { |
Konrad Rzeszutek Wilk | 22b20f2 | 2011-05-12 16:43:12 -0400 | [diff] [blame] | 702 | pr_debug(DRV_PFX "Buffer not up-to-date at end of operation," |
Konrad Rzeszutek Wilk | ebe8190 | 2011-05-12 16:42:31 -0400 | [diff] [blame] | 703 | " error=%d\n", error); |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 704 | pending_req->status = BLKIF_RSP_ERROR; |
| 705 | } |
| 706 | |
Konrad Rzeszutek Wilk | 01f37f2 | 2011-05-11 15:57:09 -0400 | [diff] [blame] | 707 | /* |
| 708 | * If all of the bio's have completed it is time to unmap |
Konrad Rzeszutek Wilk | a1397fa | 2011-04-14 17:05:23 -0400 | [diff] [blame] | 709 | * the grant references associated with 'request' and provide |
Konrad Rzeszutek Wilk | 2e9977c | 2011-04-14 17:42:07 -0400 | [diff] [blame] | 710 | * the proper response on the ring. |
| 711 | */ |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 712 | if (atomic_dec_and_test(&pending_req->pendcnt)) { |
Konrad Rzeszutek Wilk | 9f3aedf | 2011-04-15 11:50:34 -0400 | [diff] [blame] | 713 | xen_blkbk_unmap(pending_req); |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 714 | make_response(pending_req->blkif, pending_req->id, |
| 715 | pending_req->operation, pending_req->status); |
Konrad Rzeszutek Wilk | 8b6bf74 | 2011-04-20 11:50:43 -0400 | [diff] [blame] | 716 | xen_blkif_put(pending_req->blkif); |
Konrad Rzeszutek Wilk | 29bde09 | 2011-10-10 00:42:22 -0400 | [diff] [blame] | 717 | if (atomic_read(&pending_req->blkif->refcnt) <= 2) { |
| 718 | if (atomic_read(&pending_req->blkif->drain)) |
| 719 | complete(&pending_req->blkif->drain_complete); |
| 720 | } |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 721 | free_req(pending_req); |
| 722 | } |
| 723 | } |
| 724 | |
Konrad Rzeszutek Wilk | a1397fa | 2011-04-14 17:05:23 -0400 | [diff] [blame] | 725 | /* |
| 726 | * bio callback. |
| 727 | */ |
Jeremy Fitzhardinge | 8812293 | 2009-02-09 12:05:51 -0800 | [diff] [blame] | 728 | static void end_block_io_op(struct bio *bio, int error) |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 729 | { |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 730 | __end_block_io_op(bio->bi_private, error); |
| 731 | bio_put(bio); |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 732 | } |
| 733 | |
| 734 | |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 735 | |
Konrad Rzeszutek Wilk | a1397fa | 2011-04-14 17:05:23 -0400 | [diff] [blame] | 736 | /* |
| 737 | * Function to copy the from the ring buffer the 'struct blkif_request' |
| 738 | * (which has the sectors we want, number of them, grant references, etc), |
| 739 | * and transmute it to the block API to hand it over to the proper block disk. |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 740 | */ |
Daniel Stodden | b4726a9 | 2011-05-28 13:21:10 -0700 | [diff] [blame] | 741 | static int |
| 742 | __do_block_io_op(struct xen_blkif *blkif) |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 743 | { |
Jeremy Fitzhardinge | 8812293 | 2009-02-09 12:05:51 -0800 | [diff] [blame] | 744 | union blkif_back_rings *blk_rings = &blkif->blk_rings; |
| 745 | struct blkif_request req; |
Konrad Rzeszutek Wilk | 2e9977c | 2011-04-14 17:42:07 -0400 | [diff] [blame] | 746 | struct pending_req *pending_req; |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 747 | RING_IDX rc, rp; |
| 748 | int more_to_do = 0; |
| 749 | |
| 750 | rc = blk_rings->common.req_cons; |
| 751 | rp = blk_rings->common.sring->req_prod; |
| 752 | rmb(); /* Ensure we see queued requests up to 'rp'. */ |
| 753 | |
| 754 | while (rc != rp) { |
| 755 | |
| 756 | if (RING_REQUEST_CONS_OVERFLOW(&blk_rings->common, rc)) |
| 757 | break; |
| 758 | |
Keir Fraser | 8270b45 | 2009-03-06 08:29:15 +0000 | [diff] [blame] | 759 | if (kthread_should_stop()) { |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 760 | more_to_do = 1; |
| 761 | break; |
| 762 | } |
| 763 | |
Keir Fraser | 8270b45 | 2009-03-06 08:29:15 +0000 | [diff] [blame] | 764 | pending_req = alloc_req(); |
| 765 | if (NULL == pending_req) { |
| 766 | blkif->st_oo_req++; |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 767 | more_to_do = 1; |
| 768 | break; |
| 769 | } |
| 770 | |
| 771 | switch (blkif->blk_protocol) { |
| 772 | case BLKIF_PROTOCOL_NATIVE: |
| 773 | memcpy(&req, RING_GET_REQUEST(&blk_rings->native, rc), sizeof(req)); |
| 774 | break; |
| 775 | case BLKIF_PROTOCOL_X86_32: |
| 776 | blkif_get_x86_32_req(&req, RING_GET_REQUEST(&blk_rings->x86_32, rc)); |
| 777 | break; |
| 778 | case BLKIF_PROTOCOL_X86_64: |
| 779 | blkif_get_x86_64_req(&req, RING_GET_REQUEST(&blk_rings->x86_64, rc)); |
| 780 | break; |
| 781 | default: |
| 782 | BUG(); |
| 783 | } |
| 784 | blk_rings->common.req_cons = ++rc; /* before make_response() */ |
| 785 | |
| 786 | /* Apply all sanity checks to /private copy/ of request. */ |
| 787 | barrier(); |
Konrad Rzeszutek Wilk | 4214635 | 2011-10-12 17:26:47 -0400 | [diff] [blame] | 788 | if (unlikely(req.operation == BLKIF_OP_DISCARD)) { |
| 789 | free_req(pending_req); |
| 790 | if (dispatch_discard_io(blkif, &req)) |
| 791 | break; |
| 792 | } else if (dispatch_rw_block_io(blkif, &req, pending_req)) |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 793 | break; |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 794 | |
| 795 | /* Yield point for this unbounded loop. */ |
| 796 | cond_resched(); |
| 797 | } |
| 798 | |
| 799 | return more_to_do; |
| 800 | } |
| 801 | |
Daniel Stodden | b4726a9 | 2011-05-28 13:21:10 -0700 | [diff] [blame] | 802 | static int |
| 803 | do_block_io_op(struct xen_blkif *blkif) |
| 804 | { |
| 805 | union blkif_back_rings *blk_rings = &blkif->blk_rings; |
| 806 | int more_to_do; |
| 807 | |
| 808 | do { |
| 809 | more_to_do = __do_block_io_op(blkif); |
| 810 | if (more_to_do) |
| 811 | break; |
| 812 | |
| 813 | RING_FINAL_CHECK_FOR_REQUESTS(&blk_rings->common, more_to_do); |
| 814 | } while (more_to_do); |
| 815 | |
| 816 | return more_to_do; |
| 817 | } |
Konrad Rzeszutek Wilk | a1397fa | 2011-04-14 17:05:23 -0400 | [diff] [blame] | 818 | /* |
Konrad Rzeszutek Wilk | 01f37f2 | 2011-05-11 15:57:09 -0400 | [diff] [blame] | 819 | * Transmutation of the 'struct blkif_request' to a proper 'struct bio' |
| 820 | * and call the 'submit_bio' to pass it to the underlying storage. |
Konrad Rzeszutek Wilk | a1397fa | 2011-04-14 17:05:23 -0400 | [diff] [blame] | 821 | */ |
Konrad Rzeszutek Wilk | 30fd150 | 2011-05-12 16:47:48 -0400 | [diff] [blame] | 822 | static int dispatch_rw_block_io(struct xen_blkif *blkif, |
| 823 | struct blkif_request *req, |
| 824 | struct pending_req *pending_req) |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 825 | { |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 826 | struct phys_req preq; |
Konrad Rzeszutek Wilk | 1a95fe6 | 2011-04-15 11:35:13 -0400 | [diff] [blame] | 827 | struct seg_buf seg[BLKIF_MAX_SEGMENTS_PER_REQUEST]; |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 828 | unsigned int nseg; |
| 829 | struct bio *bio = NULL; |
Konrad Rzeszutek Wilk | 7708992 | 2011-04-15 10:51:27 -0400 | [diff] [blame] | 830 | struct bio *biolist[BLKIF_MAX_SEGMENTS_PER_REQUEST]; |
Konrad Rzeszutek Wilk | 1a95fe6 | 2011-04-15 11:35:13 -0400 | [diff] [blame] | 831 | int i, nbio = 0; |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 832 | int operation; |
Konrad Rzeszutek Wilk | a19be5f | 2011-04-27 12:40:11 -0400 | [diff] [blame] | 833 | struct blk_plug plug; |
Konrad Rzeszutek Wilk | 29bde09 | 2011-10-10 00:42:22 -0400 | [diff] [blame] | 834 | bool drain = false; |
Roger Pau Monne | 0a8704a | 2012-10-24 18:58:45 +0200 | [diff] [blame^] | 835 | struct page *pages[BLKIF_MAX_SEGMENTS_PER_REQUEST]; |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 836 | |
| 837 | switch (req->operation) { |
| 838 | case BLKIF_OP_READ: |
Konrad Rzeszutek Wilk | fc53bf7 | 2011-05-05 13:37:23 -0400 | [diff] [blame] | 839 | blkif->st_rd_req++; |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 840 | operation = READ; |
| 841 | break; |
| 842 | case BLKIF_OP_WRITE: |
Konrad Rzeszutek Wilk | fc53bf7 | 2011-05-05 13:37:23 -0400 | [diff] [blame] | 843 | blkif->st_wr_req++; |
Konrad Rzeszutek Wilk | 013c3ca | 2011-04-26 16:24:18 -0400 | [diff] [blame] | 844 | operation = WRITE_ODIRECT; |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 845 | break; |
Konrad Rzeszutek Wilk | 29bde09 | 2011-10-10 00:42:22 -0400 | [diff] [blame] | 846 | case BLKIF_OP_WRITE_BARRIER: |
| 847 | drain = true; |
Konrad Rzeszutek Wilk | 24f567f | 2011-05-04 17:07:27 -0400 | [diff] [blame] | 848 | case BLKIF_OP_FLUSH_DISKCACHE: |
Konrad Rzeszutek Wilk | fc53bf7 | 2011-05-05 13:37:23 -0400 | [diff] [blame] | 849 | blkif->st_f_req++; |
Konrad Rzeszutek Wilk | 24f567f | 2011-05-04 17:07:27 -0400 | [diff] [blame] | 850 | operation = WRITE_FLUSH; |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 851 | break; |
| 852 | default: |
| 853 | operation = 0; /* make gcc happy */ |
Konrad Rzeszutek Wilk | fc53bf7 | 2011-05-05 13:37:23 -0400 | [diff] [blame] | 854 | goto fail_response; |
| 855 | break; |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 856 | } |
| 857 | |
Konrad Rzeszutek Wilk | 4214635 | 2011-10-12 17:26:47 -0400 | [diff] [blame] | 858 | /* Check that the number of segments is sane. */ |
| 859 | nseg = req->u.rw.nr_segments; |
Konrad Rzeszutek Wilk | 97e3683 | 2011-10-12 12:12:36 -0400 | [diff] [blame] | 860 | |
Konrad Rzeszutek Wilk | 4214635 | 2011-10-12 17:26:47 -0400 | [diff] [blame] | 861 | if (unlikely(nseg == 0 && operation != WRITE_FLUSH) || |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 862 | unlikely(nseg > BLKIF_MAX_SEGMENTS_PER_REQUEST)) { |
Konrad Rzeszutek Wilk | 22b20f2 | 2011-05-12 16:43:12 -0400 | [diff] [blame] | 863 | pr_debug(DRV_PFX "Bad number of segments in request (%d)\n", |
Konrad Rzeszutek Wilk | ebe8190 | 2011-05-12 16:42:31 -0400 | [diff] [blame] | 864 | nseg); |
Konrad Rzeszutek Wilk | 1a95fe6 | 2011-04-15 11:35:13 -0400 | [diff] [blame] | 865 | /* Haven't submitted any bio's yet. */ |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 866 | goto fail_response; |
| 867 | } |
| 868 | |
Konrad Rzeszutek Wilk | 97e3683 | 2011-10-12 12:12:36 -0400 | [diff] [blame] | 869 | preq.dev = req->u.rw.handle; |
Konrad Rzeszutek Wilk | c35950b | 2011-03-01 16:22:28 -0500 | [diff] [blame] | 870 | preq.sector_number = req->u.rw.sector_number; |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 871 | preq.nr_sects = 0; |
| 872 | |
| 873 | pending_req->blkif = blkif; |
Konrad Rzeszutek Wilk | 97e3683 | 2011-10-12 12:12:36 -0400 | [diff] [blame] | 874 | pending_req->id = req->u.rw.id; |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 875 | pending_req->operation = req->operation; |
| 876 | pending_req->status = BLKIF_RSP_OKAY; |
| 877 | pending_req->nr_pages = nseg; |
Konrad Rzeszutek Wilk | e935049 | 2011-04-18 11:34:55 -0400 | [diff] [blame] | 878 | |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 879 | for (i = 0; i < nseg; i++) { |
Konrad Rzeszutek Wilk | c35950b | 2011-03-01 16:22:28 -0500 | [diff] [blame] | 880 | seg[i].nsec = req->u.rw.seg[i].last_sect - |
| 881 | req->u.rw.seg[i].first_sect + 1; |
Konrad Rzeszutek Wilk | c35950b | 2011-03-01 16:22:28 -0500 | [diff] [blame] | 882 | if ((req->u.rw.seg[i].last_sect >= (PAGE_SIZE >> 9)) || |
| 883 | (req->u.rw.seg[i].last_sect < req->u.rw.seg[i].first_sect)) |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 884 | goto fail_response; |
| 885 | preq.nr_sects += seg[i].nsec; |
Konrad Rzeszutek Wilk | 976222e | 2011-04-15 11:38:29 -0400 | [diff] [blame] | 886 | |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 887 | } |
| 888 | |
Konrad Rzeszutek Wilk | 3d81473 | 2011-05-12 16:53:56 -0400 | [diff] [blame] | 889 | if (xen_vbd_translate(&preq, blkif, operation) != 0) { |
Konrad Rzeszutek Wilk | 22b20f2 | 2011-05-12 16:43:12 -0400 | [diff] [blame] | 890 | pr_debug(DRV_PFX "access denied: %s of [%llu,%llu] on dev=%04x\n", |
Konrad Rzeszutek Wilk | ebe8190 | 2011-05-12 16:42:31 -0400 | [diff] [blame] | 891 | operation == READ ? "read" : "write", |
| 892 | preq.sector_number, |
| 893 | preq.sector_number + preq.nr_sects, preq.dev); |
Konrad Rzeszutek Wilk | 1a95fe6 | 2011-04-15 11:35:13 -0400 | [diff] [blame] | 894 | goto fail_response; |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 895 | } |
Konrad Rzeszutek Wilk | 01f37f2 | 2011-05-11 15:57:09 -0400 | [diff] [blame] | 896 | |
| 897 | /* |
Konrad Rzeszutek Wilk | 3d81473 | 2011-05-12 16:53:56 -0400 | [diff] [blame] | 898 | * This check _MUST_ be done after xen_vbd_translate as the preq.bdev |
Konrad Rzeszutek Wilk | 01f37f2 | 2011-05-11 15:57:09 -0400 | [diff] [blame] | 899 | * is set there. |
| 900 | */ |
Konrad Rzeszutek Wilk | e935049 | 2011-04-18 11:34:55 -0400 | [diff] [blame] | 901 | for (i = 0; i < nseg; i++) { |
| 902 | if (((int)preq.sector_number|(int)seg[i].nsec) & |
| 903 | ((bdev_logical_block_size(preq.bdev) >> 9) - 1)) { |
Konrad Rzeszutek Wilk | 22b20f2 | 2011-05-12 16:43:12 -0400 | [diff] [blame] | 904 | pr_debug(DRV_PFX "Misaligned I/O request from domain %d", |
Konrad Rzeszutek Wilk | ebe8190 | 2011-05-12 16:42:31 -0400 | [diff] [blame] | 905 | blkif->domid); |
Konrad Rzeszutek Wilk | e935049 | 2011-04-18 11:34:55 -0400 | [diff] [blame] | 906 | goto fail_response; |
| 907 | } |
| 908 | } |
Konrad Rzeszutek Wilk | 01f37f2 | 2011-05-11 15:57:09 -0400 | [diff] [blame] | 909 | |
Konrad Rzeszutek Wilk | 29bde09 | 2011-10-10 00:42:22 -0400 | [diff] [blame] | 910 | /* Wait on all outstanding I/O's and once that has been completed |
| 911 | * issue the WRITE_FLUSH. |
| 912 | */ |
| 913 | if (drain) |
| 914 | xen_blk_drain_io(pending_req->blkif); |
| 915 | |
Konrad Rzeszutek Wilk | 01f37f2 | 2011-05-11 15:57:09 -0400 | [diff] [blame] | 916 | /* |
| 917 | * If we have failed at this point, we need to undo the M2P override, |
Konrad Rzeszutek Wilk | 1a95fe6 | 2011-04-15 11:35:13 -0400 | [diff] [blame] | 918 | * set gnttab_set_unmap_op on all of the grant references and perform |
| 919 | * the hypercall to unmap the grants - that is all done in |
Konrad Rzeszutek Wilk | 9f3aedf | 2011-04-15 11:50:34 -0400 | [diff] [blame] | 920 | * xen_blkbk_unmap. |
Konrad Rzeszutek Wilk | 1a95fe6 | 2011-04-15 11:35:13 -0400 | [diff] [blame] | 921 | */ |
Roger Pau Monne | 0a8704a | 2012-10-24 18:58:45 +0200 | [diff] [blame^] | 922 | if (xen_blkbk_map(req, pending_req, seg, pages)) |
Konrad Rzeszutek Wilk | 1a95fe6 | 2011-04-15 11:35:13 -0400 | [diff] [blame] | 923 | goto fail_flush; |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 924 | |
Li Dongyang | b3cb0d6 | 2011-09-01 18:39:10 +0800 | [diff] [blame] | 925 | /* |
| 926 | * This corresponding xen_blkif_put is done in __end_block_io_op, or |
| 927 | * below (in "!bio") if we are handling a BLKIF_OP_DISCARD. |
| 928 | */ |
Konrad Rzeszutek Wilk | 8b6bf74 | 2011-04-20 11:50:43 -0400 | [diff] [blame] | 929 | xen_blkif_get(blkif); |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 930 | |
| 931 | for (i = 0; i < nseg; i++) { |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 932 | while ((bio == NULL) || |
| 933 | (bio_add_page(bio, |
Roger Pau Monne | 0a8704a | 2012-10-24 18:58:45 +0200 | [diff] [blame^] | 934 | pages[i], |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 935 | seg[i].nsec << 9, |
| 936 | seg[i].buf & ~PAGE_MASK) == 0)) { |
Konrad Rzeszutek Wilk | 2e9977c | 2011-04-14 17:42:07 -0400 | [diff] [blame] | 937 | |
Konrad Rzeszutek Wilk | 03e0edf | 2011-05-12 16:19:23 -0400 | [diff] [blame] | 938 | bio = bio_alloc(GFP_KERNEL, nseg-i); |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 939 | if (unlikely(bio == NULL)) |
| 940 | goto fail_put_bio; |
| 941 | |
Konrad Rzeszutek Wilk | 03e0edf | 2011-05-12 16:19:23 -0400 | [diff] [blame] | 942 | biolist[nbio++] = bio; |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 943 | bio->bi_bdev = preq.bdev; |
| 944 | bio->bi_private = pending_req; |
| 945 | bio->bi_end_io = end_block_io_op; |
| 946 | bio->bi_sector = preq.sector_number; |
| 947 | } |
| 948 | |
| 949 | preq.sector_number += seg[i].nsec; |
| 950 | } |
| 951 | |
Li Dongyang | b3cb0d6 | 2011-09-01 18:39:10 +0800 | [diff] [blame] | 952 | /* This will be hit if the operation was a flush or discard. */ |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 953 | if (!bio) { |
Konrad Rzeszutek Wilk | 4214635 | 2011-10-12 17:26:47 -0400 | [diff] [blame] | 954 | BUG_ON(operation != WRITE_FLUSH); |
Konrad Rzeszutek Wilk | b0f8012 | 2011-05-12 16:23:06 -0400 | [diff] [blame] | 955 | |
Konrad Rzeszutek Wilk | 4214635 | 2011-10-12 17:26:47 -0400 | [diff] [blame] | 956 | bio = bio_alloc(GFP_KERNEL, 0); |
| 957 | if (unlikely(bio == NULL)) |
| 958 | goto fail_put_bio; |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 959 | |
Konrad Rzeszutek Wilk | 4214635 | 2011-10-12 17:26:47 -0400 | [diff] [blame] | 960 | biolist[nbio++] = bio; |
| 961 | bio->bi_bdev = preq.bdev; |
| 962 | bio->bi_private = pending_req; |
| 963 | bio->bi_end_io = end_block_io_op; |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 964 | } |
| 965 | |
Konrad Rzeszutek Wilk | 01f37f2 | 2011-05-11 15:57:09 -0400 | [diff] [blame] | 966 | /* |
| 967 | * We set it one so that the last submit_bio does not have to call |
Konrad Rzeszutek Wilk | 7708992 | 2011-04-15 10:51:27 -0400 | [diff] [blame] | 968 | * atomic_inc. |
| 969 | */ |
| 970 | atomic_set(&pending_req->pendcnt, nbio); |
| 971 | |
Konrad Rzeszutek Wilk | a19be5f | 2011-04-27 12:40:11 -0400 | [diff] [blame] | 972 | /* Get a reference count for the disk queue and start sending I/O */ |
| 973 | blk_start_plug(&plug); |
| 974 | |
Konrad Rzeszutek Wilk | 7708992 | 2011-04-15 10:51:27 -0400 | [diff] [blame] | 975 | for (i = 0; i < nbio; i++) |
| 976 | submit_bio(operation, biolist[i]); |
| 977 | |
Konrad Rzeszutek Wilk | a19be5f | 2011-04-27 12:40:11 -0400 | [diff] [blame] | 978 | /* Let the I/Os go.. */ |
Konrad Rzeszutek Wilk | 3d68b39 | 2011-05-05 13:42:10 -0400 | [diff] [blame] | 979 | blk_finish_plug(&plug); |
Konrad Rzeszutek Wilk | a19be5f | 2011-04-27 12:40:11 -0400 | [diff] [blame] | 980 | |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 981 | if (operation == READ) |
| 982 | blkif->st_rd_sect += preq.nr_sects; |
Konrad Rzeszutek Wilk | 5c62cb4 | 2011-10-10 12:33:21 -0400 | [diff] [blame] | 983 | else if (operation & WRITE) |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 984 | blkif->st_wr_sect += preq.nr_sects; |
| 985 | |
Konrad Rzeszutek Wilk | fc53bf7 | 2011-05-05 13:37:23 -0400 | [diff] [blame] | 986 | return 0; |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 987 | |
| 988 | fail_flush: |
Konrad Rzeszutek Wilk | 9f3aedf | 2011-04-15 11:50:34 -0400 | [diff] [blame] | 989 | xen_blkbk_unmap(pending_req); |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 990 | fail_response: |
Konrad Rzeszutek Wilk | 0faa8cc | 2011-04-14 17:58:19 -0400 | [diff] [blame] | 991 | /* Haven't submitted any bio's yet. */ |
Konrad Rzeszutek Wilk | 97e3683 | 2011-10-12 12:12:36 -0400 | [diff] [blame] | 992 | make_response(blkif, req->u.rw.id, req->operation, BLKIF_RSP_ERROR); |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 993 | free_req(pending_req); |
| 994 | msleep(1); /* back off a bit */ |
Konrad Rzeszutek Wilk | fc53bf7 | 2011-05-05 13:37:23 -0400 | [diff] [blame] | 995 | return -EIO; |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 996 | |
| 997 | fail_put_bio: |
Konrad Rzeszutek Wilk | 03e0edf | 2011-05-12 16:19:23 -0400 | [diff] [blame] | 998 | for (i = 0; i < nbio; i++) |
Konrad Rzeszutek Wilk | 7708992 | 2011-04-15 10:51:27 -0400 | [diff] [blame] | 999 | bio_put(biolist[i]); |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 1000 | __end_block_io_op(pending_req, -EINVAL); |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 1001 | msleep(1); /* back off a bit */ |
Konrad Rzeszutek Wilk | fc53bf7 | 2011-05-05 13:37:23 -0400 | [diff] [blame] | 1002 | return -EIO; |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 1003 | } |
| 1004 | |
| 1005 | |
| 1006 | |
Konrad Rzeszutek Wilk | a1397fa | 2011-04-14 17:05:23 -0400 | [diff] [blame] | 1007 | /* |
| 1008 | * Put a response on the ring on how the operation fared. |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 1009 | */ |
Konrad Rzeszutek Wilk | 30fd150 | 2011-05-12 16:47:48 -0400 | [diff] [blame] | 1010 | static void make_response(struct xen_blkif *blkif, u64 id, |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 1011 | unsigned short op, int st) |
| 1012 | { |
Jeremy Fitzhardinge | 8812293 | 2009-02-09 12:05:51 -0800 | [diff] [blame] | 1013 | struct blkif_response resp; |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 1014 | unsigned long flags; |
Jeremy Fitzhardinge | 8812293 | 2009-02-09 12:05:51 -0800 | [diff] [blame] | 1015 | union blkif_back_rings *blk_rings = &blkif->blk_rings; |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 1016 | int notify; |
| 1017 | |
| 1018 | resp.id = id; |
| 1019 | resp.operation = op; |
| 1020 | resp.status = st; |
| 1021 | |
| 1022 | spin_lock_irqsave(&blkif->blk_ring_lock, flags); |
| 1023 | /* Place on the response ring for the relevant domain. */ |
| 1024 | switch (blkif->blk_protocol) { |
| 1025 | case BLKIF_PROTOCOL_NATIVE: |
| 1026 | memcpy(RING_GET_RESPONSE(&blk_rings->native, blk_rings->native.rsp_prod_pvt), |
| 1027 | &resp, sizeof(resp)); |
| 1028 | break; |
| 1029 | case BLKIF_PROTOCOL_X86_32: |
| 1030 | memcpy(RING_GET_RESPONSE(&blk_rings->x86_32, blk_rings->x86_32.rsp_prod_pvt), |
| 1031 | &resp, sizeof(resp)); |
| 1032 | break; |
| 1033 | case BLKIF_PROTOCOL_X86_64: |
| 1034 | memcpy(RING_GET_RESPONSE(&blk_rings->x86_64, blk_rings->x86_64.rsp_prod_pvt), |
| 1035 | &resp, sizeof(resp)); |
| 1036 | break; |
| 1037 | default: |
| 1038 | BUG(); |
| 1039 | } |
| 1040 | blk_rings->common.rsp_prod_pvt++; |
| 1041 | RING_PUSH_RESPONSES_AND_CHECK_NOTIFY(&blk_rings->common, notify); |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 1042 | spin_unlock_irqrestore(&blkif->blk_ring_lock, flags); |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 1043 | if (notify) |
| 1044 | notify_remote_via_irq(blkif->irq); |
| 1045 | } |
| 1046 | |
Konrad Rzeszutek Wilk | 8b6bf74 | 2011-04-20 11:50:43 -0400 | [diff] [blame] | 1047 | static int __init xen_blkif_init(void) |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 1048 | { |
| 1049 | int i, mmap_pages; |
Konrad Rzeszutek Wilk | 8770b26 | 2009-10-08 13:23:09 -0400 | [diff] [blame] | 1050 | int rc = 0; |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 1051 | |
Daniel De Graaf | b2167ba | 2011-11-28 11:49:05 -0500 | [diff] [blame] | 1052 | if (!xen_domain()) |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 1053 | return -ENODEV; |
| 1054 | |
Konrad Rzeszutek Wilk | 2e9977c | 2011-04-14 17:42:07 -0400 | [diff] [blame] | 1055 | blkbk = kzalloc(sizeof(struct xen_blkbk), GFP_KERNEL); |
Konrad Rzeszutek Wilk | e8e2887 | 2011-02-25 10:51:29 -0500 | [diff] [blame] | 1056 | if (!blkbk) { |
Konrad Rzeszutek Wilk | 22b20f2 | 2011-05-12 16:43:12 -0400 | [diff] [blame] | 1057 | pr_alert(DRV_PFX "%s: out of memory!\n", __func__); |
Konrad Rzeszutek Wilk | e8e2887 | 2011-02-25 10:51:29 -0500 | [diff] [blame] | 1058 | return -ENOMEM; |
| 1059 | } |
| 1060 | |
Konrad Rzeszutek Wilk | 8b6bf74 | 2011-04-20 11:50:43 -0400 | [diff] [blame] | 1061 | mmap_pages = xen_blkif_reqs * BLKIF_MAX_SEGMENTS_PER_REQUEST; |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 1062 | |
Jan Beulich | 8e6dc6f | 2011-09-16 08:38:09 +0100 | [diff] [blame] | 1063 | blkbk->pending_reqs = kzalloc(sizeof(blkbk->pending_reqs[0]) * |
Konrad Rzeszutek Wilk | 8b6bf74 | 2011-04-20 11:50:43 -0400 | [diff] [blame] | 1064 | xen_blkif_reqs, GFP_KERNEL); |
Jan Beulich | 8e6dc6f | 2011-09-16 08:38:09 +0100 | [diff] [blame] | 1065 | blkbk->pending_grant_handles = kmalloc(sizeof(blkbk->pending_grant_handles[0]) * |
Konrad Rzeszutek Wilk | a742b02 | 2011-03-14 12:41:26 -0400 | [diff] [blame] | 1066 | mmap_pages, GFP_KERNEL); |
| 1067 | blkbk->pending_pages = kzalloc(sizeof(blkbk->pending_pages[0]) * |
| 1068 | mmap_pages, GFP_KERNEL); |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 1069 | |
Konrad Rzeszutek Wilk | 2e9977c | 2011-04-14 17:42:07 -0400 | [diff] [blame] | 1070 | if (!blkbk->pending_reqs || !blkbk->pending_grant_handles || |
| 1071 | !blkbk->pending_pages) { |
Konrad Rzeszutek Wilk | 8770b26 | 2009-10-08 13:23:09 -0400 | [diff] [blame] | 1072 | rc = -ENOMEM; |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 1073 | goto out_of_memory; |
Konrad Rzeszutek Wilk | 8770b26 | 2009-10-08 13:23:09 -0400 | [diff] [blame] | 1074 | } |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 1075 | |
Konrad Rzeszutek Wilk | 464fb41 | 2011-03-01 16:26:10 -0500 | [diff] [blame] | 1076 | for (i = 0; i < mmap_pages; i++) { |
Konrad Rzeszutek Wilk | e8e2887 | 2011-02-25 10:51:29 -0500 | [diff] [blame] | 1077 | blkbk->pending_grant_handles[i] = BLKBACK_INVALID_HANDLE; |
Konrad Rzeszutek Wilk | a742b02 | 2011-03-14 12:41:26 -0400 | [diff] [blame] | 1078 | blkbk->pending_pages[i] = alloc_page(GFP_KERNEL); |
Konrad Rzeszutek Wilk | 464fb41 | 2011-03-01 16:26:10 -0500 | [diff] [blame] | 1079 | if (blkbk->pending_pages[i] == NULL) { |
| 1080 | rc = -ENOMEM; |
| 1081 | goto out_of_memory; |
| 1082 | } |
| 1083 | } |
Konrad Rzeszutek Wilk | 8b6bf74 | 2011-04-20 11:50:43 -0400 | [diff] [blame] | 1084 | rc = xen_blkif_interface_init(); |
Konrad Rzeszutek Wilk | 8770b26 | 2009-10-08 13:23:09 -0400 | [diff] [blame] | 1085 | if (rc) |
| 1086 | goto failed_init; |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 1087 | |
Konrad Rzeszutek Wilk | e8e2887 | 2011-02-25 10:51:29 -0500 | [diff] [blame] | 1088 | INIT_LIST_HEAD(&blkbk->pending_free); |
| 1089 | spin_lock_init(&blkbk->pending_free_lock); |
| 1090 | init_waitqueue_head(&blkbk->pending_free_wq); |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 1091 | |
Konrad Rzeszutek Wilk | 8b6bf74 | 2011-04-20 11:50:43 -0400 | [diff] [blame] | 1092 | for (i = 0; i < xen_blkif_reqs; i++) |
Konrad Rzeszutek Wilk | 2e9977c | 2011-04-14 17:42:07 -0400 | [diff] [blame] | 1093 | list_add_tail(&blkbk->pending_reqs[i].free_list, |
| 1094 | &blkbk->pending_free); |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 1095 | |
Konrad Rzeszutek Wilk | 8b6bf74 | 2011-04-20 11:50:43 -0400 | [diff] [blame] | 1096 | rc = xen_blkif_xenbus_init(); |
Konrad Rzeszutek Wilk | 8770b26 | 2009-10-08 13:23:09 -0400 | [diff] [blame] | 1097 | if (rc) |
| 1098 | goto failed_init; |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 1099 | |
| 1100 | return 0; |
| 1101 | |
| 1102 | out_of_memory: |
Konrad Rzeszutek Wilk | 22b20f2 | 2011-05-12 16:43:12 -0400 | [diff] [blame] | 1103 | pr_alert(DRV_PFX "%s: out of memory\n", __func__); |
Konrad Rzeszutek Wilk | 8770b26 | 2009-10-08 13:23:09 -0400 | [diff] [blame] | 1104 | failed_init: |
Konrad Rzeszutek Wilk | e8e2887 | 2011-02-25 10:51:29 -0500 | [diff] [blame] | 1105 | kfree(blkbk->pending_reqs); |
Konrad Rzeszutek Wilk | a742b02 | 2011-03-14 12:41:26 -0400 | [diff] [blame] | 1106 | kfree(blkbk->pending_grant_handles); |
Dan Carpenter | 9b83c77 | 2011-05-27 09:27:16 +0300 | [diff] [blame] | 1107 | if (blkbk->pending_pages) { |
| 1108 | for (i = 0; i < mmap_pages; i++) { |
| 1109 | if (blkbk->pending_pages[i]) |
| 1110 | __free_page(blkbk->pending_pages[i]); |
| 1111 | } |
| 1112 | kfree(blkbk->pending_pages); |
Konrad Rzeszutek Wilk | 464fb41 | 2011-03-01 16:26:10 -0500 | [diff] [blame] | 1113 | } |
Konrad Rzeszutek Wilk | a742b02 | 2011-03-14 12:41:26 -0400 | [diff] [blame] | 1114 | kfree(blkbk); |
Konrad Rzeszutek Wilk | e8e2887 | 2011-02-25 10:51:29 -0500 | [diff] [blame] | 1115 | blkbk = NULL; |
Konrad Rzeszutek Wilk | 8770b26 | 2009-10-08 13:23:09 -0400 | [diff] [blame] | 1116 | return rc; |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 1117 | } |
| 1118 | |
Konrad Rzeszutek Wilk | 8b6bf74 | 2011-04-20 11:50:43 -0400 | [diff] [blame] | 1119 | module_init(xen_blkif_init); |
Konrad Rzeszutek Wilk | 4d05a28 | 2011-04-14 18:25:47 -0400 | [diff] [blame] | 1120 | |
| 1121 | MODULE_LICENSE("Dual BSD/GPL"); |
Bastian Blank | a7e9357 | 2011-06-29 14:40:50 +0200 | [diff] [blame] | 1122 | MODULE_ALIAS("xen-backend:vbd"); |