blob: 68b9136847afb65687a426a0c16e6cca6760caf8 [file] [log] [blame]
Rusty Russell0a8a69d2007-10-22 11:03:40 +10001/* Virtio ring implementation.
2 *
3 * Copyright 2007 Rusty Russell IBM Corporation
4 *
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation; either version 2 of the License, or
8 * (at your option) any later version.
9 *
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
14 *
15 * You should have received a copy of the GNU General Public License
16 * along with this program; if not, write to the Free Software
17 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
18 */
19#include <linux/virtio.h>
20#include <linux/virtio_ring.h>
Rusty Russelle34f8722008-07-25 12:06:13 -050021#include <linux/virtio_config.h>
Rusty Russell0a8a69d2007-10-22 11:03:40 +100022#include <linux/device.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090023#include <linux/slab.h>
Rusty Russell0a8a69d2007-10-22 11:03:40 +100024
Michael S. Tsirkind57ed952010-01-28 00:42:23 +020025/* virtio guest is communicating with a virtual "device" that actually runs on
26 * a host processor. Memory barriers are used to control SMP effects. */
27#ifdef CONFIG_SMP
28/* Where possible, use SMP barriers which are more lightweight than mandatory
29 * barriers, because mandatory barriers control MMIO effects on accesses
30 * through relaxed memory I/O windows (which virtio does not use). */
31#define virtio_mb() smp_mb()
32#define virtio_rmb() smp_rmb()
33#define virtio_wmb() smp_wmb()
34#else
35/* We must force memory ordering even if guest is UP since host could be
36 * running on another CPU, but SMP barriers are defined to barrier() in that
37 * configuration. So fall back to mandatory barriers instead. */
38#define virtio_mb() mb()
39#define virtio_rmb() rmb()
40#define virtio_wmb() wmb()
41#endif
42
Rusty Russell0a8a69d2007-10-22 11:03:40 +100043#ifdef DEBUG
44/* For development, we want to crash whenever the ring is screwed. */
Rusty Russell9499f5e2009-06-12 22:16:35 -060045#define BAD_RING(_vq, fmt, args...) \
46 do { \
47 dev_err(&(_vq)->vq.vdev->dev, \
48 "%s:"fmt, (_vq)->vq.name, ##args); \
49 BUG(); \
50 } while (0)
Rusty Russellc5f841f2009-03-30 21:55:22 -060051/* Caller is supposed to guarantee no reentry. */
52#define START_USE(_vq) \
53 do { \
54 if ((_vq)->in_use) \
Rusty Russell9499f5e2009-06-12 22:16:35 -060055 panic("%s:in_use = %i\n", \
56 (_vq)->vq.name, (_vq)->in_use); \
Rusty Russellc5f841f2009-03-30 21:55:22 -060057 (_vq)->in_use = __LINE__; \
Rusty Russell9499f5e2009-06-12 22:16:35 -060058 } while (0)
Roel Kluin3a35ce72009-01-22 16:42:57 +010059#define END_USE(_vq) \
Rusty Russell97a545a2010-02-24 14:22:22 -060060 do { BUG_ON(!(_vq)->in_use); (_vq)->in_use = 0; } while(0)
Rusty Russell0a8a69d2007-10-22 11:03:40 +100061#else
Rusty Russell9499f5e2009-06-12 22:16:35 -060062#define BAD_RING(_vq, fmt, args...) \
63 do { \
64 dev_err(&_vq->vq.vdev->dev, \
65 "%s:"fmt, (_vq)->vq.name, ##args); \
66 (_vq)->broken = true; \
67 } while (0)
Rusty Russell0a8a69d2007-10-22 11:03:40 +100068#define START_USE(vq)
69#define END_USE(vq)
70#endif
71
72struct vring_virtqueue
73{
74 struct virtqueue vq;
75
76 /* Actual memory layout for this queue */
77 struct vring vring;
78
79 /* Other side has made a mess, don't try any more. */
80 bool broken;
81
Mark McLoughlin9fa29b9d2009-05-11 18:11:45 +010082 /* Host supports indirect buffers */
83 bool indirect;
84
Michael S. Tsirkina5c262c2011-05-20 02:10:44 +030085 /* Host publishes avail event idx */
86 bool event;
87
Rusty Russell0a8a69d2007-10-22 11:03:40 +100088 /* Number of free buffers */
89 unsigned int num_free;
90 /* Head of free buffer list. */
91 unsigned int free_head;
92 /* Number we've added since last sync. */
93 unsigned int num_added;
94
95 /* Last used index we've seen. */
Anthony Liguori1bc49532007-11-07 15:49:24 -060096 u16 last_used_idx;
Rusty Russell0a8a69d2007-10-22 11:03:40 +100097
98 /* How to notify other side. FIXME: commonalize hcalls! */
99 void (*notify)(struct virtqueue *vq);
100
101#ifdef DEBUG
102 /* They're supposed to lock for us. */
103 unsigned int in_use;
104#endif
105
106 /* Tokens for callbacks. */
107 void *data[];
108};
109
110#define to_vvq(_vq) container_of(_vq, struct vring_virtqueue, vq)
111
Mark McLoughlin9fa29b9d2009-05-11 18:11:45 +0100112/* Set up an indirect table of descriptors and add it to the queue. */
113static int vring_add_indirect(struct vring_virtqueue *vq,
114 struct scatterlist sg[],
115 unsigned int out,
Michael S. Tsirkinbbd603e2010-04-29 17:26:37 +0300116 unsigned int in,
117 gfp_t gfp)
Mark McLoughlin9fa29b9d2009-05-11 18:11:45 +0100118{
119 struct vring_desc *desc;
120 unsigned head;
121 int i;
122
Michael S. Tsirkinbbd603e2010-04-29 17:26:37 +0300123 desc = kmalloc((out + in) * sizeof(struct vring_desc), gfp);
Mark McLoughlin9fa29b9d2009-05-11 18:11:45 +0100124 if (!desc)
Michael S. Tsirkin686d3632010-06-10 18:16:11 +0300125 return -ENOMEM;
Mark McLoughlin9fa29b9d2009-05-11 18:11:45 +0100126
127 /* Transfer entries from the sg list into the indirect page */
128 for (i = 0; i < out; i++) {
129 desc[i].flags = VRING_DESC_F_NEXT;
130 desc[i].addr = sg_phys(sg);
131 desc[i].len = sg->length;
132 desc[i].next = i+1;
133 sg++;
134 }
135 for (; i < (out + in); i++) {
136 desc[i].flags = VRING_DESC_F_NEXT|VRING_DESC_F_WRITE;
137 desc[i].addr = sg_phys(sg);
138 desc[i].len = sg->length;
139 desc[i].next = i+1;
140 sg++;
141 }
142
143 /* Last one doesn't continue. */
144 desc[i-1].flags &= ~VRING_DESC_F_NEXT;
145 desc[i-1].next = 0;
146
147 /* We're about to use a buffer */
148 vq->num_free--;
149
150 /* Use a single buffer which doesn't continue */
151 head = vq->free_head;
152 vq->vring.desc[head].flags = VRING_DESC_F_INDIRECT;
153 vq->vring.desc[head].addr = virt_to_phys(desc);
154 vq->vring.desc[head].len = i * sizeof(struct vring_desc);
155
156 /* Update free pointer */
157 vq->free_head = vq->vring.desc[head].next;
158
159 return head;
160}
161
Michael S. Tsirkinbbd603e2010-04-29 17:26:37 +0300162int virtqueue_add_buf_gfp(struct virtqueue *_vq,
163 struct scatterlist sg[],
164 unsigned int out,
165 unsigned int in,
166 void *data,
167 gfp_t gfp)
Rusty Russell0a8a69d2007-10-22 11:03:40 +1000168{
169 struct vring_virtqueue *vq = to_vvq(_vq);
Michael S. Tsirkin1fe9b6f2010-07-26 16:55:30 +0930170 unsigned int i, avail, uninitialized_var(prev);
171 int head;
Rusty Russell0a8a69d2007-10-22 11:03:40 +1000172
Mark McLoughlin9fa29b9d2009-05-11 18:11:45 +0100173 START_USE(vq);
174
Rusty Russell0a8a69d2007-10-22 11:03:40 +1000175 BUG_ON(data == NULL);
Mark McLoughlin9fa29b9d2009-05-11 18:11:45 +0100176
177 /* If the host supports indirect descriptor tables, and we have multiple
178 * buffers, then go indirect. FIXME: tune this threshold */
179 if (vq->indirect && (out + in) > 1 && vq->num_free) {
Michael S. Tsirkinbbd603e2010-04-29 17:26:37 +0300180 head = vring_add_indirect(vq, sg, out, in, gfp);
Michael S. Tsirkin1fe9b6f2010-07-26 16:55:30 +0930181 if (likely(head >= 0))
Mark McLoughlin9fa29b9d2009-05-11 18:11:45 +0100182 goto add_head;
183 }
184
Rusty Russell0a8a69d2007-10-22 11:03:40 +1000185 BUG_ON(out + in > vq->vring.num);
186 BUG_ON(out + in == 0);
187
Rusty Russell0a8a69d2007-10-22 11:03:40 +1000188 if (vq->num_free < out + in) {
189 pr_debug("Can't add buf len %i - avail = %i\n",
190 out + in, vq->num_free);
Rusty Russell44653ea2008-07-25 12:06:04 -0500191 /* FIXME: for historical reasons, we force a notify here if
192 * there are outgoing parts to the buffer. Presumably the
193 * host should service the ring ASAP. */
194 if (out)
195 vq->notify(&vq->vq);
Rusty Russell0a8a69d2007-10-22 11:03:40 +1000196 END_USE(vq);
197 return -ENOSPC;
198 }
199
200 /* We're about to use some buffers from the free list. */
201 vq->num_free -= out + in;
202
203 head = vq->free_head;
204 for (i = vq->free_head; out; i = vq->vring.desc[i].next, out--) {
205 vq->vring.desc[i].flags = VRING_DESC_F_NEXT;
Rusty Russell15f9c892008-02-04 23:50:05 -0500206 vq->vring.desc[i].addr = sg_phys(sg);
Rusty Russell0a8a69d2007-10-22 11:03:40 +1000207 vq->vring.desc[i].len = sg->length;
208 prev = i;
209 sg++;
210 }
211 for (; in; i = vq->vring.desc[i].next, in--) {
212 vq->vring.desc[i].flags = VRING_DESC_F_NEXT|VRING_DESC_F_WRITE;
Rusty Russell15f9c892008-02-04 23:50:05 -0500213 vq->vring.desc[i].addr = sg_phys(sg);
Rusty Russell0a8a69d2007-10-22 11:03:40 +1000214 vq->vring.desc[i].len = sg->length;
215 prev = i;
216 sg++;
217 }
218 /* Last one doesn't continue. */
219 vq->vring.desc[prev].flags &= ~VRING_DESC_F_NEXT;
220
221 /* Update free pointer */
222 vq->free_head = i;
223
Mark McLoughlin9fa29b9d2009-05-11 18:11:45 +0100224add_head:
Rusty Russell0a8a69d2007-10-22 11:03:40 +1000225 /* Set token. */
226 vq->data[head] = data;
227
228 /* Put entry in available array (but don't update avail->idx until they
229 * do sync). FIXME: avoid modulus here? */
230 avail = (vq->vring.avail->idx + vq->num_added++) % vq->vring.num;
231 vq->vring.avail->ring[avail] = head;
232
233 pr_debug("Added buffer head %i to %p\n", head, vq);
234 END_USE(vq);
Rusty Russell3c1b27d2009-09-23 22:26:31 -0600235
Rusty Russell3c1b27d2009-09-23 22:26:31 -0600236 return vq->num_free;
Rusty Russell0a8a69d2007-10-22 11:03:40 +1000237}
Michael S. Tsirkinbbd603e2010-04-29 17:26:37 +0300238EXPORT_SYMBOL_GPL(virtqueue_add_buf_gfp);
Rusty Russell0a8a69d2007-10-22 11:03:40 +1000239
Michael S. Tsirkin7c5e9ed2010-04-12 16:19:07 +0300240void virtqueue_kick(struct virtqueue *_vq)
Rusty Russell0a8a69d2007-10-22 11:03:40 +1000241{
242 struct vring_virtqueue *vq = to_vvq(_vq);
Michael S. Tsirkina5c262c2011-05-20 02:10:44 +0300243 u16 new, old;
Rusty Russell0a8a69d2007-10-22 11:03:40 +1000244 START_USE(vq);
245 /* Descriptors and available array need to be set before we expose the
246 * new available array entries. */
Michael S. Tsirkind57ed952010-01-28 00:42:23 +0200247 virtio_wmb();
Rusty Russell0a8a69d2007-10-22 11:03:40 +1000248
Michael S. Tsirkina5c262c2011-05-20 02:10:44 +0300249 old = vq->vring.avail->idx;
250 new = vq->vring.avail->idx = old + vq->num_added;
Rusty Russell0a8a69d2007-10-22 11:03:40 +1000251 vq->num_added = 0;
252
253 /* Need to update avail index before checking if we should notify */
Michael S. Tsirkind57ed952010-01-28 00:42:23 +0200254 virtio_mb();
Rusty Russell0a8a69d2007-10-22 11:03:40 +1000255
Michael S. Tsirkina5c262c2011-05-20 02:10:44 +0300256 if (vq->event ?
257 vring_need_event(vring_avail_event(&vq->vring), new, old) :
258 !(vq->vring.used->flags & VRING_USED_F_NO_NOTIFY))
Rusty Russell0a8a69d2007-10-22 11:03:40 +1000259 /* Prod other side to tell it about changes. */
260 vq->notify(&vq->vq);
261
262 END_USE(vq);
263}
Michael S. Tsirkin7c5e9ed2010-04-12 16:19:07 +0300264EXPORT_SYMBOL_GPL(virtqueue_kick);
Rusty Russell0a8a69d2007-10-22 11:03:40 +1000265
266static void detach_buf(struct vring_virtqueue *vq, unsigned int head)
267{
268 unsigned int i;
269
270 /* Clear data ptr. */
271 vq->data[head] = NULL;
272
273 /* Put back on free list: find end */
274 i = head;
Mark McLoughlin9fa29b9d2009-05-11 18:11:45 +0100275
276 /* Free the indirect table */
277 if (vq->vring.desc[i].flags & VRING_DESC_F_INDIRECT)
278 kfree(phys_to_virt(vq->vring.desc[i].addr));
279
Rusty Russell0a8a69d2007-10-22 11:03:40 +1000280 while (vq->vring.desc[i].flags & VRING_DESC_F_NEXT) {
281 i = vq->vring.desc[i].next;
282 vq->num_free++;
283 }
284
285 vq->vring.desc[i].next = vq->free_head;
286 vq->free_head = head;
287 /* Plus final descriptor */
288 vq->num_free++;
289}
290
Rusty Russell0a8a69d2007-10-22 11:03:40 +1000291static inline bool more_used(const struct vring_virtqueue *vq)
292{
293 return vq->last_used_idx != vq->vring.used->idx;
294}
295
Michael S. Tsirkin7c5e9ed2010-04-12 16:19:07 +0300296void *virtqueue_get_buf(struct virtqueue *_vq, unsigned int *len)
Rusty Russell0a8a69d2007-10-22 11:03:40 +1000297{
298 struct vring_virtqueue *vq = to_vvq(_vq);
299 void *ret;
300 unsigned int i;
301
302 START_USE(vq);
303
Rusty Russell5ef82752008-05-02 21:50:43 -0500304 if (unlikely(vq->broken)) {
305 END_USE(vq);
306 return NULL;
307 }
308
Rusty Russell0a8a69d2007-10-22 11:03:40 +1000309 if (!more_used(vq)) {
310 pr_debug("No more buffers in queue\n");
311 END_USE(vq);
312 return NULL;
313 }
314
Michael S. Tsirkin2d61ba92009-10-25 15:28:53 +0200315 /* Only get used array entries after they have been exposed by host. */
Michael S. Tsirkind57ed952010-01-28 00:42:23 +0200316 virtio_rmb();
Michael S. Tsirkin2d61ba92009-10-25 15:28:53 +0200317
Rusty Russell0a8a69d2007-10-22 11:03:40 +1000318 i = vq->vring.used->ring[vq->last_used_idx%vq->vring.num].id;
319 *len = vq->vring.used->ring[vq->last_used_idx%vq->vring.num].len;
320
321 if (unlikely(i >= vq->vring.num)) {
322 BAD_RING(vq, "id %u out of range\n", i);
323 return NULL;
324 }
325 if (unlikely(!vq->data[i])) {
326 BAD_RING(vq, "id %u is not a head!\n", i);
327 return NULL;
328 }
329
330 /* detach_buf clears data, so grab it now. */
331 ret = vq->data[i];
332 detach_buf(vq, i);
333 vq->last_used_idx++;
Michael S. Tsirkina5c262c2011-05-20 02:10:44 +0300334 /* If we expect an interrupt for the next entry, tell host
335 * by writing event index and flush out the write before
336 * the read in the next get_buf call. */
337 if (!(vq->vring.avail->flags & VRING_AVAIL_F_NO_INTERRUPT)) {
338 vring_used_event(&vq->vring) = vq->last_used_idx;
339 virtio_mb();
340 }
341
Rusty Russell0a8a69d2007-10-22 11:03:40 +1000342 END_USE(vq);
343 return ret;
344}
Michael S. Tsirkin7c5e9ed2010-04-12 16:19:07 +0300345EXPORT_SYMBOL_GPL(virtqueue_get_buf);
Rusty Russell0a8a69d2007-10-22 11:03:40 +1000346
Michael S. Tsirkin7c5e9ed2010-04-12 16:19:07 +0300347void virtqueue_disable_cb(struct virtqueue *_vq)
Rusty Russell18445c42008-02-04 23:49:57 -0500348{
349 struct vring_virtqueue *vq = to_vvq(_vq);
350
Rusty Russell18445c42008-02-04 23:49:57 -0500351 vq->vring.avail->flags |= VRING_AVAIL_F_NO_INTERRUPT;
Rusty Russell18445c42008-02-04 23:49:57 -0500352}
Michael S. Tsirkin7c5e9ed2010-04-12 16:19:07 +0300353EXPORT_SYMBOL_GPL(virtqueue_disable_cb);
Rusty Russell18445c42008-02-04 23:49:57 -0500354
Michael S. Tsirkin7c5e9ed2010-04-12 16:19:07 +0300355bool virtqueue_enable_cb(struct virtqueue *_vq)
Rusty Russell0a8a69d2007-10-22 11:03:40 +1000356{
357 struct vring_virtqueue *vq = to_vvq(_vq);
358
359 START_USE(vq);
Rusty Russell0a8a69d2007-10-22 11:03:40 +1000360
361 /* We optimistically turn back on interrupts, then check if there was
362 * more to do. */
Michael S. Tsirkina5c262c2011-05-20 02:10:44 +0300363 /* Depending on the VIRTIO_RING_F_EVENT_IDX feature, we need to
364 * either clear the flags bit or point the event index at the next
365 * entry. Always do both to keep code simple. */
Rusty Russell0a8a69d2007-10-22 11:03:40 +1000366 vq->vring.avail->flags &= ~VRING_AVAIL_F_NO_INTERRUPT;
Michael S. Tsirkina5c262c2011-05-20 02:10:44 +0300367 vring_used_event(&vq->vring) = vq->last_used_idx;
Michael S. Tsirkind57ed952010-01-28 00:42:23 +0200368 virtio_mb();
Rusty Russell0a8a69d2007-10-22 11:03:40 +1000369 if (unlikely(more_used(vq))) {
Rusty Russell0a8a69d2007-10-22 11:03:40 +1000370 END_USE(vq);
371 return false;
372 }
373
374 END_USE(vq);
375 return true;
376}
Michael S. Tsirkin7c5e9ed2010-04-12 16:19:07 +0300377EXPORT_SYMBOL_GPL(virtqueue_enable_cb);
Rusty Russell0a8a69d2007-10-22 11:03:40 +1000378
Michael S. Tsirkin7ab358c2011-05-20 02:11:14 +0300379bool virtqueue_enable_cb_delayed(struct virtqueue *_vq)
380{
381 struct vring_virtqueue *vq = to_vvq(_vq);
382 u16 bufs;
383
384 START_USE(vq);
385
386 /* We optimistically turn back on interrupts, then check if there was
387 * more to do. */
388 /* Depending on the VIRTIO_RING_F_USED_EVENT_IDX feature, we need to
389 * either clear the flags bit or point the event index at the next
390 * entry. Always do both to keep code simple. */
391 vq->vring.avail->flags &= ~VRING_AVAIL_F_NO_INTERRUPT;
392 /* TODO: tune this threshold */
393 bufs = (u16)(vq->vring.avail->idx - vq->last_used_idx) * 3 / 4;
394 vring_used_event(&vq->vring) = vq->last_used_idx + bufs;
395 virtio_mb();
396 if (unlikely((u16)(vq->vring.used->idx - vq->last_used_idx) > bufs)) {
397 END_USE(vq);
398 return false;
399 }
400
401 END_USE(vq);
402 return true;
403}
404EXPORT_SYMBOL_GPL(virtqueue_enable_cb_delayed);
405
Michael S. Tsirkin7c5e9ed2010-04-12 16:19:07 +0300406void *virtqueue_detach_unused_buf(struct virtqueue *_vq)
Shirley Mac021eac2010-01-18 19:15:23 +0530407{
408 struct vring_virtqueue *vq = to_vvq(_vq);
409 unsigned int i;
410 void *buf;
411
412 START_USE(vq);
413
414 for (i = 0; i < vq->vring.num; i++) {
415 if (!vq->data[i])
416 continue;
417 /* detach_buf clears data, so grab it now. */
418 buf = vq->data[i];
419 detach_buf(vq, i);
Amit Shahb3258ff2011-03-16 19:12:10 +0530420 vq->vring.avail->idx--;
Shirley Mac021eac2010-01-18 19:15:23 +0530421 END_USE(vq);
422 return buf;
423 }
424 /* That should have freed everything. */
425 BUG_ON(vq->num_free != vq->vring.num);
426
427 END_USE(vq);
428 return NULL;
429}
Michael S. Tsirkin7c5e9ed2010-04-12 16:19:07 +0300430EXPORT_SYMBOL_GPL(virtqueue_detach_unused_buf);
Shirley Mac021eac2010-01-18 19:15:23 +0530431
Rusty Russell0a8a69d2007-10-22 11:03:40 +1000432irqreturn_t vring_interrupt(int irq, void *_vq)
433{
434 struct vring_virtqueue *vq = to_vvq(_vq);
435
436 if (!more_used(vq)) {
437 pr_debug("virtqueue interrupt with no work for %p\n", vq);
438 return IRQ_NONE;
439 }
440
441 if (unlikely(vq->broken))
442 return IRQ_HANDLED;
443
444 pr_debug("virtqueue callback for %p (%p)\n", vq, vq->vq.callback);
Rusty Russell18445c42008-02-04 23:49:57 -0500445 if (vq->vq.callback)
446 vq->vq.callback(&vq->vq);
Rusty Russell0a8a69d2007-10-22 11:03:40 +1000447
448 return IRQ_HANDLED;
449}
Rusty Russellc6fd4702008-02-04 23:50:05 -0500450EXPORT_SYMBOL_GPL(vring_interrupt);
Rusty Russell0a8a69d2007-10-22 11:03:40 +1000451
Rusty Russell0a8a69d2007-10-22 11:03:40 +1000452struct virtqueue *vring_new_virtqueue(unsigned int num,
Rusty Russell87c7d572008-12-30 09:26:03 -0600453 unsigned int vring_align,
Rusty Russell0a8a69d2007-10-22 11:03:40 +1000454 struct virtio_device *vdev,
455 void *pages,
456 void (*notify)(struct virtqueue *),
Rusty Russell9499f5e2009-06-12 22:16:35 -0600457 void (*callback)(struct virtqueue *),
458 const char *name)
Rusty Russell0a8a69d2007-10-22 11:03:40 +1000459{
460 struct vring_virtqueue *vq;
461 unsigned int i;
462
Rusty Russell42b36cc2007-11-12 13:39:18 +1100463 /* We assume num is a power of 2. */
464 if (num & (num - 1)) {
465 dev_warn(&vdev->dev, "Bad virtqueue length %u\n", num);
466 return NULL;
467 }
468
Rusty Russell0a8a69d2007-10-22 11:03:40 +1000469 vq = kmalloc(sizeof(*vq) + sizeof(void *)*num, GFP_KERNEL);
470 if (!vq)
471 return NULL;
472
Rusty Russell87c7d572008-12-30 09:26:03 -0600473 vring_init(&vq->vring, num, pages, vring_align);
Rusty Russell0a8a69d2007-10-22 11:03:40 +1000474 vq->vq.callback = callback;
475 vq->vq.vdev = vdev;
Rusty Russell9499f5e2009-06-12 22:16:35 -0600476 vq->vq.name = name;
Rusty Russell0a8a69d2007-10-22 11:03:40 +1000477 vq->notify = notify;
478 vq->broken = false;
479 vq->last_used_idx = 0;
480 vq->num_added = 0;
Rusty Russell9499f5e2009-06-12 22:16:35 -0600481 list_add_tail(&vq->vq.list, &vdev->vqs);
Rusty Russell0a8a69d2007-10-22 11:03:40 +1000482#ifdef DEBUG
483 vq->in_use = false;
484#endif
485
Mark McLoughlin9fa29b9d2009-05-11 18:11:45 +0100486 vq->indirect = virtio_has_feature(vdev, VIRTIO_RING_F_INDIRECT_DESC);
Michael S. Tsirkina5c262c2011-05-20 02:10:44 +0300487 vq->event = virtio_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX);
Mark McLoughlin9fa29b9d2009-05-11 18:11:45 +0100488
Rusty Russell0a8a69d2007-10-22 11:03:40 +1000489 /* No callback? Tell other side not to bother us. */
490 if (!callback)
491 vq->vring.avail->flags |= VRING_AVAIL_F_NO_INTERRUPT;
492
493 /* Put everything in free lists. */
494 vq->num_free = num;
495 vq->free_head = 0;
Amit Shah3b870622010-02-12 10:32:14 +0530496 for (i = 0; i < num-1; i++) {
Rusty Russell0a8a69d2007-10-22 11:03:40 +1000497 vq->vring.desc[i].next = i+1;
Amit Shah3b870622010-02-12 10:32:14 +0530498 vq->data[i] = NULL;
499 }
500 vq->data[i] = NULL;
Rusty Russell0a8a69d2007-10-22 11:03:40 +1000501
502 return &vq->vq;
503}
Rusty Russellc6fd4702008-02-04 23:50:05 -0500504EXPORT_SYMBOL_GPL(vring_new_virtqueue);
Rusty Russell0a8a69d2007-10-22 11:03:40 +1000505
506void vring_del_virtqueue(struct virtqueue *vq)
507{
Rusty Russell9499f5e2009-06-12 22:16:35 -0600508 list_del(&vq->list);
Rusty Russell0a8a69d2007-10-22 11:03:40 +1000509 kfree(to_vvq(vq));
510}
Rusty Russellc6fd4702008-02-04 23:50:05 -0500511EXPORT_SYMBOL_GPL(vring_del_virtqueue);
Rusty Russell0a8a69d2007-10-22 11:03:40 +1000512
Rusty Russelle34f8722008-07-25 12:06:13 -0500513/* Manipulates transport-specific feature bits. */
514void vring_transport_features(struct virtio_device *vdev)
515{
516 unsigned int i;
517
518 for (i = VIRTIO_TRANSPORT_F_START; i < VIRTIO_TRANSPORT_F_END; i++) {
519 switch (i) {
Mark McLoughlin9fa29b9d2009-05-11 18:11:45 +0100520 case VIRTIO_RING_F_INDIRECT_DESC:
521 break;
Michael S. Tsirkina5c262c2011-05-20 02:10:44 +0300522 case VIRTIO_RING_F_EVENT_IDX:
523 break;
Rusty Russelle34f8722008-07-25 12:06:13 -0500524 default:
525 /* We don't understand this bit. */
526 clear_bit(i, vdev->features);
527 }
528 }
529}
530EXPORT_SYMBOL_GPL(vring_transport_features);
531
Rusty Russellc6fd4702008-02-04 23:50:05 -0500532MODULE_LICENSE("GPL");