blob: 9eda69e406787a55a347370328cdf07d5375dc0a [file] [log] [blame]
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +00001/* Copyright (C) 2009 Red Hat, Inc.
2 * Author: Michael S. Tsirkin <mst@redhat.com>
3 *
4 * This work is licensed under the terms of the GNU GPL, version 2.
5 *
6 * virtio-net server in host kernel.
7 */
8
9#include <linux/compat.h>
10#include <linux/eventfd.h>
11#include <linux/vhost.h>
12#include <linux/virtio_net.h>
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +000013#include <linux/miscdevice.h>
14#include <linux/module.h>
Michael S. Tsirkinbab632d2011-07-18 03:48:46 +000015#include <linux/moduleparam.h>
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +000016#include <linux/mutex.h>
17#include <linux/workqueue.h>
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +000018#include <linux/file.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090019#include <linux/slab.h>
Michael S. Tsirkin23cc5a92013-01-23 21:46:47 +010020#include <linux/vmalloc.h>
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +000021
22#include <linux/net.h>
23#include <linux/if_packet.h>
24#include <linux/if_arp.h>
25#include <linux/if_tun.h>
Arnd Bergmann501c7742010-02-18 05:46:50 +000026#include <linux/if_macvlan.h>
Basil Gorc53cff5e2012-05-03 22:55:23 +000027#include <linux/if_vlan.h>
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +000028
29#include <net/sock.h>
30
31#include "vhost.h"
32
Michael S. Tsirkinf9611c42012-12-06 14:56:00 +020033static int experimental_zcopytx = 1;
Michael S. Tsirkinbab632d2011-07-18 03:48:46 +000034module_param(experimental_zcopytx, int, 0444);
Michael S. Tsirkinf9611c42012-12-06 14:56:00 +020035MODULE_PARM_DESC(experimental_zcopytx, "Enable Zero Copy TX;"
36 " 1 -Enable; 0 - Disable");
Michael S. Tsirkinbab632d2011-07-18 03:48:46 +000037
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +000038/* Max number of bytes transferred before requeueing the job.
39 * Using this limit prevents one virtqueue from starving others. */
40#define VHOST_NET_WEIGHT 0x80000
41
Michael S. Tsirkinbab632d2011-07-18 03:48:46 +000042/* MAX number of TX used buffers for outstanding zerocopy */
43#define VHOST_MAX_PEND 128
44#define VHOST_GOODCOPY_LEN 256
45
Michael S. Tsirkineaae8132012-11-01 09:16:51 +000046/*
47 * For transmit, used buffer len is unused; we override it to track buffer
48 * status internally; used for zerocopy tx only.
49 */
50/* Lower device DMA failed */
Michael S. Tsirkinbf995732014-10-24 11:49:27 +030051#define VHOST_DMA_FAILED_LEN ((__force __virtio32)3)
Michael S. Tsirkineaae8132012-11-01 09:16:51 +000052/* Lower device DMA done */
Michael S. Tsirkinbf995732014-10-24 11:49:27 +030053#define VHOST_DMA_DONE_LEN ((__force __virtio32)2)
Michael S. Tsirkineaae8132012-11-01 09:16:51 +000054/* Lower device DMA in progress */
Michael S. Tsirkinbf995732014-10-24 11:49:27 +030055#define VHOST_DMA_IN_PROGRESS ((__force __virtio32)1)
Michael S. Tsirkineaae8132012-11-01 09:16:51 +000056/* Buffer unused */
Michael S. Tsirkinbf995732014-10-24 11:49:27 +030057#define VHOST_DMA_CLEAR_LEN ((__force __virtio32)0)
Michael S. Tsirkineaae8132012-11-01 09:16:51 +000058
Michael S. Tsirkinbf995732014-10-24 11:49:27 +030059#define VHOST_DMA_IS_DONE(len) ((__force u32)(len) >= (__force u32)VHOST_DMA_DONE_LEN)
Michael S. Tsirkineaae8132012-11-01 09:16:51 +000060
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +000061enum {
Asias He8570a6e2013-05-06 16:38:20 +080062 VHOST_NET_FEATURES = VHOST_FEATURES |
63 (1ULL << VHOST_NET_F_VIRTIO_NET_HDR) |
Michael S. Tsirkin4e9fa502015-09-09 22:24:56 +030064 (1ULL << VIRTIO_NET_F_MRG_RXBUF)
Asias He8570a6e2013-05-06 16:38:20 +080065};
66
67enum {
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +000068 VHOST_NET_VQ_RX = 0,
69 VHOST_NET_VQ_TX = 1,
70 VHOST_NET_VQ_MAX = 2,
71};
72
Asias Hefe729a52013-05-06 16:38:24 +080073struct vhost_net_ubuf_ref {
Michael S. Tsirkin0ad8b482014-02-13 11:42:05 +020074 /* refcount follows semantics similar to kref:
75 * 0: object is released
76 * 1: no outstanding ubufs
77 * >1: outstanding ubufs
78 */
79 atomic_t refcount;
Asias He28394002013-04-27 15:07:46 +080080 wait_queue_head_t wait;
81 struct vhost_virtqueue *vq;
82};
83
Asias He3ab2e422013-04-27 11:16:48 +080084struct vhost_net_virtqueue {
85 struct vhost_virtqueue vq;
Michael S. Tsirkin81f95a52013-04-28 15:51:40 +030086 size_t vhost_hlen;
87 size_t sock_hlen;
Asias He28394002013-04-27 15:07:46 +080088 /* vhost zerocopy support fields below: */
89 /* last used idx for outstanding DMA zerocopy buffers */
90 int upend_idx;
91 /* first used idx for DMA done zerocopy buffers */
92 int done_idx;
93 /* an array of userspace buffers info */
94 struct ubuf_info *ubuf_info;
95 /* Reference counting for outstanding ubufs.
96 * Protected by vq mutex. Writers must also take device mutex. */
Asias Hefe729a52013-05-06 16:38:24 +080097 struct vhost_net_ubuf_ref *ubufs;
Asias He3ab2e422013-04-27 11:16:48 +080098};
99
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +0000100struct vhost_net {
101 struct vhost_dev dev;
Asias He3ab2e422013-04-27 11:16:48 +0800102 struct vhost_net_virtqueue vqs[VHOST_NET_VQ_MAX];
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +0000103 struct vhost_poll poll[VHOST_NET_VQ_MAX];
Michael S. Tsirkineaae8132012-11-01 09:16:51 +0000104 /* Number of TX recently submitted.
105 * Protected by tx vq lock. */
106 unsigned tx_packets;
107 /* Number of times zerocopy TX recently failed.
108 * Protected by tx vq lock. */
109 unsigned tx_zcopy_err;
Michael S. Tsirkin1280c272012-12-04 00:17:14 +0200110 /* Flush in progress. Protected by tx vq lock. */
111 bool tx_flush;
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +0000112};
113
Asias Hefe729a52013-05-06 16:38:24 +0800114static unsigned vhost_net_zcopy_mask __read_mostly;
Asias He28394002013-04-27 15:07:46 +0800115
Asias Hefe729a52013-05-06 16:38:24 +0800116static void vhost_net_enable_zcopy(int vq)
Asias He28394002013-04-27 15:07:46 +0800117{
Asias Hefe729a52013-05-06 16:38:24 +0800118 vhost_net_zcopy_mask |= 0x1 << vq;
Asias He28394002013-04-27 15:07:46 +0800119}
120
Asias Hefe729a52013-05-06 16:38:24 +0800121static struct vhost_net_ubuf_ref *
122vhost_net_ubuf_alloc(struct vhost_virtqueue *vq, bool zcopy)
Asias He28394002013-04-27 15:07:46 +0800123{
Asias Hefe729a52013-05-06 16:38:24 +0800124 struct vhost_net_ubuf_ref *ubufs;
Asias He28394002013-04-27 15:07:46 +0800125 /* No zero copy backend? Nothing to count. */
126 if (!zcopy)
127 return NULL;
128 ubufs = kmalloc(sizeof(*ubufs), GFP_KERNEL);
129 if (!ubufs)
130 return ERR_PTR(-ENOMEM);
Michael S. Tsirkin0ad8b482014-02-13 11:42:05 +0200131 atomic_set(&ubufs->refcount, 1);
Asias He28394002013-04-27 15:07:46 +0800132 init_waitqueue_head(&ubufs->wait);
133 ubufs->vq = vq;
134 return ubufs;
135}
136
Michael S. Tsirkin0ad8b482014-02-13 11:42:05 +0200137static int vhost_net_ubuf_put(struct vhost_net_ubuf_ref *ubufs)
Asias He28394002013-04-27 15:07:46 +0800138{
Michael S. Tsirkin0ad8b482014-02-13 11:42:05 +0200139 int r = atomic_sub_return(1, &ubufs->refcount);
140 if (unlikely(!r))
141 wake_up(&ubufs->wait);
142 return r;
Asias He28394002013-04-27 15:07:46 +0800143}
144
Asias Hefe729a52013-05-06 16:38:24 +0800145static void vhost_net_ubuf_put_and_wait(struct vhost_net_ubuf_ref *ubufs)
Asias He28394002013-04-27 15:07:46 +0800146{
Michael S. Tsirkin0ad8b482014-02-13 11:42:05 +0200147 vhost_net_ubuf_put(ubufs);
148 wait_event(ubufs->wait, !atomic_read(&ubufs->refcount));
Michael S. Tsirkinc38e39c2013-06-25 17:29:46 +0300149}
150
151static void vhost_net_ubuf_put_wait_and_free(struct vhost_net_ubuf_ref *ubufs)
152{
153 vhost_net_ubuf_put_and_wait(ubufs);
Asias He28394002013-04-27 15:07:46 +0800154 kfree(ubufs);
155}
156
Asias Heb1ad8492013-05-06 11:16:00 +0800157static void vhost_net_clear_ubuf_info(struct vhost_net *n)
158{
Asias Heb1ad8492013-05-06 11:16:00 +0800159 int i;
160
Michael S. Tsirkin288cfe72013-06-06 15:20:46 +0300161 for (i = 0; i < VHOST_NET_VQ_MAX; ++i) {
162 kfree(n->vqs[i].ubuf_info);
163 n->vqs[i].ubuf_info = NULL;
Asias Heb1ad8492013-05-06 11:16:00 +0800164 }
165}
166
Asias He0a1febf2013-06-05 21:17:38 +0800167static int vhost_net_set_ubuf_info(struct vhost_net *n)
Asias He28394002013-04-27 15:07:46 +0800168{
169 bool zcopy;
170 int i;
171
Michael S. Tsirkin288cfe72013-06-06 15:20:46 +0300172 for (i = 0; i < VHOST_NET_VQ_MAX; ++i) {
Asias Hefe729a52013-05-06 16:38:24 +0800173 zcopy = vhost_net_zcopy_mask & (0x1 << i);
Asias He28394002013-04-27 15:07:46 +0800174 if (!zcopy)
175 continue;
176 n->vqs[i].ubuf_info = kmalloc(sizeof(*n->vqs[i].ubuf_info) *
177 UIO_MAXIOV, GFP_KERNEL);
178 if (!n->vqs[i].ubuf_info)
179 goto err;
180 }
181 return 0;
182
183err:
Michael S. Tsirkin288cfe72013-06-06 15:20:46 +0300184 vhost_net_clear_ubuf_info(n);
Asias He28394002013-04-27 15:07:46 +0800185 return -ENOMEM;
186}
187
Asias He0a1febf2013-06-05 21:17:38 +0800188static void vhost_net_vq_reset(struct vhost_net *n)
Asias He28394002013-04-27 15:07:46 +0800189{
190 int i;
191
Michael S. Tsirkin288cfe72013-06-06 15:20:46 +0300192 vhost_net_clear_ubuf_info(n);
193
Asias He28394002013-04-27 15:07:46 +0800194 for (i = 0; i < VHOST_NET_VQ_MAX; i++) {
195 n->vqs[i].done_idx = 0;
196 n->vqs[i].upend_idx = 0;
197 n->vqs[i].ubufs = NULL;
Michael S. Tsirkin81f95a52013-04-28 15:51:40 +0300198 n->vqs[i].vhost_hlen = 0;
199 n->vqs[i].sock_hlen = 0;
Asias He28394002013-04-27 15:07:46 +0800200 }
201
202}
203
Michael S. Tsirkineaae8132012-11-01 09:16:51 +0000204static void vhost_net_tx_packet(struct vhost_net *net)
205{
206 ++net->tx_packets;
207 if (net->tx_packets < 1024)
208 return;
209 net->tx_packets = 0;
210 net->tx_zcopy_err = 0;
211}
212
213static void vhost_net_tx_err(struct vhost_net *net)
214{
215 ++net->tx_zcopy_err;
216}
217
218static bool vhost_net_tx_select_zcopy(struct vhost_net *net)
219{
Michael S. Tsirkin1280c272012-12-04 00:17:14 +0200220 /* TX flush waits for outstanding DMAs to be done.
221 * Don't start new DMAs.
222 */
223 return !net->tx_flush &&
224 net->tx_packets / 64 >= net->tx_zcopy_err;
Michael S. Tsirkineaae8132012-11-01 09:16:51 +0000225}
226
Michael S. Tsirkinbab632d2011-07-18 03:48:46 +0000227static bool vhost_sock_zcopy(struct socket *sock)
228{
229 return unlikely(experimental_zcopytx) &&
230 sock_flag(sock->sk, SOCK_ZEROCOPY);
231}
232
Michael S. Tsirkinb2116162012-11-01 09:16:46 +0000233/* In case of DMA done not in order in lower device driver for some reason.
234 * upend_idx is used to track end of used idx, done_idx is used to track head
235 * of used idx. Once lower device DMA done contiguously, we will signal KVM
236 * guest used idx.
237 */
Jason Wang094afe72013-09-02 16:40:56 +0800238static void vhost_zerocopy_signal_used(struct vhost_net *net,
239 struct vhost_virtqueue *vq)
Michael S. Tsirkinb2116162012-11-01 09:16:46 +0000240{
Asias He28394002013-04-27 15:07:46 +0800241 struct vhost_net_virtqueue *nvq =
242 container_of(vq, struct vhost_net_virtqueue, vq);
Jason Wangc92112a2013-09-02 16:40:57 +0800243 int i, add;
Michael S. Tsirkinb2116162012-11-01 09:16:46 +0000244 int j = 0;
245
Asias He28394002013-04-27 15:07:46 +0800246 for (i = nvq->done_idx; i != nvq->upend_idx; i = (i + 1) % UIO_MAXIOV) {
Michael S. Tsirkineaae8132012-11-01 09:16:51 +0000247 if (vq->heads[i].len == VHOST_DMA_FAILED_LEN)
248 vhost_net_tx_err(net);
Michael S. Tsirkinb2116162012-11-01 09:16:46 +0000249 if (VHOST_DMA_IS_DONE(vq->heads[i].len)) {
250 vq->heads[i].len = VHOST_DMA_CLEAR_LEN;
Michael S. Tsirkinb2116162012-11-01 09:16:46 +0000251 ++j;
252 } else
253 break;
254 }
Jason Wangc92112a2013-09-02 16:40:57 +0800255 while (j) {
256 add = min(UIO_MAXIOV - nvq->done_idx, j);
257 vhost_add_used_and_signal_n(vq->dev, vq,
258 &vq->heads[nvq->done_idx], add);
259 nvq->done_idx = (nvq->done_idx + add) % UIO_MAXIOV;
260 j -= add;
261 }
Michael S. Tsirkinb2116162012-11-01 09:16:46 +0000262}
263
Michael S. Tsirkineaae8132012-11-01 09:16:51 +0000264static void vhost_zerocopy_callback(struct ubuf_info *ubuf, bool success)
Michael S. Tsirkinb2116162012-11-01 09:16:46 +0000265{
Asias Hefe729a52013-05-06 16:38:24 +0800266 struct vhost_net_ubuf_ref *ubufs = ubuf->ctx;
Michael S. Tsirkinb2116162012-11-01 09:16:46 +0000267 struct vhost_virtqueue *vq = ubufs->vq;
Michael S. Tsirkin0ad8b482014-02-13 11:42:05 +0200268 int cnt;
Michael S. Tsirkinb2116162012-11-01 09:16:46 +0000269
Michael S. Tsirkinb0c057c2014-02-13 11:45:11 +0200270 rcu_read_lock_bh();
271
Jason Wang19c73b32013-09-02 16:41:00 +0800272 /* set len to mark this desc buffers done DMA */
273 vq->heads[ubuf->desc].len = success ?
274 VHOST_DMA_DONE_LEN : VHOST_DMA_FAILED_LEN;
Michael S. Tsirkin0ad8b482014-02-13 11:42:05 +0200275 cnt = vhost_net_ubuf_put(ubufs);
Jason Wang19c73b32013-09-02 16:41:00 +0800276
Michael S. Tsirkin24eb21a2012-11-01 09:16:55 +0000277 /*
278 * Trigger polling thread if guest stopped submitting new buffers:
Michael S. Tsirkin0ad8b482014-02-13 11:42:05 +0200279 * in this case, the refcount after decrement will eventually reach 1.
Michael S. Tsirkin24eb21a2012-11-01 09:16:55 +0000280 * We also trigger polling periodically after each 16 packets
281 * (the value 16 here is more or less arbitrary, it's tuned to trigger
282 * less than 10% of times).
283 */
Michael S. Tsirkin0ad8b482014-02-13 11:42:05 +0200284 if (cnt <= 1 || !(cnt % 16))
Michael S. Tsirkin24eb21a2012-11-01 09:16:55 +0000285 vhost_poll_queue(&vq->poll);
Michael S. Tsirkinb0c057c2014-02-13 11:45:11 +0200286
287 rcu_read_unlock_bh();
Michael S. Tsirkinb2116162012-11-01 09:16:46 +0000288}
289
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +0000290/* Expects to be always run from workqueue - which acts as
291 * read-size critical section for our kind of RCU. */
292static void handle_tx(struct vhost_net *net)
293{
Asias He28394002013-04-27 15:07:46 +0800294 struct vhost_net_virtqueue *nvq = &net->vqs[VHOST_NET_VQ_TX];
Michael S. Tsirkin81f95a52013-04-28 15:51:40 +0300295 struct vhost_virtqueue *vq = &nvq->vq;
Al Viro98a527a2014-12-10 15:00:58 -0500296 unsigned out, in;
Michael S. Tsirkind5675bd2010-06-24 16:59:59 +0300297 int head;
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +0000298 struct msghdr msg = {
299 .msg_name = NULL,
300 .msg_namelen = 0,
301 .msg_control = NULL,
302 .msg_controllen = 0,
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +0000303 .msg_flags = MSG_DONTWAIT,
304 };
305 size_t len, total_len = 0;
Jason Wang70181d512013-04-10 20:50:48 +0000306 int err;
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +0000307 size_t hdr_size;
Arnd Bergmann28457ee2010-03-09 19:24:45 +0100308 struct socket *sock;
Asias Hefe729a52013-05-06 16:38:24 +0800309 struct vhost_net_ubuf_ref *uninitialized_var(ubufs);
Michael S. Tsirkincedb9bd2012-12-06 17:00:18 +0200310 bool zcopy, zcopy_used;
Arnd Bergmann28457ee2010-03-09 19:24:45 +0100311
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +0000312 mutex_lock(&vq->mutex);
Asias He2e26af72013-05-07 14:54:33 +0800313 sock = vq->private_data;
314 if (!sock)
315 goto out;
316
Michael S. Tsirkin8ea8cf82011-05-20 02:10:54 +0300317 vhost_disable_notify(&net->dev, vq);
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +0000318
Michael S. Tsirkin81f95a52013-04-28 15:51:40 +0300319 hdr_size = nvq->vhost_hlen;
Asias He28394002013-04-27 15:07:46 +0800320 zcopy = nvq->ubufs;
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +0000321
322 for (;;) {
Michael S. Tsirkinbab632d2011-07-18 03:48:46 +0000323 /* Release DMAs done buffers first */
324 if (zcopy)
Michael S. Tsirkineaae8132012-11-01 09:16:51 +0000325 vhost_zerocopy_signal_used(net, vq);
Michael S. Tsirkinbab632d2011-07-18 03:48:46 +0000326
Jason Wangf7c6be42013-09-02 16:41:01 +0800327 /* If more outstanding DMAs, queue the work.
328 * Handle upend_idx wrap around
329 */
330 if (unlikely((nvq->upend_idx + vq->num - VHOST_MAX_PEND)
331 % UIO_MAXIOV == nvq->done_idx))
332 break;
333
Michael S. Tsirkin47283be2014-06-05 15:20:27 +0300334 head = vhost_get_vq_desc(vq, vq->iov,
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +0000335 ARRAY_SIZE(vq->iov),
336 &out, &in,
337 NULL, NULL);
Michael S. Tsirkind5675bd2010-06-24 16:59:59 +0300338 /* On error, stop handling until the next kick. */
Michael S. Tsirkin7b3384f2010-07-01 18:40:12 +0300339 if (unlikely(head < 0))
Michael S. Tsirkind5675bd2010-06-24 16:59:59 +0300340 break;
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +0000341 /* Nothing new? Wait for eventfd to tell us they refilled. */
342 if (head == vq->num) {
Michael S. Tsirkin8ea8cf82011-05-20 02:10:54 +0300343 if (unlikely(vhost_enable_notify(&net->dev, vq))) {
344 vhost_disable_notify(&net->dev, vq);
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +0000345 continue;
346 }
347 break;
348 }
349 if (in) {
350 vq_err(vq, "Unexpected descriptor format for TX: "
351 "out %d, int %d\n", out, in);
352 break;
353 }
354 /* Skip header. TODO: support TSO. */
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +0000355 len = iov_length(vq->iov, out);
Al Viroc0371da2014-11-24 10:42:55 -0500356 iov_iter_init(&msg.msg_iter, WRITE, vq->iov, out, len);
Al Viro98a527a2014-12-10 15:00:58 -0500357 iov_iter_advance(&msg.msg_iter, hdr_size);
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +0000358 /* Sanity check */
Al Viro01e97e62014-12-15 21:39:31 -0500359 if (!msg_data_left(&msg)) {
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +0000360 vq_err(vq, "Unexpected header len for TX: "
361 "%zd expected %zd\n",
Al Viro98a527a2014-12-10 15:00:58 -0500362 len, hdr_size);
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +0000363 break;
364 }
Al Viro01e97e62014-12-15 21:39:31 -0500365 len = msg_data_left(&msg);
Jason Wangce21a022013-09-02 16:40:59 +0800366
367 zcopy_used = zcopy && len >= VHOST_GOODCOPY_LEN
368 && (nvq->upend_idx + 1) % UIO_MAXIOV !=
369 nvq->done_idx
370 && vhost_net_tx_select_zcopy(net);
Michael S. Tsirkincedb9bd2012-12-06 17:00:18 +0200371
Michael S. Tsirkinbab632d2011-07-18 03:48:46 +0000372 /* use msg_control to pass vhost zerocopy ubuf info to skb */
Michael S. Tsirkincedb9bd2012-12-06 17:00:18 +0200373 if (zcopy_used) {
Jason Wangce21a022013-09-02 16:40:59 +0800374 struct ubuf_info *ubuf;
375 ubuf = nvq->ubuf_info + nvq->upend_idx;
Michael S. Tsirkinbab632d2011-07-18 03:48:46 +0000376
Michael S. Tsirkin8b386942014-10-24 14:19:48 +0300377 vq->heads[nvq->upend_idx].id = cpu_to_vhost32(vq, head);
Jason Wangce21a022013-09-02 16:40:59 +0800378 vq->heads[nvq->upend_idx].len = VHOST_DMA_IN_PROGRESS;
379 ubuf->callback = vhost_zerocopy_callback;
380 ubuf->ctx = nvq->ubufs;
381 ubuf->desc = nvq->upend_idx;
382 msg.msg_control = ubuf;
383 msg.msg_controllen = sizeof(ubuf);
384 ubufs = nvq->ubufs;
Michael S. Tsirkin0ad8b482014-02-13 11:42:05 +0200385 atomic_inc(&ubufs->refcount);
Asias He28394002013-04-27 15:07:46 +0800386 nvq->upend_idx = (nvq->upend_idx + 1) % UIO_MAXIOV;
Jason Wangce21a022013-09-02 16:40:59 +0800387 } else {
Jason Wang4364d5f2013-06-05 15:40:46 +0800388 msg.msg_control = NULL;
Jason Wangce21a022013-09-02 16:40:59 +0800389 ubufs = NULL;
390 }
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +0000391 /* TODO: Check specific error and bomb out unless ENOBUFS? */
Ying Xue1b784142015-03-02 15:37:48 +0800392 err = sock->ops->sendmsg(sock, &msg, len);
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +0000393 if (unlikely(err < 0)) {
Michael S. Tsirkincedb9bd2012-12-06 17:00:18 +0200394 if (zcopy_used) {
Jason Wangce21a022013-09-02 16:40:59 +0800395 vhost_net_ubuf_put(ubufs);
Asias He28394002013-04-27 15:07:46 +0800396 nvq->upend_idx = ((unsigned)nvq->upend_idx - 1)
397 % UIO_MAXIOV;
Michael S. Tsirkinbab632d2011-07-18 03:48:46 +0000398 }
David Stevens8dd014a2010-07-27 18:52:21 +0300399 vhost_discard_vq_desc(vq, 1);
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +0000400 break;
401 }
402 if (err != len)
Michael S. Tsirkin95c0ec62010-06-24 17:10:25 +0300403 pr_debug("Truncated TX packet: "
404 " len %d != %zd\n", err, len);
Michael S. Tsirkincedb9bd2012-12-06 17:00:18 +0200405 if (!zcopy_used)
Michael S. Tsirkinbab632d2011-07-18 03:48:46 +0000406 vhost_add_used_and_signal(&net->dev, vq, head, 0);
Jason Wangc8fb2172012-05-02 11:42:41 +0800407 else
Michael S. Tsirkineaae8132012-11-01 09:16:51 +0000408 vhost_zerocopy_signal_used(net, vq);
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +0000409 total_len += len;
Michael S. Tsirkineaae8132012-11-01 09:16:51 +0000410 vhost_net_tx_packet(net);
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +0000411 if (unlikely(total_len >= VHOST_NET_WEIGHT)) {
412 vhost_poll_queue(&vq->poll);
413 break;
414 }
415 }
Asias He2e26af72013-05-07 14:54:33 +0800416out:
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +0000417 mutex_unlock(&vq->mutex);
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +0000418}
419
David Stevens8dd014a2010-07-27 18:52:21 +0300420static int peek_head_len(struct sock *sk)
421{
422 struct sk_buff *head;
423 int len = 0;
Jason Wang783e3982011-01-17 16:11:17 +0800424 unsigned long flags;
David Stevens8dd014a2010-07-27 18:52:21 +0300425
Jason Wang783e3982011-01-17 16:11:17 +0800426 spin_lock_irqsave(&sk->sk_receive_queue.lock, flags);
David Stevens8dd014a2010-07-27 18:52:21 +0300427 head = skb_peek(&sk->sk_receive_queue);
Basil Gorc53cff5e2012-05-03 22:55:23 +0000428 if (likely(head)) {
David Stevens8dd014a2010-07-27 18:52:21 +0300429 len = head->len;
Jiri Pirkodf8a39d2015-01-13 17:13:44 +0100430 if (skb_vlan_tag_present(head))
Basil Gorc53cff5e2012-05-03 22:55:23 +0000431 len += VLAN_HLEN;
432 }
433
Jason Wang783e3982011-01-17 16:11:17 +0800434 spin_unlock_irqrestore(&sk->sk_receive_queue.lock, flags);
David Stevens8dd014a2010-07-27 18:52:21 +0300435 return len;
436}
437
438/* This is a multi-buffer version of vhost_get_desc, that works if
439 * vq has read descriptors only.
440 * @vq - the relevant virtqueue
441 * @datalen - data length we'll be reading
442 * @iovcount - returned count of io vectors we fill
443 * @log - vhost log
444 * @log_num - log offset
Jason Wang94249362011-01-17 16:11:08 +0800445 * @quota - headcount quota, 1 for big buffer
David Stevens8dd014a2010-07-27 18:52:21 +0300446 * returns number of buffer heads allocated, negative on error
447 */
448static int get_rx_bufs(struct vhost_virtqueue *vq,
449 struct vring_used_elem *heads,
450 int datalen,
451 unsigned *iovcount,
452 struct vhost_log *log,
Jason Wang94249362011-01-17 16:11:08 +0800453 unsigned *log_num,
454 unsigned int quota)
David Stevens8dd014a2010-07-27 18:52:21 +0300455{
456 unsigned int out, in;
457 int seg = 0;
458 int headcount = 0;
459 unsigned d;
460 int r, nlogs = 0;
Michael S. Tsirkin8b386942014-10-24 14:19:48 +0300461 /* len is always initialized before use since we are always called with
462 * datalen > 0.
463 */
464 u32 uninitialized_var(len);
David Stevens8dd014a2010-07-27 18:52:21 +0300465
Jason Wang94249362011-01-17 16:11:08 +0800466 while (datalen > 0 && headcount < quota) {
Jason Wange0e9b402010-09-14 23:53:05 +0800467 if (unlikely(seg >= UIO_MAXIOV)) {
David Stevens8dd014a2010-07-27 18:52:21 +0300468 r = -ENOBUFS;
469 goto err;
470 }
Michael S. Tsirkin47283be2014-06-05 15:20:27 +0300471 r = vhost_get_vq_desc(vq, vq->iov + seg,
David Stevens8dd014a2010-07-27 18:52:21 +0300472 ARRAY_SIZE(vq->iov) - seg, &out,
473 &in, log, log_num);
Michael S. Tsirkina39ee442014-03-27 12:53:37 +0200474 if (unlikely(r < 0))
475 goto err;
476
477 d = r;
David Stevens8dd014a2010-07-27 18:52:21 +0300478 if (d == vq->num) {
479 r = 0;
480 goto err;
481 }
482 if (unlikely(out || in <= 0)) {
483 vq_err(vq, "unexpected descriptor format for RX: "
484 "out %d, in %d\n", out, in);
485 r = -EINVAL;
486 goto err;
487 }
488 if (unlikely(log)) {
489 nlogs += *log_num;
490 log += *log_num;
491 }
Michael S. Tsirkin8b386942014-10-24 14:19:48 +0300492 heads[headcount].id = cpu_to_vhost32(vq, d);
493 len = iov_length(vq->iov + seg, in);
494 heads[headcount].len = cpu_to_vhost32(vq, len);
495 datalen -= len;
David Stevens8dd014a2010-07-27 18:52:21 +0300496 ++headcount;
497 seg += in;
498 }
Michael S. Tsirkin99975cc2015-01-07 10:51:00 +0200499 heads[headcount - 1].len = cpu_to_vhost32(vq, len + datalen);
David Stevens8dd014a2010-07-27 18:52:21 +0300500 *iovcount = seg;
501 if (unlikely(log))
502 *log_num = nlogs;
Michael S. Tsirkind8316f32014-03-27 12:00:26 +0200503
504 /* Detect overrun */
505 if (unlikely(datalen > 0)) {
506 r = UIO_MAXIOV + 1;
507 goto err;
508 }
David Stevens8dd014a2010-07-27 18:52:21 +0300509 return headcount;
510err:
511 vhost_discard_vq_desc(vq, headcount);
512 return r;
513}
514
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +0000515/* Expects to be always run from workqueue - which acts as
516 * read-size critical section for our kind of RCU. */
Jason Wang94249362011-01-17 16:11:08 +0800517static void handle_rx(struct vhost_net *net)
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +0000518{
Michael S. Tsirkin81f95a52013-04-28 15:51:40 +0300519 struct vhost_net_virtqueue *nvq = &net->vqs[VHOST_NET_VQ_RX];
520 struct vhost_virtqueue *vq = &nvq->vq;
David Stevens8dd014a2010-07-27 18:52:21 +0300521 unsigned uninitialized_var(in), log;
522 struct vhost_log *vq_log;
523 struct msghdr msg = {
524 .msg_name = NULL,
525 .msg_namelen = 0,
526 .msg_control = NULL, /* FIXME: get and handle RX aux data. */
527 .msg_controllen = 0,
David Stevens8dd014a2010-07-27 18:52:21 +0300528 .msg_flags = MSG_DONTWAIT,
529 };
Jason Wang0960b642015-02-15 16:35:17 +0800530 struct virtio_net_hdr hdr = {
531 .flags = 0,
532 .gso_type = VIRTIO_NET_HDR_GSO_NONE
David Stevens8dd014a2010-07-27 18:52:21 +0300533 };
David Stevens8dd014a2010-07-27 18:52:21 +0300534 size_t total_len = 0;
Michael S. Tsirkin910a5782012-10-24 20:37:51 +0200535 int err, mergeable;
536 s16 headcount;
David Stevens8dd014a2010-07-27 18:52:21 +0300537 size_t vhost_hlen, sock_hlen;
538 size_t vhost_len, sock_len;
Asias He2e26af72013-05-07 14:54:33 +0800539 struct socket *sock;
Al Viroba7438a2014-12-10 15:51:28 -0500540 struct iov_iter fixup;
Jason Wang0960b642015-02-15 16:35:17 +0800541 __virtio16 num_buffers;
David Stevens8dd014a2010-07-27 18:52:21 +0300542
David Stevens8dd014a2010-07-27 18:52:21 +0300543 mutex_lock(&vq->mutex);
Asias He2e26af72013-05-07 14:54:33 +0800544 sock = vq->private_data;
545 if (!sock)
546 goto out;
Michael S. Tsirkin8ea8cf82011-05-20 02:10:54 +0300547 vhost_disable_notify(&net->dev, vq);
Asias He2e26af72013-05-07 14:54:33 +0800548
Michael S. Tsirkin81f95a52013-04-28 15:51:40 +0300549 vhost_hlen = nvq->vhost_hlen;
550 sock_hlen = nvq->sock_hlen;
David Stevens8dd014a2010-07-27 18:52:21 +0300551
Michael S. Tsirkinea16c512014-06-05 15:20:23 +0300552 vq_log = unlikely(vhost_has_feature(vq, VHOST_F_LOG_ALL)) ?
David Stevens8dd014a2010-07-27 18:52:21 +0300553 vq->log : NULL;
Michael S. Tsirkinea16c512014-06-05 15:20:23 +0300554 mergeable = vhost_has_feature(vq, VIRTIO_NET_F_MRG_RXBUF);
David Stevens8dd014a2010-07-27 18:52:21 +0300555
556 while ((sock_len = peek_head_len(sock->sk))) {
557 sock_len += sock_hlen;
558 vhost_len = sock_len + vhost_hlen;
559 headcount = get_rx_bufs(vq, vq->heads, vhost_len,
Jason Wang94249362011-01-17 16:11:08 +0800560 &in, vq_log, &log,
561 likely(mergeable) ? UIO_MAXIOV : 1);
David Stevens8dd014a2010-07-27 18:52:21 +0300562 /* On error, stop handling until the next kick. */
563 if (unlikely(headcount < 0))
564 break;
Michael S. Tsirkind8316f32014-03-27 12:00:26 +0200565 /* On overrun, truncate and discard */
566 if (unlikely(headcount > UIO_MAXIOV)) {
Al Viroc0371da2014-11-24 10:42:55 -0500567 iov_iter_init(&msg.msg_iter, READ, vq->iov, 1, 1);
Ying Xue1b784142015-03-02 15:37:48 +0800568 err = sock->ops->recvmsg(sock, &msg,
Michael S. Tsirkind8316f32014-03-27 12:00:26 +0200569 1, MSG_DONTWAIT | MSG_TRUNC);
570 pr_debug("Discarded rx packet: len %zd\n", sock_len);
571 continue;
572 }
David Stevens8dd014a2010-07-27 18:52:21 +0300573 /* OK, now we need to know about added descriptors. */
574 if (!headcount) {
Michael S. Tsirkin8ea8cf82011-05-20 02:10:54 +0300575 if (unlikely(vhost_enable_notify(&net->dev, vq))) {
David Stevens8dd014a2010-07-27 18:52:21 +0300576 /* They have slipped one in as we were
577 * doing that: check again. */
Michael S. Tsirkin8ea8cf82011-05-20 02:10:54 +0300578 vhost_disable_notify(&net->dev, vq);
David Stevens8dd014a2010-07-27 18:52:21 +0300579 continue;
580 }
581 /* Nothing new? Wait for eventfd to tell us
582 * they refilled. */
583 break;
584 }
585 /* We don't need to be notified again. */
Al Viroba7438a2014-12-10 15:51:28 -0500586 iov_iter_init(&msg.msg_iter, READ, vq->iov, in, vhost_len);
587 fixup = msg.msg_iter;
588 if (unlikely((vhost_hlen))) {
589 /* We will supply the header ourselves
590 * TODO: support TSO.
591 */
592 iov_iter_advance(&msg.msg_iter, vhost_hlen);
Al Viroba7438a2014-12-10 15:51:28 -0500593 }
Ying Xue1b784142015-03-02 15:37:48 +0800594 err = sock->ops->recvmsg(sock, &msg,
David Stevens8dd014a2010-07-27 18:52:21 +0300595 sock_len, MSG_DONTWAIT | MSG_TRUNC);
596 /* Userspace might have consumed the packet meanwhile:
597 * it's not supposed to do this usually, but might be hard
598 * to prevent. Discard data we got (if any) and keep going. */
599 if (unlikely(err != sock_len)) {
600 pr_debug("Discarded rx packet: "
601 " len %d, expected %zd\n", err, sock_len);
602 vhost_discard_vq_desc(vq, headcount);
603 continue;
604 }
Al Viroba7438a2014-12-10 15:51:28 -0500605 /* Supply virtio_net_hdr if VHOST_NET_F_VIRTIO_NET_HDR */
Michael S. Tsirkin4c5a8442015-02-25 15:19:28 +0100606 if (unlikely(vhost_hlen)) {
607 if (copy_to_iter(&hdr, sizeof(hdr),
608 &fixup) != sizeof(hdr)) {
609 vq_err(vq, "Unable to write vnet_hdr "
610 "at addr %p\n", vq->iov->iov_base);
611 break;
612 }
613 } else {
614 /* Header came from socket; we'll need to patch
615 * ->num_buffers over if VIRTIO_NET_F_MRG_RXBUF
616 */
617 iov_iter_advance(&fixup, sizeof(hdr));
David Stevens8dd014a2010-07-27 18:52:21 +0300618 }
619 /* TODO: Should check and handle checksum. */
Michael S. Tsirkin5201aa42015-02-03 11:07:06 +0200620
Jason Wang0960b642015-02-15 16:35:17 +0800621 num_buffers = cpu_to_vhost16(vq, headcount);
Jason Wangcfbdab92011-01-17 16:10:59 +0800622 if (likely(mergeable) &&
Michael S. Tsirkin0d79a492015-02-25 15:20:01 +0100623 copy_to_iter(&num_buffers, sizeof num_buffers,
624 &fixup) != sizeof num_buffers) {
David Stevens8dd014a2010-07-27 18:52:21 +0300625 vq_err(vq, "Failed num_buffers write");
626 vhost_discard_vq_desc(vq, headcount);
627 break;
628 }
629 vhost_add_used_and_signal_n(&net->dev, vq, vq->heads,
630 headcount);
631 if (unlikely(vq_log))
632 vhost_log_write(vq, vq_log, log, vhost_len);
633 total_len += vhost_len;
634 if (unlikely(total_len >= VHOST_NET_WEIGHT)) {
635 vhost_poll_queue(&vq->poll);
636 break;
637 }
638 }
Asias He2e26af72013-05-07 14:54:33 +0800639out:
David Stevens8dd014a2010-07-27 18:52:21 +0300640 mutex_unlock(&vq->mutex);
David Stevens8dd014a2010-07-27 18:52:21 +0300641}
642
Tejun Heoc23f34452010-06-02 20:40:00 +0200643static void handle_tx_kick(struct vhost_work *work)
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +0000644{
Tejun Heoc23f34452010-06-02 20:40:00 +0200645 struct vhost_virtqueue *vq = container_of(work, struct vhost_virtqueue,
646 poll.work);
647 struct vhost_net *net = container_of(vq->dev, struct vhost_net, dev);
648
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +0000649 handle_tx(net);
650}
651
Tejun Heoc23f34452010-06-02 20:40:00 +0200652static void handle_rx_kick(struct vhost_work *work)
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +0000653{
Tejun Heoc23f34452010-06-02 20:40:00 +0200654 struct vhost_virtqueue *vq = container_of(work, struct vhost_virtqueue,
655 poll.work);
656 struct vhost_net *net = container_of(vq->dev, struct vhost_net, dev);
657
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +0000658 handle_rx(net);
659}
660
Tejun Heoc23f34452010-06-02 20:40:00 +0200661static void handle_tx_net(struct vhost_work *work)
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +0000662{
Tejun Heoc23f34452010-06-02 20:40:00 +0200663 struct vhost_net *net = container_of(work, struct vhost_net,
664 poll[VHOST_NET_VQ_TX].work);
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +0000665 handle_tx(net);
666}
667
Tejun Heoc23f34452010-06-02 20:40:00 +0200668static void handle_rx_net(struct vhost_work *work)
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +0000669{
Tejun Heoc23f34452010-06-02 20:40:00 +0200670 struct vhost_net *net = container_of(work, struct vhost_net,
671 poll[VHOST_NET_VQ_RX].work);
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +0000672 handle_rx(net);
673}
674
675static int vhost_net_open(struct inode *inode, struct file *f)
676{
Michael S. Tsirkin23cc5a92013-01-23 21:46:47 +0100677 struct vhost_net *n;
Tejun Heoc23f34452010-06-02 20:40:00 +0200678 struct vhost_dev *dev;
Asias He3ab2e422013-04-27 11:16:48 +0800679 struct vhost_virtqueue **vqs;
Zhi Yong Wu59566b6e2013-12-07 04:13:03 +0800680 int i;
Tejun Heoc23f34452010-06-02 20:40:00 +0200681
Michael S. Tsirkin23cc5a92013-01-23 21:46:47 +0100682 n = kmalloc(sizeof *n, GFP_KERNEL | __GFP_NOWARN | __GFP_REPEAT);
683 if (!n) {
684 n = vmalloc(sizeof *n);
685 if (!n)
686 return -ENOMEM;
687 }
Asias He3ab2e422013-04-27 11:16:48 +0800688 vqs = kmalloc(VHOST_NET_VQ_MAX * sizeof(*vqs), GFP_KERNEL);
689 if (!vqs) {
Romain Francoised04257b2014-06-12 10:42:34 +0200690 kvfree(n);
Asias He3ab2e422013-04-27 11:16:48 +0800691 return -ENOMEM;
692 }
Tejun Heoc23f34452010-06-02 20:40:00 +0200693
694 dev = &n->dev;
Asias He3ab2e422013-04-27 11:16:48 +0800695 vqs[VHOST_NET_VQ_TX] = &n->vqs[VHOST_NET_VQ_TX].vq;
696 vqs[VHOST_NET_VQ_RX] = &n->vqs[VHOST_NET_VQ_RX].vq;
697 n->vqs[VHOST_NET_VQ_TX].vq.handle_kick = handle_tx_kick;
698 n->vqs[VHOST_NET_VQ_RX].vq.handle_kick = handle_rx_kick;
Asias He28394002013-04-27 15:07:46 +0800699 for (i = 0; i < VHOST_NET_VQ_MAX; i++) {
700 n->vqs[i].ubufs = NULL;
701 n->vqs[i].ubuf_info = NULL;
702 n->vqs[i].upend_idx = 0;
703 n->vqs[i].done_idx = 0;
Michael S. Tsirkin81f95a52013-04-28 15:51:40 +0300704 n->vqs[i].vhost_hlen = 0;
705 n->vqs[i].sock_hlen = 0;
Asias He28394002013-04-27 15:07:46 +0800706 }
Zhi Yong Wu59566b6e2013-12-07 04:13:03 +0800707 vhost_dev_init(dev, vqs, VHOST_NET_VQ_MAX);
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +0000708
Tejun Heoc23f34452010-06-02 20:40:00 +0200709 vhost_poll_init(n->poll + VHOST_NET_VQ_TX, handle_tx_net, POLLOUT, dev);
710 vhost_poll_init(n->poll + VHOST_NET_VQ_RX, handle_rx_net, POLLIN, dev);
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +0000711
712 f->private_data = n;
713
714 return 0;
715}
716
717static void vhost_net_disable_vq(struct vhost_net *n,
718 struct vhost_virtqueue *vq)
719{
Asias He3ab2e422013-04-27 11:16:48 +0800720 struct vhost_net_virtqueue *nvq =
721 container_of(vq, struct vhost_net_virtqueue, vq);
722 struct vhost_poll *poll = n->poll + (nvq - n->vqs);
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +0000723 if (!vq->private_data)
724 return;
Jason Wang70181d512013-04-10 20:50:48 +0000725 vhost_poll_stop(poll);
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +0000726}
727
Jason Wang2b8b3282013-01-28 01:05:18 +0000728static int vhost_net_enable_vq(struct vhost_net *n,
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +0000729 struct vhost_virtqueue *vq)
730{
Asias He3ab2e422013-04-27 11:16:48 +0800731 struct vhost_net_virtqueue *nvq =
732 container_of(vq, struct vhost_net_virtqueue, vq);
733 struct vhost_poll *poll = n->poll + (nvq - n->vqs);
Arnd Bergmann28457ee2010-03-09 19:24:45 +0100734 struct socket *sock;
735
Asias He22fa90c2013-05-07 14:54:36 +0800736 sock = vq->private_data;
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +0000737 if (!sock)
Jason Wang2b8b3282013-01-28 01:05:18 +0000738 return 0;
Jason Wang2b8b3282013-01-28 01:05:18 +0000739
Jason Wang70181d512013-04-10 20:50:48 +0000740 return vhost_poll_start(poll, sock->file);
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +0000741}
742
743static struct socket *vhost_net_stop_vq(struct vhost_net *n,
744 struct vhost_virtqueue *vq)
745{
746 struct socket *sock;
747
748 mutex_lock(&vq->mutex);
Asias He22fa90c2013-05-07 14:54:36 +0800749 sock = vq->private_data;
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +0000750 vhost_net_disable_vq(n, vq);
Asias He22fa90c2013-05-07 14:54:36 +0800751 vq->private_data = NULL;
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +0000752 mutex_unlock(&vq->mutex);
753 return sock;
754}
755
756static void vhost_net_stop(struct vhost_net *n, struct socket **tx_sock,
757 struct socket **rx_sock)
758{
Asias He3ab2e422013-04-27 11:16:48 +0800759 *tx_sock = vhost_net_stop_vq(n, &n->vqs[VHOST_NET_VQ_TX].vq);
760 *rx_sock = vhost_net_stop_vq(n, &n->vqs[VHOST_NET_VQ_RX].vq);
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +0000761}
762
763static void vhost_net_flush_vq(struct vhost_net *n, int index)
764{
765 vhost_poll_flush(n->poll + index);
Asias He3ab2e422013-04-27 11:16:48 +0800766 vhost_poll_flush(&n->vqs[index].vq.poll);
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +0000767}
768
769static void vhost_net_flush(struct vhost_net *n)
770{
771 vhost_net_flush_vq(n, VHOST_NET_VQ_TX);
772 vhost_net_flush_vq(n, VHOST_NET_VQ_RX);
Asias He28394002013-04-27 15:07:46 +0800773 if (n->vqs[VHOST_NET_VQ_TX].ubufs) {
Asias He3ab2e422013-04-27 11:16:48 +0800774 mutex_lock(&n->vqs[VHOST_NET_VQ_TX].vq.mutex);
Michael S. Tsirkin1280c272012-12-04 00:17:14 +0200775 n->tx_flush = true;
Asias He3ab2e422013-04-27 11:16:48 +0800776 mutex_unlock(&n->vqs[VHOST_NET_VQ_TX].vq.mutex);
Michael S. Tsirkin1280c272012-12-04 00:17:14 +0200777 /* Wait for all lower device DMAs done. */
Asias Hefe729a52013-05-06 16:38:24 +0800778 vhost_net_ubuf_put_and_wait(n->vqs[VHOST_NET_VQ_TX].ubufs);
Asias He3ab2e422013-04-27 11:16:48 +0800779 mutex_lock(&n->vqs[VHOST_NET_VQ_TX].vq.mutex);
Michael S. Tsirkin1280c272012-12-04 00:17:14 +0200780 n->tx_flush = false;
Michael S. Tsirkin0ad8b482014-02-13 11:42:05 +0200781 atomic_set(&n->vqs[VHOST_NET_VQ_TX].ubufs->refcount, 1);
Asias He3ab2e422013-04-27 11:16:48 +0800782 mutex_unlock(&n->vqs[VHOST_NET_VQ_TX].vq.mutex);
Michael S. Tsirkin1280c272012-12-04 00:17:14 +0200783 }
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +0000784}
785
786static int vhost_net_release(struct inode *inode, struct file *f)
787{
788 struct vhost_net *n = f->private_data;
789 struct socket *tx_sock;
790 struct socket *rx_sock;
791
792 vhost_net_stop(n, &tx_sock, &rx_sock);
793 vhost_net_flush(n);
Michael S. Tsirkinb2116162012-11-01 09:16:46 +0000794 vhost_dev_stop(&n->dev);
Michael S. Tsirkinea5d4042011-11-27 19:05:58 +0200795 vhost_dev_cleanup(&n->dev, false);
Michael S. Tsirkin81f95a52013-04-28 15:51:40 +0300796 vhost_net_vq_reset(n);
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +0000797 if (tx_sock)
Al Viro09aaacf2014-03-05 20:39:00 -0500798 sockfd_put(tx_sock);
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +0000799 if (rx_sock)
Al Viro09aaacf2014-03-05 20:39:00 -0500800 sockfd_put(rx_sock);
Michael S. Tsirkinb0c057c2014-02-13 11:45:11 +0200801 /* Make sure no callbacks are outstanding */
802 synchronize_rcu_bh();
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +0000803 /* We do an extra flush before freeing memory,
804 * since jobs can re-queue themselves. */
805 vhost_net_flush(n);
Asias He3ab2e422013-04-27 11:16:48 +0800806 kfree(n->dev.vqs);
Romain Francoised04257b2014-06-12 10:42:34 +0200807 kvfree(n);
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +0000808 return 0;
809}
810
811static struct socket *get_raw_socket(int fd)
812{
813 struct {
814 struct sockaddr_ll sa;
815 char buf[MAX_ADDR_LEN];
816 } uaddr;
817 int uaddr_len = sizeof uaddr, r;
818 struct socket *sock = sockfd_lookup(fd, &r);
Krishna Kumard47effe2011-03-01 17:06:37 +0530819
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +0000820 if (!sock)
821 return ERR_PTR(-ENOTSOCK);
822
823 /* Parameter checking */
824 if (sock->sk->sk_type != SOCK_RAW) {
825 r = -ESOCKTNOSUPPORT;
826 goto err;
827 }
828
829 r = sock->ops->getname(sock, (struct sockaddr *)&uaddr.sa,
830 &uaddr_len, 0);
831 if (r)
832 goto err;
833
834 if (uaddr.sa.sll_family != AF_PACKET) {
835 r = -EPFNOSUPPORT;
836 goto err;
837 }
838 return sock;
839err:
Al Viro09aaacf2014-03-05 20:39:00 -0500840 sockfd_put(sock);
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +0000841 return ERR_PTR(r);
842}
843
Arnd Bergmann501c7742010-02-18 05:46:50 +0000844static struct socket *get_tap_socket(int fd)
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +0000845{
846 struct file *file = fget(fd);
847 struct socket *sock;
Krishna Kumard47effe2011-03-01 17:06:37 +0530848
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +0000849 if (!file)
850 return ERR_PTR(-EBADF);
851 sock = tun_get_socket(file);
Arnd Bergmann501c7742010-02-18 05:46:50 +0000852 if (!IS_ERR(sock))
853 return sock;
854 sock = macvtap_get_socket(file);
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +0000855 if (IS_ERR(sock))
856 fput(file);
857 return sock;
858}
859
860static struct socket *get_socket(int fd)
861{
862 struct socket *sock;
Krishna Kumard47effe2011-03-01 17:06:37 +0530863
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +0000864 /* special case to disable backend */
865 if (fd == -1)
866 return NULL;
867 sock = get_raw_socket(fd);
868 if (!IS_ERR(sock))
869 return sock;
Arnd Bergmann501c7742010-02-18 05:46:50 +0000870 sock = get_tap_socket(fd);
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +0000871 if (!IS_ERR(sock))
872 return sock;
873 return ERR_PTR(-ENOTSOCK);
874}
875
876static long vhost_net_set_backend(struct vhost_net *n, unsigned index, int fd)
877{
878 struct socket *sock, *oldsock;
879 struct vhost_virtqueue *vq;
Asias He28394002013-04-27 15:07:46 +0800880 struct vhost_net_virtqueue *nvq;
Asias Hefe729a52013-05-06 16:38:24 +0800881 struct vhost_net_ubuf_ref *ubufs, *oldubufs = NULL;
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +0000882 int r;
883
884 mutex_lock(&n->dev.mutex);
885 r = vhost_dev_check_owner(&n->dev);
886 if (r)
887 goto err;
888
889 if (index >= VHOST_NET_VQ_MAX) {
890 r = -ENOBUFS;
891 goto err;
892 }
Asias He3ab2e422013-04-27 11:16:48 +0800893 vq = &n->vqs[index].vq;
Asias He28394002013-04-27 15:07:46 +0800894 nvq = &n->vqs[index];
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +0000895 mutex_lock(&vq->mutex);
896
897 /* Verify that ring has been setup correctly. */
898 if (!vhost_vq_access_ok(vq)) {
899 r = -EFAULT;
Jeff Dike1dace8c2010-03-04 16:10:14 -0500900 goto err_vq;
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +0000901 }
902 sock = get_socket(fd);
903 if (IS_ERR(sock)) {
904 r = PTR_ERR(sock);
Jeff Dike1dace8c2010-03-04 16:10:14 -0500905 goto err_vq;
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +0000906 }
907
908 /* start polling new socket */
Asias He22fa90c2013-05-07 14:54:36 +0800909 oldsock = vq->private_data;
David S. Miller11fe8832010-07-20 18:25:24 -0700910 if (sock != oldsock) {
Asias Hefe729a52013-05-06 16:38:24 +0800911 ubufs = vhost_net_ubuf_alloc(vq,
912 sock && vhost_sock_zcopy(sock));
Michael S. Tsirkinbab632d2011-07-18 03:48:46 +0000913 if (IS_ERR(ubufs)) {
914 r = PTR_ERR(ubufs);
915 goto err_ubufs;
916 }
Jason Wang692a9982013-01-28 01:05:17 +0000917
Krishna Kumard47effe2011-03-01 17:06:37 +0530918 vhost_net_disable_vq(n, vq);
Asias He22fa90c2013-05-07 14:54:36 +0800919 vq->private_data = sock;
Jason Wangf59281d2011-06-21 18:04:27 +0800920 r = vhost_init_used(vq);
921 if (r)
Jason Wang692a9982013-01-28 01:05:17 +0000922 goto err_used;
Jason Wang2b8b3282013-01-28 01:05:18 +0000923 r = vhost_net_enable_vq(n, vq);
924 if (r)
925 goto err_used;
Jason Wang692a9982013-01-28 01:05:17 +0000926
Asias He28394002013-04-27 15:07:46 +0800927 oldubufs = nvq->ubufs;
928 nvq->ubufs = ubufs;
Michael S. Tsirkin64e9a9b2012-12-03 07:31:51 +0000929
930 n->tx_packets = 0;
931 n->tx_zcopy_err = 0;
Michael S. Tsirkin1280c272012-12-04 00:17:14 +0200932 n->tx_flush = false;
Jeff Dikedd1f4072010-03-04 16:10:14 -0500933 }
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +0000934
Michael S. Tsirkin1680e902010-07-15 15:19:12 +0300935 mutex_unlock(&vq->mutex);
936
Michael S. Tsirkinc047e5f2011-07-20 13:41:31 +0300937 if (oldubufs) {
Michael S. Tsirkinc38e39c2013-06-25 17:29:46 +0300938 vhost_net_ubuf_put_wait_and_free(oldubufs);
Michael S. Tsirkinc047e5f2011-07-20 13:41:31 +0300939 mutex_lock(&vq->mutex);
Michael S. Tsirkineaae8132012-11-01 09:16:51 +0000940 vhost_zerocopy_signal_used(n, vq);
Michael S. Tsirkinc047e5f2011-07-20 13:41:31 +0300941 mutex_unlock(&vq->mutex);
942 }
Michael S. Tsirkinbab632d2011-07-18 03:48:46 +0000943
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +0000944 if (oldsock) {
945 vhost_net_flush_vq(n, index);
Al Viro09aaacf2014-03-05 20:39:00 -0500946 sockfd_put(oldsock);
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +0000947 }
Jeff Dike1dace8c2010-03-04 16:10:14 -0500948
Michael S. Tsirkin1680e902010-07-15 15:19:12 +0300949 mutex_unlock(&n->dev.mutex);
950 return 0;
951
Jason Wang692a9982013-01-28 01:05:17 +0000952err_used:
Asias He22fa90c2013-05-07 14:54:36 +0800953 vq->private_data = oldsock;
Jason Wang692a9982013-01-28 01:05:17 +0000954 vhost_net_enable_vq(n, vq);
955 if (ubufs)
Michael S. Tsirkinc38e39c2013-06-25 17:29:46 +0300956 vhost_net_ubuf_put_wait_and_free(ubufs);
Michael S. Tsirkinbab632d2011-07-18 03:48:46 +0000957err_ubufs:
Al Viro09aaacf2014-03-05 20:39:00 -0500958 sockfd_put(sock);
Jeff Dike1dace8c2010-03-04 16:10:14 -0500959err_vq:
960 mutex_unlock(&vq->mutex);
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +0000961err:
962 mutex_unlock(&n->dev.mutex);
963 return r;
964}
965
966static long vhost_net_reset_owner(struct vhost_net *n)
967{
968 struct socket *tx_sock = NULL;
969 struct socket *rx_sock = NULL;
970 long err;
Michael S. Tsirkin150b9e52013-04-28 17:12:08 +0300971 struct vhost_memory *memory;
Krishna Kumard47effe2011-03-01 17:06:37 +0530972
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +0000973 mutex_lock(&n->dev.mutex);
974 err = vhost_dev_check_owner(&n->dev);
975 if (err)
976 goto done;
Michael S. Tsirkin150b9e52013-04-28 17:12:08 +0300977 memory = vhost_dev_reset_owner_prepare();
978 if (!memory) {
979 err = -ENOMEM;
980 goto done;
981 }
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +0000982 vhost_net_stop(n, &tx_sock, &rx_sock);
983 vhost_net_flush(n);
Michael S. Tsirkin150b9e52013-04-28 17:12:08 +0300984 vhost_dev_reset_owner(&n->dev, memory);
Michael S. Tsirkin81f95a52013-04-28 15:51:40 +0300985 vhost_net_vq_reset(n);
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +0000986done:
987 mutex_unlock(&n->dev.mutex);
988 if (tx_sock)
Al Viro09aaacf2014-03-05 20:39:00 -0500989 sockfd_put(tx_sock);
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +0000990 if (rx_sock)
Al Viro09aaacf2014-03-05 20:39:00 -0500991 sockfd_put(rx_sock);
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +0000992 return err;
993}
994
995static int vhost_net_set_features(struct vhost_net *n, u64 features)
996{
David Stevens8dd014a2010-07-27 18:52:21 +0300997 size_t vhost_hlen, sock_hlen, hdr_len;
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +0000998 int i;
David Stevens8dd014a2010-07-27 18:52:21 +0300999
Michael S. Tsirkine4fca7d2014-10-24 14:23:52 +03001000 hdr_len = (features & ((1ULL << VIRTIO_NET_F_MRG_RXBUF) |
1001 (1ULL << VIRTIO_F_VERSION_1))) ?
David Stevens8dd014a2010-07-27 18:52:21 +03001002 sizeof(struct virtio_net_hdr_mrg_rxbuf) :
1003 sizeof(struct virtio_net_hdr);
1004 if (features & (1 << VHOST_NET_F_VIRTIO_NET_HDR)) {
1005 /* vhost provides vnet_hdr */
1006 vhost_hlen = hdr_len;
1007 sock_hlen = 0;
1008 } else {
1009 /* socket provides vnet_hdr */
1010 vhost_hlen = 0;
1011 sock_hlen = hdr_len;
1012 }
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +00001013 mutex_lock(&n->dev.mutex);
1014 if ((features & (1 << VHOST_F_LOG_ALL)) &&
1015 !vhost_log_access_ok(&n->dev)) {
1016 mutex_unlock(&n->dev.mutex);
1017 return -EFAULT;
1018 }
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +00001019 for (i = 0; i < VHOST_NET_VQ_MAX; ++i) {
Asias He3ab2e422013-04-27 11:16:48 +08001020 mutex_lock(&n->vqs[i].vq.mutex);
Michael S. Tsirkinea16c512014-06-05 15:20:23 +03001021 n->vqs[i].vq.acked_features = features;
Michael S. Tsirkin81f95a52013-04-28 15:51:40 +03001022 n->vqs[i].vhost_hlen = vhost_hlen;
1023 n->vqs[i].sock_hlen = sock_hlen;
Asias He3ab2e422013-04-27 11:16:48 +08001024 mutex_unlock(&n->vqs[i].vq.mutex);
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +00001025 }
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +00001026 mutex_unlock(&n->dev.mutex);
1027 return 0;
1028}
1029
Asias Heb1ad8492013-05-06 11:16:00 +08001030static long vhost_net_set_owner(struct vhost_net *n)
1031{
1032 int r;
1033
1034 mutex_lock(&n->dev.mutex);
Michael S. Tsirkin05c05352013-06-06 15:20:39 +03001035 if (vhost_dev_has_owner(&n->dev)) {
1036 r = -EBUSY;
1037 goto out;
1038 }
Asias Heb1ad8492013-05-06 11:16:00 +08001039 r = vhost_net_set_ubuf_info(n);
1040 if (r)
1041 goto out;
1042 r = vhost_dev_set_owner(&n->dev);
1043 if (r)
1044 vhost_net_clear_ubuf_info(n);
1045 vhost_net_flush(n);
1046out:
1047 mutex_unlock(&n->dev.mutex);
1048 return r;
1049}
1050
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +00001051static long vhost_net_ioctl(struct file *f, unsigned int ioctl,
1052 unsigned long arg)
1053{
1054 struct vhost_net *n = f->private_data;
1055 void __user *argp = (void __user *)arg;
1056 u64 __user *featurep = argp;
1057 struct vhost_vring_file backend;
1058 u64 features;
1059 int r;
Krishna Kumard47effe2011-03-01 17:06:37 +05301060
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +00001061 switch (ioctl) {
1062 case VHOST_NET_SET_BACKEND:
Takuya Yoshikawad3553a52010-05-27 19:01:58 +09001063 if (copy_from_user(&backend, argp, sizeof backend))
1064 return -EFAULT;
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +00001065 return vhost_net_set_backend(n, backend.index, backend.fd);
1066 case VHOST_GET_FEATURES:
Stefan Hajnoczi0dd05a32012-07-21 06:55:36 +00001067 features = VHOST_NET_FEATURES;
Takuya Yoshikawad3553a52010-05-27 19:01:58 +09001068 if (copy_to_user(featurep, &features, sizeof features))
1069 return -EFAULT;
1070 return 0;
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +00001071 case VHOST_SET_FEATURES:
Takuya Yoshikawad3553a52010-05-27 19:01:58 +09001072 if (copy_from_user(&features, featurep, sizeof features))
1073 return -EFAULT;
Stefan Hajnoczi0dd05a32012-07-21 06:55:36 +00001074 if (features & ~VHOST_NET_FEATURES)
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +00001075 return -EOPNOTSUPP;
1076 return vhost_net_set_features(n, features);
1077 case VHOST_RESET_OWNER:
1078 return vhost_net_reset_owner(n);
Asias Heb1ad8492013-05-06 11:16:00 +08001079 case VHOST_SET_OWNER:
1080 return vhost_net_set_owner(n);
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +00001081 default:
1082 mutex_lock(&n->dev.mutex);
Michael S. Tsirkin935cdee2012-12-06 14:03:34 +02001083 r = vhost_dev_ioctl(&n->dev, ioctl, argp);
1084 if (r == -ENOIOCTLCMD)
1085 r = vhost_vring_ioctl(&n->dev, ioctl, argp);
1086 else
1087 vhost_net_flush(n);
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +00001088 mutex_unlock(&n->dev.mutex);
1089 return r;
1090 }
1091}
1092
1093#ifdef CONFIG_COMPAT
1094static long vhost_net_compat_ioctl(struct file *f, unsigned int ioctl,
1095 unsigned long arg)
1096{
1097 return vhost_net_ioctl(f, ioctl, (unsigned long)compat_ptr(arg));
1098}
1099#endif
1100
Tobias Klauser373a83a2010-05-17 15:12:49 +02001101static const struct file_operations vhost_net_fops = {
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +00001102 .owner = THIS_MODULE,
1103 .release = vhost_net_release,
1104 .unlocked_ioctl = vhost_net_ioctl,
1105#ifdef CONFIG_COMPAT
1106 .compat_ioctl = vhost_net_compat_ioctl,
1107#endif
1108 .open = vhost_net_open,
Arnd Bergmann6038f372010-08-15 18:52:59 +02001109 .llseek = noop_llseek,
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +00001110};
1111
1112static struct miscdevice vhost_net_misc = {
stephen hemminger7c7c7f02012-01-11 19:30:38 +00001113 .minor = VHOST_NET_MINOR,
1114 .name = "vhost-net",
1115 .fops = &vhost_net_fops,
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +00001116};
1117
Christoph Hellwiga8d37822010-04-13 14:11:25 -04001118static int vhost_net_init(void)
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +00001119{
Michael S. Tsirkinbab632d2011-07-18 03:48:46 +00001120 if (experimental_zcopytx)
Asias Hefe729a52013-05-06 16:38:24 +08001121 vhost_net_enable_zcopy(VHOST_NET_VQ_TX);
Tejun Heoc23f34452010-06-02 20:40:00 +02001122 return misc_register(&vhost_net_misc);
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +00001123}
1124module_init(vhost_net_init);
1125
Christoph Hellwiga8d37822010-04-13 14:11:25 -04001126static void vhost_net_exit(void)
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +00001127{
1128 misc_deregister(&vhost_net_misc);
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +00001129}
1130module_exit(vhost_net_exit);
1131
1132MODULE_VERSION("0.0.1");
1133MODULE_LICENSE("GPL v2");
1134MODULE_AUTHOR("Michael S. Tsirkin");
1135MODULE_DESCRIPTION("Host kernel accelerator for virtio net");
stephen hemminger7c7c7f02012-01-11 19:30:38 +00001136MODULE_ALIAS_MISCDEV(VHOST_NET_MINOR);
1137MODULE_ALIAS("devname:vhost-net");