blob: dd8798bf88e7c0b9095248227368c7a981f8d562 [file] [log] [blame]
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +00001/* Copyright (C) 2009 Red Hat, Inc.
2 * Author: Michael S. Tsirkin <mst@redhat.com>
3 *
4 * This work is licensed under the terms of the GNU GPL, version 2.
5 *
6 * virtio-net server in host kernel.
7 */
8
9#include <linux/compat.h>
10#include <linux/eventfd.h>
11#include <linux/vhost.h>
12#include <linux/virtio_net.h>
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +000013#include <linux/miscdevice.h>
14#include <linux/module.h>
Michael S. Tsirkinbab632d2011-07-18 03:48:46 +000015#include <linux/moduleparam.h>
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +000016#include <linux/mutex.h>
17#include <linux/workqueue.h>
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +000018#include <linux/file.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090019#include <linux/slab.h>
Michael S. Tsirkin23cc5a92013-01-23 21:46:47 +010020#include <linux/vmalloc.h>
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +000021
22#include <linux/net.h>
23#include <linux/if_packet.h>
24#include <linux/if_arp.h>
25#include <linux/if_tun.h>
Arnd Bergmann501c7742010-02-18 05:46:50 +000026#include <linux/if_macvlan.h>
Basil Gorc53cff5e2012-05-03 22:55:23 +000027#include <linux/if_vlan.h>
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +000028
29#include <net/sock.h>
30
31#include "vhost.h"
32
Jason Wange2c28912019-06-17 05:20:54 -040033static int experimental_zcopytx = 0;
Michael S. Tsirkinbab632d2011-07-18 03:48:46 +000034module_param(experimental_zcopytx, int, 0444);
Michael S. Tsirkinf9611c42012-12-06 14:56:00 +020035MODULE_PARM_DESC(experimental_zcopytx, "Enable Zero Copy TX;"
36 " 1 -Enable; 0 - Disable");
Michael S. Tsirkinbab632d2011-07-18 03:48:46 +000037
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +000038/* Max number of bytes transferred before requeueing the job.
39 * Using this limit prevents one virtqueue from starving others. */
40#define VHOST_NET_WEIGHT 0x80000
41
haibinzhang(张海斌)43f7e9b2019-08-17 00:00:19 +010042/* Max number of packets transferred before requeueing the job.
Paolo Abeni73f768b2019-08-17 00:00:28 +010043 * Using this limit prevents one virtqueue from starving others with small
44 * pkts.
45 */
46#define VHOST_NET_PKT_WEIGHT 256
haibinzhang(张海斌)43f7e9b2019-08-17 00:00:19 +010047
Michael S. Tsirkinbab632d2011-07-18 03:48:46 +000048/* MAX number of TX used buffers for outstanding zerocopy */
49#define VHOST_MAX_PEND 128
50#define VHOST_GOODCOPY_LEN 256
51
Michael S. Tsirkineaae8132012-11-01 09:16:51 +000052/*
53 * For transmit, used buffer len is unused; we override it to track buffer
54 * status internally; used for zerocopy tx only.
55 */
56/* Lower device DMA failed */
Michael S. Tsirkinbf995732014-10-24 11:49:27 +030057#define VHOST_DMA_FAILED_LEN ((__force __virtio32)3)
Michael S. Tsirkineaae8132012-11-01 09:16:51 +000058/* Lower device DMA done */
Michael S. Tsirkinbf995732014-10-24 11:49:27 +030059#define VHOST_DMA_DONE_LEN ((__force __virtio32)2)
Michael S. Tsirkineaae8132012-11-01 09:16:51 +000060/* Lower device DMA in progress */
Michael S. Tsirkinbf995732014-10-24 11:49:27 +030061#define VHOST_DMA_IN_PROGRESS ((__force __virtio32)1)
Michael S. Tsirkineaae8132012-11-01 09:16:51 +000062/* Buffer unused */
Michael S. Tsirkinbf995732014-10-24 11:49:27 +030063#define VHOST_DMA_CLEAR_LEN ((__force __virtio32)0)
Michael S. Tsirkineaae8132012-11-01 09:16:51 +000064
Michael S. Tsirkinbf995732014-10-24 11:49:27 +030065#define VHOST_DMA_IS_DONE(len) ((__force u32)(len) >= (__force u32)VHOST_DMA_DONE_LEN)
Michael S. Tsirkineaae8132012-11-01 09:16:51 +000066
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +000067enum {
Asias He8570a6e2013-05-06 16:38:20 +080068 VHOST_NET_FEATURES = VHOST_FEATURES |
69 (1ULL << VHOST_NET_F_VIRTIO_NET_HDR) |
Jason Wang6b1e6cc2016-06-23 02:04:32 -040070 (1ULL << VIRTIO_NET_F_MRG_RXBUF) |
71 (1ULL << VIRTIO_F_IOMMU_PLATFORM)
Asias He8570a6e2013-05-06 16:38:20 +080072};
73
74enum {
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +000075 VHOST_NET_VQ_RX = 0,
76 VHOST_NET_VQ_TX = 1,
77 VHOST_NET_VQ_MAX = 2,
78};
79
Asias Hefe729a52013-05-06 16:38:24 +080080struct vhost_net_ubuf_ref {
Michael S. Tsirkin0ad8b482014-02-13 11:42:05 +020081 /* refcount follows semantics similar to kref:
82 * 0: object is released
83 * 1: no outstanding ubufs
84 * >1: outstanding ubufs
85 */
86 atomic_t refcount;
Asias He28394002013-04-27 15:07:46 +080087 wait_queue_head_t wait;
88 struct vhost_virtqueue *vq;
89};
90
Asias He3ab2e422013-04-27 11:16:48 +080091struct vhost_net_virtqueue {
92 struct vhost_virtqueue vq;
Michael S. Tsirkin81f95a52013-04-28 15:51:40 +030093 size_t vhost_hlen;
94 size_t sock_hlen;
Asias He28394002013-04-27 15:07:46 +080095 /* vhost zerocopy support fields below: */
96 /* last used idx for outstanding DMA zerocopy buffers */
97 int upend_idx;
98 /* first used idx for DMA done zerocopy buffers */
99 int done_idx;
100 /* an array of userspace buffers info */
101 struct ubuf_info *ubuf_info;
102 /* Reference counting for outstanding ubufs.
103 * Protected by vq mutex. Writers must also take device mutex. */
Asias Hefe729a52013-05-06 16:38:24 +0800104 struct vhost_net_ubuf_ref *ubufs;
Asias He3ab2e422013-04-27 11:16:48 +0800105};
106
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +0000107struct vhost_net {
108 struct vhost_dev dev;
Asias He3ab2e422013-04-27 11:16:48 +0800109 struct vhost_net_virtqueue vqs[VHOST_NET_VQ_MAX];
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +0000110 struct vhost_poll poll[VHOST_NET_VQ_MAX];
Michael S. Tsirkineaae8132012-11-01 09:16:51 +0000111 /* Number of TX recently submitted.
112 * Protected by tx vq lock. */
113 unsigned tx_packets;
114 /* Number of times zerocopy TX recently failed.
115 * Protected by tx vq lock. */
116 unsigned tx_zcopy_err;
Michael S. Tsirkin1280c272012-12-04 00:17:14 +0200117 /* Flush in progress. Protected by tx vq lock. */
118 bool tx_flush;
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +0000119};
120
Asias Hefe729a52013-05-06 16:38:24 +0800121static unsigned vhost_net_zcopy_mask __read_mostly;
Asias He28394002013-04-27 15:07:46 +0800122
Asias Hefe729a52013-05-06 16:38:24 +0800123static void vhost_net_enable_zcopy(int vq)
Asias He28394002013-04-27 15:07:46 +0800124{
Asias Hefe729a52013-05-06 16:38:24 +0800125 vhost_net_zcopy_mask |= 0x1 << vq;
Asias He28394002013-04-27 15:07:46 +0800126}
127
Asias Hefe729a52013-05-06 16:38:24 +0800128static struct vhost_net_ubuf_ref *
129vhost_net_ubuf_alloc(struct vhost_virtqueue *vq, bool zcopy)
Asias He28394002013-04-27 15:07:46 +0800130{
Asias Hefe729a52013-05-06 16:38:24 +0800131 struct vhost_net_ubuf_ref *ubufs;
Asias He28394002013-04-27 15:07:46 +0800132 /* No zero copy backend? Nothing to count. */
133 if (!zcopy)
134 return NULL;
135 ubufs = kmalloc(sizeof(*ubufs), GFP_KERNEL);
136 if (!ubufs)
137 return ERR_PTR(-ENOMEM);
Michael S. Tsirkin0ad8b482014-02-13 11:42:05 +0200138 atomic_set(&ubufs->refcount, 1);
Asias He28394002013-04-27 15:07:46 +0800139 init_waitqueue_head(&ubufs->wait);
140 ubufs->vq = vq;
141 return ubufs;
142}
143
Michael S. Tsirkin0ad8b482014-02-13 11:42:05 +0200144static int vhost_net_ubuf_put(struct vhost_net_ubuf_ref *ubufs)
Asias He28394002013-04-27 15:07:46 +0800145{
Michael S. Tsirkin0ad8b482014-02-13 11:42:05 +0200146 int r = atomic_sub_return(1, &ubufs->refcount);
147 if (unlikely(!r))
148 wake_up(&ubufs->wait);
149 return r;
Asias He28394002013-04-27 15:07:46 +0800150}
151
Asias Hefe729a52013-05-06 16:38:24 +0800152static void vhost_net_ubuf_put_and_wait(struct vhost_net_ubuf_ref *ubufs)
Asias He28394002013-04-27 15:07:46 +0800153{
Michael S. Tsirkin0ad8b482014-02-13 11:42:05 +0200154 vhost_net_ubuf_put(ubufs);
155 wait_event(ubufs->wait, !atomic_read(&ubufs->refcount));
Michael S. Tsirkinc38e39c2013-06-25 17:29:46 +0300156}
157
158static void vhost_net_ubuf_put_wait_and_free(struct vhost_net_ubuf_ref *ubufs)
159{
160 vhost_net_ubuf_put_and_wait(ubufs);
Asias He28394002013-04-27 15:07:46 +0800161 kfree(ubufs);
162}
163
Asias Heb1ad8492013-05-06 11:16:00 +0800164static void vhost_net_clear_ubuf_info(struct vhost_net *n)
165{
Asias Heb1ad8492013-05-06 11:16:00 +0800166 int i;
167
Michael S. Tsirkin288cfe72013-06-06 15:20:46 +0300168 for (i = 0; i < VHOST_NET_VQ_MAX; ++i) {
169 kfree(n->vqs[i].ubuf_info);
170 n->vqs[i].ubuf_info = NULL;
Asias Heb1ad8492013-05-06 11:16:00 +0800171 }
172}
173
Asias He0a1febf2013-06-05 21:17:38 +0800174static int vhost_net_set_ubuf_info(struct vhost_net *n)
Asias He28394002013-04-27 15:07:46 +0800175{
176 bool zcopy;
177 int i;
178
Michael S. Tsirkin288cfe72013-06-06 15:20:46 +0300179 for (i = 0; i < VHOST_NET_VQ_MAX; ++i) {
Asias Hefe729a52013-05-06 16:38:24 +0800180 zcopy = vhost_net_zcopy_mask & (0x1 << i);
Asias He28394002013-04-27 15:07:46 +0800181 if (!zcopy)
182 continue;
183 n->vqs[i].ubuf_info = kmalloc(sizeof(*n->vqs[i].ubuf_info) *
184 UIO_MAXIOV, GFP_KERNEL);
185 if (!n->vqs[i].ubuf_info)
186 goto err;
187 }
188 return 0;
189
190err:
Michael S. Tsirkin288cfe72013-06-06 15:20:46 +0300191 vhost_net_clear_ubuf_info(n);
Asias He28394002013-04-27 15:07:46 +0800192 return -ENOMEM;
193}
194
Asias He0a1febf2013-06-05 21:17:38 +0800195static void vhost_net_vq_reset(struct vhost_net *n)
Asias He28394002013-04-27 15:07:46 +0800196{
197 int i;
198
Michael S. Tsirkin288cfe72013-06-06 15:20:46 +0300199 vhost_net_clear_ubuf_info(n);
200
Asias He28394002013-04-27 15:07:46 +0800201 for (i = 0; i < VHOST_NET_VQ_MAX; i++) {
202 n->vqs[i].done_idx = 0;
203 n->vqs[i].upend_idx = 0;
204 n->vqs[i].ubufs = NULL;
Michael S. Tsirkin81f95a52013-04-28 15:51:40 +0300205 n->vqs[i].vhost_hlen = 0;
206 n->vqs[i].sock_hlen = 0;
Asias He28394002013-04-27 15:07:46 +0800207 }
208
209}
210
Michael S. Tsirkineaae8132012-11-01 09:16:51 +0000211static void vhost_net_tx_packet(struct vhost_net *net)
212{
213 ++net->tx_packets;
214 if (net->tx_packets < 1024)
215 return;
216 net->tx_packets = 0;
217 net->tx_zcopy_err = 0;
218}
219
220static void vhost_net_tx_err(struct vhost_net *net)
221{
222 ++net->tx_zcopy_err;
223}
224
225static bool vhost_net_tx_select_zcopy(struct vhost_net *net)
226{
Michael S. Tsirkin1280c272012-12-04 00:17:14 +0200227 /* TX flush waits for outstanding DMAs to be done.
228 * Don't start new DMAs.
229 */
230 return !net->tx_flush &&
231 net->tx_packets / 64 >= net->tx_zcopy_err;
Michael S. Tsirkineaae8132012-11-01 09:16:51 +0000232}
233
Michael S. Tsirkinbab632d2011-07-18 03:48:46 +0000234static bool vhost_sock_zcopy(struct socket *sock)
235{
236 return unlikely(experimental_zcopytx) &&
237 sock_flag(sock->sk, SOCK_ZEROCOPY);
238}
239
Michael S. Tsirkinb2116162012-11-01 09:16:46 +0000240/* In case of DMA done not in order in lower device driver for some reason.
241 * upend_idx is used to track end of used idx, done_idx is used to track head
242 * of used idx. Once lower device DMA done contiguously, we will signal KVM
243 * guest used idx.
244 */
Jason Wang094afe72013-09-02 16:40:56 +0800245static void vhost_zerocopy_signal_used(struct vhost_net *net,
246 struct vhost_virtqueue *vq)
Michael S. Tsirkinb2116162012-11-01 09:16:46 +0000247{
Asias He28394002013-04-27 15:07:46 +0800248 struct vhost_net_virtqueue *nvq =
249 container_of(vq, struct vhost_net_virtqueue, vq);
Jason Wangc92112a2013-09-02 16:40:57 +0800250 int i, add;
Michael S. Tsirkinb2116162012-11-01 09:16:46 +0000251 int j = 0;
252
Asias He28394002013-04-27 15:07:46 +0800253 for (i = nvq->done_idx; i != nvq->upend_idx; i = (i + 1) % UIO_MAXIOV) {
Michael S. Tsirkineaae8132012-11-01 09:16:51 +0000254 if (vq->heads[i].len == VHOST_DMA_FAILED_LEN)
255 vhost_net_tx_err(net);
Michael S. Tsirkinb2116162012-11-01 09:16:46 +0000256 if (VHOST_DMA_IS_DONE(vq->heads[i].len)) {
257 vq->heads[i].len = VHOST_DMA_CLEAR_LEN;
Michael S. Tsirkinb2116162012-11-01 09:16:46 +0000258 ++j;
259 } else
260 break;
261 }
Jason Wangc92112a2013-09-02 16:40:57 +0800262 while (j) {
263 add = min(UIO_MAXIOV - nvq->done_idx, j);
264 vhost_add_used_and_signal_n(vq->dev, vq,
265 &vq->heads[nvq->done_idx], add);
266 nvq->done_idx = (nvq->done_idx + add) % UIO_MAXIOV;
267 j -= add;
268 }
Michael S. Tsirkinb2116162012-11-01 09:16:46 +0000269}
270
Michael S. Tsirkineaae8132012-11-01 09:16:51 +0000271static void vhost_zerocopy_callback(struct ubuf_info *ubuf, bool success)
Michael S. Tsirkinb2116162012-11-01 09:16:46 +0000272{
Asias Hefe729a52013-05-06 16:38:24 +0800273 struct vhost_net_ubuf_ref *ubufs = ubuf->ctx;
Michael S. Tsirkinb2116162012-11-01 09:16:46 +0000274 struct vhost_virtqueue *vq = ubufs->vq;
Michael S. Tsirkin0ad8b482014-02-13 11:42:05 +0200275 int cnt;
Michael S. Tsirkinb2116162012-11-01 09:16:46 +0000276
Michael S. Tsirkinb0c057c2014-02-13 11:45:11 +0200277 rcu_read_lock_bh();
278
Jason Wang19c73b32013-09-02 16:41:00 +0800279 /* set len to mark this desc buffers done DMA */
280 vq->heads[ubuf->desc].len = success ?
281 VHOST_DMA_DONE_LEN : VHOST_DMA_FAILED_LEN;
Michael S. Tsirkin0ad8b482014-02-13 11:42:05 +0200282 cnt = vhost_net_ubuf_put(ubufs);
Jason Wang19c73b32013-09-02 16:41:00 +0800283
Michael S. Tsirkin24eb21a2012-11-01 09:16:55 +0000284 /*
285 * Trigger polling thread if guest stopped submitting new buffers:
Michael S. Tsirkin0ad8b482014-02-13 11:42:05 +0200286 * in this case, the refcount after decrement will eventually reach 1.
Michael S. Tsirkin24eb21a2012-11-01 09:16:55 +0000287 * We also trigger polling periodically after each 16 packets
288 * (the value 16 here is more or less arbitrary, it's tuned to trigger
289 * less than 10% of times).
290 */
Michael S. Tsirkin0ad8b482014-02-13 11:42:05 +0200291 if (cnt <= 1 || !(cnt % 16))
Michael S. Tsirkin24eb21a2012-11-01 09:16:55 +0000292 vhost_poll_queue(&vq->poll);
Michael S. Tsirkinb0c057c2014-02-13 11:45:11 +0200293
294 rcu_read_unlock_bh();
Michael S. Tsirkinb2116162012-11-01 09:16:46 +0000295}
296
Jason Wang03088132016-03-04 06:24:53 -0500297static inline unsigned long busy_clock(void)
298{
299 return local_clock() >> 10;
300}
301
302static bool vhost_can_busy_poll(struct vhost_dev *dev,
303 unsigned long endtime)
304{
305 return likely(!need_resched()) &&
306 likely(!time_after(busy_clock(), endtime)) &&
307 likely(!signal_pending(current)) &&
308 !vhost_has_work(dev);
309}
310
Jason Wang8241a1e2016-06-01 01:56:33 -0400311static void vhost_net_disable_vq(struct vhost_net *n,
312 struct vhost_virtqueue *vq)
313{
314 struct vhost_net_virtqueue *nvq =
315 container_of(vq, struct vhost_net_virtqueue, vq);
316 struct vhost_poll *poll = n->poll + (nvq - n->vqs);
317 if (!vq->private_data)
318 return;
319 vhost_poll_stop(poll);
320}
321
322static int vhost_net_enable_vq(struct vhost_net *n,
323 struct vhost_virtqueue *vq)
324{
325 struct vhost_net_virtqueue *nvq =
326 container_of(vq, struct vhost_net_virtqueue, vq);
327 struct vhost_poll *poll = n->poll + (nvq - n->vqs);
328 struct socket *sock;
329
330 sock = vq->private_data;
331 if (!sock)
332 return 0;
333
334 return vhost_poll_start(poll, sock->file);
335}
336
Jason Wang03088132016-03-04 06:24:53 -0500337static int vhost_net_tx_get_vq_desc(struct vhost_net *net,
338 struct vhost_virtqueue *vq,
339 struct iovec iov[], unsigned int iov_size,
340 unsigned int *out_num, unsigned int *in_num)
341{
342 unsigned long uninitialized_var(endtime);
343 int r = vhost_get_vq_desc(vq, vq->iov, ARRAY_SIZE(vq->iov),
Jason Wang6b1e6cc2016-06-23 02:04:32 -0400344 out_num, in_num, NULL, NULL);
Jason Wang03088132016-03-04 06:24:53 -0500345
346 if (r == vq->num && vq->busyloop_timeout) {
347 preempt_disable();
348 endtime = busy_clock() + vq->busyloop_timeout;
349 while (vhost_can_busy_poll(vq->dev, endtime) &&
350 vhost_vq_avail_empty(vq->dev, vq))
351 cpu_relax_lowlatency();
352 preempt_enable();
353 r = vhost_get_vq_desc(vq, vq->iov, ARRAY_SIZE(vq->iov),
Jason Wang6b1e6cc2016-06-23 02:04:32 -0400354 out_num, in_num, NULL, NULL);
Jason Wang03088132016-03-04 06:24:53 -0500355 }
356
357 return r;
358}
359
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +0000360/* Expects to be always run from workqueue - which acts as
361 * read-size critical section for our kind of RCU. */
362static void handle_tx(struct vhost_net *net)
363{
Asias He28394002013-04-27 15:07:46 +0800364 struct vhost_net_virtqueue *nvq = &net->vqs[VHOST_NET_VQ_TX];
Michael S. Tsirkin81f95a52013-04-28 15:51:40 +0300365 struct vhost_virtqueue *vq = &nvq->vq;
Al Viro98a527a2014-12-10 15:00:58 -0500366 unsigned out, in;
Michael S. Tsirkind5675bd2010-06-24 16:59:59 +0300367 int head;
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +0000368 struct msghdr msg = {
369 .msg_name = NULL,
370 .msg_namelen = 0,
371 .msg_control = NULL,
372 .msg_controllen = 0,
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +0000373 .msg_flags = MSG_DONTWAIT,
374 };
375 size_t len, total_len = 0;
Jason Wang70181d512013-04-10 20:50:48 +0000376 int err;
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +0000377 size_t hdr_size;
Arnd Bergmann28457ee2010-03-09 19:24:45 +0100378 struct socket *sock;
Asias Hefe729a52013-05-06 16:38:24 +0800379 struct vhost_net_ubuf_ref *uninitialized_var(ubufs);
Michael S. Tsirkincedb9bd2012-12-06 17:00:18 +0200380 bool zcopy, zcopy_used;
haibinzhang(张海斌)43f7e9b2019-08-17 00:00:19 +0100381 int sent_pkts = 0;
Arnd Bergmann28457ee2010-03-09 19:24:45 +0100382
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +0000383 mutex_lock(&vq->mutex);
Asias He2e26af72013-05-07 14:54:33 +0800384 sock = vq->private_data;
385 if (!sock)
386 goto out;
387
Jason Wang6b1e6cc2016-06-23 02:04:32 -0400388 if (!vq_iotlb_prefetch(vq))
389 goto out;
390
Michael S. Tsirkin8ea8cf82011-05-20 02:10:54 +0300391 vhost_disable_notify(&net->dev, vq);
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +0000392
Michael S. Tsirkin81f95a52013-04-28 15:51:40 +0300393 hdr_size = nvq->vhost_hlen;
Asias He28394002013-04-27 15:07:46 +0800394 zcopy = nvq->ubufs;
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +0000395
Jason Wang4b586282019-08-17 00:00:53 +0100396 do {
Michael S. Tsirkinbab632d2011-07-18 03:48:46 +0000397 /* Release DMAs done buffers first */
398 if (zcopy)
Michael S. Tsirkineaae8132012-11-01 09:16:51 +0000399 vhost_zerocopy_signal_used(net, vq);
Michael S. Tsirkinbab632d2011-07-18 03:48:46 +0000400
Jason Wangf7c6be42013-09-02 16:41:01 +0800401 /* If more outstanding DMAs, queue the work.
402 * Handle upend_idx wrap around
403 */
404 if (unlikely((nvq->upend_idx + vq->num - VHOST_MAX_PEND)
405 % UIO_MAXIOV == nvq->done_idx))
406 break;
407
Jason Wang03088132016-03-04 06:24:53 -0500408 head = vhost_net_tx_get_vq_desc(net, vq, vq->iov,
409 ARRAY_SIZE(vq->iov),
410 &out, &in);
Michael S. Tsirkind5675bd2010-06-24 16:59:59 +0300411 /* On error, stop handling until the next kick. */
Michael S. Tsirkin7b3384f2010-07-01 18:40:12 +0300412 if (unlikely(head < 0))
Michael S. Tsirkind5675bd2010-06-24 16:59:59 +0300413 break;
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +0000414 /* Nothing new? Wait for eventfd to tell us they refilled. */
415 if (head == vq->num) {
Michael S. Tsirkin8ea8cf82011-05-20 02:10:54 +0300416 if (unlikely(vhost_enable_notify(&net->dev, vq))) {
417 vhost_disable_notify(&net->dev, vq);
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +0000418 continue;
419 }
420 break;
421 }
422 if (in) {
423 vq_err(vq, "Unexpected descriptor format for TX: "
424 "out %d, int %d\n", out, in);
425 break;
426 }
427 /* Skip header. TODO: support TSO. */
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +0000428 len = iov_length(vq->iov, out);
Al Viroc0371da2014-11-24 10:42:55 -0500429 iov_iter_init(&msg.msg_iter, WRITE, vq->iov, out, len);
Al Viro98a527a2014-12-10 15:00:58 -0500430 iov_iter_advance(&msg.msg_iter, hdr_size);
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +0000431 /* Sanity check */
Al Viro01e97e62014-12-15 21:39:31 -0500432 if (!msg_data_left(&msg)) {
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +0000433 vq_err(vq, "Unexpected header len for TX: "
434 "%zd expected %zd\n",
Al Viro98a527a2014-12-10 15:00:58 -0500435 len, hdr_size);
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +0000436 break;
437 }
Al Viro01e97e62014-12-15 21:39:31 -0500438 len = msg_data_left(&msg);
Jason Wangce21a022013-09-02 16:40:59 +0800439
440 zcopy_used = zcopy && len >= VHOST_GOODCOPY_LEN
441 && (nvq->upend_idx + 1) % UIO_MAXIOV !=
442 nvq->done_idx
443 && vhost_net_tx_select_zcopy(net);
Michael S. Tsirkincedb9bd2012-12-06 17:00:18 +0200444
Michael S. Tsirkinbab632d2011-07-18 03:48:46 +0000445 /* use msg_control to pass vhost zerocopy ubuf info to skb */
Michael S. Tsirkincedb9bd2012-12-06 17:00:18 +0200446 if (zcopy_used) {
Jason Wangce21a022013-09-02 16:40:59 +0800447 struct ubuf_info *ubuf;
448 ubuf = nvq->ubuf_info + nvq->upend_idx;
Michael S. Tsirkinbab632d2011-07-18 03:48:46 +0000449
Michael S. Tsirkin8b386942014-10-24 14:19:48 +0300450 vq->heads[nvq->upend_idx].id = cpu_to_vhost32(vq, head);
Jason Wangce21a022013-09-02 16:40:59 +0800451 vq->heads[nvq->upend_idx].len = VHOST_DMA_IN_PROGRESS;
452 ubuf->callback = vhost_zerocopy_callback;
453 ubuf->ctx = nvq->ubufs;
454 ubuf->desc = nvq->upend_idx;
455 msg.msg_control = ubuf;
456 msg.msg_controllen = sizeof(ubuf);
457 ubufs = nvq->ubufs;
Michael S. Tsirkin0ad8b482014-02-13 11:42:05 +0200458 atomic_inc(&ubufs->refcount);
Asias He28394002013-04-27 15:07:46 +0800459 nvq->upend_idx = (nvq->upend_idx + 1) % UIO_MAXIOV;
Jason Wangce21a022013-09-02 16:40:59 +0800460 } else {
Jason Wang4364d5f2013-06-05 15:40:46 +0800461 msg.msg_control = NULL;
Jason Wangce21a022013-09-02 16:40:59 +0800462 ubufs = NULL;
463 }
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +0000464 /* TODO: Check specific error and bomb out unless ENOBUFS? */
Ying Xue1b784142015-03-02 15:37:48 +0800465 err = sock->ops->sendmsg(sock, &msg, len);
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +0000466 if (unlikely(err < 0)) {
Michael S. Tsirkincedb9bd2012-12-06 17:00:18 +0200467 if (zcopy_used) {
Jason Wangce21a022013-09-02 16:40:59 +0800468 vhost_net_ubuf_put(ubufs);
Asias He28394002013-04-27 15:07:46 +0800469 nvq->upend_idx = ((unsigned)nvq->upend_idx - 1)
470 % UIO_MAXIOV;
Michael S. Tsirkinbab632d2011-07-18 03:48:46 +0000471 }
David Stevens8dd014a2010-07-27 18:52:21 +0300472 vhost_discard_vq_desc(vq, 1);
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +0000473 break;
474 }
475 if (err != len)
Michael S. Tsirkin95c0ec62010-06-24 17:10:25 +0300476 pr_debug("Truncated TX packet: "
477 " len %d != %zd\n", err, len);
Michael S. Tsirkincedb9bd2012-12-06 17:00:18 +0200478 if (!zcopy_used)
Michael S. Tsirkinbab632d2011-07-18 03:48:46 +0000479 vhost_add_used_and_signal(&net->dev, vq, head, 0);
Jason Wangc8fb2172012-05-02 11:42:41 +0800480 else
Michael S. Tsirkineaae8132012-11-01 09:16:51 +0000481 vhost_zerocopy_signal_used(net, vq);
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +0000482 total_len += len;
Michael S. Tsirkineaae8132012-11-01 09:16:51 +0000483 vhost_net_tx_packet(net);
Jason Wang4b586282019-08-17 00:00:53 +0100484 } while (likely(!vhost_exceeds_weight(vq, ++sent_pkts, total_len)));
Asias He2e26af72013-05-07 14:54:33 +0800485out:
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +0000486 mutex_unlock(&vq->mutex);
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +0000487}
488
David Stevens8dd014a2010-07-27 18:52:21 +0300489static int peek_head_len(struct sock *sk)
490{
Jason Wang1576d982016-06-30 14:45:36 +0800491 struct socket *sock = sk->sk_socket;
David Stevens8dd014a2010-07-27 18:52:21 +0300492 struct sk_buff *head;
493 int len = 0;
Jason Wang783e3982011-01-17 16:11:17 +0800494 unsigned long flags;
David Stevens8dd014a2010-07-27 18:52:21 +0300495
Jason Wang1576d982016-06-30 14:45:36 +0800496 if (sock->ops->peek_len)
497 return sock->ops->peek_len(sock);
498
Jason Wang783e3982011-01-17 16:11:17 +0800499 spin_lock_irqsave(&sk->sk_receive_queue.lock, flags);
David Stevens8dd014a2010-07-27 18:52:21 +0300500 head = skb_peek(&sk->sk_receive_queue);
Basil Gorc53cff5e2012-05-03 22:55:23 +0000501 if (likely(head)) {
David Stevens8dd014a2010-07-27 18:52:21 +0300502 len = head->len;
Jiri Pirkodf8a39d2015-01-13 17:13:44 +0100503 if (skb_vlan_tag_present(head))
Basil Gorc53cff5e2012-05-03 22:55:23 +0000504 len += VLAN_HLEN;
505 }
506
Jason Wang783e3982011-01-17 16:11:17 +0800507 spin_unlock_irqrestore(&sk->sk_receive_queue.lock, flags);
David Stevens8dd014a2010-07-27 18:52:21 +0300508 return len;
509}
510
Jason Wang1576d982016-06-30 14:45:36 +0800511static int sk_has_rx_data(struct sock *sk)
512{
513 struct socket *sock = sk->sk_socket;
514
515 if (sock->ops->peek_len)
516 return sock->ops->peek_len(sock);
517
518 return skb_queue_empty(&sk->sk_receive_queue);
519}
520
Jason Wang03088132016-03-04 06:24:53 -0500521static int vhost_net_rx_peek_head_len(struct vhost_net *net, struct sock *sk)
522{
523 struct vhost_net_virtqueue *nvq = &net->vqs[VHOST_NET_VQ_TX];
524 struct vhost_virtqueue *vq = &nvq->vq;
525 unsigned long uninitialized_var(endtime);
526 int len = peek_head_len(sk);
527
528 if (!len && vq->busyloop_timeout) {
529 /* Both tx vq and rx socket were polled here */
Jason Wang1cb81752018-03-26 16:10:23 +0800530 mutex_lock_nested(&vq->mutex, 1);
Jason Wang03088132016-03-04 06:24:53 -0500531 vhost_disable_notify(&net->dev, vq);
532
533 preempt_disable();
534 endtime = busy_clock() + vq->busyloop_timeout;
535
536 while (vhost_can_busy_poll(&net->dev, endtime) &&
Jason Wang1576d982016-06-30 14:45:36 +0800537 !sk_has_rx_data(sk) &&
Jason Wang03088132016-03-04 06:24:53 -0500538 vhost_vq_avail_empty(&net->dev, vq))
539 cpu_relax_lowlatency();
540
541 preempt_enable();
542
Jason Wangf5755c02017-09-05 09:22:05 +0800543 if (!vhost_vq_avail_empty(&net->dev, vq))
Jason Wang03088132016-03-04 06:24:53 -0500544 vhost_poll_queue(&vq->poll);
Jason Wangf5755c02017-09-05 09:22:05 +0800545 else if (unlikely(vhost_enable_notify(&net->dev, vq))) {
546 vhost_disable_notify(&net->dev, vq);
547 vhost_poll_queue(&vq->poll);
548 }
549
Jason Wang03088132016-03-04 06:24:53 -0500550 mutex_unlock(&vq->mutex);
551
552 len = peek_head_len(sk);
553 }
554
555 return len;
556}
557
David Stevens8dd014a2010-07-27 18:52:21 +0300558/* This is a multi-buffer version of vhost_get_desc, that works if
559 * vq has read descriptors only.
560 * @vq - the relevant virtqueue
561 * @datalen - data length we'll be reading
562 * @iovcount - returned count of io vectors we fill
563 * @log - vhost log
564 * @log_num - log offset
Jason Wang94249362011-01-17 16:11:08 +0800565 * @quota - headcount quota, 1 for big buffer
David Stevens8dd014a2010-07-27 18:52:21 +0300566 * returns number of buffer heads allocated, negative on error
567 */
568static int get_rx_bufs(struct vhost_virtqueue *vq,
569 struct vring_used_elem *heads,
570 int datalen,
571 unsigned *iovcount,
572 struct vhost_log *log,
Jason Wang94249362011-01-17 16:11:08 +0800573 unsigned *log_num,
574 unsigned int quota)
David Stevens8dd014a2010-07-27 18:52:21 +0300575{
576 unsigned int out, in;
577 int seg = 0;
578 int headcount = 0;
579 unsigned d;
580 int r, nlogs = 0;
Michael S. Tsirkin8b386942014-10-24 14:19:48 +0300581 /* len is always initialized before use since we are always called with
582 * datalen > 0.
583 */
584 u32 uninitialized_var(len);
David Stevens8dd014a2010-07-27 18:52:21 +0300585
Jason Wang94249362011-01-17 16:11:08 +0800586 while (datalen > 0 && headcount < quota) {
Jason Wange0e9b402010-09-14 23:53:05 +0800587 if (unlikely(seg >= UIO_MAXIOV)) {
David Stevens8dd014a2010-07-27 18:52:21 +0300588 r = -ENOBUFS;
589 goto err;
590 }
Michael S. Tsirkin47283be2014-06-05 15:20:27 +0300591 r = vhost_get_vq_desc(vq, vq->iov + seg,
David Stevens8dd014a2010-07-27 18:52:21 +0300592 ARRAY_SIZE(vq->iov) - seg, &out,
593 &in, log, log_num);
Michael S. Tsirkina39ee442014-03-27 12:53:37 +0200594 if (unlikely(r < 0))
595 goto err;
596
597 d = r;
David Stevens8dd014a2010-07-27 18:52:21 +0300598 if (d == vq->num) {
599 r = 0;
600 goto err;
601 }
602 if (unlikely(out || in <= 0)) {
603 vq_err(vq, "unexpected descriptor format for RX: "
604 "out %d, in %d\n", out, in);
605 r = -EINVAL;
606 goto err;
607 }
608 if (unlikely(log)) {
609 nlogs += *log_num;
610 log += *log_num;
611 }
Michael S. Tsirkin8b386942014-10-24 14:19:48 +0300612 heads[headcount].id = cpu_to_vhost32(vq, d);
613 len = iov_length(vq->iov + seg, in);
614 heads[headcount].len = cpu_to_vhost32(vq, len);
615 datalen -= len;
David Stevens8dd014a2010-07-27 18:52:21 +0300616 ++headcount;
617 seg += in;
618 }
Michael S. Tsirkin99975cc2015-01-07 10:51:00 +0200619 heads[headcount - 1].len = cpu_to_vhost32(vq, len + datalen);
David Stevens8dd014a2010-07-27 18:52:21 +0300620 *iovcount = seg;
621 if (unlikely(log))
622 *log_num = nlogs;
Michael S. Tsirkind8316f32014-03-27 12:00:26 +0200623
624 /* Detect overrun */
625 if (unlikely(datalen > 0)) {
626 r = UIO_MAXIOV + 1;
627 goto err;
628 }
David Stevens8dd014a2010-07-27 18:52:21 +0300629 return headcount;
630err:
631 vhost_discard_vq_desc(vq, headcount);
632 return r;
633}
634
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +0000635/* Expects to be always run from workqueue - which acts as
636 * read-size critical section for our kind of RCU. */
Jason Wang94249362011-01-17 16:11:08 +0800637static void handle_rx(struct vhost_net *net)
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +0000638{
Michael S. Tsirkin81f95a52013-04-28 15:51:40 +0300639 struct vhost_net_virtqueue *nvq = &net->vqs[VHOST_NET_VQ_RX];
640 struct vhost_virtqueue *vq = &nvq->vq;
David Stevens8dd014a2010-07-27 18:52:21 +0300641 unsigned uninitialized_var(in), log;
642 struct vhost_log *vq_log;
643 struct msghdr msg = {
644 .msg_name = NULL,
645 .msg_namelen = 0,
646 .msg_control = NULL, /* FIXME: get and handle RX aux data. */
647 .msg_controllen = 0,
David Stevens8dd014a2010-07-27 18:52:21 +0300648 .msg_flags = MSG_DONTWAIT,
649 };
Jason Wang0960b642015-02-15 16:35:17 +0800650 struct virtio_net_hdr hdr = {
651 .flags = 0,
652 .gso_type = VIRTIO_NET_HDR_GSO_NONE
David Stevens8dd014a2010-07-27 18:52:21 +0300653 };
David Stevens8dd014a2010-07-27 18:52:21 +0300654 size_t total_len = 0;
Michael S. Tsirkin910a5782012-10-24 20:37:51 +0200655 int err, mergeable;
656 s16 headcount;
David Stevens8dd014a2010-07-27 18:52:21 +0300657 size_t vhost_hlen, sock_hlen;
658 size_t vhost_len, sock_len;
Asias He2e26af72013-05-07 14:54:33 +0800659 struct socket *sock;
Al Viroba7438a2014-12-10 15:51:28 -0500660 struct iov_iter fixup;
Jason Wang0960b642015-02-15 16:35:17 +0800661 __virtio16 num_buffers;
Paolo Abeni73f768b2019-08-17 00:00:28 +0100662 int recv_pkts = 0;
David Stevens8dd014a2010-07-27 18:52:21 +0300663
Jason Wang1cb81752018-03-26 16:10:23 +0800664 mutex_lock_nested(&vq->mutex, 0);
Asias He2e26af72013-05-07 14:54:33 +0800665 sock = vq->private_data;
666 if (!sock)
667 goto out;
Jason Wang6b1e6cc2016-06-23 02:04:32 -0400668
669 if (!vq_iotlb_prefetch(vq))
670 goto out;
671
Michael S. Tsirkin8ea8cf82011-05-20 02:10:54 +0300672 vhost_disable_notify(&net->dev, vq);
Jason Wang8241a1e2016-06-01 01:56:33 -0400673 vhost_net_disable_vq(net, vq);
Asias He2e26af72013-05-07 14:54:33 +0800674
Michael S. Tsirkin81f95a52013-04-28 15:51:40 +0300675 vhost_hlen = nvq->vhost_hlen;
676 sock_hlen = nvq->sock_hlen;
David Stevens8dd014a2010-07-27 18:52:21 +0300677
Michael S. Tsirkinea16c512014-06-05 15:20:23 +0300678 vq_log = unlikely(vhost_has_feature(vq, VHOST_F_LOG_ALL)) ?
David Stevens8dd014a2010-07-27 18:52:21 +0300679 vq->log : NULL;
Michael S. Tsirkinea16c512014-06-05 15:20:23 +0300680 mergeable = vhost_has_feature(vq, VIRTIO_NET_F_MRG_RXBUF);
David Stevens8dd014a2010-07-27 18:52:21 +0300681
Jason Wang4b586282019-08-17 00:00:53 +0100682 do {
683 sock_len = vhost_net_rx_peek_head_len(net, sock->sk);
684 if (!sock_len)
685 break;
David Stevens8dd014a2010-07-27 18:52:21 +0300686 sock_len += sock_hlen;
687 vhost_len = sock_len + vhost_hlen;
688 headcount = get_rx_bufs(vq, vq->heads, vhost_len,
Jason Wang94249362011-01-17 16:11:08 +0800689 &in, vq_log, &log,
690 likely(mergeable) ? UIO_MAXIOV : 1);
David Stevens8dd014a2010-07-27 18:52:21 +0300691 /* On error, stop handling until the next kick. */
692 if (unlikely(headcount < 0))
Jason Wang8241a1e2016-06-01 01:56:33 -0400693 goto out;
Michael S. Tsirkind8316f32014-03-27 12:00:26 +0200694 /* On overrun, truncate and discard */
695 if (unlikely(headcount > UIO_MAXIOV)) {
Al Viroc0371da2014-11-24 10:42:55 -0500696 iov_iter_init(&msg.msg_iter, READ, vq->iov, 1, 1);
Ying Xue1b784142015-03-02 15:37:48 +0800697 err = sock->ops->recvmsg(sock, &msg,
Michael S. Tsirkind8316f32014-03-27 12:00:26 +0200698 1, MSG_DONTWAIT | MSG_TRUNC);
699 pr_debug("Discarded rx packet: len %zd\n", sock_len);
700 continue;
701 }
David Stevens8dd014a2010-07-27 18:52:21 +0300702 /* OK, now we need to know about added descriptors. */
703 if (!headcount) {
Michael S. Tsirkin8ea8cf82011-05-20 02:10:54 +0300704 if (unlikely(vhost_enable_notify(&net->dev, vq))) {
David Stevens8dd014a2010-07-27 18:52:21 +0300705 /* They have slipped one in as we were
706 * doing that: check again. */
Michael S. Tsirkin8ea8cf82011-05-20 02:10:54 +0300707 vhost_disable_notify(&net->dev, vq);
David Stevens8dd014a2010-07-27 18:52:21 +0300708 continue;
709 }
710 /* Nothing new? Wait for eventfd to tell us
711 * they refilled. */
Jason Wang8241a1e2016-06-01 01:56:33 -0400712 goto out;
David Stevens8dd014a2010-07-27 18:52:21 +0300713 }
714 /* We don't need to be notified again. */
Al Viroba7438a2014-12-10 15:51:28 -0500715 iov_iter_init(&msg.msg_iter, READ, vq->iov, in, vhost_len);
716 fixup = msg.msg_iter;
717 if (unlikely((vhost_hlen))) {
718 /* We will supply the header ourselves
719 * TODO: support TSO.
720 */
721 iov_iter_advance(&msg.msg_iter, vhost_hlen);
Al Viroba7438a2014-12-10 15:51:28 -0500722 }
Ying Xue1b784142015-03-02 15:37:48 +0800723 err = sock->ops->recvmsg(sock, &msg,
David Stevens8dd014a2010-07-27 18:52:21 +0300724 sock_len, MSG_DONTWAIT | MSG_TRUNC);
725 /* Userspace might have consumed the packet meanwhile:
726 * it's not supposed to do this usually, but might be hard
727 * to prevent. Discard data we got (if any) and keep going. */
728 if (unlikely(err != sock_len)) {
729 pr_debug("Discarded rx packet: "
730 " len %d, expected %zd\n", err, sock_len);
731 vhost_discard_vq_desc(vq, headcount);
732 continue;
733 }
Al Viroba7438a2014-12-10 15:51:28 -0500734 /* Supply virtio_net_hdr if VHOST_NET_F_VIRTIO_NET_HDR */
Michael S. Tsirkin4c5a8442015-02-25 15:19:28 +0100735 if (unlikely(vhost_hlen)) {
736 if (copy_to_iter(&hdr, sizeof(hdr),
737 &fixup) != sizeof(hdr)) {
738 vq_err(vq, "Unable to write vnet_hdr "
739 "at addr %p\n", vq->iov->iov_base);
Jason Wang8241a1e2016-06-01 01:56:33 -0400740 goto out;
Michael S. Tsirkin4c5a8442015-02-25 15:19:28 +0100741 }
742 } else {
743 /* Header came from socket; we'll need to patch
744 * ->num_buffers over if VIRTIO_NET_F_MRG_RXBUF
745 */
746 iov_iter_advance(&fixup, sizeof(hdr));
David Stevens8dd014a2010-07-27 18:52:21 +0300747 }
748 /* TODO: Should check and handle checksum. */
Michael S. Tsirkin5201aa42015-02-03 11:07:06 +0200749
Jason Wang0960b642015-02-15 16:35:17 +0800750 num_buffers = cpu_to_vhost16(vq, headcount);
Jason Wangcfbdab92011-01-17 16:10:59 +0800751 if (likely(mergeable) &&
Michael S. Tsirkin0d79a492015-02-25 15:20:01 +0100752 copy_to_iter(&num_buffers, sizeof num_buffers,
753 &fixup) != sizeof num_buffers) {
David Stevens8dd014a2010-07-27 18:52:21 +0300754 vq_err(vq, "Failed num_buffers write");
755 vhost_discard_vq_desc(vq, headcount);
Jason Wang8241a1e2016-06-01 01:56:33 -0400756 goto out;
David Stevens8dd014a2010-07-27 18:52:21 +0300757 }
758 vhost_add_used_and_signal_n(&net->dev, vq, vq->heads,
759 headcount);
760 if (unlikely(vq_log))
Jason Wangb35efeb2019-01-16 16:54:42 +0800761 vhost_log_write(vq, vq_log, log, vhost_len,
762 vq->iov, in);
David Stevens8dd014a2010-07-27 18:52:21 +0300763 total_len += vhost_len;
Jason Wang4b586282019-08-17 00:00:53 +0100764 } while (likely(!vhost_exceeds_weight(vq, ++recv_pkts, total_len)));
765
766 if (!sock_len)
767 vhost_net_enable_vq(net, vq);
Asias He2e26af72013-05-07 14:54:33 +0800768out:
David Stevens8dd014a2010-07-27 18:52:21 +0300769 mutex_unlock(&vq->mutex);
David Stevens8dd014a2010-07-27 18:52:21 +0300770}
771
Tejun Heoc23f34452010-06-02 20:40:00 +0200772static void handle_tx_kick(struct vhost_work *work)
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +0000773{
Tejun Heoc23f34452010-06-02 20:40:00 +0200774 struct vhost_virtqueue *vq = container_of(work, struct vhost_virtqueue,
775 poll.work);
776 struct vhost_net *net = container_of(vq->dev, struct vhost_net, dev);
777
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +0000778 handle_tx(net);
779}
780
Tejun Heoc23f34452010-06-02 20:40:00 +0200781static void handle_rx_kick(struct vhost_work *work)
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +0000782{
Tejun Heoc23f34452010-06-02 20:40:00 +0200783 struct vhost_virtqueue *vq = container_of(work, struct vhost_virtqueue,
784 poll.work);
785 struct vhost_net *net = container_of(vq->dev, struct vhost_net, dev);
786
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +0000787 handle_rx(net);
788}
789
Tejun Heoc23f34452010-06-02 20:40:00 +0200790static void handle_tx_net(struct vhost_work *work)
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +0000791{
Tejun Heoc23f34452010-06-02 20:40:00 +0200792 struct vhost_net *net = container_of(work, struct vhost_net,
793 poll[VHOST_NET_VQ_TX].work);
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +0000794 handle_tx(net);
795}
796
Tejun Heoc23f34452010-06-02 20:40:00 +0200797static void handle_rx_net(struct vhost_work *work)
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +0000798{
Tejun Heoc23f34452010-06-02 20:40:00 +0200799 struct vhost_net *net = container_of(work, struct vhost_net,
800 poll[VHOST_NET_VQ_RX].work);
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +0000801 handle_rx(net);
802}
803
804static int vhost_net_open(struct inode *inode, struct file *f)
805{
Michael S. Tsirkin23cc5a92013-01-23 21:46:47 +0100806 struct vhost_net *n;
Tejun Heoc23f34452010-06-02 20:40:00 +0200807 struct vhost_dev *dev;
Asias He3ab2e422013-04-27 11:16:48 +0800808 struct vhost_virtqueue **vqs;
Zhi Yong Wu59566b6e2013-12-07 04:13:03 +0800809 int i;
Tejun Heoc23f34452010-06-02 20:40:00 +0200810
Michael S. Tsirkin23cc5a92013-01-23 21:46:47 +0100811 n = kmalloc(sizeof *n, GFP_KERNEL | __GFP_NOWARN | __GFP_REPEAT);
812 if (!n) {
813 n = vmalloc(sizeof *n);
814 if (!n)
815 return -ENOMEM;
816 }
Asias He3ab2e422013-04-27 11:16:48 +0800817 vqs = kmalloc(VHOST_NET_VQ_MAX * sizeof(*vqs), GFP_KERNEL);
818 if (!vqs) {
Romain Francoised04257b2014-06-12 10:42:34 +0200819 kvfree(n);
Asias He3ab2e422013-04-27 11:16:48 +0800820 return -ENOMEM;
821 }
Tejun Heoc23f34452010-06-02 20:40:00 +0200822
823 dev = &n->dev;
Asias He3ab2e422013-04-27 11:16:48 +0800824 vqs[VHOST_NET_VQ_TX] = &n->vqs[VHOST_NET_VQ_TX].vq;
825 vqs[VHOST_NET_VQ_RX] = &n->vqs[VHOST_NET_VQ_RX].vq;
826 n->vqs[VHOST_NET_VQ_TX].vq.handle_kick = handle_tx_kick;
827 n->vqs[VHOST_NET_VQ_RX].vq.handle_kick = handle_rx_kick;
Asias He28394002013-04-27 15:07:46 +0800828 for (i = 0; i < VHOST_NET_VQ_MAX; i++) {
829 n->vqs[i].ubufs = NULL;
830 n->vqs[i].ubuf_info = NULL;
831 n->vqs[i].upend_idx = 0;
832 n->vqs[i].done_idx = 0;
Michael S. Tsirkin81f95a52013-04-28 15:51:40 +0300833 n->vqs[i].vhost_hlen = 0;
834 n->vqs[i].sock_hlen = 0;
Asias He28394002013-04-27 15:07:46 +0800835 }
Jason Wang66c8d9d2019-08-17 00:00:44 +0100836 vhost_dev_init(dev, vqs, VHOST_NET_VQ_MAX,
Jason Wang4b586282019-08-17 00:00:53 +0100837 VHOST_NET_PKT_WEIGHT, VHOST_NET_WEIGHT);
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +0000838
Tejun Heoc23f34452010-06-02 20:40:00 +0200839 vhost_poll_init(n->poll + VHOST_NET_VQ_TX, handle_tx_net, POLLOUT, dev);
840 vhost_poll_init(n->poll + VHOST_NET_VQ_RX, handle_rx_net, POLLIN, dev);
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +0000841
842 f->private_data = n;
843
844 return 0;
845}
846
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +0000847static struct socket *vhost_net_stop_vq(struct vhost_net *n,
848 struct vhost_virtqueue *vq)
849{
850 struct socket *sock;
851
852 mutex_lock(&vq->mutex);
Asias He22fa90c2013-05-07 14:54:36 +0800853 sock = vq->private_data;
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +0000854 vhost_net_disable_vq(n, vq);
Asias He22fa90c2013-05-07 14:54:36 +0800855 vq->private_data = NULL;
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +0000856 mutex_unlock(&vq->mutex);
857 return sock;
858}
859
860static void vhost_net_stop(struct vhost_net *n, struct socket **tx_sock,
861 struct socket **rx_sock)
862{
Asias He3ab2e422013-04-27 11:16:48 +0800863 *tx_sock = vhost_net_stop_vq(n, &n->vqs[VHOST_NET_VQ_TX].vq);
864 *rx_sock = vhost_net_stop_vq(n, &n->vqs[VHOST_NET_VQ_RX].vq);
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +0000865}
866
867static void vhost_net_flush_vq(struct vhost_net *n, int index)
868{
869 vhost_poll_flush(n->poll + index);
Asias He3ab2e422013-04-27 11:16:48 +0800870 vhost_poll_flush(&n->vqs[index].vq.poll);
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +0000871}
872
873static void vhost_net_flush(struct vhost_net *n)
874{
875 vhost_net_flush_vq(n, VHOST_NET_VQ_TX);
876 vhost_net_flush_vq(n, VHOST_NET_VQ_RX);
Asias He28394002013-04-27 15:07:46 +0800877 if (n->vqs[VHOST_NET_VQ_TX].ubufs) {
Asias He3ab2e422013-04-27 11:16:48 +0800878 mutex_lock(&n->vqs[VHOST_NET_VQ_TX].vq.mutex);
Michael S. Tsirkin1280c272012-12-04 00:17:14 +0200879 n->tx_flush = true;
Asias He3ab2e422013-04-27 11:16:48 +0800880 mutex_unlock(&n->vqs[VHOST_NET_VQ_TX].vq.mutex);
Michael S. Tsirkin1280c272012-12-04 00:17:14 +0200881 /* Wait for all lower device DMAs done. */
Asias Hefe729a52013-05-06 16:38:24 +0800882 vhost_net_ubuf_put_and_wait(n->vqs[VHOST_NET_VQ_TX].ubufs);
Asias He3ab2e422013-04-27 11:16:48 +0800883 mutex_lock(&n->vqs[VHOST_NET_VQ_TX].vq.mutex);
Michael S. Tsirkin1280c272012-12-04 00:17:14 +0200884 n->tx_flush = false;
Michael S. Tsirkin0ad8b482014-02-13 11:42:05 +0200885 atomic_set(&n->vqs[VHOST_NET_VQ_TX].ubufs->refcount, 1);
Asias He3ab2e422013-04-27 11:16:48 +0800886 mutex_unlock(&n->vqs[VHOST_NET_VQ_TX].vq.mutex);
Michael S. Tsirkin1280c272012-12-04 00:17:14 +0200887 }
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +0000888}
889
890static int vhost_net_release(struct inode *inode, struct file *f)
891{
892 struct vhost_net *n = f->private_data;
893 struct socket *tx_sock;
894 struct socket *rx_sock;
895
896 vhost_net_stop(n, &tx_sock, &rx_sock);
897 vhost_net_flush(n);
Michael S. Tsirkinb2116162012-11-01 09:16:46 +0000898 vhost_dev_stop(&n->dev);
Michael S. Tsirkinea5d4042011-11-27 19:05:58 +0200899 vhost_dev_cleanup(&n->dev, false);
Michael S. Tsirkin81f95a52013-04-28 15:51:40 +0300900 vhost_net_vq_reset(n);
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +0000901 if (tx_sock)
Al Viro09aaacf2014-03-05 20:39:00 -0500902 sockfd_put(tx_sock);
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +0000903 if (rx_sock)
Al Viro09aaacf2014-03-05 20:39:00 -0500904 sockfd_put(rx_sock);
Michael S. Tsirkinb0c057c2014-02-13 11:45:11 +0200905 /* Make sure no callbacks are outstanding */
906 synchronize_rcu_bh();
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +0000907 /* We do an extra flush before freeing memory,
908 * since jobs can re-queue themselves. */
909 vhost_net_flush(n);
Asias He3ab2e422013-04-27 11:16:48 +0800910 kfree(n->dev.vqs);
Romain Francoised04257b2014-06-12 10:42:34 +0200911 kvfree(n);
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +0000912 return 0;
913}
914
915static struct socket *get_raw_socket(int fd)
916{
917 struct {
918 struct sockaddr_ll sa;
919 char buf[MAX_ADDR_LEN];
920 } uaddr;
921 int uaddr_len = sizeof uaddr, r;
922 struct socket *sock = sockfd_lookup(fd, &r);
Krishna Kumard47effe2011-03-01 17:06:37 +0530923
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +0000924 if (!sock)
925 return ERR_PTR(-ENOTSOCK);
926
927 /* Parameter checking */
928 if (sock->sk->sk_type != SOCK_RAW) {
929 r = -ESOCKTNOSUPPORT;
930 goto err;
931 }
932
933 r = sock->ops->getname(sock, (struct sockaddr *)&uaddr.sa,
934 &uaddr_len, 0);
935 if (r)
936 goto err;
937
938 if (uaddr.sa.sll_family != AF_PACKET) {
939 r = -EPFNOSUPPORT;
940 goto err;
941 }
942 return sock;
943err:
Al Viro09aaacf2014-03-05 20:39:00 -0500944 sockfd_put(sock);
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +0000945 return ERR_PTR(r);
946}
947
Arnd Bergmann501c7742010-02-18 05:46:50 +0000948static struct socket *get_tap_socket(int fd)
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +0000949{
950 struct file *file = fget(fd);
951 struct socket *sock;
Krishna Kumard47effe2011-03-01 17:06:37 +0530952
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +0000953 if (!file)
954 return ERR_PTR(-EBADF);
955 sock = tun_get_socket(file);
Arnd Bergmann501c7742010-02-18 05:46:50 +0000956 if (!IS_ERR(sock))
957 return sock;
958 sock = macvtap_get_socket(file);
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +0000959 if (IS_ERR(sock))
960 fput(file);
961 return sock;
962}
963
964static struct socket *get_socket(int fd)
965{
966 struct socket *sock;
Krishna Kumard47effe2011-03-01 17:06:37 +0530967
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +0000968 /* special case to disable backend */
969 if (fd == -1)
970 return NULL;
971 sock = get_raw_socket(fd);
972 if (!IS_ERR(sock))
973 return sock;
Arnd Bergmann501c7742010-02-18 05:46:50 +0000974 sock = get_tap_socket(fd);
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +0000975 if (!IS_ERR(sock))
976 return sock;
977 return ERR_PTR(-ENOTSOCK);
978}
979
980static long vhost_net_set_backend(struct vhost_net *n, unsigned index, int fd)
981{
982 struct socket *sock, *oldsock;
983 struct vhost_virtqueue *vq;
Asias He28394002013-04-27 15:07:46 +0800984 struct vhost_net_virtqueue *nvq;
Asias Hefe729a52013-05-06 16:38:24 +0800985 struct vhost_net_ubuf_ref *ubufs, *oldubufs = NULL;
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +0000986 int r;
987
988 mutex_lock(&n->dev.mutex);
989 r = vhost_dev_check_owner(&n->dev);
990 if (r)
991 goto err;
992
993 if (index >= VHOST_NET_VQ_MAX) {
994 r = -ENOBUFS;
995 goto err;
996 }
Asias He3ab2e422013-04-27 11:16:48 +0800997 vq = &n->vqs[index].vq;
Asias He28394002013-04-27 15:07:46 +0800998 nvq = &n->vqs[index];
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +0000999 mutex_lock(&vq->mutex);
1000
1001 /* Verify that ring has been setup correctly. */
1002 if (!vhost_vq_access_ok(vq)) {
1003 r = -EFAULT;
Jeff Dike1dace8c2010-03-04 16:10:14 -05001004 goto err_vq;
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +00001005 }
1006 sock = get_socket(fd);
1007 if (IS_ERR(sock)) {
1008 r = PTR_ERR(sock);
Jeff Dike1dace8c2010-03-04 16:10:14 -05001009 goto err_vq;
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +00001010 }
1011
1012 /* start polling new socket */
Asias He22fa90c2013-05-07 14:54:36 +08001013 oldsock = vq->private_data;
David S. Miller11fe8832010-07-20 18:25:24 -07001014 if (sock != oldsock) {
Asias Hefe729a52013-05-06 16:38:24 +08001015 ubufs = vhost_net_ubuf_alloc(vq,
1016 sock && vhost_sock_zcopy(sock));
Michael S. Tsirkinbab632d2011-07-18 03:48:46 +00001017 if (IS_ERR(ubufs)) {
1018 r = PTR_ERR(ubufs);
1019 goto err_ubufs;
1020 }
Jason Wang692a9982013-01-28 01:05:17 +00001021
Krishna Kumard47effe2011-03-01 17:06:37 +05301022 vhost_net_disable_vq(n, vq);
Asias He22fa90c2013-05-07 14:54:36 +08001023 vq->private_data = sock;
Greg Kurz80f7d032016-02-16 15:59:44 +01001024 r = vhost_vq_init_access(vq);
Jason Wangf59281d2011-06-21 18:04:27 +08001025 if (r)
Jason Wang692a9982013-01-28 01:05:17 +00001026 goto err_used;
Jason Wang2b8b3282013-01-28 01:05:18 +00001027 r = vhost_net_enable_vq(n, vq);
1028 if (r)
1029 goto err_used;
Jason Wang692a9982013-01-28 01:05:17 +00001030
Asias He28394002013-04-27 15:07:46 +08001031 oldubufs = nvq->ubufs;
1032 nvq->ubufs = ubufs;
Michael S. Tsirkin64e9a9b2012-12-03 07:31:51 +00001033
1034 n->tx_packets = 0;
1035 n->tx_zcopy_err = 0;
Michael S. Tsirkin1280c272012-12-04 00:17:14 +02001036 n->tx_flush = false;
Jeff Dikedd1f4072010-03-04 16:10:14 -05001037 }
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +00001038
Michael S. Tsirkin1680e902010-07-15 15:19:12 +03001039 mutex_unlock(&vq->mutex);
1040
Michael S. Tsirkinc047e5f2011-07-20 13:41:31 +03001041 if (oldubufs) {
Michael S. Tsirkinc38e39c2013-06-25 17:29:46 +03001042 vhost_net_ubuf_put_wait_and_free(oldubufs);
Michael S. Tsirkinc047e5f2011-07-20 13:41:31 +03001043 mutex_lock(&vq->mutex);
Michael S. Tsirkineaae8132012-11-01 09:16:51 +00001044 vhost_zerocopy_signal_used(n, vq);
Michael S. Tsirkinc047e5f2011-07-20 13:41:31 +03001045 mutex_unlock(&vq->mutex);
1046 }
Michael S. Tsirkinbab632d2011-07-18 03:48:46 +00001047
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +00001048 if (oldsock) {
1049 vhost_net_flush_vq(n, index);
Al Viro09aaacf2014-03-05 20:39:00 -05001050 sockfd_put(oldsock);
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +00001051 }
Jeff Dike1dace8c2010-03-04 16:10:14 -05001052
Michael S. Tsirkin1680e902010-07-15 15:19:12 +03001053 mutex_unlock(&n->dev.mutex);
1054 return 0;
1055
Jason Wang692a9982013-01-28 01:05:17 +00001056err_used:
Asias He22fa90c2013-05-07 14:54:36 +08001057 vq->private_data = oldsock;
Jason Wang692a9982013-01-28 01:05:17 +00001058 vhost_net_enable_vq(n, vq);
1059 if (ubufs)
Michael S. Tsirkinc38e39c2013-06-25 17:29:46 +03001060 vhost_net_ubuf_put_wait_and_free(ubufs);
Michael S. Tsirkinbab632d2011-07-18 03:48:46 +00001061err_ubufs:
Jason Wange11eb6a2018-06-21 13:11:31 +08001062 if (sock)
1063 sockfd_put(sock);
Jeff Dike1dace8c2010-03-04 16:10:14 -05001064err_vq:
1065 mutex_unlock(&vq->mutex);
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +00001066err:
1067 mutex_unlock(&n->dev.mutex);
1068 return r;
1069}
1070
1071static long vhost_net_reset_owner(struct vhost_net *n)
1072{
1073 struct socket *tx_sock = NULL;
1074 struct socket *rx_sock = NULL;
1075 long err;
Jason Wanga9709d62016-06-23 02:04:31 -04001076 struct vhost_umem *umem;
Krishna Kumard47effe2011-03-01 17:06:37 +05301077
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +00001078 mutex_lock(&n->dev.mutex);
1079 err = vhost_dev_check_owner(&n->dev);
1080 if (err)
1081 goto done;
Jason Wanga9709d62016-06-23 02:04:31 -04001082 umem = vhost_dev_reset_owner_prepare();
1083 if (!umem) {
Michael S. Tsirkin150b9e52013-04-28 17:12:08 +03001084 err = -ENOMEM;
1085 goto done;
1086 }
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +00001087 vhost_net_stop(n, &tx_sock, &rx_sock);
1088 vhost_net_flush(n);
Jason Wang73adb3b2018-01-25 22:03:52 +08001089 vhost_dev_stop(&n->dev);
Jason Wanga9709d62016-06-23 02:04:31 -04001090 vhost_dev_reset_owner(&n->dev, umem);
Michael S. Tsirkin81f95a52013-04-28 15:51:40 +03001091 vhost_net_vq_reset(n);
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +00001092done:
1093 mutex_unlock(&n->dev.mutex);
1094 if (tx_sock)
Al Viro09aaacf2014-03-05 20:39:00 -05001095 sockfd_put(tx_sock);
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +00001096 if (rx_sock)
Al Viro09aaacf2014-03-05 20:39:00 -05001097 sockfd_put(rx_sock);
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +00001098 return err;
1099}
1100
1101static int vhost_net_set_features(struct vhost_net *n, u64 features)
1102{
David Stevens8dd014a2010-07-27 18:52:21 +03001103 size_t vhost_hlen, sock_hlen, hdr_len;
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +00001104 int i;
David Stevens8dd014a2010-07-27 18:52:21 +03001105
Michael S. Tsirkine4fca7d2014-10-24 14:23:52 +03001106 hdr_len = (features & ((1ULL << VIRTIO_NET_F_MRG_RXBUF) |
1107 (1ULL << VIRTIO_F_VERSION_1))) ?
David Stevens8dd014a2010-07-27 18:52:21 +03001108 sizeof(struct virtio_net_hdr_mrg_rxbuf) :
1109 sizeof(struct virtio_net_hdr);
1110 if (features & (1 << VHOST_NET_F_VIRTIO_NET_HDR)) {
1111 /* vhost provides vnet_hdr */
1112 vhost_hlen = hdr_len;
1113 sock_hlen = 0;
1114 } else {
1115 /* socket provides vnet_hdr */
1116 vhost_hlen = 0;
1117 sock_hlen = hdr_len;
1118 }
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +00001119 mutex_lock(&n->dev.mutex);
1120 if ((features & (1 << VHOST_F_LOG_ALL)) &&
Jason Wang6b1e6cc2016-06-23 02:04:32 -04001121 !vhost_log_access_ok(&n->dev))
1122 goto out_unlock;
1123
1124 if ((features & (1ULL << VIRTIO_F_IOMMU_PLATFORM))) {
1125 if (vhost_init_device_iotlb(&n->dev, true))
1126 goto out_unlock;
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +00001127 }
Jason Wang6b1e6cc2016-06-23 02:04:32 -04001128
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +00001129 for (i = 0; i < VHOST_NET_VQ_MAX; ++i) {
Asias He3ab2e422013-04-27 11:16:48 +08001130 mutex_lock(&n->vqs[i].vq.mutex);
Michael S. Tsirkinea16c512014-06-05 15:20:23 +03001131 n->vqs[i].vq.acked_features = features;
Michael S. Tsirkin81f95a52013-04-28 15:51:40 +03001132 n->vqs[i].vhost_hlen = vhost_hlen;
1133 n->vqs[i].sock_hlen = sock_hlen;
Asias He3ab2e422013-04-27 11:16:48 +08001134 mutex_unlock(&n->vqs[i].vq.mutex);
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +00001135 }
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +00001136 mutex_unlock(&n->dev.mutex);
1137 return 0;
Jason Wang6b1e6cc2016-06-23 02:04:32 -04001138
1139out_unlock:
1140 mutex_unlock(&n->dev.mutex);
1141 return -EFAULT;
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +00001142}
1143
Asias Heb1ad8492013-05-06 11:16:00 +08001144static long vhost_net_set_owner(struct vhost_net *n)
1145{
1146 int r;
1147
1148 mutex_lock(&n->dev.mutex);
Michael S. Tsirkin05c05352013-06-06 15:20:39 +03001149 if (vhost_dev_has_owner(&n->dev)) {
1150 r = -EBUSY;
1151 goto out;
1152 }
Asias Heb1ad8492013-05-06 11:16:00 +08001153 r = vhost_net_set_ubuf_info(n);
1154 if (r)
1155 goto out;
1156 r = vhost_dev_set_owner(&n->dev);
1157 if (r)
1158 vhost_net_clear_ubuf_info(n);
1159 vhost_net_flush(n);
1160out:
1161 mutex_unlock(&n->dev.mutex);
1162 return r;
1163}
1164
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +00001165static long vhost_net_ioctl(struct file *f, unsigned int ioctl,
1166 unsigned long arg)
1167{
1168 struct vhost_net *n = f->private_data;
1169 void __user *argp = (void __user *)arg;
1170 u64 __user *featurep = argp;
1171 struct vhost_vring_file backend;
1172 u64 features;
1173 int r;
Krishna Kumard47effe2011-03-01 17:06:37 +05301174
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +00001175 switch (ioctl) {
1176 case VHOST_NET_SET_BACKEND:
Takuya Yoshikawad3553a52010-05-27 19:01:58 +09001177 if (copy_from_user(&backend, argp, sizeof backend))
1178 return -EFAULT;
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +00001179 return vhost_net_set_backend(n, backend.index, backend.fd);
1180 case VHOST_GET_FEATURES:
Stefan Hajnoczi0dd05a32012-07-21 06:55:36 +00001181 features = VHOST_NET_FEATURES;
Takuya Yoshikawad3553a52010-05-27 19:01:58 +09001182 if (copy_to_user(featurep, &features, sizeof features))
1183 return -EFAULT;
1184 return 0;
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +00001185 case VHOST_SET_FEATURES:
Takuya Yoshikawad3553a52010-05-27 19:01:58 +09001186 if (copy_from_user(&features, featurep, sizeof features))
1187 return -EFAULT;
Stefan Hajnoczi0dd05a32012-07-21 06:55:36 +00001188 if (features & ~VHOST_NET_FEATURES)
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +00001189 return -EOPNOTSUPP;
1190 return vhost_net_set_features(n, features);
1191 case VHOST_RESET_OWNER:
1192 return vhost_net_reset_owner(n);
Asias Heb1ad8492013-05-06 11:16:00 +08001193 case VHOST_SET_OWNER:
1194 return vhost_net_set_owner(n);
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +00001195 default:
1196 mutex_lock(&n->dev.mutex);
Michael S. Tsirkin935cdee2012-12-06 14:03:34 +02001197 r = vhost_dev_ioctl(&n->dev, ioctl, argp);
1198 if (r == -ENOIOCTLCMD)
1199 r = vhost_vring_ioctl(&n->dev, ioctl, argp);
1200 else
1201 vhost_net_flush(n);
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +00001202 mutex_unlock(&n->dev.mutex);
1203 return r;
1204 }
1205}
1206
1207#ifdef CONFIG_COMPAT
1208static long vhost_net_compat_ioctl(struct file *f, unsigned int ioctl,
1209 unsigned long arg)
1210{
1211 return vhost_net_ioctl(f, ioctl, (unsigned long)compat_ptr(arg));
1212}
1213#endif
1214
Jason Wang6b1e6cc2016-06-23 02:04:32 -04001215static ssize_t vhost_net_chr_read_iter(struct kiocb *iocb, struct iov_iter *to)
1216{
1217 struct file *file = iocb->ki_filp;
1218 struct vhost_net *n = file->private_data;
1219 struct vhost_dev *dev = &n->dev;
1220 int noblock = file->f_flags & O_NONBLOCK;
1221
1222 return vhost_chr_read_iter(dev, to, noblock);
1223}
1224
1225static ssize_t vhost_net_chr_write_iter(struct kiocb *iocb,
1226 struct iov_iter *from)
1227{
1228 struct file *file = iocb->ki_filp;
1229 struct vhost_net *n = file->private_data;
1230 struct vhost_dev *dev = &n->dev;
1231
1232 return vhost_chr_write_iter(dev, from);
1233}
1234
1235static unsigned int vhost_net_chr_poll(struct file *file, poll_table *wait)
1236{
1237 struct vhost_net *n = file->private_data;
1238 struct vhost_dev *dev = &n->dev;
1239
1240 return vhost_chr_poll(file, dev, wait);
1241}
1242
Tobias Klauser373a83a2010-05-17 15:12:49 +02001243static const struct file_operations vhost_net_fops = {
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +00001244 .owner = THIS_MODULE,
1245 .release = vhost_net_release,
Jason Wang6b1e6cc2016-06-23 02:04:32 -04001246 .read_iter = vhost_net_chr_read_iter,
1247 .write_iter = vhost_net_chr_write_iter,
1248 .poll = vhost_net_chr_poll,
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +00001249 .unlocked_ioctl = vhost_net_ioctl,
1250#ifdef CONFIG_COMPAT
1251 .compat_ioctl = vhost_net_compat_ioctl,
1252#endif
1253 .open = vhost_net_open,
Arnd Bergmann6038f372010-08-15 18:52:59 +02001254 .llseek = noop_llseek,
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +00001255};
1256
1257static struct miscdevice vhost_net_misc = {
stephen hemminger7c7c7f02012-01-11 19:30:38 +00001258 .minor = VHOST_NET_MINOR,
1259 .name = "vhost-net",
1260 .fops = &vhost_net_fops,
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +00001261};
1262
Christoph Hellwiga8d37822010-04-13 14:11:25 -04001263static int vhost_net_init(void)
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +00001264{
Michael S. Tsirkinbab632d2011-07-18 03:48:46 +00001265 if (experimental_zcopytx)
Asias Hefe729a52013-05-06 16:38:24 +08001266 vhost_net_enable_zcopy(VHOST_NET_VQ_TX);
Tejun Heoc23f34452010-06-02 20:40:00 +02001267 return misc_register(&vhost_net_misc);
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +00001268}
1269module_init(vhost_net_init);
1270
Christoph Hellwiga8d37822010-04-13 14:11:25 -04001271static void vhost_net_exit(void)
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +00001272{
1273 misc_deregister(&vhost_net_misc);
Michael S. Tsirkin3a4d5c94e2010-01-14 06:17:27 +00001274}
1275module_exit(vhost_net_exit);
1276
1277MODULE_VERSION("0.0.1");
1278MODULE_LICENSE("GPL v2");
1279MODULE_AUTHOR("Michael S. Tsirkin");
1280MODULE_DESCRIPTION("Host kernel accelerator for virtio net");
stephen hemminger7c7c7f02012-01-11 19:30:38 +00001281MODULE_ALIAS_MISCDEV(VHOST_NET_MINOR);
1282MODULE_ALIAS("devname:vhost-net");