blob: f442668a1e52e137b347419f9938d386f638b16b [file] [log] [blame]
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +00001/* Copyright (C) 2009 Red Hat, Inc.
2 * Author: Michael S. Tsirkin <mst@redhat.com>
3 *
4 * This work is licensed under the terms of the GNU GPL, version 2.
5 *
6 * virtio-net server in host kernel.
7 */
8
9#include <linux/compat.h>
10#include <linux/eventfd.h>
11#include <linux/vhost.h>
12#include <linux/virtio_net.h>
13#include <linux/mmu_context.h>
14#include <linux/miscdevice.h>
15#include <linux/module.h>
16#include <linux/mutex.h>
17#include <linux/workqueue.h>
18#include <linux/rcupdate.h>
19#include <linux/file.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090020#include <linux/slab.h>
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +000021
22#include <linux/net.h>
23#include <linux/if_packet.h>
24#include <linux/if_arp.h>
25#include <linux/if_tun.h>
Arnd Bergmann501c7742010-02-18 05:46:50 +000026#include <linux/if_macvlan.h>
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +000027
28#include <net/sock.h>
29
30#include "vhost.h"
31
32/* Max number of bytes transferred before requeueing the job.
33 * Using this limit prevents one virtqueue from starving others. */
34#define VHOST_NET_WEIGHT 0x80000
35
36enum {
37 VHOST_NET_VQ_RX = 0,
38 VHOST_NET_VQ_TX = 1,
39 VHOST_NET_VQ_MAX = 2,
40};
41
42enum vhost_net_poll_state {
43 VHOST_NET_POLL_DISABLED = 0,
44 VHOST_NET_POLL_STARTED = 1,
45 VHOST_NET_POLL_STOPPED = 2,
46};
47
48struct vhost_net {
49 struct vhost_dev dev;
50 struct vhost_virtqueue vqs[VHOST_NET_VQ_MAX];
51 struct vhost_poll poll[VHOST_NET_VQ_MAX];
52 /* Tells us whether we are polling a socket for TX.
53 * We only do this when socket buffer fills up.
54 * Protected by tx vq lock. */
55 enum vhost_net_poll_state tx_poll_state;
56};
57
58/* Pop first len bytes from iovec. Return number of segments used. */
59static int move_iovec_hdr(struct iovec *from, struct iovec *to,
60 size_t len, int iov_count)
61{
62 int seg = 0;
63 size_t size;
64 while (len && seg < iov_count) {
65 size = min(from->iov_len, len);
66 to->iov_base = from->iov_base;
67 to->iov_len = size;
68 from->iov_len -= size;
69 from->iov_base += size;
70 len -= size;
71 ++from;
72 ++to;
73 ++seg;
74 }
75 return seg;
76}
David Stevens8dd014a2010-07-27 18:52:21 +030077/* Copy iovec entries for len bytes from iovec. */
78static void copy_iovec_hdr(const struct iovec *from, struct iovec *to,
79 size_t len, int iovcount)
80{
81 int seg = 0;
82 size_t size;
83 while (len && seg < iovcount) {
84 size = min(from->iov_len, len);
85 to->iov_base = from->iov_base;
86 to->iov_len = size;
87 len -= size;
88 ++from;
89 ++to;
90 ++seg;
91 }
92}
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +000093
94/* Caller must have TX VQ lock */
95static void tx_poll_stop(struct vhost_net *net)
96{
97 if (likely(net->tx_poll_state != VHOST_NET_POLL_STARTED))
98 return;
99 vhost_poll_stop(net->poll + VHOST_NET_VQ_TX);
100 net->tx_poll_state = VHOST_NET_POLL_STOPPED;
101}
102
103/* Caller must have TX VQ lock */
104static void tx_poll_start(struct vhost_net *net, struct socket *sock)
105{
106 if (unlikely(net->tx_poll_state != VHOST_NET_POLL_STOPPED))
107 return;
108 vhost_poll_start(net->poll + VHOST_NET_VQ_TX, sock->file);
109 net->tx_poll_state = VHOST_NET_POLL_STARTED;
110}
111
112/* Expects to be always run from workqueue - which acts as
113 * read-size critical section for our kind of RCU. */
114static void handle_tx(struct vhost_net *net)
115{
116 struct vhost_virtqueue *vq = &net->dev.vqs[VHOST_NET_VQ_TX];
Michael S. Tsirkind5675bd2010-06-24 16:59:59 +0300117 unsigned out, in, s;
118 int head;
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +0000119 struct msghdr msg = {
120 .msg_name = NULL,
121 .msg_namelen = 0,
122 .msg_control = NULL,
123 .msg_controllen = 0,
124 .msg_iov = vq->iov,
125 .msg_flags = MSG_DONTWAIT,
126 };
127 size_t len, total_len = 0;
128 int err, wmem;
129 size_t hdr_size;
Arnd Bergmann28457ee2010-03-09 19:24:45 +0100130 struct socket *sock;
131
Michael S. Tsirkin11cd1a82010-11-14 17:31:52 +0200132 /* TODO: check that we are running from vhost_worker?
133 * Not sure it's worth it, it's straight-forward enough. */
134 sock = rcu_dereference_check(vq->private_data, 1);
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +0000135 if (!sock)
136 return;
137
138 wmem = atomic_read(&sock->sk->sk_wmem_alloc);
Sridhar Samudrala39286fa2010-02-28 19:39:16 +0200139 if (wmem >= sock->sk->sk_sndbuf) {
140 mutex_lock(&vq->mutex);
141 tx_poll_start(net, sock);
142 mutex_unlock(&vq->mutex);
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +0000143 return;
Sridhar Samudrala39286fa2010-02-28 19:39:16 +0200144 }
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +0000145
146 use_mm(net->dev.mm);
147 mutex_lock(&vq->mutex);
148 vhost_disable_notify(vq);
149
Michael S. Tsirkin0e255572010-03-08 23:24:22 +0200150 if (wmem < sock->sk->sk_sndbuf / 2)
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +0000151 tx_poll_stop(net);
David Stevens8dd014a2010-07-27 18:52:21 +0300152 hdr_size = vq->vhost_hlen;
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +0000153
154 for (;;) {
155 head = vhost_get_vq_desc(&net->dev, vq, vq->iov,
156 ARRAY_SIZE(vq->iov),
157 &out, &in,
158 NULL, NULL);
Michael S. Tsirkind5675bd2010-06-24 16:59:59 +0300159 /* On error, stop handling until the next kick. */
Michael S. Tsirkin7b3384f2010-07-01 18:40:12 +0300160 if (unlikely(head < 0))
Michael S. Tsirkind5675bd2010-06-24 16:59:59 +0300161 break;
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +0000162 /* Nothing new? Wait for eventfd to tell us they refilled. */
163 if (head == vq->num) {
164 wmem = atomic_read(&sock->sk->sk_wmem_alloc);
165 if (wmem >= sock->sk->sk_sndbuf * 3 / 4) {
166 tx_poll_start(net, sock);
167 set_bit(SOCK_ASYNC_NOSPACE, &sock->flags);
168 break;
169 }
170 if (unlikely(vhost_enable_notify(vq))) {
171 vhost_disable_notify(vq);
172 continue;
173 }
174 break;
175 }
176 if (in) {
177 vq_err(vq, "Unexpected descriptor format for TX: "
178 "out %d, int %d\n", out, in);
179 break;
180 }
181 /* Skip header. TODO: support TSO. */
182 s = move_iovec_hdr(vq->iov, vq->hdr, hdr_size, out);
183 msg.msg_iovlen = out;
184 len = iov_length(vq->iov, out);
185 /* Sanity check */
186 if (!len) {
187 vq_err(vq, "Unexpected header len for TX: "
188 "%zd expected %zd\n",
189 iov_length(vq->hdr, s), hdr_size);
190 break;
191 }
192 /* TODO: Check specific error and bomb out unless ENOBUFS? */
193 err = sock->ops->sendmsg(NULL, sock, &msg, len);
194 if (unlikely(err < 0)) {
David Stevens8dd014a2010-07-27 18:52:21 +0300195 vhost_discard_vq_desc(vq, 1);
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +0000196 tx_poll_start(net, sock);
197 break;
198 }
199 if (err != len)
Michael S. Tsirkin95c0ec62010-06-24 17:10:25 +0300200 pr_debug("Truncated TX packet: "
201 " len %d != %zd\n", err, len);
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +0000202 vhost_add_used_and_signal(&net->dev, vq, head, 0);
203 total_len += len;
204 if (unlikely(total_len >= VHOST_NET_WEIGHT)) {
205 vhost_poll_queue(&vq->poll);
206 break;
207 }
208 }
209
210 mutex_unlock(&vq->mutex);
211 unuse_mm(net->dev.mm);
212}
213
David Stevens8dd014a2010-07-27 18:52:21 +0300214static int peek_head_len(struct sock *sk)
215{
216 struct sk_buff *head;
217 int len = 0;
218
219 lock_sock(sk);
220 head = skb_peek(&sk->sk_receive_queue);
221 if (head)
222 len = head->len;
223 release_sock(sk);
224 return len;
225}
226
227/* This is a multi-buffer version of vhost_get_desc, that works if
228 * vq has read descriptors only.
229 * @vq - the relevant virtqueue
230 * @datalen - data length we'll be reading
231 * @iovcount - returned count of io vectors we fill
232 * @log - vhost log
233 * @log_num - log offset
234 * returns number of buffer heads allocated, negative on error
235 */
236static int get_rx_bufs(struct vhost_virtqueue *vq,
237 struct vring_used_elem *heads,
238 int datalen,
239 unsigned *iovcount,
240 struct vhost_log *log,
241 unsigned *log_num)
242{
243 unsigned int out, in;
244 int seg = 0;
245 int headcount = 0;
246 unsigned d;
247 int r, nlogs = 0;
248
249 while (datalen > 0) {
Jason Wange0e9b402010-09-14 23:53:05 +0800250 if (unlikely(seg >= UIO_MAXIOV)) {
David Stevens8dd014a2010-07-27 18:52:21 +0300251 r = -ENOBUFS;
252 goto err;
253 }
254 d = vhost_get_vq_desc(vq->dev, vq, vq->iov + seg,
255 ARRAY_SIZE(vq->iov) - seg, &out,
256 &in, log, log_num);
257 if (d == vq->num) {
258 r = 0;
259 goto err;
260 }
261 if (unlikely(out || in <= 0)) {
262 vq_err(vq, "unexpected descriptor format for RX: "
263 "out %d, in %d\n", out, in);
264 r = -EINVAL;
265 goto err;
266 }
267 if (unlikely(log)) {
268 nlogs += *log_num;
269 log += *log_num;
270 }
271 heads[headcount].id = d;
272 heads[headcount].len = iov_length(vq->iov + seg, in);
273 datalen -= heads[headcount].len;
274 ++headcount;
275 seg += in;
276 }
277 heads[headcount - 1].len += datalen;
278 *iovcount = seg;
279 if (unlikely(log))
280 *log_num = nlogs;
281 return headcount;
282err:
283 vhost_discard_vq_desc(vq, headcount);
284 return r;
285}
286
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +0000287/* Expects to be always run from workqueue - which acts as
288 * read-size critical section for our kind of RCU. */
David Stevens8dd014a2010-07-27 18:52:21 +0300289static void handle_rx_big(struct vhost_net *net)
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +0000290{
291 struct vhost_virtqueue *vq = &net->dev.vqs[VHOST_NET_VQ_RX];
Michael S. Tsirkind5675bd2010-06-24 16:59:59 +0300292 unsigned out, in, log, s;
293 int head;
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +0000294 struct vhost_log *vq_log;
295 struct msghdr msg = {
296 .msg_name = NULL,
297 .msg_namelen = 0,
298 .msg_control = NULL, /* FIXME: get and handle RX aux data. */
299 .msg_controllen = 0,
300 .msg_iov = vq->iov,
301 .msg_flags = MSG_DONTWAIT,
302 };
303
304 struct virtio_net_hdr hdr = {
305 .flags = 0,
306 .gso_type = VIRTIO_NET_HDR_GSO_NONE
307 };
308
309 size_t len, total_len = 0;
310 int err;
311 size_t hdr_size;
312 struct socket *sock = rcu_dereference(vq->private_data);
313 if (!sock || skb_queue_empty(&sock->sk->sk_receive_queue))
314 return;
315
316 use_mm(net->dev.mm);
317 mutex_lock(&vq->mutex);
318 vhost_disable_notify(vq);
David Stevens8dd014a2010-07-27 18:52:21 +0300319 hdr_size = vq->vhost_hlen;
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +0000320
321 vq_log = unlikely(vhost_has_feature(&net->dev, VHOST_F_LOG_ALL)) ?
322 vq->log : NULL;
323
324 for (;;) {
325 head = vhost_get_vq_desc(&net->dev, vq, vq->iov,
326 ARRAY_SIZE(vq->iov),
327 &out, &in,
328 vq_log, &log);
Michael S. Tsirkind5675bd2010-06-24 16:59:59 +0300329 /* On error, stop handling until the next kick. */
Michael S. Tsirkin7b3384f2010-07-01 18:40:12 +0300330 if (unlikely(head < 0))
Michael S. Tsirkind5675bd2010-06-24 16:59:59 +0300331 break;
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +0000332 /* OK, now we need to know about added descriptors. */
333 if (head == vq->num) {
334 if (unlikely(vhost_enable_notify(vq))) {
335 /* They have slipped one in as we were
336 * doing that: check again. */
337 vhost_disable_notify(vq);
338 continue;
339 }
340 /* Nothing new? Wait for eventfd to tell us
341 * they refilled. */
342 break;
343 }
344 /* We don't need to be notified again. */
345 if (out) {
346 vq_err(vq, "Unexpected descriptor format for RX: "
347 "out %d, int %d\n",
348 out, in);
349 break;
350 }
351 /* Skip header. TODO: support TSO/mergeable rx buffers. */
352 s = move_iovec_hdr(vq->iov, vq->hdr, hdr_size, in);
353 msg.msg_iovlen = in;
354 len = iov_length(vq->iov, in);
355 /* Sanity check */
356 if (!len) {
357 vq_err(vq, "Unexpected header len for RX: "
358 "%zd expected %zd\n",
359 iov_length(vq->hdr, s), hdr_size);
360 break;
361 }
362 err = sock->ops->recvmsg(NULL, sock, &msg,
363 len, MSG_DONTWAIT | MSG_TRUNC);
364 /* TODO: Check specific error and bomb out unless EAGAIN? */
365 if (err < 0) {
David Stevens8dd014a2010-07-27 18:52:21 +0300366 vhost_discard_vq_desc(vq, 1);
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +0000367 break;
368 }
369 /* TODO: Should check and handle checksum. */
370 if (err > len) {
Michael S. Tsirkin95c0ec62010-06-24 17:10:25 +0300371 pr_debug("Discarded truncated rx packet: "
372 " len %d > %zd\n", err, len);
David Stevens8dd014a2010-07-27 18:52:21 +0300373 vhost_discard_vq_desc(vq, 1);
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +0000374 continue;
375 }
376 len = err;
377 err = memcpy_toiovec(vq->hdr, (unsigned char *)&hdr, hdr_size);
378 if (err) {
379 vq_err(vq, "Unable to write vnet_hdr at addr %p: %d\n",
380 vq->iov->iov_base, err);
381 break;
382 }
383 len += hdr_size;
384 vhost_add_used_and_signal(&net->dev, vq, head, len);
385 if (unlikely(vq_log))
386 vhost_log_write(vq, vq_log, log, len);
387 total_len += len;
388 if (unlikely(total_len >= VHOST_NET_WEIGHT)) {
389 vhost_poll_queue(&vq->poll);
390 break;
391 }
392 }
393
394 mutex_unlock(&vq->mutex);
395 unuse_mm(net->dev.mm);
396}
397
David Stevens8dd014a2010-07-27 18:52:21 +0300398/* Expects to be always run from workqueue - which acts as
399 * read-size critical section for our kind of RCU. */
400static void handle_rx_mergeable(struct vhost_net *net)
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +0000401{
David Stevens8dd014a2010-07-27 18:52:21 +0300402 struct vhost_virtqueue *vq = &net->dev.vqs[VHOST_NET_VQ_RX];
403 unsigned uninitialized_var(in), log;
404 struct vhost_log *vq_log;
405 struct msghdr msg = {
406 .msg_name = NULL,
407 .msg_namelen = 0,
408 .msg_control = NULL, /* FIXME: get and handle RX aux data. */
409 .msg_controllen = 0,
410 .msg_iov = vq->iov,
411 .msg_flags = MSG_DONTWAIT,
412 };
413
414 struct virtio_net_hdr_mrg_rxbuf hdr = {
415 .hdr.flags = 0,
416 .hdr.gso_type = VIRTIO_NET_HDR_GSO_NONE
417 };
418
419 size_t total_len = 0;
420 int err, headcount;
421 size_t vhost_hlen, sock_hlen;
422 size_t vhost_len, sock_len;
423 struct socket *sock = rcu_dereference(vq->private_data);
424 if (!sock || skb_queue_empty(&sock->sk->sk_receive_queue))
425 return;
426
427 use_mm(net->dev.mm);
428 mutex_lock(&vq->mutex);
429 vhost_disable_notify(vq);
430 vhost_hlen = vq->vhost_hlen;
431 sock_hlen = vq->sock_hlen;
432
433 vq_log = unlikely(vhost_has_feature(&net->dev, VHOST_F_LOG_ALL)) ?
434 vq->log : NULL;
435
436 while ((sock_len = peek_head_len(sock->sk))) {
437 sock_len += sock_hlen;
438 vhost_len = sock_len + vhost_hlen;
439 headcount = get_rx_bufs(vq, vq->heads, vhost_len,
440 &in, vq_log, &log);
441 /* On error, stop handling until the next kick. */
442 if (unlikely(headcount < 0))
443 break;
444 /* OK, now we need to know about added descriptors. */
445 if (!headcount) {
446 if (unlikely(vhost_enable_notify(vq))) {
447 /* They have slipped one in as we were
448 * doing that: check again. */
449 vhost_disable_notify(vq);
450 continue;
451 }
452 /* Nothing new? Wait for eventfd to tell us
453 * they refilled. */
454 break;
455 }
456 /* We don't need to be notified again. */
457 if (unlikely((vhost_hlen)))
458 /* Skip header. TODO: support TSO. */
459 move_iovec_hdr(vq->iov, vq->hdr, vhost_hlen, in);
460 else
461 /* Copy the header for use in VIRTIO_NET_F_MRG_RXBUF:
462 * needed because sendmsg can modify msg_iov. */
463 copy_iovec_hdr(vq->iov, vq->hdr, sock_hlen, in);
464 msg.msg_iovlen = in;
465 err = sock->ops->recvmsg(NULL, sock, &msg,
466 sock_len, MSG_DONTWAIT | MSG_TRUNC);
467 /* Userspace might have consumed the packet meanwhile:
468 * it's not supposed to do this usually, but might be hard
469 * to prevent. Discard data we got (if any) and keep going. */
470 if (unlikely(err != sock_len)) {
471 pr_debug("Discarded rx packet: "
472 " len %d, expected %zd\n", err, sock_len);
473 vhost_discard_vq_desc(vq, headcount);
474 continue;
475 }
476 if (unlikely(vhost_hlen) &&
477 memcpy_toiovecend(vq->hdr, (unsigned char *)&hdr, 0,
478 vhost_hlen)) {
479 vq_err(vq, "Unable to write vnet_hdr at addr %p\n",
480 vq->iov->iov_base);
481 break;
482 }
483 /* TODO: Should check and handle checksum. */
484 if (vhost_has_feature(&net->dev, VIRTIO_NET_F_MRG_RXBUF) &&
485 memcpy_toiovecend(vq->hdr, (unsigned char *)&headcount,
486 offsetof(typeof(hdr), num_buffers),
487 sizeof hdr.num_buffers)) {
488 vq_err(vq, "Failed num_buffers write");
489 vhost_discard_vq_desc(vq, headcount);
490 break;
491 }
492 vhost_add_used_and_signal_n(&net->dev, vq, vq->heads,
493 headcount);
494 if (unlikely(vq_log))
495 vhost_log_write(vq, vq_log, log, vhost_len);
496 total_len += vhost_len;
497 if (unlikely(total_len >= VHOST_NET_WEIGHT)) {
498 vhost_poll_queue(&vq->poll);
499 break;
500 }
501 }
502
503 mutex_unlock(&vq->mutex);
504 unuse_mm(net->dev.mm);
505}
506
507static void handle_rx(struct vhost_net *net)
508{
509 if (vhost_has_feature(&net->dev, VIRTIO_NET_F_MRG_RXBUF))
510 handle_rx_mergeable(net);
511 else
512 handle_rx_big(net);
513}
514
Tejun Heoc23f3442010-06-02 20:40:00 +0200515static void handle_tx_kick(struct vhost_work *work)
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +0000516{
Tejun Heoc23f3442010-06-02 20:40:00 +0200517 struct vhost_virtqueue *vq = container_of(work, struct vhost_virtqueue,
518 poll.work);
519 struct vhost_net *net = container_of(vq->dev, struct vhost_net, dev);
520
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +0000521 handle_tx(net);
522}
523
Tejun Heoc23f3442010-06-02 20:40:00 +0200524static void handle_rx_kick(struct vhost_work *work)
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +0000525{
Tejun Heoc23f3442010-06-02 20:40:00 +0200526 struct vhost_virtqueue *vq = container_of(work, struct vhost_virtqueue,
527 poll.work);
528 struct vhost_net *net = container_of(vq->dev, struct vhost_net, dev);
529
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +0000530 handle_rx(net);
531}
532
Tejun Heoc23f3442010-06-02 20:40:00 +0200533static void handle_tx_net(struct vhost_work *work)
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +0000534{
Tejun Heoc23f3442010-06-02 20:40:00 +0200535 struct vhost_net *net = container_of(work, struct vhost_net,
536 poll[VHOST_NET_VQ_TX].work);
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +0000537 handle_tx(net);
538}
539
Tejun Heoc23f3442010-06-02 20:40:00 +0200540static void handle_rx_net(struct vhost_work *work)
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +0000541{
Tejun Heoc23f3442010-06-02 20:40:00 +0200542 struct vhost_net *net = container_of(work, struct vhost_net,
543 poll[VHOST_NET_VQ_RX].work);
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +0000544 handle_rx(net);
545}
546
547static int vhost_net_open(struct inode *inode, struct file *f)
548{
549 struct vhost_net *n = kmalloc(sizeof *n, GFP_KERNEL);
Tejun Heoc23f3442010-06-02 20:40:00 +0200550 struct vhost_dev *dev;
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +0000551 int r;
Tejun Heoc23f3442010-06-02 20:40:00 +0200552
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +0000553 if (!n)
554 return -ENOMEM;
Tejun Heoc23f3442010-06-02 20:40:00 +0200555
556 dev = &n->dev;
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +0000557 n->vqs[VHOST_NET_VQ_TX].handle_kick = handle_tx_kick;
558 n->vqs[VHOST_NET_VQ_RX].handle_kick = handle_rx_kick;
Tejun Heoc23f3442010-06-02 20:40:00 +0200559 r = vhost_dev_init(dev, n->vqs, VHOST_NET_VQ_MAX);
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +0000560 if (r < 0) {
561 kfree(n);
562 return r;
563 }
564
Tejun Heoc23f3442010-06-02 20:40:00 +0200565 vhost_poll_init(n->poll + VHOST_NET_VQ_TX, handle_tx_net, POLLOUT, dev);
566 vhost_poll_init(n->poll + VHOST_NET_VQ_RX, handle_rx_net, POLLIN, dev);
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +0000567 n->tx_poll_state = VHOST_NET_POLL_DISABLED;
568
569 f->private_data = n;
570
571 return 0;
572}
573
574static void vhost_net_disable_vq(struct vhost_net *n,
575 struct vhost_virtqueue *vq)
576{
577 if (!vq->private_data)
578 return;
579 if (vq == n->vqs + VHOST_NET_VQ_TX) {
580 tx_poll_stop(n);
581 n->tx_poll_state = VHOST_NET_POLL_DISABLED;
582 } else
583 vhost_poll_stop(n->poll + VHOST_NET_VQ_RX);
584}
585
586static void vhost_net_enable_vq(struct vhost_net *n,
587 struct vhost_virtqueue *vq)
588{
Arnd Bergmann28457ee2010-03-09 19:24:45 +0100589 struct socket *sock;
590
591 sock = rcu_dereference_protected(vq->private_data,
592 lockdep_is_held(&vq->mutex));
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +0000593 if (!sock)
594 return;
595 if (vq == n->vqs + VHOST_NET_VQ_TX) {
596 n->tx_poll_state = VHOST_NET_POLL_STOPPED;
597 tx_poll_start(n, sock);
598 } else
599 vhost_poll_start(n->poll + VHOST_NET_VQ_RX, sock->file);
600}
601
602static struct socket *vhost_net_stop_vq(struct vhost_net *n,
603 struct vhost_virtqueue *vq)
604{
605 struct socket *sock;
606
607 mutex_lock(&vq->mutex);
Arnd Bergmann28457ee2010-03-09 19:24:45 +0100608 sock = rcu_dereference_protected(vq->private_data,
609 lockdep_is_held(&vq->mutex));
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +0000610 vhost_net_disable_vq(n, vq);
611 rcu_assign_pointer(vq->private_data, NULL);
612 mutex_unlock(&vq->mutex);
613 return sock;
614}
615
616static void vhost_net_stop(struct vhost_net *n, struct socket **tx_sock,
617 struct socket **rx_sock)
618{
619 *tx_sock = vhost_net_stop_vq(n, n->vqs + VHOST_NET_VQ_TX);
620 *rx_sock = vhost_net_stop_vq(n, n->vqs + VHOST_NET_VQ_RX);
621}
622
623static void vhost_net_flush_vq(struct vhost_net *n, int index)
624{
625 vhost_poll_flush(n->poll + index);
626 vhost_poll_flush(&n->dev.vqs[index].poll);
627}
628
629static void vhost_net_flush(struct vhost_net *n)
630{
631 vhost_net_flush_vq(n, VHOST_NET_VQ_TX);
632 vhost_net_flush_vq(n, VHOST_NET_VQ_RX);
633}
634
635static int vhost_net_release(struct inode *inode, struct file *f)
636{
637 struct vhost_net *n = f->private_data;
638 struct socket *tx_sock;
639 struct socket *rx_sock;
640
641 vhost_net_stop(n, &tx_sock, &rx_sock);
642 vhost_net_flush(n);
643 vhost_dev_cleanup(&n->dev);
644 if (tx_sock)
645 fput(tx_sock->file);
646 if (rx_sock)
647 fput(rx_sock->file);
648 /* We do an extra flush before freeing memory,
649 * since jobs can re-queue themselves. */
650 vhost_net_flush(n);
651 kfree(n);
652 return 0;
653}
654
655static struct socket *get_raw_socket(int fd)
656{
657 struct {
658 struct sockaddr_ll sa;
659 char buf[MAX_ADDR_LEN];
660 } uaddr;
661 int uaddr_len = sizeof uaddr, r;
662 struct socket *sock = sockfd_lookup(fd, &r);
663 if (!sock)
664 return ERR_PTR(-ENOTSOCK);
665
666 /* Parameter checking */
667 if (sock->sk->sk_type != SOCK_RAW) {
668 r = -ESOCKTNOSUPPORT;
669 goto err;
670 }
671
672 r = sock->ops->getname(sock, (struct sockaddr *)&uaddr.sa,
673 &uaddr_len, 0);
674 if (r)
675 goto err;
676
677 if (uaddr.sa.sll_family != AF_PACKET) {
678 r = -EPFNOSUPPORT;
679 goto err;
680 }
681 return sock;
682err:
683 fput(sock->file);
684 return ERR_PTR(r);
685}
686
Arnd Bergmann501c7742010-02-18 05:46:50 +0000687static struct socket *get_tap_socket(int fd)
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +0000688{
689 struct file *file = fget(fd);
690 struct socket *sock;
691 if (!file)
692 return ERR_PTR(-EBADF);
693 sock = tun_get_socket(file);
Arnd Bergmann501c7742010-02-18 05:46:50 +0000694 if (!IS_ERR(sock))
695 return sock;
696 sock = macvtap_get_socket(file);
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +0000697 if (IS_ERR(sock))
698 fput(file);
699 return sock;
700}
701
702static struct socket *get_socket(int fd)
703{
704 struct socket *sock;
705 /* special case to disable backend */
706 if (fd == -1)
707 return NULL;
708 sock = get_raw_socket(fd);
709 if (!IS_ERR(sock))
710 return sock;
Arnd Bergmann501c7742010-02-18 05:46:50 +0000711 sock = get_tap_socket(fd);
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +0000712 if (!IS_ERR(sock))
713 return sock;
714 return ERR_PTR(-ENOTSOCK);
715}
716
717static long vhost_net_set_backend(struct vhost_net *n, unsigned index, int fd)
718{
719 struct socket *sock, *oldsock;
720 struct vhost_virtqueue *vq;
721 int r;
722
723 mutex_lock(&n->dev.mutex);
724 r = vhost_dev_check_owner(&n->dev);
725 if (r)
726 goto err;
727
728 if (index >= VHOST_NET_VQ_MAX) {
729 r = -ENOBUFS;
730 goto err;
731 }
732 vq = n->vqs + index;
733 mutex_lock(&vq->mutex);
734
735 /* Verify that ring has been setup correctly. */
736 if (!vhost_vq_access_ok(vq)) {
737 r = -EFAULT;
Jeff Dike1dace8c2010-03-04 16:10:14 -0500738 goto err_vq;
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +0000739 }
740 sock = get_socket(fd);
741 if (IS_ERR(sock)) {
742 r = PTR_ERR(sock);
Jeff Dike1dace8c2010-03-04 16:10:14 -0500743 goto err_vq;
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +0000744 }
745
746 /* start polling new socket */
Arnd Bergmann28457ee2010-03-09 19:24:45 +0100747 oldsock = rcu_dereference_protected(vq->private_data,
748 lockdep_is_held(&vq->mutex));
David S. Miller11fe8832010-07-20 18:25:24 -0700749 if (sock != oldsock) {
Jeff Dikedd1f4072010-03-04 16:10:14 -0500750 vhost_net_disable_vq(n, vq);
751 rcu_assign_pointer(vq->private_data, sock);
752 vhost_net_enable_vq(n, vq);
753 }
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +0000754
Michael S. Tsirkin1680e902010-07-15 15:19:12 +0300755 mutex_unlock(&vq->mutex);
756
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +0000757 if (oldsock) {
758 vhost_net_flush_vq(n, index);
759 fput(oldsock->file);
760 }
Jeff Dike1dace8c2010-03-04 16:10:14 -0500761
Michael S. Tsirkin1680e902010-07-15 15:19:12 +0300762 mutex_unlock(&n->dev.mutex);
763 return 0;
764
Jeff Dike1dace8c2010-03-04 16:10:14 -0500765err_vq:
766 mutex_unlock(&vq->mutex);
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +0000767err:
768 mutex_unlock(&n->dev.mutex);
769 return r;
770}
771
772static long vhost_net_reset_owner(struct vhost_net *n)
773{
774 struct socket *tx_sock = NULL;
775 struct socket *rx_sock = NULL;
776 long err;
777 mutex_lock(&n->dev.mutex);
778 err = vhost_dev_check_owner(&n->dev);
779 if (err)
780 goto done;
781 vhost_net_stop(n, &tx_sock, &rx_sock);
782 vhost_net_flush(n);
783 err = vhost_dev_reset_owner(&n->dev);
784done:
785 mutex_unlock(&n->dev.mutex);
786 if (tx_sock)
787 fput(tx_sock->file);
788 if (rx_sock)
789 fput(rx_sock->file);
790 return err;
791}
792
793static int vhost_net_set_features(struct vhost_net *n, u64 features)
794{
David Stevens8dd014a2010-07-27 18:52:21 +0300795 size_t vhost_hlen, sock_hlen, hdr_len;
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +0000796 int i;
David Stevens8dd014a2010-07-27 18:52:21 +0300797
798 hdr_len = (features & (1 << VIRTIO_NET_F_MRG_RXBUF)) ?
799 sizeof(struct virtio_net_hdr_mrg_rxbuf) :
800 sizeof(struct virtio_net_hdr);
801 if (features & (1 << VHOST_NET_F_VIRTIO_NET_HDR)) {
802 /* vhost provides vnet_hdr */
803 vhost_hlen = hdr_len;
804 sock_hlen = 0;
805 } else {
806 /* socket provides vnet_hdr */
807 vhost_hlen = 0;
808 sock_hlen = hdr_len;
809 }
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +0000810 mutex_lock(&n->dev.mutex);
811 if ((features & (1 << VHOST_F_LOG_ALL)) &&
812 !vhost_log_access_ok(&n->dev)) {
813 mutex_unlock(&n->dev.mutex);
814 return -EFAULT;
815 }
816 n->dev.acked_features = features;
817 smp_wmb();
818 for (i = 0; i < VHOST_NET_VQ_MAX; ++i) {
819 mutex_lock(&n->vqs[i].mutex);
David Stevens8dd014a2010-07-27 18:52:21 +0300820 n->vqs[i].vhost_hlen = vhost_hlen;
821 n->vqs[i].sock_hlen = sock_hlen;
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +0000822 mutex_unlock(&n->vqs[i].mutex);
823 }
824 vhost_net_flush(n);
825 mutex_unlock(&n->dev.mutex);
826 return 0;
827}
828
829static long vhost_net_ioctl(struct file *f, unsigned int ioctl,
830 unsigned long arg)
831{
832 struct vhost_net *n = f->private_data;
833 void __user *argp = (void __user *)arg;
834 u64 __user *featurep = argp;
835 struct vhost_vring_file backend;
836 u64 features;
837 int r;
838 switch (ioctl) {
839 case VHOST_NET_SET_BACKEND:
Takuya Yoshikawad3553a52010-05-27 19:01:58 +0900840 if (copy_from_user(&backend, argp, sizeof backend))
841 return -EFAULT;
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +0000842 return vhost_net_set_backend(n, backend.index, backend.fd);
843 case VHOST_GET_FEATURES:
844 features = VHOST_FEATURES;
Takuya Yoshikawad3553a52010-05-27 19:01:58 +0900845 if (copy_to_user(featurep, &features, sizeof features))
846 return -EFAULT;
847 return 0;
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +0000848 case VHOST_SET_FEATURES:
Takuya Yoshikawad3553a52010-05-27 19:01:58 +0900849 if (copy_from_user(&features, featurep, sizeof features))
850 return -EFAULT;
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +0000851 if (features & ~VHOST_FEATURES)
852 return -EOPNOTSUPP;
853 return vhost_net_set_features(n, features);
854 case VHOST_RESET_OWNER:
855 return vhost_net_reset_owner(n);
856 default:
857 mutex_lock(&n->dev.mutex);
858 r = vhost_dev_ioctl(&n->dev, ioctl, arg);
859 vhost_net_flush(n);
860 mutex_unlock(&n->dev.mutex);
861 return r;
862 }
863}
864
865#ifdef CONFIG_COMPAT
866static long vhost_net_compat_ioctl(struct file *f, unsigned int ioctl,
867 unsigned long arg)
868{
869 return vhost_net_ioctl(f, ioctl, (unsigned long)compat_ptr(arg));
870}
871#endif
872
Tobias Klauser373a83a2010-05-17 15:12:49 +0200873static const struct file_operations vhost_net_fops = {
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +0000874 .owner = THIS_MODULE,
875 .release = vhost_net_release,
876 .unlocked_ioctl = vhost_net_ioctl,
877#ifdef CONFIG_COMPAT
878 .compat_ioctl = vhost_net_compat_ioctl,
879#endif
880 .open = vhost_net_open,
Arnd Bergmann6038f372010-08-15 18:52:59 +0200881 .llseek = noop_llseek,
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +0000882};
883
884static struct miscdevice vhost_net_misc = {
Alan Cox79907d82010-06-09 09:39:49 +0100885 MISC_DYNAMIC_MINOR,
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +0000886 "vhost-net",
887 &vhost_net_fops,
888};
889
Christoph Hellwiga8d37822010-04-13 14:11:25 -0400890static int vhost_net_init(void)
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +0000891{
Tejun Heoc23f3442010-06-02 20:40:00 +0200892 return misc_register(&vhost_net_misc);
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +0000893}
894module_init(vhost_net_init);
895
Christoph Hellwiga8d37822010-04-13 14:11:25 -0400896static void vhost_net_exit(void)
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +0000897{
898 misc_deregister(&vhost_net_misc);
Michael S. Tsirkin3a4d5c92010-01-14 06:17:27 +0000899}
900module_exit(vhost_net_exit);
901
902MODULE_VERSION("0.0.1");
903MODULE_LICENSE("GPL v2");
904MODULE_AUTHOR("Michael S. Tsirkin");
905MODULE_DESCRIPTION("Host kernel accelerator for virtio net");