blob: 59e9605de316809a702951a5c6de3dbee3a43a69 [file] [log] [blame]
Arnd Bergmann20d29d72010-01-30 12:24:26 +00001#include <linux/etherdevice.h>
2#include <linux/if_macvlan.h>
Basil Gorf09e2242012-05-03 22:55:24 +00003#include <linux/if_vlan.h>
Arnd Bergmann20d29d72010-01-30 12:24:26 +00004#include <linux/interrupt.h>
5#include <linux/nsproxy.h>
6#include <linux/compat.h>
7#include <linux/if_tun.h>
8#include <linux/module.h>
9#include <linux/skbuff.h>
10#include <linux/cache.h>
11#include <linux/sched.h>
12#include <linux/types.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090013#include <linux/slab.h>
Arnd Bergmann20d29d72010-01-30 12:24:26 +000014#include <linux/init.h>
15#include <linux/wait.h>
16#include <linux/cdev.h>
Al Viro40401532012-02-13 03:58:52 +000017#include <linux/idr.h>
Arnd Bergmann20d29d72010-01-30 12:24:26 +000018#include <linux/fs.h>
19
20#include <net/net_namespace.h>
21#include <net/rtnetlink.h>
22#include <net/sock.h>
Arnd Bergmannb9fb9ee2010-02-18 05:48:17 +000023#include <linux/virtio_net.h>
Arnd Bergmann20d29d72010-01-30 12:24:26 +000024
25/*
26 * A macvtap queue is the central object of this driver, it connects
27 * an open character device to a macvlan interface. There can be
28 * multiple queues on one interface, which map back to queues
29 * implemented in hardware on the underlying device.
30 *
31 * macvtap_proto is used to allocate queues through the sock allocation
32 * mechanism.
33 *
34 * TODO: multiqueue support is currently not implemented, even though
35 * macvtap is basically prepared for that. We will need to add this
36 * here as well as in virtio-net and qemu to get line rate on 10gbit
37 * adapters from a guest.
38 */
39struct macvtap_queue {
40 struct sock sk;
41 struct socket sock;
Eric Dumazet43815482010-04-29 11:01:49 +000042 struct socket_wq wq;
Michael S. Tsirkin55afbd02010-04-29 13:50:48 +030043 int vnet_hdr_sz;
Eric Dumazet13707f92011-01-26 19:28:23 +000044 struct macvlan_dev __rcu *vlan;
Arnd Bergmann20d29d72010-01-30 12:24:26 +000045 struct file *file;
Arnd Bergmannb9fb9ee2010-02-18 05:48:17 +000046 unsigned int flags;
Arnd Bergmann20d29d72010-01-30 12:24:26 +000047};
48
49static struct proto macvtap_proto = {
50 .name = "macvtap",
51 .owner = THIS_MODULE,
52 .obj_size = sizeof (struct macvtap_queue),
53};
54
55/*
Eric W. Biedermane09eff72011-10-20 04:29:24 +000056 * Variables for dealing with macvtaps device numbers.
Arnd Bergmann20d29d72010-01-30 12:24:26 +000057 */
David S. Miller1ebed712010-07-10 19:25:50 -070058static dev_t macvtap_major;
Eric W. Biedermane09eff72011-10-20 04:29:24 +000059#define MACVTAP_NUM_DEVS (1U << MINORBITS)
60static DEFINE_MUTEX(minor_lock);
61static DEFINE_IDR(minor_idr);
62
Shirley Ma97bc3632011-07-06 12:26:11 +000063#define GOODCOPY_LEN 128
Arnd Bergmann20d29d72010-01-30 12:24:26 +000064static struct class *macvtap_class;
65static struct cdev macvtap_cdev;
66
Arnd Bergmann501c7742010-02-18 05:46:50 +000067static const struct proto_ops macvtap_socket_ops;
68
Arnd Bergmann20d29d72010-01-30 12:24:26 +000069/*
70 * RCU usage:
Arnd Bergmann02df55d2010-02-18 05:45:36 +000071 * The macvtap_queue and the macvlan_dev are loosely coupled, the
72 * pointers from one to the other can only be read while rcu_read_lock
73 * or macvtap_lock is held.
Arnd Bergmann20d29d72010-01-30 12:24:26 +000074 *
Arnd Bergmann02df55d2010-02-18 05:45:36 +000075 * Both the file and the macvlan_dev hold a reference on the macvtap_queue
76 * through sock_hold(&q->sk). When the macvlan_dev goes away first,
77 * q->vlan becomes inaccessible. When the files gets closed,
78 * macvtap_get_queue() fails.
Arnd Bergmann20d29d72010-01-30 12:24:26 +000079 *
Arnd Bergmann02df55d2010-02-18 05:45:36 +000080 * There may still be references to the struct sock inside of the
81 * queue from outbound SKBs, but these never reference back to the
82 * file or the dev. The data structure is freed through __sk_free
83 * when both our references and any pending SKBs are gone.
Arnd Bergmann20d29d72010-01-30 12:24:26 +000084 */
85static DEFINE_SPINLOCK(macvtap_lock);
86
87/*
Krishna Kumar1565c7c2010-08-04 06:15:59 +000088 * get_slot: return a [unused/occupied] slot in vlan->taps[]:
89 * - if 'q' is NULL, return the first empty slot;
90 * - otherwise, return the slot this pointer occupies.
Arnd Bergmann20d29d72010-01-30 12:24:26 +000091 */
Krishna Kumar1565c7c2010-08-04 06:15:59 +000092static int get_slot(struct macvlan_dev *vlan, struct macvtap_queue *q)
93{
94 int i;
95
96 for (i = 0; i < MAX_MACVTAP_QUEUES; i++) {
Denis Efremov3a7f8c32012-08-11 20:26:31 +000097 if (rcu_dereference_protected(vlan->taps[i],
98 lockdep_is_held(&macvtap_lock)) == q)
Krishna Kumar1565c7c2010-08-04 06:15:59 +000099 return i;
100 }
101
102 /* Should never happen */
103 BUG_ON(1);
104}
105
Arnd Bergmann20d29d72010-01-30 12:24:26 +0000106static int macvtap_set_queue(struct net_device *dev, struct file *file,
107 struct macvtap_queue *q)
108{
109 struct macvlan_dev *vlan = netdev_priv(dev);
Krishna Kumar1565c7c2010-08-04 06:15:59 +0000110 int index;
Arnd Bergmann20d29d72010-01-30 12:24:26 +0000111 int err = -EBUSY;
112
113 spin_lock(&macvtap_lock);
Krishna Kumar1565c7c2010-08-04 06:15:59 +0000114 if (vlan->numvtaps == MAX_MACVTAP_QUEUES)
Arnd Bergmann20d29d72010-01-30 12:24:26 +0000115 goto out;
116
117 err = 0;
Krishna Kumar1565c7c2010-08-04 06:15:59 +0000118 index = get_slot(vlan, NULL);
Arnd Bergmann02df55d2010-02-18 05:45:36 +0000119 rcu_assign_pointer(q->vlan, vlan);
Krishna Kumar1565c7c2010-08-04 06:15:59 +0000120 rcu_assign_pointer(vlan->taps[index], q);
Arnd Bergmann02df55d2010-02-18 05:45:36 +0000121 sock_hold(&q->sk);
Arnd Bergmann20d29d72010-01-30 12:24:26 +0000122
123 q->file = file;
Arnd Bergmann02df55d2010-02-18 05:45:36 +0000124 file->private_data = q;
Arnd Bergmann20d29d72010-01-30 12:24:26 +0000125
Krishna Kumar1565c7c2010-08-04 06:15:59 +0000126 vlan->numvtaps++;
127
Arnd Bergmann20d29d72010-01-30 12:24:26 +0000128out:
129 spin_unlock(&macvtap_lock);
130 return err;
131}
132
133/*
Arnd Bergmann02df55d2010-02-18 05:45:36 +0000134 * The file owning the queue got closed, give up both
135 * the reference that the files holds as well as the
136 * one from the macvlan_dev if that still exists.
Arnd Bergmann20d29d72010-01-30 12:24:26 +0000137 *
138 * Using the spinlock makes sure that we don't get
139 * to the queue again after destroying it.
Arnd Bergmann20d29d72010-01-30 12:24:26 +0000140 */
Arnd Bergmann02df55d2010-02-18 05:45:36 +0000141static void macvtap_put_queue(struct macvtap_queue *q)
Arnd Bergmann20d29d72010-01-30 12:24:26 +0000142{
Arnd Bergmann02df55d2010-02-18 05:45:36 +0000143 struct macvlan_dev *vlan;
Arnd Bergmann20d29d72010-01-30 12:24:26 +0000144
145 spin_lock(&macvtap_lock);
Eric Dumazet13707f92011-01-26 19:28:23 +0000146 vlan = rcu_dereference_protected(q->vlan,
147 lockdep_is_held(&macvtap_lock));
Arnd Bergmann02df55d2010-02-18 05:45:36 +0000148 if (vlan) {
Krishna Kumar1565c7c2010-08-04 06:15:59 +0000149 int index = get_slot(vlan, q);
150
Eric Dumazet2cfa5a02011-11-23 07:09:32 +0000151 RCU_INIT_POINTER(vlan->taps[index], NULL);
152 RCU_INIT_POINTER(q->vlan, NULL);
Arnd Bergmann02df55d2010-02-18 05:45:36 +0000153 sock_put(&q->sk);
Krishna Kumar1565c7c2010-08-04 06:15:59 +0000154 --vlan->numvtaps;
Arnd Bergmann20d29d72010-01-30 12:24:26 +0000155 }
156
Arnd Bergmann20d29d72010-01-30 12:24:26 +0000157 spin_unlock(&macvtap_lock);
158
159 synchronize_rcu();
160 sock_put(&q->sk);
161}
162
163/*
Krishna Kumar1565c7c2010-08-04 06:15:59 +0000164 * Select a queue based on the rxq of the device on which this packet
165 * arrived. If the incoming device is not mq, calculate a flow hash
166 * to select a queue. If all fails, find the first available queue.
167 * Cache vlan->numvtaps since it can become zero during the execution
168 * of this function.
Arnd Bergmann20d29d72010-01-30 12:24:26 +0000169 */
170static struct macvtap_queue *macvtap_get_queue(struct net_device *dev,
171 struct sk_buff *skb)
172{
173 struct macvlan_dev *vlan = netdev_priv(dev);
Krishna Kumar1565c7c2010-08-04 06:15:59 +0000174 struct macvtap_queue *tap = NULL;
175 int numvtaps = vlan->numvtaps;
176 __u32 rxq;
Arnd Bergmann20d29d72010-01-30 12:24:26 +0000177
Krishna Kumar1565c7c2010-08-04 06:15:59 +0000178 if (!numvtaps)
179 goto out;
180
Krishna Kumaref0002b2011-11-23 22:17:14 +0000181 /* Check if we can use flow to select a queue */
182 rxq = skb_get_rxhash(skb);
183 if (rxq) {
184 tap = rcu_dereference(vlan->taps[rxq % numvtaps]);
185 if (tap)
186 goto out;
187 }
188
Krishna Kumar1565c7c2010-08-04 06:15:59 +0000189 if (likely(skb_rx_queue_recorded(skb))) {
190 rxq = skb_get_rx_queue(skb);
191
192 while (unlikely(rxq >= numvtaps))
193 rxq -= numvtaps;
194
195 tap = rcu_dereference(vlan->taps[rxq]);
196 if (tap)
197 goto out;
198 }
199
Krishna Kumar1565c7c2010-08-04 06:15:59 +0000200 /* Everything failed - find first available queue */
201 for (rxq = 0; rxq < MAX_MACVTAP_QUEUES; rxq++) {
202 tap = rcu_dereference(vlan->taps[rxq]);
203 if (tap)
204 break;
205 }
206
207out:
208 return tap;
Arnd Bergmann20d29d72010-01-30 12:24:26 +0000209}
210
Arnd Bergmann02df55d2010-02-18 05:45:36 +0000211/*
212 * The net_device is going away, give up the reference
Krishna Kumar1565c7c2010-08-04 06:15:59 +0000213 * that it holds on all queues and safely set the pointer
214 * from the queues to NULL.
Arnd Bergmann02df55d2010-02-18 05:45:36 +0000215 */
Arnd Bergmann20d29d72010-01-30 12:24:26 +0000216static void macvtap_del_queues(struct net_device *dev)
217{
218 struct macvlan_dev *vlan = netdev_priv(dev);
Krishna Kumar1565c7c2010-08-04 06:15:59 +0000219 struct macvtap_queue *q, *qlist[MAX_MACVTAP_QUEUES];
220 int i, j = 0;
Arnd Bergmann20d29d72010-01-30 12:24:26 +0000221
Krishna Kumar1565c7c2010-08-04 06:15:59 +0000222 /* macvtap_put_queue can free some slots, so go through all slots */
Arnd Bergmann02df55d2010-02-18 05:45:36 +0000223 spin_lock(&macvtap_lock);
Krishna Kumar1565c7c2010-08-04 06:15:59 +0000224 for (i = 0; i < MAX_MACVTAP_QUEUES && vlan->numvtaps; i++) {
Eric Dumazet13707f92011-01-26 19:28:23 +0000225 q = rcu_dereference_protected(vlan->taps[i],
226 lockdep_is_held(&macvtap_lock));
Krishna Kumar1565c7c2010-08-04 06:15:59 +0000227 if (q) {
228 qlist[j++] = q;
Eric Dumazet2cfa5a02011-11-23 07:09:32 +0000229 RCU_INIT_POINTER(vlan->taps[i], NULL);
230 RCU_INIT_POINTER(q->vlan, NULL);
Krishna Kumar1565c7c2010-08-04 06:15:59 +0000231 vlan->numvtaps--;
232 }
Arnd Bergmann02df55d2010-02-18 05:45:36 +0000233 }
Krishna Kumar1565c7c2010-08-04 06:15:59 +0000234 BUG_ON(vlan->numvtaps != 0);
Eric W. Biederman99f34b32011-10-20 04:26:01 +0000235 /* guarantee that any future macvtap_set_queue will fail */
236 vlan->numvtaps = MAX_MACVTAP_QUEUES;
Arnd Bergmann02df55d2010-02-18 05:45:36 +0000237 spin_unlock(&macvtap_lock);
238
239 synchronize_rcu();
Krishna Kumar1565c7c2010-08-04 06:15:59 +0000240
241 for (--j; j >= 0; j--)
242 sock_put(&qlist[j]->sk);
Arnd Bergmann20d29d72010-01-30 12:24:26 +0000243}
244
245/*
246 * Forward happens for data that gets sent from one macvlan
247 * endpoint to another one in bridge mode. We just take
248 * the skb and put it into the receive queue.
249 */
250static int macvtap_forward(struct net_device *dev, struct sk_buff *skb)
251{
252 struct macvtap_queue *q = macvtap_get_queue(dev, skb);
253 if (!q)
Herbert Xu8a357472010-07-21 21:44:31 +0000254 goto drop;
255
256 if (skb_queue_len(&q->sk.sk_receive_queue) >= dev->tx_queue_len)
257 goto drop;
Arnd Bergmann20d29d72010-01-30 12:24:26 +0000258
259 skb_queue_tail(&q->sk.sk_receive_queue, skb);
Eric Dumazet4a4771a2010-04-25 22:20:06 +0000260 wake_up_interruptible_poll(sk_sleep(&q->sk), POLLIN | POLLRDNORM | POLLRDBAND);
Herbert Xu8a357472010-07-21 21:44:31 +0000261 return NET_RX_SUCCESS;
262
263drop:
264 kfree_skb(skb);
265 return NET_RX_DROP;
Arnd Bergmann20d29d72010-01-30 12:24:26 +0000266}
267
268/*
269 * Receive is for data from the external interface (lowerdev),
270 * in case of macvtap, we can treat that the same way as
271 * forward, which macvlan cannot.
272 */
273static int macvtap_receive(struct sk_buff *skb)
274{
275 skb_push(skb, ETH_HLEN);
276 return macvtap_forward(skb->dev, skb);
277}
278
Eric W. Biedermane09eff72011-10-20 04:29:24 +0000279static int macvtap_get_minor(struct macvlan_dev *vlan)
280{
281 int retval = -ENOMEM;
Eric W. Biedermane09eff72011-10-20 04:29:24 +0000282
283 mutex_lock(&minor_lock);
Tejun Heoec09ebc2013-02-27 17:04:34 -0800284 retval = idr_alloc(&minor_idr, vlan, 1, MACVTAP_NUM_DEVS, GFP_KERNEL);
285 if (retval >= 0) {
286 vlan->minor = retval;
287 } else if (retval == -ENOSPC) {
Eric W. Biedermane09eff72011-10-20 04:29:24 +0000288 printk(KERN_ERR "too many macvtap devices\n");
289 retval = -EINVAL;
Eric W. Biedermane09eff72011-10-20 04:29:24 +0000290 }
Eric W. Biedermane09eff72011-10-20 04:29:24 +0000291 mutex_unlock(&minor_lock);
Tejun Heoec09ebc2013-02-27 17:04:34 -0800292 return retval < 0 ? retval : 0;
Eric W. Biedermane09eff72011-10-20 04:29:24 +0000293}
294
295static void macvtap_free_minor(struct macvlan_dev *vlan)
296{
297 mutex_lock(&minor_lock);
298 if (vlan->minor) {
299 idr_remove(&minor_idr, vlan->minor);
300 vlan->minor = 0;
301 }
302 mutex_unlock(&minor_lock);
303}
304
305static struct net_device *dev_get_by_macvtap_minor(int minor)
306{
307 struct net_device *dev = NULL;
308 struct macvlan_dev *vlan;
309
310 mutex_lock(&minor_lock);
311 vlan = idr_find(&minor_idr, minor);
312 if (vlan) {
313 dev = vlan->dev;
314 dev_hold(dev);
315 }
316 mutex_unlock(&minor_lock);
317 return dev;
318}
319
Arnd Bergmann20d29d72010-01-30 12:24:26 +0000320static int macvtap_newlink(struct net *src_net,
321 struct net_device *dev,
322 struct nlattr *tb[],
323 struct nlattr *data[])
324{
Eric W. Biederman9bf19072011-10-20 04:28:46 +0000325 /* Don't put anything that may fail after macvlan_common_newlink
326 * because we can't undo what it does.
327 */
328 return macvlan_common_newlink(src_net, dev, tb, data,
329 macvtap_receive, macvtap_forward);
Arnd Bergmann20d29d72010-01-30 12:24:26 +0000330}
331
332static void macvtap_dellink(struct net_device *dev,
333 struct list_head *head)
334{
Arnd Bergmann20d29d72010-01-30 12:24:26 +0000335 macvtap_del_queues(dev);
336 macvlan_dellink(dev, head);
337}
338
Herbert Xu8a357472010-07-21 21:44:31 +0000339static void macvtap_setup(struct net_device *dev)
340{
341 macvlan_common_setup(dev);
342 dev->tx_queue_len = TUN_READQ_SIZE;
343}
344
Arnd Bergmann20d29d72010-01-30 12:24:26 +0000345static struct rtnl_link_ops macvtap_link_ops __read_mostly = {
346 .kind = "macvtap",
Herbert Xu8a357472010-07-21 21:44:31 +0000347 .setup = macvtap_setup,
Arnd Bergmann20d29d72010-01-30 12:24:26 +0000348 .newlink = macvtap_newlink,
349 .dellink = macvtap_dellink,
350};
351
352
353static void macvtap_sock_write_space(struct sock *sk)
354{
Eric Dumazet43815482010-04-29 11:01:49 +0000355 wait_queue_head_t *wqueue;
356
Arnd Bergmann20d29d72010-01-30 12:24:26 +0000357 if (!sock_writeable(sk) ||
358 !test_and_clear_bit(SOCK_ASYNC_NOSPACE, &sk->sk_socket->flags))
359 return;
360
Eric Dumazet43815482010-04-29 11:01:49 +0000361 wqueue = sk_sleep(sk);
362 if (wqueue && waitqueue_active(wqueue))
363 wake_up_interruptible_poll(wqueue, POLLOUT | POLLWRNORM | POLLWRBAND);
Arnd Bergmann20d29d72010-01-30 12:24:26 +0000364}
365
Eric W. Biederman2259fef2011-10-20 04:27:24 +0000366static void macvtap_sock_destruct(struct sock *sk)
367{
368 skb_queue_purge(&sk->sk_receive_queue);
369}
370
Arnd Bergmann20d29d72010-01-30 12:24:26 +0000371static int macvtap_open(struct inode *inode, struct file *file)
372{
373 struct net *net = current->nsproxy->net_ns;
Eric W. Biedermane09eff72011-10-20 04:29:24 +0000374 struct net_device *dev = dev_get_by_macvtap_minor(iminor(inode));
Arnd Bergmann20d29d72010-01-30 12:24:26 +0000375 struct macvtap_queue *q;
376 int err;
377
378 err = -ENODEV;
379 if (!dev)
380 goto out;
381
Arnd Bergmann20d29d72010-01-30 12:24:26 +0000382 err = -ENOMEM;
383 q = (struct macvtap_queue *)sk_alloc(net, AF_UNSPEC, GFP_KERNEL,
384 &macvtap_proto);
385 if (!q)
386 goto out;
387
Eric Dumazet43815482010-04-29 11:01:49 +0000388 q->sock.wq = &q->wq;
389 init_waitqueue_head(&q->wq.wait);
Arnd Bergmann20d29d72010-01-30 12:24:26 +0000390 q->sock.type = SOCK_RAW;
391 q->sock.state = SS_CONNECTED;
Arnd Bergmann501c7742010-02-18 05:46:50 +0000392 q->sock.file = file;
393 q->sock.ops = &macvtap_socket_ops;
Arnd Bergmann20d29d72010-01-30 12:24:26 +0000394 sock_init_data(&q->sock, &q->sk);
Arnd Bergmann20d29d72010-01-30 12:24:26 +0000395 q->sk.sk_write_space = macvtap_sock_write_space;
Eric W. Biederman2259fef2011-10-20 04:27:24 +0000396 q->sk.sk_destruct = macvtap_sock_destruct;
Arnd Bergmannb9fb9ee2010-02-18 05:48:17 +0000397 q->flags = IFF_VNET_HDR | IFF_NO_PI | IFF_TAP;
Michael S. Tsirkin55afbd02010-04-29 13:50:48 +0300398 q->vnet_hdr_sz = sizeof(struct virtio_net_hdr);
Arnd Bergmann20d29d72010-01-30 12:24:26 +0000399
Shirley Ma97bc3632011-07-06 12:26:11 +0000400 /*
401 * so far only KVM virtio_net uses macvtap, enable zero copy between
402 * guest kernel and host kernel when lower device supports zerocopy
Eric W. Biederman047af9cf2011-10-20 04:26:39 +0000403 *
404 * The macvlan supports zerocopy iff the lower device supports zero
405 * copy so we don't have to look at the lower device directly.
Shirley Ma97bc3632011-07-06 12:26:11 +0000406 */
Eric W. Biederman047af9cf2011-10-20 04:26:39 +0000407 if ((dev->features & NETIF_F_HIGHDMA) && (dev->features & NETIF_F_SG))
408 sock_set_flag(&q->sk, SOCK_ZEROCOPY);
Shirley Ma97bc3632011-07-06 12:26:11 +0000409
Arnd Bergmann20d29d72010-01-30 12:24:26 +0000410 err = macvtap_set_queue(dev, file, q);
411 if (err)
412 sock_put(&q->sk);
413
414out:
415 if (dev)
416 dev_put(dev);
417
418 return err;
419}
420
421static int macvtap_release(struct inode *inode, struct file *file)
422{
Arnd Bergmann02df55d2010-02-18 05:45:36 +0000423 struct macvtap_queue *q = file->private_data;
424 macvtap_put_queue(q);
Arnd Bergmann20d29d72010-01-30 12:24:26 +0000425 return 0;
426}
427
428static unsigned int macvtap_poll(struct file *file, poll_table * wait)
429{
Arnd Bergmann02df55d2010-02-18 05:45:36 +0000430 struct macvtap_queue *q = file->private_data;
Arnd Bergmann20d29d72010-01-30 12:24:26 +0000431 unsigned int mask = POLLERR;
432
433 if (!q)
434 goto out;
435
436 mask = 0;
Eric Dumazet43815482010-04-29 11:01:49 +0000437 poll_wait(file, &q->wq.wait, wait);
Arnd Bergmann20d29d72010-01-30 12:24:26 +0000438
439 if (!skb_queue_empty(&q->sk.sk_receive_queue))
440 mask |= POLLIN | POLLRDNORM;
441
442 if (sock_writeable(&q->sk) ||
443 (!test_and_set_bit(SOCK_ASYNC_NOSPACE, &q->sock.flags) &&
444 sock_writeable(&q->sk)))
445 mask |= POLLOUT | POLLWRNORM;
446
447out:
Arnd Bergmann20d29d72010-01-30 12:24:26 +0000448 return mask;
449}
450
Arnd Bergmannb9fb9ee2010-02-18 05:48:17 +0000451static inline struct sk_buff *macvtap_alloc_skb(struct sock *sk, size_t prepad,
452 size_t len, size_t linear,
453 int noblock, int *err)
454{
455 struct sk_buff *skb;
456
457 /* Under a page? Don't bother with paged skb. */
458 if (prepad + len < PAGE_SIZE || !linear)
459 linear = len;
460
461 skb = sock_alloc_send_pskb(sk, prepad + linear, len - linear, noblock,
462 err);
463 if (!skb)
464 return NULL;
465
466 skb_reserve(skb, prepad);
467 skb_put(skb, linear);
468 skb->data_len = len - linear;
469 skb->len += len - linear;
470
471 return skb;
472}
473
Shirley Ma97bc3632011-07-06 12:26:11 +0000474/* set skb frags from iovec, this can move to core network code for reuse */
475static int zerocopy_sg_from_iovec(struct sk_buff *skb, const struct iovec *from,
476 int offset, size_t count)
477{
478 int len = iov_length(from, count) - offset;
479 int copy = skb_headlen(skb);
480 int size, offset1 = 0;
481 int i = 0;
Shirley Ma97bc3632011-07-06 12:26:11 +0000482
483 /* Skip over from offset */
484 while (count && (offset >= from->iov_len)) {
485 offset -= from->iov_len;
486 ++from;
487 --count;
488 }
489
490 /* copy up to skb headlen */
491 while (count && (copy > 0)) {
492 size = min_t(unsigned int, copy, from->iov_len - offset);
493 if (copy_from_user(skb->data + offset1, from->iov_base + offset,
494 size))
495 return -EFAULT;
496 if (copy > size) {
497 ++from;
498 --count;
Jason Wang3afc9622012-05-02 11:41:30 +0800499 offset = 0;
500 } else
501 offset += size;
Shirley Ma97bc3632011-07-06 12:26:11 +0000502 copy -= size;
503 offset1 += size;
Shirley Ma97bc3632011-07-06 12:26:11 +0000504 }
505
506 if (len == offset1)
507 return 0;
508
509 while (count--) {
510 struct page *page[MAX_SKB_FRAGS];
511 int num_pages;
512 unsigned long base;
Jason Wang4ef67eb2012-05-02 11:41:44 +0800513 unsigned long truesize;
Shirley Ma97bc3632011-07-06 12:26:11 +0000514
Jason Wang3afc9622012-05-02 11:41:30 +0800515 len = from->iov_len - offset;
Shirley Ma97bc3632011-07-06 12:26:11 +0000516 if (!len) {
Jason Wang3afc9622012-05-02 11:41:30 +0800517 offset = 0;
Shirley Ma97bc3632011-07-06 12:26:11 +0000518 ++from;
519 continue;
520 }
Jason Wang3afc9622012-05-02 11:41:30 +0800521 base = (unsigned long)from->iov_base + offset;
Shirley Ma97bc3632011-07-06 12:26:11 +0000522 size = ((base & ~PAGE_MASK) + len + ~PAGE_MASK) >> PAGE_SHIFT;
Jason Wangb92946e2012-05-02 11:42:15 +0800523 if (i + size > MAX_SKB_FRAGS)
524 return -EMSGSIZE;
Shirley Ma97bc3632011-07-06 12:26:11 +0000525 num_pages = get_user_pages_fast(base, size, 0, &page[i]);
Jason Wangb92946e2012-05-02 11:42:15 +0800526 if (num_pages != size) {
Jason Wang02ce04b2012-05-02 11:41:58 +0800527 for (i = 0; i < num_pages; i++)
528 put_page(page[i]);
Shirley Ma97bc3632011-07-06 12:26:11 +0000529 return -EFAULT;
Jason Wang02ce04b2012-05-02 11:41:58 +0800530 }
Jason Wang4ef67eb2012-05-02 11:41:44 +0800531 truesize = size * PAGE_SIZE;
Shirley Ma97bc3632011-07-06 12:26:11 +0000532 skb->data_len += len;
533 skb->len += len;
Jason Wang4ef67eb2012-05-02 11:41:44 +0800534 skb->truesize += truesize;
535 atomic_add(truesize, &skb->sk->sk_wmem_alloc);
Shirley Ma97bc3632011-07-06 12:26:11 +0000536 while (len) {
Jason Wang653fc912011-09-18 23:48:31 +0000537 int off = base & ~PAGE_MASK;
538 int size = min_t(int, len, PAGE_SIZE - off);
539 __skb_fill_page_desc(skb, i, page[i], off, size);
Shirley Ma97bc3632011-07-06 12:26:11 +0000540 skb_shinfo(skb)->nr_frags++;
541 /* increase sk_wmem_alloc */
Jason Wang653fc912011-09-18 23:48:31 +0000542 base += size;
543 len -= size;
Shirley Ma97bc3632011-07-06 12:26:11 +0000544 i++;
545 }
Jason Wang3afc9622012-05-02 11:41:30 +0800546 offset = 0;
Shirley Ma97bc3632011-07-06 12:26:11 +0000547 ++from;
548 }
549 return 0;
550}
551
Arnd Bergmannb9fb9ee2010-02-18 05:48:17 +0000552/*
553 * macvtap_skb_from_vnet_hdr and macvtap_skb_to_vnet_hdr should
554 * be shared with the tun/tap driver.
555 */
556static int macvtap_skb_from_vnet_hdr(struct sk_buff *skb,
557 struct virtio_net_hdr *vnet_hdr)
558{
559 unsigned short gso_type = 0;
560 if (vnet_hdr->gso_type != VIRTIO_NET_HDR_GSO_NONE) {
561 switch (vnet_hdr->gso_type & ~VIRTIO_NET_HDR_GSO_ECN) {
562 case VIRTIO_NET_HDR_GSO_TCPV4:
563 gso_type = SKB_GSO_TCPV4;
564 break;
565 case VIRTIO_NET_HDR_GSO_TCPV6:
566 gso_type = SKB_GSO_TCPV6;
567 break;
568 case VIRTIO_NET_HDR_GSO_UDP:
569 gso_type = SKB_GSO_UDP;
570 break;
571 default:
572 return -EINVAL;
573 }
574
575 if (vnet_hdr->gso_type & VIRTIO_NET_HDR_GSO_ECN)
576 gso_type |= SKB_GSO_TCP_ECN;
577
578 if (vnet_hdr->gso_size == 0)
579 return -EINVAL;
580 }
581
582 if (vnet_hdr->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) {
583 if (!skb_partial_csum_set(skb, vnet_hdr->csum_start,
584 vnet_hdr->csum_offset))
585 return -EINVAL;
586 }
587
588 if (vnet_hdr->gso_type != VIRTIO_NET_HDR_GSO_NONE) {
589 skb_shinfo(skb)->gso_size = vnet_hdr->gso_size;
Pravin B Shelarc9af6db2013-02-11 09:27:41 +0000590 skb_shinfo(skb)->gso_type = gso_type;
Arnd Bergmannb9fb9ee2010-02-18 05:48:17 +0000591
592 /* Header must be checked, and gso_segs computed. */
593 skb_shinfo(skb)->gso_type |= SKB_GSO_DODGY;
594 skb_shinfo(skb)->gso_segs = 0;
595 }
596 return 0;
597}
598
599static int macvtap_skb_to_vnet_hdr(const struct sk_buff *skb,
600 struct virtio_net_hdr *vnet_hdr)
601{
602 memset(vnet_hdr, 0, sizeof(*vnet_hdr));
603
604 if (skb_is_gso(skb)) {
605 struct skb_shared_info *sinfo = skb_shinfo(skb);
606
607 /* This is a hint as to how much should be linear. */
608 vnet_hdr->hdr_len = skb_headlen(skb);
609 vnet_hdr->gso_size = sinfo->gso_size;
610 if (sinfo->gso_type & SKB_GSO_TCPV4)
611 vnet_hdr->gso_type = VIRTIO_NET_HDR_GSO_TCPV4;
612 else if (sinfo->gso_type & SKB_GSO_TCPV6)
613 vnet_hdr->gso_type = VIRTIO_NET_HDR_GSO_TCPV6;
614 else if (sinfo->gso_type & SKB_GSO_UDP)
615 vnet_hdr->gso_type = VIRTIO_NET_HDR_GSO_UDP;
616 else
617 BUG();
618 if (sinfo->gso_type & SKB_GSO_TCP_ECN)
619 vnet_hdr->gso_type |= VIRTIO_NET_HDR_GSO_ECN;
620 } else
621 vnet_hdr->gso_type = VIRTIO_NET_HDR_GSO_NONE;
622
623 if (skb->ip_summed == CHECKSUM_PARTIAL) {
624 vnet_hdr->flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
Michał Mirosław55508d62010-12-14 15:24:08 +0000625 vnet_hdr->csum_start = skb_checksum_start_offset(skb);
Arnd Bergmannb9fb9ee2010-02-18 05:48:17 +0000626 vnet_hdr->csum_offset = skb->csum_offset;
Jason Wang10a8d942011-06-10 00:56:17 +0000627 } else if (skb->ip_summed == CHECKSUM_UNNECESSARY) {
628 vnet_hdr->flags = VIRTIO_NET_HDR_F_DATA_VALID;
Arnd Bergmannb9fb9ee2010-02-18 05:48:17 +0000629 } /* else everything is zero */
630
631 return 0;
632}
633
634
Arnd Bergmann20d29d72010-01-30 12:24:26 +0000635/* Get packet from user space buffer */
Shirley Ma97bc3632011-07-06 12:26:11 +0000636static ssize_t macvtap_get_user(struct macvtap_queue *q, struct msghdr *m,
637 const struct iovec *iv, unsigned long total_len,
638 size_t count, int noblock)
Arnd Bergmann20d29d72010-01-30 12:24:26 +0000639{
640 struct sk_buff *skb;
Arnd Bergmann02df55d2010-02-18 05:45:36 +0000641 struct macvlan_dev *vlan;
Shirley Ma97bc3632011-07-06 12:26:11 +0000642 unsigned long len = total_len;
Arnd Bergmann20d29d72010-01-30 12:24:26 +0000643 int err;
Arnd Bergmannb9fb9ee2010-02-18 05:48:17 +0000644 struct virtio_net_hdr vnet_hdr = { 0 };
645 int vnet_hdr_len = 0;
Jason Wangb92946e2012-05-02 11:42:15 +0800646 int copylen = 0;
Shirley Ma97bc3632011-07-06 12:26:11 +0000647 bool zerocopy = false;
Arnd Bergmann20d29d72010-01-30 12:24:26 +0000648
Arnd Bergmannb9fb9ee2010-02-18 05:48:17 +0000649 if (q->flags & IFF_VNET_HDR) {
Michael S. Tsirkin55afbd02010-04-29 13:50:48 +0300650 vnet_hdr_len = q->vnet_hdr_sz;
Arnd Bergmannb9fb9ee2010-02-18 05:48:17 +0000651
652 err = -EINVAL;
Nicolas Kaiserce3c8692011-03-04 13:49:41 +0000653 if (len < vnet_hdr_len)
Arnd Bergmannb9fb9ee2010-02-18 05:48:17 +0000654 goto err;
Nicolas Kaiserce3c8692011-03-04 13:49:41 +0000655 len -= vnet_hdr_len;
Arnd Bergmannb9fb9ee2010-02-18 05:48:17 +0000656
657 err = memcpy_fromiovecend((void *)&vnet_hdr, iv, 0,
Michael S. Tsirkin55afbd02010-04-29 13:50:48 +0300658 sizeof(vnet_hdr));
Arnd Bergmannb9fb9ee2010-02-18 05:48:17 +0000659 if (err < 0)
660 goto err;
661 if ((vnet_hdr.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) &&
662 vnet_hdr.csum_start + vnet_hdr.csum_offset + 2 >
663 vnet_hdr.hdr_len)
664 vnet_hdr.hdr_len = vnet_hdr.csum_start +
665 vnet_hdr.csum_offset + 2;
666 err = -EINVAL;
667 if (vnet_hdr.hdr_len > len)
668 goto err;
669 }
670
671 err = -EINVAL;
Arnd Bergmann20d29d72010-01-30 12:24:26 +0000672 if (unlikely(len < ETH_HLEN))
Arnd Bergmannb9fb9ee2010-02-18 05:48:17 +0000673 goto err;
Arnd Bergmann20d29d72010-01-30 12:24:26 +0000674
Jason Wangb92946e2012-05-02 11:42:15 +0800675 err = -EMSGSIZE;
676 if (unlikely(count > UIO_MAXIOV))
677 goto err;
678
Shirley Ma97bc3632011-07-06 12:26:11 +0000679 if (m && m->msg_control && sock_flag(&q->sk, SOCK_ZEROCOPY))
680 zerocopy = true;
681
682 if (zerocopy) {
Jason Wangb92946e2012-05-02 11:42:15 +0800683 /* Userspace may produce vectors with count greater than
684 * MAX_SKB_FRAGS, so we need to linearize parts of the skb
685 * to let the rest of data to be fit in the frags.
686 */
687 if (count > MAX_SKB_FRAGS) {
688 copylen = iov_length(iv, count - MAX_SKB_FRAGS);
689 if (copylen < vnet_hdr_len)
690 copylen = 0;
691 else
692 copylen -= vnet_hdr_len;
693 }
Shirley Ma97bc3632011-07-06 12:26:11 +0000694 /* There are 256 bytes to be copied in skb, so there is enough
695 * room for skb expand head in case it is used.
696 * The rest buffer is mapped from userspace.
697 */
Jason Wangb92946e2012-05-02 11:42:15 +0800698 if (copylen < vnet_hdr.hdr_len)
699 copylen = vnet_hdr.hdr_len;
Shirley Ma97bc3632011-07-06 12:26:11 +0000700 if (!copylen)
701 copylen = GOODCOPY_LEN;
702 } else
703 copylen = len;
704
705 skb = macvtap_alloc_skb(&q->sk, NET_IP_ALIGN, copylen,
706 vnet_hdr.hdr_len, noblock, &err);
Arnd Bergmann02df55d2010-02-18 05:45:36 +0000707 if (!skb)
708 goto err;
Arnd Bergmann20d29d72010-01-30 12:24:26 +0000709
Jason Wang01d66572012-05-02 11:42:06 +0800710 if (zerocopy)
Shirley Ma97bc3632011-07-06 12:26:11 +0000711 err = zerocopy_sg_from_iovec(skb, iv, vnet_hdr_len, count);
Jason Wang01d66572012-05-02 11:42:06 +0800712 else
Shirley Ma97bc3632011-07-06 12:26:11 +0000713 err = skb_copy_datagram_from_iovec(skb, 0, iv, vnet_hdr_len,
714 len);
Arnd Bergmann02df55d2010-02-18 05:45:36 +0000715 if (err)
Arnd Bergmannb9fb9ee2010-02-18 05:48:17 +0000716 goto err_kfree;
Arnd Bergmann20d29d72010-01-30 12:24:26 +0000717
718 skb_set_network_header(skb, ETH_HLEN);
Arnd Bergmannb9fb9ee2010-02-18 05:48:17 +0000719 skb_reset_mac_header(skb);
720 skb->protocol = eth_hdr(skb)->h_proto;
721
722 if (vnet_hdr_len) {
723 err = macvtap_skb_from_vnet_hdr(skb, &vnet_hdr);
724 if (err)
725 goto err_kfree;
726 }
727
Jason Wang40893fd2013-03-26 23:11:22 +0000728 skb_probe_transport_header(skb, ETH_HLEN);
Jason Wang9b4d6692013-03-25 20:19:55 +0000729
Arnd Bergmann02df55d2010-02-18 05:45:36 +0000730 rcu_read_lock_bh();
Eric Dumazet13707f92011-01-26 19:28:23 +0000731 vlan = rcu_dereference_bh(q->vlan);
Shirley Ma97bc3632011-07-06 12:26:11 +0000732 /* copy skb_ubuf_info for callback when skb has no error */
Jason Wang01d66572012-05-02 11:42:06 +0800733 if (zerocopy) {
Shirley Ma97bc3632011-07-06 12:26:11 +0000734 skb_shinfo(skb)->destructor_arg = m->msg_control;
Jason Wang01d66572012-05-02 11:42:06 +0800735 skb_shinfo(skb)->tx_flags |= SKBTX_DEV_ZEROCOPY;
Pravin B Shelarc9af6db2013-02-11 09:27:41 +0000736 skb_shinfo(skb)->tx_flags |= SKBTX_SHARED_FRAG;
Jason Wang01d66572012-05-02 11:42:06 +0800737 }
Arnd Bergmann02df55d2010-02-18 05:45:36 +0000738 if (vlan)
739 macvlan_start_xmit(skb, vlan->dev);
740 else
741 kfree_skb(skb);
742 rcu_read_unlock_bh();
Arnd Bergmann20d29d72010-01-30 12:24:26 +0000743
Shirley Ma97bc3632011-07-06 12:26:11 +0000744 return total_len;
Arnd Bergmann02df55d2010-02-18 05:45:36 +0000745
Arnd Bergmannb9fb9ee2010-02-18 05:48:17 +0000746err_kfree:
747 kfree_skb(skb);
748
Arnd Bergmann02df55d2010-02-18 05:45:36 +0000749err:
750 rcu_read_lock_bh();
Eric Dumazet13707f92011-01-26 19:28:23 +0000751 vlan = rcu_dereference_bh(q->vlan);
Arnd Bergmann02df55d2010-02-18 05:45:36 +0000752 if (vlan)
Eric Dumazet1ac9ad12011-01-12 12:13:14 +0000753 vlan->dev->stats.tx_dropped++;
Arnd Bergmann02df55d2010-02-18 05:45:36 +0000754 rcu_read_unlock_bh();
755
Arnd Bergmann02df55d2010-02-18 05:45:36 +0000756 return err;
Arnd Bergmann20d29d72010-01-30 12:24:26 +0000757}
758
759static ssize_t macvtap_aio_write(struct kiocb *iocb, const struct iovec *iv,
760 unsigned long count, loff_t pos)
761{
762 struct file *file = iocb->ki_filp;
763 ssize_t result = -ENOLINK;
Arnd Bergmann02df55d2010-02-18 05:45:36 +0000764 struct macvtap_queue *q = file->private_data;
Arnd Bergmann20d29d72010-01-30 12:24:26 +0000765
Shirley Ma97bc3632011-07-06 12:26:11 +0000766 result = macvtap_get_user(q, NULL, iv, iov_length(iv, count), count,
767 file->f_flags & O_NONBLOCK);
Arnd Bergmann20d29d72010-01-30 12:24:26 +0000768 return result;
769}
770
771/* Put packet to the user space buffer */
772static ssize_t macvtap_put_user(struct macvtap_queue *q,
773 const struct sk_buff *skb,
774 const struct iovec *iv, int len)
775{
Arnd Bergmann02df55d2010-02-18 05:45:36 +0000776 struct macvlan_dev *vlan;
Arnd Bergmann20d29d72010-01-30 12:24:26 +0000777 int ret;
Arnd Bergmannb9fb9ee2010-02-18 05:48:17 +0000778 int vnet_hdr_len = 0;
Basil Gorf09e2242012-05-03 22:55:24 +0000779 int vlan_offset = 0;
780 int copied;
Arnd Bergmannb9fb9ee2010-02-18 05:48:17 +0000781
782 if (q->flags & IFF_VNET_HDR) {
783 struct virtio_net_hdr vnet_hdr;
Michael S. Tsirkin55afbd02010-04-29 13:50:48 +0300784 vnet_hdr_len = q->vnet_hdr_sz;
Arnd Bergmannb9fb9ee2010-02-18 05:48:17 +0000785 if ((len -= vnet_hdr_len) < 0)
786 return -EINVAL;
787
788 ret = macvtap_skb_to_vnet_hdr(skb, &vnet_hdr);
789 if (ret)
790 return ret;
791
Michael S. Tsirkin55afbd02010-04-29 13:50:48 +0300792 if (memcpy_toiovecend(iv, (void *)&vnet_hdr, 0, sizeof(vnet_hdr)))
Arnd Bergmannb9fb9ee2010-02-18 05:48:17 +0000793 return -EFAULT;
794 }
Basil Gorf09e2242012-05-03 22:55:24 +0000795 copied = vnet_hdr_len;
Arnd Bergmann20d29d72010-01-30 12:24:26 +0000796
Basil Gorf09e2242012-05-03 22:55:24 +0000797 if (!vlan_tx_tag_present(skb))
798 len = min_t(int, skb->len, len);
799 else {
800 int copy;
801 struct {
802 __be16 h_vlan_proto;
803 __be16 h_vlan_TCI;
804 } veth;
805 veth.h_vlan_proto = htons(ETH_P_8021Q);
806 veth.h_vlan_TCI = htons(vlan_tx_tag_get(skb));
Arnd Bergmann20d29d72010-01-30 12:24:26 +0000807
Basil Gorf09e2242012-05-03 22:55:24 +0000808 vlan_offset = offsetof(struct vlan_ethhdr, h_vlan_proto);
809 len = min_t(int, skb->len + VLAN_HLEN, len);
Arnd Bergmann20d29d72010-01-30 12:24:26 +0000810
Basil Gorf09e2242012-05-03 22:55:24 +0000811 copy = min_t(int, vlan_offset, len);
812 ret = skb_copy_datagram_const_iovec(skb, 0, iv, copied, copy);
813 len -= copy;
814 copied += copy;
815 if (ret || !len)
816 goto done;
817
818 copy = min_t(int, sizeof(veth), len);
819 ret = memcpy_toiovecend(iv, (void *)&veth, copied, copy);
820 len -= copy;
821 copied += copy;
822 if (ret || !len)
823 goto done;
824 }
825
826 ret = skb_copy_datagram_const_iovec(skb, vlan_offset, iv, copied, len);
827 copied += len;
828
829done:
Arnd Bergmann02df55d2010-02-18 05:45:36 +0000830 rcu_read_lock_bh();
Eric Dumazet13707f92011-01-26 19:28:23 +0000831 vlan = rcu_dereference_bh(q->vlan);
Arnd Bergmann501c7742010-02-18 05:46:50 +0000832 if (vlan)
Basil Gorf09e2242012-05-03 22:55:24 +0000833 macvlan_count_rx(vlan, copied - vnet_hdr_len, ret == 0, 0);
Arnd Bergmann02df55d2010-02-18 05:45:36 +0000834 rcu_read_unlock_bh();
Arnd Bergmann20d29d72010-01-30 12:24:26 +0000835
Basil Gorf09e2242012-05-03 22:55:24 +0000836 return ret ? ret : copied;
Arnd Bergmann20d29d72010-01-30 12:24:26 +0000837}
838
Arnd Bergmann501c7742010-02-18 05:46:50 +0000839static ssize_t macvtap_do_read(struct macvtap_queue *q, struct kiocb *iocb,
840 const struct iovec *iv, unsigned long len,
841 int noblock)
Arnd Bergmann20d29d72010-01-30 12:24:26 +0000842{
Hong zhi guoccf7e722012-06-06 22:36:27 +0000843 DEFINE_WAIT(wait);
Arnd Bergmann20d29d72010-01-30 12:24:26 +0000844 struct sk_buff *skb;
Arnd Bergmann501c7742010-02-18 05:46:50 +0000845 ssize_t ret = 0;
Arnd Bergmann20d29d72010-01-30 12:24:26 +0000846
Arnd Bergmann20d29d72010-01-30 12:24:26 +0000847 while (len) {
Hong zhi guoccf7e722012-06-06 22:36:27 +0000848 prepare_to_wait(sk_sleep(&q->sk), &wait, TASK_INTERRUPTIBLE);
Arnd Bergmann20d29d72010-01-30 12:24:26 +0000849
850 /* Read frames from the queue */
851 skb = skb_dequeue(&q->sk.sk_receive_queue);
852 if (!skb) {
Arnd Bergmann501c7742010-02-18 05:46:50 +0000853 if (noblock) {
Arnd Bergmann20d29d72010-01-30 12:24:26 +0000854 ret = -EAGAIN;
855 break;
856 }
857 if (signal_pending(current)) {
858 ret = -ERESTARTSYS;
859 break;
860 }
861 /* Nothing to read, let's sleep */
862 schedule();
863 continue;
864 }
865 ret = macvtap_put_user(q, skb, iv, len);
866 kfree_skb(skb);
867 break;
868 }
869
Hong zhi guoccf7e722012-06-06 22:36:27 +0000870 finish_wait(sk_sleep(&q->sk), &wait);
Arnd Bergmann501c7742010-02-18 05:46:50 +0000871 return ret;
872}
Arnd Bergmann20d29d72010-01-30 12:24:26 +0000873
Arnd Bergmann501c7742010-02-18 05:46:50 +0000874static ssize_t macvtap_aio_read(struct kiocb *iocb, const struct iovec *iv,
875 unsigned long count, loff_t pos)
876{
877 struct file *file = iocb->ki_filp;
878 struct macvtap_queue *q = file->private_data;
879 ssize_t len, ret = 0;
880
881 len = iov_length(iv, count);
882 if (len < 0) {
883 ret = -EINVAL;
884 goto out;
885 }
886
887 ret = macvtap_do_read(q, iocb, iv, len, file->f_flags & O_NONBLOCK);
888 ret = min_t(ssize_t, ret, len); /* XXX copied from tun.c. Why? */
Arnd Bergmann20d29d72010-01-30 12:24:26 +0000889out:
Arnd Bergmann20d29d72010-01-30 12:24:26 +0000890 return ret;
891}
892
893/*
894 * provide compatibility with generic tun/tap interface
895 */
896static long macvtap_ioctl(struct file *file, unsigned int cmd,
897 unsigned long arg)
898{
Arnd Bergmann02df55d2010-02-18 05:45:36 +0000899 struct macvtap_queue *q = file->private_data;
900 struct macvlan_dev *vlan;
Arnd Bergmann20d29d72010-01-30 12:24:26 +0000901 void __user *argp = (void __user *)arg;
902 struct ifreq __user *ifr = argp;
903 unsigned int __user *up = argp;
904 unsigned int u;
Michael S. Tsirkin55afbd02010-04-29 13:50:48 +0300905 int __user *sp = argp;
906 int s;
Arnd Bergmann02df55d2010-02-18 05:45:36 +0000907 int ret;
Arnd Bergmann20d29d72010-01-30 12:24:26 +0000908
909 switch (cmd) {
910 case TUNSETIFF:
911 /* ignore the name, just look at flags */
912 if (get_user(u, &ifr->ifr_flags))
913 return -EFAULT;
Arnd Bergmannb9fb9ee2010-02-18 05:48:17 +0000914
915 ret = 0;
916 if ((u & ~IFF_VNET_HDR) != (IFF_NO_PI | IFF_TAP))
917 ret = -EINVAL;
918 else
919 q->flags = u;
920
921 return ret;
Arnd Bergmann20d29d72010-01-30 12:24:26 +0000922
923 case TUNGETIFF:
Arnd Bergmann02df55d2010-02-18 05:45:36 +0000924 rcu_read_lock_bh();
Eric Dumazet13707f92011-01-26 19:28:23 +0000925 vlan = rcu_dereference_bh(q->vlan);
Arnd Bergmann02df55d2010-02-18 05:45:36 +0000926 if (vlan)
927 dev_hold(vlan->dev);
928 rcu_read_unlock_bh();
Arnd Bergmann20d29d72010-01-30 12:24:26 +0000929
Arnd Bergmann02df55d2010-02-18 05:45:36 +0000930 if (!vlan)
931 return -ENOLINK;
932
933 ret = 0;
Eric Dumazet13707f92011-01-26 19:28:23 +0000934 if (copy_to_user(&ifr->ifr_name, vlan->dev->name, IFNAMSIZ) ||
Arnd Bergmannb9fb9ee2010-02-18 05:48:17 +0000935 put_user(q->flags, &ifr->ifr_flags))
Arnd Bergmann02df55d2010-02-18 05:45:36 +0000936 ret = -EFAULT;
937 dev_put(vlan->dev);
938 return ret;
Arnd Bergmann20d29d72010-01-30 12:24:26 +0000939
940 case TUNGETFEATURES:
Arnd Bergmannb9fb9ee2010-02-18 05:48:17 +0000941 if (put_user(IFF_TAP | IFF_NO_PI | IFF_VNET_HDR, up))
Arnd Bergmann20d29d72010-01-30 12:24:26 +0000942 return -EFAULT;
943 return 0;
944
945 case TUNSETSNDBUF:
946 if (get_user(u, up))
947 return -EFAULT;
948
Arnd Bergmann20d29d72010-01-30 12:24:26 +0000949 q->sk.sk_sndbuf = u;
Arnd Bergmann20d29d72010-01-30 12:24:26 +0000950 return 0;
951
Michael S. Tsirkin55afbd02010-04-29 13:50:48 +0300952 case TUNGETVNETHDRSZ:
953 s = q->vnet_hdr_sz;
954 if (put_user(s, sp))
955 return -EFAULT;
956 return 0;
957
958 case TUNSETVNETHDRSZ:
959 if (get_user(s, sp))
960 return -EFAULT;
961 if (s < (int)sizeof(struct virtio_net_hdr))
962 return -EINVAL;
963
964 q->vnet_hdr_sz = s;
965 return 0;
966
Arnd Bergmann20d29d72010-01-30 12:24:26 +0000967 case TUNSETOFFLOAD:
968 /* let the user check for future flags */
969 if (arg & ~(TUN_F_CSUM | TUN_F_TSO4 | TUN_F_TSO6 |
Arnd Bergmannb9fb9ee2010-02-18 05:48:17 +0000970 TUN_F_TSO_ECN | TUN_F_UFO))
Arnd Bergmann20d29d72010-01-30 12:24:26 +0000971 return -EINVAL;
972
Arnd Bergmannb9fb9ee2010-02-18 05:48:17 +0000973 /* TODO: only accept frames with the features that
974 got enabled for forwarded frames */
975 if (!(q->flags & IFF_VNET_HDR))
976 return -EINVAL;
Arnd Bergmann20d29d72010-01-30 12:24:26 +0000977 return 0;
978
979 default:
980 return -EINVAL;
981 }
982}
983
984#ifdef CONFIG_COMPAT
985static long macvtap_compat_ioctl(struct file *file, unsigned int cmd,
986 unsigned long arg)
987{
988 return macvtap_ioctl(file, cmd, (unsigned long)compat_ptr(arg));
989}
990#endif
991
992static const struct file_operations macvtap_fops = {
993 .owner = THIS_MODULE,
994 .open = macvtap_open,
995 .release = macvtap_release,
996 .aio_read = macvtap_aio_read,
997 .aio_write = macvtap_aio_write,
998 .poll = macvtap_poll,
999 .llseek = no_llseek,
1000 .unlocked_ioctl = macvtap_ioctl,
1001#ifdef CONFIG_COMPAT
1002 .compat_ioctl = macvtap_compat_ioctl,
1003#endif
1004};
1005
Arnd Bergmann501c7742010-02-18 05:46:50 +00001006static int macvtap_sendmsg(struct kiocb *iocb, struct socket *sock,
1007 struct msghdr *m, size_t total_len)
1008{
1009 struct macvtap_queue *q = container_of(sock, struct macvtap_queue, sock);
Shirley Ma97bc3632011-07-06 12:26:11 +00001010 return macvtap_get_user(q, m, m->msg_iov, total_len, m->msg_iovlen,
Arnd Bergmann501c7742010-02-18 05:46:50 +00001011 m->msg_flags & MSG_DONTWAIT);
1012}
1013
1014static int macvtap_recvmsg(struct kiocb *iocb, struct socket *sock,
1015 struct msghdr *m, size_t total_len,
1016 int flags)
1017{
1018 struct macvtap_queue *q = container_of(sock, struct macvtap_queue, sock);
1019 int ret;
1020 if (flags & ~(MSG_DONTWAIT|MSG_TRUNC))
1021 return -EINVAL;
1022 ret = macvtap_do_read(q, iocb, m->msg_iov, total_len,
1023 flags & MSG_DONTWAIT);
1024 if (ret > total_len) {
1025 m->msg_flags |= MSG_TRUNC;
1026 ret = flags & MSG_TRUNC ? ret : total_len;
1027 }
1028 return ret;
1029}
1030
1031/* Ops structure to mimic raw sockets with tun */
1032static const struct proto_ops macvtap_socket_ops = {
1033 .sendmsg = macvtap_sendmsg,
1034 .recvmsg = macvtap_recvmsg,
1035};
1036
1037/* Get an underlying socket object from tun file. Returns error unless file is
1038 * attached to a device. The returned object works like a packet socket, it
1039 * can be used for sock_sendmsg/sock_recvmsg. The caller is responsible for
1040 * holding a reference to the file for as long as the socket is in use. */
1041struct socket *macvtap_get_socket(struct file *file)
1042{
1043 struct macvtap_queue *q;
1044 if (file->f_op != &macvtap_fops)
1045 return ERR_PTR(-EINVAL);
1046 q = file->private_data;
1047 if (!q)
1048 return ERR_PTR(-EBADFD);
1049 return &q->sock;
1050}
1051EXPORT_SYMBOL_GPL(macvtap_get_socket);
1052
Eric W. Biederman9bf19072011-10-20 04:28:46 +00001053static int macvtap_device_event(struct notifier_block *unused,
1054 unsigned long event, void *ptr)
1055{
1056 struct net_device *dev = ptr;
Eric W. Biedermane09eff72011-10-20 04:29:24 +00001057 struct macvlan_dev *vlan;
Eric W. Biederman9bf19072011-10-20 04:28:46 +00001058 struct device *classdev;
1059 dev_t devt;
Eric W. Biedermane09eff72011-10-20 04:29:24 +00001060 int err;
Eric W. Biederman9bf19072011-10-20 04:28:46 +00001061
1062 if (dev->rtnl_link_ops != &macvtap_link_ops)
1063 return NOTIFY_DONE;
1064
Eric W. Biedermane09eff72011-10-20 04:29:24 +00001065 vlan = netdev_priv(dev);
Eric W. Biederman9bf19072011-10-20 04:28:46 +00001066
1067 switch (event) {
1068 case NETDEV_REGISTER:
1069 /* Create the device node here after the network device has
1070 * been registered but before register_netdevice has
1071 * finished running.
1072 */
Eric W. Biedermane09eff72011-10-20 04:29:24 +00001073 err = macvtap_get_minor(vlan);
1074 if (err)
1075 return notifier_from_errno(err);
1076
1077 devt = MKDEV(MAJOR(macvtap_major), vlan->minor);
Eric W. Biederman9bf19072011-10-20 04:28:46 +00001078 classdev = device_create(macvtap_class, &dev->dev, devt,
1079 dev, "tap%d", dev->ifindex);
Eric W. Biedermane09eff72011-10-20 04:29:24 +00001080 if (IS_ERR(classdev)) {
1081 macvtap_free_minor(vlan);
Eric W. Biederman9bf19072011-10-20 04:28:46 +00001082 return notifier_from_errno(PTR_ERR(classdev));
Eric W. Biedermane09eff72011-10-20 04:29:24 +00001083 }
Eric W. Biederman9bf19072011-10-20 04:28:46 +00001084 break;
1085 case NETDEV_UNREGISTER:
Eric W. Biedermane09eff72011-10-20 04:29:24 +00001086 devt = MKDEV(MAJOR(macvtap_major), vlan->minor);
Eric W. Biederman9bf19072011-10-20 04:28:46 +00001087 device_destroy(macvtap_class, devt);
Eric W. Biedermane09eff72011-10-20 04:29:24 +00001088 macvtap_free_minor(vlan);
Eric W. Biederman9bf19072011-10-20 04:28:46 +00001089 break;
1090 }
1091
1092 return NOTIFY_DONE;
1093}
1094
1095static struct notifier_block macvtap_notifier_block __read_mostly = {
1096 .notifier_call = macvtap_device_event,
1097};
1098
Arnd Bergmann20d29d72010-01-30 12:24:26 +00001099static int macvtap_init(void)
1100{
1101 int err;
1102
1103 err = alloc_chrdev_region(&macvtap_major, 0,
1104 MACVTAP_NUM_DEVS, "macvtap");
1105 if (err)
1106 goto out1;
1107
1108 cdev_init(&macvtap_cdev, &macvtap_fops);
1109 err = cdev_add(&macvtap_cdev, macvtap_major, MACVTAP_NUM_DEVS);
1110 if (err)
1111 goto out2;
1112
1113 macvtap_class = class_create(THIS_MODULE, "macvtap");
1114 if (IS_ERR(macvtap_class)) {
1115 err = PTR_ERR(macvtap_class);
1116 goto out3;
1117 }
1118
Eric W. Biederman9bf19072011-10-20 04:28:46 +00001119 err = register_netdevice_notifier(&macvtap_notifier_block);
Arnd Bergmann20d29d72010-01-30 12:24:26 +00001120 if (err)
1121 goto out4;
1122
Eric W. Biederman9bf19072011-10-20 04:28:46 +00001123 err = macvlan_link_register(&macvtap_link_ops);
1124 if (err)
1125 goto out5;
1126
Arnd Bergmann20d29d72010-01-30 12:24:26 +00001127 return 0;
1128
Eric W. Biederman9bf19072011-10-20 04:28:46 +00001129out5:
1130 unregister_netdevice_notifier(&macvtap_notifier_block);
Arnd Bergmann20d29d72010-01-30 12:24:26 +00001131out4:
1132 class_unregister(macvtap_class);
1133out3:
1134 cdev_del(&macvtap_cdev);
1135out2:
1136 unregister_chrdev_region(macvtap_major, MACVTAP_NUM_DEVS);
1137out1:
1138 return err;
1139}
1140module_init(macvtap_init);
1141
1142static void macvtap_exit(void)
1143{
1144 rtnl_link_unregister(&macvtap_link_ops);
Eric W. Biederman9bf19072011-10-20 04:28:46 +00001145 unregister_netdevice_notifier(&macvtap_notifier_block);
Arnd Bergmann20d29d72010-01-30 12:24:26 +00001146 class_unregister(macvtap_class);
1147 cdev_del(&macvtap_cdev);
1148 unregister_chrdev_region(macvtap_major, MACVTAP_NUM_DEVS);
1149}
1150module_exit(macvtap_exit);
1151
1152MODULE_ALIAS_RTNL_LINK("macvtap");
1153MODULE_AUTHOR("Arnd Bergmann <arnd@arndb.de>");
1154MODULE_LICENSE("GPL");