blob: cb4ef331567c5494aec527015be0bfd191598711 [file] [log] [blame]
Rusty Russell48925e32009-09-24 09:59:20 -06001/* A network driver using virtio.
Rusty Russell296f96f2007-10-22 11:03:37 +10002 *
3 * Copyright 2007 Rusty Russell <rusty@rustcorp.com.au> IBM Corporation
4 *
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation; either version 2 of the License, or
8 * (at your option) any later version.
9 *
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
14 *
15 * You should have received a copy of the GNU General Public License
Jeff Kirsheradf8d3f2013-12-06 06:28:47 -080016 * along with this program; if not, see <http://www.gnu.org/licenses/>.
Rusty Russell296f96f2007-10-22 11:03:37 +100017 */
18//#define DEBUG
19#include <linux/netdevice.h>
20#include <linux/etherdevice.h>
Herbert Xua9ea3fc2008-04-18 11:21:42 +080021#include <linux/ethtool.h>
Rusty Russell296f96f2007-10-22 11:03:37 +100022#include <linux/module.h>
23#include <linux/virtio.h>
24#include <linux/virtio_net.h>
John Fastabendf600b692016-12-15 12:13:24 -080025#include <linux/bpf.h>
Daniel Borkmanna67edbf2017-01-25 02:28:18 +010026#include <linux/bpf_trace.h>
Rusty Russell296f96f2007-10-22 11:03:37 +100027#include <linux/scatterlist.h>
Alex Williamsone918085a2009-01-25 18:06:26 -080028#include <linux/if_vlan.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090029#include <linux/slab.h>
Wanlong Gao8de4b2f2013-01-24 23:51:31 +000030#include <linux/cpu.h>
Michael Daltonab7db912014-01-16 22:23:27 -080031#include <linux/average.h>
Jason Wang186b3c92017-09-19 17:42:43 +080032#include <linux/filter.h>
Sridhar Samudralaba5e4422018-05-24 09:55:17 -070033#include <linux/netdevice.h>
34#include <linux/pci.h>
Michael S. Tsirkind85b758f72017-03-09 02:21:21 +020035#include <net/route.h>
Jesper Dangaard Brouer754b8a22018-01-03 11:26:04 +010036#include <net/xdp.h>
Sridhar Samudralaba5e4422018-05-24 09:55:17 -070037#include <net/net_failover.h>
Rusty Russell296f96f2007-10-22 11:03:37 +100038
Amerigo Wangd34710e2013-05-09 19:50:51 +000039static int napi_weight = NAPI_POLL_WEIGHT;
Dor Laor6c0cd7c2007-12-16 15:19:43 +020040module_param(napi_weight, int, 0444);
41
Willem de Bruijnb92f1e62017-04-24 13:49:27 -040042static bool csum = true, gso = true, napi_tx;
Rusty Russell34a48572008-02-04 23:50:02 -050043module_param(csum, bool, 0444);
44module_param(gso, bool, 0444);
Willem de Bruijnb92f1e62017-04-24 13:49:27 -040045module_param(napi_tx, bool, 0644);
Rusty Russell34a48572008-02-04 23:50:02 -050046
Rusty Russell296f96f2007-10-22 11:03:37 +100047/* FIXME: MTU in config. */
Michael Dalton5061de32013-11-14 10:41:04 -080048#define GOOD_PACKET_LEN (ETH_HLEN + VLAN_HLEN + ETH_DATA_LEN)
Mark McLoughlin3f2c31d2008-11-16 22:41:34 -080049#define GOOD_COPY_LEN 128
Rusty Russell296f96f2007-10-22 11:03:37 +100050
Jason Wangf6b10202017-02-21 16:46:28 +080051#define VIRTNET_RX_PAD (NET_IP_ALIGN + NET_SKB_PAD)
52
John Fastabend2de2f7f2017-02-02 19:16:29 -080053/* Amount of XDP headroom to prepend to packets for use by xdp_adjust_head */
54#define VIRTIO_XDP_HEADROOM 256
55
Jesper Dangaard Brouer2471c752018-06-26 17:39:58 +020056/* Separating two types of XDP xmit */
57#define VIRTIO_XDP_TX BIT(0)
58#define VIRTIO_XDP_REDIR BIT(1)
59
Johannes Berg5377d7582015-08-19 09:48:40 +020060/* RX packet size EWMA. The average packet size is used to determine the packet
61 * buffer size when refilling RX rings. As the entire RX ring may be refilled
62 * at once, the weight is chosen so that the EWMA will be insensitive to short-
63 * term, transient changes in packet size.
Michael Daltonab7db912014-01-16 22:23:27 -080064 */
Johannes Bergeb1e0112017-02-15 09:49:26 +010065DECLARE_EWMA(pkt_len, 0, 64)
Michael Daltonab7db912014-01-16 22:23:27 -080066
Rick Jones66846042011-11-14 14:17:08 +000067#define VIRTNET_DRIVER_VERSION "1.0.0"
Alex Williamson2a41f712009-02-04 09:02:34 +000068
Colin Ian King7acd4322017-08-12 22:45:53 +010069static const unsigned long guest_offloads[] = {
70 VIRTIO_NET_F_GUEST_TSO4,
71 VIRTIO_NET_F_GUEST_TSO6,
72 VIRTIO_NET_F_GUEST_ECN,
73 VIRTIO_NET_F_GUEST_UFO
74};
Jason Wang3f935222017-07-19 16:54:49 +080075
Toshiaki Makitad7dfc5c2018-01-17 15:38:25 +090076struct virtnet_stat_desc {
77 char desc[ETH_GSTRING_LEN];
78 size_t offset;
stephen hemminger3fa2a1d2011-06-15 06:36:29 +000079};
80
Toshiaki Makitad7dfc5c2018-01-17 15:38:25 +090081struct virtnet_sq_stats {
82 struct u64_stats_sync syncp;
83 u64 packets;
84 u64 bytes;
Toshiaki Makita5b8f3c82018-07-23 23:36:08 +090085 u64 xdp_tx;
86 u64 xdp_tx_drops;
Toshiaki Makitad7dfc5c2018-01-17 15:38:25 +090087};
88
Toshiaki Makitaa0929a42018-07-23 23:36:05 +090089struct virtnet_rq_stat_items {
Toshiaki Makitad7dfc5c2018-01-17 15:38:25 +090090 u64 packets;
91 u64 bytes;
Toshiaki Makita2c4a2f72018-07-23 23:36:06 +090092 u64 drops;
Toshiaki Makita5b8f3c82018-07-23 23:36:08 +090093 u64 xdp_packets;
94 u64 xdp_tx;
95 u64 xdp_redirects;
96 u64 xdp_drops;
Toshiaki Makitad7dfc5c2018-01-17 15:38:25 +090097};
98
Toshiaki Makitaa0929a42018-07-23 23:36:05 +090099struct virtnet_rq_stats {
100 struct u64_stats_sync syncp;
101 struct virtnet_rq_stat_items items;
102};
103
104struct virtnet_rx_stats {
105 struct virtnet_rq_stat_items rx;
Toshiaki Makita5b8f3c82018-07-23 23:36:08 +0900106 struct {
107 unsigned int xdp_tx;
108 unsigned int xdp_tx_drops;
109 } tx;
Toshiaki Makitaa0929a42018-07-23 23:36:05 +0900110};
111
Toshiaki Makitad7dfc5c2018-01-17 15:38:25 +0900112#define VIRTNET_SQ_STAT(m) offsetof(struct virtnet_sq_stats, m)
Toshiaki Makitaa0929a42018-07-23 23:36:05 +0900113#define VIRTNET_RQ_STAT(m) offsetof(struct virtnet_rq_stat_items, m)
Toshiaki Makitad7dfc5c2018-01-17 15:38:25 +0900114
115static const struct virtnet_stat_desc virtnet_sq_stats_desc[] = {
Toshiaki Makita5b8f3c82018-07-23 23:36:08 +0900116 { "packets", VIRTNET_SQ_STAT(packets) },
117 { "bytes", VIRTNET_SQ_STAT(bytes) },
118 { "xdp_tx", VIRTNET_SQ_STAT(xdp_tx) },
119 { "xdp_tx_drops", VIRTNET_SQ_STAT(xdp_tx_drops) },
Toshiaki Makitad7dfc5c2018-01-17 15:38:25 +0900120};
121
122static const struct virtnet_stat_desc virtnet_rq_stats_desc[] = {
Toshiaki Makita5b8f3c82018-07-23 23:36:08 +0900123 { "packets", VIRTNET_RQ_STAT(packets) },
124 { "bytes", VIRTNET_RQ_STAT(bytes) },
125 { "drops", VIRTNET_RQ_STAT(drops) },
126 { "xdp_packets", VIRTNET_RQ_STAT(xdp_packets) },
127 { "xdp_tx", VIRTNET_RQ_STAT(xdp_tx) },
128 { "xdp_redirects", VIRTNET_RQ_STAT(xdp_redirects) },
129 { "xdp_drops", VIRTNET_RQ_STAT(xdp_drops) },
Toshiaki Makitad7dfc5c2018-01-17 15:38:25 +0900130};
131
132#define VIRTNET_SQ_STATS_LEN ARRAY_SIZE(virtnet_sq_stats_desc)
133#define VIRTNET_RQ_STATS_LEN ARRAY_SIZE(virtnet_rq_stats_desc)
134
Jason Wange9d74172012-12-07 07:04:55 +0000135/* Internal representation of a send virtqueue */
136struct send_queue {
137 /* Virtqueue associated with this send _queue */
138 struct virtqueue *vq;
139
140 /* TX: fragments + linear part + virtio header */
141 struct scatterlist sg[MAX_SKB_FRAGS + 2];
Jason Wang986a4f42012-12-07 07:04:56 +0000142
143 /* Name of the send queue: output.$index */
144 char name[40];
Willem de Bruijnb92f1e62017-04-24 13:49:27 -0400145
Toshiaki Makitad7dfc5c2018-01-17 15:38:25 +0900146 struct virtnet_sq_stats stats;
147
Willem de Bruijnb92f1e62017-04-24 13:49:27 -0400148 struct napi_struct napi;
Jason Wange9d74172012-12-07 07:04:55 +0000149};
150
151/* Internal representation of a receive virtqueue */
152struct receive_queue {
153 /* Virtqueue associated with this receive_queue */
154 struct virtqueue *vq;
155
Rusty Russell296f96f2007-10-22 11:03:37 +1000156 struct napi_struct napi;
157
John Fastabendf600b692016-12-15 12:13:24 -0800158 struct bpf_prog __rcu *xdp_prog;
159
Toshiaki Makitad7dfc5c2018-01-17 15:38:25 +0900160 struct virtnet_rq_stats stats;
161
Jason Wange9d74172012-12-07 07:04:55 +0000162 /* Chain pages by the private ptr. */
163 struct page *pages;
164
Michael Daltonab7db912014-01-16 22:23:27 -0800165 /* Average packet length for mergeable receive buffers. */
Johannes Berg5377d7582015-08-19 09:48:40 +0200166 struct ewma_pkt_len mrg_avg_pkt_len;
Michael Daltonab7db912014-01-16 22:23:27 -0800167
Michael Daltonfb518792014-01-16 22:23:26 -0800168 /* Page frag for packet buffer allocation. */
169 struct page_frag alloc_frag;
170
Jason Wange9d74172012-12-07 07:04:55 +0000171 /* RX: fragments + linear part + virtio header */
172 struct scatterlist sg[MAX_SKB_FRAGS + 2];
Jason Wang986a4f42012-12-07 07:04:56 +0000173
Michael S. Tsirkind85b758f72017-03-09 02:21:21 +0200174 /* Min single buffer size for mergeable buffers case. */
175 unsigned int min_buf_len;
176
Jason Wang986a4f42012-12-07 07:04:56 +0000177 /* Name of this receive queue: input.$index */
178 char name[40];
Jesper Dangaard Brouer754b8a22018-01-03 11:26:04 +0100179
180 struct xdp_rxq_info xdp_rxq;
Jason Wange9d74172012-12-07 07:04:55 +0000181};
182
Michael S. Tsirkin12e57162018-04-19 08:30:48 +0300183/* Control VQ buffers: protected by the rtnl lock */
184struct control_buf {
185 struct virtio_net_ctrl_hdr hdr;
186 virtio_net_ctrl_ack status;
187 struct virtio_net_ctrl_mq mq;
188 u8 promisc;
189 u8 allmulti;
Michael S. Tsirkind7fad4c2018-04-19 08:30:49 +0300190 __virtio16 vid;
Michael S. Tsirkinf4ee7032018-04-19 08:30:50 +0300191 __virtio64 offloads;
Michael S. Tsirkin12e57162018-04-19 08:30:48 +0300192};
193
Jason Wange9d74172012-12-07 07:04:55 +0000194struct virtnet_info {
195 struct virtio_device *vdev;
196 struct virtqueue *cvq;
197 struct net_device *dev;
Jason Wang986a4f42012-12-07 07:04:56 +0000198 struct send_queue *sq;
199 struct receive_queue *rq;
Jason Wange9d74172012-12-07 07:04:55 +0000200 unsigned int status;
201
Jason Wang986a4f42012-12-07 07:04:56 +0000202 /* Max # of queue pairs supported by the device */
203 u16 max_queue_pairs;
204
205 /* # of queue pairs currently used by the driver */
206 u16 curr_queue_pairs;
207
John Fastabend672aafd2016-12-15 12:13:49 -0800208 /* # of XDP queue pairs currently used by the driver */
209 u16 xdp_queue_pairs;
210
Herbert Xu97402b92008-04-18 11:24:27 +0800211 /* I like... big packets and I cannot lie! */
212 bool big_packets;
213
Mark McLoughlin3f2c31d2008-11-16 22:41:34 -0800214 /* Host will merge rx buffers for big packets (shake it! shake it!) */
215 bool mergeable_rx_bufs;
216
Jason Wang986a4f42012-12-07 07:04:56 +0000217 /* Has control virtqueue */
218 bool has_cvq;
219
Michael S. Tsirkine7428e92013-07-25 10:20:23 +0930220 /* Host can handle any s/g split between our header and packet data */
221 bool any_header_sg;
222
Michael S. Tsirkin012873d2014-10-24 16:55:57 +0300223 /* Packet virtio header size */
224 u8 hdr_len;
225
Rusty Russell3161e452009-08-26 12:22:32 -0700226 /* Work struct for refilling if we run low on memory. */
227 struct delayed_work refill;
228
Jason Wang586d17c2012-04-11 20:43:52 +0000229 /* Work struct for config space updates */
230 struct work_struct config_work;
231
Jason Wang986a4f42012-12-07 07:04:56 +0000232 /* Does the affinity hint is set for virtqueues? */
233 bool affinity_hint_set;
Wanlong Gao47be2472013-01-24 23:51:29 +0000234
Sebastian Andrzej Siewior8017c272016-08-12 19:49:43 +0200235 /* CPU hotplug instances for online & dead */
236 struct hlist_node node;
237 struct hlist_node node_dead;
Michael S. Tsirkin2ac46032015-11-15 15:11:00 +0200238
Michael S. Tsirkin12e57162018-04-19 08:30:48 +0300239 struct control_buf *ctrl;
Nikolay Aleksandrov16032be2016-02-03 04:04:37 +0100240
241 /* Ethtool settings */
242 u8 duplex;
243 u32 speed;
Jason Wang3f935222017-07-19 16:54:49 +0800244
245 unsigned long guest_offloads;
Sridhar Samudralaba5e4422018-05-24 09:55:17 -0700246
247 /* failover when STANDBY feature enabled */
248 struct failover *failover;
Rusty Russell296f96f2007-10-22 11:03:37 +1000249};
250
Shirley Ma9ab86bb2010-01-29 03:20:04 +0000251struct padded_vnet_hdr {
Michael S. Tsirkin012873d2014-10-24 16:55:57 +0300252 struct virtio_net_hdr_mrg_rxbuf hdr;
Shirley Ma9ab86bb2010-01-29 03:20:04 +0000253 /*
Michael S. Tsirkin012873d2014-10-24 16:55:57 +0300254 * hdr is in a separate sg buffer, and data sg buffer shares same page
255 * with this header sg. This padding makes next sg 16 byte aligned
256 * after the header.
Shirley Ma9ab86bb2010-01-29 03:20:04 +0000257 */
Michael S. Tsirkin012873d2014-10-24 16:55:57 +0300258 char padding[4];
Shirley Ma9ab86bb2010-01-29 03:20:04 +0000259};
260
Jason Wang986a4f42012-12-07 07:04:56 +0000261/* Converting between virtqueue no. and kernel tx/rx queue no.
262 * 0:rx0 1:tx0 2:rx1 3:tx1 ... 2N:rxN 2N+1:txN 2N+2:cvq
263 */
264static int vq2txq(struct virtqueue *vq)
265{
Rusty Russell9d0ca6e2013-03-21 14:17:34 +0000266 return (vq->index - 1) / 2;
Jason Wang986a4f42012-12-07 07:04:56 +0000267}
268
269static int txq2vq(int txq)
270{
271 return txq * 2 + 1;
272}
273
274static int vq2rxq(struct virtqueue *vq)
275{
Rusty Russell9d0ca6e2013-03-21 14:17:34 +0000276 return vq->index / 2;
Jason Wang986a4f42012-12-07 07:04:56 +0000277}
278
279static int rxq2vq(int rxq)
280{
281 return rxq * 2;
282}
283
Michael S. Tsirkin012873d2014-10-24 16:55:57 +0300284static inline struct virtio_net_hdr_mrg_rxbuf *skb_vnet_hdr(struct sk_buff *skb)
Rusty Russell296f96f2007-10-22 11:03:37 +1000285{
Michael S. Tsirkin012873d2014-10-24 16:55:57 +0300286 return (struct virtio_net_hdr_mrg_rxbuf *)skb->cb;
Rusty Russell296f96f2007-10-22 11:03:37 +1000287}
288
Shirley Ma9ab86bb2010-01-29 03:20:04 +0000289/*
290 * private is used to chain pages for big packets, put the whole
291 * most recent used list in the beginning for reuse
292 */
Jason Wange9d74172012-12-07 07:04:55 +0000293static void give_pages(struct receive_queue *rq, struct page *page)
Rusty Russellfb6813f2008-07-25 12:06:01 -0500294{
Shirley Ma9ab86bb2010-01-29 03:20:04 +0000295 struct page *end;
296
Jason Wange9d74172012-12-07 07:04:55 +0000297 /* Find end of list, sew whole thing into vi->rq.pages. */
Shirley Ma9ab86bb2010-01-29 03:20:04 +0000298 for (end = page; end->private; end = (struct page *)end->private);
Jason Wange9d74172012-12-07 07:04:55 +0000299 end->private = (unsigned long)rq->pages;
300 rq->pages = page;
Rusty Russellfb6813f2008-07-25 12:06:01 -0500301}
302
Jason Wange9d74172012-12-07 07:04:55 +0000303static struct page *get_a_page(struct receive_queue *rq, gfp_t gfp_mask)
Rusty Russellfb6813f2008-07-25 12:06:01 -0500304{
Jason Wange9d74172012-12-07 07:04:55 +0000305 struct page *p = rq->pages;
Rusty Russellfb6813f2008-07-25 12:06:01 -0500306
Shirley Ma9ab86bb2010-01-29 03:20:04 +0000307 if (p) {
Jason Wange9d74172012-12-07 07:04:55 +0000308 rq->pages = (struct page *)p->private;
Shirley Ma9ab86bb2010-01-29 03:20:04 +0000309 /* clear private here, it is used to chain pages */
310 p->private = 0;
311 } else
Rusty Russellfb6813f2008-07-25 12:06:01 -0500312 p = alloc_page(gfp_mask);
313 return p;
314}
315
Willem de Bruijne4e84522017-04-24 13:49:26 -0400316static void virtqueue_napi_schedule(struct napi_struct *napi,
317 struct virtqueue *vq)
318{
319 if (napi_schedule_prep(napi)) {
320 virtqueue_disable_cb(vq);
321 __napi_schedule(napi);
322 }
323}
324
325static void virtqueue_napi_complete(struct napi_struct *napi,
326 struct virtqueue *vq, int processed)
327{
328 int opaque;
329
330 opaque = virtqueue_enable_cb_prepare(vq);
Toshiaki Makitafdaa7672017-12-07 13:15:15 +0900331 if (napi_complete_done(napi, processed)) {
332 if (unlikely(virtqueue_poll(vq, opaque)))
333 virtqueue_napi_schedule(napi, vq);
334 } else {
335 virtqueue_disable_cb(vq);
336 }
Willem de Bruijne4e84522017-04-24 13:49:26 -0400337}
338
Jason Wange9d74172012-12-07 07:04:55 +0000339static void skb_xmit_done(struct virtqueue *vq)
Rusty Russell296f96f2007-10-22 11:03:37 +1000340{
Jason Wange9d74172012-12-07 07:04:55 +0000341 struct virtnet_info *vi = vq->vdev->priv;
Willem de Bruijnb92f1e62017-04-24 13:49:27 -0400342 struct napi_struct *napi = &vi->sq[vq2txq(vq)].napi;
Rusty Russell296f96f2007-10-22 11:03:37 +1000343
Rusty Russell2cb9c6b2008-02-04 23:50:07 -0500344 /* Suppress further interrupts. */
Jason Wange9d74172012-12-07 07:04:55 +0000345 virtqueue_disable_cb(vq);
Rusty Russell11a3a152008-05-26 17:48:13 +1000346
Willem de Bruijnb92f1e62017-04-24 13:49:27 -0400347 if (napi->weight)
348 virtqueue_napi_schedule(napi, vq);
349 else
350 /* We were probably waiting for more output buffers. */
351 netif_wake_subqueue(vi->dev, vq2txq(vq));
Rusty Russell296f96f2007-10-22 11:03:37 +1000352}
353
Jason Wang28b39bc2017-07-19 16:54:46 +0800354#define MRG_CTX_HEADER_SHIFT 22
355static void *mergeable_len_to_ctx(unsigned int truesize,
356 unsigned int headroom)
357{
358 return (void *)(unsigned long)((headroom << MRG_CTX_HEADER_SHIFT) | truesize);
359}
360
361static unsigned int mergeable_ctx_to_headroom(void *mrg_ctx)
362{
363 return (unsigned long)mrg_ctx >> MRG_CTX_HEADER_SHIFT;
364}
365
366static unsigned int mergeable_ctx_to_truesize(void *mrg_ctx)
367{
368 return (unsigned long)mrg_ctx & ((1 << MRG_CTX_HEADER_SHIFT) - 1);
369}
370
Mike Waychison34646452012-01-04 12:52:32 +0000371/* Called from bottom half context */
Michael S. Tsirkin946fa562014-10-24 00:12:10 +0300372static struct sk_buff *page_to_skb(struct virtnet_info *vi,
373 struct receive_queue *rq,
Michael Dalton2613af02013-10-28 15:44:18 -0700374 struct page *page, unsigned int offset,
375 unsigned int len, unsigned int truesize)
Shirley Ma9ab86bb2010-01-29 03:20:04 +0000376{
377 struct sk_buff *skb;
Michael S. Tsirkin012873d2014-10-24 16:55:57 +0300378 struct virtio_net_hdr_mrg_rxbuf *hdr;
Michael Dalton2613af02013-10-28 15:44:18 -0700379 unsigned int copy, hdr_len, hdr_padded_len;
Shirley Ma9ab86bb2010-01-29 03:20:04 +0000380 char *p;
381
Michael Dalton2613af02013-10-28 15:44:18 -0700382 p = page_address(page) + offset;
Shirley Ma9ab86bb2010-01-29 03:20:04 +0000383
384 /* copy small packet so we can reuse these pages for small data */
Paolo Abenic67f5db2016-03-17 15:44:00 +0100385 skb = napi_alloc_skb(&rq->napi, GOOD_COPY_LEN);
Shirley Ma9ab86bb2010-01-29 03:20:04 +0000386 if (unlikely(!skb))
387 return NULL;
388
389 hdr = skb_vnet_hdr(skb);
390
Michael S. Tsirkin012873d2014-10-24 16:55:57 +0300391 hdr_len = vi->hdr_len;
392 if (vi->mergeable_rx_bufs)
stephen hemmingera4a76502017-08-15 10:29:17 -0700393 hdr_padded_len = sizeof(*hdr);
Michael S. Tsirkin012873d2014-10-24 16:55:57 +0300394 else
Michael Dalton2613af02013-10-28 15:44:18 -0700395 hdr_padded_len = sizeof(struct padded_vnet_hdr);
Shirley Ma9ab86bb2010-01-29 03:20:04 +0000396
397 memcpy(hdr, p, hdr_len);
398
399 len -= hdr_len;
Michael Dalton2613af02013-10-28 15:44:18 -0700400 offset += hdr_padded_len;
401 p += hdr_padded_len;
Shirley Ma9ab86bb2010-01-29 03:20:04 +0000402
403 copy = len;
404 if (copy > skb_tailroom(skb))
405 copy = skb_tailroom(skb);
Johannes Berg59ae1d12017-06-16 14:29:20 +0200406 skb_put_data(skb, p, copy);
Shirley Ma9ab86bb2010-01-29 03:20:04 +0000407
408 len -= copy;
409 offset += copy;
410
Michael Dalton2613af02013-10-28 15:44:18 -0700411 if (vi->mergeable_rx_bufs) {
412 if (len)
413 skb_add_rx_frag(skb, 0, page, offset, len, truesize);
414 else
415 put_page(page);
416 return skb;
417 }
418
Sasha Levine878d782011-09-28 04:40:54 +0000419 /*
420 * Verify that we can indeed put this data into a skb.
421 * This is here to handle cases when the device erroneously
422 * tries to receive more than is possible. This is usually
423 * the case of a broken device.
424 */
425 if (unlikely(len > MAX_SKB_FRAGS * PAGE_SIZE)) {
Amerigo Wangbe443892012-11-08 17:47:28 +0000426 net_dbg_ratelimited("%s: too much data\n", skb->dev->name);
Sasha Levine878d782011-09-28 04:40:54 +0000427 dev_kfree_skb(skb);
428 return NULL;
429 }
Michael Dalton2613af02013-10-28 15:44:18 -0700430 BUG_ON(offset >= PAGE_SIZE);
Shirley Ma9ab86bb2010-01-29 03:20:04 +0000431 while (len) {
Michael Dalton2613af02013-10-28 15:44:18 -0700432 unsigned int frag_size = min((unsigned)PAGE_SIZE - offset, len);
433 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, page, offset,
434 frag_size, truesize);
435 len -= frag_size;
Shirley Ma9ab86bb2010-01-29 03:20:04 +0000436 page = (struct page *)page->private;
437 offset = 0;
438 }
439
440 if (page)
Jason Wange9d74172012-12-07 07:04:55 +0000441 give_pages(rq, page);
Shirley Ma9ab86bb2010-01-29 03:20:04 +0000442
443 return skb;
444}
445
Jesper Dangaard Brouer735fc402018-05-24 16:46:12 +0200446static int __virtnet_xdp_xmit_one(struct virtnet_info *vi,
447 struct send_queue *sq,
448 struct xdp_frame *xdpf)
John Fastabend56434a02016-12-15 12:14:13 -0800449{
John Fastabend56434a02016-12-15 12:14:13 -0800450 struct virtio_net_hdr_mrg_rxbuf *hdr;
John Fastabend56434a02016-12-15 12:14:13 -0800451 int err;
452
Jesper Dangaard Brouercac320c2018-04-17 16:45:52 +0200453 /* virtqueue want to use data area in-front of packet */
454 if (unlikely(xdpf->metasize > 0))
455 return -EOPNOTSUPP;
456
457 if (unlikely(xdpf->headroom < vi->hdr_len))
458 return -EOVERFLOW;
459
460 /* Make room for virtqueue hdr (also change xdpf->headroom?) */
461 xdpf->data -= vi->hdr_len;
Jason Wangf6b10202017-02-21 16:46:28 +0800462 /* Zero header and leave csum up to XDP layers */
Jesper Dangaard Brouercac320c2018-04-17 16:45:52 +0200463 hdr = xdpf->data;
Jason Wangf6b10202017-02-21 16:46:28 +0800464 memset(hdr, 0, vi->hdr_len);
Jesper Dangaard Brouercac320c2018-04-17 16:45:52 +0200465 xdpf->len += vi->hdr_len;
John Fastabend56434a02016-12-15 12:14:13 -0800466
Jesper Dangaard Brouercac320c2018-04-17 16:45:52 +0200467 sg_init_one(sq->sg, xdpf->data, xdpf->len);
Jason Wangbb91acc2016-12-23 22:37:32 +0800468
Jesper Dangaard Brouercac320c2018-04-17 16:45:52 +0200469 err = virtqueue_add_outbuf(sq->vq, sq->sg, 1, xdpf, GFP_ATOMIC);
Jesper Dangaard Brouer11b7d8972018-02-20 14:32:15 +0100470 if (unlikely(err))
Jesper Dangaard Brouercac320c2018-04-17 16:45:52 +0200471 return -ENOSPC; /* Caller handle free/refcnt */
John Fastabend56434a02016-12-15 12:14:13 -0800472
Jesper Dangaard Brouercac320c2018-04-17 16:45:52 +0200473 return 0;
John Fastabend56434a02016-12-15 12:14:13 -0800474}
475
Toshiaki Makita2a435652018-07-23 23:36:07 +0900476static struct send_queue *virtnet_xdp_sq(struct virtnet_info *vi)
477{
478 unsigned int qp;
479
480 qp = vi->curr_queue_pairs - vi->xdp_queue_pairs + smp_processor_id();
481 return &vi->sq[qp];
482}
483
Jesper Dangaard Brouer735fc402018-05-24 16:46:12 +0200484static int __virtnet_xdp_tx_xmit(struct virtnet_info *vi,
485 struct xdp_frame *xdpf)
486{
487 struct xdp_frame *xdpf_sent;
488 struct send_queue *sq;
489 unsigned int len;
Jesper Dangaard Brouer735fc402018-05-24 16:46:12 +0200490
Toshiaki Makita2a435652018-07-23 23:36:07 +0900491 sq = virtnet_xdp_sq(vi);
Jesper Dangaard Brouer735fc402018-05-24 16:46:12 +0200492
493 /* Free up any pending old buffers before queueing new ones. */
494 while ((xdpf_sent = virtqueue_get_buf(sq->vq, &len)) != NULL)
495 xdp_return_frame(xdpf_sent);
496
497 return __virtnet_xdp_xmit_one(vi, sq, xdpf);
498}
499
500static int virtnet_xdp_xmit(struct net_device *dev,
Jesper Dangaard Brouer42b33462018-05-31 10:59:47 +0200501 int n, struct xdp_frame **frames, u32 flags)
Jason Wang186b3c92017-09-19 17:42:43 +0800502{
503 struct virtnet_info *vi = netdev_priv(dev);
Jesper Dangaard Brouer8dcc5b02018-02-20 14:32:20 +0100504 struct receive_queue *rq = vi->rq;
Jesper Dangaard Brouer735fc402018-05-24 16:46:12 +0200505 struct xdp_frame *xdpf_sent;
Jesper Dangaard Brouer8dcc5b02018-02-20 14:32:20 +0100506 struct bpf_prog *xdp_prog;
Jesper Dangaard Brouer735fc402018-05-24 16:46:12 +0200507 struct send_queue *sq;
508 unsigned int len;
Jesper Dangaard Brouer735fc402018-05-24 16:46:12 +0200509 int drops = 0;
Toshiaki Makita5b8f3c82018-07-23 23:36:08 +0900510 int ret, err;
Jesper Dangaard Brouer735fc402018-05-24 16:46:12 +0200511 int i;
512
Toshiaki Makita2a435652018-07-23 23:36:07 +0900513 sq = virtnet_xdp_sq(vi);
Jason Wang186b3c92017-09-19 17:42:43 +0800514
Toshiaki Makita5b8f3c82018-07-23 23:36:08 +0900515 if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK)) {
516 ret = -EINVAL;
517 drops = n;
518 goto out;
519 }
520
Jesper Dangaard Brouer8dcc5b02018-02-20 14:32:20 +0100521 /* Only allow ndo_xdp_xmit if XDP is loaded on dev, as this
522 * indicate XDP resources have been successfully allocated.
523 */
524 xdp_prog = rcu_dereference(rq->xdp_prog);
Toshiaki Makita5b8f3c82018-07-23 23:36:08 +0900525 if (!xdp_prog) {
526 ret = -ENXIO;
527 drops = n;
528 goto out;
529 }
Jesper Dangaard Brouer8dcc5b02018-02-20 14:32:20 +0100530
Jesper Dangaard Brouer735fc402018-05-24 16:46:12 +0200531 /* Free up any pending old buffers before queueing new ones. */
532 while ((xdpf_sent = virtqueue_get_buf(sq->vq, &len)) != NULL)
533 xdp_return_frame(xdpf_sent);
534
535 for (i = 0; i < n; i++) {
536 struct xdp_frame *xdpf = frames[i];
537
538 err = __virtnet_xdp_xmit_one(vi, sq, xdpf);
539 if (err) {
540 xdp_return_frame_rx_napi(xdpf);
541 drops++;
542 }
543 }
Toshiaki Makita5b8f3c82018-07-23 23:36:08 +0900544 ret = n - drops;
Jesper Dangaard Brouer5d274cb2018-05-31 11:00:08 +0200545
546 if (flags & XDP_XMIT_FLUSH)
547 virtqueue_kick(sq->vq);
Toshiaki Makita5b8f3c82018-07-23 23:36:08 +0900548out:
549 u64_stats_update_begin(&sq->stats.syncp);
550 sq->stats.xdp_tx += n;
551 sq->stats.xdp_tx_drops += drops;
552 u64_stats_update_end(&sq->stats.syncp);
Jesper Dangaard Brouer5d274cb2018-05-31 11:00:08 +0200553
Toshiaki Makita5b8f3c82018-07-23 23:36:08 +0900554 return ret;
Jason Wang186b3c92017-09-19 17:42:43 +0800555}
556
Jason Wangf6b10202017-02-21 16:46:28 +0800557static unsigned int virtnet_get_headroom(struct virtnet_info *vi)
558{
559 return vi->xdp_queue_pairs ? VIRTIO_XDP_HEADROOM : 0;
560}
561
Jason Wang4941d472017-07-19 16:54:48 +0800562/* We copy the packet for XDP in the following cases:
563 *
564 * 1) Packet is scattered across multiple rx buffers.
565 * 2) Headroom space is insufficient.
566 *
567 * This is inefficient but it's a temporary condition that
568 * we hit right after XDP is enabled and until queue is refilled
569 * with large buffers with sufficient headroom - so it should affect
570 * at most queue size packets.
571 * Afterwards, the conditions to enable
572 * XDP should preclude the underlying device from sending packets
573 * across multiple buffers (num_buf > 1), and we make sure buffers
574 * have enough headroom.
John Fastabend72979a62016-12-15 12:14:36 -0800575 */
576static struct page *xdp_linearize_page(struct receive_queue *rq,
Jason Wang56a86f82016-12-23 22:37:26 +0800577 u16 *num_buf,
John Fastabend72979a62016-12-15 12:14:36 -0800578 struct page *p,
579 int offset,
Jason Wang4941d472017-07-19 16:54:48 +0800580 int page_off,
John Fastabend72979a62016-12-15 12:14:36 -0800581 unsigned int *len)
582{
583 struct page *page = alloc_page(GFP_ATOMIC);
John Fastabend72979a62016-12-15 12:14:36 -0800584
585 if (!page)
586 return NULL;
587
588 memcpy(page_address(page) + page_off, page_address(p) + offset, *len);
589 page_off += *len;
590
Jason Wang56a86f82016-12-23 22:37:26 +0800591 while (--*num_buf) {
Jason Wang3cc81a92018-03-02 17:29:14 +0800592 int tailroom = SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
John Fastabend72979a62016-12-15 12:14:36 -0800593 unsigned int buflen;
John Fastabend72979a62016-12-15 12:14:36 -0800594 void *buf;
595 int off;
596
Michael S. Tsirkin680557c2017-03-06 21:29:47 +0200597 buf = virtqueue_get_buf(rq->vq, &buflen);
598 if (unlikely(!buf))
John Fastabend72979a62016-12-15 12:14:36 -0800599 goto err_buf;
600
John Fastabend72979a62016-12-15 12:14:36 -0800601 p = virt_to_head_page(buf);
602 off = buf - page_address(p);
603
Jason Wang56a86f82016-12-23 22:37:26 +0800604 /* guard against a misconfigured or uncooperative backend that
605 * is sending packet larger than the MTU.
606 */
Jason Wang3cc81a92018-03-02 17:29:14 +0800607 if ((page_off + buflen + tailroom) > PAGE_SIZE) {
Jason Wang56a86f82016-12-23 22:37:26 +0800608 put_page(p);
609 goto err_buf;
610 }
611
John Fastabend72979a62016-12-15 12:14:36 -0800612 memcpy(page_address(page) + page_off,
613 page_address(p) + off, buflen);
614 page_off += buflen;
Jason Wang56a86f82016-12-23 22:37:26 +0800615 put_page(p);
John Fastabend72979a62016-12-15 12:14:36 -0800616 }
617
John Fastabend2de2f7f2017-02-02 19:16:29 -0800618 /* Headroom does not contribute to packet length */
619 *len = page_off - VIRTIO_XDP_HEADROOM;
John Fastabend72979a62016-12-15 12:14:36 -0800620 return page;
621err_buf:
622 __free_pages(page, 0);
623 return NULL;
624}
625
Jason Wang4941d472017-07-19 16:54:48 +0800626static struct sk_buff *receive_small(struct net_device *dev,
627 struct virtnet_info *vi,
628 struct receive_queue *rq,
629 void *buf, void *ctx,
Jason Wang186b3c92017-09-19 17:42:43 +0800630 unsigned int len,
Toshiaki Makita7d9d60f2018-07-23 23:36:04 +0900631 unsigned int *xdp_xmit,
Toshiaki Makitaa0929a42018-07-23 23:36:05 +0900632 struct virtnet_rx_stats *stats)
Jason Wang4941d472017-07-19 16:54:48 +0800633{
634 struct sk_buff *skb;
635 struct bpf_prog *xdp_prog;
636 unsigned int xdp_headroom = (unsigned long)ctx;
637 unsigned int header_offset = VIRTNET_RX_PAD + xdp_headroom;
638 unsigned int headroom = vi->hdr_len + header_offset;
639 unsigned int buflen = SKB_DATA_ALIGN(GOOD_PACKET_LEN + headroom) +
640 SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
641 struct page *page = virt_to_head_page(buf);
Jesper Dangaard Brouer11b7d8972018-02-20 14:32:15 +0100642 unsigned int delta = 0;
Jason Wang4941d472017-07-19 16:54:48 +0800643 struct page *xdp_page;
Jesper Dangaard Brouer11b7d8972018-02-20 14:32:15 +0100644 int err;
645
Jason Wang4941d472017-07-19 16:54:48 +0800646 len -= vi->hdr_len;
Toshiaki Makitaa0929a42018-07-23 23:36:05 +0900647 stats->rx.bytes += len;
Jason Wang4941d472017-07-19 16:54:48 +0800648
649 rcu_read_lock();
650 xdp_prog = rcu_dereference(rq->xdp_prog);
651 if (xdp_prog) {
652 struct virtio_net_hdr_mrg_rxbuf *hdr = buf + header_offset;
Jesper Dangaard Brouer44fa2db2018-04-17 16:46:37 +0200653 struct xdp_frame *xdpf;
Jason Wang4941d472017-07-19 16:54:48 +0800654 struct xdp_buff xdp;
655 void *orig_data;
656 u32 act;
657
Jesper Dangaard Brouer95dbe9e2018-02-20 14:32:10 +0100658 if (unlikely(hdr->hdr.gso_type))
Jason Wang4941d472017-07-19 16:54:48 +0800659 goto err_xdp;
660
661 if (unlikely(xdp_headroom < virtnet_get_headroom(vi))) {
662 int offset = buf - page_address(page) + header_offset;
663 unsigned int tlen = len + vi->hdr_len;
664 u16 num_buf = 1;
665
666 xdp_headroom = virtnet_get_headroom(vi);
667 header_offset = VIRTNET_RX_PAD + xdp_headroom;
668 headroom = vi->hdr_len + header_offset;
669 buflen = SKB_DATA_ALIGN(GOOD_PACKET_LEN + headroom) +
670 SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
671 xdp_page = xdp_linearize_page(rq, &num_buf, page,
672 offset, header_offset,
673 &tlen);
674 if (!xdp_page)
675 goto err_xdp;
676
677 buf = page_address(xdp_page);
678 put_page(page);
679 page = xdp_page;
680 }
681
682 xdp.data_hard_start = buf + VIRTNET_RX_PAD + vi->hdr_len;
683 xdp.data = xdp.data_hard_start + xdp_headroom;
Daniel Borkmannde8f3a82017-09-25 02:25:51 +0200684 xdp_set_data_meta_invalid(&xdp);
Jason Wang4941d472017-07-19 16:54:48 +0800685 xdp.data_end = xdp.data + len;
Jesper Dangaard Brouer754b8a22018-01-03 11:26:04 +0100686 xdp.rxq = &rq->xdp_rxq;
Jason Wang4941d472017-07-19 16:54:48 +0800687 orig_data = xdp.data;
688 act = bpf_prog_run_xdp(xdp_prog, &xdp);
Toshiaki Makita5b8f3c82018-07-23 23:36:08 +0900689 stats->rx.xdp_packets++;
Jason Wang4941d472017-07-19 16:54:48 +0800690
691 switch (act) {
692 case XDP_PASS:
693 /* Recalculate length in case bpf program changed it */
694 delta = orig_data - xdp.data;
Nikita V. Shirokov6870de42018-04-17 21:42:20 -0700695 len = xdp.data_end - xdp.data;
Jason Wang4941d472017-07-19 16:54:48 +0800696 break;
697 case XDP_TX:
Toshiaki Makita5b8f3c82018-07-23 23:36:08 +0900698 stats->rx.xdp_tx++;
Jesper Dangaard Brouer44fa2db2018-04-17 16:46:37 +0200699 xdpf = convert_to_xdp_frame(&xdp);
700 if (unlikely(!xdpf))
701 goto err_xdp;
Toshiaki Makita5b8f3c82018-07-23 23:36:08 +0900702 stats->tx.xdp_tx++;
Jesper Dangaard Brouer735fc402018-05-24 16:46:12 +0200703 err = __virtnet_xdp_tx_xmit(vi, xdpf);
Jesper Dangaard Brouercac320c2018-04-17 16:45:52 +0200704 if (unlikely(err)) {
Toshiaki Makita5b8f3c82018-07-23 23:36:08 +0900705 stats->tx.xdp_tx_drops++;
Jason Wang4941d472017-07-19 16:54:48 +0800706 trace_xdp_exception(vi->dev, xdp_prog, act);
Jesper Dangaard Brouer11b7d8972018-02-20 14:32:15 +0100707 goto err_xdp;
708 }
Jesper Dangaard Brouer2471c752018-06-26 17:39:58 +0200709 *xdp_xmit |= VIRTIO_XDP_TX;
Jason Wang186b3c92017-09-19 17:42:43 +0800710 rcu_read_unlock();
711 goto xdp_xmit;
712 case XDP_REDIRECT:
Toshiaki Makita5b8f3c82018-07-23 23:36:08 +0900713 stats->rx.xdp_redirects++;
Jason Wang186b3c92017-09-19 17:42:43 +0800714 err = xdp_do_redirect(dev, &xdp, xdp_prog);
Jesper Dangaard Brouer11b7d8972018-02-20 14:32:15 +0100715 if (err)
716 goto err_xdp;
Jesper Dangaard Brouer2471c752018-06-26 17:39:58 +0200717 *xdp_xmit |= VIRTIO_XDP_REDIR;
Jason Wang4941d472017-07-19 16:54:48 +0800718 rcu_read_unlock();
719 goto xdp_xmit;
720 default:
721 bpf_warn_invalid_xdp_action(act);
722 case XDP_ABORTED:
723 trace_xdp_exception(vi->dev, xdp_prog, act);
724 case XDP_DROP:
725 goto err_xdp;
726 }
727 }
728 rcu_read_unlock();
729
730 skb = build_skb(buf, buflen);
731 if (!skb) {
732 put_page(page);
733 goto err;
734 }
735 skb_reserve(skb, headroom - delta);
Nikita V. Shirokov6870de42018-04-17 21:42:20 -0700736 skb_put(skb, len);
Jason Wang4941d472017-07-19 16:54:48 +0800737 if (!delta) {
738 buf += header_offset;
739 memcpy(skb_vnet_hdr(skb), buf, vi->hdr_len);
740 } /* keep zeroed vnet hdr since packet was changed by bpf */
741
742err:
743 return skb;
744
745err_xdp:
746 rcu_read_unlock();
Toshiaki Makita5b8f3c82018-07-23 23:36:08 +0900747 stats->rx.xdp_drops++;
Toshiaki Makita2c4a2f72018-07-23 23:36:06 +0900748 stats->rx.drops++;
Jason Wang4941d472017-07-19 16:54:48 +0800749 put_page(page);
750xdp_xmit:
751 return NULL;
752}
753
754static struct sk_buff *receive_big(struct net_device *dev,
755 struct virtnet_info *vi,
756 struct receive_queue *rq,
757 void *buf,
Toshiaki Makita7d9d60f2018-07-23 23:36:04 +0900758 unsigned int len,
Toshiaki Makitaa0929a42018-07-23 23:36:05 +0900759 struct virtnet_rx_stats *stats)
Jason Wang4941d472017-07-19 16:54:48 +0800760{
761 struct page *page = buf;
762 struct sk_buff *skb = page_to_skb(vi, rq, page, 0, len, PAGE_SIZE);
763
Toshiaki Makitaa0929a42018-07-23 23:36:05 +0900764 stats->rx.bytes += len - vi->hdr_len;
Jason Wang4941d472017-07-19 16:54:48 +0800765 if (unlikely(!skb))
766 goto err;
767
768 return skb;
769
770err:
Toshiaki Makita2c4a2f72018-07-23 23:36:06 +0900771 stats->rx.drops++;
Jason Wang4941d472017-07-19 16:54:48 +0800772 give_pages(rq, page);
773 return NULL;
774}
775
Michael S. Tsirkin8fc3b9e2013-11-28 13:30:55 +0200776static struct sk_buff *receive_mergeable(struct net_device *dev,
Michael S. Tsirkinfdd819b2014-10-07 16:39:48 +0200777 struct virtnet_info *vi,
Michael S. Tsirkin8fc3b9e2013-11-28 13:30:55 +0200778 struct receive_queue *rq,
Michael S. Tsirkin680557c2017-03-06 21:29:47 +0200779 void *buf,
780 void *ctx,
Jason Wang186b3c92017-09-19 17:42:43 +0800781 unsigned int len,
Toshiaki Makita7d9d60f2018-07-23 23:36:04 +0900782 unsigned int *xdp_xmit,
Toshiaki Makitaa0929a42018-07-23 23:36:05 +0900783 struct virtnet_rx_stats *stats)
Shirley Ma9ab86bb2010-01-29 03:20:04 +0000784{
Michael S. Tsirkin012873d2014-10-24 16:55:57 +0300785 struct virtio_net_hdr_mrg_rxbuf *hdr = buf;
786 u16 num_buf = virtio16_to_cpu(vi->vdev, hdr->num_buffers);
Michael S. Tsirkin8fc3b9e2013-11-28 13:30:55 +0200787 struct page *page = virt_to_head_page(buf);
788 int offset = buf - page_address(page);
John Fastabendf600b692016-12-15 12:13:24 -0800789 struct sk_buff *head_skb, *curr_skb;
790 struct bpf_prog *xdp_prog;
791 unsigned int truesize;
Jason Wang4941d472017-07-19 16:54:48 +0800792 unsigned int headroom = mergeable_ctx_to_headroom(ctx);
Jason Wang3cc81a92018-03-02 17:29:14 +0800793 int err;
Michael Daltonab7db912014-01-16 22:23:27 -0800794
John Fastabend56434a02016-12-15 12:14:13 -0800795 head_skb = NULL;
Toshiaki Makitaa0929a42018-07-23 23:36:05 +0900796 stats->rx.bytes += len - vi->hdr_len;
John Fastabend56434a02016-12-15 12:14:13 -0800797
John Fastabendf600b692016-12-15 12:13:24 -0800798 rcu_read_lock();
799 xdp_prog = rcu_dereference(rq->xdp_prog);
800 if (xdp_prog) {
Jesper Dangaard Brouer44fa2db2018-04-17 16:46:37 +0200801 struct xdp_frame *xdpf;
John Fastabend72979a62016-12-15 12:14:36 -0800802 struct page *xdp_page;
John Fastabend0354e4d2017-02-02 19:15:01 -0800803 struct xdp_buff xdp;
John Fastabend0354e4d2017-02-02 19:15:01 -0800804 void *data;
John Fastabendf600b692016-12-15 12:13:24 -0800805 u32 act;
806
Jason Wang3d62b2a2018-05-22 11:44:31 +0800807 /* Transient failure which in theory could occur if
808 * in-flight packets from before XDP was enabled reach
809 * the receive path after XDP is loaded.
810 */
811 if (unlikely(hdr->hdr.gso_type))
812 goto err_xdp;
813
Jason Wang3cc81a92018-03-02 17:29:14 +0800814 /* This happens when rx buffer size is underestimated
815 * or headroom is not enough because of the buffer
816 * was refilled before XDP is set. This should only
817 * happen for the first several packets, so we don't
818 * care much about its performance.
819 */
Jason Wang4941d472017-07-19 16:54:48 +0800820 if (unlikely(num_buf > 1 ||
821 headroom < virtnet_get_headroom(vi))) {
John Fastabend72979a62016-12-15 12:14:36 -0800822 /* linearize data for XDP */
Jason Wang56a86f82016-12-23 22:37:26 +0800823 xdp_page = xdp_linearize_page(rq, &num_buf,
Jason Wang4941d472017-07-19 16:54:48 +0800824 page, offset,
825 VIRTIO_XDP_HEADROOM,
826 &len);
John Fastabend72979a62016-12-15 12:14:36 -0800827 if (!xdp_page)
828 goto err_xdp;
John Fastabend2de2f7f2017-02-02 19:16:29 -0800829 offset = VIRTIO_XDP_HEADROOM;
John Fastabend72979a62016-12-15 12:14:36 -0800830 } else {
831 xdp_page = page;
John Fastabendf600b692016-12-15 12:13:24 -0800832 }
833
John Fastabend2de2f7f2017-02-02 19:16:29 -0800834 /* Allow consuming headroom but reserve enough space to push
835 * the descriptor on if we get an XDP_TX return code.
836 */
John Fastabend0354e4d2017-02-02 19:15:01 -0800837 data = page_address(xdp_page) + offset;
John Fastabend2de2f7f2017-02-02 19:16:29 -0800838 xdp.data_hard_start = data - VIRTIO_XDP_HEADROOM + vi->hdr_len;
John Fastabend0354e4d2017-02-02 19:15:01 -0800839 xdp.data = data + vi->hdr_len;
Daniel Borkmannde8f3a82017-09-25 02:25:51 +0200840 xdp_set_data_meta_invalid(&xdp);
John Fastabend0354e4d2017-02-02 19:15:01 -0800841 xdp.data_end = xdp.data + (len - vi->hdr_len);
Jesper Dangaard Brouer754b8a22018-01-03 11:26:04 +0100842 xdp.rxq = &rq->xdp_rxq;
843
John Fastabend0354e4d2017-02-02 19:15:01 -0800844 act = bpf_prog_run_xdp(xdp_prog, &xdp);
Toshiaki Makita5b8f3c82018-07-23 23:36:08 +0900845 stats->rx.xdp_packets++;
John Fastabend0354e4d2017-02-02 19:15:01 -0800846
John Fastabend56434a02016-12-15 12:14:13 -0800847 switch (act) {
848 case XDP_PASS:
John Fastabend2de2f7f2017-02-02 19:16:29 -0800849 /* recalculate offset to account for any header
850 * adjustments. Note other cases do not build an
851 * skb and avoid using offset
852 */
853 offset = xdp.data -
854 page_address(xdp_page) - vi->hdr_len;
855
Nikita V. Shirokov6870de42018-04-17 21:42:20 -0700856 /* recalculate len if xdp.data or xdp.data_end were
857 * adjusted
858 */
Nikita V. Shirokovaaa64522018-04-22 21:16:48 -0700859 len = xdp.data_end - xdp.data + vi->hdr_len;
Jason Wang1830f892016-12-23 22:37:27 +0800860 /* We can only create skb based on xdp_page. */
861 if (unlikely(xdp_page != page)) {
862 rcu_read_unlock();
863 put_page(page);
864 head_skb = page_to_skb(vi, rq, xdp_page,
John Fastabend2de2f7f2017-02-02 19:16:29 -0800865 offset, len, PAGE_SIZE);
Jason Wang1830f892016-12-23 22:37:27 +0800866 return head_skb;
867 }
John Fastabend56434a02016-12-15 12:14:13 -0800868 break;
869 case XDP_TX:
Toshiaki Makita5b8f3c82018-07-23 23:36:08 +0900870 stats->rx.xdp_tx++;
Jesper Dangaard Brouer44fa2db2018-04-17 16:46:37 +0200871 xdpf = convert_to_xdp_frame(&xdp);
872 if (unlikely(!xdpf))
873 goto err_xdp;
Toshiaki Makita5b8f3c82018-07-23 23:36:08 +0900874 stats->tx.xdp_tx++;
Jesper Dangaard Brouer735fc402018-05-24 16:46:12 +0200875 err = __virtnet_xdp_tx_xmit(vi, xdpf);
Jesper Dangaard Brouercac320c2018-04-17 16:45:52 +0200876 if (unlikely(err)) {
Toshiaki Makita5b8f3c82018-07-23 23:36:08 +0900877 stats->tx.xdp_tx_drops++;
John Fastabend0354e4d2017-02-02 19:15:01 -0800878 trace_xdp_exception(vi->dev, xdp_prog, act);
Jesper Dangaard Brouer11b7d8972018-02-20 14:32:15 +0100879 if (unlikely(xdp_page != page))
880 put_page(xdp_page);
881 goto err_xdp;
882 }
Jesper Dangaard Brouer2471c752018-06-26 17:39:58 +0200883 *xdp_xmit |= VIRTIO_XDP_TX;
John Fastabend72979a62016-12-15 12:14:36 -0800884 if (unlikely(xdp_page != page))
Jason Wang5d458a12018-05-22 11:44:29 +0800885 put_page(page);
John Fastabend56434a02016-12-15 12:14:13 -0800886 rcu_read_unlock();
887 goto xdp_xmit;
Jason Wang3cc81a92018-03-02 17:29:14 +0800888 case XDP_REDIRECT:
Toshiaki Makita5b8f3c82018-07-23 23:36:08 +0900889 stats->rx.xdp_redirects++;
Jason Wang3cc81a92018-03-02 17:29:14 +0800890 err = xdp_do_redirect(dev, &xdp, xdp_prog);
891 if (err) {
892 if (unlikely(xdp_page != page))
893 put_page(xdp_page);
894 goto err_xdp;
895 }
Jesper Dangaard Brouer2471c752018-06-26 17:39:58 +0200896 *xdp_xmit |= VIRTIO_XDP_REDIR;
Jason Wang3cc81a92018-03-02 17:29:14 +0800897 if (unlikely(xdp_page != page))
Jason Wang68904182018-05-22 11:44:28 +0800898 put_page(page);
Jason Wang3cc81a92018-03-02 17:29:14 +0800899 rcu_read_unlock();
900 goto xdp_xmit;
John Fastabend56434a02016-12-15 12:14:13 -0800901 default:
John Fastabend0354e4d2017-02-02 19:15:01 -0800902 bpf_warn_invalid_xdp_action(act);
903 case XDP_ABORTED:
904 trace_xdp_exception(vi->dev, xdp_prog, act);
905 case XDP_DROP:
John Fastabend72979a62016-12-15 12:14:36 -0800906 if (unlikely(xdp_page != page))
907 __free_pages(xdp_page, 0);
John Fastabendf600b692016-12-15 12:13:24 -0800908 goto err_xdp;
John Fastabend56434a02016-12-15 12:14:13 -0800909 }
John Fastabendf600b692016-12-15 12:13:24 -0800910 }
911 rcu_read_unlock();
912
Jason Wang28b39bc2017-07-19 16:54:46 +0800913 truesize = mergeable_ctx_to_truesize(ctx);
914 if (unlikely(len > truesize)) {
Dan Carpenter56da5fd2017-04-06 12:04:47 +0300915 pr_debug("%s: rx error: len %u exceeds truesize %lu\n",
Michael S. Tsirkin680557c2017-03-06 21:29:47 +0200916 dev->name, len, (unsigned long)ctx);
917 dev->stats.rx_length_errors++;
918 goto err_skb;
919 }
Jason Wang28b39bc2017-07-19 16:54:46 +0800920
John Fastabendf600b692016-12-15 12:13:24 -0800921 head_skb = page_to_skb(vi, rq, page, offset, len, truesize);
922 curr_skb = head_skb;
Shirley Ma9ab86bb2010-01-29 03:20:04 +0000923
Michael S. Tsirkin8fc3b9e2013-11-28 13:30:55 +0200924 if (unlikely(!curr_skb))
925 goto err_skb;
Shirley Ma9ab86bb2010-01-29 03:20:04 +0000926 while (--num_buf) {
Michael S. Tsirkin8fc3b9e2013-11-28 13:30:55 +0200927 int num_skb_frags;
928
Michael S. Tsirkin680557c2017-03-06 21:29:47 +0200929 buf = virtqueue_get_buf_ctx(rq->vq, &len, &ctx);
Yunjian Wang03e9f8a2017-12-04 14:02:19 +0800930 if (unlikely(!buf)) {
Michael S. Tsirkin8fc3b9e2013-11-28 13:30:55 +0200931 pr_debug("%s: rx error: %d buffers out of %d missing\n",
Michael S. Tsirkinfdd819b2014-10-07 16:39:48 +0200932 dev->name, num_buf,
Michael S. Tsirkin012873d2014-10-24 16:55:57 +0300933 virtio16_to_cpu(vi->vdev,
934 hdr->num_buffers));
Michael S. Tsirkin8fc3b9e2013-11-28 13:30:55 +0200935 dev->stats.rx_length_errors++;
936 goto err_buf;
Shirley Ma9ab86bb2010-01-29 03:20:04 +0000937 }
Michael S. Tsirkin8fc3b9e2013-11-28 13:30:55 +0200938
Toshiaki Makitaa0929a42018-07-23 23:36:05 +0900939 stats->rx.bytes += len;
Michael S. Tsirkin8fc3b9e2013-11-28 13:30:55 +0200940 page = virt_to_head_page(buf);
Jason Wang28b39bc2017-07-19 16:54:46 +0800941
942 truesize = mergeable_ctx_to_truesize(ctx);
943 if (unlikely(len > truesize)) {
Dan Carpenter56da5fd2017-04-06 12:04:47 +0300944 pr_debug("%s: rx error: len %u exceeds truesize %lu\n",
Michael S. Tsirkin680557c2017-03-06 21:29:47 +0200945 dev->name, len, (unsigned long)ctx);
946 dev->stats.rx_length_errors++;
947 goto err_skb;
948 }
Michael S. Tsirkin8fc3b9e2013-11-28 13:30:55 +0200949
950 num_skb_frags = skb_shinfo(curr_skb)->nr_frags;
Michael Dalton2613af02013-10-28 15:44:18 -0700951 if (unlikely(num_skb_frags == MAX_SKB_FRAGS)) {
952 struct sk_buff *nskb = alloc_skb(0, GFP_ATOMIC);
Michael S. Tsirkin8fc3b9e2013-11-28 13:30:55 +0200953
954 if (unlikely(!nskb))
955 goto err_skb;
Michael Dalton2613af02013-10-28 15:44:18 -0700956 if (curr_skb == head_skb)
957 skb_shinfo(curr_skb)->frag_list = nskb;
958 else
959 curr_skb->next = nskb;
960 curr_skb = nskb;
961 head_skb->truesize += nskb->truesize;
962 num_skb_frags = 0;
963 }
964 if (curr_skb != head_skb) {
965 head_skb->data_len += len;
966 head_skb->len += len;
Michael Daltonfb518792014-01-16 22:23:26 -0800967 head_skb->truesize += truesize;
Michael Dalton2613af02013-10-28 15:44:18 -0700968 }
Michael S. Tsirkin8fc3b9e2013-11-28 13:30:55 +0200969 offset = buf - page_address(page);
Jason Wangba275242013-11-01 14:07:48 +0800970 if (skb_can_coalesce(curr_skb, num_skb_frags, page, offset)) {
971 put_page(page);
972 skb_coalesce_rx_frag(curr_skb, num_skb_frags - 1,
Michael Daltonfb518792014-01-16 22:23:26 -0800973 len, truesize);
Jason Wangba275242013-11-01 14:07:48 +0800974 } else {
975 skb_add_rx_frag(curr_skb, num_skb_frags, page,
Michael Daltonfb518792014-01-16 22:23:26 -0800976 offset, len, truesize);
Jason Wangba275242013-11-01 14:07:48 +0800977 }
Michael S. Tsirkin8fc3b9e2013-11-28 13:30:55 +0200978 }
979
Johannes Berg5377d7582015-08-19 09:48:40 +0200980 ewma_pkt_len_add(&rq->mrg_avg_pkt_len, head_skb->len);
Michael S. Tsirkin8fc3b9e2013-11-28 13:30:55 +0200981 return head_skb;
982
John Fastabendf600b692016-12-15 12:13:24 -0800983err_xdp:
984 rcu_read_unlock();
Toshiaki Makita5b8f3c82018-07-23 23:36:08 +0900985 stats->rx.xdp_drops++;
Michael S. Tsirkin8fc3b9e2013-11-28 13:30:55 +0200986err_skb:
987 put_page(page);
Jason Wang850e0882018-05-22 11:44:30 +0800988 while (num_buf-- > 1) {
Michael S. Tsirkin680557c2017-03-06 21:29:47 +0200989 buf = virtqueue_get_buf(rq->vq, &len);
990 if (unlikely(!buf)) {
Michael S. Tsirkin8fc3b9e2013-11-28 13:30:55 +0200991 pr_debug("%s: rx error: %d buffers missing\n",
992 dev->name, num_buf);
993 dev->stats.rx_length_errors++;
994 break;
995 }
Toshiaki Makitaa0929a42018-07-23 23:36:05 +0900996 stats->rx.bytes += len;
Michael S. Tsirkin680557c2017-03-06 21:29:47 +0200997 page = virt_to_head_page(buf);
Michael S. Tsirkin8fc3b9e2013-11-28 13:30:55 +0200998 put_page(page);
Shirley Ma9ab86bb2010-01-29 03:20:04 +0000999 }
Michael S. Tsirkin8fc3b9e2013-11-28 13:30:55 +02001000err_buf:
Toshiaki Makita2c4a2f72018-07-23 23:36:06 +09001001 stats->rx.drops++;
Michael S. Tsirkin8fc3b9e2013-11-28 13:30:55 +02001002 dev_kfree_skb(head_skb);
John Fastabend56434a02016-12-15 12:14:13 -08001003xdp_xmit:
Michael S. Tsirkin8fc3b9e2013-11-28 13:30:55 +02001004 return NULL;
Shirley Ma9ab86bb2010-01-29 03:20:04 +00001005}
1006
Toshiaki Makita7d9d60f2018-07-23 23:36:04 +09001007static void receive_buf(struct virtnet_info *vi, struct receive_queue *rq,
1008 void *buf, unsigned int len, void **ctx,
Toshiaki Makitaa0929a42018-07-23 23:36:05 +09001009 unsigned int *xdp_xmit,
1010 struct virtnet_rx_stats *stats)
Rusty Russell296f96f2007-10-22 11:03:37 +10001011{
Jason Wange9d74172012-12-07 07:04:55 +00001012 struct net_device *dev = vi->dev;
Shirley Ma9ab86bb2010-01-29 03:20:04 +00001013 struct sk_buff *skb;
Michael S. Tsirkin012873d2014-10-24 16:55:57 +03001014 struct virtio_net_hdr_mrg_rxbuf *hdr;
Rusty Russell296f96f2007-10-22 11:03:37 +10001015
Michael S. Tsirkinbcff3162014-10-24 00:22:11 +03001016 if (unlikely(len < vi->hdr_len + ETH_HLEN)) {
Rusty Russell296f96f2007-10-22 11:03:37 +10001017 pr_debug("%s: short packet %i\n", dev->name, len);
1018 dev->stats.rx_length_errors++;
Michael Daltonab7db912014-01-16 22:23:27 -08001019 if (vi->mergeable_rx_bufs) {
Michael S. Tsirkin680557c2017-03-06 21:29:47 +02001020 put_page(virt_to_head_page(buf));
Michael Daltonab7db912014-01-16 22:23:27 -08001021 } else if (vi->big_packets) {
Michael Dalton98bfd232013-12-05 13:14:05 -08001022 give_pages(rq, buf);
Michael Daltonab7db912014-01-16 22:23:27 -08001023 } else {
Jason Wangf6b10202017-02-21 16:46:28 +08001024 put_page(virt_to_head_page(buf));
Michael Daltonab7db912014-01-16 22:23:27 -08001025 }
Toshiaki Makita7d9d60f2018-07-23 23:36:04 +09001026 return;
Rusty Russell296f96f2007-10-22 11:03:37 +10001027 }
Rusty Russell296f96f2007-10-22 11:03:37 +10001028
Michael S. Tsirkinf1211592013-11-28 13:30:59 +02001029 if (vi->mergeable_rx_bufs)
Toshiaki Makita7d9d60f2018-07-23 23:36:04 +09001030 skb = receive_mergeable(dev, vi, rq, buf, ctx, len, xdp_xmit,
Toshiaki Makitaa0929a42018-07-23 23:36:05 +09001031 stats);
Michael S. Tsirkinf1211592013-11-28 13:30:59 +02001032 else if (vi->big_packets)
Toshiaki Makitaa0929a42018-07-23 23:36:05 +09001033 skb = receive_big(dev, vi, rq, buf, len, stats);
Michael S. Tsirkinf1211592013-11-28 13:30:59 +02001034 else
Toshiaki Makitaa0929a42018-07-23 23:36:05 +09001035 skb = receive_small(dev, vi, rq, buf, ctx, len, xdp_xmit, stats);
Michael S. Tsirkinf1211592013-11-28 13:30:59 +02001036
1037 if (unlikely(!skb))
Toshiaki Makita7d9d60f2018-07-23 23:36:04 +09001038 return;
Mark McLoughlin3f2c31d2008-11-16 22:41:34 -08001039
Shirley Ma9ab86bb2010-01-29 03:20:04 +00001040 hdr = skb_vnet_hdr(skb);
stephen hemminger3fa2a1d2011-06-15 06:36:29 +00001041
Mike Rapoporte858fae2016-06-08 16:09:21 +03001042 if (hdr->hdr.flags & VIRTIO_NET_HDR_F_DATA_VALID)
Jason Wang10a8d942011-06-10 00:56:17 +00001043 skb->ip_summed = CHECKSUM_UNNECESSARY;
Rusty Russell296f96f2007-10-22 11:03:37 +10001044
Mike Rapoporte858fae2016-06-08 16:09:21 +03001045 if (virtio_net_hdr_to_skb(skb, &hdr->hdr,
1046 virtio_is_little_endian(vi->vdev))) {
1047 net_warn_ratelimited("%s: bad gso: type: %u, size: %u\n",
1048 dev->name, hdr->hdr.gso_type,
1049 hdr->hdr.gso_size);
1050 goto frame_err;
Rusty Russell296f96f2007-10-22 11:03:37 +10001051 }
1052
Mike Rapoportd1dc06d2016-06-14 08:29:38 +03001053 skb->protocol = eth_type_trans(skb, dev);
1054 pr_debug("Receiving skb proto 0x%04x len %i type %i\n",
1055 ntohs(skb->protocol), skb->len, skb->pkt_type);
1056
Eric Dumazet0fbd0502015-07-31 18:25:17 +02001057 napi_gro_receive(&rq->napi, skb);
Toshiaki Makita7d9d60f2018-07-23 23:36:04 +09001058 return;
Rusty Russell296f96f2007-10-22 11:03:37 +10001059
1060frame_err:
1061 dev->stats.rx_frame_errors++;
Rusty Russell296f96f2007-10-22 11:03:37 +10001062 dev_kfree_skb(skb);
1063}
1064
Jason Wang192f68c2017-07-19 16:54:47 +08001065/* Unlike mergeable buffers, all buffers are allocated to the
1066 * same size, except for the headroom. For this reason we do
1067 * not need to use mergeable_len_to_ctx here - it is enough
1068 * to store the headroom as the context ignoring the truesize.
1069 */
Michael S. Tsirkin946fa562014-10-24 00:12:10 +03001070static int add_recvbuf_small(struct virtnet_info *vi, struct receive_queue *rq,
1071 gfp_t gfp)
Rusty Russell296f96f2007-10-22 11:03:37 +10001072{
Jason Wangf6b10202017-02-21 16:46:28 +08001073 struct page_frag *alloc_frag = &rq->alloc_frag;
1074 char *buf;
John Fastabend2de2f7f2017-02-02 19:16:29 -08001075 unsigned int xdp_headroom = virtnet_get_headroom(vi);
Jason Wang192f68c2017-07-19 16:54:47 +08001076 void *ctx = (void *)(unsigned long)xdp_headroom;
Jason Wangf6b10202017-02-21 16:46:28 +08001077 int len = vi->hdr_len + VIRTNET_RX_PAD + GOOD_PACKET_LEN + xdp_headroom;
Shirley Ma9ab86bb2010-01-29 03:20:04 +00001078 int err;
Rusty Russell296f96f2007-10-22 11:03:37 +10001079
Jason Wangf6b10202017-02-21 16:46:28 +08001080 len = SKB_DATA_ALIGN(len) +
1081 SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
1082 if (unlikely(!skb_page_frag_refill(len, alloc_frag, gfp)))
Shirley Ma9ab86bb2010-01-29 03:20:04 +00001083 return -ENOMEM;
Mark McLoughlin3f2c31d2008-11-16 22:41:34 -08001084
Jason Wangf6b10202017-02-21 16:46:28 +08001085 buf = (char *)page_address(alloc_frag->page) + alloc_frag->offset;
1086 get_page(alloc_frag->page);
1087 alloc_frag->offset += len;
1088 sg_init_one(rq->sg, buf + VIRTNET_RX_PAD + xdp_headroom,
1089 vi->hdr_len + GOOD_PACKET_LEN);
Jason Wang192f68c2017-07-19 16:54:47 +08001090 err = virtqueue_add_inbuf_ctx(rq->vq, rq->sg, 1, buf, ctx, gfp);
Shirley Ma9ab86bb2010-01-29 03:20:04 +00001091 if (err < 0)
Jason Wangf6b10202017-02-21 16:46:28 +08001092 put_page(virt_to_head_page(buf));
Shirley Ma9ab86bb2010-01-29 03:20:04 +00001093 return err;
1094}
1095
Michael S. Tsirkin012873d2014-10-24 16:55:57 +03001096static int add_recvbuf_big(struct virtnet_info *vi, struct receive_queue *rq,
1097 gfp_t gfp)
Shirley Ma9ab86bb2010-01-29 03:20:04 +00001098{
Shirley Ma9ab86bb2010-01-29 03:20:04 +00001099 struct page *first, *list = NULL;
1100 char *p;
1101 int i, err, offset;
1102
Rusty Russella5835442014-09-11 10:17:36 +09301103 sg_init_table(rq->sg, MAX_SKB_FRAGS + 2);
1104
Jason Wange9d74172012-12-07 07:04:55 +00001105 /* page in rq->sg[MAX_SKB_FRAGS + 1] is list tail */
Shirley Ma9ab86bb2010-01-29 03:20:04 +00001106 for (i = MAX_SKB_FRAGS + 1; i > 1; --i) {
Jason Wange9d74172012-12-07 07:04:55 +00001107 first = get_a_page(rq, gfp);
Shirley Ma9ab86bb2010-01-29 03:20:04 +00001108 if (!first) {
1109 if (list)
Jason Wange9d74172012-12-07 07:04:55 +00001110 give_pages(rq, list);
Shirley Ma9ab86bb2010-01-29 03:20:04 +00001111 return -ENOMEM;
Rusty Russell3161e452009-08-26 12:22:32 -07001112 }
Jason Wange9d74172012-12-07 07:04:55 +00001113 sg_set_buf(&rq->sg[i], page_address(first), PAGE_SIZE);
Rusty Russell296f96f2007-10-22 11:03:37 +10001114
Shirley Ma9ab86bb2010-01-29 03:20:04 +00001115 /* chain new page in list head to match sg */
1116 first->private = (unsigned long)list;
1117 list = first;
1118 }
Mark McLoughlin3f2c31d2008-11-16 22:41:34 -08001119
Jason Wange9d74172012-12-07 07:04:55 +00001120 first = get_a_page(rq, gfp);
Shirley Ma9ab86bb2010-01-29 03:20:04 +00001121 if (!first) {
Jason Wange9d74172012-12-07 07:04:55 +00001122 give_pages(rq, list);
Shirley Ma9ab86bb2010-01-29 03:20:04 +00001123 return -ENOMEM;
1124 }
1125 p = page_address(first);
Herbert Xu97402b92008-04-18 11:24:27 +08001126
Jason Wange9d74172012-12-07 07:04:55 +00001127 /* rq->sg[0], rq->sg[1] share the same page */
Michael S. Tsirkin012873d2014-10-24 16:55:57 +03001128 /* a separated rq->sg[0] for header - required in case !any_header_sg */
1129 sg_set_buf(&rq->sg[0], p, vi->hdr_len);
Herbert Xu97402b92008-04-18 11:24:27 +08001130
Jason Wange9d74172012-12-07 07:04:55 +00001131 /* rq->sg[1] for data packet, from offset */
Shirley Ma9ab86bb2010-01-29 03:20:04 +00001132 offset = sizeof(struct padded_vnet_hdr);
Jason Wange9d74172012-12-07 07:04:55 +00001133 sg_set_buf(&rq->sg[1], p + offset, PAGE_SIZE - offset);
Herbert Xu97402b92008-04-18 11:24:27 +08001134
Shirley Ma9ab86bb2010-01-29 03:20:04 +00001135 /* chain first in list head */
1136 first->private = (unsigned long)list;
Rusty Russell9dc7b9e2013-03-20 15:44:28 +10301137 err = virtqueue_add_inbuf(rq->vq, rq->sg, MAX_SKB_FRAGS + 2,
1138 first, gfp);
Shirley Ma9ab86bb2010-01-29 03:20:04 +00001139 if (err < 0)
Jason Wange9d74172012-12-07 07:04:55 +00001140 give_pages(rq, first);
Herbert Xu97402b92008-04-18 11:24:27 +08001141
Shirley Ma9ab86bb2010-01-29 03:20:04 +00001142 return err;
1143}
Herbert Xu97402b92008-04-18 11:24:27 +08001144
Michael S. Tsirkind85b758f72017-03-09 02:21:21 +02001145static unsigned int get_mergeable_buf_len(struct receive_queue *rq,
Jason Wang3cc81a92018-03-02 17:29:14 +08001146 struct ewma_pkt_len *avg_pkt_len,
1147 unsigned int room)
Shirley Ma9ab86bb2010-01-29 03:20:04 +00001148{
Michael Daltonab7db912014-01-16 22:23:27 -08001149 const size_t hdr_len = sizeof(struct virtio_net_hdr_mrg_rxbuf);
Michael Daltonfbf28d72014-01-16 22:23:30 -08001150 unsigned int len;
1151
Jason Wang3cc81a92018-03-02 17:29:14 +08001152 if (room)
1153 return PAGE_SIZE - room;
1154
1155 len = hdr_len + clamp_t(unsigned int, ewma_pkt_len_read(avg_pkt_len),
Michael S. Tsirkinf0c31922017-06-02 17:54:33 +03001156 rq->min_buf_len, PAGE_SIZE - hdr_len);
Jason Wang3cc81a92018-03-02 17:29:14 +08001157
Michael S. Tsirkine377fcc2017-03-06 22:21:35 +02001158 return ALIGN(len, L1_CACHE_BYTES);
Michael Daltonfbf28d72014-01-16 22:23:30 -08001159}
1160
John Fastabend2de2f7f2017-02-02 19:16:29 -08001161static int add_recvbuf_mergeable(struct virtnet_info *vi,
1162 struct receive_queue *rq, gfp_t gfp)
Michael Daltonfbf28d72014-01-16 22:23:30 -08001163{
Michael Daltonfb518792014-01-16 22:23:26 -08001164 struct page_frag *alloc_frag = &rq->alloc_frag;
John Fastabend2de2f7f2017-02-02 19:16:29 -08001165 unsigned int headroom = virtnet_get_headroom(vi);
Jason Wang3cc81a92018-03-02 17:29:14 +08001166 unsigned int tailroom = headroom ? sizeof(struct skb_shared_info) : 0;
1167 unsigned int room = SKB_DATA_ALIGN(headroom + tailroom);
Michael Daltonfb518792014-01-16 22:23:26 -08001168 char *buf;
Michael S. Tsirkin680557c2017-03-06 21:29:47 +02001169 void *ctx;
Shirley Ma9ab86bb2010-01-29 03:20:04 +00001170 int err;
Michael Daltonfb518792014-01-16 22:23:26 -08001171 unsigned int len, hole;
Rusty Russell296f96f2007-10-22 11:03:37 +10001172
Jason Wang3cc81a92018-03-02 17:29:14 +08001173 /* Extra tailroom is needed to satisfy XDP's assumption. This
1174 * means rx frags coalescing won't work, but consider we've
1175 * disabled GSO for XDP, it won't be a big issue.
1176 */
1177 len = get_mergeable_buf_len(rq, &rq->mrg_avg_pkt_len, room);
1178 if (unlikely(!skb_page_frag_refill(len + room, alloc_frag, gfp)))
Shirley Ma9ab86bb2010-01-29 03:20:04 +00001179 return -ENOMEM;
Michael Daltonab7db912014-01-16 22:23:27 -08001180
Michael Daltonfb518792014-01-16 22:23:26 -08001181 buf = (char *)page_address(alloc_frag->page) + alloc_frag->offset;
John Fastabend2de2f7f2017-02-02 19:16:29 -08001182 buf += headroom; /* advance address leaving hole at front of pkt */
Michael Daltonfb518792014-01-16 22:23:26 -08001183 get_page(alloc_frag->page);
Jason Wang3cc81a92018-03-02 17:29:14 +08001184 alloc_frag->offset += len + room;
Michael Daltonfb518792014-01-16 22:23:26 -08001185 hole = alloc_frag->size - alloc_frag->offset;
Jason Wang3cc81a92018-03-02 17:29:14 +08001186 if (hole < len + room) {
Michael Daltonab7db912014-01-16 22:23:27 -08001187 /* To avoid internal fragmentation, if there is very likely not
1188 * enough space for another buffer, add the remaining space to
Michael S. Tsirkin1daa8792017-07-31 21:49:49 +03001189 * the current buffer.
Michael Daltonab7db912014-01-16 22:23:27 -08001190 */
Michael Daltonfb518792014-01-16 22:23:26 -08001191 len += hole;
1192 alloc_frag->offset += hole;
1193 }
Shirley Ma9ab86bb2010-01-29 03:20:04 +00001194
Michael Daltonfb518792014-01-16 22:23:26 -08001195 sg_init_one(rq->sg, buf, len);
David S. Miller29fda252017-08-01 10:07:50 -07001196 ctx = mergeable_len_to_ctx(len, headroom);
Michael S. Tsirkin680557c2017-03-06 21:29:47 +02001197 err = virtqueue_add_inbuf_ctx(rq->vq, rq->sg, 1, buf, ctx, gfp);
Shirley Ma9ab86bb2010-01-29 03:20:04 +00001198 if (err < 0)
Michael Dalton2613af02013-10-28 15:44:18 -07001199 put_page(virt_to_head_page(buf));
Shirley Ma9ab86bb2010-01-29 03:20:04 +00001200
1201 return err;
Rusty Russell296f96f2007-10-22 11:03:37 +10001202}
1203
Rusty Russellb2baed62011-12-29 00:42:38 +00001204/*
1205 * Returns false if we couldn't fill entirely (OOM).
1206 *
1207 * Normally run in the receive path, but can also be run from ndo_open
1208 * before we're receiving packets, or from refill_work which is
1209 * careful to disable receiving (using napi_disable).
1210 */
Michael S. Tsirkin946fa562014-10-24 00:12:10 +03001211static bool try_fill_recv(struct virtnet_info *vi, struct receive_queue *rq,
1212 gfp_t gfp)
Mark McLoughlin3f2c31d2008-11-16 22:41:34 -08001213{
Mark McLoughlin3f2c31d2008-11-16 22:41:34 -08001214 int err;
Michael S. Tsirkin1788f4952010-07-02 16:32:55 +00001215 bool oom;
Mark McLoughlin3f2c31d2008-11-16 22:41:34 -08001216
Amit Shah0aea51c2009-08-26 14:58:28 +05301217 do {
Shirley Ma9ab86bb2010-01-29 03:20:04 +00001218 if (vi->mergeable_rx_bufs)
John Fastabend2de2f7f2017-02-02 19:16:29 -08001219 err = add_recvbuf_mergeable(vi, rq, gfp);
Shirley Ma9ab86bb2010-01-29 03:20:04 +00001220 else if (vi->big_packets)
Michael S. Tsirkin012873d2014-10-24 16:55:57 +03001221 err = add_recvbuf_big(vi, rq, gfp);
Shirley Ma9ab86bb2010-01-29 03:20:04 +00001222 else
Michael S. Tsirkin946fa562014-10-24 00:12:10 +03001223 err = add_recvbuf_small(vi, rq, gfp);
Mark McLoughlin3f2c31d2008-11-16 22:41:34 -08001224
Michael S. Tsirkin1788f4952010-07-02 16:32:55 +00001225 oom = err == -ENOMEM;
Rusty Russell9ed4cb02012-10-16 23:56:14 +10301226 if (err)
Mark McLoughlin3f2c31d2008-11-16 22:41:34 -08001227 break;
Linus Torvaldsb7dfde92012-12-20 08:37:04 -08001228 } while (rq->vq->num_free);
Jason Wang681daee22014-03-26 13:03:00 +08001229 virtqueue_kick(rq->vq);
Rusty Russell3161e452009-08-26 12:22:32 -07001230 return !oom;
Mark McLoughlin3f2c31d2008-11-16 22:41:34 -08001231}
1232
Rusty Russell18445c42008-02-04 23:49:57 -05001233static void skb_recv_done(struct virtqueue *rvq)
Rusty Russell296f96f2007-10-22 11:03:37 +10001234{
1235 struct virtnet_info *vi = rvq->vdev->priv;
Jason Wang986a4f42012-12-07 07:04:56 +00001236 struct receive_queue *rq = &vi->rq[vq2rxq(rvq)];
Jason Wange9d74172012-12-07 07:04:55 +00001237
Willem de Bruijne4e84522017-04-24 13:49:26 -04001238 virtqueue_napi_schedule(&rq->napi, rvq);
Rusty Russell296f96f2007-10-22 11:03:37 +10001239}
1240
Willem de Bruijne4e84522017-04-24 13:49:26 -04001241static void virtnet_napi_enable(struct virtqueue *vq, struct napi_struct *napi)
Bruce Rogers3e9d08e2011-02-10 11:03:31 -08001242{
Willem de Bruijne4e84522017-04-24 13:49:26 -04001243 napi_enable(napi);
Bruce Rogers3e9d08e2011-02-10 11:03:31 -08001244
1245 /* If all buffers were filled by other side before we napi_enabled, we
Willem de Bruijne4e84522017-04-24 13:49:26 -04001246 * won't get another interrupt, so process any outstanding packets now.
1247 * Call local_bh_enable after to trigger softIRQ processing.
1248 */
1249 local_bh_disable();
1250 virtqueue_napi_schedule(napi, vq);
1251 local_bh_enable();
Bruce Rogers3e9d08e2011-02-10 11:03:31 -08001252}
1253
Willem de Bruijnb92f1e62017-04-24 13:49:27 -04001254static void virtnet_napi_tx_enable(struct virtnet_info *vi,
1255 struct virtqueue *vq,
1256 struct napi_struct *napi)
1257{
1258 if (!napi->weight)
1259 return;
1260
1261 /* Tx napi touches cachelines on the cpu handling tx interrupts. Only
1262 * enable the feature if this is likely affine with the transmit path.
1263 */
1264 if (!vi->affinity_hint_set) {
1265 napi->weight = 0;
1266 return;
1267 }
1268
1269 return virtnet_napi_enable(vq, napi);
1270}
1271
Willem de Bruijn78a57b42017-04-25 15:59:17 -04001272static void virtnet_napi_tx_disable(struct napi_struct *napi)
1273{
1274 if (napi->weight)
1275 napi_disable(napi);
1276}
1277
Rusty Russell3161e452009-08-26 12:22:32 -07001278static void refill_work(struct work_struct *work)
1279{
Jason Wange9d74172012-12-07 07:04:55 +00001280 struct virtnet_info *vi =
1281 container_of(work, struct virtnet_info, refill.work);
Rusty Russell3161e452009-08-26 12:22:32 -07001282 bool still_empty;
Jason Wang986a4f42012-12-07 07:04:56 +00001283 int i;
Rusty Russell3161e452009-08-26 12:22:32 -07001284
Sasha Levin55257d72013-04-29 12:00:08 +09301285 for (i = 0; i < vi->curr_queue_pairs; i++) {
Jason Wang986a4f42012-12-07 07:04:56 +00001286 struct receive_queue *rq = &vi->rq[i];
Rusty Russell3161e452009-08-26 12:22:32 -07001287
Jason Wang986a4f42012-12-07 07:04:56 +00001288 napi_disable(&rq->napi);
Michael S. Tsirkin946fa562014-10-24 00:12:10 +03001289 still_empty = !try_fill_recv(vi, rq, GFP_KERNEL);
Willem de Bruijne4e84522017-04-24 13:49:26 -04001290 virtnet_napi_enable(rq->vq, &rq->napi);
Jason Wang986a4f42012-12-07 07:04:56 +00001291
1292 /* In theory, this can happen: if we don't get any buffers in
1293 * we will *never* try to fill again.
1294 */
1295 if (still_empty)
1296 schedule_delayed_work(&vi->refill, HZ/2);
1297 }
Rusty Russell3161e452009-08-26 12:22:32 -07001298}
1299
Jesper Dangaard Brouer2471c752018-06-26 17:39:58 +02001300static int virtnet_receive(struct receive_queue *rq, int budget,
1301 unsigned int *xdp_xmit)
Rusty Russell296f96f2007-10-22 11:03:37 +10001302{
Jason Wange9d74172012-12-07 07:04:55 +00001303 struct virtnet_info *vi = rq->vq->vdev->priv;
Toshiaki Makitaa0929a42018-07-23 23:36:05 +09001304 struct virtnet_rx_stats stats = {};
Toshiaki Makita5b8f3c82018-07-23 23:36:08 +09001305 struct send_queue *sq;
Toshiaki Makitaa0929a42018-07-23 23:36:05 +09001306 unsigned int len;
Shirley Ma9ab86bb2010-01-29 03:20:04 +00001307 void *buf;
Toshiaki Makitaa0929a42018-07-23 23:36:05 +09001308 int i;
Rusty Russell296f96f2007-10-22 11:03:37 +10001309
Jason Wang192f68c2017-07-19 16:54:47 +08001310 if (!vi->big_packets || vi->mergeable_rx_bufs) {
Michael S. Tsirkin680557c2017-03-06 21:29:47 +02001311 void *ctx;
1312
Toshiaki Makitaa0929a42018-07-23 23:36:05 +09001313 while (stats.rx.packets < budget &&
Michael S. Tsirkin680557c2017-03-06 21:29:47 +02001314 (buf = virtqueue_get_buf_ctx(rq->vq, &len, &ctx))) {
Toshiaki Makitaa0929a42018-07-23 23:36:05 +09001315 receive_buf(vi, rq, buf, len, ctx, xdp_xmit, &stats);
1316 stats.rx.packets++;
Michael S. Tsirkin680557c2017-03-06 21:29:47 +02001317 }
1318 } else {
Toshiaki Makitaa0929a42018-07-23 23:36:05 +09001319 while (stats.rx.packets < budget &&
Michael S. Tsirkin680557c2017-03-06 21:29:47 +02001320 (buf = virtqueue_get_buf(rq->vq, &len)) != NULL) {
Toshiaki Makitaa0929a42018-07-23 23:36:05 +09001321 receive_buf(vi, rq, buf, len, NULL, xdp_xmit, &stats);
1322 stats.rx.packets++;
Michael S. Tsirkin680557c2017-03-06 21:29:47 +02001323 }
Rusty Russell296f96f2007-10-22 11:03:37 +10001324 }
1325
Jason Wangbe121f42014-01-16 14:45:24 +08001326 if (rq->vq->num_free > virtqueue_get_vring_size(rq->vq) / 2) {
Michael S. Tsirkin946fa562014-10-24 00:12:10 +03001327 if (!try_fill_recv(vi, rq, GFP_ATOMIC))
Tejun Heo3b07e9c2012-08-20 14:51:24 -07001328 schedule_delayed_work(&vi->refill, 0);
Rusty Russell3161e452009-08-26 12:22:32 -07001329 }
Rusty Russell296f96f2007-10-22 11:03:37 +10001330
Toshiaki Makitad7dfc5c2018-01-17 15:38:25 +09001331 u64_stats_update_begin(&rq->stats.syncp);
Toshiaki Makitaa0929a42018-07-23 23:36:05 +09001332 for (i = 0; i < VIRTNET_RQ_STATS_LEN; i++) {
1333 size_t offset = virtnet_rq_stats_desc[i].offset;
1334 u64 *item;
1335
1336 item = (u64 *)((u8 *)&rq->stats.items + offset);
1337 *item += *(u64 *)((u8 *)&stats.rx + offset);
1338 }
Toshiaki Makitad7dfc5c2018-01-17 15:38:25 +09001339 u64_stats_update_end(&rq->stats.syncp);
Jason Wang61845d22017-02-17 11:33:09 +08001340
Toshiaki Makita5b8f3c82018-07-23 23:36:08 +09001341 sq = virtnet_xdp_sq(vi);
1342 u64_stats_update_begin(&sq->stats.syncp);
1343 sq->stats.xdp_tx += stats.tx.xdp_tx;
1344 sq->stats.xdp_tx_drops += stats.tx.xdp_tx_drops;
1345 u64_stats_update_end(&sq->stats.syncp);
1346
Toshiaki Makitaa0929a42018-07-23 23:36:05 +09001347 return stats.rx.packets;
Jason Wang2ffa7592014-07-23 16:33:54 +08001348}
1349
Willem de Bruijnea7735d2017-04-24 13:49:28 -04001350static void free_old_xmit_skbs(struct send_queue *sq)
1351{
1352 struct sk_buff *skb;
1353 unsigned int len;
Willem de Bruijnea7735d2017-04-24 13:49:28 -04001354 unsigned int packets = 0;
1355 unsigned int bytes = 0;
1356
1357 while ((skb = virtqueue_get_buf(sq->vq, &len)) != NULL) {
1358 pr_debug("Sent skb %p\n", skb);
1359
1360 bytes += skb->len;
1361 packets++;
1362
Eric Dumazetdadc0732017-08-24 09:02:49 -07001363 dev_consume_skb_any(skb);
Willem de Bruijnea7735d2017-04-24 13:49:28 -04001364 }
1365
1366 /* Avoid overhead when no packets have been processed
1367 * happens when called speculatively from start_xmit.
1368 */
1369 if (!packets)
1370 return;
1371
Toshiaki Makitad7dfc5c2018-01-17 15:38:25 +09001372 u64_stats_update_begin(&sq->stats.syncp);
1373 sq->stats.bytes += bytes;
1374 sq->stats.packets += packets;
1375 u64_stats_update_end(&sq->stats.syncp);
Willem de Bruijnea7735d2017-04-24 13:49:28 -04001376}
1377
Willem de Bruijn7b0411e2017-04-24 13:49:29 -04001378static void virtnet_poll_cleantx(struct receive_queue *rq)
1379{
1380 struct virtnet_info *vi = rq->vq->vdev->priv;
1381 unsigned int index = vq2rxq(rq->vq);
1382 struct send_queue *sq = &vi->sq[index];
1383 struct netdev_queue *txq = netdev_get_tx_queue(vi->dev, index);
1384
1385 if (!sq->napi.weight)
1386 return;
1387
1388 if (__netif_tx_trylock(txq)) {
1389 free_old_xmit_skbs(sq);
1390 __netif_tx_unlock(txq);
1391 }
1392
1393 if (sq->vq->num_free >= 2 + MAX_SKB_FRAGS)
1394 netif_tx_wake_queue(txq);
1395}
1396
Jason Wang2ffa7592014-07-23 16:33:54 +08001397static int virtnet_poll(struct napi_struct *napi, int budget)
1398{
1399 struct receive_queue *rq =
1400 container_of(napi, struct receive_queue, napi);
Jason Wang9267c432018-04-13 14:58:25 +08001401 struct virtnet_info *vi = rq->vq->vdev->priv;
1402 struct send_queue *sq;
Toshiaki Makita2a435652018-07-23 23:36:07 +09001403 unsigned int received;
Jesper Dangaard Brouer2471c752018-06-26 17:39:58 +02001404 unsigned int xdp_xmit = 0;
Jason Wang2ffa7592014-07-23 16:33:54 +08001405
Willem de Bruijn7b0411e2017-04-24 13:49:29 -04001406 virtnet_poll_cleantx(rq);
1407
Jason Wang186b3c92017-09-19 17:42:43 +08001408 received = virtnet_receive(rq, budget, &xdp_xmit);
Jason Wang2ffa7592014-07-23 16:33:54 +08001409
Rusty Russell8329d982007-11-19 11:20:43 -05001410 /* Out of packets? */
Willem de Bruijne4e84522017-04-24 13:49:26 -04001411 if (received < budget)
1412 virtqueue_napi_complete(napi, rq->vq, received);
Rusty Russell296f96f2007-10-22 11:03:37 +10001413
Jesper Dangaard Brouer2471c752018-06-26 17:39:58 +02001414 if (xdp_xmit & VIRTIO_XDP_REDIR)
1415 xdp_do_flush_map();
1416
1417 if (xdp_xmit & VIRTIO_XDP_TX) {
Toshiaki Makita2a435652018-07-23 23:36:07 +09001418 sq = virtnet_xdp_sq(vi);
Jason Wang9267c432018-04-13 14:58:25 +08001419 virtqueue_kick(sq->vq);
Jason Wang9267c432018-04-13 14:58:25 +08001420 }
Jason Wang186b3c92017-09-19 17:42:43 +08001421
Rusty Russell296f96f2007-10-22 11:03:37 +10001422 return received;
1423}
1424
Jason Wang986a4f42012-12-07 07:04:56 +00001425static int virtnet_open(struct net_device *dev)
1426{
1427 struct virtnet_info *vi = netdev_priv(dev);
Jesper Dangaard Brouer754b8a22018-01-03 11:26:04 +01001428 int i, err;
Jason Wang986a4f42012-12-07 07:04:56 +00001429
Jason Wange4166622013-05-21 20:03:58 +00001430 for (i = 0; i < vi->max_queue_pairs; i++) {
1431 if (i < vi->curr_queue_pairs)
1432 /* Make sure we have some buffers: if oom use wq. */
Michael S. Tsirkin946fa562014-10-24 00:12:10 +03001433 if (!try_fill_recv(vi, &vi->rq[i], GFP_KERNEL))
Jason Wange4166622013-05-21 20:03:58 +00001434 schedule_delayed_work(&vi->refill, 0);
Jesper Dangaard Brouer754b8a22018-01-03 11:26:04 +01001435
1436 err = xdp_rxq_info_reg(&vi->rq[i].xdp_rxq, dev, i);
1437 if (err < 0)
1438 return err;
1439
Jesper Dangaard Brouer8d5d8852018-04-17 16:46:12 +02001440 err = xdp_rxq_info_reg_mem_model(&vi->rq[i].xdp_rxq,
1441 MEM_TYPE_PAGE_SHARED, NULL);
1442 if (err < 0) {
1443 xdp_rxq_info_unreg(&vi->rq[i].xdp_rxq);
1444 return err;
1445 }
1446
Willem de Bruijne4e84522017-04-24 13:49:26 -04001447 virtnet_napi_enable(vi->rq[i].vq, &vi->rq[i].napi);
Willem de Bruijnb92f1e62017-04-24 13:49:27 -04001448 virtnet_napi_tx_enable(vi, vi->sq[i].vq, &vi->sq[i].napi);
Jason Wang986a4f42012-12-07 07:04:56 +00001449 }
1450
1451 return 0;
1452}
1453
Willem de Bruijnb92f1e62017-04-24 13:49:27 -04001454static int virtnet_poll_tx(struct napi_struct *napi, int budget)
1455{
1456 struct send_queue *sq = container_of(napi, struct send_queue, napi);
1457 struct virtnet_info *vi = sq->vq->vdev->priv;
1458 struct netdev_queue *txq = netdev_get_tx_queue(vi->dev, vq2txq(sq->vq));
1459
1460 __netif_tx_lock(txq, raw_smp_processor_id());
1461 free_old_xmit_skbs(sq);
1462 __netif_tx_unlock(txq);
1463
1464 virtqueue_napi_complete(napi, sq->vq, 0);
1465
1466 if (sq->vq->num_free >= 2 + MAX_SKB_FRAGS)
1467 netif_tx_wake_queue(txq);
1468
1469 return 0;
1470}
1471
Jason Wange9d74172012-12-07 07:04:55 +00001472static int xmit_skb(struct send_queue *sq, struct sk_buff *skb)
Rusty Russell296f96f2007-10-22 11:03:37 +10001473{
Michael S. Tsirkin012873d2014-10-24 16:55:57 +03001474 struct virtio_net_hdr_mrg_rxbuf *hdr;
Rusty Russell296f96f2007-10-22 11:03:37 +10001475 const unsigned char *dest = ((struct ethhdr *)skb->data)->h_dest;
Jason Wange9d74172012-12-07 07:04:55 +00001476 struct virtnet_info *vi = sq->vq->vdev->priv;
Jason A. Donenfelde2fcad52017-06-04 04:16:26 +02001477 int num_sg;
Michael S. Tsirkin012873d2014-10-24 16:55:57 +03001478 unsigned hdr_len = vi->hdr_len;
Michael S. Tsirkine7428e92013-07-25 10:20:23 +09301479 bool can_push;
Rusty Russell296f96f2007-10-22 11:03:37 +10001480
Johannes Berge1749612008-10-27 15:59:26 -07001481 pr_debug("%s: xmit %p %pM\n", vi->dev->name, skb, dest);
Michael S. Tsirkine7428e92013-07-25 10:20:23 +09301482
1483 can_push = vi->any_header_sg &&
1484 !((unsigned long)skb->data & (__alignof__(*hdr) - 1)) &&
1485 !skb_header_cloned(skb) && skb_headroom(skb) >= hdr_len;
1486 /* Even if we can, don't push here yet as this would skew
1487 * csum_start offset below. */
1488 if (can_push)
Michael S. Tsirkin012873d2014-10-24 16:55:57 +03001489 hdr = (struct virtio_net_hdr_mrg_rxbuf *)(skb->data - hdr_len);
Michael S. Tsirkine7428e92013-07-25 10:20:23 +09301490 else
1491 hdr = skb_vnet_hdr(skb);
Rusty Russell296f96f2007-10-22 11:03:37 +10001492
Mike Rapoporte858fae2016-06-08 16:09:21 +03001493 if (virtio_net_hdr_from_skb(skb, &hdr->hdr,
Willem de Bruijnfd3a8862018-06-06 11:23:01 -04001494 virtio_is_little_endian(vi->vdev), false,
1495 0))
Mike Rapoporte858fae2016-06-08 16:09:21 +03001496 BUG();
Rusty Russell296f96f2007-10-22 11:03:37 +10001497
Mark McLoughlin3f2c31d2008-11-16 22:41:34 -08001498 if (vi->mergeable_rx_bufs)
Michael S. Tsirkin012873d2014-10-24 16:55:57 +03001499 hdr->num_buffers = 0;
Mark McLoughlin3f2c31d2008-11-16 22:41:34 -08001500
Jason Wang547c8902015-08-27 14:53:06 +08001501 sg_init_table(sq->sg, skb_shinfo(skb)->nr_frags + (can_push ? 1 : 2));
Michael S. Tsirkine7428e92013-07-25 10:20:23 +09301502 if (can_push) {
1503 __skb_push(skb, hdr_len);
1504 num_sg = skb_to_sgvec(skb, sq->sg, 0, skb->len);
Jason A. Donenfelde2fcad52017-06-04 04:16:26 +02001505 if (unlikely(num_sg < 0))
1506 return num_sg;
Michael S. Tsirkine7428e92013-07-25 10:20:23 +09301507 /* Pull header back to avoid skew in tx bytes calculations. */
1508 __skb_pull(skb, hdr_len);
1509 } else {
1510 sg_set_buf(sq->sg, hdr, hdr_len);
Jason A. Donenfelde2fcad52017-06-04 04:16:26 +02001511 num_sg = skb_to_sgvec(skb, sq->sg + 1, 0, skb->len);
1512 if (unlikely(num_sg < 0))
1513 return num_sg;
1514 num_sg++;
Michael S. Tsirkine7428e92013-07-25 10:20:23 +09301515 }
Rusty Russell9dc7b9e2013-03-20 15:44:28 +10301516 return virtqueue_add_outbuf(sq->vq, sq->sg, num_sg, skb, GFP_ATOMIC);
Rusty Russell11a3a152008-05-26 17:48:13 +10001517}
1518
Stephen Hemminger424efe92009-08-31 19:50:51 +00001519static netdev_tx_t start_xmit(struct sk_buff *skb, struct net_device *dev)
Rusty Russell99ffc692008-05-02 21:50:46 -05001520{
1521 struct virtnet_info *vi = netdev_priv(dev);
Jason Wang986a4f42012-12-07 07:04:56 +00001522 int qnum = skb_get_queue_mapping(skb);
1523 struct send_queue *sq = &vi->sq[qnum];
Rusty Russell9ed4cb02012-10-16 23:56:14 +10301524 int err;
Michael S. Tsirkin4b7fd2e62014-10-15 16:23:28 +03001525 struct netdev_queue *txq = netdev_get_tx_queue(dev, qnum);
1526 bool kick = !skb->xmit_more;
Willem de Bruijnb92f1e62017-04-24 13:49:27 -04001527 bool use_napi = sq->napi.weight;
Rusty Russell2cb9c6b2008-02-04 23:50:07 -05001528
Rusty Russell2cb9c6b2008-02-04 23:50:07 -05001529 /* Free up any pending old buffers before queueing new ones. */
Jason Wange9d74172012-12-07 07:04:55 +00001530 free_old_xmit_skbs(sq);
Rusty Russell2cb9c6b2008-02-04 23:50:07 -05001531
Willem de Bruijnbdb12e02017-04-24 13:49:30 -04001532 if (use_napi && kick)
1533 virtqueue_enable_cb_delayed(sq->vq);
1534
Jacob Keller074c3582014-06-25 02:37:13 +00001535 /* timestamp packet in software */
1536 skb_tx_timestamp(skb);
1537
Michael S. Tsirkin03f191b2009-10-28 04:03:38 -07001538 /* Try to transmit */
Linus Torvaldsb7dfde92012-12-20 08:37:04 -08001539 err = xmit_skb(sq, skb);
Rusty Russell48925e32009-09-24 09:59:20 -06001540
Rusty Russell9ed4cb02012-10-16 23:56:14 +10301541 /* This should not happen! */
Jason Wang681daee22014-03-26 13:03:00 +08001542 if (unlikely(err)) {
Rusty Russell9ed4cb02012-10-16 23:56:14 +10301543 dev->stats.tx_fifo_errors++;
1544 if (net_ratelimit())
1545 dev_warn(&dev->dev,
Linus Torvaldsb7dfde92012-12-20 08:37:04 -08001546 "Unexpected TXQ (%d) queue failure: %d\n", qnum, err);
Rusty Russell58eba97d2010-07-02 16:34:01 +00001547 dev->stats.tx_dropped++;
Eric W. Biederman85e94522014-03-15 18:43:33 -07001548 dev_kfree_skb_any(skb);
Rusty Russell58eba97d2010-07-02 16:34:01 +00001549 return NETDEV_TX_OK;
Rusty Russell99ffc692008-05-02 21:50:46 -05001550 }
Michael S. Tsirkin03f191b2009-10-28 04:03:38 -07001551
Rusty Russell48925e32009-09-24 09:59:20 -06001552 /* Don't wait up for transmitted skbs to be freed. */
Willem de Bruijnb92f1e62017-04-24 13:49:27 -04001553 if (!use_napi) {
1554 skb_orphan(skb);
1555 nf_reset(skb);
1556 }
Rusty Russell99ffc692008-05-02 21:50:46 -05001557
Michael S. Tsirkin60302ff2015-04-02 13:05:47 +02001558 /* If running out of space, stop queue to avoid getting packets that we
1559 * are then unable to transmit.
1560 * An alternative would be to force queuing layer to requeue the skb by
1561 * returning NETDEV_TX_BUSY. However, NETDEV_TX_BUSY should not be
1562 * returned in a normal path of operation: it means that driver is not
1563 * maintaining the TX queue stop/start state properly, and causes
1564 * the stack to do a non-trivial amount of useless work.
1565 * Since most packets only take 1 or 2 ring slots, stopping the queue
1566 * early means 16 slots are typically wasted.
stephen hemmingerd631b942015-03-24 16:22:07 -07001567 */
Linus Torvaldsb7dfde92012-12-20 08:37:04 -08001568 if (sq->vq->num_free < 2+MAX_SKB_FRAGS) {
Jason Wang986a4f42012-12-07 07:04:56 +00001569 netif_stop_subqueue(dev, qnum);
Willem de Bruijnb92f1e62017-04-24 13:49:27 -04001570 if (!use_napi &&
1571 unlikely(!virtqueue_enable_cb_delayed(sq->vq))) {
Rusty Russell48925e32009-09-24 09:59:20 -06001572 /* More just got used, free them then recheck. */
Linus Torvaldsb7dfde92012-12-20 08:37:04 -08001573 free_old_xmit_skbs(sq);
1574 if (sq->vq->num_free >= 2+MAX_SKB_FRAGS) {
Jason Wang986a4f42012-12-07 07:04:56 +00001575 netif_start_subqueue(dev, qnum);
Jason Wange9d74172012-12-07 07:04:55 +00001576 virtqueue_disable_cb(sq->vq);
Rusty Russell48925e32009-09-24 09:59:20 -06001577 }
1578 }
Rusty Russell99ffc692008-05-02 21:50:46 -05001579 }
Rusty Russell48925e32009-09-24 09:59:20 -06001580
Michael S. Tsirkin4b7fd2e62014-10-15 16:23:28 +03001581 if (kick || netif_xmit_stopped(txq))
David S. Miller0b725a22014-08-25 15:51:53 -07001582 virtqueue_kick(sq->vq);
1583
Rusty Russell48925e32009-09-24 09:59:20 -06001584 return NETDEV_TX_OK;
Rusty Russell296f96f2007-10-22 11:03:37 +10001585}
1586
Amos Kong40cbfc32013-01-21 01:17:21 +00001587/*
1588 * Send command via the control virtqueue and check status. Commands
1589 * supported by the hypervisor, as indicated by feature bits, should
stephen hemminger788a8b62013-12-09 16:18:45 -08001590 * never fail unless improperly formatted.
Amos Kong40cbfc32013-01-21 01:17:21 +00001591 */
1592static bool virtnet_send_command(struct virtnet_info *vi, u8 class, u8 cmd,
stephen hemmingerd24bae32013-12-09 16:17:40 -08001593 struct scatterlist *out)
Amos Kong40cbfc32013-01-21 01:17:21 +00001594{
Rusty Russellf7bc9592013-03-20 15:44:28 +10301595 struct scatterlist *sgs[4], hdr, stat;
stephen hemmingerd24bae32013-12-09 16:17:40 -08001596 unsigned out_num = 0, tmp;
Amos Kong40cbfc32013-01-21 01:17:21 +00001597
1598 /* Caller should know better */
Rusty Russellf7bc9592013-03-20 15:44:28 +10301599 BUG_ON(!virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_VQ));
Amos Kong40cbfc32013-01-21 01:17:21 +00001600
Michael S. Tsirkin12e57162018-04-19 08:30:48 +03001601 vi->ctrl->status = ~0;
1602 vi->ctrl->hdr.class = class;
1603 vi->ctrl->hdr.cmd = cmd;
Rusty Russellf7bc9592013-03-20 15:44:28 +10301604 /* Add header */
Michael S. Tsirkin12e57162018-04-19 08:30:48 +03001605 sg_init_one(&hdr, &vi->ctrl->hdr, sizeof(vi->ctrl->hdr));
Rusty Russellf7bc9592013-03-20 15:44:28 +10301606 sgs[out_num++] = &hdr;
Amos Kong40cbfc32013-01-21 01:17:21 +00001607
Rusty Russellf7bc9592013-03-20 15:44:28 +10301608 if (out)
1609 sgs[out_num++] = out;
Amos Kong40cbfc32013-01-21 01:17:21 +00001610
Rusty Russellf7bc9592013-03-20 15:44:28 +10301611 /* Add return status. */
Michael S. Tsirkin12e57162018-04-19 08:30:48 +03001612 sg_init_one(&stat, &vi->ctrl->status, sizeof(vi->ctrl->status));
stephen hemmingerd24bae32013-12-09 16:17:40 -08001613 sgs[out_num] = &stat;
Amos Kong40cbfc32013-01-21 01:17:21 +00001614
stephen hemmingerd24bae32013-12-09 16:17:40 -08001615 BUG_ON(out_num + 1 > ARRAY_SIZE(sgs));
Rusty Russella7c58142014-03-13 11:23:39 +10301616 virtqueue_add_sgs(vi->cvq, sgs, out_num, 1, vi, GFP_ATOMIC);
Amos Kong40cbfc32013-01-21 01:17:21 +00001617
Heinz Graalfs67975902013-10-29 09:40:02 +10301618 if (unlikely(!virtqueue_kick(vi->cvq)))
Michael S. Tsirkin12e57162018-04-19 08:30:48 +03001619 return vi->ctrl->status == VIRTIO_NET_OK;
Amos Kong40cbfc32013-01-21 01:17:21 +00001620
1621 /* Spin for a response, the kick causes an ioport write, trapping
1622 * into the hypervisor, so the request should be handled immediately.
1623 */
Heinz Graalfs047b9b92013-10-29 09:40:47 +10301624 while (!virtqueue_get_buf(vi->cvq, &tmp) &&
1625 !virtqueue_is_broken(vi->cvq))
Amos Kong40cbfc32013-01-21 01:17:21 +00001626 cpu_relax();
1627
Michael S. Tsirkin12e57162018-04-19 08:30:48 +03001628 return vi->ctrl->status == VIRTIO_NET_OK;
Amos Kong40cbfc32013-01-21 01:17:21 +00001629}
1630
Alex Williamson9c46f6d2009-02-04 16:36:34 -08001631static int virtnet_set_mac_address(struct net_device *dev, void *p)
1632{
1633 struct virtnet_info *vi = netdev_priv(dev);
1634 struct virtio_device *vdev = vi->vdev;
Jiri Pirkof2f2c8b2012-06-29 05:10:06 +00001635 int ret;
Andy Lutomirskie37e2ff2016-12-05 18:10:58 -08001636 struct sockaddr *addr;
Amos Kong7e58d5a2013-01-21 01:17:23 +00001637 struct scatterlist sg;
Alex Williamson9c46f6d2009-02-04 16:36:34 -08001638
Sridhar Samudralaba5e4422018-05-24 09:55:17 -07001639 if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_STANDBY))
1640 return -EOPNOTSUPP;
1641
Shyam Saini801822d2016-12-24 00:44:58 +05301642 addr = kmemdup(p, sizeof(*addr), GFP_KERNEL);
Andy Lutomirskie37e2ff2016-12-05 18:10:58 -08001643 if (!addr)
1644 return -ENOMEM;
Andy Lutomirskie37e2ff2016-12-05 18:10:58 -08001645
1646 ret = eth_prepare_mac_addr_change(dev, addr);
Jiri Pirkof2f2c8b2012-06-29 05:10:06 +00001647 if (ret)
Andy Lutomirskie37e2ff2016-12-05 18:10:58 -08001648 goto out;
Alex Williamson9c46f6d2009-02-04 16:36:34 -08001649
Amos Kong7e58d5a2013-01-21 01:17:23 +00001650 if (virtio_has_feature(vdev, VIRTIO_NET_F_CTRL_MAC_ADDR)) {
1651 sg_init_one(&sg, addr->sa_data, dev->addr_len);
1652 if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_MAC,
stephen hemmingerd24bae32013-12-09 16:17:40 -08001653 VIRTIO_NET_CTRL_MAC_ADDR_SET, &sg)) {
Amos Kong7e58d5a2013-01-21 01:17:23 +00001654 dev_warn(&vdev->dev,
1655 "Failed to set mac address by vq command.\n");
Andy Lutomirskie37e2ff2016-12-05 18:10:58 -08001656 ret = -EINVAL;
1657 goto out;
Amos Kong7e58d5a2013-01-21 01:17:23 +00001658 }
Michael S. Tsirkin7e93a022014-11-26 15:58:28 +02001659 } else if (virtio_has_feature(vdev, VIRTIO_NET_F_MAC) &&
1660 !virtio_has_feature(vdev, VIRTIO_F_VERSION_1)) {
Rusty Russell855e0c52013-10-14 18:11:51 +10301661 unsigned int i;
1662
1663 /* Naturally, this has an atomicity problem. */
1664 for (i = 0; i < dev->addr_len; i++)
1665 virtio_cwrite8(vdev,
1666 offsetof(struct virtio_net_config, mac) +
1667 i, addr->sa_data[i]);
Amos Kong7e58d5a2013-01-21 01:17:23 +00001668 }
1669
1670 eth_commit_mac_addr_change(dev, p);
Andy Lutomirskie37e2ff2016-12-05 18:10:58 -08001671 ret = 0;
Alex Williamson9c46f6d2009-02-04 16:36:34 -08001672
Andy Lutomirskie37e2ff2016-12-05 18:10:58 -08001673out:
1674 kfree(addr);
1675 return ret;
Alex Williamson9c46f6d2009-02-04 16:36:34 -08001676}
1677
stephen hemmingerbc1f4472017-01-06 19:12:52 -08001678static void virtnet_stats(struct net_device *dev,
1679 struct rtnl_link_stats64 *tot)
stephen hemminger3fa2a1d2011-06-15 06:36:29 +00001680{
1681 struct virtnet_info *vi = netdev_priv(dev);
stephen hemminger3fa2a1d2011-06-15 06:36:29 +00001682 unsigned int start;
Toshiaki Makitad7dfc5c2018-01-17 15:38:25 +09001683 int i;
stephen hemminger3fa2a1d2011-06-15 06:36:29 +00001684
Toshiaki Makitad7dfc5c2018-01-17 15:38:25 +09001685 for (i = 0; i < vi->max_queue_pairs; i++) {
Toshiaki Makita2c4a2f72018-07-23 23:36:06 +09001686 u64 tpackets, tbytes, rpackets, rbytes, rdrops;
Toshiaki Makitad7dfc5c2018-01-17 15:38:25 +09001687 struct receive_queue *rq = &vi->rq[i];
1688 struct send_queue *sq = &vi->sq[i];
stephen hemminger3fa2a1d2011-06-15 06:36:29 +00001689
1690 do {
Toshiaki Makitad7dfc5c2018-01-17 15:38:25 +09001691 start = u64_stats_fetch_begin_irq(&sq->stats.syncp);
1692 tpackets = sq->stats.packets;
1693 tbytes = sq->stats.bytes;
1694 } while (u64_stats_fetch_retry_irq(&sq->stats.syncp, start));
Eric Dumazet83a27052012-06-05 22:35:24 +00001695
1696 do {
Toshiaki Makitad7dfc5c2018-01-17 15:38:25 +09001697 start = u64_stats_fetch_begin_irq(&rq->stats.syncp);
Toshiaki Makitaa0929a42018-07-23 23:36:05 +09001698 rpackets = rq->stats.items.packets;
1699 rbytes = rq->stats.items.bytes;
Toshiaki Makita2c4a2f72018-07-23 23:36:06 +09001700 rdrops = rq->stats.items.drops;
Toshiaki Makitad7dfc5c2018-01-17 15:38:25 +09001701 } while (u64_stats_fetch_retry_irq(&rq->stats.syncp, start));
stephen hemminger3fa2a1d2011-06-15 06:36:29 +00001702
1703 tot->rx_packets += rpackets;
1704 tot->tx_packets += tpackets;
1705 tot->rx_bytes += rbytes;
1706 tot->tx_bytes += tbytes;
Toshiaki Makita2c4a2f72018-07-23 23:36:06 +09001707 tot->rx_dropped += rdrops;
stephen hemminger3fa2a1d2011-06-15 06:36:29 +00001708 }
1709
1710 tot->tx_dropped = dev->stats.tx_dropped;
Rick Jones021ac8d2011-11-21 09:28:17 +00001711 tot->tx_fifo_errors = dev->stats.tx_fifo_errors;
stephen hemminger3fa2a1d2011-06-15 06:36:29 +00001712 tot->rx_length_errors = dev->stats.rx_length_errors;
1713 tot->rx_frame_errors = dev->stats.rx_frame_errors;
stephen hemminger3fa2a1d2011-06-15 06:36:29 +00001714}
1715
Amit Shahda74e892008-02-29 16:24:50 +05301716#ifdef CONFIG_NET_POLL_CONTROLLER
1717static void virtnet_netpoll(struct net_device *dev)
1718{
1719 struct virtnet_info *vi = netdev_priv(dev);
Jason Wang986a4f42012-12-07 07:04:56 +00001720 int i;
Amit Shahda74e892008-02-29 16:24:50 +05301721
Jason Wang986a4f42012-12-07 07:04:56 +00001722 for (i = 0; i < vi->curr_queue_pairs; i++)
1723 napi_schedule(&vi->rq[i].napi);
Amit Shahda74e892008-02-29 16:24:50 +05301724}
1725#endif
1726
Jason Wang586d17c2012-04-11 20:43:52 +00001727static void virtnet_ack_link_announce(struct virtnet_info *vi)
1728{
1729 rtnl_lock();
1730 if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_ANNOUNCE,
stephen hemmingerd24bae32013-12-09 16:17:40 -08001731 VIRTIO_NET_CTRL_ANNOUNCE_ACK, NULL))
Jason Wang586d17c2012-04-11 20:43:52 +00001732 dev_warn(&vi->dev->dev, "Failed to ack link announce.\n");
1733 rtnl_unlock();
1734}
1735
John Fastabend473153292017-02-02 19:14:32 -08001736static int _virtnet_set_queues(struct virtnet_info *vi, u16 queue_pairs)
Jason Wang986a4f42012-12-07 07:04:56 +00001737{
1738 struct scatterlist sg;
Jason Wang986a4f42012-12-07 07:04:56 +00001739 struct net_device *dev = vi->dev;
1740
1741 if (!vi->has_cvq || !virtio_has_feature(vi->vdev, VIRTIO_NET_F_MQ))
1742 return 0;
1743
Michael S. Tsirkin12e57162018-04-19 08:30:48 +03001744 vi->ctrl->mq.virtqueue_pairs = cpu_to_virtio16(vi->vdev, queue_pairs);
1745 sg_init_one(&sg, &vi->ctrl->mq, sizeof(vi->ctrl->mq));
Jason Wang986a4f42012-12-07 07:04:56 +00001746
1747 if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_MQ,
stephen hemmingerd24bae32013-12-09 16:17:40 -08001748 VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET, &sg)) {
Jason Wang986a4f42012-12-07 07:04:56 +00001749 dev_warn(&dev->dev, "Fail to set num of queue pairs to %d\n",
1750 queue_pairs);
1751 return -EINVAL;
Sasha Levin55257d72013-04-29 12:00:08 +09301752 } else {
Jason Wang986a4f42012-12-07 07:04:56 +00001753 vi->curr_queue_pairs = queue_pairs;
Jason Wang35ed1592013-10-15 11:18:59 +08001754 /* virtnet_open() will refill when device is going to up. */
1755 if (dev->flags & IFF_UP)
1756 schedule_delayed_work(&vi->refill, 0);
Sasha Levin55257d72013-04-29 12:00:08 +09301757 }
Jason Wang986a4f42012-12-07 07:04:56 +00001758
1759 return 0;
1760}
1761
John Fastabend473153292017-02-02 19:14:32 -08001762static int virtnet_set_queues(struct virtnet_info *vi, u16 queue_pairs)
1763{
1764 int err;
1765
1766 rtnl_lock();
1767 err = _virtnet_set_queues(vi, queue_pairs);
1768 rtnl_unlock();
1769 return err;
1770}
1771
Rusty Russell296f96f2007-10-22 11:03:37 +10001772static int virtnet_close(struct net_device *dev)
1773{
1774 struct virtnet_info *vi = netdev_priv(dev);
Jason Wang986a4f42012-12-07 07:04:56 +00001775 int i;
Rusty Russell296f96f2007-10-22 11:03:37 +10001776
Rusty Russellb2baed62011-12-29 00:42:38 +00001777 /* Make sure refill_work doesn't re-enable napi! */
1778 cancel_delayed_work_sync(&vi->refill);
Jason Wang986a4f42012-12-07 07:04:56 +00001779
Willem de Bruijnb92f1e62017-04-24 13:49:27 -04001780 for (i = 0; i < vi->max_queue_pairs; i++) {
Jesper Dangaard Brouer754b8a22018-01-03 11:26:04 +01001781 xdp_rxq_info_unreg(&vi->rq[i].xdp_rxq);
Jason Wang986a4f42012-12-07 07:04:56 +00001782 napi_disable(&vi->rq[i].napi);
Willem de Bruijn78a57b42017-04-25 15:59:17 -04001783 virtnet_napi_tx_disable(&vi->sq[i].napi);
Willem de Bruijnb92f1e62017-04-24 13:49:27 -04001784 }
Rusty Russell296f96f2007-10-22 11:03:37 +10001785
Rusty Russell296f96f2007-10-22 11:03:37 +10001786 return 0;
1787}
1788
Alex Williamson2af76982009-02-04 09:02:40 +00001789static void virtnet_set_rx_mode(struct net_device *dev)
1790{
1791 struct virtnet_info *vi = netdev_priv(dev);
Alex Williamsonf565a7c2009-02-04 09:02:45 +00001792 struct scatterlist sg[2];
Alex Williamsonf565a7c2009-02-04 09:02:45 +00001793 struct virtio_net_ctrl_mac *mac_data;
Jiri Pirkoccffad252009-05-22 23:22:17 +00001794 struct netdev_hw_addr *ha;
Jiri Pirko32e7bfc2010-01-25 13:36:10 -08001795 int uc_count;
Jiri Pirko4cd24ea2010-02-08 04:30:35 +00001796 int mc_count;
Alex Williamsonf565a7c2009-02-04 09:02:45 +00001797 void *buf;
1798 int i;
Alex Williamson2af76982009-02-04 09:02:40 +00001799
stephen hemminger788a8b62013-12-09 16:18:45 -08001800 /* We can't dynamically set ndo_set_rx_mode, so return gracefully */
Alex Williamson2af76982009-02-04 09:02:40 +00001801 if (!virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_RX))
1802 return;
1803
Michael S. Tsirkin12e57162018-04-19 08:30:48 +03001804 vi->ctrl->promisc = ((dev->flags & IFF_PROMISC) != 0);
1805 vi->ctrl->allmulti = ((dev->flags & IFF_ALLMULTI) != 0);
Alex Williamson2af76982009-02-04 09:02:40 +00001806
Michael S. Tsirkin12e57162018-04-19 08:30:48 +03001807 sg_init_one(sg, &vi->ctrl->promisc, sizeof(vi->ctrl->promisc));
Alex Williamson2af76982009-02-04 09:02:40 +00001808
1809 if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_RX,
stephen hemmingerd24bae32013-12-09 16:17:40 -08001810 VIRTIO_NET_CTRL_RX_PROMISC, sg))
Alex Williamson2af76982009-02-04 09:02:40 +00001811 dev_warn(&dev->dev, "Failed to %sable promisc mode.\n",
Michael S. Tsirkin12e57162018-04-19 08:30:48 +03001812 vi->ctrl->promisc ? "en" : "dis");
Alex Williamson2af76982009-02-04 09:02:40 +00001813
Michael S. Tsirkin12e57162018-04-19 08:30:48 +03001814 sg_init_one(sg, &vi->ctrl->allmulti, sizeof(vi->ctrl->allmulti));
Alex Williamson2af76982009-02-04 09:02:40 +00001815
1816 if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_RX,
stephen hemmingerd24bae32013-12-09 16:17:40 -08001817 VIRTIO_NET_CTRL_RX_ALLMULTI, sg))
Alex Williamson2af76982009-02-04 09:02:40 +00001818 dev_warn(&dev->dev, "Failed to %sable allmulti mode.\n",
Michael S. Tsirkin12e57162018-04-19 08:30:48 +03001819 vi->ctrl->allmulti ? "en" : "dis");
Alex Williamsonf565a7c2009-02-04 09:02:45 +00001820
Jiri Pirko32e7bfc2010-01-25 13:36:10 -08001821 uc_count = netdev_uc_count(dev);
Jiri Pirko4cd24ea2010-02-08 04:30:35 +00001822 mc_count = netdev_mc_count(dev);
Alex Williamsonf565a7c2009-02-04 09:02:45 +00001823 /* MAC filter - use one buffer for both lists */
Jiri Pirko4cd24ea2010-02-08 04:30:35 +00001824 buf = kzalloc(((uc_count + mc_count) * ETH_ALEN) +
1825 (2 * sizeof(mac_data->entries)), GFP_ATOMIC);
1826 mac_data = buf;
Joe Perchese68ed8f2013-02-03 17:28:15 +00001827 if (!buf)
Alex Williamsonf565a7c2009-02-04 09:02:45 +00001828 return;
Alex Williamsonf565a7c2009-02-04 09:02:45 +00001829
Alex Williamson23e258e2009-05-01 17:27:56 +00001830 sg_init_table(sg, 2);
1831
Alex Williamsonf565a7c2009-02-04 09:02:45 +00001832 /* Store the unicast list and count in the front of the buffer */
Michael S. Tsirkinfdd819b2014-10-07 16:39:48 +02001833 mac_data->entries = cpu_to_virtio32(vi->vdev, uc_count);
Jiri Pirkoccffad252009-05-22 23:22:17 +00001834 i = 0;
Jiri Pirko32e7bfc2010-01-25 13:36:10 -08001835 netdev_for_each_uc_addr(ha, dev)
Jiri Pirkoccffad252009-05-22 23:22:17 +00001836 memcpy(&mac_data->macs[i++][0], ha->addr, ETH_ALEN);
Alex Williamsonf565a7c2009-02-04 09:02:45 +00001837
1838 sg_set_buf(&sg[0], mac_data,
Jiri Pirko32e7bfc2010-01-25 13:36:10 -08001839 sizeof(mac_data->entries) + (uc_count * ETH_ALEN));
Alex Williamsonf565a7c2009-02-04 09:02:45 +00001840
1841 /* multicast list and count fill the end */
Jiri Pirko32e7bfc2010-01-25 13:36:10 -08001842 mac_data = (void *)&mac_data->macs[uc_count][0];
Alex Williamsonf565a7c2009-02-04 09:02:45 +00001843
Michael S. Tsirkinfdd819b2014-10-07 16:39:48 +02001844 mac_data->entries = cpu_to_virtio32(vi->vdev, mc_count);
Jiri Pirko567ec872010-02-23 23:17:07 +00001845 i = 0;
Jiri Pirko22bedad32010-04-01 21:22:57 +00001846 netdev_for_each_mc_addr(ha, dev)
1847 memcpy(&mac_data->macs[i++][0], ha->addr, ETH_ALEN);
Alex Williamsonf565a7c2009-02-04 09:02:45 +00001848
1849 sg_set_buf(&sg[1], mac_data,
Jiri Pirko4cd24ea2010-02-08 04:30:35 +00001850 sizeof(mac_data->entries) + (mc_count * ETH_ALEN));
Alex Williamsonf565a7c2009-02-04 09:02:45 +00001851
1852 if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_MAC,
stephen hemmingerd24bae32013-12-09 16:17:40 -08001853 VIRTIO_NET_CTRL_MAC_TABLE_SET, sg))
Thomas Huth99e872a2013-11-29 10:02:19 +01001854 dev_warn(&dev->dev, "Failed to set MAC filter table.\n");
Alex Williamsonf565a7c2009-02-04 09:02:45 +00001855
1856 kfree(buf);
Alex Williamson2af76982009-02-04 09:02:40 +00001857}
1858
Patrick McHardy80d5c362013-04-19 02:04:28 +00001859static int virtnet_vlan_rx_add_vid(struct net_device *dev,
1860 __be16 proto, u16 vid)
Alex Williamson0bde95692009-02-04 09:02:50 +00001861{
1862 struct virtnet_info *vi = netdev_priv(dev);
1863 struct scatterlist sg;
1864
Michael S. Tsirkind7fad4c2018-04-19 08:30:49 +03001865 vi->ctrl->vid = cpu_to_virtio16(vi->vdev, vid);
Michael S. Tsirkin12e57162018-04-19 08:30:48 +03001866 sg_init_one(&sg, &vi->ctrl->vid, sizeof(vi->ctrl->vid));
Alex Williamson0bde95692009-02-04 09:02:50 +00001867
1868 if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_VLAN,
stephen hemmingerd24bae32013-12-09 16:17:40 -08001869 VIRTIO_NET_CTRL_VLAN_ADD, &sg))
Alex Williamson0bde95692009-02-04 09:02:50 +00001870 dev_warn(&dev->dev, "Failed to add VLAN ID %d.\n", vid);
Jiri Pirko8e586132011-12-08 19:52:37 -05001871 return 0;
Alex Williamson0bde95692009-02-04 09:02:50 +00001872}
1873
Patrick McHardy80d5c362013-04-19 02:04:28 +00001874static int virtnet_vlan_rx_kill_vid(struct net_device *dev,
1875 __be16 proto, u16 vid)
Alex Williamson0bde95692009-02-04 09:02:50 +00001876{
1877 struct virtnet_info *vi = netdev_priv(dev);
1878 struct scatterlist sg;
1879
Michael S. Tsirkind7fad4c2018-04-19 08:30:49 +03001880 vi->ctrl->vid = cpu_to_virtio16(vi->vdev, vid);
Michael S. Tsirkin12e57162018-04-19 08:30:48 +03001881 sg_init_one(&sg, &vi->ctrl->vid, sizeof(vi->ctrl->vid));
Alex Williamson0bde95692009-02-04 09:02:50 +00001882
1883 if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_VLAN,
stephen hemmingerd24bae32013-12-09 16:17:40 -08001884 VIRTIO_NET_CTRL_VLAN_DEL, &sg))
Alex Williamson0bde95692009-02-04 09:02:50 +00001885 dev_warn(&dev->dev, "Failed to kill VLAN ID %d.\n", vid);
Jiri Pirko8e586132011-12-08 19:52:37 -05001886 return 0;
Alex Williamson0bde95692009-02-04 09:02:50 +00001887}
1888
Wanlong Gao8898c212013-01-24 23:51:30 +00001889static void virtnet_clean_affinity(struct virtnet_info *vi, long hcpu)
Jason Wang986a4f42012-12-07 07:04:56 +00001890{
1891 int i;
Wanlong Gao8898c212013-01-24 23:51:30 +00001892
1893 if (vi->affinity_hint_set) {
1894 for (i = 0; i < vi->max_queue_pairs; i++) {
1895 virtqueue_set_affinity(vi->rq[i].vq, -1);
1896 virtqueue_set_affinity(vi->sq[i].vq, -1);
1897 }
1898
1899 vi->affinity_hint_set = false;
1900 }
Wanlong Gao8898c212013-01-24 23:51:30 +00001901}
1902
1903static void virtnet_set_affinity(struct virtnet_info *vi)
Jason Wang986a4f42012-12-07 07:04:56 +00001904{
1905 int i;
Wanlong Gao47be2472013-01-24 23:51:29 +00001906 int cpu;
Jason Wang986a4f42012-12-07 07:04:56 +00001907
1908 /* In multiqueue mode, when the number of cpu is equal to the number of
1909 * queue pairs, we let the queue pairs to be private to one cpu by
1910 * setting the affinity hint to eliminate the contention.
1911 */
Wanlong Gao8898c212013-01-24 23:51:30 +00001912 if (vi->curr_queue_pairs == 1 ||
1913 vi->max_queue_pairs != num_online_cpus()) {
1914 virtnet_clean_affinity(vi, -1);
1915 return;
Jason Wang986a4f42012-12-07 07:04:56 +00001916 }
1917
Wanlong Gao8898c212013-01-24 23:51:30 +00001918 i = 0;
1919 for_each_online_cpu(cpu) {
Jason Wang986a4f42012-12-07 07:04:56 +00001920 virtqueue_set_affinity(vi->rq[i].vq, cpu);
1921 virtqueue_set_affinity(vi->sq[i].vq, cpu);
Jason Wang9bb8ca82013-11-05 18:19:45 +08001922 netif_set_xps_queue(vi->dev, cpumask_of(cpu), i);
Wanlong Gao8898c212013-01-24 23:51:30 +00001923 i++;
Jason Wang986a4f42012-12-07 07:04:56 +00001924 }
1925
Wanlong Gao8898c212013-01-24 23:51:30 +00001926 vi->affinity_hint_set = true;
Jason Wang986a4f42012-12-07 07:04:56 +00001927}
1928
Sebastian Andrzej Siewior8017c272016-08-12 19:49:43 +02001929static int virtnet_cpu_online(unsigned int cpu, struct hlist_node *node)
Wanlong Gao8de4b2f2013-01-24 23:51:31 +00001930{
Sebastian Andrzej Siewior8017c272016-08-12 19:49:43 +02001931 struct virtnet_info *vi = hlist_entry_safe(node, struct virtnet_info,
1932 node);
1933 virtnet_set_affinity(vi);
1934 return 0;
1935}
Wanlong Gao8de4b2f2013-01-24 23:51:31 +00001936
Sebastian Andrzej Siewior8017c272016-08-12 19:49:43 +02001937static int virtnet_cpu_dead(unsigned int cpu, struct hlist_node *node)
1938{
1939 struct virtnet_info *vi = hlist_entry_safe(node, struct virtnet_info,
1940 node_dead);
1941 virtnet_set_affinity(vi);
1942 return 0;
1943}
Jason Wang3ab098d2013-10-15 11:18:58 +08001944
Sebastian Andrzej Siewior8017c272016-08-12 19:49:43 +02001945static int virtnet_cpu_down_prep(unsigned int cpu, struct hlist_node *node)
1946{
1947 struct virtnet_info *vi = hlist_entry_safe(node, struct virtnet_info,
1948 node);
1949
1950 virtnet_clean_affinity(vi, cpu);
1951 return 0;
1952}
1953
1954static enum cpuhp_state virtionet_online;
1955
1956static int virtnet_cpu_notif_add(struct virtnet_info *vi)
1957{
1958 int ret;
1959
1960 ret = cpuhp_state_add_instance_nocalls(virtionet_online, &vi->node);
1961 if (ret)
1962 return ret;
1963 ret = cpuhp_state_add_instance_nocalls(CPUHP_VIRT_NET_DEAD,
1964 &vi->node_dead);
1965 if (!ret)
1966 return ret;
1967 cpuhp_state_remove_instance_nocalls(virtionet_online, &vi->node);
1968 return ret;
1969}
1970
1971static void virtnet_cpu_notif_remove(struct virtnet_info *vi)
1972{
1973 cpuhp_state_remove_instance_nocalls(virtionet_online, &vi->node);
1974 cpuhp_state_remove_instance_nocalls(CPUHP_VIRT_NET_DEAD,
1975 &vi->node_dead);
Herbert Xua9ea3fc2008-04-18 11:21:42 +08001976}
1977
Rick Jones8f9f4662011-10-19 08:10:59 +00001978static void virtnet_get_ringparam(struct net_device *dev,
1979 struct ethtool_ringparam *ring)
1980{
1981 struct virtnet_info *vi = netdev_priv(dev);
1982
Jason Wang986a4f42012-12-07 07:04:56 +00001983 ring->rx_max_pending = virtqueue_get_vring_size(vi->rq[0].vq);
1984 ring->tx_max_pending = virtqueue_get_vring_size(vi->sq[0].vq);
Rick Jones8f9f4662011-10-19 08:10:59 +00001985 ring->rx_pending = ring->rx_max_pending;
1986 ring->tx_pending = ring->tx_max_pending;
Rick Jones8f9f4662011-10-19 08:10:59 +00001987}
1988
Rick Jones66846042011-11-14 14:17:08 +00001989
1990static void virtnet_get_drvinfo(struct net_device *dev,
1991 struct ethtool_drvinfo *info)
1992{
1993 struct virtnet_info *vi = netdev_priv(dev);
1994 struct virtio_device *vdev = vi->vdev;
1995
1996 strlcpy(info->driver, KBUILD_MODNAME, sizeof(info->driver));
1997 strlcpy(info->version, VIRTNET_DRIVER_VERSION, sizeof(info->version));
1998 strlcpy(info->bus_info, virtio_bus_name(vdev), sizeof(info->bus_info));
1999
2000}
2001
Jason Wangd73bcd22012-12-07 07:04:57 +00002002/* TODO: Eliminate OOO packets during switching */
2003static int virtnet_set_channels(struct net_device *dev,
2004 struct ethtool_channels *channels)
2005{
2006 struct virtnet_info *vi = netdev_priv(dev);
2007 u16 queue_pairs = channels->combined_count;
2008 int err;
2009
2010 /* We don't support separate rx/tx channels.
2011 * We don't allow setting 'other' channels.
2012 */
2013 if (channels->rx_count || channels->tx_count || channels->other_count)
2014 return -EINVAL;
2015
Amos Kongc18e9cd2014-04-18 13:45:41 +08002016 if (queue_pairs > vi->max_queue_pairs || queue_pairs == 0)
Jason Wangd73bcd22012-12-07 07:04:57 +00002017 return -EINVAL;
2018
John Fastabendf600b692016-12-15 12:13:24 -08002019 /* For now we don't support modifying channels while XDP is loaded
2020 * also when XDP is loaded all RX queues have XDP programs so we only
2021 * need to check a single RX queue.
2022 */
2023 if (vi->rq[0].xdp_prog)
2024 return -EINVAL;
2025
Wanlong Gao47be2472013-01-24 23:51:29 +00002026 get_online_cpus();
John Fastabend473153292017-02-02 19:14:32 -08002027 err = _virtnet_set_queues(vi, queue_pairs);
Jason Wangd73bcd22012-12-07 07:04:57 +00002028 if (!err) {
2029 netif_set_real_num_tx_queues(dev, queue_pairs);
2030 netif_set_real_num_rx_queues(dev, queue_pairs);
2031
Wanlong Gao8898c212013-01-24 23:51:30 +00002032 virtnet_set_affinity(vi);
Jason Wangd73bcd22012-12-07 07:04:57 +00002033 }
Wanlong Gao47be2472013-01-24 23:51:29 +00002034 put_online_cpus();
Jason Wangd73bcd22012-12-07 07:04:57 +00002035
2036 return err;
2037}
2038
Toshiaki Makitad7dfc5c2018-01-17 15:38:25 +09002039static void virtnet_get_strings(struct net_device *dev, u32 stringset, u8 *data)
2040{
2041 struct virtnet_info *vi = netdev_priv(dev);
2042 char *p = (char *)data;
2043 unsigned int i, j;
2044
2045 switch (stringset) {
2046 case ETH_SS_STATS:
2047 for (i = 0; i < vi->curr_queue_pairs; i++) {
2048 for (j = 0; j < VIRTNET_RQ_STATS_LEN; j++) {
2049 snprintf(p, ETH_GSTRING_LEN, "rx_queue_%u_%s",
2050 i, virtnet_rq_stats_desc[j].desc);
2051 p += ETH_GSTRING_LEN;
2052 }
2053 }
2054
2055 for (i = 0; i < vi->curr_queue_pairs; i++) {
2056 for (j = 0; j < VIRTNET_SQ_STATS_LEN; j++) {
2057 snprintf(p, ETH_GSTRING_LEN, "tx_queue_%u_%s",
2058 i, virtnet_sq_stats_desc[j].desc);
2059 p += ETH_GSTRING_LEN;
2060 }
2061 }
2062 break;
2063 }
2064}
2065
2066static int virtnet_get_sset_count(struct net_device *dev, int sset)
2067{
2068 struct virtnet_info *vi = netdev_priv(dev);
2069
2070 switch (sset) {
2071 case ETH_SS_STATS:
2072 return vi->curr_queue_pairs * (VIRTNET_RQ_STATS_LEN +
2073 VIRTNET_SQ_STATS_LEN);
2074 default:
2075 return -EOPNOTSUPP;
2076 }
2077}
2078
2079static void virtnet_get_ethtool_stats(struct net_device *dev,
2080 struct ethtool_stats *stats, u64 *data)
2081{
2082 struct virtnet_info *vi = netdev_priv(dev);
2083 unsigned int idx = 0, start, i, j;
2084 const u8 *stats_base;
2085 size_t offset;
2086
2087 for (i = 0; i < vi->curr_queue_pairs; i++) {
2088 struct receive_queue *rq = &vi->rq[i];
2089
Toshiaki Makitaa0929a42018-07-23 23:36:05 +09002090 stats_base = (u8 *)&rq->stats.items;
Toshiaki Makitad7dfc5c2018-01-17 15:38:25 +09002091 do {
2092 start = u64_stats_fetch_begin_irq(&rq->stats.syncp);
2093 for (j = 0; j < VIRTNET_RQ_STATS_LEN; j++) {
2094 offset = virtnet_rq_stats_desc[j].offset;
2095 data[idx + j] = *(u64 *)(stats_base + offset);
2096 }
2097 } while (u64_stats_fetch_retry_irq(&rq->stats.syncp, start));
2098 idx += VIRTNET_RQ_STATS_LEN;
2099 }
2100
2101 for (i = 0; i < vi->curr_queue_pairs; i++) {
2102 struct send_queue *sq = &vi->sq[i];
2103
2104 stats_base = (u8 *)&sq->stats;
2105 do {
2106 start = u64_stats_fetch_begin_irq(&sq->stats.syncp);
2107 for (j = 0; j < VIRTNET_SQ_STATS_LEN; j++) {
2108 offset = virtnet_sq_stats_desc[j].offset;
2109 data[idx + j] = *(u64 *)(stats_base + offset);
2110 }
2111 } while (u64_stats_fetch_retry_irq(&sq->stats.syncp, start));
2112 idx += VIRTNET_SQ_STATS_LEN;
2113 }
2114}
2115
Jason Wangd73bcd22012-12-07 07:04:57 +00002116static void virtnet_get_channels(struct net_device *dev,
2117 struct ethtool_channels *channels)
2118{
2119 struct virtnet_info *vi = netdev_priv(dev);
2120
2121 channels->combined_count = vi->curr_queue_pairs;
2122 channels->max_combined = vi->max_queue_pairs;
2123 channels->max_other = 0;
2124 channels->rx_count = 0;
2125 channels->tx_count = 0;
2126 channels->other_count = 0;
2127}
2128
Nikolay Aleksandrov16032be2016-02-03 04:04:37 +01002129/* Check if the user is trying to change anything besides speed/duplex */
Philippe Reynesebb6b4b2017-03-21 23:24:24 +01002130static bool
2131virtnet_validate_ethtool_cmd(const struct ethtool_link_ksettings *cmd)
Nikolay Aleksandrov16032be2016-02-03 04:04:37 +01002132{
Philippe Reynesebb6b4b2017-03-21 23:24:24 +01002133 struct ethtool_link_ksettings diff1 = *cmd;
2134 struct ethtool_link_ksettings diff2 = {};
Nikolay Aleksandrov16032be2016-02-03 04:04:37 +01002135
Nikolay Aleksandrov0cf3ace2016-02-07 21:52:24 +01002136 /* cmd is always set so we need to clear it, validate the port type
2137 * and also without autonegotiation we can ignore advertising
2138 */
Philippe Reynesebb6b4b2017-03-21 23:24:24 +01002139 diff1.base.speed = 0;
2140 diff2.base.port = PORT_OTHER;
2141 ethtool_link_ksettings_zero_link_mode(&diff1, advertising);
2142 diff1.base.duplex = 0;
2143 diff1.base.cmd = 0;
2144 diff1.base.link_mode_masks_nwords = 0;
Nikolay Aleksandrov16032be2016-02-03 04:04:37 +01002145
Philippe Reynesebb6b4b2017-03-21 23:24:24 +01002146 return !memcmp(&diff1.base, &diff2.base, sizeof(diff1.base)) &&
2147 bitmap_empty(diff1.link_modes.supported,
2148 __ETHTOOL_LINK_MODE_MASK_NBITS) &&
2149 bitmap_empty(diff1.link_modes.advertising,
2150 __ETHTOOL_LINK_MODE_MASK_NBITS) &&
2151 bitmap_empty(diff1.link_modes.lp_advertising,
2152 __ETHTOOL_LINK_MODE_MASK_NBITS);
Nikolay Aleksandrov16032be2016-02-03 04:04:37 +01002153}
2154
Philippe Reynesebb6b4b2017-03-21 23:24:24 +01002155static int virtnet_set_link_ksettings(struct net_device *dev,
2156 const struct ethtool_link_ksettings *cmd)
Nikolay Aleksandrov16032be2016-02-03 04:04:37 +01002157{
2158 struct virtnet_info *vi = netdev_priv(dev);
2159 u32 speed;
2160
Philippe Reynesebb6b4b2017-03-21 23:24:24 +01002161 speed = cmd->base.speed;
Nikolay Aleksandrov16032be2016-02-03 04:04:37 +01002162 /* don't allow custom speed and duplex */
2163 if (!ethtool_validate_speed(speed) ||
Philippe Reynesebb6b4b2017-03-21 23:24:24 +01002164 !ethtool_validate_duplex(cmd->base.duplex) ||
Nikolay Aleksandrov16032be2016-02-03 04:04:37 +01002165 !virtnet_validate_ethtool_cmd(cmd))
2166 return -EINVAL;
2167 vi->speed = speed;
Philippe Reynesebb6b4b2017-03-21 23:24:24 +01002168 vi->duplex = cmd->base.duplex;
Nikolay Aleksandrov16032be2016-02-03 04:04:37 +01002169
2170 return 0;
2171}
2172
Philippe Reynesebb6b4b2017-03-21 23:24:24 +01002173static int virtnet_get_link_ksettings(struct net_device *dev,
2174 struct ethtool_link_ksettings *cmd)
Nikolay Aleksandrov16032be2016-02-03 04:04:37 +01002175{
2176 struct virtnet_info *vi = netdev_priv(dev);
2177
Philippe Reynesebb6b4b2017-03-21 23:24:24 +01002178 cmd->base.speed = vi->speed;
2179 cmd->base.duplex = vi->duplex;
2180 cmd->base.port = PORT_OTHER;
Nikolay Aleksandrov16032be2016-02-03 04:04:37 +01002181
2182 return 0;
2183}
2184
2185static void virtnet_init_settings(struct net_device *dev)
2186{
2187 struct virtnet_info *vi = netdev_priv(dev);
2188
2189 vi->speed = SPEED_UNKNOWN;
2190 vi->duplex = DUPLEX_UNKNOWN;
2191}
2192
Jason Baronfaa9b392018-01-05 17:44:54 -05002193static void virtnet_update_settings(struct virtnet_info *vi)
2194{
2195 u32 speed;
2196 u8 duplex;
2197
2198 if (!virtio_has_feature(vi->vdev, VIRTIO_NET_F_SPEED_DUPLEX))
2199 return;
2200
2201 speed = virtio_cread32(vi->vdev, offsetof(struct virtio_net_config,
2202 speed));
2203 if (ethtool_validate_speed(speed))
2204 vi->speed = speed;
2205 duplex = virtio_cread8(vi->vdev, offsetof(struct virtio_net_config,
2206 duplex));
2207 if (ethtool_validate_duplex(duplex))
2208 vi->duplex = duplex;
2209}
2210
Stephen Hemminger0fc0b732009-09-02 01:03:33 -07002211static const struct ethtool_ops virtnet_ethtool_ops = {
Rick Jones66846042011-11-14 14:17:08 +00002212 .get_drvinfo = virtnet_get_drvinfo,
Mark McLoughlin9f4d26d2009-01-19 17:09:49 -08002213 .get_link = ethtool_op_get_link,
Rick Jones8f9f4662011-10-19 08:10:59 +00002214 .get_ringparam = virtnet_get_ringparam,
Toshiaki Makitad7dfc5c2018-01-17 15:38:25 +09002215 .get_strings = virtnet_get_strings,
2216 .get_sset_count = virtnet_get_sset_count,
2217 .get_ethtool_stats = virtnet_get_ethtool_stats,
Jason Wangd73bcd22012-12-07 07:04:57 +00002218 .set_channels = virtnet_set_channels,
2219 .get_channels = virtnet_get_channels,
Jacob Keller074c3582014-06-25 02:37:13 +00002220 .get_ts_info = ethtool_op_get_ts_info,
Philippe Reynesebb6b4b2017-03-21 23:24:24 +01002221 .get_link_ksettings = virtnet_get_link_ksettings,
2222 .set_link_ksettings = virtnet_set_link_ksettings,
Herbert Xua9ea3fc2008-04-18 11:21:42 +08002223};
2224
John Fastabend9fe7bfc2017-02-02 19:16:01 -08002225static void virtnet_freeze_down(struct virtio_device *vdev)
2226{
2227 struct virtnet_info *vi = vdev->priv;
2228 int i;
2229
2230 /* Make sure no work handler is accessing the device */
2231 flush_work(&vi->config_work);
2232
2233 netif_device_detach(vi->dev);
Jason Wang713a98d2017-06-28 09:51:03 +08002234 netif_tx_disable(vi->dev);
John Fastabend9fe7bfc2017-02-02 19:16:01 -08002235 cancel_delayed_work_sync(&vi->refill);
2236
2237 if (netif_running(vi->dev)) {
Willem de Bruijnb92f1e62017-04-24 13:49:27 -04002238 for (i = 0; i < vi->max_queue_pairs; i++) {
John Fastabend9fe7bfc2017-02-02 19:16:01 -08002239 napi_disable(&vi->rq[i].napi);
Willem de Bruijn78a57b42017-04-25 15:59:17 -04002240 virtnet_napi_tx_disable(&vi->sq[i].napi);
Willem de Bruijnb92f1e62017-04-24 13:49:27 -04002241 }
John Fastabend9fe7bfc2017-02-02 19:16:01 -08002242 }
2243}
2244
2245static int init_vqs(struct virtnet_info *vi);
2246
2247static int virtnet_restore_up(struct virtio_device *vdev)
2248{
2249 struct virtnet_info *vi = vdev->priv;
2250 int err, i;
2251
2252 err = init_vqs(vi);
2253 if (err)
2254 return err;
2255
2256 virtio_device_ready(vdev);
2257
2258 if (netif_running(vi->dev)) {
2259 for (i = 0; i < vi->curr_queue_pairs; i++)
2260 if (!try_fill_recv(vi, &vi->rq[i], GFP_KERNEL))
2261 schedule_delayed_work(&vi->refill, 0);
2262
Willem de Bruijnb92f1e62017-04-24 13:49:27 -04002263 for (i = 0; i < vi->max_queue_pairs; i++) {
Willem de Bruijne4e84522017-04-24 13:49:26 -04002264 virtnet_napi_enable(vi->rq[i].vq, &vi->rq[i].napi);
Willem de Bruijnb92f1e62017-04-24 13:49:27 -04002265 virtnet_napi_tx_enable(vi, vi->sq[i].vq,
2266 &vi->sq[i].napi);
2267 }
John Fastabend9fe7bfc2017-02-02 19:16:01 -08002268 }
2269
2270 netif_device_attach(vi->dev);
2271 return err;
2272}
2273
Jason Wang3f935222017-07-19 16:54:49 +08002274static int virtnet_set_guest_offloads(struct virtnet_info *vi, u64 offloads)
2275{
2276 struct scatterlist sg;
Michael S. Tsirkin12e57162018-04-19 08:30:48 +03002277 vi->ctrl->offloads = cpu_to_virtio64(vi->vdev, offloads);
Jason Wang3f935222017-07-19 16:54:49 +08002278
Michael S. Tsirkin12e57162018-04-19 08:30:48 +03002279 sg_init_one(&sg, &vi->ctrl->offloads, sizeof(vi->ctrl->offloads));
Jason Wang3f935222017-07-19 16:54:49 +08002280
2281 if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_GUEST_OFFLOADS,
2282 VIRTIO_NET_CTRL_GUEST_OFFLOADS_SET, &sg)) {
2283 dev_warn(&vi->dev->dev, "Fail to set guest offload. \n");
2284 return -EINVAL;
2285 }
2286
2287 return 0;
2288}
2289
2290static int virtnet_clear_guest_offloads(struct virtnet_info *vi)
2291{
2292 u64 offloads = 0;
2293
2294 if (!vi->guest_offloads)
2295 return 0;
2296
2297 if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_CSUM))
2298 offloads = 1ULL << VIRTIO_NET_F_GUEST_CSUM;
2299
2300 return virtnet_set_guest_offloads(vi, offloads);
2301}
2302
2303static int virtnet_restore_guest_offloads(struct virtnet_info *vi)
2304{
2305 u64 offloads = vi->guest_offloads;
2306
2307 if (!vi->guest_offloads)
2308 return 0;
2309 if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_CSUM))
2310 offloads |= 1ULL << VIRTIO_NET_F_GUEST_CSUM;
2311
2312 return virtnet_set_guest_offloads(vi, offloads);
2313}
2314
Jakub Kicinski9861ce02017-04-30 21:46:48 -07002315static int virtnet_xdp_set(struct net_device *dev, struct bpf_prog *prog,
2316 struct netlink_ext_ack *extack)
John Fastabendf600b692016-12-15 12:13:24 -08002317{
2318 unsigned long int max_sz = PAGE_SIZE - sizeof(struct padded_vnet_hdr);
2319 struct virtnet_info *vi = netdev_priv(dev);
2320 struct bpf_prog *old_prog;
Jason Wang017b29c2017-02-20 11:50:20 +08002321 u16 xdp_qp = 0, curr_qp;
John Fastabend672aafd2016-12-15 12:13:49 -08002322 int i, err;
John Fastabendf600b692016-12-15 12:13:24 -08002323
Jason Wang3f935222017-07-19 16:54:49 +08002324 if (!virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_GUEST_OFFLOADS)
2325 && (virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_TSO4) ||
2326 virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_TSO6) ||
2327 virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_ECN) ||
2328 virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_UFO))) {
Daniel Borkmann4d463c42017-05-03 00:39:17 +02002329 NL_SET_ERR_MSG_MOD(extack, "Can't set XDP while host is implementing LRO, disable LRO first");
John Fastabendf600b692016-12-15 12:13:24 -08002330 return -EOPNOTSUPP;
2331 }
2332
2333 if (vi->mergeable_rx_bufs && !vi->any_header_sg) {
Daniel Borkmann4d463c42017-05-03 00:39:17 +02002334 NL_SET_ERR_MSG_MOD(extack, "XDP expects header/data in single page, any_header_sg required");
John Fastabendf600b692016-12-15 12:13:24 -08002335 return -EINVAL;
2336 }
2337
2338 if (dev->mtu > max_sz) {
Daniel Borkmann4d463c42017-05-03 00:39:17 +02002339 NL_SET_ERR_MSG_MOD(extack, "MTU too large to enable XDP");
John Fastabendf600b692016-12-15 12:13:24 -08002340 netdev_warn(dev, "XDP requires MTU less than %lu\n", max_sz);
2341 return -EINVAL;
2342 }
2343
John Fastabend672aafd2016-12-15 12:13:49 -08002344 curr_qp = vi->curr_queue_pairs - vi->xdp_queue_pairs;
2345 if (prog)
2346 xdp_qp = nr_cpu_ids;
2347
2348 /* XDP requires extra queues for XDP_TX */
2349 if (curr_qp + xdp_qp > vi->max_queue_pairs) {
Daniel Borkmann4d463c42017-05-03 00:39:17 +02002350 NL_SET_ERR_MSG_MOD(extack, "Too few free TX rings available");
John Fastabend672aafd2016-12-15 12:13:49 -08002351 netdev_warn(dev, "request %i queues but max is %i\n",
2352 curr_qp + xdp_qp, vi->max_queue_pairs);
2353 return -ENOMEM;
2354 }
2355
John Fastabend2de2f7f2017-02-02 19:16:29 -08002356 if (prog) {
2357 prog = bpf_prog_add(prog, vi->max_queue_pairs - 1);
2358 if (IS_ERR(prog))
2359 return PTR_ERR(prog);
2360 }
2361
Jason Wang4941d472017-07-19 16:54:48 +08002362 /* Make sure NAPI is not using any XDP TX queues for RX. */
Jason Wang4e09ff52018-02-28 18:20:04 +08002363 if (netif_running(dev))
2364 for (i = 0; i < vi->max_queue_pairs; i++)
2365 napi_disable(&vi->rq[i].napi);
John Fastabendf600b692016-12-15 12:13:24 -08002366
John Fastabend672aafd2016-12-15 12:13:49 -08002367 netif_set_real_num_rx_queues(dev, curr_qp + xdp_qp);
Jason Wang4941d472017-07-19 16:54:48 +08002368 err = _virtnet_set_queues(vi, curr_qp + xdp_qp);
2369 if (err)
2370 goto err;
2371 vi->xdp_queue_pairs = xdp_qp;
John Fastabend672aafd2016-12-15 12:13:49 -08002372
John Fastabendf600b692016-12-15 12:13:24 -08002373 for (i = 0; i < vi->max_queue_pairs; i++) {
2374 old_prog = rtnl_dereference(vi->rq[i].xdp_prog);
2375 rcu_assign_pointer(vi->rq[i].xdp_prog, prog);
Jason Wang3f935222017-07-19 16:54:49 +08002376 if (i == 0) {
2377 if (!old_prog)
2378 virtnet_clear_guest_offloads(vi);
2379 if (!prog)
2380 virtnet_restore_guest_offloads(vi);
2381 }
John Fastabendf600b692016-12-15 12:13:24 -08002382 if (old_prog)
2383 bpf_prog_put(old_prog);
Jason Wang4e09ff52018-02-28 18:20:04 +08002384 if (netif_running(dev))
2385 virtnet_napi_enable(vi->rq[i].vq, &vi->rq[i].napi);
John Fastabendf600b692016-12-15 12:13:24 -08002386 }
2387
2388 return 0;
John Fastabend2de2f7f2017-02-02 19:16:29 -08002389
Jason Wang4941d472017-07-19 16:54:48 +08002390err:
2391 for (i = 0; i < vi->max_queue_pairs; i++)
2392 virtnet_napi_enable(vi->rq[i].vq, &vi->rq[i].napi);
John Fastabend2de2f7f2017-02-02 19:16:29 -08002393 if (prog)
2394 bpf_prog_sub(prog, vi->max_queue_pairs - 1);
2395 return err;
John Fastabendf600b692016-12-15 12:13:24 -08002396}
2397
Martin KaFai Lau5b0e6622017-06-15 17:29:12 -07002398static u32 virtnet_xdp_query(struct net_device *dev)
John Fastabendf600b692016-12-15 12:13:24 -08002399{
2400 struct virtnet_info *vi = netdev_priv(dev);
Martin KaFai Lau5b0e6622017-06-15 17:29:12 -07002401 const struct bpf_prog *xdp_prog;
John Fastabendf600b692016-12-15 12:13:24 -08002402 int i;
2403
2404 for (i = 0; i < vi->max_queue_pairs; i++) {
Martin KaFai Lau5b0e6622017-06-15 17:29:12 -07002405 xdp_prog = rtnl_dereference(vi->rq[i].xdp_prog);
2406 if (xdp_prog)
2407 return xdp_prog->aux->id;
John Fastabendf600b692016-12-15 12:13:24 -08002408 }
Martin KaFai Lau5b0e6622017-06-15 17:29:12 -07002409 return 0;
John Fastabendf600b692016-12-15 12:13:24 -08002410}
2411
Jakub Kicinskif4e63522017-11-03 13:56:16 -07002412static int virtnet_xdp(struct net_device *dev, struct netdev_bpf *xdp)
John Fastabendf600b692016-12-15 12:13:24 -08002413{
2414 switch (xdp->command) {
2415 case XDP_SETUP_PROG:
Jakub Kicinski9861ce02017-04-30 21:46:48 -07002416 return virtnet_xdp_set(dev, xdp->prog, xdp->extack);
John Fastabendf600b692016-12-15 12:13:24 -08002417 case XDP_QUERY_PROG:
Martin KaFai Lau5b0e6622017-06-15 17:29:12 -07002418 xdp->prog_id = virtnet_xdp_query(dev);
John Fastabendf600b692016-12-15 12:13:24 -08002419 return 0;
2420 default:
2421 return -EINVAL;
2422 }
2423}
2424
Sridhar Samudralaba5e4422018-05-24 09:55:17 -07002425static int virtnet_get_phys_port_name(struct net_device *dev, char *buf,
2426 size_t len)
2427{
2428 struct virtnet_info *vi = netdev_priv(dev);
2429 int ret;
2430
2431 if (!virtio_has_feature(vi->vdev, VIRTIO_NET_F_STANDBY))
2432 return -EOPNOTSUPP;
2433
2434 ret = snprintf(buf, len, "sby");
2435 if (ret >= len)
2436 return -EOPNOTSUPP;
2437
2438 return 0;
2439}
2440
Stephen Hemminger76288b42009-01-06 10:44:22 -08002441static const struct net_device_ops virtnet_netdev = {
2442 .ndo_open = virtnet_open,
2443 .ndo_stop = virtnet_close,
2444 .ndo_start_xmit = start_xmit,
2445 .ndo_validate_addr = eth_validate_addr,
Alex Williamson9c46f6d2009-02-04 16:36:34 -08002446 .ndo_set_mac_address = virtnet_set_mac_address,
Alex Williamson2af76982009-02-04 09:02:40 +00002447 .ndo_set_rx_mode = virtnet_set_rx_mode,
stephen hemminger3fa2a1d2011-06-15 06:36:29 +00002448 .ndo_get_stats64 = virtnet_stats,
Alex Williamson1824a982009-05-01 17:31:10 +00002449 .ndo_vlan_rx_add_vid = virtnet_vlan_rx_add_vid,
2450 .ndo_vlan_rx_kill_vid = virtnet_vlan_rx_kill_vid,
Stephen Hemminger76288b42009-01-06 10:44:22 -08002451#ifdef CONFIG_NET_POLL_CONTROLLER
2452 .ndo_poll_controller = virtnet_netpoll,
2453#endif
Jakub Kicinskif4e63522017-11-03 13:56:16 -07002454 .ndo_bpf = virtnet_xdp,
Jason Wang186b3c92017-09-19 17:42:43 +08002455 .ndo_xdp_xmit = virtnet_xdp_xmit,
Vlad Yasevich2836b4f2017-05-23 13:38:43 -04002456 .ndo_features_check = passthru_features_check,
Sridhar Samudralaba5e4422018-05-24 09:55:17 -07002457 .ndo_get_phys_port_name = virtnet_get_phys_port_name,
Stephen Hemminger76288b42009-01-06 10:44:22 -08002458};
2459
Jason Wang586d17c2012-04-11 20:43:52 +00002460static void virtnet_config_changed_work(struct work_struct *work)
Mark McLoughlin9f4d26d2009-01-19 17:09:49 -08002461{
Jason Wang586d17c2012-04-11 20:43:52 +00002462 struct virtnet_info *vi =
2463 container_of(work, struct virtnet_info, config_work);
Mark McLoughlin9f4d26d2009-01-19 17:09:49 -08002464 u16 v;
2465
Rusty Russell855e0c52013-10-14 18:11:51 +10302466 if (virtio_cread_feature(vi->vdev, VIRTIO_NET_F_STATUS,
2467 struct virtio_net_config, status, &v) < 0)
Michael S. Tsirkin507613b2014-10-15 10:22:30 +10302468 return;
Jason Wang586d17c2012-04-11 20:43:52 +00002469
2470 if (v & VIRTIO_NET_S_ANNOUNCE) {
Amerigo Wangee89bab2012-08-09 22:14:56 +00002471 netdev_notify_peers(vi->dev);
Jason Wang586d17c2012-04-11 20:43:52 +00002472 virtnet_ack_link_announce(vi);
2473 }
Mark McLoughlin9f4d26d2009-01-19 17:09:49 -08002474
2475 /* Ignore unknown (future) status bits */
2476 v &= VIRTIO_NET_S_LINK_UP;
2477
2478 if (vi->status == v)
Michael S. Tsirkin507613b2014-10-15 10:22:30 +10302479 return;
Mark McLoughlin9f4d26d2009-01-19 17:09:49 -08002480
2481 vi->status = v;
2482
2483 if (vi->status & VIRTIO_NET_S_LINK_UP) {
Jason Baronfaa9b392018-01-05 17:44:54 -05002484 virtnet_update_settings(vi);
Mark McLoughlin9f4d26d2009-01-19 17:09:49 -08002485 netif_carrier_on(vi->dev);
Jason Wang986a4f42012-12-07 07:04:56 +00002486 netif_tx_wake_all_queues(vi->dev);
Mark McLoughlin9f4d26d2009-01-19 17:09:49 -08002487 } else {
2488 netif_carrier_off(vi->dev);
Jason Wang986a4f42012-12-07 07:04:56 +00002489 netif_tx_stop_all_queues(vi->dev);
Mark McLoughlin9f4d26d2009-01-19 17:09:49 -08002490 }
2491}
2492
2493static void virtnet_config_changed(struct virtio_device *vdev)
2494{
2495 struct virtnet_info *vi = vdev->priv;
2496
Tejun Heo3b07e9c2012-08-20 14:51:24 -07002497 schedule_work(&vi->config_work);
Mark McLoughlin9f4d26d2009-01-19 17:09:49 -08002498}
2499
Jason Wang986a4f42012-12-07 07:04:56 +00002500static void virtnet_free_queues(struct virtnet_info *vi)
2501{
Andrey Vagind4fb84e2013-12-05 18:36:21 +04002502 int i;
2503
Jason Wangab3971b2015-03-12 13:57:44 +08002504 for (i = 0; i < vi->max_queue_pairs; i++) {
2505 napi_hash_del(&vi->rq[i].napi);
Andrey Vagind4fb84e2013-12-05 18:36:21 +04002506 netif_napi_del(&vi->rq[i].napi);
Willem de Bruijnb92f1e62017-04-24 13:49:27 -04002507 netif_napi_del(&vi->sq[i].napi);
Jason Wangab3971b2015-03-12 13:57:44 +08002508 }
Andrey Vagind4fb84e2013-12-05 18:36:21 +04002509
Eric Dumazet963abe52016-11-15 22:24:12 -08002510 /* We called napi_hash_del() before netif_napi_del(),
2511 * we need to respect an RCU grace period before freeing vi->rq
2512 */
2513 synchronize_net();
2514
Jason Wang986a4f42012-12-07 07:04:56 +00002515 kfree(vi->rq);
2516 kfree(vi->sq);
Michael S. Tsirkin12e57162018-04-19 08:30:48 +03002517 kfree(vi->ctrl);
Jason Wang986a4f42012-12-07 07:04:56 +00002518}
2519
John Fastabend473153292017-02-02 19:14:32 -08002520static void _free_receive_bufs(struct virtnet_info *vi)
Jason Wang986a4f42012-12-07 07:04:56 +00002521{
John Fastabendf600b692016-12-15 12:13:24 -08002522 struct bpf_prog *old_prog;
Jason Wang986a4f42012-12-07 07:04:56 +00002523 int i;
2524
2525 for (i = 0; i < vi->max_queue_pairs; i++) {
2526 while (vi->rq[i].pages)
2527 __free_pages(get_a_page(&vi->rq[i], GFP_KERNEL), 0);
John Fastabendf600b692016-12-15 12:13:24 -08002528
2529 old_prog = rtnl_dereference(vi->rq[i].xdp_prog);
2530 RCU_INIT_POINTER(vi->rq[i].xdp_prog, NULL);
2531 if (old_prog)
2532 bpf_prog_put(old_prog);
Jason Wang986a4f42012-12-07 07:04:56 +00002533 }
John Fastabend473153292017-02-02 19:14:32 -08002534}
2535
2536static void free_receive_bufs(struct virtnet_info *vi)
2537{
2538 rtnl_lock();
2539 _free_receive_bufs(vi);
John Fastabendf600b692016-12-15 12:13:24 -08002540 rtnl_unlock();
Jason Wang986a4f42012-12-07 07:04:56 +00002541}
2542
Michael Daltonfb518792014-01-16 22:23:26 -08002543static void free_receive_page_frags(struct virtnet_info *vi)
2544{
2545 int i;
2546 for (i = 0; i < vi->max_queue_pairs; i++)
2547 if (vi->rq[i].alloc_frag.page)
2548 put_page(vi->rq[i].alloc_frag.page);
2549}
2550
John Fastabendb68df012017-01-25 18:22:48 -08002551static bool is_xdp_raw_buffer_queue(struct virtnet_info *vi, int q)
John Fastabend56434a02016-12-15 12:14:13 -08002552{
2553 if (q < (vi->curr_queue_pairs - vi->xdp_queue_pairs))
2554 return false;
2555 else if (q < vi->curr_queue_pairs)
2556 return true;
2557 else
2558 return false;
2559}
2560
Jason Wang986a4f42012-12-07 07:04:56 +00002561static void free_unused_bufs(struct virtnet_info *vi)
2562{
2563 void *buf;
2564 int i;
2565
2566 for (i = 0; i < vi->max_queue_pairs; i++) {
2567 struct virtqueue *vq = vi->sq[i].vq;
John Fastabend56434a02016-12-15 12:14:13 -08002568 while ((buf = virtqueue_detach_unused_buf(vq)) != NULL) {
John Fastabendb68df012017-01-25 18:22:48 -08002569 if (!is_xdp_raw_buffer_queue(vi, i))
John Fastabend56434a02016-12-15 12:14:13 -08002570 dev_kfree_skb(buf);
2571 else
2572 put_page(virt_to_head_page(buf));
2573 }
Jason Wang986a4f42012-12-07 07:04:56 +00002574 }
2575
2576 for (i = 0; i < vi->max_queue_pairs; i++) {
2577 struct virtqueue *vq = vi->rq[i].vq;
2578
2579 while ((buf = virtqueue_detach_unused_buf(vq)) != NULL) {
Michael Daltonab7db912014-01-16 22:23:27 -08002580 if (vi->mergeable_rx_bufs) {
Michael S. Tsirkin680557c2017-03-06 21:29:47 +02002581 put_page(virt_to_head_page(buf));
Michael Daltonab7db912014-01-16 22:23:27 -08002582 } else if (vi->big_packets) {
Andrey Vaginfa9fac12013-12-05 18:36:20 +04002583 give_pages(&vi->rq[i], buf);
Michael Daltonab7db912014-01-16 22:23:27 -08002584 } else {
Jason Wangf6b10202017-02-21 16:46:28 +08002585 put_page(virt_to_head_page(buf));
Michael Daltonab7db912014-01-16 22:23:27 -08002586 }
Jason Wang986a4f42012-12-07 07:04:56 +00002587 }
Jason Wang986a4f42012-12-07 07:04:56 +00002588 }
2589}
2590
Jason Wange9d74172012-12-07 07:04:55 +00002591static void virtnet_del_vqs(struct virtnet_info *vi)
2592{
2593 struct virtio_device *vdev = vi->vdev;
2594
Wanlong Gao8898c212013-01-24 23:51:30 +00002595 virtnet_clean_affinity(vi, -1);
Jason Wang986a4f42012-12-07 07:04:56 +00002596
Jason Wange9d74172012-12-07 07:04:55 +00002597 vdev->config->del_vqs(vdev);
Jason Wang986a4f42012-12-07 07:04:56 +00002598
2599 virtnet_free_queues(vi);
2600}
2601
Michael S. Tsirkind85b758f72017-03-09 02:21:21 +02002602/* How large should a single buffer be so a queue full of these can fit at
2603 * least one full packet?
2604 * Logic below assumes the mergeable buffer header is used.
2605 */
2606static unsigned int mergeable_min_buf_len(struct virtnet_info *vi, struct virtqueue *vq)
2607{
2608 const unsigned int hdr_len = sizeof(struct virtio_net_hdr_mrg_rxbuf);
2609 unsigned int rq_size = virtqueue_get_vring_size(vq);
2610 unsigned int packet_len = vi->big_packets ? IP_MAX_MTU : vi->dev->max_mtu;
2611 unsigned int buf_len = hdr_len + ETH_HLEN + VLAN_HLEN + packet_len;
2612 unsigned int min_buf_len = DIV_ROUND_UP(buf_len, rq_size);
2613
Michael S. Tsirkinf0c31922017-06-02 17:54:33 +03002614 return max(max(min_buf_len, hdr_len) - hdr_len,
2615 (unsigned int)GOOD_PACKET_LEN);
Michael S. Tsirkind85b758f72017-03-09 02:21:21 +02002616}
2617
Jason Wang986a4f42012-12-07 07:04:56 +00002618static int virtnet_find_vqs(struct virtnet_info *vi)
2619{
2620 vq_callback_t **callbacks;
2621 struct virtqueue **vqs;
2622 int ret = -ENOMEM;
2623 int i, total_vqs;
2624 const char **names;
Michael S. Tsirkind45b8972017-03-06 20:31:21 +02002625 bool *ctx;
Jason Wang986a4f42012-12-07 07:04:56 +00002626
2627 /* We expect 1 RX virtqueue followed by 1 TX virtqueue, followed by
2628 * possible N-1 RX/TX queue pairs used in multiqueue mode, followed by
2629 * possible control vq.
2630 */
2631 total_vqs = vi->max_queue_pairs * 2 +
2632 virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_VQ);
2633
2634 /* Allocate space for find_vqs parameters */
Kees Cook6396bb22018-06-12 14:03:40 -07002635 vqs = kcalloc(total_vqs, sizeof(*vqs), GFP_KERNEL);
Jason Wang986a4f42012-12-07 07:04:56 +00002636 if (!vqs)
2637 goto err_vq;
Kees Cook6da2ec52018-06-12 13:55:00 -07002638 callbacks = kmalloc_array(total_vqs, sizeof(*callbacks), GFP_KERNEL);
Jason Wang986a4f42012-12-07 07:04:56 +00002639 if (!callbacks)
2640 goto err_callback;
Kees Cook6da2ec52018-06-12 13:55:00 -07002641 names = kmalloc_array(total_vqs, sizeof(*names), GFP_KERNEL);
Jason Wang986a4f42012-12-07 07:04:56 +00002642 if (!names)
2643 goto err_names;
Jason Wang192f68c2017-07-19 16:54:47 +08002644 if (!vi->big_packets || vi->mergeable_rx_bufs) {
Kees Cook6396bb22018-06-12 14:03:40 -07002645 ctx = kcalloc(total_vqs, sizeof(*ctx), GFP_KERNEL);
Michael S. Tsirkind45b8972017-03-06 20:31:21 +02002646 if (!ctx)
2647 goto err_ctx;
2648 } else {
2649 ctx = NULL;
2650 }
Jason Wang986a4f42012-12-07 07:04:56 +00002651
2652 /* Parameters for control virtqueue, if any */
2653 if (vi->has_cvq) {
2654 callbacks[total_vqs - 1] = NULL;
2655 names[total_vqs - 1] = "control";
2656 }
2657
2658 /* Allocate/initialize parameters for send/receive virtqueues */
2659 for (i = 0; i < vi->max_queue_pairs; i++) {
2660 callbacks[rxq2vq(i)] = skb_recv_done;
2661 callbacks[txq2vq(i)] = skb_xmit_done;
2662 sprintf(vi->rq[i].name, "input.%d", i);
2663 sprintf(vi->sq[i].name, "output.%d", i);
2664 names[rxq2vq(i)] = vi->rq[i].name;
2665 names[txq2vq(i)] = vi->sq[i].name;
Michael S. Tsirkind45b8972017-03-06 20:31:21 +02002666 if (ctx)
2667 ctx[rxq2vq(i)] = true;
Jason Wang986a4f42012-12-07 07:04:56 +00002668 }
2669
2670 ret = vi->vdev->config->find_vqs(vi->vdev, total_vqs, vqs, callbacks,
Michael S. Tsirkind45b8972017-03-06 20:31:21 +02002671 names, ctx, NULL);
Jason Wang986a4f42012-12-07 07:04:56 +00002672 if (ret)
2673 goto err_find;
2674
2675 if (vi->has_cvq) {
2676 vi->cvq = vqs[total_vqs - 1];
2677 if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_VLAN))
Patrick McHardyf6469682013-04-19 02:04:27 +00002678 vi->dev->features |= NETIF_F_HW_VLAN_CTAG_FILTER;
Jason Wang986a4f42012-12-07 07:04:56 +00002679 }
2680
2681 for (i = 0; i < vi->max_queue_pairs; i++) {
2682 vi->rq[i].vq = vqs[rxq2vq(i)];
Michael S. Tsirkind85b758f72017-03-09 02:21:21 +02002683 vi->rq[i].min_buf_len = mergeable_min_buf_len(vi, vi->rq[i].vq);
Jason Wang986a4f42012-12-07 07:04:56 +00002684 vi->sq[i].vq = vqs[txq2vq(i)];
2685 }
2686
Tonghao Zhang2fa3c8a2018-05-31 07:16:32 -07002687 /* run here: ret == 0. */
Jason Wang986a4f42012-12-07 07:04:56 +00002688
Jason Wang986a4f42012-12-07 07:04:56 +00002689
2690err_find:
Michael S. Tsirkind45b8972017-03-06 20:31:21 +02002691 kfree(ctx);
2692err_ctx:
Jason Wang986a4f42012-12-07 07:04:56 +00002693 kfree(names);
2694err_names:
2695 kfree(callbacks);
2696err_callback:
2697 kfree(vqs);
2698err_vq:
2699 return ret;
2700}
2701
2702static int virtnet_alloc_queues(struct virtnet_info *vi)
2703{
2704 int i;
2705
Michael S. Tsirkin12e57162018-04-19 08:30:48 +03002706 vi->ctrl = kzalloc(sizeof(*vi->ctrl), GFP_KERNEL);
2707 if (!vi->ctrl)
2708 goto err_ctrl;
Kees Cook6396bb22018-06-12 14:03:40 -07002709 vi->sq = kcalloc(vi->max_queue_pairs, sizeof(*vi->sq), GFP_KERNEL);
Jason Wang986a4f42012-12-07 07:04:56 +00002710 if (!vi->sq)
2711 goto err_sq;
Kees Cook6396bb22018-06-12 14:03:40 -07002712 vi->rq = kcalloc(vi->max_queue_pairs, sizeof(*vi->rq), GFP_KERNEL);
Amerigo Wang008d4272012-12-10 02:24:08 +00002713 if (!vi->rq)
Jason Wang986a4f42012-12-07 07:04:56 +00002714 goto err_rq;
2715
2716 INIT_DELAYED_WORK(&vi->refill, refill_work);
2717 for (i = 0; i < vi->max_queue_pairs; i++) {
2718 vi->rq[i].pages = NULL;
2719 netif_napi_add(vi->dev, &vi->rq[i].napi, virtnet_poll,
2720 napi_weight);
Willem de Bruijn1d11e732017-04-27 20:37:58 -04002721 netif_tx_napi_add(vi->dev, &vi->sq[i].napi, virtnet_poll_tx,
2722 napi_tx ? napi_weight : 0);
Jason Wang986a4f42012-12-07 07:04:56 +00002723
2724 sg_init_table(vi->rq[i].sg, ARRAY_SIZE(vi->rq[i].sg));
Johannes Berg5377d7582015-08-19 09:48:40 +02002725 ewma_pkt_len_init(&vi->rq[i].mrg_avg_pkt_len);
Jason Wang986a4f42012-12-07 07:04:56 +00002726 sg_init_table(vi->sq[i].sg, ARRAY_SIZE(vi->sq[i].sg));
Toshiaki Makitad7dfc5c2018-01-17 15:38:25 +09002727
2728 u64_stats_init(&vi->rq[i].stats.syncp);
2729 u64_stats_init(&vi->sq[i].stats.syncp);
Jason Wang986a4f42012-12-07 07:04:56 +00002730 }
2731
2732 return 0;
2733
2734err_rq:
2735 kfree(vi->sq);
2736err_sq:
Michael S. Tsirkin12e57162018-04-19 08:30:48 +03002737 kfree(vi->ctrl);
2738err_ctrl:
Jason Wang986a4f42012-12-07 07:04:56 +00002739 return -ENOMEM;
Jason Wange9d74172012-12-07 07:04:55 +00002740}
2741
Amit Shah3f9c10b2011-12-22 16:58:31 +05302742static int init_vqs(struct virtnet_info *vi)
2743{
Jason Wang986a4f42012-12-07 07:04:56 +00002744 int ret;
Amit Shah3f9c10b2011-12-22 16:58:31 +05302745
Jason Wang986a4f42012-12-07 07:04:56 +00002746 /* Allocate send & receive queues */
2747 ret = virtnet_alloc_queues(vi);
2748 if (ret)
2749 goto err;
Amit Shah3f9c10b2011-12-22 16:58:31 +05302750
Jason Wang986a4f42012-12-07 07:04:56 +00002751 ret = virtnet_find_vqs(vi);
2752 if (ret)
2753 goto err_free;
Amit Shah3f9c10b2011-12-22 16:58:31 +05302754
Wanlong Gao47be2472013-01-24 23:51:29 +00002755 get_online_cpus();
Wanlong Gao8898c212013-01-24 23:51:30 +00002756 virtnet_set_affinity(vi);
Wanlong Gao47be2472013-01-24 23:51:29 +00002757 put_online_cpus();
2758
Amit Shah3f9c10b2011-12-22 16:58:31 +05302759 return 0;
Jason Wang986a4f42012-12-07 07:04:56 +00002760
2761err_free:
2762 virtnet_free_queues(vi);
2763err:
2764 return ret;
Amit Shah3f9c10b2011-12-22 16:58:31 +05302765}
2766
Michael Daltonfbf28d72014-01-16 22:23:30 -08002767#ifdef CONFIG_SYSFS
2768static ssize_t mergeable_rx_buffer_size_show(struct netdev_rx_queue *queue,
stephen hemminger718ad682017-08-18 13:46:24 -07002769 char *buf)
Michael Daltonfbf28d72014-01-16 22:23:30 -08002770{
2771 struct virtnet_info *vi = netdev_priv(queue->dev);
2772 unsigned int queue_index = get_netdev_rx_queue_index(queue);
Jason Wang3cc81a92018-03-02 17:29:14 +08002773 unsigned int headroom = virtnet_get_headroom(vi);
2774 unsigned int tailroom = headroom ? sizeof(struct skb_shared_info) : 0;
Johannes Berg5377d7582015-08-19 09:48:40 +02002775 struct ewma_pkt_len *avg;
Michael Daltonfbf28d72014-01-16 22:23:30 -08002776
2777 BUG_ON(queue_index >= vi->max_queue_pairs);
2778 avg = &vi->rq[queue_index].mrg_avg_pkt_len;
Michael S. Tsirkind85b758f72017-03-09 02:21:21 +02002779 return sprintf(buf, "%u\n",
Jason Wang3cc81a92018-03-02 17:29:14 +08002780 get_mergeable_buf_len(&vi->rq[queue_index], avg,
2781 SKB_DATA_ALIGN(headroom + tailroom)));
Michael Daltonfbf28d72014-01-16 22:23:30 -08002782}
2783
2784static struct rx_queue_attribute mergeable_rx_buffer_size_attribute =
2785 __ATTR_RO(mergeable_rx_buffer_size);
2786
2787static struct attribute *virtio_net_mrg_rx_attrs[] = {
2788 &mergeable_rx_buffer_size_attribute.attr,
2789 NULL
2790};
2791
2792static const struct attribute_group virtio_net_mrg_rx_group = {
2793 .name = "virtio_net",
2794 .attrs = virtio_net_mrg_rx_attrs
2795};
2796#endif
2797
Jason Wang892d6eb2014-11-20 17:03:05 +08002798static bool virtnet_fail_on_feature(struct virtio_device *vdev,
2799 unsigned int fbit,
2800 const char *fname, const char *dname)
2801{
2802 if (!virtio_has_feature(vdev, fbit))
2803 return false;
2804
2805 dev_err(&vdev->dev, "device advertises feature %s but not %s",
2806 fname, dname);
2807
2808 return true;
2809}
2810
2811#define VIRTNET_FAIL_ON(vdev, fbit, dbit) \
2812 virtnet_fail_on_feature(vdev, fbit, #fbit, dbit)
2813
2814static bool virtnet_validate_features(struct virtio_device *vdev)
2815{
2816 if (!virtio_has_feature(vdev, VIRTIO_NET_F_CTRL_VQ) &&
2817 (VIRTNET_FAIL_ON(vdev, VIRTIO_NET_F_CTRL_RX,
2818 "VIRTIO_NET_F_CTRL_VQ") ||
2819 VIRTNET_FAIL_ON(vdev, VIRTIO_NET_F_CTRL_VLAN,
2820 "VIRTIO_NET_F_CTRL_VQ") ||
2821 VIRTNET_FAIL_ON(vdev, VIRTIO_NET_F_GUEST_ANNOUNCE,
2822 "VIRTIO_NET_F_CTRL_VQ") ||
2823 VIRTNET_FAIL_ON(vdev, VIRTIO_NET_F_MQ, "VIRTIO_NET_F_CTRL_VQ") ||
2824 VIRTNET_FAIL_ON(vdev, VIRTIO_NET_F_CTRL_MAC_ADDR,
2825 "VIRTIO_NET_F_CTRL_VQ"))) {
2826 return false;
2827 }
2828
2829 return true;
2830}
2831
Jarod Wilsond0c2c992016-10-20 13:55:21 -04002832#define MIN_MTU ETH_MIN_MTU
2833#define MAX_MTU ETH_MAX_MTU
2834
Michael S. Tsirkinfe36cbe2017-03-29 19:09:14 +03002835static int virtnet_validate(struct virtio_device *vdev)
Rusty Russell296f96f2007-10-22 11:03:37 +10002836{
Michael S. Tsirkin6ba42242015-01-12 16:23:37 +02002837 if (!vdev->config->get) {
2838 dev_err(&vdev->dev, "%s failure: config access disabled\n",
2839 __func__);
2840 return -EINVAL;
2841 }
2842
Jason Wang892d6eb2014-11-20 17:03:05 +08002843 if (!virtnet_validate_features(vdev))
2844 return -EINVAL;
2845
Michael S. Tsirkinfe36cbe2017-03-29 19:09:14 +03002846 if (virtio_has_feature(vdev, VIRTIO_NET_F_MTU)) {
2847 int mtu = virtio_cread16(vdev,
2848 offsetof(struct virtio_net_config,
2849 mtu));
2850 if (mtu < MIN_MTU)
2851 __virtio_clear_bit(vdev, VIRTIO_NET_F_MTU);
2852 }
2853
2854 return 0;
2855}
2856
2857static int virtnet_probe(struct virtio_device *vdev)
2858{
Toshiaki Makitad7dfc5c2018-01-17 15:38:25 +09002859 int i, err = -ENOMEM;
Michael S. Tsirkinfe36cbe2017-03-29 19:09:14 +03002860 struct net_device *dev;
2861 struct virtnet_info *vi;
2862 u16 max_queue_pairs;
2863 int mtu;
2864
Jason Wang986a4f42012-12-07 07:04:56 +00002865 /* Find if host supports multiqueue virtio_net device */
Rusty Russell855e0c52013-10-14 18:11:51 +10302866 err = virtio_cread_feature(vdev, VIRTIO_NET_F_MQ,
2867 struct virtio_net_config,
2868 max_virtqueue_pairs, &max_queue_pairs);
Jason Wang986a4f42012-12-07 07:04:56 +00002869
2870 /* We need at least 2 queue's */
2871 if (err || max_queue_pairs < VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MIN ||
2872 max_queue_pairs > VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX ||
2873 !virtio_has_feature(vdev, VIRTIO_NET_F_CTRL_VQ))
2874 max_queue_pairs = 1;
Rusty Russell296f96f2007-10-22 11:03:37 +10002875
2876 /* Allocate ourselves a network device with room for our info */
Jason Wang986a4f42012-12-07 07:04:56 +00002877 dev = alloc_etherdev_mq(sizeof(struct virtnet_info), max_queue_pairs);
Rusty Russell296f96f2007-10-22 11:03:37 +10002878 if (!dev)
2879 return -ENOMEM;
2880
2881 /* Set up network device as normal. */
Jiri Pirkof2f2c8b2012-06-29 05:10:06 +00002882 dev->priv_flags |= IFF_UNICAST_FLT | IFF_LIVE_ADDR_CHANGE;
Stephen Hemminger76288b42009-01-06 10:44:22 -08002883 dev->netdev_ops = &virtnet_netdev;
Rusty Russell296f96f2007-10-22 11:03:37 +10002884 dev->features = NETIF_F_HIGHDMA;
stephen hemminger3fa2a1d2011-06-15 06:36:29 +00002885
Wilfried Klaebe7ad24ea2014-05-11 00:12:32 +00002886 dev->ethtool_ops = &virtnet_ethtool_ops;
Rusty Russell296f96f2007-10-22 11:03:37 +10002887 SET_NETDEV_DEV(dev, &vdev->dev);
2888
2889 /* Do we support "hardware" checksums? */
Michał Mirosław98e778c2011-03-31 01:01:35 +00002890 if (virtio_has_feature(vdev, VIRTIO_NET_F_CSUM)) {
Rusty Russell296f96f2007-10-22 11:03:37 +10002891 /* This opens up the world of extra features. */
Jason Wang48900cb2015-08-05 10:34:04 +08002892 dev->hw_features |= NETIF_F_HW_CSUM | NETIF_F_SG;
Michał Mirosław98e778c2011-03-31 01:01:35 +00002893 if (csum)
Jason Wang48900cb2015-08-05 10:34:04 +08002894 dev->features |= NETIF_F_HW_CSUM | NETIF_F_SG;
Michał Mirosław98e778c2011-03-31 01:01:35 +00002895
2896 if (virtio_has_feature(vdev, VIRTIO_NET_F_GSO)) {
David S. Millere078de02017-07-03 06:37:32 -07002897 dev->hw_features |= NETIF_F_TSO
Rusty Russell34a48572008-02-04 23:50:02 -05002898 | NETIF_F_TSO_ECN | NETIF_F_TSO6;
2899 }
Rusty Russell5539ae962008-05-02 21:50:46 -05002900 /* Individual feature bits: what can host handle? */
Michał Mirosław98e778c2011-03-31 01:01:35 +00002901 if (virtio_has_feature(vdev, VIRTIO_NET_F_HOST_TSO4))
2902 dev->hw_features |= NETIF_F_TSO;
2903 if (virtio_has_feature(vdev, VIRTIO_NET_F_HOST_TSO6))
2904 dev->hw_features |= NETIF_F_TSO6;
2905 if (virtio_has_feature(vdev, VIRTIO_NET_F_HOST_ECN))
2906 dev->hw_features |= NETIF_F_TSO_ECN;
Michał Mirosław98e778c2011-03-31 01:01:35 +00002907
Jason Wang41f2f122014-12-24 11:03:52 +08002908 dev->features |= NETIF_F_GSO_ROBUST;
2909
Michał Mirosław98e778c2011-03-31 01:01:35 +00002910 if (gso)
David S. Millere078de02017-07-03 06:37:32 -07002911 dev->features |= dev->hw_features & NETIF_F_ALL_TSO;
Michał Mirosław98e778c2011-03-31 01:01:35 +00002912 /* (!csum && gso) case will be fixed by register_netdev() */
Rusty Russell296f96f2007-10-22 11:03:37 +10002913 }
Thomas Huth4f491292013-08-27 17:09:02 +02002914 if (virtio_has_feature(vdev, VIRTIO_NET_F_GUEST_CSUM))
2915 dev->features |= NETIF_F_RXCSUM;
Rusty Russell296f96f2007-10-22 11:03:37 +10002916
Jason Wang4fda8302013-04-10 23:32:21 +00002917 dev->vlan_features = dev->features;
2918
Jarod Wilsond0c2c992016-10-20 13:55:21 -04002919 /* MTU range: 68 - 65535 */
2920 dev->min_mtu = MIN_MTU;
2921 dev->max_mtu = MAX_MTU;
2922
Rusty Russell296f96f2007-10-22 11:03:37 +10002923 /* Configuration may specify what MAC to use. Otherwise random. */
Rusty Russell855e0c52013-10-14 18:11:51 +10302924 if (virtio_has_feature(vdev, VIRTIO_NET_F_MAC))
2925 virtio_cread_bytes(vdev,
2926 offsetof(struct virtio_net_config, mac),
2927 dev->dev_addr, dev->addr_len);
2928 else
Danny Kukawkaf2cedb62012-02-15 06:45:39 +00002929 eth_hw_addr_random(dev);
Rusty Russell296f96f2007-10-22 11:03:37 +10002930
2931 /* Set up our device-specific information */
2932 vi = netdev_priv(dev);
Rusty Russell296f96f2007-10-22 11:03:37 +10002933 vi->dev = dev;
2934 vi->vdev = vdev;
Christian Borntraegerd9d5dcc2008-02-18 10:02:51 +01002935 vdev->priv = vi;
John Stultz827da442013-10-07 15:51:58 -07002936
Jason Wang586d17c2012-04-11 20:43:52 +00002937 INIT_WORK(&vi->config_work, virtnet_config_changed_work);
Rusty Russell296f96f2007-10-22 11:03:37 +10002938
Herbert Xu97402b92008-04-18 11:24:27 +08002939 /* If we can receive ANY GSO packets, we must allocate large ones. */
Joe Perches8e95a202009-12-03 07:58:21 +00002940 if (virtio_has_feature(vdev, VIRTIO_NET_F_GUEST_TSO4) ||
2941 virtio_has_feature(vdev, VIRTIO_NET_F_GUEST_TSO6) ||
Vlad Yaseviche3e3c422015-02-03 16:36:17 -05002942 virtio_has_feature(vdev, VIRTIO_NET_F_GUEST_ECN) ||
2943 virtio_has_feature(vdev, VIRTIO_NET_F_GUEST_UFO))
Herbert Xu97402b92008-04-18 11:24:27 +08002944 vi->big_packets = true;
2945
Mark McLoughlin3f2c31d2008-11-16 22:41:34 -08002946 if (virtio_has_feature(vdev, VIRTIO_NET_F_MRG_RXBUF))
2947 vi->mergeable_rx_bufs = true;
2948
Michael S. Tsirkind04302b2014-10-24 00:24:03 +03002949 if (virtio_has_feature(vdev, VIRTIO_NET_F_MRG_RXBUF) ||
2950 virtio_has_feature(vdev, VIRTIO_F_VERSION_1))
Michael S. Tsirkin012873d2014-10-24 16:55:57 +03002951 vi->hdr_len = sizeof(struct virtio_net_hdr_mrg_rxbuf);
2952 else
2953 vi->hdr_len = sizeof(struct virtio_net_hdr);
2954
Michael S. Tsirkin75993302015-07-15 15:26:19 +03002955 if (virtio_has_feature(vdev, VIRTIO_F_ANY_LAYOUT) ||
2956 virtio_has_feature(vdev, VIRTIO_F_VERSION_1))
Michael S. Tsirkine7428e92013-07-25 10:20:23 +09302957 vi->any_header_sg = true;
2958
Jason Wang986a4f42012-12-07 07:04:56 +00002959 if (virtio_has_feature(vdev, VIRTIO_NET_F_CTRL_VQ))
2960 vi->has_cvq = true;
2961
Aaron Conole14de9d12016-06-03 16:57:12 -04002962 if (virtio_has_feature(vdev, VIRTIO_NET_F_MTU)) {
2963 mtu = virtio_cread16(vdev,
2964 offsetof(struct virtio_net_config,
2965 mtu));
Aaron Conole93a205e2016-10-25 16:12:12 -04002966 if (mtu < dev->min_mtu) {
Michael S. Tsirkinfe36cbe2017-03-29 19:09:14 +03002967 /* Should never trigger: MTU was previously validated
2968 * in virtnet_validate.
2969 */
2970 dev_err(&vdev->dev, "device MTU appears to have changed "
2971 "it is now %d < %d", mtu, dev->min_mtu);
Toshiaki Makitad7dfc5c2018-01-17 15:38:25 +09002972 goto free;
Aaron Conole93a205e2016-10-25 16:12:12 -04002973 }
Michael S. Tsirkin2e123b42017-03-08 02:14:25 +02002974
Michael S. Tsirkinfe36cbe2017-03-29 19:09:14 +03002975 dev->mtu = mtu;
2976 dev->max_mtu = mtu;
2977
Michael S. Tsirkin2e123b42017-03-08 02:14:25 +02002978 /* TODO: size buffers correctly in this case. */
2979 if (dev->mtu > ETH_DATA_LEN)
2980 vi->big_packets = true;
Aaron Conole14de9d12016-06-03 16:57:12 -04002981 }
2982
Michael S. Tsirkin012873d2014-10-24 16:55:57 +03002983 if (vi->any_header_sg)
2984 dev->needed_headroom = vi->hdr_len;
Zhangjie \(HZ\)6ebbc1a2014-04-29 18:43:22 +08002985
Jason Wang44900012016-11-25 12:37:26 +08002986 /* Enable multiqueue by default */
2987 if (num_online_cpus() >= max_queue_pairs)
2988 vi->curr_queue_pairs = max_queue_pairs;
2989 else
2990 vi->curr_queue_pairs = num_online_cpus();
Jason Wang986a4f42012-12-07 07:04:56 +00002991 vi->max_queue_pairs = max_queue_pairs;
2992
2993 /* Allocate/initialize the rx/tx queues, and invoke find_vqs */
Amit Shah3f9c10b2011-12-22 16:58:31 +05302994 err = init_vqs(vi);
Michael S. Tsirkind2a7ddd2009-06-12 22:16:36 -06002995 if (err)
Toshiaki Makitad7dfc5c2018-01-17 15:38:25 +09002996 goto free;
Michael S. Tsirkind2a7ddd2009-06-12 22:16:36 -06002997
Michael Daltonfbf28d72014-01-16 22:23:30 -08002998#ifdef CONFIG_SYSFS
2999 if (vi->mergeable_rx_bufs)
3000 dev->sysfs_rx_queue_group = &virtio_net_mrg_rx_group;
3001#endif
Zhi Yong Wu0f13b662013-11-18 21:19:27 +08003002 netif_set_real_num_tx_queues(dev, vi->curr_queue_pairs);
3003 netif_set_real_num_rx_queues(dev, vi->curr_queue_pairs);
Jason Wang986a4f42012-12-07 07:04:56 +00003004
Nikolay Aleksandrov16032be2016-02-03 04:04:37 +01003005 virtnet_init_settings(dev);
3006
Sridhar Samudralaba5e4422018-05-24 09:55:17 -07003007 if (virtio_has_feature(vdev, VIRTIO_NET_F_STANDBY)) {
3008 vi->failover = net_failover_create(vi->dev);
Wei Yongjun4b8e6ac2018-05-31 02:05:07 +00003009 if (IS_ERR(vi->failover)) {
3010 err = PTR_ERR(vi->failover);
Sridhar Samudralaba5e4422018-05-24 09:55:17 -07003011 goto free_vqs;
Wei Yongjun4b8e6ac2018-05-31 02:05:07 +00003012 }
Sridhar Samudralaba5e4422018-05-24 09:55:17 -07003013 }
3014
Rusty Russell296f96f2007-10-22 11:03:37 +10003015 err = register_netdev(dev);
3016 if (err) {
3017 pr_debug("virtio_net: registering device failed\n");
Sridhar Samudralaba5e4422018-05-24 09:55:17 -07003018 goto free_failover;
Rusty Russell296f96f2007-10-22 11:03:37 +10003019 }
Rusty Russellb3369c12008-02-04 23:50:02 -05003020
Michael S. Tsirkin4baf1e32014-10-15 10:22:30 +10303021 virtio_device_ready(vdev);
3022
Sebastian Andrzej Siewior8017c272016-08-12 19:49:43 +02003023 err = virtnet_cpu_notif_add(vi);
Wanlong Gao8de4b2f2013-01-24 23:51:31 +00003024 if (err) {
3025 pr_debug("virtio_net: registering cpu notifier failed\n");
wangyunjianf00e35e2016-05-31 11:52:43 +08003026 goto free_unregister_netdev;
Wanlong Gao8de4b2f2013-01-24 23:51:31 +00003027 }
3028
Jason Wanga2208712016-12-13 14:23:05 +08003029 virtnet_set_queues(vi, vi->curr_queue_pairs);
Jason Wang44900012016-11-25 12:37:26 +08003030
Jason Wang167c25e2010-11-10 14:45:41 +00003031 /* Assume link up if device can't report link status,
3032 otherwise get link status from config. */
Jay Vosburghbda7fab2018-03-22 14:42:41 +00003033 netif_carrier_off(dev);
Jason Wang167c25e2010-11-10 14:45:41 +00003034 if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_STATUS)) {
Tejun Heo3b07e9c2012-08-20 14:51:24 -07003035 schedule_work(&vi->config_work);
Jason Wang167c25e2010-11-10 14:45:41 +00003036 } else {
3037 vi->status = VIRTIO_NET_S_LINK_UP;
Jason Baronfaa9b392018-01-05 17:44:54 -05003038 virtnet_update_settings(vi);
Jason Wang167c25e2010-11-10 14:45:41 +00003039 netif_carrier_on(dev);
3040 }
Mark McLoughlin9f4d26d2009-01-19 17:09:49 -08003041
Jason Wang3f935222017-07-19 16:54:49 +08003042 for (i = 0; i < ARRAY_SIZE(guest_offloads); i++)
3043 if (virtio_has_feature(vi->vdev, guest_offloads[i]))
3044 set_bit(guest_offloads[i], &vi->guest_offloads);
3045
Jason Wang986a4f42012-12-07 07:04:56 +00003046 pr_debug("virtnet: registered device %s with %d RX and TX vq's\n",
3047 dev->name, max_queue_pairs);
3048
Rusty Russell296f96f2007-10-22 11:03:37 +10003049 return 0;
3050
wangyunjianf00e35e2016-05-31 11:52:43 +08003051free_unregister_netdev:
Michael S. Tsirkin02465552014-10-15 10:22:31 +10303052 vi->vdev->config->reset(vdev);
3053
Rusty Russellb3369c12008-02-04 23:50:02 -05003054 unregister_netdev(dev);
Sridhar Samudralaba5e4422018-05-24 09:55:17 -07003055free_failover:
3056 net_failover_destroy(vi->failover);
Michael S. Tsirkind2a7ddd2009-06-12 22:16:36 -06003057free_vqs:
Jason Wang986a4f42012-12-07 07:04:56 +00003058 cancel_delayed_work_sync(&vi->refill);
Michael Daltonfb518792014-01-16 22:23:26 -08003059 free_receive_page_frags(vi);
Jason Wange9d74172012-12-07 07:04:55 +00003060 virtnet_del_vqs(vi);
Rusty Russell296f96f2007-10-22 11:03:37 +10003061free:
3062 free_netdev(dev);
3063 return err;
3064}
3065
Amit Shah04486ed2011-12-22 16:58:32 +05303066static void remove_vq_common(struct virtnet_info *vi)
Rusty Russell296f96f2007-10-22 11:03:37 +10003067{
Amit Shah04486ed2011-12-22 16:58:32 +05303068 vi->vdev->config->reset(vi->vdev);
Shirley Ma830a8a92010-02-08 14:14:42 +00003069
3070 /* Free unused buffers in both send and recv, if any. */
Shirley Ma9ab86bb2010-01-29 03:20:04 +00003071 free_unused_bufs(vi);
Rusty Russellfb6813f2008-07-25 12:06:01 -05003072
Jason Wang986a4f42012-12-07 07:04:56 +00003073 free_receive_bufs(vi);
Michael S. Tsirkind2a7ddd2009-06-12 22:16:36 -06003074
Michael Daltonfb518792014-01-16 22:23:26 -08003075 free_receive_page_frags(vi);
3076
Jason Wang986a4f42012-12-07 07:04:56 +00003077 virtnet_del_vqs(vi);
Amit Shah04486ed2011-12-22 16:58:32 +05303078}
3079
Bill Pemberton8cc085d2012-12-03 09:24:15 -05003080static void virtnet_remove(struct virtio_device *vdev)
Amit Shah04486ed2011-12-22 16:58:32 +05303081{
3082 struct virtnet_info *vi = vdev->priv;
3083
Sebastian Andrzej Siewior8017c272016-08-12 19:49:43 +02003084 virtnet_cpu_notif_remove(vi);
Wanlong Gao8de4b2f2013-01-24 23:51:31 +00003085
Michael S. Tsirkin102a2782014-10-15 10:22:29 +10303086 /* Make sure no work handler is accessing the device. */
3087 flush_work(&vi->config_work);
Jason Wang586d17c2012-04-11 20:43:52 +00003088
Amit Shah04486ed2011-12-22 16:58:32 +05303089 unregister_netdev(vi->dev);
3090
Sridhar Samudralaba5e4422018-05-24 09:55:17 -07003091 net_failover_destroy(vi->failover);
3092
Amit Shah04486ed2011-12-22 16:58:32 +05303093 remove_vq_common(vi);
Rusty Russellfb6813f2008-07-25 12:06:01 -05003094
Rusty Russell74b25532007-11-19 11:20:42 -05003095 free_netdev(vi->dev);
Rusty Russell296f96f2007-10-22 11:03:37 +10003096}
3097
Arnd Bergmann67a75192017-07-25 17:35:50 +02003098static __maybe_unused int virtnet_freeze(struct virtio_device *vdev)
Amit Shah0741bcb2011-12-22 16:58:33 +05303099{
3100 struct virtnet_info *vi = vdev->priv;
3101
Sebastian Andrzej Siewior8017c272016-08-12 19:49:43 +02003102 virtnet_cpu_notif_remove(vi);
John Fastabend9fe7bfc2017-02-02 19:16:01 -08003103 virtnet_freeze_down(vdev);
Amit Shah0741bcb2011-12-22 16:58:33 +05303104 remove_vq_common(vi);
3105
3106 return 0;
3107}
3108
Arnd Bergmann67a75192017-07-25 17:35:50 +02003109static __maybe_unused int virtnet_restore(struct virtio_device *vdev)
Amit Shah0741bcb2011-12-22 16:58:33 +05303110{
3111 struct virtnet_info *vi = vdev->priv;
John Fastabend9fe7bfc2017-02-02 19:16:01 -08003112 int err;
Amit Shah0741bcb2011-12-22 16:58:33 +05303113
John Fastabend9fe7bfc2017-02-02 19:16:01 -08003114 err = virtnet_restore_up(vdev);
Amit Shah0741bcb2011-12-22 16:58:33 +05303115 if (err)
3116 return err;
Jason Wang986a4f42012-12-07 07:04:56 +00003117 virtnet_set_queues(vi, vi->curr_queue_pairs);
3118
Sebastian Andrzej Siewior8017c272016-08-12 19:49:43 +02003119 err = virtnet_cpu_notif_add(vi);
Jason Wangec9debb2013-10-29 15:11:07 +08003120 if (err)
3121 return err;
3122
Amit Shah0741bcb2011-12-22 16:58:33 +05303123 return 0;
3124}
Amit Shah0741bcb2011-12-22 16:58:33 +05303125
Rusty Russell296f96f2007-10-22 11:03:37 +10003126static struct virtio_device_id id_table[] = {
3127 { VIRTIO_ID_NET, VIRTIO_DEV_ANY_ID },
3128 { 0 },
3129};
3130
Michael S. Tsirkinf3358502016-11-04 12:55:36 +02003131#define VIRTNET_FEATURES \
3132 VIRTIO_NET_F_CSUM, VIRTIO_NET_F_GUEST_CSUM, \
3133 VIRTIO_NET_F_MAC, \
3134 VIRTIO_NET_F_HOST_TSO4, VIRTIO_NET_F_HOST_UFO, VIRTIO_NET_F_HOST_TSO6, \
3135 VIRTIO_NET_F_HOST_ECN, VIRTIO_NET_F_GUEST_TSO4, VIRTIO_NET_F_GUEST_TSO6, \
3136 VIRTIO_NET_F_GUEST_ECN, VIRTIO_NET_F_GUEST_UFO, \
3137 VIRTIO_NET_F_MRG_RXBUF, VIRTIO_NET_F_STATUS, VIRTIO_NET_F_CTRL_VQ, \
3138 VIRTIO_NET_F_CTRL_RX, VIRTIO_NET_F_CTRL_VLAN, \
3139 VIRTIO_NET_F_GUEST_ANNOUNCE, VIRTIO_NET_F_MQ, \
3140 VIRTIO_NET_F_CTRL_MAC_ADDR, \
Jason Baronfaa9b392018-01-05 17:44:54 -05003141 VIRTIO_NET_F_MTU, VIRTIO_NET_F_CTRL_GUEST_OFFLOADS, \
Sridhar Samudrala98050692018-05-24 09:55:16 -07003142 VIRTIO_NET_F_SPEED_DUPLEX, VIRTIO_NET_F_STANDBY
Michael S. Tsirkinf3358502016-11-04 12:55:36 +02003143
Rusty Russellc45a6812008-05-02 21:50:50 -05003144static unsigned int features[] = {
Michael S. Tsirkinf3358502016-11-04 12:55:36 +02003145 VIRTNET_FEATURES,
3146};
3147
3148static unsigned int features_legacy[] = {
3149 VIRTNET_FEATURES,
3150 VIRTIO_NET_F_GSO,
Michael S. Tsirkine7428e92013-07-25 10:20:23 +09303151 VIRTIO_F_ANY_LAYOUT,
Rusty Russellc45a6812008-05-02 21:50:50 -05003152};
3153
Uwe Kleine-König22402522009-11-05 01:32:44 -08003154static struct virtio_driver virtio_net_driver = {
Rusty Russellc45a6812008-05-02 21:50:50 -05003155 .feature_table = features,
3156 .feature_table_size = ARRAY_SIZE(features),
Michael S. Tsirkinf3358502016-11-04 12:55:36 +02003157 .feature_table_legacy = features_legacy,
3158 .feature_table_size_legacy = ARRAY_SIZE(features_legacy),
Rusty Russell296f96f2007-10-22 11:03:37 +10003159 .driver.name = KBUILD_MODNAME,
3160 .driver.owner = THIS_MODULE,
3161 .id_table = id_table,
Michael S. Tsirkinfe36cbe2017-03-29 19:09:14 +03003162 .validate = virtnet_validate,
Rusty Russell296f96f2007-10-22 11:03:37 +10003163 .probe = virtnet_probe,
Bill Pemberton8cc085d2012-12-03 09:24:15 -05003164 .remove = virtnet_remove,
Mark McLoughlin9f4d26d2009-01-19 17:09:49 -08003165 .config_changed = virtnet_config_changed,
Aaron Lu89107002013-09-17 09:25:23 +09303166#ifdef CONFIG_PM_SLEEP
Amit Shah0741bcb2011-12-22 16:58:33 +05303167 .freeze = virtnet_freeze,
3168 .restore = virtnet_restore,
3169#endif
Rusty Russell296f96f2007-10-22 11:03:37 +10003170};
3171
Sebastian Andrzej Siewior8017c272016-08-12 19:49:43 +02003172static __init int virtio_net_driver_init(void)
3173{
3174 int ret;
3175
Thomas Gleixner73c1b412016-12-21 20:19:54 +01003176 ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN, "virtio/net:online",
Sebastian Andrzej Siewior8017c272016-08-12 19:49:43 +02003177 virtnet_cpu_online,
3178 virtnet_cpu_down_prep);
3179 if (ret < 0)
3180 goto out;
3181 virtionet_online = ret;
Thomas Gleixner73c1b412016-12-21 20:19:54 +01003182 ret = cpuhp_setup_state_multi(CPUHP_VIRT_NET_DEAD, "virtio/net:dead",
Sebastian Andrzej Siewior8017c272016-08-12 19:49:43 +02003183 NULL, virtnet_cpu_dead);
3184 if (ret)
3185 goto err_dead;
3186
3187 ret = register_virtio_driver(&virtio_net_driver);
3188 if (ret)
3189 goto err_virtio;
3190 return 0;
3191err_virtio:
3192 cpuhp_remove_multi_state(CPUHP_VIRT_NET_DEAD);
3193err_dead:
3194 cpuhp_remove_multi_state(virtionet_online);
3195out:
3196 return ret;
3197}
3198module_init(virtio_net_driver_init);
3199
3200static __exit void virtio_net_driver_exit(void)
3201{
Andrew Jonescfa0ebc2017-07-24 15:38:32 +02003202 unregister_virtio_driver(&virtio_net_driver);
Sebastian Andrzej Siewior8017c272016-08-12 19:49:43 +02003203 cpuhp_remove_multi_state(CPUHP_VIRT_NET_DEAD);
3204 cpuhp_remove_multi_state(virtionet_online);
Sebastian Andrzej Siewior8017c272016-08-12 19:49:43 +02003205}
3206module_exit(virtio_net_driver_exit);
Rusty Russell296f96f2007-10-22 11:03:37 +10003207
3208MODULE_DEVICE_TABLE(virtio, id_table);
3209MODULE_DESCRIPTION("Virtio network driver");
3210MODULE_LICENSE("GPL");