blob: 03ed492c4e14ade6cfee53b8cf9a65fd1ec62d16 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * INET An implementation of the TCP/IP protocol suite for the LINUX
3 * operating system. INET is implemented using the BSD Socket
4 * interface as the means of communication with the user level.
5 *
6 * Definitions for the Interfaces handler.
7 *
8 * Version: @(#)dev.h 1.0.10 08/12/93
9 *
Jesper Juhl02c30a82005-05-05 16:16:16 -070010 * Authors: Ross Biro
Linus Torvalds1da177e2005-04-16 15:20:36 -070011 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
12 * Corey Minyard <wf-rch!minyard@relay.EU.net>
13 * Donald J. Becker, <becker@cesdis.gsfc.nasa.gov>
Alan Cox113aa832008-10-13 19:01:08 -070014 * Alan Cox, <alan@lxorguk.ukuu.org.uk>
Linus Torvalds1da177e2005-04-16 15:20:36 -070015 * Bjorn Ekwall. <bj0rn@blox.se>
16 * Pekka Riikonen <priikone@poseidon.pspt.fi>
17 *
18 * This program is free software; you can redistribute it and/or
19 * modify it under the terms of the GNU General Public License
20 * as published by the Free Software Foundation; either version
21 * 2 of the License, or (at your option) any later version.
22 *
23 * Moved to /usr/include/linux for NET3
24 */
25#ifndef _LINUX_NETDEVICE_H
26#define _LINUX_NETDEVICE_H
27
Al Virod7fe0f22006-12-03 23:15:30 -050028#include <linux/timer.h>
Paul Gortmaker187f1882011-11-23 20:12:59 -050029#include <linux/bug.h>
Stephen Hemmingerbea33482007-10-03 16:41:36 -070030#include <linux/delay.h>
Arun Sharma600634972011-07-26 16:09:06 -070031#include <linux/atomic.h>
Eric Dumazet53511452014-10-08 08:19:27 -070032#include <linux/prefetch.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070033#include <asm/cache.h>
34#include <asm/byteorder.h>
35
Linus Torvalds1da177e2005-04-16 15:20:36 -070036#include <linux/percpu.h>
David S. Miller4d5b78c2009-05-06 16:52:51 -070037#include <linux/rculist.h>
Stephen Hemmingerbea33482007-10-03 16:41:36 -070038#include <linux/workqueue.h>
Tom Herbert114cf582011-11-28 16:33:09 +000039#include <linux/dynamic_queue_limits.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070040
Patrick McHardyb1b67dd2009-04-20 04:49:28 +000041#include <linux/ethtool.h>
Daniel Lezcanoa050c332007-09-12 14:57:09 +020042#include <net/net_namespace.h>
Jeff Kirsher7a6b6f52008-11-25 01:02:08 -080043#ifdef CONFIG_DCB
Alexander Duyck2f90b862008-11-20 20:52:10 -080044#include <net/dcbnl.h>
45#endif
Neil Horman5bc14212011-11-22 05:10:51 +000046#include <net/netprio_cgroup.h>
Jesper Dangaard Brouere817f852018-01-03 11:26:09 +010047#include <net/xdp.h>
Daniel Lezcanoa050c332007-09-12 14:57:09 +020048
Michał Mirosława59e2ec2011-11-15 15:29:55 +000049#include <linux/netdev_features.h>
John Fastabend77162022012-04-15 06:43:56 +000050#include <linux/neighbour.h>
David Howells607ca462012-10-13 10:46:48 +010051#include <uapi/linux/netdevice.h>
Moni Shoua61bd3852015-02-03 16:48:29 +020052#include <uapi/linux/if_bonding.h>
John Fastabende4c67342016-02-16 21:16:15 -080053#include <uapi/linux/pkt_cls.h>
Jiri Kosina59cc1f62016-08-10 11:05:15 +020054#include <linux/hashtable.h>
Michał Mirosława59e2ec2011-11-15 15:29:55 +000055
Jeff Moyer115c1d62005-06-22 22:05:31 -070056struct netpoll_info;
Paul Gortmaker313162d2012-01-30 11:46:54 -050057struct device;
Richard Cochranc1f19b52010-07-17 08:49:36 +000058struct phy_device;
Vivien Didelot2f657a62017-09-29 17:19:20 -040059struct dsa_port;
Andrew Lunnc6e970a2017-03-28 23:45:06 +020060
Russell Kinge679c9c2018-03-28 15:44:16 -070061struct sfp_bus;
Johannes Berg704232c2007-04-23 12:20:05 -070062/* 802.11 specific */
63struct wireless_dev;
Alexander Aring98a18b62014-11-02 06:44:54 +010064/* 802.15.4 specific */
65struct wpan_dev;
Robert Shearman03c57742015-04-22 11:14:37 +010066struct mpls_dev;
Alexander Duyck7c46a642016-06-16 12:21:00 -070067/* UDP Tunnel offloads */
68struct udp_tunnel_info;
Brenden Blancoa7862b42016-07-19 12:16:48 -070069struct bpf_prog;
John Fastabend814abfa2017-07-17 09:27:07 -070070struct xdp_buff;
Linus Torvalds1da177e2005-04-16 15:20:36 -070071
Joe Perchesf629d202013-09-26 14:48:15 -070072void netdev_set_default_ethtool_ops(struct net_device *dev,
73 const struct ethtool_ops *ops);
Stanislaw Gruszkad07d7502013-01-10 23:19:10 +000074
Jarek Poplawski9a1654b2009-11-15 07:20:12 +000075/* Backlog congestion levels */
76#define NET_RX_SUCCESS 0 /* keep 'em coming, baby */
77#define NET_RX_DROP 1 /* packet dropped */
78
Patrick McHardy572a9d72009-11-10 06:14:14 +000079/*
80 * Transmit return codes: transmit return codes originate from three different
81 * namespaces:
82 *
83 * - qdisc return codes
84 * - driver transmit return codes
85 * - errno values
86 *
87 * Drivers are allowed to return any one of those in their hard_start_xmit()
88 * function. Real network devices commonly used with qdiscs should only return
89 * the driver transmit return codes though - when qdiscs are used, the actual
90 * transmission happens asynchronously, so the value is not propagated to
Bjorn Helgaas5e82b4b2016-03-23 13:47:23 -050091 * higher layers. Virtual network devices transmit synchronously; in this case
92 * the driver transmit return codes are consumed by dev_queue_xmit(), and all
Patrick McHardy572a9d72009-11-10 06:14:14 +000093 * others are propagated to higher layers.
94 */
95
96/* qdisc ->enqueue() return codes. */
97#define NET_XMIT_SUCCESS 0x00
Jarek Poplawski9a1654b2009-11-15 07:20:12 +000098#define NET_XMIT_DROP 0x01 /* skb dropped */
99#define NET_XMIT_CN 0x02 /* congestion notification */
Jarek Poplawski9a1654b2009-11-15 07:20:12 +0000100#define NET_XMIT_MASK 0x0f /* qdisc flags in net/sch_generic.h */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700101
Gerrit Renkerb9df3cb2006-11-14 11:21:36 -0200102/* NET_XMIT_CN is special. It does not guarantee that this packet is lost. It
103 * indicates that the device will soon be dropping packets, or already drops
104 * some packets of the same priority; prompting us to send less aggressively. */
Patrick McHardy572a9d72009-11-10 06:14:14 +0000105#define net_xmit_eval(e) ((e) == NET_XMIT_CN ? 0 : (e))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700106#define net_xmit_errno(e) ((e) != NET_XMIT_CN ? -ENOBUFS : 0)
107
Stephen Hemmingerdc1f8bf2009-08-31 19:50:40 +0000108/* Driver transmit return codes */
Jarek Poplawski9a1654b2009-11-15 07:20:12 +0000109#define NETDEV_TX_MASK 0xf0
Patrick McHardy572a9d72009-11-10 06:14:14 +0000110
Stephen Hemmingerdc1f8bf2009-08-31 19:50:40 +0000111enum netdev_tx {
Patrick McHardy572a9d72009-11-10 06:14:14 +0000112 __NETDEV_TX_MIN = INT_MIN, /* make sure enum is signed */
Jarek Poplawski9a1654b2009-11-15 07:20:12 +0000113 NETDEV_TX_OK = 0x00, /* driver took care of packet */
114 NETDEV_TX_BUSY = 0x10, /* driver tx path was busy*/
Stephen Hemmingerdc1f8bf2009-08-31 19:50:40 +0000115};
116typedef enum netdev_tx netdev_tx_t;
117
Jarek Poplawski9a1654b2009-11-15 07:20:12 +0000118/*
119 * Current order: NETDEV_TX_MASK > NET_XMIT_MASK >= 0 is significant;
120 * hard_start_xmit() return < NET_XMIT_MASK means skb was consumed.
121 */
122static inline bool dev_xmit_complete(int rc)
123{
124 /*
125 * Positive cases with an skb consumed by a driver:
126 * - successful transmission (rc == NETDEV_TX_OK)
127 * - error while transmitting (rc < 0)
128 * - error while queueing to a different device (rc & NET_XMIT_MASK)
129 */
130 if (likely(rc < NET_XMIT_MASK))
131 return true;
132
133 return false;
134}
135
Linus Torvalds1da177e2005-04-16 15:20:36 -0700136/*
Bjorn Helgaas5e82b4b2016-03-23 13:47:23 -0500137 * Compute the worst-case header length according to the protocols
Linus Torvalds1da177e2005-04-16 15:20:36 -0700138 * used.
139 */
Graf Yangfe2918b2009-02-05 21:26:19 -0800140
KY Srinivasanc0eb4542015-12-01 16:43:10 -0800141#if defined(CONFIG_HYPERV_NET)
142# define LL_MAX_HEADER 128
143#elif defined(CONFIG_WLAN) || IS_ENABLED(CONFIG_AX25)
David S. Miller8388e3d2008-05-12 20:17:33 -0700144# if defined(CONFIG_MAC80211_MESH)
145# define LL_MAX_HEADER 128
146# else
147# define LL_MAX_HEADER 96
148# endif
Linus Torvalds1da177e2005-04-16 15:20:36 -0700149#else
David S. Miller8388e3d2008-05-12 20:17:33 -0700150# define LL_MAX_HEADER 32
Linus Torvalds1da177e2005-04-16 15:20:36 -0700151#endif
152
Ben Hutchingsd11ead72011-11-25 14:40:26 +0000153#if !IS_ENABLED(CONFIG_NET_IPIP) && !IS_ENABLED(CONFIG_NET_IPGRE) && \
154 !IS_ENABLED(CONFIG_IPV6_SIT) && !IS_ENABLED(CONFIG_IPV6_TUNNEL)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700155#define MAX_HEADER LL_MAX_HEADER
156#else
157#define MAX_HEADER (LL_MAX_HEADER + 48)
158#endif
159
160/*
Ben Hutchingsbe1f3c22010-06-08 07:19:54 +0000161 * Old network device statistics. Fields are native words
162 * (unsigned long) so they can be read and written atomically.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700163 */
Graf Yangfe2918b2009-02-05 21:26:19 -0800164
Eric Dumazetd94d9fe2009-11-04 09:50:58 -0800165struct net_device_stats {
Ben Hutchings3cfde792010-07-09 09:11:52 +0000166 unsigned long rx_packets;
167 unsigned long tx_packets;
168 unsigned long rx_bytes;
169 unsigned long tx_bytes;
170 unsigned long rx_errors;
171 unsigned long tx_errors;
172 unsigned long rx_dropped;
173 unsigned long tx_dropped;
174 unsigned long multicast;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700175 unsigned long collisions;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700176 unsigned long rx_length_errors;
Ben Hutchings3cfde792010-07-09 09:11:52 +0000177 unsigned long rx_over_errors;
178 unsigned long rx_crc_errors;
179 unsigned long rx_frame_errors;
180 unsigned long rx_fifo_errors;
181 unsigned long rx_missed_errors;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700182 unsigned long tx_aborted_errors;
183 unsigned long tx_carrier_errors;
184 unsigned long tx_fifo_errors;
185 unsigned long tx_heartbeat_errors;
186 unsigned long tx_window_errors;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700187 unsigned long rx_compressed;
188 unsigned long tx_compressed;
189};
190
Linus Torvalds1da177e2005-04-16 15:20:36 -0700191
192#include <linux/cache.h>
193#include <linux/skbuff.h>
194
Eric Dumazetadc93002011-11-17 03:13:26 +0000195#ifdef CONFIG_RPS
Ingo Molnarc5905af2012-02-24 08:31:31 +0100196#include <linux/static_key.h>
197extern struct static_key rps_needed;
Eric Dumazet13bfff22016-12-07 08:29:10 -0800198extern struct static_key rfs_needed;
Eric Dumazetadc93002011-11-17 03:13:26 +0000199#endif
200
Linus Torvalds1da177e2005-04-16 15:20:36 -0700201struct neighbour;
202struct neigh_parms;
203struct sk_buff;
204
Jiri Pirkof001fde2009-05-05 02:48:28 +0000205struct netdev_hw_addr {
206 struct list_head list;
207 unsigned char addr[MAX_ADDR_LEN];
208 unsigned char type;
Jiri Pirkoccffad252009-05-22 23:22:17 +0000209#define NETDEV_HW_ADDR_T_LAN 1
210#define NETDEV_HW_ADDR_T_SAN 2
211#define NETDEV_HW_ADDR_T_SLAVE 3
212#define NETDEV_HW_ADDR_T_UNICAST 4
Jiri Pirko22bedad32010-04-01 21:22:57 +0000213#define NETDEV_HW_ADDR_T_MULTICAST 5
Jiri Pirko22bedad32010-04-01 21:22:57 +0000214 bool global_use;
Vlad Yasevich4cd729b02013-04-15 09:54:25 +0000215 int sync_cnt;
Eric Dumazet8f8f1032010-09-19 11:24:02 -0700216 int refcount;
Vlad Yasevich4543fbe2013-04-02 17:10:07 -0400217 int synced;
Jiri Pirkof001fde2009-05-05 02:48:28 +0000218 struct rcu_head rcu_head;
219};
220
Jiri Pirko31278e72009-06-17 01:12:19 +0000221struct netdev_hw_addr_list {
222 struct list_head list;
223 int count;
224};
225
Jiri Pirko22bedad32010-04-01 21:22:57 +0000226#define netdev_hw_addr_list_count(l) ((l)->count)
227#define netdev_hw_addr_list_empty(l) (netdev_hw_addr_list_count(l) == 0)
228#define netdev_hw_addr_list_for_each(ha, l) \
229 list_for_each_entry(ha, &(l)->list, list)
230
231#define netdev_uc_count(dev) netdev_hw_addr_list_count(&(dev)->uc)
232#define netdev_uc_empty(dev) netdev_hw_addr_list_empty(&(dev)->uc)
Jiri Pirko32e7bfc2010-01-25 13:36:10 -0800233#define netdev_for_each_uc_addr(ha, dev) \
Jiri Pirko22bedad32010-04-01 21:22:57 +0000234 netdev_hw_addr_list_for_each(ha, &(dev)->uc)
Jiri Pirko32e7bfc2010-01-25 13:36:10 -0800235
Jiri Pirko22bedad32010-04-01 21:22:57 +0000236#define netdev_mc_count(dev) netdev_hw_addr_list_count(&(dev)->mc)
237#define netdev_mc_empty(dev) netdev_hw_addr_list_empty(&(dev)->mc)
Pavel Roskin18e225f2010-04-07 16:40:09 -0700238#define netdev_for_each_mc_addr(ha, dev) \
Jiri Pirko22bedad32010-04-01 21:22:57 +0000239 netdev_hw_addr_list_for_each(ha, &(dev)->mc)
Jiri Pirko6683ece2010-02-04 10:22:25 -0800240
Eric Dumazetd94d9fe2009-11-04 09:50:58 -0800241struct hh_cache {
Alexey Dobriyan5b3dc2f2017-04-10 11:11:17 +0300242 unsigned int hh_len;
Stephen Hemminger3644f0c2006-12-07 15:08:17 -0800243 seqlock_t hh_lock;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700244
245 /* cached hardware header; allow for machine alignment needs. */
246#define HH_DATA_MOD 16
247#define HH_DATA_OFF(__len) \
Jiri Benc5ba0eac2005-06-02 16:48:05 -0700248 (HH_DATA_MOD - (((__len - 1) & (HH_DATA_MOD - 1)) + 1))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700249#define HH_DATA_ALIGN(__len) \
250 (((__len)+(HH_DATA_MOD-1))&~(HH_DATA_MOD - 1))
251 unsigned long hh_data[HH_DATA_ALIGN(LL_MAX_HEADER) / sizeof(long)];
252};
253
Bjorn Helgaas5e82b4b2016-03-23 13:47:23 -0500254/* Reserve HH_DATA_MOD byte-aligned hard_header_len, but at least that much.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700255 * Alternative is:
256 * dev->hard_header_len ? (dev->hard_header_len +
257 * (HH_DATA_MOD - 1)) & ~(HH_DATA_MOD - 1) : 0
258 *
259 * We could use other alignment values, but we must maintain the
260 * relationship HH alignment <= LL alignment.
261 */
262#define LL_RESERVED_SPACE(dev) \
Johannes Bergf5184d22008-05-12 20:48:31 -0700263 ((((dev)->hard_header_len+(dev)->needed_headroom)&~(HH_DATA_MOD - 1)) + HH_DATA_MOD)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700264#define LL_RESERVED_SPACE_EXTRA(dev,extra) \
Johannes Bergf5184d22008-05-12 20:48:31 -0700265 ((((dev)->hard_header_len+(dev)->needed_headroom+(extra))&~(HH_DATA_MOD - 1)) + HH_DATA_MOD)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700266
Stephen Hemminger3b04ddd2007-10-09 01:40:57 -0700267struct header_ops {
268 int (*create) (struct sk_buff *skb, struct net_device *dev,
269 unsigned short type, const void *daddr,
Eric Dumazet95c96172012-04-15 05:58:06 +0000270 const void *saddr, unsigned int len);
Stephen Hemminger3b04ddd2007-10-09 01:40:57 -0700271 int (*parse)(const struct sk_buff *skb, unsigned char *haddr);
David S. Millere69dd332011-07-12 23:28:12 -0700272 int (*cache)(const struct neighbour *neigh, struct hh_cache *hh, __be16 type);
Stephen Hemminger3b04ddd2007-10-09 01:40:57 -0700273 void (*cache_update)(struct hh_cache *hh,
274 const struct net_device *dev,
275 const unsigned char *haddr);
Willem de Bruijn2793a232016-03-09 21:58:32 -0500276 bool (*validate)(const char *ll_header, unsigned int len);
Stephen Hemminger3b04ddd2007-10-09 01:40:57 -0700277};
278
Linus Torvalds1da177e2005-04-16 15:20:36 -0700279/* These flag bits are private to the generic network queueing
Bjorn Helgaas5e82b4b2016-03-23 13:47:23 -0500280 * layer; they may not be explicitly referenced by any other
Linus Torvalds1da177e2005-04-16 15:20:36 -0700281 * code.
282 */
283
Eric Dumazetd94d9fe2009-11-04 09:50:58 -0800284enum netdev_state_t {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700285 __LINK_STATE_START,
286 __LINK_STATE_PRESENT,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700287 __LINK_STATE_NOCARRIER,
Stefan Rompfb00055a2006-03-20 17:09:11 -0800288 __LINK_STATE_LINKWATCH_PENDING,
289 __LINK_STATE_DORMANT,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700290};
291
292
293/*
Bjorn Helgaas5e82b4b2016-03-23 13:47:23 -0500294 * This structure holds boot-time configured netdevice settings. They
Graf Yangfe2918b2009-02-05 21:26:19 -0800295 * are then used in the device probing.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700296 */
297struct netdev_boot_setup {
298 char name[IFNAMSIZ];
299 struct ifmap map;
300};
301#define NETDEV_BOOT_SETUP_MAX 8
302
Joe Perchesf629d202013-09-26 14:48:15 -0700303int __init netdev_boot_setup(char *str);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700304
305/*
Stephen Hemmingerbea33482007-10-03 16:41:36 -0700306 * Structure for NAPI scheduling similar to tasklet but with weighting
307 */
308struct napi_struct {
309 /* The poll_list must only be managed by the entity which
310 * changes the state of the NAPI_STATE_SCHED bit. This means
311 * whoever atomically sets that bit can add this napi_struct
Bjorn Helgaas5e82b4b2016-03-23 13:47:23 -0500312 * to the per-CPU poll_list, and whoever clears that bit
Stephen Hemmingerbea33482007-10-03 16:41:36 -0700313 * can remove from the list right before clearing the bit.
314 */
315 struct list_head poll_list;
316
317 unsigned long state;
318 int weight;
Eric Dumazet404f7c92012-09-26 07:07:47 +0000319 unsigned int gro_count;
Stephen Hemmingerbea33482007-10-03 16:41:36 -0700320 int (*poll)(struct napi_struct *, int);
321#ifdef CONFIG_NETPOLL
Stephen Hemmingerbea33482007-10-03 16:41:36 -0700322 int poll_owner;
Stephen Hemmingerbea33482007-10-03 16:41:36 -0700323#endif
Herbert Xu5d38a072009-01-04 16:13:40 -0800324 struct net_device *dev;
Herbert Xud565b0a2008-12-15 23:38:52 -0800325 struct sk_buff *gro_list;
Herbert Xu5d38a072009-01-04 16:13:40 -0800326 struct sk_buff *skb;
Eric Dumazet3b47d302014-11-06 21:09:44 -0800327 struct hrtimer timer;
Eric Dumazet404f7c92012-09-26 07:07:47 +0000328 struct list_head dev_list;
Eliezer Tamiraf12fa62013-06-10 11:39:41 +0300329 struct hlist_node napi_hash_node;
330 unsigned int napi_id;
Stephen Hemmingerbea33482007-10-03 16:41:36 -0700331};
332
Eric Dumazetd94d9fe2009-11-04 09:50:58 -0800333enum {
Stephen Hemmingerbea33482007-10-03 16:41:36 -0700334 NAPI_STATE_SCHED, /* Poll is scheduled */
Eric Dumazet39e6c822017-02-28 10:34:50 -0800335 NAPI_STATE_MISSED, /* reschedule a napi */
David S. Millera0a46192008-01-07 20:35:07 -0800336 NAPI_STATE_DISABLE, /* Disable pending */
Neil Horman7b363e42008-12-09 23:22:26 -0800337 NAPI_STATE_NPSVC, /* Netpoll - don't dequeue from poll_list */
Eric Dumazetd64b5e82015-11-18 06:31:00 -0800338 NAPI_STATE_HASHED, /* In NAPI hash (busy polling possible) */
339 NAPI_STATE_NO_BUSY_POLL,/* Do not add in napi_hash, no busy polling */
Eric Dumazet217f6972016-11-15 10:15:11 -0800340 NAPI_STATE_IN_BUSY_POLL,/* sk_busy_loop() owns this NAPI */
341};
342
343enum {
Eric Dumazet39e6c822017-02-28 10:34:50 -0800344 NAPIF_STATE_SCHED = BIT(NAPI_STATE_SCHED),
345 NAPIF_STATE_MISSED = BIT(NAPI_STATE_MISSED),
346 NAPIF_STATE_DISABLE = BIT(NAPI_STATE_DISABLE),
347 NAPIF_STATE_NPSVC = BIT(NAPI_STATE_NPSVC),
348 NAPIF_STATE_HASHED = BIT(NAPI_STATE_HASHED),
349 NAPIF_STATE_NO_BUSY_POLL = BIT(NAPI_STATE_NO_BUSY_POLL),
350 NAPIF_STATE_IN_BUSY_POLL = BIT(NAPI_STATE_IN_BUSY_POLL),
Stephen Hemmingerbea33482007-10-03 16:41:36 -0700351};
352
Ben Hutchings5b252f02009-10-29 07:17:09 +0000353enum gro_result {
Herbert Xud1c76af2009-03-16 10:50:02 -0700354 GRO_MERGED,
355 GRO_MERGED_FREE,
356 GRO_HELD,
357 GRO_NORMAL,
358 GRO_DROP,
Steffen Klassert25393d32017-02-15 09:39:44 +0100359 GRO_CONSUMED,
Herbert Xud1c76af2009-03-16 10:50:02 -0700360};
Ben Hutchings5b252f02009-10-29 07:17:09 +0000361typedef enum gro_result gro_result_t;
Herbert Xud1c76af2009-03-16 10:50:02 -0700362
Jiri Pirko8a4eb572011-03-12 03:14:39 +0000363/*
364 * enum rx_handler_result - Possible return values for rx_handlers.
365 * @RX_HANDLER_CONSUMED: skb was consumed by rx_handler, do not process it
366 * further.
367 * @RX_HANDLER_ANOTHER: Do another round in receive path. This is indicated in
368 * case skb->dev was changed by rx_handler.
369 * @RX_HANDLER_EXACT: Force exact delivery, no wildcard.
Bjorn Helgaas5e82b4b2016-03-23 13:47:23 -0500370 * @RX_HANDLER_PASS: Do nothing, pass the skb as if no rx_handler was called.
Jiri Pirko8a4eb572011-03-12 03:14:39 +0000371 *
372 * rx_handlers are functions called from inside __netif_receive_skb(), to do
373 * special processing of the skb, prior to delivery to protocol handlers.
374 *
375 * Currently, a net_device can only have a single rx_handler registered. Trying
376 * to register a second rx_handler will return -EBUSY.
377 *
378 * To register a rx_handler on a net_device, use netdev_rx_handler_register().
379 * To unregister a rx_handler on a net_device, use
380 * netdev_rx_handler_unregister().
381 *
382 * Upon return, rx_handler is expected to tell __netif_receive_skb() what to
383 * do with the skb.
384 *
Bjorn Helgaas5e82b4b2016-03-23 13:47:23 -0500385 * If the rx_handler consumed the skb in some way, it should return
Jiri Pirko8a4eb572011-03-12 03:14:39 +0000386 * RX_HANDLER_CONSUMED. This is appropriate when the rx_handler arranged for
Bjorn Helgaas5e82b4b2016-03-23 13:47:23 -0500387 * the skb to be delivered in some other way.
Jiri Pirko8a4eb572011-03-12 03:14:39 +0000388 *
389 * If the rx_handler changed skb->dev, to divert the skb to another
390 * net_device, it should return RX_HANDLER_ANOTHER. The rx_handler for the
391 * new device will be called if it exists.
392 *
Bjorn Helgaas5e82b4b2016-03-23 13:47:23 -0500393 * If the rx_handler decides the skb should be ignored, it should return
Jiri Pirko8a4eb572011-03-12 03:14:39 +0000394 * RX_HANDLER_EXACT. The skb will only be delivered to protocol handlers that
Adam Buchbinderd93cf062012-09-19 21:47:58 -0400395 * are registered on exact device (ptype->dev == skb->dev).
Jiri Pirko8a4eb572011-03-12 03:14:39 +0000396 *
Bjorn Helgaas5e82b4b2016-03-23 13:47:23 -0500397 * If the rx_handler didn't change skb->dev, but wants the skb to be normally
Jiri Pirko8a4eb572011-03-12 03:14:39 +0000398 * delivered, it should return RX_HANDLER_PASS.
399 *
400 * A device without a registered rx_handler will behave as if rx_handler
401 * returned RX_HANDLER_PASS.
402 */
403
404enum rx_handler_result {
405 RX_HANDLER_CONSUMED,
406 RX_HANDLER_ANOTHER,
407 RX_HANDLER_EXACT,
408 RX_HANDLER_PASS,
409};
410typedef enum rx_handler_result rx_handler_result_t;
411typedef rx_handler_result_t rx_handler_func_t(struct sk_buff **pskb);
Jiri Pirkoab95bfe2010-06-01 21:52:08 +0000412
Joe Perchesf629d202013-09-26 14:48:15 -0700413void __napi_schedule(struct napi_struct *n);
Eric Dumazetbc9ad162014-10-28 18:05:13 -0700414void __napi_schedule_irqoff(struct napi_struct *n);
Stephen Hemmingerbea33482007-10-03 16:41:36 -0700415
David S. Miller4d295152012-03-07 21:02:35 -0500416static inline bool napi_disable_pending(struct napi_struct *n)
David S. Millera0a46192008-01-07 20:35:07 -0800417{
418 return test_bit(NAPI_STATE_DISABLE, &n->state);
419}
420
Eric Dumazet39e6c822017-02-28 10:34:50 -0800421bool napi_schedule_prep(struct napi_struct *n);
Stephen Hemmingerbea33482007-10-03 16:41:36 -0700422
423/**
424 * napi_schedule - schedule NAPI poll
Bjorn Helgaas5e82b4b2016-03-23 13:47:23 -0500425 * @n: NAPI context
Stephen Hemmingerbea33482007-10-03 16:41:36 -0700426 *
427 * Schedule NAPI poll routine to be called if it is not already
428 * running.
429 */
430static inline void napi_schedule(struct napi_struct *n)
431{
432 if (napi_schedule_prep(n))
433 __napi_schedule(n);
434}
435
Eric Dumazetbc9ad162014-10-28 18:05:13 -0700436/**
437 * napi_schedule_irqoff - schedule NAPI poll
Bjorn Helgaas5e82b4b2016-03-23 13:47:23 -0500438 * @n: NAPI context
Eric Dumazetbc9ad162014-10-28 18:05:13 -0700439 *
440 * Variant of napi_schedule(), assuming hard irqs are masked.
441 */
442static inline void napi_schedule_irqoff(struct napi_struct *n)
443{
444 if (napi_schedule_prep(n))
445 __napi_schedule_irqoff(n);
446}
447
Roland Dreierbfe13f52007-10-09 15:47:37 -0700448/* Try to reschedule poll. Called by dev->poll() after napi_complete(). */
David S. Miller4d295152012-03-07 21:02:35 -0500449static inline bool napi_reschedule(struct napi_struct *napi)
Roland Dreierbfe13f52007-10-09 15:47:37 -0700450{
451 if (napi_schedule_prep(napi)) {
452 __napi_schedule(napi);
David S. Miller4d295152012-03-07 21:02:35 -0500453 return true;
Roland Dreierbfe13f52007-10-09 15:47:37 -0700454 }
David S. Miller4d295152012-03-07 21:02:35 -0500455 return false;
Roland Dreierbfe13f52007-10-09 15:47:37 -0700456}
457
Eric Dumazet364b6052016-11-15 10:15:13 -0800458bool napi_complete_done(struct napi_struct *n, int work_done);
Stephen Hemmingerbea33482007-10-03 16:41:36 -0700459/**
460 * napi_complete - NAPI processing complete
Bjorn Helgaas5e82b4b2016-03-23 13:47:23 -0500461 * @n: NAPI context
Stephen Hemmingerbea33482007-10-03 16:41:36 -0700462 *
463 * Mark NAPI processing as complete.
Eric Dumazet3b47d302014-11-06 21:09:44 -0800464 * Consider using napi_complete_done() instead.
Eric Dumazet364b6052016-11-15 10:15:13 -0800465 * Return false if device should avoid rearming interrupts.
Stephen Hemmingerbea33482007-10-03 16:41:36 -0700466 */
Eric Dumazet364b6052016-11-15 10:15:13 -0800467static inline bool napi_complete(struct napi_struct *n)
Eric Dumazet3b47d302014-11-06 21:09:44 -0800468{
469 return napi_complete_done(n, 0);
470}
Stephen Hemmingerbea33482007-10-03 16:41:36 -0700471
472/**
Eliezer Tamiraf12fa62013-06-10 11:39:41 +0300473 * napi_hash_del - remove a NAPI from global table
Bjorn Helgaas5e82b4b2016-03-23 13:47:23 -0500474 * @napi: NAPI context
Eliezer Tamiraf12fa62013-06-10 11:39:41 +0300475 *
Bjorn Helgaas5e82b4b2016-03-23 13:47:23 -0500476 * Warning: caller must observe RCU grace period
Eric Dumazet34cbe272015-11-18 06:31:02 -0800477 * before freeing memory containing @napi, if
478 * this function returns true.
Eric Dumazet93d05d42015-11-18 06:31:03 -0800479 * Note: core networking stack automatically calls it
Bjorn Helgaas5e82b4b2016-03-23 13:47:23 -0500480 * from netif_napi_del().
Eric Dumazet93d05d42015-11-18 06:31:03 -0800481 * Drivers might want to call this helper to combine all
Bjorn Helgaas5e82b4b2016-03-23 13:47:23 -0500482 * the needed RCU grace periods into a single one.
Eliezer Tamiraf12fa62013-06-10 11:39:41 +0300483 */
Eric Dumazet34cbe272015-11-18 06:31:02 -0800484bool napi_hash_del(struct napi_struct *napi);
Eliezer Tamiraf12fa62013-06-10 11:39:41 +0300485
486/**
Stephen Hemmingerbea33482007-10-03 16:41:36 -0700487 * napi_disable - prevent NAPI from scheduling
Bjorn Helgaas5e82b4b2016-03-23 13:47:23 -0500488 * @n: NAPI context
Stephen Hemmingerbea33482007-10-03 16:41:36 -0700489 *
490 * Stop NAPI from being scheduled on this context.
491 * Waits till any outstanding processing completes.
492 */
Eric Dumazet3b47d302014-11-06 21:09:44 -0800493void napi_disable(struct napi_struct *n);
Stephen Hemmingerbea33482007-10-03 16:41:36 -0700494
495/**
496 * napi_enable - enable NAPI scheduling
Bjorn Helgaas5e82b4b2016-03-23 13:47:23 -0500497 * @n: NAPI context
Stephen Hemmingerbea33482007-10-03 16:41:36 -0700498 *
499 * Resume NAPI from being scheduled on this context.
500 * Must be paired with napi_disable.
501 */
502static inline void napi_enable(struct napi_struct *n)
503{
504 BUG_ON(!test_bit(NAPI_STATE_SCHED, &n->state));
Peter Zijlstra4e857c52014-03-17 18:06:10 +0100505 smp_mb__before_atomic();
Stephen Hemmingerbea33482007-10-03 16:41:36 -0700506 clear_bit(NAPI_STATE_SCHED, &n->state);
Neil Horman2d8bff1262015-09-23 14:57:58 -0400507 clear_bit(NAPI_STATE_NPSVC, &n->state);
Stephen Hemmingerbea33482007-10-03 16:41:36 -0700508}
509
Stephen Hemmingerc264c3d2007-10-17 13:26:41 -0700510/**
511 * napi_synchronize - wait until NAPI is not running
Bjorn Helgaas5e82b4b2016-03-23 13:47:23 -0500512 * @n: NAPI context
Stephen Hemmingerc264c3d2007-10-17 13:26:41 -0700513 *
514 * Wait until NAPI is done being scheduled on this context.
515 * Waits till any outstanding processing completes but
516 * does not disable future activations.
517 */
518static inline void napi_synchronize(const struct napi_struct *n)
519{
Arnd Bergmannfacc4322016-01-22 11:43:44 +0100520 if (IS_ENABLED(CONFIG_SMP))
521 while (test_bit(NAPI_STATE_SCHED, &n->state))
522 msleep(1);
523 else
524 barrier();
Stephen Hemmingerc264c3d2007-10-17 13:26:41 -0700525}
Stephen Hemmingerc264c3d2007-10-17 13:26:41 -0700526
Eric Dumazetd94d9fe2009-11-04 09:50:58 -0800527enum netdev_queue_state_t {
Tom Herbert734664982011-11-28 16:32:44 +0000528 __QUEUE_STATE_DRV_XOFF,
529 __QUEUE_STATE_STACK_XOFF,
David S. Millerc3f26a22008-07-31 16:58:50 -0700530 __QUEUE_STATE_FROZEN,
David S. Miller79d16382008-07-08 23:14:46 -0700531};
Daniel Borkmann8e2f1a62014-04-02 20:52:57 +0200532
533#define QUEUE_STATE_DRV_XOFF (1 << __QUEUE_STATE_DRV_XOFF)
534#define QUEUE_STATE_STACK_XOFF (1 << __QUEUE_STATE_STACK_XOFF)
535#define QUEUE_STATE_FROZEN (1 << __QUEUE_STATE_FROZEN)
536
537#define QUEUE_STATE_ANY_XOFF (QUEUE_STATE_DRV_XOFF | QUEUE_STATE_STACK_XOFF)
538#define QUEUE_STATE_ANY_XOFF_OR_FROZEN (QUEUE_STATE_ANY_XOFF | \
539 QUEUE_STATE_FROZEN)
540#define QUEUE_STATE_DRV_XOFF_OR_FROZEN (QUEUE_STATE_DRV_XOFF | \
541 QUEUE_STATE_FROZEN)
542
Tom Herbert734664982011-11-28 16:32:44 +0000543/*
544 * __QUEUE_STATE_DRV_XOFF is used by drivers to stop the transmit queue. The
545 * netif_tx_* functions below are used to manipulate this flag. The
546 * __QUEUE_STATE_STACK_XOFF flag is used by the stack to stop the transmit
547 * queue independently. The netif_xmit_*stopped functions below are called
548 * to check if the queue has been stopped by the driver or stack (either
549 * of the XOFF bits are set in the state). Drivers should not need to call
550 * netif_xmit*stopped functions, they should only be using netif_tx_*.
551 */
David S. Miller79d16382008-07-08 23:14:46 -0700552
David S. Millerbb949fb2008-07-08 16:55:56 -0700553struct netdev_queue {
Eric Dumazet6a321cb2009-04-28 04:43:42 -0700554/*
Bjorn Helgaas5e82b4b2016-03-23 13:47:23 -0500555 * read-mostly part
Eric Dumazet6a321cb2009-04-28 04:43:42 -0700556 */
David S. Millerbb949fb2008-07-08 16:55:56 -0700557 struct net_device *dev;
John Fastabend46e5da40a2014-09-12 20:04:52 -0700558 struct Qdisc __rcu *qdisc;
David S. Millerb0e1e642008-07-08 17:42:10 -0700559 struct Qdisc *qdisc_sleeping;
david decotignyccf5ff62011-11-16 12:15:10 +0000560#ifdef CONFIG_SYSFS
Tom Herbert1d24eb42010-11-21 13:17:27 +0000561 struct kobject kobj;
562#endif
Eric Dumazetf2cd2d32010-11-29 08:14:37 +0000563#if defined(CONFIG_XPS) && defined(CONFIG_NUMA)
564 int numa_node;
565#endif
Florian Westphalc0ef0792016-05-03 03:29:09 +0200566 unsigned long tx_maxrate;
567 /*
568 * Number of TX timeouts for this queue
569 * (/sys/class/net/DEV/Q/trans_timeout)
570 */
571 unsigned long trans_timeout;
Eric Dumazet6a321cb2009-04-28 04:43:42 -0700572/*
Bjorn Helgaas5e82b4b2016-03-23 13:47:23 -0500573 * write-mostly part
Eric Dumazet6a321cb2009-04-28 04:43:42 -0700574 */
575 spinlock_t _xmit_lock ____cacheline_aligned_in_smp;
576 int xmit_lock_owner;
Eric Dumazet9d214932009-05-17 20:55:16 -0700577 /*
Florian Westphal9b366272016-05-03 16:33:14 +0200578 * Time (in jiffies) of last Tx
Eric Dumazet9d214932009-05-17 20:55:16 -0700579 */
580 unsigned long trans_start;
david decotignyccf5ff62011-11-16 12:15:10 +0000581
Tom Herbert114cf582011-11-28 16:33:09 +0000582 unsigned long state;
583
584#ifdef CONFIG_BQL
585 struct dql dql;
586#endif
David S. Millere8a04642008-07-17 00:34:19 -0700587} ____cacheline_aligned_in_smp;
David S. Millerbb949fb2008-07-08 16:55:56 -0700588
Eric Dumazet79134e62018-03-08 12:51:41 -0800589extern int sysctl_fb_tunnels_only_for_init_net;
590
591static inline bool net_has_fallback_tunnels(const struct net *net)
592{
Arnd Bergmannbe9fc092018-03-13 12:44:53 +0100593 return net == &init_net ||
594 !IS_ENABLED(CONFIG_SYSCTL) ||
595 !sysctl_fb_tunnels_only_for_init_net;
Eric Dumazet79134e62018-03-08 12:51:41 -0800596}
597
Eric Dumazetf2cd2d32010-11-29 08:14:37 +0000598static inline int netdev_queue_numa_node_read(const struct netdev_queue *q)
599{
600#if defined(CONFIG_XPS) && defined(CONFIG_NUMA)
601 return q->numa_node;
602#else
Changli Gaob236da62010-12-14 03:09:15 +0000603 return NUMA_NO_NODE;
Eric Dumazetf2cd2d32010-11-29 08:14:37 +0000604#endif
605}
606
607static inline void netdev_queue_numa_node_write(struct netdev_queue *q, int node)
608{
609#if defined(CONFIG_XPS) && defined(CONFIG_NUMA)
610 q->numa_node = node;
611#endif
612}
613
Eric Dumazetdf334542010-03-24 19:13:54 +0000614#ifdef CONFIG_RPS
Tom Herbert0a9627f2010-03-16 08:03:29 +0000615/*
616 * This structure holds an RPS map which can be of variable length. The
617 * map is an array of CPUs.
618 */
619struct rps_map {
620 unsigned int len;
621 struct rcu_head rcu;
622 u16 cpus[0];
623};
Eric Dumazet60b778c2011-12-24 06:56:49 +0000624#define RPS_MAP_SIZE(_num) (sizeof(struct rps_map) + ((_num) * sizeof(u16)))
Tom Herbert0a9627f2010-03-16 08:03:29 +0000625
Tom Herbertfec5e652010-04-16 16:01:27 -0700626/*
Ben Hutchingsc4454772011-01-19 11:03:53 +0000627 * The rps_dev_flow structure contains the mapping of a flow to a CPU, the
628 * tail pointer for that CPU's input queue at the time of last enqueue, and
629 * a hardware filter index.
Tom Herbertfec5e652010-04-16 16:01:27 -0700630 */
631struct rps_dev_flow {
632 u16 cpu;
Ben Hutchingsc4454772011-01-19 11:03:53 +0000633 u16 filter;
Tom Herbertfec5e652010-04-16 16:01:27 -0700634 unsigned int last_qtail;
635};
Ben Hutchingsc4454772011-01-19 11:03:53 +0000636#define RPS_NO_FILTER 0xffff
Tom Herbertfec5e652010-04-16 16:01:27 -0700637
638/*
639 * The rps_dev_flow_table structure contains a table of flow mappings.
640 */
641struct rps_dev_flow_table {
642 unsigned int mask;
643 struct rcu_head rcu;
Tom Herbertfec5e652010-04-16 16:01:27 -0700644 struct rps_dev_flow flows[0];
645};
646#define RPS_DEV_FLOW_TABLE_SIZE(_num) (sizeof(struct rps_dev_flow_table) + \
Eric Dumazet60b778c2011-12-24 06:56:49 +0000647 ((_num) * sizeof(struct rps_dev_flow)))
Tom Herbertfec5e652010-04-16 16:01:27 -0700648
649/*
650 * The rps_sock_flow_table contains mappings of flows to the last CPU
651 * on which they were processed by the application (set in recvmsg).
Bjorn Helgaas5e82b4b2016-03-23 13:47:23 -0500652 * Each entry is a 32bit value. Upper part is the high-order bits
653 * of flow hash, lower part is CPU number.
Eric Dumazet567e4b72015-02-06 12:59:01 -0800654 * rps_cpu_mask is used to partition the space, depending on number of
Bjorn Helgaas5e82b4b2016-03-23 13:47:23 -0500655 * possible CPUs : rps_cpu_mask = roundup_pow_of_two(nr_cpu_ids) - 1
656 * For example, if 64 CPUs are possible, rps_cpu_mask = 0x3f,
Eric Dumazet567e4b72015-02-06 12:59:01 -0800657 * meaning we use 32-6=26 bits for the hash.
Tom Herbertfec5e652010-04-16 16:01:27 -0700658 */
659struct rps_sock_flow_table {
Eric Dumazet567e4b72015-02-06 12:59:01 -0800660 u32 mask;
Eric Dumazet93c1af62015-02-08 20:39:13 -0800661
662 u32 ents[0] ____cacheline_aligned_in_smp;
Tom Herbertfec5e652010-04-16 16:01:27 -0700663};
Eric Dumazet567e4b72015-02-06 12:59:01 -0800664#define RPS_SOCK_FLOW_TABLE_SIZE(_num) (offsetof(struct rps_sock_flow_table, ents[_num]))
Tom Herbertfec5e652010-04-16 16:01:27 -0700665
666#define RPS_NO_CPU 0xffff
667
Eric Dumazet567e4b72015-02-06 12:59:01 -0800668extern u32 rps_cpu_mask;
669extern struct rps_sock_flow_table __rcu *rps_sock_flow_table;
670
Tom Herbertfec5e652010-04-16 16:01:27 -0700671static inline void rps_record_sock_flow(struct rps_sock_flow_table *table,
672 u32 hash)
673{
674 if (table && hash) {
Eric Dumazet567e4b72015-02-06 12:59:01 -0800675 unsigned int index = hash & table->mask;
676 u32 val = hash & ~rps_cpu_mask;
Tom Herbertfec5e652010-04-16 16:01:27 -0700677
Bjorn Helgaas5e82b4b2016-03-23 13:47:23 -0500678 /* We only give a hint, preemption can change CPU under us */
Eric Dumazet567e4b72015-02-06 12:59:01 -0800679 val |= raw_smp_processor_id();
Tom Herbertfec5e652010-04-16 16:01:27 -0700680
Eric Dumazet567e4b72015-02-06 12:59:01 -0800681 if (table->ents[index] != val)
682 table->ents[index] = val;
Tom Herbertfec5e652010-04-16 16:01:27 -0700683 }
684}
685
Ben Hutchingsc4454772011-01-19 11:03:53 +0000686#ifdef CONFIG_RFS_ACCEL
Joe Perchesf629d202013-09-26 14:48:15 -0700687bool rps_may_expire_flow(struct net_device *dev, u16 rxq_index, u32 flow_id,
688 u16 filter_id);
Ben Hutchingsc4454772011-01-19 11:03:53 +0000689#endif
Michael Daltona953be52014-01-16 22:23:28 -0800690#endif /* CONFIG_RPS */
Ben Hutchingsc4454772011-01-19 11:03:53 +0000691
Tom Herbert0a9627f2010-03-16 08:03:29 +0000692/* This structure contains an instance of an RX queue. */
693struct netdev_rx_queue {
Michael Daltona953be52014-01-16 22:23:28 -0800694#ifdef CONFIG_RPS
Eric Dumazet6e3f7fa2010-10-25 03:02:02 +0000695 struct rps_map __rcu *rps_map;
696 struct rps_dev_flow_table __rcu *rps_flow_table;
Michael Daltona953be52014-01-16 22:23:28 -0800697#endif
Eric Dumazet6e3f7fa2010-10-25 03:02:02 +0000698 struct kobject kobj;
Tom Herbertfe822242010-11-09 10:47:38 +0000699 struct net_device *dev;
Jesper Dangaard Brouere817f852018-01-03 11:26:09 +0100700 struct xdp_rxq_info xdp_rxq;
Tom Herbert0a9627f2010-03-16 08:03:29 +0000701} ____cacheline_aligned_in_smp;
Michael Daltona953be52014-01-16 22:23:28 -0800702
703/*
704 * RX queue sysfs structures and functions.
705 */
706struct rx_queue_attribute {
707 struct attribute attr;
stephen hemminger718ad682017-08-18 13:46:24 -0700708 ssize_t (*show)(struct netdev_rx_queue *queue, char *buf);
Michael Daltona953be52014-01-16 22:23:28 -0800709 ssize_t (*store)(struct netdev_rx_queue *queue,
stephen hemminger718ad682017-08-18 13:46:24 -0700710 const char *buf, size_t len);
Michael Daltona953be52014-01-16 22:23:28 -0800711};
Stephen Hemmingerd3147742008-11-19 21:32:24 -0800712
Tom Herbertbf264142010-11-26 08:36:09 +0000713#ifdef CONFIG_XPS
714/*
715 * This structure holds an XPS map which can be of variable length. The
716 * map is an array of queues.
717 */
718struct xps_map {
719 unsigned int len;
720 unsigned int alloc_len;
721 struct rcu_head rcu;
722 u16 queues[0];
723};
Eric Dumazet60b778c2011-12-24 06:56:49 +0000724#define XPS_MAP_SIZE(_num) (sizeof(struct xps_map) + ((_num) * sizeof(u16)))
Helge Dellerc59f4192015-10-25 10:00:32 +0100725#define XPS_MIN_MAP_ALLOC ((L1_CACHE_ALIGN(offsetof(struct xps_map, queues[1])) \
726 - sizeof(struct xps_map)) / sizeof(u16))
Tom Herbertbf264142010-11-26 08:36:09 +0000727
728/*
729 * This structure holds all XPS maps for device. Maps are indexed by CPU.
730 */
731struct xps_dev_maps {
732 struct rcu_head rcu;
Eric Dumazeta4177862010-11-28 21:43:02 +0000733 struct xps_map __rcu *cpu_map[0];
Tom Herbertbf264142010-11-26 08:36:09 +0000734};
Alexander Duyck184c4492016-10-28 11:50:13 -0400735#define XPS_DEV_MAPS_SIZE(_tcs) (sizeof(struct xps_dev_maps) + \
736 (nr_cpu_ids * (_tcs) * sizeof(struct xps_map *)))
Tom Herbertbf264142010-11-26 08:36:09 +0000737#endif /* CONFIG_XPS */
738
John Fastabend4f57c082011-01-17 08:06:04 +0000739#define TC_MAX_QUEUE 16
740#define TC_BITMASK 15
741/* HW offloaded queuing disciplines txq count and offset maps */
742struct netdev_tc_txq {
743 u16 count;
744 u16 offset;
745};
746
Neerav Parikh68bad942012-01-04 20:23:39 +0000747#if defined(CONFIG_FCOE) || defined(CONFIG_FCOE_MODULE)
748/*
749 * This structure is to hold information about the device
750 * configured to run FCoE protocol stack.
751 */
752struct netdev_fcoe_hbainfo {
753 char manufacturer[64];
754 char serial_number[64];
755 char hardware_version[64];
756 char driver_version[64];
757 char optionrom_version[64];
758 char firmware_version[64];
759 char model[256];
760 char model_description[256];
761};
762#endif
763
Jiri Pirko02637fc2014-11-28 14:34:16 +0100764#define MAX_PHYS_ITEM_ID_LEN 32
Jiri Pirko66b52b02013-07-29 18:16:49 +0200765
Jiri Pirko02637fc2014-11-28 14:34:16 +0100766/* This structure holds a unique identifier to identify some
767 * physical item (port for example) used by a netdevice.
Jiri Pirko66b52b02013-07-29 18:16:49 +0200768 */
Jiri Pirko02637fc2014-11-28 14:34:16 +0100769struct netdev_phys_item_id {
770 unsigned char id[MAX_PHYS_ITEM_ID_LEN];
Jiri Pirko66b52b02013-07-29 18:16:49 +0200771 unsigned char id_len;
772};
773
Scott Feldmand754f982015-07-18 18:24:49 -0700774static inline bool netdev_phys_item_id_same(struct netdev_phys_item_id *a,
775 struct netdev_phys_item_id *b)
776{
777 return a->id_len == b->id_len &&
778 memcmp(a->id, b->id, a->id_len) == 0;
779}
780
Daniel Borkmann99932d42014-02-16 15:55:20 +0100781typedef u16 (*select_queue_fallback_t)(struct net_device *dev,
782 struct sk_buff *skb);
783
Jiri Pirko2572ac52017-08-07 10:15:17 +0200784enum tc_setup_type {
Nogah Frankel575ed7d2017-11-06 07:23:42 +0100785 TC_SETUP_QDISC_MQPRIO,
John Fastabenda1b7c5f2016-02-16 21:17:09 -0800786 TC_SETUP_CLSU32,
Amir Vadai5b33f482016-03-08 12:42:29 +0200787 TC_SETUP_CLSFLOWER,
Jiri Pirkoade9b652017-08-07 10:15:18 +0200788 TC_SETUP_CLSMATCHALL,
Jakub Kicinski332ae8e2016-09-21 11:43:53 +0100789 TC_SETUP_CLSBPF,
Jiri Pirko8c4083b2017-10-19 15:50:29 +0200790 TC_SETUP_BLOCK,
Nogah Frankel8521db42017-11-06 07:23:43 +0100791 TC_SETUP_QDISC_CBS,
Nogah Frankel602f3ba2017-11-06 07:23:41 +0100792 TC_SETUP_QDISC_RED,
Nogah Frankel7fdb61b2018-01-14 12:33:15 +0100793 TC_SETUP_QDISC_PRIO,
John Fastabend16e5cc62016-02-16 21:16:43 -0800794};
795
Jakub Kicinskif4e63522017-11-03 13:56:16 -0700796/* These structures hold the attributes of bpf state that are being passed
797 * to the netdevice through the bpf op.
Brenden Blancoa7862b42016-07-19 12:16:48 -0700798 */
Jakub Kicinskif4e63522017-11-03 13:56:16 -0700799enum bpf_netdev_command {
Brenden Blancoa7862b42016-07-19 12:16:48 -0700800 /* Set or clear a bpf program used in the earliest stages of packet
801 * rx. The prog will have been loaded as BPF_PROG_TYPE_XDP. The callee
802 * is responsible for calling bpf_prog_put on any old progs that are
803 * stored. In case of error, the callee need not release the new prog
804 * reference, but on success it takes ownership and must bpf_prog_put
805 * when it is no longer used.
806 */
807 XDP_SETUP_PROG,
Jakub Kicinskiee5d0322017-06-21 18:25:04 -0700808 XDP_SETUP_PROG_HW,
Brenden Blancoa7862b42016-07-19 12:16:48 -0700809 /* Check if a bpf program is set on the device. The callee should
Jakub Kicinskice158e52017-06-21 18:25:09 -0700810 * set @prog_attached to one of XDP_ATTACHED_* values, note that "true"
811 * is equivalent to XDP_ATTACHED_DRV.
Brenden Blancoa7862b42016-07-19 12:16:48 -0700812 */
813 XDP_QUERY_PROG,
Jakub Kicinskiab3f0062017-11-03 13:56:17 -0700814 /* BPF program for offload callbacks, invoked at program load time. */
815 BPF_OFFLOAD_VERIFIER_PREP,
816 BPF_OFFLOAD_TRANSLATE,
817 BPF_OFFLOAD_DESTROY,
Jakub Kicinskia3884572018-01-11 20:29:09 -0800818 BPF_OFFLOAD_MAP_ALLOC,
819 BPF_OFFLOAD_MAP_FREE,
Brenden Blancoa7862b42016-07-19 12:16:48 -0700820};
821
Jakub Kicinskicae19272017-12-27 18:39:05 -0800822struct bpf_prog_offload_ops;
Jakub Kicinskiddf9f972017-04-30 21:46:46 -0700823struct netlink_ext_ack;
824
Jakub Kicinskif4e63522017-11-03 13:56:16 -0700825struct netdev_bpf {
826 enum bpf_netdev_command command;
Brenden Blancoa7862b42016-07-19 12:16:48 -0700827 union {
828 /* XDP_SETUP_PROG */
Jakub Kicinskiddf9f972017-04-30 21:46:46 -0700829 struct {
Jakub Kicinski32d60272017-06-21 18:25:03 -0700830 u32 flags;
Jakub Kicinskiddf9f972017-04-30 21:46:46 -0700831 struct bpf_prog *prog;
832 struct netlink_ext_ack *extack;
833 };
Brenden Blancoa7862b42016-07-19 12:16:48 -0700834 /* XDP_QUERY_PROG */
Martin KaFai Lau58038692017-06-15 17:29:09 -0700835 struct {
Jakub Kicinskice158e52017-06-21 18:25:09 -0700836 u8 prog_attached;
Martin KaFai Lau58038692017-06-15 17:29:09 -0700837 u32 prog_id;
Jakub Kicinski92f02922017-12-01 15:08:56 -0800838 /* flags with which program was installed */
839 u32 prog_flags;
Martin KaFai Lau58038692017-06-15 17:29:09 -0700840 };
Jakub Kicinskiab3f0062017-11-03 13:56:17 -0700841 /* BPF_OFFLOAD_VERIFIER_PREP */
842 struct {
843 struct bpf_prog *prog;
Jakub Kicinskicae19272017-12-27 18:39:05 -0800844 const struct bpf_prog_offload_ops *ops; /* callee set */
Jakub Kicinskiab3f0062017-11-03 13:56:17 -0700845 } verifier;
846 /* BPF_OFFLOAD_TRANSLATE, BPF_OFFLOAD_DESTROY */
847 struct {
848 struct bpf_prog *prog;
849 } offload;
Jakub Kicinskia3884572018-01-11 20:29:09 -0800850 /* BPF_OFFLOAD_MAP_ALLOC, BPF_OFFLOAD_MAP_FREE */
851 struct {
852 struct bpf_offloaded_map *offmap;
853 };
Brenden Blancoa7862b42016-07-19 12:16:48 -0700854 };
855};
John Fastabend16e5cc62016-02-16 21:16:43 -0800856
Steffen Klassertd77e38e2017-04-14 10:06:10 +0200857#ifdef CONFIG_XFRM_OFFLOAD
858struct xfrmdev_ops {
859 int (*xdo_dev_state_add) (struct xfrm_state *x);
860 void (*xdo_dev_state_delete) (struct xfrm_state *x);
861 void (*xdo_dev_state_free) (struct xfrm_state *x);
862 bool (*xdo_dev_offload_ok) (struct sk_buff *skb,
863 struct xfrm_state *x);
Yossef Efraim50bd8702018-01-14 11:39:10 +0200864 void (*xdo_dev_state_advance_esn) (struct xfrm_state *x);
Steffen Klassertd77e38e2017-04-14 10:06:10 +0200865};
866#endif
867
Ilya Lesokhina5c37c62018-04-30 10:16:13 +0300868#if IS_ENABLED(CONFIG_TLS_DEVICE)
869enum tls_offload_ctx_dir {
870 TLS_OFFLOAD_CTX_DIR_RX,
871 TLS_OFFLOAD_CTX_DIR_TX,
872};
873
874struct tls_crypto_info;
875struct tls_context;
876
877struct tlsdev_ops {
878 int (*tls_dev_add)(struct net_device *netdev, struct sock *sk,
879 enum tls_offload_ctx_dir direction,
880 struct tls_crypto_info *crypto_info,
881 u32 start_offload_tcp_sn);
882 void (*tls_dev_del)(struct net_device *netdev,
883 struct tls_context *ctx,
884 enum tls_offload_ctx_dir direction);
885};
886#endif
887
Florian Westphal6c557002017-10-02 23:50:05 +0200888struct dev_ifalias {
889 struct rcu_head rcuhead;
890 char ifalias[];
891};
892
Stephen Hemmingerd3147742008-11-19 21:32:24 -0800893/*
894 * This structure defines the management hooks for network devices.
Stephen Hemminger00829822008-11-20 20:14:53 -0800895 * The following hooks can be defined; unless noted otherwise, they are
896 * optional and can be filled with a null pointer.
Stephen Hemmingerd3147742008-11-19 21:32:24 -0800897 *
898 * int (*ndo_init)(struct net_device *dev);
Bjorn Helgaas5e82b4b2016-03-23 13:47:23 -0500899 * This function is called once when a network device is registered.
900 * The network device can use this for any late stage initialization
901 * or semantic validation. It can fail with an error code which will
902 * be propagated back to register_netdev.
Stephen Hemmingerd3147742008-11-19 21:32:24 -0800903 *
904 * void (*ndo_uninit)(struct net_device *dev);
905 * This function is called when device is unregistered or when registration
906 * fails. It is not called if init fails.
907 *
908 * int (*ndo_open)(struct net_device *dev);
Bjorn Helgaas5e82b4b2016-03-23 13:47:23 -0500909 * This function is called when a network device transitions to the up
Stephen Hemmingerd3147742008-11-19 21:32:24 -0800910 * state.
911 *
912 * int (*ndo_stop)(struct net_device *dev);
Bjorn Helgaas5e82b4b2016-03-23 13:47:23 -0500913 * This function is called when a network device transitions to the down
Stephen Hemmingerd3147742008-11-19 21:32:24 -0800914 * state.
915 *
Stephen Hemmingerdc1f8bf2009-08-31 19:50:40 +0000916 * netdev_tx_t (*ndo_start_xmit)(struct sk_buff *skb,
917 * struct net_device *dev);
Stephen Hemminger00829822008-11-20 20:14:53 -0800918 * Called when a packet needs to be transmitted.
Rusty Russelle79d8422015-04-03 22:17:17 +1030919 * Returns NETDEV_TX_OK. Can return NETDEV_TX_BUSY, but you should stop
920 * the queue before that can happen; it's for obsolete devices and weird
921 * corner cases, but the stack really does a non-trivial amount
922 * of useless work if you return NETDEV_TX_BUSY.
Bjorn Helgaas5e82b4b2016-03-23 13:47:23 -0500923 * Required; cannot be NULL.
Stephen Hemminger00829822008-11-20 20:14:53 -0800924 *
Dimitris Michailidis1a2a1442017-01-31 16:03:13 -0800925 * netdev_features_t (*ndo_features_check)(struct sk_buff *skb,
926 * struct net_device *dev
927 * netdev_features_t features);
928 * Called by core transmit path to determine if device is capable of
929 * performing offload operations on a given packet. This is to give
930 * the device an opportunity to implement any restrictions that cannot
931 * be otherwise expressed by feature flags. The check is called with
932 * the set of features that the stack has calculated and it returns
933 * those the driver believes to be appropriate.
Eric Dumazetcdba7562016-01-06 06:53:50 -0800934 *
Jason Wangf663dd92014-01-10 16:18:26 +0800935 * u16 (*ndo_select_queue)(struct net_device *dev, struct sk_buff *skb,
Daniel Borkmann99932d42014-02-16 15:55:20 +0100936 * void *accel_priv, select_queue_fallback_t fallback);
Bjorn Helgaas5e82b4b2016-03-23 13:47:23 -0500937 * Called to decide which queue to use when device supports multiple
Stephen Hemminger00829822008-11-20 20:14:53 -0800938 * transmit queues.
939 *
Stephen Hemmingerd3147742008-11-19 21:32:24 -0800940 * void (*ndo_change_rx_flags)(struct net_device *dev, int flags);
941 * This function is called to allow device receiver to make
Bjorn Helgaas5e82b4b2016-03-23 13:47:23 -0500942 * changes to configuration when multicast or promiscuous is enabled.
Stephen Hemmingerd3147742008-11-19 21:32:24 -0800943 *
944 * void (*ndo_set_rx_mode)(struct net_device *dev);
945 * This function is called device changes address list filtering.
Jiri Pirko01789342011-08-16 06:29:00 +0000946 * If driver handles unicast address filtering, it should set
Bjorn Helgaas5e82b4b2016-03-23 13:47:23 -0500947 * IFF_UNICAST_FLT in its priv_flags.
Stephen Hemmingerd3147742008-11-19 21:32:24 -0800948 *
949 * int (*ndo_set_mac_address)(struct net_device *dev, void *addr);
950 * This function is called when the Media Access Control address
Mike Rapoport37b607c2009-04-27 05:45:54 -0700951 * needs to be changed. If this interface is not defined, the
Bjorn Helgaas5e82b4b2016-03-23 13:47:23 -0500952 * MAC address can not be changed.
Stephen Hemmingerd3147742008-11-19 21:32:24 -0800953 *
954 * int (*ndo_validate_addr)(struct net_device *dev);
955 * Test if Media Access Control address is valid for the device.
956 *
957 * int (*ndo_do_ioctl)(struct net_device *dev, struct ifreq *ifr, int cmd);
Bjorn Helgaas5e82b4b2016-03-23 13:47:23 -0500958 * Called when a user requests an ioctl which can't be handled by
959 * the generic interface code. If not defined ioctls return
Stephen Hemmingerd3147742008-11-19 21:32:24 -0800960 * not supported error code.
961 *
962 * int (*ndo_set_config)(struct net_device *dev, struct ifmap *map);
963 * Used to set network devices bus interface parameters. This interface
Bjorn Helgaas5e82b4b2016-03-23 13:47:23 -0500964 * is retained for legacy reasons; new devices should use the bus
Stephen Hemmingerd3147742008-11-19 21:32:24 -0800965 * interface (PCI) for low level management.
966 *
967 * int (*ndo_change_mtu)(struct net_device *dev, int new_mtu);
968 * Called when a user wants to change the Maximum Transfer Unit
Magnus Dammdb46a0e2017-06-14 16:15:24 +0900969 * of a device.
Stephen Hemmingerd3147742008-11-19 21:32:24 -0800970 *
Stephen Hemminger00829822008-11-20 20:14:53 -0800971 * void (*ndo_tx_timeout)(struct net_device *dev);
Bjorn Helgaas5e82b4b2016-03-23 13:47:23 -0500972 * Callback used when the transmitter has not made any progress
Stephen Hemmingerd3147742008-11-19 21:32:24 -0800973 * for dev->watchdog ticks.
974 *
stephen hemmingerbc1f4472017-01-06 19:12:52 -0800975 * void (*ndo_get_stats64)(struct net_device *dev,
976 * struct rtnl_link_stats64 *storage);
Wolfram Sangd308e382009-10-07 13:53:11 -0700977 * struct net_device_stats* (*ndo_get_stats)(struct net_device *dev);
Stephen Hemmingerd3147742008-11-19 21:32:24 -0800978 * Called when a user wants to get the network device usage
Ben Hutchingsbe1f3c22010-06-08 07:19:54 +0000979 * statistics. Drivers must do one of the following:
Ben Hutchings3cfde792010-07-09 09:11:52 +0000980 * 1. Define @ndo_get_stats64 to fill in a zero-initialised
981 * rtnl_link_stats64 structure passed by the caller.
Ben Hutchings82695d92010-06-15 15:08:48 -0700982 * 2. Define @ndo_get_stats to update a net_device_stats structure
Ben Hutchingsbe1f3c22010-06-08 07:19:54 +0000983 * (which should normally be dev->stats) and return a pointer to
984 * it. The structure may be changed asynchronously only if each
985 * field is written atomically.
986 * 3. Update dev->stats asynchronously and atomically, and define
987 * neither operation.
Stephen Hemmingerd3147742008-11-19 21:32:24 -0800988 *
Or Gerlitz3df5b3c2016-11-22 23:09:54 +0200989 * bool (*ndo_has_offload_stats)(const struct net_device *dev, int attr_id)
Nogah Frankel2c9d85d2016-09-16 15:05:36 +0200990 * Return true if this device supports offload stats of this attr_id.
991 *
992 * int (*ndo_get_offload_stats)(int attr_id, const struct net_device *dev,
993 * void *attr_data)
994 * Get statistics for offload operations by attr_id. Write it into the
995 * attr_data pointer.
996 *
B Viswanath5d632cb2015-01-12 14:46:25 +0530997 * int (*ndo_vlan_rx_add_vid)(struct net_device *dev, __be16 proto, u16 vid);
Bjorn Helgaas5e82b4b2016-03-23 13:47:23 -0500998 * If device supports VLAN filtering this function is called when a
Patrick McHardy80d5c362013-04-19 02:04:28 +0000999 * VLAN id is registered.
Stephen Hemmingerd3147742008-11-19 21:32:24 -08001000 *
B Viswanath5d632cb2015-01-12 14:46:25 +05301001 * int (*ndo_vlan_rx_kill_vid)(struct net_device *dev, __be16 proto, u16 vid);
Bjorn Helgaas5e82b4b2016-03-23 13:47:23 -05001002 * If device supports VLAN filtering this function is called when a
Patrick McHardy80d5c362013-04-19 02:04:28 +00001003 * VLAN id is unregistered.
Stephen Hemmingerd3147742008-11-19 21:32:24 -08001004 *
1005 * void (*ndo_poll_controller)(struct net_device *dev);
Williams, Mitch A95c26df2010-02-10 01:43:46 +00001006 *
1007 * SR-IOV management functions.
1008 * int (*ndo_set_vf_mac)(struct net_device *dev, int vf, u8* mac);
Moshe Shemesh79aab092016-09-22 12:11:15 +03001009 * int (*ndo_set_vf_vlan)(struct net_device *dev, int vf, u16 vlan,
1010 * u8 qos, __be16 proto);
Sucheta Chakrabortyed616682014-05-22 09:59:05 -04001011 * int (*ndo_set_vf_rate)(struct net_device *dev, int vf, int min_tx_rate,
1012 * int max_tx_rate);
Greg Rose5f8444a2011-10-08 03:05:24 +00001013 * int (*ndo_set_vf_spoofchk)(struct net_device *dev, int vf, bool setting);
Hiroshi Shimamotodd461d62015-08-28 06:57:55 +00001014 * int (*ndo_set_vf_trust)(struct net_device *dev, int vf, bool setting);
Williams, Mitch A95c26df2010-02-10 01:43:46 +00001015 * int (*ndo_get_vf_config)(struct net_device *dev,
1016 * int vf, struct ifla_vf_info *ivf);
Rony Efraim1d8faf42013-06-13 13:19:10 +03001017 * int (*ndo_set_vf_link_state)(struct net_device *dev, int vf, int link_state);
Scott Feldman57b61082010-05-17 22:49:55 -07001018 * int (*ndo_set_vf_port)(struct net_device *dev, int vf,
1019 * struct nlattr *port[]);
Vlad Zolotarov01a3d792015-03-30 21:35:23 +03001020 *
1021 * Enable or disable the VF ability to query its RSS Redirection Table and
1022 * Hash Key. This is needed since on some devices VF share this information
Bjorn Helgaas5e82b4b2016-03-23 13:47:23 -05001023 * with PF and querying it may introduce a theoretical security risk.
Vlad Zolotarov01a3d792015-03-30 21:35:23 +03001024 * int (*ndo_set_vf_rss_query_en)(struct net_device *dev, int vf, bool setting);
Scott Feldman57b61082010-05-17 22:49:55 -07001025 * int (*ndo_get_vf_port)(struct net_device *dev, int vf, struct sk_buff *skb);
Jiri Pirko2572ac52017-08-07 10:15:17 +02001026 * int (*ndo_setup_tc)(struct net_device *dev, enum tc_setup_type type,
Jiri Pirkode4784c2017-08-07 10:15:32 +02001027 * void *type_data);
Florian Fainelli6a4bc2b2017-01-26 14:44:17 -08001028 * Called to setup any 'tc' scheduler, classifier or action on @dev.
1029 * This is always called from the stack with the rtnl lock held and netif
1030 * tx queues stopped. This allows the netdevice to perform queue
1031 * management safely.
Ben Hutchingsc4454772011-01-19 11:03:53 +00001032 *
Yi Zoue9bce842011-03-09 08:48:03 +00001033 * Fiber Channel over Ethernet (FCoE) offload functions.
1034 * int (*ndo_fcoe_enable)(struct net_device *dev);
1035 * Called when the FCoE protocol stack wants to start using LLD for FCoE
1036 * so the underlying device can perform whatever needed configuration or
1037 * initialization to support acceleration of FCoE traffic.
1038 *
1039 * int (*ndo_fcoe_disable)(struct net_device *dev);
1040 * Called when the FCoE protocol stack wants to stop using LLD for FCoE
1041 * so the underlying device can perform whatever needed clean-ups to
1042 * stop supporting acceleration of FCoE traffic.
1043 *
1044 * int (*ndo_fcoe_ddp_setup)(struct net_device *dev, u16 xid,
1045 * struct scatterlist *sgl, unsigned int sgc);
1046 * Called when the FCoE Initiator wants to initialize an I/O that
1047 * is a possible candidate for Direct Data Placement (DDP). The LLD can
1048 * perform necessary setup and returns 1 to indicate the device is set up
1049 * successfully to perform DDP on this I/O, otherwise this returns 0.
1050 *
1051 * int (*ndo_fcoe_ddp_done)(struct net_device *dev, u16 xid);
1052 * Called when the FCoE Initiator/Target is done with the DDPed I/O as
1053 * indicated by the FC exchange id 'xid', so the underlying device can
1054 * clean up and reuse resources for later DDP requests.
1055 *
1056 * int (*ndo_fcoe_ddp_target)(struct net_device *dev, u16 xid,
1057 * struct scatterlist *sgl, unsigned int sgc);
1058 * Called when the FCoE Target wants to initialize an I/O that
1059 * is a possible candidate for Direct Data Placement (DDP). The LLD can
1060 * perform necessary setup and returns 1 to indicate the device is set up
1061 * successfully to perform DDP on this I/O, otherwise this returns 0.
1062 *
Neerav Parikh68bad942012-01-04 20:23:39 +00001063 * int (*ndo_fcoe_get_hbainfo)(struct net_device *dev,
1064 * struct netdev_fcoe_hbainfo *hbainfo);
1065 * Called when the FCoE Protocol stack wants information on the underlying
1066 * device. This information is utilized by the FCoE protocol stack to
1067 * register attributes with Fiber Channel management service as per the
1068 * FC-GS Fabric Device Management Information(FDMI) specification.
1069 *
Yi Zoue9bce842011-03-09 08:48:03 +00001070 * int (*ndo_fcoe_get_wwn)(struct net_device *dev, u64 *wwn, int type);
1071 * Called when the underlying device wants to override default World Wide
1072 * Name (WWN) generation mechanism in FCoE protocol stack to pass its own
1073 * World Wide Port Name (WWPN) or World Wide Node Name (WWNN) to the FCoE
1074 * protocol stack to use.
1075 *
Ben Hutchingsc4454772011-01-19 11:03:53 +00001076 * RFS acceleration.
1077 * int (*ndo_rx_flow_steer)(struct net_device *dev, const struct sk_buff *skb,
1078 * u16 rxq_index, u32 flow_id);
1079 * Set hardware filter for RFS. rxq_index is the target queue index;
1080 * flow_id is a flow ID to be passed to rps_may_expire_flow() later.
1081 * Return the filter ID on success, or a negative error code.
Jiri Pirkofbaec0e2011-02-13 10:15:37 +00001082 *
Jiri Pirko8b98a702013-01-03 22:49:02 +00001083 * Slave management functions (for bridge, bonding, etc).
Jiri Pirkofbaec0e2011-02-13 10:15:37 +00001084 * int (*ndo_add_slave)(struct net_device *dev, struct net_device *slave_dev);
1085 * Called to make another netdev an underling.
1086 *
1087 * int (*ndo_del_slave)(struct net_device *dev, struct net_device *slave_dev);
1088 * Called to release previously enslaved netdev.
Michał Mirosław5455c692011-02-15 16:59:17 +00001089 *
1090 * Feature/offload setting functions.
Dimitris Michailidis1a2a1442017-01-31 16:03:13 -08001091 * netdev_features_t (*ndo_fix_features)(struct net_device *dev,
1092 * netdev_features_t features);
1093 * Adjusts the requested feature flags according to device-specific
1094 * constraints, and returns the resulting flags. Must not modify
1095 * the device state.
1096 *
Michał Mirosławc8f44af2011-11-15 15:29:55 +00001097 * int (*ndo_set_features)(struct net_device *dev, netdev_features_t features);
Michał Mirosław5455c692011-02-15 16:59:17 +00001098 * Called to update device configuration to new features. Passed
1099 * feature set might be less than what was returned by ndo_fix_features()).
1100 * Must return >0 or -errno if it changed dev->features itself.
1101 *
stephen hemmingeredc7d572012-10-01 12:32:33 +00001102 * int (*ndo_fdb_add)(struct ndmsg *ndm, struct nlattr *tb[],
1103 * struct net_device *dev,
Jiri Pirkof6f64242014-11-28 14:34:15 +01001104 * const unsigned char *addr, u16 vid, u16 flags)
John Fastabend77162022012-04-15 06:43:56 +00001105 * Adds an FDB entry to dev for addr.
Vlad Yasevich1690be62013-02-13 12:00:18 +00001106 * int (*ndo_fdb_del)(struct ndmsg *ndm, struct nlattr *tb[],
1107 * struct net_device *dev,
Jiri Pirkof6f64242014-11-28 14:34:15 +01001108 * const unsigned char *addr, u16 vid)
John Fastabend77162022012-04-15 06:43:56 +00001109 * Deletes the FDB entry from dev coresponding to addr.
1110 * int (*ndo_fdb_dump)(struct sk_buff *skb, struct netlink_callback *cb,
Jamal Hadi Salim5d5eacb2014-07-10 07:01:58 -04001111 * struct net_device *dev, struct net_device *filter_dev,
Roopa Prabhud2976532016-08-30 21:56:45 -07001112 * int *idx)
John Fastabend77162022012-04-15 06:43:56 +00001113 * Used to add FDB entries to dump requests. Implementers should add
1114 * entries to skb and update idx with the number of entries.
John Fastabende5a55a82012-10-24 08:12:57 +00001115 *
Nicolas Dichtelad41faa2015-03-17 11:16:00 +01001116 * int (*ndo_bridge_setlink)(struct net_device *dev, struct nlmsghdr *nlh,
1117 * u16 flags)
John Fastabende5a55a82012-10-24 08:12:57 +00001118 * int (*ndo_bridge_getlink)(struct sk_buff *skb, u32 pid, u32 seq,
Nicolas Dichtel46c264d2015-04-28 18:33:49 +02001119 * struct net_device *dev, u32 filter_mask,
1120 * int nlflags)
Nicolas Dichtelad41faa2015-03-17 11:16:00 +01001121 * int (*ndo_bridge_dellink)(struct net_device *dev, struct nlmsghdr *nlh,
1122 * u16 flags);
Jiri Pirko4bf84c32012-12-27 23:49:37 +00001123 *
1124 * int (*ndo_change_carrier)(struct net_device *dev, bool new_carrier);
1125 * Called to change device carrier. Soft-devices (like dummy, team, etc)
1126 * which do not represent real hardware may define this to allow their
1127 * userspace components to manage their virtual carrier state. Devices
1128 * that determine carrier state from physical hardware properties (eg
1129 * network cables) or protocol-dependent mechanisms (eg
1130 * USB_CDC_NOTIFY_NETWORK_CONNECTION) should NOT implement this function.
Jiri Pirko66b52b02013-07-29 18:16:49 +02001131 *
1132 * int (*ndo_get_phys_port_id)(struct net_device *dev,
Jiri Pirko02637fc2014-11-28 14:34:16 +01001133 * struct netdev_phys_item_id *ppid);
Jiri Pirko66b52b02013-07-29 18:16:49 +02001134 * Called to get ID of physical port of this device. If driver does
1135 * not implement this, it is assumed that the hw is not able to have
1136 * multiple net devices on single physical port.
Joseph Gasparakis53cf52752013-09-04 02:13:38 -07001137 *
Alexander Duyck7c46a642016-06-16 12:21:00 -07001138 * void (*ndo_udp_tunnel_add)(struct net_device *dev,
1139 * struct udp_tunnel_info *ti);
1140 * Called by UDP tunnel to notify a driver about the UDP port and socket
1141 * address family that a UDP tunnel is listnening to. It is called only
1142 * when a new port starts listening. The operation is protected by the
1143 * RTNL.
1144 *
1145 * void (*ndo_udp_tunnel_del)(struct net_device *dev,
1146 * struct udp_tunnel_info *ti);
1147 * Called by UDP tunnel to notify the driver about a UDP port and socket
1148 * address family that the UDP tunnel is not listening to anymore. The
1149 * operation is protected by the RTNL.
1150 *
John Fastabenda6cc0cf2013-11-06 09:54:46 -08001151 * void* (*ndo_dfwd_add_station)(struct net_device *pdev,
1152 * struct net_device *dev)
1153 * Called by upper layer devices to accelerate switching or other
1154 * station functionality into hardware. 'pdev is the lowerdev
1155 * to use for the offload and 'dev' is the net device that will
1156 * back the offload. Returns a pointer to the private structure
1157 * the upper layer will maintain.
1158 * void (*ndo_dfwd_del_station)(struct net_device *pdev, void *priv)
1159 * Called by upper layer device to delete the station created
1160 * by 'ndo_dfwd_add_station'. 'pdev' is the net device backing
1161 * the station and priv is the structure returned by the add
1162 * operation.
John Fastabend822b3b22015-03-18 14:57:33 +02001163 * int (*ndo_set_tx_maxrate)(struct net_device *dev,
1164 * int queue_index, u32 maxrate);
1165 * Called when a user wants to set a max-rate limitation of specific
1166 * TX queue.
Nicolas Dichtela54acb32015-04-02 17:07:00 +02001167 * int (*ndo_get_iflink)(const struct net_device *dev);
1168 * Called to get the iflink value of this device.
Anuradha Karuppiahd746d702015-07-14 13:43:19 -07001169 * void (*ndo_change_proto_down)(struct net_device *dev,
Bjorn Helgaas5e82b4b2016-03-23 13:47:23 -05001170 * bool proto_down);
Anuradha Karuppiahd746d702015-07-14 13:43:19 -07001171 * This function is used to pass protocol port error state information
1172 * to the switch driver. The switch driver can react to the proto_down
1173 * by doing a phys down on the associated switch port.
Pravin B Shelarfc4099f2015-10-22 18:17:16 -07001174 * int (*ndo_fill_metadata_dst)(struct net_device *dev, struct sk_buff *skb);
1175 * This function is used to get egress tunnel information for given skb.
1176 * This is useful for retrieving outer tunnel header parameters while
1177 * sampling packet.
Paolo Abeni871b6422016-02-26 10:45:37 +01001178 * void (*ndo_set_rx_headroom)(struct net_device *dev, int needed_headroom);
1179 * This function is used to specify the headroom that the skb must
1180 * consider when allocation skb during packet reception. Setting
1181 * appropriate rx headroom value allows avoiding skb head copy on
Bjorn Helgaas5e82b4b2016-03-23 13:47:23 -05001182 * forward. Setting a negative value resets the rx headroom to the
Paolo Abeni871b6422016-02-26 10:45:37 +01001183 * default value.
Jakub Kicinskif4e63522017-11-03 13:56:16 -07001184 * int (*ndo_bpf)(struct net_device *dev, struct netdev_bpf *bpf);
Brenden Blancoa7862b42016-07-19 12:16:48 -07001185 * This function is used to set or query state related to XDP on the
Jakub Kicinskif4e63522017-11-03 13:56:16 -07001186 * netdevice and manage BPF offload. See definition of
1187 * enum bpf_netdev_command for details.
Jesper Dangaard Brouer44fa2db2018-04-17 16:46:37 +02001188 * int (*ndo_xdp_xmit)(struct net_device *dev, struct xdp_frame *xdp);
John Fastabend814abfa2017-07-17 09:27:07 -07001189 * This function is used to submit a XDP packet for transmit on a
1190 * netdevice.
John Fastabend11393cc2017-07-17 09:29:40 -07001191 * void (*ndo_xdp_flush)(struct net_device *dev);
Rami Rosenf16ded52017-08-31 13:41:40 +03001192 * This function is used to inform the driver to flush a particular
1193 * xdp tx queue. Must be called on same CPU as xdp_xmit.
Stephen Hemmingerd3147742008-11-19 21:32:24 -08001194 */
1195struct net_device_ops {
1196 int (*ndo_init)(struct net_device *dev);
1197 void (*ndo_uninit)(struct net_device *dev);
1198 int (*ndo_open)(struct net_device *dev);
1199 int (*ndo_stop)(struct net_device *dev);
Eric Dumazetcdba7562016-01-06 06:53:50 -08001200 netdev_tx_t (*ndo_start_xmit)(struct sk_buff *skb,
1201 struct net_device *dev);
1202 netdev_features_t (*ndo_features_check)(struct sk_buff *skb,
1203 struct net_device *dev,
1204 netdev_features_t features);
Stephen Hemminger00829822008-11-20 20:14:53 -08001205 u16 (*ndo_select_queue)(struct net_device *dev,
Jason Wangf663dd92014-01-10 16:18:26 +08001206 struct sk_buff *skb,
Daniel Borkmann99932d42014-02-16 15:55:20 +01001207 void *accel_priv,
1208 select_queue_fallback_t fallback);
Stephen Hemmingerd3147742008-11-19 21:32:24 -08001209 void (*ndo_change_rx_flags)(struct net_device *dev,
1210 int flags);
Stephen Hemmingerd3147742008-11-19 21:32:24 -08001211 void (*ndo_set_rx_mode)(struct net_device *dev);
Stephen Hemmingerd3147742008-11-19 21:32:24 -08001212 int (*ndo_set_mac_address)(struct net_device *dev,
1213 void *addr);
Stephen Hemmingerd3147742008-11-19 21:32:24 -08001214 int (*ndo_validate_addr)(struct net_device *dev);
Stephen Hemmingerd3147742008-11-19 21:32:24 -08001215 int (*ndo_do_ioctl)(struct net_device *dev,
1216 struct ifreq *ifr, int cmd);
Stephen Hemmingerd3147742008-11-19 21:32:24 -08001217 int (*ndo_set_config)(struct net_device *dev,
1218 struct ifmap *map);
Stephen Hemminger00829822008-11-20 20:14:53 -08001219 int (*ndo_change_mtu)(struct net_device *dev,
1220 int new_mtu);
1221 int (*ndo_neigh_setup)(struct net_device *dev,
1222 struct neigh_parms *);
Stephen Hemmingerd3147742008-11-19 21:32:24 -08001223 void (*ndo_tx_timeout) (struct net_device *dev);
1224
stephen hemmingerbc1f4472017-01-06 19:12:52 -08001225 void (*ndo_get_stats64)(struct net_device *dev,
1226 struct rtnl_link_stats64 *storage);
Or Gerlitz3df5b3c2016-11-22 23:09:54 +02001227 bool (*ndo_has_offload_stats)(const struct net_device *dev, int attr_id);
Nogah Frankel2c9d85d2016-09-16 15:05:36 +02001228 int (*ndo_get_offload_stats)(int attr_id,
1229 const struct net_device *dev,
1230 void *attr_data);
Stephen Hemmingerd3147742008-11-19 21:32:24 -08001231 struct net_device_stats* (*ndo_get_stats)(struct net_device *dev);
1232
Jiri Pirko8e586132011-12-08 19:52:37 -05001233 int (*ndo_vlan_rx_add_vid)(struct net_device *dev,
Patrick McHardy80d5c362013-04-19 02:04:28 +00001234 __be16 proto, u16 vid);
Jiri Pirko8e586132011-12-08 19:52:37 -05001235 int (*ndo_vlan_rx_kill_vid)(struct net_device *dev,
Patrick McHardy80d5c362013-04-19 02:04:28 +00001236 __be16 proto, u16 vid);
Stephen Hemmingerd3147742008-11-19 21:32:24 -08001237#ifdef CONFIG_NET_POLL_CONTROLLER
Stephen Hemmingerd3147742008-11-19 21:32:24 -08001238 void (*ndo_poll_controller)(struct net_device *dev);
Herbert Xu4247e162010-06-10 16:12:47 +00001239 int (*ndo_netpoll_setup)(struct net_device *dev,
Eric W. Biedermana8779ec2014-03-27 15:36:38 -07001240 struct netpoll_info *info);
WANG Cong0e34e932010-05-06 00:47:21 -07001241 void (*ndo_netpoll_cleanup)(struct net_device *dev);
Stephen Hemmingerd3147742008-11-19 21:32:24 -08001242#endif
Williams, Mitch A95c26df2010-02-10 01:43:46 +00001243 int (*ndo_set_vf_mac)(struct net_device *dev,
1244 int queue, u8 *mac);
1245 int (*ndo_set_vf_vlan)(struct net_device *dev,
Moshe Shemesh79aab092016-09-22 12:11:15 +03001246 int queue, u16 vlan,
1247 u8 qos, __be16 proto);
Sucheta Chakrabortyed616682014-05-22 09:59:05 -04001248 int (*ndo_set_vf_rate)(struct net_device *dev,
1249 int vf, int min_tx_rate,
1250 int max_tx_rate);
Greg Rose5f8444a2011-10-08 03:05:24 +00001251 int (*ndo_set_vf_spoofchk)(struct net_device *dev,
1252 int vf, bool setting);
Hiroshi Shimamotodd461d62015-08-28 06:57:55 +00001253 int (*ndo_set_vf_trust)(struct net_device *dev,
1254 int vf, bool setting);
Williams, Mitch A95c26df2010-02-10 01:43:46 +00001255 int (*ndo_get_vf_config)(struct net_device *dev,
1256 int vf,
1257 struct ifla_vf_info *ivf);
Rony Efraim1d8faf42013-06-13 13:19:10 +03001258 int (*ndo_set_vf_link_state)(struct net_device *dev,
1259 int vf, int link_state);
Eran Ben Elisha3b766cd2015-06-15 17:59:07 +03001260 int (*ndo_get_vf_stats)(struct net_device *dev,
1261 int vf,
1262 struct ifla_vf_stats
1263 *vf_stats);
Scott Feldman57b61082010-05-17 22:49:55 -07001264 int (*ndo_set_vf_port)(struct net_device *dev,
1265 int vf,
1266 struct nlattr *port[]);
1267 int (*ndo_get_vf_port)(struct net_device *dev,
1268 int vf, struct sk_buff *skb);
Eli Cohencc8e27c2016-03-11 22:58:34 +02001269 int (*ndo_set_vf_guid)(struct net_device *dev,
1270 int vf, u64 guid,
1271 int guid_type);
Vlad Zolotarov01a3d792015-03-30 21:35:23 +03001272 int (*ndo_set_vf_rss_query_en)(
1273 struct net_device *dev,
1274 int vf, bool setting);
John Fastabend16e5cc62016-02-16 21:16:43 -08001275 int (*ndo_setup_tc)(struct net_device *dev,
Jiri Pirko2572ac52017-08-07 10:15:17 +02001276 enum tc_setup_type type,
Jiri Pirkode4784c2017-08-07 10:15:32 +02001277 void *type_data);
Ben Hutchingsd11ead72011-11-25 14:40:26 +00001278#if IS_ENABLED(CONFIG_FCOE)
Yi Zoucb454392009-08-31 12:31:36 +00001279 int (*ndo_fcoe_enable)(struct net_device *dev);
1280 int (*ndo_fcoe_disable)(struct net_device *dev);
Yi Zou4d288d52009-02-27 14:06:59 -08001281 int (*ndo_fcoe_ddp_setup)(struct net_device *dev,
1282 u16 xid,
1283 struct scatterlist *sgl,
1284 unsigned int sgc);
1285 int (*ndo_fcoe_ddp_done)(struct net_device *dev,
1286 u16 xid);
Yi Zou6247e082011-02-01 07:22:06 +00001287 int (*ndo_fcoe_ddp_target)(struct net_device *dev,
1288 u16 xid,
1289 struct scatterlist *sgl,
1290 unsigned int sgc);
Neerav Parikh68bad942012-01-04 20:23:39 +00001291 int (*ndo_fcoe_get_hbainfo)(struct net_device *dev,
1292 struct netdev_fcoe_hbainfo *hbainfo);
Bhanu Prakash Gollapudi3c9c36bc2011-08-26 09:45:41 +00001293#endif
1294
Ben Hutchingsd11ead72011-11-25 14:40:26 +00001295#if IS_ENABLED(CONFIG_LIBFCOE)
Yi Zoudf5c7942009-10-28 18:24:35 +00001296#define NETDEV_FCOE_WWNN 0
1297#define NETDEV_FCOE_WWPN 1
1298 int (*ndo_fcoe_get_wwn)(struct net_device *dev,
1299 u64 *wwn, int type);
Yi Zou4d288d52009-02-27 14:06:59 -08001300#endif
Bhanu Prakash Gollapudi3c9c36bc2011-08-26 09:45:41 +00001301
Ben Hutchingsc4454772011-01-19 11:03:53 +00001302#ifdef CONFIG_RFS_ACCEL
1303 int (*ndo_rx_flow_steer)(struct net_device *dev,
1304 const struct sk_buff *skb,
1305 u16 rxq_index,
1306 u32 flow_id);
1307#endif
Jiri Pirkofbaec0e2011-02-13 10:15:37 +00001308 int (*ndo_add_slave)(struct net_device *dev,
David Ahern33eaf2a2017-10-04 17:48:46 -07001309 struct net_device *slave_dev,
1310 struct netlink_ext_ack *extack);
Jiri Pirkofbaec0e2011-02-13 10:15:37 +00001311 int (*ndo_del_slave)(struct net_device *dev,
1312 struct net_device *slave_dev);
Michał Mirosławc8f44af2011-11-15 15:29:55 +00001313 netdev_features_t (*ndo_fix_features)(struct net_device *dev,
1314 netdev_features_t features);
Michał Mirosław5455c692011-02-15 16:59:17 +00001315 int (*ndo_set_features)(struct net_device *dev,
Michał Mirosławc8f44af2011-11-15 15:29:55 +00001316 netdev_features_t features);
Jiri Pirko503eebc2016-07-05 11:27:37 +02001317 int (*ndo_neigh_construct)(struct net_device *dev,
1318 struct neighbour *n);
1319 void (*ndo_neigh_destroy)(struct net_device *dev,
1320 struct neighbour *n);
John Fastabend77162022012-04-15 06:43:56 +00001321
1322 int (*ndo_fdb_add)(struct ndmsg *ndm,
stephen hemmingeredc7d572012-10-01 12:32:33 +00001323 struct nlattr *tb[],
John Fastabend77162022012-04-15 06:43:56 +00001324 struct net_device *dev,
stephen hemminger6b6e2722012-09-17 10:03:26 +00001325 const unsigned char *addr,
Jiri Pirkof6f64242014-11-28 14:34:15 +01001326 u16 vid,
John Fastabend77162022012-04-15 06:43:56 +00001327 u16 flags);
1328 int (*ndo_fdb_del)(struct ndmsg *ndm,
Vlad Yasevich1690be62013-02-13 12:00:18 +00001329 struct nlattr *tb[],
John Fastabend77162022012-04-15 06:43:56 +00001330 struct net_device *dev,
Jiri Pirkof6f64242014-11-28 14:34:15 +01001331 const unsigned char *addr,
1332 u16 vid);
John Fastabend77162022012-04-15 06:43:56 +00001333 int (*ndo_fdb_dump)(struct sk_buff *skb,
1334 struct netlink_callback *cb,
1335 struct net_device *dev,
Jamal Hadi Salim5d5eacb2014-07-10 07:01:58 -04001336 struct net_device *filter_dev,
Roopa Prabhud2976532016-08-30 21:56:45 -07001337 int *idx);
John Fastabende5a55a82012-10-24 08:12:57 +00001338
1339 int (*ndo_bridge_setlink)(struct net_device *dev,
Roopa Prabhuadd511b2015-01-29 22:40:12 -08001340 struct nlmsghdr *nlh,
1341 u16 flags);
John Fastabende5a55a82012-10-24 08:12:57 +00001342 int (*ndo_bridge_getlink)(struct sk_buff *skb,
1343 u32 pid, u32 seq,
Vlad Yasevich6cbdcee2013-02-13 12:00:13 +00001344 struct net_device *dev,
Nicolas Dichtel46c264d2015-04-28 18:33:49 +02001345 u32 filter_mask,
1346 int nlflags);
Vlad Yasevich407af322013-02-13 12:00:12 +00001347 int (*ndo_bridge_dellink)(struct net_device *dev,
Roopa Prabhuadd511b2015-01-29 22:40:12 -08001348 struct nlmsghdr *nlh,
1349 u16 flags);
Jiri Pirko4bf84c32012-12-27 23:49:37 +00001350 int (*ndo_change_carrier)(struct net_device *dev,
1351 bool new_carrier);
Jiri Pirko66b52b02013-07-29 18:16:49 +02001352 int (*ndo_get_phys_port_id)(struct net_device *dev,
Jiri Pirko02637fc2014-11-28 14:34:16 +01001353 struct netdev_phys_item_id *ppid);
David Aherndb24a902015-03-17 20:23:15 -06001354 int (*ndo_get_phys_port_name)(struct net_device *dev,
1355 char *name, size_t len);
Alexander Duyck7c46a642016-06-16 12:21:00 -07001356 void (*ndo_udp_tunnel_add)(struct net_device *dev,
1357 struct udp_tunnel_info *ti);
1358 void (*ndo_udp_tunnel_del)(struct net_device *dev,
1359 struct udp_tunnel_info *ti);
John Fastabenda6cc0cf2013-11-06 09:54:46 -08001360 void* (*ndo_dfwd_add_station)(struct net_device *pdev,
1361 struct net_device *dev);
1362 void (*ndo_dfwd_del_station)(struct net_device *pdev,
1363 void *priv);
1364
Vlad Yasevich25175ba2014-05-16 17:04:54 -04001365 int (*ndo_get_lock_subclass)(struct net_device *dev);
John Fastabend822b3b22015-03-18 14:57:33 +02001366 int (*ndo_set_tx_maxrate)(struct net_device *dev,
1367 int queue_index,
1368 u32 maxrate);
Nicolas Dichtela54acb32015-04-02 17:07:00 +02001369 int (*ndo_get_iflink)(const struct net_device *dev);
Anuradha Karuppiahd746d702015-07-14 13:43:19 -07001370 int (*ndo_change_proto_down)(struct net_device *dev,
1371 bool proto_down);
Pravin B Shelarfc4099f2015-10-22 18:17:16 -07001372 int (*ndo_fill_metadata_dst)(struct net_device *dev,
1373 struct sk_buff *skb);
Paolo Abeni871b6422016-02-26 10:45:37 +01001374 void (*ndo_set_rx_headroom)(struct net_device *dev,
1375 int needed_headroom);
Jakub Kicinskif4e63522017-11-03 13:56:16 -07001376 int (*ndo_bpf)(struct net_device *dev,
1377 struct netdev_bpf *bpf);
John Fastabend814abfa2017-07-17 09:27:07 -07001378 int (*ndo_xdp_xmit)(struct net_device *dev,
Jesper Dangaard Brouer44fa2db2018-04-17 16:46:37 +02001379 struct xdp_frame *xdp);
John Fastabend11393cc2017-07-17 09:29:40 -07001380 void (*ndo_xdp_flush)(struct net_device *dev);
Stephen Hemmingerd3147742008-11-19 21:32:24 -08001381};
1382
Luis R. Rodriguez7aa98042014-02-25 17:15:13 -08001383/**
1384 * enum net_device_priv_flags - &struct net_device priv_flags
1385 *
1386 * These are the &struct net_device, they are only set internally
1387 * by drivers and used in the kernel. These flags are invisible to
Bjorn Helgaas5e82b4b2016-03-23 13:47:23 -05001388 * userspace; this means that the order of these flags can change
Luis R. Rodriguez7aa98042014-02-25 17:15:13 -08001389 * during any kernel release.
1390 *
1391 * You should have a pretty good reason to be extending these flags.
1392 *
1393 * @IFF_802_1Q_VLAN: 802.1Q VLAN device
1394 * @IFF_EBRIDGE: Ethernet bridging device
Luis R. Rodriguez7aa98042014-02-25 17:15:13 -08001395 * @IFF_BONDING: bonding master or slave
Luis R. Rodriguez7aa98042014-02-25 17:15:13 -08001396 * @IFF_ISATAP: ISATAP interface (RFC4214)
Luis R. Rodriguez7aa98042014-02-25 17:15:13 -08001397 * @IFF_WAN_HDLC: WAN HDLC device
1398 * @IFF_XMIT_DST_RELEASE: dev_hard_start_xmit() is allowed to
1399 * release skb->dst
1400 * @IFF_DONT_BRIDGE: disallow bridging this ether dev
1401 * @IFF_DISABLE_NETPOLL: disable netpoll at run-time
1402 * @IFF_MACVLAN_PORT: device used as macvlan port
1403 * @IFF_BRIDGE_PORT: device used as bridge port
1404 * @IFF_OVS_DATAPATH: device used as Open vSwitch datapath port
1405 * @IFF_TX_SKB_SHARING: The interface supports sharing skbs on transmit
1406 * @IFF_UNICAST_FLT: Supports unicast filtering
1407 * @IFF_TEAM_PORT: device used as team port
1408 * @IFF_SUPP_NOFCS: device supports sending custom FCS
1409 * @IFF_LIVE_ADDR_CHANGE: device supports hardware address
1410 * change when it's running
1411 * @IFF_MACVLAN: Macvlan device
Luis de Bethencourt6d0e24c2016-03-21 20:58:28 +00001412 * @IFF_XMIT_DST_RELEASE_PERM: IFF_XMIT_DST_RELEASE not taking into account
1413 * underlying stacked devices
David Ahern007979e2015-09-29 20:07:10 -07001414 * @IFF_L3MDEV_MASTER: device is an L3 master device
Phil Sutterfa8187c2015-08-13 19:01:06 +02001415 * @IFF_NO_QUEUE: device can run without qdisc attached
Jiri Pirko35d4e172015-08-27 09:31:20 +02001416 * @IFF_OPENVSWITCH: device is a Open vSwitch master
David Ahernfee6d4c2015-10-05 08:51:24 -07001417 * @IFF_L3MDEV_SLAVE: device is enslaved to an L3 master device
Jiri Pirkoc981e422015-12-03 12:12:06 +01001418 * @IFF_TEAM: device is a team device
Keller, Jacob Ed4ab4282016-02-08 16:05:03 -08001419 * @IFF_RXFH_CONFIGURED: device has had Rx Flow indirection table configured
Paolo Abeni871b6422016-02-26 10:45:37 +01001420 * @IFF_PHONY_HEADROOM: the headroom value is controlled by an external
1421 * entity (i.e. the master device for bridged veth)
Sabrina Dubroca3c175782016-03-11 18:07:32 +01001422 * @IFF_MACSEC: device is a MACsec device
Paolo Abenif54262502018-03-09 10:39:24 +01001423 * @IFF_NO_RX_HANDLER: device doesn't support the rx_handler hook
Luis R. Rodriguez7aa98042014-02-25 17:15:13 -08001424 */
1425enum netdev_priv_flags {
1426 IFF_802_1Q_VLAN = 1<<0,
1427 IFF_EBRIDGE = 1<<1,
Jiri Pirko0dc15492015-08-27 09:31:21 +02001428 IFF_BONDING = 1<<2,
1429 IFF_ISATAP = 1<<3,
1430 IFF_WAN_HDLC = 1<<4,
1431 IFF_XMIT_DST_RELEASE = 1<<5,
1432 IFF_DONT_BRIDGE = 1<<6,
1433 IFF_DISABLE_NETPOLL = 1<<7,
1434 IFF_MACVLAN_PORT = 1<<8,
1435 IFF_BRIDGE_PORT = 1<<9,
1436 IFF_OVS_DATAPATH = 1<<10,
1437 IFF_TX_SKB_SHARING = 1<<11,
1438 IFF_UNICAST_FLT = 1<<12,
1439 IFF_TEAM_PORT = 1<<13,
1440 IFF_SUPP_NOFCS = 1<<14,
1441 IFF_LIVE_ADDR_CHANGE = 1<<15,
1442 IFF_MACVLAN = 1<<16,
1443 IFF_XMIT_DST_RELEASE_PERM = 1<<17,
Paolo Abeni1ec54cb2018-03-06 10:56:31 +01001444 IFF_L3MDEV_MASTER = 1<<18,
1445 IFF_NO_QUEUE = 1<<19,
1446 IFF_OPENVSWITCH = 1<<20,
1447 IFF_L3MDEV_SLAVE = 1<<21,
1448 IFF_TEAM = 1<<22,
1449 IFF_RXFH_CONFIGURED = 1<<23,
1450 IFF_PHONY_HEADROOM = 1<<24,
1451 IFF_MACSEC = 1<<25,
Paolo Abenif54262502018-03-09 10:39:24 +01001452 IFF_NO_RX_HANDLER = 1<<26,
Luis R. Rodriguez7aa98042014-02-25 17:15:13 -08001453};
1454
1455#define IFF_802_1Q_VLAN IFF_802_1Q_VLAN
1456#define IFF_EBRIDGE IFF_EBRIDGE
Luis R. Rodriguez7aa98042014-02-25 17:15:13 -08001457#define IFF_BONDING IFF_BONDING
Luis R. Rodriguez7aa98042014-02-25 17:15:13 -08001458#define IFF_ISATAP IFF_ISATAP
Luis R. Rodriguez7aa98042014-02-25 17:15:13 -08001459#define IFF_WAN_HDLC IFF_WAN_HDLC
1460#define IFF_XMIT_DST_RELEASE IFF_XMIT_DST_RELEASE
1461#define IFF_DONT_BRIDGE IFF_DONT_BRIDGE
1462#define IFF_DISABLE_NETPOLL IFF_DISABLE_NETPOLL
1463#define IFF_MACVLAN_PORT IFF_MACVLAN_PORT
1464#define IFF_BRIDGE_PORT IFF_BRIDGE_PORT
1465#define IFF_OVS_DATAPATH IFF_OVS_DATAPATH
1466#define IFF_TX_SKB_SHARING IFF_TX_SKB_SHARING
1467#define IFF_UNICAST_FLT IFF_UNICAST_FLT
1468#define IFF_TEAM_PORT IFF_TEAM_PORT
1469#define IFF_SUPP_NOFCS IFF_SUPP_NOFCS
1470#define IFF_LIVE_ADDR_CHANGE IFF_LIVE_ADDR_CHANGE
1471#define IFF_MACVLAN IFF_MACVLAN
Eric Dumazet02875872014-10-05 18:38:35 -07001472#define IFF_XMIT_DST_RELEASE_PERM IFF_XMIT_DST_RELEASE_PERM
David Ahern007979e2015-09-29 20:07:10 -07001473#define IFF_L3MDEV_MASTER IFF_L3MDEV_MASTER
Phil Sutterfa8187c2015-08-13 19:01:06 +02001474#define IFF_NO_QUEUE IFF_NO_QUEUE
Jiri Pirko35d4e172015-08-27 09:31:20 +02001475#define IFF_OPENVSWITCH IFF_OPENVSWITCH
Jiri Pirko8f253482015-11-04 14:59:06 +01001476#define IFF_L3MDEV_SLAVE IFF_L3MDEV_SLAVE
Jiri Pirkoc981e422015-12-03 12:12:06 +01001477#define IFF_TEAM IFF_TEAM
Keller, Jacob Ed4ab4282016-02-08 16:05:03 -08001478#define IFF_RXFH_CONFIGURED IFF_RXFH_CONFIGURED
Sabrina Dubroca3c175782016-03-11 18:07:32 +01001479#define IFF_MACSEC IFF_MACSEC
Paolo Abenif54262502018-03-09 10:39:24 +01001480#define IFF_NO_RX_HANDLER IFF_NO_RX_HANDLER
Luis R. Rodriguez7aa98042014-02-25 17:15:13 -08001481
Karoly Kemeny536721b2014-07-30 20:27:36 +02001482/**
1483 * struct net_device - The DEVICE structure.
Mauro Carvalho Chehabd6519832017-05-12 09:35:46 -03001484 *
1485 * Actually, this whole structure is a big mistake. It mixes I/O
1486 * data with strictly "high-level" data, and it has to know about
1487 * almost every data structure used in the INET module.
Karoly Kemeny536721b2014-07-30 20:27:36 +02001488 *
1489 * @name: This is the first field of the "visible" part of this structure
1490 * (i.e. as seen by users in the "Space.c" file). It is the name
Mauro Carvalho Chehabd6519832017-05-12 09:35:46 -03001491 * of the interface.
Karoly Kemeny536721b2014-07-30 20:27:36 +02001492 *
1493 * @name_hlist: Device name hash chain, please keep it close to name[]
1494 * @ifalias: SNMP alias
1495 * @mem_end: Shared memory end
1496 * @mem_start: Shared memory start
1497 * @base_addr: Device I/O address
1498 * @irq: Device IRQ number
1499 *
1500 * @state: Generic network queuing layer state, see netdev_state_t
1501 * @dev_list: The global list of network devices
Bjorn Helgaas5e82b4b2016-03-23 13:47:23 -05001502 * @napi_list: List entry used for polling NAPI devices
1503 * @unreg_list: List entry when we are unregistering the
1504 * device; see the function unregister_netdev
1505 * @close_list: List entry used when we are closing the device
Benjamin Poirier62d885fe2016-03-21 14:08:28 -07001506 * @ptype_all: Device-specific packet handlers for all protocols
1507 * @ptype_specific: Device-specific, protocol-specific packet handlers
Karoly Kemeny536721b2014-07-30 20:27:36 +02001508 *
1509 * @adj_list: Directly linked devices, like slaves for bonding
Karoly Kemeny536721b2014-07-30 20:27:36 +02001510 * @features: Currently active device features
1511 * @hw_features: User-changeable features
1512 *
1513 * @wanted_features: User-requested features
1514 * @vlan_features: Mask of features inheritable by VLAN devices
1515 *
1516 * @hw_enc_features: Mask of features inherited by encapsulating devices
1517 * This field indicates what encapsulation
1518 * offloads the hardware is capable of doing,
1519 * and drivers will need to set them appropriately.
1520 *
1521 * @mpls_features: Mask of features inheritable by MPLS
1522 *
1523 * @ifindex: interface index
Bjorn Helgaas5e82b4b2016-03-23 13:47:23 -05001524 * @group: The group the device belongs to
Karoly Kemeny536721b2014-07-30 20:27:36 +02001525 *
1526 * @stats: Statistics struct, which was left as a legacy, use
1527 * rtnl_link_stats64 instead
1528 *
1529 * @rx_dropped: Dropped packets by core network,
1530 * do not use this in drivers
1531 * @tx_dropped: Dropped packets by core network,
1532 * do not use this in drivers
Jarod Wilson6e7333d2016-02-01 18:51:05 -05001533 * @rx_nohandler: nohandler dropped packets by core network on
1534 * inactive devices, do not use this in drivers
Florian Fainelli9e55e5d2018-01-22 19:14:25 -08001535 * @carrier_up_count: Number of times the carrier has been up
1536 * @carrier_down_count: Number of times the carrier has been down
Karoly Kemeny536721b2014-07-30 20:27:36 +02001537 *
Karoly Kemeny536721b2014-07-30 20:27:36 +02001538 * @wireless_handlers: List of functions to handle Wireless Extensions,
1539 * instead of ioctl,
1540 * see <net/iw_handler.h> for details.
1541 * @wireless_data: Instance data managed by the core of wireless extensions
1542 *
1543 * @netdev_ops: Includes several pointers to callbacks,
1544 * if one wants to override the ndo_*() functions
1545 * @ethtool_ops: Management operations
Alexander Aringf997c552016-06-15 21:20:23 +02001546 * @ndisc_ops: Includes callbacks for different IPv6 neighbour
1547 * discovery handling. Necessary for e.g. 6LoWPAN.
Eric W. Biedermand4760592015-03-02 00:11:09 -06001548 * @header_ops: Includes callbacks for creating,parsing,caching,etc
Karoly Kemeny536721b2014-07-30 20:27:36 +02001549 * of Layer 2 headers.
1550 *
1551 * @flags: Interface flags (a la BSD)
1552 * @priv_flags: Like 'flags' but invisible to userspace,
1553 * see if.h for the definitions
1554 * @gflags: Global flags ( kept as legacy )
1555 * @padded: How much padding added by alloc_netdev()
1556 * @operstate: RFC2863 operstate
1557 * @link_mode: Mapping policy to operstate
1558 * @if_port: Selectable AUI, TP, ...
1559 * @dma: DMA channel
1560 * @mtu: Interface MTU value
Jarod Wilson61e84622016-10-07 22:04:33 -04001561 * @min_mtu: Interface Minimum MTU value
1562 * @max_mtu: Interface Maximum MTU value
Karoly Kemeny536721b2014-07-30 20:27:36 +02001563 * @type: Interface hardware type
Willem de Bruijn2793a232016-03-09 21:58:32 -05001564 * @hard_header_len: Maximum hardware header length.
Willem de Bruijn217e6fa2017-02-07 15:57:20 -05001565 * @min_header_len: Minimum hardware header length
Karoly Kemeny536721b2014-07-30 20:27:36 +02001566 *
1567 * @needed_headroom: Extra headroom the hardware may need, but not in all
1568 * cases can this be guaranteed
1569 * @needed_tailroom: Extra tailroom the hardware may need, but not in all
1570 * cases can this be guaranteed. Some cases also use
1571 * LL_MAX_HEADER instead to allocate the skb
1572 *
1573 * interface address info:
1574 *
1575 * @perm_addr: Permanent hw address
1576 * @addr_assign_type: Hw address assignment type
1577 * @addr_len: Hardware address length
Alexander Aring8626a0c2016-06-15 21:20:16 +02001578 * @neigh_priv_len: Used in neigh_alloc()
Karoly Kemeny536721b2014-07-30 20:27:36 +02001579 * @dev_id: Used to differentiate devices that share
1580 * the same link layer address
1581 * @dev_port: Used to differentiate devices that share
1582 * the same function
1583 * @addr_list_lock: XXX: need comments on this one
Bjorn Helgaas5e82b4b2016-03-23 13:47:23 -05001584 * @uc_promisc: Counter that indicates promiscuous mode
Karoly Kemeny536721b2014-07-30 20:27:36 +02001585 * has been enabled due to the need to listen to
1586 * additional unicast addresses in a device that
1587 * does not implement ndo_set_rx_mode()
Thomas Graf14ffbbb2015-04-10 15:52:38 +02001588 * @uc: unicast mac addresses
1589 * @mc: multicast mac addresses
1590 * @dev_addrs: list of device hw addresses
1591 * @queues_kset: Group of all Kobjects in the Tx and RX queues
Bjorn Helgaas5e82b4b2016-03-23 13:47:23 -05001592 * @promiscuity: Number of times the NIC is told to work in
1593 * promiscuous mode; if it becomes 0 the NIC will
1594 * exit promiscuous mode
Karoly Kemeny536721b2014-07-30 20:27:36 +02001595 * @allmulti: Counter, enables or disables allmulticast mode
1596 *
1597 * @vlan_info: VLAN info
1598 * @dsa_ptr: dsa specific data
1599 * @tipc_ptr: TIPC specific data
1600 * @atalk_ptr: AppleTalk link
1601 * @ip_ptr: IPv4 specific data
1602 * @dn_ptr: DECnet specific data
1603 * @ip6_ptr: IPv6 specific data
1604 * @ax25_ptr: AX.25 specific data
1605 * @ieee80211_ptr: IEEE 802.11 specific data, assign before registering
1606 *
Karoly Kemeny536721b2014-07-30 20:27:36 +02001607 * @dev_addr: Hw address (before bcast,
1608 * because most packets are unicast)
1609 *
1610 * @_rx: Array of RX queues
1611 * @num_rx_queues: Number of RX queues
1612 * allocated at register_netdev() time
1613 * @real_num_rx_queues: Number of RX queues currently active in device
1614 *
1615 * @rx_handler: handler for received packets
1616 * @rx_handler_data: XXX: need comments on this one
Jiri Pirko46209402017-11-03 11:46:25 +01001617 * @miniq_ingress: ingress/clsact qdisc specific data for
1618 * ingress processing
Karoly Kemeny536721b2014-07-30 20:27:36 +02001619 * @ingress_queue: XXX: need comments on this one
1620 * @broadcast: hw bcast address
1621 *
Thomas Graf14ffbbb2015-04-10 15:52:38 +02001622 * @rx_cpu_rmap: CPU reverse-mapping for RX completion interrupts,
1623 * indexed by RX queue number. Assigned by driver.
1624 * This must only be set if the ndo_rx_flow_steer
1625 * operation is defined
1626 * @index_hlist: Device index hash chain
1627 *
Karoly Kemeny536721b2014-07-30 20:27:36 +02001628 * @_tx: Array of TX queues
1629 * @num_tx_queues: Number of TX queues allocated at alloc_netdev_mq() time
1630 * @real_num_tx_queues: Number of TX queues currently active in device
1631 * @qdisc: Root qdisc from userspace point of view
1632 * @tx_queue_len: Max frames per queue allowed
1633 * @tx_global_lock: XXX: need comments on this one
1634 *
1635 * @xps_maps: XXX: need comments on this one
Jiri Pirko46209402017-11-03 11:46:25 +01001636 * @miniq_egress: clsact qdisc specific data for
1637 * egress processing
Karoly Kemeny536721b2014-07-30 20:27:36 +02001638 * @watchdog_timeo: Represents the timeout that is used by
Bjorn Helgaas5e82b4b2016-03-23 13:47:23 -05001639 * the watchdog (see dev_watchdog())
Karoly Kemeny536721b2014-07-30 20:27:36 +02001640 * @watchdog_timer: List of timers
1641 *
1642 * @pcpu_refcnt: Number of references to this device
1643 * @todo_list: Delayed register/unregister
Karoly Kemeny536721b2014-07-30 20:27:36 +02001644 * @link_watch_list: XXX: need comments on this one
1645 *
1646 * @reg_state: Register/unregister state machine
1647 * @dismantle: Device is going to be freed
1648 * @rtnl_link_state: This enum represents the phases of creating
1649 * a new link
1650 *
David S. Millercf124db2017-05-08 12:52:56 -04001651 * @needs_free_netdev: Should unregister perform free_netdev?
1652 * @priv_destructor: Called from unregister
Karoly Kemeny536721b2014-07-30 20:27:36 +02001653 * @npinfo: XXX: need comments on this one
1654 * @nd_net: Network namespace this network device is inside
1655 *
1656 * @ml_priv: Mid-layer private
1657 * @lstats: Loopback statistics
1658 * @tstats: Tunnel statistics
1659 * @dstats: Dummy statistics
1660 * @vstats: Virtual ethernet statistics
1661 *
1662 * @garp_port: GARP
1663 * @mrp_port: MRP
1664 *
1665 * @dev: Class/net/name entry
1666 * @sysfs_groups: Space for optional device, statistics and wireless
1667 * sysfs groups
1668 *
1669 * @sysfs_rx_queue_group: Space for optional per-rx queue attributes
1670 * @rtnl_link_ops: Rtnl_link_ops
1671 *
1672 * @gso_max_size: Maximum size of generic segmentation offload
1673 * @gso_max_segs: Maximum number of segments that can be passed to the
1674 * NIC for GSO
1675 *
1676 * @dcbnl_ops: Data Center Bridging netlink ops
1677 * @num_tc: Number of traffic classes in the net device
1678 * @tc_to_txq: XXX: need comments on this one
Randy Dunlap920c1cd2016-11-21 18:28:36 -08001679 * @prio_tc_map: XXX: need comments on this one
Karoly Kemeny536721b2014-07-30 20:27:36 +02001680 *
1681 * @fcoe_ddp_xid: Max exchange id for FCoE LRO by ddp
1682 *
1683 * @priomap: XXX: need comments on this one
1684 * @phydev: Physical device may attach itself
1685 * for hardware timestamping
Russell Kinge679c9c2018-03-28 15:44:16 -07001686 * @sfp_bus: attached &struct sfp_bus structure.
Karoly Kemeny536721b2014-07-30 20:27:36 +02001687 *
Eric Dumazet123b3652016-06-08 07:22:49 -07001688 * @qdisc_tx_busylock: lockdep class annotating Qdisc->busylock spinlock
1689 * @qdisc_running_key: lockdep class annotating Qdisc->running seqcount
Karoly Kemeny536721b2014-07-30 20:27:36 +02001690 *
Anuradha Karuppiahd746d702015-07-14 13:43:19 -07001691 * @proto_down: protocol port state information can be sent to the
1692 * switch driver and used to set the phys state of the
1693 * switch port.
1694 *
Linus Torvalds1da177e2005-04-16 15:20:36 -07001695 * FIXME: cleanup struct net_device such that network protocol info
1696 * moves out.
1697 */
1698
Eric Dumazetd94d9fe2009-11-04 09:50:58 -08001699struct net_device {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001700 char name[IFNAMSIZ];
Eric Dumazet9356b8f2005-09-27 15:23:16 -07001701 struct hlist_node name_hlist;
Florian Westphal6c557002017-10-02 23:50:05 +02001702 struct dev_ifalias __rcu *ifalias;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001703 /*
1704 * I/O specific fields
1705 * FIXME: Merge these and struct ifmap into one
1706 */
Karoly Kemeny536721b2014-07-30 20:27:36 +02001707 unsigned long mem_end;
1708 unsigned long mem_start;
1709 unsigned long base_addr;
1710 int irq;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001711
1712 /*
Karoly Kemeny536721b2014-07-30 20:27:36 +02001713 * Some hardware also needs these fields (state,dev_list,
1714 * napi_list,unreg_list,close_list) but they are not
Linus Torvalds1da177e2005-04-16 15:20:36 -07001715 * part of the usual set specified in Space.c.
1716 */
1717
Linus Torvalds1da177e2005-04-16 15:20:36 -07001718 unsigned long state;
1719
Pavel Emelianov7562f872007-05-03 15:13:45 -07001720 struct list_head dev_list;
Stephen Hemmingerbea33482007-10-03 16:41:36 -07001721 struct list_head napi_list;
Eric Dumazet44a08732009-10-27 07:03:04 +00001722 struct list_head unreg_list;
Eric W. Biederman5cde2822013-10-05 19:26:05 -07001723 struct list_head close_list;
Salam Noureddine7866a622015-01-27 11:35:48 -08001724 struct list_head ptype_all;
1725 struct list_head ptype_specific;
Veaceslav Falico2f268f12013-09-25 09:20:07 +02001726
Veaceslav Falico2f268f12013-09-25 09:20:07 +02001727 struct {
1728 struct list_head upper;
1729 struct list_head lower;
1730 } adj_list;
1731
Michał Mirosławc8f44af2011-11-15 15:29:55 +00001732 netdev_features_t features;
Michał Mirosławc8f44af2011-11-15 15:29:55 +00001733 netdev_features_t hw_features;
Michał Mirosławc8f44af2011-11-15 15:29:55 +00001734 netdev_features_t wanted_features;
Michał Mirosławc8f44af2011-11-15 15:29:55 +00001735 netdev_features_t vlan_features;
Joseph Gasparakis6a674e92012-12-07 14:14:14 +00001736 netdev_features_t hw_enc_features;
Simon Horman0d89d202013-05-23 21:02:52 +00001737 netdev_features_t mpls_features;
Alexander Duyck802ab552016-04-10 21:45:03 -04001738 netdev_features_t gso_partial_features;
Michał Mirosław04ed3e72011-01-24 15:32:47 -08001739
Linus Torvalds1da177e2005-04-16 15:20:36 -07001740 int ifindex;
Nicolas Dichtel7a66bbc2015-04-02 17:07:09 +02001741 int group;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001742
Rusty Russellc45d2862007-03-28 14:29:08 -07001743 struct net_device_stats stats;
Eric Dumazet015f0682014-03-27 08:45:56 -07001744
Eric Dumazet015f0682014-03-27 08:45:56 -07001745 atomic_long_t rx_dropped;
1746 atomic_long_t tx_dropped;
Jarod Wilson6e7333d2016-02-01 18:51:05 -05001747 atomic_long_t rx_nohandler;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001748
David Decotignyb2d3bcf2018-01-18 09:59:13 -08001749 /* Stats to monitor link on/off, flapping */
1750 atomic_t carrier_up_count;
1751 atomic_t carrier_down_count;
1752
Johannes Bergb86e0282007-04-26 20:48:23 -07001753#ifdef CONFIG_WIRELESS_EXT
Bjorn Helgaas5e82b4b2016-03-23 13:47:23 -05001754 const struct iw_handler_def *wireless_handlers;
1755 struct iw_public_data *wireless_data;
Johannes Bergb86e0282007-04-26 20:48:23 -07001756#endif
Stephen Hemmingerd3147742008-11-19 21:32:24 -08001757 const struct net_device_ops *netdev_ops;
Stephen Hemminger76fd8592006-09-08 11:16:13 -07001758 const struct ethtool_ops *ethtool_ops;
Scott Feldman41706042015-03-15 21:07:14 -07001759#ifdef CONFIG_NET_SWITCHDEV
Jiri Pirko9d47c0a2015-05-10 09:47:47 -07001760 const struct switchdev_ops *switchdev_ops;
Scott Feldman41706042015-03-15 21:07:14 -07001761#endif
David Ahern1b69c6d2015-09-29 20:07:11 -07001762#ifdef CONFIG_NET_L3_MASTER_DEV
1763 const struct l3mdev_ops *l3mdev_ops;
1764#endif
Alexander Aringf997c552016-06-15 21:20:23 +02001765#if IS_ENABLED(CONFIG_IPV6)
1766 const struct ndisc_ops *ndisc_ops;
1767#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07001768
Shannon Nelson9cb0d212017-12-19 15:35:49 -08001769#ifdef CONFIG_XFRM_OFFLOAD
Steffen Klassertd77e38e2017-04-14 10:06:10 +02001770 const struct xfrmdev_ops *xfrmdev_ops;
1771#endif
1772
Ilya Lesokhina5c37c62018-04-30 10:16:13 +03001773#if IS_ENABLED(CONFIG_TLS_DEVICE)
1774 const struct tlsdev_ops *tlsdev_ops;
1775#endif
1776
Stephen Hemminger3b04ddd2007-10-09 01:40:57 -07001777 const struct header_ops *header_ops;
1778
Karoly Kemeny536721b2014-07-30 20:27:36 +02001779 unsigned int flags;
1780 unsigned int priv_flags;
1781
Linus Torvalds1da177e2005-04-16 15:20:36 -07001782 unsigned short gflags;
Karoly Kemeny536721b2014-07-30 20:27:36 +02001783 unsigned short padded;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001784
Karoly Kemeny536721b2014-07-30 20:27:36 +02001785 unsigned char operstate;
1786 unsigned char link_mode;
Stefan Rompfb00055a2006-03-20 17:09:11 -08001787
Karoly Kemeny536721b2014-07-30 20:27:36 +02001788 unsigned char if_port;
1789 unsigned char dma;
Joe Perchesbdc220d2011-05-09 17:42:46 +00001790
Karoly Kemeny536721b2014-07-30 20:27:36 +02001791 unsigned int mtu;
Jarod Wilson61e84622016-10-07 22:04:33 -04001792 unsigned int min_mtu;
1793 unsigned int max_mtu;
Karoly Kemeny536721b2014-07-30 20:27:36 +02001794 unsigned short type;
1795 unsigned short hard_header_len;
Alexey Dobriyand92be7a2017-04-10 11:25:26 +03001796 unsigned char min_header_len;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001797
Johannes Bergf5184d22008-05-12 20:48:31 -07001798 unsigned short needed_headroom;
1799 unsigned short needed_tailroom;
1800
Linus Torvalds1da177e2005-04-16 15:20:36 -07001801 /* Interface address info. */
Karoly Kemeny536721b2014-07-30 20:27:36 +02001802 unsigned char perm_addr[MAX_ADDR_LEN];
1803 unsigned char addr_assign_type;
1804 unsigned char addr_len;
Sebastian Siewiora0a96632013-12-12 10:15:59 +01001805 unsigned short neigh_priv_len;
Karoly Kemeny536721b2014-07-30 20:27:36 +02001806 unsigned short dev_id;
1807 unsigned short dev_port;
Jiri Pirkoccffad252009-05-22 23:22:17 +00001808 spinlock_t addr_list_lock;
Thomas Graf14ffbbb2015-04-10 15:52:38 +02001809 unsigned char name_assign_type;
1810 bool uc_promisc;
Karoly Kemeny536721b2014-07-30 20:27:36 +02001811 struct netdev_hw_addr_list uc;
1812 struct netdev_hw_addr_list mc;
1813 struct netdev_hw_addr_list dev_addrs;
1814
Eric Dumazet4c3d5e72013-03-30 06:31:03 +00001815#ifdef CONFIG_SYSFS
1816 struct kset *queues_kset;
1817#endif
Wang Chen9d45abe2008-06-17 21:12:48 -07001818 unsigned int promiscuity;
1819 unsigned int allmulti;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001820
Linus Torvalds1da177e2005-04-16 15:20:36 -07001821
Bjorn Helgaas5e82b4b2016-03-23 13:47:23 -05001822 /* Protocol-specific pointers */
Jesse Gross65ac6a52010-10-20 13:56:05 +00001823
Ben Hutchingsd11ead72011-11-25 14:40:26 +00001824#if IS_ENABLED(CONFIG_VLAN_8021Q)
Karoly Kemeny536721b2014-07-30 20:27:36 +02001825 struct vlan_info __rcu *vlan_info;
Jesse Gross65ac6a52010-10-20 13:56:05 +00001826#endif
Ben Hutchings34a430d2011-11-25 14:38:38 +00001827#if IS_ENABLED(CONFIG_NET_DSA)
Vivien Didelot2f657a62017-09-29 17:19:20 -04001828 struct dsa_port *dsa_ptr;
Lennert Buytenhek91da11f2008-10-07 13:44:02 +00001829#endif
Ying Xue37cb0622013-12-10 20:45:41 -08001830#if IS_ENABLED(CONFIG_TIPC)
Karoly Kemeny536721b2014-07-30 20:27:36 +02001831 struct tipc_bearer __rcu *tipc_ptr;
Ying Xue37cb0622013-12-10 20:45:41 -08001832#endif
David Ahern89e58142018-02-13 08:52:02 -08001833#if IS_ENABLED(CONFIG_IRDA) || IS_ENABLED(CONFIG_ATALK)
Karoly Kemeny536721b2014-07-30 20:27:36 +02001834 void *atalk_ptr;
David Ahern89e58142018-02-13 08:52:02 -08001835#endif
Karoly Kemeny536721b2014-07-30 20:27:36 +02001836 struct in_device __rcu *ip_ptr;
David Ahern330c7272018-02-13 08:52:00 -08001837#if IS_ENABLED(CONFIG_DECNET)
Karoly Kemeny536721b2014-07-30 20:27:36 +02001838 struct dn_dev __rcu *dn_ptr;
David Ahern330c7272018-02-13 08:52:00 -08001839#endif
Karoly Kemeny536721b2014-07-30 20:27:36 +02001840 struct inet6_dev __rcu *ip6_ptr;
David Ahern19ff13f2018-02-13 08:52:01 -08001841#if IS_ENABLED(CONFIG_AX25)
Karoly Kemeny536721b2014-07-30 20:27:36 +02001842 void *ax25_ptr;
David Ahern19ff13f2018-02-13 08:52:01 -08001843#endif
Karoly Kemeny536721b2014-07-30 20:27:36 +02001844 struct wireless_dev *ieee80211_ptr;
Alexander Aring98a18b62014-11-02 06:44:54 +01001845 struct wpan_dev *ieee802154_ptr;
Robert Shearman03c57742015-04-22 11:14:37 +01001846#if IS_ENABLED(CONFIG_MPLS_ROUTING)
1847 struct mpls_dev __rcu *mpls_ptr;
1848#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07001849
Eric Dumazet9356b8f2005-09-27 15:23:16 -07001850/*
Eric Dumazetcd135392010-09-16 02:58:13 +00001851 * Cache lines mostly used on receive path (including eth_type_trans())
Eric Dumazet9356b8f2005-09-27 15:23:16 -07001852 */
Eric Dumazet9356b8f2005-09-27 15:23:16 -07001853 /* Interface address info used in eth_type_trans() */
Karoly Kemeny536721b2014-07-30 20:27:36 +02001854 unsigned char *dev_addr;
Jiri Pirkof001fde2009-05-05 02:48:28 +00001855
Tom Herbert0a9627f2010-03-16 08:03:29 +00001856 struct netdev_rx_queue *_rx;
Tom Herbert0a9627f2010-03-16 08:03:29 +00001857 unsigned int num_rx_queues;
Ben Hutchings62fe0b42010-09-27 08:24:33 +00001858 unsigned int real_num_rx_queues;
Tom Herbert0a9627f2010-03-16 08:03:29 +00001859
Eric Dumazet7acedaf2017-04-25 11:36:52 -07001860 struct bpf_prog __rcu *xdp_prog;
Eric Dumazet3b47d302014-11-06 21:09:44 -08001861 unsigned long gro_flush_timeout;
stephen hemminger61391cd2010-11-15 06:38:12 +00001862 rx_handler_func_t __rcu *rx_handler;
1863 void __rcu *rx_handler_data;
David S. Millere8a04642008-07-17 00:34:19 -07001864
Daniel Borkmann4cda01e2015-05-11 19:28:49 +02001865#ifdef CONFIG_NET_CLS_ACT
Jiri Pirko46209402017-11-03 11:46:25 +01001866 struct mini_Qdisc __rcu *miniq_ingress;
Daniel Borkmannd2788d32015-05-09 22:51:32 +02001867#endif
Eric Dumazet24824a02010-10-02 06:11:55 +00001868 struct netdev_queue __rcu *ingress_queue;
Pablo Neirae687ad62015-05-13 18:19:38 +02001869#ifdef CONFIG_NETFILTER_INGRESS
Aaron Conole960632e2017-08-24 00:08:32 +02001870 struct nf_hook_entries __rcu *nf_hooks_ingress;
Pablo Neirae687ad62015-05-13 18:19:38 +02001871#endif
Daniel Borkmannd2788d32015-05-09 22:51:32 +02001872
Karoly Kemeny536721b2014-07-30 20:27:36 +02001873 unsigned char broadcast[MAX_ADDR_LEN];
Thomas Graf14ffbbb2015-04-10 15:52:38 +02001874#ifdef CONFIG_RFS_ACCEL
1875 struct cpu_rmap *rx_cpu_rmap;
1876#endif
1877 struct hlist_node index_hlist;
Eric Dumazetcd135392010-09-16 02:58:13 +00001878
1879/*
1880 * Cache lines mostly used on transmit path
1881 */
David S. Millere8a04642008-07-17 00:34:19 -07001882 struct netdev_queue *_tx ____cacheline_aligned_in_smp;
1883 unsigned int num_tx_queues;
David S. Millerfd2ea0a2008-07-17 01:56:23 -07001884 unsigned int real_num_tx_queues;
Patrick McHardyaf356af2009-09-04 06:41:18 +00001885 struct Qdisc *qdisc;
Jiri Kosina59cc1f62016-08-10 11:05:15 +02001886#ifdef CONFIG_NET_SCHED
1887 DECLARE_HASHTABLE (qdisc_hash, 4);
1888#endif
Alexey Dobriyan0cd29502017-05-17 13:30:44 +03001889 unsigned int tx_queue_len;
David S. Millerc3f26a22008-07-31 16:58:50 -07001890 spinlock_t tx_global_lock;
Thomas Graf14ffbbb2015-04-10 15:52:38 +02001891 int watchdog_timeo;
Eric Dumazetcd135392010-09-16 02:58:13 +00001892
Tom Herbertbf264142010-11-26 08:36:09 +00001893#ifdef CONFIG_XPS
Eric Dumazeta4177862010-11-28 21:43:02 +00001894 struct xps_dev_maps __rcu *xps_maps;
Tom Herbertbf264142010-11-26 08:36:09 +00001895#endif
Daniel Borkmann1f211a12016-01-07 22:29:47 +01001896#ifdef CONFIG_NET_CLS_ACT
Jiri Pirko46209402017-11-03 11:46:25 +01001897 struct mini_Qdisc __rcu *miniq_egress;
Daniel Borkmann1f211a12016-01-07 22:29:47 +01001898#endif
Scott Feldman0c4f6912015-07-18 18:24:48 -07001899
Eric Dumazet9356b8f2005-09-27 15:23:16 -07001900 /* These may be needed for future network-power-down code. */
Eric Dumazet9356b8f2005-09-27 15:23:16 -07001901 struct timer_list watchdog_timer;
1902
Eric Dumazet29b44332010-10-11 10:22:12 +00001903 int __percpu *pcpu_refcnt;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001904 struct list_head todo_list;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001905
Eric Dumazete014deb2009-11-17 05:59:21 +00001906 struct list_head link_watch_list;
Herbert Xu572a1032007-05-08 18:34:17 -07001907
Linus Torvalds1da177e2005-04-16 15:20:36 -07001908 enum { NETREG_UNINITIALIZED=0,
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07001909 NETREG_REGISTERED, /* completed register_netdevice */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001910 NETREG_UNREGISTERING, /* called unregister_netdevice */
1911 NETREG_UNREGISTERED, /* completed unregister todo */
1912 NETREG_RELEASED, /* called free_netdev */
Benjamin Herrenschmidt937f1ba2009-01-14 21:05:05 -08001913 NETREG_DUMMY, /* dummy device for NAPI poll */
Eric Dumazet449f4542011-05-19 12:24:16 +00001914 } reg_state:8;
1915
Karoly Kemeny536721b2014-07-30 20:27:36 +02001916 bool dismantle;
Patrick McHardya2835762010-02-26 06:34:51 +00001917
1918 enum {
1919 RTNL_LINK_INITIALIZED,
1920 RTNL_LINK_INITIALIZING,
1921 } rtnl_link_state:16;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001922
David S. Millercf124db2017-05-08 12:52:56 -04001923 bool needs_free_netdev;
1924 void (*priv_destructor)(struct net_device *dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001925
Linus Torvalds1da177e2005-04-16 15:20:36 -07001926#ifdef CONFIG_NETPOLL
Cong Wang5fbee842013-01-22 21:29:39 +00001927 struct netpoll_info __rcu *npinfo;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001928#endif
David S. Millereae792b2008-07-15 03:03:33 -07001929
Eric W. Biederman0c5c9fb2015-03-11 23:06:44 -05001930 possible_net_t nd_net;
Eric W. Biederman4a1c5372007-09-12 11:56:32 +02001931
David S. Miller49517042008-05-12 03:29:11 -07001932 /* mid-layer private */
Eric Dumazeta7855c72010-09-23 23:51:51 +00001933 union {
Karoly Kemeny536721b2014-07-30 20:27:36 +02001934 void *ml_priv;
1935 struct pcpu_lstats __percpu *lstats;
Li RongQing8f849852014-01-04 13:57:59 +08001936 struct pcpu_sw_netstats __percpu *tstats;
Karoly Kemeny536721b2014-07-30 20:27:36 +02001937 struct pcpu_dstats __percpu *dstats;
1938 struct pcpu_vstats __percpu *vstats;
Eric Dumazeta7855c72010-09-23 23:51:51 +00001939 };
Karoly Kemeny536721b2014-07-30 20:27:36 +02001940
Tobias Klauserfb585b42017-02-10 16:43:50 +01001941#if IS_ENABLED(CONFIG_GARP)
Eric Dumazet3cc77ec2010-10-24 21:32:36 +00001942 struct garp_port __rcu *garp_port;
Tobias Klauserfb585b42017-02-10 16:43:50 +01001943#endif
1944#if IS_ENABLED(CONFIG_MRP)
David Wardfebf0182013-02-08 17:17:06 +00001945 struct mrp_port __rcu *mrp_port;
Tobias Klauserfb585b42017-02-10 16:43:50 +01001946#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07001947
Bjorn Helgaas5e82b4b2016-03-23 13:47:23 -05001948 struct device dev;
Eric W. Biederman0c509a62009-10-29 14:18:21 +00001949 const struct attribute_group *sysfs_groups[4];
Michael Daltona953be52014-01-16 22:23:28 -08001950 const struct attribute_group *sysfs_rx_queue_group;
Patrick McHardy38f7b872007-06-13 12:03:51 -07001951
Patrick McHardy38f7b872007-06-13 12:03:51 -07001952 const struct rtnl_link_ops *rtnl_link_ops;
Peter P Waskiewicz Jrf25f4e42007-07-06 13:36:20 -07001953
Peter P Waskiewicz Jr82cc1a72008-03-21 03:43:19 -07001954 /* for setting kernel sock attribute on TCP connection setup */
1955#define GSO_MAX_SIZE 65536
1956 unsigned int gso_max_size;
Ben Hutchings30b678d2012-07-30 15:57:00 +00001957#define GSO_MAX_SEGS 65535
1958 u16 gso_max_segs;
Eric Dumazet743b03a2016-04-09 11:29:58 -07001959
Jeff Kirsher7a6b6f52008-11-25 01:02:08 -08001960#ifdef CONFIG_DCB
Stephen Hemminger32953542009-10-05 06:01:03 +00001961 const struct dcbnl_rtnl_ops *dcbnl_ops;
Alexander Duyck2f90b862008-11-20 20:52:10 -08001962#endif
Bjorn Helgaas5e82b4b2016-03-23 13:47:23 -05001963 u8 num_tc;
1964 struct netdev_tc_txq tc_to_txq[TC_MAX_QUEUE];
1965 u8 prio_tc_map[TC_BITMASK + 1];
Alexander Duyck2f90b862008-11-20 20:52:10 -08001966
Ben Hutchingsd11ead72011-11-25 14:40:26 +00001967#if IS_ENABLED(CONFIG_FCOE)
Yi Zou4d288d52009-02-27 14:06:59 -08001968 unsigned int fcoe_ddp_xid;
1969#endif
Daniel Borkmann86f85152013-12-29 17:27:11 +01001970#if IS_ENABLED(CONFIG_CGROUP_NET_PRIO)
Neil Horman5bc14212011-11-22 05:10:51 +00001971 struct netprio_map __rcu *priomap;
1972#endif
Bjorn Helgaas5e82b4b2016-03-23 13:47:23 -05001973 struct phy_device *phydev;
Russell Kinge679c9c2018-03-28 15:44:16 -07001974 struct sfp_bus *sfp_bus;
Bjorn Helgaas5e82b4b2016-03-23 13:47:23 -05001975 struct lock_class_key *qdisc_tx_busylock;
Eric Dumazetf9eb8ae2016-06-06 09:37:15 -07001976 struct lock_class_key *qdisc_running_key;
Bjorn Helgaas5e82b4b2016-03-23 13:47:23 -05001977 bool proto_down;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001978};
Greg Kroah-Hartman43cb76d2002-04-09 12:14:34 -07001979#define to_net_dev(d) container_of(d, struct net_device, dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001980
David S. Millerb5cdae32017-04-18 15:36:58 -04001981static inline bool netif_elide_gro(const struct net_device *dev)
1982{
1983 if (!(dev->features & NETIF_F_GRO) || dev->xdp_prog)
1984 return true;
1985 return false;
1986}
1987
Linus Torvalds1da177e2005-04-16 15:20:36 -07001988#define NETDEV_ALIGN 32
Linus Torvalds1da177e2005-04-16 15:20:36 -07001989
David S. Millere8a04642008-07-17 00:34:19 -07001990static inline
John Fastabend4f57c082011-01-17 08:06:04 +00001991int netdev_get_prio_tc_map(const struct net_device *dev, u32 prio)
1992{
1993 return dev->prio_tc_map[prio & TC_BITMASK];
1994}
1995
1996static inline
1997int netdev_set_prio_tc_map(struct net_device *dev, u8 prio, u8 tc)
1998{
1999 if (tc >= dev->num_tc)
2000 return -EINVAL;
2001
2002 dev->prio_tc_map[prio & TC_BITMASK] = tc & TC_BITMASK;
2003 return 0;
2004}
2005
Alexander Duyck8d059b02016-10-28 11:43:49 -04002006int netdev_txq_to_tc(struct net_device *dev, unsigned int txq);
Alexander Duyck9cf1f6a2016-10-28 11:43:20 -04002007void netdev_reset_tc(struct net_device *dev);
2008int netdev_set_tc_queue(struct net_device *dev, u8 tc, u16 count, u16 offset);
2009int netdev_set_num_tc(struct net_device *dev, u8 num_tc);
John Fastabend4f57c082011-01-17 08:06:04 +00002010
2011static inline
2012int netdev_get_num_tc(struct net_device *dev)
2013{
2014 return dev->num_tc;
2015}
2016
2017static inline
David S. Millere8a04642008-07-17 00:34:19 -07002018struct netdev_queue *netdev_get_tx_queue(const struct net_device *dev,
2019 unsigned int index)
2020{
2021 return &dev->_tx[index];
2022}
2023
Daniel Borkmann10c51b56232014-08-27 11:11:27 +02002024static inline struct netdev_queue *skb_get_tx_queue(const struct net_device *dev,
2025 const struct sk_buff *skb)
2026{
2027 return netdev_get_tx_queue(dev, skb_get_queue_mapping(skb));
2028}
2029
David S. Millere8a04642008-07-17 00:34:19 -07002030static inline void netdev_for_each_tx_queue(struct net_device *dev,
2031 void (*f)(struct net_device *,
2032 struct netdev_queue *,
2033 void *),
2034 void *arg)
2035{
2036 unsigned int i;
2037
2038 for (i = 0; i < dev->num_tx_queues; i++)
2039 f(dev, &dev->_tx[i], arg);
2040}
2041
Eric Dumazetd3fff6c2016-06-09 07:45:12 -07002042#define netdev_lockdep_set_classes(dev) \
2043{ \
2044 static struct lock_class_key qdisc_tx_busylock_key; \
2045 static struct lock_class_key qdisc_running_key; \
2046 static struct lock_class_key qdisc_xmit_lock_key; \
2047 static struct lock_class_key dev_addr_list_lock_key; \
2048 unsigned int i; \
2049 \
2050 (dev)->qdisc_tx_busylock = &qdisc_tx_busylock_key; \
2051 (dev)->qdisc_running_key = &qdisc_running_key; \
2052 lockdep_set_class(&(dev)->addr_list_lock, \
2053 &dev_addr_list_lock_key); \
2054 for (i = 0; i < (dev)->num_tx_queues; i++) \
2055 lockdep_set_class(&(dev)->_tx[i]._xmit_lock, \
2056 &qdisc_xmit_lock_key); \
2057}
2058
Joe Perchesf629d202013-09-26 14:48:15 -07002059struct netdev_queue *netdev_pick_tx(struct net_device *dev,
Jason Wangf663dd92014-01-10 16:18:26 +08002060 struct sk_buff *skb,
2061 void *accel_priv);
Amerigo Wang8c4c49d2012-09-17 20:16:31 +00002062
Paolo Abeni871b6422016-02-26 10:45:37 +01002063/* returns the headroom that the master device needs to take in account
2064 * when forwarding to this dev
2065 */
2066static inline unsigned netdev_get_fwd_headroom(struct net_device *dev)
2067{
2068 return dev->priv_flags & IFF_PHONY_HEADROOM ? 0 : dev->needed_headroom;
2069}
2070
2071static inline void netdev_set_rx_headroom(struct net_device *dev, int new_hr)
2072{
2073 if (dev->netdev_ops->ndo_set_rx_headroom)
2074 dev->netdev_ops->ndo_set_rx_headroom(dev, new_hr);
2075}
2076
2077/* set the device rx headroom to the dev's default */
2078static inline void netdev_reset_rx_headroom(struct net_device *dev)
2079{
2080 netdev_set_rx_headroom(dev, -1);
2081}
2082
YOSHIFUJI Hideakic346dca2008-03-25 21:47:49 +09002083/*
2084 * Net namespace inlines
2085 */
2086static inline
2087struct net *dev_net(const struct net_device *dev)
2088{
Eric Dumazetc2d9ba92010-06-01 06:51:19 +00002089 return read_pnet(&dev->nd_net);
YOSHIFUJI Hideakic346dca2008-03-25 21:47:49 +09002090}
2091
2092static inline
Denis V. Lunevf5aa23f2008-03-26 00:48:17 -07002093void dev_net_set(struct net_device *dev, struct net *net)
YOSHIFUJI Hideakic346dca2008-03-25 21:47:49 +09002094{
Eric W. Biederman0c5c9fb2015-03-11 23:06:44 -05002095 write_pnet(&dev->nd_net, net);
YOSHIFUJI Hideakic346dca2008-03-25 21:47:49 +09002096}
2097
Stephen Hemmingerbea33482007-10-03 16:41:36 -07002098/**
2099 * netdev_priv - access network device private data
2100 * @dev: network device
2101 *
2102 * Get network device private data
2103 */
Patrick McHardy6472ce62007-06-13 12:03:21 -07002104static inline void *netdev_priv(const struct net_device *dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002105{
Eric Dumazet1ce8e7b2009-05-27 04:42:37 +00002106 return (char *)dev + ALIGN(sizeof(struct net_device), NETDEV_ALIGN);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002107}
2108
Linus Torvalds1da177e2005-04-16 15:20:36 -07002109/* Set the sysfs physical device reference for the network logical device
2110 * if set prior to registration will cause a symlink during initialization.
2111 */
Greg Kroah-Hartman43cb76d2002-04-09 12:14:34 -07002112#define SET_NETDEV_DEV(net, pdev) ((net)->dev.parent = (pdev))
Linus Torvalds1da177e2005-04-16 15:20:36 -07002113
Marcel Holtmann384912e2009-08-31 21:08:19 +00002114/* Set the sysfs device type for the network logical device to allow
Maxime Jayat3f794102013-10-12 01:29:46 +02002115 * fine-grained identification of different network device types. For
Bjorn Helgaas5e82b4b2016-03-23 13:47:23 -05002116 * example Ethernet, Wireless LAN, Bluetooth, WiMAX etc.
Marcel Holtmann384912e2009-08-31 21:08:19 +00002117 */
2118#define SET_NETDEV_DEVTYPE(net, devtype) ((net)->dev.type = (devtype))
2119
Eric Dumazet82dc3c63c2013-03-05 15:57:22 +00002120/* Default NAPI poll() weight
2121 * Device drivers are strongly advised to not use bigger value
2122 */
2123#define NAPI_POLL_WEIGHT 64
2124
Stephen Hemminger3b582cc2007-11-01 02:21:47 -07002125/**
Bjorn Helgaas5e82b4b2016-03-23 13:47:23 -05002126 * netif_napi_add - initialize a NAPI context
Stephen Hemminger3b582cc2007-11-01 02:21:47 -07002127 * @dev: network device
Bjorn Helgaas5e82b4b2016-03-23 13:47:23 -05002128 * @napi: NAPI context
Stephen Hemminger3b582cc2007-11-01 02:21:47 -07002129 * @poll: polling function
2130 * @weight: default weight
2131 *
Bjorn Helgaas5e82b4b2016-03-23 13:47:23 -05002132 * netif_napi_add() must be used to initialize a NAPI context prior to calling
2133 * *any* of the other NAPI-related functions.
Stephen Hemminger3b582cc2007-11-01 02:21:47 -07002134 */
Herbert Xud565b0a2008-12-15 23:38:52 -08002135void netif_napi_add(struct net_device *dev, struct napi_struct *napi,
2136 int (*poll)(struct napi_struct *, int), int weight);
Stephen Hemmingerbea33482007-10-03 16:41:36 -07002137
Alexander Duyckd8156532008-07-08 15:13:05 -07002138/**
Bjorn Helgaas5e82b4b2016-03-23 13:47:23 -05002139 * netif_tx_napi_add - initialize a NAPI context
Eric Dumazetd64b5e82015-11-18 06:31:00 -08002140 * @dev: network device
Bjorn Helgaas5e82b4b2016-03-23 13:47:23 -05002141 * @napi: NAPI context
Eric Dumazetd64b5e82015-11-18 06:31:00 -08002142 * @poll: polling function
2143 * @weight: default weight
2144 *
2145 * This variant of netif_napi_add() should be used from drivers using NAPI
2146 * to exclusively poll a TX queue.
2147 * This will avoid we add it into napi_hash[], thus polluting this hash table.
2148 */
2149static inline void netif_tx_napi_add(struct net_device *dev,
2150 struct napi_struct *napi,
2151 int (*poll)(struct napi_struct *, int),
2152 int weight)
2153{
2154 set_bit(NAPI_STATE_NO_BUSY_POLL, &napi->state);
2155 netif_napi_add(dev, napi, poll, weight);
2156}
2157
2158/**
Bjorn Helgaas5e82b4b2016-03-23 13:47:23 -05002159 * netif_napi_del - remove a NAPI context
2160 * @napi: NAPI context
Alexander Duyckd8156532008-07-08 15:13:05 -07002161 *
Bjorn Helgaas5e82b4b2016-03-23 13:47:23 -05002162 * netif_napi_del() removes a NAPI context from the network device NAPI list
Alexander Duyckd8156532008-07-08 15:13:05 -07002163 */
Herbert Xud565b0a2008-12-15 23:38:52 -08002164void netif_napi_del(struct napi_struct *napi);
2165
2166struct napi_gro_cb {
Herbert Xu78a478d2009-05-26 18:50:21 +00002167 /* Virtual address of skb_shinfo(skb)->frags[0].page + offset. */
Bjorn Helgaas5e82b4b2016-03-23 13:47:23 -05002168 void *frag0;
Herbert Xu78a478d2009-05-26 18:50:21 +00002169
Herbert Xu74895942009-05-26 18:50:27 +00002170 /* Length of frag0. */
2171 unsigned int frag0_len;
2172
Herbert Xu86911732009-01-29 14:19:50 +00002173 /* This indicates where we are processing relative to skb->data. */
Bjorn Helgaas5e82b4b2016-03-23 13:47:23 -05002174 int data_offset;
Herbert Xu86911732009-01-29 14:19:50 +00002175
Herbert Xud565b0a2008-12-15 23:38:52 -08002176 /* This is non-zero if the packet cannot be merged with the new skb. */
Jerry Chubf5a7552014-01-07 10:23:19 -08002177 u16 flush;
2178
2179 /* Save the IP ID here and check when we get to the transport layer */
2180 u16 flush_id;
Herbert Xud565b0a2008-12-15 23:38:52 -08002181
2182 /* Number of segments aggregated. */
Eric Dumazet2e71a6f2012-10-06 08:08:49 +00002183 u16 count;
2184
Tom Herbert15e23962015-02-10 16:30:31 -08002185 /* Start offset for remote checksum offload */
2186 u16 gro_remcsum_start;
2187
Eric Dumazet2e71a6f2012-10-06 08:08:49 +00002188 /* jiffies when first packet was created/queued */
2189 unsigned long age;
Eric Dumazet86347242012-10-08 21:38:50 +02002190
Tom Herbertafe93322014-09-17 12:25:57 -07002191 /* Used in ipv6_gro_receive() and foo-over-udp */
Or Gerlitzb582ef02014-01-20 13:59:19 +02002192 u16 proto;
2193
Tom Herbertbaa32ff2015-02-10 16:30:30 -08002194 /* This is non-zero if the packet may be of the same flow. */
2195 u8 same_flow:1;
2196
Jesse Grossfac8e0f2016-03-19 09:32:01 -07002197 /* Used in tunnel GRO receive */
2198 u8 encap_mark:1;
Tom Herbert573e8fc2014-08-22 13:33:47 -07002199
2200 /* GRO checksum is valid */
2201 u8 csum_valid:1;
2202
Tom Herbert662880f2014-08-27 21:26:56 -07002203 /* Number of checksums via CHECKSUM_UNNECESSARY */
2204 u8 csum_cnt:3;
Eric Dumazetc3c7c252012-12-06 13:54:59 +00002205
Tom Herbertbaa32ff2015-02-10 16:30:30 -08002206 /* Free the skb? */
2207 u8 free:2;
2208#define NAPI_GRO_FREE 1
2209#define NAPI_GRO_FREE_STOLEN_HEAD 2
2210
Tom Herbertefc98d02014-10-03 15:48:08 -07002211 /* Used in foo-over-udp, set in udp[46]_gro_receive */
2212 u8 is_ipv6:1;
2213
Alexander Duycka0ca1532016-04-05 09:13:39 -07002214 /* Used in GRE, set in fou/gue_gro_receive */
2215 u8 is_fou:1;
2216
Alexander Duyck15305452016-04-10 21:44:57 -04002217 /* Used to determine if flush_id can be ignored */
2218 u8 is_atomic:1;
2219
Sabrina Dubrocafcd91dd2016-10-20 15:58:02 +02002220 /* Number of gro_receive callbacks this packet already went through */
2221 u8 recursion_counter:4;
2222
2223 /* 1 bit hole */
Tom Herbertbaa32ff2015-02-10 16:30:30 -08002224
Jerry Chubf5a7552014-01-07 10:23:19 -08002225 /* used to support CHECKSUM_COMPLETE for tunneling protocols */
2226 __wsum csum;
2227
Eric Dumazetc3c7c252012-12-06 13:54:59 +00002228 /* used in skb_gro_receive() slow path */
2229 struct sk_buff *last;
Herbert Xud565b0a2008-12-15 23:38:52 -08002230};
2231
2232#define NAPI_GRO_CB(skb) ((struct napi_gro_cb *)(skb)->cb)
Alexander Duyckd8156532008-07-08 15:13:05 -07002233
Sabrina Dubrocafcd91dd2016-10-20 15:58:02 +02002234#define GRO_RECURSION_LIMIT 15
2235static inline int gro_recursion_inc_test(struct sk_buff *skb)
2236{
2237 return ++NAPI_GRO_CB(skb)->recursion_counter == GRO_RECURSION_LIMIT;
2238}
2239
2240typedef struct sk_buff **(*gro_receive_t)(struct sk_buff **, struct sk_buff *);
2241static inline struct sk_buff **call_gro_receive(gro_receive_t cb,
2242 struct sk_buff **head,
2243 struct sk_buff *skb)
2244{
2245 if (unlikely(gro_recursion_inc_test(skb))) {
2246 NAPI_GRO_CB(skb)->flush |= 1;
2247 return NULL;
2248 }
2249
2250 return cb(head, skb);
2251}
2252
2253typedef struct sk_buff **(*gro_receive_sk_t)(struct sock *, struct sk_buff **,
2254 struct sk_buff *);
2255static inline struct sk_buff **call_gro_receive_sk(gro_receive_sk_t cb,
2256 struct sock *sk,
2257 struct sk_buff **head,
2258 struct sk_buff *skb)
2259{
2260 if (unlikely(gro_recursion_inc_test(skb))) {
2261 NAPI_GRO_CB(skb)->flush |= 1;
2262 return NULL;
2263 }
2264
2265 return cb(sk, head, skb);
2266}
2267
Linus Torvalds1da177e2005-04-16 15:20:36 -07002268struct packet_type {
David S. Millerf2ccd8f2005-08-09 19:34:12 -07002269 __be16 type; /* This is really htons(ether_type). */
2270 struct net_device *dev; /* NULL is wildcarded here */
2271 int (*func) (struct sk_buff *,
2272 struct net_device *,
2273 struct packet_type *,
2274 struct net_device *);
Eric Leblondc0de08d2012-08-16 22:02:58 +00002275 bool (*id_match)(struct packet_type *ptype,
2276 struct sock *sk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002277 void *af_packet_priv;
2278 struct list_head list;
2279};
2280
Vlad Yasevichf191a1d2012-11-15 08:49:23 +00002281struct offload_callbacks {
Linus Torvalds1da177e2005-04-16 15:20:36 -07002282 struct sk_buff *(*gso_segment)(struct sk_buff *skb,
2283 netdev_features_t features);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002284 struct sk_buff **(*gro_receive)(struct sk_buff **head,
Tom Herberta2b12f32015-01-12 17:00:37 -08002285 struct sk_buff *skb);
Jerry Chu299603e82013-12-11 20:53:45 -08002286 int (*gro_complete)(struct sk_buff *skb, int nhoff);
Vlad Yasevichf191a1d2012-11-15 08:49:23 +00002287};
2288
2289struct packet_offload {
2290 __be16 type; /* This is really htons(ether_type). */
David S. Millerbdef7de2015-06-01 14:56:09 -07002291 u16 priority;
Vlad Yasevichf191a1d2012-11-15 08:49:23 +00002292 struct offload_callbacks callbacks;
2293 struct list_head list;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002294};
2295
Bjorn Helgaas5e82b4b2016-03-23 13:47:23 -05002296/* often modified stats are per-CPU, other are shared (netdev->stats) */
Li RongQing8f849852014-01-04 13:57:59 +08002297struct pcpu_sw_netstats {
2298 u64 rx_packets;
2299 u64 rx_bytes;
2300 u64 tx_packets;
2301 u64 tx_bytes;
2302 struct u64_stats_sync syncp;
2303};
2304
Pablo Neira Ayusoaabc92b2015-11-10 14:31:18 +01002305#define __netdev_alloc_pcpu_stats(type, gfp) \
2306({ \
2307 typeof(type) __percpu *pcpu_stats = alloc_percpu_gfp(type, gfp);\
2308 if (pcpu_stats) { \
2309 int __cpu; \
2310 for_each_possible_cpu(__cpu) { \
2311 typeof(type) *stat; \
2312 stat = per_cpu_ptr(pcpu_stats, __cpu); \
2313 u64_stats_init(&stat->syncp); \
2314 } \
2315 } \
2316 pcpu_stats; \
WANG Cong1c213bd2014-02-13 11:46:28 -08002317})
2318
Pablo Neira Ayusoaabc92b2015-11-10 14:31:18 +01002319#define netdev_alloc_pcpu_stats(type) \
Felix Fietkau326fcfa2015-12-05 13:58:11 +01002320 __netdev_alloc_pcpu_stats(type, GFP_KERNEL)
Pablo Neira Ayusoaabc92b2015-11-10 14:31:18 +01002321
Jiri Pirko764f5e52015-12-03 12:12:12 +01002322enum netdev_lag_tx_type {
2323 NETDEV_LAG_TX_TYPE_UNKNOWN,
2324 NETDEV_LAG_TX_TYPE_RANDOM,
2325 NETDEV_LAG_TX_TYPE_BROADCAST,
2326 NETDEV_LAG_TX_TYPE_ROUNDROBIN,
2327 NETDEV_LAG_TX_TYPE_ACTIVEBACKUP,
2328 NETDEV_LAG_TX_TYPE_HASH,
2329};
2330
2331struct netdev_lag_upper_info {
2332 enum netdev_lag_tx_type tx_type;
2333};
2334
Jiri Pirkofb1b2e32015-12-03 12:12:16 +01002335struct netdev_lag_lower_state_info {
2336 u8 link_up : 1,
2337 tx_enabled : 1;
2338};
2339
Linus Torvalds1da177e2005-04-16 15:20:36 -07002340#include <linux/notifier.h>
2341
Kirill Tkhaiede27622018-03-23 19:47:19 +03002342/* netdevice notifier chain. Please remember to update netdev_cmd_to_name()
2343 * and the rtnetlink notification exclusion list in rtnetlink_event() when
2344 * adding new types.
Amerigo Wangdcfe1422011-07-25 17:13:09 -07002345 */
Kirill Tkhaiede27622018-03-23 19:47:19 +03002346enum netdev_cmd {
2347 NETDEV_UP = 1, /* For now you can't veto a device up/down */
2348 NETDEV_DOWN,
2349 NETDEV_REBOOT, /* Tell a protocol stack a network interface
Amerigo Wangdcfe1422011-07-25 17:13:09 -07002350 detected a hardware crash and restarted
2351 - we can use this eg to kick tcp sessions
2352 once done */
Kirill Tkhaiede27622018-03-23 19:47:19 +03002353 NETDEV_CHANGE, /* Notify device state change */
2354 NETDEV_REGISTER,
2355 NETDEV_UNREGISTER,
2356 NETDEV_CHANGEMTU, /* notify after mtu change happened */
2357 NETDEV_CHANGEADDR,
2358 NETDEV_GOING_DOWN,
2359 NETDEV_CHANGENAME,
2360 NETDEV_FEAT_CHANGE,
2361 NETDEV_BONDING_FAILOVER,
2362 NETDEV_PRE_UP,
2363 NETDEV_PRE_TYPE_CHANGE,
2364 NETDEV_POST_TYPE_CHANGE,
2365 NETDEV_POST_INIT,
Kirill Tkhaiede27622018-03-23 19:47:19 +03002366 NETDEV_RELEASE,
2367 NETDEV_NOTIFY_PEERS,
2368 NETDEV_JOIN,
2369 NETDEV_CHANGEUPPER,
2370 NETDEV_RESEND_IGMP,
2371 NETDEV_PRECHANGEMTU, /* notify before mtu change happened */
2372 NETDEV_CHANGEINFODATA,
2373 NETDEV_BONDING_INFO,
2374 NETDEV_PRECHANGEUPPER,
2375 NETDEV_CHANGELOWERSTATE,
2376 NETDEV_UDP_TUNNEL_PUSH_INFO,
2377 NETDEV_UDP_TUNNEL_DROP_INFO,
2378 NETDEV_CHANGE_TX_QUEUE_LEN,
Gal Pressman9daae9b2018-03-28 17:46:54 +03002379 NETDEV_CVLAN_FILTER_PUSH_INFO,
2380 NETDEV_CVLAN_FILTER_DROP_INFO,
2381 NETDEV_SVLAN_FILTER_PUSH_INFO,
2382 NETDEV_SVLAN_FILTER_DROP_INFO,
Kirill Tkhaiede27622018-03-23 19:47:19 +03002383};
2384const char *netdev_cmd_to_name(enum netdev_cmd cmd);
Amerigo Wangdcfe1422011-07-25 17:13:09 -07002385
Joe Perchesf629d202013-09-26 14:48:15 -07002386int register_netdevice_notifier(struct notifier_block *nb);
2387int unregister_netdevice_notifier(struct notifier_block *nb);
Jiri Pirko351638e2013-05-28 01:30:21 +00002388
2389struct netdev_notifier_info {
David Ahern51d0c0472017-10-04 17:48:45 -07002390 struct net_device *dev;
2391 struct netlink_ext_ack *extack;
Jiri Pirko351638e2013-05-28 01:30:21 +00002392};
2393
Jiri Pirkobe9efd32013-05-28 01:30:22 +00002394struct netdev_notifier_change_info {
2395 struct netdev_notifier_info info; /* must be first */
2396 unsigned int flags_changed;
2397};
2398
Jiri Pirko0e4ead92015-08-27 09:31:18 +02002399struct netdev_notifier_changeupper_info {
2400 struct netdev_notifier_info info; /* must be first */
2401 struct net_device *upper_dev; /* new upper dev */
2402 bool master; /* is upper dev master */
Bjorn Helgaas5e82b4b2016-03-23 13:47:23 -05002403 bool linking; /* is the notification for link or unlink */
Jiri Pirko29bf24a2015-12-03 12:12:11 +01002404 void *upper_info; /* upper dev info */
Jiri Pirko0e4ead92015-08-27 09:31:18 +02002405};
2406
Jiri Pirko04d48262015-12-03 12:12:15 +01002407struct netdev_notifier_changelowerstate_info {
2408 struct netdev_notifier_info info; /* must be first */
2409 void *lower_state_info; /* is lower dev state */
2410};
2411
Cong Wang75538c22013-05-29 11:30:50 +08002412static inline void netdev_notifier_info_init(struct netdev_notifier_info *info,
2413 struct net_device *dev)
2414{
2415 info->dev = dev;
David Ahern51d0c0472017-10-04 17:48:45 -07002416 info->extack = NULL;
Cong Wang75538c22013-05-29 11:30:50 +08002417}
2418
Jiri Pirko351638e2013-05-28 01:30:21 +00002419static inline struct net_device *
2420netdev_notifier_info_to_dev(const struct netdev_notifier_info *info)
2421{
2422 return info->dev;
2423}
2424
David Ahern51d0c0472017-10-04 17:48:45 -07002425static inline struct netlink_ext_ack *
2426netdev_notifier_info_to_extack(const struct netdev_notifier_info *info)
2427{
2428 return info->extack;
2429}
2430
Joe Perchesf629d202013-09-26 14:48:15 -07002431int call_netdevice_notifiers(unsigned long val, struct net_device *dev);
Amerigo Wangdcfe1422011-07-25 17:13:09 -07002432
2433
Linus Torvalds1da177e2005-04-16 15:20:36 -07002434extern rwlock_t dev_base_lock; /* Device list lock */
2435
Eric W. Biederman881d9662007-09-17 11:56:21 -07002436#define for_each_netdev(net, d) \
2437 list_for_each_entry(d, &(net)->dev_base_head, dev_list)
Eric W. Biedermandcbccbd42009-11-29 22:25:26 +00002438#define for_each_netdev_reverse(net, d) \
2439 list_for_each_entry_reverse(d, &(net)->dev_base_head, dev_list)
Eric Dumazetc6d14c82009-11-04 05:43:23 -08002440#define for_each_netdev_rcu(net, d) \
2441 list_for_each_entry_rcu(d, &(net)->dev_base_head, dev_list)
Eric W. Biederman881d9662007-09-17 11:56:21 -07002442#define for_each_netdev_safe(net, d, n) \
2443 list_for_each_entry_safe(d, n, &(net)->dev_base_head, dev_list)
2444#define for_each_netdev_continue(net, d) \
2445 list_for_each_entry_continue(d, &(net)->dev_base_head, dev_list)
stephen hemminger254245d2009-11-10 07:54:47 +00002446#define for_each_netdev_continue_rcu(net, d) \
2447 list_for_each_entry_continue_rcu(d, &(net)->dev_base_head, dev_list)
nikolay@redhat.com8a7fbfa2013-03-12 02:49:01 +00002448#define for_each_netdev_in_bond_rcu(bond, slave) \
2449 for_each_netdev_rcu(&init_net, slave) \
Benjamin Poirier4ccce022015-01-14 16:52:35 +09002450 if (netdev_master_upper_dev_get_rcu(slave) == (bond))
Pavel Emelianov7562f872007-05-03 15:13:45 -07002451#define net_device_entry(lh) list_entry(lh, struct net_device, dev_list)
2452
Daniel Lezcanoa050c332007-09-12 14:57:09 +02002453static inline struct net_device *next_net_device(struct net_device *dev)
2454{
2455 struct list_head *lh;
2456 struct net *net;
Pavel Emelianov7562f872007-05-03 15:13:45 -07002457
YOSHIFUJI Hideakic346dca2008-03-25 21:47:49 +09002458 net = dev_net(dev);
Daniel Lezcanoa050c332007-09-12 14:57:09 +02002459 lh = dev->dev_list.next;
2460 return lh == &net->dev_base_head ? NULL : net_device_entry(lh);
2461}
2462
Eric Dumazetce81b762009-11-11 17:34:30 +00002463static inline struct net_device *next_net_device_rcu(struct net_device *dev)
2464{
2465 struct list_head *lh;
2466 struct net *net;
2467
2468 net = dev_net(dev);
Eric Dumazetccf43432011-01-26 18:08:02 +00002469 lh = rcu_dereference(list_next_rcu(&dev->dev_list));
Eric Dumazetce81b762009-11-11 17:34:30 +00002470 return lh == &net->dev_base_head ? NULL : net_device_entry(lh);
2471}
2472
Daniel Lezcanoa050c332007-09-12 14:57:09 +02002473static inline struct net_device *first_net_device(struct net *net)
2474{
2475 return list_empty(&net->dev_base_head) ? NULL :
2476 net_device_entry(net->dev_base_head.next);
2477}
Pavel Emelianov7562f872007-05-03 15:13:45 -07002478
Eric Dumazetccf43432011-01-26 18:08:02 +00002479static inline struct net_device *first_net_device_rcu(struct net *net)
2480{
2481 struct list_head *lh = rcu_dereference(list_next_rcu(&net->dev_base_head));
2482
2483 return lh == &net->dev_base_head ? NULL : net_device_entry(lh);
2484}
2485
Joe Perchesf629d202013-09-26 14:48:15 -07002486int netdev_boot_setup_check(struct net_device *dev);
2487unsigned long netdev_boot_base(const char *prefix, int unit);
2488struct net_device *dev_getbyhwaddr_rcu(struct net *net, unsigned short type,
2489 const char *hwaddr);
2490struct net_device *dev_getfirstbyhwtype(struct net *net, unsigned short type);
2491struct net_device *__dev_getfirstbyhwtype(struct net *net, unsigned short type);
2492void dev_add_pack(struct packet_type *pt);
2493void dev_remove_pack(struct packet_type *pt);
2494void __dev_remove_pack(struct packet_type *pt);
2495void dev_add_offload(struct packet_offload *po);
2496void dev_remove_offload(struct packet_offload *po);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002497
Nicolas Dichtela54acb32015-04-02 17:07:00 +02002498int dev_get_iflink(const struct net_device *dev);
Pravin B Shelarfc4099f2015-10-22 18:17:16 -07002499int dev_fill_metadata_dst(struct net_device *dev, struct sk_buff *skb);
WANG Cong6c555492014-09-11 15:35:09 -07002500struct net_device *__dev_get_by_flags(struct net *net, unsigned short flags,
2501 unsigned short mask);
Joe Perchesf629d202013-09-26 14:48:15 -07002502struct net_device *dev_get_by_name(struct net *net, const char *name);
2503struct net_device *dev_get_by_name_rcu(struct net *net, const char *name);
2504struct net_device *__dev_get_by_name(struct net *net, const char *name);
2505int dev_alloc_name(struct net_device *dev, const char *name);
2506int dev_open(struct net_device *dev);
stephen hemminger7051b882017-07-18 15:59:27 -07002507void dev_close(struct net_device *dev);
2508void dev_close_many(struct list_head *head, bool unlink);
Joe Perchesf629d202013-09-26 14:48:15 -07002509void dev_disable_lro(struct net_device *dev);
Eric W. Biederman0c4b51f2015-09-15 20:04:18 -05002510int dev_loopback_xmit(struct net *net, struct sock *sk, struct sk_buff *newskb);
Eric W. Biederman2b4aa3c2015-09-15 20:04:07 -05002511int dev_queue_xmit(struct sk_buff *skb);
Jason Wangf663dd92014-01-10 16:18:26 +08002512int dev_queue_xmit_accel(struct sk_buff *skb, void *accel_priv);
Magnus Karlsson865b03f2018-05-02 13:01:33 +02002513int dev_direct_xmit(struct sk_buff *skb, u16 queue_id);
Joe Perchesf629d202013-09-26 14:48:15 -07002514int register_netdevice(struct net_device *dev);
2515void unregister_netdevice_queue(struct net_device *dev, struct list_head *head);
2516void unregister_netdevice_many(struct list_head *head);
Eric Dumazet44a08732009-10-27 07:03:04 +00002517static inline void unregister_netdevice(struct net_device *dev)
2518{
2519 unregister_netdevice_queue(dev, NULL);
2520}
2521
Joe Perchesf629d202013-09-26 14:48:15 -07002522int netdev_refcnt_read(const struct net_device *dev);
2523void free_netdev(struct net_device *dev);
Eric Dumazet74d332c2013-10-30 13:10:44 -07002524void netdev_freemem(struct net_device *dev);
Joe Perchesf629d202013-09-26 14:48:15 -07002525void synchronize_net(void);
2526int init_dummy_netdev(struct net_device *dev);
Benjamin Herrenschmidt937f1ba2009-01-14 21:05:05 -08002527
hannes@stressinduktion.orgf60e5992015-04-01 17:07:44 +02002528DECLARE_PER_CPU(int, xmit_recursion);
Daniel Borkmanna70b5062016-06-10 21:19:06 +02002529#define XMIT_RECURSION_LIMIT 10
2530
hannes@stressinduktion.orgf60e5992015-04-01 17:07:44 +02002531static inline int dev_recursion_level(void)
2532{
2533 return this_cpu_read(xmit_recursion);
2534}
2535
Joe Perchesf629d202013-09-26 14:48:15 -07002536struct net_device *dev_get_by_index(struct net *net, int ifindex);
2537struct net_device *__dev_get_by_index(struct net *net, int ifindex);
2538struct net_device *dev_get_by_index_rcu(struct net *net, int ifindex);
Miroslav Lichvar90b602f2017-05-19 17:52:37 +02002539struct net_device *dev_get_by_napi_id(unsigned int napi_id);
Joe Perchesf629d202013-09-26 14:48:15 -07002540int netdev_get_name(struct net *net, char *name, int ifindex);
2541int dev_restart(struct net_device *dev);
Joe Perchesf629d202013-09-26 14:48:15 -07002542int skb_gro_receive(struct sk_buff **head, struct sk_buff *skb);
Herbert Xu86911732009-01-29 14:19:50 +00002543
2544static inline unsigned int skb_gro_offset(const struct sk_buff *skb)
2545{
2546 return NAPI_GRO_CB(skb)->data_offset;
2547}
2548
2549static inline unsigned int skb_gro_len(const struct sk_buff *skb)
2550{
2551 return skb->len - NAPI_GRO_CB(skb)->data_offset;
2552}
2553
2554static inline void skb_gro_pull(struct sk_buff *skb, unsigned int len)
2555{
2556 NAPI_GRO_CB(skb)->data_offset += len;
2557}
2558
Herbert Xua5b1cf22009-05-26 18:50:28 +00002559static inline void *skb_gro_header_fast(struct sk_buff *skb,
2560 unsigned int offset)
Herbert Xu86911732009-01-29 14:19:50 +00002561{
Herbert Xu78a478d2009-05-26 18:50:21 +00002562 return NAPI_GRO_CB(skb)->frag0 + offset;
Herbert Xu86911732009-01-29 14:19:50 +00002563}
Linus Torvalds1da177e2005-04-16 15:20:36 -07002564
Herbert Xua5b1cf22009-05-26 18:50:28 +00002565static inline int skb_gro_header_hard(struct sk_buff *skb, unsigned int hlen)
2566{
2567 return NAPI_GRO_CB(skb)->frag0_len < hlen;
2568}
2569
Herbert Xu57ea52a2017-01-10 12:24:15 -08002570static inline void skb_gro_frag0_invalidate(struct sk_buff *skb)
2571{
2572 NAPI_GRO_CB(skb)->frag0 = NULL;
2573 NAPI_GRO_CB(skb)->frag0_len = 0;
2574}
2575
Herbert Xua5b1cf22009-05-26 18:50:28 +00002576static inline void *skb_gro_header_slow(struct sk_buff *skb, unsigned int hlen,
2577 unsigned int offset)
2578{
Herbert Xu17dd7592011-07-27 06:16:28 -07002579 if (!pskb_may_pull(skb, hlen))
2580 return NULL;
2581
Herbert Xu57ea52a2017-01-10 12:24:15 -08002582 skb_gro_frag0_invalidate(skb);
Herbert Xu17dd7592011-07-27 06:16:28 -07002583 return skb->data + offset;
Herbert Xua5b1cf22009-05-26 18:50:28 +00002584}
2585
Herbert Xu36e7b1b2009-04-27 05:44:45 -07002586static inline void *skb_gro_network_header(struct sk_buff *skb)
2587{
Herbert Xu78d3fd02009-05-26 18:50:23 +00002588 return (NAPI_GRO_CB(skb)->frag0 ?: skb->data) +
2589 skb_network_offset(skb);
Herbert Xu36e7b1b2009-04-27 05:44:45 -07002590}
2591
Jerry Chubf5a7552014-01-07 10:23:19 -08002592static inline void skb_gro_postpull_rcsum(struct sk_buff *skb,
2593 const void *start, unsigned int len)
2594{
Tom Herbert573e8fc2014-08-22 13:33:47 -07002595 if (NAPI_GRO_CB(skb)->csum_valid)
Jerry Chubf5a7552014-01-07 10:23:19 -08002596 NAPI_GRO_CB(skb)->csum = csum_sub(NAPI_GRO_CB(skb)->csum,
2597 csum_partial(start, len, 0));
2598}
2599
Tom Herbert573e8fc2014-08-22 13:33:47 -07002600/* GRO checksum functions. These are logical equivalents of the normal
2601 * checksum functions (in skbuff.h) except that they operate on the GRO
2602 * offsets and fields in sk_buff.
2603 */
2604
2605__sum16 __skb_gro_checksum_complete(struct sk_buff *skb);
2606
Tom Herbert15e23962015-02-10 16:30:31 -08002607static inline bool skb_at_gro_remcsum_start(struct sk_buff *skb)
2608{
Tom Herbertb7fe10e2015-08-19 17:07:32 -07002609 return (NAPI_GRO_CB(skb)->gro_remcsum_start == skb_gro_offset(skb));
Tom Herbert15e23962015-02-10 16:30:31 -08002610}
2611
Tom Herbert573e8fc2014-08-22 13:33:47 -07002612static inline bool __skb_gro_checksum_validate_needed(struct sk_buff *skb,
2613 bool zero_okay,
2614 __sum16 check)
2615{
Tom Herbert6edec0e2015-02-10 16:30:28 -08002616 return ((skb->ip_summed != CHECKSUM_PARTIAL ||
2617 skb_checksum_start_offset(skb) <
2618 skb_gro_offset(skb)) &&
Tom Herbert15e23962015-02-10 16:30:31 -08002619 !skb_at_gro_remcsum_start(skb) &&
Tom Herbert662880f2014-08-27 21:26:56 -07002620 NAPI_GRO_CB(skb)->csum_cnt == 0 &&
Tom Herbert573e8fc2014-08-22 13:33:47 -07002621 (!zero_okay || check));
2622}
2623
2624static inline __sum16 __skb_gro_checksum_validate_complete(struct sk_buff *skb,
2625 __wsum psum)
2626{
2627 if (NAPI_GRO_CB(skb)->csum_valid &&
2628 !csum_fold(csum_add(psum, NAPI_GRO_CB(skb)->csum)))
2629 return 0;
2630
2631 NAPI_GRO_CB(skb)->csum = psum;
2632
2633 return __skb_gro_checksum_complete(skb);
2634}
2635
Tom Herbert573e8fc2014-08-22 13:33:47 -07002636static inline void skb_gro_incr_csum_unnecessary(struct sk_buff *skb)
2637{
Tom Herbert662880f2014-08-27 21:26:56 -07002638 if (NAPI_GRO_CB(skb)->csum_cnt > 0) {
2639 /* Consume a checksum from CHECKSUM_UNNECESSARY */
2640 NAPI_GRO_CB(skb)->csum_cnt--;
2641 } else {
2642 /* Update skb for CHECKSUM_UNNECESSARY and csum_level when we
2643 * verified a new top level checksum or an encapsulated one
2644 * during GRO. This saves work if we fallback to normal path.
2645 */
2646 __skb_incr_checksum_unnecessary(skb);
Tom Herbert573e8fc2014-08-22 13:33:47 -07002647 }
2648}
2649
2650#define __skb_gro_checksum_validate(skb, proto, zero_okay, check, \
2651 compute_pseudo) \
2652({ \
2653 __sum16 __ret = 0; \
2654 if (__skb_gro_checksum_validate_needed(skb, zero_okay, check)) \
2655 __ret = __skb_gro_checksum_validate_complete(skb, \
2656 compute_pseudo(skb, proto)); \
Davide Caratti219f1d792017-05-18 15:44:39 +02002657 if (!__ret) \
Tom Herbert573e8fc2014-08-22 13:33:47 -07002658 skb_gro_incr_csum_unnecessary(skb); \
2659 __ret; \
2660})
2661
2662#define skb_gro_checksum_validate(skb, proto, compute_pseudo) \
2663 __skb_gro_checksum_validate(skb, proto, false, 0, compute_pseudo)
2664
2665#define skb_gro_checksum_validate_zero_check(skb, proto, check, \
2666 compute_pseudo) \
2667 __skb_gro_checksum_validate(skb, proto, true, check, compute_pseudo)
2668
2669#define skb_gro_checksum_simple_validate(skb) \
2670 __skb_gro_checksum_validate(skb, 0, false, 0, null_compute_pseudo)
2671
Tom Herbertd96535a2014-08-31 15:12:42 -07002672static inline bool __skb_gro_checksum_convert_check(struct sk_buff *skb)
2673{
2674 return (NAPI_GRO_CB(skb)->csum_cnt == 0 &&
2675 !NAPI_GRO_CB(skb)->csum_valid);
2676}
2677
2678static inline void __skb_gro_checksum_convert(struct sk_buff *skb,
2679 __sum16 check, __wsum pseudo)
2680{
2681 NAPI_GRO_CB(skb)->csum = ~pseudo;
2682 NAPI_GRO_CB(skb)->csum_valid = 1;
2683}
2684
2685#define skb_gro_checksum_try_convert(skb, proto, check, compute_pseudo) \
2686do { \
2687 if (__skb_gro_checksum_convert_check(skb)) \
2688 __skb_gro_checksum_convert(skb, check, \
2689 compute_pseudo(skb, proto)); \
2690} while (0)
2691
Tom Herbert26c4f7d2015-02-10 16:30:27 -08002692struct gro_remcsum {
2693 int offset;
2694 __wsum delta;
2695};
2696
2697static inline void skb_gro_remcsum_init(struct gro_remcsum *grc)
2698{
Geert Uytterhoeven846cd662015-02-18 11:38:06 +01002699 grc->offset = 0;
Tom Herbert26c4f7d2015-02-10 16:30:27 -08002700 grc->delta = 0;
2701}
2702
Tom Herbertb7fe10e2015-08-19 17:07:32 -07002703static inline void *skb_gro_remcsum_process(struct sk_buff *skb, void *ptr,
2704 unsigned int off, size_t hdrlen,
2705 int start, int offset,
2706 struct gro_remcsum *grc,
2707 bool nopartial)
Tom Herbertdcdc8992015-02-02 16:07:34 -08002708{
2709 __wsum delta;
Tom Herbertb7fe10e2015-08-19 17:07:32 -07002710 size_t plen = hdrlen + max_t(size_t, offset + sizeof(u16), start);
Tom Herbertdcdc8992015-02-02 16:07:34 -08002711
2712 BUG_ON(!NAPI_GRO_CB(skb)->csum_valid);
2713
Tom Herbert15e23962015-02-10 16:30:31 -08002714 if (!nopartial) {
Tom Herbertb7fe10e2015-08-19 17:07:32 -07002715 NAPI_GRO_CB(skb)->gro_remcsum_start = off + hdrlen + start;
2716 return ptr;
Tom Herbert15e23962015-02-10 16:30:31 -08002717 }
2718
Tom Herbertb7fe10e2015-08-19 17:07:32 -07002719 ptr = skb_gro_header_fast(skb, off);
2720 if (skb_gro_header_hard(skb, off + plen)) {
2721 ptr = skb_gro_header_slow(skb, off + plen, off);
2722 if (!ptr)
2723 return NULL;
2724 }
2725
2726 delta = remcsum_adjust(ptr + hdrlen, NAPI_GRO_CB(skb)->csum,
2727 start, offset);
Tom Herbertdcdc8992015-02-02 16:07:34 -08002728
2729 /* Adjust skb->csum since we changed the packet */
Tom Herbertdcdc8992015-02-02 16:07:34 -08002730 NAPI_GRO_CB(skb)->csum = csum_add(NAPI_GRO_CB(skb)->csum, delta);
Tom Herbert26c4f7d2015-02-10 16:30:27 -08002731
Tom Herbertb7fe10e2015-08-19 17:07:32 -07002732 grc->offset = off + hdrlen + offset;
Tom Herbert26c4f7d2015-02-10 16:30:27 -08002733 grc->delta = delta;
Tom Herbertb7fe10e2015-08-19 17:07:32 -07002734
2735 return ptr;
Tom Herbertdcdc8992015-02-02 16:07:34 -08002736}
2737
Tom Herbert26c4f7d2015-02-10 16:30:27 -08002738static inline void skb_gro_remcsum_cleanup(struct sk_buff *skb,
2739 struct gro_remcsum *grc)
2740{
Tom Herbertb7fe10e2015-08-19 17:07:32 -07002741 void *ptr;
2742 size_t plen = grc->offset + sizeof(u16);
2743
Tom Herbert26c4f7d2015-02-10 16:30:27 -08002744 if (!grc->delta)
2745 return;
2746
Tom Herbertb7fe10e2015-08-19 17:07:32 -07002747 ptr = skb_gro_header_fast(skb, grc->offset);
2748 if (skb_gro_header_hard(skb, grc->offset + sizeof(u16))) {
2749 ptr = skb_gro_header_slow(skb, plen, grc->offset);
2750 if (!ptr)
2751 return;
2752 }
2753
2754 remcsum_unadjust((__sum16 *)ptr, grc->delta);
Tom Herbert26c4f7d2015-02-10 16:30:27 -08002755}
Tom Herbertdcdc8992015-02-02 16:07:34 -08002756
Steffen Klassert25393d32017-02-15 09:39:44 +01002757#ifdef CONFIG_XFRM_OFFLOAD
2758static inline void skb_gro_flush_final(struct sk_buff *skb, struct sk_buff **pp, int flush)
2759{
2760 if (PTR_ERR(pp) != -EINPROGRESS)
2761 NAPI_GRO_CB(skb)->flush |= flush;
2762}
2763#else
Steffen Klassert5f114162017-02-15 09:39:39 +01002764static inline void skb_gro_flush_final(struct sk_buff *skb, struct sk_buff **pp, int flush)
2765{
2766 NAPI_GRO_CB(skb)->flush |= flush;
2767}
Steffen Klassert25393d32017-02-15 09:39:44 +01002768#endif
Steffen Klassert5f114162017-02-15 09:39:39 +01002769
Stephen Hemminger0c4e8582007-10-09 01:36:32 -07002770static inline int dev_hard_header(struct sk_buff *skb, struct net_device *dev,
2771 unsigned short type,
Stephen Hemminger3b04ddd2007-10-09 01:40:57 -07002772 const void *daddr, const void *saddr,
Eric Dumazet95c96172012-04-15 05:58:06 +00002773 unsigned int len)
Stephen Hemminger0c4e8582007-10-09 01:36:32 -07002774{
Ursula Braunf1ecfd52007-10-22 16:16:14 +02002775 if (!dev->header_ops || !dev->header_ops->create)
Stephen Hemminger0c4e8582007-10-09 01:36:32 -07002776 return 0;
Stephen Hemminger3b04ddd2007-10-09 01:40:57 -07002777
2778 return dev->header_ops->create(skb, dev, type, daddr, saddr, len);
Stephen Hemminger0c4e8582007-10-09 01:36:32 -07002779}
2780
Stephen Hemmingerb95cce32007-09-26 22:13:38 -07002781static inline int dev_parse_header(const struct sk_buff *skb,
2782 unsigned char *haddr)
2783{
2784 const struct net_device *dev = skb->dev;
2785
Patrick McHardy1b833362007-10-18 05:09:28 -07002786 if (!dev->header_ops || !dev->header_ops->parse)
Stephen Hemmingerb95cce32007-09-26 22:13:38 -07002787 return 0;
Stephen Hemminger3b04ddd2007-10-09 01:40:57 -07002788 return dev->header_ops->parse(skb, haddr);
Stephen Hemmingerb95cce32007-09-26 22:13:38 -07002789}
2790
Willem de Bruijn2793a232016-03-09 21:58:32 -05002791/* ll_header must have at least hard_header_len allocated */
2792static inline bool dev_validate_header(const struct net_device *dev,
2793 char *ll_header, int len)
2794{
2795 if (likely(len >= dev->hard_header_len))
2796 return true;
Willem de Bruijn217e6fa2017-02-07 15:57:20 -05002797 if (len < dev->min_header_len)
2798 return false;
Willem de Bruijn2793a232016-03-09 21:58:32 -05002799
2800 if (capable(CAP_SYS_RAWIO)) {
2801 memset(ll_header + len, 0, dev->hard_header_len - len);
2802 return true;
2803 }
2804
2805 if (dev->header_ops && dev->header_ops->validate)
2806 return dev->header_ops->validate(ll_header, len);
2807
2808 return false;
2809}
2810
Al Viro36fd6332017-06-26 13:19:16 -04002811typedef int gifconf_func_t(struct net_device * dev, char __user * bufptr,
2812 int len, int size);
Joe Perchesf629d202013-09-26 14:48:15 -07002813int register_gifconf(unsigned int family, gifconf_func_t *gifconf);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002814static inline int unregister_gifconf(unsigned int family)
2815{
2816 return register_gifconf(family, NULL);
2817}
2818
Willem de Bruijn99bbc702013-05-20 04:02:32 +00002819#ifdef CONFIG_NET_FLOW_LIMIT
Willem de Bruijn5f121b92013-06-13 15:29:38 -04002820#define FLOW_LIMIT_HISTORY (1 << 7) /* must be ^2 and !overflow buckets */
Willem de Bruijn99bbc702013-05-20 04:02:32 +00002821struct sd_flow_limit {
2822 u64 count;
2823 unsigned int num_buckets;
2824 unsigned int history_head;
2825 u16 history[FLOW_LIMIT_HISTORY];
2826 u8 buckets[];
2827};
2828
2829extern int netdev_flow_limit_table_len;
2830#endif /* CONFIG_NET_FLOW_LIMIT */
2831
Linus Torvalds1da177e2005-04-16 15:20:36 -07002832/*
Bjorn Helgaas5e82b4b2016-03-23 13:47:23 -05002833 * Incoming packets are placed on per-CPU queues
Linus Torvalds1da177e2005-04-16 15:20:36 -07002834 */
Eric Dumazetd94d9fe2009-11-04 09:50:58 -08002835struct softnet_data {
Linus Torvalds1da177e2005-04-16 15:20:36 -07002836 struct list_head poll_list;
Changli Gao6e7676c2010-04-27 15:07:33 -07002837 struct sk_buff_head process_queue;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002838
Changli Gaodee42872010-05-02 05:42:16 +00002839 /* stats */
David S. Millercd7b5392010-05-02 22:27:59 -07002840 unsigned int processed;
2841 unsigned int time_squeeze;
David S. Millercd7b5392010-05-02 22:27:59 -07002842 unsigned int received_rps;
Changli Gaofd793d82010-04-15 00:16:59 -07002843#ifdef CONFIG_RPS
Eric Dumazet88751272010-04-19 05:07:33 +00002844 struct softnet_data *rps_ipi_list;
Eric Dumazet4cdb1e22014-11-02 06:00:12 -08002845#endif
2846#ifdef CONFIG_NET_FLOW_LIMIT
2847 struct sd_flow_limit __rcu *flow_limit;
2848#endif
2849 struct Qdisc *output_queue;
2850 struct Qdisc **output_queue_tailp;
2851 struct sk_buff *completion_queue;
Steffen Klassertf53c7232017-12-20 10:41:36 +01002852#ifdef CONFIG_XFRM_OFFLOAD
2853 struct sk_buff_head xfrm_backlog;
2854#endif
Eric Dumazet4cdb1e22014-11-02 06:00:12 -08002855#ifdef CONFIG_RPS
Eric Dumazet501e7ef2016-04-26 15:30:07 -07002856 /* input_queue_head should be written by cpu owning this struct,
2857 * and only read by other cpus. Worth using a cache line.
2858 */
2859 unsigned int input_queue_head ____cacheline_aligned_in_smp;
2860
2861 /* Elements below can be accessed between CPUs for RPS/RFS */
Ying Huang966a9672017-08-08 12:30:00 +08002862 call_single_data_t csd ____cacheline_aligned_in_smp;
Eric Dumazet88751272010-04-19 05:07:33 +00002863 struct softnet_data *rps_ipi_next;
2864 unsigned int cpu;
Tom Herbert76cc8b12010-05-20 18:37:59 +00002865 unsigned int input_queue_tail;
Tom Herbert1e94d722010-03-18 17:45:44 -07002866#endif
Eric Dumazet95c96172012-04-15 05:58:06 +00002867 unsigned int dropped;
Tom Herbert0a9627f2010-03-16 08:03:29 +00002868 struct sk_buff_head input_pkt_queue;
Stephen Hemmingerbea33482007-10-03 16:41:36 -07002869 struct napi_struct backlog;
Willem de Bruijn99bbc702013-05-20 04:02:32 +00002870
Linus Torvalds1da177e2005-04-16 15:20:36 -07002871};
2872
Tom Herbert76cc8b12010-05-20 18:37:59 +00002873static inline void input_queue_head_incr(struct softnet_data *sd)
Tom Herbertfec5e652010-04-16 16:01:27 -07002874{
2875#ifdef CONFIG_RPS
Tom Herbert76cc8b12010-05-20 18:37:59 +00002876 sd->input_queue_head++;
2877#endif
2878}
2879
2880static inline void input_queue_tail_incr_save(struct softnet_data *sd,
2881 unsigned int *qtail)
2882{
2883#ifdef CONFIG_RPS
2884 *qtail = ++sd->input_queue_tail;
Tom Herbertfec5e652010-04-16 16:01:27 -07002885#endif
2886}
2887
Tom Herbert0a9627f2010-03-16 08:03:29 +00002888DECLARE_PER_CPU_ALIGNED(struct softnet_data, softnet_data);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002889
Joe Perchesf629d202013-09-26 14:48:15 -07002890void __netif_schedule(struct Qdisc *q);
John Fastabend46e5da40a2014-09-12 20:04:52 -07002891void netif_schedule_queue(struct netdev_queue *txq);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002892
David S. Millerfd2ea0a2008-07-17 01:56:23 -07002893static inline void netif_tx_schedule_all(struct net_device *dev)
2894{
2895 unsigned int i;
2896
2897 for (i = 0; i < dev->num_tx_queues; i++)
2898 netif_schedule_queue(netdev_get_tx_queue(dev, i));
2899}
2900
Denys Vlasenkof9a7cbb2016-04-08 17:51:54 +02002901static __always_inline void netif_tx_start_queue(struct netdev_queue *dev_queue)
Dave Jonesd29f7492008-07-22 14:09:06 -07002902{
Tom Herbert734664982011-11-28 16:32:44 +00002903 clear_bit(__QUEUE_STATE_DRV_XOFF, &dev_queue->state);
Dave Jonesd29f7492008-07-22 14:09:06 -07002904}
2905
Stephen Hemmingerbea33482007-10-03 16:41:36 -07002906/**
2907 * netif_start_queue - allow transmit
2908 * @dev: network device
2909 *
2910 * Allow upper layers to call the device hard_start_xmit routine.
2911 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07002912static inline void netif_start_queue(struct net_device *dev)
2913{
David S. Millere8a04642008-07-17 00:34:19 -07002914 netif_tx_start_queue(netdev_get_tx_queue(dev, 0));
Linus Torvalds1da177e2005-04-16 15:20:36 -07002915}
2916
David S. Millerfd2ea0a2008-07-17 01:56:23 -07002917static inline void netif_tx_start_all_queues(struct net_device *dev)
2918{
2919 unsigned int i;
2920
2921 for (i = 0; i < dev->num_tx_queues; i++) {
2922 struct netdev_queue *txq = netdev_get_tx_queue(dev, i);
2923 netif_tx_start_queue(txq);
2924 }
2925}
2926
John Fastabend46e5da40a2014-09-12 20:04:52 -07002927void netif_tx_wake_queue(struct netdev_queue *dev_queue);
David S. Miller79d16382008-07-08 23:14:46 -07002928
Dave Jonesd29f7492008-07-22 14:09:06 -07002929/**
2930 * netif_wake_queue - restart transmit
2931 * @dev: network device
2932 *
2933 * Allow upper layers to call the device hard_start_xmit routine.
2934 * Used for flow control when transmit resources are available.
2935 */
David S. Miller79d16382008-07-08 23:14:46 -07002936static inline void netif_wake_queue(struct net_device *dev)
2937{
David S. Millere8a04642008-07-17 00:34:19 -07002938 netif_tx_wake_queue(netdev_get_tx_queue(dev, 0));
Linus Torvalds1da177e2005-04-16 15:20:36 -07002939}
2940
David S. Millerfd2ea0a2008-07-17 01:56:23 -07002941static inline void netif_tx_wake_all_queues(struct net_device *dev)
2942{
2943 unsigned int i;
2944
2945 for (i = 0; i < dev->num_tx_queues; i++) {
2946 struct netdev_queue *txq = netdev_get_tx_queue(dev, i);
2947 netif_tx_wake_queue(txq);
2948 }
2949}
2950
Denys Vlasenkof9a7cbb2016-04-08 17:51:54 +02002951static __always_inline void netif_tx_stop_queue(struct netdev_queue *dev_queue)
Dave Jonesd29f7492008-07-22 14:09:06 -07002952{
Tom Herbert734664982011-11-28 16:32:44 +00002953 set_bit(__QUEUE_STATE_DRV_XOFF, &dev_queue->state);
Dave Jonesd29f7492008-07-22 14:09:06 -07002954}
2955
Stephen Hemmingerbea33482007-10-03 16:41:36 -07002956/**
2957 * netif_stop_queue - stop transmitted packets
2958 * @dev: network device
2959 *
2960 * Stop upper layers calling the device hard_start_xmit routine.
2961 * Used for flow control when transmit resources are unavailable.
2962 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07002963static inline void netif_stop_queue(struct net_device *dev)
2964{
David S. Millere8a04642008-07-17 00:34:19 -07002965 netif_tx_stop_queue(netdev_get_tx_queue(dev, 0));
Linus Torvalds1da177e2005-04-16 15:20:36 -07002966}
2967
Denys Vlasenkoa2029242015-05-11 21:17:53 +02002968void netif_tx_stop_all_queues(struct net_device *dev);
David S. Millerfd2ea0a2008-07-17 01:56:23 -07002969
David S. Miller4d295152012-03-07 21:02:35 -05002970static inline bool netif_tx_queue_stopped(const struct netdev_queue *dev_queue)
Dave Jonesd29f7492008-07-22 14:09:06 -07002971{
Tom Herbert734664982011-11-28 16:32:44 +00002972 return test_bit(__QUEUE_STATE_DRV_XOFF, &dev_queue->state);
Dave Jonesd29f7492008-07-22 14:09:06 -07002973}
2974
Stephen Hemmingerbea33482007-10-03 16:41:36 -07002975/**
2976 * netif_queue_stopped - test if transmit queue is flowblocked
2977 * @dev: network device
2978 *
2979 * Test if transmit queue on device is currently unable to send.
2980 */
David S. Miller4d295152012-03-07 21:02:35 -05002981static inline bool netif_queue_stopped(const struct net_device *dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002982{
David S. Millere8a04642008-07-17 00:34:19 -07002983 return netif_tx_queue_stopped(netdev_get_tx_queue(dev, 0));
Linus Torvalds1da177e2005-04-16 15:20:36 -07002984}
2985
David S. Miller4d295152012-03-07 21:02:35 -05002986static inline bool netif_xmit_stopped(const struct netdev_queue *dev_queue)
David S. Millerc3f26a22008-07-31 16:58:50 -07002987{
Tom Herbert734664982011-11-28 16:32:44 +00002988 return dev_queue->state & QUEUE_STATE_ANY_XOFF;
2989}
2990
Daniel Borkmann8e2f1a62014-04-02 20:52:57 +02002991static inline bool
2992netif_xmit_frozen_or_stopped(const struct netdev_queue *dev_queue)
Tom Herbert734664982011-11-28 16:32:44 +00002993{
2994 return dev_queue->state & QUEUE_STATE_ANY_XOFF_OR_FROZEN;
2995}
2996
Daniel Borkmann8e2f1a62014-04-02 20:52:57 +02002997static inline bool
2998netif_xmit_frozen_or_drv_stopped(const struct netdev_queue *dev_queue)
2999{
3000 return dev_queue->state & QUEUE_STATE_DRV_XOFF_OR_FROZEN;
3001}
3002
Eric Dumazet53511452014-10-08 08:19:27 -07003003/**
3004 * netdev_txq_bql_enqueue_prefetchw - prefetch bql data for write
3005 * @dev_queue: pointer to transmit queue
3006 *
3007 * BQL enabled drivers might use this helper in their ndo_start_xmit(),
Bjorn Helgaas5e82b4b2016-03-23 13:47:23 -05003008 * to give appropriate hint to the CPU.
Eric Dumazet53511452014-10-08 08:19:27 -07003009 */
3010static inline void netdev_txq_bql_enqueue_prefetchw(struct netdev_queue *dev_queue)
3011{
3012#ifdef CONFIG_BQL
3013 prefetchw(&dev_queue->dql.num_queued);
3014#endif
3015}
3016
3017/**
3018 * netdev_txq_bql_complete_prefetchw - prefetch bql data for write
3019 * @dev_queue: pointer to transmit queue
3020 *
3021 * BQL enabled drivers might use this helper in their TX completion path,
Bjorn Helgaas5e82b4b2016-03-23 13:47:23 -05003022 * to give appropriate hint to the CPU.
Eric Dumazet53511452014-10-08 08:19:27 -07003023 */
3024static inline void netdev_txq_bql_complete_prefetchw(struct netdev_queue *dev_queue)
3025{
3026#ifdef CONFIG_BQL
3027 prefetchw(&dev_queue->dql.limit);
3028#endif
3029}
3030
Tom Herbertc5d67bd2011-11-28 16:32:52 +00003031static inline void netdev_tx_sent_queue(struct netdev_queue *dev_queue,
3032 unsigned int bytes)
3033{
Tom Herbert114cf582011-11-28 16:33:09 +00003034#ifdef CONFIG_BQL
3035 dql_queued(&dev_queue->dql, bytes);
Alexander Duyckb37c0fb2012-02-07 02:29:06 +00003036
3037 if (likely(dql_avail(&dev_queue->dql) >= 0))
3038 return;
3039
3040 set_bit(__QUEUE_STATE_STACK_XOFF, &dev_queue->state);
3041
3042 /*
3043 * The XOFF flag must be set before checking the dql_avail below,
3044 * because in netdev_tx_completed_queue we update the dql_completed
3045 * before checking the XOFF flag.
3046 */
3047 smp_mb();
3048
3049 /* check again in case another CPU has just made room avail */
3050 if (unlikely(dql_avail(&dev_queue->dql) >= 0))
3051 clear_bit(__QUEUE_STATE_STACK_XOFF, &dev_queue->state);
Tom Herbert114cf582011-11-28 16:33:09 +00003052#endif
Tom Herbertc5d67bd2011-11-28 16:32:52 +00003053}
3054
Florian Fainelli0042d0c2013-09-06 16:58:00 +01003055/**
3056 * netdev_sent_queue - report the number of bytes queued to hardware
3057 * @dev: network device
3058 * @bytes: number of bytes queued to the hardware device queue
3059 *
3060 * Report the number of bytes queued for sending/completion to the network
3061 * device hardware queue. @bytes should be a good approximation and should
3062 * exactly match netdev_completed_queue() @bytes
3063 */
Tom Herbertc5d67bd2011-11-28 16:32:52 +00003064static inline void netdev_sent_queue(struct net_device *dev, unsigned int bytes)
3065{
3066 netdev_tx_sent_queue(netdev_get_tx_queue(dev, 0), bytes);
3067}
3068
3069static inline void netdev_tx_completed_queue(struct netdev_queue *dev_queue,
Eric Dumazet95c96172012-04-15 05:58:06 +00003070 unsigned int pkts, unsigned int bytes)
Tom Herbertc5d67bd2011-11-28 16:32:52 +00003071{
Tom Herbert114cf582011-11-28 16:33:09 +00003072#ifdef CONFIG_BQL
Alexander Duyckb37c0fb2012-02-07 02:29:06 +00003073 if (unlikely(!bytes))
3074 return;
3075
3076 dql_completed(&dev_queue->dql, bytes);
3077
3078 /*
3079 * Without the memory barrier there is a small possiblity that
3080 * netdev_tx_sent_queue will miss the update and cause the queue to
3081 * be stopped forever
3082 */
3083 smp_mb();
3084
3085 if (dql_avail(&dev_queue->dql) < 0)
3086 return;
3087
3088 if (test_and_clear_bit(__QUEUE_STATE_STACK_XOFF, &dev_queue->state))
3089 netif_schedule_queue(dev_queue);
Tom Herbert114cf582011-11-28 16:33:09 +00003090#endif
Tom Herbertc5d67bd2011-11-28 16:32:52 +00003091}
3092
Florian Fainelli0042d0c2013-09-06 16:58:00 +01003093/**
3094 * netdev_completed_queue - report bytes and packets completed by device
3095 * @dev: network device
3096 * @pkts: actual number of packets sent over the medium
3097 * @bytes: actual number of bytes sent over the medium
3098 *
3099 * Report the number of bytes and packets transmitted by the network device
3100 * hardware queue over the physical medium, @bytes must exactly match the
3101 * @bytes amount passed to netdev_sent_queue()
3102 */
Tom Herbertc5d67bd2011-11-28 16:32:52 +00003103static inline void netdev_completed_queue(struct net_device *dev,
Eric Dumazet95c96172012-04-15 05:58:06 +00003104 unsigned int pkts, unsigned int bytes)
Tom Herbertc5d67bd2011-11-28 16:32:52 +00003105{
3106 netdev_tx_completed_queue(netdev_get_tx_queue(dev, 0), pkts, bytes);
3107}
3108
3109static inline void netdev_tx_reset_queue(struct netdev_queue *q)
3110{
Tom Herbert114cf582011-11-28 16:33:09 +00003111#ifdef CONFIG_BQL
Alexander Duyck5c490352012-02-07 02:29:01 +00003112 clear_bit(__QUEUE_STATE_STACK_XOFF, &q->state);
Tom Herbert114cf582011-11-28 16:33:09 +00003113 dql_reset(&q->dql);
3114#endif
Tom Herbertc5d67bd2011-11-28 16:32:52 +00003115}
3116
Florian Fainelli0042d0c2013-09-06 16:58:00 +01003117/**
3118 * netdev_reset_queue - reset the packets and bytes count of a network device
3119 * @dev_queue: network device
3120 *
3121 * Reset the bytes and packet count of a network device and clear the
3122 * software flow control OFF bit for this network device
3123 */
Tom Herbertc5d67bd2011-11-28 16:32:52 +00003124static inline void netdev_reset_queue(struct net_device *dev_queue)
3125{
3126 netdev_tx_reset_queue(netdev_get_tx_queue(dev_queue, 0));
David S. Millerc3f26a22008-07-31 16:58:50 -07003127}
3128
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003129/**
Daniel Borkmannb9507bd2014-02-16 15:55:21 +01003130 * netdev_cap_txqueue - check if selected tx queue exceeds device queues
3131 * @dev: network device
3132 * @queue_index: given tx queue index
3133 *
3134 * Returns 0 if given tx queue index >= number of device tx queues,
3135 * otherwise returns the originally passed tx queue index.
3136 */
3137static inline u16 netdev_cap_txqueue(struct net_device *dev, u16 queue_index)
3138{
3139 if (unlikely(queue_index >= dev->real_num_tx_queues)) {
3140 net_warn_ratelimited("%s selects TX queue %d, but real number of TX queues is %d\n",
3141 dev->name, queue_index,
3142 dev->real_num_tx_queues);
3143 return 0;
3144 }
3145
3146 return queue_index;
3147}
3148
3149/**
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003150 * netif_running - test if up
3151 * @dev: network device
3152 *
3153 * Test if the device has been brought up.
3154 */
David S. Miller4d295152012-03-07 21:02:35 -05003155static inline bool netif_running(const struct net_device *dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003156{
3157 return test_bit(__LINK_STATE_START, &dev->state);
3158}
3159
Peter P Waskiewicz Jrf25f4e42007-07-06 13:36:20 -07003160/*
Bjorn Helgaas5e82b4b2016-03-23 13:47:23 -05003161 * Routines to manage the subqueues on a device. We only need start,
Peter P Waskiewicz Jrf25f4e42007-07-06 13:36:20 -07003162 * stop, and a check if it's stopped. All other device management is
3163 * done at the overall netdevice level.
3164 * Also test the device if we're multiqueue.
3165 */
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003166
3167/**
3168 * netif_start_subqueue - allow sending packets on subqueue
3169 * @dev: network device
3170 * @queue_index: sub queue index
3171 *
3172 * Start individual transmit queue of a device with multiple transmit queues.
3173 */
Peter P Waskiewicz Jrf25f4e42007-07-06 13:36:20 -07003174static inline void netif_start_subqueue(struct net_device *dev, u16 queue_index)
3175{
David S. Millerfd2ea0a2008-07-17 01:56:23 -07003176 struct netdev_queue *txq = netdev_get_tx_queue(dev, queue_index);
Krishna Kumar7b3d3e42009-08-29 20:21:21 +00003177
3178 netif_tx_start_queue(txq);
Peter P Waskiewicz Jrf25f4e42007-07-06 13:36:20 -07003179}
3180
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003181/**
3182 * netif_stop_subqueue - stop sending packets on subqueue
3183 * @dev: network device
3184 * @queue_index: sub queue index
3185 *
3186 * Stop individual transmit queue of a device with multiple transmit queues.
3187 */
Peter P Waskiewicz Jrf25f4e42007-07-06 13:36:20 -07003188static inline void netif_stop_subqueue(struct net_device *dev, u16 queue_index)
3189{
David S. Millerfd2ea0a2008-07-17 01:56:23 -07003190 struct netdev_queue *txq = netdev_get_tx_queue(dev, queue_index);
Krishna Kumar7b3d3e42009-08-29 20:21:21 +00003191 netif_tx_stop_queue(txq);
Peter P Waskiewicz Jrf25f4e42007-07-06 13:36:20 -07003192}
3193
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003194/**
3195 * netif_subqueue_stopped - test status of subqueue
3196 * @dev: network device
3197 * @queue_index: sub queue index
3198 *
3199 * Check individual transmit queue of a device with multiple transmit queues.
3200 */
David S. Miller4d295152012-03-07 21:02:35 -05003201static inline bool __netif_subqueue_stopped(const struct net_device *dev,
3202 u16 queue_index)
Peter P Waskiewicz Jrf25f4e42007-07-06 13:36:20 -07003203{
David S. Millerfd2ea0a2008-07-17 01:56:23 -07003204 struct netdev_queue *txq = netdev_get_tx_queue(dev, queue_index);
Krishna Kumar7b3d3e42009-08-29 20:21:21 +00003205
3206 return netif_tx_queue_stopped(txq);
Peter P Waskiewicz Jrf25f4e42007-07-06 13:36:20 -07003207}
3208
David S. Miller4d295152012-03-07 21:02:35 -05003209static inline bool netif_subqueue_stopped(const struct net_device *dev,
3210 struct sk_buff *skb)
Pavel Emelyanov668f8952007-10-21 17:01:56 -07003211{
3212 return __netif_subqueue_stopped(dev, skb_get_queue_mapping(skb));
3213}
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003214
Florian Fainelli738b35c2017-01-11 21:13:02 -08003215/**
3216 * netif_wake_subqueue - allow sending packets on subqueue
3217 * @dev: network device
3218 * @queue_index: sub queue index
3219 *
3220 * Resume individual transmit queue of a device with multiple transmit queues.
3221 */
3222static inline void netif_wake_subqueue(struct net_device *dev, u16 queue_index)
3223{
3224 struct netdev_queue *txq = netdev_get_tx_queue(dev, queue_index);
3225
3226 netif_tx_wake_queue(txq);
3227}
Peter P Waskiewicz Jrf25f4e42007-07-06 13:36:20 -07003228
Alexander Duyck537c00d2013-01-10 08:57:02 +00003229#ifdef CONFIG_XPS
David S. Miller53af53a2013-10-08 23:07:53 -04003230int netif_set_xps_queue(struct net_device *dev, const struct cpumask *mask,
Joe Perchesf629d202013-09-26 14:48:15 -07003231 u16 index);
Alexander Duyck537c00d2013-01-10 08:57:02 +00003232#else
3233static inline int netif_set_xps_queue(struct net_device *dev,
Michael S. Tsirkin35735402013-10-02 09:14:06 +03003234 const struct cpumask *mask,
Alexander Duyck537c00d2013-01-10 08:57:02 +00003235 u16 index)
3236{
3237 return 0;
3238}
3239#endif
3240
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003241/**
3242 * netif_is_multiqueue - test if device has multiple transmit queues
3243 * @dev: network device
3244 *
3245 * Check if device has multiple transmit queues
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003246 */
David S. Miller4d295152012-03-07 21:02:35 -05003247static inline bool netif_is_multiqueue(const struct net_device *dev)
Peter P Waskiewicz Jrf25f4e42007-07-06 13:36:20 -07003248{
Eric Dumazeta02cec22010-09-22 20:43:57 +00003249 return dev->num_tx_queues > 1;
Peter P Waskiewicz Jrf25f4e42007-07-06 13:36:20 -07003250}
Linus Torvalds1da177e2005-04-16 15:20:36 -07003251
Joe Perchesf629d202013-09-26 14:48:15 -07003252int netif_set_real_num_tx_queues(struct net_device *dev, unsigned int txq);
John Fastabendf0796d52010-07-01 13:21:57 +00003253
Michael Daltona953be52014-01-16 22:23:28 -08003254#ifdef CONFIG_SYSFS
Joe Perchesf629d202013-09-26 14:48:15 -07003255int netif_set_real_num_rx_queues(struct net_device *dev, unsigned int rxq);
Ben Hutchings62fe0b42010-09-27 08:24:33 +00003256#else
3257static inline int netif_set_real_num_rx_queues(struct net_device *dev,
3258 unsigned int rxq)
3259{
3260 return 0;
3261}
3262#endif
3263
Daniel Borkmann65073a62018-01-31 12:58:56 +01003264static inline struct netdev_rx_queue *
3265__netif_get_rx_queue(struct net_device *dev, unsigned int rxq)
3266{
3267 return dev->_rx + rxq;
3268}
3269
Michael Daltona953be52014-01-16 22:23:28 -08003270#ifdef CONFIG_SYSFS
3271static inline unsigned int get_netdev_rx_queue_index(
3272 struct netdev_rx_queue *queue)
3273{
3274 struct net_device *dev = queue->dev;
3275 int index = queue - dev->_rx;
3276
3277 BUG_ON(index >= dev->num_rx_queues);
3278 return index;
3279}
3280#endif
3281
Yuval Mintz16917b82012-07-01 03:18:50 +00003282#define DEFAULT_MAX_NUM_RSS_QUEUES (8)
Joe Perchesf629d202013-09-26 14:48:15 -07003283int netif_get_num_default_rss_queues(void);
Yuval Mintz16917b82012-07-01 03:18:50 +00003284
Eric Dumazete6247022013-12-05 04:45:08 -08003285enum skb_free_reason {
3286 SKB_REASON_CONSUMED,
3287 SKB_REASON_DROPPED,
3288};
Linus Torvalds1da177e2005-04-16 15:20:36 -07003289
Eric Dumazete6247022013-12-05 04:45:08 -08003290void __dev_kfree_skb_irq(struct sk_buff *skb, enum skb_free_reason reason);
3291void __dev_kfree_skb_any(struct sk_buff *skb, enum skb_free_reason reason);
3292
3293/*
3294 * It is not allowed to call kfree_skb() or consume_skb() from hardware
3295 * interrupt context or with hardware interrupts being disabled.
3296 * (in_irq() || irqs_disabled())
3297 *
3298 * We provide four helpers that can be used in following contexts :
3299 *
3300 * dev_kfree_skb_irq(skb) when caller drops a packet from irq context,
3301 * replacing kfree_skb(skb)
3302 *
3303 * dev_consume_skb_irq(skb) when caller consumes a packet from irq context.
3304 * Typically used in place of consume_skb(skb) in TX completion path
3305 *
3306 * dev_kfree_skb_any(skb) when caller doesn't know its current irq context,
3307 * replacing kfree_skb(skb)
3308 *
3309 * dev_consume_skb_any(skb) when caller doesn't know its current irq context,
3310 * and consumed a packet. Used in place of consume_skb(skb)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003311 */
Eric Dumazete6247022013-12-05 04:45:08 -08003312static inline void dev_kfree_skb_irq(struct sk_buff *skb)
3313{
3314 __dev_kfree_skb_irq(skb, SKB_REASON_DROPPED);
3315}
3316
3317static inline void dev_consume_skb_irq(struct sk_buff *skb)
3318{
3319 __dev_kfree_skb_irq(skb, SKB_REASON_CONSUMED);
3320}
3321
3322static inline void dev_kfree_skb_any(struct sk_buff *skb)
3323{
3324 __dev_kfree_skb_any(skb, SKB_REASON_DROPPED);
3325}
3326
3327static inline void dev_consume_skb_any(struct sk_buff *skb)
3328{
3329 __dev_kfree_skb_any(skb, SKB_REASON_CONSUMED);
3330}
Linus Torvalds1da177e2005-04-16 15:20:36 -07003331
Jason Wang7c497472017-08-11 19:41:17 +08003332void generic_xdp_tx(struct sk_buff *skb, struct bpf_prog *xdp_prog);
3333int do_xdp_generic(struct bpf_prog *xdp_prog, struct sk_buff *skb);
Joe Perchesf629d202013-09-26 14:48:15 -07003334int netif_rx(struct sk_buff *skb);
3335int netif_rx_ni(struct sk_buff *skb);
Eric W. Biederman04eb4482015-09-15 20:04:15 -05003336int netif_receive_skb(struct sk_buff *skb);
Jesper Dangaard Brouer1c601d82017-10-16 12:19:39 +02003337int netif_receive_skb_core(struct sk_buff *skb);
Joe Perchesf629d202013-09-26 14:48:15 -07003338gro_result_t napi_gro_receive(struct napi_struct *napi, struct sk_buff *skb);
3339void napi_gro_flush(struct napi_struct *napi, bool flush_old);
3340struct sk_buff *napi_get_frags(struct napi_struct *napi);
3341gro_result_t napi_gro_frags(struct napi_struct *napi);
Jerry Chubf5a7552014-01-07 10:23:19 -08003342struct packet_offload *gro_find_receive_by_type(__be16 type);
3343struct packet_offload *gro_find_complete_by_type(__be16 type);
Herbert Xu76620aa2009-04-16 02:02:07 -07003344
3345static inline void napi_free_frags(struct napi_struct *napi)
3346{
3347 kfree_skb(napi->skb);
3348 napi->skb = NULL;
3349}
3350
Mahesh Bandewar24b27fc2016-09-01 22:18:34 -07003351bool netdev_is_rx_handler_busy(struct net_device *dev);
Joe Perchesf629d202013-09-26 14:48:15 -07003352int netdev_rx_handler_register(struct net_device *dev,
3353 rx_handler_func_t *rx_handler,
3354 void *rx_handler_data);
3355void netdev_rx_handler_unregister(struct net_device *dev);
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00003356
Joe Perchesf629d202013-09-26 14:48:15 -07003357bool dev_valid_name(const char *name);
Al Viro44c02a22017-10-05 12:59:44 -04003358int dev_ioctl(struct net *net, unsigned int cmd, struct ifreq *ifr,
3359 bool *need_copyout);
Al Viro36fd6332017-06-26 13:19:16 -04003360int dev_ifconf(struct net *net, struct ifconf *, int);
Joe Perchesf629d202013-09-26 14:48:15 -07003361int dev_ethtool(struct net *net, struct ifreq *);
3362unsigned int dev_get_flags(const struct net_device *);
3363int __dev_change_flags(struct net_device *, unsigned int flags);
3364int dev_change_flags(struct net_device *, unsigned int);
David S. Millercb178192013-09-30 15:36:45 -04003365void __dev_notify_flags(struct net_device *, unsigned int old_flags,
3366 unsigned int gchanges);
Joe Perchesf629d202013-09-26 14:48:15 -07003367int dev_change_name(struct net_device *, const char *);
3368int dev_set_alias(struct net_device *, const char *, size_t);
Florian Westphal6c557002017-10-02 23:50:05 +02003369int dev_get_alias(const struct net_device *, char *, size_t);
Joe Perchesf629d202013-09-26 14:48:15 -07003370int dev_change_net_namespace(struct net_device *, struct net *, const char *);
WANG Congf51048c2017-07-06 15:01:57 -07003371int __dev_set_mtu(struct net_device *, int);
Joe Perchesf629d202013-09-26 14:48:15 -07003372int dev_set_mtu(struct net_device *, int);
Cong Wang6a643dd2018-01-25 18:26:22 -08003373int dev_change_tx_queue_len(struct net_device *, unsigned long);
Joe Perchesf629d202013-09-26 14:48:15 -07003374void dev_set_group(struct net_device *, int);
3375int dev_set_mac_address(struct net_device *, struct sockaddr *);
3376int dev_change_carrier(struct net_device *, bool new_carrier);
3377int dev_get_phys_port_id(struct net_device *dev,
Jiri Pirko02637fc2014-11-28 14:34:16 +01003378 struct netdev_phys_item_id *ppid);
David Aherndb24a902015-03-17 20:23:15 -06003379int dev_get_phys_port_name(struct net_device *dev,
3380 char *name, size_t len);
Anuradha Karuppiahd746d702015-07-14 13:43:19 -07003381int dev_change_proto_down(struct net_device *dev, bool proto_down);
Steffen Klassertf53c7232017-12-20 10:41:36 +01003382struct sk_buff *validate_xmit_skb_list(struct sk_buff *skb, struct net_device *dev, bool *again);
David S. Millerce937182014-08-30 19:22:20 -07003383struct sk_buff *dev_hard_start_xmit(struct sk_buff *skb, struct net_device *dev,
3384 struct netdev_queue *txq, int *ret);
Daniel Borkmannd67b9cd2017-05-12 01:04:46 +02003385
Jakub Kicinskif4e63522017-11-03 13:56:16 -07003386typedef int (*bpf_op_t)(struct net_device *dev, struct netdev_bpf *bpf);
Daniel Borkmannd67b9cd2017-05-12 01:04:46 +02003387int dev_change_xdp_fd(struct net_device *dev, struct netlink_ext_ack *extack,
3388 int fd, u32 flags);
Jakub Kicinski118b4aa2017-12-01 15:08:55 -08003389void __dev_xdp_query(struct net_device *dev, bpf_op_t xdp_op,
3390 struct netdev_bpf *xdp);
Daniel Borkmannd67b9cd2017-05-12 01:04:46 +02003391
Herbert Xua0265d22014-04-17 13:45:03 +08003392int __dev_forward_skb(struct net_device *dev, struct sk_buff *skb);
Joe Perchesf629d202013-09-26 14:48:15 -07003393int dev_forward_skb(struct net_device *dev, struct sk_buff *skb);
Nikolay Aleksandrovf4b05d22016-04-28 17:59:28 +02003394bool is_skb_forwardable(const struct net_device *dev,
3395 const struct sk_buff *skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003396
Martin KaFai Lau4e3264d2016-11-09 15:36:33 -08003397static __always_inline int ____dev_forward_skb(struct net_device *dev,
3398 struct sk_buff *skb)
3399{
3400 if (skb_orphan_frags(skb, GFP_ATOMIC) ||
3401 unlikely(!is_skb_forwardable(dev, skb))) {
3402 atomic_long_inc(&dev->rx_dropped);
3403 kfree_skb(skb);
3404 return NET_RX_DROP;
3405 }
3406
3407 skb_scrub_packet(skb, true);
3408 skb->priority = 0;
3409 return 0;
3410}
3411
David Ahern74b20582016-05-10 11:19:50 -07003412void dev_queue_xmit_nit(struct sk_buff *skb, struct net_device *dev);
3413
Arnaldo Carvalho de Melo20380732005-08-16 02:18:02 -03003414extern int netdev_budget;
Matthew Whitehead7acf8a12017-04-19 12:37:10 -04003415extern unsigned int netdev_budget_usecs;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003416
3417/* Called by rtnetlink.c:rtnl_unlock() */
Joe Perchesf629d202013-09-26 14:48:15 -07003418void netdev_run_todo(void);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003419
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003420/**
3421 * dev_put - release reference to device
3422 * @dev: network device
3423 *
Benjamin Thery9ef44292007-10-10 21:18:17 -07003424 * Release reference to device to allow it to be freed.
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003425 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07003426static inline void dev_put(struct net_device *dev)
3427{
Christoph Lameter933393f2011-12-22 11:58:51 -06003428 this_cpu_dec(*dev->pcpu_refcnt);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003429}
3430
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003431/**
3432 * dev_hold - get reference to device
3433 * @dev: network device
3434 *
Benjamin Thery9ef44292007-10-10 21:18:17 -07003435 * Hold reference to device to keep it from being freed.
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003436 */
Stephen Hemminger15333062006-03-20 22:32:28 -08003437static inline void dev_hold(struct net_device *dev)
3438{
Christoph Lameter933393f2011-12-22 11:58:51 -06003439 this_cpu_inc(*dev->pcpu_refcnt);
Stephen Hemminger15333062006-03-20 22:32:28 -08003440}
Linus Torvalds1da177e2005-04-16 15:20:36 -07003441
3442/* Carrier loss detection, dial on demand. The functions netif_carrier_on
3443 * and _off may be called from IRQ context, but it is caller
3444 * who is responsible for serialization of these calls.
Stefan Rompfb00055a2006-03-20 17:09:11 -08003445 *
3446 * The name carrier is inappropriate, these functions should really be
3447 * called netif_lowerlayer_*() because they represent the state of any
3448 * kind of lower layer not just hardware media.
Linus Torvalds1da177e2005-04-16 15:20:36 -07003449 */
3450
Joe Perchesf629d202013-09-26 14:48:15 -07003451void linkwatch_init_dev(struct net_device *dev);
3452void linkwatch_fire_event(struct net_device *dev);
3453void linkwatch_forget_dev(struct net_device *dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003454
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003455/**
3456 * netif_carrier_ok - test if carrier present
3457 * @dev: network device
3458 *
3459 * Check if carrier is present on device
3460 */
David S. Miller4d295152012-03-07 21:02:35 -05003461static inline bool netif_carrier_ok(const struct net_device *dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003462{
3463 return !test_bit(__LINK_STATE_NOCARRIER, &dev->state);
3464}
3465
Joe Perchesf629d202013-09-26 14:48:15 -07003466unsigned long dev_trans_start(struct net_device *dev);
Eric Dumazet9d214932009-05-17 20:55:16 -07003467
Joe Perchesf629d202013-09-26 14:48:15 -07003468void __netdev_watchdog_up(struct net_device *dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003469
Joe Perchesf629d202013-09-26 14:48:15 -07003470void netif_carrier_on(struct net_device *dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003471
Joe Perchesf629d202013-09-26 14:48:15 -07003472void netif_carrier_off(struct net_device *dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003473
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003474/**
3475 * netif_dormant_on - mark device as dormant.
3476 * @dev: network device
3477 *
3478 * Mark device as dormant (as per RFC2863).
3479 *
3480 * The dormant state indicates that the relevant interface is not
3481 * actually in a condition to pass packets (i.e., it is not 'up') but is
3482 * in a "pending" state, waiting for some external event. For "on-
3483 * demand" interfaces, this new state identifies the situation where the
3484 * interface is waiting for events to place it in the up state.
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003485 */
Stefan Rompfb00055a2006-03-20 17:09:11 -08003486static inline void netif_dormant_on(struct net_device *dev)
3487{
3488 if (!test_and_set_bit(__LINK_STATE_DORMANT, &dev->state))
3489 linkwatch_fire_event(dev);
3490}
3491
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003492/**
3493 * netif_dormant_off - set device as not dormant.
3494 * @dev: network device
3495 *
3496 * Device is not in dormant state.
3497 */
Stefan Rompfb00055a2006-03-20 17:09:11 -08003498static inline void netif_dormant_off(struct net_device *dev)
3499{
3500 if (test_and_clear_bit(__LINK_STATE_DORMANT, &dev->state))
3501 linkwatch_fire_event(dev);
3502}
3503
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003504/**
Zhang Shengju8ecbc402017-04-26 11:05:12 +08003505 * netif_dormant - test if device is dormant
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003506 * @dev: network device
3507 *
Zhang Shengju8ecbc402017-04-26 11:05:12 +08003508 * Check if device is dormant.
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003509 */
David S. Miller4d295152012-03-07 21:02:35 -05003510static inline bool netif_dormant(const struct net_device *dev)
Stefan Rompfb00055a2006-03-20 17:09:11 -08003511{
3512 return test_bit(__LINK_STATE_DORMANT, &dev->state);
3513}
3514
3515
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003516/**
3517 * netif_oper_up - test if device is operational
3518 * @dev: network device
3519 *
3520 * Check if carrier is operational
3521 */
David S. Miller4d295152012-03-07 21:02:35 -05003522static inline bool netif_oper_up(const struct net_device *dev)
Eric Dumazetd94d9fe2009-11-04 09:50:58 -08003523{
Stefan Rompfb00055a2006-03-20 17:09:11 -08003524 return (dev->operstate == IF_OPER_UP ||
3525 dev->operstate == IF_OPER_UNKNOWN /* backward compat */);
3526}
3527
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003528/**
3529 * netif_device_present - is device available or removed
3530 * @dev: network device
3531 *
3532 * Check if device has not been removed from system.
3533 */
David S. Miller4d295152012-03-07 21:02:35 -05003534static inline bool netif_device_present(struct net_device *dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003535{
3536 return test_bit(__LINK_STATE_PRESENT, &dev->state);
3537}
3538
Joe Perchesf629d202013-09-26 14:48:15 -07003539void netif_device_detach(struct net_device *dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003540
Joe Perchesf629d202013-09-26 14:48:15 -07003541void netif_device_attach(struct net_device *dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003542
3543/*
3544 * Network interface message level settings
3545 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07003546
3547enum {
3548 NETIF_MSG_DRV = 0x0001,
3549 NETIF_MSG_PROBE = 0x0002,
3550 NETIF_MSG_LINK = 0x0004,
3551 NETIF_MSG_TIMER = 0x0008,
3552 NETIF_MSG_IFDOWN = 0x0010,
3553 NETIF_MSG_IFUP = 0x0020,
3554 NETIF_MSG_RX_ERR = 0x0040,
3555 NETIF_MSG_TX_ERR = 0x0080,
3556 NETIF_MSG_TX_QUEUED = 0x0100,
3557 NETIF_MSG_INTR = 0x0200,
3558 NETIF_MSG_TX_DONE = 0x0400,
3559 NETIF_MSG_RX_STATUS = 0x0800,
3560 NETIF_MSG_PKTDATA = 0x1000,
3561 NETIF_MSG_HW = 0x2000,
3562 NETIF_MSG_WOL = 0x4000,
3563};
3564
3565#define netif_msg_drv(p) ((p)->msg_enable & NETIF_MSG_DRV)
3566#define netif_msg_probe(p) ((p)->msg_enable & NETIF_MSG_PROBE)
3567#define netif_msg_link(p) ((p)->msg_enable & NETIF_MSG_LINK)
3568#define netif_msg_timer(p) ((p)->msg_enable & NETIF_MSG_TIMER)
3569#define netif_msg_ifdown(p) ((p)->msg_enable & NETIF_MSG_IFDOWN)
3570#define netif_msg_ifup(p) ((p)->msg_enable & NETIF_MSG_IFUP)
3571#define netif_msg_rx_err(p) ((p)->msg_enable & NETIF_MSG_RX_ERR)
3572#define netif_msg_tx_err(p) ((p)->msg_enable & NETIF_MSG_TX_ERR)
3573#define netif_msg_tx_queued(p) ((p)->msg_enable & NETIF_MSG_TX_QUEUED)
3574#define netif_msg_intr(p) ((p)->msg_enable & NETIF_MSG_INTR)
3575#define netif_msg_tx_done(p) ((p)->msg_enable & NETIF_MSG_TX_DONE)
3576#define netif_msg_rx_status(p) ((p)->msg_enable & NETIF_MSG_RX_STATUS)
3577#define netif_msg_pktdata(p) ((p)->msg_enable & NETIF_MSG_PKTDATA)
3578#define netif_msg_hw(p) ((p)->msg_enable & NETIF_MSG_HW)
3579#define netif_msg_wol(p) ((p)->msg_enable & NETIF_MSG_WOL)
3580
3581static inline u32 netif_msg_init(int debug_value, int default_msg_enable_bits)
3582{
3583 /* use default */
3584 if (debug_value < 0 || debug_value >= (sizeof(u32) * 8))
3585 return default_msg_enable_bits;
3586 if (debug_value == 0) /* no output */
3587 return 0;
3588 /* set low N bits */
3589 return (1 << debug_value) - 1;
3590}
3591
David S. Millerc773e842008-07-08 23:13:53 -07003592static inline void __netif_tx_lock(struct netdev_queue *txq, int cpu)
Herbert Xu932ff272006-06-09 12:20:56 -07003593{
David S. Millerc773e842008-07-08 23:13:53 -07003594 spin_lock(&txq->_xmit_lock);
3595 txq->xmit_lock_owner = cpu;
Jamal Hadi Salim22dd7492007-09-16 14:40:49 -07003596}
3597
Michael S. Tsirkin5a717f42016-11-24 07:04:08 +02003598static inline bool __netif_tx_acquire(struct netdev_queue *txq)
3599{
3600 __acquire(&txq->_xmit_lock);
3601 return true;
3602}
3603
3604static inline void __netif_tx_release(struct netdev_queue *txq)
3605{
3606 __release(&txq->_xmit_lock);
3607}
3608
David S. Millerfd2ea0a2008-07-17 01:56:23 -07003609static inline void __netif_tx_lock_bh(struct netdev_queue *txq)
3610{
3611 spin_lock_bh(&txq->_xmit_lock);
3612 txq->xmit_lock_owner = smp_processor_id();
3613}
3614
David S. Miller4d295152012-03-07 21:02:35 -05003615static inline bool __netif_tx_trylock(struct netdev_queue *txq)
David S. Millerc773e842008-07-08 23:13:53 -07003616{
David S. Miller4d295152012-03-07 21:02:35 -05003617 bool ok = spin_trylock(&txq->_xmit_lock);
David S. Millerc773e842008-07-08 23:13:53 -07003618 if (likely(ok))
3619 txq->xmit_lock_owner = smp_processor_id();
3620 return ok;
Herbert Xu932ff272006-06-09 12:20:56 -07003621}
3622
David S. Millerc773e842008-07-08 23:13:53 -07003623static inline void __netif_tx_unlock(struct netdev_queue *txq)
3624{
3625 txq->xmit_lock_owner = -1;
3626 spin_unlock(&txq->_xmit_lock);
Herbert Xu932ff272006-06-09 12:20:56 -07003627}
3628
David S. Millerfd2ea0a2008-07-17 01:56:23 -07003629static inline void __netif_tx_unlock_bh(struct netdev_queue *txq)
3630{
3631 txq->xmit_lock_owner = -1;
3632 spin_unlock_bh(&txq->_xmit_lock);
3633}
3634
Eric Dumazet08baf562009-05-25 22:58:01 -07003635static inline void txq_trans_update(struct netdev_queue *txq)
3636{
3637 if (txq->xmit_lock_owner != -1)
3638 txq->trans_start = jiffies;
3639}
3640
Florian Westphalba162f82016-05-03 16:31:00 +02003641/* legacy drivers only, netdev_start_xmit() sets txq->trans_start */
3642static inline void netif_trans_update(struct net_device *dev)
3643{
Florian Westphal9b366272016-05-03 16:33:14 +02003644 struct netdev_queue *txq = netdev_get_tx_queue(dev, 0);
3645
3646 if (txq->trans_start != jiffies)
3647 txq->trans_start = jiffies;
Florian Westphalba162f82016-05-03 16:31:00 +02003648}
3649
David S. Millerc3f26a22008-07-31 16:58:50 -07003650/**
3651 * netif_tx_lock - grab network device transmit lock
3652 * @dev: network device
David S. Millerc3f26a22008-07-31 16:58:50 -07003653 *
3654 * Get network device transmit lock
3655 */
3656static inline void netif_tx_lock(struct net_device *dev)
3657{
3658 unsigned int i;
3659 int cpu;
3660
3661 spin_lock(&dev->tx_global_lock);
3662 cpu = smp_processor_id();
3663 for (i = 0; i < dev->num_tx_queues; i++) {
3664 struct netdev_queue *txq = netdev_get_tx_queue(dev, i);
3665
3666 /* We are the only thread of execution doing a
3667 * freeze, but we have to grab the _xmit_lock in
3668 * order to synchronize with threads which are in
3669 * the ->hard_start_xmit() handler and already
3670 * checked the frozen bit.
3671 */
3672 __netif_tx_lock(txq, cpu);
3673 set_bit(__QUEUE_STATE_FROZEN, &txq->state);
3674 __netif_tx_unlock(txq);
3675 }
3676}
3677
3678static inline void netif_tx_lock_bh(struct net_device *dev)
3679{
3680 local_bh_disable();
3681 netif_tx_lock(dev);
3682}
3683
Herbert Xu932ff272006-06-09 12:20:56 -07003684static inline void netif_tx_unlock(struct net_device *dev)
3685{
David S. Millere8a04642008-07-17 00:34:19 -07003686 unsigned int i;
David S. Millerc773e842008-07-08 23:13:53 -07003687
David S. Millere8a04642008-07-17 00:34:19 -07003688 for (i = 0; i < dev->num_tx_queues; i++) {
3689 struct netdev_queue *txq = netdev_get_tx_queue(dev, i);
David S. Millere8a04642008-07-17 00:34:19 -07003690
David S. Millerc3f26a22008-07-31 16:58:50 -07003691 /* No need to grab the _xmit_lock here. If the
3692 * queue is not stopped for another reason, we
3693 * force a schedule.
3694 */
3695 clear_bit(__QUEUE_STATE_FROZEN, &txq->state);
Krishna Kumar7b3d3e42009-08-29 20:21:21 +00003696 netif_schedule_queue(txq);
David S. Millerc3f26a22008-07-31 16:58:50 -07003697 }
3698 spin_unlock(&dev->tx_global_lock);
Herbert Xu932ff272006-06-09 12:20:56 -07003699}
3700
3701static inline void netif_tx_unlock_bh(struct net_device *dev)
3702{
David S. Millere8a04642008-07-17 00:34:19 -07003703 netif_tx_unlock(dev);
3704 local_bh_enable();
Herbert Xu932ff272006-06-09 12:20:56 -07003705}
3706
David S. Millerc773e842008-07-08 23:13:53 -07003707#define HARD_TX_LOCK(dev, txq, cpu) { \
Jamal Hadi Salim22dd7492007-09-16 14:40:49 -07003708 if ((dev->features & NETIF_F_LLTX) == 0) { \
David S. Millerc773e842008-07-08 23:13:53 -07003709 __netif_tx_lock(txq, cpu); \
Michael S. Tsirkin5a717f42016-11-24 07:04:08 +02003710 } else { \
3711 __netif_tx_acquire(txq); \
Jamal Hadi Salim22dd7492007-09-16 14:40:49 -07003712 } \
3713}
3714
Eric W. Biederman5efeac42014-03-27 15:42:20 -07003715#define HARD_TX_TRYLOCK(dev, txq) \
3716 (((dev->features & NETIF_F_LLTX) == 0) ? \
3717 __netif_tx_trylock(txq) : \
Michael S. Tsirkin5a717f42016-11-24 07:04:08 +02003718 __netif_tx_acquire(txq))
Eric W. Biederman5efeac42014-03-27 15:42:20 -07003719
David S. Millerc773e842008-07-08 23:13:53 -07003720#define HARD_TX_UNLOCK(dev, txq) { \
Jamal Hadi Salim22dd7492007-09-16 14:40:49 -07003721 if ((dev->features & NETIF_F_LLTX) == 0) { \
David S. Millerc773e842008-07-08 23:13:53 -07003722 __netif_tx_unlock(txq); \
Michael S. Tsirkin5a717f42016-11-24 07:04:08 +02003723 } else { \
3724 __netif_tx_release(txq); \
Jamal Hadi Salim22dd7492007-09-16 14:40:49 -07003725 } \
3726}
3727
Linus Torvalds1da177e2005-04-16 15:20:36 -07003728static inline void netif_tx_disable(struct net_device *dev)
3729{
David S. Millerfd2ea0a2008-07-17 01:56:23 -07003730 unsigned int i;
David S. Millerc3f26a22008-07-31 16:58:50 -07003731 int cpu;
David S. Millerfd2ea0a2008-07-17 01:56:23 -07003732
David S. Millerc3f26a22008-07-31 16:58:50 -07003733 local_bh_disable();
3734 cpu = smp_processor_id();
David S. Millerfd2ea0a2008-07-17 01:56:23 -07003735 for (i = 0; i < dev->num_tx_queues; i++) {
3736 struct netdev_queue *txq = netdev_get_tx_queue(dev, i);
David S. Millerc3f26a22008-07-31 16:58:50 -07003737
3738 __netif_tx_lock(txq, cpu);
David S. Millerfd2ea0a2008-07-17 01:56:23 -07003739 netif_tx_stop_queue(txq);
David S. Millerc3f26a22008-07-31 16:58:50 -07003740 __netif_tx_unlock(txq);
David S. Millerfd2ea0a2008-07-17 01:56:23 -07003741 }
David S. Millerc3f26a22008-07-31 16:58:50 -07003742 local_bh_enable();
Linus Torvalds1da177e2005-04-16 15:20:36 -07003743}
3744
David S. Millere308a5d2008-07-15 00:13:44 -07003745static inline void netif_addr_lock(struct net_device *dev)
3746{
3747 spin_lock(&dev->addr_list_lock);
3748}
3749
Jiri Pirko2429f7a2012-01-09 06:36:54 +00003750static inline void netif_addr_lock_nested(struct net_device *dev)
3751{
Vlad Yasevich25175ba2014-05-16 17:04:54 -04003752 int subclass = SINGLE_DEPTH_NESTING;
3753
3754 if (dev->netdev_ops->ndo_get_lock_subclass)
3755 subclass = dev->netdev_ops->ndo_get_lock_subclass(dev);
3756
3757 spin_lock_nested(&dev->addr_list_lock, subclass);
Jiri Pirko2429f7a2012-01-09 06:36:54 +00003758}
3759
David S. Millere308a5d2008-07-15 00:13:44 -07003760static inline void netif_addr_lock_bh(struct net_device *dev)
3761{
3762 spin_lock_bh(&dev->addr_list_lock);
3763}
3764
3765static inline void netif_addr_unlock(struct net_device *dev)
3766{
3767 spin_unlock(&dev->addr_list_lock);
3768}
3769
3770static inline void netif_addr_unlock_bh(struct net_device *dev)
3771{
3772 spin_unlock_bh(&dev->addr_list_lock);
3773}
3774
Jiri Pirkof001fde2009-05-05 02:48:28 +00003775/*
Jiri Pirko31278e72009-06-17 01:12:19 +00003776 * dev_addrs walker. Should be used only for read access. Call with
Jiri Pirkof001fde2009-05-05 02:48:28 +00003777 * rcu_read_lock held.
3778 */
3779#define for_each_dev_addr(dev, ha) \
Jiri Pirko31278e72009-06-17 01:12:19 +00003780 list_for_each_entry_rcu(ha, &dev->dev_addrs.list, list)
Jiri Pirkof001fde2009-05-05 02:48:28 +00003781
Linus Torvalds1da177e2005-04-16 15:20:36 -07003782/* These functions live elsewhere (drivers/net/net_init.c, but related) */
3783
Joe Perchesf629d202013-09-26 14:48:15 -07003784void ether_setup(struct net_device *dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003785
3786/* Support for loadable net-drivers */
Joe Perchesf629d202013-09-26 14:48:15 -07003787struct net_device *alloc_netdev_mqs(int sizeof_priv, const char *name,
Tom Gundersenc835a672014-07-14 16:37:24 +02003788 unsigned char name_assign_type,
Joe Perchesf629d202013-09-26 14:48:15 -07003789 void (*setup)(struct net_device *),
3790 unsigned int txqs, unsigned int rxqs);
Cong Wang0ad646c2017-10-13 11:58:53 -07003791int dev_get_valid_name(struct net *net, struct net_device *dev,
3792 const char *name);
3793
Tom Gundersenc835a672014-07-14 16:37:24 +02003794#define alloc_netdev(sizeof_priv, name, name_assign_type, setup) \
3795 alloc_netdev_mqs(sizeof_priv, name, name_assign_type, setup, 1, 1)
Tom Herbert36909ea2011-01-09 19:36:31 +00003796
Tom Gundersenc835a672014-07-14 16:37:24 +02003797#define alloc_netdev_mq(sizeof_priv, name, name_assign_type, setup, count) \
3798 alloc_netdev_mqs(sizeof_priv, name, name_assign_type, setup, count, \
3799 count)
Tom Herbert36909ea2011-01-09 19:36:31 +00003800
Joe Perchesf629d202013-09-26 14:48:15 -07003801int register_netdev(struct net_device *dev);
3802void unregister_netdev(struct net_device *dev);
Jiri Pirkof001fde2009-05-05 02:48:28 +00003803
Jiri Pirko22bedad32010-04-01 21:22:57 +00003804/* General hardware address lists handling functions */
Joe Perchesf629d202013-09-26 14:48:15 -07003805int __hw_addr_sync(struct netdev_hw_addr_list *to_list,
3806 struct netdev_hw_addr_list *from_list, int addr_len);
3807void __hw_addr_unsync(struct netdev_hw_addr_list *to_list,
3808 struct netdev_hw_addr_list *from_list, int addr_len);
Alexander Duyck670e5b82014-05-28 18:44:46 -07003809int __hw_addr_sync_dev(struct netdev_hw_addr_list *list,
3810 struct net_device *dev,
3811 int (*sync)(struct net_device *, const unsigned char *),
3812 int (*unsync)(struct net_device *,
3813 const unsigned char *));
3814void __hw_addr_unsync_dev(struct netdev_hw_addr_list *list,
3815 struct net_device *dev,
3816 int (*unsync)(struct net_device *,
3817 const unsigned char *));
Joe Perchesf629d202013-09-26 14:48:15 -07003818void __hw_addr_init(struct netdev_hw_addr_list *list);
Jiri Pirko22bedad32010-04-01 21:22:57 +00003819
Jiri Pirkof001fde2009-05-05 02:48:28 +00003820/* Functions used for device addresses handling */
Joe Perchesf629d202013-09-26 14:48:15 -07003821int dev_addr_add(struct net_device *dev, const unsigned char *addr,
3822 unsigned char addr_type);
3823int dev_addr_del(struct net_device *dev, const unsigned char *addr,
3824 unsigned char addr_type);
Joe Perchesf629d202013-09-26 14:48:15 -07003825void dev_addr_flush(struct net_device *dev);
3826int dev_addr_init(struct net_device *dev);
Jiri Pirkoa748ee22010-04-01 21:22:09 +00003827
3828/* Functions used for unicast addresses handling */
Joe Perchesf629d202013-09-26 14:48:15 -07003829int dev_uc_add(struct net_device *dev, const unsigned char *addr);
3830int dev_uc_add_excl(struct net_device *dev, const unsigned char *addr);
3831int dev_uc_del(struct net_device *dev, const unsigned char *addr);
3832int dev_uc_sync(struct net_device *to, struct net_device *from);
3833int dev_uc_sync_multiple(struct net_device *to, struct net_device *from);
3834void dev_uc_unsync(struct net_device *to, struct net_device *from);
3835void dev_uc_flush(struct net_device *dev);
3836void dev_uc_init(struct net_device *dev);
Jiri Pirkof001fde2009-05-05 02:48:28 +00003837
Alexander Duyck670e5b82014-05-28 18:44:46 -07003838/**
3839 * __dev_uc_sync - Synchonize device's unicast list
3840 * @dev: device to sync
3841 * @sync: function to call if address should be added
3842 * @unsync: function to call if address should be removed
3843 *
3844 * Add newly added addresses to the interface, and release
3845 * addresses that have been deleted.
Bjorn Helgaas5e82b4b2016-03-23 13:47:23 -05003846 */
Alexander Duyck670e5b82014-05-28 18:44:46 -07003847static inline int __dev_uc_sync(struct net_device *dev,
3848 int (*sync)(struct net_device *,
3849 const unsigned char *),
3850 int (*unsync)(struct net_device *,
3851 const unsigned char *))
3852{
3853 return __hw_addr_sync_dev(&dev->uc, dev, sync, unsync);
3854}
3855
3856/**
Masanari Iidae793c0f2014-09-04 23:44:36 +09003857 * __dev_uc_unsync - Remove synchronized addresses from device
Alexander Duyck670e5b82014-05-28 18:44:46 -07003858 * @dev: device to sync
3859 * @unsync: function to call if address should be removed
3860 *
3861 * Remove all addresses that were added to the device by dev_uc_sync().
Bjorn Helgaas5e82b4b2016-03-23 13:47:23 -05003862 */
Alexander Duyck670e5b82014-05-28 18:44:46 -07003863static inline void __dev_uc_unsync(struct net_device *dev,
3864 int (*unsync)(struct net_device *,
3865 const unsigned char *))
3866{
3867 __hw_addr_unsync_dev(&dev->uc, dev, unsync);
3868}
3869
Jiri Pirko22bedad32010-04-01 21:22:57 +00003870/* Functions used for multicast addresses handling */
Joe Perchesf629d202013-09-26 14:48:15 -07003871int dev_mc_add(struct net_device *dev, const unsigned char *addr);
3872int dev_mc_add_global(struct net_device *dev, const unsigned char *addr);
3873int dev_mc_add_excl(struct net_device *dev, const unsigned char *addr);
3874int dev_mc_del(struct net_device *dev, const unsigned char *addr);
3875int dev_mc_del_global(struct net_device *dev, const unsigned char *addr);
3876int dev_mc_sync(struct net_device *to, struct net_device *from);
3877int dev_mc_sync_multiple(struct net_device *to, struct net_device *from);
3878void dev_mc_unsync(struct net_device *to, struct net_device *from);
3879void dev_mc_flush(struct net_device *dev);
3880void dev_mc_init(struct net_device *dev);
Herbert Xufb286bb2005-11-10 13:01:24 -08003881
Alexander Duyck670e5b82014-05-28 18:44:46 -07003882/**
3883 * __dev_mc_sync - Synchonize device's multicast list
3884 * @dev: device to sync
3885 * @sync: function to call if address should be added
3886 * @unsync: function to call if address should be removed
3887 *
3888 * Add newly added addresses to the interface, and release
3889 * addresses that have been deleted.
Bjorn Helgaas5e82b4b2016-03-23 13:47:23 -05003890 */
Alexander Duyck670e5b82014-05-28 18:44:46 -07003891static inline int __dev_mc_sync(struct net_device *dev,
3892 int (*sync)(struct net_device *,
3893 const unsigned char *),
3894 int (*unsync)(struct net_device *,
3895 const unsigned char *))
3896{
3897 return __hw_addr_sync_dev(&dev->mc, dev, sync, unsync);
3898}
3899
3900/**
Masanari Iidae793c0f2014-09-04 23:44:36 +09003901 * __dev_mc_unsync - Remove synchronized addresses from device
Alexander Duyck670e5b82014-05-28 18:44:46 -07003902 * @dev: device to sync
3903 * @unsync: function to call if address should be removed
3904 *
3905 * Remove all addresses that were added to the device by dev_mc_sync().
Bjorn Helgaas5e82b4b2016-03-23 13:47:23 -05003906 */
Alexander Duyck670e5b82014-05-28 18:44:46 -07003907static inline void __dev_mc_unsync(struct net_device *dev,
3908 int (*unsync)(struct net_device *,
3909 const unsigned char *))
3910{
3911 __hw_addr_unsync_dev(&dev->mc, dev, unsync);
3912}
3913
Linus Torvalds1da177e2005-04-16 15:20:36 -07003914/* Functions used for secondary unicast and multicast support */
Joe Perchesf629d202013-09-26 14:48:15 -07003915void dev_set_rx_mode(struct net_device *dev);
3916void __dev_set_rx_mode(struct net_device *dev);
3917int dev_set_promiscuity(struct net_device *dev, int inc);
3918int dev_set_allmulti(struct net_device *dev, int inc);
3919void netdev_state_change(struct net_device *dev);
3920void netdev_notify_peers(struct net_device *dev);
3921void netdev_features_change(struct net_device *dev);
Herbert Xufb286bb2005-11-10 13:01:24 -08003922/* Load a device via the kmod */
Joe Perchesf629d202013-09-26 14:48:15 -07003923void dev_load(struct net *net, const char *name);
3924struct rtnl_link_stats64 *dev_get_stats(struct net_device *dev,
3925 struct rtnl_link_stats64 *storage);
3926void netdev_stats_to_stats64(struct rtnl_link_stats64 *stats64,
3927 const struct net_device_stats *netdev_stats);
Herbert Xufb286bb2005-11-10 13:01:24 -08003928
3929extern int netdev_max_backlog;
Eric Dumazet3b098e22010-05-15 23:57:10 -07003930extern int netdev_tstamp_prequeue;
Herbert Xufb286bb2005-11-10 13:01:24 -08003931extern int weight_p;
Matthias Tafelmeier3d48b532016-12-29 21:37:21 +01003932extern int dev_weight_rx_bias;
3933extern int dev_weight_tx_bias;
3934extern int dev_rx_weight;
3935extern int dev_tx_weight;
Jiri Pirko9ff162a2013-01-03 22:48:49 +00003936
Joe Perchesf629d202013-09-26 14:48:15 -07003937bool netdev_has_upper_dev(struct net_device *dev, struct net_device *upper_dev);
Vlad Yasevich44a40852014-05-16 17:20:38 -04003938struct net_device *netdev_upper_get_next_dev_rcu(struct net_device *dev,
3939 struct list_head **iter);
Joe Perchesf629d202013-09-26 14:48:15 -07003940struct net_device *netdev_all_upper_get_next_dev_rcu(struct net_device *dev,
3941 struct list_head **iter);
Veaceslav Falico8b5be852013-08-28 23:25:08 +02003942
3943/* iterate through upper list, must be called under RCU read lock */
Vlad Yasevich44a40852014-05-16 17:20:38 -04003944#define netdev_for_each_upper_dev_rcu(dev, updev, iter) \
3945 for (iter = &(dev)->adj_list.upper, \
3946 updev = netdev_upper_get_next_dev_rcu(dev, &(iter)); \
3947 updev; \
3948 updev = netdev_upper_get_next_dev_rcu(dev, &(iter)))
3949
David Ahern1a3f0602016-10-17 19:15:44 -07003950int netdev_walk_all_upper_dev_rcu(struct net_device *dev,
3951 int (*fn)(struct net_device *upper_dev,
3952 void *data),
3953 void *data);
3954
3955bool netdev_has_upper_dev_all_rcu(struct net_device *dev,
3956 struct net_device *upper_dev);
3957
Ido Schimmel25cc72a2017-09-01 10:52:31 +02003958bool netdev_has_any_upper_dev(struct net_device *dev);
3959
Joe Perchesf629d202013-09-26 14:48:15 -07003960void *netdev_lower_get_next_private(struct net_device *dev,
3961 struct list_head **iter);
3962void *netdev_lower_get_next_private_rcu(struct net_device *dev,
3963 struct list_head **iter);
Veaceslav Falico31088a12013-09-25 09:20:12 +02003964
3965#define netdev_for_each_lower_private(dev, priv, iter) \
3966 for (iter = (dev)->adj_list.lower.next, \
3967 priv = netdev_lower_get_next_private(dev, &(iter)); \
3968 priv; \
3969 priv = netdev_lower_get_next_private(dev, &(iter)))
3970
3971#define netdev_for_each_lower_private_rcu(dev, priv, iter) \
3972 for (iter = &(dev)->adj_list.lower, \
3973 priv = netdev_lower_get_next_private_rcu(dev, &(iter)); \
3974 priv; \
3975 priv = netdev_lower_get_next_private_rcu(dev, &(iter)))
3976
Vlad Yasevich4085ebe2014-05-16 17:04:53 -04003977void *netdev_lower_get_next(struct net_device *dev,
3978 struct list_head **iter);
Jiri Pirko7ce856a2016-07-04 08:23:12 +02003979
Vlad Yasevich4085ebe2014-05-16 17:04:53 -04003980#define netdev_for_each_lower_dev(dev, ldev, iter) \
Nikolay Aleksandrovcfdd28b2016-02-17 18:00:31 +01003981 for (iter = (dev)->adj_list.lower.next, \
Vlad Yasevich4085ebe2014-05-16 17:04:53 -04003982 ldev = netdev_lower_get_next(dev, &(iter)); \
3983 ldev; \
3984 ldev = netdev_lower_get_next(dev, &(iter)))
3985
Jiri Pirko7ce856a2016-07-04 08:23:12 +02003986struct net_device *netdev_all_lower_get_next(struct net_device *dev,
3987 struct list_head **iter);
3988struct net_device *netdev_all_lower_get_next_rcu(struct net_device *dev,
3989 struct list_head **iter);
3990
David Ahern1a3f0602016-10-17 19:15:44 -07003991int netdev_walk_all_lower_dev(struct net_device *dev,
3992 int (*fn)(struct net_device *lower_dev,
3993 void *data),
3994 void *data);
3995int netdev_walk_all_lower_dev_rcu(struct net_device *dev,
3996 int (*fn)(struct net_device *lower_dev,
3997 void *data),
3998 void *data);
3999
Joe Perchesf629d202013-09-26 14:48:15 -07004000void *netdev_adjacent_get_private(struct list_head *adj_list);
dingtianhonge001bfa2013-12-13 10:19:55 +08004001void *netdev_lower_get_first_private_rcu(struct net_device *dev);
Joe Perchesf629d202013-09-26 14:48:15 -07004002struct net_device *netdev_master_upper_dev_get(struct net_device *dev);
4003struct net_device *netdev_master_upper_dev_get_rcu(struct net_device *dev);
David Ahern42ab19e2017-10-04 17:48:47 -07004004int netdev_upper_dev_link(struct net_device *dev, struct net_device *upper_dev,
4005 struct netlink_ext_ack *extack);
Joe Perchesf629d202013-09-26 14:48:15 -07004006int netdev_master_upper_dev_link(struct net_device *dev,
Jiri Pirko6dffb042015-12-03 12:12:10 +01004007 struct net_device *upper_dev,
David Ahern42ab19e2017-10-04 17:48:47 -07004008 void *upper_priv, void *upper_info,
4009 struct netlink_ext_ack *extack);
Joe Perchesf629d202013-09-26 14:48:15 -07004010void netdev_upper_dev_unlink(struct net_device *dev,
4011 struct net_device *upper_dev);
Veaceslav Falico5bb025f2014-01-14 21:58:51 +01004012void netdev_adjacent_rename_links(struct net_device *dev, char *oldname);
Joe Perchesf629d202013-09-26 14:48:15 -07004013void *netdev_lower_dev_get_private(struct net_device *dev,
4014 struct net_device *lower_dev);
Jiri Pirko04d48262015-12-03 12:12:15 +01004015void netdev_lower_state_changed(struct net_device *lower_dev,
4016 void *lower_state_info);
Eric Dumazet960fb622014-11-16 06:23:05 -08004017
4018/* RSS keys are 40 or 52 bytes long */
4019#define NETDEV_RSS_KEY_LEN 52
Kim Jonesba905f52016-02-02 03:51:16 +00004020extern u8 netdev_rss_key[NETDEV_RSS_KEY_LEN] __read_mostly;
Eric Dumazet960fb622014-11-16 06:23:05 -08004021void netdev_rss_key_fill(void *buffer, size_t len);
4022
Sabrina Dubroca952fcfd2016-08-12 16:10:33 +02004023int dev_get_nest_level(struct net_device *dev);
Joe Perchesf629d202013-09-26 14:48:15 -07004024int skb_checksum_help(struct sk_buff *skb);
Davide Carattib72b5bf2017-05-18 15:44:38 +02004025int skb_crc32c_csum_help(struct sk_buff *skb);
Davide Caratti43c26a12017-05-18 15:44:41 +02004026int skb_csum_hwoffload_help(struct sk_buff *skb,
4027 const netdev_features_t features);
4028
Joe Perchesf629d202013-09-26 14:48:15 -07004029struct sk_buff *__skb_gso_segment(struct sk_buff *skb,
4030 netdev_features_t features, bool tx_path);
4031struct sk_buff *skb_mac_gso_segment(struct sk_buff *skb,
4032 netdev_features_t features);
Cong Wang12b00042013-02-05 16:36:38 +00004033
Moni Shoua61bd3852015-02-03 16:48:29 +02004034struct netdev_bonding_info {
4035 ifslave slave;
4036 ifbond master;
4037};
4038
4039struct netdev_notifier_bonding_info {
4040 struct netdev_notifier_info info; /* must be first */
4041 struct netdev_bonding_info bonding_info;
4042};
4043
4044void netdev_bonding_info_change(struct net_device *dev,
4045 struct netdev_bonding_info *bonding_info);
4046
Cong Wang12b00042013-02-05 16:36:38 +00004047static inline
4048struct sk_buff *skb_gso_segment(struct sk_buff *skb, netdev_features_t features)
4049{
4050 return __skb_gso_segment(skb, features, true);
4051}
Vlad Yasevich53d64712014-03-27 17:26:18 -04004052__be16 skb_network_protocol(struct sk_buff *skb, int *depth);
Pravin B Shelarec5f0612013-03-07 09:28:01 +00004053
4054static inline bool can_checksum_protocol(netdev_features_t features,
4055 __be16 protocol)
4056{
Tom Herbertc8cd0982015-12-14 11:19:44 -08004057 if (protocol == htons(ETH_P_FCOE))
4058 return !!(features & NETIF_F_FCOE_CRC);
4059
4060 /* Assume this is an IP checksum (not SCTP CRC) */
4061
4062 if (features & NETIF_F_HW_CSUM) {
4063 /* Can checksum everything */
4064 return true;
4065 }
4066
4067 switch (protocol) {
4068 case htons(ETH_P_IP):
4069 return !!(features & NETIF_F_IP_CSUM);
4070 case htons(ETH_P_IPV6):
4071 return !!(features & NETIF_F_IPV6_CSUM);
4072 default:
4073 return false;
4074 }
Pravin B Shelarec5f0612013-03-07 09:28:01 +00004075}
Cong Wang12b00042013-02-05 16:36:38 +00004076
Herbert Xufb286bb2005-11-10 13:01:24 -08004077#ifdef CONFIG_BUG
Joe Perchesf629d202013-09-26 14:48:15 -07004078void netdev_rx_csum_fault(struct net_device *dev);
Herbert Xufb286bb2005-11-10 13:01:24 -08004079#else
4080static inline void netdev_rx_csum_fault(struct net_device *dev)
4081{
4082}
4083#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07004084/* rx skb timestamps */
Joe Perchesf629d202013-09-26 14:48:15 -07004085void net_enable_timestamp(void);
4086void net_disable_timestamp(void);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004087
Arnaldo Carvalho de Melo20380732005-08-16 02:18:02 -03004088#ifdef CONFIG_PROC_FS
Joe Perchesf629d202013-09-26 14:48:15 -07004089int __init dev_proc_init(void);
Cong Wang900ff8c2013-02-18 19:20:33 +00004090#else
4091#define dev_proc_init() 0
Arnaldo Carvalho de Melo20380732005-08-16 02:18:02 -03004092#endif
4093
David S. Miller47982482014-08-22 16:21:53 -07004094static inline netdev_tx_t __netdev_start_xmit(const struct net_device_ops *ops,
David S. Millerfa2dbdc2014-08-29 21:55:22 -07004095 struct sk_buff *skb, struct net_device *dev,
4096 bool more)
David S. Miller47982482014-08-22 16:21:53 -07004097{
David S. Millerfa2dbdc2014-08-29 21:55:22 -07004098 skb->xmit_more = more ? 1 : 0;
David S. Miller0b725a22014-08-25 15:51:53 -07004099 return ops->ndo_start_xmit(skb, dev);
David S. Miller47982482014-08-22 16:21:53 -07004100}
4101
David S. Miller10b3ad82014-08-29 21:07:24 -07004102static inline netdev_tx_t netdev_start_xmit(struct sk_buff *skb, struct net_device *dev,
David S. Millerfa2dbdc2014-08-29 21:55:22 -07004103 struct netdev_queue *txq, bool more)
David S. Miller47982482014-08-22 16:21:53 -07004104{
4105 const struct net_device_ops *ops = dev->netdev_ops;
David S. Miller10b3ad82014-08-29 21:07:24 -07004106 int rc;
David S. Miller47982482014-08-22 16:21:53 -07004107
David S. Millerfa2dbdc2014-08-29 21:55:22 -07004108 rc = __netdev_start_xmit(ops, skb, dev, more);
David S. Miller10b3ad82014-08-29 21:07:24 -07004109 if (rc == NETDEV_TX_OK)
4110 txq_trans_update(txq);
4111
4112 return rc;
David S. Miller47982482014-08-22 16:21:53 -07004113}
4114
stephen hemmingerb793dc52017-08-18 13:46:20 -07004115int netdev_class_create_file_ns(const struct class_attribute *class_attr,
Linus Torvalds42a2d922013-11-13 17:40:34 +09004116 const void *ns);
stephen hemmingerb793dc52017-08-18 13:46:20 -07004117void netdev_class_remove_file_ns(const struct class_attribute *class_attr,
Linus Torvalds42a2d922013-11-13 17:40:34 +09004118 const void *ns);
Tejun Heo58292cbe2013-09-11 22:29:04 -04004119
stephen hemmingerb793dc52017-08-18 13:46:20 -07004120static inline int netdev_class_create_file(const struct class_attribute *class_attr)
Tejun Heo58292cbe2013-09-11 22:29:04 -04004121{
4122 return netdev_class_create_file_ns(class_attr, NULL);
4123}
4124
stephen hemmingerb793dc52017-08-18 13:46:20 -07004125static inline void netdev_class_remove_file(const struct class_attribute *class_attr)
Tejun Heo58292cbe2013-09-11 22:29:04 -04004126{
4127 netdev_class_remove_file_ns(class_attr, NULL);
4128}
Jay Vosburghb8a97872008-06-13 18:12:04 -07004129
stephen hemminger737aec52017-08-18 13:46:22 -07004130extern const struct kobj_ns_type_operations net_ns_type_operations;
Johannes Berg04600792010-08-05 17:45:15 +02004131
Joe Perchesf629d202013-09-26 14:48:15 -07004132const char *netdev_drivername(const struct net_device *dev);
Arjan van de Ven6579e572008-07-21 13:31:48 -07004133
Joe Perchesf629d202013-09-26 14:48:15 -07004134void linkwatch_run_queue(void);
Arnaldo Carvalho de Melo20380732005-08-16 02:18:02 -03004135
Michal Kubečekda081432014-05-20 08:29:25 +02004136static inline netdev_features_t netdev_intersect_features(netdev_features_t f1,
4137 netdev_features_t f2)
4138{
Tom Herbertc8cd0982015-12-14 11:19:44 -08004139 if ((f1 ^ f2) & NETIF_F_HW_CSUM) {
4140 if (f1 & NETIF_F_HW_CSUM)
Tom Herbertb6a0e722016-01-11 10:19:10 -08004141 f1 |= (NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM);
Tom Herbertc8cd0982015-12-14 11:19:44 -08004142 else
Tom Herbertb6a0e722016-01-11 10:19:10 -08004143 f2 |= (NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM);
Tom Herbertc8cd0982015-12-14 11:19:44 -08004144 }
Michal Kubečekda081432014-05-20 08:29:25 +02004145
Tom Herbertc8cd0982015-12-14 11:19:44 -08004146 return f1 & f2;
Michal Kubečekda081432014-05-20 08:29:25 +02004147}
4148
Michał Mirosławc8f44af2011-11-15 15:29:55 +00004149static inline netdev_features_t netdev_get_wanted_features(
4150 struct net_device *dev)
Michał Mirosław5455c692011-02-15 16:59:17 +00004151{
4152 return (dev->features & ~dev->hw_features) | dev->wanted_features;
4153}
Michał Mirosławc8f44af2011-11-15 15:29:55 +00004154netdev_features_t netdev_increment_features(netdev_features_t all,
4155 netdev_features_t one, netdev_features_t mask);
Eric Dumazetb0ce3502013-05-16 07:34:53 +00004156
4157/* Allow TSO being used on stacked device :
4158 * Performing the GSO segmentation before last device
4159 * is a performance improvement.
4160 */
4161static inline netdev_features_t netdev_add_tso_features(netdev_features_t features,
4162 netdev_features_t mask)
4163{
4164 return netdev_increment_features(features, NETIF_F_ALL_TSO, mask);
4165}
4166
Michał Mirosław6cb6a272011-04-02 22:48:47 -07004167int __netdev_update_features(struct net_device *dev);
Michał Mirosław5455c692011-02-15 16:59:17 +00004168void netdev_update_features(struct net_device *dev);
Michał Mirosławafe12cc2011-05-07 03:22:17 +00004169void netdev_change_features(struct net_device *dev);
Herbert Xu7f353bf2007-08-10 15:47:58 -07004170
Patrick Mullaneyfc4a7482009-12-03 15:59:22 -08004171void netif_stacked_transfer_operstate(const struct net_device *rootdev,
4172 struct net_device *dev);
4173
Toshiaki Makitae38f3022015-03-27 14:31:13 +09004174netdev_features_t passthru_features_check(struct sk_buff *skb,
4175 struct net_device *dev,
4176 netdev_features_t features);
Florian Westphalc1e756b2014-05-05 15:00:44 +02004177netdev_features_t netif_skb_features(struct sk_buff *skb);
Jesse Gross58e998c2010-10-29 12:14:55 +00004178
David S. Miller4d295152012-03-07 21:02:35 -05004179static inline bool net_gso_ok(netdev_features_t features, int gso_type)
Herbert Xubcd76112006-06-30 13:36:35 -07004180{
Marcelo Ricardo Leitner7b748342016-04-25 15:13:17 -03004181 netdev_features_t feature = (netdev_features_t)gso_type << NETIF_F_GSO_SHIFT;
Michał Mirosław0345e182011-11-16 14:05:33 +00004182
4183 /* check flags correspondence */
4184 BUILD_BUG_ON(SKB_GSO_TCPV4 != (NETIF_F_TSO >> NETIF_F_GSO_SHIFT));
Michał Mirosław0345e182011-11-16 14:05:33 +00004185 BUILD_BUG_ON(SKB_GSO_DODGY != (NETIF_F_GSO_ROBUST >> NETIF_F_GSO_SHIFT));
4186 BUILD_BUG_ON(SKB_GSO_TCP_ECN != (NETIF_F_TSO_ECN >> NETIF_F_GSO_SHIFT));
Alexander Duyckcbc53e02016-04-10 21:44:51 -04004187 BUILD_BUG_ON(SKB_GSO_TCP_FIXEDID != (NETIF_F_TSO_MANGLEID >> NETIF_F_GSO_SHIFT));
Michał Mirosław0345e182011-11-16 14:05:33 +00004188 BUILD_BUG_ON(SKB_GSO_TCPV6 != (NETIF_F_TSO6 >> NETIF_F_GSO_SHIFT));
4189 BUILD_BUG_ON(SKB_GSO_FCOE != (NETIF_F_FSO >> NETIF_F_GSO_SHIFT));
Tom Herbert4b282522014-06-14 23:23:52 -07004190 BUILD_BUG_ON(SKB_GSO_GRE != (NETIF_F_GSO_GRE >> NETIF_F_GSO_SHIFT));
4191 BUILD_BUG_ON(SKB_GSO_GRE_CSUM != (NETIF_F_GSO_GRE_CSUM >> NETIF_F_GSO_SHIFT));
Tom Herbert7e133182016-05-18 09:06:10 -07004192 BUILD_BUG_ON(SKB_GSO_IPXIP4 != (NETIF_F_GSO_IPXIP4 >> NETIF_F_GSO_SHIFT));
4193 BUILD_BUG_ON(SKB_GSO_IPXIP6 != (NETIF_F_GSO_IPXIP6 >> NETIF_F_GSO_SHIFT));
Tom Herbert4b282522014-06-14 23:23:52 -07004194 BUILD_BUG_ON(SKB_GSO_UDP_TUNNEL != (NETIF_F_GSO_UDP_TUNNEL >> NETIF_F_GSO_SHIFT));
4195 BUILD_BUG_ON(SKB_GSO_UDP_TUNNEL_CSUM != (NETIF_F_GSO_UDP_TUNNEL_CSUM >> NETIF_F_GSO_SHIFT));
Alexander Duyck802ab552016-04-10 21:45:03 -04004196 BUILD_BUG_ON(SKB_GSO_PARTIAL != (NETIF_F_GSO_PARTIAL >> NETIF_F_GSO_SHIFT));
Tom Herberte585f232014-11-04 09:06:54 -08004197 BUILD_BUG_ON(SKB_GSO_TUNNEL_REMCSUM != (NETIF_F_GSO_TUNNEL_REMCSUM >> NETIF_F_GSO_SHIFT));
Marcelo Ricardo Leitner90017ac2016-06-02 15:05:43 -03004198 BUILD_BUG_ON(SKB_GSO_SCTP != (NETIF_F_GSO_SCTP >> NETIF_F_GSO_SHIFT));
Steffen Klassertc7ef8f02017-04-14 10:05:36 +02004199 BUILD_BUG_ON(SKB_GSO_ESP != (NETIF_F_GSO_ESP >> NETIF_F_GSO_SHIFT));
Willem de Bruijn0c19f8462017-11-21 10:22:25 -05004200 BUILD_BUG_ON(SKB_GSO_UDP != (NETIF_F_GSO_UDP >> NETIF_F_GSO_SHIFT));
Willem de Bruijn83aa0252018-04-26 13:42:21 -04004201 BUILD_BUG_ON(SKB_GSO_UDP_L4 != (NETIF_F_GSO_UDP_L4 >> NETIF_F_GSO_SHIFT));
Michał Mirosław0345e182011-11-16 14:05:33 +00004202
Herbert Xubcd76112006-06-30 13:36:35 -07004203 return (features & feature) == feature;
4204}
4205
David S. Miller4d295152012-03-07 21:02:35 -05004206static inline bool skb_gso_ok(struct sk_buff *skb, netdev_features_t features)
Herbert Xu576a30e2006-06-27 13:22:38 -07004207{
Herbert Xu278b2512009-06-03 21:20:51 -07004208 return net_gso_ok(features, skb_shinfo(skb)->gso_type) &&
David S. Miller21dc3302010-08-23 00:13:46 -07004209 (!skb_has_frag_list(skb) || (features & NETIF_F_FRAGLIST));
Herbert Xu576a30e2006-06-27 13:22:38 -07004210}
4211
Johannes Berg8b86a612015-04-17 15:45:04 +02004212static inline bool netif_needs_gso(struct sk_buff *skb,
David S. Miller4d295152012-03-07 21:02:35 -05004213 netdev_features_t features)
Herbert Xu79671682006-06-22 02:40:14 -07004214{
Jesse Grossfc741212011-01-09 06:23:32 +00004215 return skb_is_gso(skb) && (!skb_gso_ok(skb, features) ||
Yi Zoucdbee742012-03-16 23:08:11 +00004216 unlikely((skb->ip_summed != CHECKSUM_PARTIAL) &&
4217 (skb->ip_summed != CHECKSUM_UNNECESSARY)));
Herbert Xu79671682006-06-22 02:40:14 -07004218}
4219
Peter P Waskiewicz Jr82cc1a72008-03-21 03:43:19 -07004220static inline void netif_set_gso_max_size(struct net_device *dev,
4221 unsigned int size)
4222{
4223 dev->gso_max_size = size;
4224}
4225
Wei-Chun Chao7a7ffba2013-12-26 13:10:22 -08004226static inline void skb_gso_error_unwind(struct sk_buff *skb, __be16 protocol,
4227 int pulled_hlen, u16 mac_offset,
4228 int mac_len)
4229{
4230 skb->protocol = protocol;
4231 skb->encapsulation = 1;
4232 skb_push(skb, pulled_hlen);
4233 skb_reset_transport_header(skb);
4234 skb->mac_header = mac_offset;
4235 skb->network_header = skb->mac_header + mac_len;
4236 skb->mac_len = mac_len;
4237}
4238
Sabrina Dubroca3c175782016-03-11 18:07:32 +01004239static inline bool netif_is_macsec(const struct net_device *dev)
4240{
4241 return dev->priv_flags & IFF_MACSEC;
4242}
4243
Jiri Pirkob618aaa2015-12-04 15:01:31 +01004244static inline bool netif_is_macvlan(const struct net_device *dev)
John Fastabenda6cc0cf2013-11-06 09:54:46 -08004245{
4246 return dev->priv_flags & IFF_MACVLAN;
4247}
4248
Jiri Pirkob618aaa2015-12-04 15:01:31 +01004249static inline bool netif_is_macvlan_port(const struct net_device *dev)
Mahesh Bandewar2f33e7d2014-12-06 15:53:04 -08004250{
4251 return dev->priv_flags & IFF_MACVLAN_PORT;
4252}
4253
Jiri Pirkob618aaa2015-12-04 15:01:31 +01004254static inline bool netif_is_bond_master(const struct net_device *dev)
nikolay@redhat.com8a7fbfa2013-03-12 02:49:01 +00004255{
4256 return dev->flags & IFF_MASTER && dev->priv_flags & IFF_BONDING;
4257}
4258
Jiri Pirkob618aaa2015-12-04 15:01:31 +01004259static inline bool netif_is_bond_slave(const struct net_device *dev)
Jiri Pirko1765a572011-02-12 06:48:36 +00004260{
4261 return dev->flags & IFF_SLAVE && dev->priv_flags & IFF_BONDING;
4262}
4263
Ben Greear3bdc0eb2012-02-11 15:39:30 +00004264static inline bool netif_supports_nofcs(struct net_device *dev)
4265{
4266 return dev->priv_flags & IFF_SUPP_NOFCS;
4267}
4268
David Ahern007979e2015-09-29 20:07:10 -07004269static inline bool netif_is_l3_master(const struct net_device *dev)
David Ahern4e3c8992015-08-13 14:59:00 -06004270{
David Ahern007979e2015-09-29 20:07:10 -07004271 return dev->priv_flags & IFF_L3MDEV_MASTER;
David Ahern4e3c8992015-08-13 14:59:00 -06004272}
4273
David Ahernfee6d4c2015-10-05 08:51:24 -07004274static inline bool netif_is_l3_slave(const struct net_device *dev)
4275{
4276 return dev->priv_flags & IFF_L3MDEV_SLAVE;
4277}
4278
Jiri Pirko0894ae32015-08-27 09:31:19 +02004279static inline bool netif_is_bridge_master(const struct net_device *dev)
4280{
4281 return dev->priv_flags & IFF_EBRIDGE;
4282}
4283
Vlad Yasevich28f9ee22015-11-16 15:43:45 -05004284static inline bool netif_is_bridge_port(const struct net_device *dev)
4285{
4286 return dev->priv_flags & IFF_BRIDGE_PORT;
4287}
4288
Jiri Pirko35d4e172015-08-27 09:31:20 +02004289static inline bool netif_is_ovs_master(const struct net_device *dev)
4290{
4291 return dev->priv_flags & IFF_OPENVSWITCH;
4292}
4293
Jiri Pirko5be66142017-04-18 16:55:36 +02004294static inline bool netif_is_ovs_port(const struct net_device *dev)
4295{
4296 return dev->priv_flags & IFF_OVS_DATAPATH;
4297}
4298
Jiri Pirkob618aaa2015-12-04 15:01:31 +01004299static inline bool netif_is_team_master(const struct net_device *dev)
Jiri Pirkoc981e422015-12-03 12:12:06 +01004300{
4301 return dev->priv_flags & IFF_TEAM;
4302}
4303
Jiri Pirkob618aaa2015-12-04 15:01:31 +01004304static inline bool netif_is_team_port(const struct net_device *dev)
Jiri Pirkof7f019e2015-12-03 12:12:07 +01004305{
4306 return dev->priv_flags & IFF_TEAM_PORT;
4307}
4308
Jiri Pirkob618aaa2015-12-04 15:01:31 +01004309static inline bool netif_is_lag_master(const struct net_device *dev)
Jiri Pirko7be61832015-12-03 12:12:08 +01004310{
4311 return netif_is_bond_master(dev) || netif_is_team_master(dev);
4312}
4313
Jiri Pirkob618aaa2015-12-04 15:01:31 +01004314static inline bool netif_is_lag_port(const struct net_device *dev)
Jiri Pirkoe0ba1412015-12-03 12:12:09 +01004315{
4316 return netif_is_bond_slave(dev) || netif_is_team_port(dev);
4317}
4318
Keller, Jacob Ed4ab4282016-02-08 16:05:03 -08004319static inline bool netif_is_rxfh_configured(const struct net_device *dev)
4320{
4321 return dev->priv_flags & IFF_RXFH_CONFIGURED;
4322}
4323
Eric Dumazet02875872014-10-05 18:38:35 -07004324/* This device needs to keep skb dst for qdisc enqueue or ndo_start_xmit() */
4325static inline void netif_keep_dst(struct net_device *dev)
4326{
4327 dev->priv_flags &= ~(IFF_XMIT_DST_RELEASE | IFF_XMIT_DST_RELEASE_PERM);
4328}
4329
Paolo Abeni18d3df32016-07-14 18:00:10 +02004330/* return true if dev can't cope with mtu frames that need vlan tag insertion */
4331static inline bool netif_reduces_vlan_mtu(struct net_device *dev)
4332{
4333 /* TODO: reserve and use an additional IFF bit, if we get more users */
4334 return dev->priv_flags & IFF_MACSEC;
4335}
4336
Eric W. Biederman505d4f72008-11-07 22:54:20 -08004337extern struct pernet_operations __net_initdata loopback_net_ops;
Patrick McHardyb1b67dd2009-04-20 04:49:28 +00004338
Joe Perches571ba422010-02-09 11:49:47 +00004339/* Logging, debugging and troubleshooting/diagnostic helpers. */
4340
4341/* netdev_printk helpers, similar to dev_printk */
4342
4343static inline const char *netdev_name(const struct net_device *dev)
4344{
Veaceslav Falicoc6f854d2014-07-17 19:46:09 +02004345 if (!dev->name[0] || strchr(dev->name, '%'))
4346 return "(unnamed net_device)";
Joe Perches571ba422010-02-09 11:49:47 +00004347 return dev->name;
4348}
4349
David Ahern8397ed32017-06-07 12:26:23 -06004350static inline bool netdev_unregistering(const struct net_device *dev)
4351{
4352 return dev->reg_state == NETREG_UNREGISTERING;
4353}
4354
Veaceslav Falicoccc7f492014-07-17 19:46:10 +02004355static inline const char *netdev_reg_state(const struct net_device *dev)
4356{
4357 switch (dev->reg_state) {
4358 case NETREG_UNINITIALIZED: return " (uninitialized)";
4359 case NETREG_REGISTERED: return "";
4360 case NETREG_UNREGISTERING: return " (unregistering)";
4361 case NETREG_UNREGISTERED: return " (unregistered)";
4362 case NETREG_RELEASED: return " (released)";
4363 case NETREG_DUMMY: return " (dummy)";
4364 }
4365
4366 WARN_ONCE(1, "%s: unknown reg_state %d\n", dev->name, dev->reg_state);
4367 return " (unknown)";
4368}
4369
Joe Perchesf629d202013-09-26 14:48:15 -07004370__printf(3, 4)
Joe Perches6ea754e2014-09-22 11:10:50 -07004371void netdev_printk(const char *level, const struct net_device *dev,
4372 const char *format, ...);
Joe Perchesf629d202013-09-26 14:48:15 -07004373__printf(2, 3)
Joe Perches6ea754e2014-09-22 11:10:50 -07004374void netdev_emerg(const struct net_device *dev, const char *format, ...);
Joe Perchesf629d202013-09-26 14:48:15 -07004375__printf(2, 3)
Joe Perches6ea754e2014-09-22 11:10:50 -07004376void netdev_alert(const struct net_device *dev, const char *format, ...);
Joe Perchesf629d202013-09-26 14:48:15 -07004377__printf(2, 3)
Joe Perches6ea754e2014-09-22 11:10:50 -07004378void netdev_crit(const struct net_device *dev, const char *format, ...);
Joe Perchesf629d202013-09-26 14:48:15 -07004379__printf(2, 3)
Joe Perches6ea754e2014-09-22 11:10:50 -07004380void netdev_err(const struct net_device *dev, const char *format, ...);
Joe Perchesf629d202013-09-26 14:48:15 -07004381__printf(2, 3)
Joe Perches6ea754e2014-09-22 11:10:50 -07004382void netdev_warn(const struct net_device *dev, const char *format, ...);
Joe Perchesf629d202013-09-26 14:48:15 -07004383__printf(2, 3)
Joe Perches6ea754e2014-09-22 11:10:50 -07004384void netdev_notice(const struct net_device *dev, const char *format, ...);
Joe Perchesf629d202013-09-26 14:48:15 -07004385__printf(2, 3)
Joe Perches6ea754e2014-09-22 11:10:50 -07004386void netdev_info(const struct net_device *dev, const char *format, ...);
Joe Perches571ba422010-02-09 11:49:47 +00004387
Gal Pressman375ef2b2017-09-17 13:43:58 +03004388#define netdev_level_once(level, dev, fmt, ...) \
4389do { \
4390 static bool __print_once __read_mostly; \
4391 \
4392 if (!__print_once) { \
4393 __print_once = true; \
4394 netdev_printk(level, dev, fmt, ##__VA_ARGS__); \
4395 } \
4396} while (0)
4397
4398#define netdev_emerg_once(dev, fmt, ...) \
4399 netdev_level_once(KERN_EMERG, dev, fmt, ##__VA_ARGS__)
4400#define netdev_alert_once(dev, fmt, ...) \
4401 netdev_level_once(KERN_ALERT, dev, fmt, ##__VA_ARGS__)
4402#define netdev_crit_once(dev, fmt, ...) \
4403 netdev_level_once(KERN_CRIT, dev, fmt, ##__VA_ARGS__)
4404#define netdev_err_once(dev, fmt, ...) \
4405 netdev_level_once(KERN_ERR, dev, fmt, ##__VA_ARGS__)
4406#define netdev_warn_once(dev, fmt, ...) \
4407 netdev_level_once(KERN_WARNING, dev, fmt, ##__VA_ARGS__)
4408#define netdev_notice_once(dev, fmt, ...) \
4409 netdev_level_once(KERN_NOTICE, dev, fmt, ##__VA_ARGS__)
4410#define netdev_info_once(dev, fmt, ...) \
4411 netdev_level_once(KERN_INFO, dev, fmt, ##__VA_ARGS__)
4412
Vasiliy Kulikov8909c9a2011-03-02 00:33:13 +03004413#define MODULE_ALIAS_NETDEV(device) \
4414 MODULE_ALIAS("netdev-" device)
4415
Jim Cromieb558c962011-12-19 17:11:18 -05004416#if defined(CONFIG_DYNAMIC_DEBUG)
Joe Perches571ba422010-02-09 11:49:47 +00004417#define netdev_dbg(__dev, format, args...) \
4418do { \
Jason Baronffa10cb2011-08-11 14:36:48 -04004419 dynamic_netdev_dbg(__dev, format, ##args); \
Joe Perches571ba422010-02-09 11:49:47 +00004420} while (0)
Jim Cromieb558c962011-12-19 17:11:18 -05004421#elif defined(DEBUG)
4422#define netdev_dbg(__dev, format, args...) \
4423 netdev_printk(KERN_DEBUG, __dev, format, ##args)
Joe Perches571ba422010-02-09 11:49:47 +00004424#else
4425#define netdev_dbg(__dev, format, args...) \
4426({ \
4427 if (0) \
4428 netdev_printk(KERN_DEBUG, __dev, format, ##args); \
Joe Perches571ba422010-02-09 11:49:47 +00004429})
4430#endif
4431
4432#if defined(VERBOSE_DEBUG)
4433#define netdev_vdbg netdev_dbg
4434#else
4435
4436#define netdev_vdbg(dev, format, args...) \
4437({ \
4438 if (0) \
4439 netdev_printk(KERN_DEBUG, dev, format, ##args); \
4440 0; \
4441})
4442#endif
4443
4444/*
4445 * netdev_WARN() acts like dev_printk(), but with the key difference
4446 * of using a WARN/WARN_ON to get the message out, including the
4447 * file/line information and a backtrace.
4448 */
4449#define netdev_WARN(dev, format, args...) \
Gal Pressmane1cfe3d2018-01-07 12:08:36 +02004450 WARN(1, "netdevice: %s%s: " format, netdev_name(dev), \
Veaceslav Falicoccc7f492014-07-17 19:46:10 +02004451 netdev_reg_state(dev), ##args)
Joe Perches571ba422010-02-09 11:49:47 +00004452
Gal Pressman72dd8312018-01-07 12:08:35 +02004453#define netdev_WARN_ONCE(dev, format, args...) \
Gal Pressmane1cfe3d2018-01-07 12:08:36 +02004454 WARN_ONCE(1, "netdevice: %s%s: " format, netdev_name(dev), \
Gal Pressman375ef2b2017-09-17 13:43:58 +03004455 netdev_reg_state(dev), ##args)
4456
Joe Perchesb3d95c52010-02-09 11:49:49 +00004457/* netif printk helpers, similar to netdev_printk */
4458
4459#define netif_printk(priv, type, level, dev, fmt, args...) \
4460do { \
4461 if (netif_msg_##type(priv)) \
4462 netdev_printk(level, (dev), fmt, ##args); \
4463} while (0)
4464
Joe Perchesf45f4322010-06-27 01:02:36 +00004465#define netif_level(level, priv, type, dev, fmt, args...) \
4466do { \
4467 if (netif_msg_##type(priv)) \
4468 netdev_##level(dev, fmt, ##args); \
4469} while (0)
4470
Joe Perchesb3d95c52010-02-09 11:49:49 +00004471#define netif_emerg(priv, type, dev, fmt, args...) \
Joe Perchesf45f4322010-06-27 01:02:36 +00004472 netif_level(emerg, priv, type, dev, fmt, ##args)
Joe Perchesb3d95c52010-02-09 11:49:49 +00004473#define netif_alert(priv, type, dev, fmt, args...) \
Joe Perchesf45f4322010-06-27 01:02:36 +00004474 netif_level(alert, priv, type, dev, fmt, ##args)
Joe Perchesb3d95c52010-02-09 11:49:49 +00004475#define netif_crit(priv, type, dev, fmt, args...) \
Joe Perchesf45f4322010-06-27 01:02:36 +00004476 netif_level(crit, priv, type, dev, fmt, ##args)
Joe Perchesb3d95c52010-02-09 11:49:49 +00004477#define netif_err(priv, type, dev, fmt, args...) \
Joe Perchesf45f4322010-06-27 01:02:36 +00004478 netif_level(err, priv, type, dev, fmt, ##args)
Joe Perchesb3d95c52010-02-09 11:49:49 +00004479#define netif_warn(priv, type, dev, fmt, args...) \
Joe Perchesf45f4322010-06-27 01:02:36 +00004480 netif_level(warn, priv, type, dev, fmt, ##args)
Joe Perchesb3d95c52010-02-09 11:49:49 +00004481#define netif_notice(priv, type, dev, fmt, args...) \
Joe Perchesf45f4322010-06-27 01:02:36 +00004482 netif_level(notice, priv, type, dev, fmt, ##args)
Joe Perchesb3d95c52010-02-09 11:49:49 +00004483#define netif_info(priv, type, dev, fmt, args...) \
Joe Perchesf45f4322010-06-27 01:02:36 +00004484 netif_level(info, priv, type, dev, fmt, ##args)
Joe Perchesb3d95c52010-02-09 11:49:49 +00004485
Joe Perches0053ea92012-05-30 07:43:34 +00004486#if defined(CONFIG_DYNAMIC_DEBUG)
Joe Perchesb3d95c52010-02-09 11:49:49 +00004487#define netif_dbg(priv, type, netdev, format, args...) \
4488do { \
4489 if (netif_msg_##type(priv)) \
Jason Baronb5fb0a02011-08-11 14:36:53 -04004490 dynamic_netdev_dbg(netdev, format, ##args); \
Joe Perchesb3d95c52010-02-09 11:49:49 +00004491} while (0)
Joe Perches0053ea92012-05-30 07:43:34 +00004492#elif defined(DEBUG)
4493#define netif_dbg(priv, type, dev, format, args...) \
4494 netif_printk(priv, type, KERN_DEBUG, dev, format, ##args)
Joe Perchesb3d95c52010-02-09 11:49:49 +00004495#else
4496#define netif_dbg(priv, type, dev, format, args...) \
4497({ \
4498 if (0) \
4499 netif_printk(priv, type, KERN_DEBUG, dev, format, ##args); \
4500 0; \
4501})
4502#endif
4503
Edward Creef617f272017-01-27 15:02:26 +00004504/* if @cond then downgrade to debug, else print at @level */
4505#define netif_cond_dbg(priv, type, netdev, cond, level, fmt, args...) \
4506 do { \
4507 if (cond) \
4508 netif_dbg(priv, type, netdev, fmt, ##args); \
4509 else \
4510 netif_ ## level(priv, type, netdev, fmt, ##args); \
4511 } while (0)
4512
Joe Perchesb3d95c52010-02-09 11:49:49 +00004513#if defined(VERBOSE_DEBUG)
Ben Hutchingsbcfcc452010-07-02 07:08:44 +00004514#define netif_vdbg netif_dbg
Joe Perchesb3d95c52010-02-09 11:49:49 +00004515#else
4516#define netif_vdbg(priv, type, dev, format, args...) \
4517({ \
4518 if (0) \
Ben Hutchingsa4ed89c2010-05-18 06:56:32 +00004519 netif_printk(priv, type, KERN_DEBUG, dev, format, ##args); \
Joe Perchesb3d95c52010-02-09 11:49:49 +00004520 0; \
4521})
4522#endif
Joe Perches571ba422010-02-09 11:49:47 +00004523
Cong Wang900ff8c2013-02-18 19:20:33 +00004524/*
4525 * The list of packet types we will receive (as opposed to discard)
4526 * and the routines to invoke.
4527 *
4528 * Why 16. Because with 16 the only overlap we get on a hash of the
4529 * low nibble of the protocol value is RARP/SNAP/X.25.
4530 *
Cong Wang900ff8c2013-02-18 19:20:33 +00004531 * 0800 IP
Cong Wang900ff8c2013-02-18 19:20:33 +00004532 * 0001 802.3
4533 * 0002 AX.25
4534 * 0004 802.2
4535 * 8035 RARP
4536 * 0005 SNAP
4537 * 0805 X.25
4538 * 0806 ARP
4539 * 8137 IPX
4540 * 0009 Localtalk
4541 * 86DD IPv6
4542 */
4543#define PTYPE_HASH_SIZE (16)
4544#define PTYPE_HASH_MASK (PTYPE_HASH_SIZE - 1)
4545
Jiri Pirko385a1542009-05-27 15:48:07 -07004546#endif /* _LINUX_NETDEVICE_H */