blob: 1024d3741d1284075ef6e4f1afb001c2ae3d693d [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
tcharding722c9a02017-02-09 17:56:04 +11002 * NET3 Protocol independent device support routines.
Linus Torvalds1da177e2005-04-16 15:20:36 -07003 *
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public License
6 * as published by the Free Software Foundation; either version
7 * 2 of the License, or (at your option) any later version.
8 *
9 * Derived from the non IP parts of dev.c 1.0.19
tcharding722c9a02017-02-09 17:56:04 +110010 * Authors: Ross Biro
Linus Torvalds1da177e2005-04-16 15:20:36 -070011 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
12 * Mark Evans, <evansmp@uhura.aston.ac.uk>
13 *
14 * Additional Authors:
15 * Florian la Roche <rzsfl@rz.uni-sb.de>
16 * Alan Cox <gw4pts@gw4pts.ampr.org>
17 * David Hinds <dahinds@users.sourceforge.net>
18 * Alexey Kuznetsov <kuznet@ms2.inr.ac.ru>
19 * Adam Sulmicki <adam@cfar.umd.edu>
20 * Pekka Riikonen <priikone@poesidon.pspt.fi>
21 *
22 * Changes:
23 * D.J. Barrow : Fixed bug where dev->refcnt gets set
tcharding722c9a02017-02-09 17:56:04 +110024 * to 2 if register_netdev gets called
25 * before net_dev_init & also removed a
26 * few lines of code in the process.
Linus Torvalds1da177e2005-04-16 15:20:36 -070027 * Alan Cox : device private ioctl copies fields back.
28 * Alan Cox : Transmit queue code does relevant
29 * stunts to keep the queue safe.
30 * Alan Cox : Fixed double lock.
31 * Alan Cox : Fixed promisc NULL pointer trap
32 * ???????? : Support the full private ioctl range
33 * Alan Cox : Moved ioctl permission check into
34 * drivers
35 * Tim Kordas : SIOCADDMULTI/SIOCDELMULTI
36 * Alan Cox : 100 backlog just doesn't cut it when
37 * you start doing multicast video 8)
38 * Alan Cox : Rewrote net_bh and list manager.
tcharding722c9a02017-02-09 17:56:04 +110039 * Alan Cox : Fix ETH_P_ALL echoback lengths.
Linus Torvalds1da177e2005-04-16 15:20:36 -070040 * Alan Cox : Took out transmit every packet pass
41 * Saved a few bytes in the ioctl handler
42 * Alan Cox : Network driver sets packet type before
43 * calling netif_rx. Saves a function
44 * call a packet.
45 * Alan Cox : Hashed net_bh()
46 * Richard Kooijman: Timestamp fixes.
47 * Alan Cox : Wrong field in SIOCGIFDSTADDR
48 * Alan Cox : Device lock protection.
tcharding722c9a02017-02-09 17:56:04 +110049 * Alan Cox : Fixed nasty side effect of device close
Linus Torvalds1da177e2005-04-16 15:20:36 -070050 * changes.
51 * Rudi Cilibrasi : Pass the right thing to
52 * set_mac_address()
53 * Dave Miller : 32bit quantity for the device lock to
54 * make it work out on a Sparc.
55 * Bjorn Ekwall : Added KERNELD hack.
56 * Alan Cox : Cleaned up the backlog initialise.
57 * Craig Metz : SIOCGIFCONF fix if space for under
58 * 1 device.
59 * Thomas Bogendoerfer : Return ENODEV for dev_open, if there
60 * is no device open function.
61 * Andi Kleen : Fix error reporting for SIOCGIFCONF
62 * Michael Chastain : Fix signed/unsigned for SIOCGIFCONF
63 * Cyrus Durgin : Cleaned for KMOD
64 * Adam Sulmicki : Bug Fix : Network Device Unload
65 * A network device unload needs to purge
66 * the backlog queue.
67 * Paul Rusty Russell : SIOCSIFNAME
68 * Pekka Riikonen : Netdev boot-time settings code
69 * Andrew Morton : Make unregister_netdevice wait
tcharding722c9a02017-02-09 17:56:04 +110070 * indefinitely on dev->refcnt
71 * J Hadi Salim : - Backlog queue sampling
Linus Torvalds1da177e2005-04-16 15:20:36 -070072 * - netif_rx() feedback
73 */
74
Linus Torvalds7c0f6ba2016-12-24 11:46:01 -080075#include <linux/uaccess.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070076#include <linux/bitops.h>
Randy Dunlap4fc268d2006-01-11 12:17:47 -080077#include <linux/capability.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070078#include <linux/cpu.h>
79#include <linux/types.h>
80#include <linux/kernel.h>
stephen hemminger08e98972009-11-10 07:20:34 +000081#include <linux/hash.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090082#include <linux/slab.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070083#include <linux/sched.h>
Vlastimil Babkaf1083042017-05-08 15:59:53 -070084#include <linux/sched/mm.h>
Arjan van de Ven4a3e2f72006-03-20 22:33:17 -080085#include <linux/mutex.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070086#include <linux/string.h>
87#include <linux/mm.h>
88#include <linux/socket.h>
89#include <linux/sockios.h>
90#include <linux/errno.h>
91#include <linux/interrupt.h>
92#include <linux/if_ether.h>
93#include <linux/netdevice.h>
94#include <linux/etherdevice.h>
Ben Hutchings0187bdf2008-06-19 16:15:47 -070095#include <linux/ethtool.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070096#include <linux/notifier.h>
97#include <linux/skbuff.h>
Brenden Blancoa7862b42016-07-19 12:16:48 -070098#include <linux/bpf.h>
David S. Millerb5cdae32017-04-18 15:36:58 -040099#include <linux/bpf_trace.h>
Eric W. Biederman457c4cb2007-09-12 12:01:34 +0200100#include <net/net_namespace.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700101#include <net/sock.h>
Eric Dumazet02d62e82015-11-18 06:30:52 -0800102#include <net/busy_poll.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700103#include <linux/rtnetlink.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700104#include <linux/stat.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700105#include <net/dst.h>
Pravin B Shelarfc4099f2015-10-22 18:17:16 -0700106#include <net/dst_metadata.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700107#include <net/pkt_sched.h>
Jiri Pirko87d83092017-05-17 11:07:54 +0200108#include <net/pkt_cls.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700109#include <net/checksum.h>
Arnd Bergmann44540962009-11-26 06:07:08 +0000110#include <net/xfrm.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700111#include <linux/highmem.h>
112#include <linux/init.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700113#include <linux/module.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700114#include <linux/netpoll.h>
115#include <linux/rcupdate.h>
116#include <linux/delay.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700117#include <net/iw_handler.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700118#include <asm/current.h>
Steve Grubb5bdb9882005-12-03 08:39:35 -0500119#include <linux/audit.h>
Chris Leechdb217332006-06-17 21:24:58 -0700120#include <linux/dmaengine.h>
Herbert Xuf6a78bf2006-06-22 02:57:17 -0700121#include <linux/err.h>
David S. Millerc7fa9d12006-08-15 16:34:13 -0700122#include <linux/ctype.h>
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700123#include <linux/if_arp.h>
Ben Hutchings6de329e2008-06-16 17:02:28 -0700124#include <linux/if_vlan.h>
David S. Miller8f0f2222008-07-15 03:47:03 -0700125#include <linux/ip.h>
Alexander Duyckad55dca2008-09-20 22:05:50 -0700126#include <net/ip.h>
Simon Horman25cd9ba2014-10-06 05:05:13 -0700127#include <net/mpls.h>
David S. Miller8f0f2222008-07-15 03:47:03 -0700128#include <linux/ipv6.h>
129#include <linux/in.h>
David S. Millerb6b2fed2008-07-21 09:48:06 -0700130#include <linux/jhash.h>
131#include <linux/random.h>
David S. Miller9cbc1cb2009-06-15 03:02:23 -0700132#include <trace/events/napi.h>
Koki Sanagicf66ba52010-08-23 18:45:02 +0900133#include <trace/events/net.h>
Koki Sanagi07dc22e2010-08-23 18:46:12 +0900134#include <trace/events/skb.h>
FUJITA Tomonori5acbbd42010-03-30 22:35:50 +0000135#include <linux/pci.h>
Stephen Rothwellcaeda9b2010-09-16 21:39:16 -0700136#include <linux/inetdevice.h>
Ben Hutchingsc4454772011-01-19 11:03:53 +0000137#include <linux/cpu_rmap.h>
Ingo Molnarc5905af2012-02-24 08:31:31 +0100138#include <linux/static_key.h>
Eliezer Tamiraf12fa62013-06-10 11:39:41 +0300139#include <linux/hashtable.h>
Eric Dumazet60877a32013-06-20 01:15:51 -0700140#include <linux/vmalloc.h>
Michal Kubeček529d0482013-11-15 06:18:50 +0100141#include <linux/if_macvlan.h>
Willem de Bruijne7fd2882014-08-04 22:11:48 -0400142#include <linux/errqueue.h>
Eric Dumazet3b47d302014-11-06 21:09:44 -0800143#include <linux/hrtimer.h>
Pablo Neirae687ad62015-05-13 18:19:38 +0200144#include <linux/netfilter_ingress.h>
Hariprasad Shenai40e4e712016-06-08 18:09:08 +0530145#include <linux/crash_dump.h>
Davide Carattib72b5bf2017-05-18 15:44:38 +0200146#include <linux/sctp.h>
Sabrina Dubrocaae847f42017-07-21 12:49:31 +0200147#include <net/udp_tunnel.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700148
Pavel Emelyanov342709e2007-10-23 21:14:45 -0700149#include "net-sysfs.h"
150
Herbert Xud565b0a2008-12-15 23:38:52 -0800151/* Instead of increasing this, you should create a hash table. */
152#define MAX_GRO_SKBS 8
153
Herbert Xu5d38a072009-01-04 16:13:40 -0800154/* This should be increased if a protocol with a bigger head is added. */
155#define GRO_MAX_HEAD (MAX_HEADER + 128)
156
Linus Torvalds1da177e2005-04-16 15:20:36 -0700157static DEFINE_SPINLOCK(ptype_lock);
Vlad Yasevich62532da2012-11-15 08:49:10 +0000158static DEFINE_SPINLOCK(offload_lock);
Cong Wang900ff8c2013-02-18 19:20:33 +0000159struct list_head ptype_base[PTYPE_HASH_SIZE] __read_mostly;
160struct list_head ptype_all __read_mostly; /* Taps */
Vlad Yasevich62532da2012-11-15 08:49:10 +0000161static struct list_head offload_base __read_mostly;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700162
Ben Hutchingsae78dbf2014-01-10 22:17:24 +0000163static int netif_rx_internal(struct sk_buff *skb);
Loic Prylli54951192014-07-01 21:39:43 -0700164static int call_netdevice_notifiers_info(unsigned long val,
165 struct net_device *dev,
166 struct netdev_notifier_info *info);
Miroslav Lichvar90b602f2017-05-19 17:52:37 +0200167static struct napi_struct *napi_by_id(unsigned int napi_id);
Ben Hutchingsae78dbf2014-01-10 22:17:24 +0000168
Linus Torvalds1da177e2005-04-16 15:20:36 -0700169/*
Pavel Emelianov7562f872007-05-03 15:13:45 -0700170 * The @dev_base_head list is protected by @dev_base_lock and the rtnl
Linus Torvalds1da177e2005-04-16 15:20:36 -0700171 * semaphore.
172 *
Eric Dumazetc6d14c82009-11-04 05:43:23 -0800173 * Pure readers hold dev_base_lock for reading, or rcu_read_lock()
Linus Torvalds1da177e2005-04-16 15:20:36 -0700174 *
175 * Writers must hold the rtnl semaphore while they loop through the
Pavel Emelianov7562f872007-05-03 15:13:45 -0700176 * dev_base_head list, and hold dev_base_lock for writing when they do the
Linus Torvalds1da177e2005-04-16 15:20:36 -0700177 * actual updates. This allows pure readers to access the list even
178 * while a writer is preparing to update it.
179 *
180 * To put it another way, dev_base_lock is held for writing only to
181 * protect against pure readers; the rtnl semaphore provides the
182 * protection against other writers.
183 *
184 * See, for example usages, register_netdevice() and
185 * unregister_netdevice(), which must be called with the rtnl
186 * semaphore held.
187 */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700188DEFINE_RWLOCK(dev_base_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700189EXPORT_SYMBOL(dev_base_lock);
190
Eliezer Tamiraf12fa62013-06-10 11:39:41 +0300191/* protects napi_hash addition/deletion and napi_gen_id */
192static DEFINE_SPINLOCK(napi_hash_lock);
193
Eric Dumazet52bd2d62015-11-18 06:30:50 -0800194static unsigned int napi_gen_id = NR_CPUS;
Eric Dumazet6180d9d2015-11-18 06:31:01 -0800195static DEFINE_READ_MOSTLY_HASHTABLE(napi_hash, 8);
Eliezer Tamiraf12fa62013-06-10 11:39:41 +0300196
Thomas Gleixner18afa4b2013-07-23 16:13:17 +0200197static seqcount_t devnet_rename_seq;
Brian Haleyc91f6df2012-11-26 05:21:08 +0000198
Thomas Graf4e985ad2011-06-21 03:11:20 +0000199static inline void dev_base_seq_inc(struct net *net)
200{
tcharding643aa9c2017-02-09 17:56:05 +1100201 while (++net->dev_base_seq == 0)
202 ;
Thomas Graf4e985ad2011-06-21 03:11:20 +0000203}
204
Eric W. Biederman881d9662007-09-17 11:56:21 -0700205static inline struct hlist_head *dev_name_hash(struct net *net, const char *name)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700206{
Linus Torvalds8387ff22016-06-10 07:51:30 -0700207 unsigned int hash = full_name_hash(net, name, strnlen(name, IFNAMSIZ));
Eric Dumazet95c96172012-04-15 05:58:06 +0000208
stephen hemminger08e98972009-11-10 07:20:34 +0000209 return &net->dev_name_head[hash_32(hash, NETDEV_HASHBITS)];
Linus Torvalds1da177e2005-04-16 15:20:36 -0700210}
211
Eric W. Biederman881d9662007-09-17 11:56:21 -0700212static inline struct hlist_head *dev_index_hash(struct net *net, int ifindex)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700213{
Eric Dumazet7c28bd02009-10-24 06:13:17 -0700214 return &net->dev_index_head[ifindex & (NETDEV_HASHENTRIES - 1)];
Linus Torvalds1da177e2005-04-16 15:20:36 -0700215}
216
Eric Dumazete36fa2f2010-04-19 21:17:14 +0000217static inline void rps_lock(struct softnet_data *sd)
Changli Gao152102c2010-03-30 20:16:22 +0000218{
219#ifdef CONFIG_RPS
Eric Dumazete36fa2f2010-04-19 21:17:14 +0000220 spin_lock(&sd->input_pkt_queue.lock);
Changli Gao152102c2010-03-30 20:16:22 +0000221#endif
222}
223
Eric Dumazete36fa2f2010-04-19 21:17:14 +0000224static inline void rps_unlock(struct softnet_data *sd)
Changli Gao152102c2010-03-30 20:16:22 +0000225{
226#ifdef CONFIG_RPS
Eric Dumazete36fa2f2010-04-19 21:17:14 +0000227 spin_unlock(&sd->input_pkt_queue.lock);
Changli Gao152102c2010-03-30 20:16:22 +0000228#endif
229}
230
Eric W. Biedermance286d32007-09-12 13:53:49 +0200231/* Device list insertion */
dingtianhong53759be2013-04-17 22:17:50 +0000232static void list_netdevice(struct net_device *dev)
Eric W. Biedermance286d32007-09-12 13:53:49 +0200233{
YOSHIFUJI Hideakic346dca2008-03-25 21:47:49 +0900234 struct net *net = dev_net(dev);
Eric W. Biedermance286d32007-09-12 13:53:49 +0200235
236 ASSERT_RTNL();
237
238 write_lock_bh(&dev_base_lock);
Eric Dumazetc6d14c82009-11-04 05:43:23 -0800239 list_add_tail_rcu(&dev->dev_list, &net->dev_base_head);
Eric Dumazet72c95282009-10-30 07:11:27 +0000240 hlist_add_head_rcu(&dev->name_hlist, dev_name_hash(net, dev->name));
Eric Dumazetfb699dfd2009-10-19 19:18:49 +0000241 hlist_add_head_rcu(&dev->index_hlist,
242 dev_index_hash(net, dev->ifindex));
Eric W. Biedermance286d32007-09-12 13:53:49 +0200243 write_unlock_bh(&dev_base_lock);
Thomas Graf4e985ad2011-06-21 03:11:20 +0000244
245 dev_base_seq_inc(net);
Eric W. Biedermance286d32007-09-12 13:53:49 +0200246}
247
Eric Dumazetfb699dfd2009-10-19 19:18:49 +0000248/* Device list removal
249 * caller must respect a RCU grace period before freeing/reusing dev
250 */
Eric W. Biedermance286d32007-09-12 13:53:49 +0200251static void unlist_netdevice(struct net_device *dev)
252{
253 ASSERT_RTNL();
254
255 /* Unlink dev from the device chain */
256 write_lock_bh(&dev_base_lock);
Eric Dumazetc6d14c82009-11-04 05:43:23 -0800257 list_del_rcu(&dev->dev_list);
Eric Dumazet72c95282009-10-30 07:11:27 +0000258 hlist_del_rcu(&dev->name_hlist);
Eric Dumazetfb699dfd2009-10-19 19:18:49 +0000259 hlist_del_rcu(&dev->index_hlist);
Eric W. Biedermance286d32007-09-12 13:53:49 +0200260 write_unlock_bh(&dev_base_lock);
Thomas Graf4e985ad2011-06-21 03:11:20 +0000261
262 dev_base_seq_inc(dev_net(dev));
Eric W. Biedermance286d32007-09-12 13:53:49 +0200263}
264
Linus Torvalds1da177e2005-04-16 15:20:36 -0700265/*
266 * Our notifier list
267 */
268
Alan Sternf07d5b92006-05-09 15:23:03 -0700269static RAW_NOTIFIER_HEAD(netdev_chain);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700270
271/*
272 * Device drivers call our routines to queue packets here. We empty the
273 * queue in the local softnet handler.
274 */
Stephen Hemmingerbea33482007-10-03 16:41:36 -0700275
Eric Dumazet9958da02010-04-17 04:17:02 +0000276DEFINE_PER_CPU_ALIGNED(struct softnet_data, softnet_data);
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700277EXPORT_PER_CPU_SYMBOL(softnet_data);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700278
David S. Millercf508b12008-07-22 14:16:42 -0700279#ifdef CONFIG_LOCKDEP
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700280/*
David S. Millerc773e842008-07-08 23:13:53 -0700281 * register_netdevice() inits txq->_xmit_lock and sets lockdep class
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700282 * according to dev->type
283 */
tcharding643aa9c2017-02-09 17:56:05 +1100284static const unsigned short netdev_lock_type[] = {
285 ARPHRD_NETROM, ARPHRD_ETHER, ARPHRD_EETHER, ARPHRD_AX25,
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700286 ARPHRD_PRONET, ARPHRD_CHAOS, ARPHRD_IEEE802, ARPHRD_ARCNET,
287 ARPHRD_APPLETLK, ARPHRD_DLCI, ARPHRD_ATM, ARPHRD_METRICOM,
288 ARPHRD_IEEE1394, ARPHRD_EUI64, ARPHRD_INFINIBAND, ARPHRD_SLIP,
289 ARPHRD_CSLIP, ARPHRD_SLIP6, ARPHRD_CSLIP6, ARPHRD_RSRVD,
290 ARPHRD_ADAPT, ARPHRD_ROSE, ARPHRD_X25, ARPHRD_HWX25,
291 ARPHRD_PPP, ARPHRD_CISCO, ARPHRD_LAPB, ARPHRD_DDCMP,
292 ARPHRD_RAWHDLC, ARPHRD_TUNNEL, ARPHRD_TUNNEL6, ARPHRD_FRAD,
293 ARPHRD_SKIP, ARPHRD_LOOPBACK, ARPHRD_LOCALTLK, ARPHRD_FDDI,
294 ARPHRD_BIF, ARPHRD_SIT, ARPHRD_IPDDP, ARPHRD_IPGRE,
295 ARPHRD_PIMREG, ARPHRD_HIPPI, ARPHRD_ASH, ARPHRD_ECONET,
296 ARPHRD_IRDA, ARPHRD_FCPP, ARPHRD_FCAL, ARPHRD_FCPL,
Paul Gortmaker211ed862012-05-10 17:14:35 -0400297 ARPHRD_FCFABRIC, ARPHRD_IEEE80211, ARPHRD_IEEE80211_PRISM,
298 ARPHRD_IEEE80211_RADIOTAP, ARPHRD_PHONET, ARPHRD_PHONET_PIPE,
299 ARPHRD_IEEE802154, ARPHRD_VOID, ARPHRD_NONE};
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700300
tcharding643aa9c2017-02-09 17:56:05 +1100301static const char *const netdev_lock_name[] = {
302 "_xmit_NETROM", "_xmit_ETHER", "_xmit_EETHER", "_xmit_AX25",
303 "_xmit_PRONET", "_xmit_CHAOS", "_xmit_IEEE802", "_xmit_ARCNET",
304 "_xmit_APPLETLK", "_xmit_DLCI", "_xmit_ATM", "_xmit_METRICOM",
305 "_xmit_IEEE1394", "_xmit_EUI64", "_xmit_INFINIBAND", "_xmit_SLIP",
306 "_xmit_CSLIP", "_xmit_SLIP6", "_xmit_CSLIP6", "_xmit_RSRVD",
307 "_xmit_ADAPT", "_xmit_ROSE", "_xmit_X25", "_xmit_HWX25",
308 "_xmit_PPP", "_xmit_CISCO", "_xmit_LAPB", "_xmit_DDCMP",
309 "_xmit_RAWHDLC", "_xmit_TUNNEL", "_xmit_TUNNEL6", "_xmit_FRAD",
310 "_xmit_SKIP", "_xmit_LOOPBACK", "_xmit_LOCALTLK", "_xmit_FDDI",
311 "_xmit_BIF", "_xmit_SIT", "_xmit_IPDDP", "_xmit_IPGRE",
312 "_xmit_PIMREG", "_xmit_HIPPI", "_xmit_ASH", "_xmit_ECONET",
313 "_xmit_IRDA", "_xmit_FCPP", "_xmit_FCAL", "_xmit_FCPL",
314 "_xmit_FCFABRIC", "_xmit_IEEE80211", "_xmit_IEEE80211_PRISM",
315 "_xmit_IEEE80211_RADIOTAP", "_xmit_PHONET", "_xmit_PHONET_PIPE",
316 "_xmit_IEEE802154", "_xmit_VOID", "_xmit_NONE"};
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700317
318static struct lock_class_key netdev_xmit_lock_key[ARRAY_SIZE(netdev_lock_type)];
David S. Millercf508b12008-07-22 14:16:42 -0700319static struct lock_class_key netdev_addr_lock_key[ARRAY_SIZE(netdev_lock_type)];
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700320
321static inline unsigned short netdev_lock_pos(unsigned short dev_type)
322{
323 int i;
324
325 for (i = 0; i < ARRAY_SIZE(netdev_lock_type); i++)
326 if (netdev_lock_type[i] == dev_type)
327 return i;
328 /* the last key is used by default */
329 return ARRAY_SIZE(netdev_lock_type) - 1;
330}
331
David S. Millercf508b12008-07-22 14:16:42 -0700332static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock,
333 unsigned short dev_type)
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700334{
335 int i;
336
337 i = netdev_lock_pos(dev_type);
338 lockdep_set_class_and_name(lock, &netdev_xmit_lock_key[i],
339 netdev_lock_name[i]);
340}
David S. Millercf508b12008-07-22 14:16:42 -0700341
342static inline void netdev_set_addr_lockdep_class(struct net_device *dev)
343{
344 int i;
345
346 i = netdev_lock_pos(dev->type);
347 lockdep_set_class_and_name(&dev->addr_list_lock,
348 &netdev_addr_lock_key[i],
349 netdev_lock_name[i]);
350}
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700351#else
David S. Millercf508b12008-07-22 14:16:42 -0700352static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock,
353 unsigned short dev_type)
354{
355}
356static inline void netdev_set_addr_lockdep_class(struct net_device *dev)
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700357{
358}
359#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -0700360
361/*******************************************************************************
tchardingeb13da12017-02-09 17:56:06 +1100362 *
363 * Protocol management and registration routines
364 *
365 *******************************************************************************/
Linus Torvalds1da177e2005-04-16 15:20:36 -0700366
Linus Torvalds1da177e2005-04-16 15:20:36 -0700367
368/*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700369 * Add a protocol ID to the list. Now that the input handler is
370 * smarter we can dispense with all the messy stuff that used to be
371 * here.
372 *
373 * BEWARE!!! Protocol handlers, mangling input packets,
374 * MUST BE last in hash buckets and checking protocol handlers
375 * MUST start from promiscuous ptype_all chain in net_bh.
376 * It is true now, do not change it.
377 * Explanation follows: if protocol handler, mangling packet, will
378 * be the first on list, it is not able to sense, that packet
379 * is cloned and should be copied-on-write, so that it will
380 * change it and subsequent readers will get broken packet.
381 * --ANK (980803)
382 */
383
Eric Dumazetc07b68e2010-09-02 03:53:46 +0000384static inline struct list_head *ptype_head(const struct packet_type *pt)
385{
386 if (pt->type == htons(ETH_P_ALL))
Salam Noureddine7866a622015-01-27 11:35:48 -0800387 return pt->dev ? &pt->dev->ptype_all : &ptype_all;
Eric Dumazetc07b68e2010-09-02 03:53:46 +0000388 else
Salam Noureddine7866a622015-01-27 11:35:48 -0800389 return pt->dev ? &pt->dev->ptype_specific :
390 &ptype_base[ntohs(pt->type) & PTYPE_HASH_MASK];
Eric Dumazetc07b68e2010-09-02 03:53:46 +0000391}
392
Linus Torvalds1da177e2005-04-16 15:20:36 -0700393/**
394 * dev_add_pack - add packet handler
395 * @pt: packet type declaration
396 *
397 * Add a protocol handler to the networking stack. The passed &packet_type
398 * is linked into kernel lists and may not be freed until it has been
399 * removed from the kernel lists.
400 *
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +0900401 * This call does not sleep therefore it can not
Linus Torvalds1da177e2005-04-16 15:20:36 -0700402 * guarantee all CPU's that are in middle of receiving packets
403 * will see the new packet type (until the next received packet).
404 */
405
406void dev_add_pack(struct packet_type *pt)
407{
Eric Dumazetc07b68e2010-09-02 03:53:46 +0000408 struct list_head *head = ptype_head(pt);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700409
Eric Dumazetc07b68e2010-09-02 03:53:46 +0000410 spin_lock(&ptype_lock);
411 list_add_rcu(&pt->list, head);
412 spin_unlock(&ptype_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700413}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700414EXPORT_SYMBOL(dev_add_pack);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700415
Linus Torvalds1da177e2005-04-16 15:20:36 -0700416/**
417 * __dev_remove_pack - remove packet handler
418 * @pt: packet type declaration
419 *
420 * Remove a protocol handler that was previously added to the kernel
421 * protocol handlers by dev_add_pack(). The passed &packet_type is removed
422 * from the kernel lists and can be freed or reused once this function
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +0900423 * returns.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700424 *
425 * The packet type might still be in use by receivers
426 * and must not be freed until after all the CPU's have gone
427 * through a quiescent state.
428 */
429void __dev_remove_pack(struct packet_type *pt)
430{
Eric Dumazetc07b68e2010-09-02 03:53:46 +0000431 struct list_head *head = ptype_head(pt);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700432 struct packet_type *pt1;
433
Eric Dumazetc07b68e2010-09-02 03:53:46 +0000434 spin_lock(&ptype_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700435
436 list_for_each_entry(pt1, head, list) {
437 if (pt == pt1) {
438 list_del_rcu(&pt->list);
439 goto out;
440 }
441 }
442
Joe Perches7b6cd1c2012-02-01 10:54:43 +0000443 pr_warn("dev_remove_pack: %p not found\n", pt);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700444out:
Eric Dumazetc07b68e2010-09-02 03:53:46 +0000445 spin_unlock(&ptype_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700446}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700447EXPORT_SYMBOL(__dev_remove_pack);
448
Linus Torvalds1da177e2005-04-16 15:20:36 -0700449/**
450 * dev_remove_pack - remove packet handler
451 * @pt: packet type declaration
452 *
453 * Remove a protocol handler that was previously added to the kernel
454 * protocol handlers by dev_add_pack(). The passed &packet_type is removed
455 * from the kernel lists and can be freed or reused once this function
456 * returns.
457 *
458 * This call sleeps to guarantee that no CPU is looking at the packet
459 * type after return.
460 */
461void dev_remove_pack(struct packet_type *pt)
462{
463 __dev_remove_pack(pt);
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +0900464
Linus Torvalds1da177e2005-04-16 15:20:36 -0700465 synchronize_net();
466}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700467EXPORT_SYMBOL(dev_remove_pack);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700468
Vlad Yasevich62532da2012-11-15 08:49:10 +0000469
470/**
471 * dev_add_offload - register offload handlers
472 * @po: protocol offload declaration
473 *
474 * Add protocol offload handlers to the networking stack. The passed
475 * &proto_offload is linked into kernel lists and may not be freed until
476 * it has been removed from the kernel lists.
477 *
478 * This call does not sleep therefore it can not
479 * guarantee all CPU's that are in middle of receiving packets
480 * will see the new offload handlers (until the next received packet).
481 */
482void dev_add_offload(struct packet_offload *po)
483{
David S. Millerbdef7de2015-06-01 14:56:09 -0700484 struct packet_offload *elem;
Vlad Yasevich62532da2012-11-15 08:49:10 +0000485
486 spin_lock(&offload_lock);
David S. Millerbdef7de2015-06-01 14:56:09 -0700487 list_for_each_entry(elem, &offload_base, list) {
488 if (po->priority < elem->priority)
489 break;
490 }
491 list_add_rcu(&po->list, elem->list.prev);
Vlad Yasevich62532da2012-11-15 08:49:10 +0000492 spin_unlock(&offload_lock);
493}
494EXPORT_SYMBOL(dev_add_offload);
495
496/**
497 * __dev_remove_offload - remove offload handler
498 * @po: packet offload declaration
499 *
500 * Remove a protocol offload handler that was previously added to the
501 * kernel offload handlers by dev_add_offload(). The passed &offload_type
502 * is removed from the kernel lists and can be freed or reused once this
503 * function returns.
504 *
505 * The packet type might still be in use by receivers
506 * and must not be freed until after all the CPU's have gone
507 * through a quiescent state.
508 */
stephen hemminger1d143d92013-12-29 14:01:29 -0800509static void __dev_remove_offload(struct packet_offload *po)
Vlad Yasevich62532da2012-11-15 08:49:10 +0000510{
511 struct list_head *head = &offload_base;
512 struct packet_offload *po1;
513
Eric Dumazetc53aa502012-11-16 08:08:23 +0000514 spin_lock(&offload_lock);
Vlad Yasevich62532da2012-11-15 08:49:10 +0000515
516 list_for_each_entry(po1, head, list) {
517 if (po == po1) {
518 list_del_rcu(&po->list);
519 goto out;
520 }
521 }
522
523 pr_warn("dev_remove_offload: %p not found\n", po);
524out:
Eric Dumazetc53aa502012-11-16 08:08:23 +0000525 spin_unlock(&offload_lock);
Vlad Yasevich62532da2012-11-15 08:49:10 +0000526}
Vlad Yasevich62532da2012-11-15 08:49:10 +0000527
528/**
529 * dev_remove_offload - remove packet offload handler
530 * @po: packet offload declaration
531 *
532 * Remove a packet offload handler that was previously added to the kernel
533 * offload handlers by dev_add_offload(). The passed &offload_type is
534 * removed from the kernel lists and can be freed or reused once this
535 * function returns.
536 *
537 * This call sleeps to guarantee that no CPU is looking at the packet
538 * type after return.
539 */
540void dev_remove_offload(struct packet_offload *po)
541{
542 __dev_remove_offload(po);
543
544 synchronize_net();
545}
546EXPORT_SYMBOL(dev_remove_offload);
547
Linus Torvalds1da177e2005-04-16 15:20:36 -0700548/******************************************************************************
tchardingeb13da12017-02-09 17:56:06 +1100549 *
550 * Device Boot-time Settings Routines
551 *
552 ******************************************************************************/
Linus Torvalds1da177e2005-04-16 15:20:36 -0700553
554/* Boot time configuration table */
555static struct netdev_boot_setup dev_boot_setup[NETDEV_BOOT_SETUP_MAX];
556
557/**
558 * netdev_boot_setup_add - add new setup entry
559 * @name: name of the device
560 * @map: configured settings for the device
561 *
562 * Adds new setup entry to the dev_boot_setup list. The function
563 * returns 0 on error and 1 on success. This is a generic routine to
564 * all netdevices.
565 */
566static int netdev_boot_setup_add(char *name, struct ifmap *map)
567{
568 struct netdev_boot_setup *s;
569 int i;
570
571 s = dev_boot_setup;
572 for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++) {
573 if (s[i].name[0] == '\0' || s[i].name[0] == ' ') {
574 memset(s[i].name, 0, sizeof(s[i].name));
Wang Chen93b3cff2008-07-01 19:57:19 -0700575 strlcpy(s[i].name, name, IFNAMSIZ);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700576 memcpy(&s[i].map, map, sizeof(s[i].map));
577 break;
578 }
579 }
580
581 return i >= NETDEV_BOOT_SETUP_MAX ? 0 : 1;
582}
583
584/**
tcharding722c9a02017-02-09 17:56:04 +1100585 * netdev_boot_setup_check - check boot time settings
586 * @dev: the netdevice
Linus Torvalds1da177e2005-04-16 15:20:36 -0700587 *
tcharding722c9a02017-02-09 17:56:04 +1100588 * Check boot time settings for the device.
589 * The found settings are set for the device to be used
590 * later in the device probing.
591 * Returns 0 if no settings found, 1 if they are.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700592 */
593int netdev_boot_setup_check(struct net_device *dev)
594{
595 struct netdev_boot_setup *s = dev_boot_setup;
596 int i;
597
598 for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++) {
599 if (s[i].name[0] != '\0' && s[i].name[0] != ' ' &&
Wang Chen93b3cff2008-07-01 19:57:19 -0700600 !strcmp(dev->name, s[i].name)) {
tcharding722c9a02017-02-09 17:56:04 +1100601 dev->irq = s[i].map.irq;
602 dev->base_addr = s[i].map.base_addr;
603 dev->mem_start = s[i].map.mem_start;
604 dev->mem_end = s[i].map.mem_end;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700605 return 1;
606 }
607 }
608 return 0;
609}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700610EXPORT_SYMBOL(netdev_boot_setup_check);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700611
612
613/**
tcharding722c9a02017-02-09 17:56:04 +1100614 * netdev_boot_base - get address from boot time settings
615 * @prefix: prefix for network device
616 * @unit: id for network device
Linus Torvalds1da177e2005-04-16 15:20:36 -0700617 *
tcharding722c9a02017-02-09 17:56:04 +1100618 * Check boot time settings for the base address of device.
619 * The found settings are set for the device to be used
620 * later in the device probing.
621 * Returns 0 if no settings found.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700622 */
623unsigned long netdev_boot_base(const char *prefix, int unit)
624{
625 const struct netdev_boot_setup *s = dev_boot_setup;
626 char name[IFNAMSIZ];
627 int i;
628
629 sprintf(name, "%s%d", prefix, unit);
630
631 /*
632 * If device already registered then return base of 1
633 * to indicate not to probe for this interface
634 */
Eric W. Biederman881d9662007-09-17 11:56:21 -0700635 if (__dev_get_by_name(&init_net, name))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700636 return 1;
637
638 for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++)
639 if (!strcmp(name, s[i].name))
640 return s[i].map.base_addr;
641 return 0;
642}
643
644/*
645 * Saves at boot time configured settings for any netdevice.
646 */
647int __init netdev_boot_setup(char *str)
648{
649 int ints[5];
650 struct ifmap map;
651
652 str = get_options(str, ARRAY_SIZE(ints), ints);
653 if (!str || !*str)
654 return 0;
655
656 /* Save settings */
657 memset(&map, 0, sizeof(map));
658 if (ints[0] > 0)
659 map.irq = ints[1];
660 if (ints[0] > 1)
661 map.base_addr = ints[2];
662 if (ints[0] > 2)
663 map.mem_start = ints[3];
664 if (ints[0] > 3)
665 map.mem_end = ints[4];
666
667 /* Add new entry to the list */
668 return netdev_boot_setup_add(str, &map);
669}
670
671__setup("netdev=", netdev_boot_setup);
672
673/*******************************************************************************
tchardingeb13da12017-02-09 17:56:06 +1100674 *
675 * Device Interface Subroutines
676 *
677 *******************************************************************************/
Linus Torvalds1da177e2005-04-16 15:20:36 -0700678
679/**
Nicolas Dichtela54acb32015-04-02 17:07:00 +0200680 * dev_get_iflink - get 'iflink' value of a interface
681 * @dev: targeted interface
682 *
683 * Indicates the ifindex the interface is linked to.
684 * Physical interfaces have the same 'ifindex' and 'iflink' values.
685 */
686
687int dev_get_iflink(const struct net_device *dev)
688{
689 if (dev->netdev_ops && dev->netdev_ops->ndo_get_iflink)
690 return dev->netdev_ops->ndo_get_iflink(dev);
691
Nicolas Dichtel7a66bbc2015-04-02 17:07:09 +0200692 return dev->ifindex;
Nicolas Dichtela54acb32015-04-02 17:07:00 +0200693}
694EXPORT_SYMBOL(dev_get_iflink);
695
696/**
Pravin B Shelarfc4099f2015-10-22 18:17:16 -0700697 * dev_fill_metadata_dst - Retrieve tunnel egress information.
698 * @dev: targeted interface
699 * @skb: The packet.
700 *
701 * For better visibility of tunnel traffic OVS needs to retrieve
702 * egress tunnel information for a packet. Following API allows
703 * user to get this info.
704 */
705int dev_fill_metadata_dst(struct net_device *dev, struct sk_buff *skb)
706{
707 struct ip_tunnel_info *info;
708
709 if (!dev->netdev_ops || !dev->netdev_ops->ndo_fill_metadata_dst)
710 return -EINVAL;
711
712 info = skb_tunnel_info_unclone(skb);
713 if (!info)
714 return -ENOMEM;
715 if (unlikely(!(info->mode & IP_TUNNEL_INFO_TX)))
716 return -EINVAL;
717
718 return dev->netdev_ops->ndo_fill_metadata_dst(dev, skb);
719}
720EXPORT_SYMBOL_GPL(dev_fill_metadata_dst);
721
722/**
Linus Torvalds1da177e2005-04-16 15:20:36 -0700723 * __dev_get_by_name - find a device by its name
Randy Dunlapc4ea43c2007-10-12 21:17:49 -0700724 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -0700725 * @name: name to find
726 *
727 * Find an interface by name. Must be called under RTNL semaphore
728 * or @dev_base_lock. If the name is found a pointer to the device
729 * is returned. If the name is not found then %NULL is returned. The
730 * reference counters are not incremented so the caller must be
731 * careful with locks.
732 */
733
Eric W. Biederman881d9662007-09-17 11:56:21 -0700734struct net_device *__dev_get_by_name(struct net *net, const char *name)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700735{
Eric Dumazet0bd8d532009-10-30 01:40:11 -0700736 struct net_device *dev;
737 struct hlist_head *head = dev_name_hash(net, name);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700738
Sasha Levinb67bfe02013-02-27 17:06:00 -0800739 hlist_for_each_entry(dev, head, name_hlist)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700740 if (!strncmp(dev->name, name, IFNAMSIZ))
741 return dev;
Eric Dumazet0bd8d532009-10-30 01:40:11 -0700742
Linus Torvalds1da177e2005-04-16 15:20:36 -0700743 return NULL;
744}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700745EXPORT_SYMBOL(__dev_get_by_name);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700746
747/**
tcharding722c9a02017-02-09 17:56:04 +1100748 * dev_get_by_name_rcu - find a device by its name
749 * @net: the applicable net namespace
750 * @name: name to find
Eric Dumazet72c95282009-10-30 07:11:27 +0000751 *
tcharding722c9a02017-02-09 17:56:04 +1100752 * Find an interface by name.
753 * If the name is found a pointer to the device is returned.
754 * If the name is not found then %NULL is returned.
755 * The reference counters are not incremented so the caller must be
756 * careful with locks. The caller must hold RCU lock.
Eric Dumazet72c95282009-10-30 07:11:27 +0000757 */
758
759struct net_device *dev_get_by_name_rcu(struct net *net, const char *name)
760{
Eric Dumazet72c95282009-10-30 07:11:27 +0000761 struct net_device *dev;
762 struct hlist_head *head = dev_name_hash(net, name);
763
Sasha Levinb67bfe02013-02-27 17:06:00 -0800764 hlist_for_each_entry_rcu(dev, head, name_hlist)
Eric Dumazet72c95282009-10-30 07:11:27 +0000765 if (!strncmp(dev->name, name, IFNAMSIZ))
766 return dev;
767
768 return NULL;
769}
770EXPORT_SYMBOL(dev_get_by_name_rcu);
771
772/**
Linus Torvalds1da177e2005-04-16 15:20:36 -0700773 * dev_get_by_name - find a device by its name
Randy Dunlapc4ea43c2007-10-12 21:17:49 -0700774 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -0700775 * @name: name to find
776 *
777 * Find an interface by name. This can be called from any
778 * context and does its own locking. The returned handle has
779 * the usage count incremented and the caller must use dev_put() to
780 * release it when it is no longer needed. %NULL is returned if no
781 * matching device is found.
782 */
783
Eric W. Biederman881d9662007-09-17 11:56:21 -0700784struct net_device *dev_get_by_name(struct net *net, const char *name)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700785{
786 struct net_device *dev;
787
Eric Dumazet72c95282009-10-30 07:11:27 +0000788 rcu_read_lock();
789 dev = dev_get_by_name_rcu(net, name);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700790 if (dev)
791 dev_hold(dev);
Eric Dumazet72c95282009-10-30 07:11:27 +0000792 rcu_read_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700793 return dev;
794}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700795EXPORT_SYMBOL(dev_get_by_name);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700796
797/**
798 * __dev_get_by_index - find a device by its ifindex
Randy Dunlapc4ea43c2007-10-12 21:17:49 -0700799 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -0700800 * @ifindex: index of device
801 *
802 * Search for an interface by index. Returns %NULL if the device
803 * is not found or a pointer to the device. The device has not
804 * had its reference counter increased so the caller must be careful
805 * about locking. The caller must hold either the RTNL semaphore
806 * or @dev_base_lock.
807 */
808
Eric W. Biederman881d9662007-09-17 11:56:21 -0700809struct net_device *__dev_get_by_index(struct net *net, int ifindex)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700810{
Eric Dumazet0bd8d532009-10-30 01:40:11 -0700811 struct net_device *dev;
812 struct hlist_head *head = dev_index_hash(net, ifindex);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700813
Sasha Levinb67bfe02013-02-27 17:06:00 -0800814 hlist_for_each_entry(dev, head, index_hlist)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700815 if (dev->ifindex == ifindex)
816 return dev;
Eric Dumazet0bd8d532009-10-30 01:40:11 -0700817
Linus Torvalds1da177e2005-04-16 15:20:36 -0700818 return NULL;
819}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700820EXPORT_SYMBOL(__dev_get_by_index);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700821
Eric Dumazetfb699dfd2009-10-19 19:18:49 +0000822/**
823 * dev_get_by_index_rcu - find a device by its ifindex
824 * @net: the applicable net namespace
825 * @ifindex: index of device
826 *
827 * Search for an interface by index. Returns %NULL if the device
828 * is not found or a pointer to the device. The device has not
829 * had its reference counter increased so the caller must be careful
830 * about locking. The caller must hold RCU lock.
831 */
832
833struct net_device *dev_get_by_index_rcu(struct net *net, int ifindex)
834{
Eric Dumazetfb699dfd2009-10-19 19:18:49 +0000835 struct net_device *dev;
836 struct hlist_head *head = dev_index_hash(net, ifindex);
837
Sasha Levinb67bfe02013-02-27 17:06:00 -0800838 hlist_for_each_entry_rcu(dev, head, index_hlist)
Eric Dumazetfb699dfd2009-10-19 19:18:49 +0000839 if (dev->ifindex == ifindex)
840 return dev;
841
842 return NULL;
843}
844EXPORT_SYMBOL(dev_get_by_index_rcu);
845
Linus Torvalds1da177e2005-04-16 15:20:36 -0700846
847/**
848 * dev_get_by_index - find a device by its ifindex
Randy Dunlapc4ea43c2007-10-12 21:17:49 -0700849 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -0700850 * @ifindex: index of device
851 *
852 * Search for an interface by index. Returns NULL if the device
853 * is not found or a pointer to the device. The device returned has
854 * had a reference added and the pointer is safe until the user calls
855 * dev_put to indicate they have finished with it.
856 */
857
Eric W. Biederman881d9662007-09-17 11:56:21 -0700858struct net_device *dev_get_by_index(struct net *net, int ifindex)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700859{
860 struct net_device *dev;
861
Eric Dumazetfb699dfd2009-10-19 19:18:49 +0000862 rcu_read_lock();
863 dev = dev_get_by_index_rcu(net, ifindex);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700864 if (dev)
865 dev_hold(dev);
Eric Dumazetfb699dfd2009-10-19 19:18:49 +0000866 rcu_read_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700867 return dev;
868}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700869EXPORT_SYMBOL(dev_get_by_index);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700870
871/**
Miroslav Lichvar90b602f2017-05-19 17:52:37 +0200872 * dev_get_by_napi_id - find a device by napi_id
873 * @napi_id: ID of the NAPI struct
874 *
875 * Search for an interface by NAPI ID. Returns %NULL if the device
876 * is not found or a pointer to the device. The device has not had
877 * its reference counter increased so the caller must be careful
878 * about locking. The caller must hold RCU lock.
879 */
880
881struct net_device *dev_get_by_napi_id(unsigned int napi_id)
882{
883 struct napi_struct *napi;
884
885 WARN_ON_ONCE(!rcu_read_lock_held());
886
887 if (napi_id < MIN_NAPI_ID)
888 return NULL;
889
890 napi = napi_by_id(napi_id);
891
892 return napi ? napi->dev : NULL;
893}
894EXPORT_SYMBOL(dev_get_by_napi_id);
895
896/**
Nicolas Schichan5dbe7c12013-06-26 17:23:42 +0200897 * netdev_get_name - get a netdevice name, knowing its ifindex.
898 * @net: network namespace
899 * @name: a pointer to the buffer where the name will be stored.
900 * @ifindex: the ifindex of the interface to get the name from.
901 *
902 * The use of raw_seqcount_begin() and cond_resched() before
903 * retrying is required as we want to give the writers a chance
904 * to complete when CONFIG_PREEMPT is not set.
905 */
906int netdev_get_name(struct net *net, char *name, int ifindex)
907{
908 struct net_device *dev;
909 unsigned int seq;
910
911retry:
912 seq = raw_seqcount_begin(&devnet_rename_seq);
913 rcu_read_lock();
914 dev = dev_get_by_index_rcu(net, ifindex);
915 if (!dev) {
916 rcu_read_unlock();
917 return -ENODEV;
918 }
919
920 strcpy(name, dev->name);
921 rcu_read_unlock();
922 if (read_seqcount_retry(&devnet_rename_seq, seq)) {
923 cond_resched();
924 goto retry;
925 }
926
927 return 0;
928}
929
930/**
Eric Dumazet941666c2010-12-05 01:23:53 +0000931 * dev_getbyhwaddr_rcu - find a device by its hardware address
Randy Dunlapc4ea43c2007-10-12 21:17:49 -0700932 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -0700933 * @type: media type of device
934 * @ha: hardware address
935 *
936 * Search for an interface by MAC address. Returns NULL if the device
Eric Dumazetc5066532011-01-24 13:16:16 -0800937 * is not found or a pointer to the device.
938 * The caller must hold RCU or RTNL.
Eric Dumazet941666c2010-12-05 01:23:53 +0000939 * The returned device has not had its ref count increased
Linus Torvalds1da177e2005-04-16 15:20:36 -0700940 * and the caller must therefore be careful about locking
941 *
Linus Torvalds1da177e2005-04-16 15:20:36 -0700942 */
943
Eric Dumazet941666c2010-12-05 01:23:53 +0000944struct net_device *dev_getbyhwaddr_rcu(struct net *net, unsigned short type,
945 const char *ha)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700946{
947 struct net_device *dev;
948
Eric Dumazet941666c2010-12-05 01:23:53 +0000949 for_each_netdev_rcu(net, dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700950 if (dev->type == type &&
951 !memcmp(dev->dev_addr, ha, dev->addr_len))
Pavel Emelianov7562f872007-05-03 15:13:45 -0700952 return dev;
953
954 return NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700955}
Eric Dumazet941666c2010-12-05 01:23:53 +0000956EXPORT_SYMBOL(dev_getbyhwaddr_rcu);
Jochen Friedrichcf309e32005-09-22 04:44:55 -0300957
Eric W. Biederman881d9662007-09-17 11:56:21 -0700958struct net_device *__dev_getfirstbyhwtype(struct net *net, unsigned short type)
Patrick McHardy4e9cac22007-05-03 03:28:13 -0700959{
960 struct net_device *dev;
961
962 ASSERT_RTNL();
Eric W. Biederman881d9662007-09-17 11:56:21 -0700963 for_each_netdev(net, dev)
Patrick McHardy4e9cac22007-05-03 03:28:13 -0700964 if (dev->type == type)
Pavel Emelianov7562f872007-05-03 15:13:45 -0700965 return dev;
966
967 return NULL;
Patrick McHardy4e9cac22007-05-03 03:28:13 -0700968}
Patrick McHardy4e9cac22007-05-03 03:28:13 -0700969EXPORT_SYMBOL(__dev_getfirstbyhwtype);
970
Eric W. Biederman881d9662007-09-17 11:56:21 -0700971struct net_device *dev_getfirstbyhwtype(struct net *net, unsigned short type)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700972{
Eric Dumazet99fe3c32010-03-18 11:27:25 +0000973 struct net_device *dev, *ret = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700974
Eric Dumazet99fe3c32010-03-18 11:27:25 +0000975 rcu_read_lock();
976 for_each_netdev_rcu(net, dev)
977 if (dev->type == type) {
978 dev_hold(dev);
979 ret = dev;
980 break;
981 }
982 rcu_read_unlock();
983 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700984}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700985EXPORT_SYMBOL(dev_getfirstbyhwtype);
986
987/**
WANG Cong6c555492014-09-11 15:35:09 -0700988 * __dev_get_by_flags - find any device with given flags
Randy Dunlapc4ea43c2007-10-12 21:17:49 -0700989 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -0700990 * @if_flags: IFF_* values
991 * @mask: bitmask of bits in if_flags to check
992 *
993 * Search for any interface with the given flags. Returns NULL if a device
Eric Dumazetbb69ae02010-06-07 11:42:13 +0000994 * is not found or a pointer to the device. Must be called inside
WANG Cong6c555492014-09-11 15:35:09 -0700995 * rtnl_lock(), and result refcount is unchanged.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700996 */
997
WANG Cong6c555492014-09-11 15:35:09 -0700998struct net_device *__dev_get_by_flags(struct net *net, unsigned short if_flags,
999 unsigned short mask)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001000{
Pavel Emelianov7562f872007-05-03 15:13:45 -07001001 struct net_device *dev, *ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001002
WANG Cong6c555492014-09-11 15:35:09 -07001003 ASSERT_RTNL();
1004
Pavel Emelianov7562f872007-05-03 15:13:45 -07001005 ret = NULL;
WANG Cong6c555492014-09-11 15:35:09 -07001006 for_each_netdev(net, dev) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001007 if (((dev->flags ^ if_flags) & mask) == 0) {
Pavel Emelianov7562f872007-05-03 15:13:45 -07001008 ret = dev;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001009 break;
1010 }
1011 }
Pavel Emelianov7562f872007-05-03 15:13:45 -07001012 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001013}
WANG Cong6c555492014-09-11 15:35:09 -07001014EXPORT_SYMBOL(__dev_get_by_flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001015
1016/**
1017 * dev_valid_name - check if name is okay for network device
1018 * @name: name string
1019 *
1020 * Network device names need to be valid file names to
David S. Millerc7fa9d12006-08-15 16:34:13 -07001021 * to allow sysfs to work. We also disallow any kind of
1022 * whitespace.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001023 */
David S. Miller95f050b2012-03-06 16:12:15 -05001024bool dev_valid_name(const char *name)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001025{
David S. Millerc7fa9d12006-08-15 16:34:13 -07001026 if (*name == '\0')
David S. Miller95f050b2012-03-06 16:12:15 -05001027 return false;
Stephen Hemmingerb6fe17d2006-08-29 17:06:13 -07001028 if (strlen(name) >= IFNAMSIZ)
David S. Miller95f050b2012-03-06 16:12:15 -05001029 return false;
David S. Millerc7fa9d12006-08-15 16:34:13 -07001030 if (!strcmp(name, ".") || !strcmp(name, ".."))
David S. Miller95f050b2012-03-06 16:12:15 -05001031 return false;
David S. Millerc7fa9d12006-08-15 16:34:13 -07001032
1033 while (*name) {
Matthew Thodea4176a92015-02-17 18:31:57 -06001034 if (*name == '/' || *name == ':' || isspace(*name))
David S. Miller95f050b2012-03-06 16:12:15 -05001035 return false;
David S. Millerc7fa9d12006-08-15 16:34:13 -07001036 name++;
1037 }
David S. Miller95f050b2012-03-06 16:12:15 -05001038 return true;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001039}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001040EXPORT_SYMBOL(dev_valid_name);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001041
1042/**
Eric W. Biedermanb267b172007-09-12 13:48:45 +02001043 * __dev_alloc_name - allocate a name for a device
1044 * @net: network namespace to allocate the device name in
Linus Torvalds1da177e2005-04-16 15:20:36 -07001045 * @name: name format string
Eric W. Biedermanb267b172007-09-12 13:48:45 +02001046 * @buf: scratch buffer and result name string
Linus Torvalds1da177e2005-04-16 15:20:36 -07001047 *
1048 * Passed a format string - eg "lt%d" it will try and find a suitable
Stephen Hemminger3041a062006-05-26 13:25:24 -07001049 * id. It scans list of devices to build up a free map, then chooses
1050 * the first empty slot. The caller must hold the dev_base or rtnl lock
1051 * while allocating the name and adding the device in order to avoid
1052 * duplicates.
1053 * Limited to bits_per_byte * page size devices (ie 32K on most platforms).
1054 * Returns the number of the unit assigned or a negative errno code.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001055 */
1056
Eric W. Biedermanb267b172007-09-12 13:48:45 +02001057static int __dev_alloc_name(struct net *net, const char *name, char *buf)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001058{
1059 int i = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001060 const char *p;
1061 const int max_netdevices = 8*PAGE_SIZE;
Stephen Hemmingercfcabdc2007-10-09 01:59:42 -07001062 unsigned long *inuse;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001063 struct net_device *d;
1064
1065 p = strnchr(name, IFNAMSIZ-1, '%');
1066 if (p) {
1067 /*
1068 * Verify the string as this thing may have come from
1069 * the user. There must be either one "%d" and no other "%"
1070 * characters.
1071 */
1072 if (p[1] != 'd' || strchr(p + 2, '%'))
1073 return -EINVAL;
1074
1075 /* Use one page as a bit array of possible slots */
Stephen Hemmingercfcabdc2007-10-09 01:59:42 -07001076 inuse = (unsigned long *) get_zeroed_page(GFP_ATOMIC);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001077 if (!inuse)
1078 return -ENOMEM;
1079
Eric W. Biederman881d9662007-09-17 11:56:21 -07001080 for_each_netdev(net, d) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001081 if (!sscanf(d->name, name, &i))
1082 continue;
1083 if (i < 0 || i >= max_netdevices)
1084 continue;
1085
1086 /* avoid cases where sscanf is not exact inverse of printf */
Eric W. Biedermanb267b172007-09-12 13:48:45 +02001087 snprintf(buf, IFNAMSIZ, name, i);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001088 if (!strncmp(buf, d->name, IFNAMSIZ))
1089 set_bit(i, inuse);
1090 }
1091
1092 i = find_first_zero_bit(inuse, max_netdevices);
1093 free_page((unsigned long) inuse);
1094 }
1095
Octavian Purdilad9031022009-11-18 02:36:59 +00001096 if (buf != name)
1097 snprintf(buf, IFNAMSIZ, name, i);
Eric W. Biedermanb267b172007-09-12 13:48:45 +02001098 if (!__dev_get_by_name(net, buf))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001099 return i;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001100
1101 /* It is possible to run out of possible slots
1102 * when the name is long and there isn't enough space left
1103 * for the digits, or if all bits are used.
1104 */
1105 return -ENFILE;
1106}
1107
Eric W. Biedermanb267b172007-09-12 13:48:45 +02001108/**
1109 * dev_alloc_name - allocate a name for a device
1110 * @dev: device
1111 * @name: name format string
1112 *
1113 * Passed a format string - eg "lt%d" it will try and find a suitable
1114 * id. It scans list of devices to build up a free map, then chooses
1115 * the first empty slot. The caller must hold the dev_base or rtnl lock
1116 * while allocating the name and adding the device in order to avoid
1117 * duplicates.
1118 * Limited to bits_per_byte * page size devices (ie 32K on most platforms).
1119 * Returns the number of the unit assigned or a negative errno code.
1120 */
1121
1122int dev_alloc_name(struct net_device *dev, const char *name)
1123{
1124 char buf[IFNAMSIZ];
1125 struct net *net;
1126 int ret;
1127
YOSHIFUJI Hideakic346dca2008-03-25 21:47:49 +09001128 BUG_ON(!dev_net(dev));
1129 net = dev_net(dev);
Eric W. Biedermanb267b172007-09-12 13:48:45 +02001130 ret = __dev_alloc_name(net, name, buf);
1131 if (ret >= 0)
1132 strlcpy(dev->name, buf, IFNAMSIZ);
1133 return ret;
1134}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001135EXPORT_SYMBOL(dev_alloc_name);
Eric W. Biedermanb267b172007-09-12 13:48:45 +02001136
Gao feng828de4f2012-09-13 20:58:27 +00001137static int dev_alloc_name_ns(struct net *net,
1138 struct net_device *dev,
1139 const char *name)
Octavian Purdilad9031022009-11-18 02:36:59 +00001140{
Gao feng828de4f2012-09-13 20:58:27 +00001141 char buf[IFNAMSIZ];
1142 int ret;
Daniel Lezcano8ce6cebc2010-05-19 10:12:19 +00001143
Gao feng828de4f2012-09-13 20:58:27 +00001144 ret = __dev_alloc_name(net, name, buf);
1145 if (ret >= 0)
1146 strlcpy(dev->name, buf, IFNAMSIZ);
1147 return ret;
1148}
1149
1150static int dev_get_valid_name(struct net *net,
1151 struct net_device *dev,
1152 const char *name)
1153{
1154 BUG_ON(!net);
Daniel Lezcano8ce6cebc2010-05-19 10:12:19 +00001155
Octavian Purdilad9031022009-11-18 02:36:59 +00001156 if (!dev_valid_name(name))
1157 return -EINVAL;
1158
Jiri Pirko1c5cae82011-04-30 01:21:32 +00001159 if (strchr(name, '%'))
Gao feng828de4f2012-09-13 20:58:27 +00001160 return dev_alloc_name_ns(net, dev, name);
Octavian Purdilad9031022009-11-18 02:36:59 +00001161 else if (__dev_get_by_name(net, name))
1162 return -EEXIST;
Daniel Lezcano8ce6cebc2010-05-19 10:12:19 +00001163 else if (dev->name != name)
1164 strlcpy(dev->name, name, IFNAMSIZ);
Octavian Purdilad9031022009-11-18 02:36:59 +00001165
1166 return 0;
1167}
Linus Torvalds1da177e2005-04-16 15:20:36 -07001168
1169/**
1170 * dev_change_name - change name of a device
1171 * @dev: device
1172 * @newname: name (or format string) must be at least IFNAMSIZ
1173 *
1174 * Change name of a device, can pass format strings "eth%d".
1175 * for wildcarding.
1176 */
Stephen Hemmingercf04a4c72008-09-30 02:22:14 -07001177int dev_change_name(struct net_device *dev, const char *newname)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001178{
Tom Gundersen238fa362014-07-14 16:37:23 +02001179 unsigned char old_assign_type;
Herbert Xufcc5a032007-07-30 17:03:38 -07001180 char oldname[IFNAMSIZ];
Linus Torvalds1da177e2005-04-16 15:20:36 -07001181 int err = 0;
Herbert Xufcc5a032007-07-30 17:03:38 -07001182 int ret;
Eric W. Biederman881d9662007-09-17 11:56:21 -07001183 struct net *net;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001184
1185 ASSERT_RTNL();
YOSHIFUJI Hideakic346dca2008-03-25 21:47:49 +09001186 BUG_ON(!dev_net(dev));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001187
YOSHIFUJI Hideakic346dca2008-03-25 21:47:49 +09001188 net = dev_net(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001189 if (dev->flags & IFF_UP)
1190 return -EBUSY;
1191
Eric Dumazet30e6c9fa2012-12-20 17:25:08 +00001192 write_seqcount_begin(&devnet_rename_seq);
Brian Haleyc91f6df2012-11-26 05:21:08 +00001193
1194 if (strncmp(newname, dev->name, IFNAMSIZ) == 0) {
Eric Dumazet30e6c9fa2012-12-20 17:25:08 +00001195 write_seqcount_end(&devnet_rename_seq);
Stephen Hemmingerc8d90dc2007-10-26 03:53:42 -07001196 return 0;
Brian Haleyc91f6df2012-11-26 05:21:08 +00001197 }
Stephen Hemmingerc8d90dc2007-10-26 03:53:42 -07001198
Herbert Xufcc5a032007-07-30 17:03:38 -07001199 memcpy(oldname, dev->name, IFNAMSIZ);
1200
Gao feng828de4f2012-09-13 20:58:27 +00001201 err = dev_get_valid_name(net, dev, newname);
Brian Haleyc91f6df2012-11-26 05:21:08 +00001202 if (err < 0) {
Eric Dumazet30e6c9fa2012-12-20 17:25:08 +00001203 write_seqcount_end(&devnet_rename_seq);
Octavian Purdilad9031022009-11-18 02:36:59 +00001204 return err;
Brian Haleyc91f6df2012-11-26 05:21:08 +00001205 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001206
Veaceslav Falico6fe82a32014-07-17 20:33:32 +02001207 if (oldname[0] && !strchr(oldname, '%'))
1208 netdev_info(dev, "renamed from %s\n", oldname);
1209
Tom Gundersen238fa362014-07-14 16:37:23 +02001210 old_assign_type = dev->name_assign_type;
1211 dev->name_assign_type = NET_NAME_RENAMED;
1212
Herbert Xufcc5a032007-07-30 17:03:38 -07001213rollback:
Eric W. Biedermana1b3f592010-05-04 17:36:49 -07001214 ret = device_rename(&dev->dev, dev->name);
1215 if (ret) {
1216 memcpy(dev->name, oldname, IFNAMSIZ);
Tom Gundersen238fa362014-07-14 16:37:23 +02001217 dev->name_assign_type = old_assign_type;
Eric Dumazet30e6c9fa2012-12-20 17:25:08 +00001218 write_seqcount_end(&devnet_rename_seq);
Eric W. Biedermana1b3f592010-05-04 17:36:49 -07001219 return ret;
Stephen Hemmingerdcc99772008-05-14 22:33:38 -07001220 }
Herbert Xu7f988ea2007-07-30 16:35:46 -07001221
Eric Dumazet30e6c9fa2012-12-20 17:25:08 +00001222 write_seqcount_end(&devnet_rename_seq);
Brian Haleyc91f6df2012-11-26 05:21:08 +00001223
Veaceslav Falico5bb025f2014-01-14 21:58:51 +01001224 netdev_adjacent_rename_links(dev, oldname);
1225
Herbert Xu7f988ea2007-07-30 16:35:46 -07001226 write_lock_bh(&dev_base_lock);
Eric Dumazet372b2312011-05-17 13:56:59 -04001227 hlist_del_rcu(&dev->name_hlist);
Eric Dumazet72c95282009-10-30 07:11:27 +00001228 write_unlock_bh(&dev_base_lock);
1229
1230 synchronize_rcu();
1231
1232 write_lock_bh(&dev_base_lock);
1233 hlist_add_head_rcu(&dev->name_hlist, dev_name_hash(net, dev->name));
Herbert Xu7f988ea2007-07-30 16:35:46 -07001234 write_unlock_bh(&dev_base_lock);
1235
Pavel Emelyanov056925a2007-09-16 15:42:43 -07001236 ret = call_netdevice_notifiers(NETDEV_CHANGENAME, dev);
Herbert Xufcc5a032007-07-30 17:03:38 -07001237 ret = notifier_to_errno(ret);
1238
1239 if (ret) {
Eric Dumazet91e9c07b2009-11-15 23:30:24 +00001240 /* err >= 0 after dev_alloc_name() or stores the first errno */
1241 if (err >= 0) {
Herbert Xufcc5a032007-07-30 17:03:38 -07001242 err = ret;
Eric Dumazet30e6c9fa2012-12-20 17:25:08 +00001243 write_seqcount_begin(&devnet_rename_seq);
Herbert Xufcc5a032007-07-30 17:03:38 -07001244 memcpy(dev->name, oldname, IFNAMSIZ);
Veaceslav Falico5bb025f2014-01-14 21:58:51 +01001245 memcpy(oldname, newname, IFNAMSIZ);
Tom Gundersen238fa362014-07-14 16:37:23 +02001246 dev->name_assign_type = old_assign_type;
1247 old_assign_type = NET_NAME_RENAMED;
Herbert Xufcc5a032007-07-30 17:03:38 -07001248 goto rollback;
Eric Dumazet91e9c07b2009-11-15 23:30:24 +00001249 } else {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00001250 pr_err("%s: name change rollback failed: %d\n",
Eric Dumazet91e9c07b2009-11-15 23:30:24 +00001251 dev->name, ret);
Herbert Xufcc5a032007-07-30 17:03:38 -07001252 }
1253 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001254
1255 return err;
1256}
1257
1258/**
Stephen Hemminger0b815a12008-09-22 21:28:11 -07001259 * dev_set_alias - change ifalias of a device
1260 * @dev: device
1261 * @alias: name up to IFALIASZ
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07001262 * @len: limit of bytes to copy from info
Stephen Hemminger0b815a12008-09-22 21:28:11 -07001263 *
1264 * Set ifalias for a device,
1265 */
1266int dev_set_alias(struct net_device *dev, const char *alias, size_t len)
1267{
Alexey Khoroshilov7364e442012-08-08 00:33:25 +00001268 char *new_ifalias;
1269
Stephen Hemminger0b815a12008-09-22 21:28:11 -07001270 ASSERT_RTNL();
1271
1272 if (len >= IFALIASZ)
1273 return -EINVAL;
1274
Oliver Hartkopp96ca4a22008-09-23 21:23:19 -07001275 if (!len) {
Sachin Kamat388dfc22012-11-20 00:57:04 +00001276 kfree(dev->ifalias);
1277 dev->ifalias = NULL;
Oliver Hartkopp96ca4a22008-09-23 21:23:19 -07001278 return 0;
1279 }
1280
Alexey Khoroshilov7364e442012-08-08 00:33:25 +00001281 new_ifalias = krealloc(dev->ifalias, len + 1, GFP_KERNEL);
1282 if (!new_ifalias)
Stephen Hemminger0b815a12008-09-22 21:28:11 -07001283 return -ENOMEM;
Alexey Khoroshilov7364e442012-08-08 00:33:25 +00001284 dev->ifalias = new_ifalias;
Alexander Potapenkoc28294b2017-06-06 15:56:54 +02001285 memcpy(dev->ifalias, alias, len);
1286 dev->ifalias[len] = 0;
Stephen Hemminger0b815a12008-09-22 21:28:11 -07001287
Stephen Hemminger0b815a12008-09-22 21:28:11 -07001288 return len;
1289}
1290
1291
1292/**
Stephen Hemminger3041a062006-05-26 13:25:24 -07001293 * netdev_features_change - device changes features
Stephen Hemmingerd8a33ac2005-05-29 14:13:47 -07001294 * @dev: device to cause notification
1295 *
1296 * Called to indicate a device has changed features.
1297 */
1298void netdev_features_change(struct net_device *dev)
1299{
Pavel Emelyanov056925a2007-09-16 15:42:43 -07001300 call_netdevice_notifiers(NETDEV_FEAT_CHANGE, dev);
Stephen Hemmingerd8a33ac2005-05-29 14:13:47 -07001301}
1302EXPORT_SYMBOL(netdev_features_change);
1303
1304/**
Linus Torvalds1da177e2005-04-16 15:20:36 -07001305 * netdev_state_change - device changes state
1306 * @dev: device to cause notification
1307 *
1308 * Called to indicate a device has changed state. This function calls
1309 * the notifier chains for netdev_chain and sends a NEWLINK message
1310 * to the routing socket.
1311 */
1312void netdev_state_change(struct net_device *dev)
1313{
1314 if (dev->flags & IFF_UP) {
Loic Prylli54951192014-07-01 21:39:43 -07001315 struct netdev_notifier_change_info change_info;
1316
1317 change_info.flags_changed = 0;
1318 call_netdevice_notifiers_info(NETDEV_CHANGE, dev,
1319 &change_info.info);
Alexei Starovoitov7f294052013-10-23 16:02:42 -07001320 rtmsg_ifinfo(RTM_NEWLINK, dev, 0, GFP_KERNEL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001321 }
1322}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001323EXPORT_SYMBOL(netdev_state_change);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001324
Amerigo Wangee89bab2012-08-09 22:14:56 +00001325/**
tcharding722c9a02017-02-09 17:56:04 +11001326 * netdev_notify_peers - notify network peers about existence of @dev
1327 * @dev: network device
Amerigo Wangee89bab2012-08-09 22:14:56 +00001328 *
1329 * Generate traffic such that interested network peers are aware of
1330 * @dev, such as by generating a gratuitous ARP. This may be used when
1331 * a device wants to inform the rest of the network about some sort of
1332 * reconfiguration such as a failover event or virtual machine
1333 * migration.
1334 */
1335void netdev_notify_peers(struct net_device *dev)
Or Gerlitzc1da4ac2008-06-13 18:12:00 -07001336{
Amerigo Wangee89bab2012-08-09 22:14:56 +00001337 rtnl_lock();
1338 call_netdevice_notifiers(NETDEV_NOTIFY_PEERS, dev);
Vlad Yasevich37c343b2017-03-14 08:58:08 -04001339 call_netdevice_notifiers(NETDEV_RESEND_IGMP, dev);
Amerigo Wangee89bab2012-08-09 22:14:56 +00001340 rtnl_unlock();
Or Gerlitzc1da4ac2008-06-13 18:12:00 -07001341}
Amerigo Wangee89bab2012-08-09 22:14:56 +00001342EXPORT_SYMBOL(netdev_notify_peers);
Or Gerlitzc1da4ac2008-06-13 18:12:00 -07001343
Patrick McHardybd380812010-02-26 06:34:53 +00001344static int __dev_open(struct net_device *dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001345{
Stephen Hemmingerd3147742008-11-19 21:32:24 -08001346 const struct net_device_ops *ops = dev->netdev_ops;
Johannes Berg3b8bcfd2009-05-30 01:39:53 +02001347 int ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001348
Ben Hutchingse46b66b2008-05-08 02:53:17 -07001349 ASSERT_RTNL();
1350
Linus Torvalds1da177e2005-04-16 15:20:36 -07001351 if (!netif_device_present(dev))
1352 return -ENODEV;
1353
Neil Hormanca99ca12013-02-05 08:05:43 +00001354 /* Block netpoll from trying to do any rx path servicing.
1355 * If we don't do this there is a chance ndo_poll_controller
1356 * or ndo_poll may be running while we open the device
1357 */
Eric W. Biederman66b55522014-03-27 15:39:03 -07001358 netpoll_poll_disable(dev);
Neil Hormanca99ca12013-02-05 08:05:43 +00001359
Johannes Berg3b8bcfd2009-05-30 01:39:53 +02001360 ret = call_netdevice_notifiers(NETDEV_PRE_UP, dev);
1361 ret = notifier_to_errno(ret);
1362 if (ret)
1363 return ret;
1364
Linus Torvalds1da177e2005-04-16 15:20:36 -07001365 set_bit(__LINK_STATE_START, &dev->state);
Jeff Garzikbada3392007-10-23 20:19:37 -07001366
Stephen Hemmingerd3147742008-11-19 21:32:24 -08001367 if (ops->ndo_validate_addr)
1368 ret = ops->ndo_validate_addr(dev);
Jeff Garzikbada3392007-10-23 20:19:37 -07001369
Stephen Hemmingerd3147742008-11-19 21:32:24 -08001370 if (!ret && ops->ndo_open)
1371 ret = ops->ndo_open(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001372
Eric W. Biederman66b55522014-03-27 15:39:03 -07001373 netpoll_poll_enable(dev);
Neil Hormanca99ca12013-02-05 08:05:43 +00001374
Jeff Garzikbada3392007-10-23 20:19:37 -07001375 if (ret)
1376 clear_bit(__LINK_STATE_START, &dev->state);
1377 else {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001378 dev->flags |= IFF_UP;
Patrick McHardy4417da62007-06-27 01:28:10 -07001379 dev_set_rx_mode(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001380 dev_activate(dev);
Theodore Ts'o7bf23572012-07-04 21:23:25 -04001381 add_device_randomness(dev->dev_addr, dev->addr_len);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001382 }
Jeff Garzikbada3392007-10-23 20:19:37 -07001383
Linus Torvalds1da177e2005-04-16 15:20:36 -07001384 return ret;
1385}
Patrick McHardybd380812010-02-26 06:34:53 +00001386
1387/**
1388 * dev_open - prepare an interface for use.
1389 * @dev: device to open
1390 *
1391 * Takes a device from down to up state. The device's private open
1392 * function is invoked and then the multicast lists are loaded. Finally
1393 * the device is moved into the up state and a %NETDEV_UP message is
1394 * sent to the netdev notifier chain.
1395 *
1396 * Calling this function on an active interface is a nop. On a failure
1397 * a negative errno code is returned.
1398 */
1399int dev_open(struct net_device *dev)
1400{
1401 int ret;
1402
Patrick McHardybd380812010-02-26 06:34:53 +00001403 if (dev->flags & IFF_UP)
1404 return 0;
1405
Patrick McHardybd380812010-02-26 06:34:53 +00001406 ret = __dev_open(dev);
1407 if (ret < 0)
1408 return ret;
1409
Alexei Starovoitov7f294052013-10-23 16:02:42 -07001410 rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP|IFF_RUNNING, GFP_KERNEL);
Patrick McHardybd380812010-02-26 06:34:53 +00001411 call_netdevice_notifiers(NETDEV_UP, dev);
1412
1413 return ret;
1414}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001415EXPORT_SYMBOL(dev_open);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001416
stephen hemminger7051b882017-07-18 15:59:27 -07001417static void __dev_close_many(struct list_head *head)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001418{
Octavian Purdila44345722010-12-13 12:44:07 +00001419 struct net_device *dev;
Patrick McHardybd380812010-02-26 06:34:53 +00001420
Ben Hutchingse46b66b2008-05-08 02:53:17 -07001421 ASSERT_RTNL();
David S. Miller9d5010d2007-09-12 14:33:25 +02001422 might_sleep();
1423
Eric W. Biederman5cde2822013-10-05 19:26:05 -07001424 list_for_each_entry(dev, head, close_list) {
Eric W. Biederman3f4df202014-03-27 15:38:17 -07001425 /* Temporarily disable netpoll until the interface is down */
Eric W. Biederman66b55522014-03-27 15:39:03 -07001426 netpoll_poll_disable(dev);
Eric W. Biederman3f4df202014-03-27 15:38:17 -07001427
Octavian Purdila44345722010-12-13 12:44:07 +00001428 call_netdevice_notifiers(NETDEV_GOING_DOWN, dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001429
Octavian Purdila44345722010-12-13 12:44:07 +00001430 clear_bit(__LINK_STATE_START, &dev->state);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001431
Octavian Purdila44345722010-12-13 12:44:07 +00001432 /* Synchronize to scheduled poll. We cannot touch poll list, it
1433 * can be even on different cpu. So just clear netif_running().
1434 *
1435 * dev->stop() will invoke napi_disable() on all of it's
1436 * napi_struct instances on this device.
1437 */
Peter Zijlstra4e857c52014-03-17 18:06:10 +01001438 smp_mb__after_atomic(); /* Commit netif_running(). */
Octavian Purdila44345722010-12-13 12:44:07 +00001439 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001440
Octavian Purdila44345722010-12-13 12:44:07 +00001441 dev_deactivate_many(head);
1442
Eric W. Biederman5cde2822013-10-05 19:26:05 -07001443 list_for_each_entry(dev, head, close_list) {
Octavian Purdila44345722010-12-13 12:44:07 +00001444 const struct net_device_ops *ops = dev->netdev_ops;
1445
1446 /*
1447 * Call the device specific close. This cannot fail.
1448 * Only if device is UP
1449 *
1450 * We allow it to be called even after a DETACH hot-plug
1451 * event.
1452 */
1453 if (ops->ndo_stop)
1454 ops->ndo_stop(dev);
1455
Octavian Purdila44345722010-12-13 12:44:07 +00001456 dev->flags &= ~IFF_UP;
Eric W. Biederman66b55522014-03-27 15:39:03 -07001457 netpoll_poll_enable(dev);
Octavian Purdila44345722010-12-13 12:44:07 +00001458 }
Octavian Purdila44345722010-12-13 12:44:07 +00001459}
1460
stephen hemminger7051b882017-07-18 15:59:27 -07001461static void __dev_close(struct net_device *dev)
Octavian Purdila44345722010-12-13 12:44:07 +00001462{
1463 LIST_HEAD(single);
1464
Eric W. Biederman5cde2822013-10-05 19:26:05 -07001465 list_add(&dev->close_list, &single);
stephen hemminger7051b882017-07-18 15:59:27 -07001466 __dev_close_many(&single);
Linus Torvaldsf87e6f42011-02-17 22:54:38 +00001467 list_del(&single);
Octavian Purdila44345722010-12-13 12:44:07 +00001468}
1469
stephen hemminger7051b882017-07-18 15:59:27 -07001470void dev_close_many(struct list_head *head, bool unlink)
Octavian Purdila44345722010-12-13 12:44:07 +00001471{
1472 struct net_device *dev, *tmp;
Octavian Purdila44345722010-12-13 12:44:07 +00001473
Eric W. Biederman5cde2822013-10-05 19:26:05 -07001474 /* Remove the devices that don't need to be closed */
1475 list_for_each_entry_safe(dev, tmp, head, close_list)
Octavian Purdila44345722010-12-13 12:44:07 +00001476 if (!(dev->flags & IFF_UP))
Eric W. Biederman5cde2822013-10-05 19:26:05 -07001477 list_del_init(&dev->close_list);
Octavian Purdila44345722010-12-13 12:44:07 +00001478
1479 __dev_close_many(head);
Matti Linnanvuorid8b2a4d2008-02-12 23:10:11 -08001480
Eric W. Biederman5cde2822013-10-05 19:26:05 -07001481 list_for_each_entry_safe(dev, tmp, head, close_list) {
Alexei Starovoitov7f294052013-10-23 16:02:42 -07001482 rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP|IFF_RUNNING, GFP_KERNEL);
Octavian Purdila44345722010-12-13 12:44:07 +00001483 call_netdevice_notifiers(NETDEV_DOWN, dev);
David S. Miller99c4a262015-03-18 22:52:33 -04001484 if (unlink)
1485 list_del_init(&dev->close_list);
Octavian Purdila44345722010-12-13 12:44:07 +00001486 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001487}
David S. Miller99c4a262015-03-18 22:52:33 -04001488EXPORT_SYMBOL(dev_close_many);
Patrick McHardybd380812010-02-26 06:34:53 +00001489
1490/**
1491 * dev_close - shutdown an interface.
1492 * @dev: device to shutdown
1493 *
1494 * This function moves an active device into down state. A
1495 * %NETDEV_GOING_DOWN is sent to the netdev notifier chain. The device
1496 * is then deactivated and finally a %NETDEV_DOWN is sent to the notifier
1497 * chain.
1498 */
stephen hemminger7051b882017-07-18 15:59:27 -07001499void dev_close(struct net_device *dev)
Patrick McHardybd380812010-02-26 06:34:53 +00001500{
Eric Dumazete14a5992011-05-10 12:26:06 -07001501 if (dev->flags & IFF_UP) {
1502 LIST_HEAD(single);
Patrick McHardybd380812010-02-26 06:34:53 +00001503
Eric W. Biederman5cde2822013-10-05 19:26:05 -07001504 list_add(&dev->close_list, &single);
David S. Miller99c4a262015-03-18 22:52:33 -04001505 dev_close_many(&single, true);
Eric Dumazete14a5992011-05-10 12:26:06 -07001506 list_del(&single);
1507 }
Patrick McHardybd380812010-02-26 06:34:53 +00001508}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001509EXPORT_SYMBOL(dev_close);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001510
1511
Ben Hutchings0187bdf2008-06-19 16:15:47 -07001512/**
1513 * dev_disable_lro - disable Large Receive Offload on a device
1514 * @dev: device
1515 *
1516 * Disable Large Receive Offload (LRO) on a net device. Must be
1517 * called under RTNL. This is needed if received packets may be
1518 * forwarded to another interface.
1519 */
1520void dev_disable_lro(struct net_device *dev)
1521{
Michal Kubečekfbe168b2014-11-13 07:54:50 +01001522 struct net_device *lower_dev;
1523 struct list_head *iter;
Michal Kubeček529d0482013-11-15 06:18:50 +01001524
Michał Mirosławbc5787c62011-11-15 15:29:55 +00001525 dev->wanted_features &= ~NETIF_F_LRO;
1526 netdev_update_features(dev);
Michał Mirosław27660512011-03-18 16:56:34 +00001527
Michał Mirosław22d59692011-04-21 12:42:15 +00001528 if (unlikely(dev->features & NETIF_F_LRO))
1529 netdev_WARN(dev, "failed to disable LRO!\n");
Michal Kubečekfbe168b2014-11-13 07:54:50 +01001530
1531 netdev_for_each_lower_dev(dev, lower_dev, iter)
1532 dev_disable_lro(lower_dev);
Ben Hutchings0187bdf2008-06-19 16:15:47 -07001533}
1534EXPORT_SYMBOL(dev_disable_lro);
1535
Jiri Pirko351638e2013-05-28 01:30:21 +00001536static int call_netdevice_notifier(struct notifier_block *nb, unsigned long val,
1537 struct net_device *dev)
1538{
1539 struct netdev_notifier_info info;
1540
1541 netdev_notifier_info_init(&info, dev);
1542 return nb->notifier_call(nb, val, &info);
1543}
Ben Hutchings0187bdf2008-06-19 16:15:47 -07001544
Eric W. Biederman881d9662007-09-17 11:56:21 -07001545static int dev_boot_phase = 1;
1546
Linus Torvalds1da177e2005-04-16 15:20:36 -07001547/**
tcharding722c9a02017-02-09 17:56:04 +11001548 * register_netdevice_notifier - register a network notifier block
1549 * @nb: notifier
Linus Torvalds1da177e2005-04-16 15:20:36 -07001550 *
tcharding722c9a02017-02-09 17:56:04 +11001551 * Register a notifier to be called when network device events occur.
1552 * The notifier passed is linked into the kernel structures and must
1553 * not be reused until it has been unregistered. A negative errno code
1554 * is returned on a failure.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001555 *
tcharding722c9a02017-02-09 17:56:04 +11001556 * When registered all registration and up events are replayed
1557 * to the new notifier to allow device to have a race free
1558 * view of the network device list.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001559 */
1560
1561int register_netdevice_notifier(struct notifier_block *nb)
1562{
1563 struct net_device *dev;
Herbert Xufcc5a032007-07-30 17:03:38 -07001564 struct net_device *last;
Eric W. Biederman881d9662007-09-17 11:56:21 -07001565 struct net *net;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001566 int err;
1567
1568 rtnl_lock();
Alan Sternf07d5b92006-05-09 15:23:03 -07001569 err = raw_notifier_chain_register(&netdev_chain, nb);
Herbert Xufcc5a032007-07-30 17:03:38 -07001570 if (err)
1571 goto unlock;
Eric W. Biederman881d9662007-09-17 11:56:21 -07001572 if (dev_boot_phase)
1573 goto unlock;
1574 for_each_net(net) {
1575 for_each_netdev(net, dev) {
Jiri Pirko351638e2013-05-28 01:30:21 +00001576 err = call_netdevice_notifier(nb, NETDEV_REGISTER, dev);
Eric W. Biederman881d9662007-09-17 11:56:21 -07001577 err = notifier_to_errno(err);
1578 if (err)
1579 goto rollback;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001580
Eric W. Biederman881d9662007-09-17 11:56:21 -07001581 if (!(dev->flags & IFF_UP))
1582 continue;
Herbert Xufcc5a032007-07-30 17:03:38 -07001583
Jiri Pirko351638e2013-05-28 01:30:21 +00001584 call_netdevice_notifier(nb, NETDEV_UP, dev);
Eric W. Biederman881d9662007-09-17 11:56:21 -07001585 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001586 }
Herbert Xufcc5a032007-07-30 17:03:38 -07001587
1588unlock:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001589 rtnl_unlock();
1590 return err;
Herbert Xufcc5a032007-07-30 17:03:38 -07001591
1592rollback:
1593 last = dev;
Eric W. Biederman881d9662007-09-17 11:56:21 -07001594 for_each_net(net) {
1595 for_each_netdev(net, dev) {
1596 if (dev == last)
RongQing.Li8f891482011-11-30 23:43:07 -05001597 goto outroll;
Herbert Xufcc5a032007-07-30 17:03:38 -07001598
Eric W. Biederman881d9662007-09-17 11:56:21 -07001599 if (dev->flags & IFF_UP) {
Jiri Pirko351638e2013-05-28 01:30:21 +00001600 call_netdevice_notifier(nb, NETDEV_GOING_DOWN,
1601 dev);
1602 call_netdevice_notifier(nb, NETDEV_DOWN, dev);
Eric W. Biederman881d9662007-09-17 11:56:21 -07001603 }
Jiri Pirko351638e2013-05-28 01:30:21 +00001604 call_netdevice_notifier(nb, NETDEV_UNREGISTER, dev);
Herbert Xufcc5a032007-07-30 17:03:38 -07001605 }
Herbert Xufcc5a032007-07-30 17:03:38 -07001606 }
Pavel Emelyanovc67625a2007-11-14 15:53:16 -08001607
RongQing.Li8f891482011-11-30 23:43:07 -05001608outroll:
Pavel Emelyanovc67625a2007-11-14 15:53:16 -08001609 raw_notifier_chain_unregister(&netdev_chain, nb);
Herbert Xufcc5a032007-07-30 17:03:38 -07001610 goto unlock;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001611}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001612EXPORT_SYMBOL(register_netdevice_notifier);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001613
1614/**
tcharding722c9a02017-02-09 17:56:04 +11001615 * unregister_netdevice_notifier - unregister a network notifier block
1616 * @nb: notifier
Linus Torvalds1da177e2005-04-16 15:20:36 -07001617 *
tcharding722c9a02017-02-09 17:56:04 +11001618 * Unregister a notifier previously registered by
1619 * register_netdevice_notifier(). The notifier is unlinked into the
1620 * kernel structures and may then be reused. A negative errno code
1621 * is returned on a failure.
Eric W. Biederman7d3d43d2012-04-06 15:33:35 +00001622 *
tcharding722c9a02017-02-09 17:56:04 +11001623 * After unregistering unregister and down device events are synthesized
1624 * for all devices on the device list to the removed notifier to remove
1625 * the need for special case cleanup code.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001626 */
1627
1628int unregister_netdevice_notifier(struct notifier_block *nb)
1629{
Eric W. Biederman7d3d43d2012-04-06 15:33:35 +00001630 struct net_device *dev;
1631 struct net *net;
Herbert Xu9f514952006-03-25 01:24:25 -08001632 int err;
1633
1634 rtnl_lock();
Alan Sternf07d5b92006-05-09 15:23:03 -07001635 err = raw_notifier_chain_unregister(&netdev_chain, nb);
Eric W. Biederman7d3d43d2012-04-06 15:33:35 +00001636 if (err)
1637 goto unlock;
1638
1639 for_each_net(net) {
1640 for_each_netdev(net, dev) {
1641 if (dev->flags & IFF_UP) {
Jiri Pirko351638e2013-05-28 01:30:21 +00001642 call_netdevice_notifier(nb, NETDEV_GOING_DOWN,
1643 dev);
1644 call_netdevice_notifier(nb, NETDEV_DOWN, dev);
Eric W. Biederman7d3d43d2012-04-06 15:33:35 +00001645 }
Jiri Pirko351638e2013-05-28 01:30:21 +00001646 call_netdevice_notifier(nb, NETDEV_UNREGISTER, dev);
Eric W. Biederman7d3d43d2012-04-06 15:33:35 +00001647 }
1648 }
1649unlock:
Herbert Xu9f514952006-03-25 01:24:25 -08001650 rtnl_unlock();
1651 return err;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001652}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001653EXPORT_SYMBOL(unregister_netdevice_notifier);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001654
1655/**
Jiri Pirko351638e2013-05-28 01:30:21 +00001656 * call_netdevice_notifiers_info - call all network notifier blocks
1657 * @val: value passed unmodified to notifier function
1658 * @dev: net_device pointer passed unmodified to notifier function
1659 * @info: notifier information data
1660 *
1661 * Call all network notifier blocks. Parameters and return value
1662 * are as for raw_notifier_call_chain().
1663 */
1664
stephen hemminger1d143d92013-12-29 14:01:29 -08001665static int call_netdevice_notifiers_info(unsigned long val,
1666 struct net_device *dev,
1667 struct netdev_notifier_info *info)
Jiri Pirko351638e2013-05-28 01:30:21 +00001668{
1669 ASSERT_RTNL();
1670 netdev_notifier_info_init(info, dev);
1671 return raw_notifier_call_chain(&netdev_chain, val, info);
1672}
Jiri Pirko351638e2013-05-28 01:30:21 +00001673
1674/**
Linus Torvalds1da177e2005-04-16 15:20:36 -07001675 * call_netdevice_notifiers - call all network notifier blocks
1676 * @val: value passed unmodified to notifier function
Randy Dunlapc4ea43c2007-10-12 21:17:49 -07001677 * @dev: net_device pointer passed unmodified to notifier function
Linus Torvalds1da177e2005-04-16 15:20:36 -07001678 *
1679 * Call all network notifier blocks. Parameters and return value
Alan Sternf07d5b92006-05-09 15:23:03 -07001680 * are as for raw_notifier_call_chain().
Linus Torvalds1da177e2005-04-16 15:20:36 -07001681 */
1682
Eric W. Biedermanad7379d2007-09-16 15:33:32 -07001683int call_netdevice_notifiers(unsigned long val, struct net_device *dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001684{
Jiri Pirko351638e2013-05-28 01:30:21 +00001685 struct netdev_notifier_info info;
1686
1687 return call_netdevice_notifiers_info(val, dev, &info);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001688}
stephen hemmingeredf947f2011-03-24 13:24:01 +00001689EXPORT_SYMBOL(call_netdevice_notifiers);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001690
Pablo Neira1cf519002015-05-13 18:19:37 +02001691#ifdef CONFIG_NET_INGRESS
Daniel Borkmann45771392015-04-10 23:07:54 +02001692static struct static_key ingress_needed __read_mostly;
1693
1694void net_inc_ingress_queue(void)
1695{
1696 static_key_slow_inc(&ingress_needed);
1697}
1698EXPORT_SYMBOL_GPL(net_inc_ingress_queue);
1699
1700void net_dec_ingress_queue(void)
1701{
1702 static_key_slow_dec(&ingress_needed);
1703}
1704EXPORT_SYMBOL_GPL(net_dec_ingress_queue);
1705#endif
1706
Daniel Borkmann1f211a12016-01-07 22:29:47 +01001707#ifdef CONFIG_NET_EGRESS
1708static struct static_key egress_needed __read_mostly;
1709
1710void net_inc_egress_queue(void)
1711{
1712 static_key_slow_inc(&egress_needed);
1713}
1714EXPORT_SYMBOL_GPL(net_inc_egress_queue);
1715
1716void net_dec_egress_queue(void)
1717{
1718 static_key_slow_dec(&egress_needed);
1719}
1720EXPORT_SYMBOL_GPL(net_dec_egress_queue);
1721#endif
1722
Ingo Molnarc5905af2012-02-24 08:31:31 +01001723static struct static_key netstamp_needed __read_mostly;
Eric Dumazetb90e5792011-11-28 11:16:50 +00001724#ifdef HAVE_JUMP_LABEL
Eric Dumazetb90e5792011-11-28 11:16:50 +00001725static atomic_t netstamp_needed_deferred;
Eric Dumazet13baa002017-03-01 14:28:39 -08001726static atomic_t netstamp_wanted;
Eric Dumazet5fa8bbd2017-02-02 10:31:35 -08001727static void netstamp_clear(struct work_struct *work)
1728{
1729 int deferred = atomic_xchg(&netstamp_needed_deferred, 0);
Eric Dumazet13baa002017-03-01 14:28:39 -08001730 int wanted;
Eric Dumazet5fa8bbd2017-02-02 10:31:35 -08001731
Eric Dumazet13baa002017-03-01 14:28:39 -08001732 wanted = atomic_add_return(deferred, &netstamp_wanted);
1733 if (wanted > 0)
1734 static_key_enable(&netstamp_needed);
1735 else
1736 static_key_disable(&netstamp_needed);
Eric Dumazet5fa8bbd2017-02-02 10:31:35 -08001737}
1738static DECLARE_WORK(netstamp_work, netstamp_clear);
Eric Dumazetb90e5792011-11-28 11:16:50 +00001739#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07001740
1741void net_enable_timestamp(void)
1742{
Eric Dumazet13baa002017-03-01 14:28:39 -08001743#ifdef HAVE_JUMP_LABEL
1744 int wanted;
1745
1746 while (1) {
1747 wanted = atomic_read(&netstamp_wanted);
1748 if (wanted <= 0)
1749 break;
1750 if (atomic_cmpxchg(&netstamp_wanted, wanted, wanted + 1) == wanted)
1751 return;
1752 }
1753 atomic_inc(&netstamp_needed_deferred);
1754 schedule_work(&netstamp_work);
1755#else
Ingo Molnarc5905af2012-02-24 08:31:31 +01001756 static_key_slow_inc(&netstamp_needed);
Eric Dumazet13baa002017-03-01 14:28:39 -08001757#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07001758}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001759EXPORT_SYMBOL(net_enable_timestamp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001760
1761void net_disable_timestamp(void)
1762{
Eric Dumazetb90e5792011-11-28 11:16:50 +00001763#ifdef HAVE_JUMP_LABEL
Eric Dumazet13baa002017-03-01 14:28:39 -08001764 int wanted;
1765
1766 while (1) {
1767 wanted = atomic_read(&netstamp_wanted);
1768 if (wanted <= 1)
1769 break;
1770 if (atomic_cmpxchg(&netstamp_wanted, wanted, wanted - 1) == wanted)
1771 return;
1772 }
1773 atomic_dec(&netstamp_needed_deferred);
Eric Dumazet5fa8bbd2017-02-02 10:31:35 -08001774 schedule_work(&netstamp_work);
1775#else
Ingo Molnarc5905af2012-02-24 08:31:31 +01001776 static_key_slow_dec(&netstamp_needed);
Eric Dumazet5fa8bbd2017-02-02 10:31:35 -08001777#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07001778}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001779EXPORT_SYMBOL(net_disable_timestamp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001780
Eric Dumazet3b098e22010-05-15 23:57:10 -07001781static inline void net_timestamp_set(struct sk_buff *skb)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001782{
Thomas Gleixner2456e852016-12-25 11:38:40 +01001783 skb->tstamp = 0;
Ingo Molnarc5905af2012-02-24 08:31:31 +01001784 if (static_key_false(&netstamp_needed))
Patrick McHardya61bbcf2005-08-14 17:24:31 -07001785 __net_timestamp(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001786}
1787
Eric Dumazet588f0332011-11-15 04:12:55 +00001788#define net_timestamp_check(COND, SKB) \
Ingo Molnarc5905af2012-02-24 08:31:31 +01001789 if (static_key_false(&netstamp_needed)) { \
Thomas Gleixner2456e852016-12-25 11:38:40 +01001790 if ((COND) && !(SKB)->tstamp) \
Eric Dumazet588f0332011-11-15 04:12:55 +00001791 __net_timestamp(SKB); \
1792 } \
Eric Dumazet3b098e22010-05-15 23:57:10 -07001793
Nikolay Aleksandrovf4b05d22016-04-28 17:59:28 +02001794bool is_skb_forwardable(const struct net_device *dev, const struct sk_buff *skb)
Daniel Lezcano79b569f2011-03-30 02:42:17 -07001795{
1796 unsigned int len;
1797
1798 if (!(dev->flags & IFF_UP))
1799 return false;
1800
1801 len = dev->mtu + dev->hard_header_len + VLAN_HLEN;
1802 if (skb->len <= len)
1803 return true;
1804
1805 /* if TSO is enabled, we don't care about the length as the packet
1806 * could be forwarded without being segmented before
1807 */
1808 if (skb_is_gso(skb))
1809 return true;
1810
1811 return false;
1812}
Vlad Yasevich1ee481f2014-03-27 17:32:29 -04001813EXPORT_SYMBOL_GPL(is_skb_forwardable);
Daniel Lezcano79b569f2011-03-30 02:42:17 -07001814
Herbert Xua0265d22014-04-17 13:45:03 +08001815int __dev_forward_skb(struct net_device *dev, struct sk_buff *skb)
1816{
Martin KaFai Lau4e3264d2016-11-09 15:36:33 -08001817 int ret = ____dev_forward_skb(dev, skb);
1818
1819 if (likely(!ret)) {
1820 skb->protocol = eth_type_trans(skb, dev);
1821 skb_postpull_rcsum(skb, eth_hdr(skb), ETH_HLEN);
Herbert Xua0265d22014-04-17 13:45:03 +08001822 }
1823
Martin KaFai Lau4e3264d2016-11-09 15:36:33 -08001824 return ret;
Herbert Xua0265d22014-04-17 13:45:03 +08001825}
1826EXPORT_SYMBOL_GPL(__dev_forward_skb);
1827
Arnd Bergmann44540962009-11-26 06:07:08 +00001828/**
1829 * dev_forward_skb - loopback an skb to another netif
1830 *
1831 * @dev: destination network device
1832 * @skb: buffer to forward
1833 *
1834 * return values:
1835 * NET_RX_SUCCESS (no congestion)
Eric Dumazet6ec82562010-05-06 00:53:53 -07001836 * NET_RX_DROP (packet was dropped, but freed)
Arnd Bergmann44540962009-11-26 06:07:08 +00001837 *
1838 * dev_forward_skb can be used for injecting an skb from the
1839 * start_xmit function of one device into the receive queue
1840 * of another device.
1841 *
1842 * The receiving device may be in another namespace, so
1843 * we have to clear all information in the skb that could
1844 * impact namespace isolation.
1845 */
1846int dev_forward_skb(struct net_device *dev, struct sk_buff *skb)
1847{
Herbert Xua0265d22014-04-17 13:45:03 +08001848 return __dev_forward_skb(dev, skb) ?: netif_rx_internal(skb);
Arnd Bergmann44540962009-11-26 06:07:08 +00001849}
1850EXPORT_SYMBOL_GPL(dev_forward_skb);
1851
Changli Gao71d9dec2010-12-15 19:57:25 +00001852static inline int deliver_skb(struct sk_buff *skb,
1853 struct packet_type *pt_prev,
1854 struct net_device *orig_dev)
1855{
Willem de Bruijn1f8b9772017-08-03 16:29:41 -04001856 if (unlikely(skb_orphan_frags_rx(skb, GFP_ATOMIC)))
Michael S. Tsirkin1080e512012-07-20 09:23:17 +00001857 return -ENOMEM;
Reshetova, Elena63354792017-06-30 13:07:58 +03001858 refcount_inc(&skb->users);
Changli Gao71d9dec2010-12-15 19:57:25 +00001859 return pt_prev->func(skb, skb->dev, pt_prev, orig_dev);
1860}
1861
Salam Noureddine7866a622015-01-27 11:35:48 -08001862static inline void deliver_ptype_list_skb(struct sk_buff *skb,
1863 struct packet_type **pt,
Jiri Pirkofbcb2172015-03-30 16:56:01 +02001864 struct net_device *orig_dev,
1865 __be16 type,
Salam Noureddine7866a622015-01-27 11:35:48 -08001866 struct list_head *ptype_list)
1867{
1868 struct packet_type *ptype, *pt_prev = *pt;
1869
1870 list_for_each_entry_rcu(ptype, ptype_list, list) {
1871 if (ptype->type != type)
1872 continue;
1873 if (pt_prev)
Jiri Pirkofbcb2172015-03-30 16:56:01 +02001874 deliver_skb(skb, pt_prev, orig_dev);
Salam Noureddine7866a622015-01-27 11:35:48 -08001875 pt_prev = ptype;
1876 }
1877 *pt = pt_prev;
1878}
1879
Eric Leblondc0de08d2012-08-16 22:02:58 +00001880static inline bool skb_loop_sk(struct packet_type *ptype, struct sk_buff *skb)
1881{
Eric Leblonda3d744e2012-11-06 02:10:10 +00001882 if (!ptype->af_packet_priv || !skb->sk)
Eric Leblondc0de08d2012-08-16 22:02:58 +00001883 return false;
1884
1885 if (ptype->id_match)
1886 return ptype->id_match(ptype, skb->sk);
1887 else if ((struct sock *)ptype->af_packet_priv == skb->sk)
1888 return true;
1889
1890 return false;
1891}
1892
Linus Torvalds1da177e2005-04-16 15:20:36 -07001893/*
1894 * Support routine. Sends outgoing frames to any network
1895 * taps currently in use.
1896 */
1897
David Ahern74b20582016-05-10 11:19:50 -07001898void dev_queue_xmit_nit(struct sk_buff *skb, struct net_device *dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001899{
1900 struct packet_type *ptype;
Changli Gao71d9dec2010-12-15 19:57:25 +00001901 struct sk_buff *skb2 = NULL;
1902 struct packet_type *pt_prev = NULL;
Salam Noureddine7866a622015-01-27 11:35:48 -08001903 struct list_head *ptype_list = &ptype_all;
Patrick McHardya61bbcf2005-08-14 17:24:31 -07001904
Linus Torvalds1da177e2005-04-16 15:20:36 -07001905 rcu_read_lock();
Salam Noureddine7866a622015-01-27 11:35:48 -08001906again:
1907 list_for_each_entry_rcu(ptype, ptype_list, list) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001908 /* Never send packets back to the socket
1909 * they originated from - MvS (miquels@drinkel.ow.org)
1910 */
Salam Noureddine7866a622015-01-27 11:35:48 -08001911 if (skb_loop_sk(ptype, skb))
1912 continue;
Changli Gao71d9dec2010-12-15 19:57:25 +00001913
Salam Noureddine7866a622015-01-27 11:35:48 -08001914 if (pt_prev) {
1915 deliver_skb(skb2, pt_prev, skb->dev);
Changli Gao71d9dec2010-12-15 19:57:25 +00001916 pt_prev = ptype;
Salam Noureddine7866a622015-01-27 11:35:48 -08001917 continue;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001918 }
Salam Noureddine7866a622015-01-27 11:35:48 -08001919
1920 /* need to clone skb, done only once */
1921 skb2 = skb_clone(skb, GFP_ATOMIC);
1922 if (!skb2)
1923 goto out_unlock;
1924
1925 net_timestamp_set(skb2);
1926
1927 /* skb->nh should be correctly
1928 * set by sender, so that the second statement is
1929 * just protection against buggy protocols.
1930 */
1931 skb_reset_mac_header(skb2);
1932
1933 if (skb_network_header(skb2) < skb2->data ||
1934 skb_network_header(skb2) > skb_tail_pointer(skb2)) {
1935 net_crit_ratelimited("protocol %04x is buggy, dev %s\n",
1936 ntohs(skb2->protocol),
1937 dev->name);
1938 skb_reset_network_header(skb2);
1939 }
1940
1941 skb2->transport_header = skb2->network_header;
1942 skb2->pkt_type = PACKET_OUTGOING;
1943 pt_prev = ptype;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001944 }
Salam Noureddine7866a622015-01-27 11:35:48 -08001945
1946 if (ptype_list == &ptype_all) {
1947 ptype_list = &dev->ptype_all;
1948 goto again;
1949 }
1950out_unlock:
Changli Gao71d9dec2010-12-15 19:57:25 +00001951 if (pt_prev)
1952 pt_prev->func(skb2, skb->dev, pt_prev, skb->dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001953 rcu_read_unlock();
1954}
David Ahern74b20582016-05-10 11:19:50 -07001955EXPORT_SYMBOL_GPL(dev_queue_xmit_nit);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001956
Ben Hutchings2c530402012-07-10 10:55:09 +00001957/**
1958 * netif_setup_tc - Handle tc mappings on real_num_tx_queues change
John Fastabend4f57c082011-01-17 08:06:04 +00001959 * @dev: Network device
1960 * @txq: number of queues available
1961 *
1962 * If real_num_tx_queues is changed the tc mappings may no longer be
1963 * valid. To resolve this verify the tc mapping remains valid and if
1964 * not NULL the mapping. With no priorities mapping to this
1965 * offset/count pair it will no longer be used. In the worst case TC0
1966 * is invalid nothing can be done so disable priority mappings. If is
1967 * expected that drivers will fix this mapping if they can before
1968 * calling netif_set_real_num_tx_queues.
1969 */
Eric Dumazetbb134d22011-01-20 19:18:08 +00001970static void netif_setup_tc(struct net_device *dev, unsigned int txq)
John Fastabend4f57c082011-01-17 08:06:04 +00001971{
1972 int i;
1973 struct netdev_tc_txq *tc = &dev->tc_to_txq[0];
1974
1975 /* If TC0 is invalidated disable TC mapping */
1976 if (tc->offset + tc->count > txq) {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00001977 pr_warn("Number of in use tx queues changed invalidating tc mappings. Priority traffic classification disabled!\n");
John Fastabend4f57c082011-01-17 08:06:04 +00001978 dev->num_tc = 0;
1979 return;
1980 }
1981
1982 /* Invalidated prio to tc mappings set to TC0 */
1983 for (i = 1; i < TC_BITMASK + 1; i++) {
1984 int q = netdev_get_prio_tc_map(dev, i);
1985
1986 tc = &dev->tc_to_txq[q];
1987 if (tc->offset + tc->count > txq) {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00001988 pr_warn("Number of in use tx queues changed. Priority %i to tc mapping %i is no longer valid. Setting map to 0\n",
1989 i, q);
John Fastabend4f57c082011-01-17 08:06:04 +00001990 netdev_set_prio_tc_map(dev, i, 0);
1991 }
1992 }
1993}
1994
Alexander Duyck8d059b02016-10-28 11:43:49 -04001995int netdev_txq_to_tc(struct net_device *dev, unsigned int txq)
1996{
1997 if (dev->num_tc) {
1998 struct netdev_tc_txq *tc = &dev->tc_to_txq[0];
1999 int i;
2000
2001 for (i = 0; i < TC_MAX_QUEUE; i++, tc++) {
2002 if ((txq - tc->offset) < tc->count)
2003 return i;
2004 }
2005
2006 return -1;
2007 }
2008
2009 return 0;
2010}
2011
Alexander Duyck537c00d2013-01-10 08:57:02 +00002012#ifdef CONFIG_XPS
2013static DEFINE_MUTEX(xps_map_mutex);
2014#define xmap_dereference(P) \
2015 rcu_dereference_protected((P), lockdep_is_held(&xps_map_mutex))
2016
Alexander Duyck6234f872016-10-28 11:46:49 -04002017static bool remove_xps_queue(struct xps_dev_maps *dev_maps,
2018 int tci, u16 index)
Alexander Duyck10cdc3f2013-01-10 08:57:17 +00002019{
2020 struct xps_map *map = NULL;
2021 int pos;
2022
2023 if (dev_maps)
Alexander Duyck6234f872016-10-28 11:46:49 -04002024 map = xmap_dereference(dev_maps->cpu_map[tci]);
2025 if (!map)
2026 return false;
Alexander Duyck10cdc3f2013-01-10 08:57:17 +00002027
Alexander Duyck6234f872016-10-28 11:46:49 -04002028 for (pos = map->len; pos--;) {
2029 if (map->queues[pos] != index)
2030 continue;
2031
2032 if (map->len > 1) {
2033 map->queues[pos] = map->queues[--map->len];
Alexander Duyck10cdc3f2013-01-10 08:57:17 +00002034 break;
2035 }
Alexander Duyck6234f872016-10-28 11:46:49 -04002036
2037 RCU_INIT_POINTER(dev_maps->cpu_map[tci], NULL);
2038 kfree_rcu(map, rcu);
2039 return false;
Alexander Duyck10cdc3f2013-01-10 08:57:17 +00002040 }
2041
Alexander Duyck6234f872016-10-28 11:46:49 -04002042 return true;
Alexander Duyck10cdc3f2013-01-10 08:57:17 +00002043}
2044
Alexander Duyck6234f872016-10-28 11:46:49 -04002045static bool remove_xps_queue_cpu(struct net_device *dev,
2046 struct xps_dev_maps *dev_maps,
2047 int cpu, u16 offset, u16 count)
2048{
Alexander Duyck184c4492016-10-28 11:50:13 -04002049 int num_tc = dev->num_tc ? : 1;
2050 bool active = false;
2051 int tci;
Alexander Duyck6234f872016-10-28 11:46:49 -04002052
Alexander Duyck184c4492016-10-28 11:50:13 -04002053 for (tci = cpu * num_tc; num_tc--; tci++) {
2054 int i, j;
2055
2056 for (i = count, j = offset; i--; j++) {
2057 if (!remove_xps_queue(dev_maps, cpu, j))
2058 break;
2059 }
2060
2061 active |= i < 0;
Alexander Duyck6234f872016-10-28 11:46:49 -04002062 }
2063
Alexander Duyck184c4492016-10-28 11:50:13 -04002064 return active;
Alexander Duyck6234f872016-10-28 11:46:49 -04002065}
2066
2067static void netif_reset_xps_queues(struct net_device *dev, u16 offset,
2068 u16 count)
Alexander Duyck537c00d2013-01-10 08:57:02 +00002069{
2070 struct xps_dev_maps *dev_maps;
Alexander Duyck024e9672013-01-10 08:57:46 +00002071 int cpu, i;
Alexander Duyck10cdc3f2013-01-10 08:57:17 +00002072 bool active = false;
Alexander Duyck537c00d2013-01-10 08:57:02 +00002073
2074 mutex_lock(&xps_map_mutex);
2075 dev_maps = xmap_dereference(dev->xps_maps);
2076
2077 if (!dev_maps)
2078 goto out_no_maps;
2079
Alexander Duyck6234f872016-10-28 11:46:49 -04002080 for_each_possible_cpu(cpu)
2081 active |= remove_xps_queue_cpu(dev, dev_maps, cpu,
2082 offset, count);
Alexander Duyck537c00d2013-01-10 08:57:02 +00002083
Alexander Duyck10cdc3f2013-01-10 08:57:17 +00002084 if (!active) {
Alexander Duyck537c00d2013-01-10 08:57:02 +00002085 RCU_INIT_POINTER(dev->xps_maps, NULL);
2086 kfree_rcu(dev_maps, rcu);
2087 }
2088
Alexander Duyck6234f872016-10-28 11:46:49 -04002089 for (i = offset + (count - 1); count--; i--)
Alexander Duyck024e9672013-01-10 08:57:46 +00002090 netdev_queue_numa_node_write(netdev_get_tx_queue(dev, i),
2091 NUMA_NO_NODE);
2092
Alexander Duyck537c00d2013-01-10 08:57:02 +00002093out_no_maps:
2094 mutex_unlock(&xps_map_mutex);
2095}
2096
Alexander Duyck6234f872016-10-28 11:46:49 -04002097static void netif_reset_xps_queues_gt(struct net_device *dev, u16 index)
2098{
2099 netif_reset_xps_queues(dev, index, dev->num_tx_queues - index);
2100}
2101
Alexander Duyck01c5f862013-01-10 08:57:35 +00002102static struct xps_map *expand_xps_map(struct xps_map *map,
2103 int cpu, u16 index)
2104{
2105 struct xps_map *new_map;
2106 int alloc_len = XPS_MIN_MAP_ALLOC;
2107 int i, pos;
2108
2109 for (pos = 0; map && pos < map->len; pos++) {
2110 if (map->queues[pos] != index)
2111 continue;
2112 return map;
2113 }
2114
2115 /* Need to add queue to this CPU's existing map */
2116 if (map) {
2117 if (pos < map->alloc_len)
2118 return map;
2119
2120 alloc_len = map->alloc_len * 2;
2121 }
2122
2123 /* Need to allocate new map to store queue on this CPU's map */
2124 new_map = kzalloc_node(XPS_MAP_SIZE(alloc_len), GFP_KERNEL,
2125 cpu_to_node(cpu));
2126 if (!new_map)
2127 return NULL;
2128
2129 for (i = 0; i < pos; i++)
2130 new_map->queues[i] = map->queues[i];
2131 new_map->alloc_len = alloc_len;
2132 new_map->len = pos;
2133
2134 return new_map;
2135}
2136
Michael S. Tsirkin35735402013-10-02 09:14:06 +03002137int netif_set_xps_queue(struct net_device *dev, const struct cpumask *mask,
2138 u16 index)
Alexander Duyck537c00d2013-01-10 08:57:02 +00002139{
Alexander Duyck01c5f862013-01-10 08:57:35 +00002140 struct xps_dev_maps *dev_maps, *new_dev_maps = NULL;
Alexander Duyck184c4492016-10-28 11:50:13 -04002141 int i, cpu, tci, numa_node_id = -2;
2142 int maps_sz, num_tc = 1, tc = 0;
Alexander Duyck537c00d2013-01-10 08:57:02 +00002143 struct xps_map *map, *new_map;
Alexander Duyck01c5f862013-01-10 08:57:35 +00002144 bool active = false;
Alexander Duyck537c00d2013-01-10 08:57:02 +00002145
Alexander Duyck184c4492016-10-28 11:50:13 -04002146 if (dev->num_tc) {
2147 num_tc = dev->num_tc;
2148 tc = netdev_txq_to_tc(dev, index);
2149 if (tc < 0)
2150 return -EINVAL;
2151 }
2152
2153 maps_sz = XPS_DEV_MAPS_SIZE(num_tc);
2154 if (maps_sz < L1_CACHE_BYTES)
2155 maps_sz = L1_CACHE_BYTES;
2156
Alexander Duyck537c00d2013-01-10 08:57:02 +00002157 mutex_lock(&xps_map_mutex);
2158
2159 dev_maps = xmap_dereference(dev->xps_maps);
2160
Alexander Duyck01c5f862013-01-10 08:57:35 +00002161 /* allocate memory for queue storage */
Alexander Duyck184c4492016-10-28 11:50:13 -04002162 for_each_cpu_and(cpu, cpu_online_mask, mask) {
Alexander Duyck01c5f862013-01-10 08:57:35 +00002163 if (!new_dev_maps)
2164 new_dev_maps = kzalloc(maps_sz, GFP_KERNEL);
Alexander Duyck2bb60cb2013-02-22 06:38:44 +00002165 if (!new_dev_maps) {
2166 mutex_unlock(&xps_map_mutex);
Alexander Duyck01c5f862013-01-10 08:57:35 +00002167 return -ENOMEM;
Alexander Duyck2bb60cb2013-02-22 06:38:44 +00002168 }
Alexander Duyck01c5f862013-01-10 08:57:35 +00002169
Alexander Duyck184c4492016-10-28 11:50:13 -04002170 tci = cpu * num_tc + tc;
2171 map = dev_maps ? xmap_dereference(dev_maps->cpu_map[tci]) :
Alexander Duyck01c5f862013-01-10 08:57:35 +00002172 NULL;
2173
2174 map = expand_xps_map(map, cpu, index);
2175 if (!map)
2176 goto error;
2177
Alexander Duyck184c4492016-10-28 11:50:13 -04002178 RCU_INIT_POINTER(new_dev_maps->cpu_map[tci], map);
Alexander Duyck01c5f862013-01-10 08:57:35 +00002179 }
2180
2181 if (!new_dev_maps)
2182 goto out_no_new_maps;
2183
2184 for_each_possible_cpu(cpu) {
Alexander Duyck184c4492016-10-28 11:50:13 -04002185 /* copy maps belonging to foreign traffic classes */
2186 for (i = tc, tci = cpu * num_tc; dev_maps && i--; tci++) {
2187 /* fill in the new device map from the old device map */
2188 map = xmap_dereference(dev_maps->cpu_map[tci]);
2189 RCU_INIT_POINTER(new_dev_maps->cpu_map[tci], map);
2190 }
2191
2192 /* We need to explicitly update tci as prevous loop
2193 * could break out early if dev_maps is NULL.
2194 */
2195 tci = cpu * num_tc + tc;
2196
Alexander Duyck01c5f862013-01-10 08:57:35 +00002197 if (cpumask_test_cpu(cpu, mask) && cpu_online(cpu)) {
2198 /* add queue to CPU maps */
2199 int pos = 0;
2200
Alexander Duyck184c4492016-10-28 11:50:13 -04002201 map = xmap_dereference(new_dev_maps->cpu_map[tci]);
Alexander Duyck01c5f862013-01-10 08:57:35 +00002202 while ((pos < map->len) && (map->queues[pos] != index))
2203 pos++;
2204
2205 if (pos == map->len)
2206 map->queues[map->len++] = index;
Alexander Duyck537c00d2013-01-10 08:57:02 +00002207#ifdef CONFIG_NUMA
Alexander Duyck537c00d2013-01-10 08:57:02 +00002208 if (numa_node_id == -2)
2209 numa_node_id = cpu_to_node(cpu);
2210 else if (numa_node_id != cpu_to_node(cpu))
2211 numa_node_id = -1;
Alexander Duyck537c00d2013-01-10 08:57:02 +00002212#endif
Alexander Duyck01c5f862013-01-10 08:57:35 +00002213 } else if (dev_maps) {
2214 /* fill in the new device map from the old device map */
Alexander Duyck184c4492016-10-28 11:50:13 -04002215 map = xmap_dereference(dev_maps->cpu_map[tci]);
2216 RCU_INIT_POINTER(new_dev_maps->cpu_map[tci], map);
Alexander Duyck537c00d2013-01-10 08:57:02 +00002217 }
Alexander Duyck01c5f862013-01-10 08:57:35 +00002218
Alexander Duyck184c4492016-10-28 11:50:13 -04002219 /* copy maps belonging to foreign traffic classes */
2220 for (i = num_tc - tc, tci++; dev_maps && --i; tci++) {
2221 /* fill in the new device map from the old device map */
2222 map = xmap_dereference(dev_maps->cpu_map[tci]);
2223 RCU_INIT_POINTER(new_dev_maps->cpu_map[tci], map);
2224 }
Alexander Duyck537c00d2013-01-10 08:57:02 +00002225 }
2226
Alexander Duyck01c5f862013-01-10 08:57:35 +00002227 rcu_assign_pointer(dev->xps_maps, new_dev_maps);
2228
Alexander Duyck537c00d2013-01-10 08:57:02 +00002229 /* Cleanup old maps */
Alexander Duyck184c4492016-10-28 11:50:13 -04002230 if (!dev_maps)
2231 goto out_no_old_maps;
2232
2233 for_each_possible_cpu(cpu) {
2234 for (i = num_tc, tci = cpu * num_tc; i--; tci++) {
2235 new_map = xmap_dereference(new_dev_maps->cpu_map[tci]);
2236 map = xmap_dereference(dev_maps->cpu_map[tci]);
Alexander Duyck01c5f862013-01-10 08:57:35 +00002237 if (map && map != new_map)
2238 kfree_rcu(map, rcu);
2239 }
Alexander Duyck01c5f862013-01-10 08:57:35 +00002240 }
Alexander Duyck537c00d2013-01-10 08:57:02 +00002241
Alexander Duyck184c4492016-10-28 11:50:13 -04002242 kfree_rcu(dev_maps, rcu);
2243
2244out_no_old_maps:
Alexander Duyck01c5f862013-01-10 08:57:35 +00002245 dev_maps = new_dev_maps;
2246 active = true;
2247
2248out_no_new_maps:
2249 /* update Tx queue numa node */
Alexander Duyck537c00d2013-01-10 08:57:02 +00002250 netdev_queue_numa_node_write(netdev_get_tx_queue(dev, index),
2251 (numa_node_id >= 0) ? numa_node_id :
2252 NUMA_NO_NODE);
2253
Alexander Duyck01c5f862013-01-10 08:57:35 +00002254 if (!dev_maps)
2255 goto out_no_maps;
2256
2257 /* removes queue from unused CPUs */
2258 for_each_possible_cpu(cpu) {
Alexander Duyck184c4492016-10-28 11:50:13 -04002259 for (i = tc, tci = cpu * num_tc; i--; tci++)
2260 active |= remove_xps_queue(dev_maps, tci, index);
2261 if (!cpumask_test_cpu(cpu, mask) || !cpu_online(cpu))
2262 active |= remove_xps_queue(dev_maps, tci, index);
2263 for (i = num_tc - tc, tci++; --i; tci++)
2264 active |= remove_xps_queue(dev_maps, tci, index);
Alexander Duyck01c5f862013-01-10 08:57:35 +00002265 }
2266
2267 /* free map if not active */
2268 if (!active) {
2269 RCU_INIT_POINTER(dev->xps_maps, NULL);
2270 kfree_rcu(dev_maps, rcu);
2271 }
2272
2273out_no_maps:
Alexander Duyck537c00d2013-01-10 08:57:02 +00002274 mutex_unlock(&xps_map_mutex);
2275
2276 return 0;
2277error:
Alexander Duyck01c5f862013-01-10 08:57:35 +00002278 /* remove any maps that we added */
2279 for_each_possible_cpu(cpu) {
Alexander Duyck184c4492016-10-28 11:50:13 -04002280 for (i = num_tc, tci = cpu * num_tc; i--; tci++) {
2281 new_map = xmap_dereference(new_dev_maps->cpu_map[tci]);
2282 map = dev_maps ?
2283 xmap_dereference(dev_maps->cpu_map[tci]) :
2284 NULL;
2285 if (new_map && new_map != map)
2286 kfree(new_map);
2287 }
Alexander Duyck01c5f862013-01-10 08:57:35 +00002288 }
2289
Alexander Duyck537c00d2013-01-10 08:57:02 +00002290 mutex_unlock(&xps_map_mutex);
2291
Alexander Duyck537c00d2013-01-10 08:57:02 +00002292 kfree(new_dev_maps);
2293 return -ENOMEM;
2294}
2295EXPORT_SYMBOL(netif_set_xps_queue);
2296
2297#endif
Alexander Duyck9cf1f6a2016-10-28 11:43:20 -04002298void netdev_reset_tc(struct net_device *dev)
2299{
Alexander Duyck6234f872016-10-28 11:46:49 -04002300#ifdef CONFIG_XPS
2301 netif_reset_xps_queues_gt(dev, 0);
2302#endif
Alexander Duyck9cf1f6a2016-10-28 11:43:20 -04002303 dev->num_tc = 0;
2304 memset(dev->tc_to_txq, 0, sizeof(dev->tc_to_txq));
2305 memset(dev->prio_tc_map, 0, sizeof(dev->prio_tc_map));
2306}
2307EXPORT_SYMBOL(netdev_reset_tc);
2308
2309int netdev_set_tc_queue(struct net_device *dev, u8 tc, u16 count, u16 offset)
2310{
2311 if (tc >= dev->num_tc)
2312 return -EINVAL;
2313
Alexander Duyck6234f872016-10-28 11:46:49 -04002314#ifdef CONFIG_XPS
2315 netif_reset_xps_queues(dev, offset, count);
2316#endif
Alexander Duyck9cf1f6a2016-10-28 11:43:20 -04002317 dev->tc_to_txq[tc].count = count;
2318 dev->tc_to_txq[tc].offset = offset;
2319 return 0;
2320}
2321EXPORT_SYMBOL(netdev_set_tc_queue);
2322
2323int netdev_set_num_tc(struct net_device *dev, u8 num_tc)
2324{
2325 if (num_tc > TC_MAX_QUEUE)
2326 return -EINVAL;
2327
Alexander Duyck6234f872016-10-28 11:46:49 -04002328#ifdef CONFIG_XPS
2329 netif_reset_xps_queues_gt(dev, 0);
2330#endif
Alexander Duyck9cf1f6a2016-10-28 11:43:20 -04002331 dev->num_tc = num_tc;
2332 return 0;
2333}
2334EXPORT_SYMBOL(netdev_set_num_tc);
2335
John Fastabendf0796d52010-07-01 13:21:57 +00002336/*
2337 * Routine to help set real_num_tx_queues. To avoid skbs mapped to queues
2338 * greater then real_num_tx_queues stale skbs on the qdisc must be flushed.
2339 */
Tom Herberte6484932010-10-18 18:04:39 +00002340int netif_set_real_num_tx_queues(struct net_device *dev, unsigned int txq)
John Fastabendf0796d52010-07-01 13:21:57 +00002341{
Tom Herbert1d24eb42010-11-21 13:17:27 +00002342 int rc;
2343
Tom Herberte6484932010-10-18 18:04:39 +00002344 if (txq < 1 || txq > dev->num_tx_queues)
2345 return -EINVAL;
John Fastabendf0796d52010-07-01 13:21:57 +00002346
Ben Hutchings5c565802011-02-15 19:39:21 +00002347 if (dev->reg_state == NETREG_REGISTERED ||
2348 dev->reg_state == NETREG_UNREGISTERING) {
Tom Herberte6484932010-10-18 18:04:39 +00002349 ASSERT_RTNL();
2350
Tom Herbert1d24eb42010-11-21 13:17:27 +00002351 rc = netdev_queue_update_kobjects(dev, dev->real_num_tx_queues,
2352 txq);
Tom Herbertbf264142010-11-26 08:36:09 +00002353 if (rc)
2354 return rc;
2355
John Fastabend4f57c082011-01-17 08:06:04 +00002356 if (dev->num_tc)
2357 netif_setup_tc(dev, txq);
2358
Alexander Duyck024e9672013-01-10 08:57:46 +00002359 if (txq < dev->real_num_tx_queues) {
Tom Herberte6484932010-10-18 18:04:39 +00002360 qdisc_reset_all_tx_gt(dev, txq);
Alexander Duyck024e9672013-01-10 08:57:46 +00002361#ifdef CONFIG_XPS
2362 netif_reset_xps_queues_gt(dev, txq);
2363#endif
2364 }
John Fastabendf0796d52010-07-01 13:21:57 +00002365 }
Tom Herberte6484932010-10-18 18:04:39 +00002366
2367 dev->real_num_tx_queues = txq;
2368 return 0;
John Fastabendf0796d52010-07-01 13:21:57 +00002369}
2370EXPORT_SYMBOL(netif_set_real_num_tx_queues);
Denis Vlasenko56079432006-03-29 15:57:29 -08002371
Michael Daltona953be52014-01-16 22:23:28 -08002372#ifdef CONFIG_SYSFS
Ben Hutchings62fe0b42010-09-27 08:24:33 +00002373/**
2374 * netif_set_real_num_rx_queues - set actual number of RX queues used
2375 * @dev: Network device
2376 * @rxq: Actual number of RX queues
2377 *
2378 * This must be called either with the rtnl_lock held or before
2379 * registration of the net device. Returns 0 on success, or a
Ben Hutchings4e7f7952010-10-08 10:33:39 -07002380 * negative error code. If called before registration, it always
2381 * succeeds.
Ben Hutchings62fe0b42010-09-27 08:24:33 +00002382 */
2383int netif_set_real_num_rx_queues(struct net_device *dev, unsigned int rxq)
2384{
2385 int rc;
2386
Tom Herbertbd25fa72010-10-18 18:00:16 +00002387 if (rxq < 1 || rxq > dev->num_rx_queues)
2388 return -EINVAL;
2389
Ben Hutchings62fe0b42010-09-27 08:24:33 +00002390 if (dev->reg_state == NETREG_REGISTERED) {
2391 ASSERT_RTNL();
2392
Ben Hutchings62fe0b42010-09-27 08:24:33 +00002393 rc = net_rx_queue_update_kobjects(dev, dev->real_num_rx_queues,
2394 rxq);
2395 if (rc)
2396 return rc;
Ben Hutchings62fe0b42010-09-27 08:24:33 +00002397 }
2398
2399 dev->real_num_rx_queues = rxq;
2400 return 0;
2401}
2402EXPORT_SYMBOL(netif_set_real_num_rx_queues);
2403#endif
2404
Ben Hutchings2c530402012-07-10 10:55:09 +00002405/**
2406 * netif_get_num_default_rss_queues - default number of RSS queues
Yuval Mintz16917b82012-07-01 03:18:50 +00002407 *
2408 * This routine should set an upper limit on the number of RSS queues
2409 * used by default by multiqueue devices.
2410 */
Ben Hutchingsa55b1382012-07-10 10:54:38 +00002411int netif_get_num_default_rss_queues(void)
Yuval Mintz16917b82012-07-01 03:18:50 +00002412{
Hariprasad Shenai40e4e712016-06-08 18:09:08 +05302413 return is_kdump_kernel() ?
2414 1 : min_t(int, DEFAULT_MAX_NUM_RSS_QUEUES, num_online_cpus());
Yuval Mintz16917b82012-07-01 03:18:50 +00002415}
2416EXPORT_SYMBOL(netif_get_num_default_rss_queues);
2417
Eric Dumazet3bcb8462016-06-04 20:02:28 -07002418static void __netif_reschedule(struct Qdisc *q)
Jarek Poplawskidef82a12008-08-17 21:54:43 -07002419{
2420 struct softnet_data *sd;
2421 unsigned long flags;
2422
2423 local_irq_save(flags);
Christoph Lameter903ceff2014-08-17 12:30:35 -05002424 sd = this_cpu_ptr(&softnet_data);
Changli Gaoa9cbd582010-04-26 23:06:24 +00002425 q->next_sched = NULL;
2426 *sd->output_queue_tailp = q;
2427 sd->output_queue_tailp = &q->next_sched;
Jarek Poplawskidef82a12008-08-17 21:54:43 -07002428 raise_softirq_irqoff(NET_TX_SOFTIRQ);
2429 local_irq_restore(flags);
2430}
2431
David S. Miller37437bb2008-07-16 02:15:04 -07002432void __netif_schedule(struct Qdisc *q)
Denis Vlasenko56079432006-03-29 15:57:29 -08002433{
Jarek Poplawskidef82a12008-08-17 21:54:43 -07002434 if (!test_and_set_bit(__QDISC_STATE_SCHED, &q->state))
2435 __netif_reschedule(q);
Denis Vlasenko56079432006-03-29 15:57:29 -08002436}
2437EXPORT_SYMBOL(__netif_schedule);
2438
Eric Dumazete6247022013-12-05 04:45:08 -08002439struct dev_kfree_skb_cb {
2440 enum skb_free_reason reason;
2441};
2442
2443static struct dev_kfree_skb_cb *get_kfree_skb_cb(const struct sk_buff *skb)
Denis Vlasenko56079432006-03-29 15:57:29 -08002444{
Eric Dumazete6247022013-12-05 04:45:08 -08002445 return (struct dev_kfree_skb_cb *)skb->cb;
Denis Vlasenko56079432006-03-29 15:57:29 -08002446}
Denis Vlasenko56079432006-03-29 15:57:29 -08002447
John Fastabend46e5da40a2014-09-12 20:04:52 -07002448void netif_schedule_queue(struct netdev_queue *txq)
2449{
2450 rcu_read_lock();
2451 if (!(txq->state & QUEUE_STATE_ANY_XOFF)) {
2452 struct Qdisc *q = rcu_dereference(txq->qdisc);
2453
2454 __netif_schedule(q);
2455 }
2456 rcu_read_unlock();
2457}
2458EXPORT_SYMBOL(netif_schedule_queue);
2459
John Fastabend46e5da40a2014-09-12 20:04:52 -07002460void netif_tx_wake_queue(struct netdev_queue *dev_queue)
2461{
2462 if (test_and_clear_bit(__QUEUE_STATE_DRV_XOFF, &dev_queue->state)) {
2463 struct Qdisc *q;
2464
2465 rcu_read_lock();
2466 q = rcu_dereference(dev_queue->qdisc);
2467 __netif_schedule(q);
2468 rcu_read_unlock();
2469 }
2470}
2471EXPORT_SYMBOL(netif_tx_wake_queue);
2472
Eric Dumazete6247022013-12-05 04:45:08 -08002473void __dev_kfree_skb_irq(struct sk_buff *skb, enum skb_free_reason reason)
2474{
2475 unsigned long flags;
2476
Myungho Jung98998862017-04-25 11:58:15 -07002477 if (unlikely(!skb))
2478 return;
2479
Reshetova, Elena63354792017-06-30 13:07:58 +03002480 if (likely(refcount_read(&skb->users) == 1)) {
Eric Dumazete6247022013-12-05 04:45:08 -08002481 smp_rmb();
Reshetova, Elena63354792017-06-30 13:07:58 +03002482 refcount_set(&skb->users, 0);
2483 } else if (likely(!refcount_dec_and_test(&skb->users))) {
Eric Dumazete6247022013-12-05 04:45:08 -08002484 return;
2485 }
2486 get_kfree_skb_cb(skb)->reason = reason;
2487 local_irq_save(flags);
2488 skb->next = __this_cpu_read(softnet_data.completion_queue);
2489 __this_cpu_write(softnet_data.completion_queue, skb);
2490 raise_softirq_irqoff(NET_TX_SOFTIRQ);
2491 local_irq_restore(flags);
2492}
2493EXPORT_SYMBOL(__dev_kfree_skb_irq);
2494
2495void __dev_kfree_skb_any(struct sk_buff *skb, enum skb_free_reason reason)
Denis Vlasenko56079432006-03-29 15:57:29 -08002496{
2497 if (in_irq() || irqs_disabled())
Eric Dumazete6247022013-12-05 04:45:08 -08002498 __dev_kfree_skb_irq(skb, reason);
Denis Vlasenko56079432006-03-29 15:57:29 -08002499 else
2500 dev_kfree_skb(skb);
2501}
Eric Dumazete6247022013-12-05 04:45:08 -08002502EXPORT_SYMBOL(__dev_kfree_skb_any);
Denis Vlasenko56079432006-03-29 15:57:29 -08002503
2504
Stephen Hemmingerbea33482007-10-03 16:41:36 -07002505/**
2506 * netif_device_detach - mark device as removed
2507 * @dev: network device
2508 *
2509 * Mark device as removed from system and therefore no longer available.
2510 */
Denis Vlasenko56079432006-03-29 15:57:29 -08002511void netif_device_detach(struct net_device *dev)
2512{
2513 if (test_and_clear_bit(__LINK_STATE_PRESENT, &dev->state) &&
2514 netif_running(dev)) {
Alexander Duyckd5431032009-04-08 13:15:22 +00002515 netif_tx_stop_all_queues(dev);
Denis Vlasenko56079432006-03-29 15:57:29 -08002516 }
2517}
2518EXPORT_SYMBOL(netif_device_detach);
2519
Stephen Hemmingerbea33482007-10-03 16:41:36 -07002520/**
2521 * netif_device_attach - mark device as attached
2522 * @dev: network device
2523 *
2524 * Mark device as attached from system and restart if needed.
2525 */
Denis Vlasenko56079432006-03-29 15:57:29 -08002526void netif_device_attach(struct net_device *dev)
2527{
2528 if (!test_and_set_bit(__LINK_STATE_PRESENT, &dev->state) &&
2529 netif_running(dev)) {
Alexander Duyckd5431032009-04-08 13:15:22 +00002530 netif_tx_wake_all_queues(dev);
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09002531 __netdev_watchdog_up(dev);
Denis Vlasenko56079432006-03-29 15:57:29 -08002532 }
2533}
2534EXPORT_SYMBOL(netif_device_attach);
2535
Jiri Pirko5605c762015-05-12 14:56:12 +02002536/*
2537 * Returns a Tx hash based on the given packet descriptor a Tx queues' number
2538 * to be used as a distribution range.
2539 */
2540u16 __skb_tx_hash(const struct net_device *dev, struct sk_buff *skb,
2541 unsigned int num_tx_queues)
2542{
2543 u32 hash;
2544 u16 qoffset = 0;
2545 u16 qcount = num_tx_queues;
2546
2547 if (skb_rx_queue_recorded(skb)) {
2548 hash = skb_get_rx_queue(skb);
2549 while (unlikely(hash >= num_tx_queues))
2550 hash -= num_tx_queues;
2551 return hash;
2552 }
2553
2554 if (dev->num_tc) {
2555 u8 tc = netdev_get_prio_tc_map(dev, skb->priority);
tchardingf4563a72017-02-09 17:56:07 +11002556
Jiri Pirko5605c762015-05-12 14:56:12 +02002557 qoffset = dev->tc_to_txq[tc].offset;
2558 qcount = dev->tc_to_txq[tc].count;
2559 }
2560
2561 return (u16) reciprocal_scale(skb_get_hash(skb), qcount) + qoffset;
2562}
2563EXPORT_SYMBOL(__skb_tx_hash);
2564
Ben Hutchings36c92472012-01-17 07:57:56 +00002565static void skb_warn_bad_offload(const struct sk_buff *skb)
2566{
Wei Tang84d15ae2016-06-16 21:17:49 +08002567 static const netdev_features_t null_features;
Ben Hutchings36c92472012-01-17 07:57:56 +00002568 struct net_device *dev = skb->dev;
Bjørn Mork88ad4172015-11-16 19:16:40 +01002569 const char *name = "";
Ben Hutchings36c92472012-01-17 07:57:56 +00002570
Ben Greearc846ad92013-04-19 10:45:52 +00002571 if (!net_ratelimit())
2572 return;
2573
Bjørn Mork88ad4172015-11-16 19:16:40 +01002574 if (dev) {
2575 if (dev->dev.parent)
2576 name = dev_driver_string(dev->dev.parent);
2577 else
2578 name = netdev_name(dev);
2579 }
Ben Hutchings36c92472012-01-17 07:57:56 +00002580 WARN(1, "%s: caps=(%pNF, %pNF) len=%d data_len=%d gso_size=%d "
2581 "gso_type=%d ip_summed=%d\n",
Bjørn Mork88ad4172015-11-16 19:16:40 +01002582 name, dev ? &dev->features : &null_features,
Michał Mirosław65e9d2f2012-01-17 10:00:40 +00002583 skb->sk ? &skb->sk->sk_route_caps : &null_features,
Ben Hutchings36c92472012-01-17 07:57:56 +00002584 skb->len, skb->data_len, skb_shinfo(skb)->gso_size,
2585 skb_shinfo(skb)->gso_type, skb->ip_summed);
2586}
2587
Linus Torvalds1da177e2005-04-16 15:20:36 -07002588/*
2589 * Invalidate hardware checksum when packet is to be mangled, and
2590 * complete checksum manually on outgoing path.
2591 */
Patrick McHardy84fa7932006-08-29 16:44:56 -07002592int skb_checksum_help(struct sk_buff *skb)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002593{
Al Virod3bc23e2006-11-14 21:24:49 -08002594 __wsum csum;
Herbert Xu663ead32007-04-09 11:59:07 -07002595 int ret = 0, offset;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002596
Patrick McHardy84fa7932006-08-29 16:44:56 -07002597 if (skb->ip_summed == CHECKSUM_COMPLETE)
Herbert Xua430a432006-07-08 13:34:56 -07002598 goto out_set_summed;
2599
2600 if (unlikely(skb_shinfo(skb)->gso_size)) {
Ben Hutchings36c92472012-01-17 07:57:56 +00002601 skb_warn_bad_offload(skb);
2602 return -EINVAL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002603 }
2604
Eric Dumazetcef401d2013-01-25 20:34:37 +00002605 /* Before computing a checksum, we should make sure no frag could
2606 * be modified by an external entity : checksum could be wrong.
2607 */
2608 if (skb_has_shared_frag(skb)) {
2609 ret = __skb_linearize(skb);
2610 if (ret)
2611 goto out;
2612 }
2613
Michał Mirosław55508d62010-12-14 15:24:08 +00002614 offset = skb_checksum_start_offset(skb);
Herbert Xua0308472007-10-15 01:47:15 -07002615 BUG_ON(offset >= skb_headlen(skb));
2616 csum = skb_checksum(skb, offset, skb->len - offset, 0);
2617
2618 offset += skb->csum_offset;
2619 BUG_ON(offset + sizeof(__sum16) > skb_headlen(skb));
2620
2621 if (skb_cloned(skb) &&
2622 !skb_clone_writable(skb, offset + sizeof(__sum16))) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07002623 ret = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
2624 if (ret)
2625 goto out;
2626 }
2627
Eric Dumazet4f2e4ad2016-10-29 11:02:36 -07002628 *(__sum16 *)(skb->data + offset) = csum_fold(csum) ?: CSUM_MANGLED_0;
Herbert Xua430a432006-07-08 13:34:56 -07002629out_set_summed:
Linus Torvalds1da177e2005-04-16 15:20:36 -07002630 skb->ip_summed = CHECKSUM_NONE;
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09002631out:
Linus Torvalds1da177e2005-04-16 15:20:36 -07002632 return ret;
2633}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07002634EXPORT_SYMBOL(skb_checksum_help);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002635
Davide Carattib72b5bf2017-05-18 15:44:38 +02002636int skb_crc32c_csum_help(struct sk_buff *skb)
2637{
2638 __le32 crc32c_csum;
2639 int ret = 0, offset, start;
2640
2641 if (skb->ip_summed != CHECKSUM_PARTIAL)
2642 goto out;
2643
2644 if (unlikely(skb_is_gso(skb)))
2645 goto out;
2646
2647 /* Before computing a checksum, we should make sure no frag could
2648 * be modified by an external entity : checksum could be wrong.
2649 */
2650 if (unlikely(skb_has_shared_frag(skb))) {
2651 ret = __skb_linearize(skb);
2652 if (ret)
2653 goto out;
2654 }
2655 start = skb_checksum_start_offset(skb);
2656 offset = start + offsetof(struct sctphdr, checksum);
2657 if (WARN_ON_ONCE(offset >= skb_headlen(skb))) {
2658 ret = -EINVAL;
2659 goto out;
2660 }
2661 if (skb_cloned(skb) &&
2662 !skb_clone_writable(skb, offset + sizeof(__le32))) {
2663 ret = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
2664 if (ret)
2665 goto out;
2666 }
2667 crc32c_csum = cpu_to_le32(~__skb_checksum(skb, start,
2668 skb->len - start, ~(__u32)0,
2669 crc32c_csum_stub));
2670 *(__le32 *)(skb->data + offset) = crc32c_csum;
2671 skb->ip_summed = CHECKSUM_NONE;
Davide Carattidba00302017-05-18 15:44:40 +02002672 skb->csum_not_inet = 0;
Davide Carattib72b5bf2017-05-18 15:44:38 +02002673out:
2674 return ret;
2675}
2676
Vlad Yasevich53d64712014-03-27 17:26:18 -04002677__be16 skb_network_protocol(struct sk_buff *skb, int *depth)
Pravin B Shelarec5f0612013-03-07 09:28:01 +00002678{
2679 __be16 type = skb->protocol;
2680
Pravin B Shelar19acc322013-05-07 20:41:07 +00002681 /* Tunnel gso handlers can set protocol to ethernet. */
2682 if (type == htons(ETH_P_TEB)) {
2683 struct ethhdr *eth;
2684
2685 if (unlikely(!pskb_may_pull(skb, sizeof(struct ethhdr))))
2686 return 0;
2687
2688 eth = (struct ethhdr *)skb_mac_header(skb);
2689 type = eth->h_proto;
2690 }
2691
Toshiaki Makitad4bcef32015-01-29 20:37:07 +09002692 return __vlan_get_protocol(skb, type, depth);
Pravin B Shelarec5f0612013-03-07 09:28:01 +00002693}
2694
Pravin B Shelar05e8ef42013-02-14 09:44:55 +00002695/**
2696 * skb_mac_gso_segment - mac layer segmentation handler.
2697 * @skb: buffer to segment
2698 * @features: features for the output path (see dev->features)
2699 */
2700struct sk_buff *skb_mac_gso_segment(struct sk_buff *skb,
2701 netdev_features_t features)
2702{
2703 struct sk_buff *segs = ERR_PTR(-EPROTONOSUPPORT);
2704 struct packet_offload *ptype;
Vlad Yasevich53d64712014-03-27 17:26:18 -04002705 int vlan_depth = skb->mac_len;
2706 __be16 type = skb_network_protocol(skb, &vlan_depth);
Pravin B Shelar05e8ef42013-02-14 09:44:55 +00002707
Pravin B Shelarec5f0612013-03-07 09:28:01 +00002708 if (unlikely(!type))
2709 return ERR_PTR(-EINVAL);
Pravin B Shelar05e8ef42013-02-14 09:44:55 +00002710
Vlad Yasevich53d64712014-03-27 17:26:18 -04002711 __skb_pull(skb, vlan_depth);
Pravin B Shelar05e8ef42013-02-14 09:44:55 +00002712
2713 rcu_read_lock();
2714 list_for_each_entry_rcu(ptype, &offload_base, list) {
2715 if (ptype->type == type && ptype->callbacks.gso_segment) {
Pravin B Shelar05e8ef42013-02-14 09:44:55 +00002716 segs = ptype->callbacks.gso_segment(skb, features);
2717 break;
2718 }
2719 }
2720 rcu_read_unlock();
2721
2722 __skb_push(skb, skb->data - skb_mac_header(skb));
2723
2724 return segs;
2725}
2726EXPORT_SYMBOL(skb_mac_gso_segment);
2727
2728
Cong Wang12b00042013-02-05 16:36:38 +00002729/* openvswitch calls this on rx path, so we need a different check.
2730 */
2731static inline bool skb_needs_check(struct sk_buff *skb, bool tx_path)
2732{
2733 if (tx_path)
Tonghao Zhang93991222017-08-10 20:16:29 -07002734 return skb->ip_summed != CHECKSUM_PARTIAL;
Eric Dumazet6e7bc472017-02-03 14:29:42 -08002735
2736 return skb->ip_summed == CHECKSUM_NONE;
Cong Wang12b00042013-02-05 16:36:38 +00002737}
2738
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002739/**
Cong Wang12b00042013-02-05 16:36:38 +00002740 * __skb_gso_segment - Perform segmentation on skb.
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002741 * @skb: buffer to segment
Herbert Xu576a30e2006-06-27 13:22:38 -07002742 * @features: features for the output path (see dev->features)
Cong Wang12b00042013-02-05 16:36:38 +00002743 * @tx_path: whether it is called in TX path
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002744 *
2745 * This function segments the given skb and returns a list of segments.
Herbert Xu576a30e2006-06-27 13:22:38 -07002746 *
2747 * It may return NULL if the skb requires no segmentation. This is
2748 * only possible when GSO is used for verifying header integrity.
Konstantin Khlebnikov9207f9d2016-01-08 15:21:46 +03002749 *
2750 * Segmentation preserves SKB_SGO_CB_OFFSET bytes of previous skb cb.
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002751 */
Cong Wang12b00042013-02-05 16:36:38 +00002752struct sk_buff *__skb_gso_segment(struct sk_buff *skb,
2753 netdev_features_t features, bool tx_path)
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002754{
Eric Dumazetb2504a52017-01-31 10:20:32 -08002755 struct sk_buff *segs;
2756
Cong Wang12b00042013-02-05 16:36:38 +00002757 if (unlikely(skb_needs_check(skb, tx_path))) {
Pravin B Shelar05e8ef42013-02-14 09:44:55 +00002758 int err;
2759
Eric Dumazetb2504a52017-01-31 10:20:32 -08002760 /* We're going to init ->check field in TCP or UDP header */
françois romieua40e0a62014-07-15 23:55:35 +02002761 err = skb_cow_head(skb, 0);
2762 if (err < 0)
Herbert Xua430a432006-07-08 13:34:56 -07002763 return ERR_PTR(err);
2764 }
2765
Alexander Duyck802ab552016-04-10 21:45:03 -04002766 /* Only report GSO partial support if it will enable us to
2767 * support segmentation on this frame without needing additional
2768 * work.
2769 */
2770 if (features & NETIF_F_GSO_PARTIAL) {
2771 netdev_features_t partial_features = NETIF_F_GSO_ROBUST;
2772 struct net_device *dev = skb->dev;
2773
2774 partial_features |= dev->features & dev->gso_partial_features;
2775 if (!skb_gso_ok(skb, features | partial_features))
2776 features &= ~NETIF_F_GSO_PARTIAL;
2777 }
2778
Konstantin Khlebnikov9207f9d2016-01-08 15:21:46 +03002779 BUILD_BUG_ON(SKB_SGO_CB_OFFSET +
2780 sizeof(*SKB_GSO_CB(skb)) > sizeof(skb->cb));
2781
Pravin B Shelar68c33162013-02-14 14:02:41 +00002782 SKB_GSO_CB(skb)->mac_offset = skb_headroom(skb);
Eric Dumazet3347c962013-10-19 11:42:56 -07002783 SKB_GSO_CB(skb)->encap_level = 0;
2784
Pravin B Shelar05e8ef42013-02-14 09:44:55 +00002785 skb_reset_mac_header(skb);
2786 skb_reset_mac_len(skb);
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002787
Eric Dumazetb2504a52017-01-31 10:20:32 -08002788 segs = skb_mac_gso_segment(skb, features);
2789
2790 if (unlikely(skb_needs_check(skb, tx_path)))
2791 skb_warn_bad_offload(skb);
2792
2793 return segs;
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002794}
Cong Wang12b00042013-02-05 16:36:38 +00002795EXPORT_SYMBOL(__skb_gso_segment);
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002796
Herbert Xufb286bb2005-11-10 13:01:24 -08002797/* Take action when hardware reception checksum errors are detected. */
2798#ifdef CONFIG_BUG
2799void netdev_rx_csum_fault(struct net_device *dev)
2800{
2801 if (net_ratelimit()) {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00002802 pr_err("%s: hw csum failure\n", dev ? dev->name : "<unknown>");
Herbert Xufb286bb2005-11-10 13:01:24 -08002803 dump_stack();
2804 }
2805}
2806EXPORT_SYMBOL(netdev_rx_csum_fault);
2807#endif
2808
Linus Torvalds1da177e2005-04-16 15:20:36 -07002809/* Actually, we should eliminate this check as soon as we know, that:
2810 * 1. IOMMU is present and allows to map all the memory.
2811 * 2. No high memory really exists on this machine.
2812 */
2813
Florian Westphalc1e756b2014-05-05 15:00:44 +02002814static int illegal_highdma(struct net_device *dev, struct sk_buff *skb)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002815{
Herbert Xu3d3a8532006-06-27 13:33:10 -07002816#ifdef CONFIG_HIGHMEM
Linus Torvalds1da177e2005-04-16 15:20:36 -07002817 int i;
tchardingf4563a72017-02-09 17:56:07 +11002818
FUJITA Tomonori5acbbd42010-03-30 22:35:50 +00002819 if (!(dev->features & NETIF_F_HIGHDMA)) {
Ian Campbellea2ab692011-08-22 23:44:58 +00002820 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
2821 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
tchardingf4563a72017-02-09 17:56:07 +11002822
Ian Campbellea2ab692011-08-22 23:44:58 +00002823 if (PageHighMem(skb_frag_page(frag)))
FUJITA Tomonori5acbbd42010-03-30 22:35:50 +00002824 return 1;
Ian Campbellea2ab692011-08-22 23:44:58 +00002825 }
FUJITA Tomonori5acbbd42010-03-30 22:35:50 +00002826 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002827
FUJITA Tomonori5acbbd42010-03-30 22:35:50 +00002828 if (PCI_DMA_BUS_IS_PHYS) {
2829 struct device *pdev = dev->dev.parent;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002830
Eric Dumazet9092c652010-04-02 13:34:49 -07002831 if (!pdev)
2832 return 0;
FUJITA Tomonori5acbbd42010-03-30 22:35:50 +00002833 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
Ian Campbellea2ab692011-08-22 23:44:58 +00002834 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
2835 dma_addr_t addr = page_to_phys(skb_frag_page(frag));
tchardingf4563a72017-02-09 17:56:07 +11002836
FUJITA Tomonori5acbbd42010-03-30 22:35:50 +00002837 if (!pdev->dma_mask || addr + PAGE_SIZE - 1 > *pdev->dma_mask)
2838 return 1;
2839 }
2840 }
Herbert Xu3d3a8532006-06-27 13:33:10 -07002841#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07002842 return 0;
2843}
Linus Torvalds1da177e2005-04-16 15:20:36 -07002844
Simon Horman3b392dd2014-06-04 08:53:17 +09002845/* If MPLS offload request, verify we are testing hardware MPLS features
2846 * instead of standard features for the netdev.
2847 */
Pravin B Shelard0edc7b2014-12-23 16:20:11 -08002848#if IS_ENABLED(CONFIG_NET_MPLS_GSO)
Simon Horman3b392dd2014-06-04 08:53:17 +09002849static netdev_features_t net_mpls_features(struct sk_buff *skb,
2850 netdev_features_t features,
2851 __be16 type)
2852{
Simon Horman25cd9ba2014-10-06 05:05:13 -07002853 if (eth_p_mpls(type))
Simon Horman3b392dd2014-06-04 08:53:17 +09002854 features &= skb->dev->mpls_features;
2855
2856 return features;
2857}
2858#else
2859static netdev_features_t net_mpls_features(struct sk_buff *skb,
2860 netdev_features_t features,
2861 __be16 type)
2862{
2863 return features;
2864}
2865#endif
2866
Michał Mirosławc8f44af2011-11-15 15:29:55 +00002867static netdev_features_t harmonize_features(struct sk_buff *skb,
Florian Westphalc1e756b2014-05-05 15:00:44 +02002868 netdev_features_t features)
Jesse Grossf01a5232011-01-09 06:23:31 +00002869{
Vlad Yasevich53d64712014-03-27 17:26:18 -04002870 int tmp;
Simon Horman3b392dd2014-06-04 08:53:17 +09002871 __be16 type;
2872
2873 type = skb_network_protocol(skb, &tmp);
2874 features = net_mpls_features(skb, features, type);
Vlad Yasevich53d64712014-03-27 17:26:18 -04002875
Ed Cashinc0d680e2012-09-19 15:49:00 +00002876 if (skb->ip_summed != CHECKSUM_NONE &&
Simon Horman3b392dd2014-06-04 08:53:17 +09002877 !can_checksum_protocol(features, type)) {
Alexander Duyck996e8022016-05-02 09:25:10 -07002878 features &= ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK);
Jesse Grossf01a5232011-01-09 06:23:31 +00002879 }
Eric Dumazet7be2c822017-01-18 12:12:17 -08002880 if (illegal_highdma(skb->dev, skb))
2881 features &= ~NETIF_F_SG;
Jesse Grossf01a5232011-01-09 06:23:31 +00002882
2883 return features;
2884}
2885
Toshiaki Makitae38f3022015-03-27 14:31:13 +09002886netdev_features_t passthru_features_check(struct sk_buff *skb,
2887 struct net_device *dev,
2888 netdev_features_t features)
2889{
2890 return features;
2891}
2892EXPORT_SYMBOL(passthru_features_check);
2893
Toshiaki Makita8cb65d02015-03-27 14:31:12 +09002894static netdev_features_t dflt_features_check(const struct sk_buff *skb,
2895 struct net_device *dev,
2896 netdev_features_t features)
2897{
2898 return vlan_features_check(skb, features);
2899}
2900
Alexander Duyckcbc53e02016-04-10 21:44:51 -04002901static netdev_features_t gso_features_check(const struct sk_buff *skb,
2902 struct net_device *dev,
2903 netdev_features_t features)
2904{
2905 u16 gso_segs = skb_shinfo(skb)->gso_segs;
2906
2907 if (gso_segs > dev->gso_max_segs)
2908 return features & ~NETIF_F_GSO_MASK;
2909
Alexander Duyck802ab552016-04-10 21:45:03 -04002910 /* Support for GSO partial features requires software
2911 * intervention before we can actually process the packets
2912 * so we need to strip support for any partial features now
2913 * and we can pull them back in after we have partially
2914 * segmented the frame.
2915 */
2916 if (!(skb_shinfo(skb)->gso_type & SKB_GSO_PARTIAL))
2917 features &= ~dev->gso_partial_features;
2918
2919 /* Make sure to clear the IPv4 ID mangling feature if the
2920 * IPv4 header has the potential to be fragmented.
Alexander Duyckcbc53e02016-04-10 21:44:51 -04002921 */
2922 if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV4) {
2923 struct iphdr *iph = skb->encapsulation ?
2924 inner_ip_hdr(skb) : ip_hdr(skb);
2925
2926 if (!(iph->frag_off & htons(IP_DF)))
2927 features &= ~NETIF_F_TSO_MANGLEID;
2928 }
2929
2930 return features;
2931}
2932
Florian Westphalc1e756b2014-05-05 15:00:44 +02002933netdev_features_t netif_skb_features(struct sk_buff *skb)
Jesse Gross58e998c2010-10-29 12:14:55 +00002934{
Jesse Gross5f352272014-12-23 22:37:26 -08002935 struct net_device *dev = skb->dev;
Eric Dumazetfcbeb972014-10-05 10:11:27 -07002936 netdev_features_t features = dev->features;
Jesse Gross58e998c2010-10-29 12:14:55 +00002937
Alexander Duyckcbc53e02016-04-10 21:44:51 -04002938 if (skb_is_gso(skb))
2939 features = gso_features_check(skb, dev, features);
Ben Hutchings30b678d2012-07-30 15:57:00 +00002940
Jesse Gross5f352272014-12-23 22:37:26 -08002941 /* If encapsulation offload request, verify we are testing
2942 * hardware encapsulation features instead of standard
2943 * features for the netdev
2944 */
2945 if (skb->encapsulation)
2946 features &= dev->hw_enc_features;
2947
Toshiaki Makitaf5a7fb82015-03-27 14:31:11 +09002948 if (skb_vlan_tagged(skb))
2949 features = netdev_intersect_features(features,
2950 dev->vlan_features |
2951 NETIF_F_HW_VLAN_CTAG_TX |
2952 NETIF_F_HW_VLAN_STAG_TX);
Jesse Gross58e998c2010-10-29 12:14:55 +00002953
Jesse Gross5f352272014-12-23 22:37:26 -08002954 if (dev->netdev_ops->ndo_features_check)
2955 features &= dev->netdev_ops->ndo_features_check(skb, dev,
2956 features);
Toshiaki Makita8cb65d02015-03-27 14:31:12 +09002957 else
2958 features &= dflt_features_check(skb, dev, features);
Jesse Gross5f352272014-12-23 22:37:26 -08002959
Florian Westphalc1e756b2014-05-05 15:00:44 +02002960 return harmonize_features(skb, features);
Jesse Gross58e998c2010-10-29 12:14:55 +00002961}
Florian Westphalc1e756b2014-05-05 15:00:44 +02002962EXPORT_SYMBOL(netif_skb_features);
Jesse Gross58e998c2010-10-29 12:14:55 +00002963
David S. Miller2ea25512014-08-29 21:10:01 -07002964static int xmit_one(struct sk_buff *skb, struct net_device *dev,
David S. Miller95f6b3d2014-08-29 21:57:30 -07002965 struct netdev_queue *txq, bool more)
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002966{
David S. Miller2ea25512014-08-29 21:10:01 -07002967 unsigned int len;
2968 int rc;
Stephen Hemminger00829822008-11-20 20:14:53 -08002969
Salam Noureddine7866a622015-01-27 11:35:48 -08002970 if (!list_empty(&ptype_all) || !list_empty(&dev->ptype_all))
David S. Miller2ea25512014-08-29 21:10:01 -07002971 dev_queue_xmit_nit(skb, dev);
Jesse Grossfc741212011-01-09 06:23:32 +00002972
David S. Miller2ea25512014-08-29 21:10:01 -07002973 len = skb->len;
2974 trace_net_dev_start_xmit(skb, dev);
David S. Miller95f6b3d2014-08-29 21:57:30 -07002975 rc = netdev_start_xmit(skb, dev, txq, more);
David S. Miller2ea25512014-08-29 21:10:01 -07002976 trace_net_dev_xmit(skb, rc, dev, len);
Eric Dumazetadf30902009-06-02 05:19:30 +00002977
Patrick McHardy572a9d72009-11-10 06:14:14 +00002978 return rc;
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002979}
David S. Miller2ea25512014-08-29 21:10:01 -07002980
David S. Miller8dcda222014-09-01 15:06:40 -07002981struct sk_buff *dev_hard_start_xmit(struct sk_buff *first, struct net_device *dev,
2982 struct netdev_queue *txq, int *ret)
David S. Miller7f2e8702014-08-29 21:19:14 -07002983{
2984 struct sk_buff *skb = first;
2985 int rc = NETDEV_TX_OK;
2986
2987 while (skb) {
2988 struct sk_buff *next = skb->next;
2989
2990 skb->next = NULL;
David S. Miller95f6b3d2014-08-29 21:57:30 -07002991 rc = xmit_one(skb, dev, txq, next != NULL);
David S. Miller7f2e8702014-08-29 21:19:14 -07002992 if (unlikely(!dev_xmit_complete(rc))) {
2993 skb->next = next;
2994 goto out;
2995 }
2996
2997 skb = next;
2998 if (netif_xmit_stopped(txq) && skb) {
2999 rc = NETDEV_TX_BUSY;
3000 break;
3001 }
3002 }
3003
3004out:
3005 *ret = rc;
3006 return skb;
3007}
3008
Eric Dumazet1ff0dc92014-10-06 11:26:27 -07003009static struct sk_buff *validate_xmit_vlan(struct sk_buff *skb,
3010 netdev_features_t features)
David S. Millereae3f882014-08-30 15:17:13 -07003011{
Jiri Pirkodf8a39d2015-01-13 17:13:44 +01003012 if (skb_vlan_tag_present(skb) &&
Jiri Pirko59682502014-11-19 14:04:59 +01003013 !vlan_hw_offload_capable(features, skb->vlan_proto))
3014 skb = __vlan_hwaccel_push_inside(skb);
David S. Millereae3f882014-08-30 15:17:13 -07003015 return skb;
3016}
3017
Davide Caratti43c26a12017-05-18 15:44:41 +02003018int skb_csum_hwoffload_help(struct sk_buff *skb,
3019 const netdev_features_t features)
3020{
3021 if (unlikely(skb->csum_not_inet))
3022 return !!(features & NETIF_F_SCTP_CRC) ? 0 :
3023 skb_crc32c_csum_help(skb);
3024
3025 return !!(features & NETIF_F_CSUM_MASK) ? 0 : skb_checksum_help(skb);
3026}
3027EXPORT_SYMBOL(skb_csum_hwoffload_help);
3028
Eric Dumazet55a93b32014-10-03 15:31:07 -07003029static struct sk_buff *validate_xmit_skb(struct sk_buff *skb, struct net_device *dev)
David S. Millereae3f882014-08-30 15:17:13 -07003030{
3031 netdev_features_t features;
3032
David S. Millereae3f882014-08-30 15:17:13 -07003033 features = netif_skb_features(skb);
3034 skb = validate_xmit_vlan(skb, features);
3035 if (unlikely(!skb))
3036 goto out_null;
3037
Johannes Berg8b86a612015-04-17 15:45:04 +02003038 if (netif_needs_gso(skb, features)) {
David S. Millerce937182014-08-30 19:22:20 -07003039 struct sk_buff *segs;
3040
3041 segs = skb_gso_segment(skb, features);
Jason Wangcecda692014-09-19 16:04:38 +08003042 if (IS_ERR(segs)) {
Jason Wangaf6dabc2014-12-19 11:09:13 +08003043 goto out_kfree_skb;
Jason Wangcecda692014-09-19 16:04:38 +08003044 } else if (segs) {
3045 consume_skb(skb);
3046 skb = segs;
3047 }
David S. Millereae3f882014-08-30 15:17:13 -07003048 } else {
3049 if (skb_needs_linearize(skb, features) &&
3050 __skb_linearize(skb))
3051 goto out_kfree_skb;
3052
Steffen Klassertf6e27112017-04-14 10:07:28 +02003053 if (validate_xmit_xfrm(skb, features))
3054 goto out_kfree_skb;
3055
David S. Millereae3f882014-08-30 15:17:13 -07003056 /* If packet is not checksummed and device does not
3057 * support checksumming for this protocol, complete
3058 * checksumming here.
3059 */
3060 if (skb->ip_summed == CHECKSUM_PARTIAL) {
3061 if (skb->encapsulation)
3062 skb_set_inner_transport_header(skb,
3063 skb_checksum_start_offset(skb));
3064 else
3065 skb_set_transport_header(skb,
3066 skb_checksum_start_offset(skb));
Davide Caratti43c26a12017-05-18 15:44:41 +02003067 if (skb_csum_hwoffload_help(skb, features))
David S. Millereae3f882014-08-30 15:17:13 -07003068 goto out_kfree_skb;
3069 }
3070 }
3071
3072 return skb;
3073
3074out_kfree_skb:
3075 kfree_skb(skb);
3076out_null:
Eric Dumazetd21fd632016-04-12 21:50:07 -07003077 atomic_long_inc(&dev->tx_dropped);
David S. Millereae3f882014-08-30 15:17:13 -07003078 return NULL;
3079}
3080
Eric Dumazet55a93b32014-10-03 15:31:07 -07003081struct sk_buff *validate_xmit_skb_list(struct sk_buff *skb, struct net_device *dev)
3082{
3083 struct sk_buff *next, *head = NULL, *tail;
3084
Eric Dumazetbec3cfd2014-10-03 20:59:19 -07003085 for (; skb != NULL; skb = next) {
Eric Dumazet55a93b32014-10-03 15:31:07 -07003086 next = skb->next;
3087 skb->next = NULL;
Eric Dumazet55a93b32014-10-03 15:31:07 -07003088
Eric Dumazetbec3cfd2014-10-03 20:59:19 -07003089 /* in case skb wont be segmented, point to itself */
3090 skb->prev = skb;
3091
3092 skb = validate_xmit_skb(skb, dev);
3093 if (!skb)
3094 continue;
3095
3096 if (!head)
3097 head = skb;
3098 else
3099 tail->next = skb;
3100 /* If skb was segmented, skb->prev points to
3101 * the last segment. If not, it still contains skb.
3102 */
3103 tail = skb->prev;
Eric Dumazet55a93b32014-10-03 15:31:07 -07003104 }
3105 return head;
3106}
Willem de Bruijn104ba782016-10-26 11:23:07 -04003107EXPORT_SYMBOL_GPL(validate_xmit_skb_list);
Herbert Xuf6a78bf2006-06-22 02:57:17 -07003108
Eric Dumazet1def9232013-01-10 12:36:42 +00003109static void qdisc_pkt_len_init(struct sk_buff *skb)
3110{
3111 const struct skb_shared_info *shinfo = skb_shinfo(skb);
3112
3113 qdisc_skb_cb(skb)->pkt_len = skb->len;
3114
3115 /* To get more precise estimation of bytes sent on wire,
3116 * we add to pkt_len the headers size of all segments
3117 */
3118 if (shinfo->gso_size) {
Eric Dumazet757b8b12013-01-15 21:14:21 -08003119 unsigned int hdr_len;
Jason Wang15e5a032013-03-25 20:19:59 +00003120 u16 gso_segs = shinfo->gso_segs;
Eric Dumazet1def9232013-01-10 12:36:42 +00003121
Eric Dumazet757b8b12013-01-15 21:14:21 -08003122 /* mac layer + network layer */
3123 hdr_len = skb_transport_header(skb) - skb_mac_header(skb);
3124
3125 /* + transport layer */
Eric Dumazet1def9232013-01-10 12:36:42 +00003126 if (likely(shinfo->gso_type & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6)))
3127 hdr_len += tcp_hdrlen(skb);
3128 else
3129 hdr_len += sizeof(struct udphdr);
Jason Wang15e5a032013-03-25 20:19:59 +00003130
3131 if (shinfo->gso_type & SKB_GSO_DODGY)
3132 gso_segs = DIV_ROUND_UP(skb->len - hdr_len,
3133 shinfo->gso_size);
3134
3135 qdisc_skb_cb(skb)->pkt_len += (gso_segs - 1) * hdr_len;
Eric Dumazet1def9232013-01-10 12:36:42 +00003136 }
3137}
3138
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00003139static inline int __dev_xmit_skb(struct sk_buff *skb, struct Qdisc *q,
3140 struct net_device *dev,
3141 struct netdev_queue *txq)
3142{
3143 spinlock_t *root_lock = qdisc_lock(q);
Eric Dumazet520ac302016-06-21 23:16:49 -07003144 struct sk_buff *to_free = NULL;
Eric Dumazeta2da5702011-01-20 03:48:19 +00003145 bool contended;
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00003146 int rc;
3147
Eric Dumazeta2da5702011-01-20 03:48:19 +00003148 qdisc_calculate_pkt_len(skb, q);
Eric Dumazet79640a42010-06-02 05:09:29 -07003149 /*
3150 * Heuristic to force contended enqueues to serialize on a
3151 * separate lock before trying to get qdisc main lock.
Eric Dumazetf9eb8ae2016-06-06 09:37:15 -07003152 * This permits qdisc->running owner to get the lock more
Ying Xue9bf2b8c2014-06-26 15:56:31 +08003153 * often and dequeue packets faster.
Eric Dumazet79640a42010-06-02 05:09:29 -07003154 */
Eric Dumazeta2da5702011-01-20 03:48:19 +00003155 contended = qdisc_is_running(q);
Eric Dumazet79640a42010-06-02 05:09:29 -07003156 if (unlikely(contended))
3157 spin_lock(&q->busylock);
3158
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00003159 spin_lock(root_lock);
3160 if (unlikely(test_bit(__QDISC_STATE_DEACTIVATED, &q->state))) {
Eric Dumazet520ac302016-06-21 23:16:49 -07003161 __qdisc_drop(skb, &to_free);
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00003162 rc = NET_XMIT_DROP;
3163 } else if ((q->flags & TCQ_F_CAN_BYPASS) && !qdisc_qlen(q) &&
Eric Dumazetbc135b22010-06-02 03:23:51 -07003164 qdisc_run_begin(q)) {
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00003165 /*
3166 * This is a work-conserving queue; there are no old skbs
3167 * waiting to be sent out; and the qdisc is not running -
3168 * xmit the skb directly.
3169 */
Eric Dumazetbfe0d022011-01-09 08:30:54 +00003170
Eric Dumazetbfe0d022011-01-09 08:30:54 +00003171 qdisc_bstats_update(q, skb);
3172
Eric Dumazet55a93b32014-10-03 15:31:07 -07003173 if (sch_direct_xmit(skb, q, dev, txq, root_lock, true)) {
Eric Dumazet79640a42010-06-02 05:09:29 -07003174 if (unlikely(contended)) {
3175 spin_unlock(&q->busylock);
3176 contended = false;
3177 }
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00003178 __qdisc_run(q);
Eric Dumazet79640a42010-06-02 05:09:29 -07003179 } else
Eric Dumazetbc135b22010-06-02 03:23:51 -07003180 qdisc_run_end(q);
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00003181
3182 rc = NET_XMIT_SUCCESS;
3183 } else {
Eric Dumazet520ac302016-06-21 23:16:49 -07003184 rc = q->enqueue(skb, q, &to_free) & NET_XMIT_MASK;
Eric Dumazet79640a42010-06-02 05:09:29 -07003185 if (qdisc_run_begin(q)) {
3186 if (unlikely(contended)) {
3187 spin_unlock(&q->busylock);
3188 contended = false;
3189 }
3190 __qdisc_run(q);
3191 }
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00003192 }
3193 spin_unlock(root_lock);
Eric Dumazet520ac302016-06-21 23:16:49 -07003194 if (unlikely(to_free))
3195 kfree_skb_list(to_free);
Eric Dumazet79640a42010-06-02 05:09:29 -07003196 if (unlikely(contended))
3197 spin_unlock(&q->busylock);
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00003198 return rc;
3199}
3200
Daniel Borkmann86f85152013-12-29 17:27:11 +01003201#if IS_ENABLED(CONFIG_CGROUP_NET_PRIO)
Neil Horman5bc14212011-11-22 05:10:51 +00003202static void skb_update_prio(struct sk_buff *skb)
3203{
Igor Maravic6977a792011-11-25 07:44:54 +00003204 struct netprio_map *map = rcu_dereference_bh(skb->dev->priomap);
Neil Horman5bc14212011-11-22 05:10:51 +00003205
Eric Dumazet91c68ce2012-07-08 21:45:10 +00003206 if (!skb->priority && skb->sk && map) {
Tejun Heo2a56a1f2015-12-07 17:38:52 -05003207 unsigned int prioidx =
3208 sock_cgroup_prioidx(&skb->sk->sk_cgrp_data);
Eric Dumazet91c68ce2012-07-08 21:45:10 +00003209
3210 if (prioidx < map->priomap_len)
3211 skb->priority = map->priomap[prioidx];
3212 }
Neil Horman5bc14212011-11-22 05:10:51 +00003213}
3214#else
3215#define skb_update_prio(skb)
3216#endif
3217
hannes@stressinduktion.orgf60e5992015-04-01 17:07:44 +02003218DEFINE_PER_CPU(int, xmit_recursion);
3219EXPORT_SYMBOL(xmit_recursion);
3220
Dave Jonesd29f7492008-07-22 14:09:06 -07003221/**
Michel Machado95603e22012-06-12 10:16:35 +00003222 * dev_loopback_xmit - loop back @skb
Eric W. Biederman0c4b51f2015-09-15 20:04:18 -05003223 * @net: network namespace this loopback is happening in
3224 * @sk: sk needed to be a netfilter okfn
Michel Machado95603e22012-06-12 10:16:35 +00003225 * @skb: buffer to transmit
3226 */
Eric W. Biederman0c4b51f2015-09-15 20:04:18 -05003227int dev_loopback_xmit(struct net *net, struct sock *sk, struct sk_buff *skb)
Michel Machado95603e22012-06-12 10:16:35 +00003228{
3229 skb_reset_mac_header(skb);
3230 __skb_pull(skb, skb_network_offset(skb));
3231 skb->pkt_type = PACKET_LOOPBACK;
3232 skb->ip_summed = CHECKSUM_UNNECESSARY;
3233 WARN_ON(!skb_dst(skb));
3234 skb_dst_force(skb);
3235 netif_rx_ni(skb);
3236 return 0;
3237}
3238EXPORT_SYMBOL(dev_loopback_xmit);
3239
Daniel Borkmann1f211a12016-01-07 22:29:47 +01003240#ifdef CONFIG_NET_EGRESS
3241static struct sk_buff *
3242sch_handle_egress(struct sk_buff *skb, int *ret, struct net_device *dev)
3243{
3244 struct tcf_proto *cl = rcu_dereference_bh(dev->egress_cl_list);
3245 struct tcf_result cl_res;
3246
3247 if (!cl)
3248 return skb;
3249
Willem de Bruijn8dc07fd2017-01-07 17:06:37 -05003250 /* qdisc_skb_cb(skb)->pkt_len was already set by the caller. */
Daniel Borkmann1f211a12016-01-07 22:29:47 +01003251 qdisc_bstats_cpu_update(cl->q, skb);
3252
Jiri Pirko87d83092017-05-17 11:07:54 +02003253 switch (tcf_classify(skb, cl, &cl_res, false)) {
Daniel Borkmann1f211a12016-01-07 22:29:47 +01003254 case TC_ACT_OK:
3255 case TC_ACT_RECLASSIFY:
3256 skb->tc_index = TC_H_MIN(cl_res.classid);
3257 break;
3258 case TC_ACT_SHOT:
3259 qdisc_qstats_cpu_drop(cl->q);
3260 *ret = NET_XMIT_DROP;
Daniel Borkmann7e2c3ae2016-05-15 23:28:29 +02003261 kfree_skb(skb);
3262 return NULL;
Daniel Borkmann1f211a12016-01-07 22:29:47 +01003263 case TC_ACT_STOLEN:
3264 case TC_ACT_QUEUED:
Jiri Pirkoe25ea212017-06-06 14:12:02 +02003265 case TC_ACT_TRAP:
Daniel Borkmann1f211a12016-01-07 22:29:47 +01003266 *ret = NET_XMIT_SUCCESS;
Daniel Borkmann7e2c3ae2016-05-15 23:28:29 +02003267 consume_skb(skb);
Daniel Borkmann1f211a12016-01-07 22:29:47 +01003268 return NULL;
3269 case TC_ACT_REDIRECT:
3270 /* No need to push/pop skb's mac_header here on egress! */
3271 skb_do_redirect(skb);
3272 *ret = NET_XMIT_SUCCESS;
3273 return NULL;
3274 default:
3275 break;
3276 }
3277
3278 return skb;
3279}
3280#endif /* CONFIG_NET_EGRESS */
3281
Jiri Pirko638b2a62015-05-12 14:56:13 +02003282static inline int get_xps_queue(struct net_device *dev, struct sk_buff *skb)
3283{
3284#ifdef CONFIG_XPS
3285 struct xps_dev_maps *dev_maps;
3286 struct xps_map *map;
3287 int queue_index = -1;
3288
3289 rcu_read_lock();
3290 dev_maps = rcu_dereference(dev->xps_maps);
3291 if (dev_maps) {
Alexander Duyck184c4492016-10-28 11:50:13 -04003292 unsigned int tci = skb->sender_cpu - 1;
3293
3294 if (dev->num_tc) {
3295 tci *= dev->num_tc;
3296 tci += netdev_get_prio_tc_map(dev, skb->priority);
3297 }
3298
3299 map = rcu_dereference(dev_maps->cpu_map[tci]);
Jiri Pirko638b2a62015-05-12 14:56:13 +02003300 if (map) {
3301 if (map->len == 1)
3302 queue_index = map->queues[0];
3303 else
3304 queue_index = map->queues[reciprocal_scale(skb_get_hash(skb),
3305 map->len)];
3306 if (unlikely(queue_index >= dev->real_num_tx_queues))
3307 queue_index = -1;
3308 }
3309 }
3310 rcu_read_unlock();
3311
3312 return queue_index;
3313#else
3314 return -1;
3315#endif
3316}
3317
3318static u16 __netdev_pick_tx(struct net_device *dev, struct sk_buff *skb)
3319{
3320 struct sock *sk = skb->sk;
3321 int queue_index = sk_tx_queue_get(sk);
3322
3323 if (queue_index < 0 || skb->ooo_okay ||
3324 queue_index >= dev->real_num_tx_queues) {
3325 int new_index = get_xps_queue(dev, skb);
tchardingf4563a72017-02-09 17:56:07 +11003326
Jiri Pirko638b2a62015-05-12 14:56:13 +02003327 if (new_index < 0)
3328 new_index = skb_tx_hash(dev, skb);
3329
3330 if (queue_index != new_index && sk &&
Eric Dumazet004a5d02015-10-04 21:08:10 -07003331 sk_fullsock(sk) &&
Jiri Pirko638b2a62015-05-12 14:56:13 +02003332 rcu_access_pointer(sk->sk_dst_cache))
3333 sk_tx_queue_set(sk, new_index);
3334
3335 queue_index = new_index;
3336 }
3337
3338 return queue_index;
3339}
3340
3341struct netdev_queue *netdev_pick_tx(struct net_device *dev,
3342 struct sk_buff *skb,
3343 void *accel_priv)
3344{
3345 int queue_index = 0;
3346
3347#ifdef CONFIG_XPS
Eric Dumazet52bd2d62015-11-18 06:30:50 -08003348 u32 sender_cpu = skb->sender_cpu - 1;
3349
3350 if (sender_cpu >= (u32)NR_CPUS)
Jiri Pirko638b2a62015-05-12 14:56:13 +02003351 skb->sender_cpu = raw_smp_processor_id() + 1;
3352#endif
3353
3354 if (dev->real_num_tx_queues != 1) {
3355 const struct net_device_ops *ops = dev->netdev_ops;
tchardingf4563a72017-02-09 17:56:07 +11003356
Jiri Pirko638b2a62015-05-12 14:56:13 +02003357 if (ops->ndo_select_queue)
3358 queue_index = ops->ndo_select_queue(dev, skb, accel_priv,
3359 __netdev_pick_tx);
3360 else
3361 queue_index = __netdev_pick_tx(dev, skb);
3362
3363 if (!accel_priv)
3364 queue_index = netdev_cap_txqueue(dev, queue_index);
3365 }
3366
3367 skb_set_queue_mapping(skb, queue_index);
3368 return netdev_get_tx_queue(dev, queue_index);
3369}
3370
Michel Machado95603e22012-06-12 10:16:35 +00003371/**
Jason Wang9d08dd32014-01-20 11:25:13 +08003372 * __dev_queue_xmit - transmit a buffer
Dave Jonesd29f7492008-07-22 14:09:06 -07003373 * @skb: buffer to transmit
Jason Wang9d08dd32014-01-20 11:25:13 +08003374 * @accel_priv: private data used for L2 forwarding offload
Dave Jonesd29f7492008-07-22 14:09:06 -07003375 *
3376 * Queue a buffer for transmission to a network device. The caller must
3377 * have set the device and priority and built the buffer before calling
3378 * this function. The function can be called from an interrupt.
3379 *
3380 * A negative errno code is returned on a failure. A success does not
3381 * guarantee the frame will be transmitted as it may be dropped due
3382 * to congestion or traffic shaping.
3383 *
3384 * -----------------------------------------------------------------------------------
3385 * I notice this method can also return errors from the queue disciplines,
3386 * including NET_XMIT_DROP, which is a positive value. So, errors can also
3387 * be positive.
3388 *
3389 * Regardless of the return value, the skb is consumed, so it is currently
3390 * difficult to retry a send to this method. (You can bump the ref count
3391 * before sending to hold a reference for retry if you are careful.)
3392 *
3393 * When calling this method, interrupts MUST be enabled. This is because
3394 * the BH enable code must have IRQs enabled so that it will not deadlock.
3395 * --BLG
3396 */
Rashika Kheria0a59f3a2014-02-09 20:26:25 +05303397static int __dev_queue_xmit(struct sk_buff *skb, void *accel_priv)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003398{
3399 struct net_device *dev = skb->dev;
David S. Millerdc2b4842008-07-08 17:18:23 -07003400 struct netdev_queue *txq;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003401 struct Qdisc *q;
3402 int rc = -ENOMEM;
3403
Eric Dumazet6d1ccff2013-02-05 20:22:20 +00003404 skb_reset_mac_header(skb);
3405
Willem de Bruijne7fd2882014-08-04 22:11:48 -04003406 if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_SCHED_TSTAMP))
3407 __skb_tstamp_tx(skb, NULL, skb->sk, SCM_TSTAMP_SCHED);
3408
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09003409 /* Disable soft irqs for various locks below. Also
3410 * stops preemption for RCU.
Linus Torvalds1da177e2005-04-16 15:20:36 -07003411 */
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09003412 rcu_read_lock_bh();
Linus Torvalds1da177e2005-04-16 15:20:36 -07003413
Neil Horman5bc14212011-11-22 05:10:51 +00003414 skb_update_prio(skb);
3415
Daniel Borkmann1f211a12016-01-07 22:29:47 +01003416 qdisc_pkt_len_init(skb);
3417#ifdef CONFIG_NET_CLS_ACT
Willem de Bruijn8dc07fd2017-01-07 17:06:37 -05003418 skb->tc_at_ingress = 0;
Daniel Borkmann1f211a12016-01-07 22:29:47 +01003419# ifdef CONFIG_NET_EGRESS
3420 if (static_key_false(&egress_needed)) {
3421 skb = sch_handle_egress(skb, &rc, dev);
3422 if (!skb)
3423 goto out;
3424 }
3425# endif
3426#endif
Eric Dumazet02875872014-10-05 18:38:35 -07003427 /* If device/qdisc don't need skb->dst, release it right now while
3428 * its hot in this cpu cache.
3429 */
3430 if (dev->priv_flags & IFF_XMIT_DST_RELEASE)
3431 skb_dst_drop(skb);
3432 else
3433 skb_dst_force(skb);
3434
Jason Wangf663dd92014-01-10 16:18:26 +08003435 txq = netdev_pick_tx(dev, skb, accel_priv);
Paul E. McKenneya898def2010-02-22 17:04:49 -08003436 q = rcu_dereference_bh(txq->qdisc);
David S. Miller37437bb2008-07-16 02:15:04 -07003437
Koki Sanagicf66ba52010-08-23 18:45:02 +09003438 trace_net_dev_queue(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003439 if (q->enqueue) {
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00003440 rc = __dev_xmit_skb(skb, q, dev, txq);
David S. Miller37437bb2008-07-16 02:15:04 -07003441 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003442 }
3443
3444 /* The device has no queue. Common case for software devices:
tchardingeb13da12017-02-09 17:56:06 +11003445 * loopback, all the sorts of tunnels...
Linus Torvalds1da177e2005-04-16 15:20:36 -07003446
tchardingeb13da12017-02-09 17:56:06 +11003447 * Really, it is unlikely that netif_tx_lock protection is necessary
3448 * here. (f.e. loopback and IP tunnels are clean ignoring statistics
3449 * counters.)
3450 * However, it is possible, that they rely on protection
3451 * made by us here.
Linus Torvalds1da177e2005-04-16 15:20:36 -07003452
tchardingeb13da12017-02-09 17:56:06 +11003453 * Check this and shot the lock. It is not prone from deadlocks.
3454 *Either shot noqueue qdisc, it is even simpler 8)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003455 */
3456 if (dev->flags & IFF_UP) {
3457 int cpu = smp_processor_id(); /* ok because BHs are off */
3458
David S. Millerc773e842008-07-08 23:13:53 -07003459 if (txq->xmit_lock_owner != cpu) {
Daniel Borkmanna70b5062016-06-10 21:19:06 +02003460 if (unlikely(__this_cpu_read(xmit_recursion) >
3461 XMIT_RECURSION_LIMIT))
Eric Dumazet745e20f2010-09-29 13:23:09 -07003462 goto recursion_alert;
3463
Jesper Dangaard Brouer1f595332014-09-03 17:56:09 +02003464 skb = validate_xmit_skb(skb, dev);
3465 if (!skb)
Eric Dumazetd21fd632016-04-12 21:50:07 -07003466 goto out;
Jesper Dangaard Brouer1f595332014-09-03 17:56:09 +02003467
David S. Millerc773e842008-07-08 23:13:53 -07003468 HARD_TX_LOCK(dev, txq, cpu);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003469
Tom Herbert734664982011-11-28 16:32:44 +00003470 if (!netif_xmit_stopped(txq)) {
Eric Dumazet745e20f2010-09-29 13:23:09 -07003471 __this_cpu_inc(xmit_recursion);
David S. Millerce937182014-08-30 19:22:20 -07003472 skb = dev_hard_start_xmit(skb, dev, txq, &rc);
Eric Dumazet745e20f2010-09-29 13:23:09 -07003473 __this_cpu_dec(xmit_recursion);
Patrick McHardy572a9d72009-11-10 06:14:14 +00003474 if (dev_xmit_complete(rc)) {
David S. Millerc773e842008-07-08 23:13:53 -07003475 HARD_TX_UNLOCK(dev, txq);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003476 goto out;
3477 }
3478 }
David S. Millerc773e842008-07-08 23:13:53 -07003479 HARD_TX_UNLOCK(dev, txq);
Joe Perchese87cc472012-05-13 21:56:26 +00003480 net_crit_ratelimited("Virtual device %s asks to queue packet!\n",
3481 dev->name);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003482 } else {
3483 /* Recursion is detected! It is possible,
Eric Dumazet745e20f2010-09-29 13:23:09 -07003484 * unfortunately
3485 */
3486recursion_alert:
Joe Perchese87cc472012-05-13 21:56:26 +00003487 net_crit_ratelimited("Dead loop on virtual device %s, fix it urgently!\n",
3488 dev->name);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003489 }
3490 }
3491
3492 rc = -ENETDOWN;
Herbert Xud4828d82006-06-22 02:28:18 -07003493 rcu_read_unlock_bh();
Linus Torvalds1da177e2005-04-16 15:20:36 -07003494
Eric Dumazet015f0682014-03-27 08:45:56 -07003495 atomic_long_inc(&dev->tx_dropped);
Jesper Dangaard Brouer1f595332014-09-03 17:56:09 +02003496 kfree_skb_list(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003497 return rc;
3498out:
Herbert Xud4828d82006-06-22 02:28:18 -07003499 rcu_read_unlock_bh();
Linus Torvalds1da177e2005-04-16 15:20:36 -07003500 return rc;
3501}
Jason Wangf663dd92014-01-10 16:18:26 +08003502
Eric W. Biederman2b4aa3c2015-09-15 20:04:07 -05003503int dev_queue_xmit(struct sk_buff *skb)
Jason Wangf663dd92014-01-10 16:18:26 +08003504{
3505 return __dev_queue_xmit(skb, NULL);
3506}
Eric W. Biederman2b4aa3c2015-09-15 20:04:07 -05003507EXPORT_SYMBOL(dev_queue_xmit);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003508
Jason Wangf663dd92014-01-10 16:18:26 +08003509int dev_queue_xmit_accel(struct sk_buff *skb, void *accel_priv)
3510{
3511 return __dev_queue_xmit(skb, accel_priv);
3512}
3513EXPORT_SYMBOL(dev_queue_xmit_accel);
3514
Linus Torvalds1da177e2005-04-16 15:20:36 -07003515
tchardingeb13da12017-02-09 17:56:06 +11003516/*************************************************************************
3517 * Receiver routines
3518 *************************************************************************/
Linus Torvalds1da177e2005-04-16 15:20:36 -07003519
Stephen Hemminger6b2bedc2007-03-12 14:33:50 -07003520int netdev_max_backlog __read_mostly = 1000;
Eric Dumazetc9e6bc62012-09-27 19:29:05 +00003521EXPORT_SYMBOL(netdev_max_backlog);
3522
Eric Dumazet3b098e22010-05-15 23:57:10 -07003523int netdev_tstamp_prequeue __read_mostly = 1;
Stephen Hemminger6b2bedc2007-03-12 14:33:50 -07003524int netdev_budget __read_mostly = 300;
Matthew Whitehead7acf8a12017-04-19 12:37:10 -04003525unsigned int __read_mostly netdev_budget_usecs = 2000;
Matthias Tafelmeier3d48b532016-12-29 21:37:21 +01003526int weight_p __read_mostly = 64; /* old backlog weight */
3527int dev_weight_rx_bias __read_mostly = 1; /* bias for backlog weight */
3528int dev_weight_tx_bias __read_mostly = 1; /* bias for output_queue quota */
3529int dev_rx_weight __read_mostly = 64;
3530int dev_tx_weight __read_mostly = 64;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003531
Eric Dumazeteecfd7c2010-05-06 22:07:48 -07003532/* Called with irq disabled */
3533static inline void ____napi_schedule(struct softnet_data *sd,
3534 struct napi_struct *napi)
3535{
3536 list_add_tail(&napi->poll_list, &sd->poll_list);
3537 __raise_softirq_irqoff(NET_RX_SOFTIRQ);
3538}
3539
Eric Dumazetdf334542010-03-24 19:13:54 +00003540#ifdef CONFIG_RPS
Tom Herbertfec5e652010-04-16 16:01:27 -07003541
3542/* One global table that all flow-based protocols share. */
Eric Dumazet6e3f7fa2010-10-25 03:02:02 +00003543struct rps_sock_flow_table __rcu *rps_sock_flow_table __read_mostly;
Tom Herbertfec5e652010-04-16 16:01:27 -07003544EXPORT_SYMBOL(rps_sock_flow_table);
Eric Dumazet567e4b72015-02-06 12:59:01 -08003545u32 rps_cpu_mask __read_mostly;
3546EXPORT_SYMBOL(rps_cpu_mask);
Tom Herbertfec5e652010-04-16 16:01:27 -07003547
Ingo Molnarc5905af2012-02-24 08:31:31 +01003548struct static_key rps_needed __read_mostly;
Jason Wang3df97ba2016-04-25 23:13:42 -04003549EXPORT_SYMBOL(rps_needed);
Eric Dumazet13bfff22016-12-07 08:29:10 -08003550struct static_key rfs_needed __read_mostly;
3551EXPORT_SYMBOL(rfs_needed);
Eric Dumazetadc93002011-11-17 03:13:26 +00003552
Ben Hutchingsc4454772011-01-19 11:03:53 +00003553static struct rps_dev_flow *
3554set_rps_cpu(struct net_device *dev, struct sk_buff *skb,
3555 struct rps_dev_flow *rflow, u16 next_cpu)
3556{
Eric Dumazeta31196b2015-04-25 09:35:24 -07003557 if (next_cpu < nr_cpu_ids) {
Ben Hutchingsc4454772011-01-19 11:03:53 +00003558#ifdef CONFIG_RFS_ACCEL
3559 struct netdev_rx_queue *rxqueue;
3560 struct rps_dev_flow_table *flow_table;
3561 struct rps_dev_flow *old_rflow;
3562 u32 flow_id;
3563 u16 rxq_index;
3564 int rc;
3565
3566 /* Should we steer this flow to a different hardware queue? */
Ben Hutchings69a19ee2011-02-15 20:32:04 +00003567 if (!skb_rx_queue_recorded(skb) || !dev->rx_cpu_rmap ||
3568 !(dev->features & NETIF_F_NTUPLE))
Ben Hutchingsc4454772011-01-19 11:03:53 +00003569 goto out;
3570 rxq_index = cpu_rmap_lookup_index(dev->rx_cpu_rmap, next_cpu);
3571 if (rxq_index == skb_get_rx_queue(skb))
3572 goto out;
3573
3574 rxqueue = dev->_rx + rxq_index;
3575 flow_table = rcu_dereference(rxqueue->rps_flow_table);
3576 if (!flow_table)
3577 goto out;
Tom Herbert61b905d2014-03-24 15:34:47 -07003578 flow_id = skb_get_hash(skb) & flow_table->mask;
Ben Hutchingsc4454772011-01-19 11:03:53 +00003579 rc = dev->netdev_ops->ndo_rx_flow_steer(dev, skb,
3580 rxq_index, flow_id);
3581 if (rc < 0)
3582 goto out;
3583 old_rflow = rflow;
3584 rflow = &flow_table->flows[flow_id];
Ben Hutchingsc4454772011-01-19 11:03:53 +00003585 rflow->filter = rc;
3586 if (old_rflow->filter == rflow->filter)
3587 old_rflow->filter = RPS_NO_FILTER;
3588 out:
3589#endif
3590 rflow->last_qtail =
Ben Hutchings09994d12011-10-03 04:42:46 +00003591 per_cpu(softnet_data, next_cpu).input_queue_head;
Ben Hutchingsc4454772011-01-19 11:03:53 +00003592 }
3593
Ben Hutchings09994d12011-10-03 04:42:46 +00003594 rflow->cpu = next_cpu;
Ben Hutchingsc4454772011-01-19 11:03:53 +00003595 return rflow;
3596}
3597
Tom Herbert0a9627f2010-03-16 08:03:29 +00003598/*
3599 * get_rps_cpu is called from netif_receive_skb and returns the target
3600 * CPU from the RPS map of the receiving queue for a given skb.
Eric Dumazetb0e28f12010-04-15 00:14:07 -07003601 * rcu_read_lock must be held on entry.
Tom Herbert0a9627f2010-03-16 08:03:29 +00003602 */
Tom Herbertfec5e652010-04-16 16:01:27 -07003603static int get_rps_cpu(struct net_device *dev, struct sk_buff *skb,
3604 struct rps_dev_flow **rflowp)
Tom Herbert0a9627f2010-03-16 08:03:29 +00003605{
Eric Dumazet567e4b72015-02-06 12:59:01 -08003606 const struct rps_sock_flow_table *sock_flow_table;
3607 struct netdev_rx_queue *rxqueue = dev->_rx;
Tom Herbertfec5e652010-04-16 16:01:27 -07003608 struct rps_dev_flow_table *flow_table;
Eric Dumazet567e4b72015-02-06 12:59:01 -08003609 struct rps_map *map;
Tom Herbert0a9627f2010-03-16 08:03:29 +00003610 int cpu = -1;
Eric Dumazet567e4b72015-02-06 12:59:01 -08003611 u32 tcpu;
Tom Herbert61b905d2014-03-24 15:34:47 -07003612 u32 hash;
Tom Herbert0a9627f2010-03-16 08:03:29 +00003613
Tom Herbert0a9627f2010-03-16 08:03:29 +00003614 if (skb_rx_queue_recorded(skb)) {
3615 u16 index = skb_get_rx_queue(skb);
Eric Dumazet567e4b72015-02-06 12:59:01 -08003616
Ben Hutchings62fe0b42010-09-27 08:24:33 +00003617 if (unlikely(index >= dev->real_num_rx_queues)) {
3618 WARN_ONCE(dev->real_num_rx_queues > 1,
3619 "%s received packet on queue %u, but number "
3620 "of RX queues is %u\n",
3621 dev->name, index, dev->real_num_rx_queues);
Tom Herbert0a9627f2010-03-16 08:03:29 +00003622 goto done;
3623 }
Eric Dumazet567e4b72015-02-06 12:59:01 -08003624 rxqueue += index;
Tom Herbert0a9627f2010-03-16 08:03:29 +00003625 }
3626
Eric Dumazet567e4b72015-02-06 12:59:01 -08003627 /* Avoid computing hash if RFS/RPS is not active for this rxqueue */
3628
3629 flow_table = rcu_dereference(rxqueue->rps_flow_table);
3630 map = rcu_dereference(rxqueue->rps_map);
3631 if (!flow_table && !map)
3632 goto done;
3633
Changli Gao2d47b452010-08-17 19:00:56 +00003634 skb_reset_network_header(skb);
Tom Herbert61b905d2014-03-24 15:34:47 -07003635 hash = skb_get_hash(skb);
3636 if (!hash)
Tom Herbert0a9627f2010-03-16 08:03:29 +00003637 goto done;
Tom Herbert0a9627f2010-03-16 08:03:29 +00003638
Tom Herbertfec5e652010-04-16 16:01:27 -07003639 sock_flow_table = rcu_dereference(rps_sock_flow_table);
3640 if (flow_table && sock_flow_table) {
Tom Herbertfec5e652010-04-16 16:01:27 -07003641 struct rps_dev_flow *rflow;
Eric Dumazet567e4b72015-02-06 12:59:01 -08003642 u32 next_cpu;
3643 u32 ident;
Tom Herbertfec5e652010-04-16 16:01:27 -07003644
Eric Dumazet567e4b72015-02-06 12:59:01 -08003645 /* First check into global flow table if there is a match */
3646 ident = sock_flow_table->ents[hash & sock_flow_table->mask];
3647 if ((ident ^ hash) & ~rps_cpu_mask)
3648 goto try_rps;
3649
3650 next_cpu = ident & rps_cpu_mask;
3651
3652 /* OK, now we know there is a match,
3653 * we can look at the local (per receive queue) flow table
3654 */
Tom Herbert61b905d2014-03-24 15:34:47 -07003655 rflow = &flow_table->flows[hash & flow_table->mask];
Tom Herbertfec5e652010-04-16 16:01:27 -07003656 tcpu = rflow->cpu;
3657
Tom Herbertfec5e652010-04-16 16:01:27 -07003658 /*
3659 * If the desired CPU (where last recvmsg was done) is
3660 * different from current CPU (one in the rx-queue flow
3661 * table entry), switch if one of the following holds:
Eric Dumazeta31196b2015-04-25 09:35:24 -07003662 * - Current CPU is unset (>= nr_cpu_ids).
Tom Herbertfec5e652010-04-16 16:01:27 -07003663 * - Current CPU is offline.
3664 * - The current CPU's queue tail has advanced beyond the
3665 * last packet that was enqueued using this table entry.
3666 * This guarantees that all previous packets for the flow
3667 * have been dequeued, thus preserving in order delivery.
3668 */
3669 if (unlikely(tcpu != next_cpu) &&
Eric Dumazeta31196b2015-04-25 09:35:24 -07003670 (tcpu >= nr_cpu_ids || !cpu_online(tcpu) ||
Tom Herbertfec5e652010-04-16 16:01:27 -07003671 ((int)(per_cpu(softnet_data, tcpu).input_queue_head -
Tom Herbertbaefa312012-11-16 09:04:15 +00003672 rflow->last_qtail)) >= 0)) {
3673 tcpu = next_cpu;
Ben Hutchingsc4454772011-01-19 11:03:53 +00003674 rflow = set_rps_cpu(dev, skb, rflow, next_cpu);
Tom Herbertbaefa312012-11-16 09:04:15 +00003675 }
Ben Hutchingsc4454772011-01-19 11:03:53 +00003676
Eric Dumazeta31196b2015-04-25 09:35:24 -07003677 if (tcpu < nr_cpu_ids && cpu_online(tcpu)) {
Tom Herbertfec5e652010-04-16 16:01:27 -07003678 *rflowp = rflow;
3679 cpu = tcpu;
3680 goto done;
3681 }
3682 }
3683
Eric Dumazet567e4b72015-02-06 12:59:01 -08003684try_rps:
3685
Tom Herbert0a9627f2010-03-16 08:03:29 +00003686 if (map) {
Daniel Borkmann8fc54f62014-08-23 20:58:54 +02003687 tcpu = map->cpus[reciprocal_scale(hash, map->len)];
Tom Herbert0a9627f2010-03-16 08:03:29 +00003688 if (cpu_online(tcpu)) {
3689 cpu = tcpu;
3690 goto done;
3691 }
3692 }
3693
3694done:
Tom Herbert0a9627f2010-03-16 08:03:29 +00003695 return cpu;
3696}
3697
Ben Hutchingsc4454772011-01-19 11:03:53 +00003698#ifdef CONFIG_RFS_ACCEL
3699
3700/**
3701 * rps_may_expire_flow - check whether an RFS hardware filter may be removed
3702 * @dev: Device on which the filter was set
3703 * @rxq_index: RX queue index
3704 * @flow_id: Flow ID passed to ndo_rx_flow_steer()
3705 * @filter_id: Filter ID returned by ndo_rx_flow_steer()
3706 *
3707 * Drivers that implement ndo_rx_flow_steer() should periodically call
3708 * this function for each installed filter and remove the filters for
3709 * which it returns %true.
3710 */
3711bool rps_may_expire_flow(struct net_device *dev, u16 rxq_index,
3712 u32 flow_id, u16 filter_id)
3713{
3714 struct netdev_rx_queue *rxqueue = dev->_rx + rxq_index;
3715 struct rps_dev_flow_table *flow_table;
3716 struct rps_dev_flow *rflow;
3717 bool expire = true;
Eric Dumazeta31196b2015-04-25 09:35:24 -07003718 unsigned int cpu;
Ben Hutchingsc4454772011-01-19 11:03:53 +00003719
3720 rcu_read_lock();
3721 flow_table = rcu_dereference(rxqueue->rps_flow_table);
3722 if (flow_table && flow_id <= flow_table->mask) {
3723 rflow = &flow_table->flows[flow_id];
3724 cpu = ACCESS_ONCE(rflow->cpu);
Eric Dumazeta31196b2015-04-25 09:35:24 -07003725 if (rflow->filter == filter_id && cpu < nr_cpu_ids &&
Ben Hutchingsc4454772011-01-19 11:03:53 +00003726 ((int)(per_cpu(softnet_data, cpu).input_queue_head -
3727 rflow->last_qtail) <
3728 (int)(10 * flow_table->mask)))
3729 expire = false;
3730 }
3731 rcu_read_unlock();
3732 return expire;
3733}
3734EXPORT_SYMBOL(rps_may_expire_flow);
3735
3736#endif /* CONFIG_RFS_ACCEL */
3737
Tom Herbert0a9627f2010-03-16 08:03:29 +00003738/* Called from hardirq (IPI) context */
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003739static void rps_trigger_softirq(void *data)
Tom Herbert0a9627f2010-03-16 08:03:29 +00003740{
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003741 struct softnet_data *sd = data;
3742
Eric Dumazeteecfd7c2010-05-06 22:07:48 -07003743 ____napi_schedule(sd, &sd->backlog);
Changli Gaodee42872010-05-02 05:42:16 +00003744 sd->received_rps++;
Tom Herbert0a9627f2010-03-16 08:03:29 +00003745}
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003746
Tom Herbertfec5e652010-04-16 16:01:27 -07003747#endif /* CONFIG_RPS */
Tom Herbert0a9627f2010-03-16 08:03:29 +00003748
3749/*
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003750 * Check if this softnet_data structure is another cpu one
3751 * If yes, queue it to our IPI list and return 1
3752 * If no, return 0
3753 */
3754static int rps_ipi_queued(struct softnet_data *sd)
3755{
3756#ifdef CONFIG_RPS
Christoph Lameter903ceff2014-08-17 12:30:35 -05003757 struct softnet_data *mysd = this_cpu_ptr(&softnet_data);
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003758
3759 if (sd != mysd) {
3760 sd->rps_ipi_next = mysd->rps_ipi_list;
3761 mysd->rps_ipi_list = sd;
3762
3763 __raise_softirq_irqoff(NET_RX_SOFTIRQ);
3764 return 1;
3765 }
3766#endif /* CONFIG_RPS */
3767 return 0;
3768}
3769
Willem de Bruijn99bbc702013-05-20 04:02:32 +00003770#ifdef CONFIG_NET_FLOW_LIMIT
3771int netdev_flow_limit_table_len __read_mostly = (1 << 12);
3772#endif
3773
3774static bool skb_flow_limit(struct sk_buff *skb, unsigned int qlen)
3775{
3776#ifdef CONFIG_NET_FLOW_LIMIT
3777 struct sd_flow_limit *fl;
3778 struct softnet_data *sd;
3779 unsigned int old_flow, new_flow;
3780
3781 if (qlen < (netdev_max_backlog >> 1))
3782 return false;
3783
Christoph Lameter903ceff2014-08-17 12:30:35 -05003784 sd = this_cpu_ptr(&softnet_data);
Willem de Bruijn99bbc702013-05-20 04:02:32 +00003785
3786 rcu_read_lock();
3787 fl = rcu_dereference(sd->flow_limit);
3788 if (fl) {
Tom Herbert3958afa1b2013-12-15 22:12:06 -08003789 new_flow = skb_get_hash(skb) & (fl->num_buckets - 1);
Willem de Bruijn99bbc702013-05-20 04:02:32 +00003790 old_flow = fl->history[fl->history_head];
3791 fl->history[fl->history_head] = new_flow;
3792
3793 fl->history_head++;
3794 fl->history_head &= FLOW_LIMIT_HISTORY - 1;
3795
3796 if (likely(fl->buckets[old_flow]))
3797 fl->buckets[old_flow]--;
3798
3799 if (++fl->buckets[new_flow] > (FLOW_LIMIT_HISTORY >> 1)) {
3800 fl->count++;
3801 rcu_read_unlock();
3802 return true;
3803 }
3804 }
3805 rcu_read_unlock();
3806#endif
3807 return false;
3808}
3809
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003810/*
Tom Herbert0a9627f2010-03-16 08:03:29 +00003811 * enqueue_to_backlog is called to queue an skb to a per CPU backlog
3812 * queue (may be a remote CPU queue).
3813 */
Tom Herbertfec5e652010-04-16 16:01:27 -07003814static int enqueue_to_backlog(struct sk_buff *skb, int cpu,
3815 unsigned int *qtail)
Tom Herbert0a9627f2010-03-16 08:03:29 +00003816{
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003817 struct softnet_data *sd;
Tom Herbert0a9627f2010-03-16 08:03:29 +00003818 unsigned long flags;
Willem de Bruijn99bbc702013-05-20 04:02:32 +00003819 unsigned int qlen;
Tom Herbert0a9627f2010-03-16 08:03:29 +00003820
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003821 sd = &per_cpu(softnet_data, cpu);
Tom Herbert0a9627f2010-03-16 08:03:29 +00003822
3823 local_irq_save(flags);
Tom Herbert0a9627f2010-03-16 08:03:29 +00003824
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003825 rps_lock(sd);
Julian Anastasove9e4dd32015-07-09 09:59:09 +03003826 if (!netif_running(skb->dev))
3827 goto drop;
Willem de Bruijn99bbc702013-05-20 04:02:32 +00003828 qlen = skb_queue_len(&sd->input_pkt_queue);
3829 if (qlen <= netdev_max_backlog && !skb_flow_limit(skb, qlen)) {
Li RongQinge008f3f2014-12-08 09:42:55 +08003830 if (qlen) {
Tom Herbert0a9627f2010-03-16 08:03:29 +00003831enqueue:
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003832 __skb_queue_tail(&sd->input_pkt_queue, skb);
Tom Herbert76cc8b12010-05-20 18:37:59 +00003833 input_queue_tail_incr_save(sd, qtail);
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003834 rps_unlock(sd);
Changli Gao152102c2010-03-30 20:16:22 +00003835 local_irq_restore(flags);
Tom Herbert0a9627f2010-03-16 08:03:29 +00003836 return NET_RX_SUCCESS;
3837 }
3838
Eric Dumazetebda37c22010-05-06 23:51:21 +00003839 /* Schedule NAPI for backlog device
3840 * We can use non atomic operation since we own the queue lock
3841 */
3842 if (!__test_and_set_bit(NAPI_STATE_SCHED, &sd->backlog.state)) {
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003843 if (!rps_ipi_queued(sd))
Eric Dumazeteecfd7c2010-05-06 22:07:48 -07003844 ____napi_schedule(sd, &sd->backlog);
Tom Herbert0a9627f2010-03-16 08:03:29 +00003845 }
3846 goto enqueue;
3847 }
3848
Julian Anastasove9e4dd32015-07-09 09:59:09 +03003849drop:
Changli Gaodee42872010-05-02 05:42:16 +00003850 sd->dropped++;
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003851 rps_unlock(sd);
Tom Herbert0a9627f2010-03-16 08:03:29 +00003852
Tom Herbert0a9627f2010-03-16 08:03:29 +00003853 local_irq_restore(flags);
3854
Eric Dumazetcaf586e2010-09-30 21:06:55 +00003855 atomic_long_inc(&skb->dev->rx_dropped);
Tom Herbert0a9627f2010-03-16 08:03:29 +00003856 kfree_skb(skb);
3857 return NET_RX_DROP;
3858}
Linus Torvalds1da177e2005-04-16 15:20:36 -07003859
John Fastabendd4455162017-07-17 09:26:45 -07003860static u32 netif_receive_generic_xdp(struct sk_buff *skb,
3861 struct bpf_prog *xdp_prog)
3862{
3863 struct xdp_buff xdp;
3864 u32 act = XDP_DROP;
3865 void *orig_data;
3866 int hlen, off;
3867 u32 mac_len;
3868
3869 /* Reinjected packets coming from act_mirred or similar should
3870 * not get XDP generic processing.
3871 */
3872 if (skb_cloned(skb))
3873 return XDP_PASS;
3874
3875 if (skb_linearize(skb))
3876 goto do_drop;
3877
3878 /* The XDP program wants to see the packet starting at the MAC
3879 * header.
3880 */
3881 mac_len = skb->data - skb_mac_header(skb);
3882 hlen = skb_headlen(skb) + mac_len;
3883 xdp.data = skb->data - mac_len;
3884 xdp.data_end = xdp.data + hlen;
3885 xdp.data_hard_start = skb->data - skb_headroom(skb);
3886 orig_data = xdp.data;
3887
3888 act = bpf_prog_run_xdp(xdp_prog, &xdp);
3889
3890 off = xdp.data - orig_data;
3891 if (off > 0)
3892 __skb_pull(skb, off);
3893 else if (off < 0)
3894 __skb_push(skb, -off);
3895
3896 switch (act) {
John Fastabend6103aa92017-07-17 09:27:50 -07003897 case XDP_REDIRECT:
John Fastabendd4455162017-07-17 09:26:45 -07003898 case XDP_TX:
3899 __skb_push(skb, mac_len);
3900 /* fall through */
3901 case XDP_PASS:
3902 break;
3903
3904 default:
3905 bpf_warn_invalid_xdp_action(act);
3906 /* fall through */
3907 case XDP_ABORTED:
3908 trace_xdp_exception(skb->dev, xdp_prog, act);
3909 /* fall through */
3910 case XDP_DROP:
3911 do_drop:
3912 kfree_skb(skb);
3913 break;
3914 }
3915
3916 return act;
3917}
3918
3919/* When doing generic XDP we have to bypass the qdisc layer and the
3920 * network taps in order to match in-driver-XDP behavior.
3921 */
3922static void generic_xdp_tx(struct sk_buff *skb, struct bpf_prog *xdp_prog)
3923{
3924 struct net_device *dev = skb->dev;
3925 struct netdev_queue *txq;
3926 bool free_skb = true;
3927 int cpu, rc;
3928
3929 txq = netdev_pick_tx(dev, skb, NULL);
3930 cpu = smp_processor_id();
3931 HARD_TX_LOCK(dev, txq, cpu);
3932 if (!netif_xmit_stopped(txq)) {
3933 rc = netdev_start_xmit(skb, dev, txq, 0);
3934 if (dev_xmit_complete(rc))
3935 free_skb = false;
3936 }
3937 HARD_TX_UNLOCK(dev, txq);
3938 if (free_skb) {
3939 trace_xdp_exception(dev, xdp_prog, XDP_TX);
3940 kfree_skb(skb);
3941 }
3942}
3943
3944static struct static_key generic_xdp_needed __read_mostly;
3945
3946static int do_xdp_generic(struct sk_buff *skb)
3947{
3948 struct bpf_prog *xdp_prog = rcu_dereference(skb->dev->xdp_prog);
3949
3950 if (xdp_prog) {
3951 u32 act = netif_receive_generic_xdp(skb, xdp_prog);
John Fastabend6103aa92017-07-17 09:27:50 -07003952 int err;
John Fastabendd4455162017-07-17 09:26:45 -07003953
3954 if (act != XDP_PASS) {
John Fastabend6103aa92017-07-17 09:27:50 -07003955 switch (act) {
3956 case XDP_REDIRECT:
3957 err = xdp_do_generic_redirect(skb->dev, skb);
3958 if (err)
3959 goto out_redir;
3960 /* fallthru to submit skb */
3961 case XDP_TX:
John Fastabendd4455162017-07-17 09:26:45 -07003962 generic_xdp_tx(skb, xdp_prog);
John Fastabend6103aa92017-07-17 09:27:50 -07003963 break;
3964 }
John Fastabendd4455162017-07-17 09:26:45 -07003965 return XDP_DROP;
3966 }
3967 }
3968 return XDP_PASS;
John Fastabend6103aa92017-07-17 09:27:50 -07003969out_redir:
3970 trace_xdp_exception(skb->dev, xdp_prog, XDP_REDIRECT);
3971 kfree_skb(skb);
3972 return XDP_DROP;
John Fastabendd4455162017-07-17 09:26:45 -07003973}
3974
Ben Hutchingsae78dbf2014-01-10 22:17:24 +00003975static int netif_rx_internal(struct sk_buff *skb)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003976{
Eric Dumazetb0e28f12010-04-15 00:14:07 -07003977 int ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003978
Eric Dumazet588f0332011-11-15 04:12:55 +00003979 net_timestamp_check(netdev_tstamp_prequeue, skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003980
Koki Sanagicf66ba52010-08-23 18:45:02 +09003981 trace_netif_rx(skb);
John Fastabendd4455162017-07-17 09:26:45 -07003982
3983 if (static_key_false(&generic_xdp_needed)) {
3984 int ret = do_xdp_generic(skb);
3985
John Fastabend6103aa92017-07-17 09:27:50 -07003986 /* Consider XDP consuming the packet a success from
3987 * the netdev point of view we do not want to count
3988 * this as an error.
3989 */
John Fastabendd4455162017-07-17 09:26:45 -07003990 if (ret != XDP_PASS)
John Fastabend6103aa92017-07-17 09:27:50 -07003991 return NET_RX_SUCCESS;
John Fastabendd4455162017-07-17 09:26:45 -07003992 }
3993
Eric Dumazetdf334542010-03-24 19:13:54 +00003994#ifdef CONFIG_RPS
Ingo Molnarc5905af2012-02-24 08:31:31 +01003995 if (static_key_false(&rps_needed)) {
Tom Herbertfec5e652010-04-16 16:01:27 -07003996 struct rps_dev_flow voidflow, *rflow = &voidflow;
Eric Dumazetb0e28f12010-04-15 00:14:07 -07003997 int cpu;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003998
Changli Gaocece1942010-08-07 20:35:43 -07003999 preempt_disable();
Eric Dumazetb0e28f12010-04-15 00:14:07 -07004000 rcu_read_lock();
Tom Herbertfec5e652010-04-16 16:01:27 -07004001
4002 cpu = get_rps_cpu(skb->dev, skb, &rflow);
Eric Dumazetb0e28f12010-04-15 00:14:07 -07004003 if (cpu < 0)
4004 cpu = smp_processor_id();
Tom Herbertfec5e652010-04-16 16:01:27 -07004005
4006 ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail);
4007
Eric Dumazetb0e28f12010-04-15 00:14:07 -07004008 rcu_read_unlock();
Changli Gaocece1942010-08-07 20:35:43 -07004009 preempt_enable();
Eric Dumazetadc93002011-11-17 03:13:26 +00004010 } else
4011#endif
Tom Herbertfec5e652010-04-16 16:01:27 -07004012 {
4013 unsigned int qtail;
tchardingf4563a72017-02-09 17:56:07 +11004014
Tom Herbertfec5e652010-04-16 16:01:27 -07004015 ret = enqueue_to_backlog(skb, get_cpu(), &qtail);
4016 put_cpu();
4017 }
Eric Dumazetb0e28f12010-04-15 00:14:07 -07004018 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004019}
Ben Hutchingsae78dbf2014-01-10 22:17:24 +00004020
4021/**
4022 * netif_rx - post buffer to the network code
4023 * @skb: buffer to post
4024 *
4025 * This function receives a packet from a device driver and queues it for
4026 * the upper (protocol) levels to process. It always succeeds. The buffer
4027 * may be dropped during processing for congestion control or by the
4028 * protocol layers.
4029 *
4030 * return values:
4031 * NET_RX_SUCCESS (no congestion)
4032 * NET_RX_DROP (packet was dropped)
4033 *
4034 */
4035
4036int netif_rx(struct sk_buff *skb)
4037{
4038 trace_netif_rx_entry(skb);
4039
4040 return netif_rx_internal(skb);
4041}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004042EXPORT_SYMBOL(netif_rx);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004043
4044int netif_rx_ni(struct sk_buff *skb)
4045{
4046 int err;
4047
Ben Hutchingsae78dbf2014-01-10 22:17:24 +00004048 trace_netif_rx_ni_entry(skb);
4049
Linus Torvalds1da177e2005-04-16 15:20:36 -07004050 preempt_disable();
Ben Hutchingsae78dbf2014-01-10 22:17:24 +00004051 err = netif_rx_internal(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004052 if (local_softirq_pending())
4053 do_softirq();
4054 preempt_enable();
4055
4056 return err;
4057}
Linus Torvalds1da177e2005-04-16 15:20:36 -07004058EXPORT_SYMBOL(netif_rx_ni);
4059
Emese Revfy0766f782016-06-20 20:42:34 +02004060static __latent_entropy void net_tx_action(struct softirq_action *h)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004061{
Christoph Lameter903ceff2014-08-17 12:30:35 -05004062 struct softnet_data *sd = this_cpu_ptr(&softnet_data);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004063
4064 if (sd->completion_queue) {
4065 struct sk_buff *clist;
4066
4067 local_irq_disable();
4068 clist = sd->completion_queue;
4069 sd->completion_queue = NULL;
4070 local_irq_enable();
4071
4072 while (clist) {
4073 struct sk_buff *skb = clist;
tchardingf4563a72017-02-09 17:56:07 +11004074
Linus Torvalds1da177e2005-04-16 15:20:36 -07004075 clist = clist->next;
4076
Reshetova, Elena63354792017-06-30 13:07:58 +03004077 WARN_ON(refcount_read(&skb->users));
Eric Dumazete6247022013-12-05 04:45:08 -08004078 if (likely(get_kfree_skb_cb(skb)->reason == SKB_REASON_CONSUMED))
4079 trace_consume_skb(skb);
4080 else
4081 trace_kfree_skb(skb, net_tx_action);
Jesper Dangaard Brouer15fad712016-02-08 13:15:04 +01004082
4083 if (skb->fclone != SKB_FCLONE_UNAVAILABLE)
4084 __kfree_skb(skb);
4085 else
4086 __kfree_skb_defer(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004087 }
Jesper Dangaard Brouer15fad712016-02-08 13:15:04 +01004088
4089 __kfree_skb_flush();
Linus Torvalds1da177e2005-04-16 15:20:36 -07004090 }
4091
4092 if (sd->output_queue) {
David S. Miller37437bb2008-07-16 02:15:04 -07004093 struct Qdisc *head;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004094
4095 local_irq_disable();
4096 head = sd->output_queue;
4097 sd->output_queue = NULL;
Changli Gaoa9cbd582010-04-26 23:06:24 +00004098 sd->output_queue_tailp = &sd->output_queue;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004099 local_irq_enable();
4100
4101 while (head) {
David S. Miller37437bb2008-07-16 02:15:04 -07004102 struct Qdisc *q = head;
4103 spinlock_t *root_lock;
4104
Linus Torvalds1da177e2005-04-16 15:20:36 -07004105 head = head->next_sched;
4106
David S. Miller5fb66222008-08-02 20:02:43 -07004107 root_lock = qdisc_lock(q);
Eric Dumazet3bcb8462016-06-04 20:02:28 -07004108 spin_lock(root_lock);
4109 /* We need to make sure head->next_sched is read
4110 * before clearing __QDISC_STATE_SCHED
4111 */
4112 smp_mb__before_atomic();
4113 clear_bit(__QDISC_STATE_SCHED, &q->state);
4114 qdisc_run(q);
4115 spin_unlock(root_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004116 }
4117 }
4118}
4119
Javier Martinez Canillas181402a2016-09-09 08:43:15 -04004120#if IS_ENABLED(CONFIG_BRIDGE) && IS_ENABLED(CONFIG_ATM_LANE)
Michał Mirosławda678292009-06-05 05:35:28 +00004121/* This hook is defined here for ATM LANE */
4122int (*br_fdb_test_addr_hook)(struct net_device *dev,
4123 unsigned char *addr) __read_mostly;
Stephen Hemminger4fb019a2009-09-11 11:50:08 -07004124EXPORT_SYMBOL_GPL(br_fdb_test_addr_hook);
Michał Mirosławda678292009-06-05 05:35:28 +00004125#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07004126
Daniel Borkmann1f211a12016-01-07 22:29:47 +01004127static inline struct sk_buff *
4128sch_handle_ingress(struct sk_buff *skb, struct packet_type **pt_prev, int *ret,
4129 struct net_device *orig_dev)
Herbert Xuf697c3e2007-10-14 00:38:47 -07004130{
Daniel Borkmanne7582ba2015-05-19 22:33:25 +02004131#ifdef CONFIG_NET_CLS_ACT
Daniel Borkmannd2788d32015-05-09 22:51:32 +02004132 struct tcf_proto *cl = rcu_dereference_bh(skb->dev->ingress_cl_list);
4133 struct tcf_result cl_res;
Eric Dumazet24824a02010-10-02 06:11:55 +00004134
Daniel Borkmannc9e99fd2015-05-09 22:51:31 +02004135 /* If there's at least one ingress present somewhere (so
4136 * we get here via enabled static key), remaining devices
4137 * that are not configured with an ingress qdisc will bail
Daniel Borkmannd2788d32015-05-09 22:51:32 +02004138 * out here.
Daniel Borkmannc9e99fd2015-05-09 22:51:31 +02004139 */
Daniel Borkmannd2788d32015-05-09 22:51:32 +02004140 if (!cl)
Daniel Borkmann45771392015-04-10 23:07:54 +02004141 return skb;
Herbert Xuf697c3e2007-10-14 00:38:47 -07004142 if (*pt_prev) {
4143 *ret = deliver_skb(skb, *pt_prev, orig_dev);
4144 *pt_prev = NULL;
Herbert Xuf697c3e2007-10-14 00:38:47 -07004145 }
4146
Florian Westphal33654952015-05-14 00:36:28 +02004147 qdisc_skb_cb(skb)->pkt_len = skb->len;
Willem de Bruijn8dc07fd2017-01-07 17:06:37 -05004148 skb->tc_at_ingress = 1;
Eric Dumazet24ea5912015-07-06 05:18:03 -07004149 qdisc_bstats_cpu_update(cl->q, skb);
Daniel Borkmannc9e99fd2015-05-09 22:51:31 +02004150
Jiri Pirko87d83092017-05-17 11:07:54 +02004151 switch (tcf_classify(skb, cl, &cl_res, false)) {
Daniel Borkmannd2788d32015-05-09 22:51:32 +02004152 case TC_ACT_OK:
4153 case TC_ACT_RECLASSIFY:
4154 skb->tc_index = TC_H_MIN(cl_res.classid);
4155 break;
4156 case TC_ACT_SHOT:
Eric Dumazet24ea5912015-07-06 05:18:03 -07004157 qdisc_qstats_cpu_drop(cl->q);
Eric Dumazet8a3a4c62016-05-06 15:55:50 -07004158 kfree_skb(skb);
4159 return NULL;
Daniel Borkmannd2788d32015-05-09 22:51:32 +02004160 case TC_ACT_STOLEN:
4161 case TC_ACT_QUEUED:
Jiri Pirkoe25ea212017-06-06 14:12:02 +02004162 case TC_ACT_TRAP:
Eric Dumazet8a3a4c62016-05-06 15:55:50 -07004163 consume_skb(skb);
Daniel Borkmannd2788d32015-05-09 22:51:32 +02004164 return NULL;
Alexei Starovoitov27b29f62015-09-15 23:05:43 -07004165 case TC_ACT_REDIRECT:
4166 /* skb_mac_header check was done by cls/act_bpf, so
4167 * we can safely push the L2 header back before
4168 * redirecting to another netdev
4169 */
4170 __skb_push(skb, skb->mac_len);
4171 skb_do_redirect(skb);
4172 return NULL;
Daniel Borkmannd2788d32015-05-09 22:51:32 +02004173 default:
4174 break;
Herbert Xuf697c3e2007-10-14 00:38:47 -07004175 }
Daniel Borkmanne7582ba2015-05-19 22:33:25 +02004176#endif /* CONFIG_NET_CLS_ACT */
Herbert Xuf697c3e2007-10-14 00:38:47 -07004177 return skb;
4178}
Linus Torvalds1da177e2005-04-16 15:20:36 -07004179
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00004180/**
Mahesh Bandewar24b27fc2016-09-01 22:18:34 -07004181 * netdev_is_rx_handler_busy - check if receive handler is registered
4182 * @dev: device to check
4183 *
4184 * Check if a receive handler is already registered for a given device.
4185 * Return true if there one.
4186 *
4187 * The caller must hold the rtnl_mutex.
4188 */
4189bool netdev_is_rx_handler_busy(struct net_device *dev)
4190{
4191 ASSERT_RTNL();
4192 return dev && rtnl_dereference(dev->rx_handler);
4193}
4194EXPORT_SYMBOL_GPL(netdev_is_rx_handler_busy);
4195
4196/**
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00004197 * netdev_rx_handler_register - register receive handler
4198 * @dev: device to register a handler for
4199 * @rx_handler: receive handler to register
Jiri Pirko93e2c322010-06-10 03:34:59 +00004200 * @rx_handler_data: data pointer that is used by rx handler
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00004201 *
Masanari Iidae2278672014-02-18 22:54:36 +09004202 * Register a receive handler for a device. This handler will then be
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00004203 * called from __netif_receive_skb. A negative errno code is returned
4204 * on a failure.
4205 *
4206 * The caller must hold the rtnl_mutex.
Jiri Pirko8a4eb572011-03-12 03:14:39 +00004207 *
4208 * For a general description of rx_handler, see enum rx_handler_result.
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00004209 */
4210int netdev_rx_handler_register(struct net_device *dev,
Jiri Pirko93e2c322010-06-10 03:34:59 +00004211 rx_handler_func_t *rx_handler,
4212 void *rx_handler_data)
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00004213{
Mahesh Bandewar1b7cd002017-01-18 15:02:49 -08004214 if (netdev_is_rx_handler_busy(dev))
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00004215 return -EBUSY;
4216
Eric Dumazet00cfec32013-03-29 03:01:22 +00004217 /* Note: rx_handler_data must be set before rx_handler */
Jiri Pirko93e2c322010-06-10 03:34:59 +00004218 rcu_assign_pointer(dev->rx_handler_data, rx_handler_data);
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00004219 rcu_assign_pointer(dev->rx_handler, rx_handler);
4220
4221 return 0;
4222}
4223EXPORT_SYMBOL_GPL(netdev_rx_handler_register);
4224
4225/**
4226 * netdev_rx_handler_unregister - unregister receive handler
4227 * @dev: device to unregister a handler from
4228 *
Kusanagi Kouichi166ec362013-03-18 02:59:52 +00004229 * Unregister a receive handler from a device.
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00004230 *
4231 * The caller must hold the rtnl_mutex.
4232 */
4233void netdev_rx_handler_unregister(struct net_device *dev)
4234{
4235
4236 ASSERT_RTNL();
Stephen Hemmingera9b3cd72011-08-01 16:19:00 +00004237 RCU_INIT_POINTER(dev->rx_handler, NULL);
Eric Dumazet00cfec32013-03-29 03:01:22 +00004238 /* a reader seeing a non NULL rx_handler in a rcu_read_lock()
4239 * section has a guarantee to see a non NULL rx_handler_data
4240 * as well.
4241 */
4242 synchronize_net();
Stephen Hemmingera9b3cd72011-08-01 16:19:00 +00004243 RCU_INIT_POINTER(dev->rx_handler_data, NULL);
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00004244}
4245EXPORT_SYMBOL_GPL(netdev_rx_handler_unregister);
4246
Mel Gormanb4b9e352012-07-31 16:44:26 -07004247/*
4248 * Limit the use of PFMEMALLOC reserves to those protocols that implement
4249 * the special handling of PFMEMALLOC skbs.
4250 */
4251static bool skb_pfmemalloc_protocol(struct sk_buff *skb)
4252{
4253 switch (skb->protocol) {
Joe Perches2b8837a2014-03-12 10:04:17 -07004254 case htons(ETH_P_ARP):
4255 case htons(ETH_P_IP):
4256 case htons(ETH_P_IPV6):
4257 case htons(ETH_P_8021Q):
4258 case htons(ETH_P_8021AD):
Mel Gormanb4b9e352012-07-31 16:44:26 -07004259 return true;
4260 default:
4261 return false;
4262 }
4263}
4264
Pablo Neirae687ad62015-05-13 18:19:38 +02004265static inline int nf_ingress(struct sk_buff *skb, struct packet_type **pt_prev,
4266 int *ret, struct net_device *orig_dev)
4267{
Daniel Borkmanne7582ba2015-05-19 22:33:25 +02004268#ifdef CONFIG_NETFILTER_INGRESS
Pablo Neirae687ad62015-05-13 18:19:38 +02004269 if (nf_hook_ingress_active(skb)) {
Aaron Conole2c1e2702016-09-21 11:35:03 -04004270 int ingress_retval;
4271
Pablo Neirae687ad62015-05-13 18:19:38 +02004272 if (*pt_prev) {
4273 *ret = deliver_skb(skb, *pt_prev, orig_dev);
4274 *pt_prev = NULL;
4275 }
4276
Aaron Conole2c1e2702016-09-21 11:35:03 -04004277 rcu_read_lock();
4278 ingress_retval = nf_hook_ingress(skb);
4279 rcu_read_unlock();
4280 return ingress_retval;
Pablo Neirae687ad62015-05-13 18:19:38 +02004281 }
Daniel Borkmanne7582ba2015-05-19 22:33:25 +02004282#endif /* CONFIG_NETFILTER_INGRESS */
Pablo Neirae687ad62015-05-13 18:19:38 +02004283 return 0;
4284}
Pablo Neirae687ad62015-05-13 18:19:38 +02004285
David S. Miller9754e292013-02-14 15:57:38 -05004286static int __netif_receive_skb_core(struct sk_buff *skb, bool pfmemalloc)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004287{
4288 struct packet_type *ptype, *pt_prev;
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00004289 rx_handler_func_t *rx_handler;
David S. Millerf2ccd8f2005-08-09 19:34:12 -07004290 struct net_device *orig_dev;
Jiri Pirko8a4eb572011-03-12 03:14:39 +00004291 bool deliver_exact = false;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004292 int ret = NET_RX_DROP;
Al Viro252e33462006-11-14 20:48:11 -08004293 __be16 type;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004294
Eric Dumazet588f0332011-11-15 04:12:55 +00004295 net_timestamp_check(!netdev_tstamp_prequeue, skb);
Eric Dumazet81bbb3d2009-09-30 16:42:42 -07004296
Koki Sanagicf66ba52010-08-23 18:45:02 +09004297 trace_netif_receive_skb(skb);
Patrick McHardy9b22ea52008-11-04 14:49:57 -08004298
Joe Eykholtcc9bd5c2008-07-02 18:22:00 -07004299 orig_dev = skb->dev;
Jiri Pirko1765a572011-02-12 06:48:36 +00004300
Arnaldo Carvalho de Meloc1d2bbe2007-04-10 20:45:18 -07004301 skb_reset_network_header(skb);
Eric Dumazetfda55ec2013-01-07 09:28:21 +00004302 if (!skb_transport_header_was_set(skb))
4303 skb_reset_transport_header(skb);
Jiri Pirko0b5c9db2011-06-10 06:56:58 +00004304 skb_reset_mac_len(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004305
4306 pt_prev = NULL;
4307
David S. Miller63d8ea72011-02-28 10:48:59 -08004308another_round:
David S. Millerb6858172012-07-23 16:27:54 -07004309 skb->skb_iif = skb->dev->ifindex;
David S. Miller63d8ea72011-02-28 10:48:59 -08004310
4311 __this_cpu_inc(softnet_data.processed);
4312
Patrick McHardy8ad227f2013-04-19 02:04:31 +00004313 if (skb->protocol == cpu_to_be16(ETH_P_8021Q) ||
4314 skb->protocol == cpu_to_be16(ETH_P_8021AD)) {
Vlad Yasevich0d5501c2014-08-08 14:42:13 -04004315 skb = skb_vlan_untag(skb);
Jiri Pirkobcc6d472011-04-07 19:48:33 +00004316 if (unlikely(!skb))
Julian Anastasov2c17d272015-07-09 09:59:10 +03004317 goto out;
Jiri Pirkobcc6d472011-04-07 19:48:33 +00004318 }
4319
Willem de Bruijne7246e12017-01-07 17:06:35 -05004320 if (skb_skip_tc_classify(skb))
4321 goto skip_classify;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004322
David S. Miller9754e292013-02-14 15:57:38 -05004323 if (pfmemalloc)
Mel Gormanb4b9e352012-07-31 16:44:26 -07004324 goto skip_taps;
4325
Linus Torvalds1da177e2005-04-16 15:20:36 -07004326 list_for_each_entry_rcu(ptype, &ptype_all, list) {
Salam Noureddine7866a622015-01-27 11:35:48 -08004327 if (pt_prev)
4328 ret = deliver_skb(skb, pt_prev, orig_dev);
4329 pt_prev = ptype;
4330 }
4331
4332 list_for_each_entry_rcu(ptype, &skb->dev->ptype_all, list) {
4333 if (pt_prev)
4334 ret = deliver_skb(skb, pt_prev, orig_dev);
4335 pt_prev = ptype;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004336 }
4337
Mel Gormanb4b9e352012-07-31 16:44:26 -07004338skip_taps:
Pablo Neira1cf519002015-05-13 18:19:37 +02004339#ifdef CONFIG_NET_INGRESS
Daniel Borkmann45771392015-04-10 23:07:54 +02004340 if (static_key_false(&ingress_needed)) {
Daniel Borkmann1f211a12016-01-07 22:29:47 +01004341 skb = sch_handle_ingress(skb, &pt_prev, &ret, orig_dev);
Daniel Borkmann45771392015-04-10 23:07:54 +02004342 if (!skb)
Julian Anastasov2c17d272015-07-09 09:59:10 +03004343 goto out;
Pablo Neirae687ad62015-05-13 18:19:38 +02004344
4345 if (nf_ingress(skb, &pt_prev, &ret, orig_dev) < 0)
Julian Anastasov2c17d272015-07-09 09:59:10 +03004346 goto out;
Daniel Borkmann45771392015-04-10 23:07:54 +02004347 }
Pablo Neira1cf519002015-05-13 18:19:37 +02004348#endif
Willem de Bruijna5135bc2017-01-07 17:06:36 -05004349 skb_reset_tc(skb);
Willem de Bruijne7246e12017-01-07 17:06:35 -05004350skip_classify:
David S. Miller9754e292013-02-14 15:57:38 -05004351 if (pfmemalloc && !skb_pfmemalloc_protocol(skb))
Mel Gormanb4b9e352012-07-31 16:44:26 -07004352 goto drop;
4353
Jiri Pirkodf8a39d2015-01-13 17:13:44 +01004354 if (skb_vlan_tag_present(skb)) {
John Fastabend24257172011-10-10 09:16:41 +00004355 if (pt_prev) {
4356 ret = deliver_skb(skb, pt_prev, orig_dev);
4357 pt_prev = NULL;
4358 }
Florian Zumbiehl48cc32d32012-10-07 15:51:58 +00004359 if (vlan_do_receive(&skb))
John Fastabend24257172011-10-10 09:16:41 +00004360 goto another_round;
4361 else if (unlikely(!skb))
Julian Anastasov2c17d272015-07-09 09:59:10 +03004362 goto out;
John Fastabend24257172011-10-10 09:16:41 +00004363 }
4364
Florian Zumbiehl48cc32d32012-10-07 15:51:58 +00004365 rx_handler = rcu_dereference(skb->dev->rx_handler);
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00004366 if (rx_handler) {
4367 if (pt_prev) {
4368 ret = deliver_skb(skb, pt_prev, orig_dev);
4369 pt_prev = NULL;
4370 }
Jiri Pirko8a4eb572011-03-12 03:14:39 +00004371 switch (rx_handler(&skb)) {
4372 case RX_HANDLER_CONSUMED:
Cristian Bercaru3bc1b1a2013-03-08 07:03:38 +00004373 ret = NET_RX_SUCCESS;
Julian Anastasov2c17d272015-07-09 09:59:10 +03004374 goto out;
Jiri Pirko8a4eb572011-03-12 03:14:39 +00004375 case RX_HANDLER_ANOTHER:
David S. Miller63d8ea72011-02-28 10:48:59 -08004376 goto another_round;
Jiri Pirko8a4eb572011-03-12 03:14:39 +00004377 case RX_HANDLER_EXACT:
4378 deliver_exact = true;
4379 case RX_HANDLER_PASS:
4380 break;
4381 default:
4382 BUG();
4383 }
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00004384 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07004385
Jiri Pirkodf8a39d2015-01-13 17:13:44 +01004386 if (unlikely(skb_vlan_tag_present(skb))) {
4387 if (skb_vlan_tag_get_id(skb))
Eric Dumazetd4b812d2013-07-18 07:19:26 -07004388 skb->pkt_type = PACKET_OTHERHOST;
4389 /* Note: we might in the future use prio bits
4390 * and set skb->priority like in vlan_do_receive()
4391 * For the time being, just ignore Priority Code Point
4392 */
4393 skb->vlan_tci = 0;
4394 }
Florian Zumbiehl48cc32d32012-10-07 15:51:58 +00004395
Linus Torvalds1da177e2005-04-16 15:20:36 -07004396 type = skb->protocol;
Salam Noureddine7866a622015-01-27 11:35:48 -08004397
4398 /* deliver only exact match when indicated */
4399 if (likely(!deliver_exact)) {
4400 deliver_ptype_list_skb(skb, &pt_prev, orig_dev, type,
4401 &ptype_base[ntohs(type) &
4402 PTYPE_HASH_MASK]);
4403 }
4404
4405 deliver_ptype_list_skb(skb, &pt_prev, orig_dev, type,
4406 &orig_dev->ptype_specific);
4407
4408 if (unlikely(skb->dev != orig_dev)) {
4409 deliver_ptype_list_skb(skb, &pt_prev, orig_dev, type,
4410 &skb->dev->ptype_specific);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004411 }
4412
4413 if (pt_prev) {
Willem de Bruijn1f8b9772017-08-03 16:29:41 -04004414 if (unlikely(skb_orphan_frags_rx(skb, GFP_ATOMIC)))
Michael S. Tsirkin0e698bf2012-09-15 22:44:16 +00004415 goto drop;
Michael S. Tsirkin1080e512012-07-20 09:23:17 +00004416 else
4417 ret = pt_prev->func(skb, skb->dev, pt_prev, orig_dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004418 } else {
Mel Gormanb4b9e352012-07-31 16:44:26 -07004419drop:
Jarod Wilson6e7333d2016-02-01 18:51:05 -05004420 if (!deliver_exact)
4421 atomic_long_inc(&skb->dev->rx_dropped);
4422 else
4423 atomic_long_inc(&skb->dev->rx_nohandler);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004424 kfree_skb(skb);
4425 /* Jamal, now you will not able to escape explaining
4426 * me how you were going to use this. :-)
4427 */
4428 ret = NET_RX_DROP;
4429 }
4430
Julian Anastasov2c17d272015-07-09 09:59:10 +03004431out:
David S. Miller9754e292013-02-14 15:57:38 -05004432 return ret;
4433}
4434
4435static int __netif_receive_skb(struct sk_buff *skb)
4436{
4437 int ret;
4438
4439 if (sk_memalloc_socks() && skb_pfmemalloc(skb)) {
Vlastimil Babkaf1083042017-05-08 15:59:53 -07004440 unsigned int noreclaim_flag;
David S. Miller9754e292013-02-14 15:57:38 -05004441
4442 /*
4443 * PFMEMALLOC skbs are special, they should
4444 * - be delivered to SOCK_MEMALLOC sockets only
4445 * - stay away from userspace
4446 * - have bounded memory usage
4447 *
4448 * Use PF_MEMALLOC as this saves us from propagating the allocation
4449 * context down to all allocation sites.
4450 */
Vlastimil Babkaf1083042017-05-08 15:59:53 -07004451 noreclaim_flag = memalloc_noreclaim_save();
David S. Miller9754e292013-02-14 15:57:38 -05004452 ret = __netif_receive_skb_core(skb, true);
Vlastimil Babkaf1083042017-05-08 15:59:53 -07004453 memalloc_noreclaim_restore(noreclaim_flag);
David S. Miller9754e292013-02-14 15:57:38 -05004454 } else
4455 ret = __netif_receive_skb_core(skb, false);
4456
Linus Torvalds1da177e2005-04-16 15:20:36 -07004457 return ret;
4458}
Tom Herbert0a9627f2010-03-16 08:03:29 +00004459
David S. Millerb5cdae32017-04-18 15:36:58 -04004460static int generic_xdp_install(struct net_device *dev, struct netdev_xdp *xdp)
4461{
Martin KaFai Lau58038692017-06-15 17:29:09 -07004462 struct bpf_prog *old = rtnl_dereference(dev->xdp_prog);
David S. Millerb5cdae32017-04-18 15:36:58 -04004463 struct bpf_prog *new = xdp->prog;
4464 int ret = 0;
4465
4466 switch (xdp->command) {
Martin KaFai Lau58038692017-06-15 17:29:09 -07004467 case XDP_SETUP_PROG:
David S. Millerb5cdae32017-04-18 15:36:58 -04004468 rcu_assign_pointer(dev->xdp_prog, new);
4469 if (old)
4470 bpf_prog_put(old);
4471
4472 if (old && !new) {
4473 static_key_slow_dec(&generic_xdp_needed);
4474 } else if (new && !old) {
4475 static_key_slow_inc(&generic_xdp_needed);
4476 dev_disable_lro(dev);
4477 }
4478 break;
David S. Millerb5cdae32017-04-18 15:36:58 -04004479
4480 case XDP_QUERY_PROG:
Martin KaFai Lau58038692017-06-15 17:29:09 -07004481 xdp->prog_attached = !!old;
4482 xdp->prog_id = old ? old->aux->id : 0;
David S. Millerb5cdae32017-04-18 15:36:58 -04004483 break;
4484
4485 default:
4486 ret = -EINVAL;
4487 break;
4488 }
4489
4490 return ret;
4491}
4492
Ben Hutchingsae78dbf2014-01-10 22:17:24 +00004493static int netif_receive_skb_internal(struct sk_buff *skb)
Tom Herbert0a9627f2010-03-16 08:03:29 +00004494{
Julian Anastasov2c17d272015-07-09 09:59:10 +03004495 int ret;
4496
Eric Dumazet588f0332011-11-15 04:12:55 +00004497 net_timestamp_check(netdev_tstamp_prequeue, skb);
Eric Dumazet3b098e22010-05-15 23:57:10 -07004498
Richard Cochranc1f19b52010-07-17 08:49:36 +00004499 if (skb_defer_rx_timestamp(skb))
4500 return NET_RX_SUCCESS;
4501
Julian Anastasov2c17d272015-07-09 09:59:10 +03004502 rcu_read_lock();
4503
David S. Millerb5cdae32017-04-18 15:36:58 -04004504 if (static_key_false(&generic_xdp_needed)) {
John Fastabendd4455162017-07-17 09:26:45 -07004505 int ret = do_xdp_generic(skb);
David S. Millerb5cdae32017-04-18 15:36:58 -04004506
John Fastabendd4455162017-07-17 09:26:45 -07004507 if (ret != XDP_PASS) {
4508 rcu_read_unlock();
4509 return NET_RX_DROP;
David S. Millerb5cdae32017-04-18 15:36:58 -04004510 }
4511 }
4512
Eric Dumazetdf334542010-03-24 19:13:54 +00004513#ifdef CONFIG_RPS
Ingo Molnarc5905af2012-02-24 08:31:31 +01004514 if (static_key_false(&rps_needed)) {
Eric Dumazet3b098e22010-05-15 23:57:10 -07004515 struct rps_dev_flow voidflow, *rflow = &voidflow;
Julian Anastasov2c17d272015-07-09 09:59:10 +03004516 int cpu = get_rps_cpu(skb->dev, skb, &rflow);
Tom Herbertfec5e652010-04-16 16:01:27 -07004517
Eric Dumazet3b098e22010-05-15 23:57:10 -07004518 if (cpu >= 0) {
4519 ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail);
4520 rcu_read_unlock();
Eric Dumazetadc93002011-11-17 03:13:26 +00004521 return ret;
Eric Dumazet3b098e22010-05-15 23:57:10 -07004522 }
Tom Herbertfec5e652010-04-16 16:01:27 -07004523 }
Tom Herbert1e94d722010-03-18 17:45:44 -07004524#endif
Julian Anastasov2c17d272015-07-09 09:59:10 +03004525 ret = __netif_receive_skb(skb);
4526 rcu_read_unlock();
4527 return ret;
Tom Herbert0a9627f2010-03-16 08:03:29 +00004528}
Ben Hutchingsae78dbf2014-01-10 22:17:24 +00004529
4530/**
4531 * netif_receive_skb - process receive buffer from network
4532 * @skb: buffer to process
4533 *
4534 * netif_receive_skb() is the main receive data processing function.
4535 * It always succeeds. The buffer may be dropped during processing
4536 * for congestion control or by the protocol layers.
4537 *
4538 * This function may only be called from softirq context and interrupts
4539 * should be enabled.
4540 *
4541 * Return values (usually ignored):
4542 * NET_RX_SUCCESS: no congestion
4543 * NET_RX_DROP: packet was dropped
4544 */
Eric W. Biederman04eb4482015-09-15 20:04:15 -05004545int netif_receive_skb(struct sk_buff *skb)
Ben Hutchingsae78dbf2014-01-10 22:17:24 +00004546{
4547 trace_netif_receive_skb_entry(skb);
4548
4549 return netif_receive_skb_internal(skb);
4550}
Eric W. Biederman04eb4482015-09-15 20:04:15 -05004551EXPORT_SYMBOL(netif_receive_skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004552
Eric Dumazet41852492016-08-26 12:50:39 -07004553DEFINE_PER_CPU(struct work_struct, flush_works);
Paolo Abeni145dd5f2016-08-25 15:58:44 +02004554
4555/* Network device is going away, flush any packets still pending */
4556static void flush_backlog(struct work_struct *work)
4557{
Paolo Abeni145dd5f2016-08-25 15:58:44 +02004558 struct sk_buff *skb, *tmp;
4559 struct softnet_data *sd;
4560
4561 local_bh_disable();
4562 sd = this_cpu_ptr(&softnet_data);
4563
4564 local_irq_disable();
Eric Dumazete36fa2f2010-04-19 21:17:14 +00004565 rps_lock(sd);
Changli Gao6e7676c2010-04-27 15:07:33 -07004566 skb_queue_walk_safe(&sd->input_pkt_queue, skb, tmp) {
Eric Dumazet41852492016-08-26 12:50:39 -07004567 if (skb->dev->reg_state == NETREG_UNREGISTERING) {
Eric Dumazete36fa2f2010-04-19 21:17:14 +00004568 __skb_unlink(skb, &sd->input_pkt_queue);
Stephen Hemminger6e583ce2008-08-03 21:29:57 -07004569 kfree_skb(skb);
Tom Herbert76cc8b12010-05-20 18:37:59 +00004570 input_queue_head_incr(sd);
Stephen Hemminger6e583ce2008-08-03 21:29:57 -07004571 }
Changli Gao6e7676c2010-04-27 15:07:33 -07004572 }
Eric Dumazete36fa2f2010-04-19 21:17:14 +00004573 rps_unlock(sd);
Paolo Abeni145dd5f2016-08-25 15:58:44 +02004574 local_irq_enable();
Changli Gao6e7676c2010-04-27 15:07:33 -07004575
4576 skb_queue_walk_safe(&sd->process_queue, skb, tmp) {
Eric Dumazet41852492016-08-26 12:50:39 -07004577 if (skb->dev->reg_state == NETREG_UNREGISTERING) {
Changli Gao6e7676c2010-04-27 15:07:33 -07004578 __skb_unlink(skb, &sd->process_queue);
4579 kfree_skb(skb);
Tom Herbert76cc8b12010-05-20 18:37:59 +00004580 input_queue_head_incr(sd);
Changli Gao6e7676c2010-04-27 15:07:33 -07004581 }
4582 }
Paolo Abeni145dd5f2016-08-25 15:58:44 +02004583 local_bh_enable();
4584}
4585
Eric Dumazet41852492016-08-26 12:50:39 -07004586static void flush_all_backlogs(void)
Paolo Abeni145dd5f2016-08-25 15:58:44 +02004587{
4588 unsigned int cpu;
4589
4590 get_online_cpus();
4591
Eric Dumazet41852492016-08-26 12:50:39 -07004592 for_each_online_cpu(cpu)
4593 queue_work_on(cpu, system_highpri_wq,
4594 per_cpu_ptr(&flush_works, cpu));
Paolo Abeni145dd5f2016-08-25 15:58:44 +02004595
4596 for_each_online_cpu(cpu)
Eric Dumazet41852492016-08-26 12:50:39 -07004597 flush_work(per_cpu_ptr(&flush_works, cpu));
Paolo Abeni145dd5f2016-08-25 15:58:44 +02004598
4599 put_online_cpus();
Stephen Hemminger6e583ce2008-08-03 21:29:57 -07004600}
4601
Herbert Xud565b0a2008-12-15 23:38:52 -08004602static int napi_gro_complete(struct sk_buff *skb)
4603{
Vlad Yasevich22061d82012-11-15 08:49:11 +00004604 struct packet_offload *ptype;
Herbert Xud565b0a2008-12-15 23:38:52 -08004605 __be16 type = skb->protocol;
Vlad Yasevich22061d82012-11-15 08:49:11 +00004606 struct list_head *head = &offload_base;
Herbert Xud565b0a2008-12-15 23:38:52 -08004607 int err = -ENOENT;
4608
Eric Dumazetc3c7c252012-12-06 13:54:59 +00004609 BUILD_BUG_ON(sizeof(struct napi_gro_cb) > sizeof(skb->cb));
4610
Herbert Xufc59f9a2009-04-14 15:11:06 -07004611 if (NAPI_GRO_CB(skb)->count == 1) {
4612 skb_shinfo(skb)->gso_size = 0;
Herbert Xud565b0a2008-12-15 23:38:52 -08004613 goto out;
Herbert Xufc59f9a2009-04-14 15:11:06 -07004614 }
Herbert Xud565b0a2008-12-15 23:38:52 -08004615
4616 rcu_read_lock();
4617 list_for_each_entry_rcu(ptype, head, list) {
Vlad Yasevichf191a1d2012-11-15 08:49:23 +00004618 if (ptype->type != type || !ptype->callbacks.gro_complete)
Herbert Xud565b0a2008-12-15 23:38:52 -08004619 continue;
4620
Jerry Chu299603e82013-12-11 20:53:45 -08004621 err = ptype->callbacks.gro_complete(skb, 0);
Herbert Xud565b0a2008-12-15 23:38:52 -08004622 break;
4623 }
4624 rcu_read_unlock();
4625
4626 if (err) {
4627 WARN_ON(&ptype->list == head);
4628 kfree_skb(skb);
4629 return NET_RX_SUCCESS;
4630 }
4631
4632out:
Ben Hutchingsae78dbf2014-01-10 22:17:24 +00004633 return netif_receive_skb_internal(skb);
Herbert Xud565b0a2008-12-15 23:38:52 -08004634}
4635
Eric Dumazet2e71a6f2012-10-06 08:08:49 +00004636/* napi->gro_list contains packets ordered by age.
4637 * youngest packets at the head of it.
4638 * Complete skbs in reverse order to reduce latencies.
4639 */
4640void napi_gro_flush(struct napi_struct *napi, bool flush_old)
Herbert Xud565b0a2008-12-15 23:38:52 -08004641{
Eric Dumazet2e71a6f2012-10-06 08:08:49 +00004642 struct sk_buff *skb, *prev = NULL;
Herbert Xud565b0a2008-12-15 23:38:52 -08004643
Eric Dumazet2e71a6f2012-10-06 08:08:49 +00004644 /* scan list and build reverse chain */
4645 for (skb = napi->gro_list; skb != NULL; skb = skb->next) {
4646 skb->prev = prev;
4647 prev = skb;
Herbert Xud565b0a2008-12-15 23:38:52 -08004648 }
4649
Eric Dumazet2e71a6f2012-10-06 08:08:49 +00004650 for (skb = prev; skb; skb = prev) {
4651 skb->next = NULL;
4652
4653 if (flush_old && NAPI_GRO_CB(skb)->age == jiffies)
4654 return;
4655
4656 prev = skb->prev;
4657 napi_gro_complete(skb);
4658 napi->gro_count--;
4659 }
4660
Herbert Xud565b0a2008-12-15 23:38:52 -08004661 napi->gro_list = NULL;
4662}
Eric Dumazet86cac582010-08-31 18:25:32 +00004663EXPORT_SYMBOL(napi_gro_flush);
Herbert Xud565b0a2008-12-15 23:38:52 -08004664
Eric Dumazet89c5fa32012-12-10 13:28:16 +00004665static void gro_list_prepare(struct napi_struct *napi, struct sk_buff *skb)
4666{
4667 struct sk_buff *p;
4668 unsigned int maclen = skb->dev->hard_header_len;
Tom Herbert0b4cec82014-01-15 08:58:06 -08004669 u32 hash = skb_get_hash_raw(skb);
Eric Dumazet89c5fa32012-12-10 13:28:16 +00004670
4671 for (p = napi->gro_list; p; p = p->next) {
4672 unsigned long diffs;
4673
Tom Herbert0b4cec82014-01-15 08:58:06 -08004674 NAPI_GRO_CB(p)->flush = 0;
4675
4676 if (hash != skb_get_hash_raw(p)) {
4677 NAPI_GRO_CB(p)->same_flow = 0;
4678 continue;
4679 }
4680
Eric Dumazet89c5fa32012-12-10 13:28:16 +00004681 diffs = (unsigned long)p->dev ^ (unsigned long)skb->dev;
4682 diffs |= p->vlan_tci ^ skb->vlan_tci;
Jesse Grossce87fc62016-01-20 17:59:49 -08004683 diffs |= skb_metadata_dst_cmp(p, skb);
Eric Dumazet89c5fa32012-12-10 13:28:16 +00004684 if (maclen == ETH_HLEN)
4685 diffs |= compare_ether_header(skb_mac_header(p),
Eric Dumazeta50e2332014-03-29 21:28:21 -07004686 skb_mac_header(skb));
Eric Dumazet89c5fa32012-12-10 13:28:16 +00004687 else if (!diffs)
4688 diffs = memcmp(skb_mac_header(p),
Eric Dumazeta50e2332014-03-29 21:28:21 -07004689 skb_mac_header(skb),
Eric Dumazet89c5fa32012-12-10 13:28:16 +00004690 maclen);
4691 NAPI_GRO_CB(p)->same_flow = !diffs;
Eric Dumazet89c5fa32012-12-10 13:28:16 +00004692 }
4693}
4694
Jerry Chu299603e82013-12-11 20:53:45 -08004695static void skb_gro_reset_offset(struct sk_buff *skb)
4696{
4697 const struct skb_shared_info *pinfo = skb_shinfo(skb);
4698 const skb_frag_t *frag0 = &pinfo->frags[0];
4699
4700 NAPI_GRO_CB(skb)->data_offset = 0;
4701 NAPI_GRO_CB(skb)->frag0 = NULL;
4702 NAPI_GRO_CB(skb)->frag0_len = 0;
4703
4704 if (skb_mac_header(skb) == skb_tail_pointer(skb) &&
4705 pinfo->nr_frags &&
4706 !PageHighMem(skb_frag_page(frag0))) {
4707 NAPI_GRO_CB(skb)->frag0 = skb_frag_address(frag0);
Eric Dumazet7cfd5fd2017-01-10 19:52:43 -08004708 NAPI_GRO_CB(skb)->frag0_len = min_t(unsigned int,
4709 skb_frag_size(frag0),
4710 skb->end - skb->tail);
Herbert Xud565b0a2008-12-15 23:38:52 -08004711 }
4712}
4713
Eric Dumazeta50e2332014-03-29 21:28:21 -07004714static void gro_pull_from_frag0(struct sk_buff *skb, int grow)
4715{
4716 struct skb_shared_info *pinfo = skb_shinfo(skb);
4717
4718 BUG_ON(skb->end - skb->tail < grow);
4719
4720 memcpy(skb_tail_pointer(skb), NAPI_GRO_CB(skb)->frag0, grow);
4721
4722 skb->data_len -= grow;
4723 skb->tail += grow;
4724
4725 pinfo->frags[0].page_offset += grow;
4726 skb_frag_size_sub(&pinfo->frags[0], grow);
4727
4728 if (unlikely(!skb_frag_size(&pinfo->frags[0]))) {
4729 skb_frag_unref(skb, 0);
4730 memmove(pinfo->frags, pinfo->frags + 1,
4731 --pinfo->nr_frags * sizeof(pinfo->frags[0]));
4732 }
4733}
4734
Rami Rosenbb728822012-11-28 21:55:25 +00004735static enum gro_result dev_gro_receive(struct napi_struct *napi, struct sk_buff *skb)
Herbert Xud565b0a2008-12-15 23:38:52 -08004736{
4737 struct sk_buff **pp = NULL;
Vlad Yasevich22061d82012-11-15 08:49:11 +00004738 struct packet_offload *ptype;
Herbert Xud565b0a2008-12-15 23:38:52 -08004739 __be16 type = skb->protocol;
Vlad Yasevich22061d82012-11-15 08:49:11 +00004740 struct list_head *head = &offload_base;
Herbert Xu0da2afd52008-12-26 14:57:42 -08004741 int same_flow;
Ben Hutchings5b252f02009-10-29 07:17:09 +00004742 enum gro_result ret;
Eric Dumazeta50e2332014-03-29 21:28:21 -07004743 int grow;
Herbert Xud565b0a2008-12-15 23:38:52 -08004744
David S. Millerb5cdae32017-04-18 15:36:58 -04004745 if (netif_elide_gro(skb->dev))
Herbert Xud565b0a2008-12-15 23:38:52 -08004746 goto normal;
4747
Eric Dumazet89c5fa32012-12-10 13:28:16 +00004748 gro_list_prepare(napi, skb);
4749
Herbert Xud565b0a2008-12-15 23:38:52 -08004750 rcu_read_lock();
4751 list_for_each_entry_rcu(ptype, head, list) {
Vlad Yasevichf191a1d2012-11-15 08:49:23 +00004752 if (ptype->type != type || !ptype->callbacks.gro_receive)
Herbert Xud565b0a2008-12-15 23:38:52 -08004753 continue;
4754
Herbert Xu86911732009-01-29 14:19:50 +00004755 skb_set_network_header(skb, skb_gro_offset(skb));
Eric Dumazetefd94502013-02-14 17:31:48 +00004756 skb_reset_mac_len(skb);
Herbert Xud565b0a2008-12-15 23:38:52 -08004757 NAPI_GRO_CB(skb)->same_flow = 0;
Eric Dumazetd61d0722016-11-07 11:12:27 -08004758 NAPI_GRO_CB(skb)->flush = skb_is_gso(skb) || skb_has_frag_list(skb);
Herbert Xu5d38a072009-01-04 16:13:40 -08004759 NAPI_GRO_CB(skb)->free = 0;
Jesse Grossfac8e0f2016-03-19 09:32:01 -07004760 NAPI_GRO_CB(skb)->encap_mark = 0;
Sabrina Dubrocafcd91dd2016-10-20 15:58:02 +02004761 NAPI_GRO_CB(skb)->recursion_counter = 0;
Alexander Duycka0ca1532016-04-05 09:13:39 -07004762 NAPI_GRO_CB(skb)->is_fou = 0;
Alexander Duyck15305452016-04-10 21:44:57 -04004763 NAPI_GRO_CB(skb)->is_atomic = 1;
Tom Herbert15e23962015-02-10 16:30:31 -08004764 NAPI_GRO_CB(skb)->gro_remcsum_start = 0;
Herbert Xud565b0a2008-12-15 23:38:52 -08004765
Tom Herbert662880f2014-08-27 21:26:56 -07004766 /* Setup for GRO checksum validation */
4767 switch (skb->ip_summed) {
4768 case CHECKSUM_COMPLETE:
4769 NAPI_GRO_CB(skb)->csum = skb->csum;
4770 NAPI_GRO_CB(skb)->csum_valid = 1;
4771 NAPI_GRO_CB(skb)->csum_cnt = 0;
4772 break;
4773 case CHECKSUM_UNNECESSARY:
4774 NAPI_GRO_CB(skb)->csum_cnt = skb->csum_level + 1;
4775 NAPI_GRO_CB(skb)->csum_valid = 0;
4776 break;
4777 default:
4778 NAPI_GRO_CB(skb)->csum_cnt = 0;
4779 NAPI_GRO_CB(skb)->csum_valid = 0;
4780 }
Herbert Xud565b0a2008-12-15 23:38:52 -08004781
Vlad Yasevichf191a1d2012-11-15 08:49:23 +00004782 pp = ptype->callbacks.gro_receive(&napi->gro_list, skb);
Herbert Xud565b0a2008-12-15 23:38:52 -08004783 break;
4784 }
4785 rcu_read_unlock();
4786
4787 if (&ptype->list == head)
4788 goto normal;
4789
Steffen Klassert25393d32017-02-15 09:39:44 +01004790 if (IS_ERR(pp) && PTR_ERR(pp) == -EINPROGRESS) {
4791 ret = GRO_CONSUMED;
4792 goto ok;
4793 }
4794
Herbert Xu0da2afd52008-12-26 14:57:42 -08004795 same_flow = NAPI_GRO_CB(skb)->same_flow;
Herbert Xu5d0d9be2009-01-29 14:19:48 +00004796 ret = NAPI_GRO_CB(skb)->free ? GRO_MERGED_FREE : GRO_MERGED;
Herbert Xu0da2afd52008-12-26 14:57:42 -08004797
Herbert Xud565b0a2008-12-15 23:38:52 -08004798 if (pp) {
4799 struct sk_buff *nskb = *pp;
4800
4801 *pp = nskb->next;
4802 nskb->next = NULL;
4803 napi_gro_complete(nskb);
Herbert Xu4ae55442009-02-08 18:00:36 +00004804 napi->gro_count--;
Herbert Xud565b0a2008-12-15 23:38:52 -08004805 }
4806
Herbert Xu0da2afd52008-12-26 14:57:42 -08004807 if (same_flow)
Herbert Xud565b0a2008-12-15 23:38:52 -08004808 goto ok;
4809
Eric Dumazet600adc12014-01-09 14:12:19 -08004810 if (NAPI_GRO_CB(skb)->flush)
Herbert Xud565b0a2008-12-15 23:38:52 -08004811 goto normal;
Herbert Xud565b0a2008-12-15 23:38:52 -08004812
Eric Dumazet600adc12014-01-09 14:12:19 -08004813 if (unlikely(napi->gro_count >= MAX_GRO_SKBS)) {
4814 struct sk_buff *nskb = napi->gro_list;
4815
4816 /* locate the end of the list to select the 'oldest' flow */
4817 while (nskb->next) {
4818 pp = &nskb->next;
4819 nskb = *pp;
4820 }
4821 *pp = NULL;
4822 nskb->next = NULL;
4823 napi_gro_complete(nskb);
4824 } else {
4825 napi->gro_count++;
4826 }
Herbert Xud565b0a2008-12-15 23:38:52 -08004827 NAPI_GRO_CB(skb)->count = 1;
Eric Dumazet2e71a6f2012-10-06 08:08:49 +00004828 NAPI_GRO_CB(skb)->age = jiffies;
Eric Dumazet29e98242014-05-16 11:34:37 -07004829 NAPI_GRO_CB(skb)->last = skb;
Herbert Xu86911732009-01-29 14:19:50 +00004830 skb_shinfo(skb)->gso_size = skb_gro_len(skb);
Herbert Xud565b0a2008-12-15 23:38:52 -08004831 skb->next = napi->gro_list;
4832 napi->gro_list = skb;
Herbert Xu5d0d9be2009-01-29 14:19:48 +00004833 ret = GRO_HELD;
Herbert Xud565b0a2008-12-15 23:38:52 -08004834
Herbert Xuad0f9902009-02-01 01:24:55 -08004835pull:
Eric Dumazeta50e2332014-03-29 21:28:21 -07004836 grow = skb_gro_offset(skb) - skb_headlen(skb);
4837 if (grow > 0)
4838 gro_pull_from_frag0(skb, grow);
Herbert Xud565b0a2008-12-15 23:38:52 -08004839ok:
Herbert Xu5d0d9be2009-01-29 14:19:48 +00004840 return ret;
Herbert Xud565b0a2008-12-15 23:38:52 -08004841
4842normal:
Herbert Xuad0f9902009-02-01 01:24:55 -08004843 ret = GRO_NORMAL;
4844 goto pull;
Herbert Xu5d38a072009-01-04 16:13:40 -08004845}
Herbert Xu96e93ea2009-01-06 10:49:34 -08004846
Jerry Chubf5a7552014-01-07 10:23:19 -08004847struct packet_offload *gro_find_receive_by_type(__be16 type)
4848{
4849 struct list_head *offload_head = &offload_base;
4850 struct packet_offload *ptype;
4851
4852 list_for_each_entry_rcu(ptype, offload_head, list) {
4853 if (ptype->type != type || !ptype->callbacks.gro_receive)
4854 continue;
4855 return ptype;
4856 }
4857 return NULL;
4858}
Or Gerlitze27a2f82014-01-20 13:59:20 +02004859EXPORT_SYMBOL(gro_find_receive_by_type);
Jerry Chubf5a7552014-01-07 10:23:19 -08004860
4861struct packet_offload *gro_find_complete_by_type(__be16 type)
4862{
4863 struct list_head *offload_head = &offload_base;
4864 struct packet_offload *ptype;
4865
4866 list_for_each_entry_rcu(ptype, offload_head, list) {
4867 if (ptype->type != type || !ptype->callbacks.gro_complete)
4868 continue;
4869 return ptype;
4870 }
4871 return NULL;
4872}
Or Gerlitze27a2f82014-01-20 13:59:20 +02004873EXPORT_SYMBOL(gro_find_complete_by_type);
Herbert Xu96e93ea2009-01-06 10:49:34 -08004874
Michal Kubečeke44699d2017-06-29 11:13:36 +02004875static void napi_skb_free_stolen_head(struct sk_buff *skb)
4876{
4877 skb_dst_drop(skb);
4878 secpath_reset(skb);
4879 kmem_cache_free(skbuff_head_cache, skb);
4880}
4881
Rami Rosenbb728822012-11-28 21:55:25 +00004882static gro_result_t napi_skb_finish(gro_result_t ret, struct sk_buff *skb)
Herbert Xu5d38a072009-01-04 16:13:40 -08004883{
Herbert Xu5d0d9be2009-01-29 14:19:48 +00004884 switch (ret) {
4885 case GRO_NORMAL:
Ben Hutchingsae78dbf2014-01-10 22:17:24 +00004886 if (netif_receive_skb_internal(skb))
Ben Hutchingsc7c4b3b2009-10-29 21:36:53 -07004887 ret = GRO_DROP;
4888 break;
Herbert Xu5d38a072009-01-04 16:13:40 -08004889
Herbert Xu5d0d9be2009-01-29 14:19:48 +00004890 case GRO_DROP:
Herbert Xu5d38a072009-01-04 16:13:40 -08004891 kfree_skb(skb);
4892 break;
Ben Hutchings5b252f02009-10-29 07:17:09 +00004893
Eric Dumazetdaa86542012-04-19 07:07:40 +00004894 case GRO_MERGED_FREE:
Michal Kubečeke44699d2017-06-29 11:13:36 +02004895 if (NAPI_GRO_CB(skb)->free == NAPI_GRO_FREE_STOLEN_HEAD)
4896 napi_skb_free_stolen_head(skb);
4897 else
Eric Dumazetd7e88832012-04-30 08:10:34 +00004898 __kfree_skb(skb);
Eric Dumazetdaa86542012-04-19 07:07:40 +00004899 break;
4900
Ben Hutchings5b252f02009-10-29 07:17:09 +00004901 case GRO_HELD:
4902 case GRO_MERGED:
Steffen Klassert25393d32017-02-15 09:39:44 +01004903 case GRO_CONSUMED:
Ben Hutchings5b252f02009-10-29 07:17:09 +00004904 break;
Herbert Xu5d38a072009-01-04 16:13:40 -08004905 }
4906
Ben Hutchingsc7c4b3b2009-10-29 21:36:53 -07004907 return ret;
Herbert Xu5d0d9be2009-01-29 14:19:48 +00004908}
Herbert Xu5d0d9be2009-01-29 14:19:48 +00004909
Ben Hutchingsc7c4b3b2009-10-29 21:36:53 -07004910gro_result_t napi_gro_receive(struct napi_struct *napi, struct sk_buff *skb)
Herbert Xu5d0d9be2009-01-29 14:19:48 +00004911{
Eric Dumazet93f93a42015-11-18 06:30:59 -08004912 skb_mark_napi_id(skb, napi);
Ben Hutchingsae78dbf2014-01-10 22:17:24 +00004913 trace_napi_gro_receive_entry(skb);
Herbert Xu86911732009-01-29 14:19:50 +00004914
Eric Dumazeta50e2332014-03-29 21:28:21 -07004915 skb_gro_reset_offset(skb);
4916
Eric Dumazet89c5fa32012-12-10 13:28:16 +00004917 return napi_skb_finish(dev_gro_receive(napi, skb), skb);
Herbert Xud565b0a2008-12-15 23:38:52 -08004918}
4919EXPORT_SYMBOL(napi_gro_receive);
4920
stephen hemmingerd0c2b0d2010-10-19 07:12:10 +00004921static void napi_reuse_skb(struct napi_struct *napi, struct sk_buff *skb)
Herbert Xu96e93ea2009-01-06 10:49:34 -08004922{
Eric Dumazet93a35f52014-10-23 06:30:30 -07004923 if (unlikely(skb->pfmemalloc)) {
4924 consume_skb(skb);
4925 return;
4926 }
Herbert Xu96e93ea2009-01-06 10:49:34 -08004927 __skb_pull(skb, skb_headlen(skb));
Eric Dumazet2a2a4592012-03-21 06:58:03 +00004928 /* restore the reserve we had after netdev_alloc_skb_ip_align() */
4929 skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN - skb_headroom(skb));
Jesse Gross3701e512010-10-20 13:56:06 +00004930 skb->vlan_tci = 0;
Herbert Xu66c46d72011-01-29 20:44:54 -08004931 skb->dev = napi->dev;
Andy Gospodarek6d152e22011-02-02 14:53:25 -08004932 skb->skb_iif = 0;
Jerry Chuc3caf112014-07-14 15:54:46 -07004933 skb->encapsulation = 0;
4934 skb_shinfo(skb)->gso_type = 0;
Eric Dumazete33d0ba2014-04-03 09:28:10 -07004935 skb->truesize = SKB_TRUESIZE(skb_end_offset(skb));
Steffen Klassertf991bb92017-01-30 06:45:38 +01004936 secpath_reset(skb);
Herbert Xu96e93ea2009-01-06 10:49:34 -08004937
4938 napi->skb = skb;
4939}
Herbert Xu96e93ea2009-01-06 10:49:34 -08004940
Herbert Xu76620aa2009-04-16 02:02:07 -07004941struct sk_buff *napi_get_frags(struct napi_struct *napi)
Herbert Xu5d38a072009-01-04 16:13:40 -08004942{
Herbert Xu5d38a072009-01-04 16:13:40 -08004943 struct sk_buff *skb = napi->skb;
Herbert Xu5d38a072009-01-04 16:13:40 -08004944
4945 if (!skb) {
Alexander Duyckfd11a832014-12-09 19:40:49 -08004946 skb = napi_alloc_skb(napi, GRO_MAX_HEAD);
Eric Dumazete2f9dc32015-11-19 12:11:23 -08004947 if (skb) {
4948 napi->skb = skb;
4949 skb_mark_napi_id(skb, napi);
4950 }
Herbert Xu5d38a072009-01-04 16:13:40 -08004951 }
Herbert Xu96e93ea2009-01-06 10:49:34 -08004952 return skb;
4953}
Herbert Xu76620aa2009-04-16 02:02:07 -07004954EXPORT_SYMBOL(napi_get_frags);
Herbert Xu96e93ea2009-01-06 10:49:34 -08004955
Eric Dumazeta50e2332014-03-29 21:28:21 -07004956static gro_result_t napi_frags_finish(struct napi_struct *napi,
4957 struct sk_buff *skb,
4958 gro_result_t ret)
Herbert Xu5d0d9be2009-01-29 14:19:48 +00004959{
Herbert Xu5d0d9be2009-01-29 14:19:48 +00004960 switch (ret) {
4961 case GRO_NORMAL:
Eric Dumazeta50e2332014-03-29 21:28:21 -07004962 case GRO_HELD:
4963 __skb_push(skb, ETH_HLEN);
4964 skb->protocol = eth_type_trans(skb, skb->dev);
4965 if (ret == GRO_NORMAL && netif_receive_skb_internal(skb))
Ben Hutchingsc7c4b3b2009-10-29 21:36:53 -07004966 ret = GRO_DROP;
Herbert Xu86911732009-01-29 14:19:50 +00004967 break;
Herbert Xu5d0d9be2009-01-29 14:19:48 +00004968
4969 case GRO_DROP:
Herbert Xu5d0d9be2009-01-29 14:19:48 +00004970 napi_reuse_skb(napi, skb);
4971 break;
Ben Hutchings5b252f02009-10-29 07:17:09 +00004972
Michal Kubečeke44699d2017-06-29 11:13:36 +02004973 case GRO_MERGED_FREE:
4974 if (NAPI_GRO_CB(skb)->free == NAPI_GRO_FREE_STOLEN_HEAD)
4975 napi_skb_free_stolen_head(skb);
4976 else
4977 napi_reuse_skb(napi, skb);
4978 break;
4979
Ben Hutchings5b252f02009-10-29 07:17:09 +00004980 case GRO_MERGED:
Steffen Klassert25393d32017-02-15 09:39:44 +01004981 case GRO_CONSUMED:
Ben Hutchings5b252f02009-10-29 07:17:09 +00004982 break;
Herbert Xu5d0d9be2009-01-29 14:19:48 +00004983 }
4984
Ben Hutchingsc7c4b3b2009-10-29 21:36:53 -07004985 return ret;
Herbert Xu5d0d9be2009-01-29 14:19:48 +00004986}
Herbert Xu5d0d9be2009-01-29 14:19:48 +00004987
Eric Dumazeta50e2332014-03-29 21:28:21 -07004988/* Upper GRO stack assumes network header starts at gro_offset=0
4989 * Drivers could call both napi_gro_frags() and napi_gro_receive()
4990 * We copy ethernet header into skb->data to have a common layout.
4991 */
Eric Dumazet4adb9c42012-05-18 20:49:06 +00004992static struct sk_buff *napi_frags_skb(struct napi_struct *napi)
Herbert Xu96e93ea2009-01-06 10:49:34 -08004993{
Herbert Xu76620aa2009-04-16 02:02:07 -07004994 struct sk_buff *skb = napi->skb;
Eric Dumazeta50e2332014-03-29 21:28:21 -07004995 const struct ethhdr *eth;
4996 unsigned int hlen = sizeof(*eth);
Herbert Xu76620aa2009-04-16 02:02:07 -07004997
4998 napi->skb = NULL;
4999
Eric Dumazeta50e2332014-03-29 21:28:21 -07005000 skb_reset_mac_header(skb);
5001 skb_gro_reset_offset(skb);
5002
5003 eth = skb_gro_header_fast(skb, 0);
5004 if (unlikely(skb_gro_header_hard(skb, hlen))) {
5005 eth = skb_gro_header_slow(skb, hlen, 0);
5006 if (unlikely(!eth)) {
Aaron Conole4da46ce2016-04-02 15:26:43 -04005007 net_warn_ratelimited("%s: dropping impossible skb from %s\n",
5008 __func__, napi->dev->name);
Eric Dumazeta50e2332014-03-29 21:28:21 -07005009 napi_reuse_skb(napi, skb);
5010 return NULL;
5011 }
5012 } else {
5013 gro_pull_from_frag0(skb, hlen);
5014 NAPI_GRO_CB(skb)->frag0 += hlen;
5015 NAPI_GRO_CB(skb)->frag0_len -= hlen;
Herbert Xu76620aa2009-04-16 02:02:07 -07005016 }
Eric Dumazeta50e2332014-03-29 21:28:21 -07005017 __skb_pull(skb, hlen);
5018
5019 /*
5020 * This works because the only protocols we care about don't require
5021 * special handling.
5022 * We'll fix it up properly in napi_frags_finish()
5023 */
5024 skb->protocol = eth->h_proto;
Herbert Xu76620aa2009-04-16 02:02:07 -07005025
Herbert Xu76620aa2009-04-16 02:02:07 -07005026 return skb;
5027}
Herbert Xu76620aa2009-04-16 02:02:07 -07005028
Ben Hutchingsc7c4b3b2009-10-29 21:36:53 -07005029gro_result_t napi_gro_frags(struct napi_struct *napi)
Herbert Xu76620aa2009-04-16 02:02:07 -07005030{
5031 struct sk_buff *skb = napi_frags_skb(napi);
Herbert Xu96e93ea2009-01-06 10:49:34 -08005032
5033 if (!skb)
Ben Hutchingsc7c4b3b2009-10-29 21:36:53 -07005034 return GRO_DROP;
Herbert Xu96e93ea2009-01-06 10:49:34 -08005035
Ben Hutchingsae78dbf2014-01-10 22:17:24 +00005036 trace_napi_gro_frags_entry(skb);
5037
Eric Dumazet89c5fa32012-12-10 13:28:16 +00005038 return napi_frags_finish(napi, skb, dev_gro_receive(napi, skb));
Herbert Xu5d38a072009-01-04 16:13:40 -08005039}
5040EXPORT_SYMBOL(napi_gro_frags);
5041
Tom Herbert573e8fc2014-08-22 13:33:47 -07005042/* Compute the checksum from gro_offset and return the folded value
5043 * after adding in any pseudo checksum.
5044 */
5045__sum16 __skb_gro_checksum_complete(struct sk_buff *skb)
5046{
5047 __wsum wsum;
5048 __sum16 sum;
5049
5050 wsum = skb_checksum(skb, skb_gro_offset(skb), skb_gro_len(skb), 0);
5051
5052 /* NAPI_GRO_CB(skb)->csum holds pseudo checksum */
5053 sum = csum_fold(csum_add(NAPI_GRO_CB(skb)->csum, wsum));
5054 if (likely(!sum)) {
5055 if (unlikely(skb->ip_summed == CHECKSUM_COMPLETE) &&
5056 !skb->csum_complete_sw)
5057 netdev_rx_csum_fault(skb->dev);
5058 }
5059
5060 NAPI_GRO_CB(skb)->csum = wsum;
5061 NAPI_GRO_CB(skb)->csum_valid = 1;
5062
5063 return sum;
5064}
5065EXPORT_SYMBOL(__skb_gro_checksum_complete);
5066
ashwanth@codeaurora.org773fc8f2017-06-09 14:24:58 +05305067static void net_rps_send_ipi(struct softnet_data *remsd)
5068{
5069#ifdef CONFIG_RPS
5070 while (remsd) {
5071 struct softnet_data *next = remsd->rps_ipi_next;
5072
5073 if (cpu_online(remsd->cpu))
5074 smp_call_function_single_async(remsd->cpu, &remsd->csd);
5075 remsd = next;
5076 }
5077#endif
5078}
5079
Eric Dumazete326bed2010-04-22 00:22:45 -07005080/*
Zhi Yong Wu855abcf2014-01-01 04:34:50 +08005081 * net_rps_action_and_irq_enable sends any pending IPI's for rps.
Eric Dumazete326bed2010-04-22 00:22:45 -07005082 * Note: called with local irq disabled, but exits with local irq enabled.
5083 */
5084static void net_rps_action_and_irq_enable(struct softnet_data *sd)
5085{
5086#ifdef CONFIG_RPS
5087 struct softnet_data *remsd = sd->rps_ipi_list;
5088
5089 if (remsd) {
5090 sd->rps_ipi_list = NULL;
5091
5092 local_irq_enable();
5093
5094 /* Send pending IPI's to kick RPS processing on remote cpus. */
ashwanth@codeaurora.org773fc8f2017-06-09 14:24:58 +05305095 net_rps_send_ipi(remsd);
Eric Dumazete326bed2010-04-22 00:22:45 -07005096 } else
5097#endif
5098 local_irq_enable();
5099}
5100
Eric Dumazetd75b1ad2014-11-02 06:19:33 -08005101static bool sd_has_rps_ipi_waiting(struct softnet_data *sd)
5102{
5103#ifdef CONFIG_RPS
5104 return sd->rps_ipi_list != NULL;
5105#else
5106 return false;
5107#endif
5108}
5109
Stephen Hemmingerbea33482007-10-03 16:41:36 -07005110static int process_backlog(struct napi_struct *napi, int quota)
Linus Torvalds1da177e2005-04-16 15:20:36 -07005111{
Eric Dumazeteecfd7c2010-05-06 22:07:48 -07005112 struct softnet_data *sd = container_of(napi, struct softnet_data, backlog);
Paolo Abeni145dd5f2016-08-25 15:58:44 +02005113 bool again = true;
5114 int work = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005115
Eric Dumazete326bed2010-04-22 00:22:45 -07005116 /* Check if we have pending ipi, its better to send them now,
5117 * not waiting net_rx_action() end.
5118 */
Eric Dumazetd75b1ad2014-11-02 06:19:33 -08005119 if (sd_has_rps_ipi_waiting(sd)) {
Eric Dumazete326bed2010-04-22 00:22:45 -07005120 local_irq_disable();
5121 net_rps_action_and_irq_enable(sd);
5122 }
Eric Dumazetd75b1ad2014-11-02 06:19:33 -08005123
Matthias Tafelmeier3d48b532016-12-29 21:37:21 +01005124 napi->weight = dev_rx_weight;
Paolo Abeni145dd5f2016-08-25 15:58:44 +02005125 while (again) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07005126 struct sk_buff *skb;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005127
Changli Gao6e7676c2010-04-27 15:07:33 -07005128 while ((skb = __skb_dequeue(&sd->process_queue))) {
Julian Anastasov2c17d272015-07-09 09:59:10 +03005129 rcu_read_lock();
Changli Gao6e7676c2010-04-27 15:07:33 -07005130 __netif_receive_skb(skb);
Julian Anastasov2c17d272015-07-09 09:59:10 +03005131 rcu_read_unlock();
Tom Herbert76cc8b12010-05-20 18:37:59 +00005132 input_queue_head_incr(sd);
Paolo Abeni145dd5f2016-08-25 15:58:44 +02005133 if (++work >= quota)
Tom Herbert76cc8b12010-05-20 18:37:59 +00005134 return work;
Paolo Abeni145dd5f2016-08-25 15:58:44 +02005135
Stephen Hemmingerbea33482007-10-03 16:41:36 -07005136 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07005137
Paolo Abeni145dd5f2016-08-25 15:58:44 +02005138 local_irq_disable();
Changli Gao6e7676c2010-04-27 15:07:33 -07005139 rps_lock(sd);
Tom Herbert11ef7a82014-06-30 09:50:40 -07005140 if (skb_queue_empty(&sd->input_pkt_queue)) {
Eric Dumazeteecfd7c2010-05-06 22:07:48 -07005141 /*
5142 * Inline a custom version of __napi_complete().
5143 * only current cpu owns and manipulates this napi,
Tom Herbert11ef7a82014-06-30 09:50:40 -07005144 * and NAPI_STATE_SCHED is the only possible flag set
5145 * on backlog.
5146 * We can use a plain write instead of clear_bit(),
Eric Dumazeteecfd7c2010-05-06 22:07:48 -07005147 * and we dont need an smp_mb() memory barrier.
5148 */
Eric Dumazeteecfd7c2010-05-06 22:07:48 -07005149 napi->state = 0;
Paolo Abeni145dd5f2016-08-25 15:58:44 +02005150 again = false;
5151 } else {
5152 skb_queue_splice_tail_init(&sd->input_pkt_queue,
5153 &sd->process_queue);
Changli Gao6e7676c2010-04-27 15:07:33 -07005154 }
5155 rps_unlock(sd);
Paolo Abeni145dd5f2016-08-25 15:58:44 +02005156 local_irq_enable();
Changli Gao6e7676c2010-04-27 15:07:33 -07005157 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07005158
Stephen Hemmingerbea33482007-10-03 16:41:36 -07005159 return work;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005160}
5161
Stephen Hemmingerbea33482007-10-03 16:41:36 -07005162/**
5163 * __napi_schedule - schedule for receive
Randy Dunlapc4ea43c2007-10-12 21:17:49 -07005164 * @n: entry to schedule
Stephen Hemmingerbea33482007-10-03 16:41:36 -07005165 *
Eric Dumazetbc9ad162014-10-28 18:05:13 -07005166 * The entry's receive function will be scheduled to run.
5167 * Consider using __napi_schedule_irqoff() if hard irqs are masked.
Stephen Hemmingerbea33482007-10-03 16:41:36 -07005168 */
Harvey Harrisonb5606c22008-02-13 15:03:16 -08005169void __napi_schedule(struct napi_struct *n)
Stephen Hemmingerbea33482007-10-03 16:41:36 -07005170{
5171 unsigned long flags;
5172
5173 local_irq_save(flags);
Christoph Lameter903ceff2014-08-17 12:30:35 -05005174 ____napi_schedule(this_cpu_ptr(&softnet_data), n);
Stephen Hemmingerbea33482007-10-03 16:41:36 -07005175 local_irq_restore(flags);
5176}
5177EXPORT_SYMBOL(__napi_schedule);
5178
Eric Dumazetbc9ad162014-10-28 18:05:13 -07005179/**
Eric Dumazet39e6c822017-02-28 10:34:50 -08005180 * napi_schedule_prep - check if napi can be scheduled
5181 * @n: napi context
5182 *
5183 * Test if NAPI routine is already running, and if not mark
5184 * it as running. This is used as a condition variable
5185 * insure only one NAPI poll instance runs. We also make
5186 * sure there is no pending NAPI disable.
5187 */
5188bool napi_schedule_prep(struct napi_struct *n)
5189{
5190 unsigned long val, new;
5191
5192 do {
5193 val = READ_ONCE(n->state);
5194 if (unlikely(val & NAPIF_STATE_DISABLE))
5195 return false;
5196 new = val | NAPIF_STATE_SCHED;
5197
5198 /* Sets STATE_MISSED bit if STATE_SCHED was already set
5199 * This was suggested by Alexander Duyck, as compiler
5200 * emits better code than :
5201 * if (val & NAPIF_STATE_SCHED)
5202 * new |= NAPIF_STATE_MISSED;
5203 */
5204 new |= (val & NAPIF_STATE_SCHED) / NAPIF_STATE_SCHED *
5205 NAPIF_STATE_MISSED;
5206 } while (cmpxchg(&n->state, val, new) != val);
5207
5208 return !(val & NAPIF_STATE_SCHED);
5209}
5210EXPORT_SYMBOL(napi_schedule_prep);
5211
5212/**
Eric Dumazetbc9ad162014-10-28 18:05:13 -07005213 * __napi_schedule_irqoff - schedule for receive
5214 * @n: entry to schedule
5215 *
5216 * Variant of __napi_schedule() assuming hard irqs are masked
5217 */
5218void __napi_schedule_irqoff(struct napi_struct *n)
5219{
5220 ____napi_schedule(this_cpu_ptr(&softnet_data), n);
5221}
5222EXPORT_SYMBOL(__napi_schedule_irqoff);
5223
Eric Dumazet364b6052016-11-15 10:15:13 -08005224bool napi_complete_done(struct napi_struct *n, int work_done)
Herbert Xud565b0a2008-12-15 23:38:52 -08005225{
Eric Dumazet39e6c822017-02-28 10:34:50 -08005226 unsigned long flags, val, new;
Herbert Xud565b0a2008-12-15 23:38:52 -08005227
5228 /*
Eric Dumazet217f6972016-11-15 10:15:11 -08005229 * 1) Don't let napi dequeue from the cpu poll list
5230 * just in case its running on a different cpu.
5231 * 2) If we are busy polling, do nothing here, we have
5232 * the guarantee we will be called later.
Herbert Xud565b0a2008-12-15 23:38:52 -08005233 */
Eric Dumazet217f6972016-11-15 10:15:11 -08005234 if (unlikely(n->state & (NAPIF_STATE_NPSVC |
5235 NAPIF_STATE_IN_BUSY_POLL)))
Eric Dumazet364b6052016-11-15 10:15:13 -08005236 return false;
Herbert Xud565b0a2008-12-15 23:38:52 -08005237
Eric Dumazet3b47d302014-11-06 21:09:44 -08005238 if (n->gro_list) {
5239 unsigned long timeout = 0;
Eric Dumazetd75b1ad2014-11-02 06:19:33 -08005240
Eric Dumazet3b47d302014-11-06 21:09:44 -08005241 if (work_done)
5242 timeout = n->dev->gro_flush_timeout;
5243
5244 if (timeout)
5245 hrtimer_start(&n->timer, ns_to_ktime(timeout),
5246 HRTIMER_MODE_REL_PINNED);
5247 else
5248 napi_gro_flush(n, false);
5249 }
Eric Dumazet02c16022017-02-04 15:25:02 -08005250 if (unlikely(!list_empty(&n->poll_list))) {
Eric Dumazetd75b1ad2014-11-02 06:19:33 -08005251 /* If n->poll_list is not empty, we need to mask irqs */
5252 local_irq_save(flags);
Eric Dumazet02c16022017-02-04 15:25:02 -08005253 list_del_init(&n->poll_list);
Eric Dumazetd75b1ad2014-11-02 06:19:33 -08005254 local_irq_restore(flags);
5255 }
Eric Dumazet39e6c822017-02-28 10:34:50 -08005256
5257 do {
5258 val = READ_ONCE(n->state);
5259
5260 WARN_ON_ONCE(!(val & NAPIF_STATE_SCHED));
5261
5262 new = val & ~(NAPIF_STATE_MISSED | NAPIF_STATE_SCHED);
5263
5264 /* If STATE_MISSED was set, leave STATE_SCHED set,
5265 * because we will call napi->poll() one more time.
5266 * This C code was suggested by Alexander Duyck to help gcc.
5267 */
5268 new |= (val & NAPIF_STATE_MISSED) / NAPIF_STATE_MISSED *
5269 NAPIF_STATE_SCHED;
5270 } while (cmpxchg(&n->state, val, new) != val);
5271
5272 if (unlikely(val & NAPIF_STATE_MISSED)) {
5273 __napi_schedule(n);
5274 return false;
5275 }
5276
Eric Dumazet364b6052016-11-15 10:15:13 -08005277 return true;
Herbert Xud565b0a2008-12-15 23:38:52 -08005278}
Eric Dumazet3b47d302014-11-06 21:09:44 -08005279EXPORT_SYMBOL(napi_complete_done);
Herbert Xud565b0a2008-12-15 23:38:52 -08005280
Eliezer Tamiraf12fa62013-06-10 11:39:41 +03005281/* must be called under rcu_read_lock(), as we dont take a reference */
Eric Dumazet02d62e82015-11-18 06:30:52 -08005282static struct napi_struct *napi_by_id(unsigned int napi_id)
Eliezer Tamiraf12fa62013-06-10 11:39:41 +03005283{
5284 unsigned int hash = napi_id % HASH_SIZE(napi_hash);
5285 struct napi_struct *napi;
5286
5287 hlist_for_each_entry_rcu(napi, &napi_hash[hash], napi_hash_node)
5288 if (napi->napi_id == napi_id)
5289 return napi;
5290
5291 return NULL;
5292}
Eric Dumazet02d62e82015-11-18 06:30:52 -08005293
5294#if defined(CONFIG_NET_RX_BUSY_POLL)
Eric Dumazet217f6972016-11-15 10:15:11 -08005295
Eric Dumazetce6aea92015-11-18 06:30:54 -08005296#define BUSY_POLL_BUDGET 8
Eric Dumazet217f6972016-11-15 10:15:11 -08005297
5298static void busy_poll_stop(struct napi_struct *napi, void *have_poll_lock)
5299{
5300 int rc;
5301
Eric Dumazet39e6c822017-02-28 10:34:50 -08005302 /* Busy polling means there is a high chance device driver hard irq
5303 * could not grab NAPI_STATE_SCHED, and that NAPI_STATE_MISSED was
5304 * set in napi_schedule_prep().
5305 * Since we are about to call napi->poll() once more, we can safely
5306 * clear NAPI_STATE_MISSED.
5307 *
5308 * Note: x86 could use a single "lock and ..." instruction
5309 * to perform these two clear_bit()
5310 */
5311 clear_bit(NAPI_STATE_MISSED, &napi->state);
Eric Dumazet217f6972016-11-15 10:15:11 -08005312 clear_bit(NAPI_STATE_IN_BUSY_POLL, &napi->state);
5313
5314 local_bh_disable();
5315
5316 /* All we really want here is to re-enable device interrupts.
5317 * Ideally, a new ndo_busy_poll_stop() could avoid another round.
5318 */
5319 rc = napi->poll(napi, BUSY_POLL_BUDGET);
5320 netpoll_poll_unlock(have_poll_lock);
5321 if (rc == BUSY_POLL_BUDGET)
5322 __napi_schedule(napi);
5323 local_bh_enable();
Eric Dumazet217f6972016-11-15 10:15:11 -08005324}
5325
Sridhar Samudrala7db6b042017-03-24 10:08:24 -07005326void napi_busy_loop(unsigned int napi_id,
5327 bool (*loop_end)(void *, unsigned long),
5328 void *loop_end_arg)
Eric Dumazet02d62e82015-11-18 06:30:52 -08005329{
Sridhar Samudrala7db6b042017-03-24 10:08:24 -07005330 unsigned long start_time = loop_end ? busy_loop_current_time() : 0;
Eric Dumazet217f6972016-11-15 10:15:11 -08005331 int (*napi_poll)(struct napi_struct *napi, int budget);
Eric Dumazet217f6972016-11-15 10:15:11 -08005332 void *have_poll_lock = NULL;
Eric Dumazet02d62e82015-11-18 06:30:52 -08005333 struct napi_struct *napi;
Eric Dumazet217f6972016-11-15 10:15:11 -08005334
5335restart:
Eric Dumazet217f6972016-11-15 10:15:11 -08005336 napi_poll = NULL;
Eric Dumazet02d62e82015-11-18 06:30:52 -08005337
Eric Dumazet2a028ec2015-11-18 06:30:53 -08005338 rcu_read_lock();
Eric Dumazet02d62e82015-11-18 06:30:52 -08005339
Alexander Duyck545cd5e2017-03-24 10:07:53 -07005340 napi = napi_by_id(napi_id);
Eric Dumazet02d62e82015-11-18 06:30:52 -08005341 if (!napi)
5342 goto out;
5343
Eric Dumazet217f6972016-11-15 10:15:11 -08005344 preempt_disable();
5345 for (;;) {
Alexander Duyck2b5cd0d2017-03-24 10:08:12 -07005346 int work = 0;
5347
Eric Dumazet2a028ec2015-11-18 06:30:53 -08005348 local_bh_disable();
Eric Dumazet217f6972016-11-15 10:15:11 -08005349 if (!napi_poll) {
5350 unsigned long val = READ_ONCE(napi->state);
5351
5352 /* If multiple threads are competing for this napi,
5353 * we avoid dirtying napi->state as much as we can.
5354 */
5355 if (val & (NAPIF_STATE_DISABLE | NAPIF_STATE_SCHED |
5356 NAPIF_STATE_IN_BUSY_POLL))
5357 goto count;
5358 if (cmpxchg(&napi->state, val,
5359 val | NAPIF_STATE_IN_BUSY_POLL |
5360 NAPIF_STATE_SCHED) != val)
5361 goto count;
5362 have_poll_lock = netpoll_poll_lock(napi);
5363 napi_poll = napi->poll;
5364 }
Alexander Duyck2b5cd0d2017-03-24 10:08:12 -07005365 work = napi_poll(napi, BUSY_POLL_BUDGET);
5366 trace_napi_poll(napi, work, BUSY_POLL_BUDGET);
Eric Dumazet217f6972016-11-15 10:15:11 -08005367count:
Alexander Duyck2b5cd0d2017-03-24 10:08:12 -07005368 if (work > 0)
Sridhar Samudrala7db6b042017-03-24 10:08:24 -07005369 __NET_ADD_STATS(dev_net(napi->dev),
Alexander Duyck2b5cd0d2017-03-24 10:08:12 -07005370 LINUX_MIB_BUSYPOLLRXPACKETS, work);
Eric Dumazet2a028ec2015-11-18 06:30:53 -08005371 local_bh_enable();
Eric Dumazet02d62e82015-11-18 06:30:52 -08005372
Sridhar Samudrala7db6b042017-03-24 10:08:24 -07005373 if (!loop_end || loop_end(loop_end_arg, start_time))
Eric Dumazet217f6972016-11-15 10:15:11 -08005374 break;
Eric Dumazet02d62e82015-11-18 06:30:52 -08005375
Eric Dumazet217f6972016-11-15 10:15:11 -08005376 if (unlikely(need_resched())) {
5377 if (napi_poll)
5378 busy_poll_stop(napi, have_poll_lock);
5379 preempt_enable();
5380 rcu_read_unlock();
5381 cond_resched();
Sridhar Samudrala7db6b042017-03-24 10:08:24 -07005382 if (loop_end(loop_end_arg, start_time))
Alexander Duyck2b5cd0d2017-03-24 10:08:12 -07005383 return;
Eric Dumazet217f6972016-11-15 10:15:11 -08005384 goto restart;
5385 }
Linus Torvalds6cdf89b2016-12-12 10:48:02 -08005386 cpu_relax();
Eric Dumazet217f6972016-11-15 10:15:11 -08005387 }
5388 if (napi_poll)
5389 busy_poll_stop(napi, have_poll_lock);
5390 preempt_enable();
Eric Dumazet02d62e82015-11-18 06:30:52 -08005391out:
Eric Dumazet2a028ec2015-11-18 06:30:53 -08005392 rcu_read_unlock();
Eric Dumazet02d62e82015-11-18 06:30:52 -08005393}
Sridhar Samudrala7db6b042017-03-24 10:08:24 -07005394EXPORT_SYMBOL(napi_busy_loop);
Eric Dumazet02d62e82015-11-18 06:30:52 -08005395
5396#endif /* CONFIG_NET_RX_BUSY_POLL */
Eliezer Tamiraf12fa62013-06-10 11:39:41 +03005397
Eric Dumazet149d6ad2016-11-08 11:07:28 -08005398static void napi_hash_add(struct napi_struct *napi)
Eliezer Tamiraf12fa62013-06-10 11:39:41 +03005399{
Eric Dumazetd64b5e82015-11-18 06:31:00 -08005400 if (test_bit(NAPI_STATE_NO_BUSY_POLL, &napi->state) ||
5401 test_and_set_bit(NAPI_STATE_HASHED, &napi->state))
Eric Dumazet52bd2d62015-11-18 06:30:50 -08005402 return;
Eliezer Tamiraf12fa62013-06-10 11:39:41 +03005403
Eric Dumazet52bd2d62015-11-18 06:30:50 -08005404 spin_lock(&napi_hash_lock);
Eliezer Tamiraf12fa62013-06-10 11:39:41 +03005405
Alexander Duyck545cd5e2017-03-24 10:07:53 -07005406 /* 0..NR_CPUS range is reserved for sender_cpu use */
Eric Dumazet52bd2d62015-11-18 06:30:50 -08005407 do {
Alexander Duyck545cd5e2017-03-24 10:07:53 -07005408 if (unlikely(++napi_gen_id < MIN_NAPI_ID))
5409 napi_gen_id = MIN_NAPI_ID;
Eric Dumazet52bd2d62015-11-18 06:30:50 -08005410 } while (napi_by_id(napi_gen_id));
5411 napi->napi_id = napi_gen_id;
Eliezer Tamiraf12fa62013-06-10 11:39:41 +03005412
Eric Dumazet52bd2d62015-11-18 06:30:50 -08005413 hlist_add_head_rcu(&napi->napi_hash_node,
5414 &napi_hash[napi->napi_id % HASH_SIZE(napi_hash)]);
Eliezer Tamiraf12fa62013-06-10 11:39:41 +03005415
Eric Dumazet52bd2d62015-11-18 06:30:50 -08005416 spin_unlock(&napi_hash_lock);
Eliezer Tamiraf12fa62013-06-10 11:39:41 +03005417}
Eliezer Tamiraf12fa62013-06-10 11:39:41 +03005418
5419/* Warning : caller is responsible to make sure rcu grace period
5420 * is respected before freeing memory containing @napi
5421 */
Eric Dumazet34cbe272015-11-18 06:31:02 -08005422bool napi_hash_del(struct napi_struct *napi)
Eliezer Tamiraf12fa62013-06-10 11:39:41 +03005423{
Eric Dumazet34cbe272015-11-18 06:31:02 -08005424 bool rcu_sync_needed = false;
5425
Eliezer Tamiraf12fa62013-06-10 11:39:41 +03005426 spin_lock(&napi_hash_lock);
5427
Eric Dumazet34cbe272015-11-18 06:31:02 -08005428 if (test_and_clear_bit(NAPI_STATE_HASHED, &napi->state)) {
5429 rcu_sync_needed = true;
Eliezer Tamiraf12fa62013-06-10 11:39:41 +03005430 hlist_del_rcu(&napi->napi_hash_node);
Eric Dumazet34cbe272015-11-18 06:31:02 -08005431 }
Eliezer Tamiraf12fa62013-06-10 11:39:41 +03005432 spin_unlock(&napi_hash_lock);
Eric Dumazet34cbe272015-11-18 06:31:02 -08005433 return rcu_sync_needed;
Eliezer Tamiraf12fa62013-06-10 11:39:41 +03005434}
5435EXPORT_SYMBOL_GPL(napi_hash_del);
5436
Eric Dumazet3b47d302014-11-06 21:09:44 -08005437static enum hrtimer_restart napi_watchdog(struct hrtimer *timer)
5438{
5439 struct napi_struct *napi;
5440
5441 napi = container_of(timer, struct napi_struct, timer);
Eric Dumazet39e6c822017-02-28 10:34:50 -08005442
5443 /* Note : we use a relaxed variant of napi_schedule_prep() not setting
5444 * NAPI_STATE_MISSED, since we do not react to a device IRQ.
5445 */
5446 if (napi->gro_list && !napi_disable_pending(napi) &&
5447 !test_and_set_bit(NAPI_STATE_SCHED, &napi->state))
5448 __napi_schedule_irqoff(napi);
Eric Dumazet3b47d302014-11-06 21:09:44 -08005449
5450 return HRTIMER_NORESTART;
5451}
5452
Herbert Xud565b0a2008-12-15 23:38:52 -08005453void netif_napi_add(struct net_device *dev, struct napi_struct *napi,
5454 int (*poll)(struct napi_struct *, int), int weight)
5455{
5456 INIT_LIST_HEAD(&napi->poll_list);
Eric Dumazet3b47d302014-11-06 21:09:44 -08005457 hrtimer_init(&napi->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL_PINNED);
5458 napi->timer.function = napi_watchdog;
Herbert Xu4ae55442009-02-08 18:00:36 +00005459 napi->gro_count = 0;
Herbert Xud565b0a2008-12-15 23:38:52 -08005460 napi->gro_list = NULL;
Herbert Xu5d38a072009-01-04 16:13:40 -08005461 napi->skb = NULL;
Herbert Xud565b0a2008-12-15 23:38:52 -08005462 napi->poll = poll;
Eric Dumazet82dc3c63c2013-03-05 15:57:22 +00005463 if (weight > NAPI_POLL_WEIGHT)
5464 pr_err_once("netif_napi_add() called with weight %d on device %s\n",
5465 weight, dev->name);
Herbert Xud565b0a2008-12-15 23:38:52 -08005466 napi->weight = weight;
5467 list_add(&napi->dev_list, &dev->napi_list);
Herbert Xud565b0a2008-12-15 23:38:52 -08005468 napi->dev = dev;
Herbert Xu5d38a072009-01-04 16:13:40 -08005469#ifdef CONFIG_NETPOLL
Herbert Xud565b0a2008-12-15 23:38:52 -08005470 napi->poll_owner = -1;
5471#endif
5472 set_bit(NAPI_STATE_SCHED, &napi->state);
Eric Dumazet93d05d42015-11-18 06:31:03 -08005473 napi_hash_add(napi);
Herbert Xud565b0a2008-12-15 23:38:52 -08005474}
5475EXPORT_SYMBOL(netif_napi_add);
5476
Eric Dumazet3b47d302014-11-06 21:09:44 -08005477void napi_disable(struct napi_struct *n)
5478{
5479 might_sleep();
5480 set_bit(NAPI_STATE_DISABLE, &n->state);
5481
5482 while (test_and_set_bit(NAPI_STATE_SCHED, &n->state))
5483 msleep(1);
Neil Horman2d8bff1262015-09-23 14:57:58 -04005484 while (test_and_set_bit(NAPI_STATE_NPSVC, &n->state))
5485 msleep(1);
Eric Dumazet3b47d302014-11-06 21:09:44 -08005486
5487 hrtimer_cancel(&n->timer);
5488
5489 clear_bit(NAPI_STATE_DISABLE, &n->state);
5490}
5491EXPORT_SYMBOL(napi_disable);
5492
Eric Dumazet93d05d42015-11-18 06:31:03 -08005493/* Must be called in process context */
Herbert Xud565b0a2008-12-15 23:38:52 -08005494void netif_napi_del(struct napi_struct *napi)
5495{
Eric Dumazet93d05d42015-11-18 06:31:03 -08005496 might_sleep();
5497 if (napi_hash_del(napi))
5498 synchronize_net();
Peter P Waskiewicz Jrd7b06632008-12-26 01:35:35 -08005499 list_del_init(&napi->dev_list);
Herbert Xu76620aa2009-04-16 02:02:07 -07005500 napi_free_frags(napi);
Herbert Xud565b0a2008-12-15 23:38:52 -08005501
Eric Dumazet289dccb2013-12-20 14:29:08 -08005502 kfree_skb_list(napi->gro_list);
Herbert Xud565b0a2008-12-15 23:38:52 -08005503 napi->gro_list = NULL;
Herbert Xu4ae55442009-02-08 18:00:36 +00005504 napi->gro_count = 0;
Herbert Xud565b0a2008-12-15 23:38:52 -08005505}
5506EXPORT_SYMBOL(netif_napi_del);
5507
Herbert Xu726ce702014-12-21 07:16:21 +11005508static int napi_poll(struct napi_struct *n, struct list_head *repoll)
5509{
5510 void *have;
5511 int work, weight;
5512
5513 list_del_init(&n->poll_list);
5514
5515 have = netpoll_poll_lock(n);
5516
5517 weight = n->weight;
5518
5519 /* This NAPI_STATE_SCHED test is for avoiding a race
5520 * with netpoll's poll_napi(). Only the entity which
5521 * obtains the lock and sees NAPI_STATE_SCHED set will
5522 * actually make the ->poll() call. Therefore we avoid
5523 * accidentally calling ->poll() when NAPI is not scheduled.
5524 */
5525 work = 0;
5526 if (test_bit(NAPI_STATE_SCHED, &n->state)) {
5527 work = n->poll(n, weight);
Jesper Dangaard Brouer1db19db2016-07-07 18:01:32 +02005528 trace_napi_poll(n, work, weight);
Herbert Xu726ce702014-12-21 07:16:21 +11005529 }
5530
5531 WARN_ON_ONCE(work > weight);
5532
5533 if (likely(work < weight))
5534 goto out_unlock;
5535
5536 /* Drivers must not modify the NAPI state if they
5537 * consume the entire weight. In such cases this code
5538 * still "owns" the NAPI instance and therefore can
5539 * move the instance around on the list at-will.
5540 */
5541 if (unlikely(napi_disable_pending(n))) {
5542 napi_complete(n);
5543 goto out_unlock;
5544 }
5545
5546 if (n->gro_list) {
5547 /* flush too old packets
5548 * If HZ < 1000, flush all packets.
5549 */
5550 napi_gro_flush(n, HZ >= 1000);
5551 }
5552
Herbert Xu001ce542014-12-21 07:16:22 +11005553 /* Some drivers may have called napi_schedule
5554 * prior to exhausting their budget.
5555 */
5556 if (unlikely(!list_empty(&n->poll_list))) {
5557 pr_warn_once("%s: Budget exhausted after napi rescheduled\n",
5558 n->dev ? n->dev->name : "backlog");
5559 goto out_unlock;
5560 }
5561
Herbert Xu726ce702014-12-21 07:16:21 +11005562 list_add_tail(&n->poll_list, repoll);
5563
5564out_unlock:
5565 netpoll_poll_unlock(have);
5566
5567 return work;
5568}
5569
Emese Revfy0766f782016-06-20 20:42:34 +02005570static __latent_entropy void net_rx_action(struct softirq_action *h)
Linus Torvalds1da177e2005-04-16 15:20:36 -07005571{
Christoph Lameter903ceff2014-08-17 12:30:35 -05005572 struct softnet_data *sd = this_cpu_ptr(&softnet_data);
Matthew Whitehead7acf8a12017-04-19 12:37:10 -04005573 unsigned long time_limit = jiffies +
5574 usecs_to_jiffies(netdev_budget_usecs);
Stephen Hemminger51b0bde2005-06-23 20:14:40 -07005575 int budget = netdev_budget;
Eric Dumazetd75b1ad2014-11-02 06:19:33 -08005576 LIST_HEAD(list);
5577 LIST_HEAD(repoll);
Matt Mackall53fb95d2005-08-11 19:27:43 -07005578
Linus Torvalds1da177e2005-04-16 15:20:36 -07005579 local_irq_disable();
Eric Dumazetd75b1ad2014-11-02 06:19:33 -08005580 list_splice_init(&sd->poll_list, &list);
5581 local_irq_enable();
Linus Torvalds1da177e2005-04-16 15:20:36 -07005582
Herbert Xuceb8d5b2014-12-21 07:16:25 +11005583 for (;;) {
Stephen Hemmingerbea33482007-10-03 16:41:36 -07005584 struct napi_struct *n;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005585
Herbert Xuceb8d5b2014-12-21 07:16:25 +11005586 if (list_empty(&list)) {
5587 if (!sd_has_rps_ipi_waiting(sd) && list_empty(&repoll))
Eric Dumazetf52dffe2016-11-23 08:44:56 -08005588 goto out;
Herbert Xuceb8d5b2014-12-21 07:16:25 +11005589 break;
5590 }
5591
Herbert Xu6bd373e2014-12-21 07:16:24 +11005592 n = list_first_entry(&list, struct napi_struct, poll_list);
5593 budget -= napi_poll(n, &repoll);
5594
Eric Dumazetd75b1ad2014-11-02 06:19:33 -08005595 /* If softirq window is exhausted then punt.
Stephen Hemminger24f8b232008-11-03 17:14:38 -08005596 * Allow this to run for 2 jiffies since which will allow
5597 * an average latency of 1.5/HZ.
Stephen Hemmingerbea33482007-10-03 16:41:36 -07005598 */
Herbert Xuceb8d5b2014-12-21 07:16:25 +11005599 if (unlikely(budget <= 0 ||
5600 time_after_eq(jiffies, time_limit))) {
5601 sd->time_squeeze++;
5602 break;
5603 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07005604 }
Eric Dumazetd75b1ad2014-11-02 06:19:33 -08005605
Eric Dumazetd75b1ad2014-11-02 06:19:33 -08005606 local_irq_disable();
5607
5608 list_splice_tail_init(&sd->poll_list, &list);
5609 list_splice_tail(&repoll, &list);
5610 list_splice(&list, &sd->poll_list);
5611 if (!list_empty(&sd->poll_list))
5612 __raise_softirq_irqoff(NET_RX_SOFTIRQ);
5613
Eric Dumazete326bed2010-04-22 00:22:45 -07005614 net_rps_action_and_irq_enable(sd);
Eric Dumazetf52dffe2016-11-23 08:44:56 -08005615out:
5616 __kfree_skb_flush();
Linus Torvalds1da177e2005-04-16 15:20:36 -07005617}
5618
Veaceslav Falicoaa9d8562013-08-28 23:25:04 +02005619struct netdev_adjacent {
Jiri Pirko9ff162a2013-01-03 22:48:49 +00005620 struct net_device *dev;
Veaceslav Falico5d261912013-08-28 23:25:05 +02005621
5622 /* upper master flag, there can only be one master device per list */
Jiri Pirko9ff162a2013-01-03 22:48:49 +00005623 bool master;
Veaceslav Falico5d261912013-08-28 23:25:05 +02005624
Veaceslav Falico5d261912013-08-28 23:25:05 +02005625 /* counter for the number of times this device was added to us */
5626 u16 ref_nr;
5627
Veaceslav Falico402dae92013-09-25 09:20:09 +02005628 /* private field for the users */
5629 void *private;
5630
Jiri Pirko9ff162a2013-01-03 22:48:49 +00005631 struct list_head list;
5632 struct rcu_head rcu;
Jiri Pirko9ff162a2013-01-03 22:48:49 +00005633};
5634
Michal Kubeček6ea29da2015-09-24 10:59:05 +02005635static struct netdev_adjacent *__netdev_find_adj(struct net_device *adj_dev,
Veaceslav Falico2f268f12013-09-25 09:20:07 +02005636 struct list_head *adj_list)
Jiri Pirko9ff162a2013-01-03 22:48:49 +00005637{
Veaceslav Falico5d261912013-08-28 23:25:05 +02005638 struct netdev_adjacent *adj;
Veaceslav Falico5d261912013-08-28 23:25:05 +02005639
Veaceslav Falico2f268f12013-09-25 09:20:07 +02005640 list_for_each_entry(adj, adj_list, list) {
Veaceslav Falico5d261912013-08-28 23:25:05 +02005641 if (adj->dev == adj_dev)
5642 return adj;
Jiri Pirko9ff162a2013-01-03 22:48:49 +00005643 }
5644 return NULL;
5645}
5646
David Ahernf1170fd2016-10-17 19:15:51 -07005647static int __netdev_has_upper_dev(struct net_device *upper_dev, void *data)
5648{
5649 struct net_device *dev = data;
5650
5651 return upper_dev == dev;
5652}
5653
Jiri Pirko9ff162a2013-01-03 22:48:49 +00005654/**
5655 * netdev_has_upper_dev - Check if device is linked to an upper device
5656 * @dev: device
5657 * @upper_dev: upper device to check
5658 *
5659 * Find out if a device is linked to specified upper device and return true
5660 * in case it is. Note that this checks only immediate upper device,
5661 * not through a complete stack of devices. The caller must hold the RTNL lock.
5662 */
5663bool netdev_has_upper_dev(struct net_device *dev,
5664 struct net_device *upper_dev)
5665{
5666 ASSERT_RTNL();
5667
David Ahernf1170fd2016-10-17 19:15:51 -07005668 return netdev_walk_all_upper_dev_rcu(dev, __netdev_has_upper_dev,
5669 upper_dev);
Jiri Pirko9ff162a2013-01-03 22:48:49 +00005670}
5671EXPORT_SYMBOL(netdev_has_upper_dev);
5672
5673/**
David Ahern1a3f0602016-10-17 19:15:44 -07005674 * netdev_has_upper_dev_all - Check if device is linked to an upper device
5675 * @dev: device
5676 * @upper_dev: upper device to check
5677 *
5678 * Find out if a device is linked to specified upper device and return true
5679 * in case it is. Note that this checks the entire upper device chain.
5680 * The caller must hold rcu lock.
5681 */
5682
David Ahern1a3f0602016-10-17 19:15:44 -07005683bool netdev_has_upper_dev_all_rcu(struct net_device *dev,
5684 struct net_device *upper_dev)
5685{
5686 return !!netdev_walk_all_upper_dev_rcu(dev, __netdev_has_upper_dev,
5687 upper_dev);
5688}
5689EXPORT_SYMBOL(netdev_has_upper_dev_all_rcu);
5690
5691/**
Jiri Pirko9ff162a2013-01-03 22:48:49 +00005692 * netdev_has_any_upper_dev - Check if device is linked to some device
5693 * @dev: device
5694 *
5695 * Find out if a device is linked to an upper device and return true in case
5696 * it is. The caller must hold the RTNL lock.
5697 */
stephen hemminger1d143d92013-12-29 14:01:29 -08005698static bool netdev_has_any_upper_dev(struct net_device *dev)
Jiri Pirko9ff162a2013-01-03 22:48:49 +00005699{
5700 ASSERT_RTNL();
5701
David Ahernf1170fd2016-10-17 19:15:51 -07005702 return !list_empty(&dev->adj_list.upper);
Jiri Pirko9ff162a2013-01-03 22:48:49 +00005703}
Jiri Pirko9ff162a2013-01-03 22:48:49 +00005704
5705/**
5706 * netdev_master_upper_dev_get - Get master upper device
5707 * @dev: device
5708 *
5709 * Find a master upper device and return pointer to it or NULL in case
5710 * it's not there. The caller must hold the RTNL lock.
5711 */
5712struct net_device *netdev_master_upper_dev_get(struct net_device *dev)
5713{
Veaceslav Falicoaa9d8562013-08-28 23:25:04 +02005714 struct netdev_adjacent *upper;
Jiri Pirko9ff162a2013-01-03 22:48:49 +00005715
5716 ASSERT_RTNL();
5717
Veaceslav Falico2f268f12013-09-25 09:20:07 +02005718 if (list_empty(&dev->adj_list.upper))
Jiri Pirko9ff162a2013-01-03 22:48:49 +00005719 return NULL;
5720
Veaceslav Falico2f268f12013-09-25 09:20:07 +02005721 upper = list_first_entry(&dev->adj_list.upper,
Veaceslav Falicoaa9d8562013-08-28 23:25:04 +02005722 struct netdev_adjacent, list);
Jiri Pirko9ff162a2013-01-03 22:48:49 +00005723 if (likely(upper->master))
5724 return upper->dev;
5725 return NULL;
5726}
5727EXPORT_SYMBOL(netdev_master_upper_dev_get);
5728
David Ahern0f524a82016-10-17 19:15:52 -07005729/**
5730 * netdev_has_any_lower_dev - Check if device is linked to some device
5731 * @dev: device
5732 *
5733 * Find out if a device is linked to a lower device and return true in case
5734 * it is. The caller must hold the RTNL lock.
5735 */
5736static bool netdev_has_any_lower_dev(struct net_device *dev)
5737{
5738 ASSERT_RTNL();
5739
5740 return !list_empty(&dev->adj_list.lower);
5741}
5742
Veaceslav Falicob6ccba42013-09-25 09:20:23 +02005743void *netdev_adjacent_get_private(struct list_head *adj_list)
5744{
5745 struct netdev_adjacent *adj;
5746
5747 adj = list_entry(adj_list, struct netdev_adjacent, list);
5748
5749 return adj->private;
5750}
5751EXPORT_SYMBOL(netdev_adjacent_get_private);
5752
Veaceslav Falico31088a12013-09-25 09:20:12 +02005753/**
Vlad Yasevich44a40852014-05-16 17:20:38 -04005754 * netdev_upper_get_next_dev_rcu - Get the next dev from upper list
5755 * @dev: device
5756 * @iter: list_head ** of the current position
5757 *
5758 * Gets the next device from the dev's upper list, starting from iter
5759 * position. The caller must hold RCU read lock.
5760 */
5761struct net_device *netdev_upper_get_next_dev_rcu(struct net_device *dev,
5762 struct list_head **iter)
5763{
5764 struct netdev_adjacent *upper;
5765
5766 WARN_ON_ONCE(!rcu_read_lock_held() && !lockdep_rtnl_is_held());
5767
5768 upper = list_entry_rcu((*iter)->next, struct netdev_adjacent, list);
5769
5770 if (&upper->list == &dev->adj_list.upper)
5771 return NULL;
5772
5773 *iter = &upper->list;
5774
5775 return upper->dev;
5776}
5777EXPORT_SYMBOL(netdev_upper_get_next_dev_rcu);
5778
David Ahern1a3f0602016-10-17 19:15:44 -07005779static struct net_device *netdev_next_upper_dev_rcu(struct net_device *dev,
5780 struct list_head **iter)
5781{
5782 struct netdev_adjacent *upper;
5783
5784 WARN_ON_ONCE(!rcu_read_lock_held() && !lockdep_rtnl_is_held());
5785
5786 upper = list_entry_rcu((*iter)->next, struct netdev_adjacent, list);
5787
5788 if (&upper->list == &dev->adj_list.upper)
5789 return NULL;
5790
5791 *iter = &upper->list;
5792
5793 return upper->dev;
5794}
5795
5796int netdev_walk_all_upper_dev_rcu(struct net_device *dev,
5797 int (*fn)(struct net_device *dev,
5798 void *data),
5799 void *data)
5800{
5801 struct net_device *udev;
5802 struct list_head *iter;
5803 int ret;
5804
5805 for (iter = &dev->adj_list.upper,
5806 udev = netdev_next_upper_dev_rcu(dev, &iter);
5807 udev;
5808 udev = netdev_next_upper_dev_rcu(dev, &iter)) {
5809 /* first is the upper device itself */
5810 ret = fn(udev, data);
5811 if (ret)
5812 return ret;
5813
5814 /* then look at all of its upper devices */
5815 ret = netdev_walk_all_upper_dev_rcu(udev, fn, data);
5816 if (ret)
5817 return ret;
5818 }
5819
5820 return 0;
5821}
5822EXPORT_SYMBOL_GPL(netdev_walk_all_upper_dev_rcu);
5823
Jiri Pirko9ff162a2013-01-03 22:48:49 +00005824/**
Veaceslav Falico31088a12013-09-25 09:20:12 +02005825 * netdev_lower_get_next_private - Get the next ->private from the
5826 * lower neighbour list
5827 * @dev: device
5828 * @iter: list_head ** of the current position
5829 *
5830 * Gets the next netdev_adjacent->private from the dev's lower neighbour
5831 * list, starting from iter position. The caller must hold either hold the
5832 * RTNL lock or its own locking that guarantees that the neighbour lower
subashab@codeaurora.orgb4691392015-07-24 03:03:29 +00005833 * list will remain unchanged.
Veaceslav Falico31088a12013-09-25 09:20:12 +02005834 */
5835void *netdev_lower_get_next_private(struct net_device *dev,
5836 struct list_head **iter)
5837{
5838 struct netdev_adjacent *lower;
5839
5840 lower = list_entry(*iter, struct netdev_adjacent, list);
5841
5842 if (&lower->list == &dev->adj_list.lower)
5843 return NULL;
5844
Veaceslav Falico6859e7d2014-04-07 11:25:12 +02005845 *iter = lower->list.next;
Veaceslav Falico31088a12013-09-25 09:20:12 +02005846
5847 return lower->private;
5848}
5849EXPORT_SYMBOL(netdev_lower_get_next_private);
5850
5851/**
5852 * netdev_lower_get_next_private_rcu - Get the next ->private from the
5853 * lower neighbour list, RCU
5854 * variant
5855 * @dev: device
5856 * @iter: list_head ** of the current position
5857 *
5858 * Gets the next netdev_adjacent->private from the dev's lower neighbour
5859 * list, starting from iter position. The caller must hold RCU read lock.
5860 */
5861void *netdev_lower_get_next_private_rcu(struct net_device *dev,
5862 struct list_head **iter)
5863{
5864 struct netdev_adjacent *lower;
5865
5866 WARN_ON_ONCE(!rcu_read_lock_held());
5867
5868 lower = list_entry_rcu((*iter)->next, struct netdev_adjacent, list);
5869
5870 if (&lower->list == &dev->adj_list.lower)
5871 return NULL;
5872
Veaceslav Falico6859e7d2014-04-07 11:25:12 +02005873 *iter = &lower->list;
Veaceslav Falico31088a12013-09-25 09:20:12 +02005874
5875 return lower->private;
5876}
5877EXPORT_SYMBOL(netdev_lower_get_next_private_rcu);
5878
5879/**
Vlad Yasevich4085ebe2014-05-16 17:04:53 -04005880 * netdev_lower_get_next - Get the next device from the lower neighbour
5881 * list
5882 * @dev: device
5883 * @iter: list_head ** of the current position
5884 *
5885 * Gets the next netdev_adjacent from the dev's lower neighbour
5886 * list, starting from iter position. The caller must hold RTNL lock or
5887 * its own locking that guarantees that the neighbour lower
subashab@codeaurora.orgb4691392015-07-24 03:03:29 +00005888 * list will remain unchanged.
Vlad Yasevich4085ebe2014-05-16 17:04:53 -04005889 */
5890void *netdev_lower_get_next(struct net_device *dev, struct list_head **iter)
5891{
5892 struct netdev_adjacent *lower;
5893
Nikolay Aleksandrovcfdd28b2016-02-17 18:00:31 +01005894 lower = list_entry(*iter, struct netdev_adjacent, list);
Vlad Yasevich4085ebe2014-05-16 17:04:53 -04005895
5896 if (&lower->list == &dev->adj_list.lower)
5897 return NULL;
5898
Nikolay Aleksandrovcfdd28b2016-02-17 18:00:31 +01005899 *iter = lower->list.next;
Vlad Yasevich4085ebe2014-05-16 17:04:53 -04005900
5901 return lower->dev;
5902}
5903EXPORT_SYMBOL(netdev_lower_get_next);
5904
David Ahern1a3f0602016-10-17 19:15:44 -07005905static struct net_device *netdev_next_lower_dev(struct net_device *dev,
5906 struct list_head **iter)
5907{
5908 struct netdev_adjacent *lower;
5909
David Ahern46b5ab12016-10-26 13:21:33 -07005910 lower = list_entry((*iter)->next, struct netdev_adjacent, list);
David Ahern1a3f0602016-10-17 19:15:44 -07005911
5912 if (&lower->list == &dev->adj_list.lower)
5913 return NULL;
5914
David Ahern46b5ab12016-10-26 13:21:33 -07005915 *iter = &lower->list;
David Ahern1a3f0602016-10-17 19:15:44 -07005916
5917 return lower->dev;
5918}
5919
5920int netdev_walk_all_lower_dev(struct net_device *dev,
5921 int (*fn)(struct net_device *dev,
5922 void *data),
5923 void *data)
5924{
5925 struct net_device *ldev;
5926 struct list_head *iter;
5927 int ret;
5928
5929 for (iter = &dev->adj_list.lower,
5930 ldev = netdev_next_lower_dev(dev, &iter);
5931 ldev;
5932 ldev = netdev_next_lower_dev(dev, &iter)) {
5933 /* first is the lower device itself */
5934 ret = fn(ldev, data);
5935 if (ret)
5936 return ret;
5937
5938 /* then look at all of its lower devices */
5939 ret = netdev_walk_all_lower_dev(ldev, fn, data);
5940 if (ret)
5941 return ret;
5942 }
5943
5944 return 0;
5945}
5946EXPORT_SYMBOL_GPL(netdev_walk_all_lower_dev);
5947
David Ahern1a3f0602016-10-17 19:15:44 -07005948static struct net_device *netdev_next_lower_dev_rcu(struct net_device *dev,
5949 struct list_head **iter)
5950{
5951 struct netdev_adjacent *lower;
5952
5953 lower = list_entry_rcu((*iter)->next, struct netdev_adjacent, list);
5954 if (&lower->list == &dev->adj_list.lower)
5955 return NULL;
5956
5957 *iter = &lower->list;
5958
5959 return lower->dev;
5960}
5961
5962int netdev_walk_all_lower_dev_rcu(struct net_device *dev,
5963 int (*fn)(struct net_device *dev,
5964 void *data),
5965 void *data)
5966{
5967 struct net_device *ldev;
5968 struct list_head *iter;
5969 int ret;
5970
5971 for (iter = &dev->adj_list.lower,
5972 ldev = netdev_next_lower_dev_rcu(dev, &iter);
5973 ldev;
5974 ldev = netdev_next_lower_dev_rcu(dev, &iter)) {
5975 /* first is the lower device itself */
5976 ret = fn(ldev, data);
5977 if (ret)
5978 return ret;
5979
5980 /* then look at all of its lower devices */
5981 ret = netdev_walk_all_lower_dev_rcu(ldev, fn, data);
5982 if (ret)
5983 return ret;
5984 }
5985
5986 return 0;
5987}
5988EXPORT_SYMBOL_GPL(netdev_walk_all_lower_dev_rcu);
5989
Jiri Pirko7ce856a2016-07-04 08:23:12 +02005990/**
dingtianhonge001bfa2013-12-13 10:19:55 +08005991 * netdev_lower_get_first_private_rcu - Get the first ->private from the
5992 * lower neighbour list, RCU
5993 * variant
5994 * @dev: device
5995 *
5996 * Gets the first netdev_adjacent->private from the dev's lower neighbour
5997 * list. The caller must hold RCU read lock.
5998 */
5999void *netdev_lower_get_first_private_rcu(struct net_device *dev)
6000{
6001 struct netdev_adjacent *lower;
6002
6003 lower = list_first_or_null_rcu(&dev->adj_list.lower,
6004 struct netdev_adjacent, list);
6005 if (lower)
6006 return lower->private;
6007 return NULL;
6008}
6009EXPORT_SYMBOL(netdev_lower_get_first_private_rcu);
6010
6011/**
Jiri Pirko9ff162a2013-01-03 22:48:49 +00006012 * netdev_master_upper_dev_get_rcu - Get master upper device
6013 * @dev: device
6014 *
6015 * Find a master upper device and return pointer to it or NULL in case
6016 * it's not there. The caller must hold the RCU read lock.
6017 */
6018struct net_device *netdev_master_upper_dev_get_rcu(struct net_device *dev)
6019{
Veaceslav Falicoaa9d8562013-08-28 23:25:04 +02006020 struct netdev_adjacent *upper;
Jiri Pirko9ff162a2013-01-03 22:48:49 +00006021
Veaceslav Falico2f268f12013-09-25 09:20:07 +02006022 upper = list_first_or_null_rcu(&dev->adj_list.upper,
Veaceslav Falicoaa9d8562013-08-28 23:25:04 +02006023 struct netdev_adjacent, list);
Jiri Pirko9ff162a2013-01-03 22:48:49 +00006024 if (upper && likely(upper->master))
6025 return upper->dev;
6026 return NULL;
6027}
6028EXPORT_SYMBOL(netdev_master_upper_dev_get_rcu);
6029
Rashika Kheria0a59f3a2014-02-09 20:26:25 +05306030static int netdev_adjacent_sysfs_add(struct net_device *dev,
Veaceslav Falico3ee32702014-01-14 21:58:50 +01006031 struct net_device *adj_dev,
6032 struct list_head *dev_list)
6033{
6034 char linkname[IFNAMSIZ+7];
tchardingf4563a72017-02-09 17:56:07 +11006035
Veaceslav Falico3ee32702014-01-14 21:58:50 +01006036 sprintf(linkname, dev_list == &dev->adj_list.upper ?
6037 "upper_%s" : "lower_%s", adj_dev->name);
6038 return sysfs_create_link(&(dev->dev.kobj), &(adj_dev->dev.kobj),
6039 linkname);
6040}
Rashika Kheria0a59f3a2014-02-09 20:26:25 +05306041static void netdev_adjacent_sysfs_del(struct net_device *dev,
Veaceslav Falico3ee32702014-01-14 21:58:50 +01006042 char *name,
6043 struct list_head *dev_list)
6044{
6045 char linkname[IFNAMSIZ+7];
tchardingf4563a72017-02-09 17:56:07 +11006046
Veaceslav Falico3ee32702014-01-14 21:58:50 +01006047 sprintf(linkname, dev_list == &dev->adj_list.upper ?
6048 "upper_%s" : "lower_%s", name);
6049 sysfs_remove_link(&(dev->dev.kobj), linkname);
6050}
6051
Alexander Y. Fomichev7ce64c72014-09-15 14:22:35 +04006052static inline bool netdev_adjacent_is_neigh_list(struct net_device *dev,
6053 struct net_device *adj_dev,
6054 struct list_head *dev_list)
6055{
6056 return (dev_list == &dev->adj_list.upper ||
6057 dev_list == &dev->adj_list.lower) &&
6058 net_eq(dev_net(dev), dev_net(adj_dev));
6059}
Veaceslav Falico3ee32702014-01-14 21:58:50 +01006060
Veaceslav Falico5d261912013-08-28 23:25:05 +02006061static int __netdev_adjacent_dev_insert(struct net_device *dev,
6062 struct net_device *adj_dev,
Veaceslav Falico7863c052013-09-25 09:20:06 +02006063 struct list_head *dev_list,
Veaceslav Falico402dae92013-09-25 09:20:09 +02006064 void *private, bool master)
Veaceslav Falico5d261912013-08-28 23:25:05 +02006065{
6066 struct netdev_adjacent *adj;
Veaceslav Falico842d67a2013-09-25 09:20:31 +02006067 int ret;
Veaceslav Falico5d261912013-08-28 23:25:05 +02006068
Michal Kubeček6ea29da2015-09-24 10:59:05 +02006069 adj = __netdev_find_adj(adj_dev, dev_list);
Veaceslav Falico5d261912013-08-28 23:25:05 +02006070
6071 if (adj) {
David Ahern790510d2016-10-17 19:15:43 -07006072 adj->ref_nr += 1;
David Ahern67b62f92016-10-17 19:15:53 -07006073 pr_debug("Insert adjacency: dev %s adj_dev %s adj->ref_nr %d\n",
6074 dev->name, adj_dev->name, adj->ref_nr);
6075
Veaceslav Falico5d261912013-08-28 23:25:05 +02006076 return 0;
6077 }
6078
6079 adj = kmalloc(sizeof(*adj), GFP_KERNEL);
6080 if (!adj)
6081 return -ENOMEM;
6082
6083 adj->dev = adj_dev;
6084 adj->master = master;
David Ahern790510d2016-10-17 19:15:43 -07006085 adj->ref_nr = 1;
Veaceslav Falico402dae92013-09-25 09:20:09 +02006086 adj->private = private;
Veaceslav Falico5d261912013-08-28 23:25:05 +02006087 dev_hold(adj_dev);
Veaceslav Falico2f268f12013-09-25 09:20:07 +02006088
David Ahern67b62f92016-10-17 19:15:53 -07006089 pr_debug("Insert adjacency: dev %s adj_dev %s adj->ref_nr %d; dev_hold on %s\n",
6090 dev->name, adj_dev->name, adj->ref_nr, adj_dev->name);
Veaceslav Falico5d261912013-08-28 23:25:05 +02006091
Alexander Y. Fomichev7ce64c72014-09-15 14:22:35 +04006092 if (netdev_adjacent_is_neigh_list(dev, adj_dev, dev_list)) {
Veaceslav Falico3ee32702014-01-14 21:58:50 +01006093 ret = netdev_adjacent_sysfs_add(dev, adj_dev, dev_list);
Veaceslav Falico5831d662013-09-25 09:20:32 +02006094 if (ret)
6095 goto free_adj;
6096 }
6097
Veaceslav Falico7863c052013-09-25 09:20:06 +02006098 /* Ensure that master link is always the first item in list. */
Veaceslav Falico842d67a2013-09-25 09:20:31 +02006099 if (master) {
6100 ret = sysfs_create_link(&(dev->dev.kobj),
6101 &(adj_dev->dev.kobj), "master");
6102 if (ret)
Veaceslav Falico5831d662013-09-25 09:20:32 +02006103 goto remove_symlinks;
Veaceslav Falico842d67a2013-09-25 09:20:31 +02006104
Veaceslav Falico7863c052013-09-25 09:20:06 +02006105 list_add_rcu(&adj->list, dev_list);
Veaceslav Falico842d67a2013-09-25 09:20:31 +02006106 } else {
Veaceslav Falico7863c052013-09-25 09:20:06 +02006107 list_add_tail_rcu(&adj->list, dev_list);
Veaceslav Falico842d67a2013-09-25 09:20:31 +02006108 }
Veaceslav Falico5d261912013-08-28 23:25:05 +02006109
6110 return 0;
Veaceslav Falico842d67a2013-09-25 09:20:31 +02006111
Veaceslav Falico5831d662013-09-25 09:20:32 +02006112remove_symlinks:
Alexander Y. Fomichev7ce64c72014-09-15 14:22:35 +04006113 if (netdev_adjacent_is_neigh_list(dev, adj_dev, dev_list))
Veaceslav Falico3ee32702014-01-14 21:58:50 +01006114 netdev_adjacent_sysfs_del(dev, adj_dev->name, dev_list);
Veaceslav Falico842d67a2013-09-25 09:20:31 +02006115free_adj:
6116 kfree(adj);
Nikolay Aleksandrov974daef2013-10-23 15:28:56 +02006117 dev_put(adj_dev);
Veaceslav Falico842d67a2013-09-25 09:20:31 +02006118
6119 return ret;
Veaceslav Falico5d261912013-08-28 23:25:05 +02006120}
6121
stephen hemminger1d143d92013-12-29 14:01:29 -08006122static void __netdev_adjacent_dev_remove(struct net_device *dev,
6123 struct net_device *adj_dev,
Andrew Collins93409032016-10-03 13:43:02 -06006124 u16 ref_nr,
stephen hemminger1d143d92013-12-29 14:01:29 -08006125 struct list_head *dev_list)
Veaceslav Falico5d261912013-08-28 23:25:05 +02006126{
6127 struct netdev_adjacent *adj;
6128
David Ahern67b62f92016-10-17 19:15:53 -07006129 pr_debug("Remove adjacency: dev %s adj_dev %s ref_nr %d\n",
6130 dev->name, adj_dev->name, ref_nr);
6131
Michal Kubeček6ea29da2015-09-24 10:59:05 +02006132 adj = __netdev_find_adj(adj_dev, dev_list);
Veaceslav Falico5d261912013-08-28 23:25:05 +02006133
Veaceslav Falico2f268f12013-09-25 09:20:07 +02006134 if (!adj) {
David Ahern67b62f92016-10-17 19:15:53 -07006135 pr_err("Adjacency does not exist for device %s from %s\n",
Veaceslav Falico2f268f12013-09-25 09:20:07 +02006136 dev->name, adj_dev->name);
David Ahern67b62f92016-10-17 19:15:53 -07006137 WARN_ON(1);
6138 return;
Veaceslav Falico2f268f12013-09-25 09:20:07 +02006139 }
Veaceslav Falico5d261912013-08-28 23:25:05 +02006140
Andrew Collins93409032016-10-03 13:43:02 -06006141 if (adj->ref_nr > ref_nr) {
David Ahern67b62f92016-10-17 19:15:53 -07006142 pr_debug("adjacency: %s to %s ref_nr - %d = %d\n",
6143 dev->name, adj_dev->name, ref_nr,
6144 adj->ref_nr - ref_nr);
Andrew Collins93409032016-10-03 13:43:02 -06006145 adj->ref_nr -= ref_nr;
Veaceslav Falico5d261912013-08-28 23:25:05 +02006146 return;
6147 }
6148
Veaceslav Falico842d67a2013-09-25 09:20:31 +02006149 if (adj->master)
6150 sysfs_remove_link(&(dev->dev.kobj), "master");
6151
Alexander Y. Fomichev7ce64c72014-09-15 14:22:35 +04006152 if (netdev_adjacent_is_neigh_list(dev, adj_dev, dev_list))
Veaceslav Falico3ee32702014-01-14 21:58:50 +01006153 netdev_adjacent_sysfs_del(dev, adj_dev->name, dev_list);
Veaceslav Falico5831d662013-09-25 09:20:32 +02006154
Veaceslav Falico5d261912013-08-28 23:25:05 +02006155 list_del_rcu(&adj->list);
David Ahern67b62f92016-10-17 19:15:53 -07006156 pr_debug("adjacency: dev_put for %s, because link removed from %s to %s\n",
Veaceslav Falico2f268f12013-09-25 09:20:07 +02006157 adj_dev->name, dev->name, adj_dev->name);
Veaceslav Falico5d261912013-08-28 23:25:05 +02006158 dev_put(adj_dev);
6159 kfree_rcu(adj, rcu);
6160}
6161
stephen hemminger1d143d92013-12-29 14:01:29 -08006162static int __netdev_adjacent_dev_link_lists(struct net_device *dev,
6163 struct net_device *upper_dev,
6164 struct list_head *up_list,
6165 struct list_head *down_list,
6166 void *private, bool master)
Veaceslav Falico5d261912013-08-28 23:25:05 +02006167{
6168 int ret;
6169
David Ahern790510d2016-10-17 19:15:43 -07006170 ret = __netdev_adjacent_dev_insert(dev, upper_dev, up_list,
Andrew Collins93409032016-10-03 13:43:02 -06006171 private, master);
Veaceslav Falico5d261912013-08-28 23:25:05 +02006172 if (ret)
6173 return ret;
6174
David Ahern790510d2016-10-17 19:15:43 -07006175 ret = __netdev_adjacent_dev_insert(upper_dev, dev, down_list,
Andrew Collins93409032016-10-03 13:43:02 -06006176 private, false);
Veaceslav Falico5d261912013-08-28 23:25:05 +02006177 if (ret) {
David Ahern790510d2016-10-17 19:15:43 -07006178 __netdev_adjacent_dev_remove(dev, upper_dev, 1, up_list);
Veaceslav Falico5d261912013-08-28 23:25:05 +02006179 return ret;
6180 }
6181
6182 return 0;
6183}
6184
stephen hemminger1d143d92013-12-29 14:01:29 -08006185static void __netdev_adjacent_dev_unlink_lists(struct net_device *dev,
6186 struct net_device *upper_dev,
Andrew Collins93409032016-10-03 13:43:02 -06006187 u16 ref_nr,
stephen hemminger1d143d92013-12-29 14:01:29 -08006188 struct list_head *up_list,
6189 struct list_head *down_list)
Veaceslav Falico5d261912013-08-28 23:25:05 +02006190{
Andrew Collins93409032016-10-03 13:43:02 -06006191 __netdev_adjacent_dev_remove(dev, upper_dev, ref_nr, up_list);
6192 __netdev_adjacent_dev_remove(upper_dev, dev, ref_nr, down_list);
Veaceslav Falico5d261912013-08-28 23:25:05 +02006193}
6194
stephen hemminger1d143d92013-12-29 14:01:29 -08006195static int __netdev_adjacent_dev_link_neighbour(struct net_device *dev,
6196 struct net_device *upper_dev,
6197 void *private, bool master)
Veaceslav Falico2f268f12013-09-25 09:20:07 +02006198{
David Ahernf1170fd2016-10-17 19:15:51 -07006199 return __netdev_adjacent_dev_link_lists(dev, upper_dev,
6200 &dev->adj_list.upper,
6201 &upper_dev->adj_list.lower,
6202 private, master);
Veaceslav Falico2f268f12013-09-25 09:20:07 +02006203}
6204
stephen hemminger1d143d92013-12-29 14:01:29 -08006205static void __netdev_adjacent_dev_unlink_neighbour(struct net_device *dev,
6206 struct net_device *upper_dev)
Veaceslav Falico2f268f12013-09-25 09:20:07 +02006207{
Andrew Collins93409032016-10-03 13:43:02 -06006208 __netdev_adjacent_dev_unlink_lists(dev, upper_dev, 1,
Veaceslav Falico2f268f12013-09-25 09:20:07 +02006209 &dev->adj_list.upper,
6210 &upper_dev->adj_list.lower);
6211}
Veaceslav Falico5d261912013-08-28 23:25:05 +02006212
Jiri Pirko9ff162a2013-01-03 22:48:49 +00006213static int __netdev_upper_dev_link(struct net_device *dev,
Veaceslav Falico402dae92013-09-25 09:20:09 +02006214 struct net_device *upper_dev, bool master,
Jiri Pirko29bf24a2015-12-03 12:12:11 +01006215 void *upper_priv, void *upper_info)
Jiri Pirko9ff162a2013-01-03 22:48:49 +00006216{
Jiri Pirko0e4ead92015-08-27 09:31:18 +02006217 struct netdev_notifier_changeupper_info changeupper_info;
Veaceslav Falico5d261912013-08-28 23:25:05 +02006218 int ret = 0;
Jiri Pirko9ff162a2013-01-03 22:48:49 +00006219
6220 ASSERT_RTNL();
6221
6222 if (dev == upper_dev)
6223 return -EBUSY;
6224
6225 /* To prevent loops, check if dev is not upper device to upper_dev. */
David Ahernf1170fd2016-10-17 19:15:51 -07006226 if (netdev_has_upper_dev(upper_dev, dev))
Jiri Pirko9ff162a2013-01-03 22:48:49 +00006227 return -EBUSY;
6228
David Ahernf1170fd2016-10-17 19:15:51 -07006229 if (netdev_has_upper_dev(dev, upper_dev))
Jiri Pirko9ff162a2013-01-03 22:48:49 +00006230 return -EEXIST;
6231
6232 if (master && netdev_master_upper_dev_get(dev))
6233 return -EBUSY;
6234
Jiri Pirko0e4ead92015-08-27 09:31:18 +02006235 changeupper_info.upper_dev = upper_dev;
6236 changeupper_info.master = master;
6237 changeupper_info.linking = true;
Jiri Pirko29bf24a2015-12-03 12:12:11 +01006238 changeupper_info.upper_info = upper_info;
Jiri Pirko0e4ead92015-08-27 09:31:18 +02006239
Jiri Pirko573c7ba2015-10-16 14:01:22 +02006240 ret = call_netdevice_notifiers_info(NETDEV_PRECHANGEUPPER, dev,
6241 &changeupper_info.info);
6242 ret = notifier_to_errno(ret);
6243 if (ret)
6244 return ret;
6245
Jiri Pirko6dffb042015-12-03 12:12:10 +01006246 ret = __netdev_adjacent_dev_link_neighbour(dev, upper_dev, upper_priv,
Veaceslav Falico402dae92013-09-25 09:20:09 +02006247 master);
Veaceslav Falico5d261912013-08-28 23:25:05 +02006248 if (ret)
6249 return ret;
Jiri Pirko9ff162a2013-01-03 22:48:49 +00006250
Ido Schimmelb03804e2015-12-03 12:12:03 +01006251 ret = call_netdevice_notifiers_info(NETDEV_CHANGEUPPER, dev,
6252 &changeupper_info.info);
6253 ret = notifier_to_errno(ret);
6254 if (ret)
David Ahernf1170fd2016-10-17 19:15:51 -07006255 goto rollback;
Ido Schimmelb03804e2015-12-03 12:12:03 +01006256
Jiri Pirko9ff162a2013-01-03 22:48:49 +00006257 return 0;
Veaceslav Falico5d261912013-08-28 23:25:05 +02006258
David Ahernf1170fd2016-10-17 19:15:51 -07006259rollback:
Veaceslav Falico2f268f12013-09-25 09:20:07 +02006260 __netdev_adjacent_dev_unlink_neighbour(dev, upper_dev);
Veaceslav Falico5d261912013-08-28 23:25:05 +02006261
6262 return ret;
Jiri Pirko9ff162a2013-01-03 22:48:49 +00006263}
6264
6265/**
6266 * netdev_upper_dev_link - Add a link to the upper device
6267 * @dev: device
6268 * @upper_dev: new upper device
6269 *
6270 * Adds a link to device which is upper to this one. The caller must hold
6271 * the RTNL lock. On a failure a negative errno code is returned.
6272 * On success the reference counts are adjusted and the function
6273 * returns zero.
6274 */
6275int netdev_upper_dev_link(struct net_device *dev,
6276 struct net_device *upper_dev)
6277{
Jiri Pirko29bf24a2015-12-03 12:12:11 +01006278 return __netdev_upper_dev_link(dev, upper_dev, false, NULL, NULL);
Jiri Pirko9ff162a2013-01-03 22:48:49 +00006279}
6280EXPORT_SYMBOL(netdev_upper_dev_link);
6281
6282/**
6283 * netdev_master_upper_dev_link - Add a master link to the upper device
6284 * @dev: device
6285 * @upper_dev: new upper device
Jiri Pirko6dffb042015-12-03 12:12:10 +01006286 * @upper_priv: upper device private
Jiri Pirko29bf24a2015-12-03 12:12:11 +01006287 * @upper_info: upper info to be passed down via notifier
Jiri Pirko9ff162a2013-01-03 22:48:49 +00006288 *
6289 * Adds a link to device which is upper to this one. In this case, only
6290 * one master upper device can be linked, although other non-master devices
6291 * might be linked as well. The caller must hold the RTNL lock.
6292 * On a failure a negative errno code is returned. On success the reference
6293 * counts are adjusted and the function returns zero.
6294 */
6295int netdev_master_upper_dev_link(struct net_device *dev,
Jiri Pirko6dffb042015-12-03 12:12:10 +01006296 struct net_device *upper_dev,
Jiri Pirko29bf24a2015-12-03 12:12:11 +01006297 void *upper_priv, void *upper_info)
Jiri Pirko9ff162a2013-01-03 22:48:49 +00006298{
Jiri Pirko29bf24a2015-12-03 12:12:11 +01006299 return __netdev_upper_dev_link(dev, upper_dev, true,
6300 upper_priv, upper_info);
Jiri Pirko9ff162a2013-01-03 22:48:49 +00006301}
6302EXPORT_SYMBOL(netdev_master_upper_dev_link);
6303
6304/**
6305 * netdev_upper_dev_unlink - Removes a link to upper device
6306 * @dev: device
6307 * @upper_dev: new upper device
6308 *
6309 * Removes a link to device which is upper to this one. The caller must hold
6310 * the RTNL lock.
6311 */
6312void netdev_upper_dev_unlink(struct net_device *dev,
6313 struct net_device *upper_dev)
6314{
Jiri Pirko0e4ead92015-08-27 09:31:18 +02006315 struct netdev_notifier_changeupper_info changeupper_info;
tchardingf4563a72017-02-09 17:56:07 +11006316
Jiri Pirko9ff162a2013-01-03 22:48:49 +00006317 ASSERT_RTNL();
6318
Jiri Pirko0e4ead92015-08-27 09:31:18 +02006319 changeupper_info.upper_dev = upper_dev;
6320 changeupper_info.master = netdev_master_upper_dev_get(dev) == upper_dev;
6321 changeupper_info.linking = false;
6322
Jiri Pirko573c7ba2015-10-16 14:01:22 +02006323 call_netdevice_notifiers_info(NETDEV_PRECHANGEUPPER, dev,
6324 &changeupper_info.info);
6325
Veaceslav Falico2f268f12013-09-25 09:20:07 +02006326 __netdev_adjacent_dev_unlink_neighbour(dev, upper_dev);
Veaceslav Falico5d261912013-08-28 23:25:05 +02006327
Jiri Pirko0e4ead92015-08-27 09:31:18 +02006328 call_netdevice_notifiers_info(NETDEV_CHANGEUPPER, dev,
6329 &changeupper_info.info);
Jiri Pirko9ff162a2013-01-03 22:48:49 +00006330}
6331EXPORT_SYMBOL(netdev_upper_dev_unlink);
6332
Moni Shoua61bd3852015-02-03 16:48:29 +02006333/**
6334 * netdev_bonding_info_change - Dispatch event about slave change
6335 * @dev: device
Masanari Iida4a26e4532015-02-14 22:26:34 +09006336 * @bonding_info: info to dispatch
Moni Shoua61bd3852015-02-03 16:48:29 +02006337 *
6338 * Send NETDEV_BONDING_INFO to netdev notifiers with info.
6339 * The caller must hold the RTNL lock.
6340 */
6341void netdev_bonding_info_change(struct net_device *dev,
6342 struct netdev_bonding_info *bonding_info)
6343{
6344 struct netdev_notifier_bonding_info info;
6345
6346 memcpy(&info.bonding_info, bonding_info,
6347 sizeof(struct netdev_bonding_info));
6348 call_netdevice_notifiers_info(NETDEV_BONDING_INFO, dev,
6349 &info.info);
6350}
6351EXPORT_SYMBOL(netdev_bonding_info_change);
6352
Eric Dumazet2ce1ee12015-02-04 13:37:44 -08006353static void netdev_adjacent_add_links(struct net_device *dev)
Alexander Y. Fomichev4c754312014-08-25 16:26:45 +04006354{
6355 struct netdev_adjacent *iter;
6356
6357 struct net *net = dev_net(dev);
6358
6359 list_for_each_entry(iter, &dev->adj_list.upper, list) {
Wei Tangbe4da0e2016-06-16 21:30:12 +08006360 if (!net_eq(net, dev_net(iter->dev)))
Alexander Y. Fomichev4c754312014-08-25 16:26:45 +04006361 continue;
6362 netdev_adjacent_sysfs_add(iter->dev, dev,
6363 &iter->dev->adj_list.lower);
6364 netdev_adjacent_sysfs_add(dev, iter->dev,
6365 &dev->adj_list.upper);
6366 }
6367
6368 list_for_each_entry(iter, &dev->adj_list.lower, list) {
Wei Tangbe4da0e2016-06-16 21:30:12 +08006369 if (!net_eq(net, dev_net(iter->dev)))
Alexander Y. Fomichev4c754312014-08-25 16:26:45 +04006370 continue;
6371 netdev_adjacent_sysfs_add(iter->dev, dev,
6372 &iter->dev->adj_list.upper);
6373 netdev_adjacent_sysfs_add(dev, iter->dev,
6374 &dev->adj_list.lower);
6375 }
6376}
6377
Eric Dumazet2ce1ee12015-02-04 13:37:44 -08006378static void netdev_adjacent_del_links(struct net_device *dev)
Alexander Y. Fomichev4c754312014-08-25 16:26:45 +04006379{
6380 struct netdev_adjacent *iter;
6381
6382 struct net *net = dev_net(dev);
6383
6384 list_for_each_entry(iter, &dev->adj_list.upper, list) {
Wei Tangbe4da0e2016-06-16 21:30:12 +08006385 if (!net_eq(net, dev_net(iter->dev)))
Alexander Y. Fomichev4c754312014-08-25 16:26:45 +04006386 continue;
6387 netdev_adjacent_sysfs_del(iter->dev, dev->name,
6388 &iter->dev->adj_list.lower);
6389 netdev_adjacent_sysfs_del(dev, iter->dev->name,
6390 &dev->adj_list.upper);
6391 }
6392
6393 list_for_each_entry(iter, &dev->adj_list.lower, list) {
Wei Tangbe4da0e2016-06-16 21:30:12 +08006394 if (!net_eq(net, dev_net(iter->dev)))
Alexander Y. Fomichev4c754312014-08-25 16:26:45 +04006395 continue;
6396 netdev_adjacent_sysfs_del(iter->dev, dev->name,
6397 &iter->dev->adj_list.upper);
6398 netdev_adjacent_sysfs_del(dev, iter->dev->name,
6399 &dev->adj_list.lower);
6400 }
6401}
6402
Veaceslav Falico5bb025f2014-01-14 21:58:51 +01006403void netdev_adjacent_rename_links(struct net_device *dev, char *oldname)
Veaceslav Falico402dae92013-09-25 09:20:09 +02006404{
Veaceslav Falico5bb025f2014-01-14 21:58:51 +01006405 struct netdev_adjacent *iter;
Veaceslav Falico402dae92013-09-25 09:20:09 +02006406
Alexander Y. Fomichev4c754312014-08-25 16:26:45 +04006407 struct net *net = dev_net(dev);
6408
Veaceslav Falico5bb025f2014-01-14 21:58:51 +01006409 list_for_each_entry(iter, &dev->adj_list.upper, list) {
Wei Tangbe4da0e2016-06-16 21:30:12 +08006410 if (!net_eq(net, dev_net(iter->dev)))
Alexander Y. Fomichev4c754312014-08-25 16:26:45 +04006411 continue;
Veaceslav Falico5bb025f2014-01-14 21:58:51 +01006412 netdev_adjacent_sysfs_del(iter->dev, oldname,
6413 &iter->dev->adj_list.lower);
6414 netdev_adjacent_sysfs_add(iter->dev, dev,
6415 &iter->dev->adj_list.lower);
6416 }
Veaceslav Falico402dae92013-09-25 09:20:09 +02006417
Veaceslav Falico5bb025f2014-01-14 21:58:51 +01006418 list_for_each_entry(iter, &dev->adj_list.lower, list) {
Wei Tangbe4da0e2016-06-16 21:30:12 +08006419 if (!net_eq(net, dev_net(iter->dev)))
Alexander Y. Fomichev4c754312014-08-25 16:26:45 +04006420 continue;
Veaceslav Falico5bb025f2014-01-14 21:58:51 +01006421 netdev_adjacent_sysfs_del(iter->dev, oldname,
6422 &iter->dev->adj_list.upper);
6423 netdev_adjacent_sysfs_add(iter->dev, dev,
6424 &iter->dev->adj_list.upper);
6425 }
Veaceslav Falico402dae92013-09-25 09:20:09 +02006426}
Veaceslav Falico402dae92013-09-25 09:20:09 +02006427
6428void *netdev_lower_dev_get_private(struct net_device *dev,
6429 struct net_device *lower_dev)
6430{
6431 struct netdev_adjacent *lower;
6432
6433 if (!lower_dev)
6434 return NULL;
Michal Kubeček6ea29da2015-09-24 10:59:05 +02006435 lower = __netdev_find_adj(lower_dev, &dev->adj_list.lower);
Veaceslav Falico402dae92013-09-25 09:20:09 +02006436 if (!lower)
6437 return NULL;
6438
6439 return lower->private;
6440}
6441EXPORT_SYMBOL(netdev_lower_dev_get_private);
6442
Vlad Yasevich4085ebe2014-05-16 17:04:53 -04006443
Sabrina Dubroca952fcfd2016-08-12 16:10:33 +02006444int dev_get_nest_level(struct net_device *dev)
Vlad Yasevich4085ebe2014-05-16 17:04:53 -04006445{
6446 struct net_device *lower = NULL;
6447 struct list_head *iter;
6448 int max_nest = -1;
6449 int nest;
6450
6451 ASSERT_RTNL();
6452
6453 netdev_for_each_lower_dev(dev, lower, iter) {
Sabrina Dubroca952fcfd2016-08-12 16:10:33 +02006454 nest = dev_get_nest_level(lower);
Vlad Yasevich4085ebe2014-05-16 17:04:53 -04006455 if (max_nest < nest)
6456 max_nest = nest;
6457 }
6458
Sabrina Dubroca952fcfd2016-08-12 16:10:33 +02006459 return max_nest + 1;
Vlad Yasevich4085ebe2014-05-16 17:04:53 -04006460}
6461EXPORT_SYMBOL(dev_get_nest_level);
6462
Jiri Pirko04d48262015-12-03 12:12:15 +01006463/**
6464 * netdev_lower_change - Dispatch event about lower device state change
6465 * @lower_dev: device
6466 * @lower_state_info: state to dispatch
6467 *
6468 * Send NETDEV_CHANGELOWERSTATE to netdev notifiers with info.
6469 * The caller must hold the RTNL lock.
6470 */
6471void netdev_lower_state_changed(struct net_device *lower_dev,
6472 void *lower_state_info)
6473{
6474 struct netdev_notifier_changelowerstate_info changelowerstate_info;
6475
6476 ASSERT_RTNL();
6477 changelowerstate_info.lower_state_info = lower_state_info;
6478 call_netdevice_notifiers_info(NETDEV_CHANGELOWERSTATE, lower_dev,
6479 &changelowerstate_info.info);
6480}
6481EXPORT_SYMBOL(netdev_lower_state_changed);
6482
Patrick McHardyb6c40d62008-10-07 15:26:48 -07006483static void dev_change_rx_flags(struct net_device *dev, int flags)
6484{
Stephen Hemmingerd3147742008-11-19 21:32:24 -08006485 const struct net_device_ops *ops = dev->netdev_ops;
6486
Vlad Yasevichd2615bf2013-11-19 20:47:15 -05006487 if (ops->ndo_change_rx_flags)
Stephen Hemmingerd3147742008-11-19 21:32:24 -08006488 ops->ndo_change_rx_flags(dev, flags);
Patrick McHardyb6c40d62008-10-07 15:26:48 -07006489}
6490
Nicolas Dichtel991fb3f2013-09-25 12:02:45 +02006491static int __dev_set_promiscuity(struct net_device *dev, int inc, bool notify)
Patrick McHardy4417da62007-06-27 01:28:10 -07006492{
Eric Dumazetb536db92011-11-30 21:42:26 +00006493 unsigned int old_flags = dev->flags;
Eric W. Biedermand04a48b2012-05-23 17:01:57 -06006494 kuid_t uid;
6495 kgid_t gid;
Patrick McHardy4417da62007-06-27 01:28:10 -07006496
Patrick McHardy24023452007-07-14 18:51:31 -07006497 ASSERT_RTNL();
6498
Wang Chendad9b332008-06-18 01:48:28 -07006499 dev->flags |= IFF_PROMISC;
6500 dev->promiscuity += inc;
6501 if (dev->promiscuity == 0) {
6502 /*
6503 * Avoid overflow.
6504 * If inc causes overflow, untouch promisc and return error.
6505 */
6506 if (inc < 0)
6507 dev->flags &= ~IFF_PROMISC;
6508 else {
6509 dev->promiscuity -= inc;
Joe Perches7b6cd1c2012-02-01 10:54:43 +00006510 pr_warn("%s: promiscuity touches roof, set promiscuity failed. promiscuity feature of device might be broken.\n",
6511 dev->name);
Wang Chendad9b332008-06-18 01:48:28 -07006512 return -EOVERFLOW;
6513 }
6514 }
Patrick McHardy4417da62007-06-27 01:28:10 -07006515 if (dev->flags != old_flags) {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00006516 pr_info("device %s %s promiscuous mode\n",
6517 dev->name,
6518 dev->flags & IFF_PROMISC ? "entered" : "left");
David Howells8192b0c2008-11-14 10:39:10 +11006519 if (audit_enabled) {
6520 current_uid_gid(&uid, &gid);
Klaus Heinrich Kiwi7759db82008-01-23 22:57:45 -05006521 audit_log(current->audit_context, GFP_ATOMIC,
6522 AUDIT_ANOM_PROMISCUOUS,
6523 "dev=%s prom=%d old_prom=%d auid=%u uid=%u gid=%u ses=%u",
6524 dev->name, (dev->flags & IFF_PROMISC),
6525 (old_flags & IFF_PROMISC),
Eric W. Biedermane1760bd2012-09-10 22:39:43 -07006526 from_kuid(&init_user_ns, audit_get_loginuid(current)),
Eric W. Biedermand04a48b2012-05-23 17:01:57 -06006527 from_kuid(&init_user_ns, uid),
6528 from_kgid(&init_user_ns, gid),
Klaus Heinrich Kiwi7759db82008-01-23 22:57:45 -05006529 audit_get_sessionid(current));
David Howells8192b0c2008-11-14 10:39:10 +11006530 }
Patrick McHardy24023452007-07-14 18:51:31 -07006531
Patrick McHardyb6c40d62008-10-07 15:26:48 -07006532 dev_change_rx_flags(dev, IFF_PROMISC);
Patrick McHardy4417da62007-06-27 01:28:10 -07006533 }
Nicolas Dichtel991fb3f2013-09-25 12:02:45 +02006534 if (notify)
6535 __dev_notify_flags(dev, old_flags, IFF_PROMISC);
Wang Chendad9b332008-06-18 01:48:28 -07006536 return 0;
Patrick McHardy4417da62007-06-27 01:28:10 -07006537}
6538
Linus Torvalds1da177e2005-04-16 15:20:36 -07006539/**
6540 * dev_set_promiscuity - update promiscuity count on a device
6541 * @dev: device
6542 * @inc: modifier
6543 *
Stephen Hemminger3041a062006-05-26 13:25:24 -07006544 * Add or remove promiscuity from a device. While the count in the device
Linus Torvalds1da177e2005-04-16 15:20:36 -07006545 * remains above zero the interface remains promiscuous. Once it hits zero
6546 * the device reverts back to normal filtering operation. A negative inc
6547 * value is used to drop promiscuity on the device.
Wang Chendad9b332008-06-18 01:48:28 -07006548 * Return 0 if successful or a negative errno code on error.
Linus Torvalds1da177e2005-04-16 15:20:36 -07006549 */
Wang Chendad9b332008-06-18 01:48:28 -07006550int dev_set_promiscuity(struct net_device *dev, int inc)
Linus Torvalds1da177e2005-04-16 15:20:36 -07006551{
Eric Dumazetb536db92011-11-30 21:42:26 +00006552 unsigned int old_flags = dev->flags;
Wang Chendad9b332008-06-18 01:48:28 -07006553 int err;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006554
Nicolas Dichtel991fb3f2013-09-25 12:02:45 +02006555 err = __dev_set_promiscuity(dev, inc, true);
Patrick McHardy4b5a6982008-07-06 15:49:08 -07006556 if (err < 0)
Wang Chendad9b332008-06-18 01:48:28 -07006557 return err;
Patrick McHardy4417da62007-06-27 01:28:10 -07006558 if (dev->flags != old_flags)
6559 dev_set_rx_mode(dev);
Wang Chendad9b332008-06-18 01:48:28 -07006560 return err;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006561}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07006562EXPORT_SYMBOL(dev_set_promiscuity);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006563
Nicolas Dichtel991fb3f2013-09-25 12:02:45 +02006564static int __dev_set_allmulti(struct net_device *dev, int inc, bool notify)
Linus Torvalds1da177e2005-04-16 15:20:36 -07006565{
Nicolas Dichtel991fb3f2013-09-25 12:02:45 +02006566 unsigned int old_flags = dev->flags, old_gflags = dev->gflags;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006567
Patrick McHardy24023452007-07-14 18:51:31 -07006568 ASSERT_RTNL();
6569
Linus Torvalds1da177e2005-04-16 15:20:36 -07006570 dev->flags |= IFF_ALLMULTI;
Wang Chendad9b332008-06-18 01:48:28 -07006571 dev->allmulti += inc;
6572 if (dev->allmulti == 0) {
6573 /*
6574 * Avoid overflow.
6575 * If inc causes overflow, untouch allmulti and return error.
6576 */
6577 if (inc < 0)
6578 dev->flags &= ~IFF_ALLMULTI;
6579 else {
6580 dev->allmulti -= inc;
Joe Perches7b6cd1c2012-02-01 10:54:43 +00006581 pr_warn("%s: allmulti touches roof, set allmulti failed. allmulti feature of device might be broken.\n",
6582 dev->name);
Wang Chendad9b332008-06-18 01:48:28 -07006583 return -EOVERFLOW;
6584 }
6585 }
Patrick McHardy24023452007-07-14 18:51:31 -07006586 if (dev->flags ^ old_flags) {
Patrick McHardyb6c40d62008-10-07 15:26:48 -07006587 dev_change_rx_flags(dev, IFF_ALLMULTI);
Patrick McHardy4417da62007-06-27 01:28:10 -07006588 dev_set_rx_mode(dev);
Nicolas Dichtel991fb3f2013-09-25 12:02:45 +02006589 if (notify)
6590 __dev_notify_flags(dev, old_flags,
6591 dev->gflags ^ old_gflags);
Patrick McHardy24023452007-07-14 18:51:31 -07006592 }
Wang Chendad9b332008-06-18 01:48:28 -07006593 return 0;
Patrick McHardy4417da62007-06-27 01:28:10 -07006594}
Nicolas Dichtel991fb3f2013-09-25 12:02:45 +02006595
6596/**
6597 * dev_set_allmulti - update allmulti count on a device
6598 * @dev: device
6599 * @inc: modifier
6600 *
6601 * Add or remove reception of all multicast frames to a device. While the
6602 * count in the device remains above zero the interface remains listening
6603 * to all interfaces. Once it hits zero the device reverts back to normal
6604 * filtering operation. A negative @inc value is used to drop the counter
6605 * when releasing a resource needing all multicasts.
6606 * Return 0 if successful or a negative errno code on error.
6607 */
6608
6609int dev_set_allmulti(struct net_device *dev, int inc)
6610{
6611 return __dev_set_allmulti(dev, inc, true);
6612}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07006613EXPORT_SYMBOL(dev_set_allmulti);
Patrick McHardy4417da62007-06-27 01:28:10 -07006614
6615/*
6616 * Upload unicast and multicast address lists to device and
6617 * configure RX filtering. When the device doesn't support unicast
Joe Perches53ccaae2007-12-20 14:02:06 -08006618 * filtering it is put in promiscuous mode while unicast addresses
Patrick McHardy4417da62007-06-27 01:28:10 -07006619 * are present.
6620 */
6621void __dev_set_rx_mode(struct net_device *dev)
6622{
Stephen Hemmingerd3147742008-11-19 21:32:24 -08006623 const struct net_device_ops *ops = dev->netdev_ops;
6624
Patrick McHardy4417da62007-06-27 01:28:10 -07006625 /* dev_open will call this function so the list will stay sane. */
6626 if (!(dev->flags&IFF_UP))
6627 return;
6628
6629 if (!netif_device_present(dev))
YOSHIFUJI Hideaki40b77c92007-07-19 10:43:23 +09006630 return;
Patrick McHardy4417da62007-06-27 01:28:10 -07006631
Jiri Pirko01789342011-08-16 06:29:00 +00006632 if (!(dev->priv_flags & IFF_UNICAST_FLT)) {
Patrick McHardy4417da62007-06-27 01:28:10 -07006633 /* Unicast addresses changes may only happen under the rtnl,
6634 * therefore calling __dev_set_promiscuity here is safe.
6635 */
Jiri Pirko32e7bfc2010-01-25 13:36:10 -08006636 if (!netdev_uc_empty(dev) && !dev->uc_promisc) {
Nicolas Dichtel991fb3f2013-09-25 12:02:45 +02006637 __dev_set_promiscuity(dev, 1, false);
Joe Perches2d348d12011-07-25 16:17:35 -07006638 dev->uc_promisc = true;
Jiri Pirko32e7bfc2010-01-25 13:36:10 -08006639 } else if (netdev_uc_empty(dev) && dev->uc_promisc) {
Nicolas Dichtel991fb3f2013-09-25 12:02:45 +02006640 __dev_set_promiscuity(dev, -1, false);
Joe Perches2d348d12011-07-25 16:17:35 -07006641 dev->uc_promisc = false;
Patrick McHardy4417da62007-06-27 01:28:10 -07006642 }
Patrick McHardy4417da62007-06-27 01:28:10 -07006643 }
Jiri Pirko01789342011-08-16 06:29:00 +00006644
6645 if (ops->ndo_set_rx_mode)
6646 ops->ndo_set_rx_mode(dev);
Patrick McHardy4417da62007-06-27 01:28:10 -07006647}
6648
6649void dev_set_rx_mode(struct net_device *dev)
6650{
David S. Millerb9e40852008-07-15 00:15:08 -07006651 netif_addr_lock_bh(dev);
Patrick McHardy4417da62007-06-27 01:28:10 -07006652 __dev_set_rx_mode(dev);
David S. Millerb9e40852008-07-15 00:15:08 -07006653 netif_addr_unlock_bh(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006654}
6655
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07006656/**
6657 * dev_get_flags - get flags reported to userspace
6658 * @dev: device
6659 *
6660 * Get the combination of flag bits exported through APIs to userspace.
6661 */
Eric Dumazet95c96172012-04-15 05:58:06 +00006662unsigned int dev_get_flags(const struct net_device *dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07006663{
Eric Dumazet95c96172012-04-15 05:58:06 +00006664 unsigned int flags;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006665
6666 flags = (dev->flags & ~(IFF_PROMISC |
6667 IFF_ALLMULTI |
Stefan Rompfb00055a2006-03-20 17:09:11 -08006668 IFF_RUNNING |
6669 IFF_LOWER_UP |
6670 IFF_DORMANT)) |
Linus Torvalds1da177e2005-04-16 15:20:36 -07006671 (dev->gflags & (IFF_PROMISC |
6672 IFF_ALLMULTI));
6673
Stefan Rompfb00055a2006-03-20 17:09:11 -08006674 if (netif_running(dev)) {
6675 if (netif_oper_up(dev))
6676 flags |= IFF_RUNNING;
6677 if (netif_carrier_ok(dev))
6678 flags |= IFF_LOWER_UP;
6679 if (netif_dormant(dev))
6680 flags |= IFF_DORMANT;
6681 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07006682
6683 return flags;
6684}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07006685EXPORT_SYMBOL(dev_get_flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006686
Patrick McHardybd380812010-02-26 06:34:53 +00006687int __dev_change_flags(struct net_device *dev, unsigned int flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -07006688{
Eric Dumazetb536db92011-11-30 21:42:26 +00006689 unsigned int old_flags = dev->flags;
Patrick McHardybd380812010-02-26 06:34:53 +00006690 int ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006691
Patrick McHardy24023452007-07-14 18:51:31 -07006692 ASSERT_RTNL();
6693
Linus Torvalds1da177e2005-04-16 15:20:36 -07006694 /*
6695 * Set the flags on our device.
6696 */
6697
6698 dev->flags = (flags & (IFF_DEBUG | IFF_NOTRAILERS | IFF_NOARP |
6699 IFF_DYNAMIC | IFF_MULTICAST | IFF_PORTSEL |
6700 IFF_AUTOMEDIA)) |
6701 (dev->flags & (IFF_UP | IFF_VOLATILE | IFF_PROMISC |
6702 IFF_ALLMULTI));
6703
6704 /*
6705 * Load in the correct multicast list now the flags have changed.
6706 */
6707
Patrick McHardyb6c40d62008-10-07 15:26:48 -07006708 if ((old_flags ^ flags) & IFF_MULTICAST)
6709 dev_change_rx_flags(dev, IFF_MULTICAST);
Patrick McHardy24023452007-07-14 18:51:31 -07006710
Patrick McHardy4417da62007-06-27 01:28:10 -07006711 dev_set_rx_mode(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006712
6713 /*
6714 * Have we downed the interface. We handle IFF_UP ourselves
6715 * according to user attempts to set it, rather than blindly
6716 * setting it.
6717 */
6718
6719 ret = 0;
stephen hemminger7051b882017-07-18 15:59:27 -07006720 if ((old_flags ^ flags) & IFF_UP) {
6721 if (old_flags & IFF_UP)
6722 __dev_close(dev);
6723 else
6724 ret = __dev_open(dev);
6725 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07006726
Linus Torvalds1da177e2005-04-16 15:20:36 -07006727 if ((flags ^ dev->gflags) & IFF_PROMISC) {
Eric Dumazetd1b19df2009-09-03 01:29:39 -07006728 int inc = (flags & IFF_PROMISC) ? 1 : -1;
Nicolas Dichtel991fb3f2013-09-25 12:02:45 +02006729 unsigned int old_flags = dev->flags;
Eric Dumazetd1b19df2009-09-03 01:29:39 -07006730
Linus Torvalds1da177e2005-04-16 15:20:36 -07006731 dev->gflags ^= IFF_PROMISC;
Nicolas Dichtel991fb3f2013-09-25 12:02:45 +02006732
6733 if (__dev_set_promiscuity(dev, inc, false) >= 0)
6734 if (dev->flags != old_flags)
6735 dev_set_rx_mode(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006736 }
6737
6738 /* NOTE: order of synchronization of IFF_PROMISC and IFF_ALLMULTI
tchardingeb13da12017-02-09 17:56:06 +11006739 * is important. Some (broken) drivers set IFF_PROMISC, when
6740 * IFF_ALLMULTI is requested not asking us and not reporting.
Linus Torvalds1da177e2005-04-16 15:20:36 -07006741 */
6742 if ((flags ^ dev->gflags) & IFF_ALLMULTI) {
Eric Dumazetd1b19df2009-09-03 01:29:39 -07006743 int inc = (flags & IFF_ALLMULTI) ? 1 : -1;
6744
Linus Torvalds1da177e2005-04-16 15:20:36 -07006745 dev->gflags ^= IFF_ALLMULTI;
Nicolas Dichtel991fb3f2013-09-25 12:02:45 +02006746 __dev_set_allmulti(dev, inc, false);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006747 }
6748
Patrick McHardybd380812010-02-26 06:34:53 +00006749 return ret;
6750}
6751
Nicolas Dichtela528c212013-09-25 12:02:44 +02006752void __dev_notify_flags(struct net_device *dev, unsigned int old_flags,
6753 unsigned int gchanges)
Patrick McHardybd380812010-02-26 06:34:53 +00006754{
6755 unsigned int changes = dev->flags ^ old_flags;
6756
Nicolas Dichtela528c212013-09-25 12:02:44 +02006757 if (gchanges)
Alexei Starovoitov7f294052013-10-23 16:02:42 -07006758 rtmsg_ifinfo(RTM_NEWLINK, dev, gchanges, GFP_ATOMIC);
Nicolas Dichtela528c212013-09-25 12:02:44 +02006759
Patrick McHardybd380812010-02-26 06:34:53 +00006760 if (changes & IFF_UP) {
6761 if (dev->flags & IFF_UP)
6762 call_netdevice_notifiers(NETDEV_UP, dev);
6763 else
6764 call_netdevice_notifiers(NETDEV_DOWN, dev);
6765 }
6766
6767 if (dev->flags & IFF_UP &&
Jiri Pirkobe9efd32013-05-28 01:30:22 +00006768 (changes & ~(IFF_UP | IFF_PROMISC | IFF_ALLMULTI | IFF_VOLATILE))) {
6769 struct netdev_notifier_change_info change_info;
6770
6771 change_info.flags_changed = changes;
6772 call_netdevice_notifiers_info(NETDEV_CHANGE, dev,
6773 &change_info.info);
6774 }
Patrick McHardybd380812010-02-26 06:34:53 +00006775}
6776
6777/**
6778 * dev_change_flags - change device settings
6779 * @dev: device
6780 * @flags: device state flags
6781 *
6782 * Change settings on device based state flags. The flags are
6783 * in the userspace exported format.
6784 */
Eric Dumazetb536db92011-11-30 21:42:26 +00006785int dev_change_flags(struct net_device *dev, unsigned int flags)
Patrick McHardybd380812010-02-26 06:34:53 +00006786{
Eric Dumazetb536db92011-11-30 21:42:26 +00006787 int ret;
Nicolas Dichtel991fb3f2013-09-25 12:02:45 +02006788 unsigned int changes, old_flags = dev->flags, old_gflags = dev->gflags;
Patrick McHardybd380812010-02-26 06:34:53 +00006789
6790 ret = __dev_change_flags(dev, flags);
6791 if (ret < 0)
6792 return ret;
6793
Nicolas Dichtel991fb3f2013-09-25 12:02:45 +02006794 changes = (old_flags ^ dev->flags) | (old_gflags ^ dev->gflags);
Nicolas Dichtela528c212013-09-25 12:02:44 +02006795 __dev_notify_flags(dev, old_flags, changes);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006796 return ret;
6797}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07006798EXPORT_SYMBOL(dev_change_flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006799
WANG Congf51048c2017-07-06 15:01:57 -07006800int __dev_set_mtu(struct net_device *dev, int new_mtu)
Veaceslav Falico2315dc92014-01-10 16:56:25 +01006801{
6802 const struct net_device_ops *ops = dev->netdev_ops;
6803
6804 if (ops->ndo_change_mtu)
6805 return ops->ndo_change_mtu(dev, new_mtu);
6806
6807 dev->mtu = new_mtu;
6808 return 0;
6809}
WANG Congf51048c2017-07-06 15:01:57 -07006810EXPORT_SYMBOL(__dev_set_mtu);
Veaceslav Falico2315dc92014-01-10 16:56:25 +01006811
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07006812/**
6813 * dev_set_mtu - Change maximum transfer unit
6814 * @dev: device
6815 * @new_mtu: new transfer unit
6816 *
6817 * Change the maximum transfer size of the network device.
6818 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07006819int dev_set_mtu(struct net_device *dev, int new_mtu)
6820{
Veaceslav Falico2315dc92014-01-10 16:56:25 +01006821 int err, orig_mtu;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006822
6823 if (new_mtu == dev->mtu)
6824 return 0;
6825
Jarod Wilson61e84622016-10-07 22:04:33 -04006826 /* MTU must be positive, and in range */
6827 if (new_mtu < 0 || new_mtu < dev->min_mtu) {
6828 net_err_ratelimited("%s: Invalid MTU %d requested, hw min %d\n",
6829 dev->name, new_mtu, dev->min_mtu);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006830 return -EINVAL;
Jarod Wilson61e84622016-10-07 22:04:33 -04006831 }
6832
6833 if (dev->max_mtu > 0 && new_mtu > dev->max_mtu) {
6834 net_err_ratelimited("%s: Invalid MTU %d requested, hw max %d\n",
Jakub Kicinskia0e65de2016-10-17 18:02:22 +01006835 dev->name, new_mtu, dev->max_mtu);
Jarod Wilson61e84622016-10-07 22:04:33 -04006836 return -EINVAL;
6837 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07006838
6839 if (!netif_device_present(dev))
6840 return -ENODEV;
6841
Veaceslav Falico1d486bf2014-01-16 00:02:18 +01006842 err = call_netdevice_notifiers(NETDEV_PRECHANGEMTU, dev);
6843 err = notifier_to_errno(err);
6844 if (err)
6845 return err;
Stephen Hemmingerd3147742008-11-19 21:32:24 -08006846
Veaceslav Falico2315dc92014-01-10 16:56:25 +01006847 orig_mtu = dev->mtu;
6848 err = __dev_set_mtu(dev, new_mtu);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006849
Veaceslav Falico2315dc92014-01-10 16:56:25 +01006850 if (!err) {
6851 err = call_netdevice_notifiers(NETDEV_CHANGEMTU, dev);
6852 err = notifier_to_errno(err);
6853 if (err) {
6854 /* setting mtu back and notifying everyone again,
6855 * so that they have a chance to revert changes.
6856 */
6857 __dev_set_mtu(dev, orig_mtu);
6858 call_netdevice_notifiers(NETDEV_CHANGEMTU, dev);
6859 }
6860 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07006861 return err;
6862}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07006863EXPORT_SYMBOL(dev_set_mtu);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006864
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07006865/**
Vlad Dogarucbda10f2011-01-13 23:38:30 +00006866 * dev_set_group - Change group this device belongs to
6867 * @dev: device
6868 * @new_group: group this device should belong to
6869 */
6870void dev_set_group(struct net_device *dev, int new_group)
6871{
6872 dev->group = new_group;
6873}
6874EXPORT_SYMBOL(dev_set_group);
6875
6876/**
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07006877 * dev_set_mac_address - Change Media Access Control Address
6878 * @dev: device
6879 * @sa: new address
6880 *
6881 * Change the hardware (MAC) address of the device
6882 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07006883int dev_set_mac_address(struct net_device *dev, struct sockaddr *sa)
6884{
Stephen Hemmingerd3147742008-11-19 21:32:24 -08006885 const struct net_device_ops *ops = dev->netdev_ops;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006886 int err;
6887
Stephen Hemmingerd3147742008-11-19 21:32:24 -08006888 if (!ops->ndo_set_mac_address)
Linus Torvalds1da177e2005-04-16 15:20:36 -07006889 return -EOPNOTSUPP;
6890 if (sa->sa_family != dev->type)
6891 return -EINVAL;
6892 if (!netif_device_present(dev))
6893 return -ENODEV;
Stephen Hemmingerd3147742008-11-19 21:32:24 -08006894 err = ops->ndo_set_mac_address(dev, sa);
Jiri Pirkof6521512013-01-01 03:30:14 +00006895 if (err)
6896 return err;
Jiri Pirkofbdeca22013-01-01 03:30:16 +00006897 dev->addr_assign_type = NET_ADDR_SET;
Jiri Pirkof6521512013-01-01 03:30:14 +00006898 call_netdevice_notifiers(NETDEV_CHANGEADDR, dev);
Theodore Ts'o7bf23572012-07-04 21:23:25 -04006899 add_device_randomness(dev->dev_addr, dev->addr_len);
Jiri Pirkof6521512013-01-01 03:30:14 +00006900 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006901}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07006902EXPORT_SYMBOL(dev_set_mac_address);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006903
Jiri Pirko4bf84c32012-12-27 23:49:37 +00006904/**
6905 * dev_change_carrier - Change device carrier
6906 * @dev: device
Randy Dunlap691b3b72013-03-04 12:32:43 +00006907 * @new_carrier: new value
Jiri Pirko4bf84c32012-12-27 23:49:37 +00006908 *
6909 * Change device carrier
6910 */
6911int dev_change_carrier(struct net_device *dev, bool new_carrier)
6912{
6913 const struct net_device_ops *ops = dev->netdev_ops;
6914
6915 if (!ops->ndo_change_carrier)
6916 return -EOPNOTSUPP;
6917 if (!netif_device_present(dev))
6918 return -ENODEV;
6919 return ops->ndo_change_carrier(dev, new_carrier);
6920}
6921EXPORT_SYMBOL(dev_change_carrier);
6922
Linus Torvalds1da177e2005-04-16 15:20:36 -07006923/**
Jiri Pirko66b52b02013-07-29 18:16:49 +02006924 * dev_get_phys_port_id - Get device physical port ID
6925 * @dev: device
6926 * @ppid: port ID
6927 *
6928 * Get device physical port ID
6929 */
6930int dev_get_phys_port_id(struct net_device *dev,
Jiri Pirko02637fc2014-11-28 14:34:16 +01006931 struct netdev_phys_item_id *ppid)
Jiri Pirko66b52b02013-07-29 18:16:49 +02006932{
6933 const struct net_device_ops *ops = dev->netdev_ops;
6934
6935 if (!ops->ndo_get_phys_port_id)
6936 return -EOPNOTSUPP;
6937 return ops->ndo_get_phys_port_id(dev, ppid);
6938}
6939EXPORT_SYMBOL(dev_get_phys_port_id);
6940
6941/**
David Aherndb24a902015-03-17 20:23:15 -06006942 * dev_get_phys_port_name - Get device physical port name
6943 * @dev: device
6944 * @name: port name
Luis de Bethencourted49e652016-03-21 16:31:14 +00006945 * @len: limit of bytes to copy to name
David Aherndb24a902015-03-17 20:23:15 -06006946 *
6947 * Get device physical port name
6948 */
6949int dev_get_phys_port_name(struct net_device *dev,
6950 char *name, size_t len)
6951{
6952 const struct net_device_ops *ops = dev->netdev_ops;
6953
6954 if (!ops->ndo_get_phys_port_name)
6955 return -EOPNOTSUPP;
6956 return ops->ndo_get_phys_port_name(dev, name, len);
6957}
6958EXPORT_SYMBOL(dev_get_phys_port_name);
6959
6960/**
Anuradha Karuppiahd746d702015-07-14 13:43:19 -07006961 * dev_change_proto_down - update protocol port state information
6962 * @dev: device
6963 * @proto_down: new value
6964 *
6965 * This info can be used by switch drivers to set the phys state of the
6966 * port.
6967 */
6968int dev_change_proto_down(struct net_device *dev, bool proto_down)
6969{
6970 const struct net_device_ops *ops = dev->netdev_ops;
6971
6972 if (!ops->ndo_change_proto_down)
6973 return -EOPNOTSUPP;
6974 if (!netif_device_present(dev))
6975 return -ENODEV;
6976 return ops->ndo_change_proto_down(dev, proto_down);
6977}
6978EXPORT_SYMBOL(dev_change_proto_down);
6979
Jakub Kicinskice158e52017-06-21 18:25:09 -07006980u8 __dev_xdp_attached(struct net_device *dev, xdp_op_t xdp_op, u32 *prog_id)
Daniel Borkmannd67b9cd2017-05-12 01:04:46 +02006981{
6982 struct netdev_xdp xdp;
6983
6984 memset(&xdp, 0, sizeof(xdp));
6985 xdp.command = XDP_QUERY_PROG;
6986
6987 /* Query must always succeed. */
6988 WARN_ON(xdp_op(dev, &xdp) < 0);
Martin KaFai Lau58038692017-06-15 17:29:09 -07006989 if (prog_id)
6990 *prog_id = xdp.prog_id;
6991
Daniel Borkmannd67b9cd2017-05-12 01:04:46 +02006992 return xdp.prog_attached;
6993}
6994
6995static int dev_xdp_install(struct net_device *dev, xdp_op_t xdp_op,
Jakub Kicinski32d60272017-06-21 18:25:03 -07006996 struct netlink_ext_ack *extack, u32 flags,
Daniel Borkmannd67b9cd2017-05-12 01:04:46 +02006997 struct bpf_prog *prog)
6998{
6999 struct netdev_xdp xdp;
7000
7001 memset(&xdp, 0, sizeof(xdp));
Jakub Kicinskiee5d0322017-06-21 18:25:04 -07007002 if (flags & XDP_FLAGS_HW_MODE)
7003 xdp.command = XDP_SETUP_PROG_HW;
7004 else
7005 xdp.command = XDP_SETUP_PROG;
Daniel Borkmannd67b9cd2017-05-12 01:04:46 +02007006 xdp.extack = extack;
Jakub Kicinski32d60272017-06-21 18:25:03 -07007007 xdp.flags = flags;
Daniel Borkmannd67b9cd2017-05-12 01:04:46 +02007008 xdp.prog = prog;
7009
7010 return xdp_op(dev, &xdp);
7011}
7012
Anuradha Karuppiahd746d702015-07-14 13:43:19 -07007013/**
Brenden Blancoa7862b42016-07-19 12:16:48 -07007014 * dev_change_xdp_fd - set or clear a bpf program for a device rx path
7015 * @dev: device
Jakub Kicinskib5d60982017-05-01 15:53:43 -07007016 * @extack: netlink extended ack
Brenden Blancoa7862b42016-07-19 12:16:48 -07007017 * @fd: new program fd or negative value to clear
Daniel Borkmann85de8572016-11-28 23:16:54 +01007018 * @flags: xdp-related flags
Brenden Blancoa7862b42016-07-19 12:16:48 -07007019 *
7020 * Set or clear a bpf program for a device
7021 */
Jakub Kicinskiddf9f972017-04-30 21:46:46 -07007022int dev_change_xdp_fd(struct net_device *dev, struct netlink_ext_ack *extack,
7023 int fd, u32 flags)
Brenden Blancoa7862b42016-07-19 12:16:48 -07007024{
7025 const struct net_device_ops *ops = dev->netdev_ops;
7026 struct bpf_prog *prog = NULL;
Daniel Borkmannd67b9cd2017-05-12 01:04:46 +02007027 xdp_op_t xdp_op, xdp_chk;
Brenden Blancoa7862b42016-07-19 12:16:48 -07007028 int err;
7029
Daniel Borkmann85de8572016-11-28 23:16:54 +01007030 ASSERT_RTNL();
7031
Daniel Borkmannd67b9cd2017-05-12 01:04:46 +02007032 xdp_op = xdp_chk = ops->ndo_xdp;
Jakub Kicinskiee5d0322017-06-21 18:25:04 -07007033 if (!xdp_op && (flags & (XDP_FLAGS_DRV_MODE | XDP_FLAGS_HW_MODE)))
Daniel Borkmann0489df92017-05-12 01:04:45 +02007034 return -EOPNOTSUPP;
David S. Millerb5cdae32017-04-18 15:36:58 -04007035 if (!xdp_op || (flags & XDP_FLAGS_SKB_MODE))
7036 xdp_op = generic_xdp_install;
Daniel Borkmannd67b9cd2017-05-12 01:04:46 +02007037 if (xdp_op == xdp_chk)
7038 xdp_chk = generic_xdp_install;
David S. Millerb5cdae32017-04-18 15:36:58 -04007039
Brenden Blancoa7862b42016-07-19 12:16:48 -07007040 if (fd >= 0) {
Martin KaFai Lau58038692017-06-15 17:29:09 -07007041 if (xdp_chk && __dev_xdp_attached(dev, xdp_chk, NULL))
Daniel Borkmannd67b9cd2017-05-12 01:04:46 +02007042 return -EEXIST;
7043 if ((flags & XDP_FLAGS_UPDATE_IF_NOEXIST) &&
Martin KaFai Lau58038692017-06-15 17:29:09 -07007044 __dev_xdp_attached(dev, xdp_op, NULL))
Daniel Borkmannd67b9cd2017-05-12 01:04:46 +02007045 return -EBUSY;
Daniel Borkmann85de8572016-11-28 23:16:54 +01007046
Brenden Blancoa7862b42016-07-19 12:16:48 -07007047 prog = bpf_prog_get_type(fd, BPF_PROG_TYPE_XDP);
7048 if (IS_ERR(prog))
7049 return PTR_ERR(prog);
7050 }
7051
Jakub Kicinski32d60272017-06-21 18:25:03 -07007052 err = dev_xdp_install(dev, xdp_op, extack, flags, prog);
Brenden Blancoa7862b42016-07-19 12:16:48 -07007053 if (err < 0 && prog)
7054 bpf_prog_put(prog);
7055
7056 return err;
7057}
Brenden Blancoa7862b42016-07-19 12:16:48 -07007058
7059/**
Linus Torvalds1da177e2005-04-16 15:20:36 -07007060 * dev_new_index - allocate an ifindex
Randy Dunlapc4ea43c2007-10-12 21:17:49 -07007061 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -07007062 *
7063 * Returns a suitable unique value for a new device interface
7064 * number. The caller must hold the rtnl semaphore or the
7065 * dev_base_lock to be sure it remains unique.
7066 */
Eric W. Biederman881d9662007-09-17 11:56:21 -07007067static int dev_new_index(struct net *net)
Linus Torvalds1da177e2005-04-16 15:20:36 -07007068{
Pavel Emelyanovaa79e662012-08-08 21:53:19 +00007069 int ifindex = net->ifindex;
tchardingf4563a72017-02-09 17:56:07 +11007070
Linus Torvalds1da177e2005-04-16 15:20:36 -07007071 for (;;) {
7072 if (++ifindex <= 0)
7073 ifindex = 1;
Eric W. Biederman881d9662007-09-17 11:56:21 -07007074 if (!__dev_get_by_index(net, ifindex))
Pavel Emelyanovaa79e662012-08-08 21:53:19 +00007075 return net->ifindex = ifindex;
Linus Torvalds1da177e2005-04-16 15:20:36 -07007076 }
7077}
7078
Linus Torvalds1da177e2005-04-16 15:20:36 -07007079/* Delayed registration/unregisteration */
Denis Cheng3b5b34f2007-12-07 00:49:17 -08007080static LIST_HEAD(net_todo_list);
Cong Wang200b9162014-05-12 15:11:20 -07007081DECLARE_WAIT_QUEUE_HEAD(netdev_unregistering_wq);
Linus Torvalds1da177e2005-04-16 15:20:36 -07007082
Stephen Hemminger6f05f622007-03-08 20:46:03 -08007083static void net_set_todo(struct net_device *dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07007084{
Linus Torvalds1da177e2005-04-16 15:20:36 -07007085 list_add_tail(&dev->todo_list, &net_todo_list);
Eric W. Biederman50624c92013-09-23 21:19:49 -07007086 dev_net(dev)->dev_unreg_count++;
Linus Torvalds1da177e2005-04-16 15:20:36 -07007087}
7088
Eric Dumazet9b5e3832009-10-27 07:04:19 +00007089static void rollback_registered_many(struct list_head *head)
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07007090{
Krishna Kumare93737b2009-12-08 22:26:02 +00007091 struct net_device *dev, *tmp;
Eric W. Biederman5cde2822013-10-05 19:26:05 -07007092 LIST_HEAD(close_head);
Eric Dumazet9b5e3832009-10-27 07:04:19 +00007093
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07007094 BUG_ON(dev_boot_phase);
7095 ASSERT_RTNL();
7096
Krishna Kumare93737b2009-12-08 22:26:02 +00007097 list_for_each_entry_safe(dev, tmp, head, unreg_list) {
Eric Dumazet9b5e3832009-10-27 07:04:19 +00007098 /* Some devices call without registering
Krishna Kumare93737b2009-12-08 22:26:02 +00007099 * for initialization unwind. Remove those
7100 * devices and proceed with the remaining.
Eric Dumazet9b5e3832009-10-27 07:04:19 +00007101 */
7102 if (dev->reg_state == NETREG_UNINITIALIZED) {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00007103 pr_debug("unregister_netdevice: device %s/%p never was registered\n",
7104 dev->name, dev);
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07007105
Eric Dumazet9b5e3832009-10-27 07:04:19 +00007106 WARN_ON(1);
Krishna Kumare93737b2009-12-08 22:26:02 +00007107 list_del(&dev->unreg_list);
7108 continue;
Eric Dumazet9b5e3832009-10-27 07:04:19 +00007109 }
Eric Dumazet449f4542011-05-19 12:24:16 +00007110 dev->dismantle = true;
Eric Dumazet9b5e3832009-10-27 07:04:19 +00007111 BUG_ON(dev->reg_state != NETREG_REGISTERED);
Octavian Purdila44345722010-12-13 12:44:07 +00007112 }
Eric Dumazet9b5e3832009-10-27 07:04:19 +00007113
Octavian Purdila44345722010-12-13 12:44:07 +00007114 /* If device is running, close it first. */
Eric W. Biederman5cde2822013-10-05 19:26:05 -07007115 list_for_each_entry(dev, head, unreg_list)
7116 list_add_tail(&dev->close_list, &close_head);
David S. Miller99c4a262015-03-18 22:52:33 -04007117 dev_close_many(&close_head, true);
Eric Dumazet9b5e3832009-10-27 07:04:19 +00007118
Octavian Purdila44345722010-12-13 12:44:07 +00007119 list_for_each_entry(dev, head, unreg_list) {
Eric Dumazet9b5e3832009-10-27 07:04:19 +00007120 /* And unlink it from device chain. */
7121 unlist_netdevice(dev);
7122
7123 dev->reg_state = NETREG_UNREGISTERING;
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07007124 }
Eric Dumazet41852492016-08-26 12:50:39 -07007125 flush_all_backlogs();
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07007126
Eric Dumazet9b5e3832009-10-27 07:04:19 +00007127 synchronize_net();
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07007128
Eric Dumazet9b5e3832009-10-27 07:04:19 +00007129 list_for_each_entry(dev, head, unreg_list) {
Mahesh Bandewar395eea62014-12-03 13:46:24 -08007130 struct sk_buff *skb = NULL;
7131
Eric Dumazet9b5e3832009-10-27 07:04:19 +00007132 /* Shutdown queueing discipline. */
7133 dev_shutdown(dev);
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07007134
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07007135
Eric Dumazet9b5e3832009-10-27 07:04:19 +00007136 /* Notify protocols, that we are about to destroy
tchardingeb13da12017-02-09 17:56:06 +11007137 * this device. They should clean all the things.
7138 */
Eric Dumazet9b5e3832009-10-27 07:04:19 +00007139 call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
7140
Mahesh Bandewar395eea62014-12-03 13:46:24 -08007141 if (!dev->rtnl_link_ops ||
7142 dev->rtnl_link_state == RTNL_LINK_INITIALIZED)
Vlad Yasevich3d3ea5a2017-05-27 10:14:34 -04007143 skb = rtmsg_ifinfo_build_skb(RTM_DELLINK, dev, ~0U, 0,
Mahesh Bandewar395eea62014-12-03 13:46:24 -08007144 GFP_KERNEL);
7145
Eric Dumazet9b5e3832009-10-27 07:04:19 +00007146 /*
7147 * Flush the unicast and multicast chains
7148 */
Jiri Pirkoa748ee22010-04-01 21:22:09 +00007149 dev_uc_flush(dev);
Jiri Pirko22bedad32010-04-01 21:22:57 +00007150 dev_mc_flush(dev);
Eric Dumazet9b5e3832009-10-27 07:04:19 +00007151
7152 if (dev->netdev_ops->ndo_uninit)
7153 dev->netdev_ops->ndo_uninit(dev);
7154
Mahesh Bandewar395eea62014-12-03 13:46:24 -08007155 if (skb)
7156 rtmsg_ifinfo_send(skb, dev, GFP_KERNEL);
Roopa Prabhu56bfa7e2014-05-01 11:40:30 -07007157
Jiri Pirko9ff162a2013-01-03 22:48:49 +00007158 /* Notifier chain MUST detach us all upper devices. */
7159 WARN_ON(netdev_has_any_upper_dev(dev));
David Ahern0f524a82016-10-17 19:15:52 -07007160 WARN_ON(netdev_has_any_lower_dev(dev));
Eric Dumazet9b5e3832009-10-27 07:04:19 +00007161
7162 /* Remove entries from kobject tree */
7163 netdev_unregister_kobject(dev);
Alexander Duyck024e9672013-01-10 08:57:46 +00007164#ifdef CONFIG_XPS
7165 /* Remove XPS queueing entries */
7166 netif_reset_xps_queues_gt(dev, 0);
7167#endif
Eric Dumazet9b5e3832009-10-27 07:04:19 +00007168 }
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07007169
Eric W. Biederman850a5452011-10-13 22:25:23 +00007170 synchronize_net();
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07007171
Eric W. Biedermana5ee1552009-11-29 15:45:58 +00007172 list_for_each_entry(dev, head, unreg_list)
Eric Dumazet9b5e3832009-10-27 07:04:19 +00007173 dev_put(dev);
7174}
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07007175
Eric Dumazet9b5e3832009-10-27 07:04:19 +00007176static void rollback_registered(struct net_device *dev)
7177{
7178 LIST_HEAD(single);
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07007179
Eric Dumazet9b5e3832009-10-27 07:04:19 +00007180 list_add(&dev->unreg_list, &single);
7181 rollback_registered_many(&single);
Eric Dumazetceaaec92011-02-17 22:59:19 +00007182 list_del(&single);
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07007183}
7184
Jarod Wilsonfd867d52015-11-02 21:55:59 -05007185static netdev_features_t netdev_sync_upper_features(struct net_device *lower,
7186 struct net_device *upper, netdev_features_t features)
7187{
7188 netdev_features_t upper_disables = NETIF_F_UPPER_DISABLES;
7189 netdev_features_t feature;
Jarod Wilson5ba3f7d2015-11-03 10:15:59 -05007190 int feature_bit;
Jarod Wilsonfd867d52015-11-02 21:55:59 -05007191
Jarod Wilson5ba3f7d2015-11-03 10:15:59 -05007192 for_each_netdev_feature(&upper_disables, feature_bit) {
7193 feature = __NETIF_F_BIT(feature_bit);
Jarod Wilsonfd867d52015-11-02 21:55:59 -05007194 if (!(upper->wanted_features & feature)
7195 && (features & feature)) {
7196 netdev_dbg(lower, "Dropping feature %pNF, upper dev %s has it off.\n",
7197 &feature, upper->name);
7198 features &= ~feature;
7199 }
7200 }
7201
7202 return features;
7203}
7204
7205static void netdev_sync_lower_features(struct net_device *upper,
7206 struct net_device *lower, netdev_features_t features)
7207{
7208 netdev_features_t upper_disables = NETIF_F_UPPER_DISABLES;
7209 netdev_features_t feature;
Jarod Wilson5ba3f7d2015-11-03 10:15:59 -05007210 int feature_bit;
Jarod Wilsonfd867d52015-11-02 21:55:59 -05007211
Jarod Wilson5ba3f7d2015-11-03 10:15:59 -05007212 for_each_netdev_feature(&upper_disables, feature_bit) {
7213 feature = __NETIF_F_BIT(feature_bit);
Jarod Wilsonfd867d52015-11-02 21:55:59 -05007214 if (!(features & feature) && (lower->features & feature)) {
7215 netdev_dbg(upper, "Disabling feature %pNF on lower dev %s.\n",
7216 &feature, lower->name);
7217 lower->wanted_features &= ~feature;
7218 netdev_update_features(lower);
7219
7220 if (unlikely(lower->features & feature))
7221 netdev_WARN(upper, "failed to disable %pNF on %s!\n",
7222 &feature, lower->name);
7223 }
7224 }
7225}
7226
Michał Mirosławc8f44af2011-11-15 15:29:55 +00007227static netdev_features_t netdev_fix_features(struct net_device *dev,
7228 netdev_features_t features)
Herbert Xub63365a2008-10-23 01:11:29 -07007229{
Michał Mirosław57422dc2011-01-22 12:14:12 +00007230 /* Fix illegal checksum combinations */
7231 if ((features & NETIF_F_HW_CSUM) &&
7232 (features & (NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM))) {
Michał Mirosław6f404e42011-05-16 15:14:21 -04007233 netdev_warn(dev, "mixed HW and IP checksum settings.\n");
Michał Mirosław57422dc2011-01-22 12:14:12 +00007234 features &= ~(NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM);
7235 }
7236
Herbert Xub63365a2008-10-23 01:11:29 -07007237 /* TSO requires that SG is present as well. */
Ben Hutchingsea2d3682011-04-12 14:38:37 +00007238 if ((features & NETIF_F_ALL_TSO) && !(features & NETIF_F_SG)) {
Michał Mirosław6f404e42011-05-16 15:14:21 -04007239 netdev_dbg(dev, "Dropping TSO features since no SG feature.\n");
Ben Hutchingsea2d3682011-04-12 14:38:37 +00007240 features &= ~NETIF_F_ALL_TSO;
Herbert Xub63365a2008-10-23 01:11:29 -07007241 }
7242
Pravin B Shelarec5f0612013-03-07 09:28:01 +00007243 if ((features & NETIF_F_TSO) && !(features & NETIF_F_HW_CSUM) &&
7244 !(features & NETIF_F_IP_CSUM)) {
7245 netdev_dbg(dev, "Dropping TSO features since no CSUM feature.\n");
7246 features &= ~NETIF_F_TSO;
7247 features &= ~NETIF_F_TSO_ECN;
7248 }
7249
7250 if ((features & NETIF_F_TSO6) && !(features & NETIF_F_HW_CSUM) &&
7251 !(features & NETIF_F_IPV6_CSUM)) {
7252 netdev_dbg(dev, "Dropping TSO6 features since no CSUM feature.\n");
7253 features &= ~NETIF_F_TSO6;
7254 }
7255
Alexander Duyckb1dc4972016-05-02 09:38:24 -07007256 /* TSO with IPv4 ID mangling requires IPv4 TSO be enabled */
7257 if ((features & NETIF_F_TSO_MANGLEID) && !(features & NETIF_F_TSO))
7258 features &= ~NETIF_F_TSO_MANGLEID;
7259
Ben Hutchings31d8b9e2011-04-12 14:47:15 +00007260 /* TSO ECN requires that TSO is present as well. */
7261 if ((features & NETIF_F_ALL_TSO) == NETIF_F_TSO_ECN)
7262 features &= ~NETIF_F_TSO_ECN;
7263
Michał Mirosław212b5732011-02-15 16:59:16 +00007264 /* Software GSO depends on SG. */
7265 if ((features & NETIF_F_GSO) && !(features & NETIF_F_SG)) {
Michał Mirosław6f404e42011-05-16 15:14:21 -04007266 netdev_dbg(dev, "Dropping NETIF_F_GSO since no SG feature.\n");
Michał Mirosław212b5732011-02-15 16:59:16 +00007267 features &= ~NETIF_F_GSO;
7268 }
7269
Alexander Duyck802ab552016-04-10 21:45:03 -04007270 /* GSO partial features require GSO partial be set */
7271 if ((features & dev->gso_partial_features) &&
7272 !(features & NETIF_F_GSO_PARTIAL)) {
7273 netdev_dbg(dev,
7274 "Dropping partially supported GSO features since no GSO partial.\n");
7275 features &= ~dev->gso_partial_features;
7276 }
7277
Herbert Xub63365a2008-10-23 01:11:29 -07007278 return features;
7279}
Herbert Xub63365a2008-10-23 01:11:29 -07007280
Michał Mirosław6cb6a272011-04-02 22:48:47 -07007281int __netdev_update_features(struct net_device *dev)
Michał Mirosław5455c692011-02-15 16:59:17 +00007282{
Jarod Wilsonfd867d52015-11-02 21:55:59 -05007283 struct net_device *upper, *lower;
Michał Mirosławc8f44af2011-11-15 15:29:55 +00007284 netdev_features_t features;
Jarod Wilsonfd867d52015-11-02 21:55:59 -05007285 struct list_head *iter;
Jarod Wilsone7868a82015-11-03 23:09:32 -05007286 int err = -1;
Michał Mirosław5455c692011-02-15 16:59:17 +00007287
Michał Mirosław87267482011-04-12 09:56:38 +00007288 ASSERT_RTNL();
7289
Michał Mirosław5455c692011-02-15 16:59:17 +00007290 features = netdev_get_wanted_features(dev);
7291
7292 if (dev->netdev_ops->ndo_fix_features)
7293 features = dev->netdev_ops->ndo_fix_features(dev, features);
7294
7295 /* driver might be less strict about feature dependencies */
7296 features = netdev_fix_features(dev, features);
7297
Jarod Wilsonfd867d52015-11-02 21:55:59 -05007298 /* some features can't be enabled if they're off an an upper device */
7299 netdev_for_each_upper_dev_rcu(dev, upper, iter)
7300 features = netdev_sync_upper_features(dev, upper, features);
7301
Michał Mirosław5455c692011-02-15 16:59:17 +00007302 if (dev->features == features)
Jarod Wilsone7868a82015-11-03 23:09:32 -05007303 goto sync_lower;
Michał Mirosław5455c692011-02-15 16:59:17 +00007304
Michał Mirosławc8f44af2011-11-15 15:29:55 +00007305 netdev_dbg(dev, "Features changed: %pNF -> %pNF\n",
7306 &dev->features, &features);
Michał Mirosław5455c692011-02-15 16:59:17 +00007307
7308 if (dev->netdev_ops->ndo_set_features)
7309 err = dev->netdev_ops->ndo_set_features(dev, features);
Nikolay Aleksandrov5f8dc332015-11-13 14:54:01 +01007310 else
7311 err = 0;
Michał Mirosław5455c692011-02-15 16:59:17 +00007312
Michał Mirosław6cb6a272011-04-02 22:48:47 -07007313 if (unlikely(err < 0)) {
Michał Mirosław5455c692011-02-15 16:59:17 +00007314 netdev_err(dev,
Michał Mirosławc8f44af2011-11-15 15:29:55 +00007315 "set_features() failed (%d); wanted %pNF, left %pNF\n",
7316 err, &features, &dev->features);
Nikolay Aleksandrov17b85d22015-11-17 15:49:06 +01007317 /* return non-0 since some features might have changed and
7318 * it's better to fire a spurious notification than miss it
7319 */
7320 return -1;
Michał Mirosław6cb6a272011-04-02 22:48:47 -07007321 }
7322
Jarod Wilsone7868a82015-11-03 23:09:32 -05007323sync_lower:
Jarod Wilsonfd867d52015-11-02 21:55:59 -05007324 /* some features must be disabled on lower devices when disabled
7325 * on an upper device (think: bonding master or bridge)
7326 */
7327 netdev_for_each_lower_dev(dev, lower, iter)
7328 netdev_sync_lower_features(dev, lower, features);
7329
Sabrina Dubrocaae847f42017-07-21 12:49:31 +02007330 if (!err) {
7331 netdev_features_t diff = features ^ dev->features;
7332
7333 if (diff & NETIF_F_RX_UDP_TUNNEL_PORT) {
7334 /* udp_tunnel_{get,drop}_rx_info both need
7335 * NETIF_F_RX_UDP_TUNNEL_PORT enabled on the
7336 * device, or they won't do anything.
7337 * Thus we need to update dev->features
7338 * *before* calling udp_tunnel_get_rx_info,
7339 * but *after* calling udp_tunnel_drop_rx_info.
7340 */
7341 if (features & NETIF_F_RX_UDP_TUNNEL_PORT) {
7342 dev->features = features;
7343 udp_tunnel_get_rx_info(dev);
7344 } else {
7345 udp_tunnel_drop_rx_info(dev);
7346 }
7347 }
7348
Michał Mirosław6cb6a272011-04-02 22:48:47 -07007349 dev->features = features;
Sabrina Dubrocaae847f42017-07-21 12:49:31 +02007350 }
Michał Mirosław6cb6a272011-04-02 22:48:47 -07007351
Jarod Wilsone7868a82015-11-03 23:09:32 -05007352 return err < 0 ? 0 : 1;
Michał Mirosław6cb6a272011-04-02 22:48:47 -07007353}
7354
Michał Mirosławafe12cc2011-05-07 03:22:17 +00007355/**
7356 * netdev_update_features - recalculate device features
7357 * @dev: the device to check
7358 *
7359 * Recalculate dev->features set and send notifications if it
7360 * has changed. Should be called after driver or hardware dependent
7361 * conditions might have changed that influence the features.
7362 */
Michał Mirosław6cb6a272011-04-02 22:48:47 -07007363void netdev_update_features(struct net_device *dev)
7364{
7365 if (__netdev_update_features(dev))
7366 netdev_features_change(dev);
Michał Mirosław5455c692011-02-15 16:59:17 +00007367}
7368EXPORT_SYMBOL(netdev_update_features);
7369
Linus Torvalds1da177e2005-04-16 15:20:36 -07007370/**
Michał Mirosławafe12cc2011-05-07 03:22:17 +00007371 * netdev_change_features - recalculate device features
7372 * @dev: the device to check
7373 *
7374 * Recalculate dev->features set and send notifications even
7375 * if they have not changed. Should be called instead of
7376 * netdev_update_features() if also dev->vlan_features might
7377 * have changed to allow the changes to be propagated to stacked
7378 * VLAN devices.
7379 */
7380void netdev_change_features(struct net_device *dev)
7381{
7382 __netdev_update_features(dev);
7383 netdev_features_change(dev);
7384}
7385EXPORT_SYMBOL(netdev_change_features);
7386
7387/**
Patrick Mullaneyfc4a7482009-12-03 15:59:22 -08007388 * netif_stacked_transfer_operstate - transfer operstate
7389 * @rootdev: the root or lower level device to transfer state from
7390 * @dev: the device to transfer operstate to
7391 *
7392 * Transfer operational state from root to device. This is normally
7393 * called when a stacking relationship exists between the root
7394 * device and the device(a leaf device).
7395 */
7396void netif_stacked_transfer_operstate(const struct net_device *rootdev,
7397 struct net_device *dev)
7398{
7399 if (rootdev->operstate == IF_OPER_DORMANT)
7400 netif_dormant_on(dev);
7401 else
7402 netif_dormant_off(dev);
7403
Zhang Shengju0575c862017-04-26 17:49:38 +08007404 if (netif_carrier_ok(rootdev))
7405 netif_carrier_on(dev);
7406 else
7407 netif_carrier_off(dev);
Patrick Mullaneyfc4a7482009-12-03 15:59:22 -08007408}
7409EXPORT_SYMBOL(netif_stacked_transfer_operstate);
7410
Michael Daltona953be52014-01-16 22:23:28 -08007411#ifdef CONFIG_SYSFS
Eric Dumazet1b4bf462010-09-23 17:26:35 +00007412static int netif_alloc_rx_queues(struct net_device *dev)
7413{
Eric Dumazet1b4bf462010-09-23 17:26:35 +00007414 unsigned int i, count = dev->num_rx_queues;
Tom Herbertbd25fa72010-10-18 18:00:16 +00007415 struct netdev_rx_queue *rx;
Pankaj Gupta10595902015-01-12 11:41:28 +05307416 size_t sz = count * sizeof(*rx);
Eric Dumazet1b4bf462010-09-23 17:26:35 +00007417
Tom Herbertbd25fa72010-10-18 18:00:16 +00007418 BUG_ON(count < 1);
Eric Dumazet1b4bf462010-09-23 17:26:35 +00007419
Michal Hockodcda9b02017-07-12 14:36:45 -07007420 rx = kvzalloc(sz, GFP_KERNEL | __GFP_RETRY_MAYFAIL);
Michal Hockoda6bc572017-05-08 15:57:31 -07007421 if (!rx)
7422 return -ENOMEM;
7423
Tom Herbertbd25fa72010-10-18 18:00:16 +00007424 dev->_rx = rx;
7425
Tom Herbertbd25fa72010-10-18 18:00:16 +00007426 for (i = 0; i < count; i++)
Tom Herbertfe822242010-11-09 10:47:38 +00007427 rx[i].dev = dev;
Eric Dumazet1b4bf462010-09-23 17:26:35 +00007428 return 0;
7429}
Tom Herbertbf264142010-11-26 08:36:09 +00007430#endif
Eric Dumazet1b4bf462010-09-23 17:26:35 +00007431
Changli Gaoaa942102010-12-04 02:31:41 +00007432static void netdev_init_one_queue(struct net_device *dev,
7433 struct netdev_queue *queue, void *_unused)
7434{
7435 /* Initialize queue lock */
7436 spin_lock_init(&queue->_xmit_lock);
7437 netdev_set_xmit_lockdep_class(&queue->_xmit_lock, dev->type);
7438 queue->xmit_lock_owner = -1;
Changli Gaob236da62010-12-14 03:09:15 +00007439 netdev_queue_numa_node_write(queue, NUMA_NO_NODE);
Changli Gaoaa942102010-12-04 02:31:41 +00007440 queue->dev = dev;
Tom Herbert114cf582011-11-28 16:33:09 +00007441#ifdef CONFIG_BQL
7442 dql_init(&queue->dql, HZ);
7443#endif
Changli Gaoaa942102010-12-04 02:31:41 +00007444}
7445
Eric Dumazet60877a32013-06-20 01:15:51 -07007446static void netif_free_tx_queues(struct net_device *dev)
7447{
WANG Cong4cb28972014-06-02 15:55:22 -07007448 kvfree(dev->_tx);
Eric Dumazet60877a32013-06-20 01:15:51 -07007449}
7450
Tom Herberte6484932010-10-18 18:04:39 +00007451static int netif_alloc_netdev_queues(struct net_device *dev)
7452{
7453 unsigned int count = dev->num_tx_queues;
7454 struct netdev_queue *tx;
Eric Dumazet60877a32013-06-20 01:15:51 -07007455 size_t sz = count * sizeof(*tx);
Tom Herberte6484932010-10-18 18:04:39 +00007456
Eric Dumazetd3397272015-07-06 17:13:26 +02007457 if (count < 1 || count > 0xffff)
7458 return -EINVAL;
Tom Herberte6484932010-10-18 18:04:39 +00007459
Michal Hockodcda9b02017-07-12 14:36:45 -07007460 tx = kvzalloc(sz, GFP_KERNEL | __GFP_RETRY_MAYFAIL);
Michal Hockoda6bc572017-05-08 15:57:31 -07007461 if (!tx)
7462 return -ENOMEM;
7463
Tom Herberte6484932010-10-18 18:04:39 +00007464 dev->_tx = tx;
Tom Herbert1d24eb42010-11-21 13:17:27 +00007465
Tom Herberte6484932010-10-18 18:04:39 +00007466 netdev_for_each_tx_queue(dev, netdev_init_one_queue, NULL);
7467 spin_lock_init(&dev->tx_global_lock);
Changli Gaoaa942102010-12-04 02:31:41 +00007468
7469 return 0;
Tom Herberte6484932010-10-18 18:04:39 +00007470}
7471
Denys Vlasenkoa2029242015-05-11 21:17:53 +02007472void netif_tx_stop_all_queues(struct net_device *dev)
7473{
7474 unsigned int i;
7475
7476 for (i = 0; i < dev->num_tx_queues; i++) {
7477 struct netdev_queue *txq = netdev_get_tx_queue(dev, i);
tchardingf4563a72017-02-09 17:56:07 +11007478
Denys Vlasenkoa2029242015-05-11 21:17:53 +02007479 netif_tx_stop_queue(txq);
7480 }
7481}
7482EXPORT_SYMBOL(netif_tx_stop_all_queues);
7483
Patrick Mullaneyfc4a7482009-12-03 15:59:22 -08007484/**
Linus Torvalds1da177e2005-04-16 15:20:36 -07007485 * register_netdevice - register a network device
7486 * @dev: device to register
7487 *
7488 * Take a completed network device structure and add it to the kernel
7489 * interfaces. A %NETDEV_REGISTER message is sent to the netdev notifier
7490 * chain. 0 is returned on success. A negative errno code is returned
7491 * on a failure to set up the device, or if the name is a duplicate.
7492 *
7493 * Callers must hold the rtnl semaphore. You may want
7494 * register_netdev() instead of this.
7495 *
7496 * BUGS:
7497 * The locking appears insufficient to guarantee two parallel registers
7498 * will not get the same name.
7499 */
7500
7501int register_netdevice(struct net_device *dev)
7502{
Linus Torvalds1da177e2005-04-16 15:20:36 -07007503 int ret;
Stephen Hemmingerd3147742008-11-19 21:32:24 -08007504 struct net *net = dev_net(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07007505
7506 BUG_ON(dev_boot_phase);
7507 ASSERT_RTNL();
7508
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07007509 might_sleep();
7510
Linus Torvalds1da177e2005-04-16 15:20:36 -07007511 /* When net_device's are persistent, this will be fatal. */
7512 BUG_ON(dev->reg_state != NETREG_UNINITIALIZED);
Stephen Hemmingerd3147742008-11-19 21:32:24 -08007513 BUG_ON(!net);
Linus Torvalds1da177e2005-04-16 15:20:36 -07007514
David S. Millerf1f28aa2008-07-15 00:08:33 -07007515 spin_lock_init(&dev->addr_list_lock);
David S. Millercf508b12008-07-22 14:16:42 -07007516 netdev_set_addr_lockdep_class(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07007517
Gao feng828de4f2012-09-13 20:58:27 +00007518 ret = dev_get_valid_name(net, dev, dev->name);
Peter Pan(潘卫平)0696c3a2011-05-12 15:46:56 +00007519 if (ret < 0)
7520 goto out;
7521
Linus Torvalds1da177e2005-04-16 15:20:36 -07007522 /* Init, if this function is available */
Stephen Hemmingerd3147742008-11-19 21:32:24 -08007523 if (dev->netdev_ops->ndo_init) {
7524 ret = dev->netdev_ops->ndo_init(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07007525 if (ret) {
7526 if (ret > 0)
7527 ret = -EIO;
Adrian Bunk90833aa2006-11-13 16:02:22 -08007528 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07007529 }
7530 }
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09007531
Patrick McHardyf6469682013-04-19 02:04:27 +00007532 if (((dev->hw_features | dev->features) &
7533 NETIF_F_HW_VLAN_CTAG_FILTER) &&
Michał Mirosławd2ed2732013-01-29 15:14:16 +00007534 (!dev->netdev_ops->ndo_vlan_rx_add_vid ||
7535 !dev->netdev_ops->ndo_vlan_rx_kill_vid)) {
7536 netdev_WARN(dev, "Buggy VLAN acceleration in driver!\n");
7537 ret = -EINVAL;
7538 goto err_uninit;
7539 }
7540
Pavel Emelyanov9c7dafb2012-08-08 21:52:46 +00007541 ret = -EBUSY;
7542 if (!dev->ifindex)
7543 dev->ifindex = dev_new_index(net);
7544 else if (__dev_get_by_index(net, dev->ifindex))
7545 goto err_uninit;
7546
Michał Mirosław5455c692011-02-15 16:59:17 +00007547 /* Transfer changeable features to wanted_features and enable
7548 * software offloads (GSO and GRO).
7549 */
7550 dev->hw_features |= NETIF_F_SOFT_FEATURES;
Michał Mirosław14d12322011-02-22 16:52:28 +00007551 dev->features |= NETIF_F_SOFT_FEATURES;
Sabrina Dubrocad764a122017-07-21 12:49:28 +02007552
7553 if (dev->netdev_ops->ndo_udp_tunnel_add) {
7554 dev->features |= NETIF_F_RX_UDP_TUNNEL_PORT;
7555 dev->hw_features |= NETIF_F_RX_UDP_TUNNEL_PORT;
7556 }
7557
Michał Mirosław14d12322011-02-22 16:52:28 +00007558 dev->wanted_features = dev->features & dev->hw_features;
Linus Torvalds1da177e2005-04-16 15:20:36 -07007559
Alexander Duyckcbc53e02016-04-10 21:44:51 -04007560 if (!(dev->flags & IFF_LOOPBACK))
Michał Mirosław34324dc2011-11-15 15:29:55 +00007561 dev->hw_features |= NETIF_F_NOCACHE_COPY;
Alexander Duyckcbc53e02016-04-10 21:44:51 -04007562
Alexander Duyck7f348a62016-04-20 16:51:00 -04007563 /* If IPv4 TCP segmentation offload is supported we should also
7564 * allow the device to enable segmenting the frame with the option
7565 * of ignoring a static IP ID value. This doesn't enable the
7566 * feature itself but allows the user to enable it later.
7567 */
Alexander Duyckcbc53e02016-04-10 21:44:51 -04007568 if (dev->hw_features & NETIF_F_TSO)
7569 dev->hw_features |= NETIF_F_TSO_MANGLEID;
Alexander Duyck7f348a62016-04-20 16:51:00 -04007570 if (dev->vlan_features & NETIF_F_TSO)
7571 dev->vlan_features |= NETIF_F_TSO_MANGLEID;
7572 if (dev->mpls_features & NETIF_F_TSO)
7573 dev->mpls_features |= NETIF_F_TSO_MANGLEID;
7574 if (dev->hw_enc_features & NETIF_F_TSO)
7575 dev->hw_enc_features |= NETIF_F_TSO_MANGLEID;
Tom Herbertc6e1a0d2011-04-04 22:30:30 -07007576
Michał Mirosław1180e7d2011-07-14 14:41:11 -07007577 /* Make NETIF_F_HIGHDMA inheritable to VLAN devices.
Brandon Philips16c3ea72010-09-15 09:24:24 +00007578 */
Michał Mirosław1180e7d2011-07-14 14:41:11 -07007579 dev->vlan_features |= NETIF_F_HIGHDMA;
Brandon Philips16c3ea72010-09-15 09:24:24 +00007580
Pravin B Shelaree579672013-03-07 09:28:08 +00007581 /* Make NETIF_F_SG inheritable to tunnel devices.
7582 */
Alexander Duyck802ab552016-04-10 21:45:03 -04007583 dev->hw_enc_features |= NETIF_F_SG | NETIF_F_GSO_PARTIAL;
Pravin B Shelaree579672013-03-07 09:28:08 +00007584
Simon Horman0d89d202013-05-23 21:02:52 +00007585 /* Make NETIF_F_SG inheritable to MPLS.
7586 */
7587 dev->mpls_features |= NETIF_F_SG;
7588
Johannes Berg7ffbe3f2009-10-02 05:15:27 +00007589 ret = call_netdevice_notifiers(NETDEV_POST_INIT, dev);
7590 ret = notifier_to_errno(ret);
7591 if (ret)
7592 goto err_uninit;
7593
Eric W. Biederman8b41d182007-09-26 22:02:53 -07007594 ret = netdev_register_kobject(dev);
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07007595 if (ret)
Herbert Xu7ce1b0e2007-07-30 16:29:40 -07007596 goto err_uninit;
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07007597 dev->reg_state = NETREG_REGISTERED;
7598
Michał Mirosław6cb6a272011-04-02 22:48:47 -07007599 __netdev_update_features(dev);
Michał Mirosław8e9b59b2011-02-22 16:52:28 +00007600
Linus Torvalds1da177e2005-04-16 15:20:36 -07007601 /*
7602 * Default initial state at registry is that the
7603 * device is present.
7604 */
7605
7606 set_bit(__LINK_STATE_PRESENT, &dev->state);
7607
Ben Hutchings8f4cccb2012-08-20 22:16:51 +01007608 linkwatch_init_dev(dev);
7609
Linus Torvalds1da177e2005-04-16 15:20:36 -07007610 dev_init_scheduler(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07007611 dev_hold(dev);
Eric W. Biedermance286d32007-09-12 13:53:49 +02007612 list_netdevice(dev);
Theodore Ts'o7bf23572012-07-04 21:23:25 -04007613 add_device_randomness(dev->dev_addr, dev->addr_len);
Linus Torvalds1da177e2005-04-16 15:20:36 -07007614
Jiri Pirko948b3372013-01-08 01:38:25 +00007615 /* If the device has permanent device address, driver should
7616 * set dev_addr and also addr_assign_type should be set to
7617 * NET_ADDR_PERM (default value).
7618 */
7619 if (dev->addr_assign_type == NET_ADDR_PERM)
7620 memcpy(dev->perm_addr, dev->dev_addr, dev->addr_len);
7621
Linus Torvalds1da177e2005-04-16 15:20:36 -07007622 /* Notify protocols, that a new device appeared. */
Pavel Emelyanov056925a2007-09-16 15:42:43 -07007623 ret = call_netdevice_notifiers(NETDEV_REGISTER, dev);
Herbert Xufcc5a032007-07-30 17:03:38 -07007624 ret = notifier_to_errno(ret);
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07007625 if (ret) {
7626 rollback_registered(dev);
7627 dev->reg_state = NETREG_UNREGISTERED;
7628 }
Eric W. Biedermand90a9092009-12-12 22:11:15 +00007629 /*
7630 * Prevent userspace races by waiting until the network
7631 * device is fully setup before sending notifications.
7632 */
Patrick McHardya2835762010-02-26 06:34:51 +00007633 if (!dev->rtnl_link_ops ||
7634 dev->rtnl_link_state == RTNL_LINK_INITIALIZED)
Alexei Starovoitov7f294052013-10-23 16:02:42 -07007635 rtmsg_ifinfo(RTM_NEWLINK, dev, ~0U, GFP_KERNEL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07007636
7637out:
7638 return ret;
Herbert Xu7ce1b0e2007-07-30 16:29:40 -07007639
7640err_uninit:
Stephen Hemmingerd3147742008-11-19 21:32:24 -08007641 if (dev->netdev_ops->ndo_uninit)
7642 dev->netdev_ops->ndo_uninit(dev);
David S. Millercf124db2017-05-08 12:52:56 -04007643 if (dev->priv_destructor)
7644 dev->priv_destructor(dev);
Herbert Xu7ce1b0e2007-07-30 16:29:40 -07007645 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07007646}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07007647EXPORT_SYMBOL(register_netdevice);
Linus Torvalds1da177e2005-04-16 15:20:36 -07007648
7649/**
Benjamin Herrenschmidt937f1ba2009-01-14 21:05:05 -08007650 * init_dummy_netdev - init a dummy network device for NAPI
7651 * @dev: device to init
7652 *
7653 * This takes a network device structure and initialize the minimum
7654 * amount of fields so it can be used to schedule NAPI polls without
7655 * registering a full blown interface. This is to be used by drivers
7656 * that need to tie several hardware interfaces to a single NAPI
7657 * poll scheduler due to HW limitations.
7658 */
7659int init_dummy_netdev(struct net_device *dev)
7660{
7661 /* Clear everything. Note we don't initialize spinlocks
7662 * are they aren't supposed to be taken by any of the
7663 * NAPI code and this dummy netdev is supposed to be
7664 * only ever used for NAPI polls
7665 */
7666 memset(dev, 0, sizeof(struct net_device));
7667
7668 /* make sure we BUG if trying to hit standard
7669 * register/unregister code path
7670 */
7671 dev->reg_state = NETREG_DUMMY;
7672
Benjamin Herrenschmidt937f1ba2009-01-14 21:05:05 -08007673 /* NAPI wants this */
7674 INIT_LIST_HEAD(&dev->napi_list);
7675
7676 /* a dummy interface is started by default */
7677 set_bit(__LINK_STATE_PRESENT, &dev->state);
7678 set_bit(__LINK_STATE_START, &dev->state);
7679
Eric Dumazet29b44332010-10-11 10:22:12 +00007680 /* Note : We dont allocate pcpu_refcnt for dummy devices,
7681 * because users of this 'device' dont need to change
7682 * its refcount.
7683 */
7684
Benjamin Herrenschmidt937f1ba2009-01-14 21:05:05 -08007685 return 0;
7686}
7687EXPORT_SYMBOL_GPL(init_dummy_netdev);
7688
7689
7690/**
Linus Torvalds1da177e2005-04-16 15:20:36 -07007691 * register_netdev - register a network device
7692 * @dev: device to register
7693 *
7694 * Take a completed network device structure and add it to the kernel
7695 * interfaces. A %NETDEV_REGISTER message is sent to the netdev notifier
7696 * chain. 0 is returned on success. A negative errno code is returned
7697 * on a failure to set up the device, or if the name is a duplicate.
7698 *
Borislav Petkov38b4da32007-04-20 22:14:10 -07007699 * This is a wrapper around register_netdevice that takes the rtnl semaphore
Linus Torvalds1da177e2005-04-16 15:20:36 -07007700 * and expands the device name if you passed a format string to
7701 * alloc_netdev.
7702 */
7703int register_netdev(struct net_device *dev)
7704{
7705 int err;
7706
7707 rtnl_lock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07007708 err = register_netdevice(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07007709 rtnl_unlock();
7710 return err;
7711}
7712EXPORT_SYMBOL(register_netdev);
7713
Eric Dumazet29b44332010-10-11 10:22:12 +00007714int netdev_refcnt_read(const struct net_device *dev)
7715{
7716 int i, refcnt = 0;
7717
7718 for_each_possible_cpu(i)
7719 refcnt += *per_cpu_ptr(dev->pcpu_refcnt, i);
7720 return refcnt;
7721}
7722EXPORT_SYMBOL(netdev_refcnt_read);
7723
Ben Hutchings2c530402012-07-10 10:55:09 +00007724/**
Linus Torvalds1da177e2005-04-16 15:20:36 -07007725 * netdev_wait_allrefs - wait until all references are gone.
Randy Dunlap3de7a372012-08-18 14:36:44 +00007726 * @dev: target net_device
Linus Torvalds1da177e2005-04-16 15:20:36 -07007727 *
7728 * This is called when unregistering network devices.
7729 *
7730 * Any protocol or device that holds a reference should register
7731 * for netdevice notification, and cleanup and put back the
7732 * reference if they receive an UNREGISTER event.
7733 * We can get stuck here if buggy protocols don't correctly
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09007734 * call dev_put.
Linus Torvalds1da177e2005-04-16 15:20:36 -07007735 */
7736static void netdev_wait_allrefs(struct net_device *dev)
7737{
7738 unsigned long rebroadcast_time, warning_time;
Eric Dumazet29b44332010-10-11 10:22:12 +00007739 int refcnt;
Linus Torvalds1da177e2005-04-16 15:20:36 -07007740
Eric Dumazete014deb2009-11-17 05:59:21 +00007741 linkwatch_forget_dev(dev);
7742
Linus Torvalds1da177e2005-04-16 15:20:36 -07007743 rebroadcast_time = warning_time = jiffies;
Eric Dumazet29b44332010-10-11 10:22:12 +00007744 refcnt = netdev_refcnt_read(dev);
7745
7746 while (refcnt != 0) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07007747 if (time_after(jiffies, rebroadcast_time + 1 * HZ)) {
Stephen Hemminger6756ae42006-03-20 22:23:58 -08007748 rtnl_lock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07007749
7750 /* Rebroadcast unregister notification */
Pavel Emelyanov056925a2007-09-16 15:42:43 -07007751 call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07007752
Eric Dumazet748e2d92012-08-22 21:50:59 +00007753 __rtnl_unlock();
Eric Dumazet0115e8e2012-08-22 17:19:46 +00007754 rcu_barrier();
Eric Dumazet748e2d92012-08-22 21:50:59 +00007755 rtnl_lock();
7756
Eric Dumazet0115e8e2012-08-22 17:19:46 +00007757 call_netdevice_notifiers(NETDEV_UNREGISTER_FINAL, dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07007758 if (test_bit(__LINK_STATE_LINKWATCH_PENDING,
7759 &dev->state)) {
7760 /* We must not have linkwatch events
7761 * pending on unregister. If this
7762 * happens, we simply run the queue
7763 * unscheduled, resulting in a noop
7764 * for this device.
7765 */
7766 linkwatch_run_queue();
7767 }
7768
Stephen Hemminger6756ae42006-03-20 22:23:58 -08007769 __rtnl_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07007770
7771 rebroadcast_time = jiffies;
7772 }
7773
7774 msleep(250);
7775
Eric Dumazet29b44332010-10-11 10:22:12 +00007776 refcnt = netdev_refcnt_read(dev);
7777
Linus Torvalds1da177e2005-04-16 15:20:36 -07007778 if (time_after(jiffies, warning_time + 10 * HZ)) {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00007779 pr_emerg("unregister_netdevice: waiting for %s to become free. Usage count = %d\n",
7780 dev->name, refcnt);
Linus Torvalds1da177e2005-04-16 15:20:36 -07007781 warning_time = jiffies;
7782 }
7783 }
7784}
7785
7786/* The sequence is:
7787 *
7788 * rtnl_lock();
7789 * ...
7790 * register_netdevice(x1);
7791 * register_netdevice(x2);
7792 * ...
7793 * unregister_netdevice(y1);
7794 * unregister_netdevice(y2);
7795 * ...
7796 * rtnl_unlock();
7797 * free_netdev(y1);
7798 * free_netdev(y2);
7799 *
Herbert Xu58ec3b42008-10-07 15:50:03 -07007800 * We are invoked by rtnl_unlock().
Linus Torvalds1da177e2005-04-16 15:20:36 -07007801 * This allows us to deal with problems:
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07007802 * 1) We can delete sysfs objects which invoke hotplug
Linus Torvalds1da177e2005-04-16 15:20:36 -07007803 * without deadlocking with linkwatch via keventd.
7804 * 2) Since we run with the RTNL semaphore not held, we can sleep
7805 * safely in order to wait for the netdev refcnt to drop to zero.
Herbert Xu58ec3b42008-10-07 15:50:03 -07007806 *
7807 * We must not return until all unregister events added during
7808 * the interval the lock was held have been completed.
Linus Torvalds1da177e2005-04-16 15:20:36 -07007809 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07007810void netdev_run_todo(void)
7811{
Oleg Nesterov626ab0e2006-06-23 02:05:55 -07007812 struct list_head list;
Linus Torvalds1da177e2005-04-16 15:20:36 -07007813
Linus Torvalds1da177e2005-04-16 15:20:36 -07007814 /* Snapshot list, allow later requests */
Oleg Nesterov626ab0e2006-06-23 02:05:55 -07007815 list_replace_init(&net_todo_list, &list);
Herbert Xu58ec3b42008-10-07 15:50:03 -07007816
7817 __rtnl_unlock();
Oleg Nesterov626ab0e2006-06-23 02:05:55 -07007818
Eric Dumazet0115e8e2012-08-22 17:19:46 +00007819
7820 /* Wait for rcu callbacks to finish before next phase */
Eric W. Biederman850a5452011-10-13 22:25:23 +00007821 if (!list_empty(&list))
7822 rcu_barrier();
7823
Linus Torvalds1da177e2005-04-16 15:20:36 -07007824 while (!list_empty(&list)) {
7825 struct net_device *dev
stephen hemmingere5e26d72010-02-24 14:01:38 +00007826 = list_first_entry(&list, struct net_device, todo_list);
Linus Torvalds1da177e2005-04-16 15:20:36 -07007827 list_del(&dev->todo_list);
7828
Eric Dumazet748e2d92012-08-22 21:50:59 +00007829 rtnl_lock();
Eric Dumazet0115e8e2012-08-22 17:19:46 +00007830 call_netdevice_notifiers(NETDEV_UNREGISTER_FINAL, dev);
Eric Dumazet748e2d92012-08-22 21:50:59 +00007831 __rtnl_unlock();
Eric Dumazet0115e8e2012-08-22 17:19:46 +00007832
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07007833 if (unlikely(dev->reg_state != NETREG_UNREGISTERING)) {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00007834 pr_err("network todo '%s' but state %d\n",
Linus Torvalds1da177e2005-04-16 15:20:36 -07007835 dev->name, dev->reg_state);
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07007836 dump_stack();
7837 continue;
Linus Torvalds1da177e2005-04-16 15:20:36 -07007838 }
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07007839
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07007840 dev->reg_state = NETREG_UNREGISTERED;
7841
7842 netdev_wait_allrefs(dev);
7843
7844 /* paranoia */
Eric Dumazet29b44332010-10-11 10:22:12 +00007845 BUG_ON(netdev_refcnt_read(dev));
Salam Noureddine7866a622015-01-27 11:35:48 -08007846 BUG_ON(!list_empty(&dev->ptype_all));
7847 BUG_ON(!list_empty(&dev->ptype_specific));
Eric Dumazet33d480c2011-08-11 19:30:52 +00007848 WARN_ON(rcu_access_pointer(dev->ip_ptr));
7849 WARN_ON(rcu_access_pointer(dev->ip6_ptr));
Ilpo Järvinen547b7922008-07-25 21:43:18 -07007850 WARN_ON(dev->dn_ptr);
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07007851
David S. Millercf124db2017-05-08 12:52:56 -04007852 if (dev->priv_destructor)
7853 dev->priv_destructor(dev);
7854 if (dev->needs_free_netdev)
7855 free_netdev(dev);
Stephen Hemminger9093bbb2007-05-19 15:39:25 -07007856
Eric W. Biederman50624c92013-09-23 21:19:49 -07007857 /* Report a network device has been unregistered */
7858 rtnl_lock();
7859 dev_net(dev)->dev_unreg_count--;
7860 __rtnl_unlock();
7861 wake_up(&netdev_unregistering_wq);
7862
Stephen Hemminger9093bbb2007-05-19 15:39:25 -07007863 /* Free network device */
7864 kobject_put(&dev->dev.kobj);
Linus Torvalds1da177e2005-04-16 15:20:36 -07007865 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07007866}
7867
Jarod Wilson92566452016-02-01 18:51:04 -05007868/* Convert net_device_stats to rtnl_link_stats64. rtnl_link_stats64 has
7869 * all the same fields in the same order as net_device_stats, with only
7870 * the type differing, but rtnl_link_stats64 may have additional fields
7871 * at the end for newer counters.
Ben Hutchings3cfde792010-07-09 09:11:52 +00007872 */
Eric Dumazet77a1abf2012-03-05 04:50:09 +00007873void netdev_stats_to_stats64(struct rtnl_link_stats64 *stats64,
7874 const struct net_device_stats *netdev_stats)
Ben Hutchings3cfde792010-07-09 09:11:52 +00007875{
7876#if BITS_PER_LONG == 64
Jarod Wilson92566452016-02-01 18:51:04 -05007877 BUILD_BUG_ON(sizeof(*stats64) < sizeof(*netdev_stats));
Alban Browaeys9af99592017-07-03 03:20:13 +02007878 memcpy(stats64, netdev_stats, sizeof(*netdev_stats));
Jarod Wilson92566452016-02-01 18:51:04 -05007879 /* zero out counters that only exist in rtnl_link_stats64 */
7880 memset((char *)stats64 + sizeof(*netdev_stats), 0,
7881 sizeof(*stats64) - sizeof(*netdev_stats));
Ben Hutchings3cfde792010-07-09 09:11:52 +00007882#else
Jarod Wilson92566452016-02-01 18:51:04 -05007883 size_t i, n = sizeof(*netdev_stats) / sizeof(unsigned long);
Ben Hutchings3cfde792010-07-09 09:11:52 +00007884 const unsigned long *src = (const unsigned long *)netdev_stats;
7885 u64 *dst = (u64 *)stats64;
7886
Jarod Wilson92566452016-02-01 18:51:04 -05007887 BUILD_BUG_ON(n > sizeof(*stats64) / sizeof(u64));
Ben Hutchings3cfde792010-07-09 09:11:52 +00007888 for (i = 0; i < n; i++)
7889 dst[i] = src[i];
Jarod Wilson92566452016-02-01 18:51:04 -05007890 /* zero out counters that only exist in rtnl_link_stats64 */
7891 memset((char *)stats64 + n * sizeof(u64), 0,
7892 sizeof(*stats64) - n * sizeof(u64));
Ben Hutchings3cfde792010-07-09 09:11:52 +00007893#endif
7894}
Eric Dumazet77a1abf2012-03-05 04:50:09 +00007895EXPORT_SYMBOL(netdev_stats_to_stats64);
Ben Hutchings3cfde792010-07-09 09:11:52 +00007896
Eric Dumazetd83345a2009-11-16 03:36:51 +00007897/**
Stephen Hemmingereeda3fd2008-11-19 21:40:23 -08007898 * dev_get_stats - get network device statistics
7899 * @dev: device to get statistics from
Eric Dumazet28172732010-07-07 14:58:56 -07007900 * @storage: place to store stats
Stephen Hemmingereeda3fd2008-11-19 21:40:23 -08007901 *
Ben Hutchingsd7753512010-07-09 09:12:41 +00007902 * Get network statistics from device. Return @storage.
7903 * The device driver may provide its own method by setting
7904 * dev->netdev_ops->get_stats64 or dev->netdev_ops->get_stats;
7905 * otherwise the internal statistics structure is used.
Stephen Hemmingereeda3fd2008-11-19 21:40:23 -08007906 */
Ben Hutchingsd7753512010-07-09 09:12:41 +00007907struct rtnl_link_stats64 *dev_get_stats(struct net_device *dev,
7908 struct rtnl_link_stats64 *storage)
Eric Dumazet7004bf22009-05-18 00:34:33 +00007909{
Stephen Hemmingereeda3fd2008-11-19 21:40:23 -08007910 const struct net_device_ops *ops = dev->netdev_ops;
7911
Eric Dumazet28172732010-07-07 14:58:56 -07007912 if (ops->ndo_get_stats64) {
7913 memset(storage, 0, sizeof(*storage));
Eric Dumazetcaf586e2010-09-30 21:06:55 +00007914 ops->ndo_get_stats64(dev, storage);
7915 } else if (ops->ndo_get_stats) {
Ben Hutchings3cfde792010-07-09 09:11:52 +00007916 netdev_stats_to_stats64(storage, ops->ndo_get_stats(dev));
Eric Dumazetcaf586e2010-09-30 21:06:55 +00007917 } else {
7918 netdev_stats_to_stats64(storage, &dev->stats);
Eric Dumazet28172732010-07-07 14:58:56 -07007919 }
Eric Dumazet6f64ec72017-06-27 07:02:20 -07007920 storage->rx_dropped += (unsigned long)atomic_long_read(&dev->rx_dropped);
7921 storage->tx_dropped += (unsigned long)atomic_long_read(&dev->tx_dropped);
7922 storage->rx_nohandler += (unsigned long)atomic_long_read(&dev->rx_nohandler);
Eric Dumazet28172732010-07-07 14:58:56 -07007923 return storage;
Rusty Russellc45d2862007-03-28 14:29:08 -07007924}
Stephen Hemmingereeda3fd2008-11-19 21:40:23 -08007925EXPORT_SYMBOL(dev_get_stats);
Rusty Russellc45d2862007-03-28 14:29:08 -07007926
Eric Dumazet24824a02010-10-02 06:11:55 +00007927struct netdev_queue *dev_ingress_queue_create(struct net_device *dev)
David S. Millerdc2b4842008-07-08 17:18:23 -07007928{
Eric Dumazet24824a02010-10-02 06:11:55 +00007929 struct netdev_queue *queue = dev_ingress_queue(dev);
David S. Millerdc2b4842008-07-08 17:18:23 -07007930
Eric Dumazet24824a02010-10-02 06:11:55 +00007931#ifdef CONFIG_NET_CLS_ACT
7932 if (queue)
7933 return queue;
7934 queue = kzalloc(sizeof(*queue), GFP_KERNEL);
7935 if (!queue)
7936 return NULL;
7937 netdev_init_one_queue(dev, queue, NULL);
Eric Dumazet2ce1ee12015-02-04 13:37:44 -08007938 RCU_INIT_POINTER(queue->qdisc, &noop_qdisc);
Eric Dumazet24824a02010-10-02 06:11:55 +00007939 queue->qdisc_sleeping = &noop_qdisc;
7940 rcu_assign_pointer(dev->ingress_queue, queue);
7941#endif
7942 return queue;
David S. Millerbb949fb2008-07-08 16:55:56 -07007943}
7944
Eric Dumazet2c60db02012-09-16 09:17:26 +00007945static const struct ethtool_ops default_ethtool_ops;
7946
Stanislaw Gruszkad07d7502013-01-10 23:19:10 +00007947void netdev_set_default_ethtool_ops(struct net_device *dev,
7948 const struct ethtool_ops *ops)
7949{
7950 if (dev->ethtool_ops == &default_ethtool_ops)
7951 dev->ethtool_ops = ops;
7952}
7953EXPORT_SYMBOL_GPL(netdev_set_default_ethtool_ops);
7954
Eric Dumazet74d332c2013-10-30 13:10:44 -07007955void netdev_freemem(struct net_device *dev)
7956{
7957 char *addr = (char *)dev - dev->padded;
7958
WANG Cong4cb28972014-06-02 15:55:22 -07007959 kvfree(addr);
Eric Dumazet74d332c2013-10-30 13:10:44 -07007960}
7961
Linus Torvalds1da177e2005-04-16 15:20:36 -07007962/**
tcharding722c9a02017-02-09 17:56:04 +11007963 * alloc_netdev_mqs - allocate network device
7964 * @sizeof_priv: size of private data to allocate space for
7965 * @name: device name format string
7966 * @name_assign_type: origin of device name
7967 * @setup: callback to initialize device
7968 * @txqs: the number of TX subqueues to allocate
7969 * @rxqs: the number of RX subqueues to allocate
Linus Torvalds1da177e2005-04-16 15:20:36 -07007970 *
tcharding722c9a02017-02-09 17:56:04 +11007971 * Allocates a struct net_device with private data area for driver use
7972 * and performs basic initialization. Also allocates subqueue structs
7973 * for each queue on the device.
Linus Torvalds1da177e2005-04-16 15:20:36 -07007974 */
Tom Herbert36909ea2011-01-09 19:36:31 +00007975struct net_device *alloc_netdev_mqs(int sizeof_priv, const char *name,
Tom Gundersenc835a672014-07-14 16:37:24 +02007976 unsigned char name_assign_type,
Tom Herbert36909ea2011-01-09 19:36:31 +00007977 void (*setup)(struct net_device *),
7978 unsigned int txqs, unsigned int rxqs)
Linus Torvalds1da177e2005-04-16 15:20:36 -07007979{
Linus Torvalds1da177e2005-04-16 15:20:36 -07007980 struct net_device *dev;
Stephen Hemminger79439862008-07-21 13:28:44 -07007981 size_t alloc_size;
Eric Dumazet1ce8e7b2009-05-27 04:42:37 +00007982 struct net_device *p;
Linus Torvalds1da177e2005-04-16 15:20:36 -07007983
Stephen Hemmingerb6fe17d2006-08-29 17:06:13 -07007984 BUG_ON(strlen(name) >= sizeof(dev->name));
7985
Tom Herbert36909ea2011-01-09 19:36:31 +00007986 if (txqs < 1) {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00007987 pr_err("alloc_netdev: Unable to allocate device with zero queues\n");
Tom Herbert55513fb2010-10-18 17:55:58 +00007988 return NULL;
7989 }
7990
Michael Daltona953be52014-01-16 22:23:28 -08007991#ifdef CONFIG_SYSFS
Tom Herbert36909ea2011-01-09 19:36:31 +00007992 if (rxqs < 1) {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00007993 pr_err("alloc_netdev: Unable to allocate device with zero RX queues\n");
Tom Herbert36909ea2011-01-09 19:36:31 +00007994 return NULL;
7995 }
7996#endif
7997
David S. Millerfd2ea0a2008-07-17 01:56:23 -07007998 alloc_size = sizeof(struct net_device);
Alexey Dobriyand1643d22008-04-18 15:43:32 -07007999 if (sizeof_priv) {
8000 /* ensure 32-byte alignment of private area */
Eric Dumazet1ce8e7b2009-05-27 04:42:37 +00008001 alloc_size = ALIGN(alloc_size, NETDEV_ALIGN);
Alexey Dobriyand1643d22008-04-18 15:43:32 -07008002 alloc_size += sizeof_priv;
8003 }
8004 /* ensure 32-byte alignment of whole construct */
Eric Dumazet1ce8e7b2009-05-27 04:42:37 +00008005 alloc_size += NETDEV_ALIGN - 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -07008006
Michal Hockodcda9b02017-07-12 14:36:45 -07008007 p = kvzalloc(alloc_size, GFP_KERNEL | __GFP_RETRY_MAYFAIL);
Joe Perches62b59422013-02-04 16:48:16 +00008008 if (!p)
Linus Torvalds1da177e2005-04-16 15:20:36 -07008009 return NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07008010
Eric Dumazet1ce8e7b2009-05-27 04:42:37 +00008011 dev = PTR_ALIGN(p, NETDEV_ALIGN);
Linus Torvalds1da177e2005-04-16 15:20:36 -07008012 dev->padded = (char *)dev - (char *)p;
Jiri Pirkoab9c73c2009-05-08 13:30:17 +00008013
Eric Dumazet29b44332010-10-11 10:22:12 +00008014 dev->pcpu_refcnt = alloc_percpu(int);
8015 if (!dev->pcpu_refcnt)
Eric Dumazet74d332c2013-10-30 13:10:44 -07008016 goto free_dev;
Jiri Pirkoab9c73c2009-05-08 13:30:17 +00008017
Linus Torvalds1da177e2005-04-16 15:20:36 -07008018 if (dev_addr_init(dev))
Eric Dumazet29b44332010-10-11 10:22:12 +00008019 goto free_pcpu;
Linus Torvalds1da177e2005-04-16 15:20:36 -07008020
Jiri Pirko22bedad32010-04-01 21:22:57 +00008021 dev_mc_init(dev);
Jiri Pirkoa748ee22010-04-01 21:22:09 +00008022 dev_uc_init(dev);
Jiri Pirkoccffad252009-05-22 23:22:17 +00008023
YOSHIFUJI Hideakic346dca2008-03-25 21:47:49 +09008024 dev_net_set(dev, &init_net);
Linus Torvalds1da177e2005-04-16 15:20:36 -07008025
Peter P Waskiewicz Jr82cc1a72008-03-21 03:43:19 -07008026 dev->gso_max_size = GSO_MAX_SIZE;
Ben Hutchings30b678d2012-07-30 15:57:00 +00008027 dev->gso_max_segs = GSO_MAX_SEGS;
Linus Torvalds1da177e2005-04-16 15:20:36 -07008028
Herbert Xud565b0a2008-12-15 23:38:52 -08008029 INIT_LIST_HEAD(&dev->napi_list);
Eric W. Biederman9fdce092009-10-30 14:51:13 +00008030 INIT_LIST_HEAD(&dev->unreg_list);
Eric W. Biederman5cde2822013-10-05 19:26:05 -07008031 INIT_LIST_HEAD(&dev->close_list);
Eric Dumazete014deb2009-11-17 05:59:21 +00008032 INIT_LIST_HEAD(&dev->link_watch_list);
Veaceslav Falico2f268f12013-09-25 09:20:07 +02008033 INIT_LIST_HEAD(&dev->adj_list.upper);
8034 INIT_LIST_HEAD(&dev->adj_list.lower);
Salam Noureddine7866a622015-01-27 11:35:48 -08008035 INIT_LIST_HEAD(&dev->ptype_all);
8036 INIT_LIST_HEAD(&dev->ptype_specific);
Jiri Kosina59cc1f62016-08-10 11:05:15 +02008037#ifdef CONFIG_NET_SCHED
8038 hash_init(dev->qdisc_hash);
8039#endif
Eric Dumazet02875872014-10-05 18:38:35 -07008040 dev->priv_flags = IFF_XMIT_DST_RELEASE | IFF_XMIT_DST_RELEASE_PERM;
Linus Torvalds1da177e2005-04-16 15:20:36 -07008041 setup(dev);
David S. Miller8d3bdbd2011-02-08 15:02:50 -08008042
Phil Suttera8131042016-02-17 15:37:43 +01008043 if (!dev->tx_queue_len) {
Phil Sutterf84bb1e2015-08-27 21:21:36 +02008044 dev->priv_flags |= IFF_NO_QUEUE;
Jesper Dangaard Brouer11597082016-11-03 14:56:06 +01008045 dev->tx_queue_len = DEFAULT_TX_QUEUE_LEN;
Phil Suttera8131042016-02-17 15:37:43 +01008046 }
Phil Sutter906470c2015-08-18 10:30:48 +02008047
David S. Miller8d3bdbd2011-02-08 15:02:50 -08008048 dev->num_tx_queues = txqs;
8049 dev->real_num_tx_queues = txqs;
8050 if (netif_alloc_netdev_queues(dev))
8051 goto free_all;
8052
Michael Daltona953be52014-01-16 22:23:28 -08008053#ifdef CONFIG_SYSFS
David S. Miller8d3bdbd2011-02-08 15:02:50 -08008054 dev->num_rx_queues = rxqs;
8055 dev->real_num_rx_queues = rxqs;
8056 if (netif_alloc_rx_queues(dev))
8057 goto free_all;
8058#endif
8059
Linus Torvalds1da177e2005-04-16 15:20:36 -07008060 strcpy(dev->name, name);
Tom Gundersenc835a672014-07-14 16:37:24 +02008061 dev->name_assign_type = name_assign_type;
Vlad Dogarucbda10f2011-01-13 23:38:30 +00008062 dev->group = INIT_NETDEV_GROUP;
Eric Dumazet2c60db02012-09-16 09:17:26 +00008063 if (!dev->ethtool_ops)
8064 dev->ethtool_ops = &default_ethtool_ops;
Pablo Neirae687ad62015-05-13 18:19:38 +02008065
8066 nf_hook_ingress_init(dev);
8067
Linus Torvalds1da177e2005-04-16 15:20:36 -07008068 return dev;
Jiri Pirkoab9c73c2009-05-08 13:30:17 +00008069
David S. Miller8d3bdbd2011-02-08 15:02:50 -08008070free_all:
8071 free_netdev(dev);
8072 return NULL;
8073
Eric Dumazet29b44332010-10-11 10:22:12 +00008074free_pcpu:
8075 free_percpu(dev->pcpu_refcnt);
Eric Dumazet74d332c2013-10-30 13:10:44 -07008076free_dev:
8077 netdev_freemem(dev);
Jiri Pirkoab9c73c2009-05-08 13:30:17 +00008078 return NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07008079}
Tom Herbert36909ea2011-01-09 19:36:31 +00008080EXPORT_SYMBOL(alloc_netdev_mqs);
Linus Torvalds1da177e2005-04-16 15:20:36 -07008081
8082/**
tcharding722c9a02017-02-09 17:56:04 +11008083 * free_netdev - free network device
8084 * @dev: device
Linus Torvalds1da177e2005-04-16 15:20:36 -07008085 *
tcharding722c9a02017-02-09 17:56:04 +11008086 * This function does the last stage of destroying an allocated device
8087 * interface. The reference to the device object is released. If this
8088 * is the last reference then it will be freed.Must be called in process
8089 * context.
Linus Torvalds1da177e2005-04-16 15:20:36 -07008090 */
8091void free_netdev(struct net_device *dev)
8092{
Herbert Xud565b0a2008-12-15 23:38:52 -08008093 struct napi_struct *p, *n;
David S. Millerb5cdae32017-04-18 15:36:58 -04008094 struct bpf_prog *prog;
Herbert Xud565b0a2008-12-15 23:38:52 -08008095
Eric Dumazet93d05d42015-11-18 06:31:03 -08008096 might_sleep();
Eric Dumazet60877a32013-06-20 01:15:51 -07008097 netif_free_tx_queues(dev);
Michael Daltona953be52014-01-16 22:23:28 -08008098#ifdef CONFIG_SYSFS
Pankaj Gupta10595902015-01-12 11:41:28 +05308099 kvfree(dev->_rx);
Tom Herbertfe822242010-11-09 10:47:38 +00008100#endif
David S. Millere8a04642008-07-17 00:34:19 -07008101
Eric Dumazet33d480c2011-08-11 19:30:52 +00008102 kfree(rcu_dereference_protected(dev->ingress_queue, 1));
Eric Dumazet24824a02010-10-02 06:11:55 +00008103
Jiri Pirkof001fde2009-05-05 02:48:28 +00008104 /* Flush device addresses */
8105 dev_addr_flush(dev);
8106
Herbert Xud565b0a2008-12-15 23:38:52 -08008107 list_for_each_entry_safe(p, n, &dev->napi_list, dev_list)
8108 netif_napi_del(p);
8109
Eric Dumazet29b44332010-10-11 10:22:12 +00008110 free_percpu(dev->pcpu_refcnt);
8111 dev->pcpu_refcnt = NULL;
8112
David S. Millerb5cdae32017-04-18 15:36:58 -04008113 prog = rcu_dereference_protected(dev->xdp_prog, 1);
8114 if (prog) {
8115 bpf_prog_put(prog);
8116 static_key_slow_dec(&generic_xdp_needed);
8117 }
8118
Stephen Hemminger3041a062006-05-26 13:25:24 -07008119 /* Compatibility with error handling in drivers */
Linus Torvalds1da177e2005-04-16 15:20:36 -07008120 if (dev->reg_state == NETREG_UNINITIALIZED) {
Eric Dumazet74d332c2013-10-30 13:10:44 -07008121 netdev_freemem(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07008122 return;
8123 }
8124
8125 BUG_ON(dev->reg_state != NETREG_UNREGISTERED);
8126 dev->reg_state = NETREG_RELEASED;
8127
Greg Kroah-Hartman43cb76d2002-04-09 12:14:34 -07008128 /* will free via device release */
8129 put_device(&dev->dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07008130}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07008131EXPORT_SYMBOL(free_netdev);
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09008132
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07008133/**
8134 * synchronize_net - Synchronize with packet receive processing
8135 *
8136 * Wait for packets currently being received to be done.
8137 * Does not block later packets from starting.
8138 */
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09008139void synchronize_net(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -07008140{
8141 might_sleep();
Eric Dumazetbe3fc412011-05-23 23:07:32 +00008142 if (rtnl_is_locked())
8143 synchronize_rcu_expedited();
8144 else
8145 synchronize_rcu();
Linus Torvalds1da177e2005-04-16 15:20:36 -07008146}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07008147EXPORT_SYMBOL(synchronize_net);
Linus Torvalds1da177e2005-04-16 15:20:36 -07008148
8149/**
Eric Dumazet44a08732009-10-27 07:03:04 +00008150 * unregister_netdevice_queue - remove device from the kernel
Linus Torvalds1da177e2005-04-16 15:20:36 -07008151 * @dev: device
Eric Dumazet44a08732009-10-27 07:03:04 +00008152 * @head: list
Jaswinder Singh Rajput6ebfbc02009-11-22 20:43:13 -08008153 *
Linus Torvalds1da177e2005-04-16 15:20:36 -07008154 * This function shuts down a device interface and removes it
Wang Chend59b54b2007-12-11 02:28:03 -08008155 * from the kernel tables.
Eric Dumazet44a08732009-10-27 07:03:04 +00008156 * If head not NULL, device is queued to be unregistered later.
Linus Torvalds1da177e2005-04-16 15:20:36 -07008157 *
8158 * Callers must hold the rtnl semaphore. You may want
8159 * unregister_netdev() instead of this.
8160 */
8161
Eric Dumazet44a08732009-10-27 07:03:04 +00008162void unregister_netdevice_queue(struct net_device *dev, struct list_head *head)
Linus Torvalds1da177e2005-04-16 15:20:36 -07008163{
Herbert Xua6620712007-12-12 19:21:56 -08008164 ASSERT_RTNL();
8165
Eric Dumazet44a08732009-10-27 07:03:04 +00008166 if (head) {
Eric W. Biederman9fdce092009-10-30 14:51:13 +00008167 list_move_tail(&dev->unreg_list, head);
Eric Dumazet44a08732009-10-27 07:03:04 +00008168 } else {
8169 rollback_registered(dev);
8170 /* Finish processing unregister after unlock */
8171 net_set_todo(dev);
8172 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07008173}
Eric Dumazet44a08732009-10-27 07:03:04 +00008174EXPORT_SYMBOL(unregister_netdevice_queue);
Linus Torvalds1da177e2005-04-16 15:20:36 -07008175
8176/**
Eric Dumazet9b5e3832009-10-27 07:04:19 +00008177 * unregister_netdevice_many - unregister many devices
8178 * @head: list of devices
Eric Dumazet87757a92014-06-06 06:44:03 -07008179 *
8180 * Note: As most callers use a stack allocated list_head,
8181 * we force a list_del() to make sure stack wont be corrupted later.
Eric Dumazet9b5e3832009-10-27 07:04:19 +00008182 */
8183void unregister_netdevice_many(struct list_head *head)
8184{
8185 struct net_device *dev;
8186
8187 if (!list_empty(head)) {
8188 rollback_registered_many(head);
8189 list_for_each_entry(dev, head, unreg_list)
8190 net_set_todo(dev);
Eric Dumazet87757a92014-06-06 06:44:03 -07008191 list_del(head);
Eric Dumazet9b5e3832009-10-27 07:04:19 +00008192 }
8193}
Eric Dumazet63c80992009-10-27 07:06:49 +00008194EXPORT_SYMBOL(unregister_netdevice_many);
Eric Dumazet9b5e3832009-10-27 07:04:19 +00008195
8196/**
Linus Torvalds1da177e2005-04-16 15:20:36 -07008197 * unregister_netdev - remove device from the kernel
8198 * @dev: device
8199 *
8200 * This function shuts down a device interface and removes it
Wang Chend59b54b2007-12-11 02:28:03 -08008201 * from the kernel tables.
Linus Torvalds1da177e2005-04-16 15:20:36 -07008202 *
8203 * This is just a wrapper for unregister_netdevice that takes
8204 * the rtnl semaphore. In general you want to use this and not
8205 * unregister_netdevice.
8206 */
8207void unregister_netdev(struct net_device *dev)
8208{
8209 rtnl_lock();
8210 unregister_netdevice(dev);
8211 rtnl_unlock();
8212}
Linus Torvalds1da177e2005-04-16 15:20:36 -07008213EXPORT_SYMBOL(unregister_netdev);
8214
Eric W. Biedermance286d32007-09-12 13:53:49 +02008215/**
8216 * dev_change_net_namespace - move device to different nethost namespace
8217 * @dev: device
8218 * @net: network namespace
8219 * @pat: If not NULL name pattern to try if the current device name
8220 * is already taken in the destination network namespace.
8221 *
8222 * This function shuts down a device interface and moves it
8223 * to a new network namespace. On success 0 is returned, on
8224 * a failure a netagive errno code is returned.
8225 *
8226 * Callers must hold the rtnl semaphore.
8227 */
8228
8229int dev_change_net_namespace(struct net_device *dev, struct net *net, const char *pat)
8230{
Eric W. Biedermance286d32007-09-12 13:53:49 +02008231 int err;
8232
8233 ASSERT_RTNL();
8234
8235 /* Don't allow namespace local devices to be moved. */
8236 err = -EINVAL;
8237 if (dev->features & NETIF_F_NETNS_LOCAL)
8238 goto out;
8239
8240 /* Ensure the device has been registrered */
Eric W. Biedermance286d32007-09-12 13:53:49 +02008241 if (dev->reg_state != NETREG_REGISTERED)
8242 goto out;
8243
8244 /* Get out if there is nothing todo */
8245 err = 0;
YOSHIFUJI Hideaki878628f2008-03-26 03:57:35 +09008246 if (net_eq(dev_net(dev), net))
Eric W. Biedermance286d32007-09-12 13:53:49 +02008247 goto out;
8248
8249 /* Pick the destination device name, and ensure
8250 * we can use it in the destination network namespace.
8251 */
8252 err = -EEXIST;
Octavian Purdilad9031022009-11-18 02:36:59 +00008253 if (__dev_get_by_name(net, dev->name)) {
Eric W. Biedermance286d32007-09-12 13:53:49 +02008254 /* We get here if we can't use the current device name */
8255 if (!pat)
8256 goto out;
Gao feng828de4f2012-09-13 20:58:27 +00008257 if (dev_get_valid_name(net, dev, pat) < 0)
Eric W. Biedermance286d32007-09-12 13:53:49 +02008258 goto out;
8259 }
8260
8261 /*
8262 * And now a mini version of register_netdevice unregister_netdevice.
8263 */
8264
8265 /* If device is running close it first. */
Pavel Emelyanov9b772652007-10-10 02:49:09 -07008266 dev_close(dev);
Eric W. Biedermance286d32007-09-12 13:53:49 +02008267
8268 /* And unlink it from device chain */
8269 err = -ENODEV;
8270 unlist_netdevice(dev);
8271
8272 synchronize_net();
8273
8274 /* Shutdown queueing discipline. */
8275 dev_shutdown(dev);
8276
8277 /* Notify protocols, that we are about to destroy
tchardingeb13da12017-02-09 17:56:06 +11008278 * this device. They should clean all the things.
8279 *
8280 * Note that dev->reg_state stays at NETREG_REGISTERED.
8281 * This is wanted because this way 8021q and macvlan know
8282 * the device is just moving and can keep their slaves up.
8283 */
Eric W. Biedermance286d32007-09-12 13:53:49 +02008284 call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
Gao feng6549dd42012-08-23 15:36:55 +00008285 rcu_barrier();
8286 call_netdevice_notifiers(NETDEV_UNREGISTER_FINAL, dev);
Alexei Starovoitov7f294052013-10-23 16:02:42 -07008287 rtmsg_ifinfo(RTM_DELLINK, dev, ~0U, GFP_KERNEL);
Eric W. Biedermance286d32007-09-12 13:53:49 +02008288
8289 /*
8290 * Flush the unicast and multicast chains
8291 */
Jiri Pirkoa748ee22010-04-01 21:22:09 +00008292 dev_uc_flush(dev);
Jiri Pirko22bedad32010-04-01 21:22:57 +00008293 dev_mc_flush(dev);
Eric W. Biedermance286d32007-09-12 13:53:49 +02008294
Serge Hallyn4e66ae22012-12-03 16:17:12 +00008295 /* Send a netdev-removed uevent to the old namespace */
8296 kobject_uevent(&dev->dev.kobj, KOBJ_REMOVE);
Alexander Y. Fomichev4c754312014-08-25 16:26:45 +04008297 netdev_adjacent_del_links(dev);
Serge Hallyn4e66ae22012-12-03 16:17:12 +00008298
Eric W. Biedermance286d32007-09-12 13:53:49 +02008299 /* Actually switch the network namespace */
YOSHIFUJI Hideakic346dca2008-03-25 21:47:49 +09008300 dev_net_set(dev, net);
Eric W. Biedermance286d32007-09-12 13:53:49 +02008301
Eric W. Biedermance286d32007-09-12 13:53:49 +02008302 /* If there is an ifindex conflict assign a new one */
Nicolas Dichtel7a66bbc2015-04-02 17:07:09 +02008303 if (__dev_get_by_index(net, dev->ifindex))
Eric W. Biedermance286d32007-09-12 13:53:49 +02008304 dev->ifindex = dev_new_index(net);
Eric W. Biedermance286d32007-09-12 13:53:49 +02008305
Serge Hallyn4e66ae22012-12-03 16:17:12 +00008306 /* Send a netdev-add uevent to the new namespace */
8307 kobject_uevent(&dev->dev.kobj, KOBJ_ADD);
Alexander Y. Fomichev4c754312014-08-25 16:26:45 +04008308 netdev_adjacent_add_links(dev);
Serge Hallyn4e66ae22012-12-03 16:17:12 +00008309
Eric W. Biederman8b41d182007-09-26 22:02:53 -07008310 /* Fixup kobjects */
Eric W. Biedermana1b3f592010-05-04 17:36:49 -07008311 err = device_rename(&dev->dev, dev->name);
Eric W. Biederman8b41d182007-09-26 22:02:53 -07008312 WARN_ON(err);
Eric W. Biedermance286d32007-09-12 13:53:49 +02008313
8314 /* Add the device back in the hashes */
8315 list_netdevice(dev);
8316
8317 /* Notify protocols, that a new device appeared. */
8318 call_netdevice_notifiers(NETDEV_REGISTER, dev);
8319
Eric W. Biedermand90a9092009-12-12 22:11:15 +00008320 /*
8321 * Prevent userspace races by waiting until the network
8322 * device is fully setup before sending notifications.
8323 */
Alexei Starovoitov7f294052013-10-23 16:02:42 -07008324 rtmsg_ifinfo(RTM_NEWLINK, dev, ~0U, GFP_KERNEL);
Eric W. Biedermand90a9092009-12-12 22:11:15 +00008325
Eric W. Biedermance286d32007-09-12 13:53:49 +02008326 synchronize_net();
8327 err = 0;
8328out:
8329 return err;
8330}
Johannes Berg463d0182009-07-14 00:33:35 +02008331EXPORT_SYMBOL_GPL(dev_change_net_namespace);
Eric W. Biedermance286d32007-09-12 13:53:49 +02008332
Sebastian Andrzej Siewiorf0bf90d2016-11-03 15:50:04 +01008333static int dev_cpu_dead(unsigned int oldcpu)
Linus Torvalds1da177e2005-04-16 15:20:36 -07008334{
8335 struct sk_buff **list_skb;
Linus Torvalds1da177e2005-04-16 15:20:36 -07008336 struct sk_buff *skb;
Sebastian Andrzej Siewiorf0bf90d2016-11-03 15:50:04 +01008337 unsigned int cpu;
Ashwanth Goli97d8b6e2017-06-13 16:54:55 +05308338 struct softnet_data *sd, *oldsd, *remsd = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07008339
Linus Torvalds1da177e2005-04-16 15:20:36 -07008340 local_irq_disable();
8341 cpu = smp_processor_id();
8342 sd = &per_cpu(softnet_data, cpu);
8343 oldsd = &per_cpu(softnet_data, oldcpu);
8344
8345 /* Find end of our completion_queue. */
8346 list_skb = &sd->completion_queue;
8347 while (*list_skb)
8348 list_skb = &(*list_skb)->next;
8349 /* Append completion queue from offline CPU. */
8350 *list_skb = oldsd->completion_queue;
8351 oldsd->completion_queue = NULL;
8352
Linus Torvalds1da177e2005-04-16 15:20:36 -07008353 /* Append output queue from offline CPU. */
Changli Gaoa9cbd582010-04-26 23:06:24 +00008354 if (oldsd->output_queue) {
8355 *sd->output_queue_tailp = oldsd->output_queue;
8356 sd->output_queue_tailp = oldsd->output_queue_tailp;
8357 oldsd->output_queue = NULL;
8358 oldsd->output_queue_tailp = &oldsd->output_queue;
8359 }
Eric Dumazetac64da02015-01-15 17:04:22 -08008360 /* Append NAPI poll list from offline CPU, with one exception :
8361 * process_backlog() must be called by cpu owning percpu backlog.
8362 * We properly handle process_queue & input_pkt_queue later.
8363 */
8364 while (!list_empty(&oldsd->poll_list)) {
8365 struct napi_struct *napi = list_first_entry(&oldsd->poll_list,
8366 struct napi_struct,
8367 poll_list);
8368
8369 list_del_init(&napi->poll_list);
8370 if (napi->poll == process_backlog)
8371 napi->state = 0;
8372 else
8373 ____napi_schedule(sd, napi);
Heiko Carstens264524d2011-06-06 20:50:03 +00008374 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07008375
8376 raise_softirq_irqoff(NET_TX_SOFTIRQ);
8377 local_irq_enable();
8378
ashwanth@codeaurora.org773fc8f2017-06-09 14:24:58 +05308379#ifdef CONFIG_RPS
8380 remsd = oldsd->rps_ipi_list;
8381 oldsd->rps_ipi_list = NULL;
8382#endif
8383 /* send out pending IPI's on offline CPU */
8384 net_rps_send_ipi(remsd);
8385
Linus Torvalds1da177e2005-04-16 15:20:36 -07008386 /* Process offline CPU's input_pkt_queue */
Tom Herbert76cc8b12010-05-20 18:37:59 +00008387 while ((skb = __skb_dequeue(&oldsd->process_queue))) {
Eric Dumazet91e83132015-02-05 14:58:14 -08008388 netif_rx_ni(skb);
Tom Herbert76cc8b12010-05-20 18:37:59 +00008389 input_queue_head_incr(oldsd);
8390 }
Eric Dumazetac64da02015-01-15 17:04:22 -08008391 while ((skb = skb_dequeue(&oldsd->input_pkt_queue))) {
Eric Dumazet91e83132015-02-05 14:58:14 -08008392 netif_rx_ni(skb);
Tom Herbert76cc8b12010-05-20 18:37:59 +00008393 input_queue_head_incr(oldsd);
Tom Herbertfec5e652010-04-16 16:01:27 -07008394 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07008395
Sebastian Andrzej Siewiorf0bf90d2016-11-03 15:50:04 +01008396 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07008397}
Linus Torvalds1da177e2005-04-16 15:20:36 -07008398
Herbert Xu7f353bf2007-08-10 15:47:58 -07008399/**
Herbert Xub63365a2008-10-23 01:11:29 -07008400 * netdev_increment_features - increment feature set by one
8401 * @all: current feature set
8402 * @one: new feature set
8403 * @mask: mask feature set
Herbert Xu7f353bf2007-08-10 15:47:58 -07008404 *
8405 * Computes a new feature set after adding a device with feature set
Herbert Xub63365a2008-10-23 01:11:29 -07008406 * @one to the master device with current feature set @all. Will not
8407 * enable anything that is off in @mask. Returns the new feature set.
Herbert Xu7f353bf2007-08-10 15:47:58 -07008408 */
Michał Mirosławc8f44af2011-11-15 15:29:55 +00008409netdev_features_t netdev_increment_features(netdev_features_t all,
8410 netdev_features_t one, netdev_features_t mask)
Herbert Xu7f353bf2007-08-10 15:47:58 -07008411{
Tom Herbertc8cd0982015-12-14 11:19:44 -08008412 if (mask & NETIF_F_HW_CSUM)
Tom Herberta1882222015-12-14 11:19:43 -08008413 mask |= NETIF_F_CSUM_MASK;
Michał Mirosław1742f182011-04-22 06:31:16 +00008414 mask |= NETIF_F_VLAN_CHALLENGED;
8415
Tom Herberta1882222015-12-14 11:19:43 -08008416 all |= one & (NETIF_F_ONE_FOR_ALL | NETIF_F_CSUM_MASK) & mask;
Michał Mirosław1742f182011-04-22 06:31:16 +00008417 all &= one | ~NETIF_F_ALL_FOR_ALL;
8418
Michał Mirosław1742f182011-04-22 06:31:16 +00008419 /* If one device supports hw checksumming, set for all. */
Tom Herbertc8cd0982015-12-14 11:19:44 -08008420 if (all & NETIF_F_HW_CSUM)
8421 all &= ~(NETIF_F_CSUM_MASK & ~NETIF_F_HW_CSUM);
Herbert Xu7f353bf2007-08-10 15:47:58 -07008422
8423 return all;
8424}
Herbert Xub63365a2008-10-23 01:11:29 -07008425EXPORT_SYMBOL(netdev_increment_features);
Herbert Xu7f353bf2007-08-10 15:47:58 -07008426
Baruch Siach430f03c2013-06-02 20:43:55 +00008427static struct hlist_head * __net_init netdev_create_hash(void)
Pavel Emelyanov30d97d32007-09-16 15:40:33 -07008428{
8429 int i;
8430 struct hlist_head *hash;
8431
8432 hash = kmalloc(sizeof(*hash) * NETDEV_HASHENTRIES, GFP_KERNEL);
8433 if (hash != NULL)
8434 for (i = 0; i < NETDEV_HASHENTRIES; i++)
8435 INIT_HLIST_HEAD(&hash[i]);
8436
8437 return hash;
8438}
8439
Eric W. Biederman881d9662007-09-17 11:56:21 -07008440/* Initialize per network namespace state */
Pavel Emelyanov46650792007-10-08 20:38:39 -07008441static int __net_init netdev_init(struct net *net)
Eric W. Biederman881d9662007-09-17 11:56:21 -07008442{
Rustad, Mark D734b6542012-07-18 09:06:07 +00008443 if (net != &init_net)
8444 INIT_LIST_HEAD(&net->dev_base_head);
Eric W. Biederman881d9662007-09-17 11:56:21 -07008445
Pavel Emelyanov30d97d32007-09-16 15:40:33 -07008446 net->dev_name_head = netdev_create_hash();
8447 if (net->dev_name_head == NULL)
8448 goto err_name;
Eric W. Biederman881d9662007-09-17 11:56:21 -07008449
Pavel Emelyanov30d97d32007-09-16 15:40:33 -07008450 net->dev_index_head = netdev_create_hash();
8451 if (net->dev_index_head == NULL)
8452 goto err_idx;
Eric W. Biederman881d9662007-09-17 11:56:21 -07008453
8454 return 0;
Pavel Emelyanov30d97d32007-09-16 15:40:33 -07008455
8456err_idx:
8457 kfree(net->dev_name_head);
8458err_name:
8459 return -ENOMEM;
Eric W. Biederman881d9662007-09-17 11:56:21 -07008460}
8461
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07008462/**
8463 * netdev_drivername - network driver for the device
8464 * @dev: network device
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07008465 *
8466 * Determine network driver for device.
8467 */
David S. Miller3019de12011-06-06 16:41:33 -07008468const char *netdev_drivername(const struct net_device *dev)
Arjan van de Ven6579e572008-07-21 13:31:48 -07008469{
Stephen Hemmingercf04a4c72008-09-30 02:22:14 -07008470 const struct device_driver *driver;
8471 const struct device *parent;
David S. Miller3019de12011-06-06 16:41:33 -07008472 const char *empty = "";
Arjan van de Ven6579e572008-07-21 13:31:48 -07008473
8474 parent = dev->dev.parent;
Arjan van de Ven6579e572008-07-21 13:31:48 -07008475 if (!parent)
David S. Miller3019de12011-06-06 16:41:33 -07008476 return empty;
Arjan van de Ven6579e572008-07-21 13:31:48 -07008477
8478 driver = parent->driver;
8479 if (driver && driver->name)
David S. Miller3019de12011-06-06 16:41:33 -07008480 return driver->name;
8481 return empty;
Arjan van de Ven6579e572008-07-21 13:31:48 -07008482}
8483
Joe Perches6ea754e2014-09-22 11:10:50 -07008484static void __netdev_printk(const char *level, const struct net_device *dev,
8485 struct va_format *vaf)
Joe Perches256df2f2010-06-27 01:02:35 +00008486{
Joe Perchesb004ff42012-09-12 20:12:19 -07008487 if (dev && dev->dev.parent) {
Joe Perches6ea754e2014-09-22 11:10:50 -07008488 dev_printk_emit(level[1] - '0',
8489 dev->dev.parent,
8490 "%s %s %s%s: %pV",
8491 dev_driver_string(dev->dev.parent),
8492 dev_name(dev->dev.parent),
8493 netdev_name(dev), netdev_reg_state(dev),
8494 vaf);
Joe Perchesb004ff42012-09-12 20:12:19 -07008495 } else if (dev) {
Joe Perches6ea754e2014-09-22 11:10:50 -07008496 printk("%s%s%s: %pV",
8497 level, netdev_name(dev), netdev_reg_state(dev), vaf);
Joe Perchesb004ff42012-09-12 20:12:19 -07008498 } else {
Joe Perches6ea754e2014-09-22 11:10:50 -07008499 printk("%s(NULL net_device): %pV", level, vaf);
Joe Perchesb004ff42012-09-12 20:12:19 -07008500 }
Joe Perches256df2f2010-06-27 01:02:35 +00008501}
8502
Joe Perches6ea754e2014-09-22 11:10:50 -07008503void netdev_printk(const char *level, const struct net_device *dev,
8504 const char *format, ...)
Joe Perches256df2f2010-06-27 01:02:35 +00008505{
8506 struct va_format vaf;
8507 va_list args;
Joe Perches256df2f2010-06-27 01:02:35 +00008508
8509 va_start(args, format);
8510
8511 vaf.fmt = format;
8512 vaf.va = &args;
8513
Joe Perches6ea754e2014-09-22 11:10:50 -07008514 __netdev_printk(level, dev, &vaf);
Joe Perchesb004ff42012-09-12 20:12:19 -07008515
Joe Perches256df2f2010-06-27 01:02:35 +00008516 va_end(args);
Joe Perches256df2f2010-06-27 01:02:35 +00008517}
8518EXPORT_SYMBOL(netdev_printk);
8519
8520#define define_netdev_printk_level(func, level) \
Joe Perches6ea754e2014-09-22 11:10:50 -07008521void func(const struct net_device *dev, const char *fmt, ...) \
Joe Perches256df2f2010-06-27 01:02:35 +00008522{ \
Joe Perches256df2f2010-06-27 01:02:35 +00008523 struct va_format vaf; \
8524 va_list args; \
8525 \
8526 va_start(args, fmt); \
8527 \
8528 vaf.fmt = fmt; \
8529 vaf.va = &args; \
8530 \
Joe Perches6ea754e2014-09-22 11:10:50 -07008531 __netdev_printk(level, dev, &vaf); \
Joe Perchesb004ff42012-09-12 20:12:19 -07008532 \
Joe Perches256df2f2010-06-27 01:02:35 +00008533 va_end(args); \
Joe Perches256df2f2010-06-27 01:02:35 +00008534} \
8535EXPORT_SYMBOL(func);
8536
8537define_netdev_printk_level(netdev_emerg, KERN_EMERG);
8538define_netdev_printk_level(netdev_alert, KERN_ALERT);
8539define_netdev_printk_level(netdev_crit, KERN_CRIT);
8540define_netdev_printk_level(netdev_err, KERN_ERR);
8541define_netdev_printk_level(netdev_warn, KERN_WARNING);
8542define_netdev_printk_level(netdev_notice, KERN_NOTICE);
8543define_netdev_printk_level(netdev_info, KERN_INFO);
8544
Pavel Emelyanov46650792007-10-08 20:38:39 -07008545static void __net_exit netdev_exit(struct net *net)
Eric W. Biederman881d9662007-09-17 11:56:21 -07008546{
8547 kfree(net->dev_name_head);
8548 kfree(net->dev_index_head);
8549}
8550
Denis V. Lunev022cbae2007-11-13 03:23:50 -08008551static struct pernet_operations __net_initdata netdev_net_ops = {
Eric W. Biederman881d9662007-09-17 11:56:21 -07008552 .init = netdev_init,
8553 .exit = netdev_exit,
8554};
8555
Pavel Emelyanov46650792007-10-08 20:38:39 -07008556static void __net_exit default_device_exit(struct net *net)
Eric W. Biedermance286d32007-09-12 13:53:49 +02008557{
Eric W. Biedermane008b5f2009-11-29 22:25:30 +00008558 struct net_device *dev, *aux;
Eric W. Biedermance286d32007-09-12 13:53:49 +02008559 /*
Eric W. Biedermane008b5f2009-11-29 22:25:30 +00008560 * Push all migratable network devices back to the
Eric W. Biedermance286d32007-09-12 13:53:49 +02008561 * initial network namespace
8562 */
8563 rtnl_lock();
Eric W. Biedermane008b5f2009-11-29 22:25:30 +00008564 for_each_netdev_safe(net, dev, aux) {
Eric W. Biedermance286d32007-09-12 13:53:49 +02008565 int err;
Pavel Emelyanovaca51392008-05-08 01:24:25 -07008566 char fb_name[IFNAMSIZ];
Eric W. Biedermance286d32007-09-12 13:53:49 +02008567
8568 /* Ignore unmoveable devices (i.e. loopback) */
8569 if (dev->features & NETIF_F_NETNS_LOCAL)
8570 continue;
8571
Eric W. Biedermane008b5f2009-11-29 22:25:30 +00008572 /* Leave virtual devices for the generic cleanup */
8573 if (dev->rtnl_link_ops)
8574 continue;
Eric W. Biedermand0c082c2008-11-05 15:59:38 -08008575
Lucas De Marchi25985ed2011-03-30 22:57:33 -03008576 /* Push remaining network devices to init_net */
Pavel Emelyanovaca51392008-05-08 01:24:25 -07008577 snprintf(fb_name, IFNAMSIZ, "dev%d", dev->ifindex);
8578 err = dev_change_net_namespace(dev, &init_net, fb_name);
Eric W. Biedermance286d32007-09-12 13:53:49 +02008579 if (err) {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00008580 pr_emerg("%s: failed to move %s to init_net: %d\n",
8581 __func__, dev->name, err);
Pavel Emelyanovaca51392008-05-08 01:24:25 -07008582 BUG();
Eric W. Biedermance286d32007-09-12 13:53:49 +02008583 }
8584 }
8585 rtnl_unlock();
8586}
8587
Eric W. Biederman50624c92013-09-23 21:19:49 -07008588static void __net_exit rtnl_lock_unregistering(struct list_head *net_list)
8589{
8590 /* Return with the rtnl_lock held when there are no network
8591 * devices unregistering in any network namespace in net_list.
8592 */
8593 struct net *net;
8594 bool unregistering;
Peter Zijlstraff960a72014-10-29 17:04:56 +01008595 DEFINE_WAIT_FUNC(wait, woken_wake_function);
Eric W. Biederman50624c92013-09-23 21:19:49 -07008596
Peter Zijlstraff960a72014-10-29 17:04:56 +01008597 add_wait_queue(&netdev_unregistering_wq, &wait);
Eric W. Biederman50624c92013-09-23 21:19:49 -07008598 for (;;) {
Eric W. Biederman50624c92013-09-23 21:19:49 -07008599 unregistering = false;
8600 rtnl_lock();
8601 list_for_each_entry(net, net_list, exit_list) {
8602 if (net->dev_unreg_count > 0) {
8603 unregistering = true;
8604 break;
8605 }
8606 }
8607 if (!unregistering)
8608 break;
8609 __rtnl_unlock();
Peter Zijlstraff960a72014-10-29 17:04:56 +01008610
8611 wait_woken(&wait, TASK_UNINTERRUPTIBLE, MAX_SCHEDULE_TIMEOUT);
Eric W. Biederman50624c92013-09-23 21:19:49 -07008612 }
Peter Zijlstraff960a72014-10-29 17:04:56 +01008613 remove_wait_queue(&netdev_unregistering_wq, &wait);
Eric W. Biederman50624c92013-09-23 21:19:49 -07008614}
8615
Eric W. Biederman04dc7f6b2009-12-03 02:29:04 +00008616static void __net_exit default_device_exit_batch(struct list_head *net_list)
8617{
8618 /* At exit all network devices most be removed from a network
Uwe Kleine-Königb5950762010-11-01 15:38:34 -04008619 * namespace. Do this in the reverse order of registration.
Eric W. Biederman04dc7f6b2009-12-03 02:29:04 +00008620 * Do this across as many network namespaces as possible to
8621 * improve batching efficiency.
8622 */
8623 struct net_device *dev;
8624 struct net *net;
8625 LIST_HEAD(dev_kill_list);
8626
Eric W. Biederman50624c92013-09-23 21:19:49 -07008627 /* To prevent network device cleanup code from dereferencing
8628 * loopback devices or network devices that have been freed
8629 * wait here for all pending unregistrations to complete,
8630 * before unregistring the loopback device and allowing the
8631 * network namespace be freed.
8632 *
8633 * The netdev todo list containing all network devices
8634 * unregistrations that happen in default_device_exit_batch
8635 * will run in the rtnl_unlock() at the end of
8636 * default_device_exit_batch.
8637 */
8638 rtnl_lock_unregistering(net_list);
Eric W. Biederman04dc7f6b2009-12-03 02:29:04 +00008639 list_for_each_entry(net, net_list, exit_list) {
8640 for_each_netdev_reverse(net, dev) {
Jiri Pirkob0ab2fa2014-06-26 09:58:25 +02008641 if (dev->rtnl_link_ops && dev->rtnl_link_ops->dellink)
Eric W. Biederman04dc7f6b2009-12-03 02:29:04 +00008642 dev->rtnl_link_ops->dellink(dev, &dev_kill_list);
8643 else
8644 unregister_netdevice_queue(dev, &dev_kill_list);
8645 }
8646 }
8647 unregister_netdevice_many(&dev_kill_list);
8648 rtnl_unlock();
8649}
8650
Denis V. Lunev022cbae2007-11-13 03:23:50 -08008651static struct pernet_operations __net_initdata default_device_ops = {
Eric W. Biedermance286d32007-09-12 13:53:49 +02008652 .exit = default_device_exit,
Eric W. Biederman04dc7f6b2009-12-03 02:29:04 +00008653 .exit_batch = default_device_exit_batch,
Eric W. Biedermance286d32007-09-12 13:53:49 +02008654};
8655
Linus Torvalds1da177e2005-04-16 15:20:36 -07008656/*
8657 * Initialize the DEV module. At boot time this walks the device list and
8658 * unhooks any devices that fail to initialise (normally hardware not
8659 * present) and leaves us with a valid list of present and active devices.
8660 *
8661 */
8662
8663/*
8664 * This is called single threaded during boot, so no need
8665 * to take the rtnl semaphore.
8666 */
8667static int __init net_dev_init(void)
8668{
8669 int i, rc = -ENOMEM;
8670
8671 BUG_ON(!dev_boot_phase);
8672
Linus Torvalds1da177e2005-04-16 15:20:36 -07008673 if (dev_proc_init())
8674 goto out;
8675
Eric W. Biederman8b41d182007-09-26 22:02:53 -07008676 if (netdev_kobject_init())
Linus Torvalds1da177e2005-04-16 15:20:36 -07008677 goto out;
8678
8679 INIT_LIST_HEAD(&ptype_all);
Pavel Emelyanov82d8a8672007-11-26 20:12:58 +08008680 for (i = 0; i < PTYPE_HASH_SIZE; i++)
Linus Torvalds1da177e2005-04-16 15:20:36 -07008681 INIT_LIST_HEAD(&ptype_base[i]);
8682
Vlad Yasevich62532da2012-11-15 08:49:10 +00008683 INIT_LIST_HEAD(&offload_base);
8684
Eric W. Biederman881d9662007-09-17 11:56:21 -07008685 if (register_pernet_subsys(&netdev_net_ops))
8686 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07008687
8688 /*
8689 * Initialise the packet receive queues.
8690 */
8691
KAMEZAWA Hiroyuki6f912042006-04-10 22:52:50 -07008692 for_each_possible_cpu(i) {
Eric Dumazet41852492016-08-26 12:50:39 -07008693 struct work_struct *flush = per_cpu_ptr(&flush_works, i);
Eric Dumazete36fa2f2010-04-19 21:17:14 +00008694 struct softnet_data *sd = &per_cpu(softnet_data, i);
Linus Torvalds1da177e2005-04-16 15:20:36 -07008695
Eric Dumazet41852492016-08-26 12:50:39 -07008696 INIT_WORK(flush, flush_backlog);
8697
Eric Dumazete36fa2f2010-04-19 21:17:14 +00008698 skb_queue_head_init(&sd->input_pkt_queue);
Changli Gao6e7676c2010-04-27 15:07:33 -07008699 skb_queue_head_init(&sd->process_queue);
Eric Dumazete36fa2f2010-04-19 21:17:14 +00008700 INIT_LIST_HEAD(&sd->poll_list);
Changli Gaoa9cbd582010-04-26 23:06:24 +00008701 sd->output_queue_tailp = &sd->output_queue;
Eric Dumazetdf334542010-03-24 19:13:54 +00008702#ifdef CONFIG_RPS
Eric Dumazete36fa2f2010-04-19 21:17:14 +00008703 sd->csd.func = rps_trigger_softirq;
8704 sd->csd.info = sd;
Eric Dumazete36fa2f2010-04-19 21:17:14 +00008705 sd->cpu = i;
Tom Herbert1e94d722010-03-18 17:45:44 -07008706#endif
Tom Herbert0a9627f2010-03-16 08:03:29 +00008707
Eric Dumazete36fa2f2010-04-19 21:17:14 +00008708 sd->backlog.poll = process_backlog;
8709 sd->backlog.weight = weight_p;
Linus Torvalds1da177e2005-04-16 15:20:36 -07008710 }
8711
Linus Torvalds1da177e2005-04-16 15:20:36 -07008712 dev_boot_phase = 0;
8713
Eric W. Biederman505d4f72008-11-07 22:54:20 -08008714 /* The loopback device is special if any other network devices
8715 * is present in a network namespace the loopback device must
8716 * be present. Since we now dynamically allocate and free the
8717 * loopback device ensure this invariant is maintained by
8718 * keeping the loopback device as the first device on the
8719 * list of network devices. Ensuring the loopback devices
8720 * is the first device that appears and the last network device
8721 * that disappears.
8722 */
8723 if (register_pernet_device(&loopback_net_ops))
8724 goto out;
8725
8726 if (register_pernet_device(&default_device_ops))
8727 goto out;
8728
Carlos R. Mafra962cf362008-05-15 11:15:37 -03008729 open_softirq(NET_TX_SOFTIRQ, net_tx_action);
8730 open_softirq(NET_RX_SOFTIRQ, net_rx_action);
Linus Torvalds1da177e2005-04-16 15:20:36 -07008731
Sebastian Andrzej Siewiorf0bf90d2016-11-03 15:50:04 +01008732 rc = cpuhp_setup_state_nocalls(CPUHP_NET_DEV_DEAD, "net/dev:dead",
8733 NULL, dev_cpu_dead);
8734 WARN_ON(rc < 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07008735 rc = 0;
8736out:
8737 return rc;
8738}
8739
8740subsys_initcall(net_dev_init);