blob: 5c925ac50b9572755e35be168cde8c858fd4ba95 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * NET3 Protocol independent device support routines.
3 *
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public License
6 * as published by the Free Software Foundation; either version
7 * 2 of the License, or (at your option) any later version.
8 *
9 * Derived from the non IP parts of dev.c 1.0.19
Jesper Juhl02c30a82005-05-05 16:16:16 -070010 * Authors: Ross Biro
Linus Torvalds1da177e2005-04-16 15:20:36 -070011 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
12 * Mark Evans, <evansmp@uhura.aston.ac.uk>
13 *
14 * Additional Authors:
15 * Florian la Roche <rzsfl@rz.uni-sb.de>
16 * Alan Cox <gw4pts@gw4pts.ampr.org>
17 * David Hinds <dahinds@users.sourceforge.net>
18 * Alexey Kuznetsov <kuznet@ms2.inr.ac.ru>
19 * Adam Sulmicki <adam@cfar.umd.edu>
20 * Pekka Riikonen <priikone@poesidon.pspt.fi>
21 *
22 * Changes:
23 * D.J. Barrow : Fixed bug where dev->refcnt gets set
24 * to 2 if register_netdev gets called
25 * before net_dev_init & also removed a
26 * few lines of code in the process.
27 * Alan Cox : device private ioctl copies fields back.
28 * Alan Cox : Transmit queue code does relevant
29 * stunts to keep the queue safe.
30 * Alan Cox : Fixed double lock.
31 * Alan Cox : Fixed promisc NULL pointer trap
32 * ???????? : Support the full private ioctl range
33 * Alan Cox : Moved ioctl permission check into
34 * drivers
35 * Tim Kordas : SIOCADDMULTI/SIOCDELMULTI
36 * Alan Cox : 100 backlog just doesn't cut it when
37 * you start doing multicast video 8)
38 * Alan Cox : Rewrote net_bh and list manager.
39 * Alan Cox : Fix ETH_P_ALL echoback lengths.
40 * Alan Cox : Took out transmit every packet pass
41 * Saved a few bytes in the ioctl handler
42 * Alan Cox : Network driver sets packet type before
43 * calling netif_rx. Saves a function
44 * call a packet.
45 * Alan Cox : Hashed net_bh()
46 * Richard Kooijman: Timestamp fixes.
47 * Alan Cox : Wrong field in SIOCGIFDSTADDR
48 * Alan Cox : Device lock protection.
49 * Alan Cox : Fixed nasty side effect of device close
50 * changes.
51 * Rudi Cilibrasi : Pass the right thing to
52 * set_mac_address()
53 * Dave Miller : 32bit quantity for the device lock to
54 * make it work out on a Sparc.
55 * Bjorn Ekwall : Added KERNELD hack.
56 * Alan Cox : Cleaned up the backlog initialise.
57 * Craig Metz : SIOCGIFCONF fix if space for under
58 * 1 device.
59 * Thomas Bogendoerfer : Return ENODEV for dev_open, if there
60 * is no device open function.
61 * Andi Kleen : Fix error reporting for SIOCGIFCONF
62 * Michael Chastain : Fix signed/unsigned for SIOCGIFCONF
63 * Cyrus Durgin : Cleaned for KMOD
64 * Adam Sulmicki : Bug Fix : Network Device Unload
65 * A network device unload needs to purge
66 * the backlog queue.
67 * Paul Rusty Russell : SIOCSIFNAME
68 * Pekka Riikonen : Netdev boot-time settings code
69 * Andrew Morton : Make unregister_netdevice wait
70 * indefinitely on dev->refcnt
71 * J Hadi Salim : - Backlog queue sampling
72 * - netif_rx() feedback
73 */
74
75#include <asm/uaccess.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070076#include <linux/bitops.h>
Randy Dunlap4fc268d2006-01-11 12:17:47 -080077#include <linux/capability.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070078#include <linux/cpu.h>
79#include <linux/types.h>
80#include <linux/kernel.h>
stephen hemminger08e98972009-11-10 07:20:34 +000081#include <linux/hash.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090082#include <linux/slab.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070083#include <linux/sched.h>
Arjan van de Ven4a3e2f72006-03-20 22:33:17 -080084#include <linux/mutex.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070085#include <linux/string.h>
86#include <linux/mm.h>
87#include <linux/socket.h>
88#include <linux/sockios.h>
89#include <linux/errno.h>
90#include <linux/interrupt.h>
91#include <linux/if_ether.h>
92#include <linux/netdevice.h>
93#include <linux/etherdevice.h>
Ben Hutchings0187bdf2008-06-19 16:15:47 -070094#include <linux/ethtool.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070095#include <linux/notifier.h>
96#include <linux/skbuff.h>
Eric W. Biederman457c4cb2007-09-12 12:01:34 +020097#include <net/net_namespace.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070098#include <net/sock.h>
Eric Dumazet02d62e82015-11-18 06:30:52 -080099#include <net/busy_poll.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700100#include <linux/rtnetlink.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700101#include <linux/stat.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700102#include <net/dst.h>
Pravin B Shelarfc4099f2015-10-22 18:17:16 -0700103#include <net/dst_metadata.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700104#include <net/pkt_sched.h>
105#include <net/checksum.h>
Arnd Bergmann44540962009-11-26 06:07:08 +0000106#include <net/xfrm.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700107#include <linux/highmem.h>
108#include <linux/init.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700109#include <linux/module.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700110#include <linux/netpoll.h>
111#include <linux/rcupdate.h>
112#include <linux/delay.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700113#include <net/iw_handler.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700114#include <asm/current.h>
Steve Grubb5bdb9882005-12-03 08:39:35 -0500115#include <linux/audit.h>
Chris Leechdb217332006-06-17 21:24:58 -0700116#include <linux/dmaengine.h>
Herbert Xuf6a78bf2006-06-22 02:57:17 -0700117#include <linux/err.h>
David S. Millerc7fa9d12006-08-15 16:34:13 -0700118#include <linux/ctype.h>
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700119#include <linux/if_arp.h>
Ben Hutchings6de329e2008-06-16 17:02:28 -0700120#include <linux/if_vlan.h>
David S. Miller8f0f2222008-07-15 03:47:03 -0700121#include <linux/ip.h>
Alexander Duyckad55dca2008-09-20 22:05:50 -0700122#include <net/ip.h>
Simon Horman25cd9ba2014-10-06 05:05:13 -0700123#include <net/mpls.h>
David S. Miller8f0f2222008-07-15 03:47:03 -0700124#include <linux/ipv6.h>
125#include <linux/in.h>
David S. Millerb6b2fed2008-07-21 09:48:06 -0700126#include <linux/jhash.h>
127#include <linux/random.h>
David S. Miller9cbc1cb2009-06-15 03:02:23 -0700128#include <trace/events/napi.h>
Koki Sanagicf66ba52010-08-23 18:45:02 +0900129#include <trace/events/net.h>
Koki Sanagi07dc22e2010-08-23 18:46:12 +0900130#include <trace/events/skb.h>
FUJITA Tomonori5acbbd42010-03-30 22:35:50 +0000131#include <linux/pci.h>
Stephen Rothwellcaeda9b2010-09-16 21:39:16 -0700132#include <linux/inetdevice.h>
Ben Hutchingsc4454772011-01-19 11:03:53 +0000133#include <linux/cpu_rmap.h>
Ingo Molnarc5905af2012-02-24 08:31:31 +0100134#include <linux/static_key.h>
Eliezer Tamiraf12fa62013-06-10 11:39:41 +0300135#include <linux/hashtable.h>
Eric Dumazet60877a32013-06-20 01:15:51 -0700136#include <linux/vmalloc.h>
Michal Kubeček529d0482013-11-15 06:18:50 +0100137#include <linux/if_macvlan.h>
Willem de Bruijne7fd2882014-08-04 22:11:48 -0400138#include <linux/errqueue.h>
Eric Dumazet3b47d302014-11-06 21:09:44 -0800139#include <linux/hrtimer.h>
Pablo Neirae687ad62015-05-13 18:19:38 +0200140#include <linux/netfilter_ingress.h>
Tom Herbert6ae23ad2015-12-14 11:19:46 -0800141#include <linux/sctp.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700142
Pavel Emelyanov342709e2007-10-23 21:14:45 -0700143#include "net-sysfs.h"
144
Herbert Xud565b0a2008-12-15 23:38:52 -0800145/* Instead of increasing this, you should create a hash table. */
146#define MAX_GRO_SKBS 8
147
Herbert Xu5d38a072009-01-04 16:13:40 -0800148/* This should be increased if a protocol with a bigger head is added. */
149#define GRO_MAX_HEAD (MAX_HEADER + 128)
150
Linus Torvalds1da177e2005-04-16 15:20:36 -0700151static DEFINE_SPINLOCK(ptype_lock);
Vlad Yasevich62532da2012-11-15 08:49:10 +0000152static DEFINE_SPINLOCK(offload_lock);
Cong Wang900ff8c2013-02-18 19:20:33 +0000153struct list_head ptype_base[PTYPE_HASH_SIZE] __read_mostly;
154struct list_head ptype_all __read_mostly; /* Taps */
Vlad Yasevich62532da2012-11-15 08:49:10 +0000155static struct list_head offload_base __read_mostly;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700156
Ben Hutchingsae78dbf2014-01-10 22:17:24 +0000157static int netif_rx_internal(struct sk_buff *skb);
Loic Prylli5495119462014-07-01 21:39:43 -0700158static int call_netdevice_notifiers_info(unsigned long val,
159 struct net_device *dev,
160 struct netdev_notifier_info *info);
Ben Hutchingsae78dbf2014-01-10 22:17:24 +0000161
Linus Torvalds1da177e2005-04-16 15:20:36 -0700162/*
Pavel Emelianov7562f872007-05-03 15:13:45 -0700163 * The @dev_base_head list is protected by @dev_base_lock and the rtnl
Linus Torvalds1da177e2005-04-16 15:20:36 -0700164 * semaphore.
165 *
Eric Dumazetc6d14c82009-11-04 05:43:23 -0800166 * Pure readers hold dev_base_lock for reading, or rcu_read_lock()
Linus Torvalds1da177e2005-04-16 15:20:36 -0700167 *
168 * Writers must hold the rtnl semaphore while they loop through the
Pavel Emelianov7562f872007-05-03 15:13:45 -0700169 * dev_base_head list, and hold dev_base_lock for writing when they do the
Linus Torvalds1da177e2005-04-16 15:20:36 -0700170 * actual updates. This allows pure readers to access the list even
171 * while a writer is preparing to update it.
172 *
173 * To put it another way, dev_base_lock is held for writing only to
174 * protect against pure readers; the rtnl semaphore provides the
175 * protection against other writers.
176 *
177 * See, for example usages, register_netdevice() and
178 * unregister_netdevice(), which must be called with the rtnl
179 * semaphore held.
180 */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700181DEFINE_RWLOCK(dev_base_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700182EXPORT_SYMBOL(dev_base_lock);
183
Eliezer Tamiraf12fa62013-06-10 11:39:41 +0300184/* protects napi_hash addition/deletion and napi_gen_id */
185static DEFINE_SPINLOCK(napi_hash_lock);
186
Eric Dumazet52bd2d62015-11-18 06:30:50 -0800187static unsigned int napi_gen_id = NR_CPUS;
Eric Dumazet6180d9d2015-11-18 06:31:01 -0800188static DEFINE_READ_MOSTLY_HASHTABLE(napi_hash, 8);
Eliezer Tamiraf12fa62013-06-10 11:39:41 +0300189
Thomas Gleixner18afa4b2013-07-23 16:13:17 +0200190static seqcount_t devnet_rename_seq;
Brian Haleyc91f6df2012-11-26 05:21:08 +0000191
Thomas Graf4e985ad2011-06-21 03:11:20 +0000192static inline void dev_base_seq_inc(struct net *net)
193{
194 while (++net->dev_base_seq == 0);
195}
196
Eric W. Biederman881d9662007-09-17 11:56:21 -0700197static inline struct hlist_head *dev_name_hash(struct net *net, const char *name)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700198{
Eric Dumazet95c96172012-04-15 05:58:06 +0000199 unsigned int hash = full_name_hash(name, strnlen(name, IFNAMSIZ));
200
stephen hemminger08e98972009-11-10 07:20:34 +0000201 return &net->dev_name_head[hash_32(hash, NETDEV_HASHBITS)];
Linus Torvalds1da177e2005-04-16 15:20:36 -0700202}
203
Eric W. Biederman881d9662007-09-17 11:56:21 -0700204static inline struct hlist_head *dev_index_hash(struct net *net, int ifindex)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700205{
Eric Dumazet7c28bd02009-10-24 06:13:17 -0700206 return &net->dev_index_head[ifindex & (NETDEV_HASHENTRIES - 1)];
Linus Torvalds1da177e2005-04-16 15:20:36 -0700207}
208
Eric Dumazete36fa2f2010-04-19 21:17:14 +0000209static inline void rps_lock(struct softnet_data *sd)
Changli Gao152102c2010-03-30 20:16:22 +0000210{
211#ifdef CONFIG_RPS
Eric Dumazete36fa2f2010-04-19 21:17:14 +0000212 spin_lock(&sd->input_pkt_queue.lock);
Changli Gao152102c2010-03-30 20:16:22 +0000213#endif
214}
215
Eric Dumazete36fa2f2010-04-19 21:17:14 +0000216static inline void rps_unlock(struct softnet_data *sd)
Changli Gao152102c2010-03-30 20:16:22 +0000217{
218#ifdef CONFIG_RPS
Eric Dumazete36fa2f2010-04-19 21:17:14 +0000219 spin_unlock(&sd->input_pkt_queue.lock);
Changli Gao152102c2010-03-30 20:16:22 +0000220#endif
221}
222
Eric W. Biedermance286d32007-09-12 13:53:49 +0200223/* Device list insertion */
dingtianhong53759be2013-04-17 22:17:50 +0000224static void list_netdevice(struct net_device *dev)
Eric W. Biedermance286d32007-09-12 13:53:49 +0200225{
YOSHIFUJI Hideakic346dca2008-03-25 21:47:49 +0900226 struct net *net = dev_net(dev);
Eric W. Biedermance286d32007-09-12 13:53:49 +0200227
228 ASSERT_RTNL();
229
230 write_lock_bh(&dev_base_lock);
Eric Dumazetc6d14c82009-11-04 05:43:23 -0800231 list_add_tail_rcu(&dev->dev_list, &net->dev_base_head);
Eric Dumazet72c95282009-10-30 07:11:27 +0000232 hlist_add_head_rcu(&dev->name_hlist, dev_name_hash(net, dev->name));
Eric Dumazetfb699dfd2009-10-19 19:18:49 +0000233 hlist_add_head_rcu(&dev->index_hlist,
234 dev_index_hash(net, dev->ifindex));
Eric W. Biedermance286d32007-09-12 13:53:49 +0200235 write_unlock_bh(&dev_base_lock);
Thomas Graf4e985ad2011-06-21 03:11:20 +0000236
237 dev_base_seq_inc(net);
Eric W. Biedermance286d32007-09-12 13:53:49 +0200238}
239
Eric Dumazetfb699dfd2009-10-19 19:18:49 +0000240/* Device list removal
241 * caller must respect a RCU grace period before freeing/reusing dev
242 */
Eric W. Biedermance286d32007-09-12 13:53:49 +0200243static void unlist_netdevice(struct net_device *dev)
244{
245 ASSERT_RTNL();
246
247 /* Unlink dev from the device chain */
248 write_lock_bh(&dev_base_lock);
Eric Dumazetc6d14c82009-11-04 05:43:23 -0800249 list_del_rcu(&dev->dev_list);
Eric Dumazet72c95282009-10-30 07:11:27 +0000250 hlist_del_rcu(&dev->name_hlist);
Eric Dumazetfb699dfd2009-10-19 19:18:49 +0000251 hlist_del_rcu(&dev->index_hlist);
Eric W. Biedermance286d32007-09-12 13:53:49 +0200252 write_unlock_bh(&dev_base_lock);
Thomas Graf4e985ad2011-06-21 03:11:20 +0000253
254 dev_base_seq_inc(dev_net(dev));
Eric W. Biedermance286d32007-09-12 13:53:49 +0200255}
256
Linus Torvalds1da177e2005-04-16 15:20:36 -0700257/*
258 * Our notifier list
259 */
260
Alan Sternf07d5b92006-05-09 15:23:03 -0700261static RAW_NOTIFIER_HEAD(netdev_chain);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700262
263/*
264 * Device drivers call our routines to queue packets here. We empty the
265 * queue in the local softnet handler.
266 */
Stephen Hemmingerbea33482007-10-03 16:41:36 -0700267
Eric Dumazet9958da02010-04-17 04:17:02 +0000268DEFINE_PER_CPU_ALIGNED(struct softnet_data, softnet_data);
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700269EXPORT_PER_CPU_SYMBOL(softnet_data);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700270
David S. Millercf508b12008-07-22 14:16:42 -0700271#ifdef CONFIG_LOCKDEP
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700272/*
David S. Millerc773e842008-07-08 23:13:53 -0700273 * register_netdevice() inits txq->_xmit_lock and sets lockdep class
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700274 * according to dev->type
275 */
276static const unsigned short netdev_lock_type[] =
277 {ARPHRD_NETROM, ARPHRD_ETHER, ARPHRD_EETHER, ARPHRD_AX25,
278 ARPHRD_PRONET, ARPHRD_CHAOS, ARPHRD_IEEE802, ARPHRD_ARCNET,
279 ARPHRD_APPLETLK, ARPHRD_DLCI, ARPHRD_ATM, ARPHRD_METRICOM,
280 ARPHRD_IEEE1394, ARPHRD_EUI64, ARPHRD_INFINIBAND, ARPHRD_SLIP,
281 ARPHRD_CSLIP, ARPHRD_SLIP6, ARPHRD_CSLIP6, ARPHRD_RSRVD,
282 ARPHRD_ADAPT, ARPHRD_ROSE, ARPHRD_X25, ARPHRD_HWX25,
283 ARPHRD_PPP, ARPHRD_CISCO, ARPHRD_LAPB, ARPHRD_DDCMP,
284 ARPHRD_RAWHDLC, ARPHRD_TUNNEL, ARPHRD_TUNNEL6, ARPHRD_FRAD,
285 ARPHRD_SKIP, ARPHRD_LOOPBACK, ARPHRD_LOCALTLK, ARPHRD_FDDI,
286 ARPHRD_BIF, ARPHRD_SIT, ARPHRD_IPDDP, ARPHRD_IPGRE,
287 ARPHRD_PIMREG, ARPHRD_HIPPI, ARPHRD_ASH, ARPHRD_ECONET,
288 ARPHRD_IRDA, ARPHRD_FCPP, ARPHRD_FCAL, ARPHRD_FCPL,
Paul Gortmaker211ed862012-05-10 17:14:35 -0400289 ARPHRD_FCFABRIC, ARPHRD_IEEE80211, ARPHRD_IEEE80211_PRISM,
290 ARPHRD_IEEE80211_RADIOTAP, ARPHRD_PHONET, ARPHRD_PHONET_PIPE,
291 ARPHRD_IEEE802154, ARPHRD_VOID, ARPHRD_NONE};
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700292
Jan Engelhardt36cbd3d2009-08-05 10:42:58 -0700293static const char *const netdev_lock_name[] =
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700294 {"_xmit_NETROM", "_xmit_ETHER", "_xmit_EETHER", "_xmit_AX25",
295 "_xmit_PRONET", "_xmit_CHAOS", "_xmit_IEEE802", "_xmit_ARCNET",
296 "_xmit_APPLETLK", "_xmit_DLCI", "_xmit_ATM", "_xmit_METRICOM",
297 "_xmit_IEEE1394", "_xmit_EUI64", "_xmit_INFINIBAND", "_xmit_SLIP",
298 "_xmit_CSLIP", "_xmit_SLIP6", "_xmit_CSLIP6", "_xmit_RSRVD",
299 "_xmit_ADAPT", "_xmit_ROSE", "_xmit_X25", "_xmit_HWX25",
300 "_xmit_PPP", "_xmit_CISCO", "_xmit_LAPB", "_xmit_DDCMP",
301 "_xmit_RAWHDLC", "_xmit_TUNNEL", "_xmit_TUNNEL6", "_xmit_FRAD",
302 "_xmit_SKIP", "_xmit_LOOPBACK", "_xmit_LOCALTLK", "_xmit_FDDI",
303 "_xmit_BIF", "_xmit_SIT", "_xmit_IPDDP", "_xmit_IPGRE",
304 "_xmit_PIMREG", "_xmit_HIPPI", "_xmit_ASH", "_xmit_ECONET",
305 "_xmit_IRDA", "_xmit_FCPP", "_xmit_FCAL", "_xmit_FCPL",
Paul Gortmaker211ed862012-05-10 17:14:35 -0400306 "_xmit_FCFABRIC", "_xmit_IEEE80211", "_xmit_IEEE80211_PRISM",
307 "_xmit_IEEE80211_RADIOTAP", "_xmit_PHONET", "_xmit_PHONET_PIPE",
308 "_xmit_IEEE802154", "_xmit_VOID", "_xmit_NONE"};
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700309
310static struct lock_class_key netdev_xmit_lock_key[ARRAY_SIZE(netdev_lock_type)];
David S. Millercf508b12008-07-22 14:16:42 -0700311static struct lock_class_key netdev_addr_lock_key[ARRAY_SIZE(netdev_lock_type)];
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700312
313static inline unsigned short netdev_lock_pos(unsigned short dev_type)
314{
315 int i;
316
317 for (i = 0; i < ARRAY_SIZE(netdev_lock_type); i++)
318 if (netdev_lock_type[i] == dev_type)
319 return i;
320 /* the last key is used by default */
321 return ARRAY_SIZE(netdev_lock_type) - 1;
322}
323
David S. Millercf508b12008-07-22 14:16:42 -0700324static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock,
325 unsigned short dev_type)
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700326{
327 int i;
328
329 i = netdev_lock_pos(dev_type);
330 lockdep_set_class_and_name(lock, &netdev_xmit_lock_key[i],
331 netdev_lock_name[i]);
332}
David S. Millercf508b12008-07-22 14:16:42 -0700333
334static inline void netdev_set_addr_lockdep_class(struct net_device *dev)
335{
336 int i;
337
338 i = netdev_lock_pos(dev->type);
339 lockdep_set_class_and_name(&dev->addr_list_lock,
340 &netdev_addr_lock_key[i],
341 netdev_lock_name[i]);
342}
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700343#else
David S. Millercf508b12008-07-22 14:16:42 -0700344static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock,
345 unsigned short dev_type)
346{
347}
348static inline void netdev_set_addr_lockdep_class(struct net_device *dev)
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700349{
350}
351#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -0700352
353/*******************************************************************************
354
355 Protocol management and registration routines
356
357*******************************************************************************/
358
359/*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700360 * Add a protocol ID to the list. Now that the input handler is
361 * smarter we can dispense with all the messy stuff that used to be
362 * here.
363 *
364 * BEWARE!!! Protocol handlers, mangling input packets,
365 * MUST BE last in hash buckets and checking protocol handlers
366 * MUST start from promiscuous ptype_all chain in net_bh.
367 * It is true now, do not change it.
368 * Explanation follows: if protocol handler, mangling packet, will
369 * be the first on list, it is not able to sense, that packet
370 * is cloned and should be copied-on-write, so that it will
371 * change it and subsequent readers will get broken packet.
372 * --ANK (980803)
373 */
374
Eric Dumazetc07b68e2010-09-02 03:53:46 +0000375static inline struct list_head *ptype_head(const struct packet_type *pt)
376{
377 if (pt->type == htons(ETH_P_ALL))
Salam Noureddine7866a622015-01-27 11:35:48 -0800378 return pt->dev ? &pt->dev->ptype_all : &ptype_all;
Eric Dumazetc07b68e2010-09-02 03:53:46 +0000379 else
Salam Noureddine7866a622015-01-27 11:35:48 -0800380 return pt->dev ? &pt->dev->ptype_specific :
381 &ptype_base[ntohs(pt->type) & PTYPE_HASH_MASK];
Eric Dumazetc07b68e2010-09-02 03:53:46 +0000382}
383
Linus Torvalds1da177e2005-04-16 15:20:36 -0700384/**
385 * dev_add_pack - add packet handler
386 * @pt: packet type declaration
387 *
388 * Add a protocol handler to the networking stack. The passed &packet_type
389 * is linked into kernel lists and may not be freed until it has been
390 * removed from the kernel lists.
391 *
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +0900392 * This call does not sleep therefore it can not
Linus Torvalds1da177e2005-04-16 15:20:36 -0700393 * guarantee all CPU's that are in middle of receiving packets
394 * will see the new packet type (until the next received packet).
395 */
396
397void dev_add_pack(struct packet_type *pt)
398{
Eric Dumazetc07b68e2010-09-02 03:53:46 +0000399 struct list_head *head = ptype_head(pt);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700400
Eric Dumazetc07b68e2010-09-02 03:53:46 +0000401 spin_lock(&ptype_lock);
402 list_add_rcu(&pt->list, head);
403 spin_unlock(&ptype_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700404}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700405EXPORT_SYMBOL(dev_add_pack);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700406
Linus Torvalds1da177e2005-04-16 15:20:36 -0700407/**
408 * __dev_remove_pack - remove packet handler
409 * @pt: packet type declaration
410 *
411 * Remove a protocol handler that was previously added to the kernel
412 * protocol handlers by dev_add_pack(). The passed &packet_type is removed
413 * from the kernel lists and can be freed or reused once this function
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +0900414 * returns.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700415 *
416 * The packet type might still be in use by receivers
417 * and must not be freed until after all the CPU's have gone
418 * through a quiescent state.
419 */
420void __dev_remove_pack(struct packet_type *pt)
421{
Eric Dumazetc07b68e2010-09-02 03:53:46 +0000422 struct list_head *head = ptype_head(pt);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700423 struct packet_type *pt1;
424
Eric Dumazetc07b68e2010-09-02 03:53:46 +0000425 spin_lock(&ptype_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700426
427 list_for_each_entry(pt1, head, list) {
428 if (pt == pt1) {
429 list_del_rcu(&pt->list);
430 goto out;
431 }
432 }
433
Joe Perches7b6cd1c2012-02-01 10:54:43 +0000434 pr_warn("dev_remove_pack: %p not found\n", pt);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700435out:
Eric Dumazetc07b68e2010-09-02 03:53:46 +0000436 spin_unlock(&ptype_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700437}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700438EXPORT_SYMBOL(__dev_remove_pack);
439
Linus Torvalds1da177e2005-04-16 15:20:36 -0700440/**
441 * dev_remove_pack - remove packet handler
442 * @pt: packet type declaration
443 *
444 * Remove a protocol handler that was previously added to the kernel
445 * protocol handlers by dev_add_pack(). The passed &packet_type is removed
446 * from the kernel lists and can be freed or reused once this function
447 * returns.
448 *
449 * This call sleeps to guarantee that no CPU is looking at the packet
450 * type after return.
451 */
452void dev_remove_pack(struct packet_type *pt)
453{
454 __dev_remove_pack(pt);
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +0900455
Linus Torvalds1da177e2005-04-16 15:20:36 -0700456 synchronize_net();
457}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700458EXPORT_SYMBOL(dev_remove_pack);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700459
Vlad Yasevich62532da2012-11-15 08:49:10 +0000460
461/**
462 * dev_add_offload - register offload handlers
463 * @po: protocol offload declaration
464 *
465 * Add protocol offload handlers to the networking stack. The passed
466 * &proto_offload is linked into kernel lists and may not be freed until
467 * it has been removed from the kernel lists.
468 *
469 * This call does not sleep therefore it can not
470 * guarantee all CPU's that are in middle of receiving packets
471 * will see the new offload handlers (until the next received packet).
472 */
473void dev_add_offload(struct packet_offload *po)
474{
David S. Millerbdef7de2015-06-01 14:56:09 -0700475 struct packet_offload *elem;
Vlad Yasevich62532da2012-11-15 08:49:10 +0000476
477 spin_lock(&offload_lock);
David S. Millerbdef7de2015-06-01 14:56:09 -0700478 list_for_each_entry(elem, &offload_base, list) {
479 if (po->priority < elem->priority)
480 break;
481 }
482 list_add_rcu(&po->list, elem->list.prev);
Vlad Yasevich62532da2012-11-15 08:49:10 +0000483 spin_unlock(&offload_lock);
484}
485EXPORT_SYMBOL(dev_add_offload);
486
487/**
488 * __dev_remove_offload - remove offload handler
489 * @po: packet offload declaration
490 *
491 * Remove a protocol offload handler that was previously added to the
492 * kernel offload handlers by dev_add_offload(). The passed &offload_type
493 * is removed from the kernel lists and can be freed or reused once this
494 * function returns.
495 *
496 * The packet type might still be in use by receivers
497 * and must not be freed until after all the CPU's have gone
498 * through a quiescent state.
499 */
stephen hemminger1d143d92013-12-29 14:01:29 -0800500static void __dev_remove_offload(struct packet_offload *po)
Vlad Yasevich62532da2012-11-15 08:49:10 +0000501{
502 struct list_head *head = &offload_base;
503 struct packet_offload *po1;
504
Eric Dumazetc53aa502012-11-16 08:08:23 +0000505 spin_lock(&offload_lock);
Vlad Yasevich62532da2012-11-15 08:49:10 +0000506
507 list_for_each_entry(po1, head, list) {
508 if (po == po1) {
509 list_del_rcu(&po->list);
510 goto out;
511 }
512 }
513
514 pr_warn("dev_remove_offload: %p not found\n", po);
515out:
Eric Dumazetc53aa502012-11-16 08:08:23 +0000516 spin_unlock(&offload_lock);
Vlad Yasevich62532da2012-11-15 08:49:10 +0000517}
Vlad Yasevich62532da2012-11-15 08:49:10 +0000518
519/**
520 * dev_remove_offload - remove packet offload handler
521 * @po: packet offload declaration
522 *
523 * Remove a packet offload handler that was previously added to the kernel
524 * offload handlers by dev_add_offload(). The passed &offload_type is
525 * removed from the kernel lists and can be freed or reused once this
526 * function returns.
527 *
528 * This call sleeps to guarantee that no CPU is looking at the packet
529 * type after return.
530 */
531void dev_remove_offload(struct packet_offload *po)
532{
533 __dev_remove_offload(po);
534
535 synchronize_net();
536}
537EXPORT_SYMBOL(dev_remove_offload);
538
Linus Torvalds1da177e2005-04-16 15:20:36 -0700539/******************************************************************************
540
541 Device Boot-time Settings Routines
542
543*******************************************************************************/
544
545/* Boot time configuration table */
546static struct netdev_boot_setup dev_boot_setup[NETDEV_BOOT_SETUP_MAX];
547
548/**
549 * netdev_boot_setup_add - add new setup entry
550 * @name: name of the device
551 * @map: configured settings for the device
552 *
553 * Adds new setup entry to the dev_boot_setup list. The function
554 * returns 0 on error and 1 on success. This is a generic routine to
555 * all netdevices.
556 */
557static int netdev_boot_setup_add(char *name, struct ifmap *map)
558{
559 struct netdev_boot_setup *s;
560 int i;
561
562 s = dev_boot_setup;
563 for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++) {
564 if (s[i].name[0] == '\0' || s[i].name[0] == ' ') {
565 memset(s[i].name, 0, sizeof(s[i].name));
Wang Chen93b3cff2008-07-01 19:57:19 -0700566 strlcpy(s[i].name, name, IFNAMSIZ);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700567 memcpy(&s[i].map, map, sizeof(s[i].map));
568 break;
569 }
570 }
571
572 return i >= NETDEV_BOOT_SETUP_MAX ? 0 : 1;
573}
574
575/**
576 * netdev_boot_setup_check - check boot time settings
577 * @dev: the netdevice
578 *
579 * Check boot time settings for the device.
580 * The found settings are set for the device to be used
581 * later in the device probing.
582 * Returns 0 if no settings found, 1 if they are.
583 */
584int netdev_boot_setup_check(struct net_device *dev)
585{
586 struct netdev_boot_setup *s = dev_boot_setup;
587 int i;
588
589 for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++) {
590 if (s[i].name[0] != '\0' && s[i].name[0] != ' ' &&
Wang Chen93b3cff2008-07-01 19:57:19 -0700591 !strcmp(dev->name, s[i].name)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700592 dev->irq = s[i].map.irq;
593 dev->base_addr = s[i].map.base_addr;
594 dev->mem_start = s[i].map.mem_start;
595 dev->mem_end = s[i].map.mem_end;
596 return 1;
597 }
598 }
599 return 0;
600}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700601EXPORT_SYMBOL(netdev_boot_setup_check);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700602
603
604/**
605 * netdev_boot_base - get address from boot time settings
606 * @prefix: prefix for network device
607 * @unit: id for network device
608 *
609 * Check boot time settings for the base address of device.
610 * The found settings are set for the device to be used
611 * later in the device probing.
612 * Returns 0 if no settings found.
613 */
614unsigned long netdev_boot_base(const char *prefix, int unit)
615{
616 const struct netdev_boot_setup *s = dev_boot_setup;
617 char name[IFNAMSIZ];
618 int i;
619
620 sprintf(name, "%s%d", prefix, unit);
621
622 /*
623 * If device already registered then return base of 1
624 * to indicate not to probe for this interface
625 */
Eric W. Biederman881d9662007-09-17 11:56:21 -0700626 if (__dev_get_by_name(&init_net, name))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700627 return 1;
628
629 for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++)
630 if (!strcmp(name, s[i].name))
631 return s[i].map.base_addr;
632 return 0;
633}
634
635/*
636 * Saves at boot time configured settings for any netdevice.
637 */
638int __init netdev_boot_setup(char *str)
639{
640 int ints[5];
641 struct ifmap map;
642
643 str = get_options(str, ARRAY_SIZE(ints), ints);
644 if (!str || !*str)
645 return 0;
646
647 /* Save settings */
648 memset(&map, 0, sizeof(map));
649 if (ints[0] > 0)
650 map.irq = ints[1];
651 if (ints[0] > 1)
652 map.base_addr = ints[2];
653 if (ints[0] > 2)
654 map.mem_start = ints[3];
655 if (ints[0] > 3)
656 map.mem_end = ints[4];
657
658 /* Add new entry to the list */
659 return netdev_boot_setup_add(str, &map);
660}
661
662__setup("netdev=", netdev_boot_setup);
663
664/*******************************************************************************
665
666 Device Interface Subroutines
667
668*******************************************************************************/
669
670/**
Nicolas Dichtela54acb32015-04-02 17:07:00 +0200671 * dev_get_iflink - get 'iflink' value of a interface
672 * @dev: targeted interface
673 *
674 * Indicates the ifindex the interface is linked to.
675 * Physical interfaces have the same 'ifindex' and 'iflink' values.
676 */
677
678int dev_get_iflink(const struct net_device *dev)
679{
680 if (dev->netdev_ops && dev->netdev_ops->ndo_get_iflink)
681 return dev->netdev_ops->ndo_get_iflink(dev);
682
Nicolas Dichtel7a66bbc2015-04-02 17:07:09 +0200683 return dev->ifindex;
Nicolas Dichtela54acb32015-04-02 17:07:00 +0200684}
685EXPORT_SYMBOL(dev_get_iflink);
686
687/**
Pravin B Shelarfc4099f2015-10-22 18:17:16 -0700688 * dev_fill_metadata_dst - Retrieve tunnel egress information.
689 * @dev: targeted interface
690 * @skb: The packet.
691 *
692 * For better visibility of tunnel traffic OVS needs to retrieve
693 * egress tunnel information for a packet. Following API allows
694 * user to get this info.
695 */
696int dev_fill_metadata_dst(struct net_device *dev, struct sk_buff *skb)
697{
698 struct ip_tunnel_info *info;
699
700 if (!dev->netdev_ops || !dev->netdev_ops->ndo_fill_metadata_dst)
701 return -EINVAL;
702
703 info = skb_tunnel_info_unclone(skb);
704 if (!info)
705 return -ENOMEM;
706 if (unlikely(!(info->mode & IP_TUNNEL_INFO_TX)))
707 return -EINVAL;
708
709 return dev->netdev_ops->ndo_fill_metadata_dst(dev, skb);
710}
711EXPORT_SYMBOL_GPL(dev_fill_metadata_dst);
712
713/**
Linus Torvalds1da177e2005-04-16 15:20:36 -0700714 * __dev_get_by_name - find a device by its name
Randy Dunlapc4ea43c2007-10-12 21:17:49 -0700715 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -0700716 * @name: name to find
717 *
718 * Find an interface by name. Must be called under RTNL semaphore
719 * or @dev_base_lock. If the name is found a pointer to the device
720 * is returned. If the name is not found then %NULL is returned. The
721 * reference counters are not incremented so the caller must be
722 * careful with locks.
723 */
724
Eric W. Biederman881d9662007-09-17 11:56:21 -0700725struct net_device *__dev_get_by_name(struct net *net, const char *name)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700726{
Eric Dumazet0bd8d532009-10-30 01:40:11 -0700727 struct net_device *dev;
728 struct hlist_head *head = dev_name_hash(net, name);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700729
Sasha Levinb67bfe02013-02-27 17:06:00 -0800730 hlist_for_each_entry(dev, head, name_hlist)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700731 if (!strncmp(dev->name, name, IFNAMSIZ))
732 return dev;
Eric Dumazet0bd8d532009-10-30 01:40:11 -0700733
Linus Torvalds1da177e2005-04-16 15:20:36 -0700734 return NULL;
735}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700736EXPORT_SYMBOL(__dev_get_by_name);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700737
738/**
Eric Dumazet72c95282009-10-30 07:11:27 +0000739 * dev_get_by_name_rcu - find a device by its name
740 * @net: the applicable net namespace
741 * @name: name to find
742 *
743 * Find an interface by name.
744 * If the name is found a pointer to the device is returned.
745 * If the name is not found then %NULL is returned.
746 * The reference counters are not incremented so the caller must be
747 * careful with locks. The caller must hold RCU lock.
748 */
749
750struct net_device *dev_get_by_name_rcu(struct net *net, const char *name)
751{
Eric Dumazet72c95282009-10-30 07:11:27 +0000752 struct net_device *dev;
753 struct hlist_head *head = dev_name_hash(net, name);
754
Sasha Levinb67bfe02013-02-27 17:06:00 -0800755 hlist_for_each_entry_rcu(dev, head, name_hlist)
Eric Dumazet72c95282009-10-30 07:11:27 +0000756 if (!strncmp(dev->name, name, IFNAMSIZ))
757 return dev;
758
759 return NULL;
760}
761EXPORT_SYMBOL(dev_get_by_name_rcu);
762
763/**
Linus Torvalds1da177e2005-04-16 15:20:36 -0700764 * dev_get_by_name - find a device by its name
Randy Dunlapc4ea43c2007-10-12 21:17:49 -0700765 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -0700766 * @name: name to find
767 *
768 * Find an interface by name. This can be called from any
769 * context and does its own locking. The returned handle has
770 * the usage count incremented and the caller must use dev_put() to
771 * release it when it is no longer needed. %NULL is returned if no
772 * matching device is found.
773 */
774
Eric W. Biederman881d9662007-09-17 11:56:21 -0700775struct net_device *dev_get_by_name(struct net *net, const char *name)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700776{
777 struct net_device *dev;
778
Eric Dumazet72c95282009-10-30 07:11:27 +0000779 rcu_read_lock();
780 dev = dev_get_by_name_rcu(net, name);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700781 if (dev)
782 dev_hold(dev);
Eric Dumazet72c95282009-10-30 07:11:27 +0000783 rcu_read_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700784 return dev;
785}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700786EXPORT_SYMBOL(dev_get_by_name);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700787
788/**
789 * __dev_get_by_index - find a device by its ifindex
Randy Dunlapc4ea43c2007-10-12 21:17:49 -0700790 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -0700791 * @ifindex: index of device
792 *
793 * Search for an interface by index. Returns %NULL if the device
794 * is not found or a pointer to the device. The device has not
795 * had its reference counter increased so the caller must be careful
796 * about locking. The caller must hold either the RTNL semaphore
797 * or @dev_base_lock.
798 */
799
Eric W. Biederman881d9662007-09-17 11:56:21 -0700800struct net_device *__dev_get_by_index(struct net *net, int ifindex)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700801{
Eric Dumazet0bd8d532009-10-30 01:40:11 -0700802 struct net_device *dev;
803 struct hlist_head *head = dev_index_hash(net, ifindex);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700804
Sasha Levinb67bfe02013-02-27 17:06:00 -0800805 hlist_for_each_entry(dev, head, index_hlist)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700806 if (dev->ifindex == ifindex)
807 return dev;
Eric Dumazet0bd8d532009-10-30 01:40:11 -0700808
Linus Torvalds1da177e2005-04-16 15:20:36 -0700809 return NULL;
810}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700811EXPORT_SYMBOL(__dev_get_by_index);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700812
Eric Dumazetfb699dfd2009-10-19 19:18:49 +0000813/**
814 * dev_get_by_index_rcu - find a device by its ifindex
815 * @net: the applicable net namespace
816 * @ifindex: index of device
817 *
818 * Search for an interface by index. Returns %NULL if the device
819 * is not found or a pointer to the device. The device has not
820 * had its reference counter increased so the caller must be careful
821 * about locking. The caller must hold RCU lock.
822 */
823
824struct net_device *dev_get_by_index_rcu(struct net *net, int ifindex)
825{
Eric Dumazetfb699dfd2009-10-19 19:18:49 +0000826 struct net_device *dev;
827 struct hlist_head *head = dev_index_hash(net, ifindex);
828
Sasha Levinb67bfe02013-02-27 17:06:00 -0800829 hlist_for_each_entry_rcu(dev, head, index_hlist)
Eric Dumazetfb699dfd2009-10-19 19:18:49 +0000830 if (dev->ifindex == ifindex)
831 return dev;
832
833 return NULL;
834}
835EXPORT_SYMBOL(dev_get_by_index_rcu);
836
Linus Torvalds1da177e2005-04-16 15:20:36 -0700837
838/**
839 * dev_get_by_index - find a device by its ifindex
Randy Dunlapc4ea43c2007-10-12 21:17:49 -0700840 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -0700841 * @ifindex: index of device
842 *
843 * Search for an interface by index. Returns NULL if the device
844 * is not found or a pointer to the device. The device returned has
845 * had a reference added and the pointer is safe until the user calls
846 * dev_put to indicate they have finished with it.
847 */
848
Eric W. Biederman881d9662007-09-17 11:56:21 -0700849struct net_device *dev_get_by_index(struct net *net, int ifindex)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700850{
851 struct net_device *dev;
852
Eric Dumazetfb699dfd2009-10-19 19:18:49 +0000853 rcu_read_lock();
854 dev = dev_get_by_index_rcu(net, ifindex);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700855 if (dev)
856 dev_hold(dev);
Eric Dumazetfb699dfd2009-10-19 19:18:49 +0000857 rcu_read_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700858 return dev;
859}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700860EXPORT_SYMBOL(dev_get_by_index);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700861
862/**
Nicolas Schichan5dbe7c12013-06-26 17:23:42 +0200863 * netdev_get_name - get a netdevice name, knowing its ifindex.
864 * @net: network namespace
865 * @name: a pointer to the buffer where the name will be stored.
866 * @ifindex: the ifindex of the interface to get the name from.
867 *
868 * The use of raw_seqcount_begin() and cond_resched() before
869 * retrying is required as we want to give the writers a chance
870 * to complete when CONFIG_PREEMPT is not set.
871 */
872int netdev_get_name(struct net *net, char *name, int ifindex)
873{
874 struct net_device *dev;
875 unsigned int seq;
876
877retry:
878 seq = raw_seqcount_begin(&devnet_rename_seq);
879 rcu_read_lock();
880 dev = dev_get_by_index_rcu(net, ifindex);
881 if (!dev) {
882 rcu_read_unlock();
883 return -ENODEV;
884 }
885
886 strcpy(name, dev->name);
887 rcu_read_unlock();
888 if (read_seqcount_retry(&devnet_rename_seq, seq)) {
889 cond_resched();
890 goto retry;
891 }
892
893 return 0;
894}
895
896/**
Eric Dumazet941666c2010-12-05 01:23:53 +0000897 * dev_getbyhwaddr_rcu - find a device by its hardware address
Randy Dunlapc4ea43c2007-10-12 21:17:49 -0700898 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -0700899 * @type: media type of device
900 * @ha: hardware address
901 *
902 * Search for an interface by MAC address. Returns NULL if the device
Eric Dumazetc5066532011-01-24 13:16:16 -0800903 * is not found or a pointer to the device.
904 * The caller must hold RCU or RTNL.
Eric Dumazet941666c2010-12-05 01:23:53 +0000905 * The returned device has not had its ref count increased
Linus Torvalds1da177e2005-04-16 15:20:36 -0700906 * and the caller must therefore be careful about locking
907 *
Linus Torvalds1da177e2005-04-16 15:20:36 -0700908 */
909
Eric Dumazet941666c2010-12-05 01:23:53 +0000910struct net_device *dev_getbyhwaddr_rcu(struct net *net, unsigned short type,
911 const char *ha)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700912{
913 struct net_device *dev;
914
Eric Dumazet941666c2010-12-05 01:23:53 +0000915 for_each_netdev_rcu(net, dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700916 if (dev->type == type &&
917 !memcmp(dev->dev_addr, ha, dev->addr_len))
Pavel Emelianov7562f872007-05-03 15:13:45 -0700918 return dev;
919
920 return NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700921}
Eric Dumazet941666c2010-12-05 01:23:53 +0000922EXPORT_SYMBOL(dev_getbyhwaddr_rcu);
Jochen Friedrichcf309e32005-09-22 04:44:55 -0300923
Eric W. Biederman881d9662007-09-17 11:56:21 -0700924struct net_device *__dev_getfirstbyhwtype(struct net *net, unsigned short type)
Patrick McHardy4e9cac22007-05-03 03:28:13 -0700925{
926 struct net_device *dev;
927
928 ASSERT_RTNL();
Eric W. Biederman881d9662007-09-17 11:56:21 -0700929 for_each_netdev(net, dev)
Patrick McHardy4e9cac22007-05-03 03:28:13 -0700930 if (dev->type == type)
Pavel Emelianov7562f872007-05-03 15:13:45 -0700931 return dev;
932
933 return NULL;
Patrick McHardy4e9cac22007-05-03 03:28:13 -0700934}
Patrick McHardy4e9cac22007-05-03 03:28:13 -0700935EXPORT_SYMBOL(__dev_getfirstbyhwtype);
936
Eric W. Biederman881d9662007-09-17 11:56:21 -0700937struct net_device *dev_getfirstbyhwtype(struct net *net, unsigned short type)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700938{
Eric Dumazet99fe3c32010-03-18 11:27:25 +0000939 struct net_device *dev, *ret = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700940
Eric Dumazet99fe3c32010-03-18 11:27:25 +0000941 rcu_read_lock();
942 for_each_netdev_rcu(net, dev)
943 if (dev->type == type) {
944 dev_hold(dev);
945 ret = dev;
946 break;
947 }
948 rcu_read_unlock();
949 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700950}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700951EXPORT_SYMBOL(dev_getfirstbyhwtype);
952
953/**
WANG Cong6c555492014-09-11 15:35:09 -0700954 * __dev_get_by_flags - find any device with given flags
Randy Dunlapc4ea43c2007-10-12 21:17:49 -0700955 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -0700956 * @if_flags: IFF_* values
957 * @mask: bitmask of bits in if_flags to check
958 *
959 * Search for any interface with the given flags. Returns NULL if a device
Eric Dumazetbb69ae02010-06-07 11:42:13 +0000960 * is not found or a pointer to the device. Must be called inside
WANG Cong6c555492014-09-11 15:35:09 -0700961 * rtnl_lock(), and result refcount is unchanged.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700962 */
963
WANG Cong6c555492014-09-11 15:35:09 -0700964struct net_device *__dev_get_by_flags(struct net *net, unsigned short if_flags,
965 unsigned short mask)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700966{
Pavel Emelianov7562f872007-05-03 15:13:45 -0700967 struct net_device *dev, *ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700968
WANG Cong6c555492014-09-11 15:35:09 -0700969 ASSERT_RTNL();
970
Pavel Emelianov7562f872007-05-03 15:13:45 -0700971 ret = NULL;
WANG Cong6c555492014-09-11 15:35:09 -0700972 for_each_netdev(net, dev) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700973 if (((dev->flags ^ if_flags) & mask) == 0) {
Pavel Emelianov7562f872007-05-03 15:13:45 -0700974 ret = dev;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700975 break;
976 }
977 }
Pavel Emelianov7562f872007-05-03 15:13:45 -0700978 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700979}
WANG Cong6c555492014-09-11 15:35:09 -0700980EXPORT_SYMBOL(__dev_get_by_flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700981
982/**
983 * dev_valid_name - check if name is okay for network device
984 * @name: name string
985 *
986 * Network device names need to be valid file names to
David S. Millerc7fa9d12006-08-15 16:34:13 -0700987 * to allow sysfs to work. We also disallow any kind of
988 * whitespace.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700989 */
David S. Miller95f050b2012-03-06 16:12:15 -0500990bool dev_valid_name(const char *name)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700991{
David S. Millerc7fa9d12006-08-15 16:34:13 -0700992 if (*name == '\0')
David S. Miller95f050b2012-03-06 16:12:15 -0500993 return false;
Stephen Hemmingerb6fe17d2006-08-29 17:06:13 -0700994 if (strlen(name) >= IFNAMSIZ)
David S. Miller95f050b2012-03-06 16:12:15 -0500995 return false;
David S. Millerc7fa9d12006-08-15 16:34:13 -0700996 if (!strcmp(name, ".") || !strcmp(name, ".."))
David S. Miller95f050b2012-03-06 16:12:15 -0500997 return false;
David S. Millerc7fa9d12006-08-15 16:34:13 -0700998
999 while (*name) {
Matthew Thodea4176a92015-02-17 18:31:57 -06001000 if (*name == '/' || *name == ':' || isspace(*name))
David S. Miller95f050b2012-03-06 16:12:15 -05001001 return false;
David S. Millerc7fa9d12006-08-15 16:34:13 -07001002 name++;
1003 }
David S. Miller95f050b2012-03-06 16:12:15 -05001004 return true;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001005}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001006EXPORT_SYMBOL(dev_valid_name);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001007
1008/**
Eric W. Biedermanb267b172007-09-12 13:48:45 +02001009 * __dev_alloc_name - allocate a name for a device
1010 * @net: network namespace to allocate the device name in
Linus Torvalds1da177e2005-04-16 15:20:36 -07001011 * @name: name format string
Eric W. Biedermanb267b172007-09-12 13:48:45 +02001012 * @buf: scratch buffer and result name string
Linus Torvalds1da177e2005-04-16 15:20:36 -07001013 *
1014 * Passed a format string - eg "lt%d" it will try and find a suitable
Stephen Hemminger3041a062006-05-26 13:25:24 -07001015 * id. It scans list of devices to build up a free map, then chooses
1016 * the first empty slot. The caller must hold the dev_base or rtnl lock
1017 * while allocating the name and adding the device in order to avoid
1018 * duplicates.
1019 * Limited to bits_per_byte * page size devices (ie 32K on most platforms).
1020 * Returns the number of the unit assigned or a negative errno code.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001021 */
1022
Eric W. Biedermanb267b172007-09-12 13:48:45 +02001023static int __dev_alloc_name(struct net *net, const char *name, char *buf)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001024{
1025 int i = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001026 const char *p;
1027 const int max_netdevices = 8*PAGE_SIZE;
Stephen Hemmingercfcabdc2007-10-09 01:59:42 -07001028 unsigned long *inuse;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001029 struct net_device *d;
1030
1031 p = strnchr(name, IFNAMSIZ-1, '%');
1032 if (p) {
1033 /*
1034 * Verify the string as this thing may have come from
1035 * the user. There must be either one "%d" and no other "%"
1036 * characters.
1037 */
1038 if (p[1] != 'd' || strchr(p + 2, '%'))
1039 return -EINVAL;
1040
1041 /* Use one page as a bit array of possible slots */
Stephen Hemmingercfcabdc2007-10-09 01:59:42 -07001042 inuse = (unsigned long *) get_zeroed_page(GFP_ATOMIC);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001043 if (!inuse)
1044 return -ENOMEM;
1045
Eric W. Biederman881d9662007-09-17 11:56:21 -07001046 for_each_netdev(net, d) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001047 if (!sscanf(d->name, name, &i))
1048 continue;
1049 if (i < 0 || i >= max_netdevices)
1050 continue;
1051
1052 /* avoid cases where sscanf is not exact inverse of printf */
Eric W. Biedermanb267b172007-09-12 13:48:45 +02001053 snprintf(buf, IFNAMSIZ, name, i);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001054 if (!strncmp(buf, d->name, IFNAMSIZ))
1055 set_bit(i, inuse);
1056 }
1057
1058 i = find_first_zero_bit(inuse, max_netdevices);
1059 free_page((unsigned long) inuse);
1060 }
1061
Octavian Purdilad9031022009-11-18 02:36:59 +00001062 if (buf != name)
1063 snprintf(buf, IFNAMSIZ, name, i);
Eric W. Biedermanb267b172007-09-12 13:48:45 +02001064 if (!__dev_get_by_name(net, buf))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001065 return i;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001066
1067 /* It is possible to run out of possible slots
1068 * when the name is long and there isn't enough space left
1069 * for the digits, or if all bits are used.
1070 */
1071 return -ENFILE;
1072}
1073
Eric W. Biedermanb267b172007-09-12 13:48:45 +02001074/**
1075 * dev_alloc_name - allocate a name for a device
1076 * @dev: device
1077 * @name: name format string
1078 *
1079 * Passed a format string - eg "lt%d" it will try and find a suitable
1080 * id. It scans list of devices to build up a free map, then chooses
1081 * the first empty slot. The caller must hold the dev_base or rtnl lock
1082 * while allocating the name and adding the device in order to avoid
1083 * duplicates.
1084 * Limited to bits_per_byte * page size devices (ie 32K on most platforms).
1085 * Returns the number of the unit assigned or a negative errno code.
1086 */
1087
1088int dev_alloc_name(struct net_device *dev, const char *name)
1089{
1090 char buf[IFNAMSIZ];
1091 struct net *net;
1092 int ret;
1093
YOSHIFUJI Hideakic346dca2008-03-25 21:47:49 +09001094 BUG_ON(!dev_net(dev));
1095 net = dev_net(dev);
Eric W. Biedermanb267b172007-09-12 13:48:45 +02001096 ret = __dev_alloc_name(net, name, buf);
1097 if (ret >= 0)
1098 strlcpy(dev->name, buf, IFNAMSIZ);
1099 return ret;
1100}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001101EXPORT_SYMBOL(dev_alloc_name);
Eric W. Biedermanb267b172007-09-12 13:48:45 +02001102
Gao feng828de4f2012-09-13 20:58:27 +00001103static int dev_alloc_name_ns(struct net *net,
1104 struct net_device *dev,
1105 const char *name)
Octavian Purdilad9031022009-11-18 02:36:59 +00001106{
Gao feng828de4f2012-09-13 20:58:27 +00001107 char buf[IFNAMSIZ];
1108 int ret;
Daniel Lezcano8ce6cebc2010-05-19 10:12:19 +00001109
Gao feng828de4f2012-09-13 20:58:27 +00001110 ret = __dev_alloc_name(net, name, buf);
1111 if (ret >= 0)
1112 strlcpy(dev->name, buf, IFNAMSIZ);
1113 return ret;
1114}
1115
1116static int dev_get_valid_name(struct net *net,
1117 struct net_device *dev,
1118 const char *name)
1119{
1120 BUG_ON(!net);
Daniel Lezcano8ce6cebc2010-05-19 10:12:19 +00001121
Octavian Purdilad9031022009-11-18 02:36:59 +00001122 if (!dev_valid_name(name))
1123 return -EINVAL;
1124
Jiri Pirko1c5cae82011-04-30 01:21:32 +00001125 if (strchr(name, '%'))
Gao feng828de4f2012-09-13 20:58:27 +00001126 return dev_alloc_name_ns(net, dev, name);
Octavian Purdilad9031022009-11-18 02:36:59 +00001127 else if (__dev_get_by_name(net, name))
1128 return -EEXIST;
Daniel Lezcano8ce6cebc2010-05-19 10:12:19 +00001129 else if (dev->name != name)
1130 strlcpy(dev->name, name, IFNAMSIZ);
Octavian Purdilad9031022009-11-18 02:36:59 +00001131
1132 return 0;
1133}
Linus Torvalds1da177e2005-04-16 15:20:36 -07001134
1135/**
1136 * dev_change_name - change name of a device
1137 * @dev: device
1138 * @newname: name (or format string) must be at least IFNAMSIZ
1139 *
1140 * Change name of a device, can pass format strings "eth%d".
1141 * for wildcarding.
1142 */
Stephen Hemmingercf04a4c72008-09-30 02:22:14 -07001143int dev_change_name(struct net_device *dev, const char *newname)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001144{
Tom Gundersen238fa362014-07-14 16:37:23 +02001145 unsigned char old_assign_type;
Herbert Xufcc5a032007-07-30 17:03:38 -07001146 char oldname[IFNAMSIZ];
Linus Torvalds1da177e2005-04-16 15:20:36 -07001147 int err = 0;
Herbert Xufcc5a032007-07-30 17:03:38 -07001148 int ret;
Eric W. Biederman881d9662007-09-17 11:56:21 -07001149 struct net *net;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001150
1151 ASSERT_RTNL();
YOSHIFUJI Hideakic346dca2008-03-25 21:47:49 +09001152 BUG_ON(!dev_net(dev));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001153
YOSHIFUJI Hideakic346dca2008-03-25 21:47:49 +09001154 net = dev_net(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001155 if (dev->flags & IFF_UP)
1156 return -EBUSY;
1157
Eric Dumazet30e6c9f2012-12-20 17:25:08 +00001158 write_seqcount_begin(&devnet_rename_seq);
Brian Haleyc91f6df2012-11-26 05:21:08 +00001159
1160 if (strncmp(newname, dev->name, IFNAMSIZ) == 0) {
Eric Dumazet30e6c9f2012-12-20 17:25:08 +00001161 write_seqcount_end(&devnet_rename_seq);
Stephen Hemmingerc8d90dc2007-10-26 03:53:42 -07001162 return 0;
Brian Haleyc91f6df2012-11-26 05:21:08 +00001163 }
Stephen Hemmingerc8d90dc2007-10-26 03:53:42 -07001164
Herbert Xufcc5a032007-07-30 17:03:38 -07001165 memcpy(oldname, dev->name, IFNAMSIZ);
1166
Gao feng828de4f2012-09-13 20:58:27 +00001167 err = dev_get_valid_name(net, dev, newname);
Brian Haleyc91f6df2012-11-26 05:21:08 +00001168 if (err < 0) {
Eric Dumazet30e6c9f2012-12-20 17:25:08 +00001169 write_seqcount_end(&devnet_rename_seq);
Octavian Purdilad9031022009-11-18 02:36:59 +00001170 return err;
Brian Haleyc91f6df2012-11-26 05:21:08 +00001171 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001172
Veaceslav Falico6fe82a32014-07-17 20:33:32 +02001173 if (oldname[0] && !strchr(oldname, '%'))
1174 netdev_info(dev, "renamed from %s\n", oldname);
1175
Tom Gundersen238fa362014-07-14 16:37:23 +02001176 old_assign_type = dev->name_assign_type;
1177 dev->name_assign_type = NET_NAME_RENAMED;
1178
Herbert Xufcc5a032007-07-30 17:03:38 -07001179rollback:
Eric W. Biedermana1b3f592010-05-04 17:36:49 -07001180 ret = device_rename(&dev->dev, dev->name);
1181 if (ret) {
1182 memcpy(dev->name, oldname, IFNAMSIZ);
Tom Gundersen238fa362014-07-14 16:37:23 +02001183 dev->name_assign_type = old_assign_type;
Eric Dumazet30e6c9f2012-12-20 17:25:08 +00001184 write_seqcount_end(&devnet_rename_seq);
Eric W. Biedermana1b3f592010-05-04 17:36:49 -07001185 return ret;
Stephen Hemmingerdcc99772008-05-14 22:33:38 -07001186 }
Herbert Xu7f988ea2007-07-30 16:35:46 -07001187
Eric Dumazet30e6c9f2012-12-20 17:25:08 +00001188 write_seqcount_end(&devnet_rename_seq);
Brian Haleyc91f6df2012-11-26 05:21:08 +00001189
Veaceslav Falico5bb025f2014-01-14 21:58:51 +01001190 netdev_adjacent_rename_links(dev, oldname);
1191
Herbert Xu7f988ea2007-07-30 16:35:46 -07001192 write_lock_bh(&dev_base_lock);
Eric Dumazet372b2312011-05-17 13:56:59 -04001193 hlist_del_rcu(&dev->name_hlist);
Eric Dumazet72c95282009-10-30 07:11:27 +00001194 write_unlock_bh(&dev_base_lock);
1195
1196 synchronize_rcu();
1197
1198 write_lock_bh(&dev_base_lock);
1199 hlist_add_head_rcu(&dev->name_hlist, dev_name_hash(net, dev->name));
Herbert Xu7f988ea2007-07-30 16:35:46 -07001200 write_unlock_bh(&dev_base_lock);
1201
Pavel Emelyanov056925a2007-09-16 15:42:43 -07001202 ret = call_netdevice_notifiers(NETDEV_CHANGENAME, dev);
Herbert Xufcc5a032007-07-30 17:03:38 -07001203 ret = notifier_to_errno(ret);
1204
1205 if (ret) {
Eric Dumazet91e9c07b2009-11-15 23:30:24 +00001206 /* err >= 0 after dev_alloc_name() or stores the first errno */
1207 if (err >= 0) {
Herbert Xufcc5a032007-07-30 17:03:38 -07001208 err = ret;
Eric Dumazet30e6c9f2012-12-20 17:25:08 +00001209 write_seqcount_begin(&devnet_rename_seq);
Herbert Xufcc5a032007-07-30 17:03:38 -07001210 memcpy(dev->name, oldname, IFNAMSIZ);
Veaceslav Falico5bb025f2014-01-14 21:58:51 +01001211 memcpy(oldname, newname, IFNAMSIZ);
Tom Gundersen238fa362014-07-14 16:37:23 +02001212 dev->name_assign_type = old_assign_type;
1213 old_assign_type = NET_NAME_RENAMED;
Herbert Xufcc5a032007-07-30 17:03:38 -07001214 goto rollback;
Eric Dumazet91e9c07b2009-11-15 23:30:24 +00001215 } else {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00001216 pr_err("%s: name change rollback failed: %d\n",
Eric Dumazet91e9c07b2009-11-15 23:30:24 +00001217 dev->name, ret);
Herbert Xufcc5a032007-07-30 17:03:38 -07001218 }
1219 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001220
1221 return err;
1222}
1223
1224/**
Stephen Hemminger0b815a12008-09-22 21:28:11 -07001225 * dev_set_alias - change ifalias of a device
1226 * @dev: device
1227 * @alias: name up to IFALIASZ
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07001228 * @len: limit of bytes to copy from info
Stephen Hemminger0b815a12008-09-22 21:28:11 -07001229 *
1230 * Set ifalias for a device,
1231 */
1232int dev_set_alias(struct net_device *dev, const char *alias, size_t len)
1233{
Alexey Khoroshilov7364e442012-08-08 00:33:25 +00001234 char *new_ifalias;
1235
Stephen Hemminger0b815a12008-09-22 21:28:11 -07001236 ASSERT_RTNL();
1237
1238 if (len >= IFALIASZ)
1239 return -EINVAL;
1240
Oliver Hartkopp96ca4a22008-09-23 21:23:19 -07001241 if (!len) {
Sachin Kamat388dfc22012-11-20 00:57:04 +00001242 kfree(dev->ifalias);
1243 dev->ifalias = NULL;
Oliver Hartkopp96ca4a22008-09-23 21:23:19 -07001244 return 0;
1245 }
1246
Alexey Khoroshilov7364e442012-08-08 00:33:25 +00001247 new_ifalias = krealloc(dev->ifalias, len + 1, GFP_KERNEL);
1248 if (!new_ifalias)
Stephen Hemminger0b815a12008-09-22 21:28:11 -07001249 return -ENOMEM;
Alexey Khoroshilov7364e442012-08-08 00:33:25 +00001250 dev->ifalias = new_ifalias;
Stephen Hemminger0b815a12008-09-22 21:28:11 -07001251
1252 strlcpy(dev->ifalias, alias, len+1);
1253 return len;
1254}
1255
1256
1257/**
Stephen Hemminger3041a062006-05-26 13:25:24 -07001258 * netdev_features_change - device changes features
Stephen Hemmingerd8a33ac2005-05-29 14:13:47 -07001259 * @dev: device to cause notification
1260 *
1261 * Called to indicate a device has changed features.
1262 */
1263void netdev_features_change(struct net_device *dev)
1264{
Pavel Emelyanov056925a2007-09-16 15:42:43 -07001265 call_netdevice_notifiers(NETDEV_FEAT_CHANGE, dev);
Stephen Hemmingerd8a33ac2005-05-29 14:13:47 -07001266}
1267EXPORT_SYMBOL(netdev_features_change);
1268
1269/**
Linus Torvalds1da177e2005-04-16 15:20:36 -07001270 * netdev_state_change - device changes state
1271 * @dev: device to cause notification
1272 *
1273 * Called to indicate a device has changed state. This function calls
1274 * the notifier chains for netdev_chain and sends a NEWLINK message
1275 * to the routing socket.
1276 */
1277void netdev_state_change(struct net_device *dev)
1278{
1279 if (dev->flags & IFF_UP) {
Loic Prylli5495119462014-07-01 21:39:43 -07001280 struct netdev_notifier_change_info change_info;
1281
1282 change_info.flags_changed = 0;
1283 call_netdevice_notifiers_info(NETDEV_CHANGE, dev,
1284 &change_info.info);
Alexei Starovoitov7f294052013-10-23 16:02:42 -07001285 rtmsg_ifinfo(RTM_NEWLINK, dev, 0, GFP_KERNEL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001286 }
1287}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001288EXPORT_SYMBOL(netdev_state_change);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001289
Amerigo Wangee89bab2012-08-09 22:14:56 +00001290/**
1291 * netdev_notify_peers - notify network peers about existence of @dev
1292 * @dev: network device
1293 *
1294 * Generate traffic such that interested network peers are aware of
1295 * @dev, such as by generating a gratuitous ARP. This may be used when
1296 * a device wants to inform the rest of the network about some sort of
1297 * reconfiguration such as a failover event or virtual machine
1298 * migration.
1299 */
1300void netdev_notify_peers(struct net_device *dev)
Or Gerlitzc1da4ac2008-06-13 18:12:00 -07001301{
Amerigo Wangee89bab2012-08-09 22:14:56 +00001302 rtnl_lock();
1303 call_netdevice_notifiers(NETDEV_NOTIFY_PEERS, dev);
1304 rtnl_unlock();
Or Gerlitzc1da4ac2008-06-13 18:12:00 -07001305}
Amerigo Wangee89bab2012-08-09 22:14:56 +00001306EXPORT_SYMBOL(netdev_notify_peers);
Or Gerlitzc1da4ac2008-06-13 18:12:00 -07001307
Patrick McHardybd380812010-02-26 06:34:53 +00001308static int __dev_open(struct net_device *dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001309{
Stephen Hemmingerd3147742008-11-19 21:32:24 -08001310 const struct net_device_ops *ops = dev->netdev_ops;
Johannes Berg3b8bcfd2009-05-30 01:39:53 +02001311 int ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001312
Ben Hutchingse46b66b2008-05-08 02:53:17 -07001313 ASSERT_RTNL();
1314
Linus Torvalds1da177e2005-04-16 15:20:36 -07001315 if (!netif_device_present(dev))
1316 return -ENODEV;
1317
Neil Hormanca99ca12013-02-05 08:05:43 +00001318 /* Block netpoll from trying to do any rx path servicing.
1319 * If we don't do this there is a chance ndo_poll_controller
1320 * or ndo_poll may be running while we open the device
1321 */
Eric W. Biederman66b55522014-03-27 15:39:03 -07001322 netpoll_poll_disable(dev);
Neil Hormanca99ca12013-02-05 08:05:43 +00001323
Johannes Berg3b8bcfd2009-05-30 01:39:53 +02001324 ret = call_netdevice_notifiers(NETDEV_PRE_UP, dev);
1325 ret = notifier_to_errno(ret);
1326 if (ret)
1327 return ret;
1328
Linus Torvalds1da177e2005-04-16 15:20:36 -07001329 set_bit(__LINK_STATE_START, &dev->state);
Jeff Garzikbada3392007-10-23 20:19:37 -07001330
Stephen Hemmingerd3147742008-11-19 21:32:24 -08001331 if (ops->ndo_validate_addr)
1332 ret = ops->ndo_validate_addr(dev);
Jeff Garzikbada3392007-10-23 20:19:37 -07001333
Stephen Hemmingerd3147742008-11-19 21:32:24 -08001334 if (!ret && ops->ndo_open)
1335 ret = ops->ndo_open(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001336
Eric W. Biederman66b55522014-03-27 15:39:03 -07001337 netpoll_poll_enable(dev);
Neil Hormanca99ca12013-02-05 08:05:43 +00001338
Jeff Garzikbada3392007-10-23 20:19:37 -07001339 if (ret)
1340 clear_bit(__LINK_STATE_START, &dev->state);
1341 else {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001342 dev->flags |= IFF_UP;
Patrick McHardy4417da62007-06-27 01:28:10 -07001343 dev_set_rx_mode(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001344 dev_activate(dev);
Theodore Ts'o7bf23572012-07-04 21:23:25 -04001345 add_device_randomness(dev->dev_addr, dev->addr_len);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001346 }
Jeff Garzikbada3392007-10-23 20:19:37 -07001347
Linus Torvalds1da177e2005-04-16 15:20:36 -07001348 return ret;
1349}
Patrick McHardybd380812010-02-26 06:34:53 +00001350
1351/**
1352 * dev_open - prepare an interface for use.
1353 * @dev: device to open
1354 *
1355 * Takes a device from down to up state. The device's private open
1356 * function is invoked and then the multicast lists are loaded. Finally
1357 * the device is moved into the up state and a %NETDEV_UP message is
1358 * sent to the netdev notifier chain.
1359 *
1360 * Calling this function on an active interface is a nop. On a failure
1361 * a negative errno code is returned.
1362 */
1363int dev_open(struct net_device *dev)
1364{
1365 int ret;
1366
Patrick McHardybd380812010-02-26 06:34:53 +00001367 if (dev->flags & IFF_UP)
1368 return 0;
1369
Patrick McHardybd380812010-02-26 06:34:53 +00001370 ret = __dev_open(dev);
1371 if (ret < 0)
1372 return ret;
1373
Alexei Starovoitov7f294052013-10-23 16:02:42 -07001374 rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP|IFF_RUNNING, GFP_KERNEL);
Patrick McHardybd380812010-02-26 06:34:53 +00001375 call_netdevice_notifiers(NETDEV_UP, dev);
1376
1377 return ret;
1378}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001379EXPORT_SYMBOL(dev_open);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001380
Octavian Purdila44345722010-12-13 12:44:07 +00001381static int __dev_close_many(struct list_head *head)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001382{
Octavian Purdila44345722010-12-13 12:44:07 +00001383 struct net_device *dev;
Patrick McHardybd380812010-02-26 06:34:53 +00001384
Ben Hutchingse46b66b2008-05-08 02:53:17 -07001385 ASSERT_RTNL();
David S. Miller9d5010d2007-09-12 14:33:25 +02001386 might_sleep();
1387
Eric W. Biederman5cde2822013-10-05 19:26:05 -07001388 list_for_each_entry(dev, head, close_list) {
Eric W. Biederman3f4df202014-03-27 15:38:17 -07001389 /* Temporarily disable netpoll until the interface is down */
Eric W. Biederman66b55522014-03-27 15:39:03 -07001390 netpoll_poll_disable(dev);
Eric W. Biederman3f4df202014-03-27 15:38:17 -07001391
Octavian Purdila44345722010-12-13 12:44:07 +00001392 call_netdevice_notifiers(NETDEV_GOING_DOWN, dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001393
Octavian Purdila44345722010-12-13 12:44:07 +00001394 clear_bit(__LINK_STATE_START, &dev->state);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001395
Octavian Purdila44345722010-12-13 12:44:07 +00001396 /* Synchronize to scheduled poll. We cannot touch poll list, it
1397 * can be even on different cpu. So just clear netif_running().
1398 *
1399 * dev->stop() will invoke napi_disable() on all of it's
1400 * napi_struct instances on this device.
1401 */
Peter Zijlstra4e857c52014-03-17 18:06:10 +01001402 smp_mb__after_atomic(); /* Commit netif_running(). */
Octavian Purdila44345722010-12-13 12:44:07 +00001403 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001404
Octavian Purdila44345722010-12-13 12:44:07 +00001405 dev_deactivate_many(head);
1406
Eric W. Biederman5cde2822013-10-05 19:26:05 -07001407 list_for_each_entry(dev, head, close_list) {
Octavian Purdila44345722010-12-13 12:44:07 +00001408 const struct net_device_ops *ops = dev->netdev_ops;
1409
1410 /*
1411 * Call the device specific close. This cannot fail.
1412 * Only if device is UP
1413 *
1414 * We allow it to be called even after a DETACH hot-plug
1415 * event.
1416 */
1417 if (ops->ndo_stop)
1418 ops->ndo_stop(dev);
1419
Octavian Purdila44345722010-12-13 12:44:07 +00001420 dev->flags &= ~IFF_UP;
Eric W. Biederman66b55522014-03-27 15:39:03 -07001421 netpoll_poll_enable(dev);
Octavian Purdila44345722010-12-13 12:44:07 +00001422 }
1423
1424 return 0;
1425}
1426
1427static int __dev_close(struct net_device *dev)
1428{
Linus Torvaldsf87e6f42011-02-17 22:54:38 +00001429 int retval;
Octavian Purdila44345722010-12-13 12:44:07 +00001430 LIST_HEAD(single);
1431
Eric W. Biederman5cde2822013-10-05 19:26:05 -07001432 list_add(&dev->close_list, &single);
Linus Torvaldsf87e6f42011-02-17 22:54:38 +00001433 retval = __dev_close_many(&single);
1434 list_del(&single);
Neil Hormanca99ca12013-02-05 08:05:43 +00001435
Linus Torvaldsf87e6f42011-02-17 22:54:38 +00001436 return retval;
Octavian Purdila44345722010-12-13 12:44:07 +00001437}
1438
David S. Miller99c4a262015-03-18 22:52:33 -04001439int dev_close_many(struct list_head *head, bool unlink)
Octavian Purdila44345722010-12-13 12:44:07 +00001440{
1441 struct net_device *dev, *tmp;
Octavian Purdila44345722010-12-13 12:44:07 +00001442
Eric W. Biederman5cde2822013-10-05 19:26:05 -07001443 /* Remove the devices that don't need to be closed */
1444 list_for_each_entry_safe(dev, tmp, head, close_list)
Octavian Purdila44345722010-12-13 12:44:07 +00001445 if (!(dev->flags & IFF_UP))
Eric W. Biederman5cde2822013-10-05 19:26:05 -07001446 list_del_init(&dev->close_list);
Octavian Purdila44345722010-12-13 12:44:07 +00001447
1448 __dev_close_many(head);
Matti Linnanvuorid8b2a4d2008-02-12 23:10:11 -08001449
Eric W. Biederman5cde2822013-10-05 19:26:05 -07001450 list_for_each_entry_safe(dev, tmp, head, close_list) {
Alexei Starovoitov7f294052013-10-23 16:02:42 -07001451 rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP|IFF_RUNNING, GFP_KERNEL);
Octavian Purdila44345722010-12-13 12:44:07 +00001452 call_netdevice_notifiers(NETDEV_DOWN, dev);
David S. Miller99c4a262015-03-18 22:52:33 -04001453 if (unlink)
1454 list_del_init(&dev->close_list);
Octavian Purdila44345722010-12-13 12:44:07 +00001455 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001456
Linus Torvalds1da177e2005-04-16 15:20:36 -07001457 return 0;
1458}
David S. Miller99c4a262015-03-18 22:52:33 -04001459EXPORT_SYMBOL(dev_close_many);
Patrick McHardybd380812010-02-26 06:34:53 +00001460
1461/**
1462 * dev_close - shutdown an interface.
1463 * @dev: device to shutdown
1464 *
1465 * This function moves an active device into down state. A
1466 * %NETDEV_GOING_DOWN is sent to the netdev notifier chain. The device
1467 * is then deactivated and finally a %NETDEV_DOWN is sent to the notifier
1468 * chain.
1469 */
1470int dev_close(struct net_device *dev)
1471{
Eric Dumazete14a5992011-05-10 12:26:06 -07001472 if (dev->flags & IFF_UP) {
1473 LIST_HEAD(single);
Patrick McHardybd380812010-02-26 06:34:53 +00001474
Eric W. Biederman5cde2822013-10-05 19:26:05 -07001475 list_add(&dev->close_list, &single);
David S. Miller99c4a262015-03-18 22:52:33 -04001476 dev_close_many(&single, true);
Eric Dumazete14a5992011-05-10 12:26:06 -07001477 list_del(&single);
1478 }
dingtianhongda6e3782013-05-27 19:53:31 +00001479 return 0;
Patrick McHardybd380812010-02-26 06:34:53 +00001480}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001481EXPORT_SYMBOL(dev_close);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001482
1483
Ben Hutchings0187bdf2008-06-19 16:15:47 -07001484/**
1485 * dev_disable_lro - disable Large Receive Offload on a device
1486 * @dev: device
1487 *
1488 * Disable Large Receive Offload (LRO) on a net device. Must be
1489 * called under RTNL. This is needed if received packets may be
1490 * forwarded to another interface.
1491 */
1492void dev_disable_lro(struct net_device *dev)
1493{
Michal Kubečekfbe168b2014-11-13 07:54:50 +01001494 struct net_device *lower_dev;
1495 struct list_head *iter;
Michal Kubeček529d0482013-11-15 06:18:50 +01001496
Michał Mirosławbc5787c62011-11-15 15:29:55 +00001497 dev->wanted_features &= ~NETIF_F_LRO;
1498 netdev_update_features(dev);
Michał Mirosław27660512011-03-18 16:56:34 +00001499
Michał Mirosław22d59692011-04-21 12:42:15 +00001500 if (unlikely(dev->features & NETIF_F_LRO))
1501 netdev_WARN(dev, "failed to disable LRO!\n");
Michal Kubečekfbe168b2014-11-13 07:54:50 +01001502
1503 netdev_for_each_lower_dev(dev, lower_dev, iter)
1504 dev_disable_lro(lower_dev);
Ben Hutchings0187bdf2008-06-19 16:15:47 -07001505}
1506EXPORT_SYMBOL(dev_disable_lro);
1507
Jiri Pirko351638e2013-05-28 01:30:21 +00001508static int call_netdevice_notifier(struct notifier_block *nb, unsigned long val,
1509 struct net_device *dev)
1510{
1511 struct netdev_notifier_info info;
1512
1513 netdev_notifier_info_init(&info, dev);
1514 return nb->notifier_call(nb, val, &info);
1515}
Ben Hutchings0187bdf2008-06-19 16:15:47 -07001516
Eric W. Biederman881d9662007-09-17 11:56:21 -07001517static int dev_boot_phase = 1;
1518
Linus Torvalds1da177e2005-04-16 15:20:36 -07001519/**
1520 * register_netdevice_notifier - register a network notifier block
1521 * @nb: notifier
1522 *
1523 * Register a notifier to be called when network device events occur.
1524 * The notifier passed is linked into the kernel structures and must
1525 * not be reused until it has been unregistered. A negative errno code
1526 * is returned on a failure.
1527 *
1528 * When registered all registration and up events are replayed
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09001529 * to the new notifier to allow device to have a race free
Linus Torvalds1da177e2005-04-16 15:20:36 -07001530 * view of the network device list.
1531 */
1532
1533int register_netdevice_notifier(struct notifier_block *nb)
1534{
1535 struct net_device *dev;
Herbert Xufcc5a032007-07-30 17:03:38 -07001536 struct net_device *last;
Eric W. Biederman881d9662007-09-17 11:56:21 -07001537 struct net *net;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001538 int err;
1539
1540 rtnl_lock();
Alan Sternf07d5b92006-05-09 15:23:03 -07001541 err = raw_notifier_chain_register(&netdev_chain, nb);
Herbert Xufcc5a032007-07-30 17:03:38 -07001542 if (err)
1543 goto unlock;
Eric W. Biederman881d9662007-09-17 11:56:21 -07001544 if (dev_boot_phase)
1545 goto unlock;
1546 for_each_net(net) {
1547 for_each_netdev(net, dev) {
Jiri Pirko351638e2013-05-28 01:30:21 +00001548 err = call_netdevice_notifier(nb, NETDEV_REGISTER, dev);
Eric W. Biederman881d9662007-09-17 11:56:21 -07001549 err = notifier_to_errno(err);
1550 if (err)
1551 goto rollback;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001552
Eric W. Biederman881d9662007-09-17 11:56:21 -07001553 if (!(dev->flags & IFF_UP))
1554 continue;
Herbert Xufcc5a032007-07-30 17:03:38 -07001555
Jiri Pirko351638e2013-05-28 01:30:21 +00001556 call_netdevice_notifier(nb, NETDEV_UP, dev);
Eric W. Biederman881d9662007-09-17 11:56:21 -07001557 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001558 }
Herbert Xufcc5a032007-07-30 17:03:38 -07001559
1560unlock:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001561 rtnl_unlock();
1562 return err;
Herbert Xufcc5a032007-07-30 17:03:38 -07001563
1564rollback:
1565 last = dev;
Eric W. Biederman881d9662007-09-17 11:56:21 -07001566 for_each_net(net) {
1567 for_each_netdev(net, dev) {
1568 if (dev == last)
RongQing.Li8f891482011-11-30 23:43:07 -05001569 goto outroll;
Herbert Xufcc5a032007-07-30 17:03:38 -07001570
Eric W. Biederman881d9662007-09-17 11:56:21 -07001571 if (dev->flags & IFF_UP) {
Jiri Pirko351638e2013-05-28 01:30:21 +00001572 call_netdevice_notifier(nb, NETDEV_GOING_DOWN,
1573 dev);
1574 call_netdevice_notifier(nb, NETDEV_DOWN, dev);
Eric W. Biederman881d9662007-09-17 11:56:21 -07001575 }
Jiri Pirko351638e2013-05-28 01:30:21 +00001576 call_netdevice_notifier(nb, NETDEV_UNREGISTER, dev);
Herbert Xufcc5a032007-07-30 17:03:38 -07001577 }
Herbert Xufcc5a032007-07-30 17:03:38 -07001578 }
Pavel Emelyanovc67625a2007-11-14 15:53:16 -08001579
RongQing.Li8f891482011-11-30 23:43:07 -05001580outroll:
Pavel Emelyanovc67625a2007-11-14 15:53:16 -08001581 raw_notifier_chain_unregister(&netdev_chain, nb);
Herbert Xufcc5a032007-07-30 17:03:38 -07001582 goto unlock;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001583}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001584EXPORT_SYMBOL(register_netdevice_notifier);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001585
1586/**
1587 * unregister_netdevice_notifier - unregister a network notifier block
1588 * @nb: notifier
1589 *
1590 * Unregister a notifier previously registered by
1591 * register_netdevice_notifier(). The notifier is unlinked into the
1592 * kernel structures and may then be reused. A negative errno code
1593 * is returned on a failure.
Eric W. Biederman7d3d43d2012-04-06 15:33:35 +00001594 *
1595 * After unregistering unregister and down device events are synthesized
1596 * for all devices on the device list to the removed notifier to remove
1597 * the need for special case cleanup code.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001598 */
1599
1600int unregister_netdevice_notifier(struct notifier_block *nb)
1601{
Eric W. Biederman7d3d43d2012-04-06 15:33:35 +00001602 struct net_device *dev;
1603 struct net *net;
Herbert Xu9f514952006-03-25 01:24:25 -08001604 int err;
1605
1606 rtnl_lock();
Alan Sternf07d5b92006-05-09 15:23:03 -07001607 err = raw_notifier_chain_unregister(&netdev_chain, nb);
Eric W. Biederman7d3d43d2012-04-06 15:33:35 +00001608 if (err)
1609 goto unlock;
1610
1611 for_each_net(net) {
1612 for_each_netdev(net, dev) {
1613 if (dev->flags & IFF_UP) {
Jiri Pirko351638e2013-05-28 01:30:21 +00001614 call_netdevice_notifier(nb, NETDEV_GOING_DOWN,
1615 dev);
1616 call_netdevice_notifier(nb, NETDEV_DOWN, dev);
Eric W. Biederman7d3d43d2012-04-06 15:33:35 +00001617 }
Jiri Pirko351638e2013-05-28 01:30:21 +00001618 call_netdevice_notifier(nb, NETDEV_UNREGISTER, dev);
Eric W. Biederman7d3d43d2012-04-06 15:33:35 +00001619 }
1620 }
1621unlock:
Herbert Xu9f514952006-03-25 01:24:25 -08001622 rtnl_unlock();
1623 return err;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001624}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001625EXPORT_SYMBOL(unregister_netdevice_notifier);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001626
1627/**
Jiri Pirko351638e2013-05-28 01:30:21 +00001628 * call_netdevice_notifiers_info - call all network notifier blocks
1629 * @val: value passed unmodified to notifier function
1630 * @dev: net_device pointer passed unmodified to notifier function
1631 * @info: notifier information data
1632 *
1633 * Call all network notifier blocks. Parameters and return value
1634 * are as for raw_notifier_call_chain().
1635 */
1636
stephen hemminger1d143d92013-12-29 14:01:29 -08001637static int call_netdevice_notifiers_info(unsigned long val,
1638 struct net_device *dev,
1639 struct netdev_notifier_info *info)
Jiri Pirko351638e2013-05-28 01:30:21 +00001640{
1641 ASSERT_RTNL();
1642 netdev_notifier_info_init(info, dev);
1643 return raw_notifier_call_chain(&netdev_chain, val, info);
1644}
Jiri Pirko351638e2013-05-28 01:30:21 +00001645
1646/**
Linus Torvalds1da177e2005-04-16 15:20:36 -07001647 * call_netdevice_notifiers - call all network notifier blocks
1648 * @val: value passed unmodified to notifier function
Randy Dunlapc4ea43c2007-10-12 21:17:49 -07001649 * @dev: net_device pointer passed unmodified to notifier function
Linus Torvalds1da177e2005-04-16 15:20:36 -07001650 *
1651 * Call all network notifier blocks. Parameters and return value
Alan Sternf07d5b92006-05-09 15:23:03 -07001652 * are as for raw_notifier_call_chain().
Linus Torvalds1da177e2005-04-16 15:20:36 -07001653 */
1654
Eric W. Biedermanad7379d2007-09-16 15:33:32 -07001655int call_netdevice_notifiers(unsigned long val, struct net_device *dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001656{
Jiri Pirko351638e2013-05-28 01:30:21 +00001657 struct netdev_notifier_info info;
1658
1659 return call_netdevice_notifiers_info(val, dev, &info);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001660}
stephen hemmingeredf947f2011-03-24 13:24:01 +00001661EXPORT_SYMBOL(call_netdevice_notifiers);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001662
Pablo Neira1cf519002015-05-13 18:19:37 +02001663#ifdef CONFIG_NET_INGRESS
Daniel Borkmann45771392015-04-10 23:07:54 +02001664static struct static_key ingress_needed __read_mostly;
1665
1666void net_inc_ingress_queue(void)
1667{
1668 static_key_slow_inc(&ingress_needed);
1669}
1670EXPORT_SYMBOL_GPL(net_inc_ingress_queue);
1671
1672void net_dec_ingress_queue(void)
1673{
1674 static_key_slow_dec(&ingress_needed);
1675}
1676EXPORT_SYMBOL_GPL(net_dec_ingress_queue);
1677#endif
1678
Daniel Borkmann1f211a12016-01-07 22:29:47 +01001679#ifdef CONFIG_NET_EGRESS
1680static struct static_key egress_needed __read_mostly;
1681
1682void net_inc_egress_queue(void)
1683{
1684 static_key_slow_inc(&egress_needed);
1685}
1686EXPORT_SYMBOL_GPL(net_inc_egress_queue);
1687
1688void net_dec_egress_queue(void)
1689{
1690 static_key_slow_dec(&egress_needed);
1691}
1692EXPORT_SYMBOL_GPL(net_dec_egress_queue);
1693#endif
1694
Ingo Molnarc5905af2012-02-24 08:31:31 +01001695static struct static_key netstamp_needed __read_mostly;
Eric Dumazetb90e5792011-11-28 11:16:50 +00001696#ifdef HAVE_JUMP_LABEL
Ingo Molnarc5905af2012-02-24 08:31:31 +01001697/* We are not allowed to call static_key_slow_dec() from irq context
Eric Dumazetb90e5792011-11-28 11:16:50 +00001698 * If net_disable_timestamp() is called from irq context, defer the
Ingo Molnarc5905af2012-02-24 08:31:31 +01001699 * static_key_slow_dec() calls.
Eric Dumazetb90e5792011-11-28 11:16:50 +00001700 */
1701static atomic_t netstamp_needed_deferred;
1702#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07001703
1704void net_enable_timestamp(void)
1705{
Eric Dumazetb90e5792011-11-28 11:16:50 +00001706#ifdef HAVE_JUMP_LABEL
1707 int deferred = atomic_xchg(&netstamp_needed_deferred, 0);
1708
1709 if (deferred) {
1710 while (--deferred)
Ingo Molnarc5905af2012-02-24 08:31:31 +01001711 static_key_slow_dec(&netstamp_needed);
Eric Dumazetb90e5792011-11-28 11:16:50 +00001712 return;
1713 }
1714#endif
Ingo Molnarc5905af2012-02-24 08:31:31 +01001715 static_key_slow_inc(&netstamp_needed);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001716}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001717EXPORT_SYMBOL(net_enable_timestamp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001718
1719void net_disable_timestamp(void)
1720{
Eric Dumazetb90e5792011-11-28 11:16:50 +00001721#ifdef HAVE_JUMP_LABEL
1722 if (in_interrupt()) {
1723 atomic_inc(&netstamp_needed_deferred);
1724 return;
1725 }
1726#endif
Ingo Molnarc5905af2012-02-24 08:31:31 +01001727 static_key_slow_dec(&netstamp_needed);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001728}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001729EXPORT_SYMBOL(net_disable_timestamp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001730
Eric Dumazet3b098e22010-05-15 23:57:10 -07001731static inline void net_timestamp_set(struct sk_buff *skb)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001732{
Eric Dumazet588f0332011-11-15 04:12:55 +00001733 skb->tstamp.tv64 = 0;
Ingo Molnarc5905af2012-02-24 08:31:31 +01001734 if (static_key_false(&netstamp_needed))
Patrick McHardya61bbcf2005-08-14 17:24:31 -07001735 __net_timestamp(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001736}
1737
Eric Dumazet588f0332011-11-15 04:12:55 +00001738#define net_timestamp_check(COND, SKB) \
Ingo Molnarc5905af2012-02-24 08:31:31 +01001739 if (static_key_false(&netstamp_needed)) { \
Eric Dumazet588f0332011-11-15 04:12:55 +00001740 if ((COND) && !(SKB)->tstamp.tv64) \
1741 __net_timestamp(SKB); \
1742 } \
Eric Dumazet3b098e22010-05-15 23:57:10 -07001743
Vlad Yasevich1ee481f2014-03-27 17:32:29 -04001744bool is_skb_forwardable(struct net_device *dev, struct sk_buff *skb)
Daniel Lezcano79b569f2011-03-30 02:42:17 -07001745{
1746 unsigned int len;
1747
1748 if (!(dev->flags & IFF_UP))
1749 return false;
1750
1751 len = dev->mtu + dev->hard_header_len + VLAN_HLEN;
1752 if (skb->len <= len)
1753 return true;
1754
1755 /* if TSO is enabled, we don't care about the length as the packet
1756 * could be forwarded without being segmented before
1757 */
1758 if (skb_is_gso(skb))
1759 return true;
1760
1761 return false;
1762}
Vlad Yasevich1ee481f2014-03-27 17:32:29 -04001763EXPORT_SYMBOL_GPL(is_skb_forwardable);
Daniel Lezcano79b569f2011-03-30 02:42:17 -07001764
Herbert Xua0265d22014-04-17 13:45:03 +08001765int __dev_forward_skb(struct net_device *dev, struct sk_buff *skb)
1766{
Willem de Bruijnbbbf2df2015-06-08 11:53:08 -04001767 if (skb_orphan_frags(skb, GFP_ATOMIC) ||
1768 unlikely(!is_skb_forwardable(dev, skb))) {
Herbert Xua0265d22014-04-17 13:45:03 +08001769 atomic_long_inc(&dev->rx_dropped);
1770 kfree_skb(skb);
1771 return NET_RX_DROP;
1772 }
1773
1774 skb_scrub_packet(skb, true);
WANG Cong08b4b8e2015-03-20 14:29:09 -07001775 skb->priority = 0;
Herbert Xua0265d22014-04-17 13:45:03 +08001776 skb->protocol = eth_type_trans(skb, dev);
Jay Vosburgh2c26d342014-12-19 15:32:00 -08001777 skb_postpull_rcsum(skb, eth_hdr(skb), ETH_HLEN);
Herbert Xua0265d22014-04-17 13:45:03 +08001778
1779 return 0;
1780}
1781EXPORT_SYMBOL_GPL(__dev_forward_skb);
1782
Arnd Bergmann44540962009-11-26 06:07:08 +00001783/**
1784 * dev_forward_skb - loopback an skb to another netif
1785 *
1786 * @dev: destination network device
1787 * @skb: buffer to forward
1788 *
1789 * return values:
1790 * NET_RX_SUCCESS (no congestion)
Eric Dumazet6ec82562010-05-06 00:53:53 -07001791 * NET_RX_DROP (packet was dropped, but freed)
Arnd Bergmann44540962009-11-26 06:07:08 +00001792 *
1793 * dev_forward_skb can be used for injecting an skb from the
1794 * start_xmit function of one device into the receive queue
1795 * of another device.
1796 *
1797 * The receiving device may be in another namespace, so
1798 * we have to clear all information in the skb that could
1799 * impact namespace isolation.
1800 */
1801int dev_forward_skb(struct net_device *dev, struct sk_buff *skb)
1802{
Herbert Xua0265d22014-04-17 13:45:03 +08001803 return __dev_forward_skb(dev, skb) ?: netif_rx_internal(skb);
Arnd Bergmann44540962009-11-26 06:07:08 +00001804}
1805EXPORT_SYMBOL_GPL(dev_forward_skb);
1806
Changli Gao71d9dec2010-12-15 19:57:25 +00001807static inline int deliver_skb(struct sk_buff *skb,
1808 struct packet_type *pt_prev,
1809 struct net_device *orig_dev)
1810{
Michael S. Tsirkin1080e512012-07-20 09:23:17 +00001811 if (unlikely(skb_orphan_frags(skb, GFP_ATOMIC)))
1812 return -ENOMEM;
Changli Gao71d9dec2010-12-15 19:57:25 +00001813 atomic_inc(&skb->users);
1814 return pt_prev->func(skb, skb->dev, pt_prev, orig_dev);
1815}
1816
Salam Noureddine7866a622015-01-27 11:35:48 -08001817static inline void deliver_ptype_list_skb(struct sk_buff *skb,
1818 struct packet_type **pt,
Jiri Pirkofbcb2172015-03-30 16:56:01 +02001819 struct net_device *orig_dev,
1820 __be16 type,
Salam Noureddine7866a622015-01-27 11:35:48 -08001821 struct list_head *ptype_list)
1822{
1823 struct packet_type *ptype, *pt_prev = *pt;
1824
1825 list_for_each_entry_rcu(ptype, ptype_list, list) {
1826 if (ptype->type != type)
1827 continue;
1828 if (pt_prev)
Jiri Pirkofbcb2172015-03-30 16:56:01 +02001829 deliver_skb(skb, pt_prev, orig_dev);
Salam Noureddine7866a622015-01-27 11:35:48 -08001830 pt_prev = ptype;
1831 }
1832 *pt = pt_prev;
1833}
1834
Eric Leblondc0de08d2012-08-16 22:02:58 +00001835static inline bool skb_loop_sk(struct packet_type *ptype, struct sk_buff *skb)
1836{
Eric Leblonda3d744e2012-11-06 02:10:10 +00001837 if (!ptype->af_packet_priv || !skb->sk)
Eric Leblondc0de08d2012-08-16 22:02:58 +00001838 return false;
1839
1840 if (ptype->id_match)
1841 return ptype->id_match(ptype, skb->sk);
1842 else if ((struct sock *)ptype->af_packet_priv == skb->sk)
1843 return true;
1844
1845 return false;
1846}
1847
Linus Torvalds1da177e2005-04-16 15:20:36 -07001848/*
1849 * Support routine. Sends outgoing frames to any network
1850 * taps currently in use.
1851 */
1852
Herbert Xuf6a78bf2006-06-22 02:57:17 -07001853static void dev_queue_xmit_nit(struct sk_buff *skb, struct net_device *dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001854{
1855 struct packet_type *ptype;
Changli Gao71d9dec2010-12-15 19:57:25 +00001856 struct sk_buff *skb2 = NULL;
1857 struct packet_type *pt_prev = NULL;
Salam Noureddine7866a622015-01-27 11:35:48 -08001858 struct list_head *ptype_list = &ptype_all;
Patrick McHardya61bbcf2005-08-14 17:24:31 -07001859
Linus Torvalds1da177e2005-04-16 15:20:36 -07001860 rcu_read_lock();
Salam Noureddine7866a622015-01-27 11:35:48 -08001861again:
1862 list_for_each_entry_rcu(ptype, ptype_list, list) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001863 /* Never send packets back to the socket
1864 * they originated from - MvS (miquels@drinkel.ow.org)
1865 */
Salam Noureddine7866a622015-01-27 11:35:48 -08001866 if (skb_loop_sk(ptype, skb))
1867 continue;
Changli Gao71d9dec2010-12-15 19:57:25 +00001868
Salam Noureddine7866a622015-01-27 11:35:48 -08001869 if (pt_prev) {
1870 deliver_skb(skb2, pt_prev, skb->dev);
Changli Gao71d9dec2010-12-15 19:57:25 +00001871 pt_prev = ptype;
Salam Noureddine7866a622015-01-27 11:35:48 -08001872 continue;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001873 }
Salam Noureddine7866a622015-01-27 11:35:48 -08001874
1875 /* need to clone skb, done only once */
1876 skb2 = skb_clone(skb, GFP_ATOMIC);
1877 if (!skb2)
1878 goto out_unlock;
1879
1880 net_timestamp_set(skb2);
1881
1882 /* skb->nh should be correctly
1883 * set by sender, so that the second statement is
1884 * just protection against buggy protocols.
1885 */
1886 skb_reset_mac_header(skb2);
1887
1888 if (skb_network_header(skb2) < skb2->data ||
1889 skb_network_header(skb2) > skb_tail_pointer(skb2)) {
1890 net_crit_ratelimited("protocol %04x is buggy, dev %s\n",
1891 ntohs(skb2->protocol),
1892 dev->name);
1893 skb_reset_network_header(skb2);
1894 }
1895
1896 skb2->transport_header = skb2->network_header;
1897 skb2->pkt_type = PACKET_OUTGOING;
1898 pt_prev = ptype;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001899 }
Salam Noureddine7866a622015-01-27 11:35:48 -08001900
1901 if (ptype_list == &ptype_all) {
1902 ptype_list = &dev->ptype_all;
1903 goto again;
1904 }
1905out_unlock:
Changli Gao71d9dec2010-12-15 19:57:25 +00001906 if (pt_prev)
1907 pt_prev->func(skb2, skb->dev, pt_prev, skb->dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001908 rcu_read_unlock();
1909}
1910
Ben Hutchings2c530402012-07-10 10:55:09 +00001911/**
1912 * netif_setup_tc - Handle tc mappings on real_num_tx_queues change
John Fastabend4f57c082011-01-17 08:06:04 +00001913 * @dev: Network device
1914 * @txq: number of queues available
1915 *
1916 * If real_num_tx_queues is changed the tc mappings may no longer be
1917 * valid. To resolve this verify the tc mapping remains valid and if
1918 * not NULL the mapping. With no priorities mapping to this
1919 * offset/count pair it will no longer be used. In the worst case TC0
1920 * is invalid nothing can be done so disable priority mappings. If is
1921 * expected that drivers will fix this mapping if they can before
1922 * calling netif_set_real_num_tx_queues.
1923 */
Eric Dumazetbb134d22011-01-20 19:18:08 +00001924static void netif_setup_tc(struct net_device *dev, unsigned int txq)
John Fastabend4f57c082011-01-17 08:06:04 +00001925{
1926 int i;
1927 struct netdev_tc_txq *tc = &dev->tc_to_txq[0];
1928
1929 /* If TC0 is invalidated disable TC mapping */
1930 if (tc->offset + tc->count > txq) {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00001931 pr_warn("Number of in use tx queues changed invalidating tc mappings. Priority traffic classification disabled!\n");
John Fastabend4f57c082011-01-17 08:06:04 +00001932 dev->num_tc = 0;
1933 return;
1934 }
1935
1936 /* Invalidated prio to tc mappings set to TC0 */
1937 for (i = 1; i < TC_BITMASK + 1; i++) {
1938 int q = netdev_get_prio_tc_map(dev, i);
1939
1940 tc = &dev->tc_to_txq[q];
1941 if (tc->offset + tc->count > txq) {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00001942 pr_warn("Number of in use tx queues changed. Priority %i to tc mapping %i is no longer valid. Setting map to 0\n",
1943 i, q);
John Fastabend4f57c082011-01-17 08:06:04 +00001944 netdev_set_prio_tc_map(dev, i, 0);
1945 }
1946 }
1947}
1948
Alexander Duyck537c00d2013-01-10 08:57:02 +00001949#ifdef CONFIG_XPS
1950static DEFINE_MUTEX(xps_map_mutex);
1951#define xmap_dereference(P) \
1952 rcu_dereference_protected((P), lockdep_is_held(&xps_map_mutex))
1953
Alexander Duyck10cdc3f2013-01-10 08:57:17 +00001954static struct xps_map *remove_xps_queue(struct xps_dev_maps *dev_maps,
1955 int cpu, u16 index)
1956{
1957 struct xps_map *map = NULL;
1958 int pos;
1959
1960 if (dev_maps)
1961 map = xmap_dereference(dev_maps->cpu_map[cpu]);
1962
1963 for (pos = 0; map && pos < map->len; pos++) {
1964 if (map->queues[pos] == index) {
1965 if (map->len > 1) {
1966 map->queues[pos] = map->queues[--map->len];
1967 } else {
1968 RCU_INIT_POINTER(dev_maps->cpu_map[cpu], NULL);
1969 kfree_rcu(map, rcu);
1970 map = NULL;
1971 }
1972 break;
1973 }
1974 }
1975
1976 return map;
1977}
1978
Alexander Duyck024e9672013-01-10 08:57:46 +00001979static void netif_reset_xps_queues_gt(struct net_device *dev, u16 index)
Alexander Duyck537c00d2013-01-10 08:57:02 +00001980{
1981 struct xps_dev_maps *dev_maps;
Alexander Duyck024e9672013-01-10 08:57:46 +00001982 int cpu, i;
Alexander Duyck10cdc3f2013-01-10 08:57:17 +00001983 bool active = false;
Alexander Duyck537c00d2013-01-10 08:57:02 +00001984
1985 mutex_lock(&xps_map_mutex);
1986 dev_maps = xmap_dereference(dev->xps_maps);
1987
1988 if (!dev_maps)
1989 goto out_no_maps;
1990
Alexander Duyck10cdc3f2013-01-10 08:57:17 +00001991 for_each_possible_cpu(cpu) {
Alexander Duyck024e9672013-01-10 08:57:46 +00001992 for (i = index; i < dev->num_tx_queues; i++) {
1993 if (!remove_xps_queue(dev_maps, cpu, i))
1994 break;
1995 }
1996 if (i == dev->num_tx_queues)
Alexander Duyck10cdc3f2013-01-10 08:57:17 +00001997 active = true;
Alexander Duyck537c00d2013-01-10 08:57:02 +00001998 }
1999
Alexander Duyck10cdc3f2013-01-10 08:57:17 +00002000 if (!active) {
Alexander Duyck537c00d2013-01-10 08:57:02 +00002001 RCU_INIT_POINTER(dev->xps_maps, NULL);
2002 kfree_rcu(dev_maps, rcu);
2003 }
2004
Alexander Duyck024e9672013-01-10 08:57:46 +00002005 for (i = index; i < dev->num_tx_queues; i++)
2006 netdev_queue_numa_node_write(netdev_get_tx_queue(dev, i),
2007 NUMA_NO_NODE);
2008
Alexander Duyck537c00d2013-01-10 08:57:02 +00002009out_no_maps:
2010 mutex_unlock(&xps_map_mutex);
2011}
2012
Alexander Duyck01c5f862013-01-10 08:57:35 +00002013static struct xps_map *expand_xps_map(struct xps_map *map,
2014 int cpu, u16 index)
2015{
2016 struct xps_map *new_map;
2017 int alloc_len = XPS_MIN_MAP_ALLOC;
2018 int i, pos;
2019
2020 for (pos = 0; map && pos < map->len; pos++) {
2021 if (map->queues[pos] != index)
2022 continue;
2023 return map;
2024 }
2025
2026 /* Need to add queue to this CPU's existing map */
2027 if (map) {
2028 if (pos < map->alloc_len)
2029 return map;
2030
2031 alloc_len = map->alloc_len * 2;
2032 }
2033
2034 /* Need to allocate new map to store queue on this CPU's map */
2035 new_map = kzalloc_node(XPS_MAP_SIZE(alloc_len), GFP_KERNEL,
2036 cpu_to_node(cpu));
2037 if (!new_map)
2038 return NULL;
2039
2040 for (i = 0; i < pos; i++)
2041 new_map->queues[i] = map->queues[i];
2042 new_map->alloc_len = alloc_len;
2043 new_map->len = pos;
2044
2045 return new_map;
2046}
2047
Michael S. Tsirkin35735402013-10-02 09:14:06 +03002048int netif_set_xps_queue(struct net_device *dev, const struct cpumask *mask,
2049 u16 index)
Alexander Duyck537c00d2013-01-10 08:57:02 +00002050{
Alexander Duyck01c5f862013-01-10 08:57:35 +00002051 struct xps_dev_maps *dev_maps, *new_dev_maps = NULL;
Alexander Duyck537c00d2013-01-10 08:57:02 +00002052 struct xps_map *map, *new_map;
Alexander Duyck537c00d2013-01-10 08:57:02 +00002053 int maps_sz = max_t(unsigned int, XPS_DEV_MAPS_SIZE, L1_CACHE_BYTES);
Alexander Duyck01c5f862013-01-10 08:57:35 +00002054 int cpu, numa_node_id = -2;
2055 bool active = false;
Alexander Duyck537c00d2013-01-10 08:57:02 +00002056
2057 mutex_lock(&xps_map_mutex);
2058
2059 dev_maps = xmap_dereference(dev->xps_maps);
2060
Alexander Duyck01c5f862013-01-10 08:57:35 +00002061 /* allocate memory for queue storage */
2062 for_each_online_cpu(cpu) {
2063 if (!cpumask_test_cpu(cpu, mask))
2064 continue;
Alexander Duyck537c00d2013-01-10 08:57:02 +00002065
Alexander Duyck01c5f862013-01-10 08:57:35 +00002066 if (!new_dev_maps)
2067 new_dev_maps = kzalloc(maps_sz, GFP_KERNEL);
Alexander Duyck2bb60cb2013-02-22 06:38:44 +00002068 if (!new_dev_maps) {
2069 mutex_unlock(&xps_map_mutex);
Alexander Duyck01c5f862013-01-10 08:57:35 +00002070 return -ENOMEM;
Alexander Duyck2bb60cb2013-02-22 06:38:44 +00002071 }
Alexander Duyck01c5f862013-01-10 08:57:35 +00002072
2073 map = dev_maps ? xmap_dereference(dev_maps->cpu_map[cpu]) :
2074 NULL;
2075
2076 map = expand_xps_map(map, cpu, index);
2077 if (!map)
2078 goto error;
2079
2080 RCU_INIT_POINTER(new_dev_maps->cpu_map[cpu], map);
2081 }
2082
2083 if (!new_dev_maps)
2084 goto out_no_new_maps;
2085
2086 for_each_possible_cpu(cpu) {
2087 if (cpumask_test_cpu(cpu, mask) && cpu_online(cpu)) {
2088 /* add queue to CPU maps */
2089 int pos = 0;
2090
2091 map = xmap_dereference(new_dev_maps->cpu_map[cpu]);
2092 while ((pos < map->len) && (map->queues[pos] != index))
2093 pos++;
2094
2095 if (pos == map->len)
2096 map->queues[map->len++] = index;
Alexander Duyck537c00d2013-01-10 08:57:02 +00002097#ifdef CONFIG_NUMA
Alexander Duyck537c00d2013-01-10 08:57:02 +00002098 if (numa_node_id == -2)
2099 numa_node_id = cpu_to_node(cpu);
2100 else if (numa_node_id != cpu_to_node(cpu))
2101 numa_node_id = -1;
Alexander Duyck537c00d2013-01-10 08:57:02 +00002102#endif
Alexander Duyck01c5f862013-01-10 08:57:35 +00002103 } else if (dev_maps) {
2104 /* fill in the new device map from the old device map */
2105 map = xmap_dereference(dev_maps->cpu_map[cpu]);
2106 RCU_INIT_POINTER(new_dev_maps->cpu_map[cpu], map);
Alexander Duyck537c00d2013-01-10 08:57:02 +00002107 }
Alexander Duyck01c5f862013-01-10 08:57:35 +00002108
Alexander Duyck537c00d2013-01-10 08:57:02 +00002109 }
2110
Alexander Duyck01c5f862013-01-10 08:57:35 +00002111 rcu_assign_pointer(dev->xps_maps, new_dev_maps);
2112
Alexander Duyck537c00d2013-01-10 08:57:02 +00002113 /* Cleanup old maps */
Alexander Duyck01c5f862013-01-10 08:57:35 +00002114 if (dev_maps) {
2115 for_each_possible_cpu(cpu) {
2116 new_map = xmap_dereference(new_dev_maps->cpu_map[cpu]);
2117 map = xmap_dereference(dev_maps->cpu_map[cpu]);
2118 if (map && map != new_map)
2119 kfree_rcu(map, rcu);
2120 }
Alexander Duyck537c00d2013-01-10 08:57:02 +00002121
Alexander Duyck537c00d2013-01-10 08:57:02 +00002122 kfree_rcu(dev_maps, rcu);
Alexander Duyck01c5f862013-01-10 08:57:35 +00002123 }
Alexander Duyck537c00d2013-01-10 08:57:02 +00002124
Alexander Duyck01c5f862013-01-10 08:57:35 +00002125 dev_maps = new_dev_maps;
2126 active = true;
2127
2128out_no_new_maps:
2129 /* update Tx queue numa node */
Alexander Duyck537c00d2013-01-10 08:57:02 +00002130 netdev_queue_numa_node_write(netdev_get_tx_queue(dev, index),
2131 (numa_node_id >= 0) ? numa_node_id :
2132 NUMA_NO_NODE);
2133
Alexander Duyck01c5f862013-01-10 08:57:35 +00002134 if (!dev_maps)
2135 goto out_no_maps;
2136
2137 /* removes queue from unused CPUs */
2138 for_each_possible_cpu(cpu) {
2139 if (cpumask_test_cpu(cpu, mask) && cpu_online(cpu))
2140 continue;
2141
2142 if (remove_xps_queue(dev_maps, cpu, index))
2143 active = true;
2144 }
2145
2146 /* free map if not active */
2147 if (!active) {
2148 RCU_INIT_POINTER(dev->xps_maps, NULL);
2149 kfree_rcu(dev_maps, rcu);
2150 }
2151
2152out_no_maps:
Alexander Duyck537c00d2013-01-10 08:57:02 +00002153 mutex_unlock(&xps_map_mutex);
2154
2155 return 0;
2156error:
Alexander Duyck01c5f862013-01-10 08:57:35 +00002157 /* remove any maps that we added */
2158 for_each_possible_cpu(cpu) {
2159 new_map = xmap_dereference(new_dev_maps->cpu_map[cpu]);
2160 map = dev_maps ? xmap_dereference(dev_maps->cpu_map[cpu]) :
2161 NULL;
2162 if (new_map && new_map != map)
2163 kfree(new_map);
2164 }
2165
Alexander Duyck537c00d2013-01-10 08:57:02 +00002166 mutex_unlock(&xps_map_mutex);
2167
Alexander Duyck537c00d2013-01-10 08:57:02 +00002168 kfree(new_dev_maps);
2169 return -ENOMEM;
2170}
2171EXPORT_SYMBOL(netif_set_xps_queue);
2172
2173#endif
John Fastabendf0796d52010-07-01 13:21:57 +00002174/*
2175 * Routine to help set real_num_tx_queues. To avoid skbs mapped to queues
2176 * greater then real_num_tx_queues stale skbs on the qdisc must be flushed.
2177 */
Tom Herberte6484932010-10-18 18:04:39 +00002178int netif_set_real_num_tx_queues(struct net_device *dev, unsigned int txq)
John Fastabendf0796d52010-07-01 13:21:57 +00002179{
Tom Herbert1d24eb42010-11-21 13:17:27 +00002180 int rc;
2181
Tom Herberte6484932010-10-18 18:04:39 +00002182 if (txq < 1 || txq > dev->num_tx_queues)
2183 return -EINVAL;
John Fastabendf0796d52010-07-01 13:21:57 +00002184
Ben Hutchings5c565802011-02-15 19:39:21 +00002185 if (dev->reg_state == NETREG_REGISTERED ||
2186 dev->reg_state == NETREG_UNREGISTERING) {
Tom Herberte6484932010-10-18 18:04:39 +00002187 ASSERT_RTNL();
2188
Tom Herbert1d24eb42010-11-21 13:17:27 +00002189 rc = netdev_queue_update_kobjects(dev, dev->real_num_tx_queues,
2190 txq);
Tom Herbertbf264142010-11-26 08:36:09 +00002191 if (rc)
2192 return rc;
2193
John Fastabend4f57c082011-01-17 08:06:04 +00002194 if (dev->num_tc)
2195 netif_setup_tc(dev, txq);
2196
Alexander Duyck024e9672013-01-10 08:57:46 +00002197 if (txq < dev->real_num_tx_queues) {
Tom Herberte6484932010-10-18 18:04:39 +00002198 qdisc_reset_all_tx_gt(dev, txq);
Alexander Duyck024e9672013-01-10 08:57:46 +00002199#ifdef CONFIG_XPS
2200 netif_reset_xps_queues_gt(dev, txq);
2201#endif
2202 }
John Fastabendf0796d52010-07-01 13:21:57 +00002203 }
Tom Herberte6484932010-10-18 18:04:39 +00002204
2205 dev->real_num_tx_queues = txq;
2206 return 0;
John Fastabendf0796d52010-07-01 13:21:57 +00002207}
2208EXPORT_SYMBOL(netif_set_real_num_tx_queues);
Denis Vlasenko56079432006-03-29 15:57:29 -08002209
Michael Daltona953be52014-01-16 22:23:28 -08002210#ifdef CONFIG_SYSFS
Ben Hutchings62fe0b42010-09-27 08:24:33 +00002211/**
2212 * netif_set_real_num_rx_queues - set actual number of RX queues used
2213 * @dev: Network device
2214 * @rxq: Actual number of RX queues
2215 *
2216 * This must be called either with the rtnl_lock held or before
2217 * registration of the net device. Returns 0 on success, or a
Ben Hutchings4e7f7952010-10-08 10:33:39 -07002218 * negative error code. If called before registration, it always
2219 * succeeds.
Ben Hutchings62fe0b42010-09-27 08:24:33 +00002220 */
2221int netif_set_real_num_rx_queues(struct net_device *dev, unsigned int rxq)
2222{
2223 int rc;
2224
Tom Herbertbd25fa72010-10-18 18:00:16 +00002225 if (rxq < 1 || rxq > dev->num_rx_queues)
2226 return -EINVAL;
2227
Ben Hutchings62fe0b42010-09-27 08:24:33 +00002228 if (dev->reg_state == NETREG_REGISTERED) {
2229 ASSERT_RTNL();
2230
Ben Hutchings62fe0b42010-09-27 08:24:33 +00002231 rc = net_rx_queue_update_kobjects(dev, dev->real_num_rx_queues,
2232 rxq);
2233 if (rc)
2234 return rc;
Ben Hutchings62fe0b42010-09-27 08:24:33 +00002235 }
2236
2237 dev->real_num_rx_queues = rxq;
2238 return 0;
2239}
2240EXPORT_SYMBOL(netif_set_real_num_rx_queues);
2241#endif
2242
Ben Hutchings2c530402012-07-10 10:55:09 +00002243/**
2244 * netif_get_num_default_rss_queues - default number of RSS queues
Yuval Mintz16917b82012-07-01 03:18:50 +00002245 *
2246 * This routine should set an upper limit on the number of RSS queues
2247 * used by default by multiqueue devices.
2248 */
Ben Hutchingsa55b1382012-07-10 10:54:38 +00002249int netif_get_num_default_rss_queues(void)
Yuval Mintz16917b82012-07-01 03:18:50 +00002250{
2251 return min_t(int, DEFAULT_MAX_NUM_RSS_QUEUES, num_online_cpus());
2252}
2253EXPORT_SYMBOL(netif_get_num_default_rss_queues);
2254
Jarek Poplawskidef82a12008-08-17 21:54:43 -07002255static inline void __netif_reschedule(struct Qdisc *q)
2256{
2257 struct softnet_data *sd;
2258 unsigned long flags;
2259
2260 local_irq_save(flags);
Christoph Lameter903ceff2014-08-17 12:30:35 -05002261 sd = this_cpu_ptr(&softnet_data);
Changli Gaoa9cbd582010-04-26 23:06:24 +00002262 q->next_sched = NULL;
2263 *sd->output_queue_tailp = q;
2264 sd->output_queue_tailp = &q->next_sched;
Jarek Poplawskidef82a12008-08-17 21:54:43 -07002265 raise_softirq_irqoff(NET_TX_SOFTIRQ);
2266 local_irq_restore(flags);
2267}
2268
David S. Miller37437bb2008-07-16 02:15:04 -07002269void __netif_schedule(struct Qdisc *q)
Denis Vlasenko56079432006-03-29 15:57:29 -08002270{
Jarek Poplawskidef82a12008-08-17 21:54:43 -07002271 if (!test_and_set_bit(__QDISC_STATE_SCHED, &q->state))
2272 __netif_reschedule(q);
Denis Vlasenko56079432006-03-29 15:57:29 -08002273}
2274EXPORT_SYMBOL(__netif_schedule);
2275
Eric Dumazete6247022013-12-05 04:45:08 -08002276struct dev_kfree_skb_cb {
2277 enum skb_free_reason reason;
2278};
2279
2280static struct dev_kfree_skb_cb *get_kfree_skb_cb(const struct sk_buff *skb)
Denis Vlasenko56079432006-03-29 15:57:29 -08002281{
Eric Dumazete6247022013-12-05 04:45:08 -08002282 return (struct dev_kfree_skb_cb *)skb->cb;
Denis Vlasenko56079432006-03-29 15:57:29 -08002283}
Denis Vlasenko56079432006-03-29 15:57:29 -08002284
John Fastabend46e5da42014-09-12 20:04:52 -07002285void netif_schedule_queue(struct netdev_queue *txq)
2286{
2287 rcu_read_lock();
2288 if (!(txq->state & QUEUE_STATE_ANY_XOFF)) {
2289 struct Qdisc *q = rcu_dereference(txq->qdisc);
2290
2291 __netif_schedule(q);
2292 }
2293 rcu_read_unlock();
2294}
2295EXPORT_SYMBOL(netif_schedule_queue);
2296
2297/**
2298 * netif_wake_subqueue - allow sending packets on subqueue
2299 * @dev: network device
2300 * @queue_index: sub queue index
2301 *
2302 * Resume individual transmit queue of a device with multiple transmit queues.
2303 */
2304void netif_wake_subqueue(struct net_device *dev, u16 queue_index)
2305{
2306 struct netdev_queue *txq = netdev_get_tx_queue(dev, queue_index);
2307
2308 if (test_and_clear_bit(__QUEUE_STATE_DRV_XOFF, &txq->state)) {
2309 struct Qdisc *q;
2310
2311 rcu_read_lock();
2312 q = rcu_dereference(txq->qdisc);
2313 __netif_schedule(q);
2314 rcu_read_unlock();
2315 }
2316}
2317EXPORT_SYMBOL(netif_wake_subqueue);
2318
2319void netif_tx_wake_queue(struct netdev_queue *dev_queue)
2320{
2321 if (test_and_clear_bit(__QUEUE_STATE_DRV_XOFF, &dev_queue->state)) {
2322 struct Qdisc *q;
2323
2324 rcu_read_lock();
2325 q = rcu_dereference(dev_queue->qdisc);
2326 __netif_schedule(q);
2327 rcu_read_unlock();
2328 }
2329}
2330EXPORT_SYMBOL(netif_tx_wake_queue);
2331
Eric Dumazete6247022013-12-05 04:45:08 -08002332void __dev_kfree_skb_irq(struct sk_buff *skb, enum skb_free_reason reason)
2333{
2334 unsigned long flags;
2335
2336 if (likely(atomic_read(&skb->users) == 1)) {
2337 smp_rmb();
2338 atomic_set(&skb->users, 0);
2339 } else if (likely(!atomic_dec_and_test(&skb->users))) {
2340 return;
2341 }
2342 get_kfree_skb_cb(skb)->reason = reason;
2343 local_irq_save(flags);
2344 skb->next = __this_cpu_read(softnet_data.completion_queue);
2345 __this_cpu_write(softnet_data.completion_queue, skb);
2346 raise_softirq_irqoff(NET_TX_SOFTIRQ);
2347 local_irq_restore(flags);
2348}
2349EXPORT_SYMBOL(__dev_kfree_skb_irq);
2350
2351void __dev_kfree_skb_any(struct sk_buff *skb, enum skb_free_reason reason)
Denis Vlasenko56079432006-03-29 15:57:29 -08002352{
2353 if (in_irq() || irqs_disabled())
Eric Dumazete6247022013-12-05 04:45:08 -08002354 __dev_kfree_skb_irq(skb, reason);
Denis Vlasenko56079432006-03-29 15:57:29 -08002355 else
2356 dev_kfree_skb(skb);
2357}
Eric Dumazete6247022013-12-05 04:45:08 -08002358EXPORT_SYMBOL(__dev_kfree_skb_any);
Denis Vlasenko56079432006-03-29 15:57:29 -08002359
2360
Stephen Hemmingerbea33482007-10-03 16:41:36 -07002361/**
2362 * netif_device_detach - mark device as removed
2363 * @dev: network device
2364 *
2365 * Mark device as removed from system and therefore no longer available.
2366 */
Denis Vlasenko56079432006-03-29 15:57:29 -08002367void netif_device_detach(struct net_device *dev)
2368{
2369 if (test_and_clear_bit(__LINK_STATE_PRESENT, &dev->state) &&
2370 netif_running(dev)) {
Alexander Duyckd5431032009-04-08 13:15:22 +00002371 netif_tx_stop_all_queues(dev);
Denis Vlasenko56079432006-03-29 15:57:29 -08002372 }
2373}
2374EXPORT_SYMBOL(netif_device_detach);
2375
Stephen Hemmingerbea33482007-10-03 16:41:36 -07002376/**
2377 * netif_device_attach - mark device as attached
2378 * @dev: network device
2379 *
2380 * Mark device as attached from system and restart if needed.
2381 */
Denis Vlasenko56079432006-03-29 15:57:29 -08002382void netif_device_attach(struct net_device *dev)
2383{
2384 if (!test_and_set_bit(__LINK_STATE_PRESENT, &dev->state) &&
2385 netif_running(dev)) {
Alexander Duyckd5431032009-04-08 13:15:22 +00002386 netif_tx_wake_all_queues(dev);
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09002387 __netdev_watchdog_up(dev);
Denis Vlasenko56079432006-03-29 15:57:29 -08002388 }
2389}
2390EXPORT_SYMBOL(netif_device_attach);
2391
Jiri Pirko5605c762015-05-12 14:56:12 +02002392/*
2393 * Returns a Tx hash based on the given packet descriptor a Tx queues' number
2394 * to be used as a distribution range.
2395 */
2396u16 __skb_tx_hash(const struct net_device *dev, struct sk_buff *skb,
2397 unsigned int num_tx_queues)
2398{
2399 u32 hash;
2400 u16 qoffset = 0;
2401 u16 qcount = num_tx_queues;
2402
2403 if (skb_rx_queue_recorded(skb)) {
2404 hash = skb_get_rx_queue(skb);
2405 while (unlikely(hash >= num_tx_queues))
2406 hash -= num_tx_queues;
2407 return hash;
2408 }
2409
2410 if (dev->num_tc) {
2411 u8 tc = netdev_get_prio_tc_map(dev, skb->priority);
2412 qoffset = dev->tc_to_txq[tc].offset;
2413 qcount = dev->tc_to_txq[tc].count;
2414 }
2415
2416 return (u16) reciprocal_scale(skb_get_hash(skb), qcount) + qoffset;
2417}
2418EXPORT_SYMBOL(__skb_tx_hash);
2419
Ben Hutchings36c92472012-01-17 07:57:56 +00002420static void skb_warn_bad_offload(const struct sk_buff *skb)
2421{
Michał Mirosław65e9d2f2012-01-17 10:00:40 +00002422 static const netdev_features_t null_features = 0;
Ben Hutchings36c92472012-01-17 07:57:56 +00002423 struct net_device *dev = skb->dev;
Bjørn Mork88ad4172015-11-16 19:16:40 +01002424 const char *name = "";
Ben Hutchings36c92472012-01-17 07:57:56 +00002425
Ben Greearc846ad92013-04-19 10:45:52 +00002426 if (!net_ratelimit())
2427 return;
2428
Bjørn Mork88ad4172015-11-16 19:16:40 +01002429 if (dev) {
2430 if (dev->dev.parent)
2431 name = dev_driver_string(dev->dev.parent);
2432 else
2433 name = netdev_name(dev);
2434 }
Ben Hutchings36c92472012-01-17 07:57:56 +00002435 WARN(1, "%s: caps=(%pNF, %pNF) len=%d data_len=%d gso_size=%d "
2436 "gso_type=%d ip_summed=%d\n",
Bjørn Mork88ad4172015-11-16 19:16:40 +01002437 name, dev ? &dev->features : &null_features,
Michał Mirosław65e9d2f2012-01-17 10:00:40 +00002438 skb->sk ? &skb->sk->sk_route_caps : &null_features,
Ben Hutchings36c92472012-01-17 07:57:56 +00002439 skb->len, skb->data_len, skb_shinfo(skb)->gso_size,
2440 skb_shinfo(skb)->gso_type, skb->ip_summed);
2441}
2442
Linus Torvalds1da177e2005-04-16 15:20:36 -07002443/*
2444 * Invalidate hardware checksum when packet is to be mangled, and
2445 * complete checksum manually on outgoing path.
2446 */
Patrick McHardy84fa7932006-08-29 16:44:56 -07002447int skb_checksum_help(struct sk_buff *skb)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002448{
Al Virod3bc23e2006-11-14 21:24:49 -08002449 __wsum csum;
Herbert Xu663ead32007-04-09 11:59:07 -07002450 int ret = 0, offset;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002451
Patrick McHardy84fa7932006-08-29 16:44:56 -07002452 if (skb->ip_summed == CHECKSUM_COMPLETE)
Herbert Xua430a432006-07-08 13:34:56 -07002453 goto out_set_summed;
2454
2455 if (unlikely(skb_shinfo(skb)->gso_size)) {
Ben Hutchings36c92472012-01-17 07:57:56 +00002456 skb_warn_bad_offload(skb);
2457 return -EINVAL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002458 }
2459
Eric Dumazetcef401d2013-01-25 20:34:37 +00002460 /* Before computing a checksum, we should make sure no frag could
2461 * be modified by an external entity : checksum could be wrong.
2462 */
2463 if (skb_has_shared_frag(skb)) {
2464 ret = __skb_linearize(skb);
2465 if (ret)
2466 goto out;
2467 }
2468
Michał Mirosław55508d62010-12-14 15:24:08 +00002469 offset = skb_checksum_start_offset(skb);
Herbert Xua0308472007-10-15 01:47:15 -07002470 BUG_ON(offset >= skb_headlen(skb));
2471 csum = skb_checksum(skb, offset, skb->len - offset, 0);
2472
2473 offset += skb->csum_offset;
2474 BUG_ON(offset + sizeof(__sum16) > skb_headlen(skb));
2475
2476 if (skb_cloned(skb) &&
2477 !skb_clone_writable(skb, offset + sizeof(__sum16))) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07002478 ret = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
2479 if (ret)
2480 goto out;
2481 }
2482
Herbert Xua0308472007-10-15 01:47:15 -07002483 *(__sum16 *)(skb->data + offset) = csum_fold(csum);
Herbert Xua430a432006-07-08 13:34:56 -07002484out_set_summed:
Linus Torvalds1da177e2005-04-16 15:20:36 -07002485 skb->ip_summed = CHECKSUM_NONE;
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09002486out:
Linus Torvalds1da177e2005-04-16 15:20:36 -07002487 return ret;
2488}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07002489EXPORT_SYMBOL(skb_checksum_help);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002490
Tom Herbert6ae23ad2015-12-14 11:19:46 -08002491/* skb_csum_offload_check - Driver helper function to determine if a device
2492 * with limited checksum offload capabilities is able to offload the checksum
2493 * for a given packet.
2494 *
2495 * Arguments:
2496 * skb - sk_buff for the packet in question
2497 * spec - contains the description of what device can offload
2498 * csum_encapped - returns true if the checksum being offloaded is
2499 * encpasulated. That is it is checksum for the transport header
2500 * in the inner headers.
2501 * checksum_help - when set indicates that helper function should
2502 * call skb_checksum_help if offload checks fail
2503 *
2504 * Returns:
2505 * true: Packet has passed the checksum checks and should be offloadable to
2506 * the device (a driver may still need to check for additional
2507 * restrictions of its device)
2508 * false: Checksum is not offloadable. If checksum_help was set then
2509 * skb_checksum_help was called to resolve checksum for non-GSO
2510 * packets and when IP protocol is not SCTP
2511 */
2512bool __skb_csum_offload_chk(struct sk_buff *skb,
2513 const struct skb_csum_offl_spec *spec,
2514 bool *csum_encapped,
2515 bool csum_help)
2516{
2517 struct iphdr *iph;
2518 struct ipv6hdr *ipv6;
2519 void *nhdr;
2520 int protocol;
2521 u8 ip_proto;
2522
2523 if (skb->protocol == htons(ETH_P_8021Q) ||
2524 skb->protocol == htons(ETH_P_8021AD)) {
2525 if (!spec->vlan_okay)
2526 goto need_help;
2527 }
2528
2529 /* We check whether the checksum refers to a transport layer checksum in
2530 * the outermost header or an encapsulated transport layer checksum that
2531 * corresponds to the inner headers of the skb. If the checksum is for
2532 * something else in the packet we need help.
2533 */
2534 if (skb_checksum_start_offset(skb) == skb_transport_offset(skb)) {
2535 /* Non-encapsulated checksum */
2536 protocol = eproto_to_ipproto(vlan_get_protocol(skb));
2537 nhdr = skb_network_header(skb);
2538 *csum_encapped = false;
2539 if (spec->no_not_encapped)
2540 goto need_help;
2541 } else if (skb->encapsulation && spec->encap_okay &&
2542 skb_checksum_start_offset(skb) ==
2543 skb_inner_transport_offset(skb)) {
2544 /* Encapsulated checksum */
2545 *csum_encapped = true;
2546 switch (skb->inner_protocol_type) {
2547 case ENCAP_TYPE_ETHER:
2548 protocol = eproto_to_ipproto(skb->inner_protocol);
2549 break;
2550 case ENCAP_TYPE_IPPROTO:
2551 protocol = skb->inner_protocol;
2552 break;
2553 }
2554 nhdr = skb_inner_network_header(skb);
2555 } else {
2556 goto need_help;
2557 }
2558
2559 switch (protocol) {
2560 case IPPROTO_IP:
2561 if (!spec->ipv4_okay)
2562 goto need_help;
2563 iph = nhdr;
2564 ip_proto = iph->protocol;
2565 if (iph->ihl != 5 && !spec->ip_options_okay)
2566 goto need_help;
2567 break;
2568 case IPPROTO_IPV6:
2569 if (!spec->ipv6_okay)
2570 goto need_help;
2571 if (spec->no_encapped_ipv6 && *csum_encapped)
2572 goto need_help;
2573 ipv6 = nhdr;
2574 nhdr += sizeof(*ipv6);
2575 ip_proto = ipv6->nexthdr;
2576 break;
2577 default:
2578 goto need_help;
2579 }
2580
2581ip_proto_again:
2582 switch (ip_proto) {
2583 case IPPROTO_TCP:
2584 if (!spec->tcp_okay ||
2585 skb->csum_offset != offsetof(struct tcphdr, check))
2586 goto need_help;
2587 break;
2588 case IPPROTO_UDP:
2589 if (!spec->udp_okay ||
2590 skb->csum_offset != offsetof(struct udphdr, check))
2591 goto need_help;
2592 break;
2593 case IPPROTO_SCTP:
2594 if (!spec->sctp_okay ||
2595 skb->csum_offset != offsetof(struct sctphdr, checksum))
2596 goto cant_help;
2597 break;
2598 case NEXTHDR_HOP:
2599 case NEXTHDR_ROUTING:
2600 case NEXTHDR_DEST: {
2601 u8 *opthdr = nhdr;
2602
2603 if (protocol != IPPROTO_IPV6 || !spec->ext_hdrs_okay)
2604 goto need_help;
2605
2606 ip_proto = opthdr[0];
2607 nhdr += (opthdr[1] + 1) << 3;
2608
2609 goto ip_proto_again;
2610 }
2611 default:
2612 goto need_help;
2613 }
2614
2615 /* Passed the tests for offloading checksum */
2616 return true;
2617
2618need_help:
2619 if (csum_help && !skb_shinfo(skb)->gso_size)
2620 skb_checksum_help(skb);
2621cant_help:
2622 return false;
2623}
2624EXPORT_SYMBOL(__skb_csum_offload_chk);
2625
Vlad Yasevich53d64712014-03-27 17:26:18 -04002626__be16 skb_network_protocol(struct sk_buff *skb, int *depth)
Pravin B Shelarec5f0612013-03-07 09:28:01 +00002627{
2628 __be16 type = skb->protocol;
2629
Pravin B Shelar19acc322013-05-07 20:41:07 +00002630 /* Tunnel gso handlers can set protocol to ethernet. */
2631 if (type == htons(ETH_P_TEB)) {
2632 struct ethhdr *eth;
2633
2634 if (unlikely(!pskb_may_pull(skb, sizeof(struct ethhdr))))
2635 return 0;
2636
2637 eth = (struct ethhdr *)skb_mac_header(skb);
2638 type = eth->h_proto;
2639 }
2640
Toshiaki Makitad4bcef32015-01-29 20:37:07 +09002641 return __vlan_get_protocol(skb, type, depth);
Pravin B Shelarec5f0612013-03-07 09:28:01 +00002642}
2643
Pravin B Shelar05e8ef42013-02-14 09:44:55 +00002644/**
2645 * skb_mac_gso_segment - mac layer segmentation handler.
2646 * @skb: buffer to segment
2647 * @features: features for the output path (see dev->features)
2648 */
2649struct sk_buff *skb_mac_gso_segment(struct sk_buff *skb,
2650 netdev_features_t features)
2651{
2652 struct sk_buff *segs = ERR_PTR(-EPROTONOSUPPORT);
2653 struct packet_offload *ptype;
Vlad Yasevich53d64712014-03-27 17:26:18 -04002654 int vlan_depth = skb->mac_len;
2655 __be16 type = skb_network_protocol(skb, &vlan_depth);
Pravin B Shelar05e8ef42013-02-14 09:44:55 +00002656
Pravin B Shelarec5f0612013-03-07 09:28:01 +00002657 if (unlikely(!type))
2658 return ERR_PTR(-EINVAL);
Pravin B Shelar05e8ef42013-02-14 09:44:55 +00002659
Vlad Yasevich53d64712014-03-27 17:26:18 -04002660 __skb_pull(skb, vlan_depth);
Pravin B Shelar05e8ef42013-02-14 09:44:55 +00002661
2662 rcu_read_lock();
2663 list_for_each_entry_rcu(ptype, &offload_base, list) {
2664 if (ptype->type == type && ptype->callbacks.gso_segment) {
Pravin B Shelar05e8ef42013-02-14 09:44:55 +00002665 segs = ptype->callbacks.gso_segment(skb, features);
2666 break;
2667 }
2668 }
2669 rcu_read_unlock();
2670
2671 __skb_push(skb, skb->data - skb_mac_header(skb));
2672
2673 return segs;
2674}
2675EXPORT_SYMBOL(skb_mac_gso_segment);
2676
2677
Cong Wang12b00042013-02-05 16:36:38 +00002678/* openvswitch calls this on rx path, so we need a different check.
2679 */
2680static inline bool skb_needs_check(struct sk_buff *skb, bool tx_path)
2681{
2682 if (tx_path)
2683 return skb->ip_summed != CHECKSUM_PARTIAL;
2684 else
2685 return skb->ip_summed == CHECKSUM_NONE;
2686}
2687
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002688/**
Cong Wang12b00042013-02-05 16:36:38 +00002689 * __skb_gso_segment - Perform segmentation on skb.
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002690 * @skb: buffer to segment
Herbert Xu576a30e2006-06-27 13:22:38 -07002691 * @features: features for the output path (see dev->features)
Cong Wang12b00042013-02-05 16:36:38 +00002692 * @tx_path: whether it is called in TX path
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002693 *
2694 * This function segments the given skb and returns a list of segments.
Herbert Xu576a30e2006-06-27 13:22:38 -07002695 *
2696 * It may return NULL if the skb requires no segmentation. This is
2697 * only possible when GSO is used for verifying header integrity.
Konstantin Khlebnikov9207f9d2016-01-08 15:21:46 +03002698 *
2699 * Segmentation preserves SKB_SGO_CB_OFFSET bytes of previous skb cb.
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002700 */
Cong Wang12b00042013-02-05 16:36:38 +00002701struct sk_buff *__skb_gso_segment(struct sk_buff *skb,
2702 netdev_features_t features, bool tx_path)
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002703{
Cong Wang12b00042013-02-05 16:36:38 +00002704 if (unlikely(skb_needs_check(skb, tx_path))) {
Pravin B Shelar05e8ef42013-02-14 09:44:55 +00002705 int err;
2706
Ben Hutchings36c92472012-01-17 07:57:56 +00002707 skb_warn_bad_offload(skb);
Herbert Xu67fd1a72009-01-19 16:26:44 -08002708
françois romieua40e0a62014-07-15 23:55:35 +02002709 err = skb_cow_head(skb, 0);
2710 if (err < 0)
Herbert Xua430a432006-07-08 13:34:56 -07002711 return ERR_PTR(err);
2712 }
2713
Konstantin Khlebnikov9207f9d2016-01-08 15:21:46 +03002714 BUILD_BUG_ON(SKB_SGO_CB_OFFSET +
2715 sizeof(*SKB_GSO_CB(skb)) > sizeof(skb->cb));
2716
Pravin B Shelar68c33162013-02-14 14:02:41 +00002717 SKB_GSO_CB(skb)->mac_offset = skb_headroom(skb);
Eric Dumazet3347c962013-10-19 11:42:56 -07002718 SKB_GSO_CB(skb)->encap_level = 0;
2719
Pravin B Shelar05e8ef42013-02-14 09:44:55 +00002720 skb_reset_mac_header(skb);
2721 skb_reset_mac_len(skb);
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002722
Pravin B Shelar05e8ef42013-02-14 09:44:55 +00002723 return skb_mac_gso_segment(skb, features);
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002724}
Cong Wang12b00042013-02-05 16:36:38 +00002725EXPORT_SYMBOL(__skb_gso_segment);
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002726
Herbert Xufb286bb2005-11-10 13:01:24 -08002727/* Take action when hardware reception checksum errors are detected. */
2728#ifdef CONFIG_BUG
2729void netdev_rx_csum_fault(struct net_device *dev)
2730{
2731 if (net_ratelimit()) {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00002732 pr_err("%s: hw csum failure\n", dev ? dev->name : "<unknown>");
Herbert Xufb286bb2005-11-10 13:01:24 -08002733 dump_stack();
2734 }
2735}
2736EXPORT_SYMBOL(netdev_rx_csum_fault);
2737#endif
2738
Linus Torvalds1da177e2005-04-16 15:20:36 -07002739/* Actually, we should eliminate this check as soon as we know, that:
2740 * 1. IOMMU is present and allows to map all the memory.
2741 * 2. No high memory really exists on this machine.
2742 */
2743
Florian Westphalc1e756b2014-05-05 15:00:44 +02002744static int illegal_highdma(struct net_device *dev, struct sk_buff *skb)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002745{
Herbert Xu3d3a8532006-06-27 13:33:10 -07002746#ifdef CONFIG_HIGHMEM
Linus Torvalds1da177e2005-04-16 15:20:36 -07002747 int i;
FUJITA Tomonori5acbbd42010-03-30 22:35:50 +00002748 if (!(dev->features & NETIF_F_HIGHDMA)) {
Ian Campbellea2ab692011-08-22 23:44:58 +00002749 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
2750 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
2751 if (PageHighMem(skb_frag_page(frag)))
FUJITA Tomonori5acbbd42010-03-30 22:35:50 +00002752 return 1;
Ian Campbellea2ab692011-08-22 23:44:58 +00002753 }
FUJITA Tomonori5acbbd42010-03-30 22:35:50 +00002754 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002755
FUJITA Tomonori5acbbd42010-03-30 22:35:50 +00002756 if (PCI_DMA_BUS_IS_PHYS) {
2757 struct device *pdev = dev->dev.parent;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002758
Eric Dumazet9092c652010-04-02 13:34:49 -07002759 if (!pdev)
2760 return 0;
FUJITA Tomonori5acbbd42010-03-30 22:35:50 +00002761 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
Ian Campbellea2ab692011-08-22 23:44:58 +00002762 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
2763 dma_addr_t addr = page_to_phys(skb_frag_page(frag));
FUJITA Tomonori5acbbd42010-03-30 22:35:50 +00002764 if (!pdev->dma_mask || addr + PAGE_SIZE - 1 > *pdev->dma_mask)
2765 return 1;
2766 }
2767 }
Herbert Xu3d3a8532006-06-27 13:33:10 -07002768#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07002769 return 0;
2770}
Linus Torvalds1da177e2005-04-16 15:20:36 -07002771
Simon Horman3b392dd2014-06-04 08:53:17 +09002772/* If MPLS offload request, verify we are testing hardware MPLS features
2773 * instead of standard features for the netdev.
2774 */
Pravin B Shelard0edc7b2014-12-23 16:20:11 -08002775#if IS_ENABLED(CONFIG_NET_MPLS_GSO)
Simon Horman3b392dd2014-06-04 08:53:17 +09002776static netdev_features_t net_mpls_features(struct sk_buff *skb,
2777 netdev_features_t features,
2778 __be16 type)
2779{
Simon Horman25cd9ba2014-10-06 05:05:13 -07002780 if (eth_p_mpls(type))
Simon Horman3b392dd2014-06-04 08:53:17 +09002781 features &= skb->dev->mpls_features;
2782
2783 return features;
2784}
2785#else
2786static netdev_features_t net_mpls_features(struct sk_buff *skb,
2787 netdev_features_t features,
2788 __be16 type)
2789{
2790 return features;
2791}
2792#endif
2793
Michał Mirosławc8f44af2011-11-15 15:29:55 +00002794static netdev_features_t harmonize_features(struct sk_buff *skb,
Florian Westphalc1e756b2014-05-05 15:00:44 +02002795 netdev_features_t features)
Jesse Grossf01a5232011-01-09 06:23:31 +00002796{
Vlad Yasevich53d64712014-03-27 17:26:18 -04002797 int tmp;
Simon Horman3b392dd2014-06-04 08:53:17 +09002798 __be16 type;
2799
2800 type = skb_network_protocol(skb, &tmp);
2801 features = net_mpls_features(skb, features, type);
Vlad Yasevich53d64712014-03-27 17:26:18 -04002802
Ed Cashinc0d680e2012-09-19 15:49:00 +00002803 if (skb->ip_summed != CHECKSUM_NONE &&
Simon Horman3b392dd2014-06-04 08:53:17 +09002804 !can_checksum_protocol(features, type)) {
Alexander Duyck996e8022016-05-02 09:25:10 -07002805 features &= ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK);
Florian Westphalc1e756b2014-05-05 15:00:44 +02002806 } else if (illegal_highdma(skb->dev, skb)) {
Jesse Grossf01a5232011-01-09 06:23:31 +00002807 features &= ~NETIF_F_SG;
2808 }
2809
2810 return features;
2811}
2812
Toshiaki Makitae38f3022015-03-27 14:31:13 +09002813netdev_features_t passthru_features_check(struct sk_buff *skb,
2814 struct net_device *dev,
2815 netdev_features_t features)
2816{
2817 return features;
2818}
2819EXPORT_SYMBOL(passthru_features_check);
2820
Toshiaki Makita8cb65d02015-03-27 14:31:12 +09002821static netdev_features_t dflt_features_check(const struct sk_buff *skb,
2822 struct net_device *dev,
2823 netdev_features_t features)
2824{
2825 return vlan_features_check(skb, features);
2826}
2827
Florian Westphalc1e756b2014-05-05 15:00:44 +02002828netdev_features_t netif_skb_features(struct sk_buff *skb)
Jesse Gross58e998c2010-10-29 12:14:55 +00002829{
Jesse Gross5f352272014-12-23 22:37:26 -08002830 struct net_device *dev = skb->dev;
Eric Dumazetfcbeb972014-10-05 10:11:27 -07002831 netdev_features_t features = dev->features;
2832 u16 gso_segs = skb_shinfo(skb)->gso_segs;
Jesse Gross58e998c2010-10-29 12:14:55 +00002833
Eric Dumazetfcbeb972014-10-05 10:11:27 -07002834 if (gso_segs > dev->gso_max_segs || gso_segs < dev->gso_min_segs)
Ben Hutchings30b678d2012-07-30 15:57:00 +00002835 features &= ~NETIF_F_GSO_MASK;
2836
Jesse Gross5f352272014-12-23 22:37:26 -08002837 /* If encapsulation offload request, verify we are testing
2838 * hardware encapsulation features instead of standard
2839 * features for the netdev
2840 */
2841 if (skb->encapsulation)
2842 features &= dev->hw_enc_features;
2843
Toshiaki Makitaf5a7fb82015-03-27 14:31:11 +09002844 if (skb_vlan_tagged(skb))
2845 features = netdev_intersect_features(features,
2846 dev->vlan_features |
2847 NETIF_F_HW_VLAN_CTAG_TX |
2848 NETIF_F_HW_VLAN_STAG_TX);
Jesse Gross58e998c2010-10-29 12:14:55 +00002849
Jesse Gross5f352272014-12-23 22:37:26 -08002850 if (dev->netdev_ops->ndo_features_check)
2851 features &= dev->netdev_ops->ndo_features_check(skb, dev,
2852 features);
Toshiaki Makita8cb65d02015-03-27 14:31:12 +09002853 else
2854 features &= dflt_features_check(skb, dev, features);
Jesse Gross5f352272014-12-23 22:37:26 -08002855
Florian Westphalc1e756b2014-05-05 15:00:44 +02002856 return harmonize_features(skb, features);
Jesse Gross58e998c2010-10-29 12:14:55 +00002857}
Florian Westphalc1e756b2014-05-05 15:00:44 +02002858EXPORT_SYMBOL(netif_skb_features);
Jesse Gross58e998c2010-10-29 12:14:55 +00002859
David S. Miller2ea25512014-08-29 21:10:01 -07002860static int xmit_one(struct sk_buff *skb, struct net_device *dev,
David S. Miller95f6b3d2014-08-29 21:57:30 -07002861 struct netdev_queue *txq, bool more)
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002862{
David S. Miller2ea25512014-08-29 21:10:01 -07002863 unsigned int len;
2864 int rc;
Stephen Hemminger00829822008-11-20 20:14:53 -08002865
Salam Noureddine7866a622015-01-27 11:35:48 -08002866 if (!list_empty(&ptype_all) || !list_empty(&dev->ptype_all))
David S. Miller2ea25512014-08-29 21:10:01 -07002867 dev_queue_xmit_nit(skb, dev);
Jesse Grossfc741212011-01-09 06:23:32 +00002868
David S. Miller2ea25512014-08-29 21:10:01 -07002869 len = skb->len;
2870 trace_net_dev_start_xmit(skb, dev);
David S. Miller95f6b3d2014-08-29 21:57:30 -07002871 rc = netdev_start_xmit(skb, dev, txq, more);
David S. Miller2ea25512014-08-29 21:10:01 -07002872 trace_net_dev_xmit(skb, rc, dev, len);
Eric Dumazetadf30902009-06-02 05:19:30 +00002873
Patrick McHardy572a9d72009-11-10 06:14:14 +00002874 return rc;
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002875}
David S. Miller2ea25512014-08-29 21:10:01 -07002876
David S. Miller8dcda222014-09-01 15:06:40 -07002877struct sk_buff *dev_hard_start_xmit(struct sk_buff *first, struct net_device *dev,
2878 struct netdev_queue *txq, int *ret)
David S. Miller7f2e8702014-08-29 21:19:14 -07002879{
2880 struct sk_buff *skb = first;
2881 int rc = NETDEV_TX_OK;
2882
2883 while (skb) {
2884 struct sk_buff *next = skb->next;
2885
2886 skb->next = NULL;
David S. Miller95f6b3d2014-08-29 21:57:30 -07002887 rc = xmit_one(skb, dev, txq, next != NULL);
David S. Miller7f2e8702014-08-29 21:19:14 -07002888 if (unlikely(!dev_xmit_complete(rc))) {
2889 skb->next = next;
2890 goto out;
2891 }
2892
2893 skb = next;
2894 if (netif_xmit_stopped(txq) && skb) {
2895 rc = NETDEV_TX_BUSY;
2896 break;
2897 }
2898 }
2899
2900out:
2901 *ret = rc;
2902 return skb;
2903}
2904
Eric Dumazet1ff0dc92014-10-06 11:26:27 -07002905static struct sk_buff *validate_xmit_vlan(struct sk_buff *skb,
2906 netdev_features_t features)
David S. Millereae3f882014-08-30 15:17:13 -07002907{
Jiri Pirkodf8a39d2015-01-13 17:13:44 +01002908 if (skb_vlan_tag_present(skb) &&
Jiri Pirko59682502014-11-19 14:04:59 +01002909 !vlan_hw_offload_capable(features, skb->vlan_proto))
2910 skb = __vlan_hwaccel_push_inside(skb);
David S. Millereae3f882014-08-30 15:17:13 -07002911 return skb;
2912}
2913
Eric Dumazet55a93b32014-10-03 15:31:07 -07002914static struct sk_buff *validate_xmit_skb(struct sk_buff *skb, struct net_device *dev)
David S. Millereae3f882014-08-30 15:17:13 -07002915{
2916 netdev_features_t features;
2917
2918 if (skb->next)
2919 return skb;
2920
David S. Millereae3f882014-08-30 15:17:13 -07002921 features = netif_skb_features(skb);
2922 skb = validate_xmit_vlan(skb, features);
2923 if (unlikely(!skb))
2924 goto out_null;
2925
Johannes Berg8b86a612015-04-17 15:45:04 +02002926 if (netif_needs_gso(skb, features)) {
David S. Millerce937182014-08-30 19:22:20 -07002927 struct sk_buff *segs;
2928
2929 segs = skb_gso_segment(skb, features);
Jason Wangcecda692014-09-19 16:04:38 +08002930 if (IS_ERR(segs)) {
Jason Wangaf6dabc2014-12-19 11:09:13 +08002931 goto out_kfree_skb;
Jason Wangcecda692014-09-19 16:04:38 +08002932 } else if (segs) {
2933 consume_skb(skb);
2934 skb = segs;
2935 }
David S. Millereae3f882014-08-30 15:17:13 -07002936 } else {
2937 if (skb_needs_linearize(skb, features) &&
2938 __skb_linearize(skb))
2939 goto out_kfree_skb;
2940
2941 /* If packet is not checksummed and device does not
2942 * support checksumming for this protocol, complete
2943 * checksumming here.
2944 */
2945 if (skb->ip_summed == CHECKSUM_PARTIAL) {
2946 if (skb->encapsulation)
2947 skb_set_inner_transport_header(skb,
2948 skb_checksum_start_offset(skb));
2949 else
2950 skb_set_transport_header(skb,
2951 skb_checksum_start_offset(skb));
Tom Herberta1882222015-12-14 11:19:43 -08002952 if (!(features & NETIF_F_CSUM_MASK) &&
David S. Millereae3f882014-08-30 15:17:13 -07002953 skb_checksum_help(skb))
2954 goto out_kfree_skb;
2955 }
2956 }
2957
2958 return skb;
2959
2960out_kfree_skb:
2961 kfree_skb(skb);
2962out_null:
2963 return NULL;
2964}
2965
Eric Dumazet55a93b32014-10-03 15:31:07 -07002966struct sk_buff *validate_xmit_skb_list(struct sk_buff *skb, struct net_device *dev)
2967{
2968 struct sk_buff *next, *head = NULL, *tail;
2969
Eric Dumazetbec3cfd2014-10-03 20:59:19 -07002970 for (; skb != NULL; skb = next) {
Eric Dumazet55a93b32014-10-03 15:31:07 -07002971 next = skb->next;
2972 skb->next = NULL;
Eric Dumazet55a93b32014-10-03 15:31:07 -07002973
Eric Dumazetbec3cfd2014-10-03 20:59:19 -07002974 /* in case skb wont be segmented, point to itself */
2975 skb->prev = skb;
2976
2977 skb = validate_xmit_skb(skb, dev);
2978 if (!skb)
2979 continue;
2980
2981 if (!head)
2982 head = skb;
2983 else
2984 tail->next = skb;
2985 /* If skb was segmented, skb->prev points to
2986 * the last segment. If not, it still contains skb.
2987 */
2988 tail = skb->prev;
Eric Dumazet55a93b32014-10-03 15:31:07 -07002989 }
2990 return head;
2991}
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002992
Eric Dumazet1def9232013-01-10 12:36:42 +00002993static void qdisc_pkt_len_init(struct sk_buff *skb)
2994{
2995 const struct skb_shared_info *shinfo = skb_shinfo(skb);
2996
2997 qdisc_skb_cb(skb)->pkt_len = skb->len;
2998
2999 /* To get more precise estimation of bytes sent on wire,
3000 * we add to pkt_len the headers size of all segments
3001 */
3002 if (shinfo->gso_size) {
Eric Dumazet757b8b12013-01-15 21:14:21 -08003003 unsigned int hdr_len;
Jason Wang15e5a032013-03-25 20:19:59 +00003004 u16 gso_segs = shinfo->gso_segs;
Eric Dumazet1def9232013-01-10 12:36:42 +00003005
Eric Dumazet757b8b12013-01-15 21:14:21 -08003006 /* mac layer + network layer */
3007 hdr_len = skb_transport_header(skb) - skb_mac_header(skb);
3008
3009 /* + transport layer */
Eric Dumazet1def9232013-01-10 12:36:42 +00003010 if (likely(shinfo->gso_type & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6)))
3011 hdr_len += tcp_hdrlen(skb);
3012 else
3013 hdr_len += sizeof(struct udphdr);
Jason Wang15e5a032013-03-25 20:19:59 +00003014
3015 if (shinfo->gso_type & SKB_GSO_DODGY)
3016 gso_segs = DIV_ROUND_UP(skb->len - hdr_len,
3017 shinfo->gso_size);
3018
3019 qdisc_skb_cb(skb)->pkt_len += (gso_segs - 1) * hdr_len;
Eric Dumazet1def9232013-01-10 12:36:42 +00003020 }
3021}
3022
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00003023static inline int __dev_xmit_skb(struct sk_buff *skb, struct Qdisc *q,
3024 struct net_device *dev,
3025 struct netdev_queue *txq)
3026{
3027 spinlock_t *root_lock = qdisc_lock(q);
Eric Dumazeta2da5702011-01-20 03:48:19 +00003028 bool contended;
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00003029 int rc;
3030
Eric Dumazeta2da5702011-01-20 03:48:19 +00003031 qdisc_calculate_pkt_len(skb, q);
Eric Dumazet79640a42010-06-02 05:09:29 -07003032 /*
3033 * Heuristic to force contended enqueues to serialize on a
3034 * separate lock before trying to get qdisc main lock.
Ying Xue9bf2b8c2014-06-26 15:56:31 +08003035 * This permits __QDISC___STATE_RUNNING owner to get the lock more
3036 * often and dequeue packets faster.
Eric Dumazet79640a42010-06-02 05:09:29 -07003037 */
Eric Dumazeta2da5702011-01-20 03:48:19 +00003038 contended = qdisc_is_running(q);
Eric Dumazet79640a42010-06-02 05:09:29 -07003039 if (unlikely(contended))
3040 spin_lock(&q->busylock);
3041
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00003042 spin_lock(root_lock);
3043 if (unlikely(test_bit(__QDISC_STATE_DEACTIVATED, &q->state))) {
3044 kfree_skb(skb);
3045 rc = NET_XMIT_DROP;
3046 } else if ((q->flags & TCQ_F_CAN_BYPASS) && !qdisc_qlen(q) &&
Eric Dumazetbc135b22010-06-02 03:23:51 -07003047 qdisc_run_begin(q)) {
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00003048 /*
3049 * This is a work-conserving queue; there are no old skbs
3050 * waiting to be sent out; and the qdisc is not running -
3051 * xmit the skb directly.
3052 */
Eric Dumazetbfe0d022011-01-09 08:30:54 +00003053
Eric Dumazetbfe0d022011-01-09 08:30:54 +00003054 qdisc_bstats_update(q, skb);
3055
Eric Dumazet55a93b32014-10-03 15:31:07 -07003056 if (sch_direct_xmit(skb, q, dev, txq, root_lock, true)) {
Eric Dumazet79640a42010-06-02 05:09:29 -07003057 if (unlikely(contended)) {
3058 spin_unlock(&q->busylock);
3059 contended = false;
3060 }
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00003061 __qdisc_run(q);
Eric Dumazet79640a42010-06-02 05:09:29 -07003062 } else
Eric Dumazetbc135b22010-06-02 03:23:51 -07003063 qdisc_run_end(q);
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00003064
3065 rc = NET_XMIT_SUCCESS;
3066 } else {
Eric Dumazeta2da5702011-01-20 03:48:19 +00003067 rc = q->enqueue(skb, q) & NET_XMIT_MASK;
Eric Dumazet79640a42010-06-02 05:09:29 -07003068 if (qdisc_run_begin(q)) {
3069 if (unlikely(contended)) {
3070 spin_unlock(&q->busylock);
3071 contended = false;
3072 }
3073 __qdisc_run(q);
3074 }
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00003075 }
3076 spin_unlock(root_lock);
Eric Dumazet79640a42010-06-02 05:09:29 -07003077 if (unlikely(contended))
3078 spin_unlock(&q->busylock);
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00003079 return rc;
3080}
3081
Daniel Borkmann86f85152013-12-29 17:27:11 +01003082#if IS_ENABLED(CONFIG_CGROUP_NET_PRIO)
Neil Horman5bc14212011-11-22 05:10:51 +00003083static void skb_update_prio(struct sk_buff *skb)
3084{
Igor Maravic6977a792011-11-25 07:44:54 +00003085 struct netprio_map *map = rcu_dereference_bh(skb->dev->priomap);
Neil Horman5bc14212011-11-22 05:10:51 +00003086
Eric Dumazet91c68ce2012-07-08 21:45:10 +00003087 if (!skb->priority && skb->sk && map) {
Tejun Heo2a56a1f2015-12-07 17:38:52 -05003088 unsigned int prioidx =
3089 sock_cgroup_prioidx(&skb->sk->sk_cgrp_data);
Eric Dumazet91c68ce2012-07-08 21:45:10 +00003090
3091 if (prioidx < map->priomap_len)
3092 skb->priority = map->priomap[prioidx];
3093 }
Neil Horman5bc14212011-11-22 05:10:51 +00003094}
3095#else
3096#define skb_update_prio(skb)
3097#endif
3098
hannes@stressinduktion.orgf60e5992015-04-01 17:07:44 +02003099DEFINE_PER_CPU(int, xmit_recursion);
3100EXPORT_SYMBOL(xmit_recursion);
3101
David S. Miller11a766c2010-10-25 12:51:55 -07003102#define RECURSION_LIMIT 10
Eric Dumazet745e20f2010-09-29 13:23:09 -07003103
Dave Jonesd29f7492008-07-22 14:09:06 -07003104/**
Michel Machado95603e22012-06-12 10:16:35 +00003105 * dev_loopback_xmit - loop back @skb
Eric W. Biederman0c4b51f2015-09-15 20:04:18 -05003106 * @net: network namespace this loopback is happening in
3107 * @sk: sk needed to be a netfilter okfn
Michel Machado95603e22012-06-12 10:16:35 +00003108 * @skb: buffer to transmit
3109 */
Eric W. Biederman0c4b51f2015-09-15 20:04:18 -05003110int dev_loopback_xmit(struct net *net, struct sock *sk, struct sk_buff *skb)
Michel Machado95603e22012-06-12 10:16:35 +00003111{
3112 skb_reset_mac_header(skb);
3113 __skb_pull(skb, skb_network_offset(skb));
3114 skb->pkt_type = PACKET_LOOPBACK;
3115 skb->ip_summed = CHECKSUM_UNNECESSARY;
3116 WARN_ON(!skb_dst(skb));
3117 skb_dst_force(skb);
3118 netif_rx_ni(skb);
3119 return 0;
3120}
3121EXPORT_SYMBOL(dev_loopback_xmit);
3122
Daniel Borkmann1f211a12016-01-07 22:29:47 +01003123#ifdef CONFIG_NET_EGRESS
3124static struct sk_buff *
3125sch_handle_egress(struct sk_buff *skb, int *ret, struct net_device *dev)
3126{
3127 struct tcf_proto *cl = rcu_dereference_bh(dev->egress_cl_list);
3128 struct tcf_result cl_res;
3129
3130 if (!cl)
3131 return skb;
3132
3133 /* skb->tc_verd and qdisc_skb_cb(skb)->pkt_len were already set
3134 * earlier by the caller.
3135 */
3136 qdisc_bstats_cpu_update(cl->q, skb);
3137
3138 switch (tc_classify(skb, cl, &cl_res, false)) {
3139 case TC_ACT_OK:
3140 case TC_ACT_RECLASSIFY:
3141 skb->tc_index = TC_H_MIN(cl_res.classid);
3142 break;
3143 case TC_ACT_SHOT:
3144 qdisc_qstats_cpu_drop(cl->q);
3145 *ret = NET_XMIT_DROP;
3146 goto drop;
3147 case TC_ACT_STOLEN:
3148 case TC_ACT_QUEUED:
3149 *ret = NET_XMIT_SUCCESS;
3150drop:
3151 kfree_skb(skb);
3152 return NULL;
3153 case TC_ACT_REDIRECT:
3154 /* No need to push/pop skb's mac_header here on egress! */
3155 skb_do_redirect(skb);
3156 *ret = NET_XMIT_SUCCESS;
3157 return NULL;
3158 default:
3159 break;
3160 }
3161
3162 return skb;
3163}
3164#endif /* CONFIG_NET_EGRESS */
3165
Jiri Pirko638b2a62015-05-12 14:56:13 +02003166static inline int get_xps_queue(struct net_device *dev, struct sk_buff *skb)
3167{
3168#ifdef CONFIG_XPS
3169 struct xps_dev_maps *dev_maps;
3170 struct xps_map *map;
3171 int queue_index = -1;
3172
3173 rcu_read_lock();
3174 dev_maps = rcu_dereference(dev->xps_maps);
3175 if (dev_maps) {
3176 map = rcu_dereference(
3177 dev_maps->cpu_map[skb->sender_cpu - 1]);
3178 if (map) {
3179 if (map->len == 1)
3180 queue_index = map->queues[0];
3181 else
3182 queue_index = map->queues[reciprocal_scale(skb_get_hash(skb),
3183 map->len)];
3184 if (unlikely(queue_index >= dev->real_num_tx_queues))
3185 queue_index = -1;
3186 }
3187 }
3188 rcu_read_unlock();
3189
3190 return queue_index;
3191#else
3192 return -1;
3193#endif
3194}
3195
3196static u16 __netdev_pick_tx(struct net_device *dev, struct sk_buff *skb)
3197{
3198 struct sock *sk = skb->sk;
3199 int queue_index = sk_tx_queue_get(sk);
3200
3201 if (queue_index < 0 || skb->ooo_okay ||
3202 queue_index >= dev->real_num_tx_queues) {
3203 int new_index = get_xps_queue(dev, skb);
3204 if (new_index < 0)
3205 new_index = skb_tx_hash(dev, skb);
3206
3207 if (queue_index != new_index && sk &&
Eric Dumazet004a5d02015-10-04 21:08:10 -07003208 sk_fullsock(sk) &&
Jiri Pirko638b2a62015-05-12 14:56:13 +02003209 rcu_access_pointer(sk->sk_dst_cache))
3210 sk_tx_queue_set(sk, new_index);
3211
3212 queue_index = new_index;
3213 }
3214
3215 return queue_index;
3216}
3217
3218struct netdev_queue *netdev_pick_tx(struct net_device *dev,
3219 struct sk_buff *skb,
3220 void *accel_priv)
3221{
3222 int queue_index = 0;
3223
3224#ifdef CONFIG_XPS
Eric Dumazet52bd2d62015-11-18 06:30:50 -08003225 u32 sender_cpu = skb->sender_cpu - 1;
3226
3227 if (sender_cpu >= (u32)NR_CPUS)
Jiri Pirko638b2a62015-05-12 14:56:13 +02003228 skb->sender_cpu = raw_smp_processor_id() + 1;
3229#endif
3230
3231 if (dev->real_num_tx_queues != 1) {
3232 const struct net_device_ops *ops = dev->netdev_ops;
3233 if (ops->ndo_select_queue)
3234 queue_index = ops->ndo_select_queue(dev, skb, accel_priv,
3235 __netdev_pick_tx);
3236 else
3237 queue_index = __netdev_pick_tx(dev, skb);
3238
3239 if (!accel_priv)
3240 queue_index = netdev_cap_txqueue(dev, queue_index);
3241 }
3242
3243 skb_set_queue_mapping(skb, queue_index);
3244 return netdev_get_tx_queue(dev, queue_index);
3245}
3246
Michel Machado95603e22012-06-12 10:16:35 +00003247/**
Jason Wang9d08dd32014-01-20 11:25:13 +08003248 * __dev_queue_xmit - transmit a buffer
Dave Jonesd29f7492008-07-22 14:09:06 -07003249 * @skb: buffer to transmit
Jason Wang9d08dd32014-01-20 11:25:13 +08003250 * @accel_priv: private data used for L2 forwarding offload
Dave Jonesd29f7492008-07-22 14:09:06 -07003251 *
3252 * Queue a buffer for transmission to a network device. The caller must
3253 * have set the device and priority and built the buffer before calling
3254 * this function. The function can be called from an interrupt.
3255 *
3256 * A negative errno code is returned on a failure. A success does not
3257 * guarantee the frame will be transmitted as it may be dropped due
3258 * to congestion or traffic shaping.
3259 *
3260 * -----------------------------------------------------------------------------------
3261 * I notice this method can also return errors from the queue disciplines,
3262 * including NET_XMIT_DROP, which is a positive value. So, errors can also
3263 * be positive.
3264 *
3265 * Regardless of the return value, the skb is consumed, so it is currently
3266 * difficult to retry a send to this method. (You can bump the ref count
3267 * before sending to hold a reference for retry if you are careful.)
3268 *
3269 * When calling this method, interrupts MUST be enabled. This is because
3270 * the BH enable code must have IRQs enabled so that it will not deadlock.
3271 * --BLG
3272 */
Rashika Kheria0a59f3a2014-02-09 20:26:25 +05303273static int __dev_queue_xmit(struct sk_buff *skb, void *accel_priv)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003274{
3275 struct net_device *dev = skb->dev;
David S. Millerdc2b4842008-07-08 17:18:23 -07003276 struct netdev_queue *txq;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003277 struct Qdisc *q;
3278 int rc = -ENOMEM;
3279
Eric Dumazet6d1ccff2013-02-05 20:22:20 +00003280 skb_reset_mac_header(skb);
3281
Willem de Bruijne7fd2882014-08-04 22:11:48 -04003282 if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_SCHED_TSTAMP))
3283 __skb_tstamp_tx(skb, NULL, skb->sk, SCM_TSTAMP_SCHED);
3284
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09003285 /* Disable soft irqs for various locks below. Also
3286 * stops preemption for RCU.
Linus Torvalds1da177e2005-04-16 15:20:36 -07003287 */
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09003288 rcu_read_lock_bh();
Linus Torvalds1da177e2005-04-16 15:20:36 -07003289
Neil Horman5bc14212011-11-22 05:10:51 +00003290 skb_update_prio(skb);
3291
Daniel Borkmann1f211a12016-01-07 22:29:47 +01003292 qdisc_pkt_len_init(skb);
3293#ifdef CONFIG_NET_CLS_ACT
3294 skb->tc_verd = SET_TC_AT(skb->tc_verd, AT_EGRESS);
3295# ifdef CONFIG_NET_EGRESS
3296 if (static_key_false(&egress_needed)) {
3297 skb = sch_handle_egress(skb, &rc, dev);
3298 if (!skb)
3299 goto out;
3300 }
3301# endif
3302#endif
Eric Dumazet02875872014-10-05 18:38:35 -07003303 /* If device/qdisc don't need skb->dst, release it right now while
3304 * its hot in this cpu cache.
3305 */
3306 if (dev->priv_flags & IFF_XMIT_DST_RELEASE)
3307 skb_dst_drop(skb);
3308 else
3309 skb_dst_force(skb);
3310
Scott Feldman0c4f6912015-07-18 18:24:48 -07003311#ifdef CONFIG_NET_SWITCHDEV
3312 /* Don't forward if offload device already forwarded */
3313 if (skb->offload_fwd_mark &&
3314 skb->offload_fwd_mark == dev->offload_fwd_mark) {
3315 consume_skb(skb);
3316 rc = NET_XMIT_SUCCESS;
3317 goto out;
3318 }
3319#endif
3320
Jason Wangf663dd92014-01-10 16:18:26 +08003321 txq = netdev_pick_tx(dev, skb, accel_priv);
Paul E. McKenneya898def2010-02-22 17:04:49 -08003322 q = rcu_dereference_bh(txq->qdisc);
David S. Miller37437bb2008-07-16 02:15:04 -07003323
Koki Sanagicf66ba52010-08-23 18:45:02 +09003324 trace_net_dev_queue(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003325 if (q->enqueue) {
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00003326 rc = __dev_xmit_skb(skb, q, dev, txq);
David S. Miller37437bb2008-07-16 02:15:04 -07003327 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003328 }
3329
3330 /* The device has no queue. Common case for software devices:
3331 loopback, all the sorts of tunnels...
3332
Herbert Xu932ff272006-06-09 12:20:56 -07003333 Really, it is unlikely that netif_tx_lock protection is necessary
3334 here. (f.e. loopback and IP tunnels are clean ignoring statistics
Linus Torvalds1da177e2005-04-16 15:20:36 -07003335 counters.)
3336 However, it is possible, that they rely on protection
3337 made by us here.
3338
3339 Check this and shot the lock. It is not prone from deadlocks.
3340 Either shot noqueue qdisc, it is even simpler 8)
3341 */
3342 if (dev->flags & IFF_UP) {
3343 int cpu = smp_processor_id(); /* ok because BHs are off */
3344
David S. Millerc773e842008-07-08 23:13:53 -07003345 if (txq->xmit_lock_owner != cpu) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07003346
Eric Dumazet745e20f2010-09-29 13:23:09 -07003347 if (__this_cpu_read(xmit_recursion) > RECURSION_LIMIT)
3348 goto recursion_alert;
3349
Jesper Dangaard Brouer1f595332014-09-03 17:56:09 +02003350 skb = validate_xmit_skb(skb, dev);
3351 if (!skb)
3352 goto drop;
3353
David S. Millerc773e842008-07-08 23:13:53 -07003354 HARD_TX_LOCK(dev, txq, cpu);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003355
Tom Herbert734664982011-11-28 16:32:44 +00003356 if (!netif_xmit_stopped(txq)) {
Eric Dumazet745e20f2010-09-29 13:23:09 -07003357 __this_cpu_inc(xmit_recursion);
David S. Millerce937182014-08-30 19:22:20 -07003358 skb = dev_hard_start_xmit(skb, dev, txq, &rc);
Eric Dumazet745e20f2010-09-29 13:23:09 -07003359 __this_cpu_dec(xmit_recursion);
Patrick McHardy572a9d72009-11-10 06:14:14 +00003360 if (dev_xmit_complete(rc)) {
David S. Millerc773e842008-07-08 23:13:53 -07003361 HARD_TX_UNLOCK(dev, txq);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003362 goto out;
3363 }
3364 }
David S. Millerc773e842008-07-08 23:13:53 -07003365 HARD_TX_UNLOCK(dev, txq);
Joe Perchese87cc472012-05-13 21:56:26 +00003366 net_crit_ratelimited("Virtual device %s asks to queue packet!\n",
3367 dev->name);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003368 } else {
3369 /* Recursion is detected! It is possible,
Eric Dumazet745e20f2010-09-29 13:23:09 -07003370 * unfortunately
3371 */
3372recursion_alert:
Joe Perchese87cc472012-05-13 21:56:26 +00003373 net_crit_ratelimited("Dead loop on virtual device %s, fix it urgently!\n",
3374 dev->name);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003375 }
3376 }
3377
3378 rc = -ENETDOWN;
Jesper Dangaard Brouer1f595332014-09-03 17:56:09 +02003379drop:
Herbert Xud4828d82006-06-22 02:28:18 -07003380 rcu_read_unlock_bh();
Linus Torvalds1da177e2005-04-16 15:20:36 -07003381
Eric Dumazet015f0682014-03-27 08:45:56 -07003382 atomic_long_inc(&dev->tx_dropped);
Jesper Dangaard Brouer1f595332014-09-03 17:56:09 +02003383 kfree_skb_list(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003384 return rc;
3385out:
Herbert Xud4828d82006-06-22 02:28:18 -07003386 rcu_read_unlock_bh();
Linus Torvalds1da177e2005-04-16 15:20:36 -07003387 return rc;
3388}
Jason Wangf663dd92014-01-10 16:18:26 +08003389
Eric W. Biederman2b4aa3c2015-09-15 20:04:07 -05003390int dev_queue_xmit(struct sk_buff *skb)
Jason Wangf663dd92014-01-10 16:18:26 +08003391{
3392 return __dev_queue_xmit(skb, NULL);
3393}
Eric W. Biederman2b4aa3c2015-09-15 20:04:07 -05003394EXPORT_SYMBOL(dev_queue_xmit);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003395
Jason Wangf663dd92014-01-10 16:18:26 +08003396int dev_queue_xmit_accel(struct sk_buff *skb, void *accel_priv)
3397{
3398 return __dev_queue_xmit(skb, accel_priv);
3399}
3400EXPORT_SYMBOL(dev_queue_xmit_accel);
3401
Linus Torvalds1da177e2005-04-16 15:20:36 -07003402
3403/*=======================================================================
3404 Receiver routines
3405 =======================================================================*/
3406
Stephen Hemminger6b2bedc2007-03-12 14:33:50 -07003407int netdev_max_backlog __read_mostly = 1000;
Eric Dumazetc9e6bc62012-09-27 19:29:05 +00003408EXPORT_SYMBOL(netdev_max_backlog);
3409
Eric Dumazet3b098e22010-05-15 23:57:10 -07003410int netdev_tstamp_prequeue __read_mostly = 1;
Stephen Hemminger6b2bedc2007-03-12 14:33:50 -07003411int netdev_budget __read_mostly = 300;
3412int weight_p __read_mostly = 64; /* old backlog weight */
Linus Torvalds1da177e2005-04-16 15:20:36 -07003413
Eric Dumazeteecfd7c2010-05-06 22:07:48 -07003414/* Called with irq disabled */
3415static inline void ____napi_schedule(struct softnet_data *sd,
3416 struct napi_struct *napi)
3417{
3418 list_add_tail(&napi->poll_list, &sd->poll_list);
3419 __raise_softirq_irqoff(NET_RX_SOFTIRQ);
3420}
3421
Eric Dumazetdf334542010-03-24 19:13:54 +00003422#ifdef CONFIG_RPS
Tom Herbertfec5e652010-04-16 16:01:27 -07003423
3424/* One global table that all flow-based protocols share. */
Eric Dumazet6e3f7fa2010-10-25 03:02:02 +00003425struct rps_sock_flow_table __rcu *rps_sock_flow_table __read_mostly;
Tom Herbertfec5e652010-04-16 16:01:27 -07003426EXPORT_SYMBOL(rps_sock_flow_table);
Eric Dumazet567e4b72015-02-06 12:59:01 -08003427u32 rps_cpu_mask __read_mostly;
3428EXPORT_SYMBOL(rps_cpu_mask);
Tom Herbertfec5e652010-04-16 16:01:27 -07003429
Ingo Molnarc5905af2012-02-24 08:31:31 +01003430struct static_key rps_needed __read_mostly;
Eric Dumazetadc93002011-11-17 03:13:26 +00003431
Ben Hutchingsc4454772011-01-19 11:03:53 +00003432static struct rps_dev_flow *
3433set_rps_cpu(struct net_device *dev, struct sk_buff *skb,
3434 struct rps_dev_flow *rflow, u16 next_cpu)
3435{
Eric Dumazeta31196b2015-04-25 09:35:24 -07003436 if (next_cpu < nr_cpu_ids) {
Ben Hutchingsc4454772011-01-19 11:03:53 +00003437#ifdef CONFIG_RFS_ACCEL
3438 struct netdev_rx_queue *rxqueue;
3439 struct rps_dev_flow_table *flow_table;
3440 struct rps_dev_flow *old_rflow;
3441 u32 flow_id;
3442 u16 rxq_index;
3443 int rc;
3444
3445 /* Should we steer this flow to a different hardware queue? */
Ben Hutchings69a19ee2011-02-15 20:32:04 +00003446 if (!skb_rx_queue_recorded(skb) || !dev->rx_cpu_rmap ||
3447 !(dev->features & NETIF_F_NTUPLE))
Ben Hutchingsc4454772011-01-19 11:03:53 +00003448 goto out;
3449 rxq_index = cpu_rmap_lookup_index(dev->rx_cpu_rmap, next_cpu);
3450 if (rxq_index == skb_get_rx_queue(skb))
3451 goto out;
3452
3453 rxqueue = dev->_rx + rxq_index;
3454 flow_table = rcu_dereference(rxqueue->rps_flow_table);
3455 if (!flow_table)
3456 goto out;
Tom Herbert61b905d2014-03-24 15:34:47 -07003457 flow_id = skb_get_hash(skb) & flow_table->mask;
Ben Hutchingsc4454772011-01-19 11:03:53 +00003458 rc = dev->netdev_ops->ndo_rx_flow_steer(dev, skb,
3459 rxq_index, flow_id);
3460 if (rc < 0)
3461 goto out;
3462 old_rflow = rflow;
3463 rflow = &flow_table->flows[flow_id];
Ben Hutchingsc4454772011-01-19 11:03:53 +00003464 rflow->filter = rc;
3465 if (old_rflow->filter == rflow->filter)
3466 old_rflow->filter = RPS_NO_FILTER;
3467 out:
3468#endif
3469 rflow->last_qtail =
Ben Hutchings09994d12011-10-03 04:42:46 +00003470 per_cpu(softnet_data, next_cpu).input_queue_head;
Ben Hutchingsc4454772011-01-19 11:03:53 +00003471 }
3472
Ben Hutchings09994d12011-10-03 04:42:46 +00003473 rflow->cpu = next_cpu;
Ben Hutchingsc4454772011-01-19 11:03:53 +00003474 return rflow;
3475}
3476
Tom Herbert0a9627f2010-03-16 08:03:29 +00003477/*
3478 * get_rps_cpu is called from netif_receive_skb and returns the target
3479 * CPU from the RPS map of the receiving queue for a given skb.
Eric Dumazetb0e28f12010-04-15 00:14:07 -07003480 * rcu_read_lock must be held on entry.
Tom Herbert0a9627f2010-03-16 08:03:29 +00003481 */
Tom Herbertfec5e652010-04-16 16:01:27 -07003482static int get_rps_cpu(struct net_device *dev, struct sk_buff *skb,
3483 struct rps_dev_flow **rflowp)
Tom Herbert0a9627f2010-03-16 08:03:29 +00003484{
Eric Dumazet567e4b72015-02-06 12:59:01 -08003485 const struct rps_sock_flow_table *sock_flow_table;
3486 struct netdev_rx_queue *rxqueue = dev->_rx;
Tom Herbertfec5e652010-04-16 16:01:27 -07003487 struct rps_dev_flow_table *flow_table;
Eric Dumazet567e4b72015-02-06 12:59:01 -08003488 struct rps_map *map;
Tom Herbert0a9627f2010-03-16 08:03:29 +00003489 int cpu = -1;
Eric Dumazet567e4b72015-02-06 12:59:01 -08003490 u32 tcpu;
Tom Herbert61b905d2014-03-24 15:34:47 -07003491 u32 hash;
Tom Herbert0a9627f2010-03-16 08:03:29 +00003492
Tom Herbert0a9627f2010-03-16 08:03:29 +00003493 if (skb_rx_queue_recorded(skb)) {
3494 u16 index = skb_get_rx_queue(skb);
Eric Dumazet567e4b72015-02-06 12:59:01 -08003495
Ben Hutchings62fe0b42010-09-27 08:24:33 +00003496 if (unlikely(index >= dev->real_num_rx_queues)) {
3497 WARN_ONCE(dev->real_num_rx_queues > 1,
3498 "%s received packet on queue %u, but number "
3499 "of RX queues is %u\n",
3500 dev->name, index, dev->real_num_rx_queues);
Tom Herbert0a9627f2010-03-16 08:03:29 +00003501 goto done;
3502 }
Eric Dumazet567e4b72015-02-06 12:59:01 -08003503 rxqueue += index;
Tom Herbert0a9627f2010-03-16 08:03:29 +00003504 }
3505
Eric Dumazet567e4b72015-02-06 12:59:01 -08003506 /* Avoid computing hash if RFS/RPS is not active for this rxqueue */
3507
3508 flow_table = rcu_dereference(rxqueue->rps_flow_table);
3509 map = rcu_dereference(rxqueue->rps_map);
3510 if (!flow_table && !map)
3511 goto done;
3512
Changli Gao2d47b452010-08-17 19:00:56 +00003513 skb_reset_network_header(skb);
Tom Herbert61b905d2014-03-24 15:34:47 -07003514 hash = skb_get_hash(skb);
3515 if (!hash)
Tom Herbert0a9627f2010-03-16 08:03:29 +00003516 goto done;
Tom Herbert0a9627f2010-03-16 08:03:29 +00003517
Tom Herbertfec5e652010-04-16 16:01:27 -07003518 sock_flow_table = rcu_dereference(rps_sock_flow_table);
3519 if (flow_table && sock_flow_table) {
Tom Herbertfec5e652010-04-16 16:01:27 -07003520 struct rps_dev_flow *rflow;
Eric Dumazet567e4b72015-02-06 12:59:01 -08003521 u32 next_cpu;
3522 u32 ident;
Tom Herbertfec5e652010-04-16 16:01:27 -07003523
Eric Dumazet567e4b72015-02-06 12:59:01 -08003524 /* First check into global flow table if there is a match */
3525 ident = sock_flow_table->ents[hash & sock_flow_table->mask];
3526 if ((ident ^ hash) & ~rps_cpu_mask)
3527 goto try_rps;
3528
3529 next_cpu = ident & rps_cpu_mask;
3530
3531 /* OK, now we know there is a match,
3532 * we can look at the local (per receive queue) flow table
3533 */
Tom Herbert61b905d2014-03-24 15:34:47 -07003534 rflow = &flow_table->flows[hash & flow_table->mask];
Tom Herbertfec5e652010-04-16 16:01:27 -07003535 tcpu = rflow->cpu;
3536
Tom Herbertfec5e652010-04-16 16:01:27 -07003537 /*
3538 * If the desired CPU (where last recvmsg was done) is
3539 * different from current CPU (one in the rx-queue flow
3540 * table entry), switch if one of the following holds:
Eric Dumazeta31196b2015-04-25 09:35:24 -07003541 * - Current CPU is unset (>= nr_cpu_ids).
Tom Herbertfec5e652010-04-16 16:01:27 -07003542 * - Current CPU is offline.
3543 * - The current CPU's queue tail has advanced beyond the
3544 * last packet that was enqueued using this table entry.
3545 * This guarantees that all previous packets for the flow
3546 * have been dequeued, thus preserving in order delivery.
3547 */
3548 if (unlikely(tcpu != next_cpu) &&
Eric Dumazeta31196b2015-04-25 09:35:24 -07003549 (tcpu >= nr_cpu_ids || !cpu_online(tcpu) ||
Tom Herbertfec5e652010-04-16 16:01:27 -07003550 ((int)(per_cpu(softnet_data, tcpu).input_queue_head -
Tom Herbertbaefa312012-11-16 09:04:15 +00003551 rflow->last_qtail)) >= 0)) {
3552 tcpu = next_cpu;
Ben Hutchingsc4454772011-01-19 11:03:53 +00003553 rflow = set_rps_cpu(dev, skb, rflow, next_cpu);
Tom Herbertbaefa312012-11-16 09:04:15 +00003554 }
Ben Hutchingsc4454772011-01-19 11:03:53 +00003555
Eric Dumazeta31196b2015-04-25 09:35:24 -07003556 if (tcpu < nr_cpu_ids && cpu_online(tcpu)) {
Tom Herbertfec5e652010-04-16 16:01:27 -07003557 *rflowp = rflow;
3558 cpu = tcpu;
3559 goto done;
3560 }
3561 }
3562
Eric Dumazet567e4b72015-02-06 12:59:01 -08003563try_rps:
3564
Tom Herbert0a9627f2010-03-16 08:03:29 +00003565 if (map) {
Daniel Borkmann8fc54f62014-08-23 20:58:54 +02003566 tcpu = map->cpus[reciprocal_scale(hash, map->len)];
Tom Herbert0a9627f2010-03-16 08:03:29 +00003567 if (cpu_online(tcpu)) {
3568 cpu = tcpu;
3569 goto done;
3570 }
3571 }
3572
3573done:
Tom Herbert0a9627f2010-03-16 08:03:29 +00003574 return cpu;
3575}
3576
Ben Hutchingsc4454772011-01-19 11:03:53 +00003577#ifdef CONFIG_RFS_ACCEL
3578
3579/**
3580 * rps_may_expire_flow - check whether an RFS hardware filter may be removed
3581 * @dev: Device on which the filter was set
3582 * @rxq_index: RX queue index
3583 * @flow_id: Flow ID passed to ndo_rx_flow_steer()
3584 * @filter_id: Filter ID returned by ndo_rx_flow_steer()
3585 *
3586 * Drivers that implement ndo_rx_flow_steer() should periodically call
3587 * this function for each installed filter and remove the filters for
3588 * which it returns %true.
3589 */
3590bool rps_may_expire_flow(struct net_device *dev, u16 rxq_index,
3591 u32 flow_id, u16 filter_id)
3592{
3593 struct netdev_rx_queue *rxqueue = dev->_rx + rxq_index;
3594 struct rps_dev_flow_table *flow_table;
3595 struct rps_dev_flow *rflow;
3596 bool expire = true;
Eric Dumazeta31196b2015-04-25 09:35:24 -07003597 unsigned int cpu;
Ben Hutchingsc4454772011-01-19 11:03:53 +00003598
3599 rcu_read_lock();
3600 flow_table = rcu_dereference(rxqueue->rps_flow_table);
3601 if (flow_table && flow_id <= flow_table->mask) {
3602 rflow = &flow_table->flows[flow_id];
3603 cpu = ACCESS_ONCE(rflow->cpu);
Eric Dumazeta31196b2015-04-25 09:35:24 -07003604 if (rflow->filter == filter_id && cpu < nr_cpu_ids &&
Ben Hutchingsc4454772011-01-19 11:03:53 +00003605 ((int)(per_cpu(softnet_data, cpu).input_queue_head -
3606 rflow->last_qtail) <
3607 (int)(10 * flow_table->mask)))
3608 expire = false;
3609 }
3610 rcu_read_unlock();
3611 return expire;
3612}
3613EXPORT_SYMBOL(rps_may_expire_flow);
3614
3615#endif /* CONFIG_RFS_ACCEL */
3616
Tom Herbert0a9627f2010-03-16 08:03:29 +00003617/* Called from hardirq (IPI) context */
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003618static void rps_trigger_softirq(void *data)
Tom Herbert0a9627f2010-03-16 08:03:29 +00003619{
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003620 struct softnet_data *sd = data;
3621
Eric Dumazeteecfd7c2010-05-06 22:07:48 -07003622 ____napi_schedule(sd, &sd->backlog);
Changli Gaodee42872010-05-02 05:42:16 +00003623 sd->received_rps++;
Tom Herbert0a9627f2010-03-16 08:03:29 +00003624}
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003625
Tom Herbertfec5e652010-04-16 16:01:27 -07003626#endif /* CONFIG_RPS */
Tom Herbert0a9627f2010-03-16 08:03:29 +00003627
3628/*
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003629 * Check if this softnet_data structure is another cpu one
3630 * If yes, queue it to our IPI list and return 1
3631 * If no, return 0
3632 */
3633static int rps_ipi_queued(struct softnet_data *sd)
3634{
3635#ifdef CONFIG_RPS
Christoph Lameter903ceff2014-08-17 12:30:35 -05003636 struct softnet_data *mysd = this_cpu_ptr(&softnet_data);
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003637
3638 if (sd != mysd) {
3639 sd->rps_ipi_next = mysd->rps_ipi_list;
3640 mysd->rps_ipi_list = sd;
3641
3642 __raise_softirq_irqoff(NET_RX_SOFTIRQ);
3643 return 1;
3644 }
3645#endif /* CONFIG_RPS */
3646 return 0;
3647}
3648
Willem de Bruijn99bbc702013-05-20 04:02:32 +00003649#ifdef CONFIG_NET_FLOW_LIMIT
3650int netdev_flow_limit_table_len __read_mostly = (1 << 12);
3651#endif
3652
3653static bool skb_flow_limit(struct sk_buff *skb, unsigned int qlen)
3654{
3655#ifdef CONFIG_NET_FLOW_LIMIT
3656 struct sd_flow_limit *fl;
3657 struct softnet_data *sd;
3658 unsigned int old_flow, new_flow;
3659
3660 if (qlen < (netdev_max_backlog >> 1))
3661 return false;
3662
Christoph Lameter903ceff2014-08-17 12:30:35 -05003663 sd = this_cpu_ptr(&softnet_data);
Willem de Bruijn99bbc702013-05-20 04:02:32 +00003664
3665 rcu_read_lock();
3666 fl = rcu_dereference(sd->flow_limit);
3667 if (fl) {
Tom Herbert3958afa1b2013-12-15 22:12:06 -08003668 new_flow = skb_get_hash(skb) & (fl->num_buckets - 1);
Willem de Bruijn99bbc702013-05-20 04:02:32 +00003669 old_flow = fl->history[fl->history_head];
3670 fl->history[fl->history_head] = new_flow;
3671
3672 fl->history_head++;
3673 fl->history_head &= FLOW_LIMIT_HISTORY - 1;
3674
3675 if (likely(fl->buckets[old_flow]))
3676 fl->buckets[old_flow]--;
3677
3678 if (++fl->buckets[new_flow] > (FLOW_LIMIT_HISTORY >> 1)) {
3679 fl->count++;
3680 rcu_read_unlock();
3681 return true;
3682 }
3683 }
3684 rcu_read_unlock();
3685#endif
3686 return false;
3687}
3688
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003689/*
Tom Herbert0a9627f2010-03-16 08:03:29 +00003690 * enqueue_to_backlog is called to queue an skb to a per CPU backlog
3691 * queue (may be a remote CPU queue).
3692 */
Tom Herbertfec5e652010-04-16 16:01:27 -07003693static int enqueue_to_backlog(struct sk_buff *skb, int cpu,
3694 unsigned int *qtail)
Tom Herbert0a9627f2010-03-16 08:03:29 +00003695{
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003696 struct softnet_data *sd;
Tom Herbert0a9627f2010-03-16 08:03:29 +00003697 unsigned long flags;
Willem de Bruijn99bbc702013-05-20 04:02:32 +00003698 unsigned int qlen;
Tom Herbert0a9627f2010-03-16 08:03:29 +00003699
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003700 sd = &per_cpu(softnet_data, cpu);
Tom Herbert0a9627f2010-03-16 08:03:29 +00003701
3702 local_irq_save(flags);
Tom Herbert0a9627f2010-03-16 08:03:29 +00003703
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003704 rps_lock(sd);
Julian Anastasove9e4dd32015-07-09 09:59:09 +03003705 if (!netif_running(skb->dev))
3706 goto drop;
Willem de Bruijn99bbc702013-05-20 04:02:32 +00003707 qlen = skb_queue_len(&sd->input_pkt_queue);
3708 if (qlen <= netdev_max_backlog && !skb_flow_limit(skb, qlen)) {
Li RongQinge008f3f2014-12-08 09:42:55 +08003709 if (qlen) {
Tom Herbert0a9627f2010-03-16 08:03:29 +00003710enqueue:
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003711 __skb_queue_tail(&sd->input_pkt_queue, skb);
Tom Herbert76cc8b12010-05-20 18:37:59 +00003712 input_queue_tail_incr_save(sd, qtail);
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003713 rps_unlock(sd);
Changli Gao152102c2010-03-30 20:16:22 +00003714 local_irq_restore(flags);
Tom Herbert0a9627f2010-03-16 08:03:29 +00003715 return NET_RX_SUCCESS;
3716 }
3717
Eric Dumazetebda37c22010-05-06 23:51:21 +00003718 /* Schedule NAPI for backlog device
3719 * We can use non atomic operation since we own the queue lock
3720 */
3721 if (!__test_and_set_bit(NAPI_STATE_SCHED, &sd->backlog.state)) {
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003722 if (!rps_ipi_queued(sd))
Eric Dumazeteecfd7c2010-05-06 22:07:48 -07003723 ____napi_schedule(sd, &sd->backlog);
Tom Herbert0a9627f2010-03-16 08:03:29 +00003724 }
3725 goto enqueue;
3726 }
3727
Julian Anastasove9e4dd32015-07-09 09:59:09 +03003728drop:
Changli Gaodee42872010-05-02 05:42:16 +00003729 sd->dropped++;
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003730 rps_unlock(sd);
Tom Herbert0a9627f2010-03-16 08:03:29 +00003731
Tom Herbert0a9627f2010-03-16 08:03:29 +00003732 local_irq_restore(flags);
3733
Eric Dumazetcaf586e2010-09-30 21:06:55 +00003734 atomic_long_inc(&skb->dev->rx_dropped);
Tom Herbert0a9627f2010-03-16 08:03:29 +00003735 kfree_skb(skb);
3736 return NET_RX_DROP;
3737}
Linus Torvalds1da177e2005-04-16 15:20:36 -07003738
Ben Hutchingsae78dbf2014-01-10 22:17:24 +00003739static int netif_rx_internal(struct sk_buff *skb)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003740{
Eric Dumazetb0e28f12010-04-15 00:14:07 -07003741 int ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003742
Eric Dumazet588f0332011-11-15 04:12:55 +00003743 net_timestamp_check(netdev_tstamp_prequeue, skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003744
Koki Sanagicf66ba52010-08-23 18:45:02 +09003745 trace_netif_rx(skb);
Eric Dumazetdf334542010-03-24 19:13:54 +00003746#ifdef CONFIG_RPS
Ingo Molnarc5905af2012-02-24 08:31:31 +01003747 if (static_key_false(&rps_needed)) {
Tom Herbertfec5e652010-04-16 16:01:27 -07003748 struct rps_dev_flow voidflow, *rflow = &voidflow;
Eric Dumazetb0e28f12010-04-15 00:14:07 -07003749 int cpu;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003750
Changli Gaocece1942010-08-07 20:35:43 -07003751 preempt_disable();
Eric Dumazetb0e28f12010-04-15 00:14:07 -07003752 rcu_read_lock();
Tom Herbertfec5e652010-04-16 16:01:27 -07003753
3754 cpu = get_rps_cpu(skb->dev, skb, &rflow);
Eric Dumazetb0e28f12010-04-15 00:14:07 -07003755 if (cpu < 0)
3756 cpu = smp_processor_id();
Tom Herbertfec5e652010-04-16 16:01:27 -07003757
3758 ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail);
3759
Eric Dumazetb0e28f12010-04-15 00:14:07 -07003760 rcu_read_unlock();
Changli Gaocece1942010-08-07 20:35:43 -07003761 preempt_enable();
Eric Dumazetadc93002011-11-17 03:13:26 +00003762 } else
3763#endif
Tom Herbertfec5e652010-04-16 16:01:27 -07003764 {
3765 unsigned int qtail;
3766 ret = enqueue_to_backlog(skb, get_cpu(), &qtail);
3767 put_cpu();
3768 }
Eric Dumazetb0e28f12010-04-15 00:14:07 -07003769 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003770}
Ben Hutchingsae78dbf2014-01-10 22:17:24 +00003771
3772/**
3773 * netif_rx - post buffer to the network code
3774 * @skb: buffer to post
3775 *
3776 * This function receives a packet from a device driver and queues it for
3777 * the upper (protocol) levels to process. It always succeeds. The buffer
3778 * may be dropped during processing for congestion control or by the
3779 * protocol layers.
3780 *
3781 * return values:
3782 * NET_RX_SUCCESS (no congestion)
3783 * NET_RX_DROP (packet was dropped)
3784 *
3785 */
3786
3787int netif_rx(struct sk_buff *skb)
3788{
3789 trace_netif_rx_entry(skb);
3790
3791 return netif_rx_internal(skb);
3792}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07003793EXPORT_SYMBOL(netif_rx);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003794
3795int netif_rx_ni(struct sk_buff *skb)
3796{
3797 int err;
3798
Ben Hutchingsae78dbf2014-01-10 22:17:24 +00003799 trace_netif_rx_ni_entry(skb);
3800
Linus Torvalds1da177e2005-04-16 15:20:36 -07003801 preempt_disable();
Ben Hutchingsae78dbf2014-01-10 22:17:24 +00003802 err = netif_rx_internal(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003803 if (local_softirq_pending())
3804 do_softirq();
3805 preempt_enable();
3806
3807 return err;
3808}
Linus Torvalds1da177e2005-04-16 15:20:36 -07003809EXPORT_SYMBOL(netif_rx_ni);
3810
Linus Torvalds1da177e2005-04-16 15:20:36 -07003811static void net_tx_action(struct softirq_action *h)
3812{
Christoph Lameter903ceff2014-08-17 12:30:35 -05003813 struct softnet_data *sd = this_cpu_ptr(&softnet_data);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003814
3815 if (sd->completion_queue) {
3816 struct sk_buff *clist;
3817
3818 local_irq_disable();
3819 clist = sd->completion_queue;
3820 sd->completion_queue = NULL;
3821 local_irq_enable();
3822
3823 while (clist) {
3824 struct sk_buff *skb = clist;
3825 clist = clist->next;
3826
Ilpo Järvinen547b7922008-07-25 21:43:18 -07003827 WARN_ON(atomic_read(&skb->users));
Eric Dumazete6247022013-12-05 04:45:08 -08003828 if (likely(get_kfree_skb_cb(skb)->reason == SKB_REASON_CONSUMED))
3829 trace_consume_skb(skb);
3830 else
3831 trace_kfree_skb(skb, net_tx_action);
Jesper Dangaard Brouer15fad712016-02-08 13:15:04 +01003832
3833 if (skb->fclone != SKB_FCLONE_UNAVAILABLE)
3834 __kfree_skb(skb);
3835 else
3836 __kfree_skb_defer(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003837 }
Jesper Dangaard Brouer15fad712016-02-08 13:15:04 +01003838
3839 __kfree_skb_flush();
Linus Torvalds1da177e2005-04-16 15:20:36 -07003840 }
3841
3842 if (sd->output_queue) {
David S. Miller37437bb2008-07-16 02:15:04 -07003843 struct Qdisc *head;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003844
3845 local_irq_disable();
3846 head = sd->output_queue;
3847 sd->output_queue = NULL;
Changli Gaoa9cbd582010-04-26 23:06:24 +00003848 sd->output_queue_tailp = &sd->output_queue;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003849 local_irq_enable();
3850
3851 while (head) {
David S. Miller37437bb2008-07-16 02:15:04 -07003852 struct Qdisc *q = head;
3853 spinlock_t *root_lock;
3854
Linus Torvalds1da177e2005-04-16 15:20:36 -07003855 head = head->next_sched;
3856
David S. Miller5fb66222008-08-02 20:02:43 -07003857 root_lock = qdisc_lock(q);
David S. Miller37437bb2008-07-16 02:15:04 -07003858 if (spin_trylock(root_lock)) {
Peter Zijlstra4e857c52014-03-17 18:06:10 +01003859 smp_mb__before_atomic();
Jarek Poplawskidef82a12008-08-17 21:54:43 -07003860 clear_bit(__QDISC_STATE_SCHED,
3861 &q->state);
David S. Miller37437bb2008-07-16 02:15:04 -07003862 qdisc_run(q);
3863 spin_unlock(root_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003864 } else {
David S. Miller195648b2008-08-19 04:00:36 -07003865 if (!test_bit(__QDISC_STATE_DEACTIVATED,
Jarek Poplawskie8a83e12008-09-07 18:41:21 -07003866 &q->state)) {
David S. Miller195648b2008-08-19 04:00:36 -07003867 __netif_reschedule(q);
Jarek Poplawskie8a83e12008-09-07 18:41:21 -07003868 } else {
Peter Zijlstra4e857c52014-03-17 18:06:10 +01003869 smp_mb__before_atomic();
Jarek Poplawskie8a83e12008-09-07 18:41:21 -07003870 clear_bit(__QDISC_STATE_SCHED,
3871 &q->state);
3872 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07003873 }
3874 }
3875 }
3876}
3877
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00003878#if (defined(CONFIG_BRIDGE) || defined(CONFIG_BRIDGE_MODULE)) && \
3879 (defined(CONFIG_ATM_LANE) || defined(CONFIG_ATM_LANE_MODULE))
Michał Mirosławda678292009-06-05 05:35:28 +00003880/* This hook is defined here for ATM LANE */
3881int (*br_fdb_test_addr_hook)(struct net_device *dev,
3882 unsigned char *addr) __read_mostly;
Stephen Hemminger4fb019a2009-09-11 11:50:08 -07003883EXPORT_SYMBOL_GPL(br_fdb_test_addr_hook);
Michał Mirosławda678292009-06-05 05:35:28 +00003884#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07003885
Daniel Borkmann1f211a12016-01-07 22:29:47 +01003886static inline struct sk_buff *
3887sch_handle_ingress(struct sk_buff *skb, struct packet_type **pt_prev, int *ret,
3888 struct net_device *orig_dev)
Herbert Xuf697c3e2007-10-14 00:38:47 -07003889{
Daniel Borkmanne7582ba2015-05-19 22:33:25 +02003890#ifdef CONFIG_NET_CLS_ACT
Daniel Borkmannd2788d32015-05-09 22:51:32 +02003891 struct tcf_proto *cl = rcu_dereference_bh(skb->dev->ingress_cl_list);
3892 struct tcf_result cl_res;
Eric Dumazet24824a02010-10-02 06:11:55 +00003893
Daniel Borkmannc9e99fd2015-05-09 22:51:31 +02003894 /* If there's at least one ingress present somewhere (so
3895 * we get here via enabled static key), remaining devices
3896 * that are not configured with an ingress qdisc will bail
Daniel Borkmannd2788d32015-05-09 22:51:32 +02003897 * out here.
Daniel Borkmannc9e99fd2015-05-09 22:51:31 +02003898 */
Daniel Borkmannd2788d32015-05-09 22:51:32 +02003899 if (!cl)
Daniel Borkmann45771392015-04-10 23:07:54 +02003900 return skb;
Herbert Xuf697c3e2007-10-14 00:38:47 -07003901 if (*pt_prev) {
3902 *ret = deliver_skb(skb, *pt_prev, orig_dev);
3903 *pt_prev = NULL;
Herbert Xuf697c3e2007-10-14 00:38:47 -07003904 }
3905
Florian Westphal33654952015-05-14 00:36:28 +02003906 qdisc_skb_cb(skb)->pkt_len = skb->len;
Daniel Borkmannc9e99fd2015-05-09 22:51:31 +02003907 skb->tc_verd = SET_TC_AT(skb->tc_verd, AT_INGRESS);
Eric Dumazet24ea5912015-07-06 05:18:03 -07003908 qdisc_bstats_cpu_update(cl->q, skb);
Daniel Borkmannc9e99fd2015-05-09 22:51:31 +02003909
Daniel Borkmann3b3ae882015-08-26 23:00:06 +02003910 switch (tc_classify(skb, cl, &cl_res, false)) {
Daniel Borkmannd2788d32015-05-09 22:51:32 +02003911 case TC_ACT_OK:
3912 case TC_ACT_RECLASSIFY:
3913 skb->tc_index = TC_H_MIN(cl_res.classid);
3914 break;
3915 case TC_ACT_SHOT:
Eric Dumazet24ea5912015-07-06 05:18:03 -07003916 qdisc_qstats_cpu_drop(cl->q);
Daniel Borkmannd2788d32015-05-09 22:51:32 +02003917 case TC_ACT_STOLEN:
3918 case TC_ACT_QUEUED:
3919 kfree_skb(skb);
3920 return NULL;
Alexei Starovoitov27b29f62015-09-15 23:05:43 -07003921 case TC_ACT_REDIRECT:
3922 /* skb_mac_header check was done by cls/act_bpf, so
3923 * we can safely push the L2 header back before
3924 * redirecting to another netdev
3925 */
3926 __skb_push(skb, skb->mac_len);
3927 skb_do_redirect(skb);
3928 return NULL;
Daniel Borkmannd2788d32015-05-09 22:51:32 +02003929 default:
3930 break;
Herbert Xuf697c3e2007-10-14 00:38:47 -07003931 }
Daniel Borkmanne7582ba2015-05-19 22:33:25 +02003932#endif /* CONFIG_NET_CLS_ACT */
Herbert Xuf697c3e2007-10-14 00:38:47 -07003933 return skb;
3934}
Linus Torvalds1da177e2005-04-16 15:20:36 -07003935
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00003936/**
3937 * netdev_rx_handler_register - register receive handler
3938 * @dev: device to register a handler for
3939 * @rx_handler: receive handler to register
Jiri Pirko93e2c322010-06-10 03:34:59 +00003940 * @rx_handler_data: data pointer that is used by rx handler
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00003941 *
Masanari Iidae2278672014-02-18 22:54:36 +09003942 * Register a receive handler for a device. This handler will then be
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00003943 * called from __netif_receive_skb. A negative errno code is returned
3944 * on a failure.
3945 *
3946 * The caller must hold the rtnl_mutex.
Jiri Pirko8a4eb572011-03-12 03:14:39 +00003947 *
3948 * For a general description of rx_handler, see enum rx_handler_result.
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00003949 */
3950int netdev_rx_handler_register(struct net_device *dev,
Jiri Pirko93e2c322010-06-10 03:34:59 +00003951 rx_handler_func_t *rx_handler,
3952 void *rx_handler_data)
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00003953{
3954 ASSERT_RTNL();
3955
3956 if (dev->rx_handler)
3957 return -EBUSY;
3958
Eric Dumazet00cfec32013-03-29 03:01:22 +00003959 /* Note: rx_handler_data must be set before rx_handler */
Jiri Pirko93e2c322010-06-10 03:34:59 +00003960 rcu_assign_pointer(dev->rx_handler_data, rx_handler_data);
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00003961 rcu_assign_pointer(dev->rx_handler, rx_handler);
3962
3963 return 0;
3964}
3965EXPORT_SYMBOL_GPL(netdev_rx_handler_register);
3966
3967/**
3968 * netdev_rx_handler_unregister - unregister receive handler
3969 * @dev: device to unregister a handler from
3970 *
Kusanagi Kouichi166ec362013-03-18 02:59:52 +00003971 * Unregister a receive handler from a device.
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00003972 *
3973 * The caller must hold the rtnl_mutex.
3974 */
3975void netdev_rx_handler_unregister(struct net_device *dev)
3976{
3977
3978 ASSERT_RTNL();
Stephen Hemmingera9b3cd72011-08-01 16:19:00 +00003979 RCU_INIT_POINTER(dev->rx_handler, NULL);
Eric Dumazet00cfec32013-03-29 03:01:22 +00003980 /* a reader seeing a non NULL rx_handler in a rcu_read_lock()
3981 * section has a guarantee to see a non NULL rx_handler_data
3982 * as well.
3983 */
3984 synchronize_net();
Stephen Hemmingera9b3cd72011-08-01 16:19:00 +00003985 RCU_INIT_POINTER(dev->rx_handler_data, NULL);
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00003986}
3987EXPORT_SYMBOL_GPL(netdev_rx_handler_unregister);
3988
Mel Gormanb4b9e352012-07-31 16:44:26 -07003989/*
3990 * Limit the use of PFMEMALLOC reserves to those protocols that implement
3991 * the special handling of PFMEMALLOC skbs.
3992 */
3993static bool skb_pfmemalloc_protocol(struct sk_buff *skb)
3994{
3995 switch (skb->protocol) {
Joe Perches2b8837a2014-03-12 10:04:17 -07003996 case htons(ETH_P_ARP):
3997 case htons(ETH_P_IP):
3998 case htons(ETH_P_IPV6):
3999 case htons(ETH_P_8021Q):
4000 case htons(ETH_P_8021AD):
Mel Gormanb4b9e352012-07-31 16:44:26 -07004001 return true;
4002 default:
4003 return false;
4004 }
4005}
4006
Pablo Neirae687ad62015-05-13 18:19:38 +02004007static inline int nf_ingress(struct sk_buff *skb, struct packet_type **pt_prev,
4008 int *ret, struct net_device *orig_dev)
4009{
Daniel Borkmanne7582ba2015-05-19 22:33:25 +02004010#ifdef CONFIG_NETFILTER_INGRESS
Pablo Neirae687ad62015-05-13 18:19:38 +02004011 if (nf_hook_ingress_active(skb)) {
4012 if (*pt_prev) {
4013 *ret = deliver_skb(skb, *pt_prev, orig_dev);
4014 *pt_prev = NULL;
4015 }
4016
4017 return nf_hook_ingress(skb);
4018 }
Daniel Borkmanne7582ba2015-05-19 22:33:25 +02004019#endif /* CONFIG_NETFILTER_INGRESS */
Pablo Neirae687ad62015-05-13 18:19:38 +02004020 return 0;
4021}
Pablo Neirae687ad62015-05-13 18:19:38 +02004022
David S. Miller9754e292013-02-14 15:57:38 -05004023static int __netif_receive_skb_core(struct sk_buff *skb, bool pfmemalloc)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004024{
4025 struct packet_type *ptype, *pt_prev;
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00004026 rx_handler_func_t *rx_handler;
David S. Millerf2ccd8f2005-08-09 19:34:12 -07004027 struct net_device *orig_dev;
Jiri Pirko8a4eb572011-03-12 03:14:39 +00004028 bool deliver_exact = false;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004029 int ret = NET_RX_DROP;
Al Viro252e3342006-11-14 20:48:11 -08004030 __be16 type;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004031
Eric Dumazet588f0332011-11-15 04:12:55 +00004032 net_timestamp_check(!netdev_tstamp_prequeue, skb);
Eric Dumazet81bbb3d2009-09-30 16:42:42 -07004033
Koki Sanagicf66ba52010-08-23 18:45:02 +09004034 trace_netif_receive_skb(skb);
Patrick McHardy9b22ea52008-11-04 14:49:57 -08004035
Joe Eykholtcc9bd5c2008-07-02 18:22:00 -07004036 orig_dev = skb->dev;
Jiri Pirko1765a572011-02-12 06:48:36 +00004037
Arnaldo Carvalho de Meloc1d2bbe2007-04-10 20:45:18 -07004038 skb_reset_network_header(skb);
Eric Dumazetfda55ec2013-01-07 09:28:21 +00004039 if (!skb_transport_header_was_set(skb))
4040 skb_reset_transport_header(skb);
Jiri Pirko0b5c9db2011-06-10 06:56:58 +00004041 skb_reset_mac_len(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004042
4043 pt_prev = NULL;
4044
David S. Miller63d8ea72011-02-28 10:48:59 -08004045another_round:
David S. Millerb6858172012-07-23 16:27:54 -07004046 skb->skb_iif = skb->dev->ifindex;
David S. Miller63d8ea72011-02-28 10:48:59 -08004047
4048 __this_cpu_inc(softnet_data.processed);
4049
Patrick McHardy8ad227f2013-04-19 02:04:31 +00004050 if (skb->protocol == cpu_to_be16(ETH_P_8021Q) ||
4051 skb->protocol == cpu_to_be16(ETH_P_8021AD)) {
Vlad Yasevich0d5501c2014-08-08 14:42:13 -04004052 skb = skb_vlan_untag(skb);
Jiri Pirkobcc6d472011-04-07 19:48:33 +00004053 if (unlikely(!skb))
Julian Anastasov2c17d272015-07-09 09:59:10 +03004054 goto out;
Jiri Pirkobcc6d472011-04-07 19:48:33 +00004055 }
4056
Linus Torvalds1da177e2005-04-16 15:20:36 -07004057#ifdef CONFIG_NET_CLS_ACT
4058 if (skb->tc_verd & TC_NCLS) {
4059 skb->tc_verd = CLR_TC_NCLS(skb->tc_verd);
4060 goto ncls;
4061 }
4062#endif
4063
David S. Miller9754e292013-02-14 15:57:38 -05004064 if (pfmemalloc)
Mel Gormanb4b9e352012-07-31 16:44:26 -07004065 goto skip_taps;
4066
Linus Torvalds1da177e2005-04-16 15:20:36 -07004067 list_for_each_entry_rcu(ptype, &ptype_all, list) {
Salam Noureddine7866a622015-01-27 11:35:48 -08004068 if (pt_prev)
4069 ret = deliver_skb(skb, pt_prev, orig_dev);
4070 pt_prev = ptype;
4071 }
4072
4073 list_for_each_entry_rcu(ptype, &skb->dev->ptype_all, list) {
4074 if (pt_prev)
4075 ret = deliver_skb(skb, pt_prev, orig_dev);
4076 pt_prev = ptype;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004077 }
4078
Mel Gormanb4b9e352012-07-31 16:44:26 -07004079skip_taps:
Pablo Neira1cf519002015-05-13 18:19:37 +02004080#ifdef CONFIG_NET_INGRESS
Daniel Borkmann45771392015-04-10 23:07:54 +02004081 if (static_key_false(&ingress_needed)) {
Daniel Borkmann1f211a12016-01-07 22:29:47 +01004082 skb = sch_handle_ingress(skb, &pt_prev, &ret, orig_dev);
Daniel Borkmann45771392015-04-10 23:07:54 +02004083 if (!skb)
Julian Anastasov2c17d272015-07-09 09:59:10 +03004084 goto out;
Pablo Neirae687ad62015-05-13 18:19:38 +02004085
4086 if (nf_ingress(skb, &pt_prev, &ret, orig_dev) < 0)
Julian Anastasov2c17d272015-07-09 09:59:10 +03004087 goto out;
Daniel Borkmann45771392015-04-10 23:07:54 +02004088 }
Pablo Neira1cf519002015-05-13 18:19:37 +02004089#endif
4090#ifdef CONFIG_NET_CLS_ACT
Daniel Borkmann45771392015-04-10 23:07:54 +02004091 skb->tc_verd = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004092ncls:
4093#endif
David S. Miller9754e292013-02-14 15:57:38 -05004094 if (pfmemalloc && !skb_pfmemalloc_protocol(skb))
Mel Gormanb4b9e352012-07-31 16:44:26 -07004095 goto drop;
4096
Jiri Pirkodf8a39d2015-01-13 17:13:44 +01004097 if (skb_vlan_tag_present(skb)) {
John Fastabend24257172011-10-10 09:16:41 +00004098 if (pt_prev) {
4099 ret = deliver_skb(skb, pt_prev, orig_dev);
4100 pt_prev = NULL;
4101 }
Florian Zumbiehl48cc32d32012-10-07 15:51:58 +00004102 if (vlan_do_receive(&skb))
John Fastabend24257172011-10-10 09:16:41 +00004103 goto another_round;
4104 else if (unlikely(!skb))
Julian Anastasov2c17d272015-07-09 09:59:10 +03004105 goto out;
John Fastabend24257172011-10-10 09:16:41 +00004106 }
4107
Florian Zumbiehl48cc32d32012-10-07 15:51:58 +00004108 rx_handler = rcu_dereference(skb->dev->rx_handler);
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00004109 if (rx_handler) {
4110 if (pt_prev) {
4111 ret = deliver_skb(skb, pt_prev, orig_dev);
4112 pt_prev = NULL;
4113 }
Jiri Pirko8a4eb572011-03-12 03:14:39 +00004114 switch (rx_handler(&skb)) {
4115 case RX_HANDLER_CONSUMED:
Cristian Bercaru3bc1b1a2013-03-08 07:03:38 +00004116 ret = NET_RX_SUCCESS;
Julian Anastasov2c17d272015-07-09 09:59:10 +03004117 goto out;
Jiri Pirko8a4eb572011-03-12 03:14:39 +00004118 case RX_HANDLER_ANOTHER:
David S. Miller63d8ea72011-02-28 10:48:59 -08004119 goto another_round;
Jiri Pirko8a4eb572011-03-12 03:14:39 +00004120 case RX_HANDLER_EXACT:
4121 deliver_exact = true;
4122 case RX_HANDLER_PASS:
4123 break;
4124 default:
4125 BUG();
4126 }
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00004127 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07004128
Jiri Pirkodf8a39d2015-01-13 17:13:44 +01004129 if (unlikely(skb_vlan_tag_present(skb))) {
4130 if (skb_vlan_tag_get_id(skb))
Eric Dumazetd4b812d2013-07-18 07:19:26 -07004131 skb->pkt_type = PACKET_OTHERHOST;
4132 /* Note: we might in the future use prio bits
4133 * and set skb->priority like in vlan_do_receive()
4134 * For the time being, just ignore Priority Code Point
4135 */
4136 skb->vlan_tci = 0;
4137 }
Florian Zumbiehl48cc32d32012-10-07 15:51:58 +00004138
Linus Torvalds1da177e2005-04-16 15:20:36 -07004139 type = skb->protocol;
Salam Noureddine7866a622015-01-27 11:35:48 -08004140
4141 /* deliver only exact match when indicated */
4142 if (likely(!deliver_exact)) {
4143 deliver_ptype_list_skb(skb, &pt_prev, orig_dev, type,
4144 &ptype_base[ntohs(type) &
4145 PTYPE_HASH_MASK]);
4146 }
4147
4148 deliver_ptype_list_skb(skb, &pt_prev, orig_dev, type,
4149 &orig_dev->ptype_specific);
4150
4151 if (unlikely(skb->dev != orig_dev)) {
4152 deliver_ptype_list_skb(skb, &pt_prev, orig_dev, type,
4153 &skb->dev->ptype_specific);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004154 }
4155
4156 if (pt_prev) {
Michael S. Tsirkin1080e512012-07-20 09:23:17 +00004157 if (unlikely(skb_orphan_frags(skb, GFP_ATOMIC)))
Michael S. Tsirkin0e698bf2012-09-15 22:44:16 +00004158 goto drop;
Michael S. Tsirkin1080e512012-07-20 09:23:17 +00004159 else
4160 ret = pt_prev->func(skb, skb->dev, pt_prev, orig_dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004161 } else {
Mel Gormanb4b9e352012-07-31 16:44:26 -07004162drop:
Jarod Wilson6e7333d2016-02-01 18:51:05 -05004163 if (!deliver_exact)
4164 atomic_long_inc(&skb->dev->rx_dropped);
4165 else
4166 atomic_long_inc(&skb->dev->rx_nohandler);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004167 kfree_skb(skb);
4168 /* Jamal, now you will not able to escape explaining
4169 * me how you were going to use this. :-)
4170 */
4171 ret = NET_RX_DROP;
4172 }
4173
Julian Anastasov2c17d272015-07-09 09:59:10 +03004174out:
David S. Miller9754e292013-02-14 15:57:38 -05004175 return ret;
4176}
4177
4178static int __netif_receive_skb(struct sk_buff *skb)
4179{
4180 int ret;
4181
4182 if (sk_memalloc_socks() && skb_pfmemalloc(skb)) {
4183 unsigned long pflags = current->flags;
4184
4185 /*
4186 * PFMEMALLOC skbs are special, they should
4187 * - be delivered to SOCK_MEMALLOC sockets only
4188 * - stay away from userspace
4189 * - have bounded memory usage
4190 *
4191 * Use PF_MEMALLOC as this saves us from propagating the allocation
4192 * context down to all allocation sites.
4193 */
4194 current->flags |= PF_MEMALLOC;
4195 ret = __netif_receive_skb_core(skb, true);
4196 tsk_restore_flags(current, pflags, PF_MEMALLOC);
4197 } else
4198 ret = __netif_receive_skb_core(skb, false);
4199
Linus Torvalds1da177e2005-04-16 15:20:36 -07004200 return ret;
4201}
Tom Herbert0a9627f2010-03-16 08:03:29 +00004202
Ben Hutchingsae78dbf2014-01-10 22:17:24 +00004203static int netif_receive_skb_internal(struct sk_buff *skb)
Tom Herbert0a9627f2010-03-16 08:03:29 +00004204{
Julian Anastasov2c17d272015-07-09 09:59:10 +03004205 int ret;
4206
Eric Dumazet588f0332011-11-15 04:12:55 +00004207 net_timestamp_check(netdev_tstamp_prequeue, skb);
Eric Dumazet3b098e22010-05-15 23:57:10 -07004208
Richard Cochranc1f19b52010-07-17 08:49:36 +00004209 if (skb_defer_rx_timestamp(skb))
4210 return NET_RX_SUCCESS;
4211
Julian Anastasov2c17d272015-07-09 09:59:10 +03004212 rcu_read_lock();
4213
Eric Dumazetdf334542010-03-24 19:13:54 +00004214#ifdef CONFIG_RPS
Ingo Molnarc5905af2012-02-24 08:31:31 +01004215 if (static_key_false(&rps_needed)) {
Eric Dumazet3b098e22010-05-15 23:57:10 -07004216 struct rps_dev_flow voidflow, *rflow = &voidflow;
Julian Anastasov2c17d272015-07-09 09:59:10 +03004217 int cpu = get_rps_cpu(skb->dev, skb, &rflow);
Tom Herbertfec5e652010-04-16 16:01:27 -07004218
Eric Dumazet3b098e22010-05-15 23:57:10 -07004219 if (cpu >= 0) {
4220 ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail);
4221 rcu_read_unlock();
Eric Dumazetadc93002011-11-17 03:13:26 +00004222 return ret;
Eric Dumazet3b098e22010-05-15 23:57:10 -07004223 }
Tom Herbertfec5e652010-04-16 16:01:27 -07004224 }
Tom Herbert1e94d722010-03-18 17:45:44 -07004225#endif
Julian Anastasov2c17d272015-07-09 09:59:10 +03004226 ret = __netif_receive_skb(skb);
4227 rcu_read_unlock();
4228 return ret;
Tom Herbert0a9627f2010-03-16 08:03:29 +00004229}
Ben Hutchingsae78dbf2014-01-10 22:17:24 +00004230
4231/**
4232 * netif_receive_skb - process receive buffer from network
4233 * @skb: buffer to process
4234 *
4235 * netif_receive_skb() is the main receive data processing function.
4236 * It always succeeds. The buffer may be dropped during processing
4237 * for congestion control or by the protocol layers.
4238 *
4239 * This function may only be called from softirq context and interrupts
4240 * should be enabled.
4241 *
4242 * Return values (usually ignored):
4243 * NET_RX_SUCCESS: no congestion
4244 * NET_RX_DROP: packet was dropped
4245 */
Eric W. Biederman04eb4482015-09-15 20:04:15 -05004246int netif_receive_skb(struct sk_buff *skb)
Ben Hutchingsae78dbf2014-01-10 22:17:24 +00004247{
4248 trace_netif_receive_skb_entry(skb);
4249
4250 return netif_receive_skb_internal(skb);
4251}
Eric W. Biederman04eb4482015-09-15 20:04:15 -05004252EXPORT_SYMBOL(netif_receive_skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004253
Eric Dumazet88751272010-04-19 05:07:33 +00004254/* Network device is going away, flush any packets still pending
4255 * Called with irqs disabled.
4256 */
Changli Gao152102c2010-03-30 20:16:22 +00004257static void flush_backlog(void *arg)
Stephen Hemminger6e583ce2008-08-03 21:29:57 -07004258{
Changli Gao152102c2010-03-30 20:16:22 +00004259 struct net_device *dev = arg;
Christoph Lameter903ceff2014-08-17 12:30:35 -05004260 struct softnet_data *sd = this_cpu_ptr(&softnet_data);
Stephen Hemminger6e583ce2008-08-03 21:29:57 -07004261 struct sk_buff *skb, *tmp;
4262
Eric Dumazete36fa2f2010-04-19 21:17:14 +00004263 rps_lock(sd);
Changli Gao6e7676c2010-04-27 15:07:33 -07004264 skb_queue_walk_safe(&sd->input_pkt_queue, skb, tmp) {
Stephen Hemminger6e583ce2008-08-03 21:29:57 -07004265 if (skb->dev == dev) {
Eric Dumazete36fa2f2010-04-19 21:17:14 +00004266 __skb_unlink(skb, &sd->input_pkt_queue);
Stephen Hemminger6e583ce2008-08-03 21:29:57 -07004267 kfree_skb(skb);
Tom Herbert76cc8b12010-05-20 18:37:59 +00004268 input_queue_head_incr(sd);
Stephen Hemminger6e583ce2008-08-03 21:29:57 -07004269 }
Changli Gao6e7676c2010-04-27 15:07:33 -07004270 }
Eric Dumazete36fa2f2010-04-19 21:17:14 +00004271 rps_unlock(sd);
Changli Gao6e7676c2010-04-27 15:07:33 -07004272
4273 skb_queue_walk_safe(&sd->process_queue, skb, tmp) {
4274 if (skb->dev == dev) {
4275 __skb_unlink(skb, &sd->process_queue);
4276 kfree_skb(skb);
Tom Herbert76cc8b12010-05-20 18:37:59 +00004277 input_queue_head_incr(sd);
Changli Gao6e7676c2010-04-27 15:07:33 -07004278 }
4279 }
Stephen Hemminger6e583ce2008-08-03 21:29:57 -07004280}
4281
Herbert Xud565b0a2008-12-15 23:38:52 -08004282static int napi_gro_complete(struct sk_buff *skb)
4283{
Vlad Yasevich22061d82012-11-15 08:49:11 +00004284 struct packet_offload *ptype;
Herbert Xud565b0a2008-12-15 23:38:52 -08004285 __be16 type = skb->protocol;
Vlad Yasevich22061d82012-11-15 08:49:11 +00004286 struct list_head *head = &offload_base;
Herbert Xud565b0a2008-12-15 23:38:52 -08004287 int err = -ENOENT;
4288
Eric Dumazetc3c7c252012-12-06 13:54:59 +00004289 BUILD_BUG_ON(sizeof(struct napi_gro_cb) > sizeof(skb->cb));
4290
Herbert Xufc59f9a2009-04-14 15:11:06 -07004291 if (NAPI_GRO_CB(skb)->count == 1) {
4292 skb_shinfo(skb)->gso_size = 0;
Herbert Xud565b0a2008-12-15 23:38:52 -08004293 goto out;
Herbert Xufc59f9a2009-04-14 15:11:06 -07004294 }
Herbert Xud565b0a2008-12-15 23:38:52 -08004295
4296 rcu_read_lock();
4297 list_for_each_entry_rcu(ptype, head, list) {
Vlad Yasevichf191a1d2012-11-15 08:49:23 +00004298 if (ptype->type != type || !ptype->callbacks.gro_complete)
Herbert Xud565b0a2008-12-15 23:38:52 -08004299 continue;
4300
Jerry Chu299603e82013-12-11 20:53:45 -08004301 err = ptype->callbacks.gro_complete(skb, 0);
Herbert Xud565b0a2008-12-15 23:38:52 -08004302 break;
4303 }
4304 rcu_read_unlock();
4305
4306 if (err) {
4307 WARN_ON(&ptype->list == head);
4308 kfree_skb(skb);
4309 return NET_RX_SUCCESS;
4310 }
4311
4312out:
Ben Hutchingsae78dbf2014-01-10 22:17:24 +00004313 return netif_receive_skb_internal(skb);
Herbert Xud565b0a2008-12-15 23:38:52 -08004314}
4315
Eric Dumazet2e71a6f2012-10-06 08:08:49 +00004316/* napi->gro_list contains packets ordered by age.
4317 * youngest packets at the head of it.
4318 * Complete skbs in reverse order to reduce latencies.
4319 */
4320void napi_gro_flush(struct napi_struct *napi, bool flush_old)
Herbert Xud565b0a2008-12-15 23:38:52 -08004321{
Eric Dumazet2e71a6f2012-10-06 08:08:49 +00004322 struct sk_buff *skb, *prev = NULL;
Herbert Xud565b0a2008-12-15 23:38:52 -08004323
Eric Dumazet2e71a6f2012-10-06 08:08:49 +00004324 /* scan list and build reverse chain */
4325 for (skb = napi->gro_list; skb != NULL; skb = skb->next) {
4326 skb->prev = prev;
4327 prev = skb;
Herbert Xud565b0a2008-12-15 23:38:52 -08004328 }
4329
Eric Dumazet2e71a6f2012-10-06 08:08:49 +00004330 for (skb = prev; skb; skb = prev) {
4331 skb->next = NULL;
4332
4333 if (flush_old && NAPI_GRO_CB(skb)->age == jiffies)
4334 return;
4335
4336 prev = skb->prev;
4337 napi_gro_complete(skb);
4338 napi->gro_count--;
4339 }
4340
Herbert Xud565b0a2008-12-15 23:38:52 -08004341 napi->gro_list = NULL;
4342}
Eric Dumazet86cac582010-08-31 18:25:32 +00004343EXPORT_SYMBOL(napi_gro_flush);
Herbert Xud565b0a2008-12-15 23:38:52 -08004344
Eric Dumazet89c5fa32012-12-10 13:28:16 +00004345static void gro_list_prepare(struct napi_struct *napi, struct sk_buff *skb)
4346{
4347 struct sk_buff *p;
4348 unsigned int maclen = skb->dev->hard_header_len;
Tom Herbert0b4cec82014-01-15 08:58:06 -08004349 u32 hash = skb_get_hash_raw(skb);
Eric Dumazet89c5fa32012-12-10 13:28:16 +00004350
4351 for (p = napi->gro_list; p; p = p->next) {
4352 unsigned long diffs;
4353
Tom Herbert0b4cec82014-01-15 08:58:06 -08004354 NAPI_GRO_CB(p)->flush = 0;
4355
4356 if (hash != skb_get_hash_raw(p)) {
4357 NAPI_GRO_CB(p)->same_flow = 0;
4358 continue;
4359 }
4360
Eric Dumazet89c5fa32012-12-10 13:28:16 +00004361 diffs = (unsigned long)p->dev ^ (unsigned long)skb->dev;
4362 diffs |= p->vlan_tci ^ skb->vlan_tci;
Jesse Grossce87fc62016-01-20 17:59:49 -08004363 diffs |= skb_metadata_dst_cmp(p, skb);
Eric Dumazet89c5fa32012-12-10 13:28:16 +00004364 if (maclen == ETH_HLEN)
4365 diffs |= compare_ether_header(skb_mac_header(p),
Eric Dumazeta50e2332014-03-29 21:28:21 -07004366 skb_mac_header(skb));
Eric Dumazet89c5fa32012-12-10 13:28:16 +00004367 else if (!diffs)
4368 diffs = memcmp(skb_mac_header(p),
Eric Dumazeta50e2332014-03-29 21:28:21 -07004369 skb_mac_header(skb),
Eric Dumazet89c5fa32012-12-10 13:28:16 +00004370 maclen);
4371 NAPI_GRO_CB(p)->same_flow = !diffs;
Eric Dumazet89c5fa32012-12-10 13:28:16 +00004372 }
4373}
4374
Jerry Chu299603e82013-12-11 20:53:45 -08004375static void skb_gro_reset_offset(struct sk_buff *skb)
4376{
4377 const struct skb_shared_info *pinfo = skb_shinfo(skb);
4378 const skb_frag_t *frag0 = &pinfo->frags[0];
4379
4380 NAPI_GRO_CB(skb)->data_offset = 0;
4381 NAPI_GRO_CB(skb)->frag0 = NULL;
4382 NAPI_GRO_CB(skb)->frag0_len = 0;
4383
4384 if (skb_mac_header(skb) == skb_tail_pointer(skb) &&
4385 pinfo->nr_frags &&
4386 !PageHighMem(skb_frag_page(frag0))) {
4387 NAPI_GRO_CB(skb)->frag0 = skb_frag_address(frag0);
4388 NAPI_GRO_CB(skb)->frag0_len = skb_frag_size(frag0);
Herbert Xud565b0a2008-12-15 23:38:52 -08004389 }
4390}
4391
Eric Dumazeta50e2332014-03-29 21:28:21 -07004392static void gro_pull_from_frag0(struct sk_buff *skb, int grow)
4393{
4394 struct skb_shared_info *pinfo = skb_shinfo(skb);
4395
4396 BUG_ON(skb->end - skb->tail < grow);
4397
4398 memcpy(skb_tail_pointer(skb), NAPI_GRO_CB(skb)->frag0, grow);
4399
4400 skb->data_len -= grow;
4401 skb->tail += grow;
4402
4403 pinfo->frags[0].page_offset += grow;
4404 skb_frag_size_sub(&pinfo->frags[0], grow);
4405
4406 if (unlikely(!skb_frag_size(&pinfo->frags[0]))) {
4407 skb_frag_unref(skb, 0);
4408 memmove(pinfo->frags, pinfo->frags + 1,
4409 --pinfo->nr_frags * sizeof(pinfo->frags[0]));
4410 }
4411}
4412
Rami Rosenbb728822012-11-28 21:55:25 +00004413static enum gro_result dev_gro_receive(struct napi_struct *napi, struct sk_buff *skb)
Herbert Xud565b0a2008-12-15 23:38:52 -08004414{
4415 struct sk_buff **pp = NULL;
Vlad Yasevich22061d82012-11-15 08:49:11 +00004416 struct packet_offload *ptype;
Herbert Xud565b0a2008-12-15 23:38:52 -08004417 __be16 type = skb->protocol;
Vlad Yasevich22061d82012-11-15 08:49:11 +00004418 struct list_head *head = &offload_base;
Herbert Xu0da2afd52008-12-26 14:57:42 -08004419 int same_flow;
Ben Hutchings5b252f02009-10-29 07:17:09 +00004420 enum gro_result ret;
Eric Dumazeta50e2332014-03-29 21:28:21 -07004421 int grow;
Herbert Xud565b0a2008-12-15 23:38:52 -08004422
Eric W. Biederman9c62a682014-03-14 20:51:52 -07004423 if (!(skb->dev->features & NETIF_F_GRO))
Herbert Xud565b0a2008-12-15 23:38:52 -08004424 goto normal;
4425
Tom Herbert5a212322014-08-31 15:12:41 -07004426 if (skb_is_gso(skb) || skb_has_frag_list(skb) || skb->csum_bad)
Herbert Xuf17f5c92009-01-14 14:36:12 -08004427 goto normal;
4428
Eric Dumazet89c5fa32012-12-10 13:28:16 +00004429 gro_list_prepare(napi, skb);
4430
Herbert Xud565b0a2008-12-15 23:38:52 -08004431 rcu_read_lock();
4432 list_for_each_entry_rcu(ptype, head, list) {
Vlad Yasevichf191a1d2012-11-15 08:49:23 +00004433 if (ptype->type != type || !ptype->callbacks.gro_receive)
Herbert Xud565b0a2008-12-15 23:38:52 -08004434 continue;
4435
Herbert Xu86911732009-01-29 14:19:50 +00004436 skb_set_network_header(skb, skb_gro_offset(skb));
Eric Dumazetefd94502013-02-14 17:31:48 +00004437 skb_reset_mac_len(skb);
Herbert Xud565b0a2008-12-15 23:38:52 -08004438 NAPI_GRO_CB(skb)->same_flow = 0;
4439 NAPI_GRO_CB(skb)->flush = 0;
Herbert Xu5d38a072009-01-04 16:13:40 -08004440 NAPI_GRO_CB(skb)->free = 0;
Jesse Grossfac8e0f2016-03-19 09:32:01 -07004441 NAPI_GRO_CB(skb)->encap_mark = 0;
Alexander Duycka0ca1532016-04-05 09:13:39 -07004442 NAPI_GRO_CB(skb)->is_fou = 0;
Tom Herbert15e23962015-02-10 16:30:31 -08004443 NAPI_GRO_CB(skb)->gro_remcsum_start = 0;
Herbert Xud565b0a2008-12-15 23:38:52 -08004444
Tom Herbert662880f2014-08-27 21:26:56 -07004445 /* Setup for GRO checksum validation */
4446 switch (skb->ip_summed) {
4447 case CHECKSUM_COMPLETE:
4448 NAPI_GRO_CB(skb)->csum = skb->csum;
4449 NAPI_GRO_CB(skb)->csum_valid = 1;
4450 NAPI_GRO_CB(skb)->csum_cnt = 0;
4451 break;
4452 case CHECKSUM_UNNECESSARY:
4453 NAPI_GRO_CB(skb)->csum_cnt = skb->csum_level + 1;
4454 NAPI_GRO_CB(skb)->csum_valid = 0;
4455 break;
4456 default:
4457 NAPI_GRO_CB(skb)->csum_cnt = 0;
4458 NAPI_GRO_CB(skb)->csum_valid = 0;
4459 }
Herbert Xud565b0a2008-12-15 23:38:52 -08004460
Vlad Yasevichf191a1d2012-11-15 08:49:23 +00004461 pp = ptype->callbacks.gro_receive(&napi->gro_list, skb);
Herbert Xud565b0a2008-12-15 23:38:52 -08004462 break;
4463 }
4464 rcu_read_unlock();
4465
4466 if (&ptype->list == head)
4467 goto normal;
4468
Herbert Xu0da2afd52008-12-26 14:57:42 -08004469 same_flow = NAPI_GRO_CB(skb)->same_flow;
Herbert Xu5d0d9be2009-01-29 14:19:48 +00004470 ret = NAPI_GRO_CB(skb)->free ? GRO_MERGED_FREE : GRO_MERGED;
Herbert Xu0da2afd52008-12-26 14:57:42 -08004471
Herbert Xud565b0a2008-12-15 23:38:52 -08004472 if (pp) {
4473 struct sk_buff *nskb = *pp;
4474
4475 *pp = nskb->next;
4476 nskb->next = NULL;
4477 napi_gro_complete(nskb);
Herbert Xu4ae55442009-02-08 18:00:36 +00004478 napi->gro_count--;
Herbert Xud565b0a2008-12-15 23:38:52 -08004479 }
4480
Herbert Xu0da2afd52008-12-26 14:57:42 -08004481 if (same_flow)
Herbert Xud565b0a2008-12-15 23:38:52 -08004482 goto ok;
4483
Eric Dumazet600adc12014-01-09 14:12:19 -08004484 if (NAPI_GRO_CB(skb)->flush)
Herbert Xud565b0a2008-12-15 23:38:52 -08004485 goto normal;
Herbert Xud565b0a2008-12-15 23:38:52 -08004486
Eric Dumazet600adc12014-01-09 14:12:19 -08004487 if (unlikely(napi->gro_count >= MAX_GRO_SKBS)) {
4488 struct sk_buff *nskb = napi->gro_list;
4489
4490 /* locate the end of the list to select the 'oldest' flow */
4491 while (nskb->next) {
4492 pp = &nskb->next;
4493 nskb = *pp;
4494 }
4495 *pp = NULL;
4496 nskb->next = NULL;
4497 napi_gro_complete(nskb);
4498 } else {
4499 napi->gro_count++;
4500 }
Herbert Xud565b0a2008-12-15 23:38:52 -08004501 NAPI_GRO_CB(skb)->count = 1;
Eric Dumazet2e71a6f2012-10-06 08:08:49 +00004502 NAPI_GRO_CB(skb)->age = jiffies;
Eric Dumazet29e98242014-05-16 11:34:37 -07004503 NAPI_GRO_CB(skb)->last = skb;
Herbert Xu86911732009-01-29 14:19:50 +00004504 skb_shinfo(skb)->gso_size = skb_gro_len(skb);
Herbert Xud565b0a2008-12-15 23:38:52 -08004505 skb->next = napi->gro_list;
4506 napi->gro_list = skb;
Herbert Xu5d0d9be2009-01-29 14:19:48 +00004507 ret = GRO_HELD;
Herbert Xud565b0a2008-12-15 23:38:52 -08004508
Herbert Xuad0f9902009-02-01 01:24:55 -08004509pull:
Eric Dumazeta50e2332014-03-29 21:28:21 -07004510 grow = skb_gro_offset(skb) - skb_headlen(skb);
4511 if (grow > 0)
4512 gro_pull_from_frag0(skb, grow);
Herbert Xud565b0a2008-12-15 23:38:52 -08004513ok:
Herbert Xu5d0d9be2009-01-29 14:19:48 +00004514 return ret;
Herbert Xud565b0a2008-12-15 23:38:52 -08004515
4516normal:
Herbert Xuad0f9902009-02-01 01:24:55 -08004517 ret = GRO_NORMAL;
4518 goto pull;
Herbert Xu5d38a072009-01-04 16:13:40 -08004519}
Herbert Xu96e93ea2009-01-06 10:49:34 -08004520
Jerry Chubf5a7552014-01-07 10:23:19 -08004521struct packet_offload *gro_find_receive_by_type(__be16 type)
4522{
4523 struct list_head *offload_head = &offload_base;
4524 struct packet_offload *ptype;
4525
4526 list_for_each_entry_rcu(ptype, offload_head, list) {
4527 if (ptype->type != type || !ptype->callbacks.gro_receive)
4528 continue;
4529 return ptype;
4530 }
4531 return NULL;
4532}
Or Gerlitze27a2f82014-01-20 13:59:20 +02004533EXPORT_SYMBOL(gro_find_receive_by_type);
Jerry Chubf5a7552014-01-07 10:23:19 -08004534
4535struct packet_offload *gro_find_complete_by_type(__be16 type)
4536{
4537 struct list_head *offload_head = &offload_base;
4538 struct packet_offload *ptype;
4539
4540 list_for_each_entry_rcu(ptype, offload_head, list) {
4541 if (ptype->type != type || !ptype->callbacks.gro_complete)
4542 continue;
4543 return ptype;
4544 }
4545 return NULL;
4546}
Or Gerlitze27a2f82014-01-20 13:59:20 +02004547EXPORT_SYMBOL(gro_find_complete_by_type);
Herbert Xu96e93ea2009-01-06 10:49:34 -08004548
Rami Rosenbb728822012-11-28 21:55:25 +00004549static gro_result_t napi_skb_finish(gro_result_t ret, struct sk_buff *skb)
Herbert Xu5d38a072009-01-04 16:13:40 -08004550{
Herbert Xu5d0d9be2009-01-29 14:19:48 +00004551 switch (ret) {
4552 case GRO_NORMAL:
Ben Hutchingsae78dbf2014-01-10 22:17:24 +00004553 if (netif_receive_skb_internal(skb))
Ben Hutchingsc7c4b3b2009-10-29 21:36:53 -07004554 ret = GRO_DROP;
4555 break;
Herbert Xu5d38a072009-01-04 16:13:40 -08004556
Herbert Xu5d0d9be2009-01-29 14:19:48 +00004557 case GRO_DROP:
Herbert Xu5d38a072009-01-04 16:13:40 -08004558 kfree_skb(skb);
4559 break;
Ben Hutchings5b252f02009-10-29 07:17:09 +00004560
Eric Dumazetdaa86542012-04-19 07:07:40 +00004561 case GRO_MERGED_FREE:
Jesse Grossce87fc62016-01-20 17:59:49 -08004562 if (NAPI_GRO_CB(skb)->free == NAPI_GRO_FREE_STOLEN_HEAD) {
4563 skb_dst_drop(skb);
Eric Dumazetd7e88832012-04-30 08:10:34 +00004564 kmem_cache_free(skbuff_head_cache, skb);
Jesse Grossce87fc62016-01-20 17:59:49 -08004565 } else {
Eric Dumazetd7e88832012-04-30 08:10:34 +00004566 __kfree_skb(skb);
Jesse Grossce87fc62016-01-20 17:59:49 -08004567 }
Eric Dumazetdaa86542012-04-19 07:07:40 +00004568 break;
4569
Ben Hutchings5b252f02009-10-29 07:17:09 +00004570 case GRO_HELD:
4571 case GRO_MERGED:
4572 break;
Herbert Xu5d38a072009-01-04 16:13:40 -08004573 }
4574
Ben Hutchingsc7c4b3b2009-10-29 21:36:53 -07004575 return ret;
Herbert Xu5d0d9be2009-01-29 14:19:48 +00004576}
Herbert Xu5d0d9be2009-01-29 14:19:48 +00004577
Ben Hutchingsc7c4b3b2009-10-29 21:36:53 -07004578gro_result_t napi_gro_receive(struct napi_struct *napi, struct sk_buff *skb)
Herbert Xu5d0d9be2009-01-29 14:19:48 +00004579{
Eric Dumazet93f93a42015-11-18 06:30:59 -08004580 skb_mark_napi_id(skb, napi);
Ben Hutchingsae78dbf2014-01-10 22:17:24 +00004581 trace_napi_gro_receive_entry(skb);
Herbert Xu86911732009-01-29 14:19:50 +00004582
Eric Dumazeta50e2332014-03-29 21:28:21 -07004583 skb_gro_reset_offset(skb);
4584
Eric Dumazet89c5fa32012-12-10 13:28:16 +00004585 return napi_skb_finish(dev_gro_receive(napi, skb), skb);
Herbert Xud565b0a2008-12-15 23:38:52 -08004586}
4587EXPORT_SYMBOL(napi_gro_receive);
4588
stephen hemmingerd0c2b0d2010-10-19 07:12:10 +00004589static void napi_reuse_skb(struct napi_struct *napi, struct sk_buff *skb)
Herbert Xu96e93ea2009-01-06 10:49:34 -08004590{
Eric Dumazet93a35f52014-10-23 06:30:30 -07004591 if (unlikely(skb->pfmemalloc)) {
4592 consume_skb(skb);
4593 return;
4594 }
Herbert Xu96e93ea2009-01-06 10:49:34 -08004595 __skb_pull(skb, skb_headlen(skb));
Eric Dumazet2a2a4592012-03-21 06:58:03 +00004596 /* restore the reserve we had after netdev_alloc_skb_ip_align() */
4597 skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN - skb_headroom(skb));
Jesse Gross3701e512010-10-20 13:56:06 +00004598 skb->vlan_tci = 0;
Herbert Xu66c46d72011-01-29 20:44:54 -08004599 skb->dev = napi->dev;
Andy Gospodarek6d152e22011-02-02 14:53:25 -08004600 skb->skb_iif = 0;
Jerry Chuc3caf112014-07-14 15:54:46 -07004601 skb->encapsulation = 0;
4602 skb_shinfo(skb)->gso_type = 0;
Eric Dumazete33d0ba2014-04-03 09:28:10 -07004603 skb->truesize = SKB_TRUESIZE(skb_end_offset(skb));
Herbert Xu96e93ea2009-01-06 10:49:34 -08004604
4605 napi->skb = skb;
4606}
Herbert Xu96e93ea2009-01-06 10:49:34 -08004607
Herbert Xu76620aa2009-04-16 02:02:07 -07004608struct sk_buff *napi_get_frags(struct napi_struct *napi)
Herbert Xu5d38a072009-01-04 16:13:40 -08004609{
Herbert Xu5d38a072009-01-04 16:13:40 -08004610 struct sk_buff *skb = napi->skb;
Herbert Xu5d38a072009-01-04 16:13:40 -08004611
4612 if (!skb) {
Alexander Duyckfd11a832014-12-09 19:40:49 -08004613 skb = napi_alloc_skb(napi, GRO_MAX_HEAD);
Eric Dumazete2f9dc32015-11-19 12:11:23 -08004614 if (skb) {
4615 napi->skb = skb;
4616 skb_mark_napi_id(skb, napi);
4617 }
Herbert Xu5d38a072009-01-04 16:13:40 -08004618 }
Herbert Xu96e93ea2009-01-06 10:49:34 -08004619 return skb;
4620}
Herbert Xu76620aa2009-04-16 02:02:07 -07004621EXPORT_SYMBOL(napi_get_frags);
Herbert Xu96e93ea2009-01-06 10:49:34 -08004622
Eric Dumazeta50e2332014-03-29 21:28:21 -07004623static gro_result_t napi_frags_finish(struct napi_struct *napi,
4624 struct sk_buff *skb,
4625 gro_result_t ret)
Herbert Xu5d0d9be2009-01-29 14:19:48 +00004626{
Herbert Xu5d0d9be2009-01-29 14:19:48 +00004627 switch (ret) {
4628 case GRO_NORMAL:
Eric Dumazeta50e2332014-03-29 21:28:21 -07004629 case GRO_HELD:
4630 __skb_push(skb, ETH_HLEN);
4631 skb->protocol = eth_type_trans(skb, skb->dev);
4632 if (ret == GRO_NORMAL && netif_receive_skb_internal(skb))
Ben Hutchingsc7c4b3b2009-10-29 21:36:53 -07004633 ret = GRO_DROP;
Herbert Xu86911732009-01-29 14:19:50 +00004634 break;
Herbert Xu5d0d9be2009-01-29 14:19:48 +00004635
4636 case GRO_DROP:
Herbert Xu5d0d9be2009-01-29 14:19:48 +00004637 case GRO_MERGED_FREE:
4638 napi_reuse_skb(napi, skb);
4639 break;
Ben Hutchings5b252f02009-10-29 07:17:09 +00004640
4641 case GRO_MERGED:
4642 break;
Herbert Xu5d0d9be2009-01-29 14:19:48 +00004643 }
4644
Ben Hutchingsc7c4b3b2009-10-29 21:36:53 -07004645 return ret;
Herbert Xu5d0d9be2009-01-29 14:19:48 +00004646}
Herbert Xu5d0d9be2009-01-29 14:19:48 +00004647
Eric Dumazeta50e2332014-03-29 21:28:21 -07004648/* Upper GRO stack assumes network header starts at gro_offset=0
4649 * Drivers could call both napi_gro_frags() and napi_gro_receive()
4650 * We copy ethernet header into skb->data to have a common layout.
4651 */
Eric Dumazet4adb9c42012-05-18 20:49:06 +00004652static struct sk_buff *napi_frags_skb(struct napi_struct *napi)
Herbert Xu96e93ea2009-01-06 10:49:34 -08004653{
Herbert Xu76620aa2009-04-16 02:02:07 -07004654 struct sk_buff *skb = napi->skb;
Eric Dumazeta50e2332014-03-29 21:28:21 -07004655 const struct ethhdr *eth;
4656 unsigned int hlen = sizeof(*eth);
Herbert Xu76620aa2009-04-16 02:02:07 -07004657
4658 napi->skb = NULL;
4659
Eric Dumazeta50e2332014-03-29 21:28:21 -07004660 skb_reset_mac_header(skb);
4661 skb_gro_reset_offset(skb);
4662
4663 eth = skb_gro_header_fast(skb, 0);
4664 if (unlikely(skb_gro_header_hard(skb, hlen))) {
4665 eth = skb_gro_header_slow(skb, hlen, 0);
4666 if (unlikely(!eth)) {
4667 napi_reuse_skb(napi, skb);
4668 return NULL;
4669 }
4670 } else {
4671 gro_pull_from_frag0(skb, hlen);
4672 NAPI_GRO_CB(skb)->frag0 += hlen;
4673 NAPI_GRO_CB(skb)->frag0_len -= hlen;
Herbert Xu76620aa2009-04-16 02:02:07 -07004674 }
Eric Dumazeta50e2332014-03-29 21:28:21 -07004675 __skb_pull(skb, hlen);
4676
4677 /*
4678 * This works because the only protocols we care about don't require
4679 * special handling.
4680 * We'll fix it up properly in napi_frags_finish()
4681 */
4682 skb->protocol = eth->h_proto;
Herbert Xu76620aa2009-04-16 02:02:07 -07004683
Herbert Xu76620aa2009-04-16 02:02:07 -07004684 return skb;
4685}
Herbert Xu76620aa2009-04-16 02:02:07 -07004686
Ben Hutchingsc7c4b3b2009-10-29 21:36:53 -07004687gro_result_t napi_gro_frags(struct napi_struct *napi)
Herbert Xu76620aa2009-04-16 02:02:07 -07004688{
4689 struct sk_buff *skb = napi_frags_skb(napi);
Herbert Xu96e93ea2009-01-06 10:49:34 -08004690
4691 if (!skb)
Ben Hutchingsc7c4b3b2009-10-29 21:36:53 -07004692 return GRO_DROP;
Herbert Xu96e93ea2009-01-06 10:49:34 -08004693
Ben Hutchingsae78dbf2014-01-10 22:17:24 +00004694 trace_napi_gro_frags_entry(skb);
4695
Eric Dumazet89c5fa32012-12-10 13:28:16 +00004696 return napi_frags_finish(napi, skb, dev_gro_receive(napi, skb));
Herbert Xu5d38a072009-01-04 16:13:40 -08004697}
4698EXPORT_SYMBOL(napi_gro_frags);
4699
Tom Herbert573e8fc2014-08-22 13:33:47 -07004700/* Compute the checksum from gro_offset and return the folded value
4701 * after adding in any pseudo checksum.
4702 */
4703__sum16 __skb_gro_checksum_complete(struct sk_buff *skb)
4704{
4705 __wsum wsum;
4706 __sum16 sum;
4707
4708 wsum = skb_checksum(skb, skb_gro_offset(skb), skb_gro_len(skb), 0);
4709
4710 /* NAPI_GRO_CB(skb)->csum holds pseudo checksum */
4711 sum = csum_fold(csum_add(NAPI_GRO_CB(skb)->csum, wsum));
4712 if (likely(!sum)) {
4713 if (unlikely(skb->ip_summed == CHECKSUM_COMPLETE) &&
4714 !skb->csum_complete_sw)
4715 netdev_rx_csum_fault(skb->dev);
4716 }
4717
4718 NAPI_GRO_CB(skb)->csum = wsum;
4719 NAPI_GRO_CB(skb)->csum_valid = 1;
4720
4721 return sum;
4722}
4723EXPORT_SYMBOL(__skb_gro_checksum_complete);
4724
Eric Dumazete326bed2010-04-22 00:22:45 -07004725/*
Zhi Yong Wu855abcf2014-01-01 04:34:50 +08004726 * net_rps_action_and_irq_enable sends any pending IPI's for rps.
Eric Dumazete326bed2010-04-22 00:22:45 -07004727 * Note: called with local irq disabled, but exits with local irq enabled.
4728 */
4729static void net_rps_action_and_irq_enable(struct softnet_data *sd)
4730{
4731#ifdef CONFIG_RPS
4732 struct softnet_data *remsd = sd->rps_ipi_list;
4733
4734 if (remsd) {
4735 sd->rps_ipi_list = NULL;
4736
4737 local_irq_enable();
4738
4739 /* Send pending IPI's to kick RPS processing on remote cpus. */
4740 while (remsd) {
4741 struct softnet_data *next = remsd->rps_ipi_next;
4742
4743 if (cpu_online(remsd->cpu))
Frederic Weisbeckerc46fff22014-02-24 16:40:02 +01004744 smp_call_function_single_async(remsd->cpu,
Frederic Weisbeckerfce8ad12014-02-24 16:40:01 +01004745 &remsd->csd);
Eric Dumazete326bed2010-04-22 00:22:45 -07004746 remsd = next;
4747 }
4748 } else
4749#endif
4750 local_irq_enable();
4751}
4752
Eric Dumazetd75b1ad2014-11-02 06:19:33 -08004753static bool sd_has_rps_ipi_waiting(struct softnet_data *sd)
4754{
4755#ifdef CONFIG_RPS
4756 return sd->rps_ipi_list != NULL;
4757#else
4758 return false;
4759#endif
4760}
4761
Stephen Hemmingerbea33482007-10-03 16:41:36 -07004762static int process_backlog(struct napi_struct *napi, int quota)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004763{
4764 int work = 0;
Eric Dumazeteecfd7c2010-05-06 22:07:48 -07004765 struct softnet_data *sd = container_of(napi, struct softnet_data, backlog);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004766
Eric Dumazete326bed2010-04-22 00:22:45 -07004767 /* Check if we have pending ipi, its better to send them now,
4768 * not waiting net_rx_action() end.
4769 */
Eric Dumazetd75b1ad2014-11-02 06:19:33 -08004770 if (sd_has_rps_ipi_waiting(sd)) {
Eric Dumazete326bed2010-04-22 00:22:45 -07004771 local_irq_disable();
4772 net_rps_action_and_irq_enable(sd);
4773 }
Eric Dumazetd75b1ad2014-11-02 06:19:33 -08004774
Stephen Hemmingerbea33482007-10-03 16:41:36 -07004775 napi->weight = weight_p;
Changli Gao6e7676c2010-04-27 15:07:33 -07004776 local_irq_disable();
Tom Herbert11ef7a82014-06-30 09:50:40 -07004777 while (1) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07004778 struct sk_buff *skb;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004779
Changli Gao6e7676c2010-04-27 15:07:33 -07004780 while ((skb = __skb_dequeue(&sd->process_queue))) {
Julian Anastasov2c17d272015-07-09 09:59:10 +03004781 rcu_read_lock();
Eric Dumazete4008272010-04-05 15:42:39 -07004782 local_irq_enable();
Changli Gao6e7676c2010-04-27 15:07:33 -07004783 __netif_receive_skb(skb);
Julian Anastasov2c17d272015-07-09 09:59:10 +03004784 rcu_read_unlock();
Changli Gao6e7676c2010-04-27 15:07:33 -07004785 local_irq_disable();
Tom Herbert76cc8b12010-05-20 18:37:59 +00004786 input_queue_head_incr(sd);
4787 if (++work >= quota) {
4788 local_irq_enable();
4789 return work;
4790 }
Stephen Hemmingerbea33482007-10-03 16:41:36 -07004791 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07004792
Changli Gao6e7676c2010-04-27 15:07:33 -07004793 rps_lock(sd);
Tom Herbert11ef7a82014-06-30 09:50:40 -07004794 if (skb_queue_empty(&sd->input_pkt_queue)) {
Eric Dumazeteecfd7c2010-05-06 22:07:48 -07004795 /*
4796 * Inline a custom version of __napi_complete().
4797 * only current cpu owns and manipulates this napi,
Tom Herbert11ef7a82014-06-30 09:50:40 -07004798 * and NAPI_STATE_SCHED is the only possible flag set
4799 * on backlog.
4800 * We can use a plain write instead of clear_bit(),
Eric Dumazeteecfd7c2010-05-06 22:07:48 -07004801 * and we dont need an smp_mb() memory barrier.
4802 */
Eric Dumazeteecfd7c2010-05-06 22:07:48 -07004803 napi->state = 0;
Tom Herbert11ef7a82014-06-30 09:50:40 -07004804 rps_unlock(sd);
Eric Dumazeteecfd7c2010-05-06 22:07:48 -07004805
Tom Herbert11ef7a82014-06-30 09:50:40 -07004806 break;
Changli Gao6e7676c2010-04-27 15:07:33 -07004807 }
Tom Herbert11ef7a82014-06-30 09:50:40 -07004808
4809 skb_queue_splice_tail_init(&sd->input_pkt_queue,
4810 &sd->process_queue);
Changli Gao6e7676c2010-04-27 15:07:33 -07004811 rps_unlock(sd);
4812 }
4813 local_irq_enable();
Linus Torvalds1da177e2005-04-16 15:20:36 -07004814
Stephen Hemmingerbea33482007-10-03 16:41:36 -07004815 return work;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004816}
4817
Stephen Hemmingerbea33482007-10-03 16:41:36 -07004818/**
4819 * __napi_schedule - schedule for receive
Randy Dunlapc4ea43c2007-10-12 21:17:49 -07004820 * @n: entry to schedule
Stephen Hemmingerbea33482007-10-03 16:41:36 -07004821 *
Eric Dumazetbc9ad162014-10-28 18:05:13 -07004822 * The entry's receive function will be scheduled to run.
4823 * Consider using __napi_schedule_irqoff() if hard irqs are masked.
Stephen Hemmingerbea33482007-10-03 16:41:36 -07004824 */
Harvey Harrisonb5606c22008-02-13 15:03:16 -08004825void __napi_schedule(struct napi_struct *n)
Stephen Hemmingerbea33482007-10-03 16:41:36 -07004826{
4827 unsigned long flags;
4828
4829 local_irq_save(flags);
Christoph Lameter903ceff2014-08-17 12:30:35 -05004830 ____napi_schedule(this_cpu_ptr(&softnet_data), n);
Stephen Hemmingerbea33482007-10-03 16:41:36 -07004831 local_irq_restore(flags);
4832}
4833EXPORT_SYMBOL(__napi_schedule);
4834
Eric Dumazetbc9ad162014-10-28 18:05:13 -07004835/**
4836 * __napi_schedule_irqoff - schedule for receive
4837 * @n: entry to schedule
4838 *
4839 * Variant of __napi_schedule() assuming hard irqs are masked
4840 */
4841void __napi_schedule_irqoff(struct napi_struct *n)
4842{
4843 ____napi_schedule(this_cpu_ptr(&softnet_data), n);
4844}
4845EXPORT_SYMBOL(__napi_schedule_irqoff);
4846
Herbert Xud565b0a2008-12-15 23:38:52 -08004847void __napi_complete(struct napi_struct *n)
4848{
4849 BUG_ON(!test_bit(NAPI_STATE_SCHED, &n->state));
Herbert Xud565b0a2008-12-15 23:38:52 -08004850
Eric Dumazetd75b1ad2014-11-02 06:19:33 -08004851 list_del_init(&n->poll_list);
Peter Zijlstra4e857c52014-03-17 18:06:10 +01004852 smp_mb__before_atomic();
Herbert Xud565b0a2008-12-15 23:38:52 -08004853 clear_bit(NAPI_STATE_SCHED, &n->state);
4854}
4855EXPORT_SYMBOL(__napi_complete);
4856
Eric Dumazet3b47d302014-11-06 21:09:44 -08004857void napi_complete_done(struct napi_struct *n, int work_done)
Herbert Xud565b0a2008-12-15 23:38:52 -08004858{
4859 unsigned long flags;
4860
4861 /*
4862 * don't let napi dequeue from the cpu poll list
4863 * just in case its running on a different cpu
4864 */
4865 if (unlikely(test_bit(NAPI_STATE_NPSVC, &n->state)))
4866 return;
4867
Eric Dumazet3b47d302014-11-06 21:09:44 -08004868 if (n->gro_list) {
4869 unsigned long timeout = 0;
Eric Dumazetd75b1ad2014-11-02 06:19:33 -08004870
Eric Dumazet3b47d302014-11-06 21:09:44 -08004871 if (work_done)
4872 timeout = n->dev->gro_flush_timeout;
4873
4874 if (timeout)
4875 hrtimer_start(&n->timer, ns_to_ktime(timeout),
4876 HRTIMER_MODE_REL_PINNED);
4877 else
4878 napi_gro_flush(n, false);
4879 }
Eric Dumazetd75b1ad2014-11-02 06:19:33 -08004880 if (likely(list_empty(&n->poll_list))) {
4881 WARN_ON_ONCE(!test_and_clear_bit(NAPI_STATE_SCHED, &n->state));
4882 } else {
4883 /* If n->poll_list is not empty, we need to mask irqs */
4884 local_irq_save(flags);
4885 __napi_complete(n);
4886 local_irq_restore(flags);
4887 }
Herbert Xud565b0a2008-12-15 23:38:52 -08004888}
Eric Dumazet3b47d302014-11-06 21:09:44 -08004889EXPORT_SYMBOL(napi_complete_done);
Herbert Xud565b0a2008-12-15 23:38:52 -08004890
Eliezer Tamiraf12fa62013-06-10 11:39:41 +03004891/* must be called under rcu_read_lock(), as we dont take a reference */
Eric Dumazet02d62e82015-11-18 06:30:52 -08004892static struct napi_struct *napi_by_id(unsigned int napi_id)
Eliezer Tamiraf12fa62013-06-10 11:39:41 +03004893{
4894 unsigned int hash = napi_id % HASH_SIZE(napi_hash);
4895 struct napi_struct *napi;
4896
4897 hlist_for_each_entry_rcu(napi, &napi_hash[hash], napi_hash_node)
4898 if (napi->napi_id == napi_id)
4899 return napi;
4900
4901 return NULL;
4902}
Eric Dumazet02d62e82015-11-18 06:30:52 -08004903
4904#if defined(CONFIG_NET_RX_BUSY_POLL)
Eric Dumazetce6aea92015-11-18 06:30:54 -08004905#define BUSY_POLL_BUDGET 8
Eric Dumazet02d62e82015-11-18 06:30:52 -08004906bool sk_busy_loop(struct sock *sk, int nonblock)
4907{
4908 unsigned long end_time = !nonblock ? sk_busy_loop_end_time(sk) : 0;
Eric Dumazetce6aea92015-11-18 06:30:54 -08004909 int (*busy_poll)(struct napi_struct *dev);
Eric Dumazet02d62e82015-11-18 06:30:52 -08004910 struct napi_struct *napi;
4911 int rc = false;
4912
Eric Dumazet2a028ec2015-11-18 06:30:53 -08004913 rcu_read_lock();
Eric Dumazet02d62e82015-11-18 06:30:52 -08004914
4915 napi = napi_by_id(sk->sk_napi_id);
4916 if (!napi)
4917 goto out;
4918
Eric Dumazetce6aea92015-11-18 06:30:54 -08004919 /* Note: ndo_busy_poll method is optional in linux-4.5 */
4920 busy_poll = napi->dev->netdev_ops->ndo_busy_poll;
Eric Dumazet02d62e82015-11-18 06:30:52 -08004921
4922 do {
Eric Dumazetce6aea92015-11-18 06:30:54 -08004923 rc = 0;
Eric Dumazet2a028ec2015-11-18 06:30:53 -08004924 local_bh_disable();
Eric Dumazetce6aea92015-11-18 06:30:54 -08004925 if (busy_poll) {
4926 rc = busy_poll(napi);
4927 } else if (napi_schedule_prep(napi)) {
4928 void *have = netpoll_poll_lock(napi);
4929
4930 if (test_bit(NAPI_STATE_SCHED, &napi->state)) {
4931 rc = napi->poll(napi, BUSY_POLL_BUDGET);
4932 trace_napi_poll(napi);
4933 if (rc == BUSY_POLL_BUDGET) {
4934 napi_complete_done(napi, rc);
4935 napi_schedule(napi);
4936 }
4937 }
4938 netpoll_poll_unlock(have);
4939 }
Eric Dumazet2a028ec2015-11-18 06:30:53 -08004940 if (rc > 0)
4941 NET_ADD_STATS_BH(sock_net(sk),
4942 LINUX_MIB_BUSYPOLLRXPACKETS, rc);
4943 local_bh_enable();
Eric Dumazet02d62e82015-11-18 06:30:52 -08004944
4945 if (rc == LL_FLUSH_FAILED)
4946 break; /* permanent failure */
4947
Eric Dumazet02d62e82015-11-18 06:30:52 -08004948 cpu_relax();
Eric Dumazet02d62e82015-11-18 06:30:52 -08004949 } while (!nonblock && skb_queue_empty(&sk->sk_receive_queue) &&
4950 !need_resched() && !busy_loop_timeout(end_time));
4951
4952 rc = !skb_queue_empty(&sk->sk_receive_queue);
4953out:
Eric Dumazet2a028ec2015-11-18 06:30:53 -08004954 rcu_read_unlock();
Eric Dumazet02d62e82015-11-18 06:30:52 -08004955 return rc;
4956}
4957EXPORT_SYMBOL(sk_busy_loop);
4958
4959#endif /* CONFIG_NET_RX_BUSY_POLL */
Eliezer Tamiraf12fa62013-06-10 11:39:41 +03004960
4961void napi_hash_add(struct napi_struct *napi)
4962{
Eric Dumazetd64b5e82015-11-18 06:31:00 -08004963 if (test_bit(NAPI_STATE_NO_BUSY_POLL, &napi->state) ||
4964 test_and_set_bit(NAPI_STATE_HASHED, &napi->state))
Eric Dumazet52bd2d62015-11-18 06:30:50 -08004965 return;
Eliezer Tamiraf12fa62013-06-10 11:39:41 +03004966
Eric Dumazet52bd2d62015-11-18 06:30:50 -08004967 spin_lock(&napi_hash_lock);
Eliezer Tamiraf12fa62013-06-10 11:39:41 +03004968
Eric Dumazet52bd2d62015-11-18 06:30:50 -08004969 /* 0..NR_CPUS+1 range is reserved for sender_cpu use */
4970 do {
4971 if (unlikely(++napi_gen_id < NR_CPUS + 1))
4972 napi_gen_id = NR_CPUS + 1;
4973 } while (napi_by_id(napi_gen_id));
4974 napi->napi_id = napi_gen_id;
Eliezer Tamiraf12fa62013-06-10 11:39:41 +03004975
Eric Dumazet52bd2d62015-11-18 06:30:50 -08004976 hlist_add_head_rcu(&napi->napi_hash_node,
4977 &napi_hash[napi->napi_id % HASH_SIZE(napi_hash)]);
Eliezer Tamiraf12fa62013-06-10 11:39:41 +03004978
Eric Dumazet52bd2d62015-11-18 06:30:50 -08004979 spin_unlock(&napi_hash_lock);
Eliezer Tamiraf12fa62013-06-10 11:39:41 +03004980}
4981EXPORT_SYMBOL_GPL(napi_hash_add);
4982
4983/* Warning : caller is responsible to make sure rcu grace period
4984 * is respected before freeing memory containing @napi
4985 */
Eric Dumazet34cbe272015-11-18 06:31:02 -08004986bool napi_hash_del(struct napi_struct *napi)
Eliezer Tamiraf12fa62013-06-10 11:39:41 +03004987{
Eric Dumazet34cbe272015-11-18 06:31:02 -08004988 bool rcu_sync_needed = false;
4989
Eliezer Tamiraf12fa62013-06-10 11:39:41 +03004990 spin_lock(&napi_hash_lock);
4991
Eric Dumazet34cbe272015-11-18 06:31:02 -08004992 if (test_and_clear_bit(NAPI_STATE_HASHED, &napi->state)) {
4993 rcu_sync_needed = true;
Eliezer Tamiraf12fa62013-06-10 11:39:41 +03004994 hlist_del_rcu(&napi->napi_hash_node);
Eric Dumazet34cbe272015-11-18 06:31:02 -08004995 }
Eliezer Tamiraf12fa62013-06-10 11:39:41 +03004996 spin_unlock(&napi_hash_lock);
Eric Dumazet34cbe272015-11-18 06:31:02 -08004997 return rcu_sync_needed;
Eliezer Tamiraf12fa62013-06-10 11:39:41 +03004998}
4999EXPORT_SYMBOL_GPL(napi_hash_del);
5000
Eric Dumazet3b47d302014-11-06 21:09:44 -08005001static enum hrtimer_restart napi_watchdog(struct hrtimer *timer)
5002{
5003 struct napi_struct *napi;
5004
5005 napi = container_of(timer, struct napi_struct, timer);
5006 if (napi->gro_list)
5007 napi_schedule(napi);
5008
5009 return HRTIMER_NORESTART;
5010}
5011
Herbert Xud565b0a2008-12-15 23:38:52 -08005012void netif_napi_add(struct net_device *dev, struct napi_struct *napi,
5013 int (*poll)(struct napi_struct *, int), int weight)
5014{
5015 INIT_LIST_HEAD(&napi->poll_list);
Eric Dumazet3b47d302014-11-06 21:09:44 -08005016 hrtimer_init(&napi->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL_PINNED);
5017 napi->timer.function = napi_watchdog;
Herbert Xu4ae55442009-02-08 18:00:36 +00005018 napi->gro_count = 0;
Herbert Xud565b0a2008-12-15 23:38:52 -08005019 napi->gro_list = NULL;
Herbert Xu5d38a072009-01-04 16:13:40 -08005020 napi->skb = NULL;
Herbert Xud565b0a2008-12-15 23:38:52 -08005021 napi->poll = poll;
Eric Dumazet82dc3c62013-03-05 15:57:22 +00005022 if (weight > NAPI_POLL_WEIGHT)
5023 pr_err_once("netif_napi_add() called with weight %d on device %s\n",
5024 weight, dev->name);
Herbert Xud565b0a2008-12-15 23:38:52 -08005025 napi->weight = weight;
5026 list_add(&napi->dev_list, &dev->napi_list);
Herbert Xud565b0a2008-12-15 23:38:52 -08005027 napi->dev = dev;
Herbert Xu5d38a072009-01-04 16:13:40 -08005028#ifdef CONFIG_NETPOLL
Herbert Xud565b0a2008-12-15 23:38:52 -08005029 spin_lock_init(&napi->poll_lock);
5030 napi->poll_owner = -1;
5031#endif
5032 set_bit(NAPI_STATE_SCHED, &napi->state);
Eric Dumazet93d05d42015-11-18 06:31:03 -08005033 napi_hash_add(napi);
Herbert Xud565b0a2008-12-15 23:38:52 -08005034}
5035EXPORT_SYMBOL(netif_napi_add);
5036
Eric Dumazet3b47d302014-11-06 21:09:44 -08005037void napi_disable(struct napi_struct *n)
5038{
5039 might_sleep();
5040 set_bit(NAPI_STATE_DISABLE, &n->state);
5041
5042 while (test_and_set_bit(NAPI_STATE_SCHED, &n->state))
5043 msleep(1);
Neil Horman2d8bff12015-09-23 14:57:58 -04005044 while (test_and_set_bit(NAPI_STATE_NPSVC, &n->state))
5045 msleep(1);
Eric Dumazet3b47d302014-11-06 21:09:44 -08005046
5047 hrtimer_cancel(&n->timer);
5048
5049 clear_bit(NAPI_STATE_DISABLE, &n->state);
5050}
5051EXPORT_SYMBOL(napi_disable);
5052
Eric Dumazet93d05d42015-11-18 06:31:03 -08005053/* Must be called in process context */
Herbert Xud565b0a2008-12-15 23:38:52 -08005054void netif_napi_del(struct napi_struct *napi)
5055{
Eric Dumazet93d05d42015-11-18 06:31:03 -08005056 might_sleep();
5057 if (napi_hash_del(napi))
5058 synchronize_net();
Peter P Waskiewicz Jrd7b06632008-12-26 01:35:35 -08005059 list_del_init(&napi->dev_list);
Herbert Xu76620aa2009-04-16 02:02:07 -07005060 napi_free_frags(napi);
Herbert Xud565b0a2008-12-15 23:38:52 -08005061
Eric Dumazet289dccb2013-12-20 14:29:08 -08005062 kfree_skb_list(napi->gro_list);
Herbert Xud565b0a2008-12-15 23:38:52 -08005063 napi->gro_list = NULL;
Herbert Xu4ae55442009-02-08 18:00:36 +00005064 napi->gro_count = 0;
Herbert Xud565b0a2008-12-15 23:38:52 -08005065}
5066EXPORT_SYMBOL(netif_napi_del);
5067
Herbert Xu726ce702014-12-21 07:16:21 +11005068static int napi_poll(struct napi_struct *n, struct list_head *repoll)
5069{
5070 void *have;
5071 int work, weight;
5072
5073 list_del_init(&n->poll_list);
5074
5075 have = netpoll_poll_lock(n);
5076
5077 weight = n->weight;
5078
5079 /* This NAPI_STATE_SCHED test is for avoiding a race
5080 * with netpoll's poll_napi(). Only the entity which
5081 * obtains the lock and sees NAPI_STATE_SCHED set will
5082 * actually make the ->poll() call. Therefore we avoid
5083 * accidentally calling ->poll() when NAPI is not scheduled.
5084 */
5085 work = 0;
5086 if (test_bit(NAPI_STATE_SCHED, &n->state)) {
5087 work = n->poll(n, weight);
5088 trace_napi_poll(n);
5089 }
5090
5091 WARN_ON_ONCE(work > weight);
5092
5093 if (likely(work < weight))
5094 goto out_unlock;
5095
5096 /* Drivers must not modify the NAPI state if they
5097 * consume the entire weight. In such cases this code
5098 * still "owns" the NAPI instance and therefore can
5099 * move the instance around on the list at-will.
5100 */
5101 if (unlikely(napi_disable_pending(n))) {
5102 napi_complete(n);
5103 goto out_unlock;
5104 }
5105
5106 if (n->gro_list) {
5107 /* flush too old packets
5108 * If HZ < 1000, flush all packets.
5109 */
5110 napi_gro_flush(n, HZ >= 1000);
5111 }
5112
Herbert Xu001ce542014-12-21 07:16:22 +11005113 /* Some drivers may have called napi_schedule
5114 * prior to exhausting their budget.
5115 */
5116 if (unlikely(!list_empty(&n->poll_list))) {
5117 pr_warn_once("%s: Budget exhausted after napi rescheduled\n",
5118 n->dev ? n->dev->name : "backlog");
5119 goto out_unlock;
5120 }
5121
Herbert Xu726ce702014-12-21 07:16:21 +11005122 list_add_tail(&n->poll_list, repoll);
5123
5124out_unlock:
5125 netpoll_poll_unlock(have);
5126
5127 return work;
5128}
5129
Linus Torvalds1da177e2005-04-16 15:20:36 -07005130static void net_rx_action(struct softirq_action *h)
5131{
Christoph Lameter903ceff2014-08-17 12:30:35 -05005132 struct softnet_data *sd = this_cpu_ptr(&softnet_data);
Stephen Hemminger24f8b232008-11-03 17:14:38 -08005133 unsigned long time_limit = jiffies + 2;
Stephen Hemminger51b0bde2005-06-23 20:14:40 -07005134 int budget = netdev_budget;
Eric Dumazetd75b1ad2014-11-02 06:19:33 -08005135 LIST_HEAD(list);
5136 LIST_HEAD(repoll);
Matt Mackall53fb95d2005-08-11 19:27:43 -07005137
Linus Torvalds1da177e2005-04-16 15:20:36 -07005138 local_irq_disable();
Eric Dumazetd75b1ad2014-11-02 06:19:33 -08005139 list_splice_init(&sd->poll_list, &list);
5140 local_irq_enable();
Linus Torvalds1da177e2005-04-16 15:20:36 -07005141
Herbert Xuceb8d5b2014-12-21 07:16:25 +11005142 for (;;) {
Stephen Hemmingerbea33482007-10-03 16:41:36 -07005143 struct napi_struct *n;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005144
Herbert Xuceb8d5b2014-12-21 07:16:25 +11005145 if (list_empty(&list)) {
5146 if (!sd_has_rps_ipi_waiting(sd) && list_empty(&repoll))
5147 return;
5148 break;
5149 }
5150
Herbert Xu6bd373e2014-12-21 07:16:24 +11005151 n = list_first_entry(&list, struct napi_struct, poll_list);
5152 budget -= napi_poll(n, &repoll);
5153
Eric Dumazetd75b1ad2014-11-02 06:19:33 -08005154 /* If softirq window is exhausted then punt.
Stephen Hemminger24f8b232008-11-03 17:14:38 -08005155 * Allow this to run for 2 jiffies since which will allow
5156 * an average latency of 1.5/HZ.
Stephen Hemmingerbea33482007-10-03 16:41:36 -07005157 */
Herbert Xuceb8d5b2014-12-21 07:16:25 +11005158 if (unlikely(budget <= 0 ||
5159 time_after_eq(jiffies, time_limit))) {
5160 sd->time_squeeze++;
5161 break;
5162 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07005163 }
Eric Dumazetd75b1ad2014-11-02 06:19:33 -08005164
Jesper Dangaard Brouer795bb1c2016-02-08 13:14:59 +01005165 __kfree_skb_flush();
Eric Dumazetd75b1ad2014-11-02 06:19:33 -08005166 local_irq_disable();
5167
5168 list_splice_tail_init(&sd->poll_list, &list);
5169 list_splice_tail(&repoll, &list);
5170 list_splice(&list, &sd->poll_list);
5171 if (!list_empty(&sd->poll_list))
5172 __raise_softirq_irqoff(NET_RX_SOFTIRQ);
5173
Eric Dumazete326bed2010-04-22 00:22:45 -07005174 net_rps_action_and_irq_enable(sd);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005175}
5176
Veaceslav Falicoaa9d8562013-08-28 23:25:04 +02005177struct netdev_adjacent {
Jiri Pirko9ff162a2013-01-03 22:48:49 +00005178 struct net_device *dev;
Veaceslav Falico5d261912013-08-28 23:25:05 +02005179
5180 /* upper master flag, there can only be one master device per list */
Jiri Pirko9ff162a2013-01-03 22:48:49 +00005181 bool master;
Veaceslav Falico5d261912013-08-28 23:25:05 +02005182
Veaceslav Falico5d261912013-08-28 23:25:05 +02005183 /* counter for the number of times this device was added to us */
5184 u16 ref_nr;
5185
Veaceslav Falico402dae92013-09-25 09:20:09 +02005186 /* private field for the users */
5187 void *private;
5188
Jiri Pirko9ff162a2013-01-03 22:48:49 +00005189 struct list_head list;
5190 struct rcu_head rcu;
Jiri Pirko9ff162a2013-01-03 22:48:49 +00005191};
5192
Michal Kubeček6ea29da2015-09-24 10:59:05 +02005193static struct netdev_adjacent *__netdev_find_adj(struct net_device *adj_dev,
Veaceslav Falico2f268f12013-09-25 09:20:07 +02005194 struct list_head *adj_list)
Jiri Pirko9ff162a2013-01-03 22:48:49 +00005195{
Veaceslav Falico5d261912013-08-28 23:25:05 +02005196 struct netdev_adjacent *adj;
Veaceslav Falico5d261912013-08-28 23:25:05 +02005197
Veaceslav Falico2f268f12013-09-25 09:20:07 +02005198 list_for_each_entry(adj, adj_list, list) {
Veaceslav Falico5d261912013-08-28 23:25:05 +02005199 if (adj->dev == adj_dev)
5200 return adj;
Jiri Pirko9ff162a2013-01-03 22:48:49 +00005201 }
5202 return NULL;
5203}
5204
5205/**
5206 * netdev_has_upper_dev - Check if device is linked to an upper device
5207 * @dev: device
5208 * @upper_dev: upper device to check
5209 *
5210 * Find out if a device is linked to specified upper device and return true
5211 * in case it is. Note that this checks only immediate upper device,
5212 * not through a complete stack of devices. The caller must hold the RTNL lock.
5213 */
5214bool netdev_has_upper_dev(struct net_device *dev,
5215 struct net_device *upper_dev)
5216{
5217 ASSERT_RTNL();
5218
Michal Kubeček6ea29da2015-09-24 10:59:05 +02005219 return __netdev_find_adj(upper_dev, &dev->all_adj_list.upper);
Jiri Pirko9ff162a2013-01-03 22:48:49 +00005220}
5221EXPORT_SYMBOL(netdev_has_upper_dev);
5222
5223/**
5224 * netdev_has_any_upper_dev - Check if device is linked to some device
5225 * @dev: device
5226 *
5227 * Find out if a device is linked to an upper device and return true in case
5228 * it is. The caller must hold the RTNL lock.
5229 */
stephen hemminger1d143d92013-12-29 14:01:29 -08005230static bool netdev_has_any_upper_dev(struct net_device *dev)
Jiri Pirko9ff162a2013-01-03 22:48:49 +00005231{
5232 ASSERT_RTNL();
5233
Veaceslav Falico2f268f12013-09-25 09:20:07 +02005234 return !list_empty(&dev->all_adj_list.upper);
Jiri Pirko9ff162a2013-01-03 22:48:49 +00005235}
Jiri Pirko9ff162a2013-01-03 22:48:49 +00005236
5237/**
5238 * netdev_master_upper_dev_get - Get master upper device
5239 * @dev: device
5240 *
5241 * Find a master upper device and return pointer to it or NULL in case
5242 * it's not there. The caller must hold the RTNL lock.
5243 */
5244struct net_device *netdev_master_upper_dev_get(struct net_device *dev)
5245{
Veaceslav Falicoaa9d8562013-08-28 23:25:04 +02005246 struct netdev_adjacent *upper;
Jiri Pirko9ff162a2013-01-03 22:48:49 +00005247
5248 ASSERT_RTNL();
5249
Veaceslav Falico2f268f12013-09-25 09:20:07 +02005250 if (list_empty(&dev->adj_list.upper))
Jiri Pirko9ff162a2013-01-03 22:48:49 +00005251 return NULL;
5252
Veaceslav Falico2f268f12013-09-25 09:20:07 +02005253 upper = list_first_entry(&dev->adj_list.upper,
Veaceslav Falicoaa9d8562013-08-28 23:25:04 +02005254 struct netdev_adjacent, list);
Jiri Pirko9ff162a2013-01-03 22:48:49 +00005255 if (likely(upper->master))
5256 return upper->dev;
5257 return NULL;
5258}
5259EXPORT_SYMBOL(netdev_master_upper_dev_get);
5260
Veaceslav Falicob6ccba42013-09-25 09:20:23 +02005261void *netdev_adjacent_get_private(struct list_head *adj_list)
5262{
5263 struct netdev_adjacent *adj;
5264
5265 adj = list_entry(adj_list, struct netdev_adjacent, list);
5266
5267 return adj->private;
5268}
5269EXPORT_SYMBOL(netdev_adjacent_get_private);
5270
Veaceslav Falico31088a12013-09-25 09:20:12 +02005271/**
Vlad Yasevich44a40852014-05-16 17:20:38 -04005272 * netdev_upper_get_next_dev_rcu - Get the next dev from upper list
5273 * @dev: device
5274 * @iter: list_head ** of the current position
5275 *
5276 * Gets the next device from the dev's upper list, starting from iter
5277 * position. The caller must hold RCU read lock.
5278 */
5279struct net_device *netdev_upper_get_next_dev_rcu(struct net_device *dev,
5280 struct list_head **iter)
5281{
5282 struct netdev_adjacent *upper;
5283
5284 WARN_ON_ONCE(!rcu_read_lock_held() && !lockdep_rtnl_is_held());
5285
5286 upper = list_entry_rcu((*iter)->next, struct netdev_adjacent, list);
5287
5288 if (&upper->list == &dev->adj_list.upper)
5289 return NULL;
5290
5291 *iter = &upper->list;
5292
5293 return upper->dev;
5294}
5295EXPORT_SYMBOL(netdev_upper_get_next_dev_rcu);
5296
5297/**
Veaceslav Falico31088a12013-09-25 09:20:12 +02005298 * netdev_all_upper_get_next_dev_rcu - Get the next dev from upper list
Veaceslav Falico48311f42013-08-28 23:25:07 +02005299 * @dev: device
5300 * @iter: list_head ** of the current position
5301 *
5302 * Gets the next device from the dev's upper list, starting from iter
5303 * position. The caller must hold RCU read lock.
5304 */
Veaceslav Falico2f268f12013-09-25 09:20:07 +02005305struct net_device *netdev_all_upper_get_next_dev_rcu(struct net_device *dev,
5306 struct list_head **iter)
Veaceslav Falico48311f42013-08-28 23:25:07 +02005307{
5308 struct netdev_adjacent *upper;
5309
John Fastabend85328242013-11-26 06:33:52 +00005310 WARN_ON_ONCE(!rcu_read_lock_held() && !lockdep_rtnl_is_held());
Veaceslav Falico48311f42013-08-28 23:25:07 +02005311
5312 upper = list_entry_rcu((*iter)->next, struct netdev_adjacent, list);
5313
Veaceslav Falico2f268f12013-09-25 09:20:07 +02005314 if (&upper->list == &dev->all_adj_list.upper)
Veaceslav Falico48311f42013-08-28 23:25:07 +02005315 return NULL;
5316
5317 *iter = &upper->list;
5318
5319 return upper->dev;
5320}
Veaceslav Falico2f268f12013-09-25 09:20:07 +02005321EXPORT_SYMBOL(netdev_all_upper_get_next_dev_rcu);
Veaceslav Falico48311f42013-08-28 23:25:07 +02005322
Jiri Pirko9ff162a2013-01-03 22:48:49 +00005323/**
Veaceslav Falico31088a12013-09-25 09:20:12 +02005324 * netdev_lower_get_next_private - Get the next ->private from the
5325 * lower neighbour list
5326 * @dev: device
5327 * @iter: list_head ** of the current position
5328 *
5329 * Gets the next netdev_adjacent->private from the dev's lower neighbour
5330 * list, starting from iter position. The caller must hold either hold the
5331 * RTNL lock or its own locking that guarantees that the neighbour lower
subashab@codeaurora.orgb4691392015-07-24 03:03:29 +00005332 * list will remain unchanged.
Veaceslav Falico31088a12013-09-25 09:20:12 +02005333 */
5334void *netdev_lower_get_next_private(struct net_device *dev,
5335 struct list_head **iter)
5336{
5337 struct netdev_adjacent *lower;
5338
5339 lower = list_entry(*iter, struct netdev_adjacent, list);
5340
5341 if (&lower->list == &dev->adj_list.lower)
5342 return NULL;
5343
Veaceslav Falico6859e7d2014-04-07 11:25:12 +02005344 *iter = lower->list.next;
Veaceslav Falico31088a12013-09-25 09:20:12 +02005345
5346 return lower->private;
5347}
5348EXPORT_SYMBOL(netdev_lower_get_next_private);
5349
5350/**
5351 * netdev_lower_get_next_private_rcu - Get the next ->private from the
5352 * lower neighbour list, RCU
5353 * variant
5354 * @dev: device
5355 * @iter: list_head ** of the current position
5356 *
5357 * Gets the next netdev_adjacent->private from the dev's lower neighbour
5358 * list, starting from iter position. The caller must hold RCU read lock.
5359 */
5360void *netdev_lower_get_next_private_rcu(struct net_device *dev,
5361 struct list_head **iter)
5362{
5363 struct netdev_adjacent *lower;
5364
5365 WARN_ON_ONCE(!rcu_read_lock_held());
5366
5367 lower = list_entry_rcu((*iter)->next, struct netdev_adjacent, list);
5368
5369 if (&lower->list == &dev->adj_list.lower)
5370 return NULL;
5371
Veaceslav Falico6859e7d2014-04-07 11:25:12 +02005372 *iter = &lower->list;
Veaceslav Falico31088a12013-09-25 09:20:12 +02005373
5374 return lower->private;
5375}
5376EXPORT_SYMBOL(netdev_lower_get_next_private_rcu);
5377
5378/**
Vlad Yasevich4085ebe2014-05-16 17:04:53 -04005379 * netdev_lower_get_next - Get the next device from the lower neighbour
5380 * list
5381 * @dev: device
5382 * @iter: list_head ** of the current position
5383 *
5384 * Gets the next netdev_adjacent from the dev's lower neighbour
5385 * list, starting from iter position. The caller must hold RTNL lock or
5386 * its own locking that guarantees that the neighbour lower
subashab@codeaurora.orgb4691392015-07-24 03:03:29 +00005387 * list will remain unchanged.
Vlad Yasevich4085ebe2014-05-16 17:04:53 -04005388 */
5389void *netdev_lower_get_next(struct net_device *dev, struct list_head **iter)
5390{
5391 struct netdev_adjacent *lower;
5392
Nikolay Aleksandrovcfdd28b2016-02-17 18:00:31 +01005393 lower = list_entry(*iter, struct netdev_adjacent, list);
Vlad Yasevich4085ebe2014-05-16 17:04:53 -04005394
5395 if (&lower->list == &dev->adj_list.lower)
5396 return NULL;
5397
Nikolay Aleksandrovcfdd28b2016-02-17 18:00:31 +01005398 *iter = lower->list.next;
Vlad Yasevich4085ebe2014-05-16 17:04:53 -04005399
5400 return lower->dev;
5401}
5402EXPORT_SYMBOL(netdev_lower_get_next);
5403
5404/**
dingtianhonge001bfa2013-12-13 10:19:55 +08005405 * netdev_lower_get_first_private_rcu - Get the first ->private from the
5406 * lower neighbour list, RCU
5407 * variant
5408 * @dev: device
5409 *
5410 * Gets the first netdev_adjacent->private from the dev's lower neighbour
5411 * list. The caller must hold RCU read lock.
5412 */
5413void *netdev_lower_get_first_private_rcu(struct net_device *dev)
5414{
5415 struct netdev_adjacent *lower;
5416
5417 lower = list_first_or_null_rcu(&dev->adj_list.lower,
5418 struct netdev_adjacent, list);
5419 if (lower)
5420 return lower->private;
5421 return NULL;
5422}
5423EXPORT_SYMBOL(netdev_lower_get_first_private_rcu);
5424
5425/**
Jiri Pirko9ff162a2013-01-03 22:48:49 +00005426 * netdev_master_upper_dev_get_rcu - Get master upper device
5427 * @dev: device
5428 *
5429 * Find a master upper device and return pointer to it or NULL in case
5430 * it's not there. The caller must hold the RCU read lock.
5431 */
5432struct net_device *netdev_master_upper_dev_get_rcu(struct net_device *dev)
5433{
Veaceslav Falicoaa9d8562013-08-28 23:25:04 +02005434 struct netdev_adjacent *upper;
Jiri Pirko9ff162a2013-01-03 22:48:49 +00005435
Veaceslav Falico2f268f12013-09-25 09:20:07 +02005436 upper = list_first_or_null_rcu(&dev->adj_list.upper,
Veaceslav Falicoaa9d8562013-08-28 23:25:04 +02005437 struct netdev_adjacent, list);
Jiri Pirko9ff162a2013-01-03 22:48:49 +00005438 if (upper && likely(upper->master))
5439 return upper->dev;
5440 return NULL;
5441}
5442EXPORT_SYMBOL(netdev_master_upper_dev_get_rcu);
5443
Rashika Kheria0a59f3a2014-02-09 20:26:25 +05305444static int netdev_adjacent_sysfs_add(struct net_device *dev,
Veaceslav Falico3ee32702014-01-14 21:58:50 +01005445 struct net_device *adj_dev,
5446 struct list_head *dev_list)
5447{
5448 char linkname[IFNAMSIZ+7];
5449 sprintf(linkname, dev_list == &dev->adj_list.upper ?
5450 "upper_%s" : "lower_%s", adj_dev->name);
5451 return sysfs_create_link(&(dev->dev.kobj), &(adj_dev->dev.kobj),
5452 linkname);
5453}
Rashika Kheria0a59f3a2014-02-09 20:26:25 +05305454static void netdev_adjacent_sysfs_del(struct net_device *dev,
Veaceslav Falico3ee32702014-01-14 21:58:50 +01005455 char *name,
5456 struct list_head *dev_list)
5457{
5458 char linkname[IFNAMSIZ+7];
5459 sprintf(linkname, dev_list == &dev->adj_list.upper ?
5460 "upper_%s" : "lower_%s", name);
5461 sysfs_remove_link(&(dev->dev.kobj), linkname);
5462}
5463
Alexander Y. Fomichev7ce64c72014-09-15 14:22:35 +04005464static inline bool netdev_adjacent_is_neigh_list(struct net_device *dev,
5465 struct net_device *adj_dev,
5466 struct list_head *dev_list)
5467{
5468 return (dev_list == &dev->adj_list.upper ||
5469 dev_list == &dev->adj_list.lower) &&
5470 net_eq(dev_net(dev), dev_net(adj_dev));
5471}
Veaceslav Falico3ee32702014-01-14 21:58:50 +01005472
Veaceslav Falico5d261912013-08-28 23:25:05 +02005473static int __netdev_adjacent_dev_insert(struct net_device *dev,
5474 struct net_device *adj_dev,
Veaceslav Falico7863c052013-09-25 09:20:06 +02005475 struct list_head *dev_list,
Veaceslav Falico402dae92013-09-25 09:20:09 +02005476 void *private, bool master)
Veaceslav Falico5d261912013-08-28 23:25:05 +02005477{
5478 struct netdev_adjacent *adj;
Veaceslav Falico842d67a2013-09-25 09:20:31 +02005479 int ret;
Veaceslav Falico5d261912013-08-28 23:25:05 +02005480
Michal Kubeček6ea29da2015-09-24 10:59:05 +02005481 adj = __netdev_find_adj(adj_dev, dev_list);
Veaceslav Falico5d261912013-08-28 23:25:05 +02005482
5483 if (adj) {
Veaceslav Falico5d261912013-08-28 23:25:05 +02005484 adj->ref_nr++;
5485 return 0;
5486 }
5487
5488 adj = kmalloc(sizeof(*adj), GFP_KERNEL);
5489 if (!adj)
5490 return -ENOMEM;
5491
5492 adj->dev = adj_dev;
5493 adj->master = master;
Veaceslav Falico5d261912013-08-28 23:25:05 +02005494 adj->ref_nr = 1;
Veaceslav Falico402dae92013-09-25 09:20:09 +02005495 adj->private = private;
Veaceslav Falico5d261912013-08-28 23:25:05 +02005496 dev_hold(adj_dev);
Veaceslav Falico2f268f12013-09-25 09:20:07 +02005497
5498 pr_debug("dev_hold for %s, because of link added from %s to %s\n",
5499 adj_dev->name, dev->name, adj_dev->name);
Veaceslav Falico5d261912013-08-28 23:25:05 +02005500
Alexander Y. Fomichev7ce64c72014-09-15 14:22:35 +04005501 if (netdev_adjacent_is_neigh_list(dev, adj_dev, dev_list)) {
Veaceslav Falico3ee32702014-01-14 21:58:50 +01005502 ret = netdev_adjacent_sysfs_add(dev, adj_dev, dev_list);
Veaceslav Falico5831d66e2013-09-25 09:20:32 +02005503 if (ret)
5504 goto free_adj;
5505 }
5506
Veaceslav Falico7863c052013-09-25 09:20:06 +02005507 /* Ensure that master link is always the first item in list. */
Veaceslav Falico842d67a2013-09-25 09:20:31 +02005508 if (master) {
5509 ret = sysfs_create_link(&(dev->dev.kobj),
5510 &(adj_dev->dev.kobj), "master");
5511 if (ret)
Veaceslav Falico5831d66e2013-09-25 09:20:32 +02005512 goto remove_symlinks;
Veaceslav Falico842d67a2013-09-25 09:20:31 +02005513
Veaceslav Falico7863c052013-09-25 09:20:06 +02005514 list_add_rcu(&adj->list, dev_list);
Veaceslav Falico842d67a2013-09-25 09:20:31 +02005515 } else {
Veaceslav Falico7863c052013-09-25 09:20:06 +02005516 list_add_tail_rcu(&adj->list, dev_list);
Veaceslav Falico842d67a2013-09-25 09:20:31 +02005517 }
Veaceslav Falico5d261912013-08-28 23:25:05 +02005518
5519 return 0;
Veaceslav Falico842d67a2013-09-25 09:20:31 +02005520
Veaceslav Falico5831d66e2013-09-25 09:20:32 +02005521remove_symlinks:
Alexander Y. Fomichev7ce64c72014-09-15 14:22:35 +04005522 if (netdev_adjacent_is_neigh_list(dev, adj_dev, dev_list))
Veaceslav Falico3ee32702014-01-14 21:58:50 +01005523 netdev_adjacent_sysfs_del(dev, adj_dev->name, dev_list);
Veaceslav Falico842d67a2013-09-25 09:20:31 +02005524free_adj:
5525 kfree(adj);
Nikolay Aleksandrov974daef2013-10-23 15:28:56 +02005526 dev_put(adj_dev);
Veaceslav Falico842d67a2013-09-25 09:20:31 +02005527
5528 return ret;
Veaceslav Falico5d261912013-08-28 23:25:05 +02005529}
5530
stephen hemminger1d143d92013-12-29 14:01:29 -08005531static void __netdev_adjacent_dev_remove(struct net_device *dev,
5532 struct net_device *adj_dev,
5533 struct list_head *dev_list)
Veaceslav Falico5d261912013-08-28 23:25:05 +02005534{
5535 struct netdev_adjacent *adj;
5536
Michal Kubeček6ea29da2015-09-24 10:59:05 +02005537 adj = __netdev_find_adj(adj_dev, dev_list);
Veaceslav Falico5d261912013-08-28 23:25:05 +02005538
Veaceslav Falico2f268f12013-09-25 09:20:07 +02005539 if (!adj) {
5540 pr_err("tried to remove device %s from %s\n",
5541 dev->name, adj_dev->name);
Veaceslav Falico5d261912013-08-28 23:25:05 +02005542 BUG();
Veaceslav Falico2f268f12013-09-25 09:20:07 +02005543 }
Veaceslav Falico5d261912013-08-28 23:25:05 +02005544
5545 if (adj->ref_nr > 1) {
Veaceslav Falico2f268f12013-09-25 09:20:07 +02005546 pr_debug("%s to %s ref_nr-- = %d\n", dev->name, adj_dev->name,
5547 adj->ref_nr-1);
Veaceslav Falico5d261912013-08-28 23:25:05 +02005548 adj->ref_nr--;
5549 return;
5550 }
5551
Veaceslav Falico842d67a2013-09-25 09:20:31 +02005552 if (adj->master)
5553 sysfs_remove_link(&(dev->dev.kobj), "master");
5554
Alexander Y. Fomichev7ce64c72014-09-15 14:22:35 +04005555 if (netdev_adjacent_is_neigh_list(dev, adj_dev, dev_list))
Veaceslav Falico3ee32702014-01-14 21:58:50 +01005556 netdev_adjacent_sysfs_del(dev, adj_dev->name, dev_list);
Veaceslav Falico5831d66e2013-09-25 09:20:32 +02005557
Veaceslav Falico5d261912013-08-28 23:25:05 +02005558 list_del_rcu(&adj->list);
Veaceslav Falico2f268f12013-09-25 09:20:07 +02005559 pr_debug("dev_put for %s, because link removed from %s to %s\n",
5560 adj_dev->name, dev->name, adj_dev->name);
Veaceslav Falico5d261912013-08-28 23:25:05 +02005561 dev_put(adj_dev);
5562 kfree_rcu(adj, rcu);
5563}
5564
stephen hemminger1d143d92013-12-29 14:01:29 -08005565static int __netdev_adjacent_dev_link_lists(struct net_device *dev,
5566 struct net_device *upper_dev,
5567 struct list_head *up_list,
5568 struct list_head *down_list,
5569 void *private, bool master)
Veaceslav Falico5d261912013-08-28 23:25:05 +02005570{
5571 int ret;
5572
Veaceslav Falico402dae92013-09-25 09:20:09 +02005573 ret = __netdev_adjacent_dev_insert(dev, upper_dev, up_list, private,
5574 master);
Veaceslav Falico5d261912013-08-28 23:25:05 +02005575 if (ret)
5576 return ret;
5577
Veaceslav Falico402dae92013-09-25 09:20:09 +02005578 ret = __netdev_adjacent_dev_insert(upper_dev, dev, down_list, private,
5579 false);
Veaceslav Falico5d261912013-08-28 23:25:05 +02005580 if (ret) {
Veaceslav Falico2f268f12013-09-25 09:20:07 +02005581 __netdev_adjacent_dev_remove(dev, upper_dev, up_list);
Veaceslav Falico5d261912013-08-28 23:25:05 +02005582 return ret;
5583 }
5584
5585 return 0;
5586}
5587
stephen hemminger1d143d92013-12-29 14:01:29 -08005588static int __netdev_adjacent_dev_link(struct net_device *dev,
5589 struct net_device *upper_dev)
Veaceslav Falico5d261912013-08-28 23:25:05 +02005590{
Veaceslav Falico2f268f12013-09-25 09:20:07 +02005591 return __netdev_adjacent_dev_link_lists(dev, upper_dev,
5592 &dev->all_adj_list.upper,
5593 &upper_dev->all_adj_list.lower,
Veaceslav Falico402dae92013-09-25 09:20:09 +02005594 NULL, false);
Veaceslav Falico5d261912013-08-28 23:25:05 +02005595}
5596
stephen hemminger1d143d92013-12-29 14:01:29 -08005597static void __netdev_adjacent_dev_unlink_lists(struct net_device *dev,
5598 struct net_device *upper_dev,
5599 struct list_head *up_list,
5600 struct list_head *down_list)
Veaceslav Falico5d261912013-08-28 23:25:05 +02005601{
Veaceslav Falico2f268f12013-09-25 09:20:07 +02005602 __netdev_adjacent_dev_remove(dev, upper_dev, up_list);
5603 __netdev_adjacent_dev_remove(upper_dev, dev, down_list);
Veaceslav Falico5d261912013-08-28 23:25:05 +02005604}
5605
stephen hemminger1d143d92013-12-29 14:01:29 -08005606static void __netdev_adjacent_dev_unlink(struct net_device *dev,
5607 struct net_device *upper_dev)
Veaceslav Falico5d261912013-08-28 23:25:05 +02005608{
Veaceslav Falico2f268f12013-09-25 09:20:07 +02005609 __netdev_adjacent_dev_unlink_lists(dev, upper_dev,
5610 &dev->all_adj_list.upper,
5611 &upper_dev->all_adj_list.lower);
Veaceslav Falico5d261912013-08-28 23:25:05 +02005612}
5613
stephen hemminger1d143d92013-12-29 14:01:29 -08005614static int __netdev_adjacent_dev_link_neighbour(struct net_device *dev,
5615 struct net_device *upper_dev,
5616 void *private, bool master)
Veaceslav Falico2f268f12013-09-25 09:20:07 +02005617{
5618 int ret = __netdev_adjacent_dev_link(dev, upper_dev);
5619
5620 if (ret)
5621 return ret;
5622
5623 ret = __netdev_adjacent_dev_link_lists(dev, upper_dev,
5624 &dev->adj_list.upper,
5625 &upper_dev->adj_list.lower,
Veaceslav Falico402dae92013-09-25 09:20:09 +02005626 private, master);
Veaceslav Falico2f268f12013-09-25 09:20:07 +02005627 if (ret) {
5628 __netdev_adjacent_dev_unlink(dev, upper_dev);
5629 return ret;
5630 }
5631
5632 return 0;
5633}
5634
stephen hemminger1d143d92013-12-29 14:01:29 -08005635static void __netdev_adjacent_dev_unlink_neighbour(struct net_device *dev,
5636 struct net_device *upper_dev)
Veaceslav Falico2f268f12013-09-25 09:20:07 +02005637{
5638 __netdev_adjacent_dev_unlink(dev, upper_dev);
5639 __netdev_adjacent_dev_unlink_lists(dev, upper_dev,
5640 &dev->adj_list.upper,
5641 &upper_dev->adj_list.lower);
5642}
Veaceslav Falico5d261912013-08-28 23:25:05 +02005643
Jiri Pirko9ff162a2013-01-03 22:48:49 +00005644static int __netdev_upper_dev_link(struct net_device *dev,
Veaceslav Falico402dae92013-09-25 09:20:09 +02005645 struct net_device *upper_dev, bool master,
Jiri Pirko29bf24a2015-12-03 12:12:11 +01005646 void *upper_priv, void *upper_info)
Jiri Pirko9ff162a2013-01-03 22:48:49 +00005647{
Jiri Pirko0e4ead92015-08-27 09:31:18 +02005648 struct netdev_notifier_changeupper_info changeupper_info;
Veaceslav Falico5d261912013-08-28 23:25:05 +02005649 struct netdev_adjacent *i, *j, *to_i, *to_j;
5650 int ret = 0;
Jiri Pirko9ff162a2013-01-03 22:48:49 +00005651
5652 ASSERT_RTNL();
5653
5654 if (dev == upper_dev)
5655 return -EBUSY;
5656
5657 /* To prevent loops, check if dev is not upper device to upper_dev. */
Michal Kubeček6ea29da2015-09-24 10:59:05 +02005658 if (__netdev_find_adj(dev, &upper_dev->all_adj_list.upper))
Jiri Pirko9ff162a2013-01-03 22:48:49 +00005659 return -EBUSY;
5660
Michal Kubeček6ea29da2015-09-24 10:59:05 +02005661 if (__netdev_find_adj(upper_dev, &dev->adj_list.upper))
Jiri Pirko9ff162a2013-01-03 22:48:49 +00005662 return -EEXIST;
5663
5664 if (master && netdev_master_upper_dev_get(dev))
5665 return -EBUSY;
5666
Jiri Pirko0e4ead92015-08-27 09:31:18 +02005667 changeupper_info.upper_dev = upper_dev;
5668 changeupper_info.master = master;
5669 changeupper_info.linking = true;
Jiri Pirko29bf24a2015-12-03 12:12:11 +01005670 changeupper_info.upper_info = upper_info;
Jiri Pirko0e4ead92015-08-27 09:31:18 +02005671
Jiri Pirko573c7ba2015-10-16 14:01:22 +02005672 ret = call_netdevice_notifiers_info(NETDEV_PRECHANGEUPPER, dev,
5673 &changeupper_info.info);
5674 ret = notifier_to_errno(ret);
5675 if (ret)
5676 return ret;
5677
Jiri Pirko6dffb042015-12-03 12:12:10 +01005678 ret = __netdev_adjacent_dev_link_neighbour(dev, upper_dev, upper_priv,
Veaceslav Falico402dae92013-09-25 09:20:09 +02005679 master);
Veaceslav Falico5d261912013-08-28 23:25:05 +02005680 if (ret)
5681 return ret;
Jiri Pirko9ff162a2013-01-03 22:48:49 +00005682
Veaceslav Falico5d261912013-08-28 23:25:05 +02005683 /* Now that we linked these devs, make all the upper_dev's
Veaceslav Falico2f268f12013-09-25 09:20:07 +02005684 * all_adj_list.upper visible to every dev's all_adj_list.lower an
Veaceslav Falico5d261912013-08-28 23:25:05 +02005685 * versa, and don't forget the devices itself. All of these
5686 * links are non-neighbours.
5687 */
Veaceslav Falico2f268f12013-09-25 09:20:07 +02005688 list_for_each_entry(i, &dev->all_adj_list.lower, list) {
5689 list_for_each_entry(j, &upper_dev->all_adj_list.upper, list) {
5690 pr_debug("Interlinking %s with %s, non-neighbour\n",
5691 i->dev->name, j->dev->name);
Veaceslav Falico5d261912013-08-28 23:25:05 +02005692 ret = __netdev_adjacent_dev_link(i->dev, j->dev);
5693 if (ret)
5694 goto rollback_mesh;
5695 }
5696 }
Jiri Pirko9ff162a2013-01-03 22:48:49 +00005697
Veaceslav Falico5d261912013-08-28 23:25:05 +02005698 /* add dev to every upper_dev's upper device */
Veaceslav Falico2f268f12013-09-25 09:20:07 +02005699 list_for_each_entry(i, &upper_dev->all_adj_list.upper, list) {
5700 pr_debug("linking %s's upper device %s with %s\n",
5701 upper_dev->name, i->dev->name, dev->name);
Veaceslav Falico5d261912013-08-28 23:25:05 +02005702 ret = __netdev_adjacent_dev_link(dev, i->dev);
5703 if (ret)
5704 goto rollback_upper_mesh;
5705 }
5706
5707 /* add upper_dev to every dev's lower device */
Veaceslav Falico2f268f12013-09-25 09:20:07 +02005708 list_for_each_entry(i, &dev->all_adj_list.lower, list) {
5709 pr_debug("linking %s's lower device %s with %s\n", dev->name,
5710 i->dev->name, upper_dev->name);
Veaceslav Falico5d261912013-08-28 23:25:05 +02005711 ret = __netdev_adjacent_dev_link(i->dev, upper_dev);
5712 if (ret)
5713 goto rollback_lower_mesh;
5714 }
5715
Ido Schimmelb03804e2015-12-03 12:12:03 +01005716 ret = call_netdevice_notifiers_info(NETDEV_CHANGEUPPER, dev,
5717 &changeupper_info.info);
5718 ret = notifier_to_errno(ret);
5719 if (ret)
5720 goto rollback_lower_mesh;
5721
Jiri Pirko9ff162a2013-01-03 22:48:49 +00005722 return 0;
Veaceslav Falico5d261912013-08-28 23:25:05 +02005723
5724rollback_lower_mesh:
5725 to_i = i;
Veaceslav Falico2f268f12013-09-25 09:20:07 +02005726 list_for_each_entry(i, &dev->all_adj_list.lower, list) {
Veaceslav Falico5d261912013-08-28 23:25:05 +02005727 if (i == to_i)
5728 break;
5729 __netdev_adjacent_dev_unlink(i->dev, upper_dev);
5730 }
5731
5732 i = NULL;
5733
5734rollback_upper_mesh:
5735 to_i = i;
Veaceslav Falico2f268f12013-09-25 09:20:07 +02005736 list_for_each_entry(i, &upper_dev->all_adj_list.upper, list) {
Veaceslav Falico5d261912013-08-28 23:25:05 +02005737 if (i == to_i)
5738 break;
5739 __netdev_adjacent_dev_unlink(dev, i->dev);
5740 }
5741
5742 i = j = NULL;
5743
5744rollback_mesh:
5745 to_i = i;
5746 to_j = j;
Veaceslav Falico2f268f12013-09-25 09:20:07 +02005747 list_for_each_entry(i, &dev->all_adj_list.lower, list) {
5748 list_for_each_entry(j, &upper_dev->all_adj_list.upper, list) {
Veaceslav Falico5d261912013-08-28 23:25:05 +02005749 if (i == to_i && j == to_j)
5750 break;
5751 __netdev_adjacent_dev_unlink(i->dev, j->dev);
5752 }
5753 if (i == to_i)
5754 break;
5755 }
5756
Veaceslav Falico2f268f12013-09-25 09:20:07 +02005757 __netdev_adjacent_dev_unlink_neighbour(dev, upper_dev);
Veaceslav Falico5d261912013-08-28 23:25:05 +02005758
5759 return ret;
Jiri Pirko9ff162a2013-01-03 22:48:49 +00005760}
5761
5762/**
5763 * netdev_upper_dev_link - Add a link to the upper device
5764 * @dev: device
5765 * @upper_dev: new upper device
5766 *
5767 * Adds a link to device which is upper to this one. The caller must hold
5768 * the RTNL lock. On a failure a negative errno code is returned.
5769 * On success the reference counts are adjusted and the function
5770 * returns zero.
5771 */
5772int netdev_upper_dev_link(struct net_device *dev,
5773 struct net_device *upper_dev)
5774{
Jiri Pirko29bf24a2015-12-03 12:12:11 +01005775 return __netdev_upper_dev_link(dev, upper_dev, false, NULL, NULL);
Jiri Pirko9ff162a2013-01-03 22:48:49 +00005776}
5777EXPORT_SYMBOL(netdev_upper_dev_link);
5778
5779/**
5780 * netdev_master_upper_dev_link - Add a master link to the upper device
5781 * @dev: device
5782 * @upper_dev: new upper device
Jiri Pirko6dffb042015-12-03 12:12:10 +01005783 * @upper_priv: upper device private
Jiri Pirko29bf24a2015-12-03 12:12:11 +01005784 * @upper_info: upper info to be passed down via notifier
Jiri Pirko9ff162a2013-01-03 22:48:49 +00005785 *
5786 * Adds a link to device which is upper to this one. In this case, only
5787 * one master upper device can be linked, although other non-master devices
5788 * might be linked as well. The caller must hold the RTNL lock.
5789 * On a failure a negative errno code is returned. On success the reference
5790 * counts are adjusted and the function returns zero.
5791 */
5792int netdev_master_upper_dev_link(struct net_device *dev,
Jiri Pirko6dffb042015-12-03 12:12:10 +01005793 struct net_device *upper_dev,
Jiri Pirko29bf24a2015-12-03 12:12:11 +01005794 void *upper_priv, void *upper_info)
Jiri Pirko9ff162a2013-01-03 22:48:49 +00005795{
Jiri Pirko29bf24a2015-12-03 12:12:11 +01005796 return __netdev_upper_dev_link(dev, upper_dev, true,
5797 upper_priv, upper_info);
Jiri Pirko9ff162a2013-01-03 22:48:49 +00005798}
5799EXPORT_SYMBOL(netdev_master_upper_dev_link);
5800
5801/**
5802 * netdev_upper_dev_unlink - Removes a link to upper device
5803 * @dev: device
5804 * @upper_dev: new upper device
5805 *
5806 * Removes a link to device which is upper to this one. The caller must hold
5807 * the RTNL lock.
5808 */
5809void netdev_upper_dev_unlink(struct net_device *dev,
5810 struct net_device *upper_dev)
5811{
Jiri Pirko0e4ead92015-08-27 09:31:18 +02005812 struct netdev_notifier_changeupper_info changeupper_info;
Veaceslav Falico5d261912013-08-28 23:25:05 +02005813 struct netdev_adjacent *i, *j;
Jiri Pirko9ff162a2013-01-03 22:48:49 +00005814 ASSERT_RTNL();
5815
Jiri Pirko0e4ead92015-08-27 09:31:18 +02005816 changeupper_info.upper_dev = upper_dev;
5817 changeupper_info.master = netdev_master_upper_dev_get(dev) == upper_dev;
5818 changeupper_info.linking = false;
5819
Jiri Pirko573c7ba2015-10-16 14:01:22 +02005820 call_netdevice_notifiers_info(NETDEV_PRECHANGEUPPER, dev,
5821 &changeupper_info.info);
5822
Veaceslav Falico2f268f12013-09-25 09:20:07 +02005823 __netdev_adjacent_dev_unlink_neighbour(dev, upper_dev);
Veaceslav Falico5d261912013-08-28 23:25:05 +02005824
5825 /* Here is the tricky part. We must remove all dev's lower
5826 * devices from all upper_dev's upper devices and vice
5827 * versa, to maintain the graph relationship.
5828 */
Veaceslav Falico2f268f12013-09-25 09:20:07 +02005829 list_for_each_entry(i, &dev->all_adj_list.lower, list)
5830 list_for_each_entry(j, &upper_dev->all_adj_list.upper, list)
Veaceslav Falico5d261912013-08-28 23:25:05 +02005831 __netdev_adjacent_dev_unlink(i->dev, j->dev);
5832
5833 /* remove also the devices itself from lower/upper device
5834 * list
5835 */
Veaceslav Falico2f268f12013-09-25 09:20:07 +02005836 list_for_each_entry(i, &dev->all_adj_list.lower, list)
Veaceslav Falico5d261912013-08-28 23:25:05 +02005837 __netdev_adjacent_dev_unlink(i->dev, upper_dev);
5838
Veaceslav Falico2f268f12013-09-25 09:20:07 +02005839 list_for_each_entry(i, &upper_dev->all_adj_list.upper, list)
Veaceslav Falico5d261912013-08-28 23:25:05 +02005840 __netdev_adjacent_dev_unlink(dev, i->dev);
5841
Jiri Pirko0e4ead92015-08-27 09:31:18 +02005842 call_netdevice_notifiers_info(NETDEV_CHANGEUPPER, dev,
5843 &changeupper_info.info);
Jiri Pirko9ff162a2013-01-03 22:48:49 +00005844}
5845EXPORT_SYMBOL(netdev_upper_dev_unlink);
5846
Moni Shoua61bd3852015-02-03 16:48:29 +02005847/**
5848 * netdev_bonding_info_change - Dispatch event about slave change
5849 * @dev: device
Masanari Iida4a26e4532015-02-14 22:26:34 +09005850 * @bonding_info: info to dispatch
Moni Shoua61bd3852015-02-03 16:48:29 +02005851 *
5852 * Send NETDEV_BONDING_INFO to netdev notifiers with info.
5853 * The caller must hold the RTNL lock.
5854 */
5855void netdev_bonding_info_change(struct net_device *dev,
5856 struct netdev_bonding_info *bonding_info)
5857{
5858 struct netdev_notifier_bonding_info info;
5859
5860 memcpy(&info.bonding_info, bonding_info,
5861 sizeof(struct netdev_bonding_info));
5862 call_netdevice_notifiers_info(NETDEV_BONDING_INFO, dev,
5863 &info.info);
5864}
5865EXPORT_SYMBOL(netdev_bonding_info_change);
5866
Eric Dumazet2ce1ee12015-02-04 13:37:44 -08005867static void netdev_adjacent_add_links(struct net_device *dev)
Alexander Y. Fomichev4c754312014-08-25 16:26:45 +04005868{
5869 struct netdev_adjacent *iter;
5870
5871 struct net *net = dev_net(dev);
5872
5873 list_for_each_entry(iter, &dev->adj_list.upper, list) {
5874 if (!net_eq(net,dev_net(iter->dev)))
5875 continue;
5876 netdev_adjacent_sysfs_add(iter->dev, dev,
5877 &iter->dev->adj_list.lower);
5878 netdev_adjacent_sysfs_add(dev, iter->dev,
5879 &dev->adj_list.upper);
5880 }
5881
5882 list_for_each_entry(iter, &dev->adj_list.lower, list) {
5883 if (!net_eq(net,dev_net(iter->dev)))
5884 continue;
5885 netdev_adjacent_sysfs_add(iter->dev, dev,
5886 &iter->dev->adj_list.upper);
5887 netdev_adjacent_sysfs_add(dev, iter->dev,
5888 &dev->adj_list.lower);
5889 }
5890}
5891
Eric Dumazet2ce1ee12015-02-04 13:37:44 -08005892static void netdev_adjacent_del_links(struct net_device *dev)
Alexander Y. Fomichev4c754312014-08-25 16:26:45 +04005893{
5894 struct netdev_adjacent *iter;
5895
5896 struct net *net = dev_net(dev);
5897
5898 list_for_each_entry(iter, &dev->adj_list.upper, list) {
5899 if (!net_eq(net,dev_net(iter->dev)))
5900 continue;
5901 netdev_adjacent_sysfs_del(iter->dev, dev->name,
5902 &iter->dev->adj_list.lower);
5903 netdev_adjacent_sysfs_del(dev, iter->dev->name,
5904 &dev->adj_list.upper);
5905 }
5906
5907 list_for_each_entry(iter, &dev->adj_list.lower, list) {
5908 if (!net_eq(net,dev_net(iter->dev)))
5909 continue;
5910 netdev_adjacent_sysfs_del(iter->dev, dev->name,
5911 &iter->dev->adj_list.upper);
5912 netdev_adjacent_sysfs_del(dev, iter->dev->name,
5913 &dev->adj_list.lower);
5914 }
5915}
5916
Veaceslav Falico5bb025f2014-01-14 21:58:51 +01005917void netdev_adjacent_rename_links(struct net_device *dev, char *oldname)
Veaceslav Falico402dae92013-09-25 09:20:09 +02005918{
Veaceslav Falico5bb025f2014-01-14 21:58:51 +01005919 struct netdev_adjacent *iter;
Veaceslav Falico402dae92013-09-25 09:20:09 +02005920
Alexander Y. Fomichev4c754312014-08-25 16:26:45 +04005921 struct net *net = dev_net(dev);
5922
Veaceslav Falico5bb025f2014-01-14 21:58:51 +01005923 list_for_each_entry(iter, &dev->adj_list.upper, list) {
Alexander Y. Fomichev4c754312014-08-25 16:26:45 +04005924 if (!net_eq(net,dev_net(iter->dev)))
5925 continue;
Veaceslav Falico5bb025f2014-01-14 21:58:51 +01005926 netdev_adjacent_sysfs_del(iter->dev, oldname,
5927 &iter->dev->adj_list.lower);
5928 netdev_adjacent_sysfs_add(iter->dev, dev,
5929 &iter->dev->adj_list.lower);
5930 }
Veaceslav Falico402dae92013-09-25 09:20:09 +02005931
Veaceslav Falico5bb025f2014-01-14 21:58:51 +01005932 list_for_each_entry(iter, &dev->adj_list.lower, list) {
Alexander Y. Fomichev4c754312014-08-25 16:26:45 +04005933 if (!net_eq(net,dev_net(iter->dev)))
5934 continue;
Veaceslav Falico5bb025f2014-01-14 21:58:51 +01005935 netdev_adjacent_sysfs_del(iter->dev, oldname,
5936 &iter->dev->adj_list.upper);
5937 netdev_adjacent_sysfs_add(iter->dev, dev,
5938 &iter->dev->adj_list.upper);
5939 }
Veaceslav Falico402dae92013-09-25 09:20:09 +02005940}
Veaceslav Falico402dae92013-09-25 09:20:09 +02005941
5942void *netdev_lower_dev_get_private(struct net_device *dev,
5943 struct net_device *lower_dev)
5944{
5945 struct netdev_adjacent *lower;
5946
5947 if (!lower_dev)
5948 return NULL;
Michal Kubeček6ea29da2015-09-24 10:59:05 +02005949 lower = __netdev_find_adj(lower_dev, &dev->adj_list.lower);
Veaceslav Falico402dae92013-09-25 09:20:09 +02005950 if (!lower)
5951 return NULL;
5952
5953 return lower->private;
5954}
5955EXPORT_SYMBOL(netdev_lower_dev_get_private);
5956
Vlad Yasevich4085ebe2014-05-16 17:04:53 -04005957
5958int dev_get_nest_level(struct net_device *dev,
Jiri Pirkob618aaa2015-12-04 15:01:31 +01005959 bool (*type_check)(const struct net_device *dev))
Vlad Yasevich4085ebe2014-05-16 17:04:53 -04005960{
5961 struct net_device *lower = NULL;
5962 struct list_head *iter;
5963 int max_nest = -1;
5964 int nest;
5965
5966 ASSERT_RTNL();
5967
5968 netdev_for_each_lower_dev(dev, lower, iter) {
5969 nest = dev_get_nest_level(lower, type_check);
5970 if (max_nest < nest)
5971 max_nest = nest;
5972 }
5973
5974 if (type_check(dev))
5975 max_nest++;
5976
5977 return max_nest;
5978}
5979EXPORT_SYMBOL(dev_get_nest_level);
5980
Jiri Pirko04d48262015-12-03 12:12:15 +01005981/**
5982 * netdev_lower_change - Dispatch event about lower device state change
5983 * @lower_dev: device
5984 * @lower_state_info: state to dispatch
5985 *
5986 * Send NETDEV_CHANGELOWERSTATE to netdev notifiers with info.
5987 * The caller must hold the RTNL lock.
5988 */
5989void netdev_lower_state_changed(struct net_device *lower_dev,
5990 void *lower_state_info)
5991{
5992 struct netdev_notifier_changelowerstate_info changelowerstate_info;
5993
5994 ASSERT_RTNL();
5995 changelowerstate_info.lower_state_info = lower_state_info;
5996 call_netdevice_notifiers_info(NETDEV_CHANGELOWERSTATE, lower_dev,
5997 &changelowerstate_info.info);
5998}
5999EXPORT_SYMBOL(netdev_lower_state_changed);
6000
Patrick McHardyb6c40d62008-10-07 15:26:48 -07006001static void dev_change_rx_flags(struct net_device *dev, int flags)
6002{
Stephen Hemmingerd3147742008-11-19 21:32:24 -08006003 const struct net_device_ops *ops = dev->netdev_ops;
6004
Vlad Yasevichd2615bf2013-11-19 20:47:15 -05006005 if (ops->ndo_change_rx_flags)
Stephen Hemmingerd3147742008-11-19 21:32:24 -08006006 ops->ndo_change_rx_flags(dev, flags);
Patrick McHardyb6c40d62008-10-07 15:26:48 -07006007}
6008
Nicolas Dichtel991fb3f2013-09-25 12:02:45 +02006009static int __dev_set_promiscuity(struct net_device *dev, int inc, bool notify)
Patrick McHardy4417da62007-06-27 01:28:10 -07006010{
Eric Dumazetb536db92011-11-30 21:42:26 +00006011 unsigned int old_flags = dev->flags;
Eric W. Biedermand04a48b2012-05-23 17:01:57 -06006012 kuid_t uid;
6013 kgid_t gid;
Patrick McHardy4417da62007-06-27 01:28:10 -07006014
Patrick McHardy24023452007-07-14 18:51:31 -07006015 ASSERT_RTNL();
6016
Wang Chendad9b332008-06-18 01:48:28 -07006017 dev->flags |= IFF_PROMISC;
6018 dev->promiscuity += inc;
6019 if (dev->promiscuity == 0) {
6020 /*
6021 * Avoid overflow.
6022 * If inc causes overflow, untouch promisc and return error.
6023 */
6024 if (inc < 0)
6025 dev->flags &= ~IFF_PROMISC;
6026 else {
6027 dev->promiscuity -= inc;
Joe Perches7b6cd1c2012-02-01 10:54:43 +00006028 pr_warn("%s: promiscuity touches roof, set promiscuity failed. promiscuity feature of device might be broken.\n",
6029 dev->name);
Wang Chendad9b332008-06-18 01:48:28 -07006030 return -EOVERFLOW;
6031 }
6032 }
Patrick McHardy4417da62007-06-27 01:28:10 -07006033 if (dev->flags != old_flags) {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00006034 pr_info("device %s %s promiscuous mode\n",
6035 dev->name,
6036 dev->flags & IFF_PROMISC ? "entered" : "left");
David Howells8192b0c2008-11-14 10:39:10 +11006037 if (audit_enabled) {
6038 current_uid_gid(&uid, &gid);
Klaus Heinrich Kiwi7759db82008-01-23 22:57:45 -05006039 audit_log(current->audit_context, GFP_ATOMIC,
6040 AUDIT_ANOM_PROMISCUOUS,
6041 "dev=%s prom=%d old_prom=%d auid=%u uid=%u gid=%u ses=%u",
6042 dev->name, (dev->flags & IFF_PROMISC),
6043 (old_flags & IFF_PROMISC),
Eric W. Biedermane1760bd2012-09-10 22:39:43 -07006044 from_kuid(&init_user_ns, audit_get_loginuid(current)),
Eric W. Biedermand04a48b2012-05-23 17:01:57 -06006045 from_kuid(&init_user_ns, uid),
6046 from_kgid(&init_user_ns, gid),
Klaus Heinrich Kiwi7759db82008-01-23 22:57:45 -05006047 audit_get_sessionid(current));
David Howells8192b0c2008-11-14 10:39:10 +11006048 }
Patrick McHardy24023452007-07-14 18:51:31 -07006049
Patrick McHardyb6c40d62008-10-07 15:26:48 -07006050 dev_change_rx_flags(dev, IFF_PROMISC);
Patrick McHardy4417da62007-06-27 01:28:10 -07006051 }
Nicolas Dichtel991fb3f2013-09-25 12:02:45 +02006052 if (notify)
6053 __dev_notify_flags(dev, old_flags, IFF_PROMISC);
Wang Chendad9b332008-06-18 01:48:28 -07006054 return 0;
Patrick McHardy4417da62007-06-27 01:28:10 -07006055}
6056
Linus Torvalds1da177e2005-04-16 15:20:36 -07006057/**
6058 * dev_set_promiscuity - update promiscuity count on a device
6059 * @dev: device
6060 * @inc: modifier
6061 *
Stephen Hemminger3041a062006-05-26 13:25:24 -07006062 * Add or remove promiscuity from a device. While the count in the device
Linus Torvalds1da177e2005-04-16 15:20:36 -07006063 * remains above zero the interface remains promiscuous. Once it hits zero
6064 * the device reverts back to normal filtering operation. A negative inc
6065 * value is used to drop promiscuity on the device.
Wang Chendad9b332008-06-18 01:48:28 -07006066 * Return 0 if successful or a negative errno code on error.
Linus Torvalds1da177e2005-04-16 15:20:36 -07006067 */
Wang Chendad9b332008-06-18 01:48:28 -07006068int dev_set_promiscuity(struct net_device *dev, int inc)
Linus Torvalds1da177e2005-04-16 15:20:36 -07006069{
Eric Dumazetb536db92011-11-30 21:42:26 +00006070 unsigned int old_flags = dev->flags;
Wang Chendad9b332008-06-18 01:48:28 -07006071 int err;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006072
Nicolas Dichtel991fb3f2013-09-25 12:02:45 +02006073 err = __dev_set_promiscuity(dev, inc, true);
Patrick McHardy4b5a6982008-07-06 15:49:08 -07006074 if (err < 0)
Wang Chendad9b332008-06-18 01:48:28 -07006075 return err;
Patrick McHardy4417da62007-06-27 01:28:10 -07006076 if (dev->flags != old_flags)
6077 dev_set_rx_mode(dev);
Wang Chendad9b332008-06-18 01:48:28 -07006078 return err;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006079}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07006080EXPORT_SYMBOL(dev_set_promiscuity);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006081
Nicolas Dichtel991fb3f2013-09-25 12:02:45 +02006082static int __dev_set_allmulti(struct net_device *dev, int inc, bool notify)
Linus Torvalds1da177e2005-04-16 15:20:36 -07006083{
Nicolas Dichtel991fb3f2013-09-25 12:02:45 +02006084 unsigned int old_flags = dev->flags, old_gflags = dev->gflags;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006085
Patrick McHardy24023452007-07-14 18:51:31 -07006086 ASSERT_RTNL();
6087
Linus Torvalds1da177e2005-04-16 15:20:36 -07006088 dev->flags |= IFF_ALLMULTI;
Wang Chendad9b332008-06-18 01:48:28 -07006089 dev->allmulti += inc;
6090 if (dev->allmulti == 0) {
6091 /*
6092 * Avoid overflow.
6093 * If inc causes overflow, untouch allmulti and return error.
6094 */
6095 if (inc < 0)
6096 dev->flags &= ~IFF_ALLMULTI;
6097 else {
6098 dev->allmulti -= inc;
Joe Perches7b6cd1c2012-02-01 10:54:43 +00006099 pr_warn("%s: allmulti touches roof, set allmulti failed. allmulti feature of device might be broken.\n",
6100 dev->name);
Wang Chendad9b332008-06-18 01:48:28 -07006101 return -EOVERFLOW;
6102 }
6103 }
Patrick McHardy24023452007-07-14 18:51:31 -07006104 if (dev->flags ^ old_flags) {
Patrick McHardyb6c40d62008-10-07 15:26:48 -07006105 dev_change_rx_flags(dev, IFF_ALLMULTI);
Patrick McHardy4417da62007-06-27 01:28:10 -07006106 dev_set_rx_mode(dev);
Nicolas Dichtel991fb3f2013-09-25 12:02:45 +02006107 if (notify)
6108 __dev_notify_flags(dev, old_flags,
6109 dev->gflags ^ old_gflags);
Patrick McHardy24023452007-07-14 18:51:31 -07006110 }
Wang Chendad9b332008-06-18 01:48:28 -07006111 return 0;
Patrick McHardy4417da62007-06-27 01:28:10 -07006112}
Nicolas Dichtel991fb3f2013-09-25 12:02:45 +02006113
6114/**
6115 * dev_set_allmulti - update allmulti count on a device
6116 * @dev: device
6117 * @inc: modifier
6118 *
6119 * Add or remove reception of all multicast frames to a device. While the
6120 * count in the device remains above zero the interface remains listening
6121 * to all interfaces. Once it hits zero the device reverts back to normal
6122 * filtering operation. A negative @inc value is used to drop the counter
6123 * when releasing a resource needing all multicasts.
6124 * Return 0 if successful or a negative errno code on error.
6125 */
6126
6127int dev_set_allmulti(struct net_device *dev, int inc)
6128{
6129 return __dev_set_allmulti(dev, inc, true);
6130}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07006131EXPORT_SYMBOL(dev_set_allmulti);
Patrick McHardy4417da62007-06-27 01:28:10 -07006132
6133/*
6134 * Upload unicast and multicast address lists to device and
6135 * configure RX filtering. When the device doesn't support unicast
Joe Perches53ccaae2007-12-20 14:02:06 -08006136 * filtering it is put in promiscuous mode while unicast addresses
Patrick McHardy4417da62007-06-27 01:28:10 -07006137 * are present.
6138 */
6139void __dev_set_rx_mode(struct net_device *dev)
6140{
Stephen Hemmingerd3147742008-11-19 21:32:24 -08006141 const struct net_device_ops *ops = dev->netdev_ops;
6142
Patrick McHardy4417da62007-06-27 01:28:10 -07006143 /* dev_open will call this function so the list will stay sane. */
6144 if (!(dev->flags&IFF_UP))
6145 return;
6146
6147 if (!netif_device_present(dev))
YOSHIFUJI Hideaki40b77c92007-07-19 10:43:23 +09006148 return;
Patrick McHardy4417da62007-06-27 01:28:10 -07006149
Jiri Pirko01789342011-08-16 06:29:00 +00006150 if (!(dev->priv_flags & IFF_UNICAST_FLT)) {
Patrick McHardy4417da62007-06-27 01:28:10 -07006151 /* Unicast addresses changes may only happen under the rtnl,
6152 * therefore calling __dev_set_promiscuity here is safe.
6153 */
Jiri Pirko32e7bfc2010-01-25 13:36:10 -08006154 if (!netdev_uc_empty(dev) && !dev->uc_promisc) {
Nicolas Dichtel991fb3f2013-09-25 12:02:45 +02006155 __dev_set_promiscuity(dev, 1, false);
Joe Perches2d348d12011-07-25 16:17:35 -07006156 dev->uc_promisc = true;
Jiri Pirko32e7bfc2010-01-25 13:36:10 -08006157 } else if (netdev_uc_empty(dev) && dev->uc_promisc) {
Nicolas Dichtel991fb3f2013-09-25 12:02:45 +02006158 __dev_set_promiscuity(dev, -1, false);
Joe Perches2d348d12011-07-25 16:17:35 -07006159 dev->uc_promisc = false;
Patrick McHardy4417da62007-06-27 01:28:10 -07006160 }
Patrick McHardy4417da62007-06-27 01:28:10 -07006161 }
Jiri Pirko01789342011-08-16 06:29:00 +00006162
6163 if (ops->ndo_set_rx_mode)
6164 ops->ndo_set_rx_mode(dev);
Patrick McHardy4417da62007-06-27 01:28:10 -07006165}
6166
6167void dev_set_rx_mode(struct net_device *dev)
6168{
David S. Millerb9e40852008-07-15 00:15:08 -07006169 netif_addr_lock_bh(dev);
Patrick McHardy4417da62007-06-27 01:28:10 -07006170 __dev_set_rx_mode(dev);
David S. Millerb9e40852008-07-15 00:15:08 -07006171 netif_addr_unlock_bh(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006172}
6173
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07006174/**
6175 * dev_get_flags - get flags reported to userspace
6176 * @dev: device
6177 *
6178 * Get the combination of flag bits exported through APIs to userspace.
6179 */
Eric Dumazet95c96172012-04-15 05:58:06 +00006180unsigned int dev_get_flags(const struct net_device *dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07006181{
Eric Dumazet95c96172012-04-15 05:58:06 +00006182 unsigned int flags;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006183
6184 flags = (dev->flags & ~(IFF_PROMISC |
6185 IFF_ALLMULTI |
Stefan Rompfb00055a2006-03-20 17:09:11 -08006186 IFF_RUNNING |
6187 IFF_LOWER_UP |
6188 IFF_DORMANT)) |
Linus Torvalds1da177e2005-04-16 15:20:36 -07006189 (dev->gflags & (IFF_PROMISC |
6190 IFF_ALLMULTI));
6191
Stefan Rompfb00055a2006-03-20 17:09:11 -08006192 if (netif_running(dev)) {
6193 if (netif_oper_up(dev))
6194 flags |= IFF_RUNNING;
6195 if (netif_carrier_ok(dev))
6196 flags |= IFF_LOWER_UP;
6197 if (netif_dormant(dev))
6198 flags |= IFF_DORMANT;
6199 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07006200
6201 return flags;
6202}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07006203EXPORT_SYMBOL(dev_get_flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006204
Patrick McHardybd380812010-02-26 06:34:53 +00006205int __dev_change_flags(struct net_device *dev, unsigned int flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -07006206{
Eric Dumazetb536db92011-11-30 21:42:26 +00006207 unsigned int old_flags = dev->flags;
Patrick McHardybd380812010-02-26 06:34:53 +00006208 int ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006209
Patrick McHardy24023452007-07-14 18:51:31 -07006210 ASSERT_RTNL();
6211
Linus Torvalds1da177e2005-04-16 15:20:36 -07006212 /*
6213 * Set the flags on our device.
6214 */
6215
6216 dev->flags = (flags & (IFF_DEBUG | IFF_NOTRAILERS | IFF_NOARP |
6217 IFF_DYNAMIC | IFF_MULTICAST | IFF_PORTSEL |
6218 IFF_AUTOMEDIA)) |
6219 (dev->flags & (IFF_UP | IFF_VOLATILE | IFF_PROMISC |
6220 IFF_ALLMULTI));
6221
6222 /*
6223 * Load in the correct multicast list now the flags have changed.
6224 */
6225
Patrick McHardyb6c40d62008-10-07 15:26:48 -07006226 if ((old_flags ^ flags) & IFF_MULTICAST)
6227 dev_change_rx_flags(dev, IFF_MULTICAST);
Patrick McHardy24023452007-07-14 18:51:31 -07006228
Patrick McHardy4417da62007-06-27 01:28:10 -07006229 dev_set_rx_mode(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006230
6231 /*
6232 * Have we downed the interface. We handle IFF_UP ourselves
6233 * according to user attempts to set it, rather than blindly
6234 * setting it.
6235 */
6236
6237 ret = 0;
Peter Pan(潘卫平)d215d102014-06-16 21:57:22 +08006238 if ((old_flags ^ flags) & IFF_UP)
Patrick McHardybd380812010-02-26 06:34:53 +00006239 ret = ((old_flags & IFF_UP) ? __dev_close : __dev_open)(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006240
Linus Torvalds1da177e2005-04-16 15:20:36 -07006241 if ((flags ^ dev->gflags) & IFF_PROMISC) {
Eric Dumazetd1b19df2009-09-03 01:29:39 -07006242 int inc = (flags & IFF_PROMISC) ? 1 : -1;
Nicolas Dichtel991fb3f2013-09-25 12:02:45 +02006243 unsigned int old_flags = dev->flags;
Eric Dumazetd1b19df2009-09-03 01:29:39 -07006244
Linus Torvalds1da177e2005-04-16 15:20:36 -07006245 dev->gflags ^= IFF_PROMISC;
Nicolas Dichtel991fb3f2013-09-25 12:02:45 +02006246
6247 if (__dev_set_promiscuity(dev, inc, false) >= 0)
6248 if (dev->flags != old_flags)
6249 dev_set_rx_mode(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006250 }
6251
6252 /* NOTE: order of synchronization of IFF_PROMISC and IFF_ALLMULTI
6253 is important. Some (broken) drivers set IFF_PROMISC, when
6254 IFF_ALLMULTI is requested not asking us and not reporting.
6255 */
6256 if ((flags ^ dev->gflags) & IFF_ALLMULTI) {
Eric Dumazetd1b19df2009-09-03 01:29:39 -07006257 int inc = (flags & IFF_ALLMULTI) ? 1 : -1;
6258
Linus Torvalds1da177e2005-04-16 15:20:36 -07006259 dev->gflags ^= IFF_ALLMULTI;
Nicolas Dichtel991fb3f2013-09-25 12:02:45 +02006260 __dev_set_allmulti(dev, inc, false);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006261 }
6262
Patrick McHardybd380812010-02-26 06:34:53 +00006263 return ret;
6264}
6265
Nicolas Dichtela528c212013-09-25 12:02:44 +02006266void __dev_notify_flags(struct net_device *dev, unsigned int old_flags,
6267 unsigned int gchanges)
Patrick McHardybd380812010-02-26 06:34:53 +00006268{
6269 unsigned int changes = dev->flags ^ old_flags;
6270
Nicolas Dichtela528c212013-09-25 12:02:44 +02006271 if (gchanges)
Alexei Starovoitov7f294052013-10-23 16:02:42 -07006272 rtmsg_ifinfo(RTM_NEWLINK, dev, gchanges, GFP_ATOMIC);
Nicolas Dichtela528c212013-09-25 12:02:44 +02006273
Patrick McHardybd380812010-02-26 06:34:53 +00006274 if (changes & IFF_UP) {
6275 if (dev->flags & IFF_UP)
6276 call_netdevice_notifiers(NETDEV_UP, dev);
6277 else
6278 call_netdevice_notifiers(NETDEV_DOWN, dev);
6279 }
6280
6281 if (dev->flags & IFF_UP &&
Jiri Pirkobe9efd32013-05-28 01:30:22 +00006282 (changes & ~(IFF_UP | IFF_PROMISC | IFF_ALLMULTI | IFF_VOLATILE))) {
6283 struct netdev_notifier_change_info change_info;
6284
6285 change_info.flags_changed = changes;
6286 call_netdevice_notifiers_info(NETDEV_CHANGE, dev,
6287 &change_info.info);
6288 }
Patrick McHardybd380812010-02-26 06:34:53 +00006289}
6290
6291/**
6292 * dev_change_flags - change device settings
6293 * @dev: device
6294 * @flags: device state flags
6295 *
6296 * Change settings on device based state flags. The flags are
6297 * in the userspace exported format.
6298 */
Eric Dumazetb536db92011-11-30 21:42:26 +00006299int dev_change_flags(struct net_device *dev, unsigned int flags)
Patrick McHardybd380812010-02-26 06:34:53 +00006300{
Eric Dumazetb536db92011-11-30 21:42:26 +00006301 int ret;
Nicolas Dichtel991fb3f2013-09-25 12:02:45 +02006302 unsigned int changes, old_flags = dev->flags, old_gflags = dev->gflags;
Patrick McHardybd380812010-02-26 06:34:53 +00006303
6304 ret = __dev_change_flags(dev, flags);
6305 if (ret < 0)
6306 return ret;
6307
Nicolas Dichtel991fb3f2013-09-25 12:02:45 +02006308 changes = (old_flags ^ dev->flags) | (old_gflags ^ dev->gflags);
Nicolas Dichtela528c212013-09-25 12:02:44 +02006309 __dev_notify_flags(dev, old_flags, changes);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006310 return ret;
6311}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07006312EXPORT_SYMBOL(dev_change_flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006313
Veaceslav Falico2315dc92014-01-10 16:56:25 +01006314static int __dev_set_mtu(struct net_device *dev, int new_mtu)
6315{
6316 const struct net_device_ops *ops = dev->netdev_ops;
6317
6318 if (ops->ndo_change_mtu)
6319 return ops->ndo_change_mtu(dev, new_mtu);
6320
6321 dev->mtu = new_mtu;
6322 return 0;
6323}
6324
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07006325/**
6326 * dev_set_mtu - Change maximum transfer unit
6327 * @dev: device
6328 * @new_mtu: new transfer unit
6329 *
6330 * Change the maximum transfer size of the network device.
6331 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07006332int dev_set_mtu(struct net_device *dev, int new_mtu)
6333{
Veaceslav Falico2315dc92014-01-10 16:56:25 +01006334 int err, orig_mtu;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006335
6336 if (new_mtu == dev->mtu)
6337 return 0;
6338
6339 /* MTU must be positive. */
6340 if (new_mtu < 0)
6341 return -EINVAL;
6342
6343 if (!netif_device_present(dev))
6344 return -ENODEV;
6345
Veaceslav Falico1d486bf2014-01-16 00:02:18 +01006346 err = call_netdevice_notifiers(NETDEV_PRECHANGEMTU, dev);
6347 err = notifier_to_errno(err);
6348 if (err)
6349 return err;
Stephen Hemmingerd3147742008-11-19 21:32:24 -08006350
Veaceslav Falico2315dc92014-01-10 16:56:25 +01006351 orig_mtu = dev->mtu;
6352 err = __dev_set_mtu(dev, new_mtu);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006353
Veaceslav Falico2315dc92014-01-10 16:56:25 +01006354 if (!err) {
6355 err = call_netdevice_notifiers(NETDEV_CHANGEMTU, dev);
6356 err = notifier_to_errno(err);
6357 if (err) {
6358 /* setting mtu back and notifying everyone again,
6359 * so that they have a chance to revert changes.
6360 */
6361 __dev_set_mtu(dev, orig_mtu);
6362 call_netdevice_notifiers(NETDEV_CHANGEMTU, dev);
6363 }
6364 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07006365 return err;
6366}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07006367EXPORT_SYMBOL(dev_set_mtu);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006368
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07006369/**
Vlad Dogarucbda10f2011-01-13 23:38:30 +00006370 * dev_set_group - Change group this device belongs to
6371 * @dev: device
6372 * @new_group: group this device should belong to
6373 */
6374void dev_set_group(struct net_device *dev, int new_group)
6375{
6376 dev->group = new_group;
6377}
6378EXPORT_SYMBOL(dev_set_group);
6379
6380/**
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07006381 * dev_set_mac_address - Change Media Access Control Address
6382 * @dev: device
6383 * @sa: new address
6384 *
6385 * Change the hardware (MAC) address of the device
6386 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07006387int dev_set_mac_address(struct net_device *dev, struct sockaddr *sa)
6388{
Stephen Hemmingerd3147742008-11-19 21:32:24 -08006389 const struct net_device_ops *ops = dev->netdev_ops;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006390 int err;
6391
Stephen Hemmingerd3147742008-11-19 21:32:24 -08006392 if (!ops->ndo_set_mac_address)
Linus Torvalds1da177e2005-04-16 15:20:36 -07006393 return -EOPNOTSUPP;
6394 if (sa->sa_family != dev->type)
6395 return -EINVAL;
6396 if (!netif_device_present(dev))
6397 return -ENODEV;
Stephen Hemmingerd3147742008-11-19 21:32:24 -08006398 err = ops->ndo_set_mac_address(dev, sa);
Jiri Pirkof6521512013-01-01 03:30:14 +00006399 if (err)
6400 return err;
Jiri Pirkofbdeca22013-01-01 03:30:16 +00006401 dev->addr_assign_type = NET_ADDR_SET;
Jiri Pirkof6521512013-01-01 03:30:14 +00006402 call_netdevice_notifiers(NETDEV_CHANGEADDR, dev);
Theodore Ts'o7bf23572012-07-04 21:23:25 -04006403 add_device_randomness(dev->dev_addr, dev->addr_len);
Jiri Pirkof6521512013-01-01 03:30:14 +00006404 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006405}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07006406EXPORT_SYMBOL(dev_set_mac_address);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006407
Jiri Pirko4bf84c32012-12-27 23:49:37 +00006408/**
6409 * dev_change_carrier - Change device carrier
6410 * @dev: device
Randy Dunlap691b3b72013-03-04 12:32:43 +00006411 * @new_carrier: new value
Jiri Pirko4bf84c32012-12-27 23:49:37 +00006412 *
6413 * Change device carrier
6414 */
6415int dev_change_carrier(struct net_device *dev, bool new_carrier)
6416{
6417 const struct net_device_ops *ops = dev->netdev_ops;
6418
6419 if (!ops->ndo_change_carrier)
6420 return -EOPNOTSUPP;
6421 if (!netif_device_present(dev))
6422 return -ENODEV;
6423 return ops->ndo_change_carrier(dev, new_carrier);
6424}
6425EXPORT_SYMBOL(dev_change_carrier);
6426
Linus Torvalds1da177e2005-04-16 15:20:36 -07006427/**
Jiri Pirko66b52b02013-07-29 18:16:49 +02006428 * dev_get_phys_port_id - Get device physical port ID
6429 * @dev: device
6430 * @ppid: port ID
6431 *
6432 * Get device physical port ID
6433 */
6434int dev_get_phys_port_id(struct net_device *dev,
Jiri Pirko02637fc2014-11-28 14:34:16 +01006435 struct netdev_phys_item_id *ppid)
Jiri Pirko66b52b02013-07-29 18:16:49 +02006436{
6437 const struct net_device_ops *ops = dev->netdev_ops;
6438
6439 if (!ops->ndo_get_phys_port_id)
6440 return -EOPNOTSUPP;
6441 return ops->ndo_get_phys_port_id(dev, ppid);
6442}
6443EXPORT_SYMBOL(dev_get_phys_port_id);
6444
6445/**
David Aherndb24a902015-03-17 20:23:15 -06006446 * dev_get_phys_port_name - Get device physical port name
6447 * @dev: device
6448 * @name: port name
Luis de Bethencourted49e652016-03-21 16:31:14 +00006449 * @len: limit of bytes to copy to name
David Aherndb24a902015-03-17 20:23:15 -06006450 *
6451 * Get device physical port name
6452 */
6453int dev_get_phys_port_name(struct net_device *dev,
6454 char *name, size_t len)
6455{
6456 const struct net_device_ops *ops = dev->netdev_ops;
6457
6458 if (!ops->ndo_get_phys_port_name)
6459 return -EOPNOTSUPP;
6460 return ops->ndo_get_phys_port_name(dev, name, len);
6461}
6462EXPORT_SYMBOL(dev_get_phys_port_name);
6463
6464/**
Anuradha Karuppiahd746d702015-07-14 13:43:19 -07006465 * dev_change_proto_down - update protocol port state information
6466 * @dev: device
6467 * @proto_down: new value
6468 *
6469 * This info can be used by switch drivers to set the phys state of the
6470 * port.
6471 */
6472int dev_change_proto_down(struct net_device *dev, bool proto_down)
6473{
6474 const struct net_device_ops *ops = dev->netdev_ops;
6475
6476 if (!ops->ndo_change_proto_down)
6477 return -EOPNOTSUPP;
6478 if (!netif_device_present(dev))
6479 return -ENODEV;
6480 return ops->ndo_change_proto_down(dev, proto_down);
6481}
6482EXPORT_SYMBOL(dev_change_proto_down);
6483
6484/**
Linus Torvalds1da177e2005-04-16 15:20:36 -07006485 * dev_new_index - allocate an ifindex
Randy Dunlapc4ea43c2007-10-12 21:17:49 -07006486 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -07006487 *
6488 * Returns a suitable unique value for a new device interface
6489 * number. The caller must hold the rtnl semaphore or the
6490 * dev_base_lock to be sure it remains unique.
6491 */
Eric W. Biederman881d9662007-09-17 11:56:21 -07006492static int dev_new_index(struct net *net)
Linus Torvalds1da177e2005-04-16 15:20:36 -07006493{
Pavel Emelyanovaa79e662012-08-08 21:53:19 +00006494 int ifindex = net->ifindex;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006495 for (;;) {
6496 if (++ifindex <= 0)
6497 ifindex = 1;
Eric W. Biederman881d9662007-09-17 11:56:21 -07006498 if (!__dev_get_by_index(net, ifindex))
Pavel Emelyanovaa79e662012-08-08 21:53:19 +00006499 return net->ifindex = ifindex;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006500 }
6501}
6502
Linus Torvalds1da177e2005-04-16 15:20:36 -07006503/* Delayed registration/unregisteration */
Denis Cheng3b5b34f2007-12-07 00:49:17 -08006504static LIST_HEAD(net_todo_list);
Cong Wang200b9162014-05-12 15:11:20 -07006505DECLARE_WAIT_QUEUE_HEAD(netdev_unregistering_wq);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006506
Stephen Hemminger6f05f622007-03-08 20:46:03 -08006507static void net_set_todo(struct net_device *dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07006508{
Linus Torvalds1da177e2005-04-16 15:20:36 -07006509 list_add_tail(&dev->todo_list, &net_todo_list);
Eric W. Biederman50624c92013-09-23 21:19:49 -07006510 dev_net(dev)->dev_unreg_count++;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006511}
6512
Eric Dumazet9b5e3832009-10-27 07:04:19 +00006513static void rollback_registered_many(struct list_head *head)
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07006514{
Krishna Kumare93737b2009-12-08 22:26:02 +00006515 struct net_device *dev, *tmp;
Eric W. Biederman5cde2822013-10-05 19:26:05 -07006516 LIST_HEAD(close_head);
Eric Dumazet9b5e3832009-10-27 07:04:19 +00006517
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07006518 BUG_ON(dev_boot_phase);
6519 ASSERT_RTNL();
6520
Krishna Kumare93737b2009-12-08 22:26:02 +00006521 list_for_each_entry_safe(dev, tmp, head, unreg_list) {
Eric Dumazet9b5e3832009-10-27 07:04:19 +00006522 /* Some devices call without registering
Krishna Kumare93737b2009-12-08 22:26:02 +00006523 * for initialization unwind. Remove those
6524 * devices and proceed with the remaining.
Eric Dumazet9b5e3832009-10-27 07:04:19 +00006525 */
6526 if (dev->reg_state == NETREG_UNINITIALIZED) {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00006527 pr_debug("unregister_netdevice: device %s/%p never was registered\n",
6528 dev->name, dev);
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07006529
Eric Dumazet9b5e3832009-10-27 07:04:19 +00006530 WARN_ON(1);
Krishna Kumare93737b2009-12-08 22:26:02 +00006531 list_del(&dev->unreg_list);
6532 continue;
Eric Dumazet9b5e3832009-10-27 07:04:19 +00006533 }
Eric Dumazet449f4542011-05-19 12:24:16 +00006534 dev->dismantle = true;
Eric Dumazet9b5e3832009-10-27 07:04:19 +00006535 BUG_ON(dev->reg_state != NETREG_REGISTERED);
Octavian Purdila44345722010-12-13 12:44:07 +00006536 }
Eric Dumazet9b5e3832009-10-27 07:04:19 +00006537
Octavian Purdila44345722010-12-13 12:44:07 +00006538 /* If device is running, close it first. */
Eric W. Biederman5cde2822013-10-05 19:26:05 -07006539 list_for_each_entry(dev, head, unreg_list)
6540 list_add_tail(&dev->close_list, &close_head);
David S. Miller99c4a262015-03-18 22:52:33 -04006541 dev_close_many(&close_head, true);
Eric Dumazet9b5e3832009-10-27 07:04:19 +00006542
Octavian Purdila44345722010-12-13 12:44:07 +00006543 list_for_each_entry(dev, head, unreg_list) {
Eric Dumazet9b5e3832009-10-27 07:04:19 +00006544 /* And unlink it from device chain. */
6545 unlist_netdevice(dev);
6546
6547 dev->reg_state = NETREG_UNREGISTERING;
Julian Anastasove9e4dd32015-07-09 09:59:09 +03006548 on_each_cpu(flush_backlog, dev, 1);
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07006549 }
6550
Eric Dumazet9b5e3832009-10-27 07:04:19 +00006551 synchronize_net();
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07006552
Eric Dumazet9b5e3832009-10-27 07:04:19 +00006553 list_for_each_entry(dev, head, unreg_list) {
Mahesh Bandewar395eea62014-12-03 13:46:24 -08006554 struct sk_buff *skb = NULL;
6555
Eric Dumazet9b5e3832009-10-27 07:04:19 +00006556 /* Shutdown queueing discipline. */
6557 dev_shutdown(dev);
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07006558
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07006559
Eric Dumazet9b5e3832009-10-27 07:04:19 +00006560 /* Notify protocols, that we are about to destroy
6561 this device. They should clean all the things.
6562 */
6563 call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
6564
Mahesh Bandewar395eea62014-12-03 13:46:24 -08006565 if (!dev->rtnl_link_ops ||
6566 dev->rtnl_link_state == RTNL_LINK_INITIALIZED)
6567 skb = rtmsg_ifinfo_build_skb(RTM_DELLINK, dev, ~0U,
6568 GFP_KERNEL);
6569
Eric Dumazet9b5e3832009-10-27 07:04:19 +00006570 /*
6571 * Flush the unicast and multicast chains
6572 */
Jiri Pirkoa748ee22010-04-01 21:22:09 +00006573 dev_uc_flush(dev);
Jiri Pirko22bedad32010-04-01 21:22:57 +00006574 dev_mc_flush(dev);
Eric Dumazet9b5e3832009-10-27 07:04:19 +00006575
6576 if (dev->netdev_ops->ndo_uninit)
6577 dev->netdev_ops->ndo_uninit(dev);
6578
Mahesh Bandewar395eea62014-12-03 13:46:24 -08006579 if (skb)
6580 rtmsg_ifinfo_send(skb, dev, GFP_KERNEL);
Roopa Prabhu56bfa7e2014-05-01 11:40:30 -07006581
Jiri Pirko9ff162a2013-01-03 22:48:49 +00006582 /* Notifier chain MUST detach us all upper devices. */
6583 WARN_ON(netdev_has_any_upper_dev(dev));
Eric Dumazet9b5e3832009-10-27 07:04:19 +00006584
6585 /* Remove entries from kobject tree */
6586 netdev_unregister_kobject(dev);
Alexander Duyck024e9672013-01-10 08:57:46 +00006587#ifdef CONFIG_XPS
6588 /* Remove XPS queueing entries */
6589 netif_reset_xps_queues_gt(dev, 0);
6590#endif
Eric Dumazet9b5e3832009-10-27 07:04:19 +00006591 }
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07006592
Eric W. Biederman850a5452011-10-13 22:25:23 +00006593 synchronize_net();
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07006594
Eric W. Biedermana5ee1552009-11-29 15:45:58 +00006595 list_for_each_entry(dev, head, unreg_list)
Eric Dumazet9b5e3832009-10-27 07:04:19 +00006596 dev_put(dev);
6597}
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07006598
Eric Dumazet9b5e3832009-10-27 07:04:19 +00006599static void rollback_registered(struct net_device *dev)
6600{
6601 LIST_HEAD(single);
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07006602
Eric Dumazet9b5e3832009-10-27 07:04:19 +00006603 list_add(&dev->unreg_list, &single);
6604 rollback_registered_many(&single);
Eric Dumazetceaaec92011-02-17 22:59:19 +00006605 list_del(&single);
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07006606}
6607
Jarod Wilsonfd867d52015-11-02 21:55:59 -05006608static netdev_features_t netdev_sync_upper_features(struct net_device *lower,
6609 struct net_device *upper, netdev_features_t features)
6610{
6611 netdev_features_t upper_disables = NETIF_F_UPPER_DISABLES;
6612 netdev_features_t feature;
Jarod Wilson5ba3f7d2015-11-03 10:15:59 -05006613 int feature_bit;
Jarod Wilsonfd867d52015-11-02 21:55:59 -05006614
Jarod Wilson5ba3f7d2015-11-03 10:15:59 -05006615 for_each_netdev_feature(&upper_disables, feature_bit) {
6616 feature = __NETIF_F_BIT(feature_bit);
Jarod Wilsonfd867d52015-11-02 21:55:59 -05006617 if (!(upper->wanted_features & feature)
6618 && (features & feature)) {
6619 netdev_dbg(lower, "Dropping feature %pNF, upper dev %s has it off.\n",
6620 &feature, upper->name);
6621 features &= ~feature;
6622 }
6623 }
6624
6625 return features;
6626}
6627
6628static void netdev_sync_lower_features(struct net_device *upper,
6629 struct net_device *lower, netdev_features_t features)
6630{
6631 netdev_features_t upper_disables = NETIF_F_UPPER_DISABLES;
6632 netdev_features_t feature;
Jarod Wilson5ba3f7d2015-11-03 10:15:59 -05006633 int feature_bit;
Jarod Wilsonfd867d52015-11-02 21:55:59 -05006634
Jarod Wilson5ba3f7d2015-11-03 10:15:59 -05006635 for_each_netdev_feature(&upper_disables, feature_bit) {
6636 feature = __NETIF_F_BIT(feature_bit);
Jarod Wilsonfd867d52015-11-02 21:55:59 -05006637 if (!(features & feature) && (lower->features & feature)) {
6638 netdev_dbg(upper, "Disabling feature %pNF on lower dev %s.\n",
6639 &feature, lower->name);
6640 lower->wanted_features &= ~feature;
6641 netdev_update_features(lower);
6642
6643 if (unlikely(lower->features & feature))
6644 netdev_WARN(upper, "failed to disable %pNF on %s!\n",
6645 &feature, lower->name);
6646 }
6647 }
6648}
6649
Michał Mirosławc8f44af2011-11-15 15:29:55 +00006650static netdev_features_t netdev_fix_features(struct net_device *dev,
6651 netdev_features_t features)
Herbert Xub63365a2008-10-23 01:11:29 -07006652{
Michał Mirosław57422dc2011-01-22 12:14:12 +00006653 /* Fix illegal checksum combinations */
6654 if ((features & NETIF_F_HW_CSUM) &&
6655 (features & (NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM))) {
Michał Mirosław6f404e42011-05-16 15:14:21 -04006656 netdev_warn(dev, "mixed HW and IP checksum settings.\n");
Michał Mirosław57422dc2011-01-22 12:14:12 +00006657 features &= ~(NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM);
6658 }
6659
Herbert Xub63365a2008-10-23 01:11:29 -07006660 /* TSO requires that SG is present as well. */
Ben Hutchingsea2d3682011-04-12 14:38:37 +00006661 if ((features & NETIF_F_ALL_TSO) && !(features & NETIF_F_SG)) {
Michał Mirosław6f404e42011-05-16 15:14:21 -04006662 netdev_dbg(dev, "Dropping TSO features since no SG feature.\n");
Ben Hutchingsea2d3682011-04-12 14:38:37 +00006663 features &= ~NETIF_F_ALL_TSO;
Herbert Xub63365a2008-10-23 01:11:29 -07006664 }
6665
Pravin B Shelarec5f0612013-03-07 09:28:01 +00006666 if ((features & NETIF_F_TSO) && !(features & NETIF_F_HW_CSUM) &&
6667 !(features & NETIF_F_IP_CSUM)) {
6668 netdev_dbg(dev, "Dropping TSO features since no CSUM feature.\n");
6669 features &= ~NETIF_F_TSO;
6670 features &= ~NETIF_F_TSO_ECN;
6671 }
6672
6673 if ((features & NETIF_F_TSO6) && !(features & NETIF_F_HW_CSUM) &&
6674 !(features & NETIF_F_IPV6_CSUM)) {
6675 netdev_dbg(dev, "Dropping TSO6 features since no CSUM feature.\n");
6676 features &= ~NETIF_F_TSO6;
6677 }
6678
Ben Hutchings31d8b9e2011-04-12 14:47:15 +00006679 /* TSO ECN requires that TSO is present as well. */
6680 if ((features & NETIF_F_ALL_TSO) == NETIF_F_TSO_ECN)
6681 features &= ~NETIF_F_TSO_ECN;
6682
Michał Mirosław212b5732011-02-15 16:59:16 +00006683 /* Software GSO depends on SG. */
6684 if ((features & NETIF_F_GSO) && !(features & NETIF_F_SG)) {
Michał Mirosław6f404e42011-05-16 15:14:21 -04006685 netdev_dbg(dev, "Dropping NETIF_F_GSO since no SG feature.\n");
Michał Mirosław212b5732011-02-15 16:59:16 +00006686 features &= ~NETIF_F_GSO;
6687 }
6688
Michał Mirosławacd11302011-01-24 15:45:15 -08006689 /* UFO needs SG and checksumming */
Herbert Xub63365a2008-10-23 01:11:29 -07006690 if (features & NETIF_F_UFO) {
Michał Mirosław79032642010-11-30 06:38:00 +00006691 /* maybe split UFO into V4 and V6? */
Tom Herbertc8cd0982015-12-14 11:19:44 -08006692 if (!(features & NETIF_F_HW_CSUM) &&
6693 ((features & (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM)) !=
6694 (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM))) {
Michał Mirosław6f404e42011-05-16 15:14:21 -04006695 netdev_dbg(dev,
Michał Mirosławacd11302011-01-24 15:45:15 -08006696 "Dropping NETIF_F_UFO since no checksum offload features.\n");
Herbert Xub63365a2008-10-23 01:11:29 -07006697 features &= ~NETIF_F_UFO;
6698 }
6699
6700 if (!(features & NETIF_F_SG)) {
Michał Mirosław6f404e42011-05-16 15:14:21 -04006701 netdev_dbg(dev,
Michał Mirosławacd11302011-01-24 15:45:15 -08006702 "Dropping NETIF_F_UFO since no NETIF_F_SG feature.\n");
Herbert Xub63365a2008-10-23 01:11:29 -07006703 features &= ~NETIF_F_UFO;
6704 }
6705 }
6706
Jiri Pirkod0290212014-04-02 23:09:31 +02006707#ifdef CONFIG_NET_RX_BUSY_POLL
6708 if (dev->netdev_ops->ndo_busy_poll)
6709 features |= NETIF_F_BUSY_POLL;
6710 else
6711#endif
6712 features &= ~NETIF_F_BUSY_POLL;
6713
Herbert Xub63365a2008-10-23 01:11:29 -07006714 return features;
6715}
Herbert Xub63365a2008-10-23 01:11:29 -07006716
Michał Mirosław6cb6a272011-04-02 22:48:47 -07006717int __netdev_update_features(struct net_device *dev)
Michał Mirosław5455c692011-02-15 16:59:17 +00006718{
Jarod Wilsonfd867d52015-11-02 21:55:59 -05006719 struct net_device *upper, *lower;
Michał Mirosławc8f44af2011-11-15 15:29:55 +00006720 netdev_features_t features;
Jarod Wilsonfd867d52015-11-02 21:55:59 -05006721 struct list_head *iter;
Jarod Wilsone7868a82015-11-03 23:09:32 -05006722 int err = -1;
Michał Mirosław5455c692011-02-15 16:59:17 +00006723
Michał Mirosław87267482011-04-12 09:56:38 +00006724 ASSERT_RTNL();
6725
Michał Mirosław5455c692011-02-15 16:59:17 +00006726 features = netdev_get_wanted_features(dev);
6727
6728 if (dev->netdev_ops->ndo_fix_features)
6729 features = dev->netdev_ops->ndo_fix_features(dev, features);
6730
6731 /* driver might be less strict about feature dependencies */
6732 features = netdev_fix_features(dev, features);
6733
Jarod Wilsonfd867d52015-11-02 21:55:59 -05006734 /* some features can't be enabled if they're off an an upper device */
6735 netdev_for_each_upper_dev_rcu(dev, upper, iter)
6736 features = netdev_sync_upper_features(dev, upper, features);
6737
Michał Mirosław5455c692011-02-15 16:59:17 +00006738 if (dev->features == features)
Jarod Wilsone7868a82015-11-03 23:09:32 -05006739 goto sync_lower;
Michał Mirosław5455c692011-02-15 16:59:17 +00006740
Michał Mirosławc8f44af2011-11-15 15:29:55 +00006741 netdev_dbg(dev, "Features changed: %pNF -> %pNF\n",
6742 &dev->features, &features);
Michał Mirosław5455c692011-02-15 16:59:17 +00006743
6744 if (dev->netdev_ops->ndo_set_features)
6745 err = dev->netdev_ops->ndo_set_features(dev, features);
Nikolay Aleksandrov5f8dc332015-11-13 14:54:01 +01006746 else
6747 err = 0;
Michał Mirosław5455c692011-02-15 16:59:17 +00006748
Michał Mirosław6cb6a272011-04-02 22:48:47 -07006749 if (unlikely(err < 0)) {
Michał Mirosław5455c692011-02-15 16:59:17 +00006750 netdev_err(dev,
Michał Mirosławc8f44af2011-11-15 15:29:55 +00006751 "set_features() failed (%d); wanted %pNF, left %pNF\n",
6752 err, &features, &dev->features);
Nikolay Aleksandrov17b85d22015-11-17 15:49:06 +01006753 /* return non-0 since some features might have changed and
6754 * it's better to fire a spurious notification than miss it
6755 */
6756 return -1;
Michał Mirosław6cb6a272011-04-02 22:48:47 -07006757 }
6758
Jarod Wilsone7868a82015-11-03 23:09:32 -05006759sync_lower:
Jarod Wilsonfd867d52015-11-02 21:55:59 -05006760 /* some features must be disabled on lower devices when disabled
6761 * on an upper device (think: bonding master or bridge)
6762 */
6763 netdev_for_each_lower_dev(dev, lower, iter)
6764 netdev_sync_lower_features(dev, lower, features);
6765
Michał Mirosław6cb6a272011-04-02 22:48:47 -07006766 if (!err)
6767 dev->features = features;
6768
Jarod Wilsone7868a82015-11-03 23:09:32 -05006769 return err < 0 ? 0 : 1;
Michał Mirosław6cb6a272011-04-02 22:48:47 -07006770}
6771
Michał Mirosławafe12cc2011-05-07 03:22:17 +00006772/**
6773 * netdev_update_features - recalculate device features
6774 * @dev: the device to check
6775 *
6776 * Recalculate dev->features set and send notifications if it
6777 * has changed. Should be called after driver or hardware dependent
6778 * conditions might have changed that influence the features.
6779 */
Michał Mirosław6cb6a272011-04-02 22:48:47 -07006780void netdev_update_features(struct net_device *dev)
6781{
6782 if (__netdev_update_features(dev))
6783 netdev_features_change(dev);
Michał Mirosław5455c692011-02-15 16:59:17 +00006784}
6785EXPORT_SYMBOL(netdev_update_features);
6786
Linus Torvalds1da177e2005-04-16 15:20:36 -07006787/**
Michał Mirosławafe12cc2011-05-07 03:22:17 +00006788 * netdev_change_features - recalculate device features
6789 * @dev: the device to check
6790 *
6791 * Recalculate dev->features set and send notifications even
6792 * if they have not changed. Should be called instead of
6793 * netdev_update_features() if also dev->vlan_features might
6794 * have changed to allow the changes to be propagated to stacked
6795 * VLAN devices.
6796 */
6797void netdev_change_features(struct net_device *dev)
6798{
6799 __netdev_update_features(dev);
6800 netdev_features_change(dev);
6801}
6802EXPORT_SYMBOL(netdev_change_features);
6803
6804/**
Patrick Mullaneyfc4a7482009-12-03 15:59:22 -08006805 * netif_stacked_transfer_operstate - transfer operstate
6806 * @rootdev: the root or lower level device to transfer state from
6807 * @dev: the device to transfer operstate to
6808 *
6809 * Transfer operational state from root to device. This is normally
6810 * called when a stacking relationship exists between the root
6811 * device and the device(a leaf device).
6812 */
6813void netif_stacked_transfer_operstate(const struct net_device *rootdev,
6814 struct net_device *dev)
6815{
6816 if (rootdev->operstate == IF_OPER_DORMANT)
6817 netif_dormant_on(dev);
6818 else
6819 netif_dormant_off(dev);
6820
6821 if (netif_carrier_ok(rootdev)) {
6822 if (!netif_carrier_ok(dev))
6823 netif_carrier_on(dev);
6824 } else {
6825 if (netif_carrier_ok(dev))
6826 netif_carrier_off(dev);
6827 }
6828}
6829EXPORT_SYMBOL(netif_stacked_transfer_operstate);
6830
Michael Daltona953be52014-01-16 22:23:28 -08006831#ifdef CONFIG_SYSFS
Eric Dumazet1b4bf462010-09-23 17:26:35 +00006832static int netif_alloc_rx_queues(struct net_device *dev)
6833{
Eric Dumazet1b4bf462010-09-23 17:26:35 +00006834 unsigned int i, count = dev->num_rx_queues;
Tom Herbertbd25fa72010-10-18 18:00:16 +00006835 struct netdev_rx_queue *rx;
Pankaj Gupta10595902015-01-12 11:41:28 +05306836 size_t sz = count * sizeof(*rx);
Eric Dumazet1b4bf462010-09-23 17:26:35 +00006837
Tom Herbertbd25fa72010-10-18 18:00:16 +00006838 BUG_ON(count < 1);
Eric Dumazet1b4bf462010-09-23 17:26:35 +00006839
Pankaj Gupta10595902015-01-12 11:41:28 +05306840 rx = kzalloc(sz, GFP_KERNEL | __GFP_NOWARN | __GFP_REPEAT);
6841 if (!rx) {
6842 rx = vzalloc(sz);
6843 if (!rx)
6844 return -ENOMEM;
6845 }
Tom Herbertbd25fa72010-10-18 18:00:16 +00006846 dev->_rx = rx;
6847
Tom Herbertbd25fa72010-10-18 18:00:16 +00006848 for (i = 0; i < count; i++)
Tom Herbertfe822242010-11-09 10:47:38 +00006849 rx[i].dev = dev;
Eric Dumazet1b4bf462010-09-23 17:26:35 +00006850 return 0;
6851}
Tom Herbertbf264142010-11-26 08:36:09 +00006852#endif
Eric Dumazet1b4bf462010-09-23 17:26:35 +00006853
Changli Gaoaa942102010-12-04 02:31:41 +00006854static void netdev_init_one_queue(struct net_device *dev,
6855 struct netdev_queue *queue, void *_unused)
6856{
6857 /* Initialize queue lock */
6858 spin_lock_init(&queue->_xmit_lock);
6859 netdev_set_xmit_lockdep_class(&queue->_xmit_lock, dev->type);
6860 queue->xmit_lock_owner = -1;
Changli Gaob236da62010-12-14 03:09:15 +00006861 netdev_queue_numa_node_write(queue, NUMA_NO_NODE);
Changli Gaoaa942102010-12-04 02:31:41 +00006862 queue->dev = dev;
Tom Herbert114cf582011-11-28 16:33:09 +00006863#ifdef CONFIG_BQL
6864 dql_init(&queue->dql, HZ);
6865#endif
Changli Gaoaa942102010-12-04 02:31:41 +00006866}
6867
Eric Dumazet60877a32013-06-20 01:15:51 -07006868static void netif_free_tx_queues(struct net_device *dev)
6869{
WANG Cong4cb28972014-06-02 15:55:22 -07006870 kvfree(dev->_tx);
Eric Dumazet60877a32013-06-20 01:15:51 -07006871}
6872
Tom Herberte6484932010-10-18 18:04:39 +00006873static int netif_alloc_netdev_queues(struct net_device *dev)
6874{
6875 unsigned int count = dev->num_tx_queues;
6876 struct netdev_queue *tx;
Eric Dumazet60877a32013-06-20 01:15:51 -07006877 size_t sz = count * sizeof(*tx);
Tom Herberte6484932010-10-18 18:04:39 +00006878
Eric Dumazetd3397272015-07-06 17:13:26 +02006879 if (count < 1 || count > 0xffff)
6880 return -EINVAL;
Tom Herberte6484932010-10-18 18:04:39 +00006881
Eric Dumazet60877a32013-06-20 01:15:51 -07006882 tx = kzalloc(sz, GFP_KERNEL | __GFP_NOWARN | __GFP_REPEAT);
6883 if (!tx) {
6884 tx = vzalloc(sz);
6885 if (!tx)
6886 return -ENOMEM;
6887 }
Tom Herberte6484932010-10-18 18:04:39 +00006888 dev->_tx = tx;
Tom Herbert1d24eb42010-11-21 13:17:27 +00006889
Tom Herberte6484932010-10-18 18:04:39 +00006890 netdev_for_each_tx_queue(dev, netdev_init_one_queue, NULL);
6891 spin_lock_init(&dev->tx_global_lock);
Changli Gaoaa942102010-12-04 02:31:41 +00006892
6893 return 0;
Tom Herberte6484932010-10-18 18:04:39 +00006894}
6895
Denys Vlasenkoa2029242015-05-11 21:17:53 +02006896void netif_tx_stop_all_queues(struct net_device *dev)
6897{
6898 unsigned int i;
6899
6900 for (i = 0; i < dev->num_tx_queues; i++) {
6901 struct netdev_queue *txq = netdev_get_tx_queue(dev, i);
6902 netif_tx_stop_queue(txq);
6903 }
6904}
6905EXPORT_SYMBOL(netif_tx_stop_all_queues);
6906
Patrick Mullaneyfc4a7482009-12-03 15:59:22 -08006907/**
Linus Torvalds1da177e2005-04-16 15:20:36 -07006908 * register_netdevice - register a network device
6909 * @dev: device to register
6910 *
6911 * Take a completed network device structure and add it to the kernel
6912 * interfaces. A %NETDEV_REGISTER message is sent to the netdev notifier
6913 * chain. 0 is returned on success. A negative errno code is returned
6914 * on a failure to set up the device, or if the name is a duplicate.
6915 *
6916 * Callers must hold the rtnl semaphore. You may want
6917 * register_netdev() instead of this.
6918 *
6919 * BUGS:
6920 * The locking appears insufficient to guarantee two parallel registers
6921 * will not get the same name.
6922 */
6923
6924int register_netdevice(struct net_device *dev)
6925{
Linus Torvalds1da177e2005-04-16 15:20:36 -07006926 int ret;
Stephen Hemmingerd3147742008-11-19 21:32:24 -08006927 struct net *net = dev_net(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006928
6929 BUG_ON(dev_boot_phase);
6930 ASSERT_RTNL();
6931
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07006932 might_sleep();
6933
Linus Torvalds1da177e2005-04-16 15:20:36 -07006934 /* When net_device's are persistent, this will be fatal. */
6935 BUG_ON(dev->reg_state != NETREG_UNINITIALIZED);
Stephen Hemmingerd3147742008-11-19 21:32:24 -08006936 BUG_ON(!net);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006937
David S. Millerf1f28aa2008-07-15 00:08:33 -07006938 spin_lock_init(&dev->addr_list_lock);
David S. Millercf508b12008-07-22 14:16:42 -07006939 netdev_set_addr_lockdep_class(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006940
Gao feng828de4f2012-09-13 20:58:27 +00006941 ret = dev_get_valid_name(net, dev, dev->name);
Peter Pan(潘卫平)0696c3a2011-05-12 15:46:56 +00006942 if (ret < 0)
6943 goto out;
6944
Linus Torvalds1da177e2005-04-16 15:20:36 -07006945 /* Init, if this function is available */
Stephen Hemmingerd3147742008-11-19 21:32:24 -08006946 if (dev->netdev_ops->ndo_init) {
6947 ret = dev->netdev_ops->ndo_init(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006948 if (ret) {
6949 if (ret > 0)
6950 ret = -EIO;
Adrian Bunk90833aa2006-11-13 16:02:22 -08006951 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006952 }
6953 }
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09006954
Patrick McHardyf6469682013-04-19 02:04:27 +00006955 if (((dev->hw_features | dev->features) &
6956 NETIF_F_HW_VLAN_CTAG_FILTER) &&
Michał Mirosławd2ed2732013-01-29 15:14:16 +00006957 (!dev->netdev_ops->ndo_vlan_rx_add_vid ||
6958 !dev->netdev_ops->ndo_vlan_rx_kill_vid)) {
6959 netdev_WARN(dev, "Buggy VLAN acceleration in driver!\n");
6960 ret = -EINVAL;
6961 goto err_uninit;
6962 }
6963
Pavel Emelyanov9c7dafb2012-08-08 21:52:46 +00006964 ret = -EBUSY;
6965 if (!dev->ifindex)
6966 dev->ifindex = dev_new_index(net);
6967 else if (__dev_get_by_index(net, dev->ifindex))
6968 goto err_uninit;
6969
Michał Mirosław5455c692011-02-15 16:59:17 +00006970 /* Transfer changeable features to wanted_features and enable
6971 * software offloads (GSO and GRO).
6972 */
6973 dev->hw_features |= NETIF_F_SOFT_FEATURES;
Michał Mirosław14d12322011-02-22 16:52:28 +00006974 dev->features |= NETIF_F_SOFT_FEATURES;
6975 dev->wanted_features = dev->features & dev->hw_features;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006976
Michał Mirosław34324dc2011-11-15 15:29:55 +00006977 if (!(dev->flags & IFF_LOOPBACK)) {
6978 dev->hw_features |= NETIF_F_NOCACHE_COPY;
Tom Herbertc6e1a0d2011-04-04 22:30:30 -07006979 }
6980
Michał Mirosław1180e7d2011-07-14 14:41:11 -07006981 /* Make NETIF_F_HIGHDMA inheritable to VLAN devices.
Brandon Philips16c3ea72010-09-15 09:24:24 +00006982 */
Michał Mirosław1180e7d2011-07-14 14:41:11 -07006983 dev->vlan_features |= NETIF_F_HIGHDMA;
Brandon Philips16c3ea72010-09-15 09:24:24 +00006984
Pravin B Shelaree579672013-03-07 09:28:08 +00006985 /* Make NETIF_F_SG inheritable to tunnel devices.
6986 */
6987 dev->hw_enc_features |= NETIF_F_SG;
6988
Simon Horman0d89d202013-05-23 21:02:52 +00006989 /* Make NETIF_F_SG inheritable to MPLS.
6990 */
6991 dev->mpls_features |= NETIF_F_SG;
6992
Johannes Berg7ffbe3f2009-10-02 05:15:27 +00006993 ret = call_netdevice_notifiers(NETDEV_POST_INIT, dev);
6994 ret = notifier_to_errno(ret);
6995 if (ret)
6996 goto err_uninit;
6997
Eric W. Biederman8b41d182007-09-26 22:02:53 -07006998 ret = netdev_register_kobject(dev);
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07006999 if (ret)
Herbert Xu7ce1b0e2007-07-30 16:29:40 -07007000 goto err_uninit;
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07007001 dev->reg_state = NETREG_REGISTERED;
7002
Michał Mirosław6cb6a272011-04-02 22:48:47 -07007003 __netdev_update_features(dev);
Michał Mirosław8e9b59b2011-02-22 16:52:28 +00007004
Linus Torvalds1da177e2005-04-16 15:20:36 -07007005 /*
7006 * Default initial state at registry is that the
7007 * device is present.
7008 */
7009
7010 set_bit(__LINK_STATE_PRESENT, &dev->state);
7011
Ben Hutchings8f4cccb2012-08-20 22:16:51 +01007012 linkwatch_init_dev(dev);
7013
Linus Torvalds1da177e2005-04-16 15:20:36 -07007014 dev_init_scheduler(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07007015 dev_hold(dev);
Eric W. Biedermance286d32007-09-12 13:53:49 +02007016 list_netdevice(dev);
Theodore Ts'o7bf23572012-07-04 21:23:25 -04007017 add_device_randomness(dev->dev_addr, dev->addr_len);
Linus Torvalds1da177e2005-04-16 15:20:36 -07007018
Jiri Pirko948b3372013-01-08 01:38:25 +00007019 /* If the device has permanent device address, driver should
7020 * set dev_addr and also addr_assign_type should be set to
7021 * NET_ADDR_PERM (default value).
7022 */
7023 if (dev->addr_assign_type == NET_ADDR_PERM)
7024 memcpy(dev->perm_addr, dev->dev_addr, dev->addr_len);
7025
Linus Torvalds1da177e2005-04-16 15:20:36 -07007026 /* Notify protocols, that a new device appeared. */
Pavel Emelyanov056925a2007-09-16 15:42:43 -07007027 ret = call_netdevice_notifiers(NETDEV_REGISTER, dev);
Herbert Xufcc5a032007-07-30 17:03:38 -07007028 ret = notifier_to_errno(ret);
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07007029 if (ret) {
7030 rollback_registered(dev);
7031 dev->reg_state = NETREG_UNREGISTERED;
7032 }
Eric W. Biedermand90a9092009-12-12 22:11:15 +00007033 /*
7034 * Prevent userspace races by waiting until the network
7035 * device is fully setup before sending notifications.
7036 */
Patrick McHardya2835762010-02-26 06:34:51 +00007037 if (!dev->rtnl_link_ops ||
7038 dev->rtnl_link_state == RTNL_LINK_INITIALIZED)
Alexei Starovoitov7f294052013-10-23 16:02:42 -07007039 rtmsg_ifinfo(RTM_NEWLINK, dev, ~0U, GFP_KERNEL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07007040
7041out:
7042 return ret;
Herbert Xu7ce1b0e2007-07-30 16:29:40 -07007043
7044err_uninit:
Stephen Hemmingerd3147742008-11-19 21:32:24 -08007045 if (dev->netdev_ops->ndo_uninit)
7046 dev->netdev_ops->ndo_uninit(dev);
Herbert Xu7ce1b0e2007-07-30 16:29:40 -07007047 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07007048}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07007049EXPORT_SYMBOL(register_netdevice);
Linus Torvalds1da177e2005-04-16 15:20:36 -07007050
7051/**
Benjamin Herrenschmidt937f1ba2009-01-14 21:05:05 -08007052 * init_dummy_netdev - init a dummy network device for NAPI
7053 * @dev: device to init
7054 *
7055 * This takes a network device structure and initialize the minimum
7056 * amount of fields so it can be used to schedule NAPI polls without
7057 * registering a full blown interface. This is to be used by drivers
7058 * that need to tie several hardware interfaces to a single NAPI
7059 * poll scheduler due to HW limitations.
7060 */
7061int init_dummy_netdev(struct net_device *dev)
7062{
7063 /* Clear everything. Note we don't initialize spinlocks
7064 * are they aren't supposed to be taken by any of the
7065 * NAPI code and this dummy netdev is supposed to be
7066 * only ever used for NAPI polls
7067 */
7068 memset(dev, 0, sizeof(struct net_device));
7069
7070 /* make sure we BUG if trying to hit standard
7071 * register/unregister code path
7072 */
7073 dev->reg_state = NETREG_DUMMY;
7074
Benjamin Herrenschmidt937f1ba2009-01-14 21:05:05 -08007075 /* NAPI wants this */
7076 INIT_LIST_HEAD(&dev->napi_list);
7077
7078 /* a dummy interface is started by default */
7079 set_bit(__LINK_STATE_PRESENT, &dev->state);
7080 set_bit(__LINK_STATE_START, &dev->state);
7081
Eric Dumazet29b44332010-10-11 10:22:12 +00007082 /* Note : We dont allocate pcpu_refcnt for dummy devices,
7083 * because users of this 'device' dont need to change
7084 * its refcount.
7085 */
7086
Benjamin Herrenschmidt937f1ba2009-01-14 21:05:05 -08007087 return 0;
7088}
7089EXPORT_SYMBOL_GPL(init_dummy_netdev);
7090
7091
7092/**
Linus Torvalds1da177e2005-04-16 15:20:36 -07007093 * register_netdev - register a network device
7094 * @dev: device to register
7095 *
7096 * Take a completed network device structure and add it to the kernel
7097 * interfaces. A %NETDEV_REGISTER message is sent to the netdev notifier
7098 * chain. 0 is returned on success. A negative errno code is returned
7099 * on a failure to set up the device, or if the name is a duplicate.
7100 *
Borislav Petkov38b4da32007-04-20 22:14:10 -07007101 * This is a wrapper around register_netdevice that takes the rtnl semaphore
Linus Torvalds1da177e2005-04-16 15:20:36 -07007102 * and expands the device name if you passed a format string to
7103 * alloc_netdev.
7104 */
7105int register_netdev(struct net_device *dev)
7106{
7107 int err;
7108
7109 rtnl_lock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07007110 err = register_netdevice(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07007111 rtnl_unlock();
7112 return err;
7113}
7114EXPORT_SYMBOL(register_netdev);
7115
Eric Dumazet29b44332010-10-11 10:22:12 +00007116int netdev_refcnt_read(const struct net_device *dev)
7117{
7118 int i, refcnt = 0;
7119
7120 for_each_possible_cpu(i)
7121 refcnt += *per_cpu_ptr(dev->pcpu_refcnt, i);
7122 return refcnt;
7123}
7124EXPORT_SYMBOL(netdev_refcnt_read);
7125
Ben Hutchings2c530402012-07-10 10:55:09 +00007126/**
Linus Torvalds1da177e2005-04-16 15:20:36 -07007127 * netdev_wait_allrefs - wait until all references are gone.
Randy Dunlap3de7a372012-08-18 14:36:44 +00007128 * @dev: target net_device
Linus Torvalds1da177e2005-04-16 15:20:36 -07007129 *
7130 * This is called when unregistering network devices.
7131 *
7132 * Any protocol or device that holds a reference should register
7133 * for netdevice notification, and cleanup and put back the
7134 * reference if they receive an UNREGISTER event.
7135 * We can get stuck here if buggy protocols don't correctly
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09007136 * call dev_put.
Linus Torvalds1da177e2005-04-16 15:20:36 -07007137 */
7138static void netdev_wait_allrefs(struct net_device *dev)
7139{
7140 unsigned long rebroadcast_time, warning_time;
Eric Dumazet29b44332010-10-11 10:22:12 +00007141 int refcnt;
Linus Torvalds1da177e2005-04-16 15:20:36 -07007142
Eric Dumazete014deb2009-11-17 05:59:21 +00007143 linkwatch_forget_dev(dev);
7144
Linus Torvalds1da177e2005-04-16 15:20:36 -07007145 rebroadcast_time = warning_time = jiffies;
Eric Dumazet29b44332010-10-11 10:22:12 +00007146 refcnt = netdev_refcnt_read(dev);
7147
7148 while (refcnt != 0) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07007149 if (time_after(jiffies, rebroadcast_time + 1 * HZ)) {
Stephen Hemminger6756ae42006-03-20 22:23:58 -08007150 rtnl_lock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07007151
7152 /* Rebroadcast unregister notification */
Pavel Emelyanov056925a2007-09-16 15:42:43 -07007153 call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07007154
Eric Dumazet748e2d92012-08-22 21:50:59 +00007155 __rtnl_unlock();
Eric Dumazet0115e8e2012-08-22 17:19:46 +00007156 rcu_barrier();
Eric Dumazet748e2d92012-08-22 21:50:59 +00007157 rtnl_lock();
7158
Eric Dumazet0115e8e2012-08-22 17:19:46 +00007159 call_netdevice_notifiers(NETDEV_UNREGISTER_FINAL, dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07007160 if (test_bit(__LINK_STATE_LINKWATCH_PENDING,
7161 &dev->state)) {
7162 /* We must not have linkwatch events
7163 * pending on unregister. If this
7164 * happens, we simply run the queue
7165 * unscheduled, resulting in a noop
7166 * for this device.
7167 */
7168 linkwatch_run_queue();
7169 }
7170
Stephen Hemminger6756ae42006-03-20 22:23:58 -08007171 __rtnl_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07007172
7173 rebroadcast_time = jiffies;
7174 }
7175
7176 msleep(250);
7177
Eric Dumazet29b44332010-10-11 10:22:12 +00007178 refcnt = netdev_refcnt_read(dev);
7179
Linus Torvalds1da177e2005-04-16 15:20:36 -07007180 if (time_after(jiffies, warning_time + 10 * HZ)) {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00007181 pr_emerg("unregister_netdevice: waiting for %s to become free. Usage count = %d\n",
7182 dev->name, refcnt);
Linus Torvalds1da177e2005-04-16 15:20:36 -07007183 warning_time = jiffies;
7184 }
7185 }
7186}
7187
7188/* The sequence is:
7189 *
7190 * rtnl_lock();
7191 * ...
7192 * register_netdevice(x1);
7193 * register_netdevice(x2);
7194 * ...
7195 * unregister_netdevice(y1);
7196 * unregister_netdevice(y2);
7197 * ...
7198 * rtnl_unlock();
7199 * free_netdev(y1);
7200 * free_netdev(y2);
7201 *
Herbert Xu58ec3b42008-10-07 15:50:03 -07007202 * We are invoked by rtnl_unlock().
Linus Torvalds1da177e2005-04-16 15:20:36 -07007203 * This allows us to deal with problems:
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07007204 * 1) We can delete sysfs objects which invoke hotplug
Linus Torvalds1da177e2005-04-16 15:20:36 -07007205 * without deadlocking with linkwatch via keventd.
7206 * 2) Since we run with the RTNL semaphore not held, we can sleep
7207 * safely in order to wait for the netdev refcnt to drop to zero.
Herbert Xu58ec3b42008-10-07 15:50:03 -07007208 *
7209 * We must not return until all unregister events added during
7210 * the interval the lock was held have been completed.
Linus Torvalds1da177e2005-04-16 15:20:36 -07007211 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07007212void netdev_run_todo(void)
7213{
Oleg Nesterov626ab0e2006-06-23 02:05:55 -07007214 struct list_head list;
Linus Torvalds1da177e2005-04-16 15:20:36 -07007215
Linus Torvalds1da177e2005-04-16 15:20:36 -07007216 /* Snapshot list, allow later requests */
Oleg Nesterov626ab0e2006-06-23 02:05:55 -07007217 list_replace_init(&net_todo_list, &list);
Herbert Xu58ec3b42008-10-07 15:50:03 -07007218
7219 __rtnl_unlock();
Oleg Nesterov626ab0e2006-06-23 02:05:55 -07007220
Eric Dumazet0115e8e2012-08-22 17:19:46 +00007221
7222 /* Wait for rcu callbacks to finish before next phase */
Eric W. Biederman850a5452011-10-13 22:25:23 +00007223 if (!list_empty(&list))
7224 rcu_barrier();
7225
Linus Torvalds1da177e2005-04-16 15:20:36 -07007226 while (!list_empty(&list)) {
7227 struct net_device *dev
stephen hemmingere5e26d72010-02-24 14:01:38 +00007228 = list_first_entry(&list, struct net_device, todo_list);
Linus Torvalds1da177e2005-04-16 15:20:36 -07007229 list_del(&dev->todo_list);
7230
Eric Dumazet748e2d92012-08-22 21:50:59 +00007231 rtnl_lock();
Eric Dumazet0115e8e2012-08-22 17:19:46 +00007232 call_netdevice_notifiers(NETDEV_UNREGISTER_FINAL, dev);
Eric Dumazet748e2d92012-08-22 21:50:59 +00007233 __rtnl_unlock();
Eric Dumazet0115e8e2012-08-22 17:19:46 +00007234
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07007235 if (unlikely(dev->reg_state != NETREG_UNREGISTERING)) {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00007236 pr_err("network todo '%s' but state %d\n",
Linus Torvalds1da177e2005-04-16 15:20:36 -07007237 dev->name, dev->reg_state);
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07007238 dump_stack();
7239 continue;
Linus Torvalds1da177e2005-04-16 15:20:36 -07007240 }
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07007241
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07007242 dev->reg_state = NETREG_UNREGISTERED;
7243
7244 netdev_wait_allrefs(dev);
7245
7246 /* paranoia */
Eric Dumazet29b44332010-10-11 10:22:12 +00007247 BUG_ON(netdev_refcnt_read(dev));
Salam Noureddine7866a622015-01-27 11:35:48 -08007248 BUG_ON(!list_empty(&dev->ptype_all));
7249 BUG_ON(!list_empty(&dev->ptype_specific));
Eric Dumazet33d480c2011-08-11 19:30:52 +00007250 WARN_ON(rcu_access_pointer(dev->ip_ptr));
7251 WARN_ON(rcu_access_pointer(dev->ip6_ptr));
Ilpo Järvinen547b7922008-07-25 21:43:18 -07007252 WARN_ON(dev->dn_ptr);
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07007253
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07007254 if (dev->destructor)
7255 dev->destructor(dev);
Stephen Hemminger9093bbb2007-05-19 15:39:25 -07007256
Eric W. Biederman50624c92013-09-23 21:19:49 -07007257 /* Report a network device has been unregistered */
7258 rtnl_lock();
7259 dev_net(dev)->dev_unreg_count--;
7260 __rtnl_unlock();
7261 wake_up(&netdev_unregistering_wq);
7262
Stephen Hemminger9093bbb2007-05-19 15:39:25 -07007263 /* Free network device */
7264 kobject_put(&dev->dev.kobj);
Linus Torvalds1da177e2005-04-16 15:20:36 -07007265 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07007266}
7267
Jarod Wilson92566452016-02-01 18:51:04 -05007268/* Convert net_device_stats to rtnl_link_stats64. rtnl_link_stats64 has
7269 * all the same fields in the same order as net_device_stats, with only
7270 * the type differing, but rtnl_link_stats64 may have additional fields
7271 * at the end for newer counters.
Ben Hutchings3cfde792010-07-09 09:11:52 +00007272 */
Eric Dumazet77a1abf2012-03-05 04:50:09 +00007273void netdev_stats_to_stats64(struct rtnl_link_stats64 *stats64,
7274 const struct net_device_stats *netdev_stats)
Ben Hutchings3cfde792010-07-09 09:11:52 +00007275{
7276#if BITS_PER_LONG == 64
Jarod Wilson92566452016-02-01 18:51:04 -05007277 BUILD_BUG_ON(sizeof(*stats64) < sizeof(*netdev_stats));
Eric Dumazet77a1abf2012-03-05 04:50:09 +00007278 memcpy(stats64, netdev_stats, sizeof(*stats64));
Jarod Wilson92566452016-02-01 18:51:04 -05007279 /* zero out counters that only exist in rtnl_link_stats64 */
7280 memset((char *)stats64 + sizeof(*netdev_stats), 0,
7281 sizeof(*stats64) - sizeof(*netdev_stats));
Ben Hutchings3cfde792010-07-09 09:11:52 +00007282#else
Jarod Wilson92566452016-02-01 18:51:04 -05007283 size_t i, n = sizeof(*netdev_stats) / sizeof(unsigned long);
Ben Hutchings3cfde792010-07-09 09:11:52 +00007284 const unsigned long *src = (const unsigned long *)netdev_stats;
7285 u64 *dst = (u64 *)stats64;
7286
Jarod Wilson92566452016-02-01 18:51:04 -05007287 BUILD_BUG_ON(n > sizeof(*stats64) / sizeof(u64));
Ben Hutchings3cfde792010-07-09 09:11:52 +00007288 for (i = 0; i < n; i++)
7289 dst[i] = src[i];
Jarod Wilson92566452016-02-01 18:51:04 -05007290 /* zero out counters that only exist in rtnl_link_stats64 */
7291 memset((char *)stats64 + n * sizeof(u64), 0,
7292 sizeof(*stats64) - n * sizeof(u64));
Ben Hutchings3cfde792010-07-09 09:11:52 +00007293#endif
7294}
Eric Dumazet77a1abf2012-03-05 04:50:09 +00007295EXPORT_SYMBOL(netdev_stats_to_stats64);
Ben Hutchings3cfde792010-07-09 09:11:52 +00007296
Eric Dumazetd83345a2009-11-16 03:36:51 +00007297/**
Stephen Hemmingereeda3fd2008-11-19 21:40:23 -08007298 * dev_get_stats - get network device statistics
7299 * @dev: device to get statistics from
Eric Dumazet28172732010-07-07 14:58:56 -07007300 * @storage: place to store stats
Stephen Hemmingereeda3fd2008-11-19 21:40:23 -08007301 *
Ben Hutchingsd7753512010-07-09 09:12:41 +00007302 * Get network statistics from device. Return @storage.
7303 * The device driver may provide its own method by setting
7304 * dev->netdev_ops->get_stats64 or dev->netdev_ops->get_stats;
7305 * otherwise the internal statistics structure is used.
Stephen Hemmingereeda3fd2008-11-19 21:40:23 -08007306 */
Ben Hutchingsd7753512010-07-09 09:12:41 +00007307struct rtnl_link_stats64 *dev_get_stats(struct net_device *dev,
7308 struct rtnl_link_stats64 *storage)
Eric Dumazet7004bf22009-05-18 00:34:33 +00007309{
Stephen Hemmingereeda3fd2008-11-19 21:40:23 -08007310 const struct net_device_ops *ops = dev->netdev_ops;
7311
Eric Dumazet28172732010-07-07 14:58:56 -07007312 if (ops->ndo_get_stats64) {
7313 memset(storage, 0, sizeof(*storage));
Eric Dumazetcaf586e2010-09-30 21:06:55 +00007314 ops->ndo_get_stats64(dev, storage);
7315 } else if (ops->ndo_get_stats) {
Ben Hutchings3cfde792010-07-09 09:11:52 +00007316 netdev_stats_to_stats64(storage, ops->ndo_get_stats(dev));
Eric Dumazetcaf586e2010-09-30 21:06:55 +00007317 } else {
7318 netdev_stats_to_stats64(storage, &dev->stats);
Eric Dumazet28172732010-07-07 14:58:56 -07007319 }
Eric Dumazetcaf586e2010-09-30 21:06:55 +00007320 storage->rx_dropped += atomic_long_read(&dev->rx_dropped);
Eric Dumazet015f0682014-03-27 08:45:56 -07007321 storage->tx_dropped += atomic_long_read(&dev->tx_dropped);
Jarod Wilson6e7333d2016-02-01 18:51:05 -05007322 storage->rx_nohandler += atomic_long_read(&dev->rx_nohandler);
Eric Dumazet28172732010-07-07 14:58:56 -07007323 return storage;
Rusty Russellc45d2862007-03-28 14:29:08 -07007324}
Stephen Hemmingereeda3fd2008-11-19 21:40:23 -08007325EXPORT_SYMBOL(dev_get_stats);
Rusty Russellc45d2862007-03-28 14:29:08 -07007326
Eric Dumazet24824a02010-10-02 06:11:55 +00007327struct netdev_queue *dev_ingress_queue_create(struct net_device *dev)
David S. Millerdc2b4842008-07-08 17:18:23 -07007328{
Eric Dumazet24824a02010-10-02 06:11:55 +00007329 struct netdev_queue *queue = dev_ingress_queue(dev);
David S. Millerdc2b4842008-07-08 17:18:23 -07007330
Eric Dumazet24824a02010-10-02 06:11:55 +00007331#ifdef CONFIG_NET_CLS_ACT
7332 if (queue)
7333 return queue;
7334 queue = kzalloc(sizeof(*queue), GFP_KERNEL);
7335 if (!queue)
7336 return NULL;
7337 netdev_init_one_queue(dev, queue, NULL);
Eric Dumazet2ce1ee12015-02-04 13:37:44 -08007338 RCU_INIT_POINTER(queue->qdisc, &noop_qdisc);
Eric Dumazet24824a02010-10-02 06:11:55 +00007339 queue->qdisc_sleeping = &noop_qdisc;
7340 rcu_assign_pointer(dev->ingress_queue, queue);
7341#endif
7342 return queue;
David S. Millerbb949fb2008-07-08 16:55:56 -07007343}
7344
Eric Dumazet2c60db02012-09-16 09:17:26 +00007345static const struct ethtool_ops default_ethtool_ops;
7346
Stanislaw Gruszkad07d7502013-01-10 23:19:10 +00007347void netdev_set_default_ethtool_ops(struct net_device *dev,
7348 const struct ethtool_ops *ops)
7349{
7350 if (dev->ethtool_ops == &default_ethtool_ops)
7351 dev->ethtool_ops = ops;
7352}
7353EXPORT_SYMBOL_GPL(netdev_set_default_ethtool_ops);
7354
Eric Dumazet74d332c2013-10-30 13:10:44 -07007355void netdev_freemem(struct net_device *dev)
7356{
7357 char *addr = (char *)dev - dev->padded;
7358
WANG Cong4cb28972014-06-02 15:55:22 -07007359 kvfree(addr);
Eric Dumazet74d332c2013-10-30 13:10:44 -07007360}
7361
Linus Torvalds1da177e2005-04-16 15:20:36 -07007362/**
Tom Herbert36909ea2011-01-09 19:36:31 +00007363 * alloc_netdev_mqs - allocate network device
Tom Gundersenc835a672014-07-14 16:37:24 +02007364 * @sizeof_priv: size of private data to allocate space for
7365 * @name: device name format string
7366 * @name_assign_type: origin of device name
7367 * @setup: callback to initialize device
7368 * @txqs: the number of TX subqueues to allocate
7369 * @rxqs: the number of RX subqueues to allocate
Linus Torvalds1da177e2005-04-16 15:20:36 -07007370 *
7371 * Allocates a struct net_device with private data area for driver use
Li Zhong90e51ad2013-11-22 15:04:46 +08007372 * and performs basic initialization. Also allocates subqueue structs
Tom Herbert36909ea2011-01-09 19:36:31 +00007373 * for each queue on the device.
Linus Torvalds1da177e2005-04-16 15:20:36 -07007374 */
Tom Herbert36909ea2011-01-09 19:36:31 +00007375struct net_device *alloc_netdev_mqs(int sizeof_priv, const char *name,
Tom Gundersenc835a672014-07-14 16:37:24 +02007376 unsigned char name_assign_type,
Tom Herbert36909ea2011-01-09 19:36:31 +00007377 void (*setup)(struct net_device *),
7378 unsigned int txqs, unsigned int rxqs)
Linus Torvalds1da177e2005-04-16 15:20:36 -07007379{
Linus Torvalds1da177e2005-04-16 15:20:36 -07007380 struct net_device *dev;
Stephen Hemminger79439862008-07-21 13:28:44 -07007381 size_t alloc_size;
Eric Dumazet1ce8e7b2009-05-27 04:42:37 +00007382 struct net_device *p;
Linus Torvalds1da177e2005-04-16 15:20:36 -07007383
Stephen Hemmingerb6fe17d2006-08-29 17:06:13 -07007384 BUG_ON(strlen(name) >= sizeof(dev->name));
7385
Tom Herbert36909ea2011-01-09 19:36:31 +00007386 if (txqs < 1) {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00007387 pr_err("alloc_netdev: Unable to allocate device with zero queues\n");
Tom Herbert55513fb2010-10-18 17:55:58 +00007388 return NULL;
7389 }
7390
Michael Daltona953be52014-01-16 22:23:28 -08007391#ifdef CONFIG_SYSFS
Tom Herbert36909ea2011-01-09 19:36:31 +00007392 if (rxqs < 1) {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00007393 pr_err("alloc_netdev: Unable to allocate device with zero RX queues\n");
Tom Herbert36909ea2011-01-09 19:36:31 +00007394 return NULL;
7395 }
7396#endif
7397
David S. Millerfd2ea0a2008-07-17 01:56:23 -07007398 alloc_size = sizeof(struct net_device);
Alexey Dobriyand1643d22008-04-18 15:43:32 -07007399 if (sizeof_priv) {
7400 /* ensure 32-byte alignment of private area */
Eric Dumazet1ce8e7b2009-05-27 04:42:37 +00007401 alloc_size = ALIGN(alloc_size, NETDEV_ALIGN);
Alexey Dobriyand1643d22008-04-18 15:43:32 -07007402 alloc_size += sizeof_priv;
7403 }
7404 /* ensure 32-byte alignment of whole construct */
Eric Dumazet1ce8e7b2009-05-27 04:42:37 +00007405 alloc_size += NETDEV_ALIGN - 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -07007406
Eric Dumazet74d332c2013-10-30 13:10:44 -07007407 p = kzalloc(alloc_size, GFP_KERNEL | __GFP_NOWARN | __GFP_REPEAT);
7408 if (!p)
7409 p = vzalloc(alloc_size);
Joe Perches62b59422013-02-04 16:48:16 +00007410 if (!p)
Linus Torvalds1da177e2005-04-16 15:20:36 -07007411 return NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07007412
Eric Dumazet1ce8e7b2009-05-27 04:42:37 +00007413 dev = PTR_ALIGN(p, NETDEV_ALIGN);
Linus Torvalds1da177e2005-04-16 15:20:36 -07007414 dev->padded = (char *)dev - (char *)p;
Jiri Pirkoab9c73c2009-05-08 13:30:17 +00007415
Eric Dumazet29b44332010-10-11 10:22:12 +00007416 dev->pcpu_refcnt = alloc_percpu(int);
7417 if (!dev->pcpu_refcnt)
Eric Dumazet74d332c2013-10-30 13:10:44 -07007418 goto free_dev;
Jiri Pirkoab9c73c2009-05-08 13:30:17 +00007419
Linus Torvalds1da177e2005-04-16 15:20:36 -07007420 if (dev_addr_init(dev))
Eric Dumazet29b44332010-10-11 10:22:12 +00007421 goto free_pcpu;
Linus Torvalds1da177e2005-04-16 15:20:36 -07007422
Jiri Pirko22bedad32010-04-01 21:22:57 +00007423 dev_mc_init(dev);
Jiri Pirkoa748ee22010-04-01 21:22:09 +00007424 dev_uc_init(dev);
Jiri Pirkoccffad252009-05-22 23:22:17 +00007425
YOSHIFUJI Hideakic346dca2008-03-25 21:47:49 +09007426 dev_net_set(dev, &init_net);
Linus Torvalds1da177e2005-04-16 15:20:36 -07007427
Peter P Waskiewicz Jr82cc1a72008-03-21 03:43:19 -07007428 dev->gso_max_size = GSO_MAX_SIZE;
Ben Hutchings30b678d2012-07-30 15:57:00 +00007429 dev->gso_max_segs = GSO_MAX_SEGS;
Eric Dumazetfcbeb972014-10-05 10:11:27 -07007430 dev->gso_min_segs = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07007431
Herbert Xud565b0a2008-12-15 23:38:52 -08007432 INIT_LIST_HEAD(&dev->napi_list);
Eric W. Biederman9fdce092009-10-30 14:51:13 +00007433 INIT_LIST_HEAD(&dev->unreg_list);
Eric W. Biederman5cde2822013-10-05 19:26:05 -07007434 INIT_LIST_HEAD(&dev->close_list);
Eric Dumazete014deb2009-11-17 05:59:21 +00007435 INIT_LIST_HEAD(&dev->link_watch_list);
Veaceslav Falico2f268f12013-09-25 09:20:07 +02007436 INIT_LIST_HEAD(&dev->adj_list.upper);
7437 INIT_LIST_HEAD(&dev->adj_list.lower);
7438 INIT_LIST_HEAD(&dev->all_adj_list.upper);
7439 INIT_LIST_HEAD(&dev->all_adj_list.lower);
Salam Noureddine7866a622015-01-27 11:35:48 -08007440 INIT_LIST_HEAD(&dev->ptype_all);
7441 INIT_LIST_HEAD(&dev->ptype_specific);
Eric Dumazet02875872014-10-05 18:38:35 -07007442 dev->priv_flags = IFF_XMIT_DST_RELEASE | IFF_XMIT_DST_RELEASE_PERM;
Linus Torvalds1da177e2005-04-16 15:20:36 -07007443 setup(dev);
David S. Miller8d3bdbd2011-02-08 15:02:50 -08007444
Phil Suttera8131042016-02-17 15:37:43 +01007445 if (!dev->tx_queue_len) {
Phil Sutterf84bb1e2015-08-27 21:21:36 +02007446 dev->priv_flags |= IFF_NO_QUEUE;
Phil Suttera8131042016-02-17 15:37:43 +01007447 dev->tx_queue_len = 1;
7448 }
Phil Sutter906470c2015-08-18 10:30:48 +02007449
David S. Miller8d3bdbd2011-02-08 15:02:50 -08007450 dev->num_tx_queues = txqs;
7451 dev->real_num_tx_queues = txqs;
7452 if (netif_alloc_netdev_queues(dev))
7453 goto free_all;
7454
Michael Daltona953be52014-01-16 22:23:28 -08007455#ifdef CONFIG_SYSFS
David S. Miller8d3bdbd2011-02-08 15:02:50 -08007456 dev->num_rx_queues = rxqs;
7457 dev->real_num_rx_queues = rxqs;
7458 if (netif_alloc_rx_queues(dev))
7459 goto free_all;
7460#endif
7461
Linus Torvalds1da177e2005-04-16 15:20:36 -07007462 strcpy(dev->name, name);
Tom Gundersenc835a672014-07-14 16:37:24 +02007463 dev->name_assign_type = name_assign_type;
Vlad Dogarucbda10f2011-01-13 23:38:30 +00007464 dev->group = INIT_NETDEV_GROUP;
Eric Dumazet2c60db02012-09-16 09:17:26 +00007465 if (!dev->ethtool_ops)
7466 dev->ethtool_ops = &default_ethtool_ops;
Pablo Neirae687ad62015-05-13 18:19:38 +02007467
7468 nf_hook_ingress_init(dev);
7469
Linus Torvalds1da177e2005-04-16 15:20:36 -07007470 return dev;
Jiri Pirkoab9c73c2009-05-08 13:30:17 +00007471
David S. Miller8d3bdbd2011-02-08 15:02:50 -08007472free_all:
7473 free_netdev(dev);
7474 return NULL;
7475
Eric Dumazet29b44332010-10-11 10:22:12 +00007476free_pcpu:
7477 free_percpu(dev->pcpu_refcnt);
Eric Dumazet74d332c2013-10-30 13:10:44 -07007478free_dev:
7479 netdev_freemem(dev);
Jiri Pirkoab9c73c2009-05-08 13:30:17 +00007480 return NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07007481}
Tom Herbert36909ea2011-01-09 19:36:31 +00007482EXPORT_SYMBOL(alloc_netdev_mqs);
Linus Torvalds1da177e2005-04-16 15:20:36 -07007483
7484/**
7485 * free_netdev - free network device
7486 * @dev: device
7487 *
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09007488 * This function does the last stage of destroying an allocated device
7489 * interface. The reference to the device object is released.
Linus Torvalds1da177e2005-04-16 15:20:36 -07007490 * If this is the last reference then it will be freed.
Eric Dumazet93d05d42015-11-18 06:31:03 -08007491 * Must be called in process context.
Linus Torvalds1da177e2005-04-16 15:20:36 -07007492 */
7493void free_netdev(struct net_device *dev)
7494{
Herbert Xud565b0a2008-12-15 23:38:52 -08007495 struct napi_struct *p, *n;
7496
Eric Dumazet93d05d42015-11-18 06:31:03 -08007497 might_sleep();
Eric Dumazet60877a32013-06-20 01:15:51 -07007498 netif_free_tx_queues(dev);
Michael Daltona953be52014-01-16 22:23:28 -08007499#ifdef CONFIG_SYSFS
Pankaj Gupta10595902015-01-12 11:41:28 +05307500 kvfree(dev->_rx);
Tom Herbertfe822242010-11-09 10:47:38 +00007501#endif
David S. Millere8a04642008-07-17 00:34:19 -07007502
Eric Dumazet33d480c2011-08-11 19:30:52 +00007503 kfree(rcu_dereference_protected(dev->ingress_queue, 1));
Eric Dumazet24824a02010-10-02 06:11:55 +00007504
Jiri Pirkof001fde2009-05-05 02:48:28 +00007505 /* Flush device addresses */
7506 dev_addr_flush(dev);
7507
Herbert Xud565b0a2008-12-15 23:38:52 -08007508 list_for_each_entry_safe(p, n, &dev->napi_list, dev_list)
7509 netif_napi_del(p);
7510
Eric Dumazet29b44332010-10-11 10:22:12 +00007511 free_percpu(dev->pcpu_refcnt);
7512 dev->pcpu_refcnt = NULL;
7513
Stephen Hemminger3041a062006-05-26 13:25:24 -07007514 /* Compatibility with error handling in drivers */
Linus Torvalds1da177e2005-04-16 15:20:36 -07007515 if (dev->reg_state == NETREG_UNINITIALIZED) {
Eric Dumazet74d332c2013-10-30 13:10:44 -07007516 netdev_freemem(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07007517 return;
7518 }
7519
7520 BUG_ON(dev->reg_state != NETREG_UNREGISTERED);
7521 dev->reg_state = NETREG_RELEASED;
7522
Greg Kroah-Hartman43cb76d2002-04-09 12:14:34 -07007523 /* will free via device release */
7524 put_device(&dev->dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07007525}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07007526EXPORT_SYMBOL(free_netdev);
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09007527
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07007528/**
7529 * synchronize_net - Synchronize with packet receive processing
7530 *
7531 * Wait for packets currently being received to be done.
7532 * Does not block later packets from starting.
7533 */
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09007534void synchronize_net(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -07007535{
7536 might_sleep();
Eric Dumazetbe3fc412011-05-23 23:07:32 +00007537 if (rtnl_is_locked())
7538 synchronize_rcu_expedited();
7539 else
7540 synchronize_rcu();
Linus Torvalds1da177e2005-04-16 15:20:36 -07007541}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07007542EXPORT_SYMBOL(synchronize_net);
Linus Torvalds1da177e2005-04-16 15:20:36 -07007543
7544/**
Eric Dumazet44a08732009-10-27 07:03:04 +00007545 * unregister_netdevice_queue - remove device from the kernel
Linus Torvalds1da177e2005-04-16 15:20:36 -07007546 * @dev: device
Eric Dumazet44a08732009-10-27 07:03:04 +00007547 * @head: list
Jaswinder Singh Rajput6ebfbc02009-11-22 20:43:13 -08007548 *
Linus Torvalds1da177e2005-04-16 15:20:36 -07007549 * This function shuts down a device interface and removes it
Wang Chend59b54b2007-12-11 02:28:03 -08007550 * from the kernel tables.
Eric Dumazet44a08732009-10-27 07:03:04 +00007551 * If head not NULL, device is queued to be unregistered later.
Linus Torvalds1da177e2005-04-16 15:20:36 -07007552 *
7553 * Callers must hold the rtnl semaphore. You may want
7554 * unregister_netdev() instead of this.
7555 */
7556
Eric Dumazet44a08732009-10-27 07:03:04 +00007557void unregister_netdevice_queue(struct net_device *dev, struct list_head *head)
Linus Torvalds1da177e2005-04-16 15:20:36 -07007558{
Herbert Xua6620712007-12-12 19:21:56 -08007559 ASSERT_RTNL();
7560
Eric Dumazet44a08732009-10-27 07:03:04 +00007561 if (head) {
Eric W. Biederman9fdce092009-10-30 14:51:13 +00007562 list_move_tail(&dev->unreg_list, head);
Eric Dumazet44a08732009-10-27 07:03:04 +00007563 } else {
7564 rollback_registered(dev);
7565 /* Finish processing unregister after unlock */
7566 net_set_todo(dev);
7567 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07007568}
Eric Dumazet44a08732009-10-27 07:03:04 +00007569EXPORT_SYMBOL(unregister_netdevice_queue);
Linus Torvalds1da177e2005-04-16 15:20:36 -07007570
7571/**
Eric Dumazet9b5e3832009-10-27 07:04:19 +00007572 * unregister_netdevice_many - unregister many devices
7573 * @head: list of devices
Eric Dumazet87757a92014-06-06 06:44:03 -07007574 *
7575 * Note: As most callers use a stack allocated list_head,
7576 * we force a list_del() to make sure stack wont be corrupted later.
Eric Dumazet9b5e3832009-10-27 07:04:19 +00007577 */
7578void unregister_netdevice_many(struct list_head *head)
7579{
7580 struct net_device *dev;
7581
7582 if (!list_empty(head)) {
7583 rollback_registered_many(head);
7584 list_for_each_entry(dev, head, unreg_list)
7585 net_set_todo(dev);
Eric Dumazet87757a92014-06-06 06:44:03 -07007586 list_del(head);
Eric Dumazet9b5e3832009-10-27 07:04:19 +00007587 }
7588}
Eric Dumazet63c80992009-10-27 07:06:49 +00007589EXPORT_SYMBOL(unregister_netdevice_many);
Eric Dumazet9b5e3832009-10-27 07:04:19 +00007590
7591/**
Linus Torvalds1da177e2005-04-16 15:20:36 -07007592 * unregister_netdev - remove device from the kernel
7593 * @dev: device
7594 *
7595 * This function shuts down a device interface and removes it
Wang Chend59b54b2007-12-11 02:28:03 -08007596 * from the kernel tables.
Linus Torvalds1da177e2005-04-16 15:20:36 -07007597 *
7598 * This is just a wrapper for unregister_netdevice that takes
7599 * the rtnl semaphore. In general you want to use this and not
7600 * unregister_netdevice.
7601 */
7602void unregister_netdev(struct net_device *dev)
7603{
7604 rtnl_lock();
7605 unregister_netdevice(dev);
7606 rtnl_unlock();
7607}
Linus Torvalds1da177e2005-04-16 15:20:36 -07007608EXPORT_SYMBOL(unregister_netdev);
7609
Eric W. Biedermance286d32007-09-12 13:53:49 +02007610/**
7611 * dev_change_net_namespace - move device to different nethost namespace
7612 * @dev: device
7613 * @net: network namespace
7614 * @pat: If not NULL name pattern to try if the current device name
7615 * is already taken in the destination network namespace.
7616 *
7617 * This function shuts down a device interface and moves it
7618 * to a new network namespace. On success 0 is returned, on
7619 * a failure a netagive errno code is returned.
7620 *
7621 * Callers must hold the rtnl semaphore.
7622 */
7623
7624int dev_change_net_namespace(struct net_device *dev, struct net *net, const char *pat)
7625{
Eric W. Biedermance286d32007-09-12 13:53:49 +02007626 int err;
7627
7628 ASSERT_RTNL();
7629
7630 /* Don't allow namespace local devices to be moved. */
7631 err = -EINVAL;
7632 if (dev->features & NETIF_F_NETNS_LOCAL)
7633 goto out;
7634
7635 /* Ensure the device has been registrered */
Eric W. Biedermance286d32007-09-12 13:53:49 +02007636 if (dev->reg_state != NETREG_REGISTERED)
7637 goto out;
7638
7639 /* Get out if there is nothing todo */
7640 err = 0;
YOSHIFUJI Hideaki878628f2008-03-26 03:57:35 +09007641 if (net_eq(dev_net(dev), net))
Eric W. Biedermance286d32007-09-12 13:53:49 +02007642 goto out;
7643
7644 /* Pick the destination device name, and ensure
7645 * we can use it in the destination network namespace.
7646 */
7647 err = -EEXIST;
Octavian Purdilad9031022009-11-18 02:36:59 +00007648 if (__dev_get_by_name(net, dev->name)) {
Eric W. Biedermance286d32007-09-12 13:53:49 +02007649 /* We get here if we can't use the current device name */
7650 if (!pat)
7651 goto out;
Gao feng828de4f2012-09-13 20:58:27 +00007652 if (dev_get_valid_name(net, dev, pat) < 0)
Eric W. Biedermance286d32007-09-12 13:53:49 +02007653 goto out;
7654 }
7655
7656 /*
7657 * And now a mini version of register_netdevice unregister_netdevice.
7658 */
7659
7660 /* If device is running close it first. */
Pavel Emelyanov9b772652007-10-10 02:49:09 -07007661 dev_close(dev);
Eric W. Biedermance286d32007-09-12 13:53:49 +02007662
7663 /* And unlink it from device chain */
7664 err = -ENODEV;
7665 unlist_netdevice(dev);
7666
7667 synchronize_net();
7668
7669 /* Shutdown queueing discipline. */
7670 dev_shutdown(dev);
7671
7672 /* Notify protocols, that we are about to destroy
7673 this device. They should clean all the things.
David Lamparter3b27e102010-09-17 03:22:19 +00007674
7675 Note that dev->reg_state stays at NETREG_REGISTERED.
7676 This is wanted because this way 8021q and macvlan know
7677 the device is just moving and can keep their slaves up.
Eric W. Biedermance286d32007-09-12 13:53:49 +02007678 */
7679 call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
Gao feng6549dd42012-08-23 15:36:55 +00007680 rcu_barrier();
7681 call_netdevice_notifiers(NETDEV_UNREGISTER_FINAL, dev);
Alexei Starovoitov7f294052013-10-23 16:02:42 -07007682 rtmsg_ifinfo(RTM_DELLINK, dev, ~0U, GFP_KERNEL);
Eric W. Biedermance286d32007-09-12 13:53:49 +02007683
7684 /*
7685 * Flush the unicast and multicast chains
7686 */
Jiri Pirkoa748ee22010-04-01 21:22:09 +00007687 dev_uc_flush(dev);
Jiri Pirko22bedad32010-04-01 21:22:57 +00007688 dev_mc_flush(dev);
Eric W. Biedermance286d32007-09-12 13:53:49 +02007689
Serge Hallyn4e66ae22012-12-03 16:17:12 +00007690 /* Send a netdev-removed uevent to the old namespace */
7691 kobject_uevent(&dev->dev.kobj, KOBJ_REMOVE);
Alexander Y. Fomichev4c754312014-08-25 16:26:45 +04007692 netdev_adjacent_del_links(dev);
Serge Hallyn4e66ae22012-12-03 16:17:12 +00007693
Eric W. Biedermance286d32007-09-12 13:53:49 +02007694 /* Actually switch the network namespace */
YOSHIFUJI Hideakic346dca2008-03-25 21:47:49 +09007695 dev_net_set(dev, net);
Eric W. Biedermance286d32007-09-12 13:53:49 +02007696
Eric W. Biedermance286d32007-09-12 13:53:49 +02007697 /* If there is an ifindex conflict assign a new one */
Nicolas Dichtel7a66bbc2015-04-02 17:07:09 +02007698 if (__dev_get_by_index(net, dev->ifindex))
Eric W. Biedermance286d32007-09-12 13:53:49 +02007699 dev->ifindex = dev_new_index(net);
Eric W. Biedermance286d32007-09-12 13:53:49 +02007700
Serge Hallyn4e66ae22012-12-03 16:17:12 +00007701 /* Send a netdev-add uevent to the new namespace */
7702 kobject_uevent(&dev->dev.kobj, KOBJ_ADD);
Alexander Y. Fomichev4c754312014-08-25 16:26:45 +04007703 netdev_adjacent_add_links(dev);
Serge Hallyn4e66ae22012-12-03 16:17:12 +00007704
Eric W. Biederman8b41d182007-09-26 22:02:53 -07007705 /* Fixup kobjects */
Eric W. Biedermana1b3f592010-05-04 17:36:49 -07007706 err = device_rename(&dev->dev, dev->name);
Eric W. Biederman8b41d182007-09-26 22:02:53 -07007707 WARN_ON(err);
Eric W. Biedermance286d32007-09-12 13:53:49 +02007708
7709 /* Add the device back in the hashes */
7710 list_netdevice(dev);
7711
7712 /* Notify protocols, that a new device appeared. */
7713 call_netdevice_notifiers(NETDEV_REGISTER, dev);
7714
Eric W. Biedermand90a9092009-12-12 22:11:15 +00007715 /*
7716 * Prevent userspace races by waiting until the network
7717 * device is fully setup before sending notifications.
7718 */
Alexei Starovoitov7f294052013-10-23 16:02:42 -07007719 rtmsg_ifinfo(RTM_NEWLINK, dev, ~0U, GFP_KERNEL);
Eric W. Biedermand90a9092009-12-12 22:11:15 +00007720
Eric W. Biedermance286d32007-09-12 13:53:49 +02007721 synchronize_net();
7722 err = 0;
7723out:
7724 return err;
7725}
Johannes Berg463d0182009-07-14 00:33:35 +02007726EXPORT_SYMBOL_GPL(dev_change_net_namespace);
Eric W. Biedermance286d32007-09-12 13:53:49 +02007727
Linus Torvalds1da177e2005-04-16 15:20:36 -07007728static int dev_cpu_callback(struct notifier_block *nfb,
7729 unsigned long action,
7730 void *ocpu)
7731{
7732 struct sk_buff **list_skb;
Linus Torvalds1da177e2005-04-16 15:20:36 -07007733 struct sk_buff *skb;
7734 unsigned int cpu, oldcpu = (unsigned long)ocpu;
7735 struct softnet_data *sd, *oldsd;
7736
Rafael J. Wysocki8bb78442007-05-09 02:35:10 -07007737 if (action != CPU_DEAD && action != CPU_DEAD_FROZEN)
Linus Torvalds1da177e2005-04-16 15:20:36 -07007738 return NOTIFY_OK;
7739
7740 local_irq_disable();
7741 cpu = smp_processor_id();
7742 sd = &per_cpu(softnet_data, cpu);
7743 oldsd = &per_cpu(softnet_data, oldcpu);
7744
7745 /* Find end of our completion_queue. */
7746 list_skb = &sd->completion_queue;
7747 while (*list_skb)
7748 list_skb = &(*list_skb)->next;
7749 /* Append completion queue from offline CPU. */
7750 *list_skb = oldsd->completion_queue;
7751 oldsd->completion_queue = NULL;
7752
Linus Torvalds1da177e2005-04-16 15:20:36 -07007753 /* Append output queue from offline CPU. */
Changli Gaoa9cbd582010-04-26 23:06:24 +00007754 if (oldsd->output_queue) {
7755 *sd->output_queue_tailp = oldsd->output_queue;
7756 sd->output_queue_tailp = oldsd->output_queue_tailp;
7757 oldsd->output_queue = NULL;
7758 oldsd->output_queue_tailp = &oldsd->output_queue;
7759 }
Eric Dumazetac64da02015-01-15 17:04:22 -08007760 /* Append NAPI poll list from offline CPU, with one exception :
7761 * process_backlog() must be called by cpu owning percpu backlog.
7762 * We properly handle process_queue & input_pkt_queue later.
7763 */
7764 while (!list_empty(&oldsd->poll_list)) {
7765 struct napi_struct *napi = list_first_entry(&oldsd->poll_list,
7766 struct napi_struct,
7767 poll_list);
7768
7769 list_del_init(&napi->poll_list);
7770 if (napi->poll == process_backlog)
7771 napi->state = 0;
7772 else
7773 ____napi_schedule(sd, napi);
Heiko Carstens264524d2011-06-06 20:50:03 +00007774 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07007775
7776 raise_softirq_irqoff(NET_TX_SOFTIRQ);
7777 local_irq_enable();
7778
7779 /* Process offline CPU's input_pkt_queue */
Tom Herbert76cc8b12010-05-20 18:37:59 +00007780 while ((skb = __skb_dequeue(&oldsd->process_queue))) {
Eric Dumazet91e83132015-02-05 14:58:14 -08007781 netif_rx_ni(skb);
Tom Herbert76cc8b12010-05-20 18:37:59 +00007782 input_queue_head_incr(oldsd);
7783 }
Eric Dumazetac64da02015-01-15 17:04:22 -08007784 while ((skb = skb_dequeue(&oldsd->input_pkt_queue))) {
Eric Dumazet91e83132015-02-05 14:58:14 -08007785 netif_rx_ni(skb);
Tom Herbert76cc8b12010-05-20 18:37:59 +00007786 input_queue_head_incr(oldsd);
Tom Herbertfec5e652010-04-16 16:01:27 -07007787 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07007788
7789 return NOTIFY_OK;
7790}
Linus Torvalds1da177e2005-04-16 15:20:36 -07007791
7792
Herbert Xu7f353bf2007-08-10 15:47:58 -07007793/**
Herbert Xub63365a2008-10-23 01:11:29 -07007794 * netdev_increment_features - increment feature set by one
7795 * @all: current feature set
7796 * @one: new feature set
7797 * @mask: mask feature set
Herbert Xu7f353bf2007-08-10 15:47:58 -07007798 *
7799 * Computes a new feature set after adding a device with feature set
Herbert Xub63365a2008-10-23 01:11:29 -07007800 * @one to the master device with current feature set @all. Will not
7801 * enable anything that is off in @mask. Returns the new feature set.
Herbert Xu7f353bf2007-08-10 15:47:58 -07007802 */
Michał Mirosławc8f44af2011-11-15 15:29:55 +00007803netdev_features_t netdev_increment_features(netdev_features_t all,
7804 netdev_features_t one, netdev_features_t mask)
Herbert Xu7f353bf2007-08-10 15:47:58 -07007805{
Tom Herbertc8cd0982015-12-14 11:19:44 -08007806 if (mask & NETIF_F_HW_CSUM)
Tom Herberta1882222015-12-14 11:19:43 -08007807 mask |= NETIF_F_CSUM_MASK;
Michał Mirosław1742f182011-04-22 06:31:16 +00007808 mask |= NETIF_F_VLAN_CHALLENGED;
7809
Tom Herberta1882222015-12-14 11:19:43 -08007810 all |= one & (NETIF_F_ONE_FOR_ALL | NETIF_F_CSUM_MASK) & mask;
Michał Mirosław1742f182011-04-22 06:31:16 +00007811 all &= one | ~NETIF_F_ALL_FOR_ALL;
7812
Michał Mirosław1742f182011-04-22 06:31:16 +00007813 /* If one device supports hw checksumming, set for all. */
Tom Herbertc8cd0982015-12-14 11:19:44 -08007814 if (all & NETIF_F_HW_CSUM)
7815 all &= ~(NETIF_F_CSUM_MASK & ~NETIF_F_HW_CSUM);
Herbert Xu7f353bf2007-08-10 15:47:58 -07007816
7817 return all;
7818}
Herbert Xub63365a2008-10-23 01:11:29 -07007819EXPORT_SYMBOL(netdev_increment_features);
Herbert Xu7f353bf2007-08-10 15:47:58 -07007820
Baruch Siach430f03c2013-06-02 20:43:55 +00007821static struct hlist_head * __net_init netdev_create_hash(void)
Pavel Emelyanov30d97d32007-09-16 15:40:33 -07007822{
7823 int i;
7824 struct hlist_head *hash;
7825
7826 hash = kmalloc(sizeof(*hash) * NETDEV_HASHENTRIES, GFP_KERNEL);
7827 if (hash != NULL)
7828 for (i = 0; i < NETDEV_HASHENTRIES; i++)
7829 INIT_HLIST_HEAD(&hash[i]);
7830
7831 return hash;
7832}
7833
Eric W. Biederman881d9662007-09-17 11:56:21 -07007834/* Initialize per network namespace state */
Pavel Emelyanov46650792007-10-08 20:38:39 -07007835static int __net_init netdev_init(struct net *net)
Eric W. Biederman881d9662007-09-17 11:56:21 -07007836{
Rustad, Mark D734b6542012-07-18 09:06:07 +00007837 if (net != &init_net)
7838 INIT_LIST_HEAD(&net->dev_base_head);
Eric W. Biederman881d9662007-09-17 11:56:21 -07007839
Pavel Emelyanov30d97d32007-09-16 15:40:33 -07007840 net->dev_name_head = netdev_create_hash();
7841 if (net->dev_name_head == NULL)
7842 goto err_name;
Eric W. Biederman881d9662007-09-17 11:56:21 -07007843
Pavel Emelyanov30d97d32007-09-16 15:40:33 -07007844 net->dev_index_head = netdev_create_hash();
7845 if (net->dev_index_head == NULL)
7846 goto err_idx;
Eric W. Biederman881d9662007-09-17 11:56:21 -07007847
7848 return 0;
Pavel Emelyanov30d97d32007-09-16 15:40:33 -07007849
7850err_idx:
7851 kfree(net->dev_name_head);
7852err_name:
7853 return -ENOMEM;
Eric W. Biederman881d9662007-09-17 11:56:21 -07007854}
7855
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07007856/**
7857 * netdev_drivername - network driver for the device
7858 * @dev: network device
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07007859 *
7860 * Determine network driver for device.
7861 */
David S. Miller3019de12011-06-06 16:41:33 -07007862const char *netdev_drivername(const struct net_device *dev)
Arjan van de Ven6579e572008-07-21 13:31:48 -07007863{
Stephen Hemmingercf04a4c72008-09-30 02:22:14 -07007864 const struct device_driver *driver;
7865 const struct device *parent;
David S. Miller3019de12011-06-06 16:41:33 -07007866 const char *empty = "";
Arjan van de Ven6579e572008-07-21 13:31:48 -07007867
7868 parent = dev->dev.parent;
Arjan van de Ven6579e572008-07-21 13:31:48 -07007869 if (!parent)
David S. Miller3019de12011-06-06 16:41:33 -07007870 return empty;
Arjan van de Ven6579e572008-07-21 13:31:48 -07007871
7872 driver = parent->driver;
7873 if (driver && driver->name)
David S. Miller3019de12011-06-06 16:41:33 -07007874 return driver->name;
7875 return empty;
Arjan van de Ven6579e572008-07-21 13:31:48 -07007876}
7877
Joe Perches6ea754e2014-09-22 11:10:50 -07007878static void __netdev_printk(const char *level, const struct net_device *dev,
7879 struct va_format *vaf)
Joe Perches256df2f2010-06-27 01:02:35 +00007880{
Joe Perchesb004ff42012-09-12 20:12:19 -07007881 if (dev && dev->dev.parent) {
Joe Perches6ea754e2014-09-22 11:10:50 -07007882 dev_printk_emit(level[1] - '0',
7883 dev->dev.parent,
7884 "%s %s %s%s: %pV",
7885 dev_driver_string(dev->dev.parent),
7886 dev_name(dev->dev.parent),
7887 netdev_name(dev), netdev_reg_state(dev),
7888 vaf);
Joe Perchesb004ff42012-09-12 20:12:19 -07007889 } else if (dev) {
Joe Perches6ea754e2014-09-22 11:10:50 -07007890 printk("%s%s%s: %pV",
7891 level, netdev_name(dev), netdev_reg_state(dev), vaf);
Joe Perchesb004ff42012-09-12 20:12:19 -07007892 } else {
Joe Perches6ea754e2014-09-22 11:10:50 -07007893 printk("%s(NULL net_device): %pV", level, vaf);
Joe Perchesb004ff42012-09-12 20:12:19 -07007894 }
Joe Perches256df2f2010-06-27 01:02:35 +00007895}
7896
Joe Perches6ea754e2014-09-22 11:10:50 -07007897void netdev_printk(const char *level, const struct net_device *dev,
7898 const char *format, ...)
Joe Perches256df2f2010-06-27 01:02:35 +00007899{
7900 struct va_format vaf;
7901 va_list args;
Joe Perches256df2f2010-06-27 01:02:35 +00007902
7903 va_start(args, format);
7904
7905 vaf.fmt = format;
7906 vaf.va = &args;
7907
Joe Perches6ea754e2014-09-22 11:10:50 -07007908 __netdev_printk(level, dev, &vaf);
Joe Perchesb004ff42012-09-12 20:12:19 -07007909
Joe Perches256df2f2010-06-27 01:02:35 +00007910 va_end(args);
Joe Perches256df2f2010-06-27 01:02:35 +00007911}
7912EXPORT_SYMBOL(netdev_printk);
7913
7914#define define_netdev_printk_level(func, level) \
Joe Perches6ea754e2014-09-22 11:10:50 -07007915void func(const struct net_device *dev, const char *fmt, ...) \
Joe Perches256df2f2010-06-27 01:02:35 +00007916{ \
Joe Perches256df2f2010-06-27 01:02:35 +00007917 struct va_format vaf; \
7918 va_list args; \
7919 \
7920 va_start(args, fmt); \
7921 \
7922 vaf.fmt = fmt; \
7923 vaf.va = &args; \
7924 \
Joe Perches6ea754e2014-09-22 11:10:50 -07007925 __netdev_printk(level, dev, &vaf); \
Joe Perchesb004ff42012-09-12 20:12:19 -07007926 \
Joe Perches256df2f2010-06-27 01:02:35 +00007927 va_end(args); \
Joe Perches256df2f2010-06-27 01:02:35 +00007928} \
7929EXPORT_SYMBOL(func);
7930
7931define_netdev_printk_level(netdev_emerg, KERN_EMERG);
7932define_netdev_printk_level(netdev_alert, KERN_ALERT);
7933define_netdev_printk_level(netdev_crit, KERN_CRIT);
7934define_netdev_printk_level(netdev_err, KERN_ERR);
7935define_netdev_printk_level(netdev_warn, KERN_WARNING);
7936define_netdev_printk_level(netdev_notice, KERN_NOTICE);
7937define_netdev_printk_level(netdev_info, KERN_INFO);
7938
Pavel Emelyanov46650792007-10-08 20:38:39 -07007939static void __net_exit netdev_exit(struct net *net)
Eric W. Biederman881d9662007-09-17 11:56:21 -07007940{
7941 kfree(net->dev_name_head);
7942 kfree(net->dev_index_head);
7943}
7944
Denis V. Lunev022cbae2007-11-13 03:23:50 -08007945static struct pernet_operations __net_initdata netdev_net_ops = {
Eric W. Biederman881d9662007-09-17 11:56:21 -07007946 .init = netdev_init,
7947 .exit = netdev_exit,
7948};
7949
Pavel Emelyanov46650792007-10-08 20:38:39 -07007950static void __net_exit default_device_exit(struct net *net)
Eric W. Biedermance286d32007-09-12 13:53:49 +02007951{
Eric W. Biedermane008b5f2009-11-29 22:25:30 +00007952 struct net_device *dev, *aux;
Eric W. Biedermance286d32007-09-12 13:53:49 +02007953 /*
Eric W. Biedermane008b5f2009-11-29 22:25:30 +00007954 * Push all migratable network devices back to the
Eric W. Biedermance286d32007-09-12 13:53:49 +02007955 * initial network namespace
7956 */
7957 rtnl_lock();
Eric W. Biedermane008b5f2009-11-29 22:25:30 +00007958 for_each_netdev_safe(net, dev, aux) {
Eric W. Biedermance286d32007-09-12 13:53:49 +02007959 int err;
Pavel Emelyanovaca51392008-05-08 01:24:25 -07007960 char fb_name[IFNAMSIZ];
Eric W. Biedermance286d32007-09-12 13:53:49 +02007961
7962 /* Ignore unmoveable devices (i.e. loopback) */
7963 if (dev->features & NETIF_F_NETNS_LOCAL)
7964 continue;
7965
Eric W. Biedermane008b5f2009-11-29 22:25:30 +00007966 /* Leave virtual devices for the generic cleanup */
7967 if (dev->rtnl_link_ops)
7968 continue;
Eric W. Biedermand0c082c2008-11-05 15:59:38 -08007969
Lucas De Marchi25985ed2011-03-30 22:57:33 -03007970 /* Push remaining network devices to init_net */
Pavel Emelyanovaca51392008-05-08 01:24:25 -07007971 snprintf(fb_name, IFNAMSIZ, "dev%d", dev->ifindex);
7972 err = dev_change_net_namespace(dev, &init_net, fb_name);
Eric W. Biedermance286d32007-09-12 13:53:49 +02007973 if (err) {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00007974 pr_emerg("%s: failed to move %s to init_net: %d\n",
7975 __func__, dev->name, err);
Pavel Emelyanovaca51392008-05-08 01:24:25 -07007976 BUG();
Eric W. Biedermance286d32007-09-12 13:53:49 +02007977 }
7978 }
7979 rtnl_unlock();
7980}
7981
Eric W. Biederman50624c92013-09-23 21:19:49 -07007982static void __net_exit rtnl_lock_unregistering(struct list_head *net_list)
7983{
7984 /* Return with the rtnl_lock held when there are no network
7985 * devices unregistering in any network namespace in net_list.
7986 */
7987 struct net *net;
7988 bool unregistering;
Peter Zijlstraff960a72014-10-29 17:04:56 +01007989 DEFINE_WAIT_FUNC(wait, woken_wake_function);
Eric W. Biederman50624c92013-09-23 21:19:49 -07007990
Peter Zijlstraff960a72014-10-29 17:04:56 +01007991 add_wait_queue(&netdev_unregistering_wq, &wait);
Eric W. Biederman50624c92013-09-23 21:19:49 -07007992 for (;;) {
Eric W. Biederman50624c92013-09-23 21:19:49 -07007993 unregistering = false;
7994 rtnl_lock();
7995 list_for_each_entry(net, net_list, exit_list) {
7996 if (net->dev_unreg_count > 0) {
7997 unregistering = true;
7998 break;
7999 }
8000 }
8001 if (!unregistering)
8002 break;
8003 __rtnl_unlock();
Peter Zijlstraff960a72014-10-29 17:04:56 +01008004
8005 wait_woken(&wait, TASK_UNINTERRUPTIBLE, MAX_SCHEDULE_TIMEOUT);
Eric W. Biederman50624c92013-09-23 21:19:49 -07008006 }
Peter Zijlstraff960a72014-10-29 17:04:56 +01008007 remove_wait_queue(&netdev_unregistering_wq, &wait);
Eric W. Biederman50624c92013-09-23 21:19:49 -07008008}
8009
Eric W. Biederman04dc7f6b2009-12-03 02:29:04 +00008010static void __net_exit default_device_exit_batch(struct list_head *net_list)
8011{
8012 /* At exit all network devices most be removed from a network
Uwe Kleine-Königb5950762010-11-01 15:38:34 -04008013 * namespace. Do this in the reverse order of registration.
Eric W. Biederman04dc7f6b2009-12-03 02:29:04 +00008014 * Do this across as many network namespaces as possible to
8015 * improve batching efficiency.
8016 */
8017 struct net_device *dev;
8018 struct net *net;
8019 LIST_HEAD(dev_kill_list);
8020
Eric W. Biederman50624c92013-09-23 21:19:49 -07008021 /* To prevent network device cleanup code from dereferencing
8022 * loopback devices or network devices that have been freed
8023 * wait here for all pending unregistrations to complete,
8024 * before unregistring the loopback device and allowing the
8025 * network namespace be freed.
8026 *
8027 * The netdev todo list containing all network devices
8028 * unregistrations that happen in default_device_exit_batch
8029 * will run in the rtnl_unlock() at the end of
8030 * default_device_exit_batch.
8031 */
8032 rtnl_lock_unregistering(net_list);
Eric W. Biederman04dc7f6b2009-12-03 02:29:04 +00008033 list_for_each_entry(net, net_list, exit_list) {
8034 for_each_netdev_reverse(net, dev) {
Jiri Pirkob0ab2fa2014-06-26 09:58:25 +02008035 if (dev->rtnl_link_ops && dev->rtnl_link_ops->dellink)
Eric W. Biederman04dc7f6b2009-12-03 02:29:04 +00008036 dev->rtnl_link_ops->dellink(dev, &dev_kill_list);
8037 else
8038 unregister_netdevice_queue(dev, &dev_kill_list);
8039 }
8040 }
8041 unregister_netdevice_many(&dev_kill_list);
8042 rtnl_unlock();
8043}
8044
Denis V. Lunev022cbae2007-11-13 03:23:50 -08008045static struct pernet_operations __net_initdata default_device_ops = {
Eric W. Biedermance286d32007-09-12 13:53:49 +02008046 .exit = default_device_exit,
Eric W. Biederman04dc7f6b2009-12-03 02:29:04 +00008047 .exit_batch = default_device_exit_batch,
Eric W. Biedermance286d32007-09-12 13:53:49 +02008048};
8049
Linus Torvalds1da177e2005-04-16 15:20:36 -07008050/*
8051 * Initialize the DEV module. At boot time this walks the device list and
8052 * unhooks any devices that fail to initialise (normally hardware not
8053 * present) and leaves us with a valid list of present and active devices.
8054 *
8055 */
8056
8057/*
8058 * This is called single threaded during boot, so no need
8059 * to take the rtnl semaphore.
8060 */
8061static int __init net_dev_init(void)
8062{
8063 int i, rc = -ENOMEM;
8064
8065 BUG_ON(!dev_boot_phase);
8066
Linus Torvalds1da177e2005-04-16 15:20:36 -07008067 if (dev_proc_init())
8068 goto out;
8069
Eric W. Biederman8b41d182007-09-26 22:02:53 -07008070 if (netdev_kobject_init())
Linus Torvalds1da177e2005-04-16 15:20:36 -07008071 goto out;
8072
8073 INIT_LIST_HEAD(&ptype_all);
Pavel Emelyanov82d8a8672007-11-26 20:12:58 +08008074 for (i = 0; i < PTYPE_HASH_SIZE; i++)
Linus Torvalds1da177e2005-04-16 15:20:36 -07008075 INIT_LIST_HEAD(&ptype_base[i]);
8076
Vlad Yasevich62532da2012-11-15 08:49:10 +00008077 INIT_LIST_HEAD(&offload_base);
8078
Eric W. Biederman881d9662007-09-17 11:56:21 -07008079 if (register_pernet_subsys(&netdev_net_ops))
8080 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07008081
8082 /*
8083 * Initialise the packet receive queues.
8084 */
8085
KAMEZAWA Hiroyuki6f912042006-04-10 22:52:50 -07008086 for_each_possible_cpu(i) {
Eric Dumazete36fa2f2010-04-19 21:17:14 +00008087 struct softnet_data *sd = &per_cpu(softnet_data, i);
Linus Torvalds1da177e2005-04-16 15:20:36 -07008088
Eric Dumazete36fa2f2010-04-19 21:17:14 +00008089 skb_queue_head_init(&sd->input_pkt_queue);
Changli Gao6e7676c2010-04-27 15:07:33 -07008090 skb_queue_head_init(&sd->process_queue);
Eric Dumazete36fa2f2010-04-19 21:17:14 +00008091 INIT_LIST_HEAD(&sd->poll_list);
Changli Gaoa9cbd582010-04-26 23:06:24 +00008092 sd->output_queue_tailp = &sd->output_queue;
Eric Dumazetdf334542010-03-24 19:13:54 +00008093#ifdef CONFIG_RPS
Eric Dumazete36fa2f2010-04-19 21:17:14 +00008094 sd->csd.func = rps_trigger_softirq;
8095 sd->csd.info = sd;
Eric Dumazete36fa2f2010-04-19 21:17:14 +00008096 sd->cpu = i;
Tom Herbert1e94d722010-03-18 17:45:44 -07008097#endif
Tom Herbert0a9627f2010-03-16 08:03:29 +00008098
Eric Dumazete36fa2f2010-04-19 21:17:14 +00008099 sd->backlog.poll = process_backlog;
8100 sd->backlog.weight = weight_p;
Linus Torvalds1da177e2005-04-16 15:20:36 -07008101 }
8102
Linus Torvalds1da177e2005-04-16 15:20:36 -07008103 dev_boot_phase = 0;
8104
Eric W. Biederman505d4f72008-11-07 22:54:20 -08008105 /* The loopback device is special if any other network devices
8106 * is present in a network namespace the loopback device must
8107 * be present. Since we now dynamically allocate and free the
8108 * loopback device ensure this invariant is maintained by
8109 * keeping the loopback device as the first device on the
8110 * list of network devices. Ensuring the loopback devices
8111 * is the first device that appears and the last network device
8112 * that disappears.
8113 */
8114 if (register_pernet_device(&loopback_net_ops))
8115 goto out;
8116
8117 if (register_pernet_device(&default_device_ops))
8118 goto out;
8119
Carlos R. Mafra962cf362008-05-15 11:15:37 -03008120 open_softirq(NET_TX_SOFTIRQ, net_tx_action);
8121 open_softirq(NET_RX_SOFTIRQ, net_rx_action);
Linus Torvalds1da177e2005-04-16 15:20:36 -07008122
8123 hotcpu_notifier(dev_cpu_callback, 0);
Thomas Graff38a9eb2015-07-21 10:43:56 +02008124 dst_subsys_init();
Linus Torvalds1da177e2005-04-16 15:20:36 -07008125 rc = 0;
8126out:
8127 return rc;
8128}
8129
8130subsys_initcall(net_dev_init);