blob: 2582c24a75c6931777b50d8c8287411de9922209 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * NET3 Protocol independent device support routines.
3 *
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public License
6 * as published by the Free Software Foundation; either version
7 * 2 of the License, or (at your option) any later version.
8 *
9 * Derived from the non IP parts of dev.c 1.0.19
Jesper Juhl02c30a82005-05-05 16:16:16 -070010 * Authors: Ross Biro
Linus Torvalds1da177e2005-04-16 15:20:36 -070011 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
12 * Mark Evans, <evansmp@uhura.aston.ac.uk>
13 *
14 * Additional Authors:
15 * Florian la Roche <rzsfl@rz.uni-sb.de>
16 * Alan Cox <gw4pts@gw4pts.ampr.org>
17 * David Hinds <dahinds@users.sourceforge.net>
18 * Alexey Kuznetsov <kuznet@ms2.inr.ac.ru>
19 * Adam Sulmicki <adam@cfar.umd.edu>
20 * Pekka Riikonen <priikone@poesidon.pspt.fi>
21 *
22 * Changes:
23 * D.J. Barrow : Fixed bug where dev->refcnt gets set
24 * to 2 if register_netdev gets called
25 * before net_dev_init & also removed a
26 * few lines of code in the process.
27 * Alan Cox : device private ioctl copies fields back.
28 * Alan Cox : Transmit queue code does relevant
29 * stunts to keep the queue safe.
30 * Alan Cox : Fixed double lock.
31 * Alan Cox : Fixed promisc NULL pointer trap
32 * ???????? : Support the full private ioctl range
33 * Alan Cox : Moved ioctl permission check into
34 * drivers
35 * Tim Kordas : SIOCADDMULTI/SIOCDELMULTI
36 * Alan Cox : 100 backlog just doesn't cut it when
37 * you start doing multicast video 8)
38 * Alan Cox : Rewrote net_bh and list manager.
39 * Alan Cox : Fix ETH_P_ALL echoback lengths.
40 * Alan Cox : Took out transmit every packet pass
41 * Saved a few bytes in the ioctl handler
42 * Alan Cox : Network driver sets packet type before
43 * calling netif_rx. Saves a function
44 * call a packet.
45 * Alan Cox : Hashed net_bh()
46 * Richard Kooijman: Timestamp fixes.
47 * Alan Cox : Wrong field in SIOCGIFDSTADDR
48 * Alan Cox : Device lock protection.
49 * Alan Cox : Fixed nasty side effect of device close
50 * changes.
51 * Rudi Cilibrasi : Pass the right thing to
52 * set_mac_address()
53 * Dave Miller : 32bit quantity for the device lock to
54 * make it work out on a Sparc.
55 * Bjorn Ekwall : Added KERNELD hack.
56 * Alan Cox : Cleaned up the backlog initialise.
57 * Craig Metz : SIOCGIFCONF fix if space for under
58 * 1 device.
59 * Thomas Bogendoerfer : Return ENODEV for dev_open, if there
60 * is no device open function.
61 * Andi Kleen : Fix error reporting for SIOCGIFCONF
62 * Michael Chastain : Fix signed/unsigned for SIOCGIFCONF
63 * Cyrus Durgin : Cleaned for KMOD
64 * Adam Sulmicki : Bug Fix : Network Device Unload
65 * A network device unload needs to purge
66 * the backlog queue.
67 * Paul Rusty Russell : SIOCSIFNAME
68 * Pekka Riikonen : Netdev boot-time settings code
69 * Andrew Morton : Make unregister_netdevice wait
70 * indefinitely on dev->refcnt
71 * J Hadi Salim : - Backlog queue sampling
72 * - netif_rx() feedback
73 */
74
75#include <asm/uaccess.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070076#include <linux/bitops.h>
Randy Dunlap4fc268d2006-01-11 12:17:47 -080077#include <linux/capability.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070078#include <linux/cpu.h>
79#include <linux/types.h>
80#include <linux/kernel.h>
stephen hemminger08e98972009-11-10 07:20:34 +000081#include <linux/hash.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090082#include <linux/slab.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070083#include <linux/sched.h>
Arjan van de Ven4a3e2f72006-03-20 22:33:17 -080084#include <linux/mutex.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070085#include <linux/string.h>
86#include <linux/mm.h>
87#include <linux/socket.h>
88#include <linux/sockios.h>
89#include <linux/errno.h>
90#include <linux/interrupt.h>
91#include <linux/if_ether.h>
92#include <linux/netdevice.h>
93#include <linux/etherdevice.h>
Ben Hutchings0187bdf2008-06-19 16:15:47 -070094#include <linux/ethtool.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070095#include <linux/notifier.h>
96#include <linux/skbuff.h>
Eric W. Biederman457c4cb2007-09-12 12:01:34 +020097#include <net/net_namespace.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070098#include <net/sock.h>
99#include <linux/rtnetlink.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700100#include <linux/stat.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700101#include <net/dst.h>
Pravin B Shelarfc4099f2015-10-22 18:17:16 -0700102#include <net/dst_metadata.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700103#include <net/pkt_sched.h>
104#include <net/checksum.h>
Arnd Bergmann44540962009-11-26 06:07:08 +0000105#include <net/xfrm.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700106#include <linux/highmem.h>
107#include <linux/init.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700108#include <linux/module.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700109#include <linux/netpoll.h>
110#include <linux/rcupdate.h>
111#include <linux/delay.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700112#include <net/iw_handler.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700113#include <asm/current.h>
Steve Grubb5bdb9882005-12-03 08:39:35 -0500114#include <linux/audit.h>
Chris Leechdb217332006-06-17 21:24:58 -0700115#include <linux/dmaengine.h>
Herbert Xuf6a78bf2006-06-22 02:57:17 -0700116#include <linux/err.h>
David S. Millerc7fa9d12006-08-15 16:34:13 -0700117#include <linux/ctype.h>
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700118#include <linux/if_arp.h>
Ben Hutchings6de329e2008-06-16 17:02:28 -0700119#include <linux/if_vlan.h>
David S. Miller8f0f2222008-07-15 03:47:03 -0700120#include <linux/ip.h>
Alexander Duyckad55dca2008-09-20 22:05:50 -0700121#include <net/ip.h>
Simon Horman25cd9ba2014-10-06 05:05:13 -0700122#include <net/mpls.h>
David S. Miller8f0f2222008-07-15 03:47:03 -0700123#include <linux/ipv6.h>
124#include <linux/in.h>
David S. Millerb6b2fed2008-07-21 09:48:06 -0700125#include <linux/jhash.h>
126#include <linux/random.h>
David S. Miller9cbc1cb2009-06-15 03:02:23 -0700127#include <trace/events/napi.h>
Koki Sanagicf66ba52010-08-23 18:45:02 +0900128#include <trace/events/net.h>
Koki Sanagi07dc22e2010-08-23 18:46:12 +0900129#include <trace/events/skb.h>
FUJITA Tomonori5acbbd42010-03-30 22:35:50 +0000130#include <linux/pci.h>
Stephen Rothwellcaeda9b2010-09-16 21:39:16 -0700131#include <linux/inetdevice.h>
Ben Hutchingsc4454772011-01-19 11:03:53 +0000132#include <linux/cpu_rmap.h>
Ingo Molnarc5905af2012-02-24 08:31:31 +0100133#include <linux/static_key.h>
Eliezer Tamiraf12fa62013-06-10 11:39:41 +0300134#include <linux/hashtable.h>
Eric Dumazet60877a32013-06-20 01:15:51 -0700135#include <linux/vmalloc.h>
Michal Kubeček529d0482013-11-15 06:18:50 +0100136#include <linux/if_macvlan.h>
Willem de Bruijne7fd2882014-08-04 22:11:48 -0400137#include <linux/errqueue.h>
Eric Dumazet3b47d302014-11-06 21:09:44 -0800138#include <linux/hrtimer.h>
Pablo Neirae687ad62015-05-13 18:19:38 +0200139#include <linux/netfilter_ingress.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700140
Pavel Emelyanov342709e2007-10-23 21:14:45 -0700141#include "net-sysfs.h"
142
Herbert Xud565b0a2008-12-15 23:38:52 -0800143/* Instead of increasing this, you should create a hash table. */
144#define MAX_GRO_SKBS 8
145
Herbert Xu5d38a072009-01-04 16:13:40 -0800146/* This should be increased if a protocol with a bigger head is added. */
147#define GRO_MAX_HEAD (MAX_HEADER + 128)
148
Linus Torvalds1da177e2005-04-16 15:20:36 -0700149static DEFINE_SPINLOCK(ptype_lock);
Vlad Yasevich62532da2012-11-15 08:49:10 +0000150static DEFINE_SPINLOCK(offload_lock);
Cong Wang900ff8c2013-02-18 19:20:33 +0000151struct list_head ptype_base[PTYPE_HASH_SIZE] __read_mostly;
152struct list_head ptype_all __read_mostly; /* Taps */
Vlad Yasevich62532da2012-11-15 08:49:10 +0000153static struct list_head offload_base __read_mostly;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700154
Ben Hutchingsae78dbf2014-01-10 22:17:24 +0000155static int netif_rx_internal(struct sk_buff *skb);
Loic Prylli54951192014-07-01 21:39:43 -0700156static int call_netdevice_notifiers_info(unsigned long val,
157 struct net_device *dev,
158 struct netdev_notifier_info *info);
Ben Hutchingsae78dbf2014-01-10 22:17:24 +0000159
Linus Torvalds1da177e2005-04-16 15:20:36 -0700160/*
Pavel Emelianov7562f872007-05-03 15:13:45 -0700161 * The @dev_base_head list is protected by @dev_base_lock and the rtnl
Linus Torvalds1da177e2005-04-16 15:20:36 -0700162 * semaphore.
163 *
Eric Dumazetc6d14c82009-11-04 05:43:23 -0800164 * Pure readers hold dev_base_lock for reading, or rcu_read_lock()
Linus Torvalds1da177e2005-04-16 15:20:36 -0700165 *
166 * Writers must hold the rtnl semaphore while they loop through the
Pavel Emelianov7562f872007-05-03 15:13:45 -0700167 * dev_base_head list, and hold dev_base_lock for writing when they do the
Linus Torvalds1da177e2005-04-16 15:20:36 -0700168 * actual updates. This allows pure readers to access the list even
169 * while a writer is preparing to update it.
170 *
171 * To put it another way, dev_base_lock is held for writing only to
172 * protect against pure readers; the rtnl semaphore provides the
173 * protection against other writers.
174 *
175 * See, for example usages, register_netdevice() and
176 * unregister_netdevice(), which must be called with the rtnl
177 * semaphore held.
178 */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700179DEFINE_RWLOCK(dev_base_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700180EXPORT_SYMBOL(dev_base_lock);
181
Eliezer Tamiraf12fa62013-06-10 11:39:41 +0300182/* protects napi_hash addition/deletion and napi_gen_id */
183static DEFINE_SPINLOCK(napi_hash_lock);
184
Eric Dumazet52bd2d62015-11-18 06:30:50 -0800185static unsigned int napi_gen_id = NR_CPUS;
Eliezer Tamiraf12fa62013-06-10 11:39:41 +0300186static DEFINE_HASHTABLE(napi_hash, 8);
187
Thomas Gleixner18afa4b2013-07-23 16:13:17 +0200188static seqcount_t devnet_rename_seq;
Brian Haleyc91f6df2012-11-26 05:21:08 +0000189
Thomas Graf4e985ad2011-06-21 03:11:20 +0000190static inline void dev_base_seq_inc(struct net *net)
191{
192 while (++net->dev_base_seq == 0);
193}
194
Eric W. Biederman881d9662007-09-17 11:56:21 -0700195static inline struct hlist_head *dev_name_hash(struct net *net, const char *name)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700196{
Eric Dumazet95c96172012-04-15 05:58:06 +0000197 unsigned int hash = full_name_hash(name, strnlen(name, IFNAMSIZ));
198
stephen hemminger08e98972009-11-10 07:20:34 +0000199 return &net->dev_name_head[hash_32(hash, NETDEV_HASHBITS)];
Linus Torvalds1da177e2005-04-16 15:20:36 -0700200}
201
Eric W. Biederman881d9662007-09-17 11:56:21 -0700202static inline struct hlist_head *dev_index_hash(struct net *net, int ifindex)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700203{
Eric Dumazet7c28bd02009-10-24 06:13:17 -0700204 return &net->dev_index_head[ifindex & (NETDEV_HASHENTRIES - 1)];
Linus Torvalds1da177e2005-04-16 15:20:36 -0700205}
206
Eric Dumazete36fa2f2010-04-19 21:17:14 +0000207static inline void rps_lock(struct softnet_data *sd)
Changli Gao152102c2010-03-30 20:16:22 +0000208{
209#ifdef CONFIG_RPS
Eric Dumazete36fa2f2010-04-19 21:17:14 +0000210 spin_lock(&sd->input_pkt_queue.lock);
Changli Gao152102c2010-03-30 20:16:22 +0000211#endif
212}
213
Eric Dumazete36fa2f2010-04-19 21:17:14 +0000214static inline void rps_unlock(struct softnet_data *sd)
Changli Gao152102c2010-03-30 20:16:22 +0000215{
216#ifdef CONFIG_RPS
Eric Dumazete36fa2f2010-04-19 21:17:14 +0000217 spin_unlock(&sd->input_pkt_queue.lock);
Changli Gao152102c2010-03-30 20:16:22 +0000218#endif
219}
220
Eric W. Biedermance286d32007-09-12 13:53:49 +0200221/* Device list insertion */
dingtianhong53759be2013-04-17 22:17:50 +0000222static void list_netdevice(struct net_device *dev)
Eric W. Biedermance286d32007-09-12 13:53:49 +0200223{
YOSHIFUJI Hideakic346dca2008-03-25 21:47:49 +0900224 struct net *net = dev_net(dev);
Eric W. Biedermance286d32007-09-12 13:53:49 +0200225
226 ASSERT_RTNL();
227
228 write_lock_bh(&dev_base_lock);
Eric Dumazetc6d14c82009-11-04 05:43:23 -0800229 list_add_tail_rcu(&dev->dev_list, &net->dev_base_head);
Eric Dumazet72c95282009-10-30 07:11:27 +0000230 hlist_add_head_rcu(&dev->name_hlist, dev_name_hash(net, dev->name));
Eric Dumazetfb699dfd2009-10-19 19:18:49 +0000231 hlist_add_head_rcu(&dev->index_hlist,
232 dev_index_hash(net, dev->ifindex));
Eric W. Biedermance286d32007-09-12 13:53:49 +0200233 write_unlock_bh(&dev_base_lock);
Thomas Graf4e985ad2011-06-21 03:11:20 +0000234
235 dev_base_seq_inc(net);
Eric W. Biedermance286d32007-09-12 13:53:49 +0200236}
237
Eric Dumazetfb699dfd2009-10-19 19:18:49 +0000238/* Device list removal
239 * caller must respect a RCU grace period before freeing/reusing dev
240 */
Eric W. Biedermance286d32007-09-12 13:53:49 +0200241static void unlist_netdevice(struct net_device *dev)
242{
243 ASSERT_RTNL();
244
245 /* Unlink dev from the device chain */
246 write_lock_bh(&dev_base_lock);
Eric Dumazetc6d14c82009-11-04 05:43:23 -0800247 list_del_rcu(&dev->dev_list);
Eric Dumazet72c95282009-10-30 07:11:27 +0000248 hlist_del_rcu(&dev->name_hlist);
Eric Dumazetfb699dfd2009-10-19 19:18:49 +0000249 hlist_del_rcu(&dev->index_hlist);
Eric W. Biedermance286d32007-09-12 13:53:49 +0200250 write_unlock_bh(&dev_base_lock);
Thomas Graf4e985ad2011-06-21 03:11:20 +0000251
252 dev_base_seq_inc(dev_net(dev));
Eric W. Biedermance286d32007-09-12 13:53:49 +0200253}
254
Linus Torvalds1da177e2005-04-16 15:20:36 -0700255/*
256 * Our notifier list
257 */
258
Alan Sternf07d5b92006-05-09 15:23:03 -0700259static RAW_NOTIFIER_HEAD(netdev_chain);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700260
261/*
262 * Device drivers call our routines to queue packets here. We empty the
263 * queue in the local softnet handler.
264 */
Stephen Hemmingerbea33482007-10-03 16:41:36 -0700265
Eric Dumazet9958da02010-04-17 04:17:02 +0000266DEFINE_PER_CPU_ALIGNED(struct softnet_data, softnet_data);
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700267EXPORT_PER_CPU_SYMBOL(softnet_data);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700268
David S. Millercf508b12008-07-22 14:16:42 -0700269#ifdef CONFIG_LOCKDEP
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700270/*
David S. Millerc773e842008-07-08 23:13:53 -0700271 * register_netdevice() inits txq->_xmit_lock and sets lockdep class
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700272 * according to dev->type
273 */
274static const unsigned short netdev_lock_type[] =
275 {ARPHRD_NETROM, ARPHRD_ETHER, ARPHRD_EETHER, ARPHRD_AX25,
276 ARPHRD_PRONET, ARPHRD_CHAOS, ARPHRD_IEEE802, ARPHRD_ARCNET,
277 ARPHRD_APPLETLK, ARPHRD_DLCI, ARPHRD_ATM, ARPHRD_METRICOM,
278 ARPHRD_IEEE1394, ARPHRD_EUI64, ARPHRD_INFINIBAND, ARPHRD_SLIP,
279 ARPHRD_CSLIP, ARPHRD_SLIP6, ARPHRD_CSLIP6, ARPHRD_RSRVD,
280 ARPHRD_ADAPT, ARPHRD_ROSE, ARPHRD_X25, ARPHRD_HWX25,
281 ARPHRD_PPP, ARPHRD_CISCO, ARPHRD_LAPB, ARPHRD_DDCMP,
282 ARPHRD_RAWHDLC, ARPHRD_TUNNEL, ARPHRD_TUNNEL6, ARPHRD_FRAD,
283 ARPHRD_SKIP, ARPHRD_LOOPBACK, ARPHRD_LOCALTLK, ARPHRD_FDDI,
284 ARPHRD_BIF, ARPHRD_SIT, ARPHRD_IPDDP, ARPHRD_IPGRE,
285 ARPHRD_PIMREG, ARPHRD_HIPPI, ARPHRD_ASH, ARPHRD_ECONET,
286 ARPHRD_IRDA, ARPHRD_FCPP, ARPHRD_FCAL, ARPHRD_FCPL,
Paul Gortmaker211ed862012-05-10 17:14:35 -0400287 ARPHRD_FCFABRIC, ARPHRD_IEEE80211, ARPHRD_IEEE80211_PRISM,
288 ARPHRD_IEEE80211_RADIOTAP, ARPHRD_PHONET, ARPHRD_PHONET_PIPE,
289 ARPHRD_IEEE802154, ARPHRD_VOID, ARPHRD_NONE};
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700290
Jan Engelhardt36cbd3d2009-08-05 10:42:58 -0700291static const char *const netdev_lock_name[] =
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700292 {"_xmit_NETROM", "_xmit_ETHER", "_xmit_EETHER", "_xmit_AX25",
293 "_xmit_PRONET", "_xmit_CHAOS", "_xmit_IEEE802", "_xmit_ARCNET",
294 "_xmit_APPLETLK", "_xmit_DLCI", "_xmit_ATM", "_xmit_METRICOM",
295 "_xmit_IEEE1394", "_xmit_EUI64", "_xmit_INFINIBAND", "_xmit_SLIP",
296 "_xmit_CSLIP", "_xmit_SLIP6", "_xmit_CSLIP6", "_xmit_RSRVD",
297 "_xmit_ADAPT", "_xmit_ROSE", "_xmit_X25", "_xmit_HWX25",
298 "_xmit_PPP", "_xmit_CISCO", "_xmit_LAPB", "_xmit_DDCMP",
299 "_xmit_RAWHDLC", "_xmit_TUNNEL", "_xmit_TUNNEL6", "_xmit_FRAD",
300 "_xmit_SKIP", "_xmit_LOOPBACK", "_xmit_LOCALTLK", "_xmit_FDDI",
301 "_xmit_BIF", "_xmit_SIT", "_xmit_IPDDP", "_xmit_IPGRE",
302 "_xmit_PIMREG", "_xmit_HIPPI", "_xmit_ASH", "_xmit_ECONET",
303 "_xmit_IRDA", "_xmit_FCPP", "_xmit_FCAL", "_xmit_FCPL",
Paul Gortmaker211ed862012-05-10 17:14:35 -0400304 "_xmit_FCFABRIC", "_xmit_IEEE80211", "_xmit_IEEE80211_PRISM",
305 "_xmit_IEEE80211_RADIOTAP", "_xmit_PHONET", "_xmit_PHONET_PIPE",
306 "_xmit_IEEE802154", "_xmit_VOID", "_xmit_NONE"};
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700307
308static struct lock_class_key netdev_xmit_lock_key[ARRAY_SIZE(netdev_lock_type)];
David S. Millercf508b12008-07-22 14:16:42 -0700309static struct lock_class_key netdev_addr_lock_key[ARRAY_SIZE(netdev_lock_type)];
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700310
311static inline unsigned short netdev_lock_pos(unsigned short dev_type)
312{
313 int i;
314
315 for (i = 0; i < ARRAY_SIZE(netdev_lock_type); i++)
316 if (netdev_lock_type[i] == dev_type)
317 return i;
318 /* the last key is used by default */
319 return ARRAY_SIZE(netdev_lock_type) - 1;
320}
321
David S. Millercf508b12008-07-22 14:16:42 -0700322static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock,
323 unsigned short dev_type)
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700324{
325 int i;
326
327 i = netdev_lock_pos(dev_type);
328 lockdep_set_class_and_name(lock, &netdev_xmit_lock_key[i],
329 netdev_lock_name[i]);
330}
David S. Millercf508b12008-07-22 14:16:42 -0700331
332static inline void netdev_set_addr_lockdep_class(struct net_device *dev)
333{
334 int i;
335
336 i = netdev_lock_pos(dev->type);
337 lockdep_set_class_and_name(&dev->addr_list_lock,
338 &netdev_addr_lock_key[i],
339 netdev_lock_name[i]);
340}
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700341#else
David S. Millercf508b12008-07-22 14:16:42 -0700342static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock,
343 unsigned short dev_type)
344{
345}
346static inline void netdev_set_addr_lockdep_class(struct net_device *dev)
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700347{
348}
349#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -0700350
351/*******************************************************************************
352
353 Protocol management and registration routines
354
355*******************************************************************************/
356
357/*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700358 * Add a protocol ID to the list. Now that the input handler is
359 * smarter we can dispense with all the messy stuff that used to be
360 * here.
361 *
362 * BEWARE!!! Protocol handlers, mangling input packets,
363 * MUST BE last in hash buckets and checking protocol handlers
364 * MUST start from promiscuous ptype_all chain in net_bh.
365 * It is true now, do not change it.
366 * Explanation follows: if protocol handler, mangling packet, will
367 * be the first on list, it is not able to sense, that packet
368 * is cloned and should be copied-on-write, so that it will
369 * change it and subsequent readers will get broken packet.
370 * --ANK (980803)
371 */
372
Eric Dumazetc07b68e2010-09-02 03:53:46 +0000373static inline struct list_head *ptype_head(const struct packet_type *pt)
374{
375 if (pt->type == htons(ETH_P_ALL))
Salam Noureddine7866a622015-01-27 11:35:48 -0800376 return pt->dev ? &pt->dev->ptype_all : &ptype_all;
Eric Dumazetc07b68e2010-09-02 03:53:46 +0000377 else
Salam Noureddine7866a622015-01-27 11:35:48 -0800378 return pt->dev ? &pt->dev->ptype_specific :
379 &ptype_base[ntohs(pt->type) & PTYPE_HASH_MASK];
Eric Dumazetc07b68e2010-09-02 03:53:46 +0000380}
381
Linus Torvalds1da177e2005-04-16 15:20:36 -0700382/**
383 * dev_add_pack - add packet handler
384 * @pt: packet type declaration
385 *
386 * Add a protocol handler to the networking stack. The passed &packet_type
387 * is linked into kernel lists and may not be freed until it has been
388 * removed from the kernel lists.
389 *
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +0900390 * This call does not sleep therefore it can not
Linus Torvalds1da177e2005-04-16 15:20:36 -0700391 * guarantee all CPU's that are in middle of receiving packets
392 * will see the new packet type (until the next received packet).
393 */
394
395void dev_add_pack(struct packet_type *pt)
396{
Eric Dumazetc07b68e2010-09-02 03:53:46 +0000397 struct list_head *head = ptype_head(pt);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700398
Eric Dumazetc07b68e2010-09-02 03:53:46 +0000399 spin_lock(&ptype_lock);
400 list_add_rcu(&pt->list, head);
401 spin_unlock(&ptype_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700402}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700403EXPORT_SYMBOL(dev_add_pack);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700404
Linus Torvalds1da177e2005-04-16 15:20:36 -0700405/**
406 * __dev_remove_pack - remove packet handler
407 * @pt: packet type declaration
408 *
409 * Remove a protocol handler that was previously added to the kernel
410 * protocol handlers by dev_add_pack(). The passed &packet_type is removed
411 * from the kernel lists and can be freed or reused once this function
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +0900412 * returns.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700413 *
414 * The packet type might still be in use by receivers
415 * and must not be freed until after all the CPU's have gone
416 * through a quiescent state.
417 */
418void __dev_remove_pack(struct packet_type *pt)
419{
Eric Dumazetc07b68e2010-09-02 03:53:46 +0000420 struct list_head *head = ptype_head(pt);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700421 struct packet_type *pt1;
422
Eric Dumazetc07b68e2010-09-02 03:53:46 +0000423 spin_lock(&ptype_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700424
425 list_for_each_entry(pt1, head, list) {
426 if (pt == pt1) {
427 list_del_rcu(&pt->list);
428 goto out;
429 }
430 }
431
Joe Perches7b6cd1c2012-02-01 10:54:43 +0000432 pr_warn("dev_remove_pack: %p not found\n", pt);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700433out:
Eric Dumazetc07b68e2010-09-02 03:53:46 +0000434 spin_unlock(&ptype_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700435}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700436EXPORT_SYMBOL(__dev_remove_pack);
437
Linus Torvalds1da177e2005-04-16 15:20:36 -0700438/**
439 * dev_remove_pack - remove packet handler
440 * @pt: packet type declaration
441 *
442 * Remove a protocol handler that was previously added to the kernel
443 * protocol handlers by dev_add_pack(). The passed &packet_type is removed
444 * from the kernel lists and can be freed or reused once this function
445 * returns.
446 *
447 * This call sleeps to guarantee that no CPU is looking at the packet
448 * type after return.
449 */
450void dev_remove_pack(struct packet_type *pt)
451{
452 __dev_remove_pack(pt);
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +0900453
Linus Torvalds1da177e2005-04-16 15:20:36 -0700454 synchronize_net();
455}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700456EXPORT_SYMBOL(dev_remove_pack);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700457
Vlad Yasevich62532da2012-11-15 08:49:10 +0000458
459/**
460 * dev_add_offload - register offload handlers
461 * @po: protocol offload declaration
462 *
463 * Add protocol offload handlers to the networking stack. The passed
464 * &proto_offload is linked into kernel lists and may not be freed until
465 * it has been removed from the kernel lists.
466 *
467 * This call does not sleep therefore it can not
468 * guarantee all CPU's that are in middle of receiving packets
469 * will see the new offload handlers (until the next received packet).
470 */
471void dev_add_offload(struct packet_offload *po)
472{
David S. Millerbdef7de2015-06-01 14:56:09 -0700473 struct packet_offload *elem;
Vlad Yasevich62532da2012-11-15 08:49:10 +0000474
475 spin_lock(&offload_lock);
David S. Millerbdef7de2015-06-01 14:56:09 -0700476 list_for_each_entry(elem, &offload_base, list) {
477 if (po->priority < elem->priority)
478 break;
479 }
480 list_add_rcu(&po->list, elem->list.prev);
Vlad Yasevich62532da2012-11-15 08:49:10 +0000481 spin_unlock(&offload_lock);
482}
483EXPORT_SYMBOL(dev_add_offload);
484
485/**
486 * __dev_remove_offload - remove offload handler
487 * @po: packet offload declaration
488 *
489 * Remove a protocol offload handler that was previously added to the
490 * kernel offload handlers by dev_add_offload(). The passed &offload_type
491 * is removed from the kernel lists and can be freed or reused once this
492 * function returns.
493 *
494 * The packet type might still be in use by receivers
495 * and must not be freed until after all the CPU's have gone
496 * through a quiescent state.
497 */
stephen hemminger1d143d92013-12-29 14:01:29 -0800498static void __dev_remove_offload(struct packet_offload *po)
Vlad Yasevich62532da2012-11-15 08:49:10 +0000499{
500 struct list_head *head = &offload_base;
501 struct packet_offload *po1;
502
Eric Dumazetc53aa502012-11-16 08:08:23 +0000503 spin_lock(&offload_lock);
Vlad Yasevich62532da2012-11-15 08:49:10 +0000504
505 list_for_each_entry(po1, head, list) {
506 if (po == po1) {
507 list_del_rcu(&po->list);
508 goto out;
509 }
510 }
511
512 pr_warn("dev_remove_offload: %p not found\n", po);
513out:
Eric Dumazetc53aa502012-11-16 08:08:23 +0000514 spin_unlock(&offload_lock);
Vlad Yasevich62532da2012-11-15 08:49:10 +0000515}
Vlad Yasevich62532da2012-11-15 08:49:10 +0000516
517/**
518 * dev_remove_offload - remove packet offload handler
519 * @po: packet offload declaration
520 *
521 * Remove a packet offload handler that was previously added to the kernel
522 * offload handlers by dev_add_offload(). The passed &offload_type is
523 * removed from the kernel lists and can be freed or reused once this
524 * function returns.
525 *
526 * This call sleeps to guarantee that no CPU is looking at the packet
527 * type after return.
528 */
529void dev_remove_offload(struct packet_offload *po)
530{
531 __dev_remove_offload(po);
532
533 synchronize_net();
534}
535EXPORT_SYMBOL(dev_remove_offload);
536
Linus Torvalds1da177e2005-04-16 15:20:36 -0700537/******************************************************************************
538
539 Device Boot-time Settings Routines
540
541*******************************************************************************/
542
543/* Boot time configuration table */
544static struct netdev_boot_setup dev_boot_setup[NETDEV_BOOT_SETUP_MAX];
545
546/**
547 * netdev_boot_setup_add - add new setup entry
548 * @name: name of the device
549 * @map: configured settings for the device
550 *
551 * Adds new setup entry to the dev_boot_setup list. The function
552 * returns 0 on error and 1 on success. This is a generic routine to
553 * all netdevices.
554 */
555static int netdev_boot_setup_add(char *name, struct ifmap *map)
556{
557 struct netdev_boot_setup *s;
558 int i;
559
560 s = dev_boot_setup;
561 for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++) {
562 if (s[i].name[0] == '\0' || s[i].name[0] == ' ') {
563 memset(s[i].name, 0, sizeof(s[i].name));
Wang Chen93b3cff2008-07-01 19:57:19 -0700564 strlcpy(s[i].name, name, IFNAMSIZ);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700565 memcpy(&s[i].map, map, sizeof(s[i].map));
566 break;
567 }
568 }
569
570 return i >= NETDEV_BOOT_SETUP_MAX ? 0 : 1;
571}
572
573/**
574 * netdev_boot_setup_check - check boot time settings
575 * @dev: the netdevice
576 *
577 * Check boot time settings for the device.
578 * The found settings are set for the device to be used
579 * later in the device probing.
580 * Returns 0 if no settings found, 1 if they are.
581 */
582int netdev_boot_setup_check(struct net_device *dev)
583{
584 struct netdev_boot_setup *s = dev_boot_setup;
585 int i;
586
587 for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++) {
588 if (s[i].name[0] != '\0' && s[i].name[0] != ' ' &&
Wang Chen93b3cff2008-07-01 19:57:19 -0700589 !strcmp(dev->name, s[i].name)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700590 dev->irq = s[i].map.irq;
591 dev->base_addr = s[i].map.base_addr;
592 dev->mem_start = s[i].map.mem_start;
593 dev->mem_end = s[i].map.mem_end;
594 return 1;
595 }
596 }
597 return 0;
598}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700599EXPORT_SYMBOL(netdev_boot_setup_check);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700600
601
602/**
603 * netdev_boot_base - get address from boot time settings
604 * @prefix: prefix for network device
605 * @unit: id for network device
606 *
607 * Check boot time settings for the base address of device.
608 * The found settings are set for the device to be used
609 * later in the device probing.
610 * Returns 0 if no settings found.
611 */
612unsigned long netdev_boot_base(const char *prefix, int unit)
613{
614 const struct netdev_boot_setup *s = dev_boot_setup;
615 char name[IFNAMSIZ];
616 int i;
617
618 sprintf(name, "%s%d", prefix, unit);
619
620 /*
621 * If device already registered then return base of 1
622 * to indicate not to probe for this interface
623 */
Eric W. Biederman881d9662007-09-17 11:56:21 -0700624 if (__dev_get_by_name(&init_net, name))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700625 return 1;
626
627 for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++)
628 if (!strcmp(name, s[i].name))
629 return s[i].map.base_addr;
630 return 0;
631}
632
633/*
634 * Saves at boot time configured settings for any netdevice.
635 */
636int __init netdev_boot_setup(char *str)
637{
638 int ints[5];
639 struct ifmap map;
640
641 str = get_options(str, ARRAY_SIZE(ints), ints);
642 if (!str || !*str)
643 return 0;
644
645 /* Save settings */
646 memset(&map, 0, sizeof(map));
647 if (ints[0] > 0)
648 map.irq = ints[1];
649 if (ints[0] > 1)
650 map.base_addr = ints[2];
651 if (ints[0] > 2)
652 map.mem_start = ints[3];
653 if (ints[0] > 3)
654 map.mem_end = ints[4];
655
656 /* Add new entry to the list */
657 return netdev_boot_setup_add(str, &map);
658}
659
660__setup("netdev=", netdev_boot_setup);
661
662/*******************************************************************************
663
664 Device Interface Subroutines
665
666*******************************************************************************/
667
668/**
Nicolas Dichtela54acb32015-04-02 17:07:00 +0200669 * dev_get_iflink - get 'iflink' value of a interface
670 * @dev: targeted interface
671 *
672 * Indicates the ifindex the interface is linked to.
673 * Physical interfaces have the same 'ifindex' and 'iflink' values.
674 */
675
676int dev_get_iflink(const struct net_device *dev)
677{
678 if (dev->netdev_ops && dev->netdev_ops->ndo_get_iflink)
679 return dev->netdev_ops->ndo_get_iflink(dev);
680
Nicolas Dichtel7a66bbc2015-04-02 17:07:09 +0200681 return dev->ifindex;
Nicolas Dichtela54acb32015-04-02 17:07:00 +0200682}
683EXPORT_SYMBOL(dev_get_iflink);
684
685/**
Pravin B Shelarfc4099f2015-10-22 18:17:16 -0700686 * dev_fill_metadata_dst - Retrieve tunnel egress information.
687 * @dev: targeted interface
688 * @skb: The packet.
689 *
690 * For better visibility of tunnel traffic OVS needs to retrieve
691 * egress tunnel information for a packet. Following API allows
692 * user to get this info.
693 */
694int dev_fill_metadata_dst(struct net_device *dev, struct sk_buff *skb)
695{
696 struct ip_tunnel_info *info;
697
698 if (!dev->netdev_ops || !dev->netdev_ops->ndo_fill_metadata_dst)
699 return -EINVAL;
700
701 info = skb_tunnel_info_unclone(skb);
702 if (!info)
703 return -ENOMEM;
704 if (unlikely(!(info->mode & IP_TUNNEL_INFO_TX)))
705 return -EINVAL;
706
707 return dev->netdev_ops->ndo_fill_metadata_dst(dev, skb);
708}
709EXPORT_SYMBOL_GPL(dev_fill_metadata_dst);
710
711/**
Linus Torvalds1da177e2005-04-16 15:20:36 -0700712 * __dev_get_by_name - find a device by its name
Randy Dunlapc4ea43c2007-10-12 21:17:49 -0700713 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -0700714 * @name: name to find
715 *
716 * Find an interface by name. Must be called under RTNL semaphore
717 * or @dev_base_lock. If the name is found a pointer to the device
718 * is returned. If the name is not found then %NULL is returned. The
719 * reference counters are not incremented so the caller must be
720 * careful with locks.
721 */
722
Eric W. Biederman881d9662007-09-17 11:56:21 -0700723struct net_device *__dev_get_by_name(struct net *net, const char *name)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700724{
Eric Dumazet0bd8d532009-10-30 01:40:11 -0700725 struct net_device *dev;
726 struct hlist_head *head = dev_name_hash(net, name);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700727
Sasha Levinb67bfe02013-02-27 17:06:00 -0800728 hlist_for_each_entry(dev, head, name_hlist)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700729 if (!strncmp(dev->name, name, IFNAMSIZ))
730 return dev;
Eric Dumazet0bd8d532009-10-30 01:40:11 -0700731
Linus Torvalds1da177e2005-04-16 15:20:36 -0700732 return NULL;
733}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700734EXPORT_SYMBOL(__dev_get_by_name);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700735
736/**
Eric Dumazet72c95282009-10-30 07:11:27 +0000737 * dev_get_by_name_rcu - find a device by its name
738 * @net: the applicable net namespace
739 * @name: name to find
740 *
741 * Find an interface by name.
742 * If the name is found a pointer to the device is returned.
743 * If the name is not found then %NULL is returned.
744 * The reference counters are not incremented so the caller must be
745 * careful with locks. The caller must hold RCU lock.
746 */
747
748struct net_device *dev_get_by_name_rcu(struct net *net, const char *name)
749{
Eric Dumazet72c95282009-10-30 07:11:27 +0000750 struct net_device *dev;
751 struct hlist_head *head = dev_name_hash(net, name);
752
Sasha Levinb67bfe02013-02-27 17:06:00 -0800753 hlist_for_each_entry_rcu(dev, head, name_hlist)
Eric Dumazet72c95282009-10-30 07:11:27 +0000754 if (!strncmp(dev->name, name, IFNAMSIZ))
755 return dev;
756
757 return NULL;
758}
759EXPORT_SYMBOL(dev_get_by_name_rcu);
760
761/**
Linus Torvalds1da177e2005-04-16 15:20:36 -0700762 * dev_get_by_name - find a device by its name
Randy Dunlapc4ea43c2007-10-12 21:17:49 -0700763 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -0700764 * @name: name to find
765 *
766 * Find an interface by name. This can be called from any
767 * context and does its own locking. The returned handle has
768 * the usage count incremented and the caller must use dev_put() to
769 * release it when it is no longer needed. %NULL is returned if no
770 * matching device is found.
771 */
772
Eric W. Biederman881d9662007-09-17 11:56:21 -0700773struct net_device *dev_get_by_name(struct net *net, const char *name)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700774{
775 struct net_device *dev;
776
Eric Dumazet72c95282009-10-30 07:11:27 +0000777 rcu_read_lock();
778 dev = dev_get_by_name_rcu(net, name);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700779 if (dev)
780 dev_hold(dev);
Eric Dumazet72c95282009-10-30 07:11:27 +0000781 rcu_read_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700782 return dev;
783}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700784EXPORT_SYMBOL(dev_get_by_name);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700785
786/**
787 * __dev_get_by_index - find a device by its ifindex
Randy Dunlapc4ea43c2007-10-12 21:17:49 -0700788 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -0700789 * @ifindex: index of device
790 *
791 * Search for an interface by index. Returns %NULL if the device
792 * is not found or a pointer to the device. The device has not
793 * had its reference counter increased so the caller must be careful
794 * about locking. The caller must hold either the RTNL semaphore
795 * or @dev_base_lock.
796 */
797
Eric W. Biederman881d9662007-09-17 11:56:21 -0700798struct net_device *__dev_get_by_index(struct net *net, int ifindex)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700799{
Eric Dumazet0bd8d532009-10-30 01:40:11 -0700800 struct net_device *dev;
801 struct hlist_head *head = dev_index_hash(net, ifindex);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700802
Sasha Levinb67bfe02013-02-27 17:06:00 -0800803 hlist_for_each_entry(dev, head, index_hlist)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700804 if (dev->ifindex == ifindex)
805 return dev;
Eric Dumazet0bd8d532009-10-30 01:40:11 -0700806
Linus Torvalds1da177e2005-04-16 15:20:36 -0700807 return NULL;
808}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700809EXPORT_SYMBOL(__dev_get_by_index);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700810
Eric Dumazetfb699dfd2009-10-19 19:18:49 +0000811/**
812 * dev_get_by_index_rcu - find a device by its ifindex
813 * @net: the applicable net namespace
814 * @ifindex: index of device
815 *
816 * Search for an interface by index. Returns %NULL if the device
817 * is not found or a pointer to the device. The device has not
818 * had its reference counter increased so the caller must be careful
819 * about locking. The caller must hold RCU lock.
820 */
821
822struct net_device *dev_get_by_index_rcu(struct net *net, int ifindex)
823{
Eric Dumazetfb699dfd2009-10-19 19:18:49 +0000824 struct net_device *dev;
825 struct hlist_head *head = dev_index_hash(net, ifindex);
826
Sasha Levinb67bfe02013-02-27 17:06:00 -0800827 hlist_for_each_entry_rcu(dev, head, index_hlist)
Eric Dumazetfb699dfd2009-10-19 19:18:49 +0000828 if (dev->ifindex == ifindex)
829 return dev;
830
831 return NULL;
832}
833EXPORT_SYMBOL(dev_get_by_index_rcu);
834
Linus Torvalds1da177e2005-04-16 15:20:36 -0700835
836/**
837 * dev_get_by_index - find a device by its ifindex
Randy Dunlapc4ea43c2007-10-12 21:17:49 -0700838 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -0700839 * @ifindex: index of device
840 *
841 * Search for an interface by index. Returns NULL if the device
842 * is not found or a pointer to the device. The device returned has
843 * had a reference added and the pointer is safe until the user calls
844 * dev_put to indicate they have finished with it.
845 */
846
Eric W. Biederman881d9662007-09-17 11:56:21 -0700847struct net_device *dev_get_by_index(struct net *net, int ifindex)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700848{
849 struct net_device *dev;
850
Eric Dumazetfb699dfd2009-10-19 19:18:49 +0000851 rcu_read_lock();
852 dev = dev_get_by_index_rcu(net, ifindex);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700853 if (dev)
854 dev_hold(dev);
Eric Dumazetfb699dfd2009-10-19 19:18:49 +0000855 rcu_read_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700856 return dev;
857}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700858EXPORT_SYMBOL(dev_get_by_index);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700859
860/**
Nicolas Schichan5dbe7c12013-06-26 17:23:42 +0200861 * netdev_get_name - get a netdevice name, knowing its ifindex.
862 * @net: network namespace
863 * @name: a pointer to the buffer where the name will be stored.
864 * @ifindex: the ifindex of the interface to get the name from.
865 *
866 * The use of raw_seqcount_begin() and cond_resched() before
867 * retrying is required as we want to give the writers a chance
868 * to complete when CONFIG_PREEMPT is not set.
869 */
870int netdev_get_name(struct net *net, char *name, int ifindex)
871{
872 struct net_device *dev;
873 unsigned int seq;
874
875retry:
876 seq = raw_seqcount_begin(&devnet_rename_seq);
877 rcu_read_lock();
878 dev = dev_get_by_index_rcu(net, ifindex);
879 if (!dev) {
880 rcu_read_unlock();
881 return -ENODEV;
882 }
883
884 strcpy(name, dev->name);
885 rcu_read_unlock();
886 if (read_seqcount_retry(&devnet_rename_seq, seq)) {
887 cond_resched();
888 goto retry;
889 }
890
891 return 0;
892}
893
894/**
Eric Dumazet941666c2010-12-05 01:23:53 +0000895 * dev_getbyhwaddr_rcu - find a device by its hardware address
Randy Dunlapc4ea43c2007-10-12 21:17:49 -0700896 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -0700897 * @type: media type of device
898 * @ha: hardware address
899 *
900 * Search for an interface by MAC address. Returns NULL if the device
Eric Dumazetc5066532011-01-24 13:16:16 -0800901 * is not found or a pointer to the device.
902 * The caller must hold RCU or RTNL.
Eric Dumazet941666c2010-12-05 01:23:53 +0000903 * The returned device has not had its ref count increased
Linus Torvalds1da177e2005-04-16 15:20:36 -0700904 * and the caller must therefore be careful about locking
905 *
Linus Torvalds1da177e2005-04-16 15:20:36 -0700906 */
907
Eric Dumazet941666c2010-12-05 01:23:53 +0000908struct net_device *dev_getbyhwaddr_rcu(struct net *net, unsigned short type,
909 const char *ha)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700910{
911 struct net_device *dev;
912
Eric Dumazet941666c2010-12-05 01:23:53 +0000913 for_each_netdev_rcu(net, dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700914 if (dev->type == type &&
915 !memcmp(dev->dev_addr, ha, dev->addr_len))
Pavel Emelianov7562f872007-05-03 15:13:45 -0700916 return dev;
917
918 return NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700919}
Eric Dumazet941666c2010-12-05 01:23:53 +0000920EXPORT_SYMBOL(dev_getbyhwaddr_rcu);
Jochen Friedrichcf309e32005-09-22 04:44:55 -0300921
Eric W. Biederman881d9662007-09-17 11:56:21 -0700922struct net_device *__dev_getfirstbyhwtype(struct net *net, unsigned short type)
Patrick McHardy4e9cac22007-05-03 03:28:13 -0700923{
924 struct net_device *dev;
925
926 ASSERT_RTNL();
Eric W. Biederman881d9662007-09-17 11:56:21 -0700927 for_each_netdev(net, dev)
Patrick McHardy4e9cac22007-05-03 03:28:13 -0700928 if (dev->type == type)
Pavel Emelianov7562f872007-05-03 15:13:45 -0700929 return dev;
930
931 return NULL;
Patrick McHardy4e9cac22007-05-03 03:28:13 -0700932}
Patrick McHardy4e9cac22007-05-03 03:28:13 -0700933EXPORT_SYMBOL(__dev_getfirstbyhwtype);
934
Eric W. Biederman881d9662007-09-17 11:56:21 -0700935struct net_device *dev_getfirstbyhwtype(struct net *net, unsigned short type)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700936{
Eric Dumazet99fe3c32010-03-18 11:27:25 +0000937 struct net_device *dev, *ret = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700938
Eric Dumazet99fe3c32010-03-18 11:27:25 +0000939 rcu_read_lock();
940 for_each_netdev_rcu(net, dev)
941 if (dev->type == type) {
942 dev_hold(dev);
943 ret = dev;
944 break;
945 }
946 rcu_read_unlock();
947 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700948}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700949EXPORT_SYMBOL(dev_getfirstbyhwtype);
950
951/**
WANG Cong6c555492014-09-11 15:35:09 -0700952 * __dev_get_by_flags - find any device with given flags
Randy Dunlapc4ea43c2007-10-12 21:17:49 -0700953 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -0700954 * @if_flags: IFF_* values
955 * @mask: bitmask of bits in if_flags to check
956 *
957 * Search for any interface with the given flags. Returns NULL if a device
Eric Dumazetbb69ae02010-06-07 11:42:13 +0000958 * is not found or a pointer to the device. Must be called inside
WANG Cong6c555492014-09-11 15:35:09 -0700959 * rtnl_lock(), and result refcount is unchanged.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700960 */
961
WANG Cong6c555492014-09-11 15:35:09 -0700962struct net_device *__dev_get_by_flags(struct net *net, unsigned short if_flags,
963 unsigned short mask)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700964{
Pavel Emelianov7562f872007-05-03 15:13:45 -0700965 struct net_device *dev, *ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700966
WANG Cong6c555492014-09-11 15:35:09 -0700967 ASSERT_RTNL();
968
Pavel Emelianov7562f872007-05-03 15:13:45 -0700969 ret = NULL;
WANG Cong6c555492014-09-11 15:35:09 -0700970 for_each_netdev(net, dev) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700971 if (((dev->flags ^ if_flags) & mask) == 0) {
Pavel Emelianov7562f872007-05-03 15:13:45 -0700972 ret = dev;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700973 break;
974 }
975 }
Pavel Emelianov7562f872007-05-03 15:13:45 -0700976 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700977}
WANG Cong6c555492014-09-11 15:35:09 -0700978EXPORT_SYMBOL(__dev_get_by_flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700979
980/**
981 * dev_valid_name - check if name is okay for network device
982 * @name: name string
983 *
984 * Network device names need to be valid file names to
David S. Millerc7fa9d12006-08-15 16:34:13 -0700985 * to allow sysfs to work. We also disallow any kind of
986 * whitespace.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700987 */
David S. Miller95f050b2012-03-06 16:12:15 -0500988bool dev_valid_name(const char *name)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700989{
David S. Millerc7fa9d12006-08-15 16:34:13 -0700990 if (*name == '\0')
David S. Miller95f050b2012-03-06 16:12:15 -0500991 return false;
Stephen Hemmingerb6fe17d2006-08-29 17:06:13 -0700992 if (strlen(name) >= IFNAMSIZ)
David S. Miller95f050b2012-03-06 16:12:15 -0500993 return false;
David S. Millerc7fa9d12006-08-15 16:34:13 -0700994 if (!strcmp(name, ".") || !strcmp(name, ".."))
David S. Miller95f050b2012-03-06 16:12:15 -0500995 return false;
David S. Millerc7fa9d12006-08-15 16:34:13 -0700996
997 while (*name) {
Matthew Thodea4176a92015-02-17 18:31:57 -0600998 if (*name == '/' || *name == ':' || isspace(*name))
David S. Miller95f050b2012-03-06 16:12:15 -0500999 return false;
David S. Millerc7fa9d12006-08-15 16:34:13 -07001000 name++;
1001 }
David S. Miller95f050b2012-03-06 16:12:15 -05001002 return true;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001003}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001004EXPORT_SYMBOL(dev_valid_name);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001005
1006/**
Eric W. Biedermanb267b172007-09-12 13:48:45 +02001007 * __dev_alloc_name - allocate a name for a device
1008 * @net: network namespace to allocate the device name in
Linus Torvalds1da177e2005-04-16 15:20:36 -07001009 * @name: name format string
Eric W. Biedermanb267b172007-09-12 13:48:45 +02001010 * @buf: scratch buffer and result name string
Linus Torvalds1da177e2005-04-16 15:20:36 -07001011 *
1012 * Passed a format string - eg "lt%d" it will try and find a suitable
Stephen Hemminger3041a062006-05-26 13:25:24 -07001013 * id. It scans list of devices to build up a free map, then chooses
1014 * the first empty slot. The caller must hold the dev_base or rtnl lock
1015 * while allocating the name and adding the device in order to avoid
1016 * duplicates.
1017 * Limited to bits_per_byte * page size devices (ie 32K on most platforms).
1018 * Returns the number of the unit assigned or a negative errno code.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001019 */
1020
Eric W. Biedermanb267b172007-09-12 13:48:45 +02001021static int __dev_alloc_name(struct net *net, const char *name, char *buf)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001022{
1023 int i = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001024 const char *p;
1025 const int max_netdevices = 8*PAGE_SIZE;
Stephen Hemmingercfcabdc2007-10-09 01:59:42 -07001026 unsigned long *inuse;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001027 struct net_device *d;
1028
1029 p = strnchr(name, IFNAMSIZ-1, '%');
1030 if (p) {
1031 /*
1032 * Verify the string as this thing may have come from
1033 * the user. There must be either one "%d" and no other "%"
1034 * characters.
1035 */
1036 if (p[1] != 'd' || strchr(p + 2, '%'))
1037 return -EINVAL;
1038
1039 /* Use one page as a bit array of possible slots */
Stephen Hemmingercfcabdc2007-10-09 01:59:42 -07001040 inuse = (unsigned long *) get_zeroed_page(GFP_ATOMIC);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001041 if (!inuse)
1042 return -ENOMEM;
1043
Eric W. Biederman881d9662007-09-17 11:56:21 -07001044 for_each_netdev(net, d) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001045 if (!sscanf(d->name, name, &i))
1046 continue;
1047 if (i < 0 || i >= max_netdevices)
1048 continue;
1049
1050 /* avoid cases where sscanf is not exact inverse of printf */
Eric W. Biedermanb267b172007-09-12 13:48:45 +02001051 snprintf(buf, IFNAMSIZ, name, i);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001052 if (!strncmp(buf, d->name, IFNAMSIZ))
1053 set_bit(i, inuse);
1054 }
1055
1056 i = find_first_zero_bit(inuse, max_netdevices);
1057 free_page((unsigned long) inuse);
1058 }
1059
Octavian Purdilad9031022009-11-18 02:36:59 +00001060 if (buf != name)
1061 snprintf(buf, IFNAMSIZ, name, i);
Eric W. Biedermanb267b172007-09-12 13:48:45 +02001062 if (!__dev_get_by_name(net, buf))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001063 return i;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001064
1065 /* It is possible to run out of possible slots
1066 * when the name is long and there isn't enough space left
1067 * for the digits, or if all bits are used.
1068 */
1069 return -ENFILE;
1070}
1071
Eric W. Biedermanb267b172007-09-12 13:48:45 +02001072/**
1073 * dev_alloc_name - allocate a name for a device
1074 * @dev: device
1075 * @name: name format string
1076 *
1077 * Passed a format string - eg "lt%d" it will try and find a suitable
1078 * id. It scans list of devices to build up a free map, then chooses
1079 * the first empty slot. The caller must hold the dev_base or rtnl lock
1080 * while allocating the name and adding the device in order to avoid
1081 * duplicates.
1082 * Limited to bits_per_byte * page size devices (ie 32K on most platforms).
1083 * Returns the number of the unit assigned or a negative errno code.
1084 */
1085
1086int dev_alloc_name(struct net_device *dev, const char *name)
1087{
1088 char buf[IFNAMSIZ];
1089 struct net *net;
1090 int ret;
1091
YOSHIFUJI Hideakic346dca2008-03-25 21:47:49 +09001092 BUG_ON(!dev_net(dev));
1093 net = dev_net(dev);
Eric W. Biedermanb267b172007-09-12 13:48:45 +02001094 ret = __dev_alloc_name(net, name, buf);
1095 if (ret >= 0)
1096 strlcpy(dev->name, buf, IFNAMSIZ);
1097 return ret;
1098}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001099EXPORT_SYMBOL(dev_alloc_name);
Eric W. Biedermanb267b172007-09-12 13:48:45 +02001100
Gao feng828de4f2012-09-13 20:58:27 +00001101static int dev_alloc_name_ns(struct net *net,
1102 struct net_device *dev,
1103 const char *name)
Octavian Purdilad9031022009-11-18 02:36:59 +00001104{
Gao feng828de4f2012-09-13 20:58:27 +00001105 char buf[IFNAMSIZ];
1106 int ret;
Daniel Lezcano8ce6cebc2010-05-19 10:12:19 +00001107
Gao feng828de4f2012-09-13 20:58:27 +00001108 ret = __dev_alloc_name(net, name, buf);
1109 if (ret >= 0)
1110 strlcpy(dev->name, buf, IFNAMSIZ);
1111 return ret;
1112}
1113
1114static int dev_get_valid_name(struct net *net,
1115 struct net_device *dev,
1116 const char *name)
1117{
1118 BUG_ON(!net);
Daniel Lezcano8ce6cebc2010-05-19 10:12:19 +00001119
Octavian Purdilad9031022009-11-18 02:36:59 +00001120 if (!dev_valid_name(name))
1121 return -EINVAL;
1122
Jiri Pirko1c5cae82011-04-30 01:21:32 +00001123 if (strchr(name, '%'))
Gao feng828de4f2012-09-13 20:58:27 +00001124 return dev_alloc_name_ns(net, dev, name);
Octavian Purdilad9031022009-11-18 02:36:59 +00001125 else if (__dev_get_by_name(net, name))
1126 return -EEXIST;
Daniel Lezcano8ce6cebc2010-05-19 10:12:19 +00001127 else if (dev->name != name)
1128 strlcpy(dev->name, name, IFNAMSIZ);
Octavian Purdilad9031022009-11-18 02:36:59 +00001129
1130 return 0;
1131}
Linus Torvalds1da177e2005-04-16 15:20:36 -07001132
1133/**
1134 * dev_change_name - change name of a device
1135 * @dev: device
1136 * @newname: name (or format string) must be at least IFNAMSIZ
1137 *
1138 * Change name of a device, can pass format strings "eth%d".
1139 * for wildcarding.
1140 */
Stephen Hemmingercf04a4c72008-09-30 02:22:14 -07001141int dev_change_name(struct net_device *dev, const char *newname)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001142{
Tom Gundersen238fa362014-07-14 16:37:23 +02001143 unsigned char old_assign_type;
Herbert Xufcc5a032007-07-30 17:03:38 -07001144 char oldname[IFNAMSIZ];
Linus Torvalds1da177e2005-04-16 15:20:36 -07001145 int err = 0;
Herbert Xufcc5a032007-07-30 17:03:38 -07001146 int ret;
Eric W. Biederman881d9662007-09-17 11:56:21 -07001147 struct net *net;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001148
1149 ASSERT_RTNL();
YOSHIFUJI Hideakic346dca2008-03-25 21:47:49 +09001150 BUG_ON(!dev_net(dev));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001151
YOSHIFUJI Hideakic346dca2008-03-25 21:47:49 +09001152 net = dev_net(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001153 if (dev->flags & IFF_UP)
1154 return -EBUSY;
1155
Eric Dumazet30e6c9f2012-12-20 17:25:08 +00001156 write_seqcount_begin(&devnet_rename_seq);
Brian Haleyc91f6df2012-11-26 05:21:08 +00001157
1158 if (strncmp(newname, dev->name, IFNAMSIZ) == 0) {
Eric Dumazet30e6c9f2012-12-20 17:25:08 +00001159 write_seqcount_end(&devnet_rename_seq);
Stephen Hemmingerc8d90dc2007-10-26 03:53:42 -07001160 return 0;
Brian Haleyc91f6df2012-11-26 05:21:08 +00001161 }
Stephen Hemmingerc8d90dc2007-10-26 03:53:42 -07001162
Herbert Xufcc5a032007-07-30 17:03:38 -07001163 memcpy(oldname, dev->name, IFNAMSIZ);
1164
Gao feng828de4f2012-09-13 20:58:27 +00001165 err = dev_get_valid_name(net, dev, newname);
Brian Haleyc91f6df2012-11-26 05:21:08 +00001166 if (err < 0) {
Eric Dumazet30e6c9f2012-12-20 17:25:08 +00001167 write_seqcount_end(&devnet_rename_seq);
Octavian Purdilad9031022009-11-18 02:36:59 +00001168 return err;
Brian Haleyc91f6df2012-11-26 05:21:08 +00001169 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001170
Veaceslav Falico6fe82a32014-07-17 20:33:32 +02001171 if (oldname[0] && !strchr(oldname, '%'))
1172 netdev_info(dev, "renamed from %s\n", oldname);
1173
Tom Gundersen238fa362014-07-14 16:37:23 +02001174 old_assign_type = dev->name_assign_type;
1175 dev->name_assign_type = NET_NAME_RENAMED;
1176
Herbert Xufcc5a032007-07-30 17:03:38 -07001177rollback:
Eric W. Biedermana1b3f592010-05-04 17:36:49 -07001178 ret = device_rename(&dev->dev, dev->name);
1179 if (ret) {
1180 memcpy(dev->name, oldname, IFNAMSIZ);
Tom Gundersen238fa362014-07-14 16:37:23 +02001181 dev->name_assign_type = old_assign_type;
Eric Dumazet30e6c9f2012-12-20 17:25:08 +00001182 write_seqcount_end(&devnet_rename_seq);
Eric W. Biedermana1b3f592010-05-04 17:36:49 -07001183 return ret;
Stephen Hemmingerdcc99772008-05-14 22:33:38 -07001184 }
Herbert Xu7f988ea2007-07-30 16:35:46 -07001185
Eric Dumazet30e6c9f2012-12-20 17:25:08 +00001186 write_seqcount_end(&devnet_rename_seq);
Brian Haleyc91f6df2012-11-26 05:21:08 +00001187
Veaceslav Falico5bb025f2014-01-14 21:58:51 +01001188 netdev_adjacent_rename_links(dev, oldname);
1189
Herbert Xu7f988ea2007-07-30 16:35:46 -07001190 write_lock_bh(&dev_base_lock);
Eric Dumazet372b2312011-05-17 13:56:59 -04001191 hlist_del_rcu(&dev->name_hlist);
Eric Dumazet72c95282009-10-30 07:11:27 +00001192 write_unlock_bh(&dev_base_lock);
1193
1194 synchronize_rcu();
1195
1196 write_lock_bh(&dev_base_lock);
1197 hlist_add_head_rcu(&dev->name_hlist, dev_name_hash(net, dev->name));
Herbert Xu7f988ea2007-07-30 16:35:46 -07001198 write_unlock_bh(&dev_base_lock);
1199
Pavel Emelyanov056925a2007-09-16 15:42:43 -07001200 ret = call_netdevice_notifiers(NETDEV_CHANGENAME, dev);
Herbert Xufcc5a032007-07-30 17:03:38 -07001201 ret = notifier_to_errno(ret);
1202
1203 if (ret) {
Eric Dumazet91e9c07b2009-11-15 23:30:24 +00001204 /* err >= 0 after dev_alloc_name() or stores the first errno */
1205 if (err >= 0) {
Herbert Xufcc5a032007-07-30 17:03:38 -07001206 err = ret;
Eric Dumazet30e6c9f2012-12-20 17:25:08 +00001207 write_seqcount_begin(&devnet_rename_seq);
Herbert Xufcc5a032007-07-30 17:03:38 -07001208 memcpy(dev->name, oldname, IFNAMSIZ);
Veaceslav Falico5bb025f2014-01-14 21:58:51 +01001209 memcpy(oldname, newname, IFNAMSIZ);
Tom Gundersen238fa362014-07-14 16:37:23 +02001210 dev->name_assign_type = old_assign_type;
1211 old_assign_type = NET_NAME_RENAMED;
Herbert Xufcc5a032007-07-30 17:03:38 -07001212 goto rollback;
Eric Dumazet91e9c07b2009-11-15 23:30:24 +00001213 } else {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00001214 pr_err("%s: name change rollback failed: %d\n",
Eric Dumazet91e9c07b2009-11-15 23:30:24 +00001215 dev->name, ret);
Herbert Xufcc5a032007-07-30 17:03:38 -07001216 }
1217 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001218
1219 return err;
1220}
1221
1222/**
Stephen Hemminger0b815a12008-09-22 21:28:11 -07001223 * dev_set_alias - change ifalias of a device
1224 * @dev: device
1225 * @alias: name up to IFALIASZ
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07001226 * @len: limit of bytes to copy from info
Stephen Hemminger0b815a12008-09-22 21:28:11 -07001227 *
1228 * Set ifalias for a device,
1229 */
1230int dev_set_alias(struct net_device *dev, const char *alias, size_t len)
1231{
Alexey Khoroshilov7364e442012-08-08 00:33:25 +00001232 char *new_ifalias;
1233
Stephen Hemminger0b815a12008-09-22 21:28:11 -07001234 ASSERT_RTNL();
1235
1236 if (len >= IFALIASZ)
1237 return -EINVAL;
1238
Oliver Hartkopp96ca4a22008-09-23 21:23:19 -07001239 if (!len) {
Sachin Kamat388dfc22012-11-20 00:57:04 +00001240 kfree(dev->ifalias);
1241 dev->ifalias = NULL;
Oliver Hartkopp96ca4a22008-09-23 21:23:19 -07001242 return 0;
1243 }
1244
Alexey Khoroshilov7364e442012-08-08 00:33:25 +00001245 new_ifalias = krealloc(dev->ifalias, len + 1, GFP_KERNEL);
1246 if (!new_ifalias)
Stephen Hemminger0b815a12008-09-22 21:28:11 -07001247 return -ENOMEM;
Alexey Khoroshilov7364e442012-08-08 00:33:25 +00001248 dev->ifalias = new_ifalias;
Stephen Hemminger0b815a12008-09-22 21:28:11 -07001249
1250 strlcpy(dev->ifalias, alias, len+1);
1251 return len;
1252}
1253
1254
1255/**
Stephen Hemminger3041a062006-05-26 13:25:24 -07001256 * netdev_features_change - device changes features
Stephen Hemmingerd8a33ac2005-05-29 14:13:47 -07001257 * @dev: device to cause notification
1258 *
1259 * Called to indicate a device has changed features.
1260 */
1261void netdev_features_change(struct net_device *dev)
1262{
Pavel Emelyanov056925a2007-09-16 15:42:43 -07001263 call_netdevice_notifiers(NETDEV_FEAT_CHANGE, dev);
Stephen Hemmingerd8a33ac2005-05-29 14:13:47 -07001264}
1265EXPORT_SYMBOL(netdev_features_change);
1266
1267/**
Linus Torvalds1da177e2005-04-16 15:20:36 -07001268 * netdev_state_change - device changes state
1269 * @dev: device to cause notification
1270 *
1271 * Called to indicate a device has changed state. This function calls
1272 * the notifier chains for netdev_chain and sends a NEWLINK message
1273 * to the routing socket.
1274 */
1275void netdev_state_change(struct net_device *dev)
1276{
1277 if (dev->flags & IFF_UP) {
Loic Prylli54951192014-07-01 21:39:43 -07001278 struct netdev_notifier_change_info change_info;
1279
1280 change_info.flags_changed = 0;
1281 call_netdevice_notifiers_info(NETDEV_CHANGE, dev,
1282 &change_info.info);
Alexei Starovoitov7f294052013-10-23 16:02:42 -07001283 rtmsg_ifinfo(RTM_NEWLINK, dev, 0, GFP_KERNEL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001284 }
1285}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001286EXPORT_SYMBOL(netdev_state_change);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001287
Amerigo Wangee89bab2012-08-09 22:14:56 +00001288/**
1289 * netdev_notify_peers - notify network peers about existence of @dev
1290 * @dev: network device
1291 *
1292 * Generate traffic such that interested network peers are aware of
1293 * @dev, such as by generating a gratuitous ARP. This may be used when
1294 * a device wants to inform the rest of the network about some sort of
1295 * reconfiguration such as a failover event or virtual machine
1296 * migration.
1297 */
1298void netdev_notify_peers(struct net_device *dev)
Or Gerlitzc1da4ac2008-06-13 18:12:00 -07001299{
Amerigo Wangee89bab2012-08-09 22:14:56 +00001300 rtnl_lock();
1301 call_netdevice_notifiers(NETDEV_NOTIFY_PEERS, dev);
1302 rtnl_unlock();
Or Gerlitzc1da4ac2008-06-13 18:12:00 -07001303}
Amerigo Wangee89bab2012-08-09 22:14:56 +00001304EXPORT_SYMBOL(netdev_notify_peers);
Or Gerlitzc1da4ac2008-06-13 18:12:00 -07001305
Patrick McHardybd380812010-02-26 06:34:53 +00001306static int __dev_open(struct net_device *dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001307{
Stephen Hemmingerd3147742008-11-19 21:32:24 -08001308 const struct net_device_ops *ops = dev->netdev_ops;
Johannes Berg3b8bcfd2009-05-30 01:39:53 +02001309 int ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001310
Ben Hutchingse46b66b2008-05-08 02:53:17 -07001311 ASSERT_RTNL();
1312
Linus Torvalds1da177e2005-04-16 15:20:36 -07001313 if (!netif_device_present(dev))
1314 return -ENODEV;
1315
Neil Hormanca99ca12013-02-05 08:05:43 +00001316 /* Block netpoll from trying to do any rx path servicing.
1317 * If we don't do this there is a chance ndo_poll_controller
1318 * or ndo_poll may be running while we open the device
1319 */
Eric W. Biederman66b55522014-03-27 15:39:03 -07001320 netpoll_poll_disable(dev);
Neil Hormanca99ca12013-02-05 08:05:43 +00001321
Johannes Berg3b8bcfd2009-05-30 01:39:53 +02001322 ret = call_netdevice_notifiers(NETDEV_PRE_UP, dev);
1323 ret = notifier_to_errno(ret);
1324 if (ret)
1325 return ret;
1326
Linus Torvalds1da177e2005-04-16 15:20:36 -07001327 set_bit(__LINK_STATE_START, &dev->state);
Jeff Garzikbada3392007-10-23 20:19:37 -07001328
Stephen Hemmingerd3147742008-11-19 21:32:24 -08001329 if (ops->ndo_validate_addr)
1330 ret = ops->ndo_validate_addr(dev);
Jeff Garzikbada3392007-10-23 20:19:37 -07001331
Stephen Hemmingerd3147742008-11-19 21:32:24 -08001332 if (!ret && ops->ndo_open)
1333 ret = ops->ndo_open(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001334
Eric W. Biederman66b55522014-03-27 15:39:03 -07001335 netpoll_poll_enable(dev);
Neil Hormanca99ca12013-02-05 08:05:43 +00001336
Jeff Garzikbada3392007-10-23 20:19:37 -07001337 if (ret)
1338 clear_bit(__LINK_STATE_START, &dev->state);
1339 else {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001340 dev->flags |= IFF_UP;
Patrick McHardy4417da62007-06-27 01:28:10 -07001341 dev_set_rx_mode(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001342 dev_activate(dev);
Theodore Ts'o7bf23572012-07-04 21:23:25 -04001343 add_device_randomness(dev->dev_addr, dev->addr_len);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001344 }
Jeff Garzikbada3392007-10-23 20:19:37 -07001345
Linus Torvalds1da177e2005-04-16 15:20:36 -07001346 return ret;
1347}
Patrick McHardybd380812010-02-26 06:34:53 +00001348
1349/**
1350 * dev_open - prepare an interface for use.
1351 * @dev: device to open
1352 *
1353 * Takes a device from down to up state. The device's private open
1354 * function is invoked and then the multicast lists are loaded. Finally
1355 * the device is moved into the up state and a %NETDEV_UP message is
1356 * sent to the netdev notifier chain.
1357 *
1358 * Calling this function on an active interface is a nop. On a failure
1359 * a negative errno code is returned.
1360 */
1361int dev_open(struct net_device *dev)
1362{
1363 int ret;
1364
Patrick McHardybd380812010-02-26 06:34:53 +00001365 if (dev->flags & IFF_UP)
1366 return 0;
1367
Patrick McHardybd380812010-02-26 06:34:53 +00001368 ret = __dev_open(dev);
1369 if (ret < 0)
1370 return ret;
1371
Alexei Starovoitov7f294052013-10-23 16:02:42 -07001372 rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP|IFF_RUNNING, GFP_KERNEL);
Patrick McHardybd380812010-02-26 06:34:53 +00001373 call_netdevice_notifiers(NETDEV_UP, dev);
1374
1375 return ret;
1376}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001377EXPORT_SYMBOL(dev_open);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001378
Octavian Purdila44345722010-12-13 12:44:07 +00001379static int __dev_close_many(struct list_head *head)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001380{
Octavian Purdila44345722010-12-13 12:44:07 +00001381 struct net_device *dev;
Patrick McHardybd380812010-02-26 06:34:53 +00001382
Ben Hutchingse46b66b2008-05-08 02:53:17 -07001383 ASSERT_RTNL();
David S. Miller9d5010d2007-09-12 14:33:25 +02001384 might_sleep();
1385
Eric W. Biederman5cde2822013-10-05 19:26:05 -07001386 list_for_each_entry(dev, head, close_list) {
Eric W. Biederman3f4df202014-03-27 15:38:17 -07001387 /* Temporarily disable netpoll until the interface is down */
Eric W. Biederman66b55522014-03-27 15:39:03 -07001388 netpoll_poll_disable(dev);
Eric W. Biederman3f4df202014-03-27 15:38:17 -07001389
Octavian Purdila44345722010-12-13 12:44:07 +00001390 call_netdevice_notifiers(NETDEV_GOING_DOWN, dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001391
Octavian Purdila44345722010-12-13 12:44:07 +00001392 clear_bit(__LINK_STATE_START, &dev->state);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001393
Octavian Purdila44345722010-12-13 12:44:07 +00001394 /* Synchronize to scheduled poll. We cannot touch poll list, it
1395 * can be even on different cpu. So just clear netif_running().
1396 *
1397 * dev->stop() will invoke napi_disable() on all of it's
1398 * napi_struct instances on this device.
1399 */
Peter Zijlstra4e857c52014-03-17 18:06:10 +01001400 smp_mb__after_atomic(); /* Commit netif_running(). */
Octavian Purdila44345722010-12-13 12:44:07 +00001401 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001402
Octavian Purdila44345722010-12-13 12:44:07 +00001403 dev_deactivate_many(head);
1404
Eric W. Biederman5cde2822013-10-05 19:26:05 -07001405 list_for_each_entry(dev, head, close_list) {
Octavian Purdila44345722010-12-13 12:44:07 +00001406 const struct net_device_ops *ops = dev->netdev_ops;
1407
1408 /*
1409 * Call the device specific close. This cannot fail.
1410 * Only if device is UP
1411 *
1412 * We allow it to be called even after a DETACH hot-plug
1413 * event.
1414 */
1415 if (ops->ndo_stop)
1416 ops->ndo_stop(dev);
1417
Octavian Purdila44345722010-12-13 12:44:07 +00001418 dev->flags &= ~IFF_UP;
Eric W. Biederman66b55522014-03-27 15:39:03 -07001419 netpoll_poll_enable(dev);
Octavian Purdila44345722010-12-13 12:44:07 +00001420 }
1421
1422 return 0;
1423}
1424
1425static int __dev_close(struct net_device *dev)
1426{
Linus Torvaldsf87e6f42011-02-17 22:54:38 +00001427 int retval;
Octavian Purdila44345722010-12-13 12:44:07 +00001428 LIST_HEAD(single);
1429
Eric W. Biederman5cde2822013-10-05 19:26:05 -07001430 list_add(&dev->close_list, &single);
Linus Torvaldsf87e6f42011-02-17 22:54:38 +00001431 retval = __dev_close_many(&single);
1432 list_del(&single);
Neil Hormanca99ca12013-02-05 08:05:43 +00001433
Linus Torvaldsf87e6f42011-02-17 22:54:38 +00001434 return retval;
Octavian Purdila44345722010-12-13 12:44:07 +00001435}
1436
David S. Miller99c4a262015-03-18 22:52:33 -04001437int dev_close_many(struct list_head *head, bool unlink)
Octavian Purdila44345722010-12-13 12:44:07 +00001438{
1439 struct net_device *dev, *tmp;
Octavian Purdila44345722010-12-13 12:44:07 +00001440
Eric W. Biederman5cde2822013-10-05 19:26:05 -07001441 /* Remove the devices that don't need to be closed */
1442 list_for_each_entry_safe(dev, tmp, head, close_list)
Octavian Purdila44345722010-12-13 12:44:07 +00001443 if (!(dev->flags & IFF_UP))
Eric W. Biederman5cde2822013-10-05 19:26:05 -07001444 list_del_init(&dev->close_list);
Octavian Purdila44345722010-12-13 12:44:07 +00001445
1446 __dev_close_many(head);
Matti Linnanvuorid8b2a4d2008-02-12 23:10:11 -08001447
Eric W. Biederman5cde2822013-10-05 19:26:05 -07001448 list_for_each_entry_safe(dev, tmp, head, close_list) {
Alexei Starovoitov7f294052013-10-23 16:02:42 -07001449 rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP|IFF_RUNNING, GFP_KERNEL);
Octavian Purdila44345722010-12-13 12:44:07 +00001450 call_netdevice_notifiers(NETDEV_DOWN, dev);
David S. Miller99c4a262015-03-18 22:52:33 -04001451 if (unlink)
1452 list_del_init(&dev->close_list);
Octavian Purdila44345722010-12-13 12:44:07 +00001453 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001454
Linus Torvalds1da177e2005-04-16 15:20:36 -07001455 return 0;
1456}
David S. Miller99c4a262015-03-18 22:52:33 -04001457EXPORT_SYMBOL(dev_close_many);
Patrick McHardybd380812010-02-26 06:34:53 +00001458
1459/**
1460 * dev_close - shutdown an interface.
1461 * @dev: device to shutdown
1462 *
1463 * This function moves an active device into down state. A
1464 * %NETDEV_GOING_DOWN is sent to the netdev notifier chain. The device
1465 * is then deactivated and finally a %NETDEV_DOWN is sent to the notifier
1466 * chain.
1467 */
1468int dev_close(struct net_device *dev)
1469{
Eric Dumazete14a5992011-05-10 12:26:06 -07001470 if (dev->flags & IFF_UP) {
1471 LIST_HEAD(single);
Patrick McHardybd380812010-02-26 06:34:53 +00001472
Eric W. Biederman5cde2822013-10-05 19:26:05 -07001473 list_add(&dev->close_list, &single);
David S. Miller99c4a262015-03-18 22:52:33 -04001474 dev_close_many(&single, true);
Eric Dumazete14a5992011-05-10 12:26:06 -07001475 list_del(&single);
1476 }
dingtianhongda6e3782013-05-27 19:53:31 +00001477 return 0;
Patrick McHardybd380812010-02-26 06:34:53 +00001478}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001479EXPORT_SYMBOL(dev_close);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001480
1481
Ben Hutchings0187bdf2008-06-19 16:15:47 -07001482/**
1483 * dev_disable_lro - disable Large Receive Offload on a device
1484 * @dev: device
1485 *
1486 * Disable Large Receive Offload (LRO) on a net device. Must be
1487 * called under RTNL. This is needed if received packets may be
1488 * forwarded to another interface.
1489 */
1490void dev_disable_lro(struct net_device *dev)
1491{
Michal Kubečekfbe168b2014-11-13 07:54:50 +01001492 struct net_device *lower_dev;
1493 struct list_head *iter;
Michal Kubeček529d0482013-11-15 06:18:50 +01001494
Michał Mirosławbc5787c62011-11-15 15:29:55 +00001495 dev->wanted_features &= ~NETIF_F_LRO;
1496 netdev_update_features(dev);
Michał Mirosław27660512011-03-18 16:56:34 +00001497
Michał Mirosław22d59692011-04-21 12:42:15 +00001498 if (unlikely(dev->features & NETIF_F_LRO))
1499 netdev_WARN(dev, "failed to disable LRO!\n");
Michal Kubečekfbe168b2014-11-13 07:54:50 +01001500
1501 netdev_for_each_lower_dev(dev, lower_dev, iter)
1502 dev_disable_lro(lower_dev);
Ben Hutchings0187bdf2008-06-19 16:15:47 -07001503}
1504EXPORT_SYMBOL(dev_disable_lro);
1505
Jiri Pirko351638e2013-05-28 01:30:21 +00001506static int call_netdevice_notifier(struct notifier_block *nb, unsigned long val,
1507 struct net_device *dev)
1508{
1509 struct netdev_notifier_info info;
1510
1511 netdev_notifier_info_init(&info, dev);
1512 return nb->notifier_call(nb, val, &info);
1513}
Ben Hutchings0187bdf2008-06-19 16:15:47 -07001514
Eric W. Biederman881d9662007-09-17 11:56:21 -07001515static int dev_boot_phase = 1;
1516
Linus Torvalds1da177e2005-04-16 15:20:36 -07001517/**
1518 * register_netdevice_notifier - register a network notifier block
1519 * @nb: notifier
1520 *
1521 * Register a notifier to be called when network device events occur.
1522 * The notifier passed is linked into the kernel structures and must
1523 * not be reused until it has been unregistered. A negative errno code
1524 * is returned on a failure.
1525 *
1526 * When registered all registration and up events are replayed
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09001527 * to the new notifier to allow device to have a race free
Linus Torvalds1da177e2005-04-16 15:20:36 -07001528 * view of the network device list.
1529 */
1530
1531int register_netdevice_notifier(struct notifier_block *nb)
1532{
1533 struct net_device *dev;
Herbert Xufcc5a032007-07-30 17:03:38 -07001534 struct net_device *last;
Eric W. Biederman881d9662007-09-17 11:56:21 -07001535 struct net *net;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001536 int err;
1537
1538 rtnl_lock();
Alan Sternf07d5b92006-05-09 15:23:03 -07001539 err = raw_notifier_chain_register(&netdev_chain, nb);
Herbert Xufcc5a032007-07-30 17:03:38 -07001540 if (err)
1541 goto unlock;
Eric W. Biederman881d9662007-09-17 11:56:21 -07001542 if (dev_boot_phase)
1543 goto unlock;
1544 for_each_net(net) {
1545 for_each_netdev(net, dev) {
Jiri Pirko351638e2013-05-28 01:30:21 +00001546 err = call_netdevice_notifier(nb, NETDEV_REGISTER, dev);
Eric W. Biederman881d9662007-09-17 11:56:21 -07001547 err = notifier_to_errno(err);
1548 if (err)
1549 goto rollback;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001550
Eric W. Biederman881d9662007-09-17 11:56:21 -07001551 if (!(dev->flags & IFF_UP))
1552 continue;
Herbert Xufcc5a032007-07-30 17:03:38 -07001553
Jiri Pirko351638e2013-05-28 01:30:21 +00001554 call_netdevice_notifier(nb, NETDEV_UP, dev);
Eric W. Biederman881d9662007-09-17 11:56:21 -07001555 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001556 }
Herbert Xufcc5a032007-07-30 17:03:38 -07001557
1558unlock:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001559 rtnl_unlock();
1560 return err;
Herbert Xufcc5a032007-07-30 17:03:38 -07001561
1562rollback:
1563 last = dev;
Eric W. Biederman881d9662007-09-17 11:56:21 -07001564 for_each_net(net) {
1565 for_each_netdev(net, dev) {
1566 if (dev == last)
RongQing.Li8f891482011-11-30 23:43:07 -05001567 goto outroll;
Herbert Xufcc5a032007-07-30 17:03:38 -07001568
Eric W. Biederman881d9662007-09-17 11:56:21 -07001569 if (dev->flags & IFF_UP) {
Jiri Pirko351638e2013-05-28 01:30:21 +00001570 call_netdevice_notifier(nb, NETDEV_GOING_DOWN,
1571 dev);
1572 call_netdevice_notifier(nb, NETDEV_DOWN, dev);
Eric W. Biederman881d9662007-09-17 11:56:21 -07001573 }
Jiri Pirko351638e2013-05-28 01:30:21 +00001574 call_netdevice_notifier(nb, NETDEV_UNREGISTER, dev);
Herbert Xufcc5a032007-07-30 17:03:38 -07001575 }
Herbert Xufcc5a032007-07-30 17:03:38 -07001576 }
Pavel Emelyanovc67625a2007-11-14 15:53:16 -08001577
RongQing.Li8f891482011-11-30 23:43:07 -05001578outroll:
Pavel Emelyanovc67625a2007-11-14 15:53:16 -08001579 raw_notifier_chain_unregister(&netdev_chain, nb);
Herbert Xufcc5a032007-07-30 17:03:38 -07001580 goto unlock;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001581}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001582EXPORT_SYMBOL(register_netdevice_notifier);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001583
1584/**
1585 * unregister_netdevice_notifier - unregister a network notifier block
1586 * @nb: notifier
1587 *
1588 * Unregister a notifier previously registered by
1589 * register_netdevice_notifier(). The notifier is unlinked into the
1590 * kernel structures and may then be reused. A negative errno code
1591 * is returned on a failure.
Eric W. Biederman7d3d43d2012-04-06 15:33:35 +00001592 *
1593 * After unregistering unregister and down device events are synthesized
1594 * for all devices on the device list to the removed notifier to remove
1595 * the need for special case cleanup code.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001596 */
1597
1598int unregister_netdevice_notifier(struct notifier_block *nb)
1599{
Eric W. Biederman7d3d43d2012-04-06 15:33:35 +00001600 struct net_device *dev;
1601 struct net *net;
Herbert Xu9f514952006-03-25 01:24:25 -08001602 int err;
1603
1604 rtnl_lock();
Alan Sternf07d5b92006-05-09 15:23:03 -07001605 err = raw_notifier_chain_unregister(&netdev_chain, nb);
Eric W. Biederman7d3d43d2012-04-06 15:33:35 +00001606 if (err)
1607 goto unlock;
1608
1609 for_each_net(net) {
1610 for_each_netdev(net, dev) {
1611 if (dev->flags & IFF_UP) {
Jiri Pirko351638e2013-05-28 01:30:21 +00001612 call_netdevice_notifier(nb, NETDEV_GOING_DOWN,
1613 dev);
1614 call_netdevice_notifier(nb, NETDEV_DOWN, dev);
Eric W. Biederman7d3d43d2012-04-06 15:33:35 +00001615 }
Jiri Pirko351638e2013-05-28 01:30:21 +00001616 call_netdevice_notifier(nb, NETDEV_UNREGISTER, dev);
Eric W. Biederman7d3d43d2012-04-06 15:33:35 +00001617 }
1618 }
1619unlock:
Herbert Xu9f514952006-03-25 01:24:25 -08001620 rtnl_unlock();
1621 return err;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001622}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001623EXPORT_SYMBOL(unregister_netdevice_notifier);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001624
1625/**
Jiri Pirko351638e2013-05-28 01:30:21 +00001626 * call_netdevice_notifiers_info - call all network notifier blocks
1627 * @val: value passed unmodified to notifier function
1628 * @dev: net_device pointer passed unmodified to notifier function
1629 * @info: notifier information data
1630 *
1631 * Call all network notifier blocks. Parameters and return value
1632 * are as for raw_notifier_call_chain().
1633 */
1634
stephen hemminger1d143d92013-12-29 14:01:29 -08001635static int call_netdevice_notifiers_info(unsigned long val,
1636 struct net_device *dev,
1637 struct netdev_notifier_info *info)
Jiri Pirko351638e2013-05-28 01:30:21 +00001638{
1639 ASSERT_RTNL();
1640 netdev_notifier_info_init(info, dev);
1641 return raw_notifier_call_chain(&netdev_chain, val, info);
1642}
Jiri Pirko351638e2013-05-28 01:30:21 +00001643
1644/**
Linus Torvalds1da177e2005-04-16 15:20:36 -07001645 * call_netdevice_notifiers - call all network notifier blocks
1646 * @val: value passed unmodified to notifier function
Randy Dunlapc4ea43c2007-10-12 21:17:49 -07001647 * @dev: net_device pointer passed unmodified to notifier function
Linus Torvalds1da177e2005-04-16 15:20:36 -07001648 *
1649 * Call all network notifier blocks. Parameters and return value
Alan Sternf07d5b92006-05-09 15:23:03 -07001650 * are as for raw_notifier_call_chain().
Linus Torvalds1da177e2005-04-16 15:20:36 -07001651 */
1652
Eric W. Biedermanad7379d2007-09-16 15:33:32 -07001653int call_netdevice_notifiers(unsigned long val, struct net_device *dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001654{
Jiri Pirko351638e2013-05-28 01:30:21 +00001655 struct netdev_notifier_info info;
1656
1657 return call_netdevice_notifiers_info(val, dev, &info);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001658}
stephen hemmingeredf947f2011-03-24 13:24:01 +00001659EXPORT_SYMBOL(call_netdevice_notifiers);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001660
Pablo Neira1cf519002015-05-13 18:19:37 +02001661#ifdef CONFIG_NET_INGRESS
Daniel Borkmann45771392015-04-10 23:07:54 +02001662static struct static_key ingress_needed __read_mostly;
1663
1664void net_inc_ingress_queue(void)
1665{
1666 static_key_slow_inc(&ingress_needed);
1667}
1668EXPORT_SYMBOL_GPL(net_inc_ingress_queue);
1669
1670void net_dec_ingress_queue(void)
1671{
1672 static_key_slow_dec(&ingress_needed);
1673}
1674EXPORT_SYMBOL_GPL(net_dec_ingress_queue);
1675#endif
1676
Ingo Molnarc5905af2012-02-24 08:31:31 +01001677static struct static_key netstamp_needed __read_mostly;
Eric Dumazetb90e5792011-11-28 11:16:50 +00001678#ifdef HAVE_JUMP_LABEL
Ingo Molnarc5905af2012-02-24 08:31:31 +01001679/* We are not allowed to call static_key_slow_dec() from irq context
Eric Dumazetb90e5792011-11-28 11:16:50 +00001680 * If net_disable_timestamp() is called from irq context, defer the
Ingo Molnarc5905af2012-02-24 08:31:31 +01001681 * static_key_slow_dec() calls.
Eric Dumazetb90e5792011-11-28 11:16:50 +00001682 */
1683static atomic_t netstamp_needed_deferred;
1684#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07001685
1686void net_enable_timestamp(void)
1687{
Eric Dumazetb90e5792011-11-28 11:16:50 +00001688#ifdef HAVE_JUMP_LABEL
1689 int deferred = atomic_xchg(&netstamp_needed_deferred, 0);
1690
1691 if (deferred) {
1692 while (--deferred)
Ingo Molnarc5905af2012-02-24 08:31:31 +01001693 static_key_slow_dec(&netstamp_needed);
Eric Dumazetb90e5792011-11-28 11:16:50 +00001694 return;
1695 }
1696#endif
Ingo Molnarc5905af2012-02-24 08:31:31 +01001697 static_key_slow_inc(&netstamp_needed);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001698}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001699EXPORT_SYMBOL(net_enable_timestamp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001700
1701void net_disable_timestamp(void)
1702{
Eric Dumazetb90e5792011-11-28 11:16:50 +00001703#ifdef HAVE_JUMP_LABEL
1704 if (in_interrupt()) {
1705 atomic_inc(&netstamp_needed_deferred);
1706 return;
1707 }
1708#endif
Ingo Molnarc5905af2012-02-24 08:31:31 +01001709 static_key_slow_dec(&netstamp_needed);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001710}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001711EXPORT_SYMBOL(net_disable_timestamp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001712
Eric Dumazet3b098e22010-05-15 23:57:10 -07001713static inline void net_timestamp_set(struct sk_buff *skb)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001714{
Eric Dumazet588f0332011-11-15 04:12:55 +00001715 skb->tstamp.tv64 = 0;
Ingo Molnarc5905af2012-02-24 08:31:31 +01001716 if (static_key_false(&netstamp_needed))
Patrick McHardya61bbcf2005-08-14 17:24:31 -07001717 __net_timestamp(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001718}
1719
Eric Dumazet588f0332011-11-15 04:12:55 +00001720#define net_timestamp_check(COND, SKB) \
Ingo Molnarc5905af2012-02-24 08:31:31 +01001721 if (static_key_false(&netstamp_needed)) { \
Eric Dumazet588f0332011-11-15 04:12:55 +00001722 if ((COND) && !(SKB)->tstamp.tv64) \
1723 __net_timestamp(SKB); \
1724 } \
Eric Dumazet3b098e22010-05-15 23:57:10 -07001725
Vlad Yasevich1ee481f2014-03-27 17:32:29 -04001726bool is_skb_forwardable(struct net_device *dev, struct sk_buff *skb)
Daniel Lezcano79b569f2011-03-30 02:42:17 -07001727{
1728 unsigned int len;
1729
1730 if (!(dev->flags & IFF_UP))
1731 return false;
1732
1733 len = dev->mtu + dev->hard_header_len + VLAN_HLEN;
1734 if (skb->len <= len)
1735 return true;
1736
1737 /* if TSO is enabled, we don't care about the length as the packet
1738 * could be forwarded without being segmented before
1739 */
1740 if (skb_is_gso(skb))
1741 return true;
1742
1743 return false;
1744}
Vlad Yasevich1ee481f2014-03-27 17:32:29 -04001745EXPORT_SYMBOL_GPL(is_skb_forwardable);
Daniel Lezcano79b569f2011-03-30 02:42:17 -07001746
Herbert Xua0265d22014-04-17 13:45:03 +08001747int __dev_forward_skb(struct net_device *dev, struct sk_buff *skb)
1748{
Willem de Bruijnbbbf2df2015-06-08 11:53:08 -04001749 if (skb_orphan_frags(skb, GFP_ATOMIC) ||
1750 unlikely(!is_skb_forwardable(dev, skb))) {
Herbert Xua0265d22014-04-17 13:45:03 +08001751 atomic_long_inc(&dev->rx_dropped);
1752 kfree_skb(skb);
1753 return NET_RX_DROP;
1754 }
1755
1756 skb_scrub_packet(skb, true);
WANG Cong08b4b8e2015-03-20 14:29:09 -07001757 skb->priority = 0;
Herbert Xua0265d22014-04-17 13:45:03 +08001758 skb->protocol = eth_type_trans(skb, dev);
Jay Vosburgh2c26d342014-12-19 15:32:00 -08001759 skb_postpull_rcsum(skb, eth_hdr(skb), ETH_HLEN);
Herbert Xua0265d22014-04-17 13:45:03 +08001760
1761 return 0;
1762}
1763EXPORT_SYMBOL_GPL(__dev_forward_skb);
1764
Arnd Bergmann44540962009-11-26 06:07:08 +00001765/**
1766 * dev_forward_skb - loopback an skb to another netif
1767 *
1768 * @dev: destination network device
1769 * @skb: buffer to forward
1770 *
1771 * return values:
1772 * NET_RX_SUCCESS (no congestion)
Eric Dumazet6ec82562010-05-06 00:53:53 -07001773 * NET_RX_DROP (packet was dropped, but freed)
Arnd Bergmann44540962009-11-26 06:07:08 +00001774 *
1775 * dev_forward_skb can be used for injecting an skb from the
1776 * start_xmit function of one device into the receive queue
1777 * of another device.
1778 *
1779 * The receiving device may be in another namespace, so
1780 * we have to clear all information in the skb that could
1781 * impact namespace isolation.
1782 */
1783int dev_forward_skb(struct net_device *dev, struct sk_buff *skb)
1784{
Herbert Xua0265d22014-04-17 13:45:03 +08001785 return __dev_forward_skb(dev, skb) ?: netif_rx_internal(skb);
Arnd Bergmann44540962009-11-26 06:07:08 +00001786}
1787EXPORT_SYMBOL_GPL(dev_forward_skb);
1788
Changli Gao71d9dec2010-12-15 19:57:25 +00001789static inline int deliver_skb(struct sk_buff *skb,
1790 struct packet_type *pt_prev,
1791 struct net_device *orig_dev)
1792{
Michael S. Tsirkin1080e512012-07-20 09:23:17 +00001793 if (unlikely(skb_orphan_frags(skb, GFP_ATOMIC)))
1794 return -ENOMEM;
Changli Gao71d9dec2010-12-15 19:57:25 +00001795 atomic_inc(&skb->users);
1796 return pt_prev->func(skb, skb->dev, pt_prev, orig_dev);
1797}
1798
Salam Noureddine7866a622015-01-27 11:35:48 -08001799static inline void deliver_ptype_list_skb(struct sk_buff *skb,
1800 struct packet_type **pt,
Jiri Pirkofbcb2172015-03-30 16:56:01 +02001801 struct net_device *orig_dev,
1802 __be16 type,
Salam Noureddine7866a622015-01-27 11:35:48 -08001803 struct list_head *ptype_list)
1804{
1805 struct packet_type *ptype, *pt_prev = *pt;
1806
1807 list_for_each_entry_rcu(ptype, ptype_list, list) {
1808 if (ptype->type != type)
1809 continue;
1810 if (pt_prev)
Jiri Pirkofbcb2172015-03-30 16:56:01 +02001811 deliver_skb(skb, pt_prev, orig_dev);
Salam Noureddine7866a622015-01-27 11:35:48 -08001812 pt_prev = ptype;
1813 }
1814 *pt = pt_prev;
1815}
1816
Eric Leblondc0de08d2012-08-16 22:02:58 +00001817static inline bool skb_loop_sk(struct packet_type *ptype, struct sk_buff *skb)
1818{
Eric Leblonda3d744e2012-11-06 02:10:10 +00001819 if (!ptype->af_packet_priv || !skb->sk)
Eric Leblondc0de08d2012-08-16 22:02:58 +00001820 return false;
1821
1822 if (ptype->id_match)
1823 return ptype->id_match(ptype, skb->sk);
1824 else if ((struct sock *)ptype->af_packet_priv == skb->sk)
1825 return true;
1826
1827 return false;
1828}
1829
Linus Torvalds1da177e2005-04-16 15:20:36 -07001830/*
1831 * Support routine. Sends outgoing frames to any network
1832 * taps currently in use.
1833 */
1834
Herbert Xuf6a78bf2006-06-22 02:57:17 -07001835static void dev_queue_xmit_nit(struct sk_buff *skb, struct net_device *dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001836{
1837 struct packet_type *ptype;
Changli Gao71d9dec2010-12-15 19:57:25 +00001838 struct sk_buff *skb2 = NULL;
1839 struct packet_type *pt_prev = NULL;
Salam Noureddine7866a622015-01-27 11:35:48 -08001840 struct list_head *ptype_list = &ptype_all;
Patrick McHardya61bbcf2005-08-14 17:24:31 -07001841
Linus Torvalds1da177e2005-04-16 15:20:36 -07001842 rcu_read_lock();
Salam Noureddine7866a622015-01-27 11:35:48 -08001843again:
1844 list_for_each_entry_rcu(ptype, ptype_list, list) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001845 /* Never send packets back to the socket
1846 * they originated from - MvS (miquels@drinkel.ow.org)
1847 */
Salam Noureddine7866a622015-01-27 11:35:48 -08001848 if (skb_loop_sk(ptype, skb))
1849 continue;
Changli Gao71d9dec2010-12-15 19:57:25 +00001850
Salam Noureddine7866a622015-01-27 11:35:48 -08001851 if (pt_prev) {
1852 deliver_skb(skb2, pt_prev, skb->dev);
Changli Gao71d9dec2010-12-15 19:57:25 +00001853 pt_prev = ptype;
Salam Noureddine7866a622015-01-27 11:35:48 -08001854 continue;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001855 }
Salam Noureddine7866a622015-01-27 11:35:48 -08001856
1857 /* need to clone skb, done only once */
1858 skb2 = skb_clone(skb, GFP_ATOMIC);
1859 if (!skb2)
1860 goto out_unlock;
1861
1862 net_timestamp_set(skb2);
1863
1864 /* skb->nh should be correctly
1865 * set by sender, so that the second statement is
1866 * just protection against buggy protocols.
1867 */
1868 skb_reset_mac_header(skb2);
1869
1870 if (skb_network_header(skb2) < skb2->data ||
1871 skb_network_header(skb2) > skb_tail_pointer(skb2)) {
1872 net_crit_ratelimited("protocol %04x is buggy, dev %s\n",
1873 ntohs(skb2->protocol),
1874 dev->name);
1875 skb_reset_network_header(skb2);
1876 }
1877
1878 skb2->transport_header = skb2->network_header;
1879 skb2->pkt_type = PACKET_OUTGOING;
1880 pt_prev = ptype;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001881 }
Salam Noureddine7866a622015-01-27 11:35:48 -08001882
1883 if (ptype_list == &ptype_all) {
1884 ptype_list = &dev->ptype_all;
1885 goto again;
1886 }
1887out_unlock:
Changli Gao71d9dec2010-12-15 19:57:25 +00001888 if (pt_prev)
1889 pt_prev->func(skb2, skb->dev, pt_prev, skb->dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001890 rcu_read_unlock();
1891}
1892
Ben Hutchings2c530402012-07-10 10:55:09 +00001893/**
1894 * netif_setup_tc - Handle tc mappings on real_num_tx_queues change
John Fastabend4f57c082011-01-17 08:06:04 +00001895 * @dev: Network device
1896 * @txq: number of queues available
1897 *
1898 * If real_num_tx_queues is changed the tc mappings may no longer be
1899 * valid. To resolve this verify the tc mapping remains valid and if
1900 * not NULL the mapping. With no priorities mapping to this
1901 * offset/count pair it will no longer be used. In the worst case TC0
1902 * is invalid nothing can be done so disable priority mappings. If is
1903 * expected that drivers will fix this mapping if they can before
1904 * calling netif_set_real_num_tx_queues.
1905 */
Eric Dumazetbb134d22011-01-20 19:18:08 +00001906static void netif_setup_tc(struct net_device *dev, unsigned int txq)
John Fastabend4f57c082011-01-17 08:06:04 +00001907{
1908 int i;
1909 struct netdev_tc_txq *tc = &dev->tc_to_txq[0];
1910
1911 /* If TC0 is invalidated disable TC mapping */
1912 if (tc->offset + tc->count > txq) {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00001913 pr_warn("Number of in use tx queues changed invalidating tc mappings. Priority traffic classification disabled!\n");
John Fastabend4f57c082011-01-17 08:06:04 +00001914 dev->num_tc = 0;
1915 return;
1916 }
1917
1918 /* Invalidated prio to tc mappings set to TC0 */
1919 for (i = 1; i < TC_BITMASK + 1; i++) {
1920 int q = netdev_get_prio_tc_map(dev, i);
1921
1922 tc = &dev->tc_to_txq[q];
1923 if (tc->offset + tc->count > txq) {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00001924 pr_warn("Number of in use tx queues changed. Priority %i to tc mapping %i is no longer valid. Setting map to 0\n",
1925 i, q);
John Fastabend4f57c082011-01-17 08:06:04 +00001926 netdev_set_prio_tc_map(dev, i, 0);
1927 }
1928 }
1929}
1930
Alexander Duyck537c00d2013-01-10 08:57:02 +00001931#ifdef CONFIG_XPS
1932static DEFINE_MUTEX(xps_map_mutex);
1933#define xmap_dereference(P) \
1934 rcu_dereference_protected((P), lockdep_is_held(&xps_map_mutex))
1935
Alexander Duyck10cdc3f2013-01-10 08:57:17 +00001936static struct xps_map *remove_xps_queue(struct xps_dev_maps *dev_maps,
1937 int cpu, u16 index)
1938{
1939 struct xps_map *map = NULL;
1940 int pos;
1941
1942 if (dev_maps)
1943 map = xmap_dereference(dev_maps->cpu_map[cpu]);
1944
1945 for (pos = 0; map && pos < map->len; pos++) {
1946 if (map->queues[pos] == index) {
1947 if (map->len > 1) {
1948 map->queues[pos] = map->queues[--map->len];
1949 } else {
1950 RCU_INIT_POINTER(dev_maps->cpu_map[cpu], NULL);
1951 kfree_rcu(map, rcu);
1952 map = NULL;
1953 }
1954 break;
1955 }
1956 }
1957
1958 return map;
1959}
1960
Alexander Duyck024e9672013-01-10 08:57:46 +00001961static void netif_reset_xps_queues_gt(struct net_device *dev, u16 index)
Alexander Duyck537c00d2013-01-10 08:57:02 +00001962{
1963 struct xps_dev_maps *dev_maps;
Alexander Duyck024e9672013-01-10 08:57:46 +00001964 int cpu, i;
Alexander Duyck10cdc3f2013-01-10 08:57:17 +00001965 bool active = false;
Alexander Duyck537c00d2013-01-10 08:57:02 +00001966
1967 mutex_lock(&xps_map_mutex);
1968 dev_maps = xmap_dereference(dev->xps_maps);
1969
1970 if (!dev_maps)
1971 goto out_no_maps;
1972
Alexander Duyck10cdc3f2013-01-10 08:57:17 +00001973 for_each_possible_cpu(cpu) {
Alexander Duyck024e9672013-01-10 08:57:46 +00001974 for (i = index; i < dev->num_tx_queues; i++) {
1975 if (!remove_xps_queue(dev_maps, cpu, i))
1976 break;
1977 }
1978 if (i == dev->num_tx_queues)
Alexander Duyck10cdc3f2013-01-10 08:57:17 +00001979 active = true;
Alexander Duyck537c00d2013-01-10 08:57:02 +00001980 }
1981
Alexander Duyck10cdc3f2013-01-10 08:57:17 +00001982 if (!active) {
Alexander Duyck537c00d2013-01-10 08:57:02 +00001983 RCU_INIT_POINTER(dev->xps_maps, NULL);
1984 kfree_rcu(dev_maps, rcu);
1985 }
1986
Alexander Duyck024e9672013-01-10 08:57:46 +00001987 for (i = index; i < dev->num_tx_queues; i++)
1988 netdev_queue_numa_node_write(netdev_get_tx_queue(dev, i),
1989 NUMA_NO_NODE);
1990
Alexander Duyck537c00d2013-01-10 08:57:02 +00001991out_no_maps:
1992 mutex_unlock(&xps_map_mutex);
1993}
1994
Alexander Duyck01c5f862013-01-10 08:57:35 +00001995static struct xps_map *expand_xps_map(struct xps_map *map,
1996 int cpu, u16 index)
1997{
1998 struct xps_map *new_map;
1999 int alloc_len = XPS_MIN_MAP_ALLOC;
2000 int i, pos;
2001
2002 for (pos = 0; map && pos < map->len; pos++) {
2003 if (map->queues[pos] != index)
2004 continue;
2005 return map;
2006 }
2007
2008 /* Need to add queue to this CPU's existing map */
2009 if (map) {
2010 if (pos < map->alloc_len)
2011 return map;
2012
2013 alloc_len = map->alloc_len * 2;
2014 }
2015
2016 /* Need to allocate new map to store queue on this CPU's map */
2017 new_map = kzalloc_node(XPS_MAP_SIZE(alloc_len), GFP_KERNEL,
2018 cpu_to_node(cpu));
2019 if (!new_map)
2020 return NULL;
2021
2022 for (i = 0; i < pos; i++)
2023 new_map->queues[i] = map->queues[i];
2024 new_map->alloc_len = alloc_len;
2025 new_map->len = pos;
2026
2027 return new_map;
2028}
2029
Michael S. Tsirkin35735402013-10-02 09:14:06 +03002030int netif_set_xps_queue(struct net_device *dev, const struct cpumask *mask,
2031 u16 index)
Alexander Duyck537c00d2013-01-10 08:57:02 +00002032{
Alexander Duyck01c5f862013-01-10 08:57:35 +00002033 struct xps_dev_maps *dev_maps, *new_dev_maps = NULL;
Alexander Duyck537c00d2013-01-10 08:57:02 +00002034 struct xps_map *map, *new_map;
Alexander Duyck537c00d2013-01-10 08:57:02 +00002035 int maps_sz = max_t(unsigned int, XPS_DEV_MAPS_SIZE, L1_CACHE_BYTES);
Alexander Duyck01c5f862013-01-10 08:57:35 +00002036 int cpu, numa_node_id = -2;
2037 bool active = false;
Alexander Duyck537c00d2013-01-10 08:57:02 +00002038
2039 mutex_lock(&xps_map_mutex);
2040
2041 dev_maps = xmap_dereference(dev->xps_maps);
2042
Alexander Duyck01c5f862013-01-10 08:57:35 +00002043 /* allocate memory for queue storage */
2044 for_each_online_cpu(cpu) {
2045 if (!cpumask_test_cpu(cpu, mask))
2046 continue;
Alexander Duyck537c00d2013-01-10 08:57:02 +00002047
Alexander Duyck01c5f862013-01-10 08:57:35 +00002048 if (!new_dev_maps)
2049 new_dev_maps = kzalloc(maps_sz, GFP_KERNEL);
Alexander Duyck2bb60cb2013-02-22 06:38:44 +00002050 if (!new_dev_maps) {
2051 mutex_unlock(&xps_map_mutex);
Alexander Duyck01c5f862013-01-10 08:57:35 +00002052 return -ENOMEM;
Alexander Duyck2bb60cb2013-02-22 06:38:44 +00002053 }
Alexander Duyck01c5f862013-01-10 08:57:35 +00002054
2055 map = dev_maps ? xmap_dereference(dev_maps->cpu_map[cpu]) :
2056 NULL;
2057
2058 map = expand_xps_map(map, cpu, index);
2059 if (!map)
2060 goto error;
2061
2062 RCU_INIT_POINTER(new_dev_maps->cpu_map[cpu], map);
2063 }
2064
2065 if (!new_dev_maps)
2066 goto out_no_new_maps;
2067
2068 for_each_possible_cpu(cpu) {
2069 if (cpumask_test_cpu(cpu, mask) && cpu_online(cpu)) {
2070 /* add queue to CPU maps */
2071 int pos = 0;
2072
2073 map = xmap_dereference(new_dev_maps->cpu_map[cpu]);
2074 while ((pos < map->len) && (map->queues[pos] != index))
2075 pos++;
2076
2077 if (pos == map->len)
2078 map->queues[map->len++] = index;
Alexander Duyck537c00d2013-01-10 08:57:02 +00002079#ifdef CONFIG_NUMA
Alexander Duyck537c00d2013-01-10 08:57:02 +00002080 if (numa_node_id == -2)
2081 numa_node_id = cpu_to_node(cpu);
2082 else if (numa_node_id != cpu_to_node(cpu))
2083 numa_node_id = -1;
Alexander Duyck537c00d2013-01-10 08:57:02 +00002084#endif
Alexander Duyck01c5f862013-01-10 08:57:35 +00002085 } else if (dev_maps) {
2086 /* fill in the new device map from the old device map */
2087 map = xmap_dereference(dev_maps->cpu_map[cpu]);
2088 RCU_INIT_POINTER(new_dev_maps->cpu_map[cpu], map);
Alexander Duyck537c00d2013-01-10 08:57:02 +00002089 }
Alexander Duyck01c5f862013-01-10 08:57:35 +00002090
Alexander Duyck537c00d2013-01-10 08:57:02 +00002091 }
2092
Alexander Duyck01c5f862013-01-10 08:57:35 +00002093 rcu_assign_pointer(dev->xps_maps, new_dev_maps);
2094
Alexander Duyck537c00d2013-01-10 08:57:02 +00002095 /* Cleanup old maps */
Alexander Duyck01c5f862013-01-10 08:57:35 +00002096 if (dev_maps) {
2097 for_each_possible_cpu(cpu) {
2098 new_map = xmap_dereference(new_dev_maps->cpu_map[cpu]);
2099 map = xmap_dereference(dev_maps->cpu_map[cpu]);
2100 if (map && map != new_map)
2101 kfree_rcu(map, rcu);
2102 }
Alexander Duyck537c00d2013-01-10 08:57:02 +00002103
Alexander Duyck537c00d2013-01-10 08:57:02 +00002104 kfree_rcu(dev_maps, rcu);
Alexander Duyck01c5f862013-01-10 08:57:35 +00002105 }
Alexander Duyck537c00d2013-01-10 08:57:02 +00002106
Alexander Duyck01c5f862013-01-10 08:57:35 +00002107 dev_maps = new_dev_maps;
2108 active = true;
2109
2110out_no_new_maps:
2111 /* update Tx queue numa node */
Alexander Duyck537c00d2013-01-10 08:57:02 +00002112 netdev_queue_numa_node_write(netdev_get_tx_queue(dev, index),
2113 (numa_node_id >= 0) ? numa_node_id :
2114 NUMA_NO_NODE);
2115
Alexander Duyck01c5f862013-01-10 08:57:35 +00002116 if (!dev_maps)
2117 goto out_no_maps;
2118
2119 /* removes queue from unused CPUs */
2120 for_each_possible_cpu(cpu) {
2121 if (cpumask_test_cpu(cpu, mask) && cpu_online(cpu))
2122 continue;
2123
2124 if (remove_xps_queue(dev_maps, cpu, index))
2125 active = true;
2126 }
2127
2128 /* free map if not active */
2129 if (!active) {
2130 RCU_INIT_POINTER(dev->xps_maps, NULL);
2131 kfree_rcu(dev_maps, rcu);
2132 }
2133
2134out_no_maps:
Alexander Duyck537c00d2013-01-10 08:57:02 +00002135 mutex_unlock(&xps_map_mutex);
2136
2137 return 0;
2138error:
Alexander Duyck01c5f862013-01-10 08:57:35 +00002139 /* remove any maps that we added */
2140 for_each_possible_cpu(cpu) {
2141 new_map = xmap_dereference(new_dev_maps->cpu_map[cpu]);
2142 map = dev_maps ? xmap_dereference(dev_maps->cpu_map[cpu]) :
2143 NULL;
2144 if (new_map && new_map != map)
2145 kfree(new_map);
2146 }
2147
Alexander Duyck537c00d2013-01-10 08:57:02 +00002148 mutex_unlock(&xps_map_mutex);
2149
Alexander Duyck537c00d2013-01-10 08:57:02 +00002150 kfree(new_dev_maps);
2151 return -ENOMEM;
2152}
2153EXPORT_SYMBOL(netif_set_xps_queue);
2154
2155#endif
John Fastabendf0796d52010-07-01 13:21:57 +00002156/*
2157 * Routine to help set real_num_tx_queues. To avoid skbs mapped to queues
2158 * greater then real_num_tx_queues stale skbs on the qdisc must be flushed.
2159 */
Tom Herberte6484932010-10-18 18:04:39 +00002160int netif_set_real_num_tx_queues(struct net_device *dev, unsigned int txq)
John Fastabendf0796d52010-07-01 13:21:57 +00002161{
Tom Herbert1d24eb42010-11-21 13:17:27 +00002162 int rc;
2163
Tom Herberte6484932010-10-18 18:04:39 +00002164 if (txq < 1 || txq > dev->num_tx_queues)
2165 return -EINVAL;
John Fastabendf0796d52010-07-01 13:21:57 +00002166
Ben Hutchings5c565802011-02-15 19:39:21 +00002167 if (dev->reg_state == NETREG_REGISTERED ||
2168 dev->reg_state == NETREG_UNREGISTERING) {
Tom Herberte6484932010-10-18 18:04:39 +00002169 ASSERT_RTNL();
2170
Tom Herbert1d24eb42010-11-21 13:17:27 +00002171 rc = netdev_queue_update_kobjects(dev, dev->real_num_tx_queues,
2172 txq);
Tom Herbertbf264142010-11-26 08:36:09 +00002173 if (rc)
2174 return rc;
2175
John Fastabend4f57c082011-01-17 08:06:04 +00002176 if (dev->num_tc)
2177 netif_setup_tc(dev, txq);
2178
Alexander Duyck024e9672013-01-10 08:57:46 +00002179 if (txq < dev->real_num_tx_queues) {
Tom Herberte6484932010-10-18 18:04:39 +00002180 qdisc_reset_all_tx_gt(dev, txq);
Alexander Duyck024e9672013-01-10 08:57:46 +00002181#ifdef CONFIG_XPS
2182 netif_reset_xps_queues_gt(dev, txq);
2183#endif
2184 }
John Fastabendf0796d52010-07-01 13:21:57 +00002185 }
Tom Herberte6484932010-10-18 18:04:39 +00002186
2187 dev->real_num_tx_queues = txq;
2188 return 0;
John Fastabendf0796d52010-07-01 13:21:57 +00002189}
2190EXPORT_SYMBOL(netif_set_real_num_tx_queues);
Denis Vlasenko56079432006-03-29 15:57:29 -08002191
Michael Daltona953be52014-01-16 22:23:28 -08002192#ifdef CONFIG_SYSFS
Ben Hutchings62fe0b42010-09-27 08:24:33 +00002193/**
2194 * netif_set_real_num_rx_queues - set actual number of RX queues used
2195 * @dev: Network device
2196 * @rxq: Actual number of RX queues
2197 *
2198 * This must be called either with the rtnl_lock held or before
2199 * registration of the net device. Returns 0 on success, or a
Ben Hutchings4e7f7952010-10-08 10:33:39 -07002200 * negative error code. If called before registration, it always
2201 * succeeds.
Ben Hutchings62fe0b42010-09-27 08:24:33 +00002202 */
2203int netif_set_real_num_rx_queues(struct net_device *dev, unsigned int rxq)
2204{
2205 int rc;
2206
Tom Herbertbd25fa72010-10-18 18:00:16 +00002207 if (rxq < 1 || rxq > dev->num_rx_queues)
2208 return -EINVAL;
2209
Ben Hutchings62fe0b42010-09-27 08:24:33 +00002210 if (dev->reg_state == NETREG_REGISTERED) {
2211 ASSERT_RTNL();
2212
Ben Hutchings62fe0b42010-09-27 08:24:33 +00002213 rc = net_rx_queue_update_kobjects(dev, dev->real_num_rx_queues,
2214 rxq);
2215 if (rc)
2216 return rc;
Ben Hutchings62fe0b42010-09-27 08:24:33 +00002217 }
2218
2219 dev->real_num_rx_queues = rxq;
2220 return 0;
2221}
2222EXPORT_SYMBOL(netif_set_real_num_rx_queues);
2223#endif
2224
Ben Hutchings2c530402012-07-10 10:55:09 +00002225/**
2226 * netif_get_num_default_rss_queues - default number of RSS queues
Yuval Mintz16917b82012-07-01 03:18:50 +00002227 *
2228 * This routine should set an upper limit on the number of RSS queues
2229 * used by default by multiqueue devices.
2230 */
Ben Hutchingsa55b1382012-07-10 10:54:38 +00002231int netif_get_num_default_rss_queues(void)
Yuval Mintz16917b82012-07-01 03:18:50 +00002232{
2233 return min_t(int, DEFAULT_MAX_NUM_RSS_QUEUES, num_online_cpus());
2234}
2235EXPORT_SYMBOL(netif_get_num_default_rss_queues);
2236
Jarek Poplawskidef82a12008-08-17 21:54:43 -07002237static inline void __netif_reschedule(struct Qdisc *q)
2238{
2239 struct softnet_data *sd;
2240 unsigned long flags;
2241
2242 local_irq_save(flags);
Christoph Lameter903ceff2014-08-17 12:30:35 -05002243 sd = this_cpu_ptr(&softnet_data);
Changli Gaoa9cbd582010-04-26 23:06:24 +00002244 q->next_sched = NULL;
2245 *sd->output_queue_tailp = q;
2246 sd->output_queue_tailp = &q->next_sched;
Jarek Poplawskidef82a12008-08-17 21:54:43 -07002247 raise_softirq_irqoff(NET_TX_SOFTIRQ);
2248 local_irq_restore(flags);
2249}
2250
David S. Miller37437bb2008-07-16 02:15:04 -07002251void __netif_schedule(struct Qdisc *q)
Denis Vlasenko56079432006-03-29 15:57:29 -08002252{
Jarek Poplawskidef82a12008-08-17 21:54:43 -07002253 if (!test_and_set_bit(__QDISC_STATE_SCHED, &q->state))
2254 __netif_reschedule(q);
Denis Vlasenko56079432006-03-29 15:57:29 -08002255}
2256EXPORT_SYMBOL(__netif_schedule);
2257
Eric Dumazete6247022013-12-05 04:45:08 -08002258struct dev_kfree_skb_cb {
2259 enum skb_free_reason reason;
2260};
2261
2262static struct dev_kfree_skb_cb *get_kfree_skb_cb(const struct sk_buff *skb)
Denis Vlasenko56079432006-03-29 15:57:29 -08002263{
Eric Dumazete6247022013-12-05 04:45:08 -08002264 return (struct dev_kfree_skb_cb *)skb->cb;
Denis Vlasenko56079432006-03-29 15:57:29 -08002265}
Denis Vlasenko56079432006-03-29 15:57:29 -08002266
John Fastabend46e5da42014-09-12 20:04:52 -07002267void netif_schedule_queue(struct netdev_queue *txq)
2268{
2269 rcu_read_lock();
2270 if (!(txq->state & QUEUE_STATE_ANY_XOFF)) {
2271 struct Qdisc *q = rcu_dereference(txq->qdisc);
2272
2273 __netif_schedule(q);
2274 }
2275 rcu_read_unlock();
2276}
2277EXPORT_SYMBOL(netif_schedule_queue);
2278
2279/**
2280 * netif_wake_subqueue - allow sending packets on subqueue
2281 * @dev: network device
2282 * @queue_index: sub queue index
2283 *
2284 * Resume individual transmit queue of a device with multiple transmit queues.
2285 */
2286void netif_wake_subqueue(struct net_device *dev, u16 queue_index)
2287{
2288 struct netdev_queue *txq = netdev_get_tx_queue(dev, queue_index);
2289
2290 if (test_and_clear_bit(__QUEUE_STATE_DRV_XOFF, &txq->state)) {
2291 struct Qdisc *q;
2292
2293 rcu_read_lock();
2294 q = rcu_dereference(txq->qdisc);
2295 __netif_schedule(q);
2296 rcu_read_unlock();
2297 }
2298}
2299EXPORT_SYMBOL(netif_wake_subqueue);
2300
2301void netif_tx_wake_queue(struct netdev_queue *dev_queue)
2302{
2303 if (test_and_clear_bit(__QUEUE_STATE_DRV_XOFF, &dev_queue->state)) {
2304 struct Qdisc *q;
2305
2306 rcu_read_lock();
2307 q = rcu_dereference(dev_queue->qdisc);
2308 __netif_schedule(q);
2309 rcu_read_unlock();
2310 }
2311}
2312EXPORT_SYMBOL(netif_tx_wake_queue);
2313
Eric Dumazete6247022013-12-05 04:45:08 -08002314void __dev_kfree_skb_irq(struct sk_buff *skb, enum skb_free_reason reason)
2315{
2316 unsigned long flags;
2317
2318 if (likely(atomic_read(&skb->users) == 1)) {
2319 smp_rmb();
2320 atomic_set(&skb->users, 0);
2321 } else if (likely(!atomic_dec_and_test(&skb->users))) {
2322 return;
2323 }
2324 get_kfree_skb_cb(skb)->reason = reason;
2325 local_irq_save(flags);
2326 skb->next = __this_cpu_read(softnet_data.completion_queue);
2327 __this_cpu_write(softnet_data.completion_queue, skb);
2328 raise_softirq_irqoff(NET_TX_SOFTIRQ);
2329 local_irq_restore(flags);
2330}
2331EXPORT_SYMBOL(__dev_kfree_skb_irq);
2332
2333void __dev_kfree_skb_any(struct sk_buff *skb, enum skb_free_reason reason)
Denis Vlasenko56079432006-03-29 15:57:29 -08002334{
2335 if (in_irq() || irqs_disabled())
Eric Dumazete6247022013-12-05 04:45:08 -08002336 __dev_kfree_skb_irq(skb, reason);
Denis Vlasenko56079432006-03-29 15:57:29 -08002337 else
2338 dev_kfree_skb(skb);
2339}
Eric Dumazete6247022013-12-05 04:45:08 -08002340EXPORT_SYMBOL(__dev_kfree_skb_any);
Denis Vlasenko56079432006-03-29 15:57:29 -08002341
2342
Stephen Hemmingerbea33482007-10-03 16:41:36 -07002343/**
2344 * netif_device_detach - mark device as removed
2345 * @dev: network device
2346 *
2347 * Mark device as removed from system and therefore no longer available.
2348 */
Denis Vlasenko56079432006-03-29 15:57:29 -08002349void netif_device_detach(struct net_device *dev)
2350{
2351 if (test_and_clear_bit(__LINK_STATE_PRESENT, &dev->state) &&
2352 netif_running(dev)) {
Alexander Duyckd5431032009-04-08 13:15:22 +00002353 netif_tx_stop_all_queues(dev);
Denis Vlasenko56079432006-03-29 15:57:29 -08002354 }
2355}
2356EXPORT_SYMBOL(netif_device_detach);
2357
Stephen Hemmingerbea33482007-10-03 16:41:36 -07002358/**
2359 * netif_device_attach - mark device as attached
2360 * @dev: network device
2361 *
2362 * Mark device as attached from system and restart if needed.
2363 */
Denis Vlasenko56079432006-03-29 15:57:29 -08002364void netif_device_attach(struct net_device *dev)
2365{
2366 if (!test_and_set_bit(__LINK_STATE_PRESENT, &dev->state) &&
2367 netif_running(dev)) {
Alexander Duyckd5431032009-04-08 13:15:22 +00002368 netif_tx_wake_all_queues(dev);
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09002369 __netdev_watchdog_up(dev);
Denis Vlasenko56079432006-03-29 15:57:29 -08002370 }
2371}
2372EXPORT_SYMBOL(netif_device_attach);
2373
Jiri Pirko5605c762015-05-12 14:56:12 +02002374/*
2375 * Returns a Tx hash based on the given packet descriptor a Tx queues' number
2376 * to be used as a distribution range.
2377 */
2378u16 __skb_tx_hash(const struct net_device *dev, struct sk_buff *skb,
2379 unsigned int num_tx_queues)
2380{
2381 u32 hash;
2382 u16 qoffset = 0;
2383 u16 qcount = num_tx_queues;
2384
2385 if (skb_rx_queue_recorded(skb)) {
2386 hash = skb_get_rx_queue(skb);
2387 while (unlikely(hash >= num_tx_queues))
2388 hash -= num_tx_queues;
2389 return hash;
2390 }
2391
2392 if (dev->num_tc) {
2393 u8 tc = netdev_get_prio_tc_map(dev, skb->priority);
2394 qoffset = dev->tc_to_txq[tc].offset;
2395 qcount = dev->tc_to_txq[tc].count;
2396 }
2397
2398 return (u16) reciprocal_scale(skb_get_hash(skb), qcount) + qoffset;
2399}
2400EXPORT_SYMBOL(__skb_tx_hash);
2401
Ben Hutchings36c92472012-01-17 07:57:56 +00002402static void skb_warn_bad_offload(const struct sk_buff *skb)
2403{
Michał Mirosław65e9d2f2012-01-17 10:00:40 +00002404 static const netdev_features_t null_features = 0;
Ben Hutchings36c92472012-01-17 07:57:56 +00002405 struct net_device *dev = skb->dev;
Bjørn Mork88ad4172015-11-16 19:16:40 +01002406 const char *name = "";
Ben Hutchings36c92472012-01-17 07:57:56 +00002407
Ben Greearc846ad92013-04-19 10:45:52 +00002408 if (!net_ratelimit())
2409 return;
2410
Bjørn Mork88ad4172015-11-16 19:16:40 +01002411 if (dev) {
2412 if (dev->dev.parent)
2413 name = dev_driver_string(dev->dev.parent);
2414 else
2415 name = netdev_name(dev);
2416 }
Ben Hutchings36c92472012-01-17 07:57:56 +00002417 WARN(1, "%s: caps=(%pNF, %pNF) len=%d data_len=%d gso_size=%d "
2418 "gso_type=%d ip_summed=%d\n",
Bjørn Mork88ad4172015-11-16 19:16:40 +01002419 name, dev ? &dev->features : &null_features,
Michał Mirosław65e9d2f2012-01-17 10:00:40 +00002420 skb->sk ? &skb->sk->sk_route_caps : &null_features,
Ben Hutchings36c92472012-01-17 07:57:56 +00002421 skb->len, skb->data_len, skb_shinfo(skb)->gso_size,
2422 skb_shinfo(skb)->gso_type, skb->ip_summed);
2423}
2424
Linus Torvalds1da177e2005-04-16 15:20:36 -07002425/*
2426 * Invalidate hardware checksum when packet is to be mangled, and
2427 * complete checksum manually on outgoing path.
2428 */
Patrick McHardy84fa7932006-08-29 16:44:56 -07002429int skb_checksum_help(struct sk_buff *skb)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002430{
Al Virod3bc23e2006-11-14 21:24:49 -08002431 __wsum csum;
Herbert Xu663ead32007-04-09 11:59:07 -07002432 int ret = 0, offset;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002433
Patrick McHardy84fa7932006-08-29 16:44:56 -07002434 if (skb->ip_summed == CHECKSUM_COMPLETE)
Herbert Xua430a432006-07-08 13:34:56 -07002435 goto out_set_summed;
2436
2437 if (unlikely(skb_shinfo(skb)->gso_size)) {
Ben Hutchings36c92472012-01-17 07:57:56 +00002438 skb_warn_bad_offload(skb);
2439 return -EINVAL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002440 }
2441
Eric Dumazetcef401d2013-01-25 20:34:37 +00002442 /* Before computing a checksum, we should make sure no frag could
2443 * be modified by an external entity : checksum could be wrong.
2444 */
2445 if (skb_has_shared_frag(skb)) {
2446 ret = __skb_linearize(skb);
2447 if (ret)
2448 goto out;
2449 }
2450
Michał Mirosław55508d62010-12-14 15:24:08 +00002451 offset = skb_checksum_start_offset(skb);
Herbert Xua0308472007-10-15 01:47:15 -07002452 BUG_ON(offset >= skb_headlen(skb));
2453 csum = skb_checksum(skb, offset, skb->len - offset, 0);
2454
2455 offset += skb->csum_offset;
2456 BUG_ON(offset + sizeof(__sum16) > skb_headlen(skb));
2457
2458 if (skb_cloned(skb) &&
2459 !skb_clone_writable(skb, offset + sizeof(__sum16))) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07002460 ret = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
2461 if (ret)
2462 goto out;
2463 }
2464
Herbert Xua0308472007-10-15 01:47:15 -07002465 *(__sum16 *)(skb->data + offset) = csum_fold(csum);
Herbert Xua430a432006-07-08 13:34:56 -07002466out_set_summed:
Linus Torvalds1da177e2005-04-16 15:20:36 -07002467 skb->ip_summed = CHECKSUM_NONE;
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09002468out:
Linus Torvalds1da177e2005-04-16 15:20:36 -07002469 return ret;
2470}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07002471EXPORT_SYMBOL(skb_checksum_help);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002472
Vlad Yasevich53d64712014-03-27 17:26:18 -04002473__be16 skb_network_protocol(struct sk_buff *skb, int *depth)
Pravin B Shelarec5f0612013-03-07 09:28:01 +00002474{
2475 __be16 type = skb->protocol;
2476
Pravin B Shelar19acc322013-05-07 20:41:07 +00002477 /* Tunnel gso handlers can set protocol to ethernet. */
2478 if (type == htons(ETH_P_TEB)) {
2479 struct ethhdr *eth;
2480
2481 if (unlikely(!pskb_may_pull(skb, sizeof(struct ethhdr))))
2482 return 0;
2483
2484 eth = (struct ethhdr *)skb_mac_header(skb);
2485 type = eth->h_proto;
2486 }
2487
Toshiaki Makitad4bcef32015-01-29 20:37:07 +09002488 return __vlan_get_protocol(skb, type, depth);
Pravin B Shelarec5f0612013-03-07 09:28:01 +00002489}
2490
Pravin B Shelar05e8ef42013-02-14 09:44:55 +00002491/**
2492 * skb_mac_gso_segment - mac layer segmentation handler.
2493 * @skb: buffer to segment
2494 * @features: features for the output path (see dev->features)
2495 */
2496struct sk_buff *skb_mac_gso_segment(struct sk_buff *skb,
2497 netdev_features_t features)
2498{
2499 struct sk_buff *segs = ERR_PTR(-EPROTONOSUPPORT);
2500 struct packet_offload *ptype;
Vlad Yasevich53d64712014-03-27 17:26:18 -04002501 int vlan_depth = skb->mac_len;
2502 __be16 type = skb_network_protocol(skb, &vlan_depth);
Pravin B Shelar05e8ef42013-02-14 09:44:55 +00002503
Pravin B Shelarec5f0612013-03-07 09:28:01 +00002504 if (unlikely(!type))
2505 return ERR_PTR(-EINVAL);
Pravin B Shelar05e8ef42013-02-14 09:44:55 +00002506
Vlad Yasevich53d64712014-03-27 17:26:18 -04002507 __skb_pull(skb, vlan_depth);
Pravin B Shelar05e8ef42013-02-14 09:44:55 +00002508
2509 rcu_read_lock();
2510 list_for_each_entry_rcu(ptype, &offload_base, list) {
2511 if (ptype->type == type && ptype->callbacks.gso_segment) {
Pravin B Shelar05e8ef42013-02-14 09:44:55 +00002512 segs = ptype->callbacks.gso_segment(skb, features);
2513 break;
2514 }
2515 }
2516 rcu_read_unlock();
2517
2518 __skb_push(skb, skb->data - skb_mac_header(skb));
2519
2520 return segs;
2521}
2522EXPORT_SYMBOL(skb_mac_gso_segment);
2523
2524
Cong Wang12b00042013-02-05 16:36:38 +00002525/* openvswitch calls this on rx path, so we need a different check.
2526 */
2527static inline bool skb_needs_check(struct sk_buff *skb, bool tx_path)
2528{
2529 if (tx_path)
2530 return skb->ip_summed != CHECKSUM_PARTIAL;
2531 else
2532 return skb->ip_summed == CHECKSUM_NONE;
2533}
2534
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002535/**
Cong Wang12b00042013-02-05 16:36:38 +00002536 * __skb_gso_segment - Perform segmentation on skb.
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002537 * @skb: buffer to segment
Herbert Xu576a30e2006-06-27 13:22:38 -07002538 * @features: features for the output path (see dev->features)
Cong Wang12b00042013-02-05 16:36:38 +00002539 * @tx_path: whether it is called in TX path
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002540 *
2541 * This function segments the given skb and returns a list of segments.
Herbert Xu576a30e2006-06-27 13:22:38 -07002542 *
2543 * It may return NULL if the skb requires no segmentation. This is
2544 * only possible when GSO is used for verifying header integrity.
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002545 */
Cong Wang12b00042013-02-05 16:36:38 +00002546struct sk_buff *__skb_gso_segment(struct sk_buff *skb,
2547 netdev_features_t features, bool tx_path)
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002548{
Cong Wang12b00042013-02-05 16:36:38 +00002549 if (unlikely(skb_needs_check(skb, tx_path))) {
Pravin B Shelar05e8ef42013-02-14 09:44:55 +00002550 int err;
2551
Ben Hutchings36c92472012-01-17 07:57:56 +00002552 skb_warn_bad_offload(skb);
Herbert Xu67fd1a72009-01-19 16:26:44 -08002553
françois romieua40e0a62014-07-15 23:55:35 +02002554 err = skb_cow_head(skb, 0);
2555 if (err < 0)
Herbert Xua430a432006-07-08 13:34:56 -07002556 return ERR_PTR(err);
2557 }
2558
Pravin B Shelar68c33162013-02-14 14:02:41 +00002559 SKB_GSO_CB(skb)->mac_offset = skb_headroom(skb);
Eric Dumazet3347c962013-10-19 11:42:56 -07002560 SKB_GSO_CB(skb)->encap_level = 0;
2561
Pravin B Shelar05e8ef42013-02-14 09:44:55 +00002562 skb_reset_mac_header(skb);
2563 skb_reset_mac_len(skb);
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002564
Pravin B Shelar05e8ef42013-02-14 09:44:55 +00002565 return skb_mac_gso_segment(skb, features);
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002566}
Cong Wang12b00042013-02-05 16:36:38 +00002567EXPORT_SYMBOL(__skb_gso_segment);
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002568
Herbert Xufb286bb2005-11-10 13:01:24 -08002569/* Take action when hardware reception checksum errors are detected. */
2570#ifdef CONFIG_BUG
2571void netdev_rx_csum_fault(struct net_device *dev)
2572{
2573 if (net_ratelimit()) {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00002574 pr_err("%s: hw csum failure\n", dev ? dev->name : "<unknown>");
Herbert Xufb286bb2005-11-10 13:01:24 -08002575 dump_stack();
2576 }
2577}
2578EXPORT_SYMBOL(netdev_rx_csum_fault);
2579#endif
2580
Linus Torvalds1da177e2005-04-16 15:20:36 -07002581/* Actually, we should eliminate this check as soon as we know, that:
2582 * 1. IOMMU is present and allows to map all the memory.
2583 * 2. No high memory really exists on this machine.
2584 */
2585
Florian Westphalc1e756b2014-05-05 15:00:44 +02002586static int illegal_highdma(struct net_device *dev, struct sk_buff *skb)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002587{
Herbert Xu3d3a8532006-06-27 13:33:10 -07002588#ifdef CONFIG_HIGHMEM
Linus Torvalds1da177e2005-04-16 15:20:36 -07002589 int i;
FUJITA Tomonori5acbbd42010-03-30 22:35:50 +00002590 if (!(dev->features & NETIF_F_HIGHDMA)) {
Ian Campbellea2ab692011-08-22 23:44:58 +00002591 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
2592 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
2593 if (PageHighMem(skb_frag_page(frag)))
FUJITA Tomonori5acbbd42010-03-30 22:35:50 +00002594 return 1;
Ian Campbellea2ab692011-08-22 23:44:58 +00002595 }
FUJITA Tomonori5acbbd42010-03-30 22:35:50 +00002596 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002597
FUJITA Tomonori5acbbd42010-03-30 22:35:50 +00002598 if (PCI_DMA_BUS_IS_PHYS) {
2599 struct device *pdev = dev->dev.parent;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002600
Eric Dumazet9092c652010-04-02 13:34:49 -07002601 if (!pdev)
2602 return 0;
FUJITA Tomonori5acbbd42010-03-30 22:35:50 +00002603 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
Ian Campbellea2ab692011-08-22 23:44:58 +00002604 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
2605 dma_addr_t addr = page_to_phys(skb_frag_page(frag));
FUJITA Tomonori5acbbd42010-03-30 22:35:50 +00002606 if (!pdev->dma_mask || addr + PAGE_SIZE - 1 > *pdev->dma_mask)
2607 return 1;
2608 }
2609 }
Herbert Xu3d3a8532006-06-27 13:33:10 -07002610#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07002611 return 0;
2612}
Linus Torvalds1da177e2005-04-16 15:20:36 -07002613
Simon Horman3b392dd2014-06-04 08:53:17 +09002614/* If MPLS offload request, verify we are testing hardware MPLS features
2615 * instead of standard features for the netdev.
2616 */
Pravin B Shelard0edc7b2014-12-23 16:20:11 -08002617#if IS_ENABLED(CONFIG_NET_MPLS_GSO)
Simon Horman3b392dd2014-06-04 08:53:17 +09002618static netdev_features_t net_mpls_features(struct sk_buff *skb,
2619 netdev_features_t features,
2620 __be16 type)
2621{
Simon Horman25cd9ba2014-10-06 05:05:13 -07002622 if (eth_p_mpls(type))
Simon Horman3b392dd2014-06-04 08:53:17 +09002623 features &= skb->dev->mpls_features;
2624
2625 return features;
2626}
2627#else
2628static netdev_features_t net_mpls_features(struct sk_buff *skb,
2629 netdev_features_t features,
2630 __be16 type)
2631{
2632 return features;
2633}
2634#endif
2635
Michał Mirosławc8f44af2011-11-15 15:29:55 +00002636static netdev_features_t harmonize_features(struct sk_buff *skb,
Florian Westphalc1e756b2014-05-05 15:00:44 +02002637 netdev_features_t features)
Jesse Grossf01a5232011-01-09 06:23:31 +00002638{
Vlad Yasevich53d64712014-03-27 17:26:18 -04002639 int tmp;
Simon Horman3b392dd2014-06-04 08:53:17 +09002640 __be16 type;
2641
2642 type = skb_network_protocol(skb, &tmp);
2643 features = net_mpls_features(skb, features, type);
Vlad Yasevich53d64712014-03-27 17:26:18 -04002644
Ed Cashinc0d680e2012-09-19 15:49:00 +00002645 if (skb->ip_summed != CHECKSUM_NONE &&
Simon Horman3b392dd2014-06-04 08:53:17 +09002646 !can_checksum_protocol(features, type)) {
Jesse Grossf01a5232011-01-09 06:23:31 +00002647 features &= ~NETIF_F_ALL_CSUM;
Florian Westphalc1e756b2014-05-05 15:00:44 +02002648 } else if (illegal_highdma(skb->dev, skb)) {
Jesse Grossf01a5232011-01-09 06:23:31 +00002649 features &= ~NETIF_F_SG;
2650 }
2651
2652 return features;
2653}
2654
Toshiaki Makitae38f3022015-03-27 14:31:13 +09002655netdev_features_t passthru_features_check(struct sk_buff *skb,
2656 struct net_device *dev,
2657 netdev_features_t features)
2658{
2659 return features;
2660}
2661EXPORT_SYMBOL(passthru_features_check);
2662
Toshiaki Makita8cb65d02015-03-27 14:31:12 +09002663static netdev_features_t dflt_features_check(const struct sk_buff *skb,
2664 struct net_device *dev,
2665 netdev_features_t features)
2666{
2667 return vlan_features_check(skb, features);
2668}
2669
Florian Westphalc1e756b2014-05-05 15:00:44 +02002670netdev_features_t netif_skb_features(struct sk_buff *skb)
Jesse Gross58e998c2010-10-29 12:14:55 +00002671{
Jesse Gross5f352272014-12-23 22:37:26 -08002672 struct net_device *dev = skb->dev;
Eric Dumazetfcbeb972014-10-05 10:11:27 -07002673 netdev_features_t features = dev->features;
2674 u16 gso_segs = skb_shinfo(skb)->gso_segs;
Jesse Gross58e998c2010-10-29 12:14:55 +00002675
Eric Dumazetfcbeb972014-10-05 10:11:27 -07002676 if (gso_segs > dev->gso_max_segs || gso_segs < dev->gso_min_segs)
Ben Hutchings30b678d2012-07-30 15:57:00 +00002677 features &= ~NETIF_F_GSO_MASK;
2678
Jesse Gross5f352272014-12-23 22:37:26 -08002679 /* If encapsulation offload request, verify we are testing
2680 * hardware encapsulation features instead of standard
2681 * features for the netdev
2682 */
2683 if (skb->encapsulation)
2684 features &= dev->hw_enc_features;
2685
Toshiaki Makitaf5a7fb82015-03-27 14:31:11 +09002686 if (skb_vlan_tagged(skb))
2687 features = netdev_intersect_features(features,
2688 dev->vlan_features |
2689 NETIF_F_HW_VLAN_CTAG_TX |
2690 NETIF_F_HW_VLAN_STAG_TX);
Jesse Gross58e998c2010-10-29 12:14:55 +00002691
Jesse Gross5f352272014-12-23 22:37:26 -08002692 if (dev->netdev_ops->ndo_features_check)
2693 features &= dev->netdev_ops->ndo_features_check(skb, dev,
2694 features);
Toshiaki Makita8cb65d02015-03-27 14:31:12 +09002695 else
2696 features &= dflt_features_check(skb, dev, features);
Jesse Gross5f352272014-12-23 22:37:26 -08002697
Florian Westphalc1e756b2014-05-05 15:00:44 +02002698 return harmonize_features(skb, features);
Jesse Gross58e998c2010-10-29 12:14:55 +00002699}
Florian Westphalc1e756b2014-05-05 15:00:44 +02002700EXPORT_SYMBOL(netif_skb_features);
Jesse Gross58e998c2010-10-29 12:14:55 +00002701
David S. Miller2ea25512014-08-29 21:10:01 -07002702static int xmit_one(struct sk_buff *skb, struct net_device *dev,
David S. Miller95f6b3d2014-08-29 21:57:30 -07002703 struct netdev_queue *txq, bool more)
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002704{
David S. Miller2ea25512014-08-29 21:10:01 -07002705 unsigned int len;
2706 int rc;
Stephen Hemminger00829822008-11-20 20:14:53 -08002707
Salam Noureddine7866a622015-01-27 11:35:48 -08002708 if (!list_empty(&ptype_all) || !list_empty(&dev->ptype_all))
David S. Miller2ea25512014-08-29 21:10:01 -07002709 dev_queue_xmit_nit(skb, dev);
Jesse Grossfc741212011-01-09 06:23:32 +00002710
David S. Miller2ea25512014-08-29 21:10:01 -07002711 len = skb->len;
2712 trace_net_dev_start_xmit(skb, dev);
David S. Miller95f6b3d2014-08-29 21:57:30 -07002713 rc = netdev_start_xmit(skb, dev, txq, more);
David S. Miller2ea25512014-08-29 21:10:01 -07002714 trace_net_dev_xmit(skb, rc, dev, len);
Eric Dumazetadf30902009-06-02 05:19:30 +00002715
Patrick McHardy572a9d72009-11-10 06:14:14 +00002716 return rc;
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002717}
David S. Miller2ea25512014-08-29 21:10:01 -07002718
David S. Miller8dcda222014-09-01 15:06:40 -07002719struct sk_buff *dev_hard_start_xmit(struct sk_buff *first, struct net_device *dev,
2720 struct netdev_queue *txq, int *ret)
David S. Miller7f2e8702014-08-29 21:19:14 -07002721{
2722 struct sk_buff *skb = first;
2723 int rc = NETDEV_TX_OK;
2724
2725 while (skb) {
2726 struct sk_buff *next = skb->next;
2727
2728 skb->next = NULL;
David S. Miller95f6b3d2014-08-29 21:57:30 -07002729 rc = xmit_one(skb, dev, txq, next != NULL);
David S. Miller7f2e8702014-08-29 21:19:14 -07002730 if (unlikely(!dev_xmit_complete(rc))) {
2731 skb->next = next;
2732 goto out;
2733 }
2734
2735 skb = next;
2736 if (netif_xmit_stopped(txq) && skb) {
2737 rc = NETDEV_TX_BUSY;
2738 break;
2739 }
2740 }
2741
2742out:
2743 *ret = rc;
2744 return skb;
2745}
2746
Eric Dumazet1ff0dc92014-10-06 11:26:27 -07002747static struct sk_buff *validate_xmit_vlan(struct sk_buff *skb,
2748 netdev_features_t features)
David S. Millereae3f882014-08-30 15:17:13 -07002749{
Jiri Pirkodf8a39d2015-01-13 17:13:44 +01002750 if (skb_vlan_tag_present(skb) &&
Jiri Pirko59682502014-11-19 14:04:59 +01002751 !vlan_hw_offload_capable(features, skb->vlan_proto))
2752 skb = __vlan_hwaccel_push_inside(skb);
David S. Millereae3f882014-08-30 15:17:13 -07002753 return skb;
2754}
2755
Eric Dumazet55a93b32014-10-03 15:31:07 -07002756static struct sk_buff *validate_xmit_skb(struct sk_buff *skb, struct net_device *dev)
David S. Millereae3f882014-08-30 15:17:13 -07002757{
2758 netdev_features_t features;
2759
2760 if (skb->next)
2761 return skb;
2762
David S. Millereae3f882014-08-30 15:17:13 -07002763 features = netif_skb_features(skb);
2764 skb = validate_xmit_vlan(skb, features);
2765 if (unlikely(!skb))
2766 goto out_null;
2767
Johannes Berg8b86a612015-04-17 15:45:04 +02002768 if (netif_needs_gso(skb, features)) {
David S. Millerce937182014-08-30 19:22:20 -07002769 struct sk_buff *segs;
2770
2771 segs = skb_gso_segment(skb, features);
Jason Wangcecda692014-09-19 16:04:38 +08002772 if (IS_ERR(segs)) {
Jason Wangaf6dabc2014-12-19 11:09:13 +08002773 goto out_kfree_skb;
Jason Wangcecda692014-09-19 16:04:38 +08002774 } else if (segs) {
2775 consume_skb(skb);
2776 skb = segs;
2777 }
David S. Millereae3f882014-08-30 15:17:13 -07002778 } else {
2779 if (skb_needs_linearize(skb, features) &&
2780 __skb_linearize(skb))
2781 goto out_kfree_skb;
2782
2783 /* If packet is not checksummed and device does not
2784 * support checksumming for this protocol, complete
2785 * checksumming here.
2786 */
2787 if (skb->ip_summed == CHECKSUM_PARTIAL) {
2788 if (skb->encapsulation)
2789 skb_set_inner_transport_header(skb,
2790 skb_checksum_start_offset(skb));
2791 else
2792 skb_set_transport_header(skb,
2793 skb_checksum_start_offset(skb));
2794 if (!(features & NETIF_F_ALL_CSUM) &&
2795 skb_checksum_help(skb))
2796 goto out_kfree_skb;
2797 }
2798 }
2799
2800 return skb;
2801
2802out_kfree_skb:
2803 kfree_skb(skb);
2804out_null:
2805 return NULL;
2806}
2807
Eric Dumazet55a93b32014-10-03 15:31:07 -07002808struct sk_buff *validate_xmit_skb_list(struct sk_buff *skb, struct net_device *dev)
2809{
2810 struct sk_buff *next, *head = NULL, *tail;
2811
Eric Dumazetbec3cfd2014-10-03 20:59:19 -07002812 for (; skb != NULL; skb = next) {
Eric Dumazet55a93b32014-10-03 15:31:07 -07002813 next = skb->next;
2814 skb->next = NULL;
Eric Dumazet55a93b32014-10-03 15:31:07 -07002815
Eric Dumazetbec3cfd2014-10-03 20:59:19 -07002816 /* in case skb wont be segmented, point to itself */
2817 skb->prev = skb;
2818
2819 skb = validate_xmit_skb(skb, dev);
2820 if (!skb)
2821 continue;
2822
2823 if (!head)
2824 head = skb;
2825 else
2826 tail->next = skb;
2827 /* If skb was segmented, skb->prev points to
2828 * the last segment. If not, it still contains skb.
2829 */
2830 tail = skb->prev;
Eric Dumazet55a93b32014-10-03 15:31:07 -07002831 }
2832 return head;
2833}
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002834
Eric Dumazet1def9232013-01-10 12:36:42 +00002835static void qdisc_pkt_len_init(struct sk_buff *skb)
2836{
2837 const struct skb_shared_info *shinfo = skb_shinfo(skb);
2838
2839 qdisc_skb_cb(skb)->pkt_len = skb->len;
2840
2841 /* To get more precise estimation of bytes sent on wire,
2842 * we add to pkt_len the headers size of all segments
2843 */
2844 if (shinfo->gso_size) {
Eric Dumazet757b8b12013-01-15 21:14:21 -08002845 unsigned int hdr_len;
Jason Wang15e5a032013-03-25 20:19:59 +00002846 u16 gso_segs = shinfo->gso_segs;
Eric Dumazet1def9232013-01-10 12:36:42 +00002847
Eric Dumazet757b8b12013-01-15 21:14:21 -08002848 /* mac layer + network layer */
2849 hdr_len = skb_transport_header(skb) - skb_mac_header(skb);
2850
2851 /* + transport layer */
Eric Dumazet1def9232013-01-10 12:36:42 +00002852 if (likely(shinfo->gso_type & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6)))
2853 hdr_len += tcp_hdrlen(skb);
2854 else
2855 hdr_len += sizeof(struct udphdr);
Jason Wang15e5a032013-03-25 20:19:59 +00002856
2857 if (shinfo->gso_type & SKB_GSO_DODGY)
2858 gso_segs = DIV_ROUND_UP(skb->len - hdr_len,
2859 shinfo->gso_size);
2860
2861 qdisc_skb_cb(skb)->pkt_len += (gso_segs - 1) * hdr_len;
Eric Dumazet1def9232013-01-10 12:36:42 +00002862 }
2863}
2864
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00002865static inline int __dev_xmit_skb(struct sk_buff *skb, struct Qdisc *q,
2866 struct net_device *dev,
2867 struct netdev_queue *txq)
2868{
2869 spinlock_t *root_lock = qdisc_lock(q);
Eric Dumazeta2da5702011-01-20 03:48:19 +00002870 bool contended;
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00002871 int rc;
2872
Eric Dumazet1def9232013-01-10 12:36:42 +00002873 qdisc_pkt_len_init(skb);
Eric Dumazeta2da5702011-01-20 03:48:19 +00002874 qdisc_calculate_pkt_len(skb, q);
Eric Dumazet79640a42010-06-02 05:09:29 -07002875 /*
2876 * Heuristic to force contended enqueues to serialize on a
2877 * separate lock before trying to get qdisc main lock.
Ying Xue9bf2b8c2014-06-26 15:56:31 +08002878 * This permits __QDISC___STATE_RUNNING owner to get the lock more
2879 * often and dequeue packets faster.
Eric Dumazet79640a42010-06-02 05:09:29 -07002880 */
Eric Dumazeta2da5702011-01-20 03:48:19 +00002881 contended = qdisc_is_running(q);
Eric Dumazet79640a42010-06-02 05:09:29 -07002882 if (unlikely(contended))
2883 spin_lock(&q->busylock);
2884
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00002885 spin_lock(root_lock);
2886 if (unlikely(test_bit(__QDISC_STATE_DEACTIVATED, &q->state))) {
2887 kfree_skb(skb);
2888 rc = NET_XMIT_DROP;
2889 } else if ((q->flags & TCQ_F_CAN_BYPASS) && !qdisc_qlen(q) &&
Eric Dumazetbc135b22010-06-02 03:23:51 -07002890 qdisc_run_begin(q)) {
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00002891 /*
2892 * This is a work-conserving queue; there are no old skbs
2893 * waiting to be sent out; and the qdisc is not running -
2894 * xmit the skb directly.
2895 */
Eric Dumazetbfe0d022011-01-09 08:30:54 +00002896
Eric Dumazetbfe0d022011-01-09 08:30:54 +00002897 qdisc_bstats_update(q, skb);
2898
Eric Dumazet55a93b32014-10-03 15:31:07 -07002899 if (sch_direct_xmit(skb, q, dev, txq, root_lock, true)) {
Eric Dumazet79640a42010-06-02 05:09:29 -07002900 if (unlikely(contended)) {
2901 spin_unlock(&q->busylock);
2902 contended = false;
2903 }
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00002904 __qdisc_run(q);
Eric Dumazet79640a42010-06-02 05:09:29 -07002905 } else
Eric Dumazetbc135b22010-06-02 03:23:51 -07002906 qdisc_run_end(q);
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00002907
2908 rc = NET_XMIT_SUCCESS;
2909 } else {
Eric Dumazeta2da5702011-01-20 03:48:19 +00002910 rc = q->enqueue(skb, q) & NET_XMIT_MASK;
Eric Dumazet79640a42010-06-02 05:09:29 -07002911 if (qdisc_run_begin(q)) {
2912 if (unlikely(contended)) {
2913 spin_unlock(&q->busylock);
2914 contended = false;
2915 }
2916 __qdisc_run(q);
2917 }
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00002918 }
2919 spin_unlock(root_lock);
Eric Dumazet79640a42010-06-02 05:09:29 -07002920 if (unlikely(contended))
2921 spin_unlock(&q->busylock);
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00002922 return rc;
2923}
2924
Daniel Borkmann86f85152013-12-29 17:27:11 +01002925#if IS_ENABLED(CONFIG_CGROUP_NET_PRIO)
Neil Horman5bc14212011-11-22 05:10:51 +00002926static void skb_update_prio(struct sk_buff *skb)
2927{
Igor Maravic6977a792011-11-25 07:44:54 +00002928 struct netprio_map *map = rcu_dereference_bh(skb->dev->priomap);
Neil Horman5bc14212011-11-22 05:10:51 +00002929
Eric Dumazet91c68ce2012-07-08 21:45:10 +00002930 if (!skb->priority && skb->sk && map) {
2931 unsigned int prioidx = skb->sk->sk_cgrp_prioidx;
2932
2933 if (prioidx < map->priomap_len)
2934 skb->priority = map->priomap[prioidx];
2935 }
Neil Horman5bc14212011-11-22 05:10:51 +00002936}
2937#else
2938#define skb_update_prio(skb)
2939#endif
2940
hannes@stressinduktion.orgf60e5992015-04-01 17:07:44 +02002941DEFINE_PER_CPU(int, xmit_recursion);
2942EXPORT_SYMBOL(xmit_recursion);
2943
David S. Miller11a766c2010-10-25 12:51:55 -07002944#define RECURSION_LIMIT 10
Eric Dumazet745e20f2010-09-29 13:23:09 -07002945
Dave Jonesd29f7492008-07-22 14:09:06 -07002946/**
Michel Machado95603e22012-06-12 10:16:35 +00002947 * dev_loopback_xmit - loop back @skb
Eric W. Biederman0c4b51f2015-09-15 20:04:18 -05002948 * @net: network namespace this loopback is happening in
2949 * @sk: sk needed to be a netfilter okfn
Michel Machado95603e22012-06-12 10:16:35 +00002950 * @skb: buffer to transmit
2951 */
Eric W. Biederman0c4b51f2015-09-15 20:04:18 -05002952int dev_loopback_xmit(struct net *net, struct sock *sk, struct sk_buff *skb)
Michel Machado95603e22012-06-12 10:16:35 +00002953{
2954 skb_reset_mac_header(skb);
2955 __skb_pull(skb, skb_network_offset(skb));
2956 skb->pkt_type = PACKET_LOOPBACK;
2957 skb->ip_summed = CHECKSUM_UNNECESSARY;
2958 WARN_ON(!skb_dst(skb));
2959 skb_dst_force(skb);
2960 netif_rx_ni(skb);
2961 return 0;
2962}
2963EXPORT_SYMBOL(dev_loopback_xmit);
2964
Jiri Pirko638b2a62015-05-12 14:56:13 +02002965static inline int get_xps_queue(struct net_device *dev, struct sk_buff *skb)
2966{
2967#ifdef CONFIG_XPS
2968 struct xps_dev_maps *dev_maps;
2969 struct xps_map *map;
2970 int queue_index = -1;
2971
2972 rcu_read_lock();
2973 dev_maps = rcu_dereference(dev->xps_maps);
2974 if (dev_maps) {
2975 map = rcu_dereference(
2976 dev_maps->cpu_map[skb->sender_cpu - 1]);
2977 if (map) {
2978 if (map->len == 1)
2979 queue_index = map->queues[0];
2980 else
2981 queue_index = map->queues[reciprocal_scale(skb_get_hash(skb),
2982 map->len)];
2983 if (unlikely(queue_index >= dev->real_num_tx_queues))
2984 queue_index = -1;
2985 }
2986 }
2987 rcu_read_unlock();
2988
2989 return queue_index;
2990#else
2991 return -1;
2992#endif
2993}
2994
2995static u16 __netdev_pick_tx(struct net_device *dev, struct sk_buff *skb)
2996{
2997 struct sock *sk = skb->sk;
2998 int queue_index = sk_tx_queue_get(sk);
2999
3000 if (queue_index < 0 || skb->ooo_okay ||
3001 queue_index >= dev->real_num_tx_queues) {
3002 int new_index = get_xps_queue(dev, skb);
3003 if (new_index < 0)
3004 new_index = skb_tx_hash(dev, skb);
3005
3006 if (queue_index != new_index && sk &&
Eric Dumazet004a5d02015-10-04 21:08:10 -07003007 sk_fullsock(sk) &&
Jiri Pirko638b2a62015-05-12 14:56:13 +02003008 rcu_access_pointer(sk->sk_dst_cache))
3009 sk_tx_queue_set(sk, new_index);
3010
3011 queue_index = new_index;
3012 }
3013
3014 return queue_index;
3015}
3016
3017struct netdev_queue *netdev_pick_tx(struct net_device *dev,
3018 struct sk_buff *skb,
3019 void *accel_priv)
3020{
3021 int queue_index = 0;
3022
3023#ifdef CONFIG_XPS
Eric Dumazet52bd2d62015-11-18 06:30:50 -08003024 u32 sender_cpu = skb->sender_cpu - 1;
3025
3026 if (sender_cpu >= (u32)NR_CPUS)
Jiri Pirko638b2a62015-05-12 14:56:13 +02003027 skb->sender_cpu = raw_smp_processor_id() + 1;
3028#endif
3029
3030 if (dev->real_num_tx_queues != 1) {
3031 const struct net_device_ops *ops = dev->netdev_ops;
3032 if (ops->ndo_select_queue)
3033 queue_index = ops->ndo_select_queue(dev, skb, accel_priv,
3034 __netdev_pick_tx);
3035 else
3036 queue_index = __netdev_pick_tx(dev, skb);
3037
3038 if (!accel_priv)
3039 queue_index = netdev_cap_txqueue(dev, queue_index);
3040 }
3041
3042 skb_set_queue_mapping(skb, queue_index);
3043 return netdev_get_tx_queue(dev, queue_index);
3044}
3045
Michel Machado95603e22012-06-12 10:16:35 +00003046/**
Jason Wang9d08dd32014-01-20 11:25:13 +08003047 * __dev_queue_xmit - transmit a buffer
Dave Jonesd29f7492008-07-22 14:09:06 -07003048 * @skb: buffer to transmit
Jason Wang9d08dd32014-01-20 11:25:13 +08003049 * @accel_priv: private data used for L2 forwarding offload
Dave Jonesd29f7492008-07-22 14:09:06 -07003050 *
3051 * Queue a buffer for transmission to a network device. The caller must
3052 * have set the device and priority and built the buffer before calling
3053 * this function. The function can be called from an interrupt.
3054 *
3055 * A negative errno code is returned on a failure. A success does not
3056 * guarantee the frame will be transmitted as it may be dropped due
3057 * to congestion or traffic shaping.
3058 *
3059 * -----------------------------------------------------------------------------------
3060 * I notice this method can also return errors from the queue disciplines,
3061 * including NET_XMIT_DROP, which is a positive value. So, errors can also
3062 * be positive.
3063 *
3064 * Regardless of the return value, the skb is consumed, so it is currently
3065 * difficult to retry a send to this method. (You can bump the ref count
3066 * before sending to hold a reference for retry if you are careful.)
3067 *
3068 * When calling this method, interrupts MUST be enabled. This is because
3069 * the BH enable code must have IRQs enabled so that it will not deadlock.
3070 * --BLG
3071 */
Rashika Kheria0a59f3a2014-02-09 20:26:25 +05303072static int __dev_queue_xmit(struct sk_buff *skb, void *accel_priv)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003073{
3074 struct net_device *dev = skb->dev;
David S. Millerdc2b4842008-07-08 17:18:23 -07003075 struct netdev_queue *txq;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003076 struct Qdisc *q;
3077 int rc = -ENOMEM;
3078
Eric Dumazet6d1ccff2013-02-05 20:22:20 +00003079 skb_reset_mac_header(skb);
3080
Willem de Bruijne7fd2882014-08-04 22:11:48 -04003081 if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_SCHED_TSTAMP))
3082 __skb_tstamp_tx(skb, NULL, skb->sk, SCM_TSTAMP_SCHED);
3083
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09003084 /* Disable soft irqs for various locks below. Also
3085 * stops preemption for RCU.
Linus Torvalds1da177e2005-04-16 15:20:36 -07003086 */
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09003087 rcu_read_lock_bh();
Linus Torvalds1da177e2005-04-16 15:20:36 -07003088
Neil Horman5bc14212011-11-22 05:10:51 +00003089 skb_update_prio(skb);
3090
Eric Dumazet02875872014-10-05 18:38:35 -07003091 /* If device/qdisc don't need skb->dst, release it right now while
3092 * its hot in this cpu cache.
3093 */
3094 if (dev->priv_flags & IFF_XMIT_DST_RELEASE)
3095 skb_dst_drop(skb);
3096 else
3097 skb_dst_force(skb);
3098
Scott Feldman0c4f6912015-07-18 18:24:48 -07003099#ifdef CONFIG_NET_SWITCHDEV
3100 /* Don't forward if offload device already forwarded */
3101 if (skb->offload_fwd_mark &&
3102 skb->offload_fwd_mark == dev->offload_fwd_mark) {
3103 consume_skb(skb);
3104 rc = NET_XMIT_SUCCESS;
3105 goto out;
3106 }
3107#endif
3108
Jason Wangf663dd92014-01-10 16:18:26 +08003109 txq = netdev_pick_tx(dev, skb, accel_priv);
Paul E. McKenneya898def2010-02-22 17:04:49 -08003110 q = rcu_dereference_bh(txq->qdisc);
David S. Miller37437bb2008-07-16 02:15:04 -07003111
Linus Torvalds1da177e2005-04-16 15:20:36 -07003112#ifdef CONFIG_NET_CLS_ACT
Eric Dumazetd1b19df2009-09-03 01:29:39 -07003113 skb->tc_verd = SET_TC_AT(skb->tc_verd, AT_EGRESS);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003114#endif
Koki Sanagicf66ba52010-08-23 18:45:02 +09003115 trace_net_dev_queue(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003116 if (q->enqueue) {
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00003117 rc = __dev_xmit_skb(skb, q, dev, txq);
David S. Miller37437bb2008-07-16 02:15:04 -07003118 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003119 }
3120
3121 /* The device has no queue. Common case for software devices:
3122 loopback, all the sorts of tunnels...
3123
Herbert Xu932ff272006-06-09 12:20:56 -07003124 Really, it is unlikely that netif_tx_lock protection is necessary
3125 here. (f.e. loopback and IP tunnels are clean ignoring statistics
Linus Torvalds1da177e2005-04-16 15:20:36 -07003126 counters.)
3127 However, it is possible, that they rely on protection
3128 made by us here.
3129
3130 Check this and shot the lock. It is not prone from deadlocks.
3131 Either shot noqueue qdisc, it is even simpler 8)
3132 */
3133 if (dev->flags & IFF_UP) {
3134 int cpu = smp_processor_id(); /* ok because BHs are off */
3135
David S. Millerc773e842008-07-08 23:13:53 -07003136 if (txq->xmit_lock_owner != cpu) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07003137
Eric Dumazet745e20f2010-09-29 13:23:09 -07003138 if (__this_cpu_read(xmit_recursion) > RECURSION_LIMIT)
3139 goto recursion_alert;
3140
Jesper Dangaard Brouer1f595332014-09-03 17:56:09 +02003141 skb = validate_xmit_skb(skb, dev);
3142 if (!skb)
3143 goto drop;
3144
David S. Millerc773e842008-07-08 23:13:53 -07003145 HARD_TX_LOCK(dev, txq, cpu);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003146
Tom Herbert734664982011-11-28 16:32:44 +00003147 if (!netif_xmit_stopped(txq)) {
Eric Dumazet745e20f2010-09-29 13:23:09 -07003148 __this_cpu_inc(xmit_recursion);
David S. Millerce937182014-08-30 19:22:20 -07003149 skb = dev_hard_start_xmit(skb, dev, txq, &rc);
Eric Dumazet745e20f2010-09-29 13:23:09 -07003150 __this_cpu_dec(xmit_recursion);
Patrick McHardy572a9d72009-11-10 06:14:14 +00003151 if (dev_xmit_complete(rc)) {
David S. Millerc773e842008-07-08 23:13:53 -07003152 HARD_TX_UNLOCK(dev, txq);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003153 goto out;
3154 }
3155 }
David S. Millerc773e842008-07-08 23:13:53 -07003156 HARD_TX_UNLOCK(dev, txq);
Joe Perchese87cc472012-05-13 21:56:26 +00003157 net_crit_ratelimited("Virtual device %s asks to queue packet!\n",
3158 dev->name);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003159 } else {
3160 /* Recursion is detected! It is possible,
Eric Dumazet745e20f2010-09-29 13:23:09 -07003161 * unfortunately
3162 */
3163recursion_alert:
Joe Perchese87cc472012-05-13 21:56:26 +00003164 net_crit_ratelimited("Dead loop on virtual device %s, fix it urgently!\n",
3165 dev->name);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003166 }
3167 }
3168
3169 rc = -ENETDOWN;
Jesper Dangaard Brouer1f595332014-09-03 17:56:09 +02003170drop:
Herbert Xud4828d82006-06-22 02:28:18 -07003171 rcu_read_unlock_bh();
Linus Torvalds1da177e2005-04-16 15:20:36 -07003172
Eric Dumazet015f0682014-03-27 08:45:56 -07003173 atomic_long_inc(&dev->tx_dropped);
Jesper Dangaard Brouer1f595332014-09-03 17:56:09 +02003174 kfree_skb_list(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003175 return rc;
3176out:
Herbert Xud4828d82006-06-22 02:28:18 -07003177 rcu_read_unlock_bh();
Linus Torvalds1da177e2005-04-16 15:20:36 -07003178 return rc;
3179}
Jason Wangf663dd92014-01-10 16:18:26 +08003180
Eric W. Biederman2b4aa3c2015-09-15 20:04:07 -05003181int dev_queue_xmit(struct sk_buff *skb)
Jason Wangf663dd92014-01-10 16:18:26 +08003182{
3183 return __dev_queue_xmit(skb, NULL);
3184}
Eric W. Biederman2b4aa3c2015-09-15 20:04:07 -05003185EXPORT_SYMBOL(dev_queue_xmit);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003186
Jason Wangf663dd92014-01-10 16:18:26 +08003187int dev_queue_xmit_accel(struct sk_buff *skb, void *accel_priv)
3188{
3189 return __dev_queue_xmit(skb, accel_priv);
3190}
3191EXPORT_SYMBOL(dev_queue_xmit_accel);
3192
Linus Torvalds1da177e2005-04-16 15:20:36 -07003193
3194/*=======================================================================
3195 Receiver routines
3196 =======================================================================*/
3197
Stephen Hemminger6b2bedc2007-03-12 14:33:50 -07003198int netdev_max_backlog __read_mostly = 1000;
Eric Dumazetc9e6bc62012-09-27 19:29:05 +00003199EXPORT_SYMBOL(netdev_max_backlog);
3200
Eric Dumazet3b098e22010-05-15 23:57:10 -07003201int netdev_tstamp_prequeue __read_mostly = 1;
Stephen Hemminger6b2bedc2007-03-12 14:33:50 -07003202int netdev_budget __read_mostly = 300;
3203int weight_p __read_mostly = 64; /* old backlog weight */
Linus Torvalds1da177e2005-04-16 15:20:36 -07003204
Eric Dumazeteecfd7c2010-05-06 22:07:48 -07003205/* Called with irq disabled */
3206static inline void ____napi_schedule(struct softnet_data *sd,
3207 struct napi_struct *napi)
3208{
3209 list_add_tail(&napi->poll_list, &sd->poll_list);
3210 __raise_softirq_irqoff(NET_RX_SOFTIRQ);
3211}
3212
Eric Dumazetdf334542010-03-24 19:13:54 +00003213#ifdef CONFIG_RPS
Tom Herbertfec5e652010-04-16 16:01:27 -07003214
3215/* One global table that all flow-based protocols share. */
Eric Dumazet6e3f7fa2010-10-25 03:02:02 +00003216struct rps_sock_flow_table __rcu *rps_sock_flow_table __read_mostly;
Tom Herbertfec5e652010-04-16 16:01:27 -07003217EXPORT_SYMBOL(rps_sock_flow_table);
Eric Dumazet567e4b72015-02-06 12:59:01 -08003218u32 rps_cpu_mask __read_mostly;
3219EXPORT_SYMBOL(rps_cpu_mask);
Tom Herbertfec5e652010-04-16 16:01:27 -07003220
Ingo Molnarc5905af2012-02-24 08:31:31 +01003221struct static_key rps_needed __read_mostly;
Eric Dumazetadc93002011-11-17 03:13:26 +00003222
Ben Hutchingsc4454772011-01-19 11:03:53 +00003223static struct rps_dev_flow *
3224set_rps_cpu(struct net_device *dev, struct sk_buff *skb,
3225 struct rps_dev_flow *rflow, u16 next_cpu)
3226{
Eric Dumazeta31196b2015-04-25 09:35:24 -07003227 if (next_cpu < nr_cpu_ids) {
Ben Hutchingsc4454772011-01-19 11:03:53 +00003228#ifdef CONFIG_RFS_ACCEL
3229 struct netdev_rx_queue *rxqueue;
3230 struct rps_dev_flow_table *flow_table;
3231 struct rps_dev_flow *old_rflow;
3232 u32 flow_id;
3233 u16 rxq_index;
3234 int rc;
3235
3236 /* Should we steer this flow to a different hardware queue? */
Ben Hutchings69a19ee2011-02-15 20:32:04 +00003237 if (!skb_rx_queue_recorded(skb) || !dev->rx_cpu_rmap ||
3238 !(dev->features & NETIF_F_NTUPLE))
Ben Hutchingsc4454772011-01-19 11:03:53 +00003239 goto out;
3240 rxq_index = cpu_rmap_lookup_index(dev->rx_cpu_rmap, next_cpu);
3241 if (rxq_index == skb_get_rx_queue(skb))
3242 goto out;
3243
3244 rxqueue = dev->_rx + rxq_index;
3245 flow_table = rcu_dereference(rxqueue->rps_flow_table);
3246 if (!flow_table)
3247 goto out;
Tom Herbert61b905d2014-03-24 15:34:47 -07003248 flow_id = skb_get_hash(skb) & flow_table->mask;
Ben Hutchingsc4454772011-01-19 11:03:53 +00003249 rc = dev->netdev_ops->ndo_rx_flow_steer(dev, skb,
3250 rxq_index, flow_id);
3251 if (rc < 0)
3252 goto out;
3253 old_rflow = rflow;
3254 rflow = &flow_table->flows[flow_id];
Ben Hutchingsc4454772011-01-19 11:03:53 +00003255 rflow->filter = rc;
3256 if (old_rflow->filter == rflow->filter)
3257 old_rflow->filter = RPS_NO_FILTER;
3258 out:
3259#endif
3260 rflow->last_qtail =
Ben Hutchings09994d12011-10-03 04:42:46 +00003261 per_cpu(softnet_data, next_cpu).input_queue_head;
Ben Hutchingsc4454772011-01-19 11:03:53 +00003262 }
3263
Ben Hutchings09994d12011-10-03 04:42:46 +00003264 rflow->cpu = next_cpu;
Ben Hutchingsc4454772011-01-19 11:03:53 +00003265 return rflow;
3266}
3267
Tom Herbert0a9627f2010-03-16 08:03:29 +00003268/*
3269 * get_rps_cpu is called from netif_receive_skb and returns the target
3270 * CPU from the RPS map of the receiving queue for a given skb.
Eric Dumazetb0e28f12010-04-15 00:14:07 -07003271 * rcu_read_lock must be held on entry.
Tom Herbert0a9627f2010-03-16 08:03:29 +00003272 */
Tom Herbertfec5e652010-04-16 16:01:27 -07003273static int get_rps_cpu(struct net_device *dev, struct sk_buff *skb,
3274 struct rps_dev_flow **rflowp)
Tom Herbert0a9627f2010-03-16 08:03:29 +00003275{
Eric Dumazet567e4b72015-02-06 12:59:01 -08003276 const struct rps_sock_flow_table *sock_flow_table;
3277 struct netdev_rx_queue *rxqueue = dev->_rx;
Tom Herbertfec5e652010-04-16 16:01:27 -07003278 struct rps_dev_flow_table *flow_table;
Eric Dumazet567e4b72015-02-06 12:59:01 -08003279 struct rps_map *map;
Tom Herbert0a9627f2010-03-16 08:03:29 +00003280 int cpu = -1;
Eric Dumazet567e4b72015-02-06 12:59:01 -08003281 u32 tcpu;
Tom Herbert61b905d2014-03-24 15:34:47 -07003282 u32 hash;
Tom Herbert0a9627f2010-03-16 08:03:29 +00003283
Tom Herbert0a9627f2010-03-16 08:03:29 +00003284 if (skb_rx_queue_recorded(skb)) {
3285 u16 index = skb_get_rx_queue(skb);
Eric Dumazet567e4b72015-02-06 12:59:01 -08003286
Ben Hutchings62fe0b42010-09-27 08:24:33 +00003287 if (unlikely(index >= dev->real_num_rx_queues)) {
3288 WARN_ONCE(dev->real_num_rx_queues > 1,
3289 "%s received packet on queue %u, but number "
3290 "of RX queues is %u\n",
3291 dev->name, index, dev->real_num_rx_queues);
Tom Herbert0a9627f2010-03-16 08:03:29 +00003292 goto done;
3293 }
Eric Dumazet567e4b72015-02-06 12:59:01 -08003294 rxqueue += index;
Tom Herbert0a9627f2010-03-16 08:03:29 +00003295 }
3296
Eric Dumazet567e4b72015-02-06 12:59:01 -08003297 /* Avoid computing hash if RFS/RPS is not active for this rxqueue */
3298
3299 flow_table = rcu_dereference(rxqueue->rps_flow_table);
3300 map = rcu_dereference(rxqueue->rps_map);
3301 if (!flow_table && !map)
3302 goto done;
3303
Changli Gao2d47b452010-08-17 19:00:56 +00003304 skb_reset_network_header(skb);
Tom Herbert61b905d2014-03-24 15:34:47 -07003305 hash = skb_get_hash(skb);
3306 if (!hash)
Tom Herbert0a9627f2010-03-16 08:03:29 +00003307 goto done;
Tom Herbert0a9627f2010-03-16 08:03:29 +00003308
Tom Herbertfec5e652010-04-16 16:01:27 -07003309 sock_flow_table = rcu_dereference(rps_sock_flow_table);
3310 if (flow_table && sock_flow_table) {
Tom Herbertfec5e652010-04-16 16:01:27 -07003311 struct rps_dev_flow *rflow;
Eric Dumazet567e4b72015-02-06 12:59:01 -08003312 u32 next_cpu;
3313 u32 ident;
Tom Herbertfec5e652010-04-16 16:01:27 -07003314
Eric Dumazet567e4b72015-02-06 12:59:01 -08003315 /* First check into global flow table if there is a match */
3316 ident = sock_flow_table->ents[hash & sock_flow_table->mask];
3317 if ((ident ^ hash) & ~rps_cpu_mask)
3318 goto try_rps;
3319
3320 next_cpu = ident & rps_cpu_mask;
3321
3322 /* OK, now we know there is a match,
3323 * we can look at the local (per receive queue) flow table
3324 */
Tom Herbert61b905d2014-03-24 15:34:47 -07003325 rflow = &flow_table->flows[hash & flow_table->mask];
Tom Herbertfec5e652010-04-16 16:01:27 -07003326 tcpu = rflow->cpu;
3327
Tom Herbertfec5e652010-04-16 16:01:27 -07003328 /*
3329 * If the desired CPU (where last recvmsg was done) is
3330 * different from current CPU (one in the rx-queue flow
3331 * table entry), switch if one of the following holds:
Eric Dumazeta31196b2015-04-25 09:35:24 -07003332 * - Current CPU is unset (>= nr_cpu_ids).
Tom Herbertfec5e652010-04-16 16:01:27 -07003333 * - Current CPU is offline.
3334 * - The current CPU's queue tail has advanced beyond the
3335 * last packet that was enqueued using this table entry.
3336 * This guarantees that all previous packets for the flow
3337 * have been dequeued, thus preserving in order delivery.
3338 */
3339 if (unlikely(tcpu != next_cpu) &&
Eric Dumazeta31196b2015-04-25 09:35:24 -07003340 (tcpu >= nr_cpu_ids || !cpu_online(tcpu) ||
Tom Herbertfec5e652010-04-16 16:01:27 -07003341 ((int)(per_cpu(softnet_data, tcpu).input_queue_head -
Tom Herbertbaefa312012-11-16 09:04:15 +00003342 rflow->last_qtail)) >= 0)) {
3343 tcpu = next_cpu;
Ben Hutchingsc4454772011-01-19 11:03:53 +00003344 rflow = set_rps_cpu(dev, skb, rflow, next_cpu);
Tom Herbertbaefa312012-11-16 09:04:15 +00003345 }
Ben Hutchingsc4454772011-01-19 11:03:53 +00003346
Eric Dumazeta31196b2015-04-25 09:35:24 -07003347 if (tcpu < nr_cpu_ids && cpu_online(tcpu)) {
Tom Herbertfec5e652010-04-16 16:01:27 -07003348 *rflowp = rflow;
3349 cpu = tcpu;
3350 goto done;
3351 }
3352 }
3353
Eric Dumazet567e4b72015-02-06 12:59:01 -08003354try_rps:
3355
Tom Herbert0a9627f2010-03-16 08:03:29 +00003356 if (map) {
Daniel Borkmann8fc54f62014-08-23 20:58:54 +02003357 tcpu = map->cpus[reciprocal_scale(hash, map->len)];
Tom Herbert0a9627f2010-03-16 08:03:29 +00003358 if (cpu_online(tcpu)) {
3359 cpu = tcpu;
3360 goto done;
3361 }
3362 }
3363
3364done:
Tom Herbert0a9627f2010-03-16 08:03:29 +00003365 return cpu;
3366}
3367
Ben Hutchingsc4454772011-01-19 11:03:53 +00003368#ifdef CONFIG_RFS_ACCEL
3369
3370/**
3371 * rps_may_expire_flow - check whether an RFS hardware filter may be removed
3372 * @dev: Device on which the filter was set
3373 * @rxq_index: RX queue index
3374 * @flow_id: Flow ID passed to ndo_rx_flow_steer()
3375 * @filter_id: Filter ID returned by ndo_rx_flow_steer()
3376 *
3377 * Drivers that implement ndo_rx_flow_steer() should periodically call
3378 * this function for each installed filter and remove the filters for
3379 * which it returns %true.
3380 */
3381bool rps_may_expire_flow(struct net_device *dev, u16 rxq_index,
3382 u32 flow_id, u16 filter_id)
3383{
3384 struct netdev_rx_queue *rxqueue = dev->_rx + rxq_index;
3385 struct rps_dev_flow_table *flow_table;
3386 struct rps_dev_flow *rflow;
3387 bool expire = true;
Eric Dumazeta31196b2015-04-25 09:35:24 -07003388 unsigned int cpu;
Ben Hutchingsc4454772011-01-19 11:03:53 +00003389
3390 rcu_read_lock();
3391 flow_table = rcu_dereference(rxqueue->rps_flow_table);
3392 if (flow_table && flow_id <= flow_table->mask) {
3393 rflow = &flow_table->flows[flow_id];
3394 cpu = ACCESS_ONCE(rflow->cpu);
Eric Dumazeta31196b2015-04-25 09:35:24 -07003395 if (rflow->filter == filter_id && cpu < nr_cpu_ids &&
Ben Hutchingsc4454772011-01-19 11:03:53 +00003396 ((int)(per_cpu(softnet_data, cpu).input_queue_head -
3397 rflow->last_qtail) <
3398 (int)(10 * flow_table->mask)))
3399 expire = false;
3400 }
3401 rcu_read_unlock();
3402 return expire;
3403}
3404EXPORT_SYMBOL(rps_may_expire_flow);
3405
3406#endif /* CONFIG_RFS_ACCEL */
3407
Tom Herbert0a9627f2010-03-16 08:03:29 +00003408/* Called from hardirq (IPI) context */
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003409static void rps_trigger_softirq(void *data)
Tom Herbert0a9627f2010-03-16 08:03:29 +00003410{
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003411 struct softnet_data *sd = data;
3412
Eric Dumazeteecfd7c2010-05-06 22:07:48 -07003413 ____napi_schedule(sd, &sd->backlog);
Changli Gaodee42872010-05-02 05:42:16 +00003414 sd->received_rps++;
Tom Herbert0a9627f2010-03-16 08:03:29 +00003415}
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003416
Tom Herbertfec5e652010-04-16 16:01:27 -07003417#endif /* CONFIG_RPS */
Tom Herbert0a9627f2010-03-16 08:03:29 +00003418
3419/*
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003420 * Check if this softnet_data structure is another cpu one
3421 * If yes, queue it to our IPI list and return 1
3422 * If no, return 0
3423 */
3424static int rps_ipi_queued(struct softnet_data *sd)
3425{
3426#ifdef CONFIG_RPS
Christoph Lameter903ceff2014-08-17 12:30:35 -05003427 struct softnet_data *mysd = this_cpu_ptr(&softnet_data);
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003428
3429 if (sd != mysd) {
3430 sd->rps_ipi_next = mysd->rps_ipi_list;
3431 mysd->rps_ipi_list = sd;
3432
3433 __raise_softirq_irqoff(NET_RX_SOFTIRQ);
3434 return 1;
3435 }
3436#endif /* CONFIG_RPS */
3437 return 0;
3438}
3439
Willem de Bruijn99bbc702013-05-20 04:02:32 +00003440#ifdef CONFIG_NET_FLOW_LIMIT
3441int netdev_flow_limit_table_len __read_mostly = (1 << 12);
3442#endif
3443
3444static bool skb_flow_limit(struct sk_buff *skb, unsigned int qlen)
3445{
3446#ifdef CONFIG_NET_FLOW_LIMIT
3447 struct sd_flow_limit *fl;
3448 struct softnet_data *sd;
3449 unsigned int old_flow, new_flow;
3450
3451 if (qlen < (netdev_max_backlog >> 1))
3452 return false;
3453
Christoph Lameter903ceff2014-08-17 12:30:35 -05003454 sd = this_cpu_ptr(&softnet_data);
Willem de Bruijn99bbc702013-05-20 04:02:32 +00003455
3456 rcu_read_lock();
3457 fl = rcu_dereference(sd->flow_limit);
3458 if (fl) {
Tom Herbert3958afa1b2013-12-15 22:12:06 -08003459 new_flow = skb_get_hash(skb) & (fl->num_buckets - 1);
Willem de Bruijn99bbc702013-05-20 04:02:32 +00003460 old_flow = fl->history[fl->history_head];
3461 fl->history[fl->history_head] = new_flow;
3462
3463 fl->history_head++;
3464 fl->history_head &= FLOW_LIMIT_HISTORY - 1;
3465
3466 if (likely(fl->buckets[old_flow]))
3467 fl->buckets[old_flow]--;
3468
3469 if (++fl->buckets[new_flow] > (FLOW_LIMIT_HISTORY >> 1)) {
3470 fl->count++;
3471 rcu_read_unlock();
3472 return true;
3473 }
3474 }
3475 rcu_read_unlock();
3476#endif
3477 return false;
3478}
3479
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003480/*
Tom Herbert0a9627f2010-03-16 08:03:29 +00003481 * enqueue_to_backlog is called to queue an skb to a per CPU backlog
3482 * queue (may be a remote CPU queue).
3483 */
Tom Herbertfec5e652010-04-16 16:01:27 -07003484static int enqueue_to_backlog(struct sk_buff *skb, int cpu,
3485 unsigned int *qtail)
Tom Herbert0a9627f2010-03-16 08:03:29 +00003486{
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003487 struct softnet_data *sd;
Tom Herbert0a9627f2010-03-16 08:03:29 +00003488 unsigned long flags;
Willem de Bruijn99bbc702013-05-20 04:02:32 +00003489 unsigned int qlen;
Tom Herbert0a9627f2010-03-16 08:03:29 +00003490
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003491 sd = &per_cpu(softnet_data, cpu);
Tom Herbert0a9627f2010-03-16 08:03:29 +00003492
3493 local_irq_save(flags);
Tom Herbert0a9627f2010-03-16 08:03:29 +00003494
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003495 rps_lock(sd);
Julian Anastasove9e4dd32015-07-09 09:59:09 +03003496 if (!netif_running(skb->dev))
3497 goto drop;
Willem de Bruijn99bbc702013-05-20 04:02:32 +00003498 qlen = skb_queue_len(&sd->input_pkt_queue);
3499 if (qlen <= netdev_max_backlog && !skb_flow_limit(skb, qlen)) {
Li RongQinge008f3f2014-12-08 09:42:55 +08003500 if (qlen) {
Tom Herbert0a9627f2010-03-16 08:03:29 +00003501enqueue:
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003502 __skb_queue_tail(&sd->input_pkt_queue, skb);
Tom Herbert76cc8b12010-05-20 18:37:59 +00003503 input_queue_tail_incr_save(sd, qtail);
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003504 rps_unlock(sd);
Changli Gao152102c2010-03-30 20:16:22 +00003505 local_irq_restore(flags);
Tom Herbert0a9627f2010-03-16 08:03:29 +00003506 return NET_RX_SUCCESS;
3507 }
3508
Eric Dumazetebda37c22010-05-06 23:51:21 +00003509 /* Schedule NAPI for backlog device
3510 * We can use non atomic operation since we own the queue lock
3511 */
3512 if (!__test_and_set_bit(NAPI_STATE_SCHED, &sd->backlog.state)) {
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003513 if (!rps_ipi_queued(sd))
Eric Dumazeteecfd7c2010-05-06 22:07:48 -07003514 ____napi_schedule(sd, &sd->backlog);
Tom Herbert0a9627f2010-03-16 08:03:29 +00003515 }
3516 goto enqueue;
3517 }
3518
Julian Anastasove9e4dd32015-07-09 09:59:09 +03003519drop:
Changli Gaodee42872010-05-02 05:42:16 +00003520 sd->dropped++;
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003521 rps_unlock(sd);
Tom Herbert0a9627f2010-03-16 08:03:29 +00003522
Tom Herbert0a9627f2010-03-16 08:03:29 +00003523 local_irq_restore(flags);
3524
Eric Dumazetcaf586e2010-09-30 21:06:55 +00003525 atomic_long_inc(&skb->dev->rx_dropped);
Tom Herbert0a9627f2010-03-16 08:03:29 +00003526 kfree_skb(skb);
3527 return NET_RX_DROP;
3528}
Linus Torvalds1da177e2005-04-16 15:20:36 -07003529
Ben Hutchingsae78dbf2014-01-10 22:17:24 +00003530static int netif_rx_internal(struct sk_buff *skb)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003531{
Eric Dumazetb0e28f12010-04-15 00:14:07 -07003532 int ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003533
Eric Dumazet588f0332011-11-15 04:12:55 +00003534 net_timestamp_check(netdev_tstamp_prequeue, skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003535
Koki Sanagicf66ba52010-08-23 18:45:02 +09003536 trace_netif_rx(skb);
Eric Dumazetdf334542010-03-24 19:13:54 +00003537#ifdef CONFIG_RPS
Ingo Molnarc5905af2012-02-24 08:31:31 +01003538 if (static_key_false(&rps_needed)) {
Tom Herbertfec5e652010-04-16 16:01:27 -07003539 struct rps_dev_flow voidflow, *rflow = &voidflow;
Eric Dumazetb0e28f12010-04-15 00:14:07 -07003540 int cpu;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003541
Changli Gaocece1942010-08-07 20:35:43 -07003542 preempt_disable();
Eric Dumazetb0e28f12010-04-15 00:14:07 -07003543 rcu_read_lock();
Tom Herbertfec5e652010-04-16 16:01:27 -07003544
3545 cpu = get_rps_cpu(skb->dev, skb, &rflow);
Eric Dumazetb0e28f12010-04-15 00:14:07 -07003546 if (cpu < 0)
3547 cpu = smp_processor_id();
Tom Herbertfec5e652010-04-16 16:01:27 -07003548
3549 ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail);
3550
Eric Dumazetb0e28f12010-04-15 00:14:07 -07003551 rcu_read_unlock();
Changli Gaocece1942010-08-07 20:35:43 -07003552 preempt_enable();
Eric Dumazetadc93002011-11-17 03:13:26 +00003553 } else
3554#endif
Tom Herbertfec5e652010-04-16 16:01:27 -07003555 {
3556 unsigned int qtail;
3557 ret = enqueue_to_backlog(skb, get_cpu(), &qtail);
3558 put_cpu();
3559 }
Eric Dumazetb0e28f12010-04-15 00:14:07 -07003560 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003561}
Ben Hutchingsae78dbf2014-01-10 22:17:24 +00003562
3563/**
3564 * netif_rx - post buffer to the network code
3565 * @skb: buffer to post
3566 *
3567 * This function receives a packet from a device driver and queues it for
3568 * the upper (protocol) levels to process. It always succeeds. The buffer
3569 * may be dropped during processing for congestion control or by the
3570 * protocol layers.
3571 *
3572 * return values:
3573 * NET_RX_SUCCESS (no congestion)
3574 * NET_RX_DROP (packet was dropped)
3575 *
3576 */
3577
3578int netif_rx(struct sk_buff *skb)
3579{
3580 trace_netif_rx_entry(skb);
3581
3582 return netif_rx_internal(skb);
3583}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07003584EXPORT_SYMBOL(netif_rx);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003585
3586int netif_rx_ni(struct sk_buff *skb)
3587{
3588 int err;
3589
Ben Hutchingsae78dbf2014-01-10 22:17:24 +00003590 trace_netif_rx_ni_entry(skb);
3591
Linus Torvalds1da177e2005-04-16 15:20:36 -07003592 preempt_disable();
Ben Hutchingsae78dbf2014-01-10 22:17:24 +00003593 err = netif_rx_internal(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003594 if (local_softirq_pending())
3595 do_softirq();
3596 preempt_enable();
3597
3598 return err;
3599}
Linus Torvalds1da177e2005-04-16 15:20:36 -07003600EXPORT_SYMBOL(netif_rx_ni);
3601
Linus Torvalds1da177e2005-04-16 15:20:36 -07003602static void net_tx_action(struct softirq_action *h)
3603{
Christoph Lameter903ceff2014-08-17 12:30:35 -05003604 struct softnet_data *sd = this_cpu_ptr(&softnet_data);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003605
3606 if (sd->completion_queue) {
3607 struct sk_buff *clist;
3608
3609 local_irq_disable();
3610 clist = sd->completion_queue;
3611 sd->completion_queue = NULL;
3612 local_irq_enable();
3613
3614 while (clist) {
3615 struct sk_buff *skb = clist;
3616 clist = clist->next;
3617
Ilpo Järvinen547b7922008-07-25 21:43:18 -07003618 WARN_ON(atomic_read(&skb->users));
Eric Dumazete6247022013-12-05 04:45:08 -08003619 if (likely(get_kfree_skb_cb(skb)->reason == SKB_REASON_CONSUMED))
3620 trace_consume_skb(skb);
3621 else
3622 trace_kfree_skb(skb, net_tx_action);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003623 __kfree_skb(skb);
3624 }
3625 }
3626
3627 if (sd->output_queue) {
David S. Miller37437bb2008-07-16 02:15:04 -07003628 struct Qdisc *head;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003629
3630 local_irq_disable();
3631 head = sd->output_queue;
3632 sd->output_queue = NULL;
Changli Gaoa9cbd582010-04-26 23:06:24 +00003633 sd->output_queue_tailp = &sd->output_queue;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003634 local_irq_enable();
3635
3636 while (head) {
David S. Miller37437bb2008-07-16 02:15:04 -07003637 struct Qdisc *q = head;
3638 spinlock_t *root_lock;
3639
Linus Torvalds1da177e2005-04-16 15:20:36 -07003640 head = head->next_sched;
3641
David S. Miller5fb66222008-08-02 20:02:43 -07003642 root_lock = qdisc_lock(q);
David S. Miller37437bb2008-07-16 02:15:04 -07003643 if (spin_trylock(root_lock)) {
Peter Zijlstra4e857c52014-03-17 18:06:10 +01003644 smp_mb__before_atomic();
Jarek Poplawskidef82a12008-08-17 21:54:43 -07003645 clear_bit(__QDISC_STATE_SCHED,
3646 &q->state);
David S. Miller37437bb2008-07-16 02:15:04 -07003647 qdisc_run(q);
3648 spin_unlock(root_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003649 } else {
David S. Miller195648b2008-08-19 04:00:36 -07003650 if (!test_bit(__QDISC_STATE_DEACTIVATED,
Jarek Poplawskie8a83e12008-09-07 18:41:21 -07003651 &q->state)) {
David S. Miller195648b2008-08-19 04:00:36 -07003652 __netif_reschedule(q);
Jarek Poplawskie8a83e12008-09-07 18:41:21 -07003653 } else {
Peter Zijlstra4e857c52014-03-17 18:06:10 +01003654 smp_mb__before_atomic();
Jarek Poplawskie8a83e12008-09-07 18:41:21 -07003655 clear_bit(__QDISC_STATE_SCHED,
3656 &q->state);
3657 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07003658 }
3659 }
3660 }
3661}
3662
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00003663#if (defined(CONFIG_BRIDGE) || defined(CONFIG_BRIDGE_MODULE)) && \
3664 (defined(CONFIG_ATM_LANE) || defined(CONFIG_ATM_LANE_MODULE))
Michał Mirosławda678292009-06-05 05:35:28 +00003665/* This hook is defined here for ATM LANE */
3666int (*br_fdb_test_addr_hook)(struct net_device *dev,
3667 unsigned char *addr) __read_mostly;
Stephen Hemminger4fb019a2009-09-11 11:50:08 -07003668EXPORT_SYMBOL_GPL(br_fdb_test_addr_hook);
Michał Mirosławda678292009-06-05 05:35:28 +00003669#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07003670
Herbert Xuf697c3e2007-10-14 00:38:47 -07003671static inline struct sk_buff *handle_ing(struct sk_buff *skb,
3672 struct packet_type **pt_prev,
3673 int *ret, struct net_device *orig_dev)
3674{
Daniel Borkmanne7582ba2015-05-19 22:33:25 +02003675#ifdef CONFIG_NET_CLS_ACT
Daniel Borkmannd2788d32015-05-09 22:51:32 +02003676 struct tcf_proto *cl = rcu_dereference_bh(skb->dev->ingress_cl_list);
3677 struct tcf_result cl_res;
Eric Dumazet24824a02010-10-02 06:11:55 +00003678
Daniel Borkmannc9e99fd2015-05-09 22:51:31 +02003679 /* If there's at least one ingress present somewhere (so
3680 * we get here via enabled static key), remaining devices
3681 * that are not configured with an ingress qdisc will bail
Daniel Borkmannd2788d32015-05-09 22:51:32 +02003682 * out here.
Daniel Borkmannc9e99fd2015-05-09 22:51:31 +02003683 */
Daniel Borkmannd2788d32015-05-09 22:51:32 +02003684 if (!cl)
Daniel Borkmann45771392015-04-10 23:07:54 +02003685 return skb;
Herbert Xuf697c3e2007-10-14 00:38:47 -07003686 if (*pt_prev) {
3687 *ret = deliver_skb(skb, *pt_prev, orig_dev);
3688 *pt_prev = NULL;
Herbert Xuf697c3e2007-10-14 00:38:47 -07003689 }
3690
Florian Westphal33654952015-05-14 00:36:28 +02003691 qdisc_skb_cb(skb)->pkt_len = skb->len;
Daniel Borkmannc9e99fd2015-05-09 22:51:31 +02003692 skb->tc_verd = SET_TC_AT(skb->tc_verd, AT_INGRESS);
Eric Dumazet24ea5912015-07-06 05:18:03 -07003693 qdisc_bstats_cpu_update(cl->q, skb);
Daniel Borkmannc9e99fd2015-05-09 22:51:31 +02003694
Daniel Borkmann3b3ae882015-08-26 23:00:06 +02003695 switch (tc_classify(skb, cl, &cl_res, false)) {
Daniel Borkmannd2788d32015-05-09 22:51:32 +02003696 case TC_ACT_OK:
3697 case TC_ACT_RECLASSIFY:
3698 skb->tc_index = TC_H_MIN(cl_res.classid);
3699 break;
3700 case TC_ACT_SHOT:
Eric Dumazet24ea5912015-07-06 05:18:03 -07003701 qdisc_qstats_cpu_drop(cl->q);
Daniel Borkmannd2788d32015-05-09 22:51:32 +02003702 case TC_ACT_STOLEN:
3703 case TC_ACT_QUEUED:
3704 kfree_skb(skb);
3705 return NULL;
Alexei Starovoitov27b29f62015-09-15 23:05:43 -07003706 case TC_ACT_REDIRECT:
3707 /* skb_mac_header check was done by cls/act_bpf, so
3708 * we can safely push the L2 header back before
3709 * redirecting to another netdev
3710 */
3711 __skb_push(skb, skb->mac_len);
3712 skb_do_redirect(skb);
3713 return NULL;
Daniel Borkmannd2788d32015-05-09 22:51:32 +02003714 default:
3715 break;
Herbert Xuf697c3e2007-10-14 00:38:47 -07003716 }
Daniel Borkmanne7582ba2015-05-19 22:33:25 +02003717#endif /* CONFIG_NET_CLS_ACT */
Herbert Xuf697c3e2007-10-14 00:38:47 -07003718 return skb;
3719}
Linus Torvalds1da177e2005-04-16 15:20:36 -07003720
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00003721/**
3722 * netdev_rx_handler_register - register receive handler
3723 * @dev: device to register a handler for
3724 * @rx_handler: receive handler to register
Jiri Pirko93e2c322010-06-10 03:34:59 +00003725 * @rx_handler_data: data pointer that is used by rx handler
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00003726 *
Masanari Iidae2278672014-02-18 22:54:36 +09003727 * Register a receive handler for a device. This handler will then be
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00003728 * called from __netif_receive_skb. A negative errno code is returned
3729 * on a failure.
3730 *
3731 * The caller must hold the rtnl_mutex.
Jiri Pirko8a4eb572011-03-12 03:14:39 +00003732 *
3733 * For a general description of rx_handler, see enum rx_handler_result.
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00003734 */
3735int netdev_rx_handler_register(struct net_device *dev,
Jiri Pirko93e2c322010-06-10 03:34:59 +00003736 rx_handler_func_t *rx_handler,
3737 void *rx_handler_data)
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00003738{
3739 ASSERT_RTNL();
3740
3741 if (dev->rx_handler)
3742 return -EBUSY;
3743
Eric Dumazet00cfec32013-03-29 03:01:22 +00003744 /* Note: rx_handler_data must be set before rx_handler */
Jiri Pirko93e2c322010-06-10 03:34:59 +00003745 rcu_assign_pointer(dev->rx_handler_data, rx_handler_data);
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00003746 rcu_assign_pointer(dev->rx_handler, rx_handler);
3747
3748 return 0;
3749}
3750EXPORT_SYMBOL_GPL(netdev_rx_handler_register);
3751
3752/**
3753 * netdev_rx_handler_unregister - unregister receive handler
3754 * @dev: device to unregister a handler from
3755 *
Kusanagi Kouichi166ec362013-03-18 02:59:52 +00003756 * Unregister a receive handler from a device.
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00003757 *
3758 * The caller must hold the rtnl_mutex.
3759 */
3760void netdev_rx_handler_unregister(struct net_device *dev)
3761{
3762
3763 ASSERT_RTNL();
Stephen Hemmingera9b3cd72011-08-01 16:19:00 +00003764 RCU_INIT_POINTER(dev->rx_handler, NULL);
Eric Dumazet00cfec32013-03-29 03:01:22 +00003765 /* a reader seeing a non NULL rx_handler in a rcu_read_lock()
3766 * section has a guarantee to see a non NULL rx_handler_data
3767 * as well.
3768 */
3769 synchronize_net();
Stephen Hemmingera9b3cd72011-08-01 16:19:00 +00003770 RCU_INIT_POINTER(dev->rx_handler_data, NULL);
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00003771}
3772EXPORT_SYMBOL_GPL(netdev_rx_handler_unregister);
3773
Mel Gormanb4b9e352012-07-31 16:44:26 -07003774/*
3775 * Limit the use of PFMEMALLOC reserves to those protocols that implement
3776 * the special handling of PFMEMALLOC skbs.
3777 */
3778static bool skb_pfmemalloc_protocol(struct sk_buff *skb)
3779{
3780 switch (skb->protocol) {
Joe Perches2b8837a2014-03-12 10:04:17 -07003781 case htons(ETH_P_ARP):
3782 case htons(ETH_P_IP):
3783 case htons(ETH_P_IPV6):
3784 case htons(ETH_P_8021Q):
3785 case htons(ETH_P_8021AD):
Mel Gormanb4b9e352012-07-31 16:44:26 -07003786 return true;
3787 default:
3788 return false;
3789 }
3790}
3791
Pablo Neirae687ad62015-05-13 18:19:38 +02003792static inline int nf_ingress(struct sk_buff *skb, struct packet_type **pt_prev,
3793 int *ret, struct net_device *orig_dev)
3794{
Daniel Borkmanne7582ba2015-05-19 22:33:25 +02003795#ifdef CONFIG_NETFILTER_INGRESS
Pablo Neirae687ad62015-05-13 18:19:38 +02003796 if (nf_hook_ingress_active(skb)) {
3797 if (*pt_prev) {
3798 *ret = deliver_skb(skb, *pt_prev, orig_dev);
3799 *pt_prev = NULL;
3800 }
3801
3802 return nf_hook_ingress(skb);
3803 }
Daniel Borkmanne7582ba2015-05-19 22:33:25 +02003804#endif /* CONFIG_NETFILTER_INGRESS */
Pablo Neirae687ad62015-05-13 18:19:38 +02003805 return 0;
3806}
Pablo Neirae687ad62015-05-13 18:19:38 +02003807
David S. Miller9754e292013-02-14 15:57:38 -05003808static int __netif_receive_skb_core(struct sk_buff *skb, bool pfmemalloc)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003809{
3810 struct packet_type *ptype, *pt_prev;
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00003811 rx_handler_func_t *rx_handler;
David S. Millerf2ccd8f2005-08-09 19:34:12 -07003812 struct net_device *orig_dev;
Jiri Pirko8a4eb572011-03-12 03:14:39 +00003813 bool deliver_exact = false;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003814 int ret = NET_RX_DROP;
Al Viro252e33462006-11-14 20:48:11 -08003815 __be16 type;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003816
Eric Dumazet588f0332011-11-15 04:12:55 +00003817 net_timestamp_check(!netdev_tstamp_prequeue, skb);
Eric Dumazet81bbb3d2009-09-30 16:42:42 -07003818
Koki Sanagicf66ba52010-08-23 18:45:02 +09003819 trace_netif_receive_skb(skb);
Patrick McHardy9b22ea52008-11-04 14:49:57 -08003820
Joe Eykholtcc9bd5c2008-07-02 18:22:00 -07003821 orig_dev = skb->dev;
Jiri Pirko1765a572011-02-12 06:48:36 +00003822
Arnaldo Carvalho de Meloc1d2bbe2007-04-10 20:45:18 -07003823 skb_reset_network_header(skb);
Eric Dumazetfda55ec2013-01-07 09:28:21 +00003824 if (!skb_transport_header_was_set(skb))
3825 skb_reset_transport_header(skb);
Jiri Pirko0b5c9db2011-06-10 06:56:58 +00003826 skb_reset_mac_len(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003827
3828 pt_prev = NULL;
3829
David S. Miller63d8ea72011-02-28 10:48:59 -08003830another_round:
David S. Millerb6858172012-07-23 16:27:54 -07003831 skb->skb_iif = skb->dev->ifindex;
David S. Miller63d8ea72011-02-28 10:48:59 -08003832
3833 __this_cpu_inc(softnet_data.processed);
3834
Patrick McHardy8ad227f2013-04-19 02:04:31 +00003835 if (skb->protocol == cpu_to_be16(ETH_P_8021Q) ||
3836 skb->protocol == cpu_to_be16(ETH_P_8021AD)) {
Vlad Yasevich0d5501c2014-08-08 14:42:13 -04003837 skb = skb_vlan_untag(skb);
Jiri Pirkobcc6d472011-04-07 19:48:33 +00003838 if (unlikely(!skb))
Julian Anastasov2c17d272015-07-09 09:59:10 +03003839 goto out;
Jiri Pirkobcc6d472011-04-07 19:48:33 +00003840 }
3841
Linus Torvalds1da177e2005-04-16 15:20:36 -07003842#ifdef CONFIG_NET_CLS_ACT
3843 if (skb->tc_verd & TC_NCLS) {
3844 skb->tc_verd = CLR_TC_NCLS(skb->tc_verd);
3845 goto ncls;
3846 }
3847#endif
3848
David S. Miller9754e292013-02-14 15:57:38 -05003849 if (pfmemalloc)
Mel Gormanb4b9e352012-07-31 16:44:26 -07003850 goto skip_taps;
3851
Linus Torvalds1da177e2005-04-16 15:20:36 -07003852 list_for_each_entry_rcu(ptype, &ptype_all, list) {
Salam Noureddine7866a622015-01-27 11:35:48 -08003853 if (pt_prev)
3854 ret = deliver_skb(skb, pt_prev, orig_dev);
3855 pt_prev = ptype;
3856 }
3857
3858 list_for_each_entry_rcu(ptype, &skb->dev->ptype_all, list) {
3859 if (pt_prev)
3860 ret = deliver_skb(skb, pt_prev, orig_dev);
3861 pt_prev = ptype;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003862 }
3863
Mel Gormanb4b9e352012-07-31 16:44:26 -07003864skip_taps:
Pablo Neira1cf519002015-05-13 18:19:37 +02003865#ifdef CONFIG_NET_INGRESS
Daniel Borkmann45771392015-04-10 23:07:54 +02003866 if (static_key_false(&ingress_needed)) {
3867 skb = handle_ing(skb, &pt_prev, &ret, orig_dev);
3868 if (!skb)
Julian Anastasov2c17d272015-07-09 09:59:10 +03003869 goto out;
Pablo Neirae687ad62015-05-13 18:19:38 +02003870
3871 if (nf_ingress(skb, &pt_prev, &ret, orig_dev) < 0)
Julian Anastasov2c17d272015-07-09 09:59:10 +03003872 goto out;
Daniel Borkmann45771392015-04-10 23:07:54 +02003873 }
Pablo Neira1cf519002015-05-13 18:19:37 +02003874#endif
3875#ifdef CONFIG_NET_CLS_ACT
Daniel Borkmann45771392015-04-10 23:07:54 +02003876 skb->tc_verd = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003877ncls:
3878#endif
David S. Miller9754e292013-02-14 15:57:38 -05003879 if (pfmemalloc && !skb_pfmemalloc_protocol(skb))
Mel Gormanb4b9e352012-07-31 16:44:26 -07003880 goto drop;
3881
Jiri Pirkodf8a39d2015-01-13 17:13:44 +01003882 if (skb_vlan_tag_present(skb)) {
John Fastabend24257172011-10-10 09:16:41 +00003883 if (pt_prev) {
3884 ret = deliver_skb(skb, pt_prev, orig_dev);
3885 pt_prev = NULL;
3886 }
Florian Zumbiehl48cc32d32012-10-07 15:51:58 +00003887 if (vlan_do_receive(&skb))
John Fastabend24257172011-10-10 09:16:41 +00003888 goto another_round;
3889 else if (unlikely(!skb))
Julian Anastasov2c17d272015-07-09 09:59:10 +03003890 goto out;
John Fastabend24257172011-10-10 09:16:41 +00003891 }
3892
Florian Zumbiehl48cc32d32012-10-07 15:51:58 +00003893 rx_handler = rcu_dereference(skb->dev->rx_handler);
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00003894 if (rx_handler) {
3895 if (pt_prev) {
3896 ret = deliver_skb(skb, pt_prev, orig_dev);
3897 pt_prev = NULL;
3898 }
Jiri Pirko8a4eb572011-03-12 03:14:39 +00003899 switch (rx_handler(&skb)) {
3900 case RX_HANDLER_CONSUMED:
Cristian Bercaru3bc1b1a2013-03-08 07:03:38 +00003901 ret = NET_RX_SUCCESS;
Julian Anastasov2c17d272015-07-09 09:59:10 +03003902 goto out;
Jiri Pirko8a4eb572011-03-12 03:14:39 +00003903 case RX_HANDLER_ANOTHER:
David S. Miller63d8ea72011-02-28 10:48:59 -08003904 goto another_round;
Jiri Pirko8a4eb572011-03-12 03:14:39 +00003905 case RX_HANDLER_EXACT:
3906 deliver_exact = true;
3907 case RX_HANDLER_PASS:
3908 break;
3909 default:
3910 BUG();
3911 }
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00003912 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07003913
Jiri Pirkodf8a39d2015-01-13 17:13:44 +01003914 if (unlikely(skb_vlan_tag_present(skb))) {
3915 if (skb_vlan_tag_get_id(skb))
Eric Dumazetd4b812d2013-07-18 07:19:26 -07003916 skb->pkt_type = PACKET_OTHERHOST;
3917 /* Note: we might in the future use prio bits
3918 * and set skb->priority like in vlan_do_receive()
3919 * For the time being, just ignore Priority Code Point
3920 */
3921 skb->vlan_tci = 0;
3922 }
Florian Zumbiehl48cc32d32012-10-07 15:51:58 +00003923
Linus Torvalds1da177e2005-04-16 15:20:36 -07003924 type = skb->protocol;
Salam Noureddine7866a622015-01-27 11:35:48 -08003925
3926 /* deliver only exact match when indicated */
3927 if (likely(!deliver_exact)) {
3928 deliver_ptype_list_skb(skb, &pt_prev, orig_dev, type,
3929 &ptype_base[ntohs(type) &
3930 PTYPE_HASH_MASK]);
3931 }
3932
3933 deliver_ptype_list_skb(skb, &pt_prev, orig_dev, type,
3934 &orig_dev->ptype_specific);
3935
3936 if (unlikely(skb->dev != orig_dev)) {
3937 deliver_ptype_list_skb(skb, &pt_prev, orig_dev, type,
3938 &skb->dev->ptype_specific);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003939 }
3940
3941 if (pt_prev) {
Michael S. Tsirkin1080e512012-07-20 09:23:17 +00003942 if (unlikely(skb_orphan_frags(skb, GFP_ATOMIC)))
Michael S. Tsirkin0e698bf2012-09-15 22:44:16 +00003943 goto drop;
Michael S. Tsirkin1080e512012-07-20 09:23:17 +00003944 else
3945 ret = pt_prev->func(skb, skb->dev, pt_prev, orig_dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003946 } else {
Mel Gormanb4b9e352012-07-31 16:44:26 -07003947drop:
Eric Dumazetcaf586e2010-09-30 21:06:55 +00003948 atomic_long_inc(&skb->dev->rx_dropped);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003949 kfree_skb(skb);
3950 /* Jamal, now you will not able to escape explaining
3951 * me how you were going to use this. :-)
3952 */
3953 ret = NET_RX_DROP;
3954 }
3955
Julian Anastasov2c17d272015-07-09 09:59:10 +03003956out:
David S. Miller9754e292013-02-14 15:57:38 -05003957 return ret;
3958}
3959
3960static int __netif_receive_skb(struct sk_buff *skb)
3961{
3962 int ret;
3963
3964 if (sk_memalloc_socks() && skb_pfmemalloc(skb)) {
3965 unsigned long pflags = current->flags;
3966
3967 /*
3968 * PFMEMALLOC skbs are special, they should
3969 * - be delivered to SOCK_MEMALLOC sockets only
3970 * - stay away from userspace
3971 * - have bounded memory usage
3972 *
3973 * Use PF_MEMALLOC as this saves us from propagating the allocation
3974 * context down to all allocation sites.
3975 */
3976 current->flags |= PF_MEMALLOC;
3977 ret = __netif_receive_skb_core(skb, true);
3978 tsk_restore_flags(current, pflags, PF_MEMALLOC);
3979 } else
3980 ret = __netif_receive_skb_core(skb, false);
3981
Linus Torvalds1da177e2005-04-16 15:20:36 -07003982 return ret;
3983}
Tom Herbert0a9627f2010-03-16 08:03:29 +00003984
Ben Hutchingsae78dbf2014-01-10 22:17:24 +00003985static int netif_receive_skb_internal(struct sk_buff *skb)
Tom Herbert0a9627f2010-03-16 08:03:29 +00003986{
Julian Anastasov2c17d272015-07-09 09:59:10 +03003987 int ret;
3988
Eric Dumazet588f0332011-11-15 04:12:55 +00003989 net_timestamp_check(netdev_tstamp_prequeue, skb);
Eric Dumazet3b098e22010-05-15 23:57:10 -07003990
Richard Cochranc1f19b52010-07-17 08:49:36 +00003991 if (skb_defer_rx_timestamp(skb))
3992 return NET_RX_SUCCESS;
3993
Julian Anastasov2c17d272015-07-09 09:59:10 +03003994 rcu_read_lock();
3995
Eric Dumazetdf334542010-03-24 19:13:54 +00003996#ifdef CONFIG_RPS
Ingo Molnarc5905af2012-02-24 08:31:31 +01003997 if (static_key_false(&rps_needed)) {
Eric Dumazet3b098e22010-05-15 23:57:10 -07003998 struct rps_dev_flow voidflow, *rflow = &voidflow;
Julian Anastasov2c17d272015-07-09 09:59:10 +03003999 int cpu = get_rps_cpu(skb->dev, skb, &rflow);
Tom Herbertfec5e652010-04-16 16:01:27 -07004000
Eric Dumazet3b098e22010-05-15 23:57:10 -07004001 if (cpu >= 0) {
4002 ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail);
4003 rcu_read_unlock();
Eric Dumazetadc93002011-11-17 03:13:26 +00004004 return ret;
Eric Dumazet3b098e22010-05-15 23:57:10 -07004005 }
Tom Herbertfec5e652010-04-16 16:01:27 -07004006 }
Tom Herbert1e94d722010-03-18 17:45:44 -07004007#endif
Julian Anastasov2c17d272015-07-09 09:59:10 +03004008 ret = __netif_receive_skb(skb);
4009 rcu_read_unlock();
4010 return ret;
Tom Herbert0a9627f2010-03-16 08:03:29 +00004011}
Ben Hutchingsae78dbf2014-01-10 22:17:24 +00004012
4013/**
4014 * netif_receive_skb - process receive buffer from network
4015 * @skb: buffer to process
4016 *
4017 * netif_receive_skb() is the main receive data processing function.
4018 * It always succeeds. The buffer may be dropped during processing
4019 * for congestion control or by the protocol layers.
4020 *
4021 * This function may only be called from softirq context and interrupts
4022 * should be enabled.
4023 *
4024 * Return values (usually ignored):
4025 * NET_RX_SUCCESS: no congestion
4026 * NET_RX_DROP: packet was dropped
4027 */
Eric W. Biederman04eb4482015-09-15 20:04:15 -05004028int netif_receive_skb(struct sk_buff *skb)
Ben Hutchingsae78dbf2014-01-10 22:17:24 +00004029{
4030 trace_netif_receive_skb_entry(skb);
4031
4032 return netif_receive_skb_internal(skb);
4033}
Eric W. Biederman04eb4482015-09-15 20:04:15 -05004034EXPORT_SYMBOL(netif_receive_skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004035
Eric Dumazet88751272010-04-19 05:07:33 +00004036/* Network device is going away, flush any packets still pending
4037 * Called with irqs disabled.
4038 */
Changli Gao152102c2010-03-30 20:16:22 +00004039static void flush_backlog(void *arg)
Stephen Hemminger6e583ce2008-08-03 21:29:57 -07004040{
Changli Gao152102c2010-03-30 20:16:22 +00004041 struct net_device *dev = arg;
Christoph Lameter903ceff2014-08-17 12:30:35 -05004042 struct softnet_data *sd = this_cpu_ptr(&softnet_data);
Stephen Hemminger6e583ce2008-08-03 21:29:57 -07004043 struct sk_buff *skb, *tmp;
4044
Eric Dumazete36fa2f2010-04-19 21:17:14 +00004045 rps_lock(sd);
Changli Gao6e7676c2010-04-27 15:07:33 -07004046 skb_queue_walk_safe(&sd->input_pkt_queue, skb, tmp) {
Stephen Hemminger6e583ce2008-08-03 21:29:57 -07004047 if (skb->dev == dev) {
Eric Dumazete36fa2f2010-04-19 21:17:14 +00004048 __skb_unlink(skb, &sd->input_pkt_queue);
Stephen Hemminger6e583ce2008-08-03 21:29:57 -07004049 kfree_skb(skb);
Tom Herbert76cc8b12010-05-20 18:37:59 +00004050 input_queue_head_incr(sd);
Stephen Hemminger6e583ce2008-08-03 21:29:57 -07004051 }
Changli Gao6e7676c2010-04-27 15:07:33 -07004052 }
Eric Dumazete36fa2f2010-04-19 21:17:14 +00004053 rps_unlock(sd);
Changli Gao6e7676c2010-04-27 15:07:33 -07004054
4055 skb_queue_walk_safe(&sd->process_queue, skb, tmp) {
4056 if (skb->dev == dev) {
4057 __skb_unlink(skb, &sd->process_queue);
4058 kfree_skb(skb);
Tom Herbert76cc8b12010-05-20 18:37:59 +00004059 input_queue_head_incr(sd);
Changli Gao6e7676c2010-04-27 15:07:33 -07004060 }
4061 }
Stephen Hemminger6e583ce2008-08-03 21:29:57 -07004062}
4063
Herbert Xud565b0a2008-12-15 23:38:52 -08004064static int napi_gro_complete(struct sk_buff *skb)
4065{
Vlad Yasevich22061d82012-11-15 08:49:11 +00004066 struct packet_offload *ptype;
Herbert Xud565b0a2008-12-15 23:38:52 -08004067 __be16 type = skb->protocol;
Vlad Yasevich22061d82012-11-15 08:49:11 +00004068 struct list_head *head = &offload_base;
Herbert Xud565b0a2008-12-15 23:38:52 -08004069 int err = -ENOENT;
4070
Eric Dumazetc3c7c252012-12-06 13:54:59 +00004071 BUILD_BUG_ON(sizeof(struct napi_gro_cb) > sizeof(skb->cb));
4072
Herbert Xufc59f9a2009-04-14 15:11:06 -07004073 if (NAPI_GRO_CB(skb)->count == 1) {
4074 skb_shinfo(skb)->gso_size = 0;
Herbert Xud565b0a2008-12-15 23:38:52 -08004075 goto out;
Herbert Xufc59f9a2009-04-14 15:11:06 -07004076 }
Herbert Xud565b0a2008-12-15 23:38:52 -08004077
4078 rcu_read_lock();
4079 list_for_each_entry_rcu(ptype, head, list) {
Vlad Yasevichf191a1d2012-11-15 08:49:23 +00004080 if (ptype->type != type || !ptype->callbacks.gro_complete)
Herbert Xud565b0a2008-12-15 23:38:52 -08004081 continue;
4082
Jerry Chu299603e82013-12-11 20:53:45 -08004083 err = ptype->callbacks.gro_complete(skb, 0);
Herbert Xud565b0a2008-12-15 23:38:52 -08004084 break;
4085 }
4086 rcu_read_unlock();
4087
4088 if (err) {
4089 WARN_ON(&ptype->list == head);
4090 kfree_skb(skb);
4091 return NET_RX_SUCCESS;
4092 }
4093
4094out:
Ben Hutchingsae78dbf2014-01-10 22:17:24 +00004095 return netif_receive_skb_internal(skb);
Herbert Xud565b0a2008-12-15 23:38:52 -08004096}
4097
Eric Dumazet2e71a6f2012-10-06 08:08:49 +00004098/* napi->gro_list contains packets ordered by age.
4099 * youngest packets at the head of it.
4100 * Complete skbs in reverse order to reduce latencies.
4101 */
4102void napi_gro_flush(struct napi_struct *napi, bool flush_old)
Herbert Xud565b0a2008-12-15 23:38:52 -08004103{
Eric Dumazet2e71a6f2012-10-06 08:08:49 +00004104 struct sk_buff *skb, *prev = NULL;
Herbert Xud565b0a2008-12-15 23:38:52 -08004105
Eric Dumazet2e71a6f2012-10-06 08:08:49 +00004106 /* scan list and build reverse chain */
4107 for (skb = napi->gro_list; skb != NULL; skb = skb->next) {
4108 skb->prev = prev;
4109 prev = skb;
Herbert Xud565b0a2008-12-15 23:38:52 -08004110 }
4111
Eric Dumazet2e71a6f2012-10-06 08:08:49 +00004112 for (skb = prev; skb; skb = prev) {
4113 skb->next = NULL;
4114
4115 if (flush_old && NAPI_GRO_CB(skb)->age == jiffies)
4116 return;
4117
4118 prev = skb->prev;
4119 napi_gro_complete(skb);
4120 napi->gro_count--;
4121 }
4122
Herbert Xud565b0a2008-12-15 23:38:52 -08004123 napi->gro_list = NULL;
4124}
Eric Dumazet86cac582010-08-31 18:25:32 +00004125EXPORT_SYMBOL(napi_gro_flush);
Herbert Xud565b0a2008-12-15 23:38:52 -08004126
Eric Dumazet89c5fa32012-12-10 13:28:16 +00004127static void gro_list_prepare(struct napi_struct *napi, struct sk_buff *skb)
4128{
4129 struct sk_buff *p;
4130 unsigned int maclen = skb->dev->hard_header_len;
Tom Herbert0b4cec82014-01-15 08:58:06 -08004131 u32 hash = skb_get_hash_raw(skb);
Eric Dumazet89c5fa32012-12-10 13:28:16 +00004132
4133 for (p = napi->gro_list; p; p = p->next) {
4134 unsigned long diffs;
4135
Tom Herbert0b4cec82014-01-15 08:58:06 -08004136 NAPI_GRO_CB(p)->flush = 0;
4137
4138 if (hash != skb_get_hash_raw(p)) {
4139 NAPI_GRO_CB(p)->same_flow = 0;
4140 continue;
4141 }
4142
Eric Dumazet89c5fa32012-12-10 13:28:16 +00004143 diffs = (unsigned long)p->dev ^ (unsigned long)skb->dev;
4144 diffs |= p->vlan_tci ^ skb->vlan_tci;
4145 if (maclen == ETH_HLEN)
4146 diffs |= compare_ether_header(skb_mac_header(p),
Eric Dumazeta50e2332014-03-29 21:28:21 -07004147 skb_mac_header(skb));
Eric Dumazet89c5fa32012-12-10 13:28:16 +00004148 else if (!diffs)
4149 diffs = memcmp(skb_mac_header(p),
Eric Dumazeta50e2332014-03-29 21:28:21 -07004150 skb_mac_header(skb),
Eric Dumazet89c5fa32012-12-10 13:28:16 +00004151 maclen);
4152 NAPI_GRO_CB(p)->same_flow = !diffs;
Eric Dumazet89c5fa32012-12-10 13:28:16 +00004153 }
4154}
4155
Jerry Chu299603e82013-12-11 20:53:45 -08004156static void skb_gro_reset_offset(struct sk_buff *skb)
4157{
4158 const struct skb_shared_info *pinfo = skb_shinfo(skb);
4159 const skb_frag_t *frag0 = &pinfo->frags[0];
4160
4161 NAPI_GRO_CB(skb)->data_offset = 0;
4162 NAPI_GRO_CB(skb)->frag0 = NULL;
4163 NAPI_GRO_CB(skb)->frag0_len = 0;
4164
4165 if (skb_mac_header(skb) == skb_tail_pointer(skb) &&
4166 pinfo->nr_frags &&
4167 !PageHighMem(skb_frag_page(frag0))) {
4168 NAPI_GRO_CB(skb)->frag0 = skb_frag_address(frag0);
4169 NAPI_GRO_CB(skb)->frag0_len = skb_frag_size(frag0);
Herbert Xud565b0a2008-12-15 23:38:52 -08004170 }
4171}
4172
Eric Dumazeta50e2332014-03-29 21:28:21 -07004173static void gro_pull_from_frag0(struct sk_buff *skb, int grow)
4174{
4175 struct skb_shared_info *pinfo = skb_shinfo(skb);
4176
4177 BUG_ON(skb->end - skb->tail < grow);
4178
4179 memcpy(skb_tail_pointer(skb), NAPI_GRO_CB(skb)->frag0, grow);
4180
4181 skb->data_len -= grow;
4182 skb->tail += grow;
4183
4184 pinfo->frags[0].page_offset += grow;
4185 skb_frag_size_sub(&pinfo->frags[0], grow);
4186
4187 if (unlikely(!skb_frag_size(&pinfo->frags[0]))) {
4188 skb_frag_unref(skb, 0);
4189 memmove(pinfo->frags, pinfo->frags + 1,
4190 --pinfo->nr_frags * sizeof(pinfo->frags[0]));
4191 }
4192}
4193
Rami Rosenbb728822012-11-28 21:55:25 +00004194static enum gro_result dev_gro_receive(struct napi_struct *napi, struct sk_buff *skb)
Herbert Xud565b0a2008-12-15 23:38:52 -08004195{
4196 struct sk_buff **pp = NULL;
Vlad Yasevich22061d82012-11-15 08:49:11 +00004197 struct packet_offload *ptype;
Herbert Xud565b0a2008-12-15 23:38:52 -08004198 __be16 type = skb->protocol;
Vlad Yasevich22061d82012-11-15 08:49:11 +00004199 struct list_head *head = &offload_base;
Herbert Xu0da2afd52008-12-26 14:57:42 -08004200 int same_flow;
Ben Hutchings5b252f02009-10-29 07:17:09 +00004201 enum gro_result ret;
Eric Dumazeta50e2332014-03-29 21:28:21 -07004202 int grow;
Herbert Xud565b0a2008-12-15 23:38:52 -08004203
Eric W. Biederman9c62a682014-03-14 20:51:52 -07004204 if (!(skb->dev->features & NETIF_F_GRO))
Herbert Xud565b0a2008-12-15 23:38:52 -08004205 goto normal;
4206
Tom Herbert5a212322014-08-31 15:12:41 -07004207 if (skb_is_gso(skb) || skb_has_frag_list(skb) || skb->csum_bad)
Herbert Xuf17f5c92009-01-14 14:36:12 -08004208 goto normal;
4209
Eric Dumazet89c5fa32012-12-10 13:28:16 +00004210 gro_list_prepare(napi, skb);
4211
Herbert Xud565b0a2008-12-15 23:38:52 -08004212 rcu_read_lock();
4213 list_for_each_entry_rcu(ptype, head, list) {
Vlad Yasevichf191a1d2012-11-15 08:49:23 +00004214 if (ptype->type != type || !ptype->callbacks.gro_receive)
Herbert Xud565b0a2008-12-15 23:38:52 -08004215 continue;
4216
Herbert Xu86911732009-01-29 14:19:50 +00004217 skb_set_network_header(skb, skb_gro_offset(skb));
Eric Dumazetefd94502013-02-14 17:31:48 +00004218 skb_reset_mac_len(skb);
Herbert Xud565b0a2008-12-15 23:38:52 -08004219 NAPI_GRO_CB(skb)->same_flow = 0;
4220 NAPI_GRO_CB(skb)->flush = 0;
Herbert Xu5d38a072009-01-04 16:13:40 -08004221 NAPI_GRO_CB(skb)->free = 0;
Or Gerlitzb582ef02014-01-20 13:59:19 +02004222 NAPI_GRO_CB(skb)->udp_mark = 0;
Tom Herbert15e23962015-02-10 16:30:31 -08004223 NAPI_GRO_CB(skb)->gro_remcsum_start = 0;
Herbert Xud565b0a2008-12-15 23:38:52 -08004224
Tom Herbert662880f2014-08-27 21:26:56 -07004225 /* Setup for GRO checksum validation */
4226 switch (skb->ip_summed) {
4227 case CHECKSUM_COMPLETE:
4228 NAPI_GRO_CB(skb)->csum = skb->csum;
4229 NAPI_GRO_CB(skb)->csum_valid = 1;
4230 NAPI_GRO_CB(skb)->csum_cnt = 0;
4231 break;
4232 case CHECKSUM_UNNECESSARY:
4233 NAPI_GRO_CB(skb)->csum_cnt = skb->csum_level + 1;
4234 NAPI_GRO_CB(skb)->csum_valid = 0;
4235 break;
4236 default:
4237 NAPI_GRO_CB(skb)->csum_cnt = 0;
4238 NAPI_GRO_CB(skb)->csum_valid = 0;
4239 }
Herbert Xud565b0a2008-12-15 23:38:52 -08004240
Vlad Yasevichf191a1d2012-11-15 08:49:23 +00004241 pp = ptype->callbacks.gro_receive(&napi->gro_list, skb);
Herbert Xud565b0a2008-12-15 23:38:52 -08004242 break;
4243 }
4244 rcu_read_unlock();
4245
4246 if (&ptype->list == head)
4247 goto normal;
4248
Herbert Xu0da2afd52008-12-26 14:57:42 -08004249 same_flow = NAPI_GRO_CB(skb)->same_flow;
Herbert Xu5d0d9be2009-01-29 14:19:48 +00004250 ret = NAPI_GRO_CB(skb)->free ? GRO_MERGED_FREE : GRO_MERGED;
Herbert Xu0da2afd52008-12-26 14:57:42 -08004251
Herbert Xud565b0a2008-12-15 23:38:52 -08004252 if (pp) {
4253 struct sk_buff *nskb = *pp;
4254
4255 *pp = nskb->next;
4256 nskb->next = NULL;
4257 napi_gro_complete(nskb);
Herbert Xu4ae55442009-02-08 18:00:36 +00004258 napi->gro_count--;
Herbert Xud565b0a2008-12-15 23:38:52 -08004259 }
4260
Herbert Xu0da2afd52008-12-26 14:57:42 -08004261 if (same_flow)
Herbert Xud565b0a2008-12-15 23:38:52 -08004262 goto ok;
4263
Eric Dumazet600adc12014-01-09 14:12:19 -08004264 if (NAPI_GRO_CB(skb)->flush)
Herbert Xud565b0a2008-12-15 23:38:52 -08004265 goto normal;
Herbert Xud565b0a2008-12-15 23:38:52 -08004266
Eric Dumazet600adc12014-01-09 14:12:19 -08004267 if (unlikely(napi->gro_count >= MAX_GRO_SKBS)) {
4268 struct sk_buff *nskb = napi->gro_list;
4269
4270 /* locate the end of the list to select the 'oldest' flow */
4271 while (nskb->next) {
4272 pp = &nskb->next;
4273 nskb = *pp;
4274 }
4275 *pp = NULL;
4276 nskb->next = NULL;
4277 napi_gro_complete(nskb);
4278 } else {
4279 napi->gro_count++;
4280 }
Herbert Xud565b0a2008-12-15 23:38:52 -08004281 NAPI_GRO_CB(skb)->count = 1;
Eric Dumazet2e71a6f2012-10-06 08:08:49 +00004282 NAPI_GRO_CB(skb)->age = jiffies;
Eric Dumazet29e98242014-05-16 11:34:37 -07004283 NAPI_GRO_CB(skb)->last = skb;
Herbert Xu86911732009-01-29 14:19:50 +00004284 skb_shinfo(skb)->gso_size = skb_gro_len(skb);
Herbert Xud565b0a2008-12-15 23:38:52 -08004285 skb->next = napi->gro_list;
4286 napi->gro_list = skb;
Herbert Xu5d0d9be2009-01-29 14:19:48 +00004287 ret = GRO_HELD;
Herbert Xud565b0a2008-12-15 23:38:52 -08004288
Herbert Xuad0f9902009-02-01 01:24:55 -08004289pull:
Eric Dumazeta50e2332014-03-29 21:28:21 -07004290 grow = skb_gro_offset(skb) - skb_headlen(skb);
4291 if (grow > 0)
4292 gro_pull_from_frag0(skb, grow);
Herbert Xud565b0a2008-12-15 23:38:52 -08004293ok:
Herbert Xu5d0d9be2009-01-29 14:19:48 +00004294 return ret;
Herbert Xud565b0a2008-12-15 23:38:52 -08004295
4296normal:
Herbert Xuad0f9902009-02-01 01:24:55 -08004297 ret = GRO_NORMAL;
4298 goto pull;
Herbert Xu5d38a072009-01-04 16:13:40 -08004299}
Herbert Xu96e93ea2009-01-06 10:49:34 -08004300
Jerry Chubf5a7552014-01-07 10:23:19 -08004301struct packet_offload *gro_find_receive_by_type(__be16 type)
4302{
4303 struct list_head *offload_head = &offload_base;
4304 struct packet_offload *ptype;
4305
4306 list_for_each_entry_rcu(ptype, offload_head, list) {
4307 if (ptype->type != type || !ptype->callbacks.gro_receive)
4308 continue;
4309 return ptype;
4310 }
4311 return NULL;
4312}
Or Gerlitze27a2f82014-01-20 13:59:20 +02004313EXPORT_SYMBOL(gro_find_receive_by_type);
Jerry Chubf5a7552014-01-07 10:23:19 -08004314
4315struct packet_offload *gro_find_complete_by_type(__be16 type)
4316{
4317 struct list_head *offload_head = &offload_base;
4318 struct packet_offload *ptype;
4319
4320 list_for_each_entry_rcu(ptype, offload_head, list) {
4321 if (ptype->type != type || !ptype->callbacks.gro_complete)
4322 continue;
4323 return ptype;
4324 }
4325 return NULL;
4326}
Or Gerlitze27a2f82014-01-20 13:59:20 +02004327EXPORT_SYMBOL(gro_find_complete_by_type);
Herbert Xu96e93ea2009-01-06 10:49:34 -08004328
Rami Rosenbb728822012-11-28 21:55:25 +00004329static gro_result_t napi_skb_finish(gro_result_t ret, struct sk_buff *skb)
Herbert Xu5d38a072009-01-04 16:13:40 -08004330{
Herbert Xu5d0d9be2009-01-29 14:19:48 +00004331 switch (ret) {
4332 case GRO_NORMAL:
Ben Hutchingsae78dbf2014-01-10 22:17:24 +00004333 if (netif_receive_skb_internal(skb))
Ben Hutchingsc7c4b3b2009-10-29 21:36:53 -07004334 ret = GRO_DROP;
4335 break;
Herbert Xu5d38a072009-01-04 16:13:40 -08004336
Herbert Xu5d0d9be2009-01-29 14:19:48 +00004337 case GRO_DROP:
Herbert Xu5d38a072009-01-04 16:13:40 -08004338 kfree_skb(skb);
4339 break;
Ben Hutchings5b252f02009-10-29 07:17:09 +00004340
Eric Dumazetdaa86542012-04-19 07:07:40 +00004341 case GRO_MERGED_FREE:
Eric Dumazetd7e88832012-04-30 08:10:34 +00004342 if (NAPI_GRO_CB(skb)->free == NAPI_GRO_FREE_STOLEN_HEAD)
4343 kmem_cache_free(skbuff_head_cache, skb);
4344 else
4345 __kfree_skb(skb);
Eric Dumazetdaa86542012-04-19 07:07:40 +00004346 break;
4347
Ben Hutchings5b252f02009-10-29 07:17:09 +00004348 case GRO_HELD:
4349 case GRO_MERGED:
4350 break;
Herbert Xu5d38a072009-01-04 16:13:40 -08004351 }
4352
Ben Hutchingsc7c4b3b2009-10-29 21:36:53 -07004353 return ret;
Herbert Xu5d0d9be2009-01-29 14:19:48 +00004354}
Herbert Xu5d0d9be2009-01-29 14:19:48 +00004355
Ben Hutchingsc7c4b3b2009-10-29 21:36:53 -07004356gro_result_t napi_gro_receive(struct napi_struct *napi, struct sk_buff *skb)
Herbert Xu5d0d9be2009-01-29 14:19:48 +00004357{
Ben Hutchingsae78dbf2014-01-10 22:17:24 +00004358 trace_napi_gro_receive_entry(skb);
Herbert Xu86911732009-01-29 14:19:50 +00004359
Eric Dumazeta50e2332014-03-29 21:28:21 -07004360 skb_gro_reset_offset(skb);
4361
Eric Dumazet89c5fa32012-12-10 13:28:16 +00004362 return napi_skb_finish(dev_gro_receive(napi, skb), skb);
Herbert Xud565b0a2008-12-15 23:38:52 -08004363}
4364EXPORT_SYMBOL(napi_gro_receive);
4365
stephen hemmingerd0c2b0d2010-10-19 07:12:10 +00004366static void napi_reuse_skb(struct napi_struct *napi, struct sk_buff *skb)
Herbert Xu96e93ea2009-01-06 10:49:34 -08004367{
Eric Dumazet93a35f52014-10-23 06:30:30 -07004368 if (unlikely(skb->pfmemalloc)) {
4369 consume_skb(skb);
4370 return;
4371 }
Herbert Xu96e93ea2009-01-06 10:49:34 -08004372 __skb_pull(skb, skb_headlen(skb));
Eric Dumazet2a2a4592012-03-21 06:58:03 +00004373 /* restore the reserve we had after netdev_alloc_skb_ip_align() */
4374 skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN - skb_headroom(skb));
Jesse Gross3701e512010-10-20 13:56:06 +00004375 skb->vlan_tci = 0;
Herbert Xu66c46d72011-01-29 20:44:54 -08004376 skb->dev = napi->dev;
Andy Gospodarek6d152e22011-02-02 14:53:25 -08004377 skb->skb_iif = 0;
Jerry Chuc3caf112014-07-14 15:54:46 -07004378 skb->encapsulation = 0;
4379 skb_shinfo(skb)->gso_type = 0;
Eric Dumazete33d0ba2014-04-03 09:28:10 -07004380 skb->truesize = SKB_TRUESIZE(skb_end_offset(skb));
Herbert Xu96e93ea2009-01-06 10:49:34 -08004381
4382 napi->skb = skb;
4383}
Herbert Xu96e93ea2009-01-06 10:49:34 -08004384
Herbert Xu76620aa2009-04-16 02:02:07 -07004385struct sk_buff *napi_get_frags(struct napi_struct *napi)
Herbert Xu5d38a072009-01-04 16:13:40 -08004386{
Herbert Xu5d38a072009-01-04 16:13:40 -08004387 struct sk_buff *skb = napi->skb;
Herbert Xu5d38a072009-01-04 16:13:40 -08004388
4389 if (!skb) {
Alexander Duyckfd11a832014-12-09 19:40:49 -08004390 skb = napi_alloc_skb(napi, GRO_MAX_HEAD);
Eric Dumazet84b9cd62013-12-05 21:44:27 -08004391 napi->skb = skb;
Herbert Xu5d38a072009-01-04 16:13:40 -08004392 }
Herbert Xu96e93ea2009-01-06 10:49:34 -08004393 return skb;
4394}
Herbert Xu76620aa2009-04-16 02:02:07 -07004395EXPORT_SYMBOL(napi_get_frags);
Herbert Xu96e93ea2009-01-06 10:49:34 -08004396
Eric Dumazeta50e2332014-03-29 21:28:21 -07004397static gro_result_t napi_frags_finish(struct napi_struct *napi,
4398 struct sk_buff *skb,
4399 gro_result_t ret)
Herbert Xu5d0d9be2009-01-29 14:19:48 +00004400{
Herbert Xu5d0d9be2009-01-29 14:19:48 +00004401 switch (ret) {
4402 case GRO_NORMAL:
Eric Dumazeta50e2332014-03-29 21:28:21 -07004403 case GRO_HELD:
4404 __skb_push(skb, ETH_HLEN);
4405 skb->protocol = eth_type_trans(skb, skb->dev);
4406 if (ret == GRO_NORMAL && netif_receive_skb_internal(skb))
Ben Hutchingsc7c4b3b2009-10-29 21:36:53 -07004407 ret = GRO_DROP;
Herbert Xu86911732009-01-29 14:19:50 +00004408 break;
Herbert Xu5d0d9be2009-01-29 14:19:48 +00004409
4410 case GRO_DROP:
Herbert Xu5d0d9be2009-01-29 14:19:48 +00004411 case GRO_MERGED_FREE:
4412 napi_reuse_skb(napi, skb);
4413 break;
Ben Hutchings5b252f02009-10-29 07:17:09 +00004414
4415 case GRO_MERGED:
4416 break;
Herbert Xu5d0d9be2009-01-29 14:19:48 +00004417 }
4418
Ben Hutchingsc7c4b3b2009-10-29 21:36:53 -07004419 return ret;
Herbert Xu5d0d9be2009-01-29 14:19:48 +00004420}
Herbert Xu5d0d9be2009-01-29 14:19:48 +00004421
Eric Dumazeta50e2332014-03-29 21:28:21 -07004422/* Upper GRO stack assumes network header starts at gro_offset=0
4423 * Drivers could call both napi_gro_frags() and napi_gro_receive()
4424 * We copy ethernet header into skb->data to have a common layout.
4425 */
Eric Dumazet4adb9c42012-05-18 20:49:06 +00004426static struct sk_buff *napi_frags_skb(struct napi_struct *napi)
Herbert Xu96e93ea2009-01-06 10:49:34 -08004427{
Herbert Xu76620aa2009-04-16 02:02:07 -07004428 struct sk_buff *skb = napi->skb;
Eric Dumazeta50e2332014-03-29 21:28:21 -07004429 const struct ethhdr *eth;
4430 unsigned int hlen = sizeof(*eth);
Herbert Xu76620aa2009-04-16 02:02:07 -07004431
4432 napi->skb = NULL;
4433
Eric Dumazeta50e2332014-03-29 21:28:21 -07004434 skb_reset_mac_header(skb);
4435 skb_gro_reset_offset(skb);
4436
4437 eth = skb_gro_header_fast(skb, 0);
4438 if (unlikely(skb_gro_header_hard(skb, hlen))) {
4439 eth = skb_gro_header_slow(skb, hlen, 0);
4440 if (unlikely(!eth)) {
4441 napi_reuse_skb(napi, skb);
4442 return NULL;
4443 }
4444 } else {
4445 gro_pull_from_frag0(skb, hlen);
4446 NAPI_GRO_CB(skb)->frag0 += hlen;
4447 NAPI_GRO_CB(skb)->frag0_len -= hlen;
Herbert Xu76620aa2009-04-16 02:02:07 -07004448 }
Eric Dumazeta50e2332014-03-29 21:28:21 -07004449 __skb_pull(skb, hlen);
4450
4451 /*
4452 * This works because the only protocols we care about don't require
4453 * special handling.
4454 * We'll fix it up properly in napi_frags_finish()
4455 */
4456 skb->protocol = eth->h_proto;
Herbert Xu76620aa2009-04-16 02:02:07 -07004457
Herbert Xu76620aa2009-04-16 02:02:07 -07004458 return skb;
4459}
Herbert Xu76620aa2009-04-16 02:02:07 -07004460
Ben Hutchingsc7c4b3b2009-10-29 21:36:53 -07004461gro_result_t napi_gro_frags(struct napi_struct *napi)
Herbert Xu76620aa2009-04-16 02:02:07 -07004462{
4463 struct sk_buff *skb = napi_frags_skb(napi);
Herbert Xu96e93ea2009-01-06 10:49:34 -08004464
4465 if (!skb)
Ben Hutchingsc7c4b3b2009-10-29 21:36:53 -07004466 return GRO_DROP;
Herbert Xu96e93ea2009-01-06 10:49:34 -08004467
Ben Hutchingsae78dbf2014-01-10 22:17:24 +00004468 trace_napi_gro_frags_entry(skb);
4469
Eric Dumazet89c5fa32012-12-10 13:28:16 +00004470 return napi_frags_finish(napi, skb, dev_gro_receive(napi, skb));
Herbert Xu5d38a072009-01-04 16:13:40 -08004471}
4472EXPORT_SYMBOL(napi_gro_frags);
4473
Tom Herbert573e8fc2014-08-22 13:33:47 -07004474/* Compute the checksum from gro_offset and return the folded value
4475 * after adding in any pseudo checksum.
4476 */
4477__sum16 __skb_gro_checksum_complete(struct sk_buff *skb)
4478{
4479 __wsum wsum;
4480 __sum16 sum;
4481
4482 wsum = skb_checksum(skb, skb_gro_offset(skb), skb_gro_len(skb), 0);
4483
4484 /* NAPI_GRO_CB(skb)->csum holds pseudo checksum */
4485 sum = csum_fold(csum_add(NAPI_GRO_CB(skb)->csum, wsum));
4486 if (likely(!sum)) {
4487 if (unlikely(skb->ip_summed == CHECKSUM_COMPLETE) &&
4488 !skb->csum_complete_sw)
4489 netdev_rx_csum_fault(skb->dev);
4490 }
4491
4492 NAPI_GRO_CB(skb)->csum = wsum;
4493 NAPI_GRO_CB(skb)->csum_valid = 1;
4494
4495 return sum;
4496}
4497EXPORT_SYMBOL(__skb_gro_checksum_complete);
4498
Eric Dumazete326bed2010-04-22 00:22:45 -07004499/*
Zhi Yong Wu855abcf2014-01-01 04:34:50 +08004500 * net_rps_action_and_irq_enable sends any pending IPI's for rps.
Eric Dumazete326bed2010-04-22 00:22:45 -07004501 * Note: called with local irq disabled, but exits with local irq enabled.
4502 */
4503static void net_rps_action_and_irq_enable(struct softnet_data *sd)
4504{
4505#ifdef CONFIG_RPS
4506 struct softnet_data *remsd = sd->rps_ipi_list;
4507
4508 if (remsd) {
4509 sd->rps_ipi_list = NULL;
4510
4511 local_irq_enable();
4512
4513 /* Send pending IPI's to kick RPS processing on remote cpus. */
4514 while (remsd) {
4515 struct softnet_data *next = remsd->rps_ipi_next;
4516
4517 if (cpu_online(remsd->cpu))
Frederic Weisbeckerc46fff22014-02-24 16:40:02 +01004518 smp_call_function_single_async(remsd->cpu,
Frederic Weisbeckerfce8ad12014-02-24 16:40:01 +01004519 &remsd->csd);
Eric Dumazete326bed2010-04-22 00:22:45 -07004520 remsd = next;
4521 }
4522 } else
4523#endif
4524 local_irq_enable();
4525}
4526
Eric Dumazetd75b1ad2014-11-02 06:19:33 -08004527static bool sd_has_rps_ipi_waiting(struct softnet_data *sd)
4528{
4529#ifdef CONFIG_RPS
4530 return sd->rps_ipi_list != NULL;
4531#else
4532 return false;
4533#endif
4534}
4535
Stephen Hemmingerbea33482007-10-03 16:41:36 -07004536static int process_backlog(struct napi_struct *napi, int quota)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004537{
4538 int work = 0;
Eric Dumazeteecfd7c2010-05-06 22:07:48 -07004539 struct softnet_data *sd = container_of(napi, struct softnet_data, backlog);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004540
Eric Dumazete326bed2010-04-22 00:22:45 -07004541 /* Check if we have pending ipi, its better to send them now,
4542 * not waiting net_rx_action() end.
4543 */
Eric Dumazetd75b1ad2014-11-02 06:19:33 -08004544 if (sd_has_rps_ipi_waiting(sd)) {
Eric Dumazete326bed2010-04-22 00:22:45 -07004545 local_irq_disable();
4546 net_rps_action_and_irq_enable(sd);
4547 }
Eric Dumazetd75b1ad2014-11-02 06:19:33 -08004548
Stephen Hemmingerbea33482007-10-03 16:41:36 -07004549 napi->weight = weight_p;
Changli Gao6e7676c2010-04-27 15:07:33 -07004550 local_irq_disable();
Tom Herbert11ef7a82014-06-30 09:50:40 -07004551 while (1) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07004552 struct sk_buff *skb;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004553
Changli Gao6e7676c2010-04-27 15:07:33 -07004554 while ((skb = __skb_dequeue(&sd->process_queue))) {
Julian Anastasov2c17d272015-07-09 09:59:10 +03004555 rcu_read_lock();
Eric Dumazete4008272010-04-05 15:42:39 -07004556 local_irq_enable();
Changli Gao6e7676c2010-04-27 15:07:33 -07004557 __netif_receive_skb(skb);
Julian Anastasov2c17d272015-07-09 09:59:10 +03004558 rcu_read_unlock();
Changli Gao6e7676c2010-04-27 15:07:33 -07004559 local_irq_disable();
Tom Herbert76cc8b12010-05-20 18:37:59 +00004560 input_queue_head_incr(sd);
4561 if (++work >= quota) {
4562 local_irq_enable();
4563 return work;
4564 }
Stephen Hemmingerbea33482007-10-03 16:41:36 -07004565 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07004566
Changli Gao6e7676c2010-04-27 15:07:33 -07004567 rps_lock(sd);
Tom Herbert11ef7a82014-06-30 09:50:40 -07004568 if (skb_queue_empty(&sd->input_pkt_queue)) {
Eric Dumazeteecfd7c2010-05-06 22:07:48 -07004569 /*
4570 * Inline a custom version of __napi_complete().
4571 * only current cpu owns and manipulates this napi,
Tom Herbert11ef7a82014-06-30 09:50:40 -07004572 * and NAPI_STATE_SCHED is the only possible flag set
4573 * on backlog.
4574 * We can use a plain write instead of clear_bit(),
Eric Dumazeteecfd7c2010-05-06 22:07:48 -07004575 * and we dont need an smp_mb() memory barrier.
4576 */
Eric Dumazeteecfd7c2010-05-06 22:07:48 -07004577 napi->state = 0;
Tom Herbert11ef7a82014-06-30 09:50:40 -07004578 rps_unlock(sd);
Eric Dumazeteecfd7c2010-05-06 22:07:48 -07004579
Tom Herbert11ef7a82014-06-30 09:50:40 -07004580 break;
Changli Gao6e7676c2010-04-27 15:07:33 -07004581 }
Tom Herbert11ef7a82014-06-30 09:50:40 -07004582
4583 skb_queue_splice_tail_init(&sd->input_pkt_queue,
4584 &sd->process_queue);
Changli Gao6e7676c2010-04-27 15:07:33 -07004585 rps_unlock(sd);
4586 }
4587 local_irq_enable();
Linus Torvalds1da177e2005-04-16 15:20:36 -07004588
Stephen Hemmingerbea33482007-10-03 16:41:36 -07004589 return work;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004590}
4591
Stephen Hemmingerbea33482007-10-03 16:41:36 -07004592/**
4593 * __napi_schedule - schedule for receive
Randy Dunlapc4ea43c2007-10-12 21:17:49 -07004594 * @n: entry to schedule
Stephen Hemmingerbea33482007-10-03 16:41:36 -07004595 *
Eric Dumazetbc9ad162014-10-28 18:05:13 -07004596 * The entry's receive function will be scheduled to run.
4597 * Consider using __napi_schedule_irqoff() if hard irqs are masked.
Stephen Hemmingerbea33482007-10-03 16:41:36 -07004598 */
Harvey Harrisonb5606c22008-02-13 15:03:16 -08004599void __napi_schedule(struct napi_struct *n)
Stephen Hemmingerbea33482007-10-03 16:41:36 -07004600{
4601 unsigned long flags;
4602
4603 local_irq_save(flags);
Christoph Lameter903ceff2014-08-17 12:30:35 -05004604 ____napi_schedule(this_cpu_ptr(&softnet_data), n);
Stephen Hemmingerbea33482007-10-03 16:41:36 -07004605 local_irq_restore(flags);
4606}
4607EXPORT_SYMBOL(__napi_schedule);
4608
Eric Dumazetbc9ad162014-10-28 18:05:13 -07004609/**
4610 * __napi_schedule_irqoff - schedule for receive
4611 * @n: entry to schedule
4612 *
4613 * Variant of __napi_schedule() assuming hard irqs are masked
4614 */
4615void __napi_schedule_irqoff(struct napi_struct *n)
4616{
4617 ____napi_schedule(this_cpu_ptr(&softnet_data), n);
4618}
4619EXPORT_SYMBOL(__napi_schedule_irqoff);
4620
Herbert Xud565b0a2008-12-15 23:38:52 -08004621void __napi_complete(struct napi_struct *n)
4622{
4623 BUG_ON(!test_bit(NAPI_STATE_SCHED, &n->state));
Herbert Xud565b0a2008-12-15 23:38:52 -08004624
Eric Dumazetd75b1ad2014-11-02 06:19:33 -08004625 list_del_init(&n->poll_list);
Peter Zijlstra4e857c52014-03-17 18:06:10 +01004626 smp_mb__before_atomic();
Herbert Xud565b0a2008-12-15 23:38:52 -08004627 clear_bit(NAPI_STATE_SCHED, &n->state);
4628}
4629EXPORT_SYMBOL(__napi_complete);
4630
Eric Dumazet3b47d302014-11-06 21:09:44 -08004631void napi_complete_done(struct napi_struct *n, int work_done)
Herbert Xud565b0a2008-12-15 23:38:52 -08004632{
4633 unsigned long flags;
4634
4635 /*
4636 * don't let napi dequeue from the cpu poll list
4637 * just in case its running on a different cpu
4638 */
4639 if (unlikely(test_bit(NAPI_STATE_NPSVC, &n->state)))
4640 return;
4641
Eric Dumazet3b47d302014-11-06 21:09:44 -08004642 if (n->gro_list) {
4643 unsigned long timeout = 0;
Eric Dumazetd75b1ad2014-11-02 06:19:33 -08004644
Eric Dumazet3b47d302014-11-06 21:09:44 -08004645 if (work_done)
4646 timeout = n->dev->gro_flush_timeout;
4647
4648 if (timeout)
4649 hrtimer_start(&n->timer, ns_to_ktime(timeout),
4650 HRTIMER_MODE_REL_PINNED);
4651 else
4652 napi_gro_flush(n, false);
4653 }
Eric Dumazetd75b1ad2014-11-02 06:19:33 -08004654 if (likely(list_empty(&n->poll_list))) {
4655 WARN_ON_ONCE(!test_and_clear_bit(NAPI_STATE_SCHED, &n->state));
4656 } else {
4657 /* If n->poll_list is not empty, we need to mask irqs */
4658 local_irq_save(flags);
4659 __napi_complete(n);
4660 local_irq_restore(flags);
4661 }
Herbert Xud565b0a2008-12-15 23:38:52 -08004662}
Eric Dumazet3b47d302014-11-06 21:09:44 -08004663EXPORT_SYMBOL(napi_complete_done);
Herbert Xud565b0a2008-12-15 23:38:52 -08004664
Eliezer Tamiraf12fa62013-06-10 11:39:41 +03004665/* must be called under rcu_read_lock(), as we dont take a reference */
4666struct napi_struct *napi_by_id(unsigned int napi_id)
4667{
4668 unsigned int hash = napi_id % HASH_SIZE(napi_hash);
4669 struct napi_struct *napi;
4670
4671 hlist_for_each_entry_rcu(napi, &napi_hash[hash], napi_hash_node)
4672 if (napi->napi_id == napi_id)
4673 return napi;
4674
4675 return NULL;
4676}
4677EXPORT_SYMBOL_GPL(napi_by_id);
4678
4679void napi_hash_add(struct napi_struct *napi)
4680{
Eric Dumazet52bd2d62015-11-18 06:30:50 -08004681 if (test_and_set_bit(NAPI_STATE_HASHED, &napi->state))
4682 return;
Eliezer Tamiraf12fa62013-06-10 11:39:41 +03004683
Eric Dumazet52bd2d62015-11-18 06:30:50 -08004684 spin_lock(&napi_hash_lock);
Eliezer Tamiraf12fa62013-06-10 11:39:41 +03004685
Eric Dumazet52bd2d62015-11-18 06:30:50 -08004686 /* 0..NR_CPUS+1 range is reserved for sender_cpu use */
4687 do {
4688 if (unlikely(++napi_gen_id < NR_CPUS + 1))
4689 napi_gen_id = NR_CPUS + 1;
4690 } while (napi_by_id(napi_gen_id));
4691 napi->napi_id = napi_gen_id;
Eliezer Tamiraf12fa62013-06-10 11:39:41 +03004692
Eric Dumazet52bd2d62015-11-18 06:30:50 -08004693 hlist_add_head_rcu(&napi->napi_hash_node,
4694 &napi_hash[napi->napi_id % HASH_SIZE(napi_hash)]);
Eliezer Tamiraf12fa62013-06-10 11:39:41 +03004695
Eric Dumazet52bd2d62015-11-18 06:30:50 -08004696 spin_unlock(&napi_hash_lock);
Eliezer Tamiraf12fa62013-06-10 11:39:41 +03004697}
4698EXPORT_SYMBOL_GPL(napi_hash_add);
4699
4700/* Warning : caller is responsible to make sure rcu grace period
4701 * is respected before freeing memory containing @napi
4702 */
4703void napi_hash_del(struct napi_struct *napi)
4704{
4705 spin_lock(&napi_hash_lock);
4706
4707 if (test_and_clear_bit(NAPI_STATE_HASHED, &napi->state))
4708 hlist_del_rcu(&napi->napi_hash_node);
4709
4710 spin_unlock(&napi_hash_lock);
4711}
4712EXPORT_SYMBOL_GPL(napi_hash_del);
4713
Eric Dumazet3b47d302014-11-06 21:09:44 -08004714static enum hrtimer_restart napi_watchdog(struct hrtimer *timer)
4715{
4716 struct napi_struct *napi;
4717
4718 napi = container_of(timer, struct napi_struct, timer);
4719 if (napi->gro_list)
4720 napi_schedule(napi);
4721
4722 return HRTIMER_NORESTART;
4723}
4724
Herbert Xud565b0a2008-12-15 23:38:52 -08004725void netif_napi_add(struct net_device *dev, struct napi_struct *napi,
4726 int (*poll)(struct napi_struct *, int), int weight)
4727{
4728 INIT_LIST_HEAD(&napi->poll_list);
Eric Dumazet3b47d302014-11-06 21:09:44 -08004729 hrtimer_init(&napi->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL_PINNED);
4730 napi->timer.function = napi_watchdog;
Herbert Xu4ae55442009-02-08 18:00:36 +00004731 napi->gro_count = 0;
Herbert Xud565b0a2008-12-15 23:38:52 -08004732 napi->gro_list = NULL;
Herbert Xu5d38a072009-01-04 16:13:40 -08004733 napi->skb = NULL;
Herbert Xud565b0a2008-12-15 23:38:52 -08004734 napi->poll = poll;
Eric Dumazet82dc3c62013-03-05 15:57:22 +00004735 if (weight > NAPI_POLL_WEIGHT)
4736 pr_err_once("netif_napi_add() called with weight %d on device %s\n",
4737 weight, dev->name);
Herbert Xud565b0a2008-12-15 23:38:52 -08004738 napi->weight = weight;
4739 list_add(&napi->dev_list, &dev->napi_list);
Herbert Xud565b0a2008-12-15 23:38:52 -08004740 napi->dev = dev;
Herbert Xu5d38a072009-01-04 16:13:40 -08004741#ifdef CONFIG_NETPOLL
Herbert Xud565b0a2008-12-15 23:38:52 -08004742 spin_lock_init(&napi->poll_lock);
4743 napi->poll_owner = -1;
4744#endif
4745 set_bit(NAPI_STATE_SCHED, &napi->state);
4746}
4747EXPORT_SYMBOL(netif_napi_add);
4748
Eric Dumazet3b47d302014-11-06 21:09:44 -08004749void napi_disable(struct napi_struct *n)
4750{
4751 might_sleep();
4752 set_bit(NAPI_STATE_DISABLE, &n->state);
4753
4754 while (test_and_set_bit(NAPI_STATE_SCHED, &n->state))
4755 msleep(1);
Neil Horman2d8bff1262015-09-23 14:57:58 -04004756 while (test_and_set_bit(NAPI_STATE_NPSVC, &n->state))
4757 msleep(1);
Eric Dumazet3b47d302014-11-06 21:09:44 -08004758
4759 hrtimer_cancel(&n->timer);
4760
4761 clear_bit(NAPI_STATE_DISABLE, &n->state);
4762}
4763EXPORT_SYMBOL(napi_disable);
4764
Herbert Xud565b0a2008-12-15 23:38:52 -08004765void netif_napi_del(struct napi_struct *napi)
4766{
Peter P Waskiewicz Jrd7b06632008-12-26 01:35:35 -08004767 list_del_init(&napi->dev_list);
Herbert Xu76620aa2009-04-16 02:02:07 -07004768 napi_free_frags(napi);
Herbert Xud565b0a2008-12-15 23:38:52 -08004769
Eric Dumazet289dccb2013-12-20 14:29:08 -08004770 kfree_skb_list(napi->gro_list);
Herbert Xud565b0a2008-12-15 23:38:52 -08004771 napi->gro_list = NULL;
Herbert Xu4ae55442009-02-08 18:00:36 +00004772 napi->gro_count = 0;
Herbert Xud565b0a2008-12-15 23:38:52 -08004773}
4774EXPORT_SYMBOL(netif_napi_del);
4775
Herbert Xu726ce702014-12-21 07:16:21 +11004776static int napi_poll(struct napi_struct *n, struct list_head *repoll)
4777{
4778 void *have;
4779 int work, weight;
4780
4781 list_del_init(&n->poll_list);
4782
4783 have = netpoll_poll_lock(n);
4784
4785 weight = n->weight;
4786
4787 /* This NAPI_STATE_SCHED test is for avoiding a race
4788 * with netpoll's poll_napi(). Only the entity which
4789 * obtains the lock and sees NAPI_STATE_SCHED set will
4790 * actually make the ->poll() call. Therefore we avoid
4791 * accidentally calling ->poll() when NAPI is not scheduled.
4792 */
4793 work = 0;
4794 if (test_bit(NAPI_STATE_SCHED, &n->state)) {
4795 work = n->poll(n, weight);
4796 trace_napi_poll(n);
4797 }
4798
4799 WARN_ON_ONCE(work > weight);
4800
4801 if (likely(work < weight))
4802 goto out_unlock;
4803
4804 /* Drivers must not modify the NAPI state if they
4805 * consume the entire weight. In such cases this code
4806 * still "owns" the NAPI instance and therefore can
4807 * move the instance around on the list at-will.
4808 */
4809 if (unlikely(napi_disable_pending(n))) {
4810 napi_complete(n);
4811 goto out_unlock;
4812 }
4813
4814 if (n->gro_list) {
4815 /* flush too old packets
4816 * If HZ < 1000, flush all packets.
4817 */
4818 napi_gro_flush(n, HZ >= 1000);
4819 }
4820
Herbert Xu001ce542014-12-21 07:16:22 +11004821 /* Some drivers may have called napi_schedule
4822 * prior to exhausting their budget.
4823 */
4824 if (unlikely(!list_empty(&n->poll_list))) {
4825 pr_warn_once("%s: Budget exhausted after napi rescheduled\n",
4826 n->dev ? n->dev->name : "backlog");
4827 goto out_unlock;
4828 }
4829
Herbert Xu726ce702014-12-21 07:16:21 +11004830 list_add_tail(&n->poll_list, repoll);
4831
4832out_unlock:
4833 netpoll_poll_unlock(have);
4834
4835 return work;
4836}
4837
Linus Torvalds1da177e2005-04-16 15:20:36 -07004838static void net_rx_action(struct softirq_action *h)
4839{
Christoph Lameter903ceff2014-08-17 12:30:35 -05004840 struct softnet_data *sd = this_cpu_ptr(&softnet_data);
Stephen Hemminger24f8b232008-11-03 17:14:38 -08004841 unsigned long time_limit = jiffies + 2;
Stephen Hemminger51b0bde2005-06-23 20:14:40 -07004842 int budget = netdev_budget;
Eric Dumazetd75b1ad2014-11-02 06:19:33 -08004843 LIST_HEAD(list);
4844 LIST_HEAD(repoll);
Matt Mackall53fb95d2005-08-11 19:27:43 -07004845
Linus Torvalds1da177e2005-04-16 15:20:36 -07004846 local_irq_disable();
Eric Dumazetd75b1ad2014-11-02 06:19:33 -08004847 list_splice_init(&sd->poll_list, &list);
4848 local_irq_enable();
Linus Torvalds1da177e2005-04-16 15:20:36 -07004849
Herbert Xuceb8d5b2014-12-21 07:16:25 +11004850 for (;;) {
Stephen Hemmingerbea33482007-10-03 16:41:36 -07004851 struct napi_struct *n;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004852
Herbert Xuceb8d5b2014-12-21 07:16:25 +11004853 if (list_empty(&list)) {
4854 if (!sd_has_rps_ipi_waiting(sd) && list_empty(&repoll))
4855 return;
4856 break;
4857 }
4858
Herbert Xu6bd373e2014-12-21 07:16:24 +11004859 n = list_first_entry(&list, struct napi_struct, poll_list);
4860 budget -= napi_poll(n, &repoll);
4861
Eric Dumazetd75b1ad2014-11-02 06:19:33 -08004862 /* If softirq window is exhausted then punt.
Stephen Hemminger24f8b232008-11-03 17:14:38 -08004863 * Allow this to run for 2 jiffies since which will allow
4864 * an average latency of 1.5/HZ.
Stephen Hemmingerbea33482007-10-03 16:41:36 -07004865 */
Herbert Xuceb8d5b2014-12-21 07:16:25 +11004866 if (unlikely(budget <= 0 ||
4867 time_after_eq(jiffies, time_limit))) {
4868 sd->time_squeeze++;
4869 break;
4870 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07004871 }
Eric Dumazetd75b1ad2014-11-02 06:19:33 -08004872
Eric Dumazetd75b1ad2014-11-02 06:19:33 -08004873 local_irq_disable();
4874
4875 list_splice_tail_init(&sd->poll_list, &list);
4876 list_splice_tail(&repoll, &list);
4877 list_splice(&list, &sd->poll_list);
4878 if (!list_empty(&sd->poll_list))
4879 __raise_softirq_irqoff(NET_RX_SOFTIRQ);
4880
Eric Dumazete326bed2010-04-22 00:22:45 -07004881 net_rps_action_and_irq_enable(sd);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004882}
4883
Veaceslav Falicoaa9d8562013-08-28 23:25:04 +02004884struct netdev_adjacent {
Jiri Pirko9ff162a2013-01-03 22:48:49 +00004885 struct net_device *dev;
Veaceslav Falico5d261912013-08-28 23:25:05 +02004886
4887 /* upper master flag, there can only be one master device per list */
Jiri Pirko9ff162a2013-01-03 22:48:49 +00004888 bool master;
Veaceslav Falico5d261912013-08-28 23:25:05 +02004889
Veaceslav Falico5d261912013-08-28 23:25:05 +02004890 /* counter for the number of times this device was added to us */
4891 u16 ref_nr;
4892
Veaceslav Falico402dae92013-09-25 09:20:09 +02004893 /* private field for the users */
4894 void *private;
4895
Jiri Pirko9ff162a2013-01-03 22:48:49 +00004896 struct list_head list;
4897 struct rcu_head rcu;
Jiri Pirko9ff162a2013-01-03 22:48:49 +00004898};
4899
Michal Kubeček6ea29da2015-09-24 10:59:05 +02004900static struct netdev_adjacent *__netdev_find_adj(struct net_device *adj_dev,
Veaceslav Falico2f268f12013-09-25 09:20:07 +02004901 struct list_head *adj_list)
Jiri Pirko9ff162a2013-01-03 22:48:49 +00004902{
Veaceslav Falico5d261912013-08-28 23:25:05 +02004903 struct netdev_adjacent *adj;
Veaceslav Falico5d261912013-08-28 23:25:05 +02004904
Veaceslav Falico2f268f12013-09-25 09:20:07 +02004905 list_for_each_entry(adj, adj_list, list) {
Veaceslav Falico5d261912013-08-28 23:25:05 +02004906 if (adj->dev == adj_dev)
4907 return adj;
Jiri Pirko9ff162a2013-01-03 22:48:49 +00004908 }
4909 return NULL;
4910}
4911
4912/**
4913 * netdev_has_upper_dev - Check if device is linked to an upper device
4914 * @dev: device
4915 * @upper_dev: upper device to check
4916 *
4917 * Find out if a device is linked to specified upper device and return true
4918 * in case it is. Note that this checks only immediate upper device,
4919 * not through a complete stack of devices. The caller must hold the RTNL lock.
4920 */
4921bool netdev_has_upper_dev(struct net_device *dev,
4922 struct net_device *upper_dev)
4923{
4924 ASSERT_RTNL();
4925
Michal Kubeček6ea29da2015-09-24 10:59:05 +02004926 return __netdev_find_adj(upper_dev, &dev->all_adj_list.upper);
Jiri Pirko9ff162a2013-01-03 22:48:49 +00004927}
4928EXPORT_SYMBOL(netdev_has_upper_dev);
4929
4930/**
4931 * netdev_has_any_upper_dev - Check if device is linked to some device
4932 * @dev: device
4933 *
4934 * Find out if a device is linked to an upper device and return true in case
4935 * it is. The caller must hold the RTNL lock.
4936 */
stephen hemminger1d143d92013-12-29 14:01:29 -08004937static bool netdev_has_any_upper_dev(struct net_device *dev)
Jiri Pirko9ff162a2013-01-03 22:48:49 +00004938{
4939 ASSERT_RTNL();
4940
Veaceslav Falico2f268f12013-09-25 09:20:07 +02004941 return !list_empty(&dev->all_adj_list.upper);
Jiri Pirko9ff162a2013-01-03 22:48:49 +00004942}
Jiri Pirko9ff162a2013-01-03 22:48:49 +00004943
4944/**
4945 * netdev_master_upper_dev_get - Get master upper device
4946 * @dev: device
4947 *
4948 * Find a master upper device and return pointer to it or NULL in case
4949 * it's not there. The caller must hold the RTNL lock.
4950 */
4951struct net_device *netdev_master_upper_dev_get(struct net_device *dev)
4952{
Veaceslav Falicoaa9d8562013-08-28 23:25:04 +02004953 struct netdev_adjacent *upper;
Jiri Pirko9ff162a2013-01-03 22:48:49 +00004954
4955 ASSERT_RTNL();
4956
Veaceslav Falico2f268f12013-09-25 09:20:07 +02004957 if (list_empty(&dev->adj_list.upper))
Jiri Pirko9ff162a2013-01-03 22:48:49 +00004958 return NULL;
4959
Veaceslav Falico2f268f12013-09-25 09:20:07 +02004960 upper = list_first_entry(&dev->adj_list.upper,
Veaceslav Falicoaa9d8562013-08-28 23:25:04 +02004961 struct netdev_adjacent, list);
Jiri Pirko9ff162a2013-01-03 22:48:49 +00004962 if (likely(upper->master))
4963 return upper->dev;
4964 return NULL;
4965}
4966EXPORT_SYMBOL(netdev_master_upper_dev_get);
4967
Veaceslav Falicob6ccba42013-09-25 09:20:23 +02004968void *netdev_adjacent_get_private(struct list_head *adj_list)
4969{
4970 struct netdev_adjacent *adj;
4971
4972 adj = list_entry(adj_list, struct netdev_adjacent, list);
4973
4974 return adj->private;
4975}
4976EXPORT_SYMBOL(netdev_adjacent_get_private);
4977
Veaceslav Falico31088a12013-09-25 09:20:12 +02004978/**
Vlad Yasevich44a40852014-05-16 17:20:38 -04004979 * netdev_upper_get_next_dev_rcu - Get the next dev from upper list
4980 * @dev: device
4981 * @iter: list_head ** of the current position
4982 *
4983 * Gets the next device from the dev's upper list, starting from iter
4984 * position. The caller must hold RCU read lock.
4985 */
4986struct net_device *netdev_upper_get_next_dev_rcu(struct net_device *dev,
4987 struct list_head **iter)
4988{
4989 struct netdev_adjacent *upper;
4990
4991 WARN_ON_ONCE(!rcu_read_lock_held() && !lockdep_rtnl_is_held());
4992
4993 upper = list_entry_rcu((*iter)->next, struct netdev_adjacent, list);
4994
4995 if (&upper->list == &dev->adj_list.upper)
4996 return NULL;
4997
4998 *iter = &upper->list;
4999
5000 return upper->dev;
5001}
5002EXPORT_SYMBOL(netdev_upper_get_next_dev_rcu);
5003
5004/**
Veaceslav Falico31088a12013-09-25 09:20:12 +02005005 * netdev_all_upper_get_next_dev_rcu - Get the next dev from upper list
Veaceslav Falico48311f42013-08-28 23:25:07 +02005006 * @dev: device
5007 * @iter: list_head ** of the current position
5008 *
5009 * Gets the next device from the dev's upper list, starting from iter
5010 * position. The caller must hold RCU read lock.
5011 */
Veaceslav Falico2f268f12013-09-25 09:20:07 +02005012struct net_device *netdev_all_upper_get_next_dev_rcu(struct net_device *dev,
5013 struct list_head **iter)
Veaceslav Falico48311f42013-08-28 23:25:07 +02005014{
5015 struct netdev_adjacent *upper;
5016
John Fastabend85328242013-11-26 06:33:52 +00005017 WARN_ON_ONCE(!rcu_read_lock_held() && !lockdep_rtnl_is_held());
Veaceslav Falico48311f42013-08-28 23:25:07 +02005018
5019 upper = list_entry_rcu((*iter)->next, struct netdev_adjacent, list);
5020
Veaceslav Falico2f268f12013-09-25 09:20:07 +02005021 if (&upper->list == &dev->all_adj_list.upper)
Veaceslav Falico48311f42013-08-28 23:25:07 +02005022 return NULL;
5023
5024 *iter = &upper->list;
5025
5026 return upper->dev;
5027}
Veaceslav Falico2f268f12013-09-25 09:20:07 +02005028EXPORT_SYMBOL(netdev_all_upper_get_next_dev_rcu);
Veaceslav Falico48311f42013-08-28 23:25:07 +02005029
Jiri Pirko9ff162a2013-01-03 22:48:49 +00005030/**
Veaceslav Falico31088a12013-09-25 09:20:12 +02005031 * netdev_lower_get_next_private - Get the next ->private from the
5032 * lower neighbour list
5033 * @dev: device
5034 * @iter: list_head ** of the current position
5035 *
5036 * Gets the next netdev_adjacent->private from the dev's lower neighbour
5037 * list, starting from iter position. The caller must hold either hold the
5038 * RTNL lock or its own locking that guarantees that the neighbour lower
subashab@codeaurora.orgb4691392015-07-24 03:03:29 +00005039 * list will remain unchanged.
Veaceslav Falico31088a12013-09-25 09:20:12 +02005040 */
5041void *netdev_lower_get_next_private(struct net_device *dev,
5042 struct list_head **iter)
5043{
5044 struct netdev_adjacent *lower;
5045
5046 lower = list_entry(*iter, struct netdev_adjacent, list);
5047
5048 if (&lower->list == &dev->adj_list.lower)
5049 return NULL;
5050
Veaceslav Falico6859e7d2014-04-07 11:25:12 +02005051 *iter = lower->list.next;
Veaceslav Falico31088a12013-09-25 09:20:12 +02005052
5053 return lower->private;
5054}
5055EXPORT_SYMBOL(netdev_lower_get_next_private);
5056
5057/**
5058 * netdev_lower_get_next_private_rcu - Get the next ->private from the
5059 * lower neighbour list, RCU
5060 * variant
5061 * @dev: device
5062 * @iter: list_head ** of the current position
5063 *
5064 * Gets the next netdev_adjacent->private from the dev's lower neighbour
5065 * list, starting from iter position. The caller must hold RCU read lock.
5066 */
5067void *netdev_lower_get_next_private_rcu(struct net_device *dev,
5068 struct list_head **iter)
5069{
5070 struct netdev_adjacent *lower;
5071
5072 WARN_ON_ONCE(!rcu_read_lock_held());
5073
5074 lower = list_entry_rcu((*iter)->next, struct netdev_adjacent, list);
5075
5076 if (&lower->list == &dev->adj_list.lower)
5077 return NULL;
5078
Veaceslav Falico6859e7d2014-04-07 11:25:12 +02005079 *iter = &lower->list;
Veaceslav Falico31088a12013-09-25 09:20:12 +02005080
5081 return lower->private;
5082}
5083EXPORT_SYMBOL(netdev_lower_get_next_private_rcu);
5084
5085/**
Vlad Yasevich4085ebe2014-05-16 17:04:53 -04005086 * netdev_lower_get_next - Get the next device from the lower neighbour
5087 * list
5088 * @dev: device
5089 * @iter: list_head ** of the current position
5090 *
5091 * Gets the next netdev_adjacent from the dev's lower neighbour
5092 * list, starting from iter position. The caller must hold RTNL lock or
5093 * its own locking that guarantees that the neighbour lower
subashab@codeaurora.orgb4691392015-07-24 03:03:29 +00005094 * list will remain unchanged.
Vlad Yasevich4085ebe2014-05-16 17:04:53 -04005095 */
5096void *netdev_lower_get_next(struct net_device *dev, struct list_head **iter)
5097{
5098 struct netdev_adjacent *lower;
5099
5100 lower = list_entry((*iter)->next, struct netdev_adjacent, list);
5101
5102 if (&lower->list == &dev->adj_list.lower)
5103 return NULL;
5104
5105 *iter = &lower->list;
5106
5107 return lower->dev;
5108}
5109EXPORT_SYMBOL(netdev_lower_get_next);
5110
5111/**
dingtianhonge001bfa2013-12-13 10:19:55 +08005112 * netdev_lower_get_first_private_rcu - Get the first ->private from the
5113 * lower neighbour list, RCU
5114 * variant
5115 * @dev: device
5116 *
5117 * Gets the first netdev_adjacent->private from the dev's lower neighbour
5118 * list. The caller must hold RCU read lock.
5119 */
5120void *netdev_lower_get_first_private_rcu(struct net_device *dev)
5121{
5122 struct netdev_adjacent *lower;
5123
5124 lower = list_first_or_null_rcu(&dev->adj_list.lower,
5125 struct netdev_adjacent, list);
5126 if (lower)
5127 return lower->private;
5128 return NULL;
5129}
5130EXPORT_SYMBOL(netdev_lower_get_first_private_rcu);
5131
5132/**
Jiri Pirko9ff162a2013-01-03 22:48:49 +00005133 * netdev_master_upper_dev_get_rcu - Get master upper device
5134 * @dev: device
5135 *
5136 * Find a master upper device and return pointer to it or NULL in case
5137 * it's not there. The caller must hold the RCU read lock.
5138 */
5139struct net_device *netdev_master_upper_dev_get_rcu(struct net_device *dev)
5140{
Veaceslav Falicoaa9d8562013-08-28 23:25:04 +02005141 struct netdev_adjacent *upper;
Jiri Pirko9ff162a2013-01-03 22:48:49 +00005142
Veaceslav Falico2f268f12013-09-25 09:20:07 +02005143 upper = list_first_or_null_rcu(&dev->adj_list.upper,
Veaceslav Falicoaa9d8562013-08-28 23:25:04 +02005144 struct netdev_adjacent, list);
Jiri Pirko9ff162a2013-01-03 22:48:49 +00005145 if (upper && likely(upper->master))
5146 return upper->dev;
5147 return NULL;
5148}
5149EXPORT_SYMBOL(netdev_master_upper_dev_get_rcu);
5150
Rashika Kheria0a59f3a2014-02-09 20:26:25 +05305151static int netdev_adjacent_sysfs_add(struct net_device *dev,
Veaceslav Falico3ee32702014-01-14 21:58:50 +01005152 struct net_device *adj_dev,
5153 struct list_head *dev_list)
5154{
5155 char linkname[IFNAMSIZ+7];
5156 sprintf(linkname, dev_list == &dev->adj_list.upper ?
5157 "upper_%s" : "lower_%s", adj_dev->name);
5158 return sysfs_create_link(&(dev->dev.kobj), &(adj_dev->dev.kobj),
5159 linkname);
5160}
Rashika Kheria0a59f3a2014-02-09 20:26:25 +05305161static void netdev_adjacent_sysfs_del(struct net_device *dev,
Veaceslav Falico3ee32702014-01-14 21:58:50 +01005162 char *name,
5163 struct list_head *dev_list)
5164{
5165 char linkname[IFNAMSIZ+7];
5166 sprintf(linkname, dev_list == &dev->adj_list.upper ?
5167 "upper_%s" : "lower_%s", name);
5168 sysfs_remove_link(&(dev->dev.kobj), linkname);
5169}
5170
Alexander Y. Fomichev7ce64c72014-09-15 14:22:35 +04005171static inline bool netdev_adjacent_is_neigh_list(struct net_device *dev,
5172 struct net_device *adj_dev,
5173 struct list_head *dev_list)
5174{
5175 return (dev_list == &dev->adj_list.upper ||
5176 dev_list == &dev->adj_list.lower) &&
5177 net_eq(dev_net(dev), dev_net(adj_dev));
5178}
Veaceslav Falico3ee32702014-01-14 21:58:50 +01005179
Veaceslav Falico5d261912013-08-28 23:25:05 +02005180static int __netdev_adjacent_dev_insert(struct net_device *dev,
5181 struct net_device *adj_dev,
Veaceslav Falico7863c052013-09-25 09:20:06 +02005182 struct list_head *dev_list,
Veaceslav Falico402dae92013-09-25 09:20:09 +02005183 void *private, bool master)
Veaceslav Falico5d261912013-08-28 23:25:05 +02005184{
5185 struct netdev_adjacent *adj;
Veaceslav Falico842d67a2013-09-25 09:20:31 +02005186 int ret;
Veaceslav Falico5d261912013-08-28 23:25:05 +02005187
Michal Kubeček6ea29da2015-09-24 10:59:05 +02005188 adj = __netdev_find_adj(adj_dev, dev_list);
Veaceslav Falico5d261912013-08-28 23:25:05 +02005189
5190 if (adj) {
Veaceslav Falico5d261912013-08-28 23:25:05 +02005191 adj->ref_nr++;
5192 return 0;
5193 }
5194
5195 adj = kmalloc(sizeof(*adj), GFP_KERNEL);
5196 if (!adj)
5197 return -ENOMEM;
5198
5199 adj->dev = adj_dev;
5200 adj->master = master;
Veaceslav Falico5d261912013-08-28 23:25:05 +02005201 adj->ref_nr = 1;
Veaceslav Falico402dae92013-09-25 09:20:09 +02005202 adj->private = private;
Veaceslav Falico5d261912013-08-28 23:25:05 +02005203 dev_hold(adj_dev);
Veaceslav Falico2f268f12013-09-25 09:20:07 +02005204
5205 pr_debug("dev_hold for %s, because of link added from %s to %s\n",
5206 adj_dev->name, dev->name, adj_dev->name);
Veaceslav Falico5d261912013-08-28 23:25:05 +02005207
Alexander Y. Fomichev7ce64c72014-09-15 14:22:35 +04005208 if (netdev_adjacent_is_neigh_list(dev, adj_dev, dev_list)) {
Veaceslav Falico3ee32702014-01-14 21:58:50 +01005209 ret = netdev_adjacent_sysfs_add(dev, adj_dev, dev_list);
Veaceslav Falico5831d662013-09-25 09:20:32 +02005210 if (ret)
5211 goto free_adj;
5212 }
5213
Veaceslav Falico7863c052013-09-25 09:20:06 +02005214 /* Ensure that master link is always the first item in list. */
Veaceslav Falico842d67a2013-09-25 09:20:31 +02005215 if (master) {
5216 ret = sysfs_create_link(&(dev->dev.kobj),
5217 &(adj_dev->dev.kobj), "master");
5218 if (ret)
Veaceslav Falico5831d662013-09-25 09:20:32 +02005219 goto remove_symlinks;
Veaceslav Falico842d67a2013-09-25 09:20:31 +02005220
Veaceslav Falico7863c052013-09-25 09:20:06 +02005221 list_add_rcu(&adj->list, dev_list);
Veaceslav Falico842d67a2013-09-25 09:20:31 +02005222 } else {
Veaceslav Falico7863c052013-09-25 09:20:06 +02005223 list_add_tail_rcu(&adj->list, dev_list);
Veaceslav Falico842d67a2013-09-25 09:20:31 +02005224 }
Veaceslav Falico5d261912013-08-28 23:25:05 +02005225
5226 return 0;
Veaceslav Falico842d67a2013-09-25 09:20:31 +02005227
Veaceslav Falico5831d662013-09-25 09:20:32 +02005228remove_symlinks:
Alexander Y. Fomichev7ce64c72014-09-15 14:22:35 +04005229 if (netdev_adjacent_is_neigh_list(dev, adj_dev, dev_list))
Veaceslav Falico3ee32702014-01-14 21:58:50 +01005230 netdev_adjacent_sysfs_del(dev, adj_dev->name, dev_list);
Veaceslav Falico842d67a2013-09-25 09:20:31 +02005231free_adj:
5232 kfree(adj);
Nikolay Aleksandrov974daef2013-10-23 15:28:56 +02005233 dev_put(adj_dev);
Veaceslav Falico842d67a2013-09-25 09:20:31 +02005234
5235 return ret;
Veaceslav Falico5d261912013-08-28 23:25:05 +02005236}
5237
stephen hemminger1d143d92013-12-29 14:01:29 -08005238static void __netdev_adjacent_dev_remove(struct net_device *dev,
5239 struct net_device *adj_dev,
5240 struct list_head *dev_list)
Veaceslav Falico5d261912013-08-28 23:25:05 +02005241{
5242 struct netdev_adjacent *adj;
5243
Michal Kubeček6ea29da2015-09-24 10:59:05 +02005244 adj = __netdev_find_adj(adj_dev, dev_list);
Veaceslav Falico5d261912013-08-28 23:25:05 +02005245
Veaceslav Falico2f268f12013-09-25 09:20:07 +02005246 if (!adj) {
5247 pr_err("tried to remove device %s from %s\n",
5248 dev->name, adj_dev->name);
Veaceslav Falico5d261912013-08-28 23:25:05 +02005249 BUG();
Veaceslav Falico2f268f12013-09-25 09:20:07 +02005250 }
Veaceslav Falico5d261912013-08-28 23:25:05 +02005251
5252 if (adj->ref_nr > 1) {
Veaceslav Falico2f268f12013-09-25 09:20:07 +02005253 pr_debug("%s to %s ref_nr-- = %d\n", dev->name, adj_dev->name,
5254 adj->ref_nr-1);
Veaceslav Falico5d261912013-08-28 23:25:05 +02005255 adj->ref_nr--;
5256 return;
5257 }
5258
Veaceslav Falico842d67a2013-09-25 09:20:31 +02005259 if (adj->master)
5260 sysfs_remove_link(&(dev->dev.kobj), "master");
5261
Alexander Y. Fomichev7ce64c72014-09-15 14:22:35 +04005262 if (netdev_adjacent_is_neigh_list(dev, adj_dev, dev_list))
Veaceslav Falico3ee32702014-01-14 21:58:50 +01005263 netdev_adjacent_sysfs_del(dev, adj_dev->name, dev_list);
Veaceslav Falico5831d662013-09-25 09:20:32 +02005264
Veaceslav Falico5d261912013-08-28 23:25:05 +02005265 list_del_rcu(&adj->list);
Veaceslav Falico2f268f12013-09-25 09:20:07 +02005266 pr_debug("dev_put for %s, because link removed from %s to %s\n",
5267 adj_dev->name, dev->name, adj_dev->name);
Veaceslav Falico5d261912013-08-28 23:25:05 +02005268 dev_put(adj_dev);
5269 kfree_rcu(adj, rcu);
5270}
5271
stephen hemminger1d143d92013-12-29 14:01:29 -08005272static int __netdev_adjacent_dev_link_lists(struct net_device *dev,
5273 struct net_device *upper_dev,
5274 struct list_head *up_list,
5275 struct list_head *down_list,
5276 void *private, bool master)
Veaceslav Falico5d261912013-08-28 23:25:05 +02005277{
5278 int ret;
5279
Veaceslav Falico402dae92013-09-25 09:20:09 +02005280 ret = __netdev_adjacent_dev_insert(dev, upper_dev, up_list, private,
5281 master);
Veaceslav Falico5d261912013-08-28 23:25:05 +02005282 if (ret)
5283 return ret;
5284
Veaceslav Falico402dae92013-09-25 09:20:09 +02005285 ret = __netdev_adjacent_dev_insert(upper_dev, dev, down_list, private,
5286 false);
Veaceslav Falico5d261912013-08-28 23:25:05 +02005287 if (ret) {
Veaceslav Falico2f268f12013-09-25 09:20:07 +02005288 __netdev_adjacent_dev_remove(dev, upper_dev, up_list);
Veaceslav Falico5d261912013-08-28 23:25:05 +02005289 return ret;
5290 }
5291
5292 return 0;
5293}
5294
stephen hemminger1d143d92013-12-29 14:01:29 -08005295static int __netdev_adjacent_dev_link(struct net_device *dev,
5296 struct net_device *upper_dev)
Veaceslav Falico5d261912013-08-28 23:25:05 +02005297{
Veaceslav Falico2f268f12013-09-25 09:20:07 +02005298 return __netdev_adjacent_dev_link_lists(dev, upper_dev,
5299 &dev->all_adj_list.upper,
5300 &upper_dev->all_adj_list.lower,
Veaceslav Falico402dae92013-09-25 09:20:09 +02005301 NULL, false);
Veaceslav Falico5d261912013-08-28 23:25:05 +02005302}
5303
stephen hemminger1d143d92013-12-29 14:01:29 -08005304static void __netdev_adjacent_dev_unlink_lists(struct net_device *dev,
5305 struct net_device *upper_dev,
5306 struct list_head *up_list,
5307 struct list_head *down_list)
Veaceslav Falico5d261912013-08-28 23:25:05 +02005308{
Veaceslav Falico2f268f12013-09-25 09:20:07 +02005309 __netdev_adjacent_dev_remove(dev, upper_dev, up_list);
5310 __netdev_adjacent_dev_remove(upper_dev, dev, down_list);
Veaceslav Falico5d261912013-08-28 23:25:05 +02005311}
5312
stephen hemminger1d143d92013-12-29 14:01:29 -08005313static void __netdev_adjacent_dev_unlink(struct net_device *dev,
5314 struct net_device *upper_dev)
Veaceslav Falico5d261912013-08-28 23:25:05 +02005315{
Veaceslav Falico2f268f12013-09-25 09:20:07 +02005316 __netdev_adjacent_dev_unlink_lists(dev, upper_dev,
5317 &dev->all_adj_list.upper,
5318 &upper_dev->all_adj_list.lower);
Veaceslav Falico5d261912013-08-28 23:25:05 +02005319}
5320
stephen hemminger1d143d92013-12-29 14:01:29 -08005321static int __netdev_adjacent_dev_link_neighbour(struct net_device *dev,
5322 struct net_device *upper_dev,
5323 void *private, bool master)
Veaceslav Falico2f268f12013-09-25 09:20:07 +02005324{
5325 int ret = __netdev_adjacent_dev_link(dev, upper_dev);
5326
5327 if (ret)
5328 return ret;
5329
5330 ret = __netdev_adjacent_dev_link_lists(dev, upper_dev,
5331 &dev->adj_list.upper,
5332 &upper_dev->adj_list.lower,
Veaceslav Falico402dae92013-09-25 09:20:09 +02005333 private, master);
Veaceslav Falico2f268f12013-09-25 09:20:07 +02005334 if (ret) {
5335 __netdev_adjacent_dev_unlink(dev, upper_dev);
5336 return ret;
5337 }
5338
5339 return 0;
5340}
5341
stephen hemminger1d143d92013-12-29 14:01:29 -08005342static void __netdev_adjacent_dev_unlink_neighbour(struct net_device *dev,
5343 struct net_device *upper_dev)
Veaceslav Falico2f268f12013-09-25 09:20:07 +02005344{
5345 __netdev_adjacent_dev_unlink(dev, upper_dev);
5346 __netdev_adjacent_dev_unlink_lists(dev, upper_dev,
5347 &dev->adj_list.upper,
5348 &upper_dev->adj_list.lower);
5349}
Veaceslav Falico5d261912013-08-28 23:25:05 +02005350
Jiri Pirko9ff162a2013-01-03 22:48:49 +00005351static int __netdev_upper_dev_link(struct net_device *dev,
Veaceslav Falico402dae92013-09-25 09:20:09 +02005352 struct net_device *upper_dev, bool master,
5353 void *private)
Jiri Pirko9ff162a2013-01-03 22:48:49 +00005354{
Jiri Pirko0e4ead92015-08-27 09:31:18 +02005355 struct netdev_notifier_changeupper_info changeupper_info;
Veaceslav Falico5d261912013-08-28 23:25:05 +02005356 struct netdev_adjacent *i, *j, *to_i, *to_j;
5357 int ret = 0;
Jiri Pirko9ff162a2013-01-03 22:48:49 +00005358
5359 ASSERT_RTNL();
5360
5361 if (dev == upper_dev)
5362 return -EBUSY;
5363
5364 /* To prevent loops, check if dev is not upper device to upper_dev. */
Michal Kubeček6ea29da2015-09-24 10:59:05 +02005365 if (__netdev_find_adj(dev, &upper_dev->all_adj_list.upper))
Jiri Pirko9ff162a2013-01-03 22:48:49 +00005366 return -EBUSY;
5367
Michal Kubeček6ea29da2015-09-24 10:59:05 +02005368 if (__netdev_find_adj(upper_dev, &dev->adj_list.upper))
Jiri Pirko9ff162a2013-01-03 22:48:49 +00005369 return -EEXIST;
5370
5371 if (master && netdev_master_upper_dev_get(dev))
5372 return -EBUSY;
5373
Jiri Pirko0e4ead92015-08-27 09:31:18 +02005374 changeupper_info.upper_dev = upper_dev;
5375 changeupper_info.master = master;
5376 changeupper_info.linking = true;
5377
Jiri Pirko573c7ba2015-10-16 14:01:22 +02005378 ret = call_netdevice_notifiers_info(NETDEV_PRECHANGEUPPER, dev,
5379 &changeupper_info.info);
5380 ret = notifier_to_errno(ret);
5381 if (ret)
5382 return ret;
5383
Veaceslav Falico402dae92013-09-25 09:20:09 +02005384 ret = __netdev_adjacent_dev_link_neighbour(dev, upper_dev, private,
5385 master);
Veaceslav Falico5d261912013-08-28 23:25:05 +02005386 if (ret)
5387 return ret;
Jiri Pirko9ff162a2013-01-03 22:48:49 +00005388
Veaceslav Falico5d261912013-08-28 23:25:05 +02005389 /* Now that we linked these devs, make all the upper_dev's
Veaceslav Falico2f268f12013-09-25 09:20:07 +02005390 * all_adj_list.upper visible to every dev's all_adj_list.lower an
Veaceslav Falico5d261912013-08-28 23:25:05 +02005391 * versa, and don't forget the devices itself. All of these
5392 * links are non-neighbours.
5393 */
Veaceslav Falico2f268f12013-09-25 09:20:07 +02005394 list_for_each_entry(i, &dev->all_adj_list.lower, list) {
5395 list_for_each_entry(j, &upper_dev->all_adj_list.upper, list) {
5396 pr_debug("Interlinking %s with %s, non-neighbour\n",
5397 i->dev->name, j->dev->name);
Veaceslav Falico5d261912013-08-28 23:25:05 +02005398 ret = __netdev_adjacent_dev_link(i->dev, j->dev);
5399 if (ret)
5400 goto rollback_mesh;
5401 }
5402 }
Jiri Pirko9ff162a2013-01-03 22:48:49 +00005403
Veaceslav Falico5d261912013-08-28 23:25:05 +02005404 /* add dev to every upper_dev's upper device */
Veaceslav Falico2f268f12013-09-25 09:20:07 +02005405 list_for_each_entry(i, &upper_dev->all_adj_list.upper, list) {
5406 pr_debug("linking %s's upper device %s with %s\n",
5407 upper_dev->name, i->dev->name, dev->name);
Veaceslav Falico5d261912013-08-28 23:25:05 +02005408 ret = __netdev_adjacent_dev_link(dev, i->dev);
5409 if (ret)
5410 goto rollback_upper_mesh;
5411 }
5412
5413 /* add upper_dev to every dev's lower device */
Veaceslav Falico2f268f12013-09-25 09:20:07 +02005414 list_for_each_entry(i, &dev->all_adj_list.lower, list) {
5415 pr_debug("linking %s's lower device %s with %s\n", dev->name,
5416 i->dev->name, upper_dev->name);
Veaceslav Falico5d261912013-08-28 23:25:05 +02005417 ret = __netdev_adjacent_dev_link(i->dev, upper_dev);
5418 if (ret)
5419 goto rollback_lower_mesh;
5420 }
5421
Jiri Pirko0e4ead92015-08-27 09:31:18 +02005422 call_netdevice_notifiers_info(NETDEV_CHANGEUPPER, dev,
5423 &changeupper_info.info);
Jiri Pirko9ff162a2013-01-03 22:48:49 +00005424 return 0;
Veaceslav Falico5d261912013-08-28 23:25:05 +02005425
5426rollback_lower_mesh:
5427 to_i = i;
Veaceslav Falico2f268f12013-09-25 09:20:07 +02005428 list_for_each_entry(i, &dev->all_adj_list.lower, list) {
Veaceslav Falico5d261912013-08-28 23:25:05 +02005429 if (i == to_i)
5430 break;
5431 __netdev_adjacent_dev_unlink(i->dev, upper_dev);
5432 }
5433
5434 i = NULL;
5435
5436rollback_upper_mesh:
5437 to_i = i;
Veaceslav Falico2f268f12013-09-25 09:20:07 +02005438 list_for_each_entry(i, &upper_dev->all_adj_list.upper, list) {
Veaceslav Falico5d261912013-08-28 23:25:05 +02005439 if (i == to_i)
5440 break;
5441 __netdev_adjacent_dev_unlink(dev, i->dev);
5442 }
5443
5444 i = j = NULL;
5445
5446rollback_mesh:
5447 to_i = i;
5448 to_j = j;
Veaceslav Falico2f268f12013-09-25 09:20:07 +02005449 list_for_each_entry(i, &dev->all_adj_list.lower, list) {
5450 list_for_each_entry(j, &upper_dev->all_adj_list.upper, list) {
Veaceslav Falico5d261912013-08-28 23:25:05 +02005451 if (i == to_i && j == to_j)
5452 break;
5453 __netdev_adjacent_dev_unlink(i->dev, j->dev);
5454 }
5455 if (i == to_i)
5456 break;
5457 }
5458
Veaceslav Falico2f268f12013-09-25 09:20:07 +02005459 __netdev_adjacent_dev_unlink_neighbour(dev, upper_dev);
Veaceslav Falico5d261912013-08-28 23:25:05 +02005460
5461 return ret;
Jiri Pirko9ff162a2013-01-03 22:48:49 +00005462}
5463
5464/**
5465 * netdev_upper_dev_link - Add a link to the upper device
5466 * @dev: device
5467 * @upper_dev: new upper device
5468 *
5469 * Adds a link to device which is upper to this one. The caller must hold
5470 * the RTNL lock. On a failure a negative errno code is returned.
5471 * On success the reference counts are adjusted and the function
5472 * returns zero.
5473 */
5474int netdev_upper_dev_link(struct net_device *dev,
5475 struct net_device *upper_dev)
5476{
Veaceslav Falico402dae92013-09-25 09:20:09 +02005477 return __netdev_upper_dev_link(dev, upper_dev, false, NULL);
Jiri Pirko9ff162a2013-01-03 22:48:49 +00005478}
5479EXPORT_SYMBOL(netdev_upper_dev_link);
5480
5481/**
5482 * netdev_master_upper_dev_link - Add a master link to the upper device
5483 * @dev: device
5484 * @upper_dev: new upper device
5485 *
5486 * Adds a link to device which is upper to this one. In this case, only
5487 * one master upper device can be linked, although other non-master devices
5488 * might be linked as well. The caller must hold the RTNL lock.
5489 * On a failure a negative errno code is returned. On success the reference
5490 * counts are adjusted and the function returns zero.
5491 */
5492int netdev_master_upper_dev_link(struct net_device *dev,
5493 struct net_device *upper_dev)
5494{
Veaceslav Falico402dae92013-09-25 09:20:09 +02005495 return __netdev_upper_dev_link(dev, upper_dev, true, NULL);
Jiri Pirko9ff162a2013-01-03 22:48:49 +00005496}
5497EXPORT_SYMBOL(netdev_master_upper_dev_link);
5498
Veaceslav Falico402dae92013-09-25 09:20:09 +02005499int netdev_master_upper_dev_link_private(struct net_device *dev,
5500 struct net_device *upper_dev,
5501 void *private)
5502{
5503 return __netdev_upper_dev_link(dev, upper_dev, true, private);
5504}
5505EXPORT_SYMBOL(netdev_master_upper_dev_link_private);
5506
Jiri Pirko9ff162a2013-01-03 22:48:49 +00005507/**
5508 * netdev_upper_dev_unlink - Removes a link to upper device
5509 * @dev: device
5510 * @upper_dev: new upper device
5511 *
5512 * Removes a link to device which is upper to this one. The caller must hold
5513 * the RTNL lock.
5514 */
5515void netdev_upper_dev_unlink(struct net_device *dev,
5516 struct net_device *upper_dev)
5517{
Jiri Pirko0e4ead92015-08-27 09:31:18 +02005518 struct netdev_notifier_changeupper_info changeupper_info;
Veaceslav Falico5d261912013-08-28 23:25:05 +02005519 struct netdev_adjacent *i, *j;
Jiri Pirko9ff162a2013-01-03 22:48:49 +00005520 ASSERT_RTNL();
5521
Jiri Pirko0e4ead92015-08-27 09:31:18 +02005522 changeupper_info.upper_dev = upper_dev;
5523 changeupper_info.master = netdev_master_upper_dev_get(dev) == upper_dev;
5524 changeupper_info.linking = false;
5525
Jiri Pirko573c7ba2015-10-16 14:01:22 +02005526 call_netdevice_notifiers_info(NETDEV_PRECHANGEUPPER, dev,
5527 &changeupper_info.info);
5528
Veaceslav Falico2f268f12013-09-25 09:20:07 +02005529 __netdev_adjacent_dev_unlink_neighbour(dev, upper_dev);
Veaceslav Falico5d261912013-08-28 23:25:05 +02005530
5531 /* Here is the tricky part. We must remove all dev's lower
5532 * devices from all upper_dev's upper devices and vice
5533 * versa, to maintain the graph relationship.
5534 */
Veaceslav Falico2f268f12013-09-25 09:20:07 +02005535 list_for_each_entry(i, &dev->all_adj_list.lower, list)
5536 list_for_each_entry(j, &upper_dev->all_adj_list.upper, list)
Veaceslav Falico5d261912013-08-28 23:25:05 +02005537 __netdev_adjacent_dev_unlink(i->dev, j->dev);
5538
5539 /* remove also the devices itself from lower/upper device
5540 * list
5541 */
Veaceslav Falico2f268f12013-09-25 09:20:07 +02005542 list_for_each_entry(i, &dev->all_adj_list.lower, list)
Veaceslav Falico5d261912013-08-28 23:25:05 +02005543 __netdev_adjacent_dev_unlink(i->dev, upper_dev);
5544
Veaceslav Falico2f268f12013-09-25 09:20:07 +02005545 list_for_each_entry(i, &upper_dev->all_adj_list.upper, list)
Veaceslav Falico5d261912013-08-28 23:25:05 +02005546 __netdev_adjacent_dev_unlink(dev, i->dev);
5547
Jiri Pirko0e4ead92015-08-27 09:31:18 +02005548 call_netdevice_notifiers_info(NETDEV_CHANGEUPPER, dev,
5549 &changeupper_info.info);
Jiri Pirko9ff162a2013-01-03 22:48:49 +00005550}
5551EXPORT_SYMBOL(netdev_upper_dev_unlink);
5552
Moni Shoua61bd3852015-02-03 16:48:29 +02005553/**
5554 * netdev_bonding_info_change - Dispatch event about slave change
5555 * @dev: device
Masanari Iida4a26e4532015-02-14 22:26:34 +09005556 * @bonding_info: info to dispatch
Moni Shoua61bd3852015-02-03 16:48:29 +02005557 *
5558 * Send NETDEV_BONDING_INFO to netdev notifiers with info.
5559 * The caller must hold the RTNL lock.
5560 */
5561void netdev_bonding_info_change(struct net_device *dev,
5562 struct netdev_bonding_info *bonding_info)
5563{
5564 struct netdev_notifier_bonding_info info;
5565
5566 memcpy(&info.bonding_info, bonding_info,
5567 sizeof(struct netdev_bonding_info));
5568 call_netdevice_notifiers_info(NETDEV_BONDING_INFO, dev,
5569 &info.info);
5570}
5571EXPORT_SYMBOL(netdev_bonding_info_change);
5572
Eric Dumazet2ce1ee12015-02-04 13:37:44 -08005573static void netdev_adjacent_add_links(struct net_device *dev)
Alexander Y. Fomichev4c754312014-08-25 16:26:45 +04005574{
5575 struct netdev_adjacent *iter;
5576
5577 struct net *net = dev_net(dev);
5578
5579 list_for_each_entry(iter, &dev->adj_list.upper, list) {
5580 if (!net_eq(net,dev_net(iter->dev)))
5581 continue;
5582 netdev_adjacent_sysfs_add(iter->dev, dev,
5583 &iter->dev->adj_list.lower);
5584 netdev_adjacent_sysfs_add(dev, iter->dev,
5585 &dev->adj_list.upper);
5586 }
5587
5588 list_for_each_entry(iter, &dev->adj_list.lower, list) {
5589 if (!net_eq(net,dev_net(iter->dev)))
5590 continue;
5591 netdev_adjacent_sysfs_add(iter->dev, dev,
5592 &iter->dev->adj_list.upper);
5593 netdev_adjacent_sysfs_add(dev, iter->dev,
5594 &dev->adj_list.lower);
5595 }
5596}
5597
Eric Dumazet2ce1ee12015-02-04 13:37:44 -08005598static void netdev_adjacent_del_links(struct net_device *dev)
Alexander Y. Fomichev4c754312014-08-25 16:26:45 +04005599{
5600 struct netdev_adjacent *iter;
5601
5602 struct net *net = dev_net(dev);
5603
5604 list_for_each_entry(iter, &dev->adj_list.upper, list) {
5605 if (!net_eq(net,dev_net(iter->dev)))
5606 continue;
5607 netdev_adjacent_sysfs_del(iter->dev, dev->name,
5608 &iter->dev->adj_list.lower);
5609 netdev_adjacent_sysfs_del(dev, iter->dev->name,
5610 &dev->adj_list.upper);
5611 }
5612
5613 list_for_each_entry(iter, &dev->adj_list.lower, list) {
5614 if (!net_eq(net,dev_net(iter->dev)))
5615 continue;
5616 netdev_adjacent_sysfs_del(iter->dev, dev->name,
5617 &iter->dev->adj_list.upper);
5618 netdev_adjacent_sysfs_del(dev, iter->dev->name,
5619 &dev->adj_list.lower);
5620 }
5621}
5622
Veaceslav Falico5bb025f2014-01-14 21:58:51 +01005623void netdev_adjacent_rename_links(struct net_device *dev, char *oldname)
Veaceslav Falico402dae92013-09-25 09:20:09 +02005624{
Veaceslav Falico5bb025f2014-01-14 21:58:51 +01005625 struct netdev_adjacent *iter;
Veaceslav Falico402dae92013-09-25 09:20:09 +02005626
Alexander Y. Fomichev4c754312014-08-25 16:26:45 +04005627 struct net *net = dev_net(dev);
5628
Veaceslav Falico5bb025f2014-01-14 21:58:51 +01005629 list_for_each_entry(iter, &dev->adj_list.upper, list) {
Alexander Y. Fomichev4c754312014-08-25 16:26:45 +04005630 if (!net_eq(net,dev_net(iter->dev)))
5631 continue;
Veaceslav Falico5bb025f2014-01-14 21:58:51 +01005632 netdev_adjacent_sysfs_del(iter->dev, oldname,
5633 &iter->dev->adj_list.lower);
5634 netdev_adjacent_sysfs_add(iter->dev, dev,
5635 &iter->dev->adj_list.lower);
5636 }
Veaceslav Falico402dae92013-09-25 09:20:09 +02005637
Veaceslav Falico5bb025f2014-01-14 21:58:51 +01005638 list_for_each_entry(iter, &dev->adj_list.lower, list) {
Alexander Y. Fomichev4c754312014-08-25 16:26:45 +04005639 if (!net_eq(net,dev_net(iter->dev)))
5640 continue;
Veaceslav Falico5bb025f2014-01-14 21:58:51 +01005641 netdev_adjacent_sysfs_del(iter->dev, oldname,
5642 &iter->dev->adj_list.upper);
5643 netdev_adjacent_sysfs_add(iter->dev, dev,
5644 &iter->dev->adj_list.upper);
5645 }
Veaceslav Falico402dae92013-09-25 09:20:09 +02005646}
Veaceslav Falico402dae92013-09-25 09:20:09 +02005647
5648void *netdev_lower_dev_get_private(struct net_device *dev,
5649 struct net_device *lower_dev)
5650{
5651 struct netdev_adjacent *lower;
5652
5653 if (!lower_dev)
5654 return NULL;
Michal Kubeček6ea29da2015-09-24 10:59:05 +02005655 lower = __netdev_find_adj(lower_dev, &dev->adj_list.lower);
Veaceslav Falico402dae92013-09-25 09:20:09 +02005656 if (!lower)
5657 return NULL;
5658
5659 return lower->private;
5660}
5661EXPORT_SYMBOL(netdev_lower_dev_get_private);
5662
Vlad Yasevich4085ebe2014-05-16 17:04:53 -04005663
5664int dev_get_nest_level(struct net_device *dev,
5665 bool (*type_check)(struct net_device *dev))
5666{
5667 struct net_device *lower = NULL;
5668 struct list_head *iter;
5669 int max_nest = -1;
5670 int nest;
5671
5672 ASSERT_RTNL();
5673
5674 netdev_for_each_lower_dev(dev, lower, iter) {
5675 nest = dev_get_nest_level(lower, type_check);
5676 if (max_nest < nest)
5677 max_nest = nest;
5678 }
5679
5680 if (type_check(dev))
5681 max_nest++;
5682
5683 return max_nest;
5684}
5685EXPORT_SYMBOL(dev_get_nest_level);
5686
Patrick McHardyb6c40d62008-10-07 15:26:48 -07005687static void dev_change_rx_flags(struct net_device *dev, int flags)
5688{
Stephen Hemmingerd3147742008-11-19 21:32:24 -08005689 const struct net_device_ops *ops = dev->netdev_ops;
5690
Vlad Yasevichd2615bf2013-11-19 20:47:15 -05005691 if (ops->ndo_change_rx_flags)
Stephen Hemmingerd3147742008-11-19 21:32:24 -08005692 ops->ndo_change_rx_flags(dev, flags);
Patrick McHardyb6c40d62008-10-07 15:26:48 -07005693}
5694
Nicolas Dichtel991fb3f2013-09-25 12:02:45 +02005695static int __dev_set_promiscuity(struct net_device *dev, int inc, bool notify)
Patrick McHardy4417da62007-06-27 01:28:10 -07005696{
Eric Dumazetb536db92011-11-30 21:42:26 +00005697 unsigned int old_flags = dev->flags;
Eric W. Biedermand04a48b2012-05-23 17:01:57 -06005698 kuid_t uid;
5699 kgid_t gid;
Patrick McHardy4417da62007-06-27 01:28:10 -07005700
Patrick McHardy24023452007-07-14 18:51:31 -07005701 ASSERT_RTNL();
5702
Wang Chendad9b332008-06-18 01:48:28 -07005703 dev->flags |= IFF_PROMISC;
5704 dev->promiscuity += inc;
5705 if (dev->promiscuity == 0) {
5706 /*
5707 * Avoid overflow.
5708 * If inc causes overflow, untouch promisc and return error.
5709 */
5710 if (inc < 0)
5711 dev->flags &= ~IFF_PROMISC;
5712 else {
5713 dev->promiscuity -= inc;
Joe Perches7b6cd1c2012-02-01 10:54:43 +00005714 pr_warn("%s: promiscuity touches roof, set promiscuity failed. promiscuity feature of device might be broken.\n",
5715 dev->name);
Wang Chendad9b332008-06-18 01:48:28 -07005716 return -EOVERFLOW;
5717 }
5718 }
Patrick McHardy4417da62007-06-27 01:28:10 -07005719 if (dev->flags != old_flags) {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00005720 pr_info("device %s %s promiscuous mode\n",
5721 dev->name,
5722 dev->flags & IFF_PROMISC ? "entered" : "left");
David Howells8192b0c2008-11-14 10:39:10 +11005723 if (audit_enabled) {
5724 current_uid_gid(&uid, &gid);
Klaus Heinrich Kiwi7759db82008-01-23 22:57:45 -05005725 audit_log(current->audit_context, GFP_ATOMIC,
5726 AUDIT_ANOM_PROMISCUOUS,
5727 "dev=%s prom=%d old_prom=%d auid=%u uid=%u gid=%u ses=%u",
5728 dev->name, (dev->flags & IFF_PROMISC),
5729 (old_flags & IFF_PROMISC),
Eric W. Biedermane1760bd2012-09-10 22:39:43 -07005730 from_kuid(&init_user_ns, audit_get_loginuid(current)),
Eric W. Biedermand04a48b2012-05-23 17:01:57 -06005731 from_kuid(&init_user_ns, uid),
5732 from_kgid(&init_user_ns, gid),
Klaus Heinrich Kiwi7759db82008-01-23 22:57:45 -05005733 audit_get_sessionid(current));
David Howells8192b0c2008-11-14 10:39:10 +11005734 }
Patrick McHardy24023452007-07-14 18:51:31 -07005735
Patrick McHardyb6c40d62008-10-07 15:26:48 -07005736 dev_change_rx_flags(dev, IFF_PROMISC);
Patrick McHardy4417da62007-06-27 01:28:10 -07005737 }
Nicolas Dichtel991fb3f2013-09-25 12:02:45 +02005738 if (notify)
5739 __dev_notify_flags(dev, old_flags, IFF_PROMISC);
Wang Chendad9b332008-06-18 01:48:28 -07005740 return 0;
Patrick McHardy4417da62007-06-27 01:28:10 -07005741}
5742
Linus Torvalds1da177e2005-04-16 15:20:36 -07005743/**
5744 * dev_set_promiscuity - update promiscuity count on a device
5745 * @dev: device
5746 * @inc: modifier
5747 *
Stephen Hemminger3041a062006-05-26 13:25:24 -07005748 * Add or remove promiscuity from a device. While the count in the device
Linus Torvalds1da177e2005-04-16 15:20:36 -07005749 * remains above zero the interface remains promiscuous. Once it hits zero
5750 * the device reverts back to normal filtering operation. A negative inc
5751 * value is used to drop promiscuity on the device.
Wang Chendad9b332008-06-18 01:48:28 -07005752 * Return 0 if successful or a negative errno code on error.
Linus Torvalds1da177e2005-04-16 15:20:36 -07005753 */
Wang Chendad9b332008-06-18 01:48:28 -07005754int dev_set_promiscuity(struct net_device *dev, int inc)
Linus Torvalds1da177e2005-04-16 15:20:36 -07005755{
Eric Dumazetb536db92011-11-30 21:42:26 +00005756 unsigned int old_flags = dev->flags;
Wang Chendad9b332008-06-18 01:48:28 -07005757 int err;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005758
Nicolas Dichtel991fb3f2013-09-25 12:02:45 +02005759 err = __dev_set_promiscuity(dev, inc, true);
Patrick McHardy4b5a6982008-07-06 15:49:08 -07005760 if (err < 0)
Wang Chendad9b332008-06-18 01:48:28 -07005761 return err;
Patrick McHardy4417da62007-06-27 01:28:10 -07005762 if (dev->flags != old_flags)
5763 dev_set_rx_mode(dev);
Wang Chendad9b332008-06-18 01:48:28 -07005764 return err;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005765}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005766EXPORT_SYMBOL(dev_set_promiscuity);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005767
Nicolas Dichtel991fb3f2013-09-25 12:02:45 +02005768static int __dev_set_allmulti(struct net_device *dev, int inc, bool notify)
Linus Torvalds1da177e2005-04-16 15:20:36 -07005769{
Nicolas Dichtel991fb3f2013-09-25 12:02:45 +02005770 unsigned int old_flags = dev->flags, old_gflags = dev->gflags;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005771
Patrick McHardy24023452007-07-14 18:51:31 -07005772 ASSERT_RTNL();
5773
Linus Torvalds1da177e2005-04-16 15:20:36 -07005774 dev->flags |= IFF_ALLMULTI;
Wang Chendad9b332008-06-18 01:48:28 -07005775 dev->allmulti += inc;
5776 if (dev->allmulti == 0) {
5777 /*
5778 * Avoid overflow.
5779 * If inc causes overflow, untouch allmulti and return error.
5780 */
5781 if (inc < 0)
5782 dev->flags &= ~IFF_ALLMULTI;
5783 else {
5784 dev->allmulti -= inc;
Joe Perches7b6cd1c2012-02-01 10:54:43 +00005785 pr_warn("%s: allmulti touches roof, set allmulti failed. allmulti feature of device might be broken.\n",
5786 dev->name);
Wang Chendad9b332008-06-18 01:48:28 -07005787 return -EOVERFLOW;
5788 }
5789 }
Patrick McHardy24023452007-07-14 18:51:31 -07005790 if (dev->flags ^ old_flags) {
Patrick McHardyb6c40d62008-10-07 15:26:48 -07005791 dev_change_rx_flags(dev, IFF_ALLMULTI);
Patrick McHardy4417da62007-06-27 01:28:10 -07005792 dev_set_rx_mode(dev);
Nicolas Dichtel991fb3f2013-09-25 12:02:45 +02005793 if (notify)
5794 __dev_notify_flags(dev, old_flags,
5795 dev->gflags ^ old_gflags);
Patrick McHardy24023452007-07-14 18:51:31 -07005796 }
Wang Chendad9b332008-06-18 01:48:28 -07005797 return 0;
Patrick McHardy4417da62007-06-27 01:28:10 -07005798}
Nicolas Dichtel991fb3f2013-09-25 12:02:45 +02005799
5800/**
5801 * dev_set_allmulti - update allmulti count on a device
5802 * @dev: device
5803 * @inc: modifier
5804 *
5805 * Add or remove reception of all multicast frames to a device. While the
5806 * count in the device remains above zero the interface remains listening
5807 * to all interfaces. Once it hits zero the device reverts back to normal
5808 * filtering operation. A negative @inc value is used to drop the counter
5809 * when releasing a resource needing all multicasts.
5810 * Return 0 if successful or a negative errno code on error.
5811 */
5812
5813int dev_set_allmulti(struct net_device *dev, int inc)
5814{
5815 return __dev_set_allmulti(dev, inc, true);
5816}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005817EXPORT_SYMBOL(dev_set_allmulti);
Patrick McHardy4417da62007-06-27 01:28:10 -07005818
5819/*
5820 * Upload unicast and multicast address lists to device and
5821 * configure RX filtering. When the device doesn't support unicast
Joe Perches53ccaae2007-12-20 14:02:06 -08005822 * filtering it is put in promiscuous mode while unicast addresses
Patrick McHardy4417da62007-06-27 01:28:10 -07005823 * are present.
5824 */
5825void __dev_set_rx_mode(struct net_device *dev)
5826{
Stephen Hemmingerd3147742008-11-19 21:32:24 -08005827 const struct net_device_ops *ops = dev->netdev_ops;
5828
Patrick McHardy4417da62007-06-27 01:28:10 -07005829 /* dev_open will call this function so the list will stay sane. */
5830 if (!(dev->flags&IFF_UP))
5831 return;
5832
5833 if (!netif_device_present(dev))
YOSHIFUJI Hideaki40b77c92007-07-19 10:43:23 +09005834 return;
Patrick McHardy4417da62007-06-27 01:28:10 -07005835
Jiri Pirko01789342011-08-16 06:29:00 +00005836 if (!(dev->priv_flags & IFF_UNICAST_FLT)) {
Patrick McHardy4417da62007-06-27 01:28:10 -07005837 /* Unicast addresses changes may only happen under the rtnl,
5838 * therefore calling __dev_set_promiscuity here is safe.
5839 */
Jiri Pirko32e7bfc2010-01-25 13:36:10 -08005840 if (!netdev_uc_empty(dev) && !dev->uc_promisc) {
Nicolas Dichtel991fb3f2013-09-25 12:02:45 +02005841 __dev_set_promiscuity(dev, 1, false);
Joe Perches2d348d12011-07-25 16:17:35 -07005842 dev->uc_promisc = true;
Jiri Pirko32e7bfc2010-01-25 13:36:10 -08005843 } else if (netdev_uc_empty(dev) && dev->uc_promisc) {
Nicolas Dichtel991fb3f2013-09-25 12:02:45 +02005844 __dev_set_promiscuity(dev, -1, false);
Joe Perches2d348d12011-07-25 16:17:35 -07005845 dev->uc_promisc = false;
Patrick McHardy4417da62007-06-27 01:28:10 -07005846 }
Patrick McHardy4417da62007-06-27 01:28:10 -07005847 }
Jiri Pirko01789342011-08-16 06:29:00 +00005848
5849 if (ops->ndo_set_rx_mode)
5850 ops->ndo_set_rx_mode(dev);
Patrick McHardy4417da62007-06-27 01:28:10 -07005851}
5852
5853void dev_set_rx_mode(struct net_device *dev)
5854{
David S. Millerb9e40852008-07-15 00:15:08 -07005855 netif_addr_lock_bh(dev);
Patrick McHardy4417da62007-06-27 01:28:10 -07005856 __dev_set_rx_mode(dev);
David S. Millerb9e40852008-07-15 00:15:08 -07005857 netif_addr_unlock_bh(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005858}
5859
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07005860/**
5861 * dev_get_flags - get flags reported to userspace
5862 * @dev: device
5863 *
5864 * Get the combination of flag bits exported through APIs to userspace.
5865 */
Eric Dumazet95c96172012-04-15 05:58:06 +00005866unsigned int dev_get_flags(const struct net_device *dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07005867{
Eric Dumazet95c96172012-04-15 05:58:06 +00005868 unsigned int flags;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005869
5870 flags = (dev->flags & ~(IFF_PROMISC |
5871 IFF_ALLMULTI |
Stefan Rompfb00055a2006-03-20 17:09:11 -08005872 IFF_RUNNING |
5873 IFF_LOWER_UP |
5874 IFF_DORMANT)) |
Linus Torvalds1da177e2005-04-16 15:20:36 -07005875 (dev->gflags & (IFF_PROMISC |
5876 IFF_ALLMULTI));
5877
Stefan Rompfb00055a2006-03-20 17:09:11 -08005878 if (netif_running(dev)) {
5879 if (netif_oper_up(dev))
5880 flags |= IFF_RUNNING;
5881 if (netif_carrier_ok(dev))
5882 flags |= IFF_LOWER_UP;
5883 if (netif_dormant(dev))
5884 flags |= IFF_DORMANT;
5885 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07005886
5887 return flags;
5888}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005889EXPORT_SYMBOL(dev_get_flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005890
Patrick McHardybd380812010-02-26 06:34:53 +00005891int __dev_change_flags(struct net_device *dev, unsigned int flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -07005892{
Eric Dumazetb536db92011-11-30 21:42:26 +00005893 unsigned int old_flags = dev->flags;
Patrick McHardybd380812010-02-26 06:34:53 +00005894 int ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005895
Patrick McHardy24023452007-07-14 18:51:31 -07005896 ASSERT_RTNL();
5897
Linus Torvalds1da177e2005-04-16 15:20:36 -07005898 /*
5899 * Set the flags on our device.
5900 */
5901
5902 dev->flags = (flags & (IFF_DEBUG | IFF_NOTRAILERS | IFF_NOARP |
5903 IFF_DYNAMIC | IFF_MULTICAST | IFF_PORTSEL |
5904 IFF_AUTOMEDIA)) |
5905 (dev->flags & (IFF_UP | IFF_VOLATILE | IFF_PROMISC |
5906 IFF_ALLMULTI));
5907
5908 /*
5909 * Load in the correct multicast list now the flags have changed.
5910 */
5911
Patrick McHardyb6c40d62008-10-07 15:26:48 -07005912 if ((old_flags ^ flags) & IFF_MULTICAST)
5913 dev_change_rx_flags(dev, IFF_MULTICAST);
Patrick McHardy24023452007-07-14 18:51:31 -07005914
Patrick McHardy4417da62007-06-27 01:28:10 -07005915 dev_set_rx_mode(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005916
5917 /*
5918 * Have we downed the interface. We handle IFF_UP ourselves
5919 * according to user attempts to set it, rather than blindly
5920 * setting it.
5921 */
5922
5923 ret = 0;
Peter Pan(潘卫平)d215d102014-06-16 21:57:22 +08005924 if ((old_flags ^ flags) & IFF_UP)
Patrick McHardybd380812010-02-26 06:34:53 +00005925 ret = ((old_flags & IFF_UP) ? __dev_close : __dev_open)(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005926
Linus Torvalds1da177e2005-04-16 15:20:36 -07005927 if ((flags ^ dev->gflags) & IFF_PROMISC) {
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005928 int inc = (flags & IFF_PROMISC) ? 1 : -1;
Nicolas Dichtel991fb3f2013-09-25 12:02:45 +02005929 unsigned int old_flags = dev->flags;
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005930
Linus Torvalds1da177e2005-04-16 15:20:36 -07005931 dev->gflags ^= IFF_PROMISC;
Nicolas Dichtel991fb3f2013-09-25 12:02:45 +02005932
5933 if (__dev_set_promiscuity(dev, inc, false) >= 0)
5934 if (dev->flags != old_flags)
5935 dev_set_rx_mode(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005936 }
5937
5938 /* NOTE: order of synchronization of IFF_PROMISC and IFF_ALLMULTI
5939 is important. Some (broken) drivers set IFF_PROMISC, when
5940 IFF_ALLMULTI is requested not asking us and not reporting.
5941 */
5942 if ((flags ^ dev->gflags) & IFF_ALLMULTI) {
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005943 int inc = (flags & IFF_ALLMULTI) ? 1 : -1;
5944
Linus Torvalds1da177e2005-04-16 15:20:36 -07005945 dev->gflags ^= IFF_ALLMULTI;
Nicolas Dichtel991fb3f2013-09-25 12:02:45 +02005946 __dev_set_allmulti(dev, inc, false);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005947 }
5948
Patrick McHardybd380812010-02-26 06:34:53 +00005949 return ret;
5950}
5951
Nicolas Dichtela528c212013-09-25 12:02:44 +02005952void __dev_notify_flags(struct net_device *dev, unsigned int old_flags,
5953 unsigned int gchanges)
Patrick McHardybd380812010-02-26 06:34:53 +00005954{
5955 unsigned int changes = dev->flags ^ old_flags;
5956
Nicolas Dichtela528c212013-09-25 12:02:44 +02005957 if (gchanges)
Alexei Starovoitov7f294052013-10-23 16:02:42 -07005958 rtmsg_ifinfo(RTM_NEWLINK, dev, gchanges, GFP_ATOMIC);
Nicolas Dichtela528c212013-09-25 12:02:44 +02005959
Patrick McHardybd380812010-02-26 06:34:53 +00005960 if (changes & IFF_UP) {
5961 if (dev->flags & IFF_UP)
5962 call_netdevice_notifiers(NETDEV_UP, dev);
5963 else
5964 call_netdevice_notifiers(NETDEV_DOWN, dev);
5965 }
5966
5967 if (dev->flags & IFF_UP &&
Jiri Pirkobe9efd32013-05-28 01:30:22 +00005968 (changes & ~(IFF_UP | IFF_PROMISC | IFF_ALLMULTI | IFF_VOLATILE))) {
5969 struct netdev_notifier_change_info change_info;
5970
5971 change_info.flags_changed = changes;
5972 call_netdevice_notifiers_info(NETDEV_CHANGE, dev,
5973 &change_info.info);
5974 }
Patrick McHardybd380812010-02-26 06:34:53 +00005975}
5976
5977/**
5978 * dev_change_flags - change device settings
5979 * @dev: device
5980 * @flags: device state flags
5981 *
5982 * Change settings on device based state flags. The flags are
5983 * in the userspace exported format.
5984 */
Eric Dumazetb536db92011-11-30 21:42:26 +00005985int dev_change_flags(struct net_device *dev, unsigned int flags)
Patrick McHardybd380812010-02-26 06:34:53 +00005986{
Eric Dumazetb536db92011-11-30 21:42:26 +00005987 int ret;
Nicolas Dichtel991fb3f2013-09-25 12:02:45 +02005988 unsigned int changes, old_flags = dev->flags, old_gflags = dev->gflags;
Patrick McHardybd380812010-02-26 06:34:53 +00005989
5990 ret = __dev_change_flags(dev, flags);
5991 if (ret < 0)
5992 return ret;
5993
Nicolas Dichtel991fb3f2013-09-25 12:02:45 +02005994 changes = (old_flags ^ dev->flags) | (old_gflags ^ dev->gflags);
Nicolas Dichtela528c212013-09-25 12:02:44 +02005995 __dev_notify_flags(dev, old_flags, changes);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005996 return ret;
5997}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005998EXPORT_SYMBOL(dev_change_flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005999
Veaceslav Falico2315dc92014-01-10 16:56:25 +01006000static int __dev_set_mtu(struct net_device *dev, int new_mtu)
6001{
6002 const struct net_device_ops *ops = dev->netdev_ops;
6003
6004 if (ops->ndo_change_mtu)
6005 return ops->ndo_change_mtu(dev, new_mtu);
6006
6007 dev->mtu = new_mtu;
6008 return 0;
6009}
6010
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07006011/**
6012 * dev_set_mtu - Change maximum transfer unit
6013 * @dev: device
6014 * @new_mtu: new transfer unit
6015 *
6016 * Change the maximum transfer size of the network device.
6017 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07006018int dev_set_mtu(struct net_device *dev, int new_mtu)
6019{
Veaceslav Falico2315dc92014-01-10 16:56:25 +01006020 int err, orig_mtu;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006021
6022 if (new_mtu == dev->mtu)
6023 return 0;
6024
6025 /* MTU must be positive. */
6026 if (new_mtu < 0)
6027 return -EINVAL;
6028
6029 if (!netif_device_present(dev))
6030 return -ENODEV;
6031
Veaceslav Falico1d486bf2014-01-16 00:02:18 +01006032 err = call_netdevice_notifiers(NETDEV_PRECHANGEMTU, dev);
6033 err = notifier_to_errno(err);
6034 if (err)
6035 return err;
Stephen Hemmingerd3147742008-11-19 21:32:24 -08006036
Veaceslav Falico2315dc92014-01-10 16:56:25 +01006037 orig_mtu = dev->mtu;
6038 err = __dev_set_mtu(dev, new_mtu);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006039
Veaceslav Falico2315dc92014-01-10 16:56:25 +01006040 if (!err) {
6041 err = call_netdevice_notifiers(NETDEV_CHANGEMTU, dev);
6042 err = notifier_to_errno(err);
6043 if (err) {
6044 /* setting mtu back and notifying everyone again,
6045 * so that they have a chance to revert changes.
6046 */
6047 __dev_set_mtu(dev, orig_mtu);
6048 call_netdevice_notifiers(NETDEV_CHANGEMTU, dev);
6049 }
6050 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07006051 return err;
6052}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07006053EXPORT_SYMBOL(dev_set_mtu);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006054
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07006055/**
Vlad Dogarucbda10f2011-01-13 23:38:30 +00006056 * dev_set_group - Change group this device belongs to
6057 * @dev: device
6058 * @new_group: group this device should belong to
6059 */
6060void dev_set_group(struct net_device *dev, int new_group)
6061{
6062 dev->group = new_group;
6063}
6064EXPORT_SYMBOL(dev_set_group);
6065
6066/**
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07006067 * dev_set_mac_address - Change Media Access Control Address
6068 * @dev: device
6069 * @sa: new address
6070 *
6071 * Change the hardware (MAC) address of the device
6072 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07006073int dev_set_mac_address(struct net_device *dev, struct sockaddr *sa)
6074{
Stephen Hemmingerd3147742008-11-19 21:32:24 -08006075 const struct net_device_ops *ops = dev->netdev_ops;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006076 int err;
6077
Stephen Hemmingerd3147742008-11-19 21:32:24 -08006078 if (!ops->ndo_set_mac_address)
Linus Torvalds1da177e2005-04-16 15:20:36 -07006079 return -EOPNOTSUPP;
6080 if (sa->sa_family != dev->type)
6081 return -EINVAL;
6082 if (!netif_device_present(dev))
6083 return -ENODEV;
Stephen Hemmingerd3147742008-11-19 21:32:24 -08006084 err = ops->ndo_set_mac_address(dev, sa);
Jiri Pirkof6521512013-01-01 03:30:14 +00006085 if (err)
6086 return err;
Jiri Pirkofbdeca22013-01-01 03:30:16 +00006087 dev->addr_assign_type = NET_ADDR_SET;
Jiri Pirkof6521512013-01-01 03:30:14 +00006088 call_netdevice_notifiers(NETDEV_CHANGEADDR, dev);
Theodore Ts'o7bf23572012-07-04 21:23:25 -04006089 add_device_randomness(dev->dev_addr, dev->addr_len);
Jiri Pirkof6521512013-01-01 03:30:14 +00006090 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006091}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07006092EXPORT_SYMBOL(dev_set_mac_address);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006093
Jiri Pirko4bf84c32012-12-27 23:49:37 +00006094/**
6095 * dev_change_carrier - Change device carrier
6096 * @dev: device
Randy Dunlap691b3b72013-03-04 12:32:43 +00006097 * @new_carrier: new value
Jiri Pirko4bf84c32012-12-27 23:49:37 +00006098 *
6099 * Change device carrier
6100 */
6101int dev_change_carrier(struct net_device *dev, bool new_carrier)
6102{
6103 const struct net_device_ops *ops = dev->netdev_ops;
6104
6105 if (!ops->ndo_change_carrier)
6106 return -EOPNOTSUPP;
6107 if (!netif_device_present(dev))
6108 return -ENODEV;
6109 return ops->ndo_change_carrier(dev, new_carrier);
6110}
6111EXPORT_SYMBOL(dev_change_carrier);
6112
Linus Torvalds1da177e2005-04-16 15:20:36 -07006113/**
Jiri Pirko66b52b02013-07-29 18:16:49 +02006114 * dev_get_phys_port_id - Get device physical port ID
6115 * @dev: device
6116 * @ppid: port ID
6117 *
6118 * Get device physical port ID
6119 */
6120int dev_get_phys_port_id(struct net_device *dev,
Jiri Pirko02637fc2014-11-28 14:34:16 +01006121 struct netdev_phys_item_id *ppid)
Jiri Pirko66b52b02013-07-29 18:16:49 +02006122{
6123 const struct net_device_ops *ops = dev->netdev_ops;
6124
6125 if (!ops->ndo_get_phys_port_id)
6126 return -EOPNOTSUPP;
6127 return ops->ndo_get_phys_port_id(dev, ppid);
6128}
6129EXPORT_SYMBOL(dev_get_phys_port_id);
6130
6131/**
David Aherndb24a902015-03-17 20:23:15 -06006132 * dev_get_phys_port_name - Get device physical port name
6133 * @dev: device
6134 * @name: port name
6135 *
6136 * Get device physical port name
6137 */
6138int dev_get_phys_port_name(struct net_device *dev,
6139 char *name, size_t len)
6140{
6141 const struct net_device_ops *ops = dev->netdev_ops;
6142
6143 if (!ops->ndo_get_phys_port_name)
6144 return -EOPNOTSUPP;
6145 return ops->ndo_get_phys_port_name(dev, name, len);
6146}
6147EXPORT_SYMBOL(dev_get_phys_port_name);
6148
6149/**
Anuradha Karuppiahd746d702015-07-14 13:43:19 -07006150 * dev_change_proto_down - update protocol port state information
6151 * @dev: device
6152 * @proto_down: new value
6153 *
6154 * This info can be used by switch drivers to set the phys state of the
6155 * port.
6156 */
6157int dev_change_proto_down(struct net_device *dev, bool proto_down)
6158{
6159 const struct net_device_ops *ops = dev->netdev_ops;
6160
6161 if (!ops->ndo_change_proto_down)
6162 return -EOPNOTSUPP;
6163 if (!netif_device_present(dev))
6164 return -ENODEV;
6165 return ops->ndo_change_proto_down(dev, proto_down);
6166}
6167EXPORT_SYMBOL(dev_change_proto_down);
6168
6169/**
Linus Torvalds1da177e2005-04-16 15:20:36 -07006170 * dev_new_index - allocate an ifindex
Randy Dunlapc4ea43c2007-10-12 21:17:49 -07006171 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -07006172 *
6173 * Returns a suitable unique value for a new device interface
6174 * number. The caller must hold the rtnl semaphore or the
6175 * dev_base_lock to be sure it remains unique.
6176 */
Eric W. Biederman881d9662007-09-17 11:56:21 -07006177static int dev_new_index(struct net *net)
Linus Torvalds1da177e2005-04-16 15:20:36 -07006178{
Pavel Emelyanovaa79e662012-08-08 21:53:19 +00006179 int ifindex = net->ifindex;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006180 for (;;) {
6181 if (++ifindex <= 0)
6182 ifindex = 1;
Eric W. Biederman881d9662007-09-17 11:56:21 -07006183 if (!__dev_get_by_index(net, ifindex))
Pavel Emelyanovaa79e662012-08-08 21:53:19 +00006184 return net->ifindex = ifindex;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006185 }
6186}
6187
Linus Torvalds1da177e2005-04-16 15:20:36 -07006188/* Delayed registration/unregisteration */
Denis Cheng3b5b34f2007-12-07 00:49:17 -08006189static LIST_HEAD(net_todo_list);
Cong Wang200b9162014-05-12 15:11:20 -07006190DECLARE_WAIT_QUEUE_HEAD(netdev_unregistering_wq);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006191
Stephen Hemminger6f05f622007-03-08 20:46:03 -08006192static void net_set_todo(struct net_device *dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07006193{
Linus Torvalds1da177e2005-04-16 15:20:36 -07006194 list_add_tail(&dev->todo_list, &net_todo_list);
Eric W. Biederman50624c92013-09-23 21:19:49 -07006195 dev_net(dev)->dev_unreg_count++;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006196}
6197
Eric Dumazet9b5e3832009-10-27 07:04:19 +00006198static void rollback_registered_many(struct list_head *head)
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07006199{
Krishna Kumare93737b2009-12-08 22:26:02 +00006200 struct net_device *dev, *tmp;
Eric W. Biederman5cde2822013-10-05 19:26:05 -07006201 LIST_HEAD(close_head);
Eric Dumazet9b5e3832009-10-27 07:04:19 +00006202
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07006203 BUG_ON(dev_boot_phase);
6204 ASSERT_RTNL();
6205
Krishna Kumare93737b2009-12-08 22:26:02 +00006206 list_for_each_entry_safe(dev, tmp, head, unreg_list) {
Eric Dumazet9b5e3832009-10-27 07:04:19 +00006207 /* Some devices call without registering
Krishna Kumare93737b2009-12-08 22:26:02 +00006208 * for initialization unwind. Remove those
6209 * devices and proceed with the remaining.
Eric Dumazet9b5e3832009-10-27 07:04:19 +00006210 */
6211 if (dev->reg_state == NETREG_UNINITIALIZED) {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00006212 pr_debug("unregister_netdevice: device %s/%p never was registered\n",
6213 dev->name, dev);
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07006214
Eric Dumazet9b5e3832009-10-27 07:04:19 +00006215 WARN_ON(1);
Krishna Kumare93737b2009-12-08 22:26:02 +00006216 list_del(&dev->unreg_list);
6217 continue;
Eric Dumazet9b5e3832009-10-27 07:04:19 +00006218 }
Eric Dumazet449f4542011-05-19 12:24:16 +00006219 dev->dismantle = true;
Eric Dumazet9b5e3832009-10-27 07:04:19 +00006220 BUG_ON(dev->reg_state != NETREG_REGISTERED);
Octavian Purdila44345722010-12-13 12:44:07 +00006221 }
Eric Dumazet9b5e3832009-10-27 07:04:19 +00006222
Octavian Purdila44345722010-12-13 12:44:07 +00006223 /* If device is running, close it first. */
Eric W. Biederman5cde2822013-10-05 19:26:05 -07006224 list_for_each_entry(dev, head, unreg_list)
6225 list_add_tail(&dev->close_list, &close_head);
David S. Miller99c4a262015-03-18 22:52:33 -04006226 dev_close_many(&close_head, true);
Eric Dumazet9b5e3832009-10-27 07:04:19 +00006227
Octavian Purdila44345722010-12-13 12:44:07 +00006228 list_for_each_entry(dev, head, unreg_list) {
Eric Dumazet9b5e3832009-10-27 07:04:19 +00006229 /* And unlink it from device chain. */
6230 unlist_netdevice(dev);
6231
6232 dev->reg_state = NETREG_UNREGISTERING;
Julian Anastasove9e4dd32015-07-09 09:59:09 +03006233 on_each_cpu(flush_backlog, dev, 1);
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07006234 }
6235
Eric Dumazet9b5e3832009-10-27 07:04:19 +00006236 synchronize_net();
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07006237
Eric Dumazet9b5e3832009-10-27 07:04:19 +00006238 list_for_each_entry(dev, head, unreg_list) {
Mahesh Bandewar395eea62014-12-03 13:46:24 -08006239 struct sk_buff *skb = NULL;
6240
Eric Dumazet9b5e3832009-10-27 07:04:19 +00006241 /* Shutdown queueing discipline. */
6242 dev_shutdown(dev);
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07006243
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07006244
Eric Dumazet9b5e3832009-10-27 07:04:19 +00006245 /* Notify protocols, that we are about to destroy
6246 this device. They should clean all the things.
6247 */
6248 call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
6249
Mahesh Bandewar395eea62014-12-03 13:46:24 -08006250 if (!dev->rtnl_link_ops ||
6251 dev->rtnl_link_state == RTNL_LINK_INITIALIZED)
6252 skb = rtmsg_ifinfo_build_skb(RTM_DELLINK, dev, ~0U,
6253 GFP_KERNEL);
6254
Eric Dumazet9b5e3832009-10-27 07:04:19 +00006255 /*
6256 * Flush the unicast and multicast chains
6257 */
Jiri Pirkoa748ee22010-04-01 21:22:09 +00006258 dev_uc_flush(dev);
Jiri Pirko22bedad32010-04-01 21:22:57 +00006259 dev_mc_flush(dev);
Eric Dumazet9b5e3832009-10-27 07:04:19 +00006260
6261 if (dev->netdev_ops->ndo_uninit)
6262 dev->netdev_ops->ndo_uninit(dev);
6263
Mahesh Bandewar395eea62014-12-03 13:46:24 -08006264 if (skb)
6265 rtmsg_ifinfo_send(skb, dev, GFP_KERNEL);
Roopa Prabhu56bfa7e2014-05-01 11:40:30 -07006266
Jiri Pirko9ff162a2013-01-03 22:48:49 +00006267 /* Notifier chain MUST detach us all upper devices. */
6268 WARN_ON(netdev_has_any_upper_dev(dev));
Eric Dumazet9b5e3832009-10-27 07:04:19 +00006269
6270 /* Remove entries from kobject tree */
6271 netdev_unregister_kobject(dev);
Alexander Duyck024e9672013-01-10 08:57:46 +00006272#ifdef CONFIG_XPS
6273 /* Remove XPS queueing entries */
6274 netif_reset_xps_queues_gt(dev, 0);
6275#endif
Eric Dumazet9b5e3832009-10-27 07:04:19 +00006276 }
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07006277
Eric W. Biederman850a5452011-10-13 22:25:23 +00006278 synchronize_net();
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07006279
Eric W. Biedermana5ee1552009-11-29 15:45:58 +00006280 list_for_each_entry(dev, head, unreg_list)
Eric Dumazet9b5e3832009-10-27 07:04:19 +00006281 dev_put(dev);
6282}
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07006283
Eric Dumazet9b5e3832009-10-27 07:04:19 +00006284static void rollback_registered(struct net_device *dev)
6285{
6286 LIST_HEAD(single);
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07006287
Eric Dumazet9b5e3832009-10-27 07:04:19 +00006288 list_add(&dev->unreg_list, &single);
6289 rollback_registered_many(&single);
Eric Dumazetceaaec92011-02-17 22:59:19 +00006290 list_del(&single);
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07006291}
6292
Jarod Wilsonfd867d52015-11-02 21:55:59 -05006293static netdev_features_t netdev_sync_upper_features(struct net_device *lower,
6294 struct net_device *upper, netdev_features_t features)
6295{
6296 netdev_features_t upper_disables = NETIF_F_UPPER_DISABLES;
6297 netdev_features_t feature;
Jarod Wilson5ba3f7d2015-11-03 10:15:59 -05006298 int feature_bit;
Jarod Wilsonfd867d52015-11-02 21:55:59 -05006299
Jarod Wilson5ba3f7d2015-11-03 10:15:59 -05006300 for_each_netdev_feature(&upper_disables, feature_bit) {
6301 feature = __NETIF_F_BIT(feature_bit);
Jarod Wilsonfd867d52015-11-02 21:55:59 -05006302 if (!(upper->wanted_features & feature)
6303 && (features & feature)) {
6304 netdev_dbg(lower, "Dropping feature %pNF, upper dev %s has it off.\n",
6305 &feature, upper->name);
6306 features &= ~feature;
6307 }
6308 }
6309
6310 return features;
6311}
6312
6313static void netdev_sync_lower_features(struct net_device *upper,
6314 struct net_device *lower, netdev_features_t features)
6315{
6316 netdev_features_t upper_disables = NETIF_F_UPPER_DISABLES;
6317 netdev_features_t feature;
Jarod Wilson5ba3f7d2015-11-03 10:15:59 -05006318 int feature_bit;
Jarod Wilsonfd867d52015-11-02 21:55:59 -05006319
Jarod Wilson5ba3f7d2015-11-03 10:15:59 -05006320 for_each_netdev_feature(&upper_disables, feature_bit) {
6321 feature = __NETIF_F_BIT(feature_bit);
Jarod Wilsonfd867d52015-11-02 21:55:59 -05006322 if (!(features & feature) && (lower->features & feature)) {
6323 netdev_dbg(upper, "Disabling feature %pNF on lower dev %s.\n",
6324 &feature, lower->name);
6325 lower->wanted_features &= ~feature;
6326 netdev_update_features(lower);
6327
6328 if (unlikely(lower->features & feature))
6329 netdev_WARN(upper, "failed to disable %pNF on %s!\n",
6330 &feature, lower->name);
6331 }
6332 }
6333}
6334
Michał Mirosławc8f44af2011-11-15 15:29:55 +00006335static netdev_features_t netdev_fix_features(struct net_device *dev,
6336 netdev_features_t features)
Herbert Xub63365a2008-10-23 01:11:29 -07006337{
Michał Mirosław57422dc2011-01-22 12:14:12 +00006338 /* Fix illegal checksum combinations */
6339 if ((features & NETIF_F_HW_CSUM) &&
6340 (features & (NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM))) {
Michał Mirosław6f404e42011-05-16 15:14:21 -04006341 netdev_warn(dev, "mixed HW and IP checksum settings.\n");
Michał Mirosław57422dc2011-01-22 12:14:12 +00006342 features &= ~(NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM);
6343 }
6344
Herbert Xub63365a2008-10-23 01:11:29 -07006345 /* TSO requires that SG is present as well. */
Ben Hutchingsea2d3682011-04-12 14:38:37 +00006346 if ((features & NETIF_F_ALL_TSO) && !(features & NETIF_F_SG)) {
Michał Mirosław6f404e42011-05-16 15:14:21 -04006347 netdev_dbg(dev, "Dropping TSO features since no SG feature.\n");
Ben Hutchingsea2d3682011-04-12 14:38:37 +00006348 features &= ~NETIF_F_ALL_TSO;
Herbert Xub63365a2008-10-23 01:11:29 -07006349 }
6350
Pravin B Shelarec5f0612013-03-07 09:28:01 +00006351 if ((features & NETIF_F_TSO) && !(features & NETIF_F_HW_CSUM) &&
6352 !(features & NETIF_F_IP_CSUM)) {
6353 netdev_dbg(dev, "Dropping TSO features since no CSUM feature.\n");
6354 features &= ~NETIF_F_TSO;
6355 features &= ~NETIF_F_TSO_ECN;
6356 }
6357
6358 if ((features & NETIF_F_TSO6) && !(features & NETIF_F_HW_CSUM) &&
6359 !(features & NETIF_F_IPV6_CSUM)) {
6360 netdev_dbg(dev, "Dropping TSO6 features since no CSUM feature.\n");
6361 features &= ~NETIF_F_TSO6;
6362 }
6363
Ben Hutchings31d8b9e2011-04-12 14:47:15 +00006364 /* TSO ECN requires that TSO is present as well. */
6365 if ((features & NETIF_F_ALL_TSO) == NETIF_F_TSO_ECN)
6366 features &= ~NETIF_F_TSO_ECN;
6367
Michał Mirosław212b5732011-02-15 16:59:16 +00006368 /* Software GSO depends on SG. */
6369 if ((features & NETIF_F_GSO) && !(features & NETIF_F_SG)) {
Michał Mirosław6f404e42011-05-16 15:14:21 -04006370 netdev_dbg(dev, "Dropping NETIF_F_GSO since no SG feature.\n");
Michał Mirosław212b5732011-02-15 16:59:16 +00006371 features &= ~NETIF_F_GSO;
6372 }
6373
Michał Mirosławacd11302011-01-24 15:45:15 -08006374 /* UFO needs SG and checksumming */
Herbert Xub63365a2008-10-23 01:11:29 -07006375 if (features & NETIF_F_UFO) {
Michał Mirosław79032642010-11-30 06:38:00 +00006376 /* maybe split UFO into V4 and V6? */
6377 if (!((features & NETIF_F_GEN_CSUM) ||
6378 (features & (NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM))
6379 == (NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM))) {
Michał Mirosław6f404e42011-05-16 15:14:21 -04006380 netdev_dbg(dev,
Michał Mirosławacd11302011-01-24 15:45:15 -08006381 "Dropping NETIF_F_UFO since no checksum offload features.\n");
Herbert Xub63365a2008-10-23 01:11:29 -07006382 features &= ~NETIF_F_UFO;
6383 }
6384
6385 if (!(features & NETIF_F_SG)) {
Michał Mirosław6f404e42011-05-16 15:14:21 -04006386 netdev_dbg(dev,
Michał Mirosławacd11302011-01-24 15:45:15 -08006387 "Dropping NETIF_F_UFO since no NETIF_F_SG feature.\n");
Herbert Xub63365a2008-10-23 01:11:29 -07006388 features &= ~NETIF_F_UFO;
6389 }
6390 }
6391
Jiri Pirkod0290212014-04-02 23:09:31 +02006392#ifdef CONFIG_NET_RX_BUSY_POLL
6393 if (dev->netdev_ops->ndo_busy_poll)
6394 features |= NETIF_F_BUSY_POLL;
6395 else
6396#endif
6397 features &= ~NETIF_F_BUSY_POLL;
6398
Herbert Xub63365a2008-10-23 01:11:29 -07006399 return features;
6400}
Herbert Xub63365a2008-10-23 01:11:29 -07006401
Michał Mirosław6cb6a272011-04-02 22:48:47 -07006402int __netdev_update_features(struct net_device *dev)
Michał Mirosław5455c692011-02-15 16:59:17 +00006403{
Jarod Wilsonfd867d52015-11-02 21:55:59 -05006404 struct net_device *upper, *lower;
Michał Mirosławc8f44af2011-11-15 15:29:55 +00006405 netdev_features_t features;
Jarod Wilsonfd867d52015-11-02 21:55:59 -05006406 struct list_head *iter;
Jarod Wilsone7868a82015-11-03 23:09:32 -05006407 int err = -1;
Michał Mirosław5455c692011-02-15 16:59:17 +00006408
Michał Mirosław87267482011-04-12 09:56:38 +00006409 ASSERT_RTNL();
6410
Michał Mirosław5455c692011-02-15 16:59:17 +00006411 features = netdev_get_wanted_features(dev);
6412
6413 if (dev->netdev_ops->ndo_fix_features)
6414 features = dev->netdev_ops->ndo_fix_features(dev, features);
6415
6416 /* driver might be less strict about feature dependencies */
6417 features = netdev_fix_features(dev, features);
6418
Jarod Wilsonfd867d52015-11-02 21:55:59 -05006419 /* some features can't be enabled if they're off an an upper device */
6420 netdev_for_each_upper_dev_rcu(dev, upper, iter)
6421 features = netdev_sync_upper_features(dev, upper, features);
6422
Michał Mirosław5455c692011-02-15 16:59:17 +00006423 if (dev->features == features)
Jarod Wilsone7868a82015-11-03 23:09:32 -05006424 goto sync_lower;
Michał Mirosław5455c692011-02-15 16:59:17 +00006425
Michał Mirosławc8f44af2011-11-15 15:29:55 +00006426 netdev_dbg(dev, "Features changed: %pNF -> %pNF\n",
6427 &dev->features, &features);
Michał Mirosław5455c692011-02-15 16:59:17 +00006428
6429 if (dev->netdev_ops->ndo_set_features)
6430 err = dev->netdev_ops->ndo_set_features(dev, features);
Nikolay Aleksandrov5f8dc332015-11-13 14:54:01 +01006431 else
6432 err = 0;
Michał Mirosław5455c692011-02-15 16:59:17 +00006433
Michał Mirosław6cb6a272011-04-02 22:48:47 -07006434 if (unlikely(err < 0)) {
Michał Mirosław5455c692011-02-15 16:59:17 +00006435 netdev_err(dev,
Michał Mirosławc8f44af2011-11-15 15:29:55 +00006436 "set_features() failed (%d); wanted %pNF, left %pNF\n",
6437 err, &features, &dev->features);
Nikolay Aleksandrov17b85d22015-11-17 15:49:06 +01006438 /* return non-0 since some features might have changed and
6439 * it's better to fire a spurious notification than miss it
6440 */
6441 return -1;
Michał Mirosław6cb6a272011-04-02 22:48:47 -07006442 }
6443
Jarod Wilsone7868a82015-11-03 23:09:32 -05006444sync_lower:
Jarod Wilsonfd867d52015-11-02 21:55:59 -05006445 /* some features must be disabled on lower devices when disabled
6446 * on an upper device (think: bonding master or bridge)
6447 */
6448 netdev_for_each_lower_dev(dev, lower, iter)
6449 netdev_sync_lower_features(dev, lower, features);
6450
Michał Mirosław6cb6a272011-04-02 22:48:47 -07006451 if (!err)
6452 dev->features = features;
6453
Jarod Wilsone7868a82015-11-03 23:09:32 -05006454 return err < 0 ? 0 : 1;
Michał Mirosław6cb6a272011-04-02 22:48:47 -07006455}
6456
Michał Mirosławafe12cc2011-05-07 03:22:17 +00006457/**
6458 * netdev_update_features - recalculate device features
6459 * @dev: the device to check
6460 *
6461 * Recalculate dev->features set and send notifications if it
6462 * has changed. Should be called after driver or hardware dependent
6463 * conditions might have changed that influence the features.
6464 */
Michał Mirosław6cb6a272011-04-02 22:48:47 -07006465void netdev_update_features(struct net_device *dev)
6466{
6467 if (__netdev_update_features(dev))
6468 netdev_features_change(dev);
Michał Mirosław5455c692011-02-15 16:59:17 +00006469}
6470EXPORT_SYMBOL(netdev_update_features);
6471
Linus Torvalds1da177e2005-04-16 15:20:36 -07006472/**
Michał Mirosławafe12cc2011-05-07 03:22:17 +00006473 * netdev_change_features - recalculate device features
6474 * @dev: the device to check
6475 *
6476 * Recalculate dev->features set and send notifications even
6477 * if they have not changed. Should be called instead of
6478 * netdev_update_features() if also dev->vlan_features might
6479 * have changed to allow the changes to be propagated to stacked
6480 * VLAN devices.
6481 */
6482void netdev_change_features(struct net_device *dev)
6483{
6484 __netdev_update_features(dev);
6485 netdev_features_change(dev);
6486}
6487EXPORT_SYMBOL(netdev_change_features);
6488
6489/**
Patrick Mullaneyfc4a7482009-12-03 15:59:22 -08006490 * netif_stacked_transfer_operstate - transfer operstate
6491 * @rootdev: the root or lower level device to transfer state from
6492 * @dev: the device to transfer operstate to
6493 *
6494 * Transfer operational state from root to device. This is normally
6495 * called when a stacking relationship exists between the root
6496 * device and the device(a leaf device).
6497 */
6498void netif_stacked_transfer_operstate(const struct net_device *rootdev,
6499 struct net_device *dev)
6500{
6501 if (rootdev->operstate == IF_OPER_DORMANT)
6502 netif_dormant_on(dev);
6503 else
6504 netif_dormant_off(dev);
6505
6506 if (netif_carrier_ok(rootdev)) {
6507 if (!netif_carrier_ok(dev))
6508 netif_carrier_on(dev);
6509 } else {
6510 if (netif_carrier_ok(dev))
6511 netif_carrier_off(dev);
6512 }
6513}
6514EXPORT_SYMBOL(netif_stacked_transfer_operstate);
6515
Michael Daltona953be52014-01-16 22:23:28 -08006516#ifdef CONFIG_SYSFS
Eric Dumazet1b4bf462010-09-23 17:26:35 +00006517static int netif_alloc_rx_queues(struct net_device *dev)
6518{
Eric Dumazet1b4bf462010-09-23 17:26:35 +00006519 unsigned int i, count = dev->num_rx_queues;
Tom Herbertbd25fa72010-10-18 18:00:16 +00006520 struct netdev_rx_queue *rx;
Pankaj Gupta10595902015-01-12 11:41:28 +05306521 size_t sz = count * sizeof(*rx);
Eric Dumazet1b4bf462010-09-23 17:26:35 +00006522
Tom Herbertbd25fa72010-10-18 18:00:16 +00006523 BUG_ON(count < 1);
Eric Dumazet1b4bf462010-09-23 17:26:35 +00006524
Pankaj Gupta10595902015-01-12 11:41:28 +05306525 rx = kzalloc(sz, GFP_KERNEL | __GFP_NOWARN | __GFP_REPEAT);
6526 if (!rx) {
6527 rx = vzalloc(sz);
6528 if (!rx)
6529 return -ENOMEM;
6530 }
Tom Herbertbd25fa72010-10-18 18:00:16 +00006531 dev->_rx = rx;
6532
Tom Herbertbd25fa72010-10-18 18:00:16 +00006533 for (i = 0; i < count; i++)
Tom Herbertfe822242010-11-09 10:47:38 +00006534 rx[i].dev = dev;
Eric Dumazet1b4bf462010-09-23 17:26:35 +00006535 return 0;
6536}
Tom Herbertbf264142010-11-26 08:36:09 +00006537#endif
Eric Dumazet1b4bf462010-09-23 17:26:35 +00006538
Changli Gaoaa942102010-12-04 02:31:41 +00006539static void netdev_init_one_queue(struct net_device *dev,
6540 struct netdev_queue *queue, void *_unused)
6541{
6542 /* Initialize queue lock */
6543 spin_lock_init(&queue->_xmit_lock);
6544 netdev_set_xmit_lockdep_class(&queue->_xmit_lock, dev->type);
6545 queue->xmit_lock_owner = -1;
Changli Gaob236da62010-12-14 03:09:15 +00006546 netdev_queue_numa_node_write(queue, NUMA_NO_NODE);
Changli Gaoaa942102010-12-04 02:31:41 +00006547 queue->dev = dev;
Tom Herbert114cf582011-11-28 16:33:09 +00006548#ifdef CONFIG_BQL
6549 dql_init(&queue->dql, HZ);
6550#endif
Changli Gaoaa942102010-12-04 02:31:41 +00006551}
6552
Eric Dumazet60877a32013-06-20 01:15:51 -07006553static void netif_free_tx_queues(struct net_device *dev)
6554{
WANG Cong4cb28972014-06-02 15:55:22 -07006555 kvfree(dev->_tx);
Eric Dumazet60877a32013-06-20 01:15:51 -07006556}
6557
Tom Herberte6484932010-10-18 18:04:39 +00006558static int netif_alloc_netdev_queues(struct net_device *dev)
6559{
6560 unsigned int count = dev->num_tx_queues;
6561 struct netdev_queue *tx;
Eric Dumazet60877a32013-06-20 01:15:51 -07006562 size_t sz = count * sizeof(*tx);
Tom Herberte6484932010-10-18 18:04:39 +00006563
Eric Dumazetd3397272015-07-06 17:13:26 +02006564 if (count < 1 || count > 0xffff)
6565 return -EINVAL;
Tom Herberte6484932010-10-18 18:04:39 +00006566
Eric Dumazet60877a32013-06-20 01:15:51 -07006567 tx = kzalloc(sz, GFP_KERNEL | __GFP_NOWARN | __GFP_REPEAT);
6568 if (!tx) {
6569 tx = vzalloc(sz);
6570 if (!tx)
6571 return -ENOMEM;
6572 }
Tom Herberte6484932010-10-18 18:04:39 +00006573 dev->_tx = tx;
Tom Herbert1d24eb42010-11-21 13:17:27 +00006574
Tom Herberte6484932010-10-18 18:04:39 +00006575 netdev_for_each_tx_queue(dev, netdev_init_one_queue, NULL);
6576 spin_lock_init(&dev->tx_global_lock);
Changli Gaoaa942102010-12-04 02:31:41 +00006577
6578 return 0;
Tom Herberte6484932010-10-18 18:04:39 +00006579}
6580
Denys Vlasenkoa2029242015-05-11 21:17:53 +02006581void netif_tx_stop_all_queues(struct net_device *dev)
6582{
6583 unsigned int i;
6584
6585 for (i = 0; i < dev->num_tx_queues; i++) {
6586 struct netdev_queue *txq = netdev_get_tx_queue(dev, i);
6587 netif_tx_stop_queue(txq);
6588 }
6589}
6590EXPORT_SYMBOL(netif_tx_stop_all_queues);
6591
Patrick Mullaneyfc4a7482009-12-03 15:59:22 -08006592/**
Linus Torvalds1da177e2005-04-16 15:20:36 -07006593 * register_netdevice - register a network device
6594 * @dev: device to register
6595 *
6596 * Take a completed network device structure and add it to the kernel
6597 * interfaces. A %NETDEV_REGISTER message is sent to the netdev notifier
6598 * chain. 0 is returned on success. A negative errno code is returned
6599 * on a failure to set up the device, or if the name is a duplicate.
6600 *
6601 * Callers must hold the rtnl semaphore. You may want
6602 * register_netdev() instead of this.
6603 *
6604 * BUGS:
6605 * The locking appears insufficient to guarantee two parallel registers
6606 * will not get the same name.
6607 */
6608
6609int register_netdevice(struct net_device *dev)
6610{
Linus Torvalds1da177e2005-04-16 15:20:36 -07006611 int ret;
Stephen Hemmingerd3147742008-11-19 21:32:24 -08006612 struct net *net = dev_net(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006613
6614 BUG_ON(dev_boot_phase);
6615 ASSERT_RTNL();
6616
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07006617 might_sleep();
6618
Linus Torvalds1da177e2005-04-16 15:20:36 -07006619 /* When net_device's are persistent, this will be fatal. */
6620 BUG_ON(dev->reg_state != NETREG_UNINITIALIZED);
Stephen Hemmingerd3147742008-11-19 21:32:24 -08006621 BUG_ON(!net);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006622
David S. Millerf1f28aa2008-07-15 00:08:33 -07006623 spin_lock_init(&dev->addr_list_lock);
David S. Millercf508b12008-07-22 14:16:42 -07006624 netdev_set_addr_lockdep_class(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006625
Gao feng828de4f2012-09-13 20:58:27 +00006626 ret = dev_get_valid_name(net, dev, dev->name);
Peter Pan(潘卫平)0696c3a2011-05-12 15:46:56 +00006627 if (ret < 0)
6628 goto out;
6629
Linus Torvalds1da177e2005-04-16 15:20:36 -07006630 /* Init, if this function is available */
Stephen Hemmingerd3147742008-11-19 21:32:24 -08006631 if (dev->netdev_ops->ndo_init) {
6632 ret = dev->netdev_ops->ndo_init(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006633 if (ret) {
6634 if (ret > 0)
6635 ret = -EIO;
Adrian Bunk90833aa2006-11-13 16:02:22 -08006636 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006637 }
6638 }
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09006639
Patrick McHardyf6469682013-04-19 02:04:27 +00006640 if (((dev->hw_features | dev->features) &
6641 NETIF_F_HW_VLAN_CTAG_FILTER) &&
Michał Mirosławd2ed2732013-01-29 15:14:16 +00006642 (!dev->netdev_ops->ndo_vlan_rx_add_vid ||
6643 !dev->netdev_ops->ndo_vlan_rx_kill_vid)) {
6644 netdev_WARN(dev, "Buggy VLAN acceleration in driver!\n");
6645 ret = -EINVAL;
6646 goto err_uninit;
6647 }
6648
Pavel Emelyanov9c7dafb2012-08-08 21:52:46 +00006649 ret = -EBUSY;
6650 if (!dev->ifindex)
6651 dev->ifindex = dev_new_index(net);
6652 else if (__dev_get_by_index(net, dev->ifindex))
6653 goto err_uninit;
6654
Michał Mirosław5455c692011-02-15 16:59:17 +00006655 /* Transfer changeable features to wanted_features and enable
6656 * software offloads (GSO and GRO).
6657 */
6658 dev->hw_features |= NETIF_F_SOFT_FEATURES;
Michał Mirosław14d12322011-02-22 16:52:28 +00006659 dev->features |= NETIF_F_SOFT_FEATURES;
6660 dev->wanted_features = dev->features & dev->hw_features;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006661
Michał Mirosław34324dc2011-11-15 15:29:55 +00006662 if (!(dev->flags & IFF_LOOPBACK)) {
6663 dev->hw_features |= NETIF_F_NOCACHE_COPY;
Tom Herbertc6e1a0d2011-04-04 22:30:30 -07006664 }
6665
Michał Mirosław1180e7d2011-07-14 14:41:11 -07006666 /* Make NETIF_F_HIGHDMA inheritable to VLAN devices.
Brandon Philips16c3ea72010-09-15 09:24:24 +00006667 */
Michał Mirosław1180e7d2011-07-14 14:41:11 -07006668 dev->vlan_features |= NETIF_F_HIGHDMA;
Brandon Philips16c3ea72010-09-15 09:24:24 +00006669
Pravin B Shelaree579672013-03-07 09:28:08 +00006670 /* Make NETIF_F_SG inheritable to tunnel devices.
6671 */
6672 dev->hw_enc_features |= NETIF_F_SG;
6673
Simon Horman0d89d202013-05-23 21:02:52 +00006674 /* Make NETIF_F_SG inheritable to MPLS.
6675 */
6676 dev->mpls_features |= NETIF_F_SG;
6677
Johannes Berg7ffbe3f2009-10-02 05:15:27 +00006678 ret = call_netdevice_notifiers(NETDEV_POST_INIT, dev);
6679 ret = notifier_to_errno(ret);
6680 if (ret)
6681 goto err_uninit;
6682
Eric W. Biederman8b41d182007-09-26 22:02:53 -07006683 ret = netdev_register_kobject(dev);
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07006684 if (ret)
Herbert Xu7ce1b0e2007-07-30 16:29:40 -07006685 goto err_uninit;
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07006686 dev->reg_state = NETREG_REGISTERED;
6687
Michał Mirosław6cb6a272011-04-02 22:48:47 -07006688 __netdev_update_features(dev);
Michał Mirosław8e9b59b2011-02-22 16:52:28 +00006689
Linus Torvalds1da177e2005-04-16 15:20:36 -07006690 /*
6691 * Default initial state at registry is that the
6692 * device is present.
6693 */
6694
6695 set_bit(__LINK_STATE_PRESENT, &dev->state);
6696
Ben Hutchings8f4cccb2012-08-20 22:16:51 +01006697 linkwatch_init_dev(dev);
6698
Linus Torvalds1da177e2005-04-16 15:20:36 -07006699 dev_init_scheduler(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006700 dev_hold(dev);
Eric W. Biedermance286d32007-09-12 13:53:49 +02006701 list_netdevice(dev);
Theodore Ts'o7bf23572012-07-04 21:23:25 -04006702 add_device_randomness(dev->dev_addr, dev->addr_len);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006703
Jiri Pirko948b3372013-01-08 01:38:25 +00006704 /* If the device has permanent device address, driver should
6705 * set dev_addr and also addr_assign_type should be set to
6706 * NET_ADDR_PERM (default value).
6707 */
6708 if (dev->addr_assign_type == NET_ADDR_PERM)
6709 memcpy(dev->perm_addr, dev->dev_addr, dev->addr_len);
6710
Linus Torvalds1da177e2005-04-16 15:20:36 -07006711 /* Notify protocols, that a new device appeared. */
Pavel Emelyanov056925a2007-09-16 15:42:43 -07006712 ret = call_netdevice_notifiers(NETDEV_REGISTER, dev);
Herbert Xufcc5a032007-07-30 17:03:38 -07006713 ret = notifier_to_errno(ret);
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07006714 if (ret) {
6715 rollback_registered(dev);
6716 dev->reg_state = NETREG_UNREGISTERED;
6717 }
Eric W. Biedermand90a9092009-12-12 22:11:15 +00006718 /*
6719 * Prevent userspace races by waiting until the network
6720 * device is fully setup before sending notifications.
6721 */
Patrick McHardya2835762010-02-26 06:34:51 +00006722 if (!dev->rtnl_link_ops ||
6723 dev->rtnl_link_state == RTNL_LINK_INITIALIZED)
Alexei Starovoitov7f294052013-10-23 16:02:42 -07006724 rtmsg_ifinfo(RTM_NEWLINK, dev, ~0U, GFP_KERNEL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006725
6726out:
6727 return ret;
Herbert Xu7ce1b0e2007-07-30 16:29:40 -07006728
6729err_uninit:
Stephen Hemmingerd3147742008-11-19 21:32:24 -08006730 if (dev->netdev_ops->ndo_uninit)
6731 dev->netdev_ops->ndo_uninit(dev);
Herbert Xu7ce1b0e2007-07-30 16:29:40 -07006732 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006733}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07006734EXPORT_SYMBOL(register_netdevice);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006735
6736/**
Benjamin Herrenschmidt937f1ba2009-01-14 21:05:05 -08006737 * init_dummy_netdev - init a dummy network device for NAPI
6738 * @dev: device to init
6739 *
6740 * This takes a network device structure and initialize the minimum
6741 * amount of fields so it can be used to schedule NAPI polls without
6742 * registering a full blown interface. This is to be used by drivers
6743 * that need to tie several hardware interfaces to a single NAPI
6744 * poll scheduler due to HW limitations.
6745 */
6746int init_dummy_netdev(struct net_device *dev)
6747{
6748 /* Clear everything. Note we don't initialize spinlocks
6749 * are they aren't supposed to be taken by any of the
6750 * NAPI code and this dummy netdev is supposed to be
6751 * only ever used for NAPI polls
6752 */
6753 memset(dev, 0, sizeof(struct net_device));
6754
6755 /* make sure we BUG if trying to hit standard
6756 * register/unregister code path
6757 */
6758 dev->reg_state = NETREG_DUMMY;
6759
Benjamin Herrenschmidt937f1ba2009-01-14 21:05:05 -08006760 /* NAPI wants this */
6761 INIT_LIST_HEAD(&dev->napi_list);
6762
6763 /* a dummy interface is started by default */
6764 set_bit(__LINK_STATE_PRESENT, &dev->state);
6765 set_bit(__LINK_STATE_START, &dev->state);
6766
Eric Dumazet29b44332010-10-11 10:22:12 +00006767 /* Note : We dont allocate pcpu_refcnt for dummy devices,
6768 * because users of this 'device' dont need to change
6769 * its refcount.
6770 */
6771
Benjamin Herrenschmidt937f1ba2009-01-14 21:05:05 -08006772 return 0;
6773}
6774EXPORT_SYMBOL_GPL(init_dummy_netdev);
6775
6776
6777/**
Linus Torvalds1da177e2005-04-16 15:20:36 -07006778 * register_netdev - register a network device
6779 * @dev: device to register
6780 *
6781 * Take a completed network device structure and add it to the kernel
6782 * interfaces. A %NETDEV_REGISTER message is sent to the netdev notifier
6783 * chain. 0 is returned on success. A negative errno code is returned
6784 * on a failure to set up the device, or if the name is a duplicate.
6785 *
Borislav Petkov38b4da32007-04-20 22:14:10 -07006786 * This is a wrapper around register_netdevice that takes the rtnl semaphore
Linus Torvalds1da177e2005-04-16 15:20:36 -07006787 * and expands the device name if you passed a format string to
6788 * alloc_netdev.
6789 */
6790int register_netdev(struct net_device *dev)
6791{
6792 int err;
6793
6794 rtnl_lock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07006795 err = register_netdevice(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006796 rtnl_unlock();
6797 return err;
6798}
6799EXPORT_SYMBOL(register_netdev);
6800
Eric Dumazet29b44332010-10-11 10:22:12 +00006801int netdev_refcnt_read(const struct net_device *dev)
6802{
6803 int i, refcnt = 0;
6804
6805 for_each_possible_cpu(i)
6806 refcnt += *per_cpu_ptr(dev->pcpu_refcnt, i);
6807 return refcnt;
6808}
6809EXPORT_SYMBOL(netdev_refcnt_read);
6810
Ben Hutchings2c530402012-07-10 10:55:09 +00006811/**
Linus Torvalds1da177e2005-04-16 15:20:36 -07006812 * netdev_wait_allrefs - wait until all references are gone.
Randy Dunlap3de7a372012-08-18 14:36:44 +00006813 * @dev: target net_device
Linus Torvalds1da177e2005-04-16 15:20:36 -07006814 *
6815 * This is called when unregistering network devices.
6816 *
6817 * Any protocol or device that holds a reference should register
6818 * for netdevice notification, and cleanup and put back the
6819 * reference if they receive an UNREGISTER event.
6820 * We can get stuck here if buggy protocols don't correctly
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09006821 * call dev_put.
Linus Torvalds1da177e2005-04-16 15:20:36 -07006822 */
6823static void netdev_wait_allrefs(struct net_device *dev)
6824{
6825 unsigned long rebroadcast_time, warning_time;
Eric Dumazet29b44332010-10-11 10:22:12 +00006826 int refcnt;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006827
Eric Dumazete014deb2009-11-17 05:59:21 +00006828 linkwatch_forget_dev(dev);
6829
Linus Torvalds1da177e2005-04-16 15:20:36 -07006830 rebroadcast_time = warning_time = jiffies;
Eric Dumazet29b44332010-10-11 10:22:12 +00006831 refcnt = netdev_refcnt_read(dev);
6832
6833 while (refcnt != 0) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07006834 if (time_after(jiffies, rebroadcast_time + 1 * HZ)) {
Stephen Hemminger6756ae42006-03-20 22:23:58 -08006835 rtnl_lock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07006836
6837 /* Rebroadcast unregister notification */
Pavel Emelyanov056925a2007-09-16 15:42:43 -07006838 call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006839
Eric Dumazet748e2d92012-08-22 21:50:59 +00006840 __rtnl_unlock();
Eric Dumazet0115e8e2012-08-22 17:19:46 +00006841 rcu_barrier();
Eric Dumazet748e2d92012-08-22 21:50:59 +00006842 rtnl_lock();
6843
Eric Dumazet0115e8e2012-08-22 17:19:46 +00006844 call_netdevice_notifiers(NETDEV_UNREGISTER_FINAL, dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006845 if (test_bit(__LINK_STATE_LINKWATCH_PENDING,
6846 &dev->state)) {
6847 /* We must not have linkwatch events
6848 * pending on unregister. If this
6849 * happens, we simply run the queue
6850 * unscheduled, resulting in a noop
6851 * for this device.
6852 */
6853 linkwatch_run_queue();
6854 }
6855
Stephen Hemminger6756ae42006-03-20 22:23:58 -08006856 __rtnl_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07006857
6858 rebroadcast_time = jiffies;
6859 }
6860
6861 msleep(250);
6862
Eric Dumazet29b44332010-10-11 10:22:12 +00006863 refcnt = netdev_refcnt_read(dev);
6864
Linus Torvalds1da177e2005-04-16 15:20:36 -07006865 if (time_after(jiffies, warning_time + 10 * HZ)) {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00006866 pr_emerg("unregister_netdevice: waiting for %s to become free. Usage count = %d\n",
6867 dev->name, refcnt);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006868 warning_time = jiffies;
6869 }
6870 }
6871}
6872
6873/* The sequence is:
6874 *
6875 * rtnl_lock();
6876 * ...
6877 * register_netdevice(x1);
6878 * register_netdevice(x2);
6879 * ...
6880 * unregister_netdevice(y1);
6881 * unregister_netdevice(y2);
6882 * ...
6883 * rtnl_unlock();
6884 * free_netdev(y1);
6885 * free_netdev(y2);
6886 *
Herbert Xu58ec3b42008-10-07 15:50:03 -07006887 * We are invoked by rtnl_unlock().
Linus Torvalds1da177e2005-04-16 15:20:36 -07006888 * This allows us to deal with problems:
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07006889 * 1) We can delete sysfs objects which invoke hotplug
Linus Torvalds1da177e2005-04-16 15:20:36 -07006890 * without deadlocking with linkwatch via keventd.
6891 * 2) Since we run with the RTNL semaphore not held, we can sleep
6892 * safely in order to wait for the netdev refcnt to drop to zero.
Herbert Xu58ec3b42008-10-07 15:50:03 -07006893 *
6894 * We must not return until all unregister events added during
6895 * the interval the lock was held have been completed.
Linus Torvalds1da177e2005-04-16 15:20:36 -07006896 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07006897void netdev_run_todo(void)
6898{
Oleg Nesterov626ab0e2006-06-23 02:05:55 -07006899 struct list_head list;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006900
Linus Torvalds1da177e2005-04-16 15:20:36 -07006901 /* Snapshot list, allow later requests */
Oleg Nesterov626ab0e2006-06-23 02:05:55 -07006902 list_replace_init(&net_todo_list, &list);
Herbert Xu58ec3b42008-10-07 15:50:03 -07006903
6904 __rtnl_unlock();
Oleg Nesterov626ab0e2006-06-23 02:05:55 -07006905
Eric Dumazet0115e8e2012-08-22 17:19:46 +00006906
6907 /* Wait for rcu callbacks to finish before next phase */
Eric W. Biederman850a5452011-10-13 22:25:23 +00006908 if (!list_empty(&list))
6909 rcu_barrier();
6910
Linus Torvalds1da177e2005-04-16 15:20:36 -07006911 while (!list_empty(&list)) {
6912 struct net_device *dev
stephen hemmingere5e26d72010-02-24 14:01:38 +00006913 = list_first_entry(&list, struct net_device, todo_list);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006914 list_del(&dev->todo_list);
6915
Eric Dumazet748e2d92012-08-22 21:50:59 +00006916 rtnl_lock();
Eric Dumazet0115e8e2012-08-22 17:19:46 +00006917 call_netdevice_notifiers(NETDEV_UNREGISTER_FINAL, dev);
Eric Dumazet748e2d92012-08-22 21:50:59 +00006918 __rtnl_unlock();
Eric Dumazet0115e8e2012-08-22 17:19:46 +00006919
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07006920 if (unlikely(dev->reg_state != NETREG_UNREGISTERING)) {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00006921 pr_err("network todo '%s' but state %d\n",
Linus Torvalds1da177e2005-04-16 15:20:36 -07006922 dev->name, dev->reg_state);
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07006923 dump_stack();
6924 continue;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006925 }
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07006926
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07006927 dev->reg_state = NETREG_UNREGISTERED;
6928
6929 netdev_wait_allrefs(dev);
6930
6931 /* paranoia */
Eric Dumazet29b44332010-10-11 10:22:12 +00006932 BUG_ON(netdev_refcnt_read(dev));
Salam Noureddine7866a622015-01-27 11:35:48 -08006933 BUG_ON(!list_empty(&dev->ptype_all));
6934 BUG_ON(!list_empty(&dev->ptype_specific));
Eric Dumazet33d480c2011-08-11 19:30:52 +00006935 WARN_ON(rcu_access_pointer(dev->ip_ptr));
6936 WARN_ON(rcu_access_pointer(dev->ip6_ptr));
Ilpo Järvinen547b7922008-07-25 21:43:18 -07006937 WARN_ON(dev->dn_ptr);
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07006938
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07006939 if (dev->destructor)
6940 dev->destructor(dev);
Stephen Hemminger9093bbb2007-05-19 15:39:25 -07006941
Eric W. Biederman50624c92013-09-23 21:19:49 -07006942 /* Report a network device has been unregistered */
6943 rtnl_lock();
6944 dev_net(dev)->dev_unreg_count--;
6945 __rtnl_unlock();
6946 wake_up(&netdev_unregistering_wq);
6947
Stephen Hemminger9093bbb2007-05-19 15:39:25 -07006948 /* Free network device */
6949 kobject_put(&dev->dev.kobj);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006950 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07006951}
6952
Ben Hutchings3cfde792010-07-09 09:11:52 +00006953/* Convert net_device_stats to rtnl_link_stats64. They have the same
6954 * fields in the same order, with only the type differing.
6955 */
Eric Dumazet77a1abf2012-03-05 04:50:09 +00006956void netdev_stats_to_stats64(struct rtnl_link_stats64 *stats64,
6957 const struct net_device_stats *netdev_stats)
Ben Hutchings3cfde792010-07-09 09:11:52 +00006958{
6959#if BITS_PER_LONG == 64
Eric Dumazet77a1abf2012-03-05 04:50:09 +00006960 BUILD_BUG_ON(sizeof(*stats64) != sizeof(*netdev_stats));
6961 memcpy(stats64, netdev_stats, sizeof(*stats64));
Ben Hutchings3cfde792010-07-09 09:11:52 +00006962#else
6963 size_t i, n = sizeof(*stats64) / sizeof(u64);
6964 const unsigned long *src = (const unsigned long *)netdev_stats;
6965 u64 *dst = (u64 *)stats64;
6966
6967 BUILD_BUG_ON(sizeof(*netdev_stats) / sizeof(unsigned long) !=
6968 sizeof(*stats64) / sizeof(u64));
6969 for (i = 0; i < n; i++)
6970 dst[i] = src[i];
6971#endif
6972}
Eric Dumazet77a1abf2012-03-05 04:50:09 +00006973EXPORT_SYMBOL(netdev_stats_to_stats64);
Ben Hutchings3cfde792010-07-09 09:11:52 +00006974
Eric Dumazetd83345a2009-11-16 03:36:51 +00006975/**
Stephen Hemmingereeda3fd2008-11-19 21:40:23 -08006976 * dev_get_stats - get network device statistics
6977 * @dev: device to get statistics from
Eric Dumazet28172732010-07-07 14:58:56 -07006978 * @storage: place to store stats
Stephen Hemmingereeda3fd2008-11-19 21:40:23 -08006979 *
Ben Hutchingsd7753512010-07-09 09:12:41 +00006980 * Get network statistics from device. Return @storage.
6981 * The device driver may provide its own method by setting
6982 * dev->netdev_ops->get_stats64 or dev->netdev_ops->get_stats;
6983 * otherwise the internal statistics structure is used.
Stephen Hemmingereeda3fd2008-11-19 21:40:23 -08006984 */
Ben Hutchingsd7753512010-07-09 09:12:41 +00006985struct rtnl_link_stats64 *dev_get_stats(struct net_device *dev,
6986 struct rtnl_link_stats64 *storage)
Eric Dumazet7004bf22009-05-18 00:34:33 +00006987{
Stephen Hemmingereeda3fd2008-11-19 21:40:23 -08006988 const struct net_device_ops *ops = dev->netdev_ops;
6989
Eric Dumazet28172732010-07-07 14:58:56 -07006990 if (ops->ndo_get_stats64) {
6991 memset(storage, 0, sizeof(*storage));
Eric Dumazetcaf586e2010-09-30 21:06:55 +00006992 ops->ndo_get_stats64(dev, storage);
6993 } else if (ops->ndo_get_stats) {
Ben Hutchings3cfde792010-07-09 09:11:52 +00006994 netdev_stats_to_stats64(storage, ops->ndo_get_stats(dev));
Eric Dumazetcaf586e2010-09-30 21:06:55 +00006995 } else {
6996 netdev_stats_to_stats64(storage, &dev->stats);
Eric Dumazet28172732010-07-07 14:58:56 -07006997 }
Eric Dumazetcaf586e2010-09-30 21:06:55 +00006998 storage->rx_dropped += atomic_long_read(&dev->rx_dropped);
Eric Dumazet015f0682014-03-27 08:45:56 -07006999 storage->tx_dropped += atomic_long_read(&dev->tx_dropped);
Eric Dumazet28172732010-07-07 14:58:56 -07007000 return storage;
Rusty Russellc45d2862007-03-28 14:29:08 -07007001}
Stephen Hemmingereeda3fd2008-11-19 21:40:23 -08007002EXPORT_SYMBOL(dev_get_stats);
Rusty Russellc45d2862007-03-28 14:29:08 -07007003
Eric Dumazet24824a02010-10-02 06:11:55 +00007004struct netdev_queue *dev_ingress_queue_create(struct net_device *dev)
David S. Millerdc2b4842008-07-08 17:18:23 -07007005{
Eric Dumazet24824a02010-10-02 06:11:55 +00007006 struct netdev_queue *queue = dev_ingress_queue(dev);
David S. Millerdc2b4842008-07-08 17:18:23 -07007007
Eric Dumazet24824a02010-10-02 06:11:55 +00007008#ifdef CONFIG_NET_CLS_ACT
7009 if (queue)
7010 return queue;
7011 queue = kzalloc(sizeof(*queue), GFP_KERNEL);
7012 if (!queue)
7013 return NULL;
7014 netdev_init_one_queue(dev, queue, NULL);
Eric Dumazet2ce1ee12015-02-04 13:37:44 -08007015 RCU_INIT_POINTER(queue->qdisc, &noop_qdisc);
Eric Dumazet24824a02010-10-02 06:11:55 +00007016 queue->qdisc_sleeping = &noop_qdisc;
7017 rcu_assign_pointer(dev->ingress_queue, queue);
7018#endif
7019 return queue;
David S. Millerbb949fb2008-07-08 16:55:56 -07007020}
7021
Eric Dumazet2c60db02012-09-16 09:17:26 +00007022static const struct ethtool_ops default_ethtool_ops;
7023
Stanislaw Gruszkad07d7502013-01-10 23:19:10 +00007024void netdev_set_default_ethtool_ops(struct net_device *dev,
7025 const struct ethtool_ops *ops)
7026{
7027 if (dev->ethtool_ops == &default_ethtool_ops)
7028 dev->ethtool_ops = ops;
7029}
7030EXPORT_SYMBOL_GPL(netdev_set_default_ethtool_ops);
7031
Eric Dumazet74d332c2013-10-30 13:10:44 -07007032void netdev_freemem(struct net_device *dev)
7033{
7034 char *addr = (char *)dev - dev->padded;
7035
WANG Cong4cb28972014-06-02 15:55:22 -07007036 kvfree(addr);
Eric Dumazet74d332c2013-10-30 13:10:44 -07007037}
7038
Linus Torvalds1da177e2005-04-16 15:20:36 -07007039/**
Tom Herbert36909ea2011-01-09 19:36:31 +00007040 * alloc_netdev_mqs - allocate network device
Tom Gundersenc835a672014-07-14 16:37:24 +02007041 * @sizeof_priv: size of private data to allocate space for
7042 * @name: device name format string
7043 * @name_assign_type: origin of device name
7044 * @setup: callback to initialize device
7045 * @txqs: the number of TX subqueues to allocate
7046 * @rxqs: the number of RX subqueues to allocate
Linus Torvalds1da177e2005-04-16 15:20:36 -07007047 *
7048 * Allocates a struct net_device with private data area for driver use
Li Zhong90e51ad2013-11-22 15:04:46 +08007049 * and performs basic initialization. Also allocates subqueue structs
Tom Herbert36909ea2011-01-09 19:36:31 +00007050 * for each queue on the device.
Linus Torvalds1da177e2005-04-16 15:20:36 -07007051 */
Tom Herbert36909ea2011-01-09 19:36:31 +00007052struct net_device *alloc_netdev_mqs(int sizeof_priv, const char *name,
Tom Gundersenc835a672014-07-14 16:37:24 +02007053 unsigned char name_assign_type,
Tom Herbert36909ea2011-01-09 19:36:31 +00007054 void (*setup)(struct net_device *),
7055 unsigned int txqs, unsigned int rxqs)
Linus Torvalds1da177e2005-04-16 15:20:36 -07007056{
Linus Torvalds1da177e2005-04-16 15:20:36 -07007057 struct net_device *dev;
Stephen Hemminger79439862008-07-21 13:28:44 -07007058 size_t alloc_size;
Eric Dumazet1ce8e7b2009-05-27 04:42:37 +00007059 struct net_device *p;
Linus Torvalds1da177e2005-04-16 15:20:36 -07007060
Stephen Hemmingerb6fe17d2006-08-29 17:06:13 -07007061 BUG_ON(strlen(name) >= sizeof(dev->name));
7062
Tom Herbert36909ea2011-01-09 19:36:31 +00007063 if (txqs < 1) {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00007064 pr_err("alloc_netdev: Unable to allocate device with zero queues\n");
Tom Herbert55513fb2010-10-18 17:55:58 +00007065 return NULL;
7066 }
7067
Michael Daltona953be52014-01-16 22:23:28 -08007068#ifdef CONFIG_SYSFS
Tom Herbert36909ea2011-01-09 19:36:31 +00007069 if (rxqs < 1) {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00007070 pr_err("alloc_netdev: Unable to allocate device with zero RX queues\n");
Tom Herbert36909ea2011-01-09 19:36:31 +00007071 return NULL;
7072 }
7073#endif
7074
David S. Millerfd2ea0a2008-07-17 01:56:23 -07007075 alloc_size = sizeof(struct net_device);
Alexey Dobriyand1643d22008-04-18 15:43:32 -07007076 if (sizeof_priv) {
7077 /* ensure 32-byte alignment of private area */
Eric Dumazet1ce8e7b2009-05-27 04:42:37 +00007078 alloc_size = ALIGN(alloc_size, NETDEV_ALIGN);
Alexey Dobriyand1643d22008-04-18 15:43:32 -07007079 alloc_size += sizeof_priv;
7080 }
7081 /* ensure 32-byte alignment of whole construct */
Eric Dumazet1ce8e7b2009-05-27 04:42:37 +00007082 alloc_size += NETDEV_ALIGN - 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -07007083
Eric Dumazet74d332c2013-10-30 13:10:44 -07007084 p = kzalloc(alloc_size, GFP_KERNEL | __GFP_NOWARN | __GFP_REPEAT);
7085 if (!p)
7086 p = vzalloc(alloc_size);
Joe Perches62b59422013-02-04 16:48:16 +00007087 if (!p)
Linus Torvalds1da177e2005-04-16 15:20:36 -07007088 return NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07007089
Eric Dumazet1ce8e7b2009-05-27 04:42:37 +00007090 dev = PTR_ALIGN(p, NETDEV_ALIGN);
Linus Torvalds1da177e2005-04-16 15:20:36 -07007091 dev->padded = (char *)dev - (char *)p;
Jiri Pirkoab9c73c2009-05-08 13:30:17 +00007092
Eric Dumazet29b44332010-10-11 10:22:12 +00007093 dev->pcpu_refcnt = alloc_percpu(int);
7094 if (!dev->pcpu_refcnt)
Eric Dumazet74d332c2013-10-30 13:10:44 -07007095 goto free_dev;
Jiri Pirkoab9c73c2009-05-08 13:30:17 +00007096
Linus Torvalds1da177e2005-04-16 15:20:36 -07007097 if (dev_addr_init(dev))
Eric Dumazet29b44332010-10-11 10:22:12 +00007098 goto free_pcpu;
Linus Torvalds1da177e2005-04-16 15:20:36 -07007099
Jiri Pirko22bedad32010-04-01 21:22:57 +00007100 dev_mc_init(dev);
Jiri Pirkoa748ee22010-04-01 21:22:09 +00007101 dev_uc_init(dev);
Jiri Pirkoccffad252009-05-22 23:22:17 +00007102
YOSHIFUJI Hideakic346dca2008-03-25 21:47:49 +09007103 dev_net_set(dev, &init_net);
Linus Torvalds1da177e2005-04-16 15:20:36 -07007104
Peter P Waskiewicz Jr82cc1a72008-03-21 03:43:19 -07007105 dev->gso_max_size = GSO_MAX_SIZE;
Ben Hutchings30b678d2012-07-30 15:57:00 +00007106 dev->gso_max_segs = GSO_MAX_SEGS;
Eric Dumazetfcbeb972014-10-05 10:11:27 -07007107 dev->gso_min_segs = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07007108
Herbert Xud565b0a2008-12-15 23:38:52 -08007109 INIT_LIST_HEAD(&dev->napi_list);
Eric W. Biederman9fdce092009-10-30 14:51:13 +00007110 INIT_LIST_HEAD(&dev->unreg_list);
Eric W. Biederman5cde2822013-10-05 19:26:05 -07007111 INIT_LIST_HEAD(&dev->close_list);
Eric Dumazete014deb2009-11-17 05:59:21 +00007112 INIT_LIST_HEAD(&dev->link_watch_list);
Veaceslav Falico2f268f12013-09-25 09:20:07 +02007113 INIT_LIST_HEAD(&dev->adj_list.upper);
7114 INIT_LIST_HEAD(&dev->adj_list.lower);
7115 INIT_LIST_HEAD(&dev->all_adj_list.upper);
7116 INIT_LIST_HEAD(&dev->all_adj_list.lower);
Salam Noureddine7866a622015-01-27 11:35:48 -08007117 INIT_LIST_HEAD(&dev->ptype_all);
7118 INIT_LIST_HEAD(&dev->ptype_specific);
Eric Dumazet02875872014-10-05 18:38:35 -07007119 dev->priv_flags = IFF_XMIT_DST_RELEASE | IFF_XMIT_DST_RELEASE_PERM;
Linus Torvalds1da177e2005-04-16 15:20:36 -07007120 setup(dev);
David S. Miller8d3bdbd2011-02-08 15:02:50 -08007121
Phil Sutter906470c2015-08-18 10:30:48 +02007122 if (!dev->tx_queue_len)
Phil Sutterf84bb1e2015-08-27 21:21:36 +02007123 dev->priv_flags |= IFF_NO_QUEUE;
Phil Sutter906470c2015-08-18 10:30:48 +02007124
David S. Miller8d3bdbd2011-02-08 15:02:50 -08007125 dev->num_tx_queues = txqs;
7126 dev->real_num_tx_queues = txqs;
7127 if (netif_alloc_netdev_queues(dev))
7128 goto free_all;
7129
Michael Daltona953be52014-01-16 22:23:28 -08007130#ifdef CONFIG_SYSFS
David S. Miller8d3bdbd2011-02-08 15:02:50 -08007131 dev->num_rx_queues = rxqs;
7132 dev->real_num_rx_queues = rxqs;
7133 if (netif_alloc_rx_queues(dev))
7134 goto free_all;
7135#endif
7136
Linus Torvalds1da177e2005-04-16 15:20:36 -07007137 strcpy(dev->name, name);
Tom Gundersenc835a672014-07-14 16:37:24 +02007138 dev->name_assign_type = name_assign_type;
Vlad Dogarucbda10f2011-01-13 23:38:30 +00007139 dev->group = INIT_NETDEV_GROUP;
Eric Dumazet2c60db02012-09-16 09:17:26 +00007140 if (!dev->ethtool_ops)
7141 dev->ethtool_ops = &default_ethtool_ops;
Pablo Neirae687ad62015-05-13 18:19:38 +02007142
7143 nf_hook_ingress_init(dev);
7144
Linus Torvalds1da177e2005-04-16 15:20:36 -07007145 return dev;
Jiri Pirkoab9c73c2009-05-08 13:30:17 +00007146
David S. Miller8d3bdbd2011-02-08 15:02:50 -08007147free_all:
7148 free_netdev(dev);
7149 return NULL;
7150
Eric Dumazet29b44332010-10-11 10:22:12 +00007151free_pcpu:
7152 free_percpu(dev->pcpu_refcnt);
Eric Dumazet74d332c2013-10-30 13:10:44 -07007153free_dev:
7154 netdev_freemem(dev);
Jiri Pirkoab9c73c2009-05-08 13:30:17 +00007155 return NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07007156}
Tom Herbert36909ea2011-01-09 19:36:31 +00007157EXPORT_SYMBOL(alloc_netdev_mqs);
Linus Torvalds1da177e2005-04-16 15:20:36 -07007158
7159/**
7160 * free_netdev - free network device
7161 * @dev: device
7162 *
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09007163 * This function does the last stage of destroying an allocated device
7164 * interface. The reference to the device object is released.
Linus Torvalds1da177e2005-04-16 15:20:36 -07007165 * If this is the last reference then it will be freed.
7166 */
7167void free_netdev(struct net_device *dev)
7168{
Herbert Xud565b0a2008-12-15 23:38:52 -08007169 struct napi_struct *p, *n;
7170
Eric Dumazet60877a32013-06-20 01:15:51 -07007171 netif_free_tx_queues(dev);
Michael Daltona953be52014-01-16 22:23:28 -08007172#ifdef CONFIG_SYSFS
Pankaj Gupta10595902015-01-12 11:41:28 +05307173 kvfree(dev->_rx);
Tom Herbertfe822242010-11-09 10:47:38 +00007174#endif
David S. Millere8a04642008-07-17 00:34:19 -07007175
Eric Dumazet33d480c2011-08-11 19:30:52 +00007176 kfree(rcu_dereference_protected(dev->ingress_queue, 1));
Eric Dumazet24824a02010-10-02 06:11:55 +00007177
Jiri Pirkof001fde2009-05-05 02:48:28 +00007178 /* Flush device addresses */
7179 dev_addr_flush(dev);
7180
Herbert Xud565b0a2008-12-15 23:38:52 -08007181 list_for_each_entry_safe(p, n, &dev->napi_list, dev_list)
7182 netif_napi_del(p);
7183
Eric Dumazet29b44332010-10-11 10:22:12 +00007184 free_percpu(dev->pcpu_refcnt);
7185 dev->pcpu_refcnt = NULL;
7186
Stephen Hemminger3041a062006-05-26 13:25:24 -07007187 /* Compatibility with error handling in drivers */
Linus Torvalds1da177e2005-04-16 15:20:36 -07007188 if (dev->reg_state == NETREG_UNINITIALIZED) {
Eric Dumazet74d332c2013-10-30 13:10:44 -07007189 netdev_freemem(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07007190 return;
7191 }
7192
7193 BUG_ON(dev->reg_state != NETREG_UNREGISTERED);
7194 dev->reg_state = NETREG_RELEASED;
7195
Greg Kroah-Hartman43cb76d2002-04-09 12:14:34 -07007196 /* will free via device release */
7197 put_device(&dev->dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07007198}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07007199EXPORT_SYMBOL(free_netdev);
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09007200
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07007201/**
7202 * synchronize_net - Synchronize with packet receive processing
7203 *
7204 * Wait for packets currently being received to be done.
7205 * Does not block later packets from starting.
7206 */
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09007207void synchronize_net(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -07007208{
7209 might_sleep();
Eric Dumazetbe3fc412011-05-23 23:07:32 +00007210 if (rtnl_is_locked())
7211 synchronize_rcu_expedited();
7212 else
7213 synchronize_rcu();
Linus Torvalds1da177e2005-04-16 15:20:36 -07007214}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07007215EXPORT_SYMBOL(synchronize_net);
Linus Torvalds1da177e2005-04-16 15:20:36 -07007216
7217/**
Eric Dumazet44a08732009-10-27 07:03:04 +00007218 * unregister_netdevice_queue - remove device from the kernel
Linus Torvalds1da177e2005-04-16 15:20:36 -07007219 * @dev: device
Eric Dumazet44a08732009-10-27 07:03:04 +00007220 * @head: list
Jaswinder Singh Rajput6ebfbc02009-11-22 20:43:13 -08007221 *
Linus Torvalds1da177e2005-04-16 15:20:36 -07007222 * This function shuts down a device interface and removes it
Wang Chend59b54b2007-12-11 02:28:03 -08007223 * from the kernel tables.
Eric Dumazet44a08732009-10-27 07:03:04 +00007224 * If head not NULL, device is queued to be unregistered later.
Linus Torvalds1da177e2005-04-16 15:20:36 -07007225 *
7226 * Callers must hold the rtnl semaphore. You may want
7227 * unregister_netdev() instead of this.
7228 */
7229
Eric Dumazet44a08732009-10-27 07:03:04 +00007230void unregister_netdevice_queue(struct net_device *dev, struct list_head *head)
Linus Torvalds1da177e2005-04-16 15:20:36 -07007231{
Herbert Xua6620712007-12-12 19:21:56 -08007232 ASSERT_RTNL();
7233
Eric Dumazet44a08732009-10-27 07:03:04 +00007234 if (head) {
Eric W. Biederman9fdce092009-10-30 14:51:13 +00007235 list_move_tail(&dev->unreg_list, head);
Eric Dumazet44a08732009-10-27 07:03:04 +00007236 } else {
7237 rollback_registered(dev);
7238 /* Finish processing unregister after unlock */
7239 net_set_todo(dev);
7240 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07007241}
Eric Dumazet44a08732009-10-27 07:03:04 +00007242EXPORT_SYMBOL(unregister_netdevice_queue);
Linus Torvalds1da177e2005-04-16 15:20:36 -07007243
7244/**
Eric Dumazet9b5e3832009-10-27 07:04:19 +00007245 * unregister_netdevice_many - unregister many devices
7246 * @head: list of devices
Eric Dumazet87757a92014-06-06 06:44:03 -07007247 *
7248 * Note: As most callers use a stack allocated list_head,
7249 * we force a list_del() to make sure stack wont be corrupted later.
Eric Dumazet9b5e3832009-10-27 07:04:19 +00007250 */
7251void unregister_netdevice_many(struct list_head *head)
7252{
7253 struct net_device *dev;
7254
7255 if (!list_empty(head)) {
7256 rollback_registered_many(head);
7257 list_for_each_entry(dev, head, unreg_list)
7258 net_set_todo(dev);
Eric Dumazet87757a92014-06-06 06:44:03 -07007259 list_del(head);
Eric Dumazet9b5e3832009-10-27 07:04:19 +00007260 }
7261}
Eric Dumazet63c80992009-10-27 07:06:49 +00007262EXPORT_SYMBOL(unregister_netdevice_many);
Eric Dumazet9b5e3832009-10-27 07:04:19 +00007263
7264/**
Linus Torvalds1da177e2005-04-16 15:20:36 -07007265 * unregister_netdev - remove device from the kernel
7266 * @dev: device
7267 *
7268 * This function shuts down a device interface and removes it
Wang Chend59b54b2007-12-11 02:28:03 -08007269 * from the kernel tables.
Linus Torvalds1da177e2005-04-16 15:20:36 -07007270 *
7271 * This is just a wrapper for unregister_netdevice that takes
7272 * the rtnl semaphore. In general you want to use this and not
7273 * unregister_netdevice.
7274 */
7275void unregister_netdev(struct net_device *dev)
7276{
7277 rtnl_lock();
7278 unregister_netdevice(dev);
7279 rtnl_unlock();
7280}
Linus Torvalds1da177e2005-04-16 15:20:36 -07007281EXPORT_SYMBOL(unregister_netdev);
7282
Eric W. Biedermance286d32007-09-12 13:53:49 +02007283/**
7284 * dev_change_net_namespace - move device to different nethost namespace
7285 * @dev: device
7286 * @net: network namespace
7287 * @pat: If not NULL name pattern to try if the current device name
7288 * is already taken in the destination network namespace.
7289 *
7290 * This function shuts down a device interface and moves it
7291 * to a new network namespace. On success 0 is returned, on
7292 * a failure a netagive errno code is returned.
7293 *
7294 * Callers must hold the rtnl semaphore.
7295 */
7296
7297int dev_change_net_namespace(struct net_device *dev, struct net *net, const char *pat)
7298{
Eric W. Biedermance286d32007-09-12 13:53:49 +02007299 int err;
7300
7301 ASSERT_RTNL();
7302
7303 /* Don't allow namespace local devices to be moved. */
7304 err = -EINVAL;
7305 if (dev->features & NETIF_F_NETNS_LOCAL)
7306 goto out;
7307
7308 /* Ensure the device has been registrered */
Eric W. Biedermance286d32007-09-12 13:53:49 +02007309 if (dev->reg_state != NETREG_REGISTERED)
7310 goto out;
7311
7312 /* Get out if there is nothing todo */
7313 err = 0;
YOSHIFUJI Hideaki878628f2008-03-26 03:57:35 +09007314 if (net_eq(dev_net(dev), net))
Eric W. Biedermance286d32007-09-12 13:53:49 +02007315 goto out;
7316
7317 /* Pick the destination device name, and ensure
7318 * we can use it in the destination network namespace.
7319 */
7320 err = -EEXIST;
Octavian Purdilad9031022009-11-18 02:36:59 +00007321 if (__dev_get_by_name(net, dev->name)) {
Eric W. Biedermance286d32007-09-12 13:53:49 +02007322 /* We get here if we can't use the current device name */
7323 if (!pat)
7324 goto out;
Gao feng828de4f2012-09-13 20:58:27 +00007325 if (dev_get_valid_name(net, dev, pat) < 0)
Eric W. Biedermance286d32007-09-12 13:53:49 +02007326 goto out;
7327 }
7328
7329 /*
7330 * And now a mini version of register_netdevice unregister_netdevice.
7331 */
7332
7333 /* If device is running close it first. */
Pavel Emelyanov9b772652007-10-10 02:49:09 -07007334 dev_close(dev);
Eric W. Biedermance286d32007-09-12 13:53:49 +02007335
7336 /* And unlink it from device chain */
7337 err = -ENODEV;
7338 unlist_netdevice(dev);
7339
7340 synchronize_net();
7341
7342 /* Shutdown queueing discipline. */
7343 dev_shutdown(dev);
7344
7345 /* Notify protocols, that we are about to destroy
7346 this device. They should clean all the things.
David Lamparter3b27e102010-09-17 03:22:19 +00007347
7348 Note that dev->reg_state stays at NETREG_REGISTERED.
7349 This is wanted because this way 8021q and macvlan know
7350 the device is just moving and can keep their slaves up.
Eric W. Biedermance286d32007-09-12 13:53:49 +02007351 */
7352 call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
Gao feng6549dd42012-08-23 15:36:55 +00007353 rcu_barrier();
7354 call_netdevice_notifiers(NETDEV_UNREGISTER_FINAL, dev);
Alexei Starovoitov7f294052013-10-23 16:02:42 -07007355 rtmsg_ifinfo(RTM_DELLINK, dev, ~0U, GFP_KERNEL);
Eric W. Biedermance286d32007-09-12 13:53:49 +02007356
7357 /*
7358 * Flush the unicast and multicast chains
7359 */
Jiri Pirkoa748ee22010-04-01 21:22:09 +00007360 dev_uc_flush(dev);
Jiri Pirko22bedad32010-04-01 21:22:57 +00007361 dev_mc_flush(dev);
Eric W. Biedermance286d32007-09-12 13:53:49 +02007362
Serge Hallyn4e66ae22012-12-03 16:17:12 +00007363 /* Send a netdev-removed uevent to the old namespace */
7364 kobject_uevent(&dev->dev.kobj, KOBJ_REMOVE);
Alexander Y. Fomichev4c754312014-08-25 16:26:45 +04007365 netdev_adjacent_del_links(dev);
Serge Hallyn4e66ae22012-12-03 16:17:12 +00007366
Eric W. Biedermance286d32007-09-12 13:53:49 +02007367 /* Actually switch the network namespace */
YOSHIFUJI Hideakic346dca2008-03-25 21:47:49 +09007368 dev_net_set(dev, net);
Eric W. Biedermance286d32007-09-12 13:53:49 +02007369
Eric W. Biedermance286d32007-09-12 13:53:49 +02007370 /* If there is an ifindex conflict assign a new one */
Nicolas Dichtel7a66bbc2015-04-02 17:07:09 +02007371 if (__dev_get_by_index(net, dev->ifindex))
Eric W. Biedermance286d32007-09-12 13:53:49 +02007372 dev->ifindex = dev_new_index(net);
Eric W. Biedermance286d32007-09-12 13:53:49 +02007373
Serge Hallyn4e66ae22012-12-03 16:17:12 +00007374 /* Send a netdev-add uevent to the new namespace */
7375 kobject_uevent(&dev->dev.kobj, KOBJ_ADD);
Alexander Y. Fomichev4c754312014-08-25 16:26:45 +04007376 netdev_adjacent_add_links(dev);
Serge Hallyn4e66ae22012-12-03 16:17:12 +00007377
Eric W. Biederman8b41d182007-09-26 22:02:53 -07007378 /* Fixup kobjects */
Eric W. Biedermana1b3f592010-05-04 17:36:49 -07007379 err = device_rename(&dev->dev, dev->name);
Eric W. Biederman8b41d182007-09-26 22:02:53 -07007380 WARN_ON(err);
Eric W. Biedermance286d32007-09-12 13:53:49 +02007381
7382 /* Add the device back in the hashes */
7383 list_netdevice(dev);
7384
7385 /* Notify protocols, that a new device appeared. */
7386 call_netdevice_notifiers(NETDEV_REGISTER, dev);
7387
Eric W. Biedermand90a9092009-12-12 22:11:15 +00007388 /*
7389 * Prevent userspace races by waiting until the network
7390 * device is fully setup before sending notifications.
7391 */
Alexei Starovoitov7f294052013-10-23 16:02:42 -07007392 rtmsg_ifinfo(RTM_NEWLINK, dev, ~0U, GFP_KERNEL);
Eric W. Biedermand90a9092009-12-12 22:11:15 +00007393
Eric W. Biedermance286d32007-09-12 13:53:49 +02007394 synchronize_net();
7395 err = 0;
7396out:
7397 return err;
7398}
Johannes Berg463d0182009-07-14 00:33:35 +02007399EXPORT_SYMBOL_GPL(dev_change_net_namespace);
Eric W. Biedermance286d32007-09-12 13:53:49 +02007400
Linus Torvalds1da177e2005-04-16 15:20:36 -07007401static int dev_cpu_callback(struct notifier_block *nfb,
7402 unsigned long action,
7403 void *ocpu)
7404{
7405 struct sk_buff **list_skb;
Linus Torvalds1da177e2005-04-16 15:20:36 -07007406 struct sk_buff *skb;
7407 unsigned int cpu, oldcpu = (unsigned long)ocpu;
7408 struct softnet_data *sd, *oldsd;
7409
Rafael J. Wysocki8bb78442007-05-09 02:35:10 -07007410 if (action != CPU_DEAD && action != CPU_DEAD_FROZEN)
Linus Torvalds1da177e2005-04-16 15:20:36 -07007411 return NOTIFY_OK;
7412
7413 local_irq_disable();
7414 cpu = smp_processor_id();
7415 sd = &per_cpu(softnet_data, cpu);
7416 oldsd = &per_cpu(softnet_data, oldcpu);
7417
7418 /* Find end of our completion_queue. */
7419 list_skb = &sd->completion_queue;
7420 while (*list_skb)
7421 list_skb = &(*list_skb)->next;
7422 /* Append completion queue from offline CPU. */
7423 *list_skb = oldsd->completion_queue;
7424 oldsd->completion_queue = NULL;
7425
Linus Torvalds1da177e2005-04-16 15:20:36 -07007426 /* Append output queue from offline CPU. */
Changli Gaoa9cbd582010-04-26 23:06:24 +00007427 if (oldsd->output_queue) {
7428 *sd->output_queue_tailp = oldsd->output_queue;
7429 sd->output_queue_tailp = oldsd->output_queue_tailp;
7430 oldsd->output_queue = NULL;
7431 oldsd->output_queue_tailp = &oldsd->output_queue;
7432 }
Eric Dumazetac64da02015-01-15 17:04:22 -08007433 /* Append NAPI poll list from offline CPU, with one exception :
7434 * process_backlog() must be called by cpu owning percpu backlog.
7435 * We properly handle process_queue & input_pkt_queue later.
7436 */
7437 while (!list_empty(&oldsd->poll_list)) {
7438 struct napi_struct *napi = list_first_entry(&oldsd->poll_list,
7439 struct napi_struct,
7440 poll_list);
7441
7442 list_del_init(&napi->poll_list);
7443 if (napi->poll == process_backlog)
7444 napi->state = 0;
7445 else
7446 ____napi_schedule(sd, napi);
Heiko Carstens264524d2011-06-06 20:50:03 +00007447 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07007448
7449 raise_softirq_irqoff(NET_TX_SOFTIRQ);
7450 local_irq_enable();
7451
7452 /* Process offline CPU's input_pkt_queue */
Tom Herbert76cc8b12010-05-20 18:37:59 +00007453 while ((skb = __skb_dequeue(&oldsd->process_queue))) {
Eric Dumazet91e83132015-02-05 14:58:14 -08007454 netif_rx_ni(skb);
Tom Herbert76cc8b12010-05-20 18:37:59 +00007455 input_queue_head_incr(oldsd);
7456 }
Eric Dumazetac64da02015-01-15 17:04:22 -08007457 while ((skb = skb_dequeue(&oldsd->input_pkt_queue))) {
Eric Dumazet91e83132015-02-05 14:58:14 -08007458 netif_rx_ni(skb);
Tom Herbert76cc8b12010-05-20 18:37:59 +00007459 input_queue_head_incr(oldsd);
Tom Herbertfec5e652010-04-16 16:01:27 -07007460 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07007461
7462 return NOTIFY_OK;
7463}
Linus Torvalds1da177e2005-04-16 15:20:36 -07007464
7465
Herbert Xu7f353bf2007-08-10 15:47:58 -07007466/**
Herbert Xub63365a2008-10-23 01:11:29 -07007467 * netdev_increment_features - increment feature set by one
7468 * @all: current feature set
7469 * @one: new feature set
7470 * @mask: mask feature set
Herbert Xu7f353bf2007-08-10 15:47:58 -07007471 *
7472 * Computes a new feature set after adding a device with feature set
Herbert Xub63365a2008-10-23 01:11:29 -07007473 * @one to the master device with current feature set @all. Will not
7474 * enable anything that is off in @mask. Returns the new feature set.
Herbert Xu7f353bf2007-08-10 15:47:58 -07007475 */
Michał Mirosławc8f44af2011-11-15 15:29:55 +00007476netdev_features_t netdev_increment_features(netdev_features_t all,
7477 netdev_features_t one, netdev_features_t mask)
Herbert Xu7f353bf2007-08-10 15:47:58 -07007478{
Michał Mirosław1742f182011-04-22 06:31:16 +00007479 if (mask & NETIF_F_GEN_CSUM)
7480 mask |= NETIF_F_ALL_CSUM;
7481 mask |= NETIF_F_VLAN_CHALLENGED;
7482
7483 all |= one & (NETIF_F_ONE_FOR_ALL|NETIF_F_ALL_CSUM) & mask;
7484 all &= one | ~NETIF_F_ALL_FOR_ALL;
7485
Michał Mirosław1742f182011-04-22 06:31:16 +00007486 /* If one device supports hw checksumming, set for all. */
7487 if (all & NETIF_F_GEN_CSUM)
7488 all &= ~(NETIF_F_ALL_CSUM & ~NETIF_F_GEN_CSUM);
Herbert Xu7f353bf2007-08-10 15:47:58 -07007489
7490 return all;
7491}
Herbert Xub63365a2008-10-23 01:11:29 -07007492EXPORT_SYMBOL(netdev_increment_features);
Herbert Xu7f353bf2007-08-10 15:47:58 -07007493
Baruch Siach430f03c2013-06-02 20:43:55 +00007494static struct hlist_head * __net_init netdev_create_hash(void)
Pavel Emelyanov30d97d32007-09-16 15:40:33 -07007495{
7496 int i;
7497 struct hlist_head *hash;
7498
7499 hash = kmalloc(sizeof(*hash) * NETDEV_HASHENTRIES, GFP_KERNEL);
7500 if (hash != NULL)
7501 for (i = 0; i < NETDEV_HASHENTRIES; i++)
7502 INIT_HLIST_HEAD(&hash[i]);
7503
7504 return hash;
7505}
7506
Eric W. Biederman881d9662007-09-17 11:56:21 -07007507/* Initialize per network namespace state */
Pavel Emelyanov46650792007-10-08 20:38:39 -07007508static int __net_init netdev_init(struct net *net)
Eric W. Biederman881d9662007-09-17 11:56:21 -07007509{
Rustad, Mark D734b6542012-07-18 09:06:07 +00007510 if (net != &init_net)
7511 INIT_LIST_HEAD(&net->dev_base_head);
Eric W. Biederman881d9662007-09-17 11:56:21 -07007512
Pavel Emelyanov30d97d32007-09-16 15:40:33 -07007513 net->dev_name_head = netdev_create_hash();
7514 if (net->dev_name_head == NULL)
7515 goto err_name;
Eric W. Biederman881d9662007-09-17 11:56:21 -07007516
Pavel Emelyanov30d97d32007-09-16 15:40:33 -07007517 net->dev_index_head = netdev_create_hash();
7518 if (net->dev_index_head == NULL)
7519 goto err_idx;
Eric W. Biederman881d9662007-09-17 11:56:21 -07007520
7521 return 0;
Pavel Emelyanov30d97d32007-09-16 15:40:33 -07007522
7523err_idx:
7524 kfree(net->dev_name_head);
7525err_name:
7526 return -ENOMEM;
Eric W. Biederman881d9662007-09-17 11:56:21 -07007527}
7528
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07007529/**
7530 * netdev_drivername - network driver for the device
7531 * @dev: network device
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07007532 *
7533 * Determine network driver for device.
7534 */
David S. Miller3019de12011-06-06 16:41:33 -07007535const char *netdev_drivername(const struct net_device *dev)
Arjan van de Ven6579e572008-07-21 13:31:48 -07007536{
Stephen Hemmingercf04a4c72008-09-30 02:22:14 -07007537 const struct device_driver *driver;
7538 const struct device *parent;
David S. Miller3019de12011-06-06 16:41:33 -07007539 const char *empty = "";
Arjan van de Ven6579e572008-07-21 13:31:48 -07007540
7541 parent = dev->dev.parent;
Arjan van de Ven6579e572008-07-21 13:31:48 -07007542 if (!parent)
David S. Miller3019de12011-06-06 16:41:33 -07007543 return empty;
Arjan van de Ven6579e572008-07-21 13:31:48 -07007544
7545 driver = parent->driver;
7546 if (driver && driver->name)
David S. Miller3019de12011-06-06 16:41:33 -07007547 return driver->name;
7548 return empty;
Arjan van de Ven6579e572008-07-21 13:31:48 -07007549}
7550
Joe Perches6ea754e2014-09-22 11:10:50 -07007551static void __netdev_printk(const char *level, const struct net_device *dev,
7552 struct va_format *vaf)
Joe Perches256df2f2010-06-27 01:02:35 +00007553{
Joe Perchesb004ff42012-09-12 20:12:19 -07007554 if (dev && dev->dev.parent) {
Joe Perches6ea754e2014-09-22 11:10:50 -07007555 dev_printk_emit(level[1] - '0',
7556 dev->dev.parent,
7557 "%s %s %s%s: %pV",
7558 dev_driver_string(dev->dev.parent),
7559 dev_name(dev->dev.parent),
7560 netdev_name(dev), netdev_reg_state(dev),
7561 vaf);
Joe Perchesb004ff42012-09-12 20:12:19 -07007562 } else if (dev) {
Joe Perches6ea754e2014-09-22 11:10:50 -07007563 printk("%s%s%s: %pV",
7564 level, netdev_name(dev), netdev_reg_state(dev), vaf);
Joe Perchesb004ff42012-09-12 20:12:19 -07007565 } else {
Joe Perches6ea754e2014-09-22 11:10:50 -07007566 printk("%s(NULL net_device): %pV", level, vaf);
Joe Perchesb004ff42012-09-12 20:12:19 -07007567 }
Joe Perches256df2f2010-06-27 01:02:35 +00007568}
7569
Joe Perches6ea754e2014-09-22 11:10:50 -07007570void netdev_printk(const char *level, const struct net_device *dev,
7571 const char *format, ...)
Joe Perches256df2f2010-06-27 01:02:35 +00007572{
7573 struct va_format vaf;
7574 va_list args;
Joe Perches256df2f2010-06-27 01:02:35 +00007575
7576 va_start(args, format);
7577
7578 vaf.fmt = format;
7579 vaf.va = &args;
7580
Joe Perches6ea754e2014-09-22 11:10:50 -07007581 __netdev_printk(level, dev, &vaf);
Joe Perchesb004ff42012-09-12 20:12:19 -07007582
Joe Perches256df2f2010-06-27 01:02:35 +00007583 va_end(args);
Joe Perches256df2f2010-06-27 01:02:35 +00007584}
7585EXPORT_SYMBOL(netdev_printk);
7586
7587#define define_netdev_printk_level(func, level) \
Joe Perches6ea754e2014-09-22 11:10:50 -07007588void func(const struct net_device *dev, const char *fmt, ...) \
Joe Perches256df2f2010-06-27 01:02:35 +00007589{ \
Joe Perches256df2f2010-06-27 01:02:35 +00007590 struct va_format vaf; \
7591 va_list args; \
7592 \
7593 va_start(args, fmt); \
7594 \
7595 vaf.fmt = fmt; \
7596 vaf.va = &args; \
7597 \
Joe Perches6ea754e2014-09-22 11:10:50 -07007598 __netdev_printk(level, dev, &vaf); \
Joe Perchesb004ff42012-09-12 20:12:19 -07007599 \
Joe Perches256df2f2010-06-27 01:02:35 +00007600 va_end(args); \
Joe Perches256df2f2010-06-27 01:02:35 +00007601} \
7602EXPORT_SYMBOL(func);
7603
7604define_netdev_printk_level(netdev_emerg, KERN_EMERG);
7605define_netdev_printk_level(netdev_alert, KERN_ALERT);
7606define_netdev_printk_level(netdev_crit, KERN_CRIT);
7607define_netdev_printk_level(netdev_err, KERN_ERR);
7608define_netdev_printk_level(netdev_warn, KERN_WARNING);
7609define_netdev_printk_level(netdev_notice, KERN_NOTICE);
7610define_netdev_printk_level(netdev_info, KERN_INFO);
7611
Pavel Emelyanov46650792007-10-08 20:38:39 -07007612static void __net_exit netdev_exit(struct net *net)
Eric W. Biederman881d9662007-09-17 11:56:21 -07007613{
7614 kfree(net->dev_name_head);
7615 kfree(net->dev_index_head);
7616}
7617
Denis V. Lunev022cbae2007-11-13 03:23:50 -08007618static struct pernet_operations __net_initdata netdev_net_ops = {
Eric W. Biederman881d9662007-09-17 11:56:21 -07007619 .init = netdev_init,
7620 .exit = netdev_exit,
7621};
7622
Pavel Emelyanov46650792007-10-08 20:38:39 -07007623static void __net_exit default_device_exit(struct net *net)
Eric W. Biedermance286d32007-09-12 13:53:49 +02007624{
Eric W. Biedermane008b5f2009-11-29 22:25:30 +00007625 struct net_device *dev, *aux;
Eric W. Biedermance286d32007-09-12 13:53:49 +02007626 /*
Eric W. Biedermane008b5f2009-11-29 22:25:30 +00007627 * Push all migratable network devices back to the
Eric W. Biedermance286d32007-09-12 13:53:49 +02007628 * initial network namespace
7629 */
7630 rtnl_lock();
Eric W. Biedermane008b5f2009-11-29 22:25:30 +00007631 for_each_netdev_safe(net, dev, aux) {
Eric W. Biedermance286d32007-09-12 13:53:49 +02007632 int err;
Pavel Emelyanovaca51392008-05-08 01:24:25 -07007633 char fb_name[IFNAMSIZ];
Eric W. Biedermance286d32007-09-12 13:53:49 +02007634
7635 /* Ignore unmoveable devices (i.e. loopback) */
7636 if (dev->features & NETIF_F_NETNS_LOCAL)
7637 continue;
7638
Eric W. Biedermane008b5f2009-11-29 22:25:30 +00007639 /* Leave virtual devices for the generic cleanup */
7640 if (dev->rtnl_link_ops)
7641 continue;
Eric W. Biedermand0c082c2008-11-05 15:59:38 -08007642
Lucas De Marchi25985ed2011-03-30 22:57:33 -03007643 /* Push remaining network devices to init_net */
Pavel Emelyanovaca51392008-05-08 01:24:25 -07007644 snprintf(fb_name, IFNAMSIZ, "dev%d", dev->ifindex);
7645 err = dev_change_net_namespace(dev, &init_net, fb_name);
Eric W. Biedermance286d32007-09-12 13:53:49 +02007646 if (err) {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00007647 pr_emerg("%s: failed to move %s to init_net: %d\n",
7648 __func__, dev->name, err);
Pavel Emelyanovaca51392008-05-08 01:24:25 -07007649 BUG();
Eric W. Biedermance286d32007-09-12 13:53:49 +02007650 }
7651 }
7652 rtnl_unlock();
7653}
7654
Eric W. Biederman50624c92013-09-23 21:19:49 -07007655static void __net_exit rtnl_lock_unregistering(struct list_head *net_list)
7656{
7657 /* Return with the rtnl_lock held when there are no network
7658 * devices unregistering in any network namespace in net_list.
7659 */
7660 struct net *net;
7661 bool unregistering;
Peter Zijlstraff960a72014-10-29 17:04:56 +01007662 DEFINE_WAIT_FUNC(wait, woken_wake_function);
Eric W. Biederman50624c92013-09-23 21:19:49 -07007663
Peter Zijlstraff960a72014-10-29 17:04:56 +01007664 add_wait_queue(&netdev_unregistering_wq, &wait);
Eric W. Biederman50624c92013-09-23 21:19:49 -07007665 for (;;) {
Eric W. Biederman50624c92013-09-23 21:19:49 -07007666 unregistering = false;
7667 rtnl_lock();
7668 list_for_each_entry(net, net_list, exit_list) {
7669 if (net->dev_unreg_count > 0) {
7670 unregistering = true;
7671 break;
7672 }
7673 }
7674 if (!unregistering)
7675 break;
7676 __rtnl_unlock();
Peter Zijlstraff960a72014-10-29 17:04:56 +01007677
7678 wait_woken(&wait, TASK_UNINTERRUPTIBLE, MAX_SCHEDULE_TIMEOUT);
Eric W. Biederman50624c92013-09-23 21:19:49 -07007679 }
Peter Zijlstraff960a72014-10-29 17:04:56 +01007680 remove_wait_queue(&netdev_unregistering_wq, &wait);
Eric W. Biederman50624c92013-09-23 21:19:49 -07007681}
7682
Eric W. Biederman04dc7f6b2009-12-03 02:29:04 +00007683static void __net_exit default_device_exit_batch(struct list_head *net_list)
7684{
7685 /* At exit all network devices most be removed from a network
Uwe Kleine-Königb5950762010-11-01 15:38:34 -04007686 * namespace. Do this in the reverse order of registration.
Eric W. Biederman04dc7f6b2009-12-03 02:29:04 +00007687 * Do this across as many network namespaces as possible to
7688 * improve batching efficiency.
7689 */
7690 struct net_device *dev;
7691 struct net *net;
7692 LIST_HEAD(dev_kill_list);
7693
Eric W. Biederman50624c92013-09-23 21:19:49 -07007694 /* To prevent network device cleanup code from dereferencing
7695 * loopback devices or network devices that have been freed
7696 * wait here for all pending unregistrations to complete,
7697 * before unregistring the loopback device and allowing the
7698 * network namespace be freed.
7699 *
7700 * The netdev todo list containing all network devices
7701 * unregistrations that happen in default_device_exit_batch
7702 * will run in the rtnl_unlock() at the end of
7703 * default_device_exit_batch.
7704 */
7705 rtnl_lock_unregistering(net_list);
Eric W. Biederman04dc7f6b2009-12-03 02:29:04 +00007706 list_for_each_entry(net, net_list, exit_list) {
7707 for_each_netdev_reverse(net, dev) {
Jiri Pirkob0ab2fa2014-06-26 09:58:25 +02007708 if (dev->rtnl_link_ops && dev->rtnl_link_ops->dellink)
Eric W. Biederman04dc7f6b2009-12-03 02:29:04 +00007709 dev->rtnl_link_ops->dellink(dev, &dev_kill_list);
7710 else
7711 unregister_netdevice_queue(dev, &dev_kill_list);
7712 }
7713 }
7714 unregister_netdevice_many(&dev_kill_list);
7715 rtnl_unlock();
7716}
7717
Denis V. Lunev022cbae2007-11-13 03:23:50 -08007718static struct pernet_operations __net_initdata default_device_ops = {
Eric W. Biedermance286d32007-09-12 13:53:49 +02007719 .exit = default_device_exit,
Eric W. Biederman04dc7f6b2009-12-03 02:29:04 +00007720 .exit_batch = default_device_exit_batch,
Eric W. Biedermance286d32007-09-12 13:53:49 +02007721};
7722
Linus Torvalds1da177e2005-04-16 15:20:36 -07007723/*
7724 * Initialize the DEV module. At boot time this walks the device list and
7725 * unhooks any devices that fail to initialise (normally hardware not
7726 * present) and leaves us with a valid list of present and active devices.
7727 *
7728 */
7729
7730/*
7731 * This is called single threaded during boot, so no need
7732 * to take the rtnl semaphore.
7733 */
7734static int __init net_dev_init(void)
7735{
7736 int i, rc = -ENOMEM;
7737
7738 BUG_ON(!dev_boot_phase);
7739
Linus Torvalds1da177e2005-04-16 15:20:36 -07007740 if (dev_proc_init())
7741 goto out;
7742
Eric W. Biederman8b41d182007-09-26 22:02:53 -07007743 if (netdev_kobject_init())
Linus Torvalds1da177e2005-04-16 15:20:36 -07007744 goto out;
7745
7746 INIT_LIST_HEAD(&ptype_all);
Pavel Emelyanov82d8a8672007-11-26 20:12:58 +08007747 for (i = 0; i < PTYPE_HASH_SIZE; i++)
Linus Torvalds1da177e2005-04-16 15:20:36 -07007748 INIT_LIST_HEAD(&ptype_base[i]);
7749
Vlad Yasevich62532da2012-11-15 08:49:10 +00007750 INIT_LIST_HEAD(&offload_base);
7751
Eric W. Biederman881d9662007-09-17 11:56:21 -07007752 if (register_pernet_subsys(&netdev_net_ops))
7753 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07007754
7755 /*
7756 * Initialise the packet receive queues.
7757 */
7758
KAMEZAWA Hiroyuki6f912042006-04-10 22:52:50 -07007759 for_each_possible_cpu(i) {
Eric Dumazete36fa2f2010-04-19 21:17:14 +00007760 struct softnet_data *sd = &per_cpu(softnet_data, i);
Linus Torvalds1da177e2005-04-16 15:20:36 -07007761
Eric Dumazete36fa2f2010-04-19 21:17:14 +00007762 skb_queue_head_init(&sd->input_pkt_queue);
Changli Gao6e7676c2010-04-27 15:07:33 -07007763 skb_queue_head_init(&sd->process_queue);
Eric Dumazete36fa2f2010-04-19 21:17:14 +00007764 INIT_LIST_HEAD(&sd->poll_list);
Changli Gaoa9cbd582010-04-26 23:06:24 +00007765 sd->output_queue_tailp = &sd->output_queue;
Eric Dumazetdf334542010-03-24 19:13:54 +00007766#ifdef CONFIG_RPS
Eric Dumazete36fa2f2010-04-19 21:17:14 +00007767 sd->csd.func = rps_trigger_softirq;
7768 sd->csd.info = sd;
Eric Dumazete36fa2f2010-04-19 21:17:14 +00007769 sd->cpu = i;
Tom Herbert1e94d722010-03-18 17:45:44 -07007770#endif
Tom Herbert0a9627f2010-03-16 08:03:29 +00007771
Eric Dumazete36fa2f2010-04-19 21:17:14 +00007772 sd->backlog.poll = process_backlog;
7773 sd->backlog.weight = weight_p;
Linus Torvalds1da177e2005-04-16 15:20:36 -07007774 }
7775
Linus Torvalds1da177e2005-04-16 15:20:36 -07007776 dev_boot_phase = 0;
7777
Eric W. Biederman505d4f72008-11-07 22:54:20 -08007778 /* The loopback device is special if any other network devices
7779 * is present in a network namespace the loopback device must
7780 * be present. Since we now dynamically allocate and free the
7781 * loopback device ensure this invariant is maintained by
7782 * keeping the loopback device as the first device on the
7783 * list of network devices. Ensuring the loopback devices
7784 * is the first device that appears and the last network device
7785 * that disappears.
7786 */
7787 if (register_pernet_device(&loopback_net_ops))
7788 goto out;
7789
7790 if (register_pernet_device(&default_device_ops))
7791 goto out;
7792
Carlos R. Mafra962cf362008-05-15 11:15:37 -03007793 open_softirq(NET_TX_SOFTIRQ, net_tx_action);
7794 open_softirq(NET_RX_SOFTIRQ, net_rx_action);
Linus Torvalds1da177e2005-04-16 15:20:36 -07007795
7796 hotcpu_notifier(dev_cpu_callback, 0);
Thomas Graff38a9eb2015-07-21 10:43:56 +02007797 dst_subsys_init();
Linus Torvalds1da177e2005-04-16 15:20:36 -07007798 rc = 0;
7799out:
7800 return rc;
7801}
7802
7803subsys_initcall(net_dev_init);