blob: 1e57efda055b58546876c2474f19719bb9e191cc [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * NET3 Protocol independent device support routines.
3 *
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public License
6 * as published by the Free Software Foundation; either version
7 * 2 of the License, or (at your option) any later version.
8 *
9 * Derived from the non IP parts of dev.c 1.0.19
Jesper Juhl02c30a82005-05-05 16:16:16 -070010 * Authors: Ross Biro
Linus Torvalds1da177e2005-04-16 15:20:36 -070011 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
12 * Mark Evans, <evansmp@uhura.aston.ac.uk>
13 *
14 * Additional Authors:
15 * Florian la Roche <rzsfl@rz.uni-sb.de>
16 * Alan Cox <gw4pts@gw4pts.ampr.org>
17 * David Hinds <dahinds@users.sourceforge.net>
18 * Alexey Kuznetsov <kuznet@ms2.inr.ac.ru>
19 * Adam Sulmicki <adam@cfar.umd.edu>
20 * Pekka Riikonen <priikone@poesidon.pspt.fi>
21 *
22 * Changes:
23 * D.J. Barrow : Fixed bug where dev->refcnt gets set
24 * to 2 if register_netdev gets called
25 * before net_dev_init & also removed a
26 * few lines of code in the process.
27 * Alan Cox : device private ioctl copies fields back.
28 * Alan Cox : Transmit queue code does relevant
29 * stunts to keep the queue safe.
30 * Alan Cox : Fixed double lock.
31 * Alan Cox : Fixed promisc NULL pointer trap
32 * ???????? : Support the full private ioctl range
33 * Alan Cox : Moved ioctl permission check into
34 * drivers
35 * Tim Kordas : SIOCADDMULTI/SIOCDELMULTI
36 * Alan Cox : 100 backlog just doesn't cut it when
37 * you start doing multicast video 8)
38 * Alan Cox : Rewrote net_bh and list manager.
39 * Alan Cox : Fix ETH_P_ALL echoback lengths.
40 * Alan Cox : Took out transmit every packet pass
41 * Saved a few bytes in the ioctl handler
42 * Alan Cox : Network driver sets packet type before
43 * calling netif_rx. Saves a function
44 * call a packet.
45 * Alan Cox : Hashed net_bh()
46 * Richard Kooijman: Timestamp fixes.
47 * Alan Cox : Wrong field in SIOCGIFDSTADDR
48 * Alan Cox : Device lock protection.
49 * Alan Cox : Fixed nasty side effect of device close
50 * changes.
51 * Rudi Cilibrasi : Pass the right thing to
52 * set_mac_address()
53 * Dave Miller : 32bit quantity for the device lock to
54 * make it work out on a Sparc.
55 * Bjorn Ekwall : Added KERNELD hack.
56 * Alan Cox : Cleaned up the backlog initialise.
57 * Craig Metz : SIOCGIFCONF fix if space for under
58 * 1 device.
59 * Thomas Bogendoerfer : Return ENODEV for dev_open, if there
60 * is no device open function.
61 * Andi Kleen : Fix error reporting for SIOCGIFCONF
62 * Michael Chastain : Fix signed/unsigned for SIOCGIFCONF
63 * Cyrus Durgin : Cleaned for KMOD
64 * Adam Sulmicki : Bug Fix : Network Device Unload
65 * A network device unload needs to purge
66 * the backlog queue.
67 * Paul Rusty Russell : SIOCSIFNAME
68 * Pekka Riikonen : Netdev boot-time settings code
69 * Andrew Morton : Make unregister_netdevice wait
70 * indefinitely on dev->refcnt
71 * J Hadi Salim : - Backlog queue sampling
72 * - netif_rx() feedback
73 */
74
75#include <asm/uaccess.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070076#include <linux/bitops.h>
Randy Dunlap4fc268d2006-01-11 12:17:47 -080077#include <linux/capability.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070078#include <linux/cpu.h>
79#include <linux/types.h>
80#include <linux/kernel.h>
stephen hemminger08e98972009-11-10 07:20:34 +000081#include <linux/hash.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090082#include <linux/slab.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070083#include <linux/sched.h>
Arjan van de Ven4a3e2f72006-03-20 22:33:17 -080084#include <linux/mutex.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070085#include <linux/string.h>
86#include <linux/mm.h>
87#include <linux/socket.h>
88#include <linux/sockios.h>
89#include <linux/errno.h>
90#include <linux/interrupt.h>
91#include <linux/if_ether.h>
92#include <linux/netdevice.h>
93#include <linux/etherdevice.h>
Ben Hutchings0187bdf2008-06-19 16:15:47 -070094#include <linux/ethtool.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070095#include <linux/notifier.h>
96#include <linux/skbuff.h>
Eric W. Biederman457c4cb2007-09-12 12:01:34 +020097#include <net/net_namespace.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070098#include <net/sock.h>
99#include <linux/rtnetlink.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700100#include <linux/stat.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700101#include <net/dst.h>
102#include <net/pkt_sched.h>
103#include <net/checksum.h>
Arnd Bergmann44540962009-11-26 06:07:08 +0000104#include <net/xfrm.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700105#include <linux/highmem.h>
106#include <linux/init.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700107#include <linux/module.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700108#include <linux/netpoll.h>
109#include <linux/rcupdate.h>
110#include <linux/delay.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700111#include <net/iw_handler.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700112#include <asm/current.h>
Steve Grubb5bdb9882005-12-03 08:39:35 -0500113#include <linux/audit.h>
Chris Leechdb217332006-06-17 21:24:58 -0700114#include <linux/dmaengine.h>
Herbert Xuf6a78bf2006-06-22 02:57:17 -0700115#include <linux/err.h>
David S. Millerc7fa9d12006-08-15 16:34:13 -0700116#include <linux/ctype.h>
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700117#include <linux/if_arp.h>
Ben Hutchings6de329e2008-06-16 17:02:28 -0700118#include <linux/if_vlan.h>
David S. Miller8f0f2222008-07-15 03:47:03 -0700119#include <linux/ip.h>
Alexander Duyckad55dca2008-09-20 22:05:50 -0700120#include <net/ip.h>
Simon Horman25cd9ba2014-10-06 05:05:13 -0700121#include <net/mpls.h>
David S. Miller8f0f2222008-07-15 03:47:03 -0700122#include <linux/ipv6.h>
123#include <linux/in.h>
David S. Millerb6b2fed2008-07-21 09:48:06 -0700124#include <linux/jhash.h>
125#include <linux/random.h>
David S. Miller9cbc1cb2009-06-15 03:02:23 -0700126#include <trace/events/napi.h>
Koki Sanagicf66ba52010-08-23 18:45:02 +0900127#include <trace/events/net.h>
Koki Sanagi07dc22e2010-08-23 18:46:12 +0900128#include <trace/events/skb.h>
FUJITA Tomonori5acbbd42010-03-30 22:35:50 +0000129#include <linux/pci.h>
Stephen Rothwellcaeda9b2010-09-16 21:39:16 -0700130#include <linux/inetdevice.h>
Ben Hutchingsc4454772011-01-19 11:03:53 +0000131#include <linux/cpu_rmap.h>
Ingo Molnarc5905af2012-02-24 08:31:31 +0100132#include <linux/static_key.h>
Eliezer Tamiraf12fa62013-06-10 11:39:41 +0300133#include <linux/hashtable.h>
Eric Dumazet60877a32013-06-20 01:15:51 -0700134#include <linux/vmalloc.h>
Michal Kubeček529d0482013-11-15 06:18:50 +0100135#include <linux/if_macvlan.h>
Willem de Bruijne7fd2882014-08-04 22:11:48 -0400136#include <linux/errqueue.h>
Eric Dumazet3b47d302014-11-06 21:09:44 -0800137#include <linux/hrtimer.h>
Pablo Neirae687ad62015-05-13 18:19:38 +0200138#include <linux/netfilter_ingress.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700139
Pavel Emelyanov342709e2007-10-23 21:14:45 -0700140#include "net-sysfs.h"
141
Herbert Xud565b0a2008-12-15 23:38:52 -0800142/* Instead of increasing this, you should create a hash table. */
143#define MAX_GRO_SKBS 8
144
Herbert Xu5d38a072009-01-04 16:13:40 -0800145/* This should be increased if a protocol with a bigger head is added. */
146#define GRO_MAX_HEAD (MAX_HEADER + 128)
147
Linus Torvalds1da177e2005-04-16 15:20:36 -0700148static DEFINE_SPINLOCK(ptype_lock);
Vlad Yasevich62532da2012-11-15 08:49:10 +0000149static DEFINE_SPINLOCK(offload_lock);
Cong Wang900ff8c2013-02-18 19:20:33 +0000150struct list_head ptype_base[PTYPE_HASH_SIZE] __read_mostly;
151struct list_head ptype_all __read_mostly; /* Taps */
Vlad Yasevich62532da2012-11-15 08:49:10 +0000152static struct list_head offload_base __read_mostly;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700153
Ben Hutchingsae78dbf2014-01-10 22:17:24 +0000154static int netif_rx_internal(struct sk_buff *skb);
Loic Prylli54951192014-07-01 21:39:43 -0700155static int call_netdevice_notifiers_info(unsigned long val,
156 struct net_device *dev,
157 struct netdev_notifier_info *info);
Ben Hutchingsae78dbf2014-01-10 22:17:24 +0000158
Linus Torvalds1da177e2005-04-16 15:20:36 -0700159/*
Pavel Emelianov7562f872007-05-03 15:13:45 -0700160 * The @dev_base_head list is protected by @dev_base_lock and the rtnl
Linus Torvalds1da177e2005-04-16 15:20:36 -0700161 * semaphore.
162 *
Eric Dumazetc6d14c82009-11-04 05:43:23 -0800163 * Pure readers hold dev_base_lock for reading, or rcu_read_lock()
Linus Torvalds1da177e2005-04-16 15:20:36 -0700164 *
165 * Writers must hold the rtnl semaphore while they loop through the
Pavel Emelianov7562f872007-05-03 15:13:45 -0700166 * dev_base_head list, and hold dev_base_lock for writing when they do the
Linus Torvalds1da177e2005-04-16 15:20:36 -0700167 * actual updates. This allows pure readers to access the list even
168 * while a writer is preparing to update it.
169 *
170 * To put it another way, dev_base_lock is held for writing only to
171 * protect against pure readers; the rtnl semaphore provides the
172 * protection against other writers.
173 *
174 * See, for example usages, register_netdevice() and
175 * unregister_netdevice(), which must be called with the rtnl
176 * semaphore held.
177 */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700178DEFINE_RWLOCK(dev_base_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700179EXPORT_SYMBOL(dev_base_lock);
180
Eliezer Tamiraf12fa62013-06-10 11:39:41 +0300181/* protects napi_hash addition/deletion and napi_gen_id */
182static DEFINE_SPINLOCK(napi_hash_lock);
183
184static unsigned int napi_gen_id;
185static DEFINE_HASHTABLE(napi_hash, 8);
186
Thomas Gleixner18afa4b2013-07-23 16:13:17 +0200187static seqcount_t devnet_rename_seq;
Brian Haleyc91f6df2012-11-26 05:21:08 +0000188
Thomas Graf4e985ad2011-06-21 03:11:20 +0000189static inline void dev_base_seq_inc(struct net *net)
190{
191 while (++net->dev_base_seq == 0);
192}
193
Eric W. Biederman881d9662007-09-17 11:56:21 -0700194static inline struct hlist_head *dev_name_hash(struct net *net, const char *name)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700195{
Eric Dumazet95c96172012-04-15 05:58:06 +0000196 unsigned int hash = full_name_hash(name, strnlen(name, IFNAMSIZ));
197
stephen hemminger08e98972009-11-10 07:20:34 +0000198 return &net->dev_name_head[hash_32(hash, NETDEV_HASHBITS)];
Linus Torvalds1da177e2005-04-16 15:20:36 -0700199}
200
Eric W. Biederman881d9662007-09-17 11:56:21 -0700201static inline struct hlist_head *dev_index_hash(struct net *net, int ifindex)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700202{
Eric Dumazet7c28bd02009-10-24 06:13:17 -0700203 return &net->dev_index_head[ifindex & (NETDEV_HASHENTRIES - 1)];
Linus Torvalds1da177e2005-04-16 15:20:36 -0700204}
205
Eric Dumazete36fa2f2010-04-19 21:17:14 +0000206static inline void rps_lock(struct softnet_data *sd)
Changli Gao152102c2010-03-30 20:16:22 +0000207{
208#ifdef CONFIG_RPS
Eric Dumazete36fa2f2010-04-19 21:17:14 +0000209 spin_lock(&sd->input_pkt_queue.lock);
Changli Gao152102c2010-03-30 20:16:22 +0000210#endif
211}
212
Eric Dumazete36fa2f2010-04-19 21:17:14 +0000213static inline void rps_unlock(struct softnet_data *sd)
Changli Gao152102c2010-03-30 20:16:22 +0000214{
215#ifdef CONFIG_RPS
Eric Dumazete36fa2f2010-04-19 21:17:14 +0000216 spin_unlock(&sd->input_pkt_queue.lock);
Changli Gao152102c2010-03-30 20:16:22 +0000217#endif
218}
219
Eric W. Biedermance286d32007-09-12 13:53:49 +0200220/* Device list insertion */
dingtianhong53759be2013-04-17 22:17:50 +0000221static void list_netdevice(struct net_device *dev)
Eric W. Biedermance286d32007-09-12 13:53:49 +0200222{
YOSHIFUJI Hideakic346dca2008-03-25 21:47:49 +0900223 struct net *net = dev_net(dev);
Eric W. Biedermance286d32007-09-12 13:53:49 +0200224
225 ASSERT_RTNL();
226
227 write_lock_bh(&dev_base_lock);
Eric Dumazetc6d14c82009-11-04 05:43:23 -0800228 list_add_tail_rcu(&dev->dev_list, &net->dev_base_head);
Eric Dumazet72c95282009-10-30 07:11:27 +0000229 hlist_add_head_rcu(&dev->name_hlist, dev_name_hash(net, dev->name));
Eric Dumazetfb699dfd2009-10-19 19:18:49 +0000230 hlist_add_head_rcu(&dev->index_hlist,
231 dev_index_hash(net, dev->ifindex));
Eric W. Biedermance286d32007-09-12 13:53:49 +0200232 write_unlock_bh(&dev_base_lock);
Thomas Graf4e985ad2011-06-21 03:11:20 +0000233
234 dev_base_seq_inc(net);
Eric W. Biedermance286d32007-09-12 13:53:49 +0200235}
236
Eric Dumazetfb699dfd2009-10-19 19:18:49 +0000237/* Device list removal
238 * caller must respect a RCU grace period before freeing/reusing dev
239 */
Eric W. Biedermance286d32007-09-12 13:53:49 +0200240static void unlist_netdevice(struct net_device *dev)
241{
242 ASSERT_RTNL();
243
244 /* Unlink dev from the device chain */
245 write_lock_bh(&dev_base_lock);
Eric Dumazetc6d14c82009-11-04 05:43:23 -0800246 list_del_rcu(&dev->dev_list);
Eric Dumazet72c95282009-10-30 07:11:27 +0000247 hlist_del_rcu(&dev->name_hlist);
Eric Dumazetfb699dfd2009-10-19 19:18:49 +0000248 hlist_del_rcu(&dev->index_hlist);
Eric W. Biedermance286d32007-09-12 13:53:49 +0200249 write_unlock_bh(&dev_base_lock);
Thomas Graf4e985ad2011-06-21 03:11:20 +0000250
251 dev_base_seq_inc(dev_net(dev));
Eric W. Biedermance286d32007-09-12 13:53:49 +0200252}
253
Linus Torvalds1da177e2005-04-16 15:20:36 -0700254/*
255 * Our notifier list
256 */
257
Alan Sternf07d5b92006-05-09 15:23:03 -0700258static RAW_NOTIFIER_HEAD(netdev_chain);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700259
260/*
261 * Device drivers call our routines to queue packets here. We empty the
262 * queue in the local softnet handler.
263 */
Stephen Hemmingerbea33482007-10-03 16:41:36 -0700264
Eric Dumazet9958da02010-04-17 04:17:02 +0000265DEFINE_PER_CPU_ALIGNED(struct softnet_data, softnet_data);
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700266EXPORT_PER_CPU_SYMBOL(softnet_data);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700267
David S. Millercf508b12008-07-22 14:16:42 -0700268#ifdef CONFIG_LOCKDEP
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700269/*
David S. Millerc773e842008-07-08 23:13:53 -0700270 * register_netdevice() inits txq->_xmit_lock and sets lockdep class
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700271 * according to dev->type
272 */
273static const unsigned short netdev_lock_type[] =
274 {ARPHRD_NETROM, ARPHRD_ETHER, ARPHRD_EETHER, ARPHRD_AX25,
275 ARPHRD_PRONET, ARPHRD_CHAOS, ARPHRD_IEEE802, ARPHRD_ARCNET,
276 ARPHRD_APPLETLK, ARPHRD_DLCI, ARPHRD_ATM, ARPHRD_METRICOM,
277 ARPHRD_IEEE1394, ARPHRD_EUI64, ARPHRD_INFINIBAND, ARPHRD_SLIP,
278 ARPHRD_CSLIP, ARPHRD_SLIP6, ARPHRD_CSLIP6, ARPHRD_RSRVD,
279 ARPHRD_ADAPT, ARPHRD_ROSE, ARPHRD_X25, ARPHRD_HWX25,
280 ARPHRD_PPP, ARPHRD_CISCO, ARPHRD_LAPB, ARPHRD_DDCMP,
281 ARPHRD_RAWHDLC, ARPHRD_TUNNEL, ARPHRD_TUNNEL6, ARPHRD_FRAD,
282 ARPHRD_SKIP, ARPHRD_LOOPBACK, ARPHRD_LOCALTLK, ARPHRD_FDDI,
283 ARPHRD_BIF, ARPHRD_SIT, ARPHRD_IPDDP, ARPHRD_IPGRE,
284 ARPHRD_PIMREG, ARPHRD_HIPPI, ARPHRD_ASH, ARPHRD_ECONET,
285 ARPHRD_IRDA, ARPHRD_FCPP, ARPHRD_FCAL, ARPHRD_FCPL,
Paul Gortmaker211ed862012-05-10 17:14:35 -0400286 ARPHRD_FCFABRIC, ARPHRD_IEEE80211, ARPHRD_IEEE80211_PRISM,
287 ARPHRD_IEEE80211_RADIOTAP, ARPHRD_PHONET, ARPHRD_PHONET_PIPE,
288 ARPHRD_IEEE802154, ARPHRD_VOID, ARPHRD_NONE};
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700289
Jan Engelhardt36cbd3d2009-08-05 10:42:58 -0700290static const char *const netdev_lock_name[] =
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700291 {"_xmit_NETROM", "_xmit_ETHER", "_xmit_EETHER", "_xmit_AX25",
292 "_xmit_PRONET", "_xmit_CHAOS", "_xmit_IEEE802", "_xmit_ARCNET",
293 "_xmit_APPLETLK", "_xmit_DLCI", "_xmit_ATM", "_xmit_METRICOM",
294 "_xmit_IEEE1394", "_xmit_EUI64", "_xmit_INFINIBAND", "_xmit_SLIP",
295 "_xmit_CSLIP", "_xmit_SLIP6", "_xmit_CSLIP6", "_xmit_RSRVD",
296 "_xmit_ADAPT", "_xmit_ROSE", "_xmit_X25", "_xmit_HWX25",
297 "_xmit_PPP", "_xmit_CISCO", "_xmit_LAPB", "_xmit_DDCMP",
298 "_xmit_RAWHDLC", "_xmit_TUNNEL", "_xmit_TUNNEL6", "_xmit_FRAD",
299 "_xmit_SKIP", "_xmit_LOOPBACK", "_xmit_LOCALTLK", "_xmit_FDDI",
300 "_xmit_BIF", "_xmit_SIT", "_xmit_IPDDP", "_xmit_IPGRE",
301 "_xmit_PIMREG", "_xmit_HIPPI", "_xmit_ASH", "_xmit_ECONET",
302 "_xmit_IRDA", "_xmit_FCPP", "_xmit_FCAL", "_xmit_FCPL",
Paul Gortmaker211ed862012-05-10 17:14:35 -0400303 "_xmit_FCFABRIC", "_xmit_IEEE80211", "_xmit_IEEE80211_PRISM",
304 "_xmit_IEEE80211_RADIOTAP", "_xmit_PHONET", "_xmit_PHONET_PIPE",
305 "_xmit_IEEE802154", "_xmit_VOID", "_xmit_NONE"};
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700306
307static struct lock_class_key netdev_xmit_lock_key[ARRAY_SIZE(netdev_lock_type)];
David S. Millercf508b12008-07-22 14:16:42 -0700308static struct lock_class_key netdev_addr_lock_key[ARRAY_SIZE(netdev_lock_type)];
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700309
310static inline unsigned short netdev_lock_pos(unsigned short dev_type)
311{
312 int i;
313
314 for (i = 0; i < ARRAY_SIZE(netdev_lock_type); i++)
315 if (netdev_lock_type[i] == dev_type)
316 return i;
317 /* the last key is used by default */
318 return ARRAY_SIZE(netdev_lock_type) - 1;
319}
320
David S. Millercf508b12008-07-22 14:16:42 -0700321static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock,
322 unsigned short dev_type)
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700323{
324 int i;
325
326 i = netdev_lock_pos(dev_type);
327 lockdep_set_class_and_name(lock, &netdev_xmit_lock_key[i],
328 netdev_lock_name[i]);
329}
David S. Millercf508b12008-07-22 14:16:42 -0700330
331static inline void netdev_set_addr_lockdep_class(struct net_device *dev)
332{
333 int i;
334
335 i = netdev_lock_pos(dev->type);
336 lockdep_set_class_and_name(&dev->addr_list_lock,
337 &netdev_addr_lock_key[i],
338 netdev_lock_name[i]);
339}
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700340#else
David S. Millercf508b12008-07-22 14:16:42 -0700341static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock,
342 unsigned short dev_type)
343{
344}
345static inline void netdev_set_addr_lockdep_class(struct net_device *dev)
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700346{
347}
348#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -0700349
350/*******************************************************************************
351
352 Protocol management and registration routines
353
354*******************************************************************************/
355
356/*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700357 * Add a protocol ID to the list. Now that the input handler is
358 * smarter we can dispense with all the messy stuff that used to be
359 * here.
360 *
361 * BEWARE!!! Protocol handlers, mangling input packets,
362 * MUST BE last in hash buckets and checking protocol handlers
363 * MUST start from promiscuous ptype_all chain in net_bh.
364 * It is true now, do not change it.
365 * Explanation follows: if protocol handler, mangling packet, will
366 * be the first on list, it is not able to sense, that packet
367 * is cloned and should be copied-on-write, so that it will
368 * change it and subsequent readers will get broken packet.
369 * --ANK (980803)
370 */
371
Eric Dumazetc07b68e2010-09-02 03:53:46 +0000372static inline struct list_head *ptype_head(const struct packet_type *pt)
373{
374 if (pt->type == htons(ETH_P_ALL))
Salam Noureddine7866a622015-01-27 11:35:48 -0800375 return pt->dev ? &pt->dev->ptype_all : &ptype_all;
Eric Dumazetc07b68e2010-09-02 03:53:46 +0000376 else
Salam Noureddine7866a622015-01-27 11:35:48 -0800377 return pt->dev ? &pt->dev->ptype_specific :
378 &ptype_base[ntohs(pt->type) & PTYPE_HASH_MASK];
Eric Dumazetc07b68e2010-09-02 03:53:46 +0000379}
380
Linus Torvalds1da177e2005-04-16 15:20:36 -0700381/**
382 * dev_add_pack - add packet handler
383 * @pt: packet type declaration
384 *
385 * Add a protocol handler to the networking stack. The passed &packet_type
386 * is linked into kernel lists and may not be freed until it has been
387 * removed from the kernel lists.
388 *
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +0900389 * This call does not sleep therefore it can not
Linus Torvalds1da177e2005-04-16 15:20:36 -0700390 * guarantee all CPU's that are in middle of receiving packets
391 * will see the new packet type (until the next received packet).
392 */
393
394void dev_add_pack(struct packet_type *pt)
395{
Eric Dumazetc07b68e2010-09-02 03:53:46 +0000396 struct list_head *head = ptype_head(pt);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700397
Eric Dumazetc07b68e2010-09-02 03:53:46 +0000398 spin_lock(&ptype_lock);
399 list_add_rcu(&pt->list, head);
400 spin_unlock(&ptype_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700401}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700402EXPORT_SYMBOL(dev_add_pack);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700403
Linus Torvalds1da177e2005-04-16 15:20:36 -0700404/**
405 * __dev_remove_pack - remove packet handler
406 * @pt: packet type declaration
407 *
408 * Remove a protocol handler that was previously added to the kernel
409 * protocol handlers by dev_add_pack(). The passed &packet_type is removed
410 * from the kernel lists and can be freed or reused once this function
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +0900411 * returns.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700412 *
413 * The packet type might still be in use by receivers
414 * and must not be freed until after all the CPU's have gone
415 * through a quiescent state.
416 */
417void __dev_remove_pack(struct packet_type *pt)
418{
Eric Dumazetc07b68e2010-09-02 03:53:46 +0000419 struct list_head *head = ptype_head(pt);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700420 struct packet_type *pt1;
421
Eric Dumazetc07b68e2010-09-02 03:53:46 +0000422 spin_lock(&ptype_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700423
424 list_for_each_entry(pt1, head, list) {
425 if (pt == pt1) {
426 list_del_rcu(&pt->list);
427 goto out;
428 }
429 }
430
Joe Perches7b6cd1c2012-02-01 10:54:43 +0000431 pr_warn("dev_remove_pack: %p not found\n", pt);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700432out:
Eric Dumazetc07b68e2010-09-02 03:53:46 +0000433 spin_unlock(&ptype_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700434}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700435EXPORT_SYMBOL(__dev_remove_pack);
436
Linus Torvalds1da177e2005-04-16 15:20:36 -0700437/**
438 * dev_remove_pack - remove packet handler
439 * @pt: packet type declaration
440 *
441 * Remove a protocol handler that was previously added to the kernel
442 * protocol handlers by dev_add_pack(). The passed &packet_type is removed
443 * from the kernel lists and can be freed or reused once this function
444 * returns.
445 *
446 * This call sleeps to guarantee that no CPU is looking at the packet
447 * type after return.
448 */
449void dev_remove_pack(struct packet_type *pt)
450{
451 __dev_remove_pack(pt);
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +0900452
Linus Torvalds1da177e2005-04-16 15:20:36 -0700453 synchronize_net();
454}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700455EXPORT_SYMBOL(dev_remove_pack);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700456
Vlad Yasevich62532da2012-11-15 08:49:10 +0000457
458/**
459 * dev_add_offload - register offload handlers
460 * @po: protocol offload declaration
461 *
462 * Add protocol offload handlers to the networking stack. The passed
463 * &proto_offload is linked into kernel lists and may not be freed until
464 * it has been removed from the kernel lists.
465 *
466 * This call does not sleep therefore it can not
467 * guarantee all CPU's that are in middle of receiving packets
468 * will see the new offload handlers (until the next received packet).
469 */
470void dev_add_offload(struct packet_offload *po)
471{
David S. Millerbdef7de2015-06-01 14:56:09 -0700472 struct packet_offload *elem;
Vlad Yasevich62532da2012-11-15 08:49:10 +0000473
474 spin_lock(&offload_lock);
David S. Millerbdef7de2015-06-01 14:56:09 -0700475 list_for_each_entry(elem, &offload_base, list) {
476 if (po->priority < elem->priority)
477 break;
478 }
479 list_add_rcu(&po->list, elem->list.prev);
Vlad Yasevich62532da2012-11-15 08:49:10 +0000480 spin_unlock(&offload_lock);
481}
482EXPORT_SYMBOL(dev_add_offload);
483
484/**
485 * __dev_remove_offload - remove offload handler
486 * @po: packet offload declaration
487 *
488 * Remove a protocol offload handler that was previously added to the
489 * kernel offload handlers by dev_add_offload(). The passed &offload_type
490 * is removed from the kernel lists and can be freed or reused once this
491 * function returns.
492 *
493 * The packet type might still be in use by receivers
494 * and must not be freed until after all the CPU's have gone
495 * through a quiescent state.
496 */
stephen hemminger1d143d92013-12-29 14:01:29 -0800497static void __dev_remove_offload(struct packet_offload *po)
Vlad Yasevich62532da2012-11-15 08:49:10 +0000498{
499 struct list_head *head = &offload_base;
500 struct packet_offload *po1;
501
Eric Dumazetc53aa502012-11-16 08:08:23 +0000502 spin_lock(&offload_lock);
Vlad Yasevich62532da2012-11-15 08:49:10 +0000503
504 list_for_each_entry(po1, head, list) {
505 if (po == po1) {
506 list_del_rcu(&po->list);
507 goto out;
508 }
509 }
510
511 pr_warn("dev_remove_offload: %p not found\n", po);
512out:
Eric Dumazetc53aa502012-11-16 08:08:23 +0000513 spin_unlock(&offload_lock);
Vlad Yasevich62532da2012-11-15 08:49:10 +0000514}
Vlad Yasevich62532da2012-11-15 08:49:10 +0000515
516/**
517 * dev_remove_offload - remove packet offload handler
518 * @po: packet offload declaration
519 *
520 * Remove a packet offload handler that was previously added to the kernel
521 * offload handlers by dev_add_offload(). The passed &offload_type is
522 * removed from the kernel lists and can be freed or reused once this
523 * function returns.
524 *
525 * This call sleeps to guarantee that no CPU is looking at the packet
526 * type after return.
527 */
528void dev_remove_offload(struct packet_offload *po)
529{
530 __dev_remove_offload(po);
531
532 synchronize_net();
533}
534EXPORT_SYMBOL(dev_remove_offload);
535
Linus Torvalds1da177e2005-04-16 15:20:36 -0700536/******************************************************************************
537
538 Device Boot-time Settings Routines
539
540*******************************************************************************/
541
542/* Boot time configuration table */
543static struct netdev_boot_setup dev_boot_setup[NETDEV_BOOT_SETUP_MAX];
544
545/**
546 * netdev_boot_setup_add - add new setup entry
547 * @name: name of the device
548 * @map: configured settings for the device
549 *
550 * Adds new setup entry to the dev_boot_setup list. The function
551 * returns 0 on error and 1 on success. This is a generic routine to
552 * all netdevices.
553 */
554static int netdev_boot_setup_add(char *name, struct ifmap *map)
555{
556 struct netdev_boot_setup *s;
557 int i;
558
559 s = dev_boot_setup;
560 for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++) {
561 if (s[i].name[0] == '\0' || s[i].name[0] == ' ') {
562 memset(s[i].name, 0, sizeof(s[i].name));
Wang Chen93b3cff2008-07-01 19:57:19 -0700563 strlcpy(s[i].name, name, IFNAMSIZ);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700564 memcpy(&s[i].map, map, sizeof(s[i].map));
565 break;
566 }
567 }
568
569 return i >= NETDEV_BOOT_SETUP_MAX ? 0 : 1;
570}
571
572/**
573 * netdev_boot_setup_check - check boot time settings
574 * @dev: the netdevice
575 *
576 * Check boot time settings for the device.
577 * The found settings are set for the device to be used
578 * later in the device probing.
579 * Returns 0 if no settings found, 1 if they are.
580 */
581int netdev_boot_setup_check(struct net_device *dev)
582{
583 struct netdev_boot_setup *s = dev_boot_setup;
584 int i;
585
586 for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++) {
587 if (s[i].name[0] != '\0' && s[i].name[0] != ' ' &&
Wang Chen93b3cff2008-07-01 19:57:19 -0700588 !strcmp(dev->name, s[i].name)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700589 dev->irq = s[i].map.irq;
590 dev->base_addr = s[i].map.base_addr;
591 dev->mem_start = s[i].map.mem_start;
592 dev->mem_end = s[i].map.mem_end;
593 return 1;
594 }
595 }
596 return 0;
597}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700598EXPORT_SYMBOL(netdev_boot_setup_check);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700599
600
601/**
602 * netdev_boot_base - get address from boot time settings
603 * @prefix: prefix for network device
604 * @unit: id for network device
605 *
606 * Check boot time settings for the base address of device.
607 * The found settings are set for the device to be used
608 * later in the device probing.
609 * Returns 0 if no settings found.
610 */
611unsigned long netdev_boot_base(const char *prefix, int unit)
612{
613 const struct netdev_boot_setup *s = dev_boot_setup;
614 char name[IFNAMSIZ];
615 int i;
616
617 sprintf(name, "%s%d", prefix, unit);
618
619 /*
620 * If device already registered then return base of 1
621 * to indicate not to probe for this interface
622 */
Eric W. Biederman881d9662007-09-17 11:56:21 -0700623 if (__dev_get_by_name(&init_net, name))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700624 return 1;
625
626 for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++)
627 if (!strcmp(name, s[i].name))
628 return s[i].map.base_addr;
629 return 0;
630}
631
632/*
633 * Saves at boot time configured settings for any netdevice.
634 */
635int __init netdev_boot_setup(char *str)
636{
637 int ints[5];
638 struct ifmap map;
639
640 str = get_options(str, ARRAY_SIZE(ints), ints);
641 if (!str || !*str)
642 return 0;
643
644 /* Save settings */
645 memset(&map, 0, sizeof(map));
646 if (ints[0] > 0)
647 map.irq = ints[1];
648 if (ints[0] > 1)
649 map.base_addr = ints[2];
650 if (ints[0] > 2)
651 map.mem_start = ints[3];
652 if (ints[0] > 3)
653 map.mem_end = ints[4];
654
655 /* Add new entry to the list */
656 return netdev_boot_setup_add(str, &map);
657}
658
659__setup("netdev=", netdev_boot_setup);
660
661/*******************************************************************************
662
663 Device Interface Subroutines
664
665*******************************************************************************/
666
667/**
Nicolas Dichtela54acb32015-04-02 17:07:00 +0200668 * dev_get_iflink - get 'iflink' value of a interface
669 * @dev: targeted interface
670 *
671 * Indicates the ifindex the interface is linked to.
672 * Physical interfaces have the same 'ifindex' and 'iflink' values.
673 */
674
675int dev_get_iflink(const struct net_device *dev)
676{
677 if (dev->netdev_ops && dev->netdev_ops->ndo_get_iflink)
678 return dev->netdev_ops->ndo_get_iflink(dev);
679
Nicolas Dichtel7a66bbc2015-04-02 17:07:09 +0200680 return dev->ifindex;
Nicolas Dichtela54acb32015-04-02 17:07:00 +0200681}
682EXPORT_SYMBOL(dev_get_iflink);
683
684/**
Linus Torvalds1da177e2005-04-16 15:20:36 -0700685 * __dev_get_by_name - find a device by its name
Randy Dunlapc4ea43c2007-10-12 21:17:49 -0700686 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -0700687 * @name: name to find
688 *
689 * Find an interface by name. Must be called under RTNL semaphore
690 * or @dev_base_lock. If the name is found a pointer to the device
691 * is returned. If the name is not found then %NULL is returned. The
692 * reference counters are not incremented so the caller must be
693 * careful with locks.
694 */
695
Eric W. Biederman881d9662007-09-17 11:56:21 -0700696struct net_device *__dev_get_by_name(struct net *net, const char *name)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700697{
Eric Dumazet0bd8d532009-10-30 01:40:11 -0700698 struct net_device *dev;
699 struct hlist_head *head = dev_name_hash(net, name);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700700
Sasha Levinb67bfe02013-02-27 17:06:00 -0800701 hlist_for_each_entry(dev, head, name_hlist)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700702 if (!strncmp(dev->name, name, IFNAMSIZ))
703 return dev;
Eric Dumazet0bd8d532009-10-30 01:40:11 -0700704
Linus Torvalds1da177e2005-04-16 15:20:36 -0700705 return NULL;
706}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700707EXPORT_SYMBOL(__dev_get_by_name);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700708
709/**
Eric Dumazet72c95282009-10-30 07:11:27 +0000710 * dev_get_by_name_rcu - find a device by its name
711 * @net: the applicable net namespace
712 * @name: name to find
713 *
714 * Find an interface by name.
715 * If the name is found a pointer to the device is returned.
716 * If the name is not found then %NULL is returned.
717 * The reference counters are not incremented so the caller must be
718 * careful with locks. The caller must hold RCU lock.
719 */
720
721struct net_device *dev_get_by_name_rcu(struct net *net, const char *name)
722{
Eric Dumazet72c95282009-10-30 07:11:27 +0000723 struct net_device *dev;
724 struct hlist_head *head = dev_name_hash(net, name);
725
Sasha Levinb67bfe02013-02-27 17:06:00 -0800726 hlist_for_each_entry_rcu(dev, head, name_hlist)
Eric Dumazet72c95282009-10-30 07:11:27 +0000727 if (!strncmp(dev->name, name, IFNAMSIZ))
728 return dev;
729
730 return NULL;
731}
732EXPORT_SYMBOL(dev_get_by_name_rcu);
733
734/**
Linus Torvalds1da177e2005-04-16 15:20:36 -0700735 * dev_get_by_name - find a device by its name
Randy Dunlapc4ea43c2007-10-12 21:17:49 -0700736 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -0700737 * @name: name to find
738 *
739 * Find an interface by name. This can be called from any
740 * context and does its own locking. The returned handle has
741 * the usage count incremented and the caller must use dev_put() to
742 * release it when it is no longer needed. %NULL is returned if no
743 * matching device is found.
744 */
745
Eric W. Biederman881d9662007-09-17 11:56:21 -0700746struct net_device *dev_get_by_name(struct net *net, const char *name)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700747{
748 struct net_device *dev;
749
Eric Dumazet72c95282009-10-30 07:11:27 +0000750 rcu_read_lock();
751 dev = dev_get_by_name_rcu(net, name);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700752 if (dev)
753 dev_hold(dev);
Eric Dumazet72c95282009-10-30 07:11:27 +0000754 rcu_read_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700755 return dev;
756}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700757EXPORT_SYMBOL(dev_get_by_name);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700758
759/**
760 * __dev_get_by_index - find a device by its ifindex
Randy Dunlapc4ea43c2007-10-12 21:17:49 -0700761 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -0700762 * @ifindex: index of device
763 *
764 * Search for an interface by index. Returns %NULL if the device
765 * is not found or a pointer to the device. The device has not
766 * had its reference counter increased so the caller must be careful
767 * about locking. The caller must hold either the RTNL semaphore
768 * or @dev_base_lock.
769 */
770
Eric W. Biederman881d9662007-09-17 11:56:21 -0700771struct net_device *__dev_get_by_index(struct net *net, int ifindex)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700772{
Eric Dumazet0bd8d532009-10-30 01:40:11 -0700773 struct net_device *dev;
774 struct hlist_head *head = dev_index_hash(net, ifindex);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700775
Sasha Levinb67bfe02013-02-27 17:06:00 -0800776 hlist_for_each_entry(dev, head, index_hlist)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700777 if (dev->ifindex == ifindex)
778 return dev;
Eric Dumazet0bd8d532009-10-30 01:40:11 -0700779
Linus Torvalds1da177e2005-04-16 15:20:36 -0700780 return NULL;
781}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700782EXPORT_SYMBOL(__dev_get_by_index);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700783
Eric Dumazetfb699dfd2009-10-19 19:18:49 +0000784/**
785 * dev_get_by_index_rcu - find a device by its ifindex
786 * @net: the applicable net namespace
787 * @ifindex: index of device
788 *
789 * Search for an interface by index. Returns %NULL if the device
790 * is not found or a pointer to the device. The device has not
791 * had its reference counter increased so the caller must be careful
792 * about locking. The caller must hold RCU lock.
793 */
794
795struct net_device *dev_get_by_index_rcu(struct net *net, int ifindex)
796{
Eric Dumazetfb699dfd2009-10-19 19:18:49 +0000797 struct net_device *dev;
798 struct hlist_head *head = dev_index_hash(net, ifindex);
799
Sasha Levinb67bfe02013-02-27 17:06:00 -0800800 hlist_for_each_entry_rcu(dev, head, index_hlist)
Eric Dumazetfb699dfd2009-10-19 19:18:49 +0000801 if (dev->ifindex == ifindex)
802 return dev;
803
804 return NULL;
805}
806EXPORT_SYMBOL(dev_get_by_index_rcu);
807
Linus Torvalds1da177e2005-04-16 15:20:36 -0700808
809/**
810 * dev_get_by_index - find a device by its ifindex
Randy Dunlapc4ea43c2007-10-12 21:17:49 -0700811 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -0700812 * @ifindex: index of device
813 *
814 * Search for an interface by index. Returns NULL if the device
815 * is not found or a pointer to the device. The device returned has
816 * had a reference added and the pointer is safe until the user calls
817 * dev_put to indicate they have finished with it.
818 */
819
Eric W. Biederman881d9662007-09-17 11:56:21 -0700820struct net_device *dev_get_by_index(struct net *net, int ifindex)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700821{
822 struct net_device *dev;
823
Eric Dumazetfb699dfd2009-10-19 19:18:49 +0000824 rcu_read_lock();
825 dev = dev_get_by_index_rcu(net, ifindex);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700826 if (dev)
827 dev_hold(dev);
Eric Dumazetfb699dfd2009-10-19 19:18:49 +0000828 rcu_read_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700829 return dev;
830}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700831EXPORT_SYMBOL(dev_get_by_index);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700832
833/**
Nicolas Schichan5dbe7c12013-06-26 17:23:42 +0200834 * netdev_get_name - get a netdevice name, knowing its ifindex.
835 * @net: network namespace
836 * @name: a pointer to the buffer where the name will be stored.
837 * @ifindex: the ifindex of the interface to get the name from.
838 *
839 * The use of raw_seqcount_begin() and cond_resched() before
840 * retrying is required as we want to give the writers a chance
841 * to complete when CONFIG_PREEMPT is not set.
842 */
843int netdev_get_name(struct net *net, char *name, int ifindex)
844{
845 struct net_device *dev;
846 unsigned int seq;
847
848retry:
849 seq = raw_seqcount_begin(&devnet_rename_seq);
850 rcu_read_lock();
851 dev = dev_get_by_index_rcu(net, ifindex);
852 if (!dev) {
853 rcu_read_unlock();
854 return -ENODEV;
855 }
856
857 strcpy(name, dev->name);
858 rcu_read_unlock();
859 if (read_seqcount_retry(&devnet_rename_seq, seq)) {
860 cond_resched();
861 goto retry;
862 }
863
864 return 0;
865}
866
867/**
Eric Dumazet941666c2010-12-05 01:23:53 +0000868 * dev_getbyhwaddr_rcu - find a device by its hardware address
Randy Dunlapc4ea43c2007-10-12 21:17:49 -0700869 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -0700870 * @type: media type of device
871 * @ha: hardware address
872 *
873 * Search for an interface by MAC address. Returns NULL if the device
Eric Dumazetc5066532011-01-24 13:16:16 -0800874 * is not found or a pointer to the device.
875 * The caller must hold RCU or RTNL.
Eric Dumazet941666c2010-12-05 01:23:53 +0000876 * The returned device has not had its ref count increased
Linus Torvalds1da177e2005-04-16 15:20:36 -0700877 * and the caller must therefore be careful about locking
878 *
Linus Torvalds1da177e2005-04-16 15:20:36 -0700879 */
880
Eric Dumazet941666c2010-12-05 01:23:53 +0000881struct net_device *dev_getbyhwaddr_rcu(struct net *net, unsigned short type,
882 const char *ha)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700883{
884 struct net_device *dev;
885
Eric Dumazet941666c2010-12-05 01:23:53 +0000886 for_each_netdev_rcu(net, dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700887 if (dev->type == type &&
888 !memcmp(dev->dev_addr, ha, dev->addr_len))
Pavel Emelianov7562f872007-05-03 15:13:45 -0700889 return dev;
890
891 return NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700892}
Eric Dumazet941666c2010-12-05 01:23:53 +0000893EXPORT_SYMBOL(dev_getbyhwaddr_rcu);
Jochen Friedrichcf309e32005-09-22 04:44:55 -0300894
Eric W. Biederman881d9662007-09-17 11:56:21 -0700895struct net_device *__dev_getfirstbyhwtype(struct net *net, unsigned short type)
Patrick McHardy4e9cac22007-05-03 03:28:13 -0700896{
897 struct net_device *dev;
898
899 ASSERT_RTNL();
Eric W. Biederman881d9662007-09-17 11:56:21 -0700900 for_each_netdev(net, dev)
Patrick McHardy4e9cac22007-05-03 03:28:13 -0700901 if (dev->type == type)
Pavel Emelianov7562f872007-05-03 15:13:45 -0700902 return dev;
903
904 return NULL;
Patrick McHardy4e9cac22007-05-03 03:28:13 -0700905}
Patrick McHardy4e9cac22007-05-03 03:28:13 -0700906EXPORT_SYMBOL(__dev_getfirstbyhwtype);
907
Eric W. Biederman881d9662007-09-17 11:56:21 -0700908struct net_device *dev_getfirstbyhwtype(struct net *net, unsigned short type)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700909{
Eric Dumazet99fe3c32010-03-18 11:27:25 +0000910 struct net_device *dev, *ret = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700911
Eric Dumazet99fe3c32010-03-18 11:27:25 +0000912 rcu_read_lock();
913 for_each_netdev_rcu(net, dev)
914 if (dev->type == type) {
915 dev_hold(dev);
916 ret = dev;
917 break;
918 }
919 rcu_read_unlock();
920 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700921}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700922EXPORT_SYMBOL(dev_getfirstbyhwtype);
923
924/**
WANG Cong6c555492014-09-11 15:35:09 -0700925 * __dev_get_by_flags - find any device with given flags
Randy Dunlapc4ea43c2007-10-12 21:17:49 -0700926 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -0700927 * @if_flags: IFF_* values
928 * @mask: bitmask of bits in if_flags to check
929 *
930 * Search for any interface with the given flags. Returns NULL if a device
Eric Dumazetbb69ae02010-06-07 11:42:13 +0000931 * is not found or a pointer to the device. Must be called inside
WANG Cong6c555492014-09-11 15:35:09 -0700932 * rtnl_lock(), and result refcount is unchanged.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700933 */
934
WANG Cong6c555492014-09-11 15:35:09 -0700935struct net_device *__dev_get_by_flags(struct net *net, unsigned short if_flags,
936 unsigned short mask)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700937{
Pavel Emelianov7562f872007-05-03 15:13:45 -0700938 struct net_device *dev, *ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700939
WANG Cong6c555492014-09-11 15:35:09 -0700940 ASSERT_RTNL();
941
Pavel Emelianov7562f872007-05-03 15:13:45 -0700942 ret = NULL;
WANG Cong6c555492014-09-11 15:35:09 -0700943 for_each_netdev(net, dev) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700944 if (((dev->flags ^ if_flags) & mask) == 0) {
Pavel Emelianov7562f872007-05-03 15:13:45 -0700945 ret = dev;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700946 break;
947 }
948 }
Pavel Emelianov7562f872007-05-03 15:13:45 -0700949 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700950}
WANG Cong6c555492014-09-11 15:35:09 -0700951EXPORT_SYMBOL(__dev_get_by_flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700952
953/**
954 * dev_valid_name - check if name is okay for network device
955 * @name: name string
956 *
957 * Network device names need to be valid file names to
David S. Millerc7fa9d12006-08-15 16:34:13 -0700958 * to allow sysfs to work. We also disallow any kind of
959 * whitespace.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700960 */
David S. Miller95f050b2012-03-06 16:12:15 -0500961bool dev_valid_name(const char *name)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700962{
David S. Millerc7fa9d12006-08-15 16:34:13 -0700963 if (*name == '\0')
David S. Miller95f050b2012-03-06 16:12:15 -0500964 return false;
Stephen Hemmingerb6fe17d2006-08-29 17:06:13 -0700965 if (strlen(name) >= IFNAMSIZ)
David S. Miller95f050b2012-03-06 16:12:15 -0500966 return false;
David S. Millerc7fa9d12006-08-15 16:34:13 -0700967 if (!strcmp(name, ".") || !strcmp(name, ".."))
David S. Miller95f050b2012-03-06 16:12:15 -0500968 return false;
David S. Millerc7fa9d12006-08-15 16:34:13 -0700969
970 while (*name) {
Matthew Thodea4176a92015-02-17 18:31:57 -0600971 if (*name == '/' || *name == ':' || isspace(*name))
David S. Miller95f050b2012-03-06 16:12:15 -0500972 return false;
David S. Millerc7fa9d12006-08-15 16:34:13 -0700973 name++;
974 }
David S. Miller95f050b2012-03-06 16:12:15 -0500975 return true;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700976}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700977EXPORT_SYMBOL(dev_valid_name);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700978
979/**
Eric W. Biedermanb267b172007-09-12 13:48:45 +0200980 * __dev_alloc_name - allocate a name for a device
981 * @net: network namespace to allocate the device name in
Linus Torvalds1da177e2005-04-16 15:20:36 -0700982 * @name: name format string
Eric W. Biedermanb267b172007-09-12 13:48:45 +0200983 * @buf: scratch buffer and result name string
Linus Torvalds1da177e2005-04-16 15:20:36 -0700984 *
985 * Passed a format string - eg "lt%d" it will try and find a suitable
Stephen Hemminger3041a062006-05-26 13:25:24 -0700986 * id. It scans list of devices to build up a free map, then chooses
987 * the first empty slot. The caller must hold the dev_base or rtnl lock
988 * while allocating the name and adding the device in order to avoid
989 * duplicates.
990 * Limited to bits_per_byte * page size devices (ie 32K on most platforms).
991 * Returns the number of the unit assigned or a negative errno code.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700992 */
993
Eric W. Biedermanb267b172007-09-12 13:48:45 +0200994static int __dev_alloc_name(struct net *net, const char *name, char *buf)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700995{
996 int i = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700997 const char *p;
998 const int max_netdevices = 8*PAGE_SIZE;
Stephen Hemmingercfcabdc2007-10-09 01:59:42 -0700999 unsigned long *inuse;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001000 struct net_device *d;
1001
1002 p = strnchr(name, IFNAMSIZ-1, '%');
1003 if (p) {
1004 /*
1005 * Verify the string as this thing may have come from
1006 * the user. There must be either one "%d" and no other "%"
1007 * characters.
1008 */
1009 if (p[1] != 'd' || strchr(p + 2, '%'))
1010 return -EINVAL;
1011
1012 /* Use one page as a bit array of possible slots */
Stephen Hemmingercfcabdc2007-10-09 01:59:42 -07001013 inuse = (unsigned long *) get_zeroed_page(GFP_ATOMIC);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001014 if (!inuse)
1015 return -ENOMEM;
1016
Eric W. Biederman881d9662007-09-17 11:56:21 -07001017 for_each_netdev(net, d) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001018 if (!sscanf(d->name, name, &i))
1019 continue;
1020 if (i < 0 || i >= max_netdevices)
1021 continue;
1022
1023 /* avoid cases where sscanf is not exact inverse of printf */
Eric W. Biedermanb267b172007-09-12 13:48:45 +02001024 snprintf(buf, IFNAMSIZ, name, i);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001025 if (!strncmp(buf, d->name, IFNAMSIZ))
1026 set_bit(i, inuse);
1027 }
1028
1029 i = find_first_zero_bit(inuse, max_netdevices);
1030 free_page((unsigned long) inuse);
1031 }
1032
Octavian Purdilad9031022009-11-18 02:36:59 +00001033 if (buf != name)
1034 snprintf(buf, IFNAMSIZ, name, i);
Eric W. Biedermanb267b172007-09-12 13:48:45 +02001035 if (!__dev_get_by_name(net, buf))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001036 return i;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001037
1038 /* It is possible to run out of possible slots
1039 * when the name is long and there isn't enough space left
1040 * for the digits, or if all bits are used.
1041 */
1042 return -ENFILE;
1043}
1044
Eric W. Biedermanb267b172007-09-12 13:48:45 +02001045/**
1046 * dev_alloc_name - allocate a name for a device
1047 * @dev: device
1048 * @name: name format string
1049 *
1050 * Passed a format string - eg "lt%d" it will try and find a suitable
1051 * id. It scans list of devices to build up a free map, then chooses
1052 * the first empty slot. The caller must hold the dev_base or rtnl lock
1053 * while allocating the name and adding the device in order to avoid
1054 * duplicates.
1055 * Limited to bits_per_byte * page size devices (ie 32K on most platforms).
1056 * Returns the number of the unit assigned or a negative errno code.
1057 */
1058
1059int dev_alloc_name(struct net_device *dev, const char *name)
1060{
1061 char buf[IFNAMSIZ];
1062 struct net *net;
1063 int ret;
1064
YOSHIFUJI Hideakic346dca2008-03-25 21:47:49 +09001065 BUG_ON(!dev_net(dev));
1066 net = dev_net(dev);
Eric W. Biedermanb267b172007-09-12 13:48:45 +02001067 ret = __dev_alloc_name(net, name, buf);
1068 if (ret >= 0)
1069 strlcpy(dev->name, buf, IFNAMSIZ);
1070 return ret;
1071}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001072EXPORT_SYMBOL(dev_alloc_name);
Eric W. Biedermanb267b172007-09-12 13:48:45 +02001073
Gao feng828de4f2012-09-13 20:58:27 +00001074static int dev_alloc_name_ns(struct net *net,
1075 struct net_device *dev,
1076 const char *name)
Octavian Purdilad9031022009-11-18 02:36:59 +00001077{
Gao feng828de4f2012-09-13 20:58:27 +00001078 char buf[IFNAMSIZ];
1079 int ret;
Daniel Lezcano8ce6cebc2010-05-19 10:12:19 +00001080
Gao feng828de4f2012-09-13 20:58:27 +00001081 ret = __dev_alloc_name(net, name, buf);
1082 if (ret >= 0)
1083 strlcpy(dev->name, buf, IFNAMSIZ);
1084 return ret;
1085}
1086
1087static int dev_get_valid_name(struct net *net,
1088 struct net_device *dev,
1089 const char *name)
1090{
1091 BUG_ON(!net);
Daniel Lezcano8ce6cebc2010-05-19 10:12:19 +00001092
Octavian Purdilad9031022009-11-18 02:36:59 +00001093 if (!dev_valid_name(name))
1094 return -EINVAL;
1095
Jiri Pirko1c5cae82011-04-30 01:21:32 +00001096 if (strchr(name, '%'))
Gao feng828de4f2012-09-13 20:58:27 +00001097 return dev_alloc_name_ns(net, dev, name);
Octavian Purdilad9031022009-11-18 02:36:59 +00001098 else if (__dev_get_by_name(net, name))
1099 return -EEXIST;
Daniel Lezcano8ce6cebc2010-05-19 10:12:19 +00001100 else if (dev->name != name)
1101 strlcpy(dev->name, name, IFNAMSIZ);
Octavian Purdilad9031022009-11-18 02:36:59 +00001102
1103 return 0;
1104}
Linus Torvalds1da177e2005-04-16 15:20:36 -07001105
1106/**
1107 * dev_change_name - change name of a device
1108 * @dev: device
1109 * @newname: name (or format string) must be at least IFNAMSIZ
1110 *
1111 * Change name of a device, can pass format strings "eth%d".
1112 * for wildcarding.
1113 */
Stephen Hemmingercf04a4c72008-09-30 02:22:14 -07001114int dev_change_name(struct net_device *dev, const char *newname)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001115{
Tom Gundersen238fa362014-07-14 16:37:23 +02001116 unsigned char old_assign_type;
Herbert Xufcc5a032007-07-30 17:03:38 -07001117 char oldname[IFNAMSIZ];
Linus Torvalds1da177e2005-04-16 15:20:36 -07001118 int err = 0;
Herbert Xufcc5a032007-07-30 17:03:38 -07001119 int ret;
Eric W. Biederman881d9662007-09-17 11:56:21 -07001120 struct net *net;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001121
1122 ASSERT_RTNL();
YOSHIFUJI Hideakic346dca2008-03-25 21:47:49 +09001123 BUG_ON(!dev_net(dev));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001124
YOSHIFUJI Hideakic346dca2008-03-25 21:47:49 +09001125 net = dev_net(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001126 if (dev->flags & IFF_UP)
1127 return -EBUSY;
1128
Eric Dumazet30e6c9f2012-12-20 17:25:08 +00001129 write_seqcount_begin(&devnet_rename_seq);
Brian Haleyc91f6df2012-11-26 05:21:08 +00001130
1131 if (strncmp(newname, dev->name, IFNAMSIZ) == 0) {
Eric Dumazet30e6c9f2012-12-20 17:25:08 +00001132 write_seqcount_end(&devnet_rename_seq);
Stephen Hemmingerc8d90dc2007-10-26 03:53:42 -07001133 return 0;
Brian Haleyc91f6df2012-11-26 05:21:08 +00001134 }
Stephen Hemmingerc8d90dc2007-10-26 03:53:42 -07001135
Herbert Xufcc5a032007-07-30 17:03:38 -07001136 memcpy(oldname, dev->name, IFNAMSIZ);
1137
Gao feng828de4f2012-09-13 20:58:27 +00001138 err = dev_get_valid_name(net, dev, newname);
Brian Haleyc91f6df2012-11-26 05:21:08 +00001139 if (err < 0) {
Eric Dumazet30e6c9f2012-12-20 17:25:08 +00001140 write_seqcount_end(&devnet_rename_seq);
Octavian Purdilad9031022009-11-18 02:36:59 +00001141 return err;
Brian Haleyc91f6df2012-11-26 05:21:08 +00001142 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001143
Veaceslav Falico6fe82a32014-07-17 20:33:32 +02001144 if (oldname[0] && !strchr(oldname, '%'))
1145 netdev_info(dev, "renamed from %s\n", oldname);
1146
Tom Gundersen238fa362014-07-14 16:37:23 +02001147 old_assign_type = dev->name_assign_type;
1148 dev->name_assign_type = NET_NAME_RENAMED;
1149
Herbert Xufcc5a032007-07-30 17:03:38 -07001150rollback:
Eric W. Biedermana1b3f592010-05-04 17:36:49 -07001151 ret = device_rename(&dev->dev, dev->name);
1152 if (ret) {
1153 memcpy(dev->name, oldname, IFNAMSIZ);
Tom Gundersen238fa362014-07-14 16:37:23 +02001154 dev->name_assign_type = old_assign_type;
Eric Dumazet30e6c9f2012-12-20 17:25:08 +00001155 write_seqcount_end(&devnet_rename_seq);
Eric W. Biedermana1b3f592010-05-04 17:36:49 -07001156 return ret;
Stephen Hemmingerdcc99772008-05-14 22:33:38 -07001157 }
Herbert Xu7f988ea2007-07-30 16:35:46 -07001158
Eric Dumazet30e6c9f2012-12-20 17:25:08 +00001159 write_seqcount_end(&devnet_rename_seq);
Brian Haleyc91f6df2012-11-26 05:21:08 +00001160
Veaceslav Falico5bb025f2014-01-14 21:58:51 +01001161 netdev_adjacent_rename_links(dev, oldname);
1162
Herbert Xu7f988ea2007-07-30 16:35:46 -07001163 write_lock_bh(&dev_base_lock);
Eric Dumazet372b2312011-05-17 13:56:59 -04001164 hlist_del_rcu(&dev->name_hlist);
Eric Dumazet72c95282009-10-30 07:11:27 +00001165 write_unlock_bh(&dev_base_lock);
1166
1167 synchronize_rcu();
1168
1169 write_lock_bh(&dev_base_lock);
1170 hlist_add_head_rcu(&dev->name_hlist, dev_name_hash(net, dev->name));
Herbert Xu7f988ea2007-07-30 16:35:46 -07001171 write_unlock_bh(&dev_base_lock);
1172
Pavel Emelyanov056925a2007-09-16 15:42:43 -07001173 ret = call_netdevice_notifiers(NETDEV_CHANGENAME, dev);
Herbert Xufcc5a032007-07-30 17:03:38 -07001174 ret = notifier_to_errno(ret);
1175
1176 if (ret) {
Eric Dumazet91e9c07b2009-11-15 23:30:24 +00001177 /* err >= 0 after dev_alloc_name() or stores the first errno */
1178 if (err >= 0) {
Herbert Xufcc5a032007-07-30 17:03:38 -07001179 err = ret;
Eric Dumazet30e6c9f2012-12-20 17:25:08 +00001180 write_seqcount_begin(&devnet_rename_seq);
Herbert Xufcc5a032007-07-30 17:03:38 -07001181 memcpy(dev->name, oldname, IFNAMSIZ);
Veaceslav Falico5bb025f2014-01-14 21:58:51 +01001182 memcpy(oldname, newname, IFNAMSIZ);
Tom Gundersen238fa362014-07-14 16:37:23 +02001183 dev->name_assign_type = old_assign_type;
1184 old_assign_type = NET_NAME_RENAMED;
Herbert Xufcc5a032007-07-30 17:03:38 -07001185 goto rollback;
Eric Dumazet91e9c07b2009-11-15 23:30:24 +00001186 } else {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00001187 pr_err("%s: name change rollback failed: %d\n",
Eric Dumazet91e9c07b2009-11-15 23:30:24 +00001188 dev->name, ret);
Herbert Xufcc5a032007-07-30 17:03:38 -07001189 }
1190 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001191
1192 return err;
1193}
1194
1195/**
Stephen Hemminger0b815a12008-09-22 21:28:11 -07001196 * dev_set_alias - change ifalias of a device
1197 * @dev: device
1198 * @alias: name up to IFALIASZ
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07001199 * @len: limit of bytes to copy from info
Stephen Hemminger0b815a12008-09-22 21:28:11 -07001200 *
1201 * Set ifalias for a device,
1202 */
1203int dev_set_alias(struct net_device *dev, const char *alias, size_t len)
1204{
Alexey Khoroshilov7364e442012-08-08 00:33:25 +00001205 char *new_ifalias;
1206
Stephen Hemminger0b815a12008-09-22 21:28:11 -07001207 ASSERT_RTNL();
1208
1209 if (len >= IFALIASZ)
1210 return -EINVAL;
1211
Oliver Hartkopp96ca4a22008-09-23 21:23:19 -07001212 if (!len) {
Sachin Kamat388dfc22012-11-20 00:57:04 +00001213 kfree(dev->ifalias);
1214 dev->ifalias = NULL;
Oliver Hartkopp96ca4a22008-09-23 21:23:19 -07001215 return 0;
1216 }
1217
Alexey Khoroshilov7364e442012-08-08 00:33:25 +00001218 new_ifalias = krealloc(dev->ifalias, len + 1, GFP_KERNEL);
1219 if (!new_ifalias)
Stephen Hemminger0b815a12008-09-22 21:28:11 -07001220 return -ENOMEM;
Alexey Khoroshilov7364e442012-08-08 00:33:25 +00001221 dev->ifalias = new_ifalias;
Stephen Hemminger0b815a12008-09-22 21:28:11 -07001222
1223 strlcpy(dev->ifalias, alias, len+1);
1224 return len;
1225}
1226
1227
1228/**
Stephen Hemminger3041a062006-05-26 13:25:24 -07001229 * netdev_features_change - device changes features
Stephen Hemmingerd8a33ac2005-05-29 14:13:47 -07001230 * @dev: device to cause notification
1231 *
1232 * Called to indicate a device has changed features.
1233 */
1234void netdev_features_change(struct net_device *dev)
1235{
Pavel Emelyanov056925a2007-09-16 15:42:43 -07001236 call_netdevice_notifiers(NETDEV_FEAT_CHANGE, dev);
Stephen Hemmingerd8a33ac2005-05-29 14:13:47 -07001237}
1238EXPORT_SYMBOL(netdev_features_change);
1239
1240/**
Linus Torvalds1da177e2005-04-16 15:20:36 -07001241 * netdev_state_change - device changes state
1242 * @dev: device to cause notification
1243 *
1244 * Called to indicate a device has changed state. This function calls
1245 * the notifier chains for netdev_chain and sends a NEWLINK message
1246 * to the routing socket.
1247 */
1248void netdev_state_change(struct net_device *dev)
1249{
1250 if (dev->flags & IFF_UP) {
Loic Prylli54951192014-07-01 21:39:43 -07001251 struct netdev_notifier_change_info change_info;
1252
1253 change_info.flags_changed = 0;
1254 call_netdevice_notifiers_info(NETDEV_CHANGE, dev,
1255 &change_info.info);
Alexei Starovoitov7f294052013-10-23 16:02:42 -07001256 rtmsg_ifinfo(RTM_NEWLINK, dev, 0, GFP_KERNEL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001257 }
1258}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001259EXPORT_SYMBOL(netdev_state_change);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001260
Amerigo Wangee89bab2012-08-09 22:14:56 +00001261/**
1262 * netdev_notify_peers - notify network peers about existence of @dev
1263 * @dev: network device
1264 *
1265 * Generate traffic such that interested network peers are aware of
1266 * @dev, such as by generating a gratuitous ARP. This may be used when
1267 * a device wants to inform the rest of the network about some sort of
1268 * reconfiguration such as a failover event or virtual machine
1269 * migration.
1270 */
1271void netdev_notify_peers(struct net_device *dev)
Or Gerlitzc1da4ac2008-06-13 18:12:00 -07001272{
Amerigo Wangee89bab2012-08-09 22:14:56 +00001273 rtnl_lock();
1274 call_netdevice_notifiers(NETDEV_NOTIFY_PEERS, dev);
1275 rtnl_unlock();
Or Gerlitzc1da4ac2008-06-13 18:12:00 -07001276}
Amerigo Wangee89bab2012-08-09 22:14:56 +00001277EXPORT_SYMBOL(netdev_notify_peers);
Or Gerlitzc1da4ac2008-06-13 18:12:00 -07001278
Patrick McHardybd380812010-02-26 06:34:53 +00001279static int __dev_open(struct net_device *dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001280{
Stephen Hemmingerd3147742008-11-19 21:32:24 -08001281 const struct net_device_ops *ops = dev->netdev_ops;
Johannes Berg3b8bcfd2009-05-30 01:39:53 +02001282 int ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001283
Ben Hutchingse46b66b2008-05-08 02:53:17 -07001284 ASSERT_RTNL();
1285
Linus Torvalds1da177e2005-04-16 15:20:36 -07001286 if (!netif_device_present(dev))
1287 return -ENODEV;
1288
Neil Hormanca99ca12013-02-05 08:05:43 +00001289 /* Block netpoll from trying to do any rx path servicing.
1290 * If we don't do this there is a chance ndo_poll_controller
1291 * or ndo_poll may be running while we open the device
1292 */
Eric W. Biederman66b55522014-03-27 15:39:03 -07001293 netpoll_poll_disable(dev);
Neil Hormanca99ca12013-02-05 08:05:43 +00001294
Johannes Berg3b8bcfd2009-05-30 01:39:53 +02001295 ret = call_netdevice_notifiers(NETDEV_PRE_UP, dev);
1296 ret = notifier_to_errno(ret);
1297 if (ret)
1298 return ret;
1299
Linus Torvalds1da177e2005-04-16 15:20:36 -07001300 set_bit(__LINK_STATE_START, &dev->state);
Jeff Garzikbada3392007-10-23 20:19:37 -07001301
Stephen Hemmingerd3147742008-11-19 21:32:24 -08001302 if (ops->ndo_validate_addr)
1303 ret = ops->ndo_validate_addr(dev);
Jeff Garzikbada3392007-10-23 20:19:37 -07001304
Stephen Hemmingerd3147742008-11-19 21:32:24 -08001305 if (!ret && ops->ndo_open)
1306 ret = ops->ndo_open(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001307
Eric W. Biederman66b55522014-03-27 15:39:03 -07001308 netpoll_poll_enable(dev);
Neil Hormanca99ca12013-02-05 08:05:43 +00001309
Jeff Garzikbada3392007-10-23 20:19:37 -07001310 if (ret)
1311 clear_bit(__LINK_STATE_START, &dev->state);
1312 else {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001313 dev->flags |= IFF_UP;
Patrick McHardy4417da62007-06-27 01:28:10 -07001314 dev_set_rx_mode(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001315 dev_activate(dev);
Theodore Ts'o7bf23572012-07-04 21:23:25 -04001316 add_device_randomness(dev->dev_addr, dev->addr_len);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001317 }
Jeff Garzikbada3392007-10-23 20:19:37 -07001318
Linus Torvalds1da177e2005-04-16 15:20:36 -07001319 return ret;
1320}
Patrick McHardybd380812010-02-26 06:34:53 +00001321
1322/**
1323 * dev_open - prepare an interface for use.
1324 * @dev: device to open
1325 *
1326 * Takes a device from down to up state. The device's private open
1327 * function is invoked and then the multicast lists are loaded. Finally
1328 * the device is moved into the up state and a %NETDEV_UP message is
1329 * sent to the netdev notifier chain.
1330 *
1331 * Calling this function on an active interface is a nop. On a failure
1332 * a negative errno code is returned.
1333 */
1334int dev_open(struct net_device *dev)
1335{
1336 int ret;
1337
Patrick McHardybd380812010-02-26 06:34:53 +00001338 if (dev->flags & IFF_UP)
1339 return 0;
1340
Patrick McHardybd380812010-02-26 06:34:53 +00001341 ret = __dev_open(dev);
1342 if (ret < 0)
1343 return ret;
1344
Alexei Starovoitov7f294052013-10-23 16:02:42 -07001345 rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP|IFF_RUNNING, GFP_KERNEL);
Patrick McHardybd380812010-02-26 06:34:53 +00001346 call_netdevice_notifiers(NETDEV_UP, dev);
1347
1348 return ret;
1349}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001350EXPORT_SYMBOL(dev_open);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001351
Octavian Purdila44345722010-12-13 12:44:07 +00001352static int __dev_close_many(struct list_head *head)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001353{
Octavian Purdila44345722010-12-13 12:44:07 +00001354 struct net_device *dev;
Patrick McHardybd380812010-02-26 06:34:53 +00001355
Ben Hutchingse46b66b2008-05-08 02:53:17 -07001356 ASSERT_RTNL();
David S. Miller9d5010d2007-09-12 14:33:25 +02001357 might_sleep();
1358
Eric W. Biederman5cde2822013-10-05 19:26:05 -07001359 list_for_each_entry(dev, head, close_list) {
Eric W. Biederman3f4df202014-03-27 15:38:17 -07001360 /* Temporarily disable netpoll until the interface is down */
Eric W. Biederman66b55522014-03-27 15:39:03 -07001361 netpoll_poll_disable(dev);
Eric W. Biederman3f4df202014-03-27 15:38:17 -07001362
Octavian Purdila44345722010-12-13 12:44:07 +00001363 call_netdevice_notifiers(NETDEV_GOING_DOWN, dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001364
Octavian Purdila44345722010-12-13 12:44:07 +00001365 clear_bit(__LINK_STATE_START, &dev->state);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001366
Octavian Purdila44345722010-12-13 12:44:07 +00001367 /* Synchronize to scheduled poll. We cannot touch poll list, it
1368 * can be even on different cpu. So just clear netif_running().
1369 *
1370 * dev->stop() will invoke napi_disable() on all of it's
1371 * napi_struct instances on this device.
1372 */
Peter Zijlstra4e857c52014-03-17 18:06:10 +01001373 smp_mb__after_atomic(); /* Commit netif_running(). */
Octavian Purdila44345722010-12-13 12:44:07 +00001374 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001375
Octavian Purdila44345722010-12-13 12:44:07 +00001376 dev_deactivate_many(head);
1377
Eric W. Biederman5cde2822013-10-05 19:26:05 -07001378 list_for_each_entry(dev, head, close_list) {
Octavian Purdila44345722010-12-13 12:44:07 +00001379 const struct net_device_ops *ops = dev->netdev_ops;
1380
1381 /*
1382 * Call the device specific close. This cannot fail.
1383 * Only if device is UP
1384 *
1385 * We allow it to be called even after a DETACH hot-plug
1386 * event.
1387 */
1388 if (ops->ndo_stop)
1389 ops->ndo_stop(dev);
1390
Octavian Purdila44345722010-12-13 12:44:07 +00001391 dev->flags &= ~IFF_UP;
Eric W. Biederman66b55522014-03-27 15:39:03 -07001392 netpoll_poll_enable(dev);
Octavian Purdila44345722010-12-13 12:44:07 +00001393 }
1394
1395 return 0;
1396}
1397
1398static int __dev_close(struct net_device *dev)
1399{
Linus Torvaldsf87e6f42011-02-17 22:54:38 +00001400 int retval;
Octavian Purdila44345722010-12-13 12:44:07 +00001401 LIST_HEAD(single);
1402
Eric W. Biederman5cde2822013-10-05 19:26:05 -07001403 list_add(&dev->close_list, &single);
Linus Torvaldsf87e6f42011-02-17 22:54:38 +00001404 retval = __dev_close_many(&single);
1405 list_del(&single);
Neil Hormanca99ca12013-02-05 08:05:43 +00001406
Linus Torvaldsf87e6f42011-02-17 22:54:38 +00001407 return retval;
Octavian Purdila44345722010-12-13 12:44:07 +00001408}
1409
David S. Miller99c4a262015-03-18 22:52:33 -04001410int dev_close_many(struct list_head *head, bool unlink)
Octavian Purdila44345722010-12-13 12:44:07 +00001411{
1412 struct net_device *dev, *tmp;
Octavian Purdila44345722010-12-13 12:44:07 +00001413
Eric W. Biederman5cde2822013-10-05 19:26:05 -07001414 /* Remove the devices that don't need to be closed */
1415 list_for_each_entry_safe(dev, tmp, head, close_list)
Octavian Purdila44345722010-12-13 12:44:07 +00001416 if (!(dev->flags & IFF_UP))
Eric W. Biederman5cde2822013-10-05 19:26:05 -07001417 list_del_init(&dev->close_list);
Octavian Purdila44345722010-12-13 12:44:07 +00001418
1419 __dev_close_many(head);
Matti Linnanvuorid8b2a4d2008-02-12 23:10:11 -08001420
Eric W. Biederman5cde2822013-10-05 19:26:05 -07001421 list_for_each_entry_safe(dev, tmp, head, close_list) {
Alexei Starovoitov7f294052013-10-23 16:02:42 -07001422 rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP|IFF_RUNNING, GFP_KERNEL);
Octavian Purdila44345722010-12-13 12:44:07 +00001423 call_netdevice_notifiers(NETDEV_DOWN, dev);
David S. Miller99c4a262015-03-18 22:52:33 -04001424 if (unlink)
1425 list_del_init(&dev->close_list);
Octavian Purdila44345722010-12-13 12:44:07 +00001426 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001427
Linus Torvalds1da177e2005-04-16 15:20:36 -07001428 return 0;
1429}
David S. Miller99c4a262015-03-18 22:52:33 -04001430EXPORT_SYMBOL(dev_close_many);
Patrick McHardybd380812010-02-26 06:34:53 +00001431
1432/**
1433 * dev_close - shutdown an interface.
1434 * @dev: device to shutdown
1435 *
1436 * This function moves an active device into down state. A
1437 * %NETDEV_GOING_DOWN is sent to the netdev notifier chain. The device
1438 * is then deactivated and finally a %NETDEV_DOWN is sent to the notifier
1439 * chain.
1440 */
1441int dev_close(struct net_device *dev)
1442{
Eric Dumazete14a5992011-05-10 12:26:06 -07001443 if (dev->flags & IFF_UP) {
1444 LIST_HEAD(single);
Patrick McHardybd380812010-02-26 06:34:53 +00001445
Eric W. Biederman5cde2822013-10-05 19:26:05 -07001446 list_add(&dev->close_list, &single);
David S. Miller99c4a262015-03-18 22:52:33 -04001447 dev_close_many(&single, true);
Eric Dumazete14a5992011-05-10 12:26:06 -07001448 list_del(&single);
1449 }
dingtianhongda6e3782013-05-27 19:53:31 +00001450 return 0;
Patrick McHardybd380812010-02-26 06:34:53 +00001451}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001452EXPORT_SYMBOL(dev_close);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001453
1454
Ben Hutchings0187bdf2008-06-19 16:15:47 -07001455/**
1456 * dev_disable_lro - disable Large Receive Offload on a device
1457 * @dev: device
1458 *
1459 * Disable Large Receive Offload (LRO) on a net device. Must be
1460 * called under RTNL. This is needed if received packets may be
1461 * forwarded to another interface.
1462 */
1463void dev_disable_lro(struct net_device *dev)
1464{
Michal Kubečekfbe168b2014-11-13 07:54:50 +01001465 struct net_device *lower_dev;
1466 struct list_head *iter;
Michal Kubeček529d0482013-11-15 06:18:50 +01001467
Michał Mirosławbc5787c62011-11-15 15:29:55 +00001468 dev->wanted_features &= ~NETIF_F_LRO;
1469 netdev_update_features(dev);
Michał Mirosław27660512011-03-18 16:56:34 +00001470
Michał Mirosław22d59692011-04-21 12:42:15 +00001471 if (unlikely(dev->features & NETIF_F_LRO))
1472 netdev_WARN(dev, "failed to disable LRO!\n");
Michal Kubečekfbe168b2014-11-13 07:54:50 +01001473
1474 netdev_for_each_lower_dev(dev, lower_dev, iter)
1475 dev_disable_lro(lower_dev);
Ben Hutchings0187bdf2008-06-19 16:15:47 -07001476}
1477EXPORT_SYMBOL(dev_disable_lro);
1478
Jiri Pirko351638e2013-05-28 01:30:21 +00001479static int call_netdevice_notifier(struct notifier_block *nb, unsigned long val,
1480 struct net_device *dev)
1481{
1482 struct netdev_notifier_info info;
1483
1484 netdev_notifier_info_init(&info, dev);
1485 return nb->notifier_call(nb, val, &info);
1486}
Ben Hutchings0187bdf2008-06-19 16:15:47 -07001487
Eric W. Biederman881d9662007-09-17 11:56:21 -07001488static int dev_boot_phase = 1;
1489
Linus Torvalds1da177e2005-04-16 15:20:36 -07001490/**
1491 * register_netdevice_notifier - register a network notifier block
1492 * @nb: notifier
1493 *
1494 * Register a notifier to be called when network device events occur.
1495 * The notifier passed is linked into the kernel structures and must
1496 * not be reused until it has been unregistered. A negative errno code
1497 * is returned on a failure.
1498 *
1499 * When registered all registration and up events are replayed
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09001500 * to the new notifier to allow device to have a race free
Linus Torvalds1da177e2005-04-16 15:20:36 -07001501 * view of the network device list.
1502 */
1503
1504int register_netdevice_notifier(struct notifier_block *nb)
1505{
1506 struct net_device *dev;
Herbert Xufcc5a032007-07-30 17:03:38 -07001507 struct net_device *last;
Eric W. Biederman881d9662007-09-17 11:56:21 -07001508 struct net *net;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001509 int err;
1510
1511 rtnl_lock();
Alan Sternf07d5b92006-05-09 15:23:03 -07001512 err = raw_notifier_chain_register(&netdev_chain, nb);
Herbert Xufcc5a032007-07-30 17:03:38 -07001513 if (err)
1514 goto unlock;
Eric W. Biederman881d9662007-09-17 11:56:21 -07001515 if (dev_boot_phase)
1516 goto unlock;
1517 for_each_net(net) {
1518 for_each_netdev(net, dev) {
Jiri Pirko351638e2013-05-28 01:30:21 +00001519 err = call_netdevice_notifier(nb, NETDEV_REGISTER, dev);
Eric W. Biederman881d9662007-09-17 11:56:21 -07001520 err = notifier_to_errno(err);
1521 if (err)
1522 goto rollback;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001523
Eric W. Biederman881d9662007-09-17 11:56:21 -07001524 if (!(dev->flags & IFF_UP))
1525 continue;
Herbert Xufcc5a032007-07-30 17:03:38 -07001526
Jiri Pirko351638e2013-05-28 01:30:21 +00001527 call_netdevice_notifier(nb, NETDEV_UP, dev);
Eric W. Biederman881d9662007-09-17 11:56:21 -07001528 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001529 }
Herbert Xufcc5a032007-07-30 17:03:38 -07001530
1531unlock:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001532 rtnl_unlock();
1533 return err;
Herbert Xufcc5a032007-07-30 17:03:38 -07001534
1535rollback:
1536 last = dev;
Eric W. Biederman881d9662007-09-17 11:56:21 -07001537 for_each_net(net) {
1538 for_each_netdev(net, dev) {
1539 if (dev == last)
RongQing.Li8f891482011-11-30 23:43:07 -05001540 goto outroll;
Herbert Xufcc5a032007-07-30 17:03:38 -07001541
Eric W. Biederman881d9662007-09-17 11:56:21 -07001542 if (dev->flags & IFF_UP) {
Jiri Pirko351638e2013-05-28 01:30:21 +00001543 call_netdevice_notifier(nb, NETDEV_GOING_DOWN,
1544 dev);
1545 call_netdevice_notifier(nb, NETDEV_DOWN, dev);
Eric W. Biederman881d9662007-09-17 11:56:21 -07001546 }
Jiri Pirko351638e2013-05-28 01:30:21 +00001547 call_netdevice_notifier(nb, NETDEV_UNREGISTER, dev);
Herbert Xufcc5a032007-07-30 17:03:38 -07001548 }
Herbert Xufcc5a032007-07-30 17:03:38 -07001549 }
Pavel Emelyanovc67625a2007-11-14 15:53:16 -08001550
RongQing.Li8f891482011-11-30 23:43:07 -05001551outroll:
Pavel Emelyanovc67625a2007-11-14 15:53:16 -08001552 raw_notifier_chain_unregister(&netdev_chain, nb);
Herbert Xufcc5a032007-07-30 17:03:38 -07001553 goto unlock;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001554}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001555EXPORT_SYMBOL(register_netdevice_notifier);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001556
1557/**
1558 * unregister_netdevice_notifier - unregister a network notifier block
1559 * @nb: notifier
1560 *
1561 * Unregister a notifier previously registered by
1562 * register_netdevice_notifier(). The notifier is unlinked into the
1563 * kernel structures and may then be reused. A negative errno code
1564 * is returned on a failure.
Eric W. Biederman7d3d43d2012-04-06 15:33:35 +00001565 *
1566 * After unregistering unregister and down device events are synthesized
1567 * for all devices on the device list to the removed notifier to remove
1568 * the need for special case cleanup code.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001569 */
1570
1571int unregister_netdevice_notifier(struct notifier_block *nb)
1572{
Eric W. Biederman7d3d43d2012-04-06 15:33:35 +00001573 struct net_device *dev;
1574 struct net *net;
Herbert Xu9f514952006-03-25 01:24:25 -08001575 int err;
1576
1577 rtnl_lock();
Alan Sternf07d5b92006-05-09 15:23:03 -07001578 err = raw_notifier_chain_unregister(&netdev_chain, nb);
Eric W. Biederman7d3d43d2012-04-06 15:33:35 +00001579 if (err)
1580 goto unlock;
1581
1582 for_each_net(net) {
1583 for_each_netdev(net, dev) {
1584 if (dev->flags & IFF_UP) {
Jiri Pirko351638e2013-05-28 01:30:21 +00001585 call_netdevice_notifier(nb, NETDEV_GOING_DOWN,
1586 dev);
1587 call_netdevice_notifier(nb, NETDEV_DOWN, dev);
Eric W. Biederman7d3d43d2012-04-06 15:33:35 +00001588 }
Jiri Pirko351638e2013-05-28 01:30:21 +00001589 call_netdevice_notifier(nb, NETDEV_UNREGISTER, dev);
Eric W. Biederman7d3d43d2012-04-06 15:33:35 +00001590 }
1591 }
1592unlock:
Herbert Xu9f514952006-03-25 01:24:25 -08001593 rtnl_unlock();
1594 return err;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001595}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001596EXPORT_SYMBOL(unregister_netdevice_notifier);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001597
1598/**
Jiri Pirko351638e2013-05-28 01:30:21 +00001599 * call_netdevice_notifiers_info - call all network notifier blocks
1600 * @val: value passed unmodified to notifier function
1601 * @dev: net_device pointer passed unmodified to notifier function
1602 * @info: notifier information data
1603 *
1604 * Call all network notifier blocks. Parameters and return value
1605 * are as for raw_notifier_call_chain().
1606 */
1607
stephen hemminger1d143d92013-12-29 14:01:29 -08001608static int call_netdevice_notifiers_info(unsigned long val,
1609 struct net_device *dev,
1610 struct netdev_notifier_info *info)
Jiri Pirko351638e2013-05-28 01:30:21 +00001611{
1612 ASSERT_RTNL();
1613 netdev_notifier_info_init(info, dev);
1614 return raw_notifier_call_chain(&netdev_chain, val, info);
1615}
Jiri Pirko351638e2013-05-28 01:30:21 +00001616
1617/**
Linus Torvalds1da177e2005-04-16 15:20:36 -07001618 * call_netdevice_notifiers - call all network notifier blocks
1619 * @val: value passed unmodified to notifier function
Randy Dunlapc4ea43c2007-10-12 21:17:49 -07001620 * @dev: net_device pointer passed unmodified to notifier function
Linus Torvalds1da177e2005-04-16 15:20:36 -07001621 *
1622 * Call all network notifier blocks. Parameters and return value
Alan Sternf07d5b92006-05-09 15:23:03 -07001623 * are as for raw_notifier_call_chain().
Linus Torvalds1da177e2005-04-16 15:20:36 -07001624 */
1625
Eric W. Biedermanad7379d2007-09-16 15:33:32 -07001626int call_netdevice_notifiers(unsigned long val, struct net_device *dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001627{
Jiri Pirko351638e2013-05-28 01:30:21 +00001628 struct netdev_notifier_info info;
1629
1630 return call_netdevice_notifiers_info(val, dev, &info);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001631}
stephen hemmingeredf947f2011-03-24 13:24:01 +00001632EXPORT_SYMBOL(call_netdevice_notifiers);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001633
Pablo Neira1cf519002015-05-13 18:19:37 +02001634#ifdef CONFIG_NET_INGRESS
Daniel Borkmann45771392015-04-10 23:07:54 +02001635static struct static_key ingress_needed __read_mostly;
1636
1637void net_inc_ingress_queue(void)
1638{
1639 static_key_slow_inc(&ingress_needed);
1640}
1641EXPORT_SYMBOL_GPL(net_inc_ingress_queue);
1642
1643void net_dec_ingress_queue(void)
1644{
1645 static_key_slow_dec(&ingress_needed);
1646}
1647EXPORT_SYMBOL_GPL(net_dec_ingress_queue);
1648#endif
1649
Ingo Molnarc5905af2012-02-24 08:31:31 +01001650static struct static_key netstamp_needed __read_mostly;
Eric Dumazetb90e5792011-11-28 11:16:50 +00001651#ifdef HAVE_JUMP_LABEL
Ingo Molnarc5905af2012-02-24 08:31:31 +01001652/* We are not allowed to call static_key_slow_dec() from irq context
Eric Dumazetb90e5792011-11-28 11:16:50 +00001653 * If net_disable_timestamp() is called from irq context, defer the
Ingo Molnarc5905af2012-02-24 08:31:31 +01001654 * static_key_slow_dec() calls.
Eric Dumazetb90e5792011-11-28 11:16:50 +00001655 */
1656static atomic_t netstamp_needed_deferred;
1657#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07001658
1659void net_enable_timestamp(void)
1660{
Eric Dumazetb90e5792011-11-28 11:16:50 +00001661#ifdef HAVE_JUMP_LABEL
1662 int deferred = atomic_xchg(&netstamp_needed_deferred, 0);
1663
1664 if (deferred) {
1665 while (--deferred)
Ingo Molnarc5905af2012-02-24 08:31:31 +01001666 static_key_slow_dec(&netstamp_needed);
Eric Dumazetb90e5792011-11-28 11:16:50 +00001667 return;
1668 }
1669#endif
Ingo Molnarc5905af2012-02-24 08:31:31 +01001670 static_key_slow_inc(&netstamp_needed);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001671}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001672EXPORT_SYMBOL(net_enable_timestamp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001673
1674void net_disable_timestamp(void)
1675{
Eric Dumazetb90e5792011-11-28 11:16:50 +00001676#ifdef HAVE_JUMP_LABEL
1677 if (in_interrupt()) {
1678 atomic_inc(&netstamp_needed_deferred);
1679 return;
1680 }
1681#endif
Ingo Molnarc5905af2012-02-24 08:31:31 +01001682 static_key_slow_dec(&netstamp_needed);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001683}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001684EXPORT_SYMBOL(net_disable_timestamp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001685
Eric Dumazet3b098e22010-05-15 23:57:10 -07001686static inline void net_timestamp_set(struct sk_buff *skb)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001687{
Eric Dumazet588f0332011-11-15 04:12:55 +00001688 skb->tstamp.tv64 = 0;
Ingo Molnarc5905af2012-02-24 08:31:31 +01001689 if (static_key_false(&netstamp_needed))
Patrick McHardya61bbcf2005-08-14 17:24:31 -07001690 __net_timestamp(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001691}
1692
Eric Dumazet588f0332011-11-15 04:12:55 +00001693#define net_timestamp_check(COND, SKB) \
Ingo Molnarc5905af2012-02-24 08:31:31 +01001694 if (static_key_false(&netstamp_needed)) { \
Eric Dumazet588f0332011-11-15 04:12:55 +00001695 if ((COND) && !(SKB)->tstamp.tv64) \
1696 __net_timestamp(SKB); \
1697 } \
Eric Dumazet3b098e22010-05-15 23:57:10 -07001698
Vlad Yasevich1ee481f2014-03-27 17:32:29 -04001699bool is_skb_forwardable(struct net_device *dev, struct sk_buff *skb)
Daniel Lezcano79b569f2011-03-30 02:42:17 -07001700{
1701 unsigned int len;
1702
1703 if (!(dev->flags & IFF_UP))
1704 return false;
1705
1706 len = dev->mtu + dev->hard_header_len + VLAN_HLEN;
1707 if (skb->len <= len)
1708 return true;
1709
1710 /* if TSO is enabled, we don't care about the length as the packet
1711 * could be forwarded without being segmented before
1712 */
1713 if (skb_is_gso(skb))
1714 return true;
1715
1716 return false;
1717}
Vlad Yasevich1ee481f2014-03-27 17:32:29 -04001718EXPORT_SYMBOL_GPL(is_skb_forwardable);
Daniel Lezcano79b569f2011-03-30 02:42:17 -07001719
Herbert Xua0265d22014-04-17 13:45:03 +08001720int __dev_forward_skb(struct net_device *dev, struct sk_buff *skb)
1721{
Willem de Bruijnbbbf2df2015-06-08 11:53:08 -04001722 if (skb_orphan_frags(skb, GFP_ATOMIC) ||
1723 unlikely(!is_skb_forwardable(dev, skb))) {
Herbert Xua0265d22014-04-17 13:45:03 +08001724 atomic_long_inc(&dev->rx_dropped);
1725 kfree_skb(skb);
1726 return NET_RX_DROP;
1727 }
1728
1729 skb_scrub_packet(skb, true);
WANG Cong08b4b8e2015-03-20 14:29:09 -07001730 skb->priority = 0;
Herbert Xua0265d22014-04-17 13:45:03 +08001731 skb->protocol = eth_type_trans(skb, dev);
Jay Vosburgh2c26d342014-12-19 15:32:00 -08001732 skb_postpull_rcsum(skb, eth_hdr(skb), ETH_HLEN);
Herbert Xua0265d22014-04-17 13:45:03 +08001733
1734 return 0;
1735}
1736EXPORT_SYMBOL_GPL(__dev_forward_skb);
1737
Arnd Bergmann44540962009-11-26 06:07:08 +00001738/**
1739 * dev_forward_skb - loopback an skb to another netif
1740 *
1741 * @dev: destination network device
1742 * @skb: buffer to forward
1743 *
1744 * return values:
1745 * NET_RX_SUCCESS (no congestion)
Eric Dumazet6ec82562010-05-06 00:53:53 -07001746 * NET_RX_DROP (packet was dropped, but freed)
Arnd Bergmann44540962009-11-26 06:07:08 +00001747 *
1748 * dev_forward_skb can be used for injecting an skb from the
1749 * start_xmit function of one device into the receive queue
1750 * of another device.
1751 *
1752 * The receiving device may be in another namespace, so
1753 * we have to clear all information in the skb that could
1754 * impact namespace isolation.
1755 */
1756int dev_forward_skb(struct net_device *dev, struct sk_buff *skb)
1757{
Herbert Xua0265d22014-04-17 13:45:03 +08001758 return __dev_forward_skb(dev, skb) ?: netif_rx_internal(skb);
Arnd Bergmann44540962009-11-26 06:07:08 +00001759}
1760EXPORT_SYMBOL_GPL(dev_forward_skb);
1761
Changli Gao71d9dec2010-12-15 19:57:25 +00001762static inline int deliver_skb(struct sk_buff *skb,
1763 struct packet_type *pt_prev,
1764 struct net_device *orig_dev)
1765{
Michael S. Tsirkin1080e512012-07-20 09:23:17 +00001766 if (unlikely(skb_orphan_frags(skb, GFP_ATOMIC)))
1767 return -ENOMEM;
Changli Gao71d9dec2010-12-15 19:57:25 +00001768 atomic_inc(&skb->users);
1769 return pt_prev->func(skb, skb->dev, pt_prev, orig_dev);
1770}
1771
Salam Noureddine7866a622015-01-27 11:35:48 -08001772static inline void deliver_ptype_list_skb(struct sk_buff *skb,
1773 struct packet_type **pt,
Jiri Pirkofbcb2172015-03-30 16:56:01 +02001774 struct net_device *orig_dev,
1775 __be16 type,
Salam Noureddine7866a622015-01-27 11:35:48 -08001776 struct list_head *ptype_list)
1777{
1778 struct packet_type *ptype, *pt_prev = *pt;
1779
1780 list_for_each_entry_rcu(ptype, ptype_list, list) {
1781 if (ptype->type != type)
1782 continue;
1783 if (pt_prev)
Jiri Pirkofbcb2172015-03-30 16:56:01 +02001784 deliver_skb(skb, pt_prev, orig_dev);
Salam Noureddine7866a622015-01-27 11:35:48 -08001785 pt_prev = ptype;
1786 }
1787 *pt = pt_prev;
1788}
1789
Eric Leblondc0de08d2012-08-16 22:02:58 +00001790static inline bool skb_loop_sk(struct packet_type *ptype, struct sk_buff *skb)
1791{
Eric Leblonda3d744e2012-11-06 02:10:10 +00001792 if (!ptype->af_packet_priv || !skb->sk)
Eric Leblondc0de08d2012-08-16 22:02:58 +00001793 return false;
1794
1795 if (ptype->id_match)
1796 return ptype->id_match(ptype, skb->sk);
1797 else if ((struct sock *)ptype->af_packet_priv == skb->sk)
1798 return true;
1799
1800 return false;
1801}
1802
Linus Torvalds1da177e2005-04-16 15:20:36 -07001803/*
1804 * Support routine. Sends outgoing frames to any network
1805 * taps currently in use.
1806 */
1807
Herbert Xuf6a78bf2006-06-22 02:57:17 -07001808static void dev_queue_xmit_nit(struct sk_buff *skb, struct net_device *dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001809{
1810 struct packet_type *ptype;
Changli Gao71d9dec2010-12-15 19:57:25 +00001811 struct sk_buff *skb2 = NULL;
1812 struct packet_type *pt_prev = NULL;
Salam Noureddine7866a622015-01-27 11:35:48 -08001813 struct list_head *ptype_list = &ptype_all;
Patrick McHardya61bbcf2005-08-14 17:24:31 -07001814
Linus Torvalds1da177e2005-04-16 15:20:36 -07001815 rcu_read_lock();
Salam Noureddine7866a622015-01-27 11:35:48 -08001816again:
1817 list_for_each_entry_rcu(ptype, ptype_list, list) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001818 /* Never send packets back to the socket
1819 * they originated from - MvS (miquels@drinkel.ow.org)
1820 */
Salam Noureddine7866a622015-01-27 11:35:48 -08001821 if (skb_loop_sk(ptype, skb))
1822 continue;
Changli Gao71d9dec2010-12-15 19:57:25 +00001823
Salam Noureddine7866a622015-01-27 11:35:48 -08001824 if (pt_prev) {
1825 deliver_skb(skb2, pt_prev, skb->dev);
Changli Gao71d9dec2010-12-15 19:57:25 +00001826 pt_prev = ptype;
Salam Noureddine7866a622015-01-27 11:35:48 -08001827 continue;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001828 }
Salam Noureddine7866a622015-01-27 11:35:48 -08001829
1830 /* need to clone skb, done only once */
1831 skb2 = skb_clone(skb, GFP_ATOMIC);
1832 if (!skb2)
1833 goto out_unlock;
1834
1835 net_timestamp_set(skb2);
1836
1837 /* skb->nh should be correctly
1838 * set by sender, so that the second statement is
1839 * just protection against buggy protocols.
1840 */
1841 skb_reset_mac_header(skb2);
1842
1843 if (skb_network_header(skb2) < skb2->data ||
1844 skb_network_header(skb2) > skb_tail_pointer(skb2)) {
1845 net_crit_ratelimited("protocol %04x is buggy, dev %s\n",
1846 ntohs(skb2->protocol),
1847 dev->name);
1848 skb_reset_network_header(skb2);
1849 }
1850
1851 skb2->transport_header = skb2->network_header;
1852 skb2->pkt_type = PACKET_OUTGOING;
1853 pt_prev = ptype;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001854 }
Salam Noureddine7866a622015-01-27 11:35:48 -08001855
1856 if (ptype_list == &ptype_all) {
1857 ptype_list = &dev->ptype_all;
1858 goto again;
1859 }
1860out_unlock:
Changli Gao71d9dec2010-12-15 19:57:25 +00001861 if (pt_prev)
1862 pt_prev->func(skb2, skb->dev, pt_prev, skb->dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001863 rcu_read_unlock();
1864}
1865
Ben Hutchings2c530402012-07-10 10:55:09 +00001866/**
1867 * netif_setup_tc - Handle tc mappings on real_num_tx_queues change
John Fastabend4f57c082011-01-17 08:06:04 +00001868 * @dev: Network device
1869 * @txq: number of queues available
1870 *
1871 * If real_num_tx_queues is changed the tc mappings may no longer be
1872 * valid. To resolve this verify the tc mapping remains valid and if
1873 * not NULL the mapping. With no priorities mapping to this
1874 * offset/count pair it will no longer be used. In the worst case TC0
1875 * is invalid nothing can be done so disable priority mappings. If is
1876 * expected that drivers will fix this mapping if they can before
1877 * calling netif_set_real_num_tx_queues.
1878 */
Eric Dumazetbb134d22011-01-20 19:18:08 +00001879static void netif_setup_tc(struct net_device *dev, unsigned int txq)
John Fastabend4f57c082011-01-17 08:06:04 +00001880{
1881 int i;
1882 struct netdev_tc_txq *tc = &dev->tc_to_txq[0];
1883
1884 /* If TC0 is invalidated disable TC mapping */
1885 if (tc->offset + tc->count > txq) {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00001886 pr_warn("Number of in use tx queues changed invalidating tc mappings. Priority traffic classification disabled!\n");
John Fastabend4f57c082011-01-17 08:06:04 +00001887 dev->num_tc = 0;
1888 return;
1889 }
1890
1891 /* Invalidated prio to tc mappings set to TC0 */
1892 for (i = 1; i < TC_BITMASK + 1; i++) {
1893 int q = netdev_get_prio_tc_map(dev, i);
1894
1895 tc = &dev->tc_to_txq[q];
1896 if (tc->offset + tc->count > txq) {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00001897 pr_warn("Number of in use tx queues changed. Priority %i to tc mapping %i is no longer valid. Setting map to 0\n",
1898 i, q);
John Fastabend4f57c082011-01-17 08:06:04 +00001899 netdev_set_prio_tc_map(dev, i, 0);
1900 }
1901 }
1902}
1903
Alexander Duyck537c00d2013-01-10 08:57:02 +00001904#ifdef CONFIG_XPS
1905static DEFINE_MUTEX(xps_map_mutex);
1906#define xmap_dereference(P) \
1907 rcu_dereference_protected((P), lockdep_is_held(&xps_map_mutex))
1908
Alexander Duyck10cdc3f2013-01-10 08:57:17 +00001909static struct xps_map *remove_xps_queue(struct xps_dev_maps *dev_maps,
1910 int cpu, u16 index)
1911{
1912 struct xps_map *map = NULL;
1913 int pos;
1914
1915 if (dev_maps)
1916 map = xmap_dereference(dev_maps->cpu_map[cpu]);
1917
1918 for (pos = 0; map && pos < map->len; pos++) {
1919 if (map->queues[pos] == index) {
1920 if (map->len > 1) {
1921 map->queues[pos] = map->queues[--map->len];
1922 } else {
1923 RCU_INIT_POINTER(dev_maps->cpu_map[cpu], NULL);
1924 kfree_rcu(map, rcu);
1925 map = NULL;
1926 }
1927 break;
1928 }
1929 }
1930
1931 return map;
1932}
1933
Alexander Duyck024e9672013-01-10 08:57:46 +00001934static void netif_reset_xps_queues_gt(struct net_device *dev, u16 index)
Alexander Duyck537c00d2013-01-10 08:57:02 +00001935{
1936 struct xps_dev_maps *dev_maps;
Alexander Duyck024e9672013-01-10 08:57:46 +00001937 int cpu, i;
Alexander Duyck10cdc3f2013-01-10 08:57:17 +00001938 bool active = false;
Alexander Duyck537c00d2013-01-10 08:57:02 +00001939
1940 mutex_lock(&xps_map_mutex);
1941 dev_maps = xmap_dereference(dev->xps_maps);
1942
1943 if (!dev_maps)
1944 goto out_no_maps;
1945
Alexander Duyck10cdc3f2013-01-10 08:57:17 +00001946 for_each_possible_cpu(cpu) {
Alexander Duyck024e9672013-01-10 08:57:46 +00001947 for (i = index; i < dev->num_tx_queues; i++) {
1948 if (!remove_xps_queue(dev_maps, cpu, i))
1949 break;
1950 }
1951 if (i == dev->num_tx_queues)
Alexander Duyck10cdc3f2013-01-10 08:57:17 +00001952 active = true;
Alexander Duyck537c00d2013-01-10 08:57:02 +00001953 }
1954
Alexander Duyck10cdc3f2013-01-10 08:57:17 +00001955 if (!active) {
Alexander Duyck537c00d2013-01-10 08:57:02 +00001956 RCU_INIT_POINTER(dev->xps_maps, NULL);
1957 kfree_rcu(dev_maps, rcu);
1958 }
1959
Alexander Duyck024e9672013-01-10 08:57:46 +00001960 for (i = index; i < dev->num_tx_queues; i++)
1961 netdev_queue_numa_node_write(netdev_get_tx_queue(dev, i),
1962 NUMA_NO_NODE);
1963
Alexander Duyck537c00d2013-01-10 08:57:02 +00001964out_no_maps:
1965 mutex_unlock(&xps_map_mutex);
1966}
1967
Alexander Duyck01c5f862013-01-10 08:57:35 +00001968static struct xps_map *expand_xps_map(struct xps_map *map,
1969 int cpu, u16 index)
1970{
1971 struct xps_map *new_map;
1972 int alloc_len = XPS_MIN_MAP_ALLOC;
1973 int i, pos;
1974
1975 for (pos = 0; map && pos < map->len; pos++) {
1976 if (map->queues[pos] != index)
1977 continue;
1978 return map;
1979 }
1980
1981 /* Need to add queue to this CPU's existing map */
1982 if (map) {
1983 if (pos < map->alloc_len)
1984 return map;
1985
1986 alloc_len = map->alloc_len * 2;
1987 }
1988
1989 /* Need to allocate new map to store queue on this CPU's map */
1990 new_map = kzalloc_node(XPS_MAP_SIZE(alloc_len), GFP_KERNEL,
1991 cpu_to_node(cpu));
1992 if (!new_map)
1993 return NULL;
1994
1995 for (i = 0; i < pos; i++)
1996 new_map->queues[i] = map->queues[i];
1997 new_map->alloc_len = alloc_len;
1998 new_map->len = pos;
1999
2000 return new_map;
2001}
2002
Michael S. Tsirkin35735402013-10-02 09:14:06 +03002003int netif_set_xps_queue(struct net_device *dev, const struct cpumask *mask,
2004 u16 index)
Alexander Duyck537c00d2013-01-10 08:57:02 +00002005{
Alexander Duyck01c5f862013-01-10 08:57:35 +00002006 struct xps_dev_maps *dev_maps, *new_dev_maps = NULL;
Alexander Duyck537c00d2013-01-10 08:57:02 +00002007 struct xps_map *map, *new_map;
Alexander Duyck537c00d2013-01-10 08:57:02 +00002008 int maps_sz = max_t(unsigned int, XPS_DEV_MAPS_SIZE, L1_CACHE_BYTES);
Alexander Duyck01c5f862013-01-10 08:57:35 +00002009 int cpu, numa_node_id = -2;
2010 bool active = false;
Alexander Duyck537c00d2013-01-10 08:57:02 +00002011
2012 mutex_lock(&xps_map_mutex);
2013
2014 dev_maps = xmap_dereference(dev->xps_maps);
2015
Alexander Duyck01c5f862013-01-10 08:57:35 +00002016 /* allocate memory for queue storage */
2017 for_each_online_cpu(cpu) {
2018 if (!cpumask_test_cpu(cpu, mask))
2019 continue;
Alexander Duyck537c00d2013-01-10 08:57:02 +00002020
Alexander Duyck01c5f862013-01-10 08:57:35 +00002021 if (!new_dev_maps)
2022 new_dev_maps = kzalloc(maps_sz, GFP_KERNEL);
Alexander Duyck2bb60cb2013-02-22 06:38:44 +00002023 if (!new_dev_maps) {
2024 mutex_unlock(&xps_map_mutex);
Alexander Duyck01c5f862013-01-10 08:57:35 +00002025 return -ENOMEM;
Alexander Duyck2bb60cb2013-02-22 06:38:44 +00002026 }
Alexander Duyck01c5f862013-01-10 08:57:35 +00002027
2028 map = dev_maps ? xmap_dereference(dev_maps->cpu_map[cpu]) :
2029 NULL;
2030
2031 map = expand_xps_map(map, cpu, index);
2032 if (!map)
2033 goto error;
2034
2035 RCU_INIT_POINTER(new_dev_maps->cpu_map[cpu], map);
2036 }
2037
2038 if (!new_dev_maps)
2039 goto out_no_new_maps;
2040
2041 for_each_possible_cpu(cpu) {
2042 if (cpumask_test_cpu(cpu, mask) && cpu_online(cpu)) {
2043 /* add queue to CPU maps */
2044 int pos = 0;
2045
2046 map = xmap_dereference(new_dev_maps->cpu_map[cpu]);
2047 while ((pos < map->len) && (map->queues[pos] != index))
2048 pos++;
2049
2050 if (pos == map->len)
2051 map->queues[map->len++] = index;
Alexander Duyck537c00d2013-01-10 08:57:02 +00002052#ifdef CONFIG_NUMA
Alexander Duyck537c00d2013-01-10 08:57:02 +00002053 if (numa_node_id == -2)
2054 numa_node_id = cpu_to_node(cpu);
2055 else if (numa_node_id != cpu_to_node(cpu))
2056 numa_node_id = -1;
Alexander Duyck537c00d2013-01-10 08:57:02 +00002057#endif
Alexander Duyck01c5f862013-01-10 08:57:35 +00002058 } else if (dev_maps) {
2059 /* fill in the new device map from the old device map */
2060 map = xmap_dereference(dev_maps->cpu_map[cpu]);
2061 RCU_INIT_POINTER(new_dev_maps->cpu_map[cpu], map);
Alexander Duyck537c00d2013-01-10 08:57:02 +00002062 }
Alexander Duyck01c5f862013-01-10 08:57:35 +00002063
Alexander Duyck537c00d2013-01-10 08:57:02 +00002064 }
2065
Alexander Duyck01c5f862013-01-10 08:57:35 +00002066 rcu_assign_pointer(dev->xps_maps, new_dev_maps);
2067
Alexander Duyck537c00d2013-01-10 08:57:02 +00002068 /* Cleanup old maps */
Alexander Duyck01c5f862013-01-10 08:57:35 +00002069 if (dev_maps) {
2070 for_each_possible_cpu(cpu) {
2071 new_map = xmap_dereference(new_dev_maps->cpu_map[cpu]);
2072 map = xmap_dereference(dev_maps->cpu_map[cpu]);
2073 if (map && map != new_map)
2074 kfree_rcu(map, rcu);
2075 }
Alexander Duyck537c00d2013-01-10 08:57:02 +00002076
Alexander Duyck537c00d2013-01-10 08:57:02 +00002077 kfree_rcu(dev_maps, rcu);
Alexander Duyck01c5f862013-01-10 08:57:35 +00002078 }
Alexander Duyck537c00d2013-01-10 08:57:02 +00002079
Alexander Duyck01c5f862013-01-10 08:57:35 +00002080 dev_maps = new_dev_maps;
2081 active = true;
2082
2083out_no_new_maps:
2084 /* update Tx queue numa node */
Alexander Duyck537c00d2013-01-10 08:57:02 +00002085 netdev_queue_numa_node_write(netdev_get_tx_queue(dev, index),
2086 (numa_node_id >= 0) ? numa_node_id :
2087 NUMA_NO_NODE);
2088
Alexander Duyck01c5f862013-01-10 08:57:35 +00002089 if (!dev_maps)
2090 goto out_no_maps;
2091
2092 /* removes queue from unused CPUs */
2093 for_each_possible_cpu(cpu) {
2094 if (cpumask_test_cpu(cpu, mask) && cpu_online(cpu))
2095 continue;
2096
2097 if (remove_xps_queue(dev_maps, cpu, index))
2098 active = true;
2099 }
2100
2101 /* free map if not active */
2102 if (!active) {
2103 RCU_INIT_POINTER(dev->xps_maps, NULL);
2104 kfree_rcu(dev_maps, rcu);
2105 }
2106
2107out_no_maps:
Alexander Duyck537c00d2013-01-10 08:57:02 +00002108 mutex_unlock(&xps_map_mutex);
2109
2110 return 0;
2111error:
Alexander Duyck01c5f862013-01-10 08:57:35 +00002112 /* remove any maps that we added */
2113 for_each_possible_cpu(cpu) {
2114 new_map = xmap_dereference(new_dev_maps->cpu_map[cpu]);
2115 map = dev_maps ? xmap_dereference(dev_maps->cpu_map[cpu]) :
2116 NULL;
2117 if (new_map && new_map != map)
2118 kfree(new_map);
2119 }
2120
Alexander Duyck537c00d2013-01-10 08:57:02 +00002121 mutex_unlock(&xps_map_mutex);
2122
Alexander Duyck537c00d2013-01-10 08:57:02 +00002123 kfree(new_dev_maps);
2124 return -ENOMEM;
2125}
2126EXPORT_SYMBOL(netif_set_xps_queue);
2127
2128#endif
John Fastabendf0796d52010-07-01 13:21:57 +00002129/*
2130 * Routine to help set real_num_tx_queues. To avoid skbs mapped to queues
2131 * greater then real_num_tx_queues stale skbs on the qdisc must be flushed.
2132 */
Tom Herberte6484932010-10-18 18:04:39 +00002133int netif_set_real_num_tx_queues(struct net_device *dev, unsigned int txq)
John Fastabendf0796d52010-07-01 13:21:57 +00002134{
Tom Herbert1d24eb42010-11-21 13:17:27 +00002135 int rc;
2136
Tom Herberte6484932010-10-18 18:04:39 +00002137 if (txq < 1 || txq > dev->num_tx_queues)
2138 return -EINVAL;
John Fastabendf0796d52010-07-01 13:21:57 +00002139
Ben Hutchings5c565802011-02-15 19:39:21 +00002140 if (dev->reg_state == NETREG_REGISTERED ||
2141 dev->reg_state == NETREG_UNREGISTERING) {
Tom Herberte6484932010-10-18 18:04:39 +00002142 ASSERT_RTNL();
2143
Tom Herbert1d24eb42010-11-21 13:17:27 +00002144 rc = netdev_queue_update_kobjects(dev, dev->real_num_tx_queues,
2145 txq);
Tom Herbertbf264142010-11-26 08:36:09 +00002146 if (rc)
2147 return rc;
2148
John Fastabend4f57c082011-01-17 08:06:04 +00002149 if (dev->num_tc)
2150 netif_setup_tc(dev, txq);
2151
Alexander Duyck024e9672013-01-10 08:57:46 +00002152 if (txq < dev->real_num_tx_queues) {
Tom Herberte6484932010-10-18 18:04:39 +00002153 qdisc_reset_all_tx_gt(dev, txq);
Alexander Duyck024e9672013-01-10 08:57:46 +00002154#ifdef CONFIG_XPS
2155 netif_reset_xps_queues_gt(dev, txq);
2156#endif
2157 }
John Fastabendf0796d52010-07-01 13:21:57 +00002158 }
Tom Herberte6484932010-10-18 18:04:39 +00002159
2160 dev->real_num_tx_queues = txq;
2161 return 0;
John Fastabendf0796d52010-07-01 13:21:57 +00002162}
2163EXPORT_SYMBOL(netif_set_real_num_tx_queues);
Denis Vlasenko56079432006-03-29 15:57:29 -08002164
Michael Daltona953be52014-01-16 22:23:28 -08002165#ifdef CONFIG_SYSFS
Ben Hutchings62fe0b42010-09-27 08:24:33 +00002166/**
2167 * netif_set_real_num_rx_queues - set actual number of RX queues used
2168 * @dev: Network device
2169 * @rxq: Actual number of RX queues
2170 *
2171 * This must be called either with the rtnl_lock held or before
2172 * registration of the net device. Returns 0 on success, or a
Ben Hutchings4e7f7952010-10-08 10:33:39 -07002173 * negative error code. If called before registration, it always
2174 * succeeds.
Ben Hutchings62fe0b42010-09-27 08:24:33 +00002175 */
2176int netif_set_real_num_rx_queues(struct net_device *dev, unsigned int rxq)
2177{
2178 int rc;
2179
Tom Herbertbd25fa72010-10-18 18:00:16 +00002180 if (rxq < 1 || rxq > dev->num_rx_queues)
2181 return -EINVAL;
2182
Ben Hutchings62fe0b42010-09-27 08:24:33 +00002183 if (dev->reg_state == NETREG_REGISTERED) {
2184 ASSERT_RTNL();
2185
Ben Hutchings62fe0b42010-09-27 08:24:33 +00002186 rc = net_rx_queue_update_kobjects(dev, dev->real_num_rx_queues,
2187 rxq);
2188 if (rc)
2189 return rc;
Ben Hutchings62fe0b42010-09-27 08:24:33 +00002190 }
2191
2192 dev->real_num_rx_queues = rxq;
2193 return 0;
2194}
2195EXPORT_SYMBOL(netif_set_real_num_rx_queues);
2196#endif
2197
Ben Hutchings2c530402012-07-10 10:55:09 +00002198/**
2199 * netif_get_num_default_rss_queues - default number of RSS queues
Yuval Mintz16917b82012-07-01 03:18:50 +00002200 *
2201 * This routine should set an upper limit on the number of RSS queues
2202 * used by default by multiqueue devices.
2203 */
Ben Hutchingsa55b1382012-07-10 10:54:38 +00002204int netif_get_num_default_rss_queues(void)
Yuval Mintz16917b82012-07-01 03:18:50 +00002205{
2206 return min_t(int, DEFAULT_MAX_NUM_RSS_QUEUES, num_online_cpus());
2207}
2208EXPORT_SYMBOL(netif_get_num_default_rss_queues);
2209
Jarek Poplawskidef82a12008-08-17 21:54:43 -07002210static inline void __netif_reschedule(struct Qdisc *q)
2211{
2212 struct softnet_data *sd;
2213 unsigned long flags;
2214
2215 local_irq_save(flags);
Christoph Lameter903ceff2014-08-17 12:30:35 -05002216 sd = this_cpu_ptr(&softnet_data);
Changli Gaoa9cbd582010-04-26 23:06:24 +00002217 q->next_sched = NULL;
2218 *sd->output_queue_tailp = q;
2219 sd->output_queue_tailp = &q->next_sched;
Jarek Poplawskidef82a12008-08-17 21:54:43 -07002220 raise_softirq_irqoff(NET_TX_SOFTIRQ);
2221 local_irq_restore(flags);
2222}
2223
David S. Miller37437bb2008-07-16 02:15:04 -07002224void __netif_schedule(struct Qdisc *q)
Denis Vlasenko56079432006-03-29 15:57:29 -08002225{
Jarek Poplawskidef82a12008-08-17 21:54:43 -07002226 if (!test_and_set_bit(__QDISC_STATE_SCHED, &q->state))
2227 __netif_reschedule(q);
Denis Vlasenko56079432006-03-29 15:57:29 -08002228}
2229EXPORT_SYMBOL(__netif_schedule);
2230
Eric Dumazete6247022013-12-05 04:45:08 -08002231struct dev_kfree_skb_cb {
2232 enum skb_free_reason reason;
2233};
2234
2235static struct dev_kfree_skb_cb *get_kfree_skb_cb(const struct sk_buff *skb)
Denis Vlasenko56079432006-03-29 15:57:29 -08002236{
Eric Dumazete6247022013-12-05 04:45:08 -08002237 return (struct dev_kfree_skb_cb *)skb->cb;
Denis Vlasenko56079432006-03-29 15:57:29 -08002238}
Denis Vlasenko56079432006-03-29 15:57:29 -08002239
John Fastabend46e5da42014-09-12 20:04:52 -07002240void netif_schedule_queue(struct netdev_queue *txq)
2241{
2242 rcu_read_lock();
2243 if (!(txq->state & QUEUE_STATE_ANY_XOFF)) {
2244 struct Qdisc *q = rcu_dereference(txq->qdisc);
2245
2246 __netif_schedule(q);
2247 }
2248 rcu_read_unlock();
2249}
2250EXPORT_SYMBOL(netif_schedule_queue);
2251
2252/**
2253 * netif_wake_subqueue - allow sending packets on subqueue
2254 * @dev: network device
2255 * @queue_index: sub queue index
2256 *
2257 * Resume individual transmit queue of a device with multiple transmit queues.
2258 */
2259void netif_wake_subqueue(struct net_device *dev, u16 queue_index)
2260{
2261 struct netdev_queue *txq = netdev_get_tx_queue(dev, queue_index);
2262
2263 if (test_and_clear_bit(__QUEUE_STATE_DRV_XOFF, &txq->state)) {
2264 struct Qdisc *q;
2265
2266 rcu_read_lock();
2267 q = rcu_dereference(txq->qdisc);
2268 __netif_schedule(q);
2269 rcu_read_unlock();
2270 }
2271}
2272EXPORT_SYMBOL(netif_wake_subqueue);
2273
2274void netif_tx_wake_queue(struct netdev_queue *dev_queue)
2275{
2276 if (test_and_clear_bit(__QUEUE_STATE_DRV_XOFF, &dev_queue->state)) {
2277 struct Qdisc *q;
2278
2279 rcu_read_lock();
2280 q = rcu_dereference(dev_queue->qdisc);
2281 __netif_schedule(q);
2282 rcu_read_unlock();
2283 }
2284}
2285EXPORT_SYMBOL(netif_tx_wake_queue);
2286
Eric Dumazete6247022013-12-05 04:45:08 -08002287void __dev_kfree_skb_irq(struct sk_buff *skb, enum skb_free_reason reason)
2288{
2289 unsigned long flags;
2290
2291 if (likely(atomic_read(&skb->users) == 1)) {
2292 smp_rmb();
2293 atomic_set(&skb->users, 0);
2294 } else if (likely(!atomic_dec_and_test(&skb->users))) {
2295 return;
2296 }
2297 get_kfree_skb_cb(skb)->reason = reason;
2298 local_irq_save(flags);
2299 skb->next = __this_cpu_read(softnet_data.completion_queue);
2300 __this_cpu_write(softnet_data.completion_queue, skb);
2301 raise_softirq_irqoff(NET_TX_SOFTIRQ);
2302 local_irq_restore(flags);
2303}
2304EXPORT_SYMBOL(__dev_kfree_skb_irq);
2305
2306void __dev_kfree_skb_any(struct sk_buff *skb, enum skb_free_reason reason)
Denis Vlasenko56079432006-03-29 15:57:29 -08002307{
2308 if (in_irq() || irqs_disabled())
Eric Dumazete6247022013-12-05 04:45:08 -08002309 __dev_kfree_skb_irq(skb, reason);
Denis Vlasenko56079432006-03-29 15:57:29 -08002310 else
2311 dev_kfree_skb(skb);
2312}
Eric Dumazete6247022013-12-05 04:45:08 -08002313EXPORT_SYMBOL(__dev_kfree_skb_any);
Denis Vlasenko56079432006-03-29 15:57:29 -08002314
2315
Stephen Hemmingerbea33482007-10-03 16:41:36 -07002316/**
2317 * netif_device_detach - mark device as removed
2318 * @dev: network device
2319 *
2320 * Mark device as removed from system and therefore no longer available.
2321 */
Denis Vlasenko56079432006-03-29 15:57:29 -08002322void netif_device_detach(struct net_device *dev)
2323{
2324 if (test_and_clear_bit(__LINK_STATE_PRESENT, &dev->state) &&
2325 netif_running(dev)) {
Alexander Duyckd5431032009-04-08 13:15:22 +00002326 netif_tx_stop_all_queues(dev);
Denis Vlasenko56079432006-03-29 15:57:29 -08002327 }
2328}
2329EXPORT_SYMBOL(netif_device_detach);
2330
Stephen Hemmingerbea33482007-10-03 16:41:36 -07002331/**
2332 * netif_device_attach - mark device as attached
2333 * @dev: network device
2334 *
2335 * Mark device as attached from system and restart if needed.
2336 */
Denis Vlasenko56079432006-03-29 15:57:29 -08002337void netif_device_attach(struct net_device *dev)
2338{
2339 if (!test_and_set_bit(__LINK_STATE_PRESENT, &dev->state) &&
2340 netif_running(dev)) {
Alexander Duyckd5431032009-04-08 13:15:22 +00002341 netif_tx_wake_all_queues(dev);
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09002342 __netdev_watchdog_up(dev);
Denis Vlasenko56079432006-03-29 15:57:29 -08002343 }
2344}
2345EXPORT_SYMBOL(netif_device_attach);
2346
Jiri Pirko5605c762015-05-12 14:56:12 +02002347/*
2348 * Returns a Tx hash based on the given packet descriptor a Tx queues' number
2349 * to be used as a distribution range.
2350 */
2351u16 __skb_tx_hash(const struct net_device *dev, struct sk_buff *skb,
2352 unsigned int num_tx_queues)
2353{
2354 u32 hash;
2355 u16 qoffset = 0;
2356 u16 qcount = num_tx_queues;
2357
2358 if (skb_rx_queue_recorded(skb)) {
2359 hash = skb_get_rx_queue(skb);
2360 while (unlikely(hash >= num_tx_queues))
2361 hash -= num_tx_queues;
2362 return hash;
2363 }
2364
2365 if (dev->num_tc) {
2366 u8 tc = netdev_get_prio_tc_map(dev, skb->priority);
2367 qoffset = dev->tc_to_txq[tc].offset;
2368 qcount = dev->tc_to_txq[tc].count;
2369 }
2370
2371 return (u16) reciprocal_scale(skb_get_hash(skb), qcount) + qoffset;
2372}
2373EXPORT_SYMBOL(__skb_tx_hash);
2374
Ben Hutchings36c92472012-01-17 07:57:56 +00002375static void skb_warn_bad_offload(const struct sk_buff *skb)
2376{
Michał Mirosław65e9d2f2012-01-17 10:00:40 +00002377 static const netdev_features_t null_features = 0;
Ben Hutchings36c92472012-01-17 07:57:56 +00002378 struct net_device *dev = skb->dev;
2379 const char *driver = "";
2380
Ben Greearc846ad92013-04-19 10:45:52 +00002381 if (!net_ratelimit())
2382 return;
2383
Ben Hutchings36c92472012-01-17 07:57:56 +00002384 if (dev && dev->dev.parent)
2385 driver = dev_driver_string(dev->dev.parent);
2386
2387 WARN(1, "%s: caps=(%pNF, %pNF) len=%d data_len=%d gso_size=%d "
2388 "gso_type=%d ip_summed=%d\n",
Michał Mirosław65e9d2f2012-01-17 10:00:40 +00002389 driver, dev ? &dev->features : &null_features,
2390 skb->sk ? &skb->sk->sk_route_caps : &null_features,
Ben Hutchings36c92472012-01-17 07:57:56 +00002391 skb->len, skb->data_len, skb_shinfo(skb)->gso_size,
2392 skb_shinfo(skb)->gso_type, skb->ip_summed);
2393}
2394
Linus Torvalds1da177e2005-04-16 15:20:36 -07002395/*
2396 * Invalidate hardware checksum when packet is to be mangled, and
2397 * complete checksum manually on outgoing path.
2398 */
Patrick McHardy84fa7932006-08-29 16:44:56 -07002399int skb_checksum_help(struct sk_buff *skb)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002400{
Al Virod3bc23e2006-11-14 21:24:49 -08002401 __wsum csum;
Herbert Xu663ead32007-04-09 11:59:07 -07002402 int ret = 0, offset;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002403
Patrick McHardy84fa7932006-08-29 16:44:56 -07002404 if (skb->ip_summed == CHECKSUM_COMPLETE)
Herbert Xua430a432006-07-08 13:34:56 -07002405 goto out_set_summed;
2406
2407 if (unlikely(skb_shinfo(skb)->gso_size)) {
Ben Hutchings36c92472012-01-17 07:57:56 +00002408 skb_warn_bad_offload(skb);
2409 return -EINVAL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002410 }
2411
Eric Dumazetcef401d2013-01-25 20:34:37 +00002412 /* Before computing a checksum, we should make sure no frag could
2413 * be modified by an external entity : checksum could be wrong.
2414 */
2415 if (skb_has_shared_frag(skb)) {
2416 ret = __skb_linearize(skb);
2417 if (ret)
2418 goto out;
2419 }
2420
Michał Mirosław55508d62010-12-14 15:24:08 +00002421 offset = skb_checksum_start_offset(skb);
Herbert Xua0308472007-10-15 01:47:15 -07002422 BUG_ON(offset >= skb_headlen(skb));
2423 csum = skb_checksum(skb, offset, skb->len - offset, 0);
2424
2425 offset += skb->csum_offset;
2426 BUG_ON(offset + sizeof(__sum16) > skb_headlen(skb));
2427
2428 if (skb_cloned(skb) &&
2429 !skb_clone_writable(skb, offset + sizeof(__sum16))) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07002430 ret = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
2431 if (ret)
2432 goto out;
2433 }
2434
Herbert Xua0308472007-10-15 01:47:15 -07002435 *(__sum16 *)(skb->data + offset) = csum_fold(csum);
Herbert Xua430a432006-07-08 13:34:56 -07002436out_set_summed:
Linus Torvalds1da177e2005-04-16 15:20:36 -07002437 skb->ip_summed = CHECKSUM_NONE;
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09002438out:
Linus Torvalds1da177e2005-04-16 15:20:36 -07002439 return ret;
2440}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07002441EXPORT_SYMBOL(skb_checksum_help);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002442
Vlad Yasevich53d64712014-03-27 17:26:18 -04002443__be16 skb_network_protocol(struct sk_buff *skb, int *depth)
Pravin B Shelarec5f0612013-03-07 09:28:01 +00002444{
2445 __be16 type = skb->protocol;
2446
Pravin B Shelar19acc322013-05-07 20:41:07 +00002447 /* Tunnel gso handlers can set protocol to ethernet. */
2448 if (type == htons(ETH_P_TEB)) {
2449 struct ethhdr *eth;
2450
2451 if (unlikely(!pskb_may_pull(skb, sizeof(struct ethhdr))))
2452 return 0;
2453
2454 eth = (struct ethhdr *)skb_mac_header(skb);
2455 type = eth->h_proto;
2456 }
2457
Toshiaki Makitad4bcef32015-01-29 20:37:07 +09002458 return __vlan_get_protocol(skb, type, depth);
Pravin B Shelarec5f0612013-03-07 09:28:01 +00002459}
2460
Pravin B Shelar05e8ef42013-02-14 09:44:55 +00002461/**
2462 * skb_mac_gso_segment - mac layer segmentation handler.
2463 * @skb: buffer to segment
2464 * @features: features for the output path (see dev->features)
2465 */
2466struct sk_buff *skb_mac_gso_segment(struct sk_buff *skb,
2467 netdev_features_t features)
2468{
2469 struct sk_buff *segs = ERR_PTR(-EPROTONOSUPPORT);
2470 struct packet_offload *ptype;
Vlad Yasevich53d64712014-03-27 17:26:18 -04002471 int vlan_depth = skb->mac_len;
2472 __be16 type = skb_network_protocol(skb, &vlan_depth);
Pravin B Shelar05e8ef42013-02-14 09:44:55 +00002473
Pravin B Shelarec5f0612013-03-07 09:28:01 +00002474 if (unlikely(!type))
2475 return ERR_PTR(-EINVAL);
Pravin B Shelar05e8ef42013-02-14 09:44:55 +00002476
Vlad Yasevich53d64712014-03-27 17:26:18 -04002477 __skb_pull(skb, vlan_depth);
Pravin B Shelar05e8ef42013-02-14 09:44:55 +00002478
2479 rcu_read_lock();
2480 list_for_each_entry_rcu(ptype, &offload_base, list) {
2481 if (ptype->type == type && ptype->callbacks.gso_segment) {
Pravin B Shelar05e8ef42013-02-14 09:44:55 +00002482 segs = ptype->callbacks.gso_segment(skb, features);
2483 break;
2484 }
2485 }
2486 rcu_read_unlock();
2487
2488 __skb_push(skb, skb->data - skb_mac_header(skb));
2489
2490 return segs;
2491}
2492EXPORT_SYMBOL(skb_mac_gso_segment);
2493
2494
Cong Wang12b00042013-02-05 16:36:38 +00002495/* openvswitch calls this on rx path, so we need a different check.
2496 */
2497static inline bool skb_needs_check(struct sk_buff *skb, bool tx_path)
2498{
2499 if (tx_path)
2500 return skb->ip_summed != CHECKSUM_PARTIAL;
2501 else
2502 return skb->ip_summed == CHECKSUM_NONE;
2503}
2504
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002505/**
Cong Wang12b00042013-02-05 16:36:38 +00002506 * __skb_gso_segment - Perform segmentation on skb.
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002507 * @skb: buffer to segment
Herbert Xu576a30e2006-06-27 13:22:38 -07002508 * @features: features for the output path (see dev->features)
Cong Wang12b00042013-02-05 16:36:38 +00002509 * @tx_path: whether it is called in TX path
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002510 *
2511 * This function segments the given skb and returns a list of segments.
Herbert Xu576a30e2006-06-27 13:22:38 -07002512 *
2513 * It may return NULL if the skb requires no segmentation. This is
2514 * only possible when GSO is used for verifying header integrity.
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002515 */
Cong Wang12b00042013-02-05 16:36:38 +00002516struct sk_buff *__skb_gso_segment(struct sk_buff *skb,
2517 netdev_features_t features, bool tx_path)
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002518{
Cong Wang12b00042013-02-05 16:36:38 +00002519 if (unlikely(skb_needs_check(skb, tx_path))) {
Pravin B Shelar05e8ef42013-02-14 09:44:55 +00002520 int err;
2521
Ben Hutchings36c92472012-01-17 07:57:56 +00002522 skb_warn_bad_offload(skb);
Herbert Xu67fd1a72009-01-19 16:26:44 -08002523
françois romieua40e0a62014-07-15 23:55:35 +02002524 err = skb_cow_head(skb, 0);
2525 if (err < 0)
Herbert Xua430a432006-07-08 13:34:56 -07002526 return ERR_PTR(err);
2527 }
2528
Pravin B Shelar68c33162013-02-14 14:02:41 +00002529 SKB_GSO_CB(skb)->mac_offset = skb_headroom(skb);
Eric Dumazet3347c962013-10-19 11:42:56 -07002530 SKB_GSO_CB(skb)->encap_level = 0;
2531
Pravin B Shelar05e8ef42013-02-14 09:44:55 +00002532 skb_reset_mac_header(skb);
2533 skb_reset_mac_len(skb);
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002534
Pravin B Shelar05e8ef42013-02-14 09:44:55 +00002535 return skb_mac_gso_segment(skb, features);
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002536}
Cong Wang12b00042013-02-05 16:36:38 +00002537EXPORT_SYMBOL(__skb_gso_segment);
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002538
Herbert Xufb286bb2005-11-10 13:01:24 -08002539/* Take action when hardware reception checksum errors are detected. */
2540#ifdef CONFIG_BUG
2541void netdev_rx_csum_fault(struct net_device *dev)
2542{
2543 if (net_ratelimit()) {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00002544 pr_err("%s: hw csum failure\n", dev ? dev->name : "<unknown>");
Herbert Xufb286bb2005-11-10 13:01:24 -08002545 dump_stack();
2546 }
2547}
2548EXPORT_SYMBOL(netdev_rx_csum_fault);
2549#endif
2550
Linus Torvalds1da177e2005-04-16 15:20:36 -07002551/* Actually, we should eliminate this check as soon as we know, that:
2552 * 1. IOMMU is present and allows to map all the memory.
2553 * 2. No high memory really exists on this machine.
2554 */
2555
Florian Westphalc1e756b2014-05-05 15:00:44 +02002556static int illegal_highdma(struct net_device *dev, struct sk_buff *skb)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002557{
Herbert Xu3d3a8532006-06-27 13:33:10 -07002558#ifdef CONFIG_HIGHMEM
Linus Torvalds1da177e2005-04-16 15:20:36 -07002559 int i;
FUJITA Tomonori5acbbd42010-03-30 22:35:50 +00002560 if (!(dev->features & NETIF_F_HIGHDMA)) {
Ian Campbellea2ab692011-08-22 23:44:58 +00002561 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
2562 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
2563 if (PageHighMem(skb_frag_page(frag)))
FUJITA Tomonori5acbbd42010-03-30 22:35:50 +00002564 return 1;
Ian Campbellea2ab692011-08-22 23:44:58 +00002565 }
FUJITA Tomonori5acbbd42010-03-30 22:35:50 +00002566 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002567
FUJITA Tomonori5acbbd42010-03-30 22:35:50 +00002568 if (PCI_DMA_BUS_IS_PHYS) {
2569 struct device *pdev = dev->dev.parent;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002570
Eric Dumazet9092c652010-04-02 13:34:49 -07002571 if (!pdev)
2572 return 0;
FUJITA Tomonori5acbbd42010-03-30 22:35:50 +00002573 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
Ian Campbellea2ab692011-08-22 23:44:58 +00002574 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
2575 dma_addr_t addr = page_to_phys(skb_frag_page(frag));
FUJITA Tomonori5acbbd42010-03-30 22:35:50 +00002576 if (!pdev->dma_mask || addr + PAGE_SIZE - 1 > *pdev->dma_mask)
2577 return 1;
2578 }
2579 }
Herbert Xu3d3a8532006-06-27 13:33:10 -07002580#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07002581 return 0;
2582}
Linus Torvalds1da177e2005-04-16 15:20:36 -07002583
Simon Horman3b392dd2014-06-04 08:53:17 +09002584/* If MPLS offload request, verify we are testing hardware MPLS features
2585 * instead of standard features for the netdev.
2586 */
Pravin B Shelard0edc7b2014-12-23 16:20:11 -08002587#if IS_ENABLED(CONFIG_NET_MPLS_GSO)
Simon Horman3b392dd2014-06-04 08:53:17 +09002588static netdev_features_t net_mpls_features(struct sk_buff *skb,
2589 netdev_features_t features,
2590 __be16 type)
2591{
Simon Horman25cd9ba2014-10-06 05:05:13 -07002592 if (eth_p_mpls(type))
Simon Horman3b392dd2014-06-04 08:53:17 +09002593 features &= skb->dev->mpls_features;
2594
2595 return features;
2596}
2597#else
2598static netdev_features_t net_mpls_features(struct sk_buff *skb,
2599 netdev_features_t features,
2600 __be16 type)
2601{
2602 return features;
2603}
2604#endif
2605
Michał Mirosławc8f44af2011-11-15 15:29:55 +00002606static netdev_features_t harmonize_features(struct sk_buff *skb,
Florian Westphalc1e756b2014-05-05 15:00:44 +02002607 netdev_features_t features)
Jesse Grossf01a5232011-01-09 06:23:31 +00002608{
Vlad Yasevich53d64712014-03-27 17:26:18 -04002609 int tmp;
Simon Horman3b392dd2014-06-04 08:53:17 +09002610 __be16 type;
2611
2612 type = skb_network_protocol(skb, &tmp);
2613 features = net_mpls_features(skb, features, type);
Vlad Yasevich53d64712014-03-27 17:26:18 -04002614
Ed Cashinc0d680e2012-09-19 15:49:00 +00002615 if (skb->ip_summed != CHECKSUM_NONE &&
Simon Horman3b392dd2014-06-04 08:53:17 +09002616 !can_checksum_protocol(features, type)) {
Jesse Grossf01a5232011-01-09 06:23:31 +00002617 features &= ~NETIF_F_ALL_CSUM;
Florian Westphalc1e756b2014-05-05 15:00:44 +02002618 } else if (illegal_highdma(skb->dev, skb)) {
Jesse Grossf01a5232011-01-09 06:23:31 +00002619 features &= ~NETIF_F_SG;
2620 }
2621
2622 return features;
2623}
2624
Toshiaki Makitae38f3022015-03-27 14:31:13 +09002625netdev_features_t passthru_features_check(struct sk_buff *skb,
2626 struct net_device *dev,
2627 netdev_features_t features)
2628{
2629 return features;
2630}
2631EXPORT_SYMBOL(passthru_features_check);
2632
Toshiaki Makita8cb65d02015-03-27 14:31:12 +09002633static netdev_features_t dflt_features_check(const struct sk_buff *skb,
2634 struct net_device *dev,
2635 netdev_features_t features)
2636{
2637 return vlan_features_check(skb, features);
2638}
2639
Florian Westphalc1e756b2014-05-05 15:00:44 +02002640netdev_features_t netif_skb_features(struct sk_buff *skb)
Jesse Gross58e998c2010-10-29 12:14:55 +00002641{
Jesse Gross5f352272014-12-23 22:37:26 -08002642 struct net_device *dev = skb->dev;
Eric Dumazetfcbeb972014-10-05 10:11:27 -07002643 netdev_features_t features = dev->features;
2644 u16 gso_segs = skb_shinfo(skb)->gso_segs;
Jesse Gross58e998c2010-10-29 12:14:55 +00002645
Eric Dumazetfcbeb972014-10-05 10:11:27 -07002646 if (gso_segs > dev->gso_max_segs || gso_segs < dev->gso_min_segs)
Ben Hutchings30b678d2012-07-30 15:57:00 +00002647 features &= ~NETIF_F_GSO_MASK;
2648
Jesse Gross5f352272014-12-23 22:37:26 -08002649 /* If encapsulation offload request, verify we are testing
2650 * hardware encapsulation features instead of standard
2651 * features for the netdev
2652 */
2653 if (skb->encapsulation)
2654 features &= dev->hw_enc_features;
2655
Toshiaki Makitaf5a7fb82015-03-27 14:31:11 +09002656 if (skb_vlan_tagged(skb))
2657 features = netdev_intersect_features(features,
2658 dev->vlan_features |
2659 NETIF_F_HW_VLAN_CTAG_TX |
2660 NETIF_F_HW_VLAN_STAG_TX);
Jesse Gross58e998c2010-10-29 12:14:55 +00002661
Jesse Gross5f352272014-12-23 22:37:26 -08002662 if (dev->netdev_ops->ndo_features_check)
2663 features &= dev->netdev_ops->ndo_features_check(skb, dev,
2664 features);
Toshiaki Makita8cb65d02015-03-27 14:31:12 +09002665 else
2666 features &= dflt_features_check(skb, dev, features);
Jesse Gross5f352272014-12-23 22:37:26 -08002667
Florian Westphalc1e756b2014-05-05 15:00:44 +02002668 return harmonize_features(skb, features);
Jesse Gross58e998c2010-10-29 12:14:55 +00002669}
Florian Westphalc1e756b2014-05-05 15:00:44 +02002670EXPORT_SYMBOL(netif_skb_features);
Jesse Gross58e998c2010-10-29 12:14:55 +00002671
David S. Miller2ea25512014-08-29 21:10:01 -07002672static int xmit_one(struct sk_buff *skb, struct net_device *dev,
David S. Miller95f6b3d2014-08-29 21:57:30 -07002673 struct netdev_queue *txq, bool more)
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002674{
David S. Miller2ea25512014-08-29 21:10:01 -07002675 unsigned int len;
2676 int rc;
Stephen Hemminger00829822008-11-20 20:14:53 -08002677
Salam Noureddine7866a622015-01-27 11:35:48 -08002678 if (!list_empty(&ptype_all) || !list_empty(&dev->ptype_all))
David S. Miller2ea25512014-08-29 21:10:01 -07002679 dev_queue_xmit_nit(skb, dev);
Jesse Grossfc741212011-01-09 06:23:32 +00002680
David S. Miller2ea25512014-08-29 21:10:01 -07002681 len = skb->len;
2682 trace_net_dev_start_xmit(skb, dev);
David S. Miller95f6b3d2014-08-29 21:57:30 -07002683 rc = netdev_start_xmit(skb, dev, txq, more);
David S. Miller2ea25512014-08-29 21:10:01 -07002684 trace_net_dev_xmit(skb, rc, dev, len);
Eric Dumazetadf30902009-06-02 05:19:30 +00002685
Patrick McHardy572a9d72009-11-10 06:14:14 +00002686 return rc;
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002687}
David S. Miller2ea25512014-08-29 21:10:01 -07002688
David S. Miller8dcda222014-09-01 15:06:40 -07002689struct sk_buff *dev_hard_start_xmit(struct sk_buff *first, struct net_device *dev,
2690 struct netdev_queue *txq, int *ret)
David S. Miller7f2e8702014-08-29 21:19:14 -07002691{
2692 struct sk_buff *skb = first;
2693 int rc = NETDEV_TX_OK;
2694
2695 while (skb) {
2696 struct sk_buff *next = skb->next;
2697
2698 skb->next = NULL;
David S. Miller95f6b3d2014-08-29 21:57:30 -07002699 rc = xmit_one(skb, dev, txq, next != NULL);
David S. Miller7f2e8702014-08-29 21:19:14 -07002700 if (unlikely(!dev_xmit_complete(rc))) {
2701 skb->next = next;
2702 goto out;
2703 }
2704
2705 skb = next;
2706 if (netif_xmit_stopped(txq) && skb) {
2707 rc = NETDEV_TX_BUSY;
2708 break;
2709 }
2710 }
2711
2712out:
2713 *ret = rc;
2714 return skb;
2715}
2716
Eric Dumazet1ff0dc92014-10-06 11:26:27 -07002717static struct sk_buff *validate_xmit_vlan(struct sk_buff *skb,
2718 netdev_features_t features)
David S. Millereae3f882014-08-30 15:17:13 -07002719{
Jiri Pirkodf8a39d2015-01-13 17:13:44 +01002720 if (skb_vlan_tag_present(skb) &&
Jiri Pirko59682502014-11-19 14:04:59 +01002721 !vlan_hw_offload_capable(features, skb->vlan_proto))
2722 skb = __vlan_hwaccel_push_inside(skb);
David S. Millereae3f882014-08-30 15:17:13 -07002723 return skb;
2724}
2725
Eric Dumazet55a93b32014-10-03 15:31:07 -07002726static struct sk_buff *validate_xmit_skb(struct sk_buff *skb, struct net_device *dev)
David S. Millereae3f882014-08-30 15:17:13 -07002727{
2728 netdev_features_t features;
2729
2730 if (skb->next)
2731 return skb;
2732
David S. Millereae3f882014-08-30 15:17:13 -07002733 features = netif_skb_features(skb);
2734 skb = validate_xmit_vlan(skb, features);
2735 if (unlikely(!skb))
2736 goto out_null;
2737
Johannes Berg8b86a612015-04-17 15:45:04 +02002738 if (netif_needs_gso(skb, features)) {
David S. Millerce937182014-08-30 19:22:20 -07002739 struct sk_buff *segs;
2740
2741 segs = skb_gso_segment(skb, features);
Jason Wangcecda692014-09-19 16:04:38 +08002742 if (IS_ERR(segs)) {
Jason Wangaf6dabc2014-12-19 11:09:13 +08002743 goto out_kfree_skb;
Jason Wangcecda692014-09-19 16:04:38 +08002744 } else if (segs) {
2745 consume_skb(skb);
2746 skb = segs;
2747 }
David S. Millereae3f882014-08-30 15:17:13 -07002748 } else {
2749 if (skb_needs_linearize(skb, features) &&
2750 __skb_linearize(skb))
2751 goto out_kfree_skb;
2752
2753 /* If packet is not checksummed and device does not
2754 * support checksumming for this protocol, complete
2755 * checksumming here.
2756 */
2757 if (skb->ip_summed == CHECKSUM_PARTIAL) {
2758 if (skb->encapsulation)
2759 skb_set_inner_transport_header(skb,
2760 skb_checksum_start_offset(skb));
2761 else
2762 skb_set_transport_header(skb,
2763 skb_checksum_start_offset(skb));
2764 if (!(features & NETIF_F_ALL_CSUM) &&
2765 skb_checksum_help(skb))
2766 goto out_kfree_skb;
2767 }
2768 }
2769
2770 return skb;
2771
2772out_kfree_skb:
2773 kfree_skb(skb);
2774out_null:
2775 return NULL;
2776}
2777
Eric Dumazet55a93b32014-10-03 15:31:07 -07002778struct sk_buff *validate_xmit_skb_list(struct sk_buff *skb, struct net_device *dev)
2779{
2780 struct sk_buff *next, *head = NULL, *tail;
2781
Eric Dumazetbec3cfd2014-10-03 20:59:19 -07002782 for (; skb != NULL; skb = next) {
Eric Dumazet55a93b32014-10-03 15:31:07 -07002783 next = skb->next;
2784 skb->next = NULL;
Eric Dumazet55a93b32014-10-03 15:31:07 -07002785
Eric Dumazetbec3cfd2014-10-03 20:59:19 -07002786 /* in case skb wont be segmented, point to itself */
2787 skb->prev = skb;
2788
2789 skb = validate_xmit_skb(skb, dev);
2790 if (!skb)
2791 continue;
2792
2793 if (!head)
2794 head = skb;
2795 else
2796 tail->next = skb;
2797 /* If skb was segmented, skb->prev points to
2798 * the last segment. If not, it still contains skb.
2799 */
2800 tail = skb->prev;
Eric Dumazet55a93b32014-10-03 15:31:07 -07002801 }
2802 return head;
2803}
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002804
Eric Dumazet1def9232013-01-10 12:36:42 +00002805static void qdisc_pkt_len_init(struct sk_buff *skb)
2806{
2807 const struct skb_shared_info *shinfo = skb_shinfo(skb);
2808
2809 qdisc_skb_cb(skb)->pkt_len = skb->len;
2810
2811 /* To get more precise estimation of bytes sent on wire,
2812 * we add to pkt_len the headers size of all segments
2813 */
2814 if (shinfo->gso_size) {
Eric Dumazet757b8b12013-01-15 21:14:21 -08002815 unsigned int hdr_len;
Jason Wang15e5a032013-03-25 20:19:59 +00002816 u16 gso_segs = shinfo->gso_segs;
Eric Dumazet1def9232013-01-10 12:36:42 +00002817
Eric Dumazet757b8b12013-01-15 21:14:21 -08002818 /* mac layer + network layer */
2819 hdr_len = skb_transport_header(skb) - skb_mac_header(skb);
2820
2821 /* + transport layer */
Eric Dumazet1def9232013-01-10 12:36:42 +00002822 if (likely(shinfo->gso_type & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6)))
2823 hdr_len += tcp_hdrlen(skb);
2824 else
2825 hdr_len += sizeof(struct udphdr);
Jason Wang15e5a032013-03-25 20:19:59 +00002826
2827 if (shinfo->gso_type & SKB_GSO_DODGY)
2828 gso_segs = DIV_ROUND_UP(skb->len - hdr_len,
2829 shinfo->gso_size);
2830
2831 qdisc_skb_cb(skb)->pkt_len += (gso_segs - 1) * hdr_len;
Eric Dumazet1def9232013-01-10 12:36:42 +00002832 }
2833}
2834
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00002835static inline int __dev_xmit_skb(struct sk_buff *skb, struct Qdisc *q,
2836 struct net_device *dev,
2837 struct netdev_queue *txq)
2838{
2839 spinlock_t *root_lock = qdisc_lock(q);
Eric Dumazeta2da5702011-01-20 03:48:19 +00002840 bool contended;
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00002841 int rc;
2842
Eric Dumazet1def9232013-01-10 12:36:42 +00002843 qdisc_pkt_len_init(skb);
Eric Dumazeta2da5702011-01-20 03:48:19 +00002844 qdisc_calculate_pkt_len(skb, q);
Eric Dumazet79640a42010-06-02 05:09:29 -07002845 /*
2846 * Heuristic to force contended enqueues to serialize on a
2847 * separate lock before trying to get qdisc main lock.
Ying Xue9bf2b8c2014-06-26 15:56:31 +08002848 * This permits __QDISC___STATE_RUNNING owner to get the lock more
2849 * often and dequeue packets faster.
Eric Dumazet79640a42010-06-02 05:09:29 -07002850 */
Eric Dumazeta2da5702011-01-20 03:48:19 +00002851 contended = qdisc_is_running(q);
Eric Dumazet79640a42010-06-02 05:09:29 -07002852 if (unlikely(contended))
2853 spin_lock(&q->busylock);
2854
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00002855 spin_lock(root_lock);
2856 if (unlikely(test_bit(__QDISC_STATE_DEACTIVATED, &q->state))) {
2857 kfree_skb(skb);
2858 rc = NET_XMIT_DROP;
2859 } else if ((q->flags & TCQ_F_CAN_BYPASS) && !qdisc_qlen(q) &&
Eric Dumazetbc135b22010-06-02 03:23:51 -07002860 qdisc_run_begin(q)) {
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00002861 /*
2862 * This is a work-conserving queue; there are no old skbs
2863 * waiting to be sent out; and the qdisc is not running -
2864 * xmit the skb directly.
2865 */
Eric Dumazetbfe0d022011-01-09 08:30:54 +00002866
Eric Dumazetbfe0d022011-01-09 08:30:54 +00002867 qdisc_bstats_update(q, skb);
2868
Eric Dumazet55a93b32014-10-03 15:31:07 -07002869 if (sch_direct_xmit(skb, q, dev, txq, root_lock, true)) {
Eric Dumazet79640a42010-06-02 05:09:29 -07002870 if (unlikely(contended)) {
2871 spin_unlock(&q->busylock);
2872 contended = false;
2873 }
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00002874 __qdisc_run(q);
Eric Dumazet79640a42010-06-02 05:09:29 -07002875 } else
Eric Dumazetbc135b22010-06-02 03:23:51 -07002876 qdisc_run_end(q);
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00002877
2878 rc = NET_XMIT_SUCCESS;
2879 } else {
Eric Dumazeta2da5702011-01-20 03:48:19 +00002880 rc = q->enqueue(skb, q) & NET_XMIT_MASK;
Eric Dumazet79640a42010-06-02 05:09:29 -07002881 if (qdisc_run_begin(q)) {
2882 if (unlikely(contended)) {
2883 spin_unlock(&q->busylock);
2884 contended = false;
2885 }
2886 __qdisc_run(q);
2887 }
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00002888 }
2889 spin_unlock(root_lock);
Eric Dumazet79640a42010-06-02 05:09:29 -07002890 if (unlikely(contended))
2891 spin_unlock(&q->busylock);
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00002892 return rc;
2893}
2894
Daniel Borkmann86f85152013-12-29 17:27:11 +01002895#if IS_ENABLED(CONFIG_CGROUP_NET_PRIO)
Neil Horman5bc14212011-11-22 05:10:51 +00002896static void skb_update_prio(struct sk_buff *skb)
2897{
Igor Maravic6977a792011-11-25 07:44:54 +00002898 struct netprio_map *map = rcu_dereference_bh(skb->dev->priomap);
Neil Horman5bc14212011-11-22 05:10:51 +00002899
Eric Dumazet91c68ce2012-07-08 21:45:10 +00002900 if (!skb->priority && skb->sk && map) {
2901 unsigned int prioidx = skb->sk->sk_cgrp_prioidx;
2902
2903 if (prioidx < map->priomap_len)
2904 skb->priority = map->priomap[prioidx];
2905 }
Neil Horman5bc14212011-11-22 05:10:51 +00002906}
2907#else
2908#define skb_update_prio(skb)
2909#endif
2910
hannes@stressinduktion.orgf60e5992015-04-01 17:07:44 +02002911DEFINE_PER_CPU(int, xmit_recursion);
2912EXPORT_SYMBOL(xmit_recursion);
2913
David S. Miller11a766c2010-10-25 12:51:55 -07002914#define RECURSION_LIMIT 10
Eric Dumazet745e20f2010-09-29 13:23:09 -07002915
Dave Jonesd29f7492008-07-22 14:09:06 -07002916/**
Michel Machado95603e22012-06-12 10:16:35 +00002917 * dev_loopback_xmit - loop back @skb
2918 * @skb: buffer to transmit
2919 */
David Miller7026b1d2015-04-05 22:19:04 -04002920int dev_loopback_xmit(struct sock *sk, struct sk_buff *skb)
Michel Machado95603e22012-06-12 10:16:35 +00002921{
2922 skb_reset_mac_header(skb);
2923 __skb_pull(skb, skb_network_offset(skb));
2924 skb->pkt_type = PACKET_LOOPBACK;
2925 skb->ip_summed = CHECKSUM_UNNECESSARY;
2926 WARN_ON(!skb_dst(skb));
2927 skb_dst_force(skb);
2928 netif_rx_ni(skb);
2929 return 0;
2930}
2931EXPORT_SYMBOL(dev_loopback_xmit);
2932
Jiri Pirko638b2a62015-05-12 14:56:13 +02002933static inline int get_xps_queue(struct net_device *dev, struct sk_buff *skb)
2934{
2935#ifdef CONFIG_XPS
2936 struct xps_dev_maps *dev_maps;
2937 struct xps_map *map;
2938 int queue_index = -1;
2939
2940 rcu_read_lock();
2941 dev_maps = rcu_dereference(dev->xps_maps);
2942 if (dev_maps) {
2943 map = rcu_dereference(
2944 dev_maps->cpu_map[skb->sender_cpu - 1]);
2945 if (map) {
2946 if (map->len == 1)
2947 queue_index = map->queues[0];
2948 else
2949 queue_index = map->queues[reciprocal_scale(skb_get_hash(skb),
2950 map->len)];
2951 if (unlikely(queue_index >= dev->real_num_tx_queues))
2952 queue_index = -1;
2953 }
2954 }
2955 rcu_read_unlock();
2956
2957 return queue_index;
2958#else
2959 return -1;
2960#endif
2961}
2962
2963static u16 __netdev_pick_tx(struct net_device *dev, struct sk_buff *skb)
2964{
2965 struct sock *sk = skb->sk;
2966 int queue_index = sk_tx_queue_get(sk);
2967
2968 if (queue_index < 0 || skb->ooo_okay ||
2969 queue_index >= dev->real_num_tx_queues) {
2970 int new_index = get_xps_queue(dev, skb);
2971 if (new_index < 0)
2972 new_index = skb_tx_hash(dev, skb);
2973
2974 if (queue_index != new_index && sk &&
2975 rcu_access_pointer(sk->sk_dst_cache))
2976 sk_tx_queue_set(sk, new_index);
2977
2978 queue_index = new_index;
2979 }
2980
2981 return queue_index;
2982}
2983
2984struct netdev_queue *netdev_pick_tx(struct net_device *dev,
2985 struct sk_buff *skb,
2986 void *accel_priv)
2987{
2988 int queue_index = 0;
2989
2990#ifdef CONFIG_XPS
2991 if (skb->sender_cpu == 0)
2992 skb->sender_cpu = raw_smp_processor_id() + 1;
2993#endif
2994
2995 if (dev->real_num_tx_queues != 1) {
2996 const struct net_device_ops *ops = dev->netdev_ops;
2997 if (ops->ndo_select_queue)
2998 queue_index = ops->ndo_select_queue(dev, skb, accel_priv,
2999 __netdev_pick_tx);
3000 else
3001 queue_index = __netdev_pick_tx(dev, skb);
3002
3003 if (!accel_priv)
3004 queue_index = netdev_cap_txqueue(dev, queue_index);
3005 }
3006
3007 skb_set_queue_mapping(skb, queue_index);
3008 return netdev_get_tx_queue(dev, queue_index);
3009}
3010
Michel Machado95603e22012-06-12 10:16:35 +00003011/**
Jason Wang9d08dd32014-01-20 11:25:13 +08003012 * __dev_queue_xmit - transmit a buffer
Dave Jonesd29f7492008-07-22 14:09:06 -07003013 * @skb: buffer to transmit
Jason Wang9d08dd32014-01-20 11:25:13 +08003014 * @accel_priv: private data used for L2 forwarding offload
Dave Jonesd29f7492008-07-22 14:09:06 -07003015 *
3016 * Queue a buffer for transmission to a network device. The caller must
3017 * have set the device and priority and built the buffer before calling
3018 * this function. The function can be called from an interrupt.
3019 *
3020 * A negative errno code is returned on a failure. A success does not
3021 * guarantee the frame will be transmitted as it may be dropped due
3022 * to congestion or traffic shaping.
3023 *
3024 * -----------------------------------------------------------------------------------
3025 * I notice this method can also return errors from the queue disciplines,
3026 * including NET_XMIT_DROP, which is a positive value. So, errors can also
3027 * be positive.
3028 *
3029 * Regardless of the return value, the skb is consumed, so it is currently
3030 * difficult to retry a send to this method. (You can bump the ref count
3031 * before sending to hold a reference for retry if you are careful.)
3032 *
3033 * When calling this method, interrupts MUST be enabled. This is because
3034 * the BH enable code must have IRQs enabled so that it will not deadlock.
3035 * --BLG
3036 */
Rashika Kheria0a59f3a2014-02-09 20:26:25 +05303037static int __dev_queue_xmit(struct sk_buff *skb, void *accel_priv)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003038{
3039 struct net_device *dev = skb->dev;
David S. Millerdc2b4842008-07-08 17:18:23 -07003040 struct netdev_queue *txq;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003041 struct Qdisc *q;
3042 int rc = -ENOMEM;
3043
Eric Dumazet6d1ccff2013-02-05 20:22:20 +00003044 skb_reset_mac_header(skb);
3045
Willem de Bruijne7fd2882014-08-04 22:11:48 -04003046 if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_SCHED_TSTAMP))
3047 __skb_tstamp_tx(skb, NULL, skb->sk, SCM_TSTAMP_SCHED);
3048
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09003049 /* Disable soft irqs for various locks below. Also
3050 * stops preemption for RCU.
Linus Torvalds1da177e2005-04-16 15:20:36 -07003051 */
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09003052 rcu_read_lock_bh();
Linus Torvalds1da177e2005-04-16 15:20:36 -07003053
Neil Horman5bc14212011-11-22 05:10:51 +00003054 skb_update_prio(skb);
3055
Eric Dumazet02875872014-10-05 18:38:35 -07003056 /* If device/qdisc don't need skb->dst, release it right now while
3057 * its hot in this cpu cache.
3058 */
3059 if (dev->priv_flags & IFF_XMIT_DST_RELEASE)
3060 skb_dst_drop(skb);
3061 else
3062 skb_dst_force(skb);
3063
Jason Wangf663dd92014-01-10 16:18:26 +08003064 txq = netdev_pick_tx(dev, skb, accel_priv);
Paul E. McKenneya898def2010-02-22 17:04:49 -08003065 q = rcu_dereference_bh(txq->qdisc);
David S. Miller37437bb2008-07-16 02:15:04 -07003066
Linus Torvalds1da177e2005-04-16 15:20:36 -07003067#ifdef CONFIG_NET_CLS_ACT
Eric Dumazetd1b19df2009-09-03 01:29:39 -07003068 skb->tc_verd = SET_TC_AT(skb->tc_verd, AT_EGRESS);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003069#endif
Koki Sanagicf66ba52010-08-23 18:45:02 +09003070 trace_net_dev_queue(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003071 if (q->enqueue) {
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00003072 rc = __dev_xmit_skb(skb, q, dev, txq);
David S. Miller37437bb2008-07-16 02:15:04 -07003073 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003074 }
3075
3076 /* The device has no queue. Common case for software devices:
3077 loopback, all the sorts of tunnels...
3078
Herbert Xu932ff272006-06-09 12:20:56 -07003079 Really, it is unlikely that netif_tx_lock protection is necessary
3080 here. (f.e. loopback and IP tunnels are clean ignoring statistics
Linus Torvalds1da177e2005-04-16 15:20:36 -07003081 counters.)
3082 However, it is possible, that they rely on protection
3083 made by us here.
3084
3085 Check this and shot the lock. It is not prone from deadlocks.
3086 Either shot noqueue qdisc, it is even simpler 8)
3087 */
3088 if (dev->flags & IFF_UP) {
3089 int cpu = smp_processor_id(); /* ok because BHs are off */
3090
David S. Millerc773e842008-07-08 23:13:53 -07003091 if (txq->xmit_lock_owner != cpu) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07003092
Eric Dumazet745e20f2010-09-29 13:23:09 -07003093 if (__this_cpu_read(xmit_recursion) > RECURSION_LIMIT)
3094 goto recursion_alert;
3095
Jesper Dangaard Brouer1f595332014-09-03 17:56:09 +02003096 skb = validate_xmit_skb(skb, dev);
3097 if (!skb)
3098 goto drop;
3099
David S. Millerc773e842008-07-08 23:13:53 -07003100 HARD_TX_LOCK(dev, txq, cpu);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003101
Tom Herbert734664982011-11-28 16:32:44 +00003102 if (!netif_xmit_stopped(txq)) {
Eric Dumazet745e20f2010-09-29 13:23:09 -07003103 __this_cpu_inc(xmit_recursion);
David S. Millerce937182014-08-30 19:22:20 -07003104 skb = dev_hard_start_xmit(skb, dev, txq, &rc);
Eric Dumazet745e20f2010-09-29 13:23:09 -07003105 __this_cpu_dec(xmit_recursion);
Patrick McHardy572a9d72009-11-10 06:14:14 +00003106 if (dev_xmit_complete(rc)) {
David S. Millerc773e842008-07-08 23:13:53 -07003107 HARD_TX_UNLOCK(dev, txq);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003108 goto out;
3109 }
3110 }
David S. Millerc773e842008-07-08 23:13:53 -07003111 HARD_TX_UNLOCK(dev, txq);
Joe Perchese87cc472012-05-13 21:56:26 +00003112 net_crit_ratelimited("Virtual device %s asks to queue packet!\n",
3113 dev->name);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003114 } else {
3115 /* Recursion is detected! It is possible,
Eric Dumazet745e20f2010-09-29 13:23:09 -07003116 * unfortunately
3117 */
3118recursion_alert:
Joe Perchese87cc472012-05-13 21:56:26 +00003119 net_crit_ratelimited("Dead loop on virtual device %s, fix it urgently!\n",
3120 dev->name);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003121 }
3122 }
3123
3124 rc = -ENETDOWN;
Jesper Dangaard Brouer1f595332014-09-03 17:56:09 +02003125drop:
Herbert Xud4828d82006-06-22 02:28:18 -07003126 rcu_read_unlock_bh();
Linus Torvalds1da177e2005-04-16 15:20:36 -07003127
Eric Dumazet015f0682014-03-27 08:45:56 -07003128 atomic_long_inc(&dev->tx_dropped);
Jesper Dangaard Brouer1f595332014-09-03 17:56:09 +02003129 kfree_skb_list(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003130 return rc;
3131out:
Herbert Xud4828d82006-06-22 02:28:18 -07003132 rcu_read_unlock_bh();
Linus Torvalds1da177e2005-04-16 15:20:36 -07003133 return rc;
3134}
Jason Wangf663dd92014-01-10 16:18:26 +08003135
David Miller7026b1d2015-04-05 22:19:04 -04003136int dev_queue_xmit_sk(struct sock *sk, struct sk_buff *skb)
Jason Wangf663dd92014-01-10 16:18:26 +08003137{
3138 return __dev_queue_xmit(skb, NULL);
3139}
David Miller7026b1d2015-04-05 22:19:04 -04003140EXPORT_SYMBOL(dev_queue_xmit_sk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003141
Jason Wangf663dd92014-01-10 16:18:26 +08003142int dev_queue_xmit_accel(struct sk_buff *skb, void *accel_priv)
3143{
3144 return __dev_queue_xmit(skb, accel_priv);
3145}
3146EXPORT_SYMBOL(dev_queue_xmit_accel);
3147
Linus Torvalds1da177e2005-04-16 15:20:36 -07003148
3149/*=======================================================================
3150 Receiver routines
3151 =======================================================================*/
3152
Stephen Hemminger6b2bedc2007-03-12 14:33:50 -07003153int netdev_max_backlog __read_mostly = 1000;
Eric Dumazetc9e6bc62012-09-27 19:29:05 +00003154EXPORT_SYMBOL(netdev_max_backlog);
3155
Eric Dumazet3b098e22010-05-15 23:57:10 -07003156int netdev_tstamp_prequeue __read_mostly = 1;
Stephen Hemminger6b2bedc2007-03-12 14:33:50 -07003157int netdev_budget __read_mostly = 300;
3158int weight_p __read_mostly = 64; /* old backlog weight */
Linus Torvalds1da177e2005-04-16 15:20:36 -07003159
Eric Dumazeteecfd7c2010-05-06 22:07:48 -07003160/* Called with irq disabled */
3161static inline void ____napi_schedule(struct softnet_data *sd,
3162 struct napi_struct *napi)
3163{
3164 list_add_tail(&napi->poll_list, &sd->poll_list);
3165 __raise_softirq_irqoff(NET_RX_SOFTIRQ);
3166}
3167
Eric Dumazetdf334542010-03-24 19:13:54 +00003168#ifdef CONFIG_RPS
Tom Herbertfec5e652010-04-16 16:01:27 -07003169
3170/* One global table that all flow-based protocols share. */
Eric Dumazet6e3f7fa2010-10-25 03:02:02 +00003171struct rps_sock_flow_table __rcu *rps_sock_flow_table __read_mostly;
Tom Herbertfec5e652010-04-16 16:01:27 -07003172EXPORT_SYMBOL(rps_sock_flow_table);
Eric Dumazet567e4b72015-02-06 12:59:01 -08003173u32 rps_cpu_mask __read_mostly;
3174EXPORT_SYMBOL(rps_cpu_mask);
Tom Herbertfec5e652010-04-16 16:01:27 -07003175
Ingo Molnarc5905af2012-02-24 08:31:31 +01003176struct static_key rps_needed __read_mostly;
Eric Dumazetadc93002011-11-17 03:13:26 +00003177
Ben Hutchingsc4454772011-01-19 11:03:53 +00003178static struct rps_dev_flow *
3179set_rps_cpu(struct net_device *dev, struct sk_buff *skb,
3180 struct rps_dev_flow *rflow, u16 next_cpu)
3181{
Eric Dumazeta31196b2015-04-25 09:35:24 -07003182 if (next_cpu < nr_cpu_ids) {
Ben Hutchingsc4454772011-01-19 11:03:53 +00003183#ifdef CONFIG_RFS_ACCEL
3184 struct netdev_rx_queue *rxqueue;
3185 struct rps_dev_flow_table *flow_table;
3186 struct rps_dev_flow *old_rflow;
3187 u32 flow_id;
3188 u16 rxq_index;
3189 int rc;
3190
3191 /* Should we steer this flow to a different hardware queue? */
Ben Hutchings69a19ee2011-02-15 20:32:04 +00003192 if (!skb_rx_queue_recorded(skb) || !dev->rx_cpu_rmap ||
3193 !(dev->features & NETIF_F_NTUPLE))
Ben Hutchingsc4454772011-01-19 11:03:53 +00003194 goto out;
3195 rxq_index = cpu_rmap_lookup_index(dev->rx_cpu_rmap, next_cpu);
3196 if (rxq_index == skb_get_rx_queue(skb))
3197 goto out;
3198
3199 rxqueue = dev->_rx + rxq_index;
3200 flow_table = rcu_dereference(rxqueue->rps_flow_table);
3201 if (!flow_table)
3202 goto out;
Tom Herbert61b905d2014-03-24 15:34:47 -07003203 flow_id = skb_get_hash(skb) & flow_table->mask;
Ben Hutchingsc4454772011-01-19 11:03:53 +00003204 rc = dev->netdev_ops->ndo_rx_flow_steer(dev, skb,
3205 rxq_index, flow_id);
3206 if (rc < 0)
3207 goto out;
3208 old_rflow = rflow;
3209 rflow = &flow_table->flows[flow_id];
Ben Hutchingsc4454772011-01-19 11:03:53 +00003210 rflow->filter = rc;
3211 if (old_rflow->filter == rflow->filter)
3212 old_rflow->filter = RPS_NO_FILTER;
3213 out:
3214#endif
3215 rflow->last_qtail =
Ben Hutchings09994d12011-10-03 04:42:46 +00003216 per_cpu(softnet_data, next_cpu).input_queue_head;
Ben Hutchingsc4454772011-01-19 11:03:53 +00003217 }
3218
Ben Hutchings09994d12011-10-03 04:42:46 +00003219 rflow->cpu = next_cpu;
Ben Hutchingsc4454772011-01-19 11:03:53 +00003220 return rflow;
3221}
3222
Tom Herbert0a9627f2010-03-16 08:03:29 +00003223/*
3224 * get_rps_cpu is called from netif_receive_skb and returns the target
3225 * CPU from the RPS map of the receiving queue for a given skb.
Eric Dumazetb0e28f12010-04-15 00:14:07 -07003226 * rcu_read_lock must be held on entry.
Tom Herbert0a9627f2010-03-16 08:03:29 +00003227 */
Tom Herbertfec5e652010-04-16 16:01:27 -07003228static int get_rps_cpu(struct net_device *dev, struct sk_buff *skb,
3229 struct rps_dev_flow **rflowp)
Tom Herbert0a9627f2010-03-16 08:03:29 +00003230{
Eric Dumazet567e4b72015-02-06 12:59:01 -08003231 const struct rps_sock_flow_table *sock_flow_table;
3232 struct netdev_rx_queue *rxqueue = dev->_rx;
Tom Herbertfec5e652010-04-16 16:01:27 -07003233 struct rps_dev_flow_table *flow_table;
Eric Dumazet567e4b72015-02-06 12:59:01 -08003234 struct rps_map *map;
Tom Herbert0a9627f2010-03-16 08:03:29 +00003235 int cpu = -1;
Eric Dumazet567e4b72015-02-06 12:59:01 -08003236 u32 tcpu;
Tom Herbert61b905d2014-03-24 15:34:47 -07003237 u32 hash;
Tom Herbert0a9627f2010-03-16 08:03:29 +00003238
Tom Herbert0a9627f2010-03-16 08:03:29 +00003239 if (skb_rx_queue_recorded(skb)) {
3240 u16 index = skb_get_rx_queue(skb);
Eric Dumazet567e4b72015-02-06 12:59:01 -08003241
Ben Hutchings62fe0b42010-09-27 08:24:33 +00003242 if (unlikely(index >= dev->real_num_rx_queues)) {
3243 WARN_ONCE(dev->real_num_rx_queues > 1,
3244 "%s received packet on queue %u, but number "
3245 "of RX queues is %u\n",
3246 dev->name, index, dev->real_num_rx_queues);
Tom Herbert0a9627f2010-03-16 08:03:29 +00003247 goto done;
3248 }
Eric Dumazet567e4b72015-02-06 12:59:01 -08003249 rxqueue += index;
Tom Herbert0a9627f2010-03-16 08:03:29 +00003250 }
3251
Eric Dumazet567e4b72015-02-06 12:59:01 -08003252 /* Avoid computing hash if RFS/RPS is not active for this rxqueue */
3253
3254 flow_table = rcu_dereference(rxqueue->rps_flow_table);
3255 map = rcu_dereference(rxqueue->rps_map);
3256 if (!flow_table && !map)
3257 goto done;
3258
Changli Gao2d47b452010-08-17 19:00:56 +00003259 skb_reset_network_header(skb);
Tom Herbert61b905d2014-03-24 15:34:47 -07003260 hash = skb_get_hash(skb);
3261 if (!hash)
Tom Herbert0a9627f2010-03-16 08:03:29 +00003262 goto done;
Tom Herbert0a9627f2010-03-16 08:03:29 +00003263
Tom Herbertfec5e652010-04-16 16:01:27 -07003264 sock_flow_table = rcu_dereference(rps_sock_flow_table);
3265 if (flow_table && sock_flow_table) {
Tom Herbertfec5e652010-04-16 16:01:27 -07003266 struct rps_dev_flow *rflow;
Eric Dumazet567e4b72015-02-06 12:59:01 -08003267 u32 next_cpu;
3268 u32 ident;
Tom Herbertfec5e652010-04-16 16:01:27 -07003269
Eric Dumazet567e4b72015-02-06 12:59:01 -08003270 /* First check into global flow table if there is a match */
3271 ident = sock_flow_table->ents[hash & sock_flow_table->mask];
3272 if ((ident ^ hash) & ~rps_cpu_mask)
3273 goto try_rps;
3274
3275 next_cpu = ident & rps_cpu_mask;
3276
3277 /* OK, now we know there is a match,
3278 * we can look at the local (per receive queue) flow table
3279 */
Tom Herbert61b905d2014-03-24 15:34:47 -07003280 rflow = &flow_table->flows[hash & flow_table->mask];
Tom Herbertfec5e652010-04-16 16:01:27 -07003281 tcpu = rflow->cpu;
3282
Tom Herbertfec5e652010-04-16 16:01:27 -07003283 /*
3284 * If the desired CPU (where last recvmsg was done) is
3285 * different from current CPU (one in the rx-queue flow
3286 * table entry), switch if one of the following holds:
Eric Dumazeta31196b2015-04-25 09:35:24 -07003287 * - Current CPU is unset (>= nr_cpu_ids).
Tom Herbertfec5e652010-04-16 16:01:27 -07003288 * - Current CPU is offline.
3289 * - The current CPU's queue tail has advanced beyond the
3290 * last packet that was enqueued using this table entry.
3291 * This guarantees that all previous packets for the flow
3292 * have been dequeued, thus preserving in order delivery.
3293 */
3294 if (unlikely(tcpu != next_cpu) &&
Eric Dumazeta31196b2015-04-25 09:35:24 -07003295 (tcpu >= nr_cpu_ids || !cpu_online(tcpu) ||
Tom Herbertfec5e652010-04-16 16:01:27 -07003296 ((int)(per_cpu(softnet_data, tcpu).input_queue_head -
Tom Herbertbaefa312012-11-16 09:04:15 +00003297 rflow->last_qtail)) >= 0)) {
3298 tcpu = next_cpu;
Ben Hutchingsc4454772011-01-19 11:03:53 +00003299 rflow = set_rps_cpu(dev, skb, rflow, next_cpu);
Tom Herbertbaefa312012-11-16 09:04:15 +00003300 }
Ben Hutchingsc4454772011-01-19 11:03:53 +00003301
Eric Dumazeta31196b2015-04-25 09:35:24 -07003302 if (tcpu < nr_cpu_ids && cpu_online(tcpu)) {
Tom Herbertfec5e652010-04-16 16:01:27 -07003303 *rflowp = rflow;
3304 cpu = tcpu;
3305 goto done;
3306 }
3307 }
3308
Eric Dumazet567e4b72015-02-06 12:59:01 -08003309try_rps:
3310
Tom Herbert0a9627f2010-03-16 08:03:29 +00003311 if (map) {
Daniel Borkmann8fc54f62014-08-23 20:58:54 +02003312 tcpu = map->cpus[reciprocal_scale(hash, map->len)];
Tom Herbert0a9627f2010-03-16 08:03:29 +00003313 if (cpu_online(tcpu)) {
3314 cpu = tcpu;
3315 goto done;
3316 }
3317 }
3318
3319done:
Tom Herbert0a9627f2010-03-16 08:03:29 +00003320 return cpu;
3321}
3322
Ben Hutchingsc4454772011-01-19 11:03:53 +00003323#ifdef CONFIG_RFS_ACCEL
3324
3325/**
3326 * rps_may_expire_flow - check whether an RFS hardware filter may be removed
3327 * @dev: Device on which the filter was set
3328 * @rxq_index: RX queue index
3329 * @flow_id: Flow ID passed to ndo_rx_flow_steer()
3330 * @filter_id: Filter ID returned by ndo_rx_flow_steer()
3331 *
3332 * Drivers that implement ndo_rx_flow_steer() should periodically call
3333 * this function for each installed filter and remove the filters for
3334 * which it returns %true.
3335 */
3336bool rps_may_expire_flow(struct net_device *dev, u16 rxq_index,
3337 u32 flow_id, u16 filter_id)
3338{
3339 struct netdev_rx_queue *rxqueue = dev->_rx + rxq_index;
3340 struct rps_dev_flow_table *flow_table;
3341 struct rps_dev_flow *rflow;
3342 bool expire = true;
Eric Dumazeta31196b2015-04-25 09:35:24 -07003343 unsigned int cpu;
Ben Hutchingsc4454772011-01-19 11:03:53 +00003344
3345 rcu_read_lock();
3346 flow_table = rcu_dereference(rxqueue->rps_flow_table);
3347 if (flow_table && flow_id <= flow_table->mask) {
3348 rflow = &flow_table->flows[flow_id];
3349 cpu = ACCESS_ONCE(rflow->cpu);
Eric Dumazeta31196b2015-04-25 09:35:24 -07003350 if (rflow->filter == filter_id && cpu < nr_cpu_ids &&
Ben Hutchingsc4454772011-01-19 11:03:53 +00003351 ((int)(per_cpu(softnet_data, cpu).input_queue_head -
3352 rflow->last_qtail) <
3353 (int)(10 * flow_table->mask)))
3354 expire = false;
3355 }
3356 rcu_read_unlock();
3357 return expire;
3358}
3359EXPORT_SYMBOL(rps_may_expire_flow);
3360
3361#endif /* CONFIG_RFS_ACCEL */
3362
Tom Herbert0a9627f2010-03-16 08:03:29 +00003363/* Called from hardirq (IPI) context */
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003364static void rps_trigger_softirq(void *data)
Tom Herbert0a9627f2010-03-16 08:03:29 +00003365{
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003366 struct softnet_data *sd = data;
3367
Eric Dumazeteecfd7c2010-05-06 22:07:48 -07003368 ____napi_schedule(sd, &sd->backlog);
Changli Gaodee42872010-05-02 05:42:16 +00003369 sd->received_rps++;
Tom Herbert0a9627f2010-03-16 08:03:29 +00003370}
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003371
Tom Herbertfec5e652010-04-16 16:01:27 -07003372#endif /* CONFIG_RPS */
Tom Herbert0a9627f2010-03-16 08:03:29 +00003373
3374/*
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003375 * Check if this softnet_data structure is another cpu one
3376 * If yes, queue it to our IPI list and return 1
3377 * If no, return 0
3378 */
3379static int rps_ipi_queued(struct softnet_data *sd)
3380{
3381#ifdef CONFIG_RPS
Christoph Lameter903ceff2014-08-17 12:30:35 -05003382 struct softnet_data *mysd = this_cpu_ptr(&softnet_data);
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003383
3384 if (sd != mysd) {
3385 sd->rps_ipi_next = mysd->rps_ipi_list;
3386 mysd->rps_ipi_list = sd;
3387
3388 __raise_softirq_irqoff(NET_RX_SOFTIRQ);
3389 return 1;
3390 }
3391#endif /* CONFIG_RPS */
3392 return 0;
3393}
3394
Willem de Bruijn99bbc702013-05-20 04:02:32 +00003395#ifdef CONFIG_NET_FLOW_LIMIT
3396int netdev_flow_limit_table_len __read_mostly = (1 << 12);
3397#endif
3398
3399static bool skb_flow_limit(struct sk_buff *skb, unsigned int qlen)
3400{
3401#ifdef CONFIG_NET_FLOW_LIMIT
3402 struct sd_flow_limit *fl;
3403 struct softnet_data *sd;
3404 unsigned int old_flow, new_flow;
3405
3406 if (qlen < (netdev_max_backlog >> 1))
3407 return false;
3408
Christoph Lameter903ceff2014-08-17 12:30:35 -05003409 sd = this_cpu_ptr(&softnet_data);
Willem de Bruijn99bbc702013-05-20 04:02:32 +00003410
3411 rcu_read_lock();
3412 fl = rcu_dereference(sd->flow_limit);
3413 if (fl) {
Tom Herbert3958afa1b2013-12-15 22:12:06 -08003414 new_flow = skb_get_hash(skb) & (fl->num_buckets - 1);
Willem de Bruijn99bbc702013-05-20 04:02:32 +00003415 old_flow = fl->history[fl->history_head];
3416 fl->history[fl->history_head] = new_flow;
3417
3418 fl->history_head++;
3419 fl->history_head &= FLOW_LIMIT_HISTORY - 1;
3420
3421 if (likely(fl->buckets[old_flow]))
3422 fl->buckets[old_flow]--;
3423
3424 if (++fl->buckets[new_flow] > (FLOW_LIMIT_HISTORY >> 1)) {
3425 fl->count++;
3426 rcu_read_unlock();
3427 return true;
3428 }
3429 }
3430 rcu_read_unlock();
3431#endif
3432 return false;
3433}
3434
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003435/*
Tom Herbert0a9627f2010-03-16 08:03:29 +00003436 * enqueue_to_backlog is called to queue an skb to a per CPU backlog
3437 * queue (may be a remote CPU queue).
3438 */
Tom Herbertfec5e652010-04-16 16:01:27 -07003439static int enqueue_to_backlog(struct sk_buff *skb, int cpu,
3440 unsigned int *qtail)
Tom Herbert0a9627f2010-03-16 08:03:29 +00003441{
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003442 struct softnet_data *sd;
Tom Herbert0a9627f2010-03-16 08:03:29 +00003443 unsigned long flags;
Willem de Bruijn99bbc702013-05-20 04:02:32 +00003444 unsigned int qlen;
Tom Herbert0a9627f2010-03-16 08:03:29 +00003445
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003446 sd = &per_cpu(softnet_data, cpu);
Tom Herbert0a9627f2010-03-16 08:03:29 +00003447
3448 local_irq_save(flags);
Tom Herbert0a9627f2010-03-16 08:03:29 +00003449
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003450 rps_lock(sd);
Willem de Bruijn99bbc702013-05-20 04:02:32 +00003451 qlen = skb_queue_len(&sd->input_pkt_queue);
3452 if (qlen <= netdev_max_backlog && !skb_flow_limit(skb, qlen)) {
Li RongQinge008f3f2014-12-08 09:42:55 +08003453 if (qlen) {
Tom Herbert0a9627f2010-03-16 08:03:29 +00003454enqueue:
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003455 __skb_queue_tail(&sd->input_pkt_queue, skb);
Tom Herbert76cc8b12010-05-20 18:37:59 +00003456 input_queue_tail_incr_save(sd, qtail);
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003457 rps_unlock(sd);
Changli Gao152102c2010-03-30 20:16:22 +00003458 local_irq_restore(flags);
Tom Herbert0a9627f2010-03-16 08:03:29 +00003459 return NET_RX_SUCCESS;
3460 }
3461
Eric Dumazetebda37c22010-05-06 23:51:21 +00003462 /* Schedule NAPI for backlog device
3463 * We can use non atomic operation since we own the queue lock
3464 */
3465 if (!__test_and_set_bit(NAPI_STATE_SCHED, &sd->backlog.state)) {
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003466 if (!rps_ipi_queued(sd))
Eric Dumazeteecfd7c2010-05-06 22:07:48 -07003467 ____napi_schedule(sd, &sd->backlog);
Tom Herbert0a9627f2010-03-16 08:03:29 +00003468 }
3469 goto enqueue;
3470 }
3471
Changli Gaodee42872010-05-02 05:42:16 +00003472 sd->dropped++;
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003473 rps_unlock(sd);
Tom Herbert0a9627f2010-03-16 08:03:29 +00003474
Tom Herbert0a9627f2010-03-16 08:03:29 +00003475 local_irq_restore(flags);
3476
Eric Dumazetcaf586e2010-09-30 21:06:55 +00003477 atomic_long_inc(&skb->dev->rx_dropped);
Tom Herbert0a9627f2010-03-16 08:03:29 +00003478 kfree_skb(skb);
3479 return NET_RX_DROP;
3480}
Linus Torvalds1da177e2005-04-16 15:20:36 -07003481
Ben Hutchingsae78dbf2014-01-10 22:17:24 +00003482static int netif_rx_internal(struct sk_buff *skb)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003483{
Eric Dumazetb0e28f12010-04-15 00:14:07 -07003484 int ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003485
Eric Dumazet588f0332011-11-15 04:12:55 +00003486 net_timestamp_check(netdev_tstamp_prequeue, skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003487
Koki Sanagicf66ba52010-08-23 18:45:02 +09003488 trace_netif_rx(skb);
Eric Dumazetdf334542010-03-24 19:13:54 +00003489#ifdef CONFIG_RPS
Ingo Molnarc5905af2012-02-24 08:31:31 +01003490 if (static_key_false(&rps_needed)) {
Tom Herbertfec5e652010-04-16 16:01:27 -07003491 struct rps_dev_flow voidflow, *rflow = &voidflow;
Eric Dumazetb0e28f12010-04-15 00:14:07 -07003492 int cpu;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003493
Changli Gaocece1942010-08-07 20:35:43 -07003494 preempt_disable();
Eric Dumazetb0e28f12010-04-15 00:14:07 -07003495 rcu_read_lock();
Tom Herbertfec5e652010-04-16 16:01:27 -07003496
3497 cpu = get_rps_cpu(skb->dev, skb, &rflow);
Eric Dumazetb0e28f12010-04-15 00:14:07 -07003498 if (cpu < 0)
3499 cpu = smp_processor_id();
Tom Herbertfec5e652010-04-16 16:01:27 -07003500
3501 ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail);
3502
Eric Dumazetb0e28f12010-04-15 00:14:07 -07003503 rcu_read_unlock();
Changli Gaocece1942010-08-07 20:35:43 -07003504 preempt_enable();
Eric Dumazetadc93002011-11-17 03:13:26 +00003505 } else
3506#endif
Tom Herbertfec5e652010-04-16 16:01:27 -07003507 {
3508 unsigned int qtail;
3509 ret = enqueue_to_backlog(skb, get_cpu(), &qtail);
3510 put_cpu();
3511 }
Eric Dumazetb0e28f12010-04-15 00:14:07 -07003512 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003513}
Ben Hutchingsae78dbf2014-01-10 22:17:24 +00003514
3515/**
3516 * netif_rx - post buffer to the network code
3517 * @skb: buffer to post
3518 *
3519 * This function receives a packet from a device driver and queues it for
3520 * the upper (protocol) levels to process. It always succeeds. The buffer
3521 * may be dropped during processing for congestion control or by the
3522 * protocol layers.
3523 *
3524 * return values:
3525 * NET_RX_SUCCESS (no congestion)
3526 * NET_RX_DROP (packet was dropped)
3527 *
3528 */
3529
3530int netif_rx(struct sk_buff *skb)
3531{
3532 trace_netif_rx_entry(skb);
3533
3534 return netif_rx_internal(skb);
3535}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07003536EXPORT_SYMBOL(netif_rx);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003537
3538int netif_rx_ni(struct sk_buff *skb)
3539{
3540 int err;
3541
Ben Hutchingsae78dbf2014-01-10 22:17:24 +00003542 trace_netif_rx_ni_entry(skb);
3543
Linus Torvalds1da177e2005-04-16 15:20:36 -07003544 preempt_disable();
Ben Hutchingsae78dbf2014-01-10 22:17:24 +00003545 err = netif_rx_internal(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003546 if (local_softirq_pending())
3547 do_softirq();
3548 preempt_enable();
3549
3550 return err;
3551}
Linus Torvalds1da177e2005-04-16 15:20:36 -07003552EXPORT_SYMBOL(netif_rx_ni);
3553
Linus Torvalds1da177e2005-04-16 15:20:36 -07003554static void net_tx_action(struct softirq_action *h)
3555{
Christoph Lameter903ceff2014-08-17 12:30:35 -05003556 struct softnet_data *sd = this_cpu_ptr(&softnet_data);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003557
3558 if (sd->completion_queue) {
3559 struct sk_buff *clist;
3560
3561 local_irq_disable();
3562 clist = sd->completion_queue;
3563 sd->completion_queue = NULL;
3564 local_irq_enable();
3565
3566 while (clist) {
3567 struct sk_buff *skb = clist;
3568 clist = clist->next;
3569
Ilpo Järvinen547b7922008-07-25 21:43:18 -07003570 WARN_ON(atomic_read(&skb->users));
Eric Dumazete6247022013-12-05 04:45:08 -08003571 if (likely(get_kfree_skb_cb(skb)->reason == SKB_REASON_CONSUMED))
3572 trace_consume_skb(skb);
3573 else
3574 trace_kfree_skb(skb, net_tx_action);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003575 __kfree_skb(skb);
3576 }
3577 }
3578
3579 if (sd->output_queue) {
David S. Miller37437bb2008-07-16 02:15:04 -07003580 struct Qdisc *head;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003581
3582 local_irq_disable();
3583 head = sd->output_queue;
3584 sd->output_queue = NULL;
Changli Gaoa9cbd582010-04-26 23:06:24 +00003585 sd->output_queue_tailp = &sd->output_queue;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003586 local_irq_enable();
3587
3588 while (head) {
David S. Miller37437bb2008-07-16 02:15:04 -07003589 struct Qdisc *q = head;
3590 spinlock_t *root_lock;
3591
Linus Torvalds1da177e2005-04-16 15:20:36 -07003592 head = head->next_sched;
3593
David S. Miller5fb66222008-08-02 20:02:43 -07003594 root_lock = qdisc_lock(q);
David S. Miller37437bb2008-07-16 02:15:04 -07003595 if (spin_trylock(root_lock)) {
Peter Zijlstra4e857c52014-03-17 18:06:10 +01003596 smp_mb__before_atomic();
Jarek Poplawskidef82a12008-08-17 21:54:43 -07003597 clear_bit(__QDISC_STATE_SCHED,
3598 &q->state);
David S. Miller37437bb2008-07-16 02:15:04 -07003599 qdisc_run(q);
3600 spin_unlock(root_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003601 } else {
David S. Miller195648b2008-08-19 04:00:36 -07003602 if (!test_bit(__QDISC_STATE_DEACTIVATED,
Jarek Poplawskie8a83e12008-09-07 18:41:21 -07003603 &q->state)) {
David S. Miller195648b2008-08-19 04:00:36 -07003604 __netif_reschedule(q);
Jarek Poplawskie8a83e12008-09-07 18:41:21 -07003605 } else {
Peter Zijlstra4e857c52014-03-17 18:06:10 +01003606 smp_mb__before_atomic();
Jarek Poplawskie8a83e12008-09-07 18:41:21 -07003607 clear_bit(__QDISC_STATE_SCHED,
3608 &q->state);
3609 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07003610 }
3611 }
3612 }
3613}
3614
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00003615#if (defined(CONFIG_BRIDGE) || defined(CONFIG_BRIDGE_MODULE)) && \
3616 (defined(CONFIG_ATM_LANE) || defined(CONFIG_ATM_LANE_MODULE))
Michał Mirosławda678292009-06-05 05:35:28 +00003617/* This hook is defined here for ATM LANE */
3618int (*br_fdb_test_addr_hook)(struct net_device *dev,
3619 unsigned char *addr) __read_mostly;
Stephen Hemminger4fb019a2009-09-11 11:50:08 -07003620EXPORT_SYMBOL_GPL(br_fdb_test_addr_hook);
Michał Mirosławda678292009-06-05 05:35:28 +00003621#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07003622
Herbert Xuf697c3e2007-10-14 00:38:47 -07003623static inline struct sk_buff *handle_ing(struct sk_buff *skb,
3624 struct packet_type **pt_prev,
3625 int *ret, struct net_device *orig_dev)
3626{
Daniel Borkmanne7582ba2015-05-19 22:33:25 +02003627#ifdef CONFIG_NET_CLS_ACT
Daniel Borkmannd2788d32015-05-09 22:51:32 +02003628 struct tcf_proto *cl = rcu_dereference_bh(skb->dev->ingress_cl_list);
3629 struct tcf_result cl_res;
Eric Dumazet24824a02010-10-02 06:11:55 +00003630
Daniel Borkmannc9e99fd2015-05-09 22:51:31 +02003631 /* If there's at least one ingress present somewhere (so
3632 * we get here via enabled static key), remaining devices
3633 * that are not configured with an ingress qdisc will bail
Daniel Borkmannd2788d32015-05-09 22:51:32 +02003634 * out here.
Daniel Borkmannc9e99fd2015-05-09 22:51:31 +02003635 */
Daniel Borkmannd2788d32015-05-09 22:51:32 +02003636 if (!cl)
Daniel Borkmann45771392015-04-10 23:07:54 +02003637 return skb;
Herbert Xuf697c3e2007-10-14 00:38:47 -07003638 if (*pt_prev) {
3639 *ret = deliver_skb(skb, *pt_prev, orig_dev);
3640 *pt_prev = NULL;
Herbert Xuf697c3e2007-10-14 00:38:47 -07003641 }
3642
Florian Westphal33654952015-05-14 00:36:28 +02003643 qdisc_skb_cb(skb)->pkt_len = skb->len;
Daniel Borkmannc9e99fd2015-05-09 22:51:31 +02003644 skb->tc_verd = SET_TC_AT(skb->tc_verd, AT_INGRESS);
Florian Westphal33654952015-05-14 00:36:28 +02003645 qdisc_bstats_update_cpu(cl->q, skb);
Daniel Borkmannc9e99fd2015-05-09 22:51:31 +02003646
Daniel Borkmannd2788d32015-05-09 22:51:32 +02003647 switch (tc_classify(skb, cl, &cl_res)) {
3648 case TC_ACT_OK:
3649 case TC_ACT_RECLASSIFY:
3650 skb->tc_index = TC_H_MIN(cl_res.classid);
3651 break;
3652 case TC_ACT_SHOT:
3653 qdisc_qstats_drop_cpu(cl->q);
3654 case TC_ACT_STOLEN:
3655 case TC_ACT_QUEUED:
3656 kfree_skb(skb);
3657 return NULL;
3658 default:
3659 break;
Herbert Xuf697c3e2007-10-14 00:38:47 -07003660 }
Daniel Borkmanne7582ba2015-05-19 22:33:25 +02003661#endif /* CONFIG_NET_CLS_ACT */
Herbert Xuf697c3e2007-10-14 00:38:47 -07003662 return skb;
3663}
Linus Torvalds1da177e2005-04-16 15:20:36 -07003664
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00003665/**
3666 * netdev_rx_handler_register - register receive handler
3667 * @dev: device to register a handler for
3668 * @rx_handler: receive handler to register
Jiri Pirko93e2c322010-06-10 03:34:59 +00003669 * @rx_handler_data: data pointer that is used by rx handler
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00003670 *
Masanari Iidae2278672014-02-18 22:54:36 +09003671 * Register a receive handler for a device. This handler will then be
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00003672 * called from __netif_receive_skb. A negative errno code is returned
3673 * on a failure.
3674 *
3675 * The caller must hold the rtnl_mutex.
Jiri Pirko8a4eb572011-03-12 03:14:39 +00003676 *
3677 * For a general description of rx_handler, see enum rx_handler_result.
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00003678 */
3679int netdev_rx_handler_register(struct net_device *dev,
Jiri Pirko93e2c322010-06-10 03:34:59 +00003680 rx_handler_func_t *rx_handler,
3681 void *rx_handler_data)
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00003682{
3683 ASSERT_RTNL();
3684
3685 if (dev->rx_handler)
3686 return -EBUSY;
3687
Eric Dumazet00cfec32013-03-29 03:01:22 +00003688 /* Note: rx_handler_data must be set before rx_handler */
Jiri Pirko93e2c322010-06-10 03:34:59 +00003689 rcu_assign_pointer(dev->rx_handler_data, rx_handler_data);
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00003690 rcu_assign_pointer(dev->rx_handler, rx_handler);
3691
3692 return 0;
3693}
3694EXPORT_SYMBOL_GPL(netdev_rx_handler_register);
3695
3696/**
3697 * netdev_rx_handler_unregister - unregister receive handler
3698 * @dev: device to unregister a handler from
3699 *
Kusanagi Kouichi166ec362013-03-18 02:59:52 +00003700 * Unregister a receive handler from a device.
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00003701 *
3702 * The caller must hold the rtnl_mutex.
3703 */
3704void netdev_rx_handler_unregister(struct net_device *dev)
3705{
3706
3707 ASSERT_RTNL();
Stephen Hemmingera9b3cd72011-08-01 16:19:00 +00003708 RCU_INIT_POINTER(dev->rx_handler, NULL);
Eric Dumazet00cfec32013-03-29 03:01:22 +00003709 /* a reader seeing a non NULL rx_handler in a rcu_read_lock()
3710 * section has a guarantee to see a non NULL rx_handler_data
3711 * as well.
3712 */
3713 synchronize_net();
Stephen Hemmingera9b3cd72011-08-01 16:19:00 +00003714 RCU_INIT_POINTER(dev->rx_handler_data, NULL);
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00003715}
3716EXPORT_SYMBOL_GPL(netdev_rx_handler_unregister);
3717
Mel Gormanb4b9e352012-07-31 16:44:26 -07003718/*
3719 * Limit the use of PFMEMALLOC reserves to those protocols that implement
3720 * the special handling of PFMEMALLOC skbs.
3721 */
3722static bool skb_pfmemalloc_protocol(struct sk_buff *skb)
3723{
3724 switch (skb->protocol) {
Joe Perches2b8837a2014-03-12 10:04:17 -07003725 case htons(ETH_P_ARP):
3726 case htons(ETH_P_IP):
3727 case htons(ETH_P_IPV6):
3728 case htons(ETH_P_8021Q):
3729 case htons(ETH_P_8021AD):
Mel Gormanb4b9e352012-07-31 16:44:26 -07003730 return true;
3731 default:
3732 return false;
3733 }
3734}
3735
Pablo Neirae687ad62015-05-13 18:19:38 +02003736static inline int nf_ingress(struct sk_buff *skb, struct packet_type **pt_prev,
3737 int *ret, struct net_device *orig_dev)
3738{
Daniel Borkmanne7582ba2015-05-19 22:33:25 +02003739#ifdef CONFIG_NETFILTER_INGRESS
Pablo Neirae687ad62015-05-13 18:19:38 +02003740 if (nf_hook_ingress_active(skb)) {
3741 if (*pt_prev) {
3742 *ret = deliver_skb(skb, *pt_prev, orig_dev);
3743 *pt_prev = NULL;
3744 }
3745
3746 return nf_hook_ingress(skb);
3747 }
Daniel Borkmanne7582ba2015-05-19 22:33:25 +02003748#endif /* CONFIG_NETFILTER_INGRESS */
Pablo Neirae687ad62015-05-13 18:19:38 +02003749 return 0;
3750}
Pablo Neirae687ad62015-05-13 18:19:38 +02003751
David S. Miller9754e292013-02-14 15:57:38 -05003752static int __netif_receive_skb_core(struct sk_buff *skb, bool pfmemalloc)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003753{
3754 struct packet_type *ptype, *pt_prev;
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00003755 rx_handler_func_t *rx_handler;
David S. Millerf2ccd8f2005-08-09 19:34:12 -07003756 struct net_device *orig_dev;
Jiri Pirko8a4eb572011-03-12 03:14:39 +00003757 bool deliver_exact = false;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003758 int ret = NET_RX_DROP;
Al Viro252e33462006-11-14 20:48:11 -08003759 __be16 type;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003760
Eric Dumazet588f0332011-11-15 04:12:55 +00003761 net_timestamp_check(!netdev_tstamp_prequeue, skb);
Eric Dumazet81bbb3d2009-09-30 16:42:42 -07003762
Koki Sanagicf66ba52010-08-23 18:45:02 +09003763 trace_netif_receive_skb(skb);
Patrick McHardy9b22ea52008-11-04 14:49:57 -08003764
Joe Eykholtcc9bd5c2008-07-02 18:22:00 -07003765 orig_dev = skb->dev;
Jiri Pirko1765a572011-02-12 06:48:36 +00003766
Arnaldo Carvalho de Meloc1d2bbe2007-04-10 20:45:18 -07003767 skb_reset_network_header(skb);
Eric Dumazetfda55ec2013-01-07 09:28:21 +00003768 if (!skb_transport_header_was_set(skb))
3769 skb_reset_transport_header(skb);
Jiri Pirko0b5c9db2011-06-10 06:56:58 +00003770 skb_reset_mac_len(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003771
3772 pt_prev = NULL;
3773
3774 rcu_read_lock();
3775
David S. Miller63d8ea72011-02-28 10:48:59 -08003776another_round:
David S. Millerb6858172012-07-23 16:27:54 -07003777 skb->skb_iif = skb->dev->ifindex;
David S. Miller63d8ea72011-02-28 10:48:59 -08003778
3779 __this_cpu_inc(softnet_data.processed);
3780
Patrick McHardy8ad227f2013-04-19 02:04:31 +00003781 if (skb->protocol == cpu_to_be16(ETH_P_8021Q) ||
3782 skb->protocol == cpu_to_be16(ETH_P_8021AD)) {
Vlad Yasevich0d5501c2014-08-08 14:42:13 -04003783 skb = skb_vlan_untag(skb);
Jiri Pirkobcc6d472011-04-07 19:48:33 +00003784 if (unlikely(!skb))
Mel Gormanb4b9e352012-07-31 16:44:26 -07003785 goto unlock;
Jiri Pirkobcc6d472011-04-07 19:48:33 +00003786 }
3787
Linus Torvalds1da177e2005-04-16 15:20:36 -07003788#ifdef CONFIG_NET_CLS_ACT
3789 if (skb->tc_verd & TC_NCLS) {
3790 skb->tc_verd = CLR_TC_NCLS(skb->tc_verd);
3791 goto ncls;
3792 }
3793#endif
3794
David S. Miller9754e292013-02-14 15:57:38 -05003795 if (pfmemalloc)
Mel Gormanb4b9e352012-07-31 16:44:26 -07003796 goto skip_taps;
3797
Linus Torvalds1da177e2005-04-16 15:20:36 -07003798 list_for_each_entry_rcu(ptype, &ptype_all, list) {
Salam Noureddine7866a622015-01-27 11:35:48 -08003799 if (pt_prev)
3800 ret = deliver_skb(skb, pt_prev, orig_dev);
3801 pt_prev = ptype;
3802 }
3803
3804 list_for_each_entry_rcu(ptype, &skb->dev->ptype_all, list) {
3805 if (pt_prev)
3806 ret = deliver_skb(skb, pt_prev, orig_dev);
3807 pt_prev = ptype;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003808 }
3809
Mel Gormanb4b9e352012-07-31 16:44:26 -07003810skip_taps:
Pablo Neira1cf519002015-05-13 18:19:37 +02003811#ifdef CONFIG_NET_INGRESS
Daniel Borkmann45771392015-04-10 23:07:54 +02003812 if (static_key_false(&ingress_needed)) {
3813 skb = handle_ing(skb, &pt_prev, &ret, orig_dev);
3814 if (!skb)
3815 goto unlock;
Pablo Neirae687ad62015-05-13 18:19:38 +02003816
3817 if (nf_ingress(skb, &pt_prev, &ret, orig_dev) < 0)
3818 goto unlock;
Daniel Borkmann45771392015-04-10 23:07:54 +02003819 }
Pablo Neira1cf519002015-05-13 18:19:37 +02003820#endif
3821#ifdef CONFIG_NET_CLS_ACT
Daniel Borkmann45771392015-04-10 23:07:54 +02003822 skb->tc_verd = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003823ncls:
3824#endif
David S. Miller9754e292013-02-14 15:57:38 -05003825 if (pfmemalloc && !skb_pfmemalloc_protocol(skb))
Mel Gormanb4b9e352012-07-31 16:44:26 -07003826 goto drop;
3827
Jiri Pirkodf8a39d2015-01-13 17:13:44 +01003828 if (skb_vlan_tag_present(skb)) {
John Fastabend24257172011-10-10 09:16:41 +00003829 if (pt_prev) {
3830 ret = deliver_skb(skb, pt_prev, orig_dev);
3831 pt_prev = NULL;
3832 }
Florian Zumbiehl48cc32d32012-10-07 15:51:58 +00003833 if (vlan_do_receive(&skb))
John Fastabend24257172011-10-10 09:16:41 +00003834 goto another_round;
3835 else if (unlikely(!skb))
Mel Gormanb4b9e352012-07-31 16:44:26 -07003836 goto unlock;
John Fastabend24257172011-10-10 09:16:41 +00003837 }
3838
Florian Zumbiehl48cc32d32012-10-07 15:51:58 +00003839 rx_handler = rcu_dereference(skb->dev->rx_handler);
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00003840 if (rx_handler) {
3841 if (pt_prev) {
3842 ret = deliver_skb(skb, pt_prev, orig_dev);
3843 pt_prev = NULL;
3844 }
Jiri Pirko8a4eb572011-03-12 03:14:39 +00003845 switch (rx_handler(&skb)) {
3846 case RX_HANDLER_CONSUMED:
Cristian Bercaru3bc1b1a2013-03-08 07:03:38 +00003847 ret = NET_RX_SUCCESS;
Mel Gormanb4b9e352012-07-31 16:44:26 -07003848 goto unlock;
Jiri Pirko8a4eb572011-03-12 03:14:39 +00003849 case RX_HANDLER_ANOTHER:
David S. Miller63d8ea72011-02-28 10:48:59 -08003850 goto another_round;
Jiri Pirko8a4eb572011-03-12 03:14:39 +00003851 case RX_HANDLER_EXACT:
3852 deliver_exact = true;
3853 case RX_HANDLER_PASS:
3854 break;
3855 default:
3856 BUG();
3857 }
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00003858 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07003859
Jiri Pirkodf8a39d2015-01-13 17:13:44 +01003860 if (unlikely(skb_vlan_tag_present(skb))) {
3861 if (skb_vlan_tag_get_id(skb))
Eric Dumazetd4b812d2013-07-18 07:19:26 -07003862 skb->pkt_type = PACKET_OTHERHOST;
3863 /* Note: we might in the future use prio bits
3864 * and set skb->priority like in vlan_do_receive()
3865 * For the time being, just ignore Priority Code Point
3866 */
3867 skb->vlan_tci = 0;
3868 }
Florian Zumbiehl48cc32d32012-10-07 15:51:58 +00003869
Linus Torvalds1da177e2005-04-16 15:20:36 -07003870 type = skb->protocol;
Salam Noureddine7866a622015-01-27 11:35:48 -08003871
3872 /* deliver only exact match when indicated */
3873 if (likely(!deliver_exact)) {
3874 deliver_ptype_list_skb(skb, &pt_prev, orig_dev, type,
3875 &ptype_base[ntohs(type) &
3876 PTYPE_HASH_MASK]);
3877 }
3878
3879 deliver_ptype_list_skb(skb, &pt_prev, orig_dev, type,
3880 &orig_dev->ptype_specific);
3881
3882 if (unlikely(skb->dev != orig_dev)) {
3883 deliver_ptype_list_skb(skb, &pt_prev, orig_dev, type,
3884 &skb->dev->ptype_specific);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003885 }
3886
3887 if (pt_prev) {
Michael S. Tsirkin1080e512012-07-20 09:23:17 +00003888 if (unlikely(skb_orphan_frags(skb, GFP_ATOMIC)))
Michael S. Tsirkin0e698bf2012-09-15 22:44:16 +00003889 goto drop;
Michael S. Tsirkin1080e512012-07-20 09:23:17 +00003890 else
3891 ret = pt_prev->func(skb, skb->dev, pt_prev, orig_dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003892 } else {
Mel Gormanb4b9e352012-07-31 16:44:26 -07003893drop:
Eric Dumazetcaf586e2010-09-30 21:06:55 +00003894 atomic_long_inc(&skb->dev->rx_dropped);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003895 kfree_skb(skb);
3896 /* Jamal, now you will not able to escape explaining
3897 * me how you were going to use this. :-)
3898 */
3899 ret = NET_RX_DROP;
3900 }
3901
Mel Gormanb4b9e352012-07-31 16:44:26 -07003902unlock:
Linus Torvalds1da177e2005-04-16 15:20:36 -07003903 rcu_read_unlock();
David S. Miller9754e292013-02-14 15:57:38 -05003904 return ret;
3905}
3906
3907static int __netif_receive_skb(struct sk_buff *skb)
3908{
3909 int ret;
3910
3911 if (sk_memalloc_socks() && skb_pfmemalloc(skb)) {
3912 unsigned long pflags = current->flags;
3913
3914 /*
3915 * PFMEMALLOC skbs are special, they should
3916 * - be delivered to SOCK_MEMALLOC sockets only
3917 * - stay away from userspace
3918 * - have bounded memory usage
3919 *
3920 * Use PF_MEMALLOC as this saves us from propagating the allocation
3921 * context down to all allocation sites.
3922 */
3923 current->flags |= PF_MEMALLOC;
3924 ret = __netif_receive_skb_core(skb, true);
3925 tsk_restore_flags(current, pflags, PF_MEMALLOC);
3926 } else
3927 ret = __netif_receive_skb_core(skb, false);
3928
Linus Torvalds1da177e2005-04-16 15:20:36 -07003929 return ret;
3930}
Tom Herbert0a9627f2010-03-16 08:03:29 +00003931
Ben Hutchingsae78dbf2014-01-10 22:17:24 +00003932static int netif_receive_skb_internal(struct sk_buff *skb)
Tom Herbert0a9627f2010-03-16 08:03:29 +00003933{
Eric Dumazet588f0332011-11-15 04:12:55 +00003934 net_timestamp_check(netdev_tstamp_prequeue, skb);
Eric Dumazet3b098e22010-05-15 23:57:10 -07003935
Richard Cochranc1f19b52010-07-17 08:49:36 +00003936 if (skb_defer_rx_timestamp(skb))
3937 return NET_RX_SUCCESS;
3938
Eric Dumazetdf334542010-03-24 19:13:54 +00003939#ifdef CONFIG_RPS
Ingo Molnarc5905af2012-02-24 08:31:31 +01003940 if (static_key_false(&rps_needed)) {
Eric Dumazet3b098e22010-05-15 23:57:10 -07003941 struct rps_dev_flow voidflow, *rflow = &voidflow;
3942 int cpu, ret;
Tom Herbert0a9627f2010-03-16 08:03:29 +00003943
Eric Dumazet3b098e22010-05-15 23:57:10 -07003944 rcu_read_lock();
Tom Herbert0a9627f2010-03-16 08:03:29 +00003945
Eric Dumazet3b098e22010-05-15 23:57:10 -07003946 cpu = get_rps_cpu(skb->dev, skb, &rflow);
Tom Herbertfec5e652010-04-16 16:01:27 -07003947
Eric Dumazet3b098e22010-05-15 23:57:10 -07003948 if (cpu >= 0) {
3949 ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail);
3950 rcu_read_unlock();
Eric Dumazetadc93002011-11-17 03:13:26 +00003951 return ret;
Eric Dumazet3b098e22010-05-15 23:57:10 -07003952 }
Eric Dumazetadc93002011-11-17 03:13:26 +00003953 rcu_read_unlock();
Tom Herbertfec5e652010-04-16 16:01:27 -07003954 }
Tom Herbert1e94d722010-03-18 17:45:44 -07003955#endif
Eric Dumazetadc93002011-11-17 03:13:26 +00003956 return __netif_receive_skb(skb);
Tom Herbert0a9627f2010-03-16 08:03:29 +00003957}
Ben Hutchingsae78dbf2014-01-10 22:17:24 +00003958
3959/**
3960 * netif_receive_skb - process receive buffer from network
3961 * @skb: buffer to process
3962 *
3963 * netif_receive_skb() is the main receive data processing function.
3964 * It always succeeds. The buffer may be dropped during processing
3965 * for congestion control or by the protocol layers.
3966 *
3967 * This function may only be called from softirq context and interrupts
3968 * should be enabled.
3969 *
3970 * Return values (usually ignored):
3971 * NET_RX_SUCCESS: no congestion
3972 * NET_RX_DROP: packet was dropped
3973 */
David Miller7026b1d2015-04-05 22:19:04 -04003974int netif_receive_skb_sk(struct sock *sk, struct sk_buff *skb)
Ben Hutchingsae78dbf2014-01-10 22:17:24 +00003975{
3976 trace_netif_receive_skb_entry(skb);
3977
3978 return netif_receive_skb_internal(skb);
3979}
David Miller7026b1d2015-04-05 22:19:04 -04003980EXPORT_SYMBOL(netif_receive_skb_sk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003981
Eric Dumazet88751272010-04-19 05:07:33 +00003982/* Network device is going away, flush any packets still pending
3983 * Called with irqs disabled.
3984 */
Changli Gao152102c2010-03-30 20:16:22 +00003985static void flush_backlog(void *arg)
Stephen Hemminger6e583ce2008-08-03 21:29:57 -07003986{
Changli Gao152102c2010-03-30 20:16:22 +00003987 struct net_device *dev = arg;
Christoph Lameter903ceff2014-08-17 12:30:35 -05003988 struct softnet_data *sd = this_cpu_ptr(&softnet_data);
Stephen Hemminger6e583ce2008-08-03 21:29:57 -07003989 struct sk_buff *skb, *tmp;
3990
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003991 rps_lock(sd);
Changli Gao6e7676c2010-04-27 15:07:33 -07003992 skb_queue_walk_safe(&sd->input_pkt_queue, skb, tmp) {
Stephen Hemminger6e583ce2008-08-03 21:29:57 -07003993 if (skb->dev == dev) {
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003994 __skb_unlink(skb, &sd->input_pkt_queue);
Stephen Hemminger6e583ce2008-08-03 21:29:57 -07003995 kfree_skb(skb);
Tom Herbert76cc8b12010-05-20 18:37:59 +00003996 input_queue_head_incr(sd);
Stephen Hemminger6e583ce2008-08-03 21:29:57 -07003997 }
Changli Gao6e7676c2010-04-27 15:07:33 -07003998 }
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003999 rps_unlock(sd);
Changli Gao6e7676c2010-04-27 15:07:33 -07004000
4001 skb_queue_walk_safe(&sd->process_queue, skb, tmp) {
4002 if (skb->dev == dev) {
4003 __skb_unlink(skb, &sd->process_queue);
4004 kfree_skb(skb);
Tom Herbert76cc8b12010-05-20 18:37:59 +00004005 input_queue_head_incr(sd);
Changli Gao6e7676c2010-04-27 15:07:33 -07004006 }
4007 }
Stephen Hemminger6e583ce2008-08-03 21:29:57 -07004008}
4009
Herbert Xud565b0a2008-12-15 23:38:52 -08004010static int napi_gro_complete(struct sk_buff *skb)
4011{
Vlad Yasevich22061d82012-11-15 08:49:11 +00004012 struct packet_offload *ptype;
Herbert Xud565b0a2008-12-15 23:38:52 -08004013 __be16 type = skb->protocol;
Vlad Yasevich22061d82012-11-15 08:49:11 +00004014 struct list_head *head = &offload_base;
Herbert Xud565b0a2008-12-15 23:38:52 -08004015 int err = -ENOENT;
4016
Eric Dumazetc3c7c252012-12-06 13:54:59 +00004017 BUILD_BUG_ON(sizeof(struct napi_gro_cb) > sizeof(skb->cb));
4018
Herbert Xufc59f9a2009-04-14 15:11:06 -07004019 if (NAPI_GRO_CB(skb)->count == 1) {
4020 skb_shinfo(skb)->gso_size = 0;
Herbert Xud565b0a2008-12-15 23:38:52 -08004021 goto out;
Herbert Xufc59f9a2009-04-14 15:11:06 -07004022 }
Herbert Xud565b0a2008-12-15 23:38:52 -08004023
4024 rcu_read_lock();
4025 list_for_each_entry_rcu(ptype, head, list) {
Vlad Yasevichf191a1d2012-11-15 08:49:23 +00004026 if (ptype->type != type || !ptype->callbacks.gro_complete)
Herbert Xud565b0a2008-12-15 23:38:52 -08004027 continue;
4028
Jerry Chu299603e82013-12-11 20:53:45 -08004029 err = ptype->callbacks.gro_complete(skb, 0);
Herbert Xud565b0a2008-12-15 23:38:52 -08004030 break;
4031 }
4032 rcu_read_unlock();
4033
4034 if (err) {
4035 WARN_ON(&ptype->list == head);
4036 kfree_skb(skb);
4037 return NET_RX_SUCCESS;
4038 }
4039
4040out:
Ben Hutchingsae78dbf2014-01-10 22:17:24 +00004041 return netif_receive_skb_internal(skb);
Herbert Xud565b0a2008-12-15 23:38:52 -08004042}
4043
Eric Dumazet2e71a6f2012-10-06 08:08:49 +00004044/* napi->gro_list contains packets ordered by age.
4045 * youngest packets at the head of it.
4046 * Complete skbs in reverse order to reduce latencies.
4047 */
4048void napi_gro_flush(struct napi_struct *napi, bool flush_old)
Herbert Xud565b0a2008-12-15 23:38:52 -08004049{
Eric Dumazet2e71a6f2012-10-06 08:08:49 +00004050 struct sk_buff *skb, *prev = NULL;
Herbert Xud565b0a2008-12-15 23:38:52 -08004051
Eric Dumazet2e71a6f2012-10-06 08:08:49 +00004052 /* scan list and build reverse chain */
4053 for (skb = napi->gro_list; skb != NULL; skb = skb->next) {
4054 skb->prev = prev;
4055 prev = skb;
Herbert Xud565b0a2008-12-15 23:38:52 -08004056 }
4057
Eric Dumazet2e71a6f2012-10-06 08:08:49 +00004058 for (skb = prev; skb; skb = prev) {
4059 skb->next = NULL;
4060
4061 if (flush_old && NAPI_GRO_CB(skb)->age == jiffies)
4062 return;
4063
4064 prev = skb->prev;
4065 napi_gro_complete(skb);
4066 napi->gro_count--;
4067 }
4068
Herbert Xud565b0a2008-12-15 23:38:52 -08004069 napi->gro_list = NULL;
4070}
Eric Dumazet86cac582010-08-31 18:25:32 +00004071EXPORT_SYMBOL(napi_gro_flush);
Herbert Xud565b0a2008-12-15 23:38:52 -08004072
Eric Dumazet89c5fa32012-12-10 13:28:16 +00004073static void gro_list_prepare(struct napi_struct *napi, struct sk_buff *skb)
4074{
4075 struct sk_buff *p;
4076 unsigned int maclen = skb->dev->hard_header_len;
Tom Herbert0b4cec82014-01-15 08:58:06 -08004077 u32 hash = skb_get_hash_raw(skb);
Eric Dumazet89c5fa32012-12-10 13:28:16 +00004078
4079 for (p = napi->gro_list; p; p = p->next) {
4080 unsigned long diffs;
4081
Tom Herbert0b4cec82014-01-15 08:58:06 -08004082 NAPI_GRO_CB(p)->flush = 0;
4083
4084 if (hash != skb_get_hash_raw(p)) {
4085 NAPI_GRO_CB(p)->same_flow = 0;
4086 continue;
4087 }
4088
Eric Dumazet89c5fa32012-12-10 13:28:16 +00004089 diffs = (unsigned long)p->dev ^ (unsigned long)skb->dev;
4090 diffs |= p->vlan_tci ^ skb->vlan_tci;
4091 if (maclen == ETH_HLEN)
4092 diffs |= compare_ether_header(skb_mac_header(p),
Eric Dumazeta50e2332014-03-29 21:28:21 -07004093 skb_mac_header(skb));
Eric Dumazet89c5fa32012-12-10 13:28:16 +00004094 else if (!diffs)
4095 diffs = memcmp(skb_mac_header(p),
Eric Dumazeta50e2332014-03-29 21:28:21 -07004096 skb_mac_header(skb),
Eric Dumazet89c5fa32012-12-10 13:28:16 +00004097 maclen);
4098 NAPI_GRO_CB(p)->same_flow = !diffs;
Eric Dumazet89c5fa32012-12-10 13:28:16 +00004099 }
4100}
4101
Jerry Chu299603e82013-12-11 20:53:45 -08004102static void skb_gro_reset_offset(struct sk_buff *skb)
4103{
4104 const struct skb_shared_info *pinfo = skb_shinfo(skb);
4105 const skb_frag_t *frag0 = &pinfo->frags[0];
4106
4107 NAPI_GRO_CB(skb)->data_offset = 0;
4108 NAPI_GRO_CB(skb)->frag0 = NULL;
4109 NAPI_GRO_CB(skb)->frag0_len = 0;
4110
4111 if (skb_mac_header(skb) == skb_tail_pointer(skb) &&
4112 pinfo->nr_frags &&
4113 !PageHighMem(skb_frag_page(frag0))) {
4114 NAPI_GRO_CB(skb)->frag0 = skb_frag_address(frag0);
4115 NAPI_GRO_CB(skb)->frag0_len = skb_frag_size(frag0);
Herbert Xud565b0a2008-12-15 23:38:52 -08004116 }
4117}
4118
Eric Dumazeta50e2332014-03-29 21:28:21 -07004119static void gro_pull_from_frag0(struct sk_buff *skb, int grow)
4120{
4121 struct skb_shared_info *pinfo = skb_shinfo(skb);
4122
4123 BUG_ON(skb->end - skb->tail < grow);
4124
4125 memcpy(skb_tail_pointer(skb), NAPI_GRO_CB(skb)->frag0, grow);
4126
4127 skb->data_len -= grow;
4128 skb->tail += grow;
4129
4130 pinfo->frags[0].page_offset += grow;
4131 skb_frag_size_sub(&pinfo->frags[0], grow);
4132
4133 if (unlikely(!skb_frag_size(&pinfo->frags[0]))) {
4134 skb_frag_unref(skb, 0);
4135 memmove(pinfo->frags, pinfo->frags + 1,
4136 --pinfo->nr_frags * sizeof(pinfo->frags[0]));
4137 }
4138}
4139
Rami Rosenbb728822012-11-28 21:55:25 +00004140static enum gro_result dev_gro_receive(struct napi_struct *napi, struct sk_buff *skb)
Herbert Xud565b0a2008-12-15 23:38:52 -08004141{
4142 struct sk_buff **pp = NULL;
Vlad Yasevich22061d82012-11-15 08:49:11 +00004143 struct packet_offload *ptype;
Herbert Xud565b0a2008-12-15 23:38:52 -08004144 __be16 type = skb->protocol;
Vlad Yasevich22061d82012-11-15 08:49:11 +00004145 struct list_head *head = &offload_base;
Herbert Xu0da2afd52008-12-26 14:57:42 -08004146 int same_flow;
Ben Hutchings5b252f02009-10-29 07:17:09 +00004147 enum gro_result ret;
Eric Dumazeta50e2332014-03-29 21:28:21 -07004148 int grow;
Herbert Xud565b0a2008-12-15 23:38:52 -08004149
Eric W. Biederman9c62a682014-03-14 20:51:52 -07004150 if (!(skb->dev->features & NETIF_F_GRO))
Herbert Xud565b0a2008-12-15 23:38:52 -08004151 goto normal;
4152
Tom Herbert5a212322014-08-31 15:12:41 -07004153 if (skb_is_gso(skb) || skb_has_frag_list(skb) || skb->csum_bad)
Herbert Xuf17f5c92009-01-14 14:36:12 -08004154 goto normal;
4155
Eric Dumazet89c5fa32012-12-10 13:28:16 +00004156 gro_list_prepare(napi, skb);
4157
Herbert Xud565b0a2008-12-15 23:38:52 -08004158 rcu_read_lock();
4159 list_for_each_entry_rcu(ptype, head, list) {
Vlad Yasevichf191a1d2012-11-15 08:49:23 +00004160 if (ptype->type != type || !ptype->callbacks.gro_receive)
Herbert Xud565b0a2008-12-15 23:38:52 -08004161 continue;
4162
Herbert Xu86911732009-01-29 14:19:50 +00004163 skb_set_network_header(skb, skb_gro_offset(skb));
Eric Dumazetefd94502013-02-14 17:31:48 +00004164 skb_reset_mac_len(skb);
Herbert Xud565b0a2008-12-15 23:38:52 -08004165 NAPI_GRO_CB(skb)->same_flow = 0;
4166 NAPI_GRO_CB(skb)->flush = 0;
Herbert Xu5d38a072009-01-04 16:13:40 -08004167 NAPI_GRO_CB(skb)->free = 0;
Or Gerlitzb582ef02014-01-20 13:59:19 +02004168 NAPI_GRO_CB(skb)->udp_mark = 0;
Tom Herbert15e23962015-02-10 16:30:31 -08004169 NAPI_GRO_CB(skb)->gro_remcsum_start = 0;
Herbert Xud565b0a2008-12-15 23:38:52 -08004170
Tom Herbert662880f2014-08-27 21:26:56 -07004171 /* Setup for GRO checksum validation */
4172 switch (skb->ip_summed) {
4173 case CHECKSUM_COMPLETE:
4174 NAPI_GRO_CB(skb)->csum = skb->csum;
4175 NAPI_GRO_CB(skb)->csum_valid = 1;
4176 NAPI_GRO_CB(skb)->csum_cnt = 0;
4177 break;
4178 case CHECKSUM_UNNECESSARY:
4179 NAPI_GRO_CB(skb)->csum_cnt = skb->csum_level + 1;
4180 NAPI_GRO_CB(skb)->csum_valid = 0;
4181 break;
4182 default:
4183 NAPI_GRO_CB(skb)->csum_cnt = 0;
4184 NAPI_GRO_CB(skb)->csum_valid = 0;
4185 }
Herbert Xud565b0a2008-12-15 23:38:52 -08004186
Vlad Yasevichf191a1d2012-11-15 08:49:23 +00004187 pp = ptype->callbacks.gro_receive(&napi->gro_list, skb);
Herbert Xud565b0a2008-12-15 23:38:52 -08004188 break;
4189 }
4190 rcu_read_unlock();
4191
4192 if (&ptype->list == head)
4193 goto normal;
4194
Herbert Xu0da2afd52008-12-26 14:57:42 -08004195 same_flow = NAPI_GRO_CB(skb)->same_flow;
Herbert Xu5d0d9be2009-01-29 14:19:48 +00004196 ret = NAPI_GRO_CB(skb)->free ? GRO_MERGED_FREE : GRO_MERGED;
Herbert Xu0da2afd52008-12-26 14:57:42 -08004197
Herbert Xud565b0a2008-12-15 23:38:52 -08004198 if (pp) {
4199 struct sk_buff *nskb = *pp;
4200
4201 *pp = nskb->next;
4202 nskb->next = NULL;
4203 napi_gro_complete(nskb);
Herbert Xu4ae55442009-02-08 18:00:36 +00004204 napi->gro_count--;
Herbert Xud565b0a2008-12-15 23:38:52 -08004205 }
4206
Herbert Xu0da2afd52008-12-26 14:57:42 -08004207 if (same_flow)
Herbert Xud565b0a2008-12-15 23:38:52 -08004208 goto ok;
4209
Eric Dumazet600adc12014-01-09 14:12:19 -08004210 if (NAPI_GRO_CB(skb)->flush)
Herbert Xud565b0a2008-12-15 23:38:52 -08004211 goto normal;
Herbert Xud565b0a2008-12-15 23:38:52 -08004212
Eric Dumazet600adc12014-01-09 14:12:19 -08004213 if (unlikely(napi->gro_count >= MAX_GRO_SKBS)) {
4214 struct sk_buff *nskb = napi->gro_list;
4215
4216 /* locate the end of the list to select the 'oldest' flow */
4217 while (nskb->next) {
4218 pp = &nskb->next;
4219 nskb = *pp;
4220 }
4221 *pp = NULL;
4222 nskb->next = NULL;
4223 napi_gro_complete(nskb);
4224 } else {
4225 napi->gro_count++;
4226 }
Herbert Xud565b0a2008-12-15 23:38:52 -08004227 NAPI_GRO_CB(skb)->count = 1;
Eric Dumazet2e71a6f2012-10-06 08:08:49 +00004228 NAPI_GRO_CB(skb)->age = jiffies;
Eric Dumazet29e98242014-05-16 11:34:37 -07004229 NAPI_GRO_CB(skb)->last = skb;
Herbert Xu86911732009-01-29 14:19:50 +00004230 skb_shinfo(skb)->gso_size = skb_gro_len(skb);
Herbert Xud565b0a2008-12-15 23:38:52 -08004231 skb->next = napi->gro_list;
4232 napi->gro_list = skb;
Herbert Xu5d0d9be2009-01-29 14:19:48 +00004233 ret = GRO_HELD;
Herbert Xud565b0a2008-12-15 23:38:52 -08004234
Herbert Xuad0f9902009-02-01 01:24:55 -08004235pull:
Eric Dumazeta50e2332014-03-29 21:28:21 -07004236 grow = skb_gro_offset(skb) - skb_headlen(skb);
4237 if (grow > 0)
4238 gro_pull_from_frag0(skb, grow);
Herbert Xud565b0a2008-12-15 23:38:52 -08004239ok:
Herbert Xu5d0d9be2009-01-29 14:19:48 +00004240 return ret;
Herbert Xud565b0a2008-12-15 23:38:52 -08004241
4242normal:
Herbert Xuad0f9902009-02-01 01:24:55 -08004243 ret = GRO_NORMAL;
4244 goto pull;
Herbert Xu5d38a072009-01-04 16:13:40 -08004245}
Herbert Xu96e93ea2009-01-06 10:49:34 -08004246
Jerry Chubf5a7552014-01-07 10:23:19 -08004247struct packet_offload *gro_find_receive_by_type(__be16 type)
4248{
4249 struct list_head *offload_head = &offload_base;
4250 struct packet_offload *ptype;
4251
4252 list_for_each_entry_rcu(ptype, offload_head, list) {
4253 if (ptype->type != type || !ptype->callbacks.gro_receive)
4254 continue;
4255 return ptype;
4256 }
4257 return NULL;
4258}
Or Gerlitze27a2f82014-01-20 13:59:20 +02004259EXPORT_SYMBOL(gro_find_receive_by_type);
Jerry Chubf5a7552014-01-07 10:23:19 -08004260
4261struct packet_offload *gro_find_complete_by_type(__be16 type)
4262{
4263 struct list_head *offload_head = &offload_base;
4264 struct packet_offload *ptype;
4265
4266 list_for_each_entry_rcu(ptype, offload_head, list) {
4267 if (ptype->type != type || !ptype->callbacks.gro_complete)
4268 continue;
4269 return ptype;
4270 }
4271 return NULL;
4272}
Or Gerlitze27a2f82014-01-20 13:59:20 +02004273EXPORT_SYMBOL(gro_find_complete_by_type);
Herbert Xu96e93ea2009-01-06 10:49:34 -08004274
Rami Rosenbb728822012-11-28 21:55:25 +00004275static gro_result_t napi_skb_finish(gro_result_t ret, struct sk_buff *skb)
Herbert Xu5d38a072009-01-04 16:13:40 -08004276{
Herbert Xu5d0d9be2009-01-29 14:19:48 +00004277 switch (ret) {
4278 case GRO_NORMAL:
Ben Hutchingsae78dbf2014-01-10 22:17:24 +00004279 if (netif_receive_skb_internal(skb))
Ben Hutchingsc7c4b3b2009-10-29 21:36:53 -07004280 ret = GRO_DROP;
4281 break;
Herbert Xu5d38a072009-01-04 16:13:40 -08004282
Herbert Xu5d0d9be2009-01-29 14:19:48 +00004283 case GRO_DROP:
Herbert Xu5d38a072009-01-04 16:13:40 -08004284 kfree_skb(skb);
4285 break;
Ben Hutchings5b252f02009-10-29 07:17:09 +00004286
Eric Dumazetdaa86542012-04-19 07:07:40 +00004287 case GRO_MERGED_FREE:
Eric Dumazetd7e88832012-04-30 08:10:34 +00004288 if (NAPI_GRO_CB(skb)->free == NAPI_GRO_FREE_STOLEN_HEAD)
4289 kmem_cache_free(skbuff_head_cache, skb);
4290 else
4291 __kfree_skb(skb);
Eric Dumazetdaa86542012-04-19 07:07:40 +00004292 break;
4293
Ben Hutchings5b252f02009-10-29 07:17:09 +00004294 case GRO_HELD:
4295 case GRO_MERGED:
4296 break;
Herbert Xu5d38a072009-01-04 16:13:40 -08004297 }
4298
Ben Hutchingsc7c4b3b2009-10-29 21:36:53 -07004299 return ret;
Herbert Xu5d0d9be2009-01-29 14:19:48 +00004300}
Herbert Xu5d0d9be2009-01-29 14:19:48 +00004301
Ben Hutchingsc7c4b3b2009-10-29 21:36:53 -07004302gro_result_t napi_gro_receive(struct napi_struct *napi, struct sk_buff *skb)
Herbert Xu5d0d9be2009-01-29 14:19:48 +00004303{
Ben Hutchingsae78dbf2014-01-10 22:17:24 +00004304 trace_napi_gro_receive_entry(skb);
Herbert Xu86911732009-01-29 14:19:50 +00004305
Eric Dumazeta50e2332014-03-29 21:28:21 -07004306 skb_gro_reset_offset(skb);
4307
Eric Dumazet89c5fa32012-12-10 13:28:16 +00004308 return napi_skb_finish(dev_gro_receive(napi, skb), skb);
Herbert Xud565b0a2008-12-15 23:38:52 -08004309}
4310EXPORT_SYMBOL(napi_gro_receive);
4311
stephen hemmingerd0c2b0d2010-10-19 07:12:10 +00004312static void napi_reuse_skb(struct napi_struct *napi, struct sk_buff *skb)
Herbert Xu96e93ea2009-01-06 10:49:34 -08004313{
Eric Dumazet93a35f52014-10-23 06:30:30 -07004314 if (unlikely(skb->pfmemalloc)) {
4315 consume_skb(skb);
4316 return;
4317 }
Herbert Xu96e93ea2009-01-06 10:49:34 -08004318 __skb_pull(skb, skb_headlen(skb));
Eric Dumazet2a2a4592012-03-21 06:58:03 +00004319 /* restore the reserve we had after netdev_alloc_skb_ip_align() */
4320 skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN - skb_headroom(skb));
Jesse Gross3701e512010-10-20 13:56:06 +00004321 skb->vlan_tci = 0;
Herbert Xu66c46d72011-01-29 20:44:54 -08004322 skb->dev = napi->dev;
Andy Gospodarek6d152e22011-02-02 14:53:25 -08004323 skb->skb_iif = 0;
Jerry Chuc3caf112014-07-14 15:54:46 -07004324 skb->encapsulation = 0;
4325 skb_shinfo(skb)->gso_type = 0;
Eric Dumazete33d0ba2014-04-03 09:28:10 -07004326 skb->truesize = SKB_TRUESIZE(skb_end_offset(skb));
Herbert Xu96e93ea2009-01-06 10:49:34 -08004327
4328 napi->skb = skb;
4329}
Herbert Xu96e93ea2009-01-06 10:49:34 -08004330
Herbert Xu76620aa2009-04-16 02:02:07 -07004331struct sk_buff *napi_get_frags(struct napi_struct *napi)
Herbert Xu5d38a072009-01-04 16:13:40 -08004332{
Herbert Xu5d38a072009-01-04 16:13:40 -08004333 struct sk_buff *skb = napi->skb;
Herbert Xu5d38a072009-01-04 16:13:40 -08004334
4335 if (!skb) {
Alexander Duyckfd11a832014-12-09 19:40:49 -08004336 skb = napi_alloc_skb(napi, GRO_MAX_HEAD);
Eric Dumazet84b9cd62013-12-05 21:44:27 -08004337 napi->skb = skb;
Herbert Xu5d38a072009-01-04 16:13:40 -08004338 }
Herbert Xu96e93ea2009-01-06 10:49:34 -08004339 return skb;
4340}
Herbert Xu76620aa2009-04-16 02:02:07 -07004341EXPORT_SYMBOL(napi_get_frags);
Herbert Xu96e93ea2009-01-06 10:49:34 -08004342
Eric Dumazeta50e2332014-03-29 21:28:21 -07004343static gro_result_t napi_frags_finish(struct napi_struct *napi,
4344 struct sk_buff *skb,
4345 gro_result_t ret)
Herbert Xu5d0d9be2009-01-29 14:19:48 +00004346{
Herbert Xu5d0d9be2009-01-29 14:19:48 +00004347 switch (ret) {
4348 case GRO_NORMAL:
Eric Dumazeta50e2332014-03-29 21:28:21 -07004349 case GRO_HELD:
4350 __skb_push(skb, ETH_HLEN);
4351 skb->protocol = eth_type_trans(skb, skb->dev);
4352 if (ret == GRO_NORMAL && netif_receive_skb_internal(skb))
Ben Hutchingsc7c4b3b2009-10-29 21:36:53 -07004353 ret = GRO_DROP;
Herbert Xu86911732009-01-29 14:19:50 +00004354 break;
Herbert Xu5d0d9be2009-01-29 14:19:48 +00004355
4356 case GRO_DROP:
Herbert Xu5d0d9be2009-01-29 14:19:48 +00004357 case GRO_MERGED_FREE:
4358 napi_reuse_skb(napi, skb);
4359 break;
Ben Hutchings5b252f02009-10-29 07:17:09 +00004360
4361 case GRO_MERGED:
4362 break;
Herbert Xu5d0d9be2009-01-29 14:19:48 +00004363 }
4364
Ben Hutchingsc7c4b3b2009-10-29 21:36:53 -07004365 return ret;
Herbert Xu5d0d9be2009-01-29 14:19:48 +00004366}
Herbert Xu5d0d9be2009-01-29 14:19:48 +00004367
Eric Dumazeta50e2332014-03-29 21:28:21 -07004368/* Upper GRO stack assumes network header starts at gro_offset=0
4369 * Drivers could call both napi_gro_frags() and napi_gro_receive()
4370 * We copy ethernet header into skb->data to have a common layout.
4371 */
Eric Dumazet4adb9c42012-05-18 20:49:06 +00004372static struct sk_buff *napi_frags_skb(struct napi_struct *napi)
Herbert Xu96e93ea2009-01-06 10:49:34 -08004373{
Herbert Xu76620aa2009-04-16 02:02:07 -07004374 struct sk_buff *skb = napi->skb;
Eric Dumazeta50e2332014-03-29 21:28:21 -07004375 const struct ethhdr *eth;
4376 unsigned int hlen = sizeof(*eth);
Herbert Xu76620aa2009-04-16 02:02:07 -07004377
4378 napi->skb = NULL;
4379
Eric Dumazeta50e2332014-03-29 21:28:21 -07004380 skb_reset_mac_header(skb);
4381 skb_gro_reset_offset(skb);
4382
4383 eth = skb_gro_header_fast(skb, 0);
4384 if (unlikely(skb_gro_header_hard(skb, hlen))) {
4385 eth = skb_gro_header_slow(skb, hlen, 0);
4386 if (unlikely(!eth)) {
4387 napi_reuse_skb(napi, skb);
4388 return NULL;
4389 }
4390 } else {
4391 gro_pull_from_frag0(skb, hlen);
4392 NAPI_GRO_CB(skb)->frag0 += hlen;
4393 NAPI_GRO_CB(skb)->frag0_len -= hlen;
Herbert Xu76620aa2009-04-16 02:02:07 -07004394 }
Eric Dumazeta50e2332014-03-29 21:28:21 -07004395 __skb_pull(skb, hlen);
4396
4397 /*
4398 * This works because the only protocols we care about don't require
4399 * special handling.
4400 * We'll fix it up properly in napi_frags_finish()
4401 */
4402 skb->protocol = eth->h_proto;
Herbert Xu76620aa2009-04-16 02:02:07 -07004403
Herbert Xu76620aa2009-04-16 02:02:07 -07004404 return skb;
4405}
Herbert Xu76620aa2009-04-16 02:02:07 -07004406
Ben Hutchingsc7c4b3b2009-10-29 21:36:53 -07004407gro_result_t napi_gro_frags(struct napi_struct *napi)
Herbert Xu76620aa2009-04-16 02:02:07 -07004408{
4409 struct sk_buff *skb = napi_frags_skb(napi);
Herbert Xu96e93ea2009-01-06 10:49:34 -08004410
4411 if (!skb)
Ben Hutchingsc7c4b3b2009-10-29 21:36:53 -07004412 return GRO_DROP;
Herbert Xu96e93ea2009-01-06 10:49:34 -08004413
Ben Hutchingsae78dbf2014-01-10 22:17:24 +00004414 trace_napi_gro_frags_entry(skb);
4415
Eric Dumazet89c5fa32012-12-10 13:28:16 +00004416 return napi_frags_finish(napi, skb, dev_gro_receive(napi, skb));
Herbert Xu5d38a072009-01-04 16:13:40 -08004417}
4418EXPORT_SYMBOL(napi_gro_frags);
4419
Tom Herbert573e8fc2014-08-22 13:33:47 -07004420/* Compute the checksum from gro_offset and return the folded value
4421 * after adding in any pseudo checksum.
4422 */
4423__sum16 __skb_gro_checksum_complete(struct sk_buff *skb)
4424{
4425 __wsum wsum;
4426 __sum16 sum;
4427
4428 wsum = skb_checksum(skb, skb_gro_offset(skb), skb_gro_len(skb), 0);
4429
4430 /* NAPI_GRO_CB(skb)->csum holds pseudo checksum */
4431 sum = csum_fold(csum_add(NAPI_GRO_CB(skb)->csum, wsum));
4432 if (likely(!sum)) {
4433 if (unlikely(skb->ip_summed == CHECKSUM_COMPLETE) &&
4434 !skb->csum_complete_sw)
4435 netdev_rx_csum_fault(skb->dev);
4436 }
4437
4438 NAPI_GRO_CB(skb)->csum = wsum;
4439 NAPI_GRO_CB(skb)->csum_valid = 1;
4440
4441 return sum;
4442}
4443EXPORT_SYMBOL(__skb_gro_checksum_complete);
4444
Eric Dumazete326bed2010-04-22 00:22:45 -07004445/*
Zhi Yong Wu855abcf2014-01-01 04:34:50 +08004446 * net_rps_action_and_irq_enable sends any pending IPI's for rps.
Eric Dumazete326bed2010-04-22 00:22:45 -07004447 * Note: called with local irq disabled, but exits with local irq enabled.
4448 */
4449static void net_rps_action_and_irq_enable(struct softnet_data *sd)
4450{
4451#ifdef CONFIG_RPS
4452 struct softnet_data *remsd = sd->rps_ipi_list;
4453
4454 if (remsd) {
4455 sd->rps_ipi_list = NULL;
4456
4457 local_irq_enable();
4458
4459 /* Send pending IPI's to kick RPS processing on remote cpus. */
4460 while (remsd) {
4461 struct softnet_data *next = remsd->rps_ipi_next;
4462
4463 if (cpu_online(remsd->cpu))
Frederic Weisbeckerc46fff22014-02-24 16:40:02 +01004464 smp_call_function_single_async(remsd->cpu,
Frederic Weisbeckerfce8ad12014-02-24 16:40:01 +01004465 &remsd->csd);
Eric Dumazete326bed2010-04-22 00:22:45 -07004466 remsd = next;
4467 }
4468 } else
4469#endif
4470 local_irq_enable();
4471}
4472
Eric Dumazetd75b1ad2014-11-02 06:19:33 -08004473static bool sd_has_rps_ipi_waiting(struct softnet_data *sd)
4474{
4475#ifdef CONFIG_RPS
4476 return sd->rps_ipi_list != NULL;
4477#else
4478 return false;
4479#endif
4480}
4481
Stephen Hemmingerbea33482007-10-03 16:41:36 -07004482static int process_backlog(struct napi_struct *napi, int quota)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004483{
4484 int work = 0;
Eric Dumazeteecfd7c2010-05-06 22:07:48 -07004485 struct softnet_data *sd = container_of(napi, struct softnet_data, backlog);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004486
Eric Dumazete326bed2010-04-22 00:22:45 -07004487 /* Check if we have pending ipi, its better to send them now,
4488 * not waiting net_rx_action() end.
4489 */
Eric Dumazetd75b1ad2014-11-02 06:19:33 -08004490 if (sd_has_rps_ipi_waiting(sd)) {
Eric Dumazete326bed2010-04-22 00:22:45 -07004491 local_irq_disable();
4492 net_rps_action_and_irq_enable(sd);
4493 }
Eric Dumazetd75b1ad2014-11-02 06:19:33 -08004494
Stephen Hemmingerbea33482007-10-03 16:41:36 -07004495 napi->weight = weight_p;
Changli Gao6e7676c2010-04-27 15:07:33 -07004496 local_irq_disable();
Tom Herbert11ef7a82014-06-30 09:50:40 -07004497 while (1) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07004498 struct sk_buff *skb;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004499
Changli Gao6e7676c2010-04-27 15:07:33 -07004500 while ((skb = __skb_dequeue(&sd->process_queue))) {
Eric Dumazete4008272010-04-05 15:42:39 -07004501 local_irq_enable();
Changli Gao6e7676c2010-04-27 15:07:33 -07004502 __netif_receive_skb(skb);
Changli Gao6e7676c2010-04-27 15:07:33 -07004503 local_irq_disable();
Tom Herbert76cc8b12010-05-20 18:37:59 +00004504 input_queue_head_incr(sd);
4505 if (++work >= quota) {
4506 local_irq_enable();
4507 return work;
4508 }
Stephen Hemmingerbea33482007-10-03 16:41:36 -07004509 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07004510
Changli Gao6e7676c2010-04-27 15:07:33 -07004511 rps_lock(sd);
Tom Herbert11ef7a82014-06-30 09:50:40 -07004512 if (skb_queue_empty(&sd->input_pkt_queue)) {
Eric Dumazeteecfd7c2010-05-06 22:07:48 -07004513 /*
4514 * Inline a custom version of __napi_complete().
4515 * only current cpu owns and manipulates this napi,
Tom Herbert11ef7a82014-06-30 09:50:40 -07004516 * and NAPI_STATE_SCHED is the only possible flag set
4517 * on backlog.
4518 * We can use a plain write instead of clear_bit(),
Eric Dumazeteecfd7c2010-05-06 22:07:48 -07004519 * and we dont need an smp_mb() memory barrier.
4520 */
Eric Dumazeteecfd7c2010-05-06 22:07:48 -07004521 napi->state = 0;
Tom Herbert11ef7a82014-06-30 09:50:40 -07004522 rps_unlock(sd);
Eric Dumazeteecfd7c2010-05-06 22:07:48 -07004523
Tom Herbert11ef7a82014-06-30 09:50:40 -07004524 break;
Changli Gao6e7676c2010-04-27 15:07:33 -07004525 }
Tom Herbert11ef7a82014-06-30 09:50:40 -07004526
4527 skb_queue_splice_tail_init(&sd->input_pkt_queue,
4528 &sd->process_queue);
Changli Gao6e7676c2010-04-27 15:07:33 -07004529 rps_unlock(sd);
4530 }
4531 local_irq_enable();
Linus Torvalds1da177e2005-04-16 15:20:36 -07004532
Stephen Hemmingerbea33482007-10-03 16:41:36 -07004533 return work;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004534}
4535
Stephen Hemmingerbea33482007-10-03 16:41:36 -07004536/**
4537 * __napi_schedule - schedule for receive
Randy Dunlapc4ea43c2007-10-12 21:17:49 -07004538 * @n: entry to schedule
Stephen Hemmingerbea33482007-10-03 16:41:36 -07004539 *
Eric Dumazetbc9ad162014-10-28 18:05:13 -07004540 * The entry's receive function will be scheduled to run.
4541 * Consider using __napi_schedule_irqoff() if hard irqs are masked.
Stephen Hemmingerbea33482007-10-03 16:41:36 -07004542 */
Harvey Harrisonb5606c22008-02-13 15:03:16 -08004543void __napi_schedule(struct napi_struct *n)
Stephen Hemmingerbea33482007-10-03 16:41:36 -07004544{
4545 unsigned long flags;
4546
4547 local_irq_save(flags);
Christoph Lameter903ceff2014-08-17 12:30:35 -05004548 ____napi_schedule(this_cpu_ptr(&softnet_data), n);
Stephen Hemmingerbea33482007-10-03 16:41:36 -07004549 local_irq_restore(flags);
4550}
4551EXPORT_SYMBOL(__napi_schedule);
4552
Eric Dumazetbc9ad162014-10-28 18:05:13 -07004553/**
4554 * __napi_schedule_irqoff - schedule for receive
4555 * @n: entry to schedule
4556 *
4557 * Variant of __napi_schedule() assuming hard irqs are masked
4558 */
4559void __napi_schedule_irqoff(struct napi_struct *n)
4560{
4561 ____napi_schedule(this_cpu_ptr(&softnet_data), n);
4562}
4563EXPORT_SYMBOL(__napi_schedule_irqoff);
4564
Herbert Xud565b0a2008-12-15 23:38:52 -08004565void __napi_complete(struct napi_struct *n)
4566{
4567 BUG_ON(!test_bit(NAPI_STATE_SCHED, &n->state));
Herbert Xud565b0a2008-12-15 23:38:52 -08004568
Eric Dumazetd75b1ad2014-11-02 06:19:33 -08004569 list_del_init(&n->poll_list);
Peter Zijlstra4e857c52014-03-17 18:06:10 +01004570 smp_mb__before_atomic();
Herbert Xud565b0a2008-12-15 23:38:52 -08004571 clear_bit(NAPI_STATE_SCHED, &n->state);
4572}
4573EXPORT_SYMBOL(__napi_complete);
4574
Eric Dumazet3b47d302014-11-06 21:09:44 -08004575void napi_complete_done(struct napi_struct *n, int work_done)
Herbert Xud565b0a2008-12-15 23:38:52 -08004576{
4577 unsigned long flags;
4578
4579 /*
4580 * don't let napi dequeue from the cpu poll list
4581 * just in case its running on a different cpu
4582 */
4583 if (unlikely(test_bit(NAPI_STATE_NPSVC, &n->state)))
4584 return;
4585
Eric Dumazet3b47d302014-11-06 21:09:44 -08004586 if (n->gro_list) {
4587 unsigned long timeout = 0;
Eric Dumazetd75b1ad2014-11-02 06:19:33 -08004588
Eric Dumazet3b47d302014-11-06 21:09:44 -08004589 if (work_done)
4590 timeout = n->dev->gro_flush_timeout;
4591
4592 if (timeout)
4593 hrtimer_start(&n->timer, ns_to_ktime(timeout),
4594 HRTIMER_MODE_REL_PINNED);
4595 else
4596 napi_gro_flush(n, false);
4597 }
Eric Dumazetd75b1ad2014-11-02 06:19:33 -08004598 if (likely(list_empty(&n->poll_list))) {
4599 WARN_ON_ONCE(!test_and_clear_bit(NAPI_STATE_SCHED, &n->state));
4600 } else {
4601 /* If n->poll_list is not empty, we need to mask irqs */
4602 local_irq_save(flags);
4603 __napi_complete(n);
4604 local_irq_restore(flags);
4605 }
Herbert Xud565b0a2008-12-15 23:38:52 -08004606}
Eric Dumazet3b47d302014-11-06 21:09:44 -08004607EXPORT_SYMBOL(napi_complete_done);
Herbert Xud565b0a2008-12-15 23:38:52 -08004608
Eliezer Tamiraf12fa62013-06-10 11:39:41 +03004609/* must be called under rcu_read_lock(), as we dont take a reference */
4610struct napi_struct *napi_by_id(unsigned int napi_id)
4611{
4612 unsigned int hash = napi_id % HASH_SIZE(napi_hash);
4613 struct napi_struct *napi;
4614
4615 hlist_for_each_entry_rcu(napi, &napi_hash[hash], napi_hash_node)
4616 if (napi->napi_id == napi_id)
4617 return napi;
4618
4619 return NULL;
4620}
4621EXPORT_SYMBOL_GPL(napi_by_id);
4622
4623void napi_hash_add(struct napi_struct *napi)
4624{
4625 if (!test_and_set_bit(NAPI_STATE_HASHED, &napi->state)) {
4626
4627 spin_lock(&napi_hash_lock);
4628
4629 /* 0 is not a valid id, we also skip an id that is taken
4630 * we expect both events to be extremely rare
4631 */
4632 napi->napi_id = 0;
4633 while (!napi->napi_id) {
4634 napi->napi_id = ++napi_gen_id;
4635 if (napi_by_id(napi->napi_id))
4636 napi->napi_id = 0;
4637 }
4638
4639 hlist_add_head_rcu(&napi->napi_hash_node,
4640 &napi_hash[napi->napi_id % HASH_SIZE(napi_hash)]);
4641
4642 spin_unlock(&napi_hash_lock);
4643 }
4644}
4645EXPORT_SYMBOL_GPL(napi_hash_add);
4646
4647/* Warning : caller is responsible to make sure rcu grace period
4648 * is respected before freeing memory containing @napi
4649 */
4650void napi_hash_del(struct napi_struct *napi)
4651{
4652 spin_lock(&napi_hash_lock);
4653
4654 if (test_and_clear_bit(NAPI_STATE_HASHED, &napi->state))
4655 hlist_del_rcu(&napi->napi_hash_node);
4656
4657 spin_unlock(&napi_hash_lock);
4658}
4659EXPORT_SYMBOL_GPL(napi_hash_del);
4660
Eric Dumazet3b47d302014-11-06 21:09:44 -08004661static enum hrtimer_restart napi_watchdog(struct hrtimer *timer)
4662{
4663 struct napi_struct *napi;
4664
4665 napi = container_of(timer, struct napi_struct, timer);
4666 if (napi->gro_list)
4667 napi_schedule(napi);
4668
4669 return HRTIMER_NORESTART;
4670}
4671
Herbert Xud565b0a2008-12-15 23:38:52 -08004672void netif_napi_add(struct net_device *dev, struct napi_struct *napi,
4673 int (*poll)(struct napi_struct *, int), int weight)
4674{
4675 INIT_LIST_HEAD(&napi->poll_list);
Eric Dumazet3b47d302014-11-06 21:09:44 -08004676 hrtimer_init(&napi->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL_PINNED);
4677 napi->timer.function = napi_watchdog;
Herbert Xu4ae55442009-02-08 18:00:36 +00004678 napi->gro_count = 0;
Herbert Xud565b0a2008-12-15 23:38:52 -08004679 napi->gro_list = NULL;
Herbert Xu5d38a072009-01-04 16:13:40 -08004680 napi->skb = NULL;
Herbert Xud565b0a2008-12-15 23:38:52 -08004681 napi->poll = poll;
Eric Dumazet82dc3c62013-03-05 15:57:22 +00004682 if (weight > NAPI_POLL_WEIGHT)
4683 pr_err_once("netif_napi_add() called with weight %d on device %s\n",
4684 weight, dev->name);
Herbert Xud565b0a2008-12-15 23:38:52 -08004685 napi->weight = weight;
4686 list_add(&napi->dev_list, &dev->napi_list);
Herbert Xud565b0a2008-12-15 23:38:52 -08004687 napi->dev = dev;
Herbert Xu5d38a072009-01-04 16:13:40 -08004688#ifdef CONFIG_NETPOLL
Herbert Xud565b0a2008-12-15 23:38:52 -08004689 spin_lock_init(&napi->poll_lock);
4690 napi->poll_owner = -1;
4691#endif
4692 set_bit(NAPI_STATE_SCHED, &napi->state);
4693}
4694EXPORT_SYMBOL(netif_napi_add);
4695
Eric Dumazet3b47d302014-11-06 21:09:44 -08004696void napi_disable(struct napi_struct *n)
4697{
4698 might_sleep();
4699 set_bit(NAPI_STATE_DISABLE, &n->state);
4700
4701 while (test_and_set_bit(NAPI_STATE_SCHED, &n->state))
4702 msleep(1);
4703
4704 hrtimer_cancel(&n->timer);
4705
4706 clear_bit(NAPI_STATE_DISABLE, &n->state);
4707}
4708EXPORT_SYMBOL(napi_disable);
4709
Herbert Xud565b0a2008-12-15 23:38:52 -08004710void netif_napi_del(struct napi_struct *napi)
4711{
Peter P Waskiewicz Jrd7b06632008-12-26 01:35:35 -08004712 list_del_init(&napi->dev_list);
Herbert Xu76620aa2009-04-16 02:02:07 -07004713 napi_free_frags(napi);
Herbert Xud565b0a2008-12-15 23:38:52 -08004714
Eric Dumazet289dccb2013-12-20 14:29:08 -08004715 kfree_skb_list(napi->gro_list);
Herbert Xud565b0a2008-12-15 23:38:52 -08004716 napi->gro_list = NULL;
Herbert Xu4ae55442009-02-08 18:00:36 +00004717 napi->gro_count = 0;
Herbert Xud565b0a2008-12-15 23:38:52 -08004718}
4719EXPORT_SYMBOL(netif_napi_del);
4720
Herbert Xu726ce702014-12-21 07:16:21 +11004721static int napi_poll(struct napi_struct *n, struct list_head *repoll)
4722{
4723 void *have;
4724 int work, weight;
4725
4726 list_del_init(&n->poll_list);
4727
4728 have = netpoll_poll_lock(n);
4729
4730 weight = n->weight;
4731
4732 /* This NAPI_STATE_SCHED test is for avoiding a race
4733 * with netpoll's poll_napi(). Only the entity which
4734 * obtains the lock and sees NAPI_STATE_SCHED set will
4735 * actually make the ->poll() call. Therefore we avoid
4736 * accidentally calling ->poll() when NAPI is not scheduled.
4737 */
4738 work = 0;
4739 if (test_bit(NAPI_STATE_SCHED, &n->state)) {
4740 work = n->poll(n, weight);
4741 trace_napi_poll(n);
4742 }
4743
4744 WARN_ON_ONCE(work > weight);
4745
4746 if (likely(work < weight))
4747 goto out_unlock;
4748
4749 /* Drivers must not modify the NAPI state if they
4750 * consume the entire weight. In such cases this code
4751 * still "owns" the NAPI instance and therefore can
4752 * move the instance around on the list at-will.
4753 */
4754 if (unlikely(napi_disable_pending(n))) {
4755 napi_complete(n);
4756 goto out_unlock;
4757 }
4758
4759 if (n->gro_list) {
4760 /* flush too old packets
4761 * If HZ < 1000, flush all packets.
4762 */
4763 napi_gro_flush(n, HZ >= 1000);
4764 }
4765
Herbert Xu001ce542014-12-21 07:16:22 +11004766 /* Some drivers may have called napi_schedule
4767 * prior to exhausting their budget.
4768 */
4769 if (unlikely(!list_empty(&n->poll_list))) {
4770 pr_warn_once("%s: Budget exhausted after napi rescheduled\n",
4771 n->dev ? n->dev->name : "backlog");
4772 goto out_unlock;
4773 }
4774
Herbert Xu726ce702014-12-21 07:16:21 +11004775 list_add_tail(&n->poll_list, repoll);
4776
4777out_unlock:
4778 netpoll_poll_unlock(have);
4779
4780 return work;
4781}
4782
Linus Torvalds1da177e2005-04-16 15:20:36 -07004783static void net_rx_action(struct softirq_action *h)
4784{
Christoph Lameter903ceff2014-08-17 12:30:35 -05004785 struct softnet_data *sd = this_cpu_ptr(&softnet_data);
Stephen Hemminger24f8b232008-11-03 17:14:38 -08004786 unsigned long time_limit = jiffies + 2;
Stephen Hemminger51b0bde2005-06-23 20:14:40 -07004787 int budget = netdev_budget;
Eric Dumazetd75b1ad2014-11-02 06:19:33 -08004788 LIST_HEAD(list);
4789 LIST_HEAD(repoll);
Matt Mackall53fb95d2005-08-11 19:27:43 -07004790
Linus Torvalds1da177e2005-04-16 15:20:36 -07004791 local_irq_disable();
Eric Dumazetd75b1ad2014-11-02 06:19:33 -08004792 list_splice_init(&sd->poll_list, &list);
4793 local_irq_enable();
Linus Torvalds1da177e2005-04-16 15:20:36 -07004794
Herbert Xuceb8d5b2014-12-21 07:16:25 +11004795 for (;;) {
Stephen Hemmingerbea33482007-10-03 16:41:36 -07004796 struct napi_struct *n;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004797
Herbert Xuceb8d5b2014-12-21 07:16:25 +11004798 if (list_empty(&list)) {
4799 if (!sd_has_rps_ipi_waiting(sd) && list_empty(&repoll))
4800 return;
4801 break;
4802 }
4803
Herbert Xu6bd373e2014-12-21 07:16:24 +11004804 n = list_first_entry(&list, struct napi_struct, poll_list);
4805 budget -= napi_poll(n, &repoll);
4806
Eric Dumazetd75b1ad2014-11-02 06:19:33 -08004807 /* If softirq window is exhausted then punt.
Stephen Hemminger24f8b232008-11-03 17:14:38 -08004808 * Allow this to run for 2 jiffies since which will allow
4809 * an average latency of 1.5/HZ.
Stephen Hemmingerbea33482007-10-03 16:41:36 -07004810 */
Herbert Xuceb8d5b2014-12-21 07:16:25 +11004811 if (unlikely(budget <= 0 ||
4812 time_after_eq(jiffies, time_limit))) {
4813 sd->time_squeeze++;
4814 break;
4815 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07004816 }
Eric Dumazetd75b1ad2014-11-02 06:19:33 -08004817
Eric Dumazetd75b1ad2014-11-02 06:19:33 -08004818 local_irq_disable();
4819
4820 list_splice_tail_init(&sd->poll_list, &list);
4821 list_splice_tail(&repoll, &list);
4822 list_splice(&list, &sd->poll_list);
4823 if (!list_empty(&sd->poll_list))
4824 __raise_softirq_irqoff(NET_RX_SOFTIRQ);
4825
Eric Dumazete326bed2010-04-22 00:22:45 -07004826 net_rps_action_and_irq_enable(sd);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004827}
4828
Veaceslav Falicoaa9d8562013-08-28 23:25:04 +02004829struct netdev_adjacent {
Jiri Pirko9ff162a2013-01-03 22:48:49 +00004830 struct net_device *dev;
Veaceslav Falico5d261912013-08-28 23:25:05 +02004831
4832 /* upper master flag, there can only be one master device per list */
Jiri Pirko9ff162a2013-01-03 22:48:49 +00004833 bool master;
Veaceslav Falico5d261912013-08-28 23:25:05 +02004834
Veaceslav Falico5d261912013-08-28 23:25:05 +02004835 /* counter for the number of times this device was added to us */
4836 u16 ref_nr;
4837
Veaceslav Falico402dae92013-09-25 09:20:09 +02004838 /* private field for the users */
4839 void *private;
4840
Jiri Pirko9ff162a2013-01-03 22:48:49 +00004841 struct list_head list;
4842 struct rcu_head rcu;
Jiri Pirko9ff162a2013-01-03 22:48:49 +00004843};
4844
Veaceslav Falico5d261912013-08-28 23:25:05 +02004845static struct netdev_adjacent *__netdev_find_adj(struct net_device *dev,
4846 struct net_device *adj_dev,
Veaceslav Falico2f268f12013-09-25 09:20:07 +02004847 struct list_head *adj_list)
Jiri Pirko9ff162a2013-01-03 22:48:49 +00004848{
Veaceslav Falico5d261912013-08-28 23:25:05 +02004849 struct netdev_adjacent *adj;
Veaceslav Falico5d261912013-08-28 23:25:05 +02004850
Veaceslav Falico2f268f12013-09-25 09:20:07 +02004851 list_for_each_entry(adj, adj_list, list) {
Veaceslav Falico5d261912013-08-28 23:25:05 +02004852 if (adj->dev == adj_dev)
4853 return adj;
Jiri Pirko9ff162a2013-01-03 22:48:49 +00004854 }
4855 return NULL;
4856}
4857
4858/**
4859 * netdev_has_upper_dev - Check if device is linked to an upper device
4860 * @dev: device
4861 * @upper_dev: upper device to check
4862 *
4863 * Find out if a device is linked to specified upper device and return true
4864 * in case it is. Note that this checks only immediate upper device,
4865 * not through a complete stack of devices. The caller must hold the RTNL lock.
4866 */
4867bool netdev_has_upper_dev(struct net_device *dev,
4868 struct net_device *upper_dev)
4869{
4870 ASSERT_RTNL();
4871
Veaceslav Falico2f268f12013-09-25 09:20:07 +02004872 return __netdev_find_adj(dev, upper_dev, &dev->all_adj_list.upper);
Jiri Pirko9ff162a2013-01-03 22:48:49 +00004873}
4874EXPORT_SYMBOL(netdev_has_upper_dev);
4875
4876/**
4877 * netdev_has_any_upper_dev - Check if device is linked to some device
4878 * @dev: device
4879 *
4880 * Find out if a device is linked to an upper device and return true in case
4881 * it is. The caller must hold the RTNL lock.
4882 */
stephen hemminger1d143d92013-12-29 14:01:29 -08004883static bool netdev_has_any_upper_dev(struct net_device *dev)
Jiri Pirko9ff162a2013-01-03 22:48:49 +00004884{
4885 ASSERT_RTNL();
4886
Veaceslav Falico2f268f12013-09-25 09:20:07 +02004887 return !list_empty(&dev->all_adj_list.upper);
Jiri Pirko9ff162a2013-01-03 22:48:49 +00004888}
Jiri Pirko9ff162a2013-01-03 22:48:49 +00004889
4890/**
4891 * netdev_master_upper_dev_get - Get master upper device
4892 * @dev: device
4893 *
4894 * Find a master upper device and return pointer to it or NULL in case
4895 * it's not there. The caller must hold the RTNL lock.
4896 */
4897struct net_device *netdev_master_upper_dev_get(struct net_device *dev)
4898{
Veaceslav Falicoaa9d8562013-08-28 23:25:04 +02004899 struct netdev_adjacent *upper;
Jiri Pirko9ff162a2013-01-03 22:48:49 +00004900
4901 ASSERT_RTNL();
4902
Veaceslav Falico2f268f12013-09-25 09:20:07 +02004903 if (list_empty(&dev->adj_list.upper))
Jiri Pirko9ff162a2013-01-03 22:48:49 +00004904 return NULL;
4905
Veaceslav Falico2f268f12013-09-25 09:20:07 +02004906 upper = list_first_entry(&dev->adj_list.upper,
Veaceslav Falicoaa9d8562013-08-28 23:25:04 +02004907 struct netdev_adjacent, list);
Jiri Pirko9ff162a2013-01-03 22:48:49 +00004908 if (likely(upper->master))
4909 return upper->dev;
4910 return NULL;
4911}
4912EXPORT_SYMBOL(netdev_master_upper_dev_get);
4913
Veaceslav Falicob6ccba42013-09-25 09:20:23 +02004914void *netdev_adjacent_get_private(struct list_head *adj_list)
4915{
4916 struct netdev_adjacent *adj;
4917
4918 adj = list_entry(adj_list, struct netdev_adjacent, list);
4919
4920 return adj->private;
4921}
4922EXPORT_SYMBOL(netdev_adjacent_get_private);
4923
Veaceslav Falico31088a12013-09-25 09:20:12 +02004924/**
Vlad Yasevich44a40852014-05-16 17:20:38 -04004925 * netdev_upper_get_next_dev_rcu - Get the next dev from upper list
4926 * @dev: device
4927 * @iter: list_head ** of the current position
4928 *
4929 * Gets the next device from the dev's upper list, starting from iter
4930 * position. The caller must hold RCU read lock.
4931 */
4932struct net_device *netdev_upper_get_next_dev_rcu(struct net_device *dev,
4933 struct list_head **iter)
4934{
4935 struct netdev_adjacent *upper;
4936
4937 WARN_ON_ONCE(!rcu_read_lock_held() && !lockdep_rtnl_is_held());
4938
4939 upper = list_entry_rcu((*iter)->next, struct netdev_adjacent, list);
4940
4941 if (&upper->list == &dev->adj_list.upper)
4942 return NULL;
4943
4944 *iter = &upper->list;
4945
4946 return upper->dev;
4947}
4948EXPORT_SYMBOL(netdev_upper_get_next_dev_rcu);
4949
4950/**
Veaceslav Falico31088a12013-09-25 09:20:12 +02004951 * netdev_all_upper_get_next_dev_rcu - Get the next dev from upper list
Veaceslav Falico48311f42013-08-28 23:25:07 +02004952 * @dev: device
4953 * @iter: list_head ** of the current position
4954 *
4955 * Gets the next device from the dev's upper list, starting from iter
4956 * position. The caller must hold RCU read lock.
4957 */
Veaceslav Falico2f268f12013-09-25 09:20:07 +02004958struct net_device *netdev_all_upper_get_next_dev_rcu(struct net_device *dev,
4959 struct list_head **iter)
Veaceslav Falico48311f42013-08-28 23:25:07 +02004960{
4961 struct netdev_adjacent *upper;
4962
John Fastabend85328242013-11-26 06:33:52 +00004963 WARN_ON_ONCE(!rcu_read_lock_held() && !lockdep_rtnl_is_held());
Veaceslav Falico48311f42013-08-28 23:25:07 +02004964
4965 upper = list_entry_rcu((*iter)->next, struct netdev_adjacent, list);
4966
Veaceslav Falico2f268f12013-09-25 09:20:07 +02004967 if (&upper->list == &dev->all_adj_list.upper)
Veaceslav Falico48311f42013-08-28 23:25:07 +02004968 return NULL;
4969
4970 *iter = &upper->list;
4971
4972 return upper->dev;
4973}
Veaceslav Falico2f268f12013-09-25 09:20:07 +02004974EXPORT_SYMBOL(netdev_all_upper_get_next_dev_rcu);
Veaceslav Falico48311f42013-08-28 23:25:07 +02004975
Jiri Pirko9ff162a2013-01-03 22:48:49 +00004976/**
Veaceslav Falico31088a12013-09-25 09:20:12 +02004977 * netdev_lower_get_next_private - Get the next ->private from the
4978 * lower neighbour list
4979 * @dev: device
4980 * @iter: list_head ** of the current position
4981 *
4982 * Gets the next netdev_adjacent->private from the dev's lower neighbour
4983 * list, starting from iter position. The caller must hold either hold the
4984 * RTNL lock or its own locking that guarantees that the neighbour lower
4985 * list will remain unchainged.
4986 */
4987void *netdev_lower_get_next_private(struct net_device *dev,
4988 struct list_head **iter)
4989{
4990 struct netdev_adjacent *lower;
4991
4992 lower = list_entry(*iter, struct netdev_adjacent, list);
4993
4994 if (&lower->list == &dev->adj_list.lower)
4995 return NULL;
4996
Veaceslav Falico6859e7d2014-04-07 11:25:12 +02004997 *iter = lower->list.next;
Veaceslav Falico31088a12013-09-25 09:20:12 +02004998
4999 return lower->private;
5000}
5001EXPORT_SYMBOL(netdev_lower_get_next_private);
5002
5003/**
5004 * netdev_lower_get_next_private_rcu - Get the next ->private from the
5005 * lower neighbour list, RCU
5006 * variant
5007 * @dev: device
5008 * @iter: list_head ** of the current position
5009 *
5010 * Gets the next netdev_adjacent->private from the dev's lower neighbour
5011 * list, starting from iter position. The caller must hold RCU read lock.
5012 */
5013void *netdev_lower_get_next_private_rcu(struct net_device *dev,
5014 struct list_head **iter)
5015{
5016 struct netdev_adjacent *lower;
5017
5018 WARN_ON_ONCE(!rcu_read_lock_held());
5019
5020 lower = list_entry_rcu((*iter)->next, struct netdev_adjacent, list);
5021
5022 if (&lower->list == &dev->adj_list.lower)
5023 return NULL;
5024
Veaceslav Falico6859e7d2014-04-07 11:25:12 +02005025 *iter = &lower->list;
Veaceslav Falico31088a12013-09-25 09:20:12 +02005026
5027 return lower->private;
5028}
5029EXPORT_SYMBOL(netdev_lower_get_next_private_rcu);
5030
5031/**
Vlad Yasevich4085ebe2014-05-16 17:04:53 -04005032 * netdev_lower_get_next - Get the next device from the lower neighbour
5033 * list
5034 * @dev: device
5035 * @iter: list_head ** of the current position
5036 *
5037 * Gets the next netdev_adjacent from the dev's lower neighbour
5038 * list, starting from iter position. The caller must hold RTNL lock or
5039 * its own locking that guarantees that the neighbour lower
5040 * list will remain unchainged.
5041 */
5042void *netdev_lower_get_next(struct net_device *dev, struct list_head **iter)
5043{
5044 struct netdev_adjacent *lower;
5045
5046 lower = list_entry((*iter)->next, struct netdev_adjacent, list);
5047
5048 if (&lower->list == &dev->adj_list.lower)
5049 return NULL;
5050
5051 *iter = &lower->list;
5052
5053 return lower->dev;
5054}
5055EXPORT_SYMBOL(netdev_lower_get_next);
5056
5057/**
dingtianhonge001bfa2013-12-13 10:19:55 +08005058 * netdev_lower_get_first_private_rcu - Get the first ->private from the
5059 * lower neighbour list, RCU
5060 * variant
5061 * @dev: device
5062 *
5063 * Gets the first netdev_adjacent->private from the dev's lower neighbour
5064 * list. The caller must hold RCU read lock.
5065 */
5066void *netdev_lower_get_first_private_rcu(struct net_device *dev)
5067{
5068 struct netdev_adjacent *lower;
5069
5070 lower = list_first_or_null_rcu(&dev->adj_list.lower,
5071 struct netdev_adjacent, list);
5072 if (lower)
5073 return lower->private;
5074 return NULL;
5075}
5076EXPORT_SYMBOL(netdev_lower_get_first_private_rcu);
5077
5078/**
Jiri Pirko9ff162a2013-01-03 22:48:49 +00005079 * netdev_master_upper_dev_get_rcu - Get master upper device
5080 * @dev: device
5081 *
5082 * Find a master upper device and return pointer to it or NULL in case
5083 * it's not there. The caller must hold the RCU read lock.
5084 */
5085struct net_device *netdev_master_upper_dev_get_rcu(struct net_device *dev)
5086{
Veaceslav Falicoaa9d8562013-08-28 23:25:04 +02005087 struct netdev_adjacent *upper;
Jiri Pirko9ff162a2013-01-03 22:48:49 +00005088
Veaceslav Falico2f268f12013-09-25 09:20:07 +02005089 upper = list_first_or_null_rcu(&dev->adj_list.upper,
Veaceslav Falicoaa9d8562013-08-28 23:25:04 +02005090 struct netdev_adjacent, list);
Jiri Pirko9ff162a2013-01-03 22:48:49 +00005091 if (upper && likely(upper->master))
5092 return upper->dev;
5093 return NULL;
5094}
5095EXPORT_SYMBOL(netdev_master_upper_dev_get_rcu);
5096
Rashika Kheria0a59f3a2014-02-09 20:26:25 +05305097static int netdev_adjacent_sysfs_add(struct net_device *dev,
Veaceslav Falico3ee32702014-01-14 21:58:50 +01005098 struct net_device *adj_dev,
5099 struct list_head *dev_list)
5100{
5101 char linkname[IFNAMSIZ+7];
5102 sprintf(linkname, dev_list == &dev->adj_list.upper ?
5103 "upper_%s" : "lower_%s", adj_dev->name);
5104 return sysfs_create_link(&(dev->dev.kobj), &(adj_dev->dev.kobj),
5105 linkname);
5106}
Rashika Kheria0a59f3a2014-02-09 20:26:25 +05305107static void netdev_adjacent_sysfs_del(struct net_device *dev,
Veaceslav Falico3ee32702014-01-14 21:58:50 +01005108 char *name,
5109 struct list_head *dev_list)
5110{
5111 char linkname[IFNAMSIZ+7];
5112 sprintf(linkname, dev_list == &dev->adj_list.upper ?
5113 "upper_%s" : "lower_%s", name);
5114 sysfs_remove_link(&(dev->dev.kobj), linkname);
5115}
5116
Alexander Y. Fomichev7ce64c72014-09-15 14:22:35 +04005117static inline bool netdev_adjacent_is_neigh_list(struct net_device *dev,
5118 struct net_device *adj_dev,
5119 struct list_head *dev_list)
5120{
5121 return (dev_list == &dev->adj_list.upper ||
5122 dev_list == &dev->adj_list.lower) &&
5123 net_eq(dev_net(dev), dev_net(adj_dev));
5124}
Veaceslav Falico3ee32702014-01-14 21:58:50 +01005125
Veaceslav Falico5d261912013-08-28 23:25:05 +02005126static int __netdev_adjacent_dev_insert(struct net_device *dev,
5127 struct net_device *adj_dev,
Veaceslav Falico7863c052013-09-25 09:20:06 +02005128 struct list_head *dev_list,
Veaceslav Falico402dae92013-09-25 09:20:09 +02005129 void *private, bool master)
Veaceslav Falico5d261912013-08-28 23:25:05 +02005130{
5131 struct netdev_adjacent *adj;
Veaceslav Falico842d67a2013-09-25 09:20:31 +02005132 int ret;
Veaceslav Falico5d261912013-08-28 23:25:05 +02005133
Veaceslav Falico7863c052013-09-25 09:20:06 +02005134 adj = __netdev_find_adj(dev, adj_dev, dev_list);
Veaceslav Falico5d261912013-08-28 23:25:05 +02005135
5136 if (adj) {
Veaceslav Falico5d261912013-08-28 23:25:05 +02005137 adj->ref_nr++;
5138 return 0;
5139 }
5140
5141 adj = kmalloc(sizeof(*adj), GFP_KERNEL);
5142 if (!adj)
5143 return -ENOMEM;
5144
5145 adj->dev = adj_dev;
5146 adj->master = master;
Veaceslav Falico5d261912013-08-28 23:25:05 +02005147 adj->ref_nr = 1;
Veaceslav Falico402dae92013-09-25 09:20:09 +02005148 adj->private = private;
Veaceslav Falico5d261912013-08-28 23:25:05 +02005149 dev_hold(adj_dev);
Veaceslav Falico2f268f12013-09-25 09:20:07 +02005150
5151 pr_debug("dev_hold for %s, because of link added from %s to %s\n",
5152 adj_dev->name, dev->name, adj_dev->name);
Veaceslav Falico5d261912013-08-28 23:25:05 +02005153
Alexander Y. Fomichev7ce64c72014-09-15 14:22:35 +04005154 if (netdev_adjacent_is_neigh_list(dev, adj_dev, dev_list)) {
Veaceslav Falico3ee32702014-01-14 21:58:50 +01005155 ret = netdev_adjacent_sysfs_add(dev, adj_dev, dev_list);
Veaceslav Falico5831d662013-09-25 09:20:32 +02005156 if (ret)
5157 goto free_adj;
5158 }
5159
Veaceslav Falico7863c052013-09-25 09:20:06 +02005160 /* Ensure that master link is always the first item in list. */
Veaceslav Falico842d67a2013-09-25 09:20:31 +02005161 if (master) {
5162 ret = sysfs_create_link(&(dev->dev.kobj),
5163 &(adj_dev->dev.kobj), "master");
5164 if (ret)
Veaceslav Falico5831d662013-09-25 09:20:32 +02005165 goto remove_symlinks;
Veaceslav Falico842d67a2013-09-25 09:20:31 +02005166
Veaceslav Falico7863c052013-09-25 09:20:06 +02005167 list_add_rcu(&adj->list, dev_list);
Veaceslav Falico842d67a2013-09-25 09:20:31 +02005168 } else {
Veaceslav Falico7863c052013-09-25 09:20:06 +02005169 list_add_tail_rcu(&adj->list, dev_list);
Veaceslav Falico842d67a2013-09-25 09:20:31 +02005170 }
Veaceslav Falico5d261912013-08-28 23:25:05 +02005171
5172 return 0;
Veaceslav Falico842d67a2013-09-25 09:20:31 +02005173
Veaceslav Falico5831d662013-09-25 09:20:32 +02005174remove_symlinks:
Alexander Y. Fomichev7ce64c72014-09-15 14:22:35 +04005175 if (netdev_adjacent_is_neigh_list(dev, adj_dev, dev_list))
Veaceslav Falico3ee32702014-01-14 21:58:50 +01005176 netdev_adjacent_sysfs_del(dev, adj_dev->name, dev_list);
Veaceslav Falico842d67a2013-09-25 09:20:31 +02005177free_adj:
5178 kfree(adj);
Nikolay Aleksandrov974daef2013-10-23 15:28:56 +02005179 dev_put(adj_dev);
Veaceslav Falico842d67a2013-09-25 09:20:31 +02005180
5181 return ret;
Veaceslav Falico5d261912013-08-28 23:25:05 +02005182}
5183
stephen hemminger1d143d92013-12-29 14:01:29 -08005184static void __netdev_adjacent_dev_remove(struct net_device *dev,
5185 struct net_device *adj_dev,
5186 struct list_head *dev_list)
Veaceslav Falico5d261912013-08-28 23:25:05 +02005187{
5188 struct netdev_adjacent *adj;
5189
Veaceslav Falico7863c052013-09-25 09:20:06 +02005190 adj = __netdev_find_adj(dev, adj_dev, dev_list);
Veaceslav Falico5d261912013-08-28 23:25:05 +02005191
Veaceslav Falico2f268f12013-09-25 09:20:07 +02005192 if (!adj) {
5193 pr_err("tried to remove device %s from %s\n",
5194 dev->name, adj_dev->name);
Veaceslav Falico5d261912013-08-28 23:25:05 +02005195 BUG();
Veaceslav Falico2f268f12013-09-25 09:20:07 +02005196 }
Veaceslav Falico5d261912013-08-28 23:25:05 +02005197
5198 if (adj->ref_nr > 1) {
Veaceslav Falico2f268f12013-09-25 09:20:07 +02005199 pr_debug("%s to %s ref_nr-- = %d\n", dev->name, adj_dev->name,
5200 adj->ref_nr-1);
Veaceslav Falico5d261912013-08-28 23:25:05 +02005201 adj->ref_nr--;
5202 return;
5203 }
5204
Veaceslav Falico842d67a2013-09-25 09:20:31 +02005205 if (adj->master)
5206 sysfs_remove_link(&(dev->dev.kobj), "master");
5207
Alexander Y. Fomichev7ce64c72014-09-15 14:22:35 +04005208 if (netdev_adjacent_is_neigh_list(dev, adj_dev, dev_list))
Veaceslav Falico3ee32702014-01-14 21:58:50 +01005209 netdev_adjacent_sysfs_del(dev, adj_dev->name, dev_list);
Veaceslav Falico5831d662013-09-25 09:20:32 +02005210
Veaceslav Falico5d261912013-08-28 23:25:05 +02005211 list_del_rcu(&adj->list);
Veaceslav Falico2f268f12013-09-25 09:20:07 +02005212 pr_debug("dev_put for %s, because link removed from %s to %s\n",
5213 adj_dev->name, dev->name, adj_dev->name);
Veaceslav Falico5d261912013-08-28 23:25:05 +02005214 dev_put(adj_dev);
5215 kfree_rcu(adj, rcu);
5216}
5217
stephen hemminger1d143d92013-12-29 14:01:29 -08005218static int __netdev_adjacent_dev_link_lists(struct net_device *dev,
5219 struct net_device *upper_dev,
5220 struct list_head *up_list,
5221 struct list_head *down_list,
5222 void *private, bool master)
Veaceslav Falico5d261912013-08-28 23:25:05 +02005223{
5224 int ret;
5225
Veaceslav Falico402dae92013-09-25 09:20:09 +02005226 ret = __netdev_adjacent_dev_insert(dev, upper_dev, up_list, private,
5227 master);
Veaceslav Falico5d261912013-08-28 23:25:05 +02005228 if (ret)
5229 return ret;
5230
Veaceslav Falico402dae92013-09-25 09:20:09 +02005231 ret = __netdev_adjacent_dev_insert(upper_dev, dev, down_list, private,
5232 false);
Veaceslav Falico5d261912013-08-28 23:25:05 +02005233 if (ret) {
Veaceslav Falico2f268f12013-09-25 09:20:07 +02005234 __netdev_adjacent_dev_remove(dev, upper_dev, up_list);
Veaceslav Falico5d261912013-08-28 23:25:05 +02005235 return ret;
5236 }
5237
5238 return 0;
5239}
5240
stephen hemminger1d143d92013-12-29 14:01:29 -08005241static int __netdev_adjacent_dev_link(struct net_device *dev,
5242 struct net_device *upper_dev)
Veaceslav Falico5d261912013-08-28 23:25:05 +02005243{
Veaceslav Falico2f268f12013-09-25 09:20:07 +02005244 return __netdev_adjacent_dev_link_lists(dev, upper_dev,
5245 &dev->all_adj_list.upper,
5246 &upper_dev->all_adj_list.lower,
Veaceslav Falico402dae92013-09-25 09:20:09 +02005247 NULL, false);
Veaceslav Falico5d261912013-08-28 23:25:05 +02005248}
5249
stephen hemminger1d143d92013-12-29 14:01:29 -08005250static void __netdev_adjacent_dev_unlink_lists(struct net_device *dev,
5251 struct net_device *upper_dev,
5252 struct list_head *up_list,
5253 struct list_head *down_list)
Veaceslav Falico5d261912013-08-28 23:25:05 +02005254{
Veaceslav Falico2f268f12013-09-25 09:20:07 +02005255 __netdev_adjacent_dev_remove(dev, upper_dev, up_list);
5256 __netdev_adjacent_dev_remove(upper_dev, dev, down_list);
Veaceslav Falico5d261912013-08-28 23:25:05 +02005257}
5258
stephen hemminger1d143d92013-12-29 14:01:29 -08005259static void __netdev_adjacent_dev_unlink(struct net_device *dev,
5260 struct net_device *upper_dev)
Veaceslav Falico5d261912013-08-28 23:25:05 +02005261{
Veaceslav Falico2f268f12013-09-25 09:20:07 +02005262 __netdev_adjacent_dev_unlink_lists(dev, upper_dev,
5263 &dev->all_adj_list.upper,
5264 &upper_dev->all_adj_list.lower);
Veaceslav Falico5d261912013-08-28 23:25:05 +02005265}
5266
stephen hemminger1d143d92013-12-29 14:01:29 -08005267static int __netdev_adjacent_dev_link_neighbour(struct net_device *dev,
5268 struct net_device *upper_dev,
5269 void *private, bool master)
Veaceslav Falico2f268f12013-09-25 09:20:07 +02005270{
5271 int ret = __netdev_adjacent_dev_link(dev, upper_dev);
5272
5273 if (ret)
5274 return ret;
5275
5276 ret = __netdev_adjacent_dev_link_lists(dev, upper_dev,
5277 &dev->adj_list.upper,
5278 &upper_dev->adj_list.lower,
Veaceslav Falico402dae92013-09-25 09:20:09 +02005279 private, master);
Veaceslav Falico2f268f12013-09-25 09:20:07 +02005280 if (ret) {
5281 __netdev_adjacent_dev_unlink(dev, upper_dev);
5282 return ret;
5283 }
5284
5285 return 0;
5286}
5287
stephen hemminger1d143d92013-12-29 14:01:29 -08005288static void __netdev_adjacent_dev_unlink_neighbour(struct net_device *dev,
5289 struct net_device *upper_dev)
Veaceslav Falico2f268f12013-09-25 09:20:07 +02005290{
5291 __netdev_adjacent_dev_unlink(dev, upper_dev);
5292 __netdev_adjacent_dev_unlink_lists(dev, upper_dev,
5293 &dev->adj_list.upper,
5294 &upper_dev->adj_list.lower);
5295}
Veaceslav Falico5d261912013-08-28 23:25:05 +02005296
Jiri Pirko9ff162a2013-01-03 22:48:49 +00005297static int __netdev_upper_dev_link(struct net_device *dev,
Veaceslav Falico402dae92013-09-25 09:20:09 +02005298 struct net_device *upper_dev, bool master,
5299 void *private)
Jiri Pirko9ff162a2013-01-03 22:48:49 +00005300{
Veaceslav Falico5d261912013-08-28 23:25:05 +02005301 struct netdev_adjacent *i, *j, *to_i, *to_j;
5302 int ret = 0;
Jiri Pirko9ff162a2013-01-03 22:48:49 +00005303
5304 ASSERT_RTNL();
5305
5306 if (dev == upper_dev)
5307 return -EBUSY;
5308
5309 /* To prevent loops, check if dev is not upper device to upper_dev. */
Veaceslav Falico2f268f12013-09-25 09:20:07 +02005310 if (__netdev_find_adj(upper_dev, dev, &upper_dev->all_adj_list.upper))
Jiri Pirko9ff162a2013-01-03 22:48:49 +00005311 return -EBUSY;
5312
Vlad Yasevichd66bf7d2015-05-02 21:33:44 -04005313 if (__netdev_find_adj(dev, upper_dev, &dev->adj_list.upper))
Jiri Pirko9ff162a2013-01-03 22:48:49 +00005314 return -EEXIST;
5315
5316 if (master && netdev_master_upper_dev_get(dev))
5317 return -EBUSY;
5318
Veaceslav Falico402dae92013-09-25 09:20:09 +02005319 ret = __netdev_adjacent_dev_link_neighbour(dev, upper_dev, private,
5320 master);
Veaceslav Falico5d261912013-08-28 23:25:05 +02005321 if (ret)
5322 return ret;
Jiri Pirko9ff162a2013-01-03 22:48:49 +00005323
Veaceslav Falico5d261912013-08-28 23:25:05 +02005324 /* Now that we linked these devs, make all the upper_dev's
Veaceslav Falico2f268f12013-09-25 09:20:07 +02005325 * all_adj_list.upper visible to every dev's all_adj_list.lower an
Veaceslav Falico5d261912013-08-28 23:25:05 +02005326 * versa, and don't forget the devices itself. All of these
5327 * links are non-neighbours.
5328 */
Veaceslav Falico2f268f12013-09-25 09:20:07 +02005329 list_for_each_entry(i, &dev->all_adj_list.lower, list) {
5330 list_for_each_entry(j, &upper_dev->all_adj_list.upper, list) {
5331 pr_debug("Interlinking %s with %s, non-neighbour\n",
5332 i->dev->name, j->dev->name);
Veaceslav Falico5d261912013-08-28 23:25:05 +02005333 ret = __netdev_adjacent_dev_link(i->dev, j->dev);
5334 if (ret)
5335 goto rollback_mesh;
5336 }
5337 }
Jiri Pirko9ff162a2013-01-03 22:48:49 +00005338
Veaceslav Falico5d261912013-08-28 23:25:05 +02005339 /* add dev to every upper_dev's upper device */
Veaceslav Falico2f268f12013-09-25 09:20:07 +02005340 list_for_each_entry(i, &upper_dev->all_adj_list.upper, list) {
5341 pr_debug("linking %s's upper device %s with %s\n",
5342 upper_dev->name, i->dev->name, dev->name);
Veaceslav Falico5d261912013-08-28 23:25:05 +02005343 ret = __netdev_adjacent_dev_link(dev, i->dev);
5344 if (ret)
5345 goto rollback_upper_mesh;
5346 }
5347
5348 /* add upper_dev to every dev's lower device */
Veaceslav Falico2f268f12013-09-25 09:20:07 +02005349 list_for_each_entry(i, &dev->all_adj_list.lower, list) {
5350 pr_debug("linking %s's lower device %s with %s\n", dev->name,
5351 i->dev->name, upper_dev->name);
Veaceslav Falico5d261912013-08-28 23:25:05 +02005352 ret = __netdev_adjacent_dev_link(i->dev, upper_dev);
5353 if (ret)
5354 goto rollback_lower_mesh;
5355 }
5356
Jiri Pirko42e52bf2013-05-25 04:12:10 +00005357 call_netdevice_notifiers(NETDEV_CHANGEUPPER, dev);
Jiri Pirko9ff162a2013-01-03 22:48:49 +00005358 return 0;
Veaceslav Falico5d261912013-08-28 23:25:05 +02005359
5360rollback_lower_mesh:
5361 to_i = i;
Veaceslav Falico2f268f12013-09-25 09:20:07 +02005362 list_for_each_entry(i, &dev->all_adj_list.lower, list) {
Veaceslav Falico5d261912013-08-28 23:25:05 +02005363 if (i == to_i)
5364 break;
5365 __netdev_adjacent_dev_unlink(i->dev, upper_dev);
5366 }
5367
5368 i = NULL;
5369
5370rollback_upper_mesh:
5371 to_i = i;
Veaceslav Falico2f268f12013-09-25 09:20:07 +02005372 list_for_each_entry(i, &upper_dev->all_adj_list.upper, list) {
Veaceslav Falico5d261912013-08-28 23:25:05 +02005373 if (i == to_i)
5374 break;
5375 __netdev_adjacent_dev_unlink(dev, i->dev);
5376 }
5377
5378 i = j = NULL;
5379
5380rollback_mesh:
5381 to_i = i;
5382 to_j = j;
Veaceslav Falico2f268f12013-09-25 09:20:07 +02005383 list_for_each_entry(i, &dev->all_adj_list.lower, list) {
5384 list_for_each_entry(j, &upper_dev->all_adj_list.upper, list) {
Veaceslav Falico5d261912013-08-28 23:25:05 +02005385 if (i == to_i && j == to_j)
5386 break;
5387 __netdev_adjacent_dev_unlink(i->dev, j->dev);
5388 }
5389 if (i == to_i)
5390 break;
5391 }
5392
Veaceslav Falico2f268f12013-09-25 09:20:07 +02005393 __netdev_adjacent_dev_unlink_neighbour(dev, upper_dev);
Veaceslav Falico5d261912013-08-28 23:25:05 +02005394
5395 return ret;
Jiri Pirko9ff162a2013-01-03 22:48:49 +00005396}
5397
5398/**
5399 * netdev_upper_dev_link - Add a link to the upper device
5400 * @dev: device
5401 * @upper_dev: new upper device
5402 *
5403 * Adds a link to device which is upper to this one. The caller must hold
5404 * the RTNL lock. On a failure a negative errno code is returned.
5405 * On success the reference counts are adjusted and the function
5406 * returns zero.
5407 */
5408int netdev_upper_dev_link(struct net_device *dev,
5409 struct net_device *upper_dev)
5410{
Veaceslav Falico402dae92013-09-25 09:20:09 +02005411 return __netdev_upper_dev_link(dev, upper_dev, false, NULL);
Jiri Pirko9ff162a2013-01-03 22:48:49 +00005412}
5413EXPORT_SYMBOL(netdev_upper_dev_link);
5414
5415/**
5416 * netdev_master_upper_dev_link - Add a master link to the upper device
5417 * @dev: device
5418 * @upper_dev: new upper device
5419 *
5420 * Adds a link to device which is upper to this one. In this case, only
5421 * one master upper device can be linked, although other non-master devices
5422 * might be linked as well. The caller must hold the RTNL lock.
5423 * On a failure a negative errno code is returned. On success the reference
5424 * counts are adjusted and the function returns zero.
5425 */
5426int netdev_master_upper_dev_link(struct net_device *dev,
5427 struct net_device *upper_dev)
5428{
Veaceslav Falico402dae92013-09-25 09:20:09 +02005429 return __netdev_upper_dev_link(dev, upper_dev, true, NULL);
Jiri Pirko9ff162a2013-01-03 22:48:49 +00005430}
5431EXPORT_SYMBOL(netdev_master_upper_dev_link);
5432
Veaceslav Falico402dae92013-09-25 09:20:09 +02005433int netdev_master_upper_dev_link_private(struct net_device *dev,
5434 struct net_device *upper_dev,
5435 void *private)
5436{
5437 return __netdev_upper_dev_link(dev, upper_dev, true, private);
5438}
5439EXPORT_SYMBOL(netdev_master_upper_dev_link_private);
5440
Jiri Pirko9ff162a2013-01-03 22:48:49 +00005441/**
5442 * netdev_upper_dev_unlink - Removes a link to upper device
5443 * @dev: device
5444 * @upper_dev: new upper device
5445 *
5446 * Removes a link to device which is upper to this one. The caller must hold
5447 * the RTNL lock.
5448 */
5449void netdev_upper_dev_unlink(struct net_device *dev,
5450 struct net_device *upper_dev)
5451{
Veaceslav Falico5d261912013-08-28 23:25:05 +02005452 struct netdev_adjacent *i, *j;
Jiri Pirko9ff162a2013-01-03 22:48:49 +00005453 ASSERT_RTNL();
5454
Veaceslav Falico2f268f12013-09-25 09:20:07 +02005455 __netdev_adjacent_dev_unlink_neighbour(dev, upper_dev);
Veaceslav Falico5d261912013-08-28 23:25:05 +02005456
5457 /* Here is the tricky part. We must remove all dev's lower
5458 * devices from all upper_dev's upper devices and vice
5459 * versa, to maintain the graph relationship.
5460 */
Veaceslav Falico2f268f12013-09-25 09:20:07 +02005461 list_for_each_entry(i, &dev->all_adj_list.lower, list)
5462 list_for_each_entry(j, &upper_dev->all_adj_list.upper, list)
Veaceslav Falico5d261912013-08-28 23:25:05 +02005463 __netdev_adjacent_dev_unlink(i->dev, j->dev);
5464
5465 /* remove also the devices itself from lower/upper device
5466 * list
5467 */
Veaceslav Falico2f268f12013-09-25 09:20:07 +02005468 list_for_each_entry(i, &dev->all_adj_list.lower, list)
Veaceslav Falico5d261912013-08-28 23:25:05 +02005469 __netdev_adjacent_dev_unlink(i->dev, upper_dev);
5470
Veaceslav Falico2f268f12013-09-25 09:20:07 +02005471 list_for_each_entry(i, &upper_dev->all_adj_list.upper, list)
Veaceslav Falico5d261912013-08-28 23:25:05 +02005472 __netdev_adjacent_dev_unlink(dev, i->dev);
5473
Jiri Pirko42e52bf2013-05-25 04:12:10 +00005474 call_netdevice_notifiers(NETDEV_CHANGEUPPER, dev);
Jiri Pirko9ff162a2013-01-03 22:48:49 +00005475}
5476EXPORT_SYMBOL(netdev_upper_dev_unlink);
5477
Moni Shoua61bd3852015-02-03 16:48:29 +02005478/**
5479 * netdev_bonding_info_change - Dispatch event about slave change
5480 * @dev: device
Masanari Iida4a26e4532015-02-14 22:26:34 +09005481 * @bonding_info: info to dispatch
Moni Shoua61bd3852015-02-03 16:48:29 +02005482 *
5483 * Send NETDEV_BONDING_INFO to netdev notifiers with info.
5484 * The caller must hold the RTNL lock.
5485 */
5486void netdev_bonding_info_change(struct net_device *dev,
5487 struct netdev_bonding_info *bonding_info)
5488{
5489 struct netdev_notifier_bonding_info info;
5490
5491 memcpy(&info.bonding_info, bonding_info,
5492 sizeof(struct netdev_bonding_info));
5493 call_netdevice_notifiers_info(NETDEV_BONDING_INFO, dev,
5494 &info.info);
5495}
5496EXPORT_SYMBOL(netdev_bonding_info_change);
5497
Eric Dumazet2ce1ee12015-02-04 13:37:44 -08005498static void netdev_adjacent_add_links(struct net_device *dev)
Alexander Y. Fomichev4c754312014-08-25 16:26:45 +04005499{
5500 struct netdev_adjacent *iter;
5501
5502 struct net *net = dev_net(dev);
5503
5504 list_for_each_entry(iter, &dev->adj_list.upper, list) {
5505 if (!net_eq(net,dev_net(iter->dev)))
5506 continue;
5507 netdev_adjacent_sysfs_add(iter->dev, dev,
5508 &iter->dev->adj_list.lower);
5509 netdev_adjacent_sysfs_add(dev, iter->dev,
5510 &dev->adj_list.upper);
5511 }
5512
5513 list_for_each_entry(iter, &dev->adj_list.lower, list) {
5514 if (!net_eq(net,dev_net(iter->dev)))
5515 continue;
5516 netdev_adjacent_sysfs_add(iter->dev, dev,
5517 &iter->dev->adj_list.upper);
5518 netdev_adjacent_sysfs_add(dev, iter->dev,
5519 &dev->adj_list.lower);
5520 }
5521}
5522
Eric Dumazet2ce1ee12015-02-04 13:37:44 -08005523static void netdev_adjacent_del_links(struct net_device *dev)
Alexander Y. Fomichev4c754312014-08-25 16:26:45 +04005524{
5525 struct netdev_adjacent *iter;
5526
5527 struct net *net = dev_net(dev);
5528
5529 list_for_each_entry(iter, &dev->adj_list.upper, list) {
5530 if (!net_eq(net,dev_net(iter->dev)))
5531 continue;
5532 netdev_adjacent_sysfs_del(iter->dev, dev->name,
5533 &iter->dev->adj_list.lower);
5534 netdev_adjacent_sysfs_del(dev, iter->dev->name,
5535 &dev->adj_list.upper);
5536 }
5537
5538 list_for_each_entry(iter, &dev->adj_list.lower, list) {
5539 if (!net_eq(net,dev_net(iter->dev)))
5540 continue;
5541 netdev_adjacent_sysfs_del(iter->dev, dev->name,
5542 &iter->dev->adj_list.upper);
5543 netdev_adjacent_sysfs_del(dev, iter->dev->name,
5544 &dev->adj_list.lower);
5545 }
5546}
5547
Veaceslav Falico5bb025f2014-01-14 21:58:51 +01005548void netdev_adjacent_rename_links(struct net_device *dev, char *oldname)
Veaceslav Falico402dae92013-09-25 09:20:09 +02005549{
Veaceslav Falico5bb025f2014-01-14 21:58:51 +01005550 struct netdev_adjacent *iter;
Veaceslav Falico402dae92013-09-25 09:20:09 +02005551
Alexander Y. Fomichev4c754312014-08-25 16:26:45 +04005552 struct net *net = dev_net(dev);
5553
Veaceslav Falico5bb025f2014-01-14 21:58:51 +01005554 list_for_each_entry(iter, &dev->adj_list.upper, list) {
Alexander Y. Fomichev4c754312014-08-25 16:26:45 +04005555 if (!net_eq(net,dev_net(iter->dev)))
5556 continue;
Veaceslav Falico5bb025f2014-01-14 21:58:51 +01005557 netdev_adjacent_sysfs_del(iter->dev, oldname,
5558 &iter->dev->adj_list.lower);
5559 netdev_adjacent_sysfs_add(iter->dev, dev,
5560 &iter->dev->adj_list.lower);
5561 }
Veaceslav Falico402dae92013-09-25 09:20:09 +02005562
Veaceslav Falico5bb025f2014-01-14 21:58:51 +01005563 list_for_each_entry(iter, &dev->adj_list.lower, list) {
Alexander Y. Fomichev4c754312014-08-25 16:26:45 +04005564 if (!net_eq(net,dev_net(iter->dev)))
5565 continue;
Veaceslav Falico5bb025f2014-01-14 21:58:51 +01005566 netdev_adjacent_sysfs_del(iter->dev, oldname,
5567 &iter->dev->adj_list.upper);
5568 netdev_adjacent_sysfs_add(iter->dev, dev,
5569 &iter->dev->adj_list.upper);
5570 }
Veaceslav Falico402dae92013-09-25 09:20:09 +02005571}
Veaceslav Falico402dae92013-09-25 09:20:09 +02005572
5573void *netdev_lower_dev_get_private(struct net_device *dev,
5574 struct net_device *lower_dev)
5575{
5576 struct netdev_adjacent *lower;
5577
5578 if (!lower_dev)
5579 return NULL;
5580 lower = __netdev_find_adj(dev, lower_dev, &dev->adj_list.lower);
5581 if (!lower)
5582 return NULL;
5583
5584 return lower->private;
5585}
5586EXPORT_SYMBOL(netdev_lower_dev_get_private);
5587
Vlad Yasevich4085ebe2014-05-16 17:04:53 -04005588
5589int dev_get_nest_level(struct net_device *dev,
5590 bool (*type_check)(struct net_device *dev))
5591{
5592 struct net_device *lower = NULL;
5593 struct list_head *iter;
5594 int max_nest = -1;
5595 int nest;
5596
5597 ASSERT_RTNL();
5598
5599 netdev_for_each_lower_dev(dev, lower, iter) {
5600 nest = dev_get_nest_level(lower, type_check);
5601 if (max_nest < nest)
5602 max_nest = nest;
5603 }
5604
5605 if (type_check(dev))
5606 max_nest++;
5607
5608 return max_nest;
5609}
5610EXPORT_SYMBOL(dev_get_nest_level);
5611
Patrick McHardyb6c40d62008-10-07 15:26:48 -07005612static void dev_change_rx_flags(struct net_device *dev, int flags)
5613{
Stephen Hemmingerd3147742008-11-19 21:32:24 -08005614 const struct net_device_ops *ops = dev->netdev_ops;
5615
Vlad Yasevichd2615bf2013-11-19 20:47:15 -05005616 if (ops->ndo_change_rx_flags)
Stephen Hemmingerd3147742008-11-19 21:32:24 -08005617 ops->ndo_change_rx_flags(dev, flags);
Patrick McHardyb6c40d62008-10-07 15:26:48 -07005618}
5619
Nicolas Dichtel991fb3f2013-09-25 12:02:45 +02005620static int __dev_set_promiscuity(struct net_device *dev, int inc, bool notify)
Patrick McHardy4417da62007-06-27 01:28:10 -07005621{
Eric Dumazetb536db92011-11-30 21:42:26 +00005622 unsigned int old_flags = dev->flags;
Eric W. Biedermand04a48b2012-05-23 17:01:57 -06005623 kuid_t uid;
5624 kgid_t gid;
Patrick McHardy4417da62007-06-27 01:28:10 -07005625
Patrick McHardy24023452007-07-14 18:51:31 -07005626 ASSERT_RTNL();
5627
Wang Chendad9b332008-06-18 01:48:28 -07005628 dev->flags |= IFF_PROMISC;
5629 dev->promiscuity += inc;
5630 if (dev->promiscuity == 0) {
5631 /*
5632 * Avoid overflow.
5633 * If inc causes overflow, untouch promisc and return error.
5634 */
5635 if (inc < 0)
5636 dev->flags &= ~IFF_PROMISC;
5637 else {
5638 dev->promiscuity -= inc;
Joe Perches7b6cd1c2012-02-01 10:54:43 +00005639 pr_warn("%s: promiscuity touches roof, set promiscuity failed. promiscuity feature of device might be broken.\n",
5640 dev->name);
Wang Chendad9b332008-06-18 01:48:28 -07005641 return -EOVERFLOW;
5642 }
5643 }
Patrick McHardy4417da62007-06-27 01:28:10 -07005644 if (dev->flags != old_flags) {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00005645 pr_info("device %s %s promiscuous mode\n",
5646 dev->name,
5647 dev->flags & IFF_PROMISC ? "entered" : "left");
David Howells8192b0c2008-11-14 10:39:10 +11005648 if (audit_enabled) {
5649 current_uid_gid(&uid, &gid);
Klaus Heinrich Kiwi7759db82008-01-23 22:57:45 -05005650 audit_log(current->audit_context, GFP_ATOMIC,
5651 AUDIT_ANOM_PROMISCUOUS,
5652 "dev=%s prom=%d old_prom=%d auid=%u uid=%u gid=%u ses=%u",
5653 dev->name, (dev->flags & IFF_PROMISC),
5654 (old_flags & IFF_PROMISC),
Eric W. Biedermane1760bd2012-09-10 22:39:43 -07005655 from_kuid(&init_user_ns, audit_get_loginuid(current)),
Eric W. Biedermand04a48b2012-05-23 17:01:57 -06005656 from_kuid(&init_user_ns, uid),
5657 from_kgid(&init_user_ns, gid),
Klaus Heinrich Kiwi7759db82008-01-23 22:57:45 -05005658 audit_get_sessionid(current));
David Howells8192b0c2008-11-14 10:39:10 +11005659 }
Patrick McHardy24023452007-07-14 18:51:31 -07005660
Patrick McHardyb6c40d62008-10-07 15:26:48 -07005661 dev_change_rx_flags(dev, IFF_PROMISC);
Patrick McHardy4417da62007-06-27 01:28:10 -07005662 }
Nicolas Dichtel991fb3f2013-09-25 12:02:45 +02005663 if (notify)
5664 __dev_notify_flags(dev, old_flags, IFF_PROMISC);
Wang Chendad9b332008-06-18 01:48:28 -07005665 return 0;
Patrick McHardy4417da62007-06-27 01:28:10 -07005666}
5667
Linus Torvalds1da177e2005-04-16 15:20:36 -07005668/**
5669 * dev_set_promiscuity - update promiscuity count on a device
5670 * @dev: device
5671 * @inc: modifier
5672 *
Stephen Hemminger3041a062006-05-26 13:25:24 -07005673 * Add or remove promiscuity from a device. While the count in the device
Linus Torvalds1da177e2005-04-16 15:20:36 -07005674 * remains above zero the interface remains promiscuous. Once it hits zero
5675 * the device reverts back to normal filtering operation. A negative inc
5676 * value is used to drop promiscuity on the device.
Wang Chendad9b332008-06-18 01:48:28 -07005677 * Return 0 if successful or a negative errno code on error.
Linus Torvalds1da177e2005-04-16 15:20:36 -07005678 */
Wang Chendad9b332008-06-18 01:48:28 -07005679int dev_set_promiscuity(struct net_device *dev, int inc)
Linus Torvalds1da177e2005-04-16 15:20:36 -07005680{
Eric Dumazetb536db92011-11-30 21:42:26 +00005681 unsigned int old_flags = dev->flags;
Wang Chendad9b332008-06-18 01:48:28 -07005682 int err;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005683
Nicolas Dichtel991fb3f2013-09-25 12:02:45 +02005684 err = __dev_set_promiscuity(dev, inc, true);
Patrick McHardy4b5a6982008-07-06 15:49:08 -07005685 if (err < 0)
Wang Chendad9b332008-06-18 01:48:28 -07005686 return err;
Patrick McHardy4417da62007-06-27 01:28:10 -07005687 if (dev->flags != old_flags)
5688 dev_set_rx_mode(dev);
Wang Chendad9b332008-06-18 01:48:28 -07005689 return err;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005690}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005691EXPORT_SYMBOL(dev_set_promiscuity);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005692
Nicolas Dichtel991fb3f2013-09-25 12:02:45 +02005693static int __dev_set_allmulti(struct net_device *dev, int inc, bool notify)
Linus Torvalds1da177e2005-04-16 15:20:36 -07005694{
Nicolas Dichtel991fb3f2013-09-25 12:02:45 +02005695 unsigned int old_flags = dev->flags, old_gflags = dev->gflags;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005696
Patrick McHardy24023452007-07-14 18:51:31 -07005697 ASSERT_RTNL();
5698
Linus Torvalds1da177e2005-04-16 15:20:36 -07005699 dev->flags |= IFF_ALLMULTI;
Wang Chendad9b332008-06-18 01:48:28 -07005700 dev->allmulti += inc;
5701 if (dev->allmulti == 0) {
5702 /*
5703 * Avoid overflow.
5704 * If inc causes overflow, untouch allmulti and return error.
5705 */
5706 if (inc < 0)
5707 dev->flags &= ~IFF_ALLMULTI;
5708 else {
5709 dev->allmulti -= inc;
Joe Perches7b6cd1c2012-02-01 10:54:43 +00005710 pr_warn("%s: allmulti touches roof, set allmulti failed. allmulti feature of device might be broken.\n",
5711 dev->name);
Wang Chendad9b332008-06-18 01:48:28 -07005712 return -EOVERFLOW;
5713 }
5714 }
Patrick McHardy24023452007-07-14 18:51:31 -07005715 if (dev->flags ^ old_flags) {
Patrick McHardyb6c40d62008-10-07 15:26:48 -07005716 dev_change_rx_flags(dev, IFF_ALLMULTI);
Patrick McHardy4417da62007-06-27 01:28:10 -07005717 dev_set_rx_mode(dev);
Nicolas Dichtel991fb3f2013-09-25 12:02:45 +02005718 if (notify)
5719 __dev_notify_flags(dev, old_flags,
5720 dev->gflags ^ old_gflags);
Patrick McHardy24023452007-07-14 18:51:31 -07005721 }
Wang Chendad9b332008-06-18 01:48:28 -07005722 return 0;
Patrick McHardy4417da62007-06-27 01:28:10 -07005723}
Nicolas Dichtel991fb3f2013-09-25 12:02:45 +02005724
5725/**
5726 * dev_set_allmulti - update allmulti count on a device
5727 * @dev: device
5728 * @inc: modifier
5729 *
5730 * Add or remove reception of all multicast frames to a device. While the
5731 * count in the device remains above zero the interface remains listening
5732 * to all interfaces. Once it hits zero the device reverts back to normal
5733 * filtering operation. A negative @inc value is used to drop the counter
5734 * when releasing a resource needing all multicasts.
5735 * Return 0 if successful or a negative errno code on error.
5736 */
5737
5738int dev_set_allmulti(struct net_device *dev, int inc)
5739{
5740 return __dev_set_allmulti(dev, inc, true);
5741}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005742EXPORT_SYMBOL(dev_set_allmulti);
Patrick McHardy4417da62007-06-27 01:28:10 -07005743
5744/*
5745 * Upload unicast and multicast address lists to device and
5746 * configure RX filtering. When the device doesn't support unicast
Joe Perches53ccaae2007-12-20 14:02:06 -08005747 * filtering it is put in promiscuous mode while unicast addresses
Patrick McHardy4417da62007-06-27 01:28:10 -07005748 * are present.
5749 */
5750void __dev_set_rx_mode(struct net_device *dev)
5751{
Stephen Hemmingerd3147742008-11-19 21:32:24 -08005752 const struct net_device_ops *ops = dev->netdev_ops;
5753
Patrick McHardy4417da62007-06-27 01:28:10 -07005754 /* dev_open will call this function so the list will stay sane. */
5755 if (!(dev->flags&IFF_UP))
5756 return;
5757
5758 if (!netif_device_present(dev))
YOSHIFUJI Hideaki40b77c92007-07-19 10:43:23 +09005759 return;
Patrick McHardy4417da62007-06-27 01:28:10 -07005760
Jiri Pirko01789342011-08-16 06:29:00 +00005761 if (!(dev->priv_flags & IFF_UNICAST_FLT)) {
Patrick McHardy4417da62007-06-27 01:28:10 -07005762 /* Unicast addresses changes may only happen under the rtnl,
5763 * therefore calling __dev_set_promiscuity here is safe.
5764 */
Jiri Pirko32e7bfc2010-01-25 13:36:10 -08005765 if (!netdev_uc_empty(dev) && !dev->uc_promisc) {
Nicolas Dichtel991fb3f2013-09-25 12:02:45 +02005766 __dev_set_promiscuity(dev, 1, false);
Joe Perches2d348d12011-07-25 16:17:35 -07005767 dev->uc_promisc = true;
Jiri Pirko32e7bfc2010-01-25 13:36:10 -08005768 } else if (netdev_uc_empty(dev) && dev->uc_promisc) {
Nicolas Dichtel991fb3f2013-09-25 12:02:45 +02005769 __dev_set_promiscuity(dev, -1, false);
Joe Perches2d348d12011-07-25 16:17:35 -07005770 dev->uc_promisc = false;
Patrick McHardy4417da62007-06-27 01:28:10 -07005771 }
Patrick McHardy4417da62007-06-27 01:28:10 -07005772 }
Jiri Pirko01789342011-08-16 06:29:00 +00005773
5774 if (ops->ndo_set_rx_mode)
5775 ops->ndo_set_rx_mode(dev);
Patrick McHardy4417da62007-06-27 01:28:10 -07005776}
5777
5778void dev_set_rx_mode(struct net_device *dev)
5779{
David S. Millerb9e40852008-07-15 00:15:08 -07005780 netif_addr_lock_bh(dev);
Patrick McHardy4417da62007-06-27 01:28:10 -07005781 __dev_set_rx_mode(dev);
David S. Millerb9e40852008-07-15 00:15:08 -07005782 netif_addr_unlock_bh(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005783}
5784
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07005785/**
5786 * dev_get_flags - get flags reported to userspace
5787 * @dev: device
5788 *
5789 * Get the combination of flag bits exported through APIs to userspace.
5790 */
Eric Dumazet95c96172012-04-15 05:58:06 +00005791unsigned int dev_get_flags(const struct net_device *dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07005792{
Eric Dumazet95c96172012-04-15 05:58:06 +00005793 unsigned int flags;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005794
5795 flags = (dev->flags & ~(IFF_PROMISC |
5796 IFF_ALLMULTI |
Stefan Rompfb00055a2006-03-20 17:09:11 -08005797 IFF_RUNNING |
5798 IFF_LOWER_UP |
5799 IFF_DORMANT)) |
Linus Torvalds1da177e2005-04-16 15:20:36 -07005800 (dev->gflags & (IFF_PROMISC |
5801 IFF_ALLMULTI));
5802
Stefan Rompfb00055a2006-03-20 17:09:11 -08005803 if (netif_running(dev)) {
5804 if (netif_oper_up(dev))
5805 flags |= IFF_RUNNING;
5806 if (netif_carrier_ok(dev))
5807 flags |= IFF_LOWER_UP;
5808 if (netif_dormant(dev))
5809 flags |= IFF_DORMANT;
5810 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07005811
5812 return flags;
5813}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005814EXPORT_SYMBOL(dev_get_flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005815
Patrick McHardybd380812010-02-26 06:34:53 +00005816int __dev_change_flags(struct net_device *dev, unsigned int flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -07005817{
Eric Dumazetb536db92011-11-30 21:42:26 +00005818 unsigned int old_flags = dev->flags;
Patrick McHardybd380812010-02-26 06:34:53 +00005819 int ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005820
Patrick McHardy24023452007-07-14 18:51:31 -07005821 ASSERT_RTNL();
5822
Linus Torvalds1da177e2005-04-16 15:20:36 -07005823 /*
5824 * Set the flags on our device.
5825 */
5826
5827 dev->flags = (flags & (IFF_DEBUG | IFF_NOTRAILERS | IFF_NOARP |
5828 IFF_DYNAMIC | IFF_MULTICAST | IFF_PORTSEL |
5829 IFF_AUTOMEDIA)) |
5830 (dev->flags & (IFF_UP | IFF_VOLATILE | IFF_PROMISC |
5831 IFF_ALLMULTI));
5832
5833 /*
5834 * Load in the correct multicast list now the flags have changed.
5835 */
5836
Patrick McHardyb6c40d62008-10-07 15:26:48 -07005837 if ((old_flags ^ flags) & IFF_MULTICAST)
5838 dev_change_rx_flags(dev, IFF_MULTICAST);
Patrick McHardy24023452007-07-14 18:51:31 -07005839
Patrick McHardy4417da62007-06-27 01:28:10 -07005840 dev_set_rx_mode(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005841
5842 /*
5843 * Have we downed the interface. We handle IFF_UP ourselves
5844 * according to user attempts to set it, rather than blindly
5845 * setting it.
5846 */
5847
5848 ret = 0;
Peter Pan(潘卫平)d215d102014-06-16 21:57:22 +08005849 if ((old_flags ^ flags) & IFF_UP)
Patrick McHardybd380812010-02-26 06:34:53 +00005850 ret = ((old_flags & IFF_UP) ? __dev_close : __dev_open)(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005851
Linus Torvalds1da177e2005-04-16 15:20:36 -07005852 if ((flags ^ dev->gflags) & IFF_PROMISC) {
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005853 int inc = (flags & IFF_PROMISC) ? 1 : -1;
Nicolas Dichtel991fb3f2013-09-25 12:02:45 +02005854 unsigned int old_flags = dev->flags;
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005855
Linus Torvalds1da177e2005-04-16 15:20:36 -07005856 dev->gflags ^= IFF_PROMISC;
Nicolas Dichtel991fb3f2013-09-25 12:02:45 +02005857
5858 if (__dev_set_promiscuity(dev, inc, false) >= 0)
5859 if (dev->flags != old_flags)
5860 dev_set_rx_mode(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005861 }
5862
5863 /* NOTE: order of synchronization of IFF_PROMISC and IFF_ALLMULTI
5864 is important. Some (broken) drivers set IFF_PROMISC, when
5865 IFF_ALLMULTI is requested not asking us and not reporting.
5866 */
5867 if ((flags ^ dev->gflags) & IFF_ALLMULTI) {
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005868 int inc = (flags & IFF_ALLMULTI) ? 1 : -1;
5869
Linus Torvalds1da177e2005-04-16 15:20:36 -07005870 dev->gflags ^= IFF_ALLMULTI;
Nicolas Dichtel991fb3f2013-09-25 12:02:45 +02005871 __dev_set_allmulti(dev, inc, false);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005872 }
5873
Patrick McHardybd380812010-02-26 06:34:53 +00005874 return ret;
5875}
5876
Nicolas Dichtela528c212013-09-25 12:02:44 +02005877void __dev_notify_flags(struct net_device *dev, unsigned int old_flags,
5878 unsigned int gchanges)
Patrick McHardybd380812010-02-26 06:34:53 +00005879{
5880 unsigned int changes = dev->flags ^ old_flags;
5881
Nicolas Dichtela528c212013-09-25 12:02:44 +02005882 if (gchanges)
Alexei Starovoitov7f294052013-10-23 16:02:42 -07005883 rtmsg_ifinfo(RTM_NEWLINK, dev, gchanges, GFP_ATOMIC);
Nicolas Dichtela528c212013-09-25 12:02:44 +02005884
Patrick McHardybd380812010-02-26 06:34:53 +00005885 if (changes & IFF_UP) {
5886 if (dev->flags & IFF_UP)
5887 call_netdevice_notifiers(NETDEV_UP, dev);
5888 else
5889 call_netdevice_notifiers(NETDEV_DOWN, dev);
5890 }
5891
5892 if (dev->flags & IFF_UP &&
Jiri Pirkobe9efd32013-05-28 01:30:22 +00005893 (changes & ~(IFF_UP | IFF_PROMISC | IFF_ALLMULTI | IFF_VOLATILE))) {
5894 struct netdev_notifier_change_info change_info;
5895
5896 change_info.flags_changed = changes;
5897 call_netdevice_notifiers_info(NETDEV_CHANGE, dev,
5898 &change_info.info);
5899 }
Patrick McHardybd380812010-02-26 06:34:53 +00005900}
5901
5902/**
5903 * dev_change_flags - change device settings
5904 * @dev: device
5905 * @flags: device state flags
5906 *
5907 * Change settings on device based state flags. The flags are
5908 * in the userspace exported format.
5909 */
Eric Dumazetb536db92011-11-30 21:42:26 +00005910int dev_change_flags(struct net_device *dev, unsigned int flags)
Patrick McHardybd380812010-02-26 06:34:53 +00005911{
Eric Dumazetb536db92011-11-30 21:42:26 +00005912 int ret;
Nicolas Dichtel991fb3f2013-09-25 12:02:45 +02005913 unsigned int changes, old_flags = dev->flags, old_gflags = dev->gflags;
Patrick McHardybd380812010-02-26 06:34:53 +00005914
5915 ret = __dev_change_flags(dev, flags);
5916 if (ret < 0)
5917 return ret;
5918
Nicolas Dichtel991fb3f2013-09-25 12:02:45 +02005919 changes = (old_flags ^ dev->flags) | (old_gflags ^ dev->gflags);
Nicolas Dichtela528c212013-09-25 12:02:44 +02005920 __dev_notify_flags(dev, old_flags, changes);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005921 return ret;
5922}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005923EXPORT_SYMBOL(dev_change_flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005924
Veaceslav Falico2315dc92014-01-10 16:56:25 +01005925static int __dev_set_mtu(struct net_device *dev, int new_mtu)
5926{
5927 const struct net_device_ops *ops = dev->netdev_ops;
5928
5929 if (ops->ndo_change_mtu)
5930 return ops->ndo_change_mtu(dev, new_mtu);
5931
5932 dev->mtu = new_mtu;
5933 return 0;
5934}
5935
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07005936/**
5937 * dev_set_mtu - Change maximum transfer unit
5938 * @dev: device
5939 * @new_mtu: new transfer unit
5940 *
5941 * Change the maximum transfer size of the network device.
5942 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07005943int dev_set_mtu(struct net_device *dev, int new_mtu)
5944{
Veaceslav Falico2315dc92014-01-10 16:56:25 +01005945 int err, orig_mtu;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005946
5947 if (new_mtu == dev->mtu)
5948 return 0;
5949
5950 /* MTU must be positive. */
5951 if (new_mtu < 0)
5952 return -EINVAL;
5953
5954 if (!netif_device_present(dev))
5955 return -ENODEV;
5956
Veaceslav Falico1d486bf2014-01-16 00:02:18 +01005957 err = call_netdevice_notifiers(NETDEV_PRECHANGEMTU, dev);
5958 err = notifier_to_errno(err);
5959 if (err)
5960 return err;
Stephen Hemmingerd3147742008-11-19 21:32:24 -08005961
Veaceslav Falico2315dc92014-01-10 16:56:25 +01005962 orig_mtu = dev->mtu;
5963 err = __dev_set_mtu(dev, new_mtu);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005964
Veaceslav Falico2315dc92014-01-10 16:56:25 +01005965 if (!err) {
5966 err = call_netdevice_notifiers(NETDEV_CHANGEMTU, dev);
5967 err = notifier_to_errno(err);
5968 if (err) {
5969 /* setting mtu back and notifying everyone again,
5970 * so that they have a chance to revert changes.
5971 */
5972 __dev_set_mtu(dev, orig_mtu);
5973 call_netdevice_notifiers(NETDEV_CHANGEMTU, dev);
5974 }
5975 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07005976 return err;
5977}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005978EXPORT_SYMBOL(dev_set_mtu);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005979
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07005980/**
Vlad Dogarucbda10f2011-01-13 23:38:30 +00005981 * dev_set_group - Change group this device belongs to
5982 * @dev: device
5983 * @new_group: group this device should belong to
5984 */
5985void dev_set_group(struct net_device *dev, int new_group)
5986{
5987 dev->group = new_group;
5988}
5989EXPORT_SYMBOL(dev_set_group);
5990
5991/**
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07005992 * dev_set_mac_address - Change Media Access Control Address
5993 * @dev: device
5994 * @sa: new address
5995 *
5996 * Change the hardware (MAC) address of the device
5997 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07005998int dev_set_mac_address(struct net_device *dev, struct sockaddr *sa)
5999{
Stephen Hemmingerd3147742008-11-19 21:32:24 -08006000 const struct net_device_ops *ops = dev->netdev_ops;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006001 int err;
6002
Stephen Hemmingerd3147742008-11-19 21:32:24 -08006003 if (!ops->ndo_set_mac_address)
Linus Torvalds1da177e2005-04-16 15:20:36 -07006004 return -EOPNOTSUPP;
6005 if (sa->sa_family != dev->type)
6006 return -EINVAL;
6007 if (!netif_device_present(dev))
6008 return -ENODEV;
Stephen Hemmingerd3147742008-11-19 21:32:24 -08006009 err = ops->ndo_set_mac_address(dev, sa);
Jiri Pirkof6521512013-01-01 03:30:14 +00006010 if (err)
6011 return err;
Jiri Pirkofbdeca22013-01-01 03:30:16 +00006012 dev->addr_assign_type = NET_ADDR_SET;
Jiri Pirkof6521512013-01-01 03:30:14 +00006013 call_netdevice_notifiers(NETDEV_CHANGEADDR, dev);
Theodore Ts'o7bf23572012-07-04 21:23:25 -04006014 add_device_randomness(dev->dev_addr, dev->addr_len);
Jiri Pirkof6521512013-01-01 03:30:14 +00006015 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006016}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07006017EXPORT_SYMBOL(dev_set_mac_address);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006018
Jiri Pirko4bf84c32012-12-27 23:49:37 +00006019/**
6020 * dev_change_carrier - Change device carrier
6021 * @dev: device
Randy Dunlap691b3b72013-03-04 12:32:43 +00006022 * @new_carrier: new value
Jiri Pirko4bf84c32012-12-27 23:49:37 +00006023 *
6024 * Change device carrier
6025 */
6026int dev_change_carrier(struct net_device *dev, bool new_carrier)
6027{
6028 const struct net_device_ops *ops = dev->netdev_ops;
6029
6030 if (!ops->ndo_change_carrier)
6031 return -EOPNOTSUPP;
6032 if (!netif_device_present(dev))
6033 return -ENODEV;
6034 return ops->ndo_change_carrier(dev, new_carrier);
6035}
6036EXPORT_SYMBOL(dev_change_carrier);
6037
Linus Torvalds1da177e2005-04-16 15:20:36 -07006038/**
Jiri Pirko66b52b02013-07-29 18:16:49 +02006039 * dev_get_phys_port_id - Get device physical port ID
6040 * @dev: device
6041 * @ppid: port ID
6042 *
6043 * Get device physical port ID
6044 */
6045int dev_get_phys_port_id(struct net_device *dev,
Jiri Pirko02637fc2014-11-28 14:34:16 +01006046 struct netdev_phys_item_id *ppid)
Jiri Pirko66b52b02013-07-29 18:16:49 +02006047{
6048 const struct net_device_ops *ops = dev->netdev_ops;
6049
6050 if (!ops->ndo_get_phys_port_id)
6051 return -EOPNOTSUPP;
6052 return ops->ndo_get_phys_port_id(dev, ppid);
6053}
6054EXPORT_SYMBOL(dev_get_phys_port_id);
6055
6056/**
David Aherndb24a902015-03-17 20:23:15 -06006057 * dev_get_phys_port_name - Get device physical port name
6058 * @dev: device
6059 * @name: port name
6060 *
6061 * Get device physical port name
6062 */
6063int dev_get_phys_port_name(struct net_device *dev,
6064 char *name, size_t len)
6065{
6066 const struct net_device_ops *ops = dev->netdev_ops;
6067
6068 if (!ops->ndo_get_phys_port_name)
6069 return -EOPNOTSUPP;
6070 return ops->ndo_get_phys_port_name(dev, name, len);
6071}
6072EXPORT_SYMBOL(dev_get_phys_port_name);
6073
6074/**
Linus Torvalds1da177e2005-04-16 15:20:36 -07006075 * dev_new_index - allocate an ifindex
Randy Dunlapc4ea43c2007-10-12 21:17:49 -07006076 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -07006077 *
6078 * Returns a suitable unique value for a new device interface
6079 * number. The caller must hold the rtnl semaphore or the
6080 * dev_base_lock to be sure it remains unique.
6081 */
Eric W. Biederman881d9662007-09-17 11:56:21 -07006082static int dev_new_index(struct net *net)
Linus Torvalds1da177e2005-04-16 15:20:36 -07006083{
Pavel Emelyanovaa79e662012-08-08 21:53:19 +00006084 int ifindex = net->ifindex;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006085 for (;;) {
6086 if (++ifindex <= 0)
6087 ifindex = 1;
Eric W. Biederman881d9662007-09-17 11:56:21 -07006088 if (!__dev_get_by_index(net, ifindex))
Pavel Emelyanovaa79e662012-08-08 21:53:19 +00006089 return net->ifindex = ifindex;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006090 }
6091}
6092
Linus Torvalds1da177e2005-04-16 15:20:36 -07006093/* Delayed registration/unregisteration */
Denis Cheng3b5b34f2007-12-07 00:49:17 -08006094static LIST_HEAD(net_todo_list);
Cong Wang200b9162014-05-12 15:11:20 -07006095DECLARE_WAIT_QUEUE_HEAD(netdev_unregistering_wq);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006096
Stephen Hemminger6f05f622007-03-08 20:46:03 -08006097static void net_set_todo(struct net_device *dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07006098{
Linus Torvalds1da177e2005-04-16 15:20:36 -07006099 list_add_tail(&dev->todo_list, &net_todo_list);
Eric W. Biederman50624c92013-09-23 21:19:49 -07006100 dev_net(dev)->dev_unreg_count++;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006101}
6102
Eric Dumazet9b5e3832009-10-27 07:04:19 +00006103static void rollback_registered_many(struct list_head *head)
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07006104{
Krishna Kumare93737b2009-12-08 22:26:02 +00006105 struct net_device *dev, *tmp;
Eric W. Biederman5cde2822013-10-05 19:26:05 -07006106 LIST_HEAD(close_head);
Eric Dumazet9b5e3832009-10-27 07:04:19 +00006107
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07006108 BUG_ON(dev_boot_phase);
6109 ASSERT_RTNL();
6110
Krishna Kumare93737b2009-12-08 22:26:02 +00006111 list_for_each_entry_safe(dev, tmp, head, unreg_list) {
Eric Dumazet9b5e3832009-10-27 07:04:19 +00006112 /* Some devices call without registering
Krishna Kumare93737b2009-12-08 22:26:02 +00006113 * for initialization unwind. Remove those
6114 * devices and proceed with the remaining.
Eric Dumazet9b5e3832009-10-27 07:04:19 +00006115 */
6116 if (dev->reg_state == NETREG_UNINITIALIZED) {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00006117 pr_debug("unregister_netdevice: device %s/%p never was registered\n",
6118 dev->name, dev);
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07006119
Eric Dumazet9b5e3832009-10-27 07:04:19 +00006120 WARN_ON(1);
Krishna Kumare93737b2009-12-08 22:26:02 +00006121 list_del(&dev->unreg_list);
6122 continue;
Eric Dumazet9b5e3832009-10-27 07:04:19 +00006123 }
Eric Dumazet449f4542011-05-19 12:24:16 +00006124 dev->dismantle = true;
Eric Dumazet9b5e3832009-10-27 07:04:19 +00006125 BUG_ON(dev->reg_state != NETREG_REGISTERED);
Octavian Purdila44345722010-12-13 12:44:07 +00006126 }
Eric Dumazet9b5e3832009-10-27 07:04:19 +00006127
Octavian Purdila44345722010-12-13 12:44:07 +00006128 /* If device is running, close it first. */
Eric W. Biederman5cde2822013-10-05 19:26:05 -07006129 list_for_each_entry(dev, head, unreg_list)
6130 list_add_tail(&dev->close_list, &close_head);
David S. Miller99c4a262015-03-18 22:52:33 -04006131 dev_close_many(&close_head, true);
Eric Dumazet9b5e3832009-10-27 07:04:19 +00006132
Octavian Purdila44345722010-12-13 12:44:07 +00006133 list_for_each_entry(dev, head, unreg_list) {
Eric Dumazet9b5e3832009-10-27 07:04:19 +00006134 /* And unlink it from device chain. */
6135 unlist_netdevice(dev);
6136
6137 dev->reg_state = NETREG_UNREGISTERING;
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07006138 }
6139
Eric Dumazet9b5e3832009-10-27 07:04:19 +00006140 synchronize_net();
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07006141
Eric Dumazet9b5e3832009-10-27 07:04:19 +00006142 list_for_each_entry(dev, head, unreg_list) {
Mahesh Bandewar395eea62014-12-03 13:46:24 -08006143 struct sk_buff *skb = NULL;
6144
Eric Dumazet9b5e3832009-10-27 07:04:19 +00006145 /* Shutdown queueing discipline. */
6146 dev_shutdown(dev);
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07006147
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07006148
Eric Dumazet9b5e3832009-10-27 07:04:19 +00006149 /* Notify protocols, that we are about to destroy
6150 this device. They should clean all the things.
6151 */
6152 call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
6153
Mahesh Bandewar395eea62014-12-03 13:46:24 -08006154 if (!dev->rtnl_link_ops ||
6155 dev->rtnl_link_state == RTNL_LINK_INITIALIZED)
6156 skb = rtmsg_ifinfo_build_skb(RTM_DELLINK, dev, ~0U,
6157 GFP_KERNEL);
6158
Eric Dumazet9b5e3832009-10-27 07:04:19 +00006159 /*
6160 * Flush the unicast and multicast chains
6161 */
Jiri Pirkoa748ee22010-04-01 21:22:09 +00006162 dev_uc_flush(dev);
Jiri Pirko22bedad32010-04-01 21:22:57 +00006163 dev_mc_flush(dev);
Eric Dumazet9b5e3832009-10-27 07:04:19 +00006164
6165 if (dev->netdev_ops->ndo_uninit)
6166 dev->netdev_ops->ndo_uninit(dev);
6167
Mahesh Bandewar395eea62014-12-03 13:46:24 -08006168 if (skb)
6169 rtmsg_ifinfo_send(skb, dev, GFP_KERNEL);
Roopa Prabhu56bfa7e2014-05-01 11:40:30 -07006170
Jiri Pirko9ff162a2013-01-03 22:48:49 +00006171 /* Notifier chain MUST detach us all upper devices. */
6172 WARN_ON(netdev_has_any_upper_dev(dev));
Eric Dumazet9b5e3832009-10-27 07:04:19 +00006173
6174 /* Remove entries from kobject tree */
6175 netdev_unregister_kobject(dev);
Alexander Duyck024e9672013-01-10 08:57:46 +00006176#ifdef CONFIG_XPS
6177 /* Remove XPS queueing entries */
6178 netif_reset_xps_queues_gt(dev, 0);
6179#endif
Eric Dumazet9b5e3832009-10-27 07:04:19 +00006180 }
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07006181
Eric W. Biederman850a5452011-10-13 22:25:23 +00006182 synchronize_net();
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07006183
Eric W. Biedermana5ee1552009-11-29 15:45:58 +00006184 list_for_each_entry(dev, head, unreg_list)
Eric Dumazet9b5e3832009-10-27 07:04:19 +00006185 dev_put(dev);
6186}
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07006187
Eric Dumazet9b5e3832009-10-27 07:04:19 +00006188static void rollback_registered(struct net_device *dev)
6189{
6190 LIST_HEAD(single);
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07006191
Eric Dumazet9b5e3832009-10-27 07:04:19 +00006192 list_add(&dev->unreg_list, &single);
6193 rollback_registered_many(&single);
Eric Dumazetceaaec92011-02-17 22:59:19 +00006194 list_del(&single);
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07006195}
6196
Michał Mirosławc8f44af2011-11-15 15:29:55 +00006197static netdev_features_t netdev_fix_features(struct net_device *dev,
6198 netdev_features_t features)
Herbert Xub63365a2008-10-23 01:11:29 -07006199{
Michał Mirosław57422dc2011-01-22 12:14:12 +00006200 /* Fix illegal checksum combinations */
6201 if ((features & NETIF_F_HW_CSUM) &&
6202 (features & (NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM))) {
Michał Mirosław6f404e42011-05-16 15:14:21 -04006203 netdev_warn(dev, "mixed HW and IP checksum settings.\n");
Michał Mirosław57422dc2011-01-22 12:14:12 +00006204 features &= ~(NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM);
6205 }
6206
Herbert Xub63365a2008-10-23 01:11:29 -07006207 /* TSO requires that SG is present as well. */
Ben Hutchingsea2d3682011-04-12 14:38:37 +00006208 if ((features & NETIF_F_ALL_TSO) && !(features & NETIF_F_SG)) {
Michał Mirosław6f404e42011-05-16 15:14:21 -04006209 netdev_dbg(dev, "Dropping TSO features since no SG feature.\n");
Ben Hutchingsea2d3682011-04-12 14:38:37 +00006210 features &= ~NETIF_F_ALL_TSO;
Herbert Xub63365a2008-10-23 01:11:29 -07006211 }
6212
Pravin B Shelarec5f0612013-03-07 09:28:01 +00006213 if ((features & NETIF_F_TSO) && !(features & NETIF_F_HW_CSUM) &&
6214 !(features & NETIF_F_IP_CSUM)) {
6215 netdev_dbg(dev, "Dropping TSO features since no CSUM feature.\n");
6216 features &= ~NETIF_F_TSO;
6217 features &= ~NETIF_F_TSO_ECN;
6218 }
6219
6220 if ((features & NETIF_F_TSO6) && !(features & NETIF_F_HW_CSUM) &&
6221 !(features & NETIF_F_IPV6_CSUM)) {
6222 netdev_dbg(dev, "Dropping TSO6 features since no CSUM feature.\n");
6223 features &= ~NETIF_F_TSO6;
6224 }
6225
Ben Hutchings31d8b9e2011-04-12 14:47:15 +00006226 /* TSO ECN requires that TSO is present as well. */
6227 if ((features & NETIF_F_ALL_TSO) == NETIF_F_TSO_ECN)
6228 features &= ~NETIF_F_TSO_ECN;
6229
Michał Mirosław212b5732011-02-15 16:59:16 +00006230 /* Software GSO depends on SG. */
6231 if ((features & NETIF_F_GSO) && !(features & NETIF_F_SG)) {
Michał Mirosław6f404e42011-05-16 15:14:21 -04006232 netdev_dbg(dev, "Dropping NETIF_F_GSO since no SG feature.\n");
Michał Mirosław212b5732011-02-15 16:59:16 +00006233 features &= ~NETIF_F_GSO;
6234 }
6235
Michał Mirosławacd11302011-01-24 15:45:15 -08006236 /* UFO needs SG and checksumming */
Herbert Xub63365a2008-10-23 01:11:29 -07006237 if (features & NETIF_F_UFO) {
Michał Mirosław79032642010-11-30 06:38:00 +00006238 /* maybe split UFO into V4 and V6? */
6239 if (!((features & NETIF_F_GEN_CSUM) ||
6240 (features & (NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM))
6241 == (NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM))) {
Michał Mirosław6f404e42011-05-16 15:14:21 -04006242 netdev_dbg(dev,
Michał Mirosławacd11302011-01-24 15:45:15 -08006243 "Dropping NETIF_F_UFO since no checksum offload features.\n");
Herbert Xub63365a2008-10-23 01:11:29 -07006244 features &= ~NETIF_F_UFO;
6245 }
6246
6247 if (!(features & NETIF_F_SG)) {
Michał Mirosław6f404e42011-05-16 15:14:21 -04006248 netdev_dbg(dev,
Michał Mirosławacd11302011-01-24 15:45:15 -08006249 "Dropping NETIF_F_UFO since no NETIF_F_SG feature.\n");
Herbert Xub63365a2008-10-23 01:11:29 -07006250 features &= ~NETIF_F_UFO;
6251 }
6252 }
6253
Jiri Pirkod0290212014-04-02 23:09:31 +02006254#ifdef CONFIG_NET_RX_BUSY_POLL
6255 if (dev->netdev_ops->ndo_busy_poll)
6256 features |= NETIF_F_BUSY_POLL;
6257 else
6258#endif
6259 features &= ~NETIF_F_BUSY_POLL;
6260
Herbert Xub63365a2008-10-23 01:11:29 -07006261 return features;
6262}
Herbert Xub63365a2008-10-23 01:11:29 -07006263
Michał Mirosław6cb6a272011-04-02 22:48:47 -07006264int __netdev_update_features(struct net_device *dev)
Michał Mirosław5455c692011-02-15 16:59:17 +00006265{
Michał Mirosławc8f44af2011-11-15 15:29:55 +00006266 netdev_features_t features;
Michał Mirosław5455c692011-02-15 16:59:17 +00006267 int err = 0;
6268
Michał Mirosław87267482011-04-12 09:56:38 +00006269 ASSERT_RTNL();
6270
Michał Mirosław5455c692011-02-15 16:59:17 +00006271 features = netdev_get_wanted_features(dev);
6272
6273 if (dev->netdev_ops->ndo_fix_features)
6274 features = dev->netdev_ops->ndo_fix_features(dev, features);
6275
6276 /* driver might be less strict about feature dependencies */
6277 features = netdev_fix_features(dev, features);
6278
6279 if (dev->features == features)
Michał Mirosław6cb6a272011-04-02 22:48:47 -07006280 return 0;
Michał Mirosław5455c692011-02-15 16:59:17 +00006281
Michał Mirosławc8f44af2011-11-15 15:29:55 +00006282 netdev_dbg(dev, "Features changed: %pNF -> %pNF\n",
6283 &dev->features, &features);
Michał Mirosław5455c692011-02-15 16:59:17 +00006284
6285 if (dev->netdev_ops->ndo_set_features)
6286 err = dev->netdev_ops->ndo_set_features(dev, features);
6287
Michał Mirosław6cb6a272011-04-02 22:48:47 -07006288 if (unlikely(err < 0)) {
Michał Mirosław5455c692011-02-15 16:59:17 +00006289 netdev_err(dev,
Michał Mirosławc8f44af2011-11-15 15:29:55 +00006290 "set_features() failed (%d); wanted %pNF, left %pNF\n",
6291 err, &features, &dev->features);
Michał Mirosław6cb6a272011-04-02 22:48:47 -07006292 return -1;
6293 }
6294
6295 if (!err)
6296 dev->features = features;
6297
6298 return 1;
6299}
6300
Michał Mirosławafe12cc2011-05-07 03:22:17 +00006301/**
6302 * netdev_update_features - recalculate device features
6303 * @dev: the device to check
6304 *
6305 * Recalculate dev->features set and send notifications if it
6306 * has changed. Should be called after driver or hardware dependent
6307 * conditions might have changed that influence the features.
6308 */
Michał Mirosław6cb6a272011-04-02 22:48:47 -07006309void netdev_update_features(struct net_device *dev)
6310{
6311 if (__netdev_update_features(dev))
6312 netdev_features_change(dev);
Michał Mirosław5455c692011-02-15 16:59:17 +00006313}
6314EXPORT_SYMBOL(netdev_update_features);
6315
Linus Torvalds1da177e2005-04-16 15:20:36 -07006316/**
Michał Mirosławafe12cc2011-05-07 03:22:17 +00006317 * netdev_change_features - recalculate device features
6318 * @dev: the device to check
6319 *
6320 * Recalculate dev->features set and send notifications even
6321 * if they have not changed. Should be called instead of
6322 * netdev_update_features() if also dev->vlan_features might
6323 * have changed to allow the changes to be propagated to stacked
6324 * VLAN devices.
6325 */
6326void netdev_change_features(struct net_device *dev)
6327{
6328 __netdev_update_features(dev);
6329 netdev_features_change(dev);
6330}
6331EXPORT_SYMBOL(netdev_change_features);
6332
6333/**
Patrick Mullaneyfc4a7482009-12-03 15:59:22 -08006334 * netif_stacked_transfer_operstate - transfer operstate
6335 * @rootdev: the root or lower level device to transfer state from
6336 * @dev: the device to transfer operstate to
6337 *
6338 * Transfer operational state from root to device. This is normally
6339 * called when a stacking relationship exists between the root
6340 * device and the device(a leaf device).
6341 */
6342void netif_stacked_transfer_operstate(const struct net_device *rootdev,
6343 struct net_device *dev)
6344{
6345 if (rootdev->operstate == IF_OPER_DORMANT)
6346 netif_dormant_on(dev);
6347 else
6348 netif_dormant_off(dev);
6349
6350 if (netif_carrier_ok(rootdev)) {
6351 if (!netif_carrier_ok(dev))
6352 netif_carrier_on(dev);
6353 } else {
6354 if (netif_carrier_ok(dev))
6355 netif_carrier_off(dev);
6356 }
6357}
6358EXPORT_SYMBOL(netif_stacked_transfer_operstate);
6359
Michael Daltona953be52014-01-16 22:23:28 -08006360#ifdef CONFIG_SYSFS
Eric Dumazet1b4bf462010-09-23 17:26:35 +00006361static int netif_alloc_rx_queues(struct net_device *dev)
6362{
Eric Dumazet1b4bf462010-09-23 17:26:35 +00006363 unsigned int i, count = dev->num_rx_queues;
Tom Herbertbd25fa72010-10-18 18:00:16 +00006364 struct netdev_rx_queue *rx;
Pankaj Gupta10595902015-01-12 11:41:28 +05306365 size_t sz = count * sizeof(*rx);
Eric Dumazet1b4bf462010-09-23 17:26:35 +00006366
Tom Herbertbd25fa72010-10-18 18:00:16 +00006367 BUG_ON(count < 1);
Eric Dumazet1b4bf462010-09-23 17:26:35 +00006368
Pankaj Gupta10595902015-01-12 11:41:28 +05306369 rx = kzalloc(sz, GFP_KERNEL | __GFP_NOWARN | __GFP_REPEAT);
6370 if (!rx) {
6371 rx = vzalloc(sz);
6372 if (!rx)
6373 return -ENOMEM;
6374 }
Tom Herbertbd25fa72010-10-18 18:00:16 +00006375 dev->_rx = rx;
6376
Tom Herbertbd25fa72010-10-18 18:00:16 +00006377 for (i = 0; i < count; i++)
Tom Herbertfe822242010-11-09 10:47:38 +00006378 rx[i].dev = dev;
Eric Dumazet1b4bf462010-09-23 17:26:35 +00006379 return 0;
6380}
Tom Herbertbf264142010-11-26 08:36:09 +00006381#endif
Eric Dumazet1b4bf462010-09-23 17:26:35 +00006382
Changli Gaoaa942102010-12-04 02:31:41 +00006383static void netdev_init_one_queue(struct net_device *dev,
6384 struct netdev_queue *queue, void *_unused)
6385{
6386 /* Initialize queue lock */
6387 spin_lock_init(&queue->_xmit_lock);
6388 netdev_set_xmit_lockdep_class(&queue->_xmit_lock, dev->type);
6389 queue->xmit_lock_owner = -1;
Changli Gaob236da62010-12-14 03:09:15 +00006390 netdev_queue_numa_node_write(queue, NUMA_NO_NODE);
Changli Gaoaa942102010-12-04 02:31:41 +00006391 queue->dev = dev;
Tom Herbert114cf582011-11-28 16:33:09 +00006392#ifdef CONFIG_BQL
6393 dql_init(&queue->dql, HZ);
6394#endif
Changli Gaoaa942102010-12-04 02:31:41 +00006395}
6396
Eric Dumazet60877a32013-06-20 01:15:51 -07006397static void netif_free_tx_queues(struct net_device *dev)
6398{
WANG Cong4cb28972014-06-02 15:55:22 -07006399 kvfree(dev->_tx);
Eric Dumazet60877a32013-06-20 01:15:51 -07006400}
6401
Tom Herberte6484932010-10-18 18:04:39 +00006402static int netif_alloc_netdev_queues(struct net_device *dev)
6403{
6404 unsigned int count = dev->num_tx_queues;
6405 struct netdev_queue *tx;
Eric Dumazet60877a32013-06-20 01:15:51 -07006406 size_t sz = count * sizeof(*tx);
Tom Herberte6484932010-10-18 18:04:39 +00006407
Eric Dumazetd3397272015-07-06 17:13:26 +02006408 if (count < 1 || count > 0xffff)
6409 return -EINVAL;
Tom Herberte6484932010-10-18 18:04:39 +00006410
Eric Dumazet60877a32013-06-20 01:15:51 -07006411 tx = kzalloc(sz, GFP_KERNEL | __GFP_NOWARN | __GFP_REPEAT);
6412 if (!tx) {
6413 tx = vzalloc(sz);
6414 if (!tx)
6415 return -ENOMEM;
6416 }
Tom Herberte6484932010-10-18 18:04:39 +00006417 dev->_tx = tx;
Tom Herbert1d24eb42010-11-21 13:17:27 +00006418
Tom Herberte6484932010-10-18 18:04:39 +00006419 netdev_for_each_tx_queue(dev, netdev_init_one_queue, NULL);
6420 spin_lock_init(&dev->tx_global_lock);
Changli Gaoaa942102010-12-04 02:31:41 +00006421
6422 return 0;
Tom Herberte6484932010-10-18 18:04:39 +00006423}
6424
Denys Vlasenkoa2029242015-05-11 21:17:53 +02006425void netif_tx_stop_all_queues(struct net_device *dev)
6426{
6427 unsigned int i;
6428
6429 for (i = 0; i < dev->num_tx_queues; i++) {
6430 struct netdev_queue *txq = netdev_get_tx_queue(dev, i);
6431 netif_tx_stop_queue(txq);
6432 }
6433}
6434EXPORT_SYMBOL(netif_tx_stop_all_queues);
6435
Patrick Mullaneyfc4a7482009-12-03 15:59:22 -08006436/**
Linus Torvalds1da177e2005-04-16 15:20:36 -07006437 * register_netdevice - register a network device
6438 * @dev: device to register
6439 *
6440 * Take a completed network device structure and add it to the kernel
6441 * interfaces. A %NETDEV_REGISTER message is sent to the netdev notifier
6442 * chain. 0 is returned on success. A negative errno code is returned
6443 * on a failure to set up the device, or if the name is a duplicate.
6444 *
6445 * Callers must hold the rtnl semaphore. You may want
6446 * register_netdev() instead of this.
6447 *
6448 * BUGS:
6449 * The locking appears insufficient to guarantee two parallel registers
6450 * will not get the same name.
6451 */
6452
6453int register_netdevice(struct net_device *dev)
6454{
Linus Torvalds1da177e2005-04-16 15:20:36 -07006455 int ret;
Stephen Hemmingerd3147742008-11-19 21:32:24 -08006456 struct net *net = dev_net(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006457
6458 BUG_ON(dev_boot_phase);
6459 ASSERT_RTNL();
6460
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07006461 might_sleep();
6462
Linus Torvalds1da177e2005-04-16 15:20:36 -07006463 /* When net_device's are persistent, this will be fatal. */
6464 BUG_ON(dev->reg_state != NETREG_UNINITIALIZED);
Stephen Hemmingerd3147742008-11-19 21:32:24 -08006465 BUG_ON(!net);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006466
David S. Millerf1f28aa2008-07-15 00:08:33 -07006467 spin_lock_init(&dev->addr_list_lock);
David S. Millercf508b12008-07-22 14:16:42 -07006468 netdev_set_addr_lockdep_class(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006469
Gao feng828de4f2012-09-13 20:58:27 +00006470 ret = dev_get_valid_name(net, dev, dev->name);
Peter Pan(潘卫平)0696c3a2011-05-12 15:46:56 +00006471 if (ret < 0)
6472 goto out;
6473
Linus Torvalds1da177e2005-04-16 15:20:36 -07006474 /* Init, if this function is available */
Stephen Hemmingerd3147742008-11-19 21:32:24 -08006475 if (dev->netdev_ops->ndo_init) {
6476 ret = dev->netdev_ops->ndo_init(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006477 if (ret) {
6478 if (ret > 0)
6479 ret = -EIO;
Adrian Bunk90833aa2006-11-13 16:02:22 -08006480 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006481 }
6482 }
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09006483
Patrick McHardyf6469682013-04-19 02:04:27 +00006484 if (((dev->hw_features | dev->features) &
6485 NETIF_F_HW_VLAN_CTAG_FILTER) &&
Michał Mirosławd2ed2732013-01-29 15:14:16 +00006486 (!dev->netdev_ops->ndo_vlan_rx_add_vid ||
6487 !dev->netdev_ops->ndo_vlan_rx_kill_vid)) {
6488 netdev_WARN(dev, "Buggy VLAN acceleration in driver!\n");
6489 ret = -EINVAL;
6490 goto err_uninit;
6491 }
6492
Pavel Emelyanov9c7dafb2012-08-08 21:52:46 +00006493 ret = -EBUSY;
6494 if (!dev->ifindex)
6495 dev->ifindex = dev_new_index(net);
6496 else if (__dev_get_by_index(net, dev->ifindex))
6497 goto err_uninit;
6498
Michał Mirosław5455c692011-02-15 16:59:17 +00006499 /* Transfer changeable features to wanted_features and enable
6500 * software offloads (GSO and GRO).
6501 */
6502 dev->hw_features |= NETIF_F_SOFT_FEATURES;
Michał Mirosław14d12322011-02-22 16:52:28 +00006503 dev->features |= NETIF_F_SOFT_FEATURES;
6504 dev->wanted_features = dev->features & dev->hw_features;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006505
Michał Mirosław34324dc2011-11-15 15:29:55 +00006506 if (!(dev->flags & IFF_LOOPBACK)) {
6507 dev->hw_features |= NETIF_F_NOCACHE_COPY;
Tom Herbertc6e1a0d2011-04-04 22:30:30 -07006508 }
6509
Michał Mirosław1180e7d2011-07-14 14:41:11 -07006510 /* Make NETIF_F_HIGHDMA inheritable to VLAN devices.
Brandon Philips16c3ea72010-09-15 09:24:24 +00006511 */
Michał Mirosław1180e7d2011-07-14 14:41:11 -07006512 dev->vlan_features |= NETIF_F_HIGHDMA;
Brandon Philips16c3ea72010-09-15 09:24:24 +00006513
Pravin B Shelaree579672013-03-07 09:28:08 +00006514 /* Make NETIF_F_SG inheritable to tunnel devices.
6515 */
6516 dev->hw_enc_features |= NETIF_F_SG;
6517
Simon Horman0d89d202013-05-23 21:02:52 +00006518 /* Make NETIF_F_SG inheritable to MPLS.
6519 */
6520 dev->mpls_features |= NETIF_F_SG;
6521
Johannes Berg7ffbe3f2009-10-02 05:15:27 +00006522 ret = call_netdevice_notifiers(NETDEV_POST_INIT, dev);
6523 ret = notifier_to_errno(ret);
6524 if (ret)
6525 goto err_uninit;
6526
Eric W. Biederman8b41d182007-09-26 22:02:53 -07006527 ret = netdev_register_kobject(dev);
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07006528 if (ret)
Herbert Xu7ce1b0e2007-07-30 16:29:40 -07006529 goto err_uninit;
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07006530 dev->reg_state = NETREG_REGISTERED;
6531
Michał Mirosław6cb6a272011-04-02 22:48:47 -07006532 __netdev_update_features(dev);
Michał Mirosław8e9b59b2011-02-22 16:52:28 +00006533
Linus Torvalds1da177e2005-04-16 15:20:36 -07006534 /*
6535 * Default initial state at registry is that the
6536 * device is present.
6537 */
6538
6539 set_bit(__LINK_STATE_PRESENT, &dev->state);
6540
Ben Hutchings8f4cccb2012-08-20 22:16:51 +01006541 linkwatch_init_dev(dev);
6542
Linus Torvalds1da177e2005-04-16 15:20:36 -07006543 dev_init_scheduler(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006544 dev_hold(dev);
Eric W. Biedermance286d32007-09-12 13:53:49 +02006545 list_netdevice(dev);
Theodore Ts'o7bf23572012-07-04 21:23:25 -04006546 add_device_randomness(dev->dev_addr, dev->addr_len);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006547
Jiri Pirko948b3372013-01-08 01:38:25 +00006548 /* If the device has permanent device address, driver should
6549 * set dev_addr and also addr_assign_type should be set to
6550 * NET_ADDR_PERM (default value).
6551 */
6552 if (dev->addr_assign_type == NET_ADDR_PERM)
6553 memcpy(dev->perm_addr, dev->dev_addr, dev->addr_len);
6554
Linus Torvalds1da177e2005-04-16 15:20:36 -07006555 /* Notify protocols, that a new device appeared. */
Pavel Emelyanov056925a2007-09-16 15:42:43 -07006556 ret = call_netdevice_notifiers(NETDEV_REGISTER, dev);
Herbert Xufcc5a032007-07-30 17:03:38 -07006557 ret = notifier_to_errno(ret);
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07006558 if (ret) {
6559 rollback_registered(dev);
6560 dev->reg_state = NETREG_UNREGISTERED;
6561 }
Eric W. Biedermand90a9092009-12-12 22:11:15 +00006562 /*
6563 * Prevent userspace races by waiting until the network
6564 * device is fully setup before sending notifications.
6565 */
Patrick McHardya2835762010-02-26 06:34:51 +00006566 if (!dev->rtnl_link_ops ||
6567 dev->rtnl_link_state == RTNL_LINK_INITIALIZED)
Alexei Starovoitov7f294052013-10-23 16:02:42 -07006568 rtmsg_ifinfo(RTM_NEWLINK, dev, ~0U, GFP_KERNEL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006569
6570out:
6571 return ret;
Herbert Xu7ce1b0e2007-07-30 16:29:40 -07006572
6573err_uninit:
Stephen Hemmingerd3147742008-11-19 21:32:24 -08006574 if (dev->netdev_ops->ndo_uninit)
6575 dev->netdev_ops->ndo_uninit(dev);
Herbert Xu7ce1b0e2007-07-30 16:29:40 -07006576 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006577}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07006578EXPORT_SYMBOL(register_netdevice);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006579
6580/**
Benjamin Herrenschmidt937f1ba2009-01-14 21:05:05 -08006581 * init_dummy_netdev - init a dummy network device for NAPI
6582 * @dev: device to init
6583 *
6584 * This takes a network device structure and initialize the minimum
6585 * amount of fields so it can be used to schedule NAPI polls without
6586 * registering a full blown interface. This is to be used by drivers
6587 * that need to tie several hardware interfaces to a single NAPI
6588 * poll scheduler due to HW limitations.
6589 */
6590int init_dummy_netdev(struct net_device *dev)
6591{
6592 /* Clear everything. Note we don't initialize spinlocks
6593 * are they aren't supposed to be taken by any of the
6594 * NAPI code and this dummy netdev is supposed to be
6595 * only ever used for NAPI polls
6596 */
6597 memset(dev, 0, sizeof(struct net_device));
6598
6599 /* make sure we BUG if trying to hit standard
6600 * register/unregister code path
6601 */
6602 dev->reg_state = NETREG_DUMMY;
6603
Benjamin Herrenschmidt937f1ba2009-01-14 21:05:05 -08006604 /* NAPI wants this */
6605 INIT_LIST_HEAD(&dev->napi_list);
6606
6607 /* a dummy interface is started by default */
6608 set_bit(__LINK_STATE_PRESENT, &dev->state);
6609 set_bit(__LINK_STATE_START, &dev->state);
6610
Eric Dumazet29b44332010-10-11 10:22:12 +00006611 /* Note : We dont allocate pcpu_refcnt for dummy devices,
6612 * because users of this 'device' dont need to change
6613 * its refcount.
6614 */
6615
Benjamin Herrenschmidt937f1ba2009-01-14 21:05:05 -08006616 return 0;
6617}
6618EXPORT_SYMBOL_GPL(init_dummy_netdev);
6619
6620
6621/**
Linus Torvalds1da177e2005-04-16 15:20:36 -07006622 * register_netdev - register a network device
6623 * @dev: device to register
6624 *
6625 * Take a completed network device structure and add it to the kernel
6626 * interfaces. A %NETDEV_REGISTER message is sent to the netdev notifier
6627 * chain. 0 is returned on success. A negative errno code is returned
6628 * on a failure to set up the device, or if the name is a duplicate.
6629 *
Borislav Petkov38b4da32007-04-20 22:14:10 -07006630 * This is a wrapper around register_netdevice that takes the rtnl semaphore
Linus Torvalds1da177e2005-04-16 15:20:36 -07006631 * and expands the device name if you passed a format string to
6632 * alloc_netdev.
6633 */
6634int register_netdev(struct net_device *dev)
6635{
6636 int err;
6637
6638 rtnl_lock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07006639 err = register_netdevice(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006640 rtnl_unlock();
6641 return err;
6642}
6643EXPORT_SYMBOL(register_netdev);
6644
Eric Dumazet29b44332010-10-11 10:22:12 +00006645int netdev_refcnt_read(const struct net_device *dev)
6646{
6647 int i, refcnt = 0;
6648
6649 for_each_possible_cpu(i)
6650 refcnt += *per_cpu_ptr(dev->pcpu_refcnt, i);
6651 return refcnt;
6652}
6653EXPORT_SYMBOL(netdev_refcnt_read);
6654
Ben Hutchings2c530402012-07-10 10:55:09 +00006655/**
Linus Torvalds1da177e2005-04-16 15:20:36 -07006656 * netdev_wait_allrefs - wait until all references are gone.
Randy Dunlap3de7a372012-08-18 14:36:44 +00006657 * @dev: target net_device
Linus Torvalds1da177e2005-04-16 15:20:36 -07006658 *
6659 * This is called when unregistering network devices.
6660 *
6661 * Any protocol or device that holds a reference should register
6662 * for netdevice notification, and cleanup and put back the
6663 * reference if they receive an UNREGISTER event.
6664 * We can get stuck here if buggy protocols don't correctly
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09006665 * call dev_put.
Linus Torvalds1da177e2005-04-16 15:20:36 -07006666 */
6667static void netdev_wait_allrefs(struct net_device *dev)
6668{
6669 unsigned long rebroadcast_time, warning_time;
Eric Dumazet29b44332010-10-11 10:22:12 +00006670 int refcnt;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006671
Eric Dumazete014deb2009-11-17 05:59:21 +00006672 linkwatch_forget_dev(dev);
6673
Linus Torvalds1da177e2005-04-16 15:20:36 -07006674 rebroadcast_time = warning_time = jiffies;
Eric Dumazet29b44332010-10-11 10:22:12 +00006675 refcnt = netdev_refcnt_read(dev);
6676
6677 while (refcnt != 0) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07006678 if (time_after(jiffies, rebroadcast_time + 1 * HZ)) {
Stephen Hemminger6756ae42006-03-20 22:23:58 -08006679 rtnl_lock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07006680
6681 /* Rebroadcast unregister notification */
Pavel Emelyanov056925a2007-09-16 15:42:43 -07006682 call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006683
Eric Dumazet748e2d92012-08-22 21:50:59 +00006684 __rtnl_unlock();
Eric Dumazet0115e8e2012-08-22 17:19:46 +00006685 rcu_barrier();
Eric Dumazet748e2d92012-08-22 21:50:59 +00006686 rtnl_lock();
6687
Eric Dumazet0115e8e2012-08-22 17:19:46 +00006688 call_netdevice_notifiers(NETDEV_UNREGISTER_FINAL, dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006689 if (test_bit(__LINK_STATE_LINKWATCH_PENDING,
6690 &dev->state)) {
6691 /* We must not have linkwatch events
6692 * pending on unregister. If this
6693 * happens, we simply run the queue
6694 * unscheduled, resulting in a noop
6695 * for this device.
6696 */
6697 linkwatch_run_queue();
6698 }
6699
Stephen Hemminger6756ae42006-03-20 22:23:58 -08006700 __rtnl_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07006701
6702 rebroadcast_time = jiffies;
6703 }
6704
6705 msleep(250);
6706
Eric Dumazet29b44332010-10-11 10:22:12 +00006707 refcnt = netdev_refcnt_read(dev);
6708
Linus Torvalds1da177e2005-04-16 15:20:36 -07006709 if (time_after(jiffies, warning_time + 10 * HZ)) {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00006710 pr_emerg("unregister_netdevice: waiting for %s to become free. Usage count = %d\n",
6711 dev->name, refcnt);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006712 warning_time = jiffies;
6713 }
6714 }
6715}
6716
6717/* The sequence is:
6718 *
6719 * rtnl_lock();
6720 * ...
6721 * register_netdevice(x1);
6722 * register_netdevice(x2);
6723 * ...
6724 * unregister_netdevice(y1);
6725 * unregister_netdevice(y2);
6726 * ...
6727 * rtnl_unlock();
6728 * free_netdev(y1);
6729 * free_netdev(y2);
6730 *
Herbert Xu58ec3b42008-10-07 15:50:03 -07006731 * We are invoked by rtnl_unlock().
Linus Torvalds1da177e2005-04-16 15:20:36 -07006732 * This allows us to deal with problems:
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07006733 * 1) We can delete sysfs objects which invoke hotplug
Linus Torvalds1da177e2005-04-16 15:20:36 -07006734 * without deadlocking with linkwatch via keventd.
6735 * 2) Since we run with the RTNL semaphore not held, we can sleep
6736 * safely in order to wait for the netdev refcnt to drop to zero.
Herbert Xu58ec3b42008-10-07 15:50:03 -07006737 *
6738 * We must not return until all unregister events added during
6739 * the interval the lock was held have been completed.
Linus Torvalds1da177e2005-04-16 15:20:36 -07006740 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07006741void netdev_run_todo(void)
6742{
Oleg Nesterov626ab0e2006-06-23 02:05:55 -07006743 struct list_head list;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006744
Linus Torvalds1da177e2005-04-16 15:20:36 -07006745 /* Snapshot list, allow later requests */
Oleg Nesterov626ab0e2006-06-23 02:05:55 -07006746 list_replace_init(&net_todo_list, &list);
Herbert Xu58ec3b42008-10-07 15:50:03 -07006747
6748 __rtnl_unlock();
Oleg Nesterov626ab0e2006-06-23 02:05:55 -07006749
Eric Dumazet0115e8e2012-08-22 17:19:46 +00006750
6751 /* Wait for rcu callbacks to finish before next phase */
Eric W. Biederman850a5452011-10-13 22:25:23 +00006752 if (!list_empty(&list))
6753 rcu_barrier();
6754
Linus Torvalds1da177e2005-04-16 15:20:36 -07006755 while (!list_empty(&list)) {
6756 struct net_device *dev
stephen hemmingere5e26d72010-02-24 14:01:38 +00006757 = list_first_entry(&list, struct net_device, todo_list);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006758 list_del(&dev->todo_list);
6759
Eric Dumazet748e2d92012-08-22 21:50:59 +00006760 rtnl_lock();
Eric Dumazet0115e8e2012-08-22 17:19:46 +00006761 call_netdevice_notifiers(NETDEV_UNREGISTER_FINAL, dev);
Eric Dumazet748e2d92012-08-22 21:50:59 +00006762 __rtnl_unlock();
Eric Dumazet0115e8e2012-08-22 17:19:46 +00006763
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07006764 if (unlikely(dev->reg_state != NETREG_UNREGISTERING)) {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00006765 pr_err("network todo '%s' but state %d\n",
Linus Torvalds1da177e2005-04-16 15:20:36 -07006766 dev->name, dev->reg_state);
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07006767 dump_stack();
6768 continue;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006769 }
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07006770
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07006771 dev->reg_state = NETREG_UNREGISTERED;
6772
Changli Gao152102c2010-03-30 20:16:22 +00006773 on_each_cpu(flush_backlog, dev, 1);
Stephen Hemminger6e583ce2008-08-03 21:29:57 -07006774
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07006775 netdev_wait_allrefs(dev);
6776
6777 /* paranoia */
Eric Dumazet29b44332010-10-11 10:22:12 +00006778 BUG_ON(netdev_refcnt_read(dev));
Salam Noureddine7866a622015-01-27 11:35:48 -08006779 BUG_ON(!list_empty(&dev->ptype_all));
6780 BUG_ON(!list_empty(&dev->ptype_specific));
Eric Dumazet33d480c2011-08-11 19:30:52 +00006781 WARN_ON(rcu_access_pointer(dev->ip_ptr));
6782 WARN_ON(rcu_access_pointer(dev->ip6_ptr));
Ilpo Järvinen547b7922008-07-25 21:43:18 -07006783 WARN_ON(dev->dn_ptr);
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07006784
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07006785 if (dev->destructor)
6786 dev->destructor(dev);
Stephen Hemminger9093bbb2007-05-19 15:39:25 -07006787
Eric W. Biederman50624c92013-09-23 21:19:49 -07006788 /* Report a network device has been unregistered */
6789 rtnl_lock();
6790 dev_net(dev)->dev_unreg_count--;
6791 __rtnl_unlock();
6792 wake_up(&netdev_unregistering_wq);
6793
Stephen Hemminger9093bbb2007-05-19 15:39:25 -07006794 /* Free network device */
6795 kobject_put(&dev->dev.kobj);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006796 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07006797}
6798
Ben Hutchings3cfde792010-07-09 09:11:52 +00006799/* Convert net_device_stats to rtnl_link_stats64. They have the same
6800 * fields in the same order, with only the type differing.
6801 */
Eric Dumazet77a1abf2012-03-05 04:50:09 +00006802void netdev_stats_to_stats64(struct rtnl_link_stats64 *stats64,
6803 const struct net_device_stats *netdev_stats)
Ben Hutchings3cfde792010-07-09 09:11:52 +00006804{
6805#if BITS_PER_LONG == 64
Eric Dumazet77a1abf2012-03-05 04:50:09 +00006806 BUILD_BUG_ON(sizeof(*stats64) != sizeof(*netdev_stats));
6807 memcpy(stats64, netdev_stats, sizeof(*stats64));
Ben Hutchings3cfde792010-07-09 09:11:52 +00006808#else
6809 size_t i, n = sizeof(*stats64) / sizeof(u64);
6810 const unsigned long *src = (const unsigned long *)netdev_stats;
6811 u64 *dst = (u64 *)stats64;
6812
6813 BUILD_BUG_ON(sizeof(*netdev_stats) / sizeof(unsigned long) !=
6814 sizeof(*stats64) / sizeof(u64));
6815 for (i = 0; i < n; i++)
6816 dst[i] = src[i];
6817#endif
6818}
Eric Dumazet77a1abf2012-03-05 04:50:09 +00006819EXPORT_SYMBOL(netdev_stats_to_stats64);
Ben Hutchings3cfde792010-07-09 09:11:52 +00006820
Eric Dumazetd83345a2009-11-16 03:36:51 +00006821/**
Stephen Hemmingereeda3fd2008-11-19 21:40:23 -08006822 * dev_get_stats - get network device statistics
6823 * @dev: device to get statistics from
Eric Dumazet28172732010-07-07 14:58:56 -07006824 * @storage: place to store stats
Stephen Hemmingereeda3fd2008-11-19 21:40:23 -08006825 *
Ben Hutchingsd7753512010-07-09 09:12:41 +00006826 * Get network statistics from device. Return @storage.
6827 * The device driver may provide its own method by setting
6828 * dev->netdev_ops->get_stats64 or dev->netdev_ops->get_stats;
6829 * otherwise the internal statistics structure is used.
Stephen Hemmingereeda3fd2008-11-19 21:40:23 -08006830 */
Ben Hutchingsd7753512010-07-09 09:12:41 +00006831struct rtnl_link_stats64 *dev_get_stats(struct net_device *dev,
6832 struct rtnl_link_stats64 *storage)
Eric Dumazet7004bf22009-05-18 00:34:33 +00006833{
Stephen Hemmingereeda3fd2008-11-19 21:40:23 -08006834 const struct net_device_ops *ops = dev->netdev_ops;
6835
Eric Dumazet28172732010-07-07 14:58:56 -07006836 if (ops->ndo_get_stats64) {
6837 memset(storage, 0, sizeof(*storage));
Eric Dumazetcaf586e2010-09-30 21:06:55 +00006838 ops->ndo_get_stats64(dev, storage);
6839 } else if (ops->ndo_get_stats) {
Ben Hutchings3cfde792010-07-09 09:11:52 +00006840 netdev_stats_to_stats64(storage, ops->ndo_get_stats(dev));
Eric Dumazetcaf586e2010-09-30 21:06:55 +00006841 } else {
6842 netdev_stats_to_stats64(storage, &dev->stats);
Eric Dumazet28172732010-07-07 14:58:56 -07006843 }
Eric Dumazetcaf586e2010-09-30 21:06:55 +00006844 storage->rx_dropped += atomic_long_read(&dev->rx_dropped);
Eric Dumazet015f0682014-03-27 08:45:56 -07006845 storage->tx_dropped += atomic_long_read(&dev->tx_dropped);
Eric Dumazet28172732010-07-07 14:58:56 -07006846 return storage;
Rusty Russellc45d2862007-03-28 14:29:08 -07006847}
Stephen Hemmingereeda3fd2008-11-19 21:40:23 -08006848EXPORT_SYMBOL(dev_get_stats);
Rusty Russellc45d2862007-03-28 14:29:08 -07006849
Eric Dumazet24824a02010-10-02 06:11:55 +00006850struct netdev_queue *dev_ingress_queue_create(struct net_device *dev)
David S. Millerdc2b4842008-07-08 17:18:23 -07006851{
Eric Dumazet24824a02010-10-02 06:11:55 +00006852 struct netdev_queue *queue = dev_ingress_queue(dev);
David S. Millerdc2b4842008-07-08 17:18:23 -07006853
Eric Dumazet24824a02010-10-02 06:11:55 +00006854#ifdef CONFIG_NET_CLS_ACT
6855 if (queue)
6856 return queue;
6857 queue = kzalloc(sizeof(*queue), GFP_KERNEL);
6858 if (!queue)
6859 return NULL;
6860 netdev_init_one_queue(dev, queue, NULL);
Eric Dumazet2ce1ee12015-02-04 13:37:44 -08006861 RCU_INIT_POINTER(queue->qdisc, &noop_qdisc);
Eric Dumazet24824a02010-10-02 06:11:55 +00006862 queue->qdisc_sleeping = &noop_qdisc;
6863 rcu_assign_pointer(dev->ingress_queue, queue);
6864#endif
6865 return queue;
David S. Millerbb949fb2008-07-08 16:55:56 -07006866}
6867
Eric Dumazet2c60db02012-09-16 09:17:26 +00006868static const struct ethtool_ops default_ethtool_ops;
6869
Stanislaw Gruszkad07d7502013-01-10 23:19:10 +00006870void netdev_set_default_ethtool_ops(struct net_device *dev,
6871 const struct ethtool_ops *ops)
6872{
6873 if (dev->ethtool_ops == &default_ethtool_ops)
6874 dev->ethtool_ops = ops;
6875}
6876EXPORT_SYMBOL_GPL(netdev_set_default_ethtool_ops);
6877
Eric Dumazet74d332c2013-10-30 13:10:44 -07006878void netdev_freemem(struct net_device *dev)
6879{
6880 char *addr = (char *)dev - dev->padded;
6881
WANG Cong4cb28972014-06-02 15:55:22 -07006882 kvfree(addr);
Eric Dumazet74d332c2013-10-30 13:10:44 -07006883}
6884
Linus Torvalds1da177e2005-04-16 15:20:36 -07006885/**
Tom Herbert36909ea2011-01-09 19:36:31 +00006886 * alloc_netdev_mqs - allocate network device
Tom Gundersenc835a672014-07-14 16:37:24 +02006887 * @sizeof_priv: size of private data to allocate space for
6888 * @name: device name format string
6889 * @name_assign_type: origin of device name
6890 * @setup: callback to initialize device
6891 * @txqs: the number of TX subqueues to allocate
6892 * @rxqs: the number of RX subqueues to allocate
Linus Torvalds1da177e2005-04-16 15:20:36 -07006893 *
6894 * Allocates a struct net_device with private data area for driver use
Li Zhong90e51ad2013-11-22 15:04:46 +08006895 * and performs basic initialization. Also allocates subqueue structs
Tom Herbert36909ea2011-01-09 19:36:31 +00006896 * for each queue on the device.
Linus Torvalds1da177e2005-04-16 15:20:36 -07006897 */
Tom Herbert36909ea2011-01-09 19:36:31 +00006898struct net_device *alloc_netdev_mqs(int sizeof_priv, const char *name,
Tom Gundersenc835a672014-07-14 16:37:24 +02006899 unsigned char name_assign_type,
Tom Herbert36909ea2011-01-09 19:36:31 +00006900 void (*setup)(struct net_device *),
6901 unsigned int txqs, unsigned int rxqs)
Linus Torvalds1da177e2005-04-16 15:20:36 -07006902{
Linus Torvalds1da177e2005-04-16 15:20:36 -07006903 struct net_device *dev;
Stephen Hemminger79439862008-07-21 13:28:44 -07006904 size_t alloc_size;
Eric Dumazet1ce8e7b2009-05-27 04:42:37 +00006905 struct net_device *p;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006906
Stephen Hemmingerb6fe17d2006-08-29 17:06:13 -07006907 BUG_ON(strlen(name) >= sizeof(dev->name));
6908
Tom Herbert36909ea2011-01-09 19:36:31 +00006909 if (txqs < 1) {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00006910 pr_err("alloc_netdev: Unable to allocate device with zero queues\n");
Tom Herbert55513fb2010-10-18 17:55:58 +00006911 return NULL;
6912 }
6913
Michael Daltona953be52014-01-16 22:23:28 -08006914#ifdef CONFIG_SYSFS
Tom Herbert36909ea2011-01-09 19:36:31 +00006915 if (rxqs < 1) {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00006916 pr_err("alloc_netdev: Unable to allocate device with zero RX queues\n");
Tom Herbert36909ea2011-01-09 19:36:31 +00006917 return NULL;
6918 }
6919#endif
6920
David S. Millerfd2ea0a2008-07-17 01:56:23 -07006921 alloc_size = sizeof(struct net_device);
Alexey Dobriyand1643d22008-04-18 15:43:32 -07006922 if (sizeof_priv) {
6923 /* ensure 32-byte alignment of private area */
Eric Dumazet1ce8e7b2009-05-27 04:42:37 +00006924 alloc_size = ALIGN(alloc_size, NETDEV_ALIGN);
Alexey Dobriyand1643d22008-04-18 15:43:32 -07006925 alloc_size += sizeof_priv;
6926 }
6927 /* ensure 32-byte alignment of whole construct */
Eric Dumazet1ce8e7b2009-05-27 04:42:37 +00006928 alloc_size += NETDEV_ALIGN - 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006929
Eric Dumazet74d332c2013-10-30 13:10:44 -07006930 p = kzalloc(alloc_size, GFP_KERNEL | __GFP_NOWARN | __GFP_REPEAT);
6931 if (!p)
6932 p = vzalloc(alloc_size);
Joe Perches62b59422013-02-04 16:48:16 +00006933 if (!p)
Linus Torvalds1da177e2005-04-16 15:20:36 -07006934 return NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006935
Eric Dumazet1ce8e7b2009-05-27 04:42:37 +00006936 dev = PTR_ALIGN(p, NETDEV_ALIGN);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006937 dev->padded = (char *)dev - (char *)p;
Jiri Pirkoab9c73c2009-05-08 13:30:17 +00006938
Eric Dumazet29b44332010-10-11 10:22:12 +00006939 dev->pcpu_refcnt = alloc_percpu(int);
6940 if (!dev->pcpu_refcnt)
Eric Dumazet74d332c2013-10-30 13:10:44 -07006941 goto free_dev;
Jiri Pirkoab9c73c2009-05-08 13:30:17 +00006942
Linus Torvalds1da177e2005-04-16 15:20:36 -07006943 if (dev_addr_init(dev))
Eric Dumazet29b44332010-10-11 10:22:12 +00006944 goto free_pcpu;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006945
Jiri Pirko22bedad32010-04-01 21:22:57 +00006946 dev_mc_init(dev);
Jiri Pirkoa748ee22010-04-01 21:22:09 +00006947 dev_uc_init(dev);
Jiri Pirkoccffad252009-05-22 23:22:17 +00006948
YOSHIFUJI Hideakic346dca2008-03-25 21:47:49 +09006949 dev_net_set(dev, &init_net);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006950
Peter P Waskiewicz Jr82cc1a72008-03-21 03:43:19 -07006951 dev->gso_max_size = GSO_MAX_SIZE;
Ben Hutchings30b678d2012-07-30 15:57:00 +00006952 dev->gso_max_segs = GSO_MAX_SEGS;
Eric Dumazetfcbeb972014-10-05 10:11:27 -07006953 dev->gso_min_segs = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006954
Herbert Xud565b0a2008-12-15 23:38:52 -08006955 INIT_LIST_HEAD(&dev->napi_list);
Eric W. Biederman9fdce092009-10-30 14:51:13 +00006956 INIT_LIST_HEAD(&dev->unreg_list);
Eric W. Biederman5cde2822013-10-05 19:26:05 -07006957 INIT_LIST_HEAD(&dev->close_list);
Eric Dumazete014deb2009-11-17 05:59:21 +00006958 INIT_LIST_HEAD(&dev->link_watch_list);
Veaceslav Falico2f268f12013-09-25 09:20:07 +02006959 INIT_LIST_HEAD(&dev->adj_list.upper);
6960 INIT_LIST_HEAD(&dev->adj_list.lower);
6961 INIT_LIST_HEAD(&dev->all_adj_list.upper);
6962 INIT_LIST_HEAD(&dev->all_adj_list.lower);
Salam Noureddine7866a622015-01-27 11:35:48 -08006963 INIT_LIST_HEAD(&dev->ptype_all);
6964 INIT_LIST_HEAD(&dev->ptype_specific);
Eric Dumazet02875872014-10-05 18:38:35 -07006965 dev->priv_flags = IFF_XMIT_DST_RELEASE | IFF_XMIT_DST_RELEASE_PERM;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006966 setup(dev);
David S. Miller8d3bdbd2011-02-08 15:02:50 -08006967
6968 dev->num_tx_queues = txqs;
6969 dev->real_num_tx_queues = txqs;
6970 if (netif_alloc_netdev_queues(dev))
6971 goto free_all;
6972
Michael Daltona953be52014-01-16 22:23:28 -08006973#ifdef CONFIG_SYSFS
David S. Miller8d3bdbd2011-02-08 15:02:50 -08006974 dev->num_rx_queues = rxqs;
6975 dev->real_num_rx_queues = rxqs;
6976 if (netif_alloc_rx_queues(dev))
6977 goto free_all;
6978#endif
6979
Linus Torvalds1da177e2005-04-16 15:20:36 -07006980 strcpy(dev->name, name);
Tom Gundersenc835a672014-07-14 16:37:24 +02006981 dev->name_assign_type = name_assign_type;
Vlad Dogarucbda10f2011-01-13 23:38:30 +00006982 dev->group = INIT_NETDEV_GROUP;
Eric Dumazet2c60db02012-09-16 09:17:26 +00006983 if (!dev->ethtool_ops)
6984 dev->ethtool_ops = &default_ethtool_ops;
Pablo Neirae687ad62015-05-13 18:19:38 +02006985
6986 nf_hook_ingress_init(dev);
6987
Linus Torvalds1da177e2005-04-16 15:20:36 -07006988 return dev;
Jiri Pirkoab9c73c2009-05-08 13:30:17 +00006989
David S. Miller8d3bdbd2011-02-08 15:02:50 -08006990free_all:
6991 free_netdev(dev);
6992 return NULL;
6993
Eric Dumazet29b44332010-10-11 10:22:12 +00006994free_pcpu:
6995 free_percpu(dev->pcpu_refcnt);
Eric Dumazet74d332c2013-10-30 13:10:44 -07006996free_dev:
6997 netdev_freemem(dev);
Jiri Pirkoab9c73c2009-05-08 13:30:17 +00006998 return NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006999}
Tom Herbert36909ea2011-01-09 19:36:31 +00007000EXPORT_SYMBOL(alloc_netdev_mqs);
Linus Torvalds1da177e2005-04-16 15:20:36 -07007001
7002/**
7003 * free_netdev - free network device
7004 * @dev: device
7005 *
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09007006 * This function does the last stage of destroying an allocated device
7007 * interface. The reference to the device object is released.
Linus Torvalds1da177e2005-04-16 15:20:36 -07007008 * If this is the last reference then it will be freed.
7009 */
7010void free_netdev(struct net_device *dev)
7011{
Herbert Xud565b0a2008-12-15 23:38:52 -08007012 struct napi_struct *p, *n;
7013
Eric Dumazet60877a32013-06-20 01:15:51 -07007014 netif_free_tx_queues(dev);
Michael Daltona953be52014-01-16 22:23:28 -08007015#ifdef CONFIG_SYSFS
Pankaj Gupta10595902015-01-12 11:41:28 +05307016 kvfree(dev->_rx);
Tom Herbertfe822242010-11-09 10:47:38 +00007017#endif
David S. Millere8a04642008-07-17 00:34:19 -07007018
Eric Dumazet33d480c2011-08-11 19:30:52 +00007019 kfree(rcu_dereference_protected(dev->ingress_queue, 1));
Eric Dumazet24824a02010-10-02 06:11:55 +00007020
Jiri Pirkof001fde2009-05-05 02:48:28 +00007021 /* Flush device addresses */
7022 dev_addr_flush(dev);
7023
Herbert Xud565b0a2008-12-15 23:38:52 -08007024 list_for_each_entry_safe(p, n, &dev->napi_list, dev_list)
7025 netif_napi_del(p);
7026
Eric Dumazet29b44332010-10-11 10:22:12 +00007027 free_percpu(dev->pcpu_refcnt);
7028 dev->pcpu_refcnt = NULL;
7029
Stephen Hemminger3041a062006-05-26 13:25:24 -07007030 /* Compatibility with error handling in drivers */
Linus Torvalds1da177e2005-04-16 15:20:36 -07007031 if (dev->reg_state == NETREG_UNINITIALIZED) {
Eric Dumazet74d332c2013-10-30 13:10:44 -07007032 netdev_freemem(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07007033 return;
7034 }
7035
7036 BUG_ON(dev->reg_state != NETREG_UNREGISTERED);
7037 dev->reg_state = NETREG_RELEASED;
7038
Greg Kroah-Hartman43cb76d2002-04-09 12:14:34 -07007039 /* will free via device release */
7040 put_device(&dev->dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07007041}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07007042EXPORT_SYMBOL(free_netdev);
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09007043
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07007044/**
7045 * synchronize_net - Synchronize with packet receive processing
7046 *
7047 * Wait for packets currently being received to be done.
7048 * Does not block later packets from starting.
7049 */
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09007050void synchronize_net(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -07007051{
7052 might_sleep();
Eric Dumazetbe3fc412011-05-23 23:07:32 +00007053 if (rtnl_is_locked())
7054 synchronize_rcu_expedited();
7055 else
7056 synchronize_rcu();
Linus Torvalds1da177e2005-04-16 15:20:36 -07007057}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07007058EXPORT_SYMBOL(synchronize_net);
Linus Torvalds1da177e2005-04-16 15:20:36 -07007059
7060/**
Eric Dumazet44a08732009-10-27 07:03:04 +00007061 * unregister_netdevice_queue - remove device from the kernel
Linus Torvalds1da177e2005-04-16 15:20:36 -07007062 * @dev: device
Eric Dumazet44a08732009-10-27 07:03:04 +00007063 * @head: list
Jaswinder Singh Rajput6ebfbc02009-11-22 20:43:13 -08007064 *
Linus Torvalds1da177e2005-04-16 15:20:36 -07007065 * This function shuts down a device interface and removes it
Wang Chend59b54b2007-12-11 02:28:03 -08007066 * from the kernel tables.
Eric Dumazet44a08732009-10-27 07:03:04 +00007067 * If head not NULL, device is queued to be unregistered later.
Linus Torvalds1da177e2005-04-16 15:20:36 -07007068 *
7069 * Callers must hold the rtnl semaphore. You may want
7070 * unregister_netdev() instead of this.
7071 */
7072
Eric Dumazet44a08732009-10-27 07:03:04 +00007073void unregister_netdevice_queue(struct net_device *dev, struct list_head *head)
Linus Torvalds1da177e2005-04-16 15:20:36 -07007074{
Herbert Xua6620712007-12-12 19:21:56 -08007075 ASSERT_RTNL();
7076
Eric Dumazet44a08732009-10-27 07:03:04 +00007077 if (head) {
Eric W. Biederman9fdce092009-10-30 14:51:13 +00007078 list_move_tail(&dev->unreg_list, head);
Eric Dumazet44a08732009-10-27 07:03:04 +00007079 } else {
7080 rollback_registered(dev);
7081 /* Finish processing unregister after unlock */
7082 net_set_todo(dev);
7083 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07007084}
Eric Dumazet44a08732009-10-27 07:03:04 +00007085EXPORT_SYMBOL(unregister_netdevice_queue);
Linus Torvalds1da177e2005-04-16 15:20:36 -07007086
7087/**
Eric Dumazet9b5e3832009-10-27 07:04:19 +00007088 * unregister_netdevice_many - unregister many devices
7089 * @head: list of devices
Eric Dumazet87757a92014-06-06 06:44:03 -07007090 *
7091 * Note: As most callers use a stack allocated list_head,
7092 * we force a list_del() to make sure stack wont be corrupted later.
Eric Dumazet9b5e3832009-10-27 07:04:19 +00007093 */
7094void unregister_netdevice_many(struct list_head *head)
7095{
7096 struct net_device *dev;
7097
7098 if (!list_empty(head)) {
7099 rollback_registered_many(head);
7100 list_for_each_entry(dev, head, unreg_list)
7101 net_set_todo(dev);
Eric Dumazet87757a92014-06-06 06:44:03 -07007102 list_del(head);
Eric Dumazet9b5e3832009-10-27 07:04:19 +00007103 }
7104}
Eric Dumazet63c80992009-10-27 07:06:49 +00007105EXPORT_SYMBOL(unregister_netdevice_many);
Eric Dumazet9b5e3832009-10-27 07:04:19 +00007106
7107/**
Linus Torvalds1da177e2005-04-16 15:20:36 -07007108 * unregister_netdev - remove device from the kernel
7109 * @dev: device
7110 *
7111 * This function shuts down a device interface and removes it
Wang Chend59b54b2007-12-11 02:28:03 -08007112 * from the kernel tables.
Linus Torvalds1da177e2005-04-16 15:20:36 -07007113 *
7114 * This is just a wrapper for unregister_netdevice that takes
7115 * the rtnl semaphore. In general you want to use this and not
7116 * unregister_netdevice.
7117 */
7118void unregister_netdev(struct net_device *dev)
7119{
7120 rtnl_lock();
7121 unregister_netdevice(dev);
7122 rtnl_unlock();
7123}
Linus Torvalds1da177e2005-04-16 15:20:36 -07007124EXPORT_SYMBOL(unregister_netdev);
7125
Eric W. Biedermance286d32007-09-12 13:53:49 +02007126/**
7127 * dev_change_net_namespace - move device to different nethost namespace
7128 * @dev: device
7129 * @net: network namespace
7130 * @pat: If not NULL name pattern to try if the current device name
7131 * is already taken in the destination network namespace.
7132 *
7133 * This function shuts down a device interface and moves it
7134 * to a new network namespace. On success 0 is returned, on
7135 * a failure a netagive errno code is returned.
7136 *
7137 * Callers must hold the rtnl semaphore.
7138 */
7139
7140int dev_change_net_namespace(struct net_device *dev, struct net *net, const char *pat)
7141{
Eric W. Biedermance286d32007-09-12 13:53:49 +02007142 int err;
7143
7144 ASSERT_RTNL();
7145
7146 /* Don't allow namespace local devices to be moved. */
7147 err = -EINVAL;
7148 if (dev->features & NETIF_F_NETNS_LOCAL)
7149 goto out;
7150
7151 /* Ensure the device has been registrered */
Eric W. Biedermance286d32007-09-12 13:53:49 +02007152 if (dev->reg_state != NETREG_REGISTERED)
7153 goto out;
7154
7155 /* Get out if there is nothing todo */
7156 err = 0;
YOSHIFUJI Hideaki878628f2008-03-26 03:57:35 +09007157 if (net_eq(dev_net(dev), net))
Eric W. Biedermance286d32007-09-12 13:53:49 +02007158 goto out;
7159
7160 /* Pick the destination device name, and ensure
7161 * we can use it in the destination network namespace.
7162 */
7163 err = -EEXIST;
Octavian Purdilad9031022009-11-18 02:36:59 +00007164 if (__dev_get_by_name(net, dev->name)) {
Eric W. Biedermance286d32007-09-12 13:53:49 +02007165 /* We get here if we can't use the current device name */
7166 if (!pat)
7167 goto out;
Gao feng828de4f2012-09-13 20:58:27 +00007168 if (dev_get_valid_name(net, dev, pat) < 0)
Eric W. Biedermance286d32007-09-12 13:53:49 +02007169 goto out;
7170 }
7171
7172 /*
7173 * And now a mini version of register_netdevice unregister_netdevice.
7174 */
7175
7176 /* If device is running close it first. */
Pavel Emelyanov9b772652007-10-10 02:49:09 -07007177 dev_close(dev);
Eric W. Biedermance286d32007-09-12 13:53:49 +02007178
7179 /* And unlink it from device chain */
7180 err = -ENODEV;
7181 unlist_netdevice(dev);
7182
7183 synchronize_net();
7184
7185 /* Shutdown queueing discipline. */
7186 dev_shutdown(dev);
7187
7188 /* Notify protocols, that we are about to destroy
7189 this device. They should clean all the things.
David Lamparter3b27e102010-09-17 03:22:19 +00007190
7191 Note that dev->reg_state stays at NETREG_REGISTERED.
7192 This is wanted because this way 8021q and macvlan know
7193 the device is just moving and can keep their slaves up.
Eric W. Biedermance286d32007-09-12 13:53:49 +02007194 */
7195 call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
Gao feng6549dd42012-08-23 15:36:55 +00007196 rcu_barrier();
7197 call_netdevice_notifiers(NETDEV_UNREGISTER_FINAL, dev);
Alexei Starovoitov7f294052013-10-23 16:02:42 -07007198 rtmsg_ifinfo(RTM_DELLINK, dev, ~0U, GFP_KERNEL);
Eric W. Biedermance286d32007-09-12 13:53:49 +02007199
7200 /*
7201 * Flush the unicast and multicast chains
7202 */
Jiri Pirkoa748ee22010-04-01 21:22:09 +00007203 dev_uc_flush(dev);
Jiri Pirko22bedad32010-04-01 21:22:57 +00007204 dev_mc_flush(dev);
Eric W. Biedermance286d32007-09-12 13:53:49 +02007205
Serge Hallyn4e66ae22012-12-03 16:17:12 +00007206 /* Send a netdev-removed uevent to the old namespace */
7207 kobject_uevent(&dev->dev.kobj, KOBJ_REMOVE);
Alexander Y. Fomichev4c754312014-08-25 16:26:45 +04007208 netdev_adjacent_del_links(dev);
Serge Hallyn4e66ae22012-12-03 16:17:12 +00007209
Eric W. Biedermance286d32007-09-12 13:53:49 +02007210 /* Actually switch the network namespace */
YOSHIFUJI Hideakic346dca2008-03-25 21:47:49 +09007211 dev_net_set(dev, net);
Eric W. Biedermance286d32007-09-12 13:53:49 +02007212
Eric W. Biedermance286d32007-09-12 13:53:49 +02007213 /* If there is an ifindex conflict assign a new one */
Nicolas Dichtel7a66bbc2015-04-02 17:07:09 +02007214 if (__dev_get_by_index(net, dev->ifindex))
Eric W. Biedermance286d32007-09-12 13:53:49 +02007215 dev->ifindex = dev_new_index(net);
Eric W. Biedermance286d32007-09-12 13:53:49 +02007216
Serge Hallyn4e66ae22012-12-03 16:17:12 +00007217 /* Send a netdev-add uevent to the new namespace */
7218 kobject_uevent(&dev->dev.kobj, KOBJ_ADD);
Alexander Y. Fomichev4c754312014-08-25 16:26:45 +04007219 netdev_adjacent_add_links(dev);
Serge Hallyn4e66ae22012-12-03 16:17:12 +00007220
Eric W. Biederman8b41d182007-09-26 22:02:53 -07007221 /* Fixup kobjects */
Eric W. Biedermana1b3f592010-05-04 17:36:49 -07007222 err = device_rename(&dev->dev, dev->name);
Eric W. Biederman8b41d182007-09-26 22:02:53 -07007223 WARN_ON(err);
Eric W. Biedermance286d32007-09-12 13:53:49 +02007224
7225 /* Add the device back in the hashes */
7226 list_netdevice(dev);
7227
7228 /* Notify protocols, that a new device appeared. */
7229 call_netdevice_notifiers(NETDEV_REGISTER, dev);
7230
Eric W. Biedermand90a9092009-12-12 22:11:15 +00007231 /*
7232 * Prevent userspace races by waiting until the network
7233 * device is fully setup before sending notifications.
7234 */
Alexei Starovoitov7f294052013-10-23 16:02:42 -07007235 rtmsg_ifinfo(RTM_NEWLINK, dev, ~0U, GFP_KERNEL);
Eric W. Biedermand90a9092009-12-12 22:11:15 +00007236
Eric W. Biedermance286d32007-09-12 13:53:49 +02007237 synchronize_net();
7238 err = 0;
7239out:
7240 return err;
7241}
Johannes Berg463d0182009-07-14 00:33:35 +02007242EXPORT_SYMBOL_GPL(dev_change_net_namespace);
Eric W. Biedermance286d32007-09-12 13:53:49 +02007243
Linus Torvalds1da177e2005-04-16 15:20:36 -07007244static int dev_cpu_callback(struct notifier_block *nfb,
7245 unsigned long action,
7246 void *ocpu)
7247{
7248 struct sk_buff **list_skb;
Linus Torvalds1da177e2005-04-16 15:20:36 -07007249 struct sk_buff *skb;
7250 unsigned int cpu, oldcpu = (unsigned long)ocpu;
7251 struct softnet_data *sd, *oldsd;
7252
Rafael J. Wysocki8bb78442007-05-09 02:35:10 -07007253 if (action != CPU_DEAD && action != CPU_DEAD_FROZEN)
Linus Torvalds1da177e2005-04-16 15:20:36 -07007254 return NOTIFY_OK;
7255
7256 local_irq_disable();
7257 cpu = smp_processor_id();
7258 sd = &per_cpu(softnet_data, cpu);
7259 oldsd = &per_cpu(softnet_data, oldcpu);
7260
7261 /* Find end of our completion_queue. */
7262 list_skb = &sd->completion_queue;
7263 while (*list_skb)
7264 list_skb = &(*list_skb)->next;
7265 /* Append completion queue from offline CPU. */
7266 *list_skb = oldsd->completion_queue;
7267 oldsd->completion_queue = NULL;
7268
Linus Torvalds1da177e2005-04-16 15:20:36 -07007269 /* Append output queue from offline CPU. */
Changli Gaoa9cbd582010-04-26 23:06:24 +00007270 if (oldsd->output_queue) {
7271 *sd->output_queue_tailp = oldsd->output_queue;
7272 sd->output_queue_tailp = oldsd->output_queue_tailp;
7273 oldsd->output_queue = NULL;
7274 oldsd->output_queue_tailp = &oldsd->output_queue;
7275 }
Eric Dumazetac64da02015-01-15 17:04:22 -08007276 /* Append NAPI poll list from offline CPU, with one exception :
7277 * process_backlog() must be called by cpu owning percpu backlog.
7278 * We properly handle process_queue & input_pkt_queue later.
7279 */
7280 while (!list_empty(&oldsd->poll_list)) {
7281 struct napi_struct *napi = list_first_entry(&oldsd->poll_list,
7282 struct napi_struct,
7283 poll_list);
7284
7285 list_del_init(&napi->poll_list);
7286 if (napi->poll == process_backlog)
7287 napi->state = 0;
7288 else
7289 ____napi_schedule(sd, napi);
Heiko Carstens264524d2011-06-06 20:50:03 +00007290 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07007291
7292 raise_softirq_irqoff(NET_TX_SOFTIRQ);
7293 local_irq_enable();
7294
7295 /* Process offline CPU's input_pkt_queue */
Tom Herbert76cc8b12010-05-20 18:37:59 +00007296 while ((skb = __skb_dequeue(&oldsd->process_queue))) {
Eric Dumazet91e83132015-02-05 14:58:14 -08007297 netif_rx_ni(skb);
Tom Herbert76cc8b12010-05-20 18:37:59 +00007298 input_queue_head_incr(oldsd);
7299 }
Eric Dumazetac64da02015-01-15 17:04:22 -08007300 while ((skb = skb_dequeue(&oldsd->input_pkt_queue))) {
Eric Dumazet91e83132015-02-05 14:58:14 -08007301 netif_rx_ni(skb);
Tom Herbert76cc8b12010-05-20 18:37:59 +00007302 input_queue_head_incr(oldsd);
Tom Herbertfec5e652010-04-16 16:01:27 -07007303 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07007304
7305 return NOTIFY_OK;
7306}
Linus Torvalds1da177e2005-04-16 15:20:36 -07007307
7308
Herbert Xu7f353bf2007-08-10 15:47:58 -07007309/**
Herbert Xub63365a2008-10-23 01:11:29 -07007310 * netdev_increment_features - increment feature set by one
7311 * @all: current feature set
7312 * @one: new feature set
7313 * @mask: mask feature set
Herbert Xu7f353bf2007-08-10 15:47:58 -07007314 *
7315 * Computes a new feature set after adding a device with feature set
Herbert Xub63365a2008-10-23 01:11:29 -07007316 * @one to the master device with current feature set @all. Will not
7317 * enable anything that is off in @mask. Returns the new feature set.
Herbert Xu7f353bf2007-08-10 15:47:58 -07007318 */
Michał Mirosławc8f44af2011-11-15 15:29:55 +00007319netdev_features_t netdev_increment_features(netdev_features_t all,
7320 netdev_features_t one, netdev_features_t mask)
Herbert Xu7f353bf2007-08-10 15:47:58 -07007321{
Michał Mirosław1742f182011-04-22 06:31:16 +00007322 if (mask & NETIF_F_GEN_CSUM)
7323 mask |= NETIF_F_ALL_CSUM;
7324 mask |= NETIF_F_VLAN_CHALLENGED;
7325
7326 all |= one & (NETIF_F_ONE_FOR_ALL|NETIF_F_ALL_CSUM) & mask;
7327 all &= one | ~NETIF_F_ALL_FOR_ALL;
7328
Michał Mirosław1742f182011-04-22 06:31:16 +00007329 /* If one device supports hw checksumming, set for all. */
7330 if (all & NETIF_F_GEN_CSUM)
7331 all &= ~(NETIF_F_ALL_CSUM & ~NETIF_F_GEN_CSUM);
Herbert Xu7f353bf2007-08-10 15:47:58 -07007332
7333 return all;
7334}
Herbert Xub63365a2008-10-23 01:11:29 -07007335EXPORT_SYMBOL(netdev_increment_features);
Herbert Xu7f353bf2007-08-10 15:47:58 -07007336
Baruch Siach430f03c2013-06-02 20:43:55 +00007337static struct hlist_head * __net_init netdev_create_hash(void)
Pavel Emelyanov30d97d32007-09-16 15:40:33 -07007338{
7339 int i;
7340 struct hlist_head *hash;
7341
7342 hash = kmalloc(sizeof(*hash) * NETDEV_HASHENTRIES, GFP_KERNEL);
7343 if (hash != NULL)
7344 for (i = 0; i < NETDEV_HASHENTRIES; i++)
7345 INIT_HLIST_HEAD(&hash[i]);
7346
7347 return hash;
7348}
7349
Eric W. Biederman881d9662007-09-17 11:56:21 -07007350/* Initialize per network namespace state */
Pavel Emelyanov46650792007-10-08 20:38:39 -07007351static int __net_init netdev_init(struct net *net)
Eric W. Biederman881d9662007-09-17 11:56:21 -07007352{
Rustad, Mark D734b6542012-07-18 09:06:07 +00007353 if (net != &init_net)
7354 INIT_LIST_HEAD(&net->dev_base_head);
Eric W. Biederman881d9662007-09-17 11:56:21 -07007355
Pavel Emelyanov30d97d32007-09-16 15:40:33 -07007356 net->dev_name_head = netdev_create_hash();
7357 if (net->dev_name_head == NULL)
7358 goto err_name;
Eric W. Biederman881d9662007-09-17 11:56:21 -07007359
Pavel Emelyanov30d97d32007-09-16 15:40:33 -07007360 net->dev_index_head = netdev_create_hash();
7361 if (net->dev_index_head == NULL)
7362 goto err_idx;
Eric W. Biederman881d9662007-09-17 11:56:21 -07007363
7364 return 0;
Pavel Emelyanov30d97d32007-09-16 15:40:33 -07007365
7366err_idx:
7367 kfree(net->dev_name_head);
7368err_name:
7369 return -ENOMEM;
Eric W. Biederman881d9662007-09-17 11:56:21 -07007370}
7371
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07007372/**
7373 * netdev_drivername - network driver for the device
7374 * @dev: network device
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07007375 *
7376 * Determine network driver for device.
7377 */
David S. Miller3019de12011-06-06 16:41:33 -07007378const char *netdev_drivername(const struct net_device *dev)
Arjan van de Ven6579e572008-07-21 13:31:48 -07007379{
Stephen Hemmingercf04a4c72008-09-30 02:22:14 -07007380 const struct device_driver *driver;
7381 const struct device *parent;
David S. Miller3019de12011-06-06 16:41:33 -07007382 const char *empty = "";
Arjan van de Ven6579e572008-07-21 13:31:48 -07007383
7384 parent = dev->dev.parent;
Arjan van de Ven6579e572008-07-21 13:31:48 -07007385 if (!parent)
David S. Miller3019de12011-06-06 16:41:33 -07007386 return empty;
Arjan van de Ven6579e572008-07-21 13:31:48 -07007387
7388 driver = parent->driver;
7389 if (driver && driver->name)
David S. Miller3019de12011-06-06 16:41:33 -07007390 return driver->name;
7391 return empty;
Arjan van de Ven6579e572008-07-21 13:31:48 -07007392}
7393
Joe Perches6ea754e2014-09-22 11:10:50 -07007394static void __netdev_printk(const char *level, const struct net_device *dev,
7395 struct va_format *vaf)
Joe Perches256df2f2010-06-27 01:02:35 +00007396{
Joe Perchesb004ff42012-09-12 20:12:19 -07007397 if (dev && dev->dev.parent) {
Joe Perches6ea754e2014-09-22 11:10:50 -07007398 dev_printk_emit(level[1] - '0',
7399 dev->dev.parent,
7400 "%s %s %s%s: %pV",
7401 dev_driver_string(dev->dev.parent),
7402 dev_name(dev->dev.parent),
7403 netdev_name(dev), netdev_reg_state(dev),
7404 vaf);
Joe Perchesb004ff42012-09-12 20:12:19 -07007405 } else if (dev) {
Joe Perches6ea754e2014-09-22 11:10:50 -07007406 printk("%s%s%s: %pV",
7407 level, netdev_name(dev), netdev_reg_state(dev), vaf);
Joe Perchesb004ff42012-09-12 20:12:19 -07007408 } else {
Joe Perches6ea754e2014-09-22 11:10:50 -07007409 printk("%s(NULL net_device): %pV", level, vaf);
Joe Perchesb004ff42012-09-12 20:12:19 -07007410 }
Joe Perches256df2f2010-06-27 01:02:35 +00007411}
7412
Joe Perches6ea754e2014-09-22 11:10:50 -07007413void netdev_printk(const char *level, const struct net_device *dev,
7414 const char *format, ...)
Joe Perches256df2f2010-06-27 01:02:35 +00007415{
7416 struct va_format vaf;
7417 va_list args;
Joe Perches256df2f2010-06-27 01:02:35 +00007418
7419 va_start(args, format);
7420
7421 vaf.fmt = format;
7422 vaf.va = &args;
7423
Joe Perches6ea754e2014-09-22 11:10:50 -07007424 __netdev_printk(level, dev, &vaf);
Joe Perchesb004ff42012-09-12 20:12:19 -07007425
Joe Perches256df2f2010-06-27 01:02:35 +00007426 va_end(args);
Joe Perches256df2f2010-06-27 01:02:35 +00007427}
7428EXPORT_SYMBOL(netdev_printk);
7429
7430#define define_netdev_printk_level(func, level) \
Joe Perches6ea754e2014-09-22 11:10:50 -07007431void func(const struct net_device *dev, const char *fmt, ...) \
Joe Perches256df2f2010-06-27 01:02:35 +00007432{ \
Joe Perches256df2f2010-06-27 01:02:35 +00007433 struct va_format vaf; \
7434 va_list args; \
7435 \
7436 va_start(args, fmt); \
7437 \
7438 vaf.fmt = fmt; \
7439 vaf.va = &args; \
7440 \
Joe Perches6ea754e2014-09-22 11:10:50 -07007441 __netdev_printk(level, dev, &vaf); \
Joe Perchesb004ff42012-09-12 20:12:19 -07007442 \
Joe Perches256df2f2010-06-27 01:02:35 +00007443 va_end(args); \
Joe Perches256df2f2010-06-27 01:02:35 +00007444} \
7445EXPORT_SYMBOL(func);
7446
7447define_netdev_printk_level(netdev_emerg, KERN_EMERG);
7448define_netdev_printk_level(netdev_alert, KERN_ALERT);
7449define_netdev_printk_level(netdev_crit, KERN_CRIT);
7450define_netdev_printk_level(netdev_err, KERN_ERR);
7451define_netdev_printk_level(netdev_warn, KERN_WARNING);
7452define_netdev_printk_level(netdev_notice, KERN_NOTICE);
7453define_netdev_printk_level(netdev_info, KERN_INFO);
7454
Pavel Emelyanov46650792007-10-08 20:38:39 -07007455static void __net_exit netdev_exit(struct net *net)
Eric W. Biederman881d9662007-09-17 11:56:21 -07007456{
7457 kfree(net->dev_name_head);
7458 kfree(net->dev_index_head);
7459}
7460
Denis V. Lunev022cbae2007-11-13 03:23:50 -08007461static struct pernet_operations __net_initdata netdev_net_ops = {
Eric W. Biederman881d9662007-09-17 11:56:21 -07007462 .init = netdev_init,
7463 .exit = netdev_exit,
7464};
7465
Pavel Emelyanov46650792007-10-08 20:38:39 -07007466static void __net_exit default_device_exit(struct net *net)
Eric W. Biedermance286d32007-09-12 13:53:49 +02007467{
Eric W. Biedermane008b5f2009-11-29 22:25:30 +00007468 struct net_device *dev, *aux;
Eric W. Biedermance286d32007-09-12 13:53:49 +02007469 /*
Eric W. Biedermane008b5f2009-11-29 22:25:30 +00007470 * Push all migratable network devices back to the
Eric W. Biedermance286d32007-09-12 13:53:49 +02007471 * initial network namespace
7472 */
7473 rtnl_lock();
Eric W. Biedermane008b5f2009-11-29 22:25:30 +00007474 for_each_netdev_safe(net, dev, aux) {
Eric W. Biedermance286d32007-09-12 13:53:49 +02007475 int err;
Pavel Emelyanovaca51392008-05-08 01:24:25 -07007476 char fb_name[IFNAMSIZ];
Eric W. Biedermance286d32007-09-12 13:53:49 +02007477
7478 /* Ignore unmoveable devices (i.e. loopback) */
7479 if (dev->features & NETIF_F_NETNS_LOCAL)
7480 continue;
7481
Eric W. Biedermane008b5f2009-11-29 22:25:30 +00007482 /* Leave virtual devices for the generic cleanup */
7483 if (dev->rtnl_link_ops)
7484 continue;
Eric W. Biedermand0c082c2008-11-05 15:59:38 -08007485
Lucas De Marchi25985ed2011-03-30 22:57:33 -03007486 /* Push remaining network devices to init_net */
Pavel Emelyanovaca51392008-05-08 01:24:25 -07007487 snprintf(fb_name, IFNAMSIZ, "dev%d", dev->ifindex);
7488 err = dev_change_net_namespace(dev, &init_net, fb_name);
Eric W. Biedermance286d32007-09-12 13:53:49 +02007489 if (err) {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00007490 pr_emerg("%s: failed to move %s to init_net: %d\n",
7491 __func__, dev->name, err);
Pavel Emelyanovaca51392008-05-08 01:24:25 -07007492 BUG();
Eric W. Biedermance286d32007-09-12 13:53:49 +02007493 }
7494 }
7495 rtnl_unlock();
7496}
7497
Eric W. Biederman50624c92013-09-23 21:19:49 -07007498static void __net_exit rtnl_lock_unregistering(struct list_head *net_list)
7499{
7500 /* Return with the rtnl_lock held when there are no network
7501 * devices unregistering in any network namespace in net_list.
7502 */
7503 struct net *net;
7504 bool unregistering;
Peter Zijlstraff960a72014-10-29 17:04:56 +01007505 DEFINE_WAIT_FUNC(wait, woken_wake_function);
Eric W. Biederman50624c92013-09-23 21:19:49 -07007506
Peter Zijlstraff960a72014-10-29 17:04:56 +01007507 add_wait_queue(&netdev_unregistering_wq, &wait);
Eric W. Biederman50624c92013-09-23 21:19:49 -07007508 for (;;) {
Eric W. Biederman50624c92013-09-23 21:19:49 -07007509 unregistering = false;
7510 rtnl_lock();
7511 list_for_each_entry(net, net_list, exit_list) {
7512 if (net->dev_unreg_count > 0) {
7513 unregistering = true;
7514 break;
7515 }
7516 }
7517 if (!unregistering)
7518 break;
7519 __rtnl_unlock();
Peter Zijlstraff960a72014-10-29 17:04:56 +01007520
7521 wait_woken(&wait, TASK_UNINTERRUPTIBLE, MAX_SCHEDULE_TIMEOUT);
Eric W. Biederman50624c92013-09-23 21:19:49 -07007522 }
Peter Zijlstraff960a72014-10-29 17:04:56 +01007523 remove_wait_queue(&netdev_unregistering_wq, &wait);
Eric W. Biederman50624c92013-09-23 21:19:49 -07007524}
7525
Eric W. Biederman04dc7f6b2009-12-03 02:29:04 +00007526static void __net_exit default_device_exit_batch(struct list_head *net_list)
7527{
7528 /* At exit all network devices most be removed from a network
Uwe Kleine-Königb5950762010-11-01 15:38:34 -04007529 * namespace. Do this in the reverse order of registration.
Eric W. Biederman04dc7f6b2009-12-03 02:29:04 +00007530 * Do this across as many network namespaces as possible to
7531 * improve batching efficiency.
7532 */
7533 struct net_device *dev;
7534 struct net *net;
7535 LIST_HEAD(dev_kill_list);
7536
Eric W. Biederman50624c92013-09-23 21:19:49 -07007537 /* To prevent network device cleanup code from dereferencing
7538 * loopback devices or network devices that have been freed
7539 * wait here for all pending unregistrations to complete,
7540 * before unregistring the loopback device and allowing the
7541 * network namespace be freed.
7542 *
7543 * The netdev todo list containing all network devices
7544 * unregistrations that happen in default_device_exit_batch
7545 * will run in the rtnl_unlock() at the end of
7546 * default_device_exit_batch.
7547 */
7548 rtnl_lock_unregistering(net_list);
Eric W. Biederman04dc7f6b2009-12-03 02:29:04 +00007549 list_for_each_entry(net, net_list, exit_list) {
7550 for_each_netdev_reverse(net, dev) {
Jiri Pirkob0ab2fa2014-06-26 09:58:25 +02007551 if (dev->rtnl_link_ops && dev->rtnl_link_ops->dellink)
Eric W. Biederman04dc7f6b2009-12-03 02:29:04 +00007552 dev->rtnl_link_ops->dellink(dev, &dev_kill_list);
7553 else
7554 unregister_netdevice_queue(dev, &dev_kill_list);
7555 }
7556 }
7557 unregister_netdevice_many(&dev_kill_list);
7558 rtnl_unlock();
7559}
7560
Denis V. Lunev022cbae2007-11-13 03:23:50 -08007561static struct pernet_operations __net_initdata default_device_ops = {
Eric W. Biedermance286d32007-09-12 13:53:49 +02007562 .exit = default_device_exit,
Eric W. Biederman04dc7f6b2009-12-03 02:29:04 +00007563 .exit_batch = default_device_exit_batch,
Eric W. Biedermance286d32007-09-12 13:53:49 +02007564};
7565
Linus Torvalds1da177e2005-04-16 15:20:36 -07007566/*
7567 * Initialize the DEV module. At boot time this walks the device list and
7568 * unhooks any devices that fail to initialise (normally hardware not
7569 * present) and leaves us with a valid list of present and active devices.
7570 *
7571 */
7572
7573/*
7574 * This is called single threaded during boot, so no need
7575 * to take the rtnl semaphore.
7576 */
7577static int __init net_dev_init(void)
7578{
7579 int i, rc = -ENOMEM;
7580
7581 BUG_ON(!dev_boot_phase);
7582
Linus Torvalds1da177e2005-04-16 15:20:36 -07007583 if (dev_proc_init())
7584 goto out;
7585
Eric W. Biederman8b41d182007-09-26 22:02:53 -07007586 if (netdev_kobject_init())
Linus Torvalds1da177e2005-04-16 15:20:36 -07007587 goto out;
7588
7589 INIT_LIST_HEAD(&ptype_all);
Pavel Emelyanov82d8a8672007-11-26 20:12:58 +08007590 for (i = 0; i < PTYPE_HASH_SIZE; i++)
Linus Torvalds1da177e2005-04-16 15:20:36 -07007591 INIT_LIST_HEAD(&ptype_base[i]);
7592
Vlad Yasevich62532da2012-11-15 08:49:10 +00007593 INIT_LIST_HEAD(&offload_base);
7594
Eric W. Biederman881d9662007-09-17 11:56:21 -07007595 if (register_pernet_subsys(&netdev_net_ops))
7596 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07007597
7598 /*
7599 * Initialise the packet receive queues.
7600 */
7601
KAMEZAWA Hiroyuki6f912042006-04-10 22:52:50 -07007602 for_each_possible_cpu(i) {
Eric Dumazete36fa2f2010-04-19 21:17:14 +00007603 struct softnet_data *sd = &per_cpu(softnet_data, i);
Linus Torvalds1da177e2005-04-16 15:20:36 -07007604
Eric Dumazete36fa2f2010-04-19 21:17:14 +00007605 skb_queue_head_init(&sd->input_pkt_queue);
Changli Gao6e7676c2010-04-27 15:07:33 -07007606 skb_queue_head_init(&sd->process_queue);
Eric Dumazete36fa2f2010-04-19 21:17:14 +00007607 INIT_LIST_HEAD(&sd->poll_list);
Changli Gaoa9cbd582010-04-26 23:06:24 +00007608 sd->output_queue_tailp = &sd->output_queue;
Eric Dumazetdf334542010-03-24 19:13:54 +00007609#ifdef CONFIG_RPS
Eric Dumazete36fa2f2010-04-19 21:17:14 +00007610 sd->csd.func = rps_trigger_softirq;
7611 sd->csd.info = sd;
Eric Dumazete36fa2f2010-04-19 21:17:14 +00007612 sd->cpu = i;
Tom Herbert1e94d722010-03-18 17:45:44 -07007613#endif
Tom Herbert0a9627f2010-03-16 08:03:29 +00007614
Eric Dumazete36fa2f2010-04-19 21:17:14 +00007615 sd->backlog.poll = process_backlog;
7616 sd->backlog.weight = weight_p;
Linus Torvalds1da177e2005-04-16 15:20:36 -07007617 }
7618
Linus Torvalds1da177e2005-04-16 15:20:36 -07007619 dev_boot_phase = 0;
7620
Eric W. Biederman505d4f72008-11-07 22:54:20 -08007621 /* The loopback device is special if any other network devices
7622 * is present in a network namespace the loopback device must
7623 * be present. Since we now dynamically allocate and free the
7624 * loopback device ensure this invariant is maintained by
7625 * keeping the loopback device as the first device on the
7626 * list of network devices. Ensuring the loopback devices
7627 * is the first device that appears and the last network device
7628 * that disappears.
7629 */
7630 if (register_pernet_device(&loopback_net_ops))
7631 goto out;
7632
7633 if (register_pernet_device(&default_device_ops))
7634 goto out;
7635
Carlos R. Mafra962cf362008-05-15 11:15:37 -03007636 open_softirq(NET_TX_SOFTIRQ, net_tx_action);
7637 open_softirq(NET_RX_SOFTIRQ, net_rx_action);
Linus Torvalds1da177e2005-04-16 15:20:36 -07007638
7639 hotcpu_notifier(dev_cpu_callback, 0);
7640 dst_init();
Linus Torvalds1da177e2005-04-16 15:20:36 -07007641 rc = 0;
7642out:
7643 return rc;
7644}
7645
7646subsys_initcall(net_dev_init);