blob: b2775f06c7102a00afbd170daae05468d660b5b1 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * NET3 Protocol independent device support routines.
3 *
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public License
6 * as published by the Free Software Foundation; either version
7 * 2 of the License, or (at your option) any later version.
8 *
9 * Derived from the non IP parts of dev.c 1.0.19
Jesper Juhl02c30a82005-05-05 16:16:16 -070010 * Authors: Ross Biro
Linus Torvalds1da177e2005-04-16 15:20:36 -070011 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
12 * Mark Evans, <evansmp@uhura.aston.ac.uk>
13 *
14 * Additional Authors:
15 * Florian la Roche <rzsfl@rz.uni-sb.de>
16 * Alan Cox <gw4pts@gw4pts.ampr.org>
17 * David Hinds <dahinds@users.sourceforge.net>
18 * Alexey Kuznetsov <kuznet@ms2.inr.ac.ru>
19 * Adam Sulmicki <adam@cfar.umd.edu>
20 * Pekka Riikonen <priikone@poesidon.pspt.fi>
21 *
22 * Changes:
23 * D.J. Barrow : Fixed bug where dev->refcnt gets set
24 * to 2 if register_netdev gets called
25 * before net_dev_init & also removed a
26 * few lines of code in the process.
27 * Alan Cox : device private ioctl copies fields back.
28 * Alan Cox : Transmit queue code does relevant
29 * stunts to keep the queue safe.
30 * Alan Cox : Fixed double lock.
31 * Alan Cox : Fixed promisc NULL pointer trap
32 * ???????? : Support the full private ioctl range
33 * Alan Cox : Moved ioctl permission check into
34 * drivers
35 * Tim Kordas : SIOCADDMULTI/SIOCDELMULTI
36 * Alan Cox : 100 backlog just doesn't cut it when
37 * you start doing multicast video 8)
38 * Alan Cox : Rewrote net_bh and list manager.
39 * Alan Cox : Fix ETH_P_ALL echoback lengths.
40 * Alan Cox : Took out transmit every packet pass
41 * Saved a few bytes in the ioctl handler
42 * Alan Cox : Network driver sets packet type before
43 * calling netif_rx. Saves a function
44 * call a packet.
45 * Alan Cox : Hashed net_bh()
46 * Richard Kooijman: Timestamp fixes.
47 * Alan Cox : Wrong field in SIOCGIFDSTADDR
48 * Alan Cox : Device lock protection.
49 * Alan Cox : Fixed nasty side effect of device close
50 * changes.
51 * Rudi Cilibrasi : Pass the right thing to
52 * set_mac_address()
53 * Dave Miller : 32bit quantity for the device lock to
54 * make it work out on a Sparc.
55 * Bjorn Ekwall : Added KERNELD hack.
56 * Alan Cox : Cleaned up the backlog initialise.
57 * Craig Metz : SIOCGIFCONF fix if space for under
58 * 1 device.
59 * Thomas Bogendoerfer : Return ENODEV for dev_open, if there
60 * is no device open function.
61 * Andi Kleen : Fix error reporting for SIOCGIFCONF
62 * Michael Chastain : Fix signed/unsigned for SIOCGIFCONF
63 * Cyrus Durgin : Cleaned for KMOD
64 * Adam Sulmicki : Bug Fix : Network Device Unload
65 * A network device unload needs to purge
66 * the backlog queue.
67 * Paul Rusty Russell : SIOCSIFNAME
68 * Pekka Riikonen : Netdev boot-time settings code
69 * Andrew Morton : Make unregister_netdevice wait
70 * indefinitely on dev->refcnt
71 * J Hadi Salim : - Backlog queue sampling
72 * - netif_rx() feedback
73 */
74
75#include <asm/uaccess.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070076#include <linux/bitops.h>
Randy Dunlap4fc268d2006-01-11 12:17:47 -080077#include <linux/capability.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070078#include <linux/cpu.h>
79#include <linux/types.h>
80#include <linux/kernel.h>
stephen hemminger08e98972009-11-10 07:20:34 +000081#include <linux/hash.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090082#include <linux/slab.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070083#include <linux/sched.h>
Arjan van de Ven4a3e2f72006-03-20 22:33:17 -080084#include <linux/mutex.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070085#include <linux/string.h>
86#include <linux/mm.h>
87#include <linux/socket.h>
88#include <linux/sockios.h>
89#include <linux/errno.h>
90#include <linux/interrupt.h>
91#include <linux/if_ether.h>
92#include <linux/netdevice.h>
93#include <linux/etherdevice.h>
Ben Hutchings0187bdf2008-06-19 16:15:47 -070094#include <linux/ethtool.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070095#include <linux/notifier.h>
96#include <linux/skbuff.h>
Eric W. Biederman457c4cb2007-09-12 12:01:34 +020097#include <net/net_namespace.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070098#include <net/sock.h>
99#include <linux/rtnetlink.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700100#include <linux/stat.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700101#include <net/dst.h>
102#include <net/pkt_sched.h>
103#include <net/checksum.h>
Arnd Bergmann44540962009-11-26 06:07:08 +0000104#include <net/xfrm.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700105#include <linux/highmem.h>
106#include <linux/init.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700107#include <linux/module.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700108#include <linux/netpoll.h>
109#include <linux/rcupdate.h>
110#include <linux/delay.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700111#include <net/iw_handler.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700112#include <asm/current.h>
Steve Grubb5bdb9882005-12-03 08:39:35 -0500113#include <linux/audit.h>
Chris Leechdb217332006-06-17 21:24:58 -0700114#include <linux/dmaengine.h>
Herbert Xuf6a78bf2006-06-22 02:57:17 -0700115#include <linux/err.h>
David S. Millerc7fa9d12006-08-15 16:34:13 -0700116#include <linux/ctype.h>
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700117#include <linux/if_arp.h>
Ben Hutchings6de329e2008-06-16 17:02:28 -0700118#include <linux/if_vlan.h>
David S. Miller8f0f2222008-07-15 03:47:03 -0700119#include <linux/ip.h>
Alexander Duyckad55dca2008-09-20 22:05:50 -0700120#include <net/ip.h>
Simon Horman25cd9ba2014-10-06 05:05:13 -0700121#include <net/mpls.h>
David S. Miller8f0f2222008-07-15 03:47:03 -0700122#include <linux/ipv6.h>
123#include <linux/in.h>
David S. Millerb6b2fed2008-07-21 09:48:06 -0700124#include <linux/jhash.h>
125#include <linux/random.h>
David S. Miller9cbc1cb2009-06-15 03:02:23 -0700126#include <trace/events/napi.h>
Koki Sanagicf66ba52010-08-23 18:45:02 +0900127#include <trace/events/net.h>
Koki Sanagi07dc22e2010-08-23 18:46:12 +0900128#include <trace/events/skb.h>
FUJITA Tomonori5acbbd42010-03-30 22:35:50 +0000129#include <linux/pci.h>
Stephen Rothwellcaeda9b2010-09-16 21:39:16 -0700130#include <linux/inetdevice.h>
Ben Hutchingsc4454772011-01-19 11:03:53 +0000131#include <linux/cpu_rmap.h>
Ingo Molnarc5905af2012-02-24 08:31:31 +0100132#include <linux/static_key.h>
Eliezer Tamiraf12fa62013-06-10 11:39:41 +0300133#include <linux/hashtable.h>
Eric Dumazet60877a32013-06-20 01:15:51 -0700134#include <linux/vmalloc.h>
Michal Kubeček529d0482013-11-15 06:18:50 +0100135#include <linux/if_macvlan.h>
Willem de Bruijne7fd2882014-08-04 22:11:48 -0400136#include <linux/errqueue.h>
Eric Dumazet3b47d302014-11-06 21:09:44 -0800137#include <linux/hrtimer.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700138
Pavel Emelyanov342709e2007-10-23 21:14:45 -0700139#include "net-sysfs.h"
140
Herbert Xud565b0a2008-12-15 23:38:52 -0800141/* Instead of increasing this, you should create a hash table. */
142#define MAX_GRO_SKBS 8
143
Herbert Xu5d38a072009-01-04 16:13:40 -0800144/* This should be increased if a protocol with a bigger head is added. */
145#define GRO_MAX_HEAD (MAX_HEADER + 128)
146
Linus Torvalds1da177e2005-04-16 15:20:36 -0700147static DEFINE_SPINLOCK(ptype_lock);
Vlad Yasevich62532da2012-11-15 08:49:10 +0000148static DEFINE_SPINLOCK(offload_lock);
Cong Wang900ff8c2013-02-18 19:20:33 +0000149struct list_head ptype_base[PTYPE_HASH_SIZE] __read_mostly;
150struct list_head ptype_all __read_mostly; /* Taps */
Vlad Yasevich62532da2012-11-15 08:49:10 +0000151static struct list_head offload_base __read_mostly;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700152
Ben Hutchingsae78dbf2014-01-10 22:17:24 +0000153static int netif_rx_internal(struct sk_buff *skb);
Loic Prylli5495119462014-07-01 21:39:43 -0700154static int call_netdevice_notifiers_info(unsigned long val,
155 struct net_device *dev,
156 struct netdev_notifier_info *info);
Ben Hutchingsae78dbf2014-01-10 22:17:24 +0000157
Linus Torvalds1da177e2005-04-16 15:20:36 -0700158/*
Pavel Emelianov7562f872007-05-03 15:13:45 -0700159 * The @dev_base_head list is protected by @dev_base_lock and the rtnl
Linus Torvalds1da177e2005-04-16 15:20:36 -0700160 * semaphore.
161 *
Eric Dumazetc6d14c82009-11-04 05:43:23 -0800162 * Pure readers hold dev_base_lock for reading, or rcu_read_lock()
Linus Torvalds1da177e2005-04-16 15:20:36 -0700163 *
164 * Writers must hold the rtnl semaphore while they loop through the
Pavel Emelianov7562f872007-05-03 15:13:45 -0700165 * dev_base_head list, and hold dev_base_lock for writing when they do the
Linus Torvalds1da177e2005-04-16 15:20:36 -0700166 * actual updates. This allows pure readers to access the list even
167 * while a writer is preparing to update it.
168 *
169 * To put it another way, dev_base_lock is held for writing only to
170 * protect against pure readers; the rtnl semaphore provides the
171 * protection against other writers.
172 *
173 * See, for example usages, register_netdevice() and
174 * unregister_netdevice(), which must be called with the rtnl
175 * semaphore held.
176 */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700177DEFINE_RWLOCK(dev_base_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700178EXPORT_SYMBOL(dev_base_lock);
179
Eliezer Tamiraf12fa62013-06-10 11:39:41 +0300180/* protects napi_hash addition/deletion and napi_gen_id */
181static DEFINE_SPINLOCK(napi_hash_lock);
182
183static unsigned int napi_gen_id;
184static DEFINE_HASHTABLE(napi_hash, 8);
185
Thomas Gleixner18afa4b2013-07-23 16:13:17 +0200186static seqcount_t devnet_rename_seq;
Brian Haleyc91f6df2012-11-26 05:21:08 +0000187
Thomas Graf4e985ad2011-06-21 03:11:20 +0000188static inline void dev_base_seq_inc(struct net *net)
189{
190 while (++net->dev_base_seq == 0);
191}
192
Eric W. Biederman881d9662007-09-17 11:56:21 -0700193static inline struct hlist_head *dev_name_hash(struct net *net, const char *name)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700194{
Eric Dumazet95c96172012-04-15 05:58:06 +0000195 unsigned int hash = full_name_hash(name, strnlen(name, IFNAMSIZ));
196
stephen hemminger08e98972009-11-10 07:20:34 +0000197 return &net->dev_name_head[hash_32(hash, NETDEV_HASHBITS)];
Linus Torvalds1da177e2005-04-16 15:20:36 -0700198}
199
Eric W. Biederman881d9662007-09-17 11:56:21 -0700200static inline struct hlist_head *dev_index_hash(struct net *net, int ifindex)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700201{
Eric Dumazet7c28bd02009-10-24 06:13:17 -0700202 return &net->dev_index_head[ifindex & (NETDEV_HASHENTRIES - 1)];
Linus Torvalds1da177e2005-04-16 15:20:36 -0700203}
204
Eric Dumazete36fa2f2010-04-19 21:17:14 +0000205static inline void rps_lock(struct softnet_data *sd)
Changli Gao152102c2010-03-30 20:16:22 +0000206{
207#ifdef CONFIG_RPS
Eric Dumazete36fa2f2010-04-19 21:17:14 +0000208 spin_lock(&sd->input_pkt_queue.lock);
Changli Gao152102c2010-03-30 20:16:22 +0000209#endif
210}
211
Eric Dumazete36fa2f2010-04-19 21:17:14 +0000212static inline void rps_unlock(struct softnet_data *sd)
Changli Gao152102c2010-03-30 20:16:22 +0000213{
214#ifdef CONFIG_RPS
Eric Dumazete36fa2f2010-04-19 21:17:14 +0000215 spin_unlock(&sd->input_pkt_queue.lock);
Changli Gao152102c2010-03-30 20:16:22 +0000216#endif
217}
218
Eric W. Biedermance286d32007-09-12 13:53:49 +0200219/* Device list insertion */
dingtianhong53759be2013-04-17 22:17:50 +0000220static void list_netdevice(struct net_device *dev)
Eric W. Biedermance286d32007-09-12 13:53:49 +0200221{
YOSHIFUJI Hideakic346dca2008-03-25 21:47:49 +0900222 struct net *net = dev_net(dev);
Eric W. Biedermance286d32007-09-12 13:53:49 +0200223
224 ASSERT_RTNL();
225
226 write_lock_bh(&dev_base_lock);
Eric Dumazetc6d14c82009-11-04 05:43:23 -0800227 list_add_tail_rcu(&dev->dev_list, &net->dev_base_head);
Eric Dumazet72c95282009-10-30 07:11:27 +0000228 hlist_add_head_rcu(&dev->name_hlist, dev_name_hash(net, dev->name));
Eric Dumazetfb699dfd2009-10-19 19:18:49 +0000229 hlist_add_head_rcu(&dev->index_hlist,
230 dev_index_hash(net, dev->ifindex));
Eric W. Biedermance286d32007-09-12 13:53:49 +0200231 write_unlock_bh(&dev_base_lock);
Thomas Graf4e985ad2011-06-21 03:11:20 +0000232
233 dev_base_seq_inc(net);
Eric W. Biedermance286d32007-09-12 13:53:49 +0200234}
235
Eric Dumazetfb699dfd2009-10-19 19:18:49 +0000236/* Device list removal
237 * caller must respect a RCU grace period before freeing/reusing dev
238 */
Eric W. Biedermance286d32007-09-12 13:53:49 +0200239static void unlist_netdevice(struct net_device *dev)
240{
241 ASSERT_RTNL();
242
243 /* Unlink dev from the device chain */
244 write_lock_bh(&dev_base_lock);
Eric Dumazetc6d14c82009-11-04 05:43:23 -0800245 list_del_rcu(&dev->dev_list);
Eric Dumazet72c95282009-10-30 07:11:27 +0000246 hlist_del_rcu(&dev->name_hlist);
Eric Dumazetfb699dfd2009-10-19 19:18:49 +0000247 hlist_del_rcu(&dev->index_hlist);
Eric W. Biedermance286d32007-09-12 13:53:49 +0200248 write_unlock_bh(&dev_base_lock);
Thomas Graf4e985ad2011-06-21 03:11:20 +0000249
250 dev_base_seq_inc(dev_net(dev));
Eric W. Biedermance286d32007-09-12 13:53:49 +0200251}
252
Linus Torvalds1da177e2005-04-16 15:20:36 -0700253/*
254 * Our notifier list
255 */
256
Alan Sternf07d5b92006-05-09 15:23:03 -0700257static RAW_NOTIFIER_HEAD(netdev_chain);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700258
259/*
260 * Device drivers call our routines to queue packets here. We empty the
261 * queue in the local softnet handler.
262 */
Stephen Hemmingerbea33482007-10-03 16:41:36 -0700263
Eric Dumazet9958da02010-04-17 04:17:02 +0000264DEFINE_PER_CPU_ALIGNED(struct softnet_data, softnet_data);
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700265EXPORT_PER_CPU_SYMBOL(softnet_data);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700266
David S. Millercf508b12008-07-22 14:16:42 -0700267#ifdef CONFIG_LOCKDEP
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700268/*
David S. Millerc773e842008-07-08 23:13:53 -0700269 * register_netdevice() inits txq->_xmit_lock and sets lockdep class
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700270 * according to dev->type
271 */
272static const unsigned short netdev_lock_type[] =
273 {ARPHRD_NETROM, ARPHRD_ETHER, ARPHRD_EETHER, ARPHRD_AX25,
274 ARPHRD_PRONET, ARPHRD_CHAOS, ARPHRD_IEEE802, ARPHRD_ARCNET,
275 ARPHRD_APPLETLK, ARPHRD_DLCI, ARPHRD_ATM, ARPHRD_METRICOM,
276 ARPHRD_IEEE1394, ARPHRD_EUI64, ARPHRD_INFINIBAND, ARPHRD_SLIP,
277 ARPHRD_CSLIP, ARPHRD_SLIP6, ARPHRD_CSLIP6, ARPHRD_RSRVD,
278 ARPHRD_ADAPT, ARPHRD_ROSE, ARPHRD_X25, ARPHRD_HWX25,
279 ARPHRD_PPP, ARPHRD_CISCO, ARPHRD_LAPB, ARPHRD_DDCMP,
280 ARPHRD_RAWHDLC, ARPHRD_TUNNEL, ARPHRD_TUNNEL6, ARPHRD_FRAD,
281 ARPHRD_SKIP, ARPHRD_LOOPBACK, ARPHRD_LOCALTLK, ARPHRD_FDDI,
282 ARPHRD_BIF, ARPHRD_SIT, ARPHRD_IPDDP, ARPHRD_IPGRE,
283 ARPHRD_PIMREG, ARPHRD_HIPPI, ARPHRD_ASH, ARPHRD_ECONET,
284 ARPHRD_IRDA, ARPHRD_FCPP, ARPHRD_FCAL, ARPHRD_FCPL,
Paul Gortmaker211ed862012-05-10 17:14:35 -0400285 ARPHRD_FCFABRIC, ARPHRD_IEEE80211, ARPHRD_IEEE80211_PRISM,
286 ARPHRD_IEEE80211_RADIOTAP, ARPHRD_PHONET, ARPHRD_PHONET_PIPE,
287 ARPHRD_IEEE802154, ARPHRD_VOID, ARPHRD_NONE};
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700288
Jan Engelhardt36cbd3d2009-08-05 10:42:58 -0700289static const char *const netdev_lock_name[] =
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700290 {"_xmit_NETROM", "_xmit_ETHER", "_xmit_EETHER", "_xmit_AX25",
291 "_xmit_PRONET", "_xmit_CHAOS", "_xmit_IEEE802", "_xmit_ARCNET",
292 "_xmit_APPLETLK", "_xmit_DLCI", "_xmit_ATM", "_xmit_METRICOM",
293 "_xmit_IEEE1394", "_xmit_EUI64", "_xmit_INFINIBAND", "_xmit_SLIP",
294 "_xmit_CSLIP", "_xmit_SLIP6", "_xmit_CSLIP6", "_xmit_RSRVD",
295 "_xmit_ADAPT", "_xmit_ROSE", "_xmit_X25", "_xmit_HWX25",
296 "_xmit_PPP", "_xmit_CISCO", "_xmit_LAPB", "_xmit_DDCMP",
297 "_xmit_RAWHDLC", "_xmit_TUNNEL", "_xmit_TUNNEL6", "_xmit_FRAD",
298 "_xmit_SKIP", "_xmit_LOOPBACK", "_xmit_LOCALTLK", "_xmit_FDDI",
299 "_xmit_BIF", "_xmit_SIT", "_xmit_IPDDP", "_xmit_IPGRE",
300 "_xmit_PIMREG", "_xmit_HIPPI", "_xmit_ASH", "_xmit_ECONET",
301 "_xmit_IRDA", "_xmit_FCPP", "_xmit_FCAL", "_xmit_FCPL",
Paul Gortmaker211ed862012-05-10 17:14:35 -0400302 "_xmit_FCFABRIC", "_xmit_IEEE80211", "_xmit_IEEE80211_PRISM",
303 "_xmit_IEEE80211_RADIOTAP", "_xmit_PHONET", "_xmit_PHONET_PIPE",
304 "_xmit_IEEE802154", "_xmit_VOID", "_xmit_NONE"};
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700305
306static struct lock_class_key netdev_xmit_lock_key[ARRAY_SIZE(netdev_lock_type)];
David S. Millercf508b12008-07-22 14:16:42 -0700307static struct lock_class_key netdev_addr_lock_key[ARRAY_SIZE(netdev_lock_type)];
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700308
309static inline unsigned short netdev_lock_pos(unsigned short dev_type)
310{
311 int i;
312
313 for (i = 0; i < ARRAY_SIZE(netdev_lock_type); i++)
314 if (netdev_lock_type[i] == dev_type)
315 return i;
316 /* the last key is used by default */
317 return ARRAY_SIZE(netdev_lock_type) - 1;
318}
319
David S. Millercf508b12008-07-22 14:16:42 -0700320static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock,
321 unsigned short dev_type)
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700322{
323 int i;
324
325 i = netdev_lock_pos(dev_type);
326 lockdep_set_class_and_name(lock, &netdev_xmit_lock_key[i],
327 netdev_lock_name[i]);
328}
David S. Millercf508b12008-07-22 14:16:42 -0700329
330static inline void netdev_set_addr_lockdep_class(struct net_device *dev)
331{
332 int i;
333
334 i = netdev_lock_pos(dev->type);
335 lockdep_set_class_and_name(&dev->addr_list_lock,
336 &netdev_addr_lock_key[i],
337 netdev_lock_name[i]);
338}
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700339#else
David S. Millercf508b12008-07-22 14:16:42 -0700340static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock,
341 unsigned short dev_type)
342{
343}
344static inline void netdev_set_addr_lockdep_class(struct net_device *dev)
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700345{
346}
347#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -0700348
349/*******************************************************************************
350
351 Protocol management and registration routines
352
353*******************************************************************************/
354
355/*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700356 * Add a protocol ID to the list. Now that the input handler is
357 * smarter we can dispense with all the messy stuff that used to be
358 * here.
359 *
360 * BEWARE!!! Protocol handlers, mangling input packets,
361 * MUST BE last in hash buckets and checking protocol handlers
362 * MUST start from promiscuous ptype_all chain in net_bh.
363 * It is true now, do not change it.
364 * Explanation follows: if protocol handler, mangling packet, will
365 * be the first on list, it is not able to sense, that packet
366 * is cloned and should be copied-on-write, so that it will
367 * change it and subsequent readers will get broken packet.
368 * --ANK (980803)
369 */
370
Eric Dumazetc07b68e2010-09-02 03:53:46 +0000371static inline struct list_head *ptype_head(const struct packet_type *pt)
372{
373 if (pt->type == htons(ETH_P_ALL))
Salam Noureddine7866a622015-01-27 11:35:48 -0800374 return pt->dev ? &pt->dev->ptype_all : &ptype_all;
Eric Dumazetc07b68e2010-09-02 03:53:46 +0000375 else
Salam Noureddine7866a622015-01-27 11:35:48 -0800376 return pt->dev ? &pt->dev->ptype_specific :
377 &ptype_base[ntohs(pt->type) & PTYPE_HASH_MASK];
Eric Dumazetc07b68e2010-09-02 03:53:46 +0000378}
379
Linus Torvalds1da177e2005-04-16 15:20:36 -0700380/**
381 * dev_add_pack - add packet handler
382 * @pt: packet type declaration
383 *
384 * Add a protocol handler to the networking stack. The passed &packet_type
385 * is linked into kernel lists and may not be freed until it has been
386 * removed from the kernel lists.
387 *
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +0900388 * This call does not sleep therefore it can not
Linus Torvalds1da177e2005-04-16 15:20:36 -0700389 * guarantee all CPU's that are in middle of receiving packets
390 * will see the new packet type (until the next received packet).
391 */
392
393void dev_add_pack(struct packet_type *pt)
394{
Eric Dumazetc07b68e2010-09-02 03:53:46 +0000395 struct list_head *head = ptype_head(pt);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700396
Eric Dumazetc07b68e2010-09-02 03:53:46 +0000397 spin_lock(&ptype_lock);
398 list_add_rcu(&pt->list, head);
399 spin_unlock(&ptype_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700400}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700401EXPORT_SYMBOL(dev_add_pack);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700402
Linus Torvalds1da177e2005-04-16 15:20:36 -0700403/**
404 * __dev_remove_pack - remove packet handler
405 * @pt: packet type declaration
406 *
407 * Remove a protocol handler that was previously added to the kernel
408 * protocol handlers by dev_add_pack(). The passed &packet_type is removed
409 * from the kernel lists and can be freed or reused once this function
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +0900410 * returns.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700411 *
412 * The packet type might still be in use by receivers
413 * and must not be freed until after all the CPU's have gone
414 * through a quiescent state.
415 */
416void __dev_remove_pack(struct packet_type *pt)
417{
Eric Dumazetc07b68e2010-09-02 03:53:46 +0000418 struct list_head *head = ptype_head(pt);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700419 struct packet_type *pt1;
420
Eric Dumazetc07b68e2010-09-02 03:53:46 +0000421 spin_lock(&ptype_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700422
423 list_for_each_entry(pt1, head, list) {
424 if (pt == pt1) {
425 list_del_rcu(&pt->list);
426 goto out;
427 }
428 }
429
Joe Perches7b6cd1c2012-02-01 10:54:43 +0000430 pr_warn("dev_remove_pack: %p not found\n", pt);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700431out:
Eric Dumazetc07b68e2010-09-02 03:53:46 +0000432 spin_unlock(&ptype_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700433}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700434EXPORT_SYMBOL(__dev_remove_pack);
435
Linus Torvalds1da177e2005-04-16 15:20:36 -0700436/**
437 * dev_remove_pack - remove packet handler
438 * @pt: packet type declaration
439 *
440 * Remove a protocol handler that was previously added to the kernel
441 * protocol handlers by dev_add_pack(). The passed &packet_type is removed
442 * from the kernel lists and can be freed or reused once this function
443 * returns.
444 *
445 * This call sleeps to guarantee that no CPU is looking at the packet
446 * type after return.
447 */
448void dev_remove_pack(struct packet_type *pt)
449{
450 __dev_remove_pack(pt);
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +0900451
Linus Torvalds1da177e2005-04-16 15:20:36 -0700452 synchronize_net();
453}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700454EXPORT_SYMBOL(dev_remove_pack);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700455
Vlad Yasevich62532da2012-11-15 08:49:10 +0000456
457/**
458 * dev_add_offload - register offload handlers
459 * @po: protocol offload declaration
460 *
461 * Add protocol offload handlers to the networking stack. The passed
462 * &proto_offload is linked into kernel lists and may not be freed until
463 * it has been removed from the kernel lists.
464 *
465 * This call does not sleep therefore it can not
466 * guarantee all CPU's that are in middle of receiving packets
467 * will see the new offload handlers (until the next received packet).
468 */
469void dev_add_offload(struct packet_offload *po)
470{
471 struct list_head *head = &offload_base;
472
473 spin_lock(&offload_lock);
474 list_add_rcu(&po->list, head);
475 spin_unlock(&offload_lock);
476}
477EXPORT_SYMBOL(dev_add_offload);
478
479/**
480 * __dev_remove_offload - remove offload handler
481 * @po: packet offload declaration
482 *
483 * Remove a protocol offload handler that was previously added to the
484 * kernel offload handlers by dev_add_offload(). The passed &offload_type
485 * is removed from the kernel lists and can be freed or reused once this
486 * function returns.
487 *
488 * The packet type might still be in use by receivers
489 * and must not be freed until after all the CPU's have gone
490 * through a quiescent state.
491 */
stephen hemminger1d143d92013-12-29 14:01:29 -0800492static void __dev_remove_offload(struct packet_offload *po)
Vlad Yasevich62532da2012-11-15 08:49:10 +0000493{
494 struct list_head *head = &offload_base;
495 struct packet_offload *po1;
496
Eric Dumazetc53aa502012-11-16 08:08:23 +0000497 spin_lock(&offload_lock);
Vlad Yasevich62532da2012-11-15 08:49:10 +0000498
499 list_for_each_entry(po1, head, list) {
500 if (po == po1) {
501 list_del_rcu(&po->list);
502 goto out;
503 }
504 }
505
506 pr_warn("dev_remove_offload: %p not found\n", po);
507out:
Eric Dumazetc53aa502012-11-16 08:08:23 +0000508 spin_unlock(&offload_lock);
Vlad Yasevich62532da2012-11-15 08:49:10 +0000509}
Vlad Yasevich62532da2012-11-15 08:49:10 +0000510
511/**
512 * dev_remove_offload - remove packet offload handler
513 * @po: packet offload declaration
514 *
515 * Remove a packet offload handler that was previously added to the kernel
516 * offload handlers by dev_add_offload(). The passed &offload_type is
517 * removed from the kernel lists and can be freed or reused once this
518 * function returns.
519 *
520 * This call sleeps to guarantee that no CPU is looking at the packet
521 * type after return.
522 */
523void dev_remove_offload(struct packet_offload *po)
524{
525 __dev_remove_offload(po);
526
527 synchronize_net();
528}
529EXPORT_SYMBOL(dev_remove_offload);
530
Linus Torvalds1da177e2005-04-16 15:20:36 -0700531/******************************************************************************
532
533 Device Boot-time Settings Routines
534
535*******************************************************************************/
536
537/* Boot time configuration table */
538static struct netdev_boot_setup dev_boot_setup[NETDEV_BOOT_SETUP_MAX];
539
540/**
541 * netdev_boot_setup_add - add new setup entry
542 * @name: name of the device
543 * @map: configured settings for the device
544 *
545 * Adds new setup entry to the dev_boot_setup list. The function
546 * returns 0 on error and 1 on success. This is a generic routine to
547 * all netdevices.
548 */
549static int netdev_boot_setup_add(char *name, struct ifmap *map)
550{
551 struct netdev_boot_setup *s;
552 int i;
553
554 s = dev_boot_setup;
555 for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++) {
556 if (s[i].name[0] == '\0' || s[i].name[0] == ' ') {
557 memset(s[i].name, 0, sizeof(s[i].name));
Wang Chen93b3cff2008-07-01 19:57:19 -0700558 strlcpy(s[i].name, name, IFNAMSIZ);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700559 memcpy(&s[i].map, map, sizeof(s[i].map));
560 break;
561 }
562 }
563
564 return i >= NETDEV_BOOT_SETUP_MAX ? 0 : 1;
565}
566
567/**
568 * netdev_boot_setup_check - check boot time settings
569 * @dev: the netdevice
570 *
571 * Check boot time settings for the device.
572 * The found settings are set for the device to be used
573 * later in the device probing.
574 * Returns 0 if no settings found, 1 if they are.
575 */
576int netdev_boot_setup_check(struct net_device *dev)
577{
578 struct netdev_boot_setup *s = dev_boot_setup;
579 int i;
580
581 for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++) {
582 if (s[i].name[0] != '\0' && s[i].name[0] != ' ' &&
Wang Chen93b3cff2008-07-01 19:57:19 -0700583 !strcmp(dev->name, s[i].name)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700584 dev->irq = s[i].map.irq;
585 dev->base_addr = s[i].map.base_addr;
586 dev->mem_start = s[i].map.mem_start;
587 dev->mem_end = s[i].map.mem_end;
588 return 1;
589 }
590 }
591 return 0;
592}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700593EXPORT_SYMBOL(netdev_boot_setup_check);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700594
595
596/**
597 * netdev_boot_base - get address from boot time settings
598 * @prefix: prefix for network device
599 * @unit: id for network device
600 *
601 * Check boot time settings for the base address of device.
602 * The found settings are set for the device to be used
603 * later in the device probing.
604 * Returns 0 if no settings found.
605 */
606unsigned long netdev_boot_base(const char *prefix, int unit)
607{
608 const struct netdev_boot_setup *s = dev_boot_setup;
609 char name[IFNAMSIZ];
610 int i;
611
612 sprintf(name, "%s%d", prefix, unit);
613
614 /*
615 * If device already registered then return base of 1
616 * to indicate not to probe for this interface
617 */
Eric W. Biederman881d9662007-09-17 11:56:21 -0700618 if (__dev_get_by_name(&init_net, name))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700619 return 1;
620
621 for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++)
622 if (!strcmp(name, s[i].name))
623 return s[i].map.base_addr;
624 return 0;
625}
626
627/*
628 * Saves at boot time configured settings for any netdevice.
629 */
630int __init netdev_boot_setup(char *str)
631{
632 int ints[5];
633 struct ifmap map;
634
635 str = get_options(str, ARRAY_SIZE(ints), ints);
636 if (!str || !*str)
637 return 0;
638
639 /* Save settings */
640 memset(&map, 0, sizeof(map));
641 if (ints[0] > 0)
642 map.irq = ints[1];
643 if (ints[0] > 1)
644 map.base_addr = ints[2];
645 if (ints[0] > 2)
646 map.mem_start = ints[3];
647 if (ints[0] > 3)
648 map.mem_end = ints[4];
649
650 /* Add new entry to the list */
651 return netdev_boot_setup_add(str, &map);
652}
653
654__setup("netdev=", netdev_boot_setup);
655
656/*******************************************************************************
657
658 Device Interface Subroutines
659
660*******************************************************************************/
661
662/**
Nicolas Dichtela54acb32015-04-02 17:07:00 +0200663 * dev_get_iflink - get 'iflink' value of a interface
664 * @dev: targeted interface
665 *
666 * Indicates the ifindex the interface is linked to.
667 * Physical interfaces have the same 'ifindex' and 'iflink' values.
668 */
669
670int dev_get_iflink(const struct net_device *dev)
671{
672 if (dev->netdev_ops && dev->netdev_ops->ndo_get_iflink)
673 return dev->netdev_ops->ndo_get_iflink(dev);
674
Nicolas Dichtele1622ba2015-04-02 17:07:10 +0200675 /* If dev->rtnl_link_ops is set, it's a virtual interface. */
676 if (dev->rtnl_link_ops)
677 return 0;
678
Nicolas Dichtel7a66bbc2015-04-02 17:07:09 +0200679 return dev->ifindex;
Nicolas Dichtela54acb32015-04-02 17:07:00 +0200680}
681EXPORT_SYMBOL(dev_get_iflink);
682
683/**
Linus Torvalds1da177e2005-04-16 15:20:36 -0700684 * __dev_get_by_name - find a device by its name
Randy Dunlapc4ea43c2007-10-12 21:17:49 -0700685 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -0700686 * @name: name to find
687 *
688 * Find an interface by name. Must be called under RTNL semaphore
689 * or @dev_base_lock. If the name is found a pointer to the device
690 * is returned. If the name is not found then %NULL is returned. The
691 * reference counters are not incremented so the caller must be
692 * careful with locks.
693 */
694
Eric W. Biederman881d9662007-09-17 11:56:21 -0700695struct net_device *__dev_get_by_name(struct net *net, const char *name)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700696{
Eric Dumazet0bd8d532009-10-30 01:40:11 -0700697 struct net_device *dev;
698 struct hlist_head *head = dev_name_hash(net, name);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700699
Sasha Levinb67bfe02013-02-27 17:06:00 -0800700 hlist_for_each_entry(dev, head, name_hlist)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700701 if (!strncmp(dev->name, name, IFNAMSIZ))
702 return dev;
Eric Dumazet0bd8d532009-10-30 01:40:11 -0700703
Linus Torvalds1da177e2005-04-16 15:20:36 -0700704 return NULL;
705}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700706EXPORT_SYMBOL(__dev_get_by_name);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700707
708/**
Eric Dumazet72c95282009-10-30 07:11:27 +0000709 * dev_get_by_name_rcu - find a device by its name
710 * @net: the applicable net namespace
711 * @name: name to find
712 *
713 * Find an interface by name.
714 * If the name is found a pointer to the device is returned.
715 * If the name is not found then %NULL is returned.
716 * The reference counters are not incremented so the caller must be
717 * careful with locks. The caller must hold RCU lock.
718 */
719
720struct net_device *dev_get_by_name_rcu(struct net *net, const char *name)
721{
Eric Dumazet72c95282009-10-30 07:11:27 +0000722 struct net_device *dev;
723 struct hlist_head *head = dev_name_hash(net, name);
724
Sasha Levinb67bfe02013-02-27 17:06:00 -0800725 hlist_for_each_entry_rcu(dev, head, name_hlist)
Eric Dumazet72c95282009-10-30 07:11:27 +0000726 if (!strncmp(dev->name, name, IFNAMSIZ))
727 return dev;
728
729 return NULL;
730}
731EXPORT_SYMBOL(dev_get_by_name_rcu);
732
733/**
Linus Torvalds1da177e2005-04-16 15:20:36 -0700734 * dev_get_by_name - find a device by its name
Randy Dunlapc4ea43c2007-10-12 21:17:49 -0700735 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -0700736 * @name: name to find
737 *
738 * Find an interface by name. This can be called from any
739 * context and does its own locking. The returned handle has
740 * the usage count incremented and the caller must use dev_put() to
741 * release it when it is no longer needed. %NULL is returned if no
742 * matching device is found.
743 */
744
Eric W. Biederman881d9662007-09-17 11:56:21 -0700745struct net_device *dev_get_by_name(struct net *net, const char *name)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700746{
747 struct net_device *dev;
748
Eric Dumazet72c95282009-10-30 07:11:27 +0000749 rcu_read_lock();
750 dev = dev_get_by_name_rcu(net, name);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700751 if (dev)
752 dev_hold(dev);
Eric Dumazet72c95282009-10-30 07:11:27 +0000753 rcu_read_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700754 return dev;
755}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700756EXPORT_SYMBOL(dev_get_by_name);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700757
758/**
759 * __dev_get_by_index - find a device by its ifindex
Randy Dunlapc4ea43c2007-10-12 21:17:49 -0700760 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -0700761 * @ifindex: index of device
762 *
763 * Search for an interface by index. Returns %NULL if the device
764 * is not found or a pointer to the device. The device has not
765 * had its reference counter increased so the caller must be careful
766 * about locking. The caller must hold either the RTNL semaphore
767 * or @dev_base_lock.
768 */
769
Eric W. Biederman881d9662007-09-17 11:56:21 -0700770struct net_device *__dev_get_by_index(struct net *net, int ifindex)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700771{
Eric Dumazet0bd8d532009-10-30 01:40:11 -0700772 struct net_device *dev;
773 struct hlist_head *head = dev_index_hash(net, ifindex);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700774
Sasha Levinb67bfe02013-02-27 17:06:00 -0800775 hlist_for_each_entry(dev, head, index_hlist)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700776 if (dev->ifindex == ifindex)
777 return dev;
Eric Dumazet0bd8d532009-10-30 01:40:11 -0700778
Linus Torvalds1da177e2005-04-16 15:20:36 -0700779 return NULL;
780}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700781EXPORT_SYMBOL(__dev_get_by_index);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700782
Eric Dumazetfb699dfd2009-10-19 19:18:49 +0000783/**
784 * dev_get_by_index_rcu - find a device by its ifindex
785 * @net: the applicable net namespace
786 * @ifindex: index of device
787 *
788 * Search for an interface by index. Returns %NULL if the device
789 * is not found or a pointer to the device. The device has not
790 * had its reference counter increased so the caller must be careful
791 * about locking. The caller must hold RCU lock.
792 */
793
794struct net_device *dev_get_by_index_rcu(struct net *net, int ifindex)
795{
Eric Dumazetfb699dfd2009-10-19 19:18:49 +0000796 struct net_device *dev;
797 struct hlist_head *head = dev_index_hash(net, ifindex);
798
Sasha Levinb67bfe02013-02-27 17:06:00 -0800799 hlist_for_each_entry_rcu(dev, head, index_hlist)
Eric Dumazetfb699dfd2009-10-19 19:18:49 +0000800 if (dev->ifindex == ifindex)
801 return dev;
802
803 return NULL;
804}
805EXPORT_SYMBOL(dev_get_by_index_rcu);
806
Linus Torvalds1da177e2005-04-16 15:20:36 -0700807
808/**
809 * dev_get_by_index - find a device by its ifindex
Randy Dunlapc4ea43c2007-10-12 21:17:49 -0700810 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -0700811 * @ifindex: index of device
812 *
813 * Search for an interface by index. Returns NULL if the device
814 * is not found or a pointer to the device. The device returned has
815 * had a reference added and the pointer is safe until the user calls
816 * dev_put to indicate they have finished with it.
817 */
818
Eric W. Biederman881d9662007-09-17 11:56:21 -0700819struct net_device *dev_get_by_index(struct net *net, int ifindex)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700820{
821 struct net_device *dev;
822
Eric Dumazetfb699dfd2009-10-19 19:18:49 +0000823 rcu_read_lock();
824 dev = dev_get_by_index_rcu(net, ifindex);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700825 if (dev)
826 dev_hold(dev);
Eric Dumazetfb699dfd2009-10-19 19:18:49 +0000827 rcu_read_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700828 return dev;
829}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700830EXPORT_SYMBOL(dev_get_by_index);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700831
832/**
Nicolas Schichan5dbe7c12013-06-26 17:23:42 +0200833 * netdev_get_name - get a netdevice name, knowing its ifindex.
834 * @net: network namespace
835 * @name: a pointer to the buffer where the name will be stored.
836 * @ifindex: the ifindex of the interface to get the name from.
837 *
838 * The use of raw_seqcount_begin() and cond_resched() before
839 * retrying is required as we want to give the writers a chance
840 * to complete when CONFIG_PREEMPT is not set.
841 */
842int netdev_get_name(struct net *net, char *name, int ifindex)
843{
844 struct net_device *dev;
845 unsigned int seq;
846
847retry:
848 seq = raw_seqcount_begin(&devnet_rename_seq);
849 rcu_read_lock();
850 dev = dev_get_by_index_rcu(net, ifindex);
851 if (!dev) {
852 rcu_read_unlock();
853 return -ENODEV;
854 }
855
856 strcpy(name, dev->name);
857 rcu_read_unlock();
858 if (read_seqcount_retry(&devnet_rename_seq, seq)) {
859 cond_resched();
860 goto retry;
861 }
862
863 return 0;
864}
865
866/**
Eric Dumazet941666c2010-12-05 01:23:53 +0000867 * dev_getbyhwaddr_rcu - find a device by its hardware address
Randy Dunlapc4ea43c2007-10-12 21:17:49 -0700868 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -0700869 * @type: media type of device
870 * @ha: hardware address
871 *
872 * Search for an interface by MAC address. Returns NULL if the device
Eric Dumazetc5066532011-01-24 13:16:16 -0800873 * is not found or a pointer to the device.
874 * The caller must hold RCU or RTNL.
Eric Dumazet941666c2010-12-05 01:23:53 +0000875 * The returned device has not had its ref count increased
Linus Torvalds1da177e2005-04-16 15:20:36 -0700876 * and the caller must therefore be careful about locking
877 *
Linus Torvalds1da177e2005-04-16 15:20:36 -0700878 */
879
Eric Dumazet941666c2010-12-05 01:23:53 +0000880struct net_device *dev_getbyhwaddr_rcu(struct net *net, unsigned short type,
881 const char *ha)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700882{
883 struct net_device *dev;
884
Eric Dumazet941666c2010-12-05 01:23:53 +0000885 for_each_netdev_rcu(net, dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700886 if (dev->type == type &&
887 !memcmp(dev->dev_addr, ha, dev->addr_len))
Pavel Emelianov7562f872007-05-03 15:13:45 -0700888 return dev;
889
890 return NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700891}
Eric Dumazet941666c2010-12-05 01:23:53 +0000892EXPORT_SYMBOL(dev_getbyhwaddr_rcu);
Jochen Friedrichcf309e32005-09-22 04:44:55 -0300893
Eric W. Biederman881d9662007-09-17 11:56:21 -0700894struct net_device *__dev_getfirstbyhwtype(struct net *net, unsigned short type)
Patrick McHardy4e9cac22007-05-03 03:28:13 -0700895{
896 struct net_device *dev;
897
898 ASSERT_RTNL();
Eric W. Biederman881d9662007-09-17 11:56:21 -0700899 for_each_netdev(net, dev)
Patrick McHardy4e9cac22007-05-03 03:28:13 -0700900 if (dev->type == type)
Pavel Emelianov7562f872007-05-03 15:13:45 -0700901 return dev;
902
903 return NULL;
Patrick McHardy4e9cac22007-05-03 03:28:13 -0700904}
Patrick McHardy4e9cac22007-05-03 03:28:13 -0700905EXPORT_SYMBOL(__dev_getfirstbyhwtype);
906
Eric W. Biederman881d9662007-09-17 11:56:21 -0700907struct net_device *dev_getfirstbyhwtype(struct net *net, unsigned short type)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700908{
Eric Dumazet99fe3c32010-03-18 11:27:25 +0000909 struct net_device *dev, *ret = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700910
Eric Dumazet99fe3c32010-03-18 11:27:25 +0000911 rcu_read_lock();
912 for_each_netdev_rcu(net, dev)
913 if (dev->type == type) {
914 dev_hold(dev);
915 ret = dev;
916 break;
917 }
918 rcu_read_unlock();
919 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700920}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700921EXPORT_SYMBOL(dev_getfirstbyhwtype);
922
923/**
WANG Cong6c555492014-09-11 15:35:09 -0700924 * __dev_get_by_flags - find any device with given flags
Randy Dunlapc4ea43c2007-10-12 21:17:49 -0700925 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -0700926 * @if_flags: IFF_* values
927 * @mask: bitmask of bits in if_flags to check
928 *
929 * Search for any interface with the given flags. Returns NULL if a device
Eric Dumazetbb69ae02010-06-07 11:42:13 +0000930 * is not found or a pointer to the device. Must be called inside
WANG Cong6c555492014-09-11 15:35:09 -0700931 * rtnl_lock(), and result refcount is unchanged.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700932 */
933
WANG Cong6c555492014-09-11 15:35:09 -0700934struct net_device *__dev_get_by_flags(struct net *net, unsigned short if_flags,
935 unsigned short mask)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700936{
Pavel Emelianov7562f872007-05-03 15:13:45 -0700937 struct net_device *dev, *ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700938
WANG Cong6c555492014-09-11 15:35:09 -0700939 ASSERT_RTNL();
940
Pavel Emelianov7562f872007-05-03 15:13:45 -0700941 ret = NULL;
WANG Cong6c555492014-09-11 15:35:09 -0700942 for_each_netdev(net, dev) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700943 if (((dev->flags ^ if_flags) & mask) == 0) {
Pavel Emelianov7562f872007-05-03 15:13:45 -0700944 ret = dev;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700945 break;
946 }
947 }
Pavel Emelianov7562f872007-05-03 15:13:45 -0700948 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700949}
WANG Cong6c555492014-09-11 15:35:09 -0700950EXPORT_SYMBOL(__dev_get_by_flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700951
952/**
953 * dev_valid_name - check if name is okay for network device
954 * @name: name string
955 *
956 * Network device names need to be valid file names to
David S. Millerc7fa9d12006-08-15 16:34:13 -0700957 * to allow sysfs to work. We also disallow any kind of
958 * whitespace.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700959 */
David S. Miller95f050b2012-03-06 16:12:15 -0500960bool dev_valid_name(const char *name)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700961{
David S. Millerc7fa9d12006-08-15 16:34:13 -0700962 if (*name == '\0')
David S. Miller95f050b2012-03-06 16:12:15 -0500963 return false;
Stephen Hemmingerb6fe17d2006-08-29 17:06:13 -0700964 if (strlen(name) >= IFNAMSIZ)
David S. Miller95f050b2012-03-06 16:12:15 -0500965 return false;
David S. Millerc7fa9d12006-08-15 16:34:13 -0700966 if (!strcmp(name, ".") || !strcmp(name, ".."))
David S. Miller95f050b2012-03-06 16:12:15 -0500967 return false;
David S. Millerc7fa9d12006-08-15 16:34:13 -0700968
969 while (*name) {
Matthew Thodea4176a92015-02-17 18:31:57 -0600970 if (*name == '/' || *name == ':' || isspace(*name))
David S. Miller95f050b2012-03-06 16:12:15 -0500971 return false;
David S. Millerc7fa9d12006-08-15 16:34:13 -0700972 name++;
973 }
David S. Miller95f050b2012-03-06 16:12:15 -0500974 return true;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700975}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700976EXPORT_SYMBOL(dev_valid_name);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700977
978/**
Eric W. Biedermanb267b172007-09-12 13:48:45 +0200979 * __dev_alloc_name - allocate a name for a device
980 * @net: network namespace to allocate the device name in
Linus Torvalds1da177e2005-04-16 15:20:36 -0700981 * @name: name format string
Eric W. Biedermanb267b172007-09-12 13:48:45 +0200982 * @buf: scratch buffer and result name string
Linus Torvalds1da177e2005-04-16 15:20:36 -0700983 *
984 * Passed a format string - eg "lt%d" it will try and find a suitable
Stephen Hemminger3041a062006-05-26 13:25:24 -0700985 * id. It scans list of devices to build up a free map, then chooses
986 * the first empty slot. The caller must hold the dev_base or rtnl lock
987 * while allocating the name and adding the device in order to avoid
988 * duplicates.
989 * Limited to bits_per_byte * page size devices (ie 32K on most platforms).
990 * Returns the number of the unit assigned or a negative errno code.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700991 */
992
Eric W. Biedermanb267b172007-09-12 13:48:45 +0200993static int __dev_alloc_name(struct net *net, const char *name, char *buf)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700994{
995 int i = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700996 const char *p;
997 const int max_netdevices = 8*PAGE_SIZE;
Stephen Hemmingercfcabdc2007-10-09 01:59:42 -0700998 unsigned long *inuse;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700999 struct net_device *d;
1000
1001 p = strnchr(name, IFNAMSIZ-1, '%');
1002 if (p) {
1003 /*
1004 * Verify the string as this thing may have come from
1005 * the user. There must be either one "%d" and no other "%"
1006 * characters.
1007 */
1008 if (p[1] != 'd' || strchr(p + 2, '%'))
1009 return -EINVAL;
1010
1011 /* Use one page as a bit array of possible slots */
Stephen Hemmingercfcabdc2007-10-09 01:59:42 -07001012 inuse = (unsigned long *) get_zeroed_page(GFP_ATOMIC);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001013 if (!inuse)
1014 return -ENOMEM;
1015
Eric W. Biederman881d9662007-09-17 11:56:21 -07001016 for_each_netdev(net, d) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001017 if (!sscanf(d->name, name, &i))
1018 continue;
1019 if (i < 0 || i >= max_netdevices)
1020 continue;
1021
1022 /* avoid cases where sscanf is not exact inverse of printf */
Eric W. Biedermanb267b172007-09-12 13:48:45 +02001023 snprintf(buf, IFNAMSIZ, name, i);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001024 if (!strncmp(buf, d->name, IFNAMSIZ))
1025 set_bit(i, inuse);
1026 }
1027
1028 i = find_first_zero_bit(inuse, max_netdevices);
1029 free_page((unsigned long) inuse);
1030 }
1031
Octavian Purdilad9031022009-11-18 02:36:59 +00001032 if (buf != name)
1033 snprintf(buf, IFNAMSIZ, name, i);
Eric W. Biedermanb267b172007-09-12 13:48:45 +02001034 if (!__dev_get_by_name(net, buf))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001035 return i;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001036
1037 /* It is possible to run out of possible slots
1038 * when the name is long and there isn't enough space left
1039 * for the digits, or if all bits are used.
1040 */
1041 return -ENFILE;
1042}
1043
Eric W. Biedermanb267b172007-09-12 13:48:45 +02001044/**
1045 * dev_alloc_name - allocate a name for a device
1046 * @dev: device
1047 * @name: name format string
1048 *
1049 * Passed a format string - eg "lt%d" it will try and find a suitable
1050 * id. It scans list of devices to build up a free map, then chooses
1051 * the first empty slot. The caller must hold the dev_base or rtnl lock
1052 * while allocating the name and adding the device in order to avoid
1053 * duplicates.
1054 * Limited to bits_per_byte * page size devices (ie 32K on most platforms).
1055 * Returns the number of the unit assigned or a negative errno code.
1056 */
1057
1058int dev_alloc_name(struct net_device *dev, const char *name)
1059{
1060 char buf[IFNAMSIZ];
1061 struct net *net;
1062 int ret;
1063
YOSHIFUJI Hideakic346dca2008-03-25 21:47:49 +09001064 BUG_ON(!dev_net(dev));
1065 net = dev_net(dev);
Eric W. Biedermanb267b172007-09-12 13:48:45 +02001066 ret = __dev_alloc_name(net, name, buf);
1067 if (ret >= 0)
1068 strlcpy(dev->name, buf, IFNAMSIZ);
1069 return ret;
1070}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001071EXPORT_SYMBOL(dev_alloc_name);
Eric W. Biedermanb267b172007-09-12 13:48:45 +02001072
Gao feng828de4f2012-09-13 20:58:27 +00001073static int dev_alloc_name_ns(struct net *net,
1074 struct net_device *dev,
1075 const char *name)
Octavian Purdilad9031022009-11-18 02:36:59 +00001076{
Gao feng828de4f2012-09-13 20:58:27 +00001077 char buf[IFNAMSIZ];
1078 int ret;
Daniel Lezcano8ce6cebc2010-05-19 10:12:19 +00001079
Gao feng828de4f2012-09-13 20:58:27 +00001080 ret = __dev_alloc_name(net, name, buf);
1081 if (ret >= 0)
1082 strlcpy(dev->name, buf, IFNAMSIZ);
1083 return ret;
1084}
1085
1086static int dev_get_valid_name(struct net *net,
1087 struct net_device *dev,
1088 const char *name)
1089{
1090 BUG_ON(!net);
Daniel Lezcano8ce6cebc2010-05-19 10:12:19 +00001091
Octavian Purdilad9031022009-11-18 02:36:59 +00001092 if (!dev_valid_name(name))
1093 return -EINVAL;
1094
Jiri Pirko1c5cae82011-04-30 01:21:32 +00001095 if (strchr(name, '%'))
Gao feng828de4f2012-09-13 20:58:27 +00001096 return dev_alloc_name_ns(net, dev, name);
Octavian Purdilad9031022009-11-18 02:36:59 +00001097 else if (__dev_get_by_name(net, name))
1098 return -EEXIST;
Daniel Lezcano8ce6cebc2010-05-19 10:12:19 +00001099 else if (dev->name != name)
1100 strlcpy(dev->name, name, IFNAMSIZ);
Octavian Purdilad9031022009-11-18 02:36:59 +00001101
1102 return 0;
1103}
Linus Torvalds1da177e2005-04-16 15:20:36 -07001104
1105/**
1106 * dev_change_name - change name of a device
1107 * @dev: device
1108 * @newname: name (or format string) must be at least IFNAMSIZ
1109 *
1110 * Change name of a device, can pass format strings "eth%d".
1111 * for wildcarding.
1112 */
Stephen Hemmingercf04a4c72008-09-30 02:22:14 -07001113int dev_change_name(struct net_device *dev, const char *newname)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001114{
Tom Gundersen238fa362014-07-14 16:37:23 +02001115 unsigned char old_assign_type;
Herbert Xufcc5a032007-07-30 17:03:38 -07001116 char oldname[IFNAMSIZ];
Linus Torvalds1da177e2005-04-16 15:20:36 -07001117 int err = 0;
Herbert Xufcc5a032007-07-30 17:03:38 -07001118 int ret;
Eric W. Biederman881d9662007-09-17 11:56:21 -07001119 struct net *net;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001120
1121 ASSERT_RTNL();
YOSHIFUJI Hideakic346dca2008-03-25 21:47:49 +09001122 BUG_ON(!dev_net(dev));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001123
YOSHIFUJI Hideakic346dca2008-03-25 21:47:49 +09001124 net = dev_net(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001125 if (dev->flags & IFF_UP)
1126 return -EBUSY;
1127
Eric Dumazet30e6c9f2012-12-20 17:25:08 +00001128 write_seqcount_begin(&devnet_rename_seq);
Brian Haleyc91f6df2012-11-26 05:21:08 +00001129
1130 if (strncmp(newname, dev->name, IFNAMSIZ) == 0) {
Eric Dumazet30e6c9f2012-12-20 17:25:08 +00001131 write_seqcount_end(&devnet_rename_seq);
Stephen Hemmingerc8d90dc2007-10-26 03:53:42 -07001132 return 0;
Brian Haleyc91f6df2012-11-26 05:21:08 +00001133 }
Stephen Hemmingerc8d90dc2007-10-26 03:53:42 -07001134
Herbert Xufcc5a032007-07-30 17:03:38 -07001135 memcpy(oldname, dev->name, IFNAMSIZ);
1136
Gao feng828de4f2012-09-13 20:58:27 +00001137 err = dev_get_valid_name(net, dev, newname);
Brian Haleyc91f6df2012-11-26 05:21:08 +00001138 if (err < 0) {
Eric Dumazet30e6c9f2012-12-20 17:25:08 +00001139 write_seqcount_end(&devnet_rename_seq);
Octavian Purdilad9031022009-11-18 02:36:59 +00001140 return err;
Brian Haleyc91f6df2012-11-26 05:21:08 +00001141 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001142
Veaceslav Falico6fe82a32014-07-17 20:33:32 +02001143 if (oldname[0] && !strchr(oldname, '%'))
1144 netdev_info(dev, "renamed from %s\n", oldname);
1145
Tom Gundersen238fa362014-07-14 16:37:23 +02001146 old_assign_type = dev->name_assign_type;
1147 dev->name_assign_type = NET_NAME_RENAMED;
1148
Herbert Xufcc5a032007-07-30 17:03:38 -07001149rollback:
Eric W. Biedermana1b3f592010-05-04 17:36:49 -07001150 ret = device_rename(&dev->dev, dev->name);
1151 if (ret) {
1152 memcpy(dev->name, oldname, IFNAMSIZ);
Tom Gundersen238fa362014-07-14 16:37:23 +02001153 dev->name_assign_type = old_assign_type;
Eric Dumazet30e6c9f2012-12-20 17:25:08 +00001154 write_seqcount_end(&devnet_rename_seq);
Eric W. Biedermana1b3f592010-05-04 17:36:49 -07001155 return ret;
Stephen Hemmingerdcc99772008-05-14 22:33:38 -07001156 }
Herbert Xu7f988ea2007-07-30 16:35:46 -07001157
Eric Dumazet30e6c9f2012-12-20 17:25:08 +00001158 write_seqcount_end(&devnet_rename_seq);
Brian Haleyc91f6df2012-11-26 05:21:08 +00001159
Veaceslav Falico5bb025f2014-01-14 21:58:51 +01001160 netdev_adjacent_rename_links(dev, oldname);
1161
Herbert Xu7f988ea2007-07-30 16:35:46 -07001162 write_lock_bh(&dev_base_lock);
Eric Dumazet372b2312011-05-17 13:56:59 -04001163 hlist_del_rcu(&dev->name_hlist);
Eric Dumazet72c95282009-10-30 07:11:27 +00001164 write_unlock_bh(&dev_base_lock);
1165
1166 synchronize_rcu();
1167
1168 write_lock_bh(&dev_base_lock);
1169 hlist_add_head_rcu(&dev->name_hlist, dev_name_hash(net, dev->name));
Herbert Xu7f988ea2007-07-30 16:35:46 -07001170 write_unlock_bh(&dev_base_lock);
1171
Pavel Emelyanov056925a2007-09-16 15:42:43 -07001172 ret = call_netdevice_notifiers(NETDEV_CHANGENAME, dev);
Herbert Xufcc5a032007-07-30 17:03:38 -07001173 ret = notifier_to_errno(ret);
1174
1175 if (ret) {
Eric Dumazet91e9c07b2009-11-15 23:30:24 +00001176 /* err >= 0 after dev_alloc_name() or stores the first errno */
1177 if (err >= 0) {
Herbert Xufcc5a032007-07-30 17:03:38 -07001178 err = ret;
Eric Dumazet30e6c9f2012-12-20 17:25:08 +00001179 write_seqcount_begin(&devnet_rename_seq);
Herbert Xufcc5a032007-07-30 17:03:38 -07001180 memcpy(dev->name, oldname, IFNAMSIZ);
Veaceslav Falico5bb025f2014-01-14 21:58:51 +01001181 memcpy(oldname, newname, IFNAMSIZ);
Tom Gundersen238fa362014-07-14 16:37:23 +02001182 dev->name_assign_type = old_assign_type;
1183 old_assign_type = NET_NAME_RENAMED;
Herbert Xufcc5a032007-07-30 17:03:38 -07001184 goto rollback;
Eric Dumazet91e9c07b2009-11-15 23:30:24 +00001185 } else {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00001186 pr_err("%s: name change rollback failed: %d\n",
Eric Dumazet91e9c07b2009-11-15 23:30:24 +00001187 dev->name, ret);
Herbert Xufcc5a032007-07-30 17:03:38 -07001188 }
1189 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001190
1191 return err;
1192}
1193
1194/**
Stephen Hemminger0b815a12008-09-22 21:28:11 -07001195 * dev_set_alias - change ifalias of a device
1196 * @dev: device
1197 * @alias: name up to IFALIASZ
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07001198 * @len: limit of bytes to copy from info
Stephen Hemminger0b815a12008-09-22 21:28:11 -07001199 *
1200 * Set ifalias for a device,
1201 */
1202int dev_set_alias(struct net_device *dev, const char *alias, size_t len)
1203{
Alexey Khoroshilov7364e442012-08-08 00:33:25 +00001204 char *new_ifalias;
1205
Stephen Hemminger0b815a12008-09-22 21:28:11 -07001206 ASSERT_RTNL();
1207
1208 if (len >= IFALIASZ)
1209 return -EINVAL;
1210
Oliver Hartkopp96ca4a22008-09-23 21:23:19 -07001211 if (!len) {
Sachin Kamat388dfc22012-11-20 00:57:04 +00001212 kfree(dev->ifalias);
1213 dev->ifalias = NULL;
Oliver Hartkopp96ca4a22008-09-23 21:23:19 -07001214 return 0;
1215 }
1216
Alexey Khoroshilov7364e442012-08-08 00:33:25 +00001217 new_ifalias = krealloc(dev->ifalias, len + 1, GFP_KERNEL);
1218 if (!new_ifalias)
Stephen Hemminger0b815a12008-09-22 21:28:11 -07001219 return -ENOMEM;
Alexey Khoroshilov7364e442012-08-08 00:33:25 +00001220 dev->ifalias = new_ifalias;
Stephen Hemminger0b815a12008-09-22 21:28:11 -07001221
1222 strlcpy(dev->ifalias, alias, len+1);
1223 return len;
1224}
1225
1226
1227/**
Stephen Hemminger3041a062006-05-26 13:25:24 -07001228 * netdev_features_change - device changes features
Stephen Hemmingerd8a33ac2005-05-29 14:13:47 -07001229 * @dev: device to cause notification
1230 *
1231 * Called to indicate a device has changed features.
1232 */
1233void netdev_features_change(struct net_device *dev)
1234{
Pavel Emelyanov056925a2007-09-16 15:42:43 -07001235 call_netdevice_notifiers(NETDEV_FEAT_CHANGE, dev);
Stephen Hemmingerd8a33ac2005-05-29 14:13:47 -07001236}
1237EXPORT_SYMBOL(netdev_features_change);
1238
1239/**
Linus Torvalds1da177e2005-04-16 15:20:36 -07001240 * netdev_state_change - device changes state
1241 * @dev: device to cause notification
1242 *
1243 * Called to indicate a device has changed state. This function calls
1244 * the notifier chains for netdev_chain and sends a NEWLINK message
1245 * to the routing socket.
1246 */
1247void netdev_state_change(struct net_device *dev)
1248{
1249 if (dev->flags & IFF_UP) {
Loic Prylli5495119462014-07-01 21:39:43 -07001250 struct netdev_notifier_change_info change_info;
1251
1252 change_info.flags_changed = 0;
1253 call_netdevice_notifiers_info(NETDEV_CHANGE, dev,
1254 &change_info.info);
Alexei Starovoitov7f294052013-10-23 16:02:42 -07001255 rtmsg_ifinfo(RTM_NEWLINK, dev, 0, GFP_KERNEL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001256 }
1257}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001258EXPORT_SYMBOL(netdev_state_change);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001259
Amerigo Wangee89bab2012-08-09 22:14:56 +00001260/**
1261 * netdev_notify_peers - notify network peers about existence of @dev
1262 * @dev: network device
1263 *
1264 * Generate traffic such that interested network peers are aware of
1265 * @dev, such as by generating a gratuitous ARP. This may be used when
1266 * a device wants to inform the rest of the network about some sort of
1267 * reconfiguration such as a failover event or virtual machine
1268 * migration.
1269 */
1270void netdev_notify_peers(struct net_device *dev)
Or Gerlitzc1da4ac2008-06-13 18:12:00 -07001271{
Amerigo Wangee89bab2012-08-09 22:14:56 +00001272 rtnl_lock();
1273 call_netdevice_notifiers(NETDEV_NOTIFY_PEERS, dev);
1274 rtnl_unlock();
Or Gerlitzc1da4ac2008-06-13 18:12:00 -07001275}
Amerigo Wangee89bab2012-08-09 22:14:56 +00001276EXPORT_SYMBOL(netdev_notify_peers);
Or Gerlitzc1da4ac2008-06-13 18:12:00 -07001277
Patrick McHardybd380812010-02-26 06:34:53 +00001278static int __dev_open(struct net_device *dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001279{
Stephen Hemmingerd3147742008-11-19 21:32:24 -08001280 const struct net_device_ops *ops = dev->netdev_ops;
Johannes Berg3b8bcfd2009-05-30 01:39:53 +02001281 int ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001282
Ben Hutchingse46b66b2008-05-08 02:53:17 -07001283 ASSERT_RTNL();
1284
Linus Torvalds1da177e2005-04-16 15:20:36 -07001285 if (!netif_device_present(dev))
1286 return -ENODEV;
1287
Neil Hormanca99ca12013-02-05 08:05:43 +00001288 /* Block netpoll from trying to do any rx path servicing.
1289 * If we don't do this there is a chance ndo_poll_controller
1290 * or ndo_poll may be running while we open the device
1291 */
Eric W. Biederman66b55522014-03-27 15:39:03 -07001292 netpoll_poll_disable(dev);
Neil Hormanca99ca12013-02-05 08:05:43 +00001293
Johannes Berg3b8bcfd2009-05-30 01:39:53 +02001294 ret = call_netdevice_notifiers(NETDEV_PRE_UP, dev);
1295 ret = notifier_to_errno(ret);
1296 if (ret)
1297 return ret;
1298
Linus Torvalds1da177e2005-04-16 15:20:36 -07001299 set_bit(__LINK_STATE_START, &dev->state);
Jeff Garzikbada3392007-10-23 20:19:37 -07001300
Stephen Hemmingerd3147742008-11-19 21:32:24 -08001301 if (ops->ndo_validate_addr)
1302 ret = ops->ndo_validate_addr(dev);
Jeff Garzikbada3392007-10-23 20:19:37 -07001303
Stephen Hemmingerd3147742008-11-19 21:32:24 -08001304 if (!ret && ops->ndo_open)
1305 ret = ops->ndo_open(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001306
Eric W. Biederman66b55522014-03-27 15:39:03 -07001307 netpoll_poll_enable(dev);
Neil Hormanca99ca12013-02-05 08:05:43 +00001308
Jeff Garzikbada3392007-10-23 20:19:37 -07001309 if (ret)
1310 clear_bit(__LINK_STATE_START, &dev->state);
1311 else {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001312 dev->flags |= IFF_UP;
Patrick McHardy4417da62007-06-27 01:28:10 -07001313 dev_set_rx_mode(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001314 dev_activate(dev);
Theodore Ts'o7bf23572012-07-04 21:23:25 -04001315 add_device_randomness(dev->dev_addr, dev->addr_len);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001316 }
Jeff Garzikbada3392007-10-23 20:19:37 -07001317
Linus Torvalds1da177e2005-04-16 15:20:36 -07001318 return ret;
1319}
Patrick McHardybd380812010-02-26 06:34:53 +00001320
1321/**
1322 * dev_open - prepare an interface for use.
1323 * @dev: device to open
1324 *
1325 * Takes a device from down to up state. The device's private open
1326 * function is invoked and then the multicast lists are loaded. Finally
1327 * the device is moved into the up state and a %NETDEV_UP message is
1328 * sent to the netdev notifier chain.
1329 *
1330 * Calling this function on an active interface is a nop. On a failure
1331 * a negative errno code is returned.
1332 */
1333int dev_open(struct net_device *dev)
1334{
1335 int ret;
1336
Patrick McHardybd380812010-02-26 06:34:53 +00001337 if (dev->flags & IFF_UP)
1338 return 0;
1339
Patrick McHardybd380812010-02-26 06:34:53 +00001340 ret = __dev_open(dev);
1341 if (ret < 0)
1342 return ret;
1343
Alexei Starovoitov7f294052013-10-23 16:02:42 -07001344 rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP|IFF_RUNNING, GFP_KERNEL);
Patrick McHardybd380812010-02-26 06:34:53 +00001345 call_netdevice_notifiers(NETDEV_UP, dev);
1346
1347 return ret;
1348}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001349EXPORT_SYMBOL(dev_open);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001350
Octavian Purdila44345722010-12-13 12:44:07 +00001351static int __dev_close_many(struct list_head *head)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001352{
Octavian Purdila44345722010-12-13 12:44:07 +00001353 struct net_device *dev;
Patrick McHardybd380812010-02-26 06:34:53 +00001354
Ben Hutchingse46b66b2008-05-08 02:53:17 -07001355 ASSERT_RTNL();
David S. Miller9d5010d2007-09-12 14:33:25 +02001356 might_sleep();
1357
Eric W. Biederman5cde2822013-10-05 19:26:05 -07001358 list_for_each_entry(dev, head, close_list) {
Eric W. Biederman3f4df202014-03-27 15:38:17 -07001359 /* Temporarily disable netpoll until the interface is down */
Eric W. Biederman66b55522014-03-27 15:39:03 -07001360 netpoll_poll_disable(dev);
Eric W. Biederman3f4df202014-03-27 15:38:17 -07001361
Octavian Purdila44345722010-12-13 12:44:07 +00001362 call_netdevice_notifiers(NETDEV_GOING_DOWN, dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001363
Octavian Purdila44345722010-12-13 12:44:07 +00001364 clear_bit(__LINK_STATE_START, &dev->state);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001365
Octavian Purdila44345722010-12-13 12:44:07 +00001366 /* Synchronize to scheduled poll. We cannot touch poll list, it
1367 * can be even on different cpu. So just clear netif_running().
1368 *
1369 * dev->stop() will invoke napi_disable() on all of it's
1370 * napi_struct instances on this device.
1371 */
Peter Zijlstra4e857c52014-03-17 18:06:10 +01001372 smp_mb__after_atomic(); /* Commit netif_running(). */
Octavian Purdila44345722010-12-13 12:44:07 +00001373 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001374
Octavian Purdila44345722010-12-13 12:44:07 +00001375 dev_deactivate_many(head);
1376
Eric W. Biederman5cde2822013-10-05 19:26:05 -07001377 list_for_each_entry(dev, head, close_list) {
Octavian Purdila44345722010-12-13 12:44:07 +00001378 const struct net_device_ops *ops = dev->netdev_ops;
1379
1380 /*
1381 * Call the device specific close. This cannot fail.
1382 * Only if device is UP
1383 *
1384 * We allow it to be called even after a DETACH hot-plug
1385 * event.
1386 */
1387 if (ops->ndo_stop)
1388 ops->ndo_stop(dev);
1389
Octavian Purdila44345722010-12-13 12:44:07 +00001390 dev->flags &= ~IFF_UP;
Eric W. Biederman66b55522014-03-27 15:39:03 -07001391 netpoll_poll_enable(dev);
Octavian Purdila44345722010-12-13 12:44:07 +00001392 }
1393
1394 return 0;
1395}
1396
1397static int __dev_close(struct net_device *dev)
1398{
Linus Torvaldsf87e6f42011-02-17 22:54:38 +00001399 int retval;
Octavian Purdila44345722010-12-13 12:44:07 +00001400 LIST_HEAD(single);
1401
Eric W. Biederman5cde2822013-10-05 19:26:05 -07001402 list_add(&dev->close_list, &single);
Linus Torvaldsf87e6f42011-02-17 22:54:38 +00001403 retval = __dev_close_many(&single);
1404 list_del(&single);
Neil Hormanca99ca12013-02-05 08:05:43 +00001405
Linus Torvaldsf87e6f42011-02-17 22:54:38 +00001406 return retval;
Octavian Purdila44345722010-12-13 12:44:07 +00001407}
1408
David S. Miller99c4a262015-03-18 22:52:33 -04001409int dev_close_many(struct list_head *head, bool unlink)
Octavian Purdila44345722010-12-13 12:44:07 +00001410{
1411 struct net_device *dev, *tmp;
Octavian Purdila44345722010-12-13 12:44:07 +00001412
Eric W. Biederman5cde2822013-10-05 19:26:05 -07001413 /* Remove the devices that don't need to be closed */
1414 list_for_each_entry_safe(dev, tmp, head, close_list)
Octavian Purdila44345722010-12-13 12:44:07 +00001415 if (!(dev->flags & IFF_UP))
Eric W. Biederman5cde2822013-10-05 19:26:05 -07001416 list_del_init(&dev->close_list);
Octavian Purdila44345722010-12-13 12:44:07 +00001417
1418 __dev_close_many(head);
Matti Linnanvuorid8b2a4d2008-02-12 23:10:11 -08001419
Eric W. Biederman5cde2822013-10-05 19:26:05 -07001420 list_for_each_entry_safe(dev, tmp, head, close_list) {
Alexei Starovoitov7f294052013-10-23 16:02:42 -07001421 rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP|IFF_RUNNING, GFP_KERNEL);
Octavian Purdila44345722010-12-13 12:44:07 +00001422 call_netdevice_notifiers(NETDEV_DOWN, dev);
David S. Miller99c4a262015-03-18 22:52:33 -04001423 if (unlink)
1424 list_del_init(&dev->close_list);
Octavian Purdila44345722010-12-13 12:44:07 +00001425 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001426
Linus Torvalds1da177e2005-04-16 15:20:36 -07001427 return 0;
1428}
David S. Miller99c4a262015-03-18 22:52:33 -04001429EXPORT_SYMBOL(dev_close_many);
Patrick McHardybd380812010-02-26 06:34:53 +00001430
1431/**
1432 * dev_close - shutdown an interface.
1433 * @dev: device to shutdown
1434 *
1435 * This function moves an active device into down state. A
1436 * %NETDEV_GOING_DOWN is sent to the netdev notifier chain. The device
1437 * is then deactivated and finally a %NETDEV_DOWN is sent to the notifier
1438 * chain.
1439 */
1440int dev_close(struct net_device *dev)
1441{
Eric Dumazete14a5992011-05-10 12:26:06 -07001442 if (dev->flags & IFF_UP) {
1443 LIST_HEAD(single);
Patrick McHardybd380812010-02-26 06:34:53 +00001444
Eric W. Biederman5cde2822013-10-05 19:26:05 -07001445 list_add(&dev->close_list, &single);
David S. Miller99c4a262015-03-18 22:52:33 -04001446 dev_close_many(&single, true);
Eric Dumazete14a5992011-05-10 12:26:06 -07001447 list_del(&single);
1448 }
dingtianhongda6e3782013-05-27 19:53:31 +00001449 return 0;
Patrick McHardybd380812010-02-26 06:34:53 +00001450}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001451EXPORT_SYMBOL(dev_close);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001452
1453
Ben Hutchings0187bdf2008-06-19 16:15:47 -07001454/**
1455 * dev_disable_lro - disable Large Receive Offload on a device
1456 * @dev: device
1457 *
1458 * Disable Large Receive Offload (LRO) on a net device. Must be
1459 * called under RTNL. This is needed if received packets may be
1460 * forwarded to another interface.
1461 */
1462void dev_disable_lro(struct net_device *dev)
1463{
Michal Kubečekfbe168b2014-11-13 07:54:50 +01001464 struct net_device *lower_dev;
1465 struct list_head *iter;
Michal Kubeček529d0482013-11-15 06:18:50 +01001466
Michał Mirosławbc5787c62011-11-15 15:29:55 +00001467 dev->wanted_features &= ~NETIF_F_LRO;
1468 netdev_update_features(dev);
Michał Mirosław27660512011-03-18 16:56:34 +00001469
Michał Mirosław22d59692011-04-21 12:42:15 +00001470 if (unlikely(dev->features & NETIF_F_LRO))
1471 netdev_WARN(dev, "failed to disable LRO!\n");
Michal Kubečekfbe168b2014-11-13 07:54:50 +01001472
1473 netdev_for_each_lower_dev(dev, lower_dev, iter)
1474 dev_disable_lro(lower_dev);
Ben Hutchings0187bdf2008-06-19 16:15:47 -07001475}
1476EXPORT_SYMBOL(dev_disable_lro);
1477
Jiri Pirko351638e2013-05-28 01:30:21 +00001478static int call_netdevice_notifier(struct notifier_block *nb, unsigned long val,
1479 struct net_device *dev)
1480{
1481 struct netdev_notifier_info info;
1482
1483 netdev_notifier_info_init(&info, dev);
1484 return nb->notifier_call(nb, val, &info);
1485}
Ben Hutchings0187bdf2008-06-19 16:15:47 -07001486
Eric W. Biederman881d9662007-09-17 11:56:21 -07001487static int dev_boot_phase = 1;
1488
Linus Torvalds1da177e2005-04-16 15:20:36 -07001489/**
1490 * register_netdevice_notifier - register a network notifier block
1491 * @nb: notifier
1492 *
1493 * Register a notifier to be called when network device events occur.
1494 * The notifier passed is linked into the kernel structures and must
1495 * not be reused until it has been unregistered. A negative errno code
1496 * is returned on a failure.
1497 *
1498 * When registered all registration and up events are replayed
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09001499 * to the new notifier to allow device to have a race free
Linus Torvalds1da177e2005-04-16 15:20:36 -07001500 * view of the network device list.
1501 */
1502
1503int register_netdevice_notifier(struct notifier_block *nb)
1504{
1505 struct net_device *dev;
Herbert Xufcc5a032007-07-30 17:03:38 -07001506 struct net_device *last;
Eric W. Biederman881d9662007-09-17 11:56:21 -07001507 struct net *net;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001508 int err;
1509
1510 rtnl_lock();
Alan Sternf07d5b92006-05-09 15:23:03 -07001511 err = raw_notifier_chain_register(&netdev_chain, nb);
Herbert Xufcc5a032007-07-30 17:03:38 -07001512 if (err)
1513 goto unlock;
Eric W. Biederman881d9662007-09-17 11:56:21 -07001514 if (dev_boot_phase)
1515 goto unlock;
1516 for_each_net(net) {
1517 for_each_netdev(net, dev) {
Jiri Pirko351638e2013-05-28 01:30:21 +00001518 err = call_netdevice_notifier(nb, NETDEV_REGISTER, dev);
Eric W. Biederman881d9662007-09-17 11:56:21 -07001519 err = notifier_to_errno(err);
1520 if (err)
1521 goto rollback;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001522
Eric W. Biederman881d9662007-09-17 11:56:21 -07001523 if (!(dev->flags & IFF_UP))
1524 continue;
Herbert Xufcc5a032007-07-30 17:03:38 -07001525
Jiri Pirko351638e2013-05-28 01:30:21 +00001526 call_netdevice_notifier(nb, NETDEV_UP, dev);
Eric W. Biederman881d9662007-09-17 11:56:21 -07001527 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001528 }
Herbert Xufcc5a032007-07-30 17:03:38 -07001529
1530unlock:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001531 rtnl_unlock();
1532 return err;
Herbert Xufcc5a032007-07-30 17:03:38 -07001533
1534rollback:
1535 last = dev;
Eric W. Biederman881d9662007-09-17 11:56:21 -07001536 for_each_net(net) {
1537 for_each_netdev(net, dev) {
1538 if (dev == last)
RongQing.Li8f891482011-11-30 23:43:07 -05001539 goto outroll;
Herbert Xufcc5a032007-07-30 17:03:38 -07001540
Eric W. Biederman881d9662007-09-17 11:56:21 -07001541 if (dev->flags & IFF_UP) {
Jiri Pirko351638e2013-05-28 01:30:21 +00001542 call_netdevice_notifier(nb, NETDEV_GOING_DOWN,
1543 dev);
1544 call_netdevice_notifier(nb, NETDEV_DOWN, dev);
Eric W. Biederman881d9662007-09-17 11:56:21 -07001545 }
Jiri Pirko351638e2013-05-28 01:30:21 +00001546 call_netdevice_notifier(nb, NETDEV_UNREGISTER, dev);
Herbert Xufcc5a032007-07-30 17:03:38 -07001547 }
Herbert Xufcc5a032007-07-30 17:03:38 -07001548 }
Pavel Emelyanovc67625a2007-11-14 15:53:16 -08001549
RongQing.Li8f891482011-11-30 23:43:07 -05001550outroll:
Pavel Emelyanovc67625a2007-11-14 15:53:16 -08001551 raw_notifier_chain_unregister(&netdev_chain, nb);
Herbert Xufcc5a032007-07-30 17:03:38 -07001552 goto unlock;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001553}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001554EXPORT_SYMBOL(register_netdevice_notifier);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001555
1556/**
1557 * unregister_netdevice_notifier - unregister a network notifier block
1558 * @nb: notifier
1559 *
1560 * Unregister a notifier previously registered by
1561 * register_netdevice_notifier(). The notifier is unlinked into the
1562 * kernel structures and may then be reused. A negative errno code
1563 * is returned on a failure.
Eric W. Biederman7d3d43d2012-04-06 15:33:35 +00001564 *
1565 * After unregistering unregister and down device events are synthesized
1566 * for all devices on the device list to the removed notifier to remove
1567 * the need for special case cleanup code.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001568 */
1569
1570int unregister_netdevice_notifier(struct notifier_block *nb)
1571{
Eric W. Biederman7d3d43d2012-04-06 15:33:35 +00001572 struct net_device *dev;
1573 struct net *net;
Herbert Xu9f514952006-03-25 01:24:25 -08001574 int err;
1575
1576 rtnl_lock();
Alan Sternf07d5b92006-05-09 15:23:03 -07001577 err = raw_notifier_chain_unregister(&netdev_chain, nb);
Eric W. Biederman7d3d43d2012-04-06 15:33:35 +00001578 if (err)
1579 goto unlock;
1580
1581 for_each_net(net) {
1582 for_each_netdev(net, dev) {
1583 if (dev->flags & IFF_UP) {
Jiri Pirko351638e2013-05-28 01:30:21 +00001584 call_netdevice_notifier(nb, NETDEV_GOING_DOWN,
1585 dev);
1586 call_netdevice_notifier(nb, NETDEV_DOWN, dev);
Eric W. Biederman7d3d43d2012-04-06 15:33:35 +00001587 }
Jiri Pirko351638e2013-05-28 01:30:21 +00001588 call_netdevice_notifier(nb, NETDEV_UNREGISTER, dev);
Eric W. Biederman7d3d43d2012-04-06 15:33:35 +00001589 }
1590 }
1591unlock:
Herbert Xu9f514952006-03-25 01:24:25 -08001592 rtnl_unlock();
1593 return err;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001594}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001595EXPORT_SYMBOL(unregister_netdevice_notifier);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001596
1597/**
Jiri Pirko351638e2013-05-28 01:30:21 +00001598 * call_netdevice_notifiers_info - call all network notifier blocks
1599 * @val: value passed unmodified to notifier function
1600 * @dev: net_device pointer passed unmodified to notifier function
1601 * @info: notifier information data
1602 *
1603 * Call all network notifier blocks. Parameters and return value
1604 * are as for raw_notifier_call_chain().
1605 */
1606
stephen hemminger1d143d92013-12-29 14:01:29 -08001607static int call_netdevice_notifiers_info(unsigned long val,
1608 struct net_device *dev,
1609 struct netdev_notifier_info *info)
Jiri Pirko351638e2013-05-28 01:30:21 +00001610{
1611 ASSERT_RTNL();
1612 netdev_notifier_info_init(info, dev);
1613 return raw_notifier_call_chain(&netdev_chain, val, info);
1614}
Jiri Pirko351638e2013-05-28 01:30:21 +00001615
1616/**
Linus Torvalds1da177e2005-04-16 15:20:36 -07001617 * call_netdevice_notifiers - call all network notifier blocks
1618 * @val: value passed unmodified to notifier function
Randy Dunlapc4ea43c2007-10-12 21:17:49 -07001619 * @dev: net_device pointer passed unmodified to notifier function
Linus Torvalds1da177e2005-04-16 15:20:36 -07001620 *
1621 * Call all network notifier blocks. Parameters and return value
Alan Sternf07d5b92006-05-09 15:23:03 -07001622 * are as for raw_notifier_call_chain().
Linus Torvalds1da177e2005-04-16 15:20:36 -07001623 */
1624
Eric W. Biedermanad7379d2007-09-16 15:33:32 -07001625int call_netdevice_notifiers(unsigned long val, struct net_device *dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001626{
Jiri Pirko351638e2013-05-28 01:30:21 +00001627 struct netdev_notifier_info info;
1628
1629 return call_netdevice_notifiers_info(val, dev, &info);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001630}
stephen hemmingeredf947f2011-03-24 13:24:01 +00001631EXPORT_SYMBOL(call_netdevice_notifiers);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001632
Ingo Molnarc5905af2012-02-24 08:31:31 +01001633static struct static_key netstamp_needed __read_mostly;
Eric Dumazetb90e5792011-11-28 11:16:50 +00001634#ifdef HAVE_JUMP_LABEL
Ingo Molnarc5905af2012-02-24 08:31:31 +01001635/* We are not allowed to call static_key_slow_dec() from irq context
Eric Dumazetb90e5792011-11-28 11:16:50 +00001636 * If net_disable_timestamp() is called from irq context, defer the
Ingo Molnarc5905af2012-02-24 08:31:31 +01001637 * static_key_slow_dec() calls.
Eric Dumazetb90e5792011-11-28 11:16:50 +00001638 */
1639static atomic_t netstamp_needed_deferred;
1640#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07001641
1642void net_enable_timestamp(void)
1643{
Eric Dumazetb90e5792011-11-28 11:16:50 +00001644#ifdef HAVE_JUMP_LABEL
1645 int deferred = atomic_xchg(&netstamp_needed_deferred, 0);
1646
1647 if (deferred) {
1648 while (--deferred)
Ingo Molnarc5905af2012-02-24 08:31:31 +01001649 static_key_slow_dec(&netstamp_needed);
Eric Dumazetb90e5792011-11-28 11:16:50 +00001650 return;
1651 }
1652#endif
Ingo Molnarc5905af2012-02-24 08:31:31 +01001653 static_key_slow_inc(&netstamp_needed);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001654}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001655EXPORT_SYMBOL(net_enable_timestamp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001656
1657void net_disable_timestamp(void)
1658{
Eric Dumazetb90e5792011-11-28 11:16:50 +00001659#ifdef HAVE_JUMP_LABEL
1660 if (in_interrupt()) {
1661 atomic_inc(&netstamp_needed_deferred);
1662 return;
1663 }
1664#endif
Ingo Molnarc5905af2012-02-24 08:31:31 +01001665 static_key_slow_dec(&netstamp_needed);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001666}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001667EXPORT_SYMBOL(net_disable_timestamp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001668
Eric Dumazet3b098e22010-05-15 23:57:10 -07001669static inline void net_timestamp_set(struct sk_buff *skb)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001670{
Eric Dumazet588f0332011-11-15 04:12:55 +00001671 skb->tstamp.tv64 = 0;
Ingo Molnarc5905af2012-02-24 08:31:31 +01001672 if (static_key_false(&netstamp_needed))
Patrick McHardya61bbcf2005-08-14 17:24:31 -07001673 __net_timestamp(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001674}
1675
Eric Dumazet588f0332011-11-15 04:12:55 +00001676#define net_timestamp_check(COND, SKB) \
Ingo Molnarc5905af2012-02-24 08:31:31 +01001677 if (static_key_false(&netstamp_needed)) { \
Eric Dumazet588f0332011-11-15 04:12:55 +00001678 if ((COND) && !(SKB)->tstamp.tv64) \
1679 __net_timestamp(SKB); \
1680 } \
Eric Dumazet3b098e22010-05-15 23:57:10 -07001681
Vlad Yasevich1ee481f2014-03-27 17:32:29 -04001682bool is_skb_forwardable(struct net_device *dev, struct sk_buff *skb)
Daniel Lezcano79b569f2011-03-30 02:42:17 -07001683{
1684 unsigned int len;
1685
1686 if (!(dev->flags & IFF_UP))
1687 return false;
1688
1689 len = dev->mtu + dev->hard_header_len + VLAN_HLEN;
1690 if (skb->len <= len)
1691 return true;
1692
1693 /* if TSO is enabled, we don't care about the length as the packet
1694 * could be forwarded without being segmented before
1695 */
1696 if (skb_is_gso(skb))
1697 return true;
1698
1699 return false;
1700}
Vlad Yasevich1ee481f2014-03-27 17:32:29 -04001701EXPORT_SYMBOL_GPL(is_skb_forwardable);
Daniel Lezcano79b569f2011-03-30 02:42:17 -07001702
Herbert Xua0265d22014-04-17 13:45:03 +08001703int __dev_forward_skb(struct net_device *dev, struct sk_buff *skb)
1704{
1705 if (skb_shinfo(skb)->tx_flags & SKBTX_DEV_ZEROCOPY) {
1706 if (skb_copy_ubufs(skb, GFP_ATOMIC)) {
1707 atomic_long_inc(&dev->rx_dropped);
1708 kfree_skb(skb);
1709 return NET_RX_DROP;
1710 }
1711 }
1712
1713 if (unlikely(!is_skb_forwardable(dev, skb))) {
1714 atomic_long_inc(&dev->rx_dropped);
1715 kfree_skb(skb);
1716 return NET_RX_DROP;
1717 }
1718
1719 skb_scrub_packet(skb, true);
WANG Cong08b4b8e2015-03-20 14:29:09 -07001720 skb->priority = 0;
Herbert Xua0265d22014-04-17 13:45:03 +08001721 skb->protocol = eth_type_trans(skb, dev);
Jay Vosburgh2c26d342014-12-19 15:32:00 -08001722 skb_postpull_rcsum(skb, eth_hdr(skb), ETH_HLEN);
Herbert Xua0265d22014-04-17 13:45:03 +08001723
1724 return 0;
1725}
1726EXPORT_SYMBOL_GPL(__dev_forward_skb);
1727
Arnd Bergmann44540962009-11-26 06:07:08 +00001728/**
1729 * dev_forward_skb - loopback an skb to another netif
1730 *
1731 * @dev: destination network device
1732 * @skb: buffer to forward
1733 *
1734 * return values:
1735 * NET_RX_SUCCESS (no congestion)
Eric Dumazet6ec82562010-05-06 00:53:53 -07001736 * NET_RX_DROP (packet was dropped, but freed)
Arnd Bergmann44540962009-11-26 06:07:08 +00001737 *
1738 * dev_forward_skb can be used for injecting an skb from the
1739 * start_xmit function of one device into the receive queue
1740 * of another device.
1741 *
1742 * The receiving device may be in another namespace, so
1743 * we have to clear all information in the skb that could
1744 * impact namespace isolation.
1745 */
1746int dev_forward_skb(struct net_device *dev, struct sk_buff *skb)
1747{
Herbert Xua0265d22014-04-17 13:45:03 +08001748 return __dev_forward_skb(dev, skb) ?: netif_rx_internal(skb);
Arnd Bergmann44540962009-11-26 06:07:08 +00001749}
1750EXPORT_SYMBOL_GPL(dev_forward_skb);
1751
Changli Gao71d9dec2010-12-15 19:57:25 +00001752static inline int deliver_skb(struct sk_buff *skb,
1753 struct packet_type *pt_prev,
1754 struct net_device *orig_dev)
1755{
Michael S. Tsirkin1080e512012-07-20 09:23:17 +00001756 if (unlikely(skb_orphan_frags(skb, GFP_ATOMIC)))
1757 return -ENOMEM;
Changli Gao71d9dec2010-12-15 19:57:25 +00001758 atomic_inc(&skb->users);
1759 return pt_prev->func(skb, skb->dev, pt_prev, orig_dev);
1760}
1761
Salam Noureddine7866a622015-01-27 11:35:48 -08001762static inline void deliver_ptype_list_skb(struct sk_buff *skb,
1763 struct packet_type **pt,
Jiri Pirkofbcb2172015-03-30 16:56:01 +02001764 struct net_device *orig_dev,
1765 __be16 type,
Salam Noureddine7866a622015-01-27 11:35:48 -08001766 struct list_head *ptype_list)
1767{
1768 struct packet_type *ptype, *pt_prev = *pt;
1769
1770 list_for_each_entry_rcu(ptype, ptype_list, list) {
1771 if (ptype->type != type)
1772 continue;
1773 if (pt_prev)
Jiri Pirkofbcb2172015-03-30 16:56:01 +02001774 deliver_skb(skb, pt_prev, orig_dev);
Salam Noureddine7866a622015-01-27 11:35:48 -08001775 pt_prev = ptype;
1776 }
1777 *pt = pt_prev;
1778}
1779
Eric Leblondc0de08d2012-08-16 22:02:58 +00001780static inline bool skb_loop_sk(struct packet_type *ptype, struct sk_buff *skb)
1781{
Eric Leblonda3d744e2012-11-06 02:10:10 +00001782 if (!ptype->af_packet_priv || !skb->sk)
Eric Leblondc0de08d2012-08-16 22:02:58 +00001783 return false;
1784
1785 if (ptype->id_match)
1786 return ptype->id_match(ptype, skb->sk);
1787 else if ((struct sock *)ptype->af_packet_priv == skb->sk)
1788 return true;
1789
1790 return false;
1791}
1792
Linus Torvalds1da177e2005-04-16 15:20:36 -07001793/*
1794 * Support routine. Sends outgoing frames to any network
1795 * taps currently in use.
1796 */
1797
Herbert Xuf6a78bf2006-06-22 02:57:17 -07001798static void dev_queue_xmit_nit(struct sk_buff *skb, struct net_device *dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001799{
1800 struct packet_type *ptype;
Changli Gao71d9dec2010-12-15 19:57:25 +00001801 struct sk_buff *skb2 = NULL;
1802 struct packet_type *pt_prev = NULL;
Salam Noureddine7866a622015-01-27 11:35:48 -08001803 struct list_head *ptype_list = &ptype_all;
Patrick McHardya61bbcf2005-08-14 17:24:31 -07001804
Linus Torvalds1da177e2005-04-16 15:20:36 -07001805 rcu_read_lock();
Salam Noureddine7866a622015-01-27 11:35:48 -08001806again:
1807 list_for_each_entry_rcu(ptype, ptype_list, list) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001808 /* Never send packets back to the socket
1809 * they originated from - MvS (miquels@drinkel.ow.org)
1810 */
Salam Noureddine7866a622015-01-27 11:35:48 -08001811 if (skb_loop_sk(ptype, skb))
1812 continue;
Changli Gao71d9dec2010-12-15 19:57:25 +00001813
Salam Noureddine7866a622015-01-27 11:35:48 -08001814 if (pt_prev) {
1815 deliver_skb(skb2, pt_prev, skb->dev);
Changli Gao71d9dec2010-12-15 19:57:25 +00001816 pt_prev = ptype;
Salam Noureddine7866a622015-01-27 11:35:48 -08001817 continue;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001818 }
Salam Noureddine7866a622015-01-27 11:35:48 -08001819
1820 /* need to clone skb, done only once */
1821 skb2 = skb_clone(skb, GFP_ATOMIC);
1822 if (!skb2)
1823 goto out_unlock;
1824
1825 net_timestamp_set(skb2);
1826
1827 /* skb->nh should be correctly
1828 * set by sender, so that the second statement is
1829 * just protection against buggy protocols.
1830 */
1831 skb_reset_mac_header(skb2);
1832
1833 if (skb_network_header(skb2) < skb2->data ||
1834 skb_network_header(skb2) > skb_tail_pointer(skb2)) {
1835 net_crit_ratelimited("protocol %04x is buggy, dev %s\n",
1836 ntohs(skb2->protocol),
1837 dev->name);
1838 skb_reset_network_header(skb2);
1839 }
1840
1841 skb2->transport_header = skb2->network_header;
1842 skb2->pkt_type = PACKET_OUTGOING;
1843 pt_prev = ptype;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001844 }
Salam Noureddine7866a622015-01-27 11:35:48 -08001845
1846 if (ptype_list == &ptype_all) {
1847 ptype_list = &dev->ptype_all;
1848 goto again;
1849 }
1850out_unlock:
Changli Gao71d9dec2010-12-15 19:57:25 +00001851 if (pt_prev)
1852 pt_prev->func(skb2, skb->dev, pt_prev, skb->dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001853 rcu_read_unlock();
1854}
1855
Ben Hutchings2c530402012-07-10 10:55:09 +00001856/**
1857 * netif_setup_tc - Handle tc mappings on real_num_tx_queues change
John Fastabend4f57c082011-01-17 08:06:04 +00001858 * @dev: Network device
1859 * @txq: number of queues available
1860 *
1861 * If real_num_tx_queues is changed the tc mappings may no longer be
1862 * valid. To resolve this verify the tc mapping remains valid and if
1863 * not NULL the mapping. With no priorities mapping to this
1864 * offset/count pair it will no longer be used. In the worst case TC0
1865 * is invalid nothing can be done so disable priority mappings. If is
1866 * expected that drivers will fix this mapping if they can before
1867 * calling netif_set_real_num_tx_queues.
1868 */
Eric Dumazetbb134d22011-01-20 19:18:08 +00001869static void netif_setup_tc(struct net_device *dev, unsigned int txq)
John Fastabend4f57c082011-01-17 08:06:04 +00001870{
1871 int i;
1872 struct netdev_tc_txq *tc = &dev->tc_to_txq[0];
1873
1874 /* If TC0 is invalidated disable TC mapping */
1875 if (tc->offset + tc->count > txq) {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00001876 pr_warn("Number of in use tx queues changed invalidating tc mappings. Priority traffic classification disabled!\n");
John Fastabend4f57c082011-01-17 08:06:04 +00001877 dev->num_tc = 0;
1878 return;
1879 }
1880
1881 /* Invalidated prio to tc mappings set to TC0 */
1882 for (i = 1; i < TC_BITMASK + 1; i++) {
1883 int q = netdev_get_prio_tc_map(dev, i);
1884
1885 tc = &dev->tc_to_txq[q];
1886 if (tc->offset + tc->count > txq) {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00001887 pr_warn("Number of in use tx queues changed. Priority %i to tc mapping %i is no longer valid. Setting map to 0\n",
1888 i, q);
John Fastabend4f57c082011-01-17 08:06:04 +00001889 netdev_set_prio_tc_map(dev, i, 0);
1890 }
1891 }
1892}
1893
Alexander Duyck537c00d2013-01-10 08:57:02 +00001894#ifdef CONFIG_XPS
1895static DEFINE_MUTEX(xps_map_mutex);
1896#define xmap_dereference(P) \
1897 rcu_dereference_protected((P), lockdep_is_held(&xps_map_mutex))
1898
Alexander Duyck10cdc3f2013-01-10 08:57:17 +00001899static struct xps_map *remove_xps_queue(struct xps_dev_maps *dev_maps,
1900 int cpu, u16 index)
1901{
1902 struct xps_map *map = NULL;
1903 int pos;
1904
1905 if (dev_maps)
1906 map = xmap_dereference(dev_maps->cpu_map[cpu]);
1907
1908 for (pos = 0; map && pos < map->len; pos++) {
1909 if (map->queues[pos] == index) {
1910 if (map->len > 1) {
1911 map->queues[pos] = map->queues[--map->len];
1912 } else {
1913 RCU_INIT_POINTER(dev_maps->cpu_map[cpu], NULL);
1914 kfree_rcu(map, rcu);
1915 map = NULL;
1916 }
1917 break;
1918 }
1919 }
1920
1921 return map;
1922}
1923
Alexander Duyck024e9672013-01-10 08:57:46 +00001924static void netif_reset_xps_queues_gt(struct net_device *dev, u16 index)
Alexander Duyck537c00d2013-01-10 08:57:02 +00001925{
1926 struct xps_dev_maps *dev_maps;
Alexander Duyck024e9672013-01-10 08:57:46 +00001927 int cpu, i;
Alexander Duyck10cdc3f2013-01-10 08:57:17 +00001928 bool active = false;
Alexander Duyck537c00d2013-01-10 08:57:02 +00001929
1930 mutex_lock(&xps_map_mutex);
1931 dev_maps = xmap_dereference(dev->xps_maps);
1932
1933 if (!dev_maps)
1934 goto out_no_maps;
1935
Alexander Duyck10cdc3f2013-01-10 08:57:17 +00001936 for_each_possible_cpu(cpu) {
Alexander Duyck024e9672013-01-10 08:57:46 +00001937 for (i = index; i < dev->num_tx_queues; i++) {
1938 if (!remove_xps_queue(dev_maps, cpu, i))
1939 break;
1940 }
1941 if (i == dev->num_tx_queues)
Alexander Duyck10cdc3f2013-01-10 08:57:17 +00001942 active = true;
Alexander Duyck537c00d2013-01-10 08:57:02 +00001943 }
1944
Alexander Duyck10cdc3f2013-01-10 08:57:17 +00001945 if (!active) {
Alexander Duyck537c00d2013-01-10 08:57:02 +00001946 RCU_INIT_POINTER(dev->xps_maps, NULL);
1947 kfree_rcu(dev_maps, rcu);
1948 }
1949
Alexander Duyck024e9672013-01-10 08:57:46 +00001950 for (i = index; i < dev->num_tx_queues; i++)
1951 netdev_queue_numa_node_write(netdev_get_tx_queue(dev, i),
1952 NUMA_NO_NODE);
1953
Alexander Duyck537c00d2013-01-10 08:57:02 +00001954out_no_maps:
1955 mutex_unlock(&xps_map_mutex);
1956}
1957
Alexander Duyck01c5f862013-01-10 08:57:35 +00001958static struct xps_map *expand_xps_map(struct xps_map *map,
1959 int cpu, u16 index)
1960{
1961 struct xps_map *new_map;
1962 int alloc_len = XPS_MIN_MAP_ALLOC;
1963 int i, pos;
1964
1965 for (pos = 0; map && pos < map->len; pos++) {
1966 if (map->queues[pos] != index)
1967 continue;
1968 return map;
1969 }
1970
1971 /* Need to add queue to this CPU's existing map */
1972 if (map) {
1973 if (pos < map->alloc_len)
1974 return map;
1975
1976 alloc_len = map->alloc_len * 2;
1977 }
1978
1979 /* Need to allocate new map to store queue on this CPU's map */
1980 new_map = kzalloc_node(XPS_MAP_SIZE(alloc_len), GFP_KERNEL,
1981 cpu_to_node(cpu));
1982 if (!new_map)
1983 return NULL;
1984
1985 for (i = 0; i < pos; i++)
1986 new_map->queues[i] = map->queues[i];
1987 new_map->alloc_len = alloc_len;
1988 new_map->len = pos;
1989
1990 return new_map;
1991}
1992
Michael S. Tsirkin35735402013-10-02 09:14:06 +03001993int netif_set_xps_queue(struct net_device *dev, const struct cpumask *mask,
1994 u16 index)
Alexander Duyck537c00d2013-01-10 08:57:02 +00001995{
Alexander Duyck01c5f862013-01-10 08:57:35 +00001996 struct xps_dev_maps *dev_maps, *new_dev_maps = NULL;
Alexander Duyck537c00d2013-01-10 08:57:02 +00001997 struct xps_map *map, *new_map;
Alexander Duyck537c00d2013-01-10 08:57:02 +00001998 int maps_sz = max_t(unsigned int, XPS_DEV_MAPS_SIZE, L1_CACHE_BYTES);
Alexander Duyck01c5f862013-01-10 08:57:35 +00001999 int cpu, numa_node_id = -2;
2000 bool active = false;
Alexander Duyck537c00d2013-01-10 08:57:02 +00002001
2002 mutex_lock(&xps_map_mutex);
2003
2004 dev_maps = xmap_dereference(dev->xps_maps);
2005
Alexander Duyck01c5f862013-01-10 08:57:35 +00002006 /* allocate memory for queue storage */
2007 for_each_online_cpu(cpu) {
2008 if (!cpumask_test_cpu(cpu, mask))
2009 continue;
Alexander Duyck537c00d2013-01-10 08:57:02 +00002010
Alexander Duyck01c5f862013-01-10 08:57:35 +00002011 if (!new_dev_maps)
2012 new_dev_maps = kzalloc(maps_sz, GFP_KERNEL);
Alexander Duyck2bb60cb2013-02-22 06:38:44 +00002013 if (!new_dev_maps) {
2014 mutex_unlock(&xps_map_mutex);
Alexander Duyck01c5f862013-01-10 08:57:35 +00002015 return -ENOMEM;
Alexander Duyck2bb60cb2013-02-22 06:38:44 +00002016 }
Alexander Duyck01c5f862013-01-10 08:57:35 +00002017
2018 map = dev_maps ? xmap_dereference(dev_maps->cpu_map[cpu]) :
2019 NULL;
2020
2021 map = expand_xps_map(map, cpu, index);
2022 if (!map)
2023 goto error;
2024
2025 RCU_INIT_POINTER(new_dev_maps->cpu_map[cpu], map);
2026 }
2027
2028 if (!new_dev_maps)
2029 goto out_no_new_maps;
2030
2031 for_each_possible_cpu(cpu) {
2032 if (cpumask_test_cpu(cpu, mask) && cpu_online(cpu)) {
2033 /* add queue to CPU maps */
2034 int pos = 0;
2035
2036 map = xmap_dereference(new_dev_maps->cpu_map[cpu]);
2037 while ((pos < map->len) && (map->queues[pos] != index))
2038 pos++;
2039
2040 if (pos == map->len)
2041 map->queues[map->len++] = index;
Alexander Duyck537c00d2013-01-10 08:57:02 +00002042#ifdef CONFIG_NUMA
Alexander Duyck537c00d2013-01-10 08:57:02 +00002043 if (numa_node_id == -2)
2044 numa_node_id = cpu_to_node(cpu);
2045 else if (numa_node_id != cpu_to_node(cpu))
2046 numa_node_id = -1;
Alexander Duyck537c00d2013-01-10 08:57:02 +00002047#endif
Alexander Duyck01c5f862013-01-10 08:57:35 +00002048 } else if (dev_maps) {
2049 /* fill in the new device map from the old device map */
2050 map = xmap_dereference(dev_maps->cpu_map[cpu]);
2051 RCU_INIT_POINTER(new_dev_maps->cpu_map[cpu], map);
Alexander Duyck537c00d2013-01-10 08:57:02 +00002052 }
Alexander Duyck01c5f862013-01-10 08:57:35 +00002053
Alexander Duyck537c00d2013-01-10 08:57:02 +00002054 }
2055
Alexander Duyck01c5f862013-01-10 08:57:35 +00002056 rcu_assign_pointer(dev->xps_maps, new_dev_maps);
2057
Alexander Duyck537c00d2013-01-10 08:57:02 +00002058 /* Cleanup old maps */
Alexander Duyck01c5f862013-01-10 08:57:35 +00002059 if (dev_maps) {
2060 for_each_possible_cpu(cpu) {
2061 new_map = xmap_dereference(new_dev_maps->cpu_map[cpu]);
2062 map = xmap_dereference(dev_maps->cpu_map[cpu]);
2063 if (map && map != new_map)
2064 kfree_rcu(map, rcu);
2065 }
Alexander Duyck537c00d2013-01-10 08:57:02 +00002066
Alexander Duyck537c00d2013-01-10 08:57:02 +00002067 kfree_rcu(dev_maps, rcu);
Alexander Duyck01c5f862013-01-10 08:57:35 +00002068 }
Alexander Duyck537c00d2013-01-10 08:57:02 +00002069
Alexander Duyck01c5f862013-01-10 08:57:35 +00002070 dev_maps = new_dev_maps;
2071 active = true;
2072
2073out_no_new_maps:
2074 /* update Tx queue numa node */
Alexander Duyck537c00d2013-01-10 08:57:02 +00002075 netdev_queue_numa_node_write(netdev_get_tx_queue(dev, index),
2076 (numa_node_id >= 0) ? numa_node_id :
2077 NUMA_NO_NODE);
2078
Alexander Duyck01c5f862013-01-10 08:57:35 +00002079 if (!dev_maps)
2080 goto out_no_maps;
2081
2082 /* removes queue from unused CPUs */
2083 for_each_possible_cpu(cpu) {
2084 if (cpumask_test_cpu(cpu, mask) && cpu_online(cpu))
2085 continue;
2086
2087 if (remove_xps_queue(dev_maps, cpu, index))
2088 active = true;
2089 }
2090
2091 /* free map if not active */
2092 if (!active) {
2093 RCU_INIT_POINTER(dev->xps_maps, NULL);
2094 kfree_rcu(dev_maps, rcu);
2095 }
2096
2097out_no_maps:
Alexander Duyck537c00d2013-01-10 08:57:02 +00002098 mutex_unlock(&xps_map_mutex);
2099
2100 return 0;
2101error:
Alexander Duyck01c5f862013-01-10 08:57:35 +00002102 /* remove any maps that we added */
2103 for_each_possible_cpu(cpu) {
2104 new_map = xmap_dereference(new_dev_maps->cpu_map[cpu]);
2105 map = dev_maps ? xmap_dereference(dev_maps->cpu_map[cpu]) :
2106 NULL;
2107 if (new_map && new_map != map)
2108 kfree(new_map);
2109 }
2110
Alexander Duyck537c00d2013-01-10 08:57:02 +00002111 mutex_unlock(&xps_map_mutex);
2112
Alexander Duyck537c00d2013-01-10 08:57:02 +00002113 kfree(new_dev_maps);
2114 return -ENOMEM;
2115}
2116EXPORT_SYMBOL(netif_set_xps_queue);
2117
2118#endif
John Fastabendf0796d52010-07-01 13:21:57 +00002119/*
2120 * Routine to help set real_num_tx_queues. To avoid skbs mapped to queues
2121 * greater then real_num_tx_queues stale skbs on the qdisc must be flushed.
2122 */
Tom Herberte6484932010-10-18 18:04:39 +00002123int netif_set_real_num_tx_queues(struct net_device *dev, unsigned int txq)
John Fastabendf0796d52010-07-01 13:21:57 +00002124{
Tom Herbert1d24eb42010-11-21 13:17:27 +00002125 int rc;
2126
Tom Herberte6484932010-10-18 18:04:39 +00002127 if (txq < 1 || txq > dev->num_tx_queues)
2128 return -EINVAL;
John Fastabendf0796d52010-07-01 13:21:57 +00002129
Ben Hutchings5c565802011-02-15 19:39:21 +00002130 if (dev->reg_state == NETREG_REGISTERED ||
2131 dev->reg_state == NETREG_UNREGISTERING) {
Tom Herberte6484932010-10-18 18:04:39 +00002132 ASSERT_RTNL();
2133
Tom Herbert1d24eb42010-11-21 13:17:27 +00002134 rc = netdev_queue_update_kobjects(dev, dev->real_num_tx_queues,
2135 txq);
Tom Herbertbf264142010-11-26 08:36:09 +00002136 if (rc)
2137 return rc;
2138
John Fastabend4f57c082011-01-17 08:06:04 +00002139 if (dev->num_tc)
2140 netif_setup_tc(dev, txq);
2141
Alexander Duyck024e9672013-01-10 08:57:46 +00002142 if (txq < dev->real_num_tx_queues) {
Tom Herberte6484932010-10-18 18:04:39 +00002143 qdisc_reset_all_tx_gt(dev, txq);
Alexander Duyck024e9672013-01-10 08:57:46 +00002144#ifdef CONFIG_XPS
2145 netif_reset_xps_queues_gt(dev, txq);
2146#endif
2147 }
John Fastabendf0796d52010-07-01 13:21:57 +00002148 }
Tom Herberte6484932010-10-18 18:04:39 +00002149
2150 dev->real_num_tx_queues = txq;
2151 return 0;
John Fastabendf0796d52010-07-01 13:21:57 +00002152}
2153EXPORT_SYMBOL(netif_set_real_num_tx_queues);
Denis Vlasenko56079432006-03-29 15:57:29 -08002154
Michael Daltona953be52014-01-16 22:23:28 -08002155#ifdef CONFIG_SYSFS
Ben Hutchings62fe0b42010-09-27 08:24:33 +00002156/**
2157 * netif_set_real_num_rx_queues - set actual number of RX queues used
2158 * @dev: Network device
2159 * @rxq: Actual number of RX queues
2160 *
2161 * This must be called either with the rtnl_lock held or before
2162 * registration of the net device. Returns 0 on success, or a
Ben Hutchings4e7f7952010-10-08 10:33:39 -07002163 * negative error code. If called before registration, it always
2164 * succeeds.
Ben Hutchings62fe0b42010-09-27 08:24:33 +00002165 */
2166int netif_set_real_num_rx_queues(struct net_device *dev, unsigned int rxq)
2167{
2168 int rc;
2169
Tom Herbertbd25fa72010-10-18 18:00:16 +00002170 if (rxq < 1 || rxq > dev->num_rx_queues)
2171 return -EINVAL;
2172
Ben Hutchings62fe0b42010-09-27 08:24:33 +00002173 if (dev->reg_state == NETREG_REGISTERED) {
2174 ASSERT_RTNL();
2175
Ben Hutchings62fe0b42010-09-27 08:24:33 +00002176 rc = net_rx_queue_update_kobjects(dev, dev->real_num_rx_queues,
2177 rxq);
2178 if (rc)
2179 return rc;
Ben Hutchings62fe0b42010-09-27 08:24:33 +00002180 }
2181
2182 dev->real_num_rx_queues = rxq;
2183 return 0;
2184}
2185EXPORT_SYMBOL(netif_set_real_num_rx_queues);
2186#endif
2187
Ben Hutchings2c530402012-07-10 10:55:09 +00002188/**
2189 * netif_get_num_default_rss_queues - default number of RSS queues
Yuval Mintz16917b82012-07-01 03:18:50 +00002190 *
2191 * This routine should set an upper limit on the number of RSS queues
2192 * used by default by multiqueue devices.
2193 */
Ben Hutchingsa55b1382012-07-10 10:54:38 +00002194int netif_get_num_default_rss_queues(void)
Yuval Mintz16917b82012-07-01 03:18:50 +00002195{
2196 return min_t(int, DEFAULT_MAX_NUM_RSS_QUEUES, num_online_cpus());
2197}
2198EXPORT_SYMBOL(netif_get_num_default_rss_queues);
2199
Jarek Poplawskidef82a12008-08-17 21:54:43 -07002200static inline void __netif_reschedule(struct Qdisc *q)
2201{
2202 struct softnet_data *sd;
2203 unsigned long flags;
2204
2205 local_irq_save(flags);
Christoph Lameter903ceff2014-08-17 12:30:35 -05002206 sd = this_cpu_ptr(&softnet_data);
Changli Gaoa9cbd582010-04-26 23:06:24 +00002207 q->next_sched = NULL;
2208 *sd->output_queue_tailp = q;
2209 sd->output_queue_tailp = &q->next_sched;
Jarek Poplawskidef82a12008-08-17 21:54:43 -07002210 raise_softirq_irqoff(NET_TX_SOFTIRQ);
2211 local_irq_restore(flags);
2212}
2213
David S. Miller37437bb2008-07-16 02:15:04 -07002214void __netif_schedule(struct Qdisc *q)
Denis Vlasenko56079432006-03-29 15:57:29 -08002215{
Jarek Poplawskidef82a12008-08-17 21:54:43 -07002216 if (!test_and_set_bit(__QDISC_STATE_SCHED, &q->state))
2217 __netif_reschedule(q);
Denis Vlasenko56079432006-03-29 15:57:29 -08002218}
2219EXPORT_SYMBOL(__netif_schedule);
2220
Eric Dumazete6247022013-12-05 04:45:08 -08002221struct dev_kfree_skb_cb {
2222 enum skb_free_reason reason;
2223};
2224
2225static struct dev_kfree_skb_cb *get_kfree_skb_cb(const struct sk_buff *skb)
Denis Vlasenko56079432006-03-29 15:57:29 -08002226{
Eric Dumazete6247022013-12-05 04:45:08 -08002227 return (struct dev_kfree_skb_cb *)skb->cb;
Denis Vlasenko56079432006-03-29 15:57:29 -08002228}
Denis Vlasenko56079432006-03-29 15:57:29 -08002229
John Fastabend46e5da42014-09-12 20:04:52 -07002230void netif_schedule_queue(struct netdev_queue *txq)
2231{
2232 rcu_read_lock();
2233 if (!(txq->state & QUEUE_STATE_ANY_XOFF)) {
2234 struct Qdisc *q = rcu_dereference(txq->qdisc);
2235
2236 __netif_schedule(q);
2237 }
2238 rcu_read_unlock();
2239}
2240EXPORT_SYMBOL(netif_schedule_queue);
2241
2242/**
2243 * netif_wake_subqueue - allow sending packets on subqueue
2244 * @dev: network device
2245 * @queue_index: sub queue index
2246 *
2247 * Resume individual transmit queue of a device with multiple transmit queues.
2248 */
2249void netif_wake_subqueue(struct net_device *dev, u16 queue_index)
2250{
2251 struct netdev_queue *txq = netdev_get_tx_queue(dev, queue_index);
2252
2253 if (test_and_clear_bit(__QUEUE_STATE_DRV_XOFF, &txq->state)) {
2254 struct Qdisc *q;
2255
2256 rcu_read_lock();
2257 q = rcu_dereference(txq->qdisc);
2258 __netif_schedule(q);
2259 rcu_read_unlock();
2260 }
2261}
2262EXPORT_SYMBOL(netif_wake_subqueue);
2263
2264void netif_tx_wake_queue(struct netdev_queue *dev_queue)
2265{
2266 if (test_and_clear_bit(__QUEUE_STATE_DRV_XOFF, &dev_queue->state)) {
2267 struct Qdisc *q;
2268
2269 rcu_read_lock();
2270 q = rcu_dereference(dev_queue->qdisc);
2271 __netif_schedule(q);
2272 rcu_read_unlock();
2273 }
2274}
2275EXPORT_SYMBOL(netif_tx_wake_queue);
2276
Eric Dumazete6247022013-12-05 04:45:08 -08002277void __dev_kfree_skb_irq(struct sk_buff *skb, enum skb_free_reason reason)
2278{
2279 unsigned long flags;
2280
2281 if (likely(atomic_read(&skb->users) == 1)) {
2282 smp_rmb();
2283 atomic_set(&skb->users, 0);
2284 } else if (likely(!atomic_dec_and_test(&skb->users))) {
2285 return;
2286 }
2287 get_kfree_skb_cb(skb)->reason = reason;
2288 local_irq_save(flags);
2289 skb->next = __this_cpu_read(softnet_data.completion_queue);
2290 __this_cpu_write(softnet_data.completion_queue, skb);
2291 raise_softirq_irqoff(NET_TX_SOFTIRQ);
2292 local_irq_restore(flags);
2293}
2294EXPORT_SYMBOL(__dev_kfree_skb_irq);
2295
2296void __dev_kfree_skb_any(struct sk_buff *skb, enum skb_free_reason reason)
Denis Vlasenko56079432006-03-29 15:57:29 -08002297{
2298 if (in_irq() || irqs_disabled())
Eric Dumazete6247022013-12-05 04:45:08 -08002299 __dev_kfree_skb_irq(skb, reason);
Denis Vlasenko56079432006-03-29 15:57:29 -08002300 else
2301 dev_kfree_skb(skb);
2302}
Eric Dumazete6247022013-12-05 04:45:08 -08002303EXPORT_SYMBOL(__dev_kfree_skb_any);
Denis Vlasenko56079432006-03-29 15:57:29 -08002304
2305
Stephen Hemmingerbea33482007-10-03 16:41:36 -07002306/**
2307 * netif_device_detach - mark device as removed
2308 * @dev: network device
2309 *
2310 * Mark device as removed from system and therefore no longer available.
2311 */
Denis Vlasenko56079432006-03-29 15:57:29 -08002312void netif_device_detach(struct net_device *dev)
2313{
2314 if (test_and_clear_bit(__LINK_STATE_PRESENT, &dev->state) &&
2315 netif_running(dev)) {
Alexander Duyckd5431032009-04-08 13:15:22 +00002316 netif_tx_stop_all_queues(dev);
Denis Vlasenko56079432006-03-29 15:57:29 -08002317 }
2318}
2319EXPORT_SYMBOL(netif_device_detach);
2320
Stephen Hemmingerbea33482007-10-03 16:41:36 -07002321/**
2322 * netif_device_attach - mark device as attached
2323 * @dev: network device
2324 *
2325 * Mark device as attached from system and restart if needed.
2326 */
Denis Vlasenko56079432006-03-29 15:57:29 -08002327void netif_device_attach(struct net_device *dev)
2328{
2329 if (!test_and_set_bit(__LINK_STATE_PRESENT, &dev->state) &&
2330 netif_running(dev)) {
Alexander Duyckd5431032009-04-08 13:15:22 +00002331 netif_tx_wake_all_queues(dev);
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09002332 __netdev_watchdog_up(dev);
Denis Vlasenko56079432006-03-29 15:57:29 -08002333 }
2334}
2335EXPORT_SYMBOL(netif_device_attach);
2336
Ben Hutchings36c92472012-01-17 07:57:56 +00002337static void skb_warn_bad_offload(const struct sk_buff *skb)
2338{
Michał Mirosław65e9d2f2012-01-17 10:00:40 +00002339 static const netdev_features_t null_features = 0;
Ben Hutchings36c92472012-01-17 07:57:56 +00002340 struct net_device *dev = skb->dev;
2341 const char *driver = "";
2342
Ben Greearc846ad92013-04-19 10:45:52 +00002343 if (!net_ratelimit())
2344 return;
2345
Ben Hutchings36c92472012-01-17 07:57:56 +00002346 if (dev && dev->dev.parent)
2347 driver = dev_driver_string(dev->dev.parent);
2348
2349 WARN(1, "%s: caps=(%pNF, %pNF) len=%d data_len=%d gso_size=%d "
2350 "gso_type=%d ip_summed=%d\n",
Michał Mirosław65e9d2f2012-01-17 10:00:40 +00002351 driver, dev ? &dev->features : &null_features,
2352 skb->sk ? &skb->sk->sk_route_caps : &null_features,
Ben Hutchings36c92472012-01-17 07:57:56 +00002353 skb->len, skb->data_len, skb_shinfo(skb)->gso_size,
2354 skb_shinfo(skb)->gso_type, skb->ip_summed);
2355}
2356
Linus Torvalds1da177e2005-04-16 15:20:36 -07002357/*
2358 * Invalidate hardware checksum when packet is to be mangled, and
2359 * complete checksum manually on outgoing path.
2360 */
Patrick McHardy84fa7932006-08-29 16:44:56 -07002361int skb_checksum_help(struct sk_buff *skb)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002362{
Al Virod3bc23e2006-11-14 21:24:49 -08002363 __wsum csum;
Herbert Xu663ead32007-04-09 11:59:07 -07002364 int ret = 0, offset;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002365
Patrick McHardy84fa7932006-08-29 16:44:56 -07002366 if (skb->ip_summed == CHECKSUM_COMPLETE)
Herbert Xua430a432006-07-08 13:34:56 -07002367 goto out_set_summed;
2368
2369 if (unlikely(skb_shinfo(skb)->gso_size)) {
Ben Hutchings36c92472012-01-17 07:57:56 +00002370 skb_warn_bad_offload(skb);
2371 return -EINVAL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002372 }
2373
Eric Dumazetcef401d2013-01-25 20:34:37 +00002374 /* Before computing a checksum, we should make sure no frag could
2375 * be modified by an external entity : checksum could be wrong.
2376 */
2377 if (skb_has_shared_frag(skb)) {
2378 ret = __skb_linearize(skb);
2379 if (ret)
2380 goto out;
2381 }
2382
Michał Mirosław55508d62010-12-14 15:24:08 +00002383 offset = skb_checksum_start_offset(skb);
Herbert Xua0308472007-10-15 01:47:15 -07002384 BUG_ON(offset >= skb_headlen(skb));
2385 csum = skb_checksum(skb, offset, skb->len - offset, 0);
2386
2387 offset += skb->csum_offset;
2388 BUG_ON(offset + sizeof(__sum16) > skb_headlen(skb));
2389
2390 if (skb_cloned(skb) &&
2391 !skb_clone_writable(skb, offset + sizeof(__sum16))) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07002392 ret = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
2393 if (ret)
2394 goto out;
2395 }
2396
Herbert Xua0308472007-10-15 01:47:15 -07002397 *(__sum16 *)(skb->data + offset) = csum_fold(csum);
Herbert Xua430a432006-07-08 13:34:56 -07002398out_set_summed:
Linus Torvalds1da177e2005-04-16 15:20:36 -07002399 skb->ip_summed = CHECKSUM_NONE;
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09002400out:
Linus Torvalds1da177e2005-04-16 15:20:36 -07002401 return ret;
2402}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07002403EXPORT_SYMBOL(skb_checksum_help);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002404
Vlad Yasevich53d64712014-03-27 17:26:18 -04002405__be16 skb_network_protocol(struct sk_buff *skb, int *depth)
Pravin B Shelarec5f0612013-03-07 09:28:01 +00002406{
2407 __be16 type = skb->protocol;
2408
Pravin B Shelar19acc322013-05-07 20:41:07 +00002409 /* Tunnel gso handlers can set protocol to ethernet. */
2410 if (type == htons(ETH_P_TEB)) {
2411 struct ethhdr *eth;
2412
2413 if (unlikely(!pskb_may_pull(skb, sizeof(struct ethhdr))))
2414 return 0;
2415
2416 eth = (struct ethhdr *)skb_mac_header(skb);
2417 type = eth->h_proto;
2418 }
2419
Toshiaki Makitad4bcef32015-01-29 20:37:07 +09002420 return __vlan_get_protocol(skb, type, depth);
Pravin B Shelarec5f0612013-03-07 09:28:01 +00002421}
2422
Pravin B Shelar05e8ef42013-02-14 09:44:55 +00002423/**
2424 * skb_mac_gso_segment - mac layer segmentation handler.
2425 * @skb: buffer to segment
2426 * @features: features for the output path (see dev->features)
2427 */
2428struct sk_buff *skb_mac_gso_segment(struct sk_buff *skb,
2429 netdev_features_t features)
2430{
2431 struct sk_buff *segs = ERR_PTR(-EPROTONOSUPPORT);
2432 struct packet_offload *ptype;
Vlad Yasevich53d64712014-03-27 17:26:18 -04002433 int vlan_depth = skb->mac_len;
2434 __be16 type = skb_network_protocol(skb, &vlan_depth);
Pravin B Shelar05e8ef42013-02-14 09:44:55 +00002435
Pravin B Shelarec5f0612013-03-07 09:28:01 +00002436 if (unlikely(!type))
2437 return ERR_PTR(-EINVAL);
Pravin B Shelar05e8ef42013-02-14 09:44:55 +00002438
Vlad Yasevich53d64712014-03-27 17:26:18 -04002439 __skb_pull(skb, vlan_depth);
Pravin B Shelar05e8ef42013-02-14 09:44:55 +00002440
2441 rcu_read_lock();
2442 list_for_each_entry_rcu(ptype, &offload_base, list) {
2443 if (ptype->type == type && ptype->callbacks.gso_segment) {
Pravin B Shelar05e8ef42013-02-14 09:44:55 +00002444 segs = ptype->callbacks.gso_segment(skb, features);
2445 break;
2446 }
2447 }
2448 rcu_read_unlock();
2449
2450 __skb_push(skb, skb->data - skb_mac_header(skb));
2451
2452 return segs;
2453}
2454EXPORT_SYMBOL(skb_mac_gso_segment);
2455
2456
Cong Wang12b00042013-02-05 16:36:38 +00002457/* openvswitch calls this on rx path, so we need a different check.
2458 */
2459static inline bool skb_needs_check(struct sk_buff *skb, bool tx_path)
2460{
2461 if (tx_path)
2462 return skb->ip_summed != CHECKSUM_PARTIAL;
2463 else
2464 return skb->ip_summed == CHECKSUM_NONE;
2465}
2466
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002467/**
Cong Wang12b00042013-02-05 16:36:38 +00002468 * __skb_gso_segment - Perform segmentation on skb.
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002469 * @skb: buffer to segment
Herbert Xu576a30e2006-06-27 13:22:38 -07002470 * @features: features for the output path (see dev->features)
Cong Wang12b00042013-02-05 16:36:38 +00002471 * @tx_path: whether it is called in TX path
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002472 *
2473 * This function segments the given skb and returns a list of segments.
Herbert Xu576a30e2006-06-27 13:22:38 -07002474 *
2475 * It may return NULL if the skb requires no segmentation. This is
2476 * only possible when GSO is used for verifying header integrity.
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002477 */
Cong Wang12b00042013-02-05 16:36:38 +00002478struct sk_buff *__skb_gso_segment(struct sk_buff *skb,
2479 netdev_features_t features, bool tx_path)
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002480{
Cong Wang12b00042013-02-05 16:36:38 +00002481 if (unlikely(skb_needs_check(skb, tx_path))) {
Pravin B Shelar05e8ef42013-02-14 09:44:55 +00002482 int err;
2483
Ben Hutchings36c92472012-01-17 07:57:56 +00002484 skb_warn_bad_offload(skb);
Herbert Xu67fd1a72009-01-19 16:26:44 -08002485
françois romieua40e0a62014-07-15 23:55:35 +02002486 err = skb_cow_head(skb, 0);
2487 if (err < 0)
Herbert Xua430a432006-07-08 13:34:56 -07002488 return ERR_PTR(err);
2489 }
2490
Pravin B Shelar68c33162013-02-14 14:02:41 +00002491 SKB_GSO_CB(skb)->mac_offset = skb_headroom(skb);
Eric Dumazet3347c962013-10-19 11:42:56 -07002492 SKB_GSO_CB(skb)->encap_level = 0;
2493
Pravin B Shelar05e8ef42013-02-14 09:44:55 +00002494 skb_reset_mac_header(skb);
2495 skb_reset_mac_len(skb);
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002496
Pravin B Shelar05e8ef42013-02-14 09:44:55 +00002497 return skb_mac_gso_segment(skb, features);
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002498}
Cong Wang12b00042013-02-05 16:36:38 +00002499EXPORT_SYMBOL(__skb_gso_segment);
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002500
Herbert Xufb286bb2005-11-10 13:01:24 -08002501/* Take action when hardware reception checksum errors are detected. */
2502#ifdef CONFIG_BUG
2503void netdev_rx_csum_fault(struct net_device *dev)
2504{
2505 if (net_ratelimit()) {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00002506 pr_err("%s: hw csum failure\n", dev ? dev->name : "<unknown>");
Herbert Xufb286bb2005-11-10 13:01:24 -08002507 dump_stack();
2508 }
2509}
2510EXPORT_SYMBOL(netdev_rx_csum_fault);
2511#endif
2512
Linus Torvalds1da177e2005-04-16 15:20:36 -07002513/* Actually, we should eliminate this check as soon as we know, that:
2514 * 1. IOMMU is present and allows to map all the memory.
2515 * 2. No high memory really exists on this machine.
2516 */
2517
Florian Westphalc1e756b2014-05-05 15:00:44 +02002518static int illegal_highdma(struct net_device *dev, struct sk_buff *skb)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002519{
Herbert Xu3d3a8532006-06-27 13:33:10 -07002520#ifdef CONFIG_HIGHMEM
Linus Torvalds1da177e2005-04-16 15:20:36 -07002521 int i;
FUJITA Tomonori5acbbd42010-03-30 22:35:50 +00002522 if (!(dev->features & NETIF_F_HIGHDMA)) {
Ian Campbellea2ab692011-08-22 23:44:58 +00002523 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
2524 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
2525 if (PageHighMem(skb_frag_page(frag)))
FUJITA Tomonori5acbbd42010-03-30 22:35:50 +00002526 return 1;
Ian Campbellea2ab692011-08-22 23:44:58 +00002527 }
FUJITA Tomonori5acbbd42010-03-30 22:35:50 +00002528 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002529
FUJITA Tomonori5acbbd42010-03-30 22:35:50 +00002530 if (PCI_DMA_BUS_IS_PHYS) {
2531 struct device *pdev = dev->dev.parent;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002532
Eric Dumazet9092c652010-04-02 13:34:49 -07002533 if (!pdev)
2534 return 0;
FUJITA Tomonori5acbbd42010-03-30 22:35:50 +00002535 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
Ian Campbellea2ab692011-08-22 23:44:58 +00002536 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
2537 dma_addr_t addr = page_to_phys(skb_frag_page(frag));
FUJITA Tomonori5acbbd42010-03-30 22:35:50 +00002538 if (!pdev->dma_mask || addr + PAGE_SIZE - 1 > *pdev->dma_mask)
2539 return 1;
2540 }
2541 }
Herbert Xu3d3a8532006-06-27 13:33:10 -07002542#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07002543 return 0;
2544}
Linus Torvalds1da177e2005-04-16 15:20:36 -07002545
Simon Horman3b392dd2014-06-04 08:53:17 +09002546/* If MPLS offload request, verify we are testing hardware MPLS features
2547 * instead of standard features for the netdev.
2548 */
Pravin B Shelard0edc7b2014-12-23 16:20:11 -08002549#if IS_ENABLED(CONFIG_NET_MPLS_GSO)
Simon Horman3b392dd2014-06-04 08:53:17 +09002550static netdev_features_t net_mpls_features(struct sk_buff *skb,
2551 netdev_features_t features,
2552 __be16 type)
2553{
Simon Horman25cd9ba2014-10-06 05:05:13 -07002554 if (eth_p_mpls(type))
Simon Horman3b392dd2014-06-04 08:53:17 +09002555 features &= skb->dev->mpls_features;
2556
2557 return features;
2558}
2559#else
2560static netdev_features_t net_mpls_features(struct sk_buff *skb,
2561 netdev_features_t features,
2562 __be16 type)
2563{
2564 return features;
2565}
2566#endif
2567
Michał Mirosławc8f44af2011-11-15 15:29:55 +00002568static netdev_features_t harmonize_features(struct sk_buff *skb,
Florian Westphalc1e756b2014-05-05 15:00:44 +02002569 netdev_features_t features)
Jesse Grossf01a5232011-01-09 06:23:31 +00002570{
Vlad Yasevich53d64712014-03-27 17:26:18 -04002571 int tmp;
Simon Horman3b392dd2014-06-04 08:53:17 +09002572 __be16 type;
2573
2574 type = skb_network_protocol(skb, &tmp);
2575 features = net_mpls_features(skb, features, type);
Vlad Yasevich53d64712014-03-27 17:26:18 -04002576
Ed Cashinc0d680e2012-09-19 15:49:00 +00002577 if (skb->ip_summed != CHECKSUM_NONE &&
Simon Horman3b392dd2014-06-04 08:53:17 +09002578 !can_checksum_protocol(features, type)) {
Jesse Grossf01a5232011-01-09 06:23:31 +00002579 features &= ~NETIF_F_ALL_CSUM;
Florian Westphalc1e756b2014-05-05 15:00:44 +02002580 } else if (illegal_highdma(skb->dev, skb)) {
Jesse Grossf01a5232011-01-09 06:23:31 +00002581 features &= ~NETIF_F_SG;
2582 }
2583
2584 return features;
2585}
2586
Toshiaki Makitae38f3022015-03-27 14:31:13 +09002587netdev_features_t passthru_features_check(struct sk_buff *skb,
2588 struct net_device *dev,
2589 netdev_features_t features)
2590{
2591 return features;
2592}
2593EXPORT_SYMBOL(passthru_features_check);
2594
Toshiaki Makita8cb65d02015-03-27 14:31:12 +09002595static netdev_features_t dflt_features_check(const struct sk_buff *skb,
2596 struct net_device *dev,
2597 netdev_features_t features)
2598{
2599 return vlan_features_check(skb, features);
2600}
2601
Florian Westphalc1e756b2014-05-05 15:00:44 +02002602netdev_features_t netif_skb_features(struct sk_buff *skb)
Jesse Gross58e998c2010-10-29 12:14:55 +00002603{
Jesse Gross5f352272014-12-23 22:37:26 -08002604 struct net_device *dev = skb->dev;
Eric Dumazetfcbeb972014-10-05 10:11:27 -07002605 netdev_features_t features = dev->features;
2606 u16 gso_segs = skb_shinfo(skb)->gso_segs;
Jesse Gross58e998c2010-10-29 12:14:55 +00002607
Eric Dumazetfcbeb972014-10-05 10:11:27 -07002608 if (gso_segs > dev->gso_max_segs || gso_segs < dev->gso_min_segs)
Ben Hutchings30b678d2012-07-30 15:57:00 +00002609 features &= ~NETIF_F_GSO_MASK;
2610
Jesse Gross5f352272014-12-23 22:37:26 -08002611 /* If encapsulation offload request, verify we are testing
2612 * hardware encapsulation features instead of standard
2613 * features for the netdev
2614 */
2615 if (skb->encapsulation)
2616 features &= dev->hw_enc_features;
2617
Toshiaki Makitaf5a7fb82015-03-27 14:31:11 +09002618 if (skb_vlan_tagged(skb))
2619 features = netdev_intersect_features(features,
2620 dev->vlan_features |
2621 NETIF_F_HW_VLAN_CTAG_TX |
2622 NETIF_F_HW_VLAN_STAG_TX);
Jesse Gross58e998c2010-10-29 12:14:55 +00002623
Jesse Gross5f352272014-12-23 22:37:26 -08002624 if (dev->netdev_ops->ndo_features_check)
2625 features &= dev->netdev_ops->ndo_features_check(skb, dev,
2626 features);
Toshiaki Makita8cb65d02015-03-27 14:31:12 +09002627 else
2628 features &= dflt_features_check(skb, dev, features);
Jesse Gross5f352272014-12-23 22:37:26 -08002629
Florian Westphalc1e756b2014-05-05 15:00:44 +02002630 return harmonize_features(skb, features);
Jesse Gross58e998c2010-10-29 12:14:55 +00002631}
Florian Westphalc1e756b2014-05-05 15:00:44 +02002632EXPORT_SYMBOL(netif_skb_features);
Jesse Gross58e998c2010-10-29 12:14:55 +00002633
David S. Miller2ea25512014-08-29 21:10:01 -07002634static int xmit_one(struct sk_buff *skb, struct net_device *dev,
David S. Miller95f6b3d2014-08-29 21:57:30 -07002635 struct netdev_queue *txq, bool more)
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002636{
David S. Miller2ea25512014-08-29 21:10:01 -07002637 unsigned int len;
2638 int rc;
Stephen Hemminger00829822008-11-20 20:14:53 -08002639
Salam Noureddine7866a622015-01-27 11:35:48 -08002640 if (!list_empty(&ptype_all) || !list_empty(&dev->ptype_all))
David S. Miller2ea25512014-08-29 21:10:01 -07002641 dev_queue_xmit_nit(skb, dev);
Jesse Grossfc741212011-01-09 06:23:32 +00002642
David S. Miller2ea25512014-08-29 21:10:01 -07002643 len = skb->len;
2644 trace_net_dev_start_xmit(skb, dev);
David S. Miller95f6b3d2014-08-29 21:57:30 -07002645 rc = netdev_start_xmit(skb, dev, txq, more);
David S. Miller2ea25512014-08-29 21:10:01 -07002646 trace_net_dev_xmit(skb, rc, dev, len);
Eric Dumazetadf30902009-06-02 05:19:30 +00002647
Patrick McHardy572a9d72009-11-10 06:14:14 +00002648 return rc;
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002649}
David S. Miller2ea25512014-08-29 21:10:01 -07002650
David S. Miller8dcda222014-09-01 15:06:40 -07002651struct sk_buff *dev_hard_start_xmit(struct sk_buff *first, struct net_device *dev,
2652 struct netdev_queue *txq, int *ret)
David S. Miller7f2e8702014-08-29 21:19:14 -07002653{
2654 struct sk_buff *skb = first;
2655 int rc = NETDEV_TX_OK;
2656
2657 while (skb) {
2658 struct sk_buff *next = skb->next;
2659
2660 skb->next = NULL;
David S. Miller95f6b3d2014-08-29 21:57:30 -07002661 rc = xmit_one(skb, dev, txq, next != NULL);
David S. Miller7f2e8702014-08-29 21:19:14 -07002662 if (unlikely(!dev_xmit_complete(rc))) {
2663 skb->next = next;
2664 goto out;
2665 }
2666
2667 skb = next;
2668 if (netif_xmit_stopped(txq) && skb) {
2669 rc = NETDEV_TX_BUSY;
2670 break;
2671 }
2672 }
2673
2674out:
2675 *ret = rc;
2676 return skb;
2677}
2678
Eric Dumazet1ff0dc92014-10-06 11:26:27 -07002679static struct sk_buff *validate_xmit_vlan(struct sk_buff *skb,
2680 netdev_features_t features)
David S. Millereae3f882014-08-30 15:17:13 -07002681{
Jiri Pirkodf8a39d2015-01-13 17:13:44 +01002682 if (skb_vlan_tag_present(skb) &&
Jiri Pirko59682502014-11-19 14:04:59 +01002683 !vlan_hw_offload_capable(features, skb->vlan_proto))
2684 skb = __vlan_hwaccel_push_inside(skb);
David S. Millereae3f882014-08-30 15:17:13 -07002685 return skb;
2686}
2687
Eric Dumazet55a93b32014-10-03 15:31:07 -07002688static struct sk_buff *validate_xmit_skb(struct sk_buff *skb, struct net_device *dev)
David S. Millereae3f882014-08-30 15:17:13 -07002689{
2690 netdev_features_t features;
2691
2692 if (skb->next)
2693 return skb;
2694
David S. Millereae3f882014-08-30 15:17:13 -07002695 features = netif_skb_features(skb);
2696 skb = validate_xmit_vlan(skb, features);
2697 if (unlikely(!skb))
2698 goto out_null;
2699
Tom Herbert04ffcb22014-10-14 15:19:06 -07002700 if (netif_needs_gso(dev, skb, features)) {
David S. Millerce937182014-08-30 19:22:20 -07002701 struct sk_buff *segs;
2702
2703 segs = skb_gso_segment(skb, features);
Jason Wangcecda692014-09-19 16:04:38 +08002704 if (IS_ERR(segs)) {
Jason Wangaf6dabc2014-12-19 11:09:13 +08002705 goto out_kfree_skb;
Jason Wangcecda692014-09-19 16:04:38 +08002706 } else if (segs) {
2707 consume_skb(skb);
2708 skb = segs;
2709 }
David S. Millereae3f882014-08-30 15:17:13 -07002710 } else {
2711 if (skb_needs_linearize(skb, features) &&
2712 __skb_linearize(skb))
2713 goto out_kfree_skb;
2714
2715 /* If packet is not checksummed and device does not
2716 * support checksumming for this protocol, complete
2717 * checksumming here.
2718 */
2719 if (skb->ip_summed == CHECKSUM_PARTIAL) {
2720 if (skb->encapsulation)
2721 skb_set_inner_transport_header(skb,
2722 skb_checksum_start_offset(skb));
2723 else
2724 skb_set_transport_header(skb,
2725 skb_checksum_start_offset(skb));
2726 if (!(features & NETIF_F_ALL_CSUM) &&
2727 skb_checksum_help(skb))
2728 goto out_kfree_skb;
2729 }
2730 }
2731
2732 return skb;
2733
2734out_kfree_skb:
2735 kfree_skb(skb);
2736out_null:
2737 return NULL;
2738}
2739
Eric Dumazet55a93b32014-10-03 15:31:07 -07002740struct sk_buff *validate_xmit_skb_list(struct sk_buff *skb, struct net_device *dev)
2741{
2742 struct sk_buff *next, *head = NULL, *tail;
2743
Eric Dumazetbec3cfd2014-10-03 20:59:19 -07002744 for (; skb != NULL; skb = next) {
Eric Dumazet55a93b32014-10-03 15:31:07 -07002745 next = skb->next;
2746 skb->next = NULL;
Eric Dumazet55a93b32014-10-03 15:31:07 -07002747
Eric Dumazetbec3cfd2014-10-03 20:59:19 -07002748 /* in case skb wont be segmented, point to itself */
2749 skb->prev = skb;
2750
2751 skb = validate_xmit_skb(skb, dev);
2752 if (!skb)
2753 continue;
2754
2755 if (!head)
2756 head = skb;
2757 else
2758 tail->next = skb;
2759 /* If skb was segmented, skb->prev points to
2760 * the last segment. If not, it still contains skb.
2761 */
2762 tail = skb->prev;
Eric Dumazet55a93b32014-10-03 15:31:07 -07002763 }
2764 return head;
2765}
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002766
Eric Dumazet1def9232013-01-10 12:36:42 +00002767static void qdisc_pkt_len_init(struct sk_buff *skb)
2768{
2769 const struct skb_shared_info *shinfo = skb_shinfo(skb);
2770
2771 qdisc_skb_cb(skb)->pkt_len = skb->len;
2772
2773 /* To get more precise estimation of bytes sent on wire,
2774 * we add to pkt_len the headers size of all segments
2775 */
2776 if (shinfo->gso_size) {
Eric Dumazet757b8b12013-01-15 21:14:21 -08002777 unsigned int hdr_len;
Jason Wang15e5a032013-03-25 20:19:59 +00002778 u16 gso_segs = shinfo->gso_segs;
Eric Dumazet1def9232013-01-10 12:36:42 +00002779
Eric Dumazet757b8b12013-01-15 21:14:21 -08002780 /* mac layer + network layer */
2781 hdr_len = skb_transport_header(skb) - skb_mac_header(skb);
2782
2783 /* + transport layer */
Eric Dumazet1def9232013-01-10 12:36:42 +00002784 if (likely(shinfo->gso_type & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6)))
2785 hdr_len += tcp_hdrlen(skb);
2786 else
2787 hdr_len += sizeof(struct udphdr);
Jason Wang15e5a032013-03-25 20:19:59 +00002788
2789 if (shinfo->gso_type & SKB_GSO_DODGY)
2790 gso_segs = DIV_ROUND_UP(skb->len - hdr_len,
2791 shinfo->gso_size);
2792
2793 qdisc_skb_cb(skb)->pkt_len += (gso_segs - 1) * hdr_len;
Eric Dumazet1def9232013-01-10 12:36:42 +00002794 }
2795}
2796
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00002797static inline int __dev_xmit_skb(struct sk_buff *skb, struct Qdisc *q,
2798 struct net_device *dev,
2799 struct netdev_queue *txq)
2800{
2801 spinlock_t *root_lock = qdisc_lock(q);
Eric Dumazeta2da5702011-01-20 03:48:19 +00002802 bool contended;
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00002803 int rc;
2804
Eric Dumazet1def9232013-01-10 12:36:42 +00002805 qdisc_pkt_len_init(skb);
Eric Dumazeta2da5702011-01-20 03:48:19 +00002806 qdisc_calculate_pkt_len(skb, q);
Eric Dumazet79640a42010-06-02 05:09:29 -07002807 /*
2808 * Heuristic to force contended enqueues to serialize on a
2809 * separate lock before trying to get qdisc main lock.
Ying Xue9bf2b8c2014-06-26 15:56:31 +08002810 * This permits __QDISC___STATE_RUNNING owner to get the lock more
2811 * often and dequeue packets faster.
Eric Dumazet79640a42010-06-02 05:09:29 -07002812 */
Eric Dumazeta2da5702011-01-20 03:48:19 +00002813 contended = qdisc_is_running(q);
Eric Dumazet79640a42010-06-02 05:09:29 -07002814 if (unlikely(contended))
2815 spin_lock(&q->busylock);
2816
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00002817 spin_lock(root_lock);
2818 if (unlikely(test_bit(__QDISC_STATE_DEACTIVATED, &q->state))) {
2819 kfree_skb(skb);
2820 rc = NET_XMIT_DROP;
2821 } else if ((q->flags & TCQ_F_CAN_BYPASS) && !qdisc_qlen(q) &&
Eric Dumazetbc135b22010-06-02 03:23:51 -07002822 qdisc_run_begin(q)) {
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00002823 /*
2824 * This is a work-conserving queue; there are no old skbs
2825 * waiting to be sent out; and the qdisc is not running -
2826 * xmit the skb directly.
2827 */
Eric Dumazetbfe0d022011-01-09 08:30:54 +00002828
Eric Dumazetbfe0d022011-01-09 08:30:54 +00002829 qdisc_bstats_update(q, skb);
2830
Eric Dumazet55a93b32014-10-03 15:31:07 -07002831 if (sch_direct_xmit(skb, q, dev, txq, root_lock, true)) {
Eric Dumazet79640a42010-06-02 05:09:29 -07002832 if (unlikely(contended)) {
2833 spin_unlock(&q->busylock);
2834 contended = false;
2835 }
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00002836 __qdisc_run(q);
Eric Dumazet79640a42010-06-02 05:09:29 -07002837 } else
Eric Dumazetbc135b22010-06-02 03:23:51 -07002838 qdisc_run_end(q);
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00002839
2840 rc = NET_XMIT_SUCCESS;
2841 } else {
Eric Dumazeta2da5702011-01-20 03:48:19 +00002842 rc = q->enqueue(skb, q) & NET_XMIT_MASK;
Eric Dumazet79640a42010-06-02 05:09:29 -07002843 if (qdisc_run_begin(q)) {
2844 if (unlikely(contended)) {
2845 spin_unlock(&q->busylock);
2846 contended = false;
2847 }
2848 __qdisc_run(q);
2849 }
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00002850 }
2851 spin_unlock(root_lock);
Eric Dumazet79640a42010-06-02 05:09:29 -07002852 if (unlikely(contended))
2853 spin_unlock(&q->busylock);
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00002854 return rc;
2855}
2856
Daniel Borkmann86f85152013-12-29 17:27:11 +01002857#if IS_ENABLED(CONFIG_CGROUP_NET_PRIO)
Neil Horman5bc14212011-11-22 05:10:51 +00002858static void skb_update_prio(struct sk_buff *skb)
2859{
Igor Maravic6977a792011-11-25 07:44:54 +00002860 struct netprio_map *map = rcu_dereference_bh(skb->dev->priomap);
Neil Horman5bc14212011-11-22 05:10:51 +00002861
Eric Dumazet91c68ce2012-07-08 21:45:10 +00002862 if (!skb->priority && skb->sk && map) {
2863 unsigned int prioidx = skb->sk->sk_cgrp_prioidx;
2864
2865 if (prioidx < map->priomap_len)
2866 skb->priority = map->priomap[prioidx];
2867 }
Neil Horman5bc14212011-11-22 05:10:51 +00002868}
2869#else
2870#define skb_update_prio(skb)
2871#endif
2872
hannes@stressinduktion.orgf60e5992015-04-01 17:07:44 +02002873DEFINE_PER_CPU(int, xmit_recursion);
2874EXPORT_SYMBOL(xmit_recursion);
2875
David S. Miller11a766c2010-10-25 12:51:55 -07002876#define RECURSION_LIMIT 10
Eric Dumazet745e20f2010-09-29 13:23:09 -07002877
Dave Jonesd29f7492008-07-22 14:09:06 -07002878/**
Michel Machado95603e22012-06-12 10:16:35 +00002879 * dev_loopback_xmit - loop back @skb
2880 * @skb: buffer to transmit
2881 */
David Miller7026b1d2015-04-05 22:19:04 -04002882int dev_loopback_xmit(struct sock *sk, struct sk_buff *skb)
Michel Machado95603e22012-06-12 10:16:35 +00002883{
2884 skb_reset_mac_header(skb);
2885 __skb_pull(skb, skb_network_offset(skb));
2886 skb->pkt_type = PACKET_LOOPBACK;
2887 skb->ip_summed = CHECKSUM_UNNECESSARY;
2888 WARN_ON(!skb_dst(skb));
2889 skb_dst_force(skb);
2890 netif_rx_ni(skb);
2891 return 0;
2892}
2893EXPORT_SYMBOL(dev_loopback_xmit);
2894
2895/**
Jason Wang9d08dd32014-01-20 11:25:13 +08002896 * __dev_queue_xmit - transmit a buffer
Dave Jonesd29f7492008-07-22 14:09:06 -07002897 * @skb: buffer to transmit
Jason Wang9d08dd32014-01-20 11:25:13 +08002898 * @accel_priv: private data used for L2 forwarding offload
Dave Jonesd29f7492008-07-22 14:09:06 -07002899 *
2900 * Queue a buffer for transmission to a network device. The caller must
2901 * have set the device and priority and built the buffer before calling
2902 * this function. The function can be called from an interrupt.
2903 *
2904 * A negative errno code is returned on a failure. A success does not
2905 * guarantee the frame will be transmitted as it may be dropped due
2906 * to congestion or traffic shaping.
2907 *
2908 * -----------------------------------------------------------------------------------
2909 * I notice this method can also return errors from the queue disciplines,
2910 * including NET_XMIT_DROP, which is a positive value. So, errors can also
2911 * be positive.
2912 *
2913 * Regardless of the return value, the skb is consumed, so it is currently
2914 * difficult to retry a send to this method. (You can bump the ref count
2915 * before sending to hold a reference for retry if you are careful.)
2916 *
2917 * When calling this method, interrupts MUST be enabled. This is because
2918 * the BH enable code must have IRQs enabled so that it will not deadlock.
2919 * --BLG
2920 */
Rashika Kheria0a59f3a2014-02-09 20:26:25 +05302921static int __dev_queue_xmit(struct sk_buff *skb, void *accel_priv)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002922{
2923 struct net_device *dev = skb->dev;
David S. Millerdc2b4842008-07-08 17:18:23 -07002924 struct netdev_queue *txq;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002925 struct Qdisc *q;
2926 int rc = -ENOMEM;
2927
Eric Dumazet6d1ccff2013-02-05 20:22:20 +00002928 skb_reset_mac_header(skb);
2929
Willem de Bruijne7fd2882014-08-04 22:11:48 -04002930 if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_SCHED_TSTAMP))
2931 __skb_tstamp_tx(skb, NULL, skb->sk, SCM_TSTAMP_SCHED);
2932
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09002933 /* Disable soft irqs for various locks below. Also
2934 * stops preemption for RCU.
Linus Torvalds1da177e2005-04-16 15:20:36 -07002935 */
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09002936 rcu_read_lock_bh();
Linus Torvalds1da177e2005-04-16 15:20:36 -07002937
Neil Horman5bc14212011-11-22 05:10:51 +00002938 skb_update_prio(skb);
2939
Eric Dumazet02875872014-10-05 18:38:35 -07002940 /* If device/qdisc don't need skb->dst, release it right now while
2941 * its hot in this cpu cache.
2942 */
2943 if (dev->priv_flags & IFF_XMIT_DST_RELEASE)
2944 skb_dst_drop(skb);
2945 else
2946 skb_dst_force(skb);
2947
Jason Wangf663dd92014-01-10 16:18:26 +08002948 txq = netdev_pick_tx(dev, skb, accel_priv);
Paul E. McKenneya898def2010-02-22 17:04:49 -08002949 q = rcu_dereference_bh(txq->qdisc);
David S. Miller37437bb2008-07-16 02:15:04 -07002950
Linus Torvalds1da177e2005-04-16 15:20:36 -07002951#ifdef CONFIG_NET_CLS_ACT
Eric Dumazetd1b19df2009-09-03 01:29:39 -07002952 skb->tc_verd = SET_TC_AT(skb->tc_verd, AT_EGRESS);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002953#endif
Koki Sanagicf66ba52010-08-23 18:45:02 +09002954 trace_net_dev_queue(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002955 if (q->enqueue) {
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00002956 rc = __dev_xmit_skb(skb, q, dev, txq);
David S. Miller37437bb2008-07-16 02:15:04 -07002957 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002958 }
2959
2960 /* The device has no queue. Common case for software devices:
2961 loopback, all the sorts of tunnels...
2962
Herbert Xu932ff272006-06-09 12:20:56 -07002963 Really, it is unlikely that netif_tx_lock protection is necessary
2964 here. (f.e. loopback and IP tunnels are clean ignoring statistics
Linus Torvalds1da177e2005-04-16 15:20:36 -07002965 counters.)
2966 However, it is possible, that they rely on protection
2967 made by us here.
2968
2969 Check this and shot the lock. It is not prone from deadlocks.
2970 Either shot noqueue qdisc, it is even simpler 8)
2971 */
2972 if (dev->flags & IFF_UP) {
2973 int cpu = smp_processor_id(); /* ok because BHs are off */
2974
David S. Millerc773e842008-07-08 23:13:53 -07002975 if (txq->xmit_lock_owner != cpu) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07002976
Eric Dumazet745e20f2010-09-29 13:23:09 -07002977 if (__this_cpu_read(xmit_recursion) > RECURSION_LIMIT)
2978 goto recursion_alert;
2979
Jesper Dangaard Brouer1f595332014-09-03 17:56:09 +02002980 skb = validate_xmit_skb(skb, dev);
2981 if (!skb)
2982 goto drop;
2983
David S. Millerc773e842008-07-08 23:13:53 -07002984 HARD_TX_LOCK(dev, txq, cpu);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002985
Tom Herbert734664982011-11-28 16:32:44 +00002986 if (!netif_xmit_stopped(txq)) {
Eric Dumazet745e20f2010-09-29 13:23:09 -07002987 __this_cpu_inc(xmit_recursion);
David S. Millerce937182014-08-30 19:22:20 -07002988 skb = dev_hard_start_xmit(skb, dev, txq, &rc);
Eric Dumazet745e20f2010-09-29 13:23:09 -07002989 __this_cpu_dec(xmit_recursion);
Patrick McHardy572a9d72009-11-10 06:14:14 +00002990 if (dev_xmit_complete(rc)) {
David S. Millerc773e842008-07-08 23:13:53 -07002991 HARD_TX_UNLOCK(dev, txq);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002992 goto out;
2993 }
2994 }
David S. Millerc773e842008-07-08 23:13:53 -07002995 HARD_TX_UNLOCK(dev, txq);
Joe Perchese87cc472012-05-13 21:56:26 +00002996 net_crit_ratelimited("Virtual device %s asks to queue packet!\n",
2997 dev->name);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002998 } else {
2999 /* Recursion is detected! It is possible,
Eric Dumazet745e20f2010-09-29 13:23:09 -07003000 * unfortunately
3001 */
3002recursion_alert:
Joe Perchese87cc472012-05-13 21:56:26 +00003003 net_crit_ratelimited("Dead loop on virtual device %s, fix it urgently!\n",
3004 dev->name);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003005 }
3006 }
3007
3008 rc = -ENETDOWN;
Jesper Dangaard Brouer1f595332014-09-03 17:56:09 +02003009drop:
Herbert Xud4828d82006-06-22 02:28:18 -07003010 rcu_read_unlock_bh();
Linus Torvalds1da177e2005-04-16 15:20:36 -07003011
Eric Dumazet015f0682014-03-27 08:45:56 -07003012 atomic_long_inc(&dev->tx_dropped);
Jesper Dangaard Brouer1f595332014-09-03 17:56:09 +02003013 kfree_skb_list(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003014 return rc;
3015out:
Herbert Xud4828d82006-06-22 02:28:18 -07003016 rcu_read_unlock_bh();
Linus Torvalds1da177e2005-04-16 15:20:36 -07003017 return rc;
3018}
Jason Wangf663dd92014-01-10 16:18:26 +08003019
David Miller7026b1d2015-04-05 22:19:04 -04003020int dev_queue_xmit_sk(struct sock *sk, struct sk_buff *skb)
Jason Wangf663dd92014-01-10 16:18:26 +08003021{
3022 return __dev_queue_xmit(skb, NULL);
3023}
David Miller7026b1d2015-04-05 22:19:04 -04003024EXPORT_SYMBOL(dev_queue_xmit_sk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003025
Jason Wangf663dd92014-01-10 16:18:26 +08003026int dev_queue_xmit_accel(struct sk_buff *skb, void *accel_priv)
3027{
3028 return __dev_queue_xmit(skb, accel_priv);
3029}
3030EXPORT_SYMBOL(dev_queue_xmit_accel);
3031
Linus Torvalds1da177e2005-04-16 15:20:36 -07003032
3033/*=======================================================================
3034 Receiver routines
3035 =======================================================================*/
3036
Stephen Hemminger6b2bedc2007-03-12 14:33:50 -07003037int netdev_max_backlog __read_mostly = 1000;
Eric Dumazetc9e6bc62012-09-27 19:29:05 +00003038EXPORT_SYMBOL(netdev_max_backlog);
3039
Eric Dumazet3b098e22010-05-15 23:57:10 -07003040int netdev_tstamp_prequeue __read_mostly = 1;
Stephen Hemminger6b2bedc2007-03-12 14:33:50 -07003041int netdev_budget __read_mostly = 300;
3042int weight_p __read_mostly = 64; /* old backlog weight */
Linus Torvalds1da177e2005-04-16 15:20:36 -07003043
Eric Dumazeteecfd7c2010-05-06 22:07:48 -07003044/* Called with irq disabled */
3045static inline void ____napi_schedule(struct softnet_data *sd,
3046 struct napi_struct *napi)
3047{
3048 list_add_tail(&napi->poll_list, &sd->poll_list);
3049 __raise_softirq_irqoff(NET_RX_SOFTIRQ);
3050}
3051
Eric Dumazetdf334542010-03-24 19:13:54 +00003052#ifdef CONFIG_RPS
Tom Herbertfec5e652010-04-16 16:01:27 -07003053
3054/* One global table that all flow-based protocols share. */
Eric Dumazet6e3f7fa2010-10-25 03:02:02 +00003055struct rps_sock_flow_table __rcu *rps_sock_flow_table __read_mostly;
Tom Herbertfec5e652010-04-16 16:01:27 -07003056EXPORT_SYMBOL(rps_sock_flow_table);
Eric Dumazet567e4b72015-02-06 12:59:01 -08003057u32 rps_cpu_mask __read_mostly;
3058EXPORT_SYMBOL(rps_cpu_mask);
Tom Herbertfec5e652010-04-16 16:01:27 -07003059
Ingo Molnarc5905af2012-02-24 08:31:31 +01003060struct static_key rps_needed __read_mostly;
Eric Dumazetadc93002011-11-17 03:13:26 +00003061
Ben Hutchingsc4454772011-01-19 11:03:53 +00003062static struct rps_dev_flow *
3063set_rps_cpu(struct net_device *dev, struct sk_buff *skb,
3064 struct rps_dev_flow *rflow, u16 next_cpu)
3065{
Ben Hutchings09994d12011-10-03 04:42:46 +00003066 if (next_cpu != RPS_NO_CPU) {
Ben Hutchingsc4454772011-01-19 11:03:53 +00003067#ifdef CONFIG_RFS_ACCEL
3068 struct netdev_rx_queue *rxqueue;
3069 struct rps_dev_flow_table *flow_table;
3070 struct rps_dev_flow *old_rflow;
3071 u32 flow_id;
3072 u16 rxq_index;
3073 int rc;
3074
3075 /* Should we steer this flow to a different hardware queue? */
Ben Hutchings69a19ee2011-02-15 20:32:04 +00003076 if (!skb_rx_queue_recorded(skb) || !dev->rx_cpu_rmap ||
3077 !(dev->features & NETIF_F_NTUPLE))
Ben Hutchingsc4454772011-01-19 11:03:53 +00003078 goto out;
3079 rxq_index = cpu_rmap_lookup_index(dev->rx_cpu_rmap, next_cpu);
3080 if (rxq_index == skb_get_rx_queue(skb))
3081 goto out;
3082
3083 rxqueue = dev->_rx + rxq_index;
3084 flow_table = rcu_dereference(rxqueue->rps_flow_table);
3085 if (!flow_table)
3086 goto out;
Tom Herbert61b905d2014-03-24 15:34:47 -07003087 flow_id = skb_get_hash(skb) & flow_table->mask;
Ben Hutchingsc4454772011-01-19 11:03:53 +00003088 rc = dev->netdev_ops->ndo_rx_flow_steer(dev, skb,
3089 rxq_index, flow_id);
3090 if (rc < 0)
3091 goto out;
3092 old_rflow = rflow;
3093 rflow = &flow_table->flows[flow_id];
Ben Hutchingsc4454772011-01-19 11:03:53 +00003094 rflow->filter = rc;
3095 if (old_rflow->filter == rflow->filter)
3096 old_rflow->filter = RPS_NO_FILTER;
3097 out:
3098#endif
3099 rflow->last_qtail =
Ben Hutchings09994d12011-10-03 04:42:46 +00003100 per_cpu(softnet_data, next_cpu).input_queue_head;
Ben Hutchingsc4454772011-01-19 11:03:53 +00003101 }
3102
Ben Hutchings09994d12011-10-03 04:42:46 +00003103 rflow->cpu = next_cpu;
Ben Hutchingsc4454772011-01-19 11:03:53 +00003104 return rflow;
3105}
3106
Tom Herbert0a9627f2010-03-16 08:03:29 +00003107/*
3108 * get_rps_cpu is called from netif_receive_skb and returns the target
3109 * CPU from the RPS map of the receiving queue for a given skb.
Eric Dumazetb0e28f12010-04-15 00:14:07 -07003110 * rcu_read_lock must be held on entry.
Tom Herbert0a9627f2010-03-16 08:03:29 +00003111 */
Tom Herbertfec5e652010-04-16 16:01:27 -07003112static int get_rps_cpu(struct net_device *dev, struct sk_buff *skb,
3113 struct rps_dev_flow **rflowp)
Tom Herbert0a9627f2010-03-16 08:03:29 +00003114{
Eric Dumazet567e4b72015-02-06 12:59:01 -08003115 const struct rps_sock_flow_table *sock_flow_table;
3116 struct netdev_rx_queue *rxqueue = dev->_rx;
Tom Herbertfec5e652010-04-16 16:01:27 -07003117 struct rps_dev_flow_table *flow_table;
Eric Dumazet567e4b72015-02-06 12:59:01 -08003118 struct rps_map *map;
Tom Herbert0a9627f2010-03-16 08:03:29 +00003119 int cpu = -1;
Eric Dumazet567e4b72015-02-06 12:59:01 -08003120 u32 tcpu;
Tom Herbert61b905d2014-03-24 15:34:47 -07003121 u32 hash;
Tom Herbert0a9627f2010-03-16 08:03:29 +00003122
Tom Herbert0a9627f2010-03-16 08:03:29 +00003123 if (skb_rx_queue_recorded(skb)) {
3124 u16 index = skb_get_rx_queue(skb);
Eric Dumazet567e4b72015-02-06 12:59:01 -08003125
Ben Hutchings62fe0b42010-09-27 08:24:33 +00003126 if (unlikely(index >= dev->real_num_rx_queues)) {
3127 WARN_ONCE(dev->real_num_rx_queues > 1,
3128 "%s received packet on queue %u, but number "
3129 "of RX queues is %u\n",
3130 dev->name, index, dev->real_num_rx_queues);
Tom Herbert0a9627f2010-03-16 08:03:29 +00003131 goto done;
3132 }
Eric Dumazet567e4b72015-02-06 12:59:01 -08003133 rxqueue += index;
Tom Herbert0a9627f2010-03-16 08:03:29 +00003134 }
3135
Eric Dumazet567e4b72015-02-06 12:59:01 -08003136 /* Avoid computing hash if RFS/RPS is not active for this rxqueue */
3137
3138 flow_table = rcu_dereference(rxqueue->rps_flow_table);
3139 map = rcu_dereference(rxqueue->rps_map);
3140 if (!flow_table && !map)
3141 goto done;
3142
Changli Gao2d47b452010-08-17 19:00:56 +00003143 skb_reset_network_header(skb);
Tom Herbert61b905d2014-03-24 15:34:47 -07003144 hash = skb_get_hash(skb);
3145 if (!hash)
Tom Herbert0a9627f2010-03-16 08:03:29 +00003146 goto done;
Tom Herbert0a9627f2010-03-16 08:03:29 +00003147
Tom Herbertfec5e652010-04-16 16:01:27 -07003148 sock_flow_table = rcu_dereference(rps_sock_flow_table);
3149 if (flow_table && sock_flow_table) {
Tom Herbertfec5e652010-04-16 16:01:27 -07003150 struct rps_dev_flow *rflow;
Eric Dumazet567e4b72015-02-06 12:59:01 -08003151 u32 next_cpu;
3152 u32 ident;
Tom Herbertfec5e652010-04-16 16:01:27 -07003153
Eric Dumazet567e4b72015-02-06 12:59:01 -08003154 /* First check into global flow table if there is a match */
3155 ident = sock_flow_table->ents[hash & sock_flow_table->mask];
3156 if ((ident ^ hash) & ~rps_cpu_mask)
3157 goto try_rps;
3158
3159 next_cpu = ident & rps_cpu_mask;
3160
3161 /* OK, now we know there is a match,
3162 * we can look at the local (per receive queue) flow table
3163 */
Tom Herbert61b905d2014-03-24 15:34:47 -07003164 rflow = &flow_table->flows[hash & flow_table->mask];
Tom Herbertfec5e652010-04-16 16:01:27 -07003165 tcpu = rflow->cpu;
3166
Tom Herbertfec5e652010-04-16 16:01:27 -07003167 /*
3168 * If the desired CPU (where last recvmsg was done) is
3169 * different from current CPU (one in the rx-queue flow
3170 * table entry), switch if one of the following holds:
3171 * - Current CPU is unset (equal to RPS_NO_CPU).
3172 * - Current CPU is offline.
3173 * - The current CPU's queue tail has advanced beyond the
3174 * last packet that was enqueued using this table entry.
3175 * This guarantees that all previous packets for the flow
3176 * have been dequeued, thus preserving in order delivery.
3177 */
3178 if (unlikely(tcpu != next_cpu) &&
3179 (tcpu == RPS_NO_CPU || !cpu_online(tcpu) ||
3180 ((int)(per_cpu(softnet_data, tcpu).input_queue_head -
Tom Herbertbaefa312012-11-16 09:04:15 +00003181 rflow->last_qtail)) >= 0)) {
3182 tcpu = next_cpu;
Ben Hutchingsc4454772011-01-19 11:03:53 +00003183 rflow = set_rps_cpu(dev, skb, rflow, next_cpu);
Tom Herbertbaefa312012-11-16 09:04:15 +00003184 }
Ben Hutchingsc4454772011-01-19 11:03:53 +00003185
Tom Herbertfec5e652010-04-16 16:01:27 -07003186 if (tcpu != RPS_NO_CPU && cpu_online(tcpu)) {
3187 *rflowp = rflow;
3188 cpu = tcpu;
3189 goto done;
3190 }
3191 }
3192
Eric Dumazet567e4b72015-02-06 12:59:01 -08003193try_rps:
3194
Tom Herbert0a9627f2010-03-16 08:03:29 +00003195 if (map) {
Daniel Borkmann8fc54f62014-08-23 20:58:54 +02003196 tcpu = map->cpus[reciprocal_scale(hash, map->len)];
Tom Herbert0a9627f2010-03-16 08:03:29 +00003197 if (cpu_online(tcpu)) {
3198 cpu = tcpu;
3199 goto done;
3200 }
3201 }
3202
3203done:
Tom Herbert0a9627f2010-03-16 08:03:29 +00003204 return cpu;
3205}
3206
Ben Hutchingsc4454772011-01-19 11:03:53 +00003207#ifdef CONFIG_RFS_ACCEL
3208
3209/**
3210 * rps_may_expire_flow - check whether an RFS hardware filter may be removed
3211 * @dev: Device on which the filter was set
3212 * @rxq_index: RX queue index
3213 * @flow_id: Flow ID passed to ndo_rx_flow_steer()
3214 * @filter_id: Filter ID returned by ndo_rx_flow_steer()
3215 *
3216 * Drivers that implement ndo_rx_flow_steer() should periodically call
3217 * this function for each installed filter and remove the filters for
3218 * which it returns %true.
3219 */
3220bool rps_may_expire_flow(struct net_device *dev, u16 rxq_index,
3221 u32 flow_id, u16 filter_id)
3222{
3223 struct netdev_rx_queue *rxqueue = dev->_rx + rxq_index;
3224 struct rps_dev_flow_table *flow_table;
3225 struct rps_dev_flow *rflow;
3226 bool expire = true;
3227 int cpu;
3228
3229 rcu_read_lock();
3230 flow_table = rcu_dereference(rxqueue->rps_flow_table);
3231 if (flow_table && flow_id <= flow_table->mask) {
3232 rflow = &flow_table->flows[flow_id];
3233 cpu = ACCESS_ONCE(rflow->cpu);
3234 if (rflow->filter == filter_id && cpu != RPS_NO_CPU &&
3235 ((int)(per_cpu(softnet_data, cpu).input_queue_head -
3236 rflow->last_qtail) <
3237 (int)(10 * flow_table->mask)))
3238 expire = false;
3239 }
3240 rcu_read_unlock();
3241 return expire;
3242}
3243EXPORT_SYMBOL(rps_may_expire_flow);
3244
3245#endif /* CONFIG_RFS_ACCEL */
3246
Tom Herbert0a9627f2010-03-16 08:03:29 +00003247/* Called from hardirq (IPI) context */
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003248static void rps_trigger_softirq(void *data)
Tom Herbert0a9627f2010-03-16 08:03:29 +00003249{
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003250 struct softnet_data *sd = data;
3251
Eric Dumazeteecfd7c2010-05-06 22:07:48 -07003252 ____napi_schedule(sd, &sd->backlog);
Changli Gaodee42872010-05-02 05:42:16 +00003253 sd->received_rps++;
Tom Herbert0a9627f2010-03-16 08:03:29 +00003254}
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003255
Tom Herbertfec5e652010-04-16 16:01:27 -07003256#endif /* CONFIG_RPS */
Tom Herbert0a9627f2010-03-16 08:03:29 +00003257
3258/*
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003259 * Check if this softnet_data structure is another cpu one
3260 * If yes, queue it to our IPI list and return 1
3261 * If no, return 0
3262 */
3263static int rps_ipi_queued(struct softnet_data *sd)
3264{
3265#ifdef CONFIG_RPS
Christoph Lameter903ceff2014-08-17 12:30:35 -05003266 struct softnet_data *mysd = this_cpu_ptr(&softnet_data);
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003267
3268 if (sd != mysd) {
3269 sd->rps_ipi_next = mysd->rps_ipi_list;
3270 mysd->rps_ipi_list = sd;
3271
3272 __raise_softirq_irqoff(NET_RX_SOFTIRQ);
3273 return 1;
3274 }
3275#endif /* CONFIG_RPS */
3276 return 0;
3277}
3278
Willem de Bruijn99bbc702013-05-20 04:02:32 +00003279#ifdef CONFIG_NET_FLOW_LIMIT
3280int netdev_flow_limit_table_len __read_mostly = (1 << 12);
3281#endif
3282
3283static bool skb_flow_limit(struct sk_buff *skb, unsigned int qlen)
3284{
3285#ifdef CONFIG_NET_FLOW_LIMIT
3286 struct sd_flow_limit *fl;
3287 struct softnet_data *sd;
3288 unsigned int old_flow, new_flow;
3289
3290 if (qlen < (netdev_max_backlog >> 1))
3291 return false;
3292
Christoph Lameter903ceff2014-08-17 12:30:35 -05003293 sd = this_cpu_ptr(&softnet_data);
Willem de Bruijn99bbc702013-05-20 04:02:32 +00003294
3295 rcu_read_lock();
3296 fl = rcu_dereference(sd->flow_limit);
3297 if (fl) {
Tom Herbert3958afa1b2013-12-15 22:12:06 -08003298 new_flow = skb_get_hash(skb) & (fl->num_buckets - 1);
Willem de Bruijn99bbc702013-05-20 04:02:32 +00003299 old_flow = fl->history[fl->history_head];
3300 fl->history[fl->history_head] = new_flow;
3301
3302 fl->history_head++;
3303 fl->history_head &= FLOW_LIMIT_HISTORY - 1;
3304
3305 if (likely(fl->buckets[old_flow]))
3306 fl->buckets[old_flow]--;
3307
3308 if (++fl->buckets[new_flow] > (FLOW_LIMIT_HISTORY >> 1)) {
3309 fl->count++;
3310 rcu_read_unlock();
3311 return true;
3312 }
3313 }
3314 rcu_read_unlock();
3315#endif
3316 return false;
3317}
3318
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003319/*
Tom Herbert0a9627f2010-03-16 08:03:29 +00003320 * enqueue_to_backlog is called to queue an skb to a per CPU backlog
3321 * queue (may be a remote CPU queue).
3322 */
Tom Herbertfec5e652010-04-16 16:01:27 -07003323static int enqueue_to_backlog(struct sk_buff *skb, int cpu,
3324 unsigned int *qtail)
Tom Herbert0a9627f2010-03-16 08:03:29 +00003325{
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003326 struct softnet_data *sd;
Tom Herbert0a9627f2010-03-16 08:03:29 +00003327 unsigned long flags;
Willem de Bruijn99bbc702013-05-20 04:02:32 +00003328 unsigned int qlen;
Tom Herbert0a9627f2010-03-16 08:03:29 +00003329
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003330 sd = &per_cpu(softnet_data, cpu);
Tom Herbert0a9627f2010-03-16 08:03:29 +00003331
3332 local_irq_save(flags);
Tom Herbert0a9627f2010-03-16 08:03:29 +00003333
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003334 rps_lock(sd);
Willem de Bruijn99bbc702013-05-20 04:02:32 +00003335 qlen = skb_queue_len(&sd->input_pkt_queue);
3336 if (qlen <= netdev_max_backlog && !skb_flow_limit(skb, qlen)) {
Li RongQinge008f3f2014-12-08 09:42:55 +08003337 if (qlen) {
Tom Herbert0a9627f2010-03-16 08:03:29 +00003338enqueue:
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003339 __skb_queue_tail(&sd->input_pkt_queue, skb);
Tom Herbert76cc8b12010-05-20 18:37:59 +00003340 input_queue_tail_incr_save(sd, qtail);
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003341 rps_unlock(sd);
Changli Gao152102c2010-03-30 20:16:22 +00003342 local_irq_restore(flags);
Tom Herbert0a9627f2010-03-16 08:03:29 +00003343 return NET_RX_SUCCESS;
3344 }
3345
Eric Dumazetebda37c22010-05-06 23:51:21 +00003346 /* Schedule NAPI for backlog device
3347 * We can use non atomic operation since we own the queue lock
3348 */
3349 if (!__test_and_set_bit(NAPI_STATE_SCHED, &sd->backlog.state)) {
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003350 if (!rps_ipi_queued(sd))
Eric Dumazeteecfd7c2010-05-06 22:07:48 -07003351 ____napi_schedule(sd, &sd->backlog);
Tom Herbert0a9627f2010-03-16 08:03:29 +00003352 }
3353 goto enqueue;
3354 }
3355
Changli Gaodee42872010-05-02 05:42:16 +00003356 sd->dropped++;
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003357 rps_unlock(sd);
Tom Herbert0a9627f2010-03-16 08:03:29 +00003358
Tom Herbert0a9627f2010-03-16 08:03:29 +00003359 local_irq_restore(flags);
3360
Eric Dumazetcaf586e2010-09-30 21:06:55 +00003361 atomic_long_inc(&skb->dev->rx_dropped);
Tom Herbert0a9627f2010-03-16 08:03:29 +00003362 kfree_skb(skb);
3363 return NET_RX_DROP;
3364}
Linus Torvalds1da177e2005-04-16 15:20:36 -07003365
Ben Hutchingsae78dbf2014-01-10 22:17:24 +00003366static int netif_rx_internal(struct sk_buff *skb)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003367{
Eric Dumazetb0e28f12010-04-15 00:14:07 -07003368 int ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003369
Eric Dumazet588f0332011-11-15 04:12:55 +00003370 net_timestamp_check(netdev_tstamp_prequeue, skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003371
Koki Sanagicf66ba52010-08-23 18:45:02 +09003372 trace_netif_rx(skb);
Eric Dumazetdf334542010-03-24 19:13:54 +00003373#ifdef CONFIG_RPS
Ingo Molnarc5905af2012-02-24 08:31:31 +01003374 if (static_key_false(&rps_needed)) {
Tom Herbertfec5e652010-04-16 16:01:27 -07003375 struct rps_dev_flow voidflow, *rflow = &voidflow;
Eric Dumazetb0e28f12010-04-15 00:14:07 -07003376 int cpu;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003377
Changli Gaocece1942010-08-07 20:35:43 -07003378 preempt_disable();
Eric Dumazetb0e28f12010-04-15 00:14:07 -07003379 rcu_read_lock();
Tom Herbertfec5e652010-04-16 16:01:27 -07003380
3381 cpu = get_rps_cpu(skb->dev, skb, &rflow);
Eric Dumazetb0e28f12010-04-15 00:14:07 -07003382 if (cpu < 0)
3383 cpu = smp_processor_id();
Tom Herbertfec5e652010-04-16 16:01:27 -07003384
3385 ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail);
3386
Eric Dumazetb0e28f12010-04-15 00:14:07 -07003387 rcu_read_unlock();
Changli Gaocece1942010-08-07 20:35:43 -07003388 preempt_enable();
Eric Dumazetadc93002011-11-17 03:13:26 +00003389 } else
3390#endif
Tom Herbertfec5e652010-04-16 16:01:27 -07003391 {
3392 unsigned int qtail;
3393 ret = enqueue_to_backlog(skb, get_cpu(), &qtail);
3394 put_cpu();
3395 }
Eric Dumazetb0e28f12010-04-15 00:14:07 -07003396 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003397}
Ben Hutchingsae78dbf2014-01-10 22:17:24 +00003398
3399/**
3400 * netif_rx - post buffer to the network code
3401 * @skb: buffer to post
3402 *
3403 * This function receives a packet from a device driver and queues it for
3404 * the upper (protocol) levels to process. It always succeeds. The buffer
3405 * may be dropped during processing for congestion control or by the
3406 * protocol layers.
3407 *
3408 * return values:
3409 * NET_RX_SUCCESS (no congestion)
3410 * NET_RX_DROP (packet was dropped)
3411 *
3412 */
3413
3414int netif_rx(struct sk_buff *skb)
3415{
3416 trace_netif_rx_entry(skb);
3417
3418 return netif_rx_internal(skb);
3419}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07003420EXPORT_SYMBOL(netif_rx);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003421
3422int netif_rx_ni(struct sk_buff *skb)
3423{
3424 int err;
3425
Ben Hutchingsae78dbf2014-01-10 22:17:24 +00003426 trace_netif_rx_ni_entry(skb);
3427
Linus Torvalds1da177e2005-04-16 15:20:36 -07003428 preempt_disable();
Ben Hutchingsae78dbf2014-01-10 22:17:24 +00003429 err = netif_rx_internal(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003430 if (local_softirq_pending())
3431 do_softirq();
3432 preempt_enable();
3433
3434 return err;
3435}
Linus Torvalds1da177e2005-04-16 15:20:36 -07003436EXPORT_SYMBOL(netif_rx_ni);
3437
Linus Torvalds1da177e2005-04-16 15:20:36 -07003438static void net_tx_action(struct softirq_action *h)
3439{
Christoph Lameter903ceff2014-08-17 12:30:35 -05003440 struct softnet_data *sd = this_cpu_ptr(&softnet_data);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003441
3442 if (sd->completion_queue) {
3443 struct sk_buff *clist;
3444
3445 local_irq_disable();
3446 clist = sd->completion_queue;
3447 sd->completion_queue = NULL;
3448 local_irq_enable();
3449
3450 while (clist) {
3451 struct sk_buff *skb = clist;
3452 clist = clist->next;
3453
Ilpo Järvinen547b7922008-07-25 21:43:18 -07003454 WARN_ON(atomic_read(&skb->users));
Eric Dumazete6247022013-12-05 04:45:08 -08003455 if (likely(get_kfree_skb_cb(skb)->reason == SKB_REASON_CONSUMED))
3456 trace_consume_skb(skb);
3457 else
3458 trace_kfree_skb(skb, net_tx_action);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003459 __kfree_skb(skb);
3460 }
3461 }
3462
3463 if (sd->output_queue) {
David S. Miller37437bb2008-07-16 02:15:04 -07003464 struct Qdisc *head;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003465
3466 local_irq_disable();
3467 head = sd->output_queue;
3468 sd->output_queue = NULL;
Changli Gaoa9cbd582010-04-26 23:06:24 +00003469 sd->output_queue_tailp = &sd->output_queue;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003470 local_irq_enable();
3471
3472 while (head) {
David S. Miller37437bb2008-07-16 02:15:04 -07003473 struct Qdisc *q = head;
3474 spinlock_t *root_lock;
3475
Linus Torvalds1da177e2005-04-16 15:20:36 -07003476 head = head->next_sched;
3477
David S. Miller5fb66222008-08-02 20:02:43 -07003478 root_lock = qdisc_lock(q);
David S. Miller37437bb2008-07-16 02:15:04 -07003479 if (spin_trylock(root_lock)) {
Peter Zijlstra4e857c52014-03-17 18:06:10 +01003480 smp_mb__before_atomic();
Jarek Poplawskidef82a12008-08-17 21:54:43 -07003481 clear_bit(__QDISC_STATE_SCHED,
3482 &q->state);
David S. Miller37437bb2008-07-16 02:15:04 -07003483 qdisc_run(q);
3484 spin_unlock(root_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003485 } else {
David S. Miller195648b2008-08-19 04:00:36 -07003486 if (!test_bit(__QDISC_STATE_DEACTIVATED,
Jarek Poplawskie8a83e12008-09-07 18:41:21 -07003487 &q->state)) {
David S. Miller195648b2008-08-19 04:00:36 -07003488 __netif_reschedule(q);
Jarek Poplawskie8a83e12008-09-07 18:41:21 -07003489 } else {
Peter Zijlstra4e857c52014-03-17 18:06:10 +01003490 smp_mb__before_atomic();
Jarek Poplawskie8a83e12008-09-07 18:41:21 -07003491 clear_bit(__QDISC_STATE_SCHED,
3492 &q->state);
3493 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07003494 }
3495 }
3496 }
3497}
3498
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00003499#if (defined(CONFIG_BRIDGE) || defined(CONFIG_BRIDGE_MODULE)) && \
3500 (defined(CONFIG_ATM_LANE) || defined(CONFIG_ATM_LANE_MODULE))
Michał Mirosławda678292009-06-05 05:35:28 +00003501/* This hook is defined here for ATM LANE */
3502int (*br_fdb_test_addr_hook)(struct net_device *dev,
3503 unsigned char *addr) __read_mostly;
Stephen Hemminger4fb019a2009-09-11 11:50:08 -07003504EXPORT_SYMBOL_GPL(br_fdb_test_addr_hook);
Michał Mirosławda678292009-06-05 05:35:28 +00003505#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07003506
Linus Torvalds1da177e2005-04-16 15:20:36 -07003507#ifdef CONFIG_NET_CLS_ACT
3508/* TODO: Maybe we should just force sch_ingress to be compiled in
3509 * when CONFIG_NET_CLS_ACT is? otherwise some useless instructions
3510 * a compare and 2 stores extra right now if we dont have it on
3511 * but have CONFIG_NET_CLS_ACT
Lucas De Marchi25985ed2011-03-30 22:57:33 -03003512 * NOTE: This doesn't stop any functionality; if you dont have
3513 * the ingress scheduler, you just can't add policies on ingress.
Linus Torvalds1da177e2005-04-16 15:20:36 -07003514 *
3515 */
Eric Dumazet24824a02010-10-02 06:11:55 +00003516static int ing_filter(struct sk_buff *skb, struct netdev_queue *rxq)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003517{
Linus Torvalds1da177e2005-04-16 15:20:36 -07003518 struct net_device *dev = skb->dev;
Herbert Xuf697c3e2007-10-14 00:38:47 -07003519 u32 ttl = G_TC_RTTL(skb->tc_verd);
David S. Miller555353c2008-07-08 17:33:13 -07003520 int result = TC_ACT_OK;
3521 struct Qdisc *q;
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09003522
Stephen Hemmingerde384832010-08-01 00:33:23 -07003523 if (unlikely(MAX_RED_LOOP < ttl++)) {
Joe Perchese87cc472012-05-13 21:56:26 +00003524 net_warn_ratelimited("Redir loop detected Dropping packet (%d->%d)\n",
3525 skb->skb_iif, dev->ifindex);
Herbert Xuf697c3e2007-10-14 00:38:47 -07003526 return TC_ACT_SHOT;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003527 }
3528
Herbert Xuf697c3e2007-10-14 00:38:47 -07003529 skb->tc_verd = SET_TC_RTTL(skb->tc_verd, ttl);
3530 skb->tc_verd = SET_TC_AT(skb->tc_verd, AT_INGRESS);
3531
John Fastabend46e5da42014-09-12 20:04:52 -07003532 q = rcu_dereference(rxq->qdisc);
David S. Miller8d50b532008-07-30 02:37:46 -07003533 if (q != &noop_qdisc) {
David S. Miller83874002008-07-17 00:53:03 -07003534 spin_lock(qdisc_lock(q));
David S. Millera9312ae2008-08-17 21:51:03 -07003535 if (likely(!test_bit(__QDISC_STATE_DEACTIVATED, &q->state)))
3536 result = qdisc_enqueue_root(skb, q);
David S. Miller83874002008-07-17 00:53:03 -07003537 spin_unlock(qdisc_lock(q));
3538 }
Herbert Xuf697c3e2007-10-14 00:38:47 -07003539
Linus Torvalds1da177e2005-04-16 15:20:36 -07003540 return result;
3541}
Herbert Xuf697c3e2007-10-14 00:38:47 -07003542
3543static inline struct sk_buff *handle_ing(struct sk_buff *skb,
3544 struct packet_type **pt_prev,
3545 int *ret, struct net_device *orig_dev)
3546{
Eric Dumazet24824a02010-10-02 06:11:55 +00003547 struct netdev_queue *rxq = rcu_dereference(skb->dev->ingress_queue);
3548
John Fastabend46e5da42014-09-12 20:04:52 -07003549 if (!rxq || rcu_access_pointer(rxq->qdisc) == &noop_qdisc)
Herbert Xuf697c3e2007-10-14 00:38:47 -07003550 goto out;
3551
3552 if (*pt_prev) {
3553 *ret = deliver_skb(skb, *pt_prev, orig_dev);
3554 *pt_prev = NULL;
Herbert Xuf697c3e2007-10-14 00:38:47 -07003555 }
3556
Eric Dumazet24824a02010-10-02 06:11:55 +00003557 switch (ing_filter(skb, rxq)) {
Herbert Xuf697c3e2007-10-14 00:38:47 -07003558 case TC_ACT_SHOT:
3559 case TC_ACT_STOLEN:
3560 kfree_skb(skb);
3561 return NULL;
3562 }
3563
3564out:
3565 skb->tc_verd = 0;
3566 return skb;
3567}
Linus Torvalds1da177e2005-04-16 15:20:36 -07003568#endif
3569
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00003570/**
3571 * netdev_rx_handler_register - register receive handler
3572 * @dev: device to register a handler for
3573 * @rx_handler: receive handler to register
Jiri Pirko93e2c322010-06-10 03:34:59 +00003574 * @rx_handler_data: data pointer that is used by rx handler
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00003575 *
Masanari Iidae2278672014-02-18 22:54:36 +09003576 * Register a receive handler for a device. This handler will then be
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00003577 * called from __netif_receive_skb. A negative errno code is returned
3578 * on a failure.
3579 *
3580 * The caller must hold the rtnl_mutex.
Jiri Pirko8a4eb572011-03-12 03:14:39 +00003581 *
3582 * For a general description of rx_handler, see enum rx_handler_result.
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00003583 */
3584int netdev_rx_handler_register(struct net_device *dev,
Jiri Pirko93e2c322010-06-10 03:34:59 +00003585 rx_handler_func_t *rx_handler,
3586 void *rx_handler_data)
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00003587{
3588 ASSERT_RTNL();
3589
3590 if (dev->rx_handler)
3591 return -EBUSY;
3592
Eric Dumazet00cfec32013-03-29 03:01:22 +00003593 /* Note: rx_handler_data must be set before rx_handler */
Jiri Pirko93e2c322010-06-10 03:34:59 +00003594 rcu_assign_pointer(dev->rx_handler_data, rx_handler_data);
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00003595 rcu_assign_pointer(dev->rx_handler, rx_handler);
3596
3597 return 0;
3598}
3599EXPORT_SYMBOL_GPL(netdev_rx_handler_register);
3600
3601/**
3602 * netdev_rx_handler_unregister - unregister receive handler
3603 * @dev: device to unregister a handler from
3604 *
Kusanagi Kouichi166ec362013-03-18 02:59:52 +00003605 * Unregister a receive handler from a device.
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00003606 *
3607 * The caller must hold the rtnl_mutex.
3608 */
3609void netdev_rx_handler_unregister(struct net_device *dev)
3610{
3611
3612 ASSERT_RTNL();
Stephen Hemmingera9b3cd72011-08-01 16:19:00 +00003613 RCU_INIT_POINTER(dev->rx_handler, NULL);
Eric Dumazet00cfec32013-03-29 03:01:22 +00003614 /* a reader seeing a non NULL rx_handler in a rcu_read_lock()
3615 * section has a guarantee to see a non NULL rx_handler_data
3616 * as well.
3617 */
3618 synchronize_net();
Stephen Hemmingera9b3cd72011-08-01 16:19:00 +00003619 RCU_INIT_POINTER(dev->rx_handler_data, NULL);
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00003620}
3621EXPORT_SYMBOL_GPL(netdev_rx_handler_unregister);
3622
Mel Gormanb4b9e352012-07-31 16:44:26 -07003623/*
3624 * Limit the use of PFMEMALLOC reserves to those protocols that implement
3625 * the special handling of PFMEMALLOC skbs.
3626 */
3627static bool skb_pfmemalloc_protocol(struct sk_buff *skb)
3628{
3629 switch (skb->protocol) {
Joe Perches2b8837a2014-03-12 10:04:17 -07003630 case htons(ETH_P_ARP):
3631 case htons(ETH_P_IP):
3632 case htons(ETH_P_IPV6):
3633 case htons(ETH_P_8021Q):
3634 case htons(ETH_P_8021AD):
Mel Gormanb4b9e352012-07-31 16:44:26 -07003635 return true;
3636 default:
3637 return false;
3638 }
3639}
3640
David S. Miller9754e292013-02-14 15:57:38 -05003641static int __netif_receive_skb_core(struct sk_buff *skb, bool pfmemalloc)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003642{
3643 struct packet_type *ptype, *pt_prev;
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00003644 rx_handler_func_t *rx_handler;
David S. Millerf2ccd8f2005-08-09 19:34:12 -07003645 struct net_device *orig_dev;
Jiri Pirko8a4eb572011-03-12 03:14:39 +00003646 bool deliver_exact = false;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003647 int ret = NET_RX_DROP;
Al Viro252e3342006-11-14 20:48:11 -08003648 __be16 type;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003649
Eric Dumazet588f0332011-11-15 04:12:55 +00003650 net_timestamp_check(!netdev_tstamp_prequeue, skb);
Eric Dumazet81bbb3d2009-09-30 16:42:42 -07003651
Koki Sanagicf66ba52010-08-23 18:45:02 +09003652 trace_netif_receive_skb(skb);
Patrick McHardy9b22ea52008-11-04 14:49:57 -08003653
Joe Eykholtcc9bd5c2008-07-02 18:22:00 -07003654 orig_dev = skb->dev;
Jiri Pirko1765a572011-02-12 06:48:36 +00003655
Arnaldo Carvalho de Meloc1d2bbe2007-04-10 20:45:18 -07003656 skb_reset_network_header(skb);
Eric Dumazetfda55ec2013-01-07 09:28:21 +00003657 if (!skb_transport_header_was_set(skb))
3658 skb_reset_transport_header(skb);
Jiri Pirko0b5c9db2011-06-10 06:56:58 +00003659 skb_reset_mac_len(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003660
3661 pt_prev = NULL;
3662
3663 rcu_read_lock();
3664
David S. Miller63d8ea72011-02-28 10:48:59 -08003665another_round:
David S. Millerb6858172012-07-23 16:27:54 -07003666 skb->skb_iif = skb->dev->ifindex;
David S. Miller63d8ea72011-02-28 10:48:59 -08003667
3668 __this_cpu_inc(softnet_data.processed);
3669
Patrick McHardy8ad227f2013-04-19 02:04:31 +00003670 if (skb->protocol == cpu_to_be16(ETH_P_8021Q) ||
3671 skb->protocol == cpu_to_be16(ETH_P_8021AD)) {
Vlad Yasevich0d5501c2014-08-08 14:42:13 -04003672 skb = skb_vlan_untag(skb);
Jiri Pirkobcc6d472011-04-07 19:48:33 +00003673 if (unlikely(!skb))
Mel Gormanb4b9e352012-07-31 16:44:26 -07003674 goto unlock;
Jiri Pirkobcc6d472011-04-07 19:48:33 +00003675 }
3676
Linus Torvalds1da177e2005-04-16 15:20:36 -07003677#ifdef CONFIG_NET_CLS_ACT
3678 if (skb->tc_verd & TC_NCLS) {
3679 skb->tc_verd = CLR_TC_NCLS(skb->tc_verd);
3680 goto ncls;
3681 }
3682#endif
3683
David S. Miller9754e292013-02-14 15:57:38 -05003684 if (pfmemalloc)
Mel Gormanb4b9e352012-07-31 16:44:26 -07003685 goto skip_taps;
3686
Linus Torvalds1da177e2005-04-16 15:20:36 -07003687 list_for_each_entry_rcu(ptype, &ptype_all, list) {
Salam Noureddine7866a622015-01-27 11:35:48 -08003688 if (pt_prev)
3689 ret = deliver_skb(skb, pt_prev, orig_dev);
3690 pt_prev = ptype;
3691 }
3692
3693 list_for_each_entry_rcu(ptype, &skb->dev->ptype_all, list) {
3694 if (pt_prev)
3695 ret = deliver_skb(skb, pt_prev, orig_dev);
3696 pt_prev = ptype;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003697 }
3698
Mel Gormanb4b9e352012-07-31 16:44:26 -07003699skip_taps:
Linus Torvalds1da177e2005-04-16 15:20:36 -07003700#ifdef CONFIG_NET_CLS_ACT
Herbert Xuf697c3e2007-10-14 00:38:47 -07003701 skb = handle_ing(skb, &pt_prev, &ret, orig_dev);
3702 if (!skb)
Mel Gormanb4b9e352012-07-31 16:44:26 -07003703 goto unlock;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003704ncls:
3705#endif
3706
David S. Miller9754e292013-02-14 15:57:38 -05003707 if (pfmemalloc && !skb_pfmemalloc_protocol(skb))
Mel Gormanb4b9e352012-07-31 16:44:26 -07003708 goto drop;
3709
Jiri Pirkodf8a39d2015-01-13 17:13:44 +01003710 if (skb_vlan_tag_present(skb)) {
John Fastabend24257172011-10-10 09:16:41 +00003711 if (pt_prev) {
3712 ret = deliver_skb(skb, pt_prev, orig_dev);
3713 pt_prev = NULL;
3714 }
Florian Zumbiehl48cc32d32012-10-07 15:51:58 +00003715 if (vlan_do_receive(&skb))
John Fastabend24257172011-10-10 09:16:41 +00003716 goto another_round;
3717 else if (unlikely(!skb))
Mel Gormanb4b9e352012-07-31 16:44:26 -07003718 goto unlock;
John Fastabend24257172011-10-10 09:16:41 +00003719 }
3720
Florian Zumbiehl48cc32d32012-10-07 15:51:58 +00003721 rx_handler = rcu_dereference(skb->dev->rx_handler);
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00003722 if (rx_handler) {
3723 if (pt_prev) {
3724 ret = deliver_skb(skb, pt_prev, orig_dev);
3725 pt_prev = NULL;
3726 }
Jiri Pirko8a4eb572011-03-12 03:14:39 +00003727 switch (rx_handler(&skb)) {
3728 case RX_HANDLER_CONSUMED:
Cristian Bercaru3bc1b1a2013-03-08 07:03:38 +00003729 ret = NET_RX_SUCCESS;
Mel Gormanb4b9e352012-07-31 16:44:26 -07003730 goto unlock;
Jiri Pirko8a4eb572011-03-12 03:14:39 +00003731 case RX_HANDLER_ANOTHER:
David S. Miller63d8ea72011-02-28 10:48:59 -08003732 goto another_round;
Jiri Pirko8a4eb572011-03-12 03:14:39 +00003733 case RX_HANDLER_EXACT:
3734 deliver_exact = true;
3735 case RX_HANDLER_PASS:
3736 break;
3737 default:
3738 BUG();
3739 }
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00003740 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07003741
Jiri Pirkodf8a39d2015-01-13 17:13:44 +01003742 if (unlikely(skb_vlan_tag_present(skb))) {
3743 if (skb_vlan_tag_get_id(skb))
Eric Dumazetd4b812d2013-07-18 07:19:26 -07003744 skb->pkt_type = PACKET_OTHERHOST;
3745 /* Note: we might in the future use prio bits
3746 * and set skb->priority like in vlan_do_receive()
3747 * For the time being, just ignore Priority Code Point
3748 */
3749 skb->vlan_tci = 0;
3750 }
Florian Zumbiehl48cc32d32012-10-07 15:51:58 +00003751
Linus Torvalds1da177e2005-04-16 15:20:36 -07003752 type = skb->protocol;
Salam Noureddine7866a622015-01-27 11:35:48 -08003753
3754 /* deliver only exact match when indicated */
3755 if (likely(!deliver_exact)) {
3756 deliver_ptype_list_skb(skb, &pt_prev, orig_dev, type,
3757 &ptype_base[ntohs(type) &
3758 PTYPE_HASH_MASK]);
3759 }
3760
3761 deliver_ptype_list_skb(skb, &pt_prev, orig_dev, type,
3762 &orig_dev->ptype_specific);
3763
3764 if (unlikely(skb->dev != orig_dev)) {
3765 deliver_ptype_list_skb(skb, &pt_prev, orig_dev, type,
3766 &skb->dev->ptype_specific);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003767 }
3768
3769 if (pt_prev) {
Michael S. Tsirkin1080e512012-07-20 09:23:17 +00003770 if (unlikely(skb_orphan_frags(skb, GFP_ATOMIC)))
Michael S. Tsirkin0e698bf2012-09-15 22:44:16 +00003771 goto drop;
Michael S. Tsirkin1080e512012-07-20 09:23:17 +00003772 else
3773 ret = pt_prev->func(skb, skb->dev, pt_prev, orig_dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003774 } else {
Mel Gormanb4b9e352012-07-31 16:44:26 -07003775drop:
Eric Dumazetcaf586e2010-09-30 21:06:55 +00003776 atomic_long_inc(&skb->dev->rx_dropped);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003777 kfree_skb(skb);
3778 /* Jamal, now you will not able to escape explaining
3779 * me how you were going to use this. :-)
3780 */
3781 ret = NET_RX_DROP;
3782 }
3783
Mel Gormanb4b9e352012-07-31 16:44:26 -07003784unlock:
Linus Torvalds1da177e2005-04-16 15:20:36 -07003785 rcu_read_unlock();
David S. Miller9754e292013-02-14 15:57:38 -05003786 return ret;
3787}
3788
3789static int __netif_receive_skb(struct sk_buff *skb)
3790{
3791 int ret;
3792
3793 if (sk_memalloc_socks() && skb_pfmemalloc(skb)) {
3794 unsigned long pflags = current->flags;
3795
3796 /*
3797 * PFMEMALLOC skbs are special, they should
3798 * - be delivered to SOCK_MEMALLOC sockets only
3799 * - stay away from userspace
3800 * - have bounded memory usage
3801 *
3802 * Use PF_MEMALLOC as this saves us from propagating the allocation
3803 * context down to all allocation sites.
3804 */
3805 current->flags |= PF_MEMALLOC;
3806 ret = __netif_receive_skb_core(skb, true);
3807 tsk_restore_flags(current, pflags, PF_MEMALLOC);
3808 } else
3809 ret = __netif_receive_skb_core(skb, false);
3810
Linus Torvalds1da177e2005-04-16 15:20:36 -07003811 return ret;
3812}
Tom Herbert0a9627f2010-03-16 08:03:29 +00003813
Ben Hutchingsae78dbf2014-01-10 22:17:24 +00003814static int netif_receive_skb_internal(struct sk_buff *skb)
Tom Herbert0a9627f2010-03-16 08:03:29 +00003815{
Eric Dumazet588f0332011-11-15 04:12:55 +00003816 net_timestamp_check(netdev_tstamp_prequeue, skb);
Eric Dumazet3b098e22010-05-15 23:57:10 -07003817
Richard Cochranc1f19b52010-07-17 08:49:36 +00003818 if (skb_defer_rx_timestamp(skb))
3819 return NET_RX_SUCCESS;
3820
Eric Dumazetdf334542010-03-24 19:13:54 +00003821#ifdef CONFIG_RPS
Ingo Molnarc5905af2012-02-24 08:31:31 +01003822 if (static_key_false(&rps_needed)) {
Eric Dumazet3b098e22010-05-15 23:57:10 -07003823 struct rps_dev_flow voidflow, *rflow = &voidflow;
3824 int cpu, ret;
Tom Herbert0a9627f2010-03-16 08:03:29 +00003825
Eric Dumazet3b098e22010-05-15 23:57:10 -07003826 rcu_read_lock();
Tom Herbert0a9627f2010-03-16 08:03:29 +00003827
Eric Dumazet3b098e22010-05-15 23:57:10 -07003828 cpu = get_rps_cpu(skb->dev, skb, &rflow);
Tom Herbertfec5e652010-04-16 16:01:27 -07003829
Eric Dumazet3b098e22010-05-15 23:57:10 -07003830 if (cpu >= 0) {
3831 ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail);
3832 rcu_read_unlock();
Eric Dumazetadc93002011-11-17 03:13:26 +00003833 return ret;
Eric Dumazet3b098e22010-05-15 23:57:10 -07003834 }
Eric Dumazetadc93002011-11-17 03:13:26 +00003835 rcu_read_unlock();
Tom Herbertfec5e652010-04-16 16:01:27 -07003836 }
Tom Herbert1e94d722010-03-18 17:45:44 -07003837#endif
Eric Dumazetadc93002011-11-17 03:13:26 +00003838 return __netif_receive_skb(skb);
Tom Herbert0a9627f2010-03-16 08:03:29 +00003839}
Ben Hutchingsae78dbf2014-01-10 22:17:24 +00003840
3841/**
3842 * netif_receive_skb - process receive buffer from network
3843 * @skb: buffer to process
3844 *
3845 * netif_receive_skb() is the main receive data processing function.
3846 * It always succeeds. The buffer may be dropped during processing
3847 * for congestion control or by the protocol layers.
3848 *
3849 * This function may only be called from softirq context and interrupts
3850 * should be enabled.
3851 *
3852 * Return values (usually ignored):
3853 * NET_RX_SUCCESS: no congestion
3854 * NET_RX_DROP: packet was dropped
3855 */
David Miller7026b1d2015-04-05 22:19:04 -04003856int netif_receive_skb_sk(struct sock *sk, struct sk_buff *skb)
Ben Hutchingsae78dbf2014-01-10 22:17:24 +00003857{
3858 trace_netif_receive_skb_entry(skb);
3859
3860 return netif_receive_skb_internal(skb);
3861}
David Miller7026b1d2015-04-05 22:19:04 -04003862EXPORT_SYMBOL(netif_receive_skb_sk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003863
Eric Dumazet88751272010-04-19 05:07:33 +00003864/* Network device is going away, flush any packets still pending
3865 * Called with irqs disabled.
3866 */
Changli Gao152102c2010-03-30 20:16:22 +00003867static void flush_backlog(void *arg)
Stephen Hemminger6e583ce2008-08-03 21:29:57 -07003868{
Changli Gao152102c2010-03-30 20:16:22 +00003869 struct net_device *dev = arg;
Christoph Lameter903ceff2014-08-17 12:30:35 -05003870 struct softnet_data *sd = this_cpu_ptr(&softnet_data);
Stephen Hemminger6e583ce2008-08-03 21:29:57 -07003871 struct sk_buff *skb, *tmp;
3872
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003873 rps_lock(sd);
Changli Gao6e7676c2010-04-27 15:07:33 -07003874 skb_queue_walk_safe(&sd->input_pkt_queue, skb, tmp) {
Stephen Hemminger6e583ce2008-08-03 21:29:57 -07003875 if (skb->dev == dev) {
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003876 __skb_unlink(skb, &sd->input_pkt_queue);
Stephen Hemminger6e583ce2008-08-03 21:29:57 -07003877 kfree_skb(skb);
Tom Herbert76cc8b12010-05-20 18:37:59 +00003878 input_queue_head_incr(sd);
Stephen Hemminger6e583ce2008-08-03 21:29:57 -07003879 }
Changli Gao6e7676c2010-04-27 15:07:33 -07003880 }
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003881 rps_unlock(sd);
Changli Gao6e7676c2010-04-27 15:07:33 -07003882
3883 skb_queue_walk_safe(&sd->process_queue, skb, tmp) {
3884 if (skb->dev == dev) {
3885 __skb_unlink(skb, &sd->process_queue);
3886 kfree_skb(skb);
Tom Herbert76cc8b12010-05-20 18:37:59 +00003887 input_queue_head_incr(sd);
Changli Gao6e7676c2010-04-27 15:07:33 -07003888 }
3889 }
Stephen Hemminger6e583ce2008-08-03 21:29:57 -07003890}
3891
Herbert Xud565b0a2008-12-15 23:38:52 -08003892static int napi_gro_complete(struct sk_buff *skb)
3893{
Vlad Yasevich22061d82012-11-15 08:49:11 +00003894 struct packet_offload *ptype;
Herbert Xud565b0a2008-12-15 23:38:52 -08003895 __be16 type = skb->protocol;
Vlad Yasevich22061d82012-11-15 08:49:11 +00003896 struct list_head *head = &offload_base;
Herbert Xud565b0a2008-12-15 23:38:52 -08003897 int err = -ENOENT;
3898
Eric Dumazetc3c7c252012-12-06 13:54:59 +00003899 BUILD_BUG_ON(sizeof(struct napi_gro_cb) > sizeof(skb->cb));
3900
Herbert Xufc59f9a2009-04-14 15:11:06 -07003901 if (NAPI_GRO_CB(skb)->count == 1) {
3902 skb_shinfo(skb)->gso_size = 0;
Herbert Xud565b0a2008-12-15 23:38:52 -08003903 goto out;
Herbert Xufc59f9a2009-04-14 15:11:06 -07003904 }
Herbert Xud565b0a2008-12-15 23:38:52 -08003905
3906 rcu_read_lock();
3907 list_for_each_entry_rcu(ptype, head, list) {
Vlad Yasevichf191a1d2012-11-15 08:49:23 +00003908 if (ptype->type != type || !ptype->callbacks.gro_complete)
Herbert Xud565b0a2008-12-15 23:38:52 -08003909 continue;
3910
Jerry Chu299603e82013-12-11 20:53:45 -08003911 err = ptype->callbacks.gro_complete(skb, 0);
Herbert Xud565b0a2008-12-15 23:38:52 -08003912 break;
3913 }
3914 rcu_read_unlock();
3915
3916 if (err) {
3917 WARN_ON(&ptype->list == head);
3918 kfree_skb(skb);
3919 return NET_RX_SUCCESS;
3920 }
3921
3922out:
Ben Hutchingsae78dbf2014-01-10 22:17:24 +00003923 return netif_receive_skb_internal(skb);
Herbert Xud565b0a2008-12-15 23:38:52 -08003924}
3925
Eric Dumazet2e71a6f2012-10-06 08:08:49 +00003926/* napi->gro_list contains packets ordered by age.
3927 * youngest packets at the head of it.
3928 * Complete skbs in reverse order to reduce latencies.
3929 */
3930void napi_gro_flush(struct napi_struct *napi, bool flush_old)
Herbert Xud565b0a2008-12-15 23:38:52 -08003931{
Eric Dumazet2e71a6f2012-10-06 08:08:49 +00003932 struct sk_buff *skb, *prev = NULL;
Herbert Xud565b0a2008-12-15 23:38:52 -08003933
Eric Dumazet2e71a6f2012-10-06 08:08:49 +00003934 /* scan list and build reverse chain */
3935 for (skb = napi->gro_list; skb != NULL; skb = skb->next) {
3936 skb->prev = prev;
3937 prev = skb;
Herbert Xud565b0a2008-12-15 23:38:52 -08003938 }
3939
Eric Dumazet2e71a6f2012-10-06 08:08:49 +00003940 for (skb = prev; skb; skb = prev) {
3941 skb->next = NULL;
3942
3943 if (flush_old && NAPI_GRO_CB(skb)->age == jiffies)
3944 return;
3945
3946 prev = skb->prev;
3947 napi_gro_complete(skb);
3948 napi->gro_count--;
3949 }
3950
Herbert Xud565b0a2008-12-15 23:38:52 -08003951 napi->gro_list = NULL;
3952}
Eric Dumazet86cac582010-08-31 18:25:32 +00003953EXPORT_SYMBOL(napi_gro_flush);
Herbert Xud565b0a2008-12-15 23:38:52 -08003954
Eric Dumazet89c5fa32012-12-10 13:28:16 +00003955static void gro_list_prepare(struct napi_struct *napi, struct sk_buff *skb)
3956{
3957 struct sk_buff *p;
3958 unsigned int maclen = skb->dev->hard_header_len;
Tom Herbert0b4cec82014-01-15 08:58:06 -08003959 u32 hash = skb_get_hash_raw(skb);
Eric Dumazet89c5fa32012-12-10 13:28:16 +00003960
3961 for (p = napi->gro_list; p; p = p->next) {
3962 unsigned long diffs;
3963
Tom Herbert0b4cec82014-01-15 08:58:06 -08003964 NAPI_GRO_CB(p)->flush = 0;
3965
3966 if (hash != skb_get_hash_raw(p)) {
3967 NAPI_GRO_CB(p)->same_flow = 0;
3968 continue;
3969 }
3970
Eric Dumazet89c5fa32012-12-10 13:28:16 +00003971 diffs = (unsigned long)p->dev ^ (unsigned long)skb->dev;
3972 diffs |= p->vlan_tci ^ skb->vlan_tci;
3973 if (maclen == ETH_HLEN)
3974 diffs |= compare_ether_header(skb_mac_header(p),
Eric Dumazeta50e2332014-03-29 21:28:21 -07003975 skb_mac_header(skb));
Eric Dumazet89c5fa32012-12-10 13:28:16 +00003976 else if (!diffs)
3977 diffs = memcmp(skb_mac_header(p),
Eric Dumazeta50e2332014-03-29 21:28:21 -07003978 skb_mac_header(skb),
Eric Dumazet89c5fa32012-12-10 13:28:16 +00003979 maclen);
3980 NAPI_GRO_CB(p)->same_flow = !diffs;
Eric Dumazet89c5fa32012-12-10 13:28:16 +00003981 }
3982}
3983
Jerry Chu299603e82013-12-11 20:53:45 -08003984static void skb_gro_reset_offset(struct sk_buff *skb)
3985{
3986 const struct skb_shared_info *pinfo = skb_shinfo(skb);
3987 const skb_frag_t *frag0 = &pinfo->frags[0];
3988
3989 NAPI_GRO_CB(skb)->data_offset = 0;
3990 NAPI_GRO_CB(skb)->frag0 = NULL;
3991 NAPI_GRO_CB(skb)->frag0_len = 0;
3992
3993 if (skb_mac_header(skb) == skb_tail_pointer(skb) &&
3994 pinfo->nr_frags &&
3995 !PageHighMem(skb_frag_page(frag0))) {
3996 NAPI_GRO_CB(skb)->frag0 = skb_frag_address(frag0);
3997 NAPI_GRO_CB(skb)->frag0_len = skb_frag_size(frag0);
Herbert Xud565b0a2008-12-15 23:38:52 -08003998 }
3999}
4000
Eric Dumazeta50e2332014-03-29 21:28:21 -07004001static void gro_pull_from_frag0(struct sk_buff *skb, int grow)
4002{
4003 struct skb_shared_info *pinfo = skb_shinfo(skb);
4004
4005 BUG_ON(skb->end - skb->tail < grow);
4006
4007 memcpy(skb_tail_pointer(skb), NAPI_GRO_CB(skb)->frag0, grow);
4008
4009 skb->data_len -= grow;
4010 skb->tail += grow;
4011
4012 pinfo->frags[0].page_offset += grow;
4013 skb_frag_size_sub(&pinfo->frags[0], grow);
4014
4015 if (unlikely(!skb_frag_size(&pinfo->frags[0]))) {
4016 skb_frag_unref(skb, 0);
4017 memmove(pinfo->frags, pinfo->frags + 1,
4018 --pinfo->nr_frags * sizeof(pinfo->frags[0]));
4019 }
4020}
4021
Rami Rosenbb728822012-11-28 21:55:25 +00004022static enum gro_result dev_gro_receive(struct napi_struct *napi, struct sk_buff *skb)
Herbert Xud565b0a2008-12-15 23:38:52 -08004023{
4024 struct sk_buff **pp = NULL;
Vlad Yasevich22061d82012-11-15 08:49:11 +00004025 struct packet_offload *ptype;
Herbert Xud565b0a2008-12-15 23:38:52 -08004026 __be16 type = skb->protocol;
Vlad Yasevich22061d82012-11-15 08:49:11 +00004027 struct list_head *head = &offload_base;
Herbert Xu0da2afd52008-12-26 14:57:42 -08004028 int same_flow;
Ben Hutchings5b252f02009-10-29 07:17:09 +00004029 enum gro_result ret;
Eric Dumazeta50e2332014-03-29 21:28:21 -07004030 int grow;
Herbert Xud565b0a2008-12-15 23:38:52 -08004031
Eric W. Biederman9c62a682014-03-14 20:51:52 -07004032 if (!(skb->dev->features & NETIF_F_GRO))
Herbert Xud565b0a2008-12-15 23:38:52 -08004033 goto normal;
4034
Tom Herbert5a212322014-08-31 15:12:41 -07004035 if (skb_is_gso(skb) || skb_has_frag_list(skb) || skb->csum_bad)
Herbert Xuf17f5c92009-01-14 14:36:12 -08004036 goto normal;
4037
Eric Dumazet89c5fa32012-12-10 13:28:16 +00004038 gro_list_prepare(napi, skb);
4039
Herbert Xud565b0a2008-12-15 23:38:52 -08004040 rcu_read_lock();
4041 list_for_each_entry_rcu(ptype, head, list) {
Vlad Yasevichf191a1d2012-11-15 08:49:23 +00004042 if (ptype->type != type || !ptype->callbacks.gro_receive)
Herbert Xud565b0a2008-12-15 23:38:52 -08004043 continue;
4044
Herbert Xu86911732009-01-29 14:19:50 +00004045 skb_set_network_header(skb, skb_gro_offset(skb));
Eric Dumazetefd94502013-02-14 17:31:48 +00004046 skb_reset_mac_len(skb);
Herbert Xud565b0a2008-12-15 23:38:52 -08004047 NAPI_GRO_CB(skb)->same_flow = 0;
4048 NAPI_GRO_CB(skb)->flush = 0;
Herbert Xu5d38a072009-01-04 16:13:40 -08004049 NAPI_GRO_CB(skb)->free = 0;
Or Gerlitzb582ef02014-01-20 13:59:19 +02004050 NAPI_GRO_CB(skb)->udp_mark = 0;
Tom Herbert15e23962015-02-10 16:30:31 -08004051 NAPI_GRO_CB(skb)->gro_remcsum_start = 0;
Herbert Xud565b0a2008-12-15 23:38:52 -08004052
Tom Herbert662880f2014-08-27 21:26:56 -07004053 /* Setup for GRO checksum validation */
4054 switch (skb->ip_summed) {
4055 case CHECKSUM_COMPLETE:
4056 NAPI_GRO_CB(skb)->csum = skb->csum;
4057 NAPI_GRO_CB(skb)->csum_valid = 1;
4058 NAPI_GRO_CB(skb)->csum_cnt = 0;
4059 break;
4060 case CHECKSUM_UNNECESSARY:
4061 NAPI_GRO_CB(skb)->csum_cnt = skb->csum_level + 1;
4062 NAPI_GRO_CB(skb)->csum_valid = 0;
4063 break;
4064 default:
4065 NAPI_GRO_CB(skb)->csum_cnt = 0;
4066 NAPI_GRO_CB(skb)->csum_valid = 0;
4067 }
Herbert Xud565b0a2008-12-15 23:38:52 -08004068
Vlad Yasevichf191a1d2012-11-15 08:49:23 +00004069 pp = ptype->callbacks.gro_receive(&napi->gro_list, skb);
Herbert Xud565b0a2008-12-15 23:38:52 -08004070 break;
4071 }
4072 rcu_read_unlock();
4073
4074 if (&ptype->list == head)
4075 goto normal;
4076
Herbert Xu0da2afd52008-12-26 14:57:42 -08004077 same_flow = NAPI_GRO_CB(skb)->same_flow;
Herbert Xu5d0d9be2009-01-29 14:19:48 +00004078 ret = NAPI_GRO_CB(skb)->free ? GRO_MERGED_FREE : GRO_MERGED;
Herbert Xu0da2afd52008-12-26 14:57:42 -08004079
Herbert Xud565b0a2008-12-15 23:38:52 -08004080 if (pp) {
4081 struct sk_buff *nskb = *pp;
4082
4083 *pp = nskb->next;
4084 nskb->next = NULL;
4085 napi_gro_complete(nskb);
Herbert Xu4ae55442009-02-08 18:00:36 +00004086 napi->gro_count--;
Herbert Xud565b0a2008-12-15 23:38:52 -08004087 }
4088
Herbert Xu0da2afd52008-12-26 14:57:42 -08004089 if (same_flow)
Herbert Xud565b0a2008-12-15 23:38:52 -08004090 goto ok;
4091
Eric Dumazet600adc12014-01-09 14:12:19 -08004092 if (NAPI_GRO_CB(skb)->flush)
Herbert Xud565b0a2008-12-15 23:38:52 -08004093 goto normal;
Herbert Xud565b0a2008-12-15 23:38:52 -08004094
Eric Dumazet600adc12014-01-09 14:12:19 -08004095 if (unlikely(napi->gro_count >= MAX_GRO_SKBS)) {
4096 struct sk_buff *nskb = napi->gro_list;
4097
4098 /* locate the end of the list to select the 'oldest' flow */
4099 while (nskb->next) {
4100 pp = &nskb->next;
4101 nskb = *pp;
4102 }
4103 *pp = NULL;
4104 nskb->next = NULL;
4105 napi_gro_complete(nskb);
4106 } else {
4107 napi->gro_count++;
4108 }
Herbert Xud565b0a2008-12-15 23:38:52 -08004109 NAPI_GRO_CB(skb)->count = 1;
Eric Dumazet2e71a6f2012-10-06 08:08:49 +00004110 NAPI_GRO_CB(skb)->age = jiffies;
Eric Dumazet29e98242014-05-16 11:34:37 -07004111 NAPI_GRO_CB(skb)->last = skb;
Herbert Xu86911732009-01-29 14:19:50 +00004112 skb_shinfo(skb)->gso_size = skb_gro_len(skb);
Herbert Xud565b0a2008-12-15 23:38:52 -08004113 skb->next = napi->gro_list;
4114 napi->gro_list = skb;
Herbert Xu5d0d9be2009-01-29 14:19:48 +00004115 ret = GRO_HELD;
Herbert Xud565b0a2008-12-15 23:38:52 -08004116
Herbert Xuad0f9902009-02-01 01:24:55 -08004117pull:
Eric Dumazeta50e2332014-03-29 21:28:21 -07004118 grow = skb_gro_offset(skb) - skb_headlen(skb);
4119 if (grow > 0)
4120 gro_pull_from_frag0(skb, grow);
Herbert Xud565b0a2008-12-15 23:38:52 -08004121ok:
Herbert Xu5d0d9be2009-01-29 14:19:48 +00004122 return ret;
Herbert Xud565b0a2008-12-15 23:38:52 -08004123
4124normal:
Herbert Xuad0f9902009-02-01 01:24:55 -08004125 ret = GRO_NORMAL;
4126 goto pull;
Herbert Xu5d38a072009-01-04 16:13:40 -08004127}
Herbert Xu96e93ea2009-01-06 10:49:34 -08004128
Jerry Chubf5a7552014-01-07 10:23:19 -08004129struct packet_offload *gro_find_receive_by_type(__be16 type)
4130{
4131 struct list_head *offload_head = &offload_base;
4132 struct packet_offload *ptype;
4133
4134 list_for_each_entry_rcu(ptype, offload_head, list) {
4135 if (ptype->type != type || !ptype->callbacks.gro_receive)
4136 continue;
4137 return ptype;
4138 }
4139 return NULL;
4140}
Or Gerlitze27a2f82014-01-20 13:59:20 +02004141EXPORT_SYMBOL(gro_find_receive_by_type);
Jerry Chubf5a7552014-01-07 10:23:19 -08004142
4143struct packet_offload *gro_find_complete_by_type(__be16 type)
4144{
4145 struct list_head *offload_head = &offload_base;
4146 struct packet_offload *ptype;
4147
4148 list_for_each_entry_rcu(ptype, offload_head, list) {
4149 if (ptype->type != type || !ptype->callbacks.gro_complete)
4150 continue;
4151 return ptype;
4152 }
4153 return NULL;
4154}
Or Gerlitze27a2f82014-01-20 13:59:20 +02004155EXPORT_SYMBOL(gro_find_complete_by_type);
Herbert Xu96e93ea2009-01-06 10:49:34 -08004156
Rami Rosenbb728822012-11-28 21:55:25 +00004157static gro_result_t napi_skb_finish(gro_result_t ret, struct sk_buff *skb)
Herbert Xu5d38a072009-01-04 16:13:40 -08004158{
Herbert Xu5d0d9be2009-01-29 14:19:48 +00004159 switch (ret) {
4160 case GRO_NORMAL:
Ben Hutchingsae78dbf2014-01-10 22:17:24 +00004161 if (netif_receive_skb_internal(skb))
Ben Hutchingsc7c4b3b2009-10-29 21:36:53 -07004162 ret = GRO_DROP;
4163 break;
Herbert Xu5d38a072009-01-04 16:13:40 -08004164
Herbert Xu5d0d9be2009-01-29 14:19:48 +00004165 case GRO_DROP:
Herbert Xu5d38a072009-01-04 16:13:40 -08004166 kfree_skb(skb);
4167 break;
Ben Hutchings5b252f02009-10-29 07:17:09 +00004168
Eric Dumazetdaa86542012-04-19 07:07:40 +00004169 case GRO_MERGED_FREE:
Eric Dumazetd7e88832012-04-30 08:10:34 +00004170 if (NAPI_GRO_CB(skb)->free == NAPI_GRO_FREE_STOLEN_HEAD)
4171 kmem_cache_free(skbuff_head_cache, skb);
4172 else
4173 __kfree_skb(skb);
Eric Dumazetdaa86542012-04-19 07:07:40 +00004174 break;
4175
Ben Hutchings5b252f02009-10-29 07:17:09 +00004176 case GRO_HELD:
4177 case GRO_MERGED:
4178 break;
Herbert Xu5d38a072009-01-04 16:13:40 -08004179 }
4180
Ben Hutchingsc7c4b3b2009-10-29 21:36:53 -07004181 return ret;
Herbert Xu5d0d9be2009-01-29 14:19:48 +00004182}
Herbert Xu5d0d9be2009-01-29 14:19:48 +00004183
Ben Hutchingsc7c4b3b2009-10-29 21:36:53 -07004184gro_result_t napi_gro_receive(struct napi_struct *napi, struct sk_buff *skb)
Herbert Xu5d0d9be2009-01-29 14:19:48 +00004185{
Ben Hutchingsae78dbf2014-01-10 22:17:24 +00004186 trace_napi_gro_receive_entry(skb);
Herbert Xu86911732009-01-29 14:19:50 +00004187
Eric Dumazeta50e2332014-03-29 21:28:21 -07004188 skb_gro_reset_offset(skb);
4189
Eric Dumazet89c5fa32012-12-10 13:28:16 +00004190 return napi_skb_finish(dev_gro_receive(napi, skb), skb);
Herbert Xud565b0a2008-12-15 23:38:52 -08004191}
4192EXPORT_SYMBOL(napi_gro_receive);
4193
stephen hemmingerd0c2b0d2010-10-19 07:12:10 +00004194static void napi_reuse_skb(struct napi_struct *napi, struct sk_buff *skb)
Herbert Xu96e93ea2009-01-06 10:49:34 -08004195{
Eric Dumazet93a35f52014-10-23 06:30:30 -07004196 if (unlikely(skb->pfmemalloc)) {
4197 consume_skb(skb);
4198 return;
4199 }
Herbert Xu96e93ea2009-01-06 10:49:34 -08004200 __skb_pull(skb, skb_headlen(skb));
Eric Dumazet2a2a4592012-03-21 06:58:03 +00004201 /* restore the reserve we had after netdev_alloc_skb_ip_align() */
4202 skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN - skb_headroom(skb));
Jesse Gross3701e512010-10-20 13:56:06 +00004203 skb->vlan_tci = 0;
Herbert Xu66c46d72011-01-29 20:44:54 -08004204 skb->dev = napi->dev;
Andy Gospodarek6d152e22011-02-02 14:53:25 -08004205 skb->skb_iif = 0;
Jerry Chuc3caf112014-07-14 15:54:46 -07004206 skb->encapsulation = 0;
4207 skb_shinfo(skb)->gso_type = 0;
Eric Dumazete33d0ba2014-04-03 09:28:10 -07004208 skb->truesize = SKB_TRUESIZE(skb_end_offset(skb));
Herbert Xu96e93ea2009-01-06 10:49:34 -08004209
4210 napi->skb = skb;
4211}
Herbert Xu96e93ea2009-01-06 10:49:34 -08004212
Herbert Xu76620aa2009-04-16 02:02:07 -07004213struct sk_buff *napi_get_frags(struct napi_struct *napi)
Herbert Xu5d38a072009-01-04 16:13:40 -08004214{
Herbert Xu5d38a072009-01-04 16:13:40 -08004215 struct sk_buff *skb = napi->skb;
Herbert Xu5d38a072009-01-04 16:13:40 -08004216
4217 if (!skb) {
Alexander Duyckfd11a832014-12-09 19:40:49 -08004218 skb = napi_alloc_skb(napi, GRO_MAX_HEAD);
Eric Dumazet84b9cd62013-12-05 21:44:27 -08004219 napi->skb = skb;
Herbert Xu5d38a072009-01-04 16:13:40 -08004220 }
Herbert Xu96e93ea2009-01-06 10:49:34 -08004221 return skb;
4222}
Herbert Xu76620aa2009-04-16 02:02:07 -07004223EXPORT_SYMBOL(napi_get_frags);
Herbert Xu96e93ea2009-01-06 10:49:34 -08004224
Eric Dumazeta50e2332014-03-29 21:28:21 -07004225static gro_result_t napi_frags_finish(struct napi_struct *napi,
4226 struct sk_buff *skb,
4227 gro_result_t ret)
Herbert Xu5d0d9be2009-01-29 14:19:48 +00004228{
Herbert Xu5d0d9be2009-01-29 14:19:48 +00004229 switch (ret) {
4230 case GRO_NORMAL:
Eric Dumazeta50e2332014-03-29 21:28:21 -07004231 case GRO_HELD:
4232 __skb_push(skb, ETH_HLEN);
4233 skb->protocol = eth_type_trans(skb, skb->dev);
4234 if (ret == GRO_NORMAL && netif_receive_skb_internal(skb))
Ben Hutchingsc7c4b3b2009-10-29 21:36:53 -07004235 ret = GRO_DROP;
Herbert Xu86911732009-01-29 14:19:50 +00004236 break;
Herbert Xu5d0d9be2009-01-29 14:19:48 +00004237
4238 case GRO_DROP:
Herbert Xu5d0d9be2009-01-29 14:19:48 +00004239 case GRO_MERGED_FREE:
4240 napi_reuse_skb(napi, skb);
4241 break;
Ben Hutchings5b252f02009-10-29 07:17:09 +00004242
4243 case GRO_MERGED:
4244 break;
Herbert Xu5d0d9be2009-01-29 14:19:48 +00004245 }
4246
Ben Hutchingsc7c4b3b2009-10-29 21:36:53 -07004247 return ret;
Herbert Xu5d0d9be2009-01-29 14:19:48 +00004248}
Herbert Xu5d0d9be2009-01-29 14:19:48 +00004249
Eric Dumazeta50e2332014-03-29 21:28:21 -07004250/* Upper GRO stack assumes network header starts at gro_offset=0
4251 * Drivers could call both napi_gro_frags() and napi_gro_receive()
4252 * We copy ethernet header into skb->data to have a common layout.
4253 */
Eric Dumazet4adb9c42012-05-18 20:49:06 +00004254static struct sk_buff *napi_frags_skb(struct napi_struct *napi)
Herbert Xu96e93ea2009-01-06 10:49:34 -08004255{
Herbert Xu76620aa2009-04-16 02:02:07 -07004256 struct sk_buff *skb = napi->skb;
Eric Dumazeta50e2332014-03-29 21:28:21 -07004257 const struct ethhdr *eth;
4258 unsigned int hlen = sizeof(*eth);
Herbert Xu76620aa2009-04-16 02:02:07 -07004259
4260 napi->skb = NULL;
4261
Eric Dumazeta50e2332014-03-29 21:28:21 -07004262 skb_reset_mac_header(skb);
4263 skb_gro_reset_offset(skb);
4264
4265 eth = skb_gro_header_fast(skb, 0);
4266 if (unlikely(skb_gro_header_hard(skb, hlen))) {
4267 eth = skb_gro_header_slow(skb, hlen, 0);
4268 if (unlikely(!eth)) {
4269 napi_reuse_skb(napi, skb);
4270 return NULL;
4271 }
4272 } else {
4273 gro_pull_from_frag0(skb, hlen);
4274 NAPI_GRO_CB(skb)->frag0 += hlen;
4275 NAPI_GRO_CB(skb)->frag0_len -= hlen;
Herbert Xu76620aa2009-04-16 02:02:07 -07004276 }
Eric Dumazeta50e2332014-03-29 21:28:21 -07004277 __skb_pull(skb, hlen);
4278
4279 /*
4280 * This works because the only protocols we care about don't require
4281 * special handling.
4282 * We'll fix it up properly in napi_frags_finish()
4283 */
4284 skb->protocol = eth->h_proto;
Herbert Xu76620aa2009-04-16 02:02:07 -07004285
Herbert Xu76620aa2009-04-16 02:02:07 -07004286 return skb;
4287}
Herbert Xu76620aa2009-04-16 02:02:07 -07004288
Ben Hutchingsc7c4b3b2009-10-29 21:36:53 -07004289gro_result_t napi_gro_frags(struct napi_struct *napi)
Herbert Xu76620aa2009-04-16 02:02:07 -07004290{
4291 struct sk_buff *skb = napi_frags_skb(napi);
Herbert Xu96e93ea2009-01-06 10:49:34 -08004292
4293 if (!skb)
Ben Hutchingsc7c4b3b2009-10-29 21:36:53 -07004294 return GRO_DROP;
Herbert Xu96e93ea2009-01-06 10:49:34 -08004295
Ben Hutchingsae78dbf2014-01-10 22:17:24 +00004296 trace_napi_gro_frags_entry(skb);
4297
Eric Dumazet89c5fa32012-12-10 13:28:16 +00004298 return napi_frags_finish(napi, skb, dev_gro_receive(napi, skb));
Herbert Xu5d38a072009-01-04 16:13:40 -08004299}
4300EXPORT_SYMBOL(napi_gro_frags);
4301
Tom Herbert573e8fc2014-08-22 13:33:47 -07004302/* Compute the checksum from gro_offset and return the folded value
4303 * after adding in any pseudo checksum.
4304 */
4305__sum16 __skb_gro_checksum_complete(struct sk_buff *skb)
4306{
4307 __wsum wsum;
4308 __sum16 sum;
4309
4310 wsum = skb_checksum(skb, skb_gro_offset(skb), skb_gro_len(skb), 0);
4311
4312 /* NAPI_GRO_CB(skb)->csum holds pseudo checksum */
4313 sum = csum_fold(csum_add(NAPI_GRO_CB(skb)->csum, wsum));
4314 if (likely(!sum)) {
4315 if (unlikely(skb->ip_summed == CHECKSUM_COMPLETE) &&
4316 !skb->csum_complete_sw)
4317 netdev_rx_csum_fault(skb->dev);
4318 }
4319
4320 NAPI_GRO_CB(skb)->csum = wsum;
4321 NAPI_GRO_CB(skb)->csum_valid = 1;
4322
4323 return sum;
4324}
4325EXPORT_SYMBOL(__skb_gro_checksum_complete);
4326
Eric Dumazete326bed2010-04-22 00:22:45 -07004327/*
Zhi Yong Wu855abcf2014-01-01 04:34:50 +08004328 * net_rps_action_and_irq_enable sends any pending IPI's for rps.
Eric Dumazete326bed2010-04-22 00:22:45 -07004329 * Note: called with local irq disabled, but exits with local irq enabled.
4330 */
4331static void net_rps_action_and_irq_enable(struct softnet_data *sd)
4332{
4333#ifdef CONFIG_RPS
4334 struct softnet_data *remsd = sd->rps_ipi_list;
4335
4336 if (remsd) {
4337 sd->rps_ipi_list = NULL;
4338
4339 local_irq_enable();
4340
4341 /* Send pending IPI's to kick RPS processing on remote cpus. */
4342 while (remsd) {
4343 struct softnet_data *next = remsd->rps_ipi_next;
4344
4345 if (cpu_online(remsd->cpu))
Frederic Weisbeckerc46fff22014-02-24 16:40:02 +01004346 smp_call_function_single_async(remsd->cpu,
Frederic Weisbeckerfce8ad12014-02-24 16:40:01 +01004347 &remsd->csd);
Eric Dumazete326bed2010-04-22 00:22:45 -07004348 remsd = next;
4349 }
4350 } else
4351#endif
4352 local_irq_enable();
4353}
4354
Eric Dumazetd75b1ad2014-11-02 06:19:33 -08004355static bool sd_has_rps_ipi_waiting(struct softnet_data *sd)
4356{
4357#ifdef CONFIG_RPS
4358 return sd->rps_ipi_list != NULL;
4359#else
4360 return false;
4361#endif
4362}
4363
Stephen Hemmingerbea33482007-10-03 16:41:36 -07004364static int process_backlog(struct napi_struct *napi, int quota)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004365{
4366 int work = 0;
Eric Dumazeteecfd7c2010-05-06 22:07:48 -07004367 struct softnet_data *sd = container_of(napi, struct softnet_data, backlog);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004368
Eric Dumazete326bed2010-04-22 00:22:45 -07004369 /* Check if we have pending ipi, its better to send them now,
4370 * not waiting net_rx_action() end.
4371 */
Eric Dumazetd75b1ad2014-11-02 06:19:33 -08004372 if (sd_has_rps_ipi_waiting(sd)) {
Eric Dumazete326bed2010-04-22 00:22:45 -07004373 local_irq_disable();
4374 net_rps_action_and_irq_enable(sd);
4375 }
Eric Dumazetd75b1ad2014-11-02 06:19:33 -08004376
Stephen Hemmingerbea33482007-10-03 16:41:36 -07004377 napi->weight = weight_p;
Changli Gao6e7676c2010-04-27 15:07:33 -07004378 local_irq_disable();
Tom Herbert11ef7a82014-06-30 09:50:40 -07004379 while (1) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07004380 struct sk_buff *skb;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004381
Changli Gao6e7676c2010-04-27 15:07:33 -07004382 while ((skb = __skb_dequeue(&sd->process_queue))) {
Eric Dumazete4008272010-04-05 15:42:39 -07004383 local_irq_enable();
Changli Gao6e7676c2010-04-27 15:07:33 -07004384 __netif_receive_skb(skb);
Changli Gao6e7676c2010-04-27 15:07:33 -07004385 local_irq_disable();
Tom Herbert76cc8b12010-05-20 18:37:59 +00004386 input_queue_head_incr(sd);
4387 if (++work >= quota) {
4388 local_irq_enable();
4389 return work;
4390 }
Stephen Hemmingerbea33482007-10-03 16:41:36 -07004391 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07004392
Changli Gao6e7676c2010-04-27 15:07:33 -07004393 rps_lock(sd);
Tom Herbert11ef7a82014-06-30 09:50:40 -07004394 if (skb_queue_empty(&sd->input_pkt_queue)) {
Eric Dumazeteecfd7c2010-05-06 22:07:48 -07004395 /*
4396 * Inline a custom version of __napi_complete().
4397 * only current cpu owns and manipulates this napi,
Tom Herbert11ef7a82014-06-30 09:50:40 -07004398 * and NAPI_STATE_SCHED is the only possible flag set
4399 * on backlog.
4400 * We can use a plain write instead of clear_bit(),
Eric Dumazeteecfd7c2010-05-06 22:07:48 -07004401 * and we dont need an smp_mb() memory barrier.
4402 */
Eric Dumazeteecfd7c2010-05-06 22:07:48 -07004403 napi->state = 0;
Tom Herbert11ef7a82014-06-30 09:50:40 -07004404 rps_unlock(sd);
Eric Dumazeteecfd7c2010-05-06 22:07:48 -07004405
Tom Herbert11ef7a82014-06-30 09:50:40 -07004406 break;
Changli Gao6e7676c2010-04-27 15:07:33 -07004407 }
Tom Herbert11ef7a82014-06-30 09:50:40 -07004408
4409 skb_queue_splice_tail_init(&sd->input_pkt_queue,
4410 &sd->process_queue);
Changli Gao6e7676c2010-04-27 15:07:33 -07004411 rps_unlock(sd);
4412 }
4413 local_irq_enable();
Linus Torvalds1da177e2005-04-16 15:20:36 -07004414
Stephen Hemmingerbea33482007-10-03 16:41:36 -07004415 return work;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004416}
4417
Stephen Hemmingerbea33482007-10-03 16:41:36 -07004418/**
4419 * __napi_schedule - schedule for receive
Randy Dunlapc4ea43c2007-10-12 21:17:49 -07004420 * @n: entry to schedule
Stephen Hemmingerbea33482007-10-03 16:41:36 -07004421 *
Eric Dumazetbc9ad162014-10-28 18:05:13 -07004422 * The entry's receive function will be scheduled to run.
4423 * Consider using __napi_schedule_irqoff() if hard irqs are masked.
Stephen Hemmingerbea33482007-10-03 16:41:36 -07004424 */
Harvey Harrisonb5606c22008-02-13 15:03:16 -08004425void __napi_schedule(struct napi_struct *n)
Stephen Hemmingerbea33482007-10-03 16:41:36 -07004426{
4427 unsigned long flags;
4428
4429 local_irq_save(flags);
Christoph Lameter903ceff2014-08-17 12:30:35 -05004430 ____napi_schedule(this_cpu_ptr(&softnet_data), n);
Stephen Hemmingerbea33482007-10-03 16:41:36 -07004431 local_irq_restore(flags);
4432}
4433EXPORT_SYMBOL(__napi_schedule);
4434
Eric Dumazetbc9ad162014-10-28 18:05:13 -07004435/**
4436 * __napi_schedule_irqoff - schedule for receive
4437 * @n: entry to schedule
4438 *
4439 * Variant of __napi_schedule() assuming hard irqs are masked
4440 */
4441void __napi_schedule_irqoff(struct napi_struct *n)
4442{
4443 ____napi_schedule(this_cpu_ptr(&softnet_data), n);
4444}
4445EXPORT_SYMBOL(__napi_schedule_irqoff);
4446
Herbert Xud565b0a2008-12-15 23:38:52 -08004447void __napi_complete(struct napi_struct *n)
4448{
4449 BUG_ON(!test_bit(NAPI_STATE_SCHED, &n->state));
Herbert Xud565b0a2008-12-15 23:38:52 -08004450
Eric Dumazetd75b1ad2014-11-02 06:19:33 -08004451 list_del_init(&n->poll_list);
Peter Zijlstra4e857c52014-03-17 18:06:10 +01004452 smp_mb__before_atomic();
Herbert Xud565b0a2008-12-15 23:38:52 -08004453 clear_bit(NAPI_STATE_SCHED, &n->state);
4454}
4455EXPORT_SYMBOL(__napi_complete);
4456
Eric Dumazet3b47d302014-11-06 21:09:44 -08004457void napi_complete_done(struct napi_struct *n, int work_done)
Herbert Xud565b0a2008-12-15 23:38:52 -08004458{
4459 unsigned long flags;
4460
4461 /*
4462 * don't let napi dequeue from the cpu poll list
4463 * just in case its running on a different cpu
4464 */
4465 if (unlikely(test_bit(NAPI_STATE_NPSVC, &n->state)))
4466 return;
4467
Eric Dumazet3b47d302014-11-06 21:09:44 -08004468 if (n->gro_list) {
4469 unsigned long timeout = 0;
Eric Dumazetd75b1ad2014-11-02 06:19:33 -08004470
Eric Dumazet3b47d302014-11-06 21:09:44 -08004471 if (work_done)
4472 timeout = n->dev->gro_flush_timeout;
4473
4474 if (timeout)
4475 hrtimer_start(&n->timer, ns_to_ktime(timeout),
4476 HRTIMER_MODE_REL_PINNED);
4477 else
4478 napi_gro_flush(n, false);
4479 }
Eric Dumazetd75b1ad2014-11-02 06:19:33 -08004480 if (likely(list_empty(&n->poll_list))) {
4481 WARN_ON_ONCE(!test_and_clear_bit(NAPI_STATE_SCHED, &n->state));
4482 } else {
4483 /* If n->poll_list is not empty, we need to mask irqs */
4484 local_irq_save(flags);
4485 __napi_complete(n);
4486 local_irq_restore(flags);
4487 }
Herbert Xud565b0a2008-12-15 23:38:52 -08004488}
Eric Dumazet3b47d302014-11-06 21:09:44 -08004489EXPORT_SYMBOL(napi_complete_done);
Herbert Xud565b0a2008-12-15 23:38:52 -08004490
Eliezer Tamiraf12fa62013-06-10 11:39:41 +03004491/* must be called under rcu_read_lock(), as we dont take a reference */
4492struct napi_struct *napi_by_id(unsigned int napi_id)
4493{
4494 unsigned int hash = napi_id % HASH_SIZE(napi_hash);
4495 struct napi_struct *napi;
4496
4497 hlist_for_each_entry_rcu(napi, &napi_hash[hash], napi_hash_node)
4498 if (napi->napi_id == napi_id)
4499 return napi;
4500
4501 return NULL;
4502}
4503EXPORT_SYMBOL_GPL(napi_by_id);
4504
4505void napi_hash_add(struct napi_struct *napi)
4506{
4507 if (!test_and_set_bit(NAPI_STATE_HASHED, &napi->state)) {
4508
4509 spin_lock(&napi_hash_lock);
4510
4511 /* 0 is not a valid id, we also skip an id that is taken
4512 * we expect both events to be extremely rare
4513 */
4514 napi->napi_id = 0;
4515 while (!napi->napi_id) {
4516 napi->napi_id = ++napi_gen_id;
4517 if (napi_by_id(napi->napi_id))
4518 napi->napi_id = 0;
4519 }
4520
4521 hlist_add_head_rcu(&napi->napi_hash_node,
4522 &napi_hash[napi->napi_id % HASH_SIZE(napi_hash)]);
4523
4524 spin_unlock(&napi_hash_lock);
4525 }
4526}
4527EXPORT_SYMBOL_GPL(napi_hash_add);
4528
4529/* Warning : caller is responsible to make sure rcu grace period
4530 * is respected before freeing memory containing @napi
4531 */
4532void napi_hash_del(struct napi_struct *napi)
4533{
4534 spin_lock(&napi_hash_lock);
4535
4536 if (test_and_clear_bit(NAPI_STATE_HASHED, &napi->state))
4537 hlist_del_rcu(&napi->napi_hash_node);
4538
4539 spin_unlock(&napi_hash_lock);
4540}
4541EXPORT_SYMBOL_GPL(napi_hash_del);
4542
Eric Dumazet3b47d302014-11-06 21:09:44 -08004543static enum hrtimer_restart napi_watchdog(struct hrtimer *timer)
4544{
4545 struct napi_struct *napi;
4546
4547 napi = container_of(timer, struct napi_struct, timer);
4548 if (napi->gro_list)
4549 napi_schedule(napi);
4550
4551 return HRTIMER_NORESTART;
4552}
4553
Herbert Xud565b0a2008-12-15 23:38:52 -08004554void netif_napi_add(struct net_device *dev, struct napi_struct *napi,
4555 int (*poll)(struct napi_struct *, int), int weight)
4556{
4557 INIT_LIST_HEAD(&napi->poll_list);
Eric Dumazet3b47d302014-11-06 21:09:44 -08004558 hrtimer_init(&napi->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL_PINNED);
4559 napi->timer.function = napi_watchdog;
Herbert Xu4ae55442009-02-08 18:00:36 +00004560 napi->gro_count = 0;
Herbert Xud565b0a2008-12-15 23:38:52 -08004561 napi->gro_list = NULL;
Herbert Xu5d38a072009-01-04 16:13:40 -08004562 napi->skb = NULL;
Herbert Xud565b0a2008-12-15 23:38:52 -08004563 napi->poll = poll;
Eric Dumazet82dc3c62013-03-05 15:57:22 +00004564 if (weight > NAPI_POLL_WEIGHT)
4565 pr_err_once("netif_napi_add() called with weight %d on device %s\n",
4566 weight, dev->name);
Herbert Xud565b0a2008-12-15 23:38:52 -08004567 napi->weight = weight;
4568 list_add(&napi->dev_list, &dev->napi_list);
Herbert Xud565b0a2008-12-15 23:38:52 -08004569 napi->dev = dev;
Herbert Xu5d38a072009-01-04 16:13:40 -08004570#ifdef CONFIG_NETPOLL
Herbert Xud565b0a2008-12-15 23:38:52 -08004571 spin_lock_init(&napi->poll_lock);
4572 napi->poll_owner = -1;
4573#endif
4574 set_bit(NAPI_STATE_SCHED, &napi->state);
4575}
4576EXPORT_SYMBOL(netif_napi_add);
4577
Eric Dumazet3b47d302014-11-06 21:09:44 -08004578void napi_disable(struct napi_struct *n)
4579{
4580 might_sleep();
4581 set_bit(NAPI_STATE_DISABLE, &n->state);
4582
4583 while (test_and_set_bit(NAPI_STATE_SCHED, &n->state))
4584 msleep(1);
4585
4586 hrtimer_cancel(&n->timer);
4587
4588 clear_bit(NAPI_STATE_DISABLE, &n->state);
4589}
4590EXPORT_SYMBOL(napi_disable);
4591
Herbert Xud565b0a2008-12-15 23:38:52 -08004592void netif_napi_del(struct napi_struct *napi)
4593{
Peter P Waskiewicz Jrd7b06632008-12-26 01:35:35 -08004594 list_del_init(&napi->dev_list);
Herbert Xu76620aa2009-04-16 02:02:07 -07004595 napi_free_frags(napi);
Herbert Xud565b0a2008-12-15 23:38:52 -08004596
Eric Dumazet289dccb2013-12-20 14:29:08 -08004597 kfree_skb_list(napi->gro_list);
Herbert Xud565b0a2008-12-15 23:38:52 -08004598 napi->gro_list = NULL;
Herbert Xu4ae55442009-02-08 18:00:36 +00004599 napi->gro_count = 0;
Herbert Xud565b0a2008-12-15 23:38:52 -08004600}
4601EXPORT_SYMBOL(netif_napi_del);
4602
Herbert Xu726ce702014-12-21 07:16:21 +11004603static int napi_poll(struct napi_struct *n, struct list_head *repoll)
4604{
4605 void *have;
4606 int work, weight;
4607
4608 list_del_init(&n->poll_list);
4609
4610 have = netpoll_poll_lock(n);
4611
4612 weight = n->weight;
4613
4614 /* This NAPI_STATE_SCHED test is for avoiding a race
4615 * with netpoll's poll_napi(). Only the entity which
4616 * obtains the lock and sees NAPI_STATE_SCHED set will
4617 * actually make the ->poll() call. Therefore we avoid
4618 * accidentally calling ->poll() when NAPI is not scheduled.
4619 */
4620 work = 0;
4621 if (test_bit(NAPI_STATE_SCHED, &n->state)) {
4622 work = n->poll(n, weight);
4623 trace_napi_poll(n);
4624 }
4625
4626 WARN_ON_ONCE(work > weight);
4627
4628 if (likely(work < weight))
4629 goto out_unlock;
4630
4631 /* Drivers must not modify the NAPI state if they
4632 * consume the entire weight. In such cases this code
4633 * still "owns" the NAPI instance and therefore can
4634 * move the instance around on the list at-will.
4635 */
4636 if (unlikely(napi_disable_pending(n))) {
4637 napi_complete(n);
4638 goto out_unlock;
4639 }
4640
4641 if (n->gro_list) {
4642 /* flush too old packets
4643 * If HZ < 1000, flush all packets.
4644 */
4645 napi_gro_flush(n, HZ >= 1000);
4646 }
4647
Herbert Xu001ce542014-12-21 07:16:22 +11004648 /* Some drivers may have called napi_schedule
4649 * prior to exhausting their budget.
4650 */
4651 if (unlikely(!list_empty(&n->poll_list))) {
4652 pr_warn_once("%s: Budget exhausted after napi rescheduled\n",
4653 n->dev ? n->dev->name : "backlog");
4654 goto out_unlock;
4655 }
4656
Herbert Xu726ce702014-12-21 07:16:21 +11004657 list_add_tail(&n->poll_list, repoll);
4658
4659out_unlock:
4660 netpoll_poll_unlock(have);
4661
4662 return work;
4663}
4664
Linus Torvalds1da177e2005-04-16 15:20:36 -07004665static void net_rx_action(struct softirq_action *h)
4666{
Christoph Lameter903ceff2014-08-17 12:30:35 -05004667 struct softnet_data *sd = this_cpu_ptr(&softnet_data);
Stephen Hemminger24f8b232008-11-03 17:14:38 -08004668 unsigned long time_limit = jiffies + 2;
Stephen Hemminger51b0bde2005-06-23 20:14:40 -07004669 int budget = netdev_budget;
Eric Dumazetd75b1ad2014-11-02 06:19:33 -08004670 LIST_HEAD(list);
4671 LIST_HEAD(repoll);
Matt Mackall53fb95d2005-08-11 19:27:43 -07004672
Linus Torvalds1da177e2005-04-16 15:20:36 -07004673 local_irq_disable();
Eric Dumazetd75b1ad2014-11-02 06:19:33 -08004674 list_splice_init(&sd->poll_list, &list);
4675 local_irq_enable();
Linus Torvalds1da177e2005-04-16 15:20:36 -07004676
Herbert Xuceb8d5b2014-12-21 07:16:25 +11004677 for (;;) {
Stephen Hemmingerbea33482007-10-03 16:41:36 -07004678 struct napi_struct *n;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004679
Herbert Xuceb8d5b2014-12-21 07:16:25 +11004680 if (list_empty(&list)) {
4681 if (!sd_has_rps_ipi_waiting(sd) && list_empty(&repoll))
4682 return;
4683 break;
4684 }
4685
Herbert Xu6bd373e2014-12-21 07:16:24 +11004686 n = list_first_entry(&list, struct napi_struct, poll_list);
4687 budget -= napi_poll(n, &repoll);
4688
Eric Dumazetd75b1ad2014-11-02 06:19:33 -08004689 /* If softirq window is exhausted then punt.
Stephen Hemminger24f8b232008-11-03 17:14:38 -08004690 * Allow this to run for 2 jiffies since which will allow
4691 * an average latency of 1.5/HZ.
Stephen Hemmingerbea33482007-10-03 16:41:36 -07004692 */
Herbert Xuceb8d5b2014-12-21 07:16:25 +11004693 if (unlikely(budget <= 0 ||
4694 time_after_eq(jiffies, time_limit))) {
4695 sd->time_squeeze++;
4696 break;
4697 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07004698 }
Eric Dumazetd75b1ad2014-11-02 06:19:33 -08004699
Eric Dumazetd75b1ad2014-11-02 06:19:33 -08004700 local_irq_disable();
4701
4702 list_splice_tail_init(&sd->poll_list, &list);
4703 list_splice_tail(&repoll, &list);
4704 list_splice(&list, &sd->poll_list);
4705 if (!list_empty(&sd->poll_list))
4706 __raise_softirq_irqoff(NET_RX_SOFTIRQ);
4707
Eric Dumazete326bed2010-04-22 00:22:45 -07004708 net_rps_action_and_irq_enable(sd);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004709}
4710
Veaceslav Falicoaa9d8562013-08-28 23:25:04 +02004711struct netdev_adjacent {
Jiri Pirko9ff162a2013-01-03 22:48:49 +00004712 struct net_device *dev;
Veaceslav Falico5d261912013-08-28 23:25:05 +02004713
4714 /* upper master flag, there can only be one master device per list */
Jiri Pirko9ff162a2013-01-03 22:48:49 +00004715 bool master;
Veaceslav Falico5d261912013-08-28 23:25:05 +02004716
Veaceslav Falico5d261912013-08-28 23:25:05 +02004717 /* counter for the number of times this device was added to us */
4718 u16 ref_nr;
4719
Veaceslav Falico402dae92013-09-25 09:20:09 +02004720 /* private field for the users */
4721 void *private;
4722
Jiri Pirko9ff162a2013-01-03 22:48:49 +00004723 struct list_head list;
4724 struct rcu_head rcu;
Jiri Pirko9ff162a2013-01-03 22:48:49 +00004725};
4726
Veaceslav Falico5d261912013-08-28 23:25:05 +02004727static struct netdev_adjacent *__netdev_find_adj(struct net_device *dev,
4728 struct net_device *adj_dev,
Veaceslav Falico2f268f12013-09-25 09:20:07 +02004729 struct list_head *adj_list)
Jiri Pirko9ff162a2013-01-03 22:48:49 +00004730{
Veaceslav Falico5d261912013-08-28 23:25:05 +02004731 struct netdev_adjacent *adj;
Veaceslav Falico5d261912013-08-28 23:25:05 +02004732
Veaceslav Falico2f268f12013-09-25 09:20:07 +02004733 list_for_each_entry(adj, adj_list, list) {
Veaceslav Falico5d261912013-08-28 23:25:05 +02004734 if (adj->dev == adj_dev)
4735 return adj;
Jiri Pirko9ff162a2013-01-03 22:48:49 +00004736 }
4737 return NULL;
4738}
4739
4740/**
4741 * netdev_has_upper_dev - Check if device is linked to an upper device
4742 * @dev: device
4743 * @upper_dev: upper device to check
4744 *
4745 * Find out if a device is linked to specified upper device and return true
4746 * in case it is. Note that this checks only immediate upper device,
4747 * not through a complete stack of devices. The caller must hold the RTNL lock.
4748 */
4749bool netdev_has_upper_dev(struct net_device *dev,
4750 struct net_device *upper_dev)
4751{
4752 ASSERT_RTNL();
4753
Veaceslav Falico2f268f12013-09-25 09:20:07 +02004754 return __netdev_find_adj(dev, upper_dev, &dev->all_adj_list.upper);
Jiri Pirko9ff162a2013-01-03 22:48:49 +00004755}
4756EXPORT_SYMBOL(netdev_has_upper_dev);
4757
4758/**
4759 * netdev_has_any_upper_dev - Check if device is linked to some device
4760 * @dev: device
4761 *
4762 * Find out if a device is linked to an upper device and return true in case
4763 * it is. The caller must hold the RTNL lock.
4764 */
stephen hemminger1d143d92013-12-29 14:01:29 -08004765static bool netdev_has_any_upper_dev(struct net_device *dev)
Jiri Pirko9ff162a2013-01-03 22:48:49 +00004766{
4767 ASSERT_RTNL();
4768
Veaceslav Falico2f268f12013-09-25 09:20:07 +02004769 return !list_empty(&dev->all_adj_list.upper);
Jiri Pirko9ff162a2013-01-03 22:48:49 +00004770}
Jiri Pirko9ff162a2013-01-03 22:48:49 +00004771
4772/**
4773 * netdev_master_upper_dev_get - Get master upper device
4774 * @dev: device
4775 *
4776 * Find a master upper device and return pointer to it or NULL in case
4777 * it's not there. The caller must hold the RTNL lock.
4778 */
4779struct net_device *netdev_master_upper_dev_get(struct net_device *dev)
4780{
Veaceslav Falicoaa9d8562013-08-28 23:25:04 +02004781 struct netdev_adjacent *upper;
Jiri Pirko9ff162a2013-01-03 22:48:49 +00004782
4783 ASSERT_RTNL();
4784
Veaceslav Falico2f268f12013-09-25 09:20:07 +02004785 if (list_empty(&dev->adj_list.upper))
Jiri Pirko9ff162a2013-01-03 22:48:49 +00004786 return NULL;
4787
Veaceslav Falico2f268f12013-09-25 09:20:07 +02004788 upper = list_first_entry(&dev->adj_list.upper,
Veaceslav Falicoaa9d8562013-08-28 23:25:04 +02004789 struct netdev_adjacent, list);
Jiri Pirko9ff162a2013-01-03 22:48:49 +00004790 if (likely(upper->master))
4791 return upper->dev;
4792 return NULL;
4793}
4794EXPORT_SYMBOL(netdev_master_upper_dev_get);
4795
Veaceslav Falicob6ccba42013-09-25 09:20:23 +02004796void *netdev_adjacent_get_private(struct list_head *adj_list)
4797{
4798 struct netdev_adjacent *adj;
4799
4800 adj = list_entry(adj_list, struct netdev_adjacent, list);
4801
4802 return adj->private;
4803}
4804EXPORT_SYMBOL(netdev_adjacent_get_private);
4805
Veaceslav Falico31088a12013-09-25 09:20:12 +02004806/**
Vlad Yasevich44a40852014-05-16 17:20:38 -04004807 * netdev_upper_get_next_dev_rcu - Get the next dev from upper list
4808 * @dev: device
4809 * @iter: list_head ** of the current position
4810 *
4811 * Gets the next device from the dev's upper list, starting from iter
4812 * position. The caller must hold RCU read lock.
4813 */
4814struct net_device *netdev_upper_get_next_dev_rcu(struct net_device *dev,
4815 struct list_head **iter)
4816{
4817 struct netdev_adjacent *upper;
4818
4819 WARN_ON_ONCE(!rcu_read_lock_held() && !lockdep_rtnl_is_held());
4820
4821 upper = list_entry_rcu((*iter)->next, struct netdev_adjacent, list);
4822
4823 if (&upper->list == &dev->adj_list.upper)
4824 return NULL;
4825
4826 *iter = &upper->list;
4827
4828 return upper->dev;
4829}
4830EXPORT_SYMBOL(netdev_upper_get_next_dev_rcu);
4831
4832/**
Veaceslav Falico31088a12013-09-25 09:20:12 +02004833 * netdev_all_upper_get_next_dev_rcu - Get the next dev from upper list
Veaceslav Falico48311f42013-08-28 23:25:07 +02004834 * @dev: device
4835 * @iter: list_head ** of the current position
4836 *
4837 * Gets the next device from the dev's upper list, starting from iter
4838 * position. The caller must hold RCU read lock.
4839 */
Veaceslav Falico2f268f12013-09-25 09:20:07 +02004840struct net_device *netdev_all_upper_get_next_dev_rcu(struct net_device *dev,
4841 struct list_head **iter)
Veaceslav Falico48311f42013-08-28 23:25:07 +02004842{
4843 struct netdev_adjacent *upper;
4844
John Fastabend85328242013-11-26 06:33:52 +00004845 WARN_ON_ONCE(!rcu_read_lock_held() && !lockdep_rtnl_is_held());
Veaceslav Falico48311f42013-08-28 23:25:07 +02004846
4847 upper = list_entry_rcu((*iter)->next, struct netdev_adjacent, list);
4848
Veaceslav Falico2f268f12013-09-25 09:20:07 +02004849 if (&upper->list == &dev->all_adj_list.upper)
Veaceslav Falico48311f42013-08-28 23:25:07 +02004850 return NULL;
4851
4852 *iter = &upper->list;
4853
4854 return upper->dev;
4855}
Veaceslav Falico2f268f12013-09-25 09:20:07 +02004856EXPORT_SYMBOL(netdev_all_upper_get_next_dev_rcu);
Veaceslav Falico48311f42013-08-28 23:25:07 +02004857
Jiri Pirko9ff162a2013-01-03 22:48:49 +00004858/**
Veaceslav Falico31088a12013-09-25 09:20:12 +02004859 * netdev_lower_get_next_private - Get the next ->private from the
4860 * lower neighbour list
4861 * @dev: device
4862 * @iter: list_head ** of the current position
4863 *
4864 * Gets the next netdev_adjacent->private from the dev's lower neighbour
4865 * list, starting from iter position. The caller must hold either hold the
4866 * RTNL lock or its own locking that guarantees that the neighbour lower
4867 * list will remain unchainged.
4868 */
4869void *netdev_lower_get_next_private(struct net_device *dev,
4870 struct list_head **iter)
4871{
4872 struct netdev_adjacent *lower;
4873
4874 lower = list_entry(*iter, struct netdev_adjacent, list);
4875
4876 if (&lower->list == &dev->adj_list.lower)
4877 return NULL;
4878
Veaceslav Falico6859e7d2014-04-07 11:25:12 +02004879 *iter = lower->list.next;
Veaceslav Falico31088a12013-09-25 09:20:12 +02004880
4881 return lower->private;
4882}
4883EXPORT_SYMBOL(netdev_lower_get_next_private);
4884
4885/**
4886 * netdev_lower_get_next_private_rcu - Get the next ->private from the
4887 * lower neighbour list, RCU
4888 * variant
4889 * @dev: device
4890 * @iter: list_head ** of the current position
4891 *
4892 * Gets the next netdev_adjacent->private from the dev's lower neighbour
4893 * list, starting from iter position. The caller must hold RCU read lock.
4894 */
4895void *netdev_lower_get_next_private_rcu(struct net_device *dev,
4896 struct list_head **iter)
4897{
4898 struct netdev_adjacent *lower;
4899
4900 WARN_ON_ONCE(!rcu_read_lock_held());
4901
4902 lower = list_entry_rcu((*iter)->next, struct netdev_adjacent, list);
4903
4904 if (&lower->list == &dev->adj_list.lower)
4905 return NULL;
4906
Veaceslav Falico6859e7d2014-04-07 11:25:12 +02004907 *iter = &lower->list;
Veaceslav Falico31088a12013-09-25 09:20:12 +02004908
4909 return lower->private;
4910}
4911EXPORT_SYMBOL(netdev_lower_get_next_private_rcu);
4912
4913/**
Vlad Yasevich4085ebe2014-05-16 17:04:53 -04004914 * netdev_lower_get_next - Get the next device from the lower neighbour
4915 * list
4916 * @dev: device
4917 * @iter: list_head ** of the current position
4918 *
4919 * Gets the next netdev_adjacent from the dev's lower neighbour
4920 * list, starting from iter position. The caller must hold RTNL lock or
4921 * its own locking that guarantees that the neighbour lower
4922 * list will remain unchainged.
4923 */
4924void *netdev_lower_get_next(struct net_device *dev, struct list_head **iter)
4925{
4926 struct netdev_adjacent *lower;
4927
4928 lower = list_entry((*iter)->next, struct netdev_adjacent, list);
4929
4930 if (&lower->list == &dev->adj_list.lower)
4931 return NULL;
4932
4933 *iter = &lower->list;
4934
4935 return lower->dev;
4936}
4937EXPORT_SYMBOL(netdev_lower_get_next);
4938
4939/**
dingtianhonge001bfa2013-12-13 10:19:55 +08004940 * netdev_lower_get_first_private_rcu - Get the first ->private from the
4941 * lower neighbour list, RCU
4942 * variant
4943 * @dev: device
4944 *
4945 * Gets the first netdev_adjacent->private from the dev's lower neighbour
4946 * list. The caller must hold RCU read lock.
4947 */
4948void *netdev_lower_get_first_private_rcu(struct net_device *dev)
4949{
4950 struct netdev_adjacent *lower;
4951
4952 lower = list_first_or_null_rcu(&dev->adj_list.lower,
4953 struct netdev_adjacent, list);
4954 if (lower)
4955 return lower->private;
4956 return NULL;
4957}
4958EXPORT_SYMBOL(netdev_lower_get_first_private_rcu);
4959
4960/**
Jiri Pirko9ff162a2013-01-03 22:48:49 +00004961 * netdev_master_upper_dev_get_rcu - Get master upper device
4962 * @dev: device
4963 *
4964 * Find a master upper device and return pointer to it or NULL in case
4965 * it's not there. The caller must hold the RCU read lock.
4966 */
4967struct net_device *netdev_master_upper_dev_get_rcu(struct net_device *dev)
4968{
Veaceslav Falicoaa9d8562013-08-28 23:25:04 +02004969 struct netdev_adjacent *upper;
Jiri Pirko9ff162a2013-01-03 22:48:49 +00004970
Veaceslav Falico2f268f12013-09-25 09:20:07 +02004971 upper = list_first_or_null_rcu(&dev->adj_list.upper,
Veaceslav Falicoaa9d8562013-08-28 23:25:04 +02004972 struct netdev_adjacent, list);
Jiri Pirko9ff162a2013-01-03 22:48:49 +00004973 if (upper && likely(upper->master))
4974 return upper->dev;
4975 return NULL;
4976}
4977EXPORT_SYMBOL(netdev_master_upper_dev_get_rcu);
4978
Rashika Kheria0a59f3a2014-02-09 20:26:25 +05304979static int netdev_adjacent_sysfs_add(struct net_device *dev,
Veaceslav Falico3ee32702014-01-14 21:58:50 +01004980 struct net_device *adj_dev,
4981 struct list_head *dev_list)
4982{
4983 char linkname[IFNAMSIZ+7];
4984 sprintf(linkname, dev_list == &dev->adj_list.upper ?
4985 "upper_%s" : "lower_%s", adj_dev->name);
4986 return sysfs_create_link(&(dev->dev.kobj), &(adj_dev->dev.kobj),
4987 linkname);
4988}
Rashika Kheria0a59f3a2014-02-09 20:26:25 +05304989static void netdev_adjacent_sysfs_del(struct net_device *dev,
Veaceslav Falico3ee32702014-01-14 21:58:50 +01004990 char *name,
4991 struct list_head *dev_list)
4992{
4993 char linkname[IFNAMSIZ+7];
4994 sprintf(linkname, dev_list == &dev->adj_list.upper ?
4995 "upper_%s" : "lower_%s", name);
4996 sysfs_remove_link(&(dev->dev.kobj), linkname);
4997}
4998
Alexander Y. Fomichev7ce64c72014-09-15 14:22:35 +04004999static inline bool netdev_adjacent_is_neigh_list(struct net_device *dev,
5000 struct net_device *adj_dev,
5001 struct list_head *dev_list)
5002{
5003 return (dev_list == &dev->adj_list.upper ||
5004 dev_list == &dev->adj_list.lower) &&
5005 net_eq(dev_net(dev), dev_net(adj_dev));
5006}
Veaceslav Falico3ee32702014-01-14 21:58:50 +01005007
Veaceslav Falico5d261912013-08-28 23:25:05 +02005008static int __netdev_adjacent_dev_insert(struct net_device *dev,
5009 struct net_device *adj_dev,
Veaceslav Falico7863c052013-09-25 09:20:06 +02005010 struct list_head *dev_list,
Veaceslav Falico402dae92013-09-25 09:20:09 +02005011 void *private, bool master)
Veaceslav Falico5d261912013-08-28 23:25:05 +02005012{
5013 struct netdev_adjacent *adj;
Veaceslav Falico842d67a2013-09-25 09:20:31 +02005014 int ret;
Veaceslav Falico5d261912013-08-28 23:25:05 +02005015
Veaceslav Falico7863c052013-09-25 09:20:06 +02005016 adj = __netdev_find_adj(dev, adj_dev, dev_list);
Veaceslav Falico5d261912013-08-28 23:25:05 +02005017
5018 if (adj) {
Veaceslav Falico5d261912013-08-28 23:25:05 +02005019 adj->ref_nr++;
5020 return 0;
5021 }
5022
5023 adj = kmalloc(sizeof(*adj), GFP_KERNEL);
5024 if (!adj)
5025 return -ENOMEM;
5026
5027 adj->dev = adj_dev;
5028 adj->master = master;
Veaceslav Falico5d261912013-08-28 23:25:05 +02005029 adj->ref_nr = 1;
Veaceslav Falico402dae92013-09-25 09:20:09 +02005030 adj->private = private;
Veaceslav Falico5d261912013-08-28 23:25:05 +02005031 dev_hold(adj_dev);
Veaceslav Falico2f268f12013-09-25 09:20:07 +02005032
5033 pr_debug("dev_hold for %s, because of link added from %s to %s\n",
5034 adj_dev->name, dev->name, adj_dev->name);
Veaceslav Falico5d261912013-08-28 23:25:05 +02005035
Alexander Y. Fomichev7ce64c72014-09-15 14:22:35 +04005036 if (netdev_adjacent_is_neigh_list(dev, adj_dev, dev_list)) {
Veaceslav Falico3ee32702014-01-14 21:58:50 +01005037 ret = netdev_adjacent_sysfs_add(dev, adj_dev, dev_list);
Veaceslav Falico5831d66e2013-09-25 09:20:32 +02005038 if (ret)
5039 goto free_adj;
5040 }
5041
Veaceslav Falico7863c052013-09-25 09:20:06 +02005042 /* Ensure that master link is always the first item in list. */
Veaceslav Falico842d67a2013-09-25 09:20:31 +02005043 if (master) {
5044 ret = sysfs_create_link(&(dev->dev.kobj),
5045 &(adj_dev->dev.kobj), "master");
5046 if (ret)
Veaceslav Falico5831d66e2013-09-25 09:20:32 +02005047 goto remove_symlinks;
Veaceslav Falico842d67a2013-09-25 09:20:31 +02005048
Veaceslav Falico7863c052013-09-25 09:20:06 +02005049 list_add_rcu(&adj->list, dev_list);
Veaceslav Falico842d67a2013-09-25 09:20:31 +02005050 } else {
Veaceslav Falico7863c052013-09-25 09:20:06 +02005051 list_add_tail_rcu(&adj->list, dev_list);
Veaceslav Falico842d67a2013-09-25 09:20:31 +02005052 }
Veaceslav Falico5d261912013-08-28 23:25:05 +02005053
5054 return 0;
Veaceslav Falico842d67a2013-09-25 09:20:31 +02005055
Veaceslav Falico5831d66e2013-09-25 09:20:32 +02005056remove_symlinks:
Alexander Y. Fomichev7ce64c72014-09-15 14:22:35 +04005057 if (netdev_adjacent_is_neigh_list(dev, adj_dev, dev_list))
Veaceslav Falico3ee32702014-01-14 21:58:50 +01005058 netdev_adjacent_sysfs_del(dev, adj_dev->name, dev_list);
Veaceslav Falico842d67a2013-09-25 09:20:31 +02005059free_adj:
5060 kfree(adj);
Nikolay Aleksandrov974daef2013-10-23 15:28:56 +02005061 dev_put(adj_dev);
Veaceslav Falico842d67a2013-09-25 09:20:31 +02005062
5063 return ret;
Veaceslav Falico5d261912013-08-28 23:25:05 +02005064}
5065
stephen hemminger1d143d92013-12-29 14:01:29 -08005066static void __netdev_adjacent_dev_remove(struct net_device *dev,
5067 struct net_device *adj_dev,
5068 struct list_head *dev_list)
Veaceslav Falico5d261912013-08-28 23:25:05 +02005069{
5070 struct netdev_adjacent *adj;
5071
Veaceslav Falico7863c052013-09-25 09:20:06 +02005072 adj = __netdev_find_adj(dev, adj_dev, dev_list);
Veaceslav Falico5d261912013-08-28 23:25:05 +02005073
Veaceslav Falico2f268f12013-09-25 09:20:07 +02005074 if (!adj) {
5075 pr_err("tried to remove device %s from %s\n",
5076 dev->name, adj_dev->name);
Veaceslav Falico5d261912013-08-28 23:25:05 +02005077 BUG();
Veaceslav Falico2f268f12013-09-25 09:20:07 +02005078 }
Veaceslav Falico5d261912013-08-28 23:25:05 +02005079
5080 if (adj->ref_nr > 1) {
Veaceslav Falico2f268f12013-09-25 09:20:07 +02005081 pr_debug("%s to %s ref_nr-- = %d\n", dev->name, adj_dev->name,
5082 adj->ref_nr-1);
Veaceslav Falico5d261912013-08-28 23:25:05 +02005083 adj->ref_nr--;
5084 return;
5085 }
5086
Veaceslav Falico842d67a2013-09-25 09:20:31 +02005087 if (adj->master)
5088 sysfs_remove_link(&(dev->dev.kobj), "master");
5089
Alexander Y. Fomichev7ce64c72014-09-15 14:22:35 +04005090 if (netdev_adjacent_is_neigh_list(dev, adj_dev, dev_list))
Veaceslav Falico3ee32702014-01-14 21:58:50 +01005091 netdev_adjacent_sysfs_del(dev, adj_dev->name, dev_list);
Veaceslav Falico5831d66e2013-09-25 09:20:32 +02005092
Veaceslav Falico5d261912013-08-28 23:25:05 +02005093 list_del_rcu(&adj->list);
Veaceslav Falico2f268f12013-09-25 09:20:07 +02005094 pr_debug("dev_put for %s, because link removed from %s to %s\n",
5095 adj_dev->name, dev->name, adj_dev->name);
Veaceslav Falico5d261912013-08-28 23:25:05 +02005096 dev_put(adj_dev);
5097 kfree_rcu(adj, rcu);
5098}
5099
stephen hemminger1d143d92013-12-29 14:01:29 -08005100static int __netdev_adjacent_dev_link_lists(struct net_device *dev,
5101 struct net_device *upper_dev,
5102 struct list_head *up_list,
5103 struct list_head *down_list,
5104 void *private, bool master)
Veaceslav Falico5d261912013-08-28 23:25:05 +02005105{
5106 int ret;
5107
Veaceslav Falico402dae92013-09-25 09:20:09 +02005108 ret = __netdev_adjacent_dev_insert(dev, upper_dev, up_list, private,
5109 master);
Veaceslav Falico5d261912013-08-28 23:25:05 +02005110 if (ret)
5111 return ret;
5112
Veaceslav Falico402dae92013-09-25 09:20:09 +02005113 ret = __netdev_adjacent_dev_insert(upper_dev, dev, down_list, private,
5114 false);
Veaceslav Falico5d261912013-08-28 23:25:05 +02005115 if (ret) {
Veaceslav Falico2f268f12013-09-25 09:20:07 +02005116 __netdev_adjacent_dev_remove(dev, upper_dev, up_list);
Veaceslav Falico5d261912013-08-28 23:25:05 +02005117 return ret;
5118 }
5119
5120 return 0;
5121}
5122
stephen hemminger1d143d92013-12-29 14:01:29 -08005123static int __netdev_adjacent_dev_link(struct net_device *dev,
5124 struct net_device *upper_dev)
Veaceslav Falico5d261912013-08-28 23:25:05 +02005125{
Veaceslav Falico2f268f12013-09-25 09:20:07 +02005126 return __netdev_adjacent_dev_link_lists(dev, upper_dev,
5127 &dev->all_adj_list.upper,
5128 &upper_dev->all_adj_list.lower,
Veaceslav Falico402dae92013-09-25 09:20:09 +02005129 NULL, false);
Veaceslav Falico5d261912013-08-28 23:25:05 +02005130}
5131
stephen hemminger1d143d92013-12-29 14:01:29 -08005132static void __netdev_adjacent_dev_unlink_lists(struct net_device *dev,
5133 struct net_device *upper_dev,
5134 struct list_head *up_list,
5135 struct list_head *down_list)
Veaceslav Falico5d261912013-08-28 23:25:05 +02005136{
Veaceslav Falico2f268f12013-09-25 09:20:07 +02005137 __netdev_adjacent_dev_remove(dev, upper_dev, up_list);
5138 __netdev_adjacent_dev_remove(upper_dev, dev, down_list);
Veaceslav Falico5d261912013-08-28 23:25:05 +02005139}
5140
stephen hemminger1d143d92013-12-29 14:01:29 -08005141static void __netdev_adjacent_dev_unlink(struct net_device *dev,
5142 struct net_device *upper_dev)
Veaceslav Falico5d261912013-08-28 23:25:05 +02005143{
Veaceslav Falico2f268f12013-09-25 09:20:07 +02005144 __netdev_adjacent_dev_unlink_lists(dev, upper_dev,
5145 &dev->all_adj_list.upper,
5146 &upper_dev->all_adj_list.lower);
Veaceslav Falico5d261912013-08-28 23:25:05 +02005147}
5148
stephen hemminger1d143d92013-12-29 14:01:29 -08005149static int __netdev_adjacent_dev_link_neighbour(struct net_device *dev,
5150 struct net_device *upper_dev,
5151 void *private, bool master)
Veaceslav Falico2f268f12013-09-25 09:20:07 +02005152{
5153 int ret = __netdev_adjacent_dev_link(dev, upper_dev);
5154
5155 if (ret)
5156 return ret;
5157
5158 ret = __netdev_adjacent_dev_link_lists(dev, upper_dev,
5159 &dev->adj_list.upper,
5160 &upper_dev->adj_list.lower,
Veaceslav Falico402dae92013-09-25 09:20:09 +02005161 private, master);
Veaceslav Falico2f268f12013-09-25 09:20:07 +02005162 if (ret) {
5163 __netdev_adjacent_dev_unlink(dev, upper_dev);
5164 return ret;
5165 }
5166
5167 return 0;
5168}
5169
stephen hemminger1d143d92013-12-29 14:01:29 -08005170static void __netdev_adjacent_dev_unlink_neighbour(struct net_device *dev,
5171 struct net_device *upper_dev)
Veaceslav Falico2f268f12013-09-25 09:20:07 +02005172{
5173 __netdev_adjacent_dev_unlink(dev, upper_dev);
5174 __netdev_adjacent_dev_unlink_lists(dev, upper_dev,
5175 &dev->adj_list.upper,
5176 &upper_dev->adj_list.lower);
5177}
Veaceslav Falico5d261912013-08-28 23:25:05 +02005178
Jiri Pirko9ff162a2013-01-03 22:48:49 +00005179static int __netdev_upper_dev_link(struct net_device *dev,
Veaceslav Falico402dae92013-09-25 09:20:09 +02005180 struct net_device *upper_dev, bool master,
5181 void *private)
Jiri Pirko9ff162a2013-01-03 22:48:49 +00005182{
Veaceslav Falico5d261912013-08-28 23:25:05 +02005183 struct netdev_adjacent *i, *j, *to_i, *to_j;
5184 int ret = 0;
Jiri Pirko9ff162a2013-01-03 22:48:49 +00005185
5186 ASSERT_RTNL();
5187
5188 if (dev == upper_dev)
5189 return -EBUSY;
5190
5191 /* To prevent loops, check if dev is not upper device to upper_dev. */
Veaceslav Falico2f268f12013-09-25 09:20:07 +02005192 if (__netdev_find_adj(upper_dev, dev, &upper_dev->all_adj_list.upper))
Jiri Pirko9ff162a2013-01-03 22:48:49 +00005193 return -EBUSY;
5194
Veaceslav Falico2f268f12013-09-25 09:20:07 +02005195 if (__netdev_find_adj(dev, upper_dev, &dev->all_adj_list.upper))
Jiri Pirko9ff162a2013-01-03 22:48:49 +00005196 return -EEXIST;
5197
5198 if (master && netdev_master_upper_dev_get(dev))
5199 return -EBUSY;
5200
Veaceslav Falico402dae92013-09-25 09:20:09 +02005201 ret = __netdev_adjacent_dev_link_neighbour(dev, upper_dev, private,
5202 master);
Veaceslav Falico5d261912013-08-28 23:25:05 +02005203 if (ret)
5204 return ret;
Jiri Pirko9ff162a2013-01-03 22:48:49 +00005205
Veaceslav Falico5d261912013-08-28 23:25:05 +02005206 /* Now that we linked these devs, make all the upper_dev's
Veaceslav Falico2f268f12013-09-25 09:20:07 +02005207 * all_adj_list.upper visible to every dev's all_adj_list.lower an
Veaceslav Falico5d261912013-08-28 23:25:05 +02005208 * versa, and don't forget the devices itself. All of these
5209 * links are non-neighbours.
5210 */
Veaceslav Falico2f268f12013-09-25 09:20:07 +02005211 list_for_each_entry(i, &dev->all_adj_list.lower, list) {
5212 list_for_each_entry(j, &upper_dev->all_adj_list.upper, list) {
5213 pr_debug("Interlinking %s with %s, non-neighbour\n",
5214 i->dev->name, j->dev->name);
Veaceslav Falico5d261912013-08-28 23:25:05 +02005215 ret = __netdev_adjacent_dev_link(i->dev, j->dev);
5216 if (ret)
5217 goto rollback_mesh;
5218 }
5219 }
Jiri Pirko9ff162a2013-01-03 22:48:49 +00005220
Veaceslav Falico5d261912013-08-28 23:25:05 +02005221 /* add dev to every upper_dev's upper device */
Veaceslav Falico2f268f12013-09-25 09:20:07 +02005222 list_for_each_entry(i, &upper_dev->all_adj_list.upper, list) {
5223 pr_debug("linking %s's upper device %s with %s\n",
5224 upper_dev->name, i->dev->name, dev->name);
Veaceslav Falico5d261912013-08-28 23:25:05 +02005225 ret = __netdev_adjacent_dev_link(dev, i->dev);
5226 if (ret)
5227 goto rollback_upper_mesh;
5228 }
5229
5230 /* add upper_dev to every dev's lower device */
Veaceslav Falico2f268f12013-09-25 09:20:07 +02005231 list_for_each_entry(i, &dev->all_adj_list.lower, list) {
5232 pr_debug("linking %s's lower device %s with %s\n", dev->name,
5233 i->dev->name, upper_dev->name);
Veaceslav Falico5d261912013-08-28 23:25:05 +02005234 ret = __netdev_adjacent_dev_link(i->dev, upper_dev);
5235 if (ret)
5236 goto rollback_lower_mesh;
5237 }
5238
Jiri Pirko42e52bf2013-05-25 04:12:10 +00005239 call_netdevice_notifiers(NETDEV_CHANGEUPPER, dev);
Jiri Pirko9ff162a2013-01-03 22:48:49 +00005240 return 0;
Veaceslav Falico5d261912013-08-28 23:25:05 +02005241
5242rollback_lower_mesh:
5243 to_i = i;
Veaceslav Falico2f268f12013-09-25 09:20:07 +02005244 list_for_each_entry(i, &dev->all_adj_list.lower, list) {
Veaceslav Falico5d261912013-08-28 23:25:05 +02005245 if (i == to_i)
5246 break;
5247 __netdev_adjacent_dev_unlink(i->dev, upper_dev);
5248 }
5249
5250 i = NULL;
5251
5252rollback_upper_mesh:
5253 to_i = i;
Veaceslav Falico2f268f12013-09-25 09:20:07 +02005254 list_for_each_entry(i, &upper_dev->all_adj_list.upper, list) {
Veaceslav Falico5d261912013-08-28 23:25:05 +02005255 if (i == to_i)
5256 break;
5257 __netdev_adjacent_dev_unlink(dev, i->dev);
5258 }
5259
5260 i = j = NULL;
5261
5262rollback_mesh:
5263 to_i = i;
5264 to_j = j;
Veaceslav Falico2f268f12013-09-25 09:20:07 +02005265 list_for_each_entry(i, &dev->all_adj_list.lower, list) {
5266 list_for_each_entry(j, &upper_dev->all_adj_list.upper, list) {
Veaceslav Falico5d261912013-08-28 23:25:05 +02005267 if (i == to_i && j == to_j)
5268 break;
5269 __netdev_adjacent_dev_unlink(i->dev, j->dev);
5270 }
5271 if (i == to_i)
5272 break;
5273 }
5274
Veaceslav Falico2f268f12013-09-25 09:20:07 +02005275 __netdev_adjacent_dev_unlink_neighbour(dev, upper_dev);
Veaceslav Falico5d261912013-08-28 23:25:05 +02005276
5277 return ret;
Jiri Pirko9ff162a2013-01-03 22:48:49 +00005278}
5279
5280/**
5281 * netdev_upper_dev_link - Add a link to the upper device
5282 * @dev: device
5283 * @upper_dev: new upper device
5284 *
5285 * Adds a link to device which is upper to this one. The caller must hold
5286 * the RTNL lock. On a failure a negative errno code is returned.
5287 * On success the reference counts are adjusted and the function
5288 * returns zero.
5289 */
5290int netdev_upper_dev_link(struct net_device *dev,
5291 struct net_device *upper_dev)
5292{
Veaceslav Falico402dae92013-09-25 09:20:09 +02005293 return __netdev_upper_dev_link(dev, upper_dev, false, NULL);
Jiri Pirko9ff162a2013-01-03 22:48:49 +00005294}
5295EXPORT_SYMBOL(netdev_upper_dev_link);
5296
5297/**
5298 * netdev_master_upper_dev_link - Add a master link to the upper device
5299 * @dev: device
5300 * @upper_dev: new upper device
5301 *
5302 * Adds a link to device which is upper to this one. In this case, only
5303 * one master upper device can be linked, although other non-master devices
5304 * might be linked as well. The caller must hold the RTNL lock.
5305 * On a failure a negative errno code is returned. On success the reference
5306 * counts are adjusted and the function returns zero.
5307 */
5308int netdev_master_upper_dev_link(struct net_device *dev,
5309 struct net_device *upper_dev)
5310{
Veaceslav Falico402dae92013-09-25 09:20:09 +02005311 return __netdev_upper_dev_link(dev, upper_dev, true, NULL);
Jiri Pirko9ff162a2013-01-03 22:48:49 +00005312}
5313EXPORT_SYMBOL(netdev_master_upper_dev_link);
5314
Veaceslav Falico402dae92013-09-25 09:20:09 +02005315int netdev_master_upper_dev_link_private(struct net_device *dev,
5316 struct net_device *upper_dev,
5317 void *private)
5318{
5319 return __netdev_upper_dev_link(dev, upper_dev, true, private);
5320}
5321EXPORT_SYMBOL(netdev_master_upper_dev_link_private);
5322
Jiri Pirko9ff162a2013-01-03 22:48:49 +00005323/**
5324 * netdev_upper_dev_unlink - Removes a link to upper device
5325 * @dev: device
5326 * @upper_dev: new upper device
5327 *
5328 * Removes a link to device which is upper to this one. The caller must hold
5329 * the RTNL lock.
5330 */
5331void netdev_upper_dev_unlink(struct net_device *dev,
5332 struct net_device *upper_dev)
5333{
Veaceslav Falico5d261912013-08-28 23:25:05 +02005334 struct netdev_adjacent *i, *j;
Jiri Pirko9ff162a2013-01-03 22:48:49 +00005335 ASSERT_RTNL();
5336
Veaceslav Falico2f268f12013-09-25 09:20:07 +02005337 __netdev_adjacent_dev_unlink_neighbour(dev, upper_dev);
Veaceslav Falico5d261912013-08-28 23:25:05 +02005338
5339 /* Here is the tricky part. We must remove all dev's lower
5340 * devices from all upper_dev's upper devices and vice
5341 * versa, to maintain the graph relationship.
5342 */
Veaceslav Falico2f268f12013-09-25 09:20:07 +02005343 list_for_each_entry(i, &dev->all_adj_list.lower, list)
5344 list_for_each_entry(j, &upper_dev->all_adj_list.upper, list)
Veaceslav Falico5d261912013-08-28 23:25:05 +02005345 __netdev_adjacent_dev_unlink(i->dev, j->dev);
5346
5347 /* remove also the devices itself from lower/upper device
5348 * list
5349 */
Veaceslav Falico2f268f12013-09-25 09:20:07 +02005350 list_for_each_entry(i, &dev->all_adj_list.lower, list)
Veaceslav Falico5d261912013-08-28 23:25:05 +02005351 __netdev_adjacent_dev_unlink(i->dev, upper_dev);
5352
Veaceslav Falico2f268f12013-09-25 09:20:07 +02005353 list_for_each_entry(i, &upper_dev->all_adj_list.upper, list)
Veaceslav Falico5d261912013-08-28 23:25:05 +02005354 __netdev_adjacent_dev_unlink(dev, i->dev);
5355
Jiri Pirko42e52bf2013-05-25 04:12:10 +00005356 call_netdevice_notifiers(NETDEV_CHANGEUPPER, dev);
Jiri Pirko9ff162a2013-01-03 22:48:49 +00005357}
5358EXPORT_SYMBOL(netdev_upper_dev_unlink);
5359
Moni Shoua61bd3852015-02-03 16:48:29 +02005360/**
5361 * netdev_bonding_info_change - Dispatch event about slave change
5362 * @dev: device
Masanari Iida4a26e4532015-02-14 22:26:34 +09005363 * @bonding_info: info to dispatch
Moni Shoua61bd3852015-02-03 16:48:29 +02005364 *
5365 * Send NETDEV_BONDING_INFO to netdev notifiers with info.
5366 * The caller must hold the RTNL lock.
5367 */
5368void netdev_bonding_info_change(struct net_device *dev,
5369 struct netdev_bonding_info *bonding_info)
5370{
5371 struct netdev_notifier_bonding_info info;
5372
5373 memcpy(&info.bonding_info, bonding_info,
5374 sizeof(struct netdev_bonding_info));
5375 call_netdevice_notifiers_info(NETDEV_BONDING_INFO, dev,
5376 &info.info);
5377}
5378EXPORT_SYMBOL(netdev_bonding_info_change);
5379
Eric Dumazet2ce1ee12015-02-04 13:37:44 -08005380static void netdev_adjacent_add_links(struct net_device *dev)
Alexander Y. Fomichev4c754312014-08-25 16:26:45 +04005381{
5382 struct netdev_adjacent *iter;
5383
5384 struct net *net = dev_net(dev);
5385
5386 list_for_each_entry(iter, &dev->adj_list.upper, list) {
5387 if (!net_eq(net,dev_net(iter->dev)))
5388 continue;
5389 netdev_adjacent_sysfs_add(iter->dev, dev,
5390 &iter->dev->adj_list.lower);
5391 netdev_adjacent_sysfs_add(dev, iter->dev,
5392 &dev->adj_list.upper);
5393 }
5394
5395 list_for_each_entry(iter, &dev->adj_list.lower, list) {
5396 if (!net_eq(net,dev_net(iter->dev)))
5397 continue;
5398 netdev_adjacent_sysfs_add(iter->dev, dev,
5399 &iter->dev->adj_list.upper);
5400 netdev_adjacent_sysfs_add(dev, iter->dev,
5401 &dev->adj_list.lower);
5402 }
5403}
5404
Eric Dumazet2ce1ee12015-02-04 13:37:44 -08005405static void netdev_adjacent_del_links(struct net_device *dev)
Alexander Y. Fomichev4c754312014-08-25 16:26:45 +04005406{
5407 struct netdev_adjacent *iter;
5408
5409 struct net *net = dev_net(dev);
5410
5411 list_for_each_entry(iter, &dev->adj_list.upper, list) {
5412 if (!net_eq(net,dev_net(iter->dev)))
5413 continue;
5414 netdev_adjacent_sysfs_del(iter->dev, dev->name,
5415 &iter->dev->adj_list.lower);
5416 netdev_adjacent_sysfs_del(dev, iter->dev->name,
5417 &dev->adj_list.upper);
5418 }
5419
5420 list_for_each_entry(iter, &dev->adj_list.lower, list) {
5421 if (!net_eq(net,dev_net(iter->dev)))
5422 continue;
5423 netdev_adjacent_sysfs_del(iter->dev, dev->name,
5424 &iter->dev->adj_list.upper);
5425 netdev_adjacent_sysfs_del(dev, iter->dev->name,
5426 &dev->adj_list.lower);
5427 }
5428}
5429
Veaceslav Falico5bb025f2014-01-14 21:58:51 +01005430void netdev_adjacent_rename_links(struct net_device *dev, char *oldname)
Veaceslav Falico402dae92013-09-25 09:20:09 +02005431{
Veaceslav Falico5bb025f2014-01-14 21:58:51 +01005432 struct netdev_adjacent *iter;
Veaceslav Falico402dae92013-09-25 09:20:09 +02005433
Alexander Y. Fomichev4c754312014-08-25 16:26:45 +04005434 struct net *net = dev_net(dev);
5435
Veaceslav Falico5bb025f2014-01-14 21:58:51 +01005436 list_for_each_entry(iter, &dev->adj_list.upper, list) {
Alexander Y. Fomichev4c754312014-08-25 16:26:45 +04005437 if (!net_eq(net,dev_net(iter->dev)))
5438 continue;
Veaceslav Falico5bb025f2014-01-14 21:58:51 +01005439 netdev_adjacent_sysfs_del(iter->dev, oldname,
5440 &iter->dev->adj_list.lower);
5441 netdev_adjacent_sysfs_add(iter->dev, dev,
5442 &iter->dev->adj_list.lower);
5443 }
Veaceslav Falico402dae92013-09-25 09:20:09 +02005444
Veaceslav Falico5bb025f2014-01-14 21:58:51 +01005445 list_for_each_entry(iter, &dev->adj_list.lower, list) {
Alexander Y. Fomichev4c754312014-08-25 16:26:45 +04005446 if (!net_eq(net,dev_net(iter->dev)))
5447 continue;
Veaceslav Falico5bb025f2014-01-14 21:58:51 +01005448 netdev_adjacent_sysfs_del(iter->dev, oldname,
5449 &iter->dev->adj_list.upper);
5450 netdev_adjacent_sysfs_add(iter->dev, dev,
5451 &iter->dev->adj_list.upper);
5452 }
Veaceslav Falico402dae92013-09-25 09:20:09 +02005453}
Veaceslav Falico402dae92013-09-25 09:20:09 +02005454
5455void *netdev_lower_dev_get_private(struct net_device *dev,
5456 struct net_device *lower_dev)
5457{
5458 struct netdev_adjacent *lower;
5459
5460 if (!lower_dev)
5461 return NULL;
5462 lower = __netdev_find_adj(dev, lower_dev, &dev->adj_list.lower);
5463 if (!lower)
5464 return NULL;
5465
5466 return lower->private;
5467}
5468EXPORT_SYMBOL(netdev_lower_dev_get_private);
5469
Vlad Yasevich4085ebe2014-05-16 17:04:53 -04005470
5471int dev_get_nest_level(struct net_device *dev,
5472 bool (*type_check)(struct net_device *dev))
5473{
5474 struct net_device *lower = NULL;
5475 struct list_head *iter;
5476 int max_nest = -1;
5477 int nest;
5478
5479 ASSERT_RTNL();
5480
5481 netdev_for_each_lower_dev(dev, lower, iter) {
5482 nest = dev_get_nest_level(lower, type_check);
5483 if (max_nest < nest)
5484 max_nest = nest;
5485 }
5486
5487 if (type_check(dev))
5488 max_nest++;
5489
5490 return max_nest;
5491}
5492EXPORT_SYMBOL(dev_get_nest_level);
5493
Patrick McHardyb6c40d62008-10-07 15:26:48 -07005494static void dev_change_rx_flags(struct net_device *dev, int flags)
5495{
Stephen Hemmingerd3147742008-11-19 21:32:24 -08005496 const struct net_device_ops *ops = dev->netdev_ops;
5497
Vlad Yasevichd2615bf2013-11-19 20:47:15 -05005498 if (ops->ndo_change_rx_flags)
Stephen Hemmingerd3147742008-11-19 21:32:24 -08005499 ops->ndo_change_rx_flags(dev, flags);
Patrick McHardyb6c40d62008-10-07 15:26:48 -07005500}
5501
Nicolas Dichtel991fb3f2013-09-25 12:02:45 +02005502static int __dev_set_promiscuity(struct net_device *dev, int inc, bool notify)
Patrick McHardy4417da62007-06-27 01:28:10 -07005503{
Eric Dumazetb536db92011-11-30 21:42:26 +00005504 unsigned int old_flags = dev->flags;
Eric W. Biedermand04a48b2012-05-23 17:01:57 -06005505 kuid_t uid;
5506 kgid_t gid;
Patrick McHardy4417da62007-06-27 01:28:10 -07005507
Patrick McHardy24023452007-07-14 18:51:31 -07005508 ASSERT_RTNL();
5509
Wang Chendad9b332008-06-18 01:48:28 -07005510 dev->flags |= IFF_PROMISC;
5511 dev->promiscuity += inc;
5512 if (dev->promiscuity == 0) {
5513 /*
5514 * Avoid overflow.
5515 * If inc causes overflow, untouch promisc and return error.
5516 */
5517 if (inc < 0)
5518 dev->flags &= ~IFF_PROMISC;
5519 else {
5520 dev->promiscuity -= inc;
Joe Perches7b6cd1c2012-02-01 10:54:43 +00005521 pr_warn("%s: promiscuity touches roof, set promiscuity failed. promiscuity feature of device might be broken.\n",
5522 dev->name);
Wang Chendad9b332008-06-18 01:48:28 -07005523 return -EOVERFLOW;
5524 }
5525 }
Patrick McHardy4417da62007-06-27 01:28:10 -07005526 if (dev->flags != old_flags) {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00005527 pr_info("device %s %s promiscuous mode\n",
5528 dev->name,
5529 dev->flags & IFF_PROMISC ? "entered" : "left");
David Howells8192b0c2008-11-14 10:39:10 +11005530 if (audit_enabled) {
5531 current_uid_gid(&uid, &gid);
Klaus Heinrich Kiwi7759db82008-01-23 22:57:45 -05005532 audit_log(current->audit_context, GFP_ATOMIC,
5533 AUDIT_ANOM_PROMISCUOUS,
5534 "dev=%s prom=%d old_prom=%d auid=%u uid=%u gid=%u ses=%u",
5535 dev->name, (dev->flags & IFF_PROMISC),
5536 (old_flags & IFF_PROMISC),
Eric W. Biedermane1760bd2012-09-10 22:39:43 -07005537 from_kuid(&init_user_ns, audit_get_loginuid(current)),
Eric W. Biedermand04a48b2012-05-23 17:01:57 -06005538 from_kuid(&init_user_ns, uid),
5539 from_kgid(&init_user_ns, gid),
Klaus Heinrich Kiwi7759db82008-01-23 22:57:45 -05005540 audit_get_sessionid(current));
David Howells8192b0c2008-11-14 10:39:10 +11005541 }
Patrick McHardy24023452007-07-14 18:51:31 -07005542
Patrick McHardyb6c40d62008-10-07 15:26:48 -07005543 dev_change_rx_flags(dev, IFF_PROMISC);
Patrick McHardy4417da62007-06-27 01:28:10 -07005544 }
Nicolas Dichtel991fb3f2013-09-25 12:02:45 +02005545 if (notify)
5546 __dev_notify_flags(dev, old_flags, IFF_PROMISC);
Wang Chendad9b332008-06-18 01:48:28 -07005547 return 0;
Patrick McHardy4417da62007-06-27 01:28:10 -07005548}
5549
Linus Torvalds1da177e2005-04-16 15:20:36 -07005550/**
5551 * dev_set_promiscuity - update promiscuity count on a device
5552 * @dev: device
5553 * @inc: modifier
5554 *
Stephen Hemminger3041a062006-05-26 13:25:24 -07005555 * Add or remove promiscuity from a device. While the count in the device
Linus Torvalds1da177e2005-04-16 15:20:36 -07005556 * remains above zero the interface remains promiscuous. Once it hits zero
5557 * the device reverts back to normal filtering operation. A negative inc
5558 * value is used to drop promiscuity on the device.
Wang Chendad9b332008-06-18 01:48:28 -07005559 * Return 0 if successful or a negative errno code on error.
Linus Torvalds1da177e2005-04-16 15:20:36 -07005560 */
Wang Chendad9b332008-06-18 01:48:28 -07005561int dev_set_promiscuity(struct net_device *dev, int inc)
Linus Torvalds1da177e2005-04-16 15:20:36 -07005562{
Eric Dumazetb536db92011-11-30 21:42:26 +00005563 unsigned int old_flags = dev->flags;
Wang Chendad9b332008-06-18 01:48:28 -07005564 int err;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005565
Nicolas Dichtel991fb3f2013-09-25 12:02:45 +02005566 err = __dev_set_promiscuity(dev, inc, true);
Patrick McHardy4b5a6982008-07-06 15:49:08 -07005567 if (err < 0)
Wang Chendad9b332008-06-18 01:48:28 -07005568 return err;
Patrick McHardy4417da62007-06-27 01:28:10 -07005569 if (dev->flags != old_flags)
5570 dev_set_rx_mode(dev);
Wang Chendad9b332008-06-18 01:48:28 -07005571 return err;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005572}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005573EXPORT_SYMBOL(dev_set_promiscuity);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005574
Nicolas Dichtel991fb3f2013-09-25 12:02:45 +02005575static int __dev_set_allmulti(struct net_device *dev, int inc, bool notify)
Linus Torvalds1da177e2005-04-16 15:20:36 -07005576{
Nicolas Dichtel991fb3f2013-09-25 12:02:45 +02005577 unsigned int old_flags = dev->flags, old_gflags = dev->gflags;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005578
Patrick McHardy24023452007-07-14 18:51:31 -07005579 ASSERT_RTNL();
5580
Linus Torvalds1da177e2005-04-16 15:20:36 -07005581 dev->flags |= IFF_ALLMULTI;
Wang Chendad9b332008-06-18 01:48:28 -07005582 dev->allmulti += inc;
5583 if (dev->allmulti == 0) {
5584 /*
5585 * Avoid overflow.
5586 * If inc causes overflow, untouch allmulti and return error.
5587 */
5588 if (inc < 0)
5589 dev->flags &= ~IFF_ALLMULTI;
5590 else {
5591 dev->allmulti -= inc;
Joe Perches7b6cd1c2012-02-01 10:54:43 +00005592 pr_warn("%s: allmulti touches roof, set allmulti failed. allmulti feature of device might be broken.\n",
5593 dev->name);
Wang Chendad9b332008-06-18 01:48:28 -07005594 return -EOVERFLOW;
5595 }
5596 }
Patrick McHardy24023452007-07-14 18:51:31 -07005597 if (dev->flags ^ old_flags) {
Patrick McHardyb6c40d62008-10-07 15:26:48 -07005598 dev_change_rx_flags(dev, IFF_ALLMULTI);
Patrick McHardy4417da62007-06-27 01:28:10 -07005599 dev_set_rx_mode(dev);
Nicolas Dichtel991fb3f2013-09-25 12:02:45 +02005600 if (notify)
5601 __dev_notify_flags(dev, old_flags,
5602 dev->gflags ^ old_gflags);
Patrick McHardy24023452007-07-14 18:51:31 -07005603 }
Wang Chendad9b332008-06-18 01:48:28 -07005604 return 0;
Patrick McHardy4417da62007-06-27 01:28:10 -07005605}
Nicolas Dichtel991fb3f2013-09-25 12:02:45 +02005606
5607/**
5608 * dev_set_allmulti - update allmulti count on a device
5609 * @dev: device
5610 * @inc: modifier
5611 *
5612 * Add or remove reception of all multicast frames to a device. While the
5613 * count in the device remains above zero the interface remains listening
5614 * to all interfaces. Once it hits zero the device reverts back to normal
5615 * filtering operation. A negative @inc value is used to drop the counter
5616 * when releasing a resource needing all multicasts.
5617 * Return 0 if successful or a negative errno code on error.
5618 */
5619
5620int dev_set_allmulti(struct net_device *dev, int inc)
5621{
5622 return __dev_set_allmulti(dev, inc, true);
5623}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005624EXPORT_SYMBOL(dev_set_allmulti);
Patrick McHardy4417da62007-06-27 01:28:10 -07005625
5626/*
5627 * Upload unicast and multicast address lists to device and
5628 * configure RX filtering. When the device doesn't support unicast
Joe Perches53ccaae2007-12-20 14:02:06 -08005629 * filtering it is put in promiscuous mode while unicast addresses
Patrick McHardy4417da62007-06-27 01:28:10 -07005630 * are present.
5631 */
5632void __dev_set_rx_mode(struct net_device *dev)
5633{
Stephen Hemmingerd3147742008-11-19 21:32:24 -08005634 const struct net_device_ops *ops = dev->netdev_ops;
5635
Patrick McHardy4417da62007-06-27 01:28:10 -07005636 /* dev_open will call this function so the list will stay sane. */
5637 if (!(dev->flags&IFF_UP))
5638 return;
5639
5640 if (!netif_device_present(dev))
YOSHIFUJI Hideaki40b77c92007-07-19 10:43:23 +09005641 return;
Patrick McHardy4417da62007-06-27 01:28:10 -07005642
Jiri Pirko01789342011-08-16 06:29:00 +00005643 if (!(dev->priv_flags & IFF_UNICAST_FLT)) {
Patrick McHardy4417da62007-06-27 01:28:10 -07005644 /* Unicast addresses changes may only happen under the rtnl,
5645 * therefore calling __dev_set_promiscuity here is safe.
5646 */
Jiri Pirko32e7bfc2010-01-25 13:36:10 -08005647 if (!netdev_uc_empty(dev) && !dev->uc_promisc) {
Nicolas Dichtel991fb3f2013-09-25 12:02:45 +02005648 __dev_set_promiscuity(dev, 1, false);
Joe Perches2d348d12011-07-25 16:17:35 -07005649 dev->uc_promisc = true;
Jiri Pirko32e7bfc2010-01-25 13:36:10 -08005650 } else if (netdev_uc_empty(dev) && dev->uc_promisc) {
Nicolas Dichtel991fb3f2013-09-25 12:02:45 +02005651 __dev_set_promiscuity(dev, -1, false);
Joe Perches2d348d12011-07-25 16:17:35 -07005652 dev->uc_promisc = false;
Patrick McHardy4417da62007-06-27 01:28:10 -07005653 }
Patrick McHardy4417da62007-06-27 01:28:10 -07005654 }
Jiri Pirko01789342011-08-16 06:29:00 +00005655
5656 if (ops->ndo_set_rx_mode)
5657 ops->ndo_set_rx_mode(dev);
Patrick McHardy4417da62007-06-27 01:28:10 -07005658}
5659
5660void dev_set_rx_mode(struct net_device *dev)
5661{
David S. Millerb9e40852008-07-15 00:15:08 -07005662 netif_addr_lock_bh(dev);
Patrick McHardy4417da62007-06-27 01:28:10 -07005663 __dev_set_rx_mode(dev);
David S. Millerb9e40852008-07-15 00:15:08 -07005664 netif_addr_unlock_bh(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005665}
5666
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07005667/**
5668 * dev_get_flags - get flags reported to userspace
5669 * @dev: device
5670 *
5671 * Get the combination of flag bits exported through APIs to userspace.
5672 */
Eric Dumazet95c96172012-04-15 05:58:06 +00005673unsigned int dev_get_flags(const struct net_device *dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07005674{
Eric Dumazet95c96172012-04-15 05:58:06 +00005675 unsigned int flags;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005676
5677 flags = (dev->flags & ~(IFF_PROMISC |
5678 IFF_ALLMULTI |
Stefan Rompfb00055a2006-03-20 17:09:11 -08005679 IFF_RUNNING |
5680 IFF_LOWER_UP |
5681 IFF_DORMANT)) |
Linus Torvalds1da177e2005-04-16 15:20:36 -07005682 (dev->gflags & (IFF_PROMISC |
5683 IFF_ALLMULTI));
5684
Stefan Rompfb00055a2006-03-20 17:09:11 -08005685 if (netif_running(dev)) {
5686 if (netif_oper_up(dev))
5687 flags |= IFF_RUNNING;
5688 if (netif_carrier_ok(dev))
5689 flags |= IFF_LOWER_UP;
5690 if (netif_dormant(dev))
5691 flags |= IFF_DORMANT;
5692 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07005693
5694 return flags;
5695}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005696EXPORT_SYMBOL(dev_get_flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005697
Patrick McHardybd380812010-02-26 06:34:53 +00005698int __dev_change_flags(struct net_device *dev, unsigned int flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -07005699{
Eric Dumazetb536db92011-11-30 21:42:26 +00005700 unsigned int old_flags = dev->flags;
Patrick McHardybd380812010-02-26 06:34:53 +00005701 int ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005702
Patrick McHardy24023452007-07-14 18:51:31 -07005703 ASSERT_RTNL();
5704
Linus Torvalds1da177e2005-04-16 15:20:36 -07005705 /*
5706 * Set the flags on our device.
5707 */
5708
5709 dev->flags = (flags & (IFF_DEBUG | IFF_NOTRAILERS | IFF_NOARP |
5710 IFF_DYNAMIC | IFF_MULTICAST | IFF_PORTSEL |
5711 IFF_AUTOMEDIA)) |
5712 (dev->flags & (IFF_UP | IFF_VOLATILE | IFF_PROMISC |
5713 IFF_ALLMULTI));
5714
5715 /*
5716 * Load in the correct multicast list now the flags have changed.
5717 */
5718
Patrick McHardyb6c40d62008-10-07 15:26:48 -07005719 if ((old_flags ^ flags) & IFF_MULTICAST)
5720 dev_change_rx_flags(dev, IFF_MULTICAST);
Patrick McHardy24023452007-07-14 18:51:31 -07005721
Patrick McHardy4417da62007-06-27 01:28:10 -07005722 dev_set_rx_mode(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005723
5724 /*
5725 * Have we downed the interface. We handle IFF_UP ourselves
5726 * according to user attempts to set it, rather than blindly
5727 * setting it.
5728 */
5729
5730 ret = 0;
Peter Pan(潘卫平)d215d102014-06-16 21:57:22 +08005731 if ((old_flags ^ flags) & IFF_UP)
Patrick McHardybd380812010-02-26 06:34:53 +00005732 ret = ((old_flags & IFF_UP) ? __dev_close : __dev_open)(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005733
Linus Torvalds1da177e2005-04-16 15:20:36 -07005734 if ((flags ^ dev->gflags) & IFF_PROMISC) {
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005735 int inc = (flags & IFF_PROMISC) ? 1 : -1;
Nicolas Dichtel991fb3f2013-09-25 12:02:45 +02005736 unsigned int old_flags = dev->flags;
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005737
Linus Torvalds1da177e2005-04-16 15:20:36 -07005738 dev->gflags ^= IFF_PROMISC;
Nicolas Dichtel991fb3f2013-09-25 12:02:45 +02005739
5740 if (__dev_set_promiscuity(dev, inc, false) >= 0)
5741 if (dev->flags != old_flags)
5742 dev_set_rx_mode(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005743 }
5744
5745 /* NOTE: order of synchronization of IFF_PROMISC and IFF_ALLMULTI
5746 is important. Some (broken) drivers set IFF_PROMISC, when
5747 IFF_ALLMULTI is requested not asking us and not reporting.
5748 */
5749 if ((flags ^ dev->gflags) & IFF_ALLMULTI) {
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005750 int inc = (flags & IFF_ALLMULTI) ? 1 : -1;
5751
Linus Torvalds1da177e2005-04-16 15:20:36 -07005752 dev->gflags ^= IFF_ALLMULTI;
Nicolas Dichtel991fb3f2013-09-25 12:02:45 +02005753 __dev_set_allmulti(dev, inc, false);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005754 }
5755
Patrick McHardybd380812010-02-26 06:34:53 +00005756 return ret;
5757}
5758
Nicolas Dichtela528c212013-09-25 12:02:44 +02005759void __dev_notify_flags(struct net_device *dev, unsigned int old_flags,
5760 unsigned int gchanges)
Patrick McHardybd380812010-02-26 06:34:53 +00005761{
5762 unsigned int changes = dev->flags ^ old_flags;
5763
Nicolas Dichtela528c212013-09-25 12:02:44 +02005764 if (gchanges)
Alexei Starovoitov7f294052013-10-23 16:02:42 -07005765 rtmsg_ifinfo(RTM_NEWLINK, dev, gchanges, GFP_ATOMIC);
Nicolas Dichtela528c212013-09-25 12:02:44 +02005766
Patrick McHardybd380812010-02-26 06:34:53 +00005767 if (changes & IFF_UP) {
5768 if (dev->flags & IFF_UP)
5769 call_netdevice_notifiers(NETDEV_UP, dev);
5770 else
5771 call_netdevice_notifiers(NETDEV_DOWN, dev);
5772 }
5773
5774 if (dev->flags & IFF_UP &&
Jiri Pirkobe9efd32013-05-28 01:30:22 +00005775 (changes & ~(IFF_UP | IFF_PROMISC | IFF_ALLMULTI | IFF_VOLATILE))) {
5776 struct netdev_notifier_change_info change_info;
5777
5778 change_info.flags_changed = changes;
5779 call_netdevice_notifiers_info(NETDEV_CHANGE, dev,
5780 &change_info.info);
5781 }
Patrick McHardybd380812010-02-26 06:34:53 +00005782}
5783
5784/**
5785 * dev_change_flags - change device settings
5786 * @dev: device
5787 * @flags: device state flags
5788 *
5789 * Change settings on device based state flags. The flags are
5790 * in the userspace exported format.
5791 */
Eric Dumazetb536db92011-11-30 21:42:26 +00005792int dev_change_flags(struct net_device *dev, unsigned int flags)
Patrick McHardybd380812010-02-26 06:34:53 +00005793{
Eric Dumazetb536db92011-11-30 21:42:26 +00005794 int ret;
Nicolas Dichtel991fb3f2013-09-25 12:02:45 +02005795 unsigned int changes, old_flags = dev->flags, old_gflags = dev->gflags;
Patrick McHardybd380812010-02-26 06:34:53 +00005796
5797 ret = __dev_change_flags(dev, flags);
5798 if (ret < 0)
5799 return ret;
5800
Nicolas Dichtel991fb3f2013-09-25 12:02:45 +02005801 changes = (old_flags ^ dev->flags) | (old_gflags ^ dev->gflags);
Nicolas Dichtela528c212013-09-25 12:02:44 +02005802 __dev_notify_flags(dev, old_flags, changes);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005803 return ret;
5804}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005805EXPORT_SYMBOL(dev_change_flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005806
Veaceslav Falico2315dc92014-01-10 16:56:25 +01005807static int __dev_set_mtu(struct net_device *dev, int new_mtu)
5808{
5809 const struct net_device_ops *ops = dev->netdev_ops;
5810
5811 if (ops->ndo_change_mtu)
5812 return ops->ndo_change_mtu(dev, new_mtu);
5813
5814 dev->mtu = new_mtu;
5815 return 0;
5816}
5817
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07005818/**
5819 * dev_set_mtu - Change maximum transfer unit
5820 * @dev: device
5821 * @new_mtu: new transfer unit
5822 *
5823 * Change the maximum transfer size of the network device.
5824 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07005825int dev_set_mtu(struct net_device *dev, int new_mtu)
5826{
Veaceslav Falico2315dc92014-01-10 16:56:25 +01005827 int err, orig_mtu;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005828
5829 if (new_mtu == dev->mtu)
5830 return 0;
5831
5832 /* MTU must be positive. */
5833 if (new_mtu < 0)
5834 return -EINVAL;
5835
5836 if (!netif_device_present(dev))
5837 return -ENODEV;
5838
Veaceslav Falico1d486bf2014-01-16 00:02:18 +01005839 err = call_netdevice_notifiers(NETDEV_PRECHANGEMTU, dev);
5840 err = notifier_to_errno(err);
5841 if (err)
5842 return err;
Stephen Hemmingerd3147742008-11-19 21:32:24 -08005843
Veaceslav Falico2315dc92014-01-10 16:56:25 +01005844 orig_mtu = dev->mtu;
5845 err = __dev_set_mtu(dev, new_mtu);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005846
Veaceslav Falico2315dc92014-01-10 16:56:25 +01005847 if (!err) {
5848 err = call_netdevice_notifiers(NETDEV_CHANGEMTU, dev);
5849 err = notifier_to_errno(err);
5850 if (err) {
5851 /* setting mtu back and notifying everyone again,
5852 * so that they have a chance to revert changes.
5853 */
5854 __dev_set_mtu(dev, orig_mtu);
5855 call_netdevice_notifiers(NETDEV_CHANGEMTU, dev);
5856 }
5857 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07005858 return err;
5859}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005860EXPORT_SYMBOL(dev_set_mtu);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005861
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07005862/**
Vlad Dogarucbda10f2011-01-13 23:38:30 +00005863 * dev_set_group - Change group this device belongs to
5864 * @dev: device
5865 * @new_group: group this device should belong to
5866 */
5867void dev_set_group(struct net_device *dev, int new_group)
5868{
5869 dev->group = new_group;
5870}
5871EXPORT_SYMBOL(dev_set_group);
5872
5873/**
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07005874 * dev_set_mac_address - Change Media Access Control Address
5875 * @dev: device
5876 * @sa: new address
5877 *
5878 * Change the hardware (MAC) address of the device
5879 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07005880int dev_set_mac_address(struct net_device *dev, struct sockaddr *sa)
5881{
Stephen Hemmingerd3147742008-11-19 21:32:24 -08005882 const struct net_device_ops *ops = dev->netdev_ops;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005883 int err;
5884
Stephen Hemmingerd3147742008-11-19 21:32:24 -08005885 if (!ops->ndo_set_mac_address)
Linus Torvalds1da177e2005-04-16 15:20:36 -07005886 return -EOPNOTSUPP;
5887 if (sa->sa_family != dev->type)
5888 return -EINVAL;
5889 if (!netif_device_present(dev))
5890 return -ENODEV;
Stephen Hemmingerd3147742008-11-19 21:32:24 -08005891 err = ops->ndo_set_mac_address(dev, sa);
Jiri Pirkof6521512013-01-01 03:30:14 +00005892 if (err)
5893 return err;
Jiri Pirkofbdeca22013-01-01 03:30:16 +00005894 dev->addr_assign_type = NET_ADDR_SET;
Jiri Pirkof6521512013-01-01 03:30:14 +00005895 call_netdevice_notifiers(NETDEV_CHANGEADDR, dev);
Theodore Ts'o7bf23572012-07-04 21:23:25 -04005896 add_device_randomness(dev->dev_addr, dev->addr_len);
Jiri Pirkof6521512013-01-01 03:30:14 +00005897 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005898}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005899EXPORT_SYMBOL(dev_set_mac_address);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005900
Jiri Pirko4bf84c32012-12-27 23:49:37 +00005901/**
5902 * dev_change_carrier - Change device carrier
5903 * @dev: device
Randy Dunlap691b3b72013-03-04 12:32:43 +00005904 * @new_carrier: new value
Jiri Pirko4bf84c32012-12-27 23:49:37 +00005905 *
5906 * Change device carrier
5907 */
5908int dev_change_carrier(struct net_device *dev, bool new_carrier)
5909{
5910 const struct net_device_ops *ops = dev->netdev_ops;
5911
5912 if (!ops->ndo_change_carrier)
5913 return -EOPNOTSUPP;
5914 if (!netif_device_present(dev))
5915 return -ENODEV;
5916 return ops->ndo_change_carrier(dev, new_carrier);
5917}
5918EXPORT_SYMBOL(dev_change_carrier);
5919
Linus Torvalds1da177e2005-04-16 15:20:36 -07005920/**
Jiri Pirko66b52b02013-07-29 18:16:49 +02005921 * dev_get_phys_port_id - Get device physical port ID
5922 * @dev: device
5923 * @ppid: port ID
5924 *
5925 * Get device physical port ID
5926 */
5927int dev_get_phys_port_id(struct net_device *dev,
Jiri Pirko02637fc2014-11-28 14:34:16 +01005928 struct netdev_phys_item_id *ppid)
Jiri Pirko66b52b02013-07-29 18:16:49 +02005929{
5930 const struct net_device_ops *ops = dev->netdev_ops;
5931
5932 if (!ops->ndo_get_phys_port_id)
5933 return -EOPNOTSUPP;
5934 return ops->ndo_get_phys_port_id(dev, ppid);
5935}
5936EXPORT_SYMBOL(dev_get_phys_port_id);
5937
5938/**
David Aherndb24a902015-03-17 20:23:15 -06005939 * dev_get_phys_port_name - Get device physical port name
5940 * @dev: device
5941 * @name: port name
5942 *
5943 * Get device physical port name
5944 */
5945int dev_get_phys_port_name(struct net_device *dev,
5946 char *name, size_t len)
5947{
5948 const struct net_device_ops *ops = dev->netdev_ops;
5949
5950 if (!ops->ndo_get_phys_port_name)
5951 return -EOPNOTSUPP;
5952 return ops->ndo_get_phys_port_name(dev, name, len);
5953}
5954EXPORT_SYMBOL(dev_get_phys_port_name);
5955
5956/**
Linus Torvalds1da177e2005-04-16 15:20:36 -07005957 * dev_new_index - allocate an ifindex
Randy Dunlapc4ea43c2007-10-12 21:17:49 -07005958 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -07005959 *
5960 * Returns a suitable unique value for a new device interface
5961 * number. The caller must hold the rtnl semaphore or the
5962 * dev_base_lock to be sure it remains unique.
5963 */
Eric W. Biederman881d9662007-09-17 11:56:21 -07005964static int dev_new_index(struct net *net)
Linus Torvalds1da177e2005-04-16 15:20:36 -07005965{
Pavel Emelyanovaa79e662012-08-08 21:53:19 +00005966 int ifindex = net->ifindex;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005967 for (;;) {
5968 if (++ifindex <= 0)
5969 ifindex = 1;
Eric W. Biederman881d9662007-09-17 11:56:21 -07005970 if (!__dev_get_by_index(net, ifindex))
Pavel Emelyanovaa79e662012-08-08 21:53:19 +00005971 return net->ifindex = ifindex;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005972 }
5973}
5974
Linus Torvalds1da177e2005-04-16 15:20:36 -07005975/* Delayed registration/unregisteration */
Denis Cheng3b5b34f2007-12-07 00:49:17 -08005976static LIST_HEAD(net_todo_list);
Cong Wang200b9162014-05-12 15:11:20 -07005977DECLARE_WAIT_QUEUE_HEAD(netdev_unregistering_wq);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005978
Stephen Hemminger6f05f622007-03-08 20:46:03 -08005979static void net_set_todo(struct net_device *dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07005980{
Linus Torvalds1da177e2005-04-16 15:20:36 -07005981 list_add_tail(&dev->todo_list, &net_todo_list);
Eric W. Biederman50624c92013-09-23 21:19:49 -07005982 dev_net(dev)->dev_unreg_count++;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005983}
5984
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005985static void rollback_registered_many(struct list_head *head)
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07005986{
Krishna Kumare93737b2009-12-08 22:26:02 +00005987 struct net_device *dev, *tmp;
Eric W. Biederman5cde2822013-10-05 19:26:05 -07005988 LIST_HEAD(close_head);
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005989
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07005990 BUG_ON(dev_boot_phase);
5991 ASSERT_RTNL();
5992
Krishna Kumare93737b2009-12-08 22:26:02 +00005993 list_for_each_entry_safe(dev, tmp, head, unreg_list) {
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005994 /* Some devices call without registering
Krishna Kumare93737b2009-12-08 22:26:02 +00005995 * for initialization unwind. Remove those
5996 * devices and proceed with the remaining.
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005997 */
5998 if (dev->reg_state == NETREG_UNINITIALIZED) {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00005999 pr_debug("unregister_netdevice: device %s/%p never was registered\n",
6000 dev->name, dev);
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07006001
Eric Dumazet9b5e3832009-10-27 07:04:19 +00006002 WARN_ON(1);
Krishna Kumare93737b2009-12-08 22:26:02 +00006003 list_del(&dev->unreg_list);
6004 continue;
Eric Dumazet9b5e3832009-10-27 07:04:19 +00006005 }
Eric Dumazet449f4542011-05-19 12:24:16 +00006006 dev->dismantle = true;
Eric Dumazet9b5e3832009-10-27 07:04:19 +00006007 BUG_ON(dev->reg_state != NETREG_REGISTERED);
Octavian Purdila44345722010-12-13 12:44:07 +00006008 }
Eric Dumazet9b5e3832009-10-27 07:04:19 +00006009
Octavian Purdila44345722010-12-13 12:44:07 +00006010 /* If device is running, close it first. */
Eric W. Biederman5cde2822013-10-05 19:26:05 -07006011 list_for_each_entry(dev, head, unreg_list)
6012 list_add_tail(&dev->close_list, &close_head);
David S. Miller99c4a262015-03-18 22:52:33 -04006013 dev_close_many(&close_head, true);
Eric Dumazet9b5e3832009-10-27 07:04:19 +00006014
Octavian Purdila44345722010-12-13 12:44:07 +00006015 list_for_each_entry(dev, head, unreg_list) {
Eric Dumazet9b5e3832009-10-27 07:04:19 +00006016 /* And unlink it from device chain. */
6017 unlist_netdevice(dev);
6018
6019 dev->reg_state = NETREG_UNREGISTERING;
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07006020 }
6021
Eric Dumazet9b5e3832009-10-27 07:04:19 +00006022 synchronize_net();
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07006023
Eric Dumazet9b5e3832009-10-27 07:04:19 +00006024 list_for_each_entry(dev, head, unreg_list) {
Mahesh Bandewar395eea62014-12-03 13:46:24 -08006025 struct sk_buff *skb = NULL;
6026
Eric Dumazet9b5e3832009-10-27 07:04:19 +00006027 /* Shutdown queueing discipline. */
6028 dev_shutdown(dev);
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07006029
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07006030
Eric Dumazet9b5e3832009-10-27 07:04:19 +00006031 /* Notify protocols, that we are about to destroy
6032 this device. They should clean all the things.
6033 */
6034 call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
6035
Mahesh Bandewar395eea62014-12-03 13:46:24 -08006036 if (!dev->rtnl_link_ops ||
6037 dev->rtnl_link_state == RTNL_LINK_INITIALIZED)
6038 skb = rtmsg_ifinfo_build_skb(RTM_DELLINK, dev, ~0U,
6039 GFP_KERNEL);
6040
Eric Dumazet9b5e3832009-10-27 07:04:19 +00006041 /*
6042 * Flush the unicast and multicast chains
6043 */
Jiri Pirkoa748ee22010-04-01 21:22:09 +00006044 dev_uc_flush(dev);
Jiri Pirko22bedad32010-04-01 21:22:57 +00006045 dev_mc_flush(dev);
Eric Dumazet9b5e3832009-10-27 07:04:19 +00006046
6047 if (dev->netdev_ops->ndo_uninit)
6048 dev->netdev_ops->ndo_uninit(dev);
6049
Mahesh Bandewar395eea62014-12-03 13:46:24 -08006050 if (skb)
6051 rtmsg_ifinfo_send(skb, dev, GFP_KERNEL);
Roopa Prabhu56bfa7e2014-05-01 11:40:30 -07006052
Jiri Pirko9ff162a2013-01-03 22:48:49 +00006053 /* Notifier chain MUST detach us all upper devices. */
6054 WARN_ON(netdev_has_any_upper_dev(dev));
Eric Dumazet9b5e3832009-10-27 07:04:19 +00006055
6056 /* Remove entries from kobject tree */
6057 netdev_unregister_kobject(dev);
Alexander Duyck024e9672013-01-10 08:57:46 +00006058#ifdef CONFIG_XPS
6059 /* Remove XPS queueing entries */
6060 netif_reset_xps_queues_gt(dev, 0);
6061#endif
Eric Dumazet9b5e3832009-10-27 07:04:19 +00006062 }
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07006063
Eric W. Biederman850a5452011-10-13 22:25:23 +00006064 synchronize_net();
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07006065
Eric W. Biedermana5ee1552009-11-29 15:45:58 +00006066 list_for_each_entry(dev, head, unreg_list)
Eric Dumazet9b5e3832009-10-27 07:04:19 +00006067 dev_put(dev);
6068}
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07006069
Eric Dumazet9b5e3832009-10-27 07:04:19 +00006070static void rollback_registered(struct net_device *dev)
6071{
6072 LIST_HEAD(single);
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07006073
Eric Dumazet9b5e3832009-10-27 07:04:19 +00006074 list_add(&dev->unreg_list, &single);
6075 rollback_registered_many(&single);
Eric Dumazetceaaec92011-02-17 22:59:19 +00006076 list_del(&single);
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07006077}
6078
Michał Mirosławc8f44af2011-11-15 15:29:55 +00006079static netdev_features_t netdev_fix_features(struct net_device *dev,
6080 netdev_features_t features)
Herbert Xub63365a2008-10-23 01:11:29 -07006081{
Michał Mirosław57422dc2011-01-22 12:14:12 +00006082 /* Fix illegal checksum combinations */
6083 if ((features & NETIF_F_HW_CSUM) &&
6084 (features & (NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM))) {
Michał Mirosław6f404e42011-05-16 15:14:21 -04006085 netdev_warn(dev, "mixed HW and IP checksum settings.\n");
Michał Mirosław57422dc2011-01-22 12:14:12 +00006086 features &= ~(NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM);
6087 }
6088
Herbert Xub63365a2008-10-23 01:11:29 -07006089 /* TSO requires that SG is present as well. */
Ben Hutchingsea2d3682011-04-12 14:38:37 +00006090 if ((features & NETIF_F_ALL_TSO) && !(features & NETIF_F_SG)) {
Michał Mirosław6f404e42011-05-16 15:14:21 -04006091 netdev_dbg(dev, "Dropping TSO features since no SG feature.\n");
Ben Hutchingsea2d3682011-04-12 14:38:37 +00006092 features &= ~NETIF_F_ALL_TSO;
Herbert Xub63365a2008-10-23 01:11:29 -07006093 }
6094
Pravin B Shelarec5f0612013-03-07 09:28:01 +00006095 if ((features & NETIF_F_TSO) && !(features & NETIF_F_HW_CSUM) &&
6096 !(features & NETIF_F_IP_CSUM)) {
6097 netdev_dbg(dev, "Dropping TSO features since no CSUM feature.\n");
6098 features &= ~NETIF_F_TSO;
6099 features &= ~NETIF_F_TSO_ECN;
6100 }
6101
6102 if ((features & NETIF_F_TSO6) && !(features & NETIF_F_HW_CSUM) &&
6103 !(features & NETIF_F_IPV6_CSUM)) {
6104 netdev_dbg(dev, "Dropping TSO6 features since no CSUM feature.\n");
6105 features &= ~NETIF_F_TSO6;
6106 }
6107
Ben Hutchings31d8b9e2011-04-12 14:47:15 +00006108 /* TSO ECN requires that TSO is present as well. */
6109 if ((features & NETIF_F_ALL_TSO) == NETIF_F_TSO_ECN)
6110 features &= ~NETIF_F_TSO_ECN;
6111
Michał Mirosław212b5732011-02-15 16:59:16 +00006112 /* Software GSO depends on SG. */
6113 if ((features & NETIF_F_GSO) && !(features & NETIF_F_SG)) {
Michał Mirosław6f404e42011-05-16 15:14:21 -04006114 netdev_dbg(dev, "Dropping NETIF_F_GSO since no SG feature.\n");
Michał Mirosław212b5732011-02-15 16:59:16 +00006115 features &= ~NETIF_F_GSO;
6116 }
6117
Michał Mirosławacd11302011-01-24 15:45:15 -08006118 /* UFO needs SG and checksumming */
Herbert Xub63365a2008-10-23 01:11:29 -07006119 if (features & NETIF_F_UFO) {
Michał Mirosław79032642010-11-30 06:38:00 +00006120 /* maybe split UFO into V4 and V6? */
6121 if (!((features & NETIF_F_GEN_CSUM) ||
6122 (features & (NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM))
6123 == (NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM))) {
Michał Mirosław6f404e42011-05-16 15:14:21 -04006124 netdev_dbg(dev,
Michał Mirosławacd11302011-01-24 15:45:15 -08006125 "Dropping NETIF_F_UFO since no checksum offload features.\n");
Herbert Xub63365a2008-10-23 01:11:29 -07006126 features &= ~NETIF_F_UFO;
6127 }
6128
6129 if (!(features & NETIF_F_SG)) {
Michał Mirosław6f404e42011-05-16 15:14:21 -04006130 netdev_dbg(dev,
Michał Mirosławacd11302011-01-24 15:45:15 -08006131 "Dropping NETIF_F_UFO since no NETIF_F_SG feature.\n");
Herbert Xub63365a2008-10-23 01:11:29 -07006132 features &= ~NETIF_F_UFO;
6133 }
6134 }
6135
Jiri Pirkod0290212014-04-02 23:09:31 +02006136#ifdef CONFIG_NET_RX_BUSY_POLL
6137 if (dev->netdev_ops->ndo_busy_poll)
6138 features |= NETIF_F_BUSY_POLL;
6139 else
6140#endif
6141 features &= ~NETIF_F_BUSY_POLL;
6142
Herbert Xub63365a2008-10-23 01:11:29 -07006143 return features;
6144}
Herbert Xub63365a2008-10-23 01:11:29 -07006145
Michał Mirosław6cb6a272011-04-02 22:48:47 -07006146int __netdev_update_features(struct net_device *dev)
Michał Mirosław5455c692011-02-15 16:59:17 +00006147{
Michał Mirosławc8f44af2011-11-15 15:29:55 +00006148 netdev_features_t features;
Michał Mirosław5455c692011-02-15 16:59:17 +00006149 int err = 0;
6150
Michał Mirosław87267482011-04-12 09:56:38 +00006151 ASSERT_RTNL();
6152
Michał Mirosław5455c692011-02-15 16:59:17 +00006153 features = netdev_get_wanted_features(dev);
6154
6155 if (dev->netdev_ops->ndo_fix_features)
6156 features = dev->netdev_ops->ndo_fix_features(dev, features);
6157
6158 /* driver might be less strict about feature dependencies */
6159 features = netdev_fix_features(dev, features);
6160
6161 if (dev->features == features)
Michał Mirosław6cb6a272011-04-02 22:48:47 -07006162 return 0;
Michał Mirosław5455c692011-02-15 16:59:17 +00006163
Michał Mirosławc8f44af2011-11-15 15:29:55 +00006164 netdev_dbg(dev, "Features changed: %pNF -> %pNF\n",
6165 &dev->features, &features);
Michał Mirosław5455c692011-02-15 16:59:17 +00006166
6167 if (dev->netdev_ops->ndo_set_features)
6168 err = dev->netdev_ops->ndo_set_features(dev, features);
6169
Michał Mirosław6cb6a272011-04-02 22:48:47 -07006170 if (unlikely(err < 0)) {
Michał Mirosław5455c692011-02-15 16:59:17 +00006171 netdev_err(dev,
Michał Mirosławc8f44af2011-11-15 15:29:55 +00006172 "set_features() failed (%d); wanted %pNF, left %pNF\n",
6173 err, &features, &dev->features);
Michał Mirosław6cb6a272011-04-02 22:48:47 -07006174 return -1;
6175 }
6176
6177 if (!err)
6178 dev->features = features;
6179
6180 return 1;
6181}
6182
Michał Mirosławafe12cc2011-05-07 03:22:17 +00006183/**
6184 * netdev_update_features - recalculate device features
6185 * @dev: the device to check
6186 *
6187 * Recalculate dev->features set and send notifications if it
6188 * has changed. Should be called after driver or hardware dependent
6189 * conditions might have changed that influence the features.
6190 */
Michał Mirosław6cb6a272011-04-02 22:48:47 -07006191void netdev_update_features(struct net_device *dev)
6192{
6193 if (__netdev_update_features(dev))
6194 netdev_features_change(dev);
Michał Mirosław5455c692011-02-15 16:59:17 +00006195}
6196EXPORT_SYMBOL(netdev_update_features);
6197
Linus Torvalds1da177e2005-04-16 15:20:36 -07006198/**
Michał Mirosławafe12cc2011-05-07 03:22:17 +00006199 * netdev_change_features - recalculate device features
6200 * @dev: the device to check
6201 *
6202 * Recalculate dev->features set and send notifications even
6203 * if they have not changed. Should be called instead of
6204 * netdev_update_features() if also dev->vlan_features might
6205 * have changed to allow the changes to be propagated to stacked
6206 * VLAN devices.
6207 */
6208void netdev_change_features(struct net_device *dev)
6209{
6210 __netdev_update_features(dev);
6211 netdev_features_change(dev);
6212}
6213EXPORT_SYMBOL(netdev_change_features);
6214
6215/**
Patrick Mullaneyfc4a7482009-12-03 15:59:22 -08006216 * netif_stacked_transfer_operstate - transfer operstate
6217 * @rootdev: the root or lower level device to transfer state from
6218 * @dev: the device to transfer operstate to
6219 *
6220 * Transfer operational state from root to device. This is normally
6221 * called when a stacking relationship exists between the root
6222 * device and the device(a leaf device).
6223 */
6224void netif_stacked_transfer_operstate(const struct net_device *rootdev,
6225 struct net_device *dev)
6226{
6227 if (rootdev->operstate == IF_OPER_DORMANT)
6228 netif_dormant_on(dev);
6229 else
6230 netif_dormant_off(dev);
6231
6232 if (netif_carrier_ok(rootdev)) {
6233 if (!netif_carrier_ok(dev))
6234 netif_carrier_on(dev);
6235 } else {
6236 if (netif_carrier_ok(dev))
6237 netif_carrier_off(dev);
6238 }
6239}
6240EXPORT_SYMBOL(netif_stacked_transfer_operstate);
6241
Michael Daltona953be52014-01-16 22:23:28 -08006242#ifdef CONFIG_SYSFS
Eric Dumazet1b4bf462010-09-23 17:26:35 +00006243static int netif_alloc_rx_queues(struct net_device *dev)
6244{
Eric Dumazet1b4bf462010-09-23 17:26:35 +00006245 unsigned int i, count = dev->num_rx_queues;
Tom Herbertbd25fa72010-10-18 18:00:16 +00006246 struct netdev_rx_queue *rx;
Pankaj Gupta10595902015-01-12 11:41:28 +05306247 size_t sz = count * sizeof(*rx);
Eric Dumazet1b4bf462010-09-23 17:26:35 +00006248
Tom Herbertbd25fa72010-10-18 18:00:16 +00006249 BUG_ON(count < 1);
Eric Dumazet1b4bf462010-09-23 17:26:35 +00006250
Pankaj Gupta10595902015-01-12 11:41:28 +05306251 rx = kzalloc(sz, GFP_KERNEL | __GFP_NOWARN | __GFP_REPEAT);
6252 if (!rx) {
6253 rx = vzalloc(sz);
6254 if (!rx)
6255 return -ENOMEM;
6256 }
Tom Herbertbd25fa72010-10-18 18:00:16 +00006257 dev->_rx = rx;
6258
Tom Herbertbd25fa72010-10-18 18:00:16 +00006259 for (i = 0; i < count; i++)
Tom Herbertfe822242010-11-09 10:47:38 +00006260 rx[i].dev = dev;
Eric Dumazet1b4bf462010-09-23 17:26:35 +00006261 return 0;
6262}
Tom Herbertbf264142010-11-26 08:36:09 +00006263#endif
Eric Dumazet1b4bf462010-09-23 17:26:35 +00006264
Changli Gaoaa942102010-12-04 02:31:41 +00006265static void netdev_init_one_queue(struct net_device *dev,
6266 struct netdev_queue *queue, void *_unused)
6267{
6268 /* Initialize queue lock */
6269 spin_lock_init(&queue->_xmit_lock);
6270 netdev_set_xmit_lockdep_class(&queue->_xmit_lock, dev->type);
6271 queue->xmit_lock_owner = -1;
Changli Gaob236da62010-12-14 03:09:15 +00006272 netdev_queue_numa_node_write(queue, NUMA_NO_NODE);
Changli Gaoaa942102010-12-04 02:31:41 +00006273 queue->dev = dev;
Tom Herbert114cf582011-11-28 16:33:09 +00006274#ifdef CONFIG_BQL
6275 dql_init(&queue->dql, HZ);
6276#endif
Changli Gaoaa942102010-12-04 02:31:41 +00006277}
6278
Eric Dumazet60877a32013-06-20 01:15:51 -07006279static void netif_free_tx_queues(struct net_device *dev)
6280{
WANG Cong4cb28972014-06-02 15:55:22 -07006281 kvfree(dev->_tx);
Eric Dumazet60877a32013-06-20 01:15:51 -07006282}
6283
Tom Herberte6484932010-10-18 18:04:39 +00006284static int netif_alloc_netdev_queues(struct net_device *dev)
6285{
6286 unsigned int count = dev->num_tx_queues;
6287 struct netdev_queue *tx;
Eric Dumazet60877a32013-06-20 01:15:51 -07006288 size_t sz = count * sizeof(*tx);
Tom Herberte6484932010-10-18 18:04:39 +00006289
Eric Dumazet60877a32013-06-20 01:15:51 -07006290 BUG_ON(count < 1 || count > 0xffff);
Tom Herberte6484932010-10-18 18:04:39 +00006291
Eric Dumazet60877a32013-06-20 01:15:51 -07006292 tx = kzalloc(sz, GFP_KERNEL | __GFP_NOWARN | __GFP_REPEAT);
6293 if (!tx) {
6294 tx = vzalloc(sz);
6295 if (!tx)
6296 return -ENOMEM;
6297 }
Tom Herberte6484932010-10-18 18:04:39 +00006298 dev->_tx = tx;
Tom Herbert1d24eb42010-11-21 13:17:27 +00006299
Tom Herberte6484932010-10-18 18:04:39 +00006300 netdev_for_each_tx_queue(dev, netdev_init_one_queue, NULL);
6301 spin_lock_init(&dev->tx_global_lock);
Changli Gaoaa942102010-12-04 02:31:41 +00006302
6303 return 0;
Tom Herberte6484932010-10-18 18:04:39 +00006304}
6305
Patrick Mullaneyfc4a7482009-12-03 15:59:22 -08006306/**
Linus Torvalds1da177e2005-04-16 15:20:36 -07006307 * register_netdevice - register a network device
6308 * @dev: device to register
6309 *
6310 * Take a completed network device structure and add it to the kernel
6311 * interfaces. A %NETDEV_REGISTER message is sent to the netdev notifier
6312 * chain. 0 is returned on success. A negative errno code is returned
6313 * on a failure to set up the device, or if the name is a duplicate.
6314 *
6315 * Callers must hold the rtnl semaphore. You may want
6316 * register_netdev() instead of this.
6317 *
6318 * BUGS:
6319 * The locking appears insufficient to guarantee two parallel registers
6320 * will not get the same name.
6321 */
6322
6323int register_netdevice(struct net_device *dev)
6324{
Linus Torvalds1da177e2005-04-16 15:20:36 -07006325 int ret;
Stephen Hemmingerd3147742008-11-19 21:32:24 -08006326 struct net *net = dev_net(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006327
6328 BUG_ON(dev_boot_phase);
6329 ASSERT_RTNL();
6330
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07006331 might_sleep();
6332
Linus Torvalds1da177e2005-04-16 15:20:36 -07006333 /* When net_device's are persistent, this will be fatal. */
6334 BUG_ON(dev->reg_state != NETREG_UNINITIALIZED);
Stephen Hemmingerd3147742008-11-19 21:32:24 -08006335 BUG_ON(!net);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006336
David S. Millerf1f28aa2008-07-15 00:08:33 -07006337 spin_lock_init(&dev->addr_list_lock);
David S. Millercf508b12008-07-22 14:16:42 -07006338 netdev_set_addr_lockdep_class(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006339
Gao feng828de4f2012-09-13 20:58:27 +00006340 ret = dev_get_valid_name(net, dev, dev->name);
Peter Pan(潘卫平)0696c3a2011-05-12 15:46:56 +00006341 if (ret < 0)
6342 goto out;
6343
Linus Torvalds1da177e2005-04-16 15:20:36 -07006344 /* Init, if this function is available */
Stephen Hemmingerd3147742008-11-19 21:32:24 -08006345 if (dev->netdev_ops->ndo_init) {
6346 ret = dev->netdev_ops->ndo_init(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006347 if (ret) {
6348 if (ret > 0)
6349 ret = -EIO;
Adrian Bunk90833aa2006-11-13 16:02:22 -08006350 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006351 }
6352 }
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09006353
Patrick McHardyf6469682013-04-19 02:04:27 +00006354 if (((dev->hw_features | dev->features) &
6355 NETIF_F_HW_VLAN_CTAG_FILTER) &&
Michał Mirosławd2ed2732013-01-29 15:14:16 +00006356 (!dev->netdev_ops->ndo_vlan_rx_add_vid ||
6357 !dev->netdev_ops->ndo_vlan_rx_kill_vid)) {
6358 netdev_WARN(dev, "Buggy VLAN acceleration in driver!\n");
6359 ret = -EINVAL;
6360 goto err_uninit;
6361 }
6362
Pavel Emelyanov9c7dafb2012-08-08 21:52:46 +00006363 ret = -EBUSY;
6364 if (!dev->ifindex)
6365 dev->ifindex = dev_new_index(net);
6366 else if (__dev_get_by_index(net, dev->ifindex))
6367 goto err_uninit;
6368
Michał Mirosław5455c692011-02-15 16:59:17 +00006369 /* Transfer changeable features to wanted_features and enable
6370 * software offloads (GSO and GRO).
6371 */
6372 dev->hw_features |= NETIF_F_SOFT_FEATURES;
Michał Mirosław14d12322011-02-22 16:52:28 +00006373 dev->features |= NETIF_F_SOFT_FEATURES;
6374 dev->wanted_features = dev->features & dev->hw_features;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006375
Michał Mirosław34324dc2011-11-15 15:29:55 +00006376 if (!(dev->flags & IFF_LOOPBACK)) {
6377 dev->hw_features |= NETIF_F_NOCACHE_COPY;
Tom Herbertc6e1a0d2011-04-04 22:30:30 -07006378 }
6379
Michał Mirosław1180e7d2011-07-14 14:41:11 -07006380 /* Make NETIF_F_HIGHDMA inheritable to VLAN devices.
Brandon Philips16c3ea72010-09-15 09:24:24 +00006381 */
Michał Mirosław1180e7d2011-07-14 14:41:11 -07006382 dev->vlan_features |= NETIF_F_HIGHDMA;
Brandon Philips16c3ea72010-09-15 09:24:24 +00006383
Pravin B Shelaree579672013-03-07 09:28:08 +00006384 /* Make NETIF_F_SG inheritable to tunnel devices.
6385 */
6386 dev->hw_enc_features |= NETIF_F_SG;
6387
Simon Horman0d89d202013-05-23 21:02:52 +00006388 /* Make NETIF_F_SG inheritable to MPLS.
6389 */
6390 dev->mpls_features |= NETIF_F_SG;
6391
Johannes Berg7ffbe3f2009-10-02 05:15:27 +00006392 ret = call_netdevice_notifiers(NETDEV_POST_INIT, dev);
6393 ret = notifier_to_errno(ret);
6394 if (ret)
6395 goto err_uninit;
6396
Eric W. Biederman8b41d182007-09-26 22:02:53 -07006397 ret = netdev_register_kobject(dev);
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07006398 if (ret)
Herbert Xu7ce1b0e2007-07-30 16:29:40 -07006399 goto err_uninit;
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07006400 dev->reg_state = NETREG_REGISTERED;
6401
Michał Mirosław6cb6a272011-04-02 22:48:47 -07006402 __netdev_update_features(dev);
Michał Mirosław8e9b59b2011-02-22 16:52:28 +00006403
Linus Torvalds1da177e2005-04-16 15:20:36 -07006404 /*
6405 * Default initial state at registry is that the
6406 * device is present.
6407 */
6408
6409 set_bit(__LINK_STATE_PRESENT, &dev->state);
6410
Ben Hutchings8f4cccb2012-08-20 22:16:51 +01006411 linkwatch_init_dev(dev);
6412
Linus Torvalds1da177e2005-04-16 15:20:36 -07006413 dev_init_scheduler(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006414 dev_hold(dev);
Eric W. Biedermance286d32007-09-12 13:53:49 +02006415 list_netdevice(dev);
Theodore Ts'o7bf23572012-07-04 21:23:25 -04006416 add_device_randomness(dev->dev_addr, dev->addr_len);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006417
Jiri Pirko948b3372013-01-08 01:38:25 +00006418 /* If the device has permanent device address, driver should
6419 * set dev_addr and also addr_assign_type should be set to
6420 * NET_ADDR_PERM (default value).
6421 */
6422 if (dev->addr_assign_type == NET_ADDR_PERM)
6423 memcpy(dev->perm_addr, dev->dev_addr, dev->addr_len);
6424
Linus Torvalds1da177e2005-04-16 15:20:36 -07006425 /* Notify protocols, that a new device appeared. */
Pavel Emelyanov056925a2007-09-16 15:42:43 -07006426 ret = call_netdevice_notifiers(NETDEV_REGISTER, dev);
Herbert Xufcc5a032007-07-30 17:03:38 -07006427 ret = notifier_to_errno(ret);
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07006428 if (ret) {
6429 rollback_registered(dev);
6430 dev->reg_state = NETREG_UNREGISTERED;
6431 }
Eric W. Biedermand90a9092009-12-12 22:11:15 +00006432 /*
6433 * Prevent userspace races by waiting until the network
6434 * device is fully setup before sending notifications.
6435 */
Patrick McHardya2835762010-02-26 06:34:51 +00006436 if (!dev->rtnl_link_ops ||
6437 dev->rtnl_link_state == RTNL_LINK_INITIALIZED)
Alexei Starovoitov7f294052013-10-23 16:02:42 -07006438 rtmsg_ifinfo(RTM_NEWLINK, dev, ~0U, GFP_KERNEL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006439
6440out:
6441 return ret;
Herbert Xu7ce1b0e2007-07-30 16:29:40 -07006442
6443err_uninit:
Stephen Hemmingerd3147742008-11-19 21:32:24 -08006444 if (dev->netdev_ops->ndo_uninit)
6445 dev->netdev_ops->ndo_uninit(dev);
Herbert Xu7ce1b0e2007-07-30 16:29:40 -07006446 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006447}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07006448EXPORT_SYMBOL(register_netdevice);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006449
6450/**
Benjamin Herrenschmidt937f1ba2009-01-14 21:05:05 -08006451 * init_dummy_netdev - init a dummy network device for NAPI
6452 * @dev: device to init
6453 *
6454 * This takes a network device structure and initialize the minimum
6455 * amount of fields so it can be used to schedule NAPI polls without
6456 * registering a full blown interface. This is to be used by drivers
6457 * that need to tie several hardware interfaces to a single NAPI
6458 * poll scheduler due to HW limitations.
6459 */
6460int init_dummy_netdev(struct net_device *dev)
6461{
6462 /* Clear everything. Note we don't initialize spinlocks
6463 * are they aren't supposed to be taken by any of the
6464 * NAPI code and this dummy netdev is supposed to be
6465 * only ever used for NAPI polls
6466 */
6467 memset(dev, 0, sizeof(struct net_device));
6468
6469 /* make sure we BUG if trying to hit standard
6470 * register/unregister code path
6471 */
6472 dev->reg_state = NETREG_DUMMY;
6473
Benjamin Herrenschmidt937f1ba2009-01-14 21:05:05 -08006474 /* NAPI wants this */
6475 INIT_LIST_HEAD(&dev->napi_list);
6476
6477 /* a dummy interface is started by default */
6478 set_bit(__LINK_STATE_PRESENT, &dev->state);
6479 set_bit(__LINK_STATE_START, &dev->state);
6480
Eric Dumazet29b44332010-10-11 10:22:12 +00006481 /* Note : We dont allocate pcpu_refcnt for dummy devices,
6482 * because users of this 'device' dont need to change
6483 * its refcount.
6484 */
6485
Benjamin Herrenschmidt937f1ba2009-01-14 21:05:05 -08006486 return 0;
6487}
6488EXPORT_SYMBOL_GPL(init_dummy_netdev);
6489
6490
6491/**
Linus Torvalds1da177e2005-04-16 15:20:36 -07006492 * register_netdev - register a network device
6493 * @dev: device to register
6494 *
6495 * Take a completed network device structure and add it to the kernel
6496 * interfaces. A %NETDEV_REGISTER message is sent to the netdev notifier
6497 * chain. 0 is returned on success. A negative errno code is returned
6498 * on a failure to set up the device, or if the name is a duplicate.
6499 *
Borislav Petkov38b4da32007-04-20 22:14:10 -07006500 * This is a wrapper around register_netdevice that takes the rtnl semaphore
Linus Torvalds1da177e2005-04-16 15:20:36 -07006501 * and expands the device name if you passed a format string to
6502 * alloc_netdev.
6503 */
6504int register_netdev(struct net_device *dev)
6505{
6506 int err;
6507
6508 rtnl_lock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07006509 err = register_netdevice(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006510 rtnl_unlock();
6511 return err;
6512}
6513EXPORT_SYMBOL(register_netdev);
6514
Eric Dumazet29b44332010-10-11 10:22:12 +00006515int netdev_refcnt_read(const struct net_device *dev)
6516{
6517 int i, refcnt = 0;
6518
6519 for_each_possible_cpu(i)
6520 refcnt += *per_cpu_ptr(dev->pcpu_refcnt, i);
6521 return refcnt;
6522}
6523EXPORT_SYMBOL(netdev_refcnt_read);
6524
Ben Hutchings2c530402012-07-10 10:55:09 +00006525/**
Linus Torvalds1da177e2005-04-16 15:20:36 -07006526 * netdev_wait_allrefs - wait until all references are gone.
Randy Dunlap3de7a372012-08-18 14:36:44 +00006527 * @dev: target net_device
Linus Torvalds1da177e2005-04-16 15:20:36 -07006528 *
6529 * This is called when unregistering network devices.
6530 *
6531 * Any protocol or device that holds a reference should register
6532 * for netdevice notification, and cleanup and put back the
6533 * reference if they receive an UNREGISTER event.
6534 * We can get stuck here if buggy protocols don't correctly
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09006535 * call dev_put.
Linus Torvalds1da177e2005-04-16 15:20:36 -07006536 */
6537static void netdev_wait_allrefs(struct net_device *dev)
6538{
6539 unsigned long rebroadcast_time, warning_time;
Eric Dumazet29b44332010-10-11 10:22:12 +00006540 int refcnt;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006541
Eric Dumazete014deb2009-11-17 05:59:21 +00006542 linkwatch_forget_dev(dev);
6543
Linus Torvalds1da177e2005-04-16 15:20:36 -07006544 rebroadcast_time = warning_time = jiffies;
Eric Dumazet29b44332010-10-11 10:22:12 +00006545 refcnt = netdev_refcnt_read(dev);
6546
6547 while (refcnt != 0) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07006548 if (time_after(jiffies, rebroadcast_time + 1 * HZ)) {
Stephen Hemminger6756ae42006-03-20 22:23:58 -08006549 rtnl_lock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07006550
6551 /* Rebroadcast unregister notification */
Pavel Emelyanov056925a2007-09-16 15:42:43 -07006552 call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006553
Eric Dumazet748e2d92012-08-22 21:50:59 +00006554 __rtnl_unlock();
Eric Dumazet0115e8e2012-08-22 17:19:46 +00006555 rcu_barrier();
Eric Dumazet748e2d92012-08-22 21:50:59 +00006556 rtnl_lock();
6557
Eric Dumazet0115e8e2012-08-22 17:19:46 +00006558 call_netdevice_notifiers(NETDEV_UNREGISTER_FINAL, dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006559 if (test_bit(__LINK_STATE_LINKWATCH_PENDING,
6560 &dev->state)) {
6561 /* We must not have linkwatch events
6562 * pending on unregister. If this
6563 * happens, we simply run the queue
6564 * unscheduled, resulting in a noop
6565 * for this device.
6566 */
6567 linkwatch_run_queue();
6568 }
6569
Stephen Hemminger6756ae42006-03-20 22:23:58 -08006570 __rtnl_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07006571
6572 rebroadcast_time = jiffies;
6573 }
6574
6575 msleep(250);
6576
Eric Dumazet29b44332010-10-11 10:22:12 +00006577 refcnt = netdev_refcnt_read(dev);
6578
Linus Torvalds1da177e2005-04-16 15:20:36 -07006579 if (time_after(jiffies, warning_time + 10 * HZ)) {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00006580 pr_emerg("unregister_netdevice: waiting for %s to become free. Usage count = %d\n",
6581 dev->name, refcnt);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006582 warning_time = jiffies;
6583 }
6584 }
6585}
6586
6587/* The sequence is:
6588 *
6589 * rtnl_lock();
6590 * ...
6591 * register_netdevice(x1);
6592 * register_netdevice(x2);
6593 * ...
6594 * unregister_netdevice(y1);
6595 * unregister_netdevice(y2);
6596 * ...
6597 * rtnl_unlock();
6598 * free_netdev(y1);
6599 * free_netdev(y2);
6600 *
Herbert Xu58ec3b42008-10-07 15:50:03 -07006601 * We are invoked by rtnl_unlock().
Linus Torvalds1da177e2005-04-16 15:20:36 -07006602 * This allows us to deal with problems:
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07006603 * 1) We can delete sysfs objects which invoke hotplug
Linus Torvalds1da177e2005-04-16 15:20:36 -07006604 * without deadlocking with linkwatch via keventd.
6605 * 2) Since we run with the RTNL semaphore not held, we can sleep
6606 * safely in order to wait for the netdev refcnt to drop to zero.
Herbert Xu58ec3b42008-10-07 15:50:03 -07006607 *
6608 * We must not return until all unregister events added during
6609 * the interval the lock was held have been completed.
Linus Torvalds1da177e2005-04-16 15:20:36 -07006610 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07006611void netdev_run_todo(void)
6612{
Oleg Nesterov626ab0e2006-06-23 02:05:55 -07006613 struct list_head list;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006614
Linus Torvalds1da177e2005-04-16 15:20:36 -07006615 /* Snapshot list, allow later requests */
Oleg Nesterov626ab0e2006-06-23 02:05:55 -07006616 list_replace_init(&net_todo_list, &list);
Herbert Xu58ec3b42008-10-07 15:50:03 -07006617
6618 __rtnl_unlock();
Oleg Nesterov626ab0e2006-06-23 02:05:55 -07006619
Eric Dumazet0115e8e2012-08-22 17:19:46 +00006620
6621 /* Wait for rcu callbacks to finish before next phase */
Eric W. Biederman850a5452011-10-13 22:25:23 +00006622 if (!list_empty(&list))
6623 rcu_barrier();
6624
Linus Torvalds1da177e2005-04-16 15:20:36 -07006625 while (!list_empty(&list)) {
6626 struct net_device *dev
stephen hemmingere5e26d72010-02-24 14:01:38 +00006627 = list_first_entry(&list, struct net_device, todo_list);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006628 list_del(&dev->todo_list);
6629
Eric Dumazet748e2d92012-08-22 21:50:59 +00006630 rtnl_lock();
Eric Dumazet0115e8e2012-08-22 17:19:46 +00006631 call_netdevice_notifiers(NETDEV_UNREGISTER_FINAL, dev);
Eric Dumazet748e2d92012-08-22 21:50:59 +00006632 __rtnl_unlock();
Eric Dumazet0115e8e2012-08-22 17:19:46 +00006633
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07006634 if (unlikely(dev->reg_state != NETREG_UNREGISTERING)) {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00006635 pr_err("network todo '%s' but state %d\n",
Linus Torvalds1da177e2005-04-16 15:20:36 -07006636 dev->name, dev->reg_state);
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07006637 dump_stack();
6638 continue;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006639 }
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07006640
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07006641 dev->reg_state = NETREG_UNREGISTERED;
6642
Changli Gao152102c2010-03-30 20:16:22 +00006643 on_each_cpu(flush_backlog, dev, 1);
Stephen Hemminger6e583ce2008-08-03 21:29:57 -07006644
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07006645 netdev_wait_allrefs(dev);
6646
6647 /* paranoia */
Eric Dumazet29b44332010-10-11 10:22:12 +00006648 BUG_ON(netdev_refcnt_read(dev));
Salam Noureddine7866a622015-01-27 11:35:48 -08006649 BUG_ON(!list_empty(&dev->ptype_all));
6650 BUG_ON(!list_empty(&dev->ptype_specific));
Eric Dumazet33d480c2011-08-11 19:30:52 +00006651 WARN_ON(rcu_access_pointer(dev->ip_ptr));
6652 WARN_ON(rcu_access_pointer(dev->ip6_ptr));
Ilpo Järvinen547b7922008-07-25 21:43:18 -07006653 WARN_ON(dev->dn_ptr);
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07006654
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07006655 if (dev->destructor)
6656 dev->destructor(dev);
Stephen Hemminger9093bbb2007-05-19 15:39:25 -07006657
Eric W. Biederman50624c92013-09-23 21:19:49 -07006658 /* Report a network device has been unregistered */
6659 rtnl_lock();
6660 dev_net(dev)->dev_unreg_count--;
6661 __rtnl_unlock();
6662 wake_up(&netdev_unregistering_wq);
6663
Stephen Hemminger9093bbb2007-05-19 15:39:25 -07006664 /* Free network device */
6665 kobject_put(&dev->dev.kobj);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006666 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07006667}
6668
Ben Hutchings3cfde792010-07-09 09:11:52 +00006669/* Convert net_device_stats to rtnl_link_stats64. They have the same
6670 * fields in the same order, with only the type differing.
6671 */
Eric Dumazet77a1abf2012-03-05 04:50:09 +00006672void netdev_stats_to_stats64(struct rtnl_link_stats64 *stats64,
6673 const struct net_device_stats *netdev_stats)
Ben Hutchings3cfde792010-07-09 09:11:52 +00006674{
6675#if BITS_PER_LONG == 64
Eric Dumazet77a1abf2012-03-05 04:50:09 +00006676 BUILD_BUG_ON(sizeof(*stats64) != sizeof(*netdev_stats));
6677 memcpy(stats64, netdev_stats, sizeof(*stats64));
Ben Hutchings3cfde792010-07-09 09:11:52 +00006678#else
6679 size_t i, n = sizeof(*stats64) / sizeof(u64);
6680 const unsigned long *src = (const unsigned long *)netdev_stats;
6681 u64 *dst = (u64 *)stats64;
6682
6683 BUILD_BUG_ON(sizeof(*netdev_stats) / sizeof(unsigned long) !=
6684 sizeof(*stats64) / sizeof(u64));
6685 for (i = 0; i < n; i++)
6686 dst[i] = src[i];
6687#endif
6688}
Eric Dumazet77a1abf2012-03-05 04:50:09 +00006689EXPORT_SYMBOL(netdev_stats_to_stats64);
Ben Hutchings3cfde792010-07-09 09:11:52 +00006690
Eric Dumazetd83345a2009-11-16 03:36:51 +00006691/**
Stephen Hemmingereeda3fd2008-11-19 21:40:23 -08006692 * dev_get_stats - get network device statistics
6693 * @dev: device to get statistics from
Eric Dumazet28172732010-07-07 14:58:56 -07006694 * @storage: place to store stats
Stephen Hemmingereeda3fd2008-11-19 21:40:23 -08006695 *
Ben Hutchingsd7753512010-07-09 09:12:41 +00006696 * Get network statistics from device. Return @storage.
6697 * The device driver may provide its own method by setting
6698 * dev->netdev_ops->get_stats64 or dev->netdev_ops->get_stats;
6699 * otherwise the internal statistics structure is used.
Stephen Hemmingereeda3fd2008-11-19 21:40:23 -08006700 */
Ben Hutchingsd7753512010-07-09 09:12:41 +00006701struct rtnl_link_stats64 *dev_get_stats(struct net_device *dev,
6702 struct rtnl_link_stats64 *storage)
Eric Dumazet7004bf22009-05-18 00:34:33 +00006703{
Stephen Hemmingereeda3fd2008-11-19 21:40:23 -08006704 const struct net_device_ops *ops = dev->netdev_ops;
6705
Eric Dumazet28172732010-07-07 14:58:56 -07006706 if (ops->ndo_get_stats64) {
6707 memset(storage, 0, sizeof(*storage));
Eric Dumazetcaf586e2010-09-30 21:06:55 +00006708 ops->ndo_get_stats64(dev, storage);
6709 } else if (ops->ndo_get_stats) {
Ben Hutchings3cfde792010-07-09 09:11:52 +00006710 netdev_stats_to_stats64(storage, ops->ndo_get_stats(dev));
Eric Dumazetcaf586e2010-09-30 21:06:55 +00006711 } else {
6712 netdev_stats_to_stats64(storage, &dev->stats);
Eric Dumazet28172732010-07-07 14:58:56 -07006713 }
Eric Dumazetcaf586e2010-09-30 21:06:55 +00006714 storage->rx_dropped += atomic_long_read(&dev->rx_dropped);
Eric Dumazet015f0682014-03-27 08:45:56 -07006715 storage->tx_dropped += atomic_long_read(&dev->tx_dropped);
Eric Dumazet28172732010-07-07 14:58:56 -07006716 return storage;
Rusty Russellc45d2862007-03-28 14:29:08 -07006717}
Stephen Hemmingereeda3fd2008-11-19 21:40:23 -08006718EXPORT_SYMBOL(dev_get_stats);
Rusty Russellc45d2862007-03-28 14:29:08 -07006719
Eric Dumazet24824a02010-10-02 06:11:55 +00006720struct netdev_queue *dev_ingress_queue_create(struct net_device *dev)
David S. Millerdc2b4842008-07-08 17:18:23 -07006721{
Eric Dumazet24824a02010-10-02 06:11:55 +00006722 struct netdev_queue *queue = dev_ingress_queue(dev);
David S. Millerdc2b4842008-07-08 17:18:23 -07006723
Eric Dumazet24824a02010-10-02 06:11:55 +00006724#ifdef CONFIG_NET_CLS_ACT
6725 if (queue)
6726 return queue;
6727 queue = kzalloc(sizeof(*queue), GFP_KERNEL);
6728 if (!queue)
6729 return NULL;
6730 netdev_init_one_queue(dev, queue, NULL);
Eric Dumazet2ce1ee12015-02-04 13:37:44 -08006731 RCU_INIT_POINTER(queue->qdisc, &noop_qdisc);
Eric Dumazet24824a02010-10-02 06:11:55 +00006732 queue->qdisc_sleeping = &noop_qdisc;
6733 rcu_assign_pointer(dev->ingress_queue, queue);
6734#endif
6735 return queue;
David S. Millerbb949fb2008-07-08 16:55:56 -07006736}
6737
Eric Dumazet2c60db02012-09-16 09:17:26 +00006738static const struct ethtool_ops default_ethtool_ops;
6739
Stanislaw Gruszkad07d7502013-01-10 23:19:10 +00006740void netdev_set_default_ethtool_ops(struct net_device *dev,
6741 const struct ethtool_ops *ops)
6742{
6743 if (dev->ethtool_ops == &default_ethtool_ops)
6744 dev->ethtool_ops = ops;
6745}
6746EXPORT_SYMBOL_GPL(netdev_set_default_ethtool_ops);
6747
Eric Dumazet74d332c2013-10-30 13:10:44 -07006748void netdev_freemem(struct net_device *dev)
6749{
6750 char *addr = (char *)dev - dev->padded;
6751
WANG Cong4cb28972014-06-02 15:55:22 -07006752 kvfree(addr);
Eric Dumazet74d332c2013-10-30 13:10:44 -07006753}
6754
Linus Torvalds1da177e2005-04-16 15:20:36 -07006755/**
Tom Herbert36909ea2011-01-09 19:36:31 +00006756 * alloc_netdev_mqs - allocate network device
Tom Gundersenc835a672014-07-14 16:37:24 +02006757 * @sizeof_priv: size of private data to allocate space for
6758 * @name: device name format string
6759 * @name_assign_type: origin of device name
6760 * @setup: callback to initialize device
6761 * @txqs: the number of TX subqueues to allocate
6762 * @rxqs: the number of RX subqueues to allocate
Linus Torvalds1da177e2005-04-16 15:20:36 -07006763 *
6764 * Allocates a struct net_device with private data area for driver use
Li Zhong90e51ad2013-11-22 15:04:46 +08006765 * and performs basic initialization. Also allocates subqueue structs
Tom Herbert36909ea2011-01-09 19:36:31 +00006766 * for each queue on the device.
Linus Torvalds1da177e2005-04-16 15:20:36 -07006767 */
Tom Herbert36909ea2011-01-09 19:36:31 +00006768struct net_device *alloc_netdev_mqs(int sizeof_priv, const char *name,
Tom Gundersenc835a672014-07-14 16:37:24 +02006769 unsigned char name_assign_type,
Tom Herbert36909ea2011-01-09 19:36:31 +00006770 void (*setup)(struct net_device *),
6771 unsigned int txqs, unsigned int rxqs)
Linus Torvalds1da177e2005-04-16 15:20:36 -07006772{
Linus Torvalds1da177e2005-04-16 15:20:36 -07006773 struct net_device *dev;
Stephen Hemminger79439862008-07-21 13:28:44 -07006774 size_t alloc_size;
Eric Dumazet1ce8e7b2009-05-27 04:42:37 +00006775 struct net_device *p;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006776
Stephen Hemmingerb6fe17d2006-08-29 17:06:13 -07006777 BUG_ON(strlen(name) >= sizeof(dev->name));
6778
Tom Herbert36909ea2011-01-09 19:36:31 +00006779 if (txqs < 1) {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00006780 pr_err("alloc_netdev: Unable to allocate device with zero queues\n");
Tom Herbert55513fb2010-10-18 17:55:58 +00006781 return NULL;
6782 }
6783
Michael Daltona953be52014-01-16 22:23:28 -08006784#ifdef CONFIG_SYSFS
Tom Herbert36909ea2011-01-09 19:36:31 +00006785 if (rxqs < 1) {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00006786 pr_err("alloc_netdev: Unable to allocate device with zero RX queues\n");
Tom Herbert36909ea2011-01-09 19:36:31 +00006787 return NULL;
6788 }
6789#endif
6790
David S. Millerfd2ea0a2008-07-17 01:56:23 -07006791 alloc_size = sizeof(struct net_device);
Alexey Dobriyand1643d22008-04-18 15:43:32 -07006792 if (sizeof_priv) {
6793 /* ensure 32-byte alignment of private area */
Eric Dumazet1ce8e7b2009-05-27 04:42:37 +00006794 alloc_size = ALIGN(alloc_size, NETDEV_ALIGN);
Alexey Dobriyand1643d22008-04-18 15:43:32 -07006795 alloc_size += sizeof_priv;
6796 }
6797 /* ensure 32-byte alignment of whole construct */
Eric Dumazet1ce8e7b2009-05-27 04:42:37 +00006798 alloc_size += NETDEV_ALIGN - 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006799
Eric Dumazet74d332c2013-10-30 13:10:44 -07006800 p = kzalloc(alloc_size, GFP_KERNEL | __GFP_NOWARN | __GFP_REPEAT);
6801 if (!p)
6802 p = vzalloc(alloc_size);
Joe Perches62b59422013-02-04 16:48:16 +00006803 if (!p)
Linus Torvalds1da177e2005-04-16 15:20:36 -07006804 return NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006805
Eric Dumazet1ce8e7b2009-05-27 04:42:37 +00006806 dev = PTR_ALIGN(p, NETDEV_ALIGN);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006807 dev->padded = (char *)dev - (char *)p;
Jiri Pirkoab9c73c2009-05-08 13:30:17 +00006808
Eric Dumazet29b44332010-10-11 10:22:12 +00006809 dev->pcpu_refcnt = alloc_percpu(int);
6810 if (!dev->pcpu_refcnt)
Eric Dumazet74d332c2013-10-30 13:10:44 -07006811 goto free_dev;
Jiri Pirkoab9c73c2009-05-08 13:30:17 +00006812
Linus Torvalds1da177e2005-04-16 15:20:36 -07006813 if (dev_addr_init(dev))
Eric Dumazet29b44332010-10-11 10:22:12 +00006814 goto free_pcpu;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006815
Jiri Pirko22bedad32010-04-01 21:22:57 +00006816 dev_mc_init(dev);
Jiri Pirkoa748ee22010-04-01 21:22:09 +00006817 dev_uc_init(dev);
Jiri Pirkoccffad252009-05-22 23:22:17 +00006818
YOSHIFUJI Hideakic346dca2008-03-25 21:47:49 +09006819 dev_net_set(dev, &init_net);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006820
Peter P Waskiewicz Jr82cc1a72008-03-21 03:43:19 -07006821 dev->gso_max_size = GSO_MAX_SIZE;
Ben Hutchings30b678d2012-07-30 15:57:00 +00006822 dev->gso_max_segs = GSO_MAX_SEGS;
Eric Dumazetfcbeb972014-10-05 10:11:27 -07006823 dev->gso_min_segs = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006824
Herbert Xud565b0a2008-12-15 23:38:52 -08006825 INIT_LIST_HEAD(&dev->napi_list);
Eric W. Biederman9fdce092009-10-30 14:51:13 +00006826 INIT_LIST_HEAD(&dev->unreg_list);
Eric W. Biederman5cde2822013-10-05 19:26:05 -07006827 INIT_LIST_HEAD(&dev->close_list);
Eric Dumazete014deb2009-11-17 05:59:21 +00006828 INIT_LIST_HEAD(&dev->link_watch_list);
Veaceslav Falico2f268f12013-09-25 09:20:07 +02006829 INIT_LIST_HEAD(&dev->adj_list.upper);
6830 INIT_LIST_HEAD(&dev->adj_list.lower);
6831 INIT_LIST_HEAD(&dev->all_adj_list.upper);
6832 INIT_LIST_HEAD(&dev->all_adj_list.lower);
Salam Noureddine7866a622015-01-27 11:35:48 -08006833 INIT_LIST_HEAD(&dev->ptype_all);
6834 INIT_LIST_HEAD(&dev->ptype_specific);
Eric Dumazet02875872014-10-05 18:38:35 -07006835 dev->priv_flags = IFF_XMIT_DST_RELEASE | IFF_XMIT_DST_RELEASE_PERM;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006836 setup(dev);
David S. Miller8d3bdbd2011-02-08 15:02:50 -08006837
6838 dev->num_tx_queues = txqs;
6839 dev->real_num_tx_queues = txqs;
6840 if (netif_alloc_netdev_queues(dev))
6841 goto free_all;
6842
Michael Daltona953be52014-01-16 22:23:28 -08006843#ifdef CONFIG_SYSFS
David S. Miller8d3bdbd2011-02-08 15:02:50 -08006844 dev->num_rx_queues = rxqs;
6845 dev->real_num_rx_queues = rxqs;
6846 if (netif_alloc_rx_queues(dev))
6847 goto free_all;
6848#endif
6849
Linus Torvalds1da177e2005-04-16 15:20:36 -07006850 strcpy(dev->name, name);
Tom Gundersenc835a672014-07-14 16:37:24 +02006851 dev->name_assign_type = name_assign_type;
Vlad Dogarucbda10f2011-01-13 23:38:30 +00006852 dev->group = INIT_NETDEV_GROUP;
Eric Dumazet2c60db02012-09-16 09:17:26 +00006853 if (!dev->ethtool_ops)
6854 dev->ethtool_ops = &default_ethtool_ops;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006855 return dev;
Jiri Pirkoab9c73c2009-05-08 13:30:17 +00006856
David S. Miller8d3bdbd2011-02-08 15:02:50 -08006857free_all:
6858 free_netdev(dev);
6859 return NULL;
6860
Eric Dumazet29b44332010-10-11 10:22:12 +00006861free_pcpu:
6862 free_percpu(dev->pcpu_refcnt);
Eric Dumazet74d332c2013-10-30 13:10:44 -07006863free_dev:
6864 netdev_freemem(dev);
Jiri Pirkoab9c73c2009-05-08 13:30:17 +00006865 return NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006866}
Tom Herbert36909ea2011-01-09 19:36:31 +00006867EXPORT_SYMBOL(alloc_netdev_mqs);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006868
6869/**
6870 * free_netdev - free network device
6871 * @dev: device
6872 *
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09006873 * This function does the last stage of destroying an allocated device
6874 * interface. The reference to the device object is released.
Linus Torvalds1da177e2005-04-16 15:20:36 -07006875 * If this is the last reference then it will be freed.
6876 */
6877void free_netdev(struct net_device *dev)
6878{
Herbert Xud565b0a2008-12-15 23:38:52 -08006879 struct napi_struct *p, *n;
6880
Eric Dumazet60877a32013-06-20 01:15:51 -07006881 netif_free_tx_queues(dev);
Michael Daltona953be52014-01-16 22:23:28 -08006882#ifdef CONFIG_SYSFS
Pankaj Gupta10595902015-01-12 11:41:28 +05306883 kvfree(dev->_rx);
Tom Herbertfe822242010-11-09 10:47:38 +00006884#endif
David S. Millere8a04642008-07-17 00:34:19 -07006885
Eric Dumazet33d480c2011-08-11 19:30:52 +00006886 kfree(rcu_dereference_protected(dev->ingress_queue, 1));
Eric Dumazet24824a02010-10-02 06:11:55 +00006887
Jiri Pirkof001fde2009-05-05 02:48:28 +00006888 /* Flush device addresses */
6889 dev_addr_flush(dev);
6890
Herbert Xud565b0a2008-12-15 23:38:52 -08006891 list_for_each_entry_safe(p, n, &dev->napi_list, dev_list)
6892 netif_napi_del(p);
6893
Eric Dumazet29b44332010-10-11 10:22:12 +00006894 free_percpu(dev->pcpu_refcnt);
6895 dev->pcpu_refcnt = NULL;
6896
Stephen Hemminger3041a062006-05-26 13:25:24 -07006897 /* Compatibility with error handling in drivers */
Linus Torvalds1da177e2005-04-16 15:20:36 -07006898 if (dev->reg_state == NETREG_UNINITIALIZED) {
Eric Dumazet74d332c2013-10-30 13:10:44 -07006899 netdev_freemem(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006900 return;
6901 }
6902
6903 BUG_ON(dev->reg_state != NETREG_UNREGISTERED);
6904 dev->reg_state = NETREG_RELEASED;
6905
Greg Kroah-Hartman43cb76d2002-04-09 12:14:34 -07006906 /* will free via device release */
6907 put_device(&dev->dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006908}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07006909EXPORT_SYMBOL(free_netdev);
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09006910
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07006911/**
6912 * synchronize_net - Synchronize with packet receive processing
6913 *
6914 * Wait for packets currently being received to be done.
6915 * Does not block later packets from starting.
6916 */
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09006917void synchronize_net(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -07006918{
6919 might_sleep();
Eric Dumazetbe3fc412011-05-23 23:07:32 +00006920 if (rtnl_is_locked())
6921 synchronize_rcu_expedited();
6922 else
6923 synchronize_rcu();
Linus Torvalds1da177e2005-04-16 15:20:36 -07006924}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07006925EXPORT_SYMBOL(synchronize_net);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006926
6927/**
Eric Dumazet44a08732009-10-27 07:03:04 +00006928 * unregister_netdevice_queue - remove device from the kernel
Linus Torvalds1da177e2005-04-16 15:20:36 -07006929 * @dev: device
Eric Dumazet44a08732009-10-27 07:03:04 +00006930 * @head: list
Jaswinder Singh Rajput6ebfbc02009-11-22 20:43:13 -08006931 *
Linus Torvalds1da177e2005-04-16 15:20:36 -07006932 * This function shuts down a device interface and removes it
Wang Chend59b54b2007-12-11 02:28:03 -08006933 * from the kernel tables.
Eric Dumazet44a08732009-10-27 07:03:04 +00006934 * If head not NULL, device is queued to be unregistered later.
Linus Torvalds1da177e2005-04-16 15:20:36 -07006935 *
6936 * Callers must hold the rtnl semaphore. You may want
6937 * unregister_netdev() instead of this.
6938 */
6939
Eric Dumazet44a08732009-10-27 07:03:04 +00006940void unregister_netdevice_queue(struct net_device *dev, struct list_head *head)
Linus Torvalds1da177e2005-04-16 15:20:36 -07006941{
Herbert Xua6620712007-12-12 19:21:56 -08006942 ASSERT_RTNL();
6943
Eric Dumazet44a08732009-10-27 07:03:04 +00006944 if (head) {
Eric W. Biederman9fdce092009-10-30 14:51:13 +00006945 list_move_tail(&dev->unreg_list, head);
Eric Dumazet44a08732009-10-27 07:03:04 +00006946 } else {
6947 rollback_registered(dev);
6948 /* Finish processing unregister after unlock */
6949 net_set_todo(dev);
6950 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07006951}
Eric Dumazet44a08732009-10-27 07:03:04 +00006952EXPORT_SYMBOL(unregister_netdevice_queue);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006953
6954/**
Eric Dumazet9b5e3832009-10-27 07:04:19 +00006955 * unregister_netdevice_many - unregister many devices
6956 * @head: list of devices
Eric Dumazet87757a92014-06-06 06:44:03 -07006957 *
6958 * Note: As most callers use a stack allocated list_head,
6959 * we force a list_del() to make sure stack wont be corrupted later.
Eric Dumazet9b5e3832009-10-27 07:04:19 +00006960 */
6961void unregister_netdevice_many(struct list_head *head)
6962{
6963 struct net_device *dev;
6964
6965 if (!list_empty(head)) {
6966 rollback_registered_many(head);
6967 list_for_each_entry(dev, head, unreg_list)
6968 net_set_todo(dev);
Eric Dumazet87757a92014-06-06 06:44:03 -07006969 list_del(head);
Eric Dumazet9b5e3832009-10-27 07:04:19 +00006970 }
6971}
Eric Dumazet63c80992009-10-27 07:06:49 +00006972EXPORT_SYMBOL(unregister_netdevice_many);
Eric Dumazet9b5e3832009-10-27 07:04:19 +00006973
6974/**
Linus Torvalds1da177e2005-04-16 15:20:36 -07006975 * unregister_netdev - remove device from the kernel
6976 * @dev: device
6977 *
6978 * This function shuts down a device interface and removes it
Wang Chend59b54b2007-12-11 02:28:03 -08006979 * from the kernel tables.
Linus Torvalds1da177e2005-04-16 15:20:36 -07006980 *
6981 * This is just a wrapper for unregister_netdevice that takes
6982 * the rtnl semaphore. In general you want to use this and not
6983 * unregister_netdevice.
6984 */
6985void unregister_netdev(struct net_device *dev)
6986{
6987 rtnl_lock();
6988 unregister_netdevice(dev);
6989 rtnl_unlock();
6990}
Linus Torvalds1da177e2005-04-16 15:20:36 -07006991EXPORT_SYMBOL(unregister_netdev);
6992
Eric W. Biedermance286d32007-09-12 13:53:49 +02006993/**
6994 * dev_change_net_namespace - move device to different nethost namespace
6995 * @dev: device
6996 * @net: network namespace
6997 * @pat: If not NULL name pattern to try if the current device name
6998 * is already taken in the destination network namespace.
6999 *
7000 * This function shuts down a device interface and moves it
7001 * to a new network namespace. On success 0 is returned, on
7002 * a failure a netagive errno code is returned.
7003 *
7004 * Callers must hold the rtnl semaphore.
7005 */
7006
7007int dev_change_net_namespace(struct net_device *dev, struct net *net, const char *pat)
7008{
Eric W. Biedermance286d32007-09-12 13:53:49 +02007009 int err;
7010
7011 ASSERT_RTNL();
7012
7013 /* Don't allow namespace local devices to be moved. */
7014 err = -EINVAL;
7015 if (dev->features & NETIF_F_NETNS_LOCAL)
7016 goto out;
7017
7018 /* Ensure the device has been registrered */
Eric W. Biedermance286d32007-09-12 13:53:49 +02007019 if (dev->reg_state != NETREG_REGISTERED)
7020 goto out;
7021
7022 /* Get out if there is nothing todo */
7023 err = 0;
YOSHIFUJI Hideaki878628f2008-03-26 03:57:35 +09007024 if (net_eq(dev_net(dev), net))
Eric W. Biedermance286d32007-09-12 13:53:49 +02007025 goto out;
7026
7027 /* Pick the destination device name, and ensure
7028 * we can use it in the destination network namespace.
7029 */
7030 err = -EEXIST;
Octavian Purdilad9031022009-11-18 02:36:59 +00007031 if (__dev_get_by_name(net, dev->name)) {
Eric W. Biedermance286d32007-09-12 13:53:49 +02007032 /* We get here if we can't use the current device name */
7033 if (!pat)
7034 goto out;
Gao feng828de4f2012-09-13 20:58:27 +00007035 if (dev_get_valid_name(net, dev, pat) < 0)
Eric W. Biedermance286d32007-09-12 13:53:49 +02007036 goto out;
7037 }
7038
7039 /*
7040 * And now a mini version of register_netdevice unregister_netdevice.
7041 */
7042
7043 /* If device is running close it first. */
Pavel Emelyanov9b772652007-10-10 02:49:09 -07007044 dev_close(dev);
Eric W. Biedermance286d32007-09-12 13:53:49 +02007045
7046 /* And unlink it from device chain */
7047 err = -ENODEV;
7048 unlist_netdevice(dev);
7049
7050 synchronize_net();
7051
7052 /* Shutdown queueing discipline. */
7053 dev_shutdown(dev);
7054
7055 /* Notify protocols, that we are about to destroy
7056 this device. They should clean all the things.
David Lamparter3b27e102010-09-17 03:22:19 +00007057
7058 Note that dev->reg_state stays at NETREG_REGISTERED.
7059 This is wanted because this way 8021q and macvlan know
7060 the device is just moving and can keep their slaves up.
Eric W. Biedermance286d32007-09-12 13:53:49 +02007061 */
7062 call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
Gao feng6549dd42012-08-23 15:36:55 +00007063 rcu_barrier();
7064 call_netdevice_notifiers(NETDEV_UNREGISTER_FINAL, dev);
Alexei Starovoitov7f294052013-10-23 16:02:42 -07007065 rtmsg_ifinfo(RTM_DELLINK, dev, ~0U, GFP_KERNEL);
Eric W. Biedermance286d32007-09-12 13:53:49 +02007066
7067 /*
7068 * Flush the unicast and multicast chains
7069 */
Jiri Pirkoa748ee22010-04-01 21:22:09 +00007070 dev_uc_flush(dev);
Jiri Pirko22bedad32010-04-01 21:22:57 +00007071 dev_mc_flush(dev);
Eric W. Biedermance286d32007-09-12 13:53:49 +02007072
Serge Hallyn4e66ae22012-12-03 16:17:12 +00007073 /* Send a netdev-removed uevent to the old namespace */
7074 kobject_uevent(&dev->dev.kobj, KOBJ_REMOVE);
Alexander Y. Fomichev4c754312014-08-25 16:26:45 +04007075 netdev_adjacent_del_links(dev);
Serge Hallyn4e66ae22012-12-03 16:17:12 +00007076
Eric W. Biedermance286d32007-09-12 13:53:49 +02007077 /* Actually switch the network namespace */
YOSHIFUJI Hideakic346dca2008-03-25 21:47:49 +09007078 dev_net_set(dev, net);
Eric W. Biedermance286d32007-09-12 13:53:49 +02007079
Eric W. Biedermance286d32007-09-12 13:53:49 +02007080 /* If there is an ifindex conflict assign a new one */
Nicolas Dichtel7a66bbc2015-04-02 17:07:09 +02007081 if (__dev_get_by_index(net, dev->ifindex))
Eric W. Biedermance286d32007-09-12 13:53:49 +02007082 dev->ifindex = dev_new_index(net);
Eric W. Biedermance286d32007-09-12 13:53:49 +02007083
Serge Hallyn4e66ae22012-12-03 16:17:12 +00007084 /* Send a netdev-add uevent to the new namespace */
7085 kobject_uevent(&dev->dev.kobj, KOBJ_ADD);
Alexander Y. Fomichev4c754312014-08-25 16:26:45 +04007086 netdev_adjacent_add_links(dev);
Serge Hallyn4e66ae22012-12-03 16:17:12 +00007087
Eric W. Biederman8b41d182007-09-26 22:02:53 -07007088 /* Fixup kobjects */
Eric W. Biedermana1b3f592010-05-04 17:36:49 -07007089 err = device_rename(&dev->dev, dev->name);
Eric W. Biederman8b41d182007-09-26 22:02:53 -07007090 WARN_ON(err);
Eric W. Biedermance286d32007-09-12 13:53:49 +02007091
7092 /* Add the device back in the hashes */
7093 list_netdevice(dev);
7094
7095 /* Notify protocols, that a new device appeared. */
7096 call_netdevice_notifiers(NETDEV_REGISTER, dev);
7097
Eric W. Biedermand90a9092009-12-12 22:11:15 +00007098 /*
7099 * Prevent userspace races by waiting until the network
7100 * device is fully setup before sending notifications.
7101 */
Alexei Starovoitov7f294052013-10-23 16:02:42 -07007102 rtmsg_ifinfo(RTM_NEWLINK, dev, ~0U, GFP_KERNEL);
Eric W. Biedermand90a9092009-12-12 22:11:15 +00007103
Eric W. Biedermance286d32007-09-12 13:53:49 +02007104 synchronize_net();
7105 err = 0;
7106out:
7107 return err;
7108}
Johannes Berg463d0182009-07-14 00:33:35 +02007109EXPORT_SYMBOL_GPL(dev_change_net_namespace);
Eric W. Biedermance286d32007-09-12 13:53:49 +02007110
Linus Torvalds1da177e2005-04-16 15:20:36 -07007111static int dev_cpu_callback(struct notifier_block *nfb,
7112 unsigned long action,
7113 void *ocpu)
7114{
7115 struct sk_buff **list_skb;
Linus Torvalds1da177e2005-04-16 15:20:36 -07007116 struct sk_buff *skb;
7117 unsigned int cpu, oldcpu = (unsigned long)ocpu;
7118 struct softnet_data *sd, *oldsd;
7119
Rafael J. Wysocki8bb78442007-05-09 02:35:10 -07007120 if (action != CPU_DEAD && action != CPU_DEAD_FROZEN)
Linus Torvalds1da177e2005-04-16 15:20:36 -07007121 return NOTIFY_OK;
7122
7123 local_irq_disable();
7124 cpu = smp_processor_id();
7125 sd = &per_cpu(softnet_data, cpu);
7126 oldsd = &per_cpu(softnet_data, oldcpu);
7127
7128 /* Find end of our completion_queue. */
7129 list_skb = &sd->completion_queue;
7130 while (*list_skb)
7131 list_skb = &(*list_skb)->next;
7132 /* Append completion queue from offline CPU. */
7133 *list_skb = oldsd->completion_queue;
7134 oldsd->completion_queue = NULL;
7135
Linus Torvalds1da177e2005-04-16 15:20:36 -07007136 /* Append output queue from offline CPU. */
Changli Gaoa9cbd582010-04-26 23:06:24 +00007137 if (oldsd->output_queue) {
7138 *sd->output_queue_tailp = oldsd->output_queue;
7139 sd->output_queue_tailp = oldsd->output_queue_tailp;
7140 oldsd->output_queue = NULL;
7141 oldsd->output_queue_tailp = &oldsd->output_queue;
7142 }
Eric Dumazetac64da02015-01-15 17:04:22 -08007143 /* Append NAPI poll list from offline CPU, with one exception :
7144 * process_backlog() must be called by cpu owning percpu backlog.
7145 * We properly handle process_queue & input_pkt_queue later.
7146 */
7147 while (!list_empty(&oldsd->poll_list)) {
7148 struct napi_struct *napi = list_first_entry(&oldsd->poll_list,
7149 struct napi_struct,
7150 poll_list);
7151
7152 list_del_init(&napi->poll_list);
7153 if (napi->poll == process_backlog)
7154 napi->state = 0;
7155 else
7156 ____napi_schedule(sd, napi);
Heiko Carstens264524d2011-06-06 20:50:03 +00007157 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07007158
7159 raise_softirq_irqoff(NET_TX_SOFTIRQ);
7160 local_irq_enable();
7161
7162 /* Process offline CPU's input_pkt_queue */
Tom Herbert76cc8b12010-05-20 18:37:59 +00007163 while ((skb = __skb_dequeue(&oldsd->process_queue))) {
Eric Dumazet91e83132015-02-05 14:58:14 -08007164 netif_rx_ni(skb);
Tom Herbert76cc8b12010-05-20 18:37:59 +00007165 input_queue_head_incr(oldsd);
7166 }
Eric Dumazetac64da02015-01-15 17:04:22 -08007167 while ((skb = skb_dequeue(&oldsd->input_pkt_queue))) {
Eric Dumazet91e83132015-02-05 14:58:14 -08007168 netif_rx_ni(skb);
Tom Herbert76cc8b12010-05-20 18:37:59 +00007169 input_queue_head_incr(oldsd);
Tom Herbertfec5e652010-04-16 16:01:27 -07007170 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07007171
7172 return NOTIFY_OK;
7173}
Linus Torvalds1da177e2005-04-16 15:20:36 -07007174
7175
Herbert Xu7f353bf2007-08-10 15:47:58 -07007176/**
Herbert Xub63365a2008-10-23 01:11:29 -07007177 * netdev_increment_features - increment feature set by one
7178 * @all: current feature set
7179 * @one: new feature set
7180 * @mask: mask feature set
Herbert Xu7f353bf2007-08-10 15:47:58 -07007181 *
7182 * Computes a new feature set after adding a device with feature set
Herbert Xub63365a2008-10-23 01:11:29 -07007183 * @one to the master device with current feature set @all. Will not
7184 * enable anything that is off in @mask. Returns the new feature set.
Herbert Xu7f353bf2007-08-10 15:47:58 -07007185 */
Michał Mirosławc8f44af2011-11-15 15:29:55 +00007186netdev_features_t netdev_increment_features(netdev_features_t all,
7187 netdev_features_t one, netdev_features_t mask)
Herbert Xu7f353bf2007-08-10 15:47:58 -07007188{
Michał Mirosław1742f182011-04-22 06:31:16 +00007189 if (mask & NETIF_F_GEN_CSUM)
7190 mask |= NETIF_F_ALL_CSUM;
7191 mask |= NETIF_F_VLAN_CHALLENGED;
7192
7193 all |= one & (NETIF_F_ONE_FOR_ALL|NETIF_F_ALL_CSUM) & mask;
7194 all &= one | ~NETIF_F_ALL_FOR_ALL;
7195
Michał Mirosław1742f182011-04-22 06:31:16 +00007196 /* If one device supports hw checksumming, set for all. */
7197 if (all & NETIF_F_GEN_CSUM)
7198 all &= ~(NETIF_F_ALL_CSUM & ~NETIF_F_GEN_CSUM);
Herbert Xu7f353bf2007-08-10 15:47:58 -07007199
7200 return all;
7201}
Herbert Xub63365a2008-10-23 01:11:29 -07007202EXPORT_SYMBOL(netdev_increment_features);
Herbert Xu7f353bf2007-08-10 15:47:58 -07007203
Baruch Siach430f03c2013-06-02 20:43:55 +00007204static struct hlist_head * __net_init netdev_create_hash(void)
Pavel Emelyanov30d97d32007-09-16 15:40:33 -07007205{
7206 int i;
7207 struct hlist_head *hash;
7208
7209 hash = kmalloc(sizeof(*hash) * NETDEV_HASHENTRIES, GFP_KERNEL);
7210 if (hash != NULL)
7211 for (i = 0; i < NETDEV_HASHENTRIES; i++)
7212 INIT_HLIST_HEAD(&hash[i]);
7213
7214 return hash;
7215}
7216
Eric W. Biederman881d9662007-09-17 11:56:21 -07007217/* Initialize per network namespace state */
Pavel Emelyanov46650792007-10-08 20:38:39 -07007218static int __net_init netdev_init(struct net *net)
Eric W. Biederman881d9662007-09-17 11:56:21 -07007219{
Rustad, Mark D734b6542012-07-18 09:06:07 +00007220 if (net != &init_net)
7221 INIT_LIST_HEAD(&net->dev_base_head);
Eric W. Biederman881d9662007-09-17 11:56:21 -07007222
Pavel Emelyanov30d97d32007-09-16 15:40:33 -07007223 net->dev_name_head = netdev_create_hash();
7224 if (net->dev_name_head == NULL)
7225 goto err_name;
Eric W. Biederman881d9662007-09-17 11:56:21 -07007226
Pavel Emelyanov30d97d32007-09-16 15:40:33 -07007227 net->dev_index_head = netdev_create_hash();
7228 if (net->dev_index_head == NULL)
7229 goto err_idx;
Eric W. Biederman881d9662007-09-17 11:56:21 -07007230
7231 return 0;
Pavel Emelyanov30d97d32007-09-16 15:40:33 -07007232
7233err_idx:
7234 kfree(net->dev_name_head);
7235err_name:
7236 return -ENOMEM;
Eric W. Biederman881d9662007-09-17 11:56:21 -07007237}
7238
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07007239/**
7240 * netdev_drivername - network driver for the device
7241 * @dev: network device
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07007242 *
7243 * Determine network driver for device.
7244 */
David S. Miller3019de12011-06-06 16:41:33 -07007245const char *netdev_drivername(const struct net_device *dev)
Arjan van de Ven6579e572008-07-21 13:31:48 -07007246{
Stephen Hemmingercf04a4c72008-09-30 02:22:14 -07007247 const struct device_driver *driver;
7248 const struct device *parent;
David S. Miller3019de12011-06-06 16:41:33 -07007249 const char *empty = "";
Arjan van de Ven6579e572008-07-21 13:31:48 -07007250
7251 parent = dev->dev.parent;
Arjan van de Ven6579e572008-07-21 13:31:48 -07007252 if (!parent)
David S. Miller3019de12011-06-06 16:41:33 -07007253 return empty;
Arjan van de Ven6579e572008-07-21 13:31:48 -07007254
7255 driver = parent->driver;
7256 if (driver && driver->name)
David S. Miller3019de12011-06-06 16:41:33 -07007257 return driver->name;
7258 return empty;
Arjan van de Ven6579e572008-07-21 13:31:48 -07007259}
7260
Joe Perches6ea754e2014-09-22 11:10:50 -07007261static void __netdev_printk(const char *level, const struct net_device *dev,
7262 struct va_format *vaf)
Joe Perches256df2f2010-06-27 01:02:35 +00007263{
Joe Perchesb004ff42012-09-12 20:12:19 -07007264 if (dev && dev->dev.parent) {
Joe Perches6ea754e2014-09-22 11:10:50 -07007265 dev_printk_emit(level[1] - '0',
7266 dev->dev.parent,
7267 "%s %s %s%s: %pV",
7268 dev_driver_string(dev->dev.parent),
7269 dev_name(dev->dev.parent),
7270 netdev_name(dev), netdev_reg_state(dev),
7271 vaf);
Joe Perchesb004ff42012-09-12 20:12:19 -07007272 } else if (dev) {
Joe Perches6ea754e2014-09-22 11:10:50 -07007273 printk("%s%s%s: %pV",
7274 level, netdev_name(dev), netdev_reg_state(dev), vaf);
Joe Perchesb004ff42012-09-12 20:12:19 -07007275 } else {
Joe Perches6ea754e2014-09-22 11:10:50 -07007276 printk("%s(NULL net_device): %pV", level, vaf);
Joe Perchesb004ff42012-09-12 20:12:19 -07007277 }
Joe Perches256df2f2010-06-27 01:02:35 +00007278}
7279
Joe Perches6ea754e2014-09-22 11:10:50 -07007280void netdev_printk(const char *level, const struct net_device *dev,
7281 const char *format, ...)
Joe Perches256df2f2010-06-27 01:02:35 +00007282{
7283 struct va_format vaf;
7284 va_list args;
Joe Perches256df2f2010-06-27 01:02:35 +00007285
7286 va_start(args, format);
7287
7288 vaf.fmt = format;
7289 vaf.va = &args;
7290
Joe Perches6ea754e2014-09-22 11:10:50 -07007291 __netdev_printk(level, dev, &vaf);
Joe Perchesb004ff42012-09-12 20:12:19 -07007292
Joe Perches256df2f2010-06-27 01:02:35 +00007293 va_end(args);
Joe Perches256df2f2010-06-27 01:02:35 +00007294}
7295EXPORT_SYMBOL(netdev_printk);
7296
7297#define define_netdev_printk_level(func, level) \
Joe Perches6ea754e2014-09-22 11:10:50 -07007298void func(const struct net_device *dev, const char *fmt, ...) \
Joe Perches256df2f2010-06-27 01:02:35 +00007299{ \
Joe Perches256df2f2010-06-27 01:02:35 +00007300 struct va_format vaf; \
7301 va_list args; \
7302 \
7303 va_start(args, fmt); \
7304 \
7305 vaf.fmt = fmt; \
7306 vaf.va = &args; \
7307 \
Joe Perches6ea754e2014-09-22 11:10:50 -07007308 __netdev_printk(level, dev, &vaf); \
Joe Perchesb004ff42012-09-12 20:12:19 -07007309 \
Joe Perches256df2f2010-06-27 01:02:35 +00007310 va_end(args); \
Joe Perches256df2f2010-06-27 01:02:35 +00007311} \
7312EXPORT_SYMBOL(func);
7313
7314define_netdev_printk_level(netdev_emerg, KERN_EMERG);
7315define_netdev_printk_level(netdev_alert, KERN_ALERT);
7316define_netdev_printk_level(netdev_crit, KERN_CRIT);
7317define_netdev_printk_level(netdev_err, KERN_ERR);
7318define_netdev_printk_level(netdev_warn, KERN_WARNING);
7319define_netdev_printk_level(netdev_notice, KERN_NOTICE);
7320define_netdev_printk_level(netdev_info, KERN_INFO);
7321
Pavel Emelyanov46650792007-10-08 20:38:39 -07007322static void __net_exit netdev_exit(struct net *net)
Eric W. Biederman881d9662007-09-17 11:56:21 -07007323{
7324 kfree(net->dev_name_head);
7325 kfree(net->dev_index_head);
7326}
7327
Denis V. Lunev022cbae2007-11-13 03:23:50 -08007328static struct pernet_operations __net_initdata netdev_net_ops = {
Eric W. Biederman881d9662007-09-17 11:56:21 -07007329 .init = netdev_init,
7330 .exit = netdev_exit,
7331};
7332
Pavel Emelyanov46650792007-10-08 20:38:39 -07007333static void __net_exit default_device_exit(struct net *net)
Eric W. Biedermance286d32007-09-12 13:53:49 +02007334{
Eric W. Biedermane008b5f2009-11-29 22:25:30 +00007335 struct net_device *dev, *aux;
Eric W. Biedermance286d32007-09-12 13:53:49 +02007336 /*
Eric W. Biedermane008b5f2009-11-29 22:25:30 +00007337 * Push all migratable network devices back to the
Eric W. Biedermance286d32007-09-12 13:53:49 +02007338 * initial network namespace
7339 */
7340 rtnl_lock();
Eric W. Biedermane008b5f2009-11-29 22:25:30 +00007341 for_each_netdev_safe(net, dev, aux) {
Eric W. Biedermance286d32007-09-12 13:53:49 +02007342 int err;
Pavel Emelyanovaca51392008-05-08 01:24:25 -07007343 char fb_name[IFNAMSIZ];
Eric W. Biedermance286d32007-09-12 13:53:49 +02007344
7345 /* Ignore unmoveable devices (i.e. loopback) */
7346 if (dev->features & NETIF_F_NETNS_LOCAL)
7347 continue;
7348
Eric W. Biedermane008b5f2009-11-29 22:25:30 +00007349 /* Leave virtual devices for the generic cleanup */
7350 if (dev->rtnl_link_ops)
7351 continue;
Eric W. Biedermand0c082c2008-11-05 15:59:38 -08007352
Lucas De Marchi25985ed2011-03-30 22:57:33 -03007353 /* Push remaining network devices to init_net */
Pavel Emelyanovaca51392008-05-08 01:24:25 -07007354 snprintf(fb_name, IFNAMSIZ, "dev%d", dev->ifindex);
7355 err = dev_change_net_namespace(dev, &init_net, fb_name);
Eric W. Biedermance286d32007-09-12 13:53:49 +02007356 if (err) {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00007357 pr_emerg("%s: failed to move %s to init_net: %d\n",
7358 __func__, dev->name, err);
Pavel Emelyanovaca51392008-05-08 01:24:25 -07007359 BUG();
Eric W. Biedermance286d32007-09-12 13:53:49 +02007360 }
7361 }
7362 rtnl_unlock();
7363}
7364
Eric W. Biederman50624c92013-09-23 21:19:49 -07007365static void __net_exit rtnl_lock_unregistering(struct list_head *net_list)
7366{
7367 /* Return with the rtnl_lock held when there are no network
7368 * devices unregistering in any network namespace in net_list.
7369 */
7370 struct net *net;
7371 bool unregistering;
Peter Zijlstraff960a72014-10-29 17:04:56 +01007372 DEFINE_WAIT_FUNC(wait, woken_wake_function);
Eric W. Biederman50624c92013-09-23 21:19:49 -07007373
Peter Zijlstraff960a72014-10-29 17:04:56 +01007374 add_wait_queue(&netdev_unregistering_wq, &wait);
Eric W. Biederman50624c92013-09-23 21:19:49 -07007375 for (;;) {
Eric W. Biederman50624c92013-09-23 21:19:49 -07007376 unregistering = false;
7377 rtnl_lock();
7378 list_for_each_entry(net, net_list, exit_list) {
7379 if (net->dev_unreg_count > 0) {
7380 unregistering = true;
7381 break;
7382 }
7383 }
7384 if (!unregistering)
7385 break;
7386 __rtnl_unlock();
Peter Zijlstraff960a72014-10-29 17:04:56 +01007387
7388 wait_woken(&wait, TASK_UNINTERRUPTIBLE, MAX_SCHEDULE_TIMEOUT);
Eric W. Biederman50624c92013-09-23 21:19:49 -07007389 }
Peter Zijlstraff960a72014-10-29 17:04:56 +01007390 remove_wait_queue(&netdev_unregistering_wq, &wait);
Eric W. Biederman50624c92013-09-23 21:19:49 -07007391}
7392
Eric W. Biederman04dc7f6b2009-12-03 02:29:04 +00007393static void __net_exit default_device_exit_batch(struct list_head *net_list)
7394{
7395 /* At exit all network devices most be removed from a network
Uwe Kleine-Königb5950762010-11-01 15:38:34 -04007396 * namespace. Do this in the reverse order of registration.
Eric W. Biederman04dc7f6b2009-12-03 02:29:04 +00007397 * Do this across as many network namespaces as possible to
7398 * improve batching efficiency.
7399 */
7400 struct net_device *dev;
7401 struct net *net;
7402 LIST_HEAD(dev_kill_list);
7403
Eric W. Biederman50624c92013-09-23 21:19:49 -07007404 /* To prevent network device cleanup code from dereferencing
7405 * loopback devices or network devices that have been freed
7406 * wait here for all pending unregistrations to complete,
7407 * before unregistring the loopback device and allowing the
7408 * network namespace be freed.
7409 *
7410 * The netdev todo list containing all network devices
7411 * unregistrations that happen in default_device_exit_batch
7412 * will run in the rtnl_unlock() at the end of
7413 * default_device_exit_batch.
7414 */
7415 rtnl_lock_unregistering(net_list);
Eric W. Biederman04dc7f6b2009-12-03 02:29:04 +00007416 list_for_each_entry(net, net_list, exit_list) {
7417 for_each_netdev_reverse(net, dev) {
Jiri Pirkob0ab2fa2014-06-26 09:58:25 +02007418 if (dev->rtnl_link_ops && dev->rtnl_link_ops->dellink)
Eric W. Biederman04dc7f6b2009-12-03 02:29:04 +00007419 dev->rtnl_link_ops->dellink(dev, &dev_kill_list);
7420 else
7421 unregister_netdevice_queue(dev, &dev_kill_list);
7422 }
7423 }
7424 unregister_netdevice_many(&dev_kill_list);
7425 rtnl_unlock();
7426}
7427
Denis V. Lunev022cbae2007-11-13 03:23:50 -08007428static struct pernet_operations __net_initdata default_device_ops = {
Eric W. Biedermance286d32007-09-12 13:53:49 +02007429 .exit = default_device_exit,
Eric W. Biederman04dc7f6b2009-12-03 02:29:04 +00007430 .exit_batch = default_device_exit_batch,
Eric W. Biedermance286d32007-09-12 13:53:49 +02007431};
7432
Linus Torvalds1da177e2005-04-16 15:20:36 -07007433/*
7434 * Initialize the DEV module. At boot time this walks the device list and
7435 * unhooks any devices that fail to initialise (normally hardware not
7436 * present) and leaves us with a valid list of present and active devices.
7437 *
7438 */
7439
7440/*
7441 * This is called single threaded during boot, so no need
7442 * to take the rtnl semaphore.
7443 */
7444static int __init net_dev_init(void)
7445{
7446 int i, rc = -ENOMEM;
7447
7448 BUG_ON(!dev_boot_phase);
7449
Linus Torvalds1da177e2005-04-16 15:20:36 -07007450 if (dev_proc_init())
7451 goto out;
7452
Eric W. Biederman8b41d182007-09-26 22:02:53 -07007453 if (netdev_kobject_init())
Linus Torvalds1da177e2005-04-16 15:20:36 -07007454 goto out;
7455
7456 INIT_LIST_HEAD(&ptype_all);
Pavel Emelyanov82d8a8672007-11-26 20:12:58 +08007457 for (i = 0; i < PTYPE_HASH_SIZE; i++)
Linus Torvalds1da177e2005-04-16 15:20:36 -07007458 INIT_LIST_HEAD(&ptype_base[i]);
7459
Vlad Yasevich62532da2012-11-15 08:49:10 +00007460 INIT_LIST_HEAD(&offload_base);
7461
Eric W. Biederman881d9662007-09-17 11:56:21 -07007462 if (register_pernet_subsys(&netdev_net_ops))
7463 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07007464
7465 /*
7466 * Initialise the packet receive queues.
7467 */
7468
KAMEZAWA Hiroyuki6f912042006-04-10 22:52:50 -07007469 for_each_possible_cpu(i) {
Eric Dumazete36fa2f2010-04-19 21:17:14 +00007470 struct softnet_data *sd = &per_cpu(softnet_data, i);
Linus Torvalds1da177e2005-04-16 15:20:36 -07007471
Eric Dumazete36fa2f2010-04-19 21:17:14 +00007472 skb_queue_head_init(&sd->input_pkt_queue);
Changli Gao6e7676c2010-04-27 15:07:33 -07007473 skb_queue_head_init(&sd->process_queue);
Eric Dumazete36fa2f2010-04-19 21:17:14 +00007474 INIT_LIST_HEAD(&sd->poll_list);
Changli Gaoa9cbd582010-04-26 23:06:24 +00007475 sd->output_queue_tailp = &sd->output_queue;
Eric Dumazetdf334542010-03-24 19:13:54 +00007476#ifdef CONFIG_RPS
Eric Dumazete36fa2f2010-04-19 21:17:14 +00007477 sd->csd.func = rps_trigger_softirq;
7478 sd->csd.info = sd;
Eric Dumazete36fa2f2010-04-19 21:17:14 +00007479 sd->cpu = i;
Tom Herbert1e94d722010-03-18 17:45:44 -07007480#endif
Tom Herbert0a9627f2010-03-16 08:03:29 +00007481
Eric Dumazete36fa2f2010-04-19 21:17:14 +00007482 sd->backlog.poll = process_backlog;
7483 sd->backlog.weight = weight_p;
Linus Torvalds1da177e2005-04-16 15:20:36 -07007484 }
7485
Linus Torvalds1da177e2005-04-16 15:20:36 -07007486 dev_boot_phase = 0;
7487
Eric W. Biederman505d4f72008-11-07 22:54:20 -08007488 /* The loopback device is special if any other network devices
7489 * is present in a network namespace the loopback device must
7490 * be present. Since we now dynamically allocate and free the
7491 * loopback device ensure this invariant is maintained by
7492 * keeping the loopback device as the first device on the
7493 * list of network devices. Ensuring the loopback devices
7494 * is the first device that appears and the last network device
7495 * that disappears.
7496 */
7497 if (register_pernet_device(&loopback_net_ops))
7498 goto out;
7499
7500 if (register_pernet_device(&default_device_ops))
7501 goto out;
7502
Carlos R. Mafra962cf362008-05-15 11:15:37 -03007503 open_softirq(NET_TX_SOFTIRQ, net_tx_action);
7504 open_softirq(NET_RX_SOFTIRQ, net_rx_action);
Linus Torvalds1da177e2005-04-16 15:20:36 -07007505
7506 hotcpu_notifier(dev_cpu_callback, 0);
7507 dst_init();
Linus Torvalds1da177e2005-04-16 15:20:36 -07007508 rc = 0;
7509out:
7510 return rc;
7511}
7512
7513subsys_initcall(net_dev_init);