blob: 65f829cfd928b3bda157b0848e13edfa5b118bae [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * NET3 Protocol independent device support routines.
3 *
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public License
6 * as published by the Free Software Foundation; either version
7 * 2 of the License, or (at your option) any later version.
8 *
9 * Derived from the non IP parts of dev.c 1.0.19
Jesper Juhl02c30a82005-05-05 16:16:16 -070010 * Authors: Ross Biro
Linus Torvalds1da177e2005-04-16 15:20:36 -070011 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
12 * Mark Evans, <evansmp@uhura.aston.ac.uk>
13 *
14 * Additional Authors:
15 * Florian la Roche <rzsfl@rz.uni-sb.de>
16 * Alan Cox <gw4pts@gw4pts.ampr.org>
17 * David Hinds <dahinds@users.sourceforge.net>
18 * Alexey Kuznetsov <kuznet@ms2.inr.ac.ru>
19 * Adam Sulmicki <adam@cfar.umd.edu>
20 * Pekka Riikonen <priikone@poesidon.pspt.fi>
21 *
22 * Changes:
23 * D.J. Barrow : Fixed bug where dev->refcnt gets set
24 * to 2 if register_netdev gets called
25 * before net_dev_init & also removed a
26 * few lines of code in the process.
27 * Alan Cox : device private ioctl copies fields back.
28 * Alan Cox : Transmit queue code does relevant
29 * stunts to keep the queue safe.
30 * Alan Cox : Fixed double lock.
31 * Alan Cox : Fixed promisc NULL pointer trap
32 * ???????? : Support the full private ioctl range
33 * Alan Cox : Moved ioctl permission check into
34 * drivers
35 * Tim Kordas : SIOCADDMULTI/SIOCDELMULTI
36 * Alan Cox : 100 backlog just doesn't cut it when
37 * you start doing multicast video 8)
38 * Alan Cox : Rewrote net_bh and list manager.
39 * Alan Cox : Fix ETH_P_ALL echoback lengths.
40 * Alan Cox : Took out transmit every packet pass
41 * Saved a few bytes in the ioctl handler
42 * Alan Cox : Network driver sets packet type before
43 * calling netif_rx. Saves a function
44 * call a packet.
45 * Alan Cox : Hashed net_bh()
46 * Richard Kooijman: Timestamp fixes.
47 * Alan Cox : Wrong field in SIOCGIFDSTADDR
48 * Alan Cox : Device lock protection.
49 * Alan Cox : Fixed nasty side effect of device close
50 * changes.
51 * Rudi Cilibrasi : Pass the right thing to
52 * set_mac_address()
53 * Dave Miller : 32bit quantity for the device lock to
54 * make it work out on a Sparc.
55 * Bjorn Ekwall : Added KERNELD hack.
56 * Alan Cox : Cleaned up the backlog initialise.
57 * Craig Metz : SIOCGIFCONF fix if space for under
58 * 1 device.
59 * Thomas Bogendoerfer : Return ENODEV for dev_open, if there
60 * is no device open function.
61 * Andi Kleen : Fix error reporting for SIOCGIFCONF
62 * Michael Chastain : Fix signed/unsigned for SIOCGIFCONF
63 * Cyrus Durgin : Cleaned for KMOD
64 * Adam Sulmicki : Bug Fix : Network Device Unload
65 * A network device unload needs to purge
66 * the backlog queue.
67 * Paul Rusty Russell : SIOCSIFNAME
68 * Pekka Riikonen : Netdev boot-time settings code
69 * Andrew Morton : Make unregister_netdevice wait
70 * indefinitely on dev->refcnt
71 * J Hadi Salim : - Backlog queue sampling
72 * - netif_rx() feedback
73 */
74
75#include <asm/uaccess.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070076#include <linux/bitops.h>
Randy Dunlap4fc268d2006-01-11 12:17:47 -080077#include <linux/capability.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070078#include <linux/cpu.h>
79#include <linux/types.h>
80#include <linux/kernel.h>
stephen hemminger08e98972009-11-10 07:20:34 +000081#include <linux/hash.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090082#include <linux/slab.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070083#include <linux/sched.h>
Arjan van de Ven4a3e2f72006-03-20 22:33:17 -080084#include <linux/mutex.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070085#include <linux/string.h>
86#include <linux/mm.h>
87#include <linux/socket.h>
88#include <linux/sockios.h>
89#include <linux/errno.h>
90#include <linux/interrupt.h>
91#include <linux/if_ether.h>
92#include <linux/netdevice.h>
93#include <linux/etherdevice.h>
Ben Hutchings0187bdf2008-06-19 16:15:47 -070094#include <linux/ethtool.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070095#include <linux/notifier.h>
96#include <linux/skbuff.h>
Eric W. Biederman457c4cb2007-09-12 12:01:34 +020097#include <net/net_namespace.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070098#include <net/sock.h>
99#include <linux/rtnetlink.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700100#include <linux/stat.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700101#include <net/dst.h>
102#include <net/pkt_sched.h>
103#include <net/checksum.h>
Arnd Bergmann44540962009-11-26 06:07:08 +0000104#include <net/xfrm.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700105#include <linux/highmem.h>
106#include <linux/init.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700107#include <linux/module.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700108#include <linux/netpoll.h>
109#include <linux/rcupdate.h>
110#include <linux/delay.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700111#include <net/iw_handler.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700112#include <asm/current.h>
Steve Grubb5bdb9882005-12-03 08:39:35 -0500113#include <linux/audit.h>
Chris Leechdb217332006-06-17 21:24:58 -0700114#include <linux/dmaengine.h>
Herbert Xuf6a78bf2006-06-22 02:57:17 -0700115#include <linux/err.h>
David S. Millerc7fa9d12006-08-15 16:34:13 -0700116#include <linux/ctype.h>
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700117#include <linux/if_arp.h>
Ben Hutchings6de329e2008-06-16 17:02:28 -0700118#include <linux/if_vlan.h>
David S. Miller8f0f2222008-07-15 03:47:03 -0700119#include <linux/ip.h>
Alexander Duyckad55dca2008-09-20 22:05:50 -0700120#include <net/ip.h>
David S. Miller8f0f2222008-07-15 03:47:03 -0700121#include <linux/ipv6.h>
122#include <linux/in.h>
David S. Millerb6b2fed2008-07-21 09:48:06 -0700123#include <linux/jhash.h>
124#include <linux/random.h>
David S. Miller9cbc1cb2009-06-15 03:02:23 -0700125#include <trace/events/napi.h>
Koki Sanagicf66ba52010-08-23 18:45:02 +0900126#include <trace/events/net.h>
Koki Sanagi07dc22e2010-08-23 18:46:12 +0900127#include <trace/events/skb.h>
FUJITA Tomonori5acbbd42010-03-30 22:35:50 +0000128#include <linux/pci.h>
Stephen Rothwellcaeda9b2010-09-16 21:39:16 -0700129#include <linux/inetdevice.h>
Ben Hutchingsc4454772011-01-19 11:03:53 +0000130#include <linux/cpu_rmap.h>
Ingo Molnarc5905af2012-02-24 08:31:31 +0100131#include <linux/static_key.h>
Eliezer Tamiraf12fa62013-06-10 11:39:41 +0300132#include <linux/hashtable.h>
Eric Dumazet60877a32013-06-20 01:15:51 -0700133#include <linux/vmalloc.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700134
Pavel Emelyanov342709e2007-10-23 21:14:45 -0700135#include "net-sysfs.h"
136
Herbert Xud565b0a2008-12-15 23:38:52 -0800137/* Instead of increasing this, you should create a hash table. */
138#define MAX_GRO_SKBS 8
139
Herbert Xu5d38a072009-01-04 16:13:40 -0800140/* This should be increased if a protocol with a bigger head is added. */
141#define GRO_MAX_HEAD (MAX_HEADER + 128)
142
Linus Torvalds1da177e2005-04-16 15:20:36 -0700143static DEFINE_SPINLOCK(ptype_lock);
Vlad Yasevich62532da2012-11-15 08:49:10 +0000144static DEFINE_SPINLOCK(offload_lock);
Cong Wang900ff8c2013-02-18 19:20:33 +0000145struct list_head ptype_base[PTYPE_HASH_SIZE] __read_mostly;
146struct list_head ptype_all __read_mostly; /* Taps */
Vlad Yasevich62532da2012-11-15 08:49:10 +0000147static struct list_head offload_base __read_mostly;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700148
Linus Torvalds1da177e2005-04-16 15:20:36 -0700149/*
Pavel Emelianov7562f872007-05-03 15:13:45 -0700150 * The @dev_base_head list is protected by @dev_base_lock and the rtnl
Linus Torvalds1da177e2005-04-16 15:20:36 -0700151 * semaphore.
152 *
Eric Dumazetc6d14c82009-11-04 05:43:23 -0800153 * Pure readers hold dev_base_lock for reading, or rcu_read_lock()
Linus Torvalds1da177e2005-04-16 15:20:36 -0700154 *
155 * Writers must hold the rtnl semaphore while they loop through the
Pavel Emelianov7562f872007-05-03 15:13:45 -0700156 * dev_base_head list, and hold dev_base_lock for writing when they do the
Linus Torvalds1da177e2005-04-16 15:20:36 -0700157 * actual updates. This allows pure readers to access the list even
158 * while a writer is preparing to update it.
159 *
160 * To put it another way, dev_base_lock is held for writing only to
161 * protect against pure readers; the rtnl semaphore provides the
162 * protection against other writers.
163 *
164 * See, for example usages, register_netdevice() and
165 * unregister_netdevice(), which must be called with the rtnl
166 * semaphore held.
167 */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700168DEFINE_RWLOCK(dev_base_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700169EXPORT_SYMBOL(dev_base_lock);
170
Eliezer Tamiraf12fa62013-06-10 11:39:41 +0300171/* protects napi_hash addition/deletion and napi_gen_id */
172static DEFINE_SPINLOCK(napi_hash_lock);
173
174static unsigned int napi_gen_id;
175static DEFINE_HASHTABLE(napi_hash, 8);
176
Thomas Gleixner18afa4b2013-07-23 16:13:17 +0200177static seqcount_t devnet_rename_seq;
Brian Haleyc91f6df2012-11-26 05:21:08 +0000178
Thomas Graf4e985ad2011-06-21 03:11:20 +0000179static inline void dev_base_seq_inc(struct net *net)
180{
181 while (++net->dev_base_seq == 0);
182}
183
Eric W. Biederman881d9662007-09-17 11:56:21 -0700184static inline struct hlist_head *dev_name_hash(struct net *net, const char *name)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700185{
Eric Dumazet95c96172012-04-15 05:58:06 +0000186 unsigned int hash = full_name_hash(name, strnlen(name, IFNAMSIZ));
187
stephen hemminger08e98972009-11-10 07:20:34 +0000188 return &net->dev_name_head[hash_32(hash, NETDEV_HASHBITS)];
Linus Torvalds1da177e2005-04-16 15:20:36 -0700189}
190
Eric W. Biederman881d9662007-09-17 11:56:21 -0700191static inline struct hlist_head *dev_index_hash(struct net *net, int ifindex)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700192{
Eric Dumazet7c28bd02009-10-24 06:13:17 -0700193 return &net->dev_index_head[ifindex & (NETDEV_HASHENTRIES - 1)];
Linus Torvalds1da177e2005-04-16 15:20:36 -0700194}
195
Eric Dumazete36fa2f2010-04-19 21:17:14 +0000196static inline void rps_lock(struct softnet_data *sd)
Changli Gao152102c2010-03-30 20:16:22 +0000197{
198#ifdef CONFIG_RPS
Eric Dumazete36fa2f2010-04-19 21:17:14 +0000199 spin_lock(&sd->input_pkt_queue.lock);
Changli Gao152102c2010-03-30 20:16:22 +0000200#endif
201}
202
Eric Dumazete36fa2f2010-04-19 21:17:14 +0000203static inline void rps_unlock(struct softnet_data *sd)
Changli Gao152102c2010-03-30 20:16:22 +0000204{
205#ifdef CONFIG_RPS
Eric Dumazete36fa2f2010-04-19 21:17:14 +0000206 spin_unlock(&sd->input_pkt_queue.lock);
Changli Gao152102c2010-03-30 20:16:22 +0000207#endif
208}
209
Eric W. Biedermance286d32007-09-12 13:53:49 +0200210/* Device list insertion */
dingtianhong53759be2013-04-17 22:17:50 +0000211static void list_netdevice(struct net_device *dev)
Eric W. Biedermance286d32007-09-12 13:53:49 +0200212{
YOSHIFUJI Hideakic346dca2008-03-25 21:47:49 +0900213 struct net *net = dev_net(dev);
Eric W. Biedermance286d32007-09-12 13:53:49 +0200214
215 ASSERT_RTNL();
216
217 write_lock_bh(&dev_base_lock);
Eric Dumazetc6d14c82009-11-04 05:43:23 -0800218 list_add_tail_rcu(&dev->dev_list, &net->dev_base_head);
Eric Dumazet72c95282009-10-30 07:11:27 +0000219 hlist_add_head_rcu(&dev->name_hlist, dev_name_hash(net, dev->name));
Eric Dumazetfb699dfd2009-10-19 19:18:49 +0000220 hlist_add_head_rcu(&dev->index_hlist,
221 dev_index_hash(net, dev->ifindex));
Eric W. Biedermance286d32007-09-12 13:53:49 +0200222 write_unlock_bh(&dev_base_lock);
Thomas Graf4e985ad2011-06-21 03:11:20 +0000223
224 dev_base_seq_inc(net);
Eric W. Biedermance286d32007-09-12 13:53:49 +0200225}
226
Eric Dumazetfb699dfd2009-10-19 19:18:49 +0000227/* Device list removal
228 * caller must respect a RCU grace period before freeing/reusing dev
229 */
Eric W. Biedermance286d32007-09-12 13:53:49 +0200230static void unlist_netdevice(struct net_device *dev)
231{
232 ASSERT_RTNL();
233
234 /* Unlink dev from the device chain */
235 write_lock_bh(&dev_base_lock);
Eric Dumazetc6d14c82009-11-04 05:43:23 -0800236 list_del_rcu(&dev->dev_list);
Eric Dumazet72c95282009-10-30 07:11:27 +0000237 hlist_del_rcu(&dev->name_hlist);
Eric Dumazetfb699dfd2009-10-19 19:18:49 +0000238 hlist_del_rcu(&dev->index_hlist);
Eric W. Biedermance286d32007-09-12 13:53:49 +0200239 write_unlock_bh(&dev_base_lock);
Thomas Graf4e985ad2011-06-21 03:11:20 +0000240
241 dev_base_seq_inc(dev_net(dev));
Eric W. Biedermance286d32007-09-12 13:53:49 +0200242}
243
Linus Torvalds1da177e2005-04-16 15:20:36 -0700244/*
245 * Our notifier list
246 */
247
Alan Sternf07d5b92006-05-09 15:23:03 -0700248static RAW_NOTIFIER_HEAD(netdev_chain);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700249
250/*
251 * Device drivers call our routines to queue packets here. We empty the
252 * queue in the local softnet handler.
253 */
Stephen Hemmingerbea33482007-10-03 16:41:36 -0700254
Eric Dumazet9958da02010-04-17 04:17:02 +0000255DEFINE_PER_CPU_ALIGNED(struct softnet_data, softnet_data);
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700256EXPORT_PER_CPU_SYMBOL(softnet_data);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700257
David S. Millercf508b12008-07-22 14:16:42 -0700258#ifdef CONFIG_LOCKDEP
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700259/*
David S. Millerc773e842008-07-08 23:13:53 -0700260 * register_netdevice() inits txq->_xmit_lock and sets lockdep class
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700261 * according to dev->type
262 */
263static const unsigned short netdev_lock_type[] =
264 {ARPHRD_NETROM, ARPHRD_ETHER, ARPHRD_EETHER, ARPHRD_AX25,
265 ARPHRD_PRONET, ARPHRD_CHAOS, ARPHRD_IEEE802, ARPHRD_ARCNET,
266 ARPHRD_APPLETLK, ARPHRD_DLCI, ARPHRD_ATM, ARPHRD_METRICOM,
267 ARPHRD_IEEE1394, ARPHRD_EUI64, ARPHRD_INFINIBAND, ARPHRD_SLIP,
268 ARPHRD_CSLIP, ARPHRD_SLIP6, ARPHRD_CSLIP6, ARPHRD_RSRVD,
269 ARPHRD_ADAPT, ARPHRD_ROSE, ARPHRD_X25, ARPHRD_HWX25,
270 ARPHRD_PPP, ARPHRD_CISCO, ARPHRD_LAPB, ARPHRD_DDCMP,
271 ARPHRD_RAWHDLC, ARPHRD_TUNNEL, ARPHRD_TUNNEL6, ARPHRD_FRAD,
272 ARPHRD_SKIP, ARPHRD_LOOPBACK, ARPHRD_LOCALTLK, ARPHRD_FDDI,
273 ARPHRD_BIF, ARPHRD_SIT, ARPHRD_IPDDP, ARPHRD_IPGRE,
274 ARPHRD_PIMREG, ARPHRD_HIPPI, ARPHRD_ASH, ARPHRD_ECONET,
275 ARPHRD_IRDA, ARPHRD_FCPP, ARPHRD_FCAL, ARPHRD_FCPL,
Paul Gortmaker211ed862012-05-10 17:14:35 -0400276 ARPHRD_FCFABRIC, ARPHRD_IEEE80211, ARPHRD_IEEE80211_PRISM,
277 ARPHRD_IEEE80211_RADIOTAP, ARPHRD_PHONET, ARPHRD_PHONET_PIPE,
278 ARPHRD_IEEE802154, ARPHRD_VOID, ARPHRD_NONE};
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700279
Jan Engelhardt36cbd3d2009-08-05 10:42:58 -0700280static const char *const netdev_lock_name[] =
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700281 {"_xmit_NETROM", "_xmit_ETHER", "_xmit_EETHER", "_xmit_AX25",
282 "_xmit_PRONET", "_xmit_CHAOS", "_xmit_IEEE802", "_xmit_ARCNET",
283 "_xmit_APPLETLK", "_xmit_DLCI", "_xmit_ATM", "_xmit_METRICOM",
284 "_xmit_IEEE1394", "_xmit_EUI64", "_xmit_INFINIBAND", "_xmit_SLIP",
285 "_xmit_CSLIP", "_xmit_SLIP6", "_xmit_CSLIP6", "_xmit_RSRVD",
286 "_xmit_ADAPT", "_xmit_ROSE", "_xmit_X25", "_xmit_HWX25",
287 "_xmit_PPP", "_xmit_CISCO", "_xmit_LAPB", "_xmit_DDCMP",
288 "_xmit_RAWHDLC", "_xmit_TUNNEL", "_xmit_TUNNEL6", "_xmit_FRAD",
289 "_xmit_SKIP", "_xmit_LOOPBACK", "_xmit_LOCALTLK", "_xmit_FDDI",
290 "_xmit_BIF", "_xmit_SIT", "_xmit_IPDDP", "_xmit_IPGRE",
291 "_xmit_PIMREG", "_xmit_HIPPI", "_xmit_ASH", "_xmit_ECONET",
292 "_xmit_IRDA", "_xmit_FCPP", "_xmit_FCAL", "_xmit_FCPL",
Paul Gortmaker211ed862012-05-10 17:14:35 -0400293 "_xmit_FCFABRIC", "_xmit_IEEE80211", "_xmit_IEEE80211_PRISM",
294 "_xmit_IEEE80211_RADIOTAP", "_xmit_PHONET", "_xmit_PHONET_PIPE",
295 "_xmit_IEEE802154", "_xmit_VOID", "_xmit_NONE"};
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700296
297static struct lock_class_key netdev_xmit_lock_key[ARRAY_SIZE(netdev_lock_type)];
David S. Millercf508b12008-07-22 14:16:42 -0700298static struct lock_class_key netdev_addr_lock_key[ARRAY_SIZE(netdev_lock_type)];
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700299
300static inline unsigned short netdev_lock_pos(unsigned short dev_type)
301{
302 int i;
303
304 for (i = 0; i < ARRAY_SIZE(netdev_lock_type); i++)
305 if (netdev_lock_type[i] == dev_type)
306 return i;
307 /* the last key is used by default */
308 return ARRAY_SIZE(netdev_lock_type) - 1;
309}
310
David S. Millercf508b12008-07-22 14:16:42 -0700311static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock,
312 unsigned short dev_type)
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700313{
314 int i;
315
316 i = netdev_lock_pos(dev_type);
317 lockdep_set_class_and_name(lock, &netdev_xmit_lock_key[i],
318 netdev_lock_name[i]);
319}
David S. Millercf508b12008-07-22 14:16:42 -0700320
321static inline void netdev_set_addr_lockdep_class(struct net_device *dev)
322{
323 int i;
324
325 i = netdev_lock_pos(dev->type);
326 lockdep_set_class_and_name(&dev->addr_list_lock,
327 &netdev_addr_lock_key[i],
328 netdev_lock_name[i]);
329}
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700330#else
David S. Millercf508b12008-07-22 14:16:42 -0700331static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock,
332 unsigned short dev_type)
333{
334}
335static inline void netdev_set_addr_lockdep_class(struct net_device *dev)
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700336{
337}
338#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -0700339
340/*******************************************************************************
341
342 Protocol management and registration routines
343
344*******************************************************************************/
345
346/*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700347 * Add a protocol ID to the list. Now that the input handler is
348 * smarter we can dispense with all the messy stuff that used to be
349 * here.
350 *
351 * BEWARE!!! Protocol handlers, mangling input packets,
352 * MUST BE last in hash buckets and checking protocol handlers
353 * MUST start from promiscuous ptype_all chain in net_bh.
354 * It is true now, do not change it.
355 * Explanation follows: if protocol handler, mangling packet, will
356 * be the first on list, it is not able to sense, that packet
357 * is cloned and should be copied-on-write, so that it will
358 * change it and subsequent readers will get broken packet.
359 * --ANK (980803)
360 */
361
Eric Dumazetc07b68e2010-09-02 03:53:46 +0000362static inline struct list_head *ptype_head(const struct packet_type *pt)
363{
364 if (pt->type == htons(ETH_P_ALL))
365 return &ptype_all;
366 else
367 return &ptype_base[ntohs(pt->type) & PTYPE_HASH_MASK];
368}
369
Linus Torvalds1da177e2005-04-16 15:20:36 -0700370/**
371 * dev_add_pack - add packet handler
372 * @pt: packet type declaration
373 *
374 * Add a protocol handler to the networking stack. The passed &packet_type
375 * is linked into kernel lists and may not be freed until it has been
376 * removed from the kernel lists.
377 *
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +0900378 * This call does not sleep therefore it can not
Linus Torvalds1da177e2005-04-16 15:20:36 -0700379 * guarantee all CPU's that are in middle of receiving packets
380 * will see the new packet type (until the next received packet).
381 */
382
383void dev_add_pack(struct packet_type *pt)
384{
Eric Dumazetc07b68e2010-09-02 03:53:46 +0000385 struct list_head *head = ptype_head(pt);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700386
Eric Dumazetc07b68e2010-09-02 03:53:46 +0000387 spin_lock(&ptype_lock);
388 list_add_rcu(&pt->list, head);
389 spin_unlock(&ptype_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700390}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700391EXPORT_SYMBOL(dev_add_pack);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700392
Linus Torvalds1da177e2005-04-16 15:20:36 -0700393/**
394 * __dev_remove_pack - remove packet handler
395 * @pt: packet type declaration
396 *
397 * Remove a protocol handler that was previously added to the kernel
398 * protocol handlers by dev_add_pack(). The passed &packet_type is removed
399 * from the kernel lists and can be freed or reused once this function
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +0900400 * returns.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700401 *
402 * The packet type might still be in use by receivers
403 * and must not be freed until after all the CPU's have gone
404 * through a quiescent state.
405 */
406void __dev_remove_pack(struct packet_type *pt)
407{
Eric Dumazetc07b68e2010-09-02 03:53:46 +0000408 struct list_head *head = ptype_head(pt);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700409 struct packet_type *pt1;
410
Eric Dumazetc07b68e2010-09-02 03:53:46 +0000411 spin_lock(&ptype_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700412
413 list_for_each_entry(pt1, head, list) {
414 if (pt == pt1) {
415 list_del_rcu(&pt->list);
416 goto out;
417 }
418 }
419
Joe Perches7b6cd1c2012-02-01 10:54:43 +0000420 pr_warn("dev_remove_pack: %p not found\n", pt);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700421out:
Eric Dumazetc07b68e2010-09-02 03:53:46 +0000422 spin_unlock(&ptype_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700423}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700424EXPORT_SYMBOL(__dev_remove_pack);
425
Linus Torvalds1da177e2005-04-16 15:20:36 -0700426/**
427 * dev_remove_pack - remove packet handler
428 * @pt: packet type declaration
429 *
430 * Remove a protocol handler that was previously added to the kernel
431 * protocol handlers by dev_add_pack(). The passed &packet_type is removed
432 * from the kernel lists and can be freed or reused once this function
433 * returns.
434 *
435 * This call sleeps to guarantee that no CPU is looking at the packet
436 * type after return.
437 */
438void dev_remove_pack(struct packet_type *pt)
439{
440 __dev_remove_pack(pt);
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +0900441
Linus Torvalds1da177e2005-04-16 15:20:36 -0700442 synchronize_net();
443}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700444EXPORT_SYMBOL(dev_remove_pack);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700445
Vlad Yasevich62532da2012-11-15 08:49:10 +0000446
447/**
448 * dev_add_offload - register offload handlers
449 * @po: protocol offload declaration
450 *
451 * Add protocol offload handlers to the networking stack. The passed
452 * &proto_offload is linked into kernel lists and may not be freed until
453 * it has been removed from the kernel lists.
454 *
455 * This call does not sleep therefore it can not
456 * guarantee all CPU's that are in middle of receiving packets
457 * will see the new offload handlers (until the next received packet).
458 */
459void dev_add_offload(struct packet_offload *po)
460{
461 struct list_head *head = &offload_base;
462
463 spin_lock(&offload_lock);
464 list_add_rcu(&po->list, head);
465 spin_unlock(&offload_lock);
466}
467EXPORT_SYMBOL(dev_add_offload);
468
469/**
470 * __dev_remove_offload - remove offload handler
471 * @po: packet offload declaration
472 *
473 * Remove a protocol offload handler that was previously added to the
474 * kernel offload handlers by dev_add_offload(). The passed &offload_type
475 * is removed from the kernel lists and can be freed or reused once this
476 * function returns.
477 *
478 * The packet type might still be in use by receivers
479 * and must not be freed until after all the CPU's have gone
480 * through a quiescent state.
481 */
482void __dev_remove_offload(struct packet_offload *po)
483{
484 struct list_head *head = &offload_base;
485 struct packet_offload *po1;
486
Eric Dumazetc53aa502012-11-16 08:08:23 +0000487 spin_lock(&offload_lock);
Vlad Yasevich62532da2012-11-15 08:49:10 +0000488
489 list_for_each_entry(po1, head, list) {
490 if (po == po1) {
491 list_del_rcu(&po->list);
492 goto out;
493 }
494 }
495
496 pr_warn("dev_remove_offload: %p not found\n", po);
497out:
Eric Dumazetc53aa502012-11-16 08:08:23 +0000498 spin_unlock(&offload_lock);
Vlad Yasevich62532da2012-11-15 08:49:10 +0000499}
500EXPORT_SYMBOL(__dev_remove_offload);
501
502/**
503 * dev_remove_offload - remove packet offload handler
504 * @po: packet offload declaration
505 *
506 * Remove a packet offload handler that was previously added to the kernel
507 * offload handlers by dev_add_offload(). The passed &offload_type is
508 * removed from the kernel lists and can be freed or reused once this
509 * function returns.
510 *
511 * This call sleeps to guarantee that no CPU is looking at the packet
512 * type after return.
513 */
514void dev_remove_offload(struct packet_offload *po)
515{
516 __dev_remove_offload(po);
517
518 synchronize_net();
519}
520EXPORT_SYMBOL(dev_remove_offload);
521
Linus Torvalds1da177e2005-04-16 15:20:36 -0700522/******************************************************************************
523
524 Device Boot-time Settings Routines
525
526*******************************************************************************/
527
528/* Boot time configuration table */
529static struct netdev_boot_setup dev_boot_setup[NETDEV_BOOT_SETUP_MAX];
530
531/**
532 * netdev_boot_setup_add - add new setup entry
533 * @name: name of the device
534 * @map: configured settings for the device
535 *
536 * Adds new setup entry to the dev_boot_setup list. The function
537 * returns 0 on error and 1 on success. This is a generic routine to
538 * all netdevices.
539 */
540static int netdev_boot_setup_add(char *name, struct ifmap *map)
541{
542 struct netdev_boot_setup *s;
543 int i;
544
545 s = dev_boot_setup;
546 for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++) {
547 if (s[i].name[0] == '\0' || s[i].name[0] == ' ') {
548 memset(s[i].name, 0, sizeof(s[i].name));
Wang Chen93b3cff2008-07-01 19:57:19 -0700549 strlcpy(s[i].name, name, IFNAMSIZ);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700550 memcpy(&s[i].map, map, sizeof(s[i].map));
551 break;
552 }
553 }
554
555 return i >= NETDEV_BOOT_SETUP_MAX ? 0 : 1;
556}
557
558/**
559 * netdev_boot_setup_check - check boot time settings
560 * @dev: the netdevice
561 *
562 * Check boot time settings for the device.
563 * The found settings are set for the device to be used
564 * later in the device probing.
565 * Returns 0 if no settings found, 1 if they are.
566 */
567int netdev_boot_setup_check(struct net_device *dev)
568{
569 struct netdev_boot_setup *s = dev_boot_setup;
570 int i;
571
572 for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++) {
573 if (s[i].name[0] != '\0' && s[i].name[0] != ' ' &&
Wang Chen93b3cff2008-07-01 19:57:19 -0700574 !strcmp(dev->name, s[i].name)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700575 dev->irq = s[i].map.irq;
576 dev->base_addr = s[i].map.base_addr;
577 dev->mem_start = s[i].map.mem_start;
578 dev->mem_end = s[i].map.mem_end;
579 return 1;
580 }
581 }
582 return 0;
583}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700584EXPORT_SYMBOL(netdev_boot_setup_check);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700585
586
587/**
588 * netdev_boot_base - get address from boot time settings
589 * @prefix: prefix for network device
590 * @unit: id for network device
591 *
592 * Check boot time settings for the base address of device.
593 * The found settings are set for the device to be used
594 * later in the device probing.
595 * Returns 0 if no settings found.
596 */
597unsigned long netdev_boot_base(const char *prefix, int unit)
598{
599 const struct netdev_boot_setup *s = dev_boot_setup;
600 char name[IFNAMSIZ];
601 int i;
602
603 sprintf(name, "%s%d", prefix, unit);
604
605 /*
606 * If device already registered then return base of 1
607 * to indicate not to probe for this interface
608 */
Eric W. Biederman881d9662007-09-17 11:56:21 -0700609 if (__dev_get_by_name(&init_net, name))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700610 return 1;
611
612 for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++)
613 if (!strcmp(name, s[i].name))
614 return s[i].map.base_addr;
615 return 0;
616}
617
618/*
619 * Saves at boot time configured settings for any netdevice.
620 */
621int __init netdev_boot_setup(char *str)
622{
623 int ints[5];
624 struct ifmap map;
625
626 str = get_options(str, ARRAY_SIZE(ints), ints);
627 if (!str || !*str)
628 return 0;
629
630 /* Save settings */
631 memset(&map, 0, sizeof(map));
632 if (ints[0] > 0)
633 map.irq = ints[1];
634 if (ints[0] > 1)
635 map.base_addr = ints[2];
636 if (ints[0] > 2)
637 map.mem_start = ints[3];
638 if (ints[0] > 3)
639 map.mem_end = ints[4];
640
641 /* Add new entry to the list */
642 return netdev_boot_setup_add(str, &map);
643}
644
645__setup("netdev=", netdev_boot_setup);
646
647/*******************************************************************************
648
649 Device Interface Subroutines
650
651*******************************************************************************/
652
653/**
654 * __dev_get_by_name - find a device by its name
Randy Dunlapc4ea43c2007-10-12 21:17:49 -0700655 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -0700656 * @name: name to find
657 *
658 * Find an interface by name. Must be called under RTNL semaphore
659 * or @dev_base_lock. If the name is found a pointer to the device
660 * is returned. If the name is not found then %NULL is returned. The
661 * reference counters are not incremented so the caller must be
662 * careful with locks.
663 */
664
Eric W. Biederman881d9662007-09-17 11:56:21 -0700665struct net_device *__dev_get_by_name(struct net *net, const char *name)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700666{
Eric Dumazet0bd8d532009-10-30 01:40:11 -0700667 struct net_device *dev;
668 struct hlist_head *head = dev_name_hash(net, name);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700669
Sasha Levinb67bfe02013-02-27 17:06:00 -0800670 hlist_for_each_entry(dev, head, name_hlist)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700671 if (!strncmp(dev->name, name, IFNAMSIZ))
672 return dev;
Eric Dumazet0bd8d532009-10-30 01:40:11 -0700673
Linus Torvalds1da177e2005-04-16 15:20:36 -0700674 return NULL;
675}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700676EXPORT_SYMBOL(__dev_get_by_name);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700677
678/**
Eric Dumazet72c95282009-10-30 07:11:27 +0000679 * dev_get_by_name_rcu - find a device by its name
680 * @net: the applicable net namespace
681 * @name: name to find
682 *
683 * Find an interface by name.
684 * If the name is found a pointer to the device is returned.
685 * If the name is not found then %NULL is returned.
686 * The reference counters are not incremented so the caller must be
687 * careful with locks. The caller must hold RCU lock.
688 */
689
690struct net_device *dev_get_by_name_rcu(struct net *net, const char *name)
691{
Eric Dumazet72c95282009-10-30 07:11:27 +0000692 struct net_device *dev;
693 struct hlist_head *head = dev_name_hash(net, name);
694
Sasha Levinb67bfe02013-02-27 17:06:00 -0800695 hlist_for_each_entry_rcu(dev, head, name_hlist)
Eric Dumazet72c95282009-10-30 07:11:27 +0000696 if (!strncmp(dev->name, name, IFNAMSIZ))
697 return dev;
698
699 return NULL;
700}
701EXPORT_SYMBOL(dev_get_by_name_rcu);
702
703/**
Linus Torvalds1da177e2005-04-16 15:20:36 -0700704 * dev_get_by_name - find a device by its name
Randy Dunlapc4ea43c2007-10-12 21:17:49 -0700705 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -0700706 * @name: name to find
707 *
708 * Find an interface by name. This can be called from any
709 * context and does its own locking. The returned handle has
710 * the usage count incremented and the caller must use dev_put() to
711 * release it when it is no longer needed. %NULL is returned if no
712 * matching device is found.
713 */
714
Eric W. Biederman881d9662007-09-17 11:56:21 -0700715struct net_device *dev_get_by_name(struct net *net, const char *name)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700716{
717 struct net_device *dev;
718
Eric Dumazet72c95282009-10-30 07:11:27 +0000719 rcu_read_lock();
720 dev = dev_get_by_name_rcu(net, name);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700721 if (dev)
722 dev_hold(dev);
Eric Dumazet72c95282009-10-30 07:11:27 +0000723 rcu_read_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700724 return dev;
725}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700726EXPORT_SYMBOL(dev_get_by_name);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700727
728/**
729 * __dev_get_by_index - find a device by its ifindex
Randy Dunlapc4ea43c2007-10-12 21:17:49 -0700730 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -0700731 * @ifindex: index of device
732 *
733 * Search for an interface by index. Returns %NULL if the device
734 * is not found or a pointer to the device. The device has not
735 * had its reference counter increased so the caller must be careful
736 * about locking. The caller must hold either the RTNL semaphore
737 * or @dev_base_lock.
738 */
739
Eric W. Biederman881d9662007-09-17 11:56:21 -0700740struct net_device *__dev_get_by_index(struct net *net, int ifindex)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700741{
Eric Dumazet0bd8d532009-10-30 01:40:11 -0700742 struct net_device *dev;
743 struct hlist_head *head = dev_index_hash(net, ifindex);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700744
Sasha Levinb67bfe02013-02-27 17:06:00 -0800745 hlist_for_each_entry(dev, head, index_hlist)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700746 if (dev->ifindex == ifindex)
747 return dev;
Eric Dumazet0bd8d532009-10-30 01:40:11 -0700748
Linus Torvalds1da177e2005-04-16 15:20:36 -0700749 return NULL;
750}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700751EXPORT_SYMBOL(__dev_get_by_index);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700752
Eric Dumazetfb699dfd2009-10-19 19:18:49 +0000753/**
754 * dev_get_by_index_rcu - find a device by its ifindex
755 * @net: the applicable net namespace
756 * @ifindex: index of device
757 *
758 * Search for an interface by index. Returns %NULL if the device
759 * is not found or a pointer to the device. The device has not
760 * had its reference counter increased so the caller must be careful
761 * about locking. The caller must hold RCU lock.
762 */
763
764struct net_device *dev_get_by_index_rcu(struct net *net, int ifindex)
765{
Eric Dumazetfb699dfd2009-10-19 19:18:49 +0000766 struct net_device *dev;
767 struct hlist_head *head = dev_index_hash(net, ifindex);
768
Sasha Levinb67bfe02013-02-27 17:06:00 -0800769 hlist_for_each_entry_rcu(dev, head, index_hlist)
Eric Dumazetfb699dfd2009-10-19 19:18:49 +0000770 if (dev->ifindex == ifindex)
771 return dev;
772
773 return NULL;
774}
775EXPORT_SYMBOL(dev_get_by_index_rcu);
776
Linus Torvalds1da177e2005-04-16 15:20:36 -0700777
778/**
779 * dev_get_by_index - find a device by its ifindex
Randy Dunlapc4ea43c2007-10-12 21:17:49 -0700780 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -0700781 * @ifindex: index of device
782 *
783 * Search for an interface by index. Returns NULL if the device
784 * is not found or a pointer to the device. The device returned has
785 * had a reference added and the pointer is safe until the user calls
786 * dev_put to indicate they have finished with it.
787 */
788
Eric W. Biederman881d9662007-09-17 11:56:21 -0700789struct net_device *dev_get_by_index(struct net *net, int ifindex)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700790{
791 struct net_device *dev;
792
Eric Dumazetfb699dfd2009-10-19 19:18:49 +0000793 rcu_read_lock();
794 dev = dev_get_by_index_rcu(net, ifindex);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700795 if (dev)
796 dev_hold(dev);
Eric Dumazetfb699dfd2009-10-19 19:18:49 +0000797 rcu_read_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700798 return dev;
799}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700800EXPORT_SYMBOL(dev_get_by_index);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700801
802/**
Nicolas Schichan5dbe7c12013-06-26 17:23:42 +0200803 * netdev_get_name - get a netdevice name, knowing its ifindex.
804 * @net: network namespace
805 * @name: a pointer to the buffer where the name will be stored.
806 * @ifindex: the ifindex of the interface to get the name from.
807 *
808 * The use of raw_seqcount_begin() and cond_resched() before
809 * retrying is required as we want to give the writers a chance
810 * to complete when CONFIG_PREEMPT is not set.
811 */
812int netdev_get_name(struct net *net, char *name, int ifindex)
813{
814 struct net_device *dev;
815 unsigned int seq;
816
817retry:
818 seq = raw_seqcount_begin(&devnet_rename_seq);
819 rcu_read_lock();
820 dev = dev_get_by_index_rcu(net, ifindex);
821 if (!dev) {
822 rcu_read_unlock();
823 return -ENODEV;
824 }
825
826 strcpy(name, dev->name);
827 rcu_read_unlock();
828 if (read_seqcount_retry(&devnet_rename_seq, seq)) {
829 cond_resched();
830 goto retry;
831 }
832
833 return 0;
834}
835
836/**
Eric Dumazet941666c2010-12-05 01:23:53 +0000837 * dev_getbyhwaddr_rcu - find a device by its hardware address
Randy Dunlapc4ea43c2007-10-12 21:17:49 -0700838 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -0700839 * @type: media type of device
840 * @ha: hardware address
841 *
842 * Search for an interface by MAC address. Returns NULL if the device
Eric Dumazetc5066532011-01-24 13:16:16 -0800843 * is not found or a pointer to the device.
844 * The caller must hold RCU or RTNL.
Eric Dumazet941666c2010-12-05 01:23:53 +0000845 * The returned device has not had its ref count increased
Linus Torvalds1da177e2005-04-16 15:20:36 -0700846 * and the caller must therefore be careful about locking
847 *
Linus Torvalds1da177e2005-04-16 15:20:36 -0700848 */
849
Eric Dumazet941666c2010-12-05 01:23:53 +0000850struct net_device *dev_getbyhwaddr_rcu(struct net *net, unsigned short type,
851 const char *ha)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700852{
853 struct net_device *dev;
854
Eric Dumazet941666c2010-12-05 01:23:53 +0000855 for_each_netdev_rcu(net, dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700856 if (dev->type == type &&
857 !memcmp(dev->dev_addr, ha, dev->addr_len))
Pavel Emelianov7562f872007-05-03 15:13:45 -0700858 return dev;
859
860 return NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700861}
Eric Dumazet941666c2010-12-05 01:23:53 +0000862EXPORT_SYMBOL(dev_getbyhwaddr_rcu);
Jochen Friedrichcf309e32005-09-22 04:44:55 -0300863
Eric W. Biederman881d9662007-09-17 11:56:21 -0700864struct net_device *__dev_getfirstbyhwtype(struct net *net, unsigned short type)
Patrick McHardy4e9cac22007-05-03 03:28:13 -0700865{
866 struct net_device *dev;
867
868 ASSERT_RTNL();
Eric W. Biederman881d9662007-09-17 11:56:21 -0700869 for_each_netdev(net, dev)
Patrick McHardy4e9cac22007-05-03 03:28:13 -0700870 if (dev->type == type)
Pavel Emelianov7562f872007-05-03 15:13:45 -0700871 return dev;
872
873 return NULL;
Patrick McHardy4e9cac22007-05-03 03:28:13 -0700874}
Patrick McHardy4e9cac22007-05-03 03:28:13 -0700875EXPORT_SYMBOL(__dev_getfirstbyhwtype);
876
Eric W. Biederman881d9662007-09-17 11:56:21 -0700877struct net_device *dev_getfirstbyhwtype(struct net *net, unsigned short type)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700878{
Eric Dumazet99fe3c32010-03-18 11:27:25 +0000879 struct net_device *dev, *ret = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700880
Eric Dumazet99fe3c32010-03-18 11:27:25 +0000881 rcu_read_lock();
882 for_each_netdev_rcu(net, dev)
883 if (dev->type == type) {
884 dev_hold(dev);
885 ret = dev;
886 break;
887 }
888 rcu_read_unlock();
889 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700890}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700891EXPORT_SYMBOL(dev_getfirstbyhwtype);
892
893/**
Eric Dumazetbb69ae02010-06-07 11:42:13 +0000894 * dev_get_by_flags_rcu - find any device with given flags
Randy Dunlapc4ea43c2007-10-12 21:17:49 -0700895 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -0700896 * @if_flags: IFF_* values
897 * @mask: bitmask of bits in if_flags to check
898 *
899 * Search for any interface with the given flags. Returns NULL if a device
Eric Dumazetbb69ae02010-06-07 11:42:13 +0000900 * is not found or a pointer to the device. Must be called inside
901 * rcu_read_lock(), and result refcount is unchanged.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700902 */
903
Eric Dumazetbb69ae02010-06-07 11:42:13 +0000904struct net_device *dev_get_by_flags_rcu(struct net *net, unsigned short if_flags,
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700905 unsigned short mask)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700906{
Pavel Emelianov7562f872007-05-03 15:13:45 -0700907 struct net_device *dev, *ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700908
Pavel Emelianov7562f872007-05-03 15:13:45 -0700909 ret = NULL;
Eric Dumazetc6d14c82009-11-04 05:43:23 -0800910 for_each_netdev_rcu(net, dev) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700911 if (((dev->flags ^ if_flags) & mask) == 0) {
Pavel Emelianov7562f872007-05-03 15:13:45 -0700912 ret = dev;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700913 break;
914 }
915 }
Pavel Emelianov7562f872007-05-03 15:13:45 -0700916 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700917}
Eric Dumazetbb69ae02010-06-07 11:42:13 +0000918EXPORT_SYMBOL(dev_get_by_flags_rcu);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700919
920/**
921 * dev_valid_name - check if name is okay for network device
922 * @name: name string
923 *
924 * Network device names need to be valid file names to
David S. Millerc7fa9d12006-08-15 16:34:13 -0700925 * to allow sysfs to work. We also disallow any kind of
926 * whitespace.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700927 */
David S. Miller95f050b2012-03-06 16:12:15 -0500928bool dev_valid_name(const char *name)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700929{
David S. Millerc7fa9d12006-08-15 16:34:13 -0700930 if (*name == '\0')
David S. Miller95f050b2012-03-06 16:12:15 -0500931 return false;
Stephen Hemmingerb6fe17d2006-08-29 17:06:13 -0700932 if (strlen(name) >= IFNAMSIZ)
David S. Miller95f050b2012-03-06 16:12:15 -0500933 return false;
David S. Millerc7fa9d12006-08-15 16:34:13 -0700934 if (!strcmp(name, ".") || !strcmp(name, ".."))
David S. Miller95f050b2012-03-06 16:12:15 -0500935 return false;
David S. Millerc7fa9d12006-08-15 16:34:13 -0700936
937 while (*name) {
938 if (*name == '/' || isspace(*name))
David S. Miller95f050b2012-03-06 16:12:15 -0500939 return false;
David S. Millerc7fa9d12006-08-15 16:34:13 -0700940 name++;
941 }
David S. Miller95f050b2012-03-06 16:12:15 -0500942 return true;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700943}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700944EXPORT_SYMBOL(dev_valid_name);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700945
946/**
Eric W. Biedermanb267b172007-09-12 13:48:45 +0200947 * __dev_alloc_name - allocate a name for a device
948 * @net: network namespace to allocate the device name in
Linus Torvalds1da177e2005-04-16 15:20:36 -0700949 * @name: name format string
Eric W. Biedermanb267b172007-09-12 13:48:45 +0200950 * @buf: scratch buffer and result name string
Linus Torvalds1da177e2005-04-16 15:20:36 -0700951 *
952 * Passed a format string - eg "lt%d" it will try and find a suitable
Stephen Hemminger3041a062006-05-26 13:25:24 -0700953 * id. It scans list of devices to build up a free map, then chooses
954 * the first empty slot. The caller must hold the dev_base or rtnl lock
955 * while allocating the name and adding the device in order to avoid
956 * duplicates.
957 * Limited to bits_per_byte * page size devices (ie 32K on most platforms).
958 * Returns the number of the unit assigned or a negative errno code.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700959 */
960
Eric W. Biedermanb267b172007-09-12 13:48:45 +0200961static int __dev_alloc_name(struct net *net, const char *name, char *buf)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700962{
963 int i = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700964 const char *p;
965 const int max_netdevices = 8*PAGE_SIZE;
Stephen Hemmingercfcabdc2007-10-09 01:59:42 -0700966 unsigned long *inuse;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700967 struct net_device *d;
968
969 p = strnchr(name, IFNAMSIZ-1, '%');
970 if (p) {
971 /*
972 * Verify the string as this thing may have come from
973 * the user. There must be either one "%d" and no other "%"
974 * characters.
975 */
976 if (p[1] != 'd' || strchr(p + 2, '%'))
977 return -EINVAL;
978
979 /* Use one page as a bit array of possible slots */
Stephen Hemmingercfcabdc2007-10-09 01:59:42 -0700980 inuse = (unsigned long *) get_zeroed_page(GFP_ATOMIC);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700981 if (!inuse)
982 return -ENOMEM;
983
Eric W. Biederman881d9662007-09-17 11:56:21 -0700984 for_each_netdev(net, d) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700985 if (!sscanf(d->name, name, &i))
986 continue;
987 if (i < 0 || i >= max_netdevices)
988 continue;
989
990 /* avoid cases where sscanf is not exact inverse of printf */
Eric W. Biedermanb267b172007-09-12 13:48:45 +0200991 snprintf(buf, IFNAMSIZ, name, i);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700992 if (!strncmp(buf, d->name, IFNAMSIZ))
993 set_bit(i, inuse);
994 }
995
996 i = find_first_zero_bit(inuse, max_netdevices);
997 free_page((unsigned long) inuse);
998 }
999
Octavian Purdilad9031022009-11-18 02:36:59 +00001000 if (buf != name)
1001 snprintf(buf, IFNAMSIZ, name, i);
Eric W. Biedermanb267b172007-09-12 13:48:45 +02001002 if (!__dev_get_by_name(net, buf))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001003 return i;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001004
1005 /* It is possible to run out of possible slots
1006 * when the name is long and there isn't enough space left
1007 * for the digits, or if all bits are used.
1008 */
1009 return -ENFILE;
1010}
1011
Eric W. Biedermanb267b172007-09-12 13:48:45 +02001012/**
1013 * dev_alloc_name - allocate a name for a device
1014 * @dev: device
1015 * @name: name format string
1016 *
1017 * Passed a format string - eg "lt%d" it will try and find a suitable
1018 * id. It scans list of devices to build up a free map, then chooses
1019 * the first empty slot. The caller must hold the dev_base or rtnl lock
1020 * while allocating the name and adding the device in order to avoid
1021 * duplicates.
1022 * Limited to bits_per_byte * page size devices (ie 32K on most platforms).
1023 * Returns the number of the unit assigned or a negative errno code.
1024 */
1025
1026int dev_alloc_name(struct net_device *dev, const char *name)
1027{
1028 char buf[IFNAMSIZ];
1029 struct net *net;
1030 int ret;
1031
YOSHIFUJI Hideakic346dca2008-03-25 21:47:49 +09001032 BUG_ON(!dev_net(dev));
1033 net = dev_net(dev);
Eric W. Biedermanb267b172007-09-12 13:48:45 +02001034 ret = __dev_alloc_name(net, name, buf);
1035 if (ret >= 0)
1036 strlcpy(dev->name, buf, IFNAMSIZ);
1037 return ret;
1038}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001039EXPORT_SYMBOL(dev_alloc_name);
Eric W. Biedermanb267b172007-09-12 13:48:45 +02001040
Gao feng828de4f2012-09-13 20:58:27 +00001041static int dev_alloc_name_ns(struct net *net,
1042 struct net_device *dev,
1043 const char *name)
Octavian Purdilad9031022009-11-18 02:36:59 +00001044{
Gao feng828de4f2012-09-13 20:58:27 +00001045 char buf[IFNAMSIZ];
1046 int ret;
Daniel Lezcano8ce6cebc2010-05-19 10:12:19 +00001047
Gao feng828de4f2012-09-13 20:58:27 +00001048 ret = __dev_alloc_name(net, name, buf);
1049 if (ret >= 0)
1050 strlcpy(dev->name, buf, IFNAMSIZ);
1051 return ret;
1052}
1053
1054static int dev_get_valid_name(struct net *net,
1055 struct net_device *dev,
1056 const char *name)
1057{
1058 BUG_ON(!net);
Daniel Lezcano8ce6cebc2010-05-19 10:12:19 +00001059
Octavian Purdilad9031022009-11-18 02:36:59 +00001060 if (!dev_valid_name(name))
1061 return -EINVAL;
1062
Jiri Pirko1c5cae82011-04-30 01:21:32 +00001063 if (strchr(name, '%'))
Gao feng828de4f2012-09-13 20:58:27 +00001064 return dev_alloc_name_ns(net, dev, name);
Octavian Purdilad9031022009-11-18 02:36:59 +00001065 else if (__dev_get_by_name(net, name))
1066 return -EEXIST;
Daniel Lezcano8ce6cebc2010-05-19 10:12:19 +00001067 else if (dev->name != name)
1068 strlcpy(dev->name, name, IFNAMSIZ);
Octavian Purdilad9031022009-11-18 02:36:59 +00001069
1070 return 0;
1071}
Linus Torvalds1da177e2005-04-16 15:20:36 -07001072
1073/**
1074 * dev_change_name - change name of a device
1075 * @dev: device
1076 * @newname: name (or format string) must be at least IFNAMSIZ
1077 *
1078 * Change name of a device, can pass format strings "eth%d".
1079 * for wildcarding.
1080 */
Stephen Hemmingercf04a4c72008-09-30 02:22:14 -07001081int dev_change_name(struct net_device *dev, const char *newname)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001082{
Herbert Xufcc5a032007-07-30 17:03:38 -07001083 char oldname[IFNAMSIZ];
Linus Torvalds1da177e2005-04-16 15:20:36 -07001084 int err = 0;
Herbert Xufcc5a032007-07-30 17:03:38 -07001085 int ret;
Eric W. Biederman881d9662007-09-17 11:56:21 -07001086 struct net *net;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001087
1088 ASSERT_RTNL();
YOSHIFUJI Hideakic346dca2008-03-25 21:47:49 +09001089 BUG_ON(!dev_net(dev));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001090
YOSHIFUJI Hideakic346dca2008-03-25 21:47:49 +09001091 net = dev_net(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001092 if (dev->flags & IFF_UP)
1093 return -EBUSY;
1094
Eric Dumazet30e6c9f2012-12-20 17:25:08 +00001095 write_seqcount_begin(&devnet_rename_seq);
Brian Haleyc91f6df2012-11-26 05:21:08 +00001096
1097 if (strncmp(newname, dev->name, IFNAMSIZ) == 0) {
Eric Dumazet30e6c9f2012-12-20 17:25:08 +00001098 write_seqcount_end(&devnet_rename_seq);
Stephen Hemmingerc8d90dc2007-10-26 03:53:42 -07001099 return 0;
Brian Haleyc91f6df2012-11-26 05:21:08 +00001100 }
Stephen Hemmingerc8d90dc2007-10-26 03:53:42 -07001101
Herbert Xufcc5a032007-07-30 17:03:38 -07001102 memcpy(oldname, dev->name, IFNAMSIZ);
1103
Gao feng828de4f2012-09-13 20:58:27 +00001104 err = dev_get_valid_name(net, dev, newname);
Brian Haleyc91f6df2012-11-26 05:21:08 +00001105 if (err < 0) {
Eric Dumazet30e6c9f2012-12-20 17:25:08 +00001106 write_seqcount_end(&devnet_rename_seq);
Octavian Purdilad9031022009-11-18 02:36:59 +00001107 return err;
Brian Haleyc91f6df2012-11-26 05:21:08 +00001108 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001109
Herbert Xufcc5a032007-07-30 17:03:38 -07001110rollback:
Eric W. Biedermana1b3f592010-05-04 17:36:49 -07001111 ret = device_rename(&dev->dev, dev->name);
1112 if (ret) {
1113 memcpy(dev->name, oldname, IFNAMSIZ);
Eric Dumazet30e6c9f2012-12-20 17:25:08 +00001114 write_seqcount_end(&devnet_rename_seq);
Eric W. Biedermana1b3f592010-05-04 17:36:49 -07001115 return ret;
Stephen Hemmingerdcc99772008-05-14 22:33:38 -07001116 }
Herbert Xu7f988ea2007-07-30 16:35:46 -07001117
Eric Dumazet30e6c9f2012-12-20 17:25:08 +00001118 write_seqcount_end(&devnet_rename_seq);
Brian Haleyc91f6df2012-11-26 05:21:08 +00001119
Herbert Xu7f988ea2007-07-30 16:35:46 -07001120 write_lock_bh(&dev_base_lock);
Eric Dumazet372b2312011-05-17 13:56:59 -04001121 hlist_del_rcu(&dev->name_hlist);
Eric Dumazet72c95282009-10-30 07:11:27 +00001122 write_unlock_bh(&dev_base_lock);
1123
1124 synchronize_rcu();
1125
1126 write_lock_bh(&dev_base_lock);
1127 hlist_add_head_rcu(&dev->name_hlist, dev_name_hash(net, dev->name));
Herbert Xu7f988ea2007-07-30 16:35:46 -07001128 write_unlock_bh(&dev_base_lock);
1129
Pavel Emelyanov056925a2007-09-16 15:42:43 -07001130 ret = call_netdevice_notifiers(NETDEV_CHANGENAME, dev);
Herbert Xufcc5a032007-07-30 17:03:38 -07001131 ret = notifier_to_errno(ret);
1132
1133 if (ret) {
Eric Dumazet91e9c07b2009-11-15 23:30:24 +00001134 /* err >= 0 after dev_alloc_name() or stores the first errno */
1135 if (err >= 0) {
Herbert Xufcc5a032007-07-30 17:03:38 -07001136 err = ret;
Eric Dumazet30e6c9f2012-12-20 17:25:08 +00001137 write_seqcount_begin(&devnet_rename_seq);
Herbert Xufcc5a032007-07-30 17:03:38 -07001138 memcpy(dev->name, oldname, IFNAMSIZ);
1139 goto rollback;
Eric Dumazet91e9c07b2009-11-15 23:30:24 +00001140 } else {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00001141 pr_err("%s: name change rollback failed: %d\n",
Eric Dumazet91e9c07b2009-11-15 23:30:24 +00001142 dev->name, ret);
Herbert Xufcc5a032007-07-30 17:03:38 -07001143 }
1144 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001145
1146 return err;
1147}
1148
1149/**
Stephen Hemminger0b815a12008-09-22 21:28:11 -07001150 * dev_set_alias - change ifalias of a device
1151 * @dev: device
1152 * @alias: name up to IFALIASZ
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07001153 * @len: limit of bytes to copy from info
Stephen Hemminger0b815a12008-09-22 21:28:11 -07001154 *
1155 * Set ifalias for a device,
1156 */
1157int dev_set_alias(struct net_device *dev, const char *alias, size_t len)
1158{
Alexey Khoroshilov7364e442012-08-08 00:33:25 +00001159 char *new_ifalias;
1160
Stephen Hemminger0b815a12008-09-22 21:28:11 -07001161 ASSERT_RTNL();
1162
1163 if (len >= IFALIASZ)
1164 return -EINVAL;
1165
Oliver Hartkopp96ca4a22008-09-23 21:23:19 -07001166 if (!len) {
Sachin Kamat388dfc22012-11-20 00:57:04 +00001167 kfree(dev->ifalias);
1168 dev->ifalias = NULL;
Oliver Hartkopp96ca4a22008-09-23 21:23:19 -07001169 return 0;
1170 }
1171
Alexey Khoroshilov7364e442012-08-08 00:33:25 +00001172 new_ifalias = krealloc(dev->ifalias, len + 1, GFP_KERNEL);
1173 if (!new_ifalias)
Stephen Hemminger0b815a12008-09-22 21:28:11 -07001174 return -ENOMEM;
Alexey Khoroshilov7364e442012-08-08 00:33:25 +00001175 dev->ifalias = new_ifalias;
Stephen Hemminger0b815a12008-09-22 21:28:11 -07001176
1177 strlcpy(dev->ifalias, alias, len+1);
1178 return len;
1179}
1180
1181
1182/**
Stephen Hemminger3041a062006-05-26 13:25:24 -07001183 * netdev_features_change - device changes features
Stephen Hemmingerd8a33ac2005-05-29 14:13:47 -07001184 * @dev: device to cause notification
1185 *
1186 * Called to indicate a device has changed features.
1187 */
1188void netdev_features_change(struct net_device *dev)
1189{
Pavel Emelyanov056925a2007-09-16 15:42:43 -07001190 call_netdevice_notifiers(NETDEV_FEAT_CHANGE, dev);
Stephen Hemmingerd8a33ac2005-05-29 14:13:47 -07001191}
1192EXPORT_SYMBOL(netdev_features_change);
1193
1194/**
Linus Torvalds1da177e2005-04-16 15:20:36 -07001195 * netdev_state_change - device changes state
1196 * @dev: device to cause notification
1197 *
1198 * Called to indicate a device has changed state. This function calls
1199 * the notifier chains for netdev_chain and sends a NEWLINK message
1200 * to the routing socket.
1201 */
1202void netdev_state_change(struct net_device *dev)
1203{
1204 if (dev->flags & IFF_UP) {
Pavel Emelyanov056925a2007-09-16 15:42:43 -07001205 call_netdevice_notifiers(NETDEV_CHANGE, dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001206 rtmsg_ifinfo(RTM_NEWLINK, dev, 0);
1207 }
1208}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001209EXPORT_SYMBOL(netdev_state_change);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001210
Amerigo Wangee89bab2012-08-09 22:14:56 +00001211/**
1212 * netdev_notify_peers - notify network peers about existence of @dev
1213 * @dev: network device
1214 *
1215 * Generate traffic such that interested network peers are aware of
1216 * @dev, such as by generating a gratuitous ARP. This may be used when
1217 * a device wants to inform the rest of the network about some sort of
1218 * reconfiguration such as a failover event or virtual machine
1219 * migration.
1220 */
1221void netdev_notify_peers(struct net_device *dev)
Or Gerlitzc1da4ac2008-06-13 18:12:00 -07001222{
Amerigo Wangee89bab2012-08-09 22:14:56 +00001223 rtnl_lock();
1224 call_netdevice_notifiers(NETDEV_NOTIFY_PEERS, dev);
1225 rtnl_unlock();
Or Gerlitzc1da4ac2008-06-13 18:12:00 -07001226}
Amerigo Wangee89bab2012-08-09 22:14:56 +00001227EXPORT_SYMBOL(netdev_notify_peers);
Or Gerlitzc1da4ac2008-06-13 18:12:00 -07001228
Patrick McHardybd380812010-02-26 06:34:53 +00001229static int __dev_open(struct net_device *dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001230{
Stephen Hemmingerd3147742008-11-19 21:32:24 -08001231 const struct net_device_ops *ops = dev->netdev_ops;
Johannes Berg3b8bcfd2009-05-30 01:39:53 +02001232 int ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001233
Ben Hutchingse46b66b2008-05-08 02:53:17 -07001234 ASSERT_RTNL();
1235
Linus Torvalds1da177e2005-04-16 15:20:36 -07001236 if (!netif_device_present(dev))
1237 return -ENODEV;
1238
Neil Hormanca99ca12013-02-05 08:05:43 +00001239 /* Block netpoll from trying to do any rx path servicing.
1240 * If we don't do this there is a chance ndo_poll_controller
1241 * or ndo_poll may be running while we open the device
1242 */
dingtianhongda6e3782013-05-27 19:53:31 +00001243 netpoll_rx_disable(dev);
Neil Hormanca99ca12013-02-05 08:05:43 +00001244
Johannes Berg3b8bcfd2009-05-30 01:39:53 +02001245 ret = call_netdevice_notifiers(NETDEV_PRE_UP, dev);
1246 ret = notifier_to_errno(ret);
1247 if (ret)
1248 return ret;
1249
Linus Torvalds1da177e2005-04-16 15:20:36 -07001250 set_bit(__LINK_STATE_START, &dev->state);
Jeff Garzikbada3392007-10-23 20:19:37 -07001251
Stephen Hemmingerd3147742008-11-19 21:32:24 -08001252 if (ops->ndo_validate_addr)
1253 ret = ops->ndo_validate_addr(dev);
Jeff Garzikbada3392007-10-23 20:19:37 -07001254
Stephen Hemmingerd3147742008-11-19 21:32:24 -08001255 if (!ret && ops->ndo_open)
1256 ret = ops->ndo_open(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001257
Neil Hormanca99ca12013-02-05 08:05:43 +00001258 netpoll_rx_enable(dev);
1259
Jeff Garzikbada3392007-10-23 20:19:37 -07001260 if (ret)
1261 clear_bit(__LINK_STATE_START, &dev->state);
1262 else {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001263 dev->flags |= IFF_UP;
David S. Millerb4bd07c2009-02-06 22:06:43 -08001264 net_dmaengine_get();
Patrick McHardy4417da62007-06-27 01:28:10 -07001265 dev_set_rx_mode(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001266 dev_activate(dev);
Theodore Ts'o7bf23572012-07-04 21:23:25 -04001267 add_device_randomness(dev->dev_addr, dev->addr_len);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001268 }
Jeff Garzikbada3392007-10-23 20:19:37 -07001269
Linus Torvalds1da177e2005-04-16 15:20:36 -07001270 return ret;
1271}
Patrick McHardybd380812010-02-26 06:34:53 +00001272
1273/**
1274 * dev_open - prepare an interface for use.
1275 * @dev: device to open
1276 *
1277 * Takes a device from down to up state. The device's private open
1278 * function is invoked and then the multicast lists are loaded. Finally
1279 * the device is moved into the up state and a %NETDEV_UP message is
1280 * sent to the netdev notifier chain.
1281 *
1282 * Calling this function on an active interface is a nop. On a failure
1283 * a negative errno code is returned.
1284 */
1285int dev_open(struct net_device *dev)
1286{
1287 int ret;
1288
Patrick McHardybd380812010-02-26 06:34:53 +00001289 if (dev->flags & IFF_UP)
1290 return 0;
1291
Patrick McHardybd380812010-02-26 06:34:53 +00001292 ret = __dev_open(dev);
1293 if (ret < 0)
1294 return ret;
1295
Patrick McHardybd380812010-02-26 06:34:53 +00001296 rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP|IFF_RUNNING);
1297 call_netdevice_notifiers(NETDEV_UP, dev);
1298
1299 return ret;
1300}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001301EXPORT_SYMBOL(dev_open);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001302
Octavian Purdila44345722010-12-13 12:44:07 +00001303static int __dev_close_many(struct list_head *head)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001304{
Octavian Purdila44345722010-12-13 12:44:07 +00001305 struct net_device *dev;
Patrick McHardybd380812010-02-26 06:34:53 +00001306
Ben Hutchingse46b66b2008-05-08 02:53:17 -07001307 ASSERT_RTNL();
David S. Miller9d5010d2007-09-12 14:33:25 +02001308 might_sleep();
1309
Octavian Purdila44345722010-12-13 12:44:07 +00001310 list_for_each_entry(dev, head, unreg_list) {
Octavian Purdila44345722010-12-13 12:44:07 +00001311 call_netdevice_notifiers(NETDEV_GOING_DOWN, dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001312
Octavian Purdila44345722010-12-13 12:44:07 +00001313 clear_bit(__LINK_STATE_START, &dev->state);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001314
Octavian Purdila44345722010-12-13 12:44:07 +00001315 /* Synchronize to scheduled poll. We cannot touch poll list, it
1316 * can be even on different cpu. So just clear netif_running().
1317 *
1318 * dev->stop() will invoke napi_disable() on all of it's
1319 * napi_struct instances on this device.
1320 */
1321 smp_mb__after_clear_bit(); /* Commit netif_running(). */
1322 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001323
Octavian Purdila44345722010-12-13 12:44:07 +00001324 dev_deactivate_many(head);
1325
1326 list_for_each_entry(dev, head, unreg_list) {
1327 const struct net_device_ops *ops = dev->netdev_ops;
1328
1329 /*
1330 * Call the device specific close. This cannot fail.
1331 * Only if device is UP
1332 *
1333 * We allow it to be called even after a DETACH hot-plug
1334 * event.
1335 */
1336 if (ops->ndo_stop)
1337 ops->ndo_stop(dev);
1338
Octavian Purdila44345722010-12-13 12:44:07 +00001339 dev->flags &= ~IFF_UP;
Octavian Purdila44345722010-12-13 12:44:07 +00001340 net_dmaengine_put();
1341 }
1342
1343 return 0;
1344}
1345
1346static int __dev_close(struct net_device *dev)
1347{
Linus Torvaldsf87e6f42011-02-17 22:54:38 +00001348 int retval;
Octavian Purdila44345722010-12-13 12:44:07 +00001349 LIST_HEAD(single);
1350
Neil Hormanca99ca12013-02-05 08:05:43 +00001351 /* Temporarily disable netpoll until the interface is down */
dingtianhongda6e3782013-05-27 19:53:31 +00001352 netpoll_rx_disable(dev);
Neil Hormanca99ca12013-02-05 08:05:43 +00001353
Octavian Purdila44345722010-12-13 12:44:07 +00001354 list_add(&dev->unreg_list, &single);
Linus Torvaldsf87e6f42011-02-17 22:54:38 +00001355 retval = __dev_close_many(&single);
1356 list_del(&single);
Neil Hormanca99ca12013-02-05 08:05:43 +00001357
1358 netpoll_rx_enable(dev);
Linus Torvaldsf87e6f42011-02-17 22:54:38 +00001359 return retval;
Octavian Purdila44345722010-12-13 12:44:07 +00001360}
1361
Eric Dumazet3fbd8752011-01-19 21:23:22 +00001362static int dev_close_many(struct list_head *head)
Octavian Purdila44345722010-12-13 12:44:07 +00001363{
1364 struct net_device *dev, *tmp;
1365 LIST_HEAD(tmp_list);
1366
1367 list_for_each_entry_safe(dev, tmp, head, unreg_list)
1368 if (!(dev->flags & IFF_UP))
1369 list_move(&dev->unreg_list, &tmp_list);
1370
1371 __dev_close_many(head);
Matti Linnanvuorid8b2a4d2008-02-12 23:10:11 -08001372
Octavian Purdila44345722010-12-13 12:44:07 +00001373 list_for_each_entry(dev, head, unreg_list) {
1374 rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP|IFF_RUNNING);
1375 call_netdevice_notifiers(NETDEV_DOWN, dev);
1376 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001377
Octavian Purdila44345722010-12-13 12:44:07 +00001378 /* rollback_registered_many needs the complete original list */
1379 list_splice(&tmp_list, head);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001380 return 0;
1381}
Patrick McHardybd380812010-02-26 06:34:53 +00001382
1383/**
1384 * dev_close - shutdown an interface.
1385 * @dev: device to shutdown
1386 *
1387 * This function moves an active device into down state. A
1388 * %NETDEV_GOING_DOWN is sent to the netdev notifier chain. The device
1389 * is then deactivated and finally a %NETDEV_DOWN is sent to the notifier
1390 * chain.
1391 */
1392int dev_close(struct net_device *dev)
1393{
Eric Dumazete14a5992011-05-10 12:26:06 -07001394 if (dev->flags & IFF_UP) {
1395 LIST_HEAD(single);
Patrick McHardybd380812010-02-26 06:34:53 +00001396
Neil Hormanca99ca12013-02-05 08:05:43 +00001397 /* Block netpoll rx while the interface is going down */
dingtianhongda6e3782013-05-27 19:53:31 +00001398 netpoll_rx_disable(dev);
Neil Hormanca99ca12013-02-05 08:05:43 +00001399
Eric Dumazete14a5992011-05-10 12:26:06 -07001400 list_add(&dev->unreg_list, &single);
1401 dev_close_many(&single);
1402 list_del(&single);
Neil Hormanca99ca12013-02-05 08:05:43 +00001403
1404 netpoll_rx_enable(dev);
Eric Dumazete14a5992011-05-10 12:26:06 -07001405 }
dingtianhongda6e3782013-05-27 19:53:31 +00001406 return 0;
Patrick McHardybd380812010-02-26 06:34:53 +00001407}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001408EXPORT_SYMBOL(dev_close);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001409
1410
Ben Hutchings0187bdf2008-06-19 16:15:47 -07001411/**
1412 * dev_disable_lro - disable Large Receive Offload on a device
1413 * @dev: device
1414 *
1415 * Disable Large Receive Offload (LRO) on a net device. Must be
1416 * called under RTNL. This is needed if received packets may be
1417 * forwarded to another interface.
1418 */
1419void dev_disable_lro(struct net_device *dev)
1420{
Neil Hormanf11970e2011-05-24 08:31:09 +00001421 /*
1422 * If we're trying to disable lro on a vlan device
1423 * use the underlying physical device instead
1424 */
1425 if (is_vlan_dev(dev))
1426 dev = vlan_dev_real_dev(dev);
1427
Michał Mirosławbc5787c62011-11-15 15:29:55 +00001428 dev->wanted_features &= ~NETIF_F_LRO;
1429 netdev_update_features(dev);
Michał Mirosław27660512011-03-18 16:56:34 +00001430
Michał Mirosław22d59692011-04-21 12:42:15 +00001431 if (unlikely(dev->features & NETIF_F_LRO))
1432 netdev_WARN(dev, "failed to disable LRO!\n");
Ben Hutchings0187bdf2008-06-19 16:15:47 -07001433}
1434EXPORT_SYMBOL(dev_disable_lro);
1435
Jiri Pirko351638e2013-05-28 01:30:21 +00001436static int call_netdevice_notifier(struct notifier_block *nb, unsigned long val,
1437 struct net_device *dev)
1438{
1439 struct netdev_notifier_info info;
1440
1441 netdev_notifier_info_init(&info, dev);
1442 return nb->notifier_call(nb, val, &info);
1443}
Ben Hutchings0187bdf2008-06-19 16:15:47 -07001444
Eric W. Biederman881d9662007-09-17 11:56:21 -07001445static int dev_boot_phase = 1;
1446
Linus Torvalds1da177e2005-04-16 15:20:36 -07001447/**
1448 * register_netdevice_notifier - register a network notifier block
1449 * @nb: notifier
1450 *
1451 * Register a notifier to be called when network device events occur.
1452 * The notifier passed is linked into the kernel structures and must
1453 * not be reused until it has been unregistered. A negative errno code
1454 * is returned on a failure.
1455 *
1456 * When registered all registration and up events are replayed
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09001457 * to the new notifier to allow device to have a race free
Linus Torvalds1da177e2005-04-16 15:20:36 -07001458 * view of the network device list.
1459 */
1460
1461int register_netdevice_notifier(struct notifier_block *nb)
1462{
1463 struct net_device *dev;
Herbert Xufcc5a032007-07-30 17:03:38 -07001464 struct net_device *last;
Eric W. Biederman881d9662007-09-17 11:56:21 -07001465 struct net *net;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001466 int err;
1467
1468 rtnl_lock();
Alan Sternf07d5b92006-05-09 15:23:03 -07001469 err = raw_notifier_chain_register(&netdev_chain, nb);
Herbert Xufcc5a032007-07-30 17:03:38 -07001470 if (err)
1471 goto unlock;
Eric W. Biederman881d9662007-09-17 11:56:21 -07001472 if (dev_boot_phase)
1473 goto unlock;
1474 for_each_net(net) {
1475 for_each_netdev(net, dev) {
Jiri Pirko351638e2013-05-28 01:30:21 +00001476 err = call_netdevice_notifier(nb, NETDEV_REGISTER, dev);
Eric W. Biederman881d9662007-09-17 11:56:21 -07001477 err = notifier_to_errno(err);
1478 if (err)
1479 goto rollback;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001480
Eric W. Biederman881d9662007-09-17 11:56:21 -07001481 if (!(dev->flags & IFF_UP))
1482 continue;
Herbert Xufcc5a032007-07-30 17:03:38 -07001483
Jiri Pirko351638e2013-05-28 01:30:21 +00001484 call_netdevice_notifier(nb, NETDEV_UP, dev);
Eric W. Biederman881d9662007-09-17 11:56:21 -07001485 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001486 }
Herbert Xufcc5a032007-07-30 17:03:38 -07001487
1488unlock:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001489 rtnl_unlock();
1490 return err;
Herbert Xufcc5a032007-07-30 17:03:38 -07001491
1492rollback:
1493 last = dev;
Eric W. Biederman881d9662007-09-17 11:56:21 -07001494 for_each_net(net) {
1495 for_each_netdev(net, dev) {
1496 if (dev == last)
RongQing.Li8f891482011-11-30 23:43:07 -05001497 goto outroll;
Herbert Xufcc5a032007-07-30 17:03:38 -07001498
Eric W. Biederman881d9662007-09-17 11:56:21 -07001499 if (dev->flags & IFF_UP) {
Jiri Pirko351638e2013-05-28 01:30:21 +00001500 call_netdevice_notifier(nb, NETDEV_GOING_DOWN,
1501 dev);
1502 call_netdevice_notifier(nb, NETDEV_DOWN, dev);
Eric W. Biederman881d9662007-09-17 11:56:21 -07001503 }
Jiri Pirko351638e2013-05-28 01:30:21 +00001504 call_netdevice_notifier(nb, NETDEV_UNREGISTER, dev);
Herbert Xufcc5a032007-07-30 17:03:38 -07001505 }
Herbert Xufcc5a032007-07-30 17:03:38 -07001506 }
Pavel Emelyanovc67625a2007-11-14 15:53:16 -08001507
RongQing.Li8f891482011-11-30 23:43:07 -05001508outroll:
Pavel Emelyanovc67625a2007-11-14 15:53:16 -08001509 raw_notifier_chain_unregister(&netdev_chain, nb);
Herbert Xufcc5a032007-07-30 17:03:38 -07001510 goto unlock;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001511}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001512EXPORT_SYMBOL(register_netdevice_notifier);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001513
1514/**
1515 * unregister_netdevice_notifier - unregister a network notifier block
1516 * @nb: notifier
1517 *
1518 * Unregister a notifier previously registered by
1519 * register_netdevice_notifier(). The notifier is unlinked into the
1520 * kernel structures and may then be reused. A negative errno code
1521 * is returned on a failure.
Eric W. Biederman7d3d43d2012-04-06 15:33:35 +00001522 *
1523 * After unregistering unregister and down device events are synthesized
1524 * for all devices on the device list to the removed notifier to remove
1525 * the need for special case cleanup code.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001526 */
1527
1528int unregister_netdevice_notifier(struct notifier_block *nb)
1529{
Eric W. Biederman7d3d43d2012-04-06 15:33:35 +00001530 struct net_device *dev;
1531 struct net *net;
Herbert Xu9f514952006-03-25 01:24:25 -08001532 int err;
1533
1534 rtnl_lock();
Alan Sternf07d5b92006-05-09 15:23:03 -07001535 err = raw_notifier_chain_unregister(&netdev_chain, nb);
Eric W. Biederman7d3d43d2012-04-06 15:33:35 +00001536 if (err)
1537 goto unlock;
1538
1539 for_each_net(net) {
1540 for_each_netdev(net, dev) {
1541 if (dev->flags & IFF_UP) {
Jiri Pirko351638e2013-05-28 01:30:21 +00001542 call_netdevice_notifier(nb, NETDEV_GOING_DOWN,
1543 dev);
1544 call_netdevice_notifier(nb, NETDEV_DOWN, dev);
Eric W. Biederman7d3d43d2012-04-06 15:33:35 +00001545 }
Jiri Pirko351638e2013-05-28 01:30:21 +00001546 call_netdevice_notifier(nb, NETDEV_UNREGISTER, dev);
Eric W. Biederman7d3d43d2012-04-06 15:33:35 +00001547 }
1548 }
1549unlock:
Herbert Xu9f514952006-03-25 01:24:25 -08001550 rtnl_unlock();
1551 return err;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001552}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001553EXPORT_SYMBOL(unregister_netdevice_notifier);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001554
1555/**
Jiri Pirko351638e2013-05-28 01:30:21 +00001556 * call_netdevice_notifiers_info - call all network notifier blocks
1557 * @val: value passed unmodified to notifier function
1558 * @dev: net_device pointer passed unmodified to notifier function
1559 * @info: notifier information data
1560 *
1561 * Call all network notifier blocks. Parameters and return value
1562 * are as for raw_notifier_call_chain().
1563 */
1564
1565int call_netdevice_notifiers_info(unsigned long val, struct net_device *dev,
1566 struct netdev_notifier_info *info)
1567{
1568 ASSERT_RTNL();
1569 netdev_notifier_info_init(info, dev);
1570 return raw_notifier_call_chain(&netdev_chain, val, info);
1571}
1572EXPORT_SYMBOL(call_netdevice_notifiers_info);
1573
1574/**
Linus Torvalds1da177e2005-04-16 15:20:36 -07001575 * call_netdevice_notifiers - call all network notifier blocks
1576 * @val: value passed unmodified to notifier function
Randy Dunlapc4ea43c2007-10-12 21:17:49 -07001577 * @dev: net_device pointer passed unmodified to notifier function
Linus Torvalds1da177e2005-04-16 15:20:36 -07001578 *
1579 * Call all network notifier blocks. Parameters and return value
Alan Sternf07d5b92006-05-09 15:23:03 -07001580 * are as for raw_notifier_call_chain().
Linus Torvalds1da177e2005-04-16 15:20:36 -07001581 */
1582
Eric W. Biedermanad7379d2007-09-16 15:33:32 -07001583int call_netdevice_notifiers(unsigned long val, struct net_device *dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001584{
Jiri Pirko351638e2013-05-28 01:30:21 +00001585 struct netdev_notifier_info info;
1586
1587 return call_netdevice_notifiers_info(val, dev, &info);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001588}
stephen hemmingeredf947f2011-03-24 13:24:01 +00001589EXPORT_SYMBOL(call_netdevice_notifiers);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001590
Ingo Molnarc5905af2012-02-24 08:31:31 +01001591static struct static_key netstamp_needed __read_mostly;
Eric Dumazetb90e5792011-11-28 11:16:50 +00001592#ifdef HAVE_JUMP_LABEL
Ingo Molnarc5905af2012-02-24 08:31:31 +01001593/* We are not allowed to call static_key_slow_dec() from irq context
Eric Dumazetb90e5792011-11-28 11:16:50 +00001594 * If net_disable_timestamp() is called from irq context, defer the
Ingo Molnarc5905af2012-02-24 08:31:31 +01001595 * static_key_slow_dec() calls.
Eric Dumazetb90e5792011-11-28 11:16:50 +00001596 */
1597static atomic_t netstamp_needed_deferred;
1598#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07001599
1600void net_enable_timestamp(void)
1601{
Eric Dumazetb90e5792011-11-28 11:16:50 +00001602#ifdef HAVE_JUMP_LABEL
1603 int deferred = atomic_xchg(&netstamp_needed_deferred, 0);
1604
1605 if (deferred) {
1606 while (--deferred)
Ingo Molnarc5905af2012-02-24 08:31:31 +01001607 static_key_slow_dec(&netstamp_needed);
Eric Dumazetb90e5792011-11-28 11:16:50 +00001608 return;
1609 }
1610#endif
Ingo Molnarc5905af2012-02-24 08:31:31 +01001611 static_key_slow_inc(&netstamp_needed);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001612}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001613EXPORT_SYMBOL(net_enable_timestamp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001614
1615void net_disable_timestamp(void)
1616{
Eric Dumazetb90e5792011-11-28 11:16:50 +00001617#ifdef HAVE_JUMP_LABEL
1618 if (in_interrupt()) {
1619 atomic_inc(&netstamp_needed_deferred);
1620 return;
1621 }
1622#endif
Ingo Molnarc5905af2012-02-24 08:31:31 +01001623 static_key_slow_dec(&netstamp_needed);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001624}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001625EXPORT_SYMBOL(net_disable_timestamp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001626
Eric Dumazet3b098e22010-05-15 23:57:10 -07001627static inline void net_timestamp_set(struct sk_buff *skb)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001628{
Eric Dumazet588f0332011-11-15 04:12:55 +00001629 skb->tstamp.tv64 = 0;
Ingo Molnarc5905af2012-02-24 08:31:31 +01001630 if (static_key_false(&netstamp_needed))
Patrick McHardya61bbcf2005-08-14 17:24:31 -07001631 __net_timestamp(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001632}
1633
Eric Dumazet588f0332011-11-15 04:12:55 +00001634#define net_timestamp_check(COND, SKB) \
Ingo Molnarc5905af2012-02-24 08:31:31 +01001635 if (static_key_false(&netstamp_needed)) { \
Eric Dumazet588f0332011-11-15 04:12:55 +00001636 if ((COND) && !(SKB)->tstamp.tv64) \
1637 __net_timestamp(SKB); \
1638 } \
Eric Dumazet3b098e22010-05-15 23:57:10 -07001639
Daniel Lezcano79b569f2011-03-30 02:42:17 -07001640static inline bool is_skb_forwardable(struct net_device *dev,
1641 struct sk_buff *skb)
1642{
1643 unsigned int len;
1644
1645 if (!(dev->flags & IFF_UP))
1646 return false;
1647
1648 len = dev->mtu + dev->hard_header_len + VLAN_HLEN;
1649 if (skb->len <= len)
1650 return true;
1651
1652 /* if TSO is enabled, we don't care about the length as the packet
1653 * could be forwarded without being segmented before
1654 */
1655 if (skb_is_gso(skb))
1656 return true;
1657
1658 return false;
1659}
1660
Arnd Bergmann44540962009-11-26 06:07:08 +00001661/**
1662 * dev_forward_skb - loopback an skb to another netif
1663 *
1664 * @dev: destination network device
1665 * @skb: buffer to forward
1666 *
1667 * return values:
1668 * NET_RX_SUCCESS (no congestion)
Eric Dumazet6ec82562010-05-06 00:53:53 -07001669 * NET_RX_DROP (packet was dropped, but freed)
Arnd Bergmann44540962009-11-26 06:07:08 +00001670 *
1671 * dev_forward_skb can be used for injecting an skb from the
1672 * start_xmit function of one device into the receive queue
1673 * of another device.
1674 *
1675 * The receiving device may be in another namespace, so
1676 * we have to clear all information in the skb that could
1677 * impact namespace isolation.
1678 */
1679int dev_forward_skb(struct net_device *dev, struct sk_buff *skb)
1680{
Michael S. Tsirkin48c83012011-08-31 08:03:29 +00001681 if (skb_shinfo(skb)->tx_flags & SKBTX_DEV_ZEROCOPY) {
1682 if (skb_copy_ubufs(skb, GFP_ATOMIC)) {
1683 atomic_long_inc(&dev->rx_dropped);
1684 kfree_skb(skb);
1685 return NET_RX_DROP;
1686 }
1687 }
1688
Daniel Lezcano79b569f2011-03-30 02:42:17 -07001689 if (unlikely(!is_skb_forwardable(dev, skb))) {
Eric Dumazetcaf586e2010-09-30 21:06:55 +00001690 atomic_long_inc(&dev->rx_dropped);
Eric Dumazet6ec82562010-05-06 00:53:53 -07001691 kfree_skb(skb);
Arnd Bergmann44540962009-11-26 06:07:08 +00001692 return NET_RX_DROP;
Eric Dumazet6ec82562010-05-06 00:53:53 -07001693 }
Arnd Bergmann44540962009-11-26 06:07:08 +00001694 skb->protocol = eth_type_trans(skb, dev);
Isaku Yamahata06a23fe2013-07-02 20:30:10 +09001695
1696 /* eth_type_trans() can set pkt_type.
Nicolas Dichtel64261f22013-08-13 17:51:09 +02001697 * call skb_scrub_packet() after it to clear pkt_type _after_ calling
1698 * eth_type_trans().
Isaku Yamahata06a23fe2013-07-02 20:30:10 +09001699 */
Nicolas Dichtel8b27f272013-09-02 15:34:56 +02001700 skb_scrub_packet(skb, true);
Isaku Yamahata06a23fe2013-07-02 20:30:10 +09001701
Arnd Bergmann44540962009-11-26 06:07:08 +00001702 return netif_rx(skb);
1703}
1704EXPORT_SYMBOL_GPL(dev_forward_skb);
1705
Changli Gao71d9dec2010-12-15 19:57:25 +00001706static inline int deliver_skb(struct sk_buff *skb,
1707 struct packet_type *pt_prev,
1708 struct net_device *orig_dev)
1709{
Michael S. Tsirkin1080e512012-07-20 09:23:17 +00001710 if (unlikely(skb_orphan_frags(skb, GFP_ATOMIC)))
1711 return -ENOMEM;
Changli Gao71d9dec2010-12-15 19:57:25 +00001712 atomic_inc(&skb->users);
1713 return pt_prev->func(skb, skb->dev, pt_prev, orig_dev);
1714}
1715
Eric Leblondc0de08d2012-08-16 22:02:58 +00001716static inline bool skb_loop_sk(struct packet_type *ptype, struct sk_buff *skb)
1717{
Eric Leblonda3d744e2012-11-06 02:10:10 +00001718 if (!ptype->af_packet_priv || !skb->sk)
Eric Leblondc0de08d2012-08-16 22:02:58 +00001719 return false;
1720
1721 if (ptype->id_match)
1722 return ptype->id_match(ptype, skb->sk);
1723 else if ((struct sock *)ptype->af_packet_priv == skb->sk)
1724 return true;
1725
1726 return false;
1727}
1728
Linus Torvalds1da177e2005-04-16 15:20:36 -07001729/*
1730 * Support routine. Sends outgoing frames to any network
1731 * taps currently in use.
1732 */
1733
Herbert Xuf6a78bf2006-06-22 02:57:17 -07001734static void dev_queue_xmit_nit(struct sk_buff *skb, struct net_device *dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001735{
1736 struct packet_type *ptype;
Changli Gao71d9dec2010-12-15 19:57:25 +00001737 struct sk_buff *skb2 = NULL;
1738 struct packet_type *pt_prev = NULL;
Patrick McHardya61bbcf2005-08-14 17:24:31 -07001739
Linus Torvalds1da177e2005-04-16 15:20:36 -07001740 rcu_read_lock();
1741 list_for_each_entry_rcu(ptype, &ptype_all, list) {
1742 /* Never send packets back to the socket
1743 * they originated from - MvS (miquels@drinkel.ow.org)
1744 */
1745 if ((ptype->dev == dev || !ptype->dev) &&
Eric Leblondc0de08d2012-08-16 22:02:58 +00001746 (!skb_loop_sk(ptype, skb))) {
Changli Gao71d9dec2010-12-15 19:57:25 +00001747 if (pt_prev) {
1748 deliver_skb(skb2, pt_prev, skb->dev);
1749 pt_prev = ptype;
1750 continue;
1751 }
1752
1753 skb2 = skb_clone(skb, GFP_ATOMIC);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001754 if (!skb2)
1755 break;
1756
Eric Dumazet70978182010-12-20 21:22:51 +00001757 net_timestamp_set(skb2);
1758
Linus Torvalds1da177e2005-04-16 15:20:36 -07001759 /* skb->nh should be correctly
1760 set by sender, so that the second statement is
1761 just protection against buggy protocols.
1762 */
Arnaldo Carvalho de Melo459a98e2007-03-19 15:30:44 -07001763 skb_reset_mac_header(skb2);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001764
Arnaldo Carvalho de Melod56f90a2007-04-10 20:50:43 -07001765 if (skb_network_header(skb2) < skb2->data ||
Simon Hormanced14f62013-05-28 20:34:25 +00001766 skb_network_header(skb2) > skb_tail_pointer(skb2)) {
Joe Perchese87cc472012-05-13 21:56:26 +00001767 net_crit_ratelimited("protocol %04x is buggy, dev %s\n",
1768 ntohs(skb2->protocol),
1769 dev->name);
Arnaldo Carvalho de Meloc1d2bbe2007-04-10 20:45:18 -07001770 skb_reset_network_header(skb2);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001771 }
1772
Arnaldo Carvalho de Melob0e380b2007-04-10 21:21:55 -07001773 skb2->transport_header = skb2->network_header;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001774 skb2->pkt_type = PACKET_OUTGOING;
Changli Gao71d9dec2010-12-15 19:57:25 +00001775 pt_prev = ptype;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001776 }
1777 }
Changli Gao71d9dec2010-12-15 19:57:25 +00001778 if (pt_prev)
1779 pt_prev->func(skb2, skb->dev, pt_prev, skb->dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001780 rcu_read_unlock();
1781}
1782
Ben Hutchings2c530402012-07-10 10:55:09 +00001783/**
1784 * netif_setup_tc - Handle tc mappings on real_num_tx_queues change
John Fastabend4f57c082011-01-17 08:06:04 +00001785 * @dev: Network device
1786 * @txq: number of queues available
1787 *
1788 * If real_num_tx_queues is changed the tc mappings may no longer be
1789 * valid. To resolve this verify the tc mapping remains valid and if
1790 * not NULL the mapping. With no priorities mapping to this
1791 * offset/count pair it will no longer be used. In the worst case TC0
1792 * is invalid nothing can be done so disable priority mappings. If is
1793 * expected that drivers will fix this mapping if they can before
1794 * calling netif_set_real_num_tx_queues.
1795 */
Eric Dumazetbb134d22011-01-20 19:18:08 +00001796static void netif_setup_tc(struct net_device *dev, unsigned int txq)
John Fastabend4f57c082011-01-17 08:06:04 +00001797{
1798 int i;
1799 struct netdev_tc_txq *tc = &dev->tc_to_txq[0];
1800
1801 /* If TC0 is invalidated disable TC mapping */
1802 if (tc->offset + tc->count > txq) {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00001803 pr_warn("Number of in use tx queues changed invalidating tc mappings. Priority traffic classification disabled!\n");
John Fastabend4f57c082011-01-17 08:06:04 +00001804 dev->num_tc = 0;
1805 return;
1806 }
1807
1808 /* Invalidated prio to tc mappings set to TC0 */
1809 for (i = 1; i < TC_BITMASK + 1; i++) {
1810 int q = netdev_get_prio_tc_map(dev, i);
1811
1812 tc = &dev->tc_to_txq[q];
1813 if (tc->offset + tc->count > txq) {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00001814 pr_warn("Number of in use tx queues changed. Priority %i to tc mapping %i is no longer valid. Setting map to 0\n",
1815 i, q);
John Fastabend4f57c082011-01-17 08:06:04 +00001816 netdev_set_prio_tc_map(dev, i, 0);
1817 }
1818 }
1819}
1820
Alexander Duyck537c00d2013-01-10 08:57:02 +00001821#ifdef CONFIG_XPS
1822static DEFINE_MUTEX(xps_map_mutex);
1823#define xmap_dereference(P) \
1824 rcu_dereference_protected((P), lockdep_is_held(&xps_map_mutex))
1825
Alexander Duyck10cdc3f2013-01-10 08:57:17 +00001826static struct xps_map *remove_xps_queue(struct xps_dev_maps *dev_maps,
1827 int cpu, u16 index)
1828{
1829 struct xps_map *map = NULL;
1830 int pos;
1831
1832 if (dev_maps)
1833 map = xmap_dereference(dev_maps->cpu_map[cpu]);
1834
1835 for (pos = 0; map && pos < map->len; pos++) {
1836 if (map->queues[pos] == index) {
1837 if (map->len > 1) {
1838 map->queues[pos] = map->queues[--map->len];
1839 } else {
1840 RCU_INIT_POINTER(dev_maps->cpu_map[cpu], NULL);
1841 kfree_rcu(map, rcu);
1842 map = NULL;
1843 }
1844 break;
1845 }
1846 }
1847
1848 return map;
1849}
1850
Alexander Duyck024e9672013-01-10 08:57:46 +00001851static void netif_reset_xps_queues_gt(struct net_device *dev, u16 index)
Alexander Duyck537c00d2013-01-10 08:57:02 +00001852{
1853 struct xps_dev_maps *dev_maps;
Alexander Duyck024e9672013-01-10 08:57:46 +00001854 int cpu, i;
Alexander Duyck10cdc3f2013-01-10 08:57:17 +00001855 bool active = false;
Alexander Duyck537c00d2013-01-10 08:57:02 +00001856
1857 mutex_lock(&xps_map_mutex);
1858 dev_maps = xmap_dereference(dev->xps_maps);
1859
1860 if (!dev_maps)
1861 goto out_no_maps;
1862
Alexander Duyck10cdc3f2013-01-10 08:57:17 +00001863 for_each_possible_cpu(cpu) {
Alexander Duyck024e9672013-01-10 08:57:46 +00001864 for (i = index; i < dev->num_tx_queues; i++) {
1865 if (!remove_xps_queue(dev_maps, cpu, i))
1866 break;
1867 }
1868 if (i == dev->num_tx_queues)
Alexander Duyck10cdc3f2013-01-10 08:57:17 +00001869 active = true;
Alexander Duyck537c00d2013-01-10 08:57:02 +00001870 }
1871
Alexander Duyck10cdc3f2013-01-10 08:57:17 +00001872 if (!active) {
Alexander Duyck537c00d2013-01-10 08:57:02 +00001873 RCU_INIT_POINTER(dev->xps_maps, NULL);
1874 kfree_rcu(dev_maps, rcu);
1875 }
1876
Alexander Duyck024e9672013-01-10 08:57:46 +00001877 for (i = index; i < dev->num_tx_queues; i++)
1878 netdev_queue_numa_node_write(netdev_get_tx_queue(dev, i),
1879 NUMA_NO_NODE);
1880
Alexander Duyck537c00d2013-01-10 08:57:02 +00001881out_no_maps:
1882 mutex_unlock(&xps_map_mutex);
1883}
1884
Alexander Duyck01c5f862013-01-10 08:57:35 +00001885static struct xps_map *expand_xps_map(struct xps_map *map,
1886 int cpu, u16 index)
1887{
1888 struct xps_map *new_map;
1889 int alloc_len = XPS_MIN_MAP_ALLOC;
1890 int i, pos;
1891
1892 for (pos = 0; map && pos < map->len; pos++) {
1893 if (map->queues[pos] != index)
1894 continue;
1895 return map;
1896 }
1897
1898 /* Need to add queue to this CPU's existing map */
1899 if (map) {
1900 if (pos < map->alloc_len)
1901 return map;
1902
1903 alloc_len = map->alloc_len * 2;
1904 }
1905
1906 /* Need to allocate new map to store queue on this CPU's map */
1907 new_map = kzalloc_node(XPS_MAP_SIZE(alloc_len), GFP_KERNEL,
1908 cpu_to_node(cpu));
1909 if (!new_map)
1910 return NULL;
1911
1912 for (i = 0; i < pos; i++)
1913 new_map->queues[i] = map->queues[i];
1914 new_map->alloc_len = alloc_len;
1915 new_map->len = pos;
1916
1917 return new_map;
1918}
1919
Alexander Duyck537c00d2013-01-10 08:57:02 +00001920int netif_set_xps_queue(struct net_device *dev, struct cpumask *mask, u16 index)
1921{
Alexander Duyck01c5f862013-01-10 08:57:35 +00001922 struct xps_dev_maps *dev_maps, *new_dev_maps = NULL;
Alexander Duyck537c00d2013-01-10 08:57:02 +00001923 struct xps_map *map, *new_map;
Alexander Duyck537c00d2013-01-10 08:57:02 +00001924 int maps_sz = max_t(unsigned int, XPS_DEV_MAPS_SIZE, L1_CACHE_BYTES);
Alexander Duyck01c5f862013-01-10 08:57:35 +00001925 int cpu, numa_node_id = -2;
1926 bool active = false;
Alexander Duyck537c00d2013-01-10 08:57:02 +00001927
1928 mutex_lock(&xps_map_mutex);
1929
1930 dev_maps = xmap_dereference(dev->xps_maps);
1931
Alexander Duyck01c5f862013-01-10 08:57:35 +00001932 /* allocate memory for queue storage */
1933 for_each_online_cpu(cpu) {
1934 if (!cpumask_test_cpu(cpu, mask))
1935 continue;
Alexander Duyck537c00d2013-01-10 08:57:02 +00001936
Alexander Duyck01c5f862013-01-10 08:57:35 +00001937 if (!new_dev_maps)
1938 new_dev_maps = kzalloc(maps_sz, GFP_KERNEL);
Alexander Duyck2bb60cb2013-02-22 06:38:44 +00001939 if (!new_dev_maps) {
1940 mutex_unlock(&xps_map_mutex);
Alexander Duyck01c5f862013-01-10 08:57:35 +00001941 return -ENOMEM;
Alexander Duyck2bb60cb2013-02-22 06:38:44 +00001942 }
Alexander Duyck01c5f862013-01-10 08:57:35 +00001943
1944 map = dev_maps ? xmap_dereference(dev_maps->cpu_map[cpu]) :
1945 NULL;
1946
1947 map = expand_xps_map(map, cpu, index);
1948 if (!map)
1949 goto error;
1950
1951 RCU_INIT_POINTER(new_dev_maps->cpu_map[cpu], map);
1952 }
1953
1954 if (!new_dev_maps)
1955 goto out_no_new_maps;
1956
1957 for_each_possible_cpu(cpu) {
1958 if (cpumask_test_cpu(cpu, mask) && cpu_online(cpu)) {
1959 /* add queue to CPU maps */
1960 int pos = 0;
1961
1962 map = xmap_dereference(new_dev_maps->cpu_map[cpu]);
1963 while ((pos < map->len) && (map->queues[pos] != index))
1964 pos++;
1965
1966 if (pos == map->len)
1967 map->queues[map->len++] = index;
Alexander Duyck537c00d2013-01-10 08:57:02 +00001968#ifdef CONFIG_NUMA
Alexander Duyck537c00d2013-01-10 08:57:02 +00001969 if (numa_node_id == -2)
1970 numa_node_id = cpu_to_node(cpu);
1971 else if (numa_node_id != cpu_to_node(cpu))
1972 numa_node_id = -1;
Alexander Duyck537c00d2013-01-10 08:57:02 +00001973#endif
Alexander Duyck01c5f862013-01-10 08:57:35 +00001974 } else if (dev_maps) {
1975 /* fill in the new device map from the old device map */
1976 map = xmap_dereference(dev_maps->cpu_map[cpu]);
1977 RCU_INIT_POINTER(new_dev_maps->cpu_map[cpu], map);
Alexander Duyck537c00d2013-01-10 08:57:02 +00001978 }
Alexander Duyck01c5f862013-01-10 08:57:35 +00001979
Alexander Duyck537c00d2013-01-10 08:57:02 +00001980 }
1981
Alexander Duyck01c5f862013-01-10 08:57:35 +00001982 rcu_assign_pointer(dev->xps_maps, new_dev_maps);
1983
Alexander Duyck537c00d2013-01-10 08:57:02 +00001984 /* Cleanup old maps */
Alexander Duyck01c5f862013-01-10 08:57:35 +00001985 if (dev_maps) {
1986 for_each_possible_cpu(cpu) {
1987 new_map = xmap_dereference(new_dev_maps->cpu_map[cpu]);
1988 map = xmap_dereference(dev_maps->cpu_map[cpu]);
1989 if (map && map != new_map)
1990 kfree_rcu(map, rcu);
1991 }
Alexander Duyck537c00d2013-01-10 08:57:02 +00001992
Alexander Duyck537c00d2013-01-10 08:57:02 +00001993 kfree_rcu(dev_maps, rcu);
Alexander Duyck01c5f862013-01-10 08:57:35 +00001994 }
Alexander Duyck537c00d2013-01-10 08:57:02 +00001995
Alexander Duyck01c5f862013-01-10 08:57:35 +00001996 dev_maps = new_dev_maps;
1997 active = true;
1998
1999out_no_new_maps:
2000 /* update Tx queue numa node */
Alexander Duyck537c00d2013-01-10 08:57:02 +00002001 netdev_queue_numa_node_write(netdev_get_tx_queue(dev, index),
2002 (numa_node_id >= 0) ? numa_node_id :
2003 NUMA_NO_NODE);
2004
Alexander Duyck01c5f862013-01-10 08:57:35 +00002005 if (!dev_maps)
2006 goto out_no_maps;
2007
2008 /* removes queue from unused CPUs */
2009 for_each_possible_cpu(cpu) {
2010 if (cpumask_test_cpu(cpu, mask) && cpu_online(cpu))
2011 continue;
2012
2013 if (remove_xps_queue(dev_maps, cpu, index))
2014 active = true;
2015 }
2016
2017 /* free map if not active */
2018 if (!active) {
2019 RCU_INIT_POINTER(dev->xps_maps, NULL);
2020 kfree_rcu(dev_maps, rcu);
2021 }
2022
2023out_no_maps:
Alexander Duyck537c00d2013-01-10 08:57:02 +00002024 mutex_unlock(&xps_map_mutex);
2025
2026 return 0;
2027error:
Alexander Duyck01c5f862013-01-10 08:57:35 +00002028 /* remove any maps that we added */
2029 for_each_possible_cpu(cpu) {
2030 new_map = xmap_dereference(new_dev_maps->cpu_map[cpu]);
2031 map = dev_maps ? xmap_dereference(dev_maps->cpu_map[cpu]) :
2032 NULL;
2033 if (new_map && new_map != map)
2034 kfree(new_map);
2035 }
2036
Alexander Duyck537c00d2013-01-10 08:57:02 +00002037 mutex_unlock(&xps_map_mutex);
2038
Alexander Duyck537c00d2013-01-10 08:57:02 +00002039 kfree(new_dev_maps);
2040 return -ENOMEM;
2041}
2042EXPORT_SYMBOL(netif_set_xps_queue);
2043
2044#endif
John Fastabendf0796d52010-07-01 13:21:57 +00002045/*
2046 * Routine to help set real_num_tx_queues. To avoid skbs mapped to queues
2047 * greater then real_num_tx_queues stale skbs on the qdisc must be flushed.
2048 */
Tom Herberte6484932010-10-18 18:04:39 +00002049int netif_set_real_num_tx_queues(struct net_device *dev, unsigned int txq)
John Fastabendf0796d52010-07-01 13:21:57 +00002050{
Tom Herbert1d24eb42010-11-21 13:17:27 +00002051 int rc;
2052
Tom Herberte6484932010-10-18 18:04:39 +00002053 if (txq < 1 || txq > dev->num_tx_queues)
2054 return -EINVAL;
John Fastabendf0796d52010-07-01 13:21:57 +00002055
Ben Hutchings5c565802011-02-15 19:39:21 +00002056 if (dev->reg_state == NETREG_REGISTERED ||
2057 dev->reg_state == NETREG_UNREGISTERING) {
Tom Herberte6484932010-10-18 18:04:39 +00002058 ASSERT_RTNL();
2059
Tom Herbert1d24eb42010-11-21 13:17:27 +00002060 rc = netdev_queue_update_kobjects(dev, dev->real_num_tx_queues,
2061 txq);
Tom Herbertbf264142010-11-26 08:36:09 +00002062 if (rc)
2063 return rc;
2064
John Fastabend4f57c082011-01-17 08:06:04 +00002065 if (dev->num_tc)
2066 netif_setup_tc(dev, txq);
2067
Alexander Duyck024e9672013-01-10 08:57:46 +00002068 if (txq < dev->real_num_tx_queues) {
Tom Herberte6484932010-10-18 18:04:39 +00002069 qdisc_reset_all_tx_gt(dev, txq);
Alexander Duyck024e9672013-01-10 08:57:46 +00002070#ifdef CONFIG_XPS
2071 netif_reset_xps_queues_gt(dev, txq);
2072#endif
2073 }
John Fastabendf0796d52010-07-01 13:21:57 +00002074 }
Tom Herberte6484932010-10-18 18:04:39 +00002075
2076 dev->real_num_tx_queues = txq;
2077 return 0;
John Fastabendf0796d52010-07-01 13:21:57 +00002078}
2079EXPORT_SYMBOL(netif_set_real_num_tx_queues);
Denis Vlasenko56079432006-03-29 15:57:29 -08002080
Ben Hutchings62fe0b42010-09-27 08:24:33 +00002081#ifdef CONFIG_RPS
2082/**
2083 * netif_set_real_num_rx_queues - set actual number of RX queues used
2084 * @dev: Network device
2085 * @rxq: Actual number of RX queues
2086 *
2087 * This must be called either with the rtnl_lock held or before
2088 * registration of the net device. Returns 0 on success, or a
Ben Hutchings4e7f7952010-10-08 10:33:39 -07002089 * negative error code. If called before registration, it always
2090 * succeeds.
Ben Hutchings62fe0b42010-09-27 08:24:33 +00002091 */
2092int netif_set_real_num_rx_queues(struct net_device *dev, unsigned int rxq)
2093{
2094 int rc;
2095
Tom Herbertbd25fa72010-10-18 18:00:16 +00002096 if (rxq < 1 || rxq > dev->num_rx_queues)
2097 return -EINVAL;
2098
Ben Hutchings62fe0b42010-09-27 08:24:33 +00002099 if (dev->reg_state == NETREG_REGISTERED) {
2100 ASSERT_RTNL();
2101
Ben Hutchings62fe0b42010-09-27 08:24:33 +00002102 rc = net_rx_queue_update_kobjects(dev, dev->real_num_rx_queues,
2103 rxq);
2104 if (rc)
2105 return rc;
Ben Hutchings62fe0b42010-09-27 08:24:33 +00002106 }
2107
2108 dev->real_num_rx_queues = rxq;
2109 return 0;
2110}
2111EXPORT_SYMBOL(netif_set_real_num_rx_queues);
2112#endif
2113
Ben Hutchings2c530402012-07-10 10:55:09 +00002114/**
2115 * netif_get_num_default_rss_queues - default number of RSS queues
Yuval Mintz16917b82012-07-01 03:18:50 +00002116 *
2117 * This routine should set an upper limit on the number of RSS queues
2118 * used by default by multiqueue devices.
2119 */
Ben Hutchingsa55b1382012-07-10 10:54:38 +00002120int netif_get_num_default_rss_queues(void)
Yuval Mintz16917b82012-07-01 03:18:50 +00002121{
2122 return min_t(int, DEFAULT_MAX_NUM_RSS_QUEUES, num_online_cpus());
2123}
2124EXPORT_SYMBOL(netif_get_num_default_rss_queues);
2125
Jarek Poplawskidef82a12008-08-17 21:54:43 -07002126static inline void __netif_reschedule(struct Qdisc *q)
2127{
2128 struct softnet_data *sd;
2129 unsigned long flags;
2130
2131 local_irq_save(flags);
2132 sd = &__get_cpu_var(softnet_data);
Changli Gaoa9cbd582010-04-26 23:06:24 +00002133 q->next_sched = NULL;
2134 *sd->output_queue_tailp = q;
2135 sd->output_queue_tailp = &q->next_sched;
Jarek Poplawskidef82a12008-08-17 21:54:43 -07002136 raise_softirq_irqoff(NET_TX_SOFTIRQ);
2137 local_irq_restore(flags);
2138}
2139
David S. Miller37437bb2008-07-16 02:15:04 -07002140void __netif_schedule(struct Qdisc *q)
Denis Vlasenko56079432006-03-29 15:57:29 -08002141{
Jarek Poplawskidef82a12008-08-17 21:54:43 -07002142 if (!test_and_set_bit(__QDISC_STATE_SCHED, &q->state))
2143 __netif_reschedule(q);
Denis Vlasenko56079432006-03-29 15:57:29 -08002144}
2145EXPORT_SYMBOL(__netif_schedule);
2146
Stephen Hemmingerbea33482007-10-03 16:41:36 -07002147void dev_kfree_skb_irq(struct sk_buff *skb)
Denis Vlasenko56079432006-03-29 15:57:29 -08002148{
David S. Miller3578b0c2010-08-03 00:24:04 -07002149 if (atomic_dec_and_test(&skb->users)) {
Stephen Hemmingerbea33482007-10-03 16:41:36 -07002150 struct softnet_data *sd;
2151 unsigned long flags;
Denis Vlasenko56079432006-03-29 15:57:29 -08002152
Stephen Hemmingerbea33482007-10-03 16:41:36 -07002153 local_irq_save(flags);
2154 sd = &__get_cpu_var(softnet_data);
2155 skb->next = sd->completion_queue;
2156 sd->completion_queue = skb;
2157 raise_softirq_irqoff(NET_TX_SOFTIRQ);
2158 local_irq_restore(flags);
2159 }
Denis Vlasenko56079432006-03-29 15:57:29 -08002160}
Stephen Hemmingerbea33482007-10-03 16:41:36 -07002161EXPORT_SYMBOL(dev_kfree_skb_irq);
Denis Vlasenko56079432006-03-29 15:57:29 -08002162
2163void dev_kfree_skb_any(struct sk_buff *skb)
2164{
2165 if (in_irq() || irqs_disabled())
2166 dev_kfree_skb_irq(skb);
2167 else
2168 dev_kfree_skb(skb);
2169}
2170EXPORT_SYMBOL(dev_kfree_skb_any);
2171
2172
Stephen Hemmingerbea33482007-10-03 16:41:36 -07002173/**
2174 * netif_device_detach - mark device as removed
2175 * @dev: network device
2176 *
2177 * Mark device as removed from system and therefore no longer available.
2178 */
Denis Vlasenko56079432006-03-29 15:57:29 -08002179void netif_device_detach(struct net_device *dev)
2180{
2181 if (test_and_clear_bit(__LINK_STATE_PRESENT, &dev->state) &&
2182 netif_running(dev)) {
Alexander Duyckd5431032009-04-08 13:15:22 +00002183 netif_tx_stop_all_queues(dev);
Denis Vlasenko56079432006-03-29 15:57:29 -08002184 }
2185}
2186EXPORT_SYMBOL(netif_device_detach);
2187
Stephen Hemmingerbea33482007-10-03 16:41:36 -07002188/**
2189 * netif_device_attach - mark device as attached
2190 * @dev: network device
2191 *
2192 * Mark device as attached from system and restart if needed.
2193 */
Denis Vlasenko56079432006-03-29 15:57:29 -08002194void netif_device_attach(struct net_device *dev)
2195{
2196 if (!test_and_set_bit(__LINK_STATE_PRESENT, &dev->state) &&
2197 netif_running(dev)) {
Alexander Duyckd5431032009-04-08 13:15:22 +00002198 netif_tx_wake_all_queues(dev);
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09002199 __netdev_watchdog_up(dev);
Denis Vlasenko56079432006-03-29 15:57:29 -08002200 }
2201}
2202EXPORT_SYMBOL(netif_device_attach);
2203
Ben Hutchings36c92472012-01-17 07:57:56 +00002204static void skb_warn_bad_offload(const struct sk_buff *skb)
2205{
Michał Mirosław65e9d2f2012-01-17 10:00:40 +00002206 static const netdev_features_t null_features = 0;
Ben Hutchings36c92472012-01-17 07:57:56 +00002207 struct net_device *dev = skb->dev;
2208 const char *driver = "";
2209
Ben Greearc846ad92013-04-19 10:45:52 +00002210 if (!net_ratelimit())
2211 return;
2212
Ben Hutchings36c92472012-01-17 07:57:56 +00002213 if (dev && dev->dev.parent)
2214 driver = dev_driver_string(dev->dev.parent);
2215
2216 WARN(1, "%s: caps=(%pNF, %pNF) len=%d data_len=%d gso_size=%d "
2217 "gso_type=%d ip_summed=%d\n",
Michał Mirosław65e9d2f2012-01-17 10:00:40 +00002218 driver, dev ? &dev->features : &null_features,
2219 skb->sk ? &skb->sk->sk_route_caps : &null_features,
Ben Hutchings36c92472012-01-17 07:57:56 +00002220 skb->len, skb->data_len, skb_shinfo(skb)->gso_size,
2221 skb_shinfo(skb)->gso_type, skb->ip_summed);
2222}
2223
Linus Torvalds1da177e2005-04-16 15:20:36 -07002224/*
2225 * Invalidate hardware checksum when packet is to be mangled, and
2226 * complete checksum manually on outgoing path.
2227 */
Patrick McHardy84fa7932006-08-29 16:44:56 -07002228int skb_checksum_help(struct sk_buff *skb)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002229{
Al Virod3bc23e2006-11-14 21:24:49 -08002230 __wsum csum;
Herbert Xu663ead32007-04-09 11:59:07 -07002231 int ret = 0, offset;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002232
Patrick McHardy84fa7932006-08-29 16:44:56 -07002233 if (skb->ip_summed == CHECKSUM_COMPLETE)
Herbert Xua430a432006-07-08 13:34:56 -07002234 goto out_set_summed;
2235
2236 if (unlikely(skb_shinfo(skb)->gso_size)) {
Ben Hutchings36c92472012-01-17 07:57:56 +00002237 skb_warn_bad_offload(skb);
2238 return -EINVAL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002239 }
2240
Eric Dumazetcef401d2013-01-25 20:34:37 +00002241 /* Before computing a checksum, we should make sure no frag could
2242 * be modified by an external entity : checksum could be wrong.
2243 */
2244 if (skb_has_shared_frag(skb)) {
2245 ret = __skb_linearize(skb);
2246 if (ret)
2247 goto out;
2248 }
2249
Michał Mirosław55508d62010-12-14 15:24:08 +00002250 offset = skb_checksum_start_offset(skb);
Herbert Xua0308472007-10-15 01:47:15 -07002251 BUG_ON(offset >= skb_headlen(skb));
2252 csum = skb_checksum(skb, offset, skb->len - offset, 0);
2253
2254 offset += skb->csum_offset;
2255 BUG_ON(offset + sizeof(__sum16) > skb_headlen(skb));
2256
2257 if (skb_cloned(skb) &&
2258 !skb_clone_writable(skb, offset + sizeof(__sum16))) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07002259 ret = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
2260 if (ret)
2261 goto out;
2262 }
2263
Herbert Xua0308472007-10-15 01:47:15 -07002264 *(__sum16 *)(skb->data + offset) = csum_fold(csum);
Herbert Xua430a432006-07-08 13:34:56 -07002265out_set_summed:
Linus Torvalds1da177e2005-04-16 15:20:36 -07002266 skb->ip_summed = CHECKSUM_NONE;
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09002267out:
Linus Torvalds1da177e2005-04-16 15:20:36 -07002268 return ret;
2269}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07002270EXPORT_SYMBOL(skb_checksum_help);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002271
Pravin B Shelarec5f0612013-03-07 09:28:01 +00002272__be16 skb_network_protocol(struct sk_buff *skb)
2273{
2274 __be16 type = skb->protocol;
David S. Miller61816592013-03-20 12:46:26 -04002275 int vlan_depth = ETH_HLEN;
Pravin B Shelarec5f0612013-03-07 09:28:01 +00002276
Pravin B Shelar19acc322013-05-07 20:41:07 +00002277 /* Tunnel gso handlers can set protocol to ethernet. */
2278 if (type == htons(ETH_P_TEB)) {
2279 struct ethhdr *eth;
2280
2281 if (unlikely(!pskb_may_pull(skb, sizeof(struct ethhdr))))
2282 return 0;
2283
2284 eth = (struct ethhdr *)skb_mac_header(skb);
2285 type = eth->h_proto;
2286 }
2287
Patrick McHardy8ad227f2013-04-19 02:04:31 +00002288 while (type == htons(ETH_P_8021Q) || type == htons(ETH_P_8021AD)) {
Pravin B Shelarec5f0612013-03-07 09:28:01 +00002289 struct vlan_hdr *vh;
2290
2291 if (unlikely(!pskb_may_pull(skb, vlan_depth + VLAN_HLEN)))
2292 return 0;
2293
2294 vh = (struct vlan_hdr *)(skb->data + vlan_depth);
2295 type = vh->h_vlan_encapsulated_proto;
2296 vlan_depth += VLAN_HLEN;
2297 }
2298
2299 return type;
2300}
2301
Pravin B Shelar05e8ef42013-02-14 09:44:55 +00002302/**
2303 * skb_mac_gso_segment - mac layer segmentation handler.
2304 * @skb: buffer to segment
2305 * @features: features for the output path (see dev->features)
2306 */
2307struct sk_buff *skb_mac_gso_segment(struct sk_buff *skb,
2308 netdev_features_t features)
2309{
2310 struct sk_buff *segs = ERR_PTR(-EPROTONOSUPPORT);
2311 struct packet_offload *ptype;
Pravin B Shelarec5f0612013-03-07 09:28:01 +00002312 __be16 type = skb_network_protocol(skb);
Pravin B Shelar05e8ef42013-02-14 09:44:55 +00002313
Pravin B Shelarec5f0612013-03-07 09:28:01 +00002314 if (unlikely(!type))
2315 return ERR_PTR(-EINVAL);
Pravin B Shelar05e8ef42013-02-14 09:44:55 +00002316
2317 __skb_pull(skb, skb->mac_len);
2318
2319 rcu_read_lock();
2320 list_for_each_entry_rcu(ptype, &offload_base, list) {
2321 if (ptype->type == type && ptype->callbacks.gso_segment) {
2322 if (unlikely(skb->ip_summed != CHECKSUM_PARTIAL)) {
2323 int err;
2324
2325 err = ptype->callbacks.gso_send_check(skb);
2326 segs = ERR_PTR(err);
2327 if (err || skb_gso_ok(skb, features))
2328 break;
2329 __skb_push(skb, (skb->data -
2330 skb_network_header(skb)));
2331 }
2332 segs = ptype->callbacks.gso_segment(skb, features);
2333 break;
2334 }
2335 }
2336 rcu_read_unlock();
2337
2338 __skb_push(skb, skb->data - skb_mac_header(skb));
2339
2340 return segs;
2341}
2342EXPORT_SYMBOL(skb_mac_gso_segment);
2343
2344
Cong Wang12b00042013-02-05 16:36:38 +00002345/* openvswitch calls this on rx path, so we need a different check.
2346 */
2347static inline bool skb_needs_check(struct sk_buff *skb, bool tx_path)
2348{
2349 if (tx_path)
2350 return skb->ip_summed != CHECKSUM_PARTIAL;
2351 else
2352 return skb->ip_summed == CHECKSUM_NONE;
2353}
2354
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002355/**
Cong Wang12b00042013-02-05 16:36:38 +00002356 * __skb_gso_segment - Perform segmentation on skb.
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002357 * @skb: buffer to segment
Herbert Xu576a30e2006-06-27 13:22:38 -07002358 * @features: features for the output path (see dev->features)
Cong Wang12b00042013-02-05 16:36:38 +00002359 * @tx_path: whether it is called in TX path
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002360 *
2361 * This function segments the given skb and returns a list of segments.
Herbert Xu576a30e2006-06-27 13:22:38 -07002362 *
2363 * It may return NULL if the skb requires no segmentation. This is
2364 * only possible when GSO is used for verifying header integrity.
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002365 */
Cong Wang12b00042013-02-05 16:36:38 +00002366struct sk_buff *__skb_gso_segment(struct sk_buff *skb,
2367 netdev_features_t features, bool tx_path)
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002368{
Cong Wang12b00042013-02-05 16:36:38 +00002369 if (unlikely(skb_needs_check(skb, tx_path))) {
Pravin B Shelar05e8ef42013-02-14 09:44:55 +00002370 int err;
2371
Ben Hutchings36c92472012-01-17 07:57:56 +00002372 skb_warn_bad_offload(skb);
Herbert Xu67fd1a72009-01-19 16:26:44 -08002373
Herbert Xua430a432006-07-08 13:34:56 -07002374 if (skb_header_cloned(skb) &&
2375 (err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC)))
2376 return ERR_PTR(err);
2377 }
2378
Pravin B Shelar68c33162013-02-14 14:02:41 +00002379 SKB_GSO_CB(skb)->mac_offset = skb_headroom(skb);
Pravin B Shelar05e8ef42013-02-14 09:44:55 +00002380 skb_reset_mac_header(skb);
2381 skb_reset_mac_len(skb);
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002382
Pravin B Shelar05e8ef42013-02-14 09:44:55 +00002383 return skb_mac_gso_segment(skb, features);
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002384}
Cong Wang12b00042013-02-05 16:36:38 +00002385EXPORT_SYMBOL(__skb_gso_segment);
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002386
Herbert Xufb286bb2005-11-10 13:01:24 -08002387/* Take action when hardware reception checksum errors are detected. */
2388#ifdef CONFIG_BUG
2389void netdev_rx_csum_fault(struct net_device *dev)
2390{
2391 if (net_ratelimit()) {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00002392 pr_err("%s: hw csum failure\n", dev ? dev->name : "<unknown>");
Herbert Xufb286bb2005-11-10 13:01:24 -08002393 dump_stack();
2394 }
2395}
2396EXPORT_SYMBOL(netdev_rx_csum_fault);
2397#endif
2398
Linus Torvalds1da177e2005-04-16 15:20:36 -07002399/* Actually, we should eliminate this check as soon as we know, that:
2400 * 1. IOMMU is present and allows to map all the memory.
2401 * 2. No high memory really exists on this machine.
2402 */
2403
Eric Dumazet9092c652010-04-02 13:34:49 -07002404static int illegal_highdma(struct net_device *dev, struct sk_buff *skb)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002405{
Herbert Xu3d3a8532006-06-27 13:33:10 -07002406#ifdef CONFIG_HIGHMEM
Linus Torvalds1da177e2005-04-16 15:20:36 -07002407 int i;
FUJITA Tomonori5acbbd42010-03-30 22:35:50 +00002408 if (!(dev->features & NETIF_F_HIGHDMA)) {
Ian Campbellea2ab692011-08-22 23:44:58 +00002409 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
2410 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
2411 if (PageHighMem(skb_frag_page(frag)))
FUJITA Tomonori5acbbd42010-03-30 22:35:50 +00002412 return 1;
Ian Campbellea2ab692011-08-22 23:44:58 +00002413 }
FUJITA Tomonori5acbbd42010-03-30 22:35:50 +00002414 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002415
FUJITA Tomonori5acbbd42010-03-30 22:35:50 +00002416 if (PCI_DMA_BUS_IS_PHYS) {
2417 struct device *pdev = dev->dev.parent;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002418
Eric Dumazet9092c652010-04-02 13:34:49 -07002419 if (!pdev)
2420 return 0;
FUJITA Tomonori5acbbd42010-03-30 22:35:50 +00002421 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
Ian Campbellea2ab692011-08-22 23:44:58 +00002422 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
2423 dma_addr_t addr = page_to_phys(skb_frag_page(frag));
FUJITA Tomonori5acbbd42010-03-30 22:35:50 +00002424 if (!pdev->dma_mask || addr + PAGE_SIZE - 1 > *pdev->dma_mask)
2425 return 1;
2426 }
2427 }
Herbert Xu3d3a8532006-06-27 13:33:10 -07002428#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07002429 return 0;
2430}
Linus Torvalds1da177e2005-04-16 15:20:36 -07002431
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002432struct dev_gso_cb {
2433 void (*destructor)(struct sk_buff *skb);
2434};
2435
2436#define DEV_GSO_CB(skb) ((struct dev_gso_cb *)(skb)->cb)
2437
2438static void dev_gso_skb_destructor(struct sk_buff *skb)
2439{
2440 struct dev_gso_cb *cb;
2441
2442 do {
2443 struct sk_buff *nskb = skb->next;
2444
2445 skb->next = nskb->next;
2446 nskb->next = NULL;
2447 kfree_skb(nskb);
2448 } while (skb->next);
2449
2450 cb = DEV_GSO_CB(skb);
2451 if (cb->destructor)
2452 cb->destructor(skb);
2453}
2454
2455/**
2456 * dev_gso_segment - Perform emulated hardware segmentation on skb.
2457 * @skb: buffer to segment
Jesse Gross91ecb632011-01-09 06:23:33 +00002458 * @features: device features as applicable to this skb
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002459 *
2460 * This function segments the given skb and stores the list of segments
2461 * in skb->next.
2462 */
Michał Mirosławc8f44af2011-11-15 15:29:55 +00002463static int dev_gso_segment(struct sk_buff *skb, netdev_features_t features)
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002464{
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002465 struct sk_buff *segs;
2466
Herbert Xu576a30e2006-06-27 13:22:38 -07002467 segs = skb_gso_segment(skb, features);
2468
2469 /* Verifying header integrity only. */
2470 if (!segs)
2471 return 0;
2472
Hirofumi Nakagawa801678c2008-04-29 01:03:09 -07002473 if (IS_ERR(segs))
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002474 return PTR_ERR(segs);
2475
2476 skb->next = segs;
2477 DEV_GSO_CB(skb)->destructor = skb->destructor;
2478 skb->destructor = dev_gso_skb_destructor;
2479
2480 return 0;
2481}
2482
Michał Mirosławc8f44af2011-11-15 15:29:55 +00002483static netdev_features_t harmonize_features(struct sk_buff *skb,
Alexander Duyckcdbaa0b2013-07-10 17:05:06 -07002484 netdev_features_t features)
Jesse Grossf01a5232011-01-09 06:23:31 +00002485{
Ed Cashinc0d680e2012-09-19 15:49:00 +00002486 if (skb->ip_summed != CHECKSUM_NONE &&
Alexander Duyckcdbaa0b2013-07-10 17:05:06 -07002487 !can_checksum_protocol(features, skb_network_protocol(skb))) {
Jesse Grossf01a5232011-01-09 06:23:31 +00002488 features &= ~NETIF_F_ALL_CSUM;
Jesse Grossf01a5232011-01-09 06:23:31 +00002489 } else if (illegal_highdma(skb->dev, skb)) {
2490 features &= ~NETIF_F_SG;
2491 }
2492
2493 return features;
2494}
2495
Michał Mirosławc8f44af2011-11-15 15:29:55 +00002496netdev_features_t netif_skb_features(struct sk_buff *skb)
Jesse Gross58e998c2010-10-29 12:14:55 +00002497{
2498 __be16 protocol = skb->protocol;
Michał Mirosławc8f44af2011-11-15 15:29:55 +00002499 netdev_features_t features = skb->dev->features;
Jesse Gross58e998c2010-10-29 12:14:55 +00002500
Ben Hutchings30b678d2012-07-30 15:57:00 +00002501 if (skb_shinfo(skb)->gso_segs > skb->dev->gso_max_segs)
2502 features &= ~NETIF_F_GSO_MASK;
2503
Patrick McHardy8ad227f2013-04-19 02:04:31 +00002504 if (protocol == htons(ETH_P_8021Q) || protocol == htons(ETH_P_8021AD)) {
Jesse Gross58e998c2010-10-29 12:14:55 +00002505 struct vlan_ethhdr *veh = (struct vlan_ethhdr *)skb->data;
2506 protocol = veh->h_vlan_encapsulated_proto;
Jesse Grossf01a5232011-01-09 06:23:31 +00002507 } else if (!vlan_tx_tag_present(skb)) {
Alexander Duyckcdbaa0b2013-07-10 17:05:06 -07002508 return harmonize_features(skb, features);
Jesse Grossf01a5232011-01-09 06:23:31 +00002509 }
Jesse Gross58e998c2010-10-29 12:14:55 +00002510
Patrick McHardy8ad227f2013-04-19 02:04:31 +00002511 features &= (skb->dev->vlan_features | NETIF_F_HW_VLAN_CTAG_TX |
2512 NETIF_F_HW_VLAN_STAG_TX);
Jesse Grossf01a5232011-01-09 06:23:31 +00002513
Alexander Duyckcdbaa0b2013-07-10 17:05:06 -07002514 if (protocol == htons(ETH_P_8021Q) || protocol == htons(ETH_P_8021AD))
Jesse Grossf01a5232011-01-09 06:23:31 +00002515 features &= NETIF_F_SG | NETIF_F_HIGHDMA | NETIF_F_FRAGLIST |
Patrick McHardy8ad227f2013-04-19 02:04:31 +00002516 NETIF_F_GEN_CSUM | NETIF_F_HW_VLAN_CTAG_TX |
2517 NETIF_F_HW_VLAN_STAG_TX;
Alexander Duyckcdbaa0b2013-07-10 17:05:06 -07002518
2519 return harmonize_features(skb, features);
Jesse Gross58e998c2010-10-29 12:14:55 +00002520}
Jesse Grossf01a5232011-01-09 06:23:31 +00002521EXPORT_SYMBOL(netif_skb_features);
Jesse Gross58e998c2010-10-29 12:14:55 +00002522
John Fastabend6afff0c2010-06-16 14:18:12 +00002523/*
2524 * Returns true if either:
2525 * 1. skb has frag_list and the device doesn't support FRAGLIST, or
Rami Rosend1a53df2012-08-27 23:39:24 +00002526 * 2. skb is fragmented and the device does not support SG.
John Fastabend6afff0c2010-06-16 14:18:12 +00002527 */
2528static inline int skb_needs_linearize(struct sk_buff *skb,
Patrick McHardy6708c9e2013-05-01 22:36:49 +00002529 netdev_features_t features)
John Fastabend6afff0c2010-06-16 14:18:12 +00002530{
Jesse Gross02932ce2011-01-09 06:23:34 +00002531 return skb_is_nonlinear(skb) &&
2532 ((skb_has_frag_list(skb) &&
2533 !(features & NETIF_F_FRAGLIST)) ||
Jesse Grosse1e78db2010-10-29 12:14:53 +00002534 (skb_shinfo(skb)->nr_frags &&
Jesse Gross02932ce2011-01-09 06:23:34 +00002535 !(features & NETIF_F_SG)));
John Fastabend6afff0c2010-06-16 14:18:12 +00002536}
2537
David S. Millerfd2ea0a2008-07-17 01:56:23 -07002538int dev_hard_start_xmit(struct sk_buff *skb, struct net_device *dev,
2539 struct netdev_queue *txq)
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002540{
Stephen Hemminger00829822008-11-20 20:14:53 -08002541 const struct net_device_ops *ops = dev->netdev_ops;
Patrick McHardy572a9d72009-11-10 06:14:14 +00002542 int rc = NETDEV_TX_OK;
Koki Sanagiec764bf2011-05-30 21:48:34 +00002543 unsigned int skb_len;
Stephen Hemminger00829822008-11-20 20:14:53 -08002544
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002545 if (likely(!skb->next)) {
Michał Mirosławc8f44af2011-11-15 15:29:55 +00002546 netdev_features_t features;
Jesse Grossfc741212011-01-09 06:23:32 +00002547
Eric Dumazet93f154b2009-05-18 22:19:19 -07002548 /*
Lucas De Marchi25985ed2011-03-30 22:57:33 -03002549 * If device doesn't need skb->dst, release it right now while
Eric Dumazet93f154b2009-05-18 22:19:19 -07002550 * its hot in this cpu cache
2551 */
Eric Dumazetadf30902009-06-02 05:19:30 +00002552 if (dev->priv_flags & IFF_XMIT_DST_RELEASE)
2553 skb_dst_drop(skb);
2554
Jesse Grossfc741212011-01-09 06:23:32 +00002555 features = netif_skb_features(skb);
2556
Jesse Gross7b9c6092010-10-20 13:56:04 +00002557 if (vlan_tx_tag_present(skb) &&
Patrick McHardy86a9bad2013-04-19 02:04:30 +00002558 !vlan_hw_offload_capable(features, skb->vlan_proto)) {
2559 skb = __vlan_put_tag(skb, skb->vlan_proto,
2560 vlan_tx_tag_get(skb));
Jesse Gross7b9c6092010-10-20 13:56:04 +00002561 if (unlikely(!skb))
2562 goto out;
2563
2564 skb->vlan_tci = 0;
2565 }
2566
Alexander Duyckfc70fb62012-12-07 14:14:15 +00002567 /* If encapsulation offload request, verify we are testing
2568 * hardware encapsulation features instead of standard
2569 * features for the netdev
2570 */
2571 if (skb->encapsulation)
2572 features &= dev->hw_enc_features;
2573
Jesse Grossfc741212011-01-09 06:23:32 +00002574 if (netif_needs_gso(skb, features)) {
Jesse Gross91ecb632011-01-09 06:23:33 +00002575 if (unlikely(dev_gso_segment(skb, features)))
David S. Miller9ccb8972010-04-22 01:02:07 -07002576 goto out_kfree_skb;
2577 if (skb->next)
2578 goto gso;
John Fastabend6afff0c2010-06-16 14:18:12 +00002579 } else {
Jesse Gross02932ce2011-01-09 06:23:34 +00002580 if (skb_needs_linearize(skb, features) &&
John Fastabend6afff0c2010-06-16 14:18:12 +00002581 __skb_linearize(skb))
2582 goto out_kfree_skb;
2583
2584 /* If packet is not checksummed and device does not
2585 * support checksumming for this protocol, complete
2586 * checksumming here.
2587 */
2588 if (skb->ip_summed == CHECKSUM_PARTIAL) {
Alexander Duyckfc70fb62012-12-07 14:14:15 +00002589 if (skb->encapsulation)
2590 skb_set_inner_transport_header(skb,
2591 skb_checksum_start_offset(skb));
2592 else
2593 skb_set_transport_header(skb,
2594 skb_checksum_start_offset(skb));
Jesse Gross03634662011-01-09 06:23:35 +00002595 if (!(features & NETIF_F_ALL_CSUM) &&
John Fastabend6afff0c2010-06-16 14:18:12 +00002596 skb_checksum_help(skb))
2597 goto out_kfree_skb;
2598 }
David S. Miller9ccb8972010-04-22 01:02:07 -07002599 }
2600
Eric Dumazetb40863c2012-09-18 20:44:49 +00002601 if (!list_empty(&ptype_all))
2602 dev_queue_xmit_nit(skb, dev);
2603
Koki Sanagiec764bf2011-05-30 21:48:34 +00002604 skb_len = skb->len;
Patrick Ohlyac45f602009-02-12 05:03:37 +00002605 rc = ops->ndo_start_xmit(skb, dev);
Koki Sanagiec764bf2011-05-30 21:48:34 +00002606 trace_net_dev_xmit(skb, rc, dev, skb_len);
Patrick McHardyec634fe2009-07-05 19:23:38 -07002607 if (rc == NETDEV_TX_OK)
Eric Dumazet08baf562009-05-25 22:58:01 -07002608 txq_trans_update(txq);
Patrick Ohlyac45f602009-02-12 05:03:37 +00002609 return rc;
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002610 }
2611
Herbert Xu576a30e2006-06-27 13:22:38 -07002612gso:
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002613 do {
2614 struct sk_buff *nskb = skb->next;
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002615
2616 skb->next = nskb->next;
2617 nskb->next = NULL;
Krishna Kumar068a2de2009-12-09 20:59:58 +00002618
Eric Dumazetb40863c2012-09-18 20:44:49 +00002619 if (!list_empty(&ptype_all))
2620 dev_queue_xmit_nit(nskb, dev);
2621
Koki Sanagiec764bf2011-05-30 21:48:34 +00002622 skb_len = nskb->len;
Stephen Hemminger00829822008-11-20 20:14:53 -08002623 rc = ops->ndo_start_xmit(nskb, dev);
Koki Sanagiec764bf2011-05-30 21:48:34 +00002624 trace_net_dev_xmit(nskb, rc, dev, skb_len);
Patrick McHardyec634fe2009-07-05 19:23:38 -07002625 if (unlikely(rc != NETDEV_TX_OK)) {
Patrick McHardy572a9d72009-11-10 06:14:14 +00002626 if (rc & ~NETDEV_TX_MASK)
2627 goto out_kfree_gso_skb;
Michael Chanf54d9e82006-06-25 23:57:04 -07002628 nskb->next = skb->next;
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002629 skb->next = nskb;
2630 return rc;
2631 }
Eric Dumazet08baf562009-05-25 22:58:01 -07002632 txq_trans_update(txq);
Tom Herbert734664982011-11-28 16:32:44 +00002633 if (unlikely(netif_xmit_stopped(txq) && skb->next))
Michael Chanf54d9e82006-06-25 23:57:04 -07002634 return NETDEV_TX_BUSY;
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002635 } while (skb->next);
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09002636
Patrick McHardy572a9d72009-11-10 06:14:14 +00002637out_kfree_gso_skb:
Sridhar Samudrala0c772152013-04-29 13:02:42 +00002638 if (likely(skb->next == NULL)) {
Patrick McHardy572a9d72009-11-10 06:14:14 +00002639 skb->destructor = DEV_GSO_CB(skb)->destructor;
Sridhar Samudrala0c772152013-04-29 13:02:42 +00002640 consume_skb(skb);
2641 return rc;
2642 }
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002643out_kfree_skb:
2644 kfree_skb(skb);
Jesse Gross7b9c6092010-10-20 13:56:04 +00002645out:
Patrick McHardy572a9d72009-11-10 06:14:14 +00002646 return rc;
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002647}
2648
Eric Dumazet1def9232013-01-10 12:36:42 +00002649static void qdisc_pkt_len_init(struct sk_buff *skb)
2650{
2651 const struct skb_shared_info *shinfo = skb_shinfo(skb);
2652
2653 qdisc_skb_cb(skb)->pkt_len = skb->len;
2654
2655 /* To get more precise estimation of bytes sent on wire,
2656 * we add to pkt_len the headers size of all segments
2657 */
2658 if (shinfo->gso_size) {
Eric Dumazet757b8b12013-01-15 21:14:21 -08002659 unsigned int hdr_len;
Jason Wang15e5a032013-03-25 20:19:59 +00002660 u16 gso_segs = shinfo->gso_segs;
Eric Dumazet1def9232013-01-10 12:36:42 +00002661
Eric Dumazet757b8b12013-01-15 21:14:21 -08002662 /* mac layer + network layer */
2663 hdr_len = skb_transport_header(skb) - skb_mac_header(skb);
2664
2665 /* + transport layer */
Eric Dumazet1def9232013-01-10 12:36:42 +00002666 if (likely(shinfo->gso_type & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6)))
2667 hdr_len += tcp_hdrlen(skb);
2668 else
2669 hdr_len += sizeof(struct udphdr);
Jason Wang15e5a032013-03-25 20:19:59 +00002670
2671 if (shinfo->gso_type & SKB_GSO_DODGY)
2672 gso_segs = DIV_ROUND_UP(skb->len - hdr_len,
2673 shinfo->gso_size);
2674
2675 qdisc_skb_cb(skb)->pkt_len += (gso_segs - 1) * hdr_len;
Eric Dumazet1def9232013-01-10 12:36:42 +00002676 }
2677}
2678
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00002679static inline int __dev_xmit_skb(struct sk_buff *skb, struct Qdisc *q,
2680 struct net_device *dev,
2681 struct netdev_queue *txq)
2682{
2683 spinlock_t *root_lock = qdisc_lock(q);
Eric Dumazeta2da5702011-01-20 03:48:19 +00002684 bool contended;
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00002685 int rc;
2686
Eric Dumazet1def9232013-01-10 12:36:42 +00002687 qdisc_pkt_len_init(skb);
Eric Dumazeta2da5702011-01-20 03:48:19 +00002688 qdisc_calculate_pkt_len(skb, q);
Eric Dumazet79640a42010-06-02 05:09:29 -07002689 /*
2690 * Heuristic to force contended enqueues to serialize on a
2691 * separate lock before trying to get qdisc main lock.
2692 * This permits __QDISC_STATE_RUNNING owner to get the lock more often
2693 * and dequeue packets faster.
2694 */
Eric Dumazeta2da5702011-01-20 03:48:19 +00002695 contended = qdisc_is_running(q);
Eric Dumazet79640a42010-06-02 05:09:29 -07002696 if (unlikely(contended))
2697 spin_lock(&q->busylock);
2698
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00002699 spin_lock(root_lock);
2700 if (unlikely(test_bit(__QDISC_STATE_DEACTIVATED, &q->state))) {
2701 kfree_skb(skb);
2702 rc = NET_XMIT_DROP;
2703 } else if ((q->flags & TCQ_F_CAN_BYPASS) && !qdisc_qlen(q) &&
Eric Dumazetbc135b22010-06-02 03:23:51 -07002704 qdisc_run_begin(q)) {
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00002705 /*
2706 * This is a work-conserving queue; there are no old skbs
2707 * waiting to be sent out; and the qdisc is not running -
2708 * xmit the skb directly.
2709 */
Eric Dumazet7fee2262010-05-11 23:19:48 +00002710 if (!(dev->priv_flags & IFF_XMIT_DST_RELEASE))
2711 skb_dst_force(skb);
Eric Dumazetbfe0d022011-01-09 08:30:54 +00002712
Eric Dumazetbfe0d022011-01-09 08:30:54 +00002713 qdisc_bstats_update(q, skb);
2714
Eric Dumazet79640a42010-06-02 05:09:29 -07002715 if (sch_direct_xmit(skb, q, dev, txq, root_lock)) {
2716 if (unlikely(contended)) {
2717 spin_unlock(&q->busylock);
2718 contended = false;
2719 }
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00002720 __qdisc_run(q);
Eric Dumazet79640a42010-06-02 05:09:29 -07002721 } else
Eric Dumazetbc135b22010-06-02 03:23:51 -07002722 qdisc_run_end(q);
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00002723
2724 rc = NET_XMIT_SUCCESS;
2725 } else {
Eric Dumazet7fee2262010-05-11 23:19:48 +00002726 skb_dst_force(skb);
Eric Dumazeta2da5702011-01-20 03:48:19 +00002727 rc = q->enqueue(skb, q) & NET_XMIT_MASK;
Eric Dumazet79640a42010-06-02 05:09:29 -07002728 if (qdisc_run_begin(q)) {
2729 if (unlikely(contended)) {
2730 spin_unlock(&q->busylock);
2731 contended = false;
2732 }
2733 __qdisc_run(q);
2734 }
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00002735 }
2736 spin_unlock(root_lock);
Eric Dumazet79640a42010-06-02 05:09:29 -07002737 if (unlikely(contended))
2738 spin_unlock(&q->busylock);
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00002739 return rc;
2740}
2741
Neil Horman5bc14212011-11-22 05:10:51 +00002742#if IS_ENABLED(CONFIG_NETPRIO_CGROUP)
2743static void skb_update_prio(struct sk_buff *skb)
2744{
Igor Maravic6977a792011-11-25 07:44:54 +00002745 struct netprio_map *map = rcu_dereference_bh(skb->dev->priomap);
Neil Horman5bc14212011-11-22 05:10:51 +00002746
Eric Dumazet91c68ce2012-07-08 21:45:10 +00002747 if (!skb->priority && skb->sk && map) {
2748 unsigned int prioidx = skb->sk->sk_cgrp_prioidx;
2749
2750 if (prioidx < map->priomap_len)
2751 skb->priority = map->priomap[prioidx];
2752 }
Neil Horman5bc14212011-11-22 05:10:51 +00002753}
2754#else
2755#define skb_update_prio(skb)
2756#endif
2757
Eric Dumazet745e20f2010-09-29 13:23:09 -07002758static DEFINE_PER_CPU(int, xmit_recursion);
David S. Miller11a766c2010-10-25 12:51:55 -07002759#define RECURSION_LIMIT 10
Eric Dumazet745e20f2010-09-29 13:23:09 -07002760
Dave Jonesd29f7492008-07-22 14:09:06 -07002761/**
Michel Machado95603e22012-06-12 10:16:35 +00002762 * dev_loopback_xmit - loop back @skb
2763 * @skb: buffer to transmit
2764 */
2765int dev_loopback_xmit(struct sk_buff *skb)
2766{
2767 skb_reset_mac_header(skb);
2768 __skb_pull(skb, skb_network_offset(skb));
2769 skb->pkt_type = PACKET_LOOPBACK;
2770 skb->ip_summed = CHECKSUM_UNNECESSARY;
2771 WARN_ON(!skb_dst(skb));
2772 skb_dst_force(skb);
2773 netif_rx_ni(skb);
2774 return 0;
2775}
2776EXPORT_SYMBOL(dev_loopback_xmit);
2777
2778/**
Dave Jonesd29f7492008-07-22 14:09:06 -07002779 * dev_queue_xmit - transmit a buffer
2780 * @skb: buffer to transmit
2781 *
2782 * Queue a buffer for transmission to a network device. The caller must
2783 * have set the device and priority and built the buffer before calling
2784 * this function. The function can be called from an interrupt.
2785 *
2786 * A negative errno code is returned on a failure. A success does not
2787 * guarantee the frame will be transmitted as it may be dropped due
2788 * to congestion or traffic shaping.
2789 *
2790 * -----------------------------------------------------------------------------------
2791 * I notice this method can also return errors from the queue disciplines,
2792 * including NET_XMIT_DROP, which is a positive value. So, errors can also
2793 * be positive.
2794 *
2795 * Regardless of the return value, the skb is consumed, so it is currently
2796 * difficult to retry a send to this method. (You can bump the ref count
2797 * before sending to hold a reference for retry if you are careful.)
2798 *
2799 * When calling this method, interrupts MUST be enabled. This is because
2800 * the BH enable code must have IRQs enabled so that it will not deadlock.
2801 * --BLG
2802 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07002803int dev_queue_xmit(struct sk_buff *skb)
2804{
2805 struct net_device *dev = skb->dev;
David S. Millerdc2b4842008-07-08 17:18:23 -07002806 struct netdev_queue *txq;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002807 struct Qdisc *q;
2808 int rc = -ENOMEM;
2809
Eric Dumazet6d1ccff2013-02-05 20:22:20 +00002810 skb_reset_mac_header(skb);
2811
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09002812 /* Disable soft irqs for various locks below. Also
2813 * stops preemption for RCU.
Linus Torvalds1da177e2005-04-16 15:20:36 -07002814 */
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09002815 rcu_read_lock_bh();
Linus Torvalds1da177e2005-04-16 15:20:36 -07002816
Neil Horman5bc14212011-11-22 05:10:51 +00002817 skb_update_prio(skb);
2818
Amerigo Wang8c4c49d2012-09-17 20:16:31 +00002819 txq = netdev_pick_tx(dev, skb);
Paul E. McKenneya898def2010-02-22 17:04:49 -08002820 q = rcu_dereference_bh(txq->qdisc);
David S. Miller37437bb2008-07-16 02:15:04 -07002821
Linus Torvalds1da177e2005-04-16 15:20:36 -07002822#ifdef CONFIG_NET_CLS_ACT
Eric Dumazetd1b19df2009-09-03 01:29:39 -07002823 skb->tc_verd = SET_TC_AT(skb->tc_verd, AT_EGRESS);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002824#endif
Koki Sanagicf66ba52010-08-23 18:45:02 +09002825 trace_net_dev_queue(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002826 if (q->enqueue) {
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00002827 rc = __dev_xmit_skb(skb, q, dev, txq);
David S. Miller37437bb2008-07-16 02:15:04 -07002828 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002829 }
2830
2831 /* The device has no queue. Common case for software devices:
2832 loopback, all the sorts of tunnels...
2833
Herbert Xu932ff272006-06-09 12:20:56 -07002834 Really, it is unlikely that netif_tx_lock protection is necessary
2835 here. (f.e. loopback and IP tunnels are clean ignoring statistics
Linus Torvalds1da177e2005-04-16 15:20:36 -07002836 counters.)
2837 However, it is possible, that they rely on protection
2838 made by us here.
2839
2840 Check this and shot the lock. It is not prone from deadlocks.
2841 Either shot noqueue qdisc, it is even simpler 8)
2842 */
2843 if (dev->flags & IFF_UP) {
2844 int cpu = smp_processor_id(); /* ok because BHs are off */
2845
David S. Millerc773e842008-07-08 23:13:53 -07002846 if (txq->xmit_lock_owner != cpu) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07002847
Eric Dumazet745e20f2010-09-29 13:23:09 -07002848 if (__this_cpu_read(xmit_recursion) > RECURSION_LIMIT)
2849 goto recursion_alert;
2850
David S. Millerc773e842008-07-08 23:13:53 -07002851 HARD_TX_LOCK(dev, txq, cpu);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002852
Tom Herbert734664982011-11-28 16:32:44 +00002853 if (!netif_xmit_stopped(txq)) {
Eric Dumazet745e20f2010-09-29 13:23:09 -07002854 __this_cpu_inc(xmit_recursion);
Patrick McHardy572a9d72009-11-10 06:14:14 +00002855 rc = dev_hard_start_xmit(skb, dev, txq);
Eric Dumazet745e20f2010-09-29 13:23:09 -07002856 __this_cpu_dec(xmit_recursion);
Patrick McHardy572a9d72009-11-10 06:14:14 +00002857 if (dev_xmit_complete(rc)) {
David S. Millerc773e842008-07-08 23:13:53 -07002858 HARD_TX_UNLOCK(dev, txq);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002859 goto out;
2860 }
2861 }
David S. Millerc773e842008-07-08 23:13:53 -07002862 HARD_TX_UNLOCK(dev, txq);
Joe Perchese87cc472012-05-13 21:56:26 +00002863 net_crit_ratelimited("Virtual device %s asks to queue packet!\n",
2864 dev->name);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002865 } else {
2866 /* Recursion is detected! It is possible,
Eric Dumazet745e20f2010-09-29 13:23:09 -07002867 * unfortunately
2868 */
2869recursion_alert:
Joe Perchese87cc472012-05-13 21:56:26 +00002870 net_crit_ratelimited("Dead loop on virtual device %s, fix it urgently!\n",
2871 dev->name);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002872 }
2873 }
2874
2875 rc = -ENETDOWN;
Herbert Xud4828d82006-06-22 02:28:18 -07002876 rcu_read_unlock_bh();
Linus Torvalds1da177e2005-04-16 15:20:36 -07002877
Linus Torvalds1da177e2005-04-16 15:20:36 -07002878 kfree_skb(skb);
2879 return rc;
2880out:
Herbert Xud4828d82006-06-22 02:28:18 -07002881 rcu_read_unlock_bh();
Linus Torvalds1da177e2005-04-16 15:20:36 -07002882 return rc;
2883}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07002884EXPORT_SYMBOL(dev_queue_xmit);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002885
2886
2887/*=======================================================================
2888 Receiver routines
2889 =======================================================================*/
2890
Stephen Hemminger6b2bedc2007-03-12 14:33:50 -07002891int netdev_max_backlog __read_mostly = 1000;
Eric Dumazetc9e6bc62012-09-27 19:29:05 +00002892EXPORT_SYMBOL(netdev_max_backlog);
2893
Eric Dumazet3b098e22010-05-15 23:57:10 -07002894int netdev_tstamp_prequeue __read_mostly = 1;
Stephen Hemminger6b2bedc2007-03-12 14:33:50 -07002895int netdev_budget __read_mostly = 300;
2896int weight_p __read_mostly = 64; /* old backlog weight */
Linus Torvalds1da177e2005-04-16 15:20:36 -07002897
Eric Dumazeteecfd7c2010-05-06 22:07:48 -07002898/* Called with irq disabled */
2899static inline void ____napi_schedule(struct softnet_data *sd,
2900 struct napi_struct *napi)
2901{
2902 list_add_tail(&napi->poll_list, &sd->poll_list);
2903 __raise_softirq_irqoff(NET_RX_SOFTIRQ);
2904}
2905
Eric Dumazetdf334542010-03-24 19:13:54 +00002906#ifdef CONFIG_RPS
Tom Herbertfec5e652010-04-16 16:01:27 -07002907
2908/* One global table that all flow-based protocols share. */
Eric Dumazet6e3f7fa2010-10-25 03:02:02 +00002909struct rps_sock_flow_table __rcu *rps_sock_flow_table __read_mostly;
Tom Herbertfec5e652010-04-16 16:01:27 -07002910EXPORT_SYMBOL(rps_sock_flow_table);
2911
Ingo Molnarc5905af2012-02-24 08:31:31 +01002912struct static_key rps_needed __read_mostly;
Eric Dumazetadc93002011-11-17 03:13:26 +00002913
Ben Hutchingsc4454772011-01-19 11:03:53 +00002914static struct rps_dev_flow *
2915set_rps_cpu(struct net_device *dev, struct sk_buff *skb,
2916 struct rps_dev_flow *rflow, u16 next_cpu)
2917{
Ben Hutchings09994d12011-10-03 04:42:46 +00002918 if (next_cpu != RPS_NO_CPU) {
Ben Hutchingsc4454772011-01-19 11:03:53 +00002919#ifdef CONFIG_RFS_ACCEL
2920 struct netdev_rx_queue *rxqueue;
2921 struct rps_dev_flow_table *flow_table;
2922 struct rps_dev_flow *old_rflow;
2923 u32 flow_id;
2924 u16 rxq_index;
2925 int rc;
2926
2927 /* Should we steer this flow to a different hardware queue? */
Ben Hutchings69a19ee2011-02-15 20:32:04 +00002928 if (!skb_rx_queue_recorded(skb) || !dev->rx_cpu_rmap ||
2929 !(dev->features & NETIF_F_NTUPLE))
Ben Hutchingsc4454772011-01-19 11:03:53 +00002930 goto out;
2931 rxq_index = cpu_rmap_lookup_index(dev->rx_cpu_rmap, next_cpu);
2932 if (rxq_index == skb_get_rx_queue(skb))
2933 goto out;
2934
2935 rxqueue = dev->_rx + rxq_index;
2936 flow_table = rcu_dereference(rxqueue->rps_flow_table);
2937 if (!flow_table)
2938 goto out;
2939 flow_id = skb->rxhash & flow_table->mask;
2940 rc = dev->netdev_ops->ndo_rx_flow_steer(dev, skb,
2941 rxq_index, flow_id);
2942 if (rc < 0)
2943 goto out;
2944 old_rflow = rflow;
2945 rflow = &flow_table->flows[flow_id];
Ben Hutchingsc4454772011-01-19 11:03:53 +00002946 rflow->filter = rc;
2947 if (old_rflow->filter == rflow->filter)
2948 old_rflow->filter = RPS_NO_FILTER;
2949 out:
2950#endif
2951 rflow->last_qtail =
Ben Hutchings09994d12011-10-03 04:42:46 +00002952 per_cpu(softnet_data, next_cpu).input_queue_head;
Ben Hutchingsc4454772011-01-19 11:03:53 +00002953 }
2954
Ben Hutchings09994d12011-10-03 04:42:46 +00002955 rflow->cpu = next_cpu;
Ben Hutchingsc4454772011-01-19 11:03:53 +00002956 return rflow;
2957}
2958
Tom Herbert0a9627f2010-03-16 08:03:29 +00002959/*
2960 * get_rps_cpu is called from netif_receive_skb and returns the target
2961 * CPU from the RPS map of the receiving queue for a given skb.
Eric Dumazetb0e28f12010-04-15 00:14:07 -07002962 * rcu_read_lock must be held on entry.
Tom Herbert0a9627f2010-03-16 08:03:29 +00002963 */
Tom Herbertfec5e652010-04-16 16:01:27 -07002964static int get_rps_cpu(struct net_device *dev, struct sk_buff *skb,
2965 struct rps_dev_flow **rflowp)
Tom Herbert0a9627f2010-03-16 08:03:29 +00002966{
Tom Herbert0a9627f2010-03-16 08:03:29 +00002967 struct netdev_rx_queue *rxqueue;
Eric Dumazet6e3f7fa2010-10-25 03:02:02 +00002968 struct rps_map *map;
Tom Herbertfec5e652010-04-16 16:01:27 -07002969 struct rps_dev_flow_table *flow_table;
2970 struct rps_sock_flow_table *sock_flow_table;
Tom Herbert0a9627f2010-03-16 08:03:29 +00002971 int cpu = -1;
Tom Herbertfec5e652010-04-16 16:01:27 -07002972 u16 tcpu;
Tom Herbert0a9627f2010-03-16 08:03:29 +00002973
Tom Herbert0a9627f2010-03-16 08:03:29 +00002974 if (skb_rx_queue_recorded(skb)) {
2975 u16 index = skb_get_rx_queue(skb);
Ben Hutchings62fe0b42010-09-27 08:24:33 +00002976 if (unlikely(index >= dev->real_num_rx_queues)) {
2977 WARN_ONCE(dev->real_num_rx_queues > 1,
2978 "%s received packet on queue %u, but number "
2979 "of RX queues is %u\n",
2980 dev->name, index, dev->real_num_rx_queues);
Tom Herbert0a9627f2010-03-16 08:03:29 +00002981 goto done;
2982 }
2983 rxqueue = dev->_rx + index;
2984 } else
2985 rxqueue = dev->_rx;
2986
Eric Dumazet6e3f7fa2010-10-25 03:02:02 +00002987 map = rcu_dereference(rxqueue->rps_map);
2988 if (map) {
Tom Herbert85875232011-01-31 16:23:42 -08002989 if (map->len == 1 &&
Eric Dumazet33d480c2011-08-11 19:30:52 +00002990 !rcu_access_pointer(rxqueue->rps_flow_table)) {
Changli Gao6febfca2010-09-03 23:12:37 +00002991 tcpu = map->cpus[0];
2992 if (cpu_online(tcpu))
2993 cpu = tcpu;
Tom Herbert0a9627f2010-03-16 08:03:29 +00002994 goto done;
Eric Dumazetb249dcb2010-04-19 21:56:38 +00002995 }
Eric Dumazet33d480c2011-08-11 19:30:52 +00002996 } else if (!rcu_access_pointer(rxqueue->rps_flow_table)) {
Tom Herbert0a9627f2010-03-16 08:03:29 +00002997 goto done;
Tom Herbert0a9627f2010-03-16 08:03:29 +00002998 }
2999
Changli Gao2d47b452010-08-17 19:00:56 +00003000 skb_reset_network_header(skb);
Krishna Kumarbfb564e2010-08-04 06:15:52 +00003001 if (!skb_get_rxhash(skb))
Tom Herbert0a9627f2010-03-16 08:03:29 +00003002 goto done;
Tom Herbert0a9627f2010-03-16 08:03:29 +00003003
Tom Herbertfec5e652010-04-16 16:01:27 -07003004 flow_table = rcu_dereference(rxqueue->rps_flow_table);
3005 sock_flow_table = rcu_dereference(rps_sock_flow_table);
3006 if (flow_table && sock_flow_table) {
3007 u16 next_cpu;
3008 struct rps_dev_flow *rflow;
3009
3010 rflow = &flow_table->flows[skb->rxhash & flow_table->mask];
3011 tcpu = rflow->cpu;
3012
3013 next_cpu = sock_flow_table->ents[skb->rxhash &
3014 sock_flow_table->mask];
3015
3016 /*
3017 * If the desired CPU (where last recvmsg was done) is
3018 * different from current CPU (one in the rx-queue flow
3019 * table entry), switch if one of the following holds:
3020 * - Current CPU is unset (equal to RPS_NO_CPU).
3021 * - Current CPU is offline.
3022 * - The current CPU's queue tail has advanced beyond the
3023 * last packet that was enqueued using this table entry.
3024 * This guarantees that all previous packets for the flow
3025 * have been dequeued, thus preserving in order delivery.
3026 */
3027 if (unlikely(tcpu != next_cpu) &&
3028 (tcpu == RPS_NO_CPU || !cpu_online(tcpu) ||
3029 ((int)(per_cpu(softnet_data, tcpu).input_queue_head -
Tom Herbertbaefa312012-11-16 09:04:15 +00003030 rflow->last_qtail)) >= 0)) {
3031 tcpu = next_cpu;
Ben Hutchingsc4454772011-01-19 11:03:53 +00003032 rflow = set_rps_cpu(dev, skb, rflow, next_cpu);
Tom Herbertbaefa312012-11-16 09:04:15 +00003033 }
Ben Hutchingsc4454772011-01-19 11:03:53 +00003034
Tom Herbertfec5e652010-04-16 16:01:27 -07003035 if (tcpu != RPS_NO_CPU && cpu_online(tcpu)) {
3036 *rflowp = rflow;
3037 cpu = tcpu;
3038 goto done;
3039 }
3040 }
3041
Tom Herbert0a9627f2010-03-16 08:03:29 +00003042 if (map) {
Tom Herbertfec5e652010-04-16 16:01:27 -07003043 tcpu = map->cpus[((u64) skb->rxhash * map->len) >> 32];
Tom Herbert0a9627f2010-03-16 08:03:29 +00003044
3045 if (cpu_online(tcpu)) {
3046 cpu = tcpu;
3047 goto done;
3048 }
3049 }
3050
3051done:
Tom Herbert0a9627f2010-03-16 08:03:29 +00003052 return cpu;
3053}
3054
Ben Hutchingsc4454772011-01-19 11:03:53 +00003055#ifdef CONFIG_RFS_ACCEL
3056
3057/**
3058 * rps_may_expire_flow - check whether an RFS hardware filter may be removed
3059 * @dev: Device on which the filter was set
3060 * @rxq_index: RX queue index
3061 * @flow_id: Flow ID passed to ndo_rx_flow_steer()
3062 * @filter_id: Filter ID returned by ndo_rx_flow_steer()
3063 *
3064 * Drivers that implement ndo_rx_flow_steer() should periodically call
3065 * this function for each installed filter and remove the filters for
3066 * which it returns %true.
3067 */
3068bool rps_may_expire_flow(struct net_device *dev, u16 rxq_index,
3069 u32 flow_id, u16 filter_id)
3070{
3071 struct netdev_rx_queue *rxqueue = dev->_rx + rxq_index;
3072 struct rps_dev_flow_table *flow_table;
3073 struct rps_dev_flow *rflow;
3074 bool expire = true;
3075 int cpu;
3076
3077 rcu_read_lock();
3078 flow_table = rcu_dereference(rxqueue->rps_flow_table);
3079 if (flow_table && flow_id <= flow_table->mask) {
3080 rflow = &flow_table->flows[flow_id];
3081 cpu = ACCESS_ONCE(rflow->cpu);
3082 if (rflow->filter == filter_id && cpu != RPS_NO_CPU &&
3083 ((int)(per_cpu(softnet_data, cpu).input_queue_head -
3084 rflow->last_qtail) <
3085 (int)(10 * flow_table->mask)))
3086 expire = false;
3087 }
3088 rcu_read_unlock();
3089 return expire;
3090}
3091EXPORT_SYMBOL(rps_may_expire_flow);
3092
3093#endif /* CONFIG_RFS_ACCEL */
3094
Tom Herbert0a9627f2010-03-16 08:03:29 +00003095/* Called from hardirq (IPI) context */
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003096static void rps_trigger_softirq(void *data)
Tom Herbert0a9627f2010-03-16 08:03:29 +00003097{
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003098 struct softnet_data *sd = data;
3099
Eric Dumazeteecfd7c2010-05-06 22:07:48 -07003100 ____napi_schedule(sd, &sd->backlog);
Changli Gaodee42872010-05-02 05:42:16 +00003101 sd->received_rps++;
Tom Herbert0a9627f2010-03-16 08:03:29 +00003102}
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003103
Tom Herbertfec5e652010-04-16 16:01:27 -07003104#endif /* CONFIG_RPS */
Tom Herbert0a9627f2010-03-16 08:03:29 +00003105
3106/*
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003107 * Check if this softnet_data structure is another cpu one
3108 * If yes, queue it to our IPI list and return 1
3109 * If no, return 0
3110 */
3111static int rps_ipi_queued(struct softnet_data *sd)
3112{
3113#ifdef CONFIG_RPS
3114 struct softnet_data *mysd = &__get_cpu_var(softnet_data);
3115
3116 if (sd != mysd) {
3117 sd->rps_ipi_next = mysd->rps_ipi_list;
3118 mysd->rps_ipi_list = sd;
3119
3120 __raise_softirq_irqoff(NET_RX_SOFTIRQ);
3121 return 1;
3122 }
3123#endif /* CONFIG_RPS */
3124 return 0;
3125}
3126
Willem de Bruijn99bbc702013-05-20 04:02:32 +00003127#ifdef CONFIG_NET_FLOW_LIMIT
3128int netdev_flow_limit_table_len __read_mostly = (1 << 12);
3129#endif
3130
3131static bool skb_flow_limit(struct sk_buff *skb, unsigned int qlen)
3132{
3133#ifdef CONFIG_NET_FLOW_LIMIT
3134 struct sd_flow_limit *fl;
3135 struct softnet_data *sd;
3136 unsigned int old_flow, new_flow;
3137
3138 if (qlen < (netdev_max_backlog >> 1))
3139 return false;
3140
3141 sd = &__get_cpu_var(softnet_data);
3142
3143 rcu_read_lock();
3144 fl = rcu_dereference(sd->flow_limit);
3145 if (fl) {
3146 new_flow = skb_get_rxhash(skb) & (fl->num_buckets - 1);
3147 old_flow = fl->history[fl->history_head];
3148 fl->history[fl->history_head] = new_flow;
3149
3150 fl->history_head++;
3151 fl->history_head &= FLOW_LIMIT_HISTORY - 1;
3152
3153 if (likely(fl->buckets[old_flow]))
3154 fl->buckets[old_flow]--;
3155
3156 if (++fl->buckets[new_flow] > (FLOW_LIMIT_HISTORY >> 1)) {
3157 fl->count++;
3158 rcu_read_unlock();
3159 return true;
3160 }
3161 }
3162 rcu_read_unlock();
3163#endif
3164 return false;
3165}
3166
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003167/*
Tom Herbert0a9627f2010-03-16 08:03:29 +00003168 * enqueue_to_backlog is called to queue an skb to a per CPU backlog
3169 * queue (may be a remote CPU queue).
3170 */
Tom Herbertfec5e652010-04-16 16:01:27 -07003171static int enqueue_to_backlog(struct sk_buff *skb, int cpu,
3172 unsigned int *qtail)
Tom Herbert0a9627f2010-03-16 08:03:29 +00003173{
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003174 struct softnet_data *sd;
Tom Herbert0a9627f2010-03-16 08:03:29 +00003175 unsigned long flags;
Willem de Bruijn99bbc702013-05-20 04:02:32 +00003176 unsigned int qlen;
Tom Herbert0a9627f2010-03-16 08:03:29 +00003177
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003178 sd = &per_cpu(softnet_data, cpu);
Tom Herbert0a9627f2010-03-16 08:03:29 +00003179
3180 local_irq_save(flags);
Tom Herbert0a9627f2010-03-16 08:03:29 +00003181
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003182 rps_lock(sd);
Willem de Bruijn99bbc702013-05-20 04:02:32 +00003183 qlen = skb_queue_len(&sd->input_pkt_queue);
3184 if (qlen <= netdev_max_backlog && !skb_flow_limit(skb, qlen)) {
Changli Gao6e7676c2010-04-27 15:07:33 -07003185 if (skb_queue_len(&sd->input_pkt_queue)) {
Tom Herbert0a9627f2010-03-16 08:03:29 +00003186enqueue:
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003187 __skb_queue_tail(&sd->input_pkt_queue, skb);
Tom Herbert76cc8b12010-05-20 18:37:59 +00003188 input_queue_tail_incr_save(sd, qtail);
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003189 rps_unlock(sd);
Changli Gao152102c2010-03-30 20:16:22 +00003190 local_irq_restore(flags);
Tom Herbert0a9627f2010-03-16 08:03:29 +00003191 return NET_RX_SUCCESS;
3192 }
3193
Eric Dumazetebda37c22010-05-06 23:51:21 +00003194 /* Schedule NAPI for backlog device
3195 * We can use non atomic operation since we own the queue lock
3196 */
3197 if (!__test_and_set_bit(NAPI_STATE_SCHED, &sd->backlog.state)) {
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003198 if (!rps_ipi_queued(sd))
Eric Dumazeteecfd7c2010-05-06 22:07:48 -07003199 ____napi_schedule(sd, &sd->backlog);
Tom Herbert0a9627f2010-03-16 08:03:29 +00003200 }
3201 goto enqueue;
3202 }
3203
Changli Gaodee42872010-05-02 05:42:16 +00003204 sd->dropped++;
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003205 rps_unlock(sd);
Tom Herbert0a9627f2010-03-16 08:03:29 +00003206
Tom Herbert0a9627f2010-03-16 08:03:29 +00003207 local_irq_restore(flags);
3208
Eric Dumazetcaf586e2010-09-30 21:06:55 +00003209 atomic_long_inc(&skb->dev->rx_dropped);
Tom Herbert0a9627f2010-03-16 08:03:29 +00003210 kfree_skb(skb);
3211 return NET_RX_DROP;
3212}
Linus Torvalds1da177e2005-04-16 15:20:36 -07003213
Linus Torvalds1da177e2005-04-16 15:20:36 -07003214/**
3215 * netif_rx - post buffer to the network code
3216 * @skb: buffer to post
3217 *
3218 * This function receives a packet from a device driver and queues it for
3219 * the upper (protocol) levels to process. It always succeeds. The buffer
3220 * may be dropped during processing for congestion control or by the
3221 * protocol layers.
3222 *
3223 * return values:
3224 * NET_RX_SUCCESS (no congestion)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003225 * NET_RX_DROP (packet was dropped)
3226 *
3227 */
3228
3229int netif_rx(struct sk_buff *skb)
3230{
Eric Dumazetb0e28f12010-04-15 00:14:07 -07003231 int ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003232
3233 /* if netpoll wants it, pretend we never saw it */
3234 if (netpoll_rx(skb))
3235 return NET_RX_DROP;
3236
Eric Dumazet588f0332011-11-15 04:12:55 +00003237 net_timestamp_check(netdev_tstamp_prequeue, skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003238
Koki Sanagicf66ba52010-08-23 18:45:02 +09003239 trace_netif_rx(skb);
Eric Dumazetdf334542010-03-24 19:13:54 +00003240#ifdef CONFIG_RPS
Ingo Molnarc5905af2012-02-24 08:31:31 +01003241 if (static_key_false(&rps_needed)) {
Tom Herbertfec5e652010-04-16 16:01:27 -07003242 struct rps_dev_flow voidflow, *rflow = &voidflow;
Eric Dumazetb0e28f12010-04-15 00:14:07 -07003243 int cpu;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003244
Changli Gaocece1942010-08-07 20:35:43 -07003245 preempt_disable();
Eric Dumazetb0e28f12010-04-15 00:14:07 -07003246 rcu_read_lock();
Tom Herbertfec5e652010-04-16 16:01:27 -07003247
3248 cpu = get_rps_cpu(skb->dev, skb, &rflow);
Eric Dumazetb0e28f12010-04-15 00:14:07 -07003249 if (cpu < 0)
3250 cpu = smp_processor_id();
Tom Herbertfec5e652010-04-16 16:01:27 -07003251
3252 ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail);
3253
Eric Dumazetb0e28f12010-04-15 00:14:07 -07003254 rcu_read_unlock();
Changli Gaocece1942010-08-07 20:35:43 -07003255 preempt_enable();
Eric Dumazetadc93002011-11-17 03:13:26 +00003256 } else
3257#endif
Tom Herbertfec5e652010-04-16 16:01:27 -07003258 {
3259 unsigned int qtail;
3260 ret = enqueue_to_backlog(skb, get_cpu(), &qtail);
3261 put_cpu();
3262 }
Eric Dumazetb0e28f12010-04-15 00:14:07 -07003263 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003264}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07003265EXPORT_SYMBOL(netif_rx);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003266
3267int netif_rx_ni(struct sk_buff *skb)
3268{
3269 int err;
3270
3271 preempt_disable();
3272 err = netif_rx(skb);
3273 if (local_softirq_pending())
3274 do_softirq();
3275 preempt_enable();
3276
3277 return err;
3278}
Linus Torvalds1da177e2005-04-16 15:20:36 -07003279EXPORT_SYMBOL(netif_rx_ni);
3280
Linus Torvalds1da177e2005-04-16 15:20:36 -07003281static void net_tx_action(struct softirq_action *h)
3282{
3283 struct softnet_data *sd = &__get_cpu_var(softnet_data);
3284
3285 if (sd->completion_queue) {
3286 struct sk_buff *clist;
3287
3288 local_irq_disable();
3289 clist = sd->completion_queue;
3290 sd->completion_queue = NULL;
3291 local_irq_enable();
3292
3293 while (clist) {
3294 struct sk_buff *skb = clist;
3295 clist = clist->next;
3296
Ilpo Järvinen547b7922008-07-25 21:43:18 -07003297 WARN_ON(atomic_read(&skb->users));
Koki Sanagi07dc22e2010-08-23 18:46:12 +09003298 trace_kfree_skb(skb, net_tx_action);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003299 __kfree_skb(skb);
3300 }
3301 }
3302
3303 if (sd->output_queue) {
David S. Miller37437bb2008-07-16 02:15:04 -07003304 struct Qdisc *head;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003305
3306 local_irq_disable();
3307 head = sd->output_queue;
3308 sd->output_queue = NULL;
Changli Gaoa9cbd582010-04-26 23:06:24 +00003309 sd->output_queue_tailp = &sd->output_queue;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003310 local_irq_enable();
3311
3312 while (head) {
David S. Miller37437bb2008-07-16 02:15:04 -07003313 struct Qdisc *q = head;
3314 spinlock_t *root_lock;
3315
Linus Torvalds1da177e2005-04-16 15:20:36 -07003316 head = head->next_sched;
3317
David S. Miller5fb66222008-08-02 20:02:43 -07003318 root_lock = qdisc_lock(q);
David S. Miller37437bb2008-07-16 02:15:04 -07003319 if (spin_trylock(root_lock)) {
Jarek Poplawskidef82a12008-08-17 21:54:43 -07003320 smp_mb__before_clear_bit();
3321 clear_bit(__QDISC_STATE_SCHED,
3322 &q->state);
David S. Miller37437bb2008-07-16 02:15:04 -07003323 qdisc_run(q);
3324 spin_unlock(root_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003325 } else {
David S. Miller195648b2008-08-19 04:00:36 -07003326 if (!test_bit(__QDISC_STATE_DEACTIVATED,
Jarek Poplawskie8a83e12008-09-07 18:41:21 -07003327 &q->state)) {
David S. Miller195648b2008-08-19 04:00:36 -07003328 __netif_reschedule(q);
Jarek Poplawskie8a83e12008-09-07 18:41:21 -07003329 } else {
3330 smp_mb__before_clear_bit();
3331 clear_bit(__QDISC_STATE_SCHED,
3332 &q->state);
3333 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07003334 }
3335 }
3336 }
3337}
3338
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00003339#if (defined(CONFIG_BRIDGE) || defined(CONFIG_BRIDGE_MODULE)) && \
3340 (defined(CONFIG_ATM_LANE) || defined(CONFIG_ATM_LANE_MODULE))
Michał Mirosławda678292009-06-05 05:35:28 +00003341/* This hook is defined here for ATM LANE */
3342int (*br_fdb_test_addr_hook)(struct net_device *dev,
3343 unsigned char *addr) __read_mostly;
Stephen Hemminger4fb019a2009-09-11 11:50:08 -07003344EXPORT_SYMBOL_GPL(br_fdb_test_addr_hook);
Michał Mirosławda678292009-06-05 05:35:28 +00003345#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07003346
Linus Torvalds1da177e2005-04-16 15:20:36 -07003347#ifdef CONFIG_NET_CLS_ACT
3348/* TODO: Maybe we should just force sch_ingress to be compiled in
3349 * when CONFIG_NET_CLS_ACT is? otherwise some useless instructions
3350 * a compare and 2 stores extra right now if we dont have it on
3351 * but have CONFIG_NET_CLS_ACT
Lucas De Marchi25985ed2011-03-30 22:57:33 -03003352 * NOTE: This doesn't stop any functionality; if you dont have
3353 * the ingress scheduler, you just can't add policies on ingress.
Linus Torvalds1da177e2005-04-16 15:20:36 -07003354 *
3355 */
Eric Dumazet24824a02010-10-02 06:11:55 +00003356static int ing_filter(struct sk_buff *skb, struct netdev_queue *rxq)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003357{
Linus Torvalds1da177e2005-04-16 15:20:36 -07003358 struct net_device *dev = skb->dev;
Herbert Xuf697c3e2007-10-14 00:38:47 -07003359 u32 ttl = G_TC_RTTL(skb->tc_verd);
David S. Miller555353c2008-07-08 17:33:13 -07003360 int result = TC_ACT_OK;
3361 struct Qdisc *q;
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09003362
Stephen Hemmingerde384832010-08-01 00:33:23 -07003363 if (unlikely(MAX_RED_LOOP < ttl++)) {
Joe Perchese87cc472012-05-13 21:56:26 +00003364 net_warn_ratelimited("Redir loop detected Dropping packet (%d->%d)\n",
3365 skb->skb_iif, dev->ifindex);
Herbert Xuf697c3e2007-10-14 00:38:47 -07003366 return TC_ACT_SHOT;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003367 }
3368
Herbert Xuf697c3e2007-10-14 00:38:47 -07003369 skb->tc_verd = SET_TC_RTTL(skb->tc_verd, ttl);
3370 skb->tc_verd = SET_TC_AT(skb->tc_verd, AT_INGRESS);
3371
David S. Miller83874002008-07-17 00:53:03 -07003372 q = rxq->qdisc;
David S. Miller8d50b532008-07-30 02:37:46 -07003373 if (q != &noop_qdisc) {
David S. Miller83874002008-07-17 00:53:03 -07003374 spin_lock(qdisc_lock(q));
David S. Millera9312ae2008-08-17 21:51:03 -07003375 if (likely(!test_bit(__QDISC_STATE_DEACTIVATED, &q->state)))
3376 result = qdisc_enqueue_root(skb, q);
David S. Miller83874002008-07-17 00:53:03 -07003377 spin_unlock(qdisc_lock(q));
3378 }
Herbert Xuf697c3e2007-10-14 00:38:47 -07003379
Linus Torvalds1da177e2005-04-16 15:20:36 -07003380 return result;
3381}
Herbert Xuf697c3e2007-10-14 00:38:47 -07003382
3383static inline struct sk_buff *handle_ing(struct sk_buff *skb,
3384 struct packet_type **pt_prev,
3385 int *ret, struct net_device *orig_dev)
3386{
Eric Dumazet24824a02010-10-02 06:11:55 +00003387 struct netdev_queue *rxq = rcu_dereference(skb->dev->ingress_queue);
3388
3389 if (!rxq || rxq->qdisc == &noop_qdisc)
Herbert Xuf697c3e2007-10-14 00:38:47 -07003390 goto out;
3391
3392 if (*pt_prev) {
3393 *ret = deliver_skb(skb, *pt_prev, orig_dev);
3394 *pt_prev = NULL;
Herbert Xuf697c3e2007-10-14 00:38:47 -07003395 }
3396
Eric Dumazet24824a02010-10-02 06:11:55 +00003397 switch (ing_filter(skb, rxq)) {
Herbert Xuf697c3e2007-10-14 00:38:47 -07003398 case TC_ACT_SHOT:
3399 case TC_ACT_STOLEN:
3400 kfree_skb(skb);
3401 return NULL;
3402 }
3403
3404out:
3405 skb->tc_verd = 0;
3406 return skb;
3407}
Linus Torvalds1da177e2005-04-16 15:20:36 -07003408#endif
3409
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00003410/**
3411 * netdev_rx_handler_register - register receive handler
3412 * @dev: device to register a handler for
3413 * @rx_handler: receive handler to register
Jiri Pirko93e2c322010-06-10 03:34:59 +00003414 * @rx_handler_data: data pointer that is used by rx handler
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00003415 *
3416 * Register a receive hander for a device. This handler will then be
3417 * called from __netif_receive_skb. A negative errno code is returned
3418 * on a failure.
3419 *
3420 * The caller must hold the rtnl_mutex.
Jiri Pirko8a4eb572011-03-12 03:14:39 +00003421 *
3422 * For a general description of rx_handler, see enum rx_handler_result.
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00003423 */
3424int netdev_rx_handler_register(struct net_device *dev,
Jiri Pirko93e2c322010-06-10 03:34:59 +00003425 rx_handler_func_t *rx_handler,
3426 void *rx_handler_data)
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00003427{
3428 ASSERT_RTNL();
3429
3430 if (dev->rx_handler)
3431 return -EBUSY;
3432
Eric Dumazet00cfec32013-03-29 03:01:22 +00003433 /* Note: rx_handler_data must be set before rx_handler */
Jiri Pirko93e2c322010-06-10 03:34:59 +00003434 rcu_assign_pointer(dev->rx_handler_data, rx_handler_data);
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00003435 rcu_assign_pointer(dev->rx_handler, rx_handler);
3436
3437 return 0;
3438}
3439EXPORT_SYMBOL_GPL(netdev_rx_handler_register);
3440
3441/**
3442 * netdev_rx_handler_unregister - unregister receive handler
3443 * @dev: device to unregister a handler from
3444 *
Kusanagi Kouichi166ec362013-03-18 02:59:52 +00003445 * Unregister a receive handler from a device.
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00003446 *
3447 * The caller must hold the rtnl_mutex.
3448 */
3449void netdev_rx_handler_unregister(struct net_device *dev)
3450{
3451
3452 ASSERT_RTNL();
Stephen Hemmingera9b3cd72011-08-01 16:19:00 +00003453 RCU_INIT_POINTER(dev->rx_handler, NULL);
Eric Dumazet00cfec32013-03-29 03:01:22 +00003454 /* a reader seeing a non NULL rx_handler in a rcu_read_lock()
3455 * section has a guarantee to see a non NULL rx_handler_data
3456 * as well.
3457 */
3458 synchronize_net();
Stephen Hemmingera9b3cd72011-08-01 16:19:00 +00003459 RCU_INIT_POINTER(dev->rx_handler_data, NULL);
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00003460}
3461EXPORT_SYMBOL_GPL(netdev_rx_handler_unregister);
3462
Mel Gormanb4b9e352012-07-31 16:44:26 -07003463/*
3464 * Limit the use of PFMEMALLOC reserves to those protocols that implement
3465 * the special handling of PFMEMALLOC skbs.
3466 */
3467static bool skb_pfmemalloc_protocol(struct sk_buff *skb)
3468{
3469 switch (skb->protocol) {
3470 case __constant_htons(ETH_P_ARP):
3471 case __constant_htons(ETH_P_IP):
3472 case __constant_htons(ETH_P_IPV6):
3473 case __constant_htons(ETH_P_8021Q):
Patrick McHardy8ad227f2013-04-19 02:04:31 +00003474 case __constant_htons(ETH_P_8021AD):
Mel Gormanb4b9e352012-07-31 16:44:26 -07003475 return true;
3476 default:
3477 return false;
3478 }
3479}
3480
David S. Miller9754e292013-02-14 15:57:38 -05003481static int __netif_receive_skb_core(struct sk_buff *skb, bool pfmemalloc)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003482{
3483 struct packet_type *ptype, *pt_prev;
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00003484 rx_handler_func_t *rx_handler;
David S. Millerf2ccd8f2005-08-09 19:34:12 -07003485 struct net_device *orig_dev;
David S. Miller63d8ea72011-02-28 10:48:59 -08003486 struct net_device *null_or_dev;
Jiri Pirko8a4eb572011-03-12 03:14:39 +00003487 bool deliver_exact = false;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003488 int ret = NET_RX_DROP;
Al Viro252e3342006-11-14 20:48:11 -08003489 __be16 type;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003490
Eric Dumazet588f0332011-11-15 04:12:55 +00003491 net_timestamp_check(!netdev_tstamp_prequeue, skb);
Eric Dumazet81bbb3d2009-09-30 16:42:42 -07003492
Koki Sanagicf66ba52010-08-23 18:45:02 +09003493 trace_netif_receive_skb(skb);
Patrick McHardy9b22ea52008-11-04 14:49:57 -08003494
Linus Torvalds1da177e2005-04-16 15:20:36 -07003495 /* if we've gotten here through NAPI, check netpoll */
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003496 if (netpoll_receive_skb(skb))
Mel Gormanb4b9e352012-07-31 16:44:26 -07003497 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003498
Joe Eykholtcc9bd5c2008-07-02 18:22:00 -07003499 orig_dev = skb->dev;
Jiri Pirko1765a572011-02-12 06:48:36 +00003500
Arnaldo Carvalho de Meloc1d2bbe2007-04-10 20:45:18 -07003501 skb_reset_network_header(skb);
Eric Dumazetfda55ec2013-01-07 09:28:21 +00003502 if (!skb_transport_header_was_set(skb))
3503 skb_reset_transport_header(skb);
Jiri Pirko0b5c9db2011-06-10 06:56:58 +00003504 skb_reset_mac_len(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003505
3506 pt_prev = NULL;
3507
3508 rcu_read_lock();
3509
David S. Miller63d8ea72011-02-28 10:48:59 -08003510another_round:
David S. Millerb6858172012-07-23 16:27:54 -07003511 skb->skb_iif = skb->dev->ifindex;
David S. Miller63d8ea72011-02-28 10:48:59 -08003512
3513 __this_cpu_inc(softnet_data.processed);
3514
Patrick McHardy8ad227f2013-04-19 02:04:31 +00003515 if (skb->protocol == cpu_to_be16(ETH_P_8021Q) ||
3516 skb->protocol == cpu_to_be16(ETH_P_8021AD)) {
Jiri Pirkobcc6d472011-04-07 19:48:33 +00003517 skb = vlan_untag(skb);
3518 if (unlikely(!skb))
Mel Gormanb4b9e352012-07-31 16:44:26 -07003519 goto unlock;
Jiri Pirkobcc6d472011-04-07 19:48:33 +00003520 }
3521
Linus Torvalds1da177e2005-04-16 15:20:36 -07003522#ifdef CONFIG_NET_CLS_ACT
3523 if (skb->tc_verd & TC_NCLS) {
3524 skb->tc_verd = CLR_TC_NCLS(skb->tc_verd);
3525 goto ncls;
3526 }
3527#endif
3528
David S. Miller9754e292013-02-14 15:57:38 -05003529 if (pfmemalloc)
Mel Gormanb4b9e352012-07-31 16:44:26 -07003530 goto skip_taps;
3531
Linus Torvalds1da177e2005-04-16 15:20:36 -07003532 list_for_each_entry_rcu(ptype, &ptype_all, list) {
David S. Miller63d8ea72011-02-28 10:48:59 -08003533 if (!ptype->dev || ptype->dev == skb->dev) {
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09003534 if (pt_prev)
David S. Millerf2ccd8f2005-08-09 19:34:12 -07003535 ret = deliver_skb(skb, pt_prev, orig_dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003536 pt_prev = ptype;
3537 }
3538 }
3539
Mel Gormanb4b9e352012-07-31 16:44:26 -07003540skip_taps:
Linus Torvalds1da177e2005-04-16 15:20:36 -07003541#ifdef CONFIG_NET_CLS_ACT
Herbert Xuf697c3e2007-10-14 00:38:47 -07003542 skb = handle_ing(skb, &pt_prev, &ret, orig_dev);
3543 if (!skb)
Mel Gormanb4b9e352012-07-31 16:44:26 -07003544 goto unlock;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003545ncls:
3546#endif
3547
David S. Miller9754e292013-02-14 15:57:38 -05003548 if (pfmemalloc && !skb_pfmemalloc_protocol(skb))
Mel Gormanb4b9e352012-07-31 16:44:26 -07003549 goto drop;
3550
John Fastabend24257172011-10-10 09:16:41 +00003551 if (vlan_tx_tag_present(skb)) {
3552 if (pt_prev) {
3553 ret = deliver_skb(skb, pt_prev, orig_dev);
3554 pt_prev = NULL;
3555 }
Florian Zumbiehl48cc32d32012-10-07 15:51:58 +00003556 if (vlan_do_receive(&skb))
John Fastabend24257172011-10-10 09:16:41 +00003557 goto another_round;
3558 else if (unlikely(!skb))
Mel Gormanb4b9e352012-07-31 16:44:26 -07003559 goto unlock;
John Fastabend24257172011-10-10 09:16:41 +00003560 }
3561
Florian Zumbiehl48cc32d32012-10-07 15:51:58 +00003562 rx_handler = rcu_dereference(skb->dev->rx_handler);
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00003563 if (rx_handler) {
3564 if (pt_prev) {
3565 ret = deliver_skb(skb, pt_prev, orig_dev);
3566 pt_prev = NULL;
3567 }
Jiri Pirko8a4eb572011-03-12 03:14:39 +00003568 switch (rx_handler(&skb)) {
3569 case RX_HANDLER_CONSUMED:
Cristian Bercaru3bc1b1a2013-03-08 07:03:38 +00003570 ret = NET_RX_SUCCESS;
Mel Gormanb4b9e352012-07-31 16:44:26 -07003571 goto unlock;
Jiri Pirko8a4eb572011-03-12 03:14:39 +00003572 case RX_HANDLER_ANOTHER:
David S. Miller63d8ea72011-02-28 10:48:59 -08003573 goto another_round;
Jiri Pirko8a4eb572011-03-12 03:14:39 +00003574 case RX_HANDLER_EXACT:
3575 deliver_exact = true;
3576 case RX_HANDLER_PASS:
3577 break;
3578 default:
3579 BUG();
3580 }
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00003581 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07003582
Eric Dumazetd4b812d2013-07-18 07:19:26 -07003583 if (unlikely(vlan_tx_tag_present(skb))) {
3584 if (vlan_tx_tag_get_id(skb))
3585 skb->pkt_type = PACKET_OTHERHOST;
3586 /* Note: we might in the future use prio bits
3587 * and set skb->priority like in vlan_do_receive()
3588 * For the time being, just ignore Priority Code Point
3589 */
3590 skb->vlan_tci = 0;
3591 }
Florian Zumbiehl48cc32d32012-10-07 15:51:58 +00003592
David S. Miller63d8ea72011-02-28 10:48:59 -08003593 /* deliver only exact match when indicated */
Jiri Pirko8a4eb572011-03-12 03:14:39 +00003594 null_or_dev = deliver_exact ? skb->dev : NULL;
Andy Gospodarek1f3c8802009-12-14 10:48:58 +00003595
Linus Torvalds1da177e2005-04-16 15:20:36 -07003596 type = skb->protocol;
Pavel Emelyanov82d8a8672007-11-26 20:12:58 +08003597 list_for_each_entry_rcu(ptype,
3598 &ptype_base[ntohs(type) & PTYPE_HASH_MASK], list) {
David S. Miller63d8ea72011-02-28 10:48:59 -08003599 if (ptype->type == type &&
Jiri Pirkoe3f48d32011-02-28 20:26:31 +00003600 (ptype->dev == null_or_dev || ptype->dev == skb->dev ||
3601 ptype->dev == orig_dev)) {
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09003602 if (pt_prev)
David S. Millerf2ccd8f2005-08-09 19:34:12 -07003603 ret = deliver_skb(skb, pt_prev, orig_dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003604 pt_prev = ptype;
3605 }
3606 }
3607
3608 if (pt_prev) {
Michael S. Tsirkin1080e512012-07-20 09:23:17 +00003609 if (unlikely(skb_orphan_frags(skb, GFP_ATOMIC)))
Michael S. Tsirkin0e698bf2012-09-15 22:44:16 +00003610 goto drop;
Michael S. Tsirkin1080e512012-07-20 09:23:17 +00003611 else
3612 ret = pt_prev->func(skb, skb->dev, pt_prev, orig_dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003613 } else {
Mel Gormanb4b9e352012-07-31 16:44:26 -07003614drop:
Eric Dumazetcaf586e2010-09-30 21:06:55 +00003615 atomic_long_inc(&skb->dev->rx_dropped);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003616 kfree_skb(skb);
3617 /* Jamal, now you will not able to escape explaining
3618 * me how you were going to use this. :-)
3619 */
3620 ret = NET_RX_DROP;
3621 }
3622
Mel Gormanb4b9e352012-07-31 16:44:26 -07003623unlock:
Linus Torvalds1da177e2005-04-16 15:20:36 -07003624 rcu_read_unlock();
Mel Gormanb4b9e352012-07-31 16:44:26 -07003625out:
David S. Miller9754e292013-02-14 15:57:38 -05003626 return ret;
3627}
3628
3629static int __netif_receive_skb(struct sk_buff *skb)
3630{
3631 int ret;
3632
3633 if (sk_memalloc_socks() && skb_pfmemalloc(skb)) {
3634 unsigned long pflags = current->flags;
3635
3636 /*
3637 * PFMEMALLOC skbs are special, they should
3638 * - be delivered to SOCK_MEMALLOC sockets only
3639 * - stay away from userspace
3640 * - have bounded memory usage
3641 *
3642 * Use PF_MEMALLOC as this saves us from propagating the allocation
3643 * context down to all allocation sites.
3644 */
3645 current->flags |= PF_MEMALLOC;
3646 ret = __netif_receive_skb_core(skb, true);
3647 tsk_restore_flags(current, pflags, PF_MEMALLOC);
3648 } else
3649 ret = __netif_receive_skb_core(skb, false);
3650
Linus Torvalds1da177e2005-04-16 15:20:36 -07003651 return ret;
3652}
Tom Herbert0a9627f2010-03-16 08:03:29 +00003653
3654/**
3655 * netif_receive_skb - process receive buffer from network
3656 * @skb: buffer to process
3657 *
3658 * netif_receive_skb() is the main receive data processing function.
3659 * It always succeeds. The buffer may be dropped during processing
3660 * for congestion control or by the protocol layers.
3661 *
3662 * This function may only be called from softirq context and interrupts
3663 * should be enabled.
3664 *
3665 * Return values (usually ignored):
3666 * NET_RX_SUCCESS: no congestion
3667 * NET_RX_DROP: packet was dropped
3668 */
3669int netif_receive_skb(struct sk_buff *skb)
3670{
Eric Dumazet588f0332011-11-15 04:12:55 +00003671 net_timestamp_check(netdev_tstamp_prequeue, skb);
Eric Dumazet3b098e22010-05-15 23:57:10 -07003672
Richard Cochranc1f19b52010-07-17 08:49:36 +00003673 if (skb_defer_rx_timestamp(skb))
3674 return NET_RX_SUCCESS;
3675
Eric Dumazetdf334542010-03-24 19:13:54 +00003676#ifdef CONFIG_RPS
Ingo Molnarc5905af2012-02-24 08:31:31 +01003677 if (static_key_false(&rps_needed)) {
Eric Dumazet3b098e22010-05-15 23:57:10 -07003678 struct rps_dev_flow voidflow, *rflow = &voidflow;
3679 int cpu, ret;
Tom Herbert0a9627f2010-03-16 08:03:29 +00003680
Eric Dumazet3b098e22010-05-15 23:57:10 -07003681 rcu_read_lock();
Tom Herbert0a9627f2010-03-16 08:03:29 +00003682
Eric Dumazet3b098e22010-05-15 23:57:10 -07003683 cpu = get_rps_cpu(skb->dev, skb, &rflow);
Tom Herbertfec5e652010-04-16 16:01:27 -07003684
Eric Dumazet3b098e22010-05-15 23:57:10 -07003685 if (cpu >= 0) {
3686 ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail);
3687 rcu_read_unlock();
Eric Dumazetadc93002011-11-17 03:13:26 +00003688 return ret;
Eric Dumazet3b098e22010-05-15 23:57:10 -07003689 }
Eric Dumazetadc93002011-11-17 03:13:26 +00003690 rcu_read_unlock();
Tom Herbertfec5e652010-04-16 16:01:27 -07003691 }
Tom Herbert1e94d722010-03-18 17:45:44 -07003692#endif
Eric Dumazetadc93002011-11-17 03:13:26 +00003693 return __netif_receive_skb(skb);
Tom Herbert0a9627f2010-03-16 08:03:29 +00003694}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07003695EXPORT_SYMBOL(netif_receive_skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003696
Eric Dumazet88751272010-04-19 05:07:33 +00003697/* Network device is going away, flush any packets still pending
3698 * Called with irqs disabled.
3699 */
Changli Gao152102c2010-03-30 20:16:22 +00003700static void flush_backlog(void *arg)
Stephen Hemminger6e583ce2008-08-03 21:29:57 -07003701{
Changli Gao152102c2010-03-30 20:16:22 +00003702 struct net_device *dev = arg;
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003703 struct softnet_data *sd = &__get_cpu_var(softnet_data);
Stephen Hemminger6e583ce2008-08-03 21:29:57 -07003704 struct sk_buff *skb, *tmp;
3705
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003706 rps_lock(sd);
Changli Gao6e7676c2010-04-27 15:07:33 -07003707 skb_queue_walk_safe(&sd->input_pkt_queue, skb, tmp) {
Stephen Hemminger6e583ce2008-08-03 21:29:57 -07003708 if (skb->dev == dev) {
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003709 __skb_unlink(skb, &sd->input_pkt_queue);
Stephen Hemminger6e583ce2008-08-03 21:29:57 -07003710 kfree_skb(skb);
Tom Herbert76cc8b12010-05-20 18:37:59 +00003711 input_queue_head_incr(sd);
Stephen Hemminger6e583ce2008-08-03 21:29:57 -07003712 }
Changli Gao6e7676c2010-04-27 15:07:33 -07003713 }
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003714 rps_unlock(sd);
Changli Gao6e7676c2010-04-27 15:07:33 -07003715
3716 skb_queue_walk_safe(&sd->process_queue, skb, tmp) {
3717 if (skb->dev == dev) {
3718 __skb_unlink(skb, &sd->process_queue);
3719 kfree_skb(skb);
Tom Herbert76cc8b12010-05-20 18:37:59 +00003720 input_queue_head_incr(sd);
Changli Gao6e7676c2010-04-27 15:07:33 -07003721 }
3722 }
Stephen Hemminger6e583ce2008-08-03 21:29:57 -07003723}
3724
Herbert Xud565b0a2008-12-15 23:38:52 -08003725static int napi_gro_complete(struct sk_buff *skb)
3726{
Vlad Yasevich22061d82012-11-15 08:49:11 +00003727 struct packet_offload *ptype;
Herbert Xud565b0a2008-12-15 23:38:52 -08003728 __be16 type = skb->protocol;
Vlad Yasevich22061d82012-11-15 08:49:11 +00003729 struct list_head *head = &offload_base;
Herbert Xud565b0a2008-12-15 23:38:52 -08003730 int err = -ENOENT;
3731
Eric Dumazetc3c7c252012-12-06 13:54:59 +00003732 BUILD_BUG_ON(sizeof(struct napi_gro_cb) > sizeof(skb->cb));
3733
Herbert Xufc59f9a2009-04-14 15:11:06 -07003734 if (NAPI_GRO_CB(skb)->count == 1) {
3735 skb_shinfo(skb)->gso_size = 0;
Herbert Xud565b0a2008-12-15 23:38:52 -08003736 goto out;
Herbert Xufc59f9a2009-04-14 15:11:06 -07003737 }
Herbert Xud565b0a2008-12-15 23:38:52 -08003738
3739 rcu_read_lock();
3740 list_for_each_entry_rcu(ptype, head, list) {
Vlad Yasevichf191a1d2012-11-15 08:49:23 +00003741 if (ptype->type != type || !ptype->callbacks.gro_complete)
Herbert Xud565b0a2008-12-15 23:38:52 -08003742 continue;
3743
Vlad Yasevichf191a1d2012-11-15 08:49:23 +00003744 err = ptype->callbacks.gro_complete(skb);
Herbert Xud565b0a2008-12-15 23:38:52 -08003745 break;
3746 }
3747 rcu_read_unlock();
3748
3749 if (err) {
3750 WARN_ON(&ptype->list == head);
3751 kfree_skb(skb);
3752 return NET_RX_SUCCESS;
3753 }
3754
3755out:
Herbert Xud565b0a2008-12-15 23:38:52 -08003756 return netif_receive_skb(skb);
3757}
3758
Eric Dumazet2e71a6f2012-10-06 08:08:49 +00003759/* napi->gro_list contains packets ordered by age.
3760 * youngest packets at the head of it.
3761 * Complete skbs in reverse order to reduce latencies.
3762 */
3763void napi_gro_flush(struct napi_struct *napi, bool flush_old)
Herbert Xud565b0a2008-12-15 23:38:52 -08003764{
Eric Dumazet2e71a6f2012-10-06 08:08:49 +00003765 struct sk_buff *skb, *prev = NULL;
Herbert Xud565b0a2008-12-15 23:38:52 -08003766
Eric Dumazet2e71a6f2012-10-06 08:08:49 +00003767 /* scan list and build reverse chain */
3768 for (skb = napi->gro_list; skb != NULL; skb = skb->next) {
3769 skb->prev = prev;
3770 prev = skb;
Herbert Xud565b0a2008-12-15 23:38:52 -08003771 }
3772
Eric Dumazet2e71a6f2012-10-06 08:08:49 +00003773 for (skb = prev; skb; skb = prev) {
3774 skb->next = NULL;
3775
3776 if (flush_old && NAPI_GRO_CB(skb)->age == jiffies)
3777 return;
3778
3779 prev = skb->prev;
3780 napi_gro_complete(skb);
3781 napi->gro_count--;
3782 }
3783
Herbert Xud565b0a2008-12-15 23:38:52 -08003784 napi->gro_list = NULL;
3785}
Eric Dumazet86cac582010-08-31 18:25:32 +00003786EXPORT_SYMBOL(napi_gro_flush);
Herbert Xud565b0a2008-12-15 23:38:52 -08003787
Eric Dumazet89c5fa32012-12-10 13:28:16 +00003788static void gro_list_prepare(struct napi_struct *napi, struct sk_buff *skb)
3789{
3790 struct sk_buff *p;
3791 unsigned int maclen = skb->dev->hard_header_len;
3792
3793 for (p = napi->gro_list; p; p = p->next) {
3794 unsigned long diffs;
3795
3796 diffs = (unsigned long)p->dev ^ (unsigned long)skb->dev;
3797 diffs |= p->vlan_tci ^ skb->vlan_tci;
3798 if (maclen == ETH_HLEN)
3799 diffs |= compare_ether_header(skb_mac_header(p),
3800 skb_gro_mac_header(skb));
3801 else if (!diffs)
3802 diffs = memcmp(skb_mac_header(p),
3803 skb_gro_mac_header(skb),
3804 maclen);
3805 NAPI_GRO_CB(p)->same_flow = !diffs;
3806 NAPI_GRO_CB(p)->flush = 0;
3807 }
3808}
3809
Rami Rosenbb728822012-11-28 21:55:25 +00003810static enum gro_result dev_gro_receive(struct napi_struct *napi, struct sk_buff *skb)
Herbert Xud565b0a2008-12-15 23:38:52 -08003811{
3812 struct sk_buff **pp = NULL;
Vlad Yasevich22061d82012-11-15 08:49:11 +00003813 struct packet_offload *ptype;
Herbert Xud565b0a2008-12-15 23:38:52 -08003814 __be16 type = skb->protocol;
Vlad Yasevich22061d82012-11-15 08:49:11 +00003815 struct list_head *head = &offload_base;
Herbert Xu0da2afd52008-12-26 14:57:42 -08003816 int same_flow;
Ben Hutchings5b252f02009-10-29 07:17:09 +00003817 enum gro_result ret;
Herbert Xud565b0a2008-12-15 23:38:52 -08003818
Jarek Poplawskice9e76c2010-08-05 01:19:11 +00003819 if (!(skb->dev->features & NETIF_F_GRO) || netpoll_rx_on(skb))
Herbert Xud565b0a2008-12-15 23:38:52 -08003820 goto normal;
3821
David S. Miller21dc3302010-08-23 00:13:46 -07003822 if (skb_is_gso(skb) || skb_has_frag_list(skb))
Herbert Xuf17f5c92009-01-14 14:36:12 -08003823 goto normal;
3824
Eric Dumazet89c5fa32012-12-10 13:28:16 +00003825 gro_list_prepare(napi, skb);
3826
Herbert Xud565b0a2008-12-15 23:38:52 -08003827 rcu_read_lock();
3828 list_for_each_entry_rcu(ptype, head, list) {
Vlad Yasevichf191a1d2012-11-15 08:49:23 +00003829 if (ptype->type != type || !ptype->callbacks.gro_receive)
Herbert Xud565b0a2008-12-15 23:38:52 -08003830 continue;
3831
Herbert Xu86911732009-01-29 14:19:50 +00003832 skb_set_network_header(skb, skb_gro_offset(skb));
Eric Dumazetefd94502013-02-14 17:31:48 +00003833 skb_reset_mac_len(skb);
Herbert Xud565b0a2008-12-15 23:38:52 -08003834 NAPI_GRO_CB(skb)->same_flow = 0;
3835 NAPI_GRO_CB(skb)->flush = 0;
Herbert Xu5d38a072009-01-04 16:13:40 -08003836 NAPI_GRO_CB(skb)->free = 0;
Herbert Xud565b0a2008-12-15 23:38:52 -08003837
Vlad Yasevichf191a1d2012-11-15 08:49:23 +00003838 pp = ptype->callbacks.gro_receive(&napi->gro_list, skb);
Herbert Xud565b0a2008-12-15 23:38:52 -08003839 break;
3840 }
3841 rcu_read_unlock();
3842
3843 if (&ptype->list == head)
3844 goto normal;
3845
Herbert Xu0da2afd52008-12-26 14:57:42 -08003846 same_flow = NAPI_GRO_CB(skb)->same_flow;
Herbert Xu5d0d9be2009-01-29 14:19:48 +00003847 ret = NAPI_GRO_CB(skb)->free ? GRO_MERGED_FREE : GRO_MERGED;
Herbert Xu0da2afd52008-12-26 14:57:42 -08003848
Herbert Xud565b0a2008-12-15 23:38:52 -08003849 if (pp) {
3850 struct sk_buff *nskb = *pp;
3851
3852 *pp = nskb->next;
3853 nskb->next = NULL;
3854 napi_gro_complete(nskb);
Herbert Xu4ae55442009-02-08 18:00:36 +00003855 napi->gro_count--;
Herbert Xud565b0a2008-12-15 23:38:52 -08003856 }
3857
Herbert Xu0da2afd52008-12-26 14:57:42 -08003858 if (same_flow)
Herbert Xud565b0a2008-12-15 23:38:52 -08003859 goto ok;
3860
Herbert Xu4ae55442009-02-08 18:00:36 +00003861 if (NAPI_GRO_CB(skb)->flush || napi->gro_count >= MAX_GRO_SKBS)
Herbert Xud565b0a2008-12-15 23:38:52 -08003862 goto normal;
Herbert Xud565b0a2008-12-15 23:38:52 -08003863
Herbert Xu4ae55442009-02-08 18:00:36 +00003864 napi->gro_count++;
Herbert Xud565b0a2008-12-15 23:38:52 -08003865 NAPI_GRO_CB(skb)->count = 1;
Eric Dumazet2e71a6f2012-10-06 08:08:49 +00003866 NAPI_GRO_CB(skb)->age = jiffies;
Herbert Xu86911732009-01-29 14:19:50 +00003867 skb_shinfo(skb)->gso_size = skb_gro_len(skb);
Herbert Xud565b0a2008-12-15 23:38:52 -08003868 skb->next = napi->gro_list;
3869 napi->gro_list = skb;
Herbert Xu5d0d9be2009-01-29 14:19:48 +00003870 ret = GRO_HELD;
Herbert Xud565b0a2008-12-15 23:38:52 -08003871
Herbert Xuad0f9902009-02-01 01:24:55 -08003872pull:
Herbert Xucb189782009-05-26 18:50:31 +00003873 if (skb_headlen(skb) < skb_gro_offset(skb)) {
3874 int grow = skb_gro_offset(skb) - skb_headlen(skb);
3875
3876 BUG_ON(skb->end - skb->tail < grow);
3877
3878 memcpy(skb_tail_pointer(skb), NAPI_GRO_CB(skb)->frag0, grow);
3879
3880 skb->tail += grow;
3881 skb->data_len -= grow;
3882
3883 skb_shinfo(skb)->frags[0].page_offset += grow;
Eric Dumazet9e903e02011-10-18 21:00:24 +00003884 skb_frag_size_sub(&skb_shinfo(skb)->frags[0], grow);
Herbert Xucb189782009-05-26 18:50:31 +00003885
Eric Dumazet9e903e02011-10-18 21:00:24 +00003886 if (unlikely(!skb_frag_size(&skb_shinfo(skb)->frags[0]))) {
Ian Campbellea2ab692011-08-22 23:44:58 +00003887 skb_frag_unref(skb, 0);
Herbert Xucb189782009-05-26 18:50:31 +00003888 memmove(skb_shinfo(skb)->frags,
3889 skb_shinfo(skb)->frags + 1,
Jarek Poplawskie5093ae2010-08-11 02:02:10 +00003890 --skb_shinfo(skb)->nr_frags * sizeof(skb_frag_t));
Herbert Xucb189782009-05-26 18:50:31 +00003891 }
Herbert Xuad0f9902009-02-01 01:24:55 -08003892 }
3893
Herbert Xud565b0a2008-12-15 23:38:52 -08003894ok:
Herbert Xu5d0d9be2009-01-29 14:19:48 +00003895 return ret;
Herbert Xud565b0a2008-12-15 23:38:52 -08003896
3897normal:
Herbert Xuad0f9902009-02-01 01:24:55 -08003898 ret = GRO_NORMAL;
3899 goto pull;
Herbert Xu5d38a072009-01-04 16:13:40 -08003900}
Herbert Xu96e93ea2009-01-06 10:49:34 -08003901
Herbert Xu96e93ea2009-01-06 10:49:34 -08003902
Rami Rosenbb728822012-11-28 21:55:25 +00003903static gro_result_t napi_skb_finish(gro_result_t ret, struct sk_buff *skb)
Herbert Xu5d38a072009-01-04 16:13:40 -08003904{
Herbert Xu5d0d9be2009-01-29 14:19:48 +00003905 switch (ret) {
3906 case GRO_NORMAL:
Ben Hutchingsc7c4b3b2009-10-29 21:36:53 -07003907 if (netif_receive_skb(skb))
3908 ret = GRO_DROP;
3909 break;
Herbert Xu5d38a072009-01-04 16:13:40 -08003910
Herbert Xu5d0d9be2009-01-29 14:19:48 +00003911 case GRO_DROP:
Herbert Xu5d38a072009-01-04 16:13:40 -08003912 kfree_skb(skb);
3913 break;
Ben Hutchings5b252f02009-10-29 07:17:09 +00003914
Eric Dumazetdaa86542012-04-19 07:07:40 +00003915 case GRO_MERGED_FREE:
Eric Dumazetd7e88832012-04-30 08:10:34 +00003916 if (NAPI_GRO_CB(skb)->free == NAPI_GRO_FREE_STOLEN_HEAD)
3917 kmem_cache_free(skbuff_head_cache, skb);
3918 else
3919 __kfree_skb(skb);
Eric Dumazetdaa86542012-04-19 07:07:40 +00003920 break;
3921
Ben Hutchings5b252f02009-10-29 07:17:09 +00003922 case GRO_HELD:
3923 case GRO_MERGED:
3924 break;
Herbert Xu5d38a072009-01-04 16:13:40 -08003925 }
3926
Ben Hutchingsc7c4b3b2009-10-29 21:36:53 -07003927 return ret;
Herbert Xu5d0d9be2009-01-29 14:19:48 +00003928}
Herbert Xu5d0d9be2009-01-29 14:19:48 +00003929
Eric Dumazetca07e432012-10-06 22:28:06 +00003930static void skb_gro_reset_offset(struct sk_buff *skb)
Herbert Xu78a478d2009-05-26 18:50:21 +00003931{
Eric Dumazetca07e432012-10-06 22:28:06 +00003932 const struct skb_shared_info *pinfo = skb_shinfo(skb);
3933 const skb_frag_t *frag0 = &pinfo->frags[0];
3934
Herbert Xu78a478d2009-05-26 18:50:21 +00003935 NAPI_GRO_CB(skb)->data_offset = 0;
3936 NAPI_GRO_CB(skb)->frag0 = NULL;
Herbert Xu74895942009-05-26 18:50:27 +00003937 NAPI_GRO_CB(skb)->frag0_len = 0;
Herbert Xu78a478d2009-05-26 18:50:21 +00003938
Simon Hormanced14f62013-05-28 20:34:25 +00003939 if (skb_mac_header(skb) == skb_tail_pointer(skb) &&
Eric Dumazetca07e432012-10-06 22:28:06 +00003940 pinfo->nr_frags &&
3941 !PageHighMem(skb_frag_page(frag0))) {
3942 NAPI_GRO_CB(skb)->frag0 = skb_frag_address(frag0);
3943 NAPI_GRO_CB(skb)->frag0_len = skb_frag_size(frag0);
Herbert Xu74895942009-05-26 18:50:27 +00003944 }
Herbert Xu78a478d2009-05-26 18:50:21 +00003945}
Herbert Xu78a478d2009-05-26 18:50:21 +00003946
Ben Hutchingsc7c4b3b2009-10-29 21:36:53 -07003947gro_result_t napi_gro_receive(struct napi_struct *napi, struct sk_buff *skb)
Herbert Xu5d0d9be2009-01-29 14:19:48 +00003948{
Herbert Xu86911732009-01-29 14:19:50 +00003949 skb_gro_reset_offset(skb);
3950
Eric Dumazet89c5fa32012-12-10 13:28:16 +00003951 return napi_skb_finish(dev_gro_receive(napi, skb), skb);
Herbert Xud565b0a2008-12-15 23:38:52 -08003952}
3953EXPORT_SYMBOL(napi_gro_receive);
3954
stephen hemmingerd0c2b0d2010-10-19 07:12:10 +00003955static void napi_reuse_skb(struct napi_struct *napi, struct sk_buff *skb)
Herbert Xu96e93ea2009-01-06 10:49:34 -08003956{
Herbert Xu96e93ea2009-01-06 10:49:34 -08003957 __skb_pull(skb, skb_headlen(skb));
Eric Dumazet2a2a4592012-03-21 06:58:03 +00003958 /* restore the reserve we had after netdev_alloc_skb_ip_align() */
3959 skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN - skb_headroom(skb));
Jesse Gross3701e512010-10-20 13:56:06 +00003960 skb->vlan_tci = 0;
Herbert Xu66c46d72011-01-29 20:44:54 -08003961 skb->dev = napi->dev;
Andy Gospodarek6d152e22011-02-02 14:53:25 -08003962 skb->skb_iif = 0;
Herbert Xu96e93ea2009-01-06 10:49:34 -08003963
3964 napi->skb = skb;
3965}
Herbert Xu96e93ea2009-01-06 10:49:34 -08003966
Herbert Xu76620aa2009-04-16 02:02:07 -07003967struct sk_buff *napi_get_frags(struct napi_struct *napi)
Herbert Xu5d38a072009-01-04 16:13:40 -08003968{
Herbert Xu5d38a072009-01-04 16:13:40 -08003969 struct sk_buff *skb = napi->skb;
Herbert Xu5d38a072009-01-04 16:13:40 -08003970
3971 if (!skb) {
Eric Dumazet89d71a62009-10-13 05:34:20 +00003972 skb = netdev_alloc_skb_ip_align(napi->dev, GRO_MAX_HEAD);
3973 if (skb)
3974 napi->skb = skb;
Herbert Xu5d38a072009-01-04 16:13:40 -08003975 }
Herbert Xu96e93ea2009-01-06 10:49:34 -08003976 return skb;
3977}
Herbert Xu76620aa2009-04-16 02:02:07 -07003978EXPORT_SYMBOL(napi_get_frags);
Herbert Xu96e93ea2009-01-06 10:49:34 -08003979
Rami Rosenbb728822012-11-28 21:55:25 +00003980static gro_result_t napi_frags_finish(struct napi_struct *napi, struct sk_buff *skb,
Ben Hutchingsc7c4b3b2009-10-29 21:36:53 -07003981 gro_result_t ret)
Herbert Xu5d0d9be2009-01-29 14:19:48 +00003982{
Herbert Xu5d0d9be2009-01-29 14:19:48 +00003983 switch (ret) {
3984 case GRO_NORMAL:
Herbert Xu86911732009-01-29 14:19:50 +00003985 case GRO_HELD:
Ajit Khapardee76b69c2010-02-16 20:25:43 +00003986 skb->protocol = eth_type_trans(skb, skb->dev);
Herbert Xu86911732009-01-29 14:19:50 +00003987
Ben Hutchingsc7c4b3b2009-10-29 21:36:53 -07003988 if (ret == GRO_HELD)
3989 skb_gro_pull(skb, -ETH_HLEN);
3990 else if (netif_receive_skb(skb))
3991 ret = GRO_DROP;
Herbert Xu86911732009-01-29 14:19:50 +00003992 break;
Herbert Xu5d0d9be2009-01-29 14:19:48 +00003993
3994 case GRO_DROP:
Herbert Xu5d0d9be2009-01-29 14:19:48 +00003995 case GRO_MERGED_FREE:
3996 napi_reuse_skb(napi, skb);
3997 break;
Ben Hutchings5b252f02009-10-29 07:17:09 +00003998
3999 case GRO_MERGED:
4000 break;
Herbert Xu5d0d9be2009-01-29 14:19:48 +00004001 }
4002
Ben Hutchingsc7c4b3b2009-10-29 21:36:53 -07004003 return ret;
Herbert Xu5d0d9be2009-01-29 14:19:48 +00004004}
Herbert Xu5d0d9be2009-01-29 14:19:48 +00004005
Eric Dumazet4adb9c42012-05-18 20:49:06 +00004006static struct sk_buff *napi_frags_skb(struct napi_struct *napi)
Herbert Xu96e93ea2009-01-06 10:49:34 -08004007{
Herbert Xu76620aa2009-04-16 02:02:07 -07004008 struct sk_buff *skb = napi->skb;
4009 struct ethhdr *eth;
Herbert Xua5b1cf22009-05-26 18:50:28 +00004010 unsigned int hlen;
4011 unsigned int off;
Herbert Xu76620aa2009-04-16 02:02:07 -07004012
4013 napi->skb = NULL;
4014
4015 skb_reset_mac_header(skb);
4016 skb_gro_reset_offset(skb);
4017
Herbert Xua5b1cf22009-05-26 18:50:28 +00004018 off = skb_gro_offset(skb);
4019 hlen = off + sizeof(*eth);
4020 eth = skb_gro_header_fast(skb, off);
4021 if (skb_gro_header_hard(skb, hlen)) {
4022 eth = skb_gro_header_slow(skb, hlen, off);
4023 if (unlikely(!eth)) {
4024 napi_reuse_skb(napi, skb);
4025 skb = NULL;
4026 goto out;
4027 }
Herbert Xu76620aa2009-04-16 02:02:07 -07004028 }
4029
4030 skb_gro_pull(skb, sizeof(*eth));
4031
4032 /*
4033 * This works because the only protocols we care about don't require
4034 * special handling. We'll fix it up properly at the end.
4035 */
4036 skb->protocol = eth->h_proto;
4037
4038out:
4039 return skb;
4040}
Herbert Xu76620aa2009-04-16 02:02:07 -07004041
Ben Hutchingsc7c4b3b2009-10-29 21:36:53 -07004042gro_result_t napi_gro_frags(struct napi_struct *napi)
Herbert Xu76620aa2009-04-16 02:02:07 -07004043{
4044 struct sk_buff *skb = napi_frags_skb(napi);
Herbert Xu96e93ea2009-01-06 10:49:34 -08004045
4046 if (!skb)
Ben Hutchingsc7c4b3b2009-10-29 21:36:53 -07004047 return GRO_DROP;
Herbert Xu96e93ea2009-01-06 10:49:34 -08004048
Eric Dumazet89c5fa32012-12-10 13:28:16 +00004049 return napi_frags_finish(napi, skb, dev_gro_receive(napi, skb));
Herbert Xu5d38a072009-01-04 16:13:40 -08004050}
4051EXPORT_SYMBOL(napi_gro_frags);
4052
Eric Dumazete326bed2010-04-22 00:22:45 -07004053/*
4054 * net_rps_action sends any pending IPI's for rps.
4055 * Note: called with local irq disabled, but exits with local irq enabled.
4056 */
4057static void net_rps_action_and_irq_enable(struct softnet_data *sd)
4058{
4059#ifdef CONFIG_RPS
4060 struct softnet_data *remsd = sd->rps_ipi_list;
4061
4062 if (remsd) {
4063 sd->rps_ipi_list = NULL;
4064
4065 local_irq_enable();
4066
4067 /* Send pending IPI's to kick RPS processing on remote cpus. */
4068 while (remsd) {
4069 struct softnet_data *next = remsd->rps_ipi_next;
4070
4071 if (cpu_online(remsd->cpu))
4072 __smp_call_function_single(remsd->cpu,
4073 &remsd->csd, 0);
4074 remsd = next;
4075 }
4076 } else
4077#endif
4078 local_irq_enable();
4079}
4080
Stephen Hemmingerbea33482007-10-03 16:41:36 -07004081static int process_backlog(struct napi_struct *napi, int quota)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004082{
4083 int work = 0;
Eric Dumazeteecfd7c2010-05-06 22:07:48 -07004084 struct softnet_data *sd = container_of(napi, struct softnet_data, backlog);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004085
Eric Dumazete326bed2010-04-22 00:22:45 -07004086#ifdef CONFIG_RPS
4087 /* Check if we have pending ipi, its better to send them now,
4088 * not waiting net_rx_action() end.
4089 */
4090 if (sd->rps_ipi_list) {
4091 local_irq_disable();
4092 net_rps_action_and_irq_enable(sd);
4093 }
4094#endif
Stephen Hemmingerbea33482007-10-03 16:41:36 -07004095 napi->weight = weight_p;
Changli Gao6e7676c2010-04-27 15:07:33 -07004096 local_irq_disable();
4097 while (work < quota) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07004098 struct sk_buff *skb;
Changli Gao6e7676c2010-04-27 15:07:33 -07004099 unsigned int qlen;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004100
Changli Gao6e7676c2010-04-27 15:07:33 -07004101 while ((skb = __skb_dequeue(&sd->process_queue))) {
Eric Dumazete4008272010-04-05 15:42:39 -07004102 local_irq_enable();
Changli Gao6e7676c2010-04-27 15:07:33 -07004103 __netif_receive_skb(skb);
Changli Gao6e7676c2010-04-27 15:07:33 -07004104 local_irq_disable();
Tom Herbert76cc8b12010-05-20 18:37:59 +00004105 input_queue_head_incr(sd);
4106 if (++work >= quota) {
4107 local_irq_enable();
4108 return work;
4109 }
Stephen Hemmingerbea33482007-10-03 16:41:36 -07004110 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07004111
Changli Gao6e7676c2010-04-27 15:07:33 -07004112 rps_lock(sd);
4113 qlen = skb_queue_len(&sd->input_pkt_queue);
Tom Herbert76cc8b12010-05-20 18:37:59 +00004114 if (qlen)
Changli Gao6e7676c2010-04-27 15:07:33 -07004115 skb_queue_splice_tail_init(&sd->input_pkt_queue,
4116 &sd->process_queue);
Tom Herbert76cc8b12010-05-20 18:37:59 +00004117
Changli Gao6e7676c2010-04-27 15:07:33 -07004118 if (qlen < quota - work) {
Eric Dumazeteecfd7c2010-05-06 22:07:48 -07004119 /*
4120 * Inline a custom version of __napi_complete().
4121 * only current cpu owns and manipulates this napi,
4122 * and NAPI_STATE_SCHED is the only possible flag set on backlog.
4123 * we can use a plain write instead of clear_bit(),
4124 * and we dont need an smp_mb() memory barrier.
4125 */
4126 list_del(&napi->poll_list);
4127 napi->state = 0;
4128
Changli Gao6e7676c2010-04-27 15:07:33 -07004129 quota = work + qlen;
4130 }
4131 rps_unlock(sd);
4132 }
4133 local_irq_enable();
Linus Torvalds1da177e2005-04-16 15:20:36 -07004134
Stephen Hemmingerbea33482007-10-03 16:41:36 -07004135 return work;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004136}
4137
Stephen Hemmingerbea33482007-10-03 16:41:36 -07004138/**
4139 * __napi_schedule - schedule for receive
Randy Dunlapc4ea43c2007-10-12 21:17:49 -07004140 * @n: entry to schedule
Stephen Hemmingerbea33482007-10-03 16:41:36 -07004141 *
4142 * The entry's receive function will be scheduled to run
4143 */
Harvey Harrisonb5606c22008-02-13 15:03:16 -08004144void __napi_schedule(struct napi_struct *n)
Stephen Hemmingerbea33482007-10-03 16:41:36 -07004145{
4146 unsigned long flags;
4147
4148 local_irq_save(flags);
Eric Dumazeteecfd7c2010-05-06 22:07:48 -07004149 ____napi_schedule(&__get_cpu_var(softnet_data), n);
Stephen Hemmingerbea33482007-10-03 16:41:36 -07004150 local_irq_restore(flags);
4151}
4152EXPORT_SYMBOL(__napi_schedule);
4153
Herbert Xud565b0a2008-12-15 23:38:52 -08004154void __napi_complete(struct napi_struct *n)
4155{
4156 BUG_ON(!test_bit(NAPI_STATE_SCHED, &n->state));
4157 BUG_ON(n->gro_list);
4158
4159 list_del(&n->poll_list);
4160 smp_mb__before_clear_bit();
4161 clear_bit(NAPI_STATE_SCHED, &n->state);
4162}
4163EXPORT_SYMBOL(__napi_complete);
4164
4165void napi_complete(struct napi_struct *n)
4166{
4167 unsigned long flags;
4168
4169 /*
4170 * don't let napi dequeue from the cpu poll list
4171 * just in case its running on a different cpu
4172 */
4173 if (unlikely(test_bit(NAPI_STATE_NPSVC, &n->state)))
4174 return;
4175
Eric Dumazet2e71a6f2012-10-06 08:08:49 +00004176 napi_gro_flush(n, false);
Herbert Xud565b0a2008-12-15 23:38:52 -08004177 local_irq_save(flags);
4178 __napi_complete(n);
4179 local_irq_restore(flags);
4180}
4181EXPORT_SYMBOL(napi_complete);
4182
Eliezer Tamiraf12fa62013-06-10 11:39:41 +03004183/* must be called under rcu_read_lock(), as we dont take a reference */
4184struct napi_struct *napi_by_id(unsigned int napi_id)
4185{
4186 unsigned int hash = napi_id % HASH_SIZE(napi_hash);
4187 struct napi_struct *napi;
4188
4189 hlist_for_each_entry_rcu(napi, &napi_hash[hash], napi_hash_node)
4190 if (napi->napi_id == napi_id)
4191 return napi;
4192
4193 return NULL;
4194}
4195EXPORT_SYMBOL_GPL(napi_by_id);
4196
4197void napi_hash_add(struct napi_struct *napi)
4198{
4199 if (!test_and_set_bit(NAPI_STATE_HASHED, &napi->state)) {
4200
4201 spin_lock(&napi_hash_lock);
4202
4203 /* 0 is not a valid id, we also skip an id that is taken
4204 * we expect both events to be extremely rare
4205 */
4206 napi->napi_id = 0;
4207 while (!napi->napi_id) {
4208 napi->napi_id = ++napi_gen_id;
4209 if (napi_by_id(napi->napi_id))
4210 napi->napi_id = 0;
4211 }
4212
4213 hlist_add_head_rcu(&napi->napi_hash_node,
4214 &napi_hash[napi->napi_id % HASH_SIZE(napi_hash)]);
4215
4216 spin_unlock(&napi_hash_lock);
4217 }
4218}
4219EXPORT_SYMBOL_GPL(napi_hash_add);
4220
4221/* Warning : caller is responsible to make sure rcu grace period
4222 * is respected before freeing memory containing @napi
4223 */
4224void napi_hash_del(struct napi_struct *napi)
4225{
4226 spin_lock(&napi_hash_lock);
4227
4228 if (test_and_clear_bit(NAPI_STATE_HASHED, &napi->state))
4229 hlist_del_rcu(&napi->napi_hash_node);
4230
4231 spin_unlock(&napi_hash_lock);
4232}
4233EXPORT_SYMBOL_GPL(napi_hash_del);
4234
Herbert Xud565b0a2008-12-15 23:38:52 -08004235void netif_napi_add(struct net_device *dev, struct napi_struct *napi,
4236 int (*poll)(struct napi_struct *, int), int weight)
4237{
4238 INIT_LIST_HEAD(&napi->poll_list);
Herbert Xu4ae55442009-02-08 18:00:36 +00004239 napi->gro_count = 0;
Herbert Xud565b0a2008-12-15 23:38:52 -08004240 napi->gro_list = NULL;
Herbert Xu5d38a072009-01-04 16:13:40 -08004241 napi->skb = NULL;
Herbert Xud565b0a2008-12-15 23:38:52 -08004242 napi->poll = poll;
Eric Dumazet82dc3c62013-03-05 15:57:22 +00004243 if (weight > NAPI_POLL_WEIGHT)
4244 pr_err_once("netif_napi_add() called with weight %d on device %s\n",
4245 weight, dev->name);
Herbert Xud565b0a2008-12-15 23:38:52 -08004246 napi->weight = weight;
4247 list_add(&napi->dev_list, &dev->napi_list);
Herbert Xud565b0a2008-12-15 23:38:52 -08004248 napi->dev = dev;
Herbert Xu5d38a072009-01-04 16:13:40 -08004249#ifdef CONFIG_NETPOLL
Herbert Xud565b0a2008-12-15 23:38:52 -08004250 spin_lock_init(&napi->poll_lock);
4251 napi->poll_owner = -1;
4252#endif
4253 set_bit(NAPI_STATE_SCHED, &napi->state);
4254}
4255EXPORT_SYMBOL(netif_napi_add);
4256
4257void netif_napi_del(struct napi_struct *napi)
4258{
4259 struct sk_buff *skb, *next;
4260
Peter P Waskiewicz Jrd7b06632008-12-26 01:35:35 -08004261 list_del_init(&napi->dev_list);
Herbert Xu76620aa2009-04-16 02:02:07 -07004262 napi_free_frags(napi);
Herbert Xud565b0a2008-12-15 23:38:52 -08004263
4264 for (skb = napi->gro_list; skb; skb = next) {
4265 next = skb->next;
4266 skb->next = NULL;
4267 kfree_skb(skb);
4268 }
4269
4270 napi->gro_list = NULL;
Herbert Xu4ae55442009-02-08 18:00:36 +00004271 napi->gro_count = 0;
Herbert Xud565b0a2008-12-15 23:38:52 -08004272}
4273EXPORT_SYMBOL(netif_napi_del);
4274
Linus Torvalds1da177e2005-04-16 15:20:36 -07004275static void net_rx_action(struct softirq_action *h)
4276{
Eric Dumazete326bed2010-04-22 00:22:45 -07004277 struct softnet_data *sd = &__get_cpu_var(softnet_data);
Stephen Hemminger24f8b232008-11-03 17:14:38 -08004278 unsigned long time_limit = jiffies + 2;
Stephen Hemminger51b0bde2005-06-23 20:14:40 -07004279 int budget = netdev_budget;
Matt Mackall53fb95d2005-08-11 19:27:43 -07004280 void *have;
4281
Linus Torvalds1da177e2005-04-16 15:20:36 -07004282 local_irq_disable();
4283
Eric Dumazete326bed2010-04-22 00:22:45 -07004284 while (!list_empty(&sd->poll_list)) {
Stephen Hemmingerbea33482007-10-03 16:41:36 -07004285 struct napi_struct *n;
4286 int work, weight;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004287
Stephen Hemmingerbea33482007-10-03 16:41:36 -07004288 /* If softirq window is exhuasted then punt.
Stephen Hemminger24f8b232008-11-03 17:14:38 -08004289 * Allow this to run for 2 jiffies since which will allow
4290 * an average latency of 1.5/HZ.
Stephen Hemmingerbea33482007-10-03 16:41:36 -07004291 */
Eric Dumazetd1f41b62013-03-05 07:15:13 +00004292 if (unlikely(budget <= 0 || time_after_eq(jiffies, time_limit)))
Linus Torvalds1da177e2005-04-16 15:20:36 -07004293 goto softnet_break;
4294
4295 local_irq_enable();
4296
Stephen Hemmingerbea33482007-10-03 16:41:36 -07004297 /* Even though interrupts have been re-enabled, this
4298 * access is safe because interrupts can only add new
4299 * entries to the tail of this list, and only ->poll()
4300 * calls can remove this head entry from the list.
4301 */
Eric Dumazete326bed2010-04-22 00:22:45 -07004302 n = list_first_entry(&sd->poll_list, struct napi_struct, poll_list);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004303
Stephen Hemmingerbea33482007-10-03 16:41:36 -07004304 have = netpoll_poll_lock(n);
4305
4306 weight = n->weight;
4307
David S. Miller0a7606c2007-10-29 21:28:47 -07004308 /* This NAPI_STATE_SCHED test is for avoiding a race
4309 * with netpoll's poll_napi(). Only the entity which
4310 * obtains the lock and sees NAPI_STATE_SCHED set will
4311 * actually make the ->poll() call. Therefore we avoid
Lucas De Marchi25985ed2011-03-30 22:57:33 -03004312 * accidentally calling ->poll() when NAPI is not scheduled.
David S. Miller0a7606c2007-10-29 21:28:47 -07004313 */
4314 work = 0;
Neil Horman4ea7e382009-05-21 07:36:08 +00004315 if (test_bit(NAPI_STATE_SCHED, &n->state)) {
David S. Miller0a7606c2007-10-29 21:28:47 -07004316 work = n->poll(n, weight);
Neil Horman4ea7e382009-05-21 07:36:08 +00004317 trace_napi_poll(n);
4318 }
Stephen Hemmingerbea33482007-10-03 16:41:36 -07004319
4320 WARN_ON_ONCE(work > weight);
4321
4322 budget -= work;
4323
4324 local_irq_disable();
4325
4326 /* Drivers must not modify the NAPI state if they
4327 * consume the entire weight. In such cases this code
4328 * still "owns" the NAPI instance and therefore can
4329 * move the instance around on the list at-will.
4330 */
David S. Millerfed17f32008-01-07 21:00:40 -08004331 if (unlikely(work == weight)) {
Herbert Xuff780cd2009-06-26 19:27:04 -07004332 if (unlikely(napi_disable_pending(n))) {
4333 local_irq_enable();
4334 napi_complete(n);
4335 local_irq_disable();
Eric Dumazet2e71a6f2012-10-06 08:08:49 +00004336 } else {
4337 if (n->gro_list) {
4338 /* flush too old packets
4339 * If HZ < 1000, flush all packets.
4340 */
4341 local_irq_enable();
4342 napi_gro_flush(n, HZ >= 1000);
4343 local_irq_disable();
4344 }
Eric Dumazete326bed2010-04-22 00:22:45 -07004345 list_move_tail(&n->poll_list, &sd->poll_list);
Eric Dumazet2e71a6f2012-10-06 08:08:49 +00004346 }
David S. Millerfed17f32008-01-07 21:00:40 -08004347 }
Stephen Hemmingerbea33482007-10-03 16:41:36 -07004348
4349 netpoll_poll_unlock(have);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004350 }
4351out:
Eric Dumazete326bed2010-04-22 00:22:45 -07004352 net_rps_action_and_irq_enable(sd);
Tom Herbert0a9627f2010-03-16 08:03:29 +00004353
Chris Leechdb217332006-06-17 21:24:58 -07004354#ifdef CONFIG_NET_DMA
4355 /*
4356 * There may not be any more sk_buffs coming right now, so push
4357 * any pending DMA copies to hardware
4358 */
Dan Williams2ba05622009-01-06 11:38:14 -07004359 dma_issue_pending_all();
Chris Leechdb217332006-06-17 21:24:58 -07004360#endif
Stephen Hemmingerbea33482007-10-03 16:41:36 -07004361
Linus Torvalds1da177e2005-04-16 15:20:36 -07004362 return;
4363
4364softnet_break:
Changli Gaodee42872010-05-02 05:42:16 +00004365 sd->time_squeeze++;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004366 __raise_softirq_irqoff(NET_RX_SOFTIRQ);
4367 goto out;
4368}
4369
Veaceslav Falicoaa9d8562013-08-28 23:25:04 +02004370struct netdev_adjacent {
Jiri Pirko9ff162a2013-01-03 22:48:49 +00004371 struct net_device *dev;
Veaceslav Falico5d261912013-08-28 23:25:05 +02004372
4373 /* upper master flag, there can only be one master device per list */
Jiri Pirko9ff162a2013-01-03 22:48:49 +00004374 bool master;
Veaceslav Falico5d261912013-08-28 23:25:05 +02004375
4376 /* indicates that this dev is our first-level lower/upper device */
4377 bool neighbour;
4378
4379 /* counter for the number of times this device was added to us */
4380 u16 ref_nr;
4381
Jiri Pirko9ff162a2013-01-03 22:48:49 +00004382 struct list_head list;
4383 struct rcu_head rcu;
Jiri Pirko9ff162a2013-01-03 22:48:49 +00004384};
4385
Veaceslav Falico5d261912013-08-28 23:25:05 +02004386static struct netdev_adjacent *__netdev_find_adj(struct net_device *dev,
4387 struct net_device *adj_dev,
4388 bool upper)
Jiri Pirko9ff162a2013-01-03 22:48:49 +00004389{
Veaceslav Falico5d261912013-08-28 23:25:05 +02004390 struct netdev_adjacent *adj;
4391 struct list_head *dev_list;
Jiri Pirko9ff162a2013-01-03 22:48:49 +00004392
Veaceslav Falico5d261912013-08-28 23:25:05 +02004393 dev_list = upper ? &dev->upper_dev_list : &dev->lower_dev_list;
4394
4395 list_for_each_entry(adj, dev_list, list) {
4396 if (adj->dev == adj_dev)
4397 return adj;
Jiri Pirko9ff162a2013-01-03 22:48:49 +00004398 }
4399 return NULL;
4400}
4401
Veaceslav Falico5d261912013-08-28 23:25:05 +02004402static inline struct netdev_adjacent *__netdev_find_upper(struct net_device *dev,
4403 struct net_device *udev)
4404{
4405 return __netdev_find_adj(dev, udev, true);
4406}
4407
4408static inline struct netdev_adjacent *__netdev_find_lower(struct net_device *dev,
4409 struct net_device *ldev)
4410{
4411 return __netdev_find_adj(dev, ldev, false);
4412}
4413
Jiri Pirko9ff162a2013-01-03 22:48:49 +00004414/**
4415 * netdev_has_upper_dev - Check if device is linked to an upper device
4416 * @dev: device
4417 * @upper_dev: upper device to check
4418 *
4419 * Find out if a device is linked to specified upper device and return true
4420 * in case it is. Note that this checks only immediate upper device,
4421 * not through a complete stack of devices. The caller must hold the RTNL lock.
4422 */
4423bool netdev_has_upper_dev(struct net_device *dev,
4424 struct net_device *upper_dev)
4425{
4426 ASSERT_RTNL();
4427
4428 return __netdev_find_upper(dev, upper_dev);
4429}
4430EXPORT_SYMBOL(netdev_has_upper_dev);
4431
4432/**
4433 * netdev_has_any_upper_dev - Check if device is linked to some device
4434 * @dev: device
4435 *
4436 * Find out if a device is linked to an upper device and return true in case
4437 * it is. The caller must hold the RTNL lock.
4438 */
4439bool netdev_has_any_upper_dev(struct net_device *dev)
4440{
4441 ASSERT_RTNL();
4442
4443 return !list_empty(&dev->upper_dev_list);
4444}
4445EXPORT_SYMBOL(netdev_has_any_upper_dev);
4446
4447/**
4448 * netdev_master_upper_dev_get - Get master upper device
4449 * @dev: device
4450 *
4451 * Find a master upper device and return pointer to it or NULL in case
4452 * it's not there. The caller must hold the RTNL lock.
4453 */
4454struct net_device *netdev_master_upper_dev_get(struct net_device *dev)
4455{
Veaceslav Falicoaa9d8562013-08-28 23:25:04 +02004456 struct netdev_adjacent *upper;
Jiri Pirko9ff162a2013-01-03 22:48:49 +00004457
4458 ASSERT_RTNL();
4459
4460 if (list_empty(&dev->upper_dev_list))
4461 return NULL;
4462
4463 upper = list_first_entry(&dev->upper_dev_list,
Veaceslav Falicoaa9d8562013-08-28 23:25:04 +02004464 struct netdev_adjacent, list);
Jiri Pirko9ff162a2013-01-03 22:48:49 +00004465 if (likely(upper->master))
4466 return upper->dev;
4467 return NULL;
4468}
4469EXPORT_SYMBOL(netdev_master_upper_dev_get);
4470
Veaceslav Falico48311f42013-08-28 23:25:07 +02004471/* netdev_upper_get_next_dev_rcu - Get the next dev from upper list
4472 * @dev: device
4473 * @iter: list_head ** of the current position
4474 *
4475 * Gets the next device from the dev's upper list, starting from iter
4476 * position. The caller must hold RCU read lock.
4477 */
4478struct net_device *netdev_upper_get_next_dev_rcu(struct net_device *dev,
4479 struct list_head **iter)
4480{
4481 struct netdev_adjacent *upper;
4482
4483 WARN_ON_ONCE(!rcu_read_lock_held());
4484
4485 upper = list_entry_rcu((*iter)->next, struct netdev_adjacent, list);
4486
4487 if (&upper->list == &dev->upper_dev_list)
4488 return NULL;
4489
4490 *iter = &upper->list;
4491
4492 return upper->dev;
4493}
4494EXPORT_SYMBOL(netdev_upper_get_next_dev_rcu);
4495
Jiri Pirko9ff162a2013-01-03 22:48:49 +00004496/**
4497 * netdev_master_upper_dev_get_rcu - Get master upper device
4498 * @dev: device
4499 *
4500 * Find a master upper device and return pointer to it or NULL in case
4501 * it's not there. The caller must hold the RCU read lock.
4502 */
4503struct net_device *netdev_master_upper_dev_get_rcu(struct net_device *dev)
4504{
Veaceslav Falicoaa9d8562013-08-28 23:25:04 +02004505 struct netdev_adjacent *upper;
Jiri Pirko9ff162a2013-01-03 22:48:49 +00004506
4507 upper = list_first_or_null_rcu(&dev->upper_dev_list,
Veaceslav Falicoaa9d8562013-08-28 23:25:04 +02004508 struct netdev_adjacent, list);
Jiri Pirko9ff162a2013-01-03 22:48:49 +00004509 if (upper && likely(upper->master))
4510 return upper->dev;
4511 return NULL;
4512}
4513EXPORT_SYMBOL(netdev_master_upper_dev_get_rcu);
4514
Veaceslav Falico5d261912013-08-28 23:25:05 +02004515static int __netdev_adjacent_dev_insert(struct net_device *dev,
4516 struct net_device *adj_dev,
4517 bool neighbour, bool master,
4518 bool upper)
4519{
4520 struct netdev_adjacent *adj;
4521
4522 adj = __netdev_find_adj(dev, adj_dev, upper);
4523
4524 if (adj) {
4525 BUG_ON(neighbour);
4526 adj->ref_nr++;
4527 return 0;
4528 }
4529
4530 adj = kmalloc(sizeof(*adj), GFP_KERNEL);
4531 if (!adj)
4532 return -ENOMEM;
4533
4534 adj->dev = adj_dev;
4535 adj->master = master;
4536 adj->neighbour = neighbour;
4537 adj->ref_nr = 1;
Veaceslav Falico5d261912013-08-28 23:25:05 +02004538
4539 dev_hold(adj_dev);
4540 pr_debug("dev_hold for %s, because of %s link added from %s to %s\n",
4541 adj_dev->name, upper ? "upper" : "lower", dev->name,
4542 adj_dev->name);
4543
4544 if (!upper) {
4545 list_add_tail_rcu(&adj->list, &dev->lower_dev_list);
4546 return 0;
4547 }
4548
4549 /* Ensure that master upper link is always the first item in list. */
4550 if (master)
4551 list_add_rcu(&adj->list, &dev->upper_dev_list);
4552 else
4553 list_add_tail_rcu(&adj->list, &dev->upper_dev_list);
4554
4555 return 0;
4556}
4557
4558static inline int __netdev_upper_dev_insert(struct net_device *dev,
4559 struct net_device *udev,
4560 bool master, bool neighbour)
4561{
4562 return __netdev_adjacent_dev_insert(dev, udev, neighbour, master,
4563 true);
4564}
4565
4566static inline int __netdev_lower_dev_insert(struct net_device *dev,
4567 struct net_device *ldev,
4568 bool neighbour)
4569{
4570 return __netdev_adjacent_dev_insert(dev, ldev, neighbour, false,
4571 false);
4572}
4573
4574void __netdev_adjacent_dev_remove(struct net_device *dev,
4575 struct net_device *adj_dev, bool upper)
4576{
4577 struct netdev_adjacent *adj;
4578
4579 if (upper)
4580 adj = __netdev_find_upper(dev, adj_dev);
4581 else
4582 adj = __netdev_find_lower(dev, adj_dev);
4583
4584 if (!adj)
4585 BUG();
4586
4587 if (adj->ref_nr > 1) {
4588 adj->ref_nr--;
4589 return;
4590 }
4591
4592 list_del_rcu(&adj->list);
4593 pr_debug("dev_put for %s, because of %s link removed from %s to %s\n",
4594 adj_dev->name, upper ? "upper" : "lower", dev->name,
4595 adj_dev->name);
4596 dev_put(adj_dev);
4597 kfree_rcu(adj, rcu);
4598}
4599
4600static inline void __netdev_upper_dev_remove(struct net_device *dev,
4601 struct net_device *udev)
4602{
4603 return __netdev_adjacent_dev_remove(dev, udev, true);
4604}
4605
4606static inline void __netdev_lower_dev_remove(struct net_device *dev,
4607 struct net_device *ldev)
4608{
4609 return __netdev_adjacent_dev_remove(dev, ldev, false);
4610}
4611
4612int __netdev_adjacent_dev_insert_link(struct net_device *dev,
4613 struct net_device *upper_dev,
4614 bool master, bool neighbour)
4615{
4616 int ret;
4617
4618 ret = __netdev_upper_dev_insert(dev, upper_dev, master, neighbour);
4619 if (ret)
4620 return ret;
4621
4622 ret = __netdev_lower_dev_insert(upper_dev, dev, neighbour);
4623 if (ret) {
4624 __netdev_upper_dev_remove(dev, upper_dev);
4625 return ret;
4626 }
4627
4628 return 0;
4629}
4630
4631static inline int __netdev_adjacent_dev_link(struct net_device *dev,
4632 struct net_device *udev)
4633{
4634 return __netdev_adjacent_dev_insert_link(dev, udev, false, false);
4635}
4636
4637static inline int __netdev_adjacent_dev_link_neighbour(struct net_device *dev,
4638 struct net_device *udev,
4639 bool master)
4640{
4641 return __netdev_adjacent_dev_insert_link(dev, udev, master, true);
4642}
4643
4644void __netdev_adjacent_dev_unlink(struct net_device *dev,
4645 struct net_device *upper_dev)
4646{
4647 __netdev_upper_dev_remove(dev, upper_dev);
4648 __netdev_lower_dev_remove(upper_dev, dev);
4649}
4650
4651
Jiri Pirko9ff162a2013-01-03 22:48:49 +00004652static int __netdev_upper_dev_link(struct net_device *dev,
4653 struct net_device *upper_dev, bool master)
4654{
Veaceslav Falico5d261912013-08-28 23:25:05 +02004655 struct netdev_adjacent *i, *j, *to_i, *to_j;
4656 int ret = 0;
Jiri Pirko9ff162a2013-01-03 22:48:49 +00004657
4658 ASSERT_RTNL();
4659
4660 if (dev == upper_dev)
4661 return -EBUSY;
4662
4663 /* To prevent loops, check if dev is not upper device to upper_dev. */
Veaceslav Falico620f3182013-08-28 23:25:06 +02004664 if (__netdev_find_upper(upper_dev, dev))
Jiri Pirko9ff162a2013-01-03 22:48:49 +00004665 return -EBUSY;
4666
4667 if (__netdev_find_upper(dev, upper_dev))
4668 return -EEXIST;
4669
4670 if (master && netdev_master_upper_dev_get(dev))
4671 return -EBUSY;
4672
Veaceslav Falico5d261912013-08-28 23:25:05 +02004673 ret = __netdev_adjacent_dev_link_neighbour(dev, upper_dev, master);
4674 if (ret)
4675 return ret;
Jiri Pirko9ff162a2013-01-03 22:48:49 +00004676
Veaceslav Falico5d261912013-08-28 23:25:05 +02004677 /* Now that we linked these devs, make all the upper_dev's
4678 * upper_dev_list visible to every dev's lower_dev_list and vice
4679 * versa, and don't forget the devices itself. All of these
4680 * links are non-neighbours.
4681 */
Veaceslav Falico82476b32013-09-02 16:26:51 +02004682 list_for_each_entry(i, &dev->lower_dev_list, list) {
4683 list_for_each_entry(j, &upper_dev->upper_dev_list, list) {
Veaceslav Falico5d261912013-08-28 23:25:05 +02004684 ret = __netdev_adjacent_dev_link(i->dev, j->dev);
4685 if (ret)
4686 goto rollback_mesh;
4687 }
4688 }
Jiri Pirko9ff162a2013-01-03 22:48:49 +00004689
Veaceslav Falico5d261912013-08-28 23:25:05 +02004690 /* add dev to every upper_dev's upper device */
4691 list_for_each_entry(i, &upper_dev->upper_dev_list, list) {
4692 ret = __netdev_adjacent_dev_link(dev, i->dev);
4693 if (ret)
4694 goto rollback_upper_mesh;
4695 }
4696
4697 /* add upper_dev to every dev's lower device */
4698 list_for_each_entry(i, &dev->lower_dev_list, list) {
4699 ret = __netdev_adjacent_dev_link(i->dev, upper_dev);
4700 if (ret)
4701 goto rollback_lower_mesh;
4702 }
4703
Jiri Pirko42e52bf2013-05-25 04:12:10 +00004704 call_netdevice_notifiers(NETDEV_CHANGEUPPER, dev);
Jiri Pirko9ff162a2013-01-03 22:48:49 +00004705 return 0;
Veaceslav Falico5d261912013-08-28 23:25:05 +02004706
4707rollback_lower_mesh:
4708 to_i = i;
4709 list_for_each_entry(i, &dev->lower_dev_list, list) {
4710 if (i == to_i)
4711 break;
4712 __netdev_adjacent_dev_unlink(i->dev, upper_dev);
4713 }
4714
4715 i = NULL;
4716
4717rollback_upper_mesh:
4718 to_i = i;
4719 list_for_each_entry(i, &upper_dev->upper_dev_list, list) {
4720 if (i == to_i)
4721 break;
4722 __netdev_adjacent_dev_unlink(dev, i->dev);
4723 }
4724
4725 i = j = NULL;
4726
4727rollback_mesh:
4728 to_i = i;
4729 to_j = j;
4730 list_for_each_entry(i, &dev->lower_dev_list, list) {
4731 list_for_each_entry(j, &upper_dev->upper_dev_list, list) {
4732 if (i == to_i && j == to_j)
4733 break;
4734 __netdev_adjacent_dev_unlink(i->dev, j->dev);
4735 }
4736 if (i == to_i)
4737 break;
4738 }
4739
4740 __netdev_adjacent_dev_unlink(dev, upper_dev);
4741
4742 return ret;
Jiri Pirko9ff162a2013-01-03 22:48:49 +00004743}
4744
4745/**
4746 * netdev_upper_dev_link - Add a link to the upper device
4747 * @dev: device
4748 * @upper_dev: new upper device
4749 *
4750 * Adds a link to device which is upper to this one. The caller must hold
4751 * the RTNL lock. On a failure a negative errno code is returned.
4752 * On success the reference counts are adjusted and the function
4753 * returns zero.
4754 */
4755int netdev_upper_dev_link(struct net_device *dev,
4756 struct net_device *upper_dev)
4757{
4758 return __netdev_upper_dev_link(dev, upper_dev, false);
4759}
4760EXPORT_SYMBOL(netdev_upper_dev_link);
4761
4762/**
4763 * netdev_master_upper_dev_link - Add a master link to the upper device
4764 * @dev: device
4765 * @upper_dev: new upper device
4766 *
4767 * Adds a link to device which is upper to this one. In this case, only
4768 * one master upper device can be linked, although other non-master devices
4769 * might be linked as well. The caller must hold the RTNL lock.
4770 * On a failure a negative errno code is returned. On success the reference
4771 * counts are adjusted and the function returns zero.
4772 */
4773int netdev_master_upper_dev_link(struct net_device *dev,
4774 struct net_device *upper_dev)
4775{
4776 return __netdev_upper_dev_link(dev, upper_dev, true);
4777}
4778EXPORT_SYMBOL(netdev_master_upper_dev_link);
4779
4780/**
4781 * netdev_upper_dev_unlink - Removes a link to upper device
4782 * @dev: device
4783 * @upper_dev: new upper device
4784 *
4785 * Removes a link to device which is upper to this one. The caller must hold
4786 * the RTNL lock.
4787 */
4788void netdev_upper_dev_unlink(struct net_device *dev,
4789 struct net_device *upper_dev)
4790{
Veaceslav Falico5d261912013-08-28 23:25:05 +02004791 struct netdev_adjacent *i, *j;
Jiri Pirko9ff162a2013-01-03 22:48:49 +00004792 ASSERT_RTNL();
4793
Veaceslav Falico5d261912013-08-28 23:25:05 +02004794 __netdev_adjacent_dev_unlink(dev, upper_dev);
4795
4796 /* Here is the tricky part. We must remove all dev's lower
4797 * devices from all upper_dev's upper devices and vice
4798 * versa, to maintain the graph relationship.
4799 */
4800 list_for_each_entry(i, &dev->lower_dev_list, list)
4801 list_for_each_entry(j, &upper_dev->upper_dev_list, list)
4802 __netdev_adjacent_dev_unlink(i->dev, j->dev);
4803
4804 /* remove also the devices itself from lower/upper device
4805 * list
4806 */
4807 list_for_each_entry(i, &dev->lower_dev_list, list)
4808 __netdev_adjacent_dev_unlink(i->dev, upper_dev);
4809
4810 list_for_each_entry(i, &upper_dev->upper_dev_list, list)
4811 __netdev_adjacent_dev_unlink(dev, i->dev);
4812
Jiri Pirko42e52bf2013-05-25 04:12:10 +00004813 call_netdevice_notifiers(NETDEV_CHANGEUPPER, dev);
Jiri Pirko9ff162a2013-01-03 22:48:49 +00004814}
4815EXPORT_SYMBOL(netdev_upper_dev_unlink);
4816
Patrick McHardyb6c40d62008-10-07 15:26:48 -07004817static void dev_change_rx_flags(struct net_device *dev, int flags)
4818{
Stephen Hemmingerd3147742008-11-19 21:32:24 -08004819 const struct net_device_ops *ops = dev->netdev_ops;
4820
4821 if ((dev->flags & IFF_UP) && ops->ndo_change_rx_flags)
4822 ops->ndo_change_rx_flags(dev, flags);
Patrick McHardyb6c40d62008-10-07 15:26:48 -07004823}
4824
Wang Chendad9b332008-06-18 01:48:28 -07004825static int __dev_set_promiscuity(struct net_device *dev, int inc)
Patrick McHardy4417da62007-06-27 01:28:10 -07004826{
Eric Dumazetb536db92011-11-30 21:42:26 +00004827 unsigned int old_flags = dev->flags;
Eric W. Biedermand04a48b2012-05-23 17:01:57 -06004828 kuid_t uid;
4829 kgid_t gid;
Patrick McHardy4417da62007-06-27 01:28:10 -07004830
Patrick McHardy24023452007-07-14 18:51:31 -07004831 ASSERT_RTNL();
4832
Wang Chendad9b332008-06-18 01:48:28 -07004833 dev->flags |= IFF_PROMISC;
4834 dev->promiscuity += inc;
4835 if (dev->promiscuity == 0) {
4836 /*
4837 * Avoid overflow.
4838 * If inc causes overflow, untouch promisc and return error.
4839 */
4840 if (inc < 0)
4841 dev->flags &= ~IFF_PROMISC;
4842 else {
4843 dev->promiscuity -= inc;
Joe Perches7b6cd1c2012-02-01 10:54:43 +00004844 pr_warn("%s: promiscuity touches roof, set promiscuity failed. promiscuity feature of device might be broken.\n",
4845 dev->name);
Wang Chendad9b332008-06-18 01:48:28 -07004846 return -EOVERFLOW;
4847 }
4848 }
Patrick McHardy4417da62007-06-27 01:28:10 -07004849 if (dev->flags != old_flags) {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00004850 pr_info("device %s %s promiscuous mode\n",
4851 dev->name,
4852 dev->flags & IFF_PROMISC ? "entered" : "left");
David Howells8192b0c2008-11-14 10:39:10 +11004853 if (audit_enabled) {
4854 current_uid_gid(&uid, &gid);
Klaus Heinrich Kiwi7759db82008-01-23 22:57:45 -05004855 audit_log(current->audit_context, GFP_ATOMIC,
4856 AUDIT_ANOM_PROMISCUOUS,
4857 "dev=%s prom=%d old_prom=%d auid=%u uid=%u gid=%u ses=%u",
4858 dev->name, (dev->flags & IFF_PROMISC),
4859 (old_flags & IFF_PROMISC),
Eric W. Biedermane1760bd2012-09-10 22:39:43 -07004860 from_kuid(&init_user_ns, audit_get_loginuid(current)),
Eric W. Biedermand04a48b2012-05-23 17:01:57 -06004861 from_kuid(&init_user_ns, uid),
4862 from_kgid(&init_user_ns, gid),
Klaus Heinrich Kiwi7759db82008-01-23 22:57:45 -05004863 audit_get_sessionid(current));
David Howells8192b0c2008-11-14 10:39:10 +11004864 }
Patrick McHardy24023452007-07-14 18:51:31 -07004865
Patrick McHardyb6c40d62008-10-07 15:26:48 -07004866 dev_change_rx_flags(dev, IFF_PROMISC);
Patrick McHardy4417da62007-06-27 01:28:10 -07004867 }
Wang Chendad9b332008-06-18 01:48:28 -07004868 return 0;
Patrick McHardy4417da62007-06-27 01:28:10 -07004869}
4870
Linus Torvalds1da177e2005-04-16 15:20:36 -07004871/**
4872 * dev_set_promiscuity - update promiscuity count on a device
4873 * @dev: device
4874 * @inc: modifier
4875 *
Stephen Hemminger3041a062006-05-26 13:25:24 -07004876 * Add or remove promiscuity from a device. While the count in the device
Linus Torvalds1da177e2005-04-16 15:20:36 -07004877 * remains above zero the interface remains promiscuous. Once it hits zero
4878 * the device reverts back to normal filtering operation. A negative inc
4879 * value is used to drop promiscuity on the device.
Wang Chendad9b332008-06-18 01:48:28 -07004880 * Return 0 if successful or a negative errno code on error.
Linus Torvalds1da177e2005-04-16 15:20:36 -07004881 */
Wang Chendad9b332008-06-18 01:48:28 -07004882int dev_set_promiscuity(struct net_device *dev, int inc)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004883{
Eric Dumazetb536db92011-11-30 21:42:26 +00004884 unsigned int old_flags = dev->flags;
Wang Chendad9b332008-06-18 01:48:28 -07004885 int err;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004886
Wang Chendad9b332008-06-18 01:48:28 -07004887 err = __dev_set_promiscuity(dev, inc);
Patrick McHardy4b5a6982008-07-06 15:49:08 -07004888 if (err < 0)
Wang Chendad9b332008-06-18 01:48:28 -07004889 return err;
Patrick McHardy4417da62007-06-27 01:28:10 -07004890 if (dev->flags != old_flags)
4891 dev_set_rx_mode(dev);
Wang Chendad9b332008-06-18 01:48:28 -07004892 return err;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004893}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004894EXPORT_SYMBOL(dev_set_promiscuity);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004895
4896/**
4897 * dev_set_allmulti - update allmulti count on a device
4898 * @dev: device
4899 * @inc: modifier
4900 *
4901 * Add or remove reception of all multicast frames to a device. While the
4902 * count in the device remains above zero the interface remains listening
4903 * to all interfaces. Once it hits zero the device reverts back to normal
4904 * filtering operation. A negative @inc value is used to drop the counter
4905 * when releasing a resource needing all multicasts.
Wang Chendad9b332008-06-18 01:48:28 -07004906 * Return 0 if successful or a negative errno code on error.
Linus Torvalds1da177e2005-04-16 15:20:36 -07004907 */
4908
Wang Chendad9b332008-06-18 01:48:28 -07004909int dev_set_allmulti(struct net_device *dev, int inc)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004910{
Eric Dumazetb536db92011-11-30 21:42:26 +00004911 unsigned int old_flags = dev->flags;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004912
Patrick McHardy24023452007-07-14 18:51:31 -07004913 ASSERT_RTNL();
4914
Linus Torvalds1da177e2005-04-16 15:20:36 -07004915 dev->flags |= IFF_ALLMULTI;
Wang Chendad9b332008-06-18 01:48:28 -07004916 dev->allmulti += inc;
4917 if (dev->allmulti == 0) {
4918 /*
4919 * Avoid overflow.
4920 * If inc causes overflow, untouch allmulti and return error.
4921 */
4922 if (inc < 0)
4923 dev->flags &= ~IFF_ALLMULTI;
4924 else {
4925 dev->allmulti -= inc;
Joe Perches7b6cd1c2012-02-01 10:54:43 +00004926 pr_warn("%s: allmulti touches roof, set allmulti failed. allmulti feature of device might be broken.\n",
4927 dev->name);
Wang Chendad9b332008-06-18 01:48:28 -07004928 return -EOVERFLOW;
4929 }
4930 }
Patrick McHardy24023452007-07-14 18:51:31 -07004931 if (dev->flags ^ old_flags) {
Patrick McHardyb6c40d62008-10-07 15:26:48 -07004932 dev_change_rx_flags(dev, IFF_ALLMULTI);
Patrick McHardy4417da62007-06-27 01:28:10 -07004933 dev_set_rx_mode(dev);
Patrick McHardy24023452007-07-14 18:51:31 -07004934 }
Wang Chendad9b332008-06-18 01:48:28 -07004935 return 0;
Patrick McHardy4417da62007-06-27 01:28:10 -07004936}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004937EXPORT_SYMBOL(dev_set_allmulti);
Patrick McHardy4417da62007-06-27 01:28:10 -07004938
4939/*
4940 * Upload unicast and multicast address lists to device and
4941 * configure RX filtering. When the device doesn't support unicast
Joe Perches53ccaae2007-12-20 14:02:06 -08004942 * filtering it is put in promiscuous mode while unicast addresses
Patrick McHardy4417da62007-06-27 01:28:10 -07004943 * are present.
4944 */
4945void __dev_set_rx_mode(struct net_device *dev)
4946{
Stephen Hemmingerd3147742008-11-19 21:32:24 -08004947 const struct net_device_ops *ops = dev->netdev_ops;
4948
Patrick McHardy4417da62007-06-27 01:28:10 -07004949 /* dev_open will call this function so the list will stay sane. */
4950 if (!(dev->flags&IFF_UP))
4951 return;
4952
4953 if (!netif_device_present(dev))
YOSHIFUJI Hideaki40b77c92007-07-19 10:43:23 +09004954 return;
Patrick McHardy4417da62007-06-27 01:28:10 -07004955
Jiri Pirko01789342011-08-16 06:29:00 +00004956 if (!(dev->priv_flags & IFF_UNICAST_FLT)) {
Patrick McHardy4417da62007-06-27 01:28:10 -07004957 /* Unicast addresses changes may only happen under the rtnl,
4958 * therefore calling __dev_set_promiscuity here is safe.
4959 */
Jiri Pirko32e7bfc2010-01-25 13:36:10 -08004960 if (!netdev_uc_empty(dev) && !dev->uc_promisc) {
Patrick McHardy4417da62007-06-27 01:28:10 -07004961 __dev_set_promiscuity(dev, 1);
Joe Perches2d348d12011-07-25 16:17:35 -07004962 dev->uc_promisc = true;
Jiri Pirko32e7bfc2010-01-25 13:36:10 -08004963 } else if (netdev_uc_empty(dev) && dev->uc_promisc) {
Patrick McHardy4417da62007-06-27 01:28:10 -07004964 __dev_set_promiscuity(dev, -1);
Joe Perches2d348d12011-07-25 16:17:35 -07004965 dev->uc_promisc = false;
Patrick McHardy4417da62007-06-27 01:28:10 -07004966 }
Patrick McHardy4417da62007-06-27 01:28:10 -07004967 }
Jiri Pirko01789342011-08-16 06:29:00 +00004968
4969 if (ops->ndo_set_rx_mode)
4970 ops->ndo_set_rx_mode(dev);
Patrick McHardy4417da62007-06-27 01:28:10 -07004971}
4972
4973void dev_set_rx_mode(struct net_device *dev)
4974{
David S. Millerb9e40852008-07-15 00:15:08 -07004975 netif_addr_lock_bh(dev);
Patrick McHardy4417da62007-06-27 01:28:10 -07004976 __dev_set_rx_mode(dev);
David S. Millerb9e40852008-07-15 00:15:08 -07004977 netif_addr_unlock_bh(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004978}
4979
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07004980/**
4981 * dev_get_flags - get flags reported to userspace
4982 * @dev: device
4983 *
4984 * Get the combination of flag bits exported through APIs to userspace.
4985 */
Eric Dumazet95c96172012-04-15 05:58:06 +00004986unsigned int dev_get_flags(const struct net_device *dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004987{
Eric Dumazet95c96172012-04-15 05:58:06 +00004988 unsigned int flags;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004989
4990 flags = (dev->flags & ~(IFF_PROMISC |
4991 IFF_ALLMULTI |
Stefan Rompfb00055a2006-03-20 17:09:11 -08004992 IFF_RUNNING |
4993 IFF_LOWER_UP |
4994 IFF_DORMANT)) |
Linus Torvalds1da177e2005-04-16 15:20:36 -07004995 (dev->gflags & (IFF_PROMISC |
4996 IFF_ALLMULTI));
4997
Stefan Rompfb00055a2006-03-20 17:09:11 -08004998 if (netif_running(dev)) {
4999 if (netif_oper_up(dev))
5000 flags |= IFF_RUNNING;
5001 if (netif_carrier_ok(dev))
5002 flags |= IFF_LOWER_UP;
5003 if (netif_dormant(dev))
5004 flags |= IFF_DORMANT;
5005 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07005006
5007 return flags;
5008}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005009EXPORT_SYMBOL(dev_get_flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005010
Patrick McHardybd380812010-02-26 06:34:53 +00005011int __dev_change_flags(struct net_device *dev, unsigned int flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -07005012{
Eric Dumazetb536db92011-11-30 21:42:26 +00005013 unsigned int old_flags = dev->flags;
Patrick McHardybd380812010-02-26 06:34:53 +00005014 int ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005015
Patrick McHardy24023452007-07-14 18:51:31 -07005016 ASSERT_RTNL();
5017
Linus Torvalds1da177e2005-04-16 15:20:36 -07005018 /*
5019 * Set the flags on our device.
5020 */
5021
5022 dev->flags = (flags & (IFF_DEBUG | IFF_NOTRAILERS | IFF_NOARP |
5023 IFF_DYNAMIC | IFF_MULTICAST | IFF_PORTSEL |
5024 IFF_AUTOMEDIA)) |
5025 (dev->flags & (IFF_UP | IFF_VOLATILE | IFF_PROMISC |
5026 IFF_ALLMULTI));
5027
5028 /*
5029 * Load in the correct multicast list now the flags have changed.
5030 */
5031
Patrick McHardyb6c40d62008-10-07 15:26:48 -07005032 if ((old_flags ^ flags) & IFF_MULTICAST)
5033 dev_change_rx_flags(dev, IFF_MULTICAST);
Patrick McHardy24023452007-07-14 18:51:31 -07005034
Patrick McHardy4417da62007-06-27 01:28:10 -07005035 dev_set_rx_mode(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005036
5037 /*
5038 * Have we downed the interface. We handle IFF_UP ourselves
5039 * according to user attempts to set it, rather than blindly
5040 * setting it.
5041 */
5042
5043 ret = 0;
5044 if ((old_flags ^ flags) & IFF_UP) { /* Bit is different ? */
Patrick McHardybd380812010-02-26 06:34:53 +00005045 ret = ((old_flags & IFF_UP) ? __dev_close : __dev_open)(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005046
5047 if (!ret)
Patrick McHardy4417da62007-06-27 01:28:10 -07005048 dev_set_rx_mode(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005049 }
5050
Linus Torvalds1da177e2005-04-16 15:20:36 -07005051 if ((flags ^ dev->gflags) & IFF_PROMISC) {
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005052 int inc = (flags & IFF_PROMISC) ? 1 : -1;
5053
Linus Torvalds1da177e2005-04-16 15:20:36 -07005054 dev->gflags ^= IFF_PROMISC;
5055 dev_set_promiscuity(dev, inc);
5056 }
5057
5058 /* NOTE: order of synchronization of IFF_PROMISC and IFF_ALLMULTI
5059 is important. Some (broken) drivers set IFF_PROMISC, when
5060 IFF_ALLMULTI is requested not asking us and not reporting.
5061 */
5062 if ((flags ^ dev->gflags) & IFF_ALLMULTI) {
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005063 int inc = (flags & IFF_ALLMULTI) ? 1 : -1;
5064
Linus Torvalds1da177e2005-04-16 15:20:36 -07005065 dev->gflags ^= IFF_ALLMULTI;
5066 dev_set_allmulti(dev, inc);
5067 }
5068
Patrick McHardybd380812010-02-26 06:34:53 +00005069 return ret;
5070}
5071
5072void __dev_notify_flags(struct net_device *dev, unsigned int old_flags)
5073{
5074 unsigned int changes = dev->flags ^ old_flags;
5075
5076 if (changes & IFF_UP) {
5077 if (dev->flags & IFF_UP)
5078 call_netdevice_notifiers(NETDEV_UP, dev);
5079 else
5080 call_netdevice_notifiers(NETDEV_DOWN, dev);
5081 }
5082
5083 if (dev->flags & IFF_UP &&
Jiri Pirkobe9efd32013-05-28 01:30:22 +00005084 (changes & ~(IFF_UP | IFF_PROMISC | IFF_ALLMULTI | IFF_VOLATILE))) {
5085 struct netdev_notifier_change_info change_info;
5086
5087 change_info.flags_changed = changes;
5088 call_netdevice_notifiers_info(NETDEV_CHANGE, dev,
5089 &change_info.info);
5090 }
Patrick McHardybd380812010-02-26 06:34:53 +00005091}
5092
5093/**
5094 * dev_change_flags - change device settings
5095 * @dev: device
5096 * @flags: device state flags
5097 *
5098 * Change settings on device based state flags. The flags are
5099 * in the userspace exported format.
5100 */
Eric Dumazetb536db92011-11-30 21:42:26 +00005101int dev_change_flags(struct net_device *dev, unsigned int flags)
Patrick McHardybd380812010-02-26 06:34:53 +00005102{
Eric Dumazetb536db92011-11-30 21:42:26 +00005103 int ret;
5104 unsigned int changes, old_flags = dev->flags;
Patrick McHardybd380812010-02-26 06:34:53 +00005105
5106 ret = __dev_change_flags(dev, flags);
5107 if (ret < 0)
5108 return ret;
5109
5110 changes = old_flags ^ dev->flags;
Thomas Graf7c355f52007-06-05 16:03:03 -07005111 if (changes)
5112 rtmsg_ifinfo(RTM_NEWLINK, dev, changes);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005113
Patrick McHardybd380812010-02-26 06:34:53 +00005114 __dev_notify_flags(dev, old_flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005115 return ret;
5116}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005117EXPORT_SYMBOL(dev_change_flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005118
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07005119/**
5120 * dev_set_mtu - Change maximum transfer unit
5121 * @dev: device
5122 * @new_mtu: new transfer unit
5123 *
5124 * Change the maximum transfer size of the network device.
5125 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07005126int dev_set_mtu(struct net_device *dev, int new_mtu)
5127{
Stephen Hemmingerd3147742008-11-19 21:32:24 -08005128 const struct net_device_ops *ops = dev->netdev_ops;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005129 int err;
5130
5131 if (new_mtu == dev->mtu)
5132 return 0;
5133
5134 /* MTU must be positive. */
5135 if (new_mtu < 0)
5136 return -EINVAL;
5137
5138 if (!netif_device_present(dev))
5139 return -ENODEV;
5140
5141 err = 0;
Stephen Hemmingerd3147742008-11-19 21:32:24 -08005142 if (ops->ndo_change_mtu)
5143 err = ops->ndo_change_mtu(dev, new_mtu);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005144 else
5145 dev->mtu = new_mtu;
Stephen Hemmingerd3147742008-11-19 21:32:24 -08005146
Jiri Pirkoe3d8fab2012-12-03 01:16:32 +00005147 if (!err)
Pavel Emelyanov056925a2007-09-16 15:42:43 -07005148 call_netdevice_notifiers(NETDEV_CHANGEMTU, dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005149 return err;
5150}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005151EXPORT_SYMBOL(dev_set_mtu);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005152
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07005153/**
Vlad Dogarucbda10f2011-01-13 23:38:30 +00005154 * dev_set_group - Change group this device belongs to
5155 * @dev: device
5156 * @new_group: group this device should belong to
5157 */
5158void dev_set_group(struct net_device *dev, int new_group)
5159{
5160 dev->group = new_group;
5161}
5162EXPORT_SYMBOL(dev_set_group);
5163
5164/**
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07005165 * dev_set_mac_address - Change Media Access Control Address
5166 * @dev: device
5167 * @sa: new address
5168 *
5169 * Change the hardware (MAC) address of the device
5170 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07005171int dev_set_mac_address(struct net_device *dev, struct sockaddr *sa)
5172{
Stephen Hemmingerd3147742008-11-19 21:32:24 -08005173 const struct net_device_ops *ops = dev->netdev_ops;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005174 int err;
5175
Stephen Hemmingerd3147742008-11-19 21:32:24 -08005176 if (!ops->ndo_set_mac_address)
Linus Torvalds1da177e2005-04-16 15:20:36 -07005177 return -EOPNOTSUPP;
5178 if (sa->sa_family != dev->type)
5179 return -EINVAL;
5180 if (!netif_device_present(dev))
5181 return -ENODEV;
Stephen Hemmingerd3147742008-11-19 21:32:24 -08005182 err = ops->ndo_set_mac_address(dev, sa);
Jiri Pirkof6521512013-01-01 03:30:14 +00005183 if (err)
5184 return err;
Jiri Pirkofbdeca22013-01-01 03:30:16 +00005185 dev->addr_assign_type = NET_ADDR_SET;
Jiri Pirkof6521512013-01-01 03:30:14 +00005186 call_netdevice_notifiers(NETDEV_CHANGEADDR, dev);
Theodore Ts'o7bf23572012-07-04 21:23:25 -04005187 add_device_randomness(dev->dev_addr, dev->addr_len);
Jiri Pirkof6521512013-01-01 03:30:14 +00005188 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005189}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005190EXPORT_SYMBOL(dev_set_mac_address);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005191
Jiri Pirko4bf84c32012-12-27 23:49:37 +00005192/**
5193 * dev_change_carrier - Change device carrier
5194 * @dev: device
Randy Dunlap691b3b72013-03-04 12:32:43 +00005195 * @new_carrier: new value
Jiri Pirko4bf84c32012-12-27 23:49:37 +00005196 *
5197 * Change device carrier
5198 */
5199int dev_change_carrier(struct net_device *dev, bool new_carrier)
5200{
5201 const struct net_device_ops *ops = dev->netdev_ops;
5202
5203 if (!ops->ndo_change_carrier)
5204 return -EOPNOTSUPP;
5205 if (!netif_device_present(dev))
5206 return -ENODEV;
5207 return ops->ndo_change_carrier(dev, new_carrier);
5208}
5209EXPORT_SYMBOL(dev_change_carrier);
5210
Linus Torvalds1da177e2005-04-16 15:20:36 -07005211/**
Jiri Pirko66b52b02013-07-29 18:16:49 +02005212 * dev_get_phys_port_id - Get device physical port ID
5213 * @dev: device
5214 * @ppid: port ID
5215 *
5216 * Get device physical port ID
5217 */
5218int dev_get_phys_port_id(struct net_device *dev,
5219 struct netdev_phys_port_id *ppid)
5220{
5221 const struct net_device_ops *ops = dev->netdev_ops;
5222
5223 if (!ops->ndo_get_phys_port_id)
5224 return -EOPNOTSUPP;
5225 return ops->ndo_get_phys_port_id(dev, ppid);
5226}
5227EXPORT_SYMBOL(dev_get_phys_port_id);
5228
5229/**
Linus Torvalds1da177e2005-04-16 15:20:36 -07005230 * dev_new_index - allocate an ifindex
Randy Dunlapc4ea43c2007-10-12 21:17:49 -07005231 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -07005232 *
5233 * Returns a suitable unique value for a new device interface
5234 * number. The caller must hold the rtnl semaphore or the
5235 * dev_base_lock to be sure it remains unique.
5236 */
Eric W. Biederman881d9662007-09-17 11:56:21 -07005237static int dev_new_index(struct net *net)
Linus Torvalds1da177e2005-04-16 15:20:36 -07005238{
Pavel Emelyanovaa79e662012-08-08 21:53:19 +00005239 int ifindex = net->ifindex;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005240 for (;;) {
5241 if (++ifindex <= 0)
5242 ifindex = 1;
Eric W. Biederman881d9662007-09-17 11:56:21 -07005243 if (!__dev_get_by_index(net, ifindex))
Pavel Emelyanovaa79e662012-08-08 21:53:19 +00005244 return net->ifindex = ifindex;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005245 }
5246}
5247
Linus Torvalds1da177e2005-04-16 15:20:36 -07005248/* Delayed registration/unregisteration */
Denis Cheng3b5b34f2007-12-07 00:49:17 -08005249static LIST_HEAD(net_todo_list);
Eric W. Biederman50624c92013-09-23 21:19:49 -07005250static DECLARE_WAIT_QUEUE_HEAD(netdev_unregistering_wq);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005251
Stephen Hemminger6f05f622007-03-08 20:46:03 -08005252static void net_set_todo(struct net_device *dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07005253{
Linus Torvalds1da177e2005-04-16 15:20:36 -07005254 list_add_tail(&dev->todo_list, &net_todo_list);
Eric W. Biederman50624c92013-09-23 21:19:49 -07005255 dev_net(dev)->dev_unreg_count++;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005256}
5257
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005258static void rollback_registered_many(struct list_head *head)
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07005259{
Krishna Kumare93737b2009-12-08 22:26:02 +00005260 struct net_device *dev, *tmp;
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005261
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07005262 BUG_ON(dev_boot_phase);
5263 ASSERT_RTNL();
5264
Krishna Kumare93737b2009-12-08 22:26:02 +00005265 list_for_each_entry_safe(dev, tmp, head, unreg_list) {
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005266 /* Some devices call without registering
Krishna Kumare93737b2009-12-08 22:26:02 +00005267 * for initialization unwind. Remove those
5268 * devices and proceed with the remaining.
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005269 */
5270 if (dev->reg_state == NETREG_UNINITIALIZED) {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00005271 pr_debug("unregister_netdevice: device %s/%p never was registered\n",
5272 dev->name, dev);
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07005273
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005274 WARN_ON(1);
Krishna Kumare93737b2009-12-08 22:26:02 +00005275 list_del(&dev->unreg_list);
5276 continue;
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005277 }
Eric Dumazet449f4542011-05-19 12:24:16 +00005278 dev->dismantle = true;
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005279 BUG_ON(dev->reg_state != NETREG_REGISTERED);
Octavian Purdila44345722010-12-13 12:44:07 +00005280 }
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005281
Octavian Purdila44345722010-12-13 12:44:07 +00005282 /* If device is running, close it first. */
5283 dev_close_many(head);
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005284
Octavian Purdila44345722010-12-13 12:44:07 +00005285 list_for_each_entry(dev, head, unreg_list) {
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005286 /* And unlink it from device chain. */
5287 unlist_netdevice(dev);
5288
5289 dev->reg_state = NETREG_UNREGISTERING;
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07005290 }
5291
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005292 synchronize_net();
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07005293
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005294 list_for_each_entry(dev, head, unreg_list) {
5295 /* Shutdown queueing discipline. */
5296 dev_shutdown(dev);
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07005297
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07005298
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005299 /* Notify protocols, that we are about to destroy
5300 this device. They should clean all the things.
5301 */
5302 call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
5303
Patrick McHardya2835762010-02-26 06:34:51 +00005304 if (!dev->rtnl_link_ops ||
5305 dev->rtnl_link_state == RTNL_LINK_INITIALIZED)
5306 rtmsg_ifinfo(RTM_DELLINK, dev, ~0U);
5307
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005308 /*
5309 * Flush the unicast and multicast chains
5310 */
Jiri Pirkoa748ee22010-04-01 21:22:09 +00005311 dev_uc_flush(dev);
Jiri Pirko22bedad32010-04-01 21:22:57 +00005312 dev_mc_flush(dev);
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005313
5314 if (dev->netdev_ops->ndo_uninit)
5315 dev->netdev_ops->ndo_uninit(dev);
5316
Jiri Pirko9ff162a2013-01-03 22:48:49 +00005317 /* Notifier chain MUST detach us all upper devices. */
5318 WARN_ON(netdev_has_any_upper_dev(dev));
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005319
5320 /* Remove entries from kobject tree */
5321 netdev_unregister_kobject(dev);
Alexander Duyck024e9672013-01-10 08:57:46 +00005322#ifdef CONFIG_XPS
5323 /* Remove XPS queueing entries */
5324 netif_reset_xps_queues_gt(dev, 0);
5325#endif
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005326 }
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07005327
Eric W. Biederman850a5452011-10-13 22:25:23 +00005328 synchronize_net();
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07005329
Eric W. Biedermana5ee1552009-11-29 15:45:58 +00005330 list_for_each_entry(dev, head, unreg_list)
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005331 dev_put(dev);
5332}
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07005333
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005334static void rollback_registered(struct net_device *dev)
5335{
5336 LIST_HEAD(single);
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07005337
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005338 list_add(&dev->unreg_list, &single);
5339 rollback_registered_many(&single);
Eric Dumazetceaaec92011-02-17 22:59:19 +00005340 list_del(&single);
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07005341}
5342
Michał Mirosławc8f44af2011-11-15 15:29:55 +00005343static netdev_features_t netdev_fix_features(struct net_device *dev,
5344 netdev_features_t features)
Herbert Xub63365a2008-10-23 01:11:29 -07005345{
Michał Mirosław57422dc2011-01-22 12:14:12 +00005346 /* Fix illegal checksum combinations */
5347 if ((features & NETIF_F_HW_CSUM) &&
5348 (features & (NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM))) {
Michał Mirosław6f404e42011-05-16 15:14:21 -04005349 netdev_warn(dev, "mixed HW and IP checksum settings.\n");
Michał Mirosław57422dc2011-01-22 12:14:12 +00005350 features &= ~(NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM);
5351 }
5352
Herbert Xub63365a2008-10-23 01:11:29 -07005353 /* TSO requires that SG is present as well. */
Ben Hutchingsea2d3682011-04-12 14:38:37 +00005354 if ((features & NETIF_F_ALL_TSO) && !(features & NETIF_F_SG)) {
Michał Mirosław6f404e42011-05-16 15:14:21 -04005355 netdev_dbg(dev, "Dropping TSO features since no SG feature.\n");
Ben Hutchingsea2d3682011-04-12 14:38:37 +00005356 features &= ~NETIF_F_ALL_TSO;
Herbert Xub63365a2008-10-23 01:11:29 -07005357 }
5358
Pravin B Shelarec5f0612013-03-07 09:28:01 +00005359 if ((features & NETIF_F_TSO) && !(features & NETIF_F_HW_CSUM) &&
5360 !(features & NETIF_F_IP_CSUM)) {
5361 netdev_dbg(dev, "Dropping TSO features since no CSUM feature.\n");
5362 features &= ~NETIF_F_TSO;
5363 features &= ~NETIF_F_TSO_ECN;
5364 }
5365
5366 if ((features & NETIF_F_TSO6) && !(features & NETIF_F_HW_CSUM) &&
5367 !(features & NETIF_F_IPV6_CSUM)) {
5368 netdev_dbg(dev, "Dropping TSO6 features since no CSUM feature.\n");
5369 features &= ~NETIF_F_TSO6;
5370 }
5371
Ben Hutchings31d8b9e2011-04-12 14:47:15 +00005372 /* TSO ECN requires that TSO is present as well. */
5373 if ((features & NETIF_F_ALL_TSO) == NETIF_F_TSO_ECN)
5374 features &= ~NETIF_F_TSO_ECN;
5375
Michał Mirosław212b5732011-02-15 16:59:16 +00005376 /* Software GSO depends on SG. */
5377 if ((features & NETIF_F_GSO) && !(features & NETIF_F_SG)) {
Michał Mirosław6f404e42011-05-16 15:14:21 -04005378 netdev_dbg(dev, "Dropping NETIF_F_GSO since no SG feature.\n");
Michał Mirosław212b5732011-02-15 16:59:16 +00005379 features &= ~NETIF_F_GSO;
5380 }
5381
Michał Mirosławacd11302011-01-24 15:45:15 -08005382 /* UFO needs SG and checksumming */
Herbert Xub63365a2008-10-23 01:11:29 -07005383 if (features & NETIF_F_UFO) {
Michał Mirosław79032642010-11-30 06:38:00 +00005384 /* maybe split UFO into V4 and V6? */
5385 if (!((features & NETIF_F_GEN_CSUM) ||
5386 (features & (NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM))
5387 == (NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM))) {
Michał Mirosław6f404e42011-05-16 15:14:21 -04005388 netdev_dbg(dev,
Michał Mirosławacd11302011-01-24 15:45:15 -08005389 "Dropping NETIF_F_UFO since no checksum offload features.\n");
Herbert Xub63365a2008-10-23 01:11:29 -07005390 features &= ~NETIF_F_UFO;
5391 }
5392
5393 if (!(features & NETIF_F_SG)) {
Michał Mirosław6f404e42011-05-16 15:14:21 -04005394 netdev_dbg(dev,
Michał Mirosławacd11302011-01-24 15:45:15 -08005395 "Dropping NETIF_F_UFO since no NETIF_F_SG feature.\n");
Herbert Xub63365a2008-10-23 01:11:29 -07005396 features &= ~NETIF_F_UFO;
5397 }
5398 }
5399
5400 return features;
5401}
Herbert Xub63365a2008-10-23 01:11:29 -07005402
Michał Mirosław6cb6a272011-04-02 22:48:47 -07005403int __netdev_update_features(struct net_device *dev)
Michał Mirosław5455c692011-02-15 16:59:17 +00005404{
Michał Mirosławc8f44af2011-11-15 15:29:55 +00005405 netdev_features_t features;
Michał Mirosław5455c692011-02-15 16:59:17 +00005406 int err = 0;
5407
Michał Mirosław87267482011-04-12 09:56:38 +00005408 ASSERT_RTNL();
5409
Michał Mirosław5455c692011-02-15 16:59:17 +00005410 features = netdev_get_wanted_features(dev);
5411
5412 if (dev->netdev_ops->ndo_fix_features)
5413 features = dev->netdev_ops->ndo_fix_features(dev, features);
5414
5415 /* driver might be less strict about feature dependencies */
5416 features = netdev_fix_features(dev, features);
5417
5418 if (dev->features == features)
Michał Mirosław6cb6a272011-04-02 22:48:47 -07005419 return 0;
Michał Mirosław5455c692011-02-15 16:59:17 +00005420
Michał Mirosławc8f44af2011-11-15 15:29:55 +00005421 netdev_dbg(dev, "Features changed: %pNF -> %pNF\n",
5422 &dev->features, &features);
Michał Mirosław5455c692011-02-15 16:59:17 +00005423
5424 if (dev->netdev_ops->ndo_set_features)
5425 err = dev->netdev_ops->ndo_set_features(dev, features);
5426
Michał Mirosław6cb6a272011-04-02 22:48:47 -07005427 if (unlikely(err < 0)) {
Michał Mirosław5455c692011-02-15 16:59:17 +00005428 netdev_err(dev,
Michał Mirosławc8f44af2011-11-15 15:29:55 +00005429 "set_features() failed (%d); wanted %pNF, left %pNF\n",
5430 err, &features, &dev->features);
Michał Mirosław6cb6a272011-04-02 22:48:47 -07005431 return -1;
5432 }
5433
5434 if (!err)
5435 dev->features = features;
5436
5437 return 1;
5438}
5439
Michał Mirosławafe12cc2011-05-07 03:22:17 +00005440/**
5441 * netdev_update_features - recalculate device features
5442 * @dev: the device to check
5443 *
5444 * Recalculate dev->features set and send notifications if it
5445 * has changed. Should be called after driver or hardware dependent
5446 * conditions might have changed that influence the features.
5447 */
Michał Mirosław6cb6a272011-04-02 22:48:47 -07005448void netdev_update_features(struct net_device *dev)
5449{
5450 if (__netdev_update_features(dev))
5451 netdev_features_change(dev);
Michał Mirosław5455c692011-02-15 16:59:17 +00005452}
5453EXPORT_SYMBOL(netdev_update_features);
5454
Linus Torvalds1da177e2005-04-16 15:20:36 -07005455/**
Michał Mirosławafe12cc2011-05-07 03:22:17 +00005456 * netdev_change_features - recalculate device features
5457 * @dev: the device to check
5458 *
5459 * Recalculate dev->features set and send notifications even
5460 * if they have not changed. Should be called instead of
5461 * netdev_update_features() if also dev->vlan_features might
5462 * have changed to allow the changes to be propagated to stacked
5463 * VLAN devices.
5464 */
5465void netdev_change_features(struct net_device *dev)
5466{
5467 __netdev_update_features(dev);
5468 netdev_features_change(dev);
5469}
5470EXPORT_SYMBOL(netdev_change_features);
5471
5472/**
Patrick Mullaneyfc4a7482009-12-03 15:59:22 -08005473 * netif_stacked_transfer_operstate - transfer operstate
5474 * @rootdev: the root or lower level device to transfer state from
5475 * @dev: the device to transfer operstate to
5476 *
5477 * Transfer operational state from root to device. This is normally
5478 * called when a stacking relationship exists between the root
5479 * device and the device(a leaf device).
5480 */
5481void netif_stacked_transfer_operstate(const struct net_device *rootdev,
5482 struct net_device *dev)
5483{
5484 if (rootdev->operstate == IF_OPER_DORMANT)
5485 netif_dormant_on(dev);
5486 else
5487 netif_dormant_off(dev);
5488
5489 if (netif_carrier_ok(rootdev)) {
5490 if (!netif_carrier_ok(dev))
5491 netif_carrier_on(dev);
5492 } else {
5493 if (netif_carrier_ok(dev))
5494 netif_carrier_off(dev);
5495 }
5496}
5497EXPORT_SYMBOL(netif_stacked_transfer_operstate);
5498
Tom Herbertbf264142010-11-26 08:36:09 +00005499#ifdef CONFIG_RPS
Eric Dumazet1b4bf462010-09-23 17:26:35 +00005500static int netif_alloc_rx_queues(struct net_device *dev)
5501{
Eric Dumazet1b4bf462010-09-23 17:26:35 +00005502 unsigned int i, count = dev->num_rx_queues;
Tom Herbertbd25fa72010-10-18 18:00:16 +00005503 struct netdev_rx_queue *rx;
Eric Dumazet1b4bf462010-09-23 17:26:35 +00005504
Tom Herbertbd25fa72010-10-18 18:00:16 +00005505 BUG_ON(count < 1);
Eric Dumazet1b4bf462010-09-23 17:26:35 +00005506
Tom Herbertbd25fa72010-10-18 18:00:16 +00005507 rx = kcalloc(count, sizeof(struct netdev_rx_queue), GFP_KERNEL);
Joe Perches62b59422013-02-04 16:48:16 +00005508 if (!rx)
Tom Herbertbd25fa72010-10-18 18:00:16 +00005509 return -ENOMEM;
Joe Perches62b59422013-02-04 16:48:16 +00005510
Tom Herbertbd25fa72010-10-18 18:00:16 +00005511 dev->_rx = rx;
5512
Tom Herbertbd25fa72010-10-18 18:00:16 +00005513 for (i = 0; i < count; i++)
Tom Herbertfe822242010-11-09 10:47:38 +00005514 rx[i].dev = dev;
Eric Dumazet1b4bf462010-09-23 17:26:35 +00005515 return 0;
5516}
Tom Herbertbf264142010-11-26 08:36:09 +00005517#endif
Eric Dumazet1b4bf462010-09-23 17:26:35 +00005518
Changli Gaoaa942102010-12-04 02:31:41 +00005519static void netdev_init_one_queue(struct net_device *dev,
5520 struct netdev_queue *queue, void *_unused)
5521{
5522 /* Initialize queue lock */
5523 spin_lock_init(&queue->_xmit_lock);
5524 netdev_set_xmit_lockdep_class(&queue->_xmit_lock, dev->type);
5525 queue->xmit_lock_owner = -1;
Changli Gaob236da62010-12-14 03:09:15 +00005526 netdev_queue_numa_node_write(queue, NUMA_NO_NODE);
Changli Gaoaa942102010-12-04 02:31:41 +00005527 queue->dev = dev;
Tom Herbert114cf582011-11-28 16:33:09 +00005528#ifdef CONFIG_BQL
5529 dql_init(&queue->dql, HZ);
5530#endif
Changli Gaoaa942102010-12-04 02:31:41 +00005531}
5532
Eric Dumazet60877a32013-06-20 01:15:51 -07005533static void netif_free_tx_queues(struct net_device *dev)
5534{
5535 if (is_vmalloc_addr(dev->_tx))
5536 vfree(dev->_tx);
5537 else
5538 kfree(dev->_tx);
5539}
5540
Tom Herberte6484932010-10-18 18:04:39 +00005541static int netif_alloc_netdev_queues(struct net_device *dev)
5542{
5543 unsigned int count = dev->num_tx_queues;
5544 struct netdev_queue *tx;
Eric Dumazet60877a32013-06-20 01:15:51 -07005545 size_t sz = count * sizeof(*tx);
Tom Herberte6484932010-10-18 18:04:39 +00005546
Eric Dumazet60877a32013-06-20 01:15:51 -07005547 BUG_ON(count < 1 || count > 0xffff);
Tom Herberte6484932010-10-18 18:04:39 +00005548
Eric Dumazet60877a32013-06-20 01:15:51 -07005549 tx = kzalloc(sz, GFP_KERNEL | __GFP_NOWARN | __GFP_REPEAT);
5550 if (!tx) {
5551 tx = vzalloc(sz);
5552 if (!tx)
5553 return -ENOMEM;
5554 }
Tom Herberte6484932010-10-18 18:04:39 +00005555 dev->_tx = tx;
Tom Herbert1d24eb42010-11-21 13:17:27 +00005556
Tom Herberte6484932010-10-18 18:04:39 +00005557 netdev_for_each_tx_queue(dev, netdev_init_one_queue, NULL);
5558 spin_lock_init(&dev->tx_global_lock);
Changli Gaoaa942102010-12-04 02:31:41 +00005559
5560 return 0;
Tom Herberte6484932010-10-18 18:04:39 +00005561}
5562
Patrick Mullaneyfc4a7482009-12-03 15:59:22 -08005563/**
Linus Torvalds1da177e2005-04-16 15:20:36 -07005564 * register_netdevice - register a network device
5565 * @dev: device to register
5566 *
5567 * Take a completed network device structure and add it to the kernel
5568 * interfaces. A %NETDEV_REGISTER message is sent to the netdev notifier
5569 * chain. 0 is returned on success. A negative errno code is returned
5570 * on a failure to set up the device, or if the name is a duplicate.
5571 *
5572 * Callers must hold the rtnl semaphore. You may want
5573 * register_netdev() instead of this.
5574 *
5575 * BUGS:
5576 * The locking appears insufficient to guarantee two parallel registers
5577 * will not get the same name.
5578 */
5579
5580int register_netdevice(struct net_device *dev)
5581{
Linus Torvalds1da177e2005-04-16 15:20:36 -07005582 int ret;
Stephen Hemmingerd3147742008-11-19 21:32:24 -08005583 struct net *net = dev_net(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005584
5585 BUG_ON(dev_boot_phase);
5586 ASSERT_RTNL();
5587
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07005588 might_sleep();
5589
Linus Torvalds1da177e2005-04-16 15:20:36 -07005590 /* When net_device's are persistent, this will be fatal. */
5591 BUG_ON(dev->reg_state != NETREG_UNINITIALIZED);
Stephen Hemmingerd3147742008-11-19 21:32:24 -08005592 BUG_ON(!net);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005593
David S. Millerf1f28aa2008-07-15 00:08:33 -07005594 spin_lock_init(&dev->addr_list_lock);
David S. Millercf508b12008-07-22 14:16:42 -07005595 netdev_set_addr_lockdep_class(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005596
Linus Torvalds1da177e2005-04-16 15:20:36 -07005597 dev->iflink = -1;
5598
Gao feng828de4f2012-09-13 20:58:27 +00005599 ret = dev_get_valid_name(net, dev, dev->name);
Peter Pan(潘卫平)0696c3a2011-05-12 15:46:56 +00005600 if (ret < 0)
5601 goto out;
5602
Linus Torvalds1da177e2005-04-16 15:20:36 -07005603 /* Init, if this function is available */
Stephen Hemmingerd3147742008-11-19 21:32:24 -08005604 if (dev->netdev_ops->ndo_init) {
5605 ret = dev->netdev_ops->ndo_init(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005606 if (ret) {
5607 if (ret > 0)
5608 ret = -EIO;
Adrian Bunk90833aa2006-11-13 16:02:22 -08005609 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005610 }
5611 }
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09005612
Patrick McHardyf6469682013-04-19 02:04:27 +00005613 if (((dev->hw_features | dev->features) &
5614 NETIF_F_HW_VLAN_CTAG_FILTER) &&
Michał Mirosławd2ed2732013-01-29 15:14:16 +00005615 (!dev->netdev_ops->ndo_vlan_rx_add_vid ||
5616 !dev->netdev_ops->ndo_vlan_rx_kill_vid)) {
5617 netdev_WARN(dev, "Buggy VLAN acceleration in driver!\n");
5618 ret = -EINVAL;
5619 goto err_uninit;
5620 }
5621
Pavel Emelyanov9c7dafb2012-08-08 21:52:46 +00005622 ret = -EBUSY;
5623 if (!dev->ifindex)
5624 dev->ifindex = dev_new_index(net);
5625 else if (__dev_get_by_index(net, dev->ifindex))
5626 goto err_uninit;
5627
Linus Torvalds1da177e2005-04-16 15:20:36 -07005628 if (dev->iflink == -1)
5629 dev->iflink = dev->ifindex;
5630
Michał Mirosław5455c692011-02-15 16:59:17 +00005631 /* Transfer changeable features to wanted_features and enable
5632 * software offloads (GSO and GRO).
5633 */
5634 dev->hw_features |= NETIF_F_SOFT_FEATURES;
Michał Mirosław14d12322011-02-22 16:52:28 +00005635 dev->features |= NETIF_F_SOFT_FEATURES;
5636 dev->wanted_features = dev->features & dev->hw_features;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005637
Tom Herbertc6e1a0d2011-04-04 22:30:30 -07005638 /* Turn on no cache copy if HW is doing checksum */
Michał Mirosław34324dc2011-11-15 15:29:55 +00005639 if (!(dev->flags & IFF_LOOPBACK)) {
5640 dev->hw_features |= NETIF_F_NOCACHE_COPY;
5641 if (dev->features & NETIF_F_ALL_CSUM) {
5642 dev->wanted_features |= NETIF_F_NOCACHE_COPY;
5643 dev->features |= NETIF_F_NOCACHE_COPY;
5644 }
Tom Herbertc6e1a0d2011-04-04 22:30:30 -07005645 }
5646
Michał Mirosław1180e7d2011-07-14 14:41:11 -07005647 /* Make NETIF_F_HIGHDMA inheritable to VLAN devices.
Brandon Philips16c3ea72010-09-15 09:24:24 +00005648 */
Michał Mirosław1180e7d2011-07-14 14:41:11 -07005649 dev->vlan_features |= NETIF_F_HIGHDMA;
Brandon Philips16c3ea72010-09-15 09:24:24 +00005650
Pravin B Shelaree579672013-03-07 09:28:08 +00005651 /* Make NETIF_F_SG inheritable to tunnel devices.
5652 */
5653 dev->hw_enc_features |= NETIF_F_SG;
5654
Simon Horman0d89d202013-05-23 21:02:52 +00005655 /* Make NETIF_F_SG inheritable to MPLS.
5656 */
5657 dev->mpls_features |= NETIF_F_SG;
5658
Johannes Berg7ffbe3f2009-10-02 05:15:27 +00005659 ret = call_netdevice_notifiers(NETDEV_POST_INIT, dev);
5660 ret = notifier_to_errno(ret);
5661 if (ret)
5662 goto err_uninit;
5663
Eric W. Biederman8b41d182007-09-26 22:02:53 -07005664 ret = netdev_register_kobject(dev);
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07005665 if (ret)
Herbert Xu7ce1b0e2007-07-30 16:29:40 -07005666 goto err_uninit;
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07005667 dev->reg_state = NETREG_REGISTERED;
5668
Michał Mirosław6cb6a272011-04-02 22:48:47 -07005669 __netdev_update_features(dev);
Michał Mirosław8e9b59b2011-02-22 16:52:28 +00005670
Linus Torvalds1da177e2005-04-16 15:20:36 -07005671 /*
5672 * Default initial state at registry is that the
5673 * device is present.
5674 */
5675
5676 set_bit(__LINK_STATE_PRESENT, &dev->state);
5677
Ben Hutchings8f4cccb2012-08-20 22:16:51 +01005678 linkwatch_init_dev(dev);
5679
Linus Torvalds1da177e2005-04-16 15:20:36 -07005680 dev_init_scheduler(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005681 dev_hold(dev);
Eric W. Biedermance286d32007-09-12 13:53:49 +02005682 list_netdevice(dev);
Theodore Ts'o7bf23572012-07-04 21:23:25 -04005683 add_device_randomness(dev->dev_addr, dev->addr_len);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005684
Jiri Pirko948b3372013-01-08 01:38:25 +00005685 /* If the device has permanent device address, driver should
5686 * set dev_addr and also addr_assign_type should be set to
5687 * NET_ADDR_PERM (default value).
5688 */
5689 if (dev->addr_assign_type == NET_ADDR_PERM)
5690 memcpy(dev->perm_addr, dev->dev_addr, dev->addr_len);
5691
Linus Torvalds1da177e2005-04-16 15:20:36 -07005692 /* Notify protocols, that a new device appeared. */
Pavel Emelyanov056925a2007-09-16 15:42:43 -07005693 ret = call_netdevice_notifiers(NETDEV_REGISTER, dev);
Herbert Xufcc5a032007-07-30 17:03:38 -07005694 ret = notifier_to_errno(ret);
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07005695 if (ret) {
5696 rollback_registered(dev);
5697 dev->reg_state = NETREG_UNREGISTERED;
5698 }
Eric W. Biedermand90a9092009-12-12 22:11:15 +00005699 /*
5700 * Prevent userspace races by waiting until the network
5701 * device is fully setup before sending notifications.
5702 */
Patrick McHardya2835762010-02-26 06:34:51 +00005703 if (!dev->rtnl_link_ops ||
5704 dev->rtnl_link_state == RTNL_LINK_INITIALIZED)
5705 rtmsg_ifinfo(RTM_NEWLINK, dev, ~0U);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005706
5707out:
5708 return ret;
Herbert Xu7ce1b0e2007-07-30 16:29:40 -07005709
5710err_uninit:
Stephen Hemmingerd3147742008-11-19 21:32:24 -08005711 if (dev->netdev_ops->ndo_uninit)
5712 dev->netdev_ops->ndo_uninit(dev);
Herbert Xu7ce1b0e2007-07-30 16:29:40 -07005713 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005714}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005715EXPORT_SYMBOL(register_netdevice);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005716
5717/**
Benjamin Herrenschmidt937f1ba2009-01-14 21:05:05 -08005718 * init_dummy_netdev - init a dummy network device for NAPI
5719 * @dev: device to init
5720 *
5721 * This takes a network device structure and initialize the minimum
5722 * amount of fields so it can be used to schedule NAPI polls without
5723 * registering a full blown interface. This is to be used by drivers
5724 * that need to tie several hardware interfaces to a single NAPI
5725 * poll scheduler due to HW limitations.
5726 */
5727int init_dummy_netdev(struct net_device *dev)
5728{
5729 /* Clear everything. Note we don't initialize spinlocks
5730 * are they aren't supposed to be taken by any of the
5731 * NAPI code and this dummy netdev is supposed to be
5732 * only ever used for NAPI polls
5733 */
5734 memset(dev, 0, sizeof(struct net_device));
5735
5736 /* make sure we BUG if trying to hit standard
5737 * register/unregister code path
5738 */
5739 dev->reg_state = NETREG_DUMMY;
5740
Benjamin Herrenschmidt937f1ba2009-01-14 21:05:05 -08005741 /* NAPI wants this */
5742 INIT_LIST_HEAD(&dev->napi_list);
5743
5744 /* a dummy interface is started by default */
5745 set_bit(__LINK_STATE_PRESENT, &dev->state);
5746 set_bit(__LINK_STATE_START, &dev->state);
5747
Eric Dumazet29b44332010-10-11 10:22:12 +00005748 /* Note : We dont allocate pcpu_refcnt for dummy devices,
5749 * because users of this 'device' dont need to change
5750 * its refcount.
5751 */
5752
Benjamin Herrenschmidt937f1ba2009-01-14 21:05:05 -08005753 return 0;
5754}
5755EXPORT_SYMBOL_GPL(init_dummy_netdev);
5756
5757
5758/**
Linus Torvalds1da177e2005-04-16 15:20:36 -07005759 * register_netdev - register a network device
5760 * @dev: device to register
5761 *
5762 * Take a completed network device structure and add it to the kernel
5763 * interfaces. A %NETDEV_REGISTER message is sent to the netdev notifier
5764 * chain. 0 is returned on success. A negative errno code is returned
5765 * on a failure to set up the device, or if the name is a duplicate.
5766 *
Borislav Petkov38b4da32007-04-20 22:14:10 -07005767 * This is a wrapper around register_netdevice that takes the rtnl semaphore
Linus Torvalds1da177e2005-04-16 15:20:36 -07005768 * and expands the device name if you passed a format string to
5769 * alloc_netdev.
5770 */
5771int register_netdev(struct net_device *dev)
5772{
5773 int err;
5774
5775 rtnl_lock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07005776 err = register_netdevice(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005777 rtnl_unlock();
5778 return err;
5779}
5780EXPORT_SYMBOL(register_netdev);
5781
Eric Dumazet29b44332010-10-11 10:22:12 +00005782int netdev_refcnt_read(const struct net_device *dev)
5783{
5784 int i, refcnt = 0;
5785
5786 for_each_possible_cpu(i)
5787 refcnt += *per_cpu_ptr(dev->pcpu_refcnt, i);
5788 return refcnt;
5789}
5790EXPORT_SYMBOL(netdev_refcnt_read);
5791
Ben Hutchings2c530402012-07-10 10:55:09 +00005792/**
Linus Torvalds1da177e2005-04-16 15:20:36 -07005793 * netdev_wait_allrefs - wait until all references are gone.
Randy Dunlap3de7a372012-08-18 14:36:44 +00005794 * @dev: target net_device
Linus Torvalds1da177e2005-04-16 15:20:36 -07005795 *
5796 * This is called when unregistering network devices.
5797 *
5798 * Any protocol or device that holds a reference should register
5799 * for netdevice notification, and cleanup and put back the
5800 * reference if they receive an UNREGISTER event.
5801 * We can get stuck here if buggy protocols don't correctly
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09005802 * call dev_put.
Linus Torvalds1da177e2005-04-16 15:20:36 -07005803 */
5804static void netdev_wait_allrefs(struct net_device *dev)
5805{
5806 unsigned long rebroadcast_time, warning_time;
Eric Dumazet29b44332010-10-11 10:22:12 +00005807 int refcnt;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005808
Eric Dumazete014deb2009-11-17 05:59:21 +00005809 linkwatch_forget_dev(dev);
5810
Linus Torvalds1da177e2005-04-16 15:20:36 -07005811 rebroadcast_time = warning_time = jiffies;
Eric Dumazet29b44332010-10-11 10:22:12 +00005812 refcnt = netdev_refcnt_read(dev);
5813
5814 while (refcnt != 0) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07005815 if (time_after(jiffies, rebroadcast_time + 1 * HZ)) {
Stephen Hemminger6756ae42006-03-20 22:23:58 -08005816 rtnl_lock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07005817
5818 /* Rebroadcast unregister notification */
Pavel Emelyanov056925a2007-09-16 15:42:43 -07005819 call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005820
Eric Dumazet748e2d92012-08-22 21:50:59 +00005821 __rtnl_unlock();
Eric Dumazet0115e8e2012-08-22 17:19:46 +00005822 rcu_barrier();
Eric Dumazet748e2d92012-08-22 21:50:59 +00005823 rtnl_lock();
5824
Eric Dumazet0115e8e2012-08-22 17:19:46 +00005825 call_netdevice_notifiers(NETDEV_UNREGISTER_FINAL, dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005826 if (test_bit(__LINK_STATE_LINKWATCH_PENDING,
5827 &dev->state)) {
5828 /* We must not have linkwatch events
5829 * pending on unregister. If this
5830 * happens, we simply run the queue
5831 * unscheduled, resulting in a noop
5832 * for this device.
5833 */
5834 linkwatch_run_queue();
5835 }
5836
Stephen Hemminger6756ae42006-03-20 22:23:58 -08005837 __rtnl_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07005838
5839 rebroadcast_time = jiffies;
5840 }
5841
5842 msleep(250);
5843
Eric Dumazet29b44332010-10-11 10:22:12 +00005844 refcnt = netdev_refcnt_read(dev);
5845
Linus Torvalds1da177e2005-04-16 15:20:36 -07005846 if (time_after(jiffies, warning_time + 10 * HZ)) {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00005847 pr_emerg("unregister_netdevice: waiting for %s to become free. Usage count = %d\n",
5848 dev->name, refcnt);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005849 warning_time = jiffies;
5850 }
5851 }
5852}
5853
5854/* The sequence is:
5855 *
5856 * rtnl_lock();
5857 * ...
5858 * register_netdevice(x1);
5859 * register_netdevice(x2);
5860 * ...
5861 * unregister_netdevice(y1);
5862 * unregister_netdevice(y2);
5863 * ...
5864 * rtnl_unlock();
5865 * free_netdev(y1);
5866 * free_netdev(y2);
5867 *
Herbert Xu58ec3b42008-10-07 15:50:03 -07005868 * We are invoked by rtnl_unlock().
Linus Torvalds1da177e2005-04-16 15:20:36 -07005869 * This allows us to deal with problems:
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07005870 * 1) We can delete sysfs objects which invoke hotplug
Linus Torvalds1da177e2005-04-16 15:20:36 -07005871 * without deadlocking with linkwatch via keventd.
5872 * 2) Since we run with the RTNL semaphore not held, we can sleep
5873 * safely in order to wait for the netdev refcnt to drop to zero.
Herbert Xu58ec3b42008-10-07 15:50:03 -07005874 *
5875 * We must not return until all unregister events added during
5876 * the interval the lock was held have been completed.
Linus Torvalds1da177e2005-04-16 15:20:36 -07005877 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07005878void netdev_run_todo(void)
5879{
Oleg Nesterov626ab0e2006-06-23 02:05:55 -07005880 struct list_head list;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005881
Linus Torvalds1da177e2005-04-16 15:20:36 -07005882 /* Snapshot list, allow later requests */
Oleg Nesterov626ab0e2006-06-23 02:05:55 -07005883 list_replace_init(&net_todo_list, &list);
Herbert Xu58ec3b42008-10-07 15:50:03 -07005884
5885 __rtnl_unlock();
Oleg Nesterov626ab0e2006-06-23 02:05:55 -07005886
Eric Dumazet0115e8e2012-08-22 17:19:46 +00005887
5888 /* Wait for rcu callbacks to finish before next phase */
Eric W. Biederman850a5452011-10-13 22:25:23 +00005889 if (!list_empty(&list))
5890 rcu_barrier();
5891
Linus Torvalds1da177e2005-04-16 15:20:36 -07005892 while (!list_empty(&list)) {
5893 struct net_device *dev
stephen hemmingere5e26d72010-02-24 14:01:38 +00005894 = list_first_entry(&list, struct net_device, todo_list);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005895 list_del(&dev->todo_list);
5896
Eric Dumazet748e2d92012-08-22 21:50:59 +00005897 rtnl_lock();
Eric Dumazet0115e8e2012-08-22 17:19:46 +00005898 call_netdevice_notifiers(NETDEV_UNREGISTER_FINAL, dev);
Eric Dumazet748e2d92012-08-22 21:50:59 +00005899 __rtnl_unlock();
Eric Dumazet0115e8e2012-08-22 17:19:46 +00005900
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07005901 if (unlikely(dev->reg_state != NETREG_UNREGISTERING)) {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00005902 pr_err("network todo '%s' but state %d\n",
Linus Torvalds1da177e2005-04-16 15:20:36 -07005903 dev->name, dev->reg_state);
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07005904 dump_stack();
5905 continue;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005906 }
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07005907
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07005908 dev->reg_state = NETREG_UNREGISTERED;
5909
Changli Gao152102c2010-03-30 20:16:22 +00005910 on_each_cpu(flush_backlog, dev, 1);
Stephen Hemminger6e583ce2008-08-03 21:29:57 -07005911
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07005912 netdev_wait_allrefs(dev);
5913
5914 /* paranoia */
Eric Dumazet29b44332010-10-11 10:22:12 +00005915 BUG_ON(netdev_refcnt_read(dev));
Eric Dumazet33d480c2011-08-11 19:30:52 +00005916 WARN_ON(rcu_access_pointer(dev->ip_ptr));
5917 WARN_ON(rcu_access_pointer(dev->ip6_ptr));
Ilpo Järvinen547b7922008-07-25 21:43:18 -07005918 WARN_ON(dev->dn_ptr);
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07005919
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07005920 if (dev->destructor)
5921 dev->destructor(dev);
Stephen Hemminger9093bbb2007-05-19 15:39:25 -07005922
Eric W. Biederman50624c92013-09-23 21:19:49 -07005923 /* Report a network device has been unregistered */
5924 rtnl_lock();
5925 dev_net(dev)->dev_unreg_count--;
5926 __rtnl_unlock();
5927 wake_up(&netdev_unregistering_wq);
5928
Stephen Hemminger9093bbb2007-05-19 15:39:25 -07005929 /* Free network device */
5930 kobject_put(&dev->dev.kobj);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005931 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07005932}
5933
Ben Hutchings3cfde792010-07-09 09:11:52 +00005934/* Convert net_device_stats to rtnl_link_stats64. They have the same
5935 * fields in the same order, with only the type differing.
5936 */
Eric Dumazet77a1abf2012-03-05 04:50:09 +00005937void netdev_stats_to_stats64(struct rtnl_link_stats64 *stats64,
5938 const struct net_device_stats *netdev_stats)
Ben Hutchings3cfde792010-07-09 09:11:52 +00005939{
5940#if BITS_PER_LONG == 64
Eric Dumazet77a1abf2012-03-05 04:50:09 +00005941 BUILD_BUG_ON(sizeof(*stats64) != sizeof(*netdev_stats));
5942 memcpy(stats64, netdev_stats, sizeof(*stats64));
Ben Hutchings3cfde792010-07-09 09:11:52 +00005943#else
5944 size_t i, n = sizeof(*stats64) / sizeof(u64);
5945 const unsigned long *src = (const unsigned long *)netdev_stats;
5946 u64 *dst = (u64 *)stats64;
5947
5948 BUILD_BUG_ON(sizeof(*netdev_stats) / sizeof(unsigned long) !=
5949 sizeof(*stats64) / sizeof(u64));
5950 for (i = 0; i < n; i++)
5951 dst[i] = src[i];
5952#endif
5953}
Eric Dumazet77a1abf2012-03-05 04:50:09 +00005954EXPORT_SYMBOL(netdev_stats_to_stats64);
Ben Hutchings3cfde792010-07-09 09:11:52 +00005955
Eric Dumazetd83345a2009-11-16 03:36:51 +00005956/**
Stephen Hemmingereeda3fd2008-11-19 21:40:23 -08005957 * dev_get_stats - get network device statistics
5958 * @dev: device to get statistics from
Eric Dumazet28172732010-07-07 14:58:56 -07005959 * @storage: place to store stats
Stephen Hemmingereeda3fd2008-11-19 21:40:23 -08005960 *
Ben Hutchingsd7753512010-07-09 09:12:41 +00005961 * Get network statistics from device. Return @storage.
5962 * The device driver may provide its own method by setting
5963 * dev->netdev_ops->get_stats64 or dev->netdev_ops->get_stats;
5964 * otherwise the internal statistics structure is used.
Stephen Hemmingereeda3fd2008-11-19 21:40:23 -08005965 */
Ben Hutchingsd7753512010-07-09 09:12:41 +00005966struct rtnl_link_stats64 *dev_get_stats(struct net_device *dev,
5967 struct rtnl_link_stats64 *storage)
Eric Dumazet7004bf22009-05-18 00:34:33 +00005968{
Stephen Hemmingereeda3fd2008-11-19 21:40:23 -08005969 const struct net_device_ops *ops = dev->netdev_ops;
5970
Eric Dumazet28172732010-07-07 14:58:56 -07005971 if (ops->ndo_get_stats64) {
5972 memset(storage, 0, sizeof(*storage));
Eric Dumazetcaf586e2010-09-30 21:06:55 +00005973 ops->ndo_get_stats64(dev, storage);
5974 } else if (ops->ndo_get_stats) {
Ben Hutchings3cfde792010-07-09 09:11:52 +00005975 netdev_stats_to_stats64(storage, ops->ndo_get_stats(dev));
Eric Dumazetcaf586e2010-09-30 21:06:55 +00005976 } else {
5977 netdev_stats_to_stats64(storage, &dev->stats);
Eric Dumazet28172732010-07-07 14:58:56 -07005978 }
Eric Dumazetcaf586e2010-09-30 21:06:55 +00005979 storage->rx_dropped += atomic_long_read(&dev->rx_dropped);
Eric Dumazet28172732010-07-07 14:58:56 -07005980 return storage;
Rusty Russellc45d2862007-03-28 14:29:08 -07005981}
Stephen Hemmingereeda3fd2008-11-19 21:40:23 -08005982EXPORT_SYMBOL(dev_get_stats);
Rusty Russellc45d2862007-03-28 14:29:08 -07005983
Eric Dumazet24824a02010-10-02 06:11:55 +00005984struct netdev_queue *dev_ingress_queue_create(struct net_device *dev)
David S. Millerdc2b4842008-07-08 17:18:23 -07005985{
Eric Dumazet24824a02010-10-02 06:11:55 +00005986 struct netdev_queue *queue = dev_ingress_queue(dev);
David S. Millerdc2b4842008-07-08 17:18:23 -07005987
Eric Dumazet24824a02010-10-02 06:11:55 +00005988#ifdef CONFIG_NET_CLS_ACT
5989 if (queue)
5990 return queue;
5991 queue = kzalloc(sizeof(*queue), GFP_KERNEL);
5992 if (!queue)
5993 return NULL;
5994 netdev_init_one_queue(dev, queue, NULL);
Eric Dumazet24824a02010-10-02 06:11:55 +00005995 queue->qdisc = &noop_qdisc;
5996 queue->qdisc_sleeping = &noop_qdisc;
5997 rcu_assign_pointer(dev->ingress_queue, queue);
5998#endif
5999 return queue;
David S. Millerbb949fb2008-07-08 16:55:56 -07006000}
6001
Eric Dumazet2c60db02012-09-16 09:17:26 +00006002static const struct ethtool_ops default_ethtool_ops;
6003
Stanislaw Gruszkad07d7502013-01-10 23:19:10 +00006004void netdev_set_default_ethtool_ops(struct net_device *dev,
6005 const struct ethtool_ops *ops)
6006{
6007 if (dev->ethtool_ops == &default_ethtool_ops)
6008 dev->ethtool_ops = ops;
6009}
6010EXPORT_SYMBOL_GPL(netdev_set_default_ethtool_ops);
6011
Linus Torvalds1da177e2005-04-16 15:20:36 -07006012/**
Tom Herbert36909ea2011-01-09 19:36:31 +00006013 * alloc_netdev_mqs - allocate network device
Linus Torvalds1da177e2005-04-16 15:20:36 -07006014 * @sizeof_priv: size of private data to allocate space for
6015 * @name: device name format string
6016 * @setup: callback to initialize device
Tom Herbert36909ea2011-01-09 19:36:31 +00006017 * @txqs: the number of TX subqueues to allocate
6018 * @rxqs: the number of RX subqueues to allocate
Linus Torvalds1da177e2005-04-16 15:20:36 -07006019 *
6020 * Allocates a struct net_device with private data area for driver use
Peter P Waskiewicz Jrf25f4e42007-07-06 13:36:20 -07006021 * and performs basic initialization. Also allocates subquue structs
Tom Herbert36909ea2011-01-09 19:36:31 +00006022 * for each queue on the device.
Linus Torvalds1da177e2005-04-16 15:20:36 -07006023 */
Tom Herbert36909ea2011-01-09 19:36:31 +00006024struct net_device *alloc_netdev_mqs(int sizeof_priv, const char *name,
6025 void (*setup)(struct net_device *),
6026 unsigned int txqs, unsigned int rxqs)
Linus Torvalds1da177e2005-04-16 15:20:36 -07006027{
Linus Torvalds1da177e2005-04-16 15:20:36 -07006028 struct net_device *dev;
Stephen Hemminger79439862008-07-21 13:28:44 -07006029 size_t alloc_size;
Eric Dumazet1ce8e7b2009-05-27 04:42:37 +00006030 struct net_device *p;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006031
Stephen Hemmingerb6fe17d2006-08-29 17:06:13 -07006032 BUG_ON(strlen(name) >= sizeof(dev->name));
6033
Tom Herbert36909ea2011-01-09 19:36:31 +00006034 if (txqs < 1) {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00006035 pr_err("alloc_netdev: Unable to allocate device with zero queues\n");
Tom Herbert55513fb2010-10-18 17:55:58 +00006036 return NULL;
6037 }
6038
Tom Herbert36909ea2011-01-09 19:36:31 +00006039#ifdef CONFIG_RPS
6040 if (rxqs < 1) {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00006041 pr_err("alloc_netdev: Unable to allocate device with zero RX queues\n");
Tom Herbert36909ea2011-01-09 19:36:31 +00006042 return NULL;
6043 }
6044#endif
6045
David S. Millerfd2ea0a2008-07-17 01:56:23 -07006046 alloc_size = sizeof(struct net_device);
Alexey Dobriyand1643d22008-04-18 15:43:32 -07006047 if (sizeof_priv) {
6048 /* ensure 32-byte alignment of private area */
Eric Dumazet1ce8e7b2009-05-27 04:42:37 +00006049 alloc_size = ALIGN(alloc_size, NETDEV_ALIGN);
Alexey Dobriyand1643d22008-04-18 15:43:32 -07006050 alloc_size += sizeof_priv;
6051 }
6052 /* ensure 32-byte alignment of whole construct */
Eric Dumazet1ce8e7b2009-05-27 04:42:37 +00006053 alloc_size += NETDEV_ALIGN - 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006054
Paolo 'Blaisorblade' Giarrusso31380de2006-04-06 22:38:28 -07006055 p = kzalloc(alloc_size, GFP_KERNEL);
Joe Perches62b59422013-02-04 16:48:16 +00006056 if (!p)
Linus Torvalds1da177e2005-04-16 15:20:36 -07006057 return NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006058
Eric Dumazet1ce8e7b2009-05-27 04:42:37 +00006059 dev = PTR_ALIGN(p, NETDEV_ALIGN);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006060 dev->padded = (char *)dev - (char *)p;
Jiri Pirkoab9c73c2009-05-08 13:30:17 +00006061
Eric Dumazet29b44332010-10-11 10:22:12 +00006062 dev->pcpu_refcnt = alloc_percpu(int);
6063 if (!dev->pcpu_refcnt)
Tom Herberte6484932010-10-18 18:04:39 +00006064 goto free_p;
Jiri Pirkoab9c73c2009-05-08 13:30:17 +00006065
Linus Torvalds1da177e2005-04-16 15:20:36 -07006066 if (dev_addr_init(dev))
Eric Dumazet29b44332010-10-11 10:22:12 +00006067 goto free_pcpu;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006068
Jiri Pirko22bedad32010-04-01 21:22:57 +00006069 dev_mc_init(dev);
Jiri Pirkoa748ee22010-04-01 21:22:09 +00006070 dev_uc_init(dev);
Jiri Pirkoccffad252009-05-22 23:22:17 +00006071
YOSHIFUJI Hideakic346dca2008-03-25 21:47:49 +09006072 dev_net_set(dev, &init_net);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006073
Peter P Waskiewicz Jr82cc1a72008-03-21 03:43:19 -07006074 dev->gso_max_size = GSO_MAX_SIZE;
Ben Hutchings30b678d2012-07-30 15:57:00 +00006075 dev->gso_max_segs = GSO_MAX_SEGS;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006076
Herbert Xud565b0a2008-12-15 23:38:52 -08006077 INIT_LIST_HEAD(&dev->napi_list);
Eric W. Biederman9fdce092009-10-30 14:51:13 +00006078 INIT_LIST_HEAD(&dev->unreg_list);
Eric Dumazete014deb2009-11-17 05:59:21 +00006079 INIT_LIST_HEAD(&dev->link_watch_list);
Jiri Pirko9ff162a2013-01-03 22:48:49 +00006080 INIT_LIST_HEAD(&dev->upper_dev_list);
Veaceslav Falico5d261912013-08-28 23:25:05 +02006081 INIT_LIST_HEAD(&dev->lower_dev_list);
Eric Dumazet93f154b2009-05-18 22:19:19 -07006082 dev->priv_flags = IFF_XMIT_DST_RELEASE;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006083 setup(dev);
David S. Miller8d3bdbd2011-02-08 15:02:50 -08006084
6085 dev->num_tx_queues = txqs;
6086 dev->real_num_tx_queues = txqs;
6087 if (netif_alloc_netdev_queues(dev))
6088 goto free_all;
6089
6090#ifdef CONFIG_RPS
6091 dev->num_rx_queues = rxqs;
6092 dev->real_num_rx_queues = rxqs;
6093 if (netif_alloc_rx_queues(dev))
6094 goto free_all;
6095#endif
6096
Linus Torvalds1da177e2005-04-16 15:20:36 -07006097 strcpy(dev->name, name);
Vlad Dogarucbda10f2011-01-13 23:38:30 +00006098 dev->group = INIT_NETDEV_GROUP;
Eric Dumazet2c60db02012-09-16 09:17:26 +00006099 if (!dev->ethtool_ops)
6100 dev->ethtool_ops = &default_ethtool_ops;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006101 return dev;
Jiri Pirkoab9c73c2009-05-08 13:30:17 +00006102
David S. Miller8d3bdbd2011-02-08 15:02:50 -08006103free_all:
6104 free_netdev(dev);
6105 return NULL;
6106
Eric Dumazet29b44332010-10-11 10:22:12 +00006107free_pcpu:
6108 free_percpu(dev->pcpu_refcnt);
Eric Dumazet60877a32013-06-20 01:15:51 -07006109 netif_free_tx_queues(dev);
Tom Herbertfe822242010-11-09 10:47:38 +00006110#ifdef CONFIG_RPS
6111 kfree(dev->_rx);
6112#endif
6113
Jiri Pirkoab9c73c2009-05-08 13:30:17 +00006114free_p:
6115 kfree(p);
6116 return NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006117}
Tom Herbert36909ea2011-01-09 19:36:31 +00006118EXPORT_SYMBOL(alloc_netdev_mqs);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006119
6120/**
6121 * free_netdev - free network device
6122 * @dev: device
6123 *
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09006124 * This function does the last stage of destroying an allocated device
6125 * interface. The reference to the device object is released.
Linus Torvalds1da177e2005-04-16 15:20:36 -07006126 * If this is the last reference then it will be freed.
6127 */
6128void free_netdev(struct net_device *dev)
6129{
Herbert Xud565b0a2008-12-15 23:38:52 -08006130 struct napi_struct *p, *n;
6131
Denis V. Lunevf3005d72008-04-16 02:02:18 -07006132 release_net(dev_net(dev));
6133
Eric Dumazet60877a32013-06-20 01:15:51 -07006134 netif_free_tx_queues(dev);
Tom Herbertfe822242010-11-09 10:47:38 +00006135#ifdef CONFIG_RPS
6136 kfree(dev->_rx);
6137#endif
David S. Millere8a04642008-07-17 00:34:19 -07006138
Eric Dumazet33d480c2011-08-11 19:30:52 +00006139 kfree(rcu_dereference_protected(dev->ingress_queue, 1));
Eric Dumazet24824a02010-10-02 06:11:55 +00006140
Jiri Pirkof001fde2009-05-05 02:48:28 +00006141 /* Flush device addresses */
6142 dev_addr_flush(dev);
6143
Herbert Xud565b0a2008-12-15 23:38:52 -08006144 list_for_each_entry_safe(p, n, &dev->napi_list, dev_list)
6145 netif_napi_del(p);
6146
Eric Dumazet29b44332010-10-11 10:22:12 +00006147 free_percpu(dev->pcpu_refcnt);
6148 dev->pcpu_refcnt = NULL;
6149
Stephen Hemminger3041a062006-05-26 13:25:24 -07006150 /* Compatibility with error handling in drivers */
Linus Torvalds1da177e2005-04-16 15:20:36 -07006151 if (dev->reg_state == NETREG_UNINITIALIZED) {
6152 kfree((char *)dev - dev->padded);
6153 return;
6154 }
6155
6156 BUG_ON(dev->reg_state != NETREG_UNREGISTERED);
6157 dev->reg_state = NETREG_RELEASED;
6158
Greg Kroah-Hartman43cb76d2002-04-09 12:14:34 -07006159 /* will free via device release */
6160 put_device(&dev->dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006161}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07006162EXPORT_SYMBOL(free_netdev);
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09006163
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07006164/**
6165 * synchronize_net - Synchronize with packet receive processing
6166 *
6167 * Wait for packets currently being received to be done.
6168 * Does not block later packets from starting.
6169 */
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09006170void synchronize_net(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -07006171{
6172 might_sleep();
Eric Dumazetbe3fc412011-05-23 23:07:32 +00006173 if (rtnl_is_locked())
6174 synchronize_rcu_expedited();
6175 else
6176 synchronize_rcu();
Linus Torvalds1da177e2005-04-16 15:20:36 -07006177}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07006178EXPORT_SYMBOL(synchronize_net);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006179
6180/**
Eric Dumazet44a08732009-10-27 07:03:04 +00006181 * unregister_netdevice_queue - remove device from the kernel
Linus Torvalds1da177e2005-04-16 15:20:36 -07006182 * @dev: device
Eric Dumazet44a08732009-10-27 07:03:04 +00006183 * @head: list
Jaswinder Singh Rajput6ebfbc02009-11-22 20:43:13 -08006184 *
Linus Torvalds1da177e2005-04-16 15:20:36 -07006185 * This function shuts down a device interface and removes it
Wang Chend59b54b2007-12-11 02:28:03 -08006186 * from the kernel tables.
Eric Dumazet44a08732009-10-27 07:03:04 +00006187 * If head not NULL, device is queued to be unregistered later.
Linus Torvalds1da177e2005-04-16 15:20:36 -07006188 *
6189 * Callers must hold the rtnl semaphore. You may want
6190 * unregister_netdev() instead of this.
6191 */
6192
Eric Dumazet44a08732009-10-27 07:03:04 +00006193void unregister_netdevice_queue(struct net_device *dev, struct list_head *head)
Linus Torvalds1da177e2005-04-16 15:20:36 -07006194{
Herbert Xua6620712007-12-12 19:21:56 -08006195 ASSERT_RTNL();
6196
Eric Dumazet44a08732009-10-27 07:03:04 +00006197 if (head) {
Eric W. Biederman9fdce092009-10-30 14:51:13 +00006198 list_move_tail(&dev->unreg_list, head);
Eric Dumazet44a08732009-10-27 07:03:04 +00006199 } else {
6200 rollback_registered(dev);
6201 /* Finish processing unregister after unlock */
6202 net_set_todo(dev);
6203 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07006204}
Eric Dumazet44a08732009-10-27 07:03:04 +00006205EXPORT_SYMBOL(unregister_netdevice_queue);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006206
6207/**
Eric Dumazet9b5e3832009-10-27 07:04:19 +00006208 * unregister_netdevice_many - unregister many devices
6209 * @head: list of devices
Eric Dumazet9b5e3832009-10-27 07:04:19 +00006210 */
6211void unregister_netdevice_many(struct list_head *head)
6212{
6213 struct net_device *dev;
6214
6215 if (!list_empty(head)) {
6216 rollback_registered_many(head);
6217 list_for_each_entry(dev, head, unreg_list)
6218 net_set_todo(dev);
6219 }
6220}
Eric Dumazet63c80992009-10-27 07:06:49 +00006221EXPORT_SYMBOL(unregister_netdevice_many);
Eric Dumazet9b5e3832009-10-27 07:04:19 +00006222
6223/**
Linus Torvalds1da177e2005-04-16 15:20:36 -07006224 * unregister_netdev - remove device from the kernel
6225 * @dev: device
6226 *
6227 * This function shuts down a device interface and removes it
Wang Chend59b54b2007-12-11 02:28:03 -08006228 * from the kernel tables.
Linus Torvalds1da177e2005-04-16 15:20:36 -07006229 *
6230 * This is just a wrapper for unregister_netdevice that takes
6231 * the rtnl semaphore. In general you want to use this and not
6232 * unregister_netdevice.
6233 */
6234void unregister_netdev(struct net_device *dev)
6235{
6236 rtnl_lock();
6237 unregister_netdevice(dev);
6238 rtnl_unlock();
6239}
Linus Torvalds1da177e2005-04-16 15:20:36 -07006240EXPORT_SYMBOL(unregister_netdev);
6241
Eric W. Biedermance286d32007-09-12 13:53:49 +02006242/**
6243 * dev_change_net_namespace - move device to different nethost namespace
6244 * @dev: device
6245 * @net: network namespace
6246 * @pat: If not NULL name pattern to try if the current device name
6247 * is already taken in the destination network namespace.
6248 *
6249 * This function shuts down a device interface and moves it
6250 * to a new network namespace. On success 0 is returned, on
6251 * a failure a netagive errno code is returned.
6252 *
6253 * Callers must hold the rtnl semaphore.
6254 */
6255
6256int dev_change_net_namespace(struct net_device *dev, struct net *net, const char *pat)
6257{
Eric W. Biedermance286d32007-09-12 13:53:49 +02006258 int err;
6259
6260 ASSERT_RTNL();
6261
6262 /* Don't allow namespace local devices to be moved. */
6263 err = -EINVAL;
6264 if (dev->features & NETIF_F_NETNS_LOCAL)
6265 goto out;
6266
6267 /* Ensure the device has been registrered */
Eric W. Biedermance286d32007-09-12 13:53:49 +02006268 if (dev->reg_state != NETREG_REGISTERED)
6269 goto out;
6270
6271 /* Get out if there is nothing todo */
6272 err = 0;
YOSHIFUJI Hideaki878628f2008-03-26 03:57:35 +09006273 if (net_eq(dev_net(dev), net))
Eric W. Biedermance286d32007-09-12 13:53:49 +02006274 goto out;
6275
6276 /* Pick the destination device name, and ensure
6277 * we can use it in the destination network namespace.
6278 */
6279 err = -EEXIST;
Octavian Purdilad9031022009-11-18 02:36:59 +00006280 if (__dev_get_by_name(net, dev->name)) {
Eric W. Biedermance286d32007-09-12 13:53:49 +02006281 /* We get here if we can't use the current device name */
6282 if (!pat)
6283 goto out;
Gao feng828de4f2012-09-13 20:58:27 +00006284 if (dev_get_valid_name(net, dev, pat) < 0)
Eric W. Biedermance286d32007-09-12 13:53:49 +02006285 goto out;
6286 }
6287
6288 /*
6289 * And now a mini version of register_netdevice unregister_netdevice.
6290 */
6291
6292 /* If device is running close it first. */
Pavel Emelyanov9b772652007-10-10 02:49:09 -07006293 dev_close(dev);
Eric W. Biedermance286d32007-09-12 13:53:49 +02006294
6295 /* And unlink it from device chain */
6296 err = -ENODEV;
6297 unlist_netdevice(dev);
6298
6299 synchronize_net();
6300
6301 /* Shutdown queueing discipline. */
6302 dev_shutdown(dev);
6303
6304 /* Notify protocols, that we are about to destroy
6305 this device. They should clean all the things.
David Lamparter3b27e102010-09-17 03:22:19 +00006306
6307 Note that dev->reg_state stays at NETREG_REGISTERED.
6308 This is wanted because this way 8021q and macvlan know
6309 the device is just moving and can keep their slaves up.
Eric W. Biedermance286d32007-09-12 13:53:49 +02006310 */
6311 call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
Gao feng6549dd42012-08-23 15:36:55 +00006312 rcu_barrier();
6313 call_netdevice_notifiers(NETDEV_UNREGISTER_FINAL, dev);
Eric W. Biedermand2237d32011-10-21 06:24:20 +00006314 rtmsg_ifinfo(RTM_DELLINK, dev, ~0U);
Eric W. Biedermance286d32007-09-12 13:53:49 +02006315
6316 /*
6317 * Flush the unicast and multicast chains
6318 */
Jiri Pirkoa748ee22010-04-01 21:22:09 +00006319 dev_uc_flush(dev);
Jiri Pirko22bedad32010-04-01 21:22:57 +00006320 dev_mc_flush(dev);
Eric W. Biedermance286d32007-09-12 13:53:49 +02006321
Serge Hallyn4e66ae22012-12-03 16:17:12 +00006322 /* Send a netdev-removed uevent to the old namespace */
6323 kobject_uevent(&dev->dev.kobj, KOBJ_REMOVE);
6324
Eric W. Biedermance286d32007-09-12 13:53:49 +02006325 /* Actually switch the network namespace */
YOSHIFUJI Hideakic346dca2008-03-25 21:47:49 +09006326 dev_net_set(dev, net);
Eric W. Biedermance286d32007-09-12 13:53:49 +02006327
Eric W. Biedermance286d32007-09-12 13:53:49 +02006328 /* If there is an ifindex conflict assign a new one */
6329 if (__dev_get_by_index(net, dev->ifindex)) {
6330 int iflink = (dev->iflink == dev->ifindex);
6331 dev->ifindex = dev_new_index(net);
6332 if (iflink)
6333 dev->iflink = dev->ifindex;
6334 }
6335
Serge Hallyn4e66ae22012-12-03 16:17:12 +00006336 /* Send a netdev-add uevent to the new namespace */
6337 kobject_uevent(&dev->dev.kobj, KOBJ_ADD);
6338
Eric W. Biederman8b41d182007-09-26 22:02:53 -07006339 /* Fixup kobjects */
Eric W. Biedermana1b3f592010-05-04 17:36:49 -07006340 err = device_rename(&dev->dev, dev->name);
Eric W. Biederman8b41d182007-09-26 22:02:53 -07006341 WARN_ON(err);
Eric W. Biedermance286d32007-09-12 13:53:49 +02006342
6343 /* Add the device back in the hashes */
6344 list_netdevice(dev);
6345
6346 /* Notify protocols, that a new device appeared. */
6347 call_netdevice_notifiers(NETDEV_REGISTER, dev);
6348
Eric W. Biedermand90a9092009-12-12 22:11:15 +00006349 /*
6350 * Prevent userspace races by waiting until the network
6351 * device is fully setup before sending notifications.
6352 */
6353 rtmsg_ifinfo(RTM_NEWLINK, dev, ~0U);
6354
Eric W. Biedermance286d32007-09-12 13:53:49 +02006355 synchronize_net();
6356 err = 0;
6357out:
6358 return err;
6359}
Johannes Berg463d0182009-07-14 00:33:35 +02006360EXPORT_SYMBOL_GPL(dev_change_net_namespace);
Eric W. Biedermance286d32007-09-12 13:53:49 +02006361
Linus Torvalds1da177e2005-04-16 15:20:36 -07006362static int dev_cpu_callback(struct notifier_block *nfb,
6363 unsigned long action,
6364 void *ocpu)
6365{
6366 struct sk_buff **list_skb;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006367 struct sk_buff *skb;
6368 unsigned int cpu, oldcpu = (unsigned long)ocpu;
6369 struct softnet_data *sd, *oldsd;
6370
Rafael J. Wysocki8bb78442007-05-09 02:35:10 -07006371 if (action != CPU_DEAD && action != CPU_DEAD_FROZEN)
Linus Torvalds1da177e2005-04-16 15:20:36 -07006372 return NOTIFY_OK;
6373
6374 local_irq_disable();
6375 cpu = smp_processor_id();
6376 sd = &per_cpu(softnet_data, cpu);
6377 oldsd = &per_cpu(softnet_data, oldcpu);
6378
6379 /* Find end of our completion_queue. */
6380 list_skb = &sd->completion_queue;
6381 while (*list_skb)
6382 list_skb = &(*list_skb)->next;
6383 /* Append completion queue from offline CPU. */
6384 *list_skb = oldsd->completion_queue;
6385 oldsd->completion_queue = NULL;
6386
Linus Torvalds1da177e2005-04-16 15:20:36 -07006387 /* Append output queue from offline CPU. */
Changli Gaoa9cbd582010-04-26 23:06:24 +00006388 if (oldsd->output_queue) {
6389 *sd->output_queue_tailp = oldsd->output_queue;
6390 sd->output_queue_tailp = oldsd->output_queue_tailp;
6391 oldsd->output_queue = NULL;
6392 oldsd->output_queue_tailp = &oldsd->output_queue;
6393 }
Heiko Carstens264524d2011-06-06 20:50:03 +00006394 /* Append NAPI poll list from offline CPU. */
6395 if (!list_empty(&oldsd->poll_list)) {
6396 list_splice_init(&oldsd->poll_list, &sd->poll_list);
6397 raise_softirq_irqoff(NET_RX_SOFTIRQ);
6398 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07006399
6400 raise_softirq_irqoff(NET_TX_SOFTIRQ);
6401 local_irq_enable();
6402
6403 /* Process offline CPU's input_pkt_queue */
Tom Herbert76cc8b12010-05-20 18:37:59 +00006404 while ((skb = __skb_dequeue(&oldsd->process_queue))) {
6405 netif_rx(skb);
6406 input_queue_head_incr(oldsd);
6407 }
Tom Herbertfec5e652010-04-16 16:01:27 -07006408 while ((skb = __skb_dequeue(&oldsd->input_pkt_queue))) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07006409 netif_rx(skb);
Tom Herbert76cc8b12010-05-20 18:37:59 +00006410 input_queue_head_incr(oldsd);
Tom Herbertfec5e652010-04-16 16:01:27 -07006411 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07006412
6413 return NOTIFY_OK;
6414}
Linus Torvalds1da177e2005-04-16 15:20:36 -07006415
6416
Herbert Xu7f353bf2007-08-10 15:47:58 -07006417/**
Herbert Xub63365a2008-10-23 01:11:29 -07006418 * netdev_increment_features - increment feature set by one
6419 * @all: current feature set
6420 * @one: new feature set
6421 * @mask: mask feature set
Herbert Xu7f353bf2007-08-10 15:47:58 -07006422 *
6423 * Computes a new feature set after adding a device with feature set
Herbert Xub63365a2008-10-23 01:11:29 -07006424 * @one to the master device with current feature set @all. Will not
6425 * enable anything that is off in @mask. Returns the new feature set.
Herbert Xu7f353bf2007-08-10 15:47:58 -07006426 */
Michał Mirosławc8f44af2011-11-15 15:29:55 +00006427netdev_features_t netdev_increment_features(netdev_features_t all,
6428 netdev_features_t one, netdev_features_t mask)
Herbert Xu7f353bf2007-08-10 15:47:58 -07006429{
Michał Mirosław1742f182011-04-22 06:31:16 +00006430 if (mask & NETIF_F_GEN_CSUM)
6431 mask |= NETIF_F_ALL_CSUM;
6432 mask |= NETIF_F_VLAN_CHALLENGED;
6433
6434 all |= one & (NETIF_F_ONE_FOR_ALL|NETIF_F_ALL_CSUM) & mask;
6435 all &= one | ~NETIF_F_ALL_FOR_ALL;
6436
Michał Mirosław1742f182011-04-22 06:31:16 +00006437 /* If one device supports hw checksumming, set for all. */
6438 if (all & NETIF_F_GEN_CSUM)
6439 all &= ~(NETIF_F_ALL_CSUM & ~NETIF_F_GEN_CSUM);
Herbert Xu7f353bf2007-08-10 15:47:58 -07006440
6441 return all;
6442}
Herbert Xub63365a2008-10-23 01:11:29 -07006443EXPORT_SYMBOL(netdev_increment_features);
Herbert Xu7f353bf2007-08-10 15:47:58 -07006444
Baruch Siach430f03c2013-06-02 20:43:55 +00006445static struct hlist_head * __net_init netdev_create_hash(void)
Pavel Emelyanov30d97d32007-09-16 15:40:33 -07006446{
6447 int i;
6448 struct hlist_head *hash;
6449
6450 hash = kmalloc(sizeof(*hash) * NETDEV_HASHENTRIES, GFP_KERNEL);
6451 if (hash != NULL)
6452 for (i = 0; i < NETDEV_HASHENTRIES; i++)
6453 INIT_HLIST_HEAD(&hash[i]);
6454
6455 return hash;
6456}
6457
Eric W. Biederman881d9662007-09-17 11:56:21 -07006458/* Initialize per network namespace state */
Pavel Emelyanov46650792007-10-08 20:38:39 -07006459static int __net_init netdev_init(struct net *net)
Eric W. Biederman881d9662007-09-17 11:56:21 -07006460{
Rustad, Mark D734b6542012-07-18 09:06:07 +00006461 if (net != &init_net)
6462 INIT_LIST_HEAD(&net->dev_base_head);
Eric W. Biederman881d9662007-09-17 11:56:21 -07006463
Pavel Emelyanov30d97d32007-09-16 15:40:33 -07006464 net->dev_name_head = netdev_create_hash();
6465 if (net->dev_name_head == NULL)
6466 goto err_name;
Eric W. Biederman881d9662007-09-17 11:56:21 -07006467
Pavel Emelyanov30d97d32007-09-16 15:40:33 -07006468 net->dev_index_head = netdev_create_hash();
6469 if (net->dev_index_head == NULL)
6470 goto err_idx;
Eric W. Biederman881d9662007-09-17 11:56:21 -07006471
6472 return 0;
Pavel Emelyanov30d97d32007-09-16 15:40:33 -07006473
6474err_idx:
6475 kfree(net->dev_name_head);
6476err_name:
6477 return -ENOMEM;
Eric W. Biederman881d9662007-09-17 11:56:21 -07006478}
6479
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07006480/**
6481 * netdev_drivername - network driver for the device
6482 * @dev: network device
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07006483 *
6484 * Determine network driver for device.
6485 */
David S. Miller3019de12011-06-06 16:41:33 -07006486const char *netdev_drivername(const struct net_device *dev)
Arjan van de Ven6579e572008-07-21 13:31:48 -07006487{
Stephen Hemmingercf04a4c72008-09-30 02:22:14 -07006488 const struct device_driver *driver;
6489 const struct device *parent;
David S. Miller3019de12011-06-06 16:41:33 -07006490 const char *empty = "";
Arjan van de Ven6579e572008-07-21 13:31:48 -07006491
6492 parent = dev->dev.parent;
Arjan van de Ven6579e572008-07-21 13:31:48 -07006493 if (!parent)
David S. Miller3019de12011-06-06 16:41:33 -07006494 return empty;
Arjan van de Ven6579e572008-07-21 13:31:48 -07006495
6496 driver = parent->driver;
6497 if (driver && driver->name)
David S. Miller3019de12011-06-06 16:41:33 -07006498 return driver->name;
6499 return empty;
Arjan van de Ven6579e572008-07-21 13:31:48 -07006500}
6501
Joe Perchesb004ff42012-09-12 20:12:19 -07006502static int __netdev_printk(const char *level, const struct net_device *dev,
Joe Perches256df2f2010-06-27 01:02:35 +00006503 struct va_format *vaf)
6504{
6505 int r;
6506
Joe Perchesb004ff42012-09-12 20:12:19 -07006507 if (dev && dev->dev.parent) {
Joe Perches666f3552012-09-12 20:14:11 -07006508 r = dev_printk_emit(level[1] - '0',
6509 dev->dev.parent,
6510 "%s %s %s: %pV",
6511 dev_driver_string(dev->dev.parent),
6512 dev_name(dev->dev.parent),
6513 netdev_name(dev), vaf);
Joe Perchesb004ff42012-09-12 20:12:19 -07006514 } else if (dev) {
Joe Perches256df2f2010-06-27 01:02:35 +00006515 r = printk("%s%s: %pV", level, netdev_name(dev), vaf);
Joe Perchesb004ff42012-09-12 20:12:19 -07006516 } else {
Joe Perches256df2f2010-06-27 01:02:35 +00006517 r = printk("%s(NULL net_device): %pV", level, vaf);
Joe Perchesb004ff42012-09-12 20:12:19 -07006518 }
Joe Perches256df2f2010-06-27 01:02:35 +00006519
6520 return r;
6521}
6522
6523int netdev_printk(const char *level, const struct net_device *dev,
6524 const char *format, ...)
6525{
6526 struct va_format vaf;
6527 va_list args;
6528 int r;
6529
6530 va_start(args, format);
6531
6532 vaf.fmt = format;
6533 vaf.va = &args;
6534
6535 r = __netdev_printk(level, dev, &vaf);
Joe Perchesb004ff42012-09-12 20:12:19 -07006536
Joe Perches256df2f2010-06-27 01:02:35 +00006537 va_end(args);
6538
6539 return r;
6540}
6541EXPORT_SYMBOL(netdev_printk);
6542
6543#define define_netdev_printk_level(func, level) \
6544int func(const struct net_device *dev, const char *fmt, ...) \
6545{ \
6546 int r; \
6547 struct va_format vaf; \
6548 va_list args; \
6549 \
6550 va_start(args, fmt); \
6551 \
6552 vaf.fmt = fmt; \
6553 vaf.va = &args; \
6554 \
6555 r = __netdev_printk(level, dev, &vaf); \
Joe Perchesb004ff42012-09-12 20:12:19 -07006556 \
Joe Perches256df2f2010-06-27 01:02:35 +00006557 va_end(args); \
6558 \
6559 return r; \
6560} \
6561EXPORT_SYMBOL(func);
6562
6563define_netdev_printk_level(netdev_emerg, KERN_EMERG);
6564define_netdev_printk_level(netdev_alert, KERN_ALERT);
6565define_netdev_printk_level(netdev_crit, KERN_CRIT);
6566define_netdev_printk_level(netdev_err, KERN_ERR);
6567define_netdev_printk_level(netdev_warn, KERN_WARNING);
6568define_netdev_printk_level(netdev_notice, KERN_NOTICE);
6569define_netdev_printk_level(netdev_info, KERN_INFO);
6570
Pavel Emelyanov46650792007-10-08 20:38:39 -07006571static void __net_exit netdev_exit(struct net *net)
Eric W. Biederman881d9662007-09-17 11:56:21 -07006572{
6573 kfree(net->dev_name_head);
6574 kfree(net->dev_index_head);
6575}
6576
Denis V. Lunev022cbae2007-11-13 03:23:50 -08006577static struct pernet_operations __net_initdata netdev_net_ops = {
Eric W. Biederman881d9662007-09-17 11:56:21 -07006578 .init = netdev_init,
6579 .exit = netdev_exit,
6580};
6581
Pavel Emelyanov46650792007-10-08 20:38:39 -07006582static void __net_exit default_device_exit(struct net *net)
Eric W. Biedermance286d32007-09-12 13:53:49 +02006583{
Eric W. Biedermane008b5f2009-11-29 22:25:30 +00006584 struct net_device *dev, *aux;
Eric W. Biedermance286d32007-09-12 13:53:49 +02006585 /*
Eric W. Biedermane008b5f2009-11-29 22:25:30 +00006586 * Push all migratable network devices back to the
Eric W. Biedermance286d32007-09-12 13:53:49 +02006587 * initial network namespace
6588 */
6589 rtnl_lock();
Eric W. Biedermane008b5f2009-11-29 22:25:30 +00006590 for_each_netdev_safe(net, dev, aux) {
Eric W. Biedermance286d32007-09-12 13:53:49 +02006591 int err;
Pavel Emelyanovaca51392008-05-08 01:24:25 -07006592 char fb_name[IFNAMSIZ];
Eric W. Biedermance286d32007-09-12 13:53:49 +02006593
6594 /* Ignore unmoveable devices (i.e. loopback) */
6595 if (dev->features & NETIF_F_NETNS_LOCAL)
6596 continue;
6597
Eric W. Biedermane008b5f2009-11-29 22:25:30 +00006598 /* Leave virtual devices for the generic cleanup */
6599 if (dev->rtnl_link_ops)
6600 continue;
Eric W. Biedermand0c082c2008-11-05 15:59:38 -08006601
Lucas De Marchi25985ed2011-03-30 22:57:33 -03006602 /* Push remaining network devices to init_net */
Pavel Emelyanovaca51392008-05-08 01:24:25 -07006603 snprintf(fb_name, IFNAMSIZ, "dev%d", dev->ifindex);
6604 err = dev_change_net_namespace(dev, &init_net, fb_name);
Eric W. Biedermance286d32007-09-12 13:53:49 +02006605 if (err) {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00006606 pr_emerg("%s: failed to move %s to init_net: %d\n",
6607 __func__, dev->name, err);
Pavel Emelyanovaca51392008-05-08 01:24:25 -07006608 BUG();
Eric W. Biedermance286d32007-09-12 13:53:49 +02006609 }
6610 }
6611 rtnl_unlock();
6612}
6613
Eric W. Biederman50624c92013-09-23 21:19:49 -07006614static void __net_exit rtnl_lock_unregistering(struct list_head *net_list)
6615{
6616 /* Return with the rtnl_lock held when there are no network
6617 * devices unregistering in any network namespace in net_list.
6618 */
6619 struct net *net;
6620 bool unregistering;
6621 DEFINE_WAIT(wait);
6622
6623 for (;;) {
6624 prepare_to_wait(&netdev_unregistering_wq, &wait,
6625 TASK_UNINTERRUPTIBLE);
6626 unregistering = false;
6627 rtnl_lock();
6628 list_for_each_entry(net, net_list, exit_list) {
6629 if (net->dev_unreg_count > 0) {
6630 unregistering = true;
6631 break;
6632 }
6633 }
6634 if (!unregistering)
6635 break;
6636 __rtnl_unlock();
6637 schedule();
6638 }
6639 finish_wait(&netdev_unregistering_wq, &wait);
6640}
6641
Eric W. Biederman04dc7f6b2009-12-03 02:29:04 +00006642static void __net_exit default_device_exit_batch(struct list_head *net_list)
6643{
6644 /* At exit all network devices most be removed from a network
Uwe Kleine-Königb5950762010-11-01 15:38:34 -04006645 * namespace. Do this in the reverse order of registration.
Eric W. Biederman04dc7f6b2009-12-03 02:29:04 +00006646 * Do this across as many network namespaces as possible to
6647 * improve batching efficiency.
6648 */
6649 struct net_device *dev;
6650 struct net *net;
6651 LIST_HEAD(dev_kill_list);
6652
Eric W. Biederman50624c92013-09-23 21:19:49 -07006653 /* To prevent network device cleanup code from dereferencing
6654 * loopback devices or network devices that have been freed
6655 * wait here for all pending unregistrations to complete,
6656 * before unregistring the loopback device and allowing the
6657 * network namespace be freed.
6658 *
6659 * The netdev todo list containing all network devices
6660 * unregistrations that happen in default_device_exit_batch
6661 * will run in the rtnl_unlock() at the end of
6662 * default_device_exit_batch.
6663 */
6664 rtnl_lock_unregistering(net_list);
Eric W. Biederman04dc7f6b2009-12-03 02:29:04 +00006665 list_for_each_entry(net, net_list, exit_list) {
6666 for_each_netdev_reverse(net, dev) {
6667 if (dev->rtnl_link_ops)
6668 dev->rtnl_link_ops->dellink(dev, &dev_kill_list);
6669 else
6670 unregister_netdevice_queue(dev, &dev_kill_list);
6671 }
6672 }
6673 unregister_netdevice_many(&dev_kill_list);
Eric Dumazetceaaec92011-02-17 22:59:19 +00006674 list_del(&dev_kill_list);
Eric W. Biederman04dc7f6b2009-12-03 02:29:04 +00006675 rtnl_unlock();
6676}
6677
Denis V. Lunev022cbae2007-11-13 03:23:50 -08006678static struct pernet_operations __net_initdata default_device_ops = {
Eric W. Biedermance286d32007-09-12 13:53:49 +02006679 .exit = default_device_exit,
Eric W. Biederman04dc7f6b2009-12-03 02:29:04 +00006680 .exit_batch = default_device_exit_batch,
Eric W. Biedermance286d32007-09-12 13:53:49 +02006681};
6682
Linus Torvalds1da177e2005-04-16 15:20:36 -07006683/*
6684 * Initialize the DEV module. At boot time this walks the device list and
6685 * unhooks any devices that fail to initialise (normally hardware not
6686 * present) and leaves us with a valid list of present and active devices.
6687 *
6688 */
6689
6690/*
6691 * This is called single threaded during boot, so no need
6692 * to take the rtnl semaphore.
6693 */
6694static int __init net_dev_init(void)
6695{
6696 int i, rc = -ENOMEM;
6697
6698 BUG_ON(!dev_boot_phase);
6699
Linus Torvalds1da177e2005-04-16 15:20:36 -07006700 if (dev_proc_init())
6701 goto out;
6702
Eric W. Biederman8b41d182007-09-26 22:02:53 -07006703 if (netdev_kobject_init())
Linus Torvalds1da177e2005-04-16 15:20:36 -07006704 goto out;
6705
6706 INIT_LIST_HEAD(&ptype_all);
Pavel Emelyanov82d8a8672007-11-26 20:12:58 +08006707 for (i = 0; i < PTYPE_HASH_SIZE; i++)
Linus Torvalds1da177e2005-04-16 15:20:36 -07006708 INIT_LIST_HEAD(&ptype_base[i]);
6709
Vlad Yasevich62532da2012-11-15 08:49:10 +00006710 INIT_LIST_HEAD(&offload_base);
6711
Eric W. Biederman881d9662007-09-17 11:56:21 -07006712 if (register_pernet_subsys(&netdev_net_ops))
6713 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006714
6715 /*
6716 * Initialise the packet receive queues.
6717 */
6718
KAMEZAWA Hiroyuki6f912042006-04-10 22:52:50 -07006719 for_each_possible_cpu(i) {
Eric Dumazete36fa2f2010-04-19 21:17:14 +00006720 struct softnet_data *sd = &per_cpu(softnet_data, i);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006721
Changli Gaodee42872010-05-02 05:42:16 +00006722 memset(sd, 0, sizeof(*sd));
Eric Dumazete36fa2f2010-04-19 21:17:14 +00006723 skb_queue_head_init(&sd->input_pkt_queue);
Changli Gao6e7676c2010-04-27 15:07:33 -07006724 skb_queue_head_init(&sd->process_queue);
Eric Dumazete36fa2f2010-04-19 21:17:14 +00006725 sd->completion_queue = NULL;
6726 INIT_LIST_HEAD(&sd->poll_list);
Changli Gaoa9cbd582010-04-26 23:06:24 +00006727 sd->output_queue = NULL;
6728 sd->output_queue_tailp = &sd->output_queue;
Eric Dumazetdf334542010-03-24 19:13:54 +00006729#ifdef CONFIG_RPS
Eric Dumazete36fa2f2010-04-19 21:17:14 +00006730 sd->csd.func = rps_trigger_softirq;
6731 sd->csd.info = sd;
6732 sd->csd.flags = 0;
6733 sd->cpu = i;
Tom Herbert1e94d722010-03-18 17:45:44 -07006734#endif
Tom Herbert0a9627f2010-03-16 08:03:29 +00006735
Eric Dumazete36fa2f2010-04-19 21:17:14 +00006736 sd->backlog.poll = process_backlog;
6737 sd->backlog.weight = weight_p;
6738 sd->backlog.gro_list = NULL;
6739 sd->backlog.gro_count = 0;
Willem de Bruijn99bbc702013-05-20 04:02:32 +00006740
6741#ifdef CONFIG_NET_FLOW_LIMIT
6742 sd->flow_limit = NULL;
6743#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07006744 }
6745
Linus Torvalds1da177e2005-04-16 15:20:36 -07006746 dev_boot_phase = 0;
6747
Eric W. Biederman505d4f72008-11-07 22:54:20 -08006748 /* The loopback device is special if any other network devices
6749 * is present in a network namespace the loopback device must
6750 * be present. Since we now dynamically allocate and free the
6751 * loopback device ensure this invariant is maintained by
6752 * keeping the loopback device as the first device on the
6753 * list of network devices. Ensuring the loopback devices
6754 * is the first device that appears and the last network device
6755 * that disappears.
6756 */
6757 if (register_pernet_device(&loopback_net_ops))
6758 goto out;
6759
6760 if (register_pernet_device(&default_device_ops))
6761 goto out;
6762
Carlos R. Mafra962cf362008-05-15 11:15:37 -03006763 open_softirq(NET_TX_SOFTIRQ, net_tx_action);
6764 open_softirq(NET_RX_SOFTIRQ, net_rx_action);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006765
6766 hotcpu_notifier(dev_cpu_callback, 0);
6767 dst_init();
Linus Torvalds1da177e2005-04-16 15:20:36 -07006768 rc = 0;
6769out:
6770 return rc;
6771}
6772
6773subsys_initcall(net_dev_init);