blob: dffbef70cd31253625da50819a171fbc59b386a5 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * NET3 Protocol independent device support routines.
3 *
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public License
6 * as published by the Free Software Foundation; either version
7 * 2 of the License, or (at your option) any later version.
8 *
9 * Derived from the non IP parts of dev.c 1.0.19
Jesper Juhl02c30a82005-05-05 16:16:16 -070010 * Authors: Ross Biro
Linus Torvalds1da177e2005-04-16 15:20:36 -070011 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
12 * Mark Evans, <evansmp@uhura.aston.ac.uk>
13 *
14 * Additional Authors:
15 * Florian la Roche <rzsfl@rz.uni-sb.de>
16 * Alan Cox <gw4pts@gw4pts.ampr.org>
17 * David Hinds <dahinds@users.sourceforge.net>
18 * Alexey Kuznetsov <kuznet@ms2.inr.ac.ru>
19 * Adam Sulmicki <adam@cfar.umd.edu>
20 * Pekka Riikonen <priikone@poesidon.pspt.fi>
21 *
22 * Changes:
23 * D.J. Barrow : Fixed bug where dev->refcnt gets set
24 * to 2 if register_netdev gets called
25 * before net_dev_init & also removed a
26 * few lines of code in the process.
27 * Alan Cox : device private ioctl copies fields back.
28 * Alan Cox : Transmit queue code does relevant
29 * stunts to keep the queue safe.
30 * Alan Cox : Fixed double lock.
31 * Alan Cox : Fixed promisc NULL pointer trap
32 * ???????? : Support the full private ioctl range
33 * Alan Cox : Moved ioctl permission check into
34 * drivers
35 * Tim Kordas : SIOCADDMULTI/SIOCDELMULTI
36 * Alan Cox : 100 backlog just doesn't cut it when
37 * you start doing multicast video 8)
38 * Alan Cox : Rewrote net_bh and list manager.
39 * Alan Cox : Fix ETH_P_ALL echoback lengths.
40 * Alan Cox : Took out transmit every packet pass
41 * Saved a few bytes in the ioctl handler
42 * Alan Cox : Network driver sets packet type before
43 * calling netif_rx. Saves a function
44 * call a packet.
45 * Alan Cox : Hashed net_bh()
46 * Richard Kooijman: Timestamp fixes.
47 * Alan Cox : Wrong field in SIOCGIFDSTADDR
48 * Alan Cox : Device lock protection.
49 * Alan Cox : Fixed nasty side effect of device close
50 * changes.
51 * Rudi Cilibrasi : Pass the right thing to
52 * set_mac_address()
53 * Dave Miller : 32bit quantity for the device lock to
54 * make it work out on a Sparc.
55 * Bjorn Ekwall : Added KERNELD hack.
56 * Alan Cox : Cleaned up the backlog initialise.
57 * Craig Metz : SIOCGIFCONF fix if space for under
58 * 1 device.
59 * Thomas Bogendoerfer : Return ENODEV for dev_open, if there
60 * is no device open function.
61 * Andi Kleen : Fix error reporting for SIOCGIFCONF
62 * Michael Chastain : Fix signed/unsigned for SIOCGIFCONF
63 * Cyrus Durgin : Cleaned for KMOD
64 * Adam Sulmicki : Bug Fix : Network Device Unload
65 * A network device unload needs to purge
66 * the backlog queue.
67 * Paul Rusty Russell : SIOCSIFNAME
68 * Pekka Riikonen : Netdev boot-time settings code
69 * Andrew Morton : Make unregister_netdevice wait
70 * indefinitely on dev->refcnt
71 * J Hadi Salim : - Backlog queue sampling
72 * - netif_rx() feedback
73 */
74
75#include <asm/uaccess.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070076#include <linux/bitops.h>
Randy Dunlap4fc268d2006-01-11 12:17:47 -080077#include <linux/capability.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070078#include <linux/cpu.h>
79#include <linux/types.h>
80#include <linux/kernel.h>
stephen hemminger08e98972009-11-10 07:20:34 +000081#include <linux/hash.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090082#include <linux/slab.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070083#include <linux/sched.h>
Arjan van de Ven4a3e2f72006-03-20 22:33:17 -080084#include <linux/mutex.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070085#include <linux/string.h>
86#include <linux/mm.h>
87#include <linux/socket.h>
88#include <linux/sockios.h>
89#include <linux/errno.h>
90#include <linux/interrupt.h>
91#include <linux/if_ether.h>
92#include <linux/netdevice.h>
93#include <linux/etherdevice.h>
Ben Hutchings0187bdf2008-06-19 16:15:47 -070094#include <linux/ethtool.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070095#include <linux/notifier.h>
96#include <linux/skbuff.h>
Eric W. Biederman457c4cb2007-09-12 12:01:34 +020097#include <net/net_namespace.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070098#include <net/sock.h>
99#include <linux/rtnetlink.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700100#include <linux/stat.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700101#include <net/dst.h>
102#include <net/pkt_sched.h>
103#include <net/checksum.h>
Arnd Bergmann44540962009-11-26 06:07:08 +0000104#include <net/xfrm.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700105#include <linux/highmem.h>
106#include <linux/init.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700107#include <linux/module.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700108#include <linux/netpoll.h>
109#include <linux/rcupdate.h>
110#include <linux/delay.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700111#include <net/iw_handler.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700112#include <asm/current.h>
Steve Grubb5bdb9882005-12-03 08:39:35 -0500113#include <linux/audit.h>
Chris Leechdb217332006-06-17 21:24:58 -0700114#include <linux/dmaengine.h>
Herbert Xuf6a78bf2006-06-22 02:57:17 -0700115#include <linux/err.h>
David S. Millerc7fa9d12006-08-15 16:34:13 -0700116#include <linux/ctype.h>
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700117#include <linux/if_arp.h>
Ben Hutchings6de329e2008-06-16 17:02:28 -0700118#include <linux/if_vlan.h>
David S. Miller8f0f2222008-07-15 03:47:03 -0700119#include <linux/ip.h>
Alexander Duyckad55dca2008-09-20 22:05:50 -0700120#include <net/ip.h>
David S. Miller8f0f2222008-07-15 03:47:03 -0700121#include <linux/ipv6.h>
122#include <linux/in.h>
David S. Millerb6b2fed2008-07-21 09:48:06 -0700123#include <linux/jhash.h>
124#include <linux/random.h>
David S. Miller9cbc1cb2009-06-15 03:02:23 -0700125#include <trace/events/napi.h>
Koki Sanagicf66ba52010-08-23 18:45:02 +0900126#include <trace/events/net.h>
Koki Sanagi07dc22e2010-08-23 18:46:12 +0900127#include <trace/events/skb.h>
FUJITA Tomonori5acbbd42010-03-30 22:35:50 +0000128#include <linux/pci.h>
Stephen Rothwellcaeda9b2010-09-16 21:39:16 -0700129#include <linux/inetdevice.h>
Ben Hutchingsc4454772011-01-19 11:03:53 +0000130#include <linux/cpu_rmap.h>
Ingo Molnarc5905af2012-02-24 08:31:31 +0100131#include <linux/static_key.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700132
Pavel Emelyanov342709e2007-10-23 21:14:45 -0700133#include "net-sysfs.h"
134
Herbert Xud565b0a2008-12-15 23:38:52 -0800135/* Instead of increasing this, you should create a hash table. */
136#define MAX_GRO_SKBS 8
137
Herbert Xu5d38a072009-01-04 16:13:40 -0800138/* This should be increased if a protocol with a bigger head is added. */
139#define GRO_MAX_HEAD (MAX_HEADER + 128)
140
Linus Torvalds1da177e2005-04-16 15:20:36 -0700141static DEFINE_SPINLOCK(ptype_lock);
Vlad Yasevich62532da2012-11-15 08:49:10 +0000142static DEFINE_SPINLOCK(offload_lock);
Cong Wang900ff8c2013-02-18 19:20:33 +0000143struct list_head ptype_base[PTYPE_HASH_SIZE] __read_mostly;
144struct list_head ptype_all __read_mostly; /* Taps */
Vlad Yasevich62532da2012-11-15 08:49:10 +0000145static struct list_head offload_base __read_mostly;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700146
Linus Torvalds1da177e2005-04-16 15:20:36 -0700147/*
Pavel Emelianov7562f872007-05-03 15:13:45 -0700148 * The @dev_base_head list is protected by @dev_base_lock and the rtnl
Linus Torvalds1da177e2005-04-16 15:20:36 -0700149 * semaphore.
150 *
Eric Dumazetc6d14c82009-11-04 05:43:23 -0800151 * Pure readers hold dev_base_lock for reading, or rcu_read_lock()
Linus Torvalds1da177e2005-04-16 15:20:36 -0700152 *
153 * Writers must hold the rtnl semaphore while they loop through the
Pavel Emelianov7562f872007-05-03 15:13:45 -0700154 * dev_base_head list, and hold dev_base_lock for writing when they do the
Linus Torvalds1da177e2005-04-16 15:20:36 -0700155 * actual updates. This allows pure readers to access the list even
156 * while a writer is preparing to update it.
157 *
158 * To put it another way, dev_base_lock is held for writing only to
159 * protect against pure readers; the rtnl semaphore provides the
160 * protection against other writers.
161 *
162 * See, for example usages, register_netdevice() and
163 * unregister_netdevice(), which must be called with the rtnl
164 * semaphore held.
165 */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700166DEFINE_RWLOCK(dev_base_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700167EXPORT_SYMBOL(dev_base_lock);
168
Eric Dumazet30e6c9f2012-12-20 17:25:08 +0000169seqcount_t devnet_rename_seq;
Brian Haleyc91f6df2012-11-26 05:21:08 +0000170
Thomas Graf4e985ad2011-06-21 03:11:20 +0000171static inline void dev_base_seq_inc(struct net *net)
172{
173 while (++net->dev_base_seq == 0);
174}
175
Eric W. Biederman881d9662007-09-17 11:56:21 -0700176static inline struct hlist_head *dev_name_hash(struct net *net, const char *name)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700177{
Eric Dumazet95c96172012-04-15 05:58:06 +0000178 unsigned int hash = full_name_hash(name, strnlen(name, IFNAMSIZ));
179
stephen hemminger08e98972009-11-10 07:20:34 +0000180 return &net->dev_name_head[hash_32(hash, NETDEV_HASHBITS)];
Linus Torvalds1da177e2005-04-16 15:20:36 -0700181}
182
Eric W. Biederman881d9662007-09-17 11:56:21 -0700183static inline struct hlist_head *dev_index_hash(struct net *net, int ifindex)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700184{
Eric Dumazet7c28bd02009-10-24 06:13:17 -0700185 return &net->dev_index_head[ifindex & (NETDEV_HASHENTRIES - 1)];
Linus Torvalds1da177e2005-04-16 15:20:36 -0700186}
187
Eric Dumazete36fa2f2010-04-19 21:17:14 +0000188static inline void rps_lock(struct softnet_data *sd)
Changli Gao152102c2010-03-30 20:16:22 +0000189{
190#ifdef CONFIG_RPS
Eric Dumazete36fa2f2010-04-19 21:17:14 +0000191 spin_lock(&sd->input_pkt_queue.lock);
Changli Gao152102c2010-03-30 20:16:22 +0000192#endif
193}
194
Eric Dumazete36fa2f2010-04-19 21:17:14 +0000195static inline void rps_unlock(struct softnet_data *sd)
Changli Gao152102c2010-03-30 20:16:22 +0000196{
197#ifdef CONFIG_RPS
Eric Dumazete36fa2f2010-04-19 21:17:14 +0000198 spin_unlock(&sd->input_pkt_queue.lock);
Changli Gao152102c2010-03-30 20:16:22 +0000199#endif
200}
201
Eric W. Biedermance286d32007-09-12 13:53:49 +0200202/* Device list insertion */
203static int list_netdevice(struct net_device *dev)
204{
YOSHIFUJI Hideakic346dca2008-03-25 21:47:49 +0900205 struct net *net = dev_net(dev);
Eric W. Biedermance286d32007-09-12 13:53:49 +0200206
207 ASSERT_RTNL();
208
209 write_lock_bh(&dev_base_lock);
Eric Dumazetc6d14c82009-11-04 05:43:23 -0800210 list_add_tail_rcu(&dev->dev_list, &net->dev_base_head);
Eric Dumazet72c95282009-10-30 07:11:27 +0000211 hlist_add_head_rcu(&dev->name_hlist, dev_name_hash(net, dev->name));
Eric Dumazetfb699dfd2009-10-19 19:18:49 +0000212 hlist_add_head_rcu(&dev->index_hlist,
213 dev_index_hash(net, dev->ifindex));
Eric W. Biedermance286d32007-09-12 13:53:49 +0200214 write_unlock_bh(&dev_base_lock);
Thomas Graf4e985ad2011-06-21 03:11:20 +0000215
216 dev_base_seq_inc(net);
217
Eric W. Biedermance286d32007-09-12 13:53:49 +0200218 return 0;
219}
220
Eric Dumazetfb699dfd2009-10-19 19:18:49 +0000221/* Device list removal
222 * caller must respect a RCU grace period before freeing/reusing dev
223 */
Eric W. Biedermance286d32007-09-12 13:53:49 +0200224static void unlist_netdevice(struct net_device *dev)
225{
226 ASSERT_RTNL();
227
228 /* Unlink dev from the device chain */
229 write_lock_bh(&dev_base_lock);
Eric Dumazetc6d14c82009-11-04 05:43:23 -0800230 list_del_rcu(&dev->dev_list);
Eric Dumazet72c95282009-10-30 07:11:27 +0000231 hlist_del_rcu(&dev->name_hlist);
Eric Dumazetfb699dfd2009-10-19 19:18:49 +0000232 hlist_del_rcu(&dev->index_hlist);
Eric W. Biedermance286d32007-09-12 13:53:49 +0200233 write_unlock_bh(&dev_base_lock);
Thomas Graf4e985ad2011-06-21 03:11:20 +0000234
235 dev_base_seq_inc(dev_net(dev));
Eric W. Biedermance286d32007-09-12 13:53:49 +0200236}
237
Linus Torvalds1da177e2005-04-16 15:20:36 -0700238/*
239 * Our notifier list
240 */
241
Alan Sternf07d5b92006-05-09 15:23:03 -0700242static RAW_NOTIFIER_HEAD(netdev_chain);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700243
244/*
245 * Device drivers call our routines to queue packets here. We empty the
246 * queue in the local softnet handler.
247 */
Stephen Hemmingerbea33482007-10-03 16:41:36 -0700248
Eric Dumazet9958da02010-04-17 04:17:02 +0000249DEFINE_PER_CPU_ALIGNED(struct softnet_data, softnet_data);
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700250EXPORT_PER_CPU_SYMBOL(softnet_data);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700251
David S. Millercf508b12008-07-22 14:16:42 -0700252#ifdef CONFIG_LOCKDEP
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700253/*
David S. Millerc773e842008-07-08 23:13:53 -0700254 * register_netdevice() inits txq->_xmit_lock and sets lockdep class
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700255 * according to dev->type
256 */
257static const unsigned short netdev_lock_type[] =
258 {ARPHRD_NETROM, ARPHRD_ETHER, ARPHRD_EETHER, ARPHRD_AX25,
259 ARPHRD_PRONET, ARPHRD_CHAOS, ARPHRD_IEEE802, ARPHRD_ARCNET,
260 ARPHRD_APPLETLK, ARPHRD_DLCI, ARPHRD_ATM, ARPHRD_METRICOM,
261 ARPHRD_IEEE1394, ARPHRD_EUI64, ARPHRD_INFINIBAND, ARPHRD_SLIP,
262 ARPHRD_CSLIP, ARPHRD_SLIP6, ARPHRD_CSLIP6, ARPHRD_RSRVD,
263 ARPHRD_ADAPT, ARPHRD_ROSE, ARPHRD_X25, ARPHRD_HWX25,
264 ARPHRD_PPP, ARPHRD_CISCO, ARPHRD_LAPB, ARPHRD_DDCMP,
265 ARPHRD_RAWHDLC, ARPHRD_TUNNEL, ARPHRD_TUNNEL6, ARPHRD_FRAD,
266 ARPHRD_SKIP, ARPHRD_LOOPBACK, ARPHRD_LOCALTLK, ARPHRD_FDDI,
267 ARPHRD_BIF, ARPHRD_SIT, ARPHRD_IPDDP, ARPHRD_IPGRE,
268 ARPHRD_PIMREG, ARPHRD_HIPPI, ARPHRD_ASH, ARPHRD_ECONET,
269 ARPHRD_IRDA, ARPHRD_FCPP, ARPHRD_FCAL, ARPHRD_FCPL,
Paul Gortmaker211ed862012-05-10 17:14:35 -0400270 ARPHRD_FCFABRIC, ARPHRD_IEEE80211, ARPHRD_IEEE80211_PRISM,
271 ARPHRD_IEEE80211_RADIOTAP, ARPHRD_PHONET, ARPHRD_PHONET_PIPE,
272 ARPHRD_IEEE802154, ARPHRD_VOID, ARPHRD_NONE};
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700273
Jan Engelhardt36cbd3d2009-08-05 10:42:58 -0700274static const char *const netdev_lock_name[] =
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700275 {"_xmit_NETROM", "_xmit_ETHER", "_xmit_EETHER", "_xmit_AX25",
276 "_xmit_PRONET", "_xmit_CHAOS", "_xmit_IEEE802", "_xmit_ARCNET",
277 "_xmit_APPLETLK", "_xmit_DLCI", "_xmit_ATM", "_xmit_METRICOM",
278 "_xmit_IEEE1394", "_xmit_EUI64", "_xmit_INFINIBAND", "_xmit_SLIP",
279 "_xmit_CSLIP", "_xmit_SLIP6", "_xmit_CSLIP6", "_xmit_RSRVD",
280 "_xmit_ADAPT", "_xmit_ROSE", "_xmit_X25", "_xmit_HWX25",
281 "_xmit_PPP", "_xmit_CISCO", "_xmit_LAPB", "_xmit_DDCMP",
282 "_xmit_RAWHDLC", "_xmit_TUNNEL", "_xmit_TUNNEL6", "_xmit_FRAD",
283 "_xmit_SKIP", "_xmit_LOOPBACK", "_xmit_LOCALTLK", "_xmit_FDDI",
284 "_xmit_BIF", "_xmit_SIT", "_xmit_IPDDP", "_xmit_IPGRE",
285 "_xmit_PIMREG", "_xmit_HIPPI", "_xmit_ASH", "_xmit_ECONET",
286 "_xmit_IRDA", "_xmit_FCPP", "_xmit_FCAL", "_xmit_FCPL",
Paul Gortmaker211ed862012-05-10 17:14:35 -0400287 "_xmit_FCFABRIC", "_xmit_IEEE80211", "_xmit_IEEE80211_PRISM",
288 "_xmit_IEEE80211_RADIOTAP", "_xmit_PHONET", "_xmit_PHONET_PIPE",
289 "_xmit_IEEE802154", "_xmit_VOID", "_xmit_NONE"};
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700290
291static struct lock_class_key netdev_xmit_lock_key[ARRAY_SIZE(netdev_lock_type)];
David S. Millercf508b12008-07-22 14:16:42 -0700292static struct lock_class_key netdev_addr_lock_key[ARRAY_SIZE(netdev_lock_type)];
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700293
294static inline unsigned short netdev_lock_pos(unsigned short dev_type)
295{
296 int i;
297
298 for (i = 0; i < ARRAY_SIZE(netdev_lock_type); i++)
299 if (netdev_lock_type[i] == dev_type)
300 return i;
301 /* the last key is used by default */
302 return ARRAY_SIZE(netdev_lock_type) - 1;
303}
304
David S. Millercf508b12008-07-22 14:16:42 -0700305static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock,
306 unsigned short dev_type)
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700307{
308 int i;
309
310 i = netdev_lock_pos(dev_type);
311 lockdep_set_class_and_name(lock, &netdev_xmit_lock_key[i],
312 netdev_lock_name[i]);
313}
David S. Millercf508b12008-07-22 14:16:42 -0700314
315static inline void netdev_set_addr_lockdep_class(struct net_device *dev)
316{
317 int i;
318
319 i = netdev_lock_pos(dev->type);
320 lockdep_set_class_and_name(&dev->addr_list_lock,
321 &netdev_addr_lock_key[i],
322 netdev_lock_name[i]);
323}
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700324#else
David S. Millercf508b12008-07-22 14:16:42 -0700325static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock,
326 unsigned short dev_type)
327{
328}
329static inline void netdev_set_addr_lockdep_class(struct net_device *dev)
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700330{
331}
332#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -0700333
334/*******************************************************************************
335
336 Protocol management and registration routines
337
338*******************************************************************************/
339
340/*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700341 * Add a protocol ID to the list. Now that the input handler is
342 * smarter we can dispense with all the messy stuff that used to be
343 * here.
344 *
345 * BEWARE!!! Protocol handlers, mangling input packets,
346 * MUST BE last in hash buckets and checking protocol handlers
347 * MUST start from promiscuous ptype_all chain in net_bh.
348 * It is true now, do not change it.
349 * Explanation follows: if protocol handler, mangling packet, will
350 * be the first on list, it is not able to sense, that packet
351 * is cloned and should be copied-on-write, so that it will
352 * change it and subsequent readers will get broken packet.
353 * --ANK (980803)
354 */
355
Eric Dumazetc07b68e2010-09-02 03:53:46 +0000356static inline struct list_head *ptype_head(const struct packet_type *pt)
357{
358 if (pt->type == htons(ETH_P_ALL))
359 return &ptype_all;
360 else
361 return &ptype_base[ntohs(pt->type) & PTYPE_HASH_MASK];
362}
363
Linus Torvalds1da177e2005-04-16 15:20:36 -0700364/**
365 * dev_add_pack - add packet handler
366 * @pt: packet type declaration
367 *
368 * Add a protocol handler to the networking stack. The passed &packet_type
369 * is linked into kernel lists and may not be freed until it has been
370 * removed from the kernel lists.
371 *
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +0900372 * This call does not sleep therefore it can not
Linus Torvalds1da177e2005-04-16 15:20:36 -0700373 * guarantee all CPU's that are in middle of receiving packets
374 * will see the new packet type (until the next received packet).
375 */
376
377void dev_add_pack(struct packet_type *pt)
378{
Eric Dumazetc07b68e2010-09-02 03:53:46 +0000379 struct list_head *head = ptype_head(pt);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700380
Eric Dumazetc07b68e2010-09-02 03:53:46 +0000381 spin_lock(&ptype_lock);
382 list_add_rcu(&pt->list, head);
383 spin_unlock(&ptype_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700384}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700385EXPORT_SYMBOL(dev_add_pack);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700386
Linus Torvalds1da177e2005-04-16 15:20:36 -0700387/**
388 * __dev_remove_pack - remove packet handler
389 * @pt: packet type declaration
390 *
391 * Remove a protocol handler that was previously added to the kernel
392 * protocol handlers by dev_add_pack(). The passed &packet_type is removed
393 * from the kernel lists and can be freed or reused once this function
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +0900394 * returns.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700395 *
396 * The packet type might still be in use by receivers
397 * and must not be freed until after all the CPU's have gone
398 * through a quiescent state.
399 */
400void __dev_remove_pack(struct packet_type *pt)
401{
Eric Dumazetc07b68e2010-09-02 03:53:46 +0000402 struct list_head *head = ptype_head(pt);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700403 struct packet_type *pt1;
404
Eric Dumazetc07b68e2010-09-02 03:53:46 +0000405 spin_lock(&ptype_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700406
407 list_for_each_entry(pt1, head, list) {
408 if (pt == pt1) {
409 list_del_rcu(&pt->list);
410 goto out;
411 }
412 }
413
Joe Perches7b6cd1c2012-02-01 10:54:43 +0000414 pr_warn("dev_remove_pack: %p not found\n", pt);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700415out:
Eric Dumazetc07b68e2010-09-02 03:53:46 +0000416 spin_unlock(&ptype_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700417}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700418EXPORT_SYMBOL(__dev_remove_pack);
419
Linus Torvalds1da177e2005-04-16 15:20:36 -0700420/**
421 * dev_remove_pack - remove packet handler
422 * @pt: packet type declaration
423 *
424 * Remove a protocol handler that was previously added to the kernel
425 * protocol handlers by dev_add_pack(). The passed &packet_type is removed
426 * from the kernel lists and can be freed or reused once this function
427 * returns.
428 *
429 * This call sleeps to guarantee that no CPU is looking at the packet
430 * type after return.
431 */
432void dev_remove_pack(struct packet_type *pt)
433{
434 __dev_remove_pack(pt);
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +0900435
Linus Torvalds1da177e2005-04-16 15:20:36 -0700436 synchronize_net();
437}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700438EXPORT_SYMBOL(dev_remove_pack);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700439
Vlad Yasevich62532da2012-11-15 08:49:10 +0000440
441/**
442 * dev_add_offload - register offload handlers
443 * @po: protocol offload declaration
444 *
445 * Add protocol offload handlers to the networking stack. The passed
446 * &proto_offload is linked into kernel lists and may not be freed until
447 * it has been removed from the kernel lists.
448 *
449 * This call does not sleep therefore it can not
450 * guarantee all CPU's that are in middle of receiving packets
451 * will see the new offload handlers (until the next received packet).
452 */
453void dev_add_offload(struct packet_offload *po)
454{
455 struct list_head *head = &offload_base;
456
457 spin_lock(&offload_lock);
458 list_add_rcu(&po->list, head);
459 spin_unlock(&offload_lock);
460}
461EXPORT_SYMBOL(dev_add_offload);
462
463/**
464 * __dev_remove_offload - remove offload handler
465 * @po: packet offload declaration
466 *
467 * Remove a protocol offload handler that was previously added to the
468 * kernel offload handlers by dev_add_offload(). The passed &offload_type
469 * is removed from the kernel lists and can be freed or reused once this
470 * function returns.
471 *
472 * The packet type might still be in use by receivers
473 * and must not be freed until after all the CPU's have gone
474 * through a quiescent state.
475 */
476void __dev_remove_offload(struct packet_offload *po)
477{
478 struct list_head *head = &offload_base;
479 struct packet_offload *po1;
480
Eric Dumazetc53aa502012-11-16 08:08:23 +0000481 spin_lock(&offload_lock);
Vlad Yasevich62532da2012-11-15 08:49:10 +0000482
483 list_for_each_entry(po1, head, list) {
484 if (po == po1) {
485 list_del_rcu(&po->list);
486 goto out;
487 }
488 }
489
490 pr_warn("dev_remove_offload: %p not found\n", po);
491out:
Eric Dumazetc53aa502012-11-16 08:08:23 +0000492 spin_unlock(&offload_lock);
Vlad Yasevich62532da2012-11-15 08:49:10 +0000493}
494EXPORT_SYMBOL(__dev_remove_offload);
495
496/**
497 * dev_remove_offload - remove packet offload handler
498 * @po: packet offload declaration
499 *
500 * Remove a packet offload handler that was previously added to the kernel
501 * offload handlers by dev_add_offload(). The passed &offload_type is
502 * removed from the kernel lists and can be freed or reused once this
503 * function returns.
504 *
505 * This call sleeps to guarantee that no CPU is looking at the packet
506 * type after return.
507 */
508void dev_remove_offload(struct packet_offload *po)
509{
510 __dev_remove_offload(po);
511
512 synchronize_net();
513}
514EXPORT_SYMBOL(dev_remove_offload);
515
Linus Torvalds1da177e2005-04-16 15:20:36 -0700516/******************************************************************************
517
518 Device Boot-time Settings Routines
519
520*******************************************************************************/
521
522/* Boot time configuration table */
523static struct netdev_boot_setup dev_boot_setup[NETDEV_BOOT_SETUP_MAX];
524
525/**
526 * netdev_boot_setup_add - add new setup entry
527 * @name: name of the device
528 * @map: configured settings for the device
529 *
530 * Adds new setup entry to the dev_boot_setup list. The function
531 * returns 0 on error and 1 on success. This is a generic routine to
532 * all netdevices.
533 */
534static int netdev_boot_setup_add(char *name, struct ifmap *map)
535{
536 struct netdev_boot_setup *s;
537 int i;
538
539 s = dev_boot_setup;
540 for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++) {
541 if (s[i].name[0] == '\0' || s[i].name[0] == ' ') {
542 memset(s[i].name, 0, sizeof(s[i].name));
Wang Chen93b3cff2008-07-01 19:57:19 -0700543 strlcpy(s[i].name, name, IFNAMSIZ);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700544 memcpy(&s[i].map, map, sizeof(s[i].map));
545 break;
546 }
547 }
548
549 return i >= NETDEV_BOOT_SETUP_MAX ? 0 : 1;
550}
551
552/**
553 * netdev_boot_setup_check - check boot time settings
554 * @dev: the netdevice
555 *
556 * Check boot time settings for the device.
557 * The found settings are set for the device to be used
558 * later in the device probing.
559 * Returns 0 if no settings found, 1 if they are.
560 */
561int netdev_boot_setup_check(struct net_device *dev)
562{
563 struct netdev_boot_setup *s = dev_boot_setup;
564 int i;
565
566 for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++) {
567 if (s[i].name[0] != '\0' && s[i].name[0] != ' ' &&
Wang Chen93b3cff2008-07-01 19:57:19 -0700568 !strcmp(dev->name, s[i].name)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700569 dev->irq = s[i].map.irq;
570 dev->base_addr = s[i].map.base_addr;
571 dev->mem_start = s[i].map.mem_start;
572 dev->mem_end = s[i].map.mem_end;
573 return 1;
574 }
575 }
576 return 0;
577}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700578EXPORT_SYMBOL(netdev_boot_setup_check);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700579
580
581/**
582 * netdev_boot_base - get address from boot time settings
583 * @prefix: prefix for network device
584 * @unit: id for network device
585 *
586 * Check boot time settings for the base address of device.
587 * The found settings are set for the device to be used
588 * later in the device probing.
589 * Returns 0 if no settings found.
590 */
591unsigned long netdev_boot_base(const char *prefix, int unit)
592{
593 const struct netdev_boot_setup *s = dev_boot_setup;
594 char name[IFNAMSIZ];
595 int i;
596
597 sprintf(name, "%s%d", prefix, unit);
598
599 /*
600 * If device already registered then return base of 1
601 * to indicate not to probe for this interface
602 */
Eric W. Biederman881d9662007-09-17 11:56:21 -0700603 if (__dev_get_by_name(&init_net, name))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700604 return 1;
605
606 for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++)
607 if (!strcmp(name, s[i].name))
608 return s[i].map.base_addr;
609 return 0;
610}
611
612/*
613 * Saves at boot time configured settings for any netdevice.
614 */
615int __init netdev_boot_setup(char *str)
616{
617 int ints[5];
618 struct ifmap map;
619
620 str = get_options(str, ARRAY_SIZE(ints), ints);
621 if (!str || !*str)
622 return 0;
623
624 /* Save settings */
625 memset(&map, 0, sizeof(map));
626 if (ints[0] > 0)
627 map.irq = ints[1];
628 if (ints[0] > 1)
629 map.base_addr = ints[2];
630 if (ints[0] > 2)
631 map.mem_start = ints[3];
632 if (ints[0] > 3)
633 map.mem_end = ints[4];
634
635 /* Add new entry to the list */
636 return netdev_boot_setup_add(str, &map);
637}
638
639__setup("netdev=", netdev_boot_setup);
640
641/*******************************************************************************
642
643 Device Interface Subroutines
644
645*******************************************************************************/
646
647/**
648 * __dev_get_by_name - find a device by its name
Randy Dunlapc4ea43c2007-10-12 21:17:49 -0700649 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -0700650 * @name: name to find
651 *
652 * Find an interface by name. Must be called under RTNL semaphore
653 * or @dev_base_lock. If the name is found a pointer to the device
654 * is returned. If the name is not found then %NULL is returned. The
655 * reference counters are not incremented so the caller must be
656 * careful with locks.
657 */
658
Eric W. Biederman881d9662007-09-17 11:56:21 -0700659struct net_device *__dev_get_by_name(struct net *net, const char *name)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700660{
Eric Dumazet0bd8d532009-10-30 01:40:11 -0700661 struct net_device *dev;
662 struct hlist_head *head = dev_name_hash(net, name);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700663
Sasha Levinb67bfe02013-02-27 17:06:00 -0800664 hlist_for_each_entry(dev, head, name_hlist)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700665 if (!strncmp(dev->name, name, IFNAMSIZ))
666 return dev;
Eric Dumazet0bd8d532009-10-30 01:40:11 -0700667
Linus Torvalds1da177e2005-04-16 15:20:36 -0700668 return NULL;
669}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700670EXPORT_SYMBOL(__dev_get_by_name);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700671
672/**
Eric Dumazet72c95282009-10-30 07:11:27 +0000673 * dev_get_by_name_rcu - find a device by its name
674 * @net: the applicable net namespace
675 * @name: name to find
676 *
677 * Find an interface by name.
678 * If the name is found a pointer to the device is returned.
679 * If the name is not found then %NULL is returned.
680 * The reference counters are not incremented so the caller must be
681 * careful with locks. The caller must hold RCU lock.
682 */
683
684struct net_device *dev_get_by_name_rcu(struct net *net, const char *name)
685{
Eric Dumazet72c95282009-10-30 07:11:27 +0000686 struct net_device *dev;
687 struct hlist_head *head = dev_name_hash(net, name);
688
Sasha Levinb67bfe02013-02-27 17:06:00 -0800689 hlist_for_each_entry_rcu(dev, head, name_hlist)
Eric Dumazet72c95282009-10-30 07:11:27 +0000690 if (!strncmp(dev->name, name, IFNAMSIZ))
691 return dev;
692
693 return NULL;
694}
695EXPORT_SYMBOL(dev_get_by_name_rcu);
696
697/**
Linus Torvalds1da177e2005-04-16 15:20:36 -0700698 * dev_get_by_name - find a device by its name
Randy Dunlapc4ea43c2007-10-12 21:17:49 -0700699 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -0700700 * @name: name to find
701 *
702 * Find an interface by name. This can be called from any
703 * context and does its own locking. The returned handle has
704 * the usage count incremented and the caller must use dev_put() to
705 * release it when it is no longer needed. %NULL is returned if no
706 * matching device is found.
707 */
708
Eric W. Biederman881d9662007-09-17 11:56:21 -0700709struct net_device *dev_get_by_name(struct net *net, const char *name)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700710{
711 struct net_device *dev;
712
Eric Dumazet72c95282009-10-30 07:11:27 +0000713 rcu_read_lock();
714 dev = dev_get_by_name_rcu(net, name);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700715 if (dev)
716 dev_hold(dev);
Eric Dumazet72c95282009-10-30 07:11:27 +0000717 rcu_read_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700718 return dev;
719}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700720EXPORT_SYMBOL(dev_get_by_name);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700721
722/**
723 * __dev_get_by_index - find a device by its ifindex
Randy Dunlapc4ea43c2007-10-12 21:17:49 -0700724 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -0700725 * @ifindex: index of device
726 *
727 * Search for an interface by index. Returns %NULL if the device
728 * is not found or a pointer to the device. The device has not
729 * had its reference counter increased so the caller must be careful
730 * about locking. The caller must hold either the RTNL semaphore
731 * or @dev_base_lock.
732 */
733
Eric W. Biederman881d9662007-09-17 11:56:21 -0700734struct net_device *__dev_get_by_index(struct net *net, int ifindex)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700735{
Eric Dumazet0bd8d532009-10-30 01:40:11 -0700736 struct net_device *dev;
737 struct hlist_head *head = dev_index_hash(net, ifindex);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700738
Sasha Levinb67bfe02013-02-27 17:06:00 -0800739 hlist_for_each_entry(dev, head, index_hlist)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700740 if (dev->ifindex == ifindex)
741 return dev;
Eric Dumazet0bd8d532009-10-30 01:40:11 -0700742
Linus Torvalds1da177e2005-04-16 15:20:36 -0700743 return NULL;
744}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700745EXPORT_SYMBOL(__dev_get_by_index);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700746
Eric Dumazetfb699dfd2009-10-19 19:18:49 +0000747/**
748 * dev_get_by_index_rcu - find a device by its ifindex
749 * @net: the applicable net namespace
750 * @ifindex: index of device
751 *
752 * Search for an interface by index. Returns %NULL if the device
753 * is not found or a pointer to the device. The device has not
754 * had its reference counter increased so the caller must be careful
755 * about locking. The caller must hold RCU lock.
756 */
757
758struct net_device *dev_get_by_index_rcu(struct net *net, int ifindex)
759{
Eric Dumazetfb699dfd2009-10-19 19:18:49 +0000760 struct net_device *dev;
761 struct hlist_head *head = dev_index_hash(net, ifindex);
762
Sasha Levinb67bfe02013-02-27 17:06:00 -0800763 hlist_for_each_entry_rcu(dev, head, index_hlist)
Eric Dumazetfb699dfd2009-10-19 19:18:49 +0000764 if (dev->ifindex == ifindex)
765 return dev;
766
767 return NULL;
768}
769EXPORT_SYMBOL(dev_get_by_index_rcu);
770
Linus Torvalds1da177e2005-04-16 15:20:36 -0700771
772/**
773 * dev_get_by_index - find a device by its ifindex
Randy Dunlapc4ea43c2007-10-12 21:17:49 -0700774 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -0700775 * @ifindex: index of device
776 *
777 * Search for an interface by index. Returns NULL if the device
778 * is not found or a pointer to the device. The device returned has
779 * had a reference added and the pointer is safe until the user calls
780 * dev_put to indicate they have finished with it.
781 */
782
Eric W. Biederman881d9662007-09-17 11:56:21 -0700783struct net_device *dev_get_by_index(struct net *net, int ifindex)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700784{
785 struct net_device *dev;
786
Eric Dumazetfb699dfd2009-10-19 19:18:49 +0000787 rcu_read_lock();
788 dev = dev_get_by_index_rcu(net, ifindex);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700789 if (dev)
790 dev_hold(dev);
Eric Dumazetfb699dfd2009-10-19 19:18:49 +0000791 rcu_read_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700792 return dev;
793}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700794EXPORT_SYMBOL(dev_get_by_index);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700795
796/**
Eric Dumazet941666c2010-12-05 01:23:53 +0000797 * dev_getbyhwaddr_rcu - find a device by its hardware address
Randy Dunlapc4ea43c2007-10-12 21:17:49 -0700798 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -0700799 * @type: media type of device
800 * @ha: hardware address
801 *
802 * Search for an interface by MAC address. Returns NULL if the device
Eric Dumazetc5066532011-01-24 13:16:16 -0800803 * is not found or a pointer to the device.
804 * The caller must hold RCU or RTNL.
Eric Dumazet941666c2010-12-05 01:23:53 +0000805 * The returned device has not had its ref count increased
Linus Torvalds1da177e2005-04-16 15:20:36 -0700806 * and the caller must therefore be careful about locking
807 *
Linus Torvalds1da177e2005-04-16 15:20:36 -0700808 */
809
Eric Dumazet941666c2010-12-05 01:23:53 +0000810struct net_device *dev_getbyhwaddr_rcu(struct net *net, unsigned short type,
811 const char *ha)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700812{
813 struct net_device *dev;
814
Eric Dumazet941666c2010-12-05 01:23:53 +0000815 for_each_netdev_rcu(net, dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700816 if (dev->type == type &&
817 !memcmp(dev->dev_addr, ha, dev->addr_len))
Pavel Emelianov7562f872007-05-03 15:13:45 -0700818 return dev;
819
820 return NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700821}
Eric Dumazet941666c2010-12-05 01:23:53 +0000822EXPORT_SYMBOL(dev_getbyhwaddr_rcu);
Jochen Friedrichcf309e32005-09-22 04:44:55 -0300823
Eric W. Biederman881d9662007-09-17 11:56:21 -0700824struct net_device *__dev_getfirstbyhwtype(struct net *net, unsigned short type)
Patrick McHardy4e9cac22007-05-03 03:28:13 -0700825{
826 struct net_device *dev;
827
828 ASSERT_RTNL();
Eric W. Biederman881d9662007-09-17 11:56:21 -0700829 for_each_netdev(net, dev)
Patrick McHardy4e9cac22007-05-03 03:28:13 -0700830 if (dev->type == type)
Pavel Emelianov7562f872007-05-03 15:13:45 -0700831 return dev;
832
833 return NULL;
Patrick McHardy4e9cac22007-05-03 03:28:13 -0700834}
Patrick McHardy4e9cac22007-05-03 03:28:13 -0700835EXPORT_SYMBOL(__dev_getfirstbyhwtype);
836
Eric W. Biederman881d9662007-09-17 11:56:21 -0700837struct net_device *dev_getfirstbyhwtype(struct net *net, unsigned short type)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700838{
Eric Dumazet99fe3c32010-03-18 11:27:25 +0000839 struct net_device *dev, *ret = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700840
Eric Dumazet99fe3c32010-03-18 11:27:25 +0000841 rcu_read_lock();
842 for_each_netdev_rcu(net, dev)
843 if (dev->type == type) {
844 dev_hold(dev);
845 ret = dev;
846 break;
847 }
848 rcu_read_unlock();
849 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700850}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700851EXPORT_SYMBOL(dev_getfirstbyhwtype);
852
853/**
Eric Dumazetbb69ae02010-06-07 11:42:13 +0000854 * dev_get_by_flags_rcu - find any device with given flags
Randy Dunlapc4ea43c2007-10-12 21:17:49 -0700855 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -0700856 * @if_flags: IFF_* values
857 * @mask: bitmask of bits in if_flags to check
858 *
859 * Search for any interface with the given flags. Returns NULL if a device
Eric Dumazetbb69ae02010-06-07 11:42:13 +0000860 * is not found or a pointer to the device. Must be called inside
861 * rcu_read_lock(), and result refcount is unchanged.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700862 */
863
Eric Dumazetbb69ae02010-06-07 11:42:13 +0000864struct net_device *dev_get_by_flags_rcu(struct net *net, unsigned short if_flags,
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700865 unsigned short mask)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700866{
Pavel Emelianov7562f872007-05-03 15:13:45 -0700867 struct net_device *dev, *ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700868
Pavel Emelianov7562f872007-05-03 15:13:45 -0700869 ret = NULL;
Eric Dumazetc6d14c82009-11-04 05:43:23 -0800870 for_each_netdev_rcu(net, dev) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700871 if (((dev->flags ^ if_flags) & mask) == 0) {
Pavel Emelianov7562f872007-05-03 15:13:45 -0700872 ret = dev;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700873 break;
874 }
875 }
Pavel Emelianov7562f872007-05-03 15:13:45 -0700876 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700877}
Eric Dumazetbb69ae02010-06-07 11:42:13 +0000878EXPORT_SYMBOL(dev_get_by_flags_rcu);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700879
880/**
881 * dev_valid_name - check if name is okay for network device
882 * @name: name string
883 *
884 * Network device names need to be valid file names to
David S. Millerc7fa9d12006-08-15 16:34:13 -0700885 * to allow sysfs to work. We also disallow any kind of
886 * whitespace.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700887 */
David S. Miller95f050b2012-03-06 16:12:15 -0500888bool dev_valid_name(const char *name)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700889{
David S. Millerc7fa9d12006-08-15 16:34:13 -0700890 if (*name == '\0')
David S. Miller95f050b2012-03-06 16:12:15 -0500891 return false;
Stephen Hemmingerb6fe17d2006-08-29 17:06:13 -0700892 if (strlen(name) >= IFNAMSIZ)
David S. Miller95f050b2012-03-06 16:12:15 -0500893 return false;
David S. Millerc7fa9d12006-08-15 16:34:13 -0700894 if (!strcmp(name, ".") || !strcmp(name, ".."))
David S. Miller95f050b2012-03-06 16:12:15 -0500895 return false;
David S. Millerc7fa9d12006-08-15 16:34:13 -0700896
897 while (*name) {
898 if (*name == '/' || isspace(*name))
David S. Miller95f050b2012-03-06 16:12:15 -0500899 return false;
David S. Millerc7fa9d12006-08-15 16:34:13 -0700900 name++;
901 }
David S. Miller95f050b2012-03-06 16:12:15 -0500902 return true;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700903}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700904EXPORT_SYMBOL(dev_valid_name);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700905
906/**
Eric W. Biedermanb267b172007-09-12 13:48:45 +0200907 * __dev_alloc_name - allocate a name for a device
908 * @net: network namespace to allocate the device name in
Linus Torvalds1da177e2005-04-16 15:20:36 -0700909 * @name: name format string
Eric W. Biedermanb267b172007-09-12 13:48:45 +0200910 * @buf: scratch buffer and result name string
Linus Torvalds1da177e2005-04-16 15:20:36 -0700911 *
912 * Passed a format string - eg "lt%d" it will try and find a suitable
Stephen Hemminger3041a062006-05-26 13:25:24 -0700913 * id. It scans list of devices to build up a free map, then chooses
914 * the first empty slot. The caller must hold the dev_base or rtnl lock
915 * while allocating the name and adding the device in order to avoid
916 * duplicates.
917 * Limited to bits_per_byte * page size devices (ie 32K on most platforms).
918 * Returns the number of the unit assigned or a negative errno code.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700919 */
920
Eric W. Biedermanb267b172007-09-12 13:48:45 +0200921static int __dev_alloc_name(struct net *net, const char *name, char *buf)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700922{
923 int i = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700924 const char *p;
925 const int max_netdevices = 8*PAGE_SIZE;
Stephen Hemmingercfcabdc2007-10-09 01:59:42 -0700926 unsigned long *inuse;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700927 struct net_device *d;
928
929 p = strnchr(name, IFNAMSIZ-1, '%');
930 if (p) {
931 /*
932 * Verify the string as this thing may have come from
933 * the user. There must be either one "%d" and no other "%"
934 * characters.
935 */
936 if (p[1] != 'd' || strchr(p + 2, '%'))
937 return -EINVAL;
938
939 /* Use one page as a bit array of possible slots */
Stephen Hemmingercfcabdc2007-10-09 01:59:42 -0700940 inuse = (unsigned long *) get_zeroed_page(GFP_ATOMIC);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700941 if (!inuse)
942 return -ENOMEM;
943
Eric W. Biederman881d9662007-09-17 11:56:21 -0700944 for_each_netdev(net, d) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700945 if (!sscanf(d->name, name, &i))
946 continue;
947 if (i < 0 || i >= max_netdevices)
948 continue;
949
950 /* avoid cases where sscanf is not exact inverse of printf */
Eric W. Biedermanb267b172007-09-12 13:48:45 +0200951 snprintf(buf, IFNAMSIZ, name, i);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700952 if (!strncmp(buf, d->name, IFNAMSIZ))
953 set_bit(i, inuse);
954 }
955
956 i = find_first_zero_bit(inuse, max_netdevices);
957 free_page((unsigned long) inuse);
958 }
959
Octavian Purdilad9031022009-11-18 02:36:59 +0000960 if (buf != name)
961 snprintf(buf, IFNAMSIZ, name, i);
Eric W. Biedermanb267b172007-09-12 13:48:45 +0200962 if (!__dev_get_by_name(net, buf))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700963 return i;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700964
965 /* It is possible to run out of possible slots
966 * when the name is long and there isn't enough space left
967 * for the digits, or if all bits are used.
968 */
969 return -ENFILE;
970}
971
Eric W. Biedermanb267b172007-09-12 13:48:45 +0200972/**
973 * dev_alloc_name - allocate a name for a device
974 * @dev: device
975 * @name: name format string
976 *
977 * Passed a format string - eg "lt%d" it will try and find a suitable
978 * id. It scans list of devices to build up a free map, then chooses
979 * the first empty slot. The caller must hold the dev_base or rtnl lock
980 * while allocating the name and adding the device in order to avoid
981 * duplicates.
982 * Limited to bits_per_byte * page size devices (ie 32K on most platforms).
983 * Returns the number of the unit assigned or a negative errno code.
984 */
985
986int dev_alloc_name(struct net_device *dev, const char *name)
987{
988 char buf[IFNAMSIZ];
989 struct net *net;
990 int ret;
991
YOSHIFUJI Hideakic346dca2008-03-25 21:47:49 +0900992 BUG_ON(!dev_net(dev));
993 net = dev_net(dev);
Eric W. Biedermanb267b172007-09-12 13:48:45 +0200994 ret = __dev_alloc_name(net, name, buf);
995 if (ret >= 0)
996 strlcpy(dev->name, buf, IFNAMSIZ);
997 return ret;
998}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700999EXPORT_SYMBOL(dev_alloc_name);
Eric W. Biedermanb267b172007-09-12 13:48:45 +02001000
Gao feng828de4f2012-09-13 20:58:27 +00001001static int dev_alloc_name_ns(struct net *net,
1002 struct net_device *dev,
1003 const char *name)
Octavian Purdilad9031022009-11-18 02:36:59 +00001004{
Gao feng828de4f2012-09-13 20:58:27 +00001005 char buf[IFNAMSIZ];
1006 int ret;
Daniel Lezcano8ce6cebc2010-05-19 10:12:19 +00001007
Gao feng828de4f2012-09-13 20:58:27 +00001008 ret = __dev_alloc_name(net, name, buf);
1009 if (ret >= 0)
1010 strlcpy(dev->name, buf, IFNAMSIZ);
1011 return ret;
1012}
1013
1014static int dev_get_valid_name(struct net *net,
1015 struct net_device *dev,
1016 const char *name)
1017{
1018 BUG_ON(!net);
Daniel Lezcano8ce6cebc2010-05-19 10:12:19 +00001019
Octavian Purdilad9031022009-11-18 02:36:59 +00001020 if (!dev_valid_name(name))
1021 return -EINVAL;
1022
Jiri Pirko1c5cae82011-04-30 01:21:32 +00001023 if (strchr(name, '%'))
Gao feng828de4f2012-09-13 20:58:27 +00001024 return dev_alloc_name_ns(net, dev, name);
Octavian Purdilad9031022009-11-18 02:36:59 +00001025 else if (__dev_get_by_name(net, name))
1026 return -EEXIST;
Daniel Lezcano8ce6cebc2010-05-19 10:12:19 +00001027 else if (dev->name != name)
1028 strlcpy(dev->name, name, IFNAMSIZ);
Octavian Purdilad9031022009-11-18 02:36:59 +00001029
1030 return 0;
1031}
Linus Torvalds1da177e2005-04-16 15:20:36 -07001032
1033/**
1034 * dev_change_name - change name of a device
1035 * @dev: device
1036 * @newname: name (or format string) must be at least IFNAMSIZ
1037 *
1038 * Change name of a device, can pass format strings "eth%d".
1039 * for wildcarding.
1040 */
Stephen Hemmingercf04a4c72008-09-30 02:22:14 -07001041int dev_change_name(struct net_device *dev, const char *newname)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001042{
Herbert Xufcc5a032007-07-30 17:03:38 -07001043 char oldname[IFNAMSIZ];
Linus Torvalds1da177e2005-04-16 15:20:36 -07001044 int err = 0;
Herbert Xufcc5a032007-07-30 17:03:38 -07001045 int ret;
Eric W. Biederman881d9662007-09-17 11:56:21 -07001046 struct net *net;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001047
1048 ASSERT_RTNL();
YOSHIFUJI Hideakic346dca2008-03-25 21:47:49 +09001049 BUG_ON(!dev_net(dev));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001050
YOSHIFUJI Hideakic346dca2008-03-25 21:47:49 +09001051 net = dev_net(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001052 if (dev->flags & IFF_UP)
1053 return -EBUSY;
1054
Eric Dumazet30e6c9f2012-12-20 17:25:08 +00001055 write_seqcount_begin(&devnet_rename_seq);
Brian Haleyc91f6df2012-11-26 05:21:08 +00001056
1057 if (strncmp(newname, dev->name, IFNAMSIZ) == 0) {
Eric Dumazet30e6c9f2012-12-20 17:25:08 +00001058 write_seqcount_end(&devnet_rename_seq);
Stephen Hemmingerc8d90dc2007-10-26 03:53:42 -07001059 return 0;
Brian Haleyc91f6df2012-11-26 05:21:08 +00001060 }
Stephen Hemmingerc8d90dc2007-10-26 03:53:42 -07001061
Herbert Xufcc5a032007-07-30 17:03:38 -07001062 memcpy(oldname, dev->name, IFNAMSIZ);
1063
Gao feng828de4f2012-09-13 20:58:27 +00001064 err = dev_get_valid_name(net, dev, newname);
Brian Haleyc91f6df2012-11-26 05:21:08 +00001065 if (err < 0) {
Eric Dumazet30e6c9f2012-12-20 17:25:08 +00001066 write_seqcount_end(&devnet_rename_seq);
Octavian Purdilad9031022009-11-18 02:36:59 +00001067 return err;
Brian Haleyc91f6df2012-11-26 05:21:08 +00001068 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001069
Herbert Xufcc5a032007-07-30 17:03:38 -07001070rollback:
Eric W. Biedermana1b3f592010-05-04 17:36:49 -07001071 ret = device_rename(&dev->dev, dev->name);
1072 if (ret) {
1073 memcpy(dev->name, oldname, IFNAMSIZ);
Eric Dumazet30e6c9f2012-12-20 17:25:08 +00001074 write_seqcount_end(&devnet_rename_seq);
Eric W. Biedermana1b3f592010-05-04 17:36:49 -07001075 return ret;
Stephen Hemmingerdcc99772008-05-14 22:33:38 -07001076 }
Herbert Xu7f988ea2007-07-30 16:35:46 -07001077
Eric Dumazet30e6c9f2012-12-20 17:25:08 +00001078 write_seqcount_end(&devnet_rename_seq);
Brian Haleyc91f6df2012-11-26 05:21:08 +00001079
Herbert Xu7f988ea2007-07-30 16:35:46 -07001080 write_lock_bh(&dev_base_lock);
Eric Dumazet372b2312011-05-17 13:56:59 -04001081 hlist_del_rcu(&dev->name_hlist);
Eric Dumazet72c95282009-10-30 07:11:27 +00001082 write_unlock_bh(&dev_base_lock);
1083
1084 synchronize_rcu();
1085
1086 write_lock_bh(&dev_base_lock);
1087 hlist_add_head_rcu(&dev->name_hlist, dev_name_hash(net, dev->name));
Herbert Xu7f988ea2007-07-30 16:35:46 -07001088 write_unlock_bh(&dev_base_lock);
1089
Pavel Emelyanov056925a2007-09-16 15:42:43 -07001090 ret = call_netdevice_notifiers(NETDEV_CHANGENAME, dev);
Herbert Xufcc5a032007-07-30 17:03:38 -07001091 ret = notifier_to_errno(ret);
1092
1093 if (ret) {
Eric Dumazet91e9c07b2009-11-15 23:30:24 +00001094 /* err >= 0 after dev_alloc_name() or stores the first errno */
1095 if (err >= 0) {
Herbert Xufcc5a032007-07-30 17:03:38 -07001096 err = ret;
Eric Dumazet30e6c9f2012-12-20 17:25:08 +00001097 write_seqcount_begin(&devnet_rename_seq);
Herbert Xufcc5a032007-07-30 17:03:38 -07001098 memcpy(dev->name, oldname, IFNAMSIZ);
1099 goto rollback;
Eric Dumazet91e9c07b2009-11-15 23:30:24 +00001100 } else {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00001101 pr_err("%s: name change rollback failed: %d\n",
Eric Dumazet91e9c07b2009-11-15 23:30:24 +00001102 dev->name, ret);
Herbert Xufcc5a032007-07-30 17:03:38 -07001103 }
1104 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001105
1106 return err;
1107}
1108
1109/**
Stephen Hemminger0b815a12008-09-22 21:28:11 -07001110 * dev_set_alias - change ifalias of a device
1111 * @dev: device
1112 * @alias: name up to IFALIASZ
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07001113 * @len: limit of bytes to copy from info
Stephen Hemminger0b815a12008-09-22 21:28:11 -07001114 *
1115 * Set ifalias for a device,
1116 */
1117int dev_set_alias(struct net_device *dev, const char *alias, size_t len)
1118{
Alexey Khoroshilov7364e442012-08-08 00:33:25 +00001119 char *new_ifalias;
1120
Stephen Hemminger0b815a12008-09-22 21:28:11 -07001121 ASSERT_RTNL();
1122
1123 if (len >= IFALIASZ)
1124 return -EINVAL;
1125
Oliver Hartkopp96ca4a22008-09-23 21:23:19 -07001126 if (!len) {
Sachin Kamat388dfc22012-11-20 00:57:04 +00001127 kfree(dev->ifalias);
1128 dev->ifalias = NULL;
Oliver Hartkopp96ca4a22008-09-23 21:23:19 -07001129 return 0;
1130 }
1131
Alexey Khoroshilov7364e442012-08-08 00:33:25 +00001132 new_ifalias = krealloc(dev->ifalias, len + 1, GFP_KERNEL);
1133 if (!new_ifalias)
Stephen Hemminger0b815a12008-09-22 21:28:11 -07001134 return -ENOMEM;
Alexey Khoroshilov7364e442012-08-08 00:33:25 +00001135 dev->ifalias = new_ifalias;
Stephen Hemminger0b815a12008-09-22 21:28:11 -07001136
1137 strlcpy(dev->ifalias, alias, len+1);
1138 return len;
1139}
1140
1141
1142/**
Stephen Hemminger3041a062006-05-26 13:25:24 -07001143 * netdev_features_change - device changes features
Stephen Hemmingerd8a33ac2005-05-29 14:13:47 -07001144 * @dev: device to cause notification
1145 *
1146 * Called to indicate a device has changed features.
1147 */
1148void netdev_features_change(struct net_device *dev)
1149{
Pavel Emelyanov056925a2007-09-16 15:42:43 -07001150 call_netdevice_notifiers(NETDEV_FEAT_CHANGE, dev);
Stephen Hemmingerd8a33ac2005-05-29 14:13:47 -07001151}
1152EXPORT_SYMBOL(netdev_features_change);
1153
1154/**
Linus Torvalds1da177e2005-04-16 15:20:36 -07001155 * netdev_state_change - device changes state
1156 * @dev: device to cause notification
1157 *
1158 * Called to indicate a device has changed state. This function calls
1159 * the notifier chains for netdev_chain and sends a NEWLINK message
1160 * to the routing socket.
1161 */
1162void netdev_state_change(struct net_device *dev)
1163{
1164 if (dev->flags & IFF_UP) {
Pavel Emelyanov056925a2007-09-16 15:42:43 -07001165 call_netdevice_notifiers(NETDEV_CHANGE, dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001166 rtmsg_ifinfo(RTM_NEWLINK, dev, 0);
1167 }
1168}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001169EXPORT_SYMBOL(netdev_state_change);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001170
Amerigo Wangee89bab2012-08-09 22:14:56 +00001171/**
1172 * netdev_notify_peers - notify network peers about existence of @dev
1173 * @dev: network device
1174 *
1175 * Generate traffic such that interested network peers are aware of
1176 * @dev, such as by generating a gratuitous ARP. This may be used when
1177 * a device wants to inform the rest of the network about some sort of
1178 * reconfiguration such as a failover event or virtual machine
1179 * migration.
1180 */
1181void netdev_notify_peers(struct net_device *dev)
Or Gerlitzc1da4ac2008-06-13 18:12:00 -07001182{
Amerigo Wangee89bab2012-08-09 22:14:56 +00001183 rtnl_lock();
1184 call_netdevice_notifiers(NETDEV_NOTIFY_PEERS, dev);
1185 rtnl_unlock();
Or Gerlitzc1da4ac2008-06-13 18:12:00 -07001186}
Amerigo Wangee89bab2012-08-09 22:14:56 +00001187EXPORT_SYMBOL(netdev_notify_peers);
Or Gerlitzc1da4ac2008-06-13 18:12:00 -07001188
Patrick McHardybd380812010-02-26 06:34:53 +00001189static int __dev_open(struct net_device *dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001190{
Stephen Hemmingerd3147742008-11-19 21:32:24 -08001191 const struct net_device_ops *ops = dev->netdev_ops;
Johannes Berg3b8bcfd2009-05-30 01:39:53 +02001192 int ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001193
Ben Hutchingse46b66b2008-05-08 02:53:17 -07001194 ASSERT_RTNL();
1195
Linus Torvalds1da177e2005-04-16 15:20:36 -07001196 if (!netif_device_present(dev))
1197 return -ENODEV;
1198
Neil Hormanca99ca12013-02-05 08:05:43 +00001199 /* Block netpoll from trying to do any rx path servicing.
1200 * If we don't do this there is a chance ndo_poll_controller
1201 * or ndo_poll may be running while we open the device
1202 */
1203 ret = netpoll_rx_disable(dev);
1204 if (ret)
1205 return ret;
1206
Johannes Berg3b8bcfd2009-05-30 01:39:53 +02001207 ret = call_netdevice_notifiers(NETDEV_PRE_UP, dev);
1208 ret = notifier_to_errno(ret);
1209 if (ret)
1210 return ret;
1211
Linus Torvalds1da177e2005-04-16 15:20:36 -07001212 set_bit(__LINK_STATE_START, &dev->state);
Jeff Garzikbada3392007-10-23 20:19:37 -07001213
Stephen Hemmingerd3147742008-11-19 21:32:24 -08001214 if (ops->ndo_validate_addr)
1215 ret = ops->ndo_validate_addr(dev);
Jeff Garzikbada3392007-10-23 20:19:37 -07001216
Stephen Hemmingerd3147742008-11-19 21:32:24 -08001217 if (!ret && ops->ndo_open)
1218 ret = ops->ndo_open(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001219
Neil Hormanca99ca12013-02-05 08:05:43 +00001220 netpoll_rx_enable(dev);
1221
Jeff Garzikbada3392007-10-23 20:19:37 -07001222 if (ret)
1223 clear_bit(__LINK_STATE_START, &dev->state);
1224 else {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001225 dev->flags |= IFF_UP;
David S. Millerb4bd07c2009-02-06 22:06:43 -08001226 net_dmaengine_get();
Patrick McHardy4417da62007-06-27 01:28:10 -07001227 dev_set_rx_mode(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001228 dev_activate(dev);
Theodore Ts'o7bf23572012-07-04 21:23:25 -04001229 add_device_randomness(dev->dev_addr, dev->addr_len);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001230 }
Jeff Garzikbada3392007-10-23 20:19:37 -07001231
Linus Torvalds1da177e2005-04-16 15:20:36 -07001232 return ret;
1233}
Patrick McHardybd380812010-02-26 06:34:53 +00001234
1235/**
1236 * dev_open - prepare an interface for use.
1237 * @dev: device to open
1238 *
1239 * Takes a device from down to up state. The device's private open
1240 * function is invoked and then the multicast lists are loaded. Finally
1241 * the device is moved into the up state and a %NETDEV_UP message is
1242 * sent to the netdev notifier chain.
1243 *
1244 * Calling this function on an active interface is a nop. On a failure
1245 * a negative errno code is returned.
1246 */
1247int dev_open(struct net_device *dev)
1248{
1249 int ret;
1250
Patrick McHardybd380812010-02-26 06:34:53 +00001251 if (dev->flags & IFF_UP)
1252 return 0;
1253
Patrick McHardybd380812010-02-26 06:34:53 +00001254 ret = __dev_open(dev);
1255 if (ret < 0)
1256 return ret;
1257
Patrick McHardybd380812010-02-26 06:34:53 +00001258 rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP|IFF_RUNNING);
1259 call_netdevice_notifiers(NETDEV_UP, dev);
1260
1261 return ret;
1262}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001263EXPORT_SYMBOL(dev_open);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001264
Octavian Purdila44345722010-12-13 12:44:07 +00001265static int __dev_close_many(struct list_head *head)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001266{
Octavian Purdila44345722010-12-13 12:44:07 +00001267 struct net_device *dev;
Patrick McHardybd380812010-02-26 06:34:53 +00001268
Ben Hutchingse46b66b2008-05-08 02:53:17 -07001269 ASSERT_RTNL();
David S. Miller9d5010d2007-09-12 14:33:25 +02001270 might_sleep();
1271
Octavian Purdila44345722010-12-13 12:44:07 +00001272 list_for_each_entry(dev, head, unreg_list) {
Octavian Purdila44345722010-12-13 12:44:07 +00001273 call_netdevice_notifiers(NETDEV_GOING_DOWN, dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001274
Octavian Purdila44345722010-12-13 12:44:07 +00001275 clear_bit(__LINK_STATE_START, &dev->state);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001276
Octavian Purdila44345722010-12-13 12:44:07 +00001277 /* Synchronize to scheduled poll. We cannot touch poll list, it
1278 * can be even on different cpu. So just clear netif_running().
1279 *
1280 * dev->stop() will invoke napi_disable() on all of it's
1281 * napi_struct instances on this device.
1282 */
1283 smp_mb__after_clear_bit(); /* Commit netif_running(). */
1284 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001285
Octavian Purdila44345722010-12-13 12:44:07 +00001286 dev_deactivate_many(head);
1287
1288 list_for_each_entry(dev, head, unreg_list) {
1289 const struct net_device_ops *ops = dev->netdev_ops;
1290
1291 /*
1292 * Call the device specific close. This cannot fail.
1293 * Only if device is UP
1294 *
1295 * We allow it to be called even after a DETACH hot-plug
1296 * event.
1297 */
1298 if (ops->ndo_stop)
1299 ops->ndo_stop(dev);
1300
Octavian Purdila44345722010-12-13 12:44:07 +00001301 dev->flags &= ~IFF_UP;
Octavian Purdila44345722010-12-13 12:44:07 +00001302 net_dmaengine_put();
1303 }
1304
1305 return 0;
1306}
1307
1308static int __dev_close(struct net_device *dev)
1309{
Linus Torvaldsf87e6f42011-02-17 22:54:38 +00001310 int retval;
Octavian Purdila44345722010-12-13 12:44:07 +00001311 LIST_HEAD(single);
1312
Neil Hormanca99ca12013-02-05 08:05:43 +00001313 /* Temporarily disable netpoll until the interface is down */
1314 retval = netpoll_rx_disable(dev);
1315 if (retval)
1316 return retval;
1317
Octavian Purdila44345722010-12-13 12:44:07 +00001318 list_add(&dev->unreg_list, &single);
Linus Torvaldsf87e6f42011-02-17 22:54:38 +00001319 retval = __dev_close_many(&single);
1320 list_del(&single);
Neil Hormanca99ca12013-02-05 08:05:43 +00001321
1322 netpoll_rx_enable(dev);
Linus Torvaldsf87e6f42011-02-17 22:54:38 +00001323 return retval;
Octavian Purdila44345722010-12-13 12:44:07 +00001324}
1325
Eric Dumazet3fbd8752011-01-19 21:23:22 +00001326static int dev_close_many(struct list_head *head)
Octavian Purdila44345722010-12-13 12:44:07 +00001327{
1328 struct net_device *dev, *tmp;
1329 LIST_HEAD(tmp_list);
1330
1331 list_for_each_entry_safe(dev, tmp, head, unreg_list)
1332 if (!(dev->flags & IFF_UP))
1333 list_move(&dev->unreg_list, &tmp_list);
1334
1335 __dev_close_many(head);
Matti Linnanvuorid8b2a4d2008-02-12 23:10:11 -08001336
Octavian Purdila44345722010-12-13 12:44:07 +00001337 list_for_each_entry(dev, head, unreg_list) {
1338 rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP|IFF_RUNNING);
1339 call_netdevice_notifiers(NETDEV_DOWN, dev);
1340 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001341
Octavian Purdila44345722010-12-13 12:44:07 +00001342 /* rollback_registered_many needs the complete original list */
1343 list_splice(&tmp_list, head);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001344 return 0;
1345}
Patrick McHardybd380812010-02-26 06:34:53 +00001346
1347/**
1348 * dev_close - shutdown an interface.
1349 * @dev: device to shutdown
1350 *
1351 * This function moves an active device into down state. A
1352 * %NETDEV_GOING_DOWN is sent to the netdev notifier chain. The device
1353 * is then deactivated and finally a %NETDEV_DOWN is sent to the notifier
1354 * chain.
1355 */
1356int dev_close(struct net_device *dev)
1357{
Neil Hormanca99ca12013-02-05 08:05:43 +00001358 int ret = 0;
Eric Dumazete14a5992011-05-10 12:26:06 -07001359 if (dev->flags & IFF_UP) {
1360 LIST_HEAD(single);
Patrick McHardybd380812010-02-26 06:34:53 +00001361
Neil Hormanca99ca12013-02-05 08:05:43 +00001362 /* Block netpoll rx while the interface is going down */
1363 ret = netpoll_rx_disable(dev);
1364 if (ret)
1365 return ret;
1366
Eric Dumazete14a5992011-05-10 12:26:06 -07001367 list_add(&dev->unreg_list, &single);
1368 dev_close_many(&single);
1369 list_del(&single);
Neil Hormanca99ca12013-02-05 08:05:43 +00001370
1371 netpoll_rx_enable(dev);
Eric Dumazete14a5992011-05-10 12:26:06 -07001372 }
Neil Hormanca99ca12013-02-05 08:05:43 +00001373 return ret;
Patrick McHardybd380812010-02-26 06:34:53 +00001374}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001375EXPORT_SYMBOL(dev_close);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001376
1377
Ben Hutchings0187bdf2008-06-19 16:15:47 -07001378/**
1379 * dev_disable_lro - disable Large Receive Offload on a device
1380 * @dev: device
1381 *
1382 * Disable Large Receive Offload (LRO) on a net device. Must be
1383 * called under RTNL. This is needed if received packets may be
1384 * forwarded to another interface.
1385 */
1386void dev_disable_lro(struct net_device *dev)
1387{
Neil Hormanf11970e2011-05-24 08:31:09 +00001388 /*
1389 * If we're trying to disable lro on a vlan device
1390 * use the underlying physical device instead
1391 */
1392 if (is_vlan_dev(dev))
1393 dev = vlan_dev_real_dev(dev);
1394
Michał Mirosławbc5787c62011-11-15 15:29:55 +00001395 dev->wanted_features &= ~NETIF_F_LRO;
1396 netdev_update_features(dev);
Michał Mirosław27660512011-03-18 16:56:34 +00001397
Michał Mirosław22d59692011-04-21 12:42:15 +00001398 if (unlikely(dev->features & NETIF_F_LRO))
1399 netdev_WARN(dev, "failed to disable LRO!\n");
Ben Hutchings0187bdf2008-06-19 16:15:47 -07001400}
1401EXPORT_SYMBOL(dev_disable_lro);
1402
1403
Eric W. Biederman881d9662007-09-17 11:56:21 -07001404static int dev_boot_phase = 1;
1405
Linus Torvalds1da177e2005-04-16 15:20:36 -07001406/**
1407 * register_netdevice_notifier - register a network notifier block
1408 * @nb: notifier
1409 *
1410 * Register a notifier to be called when network device events occur.
1411 * The notifier passed is linked into the kernel structures and must
1412 * not be reused until it has been unregistered. A negative errno code
1413 * is returned on a failure.
1414 *
1415 * When registered all registration and up events are replayed
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09001416 * to the new notifier to allow device to have a race free
Linus Torvalds1da177e2005-04-16 15:20:36 -07001417 * view of the network device list.
1418 */
1419
1420int register_netdevice_notifier(struct notifier_block *nb)
1421{
1422 struct net_device *dev;
Herbert Xufcc5a032007-07-30 17:03:38 -07001423 struct net_device *last;
Eric W. Biederman881d9662007-09-17 11:56:21 -07001424 struct net *net;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001425 int err;
1426
1427 rtnl_lock();
Alan Sternf07d5b92006-05-09 15:23:03 -07001428 err = raw_notifier_chain_register(&netdev_chain, nb);
Herbert Xufcc5a032007-07-30 17:03:38 -07001429 if (err)
1430 goto unlock;
Eric W. Biederman881d9662007-09-17 11:56:21 -07001431 if (dev_boot_phase)
1432 goto unlock;
1433 for_each_net(net) {
1434 for_each_netdev(net, dev) {
1435 err = nb->notifier_call(nb, NETDEV_REGISTER, dev);
1436 err = notifier_to_errno(err);
1437 if (err)
1438 goto rollback;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001439
Eric W. Biederman881d9662007-09-17 11:56:21 -07001440 if (!(dev->flags & IFF_UP))
1441 continue;
Herbert Xufcc5a032007-07-30 17:03:38 -07001442
Eric W. Biederman881d9662007-09-17 11:56:21 -07001443 nb->notifier_call(nb, NETDEV_UP, dev);
1444 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001445 }
Herbert Xufcc5a032007-07-30 17:03:38 -07001446
1447unlock:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001448 rtnl_unlock();
1449 return err;
Herbert Xufcc5a032007-07-30 17:03:38 -07001450
1451rollback:
1452 last = dev;
Eric W. Biederman881d9662007-09-17 11:56:21 -07001453 for_each_net(net) {
1454 for_each_netdev(net, dev) {
1455 if (dev == last)
RongQing.Li8f891482011-11-30 23:43:07 -05001456 goto outroll;
Herbert Xufcc5a032007-07-30 17:03:38 -07001457
Eric W. Biederman881d9662007-09-17 11:56:21 -07001458 if (dev->flags & IFF_UP) {
1459 nb->notifier_call(nb, NETDEV_GOING_DOWN, dev);
1460 nb->notifier_call(nb, NETDEV_DOWN, dev);
1461 }
1462 nb->notifier_call(nb, NETDEV_UNREGISTER, dev);
Herbert Xufcc5a032007-07-30 17:03:38 -07001463 }
Herbert Xufcc5a032007-07-30 17:03:38 -07001464 }
Pavel Emelyanovc67625a2007-11-14 15:53:16 -08001465
RongQing.Li8f891482011-11-30 23:43:07 -05001466outroll:
Pavel Emelyanovc67625a2007-11-14 15:53:16 -08001467 raw_notifier_chain_unregister(&netdev_chain, nb);
Herbert Xufcc5a032007-07-30 17:03:38 -07001468 goto unlock;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001469}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001470EXPORT_SYMBOL(register_netdevice_notifier);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001471
1472/**
1473 * unregister_netdevice_notifier - unregister a network notifier block
1474 * @nb: notifier
1475 *
1476 * Unregister a notifier previously registered by
1477 * register_netdevice_notifier(). The notifier is unlinked into the
1478 * kernel structures and may then be reused. A negative errno code
1479 * is returned on a failure.
Eric W. Biederman7d3d43d2012-04-06 15:33:35 +00001480 *
1481 * After unregistering unregister and down device events are synthesized
1482 * for all devices on the device list to the removed notifier to remove
1483 * the need for special case cleanup code.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001484 */
1485
1486int unregister_netdevice_notifier(struct notifier_block *nb)
1487{
Eric W. Biederman7d3d43d2012-04-06 15:33:35 +00001488 struct net_device *dev;
1489 struct net *net;
Herbert Xu9f514952006-03-25 01:24:25 -08001490 int err;
1491
1492 rtnl_lock();
Alan Sternf07d5b92006-05-09 15:23:03 -07001493 err = raw_notifier_chain_unregister(&netdev_chain, nb);
Eric W. Biederman7d3d43d2012-04-06 15:33:35 +00001494 if (err)
1495 goto unlock;
1496
1497 for_each_net(net) {
1498 for_each_netdev(net, dev) {
1499 if (dev->flags & IFF_UP) {
1500 nb->notifier_call(nb, NETDEV_GOING_DOWN, dev);
1501 nb->notifier_call(nb, NETDEV_DOWN, dev);
1502 }
1503 nb->notifier_call(nb, NETDEV_UNREGISTER, dev);
Eric W. Biederman7d3d43d2012-04-06 15:33:35 +00001504 }
1505 }
1506unlock:
Herbert Xu9f514952006-03-25 01:24:25 -08001507 rtnl_unlock();
1508 return err;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001509}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001510EXPORT_SYMBOL(unregister_netdevice_notifier);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001511
1512/**
1513 * call_netdevice_notifiers - call all network notifier blocks
1514 * @val: value passed unmodified to notifier function
Randy Dunlapc4ea43c2007-10-12 21:17:49 -07001515 * @dev: net_device pointer passed unmodified to notifier function
Linus Torvalds1da177e2005-04-16 15:20:36 -07001516 *
1517 * Call all network notifier blocks. Parameters and return value
Alan Sternf07d5b92006-05-09 15:23:03 -07001518 * are as for raw_notifier_call_chain().
Linus Torvalds1da177e2005-04-16 15:20:36 -07001519 */
1520
Eric W. Biedermanad7379d2007-09-16 15:33:32 -07001521int call_netdevice_notifiers(unsigned long val, struct net_device *dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001522{
Jiri Pirkoab930472010-04-20 01:45:37 -07001523 ASSERT_RTNL();
Eric W. Biedermanad7379d2007-09-16 15:33:32 -07001524 return raw_notifier_call_chain(&netdev_chain, val, dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001525}
stephen hemmingeredf947f2011-03-24 13:24:01 +00001526EXPORT_SYMBOL(call_netdevice_notifiers);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001527
Ingo Molnarc5905af2012-02-24 08:31:31 +01001528static struct static_key netstamp_needed __read_mostly;
Eric Dumazetb90e5792011-11-28 11:16:50 +00001529#ifdef HAVE_JUMP_LABEL
Ingo Molnarc5905af2012-02-24 08:31:31 +01001530/* We are not allowed to call static_key_slow_dec() from irq context
Eric Dumazetb90e5792011-11-28 11:16:50 +00001531 * If net_disable_timestamp() is called from irq context, defer the
Ingo Molnarc5905af2012-02-24 08:31:31 +01001532 * static_key_slow_dec() calls.
Eric Dumazetb90e5792011-11-28 11:16:50 +00001533 */
1534static atomic_t netstamp_needed_deferred;
1535#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07001536
1537void net_enable_timestamp(void)
1538{
Eric Dumazetb90e5792011-11-28 11:16:50 +00001539#ifdef HAVE_JUMP_LABEL
1540 int deferred = atomic_xchg(&netstamp_needed_deferred, 0);
1541
1542 if (deferred) {
1543 while (--deferred)
Ingo Molnarc5905af2012-02-24 08:31:31 +01001544 static_key_slow_dec(&netstamp_needed);
Eric Dumazetb90e5792011-11-28 11:16:50 +00001545 return;
1546 }
1547#endif
1548 WARN_ON(in_interrupt());
Ingo Molnarc5905af2012-02-24 08:31:31 +01001549 static_key_slow_inc(&netstamp_needed);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001550}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001551EXPORT_SYMBOL(net_enable_timestamp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001552
1553void net_disable_timestamp(void)
1554{
Eric Dumazetb90e5792011-11-28 11:16:50 +00001555#ifdef HAVE_JUMP_LABEL
1556 if (in_interrupt()) {
1557 atomic_inc(&netstamp_needed_deferred);
1558 return;
1559 }
1560#endif
Ingo Molnarc5905af2012-02-24 08:31:31 +01001561 static_key_slow_dec(&netstamp_needed);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001562}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001563EXPORT_SYMBOL(net_disable_timestamp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001564
Eric Dumazet3b098e22010-05-15 23:57:10 -07001565static inline void net_timestamp_set(struct sk_buff *skb)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001566{
Eric Dumazet588f0332011-11-15 04:12:55 +00001567 skb->tstamp.tv64 = 0;
Ingo Molnarc5905af2012-02-24 08:31:31 +01001568 if (static_key_false(&netstamp_needed))
Patrick McHardya61bbcf2005-08-14 17:24:31 -07001569 __net_timestamp(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001570}
1571
Eric Dumazet588f0332011-11-15 04:12:55 +00001572#define net_timestamp_check(COND, SKB) \
Ingo Molnarc5905af2012-02-24 08:31:31 +01001573 if (static_key_false(&netstamp_needed)) { \
Eric Dumazet588f0332011-11-15 04:12:55 +00001574 if ((COND) && !(SKB)->tstamp.tv64) \
1575 __net_timestamp(SKB); \
1576 } \
Eric Dumazet3b098e22010-05-15 23:57:10 -07001577
Daniel Lezcano79b569f2011-03-30 02:42:17 -07001578static inline bool is_skb_forwardable(struct net_device *dev,
1579 struct sk_buff *skb)
1580{
1581 unsigned int len;
1582
1583 if (!(dev->flags & IFF_UP))
1584 return false;
1585
1586 len = dev->mtu + dev->hard_header_len + VLAN_HLEN;
1587 if (skb->len <= len)
1588 return true;
1589
1590 /* if TSO is enabled, we don't care about the length as the packet
1591 * could be forwarded without being segmented before
1592 */
1593 if (skb_is_gso(skb))
1594 return true;
1595
1596 return false;
1597}
1598
Arnd Bergmann44540962009-11-26 06:07:08 +00001599/**
1600 * dev_forward_skb - loopback an skb to another netif
1601 *
1602 * @dev: destination network device
1603 * @skb: buffer to forward
1604 *
1605 * return values:
1606 * NET_RX_SUCCESS (no congestion)
Eric Dumazet6ec82562010-05-06 00:53:53 -07001607 * NET_RX_DROP (packet was dropped, but freed)
Arnd Bergmann44540962009-11-26 06:07:08 +00001608 *
1609 * dev_forward_skb can be used for injecting an skb from the
1610 * start_xmit function of one device into the receive queue
1611 * of another device.
1612 *
1613 * The receiving device may be in another namespace, so
1614 * we have to clear all information in the skb that could
1615 * impact namespace isolation.
1616 */
1617int dev_forward_skb(struct net_device *dev, struct sk_buff *skb)
1618{
Michael S. Tsirkin48c83012011-08-31 08:03:29 +00001619 if (skb_shinfo(skb)->tx_flags & SKBTX_DEV_ZEROCOPY) {
1620 if (skb_copy_ubufs(skb, GFP_ATOMIC)) {
1621 atomic_long_inc(&dev->rx_dropped);
1622 kfree_skb(skb);
1623 return NET_RX_DROP;
1624 }
1625 }
1626
Arnd Bergmann44540962009-11-26 06:07:08 +00001627 skb_orphan(skb);
Ben Greearc736eef2010-07-22 09:54:47 +00001628 nf_reset(skb);
Arnd Bergmann44540962009-11-26 06:07:08 +00001629
Daniel Lezcano79b569f2011-03-30 02:42:17 -07001630 if (unlikely(!is_skb_forwardable(dev, skb))) {
Eric Dumazetcaf586e2010-09-30 21:06:55 +00001631 atomic_long_inc(&dev->rx_dropped);
Eric Dumazet6ec82562010-05-06 00:53:53 -07001632 kfree_skb(skb);
Arnd Bergmann44540962009-11-26 06:07:08 +00001633 return NET_RX_DROP;
Eric Dumazet6ec82562010-05-06 00:53:53 -07001634 }
Benjamin LaHaise3b9785c2012-03-27 15:55:44 +00001635 skb->skb_iif = 0;
David S. Miller59b99972012-05-10 23:03:34 -04001636 skb->dev = dev;
1637 skb_dst_drop(skb);
Arnd Bergmann44540962009-11-26 06:07:08 +00001638 skb->tstamp.tv64 = 0;
1639 skb->pkt_type = PACKET_HOST;
1640 skb->protocol = eth_type_trans(skb, dev);
David S. Miller59b99972012-05-10 23:03:34 -04001641 skb->mark = 0;
1642 secpath_reset(skb);
1643 nf_reset(skb);
Arnd Bergmann44540962009-11-26 06:07:08 +00001644 return netif_rx(skb);
1645}
1646EXPORT_SYMBOL_GPL(dev_forward_skb);
1647
Changli Gao71d9dec2010-12-15 19:57:25 +00001648static inline int deliver_skb(struct sk_buff *skb,
1649 struct packet_type *pt_prev,
1650 struct net_device *orig_dev)
1651{
Michael S. Tsirkin1080e512012-07-20 09:23:17 +00001652 if (unlikely(skb_orphan_frags(skb, GFP_ATOMIC)))
1653 return -ENOMEM;
Changli Gao71d9dec2010-12-15 19:57:25 +00001654 atomic_inc(&skb->users);
1655 return pt_prev->func(skb, skb->dev, pt_prev, orig_dev);
1656}
1657
Eric Leblondc0de08d2012-08-16 22:02:58 +00001658static inline bool skb_loop_sk(struct packet_type *ptype, struct sk_buff *skb)
1659{
Eric Leblonda3d744e2012-11-06 02:10:10 +00001660 if (!ptype->af_packet_priv || !skb->sk)
Eric Leblondc0de08d2012-08-16 22:02:58 +00001661 return false;
1662
1663 if (ptype->id_match)
1664 return ptype->id_match(ptype, skb->sk);
1665 else if ((struct sock *)ptype->af_packet_priv == skb->sk)
1666 return true;
1667
1668 return false;
1669}
1670
Linus Torvalds1da177e2005-04-16 15:20:36 -07001671/*
1672 * Support routine. Sends outgoing frames to any network
1673 * taps currently in use.
1674 */
1675
Herbert Xuf6a78bf2006-06-22 02:57:17 -07001676static void dev_queue_xmit_nit(struct sk_buff *skb, struct net_device *dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001677{
1678 struct packet_type *ptype;
Changli Gao71d9dec2010-12-15 19:57:25 +00001679 struct sk_buff *skb2 = NULL;
1680 struct packet_type *pt_prev = NULL;
Patrick McHardya61bbcf2005-08-14 17:24:31 -07001681
Linus Torvalds1da177e2005-04-16 15:20:36 -07001682 rcu_read_lock();
1683 list_for_each_entry_rcu(ptype, &ptype_all, list) {
1684 /* Never send packets back to the socket
1685 * they originated from - MvS (miquels@drinkel.ow.org)
1686 */
1687 if ((ptype->dev == dev || !ptype->dev) &&
Eric Leblondc0de08d2012-08-16 22:02:58 +00001688 (!skb_loop_sk(ptype, skb))) {
Changli Gao71d9dec2010-12-15 19:57:25 +00001689 if (pt_prev) {
1690 deliver_skb(skb2, pt_prev, skb->dev);
1691 pt_prev = ptype;
1692 continue;
1693 }
1694
1695 skb2 = skb_clone(skb, GFP_ATOMIC);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001696 if (!skb2)
1697 break;
1698
Eric Dumazet70978182010-12-20 21:22:51 +00001699 net_timestamp_set(skb2);
1700
Linus Torvalds1da177e2005-04-16 15:20:36 -07001701 /* skb->nh should be correctly
1702 set by sender, so that the second statement is
1703 just protection against buggy protocols.
1704 */
Arnaldo Carvalho de Melo459a98e2007-03-19 15:30:44 -07001705 skb_reset_mac_header(skb2);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001706
Arnaldo Carvalho de Melod56f90a2007-04-10 20:50:43 -07001707 if (skb_network_header(skb2) < skb2->data ||
Arnaldo Carvalho de Melo27a884d2007-04-19 20:29:13 -07001708 skb2->network_header > skb2->tail) {
Joe Perchese87cc472012-05-13 21:56:26 +00001709 net_crit_ratelimited("protocol %04x is buggy, dev %s\n",
1710 ntohs(skb2->protocol),
1711 dev->name);
Arnaldo Carvalho de Meloc1d2bbe2007-04-10 20:45:18 -07001712 skb_reset_network_header(skb2);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001713 }
1714
Arnaldo Carvalho de Melob0e380b2007-04-10 21:21:55 -07001715 skb2->transport_header = skb2->network_header;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001716 skb2->pkt_type = PACKET_OUTGOING;
Changli Gao71d9dec2010-12-15 19:57:25 +00001717 pt_prev = ptype;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001718 }
1719 }
Changli Gao71d9dec2010-12-15 19:57:25 +00001720 if (pt_prev)
1721 pt_prev->func(skb2, skb->dev, pt_prev, skb->dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001722 rcu_read_unlock();
1723}
1724
Ben Hutchings2c530402012-07-10 10:55:09 +00001725/**
1726 * netif_setup_tc - Handle tc mappings on real_num_tx_queues change
John Fastabend4f57c082011-01-17 08:06:04 +00001727 * @dev: Network device
1728 * @txq: number of queues available
1729 *
1730 * If real_num_tx_queues is changed the tc mappings may no longer be
1731 * valid. To resolve this verify the tc mapping remains valid and if
1732 * not NULL the mapping. With no priorities mapping to this
1733 * offset/count pair it will no longer be used. In the worst case TC0
1734 * is invalid nothing can be done so disable priority mappings. If is
1735 * expected that drivers will fix this mapping if they can before
1736 * calling netif_set_real_num_tx_queues.
1737 */
Eric Dumazetbb134d22011-01-20 19:18:08 +00001738static void netif_setup_tc(struct net_device *dev, unsigned int txq)
John Fastabend4f57c082011-01-17 08:06:04 +00001739{
1740 int i;
1741 struct netdev_tc_txq *tc = &dev->tc_to_txq[0];
1742
1743 /* If TC0 is invalidated disable TC mapping */
1744 if (tc->offset + tc->count > txq) {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00001745 pr_warn("Number of in use tx queues changed invalidating tc mappings. Priority traffic classification disabled!\n");
John Fastabend4f57c082011-01-17 08:06:04 +00001746 dev->num_tc = 0;
1747 return;
1748 }
1749
1750 /* Invalidated prio to tc mappings set to TC0 */
1751 for (i = 1; i < TC_BITMASK + 1; i++) {
1752 int q = netdev_get_prio_tc_map(dev, i);
1753
1754 tc = &dev->tc_to_txq[q];
1755 if (tc->offset + tc->count > txq) {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00001756 pr_warn("Number of in use tx queues changed. Priority %i to tc mapping %i is no longer valid. Setting map to 0\n",
1757 i, q);
John Fastabend4f57c082011-01-17 08:06:04 +00001758 netdev_set_prio_tc_map(dev, i, 0);
1759 }
1760 }
1761}
1762
Alexander Duyck537c00d2013-01-10 08:57:02 +00001763#ifdef CONFIG_XPS
1764static DEFINE_MUTEX(xps_map_mutex);
1765#define xmap_dereference(P) \
1766 rcu_dereference_protected((P), lockdep_is_held(&xps_map_mutex))
1767
Alexander Duyck10cdc3f2013-01-10 08:57:17 +00001768static struct xps_map *remove_xps_queue(struct xps_dev_maps *dev_maps,
1769 int cpu, u16 index)
1770{
1771 struct xps_map *map = NULL;
1772 int pos;
1773
1774 if (dev_maps)
1775 map = xmap_dereference(dev_maps->cpu_map[cpu]);
1776
1777 for (pos = 0; map && pos < map->len; pos++) {
1778 if (map->queues[pos] == index) {
1779 if (map->len > 1) {
1780 map->queues[pos] = map->queues[--map->len];
1781 } else {
1782 RCU_INIT_POINTER(dev_maps->cpu_map[cpu], NULL);
1783 kfree_rcu(map, rcu);
1784 map = NULL;
1785 }
1786 break;
1787 }
1788 }
1789
1790 return map;
1791}
1792
Alexander Duyck024e9672013-01-10 08:57:46 +00001793static void netif_reset_xps_queues_gt(struct net_device *dev, u16 index)
Alexander Duyck537c00d2013-01-10 08:57:02 +00001794{
1795 struct xps_dev_maps *dev_maps;
Alexander Duyck024e9672013-01-10 08:57:46 +00001796 int cpu, i;
Alexander Duyck10cdc3f2013-01-10 08:57:17 +00001797 bool active = false;
Alexander Duyck537c00d2013-01-10 08:57:02 +00001798
1799 mutex_lock(&xps_map_mutex);
1800 dev_maps = xmap_dereference(dev->xps_maps);
1801
1802 if (!dev_maps)
1803 goto out_no_maps;
1804
Alexander Duyck10cdc3f2013-01-10 08:57:17 +00001805 for_each_possible_cpu(cpu) {
Alexander Duyck024e9672013-01-10 08:57:46 +00001806 for (i = index; i < dev->num_tx_queues; i++) {
1807 if (!remove_xps_queue(dev_maps, cpu, i))
1808 break;
1809 }
1810 if (i == dev->num_tx_queues)
Alexander Duyck10cdc3f2013-01-10 08:57:17 +00001811 active = true;
Alexander Duyck537c00d2013-01-10 08:57:02 +00001812 }
1813
Alexander Duyck10cdc3f2013-01-10 08:57:17 +00001814 if (!active) {
Alexander Duyck537c00d2013-01-10 08:57:02 +00001815 RCU_INIT_POINTER(dev->xps_maps, NULL);
1816 kfree_rcu(dev_maps, rcu);
1817 }
1818
Alexander Duyck024e9672013-01-10 08:57:46 +00001819 for (i = index; i < dev->num_tx_queues; i++)
1820 netdev_queue_numa_node_write(netdev_get_tx_queue(dev, i),
1821 NUMA_NO_NODE);
1822
Alexander Duyck537c00d2013-01-10 08:57:02 +00001823out_no_maps:
1824 mutex_unlock(&xps_map_mutex);
1825}
1826
Alexander Duyck01c5f862013-01-10 08:57:35 +00001827static struct xps_map *expand_xps_map(struct xps_map *map,
1828 int cpu, u16 index)
1829{
1830 struct xps_map *new_map;
1831 int alloc_len = XPS_MIN_MAP_ALLOC;
1832 int i, pos;
1833
1834 for (pos = 0; map && pos < map->len; pos++) {
1835 if (map->queues[pos] != index)
1836 continue;
1837 return map;
1838 }
1839
1840 /* Need to add queue to this CPU's existing map */
1841 if (map) {
1842 if (pos < map->alloc_len)
1843 return map;
1844
1845 alloc_len = map->alloc_len * 2;
1846 }
1847
1848 /* Need to allocate new map to store queue on this CPU's map */
1849 new_map = kzalloc_node(XPS_MAP_SIZE(alloc_len), GFP_KERNEL,
1850 cpu_to_node(cpu));
1851 if (!new_map)
1852 return NULL;
1853
1854 for (i = 0; i < pos; i++)
1855 new_map->queues[i] = map->queues[i];
1856 new_map->alloc_len = alloc_len;
1857 new_map->len = pos;
1858
1859 return new_map;
1860}
1861
Alexander Duyck537c00d2013-01-10 08:57:02 +00001862int netif_set_xps_queue(struct net_device *dev, struct cpumask *mask, u16 index)
1863{
Alexander Duyck01c5f862013-01-10 08:57:35 +00001864 struct xps_dev_maps *dev_maps, *new_dev_maps = NULL;
Alexander Duyck537c00d2013-01-10 08:57:02 +00001865 struct xps_map *map, *new_map;
Alexander Duyck537c00d2013-01-10 08:57:02 +00001866 int maps_sz = max_t(unsigned int, XPS_DEV_MAPS_SIZE, L1_CACHE_BYTES);
Alexander Duyck01c5f862013-01-10 08:57:35 +00001867 int cpu, numa_node_id = -2;
1868 bool active = false;
Alexander Duyck537c00d2013-01-10 08:57:02 +00001869
1870 mutex_lock(&xps_map_mutex);
1871
1872 dev_maps = xmap_dereference(dev->xps_maps);
1873
Alexander Duyck01c5f862013-01-10 08:57:35 +00001874 /* allocate memory for queue storage */
1875 for_each_online_cpu(cpu) {
1876 if (!cpumask_test_cpu(cpu, mask))
1877 continue;
Alexander Duyck537c00d2013-01-10 08:57:02 +00001878
Alexander Duyck01c5f862013-01-10 08:57:35 +00001879 if (!new_dev_maps)
1880 new_dev_maps = kzalloc(maps_sz, GFP_KERNEL);
Alexander Duyck2bb60cb2013-02-22 06:38:44 +00001881 if (!new_dev_maps) {
1882 mutex_unlock(&xps_map_mutex);
Alexander Duyck01c5f862013-01-10 08:57:35 +00001883 return -ENOMEM;
Alexander Duyck2bb60cb2013-02-22 06:38:44 +00001884 }
Alexander Duyck01c5f862013-01-10 08:57:35 +00001885
1886 map = dev_maps ? xmap_dereference(dev_maps->cpu_map[cpu]) :
1887 NULL;
1888
1889 map = expand_xps_map(map, cpu, index);
1890 if (!map)
1891 goto error;
1892
1893 RCU_INIT_POINTER(new_dev_maps->cpu_map[cpu], map);
1894 }
1895
1896 if (!new_dev_maps)
1897 goto out_no_new_maps;
1898
1899 for_each_possible_cpu(cpu) {
1900 if (cpumask_test_cpu(cpu, mask) && cpu_online(cpu)) {
1901 /* add queue to CPU maps */
1902 int pos = 0;
1903
1904 map = xmap_dereference(new_dev_maps->cpu_map[cpu]);
1905 while ((pos < map->len) && (map->queues[pos] != index))
1906 pos++;
1907
1908 if (pos == map->len)
1909 map->queues[map->len++] = index;
Alexander Duyck537c00d2013-01-10 08:57:02 +00001910#ifdef CONFIG_NUMA
Alexander Duyck537c00d2013-01-10 08:57:02 +00001911 if (numa_node_id == -2)
1912 numa_node_id = cpu_to_node(cpu);
1913 else if (numa_node_id != cpu_to_node(cpu))
1914 numa_node_id = -1;
Alexander Duyck537c00d2013-01-10 08:57:02 +00001915#endif
Alexander Duyck01c5f862013-01-10 08:57:35 +00001916 } else if (dev_maps) {
1917 /* fill in the new device map from the old device map */
1918 map = xmap_dereference(dev_maps->cpu_map[cpu]);
1919 RCU_INIT_POINTER(new_dev_maps->cpu_map[cpu], map);
Alexander Duyck537c00d2013-01-10 08:57:02 +00001920 }
Alexander Duyck01c5f862013-01-10 08:57:35 +00001921
Alexander Duyck537c00d2013-01-10 08:57:02 +00001922 }
1923
Alexander Duyck01c5f862013-01-10 08:57:35 +00001924 rcu_assign_pointer(dev->xps_maps, new_dev_maps);
1925
Alexander Duyck537c00d2013-01-10 08:57:02 +00001926 /* Cleanup old maps */
Alexander Duyck01c5f862013-01-10 08:57:35 +00001927 if (dev_maps) {
1928 for_each_possible_cpu(cpu) {
1929 new_map = xmap_dereference(new_dev_maps->cpu_map[cpu]);
1930 map = xmap_dereference(dev_maps->cpu_map[cpu]);
1931 if (map && map != new_map)
1932 kfree_rcu(map, rcu);
1933 }
Alexander Duyck537c00d2013-01-10 08:57:02 +00001934
Alexander Duyck537c00d2013-01-10 08:57:02 +00001935 kfree_rcu(dev_maps, rcu);
Alexander Duyck01c5f862013-01-10 08:57:35 +00001936 }
Alexander Duyck537c00d2013-01-10 08:57:02 +00001937
Alexander Duyck01c5f862013-01-10 08:57:35 +00001938 dev_maps = new_dev_maps;
1939 active = true;
1940
1941out_no_new_maps:
1942 /* update Tx queue numa node */
Alexander Duyck537c00d2013-01-10 08:57:02 +00001943 netdev_queue_numa_node_write(netdev_get_tx_queue(dev, index),
1944 (numa_node_id >= 0) ? numa_node_id :
1945 NUMA_NO_NODE);
1946
Alexander Duyck01c5f862013-01-10 08:57:35 +00001947 if (!dev_maps)
1948 goto out_no_maps;
1949
1950 /* removes queue from unused CPUs */
1951 for_each_possible_cpu(cpu) {
1952 if (cpumask_test_cpu(cpu, mask) && cpu_online(cpu))
1953 continue;
1954
1955 if (remove_xps_queue(dev_maps, cpu, index))
1956 active = true;
1957 }
1958
1959 /* free map if not active */
1960 if (!active) {
1961 RCU_INIT_POINTER(dev->xps_maps, NULL);
1962 kfree_rcu(dev_maps, rcu);
1963 }
1964
1965out_no_maps:
Alexander Duyck537c00d2013-01-10 08:57:02 +00001966 mutex_unlock(&xps_map_mutex);
1967
1968 return 0;
1969error:
Alexander Duyck01c5f862013-01-10 08:57:35 +00001970 /* remove any maps that we added */
1971 for_each_possible_cpu(cpu) {
1972 new_map = xmap_dereference(new_dev_maps->cpu_map[cpu]);
1973 map = dev_maps ? xmap_dereference(dev_maps->cpu_map[cpu]) :
1974 NULL;
1975 if (new_map && new_map != map)
1976 kfree(new_map);
1977 }
1978
Alexander Duyck537c00d2013-01-10 08:57:02 +00001979 mutex_unlock(&xps_map_mutex);
1980
Alexander Duyck537c00d2013-01-10 08:57:02 +00001981 kfree(new_dev_maps);
1982 return -ENOMEM;
1983}
1984EXPORT_SYMBOL(netif_set_xps_queue);
1985
1986#endif
John Fastabendf0796d52010-07-01 13:21:57 +00001987/*
1988 * Routine to help set real_num_tx_queues. To avoid skbs mapped to queues
1989 * greater then real_num_tx_queues stale skbs on the qdisc must be flushed.
1990 */
Tom Herberte6484932010-10-18 18:04:39 +00001991int netif_set_real_num_tx_queues(struct net_device *dev, unsigned int txq)
John Fastabendf0796d52010-07-01 13:21:57 +00001992{
Tom Herbert1d24eb42010-11-21 13:17:27 +00001993 int rc;
1994
Tom Herberte6484932010-10-18 18:04:39 +00001995 if (txq < 1 || txq > dev->num_tx_queues)
1996 return -EINVAL;
John Fastabendf0796d52010-07-01 13:21:57 +00001997
Ben Hutchings5c565802011-02-15 19:39:21 +00001998 if (dev->reg_state == NETREG_REGISTERED ||
1999 dev->reg_state == NETREG_UNREGISTERING) {
Tom Herberte6484932010-10-18 18:04:39 +00002000 ASSERT_RTNL();
2001
Tom Herbert1d24eb42010-11-21 13:17:27 +00002002 rc = netdev_queue_update_kobjects(dev, dev->real_num_tx_queues,
2003 txq);
Tom Herbertbf264142010-11-26 08:36:09 +00002004 if (rc)
2005 return rc;
2006
John Fastabend4f57c082011-01-17 08:06:04 +00002007 if (dev->num_tc)
2008 netif_setup_tc(dev, txq);
2009
Alexander Duyck024e9672013-01-10 08:57:46 +00002010 if (txq < dev->real_num_tx_queues) {
Tom Herberte6484932010-10-18 18:04:39 +00002011 qdisc_reset_all_tx_gt(dev, txq);
Alexander Duyck024e9672013-01-10 08:57:46 +00002012#ifdef CONFIG_XPS
2013 netif_reset_xps_queues_gt(dev, txq);
2014#endif
2015 }
John Fastabendf0796d52010-07-01 13:21:57 +00002016 }
Tom Herberte6484932010-10-18 18:04:39 +00002017
2018 dev->real_num_tx_queues = txq;
2019 return 0;
John Fastabendf0796d52010-07-01 13:21:57 +00002020}
2021EXPORT_SYMBOL(netif_set_real_num_tx_queues);
Denis Vlasenko56079432006-03-29 15:57:29 -08002022
Ben Hutchings62fe0b42010-09-27 08:24:33 +00002023#ifdef CONFIG_RPS
2024/**
2025 * netif_set_real_num_rx_queues - set actual number of RX queues used
2026 * @dev: Network device
2027 * @rxq: Actual number of RX queues
2028 *
2029 * This must be called either with the rtnl_lock held or before
2030 * registration of the net device. Returns 0 on success, or a
Ben Hutchings4e7f7952010-10-08 10:33:39 -07002031 * negative error code. If called before registration, it always
2032 * succeeds.
Ben Hutchings62fe0b42010-09-27 08:24:33 +00002033 */
2034int netif_set_real_num_rx_queues(struct net_device *dev, unsigned int rxq)
2035{
2036 int rc;
2037
Tom Herbertbd25fa72010-10-18 18:00:16 +00002038 if (rxq < 1 || rxq > dev->num_rx_queues)
2039 return -EINVAL;
2040
Ben Hutchings62fe0b42010-09-27 08:24:33 +00002041 if (dev->reg_state == NETREG_REGISTERED) {
2042 ASSERT_RTNL();
2043
Ben Hutchings62fe0b42010-09-27 08:24:33 +00002044 rc = net_rx_queue_update_kobjects(dev, dev->real_num_rx_queues,
2045 rxq);
2046 if (rc)
2047 return rc;
Ben Hutchings62fe0b42010-09-27 08:24:33 +00002048 }
2049
2050 dev->real_num_rx_queues = rxq;
2051 return 0;
2052}
2053EXPORT_SYMBOL(netif_set_real_num_rx_queues);
2054#endif
2055
Ben Hutchings2c530402012-07-10 10:55:09 +00002056/**
2057 * netif_get_num_default_rss_queues - default number of RSS queues
Yuval Mintz16917b82012-07-01 03:18:50 +00002058 *
2059 * This routine should set an upper limit on the number of RSS queues
2060 * used by default by multiqueue devices.
2061 */
Ben Hutchingsa55b1382012-07-10 10:54:38 +00002062int netif_get_num_default_rss_queues(void)
Yuval Mintz16917b82012-07-01 03:18:50 +00002063{
2064 return min_t(int, DEFAULT_MAX_NUM_RSS_QUEUES, num_online_cpus());
2065}
2066EXPORT_SYMBOL(netif_get_num_default_rss_queues);
2067
Jarek Poplawskidef82a12008-08-17 21:54:43 -07002068static inline void __netif_reschedule(struct Qdisc *q)
2069{
2070 struct softnet_data *sd;
2071 unsigned long flags;
2072
2073 local_irq_save(flags);
2074 sd = &__get_cpu_var(softnet_data);
Changli Gaoa9cbd582010-04-26 23:06:24 +00002075 q->next_sched = NULL;
2076 *sd->output_queue_tailp = q;
2077 sd->output_queue_tailp = &q->next_sched;
Jarek Poplawskidef82a12008-08-17 21:54:43 -07002078 raise_softirq_irqoff(NET_TX_SOFTIRQ);
2079 local_irq_restore(flags);
2080}
2081
David S. Miller37437bb2008-07-16 02:15:04 -07002082void __netif_schedule(struct Qdisc *q)
Denis Vlasenko56079432006-03-29 15:57:29 -08002083{
Jarek Poplawskidef82a12008-08-17 21:54:43 -07002084 if (!test_and_set_bit(__QDISC_STATE_SCHED, &q->state))
2085 __netif_reschedule(q);
Denis Vlasenko56079432006-03-29 15:57:29 -08002086}
2087EXPORT_SYMBOL(__netif_schedule);
2088
Stephen Hemmingerbea33482007-10-03 16:41:36 -07002089void dev_kfree_skb_irq(struct sk_buff *skb)
Denis Vlasenko56079432006-03-29 15:57:29 -08002090{
David S. Miller3578b0c2010-08-03 00:24:04 -07002091 if (atomic_dec_and_test(&skb->users)) {
Stephen Hemmingerbea33482007-10-03 16:41:36 -07002092 struct softnet_data *sd;
2093 unsigned long flags;
Denis Vlasenko56079432006-03-29 15:57:29 -08002094
Stephen Hemmingerbea33482007-10-03 16:41:36 -07002095 local_irq_save(flags);
2096 sd = &__get_cpu_var(softnet_data);
2097 skb->next = sd->completion_queue;
2098 sd->completion_queue = skb;
2099 raise_softirq_irqoff(NET_TX_SOFTIRQ);
2100 local_irq_restore(flags);
2101 }
Denis Vlasenko56079432006-03-29 15:57:29 -08002102}
Stephen Hemmingerbea33482007-10-03 16:41:36 -07002103EXPORT_SYMBOL(dev_kfree_skb_irq);
Denis Vlasenko56079432006-03-29 15:57:29 -08002104
2105void dev_kfree_skb_any(struct sk_buff *skb)
2106{
2107 if (in_irq() || irqs_disabled())
2108 dev_kfree_skb_irq(skb);
2109 else
2110 dev_kfree_skb(skb);
2111}
2112EXPORT_SYMBOL(dev_kfree_skb_any);
2113
2114
Stephen Hemmingerbea33482007-10-03 16:41:36 -07002115/**
2116 * netif_device_detach - mark device as removed
2117 * @dev: network device
2118 *
2119 * Mark device as removed from system and therefore no longer available.
2120 */
Denis Vlasenko56079432006-03-29 15:57:29 -08002121void netif_device_detach(struct net_device *dev)
2122{
2123 if (test_and_clear_bit(__LINK_STATE_PRESENT, &dev->state) &&
2124 netif_running(dev)) {
Alexander Duyckd5431032009-04-08 13:15:22 +00002125 netif_tx_stop_all_queues(dev);
Denis Vlasenko56079432006-03-29 15:57:29 -08002126 }
2127}
2128EXPORT_SYMBOL(netif_device_detach);
2129
Stephen Hemmingerbea33482007-10-03 16:41:36 -07002130/**
2131 * netif_device_attach - mark device as attached
2132 * @dev: network device
2133 *
2134 * Mark device as attached from system and restart if needed.
2135 */
Denis Vlasenko56079432006-03-29 15:57:29 -08002136void netif_device_attach(struct net_device *dev)
2137{
2138 if (!test_and_set_bit(__LINK_STATE_PRESENT, &dev->state) &&
2139 netif_running(dev)) {
Alexander Duyckd5431032009-04-08 13:15:22 +00002140 netif_tx_wake_all_queues(dev);
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09002141 __netdev_watchdog_up(dev);
Denis Vlasenko56079432006-03-29 15:57:29 -08002142 }
2143}
2144EXPORT_SYMBOL(netif_device_attach);
2145
Ben Hutchings36c92472012-01-17 07:57:56 +00002146static void skb_warn_bad_offload(const struct sk_buff *skb)
2147{
Michał Mirosław65e9d2f2012-01-17 10:00:40 +00002148 static const netdev_features_t null_features = 0;
Ben Hutchings36c92472012-01-17 07:57:56 +00002149 struct net_device *dev = skb->dev;
2150 const char *driver = "";
2151
2152 if (dev && dev->dev.parent)
2153 driver = dev_driver_string(dev->dev.parent);
2154
2155 WARN(1, "%s: caps=(%pNF, %pNF) len=%d data_len=%d gso_size=%d "
2156 "gso_type=%d ip_summed=%d\n",
Michał Mirosław65e9d2f2012-01-17 10:00:40 +00002157 driver, dev ? &dev->features : &null_features,
2158 skb->sk ? &skb->sk->sk_route_caps : &null_features,
Ben Hutchings36c92472012-01-17 07:57:56 +00002159 skb->len, skb->data_len, skb_shinfo(skb)->gso_size,
2160 skb_shinfo(skb)->gso_type, skb->ip_summed);
2161}
2162
Linus Torvalds1da177e2005-04-16 15:20:36 -07002163/*
2164 * Invalidate hardware checksum when packet is to be mangled, and
2165 * complete checksum manually on outgoing path.
2166 */
Patrick McHardy84fa7932006-08-29 16:44:56 -07002167int skb_checksum_help(struct sk_buff *skb)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002168{
Al Virod3bc23e2006-11-14 21:24:49 -08002169 __wsum csum;
Herbert Xu663ead32007-04-09 11:59:07 -07002170 int ret = 0, offset;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002171
Patrick McHardy84fa7932006-08-29 16:44:56 -07002172 if (skb->ip_summed == CHECKSUM_COMPLETE)
Herbert Xua430a432006-07-08 13:34:56 -07002173 goto out_set_summed;
2174
2175 if (unlikely(skb_shinfo(skb)->gso_size)) {
Ben Hutchings36c92472012-01-17 07:57:56 +00002176 skb_warn_bad_offload(skb);
2177 return -EINVAL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002178 }
2179
Eric Dumazetcef401d2013-01-25 20:34:37 +00002180 /* Before computing a checksum, we should make sure no frag could
2181 * be modified by an external entity : checksum could be wrong.
2182 */
2183 if (skb_has_shared_frag(skb)) {
2184 ret = __skb_linearize(skb);
2185 if (ret)
2186 goto out;
2187 }
2188
Michał Mirosław55508d62010-12-14 15:24:08 +00002189 offset = skb_checksum_start_offset(skb);
Herbert Xua0308472007-10-15 01:47:15 -07002190 BUG_ON(offset >= skb_headlen(skb));
2191 csum = skb_checksum(skb, offset, skb->len - offset, 0);
2192
2193 offset += skb->csum_offset;
2194 BUG_ON(offset + sizeof(__sum16) > skb_headlen(skb));
2195
2196 if (skb_cloned(skb) &&
2197 !skb_clone_writable(skb, offset + sizeof(__sum16))) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07002198 ret = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
2199 if (ret)
2200 goto out;
2201 }
2202
Herbert Xua0308472007-10-15 01:47:15 -07002203 *(__sum16 *)(skb->data + offset) = csum_fold(csum);
Herbert Xua430a432006-07-08 13:34:56 -07002204out_set_summed:
Linus Torvalds1da177e2005-04-16 15:20:36 -07002205 skb->ip_summed = CHECKSUM_NONE;
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09002206out:
Linus Torvalds1da177e2005-04-16 15:20:36 -07002207 return ret;
2208}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07002209EXPORT_SYMBOL(skb_checksum_help);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002210
Pravin B Shelar05e8ef42013-02-14 09:44:55 +00002211/**
2212 * skb_mac_gso_segment - mac layer segmentation handler.
2213 * @skb: buffer to segment
2214 * @features: features for the output path (see dev->features)
2215 */
2216struct sk_buff *skb_mac_gso_segment(struct sk_buff *skb,
2217 netdev_features_t features)
2218{
2219 struct sk_buff *segs = ERR_PTR(-EPROTONOSUPPORT);
2220 struct packet_offload *ptype;
2221 __be16 type = skb->protocol;
2222
2223 while (type == htons(ETH_P_8021Q)) {
2224 int vlan_depth = ETH_HLEN;
2225 struct vlan_hdr *vh;
2226
2227 if (unlikely(!pskb_may_pull(skb, vlan_depth + VLAN_HLEN)))
2228 return ERR_PTR(-EINVAL);
2229
2230 vh = (struct vlan_hdr *)(skb->data + vlan_depth);
2231 type = vh->h_vlan_encapsulated_proto;
2232 vlan_depth += VLAN_HLEN;
2233 }
2234
2235 __skb_pull(skb, skb->mac_len);
2236
2237 rcu_read_lock();
2238 list_for_each_entry_rcu(ptype, &offload_base, list) {
2239 if (ptype->type == type && ptype->callbacks.gso_segment) {
2240 if (unlikely(skb->ip_summed != CHECKSUM_PARTIAL)) {
2241 int err;
2242
2243 err = ptype->callbacks.gso_send_check(skb);
2244 segs = ERR_PTR(err);
2245 if (err || skb_gso_ok(skb, features))
2246 break;
2247 __skb_push(skb, (skb->data -
2248 skb_network_header(skb)));
2249 }
2250 segs = ptype->callbacks.gso_segment(skb, features);
2251 break;
2252 }
2253 }
2254 rcu_read_unlock();
2255
2256 __skb_push(skb, skb->data - skb_mac_header(skb));
2257
2258 return segs;
2259}
2260EXPORT_SYMBOL(skb_mac_gso_segment);
2261
2262
Cong Wang12b00042013-02-05 16:36:38 +00002263/* openvswitch calls this on rx path, so we need a different check.
2264 */
2265static inline bool skb_needs_check(struct sk_buff *skb, bool tx_path)
2266{
2267 if (tx_path)
2268 return skb->ip_summed != CHECKSUM_PARTIAL;
2269 else
2270 return skb->ip_summed == CHECKSUM_NONE;
2271}
2272
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002273/**
Cong Wang12b00042013-02-05 16:36:38 +00002274 * __skb_gso_segment - Perform segmentation on skb.
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002275 * @skb: buffer to segment
Herbert Xu576a30e2006-06-27 13:22:38 -07002276 * @features: features for the output path (see dev->features)
Cong Wang12b00042013-02-05 16:36:38 +00002277 * @tx_path: whether it is called in TX path
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002278 *
2279 * This function segments the given skb and returns a list of segments.
Herbert Xu576a30e2006-06-27 13:22:38 -07002280 *
2281 * It may return NULL if the skb requires no segmentation. This is
2282 * only possible when GSO is used for verifying header integrity.
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002283 */
Cong Wang12b00042013-02-05 16:36:38 +00002284struct sk_buff *__skb_gso_segment(struct sk_buff *skb,
2285 netdev_features_t features, bool tx_path)
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002286{
Cong Wang12b00042013-02-05 16:36:38 +00002287 if (unlikely(skb_needs_check(skb, tx_path))) {
Pravin B Shelar05e8ef42013-02-14 09:44:55 +00002288 int err;
2289
Ben Hutchings36c92472012-01-17 07:57:56 +00002290 skb_warn_bad_offload(skb);
Herbert Xu67fd1a72009-01-19 16:26:44 -08002291
Herbert Xua430a432006-07-08 13:34:56 -07002292 if (skb_header_cloned(skb) &&
2293 (err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC)))
2294 return ERR_PTR(err);
2295 }
2296
Pravin B Shelar68c33162013-02-14 14:02:41 +00002297 SKB_GSO_CB(skb)->mac_offset = skb_headroom(skb);
Pravin B Shelar05e8ef42013-02-14 09:44:55 +00002298 skb_reset_mac_header(skb);
2299 skb_reset_mac_len(skb);
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002300
Pravin B Shelar05e8ef42013-02-14 09:44:55 +00002301 return skb_mac_gso_segment(skb, features);
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002302}
Cong Wang12b00042013-02-05 16:36:38 +00002303EXPORT_SYMBOL(__skb_gso_segment);
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002304
Herbert Xufb286bb2005-11-10 13:01:24 -08002305/* Take action when hardware reception checksum errors are detected. */
2306#ifdef CONFIG_BUG
2307void netdev_rx_csum_fault(struct net_device *dev)
2308{
2309 if (net_ratelimit()) {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00002310 pr_err("%s: hw csum failure\n", dev ? dev->name : "<unknown>");
Herbert Xufb286bb2005-11-10 13:01:24 -08002311 dump_stack();
2312 }
2313}
2314EXPORT_SYMBOL(netdev_rx_csum_fault);
2315#endif
2316
Linus Torvalds1da177e2005-04-16 15:20:36 -07002317/* Actually, we should eliminate this check as soon as we know, that:
2318 * 1. IOMMU is present and allows to map all the memory.
2319 * 2. No high memory really exists on this machine.
2320 */
2321
Eric Dumazet9092c652010-04-02 13:34:49 -07002322static int illegal_highdma(struct net_device *dev, struct sk_buff *skb)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002323{
Herbert Xu3d3a8532006-06-27 13:33:10 -07002324#ifdef CONFIG_HIGHMEM
Linus Torvalds1da177e2005-04-16 15:20:36 -07002325 int i;
FUJITA Tomonori5acbbd42010-03-30 22:35:50 +00002326 if (!(dev->features & NETIF_F_HIGHDMA)) {
Ian Campbellea2ab692011-08-22 23:44:58 +00002327 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
2328 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
2329 if (PageHighMem(skb_frag_page(frag)))
FUJITA Tomonori5acbbd42010-03-30 22:35:50 +00002330 return 1;
Ian Campbellea2ab692011-08-22 23:44:58 +00002331 }
FUJITA Tomonori5acbbd42010-03-30 22:35:50 +00002332 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002333
FUJITA Tomonori5acbbd42010-03-30 22:35:50 +00002334 if (PCI_DMA_BUS_IS_PHYS) {
2335 struct device *pdev = dev->dev.parent;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002336
Eric Dumazet9092c652010-04-02 13:34:49 -07002337 if (!pdev)
2338 return 0;
FUJITA Tomonori5acbbd42010-03-30 22:35:50 +00002339 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
Ian Campbellea2ab692011-08-22 23:44:58 +00002340 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
2341 dma_addr_t addr = page_to_phys(skb_frag_page(frag));
FUJITA Tomonori5acbbd42010-03-30 22:35:50 +00002342 if (!pdev->dma_mask || addr + PAGE_SIZE - 1 > *pdev->dma_mask)
2343 return 1;
2344 }
2345 }
Herbert Xu3d3a8532006-06-27 13:33:10 -07002346#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07002347 return 0;
2348}
Linus Torvalds1da177e2005-04-16 15:20:36 -07002349
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002350struct dev_gso_cb {
2351 void (*destructor)(struct sk_buff *skb);
2352};
2353
2354#define DEV_GSO_CB(skb) ((struct dev_gso_cb *)(skb)->cb)
2355
2356static void dev_gso_skb_destructor(struct sk_buff *skb)
2357{
2358 struct dev_gso_cb *cb;
2359
2360 do {
2361 struct sk_buff *nskb = skb->next;
2362
2363 skb->next = nskb->next;
2364 nskb->next = NULL;
2365 kfree_skb(nskb);
2366 } while (skb->next);
2367
2368 cb = DEV_GSO_CB(skb);
2369 if (cb->destructor)
2370 cb->destructor(skb);
2371}
2372
2373/**
2374 * dev_gso_segment - Perform emulated hardware segmentation on skb.
2375 * @skb: buffer to segment
Jesse Gross91ecb632011-01-09 06:23:33 +00002376 * @features: device features as applicable to this skb
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002377 *
2378 * This function segments the given skb and stores the list of segments
2379 * in skb->next.
2380 */
Michał Mirosławc8f44af2011-11-15 15:29:55 +00002381static int dev_gso_segment(struct sk_buff *skb, netdev_features_t features)
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002382{
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002383 struct sk_buff *segs;
2384
Herbert Xu576a30e2006-06-27 13:22:38 -07002385 segs = skb_gso_segment(skb, features);
2386
2387 /* Verifying header integrity only. */
2388 if (!segs)
2389 return 0;
2390
Hirofumi Nakagawa801678c2008-04-29 01:03:09 -07002391 if (IS_ERR(segs))
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002392 return PTR_ERR(segs);
2393
2394 skb->next = segs;
2395 DEV_GSO_CB(skb)->destructor = skb->destructor;
2396 skb->destructor = dev_gso_skb_destructor;
2397
2398 return 0;
2399}
2400
Michał Mirosławc8f44af2011-11-15 15:29:55 +00002401static bool can_checksum_protocol(netdev_features_t features, __be16 protocol)
Jesse Gross03634662011-01-09 06:23:35 +00002402{
2403 return ((features & NETIF_F_GEN_CSUM) ||
2404 ((features & NETIF_F_V4_CSUM) &&
2405 protocol == htons(ETH_P_IP)) ||
2406 ((features & NETIF_F_V6_CSUM) &&
2407 protocol == htons(ETH_P_IPV6)) ||
2408 ((features & NETIF_F_FCOE_CRC) &&
2409 protocol == htons(ETH_P_FCOE)));
2410}
2411
Michał Mirosławc8f44af2011-11-15 15:29:55 +00002412static netdev_features_t harmonize_features(struct sk_buff *skb,
2413 __be16 protocol, netdev_features_t features)
Jesse Grossf01a5232011-01-09 06:23:31 +00002414{
Ed Cashinc0d680e2012-09-19 15:49:00 +00002415 if (skb->ip_summed != CHECKSUM_NONE &&
2416 !can_checksum_protocol(features, protocol)) {
Jesse Grossf01a5232011-01-09 06:23:31 +00002417 features &= ~NETIF_F_ALL_CSUM;
2418 features &= ~NETIF_F_SG;
2419 } else if (illegal_highdma(skb->dev, skb)) {
2420 features &= ~NETIF_F_SG;
2421 }
2422
2423 return features;
2424}
2425
Michał Mirosławc8f44af2011-11-15 15:29:55 +00002426netdev_features_t netif_skb_features(struct sk_buff *skb)
Jesse Gross58e998c2010-10-29 12:14:55 +00002427{
2428 __be16 protocol = skb->protocol;
Michał Mirosławc8f44af2011-11-15 15:29:55 +00002429 netdev_features_t features = skb->dev->features;
Jesse Gross58e998c2010-10-29 12:14:55 +00002430
Ben Hutchings30b678d2012-07-30 15:57:00 +00002431 if (skb_shinfo(skb)->gso_segs > skb->dev->gso_max_segs)
2432 features &= ~NETIF_F_GSO_MASK;
2433
Jesse Gross58e998c2010-10-29 12:14:55 +00002434 if (protocol == htons(ETH_P_8021Q)) {
2435 struct vlan_ethhdr *veh = (struct vlan_ethhdr *)skb->data;
2436 protocol = veh->h_vlan_encapsulated_proto;
Jesse Grossf01a5232011-01-09 06:23:31 +00002437 } else if (!vlan_tx_tag_present(skb)) {
2438 return harmonize_features(skb, protocol, features);
2439 }
Jesse Gross58e998c2010-10-29 12:14:55 +00002440
Jesse Gross6ee400a2011-01-17 20:46:00 +00002441 features &= (skb->dev->vlan_features | NETIF_F_HW_VLAN_TX);
Jesse Grossf01a5232011-01-09 06:23:31 +00002442
2443 if (protocol != htons(ETH_P_8021Q)) {
2444 return harmonize_features(skb, protocol, features);
2445 } else {
2446 features &= NETIF_F_SG | NETIF_F_HIGHDMA | NETIF_F_FRAGLIST |
Jesse Gross6ee400a2011-01-17 20:46:00 +00002447 NETIF_F_GEN_CSUM | NETIF_F_HW_VLAN_TX;
Jesse Grossf01a5232011-01-09 06:23:31 +00002448 return harmonize_features(skb, protocol, features);
2449 }
Jesse Gross58e998c2010-10-29 12:14:55 +00002450}
Jesse Grossf01a5232011-01-09 06:23:31 +00002451EXPORT_SYMBOL(netif_skb_features);
Jesse Gross58e998c2010-10-29 12:14:55 +00002452
John Fastabend6afff0c2010-06-16 14:18:12 +00002453/*
2454 * Returns true if either:
2455 * 1. skb has frag_list and the device doesn't support FRAGLIST, or
Rami Rosend1a53df2012-08-27 23:39:24 +00002456 * 2. skb is fragmented and the device does not support SG.
John Fastabend6afff0c2010-06-16 14:18:12 +00002457 */
2458static inline int skb_needs_linearize(struct sk_buff *skb,
Jesse Gross02932ce2011-01-09 06:23:34 +00002459 int features)
John Fastabend6afff0c2010-06-16 14:18:12 +00002460{
Jesse Gross02932ce2011-01-09 06:23:34 +00002461 return skb_is_nonlinear(skb) &&
2462 ((skb_has_frag_list(skb) &&
2463 !(features & NETIF_F_FRAGLIST)) ||
Jesse Grosse1e78db2010-10-29 12:14:53 +00002464 (skb_shinfo(skb)->nr_frags &&
Jesse Gross02932ce2011-01-09 06:23:34 +00002465 !(features & NETIF_F_SG)));
John Fastabend6afff0c2010-06-16 14:18:12 +00002466}
2467
David S. Millerfd2ea0a2008-07-17 01:56:23 -07002468int dev_hard_start_xmit(struct sk_buff *skb, struct net_device *dev,
2469 struct netdev_queue *txq)
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002470{
Stephen Hemminger00829822008-11-20 20:14:53 -08002471 const struct net_device_ops *ops = dev->netdev_ops;
Patrick McHardy572a9d72009-11-10 06:14:14 +00002472 int rc = NETDEV_TX_OK;
Koki Sanagiec764bf2011-05-30 21:48:34 +00002473 unsigned int skb_len;
Stephen Hemminger00829822008-11-20 20:14:53 -08002474
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002475 if (likely(!skb->next)) {
Michał Mirosławc8f44af2011-11-15 15:29:55 +00002476 netdev_features_t features;
Jesse Grossfc741212011-01-09 06:23:32 +00002477
Eric Dumazet93f154b2009-05-18 22:19:19 -07002478 /*
Lucas De Marchi25985ed2011-03-30 22:57:33 -03002479 * If device doesn't need skb->dst, release it right now while
Eric Dumazet93f154b2009-05-18 22:19:19 -07002480 * its hot in this cpu cache
2481 */
Eric Dumazetadf30902009-06-02 05:19:30 +00002482 if (dev->priv_flags & IFF_XMIT_DST_RELEASE)
2483 skb_dst_drop(skb);
2484
Jesse Grossfc741212011-01-09 06:23:32 +00002485 features = netif_skb_features(skb);
2486
Jesse Gross7b9c6092010-10-20 13:56:04 +00002487 if (vlan_tx_tag_present(skb) &&
Jesse Grossfc741212011-01-09 06:23:32 +00002488 !(features & NETIF_F_HW_VLAN_TX)) {
Jesse Gross7b9c6092010-10-20 13:56:04 +00002489 skb = __vlan_put_tag(skb, vlan_tx_tag_get(skb));
2490 if (unlikely(!skb))
2491 goto out;
2492
2493 skb->vlan_tci = 0;
2494 }
2495
Alexander Duyckfc70fb62012-12-07 14:14:15 +00002496 /* If encapsulation offload request, verify we are testing
2497 * hardware encapsulation features instead of standard
2498 * features for the netdev
2499 */
2500 if (skb->encapsulation)
2501 features &= dev->hw_enc_features;
2502
Jesse Grossfc741212011-01-09 06:23:32 +00002503 if (netif_needs_gso(skb, features)) {
Jesse Gross91ecb632011-01-09 06:23:33 +00002504 if (unlikely(dev_gso_segment(skb, features)))
David S. Miller9ccb8972010-04-22 01:02:07 -07002505 goto out_kfree_skb;
2506 if (skb->next)
2507 goto gso;
John Fastabend6afff0c2010-06-16 14:18:12 +00002508 } else {
Jesse Gross02932ce2011-01-09 06:23:34 +00002509 if (skb_needs_linearize(skb, features) &&
John Fastabend6afff0c2010-06-16 14:18:12 +00002510 __skb_linearize(skb))
2511 goto out_kfree_skb;
2512
2513 /* If packet is not checksummed and device does not
2514 * support checksumming for this protocol, complete
2515 * checksumming here.
2516 */
2517 if (skb->ip_summed == CHECKSUM_PARTIAL) {
Alexander Duyckfc70fb62012-12-07 14:14:15 +00002518 if (skb->encapsulation)
2519 skb_set_inner_transport_header(skb,
2520 skb_checksum_start_offset(skb));
2521 else
2522 skb_set_transport_header(skb,
2523 skb_checksum_start_offset(skb));
Jesse Gross03634662011-01-09 06:23:35 +00002524 if (!(features & NETIF_F_ALL_CSUM) &&
John Fastabend6afff0c2010-06-16 14:18:12 +00002525 skb_checksum_help(skb))
2526 goto out_kfree_skb;
2527 }
David S. Miller9ccb8972010-04-22 01:02:07 -07002528 }
2529
Eric Dumazetb40863c2012-09-18 20:44:49 +00002530 if (!list_empty(&ptype_all))
2531 dev_queue_xmit_nit(skb, dev);
2532
Koki Sanagiec764bf2011-05-30 21:48:34 +00002533 skb_len = skb->len;
Patrick Ohlyac45f602009-02-12 05:03:37 +00002534 rc = ops->ndo_start_xmit(skb, dev);
Koki Sanagiec764bf2011-05-30 21:48:34 +00002535 trace_net_dev_xmit(skb, rc, dev, skb_len);
Patrick McHardyec634fe2009-07-05 19:23:38 -07002536 if (rc == NETDEV_TX_OK)
Eric Dumazet08baf562009-05-25 22:58:01 -07002537 txq_trans_update(txq);
Patrick Ohlyac45f602009-02-12 05:03:37 +00002538 return rc;
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002539 }
2540
Herbert Xu576a30e2006-06-27 13:22:38 -07002541gso:
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002542 do {
2543 struct sk_buff *nskb = skb->next;
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002544
2545 skb->next = nskb->next;
2546 nskb->next = NULL;
Krishna Kumar068a2de2009-12-09 20:59:58 +00002547
2548 /*
Lucas De Marchi25985ed2011-03-30 22:57:33 -03002549 * If device doesn't need nskb->dst, release it right now while
Krishna Kumar068a2de2009-12-09 20:59:58 +00002550 * its hot in this cpu cache
2551 */
2552 if (dev->priv_flags & IFF_XMIT_DST_RELEASE)
2553 skb_dst_drop(nskb);
2554
Eric Dumazetb40863c2012-09-18 20:44:49 +00002555 if (!list_empty(&ptype_all))
2556 dev_queue_xmit_nit(nskb, dev);
2557
Koki Sanagiec764bf2011-05-30 21:48:34 +00002558 skb_len = nskb->len;
Stephen Hemminger00829822008-11-20 20:14:53 -08002559 rc = ops->ndo_start_xmit(nskb, dev);
Koki Sanagiec764bf2011-05-30 21:48:34 +00002560 trace_net_dev_xmit(nskb, rc, dev, skb_len);
Patrick McHardyec634fe2009-07-05 19:23:38 -07002561 if (unlikely(rc != NETDEV_TX_OK)) {
Patrick McHardy572a9d72009-11-10 06:14:14 +00002562 if (rc & ~NETDEV_TX_MASK)
2563 goto out_kfree_gso_skb;
Michael Chanf54d9e82006-06-25 23:57:04 -07002564 nskb->next = skb->next;
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002565 skb->next = nskb;
2566 return rc;
2567 }
Eric Dumazet08baf562009-05-25 22:58:01 -07002568 txq_trans_update(txq);
Tom Herbert734664982011-11-28 16:32:44 +00002569 if (unlikely(netif_xmit_stopped(txq) && skb->next))
Michael Chanf54d9e82006-06-25 23:57:04 -07002570 return NETDEV_TX_BUSY;
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002571 } while (skb->next);
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09002572
Patrick McHardy572a9d72009-11-10 06:14:14 +00002573out_kfree_gso_skb:
2574 if (likely(skb->next == NULL))
2575 skb->destructor = DEV_GSO_CB(skb)->destructor;
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002576out_kfree_skb:
2577 kfree_skb(skb);
Jesse Gross7b9c6092010-10-20 13:56:04 +00002578out:
Patrick McHardy572a9d72009-11-10 06:14:14 +00002579 return rc;
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002580}
2581
Eric Dumazet1def9232013-01-10 12:36:42 +00002582static void qdisc_pkt_len_init(struct sk_buff *skb)
2583{
2584 const struct skb_shared_info *shinfo = skb_shinfo(skb);
2585
2586 qdisc_skb_cb(skb)->pkt_len = skb->len;
2587
2588 /* To get more precise estimation of bytes sent on wire,
2589 * we add to pkt_len the headers size of all segments
2590 */
2591 if (shinfo->gso_size) {
Eric Dumazet757b8b12013-01-15 21:14:21 -08002592 unsigned int hdr_len;
Eric Dumazet1def9232013-01-10 12:36:42 +00002593
Eric Dumazet757b8b12013-01-15 21:14:21 -08002594 /* mac layer + network layer */
2595 hdr_len = skb_transport_header(skb) - skb_mac_header(skb);
2596
2597 /* + transport layer */
Eric Dumazet1def9232013-01-10 12:36:42 +00002598 if (likely(shinfo->gso_type & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6)))
2599 hdr_len += tcp_hdrlen(skb);
2600 else
2601 hdr_len += sizeof(struct udphdr);
2602 qdisc_skb_cb(skb)->pkt_len += (shinfo->gso_segs - 1) * hdr_len;
2603 }
2604}
2605
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00002606static inline int __dev_xmit_skb(struct sk_buff *skb, struct Qdisc *q,
2607 struct net_device *dev,
2608 struct netdev_queue *txq)
2609{
2610 spinlock_t *root_lock = qdisc_lock(q);
Eric Dumazeta2da5702011-01-20 03:48:19 +00002611 bool contended;
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00002612 int rc;
2613
Eric Dumazet1def9232013-01-10 12:36:42 +00002614 qdisc_pkt_len_init(skb);
Eric Dumazeta2da5702011-01-20 03:48:19 +00002615 qdisc_calculate_pkt_len(skb, q);
Eric Dumazet79640a42010-06-02 05:09:29 -07002616 /*
2617 * Heuristic to force contended enqueues to serialize on a
2618 * separate lock before trying to get qdisc main lock.
2619 * This permits __QDISC_STATE_RUNNING owner to get the lock more often
2620 * and dequeue packets faster.
2621 */
Eric Dumazeta2da5702011-01-20 03:48:19 +00002622 contended = qdisc_is_running(q);
Eric Dumazet79640a42010-06-02 05:09:29 -07002623 if (unlikely(contended))
2624 spin_lock(&q->busylock);
2625
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00002626 spin_lock(root_lock);
2627 if (unlikely(test_bit(__QDISC_STATE_DEACTIVATED, &q->state))) {
2628 kfree_skb(skb);
2629 rc = NET_XMIT_DROP;
2630 } else if ((q->flags & TCQ_F_CAN_BYPASS) && !qdisc_qlen(q) &&
Eric Dumazetbc135b22010-06-02 03:23:51 -07002631 qdisc_run_begin(q)) {
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00002632 /*
2633 * This is a work-conserving queue; there are no old skbs
2634 * waiting to be sent out; and the qdisc is not running -
2635 * xmit the skb directly.
2636 */
Eric Dumazet7fee2262010-05-11 23:19:48 +00002637 if (!(dev->priv_flags & IFF_XMIT_DST_RELEASE))
2638 skb_dst_force(skb);
Eric Dumazetbfe0d022011-01-09 08:30:54 +00002639
Eric Dumazetbfe0d022011-01-09 08:30:54 +00002640 qdisc_bstats_update(q, skb);
2641
Eric Dumazet79640a42010-06-02 05:09:29 -07002642 if (sch_direct_xmit(skb, q, dev, txq, root_lock)) {
2643 if (unlikely(contended)) {
2644 spin_unlock(&q->busylock);
2645 contended = false;
2646 }
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00002647 __qdisc_run(q);
Eric Dumazet79640a42010-06-02 05:09:29 -07002648 } else
Eric Dumazetbc135b22010-06-02 03:23:51 -07002649 qdisc_run_end(q);
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00002650
2651 rc = NET_XMIT_SUCCESS;
2652 } else {
Eric Dumazet7fee2262010-05-11 23:19:48 +00002653 skb_dst_force(skb);
Eric Dumazeta2da5702011-01-20 03:48:19 +00002654 rc = q->enqueue(skb, q) & NET_XMIT_MASK;
Eric Dumazet79640a42010-06-02 05:09:29 -07002655 if (qdisc_run_begin(q)) {
2656 if (unlikely(contended)) {
2657 spin_unlock(&q->busylock);
2658 contended = false;
2659 }
2660 __qdisc_run(q);
2661 }
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00002662 }
2663 spin_unlock(root_lock);
Eric Dumazet79640a42010-06-02 05:09:29 -07002664 if (unlikely(contended))
2665 spin_unlock(&q->busylock);
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00002666 return rc;
2667}
2668
Neil Horman5bc14212011-11-22 05:10:51 +00002669#if IS_ENABLED(CONFIG_NETPRIO_CGROUP)
2670static void skb_update_prio(struct sk_buff *skb)
2671{
Igor Maravic6977a792011-11-25 07:44:54 +00002672 struct netprio_map *map = rcu_dereference_bh(skb->dev->priomap);
Neil Horman5bc14212011-11-22 05:10:51 +00002673
Eric Dumazet91c68ce2012-07-08 21:45:10 +00002674 if (!skb->priority && skb->sk && map) {
2675 unsigned int prioidx = skb->sk->sk_cgrp_prioidx;
2676
2677 if (prioidx < map->priomap_len)
2678 skb->priority = map->priomap[prioidx];
2679 }
Neil Horman5bc14212011-11-22 05:10:51 +00002680}
2681#else
2682#define skb_update_prio(skb)
2683#endif
2684
Eric Dumazet745e20f2010-09-29 13:23:09 -07002685static DEFINE_PER_CPU(int, xmit_recursion);
David S. Miller11a766c2010-10-25 12:51:55 -07002686#define RECURSION_LIMIT 10
Eric Dumazet745e20f2010-09-29 13:23:09 -07002687
Dave Jonesd29f7492008-07-22 14:09:06 -07002688/**
Michel Machado95603e22012-06-12 10:16:35 +00002689 * dev_loopback_xmit - loop back @skb
2690 * @skb: buffer to transmit
2691 */
2692int dev_loopback_xmit(struct sk_buff *skb)
2693{
2694 skb_reset_mac_header(skb);
2695 __skb_pull(skb, skb_network_offset(skb));
2696 skb->pkt_type = PACKET_LOOPBACK;
2697 skb->ip_summed = CHECKSUM_UNNECESSARY;
2698 WARN_ON(!skb_dst(skb));
2699 skb_dst_force(skb);
2700 netif_rx_ni(skb);
2701 return 0;
2702}
2703EXPORT_SYMBOL(dev_loopback_xmit);
2704
2705/**
Dave Jonesd29f7492008-07-22 14:09:06 -07002706 * dev_queue_xmit - transmit a buffer
2707 * @skb: buffer to transmit
2708 *
2709 * Queue a buffer for transmission to a network device. The caller must
2710 * have set the device and priority and built the buffer before calling
2711 * this function. The function can be called from an interrupt.
2712 *
2713 * A negative errno code is returned on a failure. A success does not
2714 * guarantee the frame will be transmitted as it may be dropped due
2715 * to congestion or traffic shaping.
2716 *
2717 * -----------------------------------------------------------------------------------
2718 * I notice this method can also return errors from the queue disciplines,
2719 * including NET_XMIT_DROP, which is a positive value. So, errors can also
2720 * be positive.
2721 *
2722 * Regardless of the return value, the skb is consumed, so it is currently
2723 * difficult to retry a send to this method. (You can bump the ref count
2724 * before sending to hold a reference for retry if you are careful.)
2725 *
2726 * When calling this method, interrupts MUST be enabled. This is because
2727 * the BH enable code must have IRQs enabled so that it will not deadlock.
2728 * --BLG
2729 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07002730int dev_queue_xmit(struct sk_buff *skb)
2731{
2732 struct net_device *dev = skb->dev;
David S. Millerdc2b4842008-07-08 17:18:23 -07002733 struct netdev_queue *txq;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002734 struct Qdisc *q;
2735 int rc = -ENOMEM;
2736
Eric Dumazet6d1ccff2013-02-05 20:22:20 +00002737 skb_reset_mac_header(skb);
2738
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09002739 /* Disable soft irqs for various locks below. Also
2740 * stops preemption for RCU.
Linus Torvalds1da177e2005-04-16 15:20:36 -07002741 */
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09002742 rcu_read_lock_bh();
Linus Torvalds1da177e2005-04-16 15:20:36 -07002743
Neil Horman5bc14212011-11-22 05:10:51 +00002744 skb_update_prio(skb);
2745
Amerigo Wang8c4c49d2012-09-17 20:16:31 +00002746 txq = netdev_pick_tx(dev, skb);
Paul E. McKenneya898def2010-02-22 17:04:49 -08002747 q = rcu_dereference_bh(txq->qdisc);
David S. Miller37437bb2008-07-16 02:15:04 -07002748
Linus Torvalds1da177e2005-04-16 15:20:36 -07002749#ifdef CONFIG_NET_CLS_ACT
Eric Dumazetd1b19df2009-09-03 01:29:39 -07002750 skb->tc_verd = SET_TC_AT(skb->tc_verd, AT_EGRESS);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002751#endif
Koki Sanagicf66ba52010-08-23 18:45:02 +09002752 trace_net_dev_queue(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002753 if (q->enqueue) {
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00002754 rc = __dev_xmit_skb(skb, q, dev, txq);
David S. Miller37437bb2008-07-16 02:15:04 -07002755 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002756 }
2757
2758 /* The device has no queue. Common case for software devices:
2759 loopback, all the sorts of tunnels...
2760
Herbert Xu932ff272006-06-09 12:20:56 -07002761 Really, it is unlikely that netif_tx_lock protection is necessary
2762 here. (f.e. loopback and IP tunnels are clean ignoring statistics
Linus Torvalds1da177e2005-04-16 15:20:36 -07002763 counters.)
2764 However, it is possible, that they rely on protection
2765 made by us here.
2766
2767 Check this and shot the lock. It is not prone from deadlocks.
2768 Either shot noqueue qdisc, it is even simpler 8)
2769 */
2770 if (dev->flags & IFF_UP) {
2771 int cpu = smp_processor_id(); /* ok because BHs are off */
2772
David S. Millerc773e842008-07-08 23:13:53 -07002773 if (txq->xmit_lock_owner != cpu) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07002774
Eric Dumazet745e20f2010-09-29 13:23:09 -07002775 if (__this_cpu_read(xmit_recursion) > RECURSION_LIMIT)
2776 goto recursion_alert;
2777
David S. Millerc773e842008-07-08 23:13:53 -07002778 HARD_TX_LOCK(dev, txq, cpu);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002779
Tom Herbert734664982011-11-28 16:32:44 +00002780 if (!netif_xmit_stopped(txq)) {
Eric Dumazet745e20f2010-09-29 13:23:09 -07002781 __this_cpu_inc(xmit_recursion);
Patrick McHardy572a9d72009-11-10 06:14:14 +00002782 rc = dev_hard_start_xmit(skb, dev, txq);
Eric Dumazet745e20f2010-09-29 13:23:09 -07002783 __this_cpu_dec(xmit_recursion);
Patrick McHardy572a9d72009-11-10 06:14:14 +00002784 if (dev_xmit_complete(rc)) {
David S. Millerc773e842008-07-08 23:13:53 -07002785 HARD_TX_UNLOCK(dev, txq);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002786 goto out;
2787 }
2788 }
David S. Millerc773e842008-07-08 23:13:53 -07002789 HARD_TX_UNLOCK(dev, txq);
Joe Perchese87cc472012-05-13 21:56:26 +00002790 net_crit_ratelimited("Virtual device %s asks to queue packet!\n",
2791 dev->name);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002792 } else {
2793 /* Recursion is detected! It is possible,
Eric Dumazet745e20f2010-09-29 13:23:09 -07002794 * unfortunately
2795 */
2796recursion_alert:
Joe Perchese87cc472012-05-13 21:56:26 +00002797 net_crit_ratelimited("Dead loop on virtual device %s, fix it urgently!\n",
2798 dev->name);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002799 }
2800 }
2801
2802 rc = -ENETDOWN;
Herbert Xud4828d82006-06-22 02:28:18 -07002803 rcu_read_unlock_bh();
Linus Torvalds1da177e2005-04-16 15:20:36 -07002804
Linus Torvalds1da177e2005-04-16 15:20:36 -07002805 kfree_skb(skb);
2806 return rc;
2807out:
Herbert Xud4828d82006-06-22 02:28:18 -07002808 rcu_read_unlock_bh();
Linus Torvalds1da177e2005-04-16 15:20:36 -07002809 return rc;
2810}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07002811EXPORT_SYMBOL(dev_queue_xmit);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002812
2813
2814/*=======================================================================
2815 Receiver routines
2816 =======================================================================*/
2817
Stephen Hemminger6b2bedc2007-03-12 14:33:50 -07002818int netdev_max_backlog __read_mostly = 1000;
Eric Dumazetc9e6bc62012-09-27 19:29:05 +00002819EXPORT_SYMBOL(netdev_max_backlog);
2820
Eric Dumazet3b098e22010-05-15 23:57:10 -07002821int netdev_tstamp_prequeue __read_mostly = 1;
Stephen Hemminger6b2bedc2007-03-12 14:33:50 -07002822int netdev_budget __read_mostly = 300;
2823int weight_p __read_mostly = 64; /* old backlog weight */
Linus Torvalds1da177e2005-04-16 15:20:36 -07002824
Eric Dumazeteecfd7c2010-05-06 22:07:48 -07002825/* Called with irq disabled */
2826static inline void ____napi_schedule(struct softnet_data *sd,
2827 struct napi_struct *napi)
2828{
2829 list_add_tail(&napi->poll_list, &sd->poll_list);
2830 __raise_softirq_irqoff(NET_RX_SOFTIRQ);
2831}
2832
Eric Dumazetdf334542010-03-24 19:13:54 +00002833#ifdef CONFIG_RPS
Tom Herbertfec5e652010-04-16 16:01:27 -07002834
2835/* One global table that all flow-based protocols share. */
Eric Dumazet6e3f7fa2010-10-25 03:02:02 +00002836struct rps_sock_flow_table __rcu *rps_sock_flow_table __read_mostly;
Tom Herbertfec5e652010-04-16 16:01:27 -07002837EXPORT_SYMBOL(rps_sock_flow_table);
2838
Ingo Molnarc5905af2012-02-24 08:31:31 +01002839struct static_key rps_needed __read_mostly;
Eric Dumazetadc93002011-11-17 03:13:26 +00002840
Ben Hutchingsc4454772011-01-19 11:03:53 +00002841static struct rps_dev_flow *
2842set_rps_cpu(struct net_device *dev, struct sk_buff *skb,
2843 struct rps_dev_flow *rflow, u16 next_cpu)
2844{
Ben Hutchings09994d12011-10-03 04:42:46 +00002845 if (next_cpu != RPS_NO_CPU) {
Ben Hutchingsc4454772011-01-19 11:03:53 +00002846#ifdef CONFIG_RFS_ACCEL
2847 struct netdev_rx_queue *rxqueue;
2848 struct rps_dev_flow_table *flow_table;
2849 struct rps_dev_flow *old_rflow;
2850 u32 flow_id;
2851 u16 rxq_index;
2852 int rc;
2853
2854 /* Should we steer this flow to a different hardware queue? */
Ben Hutchings69a19ee2011-02-15 20:32:04 +00002855 if (!skb_rx_queue_recorded(skb) || !dev->rx_cpu_rmap ||
2856 !(dev->features & NETIF_F_NTUPLE))
Ben Hutchingsc4454772011-01-19 11:03:53 +00002857 goto out;
2858 rxq_index = cpu_rmap_lookup_index(dev->rx_cpu_rmap, next_cpu);
2859 if (rxq_index == skb_get_rx_queue(skb))
2860 goto out;
2861
2862 rxqueue = dev->_rx + rxq_index;
2863 flow_table = rcu_dereference(rxqueue->rps_flow_table);
2864 if (!flow_table)
2865 goto out;
2866 flow_id = skb->rxhash & flow_table->mask;
2867 rc = dev->netdev_ops->ndo_rx_flow_steer(dev, skb,
2868 rxq_index, flow_id);
2869 if (rc < 0)
2870 goto out;
2871 old_rflow = rflow;
2872 rflow = &flow_table->flows[flow_id];
Ben Hutchingsc4454772011-01-19 11:03:53 +00002873 rflow->filter = rc;
2874 if (old_rflow->filter == rflow->filter)
2875 old_rflow->filter = RPS_NO_FILTER;
2876 out:
2877#endif
2878 rflow->last_qtail =
Ben Hutchings09994d12011-10-03 04:42:46 +00002879 per_cpu(softnet_data, next_cpu).input_queue_head;
Ben Hutchingsc4454772011-01-19 11:03:53 +00002880 }
2881
Ben Hutchings09994d12011-10-03 04:42:46 +00002882 rflow->cpu = next_cpu;
Ben Hutchingsc4454772011-01-19 11:03:53 +00002883 return rflow;
2884}
2885
Tom Herbert0a9627f2010-03-16 08:03:29 +00002886/*
2887 * get_rps_cpu is called from netif_receive_skb and returns the target
2888 * CPU from the RPS map of the receiving queue for a given skb.
Eric Dumazetb0e28f12010-04-15 00:14:07 -07002889 * rcu_read_lock must be held on entry.
Tom Herbert0a9627f2010-03-16 08:03:29 +00002890 */
Tom Herbertfec5e652010-04-16 16:01:27 -07002891static int get_rps_cpu(struct net_device *dev, struct sk_buff *skb,
2892 struct rps_dev_flow **rflowp)
Tom Herbert0a9627f2010-03-16 08:03:29 +00002893{
Tom Herbert0a9627f2010-03-16 08:03:29 +00002894 struct netdev_rx_queue *rxqueue;
Eric Dumazet6e3f7fa2010-10-25 03:02:02 +00002895 struct rps_map *map;
Tom Herbertfec5e652010-04-16 16:01:27 -07002896 struct rps_dev_flow_table *flow_table;
2897 struct rps_sock_flow_table *sock_flow_table;
Tom Herbert0a9627f2010-03-16 08:03:29 +00002898 int cpu = -1;
Tom Herbertfec5e652010-04-16 16:01:27 -07002899 u16 tcpu;
Tom Herbert0a9627f2010-03-16 08:03:29 +00002900
Tom Herbert0a9627f2010-03-16 08:03:29 +00002901 if (skb_rx_queue_recorded(skb)) {
2902 u16 index = skb_get_rx_queue(skb);
Ben Hutchings62fe0b42010-09-27 08:24:33 +00002903 if (unlikely(index >= dev->real_num_rx_queues)) {
2904 WARN_ONCE(dev->real_num_rx_queues > 1,
2905 "%s received packet on queue %u, but number "
2906 "of RX queues is %u\n",
2907 dev->name, index, dev->real_num_rx_queues);
Tom Herbert0a9627f2010-03-16 08:03:29 +00002908 goto done;
2909 }
2910 rxqueue = dev->_rx + index;
2911 } else
2912 rxqueue = dev->_rx;
2913
Eric Dumazet6e3f7fa2010-10-25 03:02:02 +00002914 map = rcu_dereference(rxqueue->rps_map);
2915 if (map) {
Tom Herbert85875232011-01-31 16:23:42 -08002916 if (map->len == 1 &&
Eric Dumazet33d480c2011-08-11 19:30:52 +00002917 !rcu_access_pointer(rxqueue->rps_flow_table)) {
Changli Gao6febfca2010-09-03 23:12:37 +00002918 tcpu = map->cpus[0];
2919 if (cpu_online(tcpu))
2920 cpu = tcpu;
Tom Herbert0a9627f2010-03-16 08:03:29 +00002921 goto done;
Eric Dumazetb249dcb2010-04-19 21:56:38 +00002922 }
Eric Dumazet33d480c2011-08-11 19:30:52 +00002923 } else if (!rcu_access_pointer(rxqueue->rps_flow_table)) {
Tom Herbert0a9627f2010-03-16 08:03:29 +00002924 goto done;
Tom Herbert0a9627f2010-03-16 08:03:29 +00002925 }
2926
Changli Gao2d47b452010-08-17 19:00:56 +00002927 skb_reset_network_header(skb);
Krishna Kumarbfb564e2010-08-04 06:15:52 +00002928 if (!skb_get_rxhash(skb))
Tom Herbert0a9627f2010-03-16 08:03:29 +00002929 goto done;
Tom Herbert0a9627f2010-03-16 08:03:29 +00002930
Tom Herbertfec5e652010-04-16 16:01:27 -07002931 flow_table = rcu_dereference(rxqueue->rps_flow_table);
2932 sock_flow_table = rcu_dereference(rps_sock_flow_table);
2933 if (flow_table && sock_flow_table) {
2934 u16 next_cpu;
2935 struct rps_dev_flow *rflow;
2936
2937 rflow = &flow_table->flows[skb->rxhash & flow_table->mask];
2938 tcpu = rflow->cpu;
2939
2940 next_cpu = sock_flow_table->ents[skb->rxhash &
2941 sock_flow_table->mask];
2942
2943 /*
2944 * If the desired CPU (where last recvmsg was done) is
2945 * different from current CPU (one in the rx-queue flow
2946 * table entry), switch if one of the following holds:
2947 * - Current CPU is unset (equal to RPS_NO_CPU).
2948 * - Current CPU is offline.
2949 * - The current CPU's queue tail has advanced beyond the
2950 * last packet that was enqueued using this table entry.
2951 * This guarantees that all previous packets for the flow
2952 * have been dequeued, thus preserving in order delivery.
2953 */
2954 if (unlikely(tcpu != next_cpu) &&
2955 (tcpu == RPS_NO_CPU || !cpu_online(tcpu) ||
2956 ((int)(per_cpu(softnet_data, tcpu).input_queue_head -
Tom Herbertbaefa312012-11-16 09:04:15 +00002957 rflow->last_qtail)) >= 0)) {
2958 tcpu = next_cpu;
Ben Hutchingsc4454772011-01-19 11:03:53 +00002959 rflow = set_rps_cpu(dev, skb, rflow, next_cpu);
Tom Herbertbaefa312012-11-16 09:04:15 +00002960 }
Ben Hutchingsc4454772011-01-19 11:03:53 +00002961
Tom Herbertfec5e652010-04-16 16:01:27 -07002962 if (tcpu != RPS_NO_CPU && cpu_online(tcpu)) {
2963 *rflowp = rflow;
2964 cpu = tcpu;
2965 goto done;
2966 }
2967 }
2968
Tom Herbert0a9627f2010-03-16 08:03:29 +00002969 if (map) {
Tom Herbertfec5e652010-04-16 16:01:27 -07002970 tcpu = map->cpus[((u64) skb->rxhash * map->len) >> 32];
Tom Herbert0a9627f2010-03-16 08:03:29 +00002971
2972 if (cpu_online(tcpu)) {
2973 cpu = tcpu;
2974 goto done;
2975 }
2976 }
2977
2978done:
Tom Herbert0a9627f2010-03-16 08:03:29 +00002979 return cpu;
2980}
2981
Ben Hutchingsc4454772011-01-19 11:03:53 +00002982#ifdef CONFIG_RFS_ACCEL
2983
2984/**
2985 * rps_may_expire_flow - check whether an RFS hardware filter may be removed
2986 * @dev: Device on which the filter was set
2987 * @rxq_index: RX queue index
2988 * @flow_id: Flow ID passed to ndo_rx_flow_steer()
2989 * @filter_id: Filter ID returned by ndo_rx_flow_steer()
2990 *
2991 * Drivers that implement ndo_rx_flow_steer() should periodically call
2992 * this function for each installed filter and remove the filters for
2993 * which it returns %true.
2994 */
2995bool rps_may_expire_flow(struct net_device *dev, u16 rxq_index,
2996 u32 flow_id, u16 filter_id)
2997{
2998 struct netdev_rx_queue *rxqueue = dev->_rx + rxq_index;
2999 struct rps_dev_flow_table *flow_table;
3000 struct rps_dev_flow *rflow;
3001 bool expire = true;
3002 int cpu;
3003
3004 rcu_read_lock();
3005 flow_table = rcu_dereference(rxqueue->rps_flow_table);
3006 if (flow_table && flow_id <= flow_table->mask) {
3007 rflow = &flow_table->flows[flow_id];
3008 cpu = ACCESS_ONCE(rflow->cpu);
3009 if (rflow->filter == filter_id && cpu != RPS_NO_CPU &&
3010 ((int)(per_cpu(softnet_data, cpu).input_queue_head -
3011 rflow->last_qtail) <
3012 (int)(10 * flow_table->mask)))
3013 expire = false;
3014 }
3015 rcu_read_unlock();
3016 return expire;
3017}
3018EXPORT_SYMBOL(rps_may_expire_flow);
3019
3020#endif /* CONFIG_RFS_ACCEL */
3021
Tom Herbert0a9627f2010-03-16 08:03:29 +00003022/* Called from hardirq (IPI) context */
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003023static void rps_trigger_softirq(void *data)
Tom Herbert0a9627f2010-03-16 08:03:29 +00003024{
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003025 struct softnet_data *sd = data;
3026
Eric Dumazeteecfd7c2010-05-06 22:07:48 -07003027 ____napi_schedule(sd, &sd->backlog);
Changli Gaodee42872010-05-02 05:42:16 +00003028 sd->received_rps++;
Tom Herbert0a9627f2010-03-16 08:03:29 +00003029}
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003030
Tom Herbertfec5e652010-04-16 16:01:27 -07003031#endif /* CONFIG_RPS */
Tom Herbert0a9627f2010-03-16 08:03:29 +00003032
3033/*
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003034 * Check if this softnet_data structure is another cpu one
3035 * If yes, queue it to our IPI list and return 1
3036 * If no, return 0
3037 */
3038static int rps_ipi_queued(struct softnet_data *sd)
3039{
3040#ifdef CONFIG_RPS
3041 struct softnet_data *mysd = &__get_cpu_var(softnet_data);
3042
3043 if (sd != mysd) {
3044 sd->rps_ipi_next = mysd->rps_ipi_list;
3045 mysd->rps_ipi_list = sd;
3046
3047 __raise_softirq_irqoff(NET_RX_SOFTIRQ);
3048 return 1;
3049 }
3050#endif /* CONFIG_RPS */
3051 return 0;
3052}
3053
3054/*
Tom Herbert0a9627f2010-03-16 08:03:29 +00003055 * enqueue_to_backlog is called to queue an skb to a per CPU backlog
3056 * queue (may be a remote CPU queue).
3057 */
Tom Herbertfec5e652010-04-16 16:01:27 -07003058static int enqueue_to_backlog(struct sk_buff *skb, int cpu,
3059 unsigned int *qtail)
Tom Herbert0a9627f2010-03-16 08:03:29 +00003060{
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003061 struct softnet_data *sd;
Tom Herbert0a9627f2010-03-16 08:03:29 +00003062 unsigned long flags;
3063
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003064 sd = &per_cpu(softnet_data, cpu);
Tom Herbert0a9627f2010-03-16 08:03:29 +00003065
3066 local_irq_save(flags);
Tom Herbert0a9627f2010-03-16 08:03:29 +00003067
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003068 rps_lock(sd);
Changli Gao6e7676c2010-04-27 15:07:33 -07003069 if (skb_queue_len(&sd->input_pkt_queue) <= netdev_max_backlog) {
3070 if (skb_queue_len(&sd->input_pkt_queue)) {
Tom Herbert0a9627f2010-03-16 08:03:29 +00003071enqueue:
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003072 __skb_queue_tail(&sd->input_pkt_queue, skb);
Tom Herbert76cc8b12010-05-20 18:37:59 +00003073 input_queue_tail_incr_save(sd, qtail);
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003074 rps_unlock(sd);
Changli Gao152102c2010-03-30 20:16:22 +00003075 local_irq_restore(flags);
Tom Herbert0a9627f2010-03-16 08:03:29 +00003076 return NET_RX_SUCCESS;
3077 }
3078
Eric Dumazetebda37c22010-05-06 23:51:21 +00003079 /* Schedule NAPI for backlog device
3080 * We can use non atomic operation since we own the queue lock
3081 */
3082 if (!__test_and_set_bit(NAPI_STATE_SCHED, &sd->backlog.state)) {
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003083 if (!rps_ipi_queued(sd))
Eric Dumazeteecfd7c2010-05-06 22:07:48 -07003084 ____napi_schedule(sd, &sd->backlog);
Tom Herbert0a9627f2010-03-16 08:03:29 +00003085 }
3086 goto enqueue;
3087 }
3088
Changli Gaodee42872010-05-02 05:42:16 +00003089 sd->dropped++;
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003090 rps_unlock(sd);
Tom Herbert0a9627f2010-03-16 08:03:29 +00003091
Tom Herbert0a9627f2010-03-16 08:03:29 +00003092 local_irq_restore(flags);
3093
Eric Dumazetcaf586e2010-09-30 21:06:55 +00003094 atomic_long_inc(&skb->dev->rx_dropped);
Tom Herbert0a9627f2010-03-16 08:03:29 +00003095 kfree_skb(skb);
3096 return NET_RX_DROP;
3097}
Linus Torvalds1da177e2005-04-16 15:20:36 -07003098
Linus Torvalds1da177e2005-04-16 15:20:36 -07003099/**
3100 * netif_rx - post buffer to the network code
3101 * @skb: buffer to post
3102 *
3103 * This function receives a packet from a device driver and queues it for
3104 * the upper (protocol) levels to process. It always succeeds. The buffer
3105 * may be dropped during processing for congestion control or by the
3106 * protocol layers.
3107 *
3108 * return values:
3109 * NET_RX_SUCCESS (no congestion)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003110 * NET_RX_DROP (packet was dropped)
3111 *
3112 */
3113
3114int netif_rx(struct sk_buff *skb)
3115{
Eric Dumazetb0e28f12010-04-15 00:14:07 -07003116 int ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003117
3118 /* if netpoll wants it, pretend we never saw it */
3119 if (netpoll_rx(skb))
3120 return NET_RX_DROP;
3121
Eric Dumazet588f0332011-11-15 04:12:55 +00003122 net_timestamp_check(netdev_tstamp_prequeue, skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003123
Koki Sanagicf66ba52010-08-23 18:45:02 +09003124 trace_netif_rx(skb);
Eric Dumazetdf334542010-03-24 19:13:54 +00003125#ifdef CONFIG_RPS
Ingo Molnarc5905af2012-02-24 08:31:31 +01003126 if (static_key_false(&rps_needed)) {
Tom Herbertfec5e652010-04-16 16:01:27 -07003127 struct rps_dev_flow voidflow, *rflow = &voidflow;
Eric Dumazetb0e28f12010-04-15 00:14:07 -07003128 int cpu;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003129
Changli Gaocece1942010-08-07 20:35:43 -07003130 preempt_disable();
Eric Dumazetb0e28f12010-04-15 00:14:07 -07003131 rcu_read_lock();
Tom Herbertfec5e652010-04-16 16:01:27 -07003132
3133 cpu = get_rps_cpu(skb->dev, skb, &rflow);
Eric Dumazetb0e28f12010-04-15 00:14:07 -07003134 if (cpu < 0)
3135 cpu = smp_processor_id();
Tom Herbertfec5e652010-04-16 16:01:27 -07003136
3137 ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail);
3138
Eric Dumazetb0e28f12010-04-15 00:14:07 -07003139 rcu_read_unlock();
Changli Gaocece1942010-08-07 20:35:43 -07003140 preempt_enable();
Eric Dumazetadc93002011-11-17 03:13:26 +00003141 } else
3142#endif
Tom Herbertfec5e652010-04-16 16:01:27 -07003143 {
3144 unsigned int qtail;
3145 ret = enqueue_to_backlog(skb, get_cpu(), &qtail);
3146 put_cpu();
3147 }
Eric Dumazetb0e28f12010-04-15 00:14:07 -07003148 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003149}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07003150EXPORT_SYMBOL(netif_rx);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003151
3152int netif_rx_ni(struct sk_buff *skb)
3153{
3154 int err;
3155
3156 preempt_disable();
3157 err = netif_rx(skb);
3158 if (local_softirq_pending())
3159 do_softirq();
3160 preempt_enable();
3161
3162 return err;
3163}
Linus Torvalds1da177e2005-04-16 15:20:36 -07003164EXPORT_SYMBOL(netif_rx_ni);
3165
Linus Torvalds1da177e2005-04-16 15:20:36 -07003166static void net_tx_action(struct softirq_action *h)
3167{
3168 struct softnet_data *sd = &__get_cpu_var(softnet_data);
3169
3170 if (sd->completion_queue) {
3171 struct sk_buff *clist;
3172
3173 local_irq_disable();
3174 clist = sd->completion_queue;
3175 sd->completion_queue = NULL;
3176 local_irq_enable();
3177
3178 while (clist) {
3179 struct sk_buff *skb = clist;
3180 clist = clist->next;
3181
Ilpo Järvinen547b7922008-07-25 21:43:18 -07003182 WARN_ON(atomic_read(&skb->users));
Koki Sanagi07dc22e2010-08-23 18:46:12 +09003183 trace_kfree_skb(skb, net_tx_action);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003184 __kfree_skb(skb);
3185 }
3186 }
3187
3188 if (sd->output_queue) {
David S. Miller37437bb2008-07-16 02:15:04 -07003189 struct Qdisc *head;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003190
3191 local_irq_disable();
3192 head = sd->output_queue;
3193 sd->output_queue = NULL;
Changli Gaoa9cbd582010-04-26 23:06:24 +00003194 sd->output_queue_tailp = &sd->output_queue;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003195 local_irq_enable();
3196
3197 while (head) {
David S. Miller37437bb2008-07-16 02:15:04 -07003198 struct Qdisc *q = head;
3199 spinlock_t *root_lock;
3200
Linus Torvalds1da177e2005-04-16 15:20:36 -07003201 head = head->next_sched;
3202
David S. Miller5fb66222008-08-02 20:02:43 -07003203 root_lock = qdisc_lock(q);
David S. Miller37437bb2008-07-16 02:15:04 -07003204 if (spin_trylock(root_lock)) {
Jarek Poplawskidef82a12008-08-17 21:54:43 -07003205 smp_mb__before_clear_bit();
3206 clear_bit(__QDISC_STATE_SCHED,
3207 &q->state);
David S. Miller37437bb2008-07-16 02:15:04 -07003208 qdisc_run(q);
3209 spin_unlock(root_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003210 } else {
David S. Miller195648b2008-08-19 04:00:36 -07003211 if (!test_bit(__QDISC_STATE_DEACTIVATED,
Jarek Poplawskie8a83e12008-09-07 18:41:21 -07003212 &q->state)) {
David S. Miller195648b2008-08-19 04:00:36 -07003213 __netif_reschedule(q);
Jarek Poplawskie8a83e12008-09-07 18:41:21 -07003214 } else {
3215 smp_mb__before_clear_bit();
3216 clear_bit(__QDISC_STATE_SCHED,
3217 &q->state);
3218 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07003219 }
3220 }
3221 }
3222}
3223
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00003224#if (defined(CONFIG_BRIDGE) || defined(CONFIG_BRIDGE_MODULE)) && \
3225 (defined(CONFIG_ATM_LANE) || defined(CONFIG_ATM_LANE_MODULE))
Michał Mirosławda678292009-06-05 05:35:28 +00003226/* This hook is defined here for ATM LANE */
3227int (*br_fdb_test_addr_hook)(struct net_device *dev,
3228 unsigned char *addr) __read_mostly;
Stephen Hemminger4fb019a2009-09-11 11:50:08 -07003229EXPORT_SYMBOL_GPL(br_fdb_test_addr_hook);
Michał Mirosławda678292009-06-05 05:35:28 +00003230#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07003231
Linus Torvalds1da177e2005-04-16 15:20:36 -07003232#ifdef CONFIG_NET_CLS_ACT
3233/* TODO: Maybe we should just force sch_ingress to be compiled in
3234 * when CONFIG_NET_CLS_ACT is? otherwise some useless instructions
3235 * a compare and 2 stores extra right now if we dont have it on
3236 * but have CONFIG_NET_CLS_ACT
Lucas De Marchi25985ed2011-03-30 22:57:33 -03003237 * NOTE: This doesn't stop any functionality; if you dont have
3238 * the ingress scheduler, you just can't add policies on ingress.
Linus Torvalds1da177e2005-04-16 15:20:36 -07003239 *
3240 */
Eric Dumazet24824a02010-10-02 06:11:55 +00003241static int ing_filter(struct sk_buff *skb, struct netdev_queue *rxq)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003242{
Linus Torvalds1da177e2005-04-16 15:20:36 -07003243 struct net_device *dev = skb->dev;
Herbert Xuf697c3e2007-10-14 00:38:47 -07003244 u32 ttl = G_TC_RTTL(skb->tc_verd);
David S. Miller555353c2008-07-08 17:33:13 -07003245 int result = TC_ACT_OK;
3246 struct Qdisc *q;
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09003247
Stephen Hemmingerde384832010-08-01 00:33:23 -07003248 if (unlikely(MAX_RED_LOOP < ttl++)) {
Joe Perchese87cc472012-05-13 21:56:26 +00003249 net_warn_ratelimited("Redir loop detected Dropping packet (%d->%d)\n",
3250 skb->skb_iif, dev->ifindex);
Herbert Xuf697c3e2007-10-14 00:38:47 -07003251 return TC_ACT_SHOT;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003252 }
3253
Herbert Xuf697c3e2007-10-14 00:38:47 -07003254 skb->tc_verd = SET_TC_RTTL(skb->tc_verd, ttl);
3255 skb->tc_verd = SET_TC_AT(skb->tc_verd, AT_INGRESS);
3256
David S. Miller83874002008-07-17 00:53:03 -07003257 q = rxq->qdisc;
David S. Miller8d50b532008-07-30 02:37:46 -07003258 if (q != &noop_qdisc) {
David S. Miller83874002008-07-17 00:53:03 -07003259 spin_lock(qdisc_lock(q));
David S. Millera9312ae2008-08-17 21:51:03 -07003260 if (likely(!test_bit(__QDISC_STATE_DEACTIVATED, &q->state)))
3261 result = qdisc_enqueue_root(skb, q);
David S. Miller83874002008-07-17 00:53:03 -07003262 spin_unlock(qdisc_lock(q));
3263 }
Herbert Xuf697c3e2007-10-14 00:38:47 -07003264
Linus Torvalds1da177e2005-04-16 15:20:36 -07003265 return result;
3266}
Herbert Xuf697c3e2007-10-14 00:38:47 -07003267
3268static inline struct sk_buff *handle_ing(struct sk_buff *skb,
3269 struct packet_type **pt_prev,
3270 int *ret, struct net_device *orig_dev)
3271{
Eric Dumazet24824a02010-10-02 06:11:55 +00003272 struct netdev_queue *rxq = rcu_dereference(skb->dev->ingress_queue);
3273
3274 if (!rxq || rxq->qdisc == &noop_qdisc)
Herbert Xuf697c3e2007-10-14 00:38:47 -07003275 goto out;
3276
3277 if (*pt_prev) {
3278 *ret = deliver_skb(skb, *pt_prev, orig_dev);
3279 *pt_prev = NULL;
Herbert Xuf697c3e2007-10-14 00:38:47 -07003280 }
3281
Eric Dumazet24824a02010-10-02 06:11:55 +00003282 switch (ing_filter(skb, rxq)) {
Herbert Xuf697c3e2007-10-14 00:38:47 -07003283 case TC_ACT_SHOT:
3284 case TC_ACT_STOLEN:
3285 kfree_skb(skb);
3286 return NULL;
3287 }
3288
3289out:
3290 skb->tc_verd = 0;
3291 return skb;
3292}
Linus Torvalds1da177e2005-04-16 15:20:36 -07003293#endif
3294
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00003295/**
3296 * netdev_rx_handler_register - register receive handler
3297 * @dev: device to register a handler for
3298 * @rx_handler: receive handler to register
Jiri Pirko93e2c322010-06-10 03:34:59 +00003299 * @rx_handler_data: data pointer that is used by rx handler
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00003300 *
3301 * Register a receive hander for a device. This handler will then be
3302 * called from __netif_receive_skb. A negative errno code is returned
3303 * on a failure.
3304 *
3305 * The caller must hold the rtnl_mutex.
Jiri Pirko8a4eb572011-03-12 03:14:39 +00003306 *
3307 * For a general description of rx_handler, see enum rx_handler_result.
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00003308 */
3309int netdev_rx_handler_register(struct net_device *dev,
Jiri Pirko93e2c322010-06-10 03:34:59 +00003310 rx_handler_func_t *rx_handler,
3311 void *rx_handler_data)
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00003312{
3313 ASSERT_RTNL();
3314
3315 if (dev->rx_handler)
3316 return -EBUSY;
3317
Jiri Pirko93e2c322010-06-10 03:34:59 +00003318 rcu_assign_pointer(dev->rx_handler_data, rx_handler_data);
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00003319 rcu_assign_pointer(dev->rx_handler, rx_handler);
3320
3321 return 0;
3322}
3323EXPORT_SYMBOL_GPL(netdev_rx_handler_register);
3324
3325/**
3326 * netdev_rx_handler_unregister - unregister receive handler
3327 * @dev: device to unregister a handler from
3328 *
3329 * Unregister a receive hander from a device.
3330 *
3331 * The caller must hold the rtnl_mutex.
3332 */
3333void netdev_rx_handler_unregister(struct net_device *dev)
3334{
3335
3336 ASSERT_RTNL();
Stephen Hemmingera9b3cd72011-08-01 16:19:00 +00003337 RCU_INIT_POINTER(dev->rx_handler, NULL);
3338 RCU_INIT_POINTER(dev->rx_handler_data, NULL);
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00003339}
3340EXPORT_SYMBOL_GPL(netdev_rx_handler_unregister);
3341
Mel Gormanb4b9e352012-07-31 16:44:26 -07003342/*
3343 * Limit the use of PFMEMALLOC reserves to those protocols that implement
3344 * the special handling of PFMEMALLOC skbs.
3345 */
3346static bool skb_pfmemalloc_protocol(struct sk_buff *skb)
3347{
3348 switch (skb->protocol) {
3349 case __constant_htons(ETH_P_ARP):
3350 case __constant_htons(ETH_P_IP):
3351 case __constant_htons(ETH_P_IPV6):
3352 case __constant_htons(ETH_P_8021Q):
3353 return true;
3354 default:
3355 return false;
3356 }
3357}
3358
David S. Miller9754e292013-02-14 15:57:38 -05003359static int __netif_receive_skb_core(struct sk_buff *skb, bool pfmemalloc)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003360{
3361 struct packet_type *ptype, *pt_prev;
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00003362 rx_handler_func_t *rx_handler;
David S. Millerf2ccd8f2005-08-09 19:34:12 -07003363 struct net_device *orig_dev;
David S. Miller63d8ea72011-02-28 10:48:59 -08003364 struct net_device *null_or_dev;
Jiri Pirko8a4eb572011-03-12 03:14:39 +00003365 bool deliver_exact = false;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003366 int ret = NET_RX_DROP;
Al Viro252e3342006-11-14 20:48:11 -08003367 __be16 type;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003368
Eric Dumazet588f0332011-11-15 04:12:55 +00003369 net_timestamp_check(!netdev_tstamp_prequeue, skb);
Eric Dumazet81bbb3d2009-09-30 16:42:42 -07003370
Koki Sanagicf66ba52010-08-23 18:45:02 +09003371 trace_netif_receive_skb(skb);
Patrick McHardy9b22ea52008-11-04 14:49:57 -08003372
Linus Torvalds1da177e2005-04-16 15:20:36 -07003373 /* if we've gotten here through NAPI, check netpoll */
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003374 if (netpoll_receive_skb(skb))
Mel Gormanb4b9e352012-07-31 16:44:26 -07003375 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003376
Joe Eykholtcc9bd5c2008-07-02 18:22:00 -07003377 orig_dev = skb->dev;
Jiri Pirko1765a572011-02-12 06:48:36 +00003378
Arnaldo Carvalho de Meloc1d2bbe2007-04-10 20:45:18 -07003379 skb_reset_network_header(skb);
Eric Dumazetfda55ec2013-01-07 09:28:21 +00003380 if (!skb_transport_header_was_set(skb))
3381 skb_reset_transport_header(skb);
Jiri Pirko0b5c9db2011-06-10 06:56:58 +00003382 skb_reset_mac_len(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003383
3384 pt_prev = NULL;
3385
3386 rcu_read_lock();
3387
David S. Miller63d8ea72011-02-28 10:48:59 -08003388another_round:
David S. Millerb6858172012-07-23 16:27:54 -07003389 skb->skb_iif = skb->dev->ifindex;
David S. Miller63d8ea72011-02-28 10:48:59 -08003390
3391 __this_cpu_inc(softnet_data.processed);
3392
Jiri Pirkobcc6d472011-04-07 19:48:33 +00003393 if (skb->protocol == cpu_to_be16(ETH_P_8021Q)) {
3394 skb = vlan_untag(skb);
3395 if (unlikely(!skb))
Mel Gormanb4b9e352012-07-31 16:44:26 -07003396 goto unlock;
Jiri Pirkobcc6d472011-04-07 19:48:33 +00003397 }
3398
Linus Torvalds1da177e2005-04-16 15:20:36 -07003399#ifdef CONFIG_NET_CLS_ACT
3400 if (skb->tc_verd & TC_NCLS) {
3401 skb->tc_verd = CLR_TC_NCLS(skb->tc_verd);
3402 goto ncls;
3403 }
3404#endif
3405
David S. Miller9754e292013-02-14 15:57:38 -05003406 if (pfmemalloc)
Mel Gormanb4b9e352012-07-31 16:44:26 -07003407 goto skip_taps;
3408
Linus Torvalds1da177e2005-04-16 15:20:36 -07003409 list_for_each_entry_rcu(ptype, &ptype_all, list) {
David S. Miller63d8ea72011-02-28 10:48:59 -08003410 if (!ptype->dev || ptype->dev == skb->dev) {
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09003411 if (pt_prev)
David S. Millerf2ccd8f2005-08-09 19:34:12 -07003412 ret = deliver_skb(skb, pt_prev, orig_dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003413 pt_prev = ptype;
3414 }
3415 }
3416
Mel Gormanb4b9e352012-07-31 16:44:26 -07003417skip_taps:
Linus Torvalds1da177e2005-04-16 15:20:36 -07003418#ifdef CONFIG_NET_CLS_ACT
Herbert Xuf697c3e2007-10-14 00:38:47 -07003419 skb = handle_ing(skb, &pt_prev, &ret, orig_dev);
3420 if (!skb)
Mel Gormanb4b9e352012-07-31 16:44:26 -07003421 goto unlock;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003422ncls:
3423#endif
3424
David S. Miller9754e292013-02-14 15:57:38 -05003425 if (pfmemalloc && !skb_pfmemalloc_protocol(skb))
Mel Gormanb4b9e352012-07-31 16:44:26 -07003426 goto drop;
3427
John Fastabend24257172011-10-10 09:16:41 +00003428 if (vlan_tx_tag_present(skb)) {
3429 if (pt_prev) {
3430 ret = deliver_skb(skb, pt_prev, orig_dev);
3431 pt_prev = NULL;
3432 }
Florian Zumbiehl48cc32d32012-10-07 15:51:58 +00003433 if (vlan_do_receive(&skb))
John Fastabend24257172011-10-10 09:16:41 +00003434 goto another_round;
3435 else if (unlikely(!skb))
Mel Gormanb4b9e352012-07-31 16:44:26 -07003436 goto unlock;
John Fastabend24257172011-10-10 09:16:41 +00003437 }
3438
Florian Zumbiehl48cc32d32012-10-07 15:51:58 +00003439 rx_handler = rcu_dereference(skb->dev->rx_handler);
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00003440 if (rx_handler) {
3441 if (pt_prev) {
3442 ret = deliver_skb(skb, pt_prev, orig_dev);
3443 pt_prev = NULL;
3444 }
Jiri Pirko8a4eb572011-03-12 03:14:39 +00003445 switch (rx_handler(&skb)) {
3446 case RX_HANDLER_CONSUMED:
Cristian Bercaru3bc1b1a2013-03-08 07:03:38 +00003447 ret = NET_RX_SUCCESS;
Mel Gormanb4b9e352012-07-31 16:44:26 -07003448 goto unlock;
Jiri Pirko8a4eb572011-03-12 03:14:39 +00003449 case RX_HANDLER_ANOTHER:
David S. Miller63d8ea72011-02-28 10:48:59 -08003450 goto another_round;
Jiri Pirko8a4eb572011-03-12 03:14:39 +00003451 case RX_HANDLER_EXACT:
3452 deliver_exact = true;
3453 case RX_HANDLER_PASS:
3454 break;
3455 default:
3456 BUG();
3457 }
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00003458 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07003459
Florian Zumbiehl48cc32d32012-10-07 15:51:58 +00003460 if (vlan_tx_nonzero_tag_present(skb))
3461 skb->pkt_type = PACKET_OTHERHOST;
3462
David S. Miller63d8ea72011-02-28 10:48:59 -08003463 /* deliver only exact match when indicated */
Jiri Pirko8a4eb572011-03-12 03:14:39 +00003464 null_or_dev = deliver_exact ? skb->dev : NULL;
Andy Gospodarek1f3c8802009-12-14 10:48:58 +00003465
Linus Torvalds1da177e2005-04-16 15:20:36 -07003466 type = skb->protocol;
Pavel Emelyanov82d8a8672007-11-26 20:12:58 +08003467 list_for_each_entry_rcu(ptype,
3468 &ptype_base[ntohs(type) & PTYPE_HASH_MASK], list) {
David S. Miller63d8ea72011-02-28 10:48:59 -08003469 if (ptype->type == type &&
Jiri Pirkoe3f48d32011-02-28 20:26:31 +00003470 (ptype->dev == null_or_dev || ptype->dev == skb->dev ||
3471 ptype->dev == orig_dev)) {
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09003472 if (pt_prev)
David S. Millerf2ccd8f2005-08-09 19:34:12 -07003473 ret = deliver_skb(skb, pt_prev, orig_dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003474 pt_prev = ptype;
3475 }
3476 }
3477
3478 if (pt_prev) {
Michael S. Tsirkin1080e512012-07-20 09:23:17 +00003479 if (unlikely(skb_orphan_frags(skb, GFP_ATOMIC)))
Michael S. Tsirkin0e698bf2012-09-15 22:44:16 +00003480 goto drop;
Michael S. Tsirkin1080e512012-07-20 09:23:17 +00003481 else
3482 ret = pt_prev->func(skb, skb->dev, pt_prev, orig_dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003483 } else {
Mel Gormanb4b9e352012-07-31 16:44:26 -07003484drop:
Eric Dumazetcaf586e2010-09-30 21:06:55 +00003485 atomic_long_inc(&skb->dev->rx_dropped);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003486 kfree_skb(skb);
3487 /* Jamal, now you will not able to escape explaining
3488 * me how you were going to use this. :-)
3489 */
3490 ret = NET_RX_DROP;
3491 }
3492
Mel Gormanb4b9e352012-07-31 16:44:26 -07003493unlock:
Linus Torvalds1da177e2005-04-16 15:20:36 -07003494 rcu_read_unlock();
Mel Gormanb4b9e352012-07-31 16:44:26 -07003495out:
David S. Miller9754e292013-02-14 15:57:38 -05003496 return ret;
3497}
3498
3499static int __netif_receive_skb(struct sk_buff *skb)
3500{
3501 int ret;
3502
3503 if (sk_memalloc_socks() && skb_pfmemalloc(skb)) {
3504 unsigned long pflags = current->flags;
3505
3506 /*
3507 * PFMEMALLOC skbs are special, they should
3508 * - be delivered to SOCK_MEMALLOC sockets only
3509 * - stay away from userspace
3510 * - have bounded memory usage
3511 *
3512 * Use PF_MEMALLOC as this saves us from propagating the allocation
3513 * context down to all allocation sites.
3514 */
3515 current->flags |= PF_MEMALLOC;
3516 ret = __netif_receive_skb_core(skb, true);
3517 tsk_restore_flags(current, pflags, PF_MEMALLOC);
3518 } else
3519 ret = __netif_receive_skb_core(skb, false);
3520
Linus Torvalds1da177e2005-04-16 15:20:36 -07003521 return ret;
3522}
Tom Herbert0a9627f2010-03-16 08:03:29 +00003523
3524/**
3525 * netif_receive_skb - process receive buffer from network
3526 * @skb: buffer to process
3527 *
3528 * netif_receive_skb() is the main receive data processing function.
3529 * It always succeeds. The buffer may be dropped during processing
3530 * for congestion control or by the protocol layers.
3531 *
3532 * This function may only be called from softirq context and interrupts
3533 * should be enabled.
3534 *
3535 * Return values (usually ignored):
3536 * NET_RX_SUCCESS: no congestion
3537 * NET_RX_DROP: packet was dropped
3538 */
3539int netif_receive_skb(struct sk_buff *skb)
3540{
Eric Dumazet588f0332011-11-15 04:12:55 +00003541 net_timestamp_check(netdev_tstamp_prequeue, skb);
Eric Dumazet3b098e22010-05-15 23:57:10 -07003542
Richard Cochranc1f19b52010-07-17 08:49:36 +00003543 if (skb_defer_rx_timestamp(skb))
3544 return NET_RX_SUCCESS;
3545
Eric Dumazetdf334542010-03-24 19:13:54 +00003546#ifdef CONFIG_RPS
Ingo Molnarc5905af2012-02-24 08:31:31 +01003547 if (static_key_false(&rps_needed)) {
Eric Dumazet3b098e22010-05-15 23:57:10 -07003548 struct rps_dev_flow voidflow, *rflow = &voidflow;
3549 int cpu, ret;
Tom Herbert0a9627f2010-03-16 08:03:29 +00003550
Eric Dumazet3b098e22010-05-15 23:57:10 -07003551 rcu_read_lock();
Tom Herbert0a9627f2010-03-16 08:03:29 +00003552
Eric Dumazet3b098e22010-05-15 23:57:10 -07003553 cpu = get_rps_cpu(skb->dev, skb, &rflow);
Tom Herbertfec5e652010-04-16 16:01:27 -07003554
Eric Dumazet3b098e22010-05-15 23:57:10 -07003555 if (cpu >= 0) {
3556 ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail);
3557 rcu_read_unlock();
Eric Dumazetadc93002011-11-17 03:13:26 +00003558 return ret;
Eric Dumazet3b098e22010-05-15 23:57:10 -07003559 }
Eric Dumazetadc93002011-11-17 03:13:26 +00003560 rcu_read_unlock();
Tom Herbertfec5e652010-04-16 16:01:27 -07003561 }
Tom Herbert1e94d722010-03-18 17:45:44 -07003562#endif
Eric Dumazetadc93002011-11-17 03:13:26 +00003563 return __netif_receive_skb(skb);
Tom Herbert0a9627f2010-03-16 08:03:29 +00003564}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07003565EXPORT_SYMBOL(netif_receive_skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003566
Eric Dumazet88751272010-04-19 05:07:33 +00003567/* Network device is going away, flush any packets still pending
3568 * Called with irqs disabled.
3569 */
Changli Gao152102c2010-03-30 20:16:22 +00003570static void flush_backlog(void *arg)
Stephen Hemminger6e583ce2008-08-03 21:29:57 -07003571{
Changli Gao152102c2010-03-30 20:16:22 +00003572 struct net_device *dev = arg;
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003573 struct softnet_data *sd = &__get_cpu_var(softnet_data);
Stephen Hemminger6e583ce2008-08-03 21:29:57 -07003574 struct sk_buff *skb, *tmp;
3575
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003576 rps_lock(sd);
Changli Gao6e7676c2010-04-27 15:07:33 -07003577 skb_queue_walk_safe(&sd->input_pkt_queue, skb, tmp) {
Stephen Hemminger6e583ce2008-08-03 21:29:57 -07003578 if (skb->dev == dev) {
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003579 __skb_unlink(skb, &sd->input_pkt_queue);
Stephen Hemminger6e583ce2008-08-03 21:29:57 -07003580 kfree_skb(skb);
Tom Herbert76cc8b12010-05-20 18:37:59 +00003581 input_queue_head_incr(sd);
Stephen Hemminger6e583ce2008-08-03 21:29:57 -07003582 }
Changli Gao6e7676c2010-04-27 15:07:33 -07003583 }
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003584 rps_unlock(sd);
Changli Gao6e7676c2010-04-27 15:07:33 -07003585
3586 skb_queue_walk_safe(&sd->process_queue, skb, tmp) {
3587 if (skb->dev == dev) {
3588 __skb_unlink(skb, &sd->process_queue);
3589 kfree_skb(skb);
Tom Herbert76cc8b12010-05-20 18:37:59 +00003590 input_queue_head_incr(sd);
Changli Gao6e7676c2010-04-27 15:07:33 -07003591 }
3592 }
Stephen Hemminger6e583ce2008-08-03 21:29:57 -07003593}
3594
Herbert Xud565b0a2008-12-15 23:38:52 -08003595static int napi_gro_complete(struct sk_buff *skb)
3596{
Vlad Yasevich22061d82012-11-15 08:49:11 +00003597 struct packet_offload *ptype;
Herbert Xud565b0a2008-12-15 23:38:52 -08003598 __be16 type = skb->protocol;
Vlad Yasevich22061d82012-11-15 08:49:11 +00003599 struct list_head *head = &offload_base;
Herbert Xud565b0a2008-12-15 23:38:52 -08003600 int err = -ENOENT;
3601
Eric Dumazetc3c7c252012-12-06 13:54:59 +00003602 BUILD_BUG_ON(sizeof(struct napi_gro_cb) > sizeof(skb->cb));
3603
Herbert Xufc59f9a2009-04-14 15:11:06 -07003604 if (NAPI_GRO_CB(skb)->count == 1) {
3605 skb_shinfo(skb)->gso_size = 0;
Herbert Xud565b0a2008-12-15 23:38:52 -08003606 goto out;
Herbert Xufc59f9a2009-04-14 15:11:06 -07003607 }
Herbert Xud565b0a2008-12-15 23:38:52 -08003608
3609 rcu_read_lock();
3610 list_for_each_entry_rcu(ptype, head, list) {
Vlad Yasevichf191a1d2012-11-15 08:49:23 +00003611 if (ptype->type != type || !ptype->callbacks.gro_complete)
Herbert Xud565b0a2008-12-15 23:38:52 -08003612 continue;
3613
Vlad Yasevichf191a1d2012-11-15 08:49:23 +00003614 err = ptype->callbacks.gro_complete(skb);
Herbert Xud565b0a2008-12-15 23:38:52 -08003615 break;
3616 }
3617 rcu_read_unlock();
3618
3619 if (err) {
3620 WARN_ON(&ptype->list == head);
3621 kfree_skb(skb);
3622 return NET_RX_SUCCESS;
3623 }
3624
3625out:
Herbert Xud565b0a2008-12-15 23:38:52 -08003626 return netif_receive_skb(skb);
3627}
3628
Eric Dumazet2e71a6f2012-10-06 08:08:49 +00003629/* napi->gro_list contains packets ordered by age.
3630 * youngest packets at the head of it.
3631 * Complete skbs in reverse order to reduce latencies.
3632 */
3633void napi_gro_flush(struct napi_struct *napi, bool flush_old)
Herbert Xud565b0a2008-12-15 23:38:52 -08003634{
Eric Dumazet2e71a6f2012-10-06 08:08:49 +00003635 struct sk_buff *skb, *prev = NULL;
Herbert Xud565b0a2008-12-15 23:38:52 -08003636
Eric Dumazet2e71a6f2012-10-06 08:08:49 +00003637 /* scan list and build reverse chain */
3638 for (skb = napi->gro_list; skb != NULL; skb = skb->next) {
3639 skb->prev = prev;
3640 prev = skb;
Herbert Xud565b0a2008-12-15 23:38:52 -08003641 }
3642
Eric Dumazet2e71a6f2012-10-06 08:08:49 +00003643 for (skb = prev; skb; skb = prev) {
3644 skb->next = NULL;
3645
3646 if (flush_old && NAPI_GRO_CB(skb)->age == jiffies)
3647 return;
3648
3649 prev = skb->prev;
3650 napi_gro_complete(skb);
3651 napi->gro_count--;
3652 }
3653
Herbert Xud565b0a2008-12-15 23:38:52 -08003654 napi->gro_list = NULL;
3655}
Eric Dumazet86cac582010-08-31 18:25:32 +00003656EXPORT_SYMBOL(napi_gro_flush);
Herbert Xud565b0a2008-12-15 23:38:52 -08003657
Eric Dumazet89c5fa32012-12-10 13:28:16 +00003658static void gro_list_prepare(struct napi_struct *napi, struct sk_buff *skb)
3659{
3660 struct sk_buff *p;
3661 unsigned int maclen = skb->dev->hard_header_len;
3662
3663 for (p = napi->gro_list; p; p = p->next) {
3664 unsigned long diffs;
3665
3666 diffs = (unsigned long)p->dev ^ (unsigned long)skb->dev;
3667 diffs |= p->vlan_tci ^ skb->vlan_tci;
3668 if (maclen == ETH_HLEN)
3669 diffs |= compare_ether_header(skb_mac_header(p),
3670 skb_gro_mac_header(skb));
3671 else if (!diffs)
3672 diffs = memcmp(skb_mac_header(p),
3673 skb_gro_mac_header(skb),
3674 maclen);
3675 NAPI_GRO_CB(p)->same_flow = !diffs;
3676 NAPI_GRO_CB(p)->flush = 0;
3677 }
3678}
3679
Rami Rosenbb728822012-11-28 21:55:25 +00003680static enum gro_result dev_gro_receive(struct napi_struct *napi, struct sk_buff *skb)
Herbert Xud565b0a2008-12-15 23:38:52 -08003681{
3682 struct sk_buff **pp = NULL;
Vlad Yasevich22061d82012-11-15 08:49:11 +00003683 struct packet_offload *ptype;
Herbert Xud565b0a2008-12-15 23:38:52 -08003684 __be16 type = skb->protocol;
Vlad Yasevich22061d82012-11-15 08:49:11 +00003685 struct list_head *head = &offload_base;
Herbert Xu0da2afd52008-12-26 14:57:42 -08003686 int same_flow;
Ben Hutchings5b252f02009-10-29 07:17:09 +00003687 enum gro_result ret;
Herbert Xud565b0a2008-12-15 23:38:52 -08003688
Jarek Poplawskice9e76c2010-08-05 01:19:11 +00003689 if (!(skb->dev->features & NETIF_F_GRO) || netpoll_rx_on(skb))
Herbert Xud565b0a2008-12-15 23:38:52 -08003690 goto normal;
3691
David S. Miller21dc3302010-08-23 00:13:46 -07003692 if (skb_is_gso(skb) || skb_has_frag_list(skb))
Herbert Xuf17f5c92009-01-14 14:36:12 -08003693 goto normal;
3694
Eric Dumazet89c5fa32012-12-10 13:28:16 +00003695 gro_list_prepare(napi, skb);
3696
Herbert Xud565b0a2008-12-15 23:38:52 -08003697 rcu_read_lock();
3698 list_for_each_entry_rcu(ptype, head, list) {
Vlad Yasevichf191a1d2012-11-15 08:49:23 +00003699 if (ptype->type != type || !ptype->callbacks.gro_receive)
Herbert Xud565b0a2008-12-15 23:38:52 -08003700 continue;
3701
Herbert Xu86911732009-01-29 14:19:50 +00003702 skb_set_network_header(skb, skb_gro_offset(skb));
Eric Dumazetefd94502013-02-14 17:31:48 +00003703 skb_reset_mac_len(skb);
Herbert Xud565b0a2008-12-15 23:38:52 -08003704 NAPI_GRO_CB(skb)->same_flow = 0;
3705 NAPI_GRO_CB(skb)->flush = 0;
Herbert Xu5d38a072009-01-04 16:13:40 -08003706 NAPI_GRO_CB(skb)->free = 0;
Herbert Xud565b0a2008-12-15 23:38:52 -08003707
Vlad Yasevichf191a1d2012-11-15 08:49:23 +00003708 pp = ptype->callbacks.gro_receive(&napi->gro_list, skb);
Herbert Xud565b0a2008-12-15 23:38:52 -08003709 break;
3710 }
3711 rcu_read_unlock();
3712
3713 if (&ptype->list == head)
3714 goto normal;
3715
Herbert Xu0da2afd52008-12-26 14:57:42 -08003716 same_flow = NAPI_GRO_CB(skb)->same_flow;
Herbert Xu5d0d9be2009-01-29 14:19:48 +00003717 ret = NAPI_GRO_CB(skb)->free ? GRO_MERGED_FREE : GRO_MERGED;
Herbert Xu0da2afd52008-12-26 14:57:42 -08003718
Herbert Xud565b0a2008-12-15 23:38:52 -08003719 if (pp) {
3720 struct sk_buff *nskb = *pp;
3721
3722 *pp = nskb->next;
3723 nskb->next = NULL;
3724 napi_gro_complete(nskb);
Herbert Xu4ae55442009-02-08 18:00:36 +00003725 napi->gro_count--;
Herbert Xud565b0a2008-12-15 23:38:52 -08003726 }
3727
Herbert Xu0da2afd52008-12-26 14:57:42 -08003728 if (same_flow)
Herbert Xud565b0a2008-12-15 23:38:52 -08003729 goto ok;
3730
Herbert Xu4ae55442009-02-08 18:00:36 +00003731 if (NAPI_GRO_CB(skb)->flush || napi->gro_count >= MAX_GRO_SKBS)
Herbert Xud565b0a2008-12-15 23:38:52 -08003732 goto normal;
Herbert Xud565b0a2008-12-15 23:38:52 -08003733
Herbert Xu4ae55442009-02-08 18:00:36 +00003734 napi->gro_count++;
Herbert Xud565b0a2008-12-15 23:38:52 -08003735 NAPI_GRO_CB(skb)->count = 1;
Eric Dumazet2e71a6f2012-10-06 08:08:49 +00003736 NAPI_GRO_CB(skb)->age = jiffies;
Herbert Xu86911732009-01-29 14:19:50 +00003737 skb_shinfo(skb)->gso_size = skb_gro_len(skb);
Herbert Xud565b0a2008-12-15 23:38:52 -08003738 skb->next = napi->gro_list;
3739 napi->gro_list = skb;
Herbert Xu5d0d9be2009-01-29 14:19:48 +00003740 ret = GRO_HELD;
Herbert Xud565b0a2008-12-15 23:38:52 -08003741
Herbert Xuad0f9902009-02-01 01:24:55 -08003742pull:
Herbert Xucb189782009-05-26 18:50:31 +00003743 if (skb_headlen(skb) < skb_gro_offset(skb)) {
3744 int grow = skb_gro_offset(skb) - skb_headlen(skb);
3745
3746 BUG_ON(skb->end - skb->tail < grow);
3747
3748 memcpy(skb_tail_pointer(skb), NAPI_GRO_CB(skb)->frag0, grow);
3749
3750 skb->tail += grow;
3751 skb->data_len -= grow;
3752
3753 skb_shinfo(skb)->frags[0].page_offset += grow;
Eric Dumazet9e903e02011-10-18 21:00:24 +00003754 skb_frag_size_sub(&skb_shinfo(skb)->frags[0], grow);
Herbert Xucb189782009-05-26 18:50:31 +00003755
Eric Dumazet9e903e02011-10-18 21:00:24 +00003756 if (unlikely(!skb_frag_size(&skb_shinfo(skb)->frags[0]))) {
Ian Campbellea2ab692011-08-22 23:44:58 +00003757 skb_frag_unref(skb, 0);
Herbert Xucb189782009-05-26 18:50:31 +00003758 memmove(skb_shinfo(skb)->frags,
3759 skb_shinfo(skb)->frags + 1,
Jarek Poplawskie5093ae2010-08-11 02:02:10 +00003760 --skb_shinfo(skb)->nr_frags * sizeof(skb_frag_t));
Herbert Xucb189782009-05-26 18:50:31 +00003761 }
Herbert Xuad0f9902009-02-01 01:24:55 -08003762 }
3763
Herbert Xud565b0a2008-12-15 23:38:52 -08003764ok:
Herbert Xu5d0d9be2009-01-29 14:19:48 +00003765 return ret;
Herbert Xud565b0a2008-12-15 23:38:52 -08003766
3767normal:
Herbert Xuad0f9902009-02-01 01:24:55 -08003768 ret = GRO_NORMAL;
3769 goto pull;
Herbert Xu5d38a072009-01-04 16:13:40 -08003770}
Herbert Xu96e93ea2009-01-06 10:49:34 -08003771
Herbert Xu96e93ea2009-01-06 10:49:34 -08003772
Rami Rosenbb728822012-11-28 21:55:25 +00003773static gro_result_t napi_skb_finish(gro_result_t ret, struct sk_buff *skb)
Herbert Xu5d38a072009-01-04 16:13:40 -08003774{
Herbert Xu5d0d9be2009-01-29 14:19:48 +00003775 switch (ret) {
3776 case GRO_NORMAL:
Ben Hutchingsc7c4b3b2009-10-29 21:36:53 -07003777 if (netif_receive_skb(skb))
3778 ret = GRO_DROP;
3779 break;
Herbert Xu5d38a072009-01-04 16:13:40 -08003780
Herbert Xu5d0d9be2009-01-29 14:19:48 +00003781 case GRO_DROP:
Herbert Xu5d38a072009-01-04 16:13:40 -08003782 kfree_skb(skb);
3783 break;
Ben Hutchings5b252f02009-10-29 07:17:09 +00003784
Eric Dumazetdaa86542012-04-19 07:07:40 +00003785 case GRO_MERGED_FREE:
Eric Dumazetd7e88832012-04-30 08:10:34 +00003786 if (NAPI_GRO_CB(skb)->free == NAPI_GRO_FREE_STOLEN_HEAD)
3787 kmem_cache_free(skbuff_head_cache, skb);
3788 else
3789 __kfree_skb(skb);
Eric Dumazetdaa86542012-04-19 07:07:40 +00003790 break;
3791
Ben Hutchings5b252f02009-10-29 07:17:09 +00003792 case GRO_HELD:
3793 case GRO_MERGED:
3794 break;
Herbert Xu5d38a072009-01-04 16:13:40 -08003795 }
3796
Ben Hutchingsc7c4b3b2009-10-29 21:36:53 -07003797 return ret;
Herbert Xu5d0d9be2009-01-29 14:19:48 +00003798}
Herbert Xu5d0d9be2009-01-29 14:19:48 +00003799
Eric Dumazetca07e432012-10-06 22:28:06 +00003800static void skb_gro_reset_offset(struct sk_buff *skb)
Herbert Xu78a478d2009-05-26 18:50:21 +00003801{
Eric Dumazetca07e432012-10-06 22:28:06 +00003802 const struct skb_shared_info *pinfo = skb_shinfo(skb);
3803 const skb_frag_t *frag0 = &pinfo->frags[0];
3804
Herbert Xu78a478d2009-05-26 18:50:21 +00003805 NAPI_GRO_CB(skb)->data_offset = 0;
3806 NAPI_GRO_CB(skb)->frag0 = NULL;
Herbert Xu74895942009-05-26 18:50:27 +00003807 NAPI_GRO_CB(skb)->frag0_len = 0;
Herbert Xu78a478d2009-05-26 18:50:21 +00003808
Herbert Xu78d3fd02009-05-26 18:50:23 +00003809 if (skb->mac_header == skb->tail &&
Eric Dumazetca07e432012-10-06 22:28:06 +00003810 pinfo->nr_frags &&
3811 !PageHighMem(skb_frag_page(frag0))) {
3812 NAPI_GRO_CB(skb)->frag0 = skb_frag_address(frag0);
3813 NAPI_GRO_CB(skb)->frag0_len = skb_frag_size(frag0);
Herbert Xu74895942009-05-26 18:50:27 +00003814 }
Herbert Xu78a478d2009-05-26 18:50:21 +00003815}
Herbert Xu78a478d2009-05-26 18:50:21 +00003816
Ben Hutchingsc7c4b3b2009-10-29 21:36:53 -07003817gro_result_t napi_gro_receive(struct napi_struct *napi, struct sk_buff *skb)
Herbert Xu5d0d9be2009-01-29 14:19:48 +00003818{
Herbert Xu86911732009-01-29 14:19:50 +00003819 skb_gro_reset_offset(skb);
3820
Eric Dumazet89c5fa32012-12-10 13:28:16 +00003821 return napi_skb_finish(dev_gro_receive(napi, skb), skb);
Herbert Xud565b0a2008-12-15 23:38:52 -08003822}
3823EXPORT_SYMBOL(napi_gro_receive);
3824
stephen hemmingerd0c2b0d2010-10-19 07:12:10 +00003825static void napi_reuse_skb(struct napi_struct *napi, struct sk_buff *skb)
Herbert Xu96e93ea2009-01-06 10:49:34 -08003826{
Herbert Xu96e93ea2009-01-06 10:49:34 -08003827 __skb_pull(skb, skb_headlen(skb));
Eric Dumazet2a2a4592012-03-21 06:58:03 +00003828 /* restore the reserve we had after netdev_alloc_skb_ip_align() */
3829 skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN - skb_headroom(skb));
Jesse Gross3701e512010-10-20 13:56:06 +00003830 skb->vlan_tci = 0;
Herbert Xu66c46d72011-01-29 20:44:54 -08003831 skb->dev = napi->dev;
Andy Gospodarek6d152e22011-02-02 14:53:25 -08003832 skb->skb_iif = 0;
Herbert Xu96e93ea2009-01-06 10:49:34 -08003833
3834 napi->skb = skb;
3835}
Herbert Xu96e93ea2009-01-06 10:49:34 -08003836
Herbert Xu76620aa2009-04-16 02:02:07 -07003837struct sk_buff *napi_get_frags(struct napi_struct *napi)
Herbert Xu5d38a072009-01-04 16:13:40 -08003838{
Herbert Xu5d38a072009-01-04 16:13:40 -08003839 struct sk_buff *skb = napi->skb;
Herbert Xu5d38a072009-01-04 16:13:40 -08003840
3841 if (!skb) {
Eric Dumazet89d71a62009-10-13 05:34:20 +00003842 skb = netdev_alloc_skb_ip_align(napi->dev, GRO_MAX_HEAD);
3843 if (skb)
3844 napi->skb = skb;
Herbert Xu5d38a072009-01-04 16:13:40 -08003845 }
Herbert Xu96e93ea2009-01-06 10:49:34 -08003846 return skb;
3847}
Herbert Xu76620aa2009-04-16 02:02:07 -07003848EXPORT_SYMBOL(napi_get_frags);
Herbert Xu96e93ea2009-01-06 10:49:34 -08003849
Rami Rosenbb728822012-11-28 21:55:25 +00003850static gro_result_t napi_frags_finish(struct napi_struct *napi, struct sk_buff *skb,
Ben Hutchingsc7c4b3b2009-10-29 21:36:53 -07003851 gro_result_t ret)
Herbert Xu5d0d9be2009-01-29 14:19:48 +00003852{
Herbert Xu5d0d9be2009-01-29 14:19:48 +00003853 switch (ret) {
3854 case GRO_NORMAL:
Herbert Xu86911732009-01-29 14:19:50 +00003855 case GRO_HELD:
Ajit Khapardee76b69c2010-02-16 20:25:43 +00003856 skb->protocol = eth_type_trans(skb, skb->dev);
Herbert Xu86911732009-01-29 14:19:50 +00003857
Ben Hutchingsc7c4b3b2009-10-29 21:36:53 -07003858 if (ret == GRO_HELD)
3859 skb_gro_pull(skb, -ETH_HLEN);
3860 else if (netif_receive_skb(skb))
3861 ret = GRO_DROP;
Herbert Xu86911732009-01-29 14:19:50 +00003862 break;
Herbert Xu5d0d9be2009-01-29 14:19:48 +00003863
3864 case GRO_DROP:
Herbert Xu5d0d9be2009-01-29 14:19:48 +00003865 case GRO_MERGED_FREE:
3866 napi_reuse_skb(napi, skb);
3867 break;
Ben Hutchings5b252f02009-10-29 07:17:09 +00003868
3869 case GRO_MERGED:
3870 break;
Herbert Xu5d0d9be2009-01-29 14:19:48 +00003871 }
3872
Ben Hutchingsc7c4b3b2009-10-29 21:36:53 -07003873 return ret;
Herbert Xu5d0d9be2009-01-29 14:19:48 +00003874}
Herbert Xu5d0d9be2009-01-29 14:19:48 +00003875
Eric Dumazet4adb9c42012-05-18 20:49:06 +00003876static struct sk_buff *napi_frags_skb(struct napi_struct *napi)
Herbert Xu96e93ea2009-01-06 10:49:34 -08003877{
Herbert Xu76620aa2009-04-16 02:02:07 -07003878 struct sk_buff *skb = napi->skb;
3879 struct ethhdr *eth;
Herbert Xua5b1cf22009-05-26 18:50:28 +00003880 unsigned int hlen;
3881 unsigned int off;
Herbert Xu76620aa2009-04-16 02:02:07 -07003882
3883 napi->skb = NULL;
3884
3885 skb_reset_mac_header(skb);
3886 skb_gro_reset_offset(skb);
3887
Herbert Xua5b1cf22009-05-26 18:50:28 +00003888 off = skb_gro_offset(skb);
3889 hlen = off + sizeof(*eth);
3890 eth = skb_gro_header_fast(skb, off);
3891 if (skb_gro_header_hard(skb, hlen)) {
3892 eth = skb_gro_header_slow(skb, hlen, off);
3893 if (unlikely(!eth)) {
3894 napi_reuse_skb(napi, skb);
3895 skb = NULL;
3896 goto out;
3897 }
Herbert Xu76620aa2009-04-16 02:02:07 -07003898 }
3899
3900 skb_gro_pull(skb, sizeof(*eth));
3901
3902 /*
3903 * This works because the only protocols we care about don't require
3904 * special handling. We'll fix it up properly at the end.
3905 */
3906 skb->protocol = eth->h_proto;
3907
3908out:
3909 return skb;
3910}
Herbert Xu76620aa2009-04-16 02:02:07 -07003911
Ben Hutchingsc7c4b3b2009-10-29 21:36:53 -07003912gro_result_t napi_gro_frags(struct napi_struct *napi)
Herbert Xu76620aa2009-04-16 02:02:07 -07003913{
3914 struct sk_buff *skb = napi_frags_skb(napi);
Herbert Xu96e93ea2009-01-06 10:49:34 -08003915
3916 if (!skb)
Ben Hutchingsc7c4b3b2009-10-29 21:36:53 -07003917 return GRO_DROP;
Herbert Xu96e93ea2009-01-06 10:49:34 -08003918
Eric Dumazet89c5fa32012-12-10 13:28:16 +00003919 return napi_frags_finish(napi, skb, dev_gro_receive(napi, skb));
Herbert Xu5d38a072009-01-04 16:13:40 -08003920}
3921EXPORT_SYMBOL(napi_gro_frags);
3922
Eric Dumazete326bed2010-04-22 00:22:45 -07003923/*
3924 * net_rps_action sends any pending IPI's for rps.
3925 * Note: called with local irq disabled, but exits with local irq enabled.
3926 */
3927static void net_rps_action_and_irq_enable(struct softnet_data *sd)
3928{
3929#ifdef CONFIG_RPS
3930 struct softnet_data *remsd = sd->rps_ipi_list;
3931
3932 if (remsd) {
3933 sd->rps_ipi_list = NULL;
3934
3935 local_irq_enable();
3936
3937 /* Send pending IPI's to kick RPS processing on remote cpus. */
3938 while (remsd) {
3939 struct softnet_data *next = remsd->rps_ipi_next;
3940
3941 if (cpu_online(remsd->cpu))
3942 __smp_call_function_single(remsd->cpu,
3943 &remsd->csd, 0);
3944 remsd = next;
3945 }
3946 } else
3947#endif
3948 local_irq_enable();
3949}
3950
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003951static int process_backlog(struct napi_struct *napi, int quota)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003952{
3953 int work = 0;
Eric Dumazeteecfd7c2010-05-06 22:07:48 -07003954 struct softnet_data *sd = container_of(napi, struct softnet_data, backlog);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003955
Eric Dumazete326bed2010-04-22 00:22:45 -07003956#ifdef CONFIG_RPS
3957 /* Check if we have pending ipi, its better to send them now,
3958 * not waiting net_rx_action() end.
3959 */
3960 if (sd->rps_ipi_list) {
3961 local_irq_disable();
3962 net_rps_action_and_irq_enable(sd);
3963 }
3964#endif
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003965 napi->weight = weight_p;
Changli Gao6e7676c2010-04-27 15:07:33 -07003966 local_irq_disable();
3967 while (work < quota) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07003968 struct sk_buff *skb;
Changli Gao6e7676c2010-04-27 15:07:33 -07003969 unsigned int qlen;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003970
Changli Gao6e7676c2010-04-27 15:07:33 -07003971 while ((skb = __skb_dequeue(&sd->process_queue))) {
Eric Dumazete4008272010-04-05 15:42:39 -07003972 local_irq_enable();
Changli Gao6e7676c2010-04-27 15:07:33 -07003973 __netif_receive_skb(skb);
Changli Gao6e7676c2010-04-27 15:07:33 -07003974 local_irq_disable();
Tom Herbert76cc8b12010-05-20 18:37:59 +00003975 input_queue_head_incr(sd);
3976 if (++work >= quota) {
3977 local_irq_enable();
3978 return work;
3979 }
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003980 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07003981
Changli Gao6e7676c2010-04-27 15:07:33 -07003982 rps_lock(sd);
3983 qlen = skb_queue_len(&sd->input_pkt_queue);
Tom Herbert76cc8b12010-05-20 18:37:59 +00003984 if (qlen)
Changli Gao6e7676c2010-04-27 15:07:33 -07003985 skb_queue_splice_tail_init(&sd->input_pkt_queue,
3986 &sd->process_queue);
Tom Herbert76cc8b12010-05-20 18:37:59 +00003987
Changli Gao6e7676c2010-04-27 15:07:33 -07003988 if (qlen < quota - work) {
Eric Dumazeteecfd7c2010-05-06 22:07:48 -07003989 /*
3990 * Inline a custom version of __napi_complete().
3991 * only current cpu owns and manipulates this napi,
3992 * and NAPI_STATE_SCHED is the only possible flag set on backlog.
3993 * we can use a plain write instead of clear_bit(),
3994 * and we dont need an smp_mb() memory barrier.
3995 */
3996 list_del(&napi->poll_list);
3997 napi->state = 0;
3998
Changli Gao6e7676c2010-04-27 15:07:33 -07003999 quota = work + qlen;
4000 }
4001 rps_unlock(sd);
4002 }
4003 local_irq_enable();
Linus Torvalds1da177e2005-04-16 15:20:36 -07004004
Stephen Hemmingerbea33482007-10-03 16:41:36 -07004005 return work;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004006}
4007
Stephen Hemmingerbea33482007-10-03 16:41:36 -07004008/**
4009 * __napi_schedule - schedule for receive
Randy Dunlapc4ea43c2007-10-12 21:17:49 -07004010 * @n: entry to schedule
Stephen Hemmingerbea33482007-10-03 16:41:36 -07004011 *
4012 * The entry's receive function will be scheduled to run
4013 */
Harvey Harrisonb5606c22008-02-13 15:03:16 -08004014void __napi_schedule(struct napi_struct *n)
Stephen Hemmingerbea33482007-10-03 16:41:36 -07004015{
4016 unsigned long flags;
4017
4018 local_irq_save(flags);
Eric Dumazeteecfd7c2010-05-06 22:07:48 -07004019 ____napi_schedule(&__get_cpu_var(softnet_data), n);
Stephen Hemmingerbea33482007-10-03 16:41:36 -07004020 local_irq_restore(flags);
4021}
4022EXPORT_SYMBOL(__napi_schedule);
4023
Herbert Xud565b0a2008-12-15 23:38:52 -08004024void __napi_complete(struct napi_struct *n)
4025{
4026 BUG_ON(!test_bit(NAPI_STATE_SCHED, &n->state));
4027 BUG_ON(n->gro_list);
4028
4029 list_del(&n->poll_list);
4030 smp_mb__before_clear_bit();
4031 clear_bit(NAPI_STATE_SCHED, &n->state);
4032}
4033EXPORT_SYMBOL(__napi_complete);
4034
4035void napi_complete(struct napi_struct *n)
4036{
4037 unsigned long flags;
4038
4039 /*
4040 * don't let napi dequeue from the cpu poll list
4041 * just in case its running on a different cpu
4042 */
4043 if (unlikely(test_bit(NAPI_STATE_NPSVC, &n->state)))
4044 return;
4045
Eric Dumazet2e71a6f2012-10-06 08:08:49 +00004046 napi_gro_flush(n, false);
Herbert Xud565b0a2008-12-15 23:38:52 -08004047 local_irq_save(flags);
4048 __napi_complete(n);
4049 local_irq_restore(flags);
4050}
4051EXPORT_SYMBOL(napi_complete);
4052
4053void netif_napi_add(struct net_device *dev, struct napi_struct *napi,
4054 int (*poll)(struct napi_struct *, int), int weight)
4055{
4056 INIT_LIST_HEAD(&napi->poll_list);
Herbert Xu4ae55442009-02-08 18:00:36 +00004057 napi->gro_count = 0;
Herbert Xud565b0a2008-12-15 23:38:52 -08004058 napi->gro_list = NULL;
Herbert Xu5d38a072009-01-04 16:13:40 -08004059 napi->skb = NULL;
Herbert Xud565b0a2008-12-15 23:38:52 -08004060 napi->poll = poll;
4061 napi->weight = weight;
4062 list_add(&napi->dev_list, &dev->napi_list);
Herbert Xud565b0a2008-12-15 23:38:52 -08004063 napi->dev = dev;
Herbert Xu5d38a072009-01-04 16:13:40 -08004064#ifdef CONFIG_NETPOLL
Herbert Xud565b0a2008-12-15 23:38:52 -08004065 spin_lock_init(&napi->poll_lock);
4066 napi->poll_owner = -1;
4067#endif
4068 set_bit(NAPI_STATE_SCHED, &napi->state);
4069}
4070EXPORT_SYMBOL(netif_napi_add);
4071
4072void netif_napi_del(struct napi_struct *napi)
4073{
4074 struct sk_buff *skb, *next;
4075
Peter P Waskiewicz Jrd7b06632008-12-26 01:35:35 -08004076 list_del_init(&napi->dev_list);
Herbert Xu76620aa2009-04-16 02:02:07 -07004077 napi_free_frags(napi);
Herbert Xud565b0a2008-12-15 23:38:52 -08004078
4079 for (skb = napi->gro_list; skb; skb = next) {
4080 next = skb->next;
4081 skb->next = NULL;
4082 kfree_skb(skb);
4083 }
4084
4085 napi->gro_list = NULL;
Herbert Xu4ae55442009-02-08 18:00:36 +00004086 napi->gro_count = 0;
Herbert Xud565b0a2008-12-15 23:38:52 -08004087}
4088EXPORT_SYMBOL(netif_napi_del);
4089
Linus Torvalds1da177e2005-04-16 15:20:36 -07004090static void net_rx_action(struct softirq_action *h)
4091{
Eric Dumazete326bed2010-04-22 00:22:45 -07004092 struct softnet_data *sd = &__get_cpu_var(softnet_data);
Stephen Hemminger24f8b232008-11-03 17:14:38 -08004093 unsigned long time_limit = jiffies + 2;
Stephen Hemminger51b0bde2005-06-23 20:14:40 -07004094 int budget = netdev_budget;
Matt Mackall53fb95d2005-08-11 19:27:43 -07004095 void *have;
4096
Linus Torvalds1da177e2005-04-16 15:20:36 -07004097 local_irq_disable();
4098
Eric Dumazete326bed2010-04-22 00:22:45 -07004099 while (!list_empty(&sd->poll_list)) {
Stephen Hemmingerbea33482007-10-03 16:41:36 -07004100 struct napi_struct *n;
4101 int work, weight;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004102
Stephen Hemmingerbea33482007-10-03 16:41:36 -07004103 /* If softirq window is exhuasted then punt.
Stephen Hemminger24f8b232008-11-03 17:14:38 -08004104 * Allow this to run for 2 jiffies since which will allow
4105 * an average latency of 1.5/HZ.
Stephen Hemmingerbea33482007-10-03 16:41:36 -07004106 */
Eric Dumazetd1f41b62013-03-05 07:15:13 +00004107 if (unlikely(budget <= 0 || time_after_eq(jiffies, time_limit)))
Linus Torvalds1da177e2005-04-16 15:20:36 -07004108 goto softnet_break;
4109
4110 local_irq_enable();
4111
Stephen Hemmingerbea33482007-10-03 16:41:36 -07004112 /* Even though interrupts have been re-enabled, this
4113 * access is safe because interrupts can only add new
4114 * entries to the tail of this list, and only ->poll()
4115 * calls can remove this head entry from the list.
4116 */
Eric Dumazete326bed2010-04-22 00:22:45 -07004117 n = list_first_entry(&sd->poll_list, struct napi_struct, poll_list);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004118
Stephen Hemmingerbea33482007-10-03 16:41:36 -07004119 have = netpoll_poll_lock(n);
4120
4121 weight = n->weight;
4122
David S. Miller0a7606c2007-10-29 21:28:47 -07004123 /* This NAPI_STATE_SCHED test is for avoiding a race
4124 * with netpoll's poll_napi(). Only the entity which
4125 * obtains the lock and sees NAPI_STATE_SCHED set will
4126 * actually make the ->poll() call. Therefore we avoid
Lucas De Marchi25985ed2011-03-30 22:57:33 -03004127 * accidentally calling ->poll() when NAPI is not scheduled.
David S. Miller0a7606c2007-10-29 21:28:47 -07004128 */
4129 work = 0;
Neil Horman4ea7e382009-05-21 07:36:08 +00004130 if (test_bit(NAPI_STATE_SCHED, &n->state)) {
David S. Miller0a7606c2007-10-29 21:28:47 -07004131 work = n->poll(n, weight);
Neil Horman4ea7e382009-05-21 07:36:08 +00004132 trace_napi_poll(n);
4133 }
Stephen Hemmingerbea33482007-10-03 16:41:36 -07004134
4135 WARN_ON_ONCE(work > weight);
4136
4137 budget -= work;
4138
4139 local_irq_disable();
4140
4141 /* Drivers must not modify the NAPI state if they
4142 * consume the entire weight. In such cases this code
4143 * still "owns" the NAPI instance and therefore can
4144 * move the instance around on the list at-will.
4145 */
David S. Millerfed17f32008-01-07 21:00:40 -08004146 if (unlikely(work == weight)) {
Herbert Xuff780cd2009-06-26 19:27:04 -07004147 if (unlikely(napi_disable_pending(n))) {
4148 local_irq_enable();
4149 napi_complete(n);
4150 local_irq_disable();
Eric Dumazet2e71a6f2012-10-06 08:08:49 +00004151 } else {
4152 if (n->gro_list) {
4153 /* flush too old packets
4154 * If HZ < 1000, flush all packets.
4155 */
4156 local_irq_enable();
4157 napi_gro_flush(n, HZ >= 1000);
4158 local_irq_disable();
4159 }
Eric Dumazete326bed2010-04-22 00:22:45 -07004160 list_move_tail(&n->poll_list, &sd->poll_list);
Eric Dumazet2e71a6f2012-10-06 08:08:49 +00004161 }
David S. Millerfed17f32008-01-07 21:00:40 -08004162 }
Stephen Hemmingerbea33482007-10-03 16:41:36 -07004163
4164 netpoll_poll_unlock(have);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004165 }
4166out:
Eric Dumazete326bed2010-04-22 00:22:45 -07004167 net_rps_action_and_irq_enable(sd);
Tom Herbert0a9627f2010-03-16 08:03:29 +00004168
Chris Leechdb217332006-06-17 21:24:58 -07004169#ifdef CONFIG_NET_DMA
4170 /*
4171 * There may not be any more sk_buffs coming right now, so push
4172 * any pending DMA copies to hardware
4173 */
Dan Williams2ba05622009-01-06 11:38:14 -07004174 dma_issue_pending_all();
Chris Leechdb217332006-06-17 21:24:58 -07004175#endif
Stephen Hemmingerbea33482007-10-03 16:41:36 -07004176
Linus Torvalds1da177e2005-04-16 15:20:36 -07004177 return;
4178
4179softnet_break:
Changli Gaodee42872010-05-02 05:42:16 +00004180 sd->time_squeeze++;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004181 __raise_softirq_irqoff(NET_RX_SOFTIRQ);
4182 goto out;
4183}
4184
Jiri Pirko9ff162a2013-01-03 22:48:49 +00004185struct netdev_upper {
4186 struct net_device *dev;
4187 bool master;
4188 struct list_head list;
4189 struct rcu_head rcu;
4190 struct list_head search_list;
4191};
4192
4193static void __append_search_uppers(struct list_head *search_list,
4194 struct net_device *dev)
4195{
4196 struct netdev_upper *upper;
4197
4198 list_for_each_entry(upper, &dev->upper_dev_list, list) {
4199 /* check if this upper is not already in search list */
4200 if (list_empty(&upper->search_list))
4201 list_add_tail(&upper->search_list, search_list);
4202 }
4203}
4204
4205static bool __netdev_search_upper_dev(struct net_device *dev,
4206 struct net_device *upper_dev)
4207{
4208 LIST_HEAD(search_list);
4209 struct netdev_upper *upper;
4210 struct netdev_upper *tmp;
4211 bool ret = false;
4212
4213 __append_search_uppers(&search_list, dev);
4214 list_for_each_entry(upper, &search_list, search_list) {
4215 if (upper->dev == upper_dev) {
4216 ret = true;
4217 break;
4218 }
4219 __append_search_uppers(&search_list, upper->dev);
4220 }
4221 list_for_each_entry_safe(upper, tmp, &search_list, search_list)
4222 INIT_LIST_HEAD(&upper->search_list);
4223 return ret;
4224}
4225
4226static struct netdev_upper *__netdev_find_upper(struct net_device *dev,
4227 struct net_device *upper_dev)
4228{
4229 struct netdev_upper *upper;
4230
4231 list_for_each_entry(upper, &dev->upper_dev_list, list) {
4232 if (upper->dev == upper_dev)
4233 return upper;
4234 }
4235 return NULL;
4236}
4237
4238/**
4239 * netdev_has_upper_dev - Check if device is linked to an upper device
4240 * @dev: device
4241 * @upper_dev: upper device to check
4242 *
4243 * Find out if a device is linked to specified upper device and return true
4244 * in case it is. Note that this checks only immediate upper device,
4245 * not through a complete stack of devices. The caller must hold the RTNL lock.
4246 */
4247bool netdev_has_upper_dev(struct net_device *dev,
4248 struct net_device *upper_dev)
4249{
4250 ASSERT_RTNL();
4251
4252 return __netdev_find_upper(dev, upper_dev);
4253}
4254EXPORT_SYMBOL(netdev_has_upper_dev);
4255
4256/**
4257 * netdev_has_any_upper_dev - Check if device is linked to some device
4258 * @dev: device
4259 *
4260 * Find out if a device is linked to an upper device and return true in case
4261 * it is. The caller must hold the RTNL lock.
4262 */
4263bool netdev_has_any_upper_dev(struct net_device *dev)
4264{
4265 ASSERT_RTNL();
4266
4267 return !list_empty(&dev->upper_dev_list);
4268}
4269EXPORT_SYMBOL(netdev_has_any_upper_dev);
4270
4271/**
4272 * netdev_master_upper_dev_get - Get master upper device
4273 * @dev: device
4274 *
4275 * Find a master upper device and return pointer to it or NULL in case
4276 * it's not there. The caller must hold the RTNL lock.
4277 */
4278struct net_device *netdev_master_upper_dev_get(struct net_device *dev)
4279{
4280 struct netdev_upper *upper;
4281
4282 ASSERT_RTNL();
4283
4284 if (list_empty(&dev->upper_dev_list))
4285 return NULL;
4286
4287 upper = list_first_entry(&dev->upper_dev_list,
4288 struct netdev_upper, list);
4289 if (likely(upper->master))
4290 return upper->dev;
4291 return NULL;
4292}
4293EXPORT_SYMBOL(netdev_master_upper_dev_get);
4294
4295/**
4296 * netdev_master_upper_dev_get_rcu - Get master upper device
4297 * @dev: device
4298 *
4299 * Find a master upper device and return pointer to it or NULL in case
4300 * it's not there. The caller must hold the RCU read lock.
4301 */
4302struct net_device *netdev_master_upper_dev_get_rcu(struct net_device *dev)
4303{
4304 struct netdev_upper *upper;
4305
4306 upper = list_first_or_null_rcu(&dev->upper_dev_list,
4307 struct netdev_upper, list);
4308 if (upper && likely(upper->master))
4309 return upper->dev;
4310 return NULL;
4311}
4312EXPORT_SYMBOL(netdev_master_upper_dev_get_rcu);
4313
4314static int __netdev_upper_dev_link(struct net_device *dev,
4315 struct net_device *upper_dev, bool master)
4316{
4317 struct netdev_upper *upper;
4318
4319 ASSERT_RTNL();
4320
4321 if (dev == upper_dev)
4322 return -EBUSY;
4323
4324 /* To prevent loops, check if dev is not upper device to upper_dev. */
4325 if (__netdev_search_upper_dev(upper_dev, dev))
4326 return -EBUSY;
4327
4328 if (__netdev_find_upper(dev, upper_dev))
4329 return -EEXIST;
4330
4331 if (master && netdev_master_upper_dev_get(dev))
4332 return -EBUSY;
4333
4334 upper = kmalloc(sizeof(*upper), GFP_KERNEL);
4335 if (!upper)
4336 return -ENOMEM;
4337
4338 upper->dev = upper_dev;
4339 upper->master = master;
4340 INIT_LIST_HEAD(&upper->search_list);
4341
4342 /* Ensure that master upper link is always the first item in list. */
4343 if (master)
4344 list_add_rcu(&upper->list, &dev->upper_dev_list);
4345 else
4346 list_add_tail_rcu(&upper->list, &dev->upper_dev_list);
4347 dev_hold(upper_dev);
4348
4349 return 0;
4350}
4351
4352/**
4353 * netdev_upper_dev_link - Add a link to the upper device
4354 * @dev: device
4355 * @upper_dev: new upper device
4356 *
4357 * Adds a link to device which is upper to this one. The caller must hold
4358 * the RTNL lock. On a failure a negative errno code is returned.
4359 * On success the reference counts are adjusted and the function
4360 * returns zero.
4361 */
4362int netdev_upper_dev_link(struct net_device *dev,
4363 struct net_device *upper_dev)
4364{
4365 return __netdev_upper_dev_link(dev, upper_dev, false);
4366}
4367EXPORT_SYMBOL(netdev_upper_dev_link);
4368
4369/**
4370 * netdev_master_upper_dev_link - Add a master link to the upper device
4371 * @dev: device
4372 * @upper_dev: new upper device
4373 *
4374 * Adds a link to device which is upper to this one. In this case, only
4375 * one master upper device can be linked, although other non-master devices
4376 * might be linked as well. The caller must hold the RTNL lock.
4377 * On a failure a negative errno code is returned. On success the reference
4378 * counts are adjusted and the function returns zero.
4379 */
4380int netdev_master_upper_dev_link(struct net_device *dev,
4381 struct net_device *upper_dev)
4382{
4383 return __netdev_upper_dev_link(dev, upper_dev, true);
4384}
4385EXPORT_SYMBOL(netdev_master_upper_dev_link);
4386
4387/**
4388 * netdev_upper_dev_unlink - Removes a link to upper device
4389 * @dev: device
4390 * @upper_dev: new upper device
4391 *
4392 * Removes a link to device which is upper to this one. The caller must hold
4393 * the RTNL lock.
4394 */
4395void netdev_upper_dev_unlink(struct net_device *dev,
4396 struct net_device *upper_dev)
4397{
4398 struct netdev_upper *upper;
4399
4400 ASSERT_RTNL();
4401
4402 upper = __netdev_find_upper(dev, upper_dev);
4403 if (!upper)
4404 return;
4405 list_del_rcu(&upper->list);
4406 dev_put(upper_dev);
4407 kfree_rcu(upper, rcu);
4408}
4409EXPORT_SYMBOL(netdev_upper_dev_unlink);
4410
Patrick McHardyb6c40d62008-10-07 15:26:48 -07004411static void dev_change_rx_flags(struct net_device *dev, int flags)
4412{
Stephen Hemmingerd3147742008-11-19 21:32:24 -08004413 const struct net_device_ops *ops = dev->netdev_ops;
4414
4415 if ((dev->flags & IFF_UP) && ops->ndo_change_rx_flags)
4416 ops->ndo_change_rx_flags(dev, flags);
Patrick McHardyb6c40d62008-10-07 15:26:48 -07004417}
4418
Wang Chendad9b332008-06-18 01:48:28 -07004419static int __dev_set_promiscuity(struct net_device *dev, int inc)
Patrick McHardy4417da62007-06-27 01:28:10 -07004420{
Eric Dumazetb536db92011-11-30 21:42:26 +00004421 unsigned int old_flags = dev->flags;
Eric W. Biedermand04a48b2012-05-23 17:01:57 -06004422 kuid_t uid;
4423 kgid_t gid;
Patrick McHardy4417da62007-06-27 01:28:10 -07004424
Patrick McHardy24023452007-07-14 18:51:31 -07004425 ASSERT_RTNL();
4426
Wang Chendad9b332008-06-18 01:48:28 -07004427 dev->flags |= IFF_PROMISC;
4428 dev->promiscuity += inc;
4429 if (dev->promiscuity == 0) {
4430 /*
4431 * Avoid overflow.
4432 * If inc causes overflow, untouch promisc and return error.
4433 */
4434 if (inc < 0)
4435 dev->flags &= ~IFF_PROMISC;
4436 else {
4437 dev->promiscuity -= inc;
Joe Perches7b6cd1c2012-02-01 10:54:43 +00004438 pr_warn("%s: promiscuity touches roof, set promiscuity failed. promiscuity feature of device might be broken.\n",
4439 dev->name);
Wang Chendad9b332008-06-18 01:48:28 -07004440 return -EOVERFLOW;
4441 }
4442 }
Patrick McHardy4417da62007-06-27 01:28:10 -07004443 if (dev->flags != old_flags) {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00004444 pr_info("device %s %s promiscuous mode\n",
4445 dev->name,
4446 dev->flags & IFF_PROMISC ? "entered" : "left");
David Howells8192b0c2008-11-14 10:39:10 +11004447 if (audit_enabled) {
4448 current_uid_gid(&uid, &gid);
Klaus Heinrich Kiwi7759db82008-01-23 22:57:45 -05004449 audit_log(current->audit_context, GFP_ATOMIC,
4450 AUDIT_ANOM_PROMISCUOUS,
4451 "dev=%s prom=%d old_prom=%d auid=%u uid=%u gid=%u ses=%u",
4452 dev->name, (dev->flags & IFF_PROMISC),
4453 (old_flags & IFF_PROMISC),
Eric W. Biedermane1760bd2012-09-10 22:39:43 -07004454 from_kuid(&init_user_ns, audit_get_loginuid(current)),
Eric W. Biedermand04a48b2012-05-23 17:01:57 -06004455 from_kuid(&init_user_ns, uid),
4456 from_kgid(&init_user_ns, gid),
Klaus Heinrich Kiwi7759db82008-01-23 22:57:45 -05004457 audit_get_sessionid(current));
David Howells8192b0c2008-11-14 10:39:10 +11004458 }
Patrick McHardy24023452007-07-14 18:51:31 -07004459
Patrick McHardyb6c40d62008-10-07 15:26:48 -07004460 dev_change_rx_flags(dev, IFF_PROMISC);
Patrick McHardy4417da62007-06-27 01:28:10 -07004461 }
Wang Chendad9b332008-06-18 01:48:28 -07004462 return 0;
Patrick McHardy4417da62007-06-27 01:28:10 -07004463}
4464
Linus Torvalds1da177e2005-04-16 15:20:36 -07004465/**
4466 * dev_set_promiscuity - update promiscuity count on a device
4467 * @dev: device
4468 * @inc: modifier
4469 *
Stephen Hemminger3041a062006-05-26 13:25:24 -07004470 * Add or remove promiscuity from a device. While the count in the device
Linus Torvalds1da177e2005-04-16 15:20:36 -07004471 * remains above zero the interface remains promiscuous. Once it hits zero
4472 * the device reverts back to normal filtering operation. A negative inc
4473 * value is used to drop promiscuity on the device.
Wang Chendad9b332008-06-18 01:48:28 -07004474 * Return 0 if successful or a negative errno code on error.
Linus Torvalds1da177e2005-04-16 15:20:36 -07004475 */
Wang Chendad9b332008-06-18 01:48:28 -07004476int dev_set_promiscuity(struct net_device *dev, int inc)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004477{
Eric Dumazetb536db92011-11-30 21:42:26 +00004478 unsigned int old_flags = dev->flags;
Wang Chendad9b332008-06-18 01:48:28 -07004479 int err;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004480
Wang Chendad9b332008-06-18 01:48:28 -07004481 err = __dev_set_promiscuity(dev, inc);
Patrick McHardy4b5a6982008-07-06 15:49:08 -07004482 if (err < 0)
Wang Chendad9b332008-06-18 01:48:28 -07004483 return err;
Patrick McHardy4417da62007-06-27 01:28:10 -07004484 if (dev->flags != old_flags)
4485 dev_set_rx_mode(dev);
Wang Chendad9b332008-06-18 01:48:28 -07004486 return err;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004487}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004488EXPORT_SYMBOL(dev_set_promiscuity);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004489
4490/**
4491 * dev_set_allmulti - update allmulti count on a device
4492 * @dev: device
4493 * @inc: modifier
4494 *
4495 * Add or remove reception of all multicast frames to a device. While the
4496 * count in the device remains above zero the interface remains listening
4497 * to all interfaces. Once it hits zero the device reverts back to normal
4498 * filtering operation. A negative @inc value is used to drop the counter
4499 * when releasing a resource needing all multicasts.
Wang Chendad9b332008-06-18 01:48:28 -07004500 * Return 0 if successful or a negative errno code on error.
Linus Torvalds1da177e2005-04-16 15:20:36 -07004501 */
4502
Wang Chendad9b332008-06-18 01:48:28 -07004503int dev_set_allmulti(struct net_device *dev, int inc)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004504{
Eric Dumazetb536db92011-11-30 21:42:26 +00004505 unsigned int old_flags = dev->flags;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004506
Patrick McHardy24023452007-07-14 18:51:31 -07004507 ASSERT_RTNL();
4508
Linus Torvalds1da177e2005-04-16 15:20:36 -07004509 dev->flags |= IFF_ALLMULTI;
Wang Chendad9b332008-06-18 01:48:28 -07004510 dev->allmulti += inc;
4511 if (dev->allmulti == 0) {
4512 /*
4513 * Avoid overflow.
4514 * If inc causes overflow, untouch allmulti and return error.
4515 */
4516 if (inc < 0)
4517 dev->flags &= ~IFF_ALLMULTI;
4518 else {
4519 dev->allmulti -= inc;
Joe Perches7b6cd1c2012-02-01 10:54:43 +00004520 pr_warn("%s: allmulti touches roof, set allmulti failed. allmulti feature of device might be broken.\n",
4521 dev->name);
Wang Chendad9b332008-06-18 01:48:28 -07004522 return -EOVERFLOW;
4523 }
4524 }
Patrick McHardy24023452007-07-14 18:51:31 -07004525 if (dev->flags ^ old_flags) {
Patrick McHardyb6c40d62008-10-07 15:26:48 -07004526 dev_change_rx_flags(dev, IFF_ALLMULTI);
Patrick McHardy4417da62007-06-27 01:28:10 -07004527 dev_set_rx_mode(dev);
Patrick McHardy24023452007-07-14 18:51:31 -07004528 }
Wang Chendad9b332008-06-18 01:48:28 -07004529 return 0;
Patrick McHardy4417da62007-06-27 01:28:10 -07004530}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004531EXPORT_SYMBOL(dev_set_allmulti);
Patrick McHardy4417da62007-06-27 01:28:10 -07004532
4533/*
4534 * Upload unicast and multicast address lists to device and
4535 * configure RX filtering. When the device doesn't support unicast
Joe Perches53ccaae2007-12-20 14:02:06 -08004536 * filtering it is put in promiscuous mode while unicast addresses
Patrick McHardy4417da62007-06-27 01:28:10 -07004537 * are present.
4538 */
4539void __dev_set_rx_mode(struct net_device *dev)
4540{
Stephen Hemmingerd3147742008-11-19 21:32:24 -08004541 const struct net_device_ops *ops = dev->netdev_ops;
4542
Patrick McHardy4417da62007-06-27 01:28:10 -07004543 /* dev_open will call this function so the list will stay sane. */
4544 if (!(dev->flags&IFF_UP))
4545 return;
4546
4547 if (!netif_device_present(dev))
YOSHIFUJI Hideaki40b77c92007-07-19 10:43:23 +09004548 return;
Patrick McHardy4417da62007-06-27 01:28:10 -07004549
Jiri Pirko01789342011-08-16 06:29:00 +00004550 if (!(dev->priv_flags & IFF_UNICAST_FLT)) {
Patrick McHardy4417da62007-06-27 01:28:10 -07004551 /* Unicast addresses changes may only happen under the rtnl,
4552 * therefore calling __dev_set_promiscuity here is safe.
4553 */
Jiri Pirko32e7bfc2010-01-25 13:36:10 -08004554 if (!netdev_uc_empty(dev) && !dev->uc_promisc) {
Patrick McHardy4417da62007-06-27 01:28:10 -07004555 __dev_set_promiscuity(dev, 1);
Joe Perches2d348d12011-07-25 16:17:35 -07004556 dev->uc_promisc = true;
Jiri Pirko32e7bfc2010-01-25 13:36:10 -08004557 } else if (netdev_uc_empty(dev) && dev->uc_promisc) {
Patrick McHardy4417da62007-06-27 01:28:10 -07004558 __dev_set_promiscuity(dev, -1);
Joe Perches2d348d12011-07-25 16:17:35 -07004559 dev->uc_promisc = false;
Patrick McHardy4417da62007-06-27 01:28:10 -07004560 }
Patrick McHardy4417da62007-06-27 01:28:10 -07004561 }
Jiri Pirko01789342011-08-16 06:29:00 +00004562
4563 if (ops->ndo_set_rx_mode)
4564 ops->ndo_set_rx_mode(dev);
Patrick McHardy4417da62007-06-27 01:28:10 -07004565}
4566
4567void dev_set_rx_mode(struct net_device *dev)
4568{
David S. Millerb9e40852008-07-15 00:15:08 -07004569 netif_addr_lock_bh(dev);
Patrick McHardy4417da62007-06-27 01:28:10 -07004570 __dev_set_rx_mode(dev);
David S. Millerb9e40852008-07-15 00:15:08 -07004571 netif_addr_unlock_bh(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004572}
4573
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07004574/**
4575 * dev_get_flags - get flags reported to userspace
4576 * @dev: device
4577 *
4578 * Get the combination of flag bits exported through APIs to userspace.
4579 */
Eric Dumazet95c96172012-04-15 05:58:06 +00004580unsigned int dev_get_flags(const struct net_device *dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004581{
Eric Dumazet95c96172012-04-15 05:58:06 +00004582 unsigned int flags;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004583
4584 flags = (dev->flags & ~(IFF_PROMISC |
4585 IFF_ALLMULTI |
Stefan Rompfb00055a2006-03-20 17:09:11 -08004586 IFF_RUNNING |
4587 IFF_LOWER_UP |
4588 IFF_DORMANT)) |
Linus Torvalds1da177e2005-04-16 15:20:36 -07004589 (dev->gflags & (IFF_PROMISC |
4590 IFF_ALLMULTI));
4591
Stefan Rompfb00055a2006-03-20 17:09:11 -08004592 if (netif_running(dev)) {
4593 if (netif_oper_up(dev))
4594 flags |= IFF_RUNNING;
4595 if (netif_carrier_ok(dev))
4596 flags |= IFF_LOWER_UP;
4597 if (netif_dormant(dev))
4598 flags |= IFF_DORMANT;
4599 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07004600
4601 return flags;
4602}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004603EXPORT_SYMBOL(dev_get_flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004604
Patrick McHardybd380812010-02-26 06:34:53 +00004605int __dev_change_flags(struct net_device *dev, unsigned int flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004606{
Eric Dumazetb536db92011-11-30 21:42:26 +00004607 unsigned int old_flags = dev->flags;
Patrick McHardybd380812010-02-26 06:34:53 +00004608 int ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004609
Patrick McHardy24023452007-07-14 18:51:31 -07004610 ASSERT_RTNL();
4611
Linus Torvalds1da177e2005-04-16 15:20:36 -07004612 /*
4613 * Set the flags on our device.
4614 */
4615
4616 dev->flags = (flags & (IFF_DEBUG | IFF_NOTRAILERS | IFF_NOARP |
4617 IFF_DYNAMIC | IFF_MULTICAST | IFF_PORTSEL |
4618 IFF_AUTOMEDIA)) |
4619 (dev->flags & (IFF_UP | IFF_VOLATILE | IFF_PROMISC |
4620 IFF_ALLMULTI));
4621
4622 /*
4623 * Load in the correct multicast list now the flags have changed.
4624 */
4625
Patrick McHardyb6c40d62008-10-07 15:26:48 -07004626 if ((old_flags ^ flags) & IFF_MULTICAST)
4627 dev_change_rx_flags(dev, IFF_MULTICAST);
Patrick McHardy24023452007-07-14 18:51:31 -07004628
Patrick McHardy4417da62007-06-27 01:28:10 -07004629 dev_set_rx_mode(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004630
4631 /*
4632 * Have we downed the interface. We handle IFF_UP ourselves
4633 * according to user attempts to set it, rather than blindly
4634 * setting it.
4635 */
4636
4637 ret = 0;
4638 if ((old_flags ^ flags) & IFF_UP) { /* Bit is different ? */
Patrick McHardybd380812010-02-26 06:34:53 +00004639 ret = ((old_flags & IFF_UP) ? __dev_close : __dev_open)(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004640
4641 if (!ret)
Patrick McHardy4417da62007-06-27 01:28:10 -07004642 dev_set_rx_mode(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004643 }
4644
Linus Torvalds1da177e2005-04-16 15:20:36 -07004645 if ((flags ^ dev->gflags) & IFF_PROMISC) {
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004646 int inc = (flags & IFF_PROMISC) ? 1 : -1;
4647
Linus Torvalds1da177e2005-04-16 15:20:36 -07004648 dev->gflags ^= IFF_PROMISC;
4649 dev_set_promiscuity(dev, inc);
4650 }
4651
4652 /* NOTE: order of synchronization of IFF_PROMISC and IFF_ALLMULTI
4653 is important. Some (broken) drivers set IFF_PROMISC, when
4654 IFF_ALLMULTI is requested not asking us and not reporting.
4655 */
4656 if ((flags ^ dev->gflags) & IFF_ALLMULTI) {
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004657 int inc = (flags & IFF_ALLMULTI) ? 1 : -1;
4658
Linus Torvalds1da177e2005-04-16 15:20:36 -07004659 dev->gflags ^= IFF_ALLMULTI;
4660 dev_set_allmulti(dev, inc);
4661 }
4662
Patrick McHardybd380812010-02-26 06:34:53 +00004663 return ret;
4664}
4665
4666void __dev_notify_flags(struct net_device *dev, unsigned int old_flags)
4667{
4668 unsigned int changes = dev->flags ^ old_flags;
4669
4670 if (changes & IFF_UP) {
4671 if (dev->flags & IFF_UP)
4672 call_netdevice_notifiers(NETDEV_UP, dev);
4673 else
4674 call_netdevice_notifiers(NETDEV_DOWN, dev);
4675 }
4676
4677 if (dev->flags & IFF_UP &&
4678 (changes & ~(IFF_UP | IFF_PROMISC | IFF_ALLMULTI | IFF_VOLATILE)))
4679 call_netdevice_notifiers(NETDEV_CHANGE, dev);
4680}
4681
4682/**
4683 * dev_change_flags - change device settings
4684 * @dev: device
4685 * @flags: device state flags
4686 *
4687 * Change settings on device based state flags. The flags are
4688 * in the userspace exported format.
4689 */
Eric Dumazetb536db92011-11-30 21:42:26 +00004690int dev_change_flags(struct net_device *dev, unsigned int flags)
Patrick McHardybd380812010-02-26 06:34:53 +00004691{
Eric Dumazetb536db92011-11-30 21:42:26 +00004692 int ret;
4693 unsigned int changes, old_flags = dev->flags;
Patrick McHardybd380812010-02-26 06:34:53 +00004694
4695 ret = __dev_change_flags(dev, flags);
4696 if (ret < 0)
4697 return ret;
4698
4699 changes = old_flags ^ dev->flags;
Thomas Graf7c355f52007-06-05 16:03:03 -07004700 if (changes)
4701 rtmsg_ifinfo(RTM_NEWLINK, dev, changes);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004702
Patrick McHardybd380812010-02-26 06:34:53 +00004703 __dev_notify_flags(dev, old_flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004704 return ret;
4705}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004706EXPORT_SYMBOL(dev_change_flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004707
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07004708/**
4709 * dev_set_mtu - Change maximum transfer unit
4710 * @dev: device
4711 * @new_mtu: new transfer unit
4712 *
4713 * Change the maximum transfer size of the network device.
4714 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07004715int dev_set_mtu(struct net_device *dev, int new_mtu)
4716{
Stephen Hemmingerd3147742008-11-19 21:32:24 -08004717 const struct net_device_ops *ops = dev->netdev_ops;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004718 int err;
4719
4720 if (new_mtu == dev->mtu)
4721 return 0;
4722
4723 /* MTU must be positive. */
4724 if (new_mtu < 0)
4725 return -EINVAL;
4726
4727 if (!netif_device_present(dev))
4728 return -ENODEV;
4729
4730 err = 0;
Stephen Hemmingerd3147742008-11-19 21:32:24 -08004731 if (ops->ndo_change_mtu)
4732 err = ops->ndo_change_mtu(dev, new_mtu);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004733 else
4734 dev->mtu = new_mtu;
Stephen Hemmingerd3147742008-11-19 21:32:24 -08004735
Jiri Pirkoe3d8fab2012-12-03 01:16:32 +00004736 if (!err)
Pavel Emelyanov056925a2007-09-16 15:42:43 -07004737 call_netdevice_notifiers(NETDEV_CHANGEMTU, dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004738 return err;
4739}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004740EXPORT_SYMBOL(dev_set_mtu);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004741
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07004742/**
Vlad Dogarucbda10f2011-01-13 23:38:30 +00004743 * dev_set_group - Change group this device belongs to
4744 * @dev: device
4745 * @new_group: group this device should belong to
4746 */
4747void dev_set_group(struct net_device *dev, int new_group)
4748{
4749 dev->group = new_group;
4750}
4751EXPORT_SYMBOL(dev_set_group);
4752
4753/**
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07004754 * dev_set_mac_address - Change Media Access Control Address
4755 * @dev: device
4756 * @sa: new address
4757 *
4758 * Change the hardware (MAC) address of the device
4759 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07004760int dev_set_mac_address(struct net_device *dev, struct sockaddr *sa)
4761{
Stephen Hemmingerd3147742008-11-19 21:32:24 -08004762 const struct net_device_ops *ops = dev->netdev_ops;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004763 int err;
4764
Stephen Hemmingerd3147742008-11-19 21:32:24 -08004765 if (!ops->ndo_set_mac_address)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004766 return -EOPNOTSUPP;
4767 if (sa->sa_family != dev->type)
4768 return -EINVAL;
4769 if (!netif_device_present(dev))
4770 return -ENODEV;
Stephen Hemmingerd3147742008-11-19 21:32:24 -08004771 err = ops->ndo_set_mac_address(dev, sa);
Jiri Pirkof6521512013-01-01 03:30:14 +00004772 if (err)
4773 return err;
Jiri Pirkofbdeca22013-01-01 03:30:16 +00004774 dev->addr_assign_type = NET_ADDR_SET;
Jiri Pirkof6521512013-01-01 03:30:14 +00004775 call_netdevice_notifiers(NETDEV_CHANGEADDR, dev);
Theodore Ts'o7bf23572012-07-04 21:23:25 -04004776 add_device_randomness(dev->dev_addr, dev->addr_len);
Jiri Pirkof6521512013-01-01 03:30:14 +00004777 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004778}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004779EXPORT_SYMBOL(dev_set_mac_address);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004780
Jiri Pirko4bf84c32012-12-27 23:49:37 +00004781/**
4782 * dev_change_carrier - Change device carrier
4783 * @dev: device
Randy Dunlap691b3b72013-03-04 12:32:43 +00004784 * @new_carrier: new value
Jiri Pirko4bf84c32012-12-27 23:49:37 +00004785 *
4786 * Change device carrier
4787 */
4788int dev_change_carrier(struct net_device *dev, bool new_carrier)
4789{
4790 const struct net_device_ops *ops = dev->netdev_ops;
4791
4792 if (!ops->ndo_change_carrier)
4793 return -EOPNOTSUPP;
4794 if (!netif_device_present(dev))
4795 return -ENODEV;
4796 return ops->ndo_change_carrier(dev, new_carrier);
4797}
4798EXPORT_SYMBOL(dev_change_carrier);
4799
Linus Torvalds1da177e2005-04-16 15:20:36 -07004800/**
4801 * dev_new_index - allocate an ifindex
Randy Dunlapc4ea43c2007-10-12 21:17:49 -07004802 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -07004803 *
4804 * Returns a suitable unique value for a new device interface
4805 * number. The caller must hold the rtnl semaphore or the
4806 * dev_base_lock to be sure it remains unique.
4807 */
Eric W. Biederman881d9662007-09-17 11:56:21 -07004808static int dev_new_index(struct net *net)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004809{
Pavel Emelyanovaa79e662012-08-08 21:53:19 +00004810 int ifindex = net->ifindex;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004811 for (;;) {
4812 if (++ifindex <= 0)
4813 ifindex = 1;
Eric W. Biederman881d9662007-09-17 11:56:21 -07004814 if (!__dev_get_by_index(net, ifindex))
Pavel Emelyanovaa79e662012-08-08 21:53:19 +00004815 return net->ifindex = ifindex;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004816 }
4817}
4818
Linus Torvalds1da177e2005-04-16 15:20:36 -07004819/* Delayed registration/unregisteration */
Denis Cheng3b5b34f2007-12-07 00:49:17 -08004820static LIST_HEAD(net_todo_list);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004821
Stephen Hemminger6f05f622007-03-08 20:46:03 -08004822static void net_set_todo(struct net_device *dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004823{
Linus Torvalds1da177e2005-04-16 15:20:36 -07004824 list_add_tail(&dev->todo_list, &net_todo_list);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004825}
4826
Eric Dumazet9b5e3832009-10-27 07:04:19 +00004827static void rollback_registered_many(struct list_head *head)
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07004828{
Krishna Kumare93737b2009-12-08 22:26:02 +00004829 struct net_device *dev, *tmp;
Eric Dumazet9b5e3832009-10-27 07:04:19 +00004830
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07004831 BUG_ON(dev_boot_phase);
4832 ASSERT_RTNL();
4833
Krishna Kumare93737b2009-12-08 22:26:02 +00004834 list_for_each_entry_safe(dev, tmp, head, unreg_list) {
Eric Dumazet9b5e3832009-10-27 07:04:19 +00004835 /* Some devices call without registering
Krishna Kumare93737b2009-12-08 22:26:02 +00004836 * for initialization unwind. Remove those
4837 * devices and proceed with the remaining.
Eric Dumazet9b5e3832009-10-27 07:04:19 +00004838 */
4839 if (dev->reg_state == NETREG_UNINITIALIZED) {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00004840 pr_debug("unregister_netdevice: device %s/%p never was registered\n",
4841 dev->name, dev);
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07004842
Eric Dumazet9b5e3832009-10-27 07:04:19 +00004843 WARN_ON(1);
Krishna Kumare93737b2009-12-08 22:26:02 +00004844 list_del(&dev->unreg_list);
4845 continue;
Eric Dumazet9b5e3832009-10-27 07:04:19 +00004846 }
Eric Dumazet449f4542011-05-19 12:24:16 +00004847 dev->dismantle = true;
Eric Dumazet9b5e3832009-10-27 07:04:19 +00004848 BUG_ON(dev->reg_state != NETREG_REGISTERED);
Octavian Purdila44345722010-12-13 12:44:07 +00004849 }
Eric Dumazet9b5e3832009-10-27 07:04:19 +00004850
Octavian Purdila44345722010-12-13 12:44:07 +00004851 /* If device is running, close it first. */
4852 dev_close_many(head);
Eric Dumazet9b5e3832009-10-27 07:04:19 +00004853
Octavian Purdila44345722010-12-13 12:44:07 +00004854 list_for_each_entry(dev, head, unreg_list) {
Eric Dumazet9b5e3832009-10-27 07:04:19 +00004855 /* And unlink it from device chain. */
4856 unlist_netdevice(dev);
4857
4858 dev->reg_state = NETREG_UNREGISTERING;
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07004859 }
4860
Eric Dumazet9b5e3832009-10-27 07:04:19 +00004861 synchronize_net();
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07004862
Eric Dumazet9b5e3832009-10-27 07:04:19 +00004863 list_for_each_entry(dev, head, unreg_list) {
4864 /* Shutdown queueing discipline. */
4865 dev_shutdown(dev);
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07004866
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07004867
Eric Dumazet9b5e3832009-10-27 07:04:19 +00004868 /* Notify protocols, that we are about to destroy
4869 this device. They should clean all the things.
4870 */
4871 call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
4872
Patrick McHardya2835762010-02-26 06:34:51 +00004873 if (!dev->rtnl_link_ops ||
4874 dev->rtnl_link_state == RTNL_LINK_INITIALIZED)
4875 rtmsg_ifinfo(RTM_DELLINK, dev, ~0U);
4876
Eric Dumazet9b5e3832009-10-27 07:04:19 +00004877 /*
4878 * Flush the unicast and multicast chains
4879 */
Jiri Pirkoa748ee22010-04-01 21:22:09 +00004880 dev_uc_flush(dev);
Jiri Pirko22bedad32010-04-01 21:22:57 +00004881 dev_mc_flush(dev);
Eric Dumazet9b5e3832009-10-27 07:04:19 +00004882
4883 if (dev->netdev_ops->ndo_uninit)
4884 dev->netdev_ops->ndo_uninit(dev);
4885
Jiri Pirko9ff162a2013-01-03 22:48:49 +00004886 /* Notifier chain MUST detach us all upper devices. */
4887 WARN_ON(netdev_has_any_upper_dev(dev));
Eric Dumazet9b5e3832009-10-27 07:04:19 +00004888
4889 /* Remove entries from kobject tree */
4890 netdev_unregister_kobject(dev);
Alexander Duyck024e9672013-01-10 08:57:46 +00004891#ifdef CONFIG_XPS
4892 /* Remove XPS queueing entries */
4893 netif_reset_xps_queues_gt(dev, 0);
4894#endif
Eric Dumazet9b5e3832009-10-27 07:04:19 +00004895 }
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07004896
Eric W. Biederman850a5452011-10-13 22:25:23 +00004897 synchronize_net();
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07004898
Eric W. Biedermana5ee1552009-11-29 15:45:58 +00004899 list_for_each_entry(dev, head, unreg_list)
Eric Dumazet9b5e3832009-10-27 07:04:19 +00004900 dev_put(dev);
4901}
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07004902
Eric Dumazet9b5e3832009-10-27 07:04:19 +00004903static void rollback_registered(struct net_device *dev)
4904{
4905 LIST_HEAD(single);
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07004906
Eric Dumazet9b5e3832009-10-27 07:04:19 +00004907 list_add(&dev->unreg_list, &single);
4908 rollback_registered_many(&single);
Eric Dumazetceaaec92011-02-17 22:59:19 +00004909 list_del(&single);
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07004910}
4911
Michał Mirosławc8f44af2011-11-15 15:29:55 +00004912static netdev_features_t netdev_fix_features(struct net_device *dev,
4913 netdev_features_t features)
Herbert Xub63365a2008-10-23 01:11:29 -07004914{
Michał Mirosław57422dc2011-01-22 12:14:12 +00004915 /* Fix illegal checksum combinations */
4916 if ((features & NETIF_F_HW_CSUM) &&
4917 (features & (NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM))) {
Michał Mirosław6f404e42011-05-16 15:14:21 -04004918 netdev_warn(dev, "mixed HW and IP checksum settings.\n");
Michał Mirosław57422dc2011-01-22 12:14:12 +00004919 features &= ~(NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM);
4920 }
4921
Herbert Xub63365a2008-10-23 01:11:29 -07004922 /* Fix illegal SG+CSUM combinations. */
4923 if ((features & NETIF_F_SG) &&
4924 !(features & NETIF_F_ALL_CSUM)) {
Michał Mirosław6f404e42011-05-16 15:14:21 -04004925 netdev_dbg(dev,
4926 "Dropping NETIF_F_SG since no checksum feature.\n");
Herbert Xub63365a2008-10-23 01:11:29 -07004927 features &= ~NETIF_F_SG;
4928 }
4929
4930 /* TSO requires that SG is present as well. */
Ben Hutchingsea2d3682011-04-12 14:38:37 +00004931 if ((features & NETIF_F_ALL_TSO) && !(features & NETIF_F_SG)) {
Michał Mirosław6f404e42011-05-16 15:14:21 -04004932 netdev_dbg(dev, "Dropping TSO features since no SG feature.\n");
Ben Hutchingsea2d3682011-04-12 14:38:37 +00004933 features &= ~NETIF_F_ALL_TSO;
Herbert Xub63365a2008-10-23 01:11:29 -07004934 }
4935
Ben Hutchings31d8b9e2011-04-12 14:47:15 +00004936 /* TSO ECN requires that TSO is present as well. */
4937 if ((features & NETIF_F_ALL_TSO) == NETIF_F_TSO_ECN)
4938 features &= ~NETIF_F_TSO_ECN;
4939
Michał Mirosław212b5732011-02-15 16:59:16 +00004940 /* Software GSO depends on SG. */
4941 if ((features & NETIF_F_GSO) && !(features & NETIF_F_SG)) {
Michał Mirosław6f404e42011-05-16 15:14:21 -04004942 netdev_dbg(dev, "Dropping NETIF_F_GSO since no SG feature.\n");
Michał Mirosław212b5732011-02-15 16:59:16 +00004943 features &= ~NETIF_F_GSO;
4944 }
4945
Michał Mirosławacd11302011-01-24 15:45:15 -08004946 /* UFO needs SG and checksumming */
Herbert Xub63365a2008-10-23 01:11:29 -07004947 if (features & NETIF_F_UFO) {
Michał Mirosław79032642010-11-30 06:38:00 +00004948 /* maybe split UFO into V4 and V6? */
4949 if (!((features & NETIF_F_GEN_CSUM) ||
4950 (features & (NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM))
4951 == (NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM))) {
Michał Mirosław6f404e42011-05-16 15:14:21 -04004952 netdev_dbg(dev,
Michał Mirosławacd11302011-01-24 15:45:15 -08004953 "Dropping NETIF_F_UFO since no checksum offload features.\n");
Herbert Xub63365a2008-10-23 01:11:29 -07004954 features &= ~NETIF_F_UFO;
4955 }
4956
4957 if (!(features & NETIF_F_SG)) {
Michał Mirosław6f404e42011-05-16 15:14:21 -04004958 netdev_dbg(dev,
Michał Mirosławacd11302011-01-24 15:45:15 -08004959 "Dropping NETIF_F_UFO since no NETIF_F_SG feature.\n");
Herbert Xub63365a2008-10-23 01:11:29 -07004960 features &= ~NETIF_F_UFO;
4961 }
4962 }
4963
4964 return features;
4965}
Herbert Xub63365a2008-10-23 01:11:29 -07004966
Michał Mirosław6cb6a272011-04-02 22:48:47 -07004967int __netdev_update_features(struct net_device *dev)
Michał Mirosław5455c692011-02-15 16:59:17 +00004968{
Michał Mirosławc8f44af2011-11-15 15:29:55 +00004969 netdev_features_t features;
Michał Mirosław5455c692011-02-15 16:59:17 +00004970 int err = 0;
4971
Michał Mirosław87267482011-04-12 09:56:38 +00004972 ASSERT_RTNL();
4973
Michał Mirosław5455c692011-02-15 16:59:17 +00004974 features = netdev_get_wanted_features(dev);
4975
4976 if (dev->netdev_ops->ndo_fix_features)
4977 features = dev->netdev_ops->ndo_fix_features(dev, features);
4978
4979 /* driver might be less strict about feature dependencies */
4980 features = netdev_fix_features(dev, features);
4981
4982 if (dev->features == features)
Michał Mirosław6cb6a272011-04-02 22:48:47 -07004983 return 0;
Michał Mirosław5455c692011-02-15 16:59:17 +00004984
Michał Mirosławc8f44af2011-11-15 15:29:55 +00004985 netdev_dbg(dev, "Features changed: %pNF -> %pNF\n",
4986 &dev->features, &features);
Michał Mirosław5455c692011-02-15 16:59:17 +00004987
4988 if (dev->netdev_ops->ndo_set_features)
4989 err = dev->netdev_ops->ndo_set_features(dev, features);
4990
Michał Mirosław6cb6a272011-04-02 22:48:47 -07004991 if (unlikely(err < 0)) {
Michał Mirosław5455c692011-02-15 16:59:17 +00004992 netdev_err(dev,
Michał Mirosławc8f44af2011-11-15 15:29:55 +00004993 "set_features() failed (%d); wanted %pNF, left %pNF\n",
4994 err, &features, &dev->features);
Michał Mirosław6cb6a272011-04-02 22:48:47 -07004995 return -1;
4996 }
4997
4998 if (!err)
4999 dev->features = features;
5000
5001 return 1;
5002}
5003
Michał Mirosławafe12cc2011-05-07 03:22:17 +00005004/**
5005 * netdev_update_features - recalculate device features
5006 * @dev: the device to check
5007 *
5008 * Recalculate dev->features set and send notifications if it
5009 * has changed. Should be called after driver or hardware dependent
5010 * conditions might have changed that influence the features.
5011 */
Michał Mirosław6cb6a272011-04-02 22:48:47 -07005012void netdev_update_features(struct net_device *dev)
5013{
5014 if (__netdev_update_features(dev))
5015 netdev_features_change(dev);
Michał Mirosław5455c692011-02-15 16:59:17 +00005016}
5017EXPORT_SYMBOL(netdev_update_features);
5018
Linus Torvalds1da177e2005-04-16 15:20:36 -07005019/**
Michał Mirosławafe12cc2011-05-07 03:22:17 +00005020 * netdev_change_features - recalculate device features
5021 * @dev: the device to check
5022 *
5023 * Recalculate dev->features set and send notifications even
5024 * if they have not changed. Should be called instead of
5025 * netdev_update_features() if also dev->vlan_features might
5026 * have changed to allow the changes to be propagated to stacked
5027 * VLAN devices.
5028 */
5029void netdev_change_features(struct net_device *dev)
5030{
5031 __netdev_update_features(dev);
5032 netdev_features_change(dev);
5033}
5034EXPORT_SYMBOL(netdev_change_features);
5035
5036/**
Patrick Mullaneyfc4a7482009-12-03 15:59:22 -08005037 * netif_stacked_transfer_operstate - transfer operstate
5038 * @rootdev: the root or lower level device to transfer state from
5039 * @dev: the device to transfer operstate to
5040 *
5041 * Transfer operational state from root to device. This is normally
5042 * called when a stacking relationship exists between the root
5043 * device and the device(a leaf device).
5044 */
5045void netif_stacked_transfer_operstate(const struct net_device *rootdev,
5046 struct net_device *dev)
5047{
5048 if (rootdev->operstate == IF_OPER_DORMANT)
5049 netif_dormant_on(dev);
5050 else
5051 netif_dormant_off(dev);
5052
5053 if (netif_carrier_ok(rootdev)) {
5054 if (!netif_carrier_ok(dev))
5055 netif_carrier_on(dev);
5056 } else {
5057 if (netif_carrier_ok(dev))
5058 netif_carrier_off(dev);
5059 }
5060}
5061EXPORT_SYMBOL(netif_stacked_transfer_operstate);
5062
Tom Herbertbf264142010-11-26 08:36:09 +00005063#ifdef CONFIG_RPS
Eric Dumazet1b4bf462010-09-23 17:26:35 +00005064static int netif_alloc_rx_queues(struct net_device *dev)
5065{
Eric Dumazet1b4bf462010-09-23 17:26:35 +00005066 unsigned int i, count = dev->num_rx_queues;
Tom Herbertbd25fa72010-10-18 18:00:16 +00005067 struct netdev_rx_queue *rx;
Eric Dumazet1b4bf462010-09-23 17:26:35 +00005068
Tom Herbertbd25fa72010-10-18 18:00:16 +00005069 BUG_ON(count < 1);
Eric Dumazet1b4bf462010-09-23 17:26:35 +00005070
Tom Herbertbd25fa72010-10-18 18:00:16 +00005071 rx = kcalloc(count, sizeof(struct netdev_rx_queue), GFP_KERNEL);
Joe Perches62b59422013-02-04 16:48:16 +00005072 if (!rx)
Tom Herbertbd25fa72010-10-18 18:00:16 +00005073 return -ENOMEM;
Joe Perches62b59422013-02-04 16:48:16 +00005074
Tom Herbertbd25fa72010-10-18 18:00:16 +00005075 dev->_rx = rx;
5076
Tom Herbertbd25fa72010-10-18 18:00:16 +00005077 for (i = 0; i < count; i++)
Tom Herbertfe822242010-11-09 10:47:38 +00005078 rx[i].dev = dev;
Eric Dumazet1b4bf462010-09-23 17:26:35 +00005079 return 0;
5080}
Tom Herbertbf264142010-11-26 08:36:09 +00005081#endif
Eric Dumazet1b4bf462010-09-23 17:26:35 +00005082
Changli Gaoaa942102010-12-04 02:31:41 +00005083static void netdev_init_one_queue(struct net_device *dev,
5084 struct netdev_queue *queue, void *_unused)
5085{
5086 /* Initialize queue lock */
5087 spin_lock_init(&queue->_xmit_lock);
5088 netdev_set_xmit_lockdep_class(&queue->_xmit_lock, dev->type);
5089 queue->xmit_lock_owner = -1;
Changli Gaob236da62010-12-14 03:09:15 +00005090 netdev_queue_numa_node_write(queue, NUMA_NO_NODE);
Changli Gaoaa942102010-12-04 02:31:41 +00005091 queue->dev = dev;
Tom Herbert114cf582011-11-28 16:33:09 +00005092#ifdef CONFIG_BQL
5093 dql_init(&queue->dql, HZ);
5094#endif
Changli Gaoaa942102010-12-04 02:31:41 +00005095}
5096
Tom Herberte6484932010-10-18 18:04:39 +00005097static int netif_alloc_netdev_queues(struct net_device *dev)
5098{
5099 unsigned int count = dev->num_tx_queues;
5100 struct netdev_queue *tx;
5101
5102 BUG_ON(count < 1);
5103
5104 tx = kcalloc(count, sizeof(struct netdev_queue), GFP_KERNEL);
Joe Perches62b59422013-02-04 16:48:16 +00005105 if (!tx)
Tom Herberte6484932010-10-18 18:04:39 +00005106 return -ENOMEM;
Joe Perches62b59422013-02-04 16:48:16 +00005107
Tom Herberte6484932010-10-18 18:04:39 +00005108 dev->_tx = tx;
Tom Herbert1d24eb42010-11-21 13:17:27 +00005109
Tom Herberte6484932010-10-18 18:04:39 +00005110 netdev_for_each_tx_queue(dev, netdev_init_one_queue, NULL);
5111 spin_lock_init(&dev->tx_global_lock);
Changli Gaoaa942102010-12-04 02:31:41 +00005112
5113 return 0;
Tom Herberte6484932010-10-18 18:04:39 +00005114}
5115
Patrick Mullaneyfc4a7482009-12-03 15:59:22 -08005116/**
Linus Torvalds1da177e2005-04-16 15:20:36 -07005117 * register_netdevice - register a network device
5118 * @dev: device to register
5119 *
5120 * Take a completed network device structure and add it to the kernel
5121 * interfaces. A %NETDEV_REGISTER message is sent to the netdev notifier
5122 * chain. 0 is returned on success. A negative errno code is returned
5123 * on a failure to set up the device, or if the name is a duplicate.
5124 *
5125 * Callers must hold the rtnl semaphore. You may want
5126 * register_netdev() instead of this.
5127 *
5128 * BUGS:
5129 * The locking appears insufficient to guarantee two parallel registers
5130 * will not get the same name.
5131 */
5132
5133int register_netdevice(struct net_device *dev)
5134{
Linus Torvalds1da177e2005-04-16 15:20:36 -07005135 int ret;
Stephen Hemmingerd3147742008-11-19 21:32:24 -08005136 struct net *net = dev_net(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005137
5138 BUG_ON(dev_boot_phase);
5139 ASSERT_RTNL();
5140
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07005141 might_sleep();
5142
Linus Torvalds1da177e2005-04-16 15:20:36 -07005143 /* When net_device's are persistent, this will be fatal. */
5144 BUG_ON(dev->reg_state != NETREG_UNINITIALIZED);
Stephen Hemmingerd3147742008-11-19 21:32:24 -08005145 BUG_ON(!net);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005146
David S. Millerf1f28aa2008-07-15 00:08:33 -07005147 spin_lock_init(&dev->addr_list_lock);
David S. Millercf508b12008-07-22 14:16:42 -07005148 netdev_set_addr_lockdep_class(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005149
Linus Torvalds1da177e2005-04-16 15:20:36 -07005150 dev->iflink = -1;
5151
Gao feng828de4f2012-09-13 20:58:27 +00005152 ret = dev_get_valid_name(net, dev, dev->name);
Peter Pan(潘卫平)0696c3a2011-05-12 15:46:56 +00005153 if (ret < 0)
5154 goto out;
5155
Linus Torvalds1da177e2005-04-16 15:20:36 -07005156 /* Init, if this function is available */
Stephen Hemmingerd3147742008-11-19 21:32:24 -08005157 if (dev->netdev_ops->ndo_init) {
5158 ret = dev->netdev_ops->ndo_init(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005159 if (ret) {
5160 if (ret > 0)
5161 ret = -EIO;
Adrian Bunk90833aa2006-11-13 16:02:22 -08005162 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005163 }
5164 }
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09005165
Michał Mirosławd2ed2732013-01-29 15:14:16 +00005166 if (((dev->hw_features | dev->features) & NETIF_F_HW_VLAN_FILTER) &&
5167 (!dev->netdev_ops->ndo_vlan_rx_add_vid ||
5168 !dev->netdev_ops->ndo_vlan_rx_kill_vid)) {
5169 netdev_WARN(dev, "Buggy VLAN acceleration in driver!\n");
5170 ret = -EINVAL;
5171 goto err_uninit;
5172 }
5173
Pavel Emelyanov9c7dafb2012-08-08 21:52:46 +00005174 ret = -EBUSY;
5175 if (!dev->ifindex)
5176 dev->ifindex = dev_new_index(net);
5177 else if (__dev_get_by_index(net, dev->ifindex))
5178 goto err_uninit;
5179
Linus Torvalds1da177e2005-04-16 15:20:36 -07005180 if (dev->iflink == -1)
5181 dev->iflink = dev->ifindex;
5182
Michał Mirosław5455c692011-02-15 16:59:17 +00005183 /* Transfer changeable features to wanted_features and enable
5184 * software offloads (GSO and GRO).
5185 */
5186 dev->hw_features |= NETIF_F_SOFT_FEATURES;
Michał Mirosław14d12322011-02-22 16:52:28 +00005187 dev->features |= NETIF_F_SOFT_FEATURES;
5188 dev->wanted_features = dev->features & dev->hw_features;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005189
Tom Herbertc6e1a0d2011-04-04 22:30:30 -07005190 /* Turn on no cache copy if HW is doing checksum */
Michał Mirosław34324dc2011-11-15 15:29:55 +00005191 if (!(dev->flags & IFF_LOOPBACK)) {
5192 dev->hw_features |= NETIF_F_NOCACHE_COPY;
5193 if (dev->features & NETIF_F_ALL_CSUM) {
5194 dev->wanted_features |= NETIF_F_NOCACHE_COPY;
5195 dev->features |= NETIF_F_NOCACHE_COPY;
5196 }
Tom Herbertc6e1a0d2011-04-04 22:30:30 -07005197 }
5198
Michał Mirosław1180e7d2011-07-14 14:41:11 -07005199 /* Make NETIF_F_HIGHDMA inheritable to VLAN devices.
Brandon Philips16c3ea72010-09-15 09:24:24 +00005200 */
Michał Mirosław1180e7d2011-07-14 14:41:11 -07005201 dev->vlan_features |= NETIF_F_HIGHDMA;
Brandon Philips16c3ea72010-09-15 09:24:24 +00005202
Johannes Berg7ffbe3f2009-10-02 05:15:27 +00005203 ret = call_netdevice_notifiers(NETDEV_POST_INIT, dev);
5204 ret = notifier_to_errno(ret);
5205 if (ret)
5206 goto err_uninit;
5207
Eric W. Biederman8b41d182007-09-26 22:02:53 -07005208 ret = netdev_register_kobject(dev);
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07005209 if (ret)
Herbert Xu7ce1b0e2007-07-30 16:29:40 -07005210 goto err_uninit;
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07005211 dev->reg_state = NETREG_REGISTERED;
5212
Michał Mirosław6cb6a272011-04-02 22:48:47 -07005213 __netdev_update_features(dev);
Michał Mirosław8e9b59b2011-02-22 16:52:28 +00005214
Linus Torvalds1da177e2005-04-16 15:20:36 -07005215 /*
5216 * Default initial state at registry is that the
5217 * device is present.
5218 */
5219
5220 set_bit(__LINK_STATE_PRESENT, &dev->state);
5221
Ben Hutchings8f4cccb2012-08-20 22:16:51 +01005222 linkwatch_init_dev(dev);
5223
Linus Torvalds1da177e2005-04-16 15:20:36 -07005224 dev_init_scheduler(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005225 dev_hold(dev);
Eric W. Biedermance286d32007-09-12 13:53:49 +02005226 list_netdevice(dev);
Theodore Ts'o7bf23572012-07-04 21:23:25 -04005227 add_device_randomness(dev->dev_addr, dev->addr_len);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005228
Jiri Pirko948b3372013-01-08 01:38:25 +00005229 /* If the device has permanent device address, driver should
5230 * set dev_addr and also addr_assign_type should be set to
5231 * NET_ADDR_PERM (default value).
5232 */
5233 if (dev->addr_assign_type == NET_ADDR_PERM)
5234 memcpy(dev->perm_addr, dev->dev_addr, dev->addr_len);
5235
Linus Torvalds1da177e2005-04-16 15:20:36 -07005236 /* Notify protocols, that a new device appeared. */
Pavel Emelyanov056925a2007-09-16 15:42:43 -07005237 ret = call_netdevice_notifiers(NETDEV_REGISTER, dev);
Herbert Xufcc5a032007-07-30 17:03:38 -07005238 ret = notifier_to_errno(ret);
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07005239 if (ret) {
5240 rollback_registered(dev);
5241 dev->reg_state = NETREG_UNREGISTERED;
5242 }
Eric W. Biedermand90a9092009-12-12 22:11:15 +00005243 /*
5244 * Prevent userspace races by waiting until the network
5245 * device is fully setup before sending notifications.
5246 */
Patrick McHardya2835762010-02-26 06:34:51 +00005247 if (!dev->rtnl_link_ops ||
5248 dev->rtnl_link_state == RTNL_LINK_INITIALIZED)
5249 rtmsg_ifinfo(RTM_NEWLINK, dev, ~0U);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005250
5251out:
5252 return ret;
Herbert Xu7ce1b0e2007-07-30 16:29:40 -07005253
5254err_uninit:
Stephen Hemmingerd3147742008-11-19 21:32:24 -08005255 if (dev->netdev_ops->ndo_uninit)
5256 dev->netdev_ops->ndo_uninit(dev);
Herbert Xu7ce1b0e2007-07-30 16:29:40 -07005257 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005258}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005259EXPORT_SYMBOL(register_netdevice);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005260
5261/**
Benjamin Herrenschmidt937f1ba2009-01-14 21:05:05 -08005262 * init_dummy_netdev - init a dummy network device for NAPI
5263 * @dev: device to init
5264 *
5265 * This takes a network device structure and initialize the minimum
5266 * amount of fields so it can be used to schedule NAPI polls without
5267 * registering a full blown interface. This is to be used by drivers
5268 * that need to tie several hardware interfaces to a single NAPI
5269 * poll scheduler due to HW limitations.
5270 */
5271int init_dummy_netdev(struct net_device *dev)
5272{
5273 /* Clear everything. Note we don't initialize spinlocks
5274 * are they aren't supposed to be taken by any of the
5275 * NAPI code and this dummy netdev is supposed to be
5276 * only ever used for NAPI polls
5277 */
5278 memset(dev, 0, sizeof(struct net_device));
5279
5280 /* make sure we BUG if trying to hit standard
5281 * register/unregister code path
5282 */
5283 dev->reg_state = NETREG_DUMMY;
5284
Benjamin Herrenschmidt937f1ba2009-01-14 21:05:05 -08005285 /* NAPI wants this */
5286 INIT_LIST_HEAD(&dev->napi_list);
5287
5288 /* a dummy interface is started by default */
5289 set_bit(__LINK_STATE_PRESENT, &dev->state);
5290 set_bit(__LINK_STATE_START, &dev->state);
5291
Eric Dumazet29b44332010-10-11 10:22:12 +00005292 /* Note : We dont allocate pcpu_refcnt for dummy devices,
5293 * because users of this 'device' dont need to change
5294 * its refcount.
5295 */
5296
Benjamin Herrenschmidt937f1ba2009-01-14 21:05:05 -08005297 return 0;
5298}
5299EXPORT_SYMBOL_GPL(init_dummy_netdev);
5300
5301
5302/**
Linus Torvalds1da177e2005-04-16 15:20:36 -07005303 * register_netdev - register a network device
5304 * @dev: device to register
5305 *
5306 * Take a completed network device structure and add it to the kernel
5307 * interfaces. A %NETDEV_REGISTER message is sent to the netdev notifier
5308 * chain. 0 is returned on success. A negative errno code is returned
5309 * on a failure to set up the device, or if the name is a duplicate.
5310 *
Borislav Petkov38b4da32007-04-20 22:14:10 -07005311 * This is a wrapper around register_netdevice that takes the rtnl semaphore
Linus Torvalds1da177e2005-04-16 15:20:36 -07005312 * and expands the device name if you passed a format string to
5313 * alloc_netdev.
5314 */
5315int register_netdev(struct net_device *dev)
5316{
5317 int err;
5318
5319 rtnl_lock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07005320 err = register_netdevice(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005321 rtnl_unlock();
5322 return err;
5323}
5324EXPORT_SYMBOL(register_netdev);
5325
Eric Dumazet29b44332010-10-11 10:22:12 +00005326int netdev_refcnt_read(const struct net_device *dev)
5327{
5328 int i, refcnt = 0;
5329
5330 for_each_possible_cpu(i)
5331 refcnt += *per_cpu_ptr(dev->pcpu_refcnt, i);
5332 return refcnt;
5333}
5334EXPORT_SYMBOL(netdev_refcnt_read);
5335
Ben Hutchings2c530402012-07-10 10:55:09 +00005336/**
Linus Torvalds1da177e2005-04-16 15:20:36 -07005337 * netdev_wait_allrefs - wait until all references are gone.
Randy Dunlap3de7a372012-08-18 14:36:44 +00005338 * @dev: target net_device
Linus Torvalds1da177e2005-04-16 15:20:36 -07005339 *
5340 * This is called when unregistering network devices.
5341 *
5342 * Any protocol or device that holds a reference should register
5343 * for netdevice notification, and cleanup and put back the
5344 * reference if they receive an UNREGISTER event.
5345 * We can get stuck here if buggy protocols don't correctly
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09005346 * call dev_put.
Linus Torvalds1da177e2005-04-16 15:20:36 -07005347 */
5348static void netdev_wait_allrefs(struct net_device *dev)
5349{
5350 unsigned long rebroadcast_time, warning_time;
Eric Dumazet29b44332010-10-11 10:22:12 +00005351 int refcnt;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005352
Eric Dumazete014deb2009-11-17 05:59:21 +00005353 linkwatch_forget_dev(dev);
5354
Linus Torvalds1da177e2005-04-16 15:20:36 -07005355 rebroadcast_time = warning_time = jiffies;
Eric Dumazet29b44332010-10-11 10:22:12 +00005356 refcnt = netdev_refcnt_read(dev);
5357
5358 while (refcnt != 0) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07005359 if (time_after(jiffies, rebroadcast_time + 1 * HZ)) {
Stephen Hemminger6756ae42006-03-20 22:23:58 -08005360 rtnl_lock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07005361
5362 /* Rebroadcast unregister notification */
Pavel Emelyanov056925a2007-09-16 15:42:43 -07005363 call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005364
Eric Dumazet748e2d92012-08-22 21:50:59 +00005365 __rtnl_unlock();
Eric Dumazet0115e8e2012-08-22 17:19:46 +00005366 rcu_barrier();
Eric Dumazet748e2d92012-08-22 21:50:59 +00005367 rtnl_lock();
5368
Eric Dumazet0115e8e2012-08-22 17:19:46 +00005369 call_netdevice_notifiers(NETDEV_UNREGISTER_FINAL, dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005370 if (test_bit(__LINK_STATE_LINKWATCH_PENDING,
5371 &dev->state)) {
5372 /* We must not have linkwatch events
5373 * pending on unregister. If this
5374 * happens, we simply run the queue
5375 * unscheduled, resulting in a noop
5376 * for this device.
5377 */
5378 linkwatch_run_queue();
5379 }
5380
Stephen Hemminger6756ae42006-03-20 22:23:58 -08005381 __rtnl_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07005382
5383 rebroadcast_time = jiffies;
5384 }
5385
5386 msleep(250);
5387
Eric Dumazet29b44332010-10-11 10:22:12 +00005388 refcnt = netdev_refcnt_read(dev);
5389
Linus Torvalds1da177e2005-04-16 15:20:36 -07005390 if (time_after(jiffies, warning_time + 10 * HZ)) {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00005391 pr_emerg("unregister_netdevice: waiting for %s to become free. Usage count = %d\n",
5392 dev->name, refcnt);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005393 warning_time = jiffies;
5394 }
5395 }
5396}
5397
5398/* The sequence is:
5399 *
5400 * rtnl_lock();
5401 * ...
5402 * register_netdevice(x1);
5403 * register_netdevice(x2);
5404 * ...
5405 * unregister_netdevice(y1);
5406 * unregister_netdevice(y2);
5407 * ...
5408 * rtnl_unlock();
5409 * free_netdev(y1);
5410 * free_netdev(y2);
5411 *
Herbert Xu58ec3b42008-10-07 15:50:03 -07005412 * We are invoked by rtnl_unlock().
Linus Torvalds1da177e2005-04-16 15:20:36 -07005413 * This allows us to deal with problems:
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07005414 * 1) We can delete sysfs objects which invoke hotplug
Linus Torvalds1da177e2005-04-16 15:20:36 -07005415 * without deadlocking with linkwatch via keventd.
5416 * 2) Since we run with the RTNL semaphore not held, we can sleep
5417 * safely in order to wait for the netdev refcnt to drop to zero.
Herbert Xu58ec3b42008-10-07 15:50:03 -07005418 *
5419 * We must not return until all unregister events added during
5420 * the interval the lock was held have been completed.
Linus Torvalds1da177e2005-04-16 15:20:36 -07005421 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07005422void netdev_run_todo(void)
5423{
Oleg Nesterov626ab0e2006-06-23 02:05:55 -07005424 struct list_head list;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005425
Linus Torvalds1da177e2005-04-16 15:20:36 -07005426 /* Snapshot list, allow later requests */
Oleg Nesterov626ab0e2006-06-23 02:05:55 -07005427 list_replace_init(&net_todo_list, &list);
Herbert Xu58ec3b42008-10-07 15:50:03 -07005428
5429 __rtnl_unlock();
Oleg Nesterov626ab0e2006-06-23 02:05:55 -07005430
Eric Dumazet0115e8e2012-08-22 17:19:46 +00005431
5432 /* Wait for rcu callbacks to finish before next phase */
Eric W. Biederman850a5452011-10-13 22:25:23 +00005433 if (!list_empty(&list))
5434 rcu_barrier();
5435
Linus Torvalds1da177e2005-04-16 15:20:36 -07005436 while (!list_empty(&list)) {
5437 struct net_device *dev
stephen hemmingere5e26d72010-02-24 14:01:38 +00005438 = list_first_entry(&list, struct net_device, todo_list);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005439 list_del(&dev->todo_list);
5440
Eric Dumazet748e2d92012-08-22 21:50:59 +00005441 rtnl_lock();
Eric Dumazet0115e8e2012-08-22 17:19:46 +00005442 call_netdevice_notifiers(NETDEV_UNREGISTER_FINAL, dev);
Eric Dumazet748e2d92012-08-22 21:50:59 +00005443 __rtnl_unlock();
Eric Dumazet0115e8e2012-08-22 17:19:46 +00005444
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07005445 if (unlikely(dev->reg_state != NETREG_UNREGISTERING)) {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00005446 pr_err("network todo '%s' but state %d\n",
Linus Torvalds1da177e2005-04-16 15:20:36 -07005447 dev->name, dev->reg_state);
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07005448 dump_stack();
5449 continue;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005450 }
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07005451
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07005452 dev->reg_state = NETREG_UNREGISTERED;
5453
Changli Gao152102c2010-03-30 20:16:22 +00005454 on_each_cpu(flush_backlog, dev, 1);
Stephen Hemminger6e583ce2008-08-03 21:29:57 -07005455
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07005456 netdev_wait_allrefs(dev);
5457
5458 /* paranoia */
Eric Dumazet29b44332010-10-11 10:22:12 +00005459 BUG_ON(netdev_refcnt_read(dev));
Eric Dumazet33d480c2011-08-11 19:30:52 +00005460 WARN_ON(rcu_access_pointer(dev->ip_ptr));
5461 WARN_ON(rcu_access_pointer(dev->ip6_ptr));
Ilpo Järvinen547b7922008-07-25 21:43:18 -07005462 WARN_ON(dev->dn_ptr);
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07005463
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07005464 if (dev->destructor)
5465 dev->destructor(dev);
Stephen Hemminger9093bbb2007-05-19 15:39:25 -07005466
5467 /* Free network device */
5468 kobject_put(&dev->dev.kobj);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005469 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07005470}
5471
Ben Hutchings3cfde792010-07-09 09:11:52 +00005472/* Convert net_device_stats to rtnl_link_stats64. They have the same
5473 * fields in the same order, with only the type differing.
5474 */
Eric Dumazet77a1abf2012-03-05 04:50:09 +00005475void netdev_stats_to_stats64(struct rtnl_link_stats64 *stats64,
5476 const struct net_device_stats *netdev_stats)
Ben Hutchings3cfde792010-07-09 09:11:52 +00005477{
5478#if BITS_PER_LONG == 64
Eric Dumazet77a1abf2012-03-05 04:50:09 +00005479 BUILD_BUG_ON(sizeof(*stats64) != sizeof(*netdev_stats));
5480 memcpy(stats64, netdev_stats, sizeof(*stats64));
Ben Hutchings3cfde792010-07-09 09:11:52 +00005481#else
5482 size_t i, n = sizeof(*stats64) / sizeof(u64);
5483 const unsigned long *src = (const unsigned long *)netdev_stats;
5484 u64 *dst = (u64 *)stats64;
5485
5486 BUILD_BUG_ON(sizeof(*netdev_stats) / sizeof(unsigned long) !=
5487 sizeof(*stats64) / sizeof(u64));
5488 for (i = 0; i < n; i++)
5489 dst[i] = src[i];
5490#endif
5491}
Eric Dumazet77a1abf2012-03-05 04:50:09 +00005492EXPORT_SYMBOL(netdev_stats_to_stats64);
Ben Hutchings3cfde792010-07-09 09:11:52 +00005493
Eric Dumazetd83345a2009-11-16 03:36:51 +00005494/**
Stephen Hemmingereeda3fd2008-11-19 21:40:23 -08005495 * dev_get_stats - get network device statistics
5496 * @dev: device to get statistics from
Eric Dumazet28172732010-07-07 14:58:56 -07005497 * @storage: place to store stats
Stephen Hemmingereeda3fd2008-11-19 21:40:23 -08005498 *
Ben Hutchingsd7753512010-07-09 09:12:41 +00005499 * Get network statistics from device. Return @storage.
5500 * The device driver may provide its own method by setting
5501 * dev->netdev_ops->get_stats64 or dev->netdev_ops->get_stats;
5502 * otherwise the internal statistics structure is used.
Stephen Hemmingereeda3fd2008-11-19 21:40:23 -08005503 */
Ben Hutchingsd7753512010-07-09 09:12:41 +00005504struct rtnl_link_stats64 *dev_get_stats(struct net_device *dev,
5505 struct rtnl_link_stats64 *storage)
Eric Dumazet7004bf22009-05-18 00:34:33 +00005506{
Stephen Hemmingereeda3fd2008-11-19 21:40:23 -08005507 const struct net_device_ops *ops = dev->netdev_ops;
5508
Eric Dumazet28172732010-07-07 14:58:56 -07005509 if (ops->ndo_get_stats64) {
5510 memset(storage, 0, sizeof(*storage));
Eric Dumazetcaf586e2010-09-30 21:06:55 +00005511 ops->ndo_get_stats64(dev, storage);
5512 } else if (ops->ndo_get_stats) {
Ben Hutchings3cfde792010-07-09 09:11:52 +00005513 netdev_stats_to_stats64(storage, ops->ndo_get_stats(dev));
Eric Dumazetcaf586e2010-09-30 21:06:55 +00005514 } else {
5515 netdev_stats_to_stats64(storage, &dev->stats);
Eric Dumazet28172732010-07-07 14:58:56 -07005516 }
Eric Dumazetcaf586e2010-09-30 21:06:55 +00005517 storage->rx_dropped += atomic_long_read(&dev->rx_dropped);
Eric Dumazet28172732010-07-07 14:58:56 -07005518 return storage;
Rusty Russellc45d2862007-03-28 14:29:08 -07005519}
Stephen Hemmingereeda3fd2008-11-19 21:40:23 -08005520EXPORT_SYMBOL(dev_get_stats);
Rusty Russellc45d2862007-03-28 14:29:08 -07005521
Eric Dumazet24824a02010-10-02 06:11:55 +00005522struct netdev_queue *dev_ingress_queue_create(struct net_device *dev)
David S. Millerdc2b4842008-07-08 17:18:23 -07005523{
Eric Dumazet24824a02010-10-02 06:11:55 +00005524 struct netdev_queue *queue = dev_ingress_queue(dev);
David S. Millerdc2b4842008-07-08 17:18:23 -07005525
Eric Dumazet24824a02010-10-02 06:11:55 +00005526#ifdef CONFIG_NET_CLS_ACT
5527 if (queue)
5528 return queue;
5529 queue = kzalloc(sizeof(*queue), GFP_KERNEL);
5530 if (!queue)
5531 return NULL;
5532 netdev_init_one_queue(dev, queue, NULL);
Eric Dumazet24824a02010-10-02 06:11:55 +00005533 queue->qdisc = &noop_qdisc;
5534 queue->qdisc_sleeping = &noop_qdisc;
5535 rcu_assign_pointer(dev->ingress_queue, queue);
5536#endif
5537 return queue;
David S. Millerbb949fb2008-07-08 16:55:56 -07005538}
5539
Eric Dumazet2c60db02012-09-16 09:17:26 +00005540static const struct ethtool_ops default_ethtool_ops;
5541
Stanislaw Gruszkad07d7502013-01-10 23:19:10 +00005542void netdev_set_default_ethtool_ops(struct net_device *dev,
5543 const struct ethtool_ops *ops)
5544{
5545 if (dev->ethtool_ops == &default_ethtool_ops)
5546 dev->ethtool_ops = ops;
5547}
5548EXPORT_SYMBOL_GPL(netdev_set_default_ethtool_ops);
5549
Linus Torvalds1da177e2005-04-16 15:20:36 -07005550/**
Tom Herbert36909ea2011-01-09 19:36:31 +00005551 * alloc_netdev_mqs - allocate network device
Linus Torvalds1da177e2005-04-16 15:20:36 -07005552 * @sizeof_priv: size of private data to allocate space for
5553 * @name: device name format string
5554 * @setup: callback to initialize device
Tom Herbert36909ea2011-01-09 19:36:31 +00005555 * @txqs: the number of TX subqueues to allocate
5556 * @rxqs: the number of RX subqueues to allocate
Linus Torvalds1da177e2005-04-16 15:20:36 -07005557 *
5558 * Allocates a struct net_device with private data area for driver use
Peter P Waskiewicz Jrf25f4e42007-07-06 13:36:20 -07005559 * and performs basic initialization. Also allocates subquue structs
Tom Herbert36909ea2011-01-09 19:36:31 +00005560 * for each queue on the device.
Linus Torvalds1da177e2005-04-16 15:20:36 -07005561 */
Tom Herbert36909ea2011-01-09 19:36:31 +00005562struct net_device *alloc_netdev_mqs(int sizeof_priv, const char *name,
5563 void (*setup)(struct net_device *),
5564 unsigned int txqs, unsigned int rxqs)
Linus Torvalds1da177e2005-04-16 15:20:36 -07005565{
Linus Torvalds1da177e2005-04-16 15:20:36 -07005566 struct net_device *dev;
Stephen Hemminger79439862008-07-21 13:28:44 -07005567 size_t alloc_size;
Eric Dumazet1ce8e7b2009-05-27 04:42:37 +00005568 struct net_device *p;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005569
Stephen Hemmingerb6fe17d2006-08-29 17:06:13 -07005570 BUG_ON(strlen(name) >= sizeof(dev->name));
5571
Tom Herbert36909ea2011-01-09 19:36:31 +00005572 if (txqs < 1) {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00005573 pr_err("alloc_netdev: Unable to allocate device with zero queues\n");
Tom Herbert55513fb2010-10-18 17:55:58 +00005574 return NULL;
5575 }
5576
Tom Herbert36909ea2011-01-09 19:36:31 +00005577#ifdef CONFIG_RPS
5578 if (rxqs < 1) {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00005579 pr_err("alloc_netdev: Unable to allocate device with zero RX queues\n");
Tom Herbert36909ea2011-01-09 19:36:31 +00005580 return NULL;
5581 }
5582#endif
5583
David S. Millerfd2ea0a2008-07-17 01:56:23 -07005584 alloc_size = sizeof(struct net_device);
Alexey Dobriyand1643d22008-04-18 15:43:32 -07005585 if (sizeof_priv) {
5586 /* ensure 32-byte alignment of private area */
Eric Dumazet1ce8e7b2009-05-27 04:42:37 +00005587 alloc_size = ALIGN(alloc_size, NETDEV_ALIGN);
Alexey Dobriyand1643d22008-04-18 15:43:32 -07005588 alloc_size += sizeof_priv;
5589 }
5590 /* ensure 32-byte alignment of whole construct */
Eric Dumazet1ce8e7b2009-05-27 04:42:37 +00005591 alloc_size += NETDEV_ALIGN - 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005592
Paolo 'Blaisorblade' Giarrusso31380de2006-04-06 22:38:28 -07005593 p = kzalloc(alloc_size, GFP_KERNEL);
Joe Perches62b59422013-02-04 16:48:16 +00005594 if (!p)
Linus Torvalds1da177e2005-04-16 15:20:36 -07005595 return NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005596
Eric Dumazet1ce8e7b2009-05-27 04:42:37 +00005597 dev = PTR_ALIGN(p, NETDEV_ALIGN);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005598 dev->padded = (char *)dev - (char *)p;
Jiri Pirkoab9c73c2009-05-08 13:30:17 +00005599
Eric Dumazet29b44332010-10-11 10:22:12 +00005600 dev->pcpu_refcnt = alloc_percpu(int);
5601 if (!dev->pcpu_refcnt)
Tom Herberte6484932010-10-18 18:04:39 +00005602 goto free_p;
Jiri Pirkoab9c73c2009-05-08 13:30:17 +00005603
Linus Torvalds1da177e2005-04-16 15:20:36 -07005604 if (dev_addr_init(dev))
Eric Dumazet29b44332010-10-11 10:22:12 +00005605 goto free_pcpu;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005606
Jiri Pirko22bedad32010-04-01 21:22:57 +00005607 dev_mc_init(dev);
Jiri Pirkoa748ee22010-04-01 21:22:09 +00005608 dev_uc_init(dev);
Jiri Pirkoccffad252009-05-22 23:22:17 +00005609
YOSHIFUJI Hideakic346dca2008-03-25 21:47:49 +09005610 dev_net_set(dev, &init_net);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005611
Peter P Waskiewicz Jr82cc1a72008-03-21 03:43:19 -07005612 dev->gso_max_size = GSO_MAX_SIZE;
Ben Hutchings30b678d2012-07-30 15:57:00 +00005613 dev->gso_max_segs = GSO_MAX_SEGS;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005614
Herbert Xud565b0a2008-12-15 23:38:52 -08005615 INIT_LIST_HEAD(&dev->napi_list);
Eric W. Biederman9fdce092009-10-30 14:51:13 +00005616 INIT_LIST_HEAD(&dev->unreg_list);
Eric Dumazete014deb2009-11-17 05:59:21 +00005617 INIT_LIST_HEAD(&dev->link_watch_list);
Jiri Pirko9ff162a2013-01-03 22:48:49 +00005618 INIT_LIST_HEAD(&dev->upper_dev_list);
Eric Dumazet93f154b2009-05-18 22:19:19 -07005619 dev->priv_flags = IFF_XMIT_DST_RELEASE;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005620 setup(dev);
David S. Miller8d3bdbd2011-02-08 15:02:50 -08005621
5622 dev->num_tx_queues = txqs;
5623 dev->real_num_tx_queues = txqs;
5624 if (netif_alloc_netdev_queues(dev))
5625 goto free_all;
5626
5627#ifdef CONFIG_RPS
5628 dev->num_rx_queues = rxqs;
5629 dev->real_num_rx_queues = rxqs;
5630 if (netif_alloc_rx_queues(dev))
5631 goto free_all;
5632#endif
5633
Linus Torvalds1da177e2005-04-16 15:20:36 -07005634 strcpy(dev->name, name);
Vlad Dogarucbda10f2011-01-13 23:38:30 +00005635 dev->group = INIT_NETDEV_GROUP;
Eric Dumazet2c60db02012-09-16 09:17:26 +00005636 if (!dev->ethtool_ops)
5637 dev->ethtool_ops = &default_ethtool_ops;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005638 return dev;
Jiri Pirkoab9c73c2009-05-08 13:30:17 +00005639
David S. Miller8d3bdbd2011-02-08 15:02:50 -08005640free_all:
5641 free_netdev(dev);
5642 return NULL;
5643
Eric Dumazet29b44332010-10-11 10:22:12 +00005644free_pcpu:
5645 free_percpu(dev->pcpu_refcnt);
Tom Herberted9af2e2010-11-09 10:47:30 +00005646 kfree(dev->_tx);
Tom Herbertfe822242010-11-09 10:47:38 +00005647#ifdef CONFIG_RPS
5648 kfree(dev->_rx);
5649#endif
5650
Jiri Pirkoab9c73c2009-05-08 13:30:17 +00005651free_p:
5652 kfree(p);
5653 return NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005654}
Tom Herbert36909ea2011-01-09 19:36:31 +00005655EXPORT_SYMBOL(alloc_netdev_mqs);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005656
5657/**
5658 * free_netdev - free network device
5659 * @dev: device
5660 *
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09005661 * This function does the last stage of destroying an allocated device
5662 * interface. The reference to the device object is released.
Linus Torvalds1da177e2005-04-16 15:20:36 -07005663 * If this is the last reference then it will be freed.
5664 */
5665void free_netdev(struct net_device *dev)
5666{
Herbert Xud565b0a2008-12-15 23:38:52 -08005667 struct napi_struct *p, *n;
5668
Denis V. Lunevf3005d72008-04-16 02:02:18 -07005669 release_net(dev_net(dev));
5670
David S. Millere8a04642008-07-17 00:34:19 -07005671 kfree(dev->_tx);
Tom Herbertfe822242010-11-09 10:47:38 +00005672#ifdef CONFIG_RPS
5673 kfree(dev->_rx);
5674#endif
David S. Millere8a04642008-07-17 00:34:19 -07005675
Eric Dumazet33d480c2011-08-11 19:30:52 +00005676 kfree(rcu_dereference_protected(dev->ingress_queue, 1));
Eric Dumazet24824a02010-10-02 06:11:55 +00005677
Jiri Pirkof001fde2009-05-05 02:48:28 +00005678 /* Flush device addresses */
5679 dev_addr_flush(dev);
5680
Herbert Xud565b0a2008-12-15 23:38:52 -08005681 list_for_each_entry_safe(p, n, &dev->napi_list, dev_list)
5682 netif_napi_del(p);
5683
Eric Dumazet29b44332010-10-11 10:22:12 +00005684 free_percpu(dev->pcpu_refcnt);
5685 dev->pcpu_refcnt = NULL;
5686
Stephen Hemminger3041a062006-05-26 13:25:24 -07005687 /* Compatibility with error handling in drivers */
Linus Torvalds1da177e2005-04-16 15:20:36 -07005688 if (dev->reg_state == NETREG_UNINITIALIZED) {
5689 kfree((char *)dev - dev->padded);
5690 return;
5691 }
5692
5693 BUG_ON(dev->reg_state != NETREG_UNREGISTERED);
5694 dev->reg_state = NETREG_RELEASED;
5695
Greg Kroah-Hartman43cb76d2002-04-09 12:14:34 -07005696 /* will free via device release */
5697 put_device(&dev->dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005698}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005699EXPORT_SYMBOL(free_netdev);
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09005700
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07005701/**
5702 * synchronize_net - Synchronize with packet receive processing
5703 *
5704 * Wait for packets currently being received to be done.
5705 * Does not block later packets from starting.
5706 */
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09005707void synchronize_net(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -07005708{
5709 might_sleep();
Eric Dumazetbe3fc412011-05-23 23:07:32 +00005710 if (rtnl_is_locked())
5711 synchronize_rcu_expedited();
5712 else
5713 synchronize_rcu();
Linus Torvalds1da177e2005-04-16 15:20:36 -07005714}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005715EXPORT_SYMBOL(synchronize_net);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005716
5717/**
Eric Dumazet44a08732009-10-27 07:03:04 +00005718 * unregister_netdevice_queue - remove device from the kernel
Linus Torvalds1da177e2005-04-16 15:20:36 -07005719 * @dev: device
Eric Dumazet44a08732009-10-27 07:03:04 +00005720 * @head: list
Jaswinder Singh Rajput6ebfbc02009-11-22 20:43:13 -08005721 *
Linus Torvalds1da177e2005-04-16 15:20:36 -07005722 * This function shuts down a device interface and removes it
Wang Chend59b54b2007-12-11 02:28:03 -08005723 * from the kernel tables.
Eric Dumazet44a08732009-10-27 07:03:04 +00005724 * If head not NULL, device is queued to be unregistered later.
Linus Torvalds1da177e2005-04-16 15:20:36 -07005725 *
5726 * Callers must hold the rtnl semaphore. You may want
5727 * unregister_netdev() instead of this.
5728 */
5729
Eric Dumazet44a08732009-10-27 07:03:04 +00005730void unregister_netdevice_queue(struct net_device *dev, struct list_head *head)
Linus Torvalds1da177e2005-04-16 15:20:36 -07005731{
Herbert Xua6620712007-12-12 19:21:56 -08005732 ASSERT_RTNL();
5733
Eric Dumazet44a08732009-10-27 07:03:04 +00005734 if (head) {
Eric W. Biederman9fdce092009-10-30 14:51:13 +00005735 list_move_tail(&dev->unreg_list, head);
Eric Dumazet44a08732009-10-27 07:03:04 +00005736 } else {
5737 rollback_registered(dev);
5738 /* Finish processing unregister after unlock */
5739 net_set_todo(dev);
5740 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07005741}
Eric Dumazet44a08732009-10-27 07:03:04 +00005742EXPORT_SYMBOL(unregister_netdevice_queue);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005743
5744/**
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005745 * unregister_netdevice_many - unregister many devices
5746 * @head: list of devices
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005747 */
5748void unregister_netdevice_many(struct list_head *head)
5749{
5750 struct net_device *dev;
5751
5752 if (!list_empty(head)) {
5753 rollback_registered_many(head);
5754 list_for_each_entry(dev, head, unreg_list)
5755 net_set_todo(dev);
5756 }
5757}
Eric Dumazet63c80992009-10-27 07:06:49 +00005758EXPORT_SYMBOL(unregister_netdevice_many);
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005759
5760/**
Linus Torvalds1da177e2005-04-16 15:20:36 -07005761 * unregister_netdev - remove device from the kernel
5762 * @dev: device
5763 *
5764 * This function shuts down a device interface and removes it
Wang Chend59b54b2007-12-11 02:28:03 -08005765 * from the kernel tables.
Linus Torvalds1da177e2005-04-16 15:20:36 -07005766 *
5767 * This is just a wrapper for unregister_netdevice that takes
5768 * the rtnl semaphore. In general you want to use this and not
5769 * unregister_netdevice.
5770 */
5771void unregister_netdev(struct net_device *dev)
5772{
5773 rtnl_lock();
5774 unregister_netdevice(dev);
5775 rtnl_unlock();
5776}
Linus Torvalds1da177e2005-04-16 15:20:36 -07005777EXPORT_SYMBOL(unregister_netdev);
5778
Eric W. Biedermance286d32007-09-12 13:53:49 +02005779/**
5780 * dev_change_net_namespace - move device to different nethost namespace
5781 * @dev: device
5782 * @net: network namespace
5783 * @pat: If not NULL name pattern to try if the current device name
5784 * is already taken in the destination network namespace.
5785 *
5786 * This function shuts down a device interface and moves it
5787 * to a new network namespace. On success 0 is returned, on
5788 * a failure a netagive errno code is returned.
5789 *
5790 * Callers must hold the rtnl semaphore.
5791 */
5792
5793int dev_change_net_namespace(struct net_device *dev, struct net *net, const char *pat)
5794{
Eric W. Biedermance286d32007-09-12 13:53:49 +02005795 int err;
5796
5797 ASSERT_RTNL();
5798
5799 /* Don't allow namespace local devices to be moved. */
5800 err = -EINVAL;
5801 if (dev->features & NETIF_F_NETNS_LOCAL)
5802 goto out;
5803
5804 /* Ensure the device has been registrered */
Eric W. Biedermance286d32007-09-12 13:53:49 +02005805 if (dev->reg_state != NETREG_REGISTERED)
5806 goto out;
5807
5808 /* Get out if there is nothing todo */
5809 err = 0;
YOSHIFUJI Hideaki878628f2008-03-26 03:57:35 +09005810 if (net_eq(dev_net(dev), net))
Eric W. Biedermance286d32007-09-12 13:53:49 +02005811 goto out;
5812
5813 /* Pick the destination device name, and ensure
5814 * we can use it in the destination network namespace.
5815 */
5816 err = -EEXIST;
Octavian Purdilad9031022009-11-18 02:36:59 +00005817 if (__dev_get_by_name(net, dev->name)) {
Eric W. Biedermance286d32007-09-12 13:53:49 +02005818 /* We get here if we can't use the current device name */
5819 if (!pat)
5820 goto out;
Gao feng828de4f2012-09-13 20:58:27 +00005821 if (dev_get_valid_name(net, dev, pat) < 0)
Eric W. Biedermance286d32007-09-12 13:53:49 +02005822 goto out;
5823 }
5824
5825 /*
5826 * And now a mini version of register_netdevice unregister_netdevice.
5827 */
5828
5829 /* If device is running close it first. */
Pavel Emelyanov9b772652007-10-10 02:49:09 -07005830 dev_close(dev);
Eric W. Biedermance286d32007-09-12 13:53:49 +02005831
5832 /* And unlink it from device chain */
5833 err = -ENODEV;
5834 unlist_netdevice(dev);
5835
5836 synchronize_net();
5837
5838 /* Shutdown queueing discipline. */
5839 dev_shutdown(dev);
5840
5841 /* Notify protocols, that we are about to destroy
5842 this device. They should clean all the things.
David Lamparter3b27e102010-09-17 03:22:19 +00005843
5844 Note that dev->reg_state stays at NETREG_REGISTERED.
5845 This is wanted because this way 8021q and macvlan know
5846 the device is just moving and can keep their slaves up.
Eric W. Biedermance286d32007-09-12 13:53:49 +02005847 */
5848 call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
Gao feng6549dd42012-08-23 15:36:55 +00005849 rcu_barrier();
5850 call_netdevice_notifiers(NETDEV_UNREGISTER_FINAL, dev);
Eric W. Biedermand2237d32011-10-21 06:24:20 +00005851 rtmsg_ifinfo(RTM_DELLINK, dev, ~0U);
Eric W. Biedermance286d32007-09-12 13:53:49 +02005852
5853 /*
5854 * Flush the unicast and multicast chains
5855 */
Jiri Pirkoa748ee22010-04-01 21:22:09 +00005856 dev_uc_flush(dev);
Jiri Pirko22bedad32010-04-01 21:22:57 +00005857 dev_mc_flush(dev);
Eric W. Biedermance286d32007-09-12 13:53:49 +02005858
Serge Hallyn4e66ae22012-12-03 16:17:12 +00005859 /* Send a netdev-removed uevent to the old namespace */
5860 kobject_uevent(&dev->dev.kobj, KOBJ_REMOVE);
5861
Eric W. Biedermance286d32007-09-12 13:53:49 +02005862 /* Actually switch the network namespace */
YOSHIFUJI Hideakic346dca2008-03-25 21:47:49 +09005863 dev_net_set(dev, net);
Eric W. Biedermance286d32007-09-12 13:53:49 +02005864
Eric W. Biedermance286d32007-09-12 13:53:49 +02005865 /* If there is an ifindex conflict assign a new one */
5866 if (__dev_get_by_index(net, dev->ifindex)) {
5867 int iflink = (dev->iflink == dev->ifindex);
5868 dev->ifindex = dev_new_index(net);
5869 if (iflink)
5870 dev->iflink = dev->ifindex;
5871 }
5872
Serge Hallyn4e66ae22012-12-03 16:17:12 +00005873 /* Send a netdev-add uevent to the new namespace */
5874 kobject_uevent(&dev->dev.kobj, KOBJ_ADD);
5875
Eric W. Biederman8b41d182007-09-26 22:02:53 -07005876 /* Fixup kobjects */
Eric W. Biedermana1b3f592010-05-04 17:36:49 -07005877 err = device_rename(&dev->dev, dev->name);
Eric W. Biederman8b41d182007-09-26 22:02:53 -07005878 WARN_ON(err);
Eric W. Biedermance286d32007-09-12 13:53:49 +02005879
5880 /* Add the device back in the hashes */
5881 list_netdevice(dev);
5882
5883 /* Notify protocols, that a new device appeared. */
5884 call_netdevice_notifiers(NETDEV_REGISTER, dev);
5885
Eric W. Biedermand90a9092009-12-12 22:11:15 +00005886 /*
5887 * Prevent userspace races by waiting until the network
5888 * device is fully setup before sending notifications.
5889 */
5890 rtmsg_ifinfo(RTM_NEWLINK, dev, ~0U);
5891
Eric W. Biedermance286d32007-09-12 13:53:49 +02005892 synchronize_net();
5893 err = 0;
5894out:
5895 return err;
5896}
Johannes Berg463d0182009-07-14 00:33:35 +02005897EXPORT_SYMBOL_GPL(dev_change_net_namespace);
Eric W. Biedermance286d32007-09-12 13:53:49 +02005898
Linus Torvalds1da177e2005-04-16 15:20:36 -07005899static int dev_cpu_callback(struct notifier_block *nfb,
5900 unsigned long action,
5901 void *ocpu)
5902{
5903 struct sk_buff **list_skb;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005904 struct sk_buff *skb;
5905 unsigned int cpu, oldcpu = (unsigned long)ocpu;
5906 struct softnet_data *sd, *oldsd;
5907
Rafael J. Wysocki8bb78442007-05-09 02:35:10 -07005908 if (action != CPU_DEAD && action != CPU_DEAD_FROZEN)
Linus Torvalds1da177e2005-04-16 15:20:36 -07005909 return NOTIFY_OK;
5910
5911 local_irq_disable();
5912 cpu = smp_processor_id();
5913 sd = &per_cpu(softnet_data, cpu);
5914 oldsd = &per_cpu(softnet_data, oldcpu);
5915
5916 /* Find end of our completion_queue. */
5917 list_skb = &sd->completion_queue;
5918 while (*list_skb)
5919 list_skb = &(*list_skb)->next;
5920 /* Append completion queue from offline CPU. */
5921 *list_skb = oldsd->completion_queue;
5922 oldsd->completion_queue = NULL;
5923
Linus Torvalds1da177e2005-04-16 15:20:36 -07005924 /* Append output queue from offline CPU. */
Changli Gaoa9cbd582010-04-26 23:06:24 +00005925 if (oldsd->output_queue) {
5926 *sd->output_queue_tailp = oldsd->output_queue;
5927 sd->output_queue_tailp = oldsd->output_queue_tailp;
5928 oldsd->output_queue = NULL;
5929 oldsd->output_queue_tailp = &oldsd->output_queue;
5930 }
Heiko Carstens264524d2011-06-06 20:50:03 +00005931 /* Append NAPI poll list from offline CPU. */
5932 if (!list_empty(&oldsd->poll_list)) {
5933 list_splice_init(&oldsd->poll_list, &sd->poll_list);
5934 raise_softirq_irqoff(NET_RX_SOFTIRQ);
5935 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07005936
5937 raise_softirq_irqoff(NET_TX_SOFTIRQ);
5938 local_irq_enable();
5939
5940 /* Process offline CPU's input_pkt_queue */
Tom Herbert76cc8b12010-05-20 18:37:59 +00005941 while ((skb = __skb_dequeue(&oldsd->process_queue))) {
5942 netif_rx(skb);
5943 input_queue_head_incr(oldsd);
5944 }
Tom Herbertfec5e652010-04-16 16:01:27 -07005945 while ((skb = __skb_dequeue(&oldsd->input_pkt_queue))) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07005946 netif_rx(skb);
Tom Herbert76cc8b12010-05-20 18:37:59 +00005947 input_queue_head_incr(oldsd);
Tom Herbertfec5e652010-04-16 16:01:27 -07005948 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07005949
5950 return NOTIFY_OK;
5951}
Linus Torvalds1da177e2005-04-16 15:20:36 -07005952
5953
Herbert Xu7f353bf2007-08-10 15:47:58 -07005954/**
Herbert Xub63365a2008-10-23 01:11:29 -07005955 * netdev_increment_features - increment feature set by one
5956 * @all: current feature set
5957 * @one: new feature set
5958 * @mask: mask feature set
Herbert Xu7f353bf2007-08-10 15:47:58 -07005959 *
5960 * Computes a new feature set after adding a device with feature set
Herbert Xub63365a2008-10-23 01:11:29 -07005961 * @one to the master device with current feature set @all. Will not
5962 * enable anything that is off in @mask. Returns the new feature set.
Herbert Xu7f353bf2007-08-10 15:47:58 -07005963 */
Michał Mirosławc8f44af2011-11-15 15:29:55 +00005964netdev_features_t netdev_increment_features(netdev_features_t all,
5965 netdev_features_t one, netdev_features_t mask)
Herbert Xu7f353bf2007-08-10 15:47:58 -07005966{
Michał Mirosław1742f182011-04-22 06:31:16 +00005967 if (mask & NETIF_F_GEN_CSUM)
5968 mask |= NETIF_F_ALL_CSUM;
5969 mask |= NETIF_F_VLAN_CHALLENGED;
5970
5971 all |= one & (NETIF_F_ONE_FOR_ALL|NETIF_F_ALL_CSUM) & mask;
5972 all &= one | ~NETIF_F_ALL_FOR_ALL;
5973
Michał Mirosław1742f182011-04-22 06:31:16 +00005974 /* If one device supports hw checksumming, set for all. */
5975 if (all & NETIF_F_GEN_CSUM)
5976 all &= ~(NETIF_F_ALL_CSUM & ~NETIF_F_GEN_CSUM);
Herbert Xu7f353bf2007-08-10 15:47:58 -07005977
5978 return all;
5979}
Herbert Xub63365a2008-10-23 01:11:29 -07005980EXPORT_SYMBOL(netdev_increment_features);
Herbert Xu7f353bf2007-08-10 15:47:58 -07005981
Pavel Emelyanov30d97d32007-09-16 15:40:33 -07005982static struct hlist_head *netdev_create_hash(void)
5983{
5984 int i;
5985 struct hlist_head *hash;
5986
5987 hash = kmalloc(sizeof(*hash) * NETDEV_HASHENTRIES, GFP_KERNEL);
5988 if (hash != NULL)
5989 for (i = 0; i < NETDEV_HASHENTRIES; i++)
5990 INIT_HLIST_HEAD(&hash[i]);
5991
5992 return hash;
5993}
5994
Eric W. Biederman881d9662007-09-17 11:56:21 -07005995/* Initialize per network namespace state */
Pavel Emelyanov46650792007-10-08 20:38:39 -07005996static int __net_init netdev_init(struct net *net)
Eric W. Biederman881d9662007-09-17 11:56:21 -07005997{
Rustad, Mark D734b6542012-07-18 09:06:07 +00005998 if (net != &init_net)
5999 INIT_LIST_HEAD(&net->dev_base_head);
Eric W. Biederman881d9662007-09-17 11:56:21 -07006000
Pavel Emelyanov30d97d32007-09-16 15:40:33 -07006001 net->dev_name_head = netdev_create_hash();
6002 if (net->dev_name_head == NULL)
6003 goto err_name;
Eric W. Biederman881d9662007-09-17 11:56:21 -07006004
Pavel Emelyanov30d97d32007-09-16 15:40:33 -07006005 net->dev_index_head = netdev_create_hash();
6006 if (net->dev_index_head == NULL)
6007 goto err_idx;
Eric W. Biederman881d9662007-09-17 11:56:21 -07006008
6009 return 0;
Pavel Emelyanov30d97d32007-09-16 15:40:33 -07006010
6011err_idx:
6012 kfree(net->dev_name_head);
6013err_name:
6014 return -ENOMEM;
Eric W. Biederman881d9662007-09-17 11:56:21 -07006015}
6016
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07006017/**
6018 * netdev_drivername - network driver for the device
6019 * @dev: network device
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07006020 *
6021 * Determine network driver for device.
6022 */
David S. Miller3019de12011-06-06 16:41:33 -07006023const char *netdev_drivername(const struct net_device *dev)
Arjan van de Ven6579e572008-07-21 13:31:48 -07006024{
Stephen Hemmingercf04a4c72008-09-30 02:22:14 -07006025 const struct device_driver *driver;
6026 const struct device *parent;
David S. Miller3019de12011-06-06 16:41:33 -07006027 const char *empty = "";
Arjan van de Ven6579e572008-07-21 13:31:48 -07006028
6029 parent = dev->dev.parent;
Arjan van de Ven6579e572008-07-21 13:31:48 -07006030 if (!parent)
David S. Miller3019de12011-06-06 16:41:33 -07006031 return empty;
Arjan van de Ven6579e572008-07-21 13:31:48 -07006032
6033 driver = parent->driver;
6034 if (driver && driver->name)
David S. Miller3019de12011-06-06 16:41:33 -07006035 return driver->name;
6036 return empty;
Arjan van de Ven6579e572008-07-21 13:31:48 -07006037}
6038
Joe Perchesb004ff42012-09-12 20:12:19 -07006039static int __netdev_printk(const char *level, const struct net_device *dev,
Joe Perches256df2f2010-06-27 01:02:35 +00006040 struct va_format *vaf)
6041{
6042 int r;
6043
Joe Perchesb004ff42012-09-12 20:12:19 -07006044 if (dev && dev->dev.parent) {
Joe Perches666f3552012-09-12 20:14:11 -07006045 r = dev_printk_emit(level[1] - '0',
6046 dev->dev.parent,
6047 "%s %s %s: %pV",
6048 dev_driver_string(dev->dev.parent),
6049 dev_name(dev->dev.parent),
6050 netdev_name(dev), vaf);
Joe Perchesb004ff42012-09-12 20:12:19 -07006051 } else if (dev) {
Joe Perches256df2f2010-06-27 01:02:35 +00006052 r = printk("%s%s: %pV", level, netdev_name(dev), vaf);
Joe Perchesb004ff42012-09-12 20:12:19 -07006053 } else {
Joe Perches256df2f2010-06-27 01:02:35 +00006054 r = printk("%s(NULL net_device): %pV", level, vaf);
Joe Perchesb004ff42012-09-12 20:12:19 -07006055 }
Joe Perches256df2f2010-06-27 01:02:35 +00006056
6057 return r;
6058}
6059
6060int netdev_printk(const char *level, const struct net_device *dev,
6061 const char *format, ...)
6062{
6063 struct va_format vaf;
6064 va_list args;
6065 int r;
6066
6067 va_start(args, format);
6068
6069 vaf.fmt = format;
6070 vaf.va = &args;
6071
6072 r = __netdev_printk(level, dev, &vaf);
Joe Perchesb004ff42012-09-12 20:12:19 -07006073
Joe Perches256df2f2010-06-27 01:02:35 +00006074 va_end(args);
6075
6076 return r;
6077}
6078EXPORT_SYMBOL(netdev_printk);
6079
6080#define define_netdev_printk_level(func, level) \
6081int func(const struct net_device *dev, const char *fmt, ...) \
6082{ \
6083 int r; \
6084 struct va_format vaf; \
6085 va_list args; \
6086 \
6087 va_start(args, fmt); \
6088 \
6089 vaf.fmt = fmt; \
6090 vaf.va = &args; \
6091 \
6092 r = __netdev_printk(level, dev, &vaf); \
Joe Perchesb004ff42012-09-12 20:12:19 -07006093 \
Joe Perches256df2f2010-06-27 01:02:35 +00006094 va_end(args); \
6095 \
6096 return r; \
6097} \
6098EXPORT_SYMBOL(func);
6099
6100define_netdev_printk_level(netdev_emerg, KERN_EMERG);
6101define_netdev_printk_level(netdev_alert, KERN_ALERT);
6102define_netdev_printk_level(netdev_crit, KERN_CRIT);
6103define_netdev_printk_level(netdev_err, KERN_ERR);
6104define_netdev_printk_level(netdev_warn, KERN_WARNING);
6105define_netdev_printk_level(netdev_notice, KERN_NOTICE);
6106define_netdev_printk_level(netdev_info, KERN_INFO);
6107
Pavel Emelyanov46650792007-10-08 20:38:39 -07006108static void __net_exit netdev_exit(struct net *net)
Eric W. Biederman881d9662007-09-17 11:56:21 -07006109{
6110 kfree(net->dev_name_head);
6111 kfree(net->dev_index_head);
6112}
6113
Denis V. Lunev022cbae2007-11-13 03:23:50 -08006114static struct pernet_operations __net_initdata netdev_net_ops = {
Eric W. Biederman881d9662007-09-17 11:56:21 -07006115 .init = netdev_init,
6116 .exit = netdev_exit,
6117};
6118
Pavel Emelyanov46650792007-10-08 20:38:39 -07006119static void __net_exit default_device_exit(struct net *net)
Eric W. Biedermance286d32007-09-12 13:53:49 +02006120{
Eric W. Biedermane008b5f2009-11-29 22:25:30 +00006121 struct net_device *dev, *aux;
Eric W. Biedermance286d32007-09-12 13:53:49 +02006122 /*
Eric W. Biedermane008b5f2009-11-29 22:25:30 +00006123 * Push all migratable network devices back to the
Eric W. Biedermance286d32007-09-12 13:53:49 +02006124 * initial network namespace
6125 */
6126 rtnl_lock();
Eric W. Biedermane008b5f2009-11-29 22:25:30 +00006127 for_each_netdev_safe(net, dev, aux) {
Eric W. Biedermance286d32007-09-12 13:53:49 +02006128 int err;
Pavel Emelyanovaca51392008-05-08 01:24:25 -07006129 char fb_name[IFNAMSIZ];
Eric W. Biedermance286d32007-09-12 13:53:49 +02006130
6131 /* Ignore unmoveable devices (i.e. loopback) */
6132 if (dev->features & NETIF_F_NETNS_LOCAL)
6133 continue;
6134
Eric W. Biedermane008b5f2009-11-29 22:25:30 +00006135 /* Leave virtual devices for the generic cleanup */
6136 if (dev->rtnl_link_ops)
6137 continue;
Eric W. Biedermand0c082c2008-11-05 15:59:38 -08006138
Lucas De Marchi25985ed2011-03-30 22:57:33 -03006139 /* Push remaining network devices to init_net */
Pavel Emelyanovaca51392008-05-08 01:24:25 -07006140 snprintf(fb_name, IFNAMSIZ, "dev%d", dev->ifindex);
6141 err = dev_change_net_namespace(dev, &init_net, fb_name);
Eric W. Biedermance286d32007-09-12 13:53:49 +02006142 if (err) {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00006143 pr_emerg("%s: failed to move %s to init_net: %d\n",
6144 __func__, dev->name, err);
Pavel Emelyanovaca51392008-05-08 01:24:25 -07006145 BUG();
Eric W. Biedermance286d32007-09-12 13:53:49 +02006146 }
6147 }
6148 rtnl_unlock();
6149}
6150
Eric W. Biederman04dc7f6b2009-12-03 02:29:04 +00006151static void __net_exit default_device_exit_batch(struct list_head *net_list)
6152{
6153 /* At exit all network devices most be removed from a network
Uwe Kleine-Königb5950762010-11-01 15:38:34 -04006154 * namespace. Do this in the reverse order of registration.
Eric W. Biederman04dc7f6b2009-12-03 02:29:04 +00006155 * Do this across as many network namespaces as possible to
6156 * improve batching efficiency.
6157 */
6158 struct net_device *dev;
6159 struct net *net;
6160 LIST_HEAD(dev_kill_list);
6161
6162 rtnl_lock();
6163 list_for_each_entry(net, net_list, exit_list) {
6164 for_each_netdev_reverse(net, dev) {
6165 if (dev->rtnl_link_ops)
6166 dev->rtnl_link_ops->dellink(dev, &dev_kill_list);
6167 else
6168 unregister_netdevice_queue(dev, &dev_kill_list);
6169 }
6170 }
6171 unregister_netdevice_many(&dev_kill_list);
Eric Dumazetceaaec92011-02-17 22:59:19 +00006172 list_del(&dev_kill_list);
Eric W. Biederman04dc7f6b2009-12-03 02:29:04 +00006173 rtnl_unlock();
6174}
6175
Denis V. Lunev022cbae2007-11-13 03:23:50 -08006176static struct pernet_operations __net_initdata default_device_ops = {
Eric W. Biedermance286d32007-09-12 13:53:49 +02006177 .exit = default_device_exit,
Eric W. Biederman04dc7f6b2009-12-03 02:29:04 +00006178 .exit_batch = default_device_exit_batch,
Eric W. Biedermance286d32007-09-12 13:53:49 +02006179};
6180
Linus Torvalds1da177e2005-04-16 15:20:36 -07006181/*
6182 * Initialize the DEV module. At boot time this walks the device list and
6183 * unhooks any devices that fail to initialise (normally hardware not
6184 * present) and leaves us with a valid list of present and active devices.
6185 *
6186 */
6187
6188/*
6189 * This is called single threaded during boot, so no need
6190 * to take the rtnl semaphore.
6191 */
6192static int __init net_dev_init(void)
6193{
6194 int i, rc = -ENOMEM;
6195
6196 BUG_ON(!dev_boot_phase);
6197
Linus Torvalds1da177e2005-04-16 15:20:36 -07006198 if (dev_proc_init())
6199 goto out;
6200
Eric W. Biederman8b41d182007-09-26 22:02:53 -07006201 if (netdev_kobject_init())
Linus Torvalds1da177e2005-04-16 15:20:36 -07006202 goto out;
6203
6204 INIT_LIST_HEAD(&ptype_all);
Pavel Emelyanov82d8a8672007-11-26 20:12:58 +08006205 for (i = 0; i < PTYPE_HASH_SIZE; i++)
Linus Torvalds1da177e2005-04-16 15:20:36 -07006206 INIT_LIST_HEAD(&ptype_base[i]);
6207
Vlad Yasevich62532da2012-11-15 08:49:10 +00006208 INIT_LIST_HEAD(&offload_base);
6209
Eric W. Biederman881d9662007-09-17 11:56:21 -07006210 if (register_pernet_subsys(&netdev_net_ops))
6211 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006212
6213 /*
6214 * Initialise the packet receive queues.
6215 */
6216
KAMEZAWA Hiroyuki6f912042006-04-10 22:52:50 -07006217 for_each_possible_cpu(i) {
Eric Dumazete36fa2f2010-04-19 21:17:14 +00006218 struct softnet_data *sd = &per_cpu(softnet_data, i);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006219
Changli Gaodee42872010-05-02 05:42:16 +00006220 memset(sd, 0, sizeof(*sd));
Eric Dumazete36fa2f2010-04-19 21:17:14 +00006221 skb_queue_head_init(&sd->input_pkt_queue);
Changli Gao6e7676c2010-04-27 15:07:33 -07006222 skb_queue_head_init(&sd->process_queue);
Eric Dumazete36fa2f2010-04-19 21:17:14 +00006223 sd->completion_queue = NULL;
6224 INIT_LIST_HEAD(&sd->poll_list);
Changli Gaoa9cbd582010-04-26 23:06:24 +00006225 sd->output_queue = NULL;
6226 sd->output_queue_tailp = &sd->output_queue;
Eric Dumazetdf334542010-03-24 19:13:54 +00006227#ifdef CONFIG_RPS
Eric Dumazete36fa2f2010-04-19 21:17:14 +00006228 sd->csd.func = rps_trigger_softirq;
6229 sd->csd.info = sd;
6230 sd->csd.flags = 0;
6231 sd->cpu = i;
Tom Herbert1e94d722010-03-18 17:45:44 -07006232#endif
Tom Herbert0a9627f2010-03-16 08:03:29 +00006233
Eric Dumazete36fa2f2010-04-19 21:17:14 +00006234 sd->backlog.poll = process_backlog;
6235 sd->backlog.weight = weight_p;
6236 sd->backlog.gro_list = NULL;
6237 sd->backlog.gro_count = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006238 }
6239
Linus Torvalds1da177e2005-04-16 15:20:36 -07006240 dev_boot_phase = 0;
6241
Eric W. Biederman505d4f72008-11-07 22:54:20 -08006242 /* The loopback device is special if any other network devices
6243 * is present in a network namespace the loopback device must
6244 * be present. Since we now dynamically allocate and free the
6245 * loopback device ensure this invariant is maintained by
6246 * keeping the loopback device as the first device on the
6247 * list of network devices. Ensuring the loopback devices
6248 * is the first device that appears and the last network device
6249 * that disappears.
6250 */
6251 if (register_pernet_device(&loopback_net_ops))
6252 goto out;
6253
6254 if (register_pernet_device(&default_device_ops))
6255 goto out;
6256
Carlos R. Mafra962cf362008-05-15 11:15:37 -03006257 open_softirq(NET_TX_SOFTIRQ, net_tx_action);
6258 open_softirq(NET_RX_SOFTIRQ, net_rx_action);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006259
6260 hotcpu_notifier(dev_cpu_callback, 0);
6261 dst_init();
Linus Torvalds1da177e2005-04-16 15:20:36 -07006262 rc = 0;
6263out:
6264 return rc;
6265}
6266
6267subsys_initcall(net_dev_init);