blob: 0dd54a69dace255fcdf54732d982e8c521c574a5 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * NET3 Protocol independent device support routines.
3 *
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public License
6 * as published by the Free Software Foundation; either version
7 * 2 of the License, or (at your option) any later version.
8 *
9 * Derived from the non IP parts of dev.c 1.0.19
Jesper Juhl02c30a82005-05-05 16:16:16 -070010 * Authors: Ross Biro
Linus Torvalds1da177e2005-04-16 15:20:36 -070011 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
12 * Mark Evans, <evansmp@uhura.aston.ac.uk>
13 *
14 * Additional Authors:
15 * Florian la Roche <rzsfl@rz.uni-sb.de>
16 * Alan Cox <gw4pts@gw4pts.ampr.org>
17 * David Hinds <dahinds@users.sourceforge.net>
18 * Alexey Kuznetsov <kuznet@ms2.inr.ac.ru>
19 * Adam Sulmicki <adam@cfar.umd.edu>
20 * Pekka Riikonen <priikone@poesidon.pspt.fi>
21 *
22 * Changes:
23 * D.J. Barrow : Fixed bug where dev->refcnt gets set
24 * to 2 if register_netdev gets called
25 * before net_dev_init & also removed a
26 * few lines of code in the process.
27 * Alan Cox : device private ioctl copies fields back.
28 * Alan Cox : Transmit queue code does relevant
29 * stunts to keep the queue safe.
30 * Alan Cox : Fixed double lock.
31 * Alan Cox : Fixed promisc NULL pointer trap
32 * ???????? : Support the full private ioctl range
33 * Alan Cox : Moved ioctl permission check into
34 * drivers
35 * Tim Kordas : SIOCADDMULTI/SIOCDELMULTI
36 * Alan Cox : 100 backlog just doesn't cut it when
37 * you start doing multicast video 8)
38 * Alan Cox : Rewrote net_bh and list manager.
39 * Alan Cox : Fix ETH_P_ALL echoback lengths.
40 * Alan Cox : Took out transmit every packet pass
41 * Saved a few bytes in the ioctl handler
42 * Alan Cox : Network driver sets packet type before
43 * calling netif_rx. Saves a function
44 * call a packet.
45 * Alan Cox : Hashed net_bh()
46 * Richard Kooijman: Timestamp fixes.
47 * Alan Cox : Wrong field in SIOCGIFDSTADDR
48 * Alan Cox : Device lock protection.
49 * Alan Cox : Fixed nasty side effect of device close
50 * changes.
51 * Rudi Cilibrasi : Pass the right thing to
52 * set_mac_address()
53 * Dave Miller : 32bit quantity for the device lock to
54 * make it work out on a Sparc.
55 * Bjorn Ekwall : Added KERNELD hack.
56 * Alan Cox : Cleaned up the backlog initialise.
57 * Craig Metz : SIOCGIFCONF fix if space for under
58 * 1 device.
59 * Thomas Bogendoerfer : Return ENODEV for dev_open, if there
60 * is no device open function.
61 * Andi Kleen : Fix error reporting for SIOCGIFCONF
62 * Michael Chastain : Fix signed/unsigned for SIOCGIFCONF
63 * Cyrus Durgin : Cleaned for KMOD
64 * Adam Sulmicki : Bug Fix : Network Device Unload
65 * A network device unload needs to purge
66 * the backlog queue.
67 * Paul Rusty Russell : SIOCSIFNAME
68 * Pekka Riikonen : Netdev boot-time settings code
69 * Andrew Morton : Make unregister_netdevice wait
70 * indefinitely on dev->refcnt
71 * J Hadi Salim : - Backlog queue sampling
72 * - netif_rx() feedback
73 */
74
75#include <asm/uaccess.h>
76#include <asm/system.h>
77#include <linux/bitops.h>
Randy Dunlap4fc268d2006-01-11 12:17:47 -080078#include <linux/capability.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070079#include <linux/cpu.h>
80#include <linux/types.h>
81#include <linux/kernel.h>
stephen hemminger08e98972009-11-10 07:20:34 +000082#include <linux/hash.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090083#include <linux/slab.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070084#include <linux/sched.h>
Arjan van de Ven4a3e2f72006-03-20 22:33:17 -080085#include <linux/mutex.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070086#include <linux/string.h>
87#include <linux/mm.h>
88#include <linux/socket.h>
89#include <linux/sockios.h>
90#include <linux/errno.h>
91#include <linux/interrupt.h>
92#include <linux/if_ether.h>
93#include <linux/netdevice.h>
94#include <linux/etherdevice.h>
Ben Hutchings0187bdf2008-06-19 16:15:47 -070095#include <linux/ethtool.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070096#include <linux/notifier.h>
97#include <linux/skbuff.h>
Eric W. Biederman457c4cb2007-09-12 12:01:34 +020098#include <net/net_namespace.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070099#include <net/sock.h>
100#include <linux/rtnetlink.h>
101#include <linux/proc_fs.h>
102#include <linux/seq_file.h>
103#include <linux/stat.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700104#include <net/dst.h>
105#include <net/pkt_sched.h>
106#include <net/checksum.h>
Arnd Bergmann44540962009-11-26 06:07:08 +0000107#include <net/xfrm.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700108#include <linux/highmem.h>
109#include <linux/init.h>
110#include <linux/kmod.h>
111#include <linux/module.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700112#include <linux/netpoll.h>
113#include <linux/rcupdate.h>
114#include <linux/delay.h>
Johannes Berg295f4a12007-04-26 20:43:56 -0700115#include <net/wext.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700116#include <net/iw_handler.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700117#include <asm/current.h>
Steve Grubb5bdb9882005-12-03 08:39:35 -0500118#include <linux/audit.h>
Chris Leechdb217332006-06-17 21:24:58 -0700119#include <linux/dmaengine.h>
Herbert Xuf6a78bf2006-06-22 02:57:17 -0700120#include <linux/err.h>
David S. Millerc7fa9d12006-08-15 16:34:13 -0700121#include <linux/ctype.h>
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700122#include <linux/if_arp.h>
Ben Hutchings6de329e2008-06-16 17:02:28 -0700123#include <linux/if_vlan.h>
David S. Miller8f0f2222008-07-15 03:47:03 -0700124#include <linux/ip.h>
Alexander Duyckad55dca2008-09-20 22:05:50 -0700125#include <net/ip.h>
David S. Miller8f0f2222008-07-15 03:47:03 -0700126#include <linux/ipv6.h>
127#include <linux/in.h>
David S. Millerb6b2fed2008-07-21 09:48:06 -0700128#include <linux/jhash.h>
129#include <linux/random.h>
David S. Miller9cbc1cb2009-06-15 03:02:23 -0700130#include <trace/events/napi.h>
Koki Sanagicf66ba52010-08-23 18:45:02 +0900131#include <trace/events/net.h>
Koki Sanagi07dc22e2010-08-23 18:46:12 +0900132#include <trace/events/skb.h>
FUJITA Tomonori5acbbd42010-03-30 22:35:50 +0000133#include <linux/pci.h>
Stephen Rothwellcaeda9b2010-09-16 21:39:16 -0700134#include <linux/inetdevice.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700135
Pavel Emelyanov342709e2007-10-23 21:14:45 -0700136#include "net-sysfs.h"
137
Herbert Xud565b0a2008-12-15 23:38:52 -0800138/* Instead of increasing this, you should create a hash table. */
139#define MAX_GRO_SKBS 8
140
Herbert Xu5d38a072009-01-04 16:13:40 -0800141/* This should be increased if a protocol with a bigger head is added. */
142#define GRO_MAX_HEAD (MAX_HEADER + 128)
143
Linus Torvalds1da177e2005-04-16 15:20:36 -0700144/*
145 * The list of packet types we will receive (as opposed to discard)
146 * and the routines to invoke.
147 *
148 * Why 16. Because with 16 the only overlap we get on a hash of the
149 * low nibble of the protocol value is RARP/SNAP/X.25.
150 *
151 * NOTE: That is no longer true with the addition of VLAN tags. Not
152 * sure which should go first, but I bet it won't make much
153 * difference if we are running VLANs. The good news is that
154 * this protocol won't be in the list unless compiled in, so
Stephen Hemminger3041a062006-05-26 13:25:24 -0700155 * the average user (w/out VLANs) will not be adversely affected.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700156 * --BLG
157 *
158 * 0800 IP
159 * 8100 802.1Q VLAN
160 * 0001 802.3
161 * 0002 AX.25
162 * 0004 802.2
163 * 8035 RARP
164 * 0005 SNAP
165 * 0805 X.25
166 * 0806 ARP
167 * 8137 IPX
168 * 0009 Localtalk
169 * 86DD IPv6
170 */
171
Pavel Emelyanov82d8a8672007-11-26 20:12:58 +0800172#define PTYPE_HASH_SIZE (16)
173#define PTYPE_HASH_MASK (PTYPE_HASH_SIZE - 1)
174
Linus Torvalds1da177e2005-04-16 15:20:36 -0700175static DEFINE_SPINLOCK(ptype_lock);
Pavel Emelyanov82d8a8672007-11-26 20:12:58 +0800176static struct list_head ptype_base[PTYPE_HASH_SIZE] __read_mostly;
Stephen Hemminger6b2bedc2007-03-12 14:33:50 -0700177static struct list_head ptype_all __read_mostly; /* Taps */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700178
Linus Torvalds1da177e2005-04-16 15:20:36 -0700179/*
Pavel Emelianov7562f872007-05-03 15:13:45 -0700180 * The @dev_base_head list is protected by @dev_base_lock and the rtnl
Linus Torvalds1da177e2005-04-16 15:20:36 -0700181 * semaphore.
182 *
Eric Dumazetc6d14c82009-11-04 05:43:23 -0800183 * Pure readers hold dev_base_lock for reading, or rcu_read_lock()
Linus Torvalds1da177e2005-04-16 15:20:36 -0700184 *
185 * Writers must hold the rtnl semaphore while they loop through the
Pavel Emelianov7562f872007-05-03 15:13:45 -0700186 * dev_base_head list, and hold dev_base_lock for writing when they do the
Linus Torvalds1da177e2005-04-16 15:20:36 -0700187 * actual updates. This allows pure readers to access the list even
188 * while a writer is preparing to update it.
189 *
190 * To put it another way, dev_base_lock is held for writing only to
191 * protect against pure readers; the rtnl semaphore provides the
192 * protection against other writers.
193 *
194 * See, for example usages, register_netdevice() and
195 * unregister_netdevice(), which must be called with the rtnl
196 * semaphore held.
197 */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700198DEFINE_RWLOCK(dev_base_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700199EXPORT_SYMBOL(dev_base_lock);
200
Eric W. Biederman881d9662007-09-17 11:56:21 -0700201static inline struct hlist_head *dev_name_hash(struct net *net, const char *name)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700202{
203 unsigned hash = full_name_hash(name, strnlen(name, IFNAMSIZ));
stephen hemminger08e98972009-11-10 07:20:34 +0000204 return &net->dev_name_head[hash_32(hash, NETDEV_HASHBITS)];
Linus Torvalds1da177e2005-04-16 15:20:36 -0700205}
206
Eric W. Biederman881d9662007-09-17 11:56:21 -0700207static inline struct hlist_head *dev_index_hash(struct net *net, int ifindex)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700208{
Eric Dumazet7c28bd02009-10-24 06:13:17 -0700209 return &net->dev_index_head[ifindex & (NETDEV_HASHENTRIES - 1)];
Linus Torvalds1da177e2005-04-16 15:20:36 -0700210}
211
Eric Dumazete36fa2f2010-04-19 21:17:14 +0000212static inline void rps_lock(struct softnet_data *sd)
Changli Gao152102c2010-03-30 20:16:22 +0000213{
214#ifdef CONFIG_RPS
Eric Dumazete36fa2f2010-04-19 21:17:14 +0000215 spin_lock(&sd->input_pkt_queue.lock);
Changli Gao152102c2010-03-30 20:16:22 +0000216#endif
217}
218
Eric Dumazete36fa2f2010-04-19 21:17:14 +0000219static inline void rps_unlock(struct softnet_data *sd)
Changli Gao152102c2010-03-30 20:16:22 +0000220{
221#ifdef CONFIG_RPS
Eric Dumazete36fa2f2010-04-19 21:17:14 +0000222 spin_unlock(&sd->input_pkt_queue.lock);
Changli Gao152102c2010-03-30 20:16:22 +0000223#endif
224}
225
Eric W. Biedermance286d32007-09-12 13:53:49 +0200226/* Device list insertion */
227static int list_netdevice(struct net_device *dev)
228{
YOSHIFUJI Hideakic346dca2008-03-25 21:47:49 +0900229 struct net *net = dev_net(dev);
Eric W. Biedermance286d32007-09-12 13:53:49 +0200230
231 ASSERT_RTNL();
232
233 write_lock_bh(&dev_base_lock);
Eric Dumazetc6d14c82009-11-04 05:43:23 -0800234 list_add_tail_rcu(&dev->dev_list, &net->dev_base_head);
Eric Dumazet72c95282009-10-30 07:11:27 +0000235 hlist_add_head_rcu(&dev->name_hlist, dev_name_hash(net, dev->name));
Eric Dumazetfb699dfd2009-10-19 19:18:49 +0000236 hlist_add_head_rcu(&dev->index_hlist,
237 dev_index_hash(net, dev->ifindex));
Eric W. Biedermance286d32007-09-12 13:53:49 +0200238 write_unlock_bh(&dev_base_lock);
239 return 0;
240}
241
Eric Dumazetfb699dfd2009-10-19 19:18:49 +0000242/* Device list removal
243 * caller must respect a RCU grace period before freeing/reusing dev
244 */
Eric W. Biedermance286d32007-09-12 13:53:49 +0200245static void unlist_netdevice(struct net_device *dev)
246{
247 ASSERT_RTNL();
248
249 /* Unlink dev from the device chain */
250 write_lock_bh(&dev_base_lock);
Eric Dumazetc6d14c82009-11-04 05:43:23 -0800251 list_del_rcu(&dev->dev_list);
Eric Dumazet72c95282009-10-30 07:11:27 +0000252 hlist_del_rcu(&dev->name_hlist);
Eric Dumazetfb699dfd2009-10-19 19:18:49 +0000253 hlist_del_rcu(&dev->index_hlist);
Eric W. Biedermance286d32007-09-12 13:53:49 +0200254 write_unlock_bh(&dev_base_lock);
255}
256
Linus Torvalds1da177e2005-04-16 15:20:36 -0700257/*
258 * Our notifier list
259 */
260
Alan Sternf07d5b92006-05-09 15:23:03 -0700261static RAW_NOTIFIER_HEAD(netdev_chain);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700262
263/*
264 * Device drivers call our routines to queue packets here. We empty the
265 * queue in the local softnet handler.
266 */
Stephen Hemmingerbea33482007-10-03 16:41:36 -0700267
Eric Dumazet9958da02010-04-17 04:17:02 +0000268DEFINE_PER_CPU_ALIGNED(struct softnet_data, softnet_data);
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700269EXPORT_PER_CPU_SYMBOL(softnet_data);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700270
David S. Millercf508b12008-07-22 14:16:42 -0700271#ifdef CONFIG_LOCKDEP
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700272/*
David S. Millerc773e842008-07-08 23:13:53 -0700273 * register_netdevice() inits txq->_xmit_lock and sets lockdep class
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700274 * according to dev->type
275 */
276static const unsigned short netdev_lock_type[] =
277 {ARPHRD_NETROM, ARPHRD_ETHER, ARPHRD_EETHER, ARPHRD_AX25,
278 ARPHRD_PRONET, ARPHRD_CHAOS, ARPHRD_IEEE802, ARPHRD_ARCNET,
279 ARPHRD_APPLETLK, ARPHRD_DLCI, ARPHRD_ATM, ARPHRD_METRICOM,
280 ARPHRD_IEEE1394, ARPHRD_EUI64, ARPHRD_INFINIBAND, ARPHRD_SLIP,
281 ARPHRD_CSLIP, ARPHRD_SLIP6, ARPHRD_CSLIP6, ARPHRD_RSRVD,
282 ARPHRD_ADAPT, ARPHRD_ROSE, ARPHRD_X25, ARPHRD_HWX25,
283 ARPHRD_PPP, ARPHRD_CISCO, ARPHRD_LAPB, ARPHRD_DDCMP,
284 ARPHRD_RAWHDLC, ARPHRD_TUNNEL, ARPHRD_TUNNEL6, ARPHRD_FRAD,
285 ARPHRD_SKIP, ARPHRD_LOOPBACK, ARPHRD_LOCALTLK, ARPHRD_FDDI,
286 ARPHRD_BIF, ARPHRD_SIT, ARPHRD_IPDDP, ARPHRD_IPGRE,
287 ARPHRD_PIMREG, ARPHRD_HIPPI, ARPHRD_ASH, ARPHRD_ECONET,
288 ARPHRD_IRDA, ARPHRD_FCPP, ARPHRD_FCAL, ARPHRD_FCPL,
289 ARPHRD_FCFABRIC, ARPHRD_IEEE802_TR, ARPHRD_IEEE80211,
Rémi Denis-Courmont2d91d782008-12-17 15:47:29 -0800290 ARPHRD_IEEE80211_PRISM, ARPHRD_IEEE80211_RADIOTAP, ARPHRD_PHONET,
Dmitry Eremin-Solenikov929122cd2009-08-14 20:00:20 +0400291 ARPHRD_PHONET_PIPE, ARPHRD_IEEE802154,
Sergey Lapinfcb94e42009-06-08 12:18:47 +0000292 ARPHRD_VOID, ARPHRD_NONE};
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700293
Jan Engelhardt36cbd3d2009-08-05 10:42:58 -0700294static const char *const netdev_lock_name[] =
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700295 {"_xmit_NETROM", "_xmit_ETHER", "_xmit_EETHER", "_xmit_AX25",
296 "_xmit_PRONET", "_xmit_CHAOS", "_xmit_IEEE802", "_xmit_ARCNET",
297 "_xmit_APPLETLK", "_xmit_DLCI", "_xmit_ATM", "_xmit_METRICOM",
298 "_xmit_IEEE1394", "_xmit_EUI64", "_xmit_INFINIBAND", "_xmit_SLIP",
299 "_xmit_CSLIP", "_xmit_SLIP6", "_xmit_CSLIP6", "_xmit_RSRVD",
300 "_xmit_ADAPT", "_xmit_ROSE", "_xmit_X25", "_xmit_HWX25",
301 "_xmit_PPP", "_xmit_CISCO", "_xmit_LAPB", "_xmit_DDCMP",
302 "_xmit_RAWHDLC", "_xmit_TUNNEL", "_xmit_TUNNEL6", "_xmit_FRAD",
303 "_xmit_SKIP", "_xmit_LOOPBACK", "_xmit_LOCALTLK", "_xmit_FDDI",
304 "_xmit_BIF", "_xmit_SIT", "_xmit_IPDDP", "_xmit_IPGRE",
305 "_xmit_PIMREG", "_xmit_HIPPI", "_xmit_ASH", "_xmit_ECONET",
306 "_xmit_IRDA", "_xmit_FCPP", "_xmit_FCAL", "_xmit_FCPL",
307 "_xmit_FCFABRIC", "_xmit_IEEE802_TR", "_xmit_IEEE80211",
Rémi Denis-Courmont2d91d782008-12-17 15:47:29 -0800308 "_xmit_IEEE80211_PRISM", "_xmit_IEEE80211_RADIOTAP", "_xmit_PHONET",
Dmitry Eremin-Solenikov929122cd2009-08-14 20:00:20 +0400309 "_xmit_PHONET_PIPE", "_xmit_IEEE802154",
Sergey Lapinfcb94e42009-06-08 12:18:47 +0000310 "_xmit_VOID", "_xmit_NONE"};
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700311
312static struct lock_class_key netdev_xmit_lock_key[ARRAY_SIZE(netdev_lock_type)];
David S. Millercf508b12008-07-22 14:16:42 -0700313static struct lock_class_key netdev_addr_lock_key[ARRAY_SIZE(netdev_lock_type)];
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700314
315static inline unsigned short netdev_lock_pos(unsigned short dev_type)
316{
317 int i;
318
319 for (i = 0; i < ARRAY_SIZE(netdev_lock_type); i++)
320 if (netdev_lock_type[i] == dev_type)
321 return i;
322 /* the last key is used by default */
323 return ARRAY_SIZE(netdev_lock_type) - 1;
324}
325
David S. Millercf508b12008-07-22 14:16:42 -0700326static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock,
327 unsigned short dev_type)
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700328{
329 int i;
330
331 i = netdev_lock_pos(dev_type);
332 lockdep_set_class_and_name(lock, &netdev_xmit_lock_key[i],
333 netdev_lock_name[i]);
334}
David S. Millercf508b12008-07-22 14:16:42 -0700335
336static inline void netdev_set_addr_lockdep_class(struct net_device *dev)
337{
338 int i;
339
340 i = netdev_lock_pos(dev->type);
341 lockdep_set_class_and_name(&dev->addr_list_lock,
342 &netdev_addr_lock_key[i],
343 netdev_lock_name[i]);
344}
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700345#else
David S. Millercf508b12008-07-22 14:16:42 -0700346static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock,
347 unsigned short dev_type)
348{
349}
350static inline void netdev_set_addr_lockdep_class(struct net_device *dev)
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700351{
352}
353#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -0700354
355/*******************************************************************************
356
357 Protocol management and registration routines
358
359*******************************************************************************/
360
361/*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700362 * Add a protocol ID to the list. Now that the input handler is
363 * smarter we can dispense with all the messy stuff that used to be
364 * here.
365 *
366 * BEWARE!!! Protocol handlers, mangling input packets,
367 * MUST BE last in hash buckets and checking protocol handlers
368 * MUST start from promiscuous ptype_all chain in net_bh.
369 * It is true now, do not change it.
370 * Explanation follows: if protocol handler, mangling packet, will
371 * be the first on list, it is not able to sense, that packet
372 * is cloned and should be copied-on-write, so that it will
373 * change it and subsequent readers will get broken packet.
374 * --ANK (980803)
375 */
376
Eric Dumazetc07b68e2010-09-02 03:53:46 +0000377static inline struct list_head *ptype_head(const struct packet_type *pt)
378{
379 if (pt->type == htons(ETH_P_ALL))
380 return &ptype_all;
381 else
382 return &ptype_base[ntohs(pt->type) & PTYPE_HASH_MASK];
383}
384
Linus Torvalds1da177e2005-04-16 15:20:36 -0700385/**
386 * dev_add_pack - add packet handler
387 * @pt: packet type declaration
388 *
389 * Add a protocol handler to the networking stack. The passed &packet_type
390 * is linked into kernel lists and may not be freed until it has been
391 * removed from the kernel lists.
392 *
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +0900393 * This call does not sleep therefore it can not
Linus Torvalds1da177e2005-04-16 15:20:36 -0700394 * guarantee all CPU's that are in middle of receiving packets
395 * will see the new packet type (until the next received packet).
396 */
397
398void dev_add_pack(struct packet_type *pt)
399{
Eric Dumazetc07b68e2010-09-02 03:53:46 +0000400 struct list_head *head = ptype_head(pt);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700401
Eric Dumazetc07b68e2010-09-02 03:53:46 +0000402 spin_lock(&ptype_lock);
403 list_add_rcu(&pt->list, head);
404 spin_unlock(&ptype_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700405}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700406EXPORT_SYMBOL(dev_add_pack);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700407
Linus Torvalds1da177e2005-04-16 15:20:36 -0700408/**
409 * __dev_remove_pack - remove packet handler
410 * @pt: packet type declaration
411 *
412 * Remove a protocol handler that was previously added to the kernel
413 * protocol handlers by dev_add_pack(). The passed &packet_type is removed
414 * from the kernel lists and can be freed or reused once this function
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +0900415 * returns.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700416 *
417 * The packet type might still be in use by receivers
418 * and must not be freed until after all the CPU's have gone
419 * through a quiescent state.
420 */
421void __dev_remove_pack(struct packet_type *pt)
422{
Eric Dumazetc07b68e2010-09-02 03:53:46 +0000423 struct list_head *head = ptype_head(pt);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700424 struct packet_type *pt1;
425
Eric Dumazetc07b68e2010-09-02 03:53:46 +0000426 spin_lock(&ptype_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700427
428 list_for_each_entry(pt1, head, list) {
429 if (pt == pt1) {
430 list_del_rcu(&pt->list);
431 goto out;
432 }
433 }
434
435 printk(KERN_WARNING "dev_remove_pack: %p not found.\n", pt);
436out:
Eric Dumazetc07b68e2010-09-02 03:53:46 +0000437 spin_unlock(&ptype_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700438}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700439EXPORT_SYMBOL(__dev_remove_pack);
440
Linus Torvalds1da177e2005-04-16 15:20:36 -0700441/**
442 * dev_remove_pack - remove packet handler
443 * @pt: packet type declaration
444 *
445 * Remove a protocol handler that was previously added to the kernel
446 * protocol handlers by dev_add_pack(). The passed &packet_type is removed
447 * from the kernel lists and can be freed or reused once this function
448 * returns.
449 *
450 * This call sleeps to guarantee that no CPU is looking at the packet
451 * type after return.
452 */
453void dev_remove_pack(struct packet_type *pt)
454{
455 __dev_remove_pack(pt);
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +0900456
Linus Torvalds1da177e2005-04-16 15:20:36 -0700457 synchronize_net();
458}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700459EXPORT_SYMBOL(dev_remove_pack);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700460
461/******************************************************************************
462
463 Device Boot-time Settings Routines
464
465*******************************************************************************/
466
467/* Boot time configuration table */
468static struct netdev_boot_setup dev_boot_setup[NETDEV_BOOT_SETUP_MAX];
469
470/**
471 * netdev_boot_setup_add - add new setup entry
472 * @name: name of the device
473 * @map: configured settings for the device
474 *
475 * Adds new setup entry to the dev_boot_setup list. The function
476 * returns 0 on error and 1 on success. This is a generic routine to
477 * all netdevices.
478 */
479static int netdev_boot_setup_add(char *name, struct ifmap *map)
480{
481 struct netdev_boot_setup *s;
482 int i;
483
484 s = dev_boot_setup;
485 for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++) {
486 if (s[i].name[0] == '\0' || s[i].name[0] == ' ') {
487 memset(s[i].name, 0, sizeof(s[i].name));
Wang Chen93b3cff2008-07-01 19:57:19 -0700488 strlcpy(s[i].name, name, IFNAMSIZ);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700489 memcpy(&s[i].map, map, sizeof(s[i].map));
490 break;
491 }
492 }
493
494 return i >= NETDEV_BOOT_SETUP_MAX ? 0 : 1;
495}
496
497/**
498 * netdev_boot_setup_check - check boot time settings
499 * @dev: the netdevice
500 *
501 * Check boot time settings for the device.
502 * The found settings are set for the device to be used
503 * later in the device probing.
504 * Returns 0 if no settings found, 1 if they are.
505 */
506int netdev_boot_setup_check(struct net_device *dev)
507{
508 struct netdev_boot_setup *s = dev_boot_setup;
509 int i;
510
511 for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++) {
512 if (s[i].name[0] != '\0' && s[i].name[0] != ' ' &&
Wang Chen93b3cff2008-07-01 19:57:19 -0700513 !strcmp(dev->name, s[i].name)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700514 dev->irq = s[i].map.irq;
515 dev->base_addr = s[i].map.base_addr;
516 dev->mem_start = s[i].map.mem_start;
517 dev->mem_end = s[i].map.mem_end;
518 return 1;
519 }
520 }
521 return 0;
522}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700523EXPORT_SYMBOL(netdev_boot_setup_check);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700524
525
526/**
527 * netdev_boot_base - get address from boot time settings
528 * @prefix: prefix for network device
529 * @unit: id for network device
530 *
531 * Check boot time settings for the base address of device.
532 * The found settings are set for the device to be used
533 * later in the device probing.
534 * Returns 0 if no settings found.
535 */
536unsigned long netdev_boot_base(const char *prefix, int unit)
537{
538 const struct netdev_boot_setup *s = dev_boot_setup;
539 char name[IFNAMSIZ];
540 int i;
541
542 sprintf(name, "%s%d", prefix, unit);
543
544 /*
545 * If device already registered then return base of 1
546 * to indicate not to probe for this interface
547 */
Eric W. Biederman881d9662007-09-17 11:56:21 -0700548 if (__dev_get_by_name(&init_net, name))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700549 return 1;
550
551 for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++)
552 if (!strcmp(name, s[i].name))
553 return s[i].map.base_addr;
554 return 0;
555}
556
557/*
558 * Saves at boot time configured settings for any netdevice.
559 */
560int __init netdev_boot_setup(char *str)
561{
562 int ints[5];
563 struct ifmap map;
564
565 str = get_options(str, ARRAY_SIZE(ints), ints);
566 if (!str || !*str)
567 return 0;
568
569 /* Save settings */
570 memset(&map, 0, sizeof(map));
571 if (ints[0] > 0)
572 map.irq = ints[1];
573 if (ints[0] > 1)
574 map.base_addr = ints[2];
575 if (ints[0] > 2)
576 map.mem_start = ints[3];
577 if (ints[0] > 3)
578 map.mem_end = ints[4];
579
580 /* Add new entry to the list */
581 return netdev_boot_setup_add(str, &map);
582}
583
584__setup("netdev=", netdev_boot_setup);
585
586/*******************************************************************************
587
588 Device Interface Subroutines
589
590*******************************************************************************/
591
592/**
593 * __dev_get_by_name - find a device by its name
Randy Dunlapc4ea43c2007-10-12 21:17:49 -0700594 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -0700595 * @name: name to find
596 *
597 * Find an interface by name. Must be called under RTNL semaphore
598 * or @dev_base_lock. If the name is found a pointer to the device
599 * is returned. If the name is not found then %NULL is returned. The
600 * reference counters are not incremented so the caller must be
601 * careful with locks.
602 */
603
Eric W. Biederman881d9662007-09-17 11:56:21 -0700604struct net_device *__dev_get_by_name(struct net *net, const char *name)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700605{
606 struct hlist_node *p;
Eric Dumazet0bd8d532009-10-30 01:40:11 -0700607 struct net_device *dev;
608 struct hlist_head *head = dev_name_hash(net, name);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700609
Eric Dumazet0bd8d532009-10-30 01:40:11 -0700610 hlist_for_each_entry(dev, p, head, name_hlist)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700611 if (!strncmp(dev->name, name, IFNAMSIZ))
612 return dev;
Eric Dumazet0bd8d532009-10-30 01:40:11 -0700613
Linus Torvalds1da177e2005-04-16 15:20:36 -0700614 return NULL;
615}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700616EXPORT_SYMBOL(__dev_get_by_name);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700617
618/**
Eric Dumazet72c95282009-10-30 07:11:27 +0000619 * dev_get_by_name_rcu - find a device by its name
620 * @net: the applicable net namespace
621 * @name: name to find
622 *
623 * Find an interface by name.
624 * If the name is found a pointer to the device is returned.
625 * If the name is not found then %NULL is returned.
626 * The reference counters are not incremented so the caller must be
627 * careful with locks. The caller must hold RCU lock.
628 */
629
630struct net_device *dev_get_by_name_rcu(struct net *net, const char *name)
631{
632 struct hlist_node *p;
633 struct net_device *dev;
634 struct hlist_head *head = dev_name_hash(net, name);
635
636 hlist_for_each_entry_rcu(dev, p, head, name_hlist)
637 if (!strncmp(dev->name, name, IFNAMSIZ))
638 return dev;
639
640 return NULL;
641}
642EXPORT_SYMBOL(dev_get_by_name_rcu);
643
644/**
Linus Torvalds1da177e2005-04-16 15:20:36 -0700645 * dev_get_by_name - find a device by its name
Randy Dunlapc4ea43c2007-10-12 21:17:49 -0700646 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -0700647 * @name: name to find
648 *
649 * Find an interface by name. This can be called from any
650 * context and does its own locking. The returned handle has
651 * the usage count incremented and the caller must use dev_put() to
652 * release it when it is no longer needed. %NULL is returned if no
653 * matching device is found.
654 */
655
Eric W. Biederman881d9662007-09-17 11:56:21 -0700656struct net_device *dev_get_by_name(struct net *net, const char *name)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700657{
658 struct net_device *dev;
659
Eric Dumazet72c95282009-10-30 07:11:27 +0000660 rcu_read_lock();
661 dev = dev_get_by_name_rcu(net, name);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700662 if (dev)
663 dev_hold(dev);
Eric Dumazet72c95282009-10-30 07:11:27 +0000664 rcu_read_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700665 return dev;
666}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700667EXPORT_SYMBOL(dev_get_by_name);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700668
669/**
670 * __dev_get_by_index - find a device by its ifindex
Randy Dunlapc4ea43c2007-10-12 21:17:49 -0700671 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -0700672 * @ifindex: index of device
673 *
674 * Search for an interface by index. Returns %NULL if the device
675 * is not found or a pointer to the device. The device has not
676 * had its reference counter increased so the caller must be careful
677 * about locking. The caller must hold either the RTNL semaphore
678 * or @dev_base_lock.
679 */
680
Eric W. Biederman881d9662007-09-17 11:56:21 -0700681struct net_device *__dev_get_by_index(struct net *net, int ifindex)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700682{
683 struct hlist_node *p;
Eric Dumazet0bd8d532009-10-30 01:40:11 -0700684 struct net_device *dev;
685 struct hlist_head *head = dev_index_hash(net, ifindex);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700686
Eric Dumazet0bd8d532009-10-30 01:40:11 -0700687 hlist_for_each_entry(dev, p, head, index_hlist)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700688 if (dev->ifindex == ifindex)
689 return dev;
Eric Dumazet0bd8d532009-10-30 01:40:11 -0700690
Linus Torvalds1da177e2005-04-16 15:20:36 -0700691 return NULL;
692}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700693EXPORT_SYMBOL(__dev_get_by_index);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700694
Eric Dumazetfb699dfd2009-10-19 19:18:49 +0000695/**
696 * dev_get_by_index_rcu - find a device by its ifindex
697 * @net: the applicable net namespace
698 * @ifindex: index of device
699 *
700 * Search for an interface by index. Returns %NULL if the device
701 * is not found or a pointer to the device. The device has not
702 * had its reference counter increased so the caller must be careful
703 * about locking. The caller must hold RCU lock.
704 */
705
706struct net_device *dev_get_by_index_rcu(struct net *net, int ifindex)
707{
708 struct hlist_node *p;
709 struct net_device *dev;
710 struct hlist_head *head = dev_index_hash(net, ifindex);
711
712 hlist_for_each_entry_rcu(dev, p, head, index_hlist)
713 if (dev->ifindex == ifindex)
714 return dev;
715
716 return NULL;
717}
718EXPORT_SYMBOL(dev_get_by_index_rcu);
719
Linus Torvalds1da177e2005-04-16 15:20:36 -0700720
721/**
722 * dev_get_by_index - find a device by its ifindex
Randy Dunlapc4ea43c2007-10-12 21:17:49 -0700723 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -0700724 * @ifindex: index of device
725 *
726 * Search for an interface by index. Returns NULL if the device
727 * is not found or a pointer to the device. The device returned has
728 * had a reference added and the pointer is safe until the user calls
729 * dev_put to indicate they have finished with it.
730 */
731
Eric W. Biederman881d9662007-09-17 11:56:21 -0700732struct net_device *dev_get_by_index(struct net *net, int ifindex)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700733{
734 struct net_device *dev;
735
Eric Dumazetfb699dfd2009-10-19 19:18:49 +0000736 rcu_read_lock();
737 dev = dev_get_by_index_rcu(net, ifindex);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700738 if (dev)
739 dev_hold(dev);
Eric Dumazetfb699dfd2009-10-19 19:18:49 +0000740 rcu_read_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700741 return dev;
742}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700743EXPORT_SYMBOL(dev_get_by_index);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700744
745/**
746 * dev_getbyhwaddr - find a device by its hardware address
Randy Dunlapc4ea43c2007-10-12 21:17:49 -0700747 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -0700748 * @type: media type of device
749 * @ha: hardware address
750 *
751 * Search for an interface by MAC address. Returns NULL if the device
752 * is not found or a pointer to the device. The caller must hold the
753 * rtnl semaphore. The returned device has not had its ref count increased
754 * and the caller must therefore be careful about locking
755 *
756 * BUGS:
757 * If the API was consistent this would be __dev_get_by_hwaddr
758 */
759
Eric W. Biederman881d9662007-09-17 11:56:21 -0700760struct net_device *dev_getbyhwaddr(struct net *net, unsigned short type, char *ha)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700761{
762 struct net_device *dev;
763
764 ASSERT_RTNL();
765
Denis V. Lunev81103a52007-12-12 10:47:38 -0800766 for_each_netdev(net, dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700767 if (dev->type == type &&
768 !memcmp(dev->dev_addr, ha, dev->addr_len))
Pavel Emelianov7562f872007-05-03 15:13:45 -0700769 return dev;
770
771 return NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700772}
Jochen Friedrichcf309e32005-09-22 04:44:55 -0300773EXPORT_SYMBOL(dev_getbyhwaddr);
774
Eric W. Biederman881d9662007-09-17 11:56:21 -0700775struct net_device *__dev_getfirstbyhwtype(struct net *net, unsigned short type)
Patrick McHardy4e9cac22007-05-03 03:28:13 -0700776{
777 struct net_device *dev;
778
779 ASSERT_RTNL();
Eric W. Biederman881d9662007-09-17 11:56:21 -0700780 for_each_netdev(net, dev)
Patrick McHardy4e9cac22007-05-03 03:28:13 -0700781 if (dev->type == type)
Pavel Emelianov7562f872007-05-03 15:13:45 -0700782 return dev;
783
784 return NULL;
Patrick McHardy4e9cac22007-05-03 03:28:13 -0700785}
Patrick McHardy4e9cac22007-05-03 03:28:13 -0700786EXPORT_SYMBOL(__dev_getfirstbyhwtype);
787
Eric W. Biederman881d9662007-09-17 11:56:21 -0700788struct net_device *dev_getfirstbyhwtype(struct net *net, unsigned short type)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700789{
Eric Dumazet99fe3c32010-03-18 11:27:25 +0000790 struct net_device *dev, *ret = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700791
Eric Dumazet99fe3c32010-03-18 11:27:25 +0000792 rcu_read_lock();
793 for_each_netdev_rcu(net, dev)
794 if (dev->type == type) {
795 dev_hold(dev);
796 ret = dev;
797 break;
798 }
799 rcu_read_unlock();
800 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700801}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700802EXPORT_SYMBOL(dev_getfirstbyhwtype);
803
804/**
Eric Dumazetbb69ae02010-06-07 11:42:13 +0000805 * dev_get_by_flags_rcu - find any device with given flags
Randy Dunlapc4ea43c2007-10-12 21:17:49 -0700806 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -0700807 * @if_flags: IFF_* values
808 * @mask: bitmask of bits in if_flags to check
809 *
810 * Search for any interface with the given flags. Returns NULL if a device
Eric Dumazetbb69ae02010-06-07 11:42:13 +0000811 * is not found or a pointer to the device. Must be called inside
812 * rcu_read_lock(), and result refcount is unchanged.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700813 */
814
Eric Dumazetbb69ae02010-06-07 11:42:13 +0000815struct net_device *dev_get_by_flags_rcu(struct net *net, unsigned short if_flags,
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700816 unsigned short mask)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700817{
Pavel Emelianov7562f872007-05-03 15:13:45 -0700818 struct net_device *dev, *ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700819
Pavel Emelianov7562f872007-05-03 15:13:45 -0700820 ret = NULL;
Eric Dumazetc6d14c82009-11-04 05:43:23 -0800821 for_each_netdev_rcu(net, dev) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700822 if (((dev->flags ^ if_flags) & mask) == 0) {
Pavel Emelianov7562f872007-05-03 15:13:45 -0700823 ret = dev;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700824 break;
825 }
826 }
Pavel Emelianov7562f872007-05-03 15:13:45 -0700827 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700828}
Eric Dumazetbb69ae02010-06-07 11:42:13 +0000829EXPORT_SYMBOL(dev_get_by_flags_rcu);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700830
831/**
832 * dev_valid_name - check if name is okay for network device
833 * @name: name string
834 *
835 * Network device names need to be valid file names to
David S. Millerc7fa9d12006-08-15 16:34:13 -0700836 * to allow sysfs to work. We also disallow any kind of
837 * whitespace.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700838 */
Mitch Williamsc2373ee2005-11-09 10:34:45 -0800839int dev_valid_name(const char *name)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700840{
David S. Millerc7fa9d12006-08-15 16:34:13 -0700841 if (*name == '\0')
842 return 0;
Stephen Hemmingerb6fe17d2006-08-29 17:06:13 -0700843 if (strlen(name) >= IFNAMSIZ)
844 return 0;
David S. Millerc7fa9d12006-08-15 16:34:13 -0700845 if (!strcmp(name, ".") || !strcmp(name, ".."))
846 return 0;
847
848 while (*name) {
849 if (*name == '/' || isspace(*name))
850 return 0;
851 name++;
852 }
853 return 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700854}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700855EXPORT_SYMBOL(dev_valid_name);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700856
857/**
Eric W. Biedermanb267b172007-09-12 13:48:45 +0200858 * __dev_alloc_name - allocate a name for a device
859 * @net: network namespace to allocate the device name in
Linus Torvalds1da177e2005-04-16 15:20:36 -0700860 * @name: name format string
Eric W. Biedermanb267b172007-09-12 13:48:45 +0200861 * @buf: scratch buffer and result name string
Linus Torvalds1da177e2005-04-16 15:20:36 -0700862 *
863 * Passed a format string - eg "lt%d" it will try and find a suitable
Stephen Hemminger3041a062006-05-26 13:25:24 -0700864 * id. It scans list of devices to build up a free map, then chooses
865 * the first empty slot. The caller must hold the dev_base or rtnl lock
866 * while allocating the name and adding the device in order to avoid
867 * duplicates.
868 * Limited to bits_per_byte * page size devices (ie 32K on most platforms).
869 * Returns the number of the unit assigned or a negative errno code.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700870 */
871
Eric W. Biedermanb267b172007-09-12 13:48:45 +0200872static int __dev_alloc_name(struct net *net, const char *name, char *buf)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700873{
874 int i = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700875 const char *p;
876 const int max_netdevices = 8*PAGE_SIZE;
Stephen Hemmingercfcabdc2007-10-09 01:59:42 -0700877 unsigned long *inuse;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700878 struct net_device *d;
879
880 p = strnchr(name, IFNAMSIZ-1, '%');
881 if (p) {
882 /*
883 * Verify the string as this thing may have come from
884 * the user. There must be either one "%d" and no other "%"
885 * characters.
886 */
887 if (p[1] != 'd' || strchr(p + 2, '%'))
888 return -EINVAL;
889
890 /* Use one page as a bit array of possible slots */
Stephen Hemmingercfcabdc2007-10-09 01:59:42 -0700891 inuse = (unsigned long *) get_zeroed_page(GFP_ATOMIC);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700892 if (!inuse)
893 return -ENOMEM;
894
Eric W. Biederman881d9662007-09-17 11:56:21 -0700895 for_each_netdev(net, d) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700896 if (!sscanf(d->name, name, &i))
897 continue;
898 if (i < 0 || i >= max_netdevices)
899 continue;
900
901 /* avoid cases where sscanf is not exact inverse of printf */
Eric W. Biedermanb267b172007-09-12 13:48:45 +0200902 snprintf(buf, IFNAMSIZ, name, i);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700903 if (!strncmp(buf, d->name, IFNAMSIZ))
904 set_bit(i, inuse);
905 }
906
907 i = find_first_zero_bit(inuse, max_netdevices);
908 free_page((unsigned long) inuse);
909 }
910
Octavian Purdilad9031022009-11-18 02:36:59 +0000911 if (buf != name)
912 snprintf(buf, IFNAMSIZ, name, i);
Eric W. Biedermanb267b172007-09-12 13:48:45 +0200913 if (!__dev_get_by_name(net, buf))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700914 return i;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700915
916 /* It is possible to run out of possible slots
917 * when the name is long and there isn't enough space left
918 * for the digits, or if all bits are used.
919 */
920 return -ENFILE;
921}
922
Eric W. Biedermanb267b172007-09-12 13:48:45 +0200923/**
924 * dev_alloc_name - allocate a name for a device
925 * @dev: device
926 * @name: name format string
927 *
928 * Passed a format string - eg "lt%d" it will try and find a suitable
929 * id. It scans list of devices to build up a free map, then chooses
930 * the first empty slot. The caller must hold the dev_base or rtnl lock
931 * while allocating the name and adding the device in order to avoid
932 * duplicates.
933 * Limited to bits_per_byte * page size devices (ie 32K on most platforms).
934 * Returns the number of the unit assigned or a negative errno code.
935 */
936
937int dev_alloc_name(struct net_device *dev, const char *name)
938{
939 char buf[IFNAMSIZ];
940 struct net *net;
941 int ret;
942
YOSHIFUJI Hideakic346dca2008-03-25 21:47:49 +0900943 BUG_ON(!dev_net(dev));
944 net = dev_net(dev);
Eric W. Biedermanb267b172007-09-12 13:48:45 +0200945 ret = __dev_alloc_name(net, name, buf);
946 if (ret >= 0)
947 strlcpy(dev->name, buf, IFNAMSIZ);
948 return ret;
949}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700950EXPORT_SYMBOL(dev_alloc_name);
Eric W. Biedermanb267b172007-09-12 13:48:45 +0200951
Daniel Lezcano8ce6cebc2010-05-19 10:12:19 +0000952static int dev_get_valid_name(struct net_device *dev, const char *name, bool fmt)
Octavian Purdilad9031022009-11-18 02:36:59 +0000953{
Daniel Lezcano8ce6cebc2010-05-19 10:12:19 +0000954 struct net *net;
955
956 BUG_ON(!dev_net(dev));
957 net = dev_net(dev);
958
Octavian Purdilad9031022009-11-18 02:36:59 +0000959 if (!dev_valid_name(name))
960 return -EINVAL;
961
962 if (fmt && strchr(name, '%'))
Daniel Lezcano8ce6cebc2010-05-19 10:12:19 +0000963 return dev_alloc_name(dev, name);
Octavian Purdilad9031022009-11-18 02:36:59 +0000964 else if (__dev_get_by_name(net, name))
965 return -EEXIST;
Daniel Lezcano8ce6cebc2010-05-19 10:12:19 +0000966 else if (dev->name != name)
967 strlcpy(dev->name, name, IFNAMSIZ);
Octavian Purdilad9031022009-11-18 02:36:59 +0000968
969 return 0;
970}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700971
972/**
973 * dev_change_name - change name of a device
974 * @dev: device
975 * @newname: name (or format string) must be at least IFNAMSIZ
976 *
977 * Change name of a device, can pass format strings "eth%d".
978 * for wildcarding.
979 */
Stephen Hemmingercf04a4c72008-09-30 02:22:14 -0700980int dev_change_name(struct net_device *dev, const char *newname)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700981{
Herbert Xufcc5a032007-07-30 17:03:38 -0700982 char oldname[IFNAMSIZ];
Linus Torvalds1da177e2005-04-16 15:20:36 -0700983 int err = 0;
Herbert Xufcc5a032007-07-30 17:03:38 -0700984 int ret;
Eric W. Biederman881d9662007-09-17 11:56:21 -0700985 struct net *net;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700986
987 ASSERT_RTNL();
YOSHIFUJI Hideakic346dca2008-03-25 21:47:49 +0900988 BUG_ON(!dev_net(dev));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700989
YOSHIFUJI Hideakic346dca2008-03-25 21:47:49 +0900990 net = dev_net(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700991 if (dev->flags & IFF_UP)
992 return -EBUSY;
993
Stephen Hemmingerc8d90dc2007-10-26 03:53:42 -0700994 if (strncmp(newname, dev->name, IFNAMSIZ) == 0)
995 return 0;
996
Herbert Xufcc5a032007-07-30 17:03:38 -0700997 memcpy(oldname, dev->name, IFNAMSIZ);
998
Daniel Lezcano8ce6cebc2010-05-19 10:12:19 +0000999 err = dev_get_valid_name(dev, newname, 1);
Octavian Purdilad9031022009-11-18 02:36:59 +00001000 if (err < 0)
1001 return err;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001002
Herbert Xufcc5a032007-07-30 17:03:38 -07001003rollback:
Eric W. Biedermana1b3f592010-05-04 17:36:49 -07001004 ret = device_rename(&dev->dev, dev->name);
1005 if (ret) {
1006 memcpy(dev->name, oldname, IFNAMSIZ);
1007 return ret;
Stephen Hemmingerdcc99772008-05-14 22:33:38 -07001008 }
Herbert Xu7f988ea2007-07-30 16:35:46 -07001009
1010 write_lock_bh(&dev_base_lock);
Eric W. Biederman92749822007-04-03 00:07:30 -06001011 hlist_del(&dev->name_hlist);
Eric Dumazet72c95282009-10-30 07:11:27 +00001012 write_unlock_bh(&dev_base_lock);
1013
1014 synchronize_rcu();
1015
1016 write_lock_bh(&dev_base_lock);
1017 hlist_add_head_rcu(&dev->name_hlist, dev_name_hash(net, dev->name));
Herbert Xu7f988ea2007-07-30 16:35:46 -07001018 write_unlock_bh(&dev_base_lock);
1019
Pavel Emelyanov056925a2007-09-16 15:42:43 -07001020 ret = call_netdevice_notifiers(NETDEV_CHANGENAME, dev);
Herbert Xufcc5a032007-07-30 17:03:38 -07001021 ret = notifier_to_errno(ret);
1022
1023 if (ret) {
Eric Dumazet91e9c07b2009-11-15 23:30:24 +00001024 /* err >= 0 after dev_alloc_name() or stores the first errno */
1025 if (err >= 0) {
Herbert Xufcc5a032007-07-30 17:03:38 -07001026 err = ret;
1027 memcpy(dev->name, oldname, IFNAMSIZ);
1028 goto rollback;
Eric Dumazet91e9c07b2009-11-15 23:30:24 +00001029 } else {
1030 printk(KERN_ERR
1031 "%s: name change rollback failed: %d.\n",
1032 dev->name, ret);
Herbert Xufcc5a032007-07-30 17:03:38 -07001033 }
1034 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001035
1036 return err;
1037}
1038
1039/**
Stephen Hemminger0b815a12008-09-22 21:28:11 -07001040 * dev_set_alias - change ifalias of a device
1041 * @dev: device
1042 * @alias: name up to IFALIASZ
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07001043 * @len: limit of bytes to copy from info
Stephen Hemminger0b815a12008-09-22 21:28:11 -07001044 *
1045 * Set ifalias for a device,
1046 */
1047int dev_set_alias(struct net_device *dev, const char *alias, size_t len)
1048{
1049 ASSERT_RTNL();
1050
1051 if (len >= IFALIASZ)
1052 return -EINVAL;
1053
Oliver Hartkopp96ca4a22008-09-23 21:23:19 -07001054 if (!len) {
1055 if (dev->ifalias) {
1056 kfree(dev->ifalias);
1057 dev->ifalias = NULL;
1058 }
1059 return 0;
1060 }
1061
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001062 dev->ifalias = krealloc(dev->ifalias, len + 1, GFP_KERNEL);
Stephen Hemminger0b815a12008-09-22 21:28:11 -07001063 if (!dev->ifalias)
1064 return -ENOMEM;
1065
1066 strlcpy(dev->ifalias, alias, len+1);
1067 return len;
1068}
1069
1070
1071/**
Stephen Hemminger3041a062006-05-26 13:25:24 -07001072 * netdev_features_change - device changes features
Stephen Hemmingerd8a33ac2005-05-29 14:13:47 -07001073 * @dev: device to cause notification
1074 *
1075 * Called to indicate a device has changed features.
1076 */
1077void netdev_features_change(struct net_device *dev)
1078{
Pavel Emelyanov056925a2007-09-16 15:42:43 -07001079 call_netdevice_notifiers(NETDEV_FEAT_CHANGE, dev);
Stephen Hemmingerd8a33ac2005-05-29 14:13:47 -07001080}
1081EXPORT_SYMBOL(netdev_features_change);
1082
1083/**
Linus Torvalds1da177e2005-04-16 15:20:36 -07001084 * netdev_state_change - device changes state
1085 * @dev: device to cause notification
1086 *
1087 * Called to indicate a device has changed state. This function calls
1088 * the notifier chains for netdev_chain and sends a NEWLINK message
1089 * to the routing socket.
1090 */
1091void netdev_state_change(struct net_device *dev)
1092{
1093 if (dev->flags & IFF_UP) {
Pavel Emelyanov056925a2007-09-16 15:42:43 -07001094 call_netdevice_notifiers(NETDEV_CHANGE, dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001095 rtmsg_ifinfo(RTM_NEWLINK, dev, 0);
1096 }
1097}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001098EXPORT_SYMBOL(netdev_state_change);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001099
Jiri Pirko3ca5b402010-03-10 10:29:35 +00001100int netdev_bonding_change(struct net_device *dev, unsigned long event)
Or Gerlitzc1da4ac2008-06-13 18:12:00 -07001101{
Jiri Pirko3ca5b402010-03-10 10:29:35 +00001102 return call_netdevice_notifiers(event, dev);
Or Gerlitzc1da4ac2008-06-13 18:12:00 -07001103}
1104EXPORT_SYMBOL(netdev_bonding_change);
1105
Linus Torvalds1da177e2005-04-16 15:20:36 -07001106/**
1107 * dev_load - load a network module
Randy Dunlapc4ea43c2007-10-12 21:17:49 -07001108 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -07001109 * @name: name of interface
1110 *
1111 * If a network interface is not present and the process has suitable
1112 * privileges this function loads the module. If module loading is not
1113 * available in this kernel then it becomes a nop.
1114 */
1115
Eric W. Biederman881d9662007-09-17 11:56:21 -07001116void dev_load(struct net *net, const char *name)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001117{
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09001118 struct net_device *dev;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001119
Eric Dumazet72c95282009-10-30 07:11:27 +00001120 rcu_read_lock();
1121 dev = dev_get_by_name_rcu(net, name);
1122 rcu_read_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07001123
Eric Parisa8f80e82009-08-13 09:44:51 -04001124 if (!dev && capable(CAP_NET_ADMIN))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001125 request_module("%s", name);
1126}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001127EXPORT_SYMBOL(dev_load);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001128
Patrick McHardybd380812010-02-26 06:34:53 +00001129static int __dev_open(struct net_device *dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001130{
Stephen Hemmingerd3147742008-11-19 21:32:24 -08001131 const struct net_device_ops *ops = dev->netdev_ops;
Johannes Berg3b8bcfd2009-05-30 01:39:53 +02001132 int ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001133
Ben Hutchingse46b66b2008-05-08 02:53:17 -07001134 ASSERT_RTNL();
1135
Linus Torvalds1da177e2005-04-16 15:20:36 -07001136 /*
Linus Torvalds1da177e2005-04-16 15:20:36 -07001137 * Is it even present?
1138 */
1139 if (!netif_device_present(dev))
1140 return -ENODEV;
1141
Johannes Berg3b8bcfd2009-05-30 01:39:53 +02001142 ret = call_netdevice_notifiers(NETDEV_PRE_UP, dev);
1143 ret = notifier_to_errno(ret);
1144 if (ret)
1145 return ret;
1146
Linus Torvalds1da177e2005-04-16 15:20:36 -07001147 /*
1148 * Call device private open method
1149 */
1150 set_bit(__LINK_STATE_START, &dev->state);
Jeff Garzikbada3392007-10-23 20:19:37 -07001151
Stephen Hemmingerd3147742008-11-19 21:32:24 -08001152 if (ops->ndo_validate_addr)
1153 ret = ops->ndo_validate_addr(dev);
Jeff Garzikbada3392007-10-23 20:19:37 -07001154
Stephen Hemmingerd3147742008-11-19 21:32:24 -08001155 if (!ret && ops->ndo_open)
1156 ret = ops->ndo_open(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001157
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09001158 /*
Linus Torvalds1da177e2005-04-16 15:20:36 -07001159 * If it went open OK then:
1160 */
1161
Jeff Garzikbada3392007-10-23 20:19:37 -07001162 if (ret)
1163 clear_bit(__LINK_STATE_START, &dev->state);
1164 else {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001165 /*
1166 * Set the flags.
1167 */
1168 dev->flags |= IFF_UP;
1169
1170 /*
Dan Williams649274d2009-01-11 00:20:39 -08001171 * Enable NET_DMA
1172 */
David S. Millerb4bd07c2009-02-06 22:06:43 -08001173 net_dmaengine_get();
Dan Williams649274d2009-01-11 00:20:39 -08001174
1175 /*
Linus Torvalds1da177e2005-04-16 15:20:36 -07001176 * Initialize multicasting status
1177 */
Patrick McHardy4417da62007-06-27 01:28:10 -07001178 dev_set_rx_mode(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001179
1180 /*
1181 * Wakeup transmit queue engine
1182 */
1183 dev_activate(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001184 }
Jeff Garzikbada3392007-10-23 20:19:37 -07001185
Linus Torvalds1da177e2005-04-16 15:20:36 -07001186 return ret;
1187}
Patrick McHardybd380812010-02-26 06:34:53 +00001188
1189/**
1190 * dev_open - prepare an interface for use.
1191 * @dev: device to open
1192 *
1193 * Takes a device from down to up state. The device's private open
1194 * function is invoked and then the multicast lists are loaded. Finally
1195 * the device is moved into the up state and a %NETDEV_UP message is
1196 * sent to the netdev notifier chain.
1197 *
1198 * Calling this function on an active interface is a nop. On a failure
1199 * a negative errno code is returned.
1200 */
1201int dev_open(struct net_device *dev)
1202{
1203 int ret;
1204
1205 /*
1206 * Is it already up?
1207 */
1208 if (dev->flags & IFF_UP)
1209 return 0;
1210
1211 /*
1212 * Open device
1213 */
1214 ret = __dev_open(dev);
1215 if (ret < 0)
1216 return ret;
1217
1218 /*
1219 * ... and announce new interface.
1220 */
1221 rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP|IFF_RUNNING);
1222 call_netdevice_notifiers(NETDEV_UP, dev);
1223
1224 return ret;
1225}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001226EXPORT_SYMBOL(dev_open);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001227
Patrick McHardybd380812010-02-26 06:34:53 +00001228static int __dev_close(struct net_device *dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001229{
Stephen Hemmingerd3147742008-11-19 21:32:24 -08001230 const struct net_device_ops *ops = dev->netdev_ops;
Patrick McHardybd380812010-02-26 06:34:53 +00001231
Ben Hutchingse46b66b2008-05-08 02:53:17 -07001232 ASSERT_RTNL();
David S. Miller9d5010d2007-09-12 14:33:25 +02001233 might_sleep();
1234
Linus Torvalds1da177e2005-04-16 15:20:36 -07001235 /*
1236 * Tell people we are going down, so that they can
1237 * prepare to death, when device is still operating.
1238 */
Pavel Emelyanov056925a2007-09-16 15:42:43 -07001239 call_netdevice_notifiers(NETDEV_GOING_DOWN, dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001240
Linus Torvalds1da177e2005-04-16 15:20:36 -07001241 clear_bit(__LINK_STATE_START, &dev->state);
1242
1243 /* Synchronize to scheduled poll. We cannot touch poll list,
Stephen Hemmingerbea33482007-10-03 16:41:36 -07001244 * it can be even on different cpu. So just clear netif_running().
1245 *
1246 * dev->stop() will invoke napi_disable() on all of it's
1247 * napi_struct instances on this device.
1248 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001249 smp_mb__after_clear_bit(); /* Commit netif_running(). */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001250
Matti Linnanvuorid8b2a4d2008-02-12 23:10:11 -08001251 dev_deactivate(dev);
1252
Linus Torvalds1da177e2005-04-16 15:20:36 -07001253 /*
1254 * Call the device specific close. This cannot fail.
1255 * Only if device is UP
1256 *
1257 * We allow it to be called even after a DETACH hot-plug
1258 * event.
1259 */
Stephen Hemmingerd3147742008-11-19 21:32:24 -08001260 if (ops->ndo_stop)
1261 ops->ndo_stop(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001262
1263 /*
1264 * Device is now down.
1265 */
1266
1267 dev->flags &= ~IFF_UP;
1268
1269 /*
Dan Williams649274d2009-01-11 00:20:39 -08001270 * Shutdown NET_DMA
1271 */
David S. Millerb4bd07c2009-02-06 22:06:43 -08001272 net_dmaengine_put();
Dan Williams649274d2009-01-11 00:20:39 -08001273
Linus Torvalds1da177e2005-04-16 15:20:36 -07001274 return 0;
1275}
Patrick McHardybd380812010-02-26 06:34:53 +00001276
1277/**
1278 * dev_close - shutdown an interface.
1279 * @dev: device to shutdown
1280 *
1281 * This function moves an active device into down state. A
1282 * %NETDEV_GOING_DOWN is sent to the netdev notifier chain. The device
1283 * is then deactivated and finally a %NETDEV_DOWN is sent to the notifier
1284 * chain.
1285 */
1286int dev_close(struct net_device *dev)
1287{
1288 if (!(dev->flags & IFF_UP))
1289 return 0;
1290
1291 __dev_close(dev);
1292
1293 /*
1294 * Tell people we are down
1295 */
1296 rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP|IFF_RUNNING);
1297 call_netdevice_notifiers(NETDEV_DOWN, dev);
1298
1299 return 0;
1300}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001301EXPORT_SYMBOL(dev_close);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001302
1303
Ben Hutchings0187bdf2008-06-19 16:15:47 -07001304/**
1305 * dev_disable_lro - disable Large Receive Offload on a device
1306 * @dev: device
1307 *
1308 * Disable Large Receive Offload (LRO) on a net device. Must be
1309 * called under RTNL. This is needed if received packets may be
1310 * forwarded to another interface.
1311 */
1312void dev_disable_lro(struct net_device *dev)
1313{
1314 if (dev->ethtool_ops && dev->ethtool_ops->get_flags &&
1315 dev->ethtool_ops->set_flags) {
1316 u32 flags = dev->ethtool_ops->get_flags(dev);
1317 if (flags & ETH_FLAG_LRO) {
1318 flags &= ~ETH_FLAG_LRO;
1319 dev->ethtool_ops->set_flags(dev, flags);
1320 }
1321 }
1322 WARN_ON(dev->features & NETIF_F_LRO);
1323}
1324EXPORT_SYMBOL(dev_disable_lro);
1325
1326
Eric W. Biederman881d9662007-09-17 11:56:21 -07001327static int dev_boot_phase = 1;
1328
Linus Torvalds1da177e2005-04-16 15:20:36 -07001329/*
1330 * Device change register/unregister. These are not inline or static
1331 * as we export them to the world.
1332 */
1333
1334/**
1335 * register_netdevice_notifier - register a network notifier block
1336 * @nb: notifier
1337 *
1338 * Register a notifier to be called when network device events occur.
1339 * The notifier passed is linked into the kernel structures and must
1340 * not be reused until it has been unregistered. A negative errno code
1341 * is returned on a failure.
1342 *
1343 * When registered all registration and up events are replayed
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09001344 * to the new notifier to allow device to have a race free
Linus Torvalds1da177e2005-04-16 15:20:36 -07001345 * view of the network device list.
1346 */
1347
1348int register_netdevice_notifier(struct notifier_block *nb)
1349{
1350 struct net_device *dev;
Herbert Xufcc5a032007-07-30 17:03:38 -07001351 struct net_device *last;
Eric W. Biederman881d9662007-09-17 11:56:21 -07001352 struct net *net;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001353 int err;
1354
1355 rtnl_lock();
Alan Sternf07d5b92006-05-09 15:23:03 -07001356 err = raw_notifier_chain_register(&netdev_chain, nb);
Herbert Xufcc5a032007-07-30 17:03:38 -07001357 if (err)
1358 goto unlock;
Eric W. Biederman881d9662007-09-17 11:56:21 -07001359 if (dev_boot_phase)
1360 goto unlock;
1361 for_each_net(net) {
1362 for_each_netdev(net, dev) {
1363 err = nb->notifier_call(nb, NETDEV_REGISTER, dev);
1364 err = notifier_to_errno(err);
1365 if (err)
1366 goto rollback;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001367
Eric W. Biederman881d9662007-09-17 11:56:21 -07001368 if (!(dev->flags & IFF_UP))
1369 continue;
Herbert Xufcc5a032007-07-30 17:03:38 -07001370
Eric W. Biederman881d9662007-09-17 11:56:21 -07001371 nb->notifier_call(nb, NETDEV_UP, dev);
1372 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001373 }
Herbert Xufcc5a032007-07-30 17:03:38 -07001374
1375unlock:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001376 rtnl_unlock();
1377 return err;
Herbert Xufcc5a032007-07-30 17:03:38 -07001378
1379rollback:
1380 last = dev;
Eric W. Biederman881d9662007-09-17 11:56:21 -07001381 for_each_net(net) {
1382 for_each_netdev(net, dev) {
1383 if (dev == last)
1384 break;
Herbert Xufcc5a032007-07-30 17:03:38 -07001385
Eric W. Biederman881d9662007-09-17 11:56:21 -07001386 if (dev->flags & IFF_UP) {
1387 nb->notifier_call(nb, NETDEV_GOING_DOWN, dev);
1388 nb->notifier_call(nb, NETDEV_DOWN, dev);
1389 }
1390 nb->notifier_call(nb, NETDEV_UNREGISTER, dev);
Eric W. Biedermana5ee1552009-11-29 15:45:58 +00001391 nb->notifier_call(nb, NETDEV_UNREGISTER_BATCH, dev);
Herbert Xufcc5a032007-07-30 17:03:38 -07001392 }
Herbert Xufcc5a032007-07-30 17:03:38 -07001393 }
Pavel Emelyanovc67625a2007-11-14 15:53:16 -08001394
1395 raw_notifier_chain_unregister(&netdev_chain, nb);
Herbert Xufcc5a032007-07-30 17:03:38 -07001396 goto unlock;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001397}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001398EXPORT_SYMBOL(register_netdevice_notifier);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001399
1400/**
1401 * unregister_netdevice_notifier - unregister a network notifier block
1402 * @nb: notifier
1403 *
1404 * Unregister a notifier previously registered by
1405 * register_netdevice_notifier(). The notifier is unlinked into the
1406 * kernel structures and may then be reused. A negative errno code
1407 * is returned on a failure.
1408 */
1409
1410int unregister_netdevice_notifier(struct notifier_block *nb)
1411{
Herbert Xu9f514952006-03-25 01:24:25 -08001412 int err;
1413
1414 rtnl_lock();
Alan Sternf07d5b92006-05-09 15:23:03 -07001415 err = raw_notifier_chain_unregister(&netdev_chain, nb);
Herbert Xu9f514952006-03-25 01:24:25 -08001416 rtnl_unlock();
1417 return err;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001418}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001419EXPORT_SYMBOL(unregister_netdevice_notifier);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001420
1421/**
1422 * call_netdevice_notifiers - call all network notifier blocks
1423 * @val: value passed unmodified to notifier function
Randy Dunlapc4ea43c2007-10-12 21:17:49 -07001424 * @dev: net_device pointer passed unmodified to notifier function
Linus Torvalds1da177e2005-04-16 15:20:36 -07001425 *
1426 * Call all network notifier blocks. Parameters and return value
Alan Sternf07d5b92006-05-09 15:23:03 -07001427 * are as for raw_notifier_call_chain().
Linus Torvalds1da177e2005-04-16 15:20:36 -07001428 */
1429
Eric W. Biedermanad7379d2007-09-16 15:33:32 -07001430int call_netdevice_notifiers(unsigned long val, struct net_device *dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001431{
Jiri Pirkoab930472010-04-20 01:45:37 -07001432 ASSERT_RTNL();
Eric W. Biedermanad7379d2007-09-16 15:33:32 -07001433 return raw_notifier_call_chain(&netdev_chain, val, dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001434}
1435
1436/* When > 0 there are consumers of rx skb time stamps */
1437static atomic_t netstamp_needed = ATOMIC_INIT(0);
1438
1439void net_enable_timestamp(void)
1440{
1441 atomic_inc(&netstamp_needed);
1442}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001443EXPORT_SYMBOL(net_enable_timestamp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001444
1445void net_disable_timestamp(void)
1446{
1447 atomic_dec(&netstamp_needed);
1448}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001449EXPORT_SYMBOL(net_disable_timestamp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001450
Eric Dumazet3b098e22010-05-15 23:57:10 -07001451static inline void net_timestamp_set(struct sk_buff *skb)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001452{
1453 if (atomic_read(&netstamp_needed))
Patrick McHardya61bbcf2005-08-14 17:24:31 -07001454 __net_timestamp(skb);
Eric Dumazetb7aa0bf2007-04-19 16:16:32 -07001455 else
1456 skb->tstamp.tv64 = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001457}
1458
Eric Dumazet3b098e22010-05-15 23:57:10 -07001459static inline void net_timestamp_check(struct sk_buff *skb)
1460{
1461 if (!skb->tstamp.tv64 && atomic_read(&netstamp_needed))
1462 __net_timestamp(skb);
1463}
1464
Arnd Bergmann44540962009-11-26 06:07:08 +00001465/**
1466 * dev_forward_skb - loopback an skb to another netif
1467 *
1468 * @dev: destination network device
1469 * @skb: buffer to forward
1470 *
1471 * return values:
1472 * NET_RX_SUCCESS (no congestion)
Eric Dumazet6ec82562010-05-06 00:53:53 -07001473 * NET_RX_DROP (packet was dropped, but freed)
Arnd Bergmann44540962009-11-26 06:07:08 +00001474 *
1475 * dev_forward_skb can be used for injecting an skb from the
1476 * start_xmit function of one device into the receive queue
1477 * of another device.
1478 *
1479 * The receiving device may be in another namespace, so
1480 * we have to clear all information in the skb that could
1481 * impact namespace isolation.
1482 */
1483int dev_forward_skb(struct net_device *dev, struct sk_buff *skb)
1484{
1485 skb_orphan(skb);
Ben Greearc736eef2010-07-22 09:54:47 +00001486 nf_reset(skb);
Arnd Bergmann44540962009-11-26 06:07:08 +00001487
Eric Dumazetcaf586e2010-09-30 21:06:55 +00001488 if (unlikely(!(dev->flags & IFF_UP) ||
David S. Miller2198a102010-10-21 08:43:05 -07001489 (skb->len > (dev->mtu + dev->hard_header_len + VLAN_HLEN)))) {
Eric Dumazetcaf586e2010-09-30 21:06:55 +00001490 atomic_long_inc(&dev->rx_dropped);
Eric Dumazet6ec82562010-05-06 00:53:53 -07001491 kfree_skb(skb);
Arnd Bergmann44540962009-11-26 06:07:08 +00001492 return NET_RX_DROP;
Eric Dumazet6ec82562010-05-06 00:53:53 -07001493 }
Arnd Bergmann8a83a002010-01-30 12:23:03 +00001494 skb_set_dev(skb, dev);
Arnd Bergmann44540962009-11-26 06:07:08 +00001495 skb->tstamp.tv64 = 0;
1496 skb->pkt_type = PACKET_HOST;
1497 skb->protocol = eth_type_trans(skb, dev);
Arnd Bergmann44540962009-11-26 06:07:08 +00001498 return netif_rx(skb);
1499}
1500EXPORT_SYMBOL_GPL(dev_forward_skb);
1501
Linus Torvalds1da177e2005-04-16 15:20:36 -07001502/*
1503 * Support routine. Sends outgoing frames to any network
1504 * taps currently in use.
1505 */
1506
Herbert Xuf6a78bf2006-06-22 02:57:17 -07001507static void dev_queue_xmit_nit(struct sk_buff *skb, struct net_device *dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001508{
1509 struct packet_type *ptype;
Patrick McHardya61bbcf2005-08-14 17:24:31 -07001510
Jarek Poplawski8caf1532009-04-17 10:08:49 +00001511#ifdef CONFIG_NET_CLS_ACT
1512 if (!(skb->tstamp.tv64 && (G_TC_FROM(skb->tc_verd) & AT_INGRESS)))
Eric Dumazet3b098e22010-05-15 23:57:10 -07001513 net_timestamp_set(skb);
Jarek Poplawski8caf1532009-04-17 10:08:49 +00001514#else
Eric Dumazet3b098e22010-05-15 23:57:10 -07001515 net_timestamp_set(skb);
Jarek Poplawski8caf1532009-04-17 10:08:49 +00001516#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07001517
1518 rcu_read_lock();
1519 list_for_each_entry_rcu(ptype, &ptype_all, list) {
1520 /* Never send packets back to the socket
1521 * they originated from - MvS (miquels@drinkel.ow.org)
1522 */
1523 if ((ptype->dev == dev || !ptype->dev) &&
1524 (ptype->af_packet_priv == NULL ||
1525 (struct sock *)ptype->af_packet_priv != skb->sk)) {
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001526 struct sk_buff *skb2 = skb_clone(skb, GFP_ATOMIC);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001527 if (!skb2)
1528 break;
1529
1530 /* skb->nh should be correctly
1531 set by sender, so that the second statement is
1532 just protection against buggy protocols.
1533 */
Arnaldo Carvalho de Melo459a98e2007-03-19 15:30:44 -07001534 skb_reset_mac_header(skb2);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001535
Arnaldo Carvalho de Melod56f90a2007-04-10 20:50:43 -07001536 if (skb_network_header(skb2) < skb2->data ||
Arnaldo Carvalho de Melo27a884d2007-04-19 20:29:13 -07001537 skb2->network_header > skb2->tail) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001538 if (net_ratelimit())
1539 printk(KERN_CRIT "protocol %04x is "
1540 "buggy, dev %s\n",
Sebastian Andrzej Siewior70777d02010-06-30 10:39:19 -07001541 ntohs(skb2->protocol),
1542 dev->name);
Arnaldo Carvalho de Meloc1d2bbe2007-04-10 20:45:18 -07001543 skb_reset_network_header(skb2);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001544 }
1545
Arnaldo Carvalho de Melob0e380b2007-04-10 21:21:55 -07001546 skb2->transport_header = skb2->network_header;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001547 skb2->pkt_type = PACKET_OUTGOING;
David S. Millerf2ccd8f2005-08-09 19:34:12 -07001548 ptype->func(skb2, skb->dev, ptype, skb->dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001549 }
1550 }
1551 rcu_read_unlock();
1552}
1553
John Fastabendf0796d52010-07-01 13:21:57 +00001554/*
1555 * Routine to help set real_num_tx_queues. To avoid skbs mapped to queues
1556 * greater then real_num_tx_queues stale skbs on the qdisc must be flushed.
1557 */
Tom Herberte6484932010-10-18 18:04:39 +00001558int netif_set_real_num_tx_queues(struct net_device *dev, unsigned int txq)
John Fastabendf0796d52010-07-01 13:21:57 +00001559{
Tom Herberte6484932010-10-18 18:04:39 +00001560 if (txq < 1 || txq > dev->num_tx_queues)
1561 return -EINVAL;
John Fastabendf0796d52010-07-01 13:21:57 +00001562
Tom Herberte6484932010-10-18 18:04:39 +00001563 if (dev->reg_state == NETREG_REGISTERED) {
1564 ASSERT_RTNL();
1565
1566 if (txq < dev->real_num_tx_queues)
1567 qdisc_reset_all_tx_gt(dev, txq);
John Fastabendf0796d52010-07-01 13:21:57 +00001568 }
Tom Herberte6484932010-10-18 18:04:39 +00001569
1570 dev->real_num_tx_queues = txq;
1571 return 0;
John Fastabendf0796d52010-07-01 13:21:57 +00001572}
1573EXPORT_SYMBOL(netif_set_real_num_tx_queues);
Denis Vlasenko56079432006-03-29 15:57:29 -08001574
Ben Hutchings62fe0b42010-09-27 08:24:33 +00001575#ifdef CONFIG_RPS
1576/**
1577 * netif_set_real_num_rx_queues - set actual number of RX queues used
1578 * @dev: Network device
1579 * @rxq: Actual number of RX queues
1580 *
1581 * This must be called either with the rtnl_lock held or before
1582 * registration of the net device. Returns 0 on success, or a
Ben Hutchings4e7f7952010-10-08 10:33:39 -07001583 * negative error code. If called before registration, it always
1584 * succeeds.
Ben Hutchings62fe0b42010-09-27 08:24:33 +00001585 */
1586int netif_set_real_num_rx_queues(struct net_device *dev, unsigned int rxq)
1587{
1588 int rc;
1589
Tom Herbertbd25fa72010-10-18 18:00:16 +00001590 if (rxq < 1 || rxq > dev->num_rx_queues)
1591 return -EINVAL;
1592
Ben Hutchings62fe0b42010-09-27 08:24:33 +00001593 if (dev->reg_state == NETREG_REGISTERED) {
1594 ASSERT_RTNL();
1595
Ben Hutchings62fe0b42010-09-27 08:24:33 +00001596 rc = net_rx_queue_update_kobjects(dev, dev->real_num_rx_queues,
1597 rxq);
1598 if (rc)
1599 return rc;
Ben Hutchings62fe0b42010-09-27 08:24:33 +00001600 }
1601
1602 dev->real_num_rx_queues = rxq;
1603 return 0;
1604}
1605EXPORT_SYMBOL(netif_set_real_num_rx_queues);
1606#endif
1607
Jarek Poplawskidef82a12008-08-17 21:54:43 -07001608static inline void __netif_reschedule(struct Qdisc *q)
1609{
1610 struct softnet_data *sd;
1611 unsigned long flags;
1612
1613 local_irq_save(flags);
1614 sd = &__get_cpu_var(softnet_data);
Changli Gaoa9cbd582010-04-26 23:06:24 +00001615 q->next_sched = NULL;
1616 *sd->output_queue_tailp = q;
1617 sd->output_queue_tailp = &q->next_sched;
Jarek Poplawskidef82a12008-08-17 21:54:43 -07001618 raise_softirq_irqoff(NET_TX_SOFTIRQ);
1619 local_irq_restore(flags);
1620}
1621
David S. Miller37437bb2008-07-16 02:15:04 -07001622void __netif_schedule(struct Qdisc *q)
Denis Vlasenko56079432006-03-29 15:57:29 -08001623{
Jarek Poplawskidef82a12008-08-17 21:54:43 -07001624 if (!test_and_set_bit(__QDISC_STATE_SCHED, &q->state))
1625 __netif_reschedule(q);
Denis Vlasenko56079432006-03-29 15:57:29 -08001626}
1627EXPORT_SYMBOL(__netif_schedule);
1628
Stephen Hemmingerbea33482007-10-03 16:41:36 -07001629void dev_kfree_skb_irq(struct sk_buff *skb)
Denis Vlasenko56079432006-03-29 15:57:29 -08001630{
David S. Miller3578b0c2010-08-03 00:24:04 -07001631 if (atomic_dec_and_test(&skb->users)) {
Stephen Hemmingerbea33482007-10-03 16:41:36 -07001632 struct softnet_data *sd;
1633 unsigned long flags;
Denis Vlasenko56079432006-03-29 15:57:29 -08001634
Stephen Hemmingerbea33482007-10-03 16:41:36 -07001635 local_irq_save(flags);
1636 sd = &__get_cpu_var(softnet_data);
1637 skb->next = sd->completion_queue;
1638 sd->completion_queue = skb;
1639 raise_softirq_irqoff(NET_TX_SOFTIRQ);
1640 local_irq_restore(flags);
1641 }
Denis Vlasenko56079432006-03-29 15:57:29 -08001642}
Stephen Hemmingerbea33482007-10-03 16:41:36 -07001643EXPORT_SYMBOL(dev_kfree_skb_irq);
Denis Vlasenko56079432006-03-29 15:57:29 -08001644
1645void dev_kfree_skb_any(struct sk_buff *skb)
1646{
1647 if (in_irq() || irqs_disabled())
1648 dev_kfree_skb_irq(skb);
1649 else
1650 dev_kfree_skb(skb);
1651}
1652EXPORT_SYMBOL(dev_kfree_skb_any);
1653
1654
Stephen Hemmingerbea33482007-10-03 16:41:36 -07001655/**
1656 * netif_device_detach - mark device as removed
1657 * @dev: network device
1658 *
1659 * Mark device as removed from system and therefore no longer available.
1660 */
Denis Vlasenko56079432006-03-29 15:57:29 -08001661void netif_device_detach(struct net_device *dev)
1662{
1663 if (test_and_clear_bit(__LINK_STATE_PRESENT, &dev->state) &&
1664 netif_running(dev)) {
Alexander Duyckd5431032009-04-08 13:15:22 +00001665 netif_tx_stop_all_queues(dev);
Denis Vlasenko56079432006-03-29 15:57:29 -08001666 }
1667}
1668EXPORT_SYMBOL(netif_device_detach);
1669
Stephen Hemmingerbea33482007-10-03 16:41:36 -07001670/**
1671 * netif_device_attach - mark device as attached
1672 * @dev: network device
1673 *
1674 * Mark device as attached from system and restart if needed.
1675 */
Denis Vlasenko56079432006-03-29 15:57:29 -08001676void netif_device_attach(struct net_device *dev)
1677{
1678 if (!test_and_set_bit(__LINK_STATE_PRESENT, &dev->state) &&
1679 netif_running(dev)) {
Alexander Duyckd5431032009-04-08 13:15:22 +00001680 netif_tx_wake_all_queues(dev);
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09001681 __netdev_watchdog_up(dev);
Denis Vlasenko56079432006-03-29 15:57:29 -08001682 }
1683}
1684EXPORT_SYMBOL(netif_device_attach);
1685
Ben Hutchings6de329e2008-06-16 17:02:28 -07001686static bool can_checksum_protocol(unsigned long features, __be16 protocol)
1687{
Ben Hutchings66c68bc2010-10-22 04:38:26 +00001688 return ((features & NETIF_F_NO_CSUM) ||
1689 ((features & NETIF_F_V4_CSUM) &&
Ben Hutchings6de329e2008-06-16 17:02:28 -07001690 protocol == htons(ETH_P_IP)) ||
Ben Hutchings66c68bc2010-10-22 04:38:26 +00001691 ((features & NETIF_F_V6_CSUM) &&
Yi Zou1c8dbcf2009-02-27 14:06:54 -08001692 protocol == htons(ETH_P_IPV6)) ||
1693 ((features & NETIF_F_FCOE_CRC) &&
1694 protocol == htons(ETH_P_FCOE)));
Ben Hutchings6de329e2008-06-16 17:02:28 -07001695}
1696
1697static bool dev_can_checksum(struct net_device *dev, struct sk_buff *skb)
1698{
Ben Hutchingsaf1905d2010-10-22 04:12:19 +00001699 __be16 protocol = skb->protocol;
Jesse Gross7b9c6092010-10-20 13:56:04 +00001700 int features = dev->features;
1701
Ben Hutchingsaf1905d2010-10-22 04:12:19 +00001702 if (vlan_tx_tag_present(skb)) {
Jesse Gross7b9c6092010-10-20 13:56:04 +00001703 features &= dev->vlan_features;
Ben Hutchingsaf1905d2010-10-22 04:12:19 +00001704 } else if (protocol == htons(ETH_P_8021Q)) {
Ben Hutchings6de329e2008-06-16 17:02:28 -07001705 struct vlan_ethhdr *veh = (struct vlan_ethhdr *)skb->data;
Ben Hutchingsaf1905d2010-10-22 04:12:19 +00001706 protocol = veh->h_vlan_encapsulated_proto;
1707 features &= dev->vlan_features;
Ben Hutchings6de329e2008-06-16 17:02:28 -07001708 }
1709
Ben Hutchingsaf1905d2010-10-22 04:12:19 +00001710 return can_checksum_protocol(features, protocol);
Ben Hutchings6de329e2008-06-16 17:02:28 -07001711}
Denis Vlasenko56079432006-03-29 15:57:29 -08001712
Arnd Bergmann8a83a002010-01-30 12:23:03 +00001713/**
1714 * skb_dev_set -- assign a new device to a buffer
1715 * @skb: buffer for the new device
1716 * @dev: network device
1717 *
1718 * If an skb is owned by a device already, we have to reset
1719 * all data private to the namespace a device belongs to
1720 * before assigning it a new device.
1721 */
1722#ifdef CONFIG_NET_NS
1723void skb_set_dev(struct sk_buff *skb, struct net_device *dev)
1724{
1725 skb_dst_drop(skb);
1726 if (skb->dev && !net_eq(dev_net(skb->dev), dev_net(dev))) {
1727 secpath_reset(skb);
1728 nf_reset(skb);
1729 skb_init_secmark(skb);
1730 skb->mark = 0;
1731 skb->priority = 0;
1732 skb->nf_trace = 0;
1733 skb->ipvs_property = 0;
1734#ifdef CONFIG_NET_SCHED
1735 skb->tc_index = 0;
1736#endif
1737 }
1738 skb->dev = dev;
1739}
1740EXPORT_SYMBOL(skb_set_dev);
1741#endif /* CONFIG_NET_NS */
1742
Linus Torvalds1da177e2005-04-16 15:20:36 -07001743/*
1744 * Invalidate hardware checksum when packet is to be mangled, and
1745 * complete checksum manually on outgoing path.
1746 */
Patrick McHardy84fa7932006-08-29 16:44:56 -07001747int skb_checksum_help(struct sk_buff *skb)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001748{
Al Virod3bc23e2006-11-14 21:24:49 -08001749 __wsum csum;
Herbert Xu663ead32007-04-09 11:59:07 -07001750 int ret = 0, offset;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001751
Patrick McHardy84fa7932006-08-29 16:44:56 -07001752 if (skb->ip_summed == CHECKSUM_COMPLETE)
Herbert Xua430a432006-07-08 13:34:56 -07001753 goto out_set_summed;
1754
1755 if (unlikely(skb_shinfo(skb)->gso_size)) {
Herbert Xua430a432006-07-08 13:34:56 -07001756 /* Let GSO fix up the checksum. */
1757 goto out_set_summed;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001758 }
1759
Herbert Xua0308472007-10-15 01:47:15 -07001760 offset = skb->csum_start - skb_headroom(skb);
1761 BUG_ON(offset >= skb_headlen(skb));
1762 csum = skb_checksum(skb, offset, skb->len - offset, 0);
1763
1764 offset += skb->csum_offset;
1765 BUG_ON(offset + sizeof(__sum16) > skb_headlen(skb));
1766
1767 if (skb_cloned(skb) &&
1768 !skb_clone_writable(skb, offset + sizeof(__sum16))) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001769 ret = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
1770 if (ret)
1771 goto out;
1772 }
1773
Herbert Xua0308472007-10-15 01:47:15 -07001774 *(__sum16 *)(skb->data + offset) = csum_fold(csum);
Herbert Xua430a432006-07-08 13:34:56 -07001775out_set_summed:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001776 skb->ip_summed = CHECKSUM_NONE;
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09001777out:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001778 return ret;
1779}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001780EXPORT_SYMBOL(skb_checksum_help);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001781
Herbert Xuf6a78bf2006-06-22 02:57:17 -07001782/**
1783 * skb_gso_segment - Perform segmentation on skb.
1784 * @skb: buffer to segment
Herbert Xu576a30e2006-06-27 13:22:38 -07001785 * @features: features for the output path (see dev->features)
Herbert Xuf6a78bf2006-06-22 02:57:17 -07001786 *
1787 * This function segments the given skb and returns a list of segments.
Herbert Xu576a30e2006-06-27 13:22:38 -07001788 *
1789 * It may return NULL if the skb requires no segmentation. This is
1790 * only possible when GSO is used for verifying header integrity.
Herbert Xuf6a78bf2006-06-22 02:57:17 -07001791 */
Herbert Xu576a30e2006-06-27 13:22:38 -07001792struct sk_buff *skb_gso_segment(struct sk_buff *skb, int features)
Herbert Xuf6a78bf2006-06-22 02:57:17 -07001793{
1794 struct sk_buff *segs = ERR_PTR(-EPROTONOSUPPORT);
1795 struct packet_type *ptype;
Al Viro252e3342006-11-14 20:48:11 -08001796 __be16 type = skb->protocol;
Herbert Xua430a432006-07-08 13:34:56 -07001797 int err;
Herbert Xuf6a78bf2006-06-22 02:57:17 -07001798
Jesse Gross7b9c6092010-10-20 13:56:04 +00001799 if (type == htons(ETH_P_8021Q)) {
1800 struct vlan_ethhdr *veh;
1801
1802 if (unlikely(!pskb_may_pull(skb, VLAN_ETH_HLEN)))
1803 return ERR_PTR(-EINVAL);
1804
1805 veh = (struct vlan_ethhdr *)skb->data;
1806 type = veh->h_vlan_encapsulated_proto;
1807 }
1808
Arnaldo Carvalho de Melo459a98e2007-03-19 15:30:44 -07001809 skb_reset_mac_header(skb);
Arnaldo Carvalho de Melob0e380b2007-04-10 21:21:55 -07001810 skb->mac_len = skb->network_header - skb->mac_header;
Herbert Xuf6a78bf2006-06-22 02:57:17 -07001811 __skb_pull(skb, skb->mac_len);
1812
Herbert Xu67fd1a72009-01-19 16:26:44 -08001813 if (unlikely(skb->ip_summed != CHECKSUM_PARTIAL)) {
1814 struct net_device *dev = skb->dev;
1815 struct ethtool_drvinfo info = {};
1816
1817 if (dev && dev->ethtool_ops && dev->ethtool_ops->get_drvinfo)
1818 dev->ethtool_ops->get_drvinfo(dev, &info);
1819
1820 WARN(1, "%s: caps=(0x%lx, 0x%lx) len=%d data_len=%d "
1821 "ip_summed=%d",
1822 info.driver, dev ? dev->features : 0L,
1823 skb->sk ? skb->sk->sk_route_caps : 0L,
1824 skb->len, skb->data_len, skb->ip_summed);
1825
Herbert Xua430a432006-07-08 13:34:56 -07001826 if (skb_header_cloned(skb) &&
1827 (err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC)))
1828 return ERR_PTR(err);
1829 }
1830
Herbert Xuf6a78bf2006-06-22 02:57:17 -07001831 rcu_read_lock();
Pavel Emelyanov82d8a8672007-11-26 20:12:58 +08001832 list_for_each_entry_rcu(ptype,
1833 &ptype_base[ntohs(type) & PTYPE_HASH_MASK], list) {
Herbert Xuf6a78bf2006-06-22 02:57:17 -07001834 if (ptype->type == type && !ptype->dev && ptype->gso_segment) {
Patrick McHardy84fa7932006-08-29 16:44:56 -07001835 if (unlikely(skb->ip_summed != CHECKSUM_PARTIAL)) {
Herbert Xua430a432006-07-08 13:34:56 -07001836 err = ptype->gso_send_check(skb);
1837 segs = ERR_PTR(err);
1838 if (err || skb_gso_ok(skb, features))
1839 break;
Arnaldo Carvalho de Melod56f90a2007-04-10 20:50:43 -07001840 __skb_push(skb, (skb->data -
1841 skb_network_header(skb)));
Herbert Xua430a432006-07-08 13:34:56 -07001842 }
Herbert Xu576a30e2006-06-27 13:22:38 -07001843 segs = ptype->gso_segment(skb, features);
Herbert Xuf6a78bf2006-06-22 02:57:17 -07001844 break;
1845 }
1846 }
1847 rcu_read_unlock();
1848
Arnaldo Carvalho de Melo98e399f2007-03-19 15:33:04 -07001849 __skb_push(skb, skb->data - skb_mac_header(skb));
Herbert Xu576a30e2006-06-27 13:22:38 -07001850
Herbert Xuf6a78bf2006-06-22 02:57:17 -07001851 return segs;
1852}
Herbert Xuf6a78bf2006-06-22 02:57:17 -07001853EXPORT_SYMBOL(skb_gso_segment);
1854
Herbert Xufb286bb2005-11-10 13:01:24 -08001855/* Take action when hardware reception checksum errors are detected. */
1856#ifdef CONFIG_BUG
1857void netdev_rx_csum_fault(struct net_device *dev)
1858{
1859 if (net_ratelimit()) {
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09001860 printk(KERN_ERR "%s: hw csum failure.\n",
Stephen Hemminger246a4212005-12-08 15:21:39 -08001861 dev ? dev->name : "<unknown>");
Herbert Xufb286bb2005-11-10 13:01:24 -08001862 dump_stack();
1863 }
1864}
1865EXPORT_SYMBOL(netdev_rx_csum_fault);
1866#endif
1867
Linus Torvalds1da177e2005-04-16 15:20:36 -07001868/* Actually, we should eliminate this check as soon as we know, that:
1869 * 1. IOMMU is present and allows to map all the memory.
1870 * 2. No high memory really exists on this machine.
1871 */
1872
Eric Dumazet9092c652010-04-02 13:34:49 -07001873static int illegal_highdma(struct net_device *dev, struct sk_buff *skb)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001874{
Herbert Xu3d3a8532006-06-27 13:33:10 -07001875#ifdef CONFIG_HIGHMEM
Linus Torvalds1da177e2005-04-16 15:20:36 -07001876 int i;
FUJITA Tomonori5acbbd42010-03-30 22:35:50 +00001877 if (!(dev->features & NETIF_F_HIGHDMA)) {
1878 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++)
1879 if (PageHighMem(skb_shinfo(skb)->frags[i].page))
1880 return 1;
1881 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001882
FUJITA Tomonori5acbbd42010-03-30 22:35:50 +00001883 if (PCI_DMA_BUS_IS_PHYS) {
1884 struct device *pdev = dev->dev.parent;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001885
Eric Dumazet9092c652010-04-02 13:34:49 -07001886 if (!pdev)
1887 return 0;
FUJITA Tomonori5acbbd42010-03-30 22:35:50 +00001888 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
1889 dma_addr_t addr = page_to_phys(skb_shinfo(skb)->frags[i].page);
1890 if (!pdev->dma_mask || addr + PAGE_SIZE - 1 > *pdev->dma_mask)
1891 return 1;
1892 }
1893 }
Herbert Xu3d3a8532006-06-27 13:33:10 -07001894#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07001895 return 0;
1896}
Linus Torvalds1da177e2005-04-16 15:20:36 -07001897
Herbert Xuf6a78bf2006-06-22 02:57:17 -07001898struct dev_gso_cb {
1899 void (*destructor)(struct sk_buff *skb);
1900};
1901
1902#define DEV_GSO_CB(skb) ((struct dev_gso_cb *)(skb)->cb)
1903
1904static void dev_gso_skb_destructor(struct sk_buff *skb)
1905{
1906 struct dev_gso_cb *cb;
1907
1908 do {
1909 struct sk_buff *nskb = skb->next;
1910
1911 skb->next = nskb->next;
1912 nskb->next = NULL;
1913 kfree_skb(nskb);
1914 } while (skb->next);
1915
1916 cb = DEV_GSO_CB(skb);
1917 if (cb->destructor)
1918 cb->destructor(skb);
1919}
1920
1921/**
1922 * dev_gso_segment - Perform emulated hardware segmentation on skb.
1923 * @skb: buffer to segment
1924 *
1925 * This function segments the given skb and stores the list of segments
1926 * in skb->next.
1927 */
1928static int dev_gso_segment(struct sk_buff *skb)
1929{
1930 struct net_device *dev = skb->dev;
1931 struct sk_buff *segs;
Herbert Xu576a30e2006-06-27 13:22:38 -07001932 int features = dev->features & ~(illegal_highdma(dev, skb) ?
1933 NETIF_F_SG : 0);
Herbert Xuf6a78bf2006-06-22 02:57:17 -07001934
Herbert Xu576a30e2006-06-27 13:22:38 -07001935 segs = skb_gso_segment(skb, features);
1936
1937 /* Verifying header integrity only. */
1938 if (!segs)
1939 return 0;
1940
Hirofumi Nakagawa801678c2008-04-29 01:03:09 -07001941 if (IS_ERR(segs))
Herbert Xuf6a78bf2006-06-22 02:57:17 -07001942 return PTR_ERR(segs);
1943
1944 skb->next = segs;
1945 DEV_GSO_CB(skb)->destructor = skb->destructor;
1946 skb->destructor = dev_gso_skb_destructor;
1947
1948 return 0;
1949}
1950
Eric Dumazetfc6055a2010-04-16 12:18:22 +00001951/*
1952 * Try to orphan skb early, right before transmission by the device.
Oliver Hartkopp2244d072010-08-17 08:59:14 +00001953 * We cannot orphan skb if tx timestamp is requested or the sk-reference
1954 * is needed on driver level for other reasons, e.g. see net/can/raw.c
Eric Dumazetfc6055a2010-04-16 12:18:22 +00001955 */
1956static inline void skb_orphan_try(struct sk_buff *skb)
1957{
Eric Dumazet87fd3082010-07-13 05:24:20 +00001958 struct sock *sk = skb->sk;
1959
Oliver Hartkopp2244d072010-08-17 08:59:14 +00001960 if (sk && !skb_shinfo(skb)->tx_flags) {
Eric Dumazet87fd3082010-07-13 05:24:20 +00001961 /* skb_tx_hash() wont be able to get sk.
1962 * We copy sk_hash into skb->rxhash
1963 */
1964 if (!skb->rxhash)
1965 skb->rxhash = sk->sk_hash;
Eric Dumazetfc6055a2010-04-16 12:18:22 +00001966 skb_orphan(skb);
Eric Dumazet87fd3082010-07-13 05:24:20 +00001967 }
Eric Dumazetfc6055a2010-04-16 12:18:22 +00001968}
1969
John Fastabend6afff0c2010-06-16 14:18:12 +00001970/*
1971 * Returns true if either:
1972 * 1. skb has frag_list and the device doesn't support FRAGLIST, or
1973 * 2. skb is fragmented and the device does not support SG, or if
1974 * at least one of fragments is in highmem and device does not
1975 * support DMA from it.
1976 */
1977static inline int skb_needs_linearize(struct sk_buff *skb,
1978 struct net_device *dev)
1979{
Jesse Gross7b9c6092010-10-20 13:56:04 +00001980 int features = dev->features;
1981
1982 if (skb->protocol == htons(ETH_P_8021Q) || vlan_tx_tag_present(skb))
1983 features &= dev->vlan_features;
1984
John Fastabend6afff0c2010-06-16 14:18:12 +00001985 return skb_is_nonlinear(skb) &&
Jesse Gross7b9c6092010-10-20 13:56:04 +00001986 ((skb_has_frag_list(skb) && !(features & NETIF_F_FRAGLIST)) ||
1987 (skb_shinfo(skb)->nr_frags && (!(features & NETIF_F_SG) ||
John Fastabend6afff0c2010-06-16 14:18:12 +00001988 illegal_highdma(dev, skb))));
1989}
1990
David S. Millerfd2ea0a2008-07-17 01:56:23 -07001991int dev_hard_start_xmit(struct sk_buff *skb, struct net_device *dev,
1992 struct netdev_queue *txq)
Herbert Xuf6a78bf2006-06-22 02:57:17 -07001993{
Stephen Hemminger00829822008-11-20 20:14:53 -08001994 const struct net_device_ops *ops = dev->netdev_ops;
Patrick McHardy572a9d72009-11-10 06:14:14 +00001995 int rc = NETDEV_TX_OK;
Stephen Hemminger00829822008-11-20 20:14:53 -08001996
Herbert Xuf6a78bf2006-06-22 02:57:17 -07001997 if (likely(!skb->next)) {
Stephen Hemminger9be9a6b2007-04-20 17:02:45 -07001998 if (!list_empty(&ptype_all))
Herbert Xuf6a78bf2006-06-22 02:57:17 -07001999 dev_queue_xmit_nit(skb, dev);
2000
Eric Dumazet93f154b2009-05-18 22:19:19 -07002001 /*
2002 * If device doesnt need skb->dst, release it right now while
2003 * its hot in this cpu cache
2004 */
Eric Dumazetadf30902009-06-02 05:19:30 +00002005 if (dev->priv_flags & IFF_XMIT_DST_RELEASE)
2006 skb_dst_drop(skb);
2007
Eric Dumazetfc6055a2010-04-16 12:18:22 +00002008 skb_orphan_try(skb);
David S. Miller9ccb8972010-04-22 01:02:07 -07002009
Jesse Gross7b9c6092010-10-20 13:56:04 +00002010 if (vlan_tx_tag_present(skb) &&
2011 !(dev->features & NETIF_F_HW_VLAN_TX)) {
2012 skb = __vlan_put_tag(skb, vlan_tx_tag_get(skb));
2013 if (unlikely(!skb))
2014 goto out;
2015
2016 skb->vlan_tci = 0;
2017 }
2018
David S. Miller9ccb8972010-04-22 01:02:07 -07002019 if (netif_needs_gso(dev, skb)) {
2020 if (unlikely(dev_gso_segment(skb)))
2021 goto out_kfree_skb;
2022 if (skb->next)
2023 goto gso;
John Fastabend6afff0c2010-06-16 14:18:12 +00002024 } else {
2025 if (skb_needs_linearize(skb, dev) &&
2026 __skb_linearize(skb))
2027 goto out_kfree_skb;
2028
2029 /* If packet is not checksummed and device does not
2030 * support checksumming for this protocol, complete
2031 * checksumming here.
2032 */
2033 if (skb->ip_summed == CHECKSUM_PARTIAL) {
2034 skb_set_transport_header(skb, skb->csum_start -
2035 skb_headroom(skb));
2036 if (!dev_can_checksum(dev, skb) &&
2037 skb_checksum_help(skb))
2038 goto out_kfree_skb;
2039 }
David S. Miller9ccb8972010-04-22 01:02:07 -07002040 }
2041
Patrick Ohlyac45f602009-02-12 05:03:37 +00002042 rc = ops->ndo_start_xmit(skb, dev);
Koki Sanagicf66ba52010-08-23 18:45:02 +09002043 trace_net_dev_xmit(skb, rc);
Patrick McHardyec634fe2009-07-05 19:23:38 -07002044 if (rc == NETDEV_TX_OK)
Eric Dumazet08baf562009-05-25 22:58:01 -07002045 txq_trans_update(txq);
Patrick Ohlyac45f602009-02-12 05:03:37 +00002046 return rc;
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002047 }
2048
Herbert Xu576a30e2006-06-27 13:22:38 -07002049gso:
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002050 do {
2051 struct sk_buff *nskb = skb->next;
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002052
2053 skb->next = nskb->next;
2054 nskb->next = NULL;
Krishna Kumar068a2de2009-12-09 20:59:58 +00002055
2056 /*
2057 * If device doesnt need nskb->dst, release it right now while
2058 * its hot in this cpu cache
2059 */
2060 if (dev->priv_flags & IFF_XMIT_DST_RELEASE)
2061 skb_dst_drop(nskb);
2062
Stephen Hemminger00829822008-11-20 20:14:53 -08002063 rc = ops->ndo_start_xmit(nskb, dev);
Koki Sanagicf66ba52010-08-23 18:45:02 +09002064 trace_net_dev_xmit(nskb, rc);
Patrick McHardyec634fe2009-07-05 19:23:38 -07002065 if (unlikely(rc != NETDEV_TX_OK)) {
Patrick McHardy572a9d72009-11-10 06:14:14 +00002066 if (rc & ~NETDEV_TX_MASK)
2067 goto out_kfree_gso_skb;
Michael Chanf54d9e82006-06-25 23:57:04 -07002068 nskb->next = skb->next;
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002069 skb->next = nskb;
2070 return rc;
2071 }
Eric Dumazet08baf562009-05-25 22:58:01 -07002072 txq_trans_update(txq);
David S. Millerfd2ea0a2008-07-17 01:56:23 -07002073 if (unlikely(netif_tx_queue_stopped(txq) && skb->next))
Michael Chanf54d9e82006-06-25 23:57:04 -07002074 return NETDEV_TX_BUSY;
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002075 } while (skb->next);
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09002076
Patrick McHardy572a9d72009-11-10 06:14:14 +00002077out_kfree_gso_skb:
2078 if (likely(skb->next == NULL))
2079 skb->destructor = DEV_GSO_CB(skb)->destructor;
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002080out_kfree_skb:
2081 kfree_skb(skb);
Jesse Gross7b9c6092010-10-20 13:56:04 +00002082out:
Patrick McHardy572a9d72009-11-10 06:14:14 +00002083 return rc;
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002084}
2085
Tom Herbert0a9627f2010-03-16 08:03:29 +00002086static u32 hashrnd __read_mostly;
David S. Millerb6b2fed2008-07-21 09:48:06 -07002087
Stephen Hemminger92477442009-03-21 13:39:26 -07002088u16 skb_tx_hash(const struct net_device *dev, const struct sk_buff *skb)
David S. Miller8f0f2222008-07-15 03:47:03 -07002089{
David S. Miller70192982009-01-27 16:34:47 -08002090 u32 hash;
David S. Millerb6b2fed2008-07-21 09:48:06 -07002091
David S. Miller513de112009-05-03 14:43:10 -07002092 if (skb_rx_queue_recorded(skb)) {
2093 hash = skb_get_rx_queue(skb);
Eric Dumazetd1b19df2009-09-03 01:29:39 -07002094 while (unlikely(hash >= dev->real_num_tx_queues))
David S. Miller513de112009-05-03 14:43:10 -07002095 hash -= dev->real_num_tx_queues;
2096 return hash;
2097 }
Eric Dumazetec581f62009-05-01 09:05:06 -07002098
2099 if (skb->sk && skb->sk->sk_hash)
David S. Miller70192982009-01-27 16:34:47 -08002100 hash = skb->sk->sk_hash;
Eric Dumazetec581f62009-05-01 09:05:06 -07002101 else
Eric Dumazet87fd3082010-07-13 05:24:20 +00002102 hash = (__force u16) skb->protocol ^ skb->rxhash;
Tom Herbert0a9627f2010-03-16 08:03:29 +00002103 hash = jhash_1word(hash, hashrnd);
David S. Millerd5a9e242009-01-27 16:22:11 -08002104
David S. Millerb6b2fed2008-07-21 09:48:06 -07002105 return (u16) (((u64) hash * dev->real_num_tx_queues) >> 32);
David S. Miller8f0f2222008-07-15 03:47:03 -07002106}
Stephen Hemminger92477442009-03-21 13:39:26 -07002107EXPORT_SYMBOL(skb_tx_hash);
David S. Miller8f0f2222008-07-15 03:47:03 -07002108
Eric Dumazeted046422009-11-13 21:54:04 +00002109static inline u16 dev_cap_txqueue(struct net_device *dev, u16 queue_index)
2110{
2111 if (unlikely(queue_index >= dev->real_num_tx_queues)) {
2112 if (net_ratelimit()) {
Eric Dumazet7a161ea2010-04-08 21:26:13 +00002113 pr_warning("%s selects TX queue %d, but "
2114 "real number of TX queues is %d\n",
2115 dev->name, queue_index, dev->real_num_tx_queues);
Eric Dumazeted046422009-11-13 21:54:04 +00002116 }
2117 return 0;
2118 }
2119 return queue_index;
2120}
2121
David S. Millere8a04642008-07-17 00:34:19 -07002122static struct netdev_queue *dev_pick_tx(struct net_device *dev,
2123 struct sk_buff *skb)
2124{
Tom Herbertb0f77d02010-07-14 20:50:29 -07002125 int queue_index;
Helmut Schaadeabc772010-09-03 02:39:56 +00002126 const struct net_device_ops *ops = dev->netdev_ops;
David S. Millerfd2ea0a2008-07-17 01:56:23 -07002127
Helmut Schaadeabc772010-09-03 02:39:56 +00002128 if (ops->ndo_select_queue) {
2129 queue_index = ops->ndo_select_queue(dev, skb);
2130 queue_index = dev_cap_txqueue(dev, queue_index);
2131 } else {
2132 struct sock *sk = skb->sk;
2133 queue_index = sk_tx_queue_get(sk);
Tom Herbertdf32cc12010-11-01 12:55:52 -07002134 if (queue_index < 0 || queue_index >= dev->real_num_tx_queues) {
Krishna Kumara4ee3ce2009-10-19 23:50:07 +00002135
Krishna Kumara4ee3ce2009-10-19 23:50:07 +00002136 queue_index = 0;
2137 if (dev->real_num_tx_queues > 1)
2138 queue_index = skb_tx_hash(dev, skb);
2139
Eric Dumazet8728c542010-04-11 21:18:17 +00002140 if (sk) {
David S. Miller87eb3672010-04-21 01:14:25 -07002141 struct dst_entry *dst = rcu_dereference_check(sk->sk_dst_cache, 1);
Eric Dumazet8728c542010-04-11 21:18:17 +00002142
2143 if (dst && skb_dst(skb) == dst)
2144 sk_tx_queue_set(sk, queue_index);
2145 }
Krishna Kumara4ee3ce2009-10-19 23:50:07 +00002146 }
2147 }
David S. Millereae792b2008-07-15 03:03:33 -07002148
David S. Millerfd2ea0a2008-07-17 01:56:23 -07002149 skb_set_queue_mapping(skb, queue_index);
2150 return netdev_get_tx_queue(dev, queue_index);
David S. Millere8a04642008-07-17 00:34:19 -07002151}
2152
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00002153static inline int __dev_xmit_skb(struct sk_buff *skb, struct Qdisc *q,
2154 struct net_device *dev,
2155 struct netdev_queue *txq)
2156{
2157 spinlock_t *root_lock = qdisc_lock(q);
Eric Dumazet79640a42010-06-02 05:09:29 -07002158 bool contended = qdisc_is_running(q);
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00002159 int rc;
2160
Eric Dumazet79640a42010-06-02 05:09:29 -07002161 /*
2162 * Heuristic to force contended enqueues to serialize on a
2163 * separate lock before trying to get qdisc main lock.
2164 * This permits __QDISC_STATE_RUNNING owner to get the lock more often
2165 * and dequeue packets faster.
2166 */
2167 if (unlikely(contended))
2168 spin_lock(&q->busylock);
2169
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00002170 spin_lock(root_lock);
2171 if (unlikely(test_bit(__QDISC_STATE_DEACTIVATED, &q->state))) {
2172 kfree_skb(skb);
2173 rc = NET_XMIT_DROP;
2174 } else if ((q->flags & TCQ_F_CAN_BYPASS) && !qdisc_qlen(q) &&
Eric Dumazetbc135b22010-06-02 03:23:51 -07002175 qdisc_run_begin(q)) {
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00002176 /*
2177 * This is a work-conserving queue; there are no old skbs
2178 * waiting to be sent out; and the qdisc is not running -
2179 * xmit the skb directly.
2180 */
Eric Dumazet7fee2262010-05-11 23:19:48 +00002181 if (!(dev->priv_flags & IFF_XMIT_DST_RELEASE))
2182 skb_dst_force(skb);
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00002183 __qdisc_update_bstats(q, skb->len);
Eric Dumazet79640a42010-06-02 05:09:29 -07002184 if (sch_direct_xmit(skb, q, dev, txq, root_lock)) {
2185 if (unlikely(contended)) {
2186 spin_unlock(&q->busylock);
2187 contended = false;
2188 }
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00002189 __qdisc_run(q);
Eric Dumazet79640a42010-06-02 05:09:29 -07002190 } else
Eric Dumazetbc135b22010-06-02 03:23:51 -07002191 qdisc_run_end(q);
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00002192
2193 rc = NET_XMIT_SUCCESS;
2194 } else {
Eric Dumazet7fee2262010-05-11 23:19:48 +00002195 skb_dst_force(skb);
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00002196 rc = qdisc_enqueue_root(skb, q);
Eric Dumazet79640a42010-06-02 05:09:29 -07002197 if (qdisc_run_begin(q)) {
2198 if (unlikely(contended)) {
2199 spin_unlock(&q->busylock);
2200 contended = false;
2201 }
2202 __qdisc_run(q);
2203 }
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00002204 }
2205 spin_unlock(root_lock);
Eric Dumazet79640a42010-06-02 05:09:29 -07002206 if (unlikely(contended))
2207 spin_unlock(&q->busylock);
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00002208 return rc;
2209}
2210
Eric Dumazet745e20f2010-09-29 13:23:09 -07002211static DEFINE_PER_CPU(int, xmit_recursion);
David S. Miller11a766c2010-10-25 12:51:55 -07002212#define RECURSION_LIMIT 10
Eric Dumazet745e20f2010-09-29 13:23:09 -07002213
Dave Jonesd29f7492008-07-22 14:09:06 -07002214/**
2215 * dev_queue_xmit - transmit a buffer
2216 * @skb: buffer to transmit
2217 *
2218 * Queue a buffer for transmission to a network device. The caller must
2219 * have set the device and priority and built the buffer before calling
2220 * this function. The function can be called from an interrupt.
2221 *
2222 * A negative errno code is returned on a failure. A success does not
2223 * guarantee the frame will be transmitted as it may be dropped due
2224 * to congestion or traffic shaping.
2225 *
2226 * -----------------------------------------------------------------------------------
2227 * I notice this method can also return errors from the queue disciplines,
2228 * including NET_XMIT_DROP, which is a positive value. So, errors can also
2229 * be positive.
2230 *
2231 * Regardless of the return value, the skb is consumed, so it is currently
2232 * difficult to retry a send to this method. (You can bump the ref count
2233 * before sending to hold a reference for retry if you are careful.)
2234 *
2235 * When calling this method, interrupts MUST be enabled. This is because
2236 * the BH enable code must have IRQs enabled so that it will not deadlock.
2237 * --BLG
2238 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07002239int dev_queue_xmit(struct sk_buff *skb)
2240{
2241 struct net_device *dev = skb->dev;
David S. Millerdc2b4842008-07-08 17:18:23 -07002242 struct netdev_queue *txq;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002243 struct Qdisc *q;
2244 int rc = -ENOMEM;
2245
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09002246 /* Disable soft irqs for various locks below. Also
2247 * stops preemption for RCU.
Linus Torvalds1da177e2005-04-16 15:20:36 -07002248 */
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09002249 rcu_read_lock_bh();
Linus Torvalds1da177e2005-04-16 15:20:36 -07002250
David S. Millereae792b2008-07-15 03:03:33 -07002251 txq = dev_pick_tx(dev, skb);
Paul E. McKenneya898def2010-02-22 17:04:49 -08002252 q = rcu_dereference_bh(txq->qdisc);
David S. Miller37437bb2008-07-16 02:15:04 -07002253
Linus Torvalds1da177e2005-04-16 15:20:36 -07002254#ifdef CONFIG_NET_CLS_ACT
Eric Dumazetd1b19df2009-09-03 01:29:39 -07002255 skb->tc_verd = SET_TC_AT(skb->tc_verd, AT_EGRESS);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002256#endif
Koki Sanagicf66ba52010-08-23 18:45:02 +09002257 trace_net_dev_queue(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002258 if (q->enqueue) {
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00002259 rc = __dev_xmit_skb(skb, q, dev, txq);
David S. Miller37437bb2008-07-16 02:15:04 -07002260 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002261 }
2262
2263 /* The device has no queue. Common case for software devices:
2264 loopback, all the sorts of tunnels...
2265
Herbert Xu932ff272006-06-09 12:20:56 -07002266 Really, it is unlikely that netif_tx_lock protection is necessary
2267 here. (f.e. loopback and IP tunnels are clean ignoring statistics
Linus Torvalds1da177e2005-04-16 15:20:36 -07002268 counters.)
2269 However, it is possible, that they rely on protection
2270 made by us here.
2271
2272 Check this and shot the lock. It is not prone from deadlocks.
2273 Either shot noqueue qdisc, it is even simpler 8)
2274 */
2275 if (dev->flags & IFF_UP) {
2276 int cpu = smp_processor_id(); /* ok because BHs are off */
2277
David S. Millerc773e842008-07-08 23:13:53 -07002278 if (txq->xmit_lock_owner != cpu) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07002279
Eric Dumazet745e20f2010-09-29 13:23:09 -07002280 if (__this_cpu_read(xmit_recursion) > RECURSION_LIMIT)
2281 goto recursion_alert;
2282
David S. Millerc773e842008-07-08 23:13:53 -07002283 HARD_TX_LOCK(dev, txq, cpu);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002284
David S. Millerfd2ea0a2008-07-17 01:56:23 -07002285 if (!netif_tx_queue_stopped(txq)) {
Eric Dumazet745e20f2010-09-29 13:23:09 -07002286 __this_cpu_inc(xmit_recursion);
Patrick McHardy572a9d72009-11-10 06:14:14 +00002287 rc = dev_hard_start_xmit(skb, dev, txq);
Eric Dumazet745e20f2010-09-29 13:23:09 -07002288 __this_cpu_dec(xmit_recursion);
Patrick McHardy572a9d72009-11-10 06:14:14 +00002289 if (dev_xmit_complete(rc)) {
David S. Millerc773e842008-07-08 23:13:53 -07002290 HARD_TX_UNLOCK(dev, txq);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002291 goto out;
2292 }
2293 }
David S. Millerc773e842008-07-08 23:13:53 -07002294 HARD_TX_UNLOCK(dev, txq);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002295 if (net_ratelimit())
2296 printk(KERN_CRIT "Virtual device %s asks to "
2297 "queue packet!\n", dev->name);
2298 } else {
2299 /* Recursion is detected! It is possible,
Eric Dumazet745e20f2010-09-29 13:23:09 -07002300 * unfortunately
2301 */
2302recursion_alert:
Linus Torvalds1da177e2005-04-16 15:20:36 -07002303 if (net_ratelimit())
2304 printk(KERN_CRIT "Dead loop on virtual device "
2305 "%s, fix it urgently!\n", dev->name);
2306 }
2307 }
2308
2309 rc = -ENETDOWN;
Herbert Xud4828d82006-06-22 02:28:18 -07002310 rcu_read_unlock_bh();
Linus Torvalds1da177e2005-04-16 15:20:36 -07002311
Linus Torvalds1da177e2005-04-16 15:20:36 -07002312 kfree_skb(skb);
2313 return rc;
2314out:
Herbert Xud4828d82006-06-22 02:28:18 -07002315 rcu_read_unlock_bh();
Linus Torvalds1da177e2005-04-16 15:20:36 -07002316 return rc;
2317}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07002318EXPORT_SYMBOL(dev_queue_xmit);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002319
2320
2321/*=======================================================================
2322 Receiver routines
2323 =======================================================================*/
2324
Stephen Hemminger6b2bedc2007-03-12 14:33:50 -07002325int netdev_max_backlog __read_mostly = 1000;
Eric Dumazet3b098e22010-05-15 23:57:10 -07002326int netdev_tstamp_prequeue __read_mostly = 1;
Stephen Hemminger6b2bedc2007-03-12 14:33:50 -07002327int netdev_budget __read_mostly = 300;
2328int weight_p __read_mostly = 64; /* old backlog weight */
Linus Torvalds1da177e2005-04-16 15:20:36 -07002329
Eric Dumazeteecfd7c2010-05-06 22:07:48 -07002330/* Called with irq disabled */
2331static inline void ____napi_schedule(struct softnet_data *sd,
2332 struct napi_struct *napi)
2333{
2334 list_add_tail(&napi->poll_list, &sd->poll_list);
2335 __raise_softirq_irqoff(NET_RX_SOFTIRQ);
2336}
2337
Krishna Kumarbfb564e2010-08-04 06:15:52 +00002338/*
2339 * __skb_get_rxhash: calculate a flow hash based on src/dst addresses
2340 * and src/dst port numbers. Returns a non-zero hash number on success
2341 * and 0 on failure.
2342 */
2343__u32 __skb_get_rxhash(struct sk_buff *skb)
2344{
Changli Gao12fcdef2010-08-17 19:04:32 +00002345 int nhoff, hash = 0, poff;
Krishna Kumarbfb564e2010-08-04 06:15:52 +00002346 struct ipv6hdr *ip6;
2347 struct iphdr *ip;
2348 u8 ip_proto;
2349 u32 addr1, addr2, ihl;
2350 union {
2351 u32 v32;
2352 u16 v16[2];
2353 } ports;
2354
2355 nhoff = skb_network_offset(skb);
2356
2357 switch (skb->protocol) {
2358 case __constant_htons(ETH_P_IP):
2359 if (!pskb_may_pull(skb, sizeof(*ip) + nhoff))
2360 goto done;
2361
Changli Gao10034892010-08-21 06:13:28 +00002362 ip = (struct iphdr *) (skb->data + nhoff);
Changli Gaodbe57752010-08-17 19:01:38 +00002363 if (ip->frag_off & htons(IP_MF | IP_OFFSET))
2364 ip_proto = 0;
2365 else
2366 ip_proto = ip->protocol;
Krishna Kumarbfb564e2010-08-04 06:15:52 +00002367 addr1 = (__force u32) ip->saddr;
2368 addr2 = (__force u32) ip->daddr;
2369 ihl = ip->ihl;
2370 break;
2371 case __constant_htons(ETH_P_IPV6):
2372 if (!pskb_may_pull(skb, sizeof(*ip6) + nhoff))
2373 goto done;
2374
Changli Gao10034892010-08-21 06:13:28 +00002375 ip6 = (struct ipv6hdr *) (skb->data + nhoff);
Krishna Kumarbfb564e2010-08-04 06:15:52 +00002376 ip_proto = ip6->nexthdr;
2377 addr1 = (__force u32) ip6->saddr.s6_addr32[3];
2378 addr2 = (__force u32) ip6->daddr.s6_addr32[3];
2379 ihl = (40 >> 2);
2380 break;
2381 default:
2382 goto done;
2383 }
2384
Changli Gao12fcdef2010-08-17 19:04:32 +00002385 ports.v32 = 0;
2386 poff = proto_ports_offset(ip_proto);
2387 if (poff >= 0) {
2388 nhoff += ihl * 4 + poff;
2389 if (pskb_may_pull(skb, nhoff + 4)) {
2390 ports.v32 = * (__force u32 *) (skb->data + nhoff);
Krishna Kumarbfb564e2010-08-04 06:15:52 +00002391 if (ports.v16[1] < ports.v16[0])
2392 swap(ports.v16[0], ports.v16[1]);
Krishna Kumarbfb564e2010-08-04 06:15:52 +00002393 }
Krishna Kumarbfb564e2010-08-04 06:15:52 +00002394 }
2395
2396 /* get a consistent hash (same value on both flow directions) */
2397 if (addr2 < addr1)
2398 swap(addr1, addr2);
2399
2400 hash = jhash_3words(addr1, addr2, ports.v32, hashrnd);
2401 if (!hash)
2402 hash = 1;
2403
2404done:
2405 return hash;
2406}
2407EXPORT_SYMBOL(__skb_get_rxhash);
2408
Eric Dumazetdf334542010-03-24 19:13:54 +00002409#ifdef CONFIG_RPS
Tom Herbertfec5e652010-04-16 16:01:27 -07002410
2411/* One global table that all flow-based protocols share. */
Eric Dumazet6e3f7fa2010-10-25 03:02:02 +00002412struct rps_sock_flow_table __rcu *rps_sock_flow_table __read_mostly;
Tom Herbertfec5e652010-04-16 16:01:27 -07002413EXPORT_SYMBOL(rps_sock_flow_table);
2414
Tom Herbert0a9627f2010-03-16 08:03:29 +00002415/*
2416 * get_rps_cpu is called from netif_receive_skb and returns the target
2417 * CPU from the RPS map of the receiving queue for a given skb.
Eric Dumazetb0e28f12010-04-15 00:14:07 -07002418 * rcu_read_lock must be held on entry.
Tom Herbert0a9627f2010-03-16 08:03:29 +00002419 */
Tom Herbertfec5e652010-04-16 16:01:27 -07002420static int get_rps_cpu(struct net_device *dev, struct sk_buff *skb,
2421 struct rps_dev_flow **rflowp)
Tom Herbert0a9627f2010-03-16 08:03:29 +00002422{
Tom Herbert0a9627f2010-03-16 08:03:29 +00002423 struct netdev_rx_queue *rxqueue;
Eric Dumazet6e3f7fa2010-10-25 03:02:02 +00002424 struct rps_map *map;
Tom Herbertfec5e652010-04-16 16:01:27 -07002425 struct rps_dev_flow_table *flow_table;
2426 struct rps_sock_flow_table *sock_flow_table;
Tom Herbert0a9627f2010-03-16 08:03:29 +00002427 int cpu = -1;
Tom Herbertfec5e652010-04-16 16:01:27 -07002428 u16 tcpu;
Tom Herbert0a9627f2010-03-16 08:03:29 +00002429
Tom Herbert0a9627f2010-03-16 08:03:29 +00002430 if (skb_rx_queue_recorded(skb)) {
2431 u16 index = skb_get_rx_queue(skb);
Ben Hutchings62fe0b42010-09-27 08:24:33 +00002432 if (unlikely(index >= dev->real_num_rx_queues)) {
2433 WARN_ONCE(dev->real_num_rx_queues > 1,
2434 "%s received packet on queue %u, but number "
2435 "of RX queues is %u\n",
2436 dev->name, index, dev->real_num_rx_queues);
Tom Herbert0a9627f2010-03-16 08:03:29 +00002437 goto done;
2438 }
2439 rxqueue = dev->_rx + index;
2440 } else
2441 rxqueue = dev->_rx;
2442
Eric Dumazet6e3f7fa2010-10-25 03:02:02 +00002443 map = rcu_dereference(rxqueue->rps_map);
2444 if (map) {
2445 if (map->len == 1) {
Changli Gao6febfca2010-09-03 23:12:37 +00002446 tcpu = map->cpus[0];
2447 if (cpu_online(tcpu))
2448 cpu = tcpu;
Tom Herbert0a9627f2010-03-16 08:03:29 +00002449 goto done;
Eric Dumazetb249dcb2010-04-19 21:56:38 +00002450 }
Eric Dumazet6e3f7fa2010-10-25 03:02:02 +00002451 } else if (!rcu_dereference_raw(rxqueue->rps_flow_table)) {
Tom Herbert0a9627f2010-03-16 08:03:29 +00002452 goto done;
Tom Herbert0a9627f2010-03-16 08:03:29 +00002453 }
2454
Changli Gao2d47b452010-08-17 19:00:56 +00002455 skb_reset_network_header(skb);
Krishna Kumarbfb564e2010-08-04 06:15:52 +00002456 if (!skb_get_rxhash(skb))
Tom Herbert0a9627f2010-03-16 08:03:29 +00002457 goto done;
Tom Herbert0a9627f2010-03-16 08:03:29 +00002458
Tom Herbertfec5e652010-04-16 16:01:27 -07002459 flow_table = rcu_dereference(rxqueue->rps_flow_table);
2460 sock_flow_table = rcu_dereference(rps_sock_flow_table);
2461 if (flow_table && sock_flow_table) {
2462 u16 next_cpu;
2463 struct rps_dev_flow *rflow;
2464
2465 rflow = &flow_table->flows[skb->rxhash & flow_table->mask];
2466 tcpu = rflow->cpu;
2467
2468 next_cpu = sock_flow_table->ents[skb->rxhash &
2469 sock_flow_table->mask];
2470
2471 /*
2472 * If the desired CPU (where last recvmsg was done) is
2473 * different from current CPU (one in the rx-queue flow
2474 * table entry), switch if one of the following holds:
2475 * - Current CPU is unset (equal to RPS_NO_CPU).
2476 * - Current CPU is offline.
2477 * - The current CPU's queue tail has advanced beyond the
2478 * last packet that was enqueued using this table entry.
2479 * This guarantees that all previous packets for the flow
2480 * have been dequeued, thus preserving in order delivery.
2481 */
2482 if (unlikely(tcpu != next_cpu) &&
2483 (tcpu == RPS_NO_CPU || !cpu_online(tcpu) ||
2484 ((int)(per_cpu(softnet_data, tcpu).input_queue_head -
2485 rflow->last_qtail)) >= 0)) {
2486 tcpu = rflow->cpu = next_cpu;
2487 if (tcpu != RPS_NO_CPU)
2488 rflow->last_qtail = per_cpu(softnet_data,
2489 tcpu).input_queue_head;
2490 }
2491 if (tcpu != RPS_NO_CPU && cpu_online(tcpu)) {
2492 *rflowp = rflow;
2493 cpu = tcpu;
2494 goto done;
2495 }
2496 }
2497
Tom Herbert0a9627f2010-03-16 08:03:29 +00002498 if (map) {
Tom Herbertfec5e652010-04-16 16:01:27 -07002499 tcpu = map->cpus[((u64) skb->rxhash * map->len) >> 32];
Tom Herbert0a9627f2010-03-16 08:03:29 +00002500
2501 if (cpu_online(tcpu)) {
2502 cpu = tcpu;
2503 goto done;
2504 }
2505 }
2506
2507done:
Tom Herbert0a9627f2010-03-16 08:03:29 +00002508 return cpu;
2509}
2510
Tom Herbert0a9627f2010-03-16 08:03:29 +00002511/* Called from hardirq (IPI) context */
Eric Dumazete36fa2f2010-04-19 21:17:14 +00002512static void rps_trigger_softirq(void *data)
Tom Herbert0a9627f2010-03-16 08:03:29 +00002513{
Eric Dumazete36fa2f2010-04-19 21:17:14 +00002514 struct softnet_data *sd = data;
2515
Eric Dumazeteecfd7c2010-05-06 22:07:48 -07002516 ____napi_schedule(sd, &sd->backlog);
Changli Gaodee42872010-05-02 05:42:16 +00002517 sd->received_rps++;
Tom Herbert0a9627f2010-03-16 08:03:29 +00002518}
Eric Dumazete36fa2f2010-04-19 21:17:14 +00002519
Tom Herbertfec5e652010-04-16 16:01:27 -07002520#endif /* CONFIG_RPS */
Tom Herbert0a9627f2010-03-16 08:03:29 +00002521
2522/*
Eric Dumazete36fa2f2010-04-19 21:17:14 +00002523 * Check if this softnet_data structure is another cpu one
2524 * If yes, queue it to our IPI list and return 1
2525 * If no, return 0
2526 */
2527static int rps_ipi_queued(struct softnet_data *sd)
2528{
2529#ifdef CONFIG_RPS
2530 struct softnet_data *mysd = &__get_cpu_var(softnet_data);
2531
2532 if (sd != mysd) {
2533 sd->rps_ipi_next = mysd->rps_ipi_list;
2534 mysd->rps_ipi_list = sd;
2535
2536 __raise_softirq_irqoff(NET_RX_SOFTIRQ);
2537 return 1;
2538 }
2539#endif /* CONFIG_RPS */
2540 return 0;
2541}
2542
2543/*
Tom Herbert0a9627f2010-03-16 08:03:29 +00002544 * enqueue_to_backlog is called to queue an skb to a per CPU backlog
2545 * queue (may be a remote CPU queue).
2546 */
Tom Herbertfec5e652010-04-16 16:01:27 -07002547static int enqueue_to_backlog(struct sk_buff *skb, int cpu,
2548 unsigned int *qtail)
Tom Herbert0a9627f2010-03-16 08:03:29 +00002549{
Eric Dumazete36fa2f2010-04-19 21:17:14 +00002550 struct softnet_data *sd;
Tom Herbert0a9627f2010-03-16 08:03:29 +00002551 unsigned long flags;
2552
Eric Dumazete36fa2f2010-04-19 21:17:14 +00002553 sd = &per_cpu(softnet_data, cpu);
Tom Herbert0a9627f2010-03-16 08:03:29 +00002554
2555 local_irq_save(flags);
Tom Herbert0a9627f2010-03-16 08:03:29 +00002556
Eric Dumazete36fa2f2010-04-19 21:17:14 +00002557 rps_lock(sd);
Changli Gao6e7676c2010-04-27 15:07:33 -07002558 if (skb_queue_len(&sd->input_pkt_queue) <= netdev_max_backlog) {
2559 if (skb_queue_len(&sd->input_pkt_queue)) {
Tom Herbert0a9627f2010-03-16 08:03:29 +00002560enqueue:
Eric Dumazete36fa2f2010-04-19 21:17:14 +00002561 __skb_queue_tail(&sd->input_pkt_queue, skb);
Tom Herbert76cc8b12010-05-20 18:37:59 +00002562 input_queue_tail_incr_save(sd, qtail);
Eric Dumazete36fa2f2010-04-19 21:17:14 +00002563 rps_unlock(sd);
Changli Gao152102c2010-03-30 20:16:22 +00002564 local_irq_restore(flags);
Tom Herbert0a9627f2010-03-16 08:03:29 +00002565 return NET_RX_SUCCESS;
2566 }
2567
Eric Dumazetebda37c22010-05-06 23:51:21 +00002568 /* Schedule NAPI for backlog device
2569 * We can use non atomic operation since we own the queue lock
2570 */
2571 if (!__test_and_set_bit(NAPI_STATE_SCHED, &sd->backlog.state)) {
Eric Dumazete36fa2f2010-04-19 21:17:14 +00002572 if (!rps_ipi_queued(sd))
Eric Dumazeteecfd7c2010-05-06 22:07:48 -07002573 ____napi_schedule(sd, &sd->backlog);
Tom Herbert0a9627f2010-03-16 08:03:29 +00002574 }
2575 goto enqueue;
2576 }
2577
Changli Gaodee42872010-05-02 05:42:16 +00002578 sd->dropped++;
Eric Dumazete36fa2f2010-04-19 21:17:14 +00002579 rps_unlock(sd);
Tom Herbert0a9627f2010-03-16 08:03:29 +00002580
Tom Herbert0a9627f2010-03-16 08:03:29 +00002581 local_irq_restore(flags);
2582
Eric Dumazetcaf586e2010-09-30 21:06:55 +00002583 atomic_long_inc(&skb->dev->rx_dropped);
Tom Herbert0a9627f2010-03-16 08:03:29 +00002584 kfree_skb(skb);
2585 return NET_RX_DROP;
2586}
Linus Torvalds1da177e2005-04-16 15:20:36 -07002587
Linus Torvalds1da177e2005-04-16 15:20:36 -07002588/**
2589 * netif_rx - post buffer to the network code
2590 * @skb: buffer to post
2591 *
2592 * This function receives a packet from a device driver and queues it for
2593 * the upper (protocol) levels to process. It always succeeds. The buffer
2594 * may be dropped during processing for congestion control or by the
2595 * protocol layers.
2596 *
2597 * return values:
2598 * NET_RX_SUCCESS (no congestion)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002599 * NET_RX_DROP (packet was dropped)
2600 *
2601 */
2602
2603int netif_rx(struct sk_buff *skb)
2604{
Eric Dumazetb0e28f12010-04-15 00:14:07 -07002605 int ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002606
2607 /* if netpoll wants it, pretend we never saw it */
2608 if (netpoll_rx(skb))
2609 return NET_RX_DROP;
2610
Eric Dumazet3b098e22010-05-15 23:57:10 -07002611 if (netdev_tstamp_prequeue)
2612 net_timestamp_check(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002613
Koki Sanagicf66ba52010-08-23 18:45:02 +09002614 trace_netif_rx(skb);
Eric Dumazetdf334542010-03-24 19:13:54 +00002615#ifdef CONFIG_RPS
Eric Dumazetb0e28f12010-04-15 00:14:07 -07002616 {
Tom Herbertfec5e652010-04-16 16:01:27 -07002617 struct rps_dev_flow voidflow, *rflow = &voidflow;
Eric Dumazetb0e28f12010-04-15 00:14:07 -07002618 int cpu;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002619
Changli Gaocece1942010-08-07 20:35:43 -07002620 preempt_disable();
Eric Dumazetb0e28f12010-04-15 00:14:07 -07002621 rcu_read_lock();
Tom Herbertfec5e652010-04-16 16:01:27 -07002622
2623 cpu = get_rps_cpu(skb->dev, skb, &rflow);
Eric Dumazetb0e28f12010-04-15 00:14:07 -07002624 if (cpu < 0)
2625 cpu = smp_processor_id();
Tom Herbertfec5e652010-04-16 16:01:27 -07002626
2627 ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail);
2628
Eric Dumazetb0e28f12010-04-15 00:14:07 -07002629 rcu_read_unlock();
Changli Gaocece1942010-08-07 20:35:43 -07002630 preempt_enable();
Eric Dumazetb0e28f12010-04-15 00:14:07 -07002631 }
2632#else
Tom Herbertfec5e652010-04-16 16:01:27 -07002633 {
2634 unsigned int qtail;
2635 ret = enqueue_to_backlog(skb, get_cpu(), &qtail);
2636 put_cpu();
2637 }
Eric Dumazetb0e28f12010-04-15 00:14:07 -07002638#endif
2639 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002640}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07002641EXPORT_SYMBOL(netif_rx);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002642
2643int netif_rx_ni(struct sk_buff *skb)
2644{
2645 int err;
2646
2647 preempt_disable();
2648 err = netif_rx(skb);
2649 if (local_softirq_pending())
2650 do_softirq();
2651 preempt_enable();
2652
2653 return err;
2654}
Linus Torvalds1da177e2005-04-16 15:20:36 -07002655EXPORT_SYMBOL(netif_rx_ni);
2656
Linus Torvalds1da177e2005-04-16 15:20:36 -07002657static void net_tx_action(struct softirq_action *h)
2658{
2659 struct softnet_data *sd = &__get_cpu_var(softnet_data);
2660
2661 if (sd->completion_queue) {
2662 struct sk_buff *clist;
2663
2664 local_irq_disable();
2665 clist = sd->completion_queue;
2666 sd->completion_queue = NULL;
2667 local_irq_enable();
2668
2669 while (clist) {
2670 struct sk_buff *skb = clist;
2671 clist = clist->next;
2672
Ilpo Järvinen547b7922008-07-25 21:43:18 -07002673 WARN_ON(atomic_read(&skb->users));
Koki Sanagi07dc22e2010-08-23 18:46:12 +09002674 trace_kfree_skb(skb, net_tx_action);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002675 __kfree_skb(skb);
2676 }
2677 }
2678
2679 if (sd->output_queue) {
David S. Miller37437bb2008-07-16 02:15:04 -07002680 struct Qdisc *head;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002681
2682 local_irq_disable();
2683 head = sd->output_queue;
2684 sd->output_queue = NULL;
Changli Gaoa9cbd582010-04-26 23:06:24 +00002685 sd->output_queue_tailp = &sd->output_queue;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002686 local_irq_enable();
2687
2688 while (head) {
David S. Miller37437bb2008-07-16 02:15:04 -07002689 struct Qdisc *q = head;
2690 spinlock_t *root_lock;
2691
Linus Torvalds1da177e2005-04-16 15:20:36 -07002692 head = head->next_sched;
2693
David S. Miller5fb66222008-08-02 20:02:43 -07002694 root_lock = qdisc_lock(q);
David S. Miller37437bb2008-07-16 02:15:04 -07002695 if (spin_trylock(root_lock)) {
Jarek Poplawskidef82a12008-08-17 21:54:43 -07002696 smp_mb__before_clear_bit();
2697 clear_bit(__QDISC_STATE_SCHED,
2698 &q->state);
David S. Miller37437bb2008-07-16 02:15:04 -07002699 qdisc_run(q);
2700 spin_unlock(root_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002701 } else {
David S. Miller195648b2008-08-19 04:00:36 -07002702 if (!test_bit(__QDISC_STATE_DEACTIVATED,
Jarek Poplawskie8a83e12008-09-07 18:41:21 -07002703 &q->state)) {
David S. Miller195648b2008-08-19 04:00:36 -07002704 __netif_reschedule(q);
Jarek Poplawskie8a83e12008-09-07 18:41:21 -07002705 } else {
2706 smp_mb__before_clear_bit();
2707 clear_bit(__QDISC_STATE_SCHED,
2708 &q->state);
2709 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002710 }
2711 }
2712 }
2713}
2714
Stephen Hemminger6f05f622007-03-08 20:46:03 -08002715static inline int deliver_skb(struct sk_buff *skb,
2716 struct packet_type *pt_prev,
2717 struct net_device *orig_dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002718{
2719 atomic_inc(&skb->users);
David S. Millerf2ccd8f2005-08-09 19:34:12 -07002720 return pt_prev->func(skb, skb->dev, pt_prev, orig_dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002721}
2722
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00002723#if (defined(CONFIG_BRIDGE) || defined(CONFIG_BRIDGE_MODULE)) && \
2724 (defined(CONFIG_ATM_LANE) || defined(CONFIG_ATM_LANE_MODULE))
Michał Mirosławda678292009-06-05 05:35:28 +00002725/* This hook is defined here for ATM LANE */
2726int (*br_fdb_test_addr_hook)(struct net_device *dev,
2727 unsigned char *addr) __read_mostly;
Stephen Hemminger4fb019a2009-09-11 11:50:08 -07002728EXPORT_SYMBOL_GPL(br_fdb_test_addr_hook);
Michał Mirosławda678292009-06-05 05:35:28 +00002729#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07002730
Linus Torvalds1da177e2005-04-16 15:20:36 -07002731#ifdef CONFIG_NET_CLS_ACT
2732/* TODO: Maybe we should just force sch_ingress to be compiled in
2733 * when CONFIG_NET_CLS_ACT is? otherwise some useless instructions
2734 * a compare and 2 stores extra right now if we dont have it on
2735 * but have CONFIG_NET_CLS_ACT
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09002736 * NOTE: This doesnt stop any functionality; if you dont have
Linus Torvalds1da177e2005-04-16 15:20:36 -07002737 * the ingress scheduler, you just cant add policies on ingress.
2738 *
2739 */
Eric Dumazet24824a02010-10-02 06:11:55 +00002740static int ing_filter(struct sk_buff *skb, struct netdev_queue *rxq)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002741{
Linus Torvalds1da177e2005-04-16 15:20:36 -07002742 struct net_device *dev = skb->dev;
Herbert Xuf697c3e2007-10-14 00:38:47 -07002743 u32 ttl = G_TC_RTTL(skb->tc_verd);
David S. Miller555353c2008-07-08 17:33:13 -07002744 int result = TC_ACT_OK;
2745 struct Qdisc *q;
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09002746
Stephen Hemmingerde384832010-08-01 00:33:23 -07002747 if (unlikely(MAX_RED_LOOP < ttl++)) {
2748 if (net_ratelimit())
2749 pr_warning( "Redir loop detected Dropping packet (%d->%d)\n",
2750 skb->skb_iif, dev->ifindex);
Herbert Xuf697c3e2007-10-14 00:38:47 -07002751 return TC_ACT_SHOT;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002752 }
2753
Herbert Xuf697c3e2007-10-14 00:38:47 -07002754 skb->tc_verd = SET_TC_RTTL(skb->tc_verd, ttl);
2755 skb->tc_verd = SET_TC_AT(skb->tc_verd, AT_INGRESS);
2756
David S. Miller83874002008-07-17 00:53:03 -07002757 q = rxq->qdisc;
David S. Miller8d50b532008-07-30 02:37:46 -07002758 if (q != &noop_qdisc) {
David S. Miller83874002008-07-17 00:53:03 -07002759 spin_lock(qdisc_lock(q));
David S. Millera9312ae2008-08-17 21:51:03 -07002760 if (likely(!test_bit(__QDISC_STATE_DEACTIVATED, &q->state)))
2761 result = qdisc_enqueue_root(skb, q);
David S. Miller83874002008-07-17 00:53:03 -07002762 spin_unlock(qdisc_lock(q));
2763 }
Herbert Xuf697c3e2007-10-14 00:38:47 -07002764
Linus Torvalds1da177e2005-04-16 15:20:36 -07002765 return result;
2766}
Herbert Xuf697c3e2007-10-14 00:38:47 -07002767
2768static inline struct sk_buff *handle_ing(struct sk_buff *skb,
2769 struct packet_type **pt_prev,
2770 int *ret, struct net_device *orig_dev)
2771{
Eric Dumazet24824a02010-10-02 06:11:55 +00002772 struct netdev_queue *rxq = rcu_dereference(skb->dev->ingress_queue);
2773
2774 if (!rxq || rxq->qdisc == &noop_qdisc)
Herbert Xuf697c3e2007-10-14 00:38:47 -07002775 goto out;
2776
2777 if (*pt_prev) {
2778 *ret = deliver_skb(skb, *pt_prev, orig_dev);
2779 *pt_prev = NULL;
Herbert Xuf697c3e2007-10-14 00:38:47 -07002780 }
2781
Eric Dumazet24824a02010-10-02 06:11:55 +00002782 switch (ing_filter(skb, rxq)) {
Herbert Xuf697c3e2007-10-14 00:38:47 -07002783 case TC_ACT_SHOT:
2784 case TC_ACT_STOLEN:
2785 kfree_skb(skb);
2786 return NULL;
2787 }
2788
2789out:
2790 skb->tc_verd = 0;
2791 return skb;
2792}
Linus Torvalds1da177e2005-04-16 15:20:36 -07002793#endif
2794
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00002795/**
2796 * netdev_rx_handler_register - register receive handler
2797 * @dev: device to register a handler for
2798 * @rx_handler: receive handler to register
Jiri Pirko93e2c322010-06-10 03:34:59 +00002799 * @rx_handler_data: data pointer that is used by rx handler
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00002800 *
2801 * Register a receive hander for a device. This handler will then be
2802 * called from __netif_receive_skb. A negative errno code is returned
2803 * on a failure.
2804 *
2805 * The caller must hold the rtnl_mutex.
2806 */
2807int netdev_rx_handler_register(struct net_device *dev,
Jiri Pirko93e2c322010-06-10 03:34:59 +00002808 rx_handler_func_t *rx_handler,
2809 void *rx_handler_data)
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00002810{
2811 ASSERT_RTNL();
2812
2813 if (dev->rx_handler)
2814 return -EBUSY;
2815
Jiri Pirko93e2c322010-06-10 03:34:59 +00002816 rcu_assign_pointer(dev->rx_handler_data, rx_handler_data);
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00002817 rcu_assign_pointer(dev->rx_handler, rx_handler);
2818
2819 return 0;
2820}
2821EXPORT_SYMBOL_GPL(netdev_rx_handler_register);
2822
2823/**
2824 * netdev_rx_handler_unregister - unregister receive handler
2825 * @dev: device to unregister a handler from
2826 *
2827 * Unregister a receive hander from a device.
2828 *
2829 * The caller must hold the rtnl_mutex.
2830 */
2831void netdev_rx_handler_unregister(struct net_device *dev)
2832{
2833
2834 ASSERT_RTNL();
2835 rcu_assign_pointer(dev->rx_handler, NULL);
Jiri Pirko93e2c322010-06-10 03:34:59 +00002836 rcu_assign_pointer(dev->rx_handler_data, NULL);
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00002837}
2838EXPORT_SYMBOL_GPL(netdev_rx_handler_unregister);
2839
Eric Dumazetacbbc072010-04-11 06:56:11 +00002840static inline void skb_bond_set_mac_by_master(struct sk_buff *skb,
2841 struct net_device *master)
2842{
2843 if (skb->pkt_type == PACKET_HOST) {
2844 u16 *dest = (u16 *) eth_hdr(skb)->h_dest;
2845
2846 memcpy(dest, master->dev_addr, ETH_ALEN);
2847 }
2848}
2849
2850/* On bonding slaves other than the currently active slave, suppress
2851 * duplicates except for 802.3ad ETH_P_SLOW, alb non-mcast/bcast, and
2852 * ARP on active-backup slaves with arp_validate enabled.
2853 */
2854int __skb_bond_should_drop(struct sk_buff *skb, struct net_device *master)
2855{
2856 struct net_device *dev = skb->dev;
2857
2858 if (master->priv_flags & IFF_MASTER_ARPMON)
2859 dev->last_rx = jiffies;
2860
Jiri Pirkof350a0a82010-06-15 06:50:45 +00002861 if ((master->priv_flags & IFF_MASTER_ALB) &&
2862 (master->priv_flags & IFF_BRIDGE_PORT)) {
Eric Dumazetacbbc072010-04-11 06:56:11 +00002863 /* Do address unmangle. The local destination address
2864 * will be always the one master has. Provides the right
2865 * functionality in a bridge.
2866 */
2867 skb_bond_set_mac_by_master(skb, master);
2868 }
2869
2870 if (dev->priv_flags & IFF_SLAVE_INACTIVE) {
2871 if ((dev->priv_flags & IFF_SLAVE_NEEDARP) &&
2872 skb->protocol == __cpu_to_be16(ETH_P_ARP))
2873 return 0;
2874
2875 if (master->priv_flags & IFF_MASTER_ALB) {
2876 if (skb->pkt_type != PACKET_BROADCAST &&
2877 skb->pkt_type != PACKET_MULTICAST)
2878 return 0;
2879 }
2880 if (master->priv_flags & IFF_MASTER_8023AD &&
2881 skb->protocol == __cpu_to_be16(ETH_P_SLOW))
2882 return 0;
2883
2884 return 1;
2885 }
2886 return 0;
2887}
2888EXPORT_SYMBOL(__skb_bond_should_drop);
2889
Eric Dumazet10f744d2010-03-28 23:07:20 -07002890static int __netif_receive_skb(struct sk_buff *skb)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002891{
2892 struct packet_type *ptype, *pt_prev;
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00002893 rx_handler_func_t *rx_handler;
David S. Millerf2ccd8f2005-08-09 19:34:12 -07002894 struct net_device *orig_dev;
Eric Dumazet0641e4f2010-03-18 21:16:45 -07002895 struct net_device *master;
Joe Eykholt0d7a3682008-07-02 18:22:01 -07002896 struct net_device *null_or_orig;
John Fastabend2df4a0f2010-05-12 21:31:11 +00002897 struct net_device *orig_or_bond;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002898 int ret = NET_RX_DROP;
Al Viro252e3342006-11-14 20:48:11 -08002899 __be16 type;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002900
Eric Dumazet3b098e22010-05-15 23:57:10 -07002901 if (!netdev_tstamp_prequeue)
2902 net_timestamp_check(skb);
Eric Dumazet81bbb3d2009-09-30 16:42:42 -07002903
Koki Sanagicf66ba52010-08-23 18:45:02 +09002904 trace_netif_receive_skb(skb);
Patrick McHardy9b22ea52008-11-04 14:49:57 -08002905
Linus Torvalds1da177e2005-04-16 15:20:36 -07002906 /* if we've gotten here through NAPI, check netpoll */
Stephen Hemmingerbea33482007-10-03 16:41:36 -07002907 if (netpoll_receive_skb(skb))
Linus Torvalds1da177e2005-04-16 15:20:36 -07002908 return NET_RX_DROP;
2909
Eric Dumazet8964be42009-11-20 15:35:04 -08002910 if (!skb->skb_iif)
2911 skb->skb_iif = skb->dev->ifindex;
David S. Miller86e65da2005-08-09 19:36:29 -07002912
John Fastabend597a2642010-06-03 09:30:11 +00002913 /*
2914 * bonding note: skbs received on inactive slaves should only
2915 * be delivered to pkt handlers that are exact matches. Also
2916 * the deliver_no_wcard flag will be set. If packet handlers
2917 * are sensitive to duplicate packets these skbs will need to
Jesse Gross3701e512010-10-20 13:56:06 +00002918 * be dropped at the handler.
John Fastabend597a2642010-06-03 09:30:11 +00002919 */
Joe Eykholt0d7a3682008-07-02 18:22:01 -07002920 null_or_orig = NULL;
Joe Eykholtcc9bd5c2008-07-02 18:22:00 -07002921 orig_dev = skb->dev;
Eric Dumazet0641e4f2010-03-18 21:16:45 -07002922 master = ACCESS_ONCE(orig_dev->master);
John Fastabend597a2642010-06-03 09:30:11 +00002923 if (skb->deliver_no_wcard)
2924 null_or_orig = orig_dev;
2925 else if (master) {
2926 if (skb_bond_should_drop(skb, master)) {
2927 skb->deliver_no_wcard = 1;
Joe Eykholt0d7a3682008-07-02 18:22:01 -07002928 null_or_orig = orig_dev; /* deliver only exact match */
John Fastabend597a2642010-06-03 09:30:11 +00002929 } else
Eric Dumazet0641e4f2010-03-18 21:16:45 -07002930 skb->dev = master;
Joe Eykholtcc9bd5c2008-07-02 18:22:00 -07002931 }
Jay Vosburgh8f903c72006-02-21 16:36:44 -08002932
Eric Dumazet27f39c73e2010-05-19 22:07:23 +00002933 __this_cpu_inc(softnet_data.processed);
Arnaldo Carvalho de Meloc1d2bbe2007-04-10 20:45:18 -07002934 skb_reset_network_header(skb);
Arnaldo Carvalho de Melobadff6d2007-03-13 13:06:52 -03002935 skb_reset_transport_header(skb);
Arnaldo Carvalho de Melob0e380b2007-04-10 21:21:55 -07002936 skb->mac_len = skb->network_header - skb->mac_header;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002937
2938 pt_prev = NULL;
2939
2940 rcu_read_lock();
2941
2942#ifdef CONFIG_NET_CLS_ACT
2943 if (skb->tc_verd & TC_NCLS) {
2944 skb->tc_verd = CLR_TC_NCLS(skb->tc_verd);
2945 goto ncls;
2946 }
2947#endif
2948
2949 list_for_each_entry_rcu(ptype, &ptype_all, list) {
Joe Eykholtf9823072008-07-02 18:22:02 -07002950 if (ptype->dev == null_or_orig || ptype->dev == skb->dev ||
2951 ptype->dev == orig_dev) {
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09002952 if (pt_prev)
David S. Millerf2ccd8f2005-08-09 19:34:12 -07002953 ret = deliver_skb(skb, pt_prev, orig_dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002954 pt_prev = ptype;
2955 }
2956 }
2957
2958#ifdef CONFIG_NET_CLS_ACT
Herbert Xuf697c3e2007-10-14 00:38:47 -07002959 skb = handle_ing(skb, &pt_prev, &ret, orig_dev);
2960 if (!skb)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002961 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002962ncls:
2963#endif
2964
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00002965 /* Handle special case of bridge or macvlan */
2966 rx_handler = rcu_dereference(skb->dev->rx_handler);
2967 if (rx_handler) {
2968 if (pt_prev) {
2969 ret = deliver_skb(skb, pt_prev, orig_dev);
2970 pt_prev = NULL;
2971 }
2972 skb = rx_handler(skb);
2973 if (!skb)
2974 goto out;
2975 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002976
Jesse Gross3701e512010-10-20 13:56:06 +00002977 if (vlan_tx_tag_present(skb)) {
2978 if (pt_prev) {
2979 ret = deliver_skb(skb, pt_prev, orig_dev);
2980 pt_prev = NULL;
2981 }
2982 if (vlan_hwaccel_do_receive(&skb)) {
2983 ret = __netif_receive_skb(skb);
2984 goto out;
2985 } else if (unlikely(!skb))
2986 goto out;
2987 }
2988
Andy Gospodarek1f3c8802009-12-14 10:48:58 +00002989 /*
2990 * Make sure frames received on VLAN interfaces stacked on
2991 * bonding interfaces still make their way to any base bonding
2992 * device that may have registered for a specific ptype. The
2993 * handler may have to adjust skb->dev and orig_dev.
Andy Gospodarek1f3c8802009-12-14 10:48:58 +00002994 */
John Fastabend2df4a0f2010-05-12 21:31:11 +00002995 orig_or_bond = orig_dev;
Andy Gospodarek1f3c8802009-12-14 10:48:58 +00002996 if ((skb->dev->priv_flags & IFF_802_1Q_VLAN) &&
2997 (vlan_dev_real_dev(skb->dev)->priv_flags & IFF_BONDING)) {
John Fastabend2df4a0f2010-05-12 21:31:11 +00002998 orig_or_bond = vlan_dev_real_dev(skb->dev);
Andy Gospodarek1f3c8802009-12-14 10:48:58 +00002999 }
3000
Linus Torvalds1da177e2005-04-16 15:20:36 -07003001 type = skb->protocol;
Pavel Emelyanov82d8a8672007-11-26 20:12:58 +08003002 list_for_each_entry_rcu(ptype,
3003 &ptype_base[ntohs(type) & PTYPE_HASH_MASK], list) {
Andy Gospodarek1f3c8802009-12-14 10:48:58 +00003004 if (ptype->type == type && (ptype->dev == null_or_orig ||
Andy Gospodarekca8d9ea2010-01-06 12:56:37 +00003005 ptype->dev == skb->dev || ptype->dev == orig_dev ||
John Fastabend2df4a0f2010-05-12 21:31:11 +00003006 ptype->dev == orig_or_bond)) {
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09003007 if (pt_prev)
David S. Millerf2ccd8f2005-08-09 19:34:12 -07003008 ret = deliver_skb(skb, pt_prev, orig_dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003009 pt_prev = ptype;
3010 }
3011 }
3012
3013 if (pt_prev) {
David S. Millerf2ccd8f2005-08-09 19:34:12 -07003014 ret = pt_prev->func(skb, skb->dev, pt_prev, orig_dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003015 } else {
Eric Dumazetcaf586e2010-09-30 21:06:55 +00003016 atomic_long_inc(&skb->dev->rx_dropped);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003017 kfree_skb(skb);
3018 /* Jamal, now you will not able to escape explaining
3019 * me how you were going to use this. :-)
3020 */
3021 ret = NET_RX_DROP;
3022 }
3023
3024out:
3025 rcu_read_unlock();
3026 return ret;
3027}
Tom Herbert0a9627f2010-03-16 08:03:29 +00003028
3029/**
3030 * netif_receive_skb - process receive buffer from network
3031 * @skb: buffer to process
3032 *
3033 * netif_receive_skb() is the main receive data processing function.
3034 * It always succeeds. The buffer may be dropped during processing
3035 * for congestion control or by the protocol layers.
3036 *
3037 * This function may only be called from softirq context and interrupts
3038 * should be enabled.
3039 *
3040 * Return values (usually ignored):
3041 * NET_RX_SUCCESS: no congestion
3042 * NET_RX_DROP: packet was dropped
3043 */
3044int netif_receive_skb(struct sk_buff *skb)
3045{
Eric Dumazet3b098e22010-05-15 23:57:10 -07003046 if (netdev_tstamp_prequeue)
3047 net_timestamp_check(skb);
3048
Richard Cochranc1f19b52010-07-17 08:49:36 +00003049 if (skb_defer_rx_timestamp(skb))
3050 return NET_RX_SUCCESS;
3051
Eric Dumazetdf334542010-03-24 19:13:54 +00003052#ifdef CONFIG_RPS
Eric Dumazet3b098e22010-05-15 23:57:10 -07003053 {
3054 struct rps_dev_flow voidflow, *rflow = &voidflow;
3055 int cpu, ret;
Tom Herbert0a9627f2010-03-16 08:03:29 +00003056
Eric Dumazet3b098e22010-05-15 23:57:10 -07003057 rcu_read_lock();
Tom Herbert0a9627f2010-03-16 08:03:29 +00003058
Eric Dumazet3b098e22010-05-15 23:57:10 -07003059 cpu = get_rps_cpu(skb->dev, skb, &rflow);
Tom Herbertfec5e652010-04-16 16:01:27 -07003060
Eric Dumazet3b098e22010-05-15 23:57:10 -07003061 if (cpu >= 0) {
3062 ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail);
3063 rcu_read_unlock();
3064 } else {
3065 rcu_read_unlock();
3066 ret = __netif_receive_skb(skb);
3067 }
3068
3069 return ret;
Tom Herbertfec5e652010-04-16 16:01:27 -07003070 }
Tom Herbert1e94d722010-03-18 17:45:44 -07003071#else
3072 return __netif_receive_skb(skb);
3073#endif
Tom Herbert0a9627f2010-03-16 08:03:29 +00003074}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07003075EXPORT_SYMBOL(netif_receive_skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003076
Eric Dumazet88751272010-04-19 05:07:33 +00003077/* Network device is going away, flush any packets still pending
3078 * Called with irqs disabled.
3079 */
Changli Gao152102c2010-03-30 20:16:22 +00003080static void flush_backlog(void *arg)
Stephen Hemminger6e583ce2008-08-03 21:29:57 -07003081{
Changli Gao152102c2010-03-30 20:16:22 +00003082 struct net_device *dev = arg;
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003083 struct softnet_data *sd = &__get_cpu_var(softnet_data);
Stephen Hemminger6e583ce2008-08-03 21:29:57 -07003084 struct sk_buff *skb, *tmp;
3085
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003086 rps_lock(sd);
Changli Gao6e7676c2010-04-27 15:07:33 -07003087 skb_queue_walk_safe(&sd->input_pkt_queue, skb, tmp) {
Stephen Hemminger6e583ce2008-08-03 21:29:57 -07003088 if (skb->dev == dev) {
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003089 __skb_unlink(skb, &sd->input_pkt_queue);
Stephen Hemminger6e583ce2008-08-03 21:29:57 -07003090 kfree_skb(skb);
Tom Herbert76cc8b12010-05-20 18:37:59 +00003091 input_queue_head_incr(sd);
Stephen Hemminger6e583ce2008-08-03 21:29:57 -07003092 }
Changli Gao6e7676c2010-04-27 15:07:33 -07003093 }
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003094 rps_unlock(sd);
Changli Gao6e7676c2010-04-27 15:07:33 -07003095
3096 skb_queue_walk_safe(&sd->process_queue, skb, tmp) {
3097 if (skb->dev == dev) {
3098 __skb_unlink(skb, &sd->process_queue);
3099 kfree_skb(skb);
Tom Herbert76cc8b12010-05-20 18:37:59 +00003100 input_queue_head_incr(sd);
Changli Gao6e7676c2010-04-27 15:07:33 -07003101 }
3102 }
Stephen Hemminger6e583ce2008-08-03 21:29:57 -07003103}
3104
Herbert Xud565b0a2008-12-15 23:38:52 -08003105static int napi_gro_complete(struct sk_buff *skb)
3106{
3107 struct packet_type *ptype;
3108 __be16 type = skb->protocol;
3109 struct list_head *head = &ptype_base[ntohs(type) & PTYPE_HASH_MASK];
3110 int err = -ENOENT;
3111
Herbert Xufc59f9a2009-04-14 15:11:06 -07003112 if (NAPI_GRO_CB(skb)->count == 1) {
3113 skb_shinfo(skb)->gso_size = 0;
Herbert Xud565b0a2008-12-15 23:38:52 -08003114 goto out;
Herbert Xufc59f9a2009-04-14 15:11:06 -07003115 }
Herbert Xud565b0a2008-12-15 23:38:52 -08003116
3117 rcu_read_lock();
3118 list_for_each_entry_rcu(ptype, head, list) {
3119 if (ptype->type != type || ptype->dev || !ptype->gro_complete)
3120 continue;
3121
3122 err = ptype->gro_complete(skb);
3123 break;
3124 }
3125 rcu_read_unlock();
3126
3127 if (err) {
3128 WARN_ON(&ptype->list == head);
3129 kfree_skb(skb);
3130 return NET_RX_SUCCESS;
3131 }
3132
3133out:
Herbert Xud565b0a2008-12-15 23:38:52 -08003134 return netif_receive_skb(skb);
3135}
3136
Eric Dumazet86cac582010-08-31 18:25:32 +00003137inline void napi_gro_flush(struct napi_struct *napi)
Herbert Xud565b0a2008-12-15 23:38:52 -08003138{
3139 struct sk_buff *skb, *next;
3140
3141 for (skb = napi->gro_list; skb; skb = next) {
3142 next = skb->next;
3143 skb->next = NULL;
3144 napi_gro_complete(skb);
3145 }
3146
Herbert Xu4ae55442009-02-08 18:00:36 +00003147 napi->gro_count = 0;
Herbert Xud565b0a2008-12-15 23:38:52 -08003148 napi->gro_list = NULL;
3149}
Eric Dumazet86cac582010-08-31 18:25:32 +00003150EXPORT_SYMBOL(napi_gro_flush);
Herbert Xud565b0a2008-12-15 23:38:52 -08003151
Ben Hutchings5b252f02009-10-29 07:17:09 +00003152enum gro_result dev_gro_receive(struct napi_struct *napi, struct sk_buff *skb)
Herbert Xud565b0a2008-12-15 23:38:52 -08003153{
3154 struct sk_buff **pp = NULL;
3155 struct packet_type *ptype;
3156 __be16 type = skb->protocol;
3157 struct list_head *head = &ptype_base[ntohs(type) & PTYPE_HASH_MASK];
Herbert Xu0da2afd52008-12-26 14:57:42 -08003158 int same_flow;
Herbert Xud565b0a2008-12-15 23:38:52 -08003159 int mac_len;
Ben Hutchings5b252f02009-10-29 07:17:09 +00003160 enum gro_result ret;
Herbert Xud565b0a2008-12-15 23:38:52 -08003161
Jarek Poplawskice9e76c2010-08-05 01:19:11 +00003162 if (!(skb->dev->features & NETIF_F_GRO) || netpoll_rx_on(skb))
Herbert Xud565b0a2008-12-15 23:38:52 -08003163 goto normal;
3164
David S. Miller21dc3302010-08-23 00:13:46 -07003165 if (skb_is_gso(skb) || skb_has_frag_list(skb))
Herbert Xuf17f5c92009-01-14 14:36:12 -08003166 goto normal;
3167
Herbert Xud565b0a2008-12-15 23:38:52 -08003168 rcu_read_lock();
3169 list_for_each_entry_rcu(ptype, head, list) {
Herbert Xud565b0a2008-12-15 23:38:52 -08003170 if (ptype->type != type || ptype->dev || !ptype->gro_receive)
3171 continue;
3172
Herbert Xu86911732009-01-29 14:19:50 +00003173 skb_set_network_header(skb, skb_gro_offset(skb));
Herbert Xud565b0a2008-12-15 23:38:52 -08003174 mac_len = skb->network_header - skb->mac_header;
3175 skb->mac_len = mac_len;
3176 NAPI_GRO_CB(skb)->same_flow = 0;
3177 NAPI_GRO_CB(skb)->flush = 0;
Herbert Xu5d38a072009-01-04 16:13:40 -08003178 NAPI_GRO_CB(skb)->free = 0;
Herbert Xud565b0a2008-12-15 23:38:52 -08003179
Herbert Xud565b0a2008-12-15 23:38:52 -08003180 pp = ptype->gro_receive(&napi->gro_list, skb);
3181 break;
3182 }
3183 rcu_read_unlock();
3184
3185 if (&ptype->list == head)
3186 goto normal;
3187
Herbert Xu0da2afd52008-12-26 14:57:42 -08003188 same_flow = NAPI_GRO_CB(skb)->same_flow;
Herbert Xu5d0d9be2009-01-29 14:19:48 +00003189 ret = NAPI_GRO_CB(skb)->free ? GRO_MERGED_FREE : GRO_MERGED;
Herbert Xu0da2afd52008-12-26 14:57:42 -08003190
Herbert Xud565b0a2008-12-15 23:38:52 -08003191 if (pp) {
3192 struct sk_buff *nskb = *pp;
3193
3194 *pp = nskb->next;
3195 nskb->next = NULL;
3196 napi_gro_complete(nskb);
Herbert Xu4ae55442009-02-08 18:00:36 +00003197 napi->gro_count--;
Herbert Xud565b0a2008-12-15 23:38:52 -08003198 }
3199
Herbert Xu0da2afd52008-12-26 14:57:42 -08003200 if (same_flow)
Herbert Xud565b0a2008-12-15 23:38:52 -08003201 goto ok;
3202
Herbert Xu4ae55442009-02-08 18:00:36 +00003203 if (NAPI_GRO_CB(skb)->flush || napi->gro_count >= MAX_GRO_SKBS)
Herbert Xud565b0a2008-12-15 23:38:52 -08003204 goto normal;
Herbert Xud565b0a2008-12-15 23:38:52 -08003205
Herbert Xu4ae55442009-02-08 18:00:36 +00003206 napi->gro_count++;
Herbert Xud565b0a2008-12-15 23:38:52 -08003207 NAPI_GRO_CB(skb)->count = 1;
Herbert Xu86911732009-01-29 14:19:50 +00003208 skb_shinfo(skb)->gso_size = skb_gro_len(skb);
Herbert Xud565b0a2008-12-15 23:38:52 -08003209 skb->next = napi->gro_list;
3210 napi->gro_list = skb;
Herbert Xu5d0d9be2009-01-29 14:19:48 +00003211 ret = GRO_HELD;
Herbert Xud565b0a2008-12-15 23:38:52 -08003212
Herbert Xuad0f9902009-02-01 01:24:55 -08003213pull:
Herbert Xucb189782009-05-26 18:50:31 +00003214 if (skb_headlen(skb) < skb_gro_offset(skb)) {
3215 int grow = skb_gro_offset(skb) - skb_headlen(skb);
3216
3217 BUG_ON(skb->end - skb->tail < grow);
3218
3219 memcpy(skb_tail_pointer(skb), NAPI_GRO_CB(skb)->frag0, grow);
3220
3221 skb->tail += grow;
3222 skb->data_len -= grow;
3223
3224 skb_shinfo(skb)->frags[0].page_offset += grow;
3225 skb_shinfo(skb)->frags[0].size -= grow;
3226
3227 if (unlikely(!skb_shinfo(skb)->frags[0].size)) {
3228 put_page(skb_shinfo(skb)->frags[0].page);
3229 memmove(skb_shinfo(skb)->frags,
3230 skb_shinfo(skb)->frags + 1,
Jarek Poplawskie5093ae2010-08-11 02:02:10 +00003231 --skb_shinfo(skb)->nr_frags * sizeof(skb_frag_t));
Herbert Xucb189782009-05-26 18:50:31 +00003232 }
Herbert Xuad0f9902009-02-01 01:24:55 -08003233 }
3234
Herbert Xud565b0a2008-12-15 23:38:52 -08003235ok:
Herbert Xu5d0d9be2009-01-29 14:19:48 +00003236 return ret;
Herbert Xud565b0a2008-12-15 23:38:52 -08003237
3238normal:
Herbert Xuad0f9902009-02-01 01:24:55 -08003239 ret = GRO_NORMAL;
3240 goto pull;
Herbert Xu5d38a072009-01-04 16:13:40 -08003241}
Herbert Xu96e93ea2009-01-06 10:49:34 -08003242EXPORT_SYMBOL(dev_gro_receive);
3243
Eric Dumazet40d08022010-08-26 22:03:08 -07003244static inline gro_result_t
Ben Hutchings5b252f02009-10-29 07:17:09 +00003245__napi_gro_receive(struct napi_struct *napi, struct sk_buff *skb)
Herbert Xu96e93ea2009-01-06 10:49:34 -08003246{
3247 struct sk_buff *p;
3248
3249 for (p = napi->gro_list; p; p = p->next) {
Eric Dumazet40d08022010-08-26 22:03:08 -07003250 unsigned long diffs;
3251
3252 diffs = (unsigned long)p->dev ^ (unsigned long)skb->dev;
Jesse Gross3701e512010-10-20 13:56:06 +00003253 diffs |= p->vlan_tci ^ skb->vlan_tci;
Eric Dumazet40d08022010-08-26 22:03:08 -07003254 diffs |= compare_ether_header(skb_mac_header(p),
Joe Perchesf64f9e72009-11-29 16:55:45 -08003255 skb_gro_mac_header(skb));
Eric Dumazet40d08022010-08-26 22:03:08 -07003256 NAPI_GRO_CB(p)->same_flow = !diffs;
Herbert Xu96e93ea2009-01-06 10:49:34 -08003257 NAPI_GRO_CB(p)->flush = 0;
3258 }
3259
3260 return dev_gro_receive(napi, skb);
3261}
Herbert Xu5d38a072009-01-04 16:13:40 -08003262
Ben Hutchingsc7c4b3b2009-10-29 21:36:53 -07003263gro_result_t napi_skb_finish(gro_result_t ret, struct sk_buff *skb)
Herbert Xu5d38a072009-01-04 16:13:40 -08003264{
Herbert Xu5d0d9be2009-01-29 14:19:48 +00003265 switch (ret) {
3266 case GRO_NORMAL:
Ben Hutchingsc7c4b3b2009-10-29 21:36:53 -07003267 if (netif_receive_skb(skb))
3268 ret = GRO_DROP;
3269 break;
Herbert Xu5d38a072009-01-04 16:13:40 -08003270
Herbert Xu5d0d9be2009-01-29 14:19:48 +00003271 case GRO_DROP:
Herbert Xu5d0d9be2009-01-29 14:19:48 +00003272 case GRO_MERGED_FREE:
Herbert Xu5d38a072009-01-04 16:13:40 -08003273 kfree_skb(skb);
3274 break;
Ben Hutchings5b252f02009-10-29 07:17:09 +00003275
3276 case GRO_HELD:
3277 case GRO_MERGED:
3278 break;
Herbert Xu5d38a072009-01-04 16:13:40 -08003279 }
3280
Ben Hutchingsc7c4b3b2009-10-29 21:36:53 -07003281 return ret;
Herbert Xu5d0d9be2009-01-29 14:19:48 +00003282}
3283EXPORT_SYMBOL(napi_skb_finish);
3284
Herbert Xu78a478d2009-05-26 18:50:21 +00003285void skb_gro_reset_offset(struct sk_buff *skb)
3286{
3287 NAPI_GRO_CB(skb)->data_offset = 0;
3288 NAPI_GRO_CB(skb)->frag0 = NULL;
Herbert Xu74895942009-05-26 18:50:27 +00003289 NAPI_GRO_CB(skb)->frag0_len = 0;
Herbert Xu78a478d2009-05-26 18:50:21 +00003290
Herbert Xu78d3fd02009-05-26 18:50:23 +00003291 if (skb->mac_header == skb->tail &&
Herbert Xu74895942009-05-26 18:50:27 +00003292 !PageHighMem(skb_shinfo(skb)->frags[0].page)) {
Herbert Xu78a478d2009-05-26 18:50:21 +00003293 NAPI_GRO_CB(skb)->frag0 =
3294 page_address(skb_shinfo(skb)->frags[0].page) +
3295 skb_shinfo(skb)->frags[0].page_offset;
Herbert Xu74895942009-05-26 18:50:27 +00003296 NAPI_GRO_CB(skb)->frag0_len = skb_shinfo(skb)->frags[0].size;
3297 }
Herbert Xu78a478d2009-05-26 18:50:21 +00003298}
3299EXPORT_SYMBOL(skb_gro_reset_offset);
3300
Ben Hutchingsc7c4b3b2009-10-29 21:36:53 -07003301gro_result_t napi_gro_receive(struct napi_struct *napi, struct sk_buff *skb)
Herbert Xu5d0d9be2009-01-29 14:19:48 +00003302{
Herbert Xu86911732009-01-29 14:19:50 +00003303 skb_gro_reset_offset(skb);
3304
Herbert Xu5d0d9be2009-01-29 14:19:48 +00003305 return napi_skb_finish(__napi_gro_receive(napi, skb), skb);
Herbert Xud565b0a2008-12-15 23:38:52 -08003306}
3307EXPORT_SYMBOL(napi_gro_receive);
3308
stephen hemmingerd0c2b0d2010-10-19 07:12:10 +00003309static void napi_reuse_skb(struct napi_struct *napi, struct sk_buff *skb)
Herbert Xu96e93ea2009-01-06 10:49:34 -08003310{
Herbert Xu96e93ea2009-01-06 10:49:34 -08003311 __skb_pull(skb, skb_headlen(skb));
3312 skb_reserve(skb, NET_IP_ALIGN - skb_headroom(skb));
Jesse Gross3701e512010-10-20 13:56:06 +00003313 skb->vlan_tci = 0;
Herbert Xu96e93ea2009-01-06 10:49:34 -08003314
3315 napi->skb = skb;
3316}
Herbert Xu96e93ea2009-01-06 10:49:34 -08003317
Herbert Xu76620aa2009-04-16 02:02:07 -07003318struct sk_buff *napi_get_frags(struct napi_struct *napi)
Herbert Xu5d38a072009-01-04 16:13:40 -08003319{
Herbert Xu5d38a072009-01-04 16:13:40 -08003320 struct sk_buff *skb = napi->skb;
Herbert Xu5d38a072009-01-04 16:13:40 -08003321
3322 if (!skb) {
Eric Dumazet89d71a62009-10-13 05:34:20 +00003323 skb = netdev_alloc_skb_ip_align(napi->dev, GRO_MAX_HEAD);
3324 if (skb)
3325 napi->skb = skb;
Herbert Xu5d38a072009-01-04 16:13:40 -08003326 }
Herbert Xu96e93ea2009-01-06 10:49:34 -08003327 return skb;
3328}
Herbert Xu76620aa2009-04-16 02:02:07 -07003329EXPORT_SYMBOL(napi_get_frags);
Herbert Xu96e93ea2009-01-06 10:49:34 -08003330
Ben Hutchingsc7c4b3b2009-10-29 21:36:53 -07003331gro_result_t napi_frags_finish(struct napi_struct *napi, struct sk_buff *skb,
3332 gro_result_t ret)
Herbert Xu5d0d9be2009-01-29 14:19:48 +00003333{
Herbert Xu5d0d9be2009-01-29 14:19:48 +00003334 switch (ret) {
3335 case GRO_NORMAL:
Herbert Xu86911732009-01-29 14:19:50 +00003336 case GRO_HELD:
Ajit Khapardee76b69c2010-02-16 20:25:43 +00003337 skb->protocol = eth_type_trans(skb, skb->dev);
Herbert Xu86911732009-01-29 14:19:50 +00003338
Ben Hutchingsc7c4b3b2009-10-29 21:36:53 -07003339 if (ret == GRO_HELD)
3340 skb_gro_pull(skb, -ETH_HLEN);
3341 else if (netif_receive_skb(skb))
3342 ret = GRO_DROP;
Herbert Xu86911732009-01-29 14:19:50 +00003343 break;
Herbert Xu5d0d9be2009-01-29 14:19:48 +00003344
3345 case GRO_DROP:
Herbert Xu5d0d9be2009-01-29 14:19:48 +00003346 case GRO_MERGED_FREE:
3347 napi_reuse_skb(napi, skb);
3348 break;
Ben Hutchings5b252f02009-10-29 07:17:09 +00003349
3350 case GRO_MERGED:
3351 break;
Herbert Xu5d0d9be2009-01-29 14:19:48 +00003352 }
3353
Ben Hutchingsc7c4b3b2009-10-29 21:36:53 -07003354 return ret;
Herbert Xu5d0d9be2009-01-29 14:19:48 +00003355}
3356EXPORT_SYMBOL(napi_frags_finish);
3357
Herbert Xu76620aa2009-04-16 02:02:07 -07003358struct sk_buff *napi_frags_skb(struct napi_struct *napi)
Herbert Xu96e93ea2009-01-06 10:49:34 -08003359{
Herbert Xu76620aa2009-04-16 02:02:07 -07003360 struct sk_buff *skb = napi->skb;
3361 struct ethhdr *eth;
Herbert Xua5b1cf22009-05-26 18:50:28 +00003362 unsigned int hlen;
3363 unsigned int off;
Herbert Xu76620aa2009-04-16 02:02:07 -07003364
3365 napi->skb = NULL;
3366
3367 skb_reset_mac_header(skb);
3368 skb_gro_reset_offset(skb);
3369
Herbert Xua5b1cf22009-05-26 18:50:28 +00003370 off = skb_gro_offset(skb);
3371 hlen = off + sizeof(*eth);
3372 eth = skb_gro_header_fast(skb, off);
3373 if (skb_gro_header_hard(skb, hlen)) {
3374 eth = skb_gro_header_slow(skb, hlen, off);
3375 if (unlikely(!eth)) {
3376 napi_reuse_skb(napi, skb);
3377 skb = NULL;
3378 goto out;
3379 }
Herbert Xu76620aa2009-04-16 02:02:07 -07003380 }
3381
3382 skb_gro_pull(skb, sizeof(*eth));
3383
3384 /*
3385 * This works because the only protocols we care about don't require
3386 * special handling. We'll fix it up properly at the end.
3387 */
3388 skb->protocol = eth->h_proto;
3389
3390out:
3391 return skb;
3392}
3393EXPORT_SYMBOL(napi_frags_skb);
3394
Ben Hutchingsc7c4b3b2009-10-29 21:36:53 -07003395gro_result_t napi_gro_frags(struct napi_struct *napi)
Herbert Xu76620aa2009-04-16 02:02:07 -07003396{
3397 struct sk_buff *skb = napi_frags_skb(napi);
Herbert Xu96e93ea2009-01-06 10:49:34 -08003398
3399 if (!skb)
Ben Hutchingsc7c4b3b2009-10-29 21:36:53 -07003400 return GRO_DROP;
Herbert Xu96e93ea2009-01-06 10:49:34 -08003401
Herbert Xu5d0d9be2009-01-29 14:19:48 +00003402 return napi_frags_finish(napi, skb, __napi_gro_receive(napi, skb));
Herbert Xu5d38a072009-01-04 16:13:40 -08003403}
3404EXPORT_SYMBOL(napi_gro_frags);
3405
Eric Dumazete326bed2010-04-22 00:22:45 -07003406/*
3407 * net_rps_action sends any pending IPI's for rps.
3408 * Note: called with local irq disabled, but exits with local irq enabled.
3409 */
3410static void net_rps_action_and_irq_enable(struct softnet_data *sd)
3411{
3412#ifdef CONFIG_RPS
3413 struct softnet_data *remsd = sd->rps_ipi_list;
3414
3415 if (remsd) {
3416 sd->rps_ipi_list = NULL;
3417
3418 local_irq_enable();
3419
3420 /* Send pending IPI's to kick RPS processing on remote cpus. */
3421 while (remsd) {
3422 struct softnet_data *next = remsd->rps_ipi_next;
3423
3424 if (cpu_online(remsd->cpu))
3425 __smp_call_function_single(remsd->cpu,
3426 &remsd->csd, 0);
3427 remsd = next;
3428 }
3429 } else
3430#endif
3431 local_irq_enable();
3432}
3433
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003434static int process_backlog(struct napi_struct *napi, int quota)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003435{
3436 int work = 0;
Eric Dumazeteecfd7c2010-05-06 22:07:48 -07003437 struct softnet_data *sd = container_of(napi, struct softnet_data, backlog);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003438
Eric Dumazete326bed2010-04-22 00:22:45 -07003439#ifdef CONFIG_RPS
3440 /* Check if we have pending ipi, its better to send them now,
3441 * not waiting net_rx_action() end.
3442 */
3443 if (sd->rps_ipi_list) {
3444 local_irq_disable();
3445 net_rps_action_and_irq_enable(sd);
3446 }
3447#endif
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003448 napi->weight = weight_p;
Changli Gao6e7676c2010-04-27 15:07:33 -07003449 local_irq_disable();
3450 while (work < quota) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07003451 struct sk_buff *skb;
Changli Gao6e7676c2010-04-27 15:07:33 -07003452 unsigned int qlen;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003453
Changli Gao6e7676c2010-04-27 15:07:33 -07003454 while ((skb = __skb_dequeue(&sd->process_queue))) {
Eric Dumazete4008272010-04-05 15:42:39 -07003455 local_irq_enable();
Changli Gao6e7676c2010-04-27 15:07:33 -07003456 __netif_receive_skb(skb);
Changli Gao6e7676c2010-04-27 15:07:33 -07003457 local_irq_disable();
Tom Herbert76cc8b12010-05-20 18:37:59 +00003458 input_queue_head_incr(sd);
3459 if (++work >= quota) {
3460 local_irq_enable();
3461 return work;
3462 }
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003463 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07003464
Changli Gao6e7676c2010-04-27 15:07:33 -07003465 rps_lock(sd);
3466 qlen = skb_queue_len(&sd->input_pkt_queue);
Tom Herbert76cc8b12010-05-20 18:37:59 +00003467 if (qlen)
Changli Gao6e7676c2010-04-27 15:07:33 -07003468 skb_queue_splice_tail_init(&sd->input_pkt_queue,
3469 &sd->process_queue);
Tom Herbert76cc8b12010-05-20 18:37:59 +00003470
Changli Gao6e7676c2010-04-27 15:07:33 -07003471 if (qlen < quota - work) {
Eric Dumazeteecfd7c2010-05-06 22:07:48 -07003472 /*
3473 * Inline a custom version of __napi_complete().
3474 * only current cpu owns and manipulates this napi,
3475 * and NAPI_STATE_SCHED is the only possible flag set on backlog.
3476 * we can use a plain write instead of clear_bit(),
3477 * and we dont need an smp_mb() memory barrier.
3478 */
3479 list_del(&napi->poll_list);
3480 napi->state = 0;
3481
Changli Gao6e7676c2010-04-27 15:07:33 -07003482 quota = work + qlen;
3483 }
3484 rps_unlock(sd);
3485 }
3486 local_irq_enable();
Linus Torvalds1da177e2005-04-16 15:20:36 -07003487
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003488 return work;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003489}
3490
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003491/**
3492 * __napi_schedule - schedule for receive
Randy Dunlapc4ea43c2007-10-12 21:17:49 -07003493 * @n: entry to schedule
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003494 *
3495 * The entry's receive function will be scheduled to run
3496 */
Harvey Harrisonb5606c22008-02-13 15:03:16 -08003497void __napi_schedule(struct napi_struct *n)
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003498{
3499 unsigned long flags;
3500
3501 local_irq_save(flags);
Eric Dumazeteecfd7c2010-05-06 22:07:48 -07003502 ____napi_schedule(&__get_cpu_var(softnet_data), n);
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003503 local_irq_restore(flags);
3504}
3505EXPORT_SYMBOL(__napi_schedule);
3506
Herbert Xud565b0a2008-12-15 23:38:52 -08003507void __napi_complete(struct napi_struct *n)
3508{
3509 BUG_ON(!test_bit(NAPI_STATE_SCHED, &n->state));
3510 BUG_ON(n->gro_list);
3511
3512 list_del(&n->poll_list);
3513 smp_mb__before_clear_bit();
3514 clear_bit(NAPI_STATE_SCHED, &n->state);
3515}
3516EXPORT_SYMBOL(__napi_complete);
3517
3518void napi_complete(struct napi_struct *n)
3519{
3520 unsigned long flags;
3521
3522 /*
3523 * don't let napi dequeue from the cpu poll list
3524 * just in case its running on a different cpu
3525 */
3526 if (unlikely(test_bit(NAPI_STATE_NPSVC, &n->state)))
3527 return;
3528
3529 napi_gro_flush(n);
3530 local_irq_save(flags);
3531 __napi_complete(n);
3532 local_irq_restore(flags);
3533}
3534EXPORT_SYMBOL(napi_complete);
3535
3536void netif_napi_add(struct net_device *dev, struct napi_struct *napi,
3537 int (*poll)(struct napi_struct *, int), int weight)
3538{
3539 INIT_LIST_HEAD(&napi->poll_list);
Herbert Xu4ae55442009-02-08 18:00:36 +00003540 napi->gro_count = 0;
Herbert Xud565b0a2008-12-15 23:38:52 -08003541 napi->gro_list = NULL;
Herbert Xu5d38a072009-01-04 16:13:40 -08003542 napi->skb = NULL;
Herbert Xud565b0a2008-12-15 23:38:52 -08003543 napi->poll = poll;
3544 napi->weight = weight;
3545 list_add(&napi->dev_list, &dev->napi_list);
Herbert Xud565b0a2008-12-15 23:38:52 -08003546 napi->dev = dev;
Herbert Xu5d38a072009-01-04 16:13:40 -08003547#ifdef CONFIG_NETPOLL
Herbert Xud565b0a2008-12-15 23:38:52 -08003548 spin_lock_init(&napi->poll_lock);
3549 napi->poll_owner = -1;
3550#endif
3551 set_bit(NAPI_STATE_SCHED, &napi->state);
3552}
3553EXPORT_SYMBOL(netif_napi_add);
3554
3555void netif_napi_del(struct napi_struct *napi)
3556{
3557 struct sk_buff *skb, *next;
3558
Peter P Waskiewicz Jrd7b06632008-12-26 01:35:35 -08003559 list_del_init(&napi->dev_list);
Herbert Xu76620aa2009-04-16 02:02:07 -07003560 napi_free_frags(napi);
Herbert Xud565b0a2008-12-15 23:38:52 -08003561
3562 for (skb = napi->gro_list; skb; skb = next) {
3563 next = skb->next;
3564 skb->next = NULL;
3565 kfree_skb(skb);
3566 }
3567
3568 napi->gro_list = NULL;
Herbert Xu4ae55442009-02-08 18:00:36 +00003569 napi->gro_count = 0;
Herbert Xud565b0a2008-12-15 23:38:52 -08003570}
3571EXPORT_SYMBOL(netif_napi_del);
3572
Linus Torvalds1da177e2005-04-16 15:20:36 -07003573static void net_rx_action(struct softirq_action *h)
3574{
Eric Dumazete326bed2010-04-22 00:22:45 -07003575 struct softnet_data *sd = &__get_cpu_var(softnet_data);
Stephen Hemminger24f8b232008-11-03 17:14:38 -08003576 unsigned long time_limit = jiffies + 2;
Stephen Hemminger51b0bde2005-06-23 20:14:40 -07003577 int budget = netdev_budget;
Matt Mackall53fb95d2005-08-11 19:27:43 -07003578 void *have;
3579
Linus Torvalds1da177e2005-04-16 15:20:36 -07003580 local_irq_disable();
3581
Eric Dumazete326bed2010-04-22 00:22:45 -07003582 while (!list_empty(&sd->poll_list)) {
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003583 struct napi_struct *n;
3584 int work, weight;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003585
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003586 /* If softirq window is exhuasted then punt.
Stephen Hemminger24f8b232008-11-03 17:14:38 -08003587 * Allow this to run for 2 jiffies since which will allow
3588 * an average latency of 1.5/HZ.
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003589 */
Stephen Hemminger24f8b232008-11-03 17:14:38 -08003590 if (unlikely(budget <= 0 || time_after(jiffies, time_limit)))
Linus Torvalds1da177e2005-04-16 15:20:36 -07003591 goto softnet_break;
3592
3593 local_irq_enable();
3594
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003595 /* Even though interrupts have been re-enabled, this
3596 * access is safe because interrupts can only add new
3597 * entries to the tail of this list, and only ->poll()
3598 * calls can remove this head entry from the list.
3599 */
Eric Dumazete326bed2010-04-22 00:22:45 -07003600 n = list_first_entry(&sd->poll_list, struct napi_struct, poll_list);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003601
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003602 have = netpoll_poll_lock(n);
3603
3604 weight = n->weight;
3605
David S. Miller0a7606c2007-10-29 21:28:47 -07003606 /* This NAPI_STATE_SCHED test is for avoiding a race
3607 * with netpoll's poll_napi(). Only the entity which
3608 * obtains the lock and sees NAPI_STATE_SCHED set will
3609 * actually make the ->poll() call. Therefore we avoid
3610 * accidently calling ->poll() when NAPI is not scheduled.
3611 */
3612 work = 0;
Neil Horman4ea7e382009-05-21 07:36:08 +00003613 if (test_bit(NAPI_STATE_SCHED, &n->state)) {
David S. Miller0a7606c2007-10-29 21:28:47 -07003614 work = n->poll(n, weight);
Neil Horman4ea7e382009-05-21 07:36:08 +00003615 trace_napi_poll(n);
3616 }
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003617
3618 WARN_ON_ONCE(work > weight);
3619
3620 budget -= work;
3621
3622 local_irq_disable();
3623
3624 /* Drivers must not modify the NAPI state if they
3625 * consume the entire weight. In such cases this code
3626 * still "owns" the NAPI instance and therefore can
3627 * move the instance around on the list at-will.
3628 */
David S. Millerfed17f32008-01-07 21:00:40 -08003629 if (unlikely(work == weight)) {
Herbert Xuff780cd2009-06-26 19:27:04 -07003630 if (unlikely(napi_disable_pending(n))) {
3631 local_irq_enable();
3632 napi_complete(n);
3633 local_irq_disable();
3634 } else
Eric Dumazete326bed2010-04-22 00:22:45 -07003635 list_move_tail(&n->poll_list, &sd->poll_list);
David S. Millerfed17f32008-01-07 21:00:40 -08003636 }
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003637
3638 netpoll_poll_unlock(have);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003639 }
3640out:
Eric Dumazete326bed2010-04-22 00:22:45 -07003641 net_rps_action_and_irq_enable(sd);
Tom Herbert0a9627f2010-03-16 08:03:29 +00003642
Chris Leechdb217332006-06-17 21:24:58 -07003643#ifdef CONFIG_NET_DMA
3644 /*
3645 * There may not be any more sk_buffs coming right now, so push
3646 * any pending DMA copies to hardware
3647 */
Dan Williams2ba05622009-01-06 11:38:14 -07003648 dma_issue_pending_all();
Chris Leechdb217332006-06-17 21:24:58 -07003649#endif
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003650
Linus Torvalds1da177e2005-04-16 15:20:36 -07003651 return;
3652
3653softnet_break:
Changli Gaodee42872010-05-02 05:42:16 +00003654 sd->time_squeeze++;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003655 __raise_softirq_irqoff(NET_RX_SOFTIRQ);
3656 goto out;
3657}
3658
Eric Dumazetd1b19df2009-09-03 01:29:39 -07003659static gifconf_func_t *gifconf_list[NPROTO];
Linus Torvalds1da177e2005-04-16 15:20:36 -07003660
3661/**
3662 * register_gifconf - register a SIOCGIF handler
3663 * @family: Address family
3664 * @gifconf: Function handler
3665 *
3666 * Register protocol dependent address dumping routines. The handler
3667 * that is passed must not be freed or reused until it has been replaced
3668 * by another handler.
3669 */
Eric Dumazetd1b19df2009-09-03 01:29:39 -07003670int register_gifconf(unsigned int family, gifconf_func_t *gifconf)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003671{
3672 if (family >= NPROTO)
3673 return -EINVAL;
3674 gifconf_list[family] = gifconf;
3675 return 0;
3676}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07003677EXPORT_SYMBOL(register_gifconf);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003678
3679
3680/*
3681 * Map an interface index to its name (SIOCGIFNAME)
3682 */
3683
3684/*
3685 * We need this ioctl for efficient implementation of the
3686 * if_indextoname() function required by the IPv6 API. Without
3687 * it, we would have to search all the interfaces to find a
3688 * match. --pb
3689 */
3690
Eric W. Biederman881d9662007-09-17 11:56:21 -07003691static int dev_ifname(struct net *net, struct ifreq __user *arg)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003692{
3693 struct net_device *dev;
3694 struct ifreq ifr;
3695
3696 /*
3697 * Fetch the caller's info block.
3698 */
3699
3700 if (copy_from_user(&ifr, arg, sizeof(struct ifreq)))
3701 return -EFAULT;
3702
Eric Dumazetfb699dfd2009-10-19 19:18:49 +00003703 rcu_read_lock();
3704 dev = dev_get_by_index_rcu(net, ifr.ifr_ifindex);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003705 if (!dev) {
Eric Dumazetfb699dfd2009-10-19 19:18:49 +00003706 rcu_read_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07003707 return -ENODEV;
3708 }
3709
3710 strcpy(ifr.ifr_name, dev->name);
Eric Dumazetfb699dfd2009-10-19 19:18:49 +00003711 rcu_read_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07003712
3713 if (copy_to_user(arg, &ifr, sizeof(struct ifreq)))
3714 return -EFAULT;
3715 return 0;
3716}
3717
3718/*
3719 * Perform a SIOCGIFCONF call. This structure will change
3720 * size eventually, and there is nothing I can do about it.
3721 * Thus we will need a 'compatibility mode'.
3722 */
3723
Eric W. Biederman881d9662007-09-17 11:56:21 -07003724static int dev_ifconf(struct net *net, char __user *arg)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003725{
3726 struct ifconf ifc;
3727 struct net_device *dev;
3728 char __user *pos;
3729 int len;
3730 int total;
3731 int i;
3732
3733 /*
3734 * Fetch the caller's info block.
3735 */
3736
3737 if (copy_from_user(&ifc, arg, sizeof(struct ifconf)))
3738 return -EFAULT;
3739
3740 pos = ifc.ifc_buf;
3741 len = ifc.ifc_len;
3742
3743 /*
3744 * Loop over the interfaces, and write an info block for each.
3745 */
3746
3747 total = 0;
Eric W. Biederman881d9662007-09-17 11:56:21 -07003748 for_each_netdev(net, dev) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07003749 for (i = 0; i < NPROTO; i++) {
3750 if (gifconf_list[i]) {
3751 int done;
3752 if (!pos)
3753 done = gifconf_list[i](dev, NULL, 0);
3754 else
3755 done = gifconf_list[i](dev, pos + total,
3756 len - total);
3757 if (done < 0)
3758 return -EFAULT;
3759 total += done;
3760 }
3761 }
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09003762 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07003763
3764 /*
3765 * All done. Write the updated control block back to the caller.
3766 */
3767 ifc.ifc_len = total;
3768
3769 /*
3770 * Both BSD and Solaris return 0 here, so we do too.
3771 */
3772 return copy_to_user(arg, &ifc, sizeof(struct ifconf)) ? -EFAULT : 0;
3773}
3774
3775#ifdef CONFIG_PROC_FS
3776/*
3777 * This is invoked by the /proc filesystem handler to display a device
3778 * in detail.
3779 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07003780void *dev_seq_start(struct seq_file *seq, loff_t *pos)
Eric Dumazetc6d14c82009-11-04 05:43:23 -08003781 __acquires(RCU)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003782{
Denis V. Luneve372c412007-11-19 22:31:54 -08003783 struct net *net = seq_file_net(seq);
Pavel Emelianov7562f872007-05-03 15:13:45 -07003784 loff_t off;
3785 struct net_device *dev;
3786
Eric Dumazetc6d14c82009-11-04 05:43:23 -08003787 rcu_read_lock();
Pavel Emelianov7562f872007-05-03 15:13:45 -07003788 if (!*pos)
3789 return SEQ_START_TOKEN;
3790
3791 off = 1;
Eric Dumazetc6d14c82009-11-04 05:43:23 -08003792 for_each_netdev_rcu(net, dev)
Pavel Emelianov7562f872007-05-03 15:13:45 -07003793 if (off++ == *pos)
3794 return dev;
3795
3796 return NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003797}
3798
3799void *dev_seq_next(struct seq_file *seq, void *v, loff_t *pos)
3800{
Eric Dumazetc6d14c82009-11-04 05:43:23 -08003801 struct net_device *dev = (v == SEQ_START_TOKEN) ?
3802 first_net_device(seq_file_net(seq)) :
3803 next_net_device((struct net_device *)v);
3804
Linus Torvalds1da177e2005-04-16 15:20:36 -07003805 ++*pos;
Eric Dumazetc6d14c82009-11-04 05:43:23 -08003806 return rcu_dereference(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003807}
3808
3809void dev_seq_stop(struct seq_file *seq, void *v)
Eric Dumazetc6d14c82009-11-04 05:43:23 -08003810 __releases(RCU)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003811{
Eric Dumazetc6d14c82009-11-04 05:43:23 -08003812 rcu_read_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07003813}
3814
3815static void dev_seq_printf_stats(struct seq_file *seq, struct net_device *dev)
3816{
Eric Dumazet28172732010-07-07 14:58:56 -07003817 struct rtnl_link_stats64 temp;
3818 const struct rtnl_link_stats64 *stats = dev_get_stats(dev, &temp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003819
Ben Hutchingsbe1f3c22010-06-08 07:19:54 +00003820 seq_printf(seq, "%6s: %7llu %7llu %4llu %4llu %4llu %5llu %10llu %9llu "
3821 "%8llu %7llu %4llu %4llu %4llu %5llu %7llu %10llu\n",
Rusty Russell5a1b5892007-04-28 21:04:03 -07003822 dev->name, stats->rx_bytes, stats->rx_packets,
3823 stats->rx_errors,
3824 stats->rx_dropped + stats->rx_missed_errors,
3825 stats->rx_fifo_errors,
3826 stats->rx_length_errors + stats->rx_over_errors +
3827 stats->rx_crc_errors + stats->rx_frame_errors,
3828 stats->rx_compressed, stats->multicast,
3829 stats->tx_bytes, stats->tx_packets,
3830 stats->tx_errors, stats->tx_dropped,
3831 stats->tx_fifo_errors, stats->collisions,
3832 stats->tx_carrier_errors +
3833 stats->tx_aborted_errors +
3834 stats->tx_window_errors +
3835 stats->tx_heartbeat_errors,
3836 stats->tx_compressed);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003837}
3838
3839/*
3840 * Called from the PROCfs module. This now uses the new arbitrary sized
3841 * /proc/net interface to create /proc/net/dev
3842 */
3843static int dev_seq_show(struct seq_file *seq, void *v)
3844{
3845 if (v == SEQ_START_TOKEN)
3846 seq_puts(seq, "Inter-| Receive "
3847 " | Transmit\n"
3848 " face |bytes packets errs drop fifo frame "
3849 "compressed multicast|bytes packets errs "
3850 "drop fifo colls carrier compressed\n");
3851 else
3852 dev_seq_printf_stats(seq, v);
3853 return 0;
3854}
3855
Changli Gaodee42872010-05-02 05:42:16 +00003856static struct softnet_data *softnet_get_online(loff_t *pos)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003857{
Changli Gaodee42872010-05-02 05:42:16 +00003858 struct softnet_data *sd = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003859
Mike Travis0c0b0ac2008-05-02 16:43:08 -07003860 while (*pos < nr_cpu_ids)
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09003861 if (cpu_online(*pos)) {
Changli Gaodee42872010-05-02 05:42:16 +00003862 sd = &per_cpu(softnet_data, *pos);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003863 break;
3864 } else
3865 ++*pos;
Changli Gaodee42872010-05-02 05:42:16 +00003866 return sd;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003867}
3868
3869static void *softnet_seq_start(struct seq_file *seq, loff_t *pos)
3870{
3871 return softnet_get_online(pos);
3872}
3873
3874static void *softnet_seq_next(struct seq_file *seq, void *v, loff_t *pos)
3875{
3876 ++*pos;
3877 return softnet_get_online(pos);
3878}
3879
3880static void softnet_seq_stop(struct seq_file *seq, void *v)
3881{
3882}
3883
3884static int softnet_seq_show(struct seq_file *seq, void *v)
3885{
Changli Gaodee42872010-05-02 05:42:16 +00003886 struct softnet_data *sd = v;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003887
Tom Herbert0a9627f2010-03-16 08:03:29 +00003888 seq_printf(seq, "%08x %08x %08x %08x %08x %08x %08x %08x %08x %08x\n",
Changli Gaodee42872010-05-02 05:42:16 +00003889 sd->processed, sd->dropped, sd->time_squeeze, 0,
Stephen Hemmingerc1ebcdb2005-06-23 20:08:59 -07003890 0, 0, 0, 0, /* was fastroute */
Changli Gaodee42872010-05-02 05:42:16 +00003891 sd->cpu_collision, sd->received_rps);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003892 return 0;
3893}
3894
Stephen Hemmingerf6908082007-03-12 14:34:29 -07003895static const struct seq_operations dev_seq_ops = {
Linus Torvalds1da177e2005-04-16 15:20:36 -07003896 .start = dev_seq_start,
3897 .next = dev_seq_next,
3898 .stop = dev_seq_stop,
3899 .show = dev_seq_show,
3900};
3901
3902static int dev_seq_open(struct inode *inode, struct file *file)
3903{
Denis V. Luneve372c412007-11-19 22:31:54 -08003904 return seq_open_net(inode, file, &dev_seq_ops,
3905 sizeof(struct seq_net_private));
Linus Torvalds1da177e2005-04-16 15:20:36 -07003906}
3907
Arjan van de Ven9a321442007-02-12 00:55:35 -08003908static const struct file_operations dev_seq_fops = {
Linus Torvalds1da177e2005-04-16 15:20:36 -07003909 .owner = THIS_MODULE,
3910 .open = dev_seq_open,
3911 .read = seq_read,
3912 .llseek = seq_lseek,
Denis V. Luneve372c412007-11-19 22:31:54 -08003913 .release = seq_release_net,
Linus Torvalds1da177e2005-04-16 15:20:36 -07003914};
3915
Stephen Hemmingerf6908082007-03-12 14:34:29 -07003916static const struct seq_operations softnet_seq_ops = {
Linus Torvalds1da177e2005-04-16 15:20:36 -07003917 .start = softnet_seq_start,
3918 .next = softnet_seq_next,
3919 .stop = softnet_seq_stop,
3920 .show = softnet_seq_show,
3921};
3922
3923static int softnet_seq_open(struct inode *inode, struct file *file)
3924{
3925 return seq_open(file, &softnet_seq_ops);
3926}
3927
Arjan van de Ven9a321442007-02-12 00:55:35 -08003928static const struct file_operations softnet_seq_fops = {
Linus Torvalds1da177e2005-04-16 15:20:36 -07003929 .owner = THIS_MODULE,
3930 .open = softnet_seq_open,
3931 .read = seq_read,
3932 .llseek = seq_lseek,
3933 .release = seq_release,
3934};
3935
Stephen Hemminger0e1256f2007-03-12 14:35:37 -07003936static void *ptype_get_idx(loff_t pos)
3937{
3938 struct packet_type *pt = NULL;
3939 loff_t i = 0;
3940 int t;
3941
3942 list_for_each_entry_rcu(pt, &ptype_all, list) {
3943 if (i == pos)
3944 return pt;
3945 ++i;
3946 }
3947
Pavel Emelyanov82d8a8672007-11-26 20:12:58 +08003948 for (t = 0; t < PTYPE_HASH_SIZE; t++) {
Stephen Hemminger0e1256f2007-03-12 14:35:37 -07003949 list_for_each_entry_rcu(pt, &ptype_base[t], list) {
3950 if (i == pos)
3951 return pt;
3952 ++i;
3953 }
3954 }
3955 return NULL;
3956}
3957
3958static void *ptype_seq_start(struct seq_file *seq, loff_t *pos)
Stephen Hemminger72348a42008-01-21 02:27:29 -08003959 __acquires(RCU)
Stephen Hemminger0e1256f2007-03-12 14:35:37 -07003960{
3961 rcu_read_lock();
3962 return *pos ? ptype_get_idx(*pos - 1) : SEQ_START_TOKEN;
3963}
3964
3965static void *ptype_seq_next(struct seq_file *seq, void *v, loff_t *pos)
3966{
3967 struct packet_type *pt;
3968 struct list_head *nxt;
3969 int hash;
3970
3971 ++*pos;
3972 if (v == SEQ_START_TOKEN)
3973 return ptype_get_idx(0);
3974
3975 pt = v;
3976 nxt = pt->list.next;
3977 if (pt->type == htons(ETH_P_ALL)) {
3978 if (nxt != &ptype_all)
3979 goto found;
3980 hash = 0;
3981 nxt = ptype_base[0].next;
3982 } else
Pavel Emelyanov82d8a8672007-11-26 20:12:58 +08003983 hash = ntohs(pt->type) & PTYPE_HASH_MASK;
Stephen Hemminger0e1256f2007-03-12 14:35:37 -07003984
3985 while (nxt == &ptype_base[hash]) {
Pavel Emelyanov82d8a8672007-11-26 20:12:58 +08003986 if (++hash >= PTYPE_HASH_SIZE)
Stephen Hemminger0e1256f2007-03-12 14:35:37 -07003987 return NULL;
3988 nxt = ptype_base[hash].next;
3989 }
3990found:
3991 return list_entry(nxt, struct packet_type, list);
3992}
3993
3994static void ptype_seq_stop(struct seq_file *seq, void *v)
Stephen Hemminger72348a42008-01-21 02:27:29 -08003995 __releases(RCU)
Stephen Hemminger0e1256f2007-03-12 14:35:37 -07003996{
3997 rcu_read_unlock();
3998}
3999
Stephen Hemminger0e1256f2007-03-12 14:35:37 -07004000static int ptype_seq_show(struct seq_file *seq, void *v)
4001{
4002 struct packet_type *pt = v;
4003
4004 if (v == SEQ_START_TOKEN)
4005 seq_puts(seq, "Type Device Function\n");
YOSHIFUJI Hideakic346dca2008-03-25 21:47:49 +09004006 else if (pt->dev == NULL || dev_net(pt->dev) == seq_file_net(seq)) {
Stephen Hemminger0e1256f2007-03-12 14:35:37 -07004007 if (pt->type == htons(ETH_P_ALL))
4008 seq_puts(seq, "ALL ");
4009 else
4010 seq_printf(seq, "%04x", ntohs(pt->type));
4011
Alexey Dobriyan908cd2d2008-11-16 19:50:35 -08004012 seq_printf(seq, " %-8s %pF\n",
4013 pt->dev ? pt->dev->name : "", pt->func);
Stephen Hemminger0e1256f2007-03-12 14:35:37 -07004014 }
4015
4016 return 0;
4017}
4018
4019static const struct seq_operations ptype_seq_ops = {
4020 .start = ptype_seq_start,
4021 .next = ptype_seq_next,
4022 .stop = ptype_seq_stop,
4023 .show = ptype_seq_show,
4024};
4025
4026static int ptype_seq_open(struct inode *inode, struct file *file)
4027{
Pavel Emelyanov2feb27d2008-03-24 14:57:45 -07004028 return seq_open_net(inode, file, &ptype_seq_ops,
4029 sizeof(struct seq_net_private));
Stephen Hemminger0e1256f2007-03-12 14:35:37 -07004030}
4031
4032static const struct file_operations ptype_seq_fops = {
4033 .owner = THIS_MODULE,
4034 .open = ptype_seq_open,
4035 .read = seq_read,
4036 .llseek = seq_lseek,
Pavel Emelyanov2feb27d2008-03-24 14:57:45 -07004037 .release = seq_release_net,
Stephen Hemminger0e1256f2007-03-12 14:35:37 -07004038};
4039
4040
Pavel Emelyanov46650792007-10-08 20:38:39 -07004041static int __net_init dev_proc_net_init(struct net *net)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004042{
4043 int rc = -ENOMEM;
4044
Eric W. Biederman881d9662007-09-17 11:56:21 -07004045 if (!proc_net_fops_create(net, "dev", S_IRUGO, &dev_seq_fops))
Linus Torvalds1da177e2005-04-16 15:20:36 -07004046 goto out;
Eric W. Biederman881d9662007-09-17 11:56:21 -07004047 if (!proc_net_fops_create(net, "softnet_stat", S_IRUGO, &softnet_seq_fops))
Linus Torvalds1da177e2005-04-16 15:20:36 -07004048 goto out_dev;
Eric W. Biederman881d9662007-09-17 11:56:21 -07004049 if (!proc_net_fops_create(net, "ptype", S_IRUGO, &ptype_seq_fops))
Eric W. Biederman457c4cb2007-09-12 12:01:34 +02004050 goto out_softnet;
Stephen Hemminger0e1256f2007-03-12 14:35:37 -07004051
Eric W. Biederman881d9662007-09-17 11:56:21 -07004052 if (wext_proc_init(net))
Eric W. Biederman457c4cb2007-09-12 12:01:34 +02004053 goto out_ptype;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004054 rc = 0;
4055out:
4056 return rc;
Eric W. Biederman457c4cb2007-09-12 12:01:34 +02004057out_ptype:
Eric W. Biederman881d9662007-09-17 11:56:21 -07004058 proc_net_remove(net, "ptype");
Linus Torvalds1da177e2005-04-16 15:20:36 -07004059out_softnet:
Eric W. Biederman881d9662007-09-17 11:56:21 -07004060 proc_net_remove(net, "softnet_stat");
Linus Torvalds1da177e2005-04-16 15:20:36 -07004061out_dev:
Eric W. Biederman881d9662007-09-17 11:56:21 -07004062 proc_net_remove(net, "dev");
Linus Torvalds1da177e2005-04-16 15:20:36 -07004063 goto out;
4064}
Eric W. Biederman881d9662007-09-17 11:56:21 -07004065
Pavel Emelyanov46650792007-10-08 20:38:39 -07004066static void __net_exit dev_proc_net_exit(struct net *net)
Eric W. Biederman881d9662007-09-17 11:56:21 -07004067{
4068 wext_proc_exit(net);
4069
4070 proc_net_remove(net, "ptype");
4071 proc_net_remove(net, "softnet_stat");
4072 proc_net_remove(net, "dev");
4073}
4074
Denis V. Lunev022cbae2007-11-13 03:23:50 -08004075static struct pernet_operations __net_initdata dev_proc_ops = {
Eric W. Biederman881d9662007-09-17 11:56:21 -07004076 .init = dev_proc_net_init,
4077 .exit = dev_proc_net_exit,
4078};
4079
4080static int __init dev_proc_init(void)
4081{
4082 return register_pernet_subsys(&dev_proc_ops);
4083}
Linus Torvalds1da177e2005-04-16 15:20:36 -07004084#else
4085#define dev_proc_init() 0
4086#endif /* CONFIG_PROC_FS */
4087
4088
4089/**
4090 * netdev_set_master - set up master/slave pair
4091 * @slave: slave device
4092 * @master: new master device
4093 *
4094 * Changes the master device of the slave. Pass %NULL to break the
4095 * bonding. The caller must hold the RTNL semaphore. On a failure
4096 * a negative errno code is returned. On success the reference counts
4097 * are adjusted, %RTM_NEWLINK is sent to the routing socket and the
4098 * function returns zero.
4099 */
4100int netdev_set_master(struct net_device *slave, struct net_device *master)
4101{
4102 struct net_device *old = slave->master;
4103
4104 ASSERT_RTNL();
4105
4106 if (master) {
4107 if (old)
4108 return -EBUSY;
4109 dev_hold(master);
4110 }
4111
4112 slave->master = master;
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09004113
Eric Dumazet283f2fe2010-03-18 13:37:40 +00004114 if (old) {
4115 synchronize_net();
Linus Torvalds1da177e2005-04-16 15:20:36 -07004116 dev_put(old);
Eric Dumazet283f2fe2010-03-18 13:37:40 +00004117 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07004118 if (master)
4119 slave->flags |= IFF_SLAVE;
4120 else
4121 slave->flags &= ~IFF_SLAVE;
4122
4123 rtmsg_ifinfo(RTM_NEWLINK, slave, IFF_SLAVE);
4124 return 0;
4125}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004126EXPORT_SYMBOL(netdev_set_master);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004127
Patrick McHardyb6c40d62008-10-07 15:26:48 -07004128static void dev_change_rx_flags(struct net_device *dev, int flags)
4129{
Stephen Hemmingerd3147742008-11-19 21:32:24 -08004130 const struct net_device_ops *ops = dev->netdev_ops;
4131
4132 if ((dev->flags & IFF_UP) && ops->ndo_change_rx_flags)
4133 ops->ndo_change_rx_flags(dev, flags);
Patrick McHardyb6c40d62008-10-07 15:26:48 -07004134}
4135
Wang Chendad9b332008-06-18 01:48:28 -07004136static int __dev_set_promiscuity(struct net_device *dev, int inc)
Patrick McHardy4417da62007-06-27 01:28:10 -07004137{
4138 unsigned short old_flags = dev->flags;
David Howells8192b0c2008-11-14 10:39:10 +11004139 uid_t uid;
4140 gid_t gid;
Patrick McHardy4417da62007-06-27 01:28:10 -07004141
Patrick McHardy24023452007-07-14 18:51:31 -07004142 ASSERT_RTNL();
4143
Wang Chendad9b332008-06-18 01:48:28 -07004144 dev->flags |= IFF_PROMISC;
4145 dev->promiscuity += inc;
4146 if (dev->promiscuity == 0) {
4147 /*
4148 * Avoid overflow.
4149 * If inc causes overflow, untouch promisc and return error.
4150 */
4151 if (inc < 0)
4152 dev->flags &= ~IFF_PROMISC;
4153 else {
4154 dev->promiscuity -= inc;
4155 printk(KERN_WARNING "%s: promiscuity touches roof, "
4156 "set promiscuity failed, promiscuity feature "
4157 "of device might be broken.\n", dev->name);
4158 return -EOVERFLOW;
4159 }
4160 }
Patrick McHardy4417da62007-06-27 01:28:10 -07004161 if (dev->flags != old_flags) {
4162 printk(KERN_INFO "device %s %s promiscuous mode\n",
4163 dev->name, (dev->flags & IFF_PROMISC) ? "entered" :
4164 "left");
David Howells8192b0c2008-11-14 10:39:10 +11004165 if (audit_enabled) {
4166 current_uid_gid(&uid, &gid);
Klaus Heinrich Kiwi7759db82008-01-23 22:57:45 -05004167 audit_log(current->audit_context, GFP_ATOMIC,
4168 AUDIT_ANOM_PROMISCUOUS,
4169 "dev=%s prom=%d old_prom=%d auid=%u uid=%u gid=%u ses=%u",
4170 dev->name, (dev->flags & IFF_PROMISC),
4171 (old_flags & IFF_PROMISC),
4172 audit_get_loginuid(current),
David Howells8192b0c2008-11-14 10:39:10 +11004173 uid, gid,
Klaus Heinrich Kiwi7759db82008-01-23 22:57:45 -05004174 audit_get_sessionid(current));
David Howells8192b0c2008-11-14 10:39:10 +11004175 }
Patrick McHardy24023452007-07-14 18:51:31 -07004176
Patrick McHardyb6c40d62008-10-07 15:26:48 -07004177 dev_change_rx_flags(dev, IFF_PROMISC);
Patrick McHardy4417da62007-06-27 01:28:10 -07004178 }
Wang Chendad9b332008-06-18 01:48:28 -07004179 return 0;
Patrick McHardy4417da62007-06-27 01:28:10 -07004180}
4181
Linus Torvalds1da177e2005-04-16 15:20:36 -07004182/**
4183 * dev_set_promiscuity - update promiscuity count on a device
4184 * @dev: device
4185 * @inc: modifier
4186 *
Stephen Hemminger3041a062006-05-26 13:25:24 -07004187 * Add or remove promiscuity from a device. While the count in the device
Linus Torvalds1da177e2005-04-16 15:20:36 -07004188 * remains above zero the interface remains promiscuous. Once it hits zero
4189 * the device reverts back to normal filtering operation. A negative inc
4190 * value is used to drop promiscuity on the device.
Wang Chendad9b332008-06-18 01:48:28 -07004191 * Return 0 if successful or a negative errno code on error.
Linus Torvalds1da177e2005-04-16 15:20:36 -07004192 */
Wang Chendad9b332008-06-18 01:48:28 -07004193int dev_set_promiscuity(struct net_device *dev, int inc)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004194{
4195 unsigned short old_flags = dev->flags;
Wang Chendad9b332008-06-18 01:48:28 -07004196 int err;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004197
Wang Chendad9b332008-06-18 01:48:28 -07004198 err = __dev_set_promiscuity(dev, inc);
Patrick McHardy4b5a6982008-07-06 15:49:08 -07004199 if (err < 0)
Wang Chendad9b332008-06-18 01:48:28 -07004200 return err;
Patrick McHardy4417da62007-06-27 01:28:10 -07004201 if (dev->flags != old_flags)
4202 dev_set_rx_mode(dev);
Wang Chendad9b332008-06-18 01:48:28 -07004203 return err;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004204}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004205EXPORT_SYMBOL(dev_set_promiscuity);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004206
4207/**
4208 * dev_set_allmulti - update allmulti count on a device
4209 * @dev: device
4210 * @inc: modifier
4211 *
4212 * Add or remove reception of all multicast frames to a device. While the
4213 * count in the device remains above zero the interface remains listening
4214 * to all interfaces. Once it hits zero the device reverts back to normal
4215 * filtering operation. A negative @inc value is used to drop the counter
4216 * when releasing a resource needing all multicasts.
Wang Chendad9b332008-06-18 01:48:28 -07004217 * Return 0 if successful or a negative errno code on error.
Linus Torvalds1da177e2005-04-16 15:20:36 -07004218 */
4219
Wang Chendad9b332008-06-18 01:48:28 -07004220int dev_set_allmulti(struct net_device *dev, int inc)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004221{
4222 unsigned short old_flags = dev->flags;
4223
Patrick McHardy24023452007-07-14 18:51:31 -07004224 ASSERT_RTNL();
4225
Linus Torvalds1da177e2005-04-16 15:20:36 -07004226 dev->flags |= IFF_ALLMULTI;
Wang Chendad9b332008-06-18 01:48:28 -07004227 dev->allmulti += inc;
4228 if (dev->allmulti == 0) {
4229 /*
4230 * Avoid overflow.
4231 * If inc causes overflow, untouch allmulti and return error.
4232 */
4233 if (inc < 0)
4234 dev->flags &= ~IFF_ALLMULTI;
4235 else {
4236 dev->allmulti -= inc;
4237 printk(KERN_WARNING "%s: allmulti touches roof, "
4238 "set allmulti failed, allmulti feature of "
4239 "device might be broken.\n", dev->name);
4240 return -EOVERFLOW;
4241 }
4242 }
Patrick McHardy24023452007-07-14 18:51:31 -07004243 if (dev->flags ^ old_flags) {
Patrick McHardyb6c40d62008-10-07 15:26:48 -07004244 dev_change_rx_flags(dev, IFF_ALLMULTI);
Patrick McHardy4417da62007-06-27 01:28:10 -07004245 dev_set_rx_mode(dev);
Patrick McHardy24023452007-07-14 18:51:31 -07004246 }
Wang Chendad9b332008-06-18 01:48:28 -07004247 return 0;
Patrick McHardy4417da62007-06-27 01:28:10 -07004248}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004249EXPORT_SYMBOL(dev_set_allmulti);
Patrick McHardy4417da62007-06-27 01:28:10 -07004250
4251/*
4252 * Upload unicast and multicast address lists to device and
4253 * configure RX filtering. When the device doesn't support unicast
Joe Perches53ccaae2007-12-20 14:02:06 -08004254 * filtering it is put in promiscuous mode while unicast addresses
Patrick McHardy4417da62007-06-27 01:28:10 -07004255 * are present.
4256 */
4257void __dev_set_rx_mode(struct net_device *dev)
4258{
Stephen Hemmingerd3147742008-11-19 21:32:24 -08004259 const struct net_device_ops *ops = dev->netdev_ops;
4260
Patrick McHardy4417da62007-06-27 01:28:10 -07004261 /* dev_open will call this function so the list will stay sane. */
4262 if (!(dev->flags&IFF_UP))
4263 return;
4264
4265 if (!netif_device_present(dev))
YOSHIFUJI Hideaki40b77c92007-07-19 10:43:23 +09004266 return;
Patrick McHardy4417da62007-06-27 01:28:10 -07004267
Stephen Hemmingerd3147742008-11-19 21:32:24 -08004268 if (ops->ndo_set_rx_mode)
4269 ops->ndo_set_rx_mode(dev);
Patrick McHardy4417da62007-06-27 01:28:10 -07004270 else {
4271 /* Unicast addresses changes may only happen under the rtnl,
4272 * therefore calling __dev_set_promiscuity here is safe.
4273 */
Jiri Pirko32e7bfc2010-01-25 13:36:10 -08004274 if (!netdev_uc_empty(dev) && !dev->uc_promisc) {
Patrick McHardy4417da62007-06-27 01:28:10 -07004275 __dev_set_promiscuity(dev, 1);
4276 dev->uc_promisc = 1;
Jiri Pirko32e7bfc2010-01-25 13:36:10 -08004277 } else if (netdev_uc_empty(dev) && dev->uc_promisc) {
Patrick McHardy4417da62007-06-27 01:28:10 -07004278 __dev_set_promiscuity(dev, -1);
4279 dev->uc_promisc = 0;
4280 }
4281
Stephen Hemmingerd3147742008-11-19 21:32:24 -08004282 if (ops->ndo_set_multicast_list)
4283 ops->ndo_set_multicast_list(dev);
Patrick McHardy4417da62007-06-27 01:28:10 -07004284 }
4285}
4286
4287void dev_set_rx_mode(struct net_device *dev)
4288{
David S. Millerb9e40852008-07-15 00:15:08 -07004289 netif_addr_lock_bh(dev);
Patrick McHardy4417da62007-06-27 01:28:10 -07004290 __dev_set_rx_mode(dev);
David S. Millerb9e40852008-07-15 00:15:08 -07004291 netif_addr_unlock_bh(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004292}
4293
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07004294/**
4295 * dev_get_flags - get flags reported to userspace
4296 * @dev: device
4297 *
4298 * Get the combination of flag bits exported through APIs to userspace.
4299 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07004300unsigned dev_get_flags(const struct net_device *dev)
4301{
4302 unsigned flags;
4303
4304 flags = (dev->flags & ~(IFF_PROMISC |
4305 IFF_ALLMULTI |
Stefan Rompfb00055a2006-03-20 17:09:11 -08004306 IFF_RUNNING |
4307 IFF_LOWER_UP |
4308 IFF_DORMANT)) |
Linus Torvalds1da177e2005-04-16 15:20:36 -07004309 (dev->gflags & (IFF_PROMISC |
4310 IFF_ALLMULTI));
4311
Stefan Rompfb00055a2006-03-20 17:09:11 -08004312 if (netif_running(dev)) {
4313 if (netif_oper_up(dev))
4314 flags |= IFF_RUNNING;
4315 if (netif_carrier_ok(dev))
4316 flags |= IFF_LOWER_UP;
4317 if (netif_dormant(dev))
4318 flags |= IFF_DORMANT;
4319 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07004320
4321 return flags;
4322}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004323EXPORT_SYMBOL(dev_get_flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004324
Patrick McHardybd380812010-02-26 06:34:53 +00004325int __dev_change_flags(struct net_device *dev, unsigned int flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004326{
Linus Torvalds1da177e2005-04-16 15:20:36 -07004327 int old_flags = dev->flags;
Patrick McHardybd380812010-02-26 06:34:53 +00004328 int ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004329
Patrick McHardy24023452007-07-14 18:51:31 -07004330 ASSERT_RTNL();
4331
Linus Torvalds1da177e2005-04-16 15:20:36 -07004332 /*
4333 * Set the flags on our device.
4334 */
4335
4336 dev->flags = (flags & (IFF_DEBUG | IFF_NOTRAILERS | IFF_NOARP |
4337 IFF_DYNAMIC | IFF_MULTICAST | IFF_PORTSEL |
4338 IFF_AUTOMEDIA)) |
4339 (dev->flags & (IFF_UP | IFF_VOLATILE | IFF_PROMISC |
4340 IFF_ALLMULTI));
4341
4342 /*
4343 * Load in the correct multicast list now the flags have changed.
4344 */
4345
Patrick McHardyb6c40d62008-10-07 15:26:48 -07004346 if ((old_flags ^ flags) & IFF_MULTICAST)
4347 dev_change_rx_flags(dev, IFF_MULTICAST);
Patrick McHardy24023452007-07-14 18:51:31 -07004348
Patrick McHardy4417da62007-06-27 01:28:10 -07004349 dev_set_rx_mode(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004350
4351 /*
4352 * Have we downed the interface. We handle IFF_UP ourselves
4353 * according to user attempts to set it, rather than blindly
4354 * setting it.
4355 */
4356
4357 ret = 0;
4358 if ((old_flags ^ flags) & IFF_UP) { /* Bit is different ? */
Patrick McHardybd380812010-02-26 06:34:53 +00004359 ret = ((old_flags & IFF_UP) ? __dev_close : __dev_open)(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004360
4361 if (!ret)
Patrick McHardy4417da62007-06-27 01:28:10 -07004362 dev_set_rx_mode(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004363 }
4364
Linus Torvalds1da177e2005-04-16 15:20:36 -07004365 if ((flags ^ dev->gflags) & IFF_PROMISC) {
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004366 int inc = (flags & IFF_PROMISC) ? 1 : -1;
4367
Linus Torvalds1da177e2005-04-16 15:20:36 -07004368 dev->gflags ^= IFF_PROMISC;
4369 dev_set_promiscuity(dev, inc);
4370 }
4371
4372 /* NOTE: order of synchronization of IFF_PROMISC and IFF_ALLMULTI
4373 is important. Some (broken) drivers set IFF_PROMISC, when
4374 IFF_ALLMULTI is requested not asking us and not reporting.
4375 */
4376 if ((flags ^ dev->gflags) & IFF_ALLMULTI) {
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004377 int inc = (flags & IFF_ALLMULTI) ? 1 : -1;
4378
Linus Torvalds1da177e2005-04-16 15:20:36 -07004379 dev->gflags ^= IFF_ALLMULTI;
4380 dev_set_allmulti(dev, inc);
4381 }
4382
Patrick McHardybd380812010-02-26 06:34:53 +00004383 return ret;
4384}
4385
4386void __dev_notify_flags(struct net_device *dev, unsigned int old_flags)
4387{
4388 unsigned int changes = dev->flags ^ old_flags;
4389
4390 if (changes & IFF_UP) {
4391 if (dev->flags & IFF_UP)
4392 call_netdevice_notifiers(NETDEV_UP, dev);
4393 else
4394 call_netdevice_notifiers(NETDEV_DOWN, dev);
4395 }
4396
4397 if (dev->flags & IFF_UP &&
4398 (changes & ~(IFF_UP | IFF_PROMISC | IFF_ALLMULTI | IFF_VOLATILE)))
4399 call_netdevice_notifiers(NETDEV_CHANGE, dev);
4400}
4401
4402/**
4403 * dev_change_flags - change device settings
4404 * @dev: device
4405 * @flags: device state flags
4406 *
4407 * Change settings on device based state flags. The flags are
4408 * in the userspace exported format.
4409 */
4410int dev_change_flags(struct net_device *dev, unsigned flags)
4411{
4412 int ret, changes;
4413 int old_flags = dev->flags;
4414
4415 ret = __dev_change_flags(dev, flags);
4416 if (ret < 0)
4417 return ret;
4418
4419 changes = old_flags ^ dev->flags;
Thomas Graf7c355f52007-06-05 16:03:03 -07004420 if (changes)
4421 rtmsg_ifinfo(RTM_NEWLINK, dev, changes);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004422
Patrick McHardybd380812010-02-26 06:34:53 +00004423 __dev_notify_flags(dev, old_flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004424 return ret;
4425}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004426EXPORT_SYMBOL(dev_change_flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004427
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07004428/**
4429 * dev_set_mtu - Change maximum transfer unit
4430 * @dev: device
4431 * @new_mtu: new transfer unit
4432 *
4433 * Change the maximum transfer size of the network device.
4434 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07004435int dev_set_mtu(struct net_device *dev, int new_mtu)
4436{
Stephen Hemmingerd3147742008-11-19 21:32:24 -08004437 const struct net_device_ops *ops = dev->netdev_ops;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004438 int err;
4439
4440 if (new_mtu == dev->mtu)
4441 return 0;
4442
4443 /* MTU must be positive. */
4444 if (new_mtu < 0)
4445 return -EINVAL;
4446
4447 if (!netif_device_present(dev))
4448 return -ENODEV;
4449
4450 err = 0;
Stephen Hemmingerd3147742008-11-19 21:32:24 -08004451 if (ops->ndo_change_mtu)
4452 err = ops->ndo_change_mtu(dev, new_mtu);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004453 else
4454 dev->mtu = new_mtu;
Stephen Hemmingerd3147742008-11-19 21:32:24 -08004455
Linus Torvalds1da177e2005-04-16 15:20:36 -07004456 if (!err && dev->flags & IFF_UP)
Pavel Emelyanov056925a2007-09-16 15:42:43 -07004457 call_netdevice_notifiers(NETDEV_CHANGEMTU, dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004458 return err;
4459}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004460EXPORT_SYMBOL(dev_set_mtu);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004461
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07004462/**
4463 * dev_set_mac_address - Change Media Access Control Address
4464 * @dev: device
4465 * @sa: new address
4466 *
4467 * Change the hardware (MAC) address of the device
4468 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07004469int dev_set_mac_address(struct net_device *dev, struct sockaddr *sa)
4470{
Stephen Hemmingerd3147742008-11-19 21:32:24 -08004471 const struct net_device_ops *ops = dev->netdev_ops;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004472 int err;
4473
Stephen Hemmingerd3147742008-11-19 21:32:24 -08004474 if (!ops->ndo_set_mac_address)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004475 return -EOPNOTSUPP;
4476 if (sa->sa_family != dev->type)
4477 return -EINVAL;
4478 if (!netif_device_present(dev))
4479 return -ENODEV;
Stephen Hemmingerd3147742008-11-19 21:32:24 -08004480 err = ops->ndo_set_mac_address(dev, sa);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004481 if (!err)
Pavel Emelyanov056925a2007-09-16 15:42:43 -07004482 call_netdevice_notifiers(NETDEV_CHANGEADDR, dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004483 return err;
4484}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004485EXPORT_SYMBOL(dev_set_mac_address);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004486
4487/*
Eric Dumazet3710bec2009-11-01 19:42:09 +00004488 * Perform the SIOCxIFxxx calls, inside rcu_read_lock()
Linus Torvalds1da177e2005-04-16 15:20:36 -07004489 */
Jeff Garzik14e3e072007-10-08 00:06:32 -07004490static int dev_ifsioc_locked(struct net *net, struct ifreq *ifr, unsigned int cmd)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004491{
4492 int err;
Eric Dumazet3710bec2009-11-01 19:42:09 +00004493 struct net_device *dev = dev_get_by_name_rcu(net, ifr->ifr_name);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004494
4495 if (!dev)
4496 return -ENODEV;
4497
4498 switch (cmd) {
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004499 case SIOCGIFFLAGS: /* Get interface flags */
4500 ifr->ifr_flags = (short) dev_get_flags(dev);
4501 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004502
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004503 case SIOCGIFMETRIC: /* Get the metric on the interface
4504 (currently unused) */
4505 ifr->ifr_metric = 0;
4506 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004507
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004508 case SIOCGIFMTU: /* Get the MTU of a device */
4509 ifr->ifr_mtu = dev->mtu;
4510 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004511
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004512 case SIOCGIFHWADDR:
4513 if (!dev->addr_len)
4514 memset(ifr->ifr_hwaddr.sa_data, 0, sizeof ifr->ifr_hwaddr.sa_data);
4515 else
4516 memcpy(ifr->ifr_hwaddr.sa_data, dev->dev_addr,
4517 min(sizeof ifr->ifr_hwaddr.sa_data, (size_t) dev->addr_len));
4518 ifr->ifr_hwaddr.sa_family = dev->type;
4519 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004520
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004521 case SIOCGIFSLAVE:
4522 err = -EINVAL;
4523 break;
Jeff Garzik14e3e072007-10-08 00:06:32 -07004524
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004525 case SIOCGIFMAP:
4526 ifr->ifr_map.mem_start = dev->mem_start;
4527 ifr->ifr_map.mem_end = dev->mem_end;
4528 ifr->ifr_map.base_addr = dev->base_addr;
4529 ifr->ifr_map.irq = dev->irq;
4530 ifr->ifr_map.dma = dev->dma;
4531 ifr->ifr_map.port = dev->if_port;
4532 return 0;
Jeff Garzik14e3e072007-10-08 00:06:32 -07004533
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004534 case SIOCGIFINDEX:
4535 ifr->ifr_ifindex = dev->ifindex;
4536 return 0;
Jeff Garzik14e3e072007-10-08 00:06:32 -07004537
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004538 case SIOCGIFTXQLEN:
4539 ifr->ifr_qlen = dev->tx_queue_len;
4540 return 0;
Jeff Garzik14e3e072007-10-08 00:06:32 -07004541
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004542 default:
4543 /* dev_ioctl() should ensure this case
4544 * is never reached
4545 */
4546 WARN_ON(1);
4547 err = -EINVAL;
4548 break;
Jeff Garzik14e3e072007-10-08 00:06:32 -07004549
4550 }
4551 return err;
4552}
4553
4554/*
4555 * Perform the SIOCxIFxxx calls, inside rtnl_lock()
4556 */
4557static int dev_ifsioc(struct net *net, struct ifreq *ifr, unsigned int cmd)
4558{
4559 int err;
4560 struct net_device *dev = __dev_get_by_name(net, ifr->ifr_name);
Jarek Poplawski5f2f6da2008-12-22 19:35:28 -08004561 const struct net_device_ops *ops;
Jeff Garzik14e3e072007-10-08 00:06:32 -07004562
4563 if (!dev)
4564 return -ENODEV;
4565
Jarek Poplawski5f2f6da2008-12-22 19:35:28 -08004566 ops = dev->netdev_ops;
4567
Jeff Garzik14e3e072007-10-08 00:06:32 -07004568 switch (cmd) {
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004569 case SIOCSIFFLAGS: /* Set interface flags */
4570 return dev_change_flags(dev, ifr->ifr_flags);
Jeff Garzik14e3e072007-10-08 00:06:32 -07004571
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004572 case SIOCSIFMETRIC: /* Set the metric on the interface
4573 (currently unused) */
4574 return -EOPNOTSUPP;
Jeff Garzik14e3e072007-10-08 00:06:32 -07004575
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004576 case SIOCSIFMTU: /* Set the MTU of a device */
4577 return dev_set_mtu(dev, ifr->ifr_mtu);
Jeff Garzik14e3e072007-10-08 00:06:32 -07004578
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004579 case SIOCSIFHWADDR:
4580 return dev_set_mac_address(dev, &ifr->ifr_hwaddr);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004581
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004582 case SIOCSIFHWBROADCAST:
4583 if (ifr->ifr_hwaddr.sa_family != dev->type)
4584 return -EINVAL;
4585 memcpy(dev->broadcast, ifr->ifr_hwaddr.sa_data,
4586 min(sizeof ifr->ifr_hwaddr.sa_data, (size_t) dev->addr_len));
4587 call_netdevice_notifiers(NETDEV_CHANGEADDR, dev);
4588 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004589
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004590 case SIOCSIFMAP:
4591 if (ops->ndo_set_config) {
4592 if (!netif_device_present(dev))
4593 return -ENODEV;
4594 return ops->ndo_set_config(dev, &ifr->ifr_map);
4595 }
4596 return -EOPNOTSUPP;
4597
4598 case SIOCADDMULTI:
4599 if ((!ops->ndo_set_multicast_list && !ops->ndo_set_rx_mode) ||
4600 ifr->ifr_hwaddr.sa_family != AF_UNSPEC)
4601 return -EINVAL;
4602 if (!netif_device_present(dev))
4603 return -ENODEV;
Jiri Pirko22bedad32010-04-01 21:22:57 +00004604 return dev_mc_add_global(dev, ifr->ifr_hwaddr.sa_data);
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004605
4606 case SIOCDELMULTI:
4607 if ((!ops->ndo_set_multicast_list && !ops->ndo_set_rx_mode) ||
4608 ifr->ifr_hwaddr.sa_family != AF_UNSPEC)
4609 return -EINVAL;
4610 if (!netif_device_present(dev))
4611 return -ENODEV;
Jiri Pirko22bedad32010-04-01 21:22:57 +00004612 return dev_mc_del_global(dev, ifr->ifr_hwaddr.sa_data);
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004613
4614 case SIOCSIFTXQLEN:
4615 if (ifr->ifr_qlen < 0)
4616 return -EINVAL;
4617 dev->tx_queue_len = ifr->ifr_qlen;
4618 return 0;
4619
4620 case SIOCSIFNAME:
4621 ifr->ifr_newname[IFNAMSIZ-1] = '\0';
4622 return dev_change_name(dev, ifr->ifr_newname);
4623
4624 /*
4625 * Unknown or private ioctl
4626 */
4627 default:
4628 if ((cmd >= SIOCDEVPRIVATE &&
4629 cmd <= SIOCDEVPRIVATE + 15) ||
4630 cmd == SIOCBONDENSLAVE ||
4631 cmd == SIOCBONDRELEASE ||
4632 cmd == SIOCBONDSETHWADDR ||
4633 cmd == SIOCBONDSLAVEINFOQUERY ||
4634 cmd == SIOCBONDINFOQUERY ||
4635 cmd == SIOCBONDCHANGEACTIVE ||
4636 cmd == SIOCGMIIPHY ||
4637 cmd == SIOCGMIIREG ||
4638 cmd == SIOCSMIIREG ||
4639 cmd == SIOCBRADDIF ||
4640 cmd == SIOCBRDELIF ||
4641 cmd == SIOCSHWTSTAMP ||
4642 cmd == SIOCWANDEV) {
4643 err = -EOPNOTSUPP;
4644 if (ops->ndo_do_ioctl) {
4645 if (netif_device_present(dev))
4646 err = ops->ndo_do_ioctl(dev, ifr, cmd);
4647 else
4648 err = -ENODEV;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004649 }
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004650 } else
4651 err = -EINVAL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004652
4653 }
4654 return err;
4655}
4656
4657/*
4658 * This function handles all "interface"-type I/O control requests. The actual
4659 * 'doing' part of this is dev_ifsioc above.
4660 */
4661
4662/**
4663 * dev_ioctl - network device ioctl
Randy Dunlapc4ea43c2007-10-12 21:17:49 -07004664 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -07004665 * @cmd: command to issue
4666 * @arg: pointer to a struct ifreq in user space
4667 *
4668 * Issue ioctl functions to devices. This is normally called by the
4669 * user space syscall interfaces but can sometimes be useful for
4670 * other purposes. The return value is the return from the syscall if
4671 * positive or a negative errno code on error.
4672 */
4673
Eric W. Biederman881d9662007-09-17 11:56:21 -07004674int dev_ioctl(struct net *net, unsigned int cmd, void __user *arg)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004675{
4676 struct ifreq ifr;
4677 int ret;
4678 char *colon;
4679
4680 /* One special case: SIOCGIFCONF takes ifconf argument
4681 and requires shared lock, because it sleeps writing
4682 to user space.
4683 */
4684
4685 if (cmd == SIOCGIFCONF) {
Stephen Hemminger6756ae42006-03-20 22:23:58 -08004686 rtnl_lock();
Eric W. Biederman881d9662007-09-17 11:56:21 -07004687 ret = dev_ifconf(net, (char __user *) arg);
Stephen Hemminger6756ae42006-03-20 22:23:58 -08004688 rtnl_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07004689 return ret;
4690 }
4691 if (cmd == SIOCGIFNAME)
Eric W. Biederman881d9662007-09-17 11:56:21 -07004692 return dev_ifname(net, (struct ifreq __user *)arg);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004693
4694 if (copy_from_user(&ifr, arg, sizeof(struct ifreq)))
4695 return -EFAULT;
4696
4697 ifr.ifr_name[IFNAMSIZ-1] = 0;
4698
4699 colon = strchr(ifr.ifr_name, ':');
4700 if (colon)
4701 *colon = 0;
4702
4703 /*
4704 * See which interface the caller is talking about.
4705 */
4706
4707 switch (cmd) {
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004708 /*
4709 * These ioctl calls:
4710 * - can be done by all.
4711 * - atomic and do not require locking.
4712 * - return a value
4713 */
4714 case SIOCGIFFLAGS:
4715 case SIOCGIFMETRIC:
4716 case SIOCGIFMTU:
4717 case SIOCGIFHWADDR:
4718 case SIOCGIFSLAVE:
4719 case SIOCGIFMAP:
4720 case SIOCGIFINDEX:
4721 case SIOCGIFTXQLEN:
4722 dev_load(net, ifr.ifr_name);
Eric Dumazet3710bec2009-11-01 19:42:09 +00004723 rcu_read_lock();
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004724 ret = dev_ifsioc_locked(net, &ifr, cmd);
Eric Dumazet3710bec2009-11-01 19:42:09 +00004725 rcu_read_unlock();
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004726 if (!ret) {
4727 if (colon)
4728 *colon = ':';
4729 if (copy_to_user(arg, &ifr,
4730 sizeof(struct ifreq)))
4731 ret = -EFAULT;
4732 }
4733 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004734
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004735 case SIOCETHTOOL:
4736 dev_load(net, ifr.ifr_name);
4737 rtnl_lock();
4738 ret = dev_ethtool(net, &ifr);
4739 rtnl_unlock();
4740 if (!ret) {
4741 if (colon)
4742 *colon = ':';
4743 if (copy_to_user(arg, &ifr,
4744 sizeof(struct ifreq)))
4745 ret = -EFAULT;
4746 }
4747 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004748
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004749 /*
4750 * These ioctl calls:
4751 * - require superuser power.
4752 * - require strict serialization.
4753 * - return a value
4754 */
4755 case SIOCGMIIPHY:
4756 case SIOCGMIIREG:
4757 case SIOCSIFNAME:
4758 if (!capable(CAP_NET_ADMIN))
4759 return -EPERM;
4760 dev_load(net, ifr.ifr_name);
4761 rtnl_lock();
4762 ret = dev_ifsioc(net, &ifr, cmd);
4763 rtnl_unlock();
4764 if (!ret) {
4765 if (colon)
4766 *colon = ':';
4767 if (copy_to_user(arg, &ifr,
4768 sizeof(struct ifreq)))
4769 ret = -EFAULT;
4770 }
4771 return ret;
4772
4773 /*
4774 * These ioctl calls:
4775 * - require superuser power.
4776 * - require strict serialization.
4777 * - do not return a value
4778 */
4779 case SIOCSIFFLAGS:
4780 case SIOCSIFMETRIC:
4781 case SIOCSIFMTU:
4782 case SIOCSIFMAP:
4783 case SIOCSIFHWADDR:
4784 case SIOCSIFSLAVE:
4785 case SIOCADDMULTI:
4786 case SIOCDELMULTI:
4787 case SIOCSIFHWBROADCAST:
4788 case SIOCSIFTXQLEN:
4789 case SIOCSMIIREG:
4790 case SIOCBONDENSLAVE:
4791 case SIOCBONDRELEASE:
4792 case SIOCBONDSETHWADDR:
4793 case SIOCBONDCHANGEACTIVE:
4794 case SIOCBRADDIF:
4795 case SIOCBRDELIF:
4796 case SIOCSHWTSTAMP:
4797 if (!capable(CAP_NET_ADMIN))
4798 return -EPERM;
4799 /* fall through */
4800 case SIOCBONDSLAVEINFOQUERY:
4801 case SIOCBONDINFOQUERY:
4802 dev_load(net, ifr.ifr_name);
4803 rtnl_lock();
4804 ret = dev_ifsioc(net, &ifr, cmd);
4805 rtnl_unlock();
4806 return ret;
4807
4808 case SIOCGIFMEM:
4809 /* Get the per device memory space. We can add this but
4810 * currently do not support it */
4811 case SIOCSIFMEM:
4812 /* Set the per device memory buffer space.
4813 * Not applicable in our case */
4814 case SIOCSIFLINK:
4815 return -EINVAL;
4816
4817 /*
4818 * Unknown or private ioctl.
4819 */
4820 default:
4821 if (cmd == SIOCWANDEV ||
4822 (cmd >= SIOCDEVPRIVATE &&
4823 cmd <= SIOCDEVPRIVATE + 15)) {
Eric W. Biederman881d9662007-09-17 11:56:21 -07004824 dev_load(net, ifr.ifr_name);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004825 rtnl_lock();
Eric W. Biederman881d9662007-09-17 11:56:21 -07004826 ret = dev_ifsioc(net, &ifr, cmd);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004827 rtnl_unlock();
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004828 if (!ret && copy_to_user(arg, &ifr,
Linus Torvalds1da177e2005-04-16 15:20:36 -07004829 sizeof(struct ifreq)))
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004830 ret = -EFAULT;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004831 return ret;
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004832 }
4833 /* Take care of Wireless Extensions */
4834 if (cmd >= SIOCIWFIRST && cmd <= SIOCIWLAST)
4835 return wext_handle_ioctl(net, &ifr, cmd, arg);
4836 return -EINVAL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004837 }
4838}
4839
4840
4841/**
4842 * dev_new_index - allocate an ifindex
Randy Dunlapc4ea43c2007-10-12 21:17:49 -07004843 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -07004844 *
4845 * Returns a suitable unique value for a new device interface
4846 * number. The caller must hold the rtnl semaphore or the
4847 * dev_base_lock to be sure it remains unique.
4848 */
Eric W. Biederman881d9662007-09-17 11:56:21 -07004849static int dev_new_index(struct net *net)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004850{
4851 static int ifindex;
4852 for (;;) {
4853 if (++ifindex <= 0)
4854 ifindex = 1;
Eric W. Biederman881d9662007-09-17 11:56:21 -07004855 if (!__dev_get_by_index(net, ifindex))
Linus Torvalds1da177e2005-04-16 15:20:36 -07004856 return ifindex;
4857 }
4858}
4859
Linus Torvalds1da177e2005-04-16 15:20:36 -07004860/* Delayed registration/unregisteration */
Denis Cheng3b5b34f2007-12-07 00:49:17 -08004861static LIST_HEAD(net_todo_list);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004862
Stephen Hemminger6f05f622007-03-08 20:46:03 -08004863static void net_set_todo(struct net_device *dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004864{
Linus Torvalds1da177e2005-04-16 15:20:36 -07004865 list_add_tail(&dev->todo_list, &net_todo_list);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004866}
4867
Eric Dumazet9b5e3832009-10-27 07:04:19 +00004868static void rollback_registered_many(struct list_head *head)
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07004869{
Krishna Kumare93737b2009-12-08 22:26:02 +00004870 struct net_device *dev, *tmp;
Eric Dumazet9b5e3832009-10-27 07:04:19 +00004871
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07004872 BUG_ON(dev_boot_phase);
4873 ASSERT_RTNL();
4874
Krishna Kumare93737b2009-12-08 22:26:02 +00004875 list_for_each_entry_safe(dev, tmp, head, unreg_list) {
Eric Dumazet9b5e3832009-10-27 07:04:19 +00004876 /* Some devices call without registering
Krishna Kumare93737b2009-12-08 22:26:02 +00004877 * for initialization unwind. Remove those
4878 * devices and proceed with the remaining.
Eric Dumazet9b5e3832009-10-27 07:04:19 +00004879 */
4880 if (dev->reg_state == NETREG_UNINITIALIZED) {
4881 pr_debug("unregister_netdevice: device %s/%p never "
4882 "was registered\n", dev->name, dev);
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07004883
Eric Dumazet9b5e3832009-10-27 07:04:19 +00004884 WARN_ON(1);
Krishna Kumare93737b2009-12-08 22:26:02 +00004885 list_del(&dev->unreg_list);
4886 continue;
Eric Dumazet9b5e3832009-10-27 07:04:19 +00004887 }
4888
4889 BUG_ON(dev->reg_state != NETREG_REGISTERED);
4890
4891 /* If device is running, close it first. */
4892 dev_close(dev);
4893
4894 /* And unlink it from device chain. */
4895 unlist_netdevice(dev);
4896
4897 dev->reg_state = NETREG_UNREGISTERING;
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07004898 }
4899
Eric Dumazet9b5e3832009-10-27 07:04:19 +00004900 synchronize_net();
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07004901
Eric Dumazet9b5e3832009-10-27 07:04:19 +00004902 list_for_each_entry(dev, head, unreg_list) {
4903 /* Shutdown queueing discipline. */
4904 dev_shutdown(dev);
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07004905
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07004906
Eric Dumazet9b5e3832009-10-27 07:04:19 +00004907 /* Notify protocols, that we are about to destroy
4908 this device. They should clean all the things.
4909 */
4910 call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
4911
Patrick McHardya2835762010-02-26 06:34:51 +00004912 if (!dev->rtnl_link_ops ||
4913 dev->rtnl_link_state == RTNL_LINK_INITIALIZED)
4914 rtmsg_ifinfo(RTM_DELLINK, dev, ~0U);
4915
Eric Dumazet9b5e3832009-10-27 07:04:19 +00004916 /*
4917 * Flush the unicast and multicast chains
4918 */
Jiri Pirkoa748ee22010-04-01 21:22:09 +00004919 dev_uc_flush(dev);
Jiri Pirko22bedad32010-04-01 21:22:57 +00004920 dev_mc_flush(dev);
Eric Dumazet9b5e3832009-10-27 07:04:19 +00004921
4922 if (dev->netdev_ops->ndo_uninit)
4923 dev->netdev_ops->ndo_uninit(dev);
4924
4925 /* Notifier chain MUST detach us from master device. */
4926 WARN_ON(dev->master);
4927
4928 /* Remove entries from kobject tree */
4929 netdev_unregister_kobject(dev);
4930 }
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07004931
Eric W. Biedermana5ee1552009-11-29 15:45:58 +00004932 /* Process any work delayed until the end of the batch */
stephen hemmingere5e26d72010-02-24 14:01:38 +00004933 dev = list_first_entry(head, struct net_device, unreg_list);
Eric W. Biedermana5ee1552009-11-29 15:45:58 +00004934 call_netdevice_notifiers(NETDEV_UNREGISTER_BATCH, dev);
4935
Eric Dumazetef885af2010-09-13 12:24:54 +00004936 rcu_barrier();
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07004937
Eric W. Biedermana5ee1552009-11-29 15:45:58 +00004938 list_for_each_entry(dev, head, unreg_list)
Eric Dumazet9b5e3832009-10-27 07:04:19 +00004939 dev_put(dev);
4940}
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07004941
Eric Dumazet9b5e3832009-10-27 07:04:19 +00004942static void rollback_registered(struct net_device *dev)
4943{
4944 LIST_HEAD(single);
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07004945
Eric Dumazet9b5e3832009-10-27 07:04:19 +00004946 list_add(&dev->unreg_list, &single);
4947 rollback_registered_many(&single);
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07004948}
4949
Herbert Xub63365a2008-10-23 01:11:29 -07004950unsigned long netdev_fix_features(unsigned long features, const char *name)
4951{
4952 /* Fix illegal SG+CSUM combinations. */
4953 if ((features & NETIF_F_SG) &&
4954 !(features & NETIF_F_ALL_CSUM)) {
4955 if (name)
4956 printk(KERN_NOTICE "%s: Dropping NETIF_F_SG since no "
4957 "checksum feature.\n", name);
4958 features &= ~NETIF_F_SG;
4959 }
4960
4961 /* TSO requires that SG is present as well. */
4962 if ((features & NETIF_F_TSO) && !(features & NETIF_F_SG)) {
4963 if (name)
4964 printk(KERN_NOTICE "%s: Dropping NETIF_F_TSO since no "
4965 "SG feature.\n", name);
4966 features &= ~NETIF_F_TSO;
4967 }
4968
4969 if (features & NETIF_F_UFO) {
4970 if (!(features & NETIF_F_GEN_CSUM)) {
4971 if (name)
4972 printk(KERN_ERR "%s: Dropping NETIF_F_UFO "
4973 "since no NETIF_F_HW_CSUM feature.\n",
4974 name);
4975 features &= ~NETIF_F_UFO;
4976 }
4977
4978 if (!(features & NETIF_F_SG)) {
4979 if (name)
4980 printk(KERN_ERR "%s: Dropping NETIF_F_UFO "
4981 "since no NETIF_F_SG feature.\n", name);
4982 features &= ~NETIF_F_UFO;
4983 }
4984 }
4985
4986 return features;
4987}
4988EXPORT_SYMBOL(netdev_fix_features);
4989
Linus Torvalds1da177e2005-04-16 15:20:36 -07004990/**
Patrick Mullaneyfc4a7482009-12-03 15:59:22 -08004991 * netif_stacked_transfer_operstate - transfer operstate
4992 * @rootdev: the root or lower level device to transfer state from
4993 * @dev: the device to transfer operstate to
4994 *
4995 * Transfer operational state from root to device. This is normally
4996 * called when a stacking relationship exists between the root
4997 * device and the device(a leaf device).
4998 */
4999void netif_stacked_transfer_operstate(const struct net_device *rootdev,
5000 struct net_device *dev)
5001{
5002 if (rootdev->operstate == IF_OPER_DORMANT)
5003 netif_dormant_on(dev);
5004 else
5005 netif_dormant_off(dev);
5006
5007 if (netif_carrier_ok(rootdev)) {
5008 if (!netif_carrier_ok(dev))
5009 netif_carrier_on(dev);
5010 } else {
5011 if (netif_carrier_ok(dev))
5012 netif_carrier_off(dev);
5013 }
5014}
5015EXPORT_SYMBOL(netif_stacked_transfer_operstate);
5016
Eric Dumazet1b4bf462010-09-23 17:26:35 +00005017static int netif_alloc_rx_queues(struct net_device *dev)
5018{
5019#ifdef CONFIG_RPS
5020 unsigned int i, count = dev->num_rx_queues;
Tom Herbertbd25fa72010-10-18 18:00:16 +00005021 struct netdev_rx_queue *rx;
Eric Dumazet1b4bf462010-09-23 17:26:35 +00005022
Tom Herbertbd25fa72010-10-18 18:00:16 +00005023 BUG_ON(count < 1);
Eric Dumazet1b4bf462010-09-23 17:26:35 +00005024
Tom Herbertbd25fa72010-10-18 18:00:16 +00005025 rx = kcalloc(count, sizeof(struct netdev_rx_queue), GFP_KERNEL);
5026 if (!rx) {
5027 pr_err("netdev: Unable to allocate %u rx queues.\n", count);
5028 return -ENOMEM;
Eric Dumazet1b4bf462010-09-23 17:26:35 +00005029 }
Tom Herbertbd25fa72010-10-18 18:00:16 +00005030 dev->_rx = rx;
5031
5032 /*
5033 * Set a pointer to first element in the array which holds the
5034 * reference count.
5035 */
5036 for (i = 0; i < count; i++)
5037 rx[i].first = rx;
Eric Dumazet1b4bf462010-09-23 17:26:35 +00005038#endif
5039 return 0;
5040}
5041
Tom Herberte6484932010-10-18 18:04:39 +00005042static int netif_alloc_netdev_queues(struct net_device *dev)
5043{
5044 unsigned int count = dev->num_tx_queues;
5045 struct netdev_queue *tx;
5046
5047 BUG_ON(count < 1);
5048
5049 tx = kcalloc(count, sizeof(struct netdev_queue), GFP_KERNEL);
5050 if (!tx) {
5051 pr_err("netdev: Unable to allocate %u tx queues.\n",
5052 count);
5053 return -ENOMEM;
5054 }
5055 dev->_tx = tx;
5056 return 0;
5057}
5058
5059static void netdev_init_one_queue(struct net_device *dev,
5060 struct netdev_queue *queue,
5061 void *_unused)
5062{
5063 queue->dev = dev;
5064
5065 /* Initialize queue lock */
5066 spin_lock_init(&queue->_xmit_lock);
5067 netdev_set_xmit_lockdep_class(&queue->_xmit_lock, dev->type);
5068 queue->xmit_lock_owner = -1;
5069}
5070
5071static void netdev_init_queues(struct net_device *dev)
5072{
5073 netdev_for_each_tx_queue(dev, netdev_init_one_queue, NULL);
5074 spin_lock_init(&dev->tx_global_lock);
5075}
5076
Patrick Mullaneyfc4a7482009-12-03 15:59:22 -08005077/**
Linus Torvalds1da177e2005-04-16 15:20:36 -07005078 * register_netdevice - register a network device
5079 * @dev: device to register
5080 *
5081 * Take a completed network device structure and add it to the kernel
5082 * interfaces. A %NETDEV_REGISTER message is sent to the netdev notifier
5083 * chain. 0 is returned on success. A negative errno code is returned
5084 * on a failure to set up the device, or if the name is a duplicate.
5085 *
5086 * Callers must hold the rtnl semaphore. You may want
5087 * register_netdev() instead of this.
5088 *
5089 * BUGS:
5090 * The locking appears insufficient to guarantee two parallel registers
5091 * will not get the same name.
5092 */
5093
5094int register_netdevice(struct net_device *dev)
5095{
Linus Torvalds1da177e2005-04-16 15:20:36 -07005096 int ret;
Stephen Hemmingerd3147742008-11-19 21:32:24 -08005097 struct net *net = dev_net(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005098
5099 BUG_ON(dev_boot_phase);
5100 ASSERT_RTNL();
5101
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07005102 might_sleep();
5103
Linus Torvalds1da177e2005-04-16 15:20:36 -07005104 /* When net_device's are persistent, this will be fatal. */
5105 BUG_ON(dev->reg_state != NETREG_UNINITIALIZED);
Stephen Hemmingerd3147742008-11-19 21:32:24 -08005106 BUG_ON(!net);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005107
David S. Millerf1f28aa2008-07-15 00:08:33 -07005108 spin_lock_init(&dev->addr_list_lock);
David S. Millercf508b12008-07-22 14:16:42 -07005109 netdev_set_addr_lockdep_class(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005110
Linus Torvalds1da177e2005-04-16 15:20:36 -07005111 dev->iflink = -1;
5112
Eric Dumazet1b4bf462010-09-23 17:26:35 +00005113 ret = netif_alloc_rx_queues(dev);
5114 if (ret)
5115 goto out;
Tom Herbert0a9627f2010-03-16 08:03:29 +00005116
Tom Herberte6484932010-10-18 18:04:39 +00005117 ret = netif_alloc_netdev_queues(dev);
5118 if (ret)
5119 goto out;
Tom Herbert0a9627f2010-03-16 08:03:29 +00005120
Tom Herberte6484932010-10-18 18:04:39 +00005121 netdev_init_queues(dev);
5122
Linus Torvalds1da177e2005-04-16 15:20:36 -07005123 /* Init, if this function is available */
Stephen Hemmingerd3147742008-11-19 21:32:24 -08005124 if (dev->netdev_ops->ndo_init) {
5125 ret = dev->netdev_ops->ndo_init(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005126 if (ret) {
5127 if (ret > 0)
5128 ret = -EIO;
Adrian Bunk90833aa2006-11-13 16:02:22 -08005129 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005130 }
5131 }
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09005132
Daniel Lezcano8ce6cebc2010-05-19 10:12:19 +00005133 ret = dev_get_valid_name(dev, dev->name, 0);
Octavian Purdilad9031022009-11-18 02:36:59 +00005134 if (ret)
Herbert Xu7ce1b0e2007-07-30 16:29:40 -07005135 goto err_uninit;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005136
Eric W. Biederman881d9662007-09-17 11:56:21 -07005137 dev->ifindex = dev_new_index(net);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005138 if (dev->iflink == -1)
5139 dev->iflink = dev->ifindex;
5140
Stephen Hemmingerd212f872007-06-27 00:47:37 -07005141 /* Fix illegal checksum combinations */
5142 if ((dev->features & NETIF_F_HW_CSUM) &&
5143 (dev->features & (NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM))) {
5144 printk(KERN_NOTICE "%s: mixed HW and IP checksum settings.\n",
5145 dev->name);
5146 dev->features &= ~(NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM);
5147 }
5148
5149 if ((dev->features & NETIF_F_NO_CSUM) &&
5150 (dev->features & (NETIF_F_HW_CSUM|NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM))) {
5151 printk(KERN_NOTICE "%s: mixed no checksumming and other settings.\n",
5152 dev->name);
5153 dev->features &= ~(NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM|NETIF_F_HW_CSUM);
5154 }
5155
Herbert Xub63365a2008-10-23 01:11:29 -07005156 dev->features = netdev_fix_features(dev->features, dev->name);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005157
Lennert Buytenheke5a4a722008-08-03 01:23:10 -07005158 /* Enable software GSO if SG is supported. */
5159 if (dev->features & NETIF_F_SG)
5160 dev->features |= NETIF_F_GSO;
5161
Eric Dumazetc5256c52010-09-23 00:46:11 +00005162 /* Enable GRO and NETIF_F_HIGHDMA for vlans by default,
5163 * vlan_dev_init() will do the dev->features check, so these features
5164 * are enabled only if supported by underlying device.
Brandon Philips16c3ea72010-09-15 09:24:24 +00005165 */
Eric Dumazetc5256c52010-09-23 00:46:11 +00005166 dev->vlan_features |= (NETIF_F_GRO | NETIF_F_HIGHDMA);
Brandon Philips16c3ea72010-09-15 09:24:24 +00005167
Johannes Berg7ffbe3f2009-10-02 05:15:27 +00005168 ret = call_netdevice_notifiers(NETDEV_POST_INIT, dev);
5169 ret = notifier_to_errno(ret);
5170 if (ret)
5171 goto err_uninit;
5172
Eric W. Biederman8b41d182007-09-26 22:02:53 -07005173 ret = netdev_register_kobject(dev);
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07005174 if (ret)
Herbert Xu7ce1b0e2007-07-30 16:29:40 -07005175 goto err_uninit;
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07005176 dev->reg_state = NETREG_REGISTERED;
5177
Linus Torvalds1da177e2005-04-16 15:20:36 -07005178 /*
5179 * Default initial state at registry is that the
5180 * device is present.
5181 */
5182
5183 set_bit(__LINK_STATE_PRESENT, &dev->state);
5184
Linus Torvalds1da177e2005-04-16 15:20:36 -07005185 dev_init_scheduler(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005186 dev_hold(dev);
Eric W. Biedermance286d32007-09-12 13:53:49 +02005187 list_netdevice(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005188
5189 /* Notify protocols, that a new device appeared. */
Pavel Emelyanov056925a2007-09-16 15:42:43 -07005190 ret = call_netdevice_notifiers(NETDEV_REGISTER, dev);
Herbert Xufcc5a032007-07-30 17:03:38 -07005191 ret = notifier_to_errno(ret);
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07005192 if (ret) {
5193 rollback_registered(dev);
5194 dev->reg_state = NETREG_UNREGISTERED;
5195 }
Eric W. Biedermand90a9092009-12-12 22:11:15 +00005196 /*
5197 * Prevent userspace races by waiting until the network
5198 * device is fully setup before sending notifications.
5199 */
Patrick McHardya2835762010-02-26 06:34:51 +00005200 if (!dev->rtnl_link_ops ||
5201 dev->rtnl_link_state == RTNL_LINK_INITIALIZED)
5202 rtmsg_ifinfo(RTM_NEWLINK, dev, ~0U);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005203
5204out:
5205 return ret;
Herbert Xu7ce1b0e2007-07-30 16:29:40 -07005206
5207err_uninit:
Stephen Hemmingerd3147742008-11-19 21:32:24 -08005208 if (dev->netdev_ops->ndo_uninit)
5209 dev->netdev_ops->ndo_uninit(dev);
Herbert Xu7ce1b0e2007-07-30 16:29:40 -07005210 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005211}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005212EXPORT_SYMBOL(register_netdevice);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005213
5214/**
Benjamin Herrenschmidt937f1ba2009-01-14 21:05:05 -08005215 * init_dummy_netdev - init a dummy network device for NAPI
5216 * @dev: device to init
5217 *
5218 * This takes a network device structure and initialize the minimum
5219 * amount of fields so it can be used to schedule NAPI polls without
5220 * registering a full blown interface. This is to be used by drivers
5221 * that need to tie several hardware interfaces to a single NAPI
5222 * poll scheduler due to HW limitations.
5223 */
5224int init_dummy_netdev(struct net_device *dev)
5225{
5226 /* Clear everything. Note we don't initialize spinlocks
5227 * are they aren't supposed to be taken by any of the
5228 * NAPI code and this dummy netdev is supposed to be
5229 * only ever used for NAPI polls
5230 */
5231 memset(dev, 0, sizeof(struct net_device));
5232
5233 /* make sure we BUG if trying to hit standard
5234 * register/unregister code path
5235 */
5236 dev->reg_state = NETREG_DUMMY;
5237
Benjamin Herrenschmidt937f1ba2009-01-14 21:05:05 -08005238 /* NAPI wants this */
5239 INIT_LIST_HEAD(&dev->napi_list);
5240
5241 /* a dummy interface is started by default */
5242 set_bit(__LINK_STATE_PRESENT, &dev->state);
5243 set_bit(__LINK_STATE_START, &dev->state);
5244
Eric Dumazet29b44332010-10-11 10:22:12 +00005245 /* Note : We dont allocate pcpu_refcnt for dummy devices,
5246 * because users of this 'device' dont need to change
5247 * its refcount.
5248 */
5249
Benjamin Herrenschmidt937f1ba2009-01-14 21:05:05 -08005250 return 0;
5251}
5252EXPORT_SYMBOL_GPL(init_dummy_netdev);
5253
5254
5255/**
Linus Torvalds1da177e2005-04-16 15:20:36 -07005256 * register_netdev - register a network device
5257 * @dev: device to register
5258 *
5259 * Take a completed network device structure and add it to the kernel
5260 * interfaces. A %NETDEV_REGISTER message is sent to the netdev notifier
5261 * chain. 0 is returned on success. A negative errno code is returned
5262 * on a failure to set up the device, or if the name is a duplicate.
5263 *
Borislav Petkov38b4da32007-04-20 22:14:10 -07005264 * This is a wrapper around register_netdevice that takes the rtnl semaphore
Linus Torvalds1da177e2005-04-16 15:20:36 -07005265 * and expands the device name if you passed a format string to
5266 * alloc_netdev.
5267 */
5268int register_netdev(struct net_device *dev)
5269{
5270 int err;
5271
5272 rtnl_lock();
5273
5274 /*
5275 * If the name is a format string the caller wants us to do a
5276 * name allocation.
5277 */
5278 if (strchr(dev->name, '%')) {
5279 err = dev_alloc_name(dev, dev->name);
5280 if (err < 0)
5281 goto out;
5282 }
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09005283
Linus Torvalds1da177e2005-04-16 15:20:36 -07005284 err = register_netdevice(dev);
5285out:
5286 rtnl_unlock();
5287 return err;
5288}
5289EXPORT_SYMBOL(register_netdev);
5290
Eric Dumazet29b44332010-10-11 10:22:12 +00005291int netdev_refcnt_read(const struct net_device *dev)
5292{
5293 int i, refcnt = 0;
5294
5295 for_each_possible_cpu(i)
5296 refcnt += *per_cpu_ptr(dev->pcpu_refcnt, i);
5297 return refcnt;
5298}
5299EXPORT_SYMBOL(netdev_refcnt_read);
5300
Linus Torvalds1da177e2005-04-16 15:20:36 -07005301/*
5302 * netdev_wait_allrefs - wait until all references are gone.
5303 *
5304 * This is called when unregistering network devices.
5305 *
5306 * Any protocol or device that holds a reference should register
5307 * for netdevice notification, and cleanup and put back the
5308 * reference if they receive an UNREGISTER event.
5309 * We can get stuck here if buggy protocols don't correctly
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09005310 * call dev_put.
Linus Torvalds1da177e2005-04-16 15:20:36 -07005311 */
5312static void netdev_wait_allrefs(struct net_device *dev)
5313{
5314 unsigned long rebroadcast_time, warning_time;
Eric Dumazet29b44332010-10-11 10:22:12 +00005315 int refcnt;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005316
Eric Dumazete014deb2009-11-17 05:59:21 +00005317 linkwatch_forget_dev(dev);
5318
Linus Torvalds1da177e2005-04-16 15:20:36 -07005319 rebroadcast_time = warning_time = jiffies;
Eric Dumazet29b44332010-10-11 10:22:12 +00005320 refcnt = netdev_refcnt_read(dev);
5321
5322 while (refcnt != 0) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07005323 if (time_after(jiffies, rebroadcast_time + 1 * HZ)) {
Stephen Hemminger6756ae42006-03-20 22:23:58 -08005324 rtnl_lock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07005325
5326 /* Rebroadcast unregister notification */
Pavel Emelyanov056925a2007-09-16 15:42:43 -07005327 call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
Eric W. Biedermana5ee1552009-11-29 15:45:58 +00005328 /* don't resend NETDEV_UNREGISTER_BATCH, _BATCH users
Octavian Purdila395264d2009-11-16 13:49:35 +00005329 * should have already handle it the first time */
Linus Torvalds1da177e2005-04-16 15:20:36 -07005330
5331 if (test_bit(__LINK_STATE_LINKWATCH_PENDING,
5332 &dev->state)) {
5333 /* We must not have linkwatch events
5334 * pending on unregister. If this
5335 * happens, we simply run the queue
5336 * unscheduled, resulting in a noop
5337 * for this device.
5338 */
5339 linkwatch_run_queue();
5340 }
5341
Stephen Hemminger6756ae42006-03-20 22:23:58 -08005342 __rtnl_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07005343
5344 rebroadcast_time = jiffies;
5345 }
5346
5347 msleep(250);
5348
Eric Dumazet29b44332010-10-11 10:22:12 +00005349 refcnt = netdev_refcnt_read(dev);
5350
Linus Torvalds1da177e2005-04-16 15:20:36 -07005351 if (time_after(jiffies, warning_time + 10 * HZ)) {
5352 printk(KERN_EMERG "unregister_netdevice: "
5353 "waiting for %s to become free. Usage "
5354 "count = %d\n",
Eric Dumazet29b44332010-10-11 10:22:12 +00005355 dev->name, refcnt);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005356 warning_time = jiffies;
5357 }
5358 }
5359}
5360
5361/* The sequence is:
5362 *
5363 * rtnl_lock();
5364 * ...
5365 * register_netdevice(x1);
5366 * register_netdevice(x2);
5367 * ...
5368 * unregister_netdevice(y1);
5369 * unregister_netdevice(y2);
5370 * ...
5371 * rtnl_unlock();
5372 * free_netdev(y1);
5373 * free_netdev(y2);
5374 *
Herbert Xu58ec3b42008-10-07 15:50:03 -07005375 * We are invoked by rtnl_unlock().
Linus Torvalds1da177e2005-04-16 15:20:36 -07005376 * This allows us to deal with problems:
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07005377 * 1) We can delete sysfs objects which invoke hotplug
Linus Torvalds1da177e2005-04-16 15:20:36 -07005378 * without deadlocking with linkwatch via keventd.
5379 * 2) Since we run with the RTNL semaphore not held, we can sleep
5380 * safely in order to wait for the netdev refcnt to drop to zero.
Herbert Xu58ec3b42008-10-07 15:50:03 -07005381 *
5382 * We must not return until all unregister events added during
5383 * the interval the lock was held have been completed.
Linus Torvalds1da177e2005-04-16 15:20:36 -07005384 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07005385void netdev_run_todo(void)
5386{
Oleg Nesterov626ab0e2006-06-23 02:05:55 -07005387 struct list_head list;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005388
Linus Torvalds1da177e2005-04-16 15:20:36 -07005389 /* Snapshot list, allow later requests */
Oleg Nesterov626ab0e2006-06-23 02:05:55 -07005390 list_replace_init(&net_todo_list, &list);
Herbert Xu58ec3b42008-10-07 15:50:03 -07005391
5392 __rtnl_unlock();
Oleg Nesterov626ab0e2006-06-23 02:05:55 -07005393
Linus Torvalds1da177e2005-04-16 15:20:36 -07005394 while (!list_empty(&list)) {
5395 struct net_device *dev
stephen hemmingere5e26d72010-02-24 14:01:38 +00005396 = list_first_entry(&list, struct net_device, todo_list);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005397 list_del(&dev->todo_list);
5398
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07005399 if (unlikely(dev->reg_state != NETREG_UNREGISTERING)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07005400 printk(KERN_ERR "network todo '%s' but state %d\n",
5401 dev->name, dev->reg_state);
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07005402 dump_stack();
5403 continue;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005404 }
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07005405
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07005406 dev->reg_state = NETREG_UNREGISTERED;
5407
Changli Gao152102c2010-03-30 20:16:22 +00005408 on_each_cpu(flush_backlog, dev, 1);
Stephen Hemminger6e583ce2008-08-03 21:29:57 -07005409
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07005410 netdev_wait_allrefs(dev);
5411
5412 /* paranoia */
Eric Dumazet29b44332010-10-11 10:22:12 +00005413 BUG_ON(netdev_refcnt_read(dev));
Eric Dumazet95ae6b22010-09-15 04:04:31 +00005414 WARN_ON(rcu_dereference_raw(dev->ip_ptr));
Eric Dumazet198caec2010-10-24 21:32:05 +00005415 WARN_ON(rcu_dereference_raw(dev->ip6_ptr));
Ilpo Järvinen547b7922008-07-25 21:43:18 -07005416 WARN_ON(dev->dn_ptr);
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07005417
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07005418 if (dev->destructor)
5419 dev->destructor(dev);
Stephen Hemminger9093bbb2007-05-19 15:39:25 -07005420
5421 /* Free network device */
5422 kobject_put(&dev->dev.kobj);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005423 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07005424}
5425
Stephen Hemmingereeda3fd2008-11-19 21:40:23 -08005426/**
Eric Dumazetd83345a2009-11-16 03:36:51 +00005427 * dev_txq_stats_fold - fold tx_queues stats
5428 * @dev: device to get statistics from
Ben Hutchings3cfde792010-07-09 09:11:52 +00005429 * @stats: struct rtnl_link_stats64 to hold results
Eric Dumazetd83345a2009-11-16 03:36:51 +00005430 */
5431void dev_txq_stats_fold(const struct net_device *dev,
Ben Hutchings3cfde792010-07-09 09:11:52 +00005432 struct rtnl_link_stats64 *stats)
Eric Dumazetd83345a2009-11-16 03:36:51 +00005433{
Eric Dumazetbd272902010-07-19 09:35:40 -07005434 u64 tx_bytes = 0, tx_packets = 0, tx_dropped = 0;
Eric Dumazetd83345a2009-11-16 03:36:51 +00005435 unsigned int i;
5436 struct netdev_queue *txq;
5437
5438 for (i = 0; i < dev->num_tx_queues; i++) {
5439 txq = netdev_get_tx_queue(dev, i);
Eric Dumazetbd272902010-07-19 09:35:40 -07005440 spin_lock_bh(&txq->_xmit_lock);
Eric Dumazetd83345a2009-11-16 03:36:51 +00005441 tx_bytes += txq->tx_bytes;
5442 tx_packets += txq->tx_packets;
5443 tx_dropped += txq->tx_dropped;
Eric Dumazetbd272902010-07-19 09:35:40 -07005444 spin_unlock_bh(&txq->_xmit_lock);
Eric Dumazetd83345a2009-11-16 03:36:51 +00005445 }
5446 if (tx_bytes || tx_packets || tx_dropped) {
5447 stats->tx_bytes = tx_bytes;
5448 stats->tx_packets = tx_packets;
5449 stats->tx_dropped = tx_dropped;
5450 }
5451}
5452EXPORT_SYMBOL(dev_txq_stats_fold);
5453
Ben Hutchings3cfde792010-07-09 09:11:52 +00005454/* Convert net_device_stats to rtnl_link_stats64. They have the same
5455 * fields in the same order, with only the type differing.
5456 */
5457static void netdev_stats_to_stats64(struct rtnl_link_stats64 *stats64,
5458 const struct net_device_stats *netdev_stats)
5459{
5460#if BITS_PER_LONG == 64
5461 BUILD_BUG_ON(sizeof(*stats64) != sizeof(*netdev_stats));
5462 memcpy(stats64, netdev_stats, sizeof(*stats64));
5463#else
5464 size_t i, n = sizeof(*stats64) / sizeof(u64);
5465 const unsigned long *src = (const unsigned long *)netdev_stats;
5466 u64 *dst = (u64 *)stats64;
5467
5468 BUILD_BUG_ON(sizeof(*netdev_stats) / sizeof(unsigned long) !=
5469 sizeof(*stats64) / sizeof(u64));
5470 for (i = 0; i < n; i++)
5471 dst[i] = src[i];
5472#endif
5473}
5474
Eric Dumazetd83345a2009-11-16 03:36:51 +00005475/**
Stephen Hemmingereeda3fd2008-11-19 21:40:23 -08005476 * dev_get_stats - get network device statistics
5477 * @dev: device to get statistics from
Eric Dumazet28172732010-07-07 14:58:56 -07005478 * @storage: place to store stats
Stephen Hemmingereeda3fd2008-11-19 21:40:23 -08005479 *
Ben Hutchingsd7753512010-07-09 09:12:41 +00005480 * Get network statistics from device. Return @storage.
5481 * The device driver may provide its own method by setting
5482 * dev->netdev_ops->get_stats64 or dev->netdev_ops->get_stats;
5483 * otherwise the internal statistics structure is used.
Stephen Hemmingereeda3fd2008-11-19 21:40:23 -08005484 */
Ben Hutchingsd7753512010-07-09 09:12:41 +00005485struct rtnl_link_stats64 *dev_get_stats(struct net_device *dev,
5486 struct rtnl_link_stats64 *storage)
Eric Dumazet7004bf22009-05-18 00:34:33 +00005487{
Stephen Hemmingereeda3fd2008-11-19 21:40:23 -08005488 const struct net_device_ops *ops = dev->netdev_ops;
5489
Eric Dumazet28172732010-07-07 14:58:56 -07005490 if (ops->ndo_get_stats64) {
5491 memset(storage, 0, sizeof(*storage));
Eric Dumazetcaf586e2010-09-30 21:06:55 +00005492 ops->ndo_get_stats64(dev, storage);
5493 } else if (ops->ndo_get_stats) {
Ben Hutchings3cfde792010-07-09 09:11:52 +00005494 netdev_stats_to_stats64(storage, ops->ndo_get_stats(dev));
Eric Dumazetcaf586e2010-09-30 21:06:55 +00005495 } else {
5496 netdev_stats_to_stats64(storage, &dev->stats);
5497 dev_txq_stats_fold(dev, storage);
Eric Dumazet28172732010-07-07 14:58:56 -07005498 }
Eric Dumazetcaf586e2010-09-30 21:06:55 +00005499 storage->rx_dropped += atomic_long_read(&dev->rx_dropped);
Eric Dumazet28172732010-07-07 14:58:56 -07005500 return storage;
Rusty Russellc45d2862007-03-28 14:29:08 -07005501}
Stephen Hemmingereeda3fd2008-11-19 21:40:23 -08005502EXPORT_SYMBOL(dev_get_stats);
Rusty Russellc45d2862007-03-28 14:29:08 -07005503
Eric Dumazet24824a02010-10-02 06:11:55 +00005504struct netdev_queue *dev_ingress_queue_create(struct net_device *dev)
David S. Millerdc2b4842008-07-08 17:18:23 -07005505{
Eric Dumazet24824a02010-10-02 06:11:55 +00005506 struct netdev_queue *queue = dev_ingress_queue(dev);
David S. Millerdc2b4842008-07-08 17:18:23 -07005507
Eric Dumazet24824a02010-10-02 06:11:55 +00005508#ifdef CONFIG_NET_CLS_ACT
5509 if (queue)
5510 return queue;
5511 queue = kzalloc(sizeof(*queue), GFP_KERNEL);
5512 if (!queue)
5513 return NULL;
5514 netdev_init_one_queue(dev, queue, NULL);
Eric Dumazet24824a02010-10-02 06:11:55 +00005515 queue->qdisc = &noop_qdisc;
5516 queue->qdisc_sleeping = &noop_qdisc;
5517 rcu_assign_pointer(dev->ingress_queue, queue);
5518#endif
5519 return queue;
David S. Millerbb949fb2008-07-08 16:55:56 -07005520}
5521
Linus Torvalds1da177e2005-04-16 15:20:36 -07005522/**
Peter P Waskiewicz Jrf25f4e42007-07-06 13:36:20 -07005523 * alloc_netdev_mq - allocate network device
Linus Torvalds1da177e2005-04-16 15:20:36 -07005524 * @sizeof_priv: size of private data to allocate space for
5525 * @name: device name format string
5526 * @setup: callback to initialize device
Peter P Waskiewicz Jrf25f4e42007-07-06 13:36:20 -07005527 * @queue_count: the number of subqueues to allocate
Linus Torvalds1da177e2005-04-16 15:20:36 -07005528 *
5529 * Allocates a struct net_device with private data area for driver use
Peter P Waskiewicz Jrf25f4e42007-07-06 13:36:20 -07005530 * and performs basic initialization. Also allocates subquue structs
5531 * for each queue on the device at the end of the netdevice.
Linus Torvalds1da177e2005-04-16 15:20:36 -07005532 */
Peter P Waskiewicz Jrf25f4e42007-07-06 13:36:20 -07005533struct net_device *alloc_netdev_mq(int sizeof_priv, const char *name,
5534 void (*setup)(struct net_device *), unsigned int queue_count)
Linus Torvalds1da177e2005-04-16 15:20:36 -07005535{
Linus Torvalds1da177e2005-04-16 15:20:36 -07005536 struct net_device *dev;
Stephen Hemminger79439862008-07-21 13:28:44 -07005537 size_t alloc_size;
Eric Dumazet1ce8e7b2009-05-27 04:42:37 +00005538 struct net_device *p;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005539
Stephen Hemmingerb6fe17d2006-08-29 17:06:13 -07005540 BUG_ON(strlen(name) >= sizeof(dev->name));
5541
Tom Herbert55513fb2010-10-18 17:55:58 +00005542 if (queue_count < 1) {
5543 pr_err("alloc_netdev: Unable to allocate device "
5544 "with zero queues.\n");
5545 return NULL;
5546 }
5547
David S. Millerfd2ea0a2008-07-17 01:56:23 -07005548 alloc_size = sizeof(struct net_device);
Alexey Dobriyand1643d22008-04-18 15:43:32 -07005549 if (sizeof_priv) {
5550 /* ensure 32-byte alignment of private area */
Eric Dumazet1ce8e7b2009-05-27 04:42:37 +00005551 alloc_size = ALIGN(alloc_size, NETDEV_ALIGN);
Alexey Dobriyand1643d22008-04-18 15:43:32 -07005552 alloc_size += sizeof_priv;
5553 }
5554 /* ensure 32-byte alignment of whole construct */
Eric Dumazet1ce8e7b2009-05-27 04:42:37 +00005555 alloc_size += NETDEV_ALIGN - 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005556
Paolo 'Blaisorblade' Giarrusso31380de2006-04-06 22:38:28 -07005557 p = kzalloc(alloc_size, GFP_KERNEL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005558 if (!p) {
Stephen Hemmingerb6fe17d2006-08-29 17:06:13 -07005559 printk(KERN_ERR "alloc_netdev: Unable to allocate device.\n");
Linus Torvalds1da177e2005-04-16 15:20:36 -07005560 return NULL;
5561 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07005562
Eric Dumazet1ce8e7b2009-05-27 04:42:37 +00005563 dev = PTR_ALIGN(p, NETDEV_ALIGN);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005564 dev->padded = (char *)dev - (char *)p;
Jiri Pirkoab9c73c2009-05-08 13:30:17 +00005565
Eric Dumazet29b44332010-10-11 10:22:12 +00005566 dev->pcpu_refcnt = alloc_percpu(int);
5567 if (!dev->pcpu_refcnt)
Tom Herberte6484932010-10-18 18:04:39 +00005568 goto free_p;
Jiri Pirkoab9c73c2009-05-08 13:30:17 +00005569
Linus Torvalds1da177e2005-04-16 15:20:36 -07005570 if (dev_addr_init(dev))
Eric Dumazet29b44332010-10-11 10:22:12 +00005571 goto free_pcpu;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005572
Jiri Pirko22bedad32010-04-01 21:22:57 +00005573 dev_mc_init(dev);
Jiri Pirkoa748ee22010-04-01 21:22:09 +00005574 dev_uc_init(dev);
Jiri Pirkoccffad252009-05-22 23:22:17 +00005575
YOSHIFUJI Hideakic346dca2008-03-25 21:47:49 +09005576 dev_net_set(dev, &init_net);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005577
David S. Millere8a04642008-07-17 00:34:19 -07005578 dev->num_tx_queues = queue_count;
David S. Millerfd2ea0a2008-07-17 01:56:23 -07005579 dev->real_num_tx_queues = queue_count;
David S. Millere8a04642008-07-17 00:34:19 -07005580
Eric Dumazetdf334542010-03-24 19:13:54 +00005581#ifdef CONFIG_RPS
Tom Herbert0a9627f2010-03-16 08:03:29 +00005582 dev->num_rx_queues = queue_count;
Ben Hutchings62fe0b42010-09-27 08:24:33 +00005583 dev->real_num_rx_queues = queue_count;
Eric Dumazetdf334542010-03-24 19:13:54 +00005584#endif
Tom Herbert0a9627f2010-03-16 08:03:29 +00005585
Peter P Waskiewicz Jr82cc1a72008-03-21 03:43:19 -07005586 dev->gso_max_size = GSO_MAX_SIZE;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005587
Peter P Waskiewicz Jr15682bc2010-02-10 20:03:05 -08005588 INIT_LIST_HEAD(&dev->ethtool_ntuple_list.list);
5589 dev->ethtool_ntuple_list.count = 0;
Herbert Xud565b0a2008-12-15 23:38:52 -08005590 INIT_LIST_HEAD(&dev->napi_list);
Eric W. Biederman9fdce092009-10-30 14:51:13 +00005591 INIT_LIST_HEAD(&dev->unreg_list);
Eric Dumazete014deb2009-11-17 05:59:21 +00005592 INIT_LIST_HEAD(&dev->link_watch_list);
Eric Dumazet93f154b2009-05-18 22:19:19 -07005593 dev->priv_flags = IFF_XMIT_DST_RELEASE;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005594 setup(dev);
5595 strcpy(dev->name, name);
5596 return dev;
Jiri Pirkoab9c73c2009-05-08 13:30:17 +00005597
Eric Dumazet29b44332010-10-11 10:22:12 +00005598free_pcpu:
5599 free_percpu(dev->pcpu_refcnt);
Jiri Pirkoab9c73c2009-05-08 13:30:17 +00005600free_p:
5601 kfree(p);
5602 return NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005603}
Peter P Waskiewicz Jrf25f4e42007-07-06 13:36:20 -07005604EXPORT_SYMBOL(alloc_netdev_mq);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005605
5606/**
5607 * free_netdev - free network device
5608 * @dev: device
5609 *
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09005610 * This function does the last stage of destroying an allocated device
5611 * interface. The reference to the device object is released.
Linus Torvalds1da177e2005-04-16 15:20:36 -07005612 * If this is the last reference then it will be freed.
5613 */
5614void free_netdev(struct net_device *dev)
5615{
Herbert Xud565b0a2008-12-15 23:38:52 -08005616 struct napi_struct *p, *n;
5617
Denis V. Lunevf3005d72008-04-16 02:02:18 -07005618 release_net(dev_net(dev));
5619
David S. Millere8a04642008-07-17 00:34:19 -07005620 kfree(dev->_tx);
5621
Eric Dumazet24824a02010-10-02 06:11:55 +00005622 kfree(rcu_dereference_raw(dev->ingress_queue));
5623
Jiri Pirkof001fde2009-05-05 02:48:28 +00005624 /* Flush device addresses */
5625 dev_addr_flush(dev);
5626
Peter P Waskiewicz Jr15682bc2010-02-10 20:03:05 -08005627 /* Clear ethtool n-tuple list */
5628 ethtool_ntuple_flush(dev);
5629
Herbert Xud565b0a2008-12-15 23:38:52 -08005630 list_for_each_entry_safe(p, n, &dev->napi_list, dev_list)
5631 netif_napi_del(p);
5632
Eric Dumazet29b44332010-10-11 10:22:12 +00005633 free_percpu(dev->pcpu_refcnt);
5634 dev->pcpu_refcnt = NULL;
5635
Stephen Hemminger3041a062006-05-26 13:25:24 -07005636 /* Compatibility with error handling in drivers */
Linus Torvalds1da177e2005-04-16 15:20:36 -07005637 if (dev->reg_state == NETREG_UNINITIALIZED) {
5638 kfree((char *)dev - dev->padded);
5639 return;
5640 }
5641
5642 BUG_ON(dev->reg_state != NETREG_UNREGISTERED);
5643 dev->reg_state = NETREG_RELEASED;
5644
Greg Kroah-Hartman43cb76d2002-04-09 12:14:34 -07005645 /* will free via device release */
5646 put_device(&dev->dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005647}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005648EXPORT_SYMBOL(free_netdev);
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09005649
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07005650/**
5651 * synchronize_net - Synchronize with packet receive processing
5652 *
5653 * Wait for packets currently being received to be done.
5654 * Does not block later packets from starting.
5655 */
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09005656void synchronize_net(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -07005657{
5658 might_sleep();
Paul E. McKenneyfbd568a3e2005-05-01 08:59:04 -07005659 synchronize_rcu();
Linus Torvalds1da177e2005-04-16 15:20:36 -07005660}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005661EXPORT_SYMBOL(synchronize_net);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005662
5663/**
Eric Dumazet44a08732009-10-27 07:03:04 +00005664 * unregister_netdevice_queue - remove device from the kernel
Linus Torvalds1da177e2005-04-16 15:20:36 -07005665 * @dev: device
Eric Dumazet44a08732009-10-27 07:03:04 +00005666 * @head: list
Jaswinder Singh Rajput6ebfbc02009-11-22 20:43:13 -08005667 *
Linus Torvalds1da177e2005-04-16 15:20:36 -07005668 * This function shuts down a device interface and removes it
Wang Chend59b54b2007-12-11 02:28:03 -08005669 * from the kernel tables.
Eric Dumazet44a08732009-10-27 07:03:04 +00005670 * If head not NULL, device is queued to be unregistered later.
Linus Torvalds1da177e2005-04-16 15:20:36 -07005671 *
5672 * Callers must hold the rtnl semaphore. You may want
5673 * unregister_netdev() instead of this.
5674 */
5675
Eric Dumazet44a08732009-10-27 07:03:04 +00005676void unregister_netdevice_queue(struct net_device *dev, struct list_head *head)
Linus Torvalds1da177e2005-04-16 15:20:36 -07005677{
Herbert Xua6620712007-12-12 19:21:56 -08005678 ASSERT_RTNL();
5679
Eric Dumazet44a08732009-10-27 07:03:04 +00005680 if (head) {
Eric W. Biederman9fdce092009-10-30 14:51:13 +00005681 list_move_tail(&dev->unreg_list, head);
Eric Dumazet44a08732009-10-27 07:03:04 +00005682 } else {
5683 rollback_registered(dev);
5684 /* Finish processing unregister after unlock */
5685 net_set_todo(dev);
5686 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07005687}
Eric Dumazet44a08732009-10-27 07:03:04 +00005688EXPORT_SYMBOL(unregister_netdevice_queue);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005689
5690/**
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005691 * unregister_netdevice_many - unregister many devices
5692 * @head: list of devices
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005693 */
5694void unregister_netdevice_many(struct list_head *head)
5695{
5696 struct net_device *dev;
5697
5698 if (!list_empty(head)) {
5699 rollback_registered_many(head);
5700 list_for_each_entry(dev, head, unreg_list)
5701 net_set_todo(dev);
5702 }
5703}
Eric Dumazet63c80992009-10-27 07:06:49 +00005704EXPORT_SYMBOL(unregister_netdevice_many);
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005705
5706/**
Linus Torvalds1da177e2005-04-16 15:20:36 -07005707 * unregister_netdev - remove device from the kernel
5708 * @dev: device
5709 *
5710 * This function shuts down a device interface and removes it
Wang Chend59b54b2007-12-11 02:28:03 -08005711 * from the kernel tables.
Linus Torvalds1da177e2005-04-16 15:20:36 -07005712 *
5713 * This is just a wrapper for unregister_netdevice that takes
5714 * the rtnl semaphore. In general you want to use this and not
5715 * unregister_netdevice.
5716 */
5717void unregister_netdev(struct net_device *dev)
5718{
5719 rtnl_lock();
5720 unregister_netdevice(dev);
5721 rtnl_unlock();
5722}
Linus Torvalds1da177e2005-04-16 15:20:36 -07005723EXPORT_SYMBOL(unregister_netdev);
5724
Eric W. Biedermance286d32007-09-12 13:53:49 +02005725/**
5726 * dev_change_net_namespace - move device to different nethost namespace
5727 * @dev: device
5728 * @net: network namespace
5729 * @pat: If not NULL name pattern to try if the current device name
5730 * is already taken in the destination network namespace.
5731 *
5732 * This function shuts down a device interface and moves it
5733 * to a new network namespace. On success 0 is returned, on
5734 * a failure a netagive errno code is returned.
5735 *
5736 * Callers must hold the rtnl semaphore.
5737 */
5738
5739int dev_change_net_namespace(struct net_device *dev, struct net *net, const char *pat)
5740{
Eric W. Biedermance286d32007-09-12 13:53:49 +02005741 int err;
5742
5743 ASSERT_RTNL();
5744
5745 /* Don't allow namespace local devices to be moved. */
5746 err = -EINVAL;
5747 if (dev->features & NETIF_F_NETNS_LOCAL)
5748 goto out;
5749
5750 /* Ensure the device has been registrered */
5751 err = -EINVAL;
5752 if (dev->reg_state != NETREG_REGISTERED)
5753 goto out;
5754
5755 /* Get out if there is nothing todo */
5756 err = 0;
YOSHIFUJI Hideaki878628f2008-03-26 03:57:35 +09005757 if (net_eq(dev_net(dev), net))
Eric W. Biedermance286d32007-09-12 13:53:49 +02005758 goto out;
5759
5760 /* Pick the destination device name, and ensure
5761 * we can use it in the destination network namespace.
5762 */
5763 err = -EEXIST;
Octavian Purdilad9031022009-11-18 02:36:59 +00005764 if (__dev_get_by_name(net, dev->name)) {
Eric W. Biedermance286d32007-09-12 13:53:49 +02005765 /* We get here if we can't use the current device name */
5766 if (!pat)
5767 goto out;
Daniel Lezcano8ce6cebc2010-05-19 10:12:19 +00005768 if (dev_get_valid_name(dev, pat, 1))
Eric W. Biedermance286d32007-09-12 13:53:49 +02005769 goto out;
5770 }
5771
5772 /*
5773 * And now a mini version of register_netdevice unregister_netdevice.
5774 */
5775
5776 /* If device is running close it first. */
Pavel Emelyanov9b772652007-10-10 02:49:09 -07005777 dev_close(dev);
Eric W. Biedermance286d32007-09-12 13:53:49 +02005778
5779 /* And unlink it from device chain */
5780 err = -ENODEV;
5781 unlist_netdevice(dev);
5782
5783 synchronize_net();
5784
5785 /* Shutdown queueing discipline. */
5786 dev_shutdown(dev);
5787
5788 /* Notify protocols, that we are about to destroy
5789 this device. They should clean all the things.
David Lamparter3b27e102010-09-17 03:22:19 +00005790
5791 Note that dev->reg_state stays at NETREG_REGISTERED.
5792 This is wanted because this way 8021q and macvlan know
5793 the device is just moving and can keep their slaves up.
Eric W. Biedermance286d32007-09-12 13:53:49 +02005794 */
5795 call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
Eric W. Biedermana5ee1552009-11-29 15:45:58 +00005796 call_netdevice_notifiers(NETDEV_UNREGISTER_BATCH, dev);
Eric W. Biedermance286d32007-09-12 13:53:49 +02005797
5798 /*
5799 * Flush the unicast and multicast chains
5800 */
Jiri Pirkoa748ee22010-04-01 21:22:09 +00005801 dev_uc_flush(dev);
Jiri Pirko22bedad32010-04-01 21:22:57 +00005802 dev_mc_flush(dev);
Eric W. Biedermance286d32007-09-12 13:53:49 +02005803
5804 /* Actually switch the network namespace */
YOSHIFUJI Hideakic346dca2008-03-25 21:47:49 +09005805 dev_net_set(dev, net);
Eric W. Biedermance286d32007-09-12 13:53:49 +02005806
Eric W. Biedermance286d32007-09-12 13:53:49 +02005807 /* If there is an ifindex conflict assign a new one */
5808 if (__dev_get_by_index(net, dev->ifindex)) {
5809 int iflink = (dev->iflink == dev->ifindex);
5810 dev->ifindex = dev_new_index(net);
5811 if (iflink)
5812 dev->iflink = dev->ifindex;
5813 }
5814
Eric W. Biederman8b41d182007-09-26 22:02:53 -07005815 /* Fixup kobjects */
Eric W. Biedermana1b3f592010-05-04 17:36:49 -07005816 err = device_rename(&dev->dev, dev->name);
Eric W. Biederman8b41d182007-09-26 22:02:53 -07005817 WARN_ON(err);
Eric W. Biedermance286d32007-09-12 13:53:49 +02005818
5819 /* Add the device back in the hashes */
5820 list_netdevice(dev);
5821
5822 /* Notify protocols, that a new device appeared. */
5823 call_netdevice_notifiers(NETDEV_REGISTER, dev);
5824
Eric W. Biedermand90a9092009-12-12 22:11:15 +00005825 /*
5826 * Prevent userspace races by waiting until the network
5827 * device is fully setup before sending notifications.
5828 */
5829 rtmsg_ifinfo(RTM_NEWLINK, dev, ~0U);
5830
Eric W. Biedermance286d32007-09-12 13:53:49 +02005831 synchronize_net();
5832 err = 0;
5833out:
5834 return err;
5835}
Johannes Berg463d0182009-07-14 00:33:35 +02005836EXPORT_SYMBOL_GPL(dev_change_net_namespace);
Eric W. Biedermance286d32007-09-12 13:53:49 +02005837
Linus Torvalds1da177e2005-04-16 15:20:36 -07005838static int dev_cpu_callback(struct notifier_block *nfb,
5839 unsigned long action,
5840 void *ocpu)
5841{
5842 struct sk_buff **list_skb;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005843 struct sk_buff *skb;
5844 unsigned int cpu, oldcpu = (unsigned long)ocpu;
5845 struct softnet_data *sd, *oldsd;
5846
Rafael J. Wysocki8bb78442007-05-09 02:35:10 -07005847 if (action != CPU_DEAD && action != CPU_DEAD_FROZEN)
Linus Torvalds1da177e2005-04-16 15:20:36 -07005848 return NOTIFY_OK;
5849
5850 local_irq_disable();
5851 cpu = smp_processor_id();
5852 sd = &per_cpu(softnet_data, cpu);
5853 oldsd = &per_cpu(softnet_data, oldcpu);
5854
5855 /* Find end of our completion_queue. */
5856 list_skb = &sd->completion_queue;
5857 while (*list_skb)
5858 list_skb = &(*list_skb)->next;
5859 /* Append completion queue from offline CPU. */
5860 *list_skb = oldsd->completion_queue;
5861 oldsd->completion_queue = NULL;
5862
Linus Torvalds1da177e2005-04-16 15:20:36 -07005863 /* Append output queue from offline CPU. */
Changli Gaoa9cbd582010-04-26 23:06:24 +00005864 if (oldsd->output_queue) {
5865 *sd->output_queue_tailp = oldsd->output_queue;
5866 sd->output_queue_tailp = oldsd->output_queue_tailp;
5867 oldsd->output_queue = NULL;
5868 oldsd->output_queue_tailp = &oldsd->output_queue;
5869 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07005870
5871 raise_softirq_irqoff(NET_TX_SOFTIRQ);
5872 local_irq_enable();
5873
5874 /* Process offline CPU's input_pkt_queue */
Tom Herbert76cc8b12010-05-20 18:37:59 +00005875 while ((skb = __skb_dequeue(&oldsd->process_queue))) {
5876 netif_rx(skb);
5877 input_queue_head_incr(oldsd);
5878 }
Tom Herbertfec5e652010-04-16 16:01:27 -07005879 while ((skb = __skb_dequeue(&oldsd->input_pkt_queue))) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07005880 netif_rx(skb);
Tom Herbert76cc8b12010-05-20 18:37:59 +00005881 input_queue_head_incr(oldsd);
Tom Herbertfec5e652010-04-16 16:01:27 -07005882 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07005883
5884 return NOTIFY_OK;
5885}
Linus Torvalds1da177e2005-04-16 15:20:36 -07005886
5887
Herbert Xu7f353bf2007-08-10 15:47:58 -07005888/**
Herbert Xub63365a2008-10-23 01:11:29 -07005889 * netdev_increment_features - increment feature set by one
5890 * @all: current feature set
5891 * @one: new feature set
5892 * @mask: mask feature set
Herbert Xu7f353bf2007-08-10 15:47:58 -07005893 *
5894 * Computes a new feature set after adding a device with feature set
Herbert Xub63365a2008-10-23 01:11:29 -07005895 * @one to the master device with current feature set @all. Will not
5896 * enable anything that is off in @mask. Returns the new feature set.
Herbert Xu7f353bf2007-08-10 15:47:58 -07005897 */
Herbert Xub63365a2008-10-23 01:11:29 -07005898unsigned long netdev_increment_features(unsigned long all, unsigned long one,
5899 unsigned long mask)
Herbert Xu7f353bf2007-08-10 15:47:58 -07005900{
Herbert Xub63365a2008-10-23 01:11:29 -07005901 /* If device needs checksumming, downgrade to it. */
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005902 if (all & NETIF_F_NO_CSUM && !(one & NETIF_F_NO_CSUM))
Herbert Xub63365a2008-10-23 01:11:29 -07005903 all ^= NETIF_F_NO_CSUM | (one & NETIF_F_ALL_CSUM);
5904 else if (mask & NETIF_F_ALL_CSUM) {
5905 /* If one device supports v4/v6 checksumming, set for all. */
5906 if (one & (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM) &&
5907 !(all & NETIF_F_GEN_CSUM)) {
5908 all &= ~NETIF_F_ALL_CSUM;
5909 all |= one & (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM);
5910 }
Herbert Xu7f353bf2007-08-10 15:47:58 -07005911
Herbert Xub63365a2008-10-23 01:11:29 -07005912 /* If one device supports hw checksumming, set for all. */
5913 if (one & NETIF_F_GEN_CSUM && !(all & NETIF_F_GEN_CSUM)) {
5914 all &= ~NETIF_F_ALL_CSUM;
5915 all |= NETIF_F_HW_CSUM;
5916 }
5917 }
Herbert Xu7f353bf2007-08-10 15:47:58 -07005918
Herbert Xub63365a2008-10-23 01:11:29 -07005919 one |= NETIF_F_ALL_CSUM;
Herbert Xu7f353bf2007-08-10 15:47:58 -07005920
Herbert Xub63365a2008-10-23 01:11:29 -07005921 one |= all & NETIF_F_ONE_FOR_ALL;
Sridhar Samudralad9f59502009-10-07 12:24:25 +00005922 all &= one | NETIF_F_LLTX | NETIF_F_GSO | NETIF_F_UFO;
Herbert Xub63365a2008-10-23 01:11:29 -07005923 all |= one & mask & NETIF_F_ONE_FOR_ALL;
Herbert Xu7f353bf2007-08-10 15:47:58 -07005924
5925 return all;
5926}
Herbert Xub63365a2008-10-23 01:11:29 -07005927EXPORT_SYMBOL(netdev_increment_features);
Herbert Xu7f353bf2007-08-10 15:47:58 -07005928
Pavel Emelyanov30d97d32007-09-16 15:40:33 -07005929static struct hlist_head *netdev_create_hash(void)
5930{
5931 int i;
5932 struct hlist_head *hash;
5933
5934 hash = kmalloc(sizeof(*hash) * NETDEV_HASHENTRIES, GFP_KERNEL);
5935 if (hash != NULL)
5936 for (i = 0; i < NETDEV_HASHENTRIES; i++)
5937 INIT_HLIST_HEAD(&hash[i]);
5938
5939 return hash;
5940}
5941
Eric W. Biederman881d9662007-09-17 11:56:21 -07005942/* Initialize per network namespace state */
Pavel Emelyanov46650792007-10-08 20:38:39 -07005943static int __net_init netdev_init(struct net *net)
Eric W. Biederman881d9662007-09-17 11:56:21 -07005944{
Eric W. Biederman881d9662007-09-17 11:56:21 -07005945 INIT_LIST_HEAD(&net->dev_base_head);
Eric W. Biederman881d9662007-09-17 11:56:21 -07005946
Pavel Emelyanov30d97d32007-09-16 15:40:33 -07005947 net->dev_name_head = netdev_create_hash();
5948 if (net->dev_name_head == NULL)
5949 goto err_name;
Eric W. Biederman881d9662007-09-17 11:56:21 -07005950
Pavel Emelyanov30d97d32007-09-16 15:40:33 -07005951 net->dev_index_head = netdev_create_hash();
5952 if (net->dev_index_head == NULL)
5953 goto err_idx;
Eric W. Biederman881d9662007-09-17 11:56:21 -07005954
5955 return 0;
Pavel Emelyanov30d97d32007-09-16 15:40:33 -07005956
5957err_idx:
5958 kfree(net->dev_name_head);
5959err_name:
5960 return -ENOMEM;
Eric W. Biederman881d9662007-09-17 11:56:21 -07005961}
5962
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07005963/**
5964 * netdev_drivername - network driver for the device
5965 * @dev: network device
5966 * @buffer: buffer for resulting name
5967 * @len: size of buffer
5968 *
5969 * Determine network driver for device.
5970 */
Stephen Hemmingercf04a4c72008-09-30 02:22:14 -07005971char *netdev_drivername(const struct net_device *dev, char *buffer, int len)
Arjan van de Ven6579e572008-07-21 13:31:48 -07005972{
Stephen Hemmingercf04a4c72008-09-30 02:22:14 -07005973 const struct device_driver *driver;
5974 const struct device *parent;
Arjan van de Ven6579e572008-07-21 13:31:48 -07005975
5976 if (len <= 0 || !buffer)
5977 return buffer;
5978 buffer[0] = 0;
5979
5980 parent = dev->dev.parent;
5981
5982 if (!parent)
5983 return buffer;
5984
5985 driver = parent->driver;
5986 if (driver && driver->name)
5987 strlcpy(buffer, driver->name, len);
5988 return buffer;
5989}
5990
Joe Perches256df2f2010-06-27 01:02:35 +00005991static int __netdev_printk(const char *level, const struct net_device *dev,
5992 struct va_format *vaf)
5993{
5994 int r;
5995
5996 if (dev && dev->dev.parent)
5997 r = dev_printk(level, dev->dev.parent, "%s: %pV",
5998 netdev_name(dev), vaf);
5999 else if (dev)
6000 r = printk("%s%s: %pV", level, netdev_name(dev), vaf);
6001 else
6002 r = printk("%s(NULL net_device): %pV", level, vaf);
6003
6004 return r;
6005}
6006
6007int netdev_printk(const char *level, const struct net_device *dev,
6008 const char *format, ...)
6009{
6010 struct va_format vaf;
6011 va_list args;
6012 int r;
6013
6014 va_start(args, format);
6015
6016 vaf.fmt = format;
6017 vaf.va = &args;
6018
6019 r = __netdev_printk(level, dev, &vaf);
6020 va_end(args);
6021
6022 return r;
6023}
6024EXPORT_SYMBOL(netdev_printk);
6025
6026#define define_netdev_printk_level(func, level) \
6027int func(const struct net_device *dev, const char *fmt, ...) \
6028{ \
6029 int r; \
6030 struct va_format vaf; \
6031 va_list args; \
6032 \
6033 va_start(args, fmt); \
6034 \
6035 vaf.fmt = fmt; \
6036 vaf.va = &args; \
6037 \
6038 r = __netdev_printk(level, dev, &vaf); \
6039 va_end(args); \
6040 \
6041 return r; \
6042} \
6043EXPORT_SYMBOL(func);
6044
6045define_netdev_printk_level(netdev_emerg, KERN_EMERG);
6046define_netdev_printk_level(netdev_alert, KERN_ALERT);
6047define_netdev_printk_level(netdev_crit, KERN_CRIT);
6048define_netdev_printk_level(netdev_err, KERN_ERR);
6049define_netdev_printk_level(netdev_warn, KERN_WARNING);
6050define_netdev_printk_level(netdev_notice, KERN_NOTICE);
6051define_netdev_printk_level(netdev_info, KERN_INFO);
6052
Pavel Emelyanov46650792007-10-08 20:38:39 -07006053static void __net_exit netdev_exit(struct net *net)
Eric W. Biederman881d9662007-09-17 11:56:21 -07006054{
6055 kfree(net->dev_name_head);
6056 kfree(net->dev_index_head);
6057}
6058
Denis V. Lunev022cbae2007-11-13 03:23:50 -08006059static struct pernet_operations __net_initdata netdev_net_ops = {
Eric W. Biederman881d9662007-09-17 11:56:21 -07006060 .init = netdev_init,
6061 .exit = netdev_exit,
6062};
6063
Pavel Emelyanov46650792007-10-08 20:38:39 -07006064static void __net_exit default_device_exit(struct net *net)
Eric W. Biedermance286d32007-09-12 13:53:49 +02006065{
Eric W. Biedermane008b5f2009-11-29 22:25:30 +00006066 struct net_device *dev, *aux;
Eric W. Biedermance286d32007-09-12 13:53:49 +02006067 /*
Eric W. Biedermane008b5f2009-11-29 22:25:30 +00006068 * Push all migratable network devices back to the
Eric W. Biedermance286d32007-09-12 13:53:49 +02006069 * initial network namespace
6070 */
6071 rtnl_lock();
Eric W. Biedermane008b5f2009-11-29 22:25:30 +00006072 for_each_netdev_safe(net, dev, aux) {
Eric W. Biedermance286d32007-09-12 13:53:49 +02006073 int err;
Pavel Emelyanovaca51392008-05-08 01:24:25 -07006074 char fb_name[IFNAMSIZ];
Eric W. Biedermance286d32007-09-12 13:53:49 +02006075
6076 /* Ignore unmoveable devices (i.e. loopback) */
6077 if (dev->features & NETIF_F_NETNS_LOCAL)
6078 continue;
6079
Eric W. Biedermane008b5f2009-11-29 22:25:30 +00006080 /* Leave virtual devices for the generic cleanup */
6081 if (dev->rtnl_link_ops)
6082 continue;
Eric W. Biedermand0c082c2008-11-05 15:59:38 -08006083
Eric W. Biedermance286d32007-09-12 13:53:49 +02006084 /* Push remaing network devices to init_net */
Pavel Emelyanovaca51392008-05-08 01:24:25 -07006085 snprintf(fb_name, IFNAMSIZ, "dev%d", dev->ifindex);
6086 err = dev_change_net_namespace(dev, &init_net, fb_name);
Eric W. Biedermance286d32007-09-12 13:53:49 +02006087 if (err) {
Pavel Emelyanovaca51392008-05-08 01:24:25 -07006088 printk(KERN_EMERG "%s: failed to move %s to init_net: %d\n",
Eric W. Biedermance286d32007-09-12 13:53:49 +02006089 __func__, dev->name, err);
Pavel Emelyanovaca51392008-05-08 01:24:25 -07006090 BUG();
Eric W. Biedermance286d32007-09-12 13:53:49 +02006091 }
6092 }
6093 rtnl_unlock();
6094}
6095
Eric W. Biederman04dc7f6b2009-12-03 02:29:04 +00006096static void __net_exit default_device_exit_batch(struct list_head *net_list)
6097{
6098 /* At exit all network devices most be removed from a network
6099 * namespace. Do this in the reverse order of registeration.
6100 * Do this across as many network namespaces as possible to
6101 * improve batching efficiency.
6102 */
6103 struct net_device *dev;
6104 struct net *net;
6105 LIST_HEAD(dev_kill_list);
6106
6107 rtnl_lock();
6108 list_for_each_entry(net, net_list, exit_list) {
6109 for_each_netdev_reverse(net, dev) {
6110 if (dev->rtnl_link_ops)
6111 dev->rtnl_link_ops->dellink(dev, &dev_kill_list);
6112 else
6113 unregister_netdevice_queue(dev, &dev_kill_list);
6114 }
6115 }
6116 unregister_netdevice_many(&dev_kill_list);
6117 rtnl_unlock();
6118}
6119
Denis V. Lunev022cbae2007-11-13 03:23:50 -08006120static struct pernet_operations __net_initdata default_device_ops = {
Eric W. Biedermance286d32007-09-12 13:53:49 +02006121 .exit = default_device_exit,
Eric W. Biederman04dc7f6b2009-12-03 02:29:04 +00006122 .exit_batch = default_device_exit_batch,
Eric W. Biedermance286d32007-09-12 13:53:49 +02006123};
6124
Linus Torvalds1da177e2005-04-16 15:20:36 -07006125/*
6126 * Initialize the DEV module. At boot time this walks the device list and
6127 * unhooks any devices that fail to initialise (normally hardware not
6128 * present) and leaves us with a valid list of present and active devices.
6129 *
6130 */
6131
6132/*
6133 * This is called single threaded during boot, so no need
6134 * to take the rtnl semaphore.
6135 */
6136static int __init net_dev_init(void)
6137{
6138 int i, rc = -ENOMEM;
6139
6140 BUG_ON(!dev_boot_phase);
6141
Linus Torvalds1da177e2005-04-16 15:20:36 -07006142 if (dev_proc_init())
6143 goto out;
6144
Eric W. Biederman8b41d182007-09-26 22:02:53 -07006145 if (netdev_kobject_init())
Linus Torvalds1da177e2005-04-16 15:20:36 -07006146 goto out;
6147
6148 INIT_LIST_HEAD(&ptype_all);
Pavel Emelyanov82d8a8672007-11-26 20:12:58 +08006149 for (i = 0; i < PTYPE_HASH_SIZE; i++)
Linus Torvalds1da177e2005-04-16 15:20:36 -07006150 INIT_LIST_HEAD(&ptype_base[i]);
6151
Eric W. Biederman881d9662007-09-17 11:56:21 -07006152 if (register_pernet_subsys(&netdev_net_ops))
6153 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006154
6155 /*
6156 * Initialise the packet receive queues.
6157 */
6158
KAMEZAWA Hiroyuki6f912042006-04-10 22:52:50 -07006159 for_each_possible_cpu(i) {
Eric Dumazete36fa2f2010-04-19 21:17:14 +00006160 struct softnet_data *sd = &per_cpu(softnet_data, i);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006161
Changli Gaodee42872010-05-02 05:42:16 +00006162 memset(sd, 0, sizeof(*sd));
Eric Dumazete36fa2f2010-04-19 21:17:14 +00006163 skb_queue_head_init(&sd->input_pkt_queue);
Changli Gao6e7676c2010-04-27 15:07:33 -07006164 skb_queue_head_init(&sd->process_queue);
Eric Dumazete36fa2f2010-04-19 21:17:14 +00006165 sd->completion_queue = NULL;
6166 INIT_LIST_HEAD(&sd->poll_list);
Changli Gaoa9cbd582010-04-26 23:06:24 +00006167 sd->output_queue = NULL;
6168 sd->output_queue_tailp = &sd->output_queue;
Eric Dumazetdf334542010-03-24 19:13:54 +00006169#ifdef CONFIG_RPS
Eric Dumazete36fa2f2010-04-19 21:17:14 +00006170 sd->csd.func = rps_trigger_softirq;
6171 sd->csd.info = sd;
6172 sd->csd.flags = 0;
6173 sd->cpu = i;
Tom Herbert1e94d722010-03-18 17:45:44 -07006174#endif
Tom Herbert0a9627f2010-03-16 08:03:29 +00006175
Eric Dumazete36fa2f2010-04-19 21:17:14 +00006176 sd->backlog.poll = process_backlog;
6177 sd->backlog.weight = weight_p;
6178 sd->backlog.gro_list = NULL;
6179 sd->backlog.gro_count = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006180 }
6181
Linus Torvalds1da177e2005-04-16 15:20:36 -07006182 dev_boot_phase = 0;
6183
Eric W. Biederman505d4f72008-11-07 22:54:20 -08006184 /* The loopback device is special if any other network devices
6185 * is present in a network namespace the loopback device must
6186 * be present. Since we now dynamically allocate and free the
6187 * loopback device ensure this invariant is maintained by
6188 * keeping the loopback device as the first device on the
6189 * list of network devices. Ensuring the loopback devices
6190 * is the first device that appears and the last network device
6191 * that disappears.
6192 */
6193 if (register_pernet_device(&loopback_net_ops))
6194 goto out;
6195
6196 if (register_pernet_device(&default_device_ops))
6197 goto out;
6198
Carlos R. Mafra962cf362008-05-15 11:15:37 -03006199 open_softirq(NET_TX_SOFTIRQ, net_tx_action);
6200 open_softirq(NET_RX_SOFTIRQ, net_rx_action);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006201
6202 hotcpu_notifier(dev_cpu_callback, 0);
6203 dst_init();
6204 dev_mcast_init();
6205 rc = 0;
6206out:
6207 return rc;
6208}
6209
6210subsys_initcall(net_dev_init);
6211
Krishna Kumare88721f2009-02-18 17:55:02 -08006212static int __init initialize_hashrnd(void)
6213{
Tom Herbert0a9627f2010-03-16 08:03:29 +00006214 get_random_bytes(&hashrnd, sizeof(hashrnd));
Krishna Kumare88721f2009-02-18 17:55:02 -08006215 return 0;
6216}
6217
6218late_initcall_sync(initialize_hashrnd);
6219